diff --git a/docs/backups/docker-installs/docker-installs.md b/docs/backups/docker-installs/docker-installs.md deleted file mode 100644 index cc211cee5e0..00000000000 --- a/docs/backups/docker-installs/docker-installs.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Backup and Restore for Rancher Installed with Docker ---- - -- [Backups](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) -- [Restores](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) \ No newline at end of file diff --git a/docs/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md b/docs/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md index be7d1b5a57a..85f57c5b1bd 100644 --- a/docs/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md +++ b/docs/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md @@ -45,6 +45,27 @@ If the requested resources exceed the remaining capacity in the project for thos However, this is not the case with namespaces created **_outside_** of Rancher's UI. For namespaces created via `kubectl`, Rancher will assign a resource quota that has a **zero** amount for any resource that requested more capacity than what remains in the project. +To create a project with a resource quota via `kubectl`, use the `spec.resourceQuota` and `spec.namespaceDefaultResourceQuota` fields: + +``` +apiVersion: management.cattle.io/v3 +kind: Project +metadata: + generateName: p- + namespace: c-m-abcde +spec: + clusterName: c-m-abcde + displayName: myproject + resourceQuota: + limit: + limitsCpu: 1000m + namespaceDefaultResourceQuota: + limit: + limitsCpu: 50m +``` + +Use `metadata.generateName` to ensure a unique project ID. Set `metadata.namespace` and `spec.clusterName` to the ID for the cluster the project belongs to. + To create a namespace in an existing project via `kubectl`, use the `field.cattle.io/projectId` annotation. To override the default requested quota limit, use the `field.cattle.io/resourceQuota` annotation. diff --git a/docs/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/docs/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md index bc5cc83d471..300e152bb22 100644 --- a/docs/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md +++ b/docs/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -109,20 +109,23 @@ This reference contains variables that you can use in `questions.yml` nested und | label | string | true | Define the UI label. | | description | string | false | Specify the description of the variable.| | type | string | false | Default to `string` if not specified (current supported types are string, multiline, boolean, int, enum, password, storageclass, hostname, pvc, and secret).| -| required | bool | false | Define if the variable is required or not (true \| false)| | default | string | false | Specify the default value. Only used if there is no corresponding value in the `values.yml` file. | | group | string | false | Group questions by input value. | -| min_length | int | false | Min character length.| -| max_length | int | false | Max character length.| -| min | int | false | Min integer length. | -| max | int | false | Max integer length. | | options | []string | false | Specify the options when the variable type is `enum`, for example: options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer"| -| valid_chars | string | false | Regular expression for input chars validation. | -| invalid_chars | string | false | Regular expression for invalid input chars validation.| | subquestions | []subquestion | false| Add an array of subquestions.| | show_if | string | false | Show current variable if conditional variable is true. For example `show_if: "serviceType=Nodeport"` | | show\_subquestion_if | string | false | Show subquestions if is true or equal to one of the options. for example `show_subquestion_if: "true"`| + + :::note `subquestions[]` cannot contain `subquestions` or `show_subquestions_if` keys, but all other keys in the above table are supported. diff --git a/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md b/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md index b65d7aa9b4b..f176acc177b 100644 --- a/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md +++ b/docs/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md @@ -8,7 +8,6 @@ title: Creating an AKS Cluster You can use Rancher to create a cluster hosted in Microsoft Azure Kubernetes Service (AKS). - ## Prerequisites in Microsoft Azure :::caution @@ -65,6 +64,14 @@ az ad sp create-for-rbac \ --role Contributor ``` +### Setting Up the Service Principal with the Azure Command Line Tool + +Create the Resource Group by running this command: + +``` +az group create --location AZURE_LOCATION_NAME --resource-group AZURE_RESOURCE_GROUP_NAME +``` + ### Setting Up the Service Principal from the Azure Portal You can also follow these instructions to set up a service principal and give it role-based access from the Azure Portal. @@ -124,6 +131,17 @@ When provisioning an AKS cluster in the Rancher UI, RBAC is not configurable bec RBAC is required for AKS clusters that are registered or imported into Rancher. +### Setting Up the Role Assignment to Service Principal with the Azure Command Line Tool + +Assign the Rancher AKSv2 role to the service principal with the Azure Command Line Tool: + +``` +az role assignment create \ +--assignee CLIENT_ID \ +--scope "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME" \ +--role "Rancher AKSv2" +``` + ## AKS Cluster Configuration Reference For more information about how to configure AKS clusters from the Rancher UI, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) @@ -146,12 +164,114 @@ Please be aware that when registering an existing AKS cluster, the cluster might For more information about connecting to an AKS private cluster, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster) +## Setting Up the Minimum Permission Role with the Azure Command Line Tool + +1. Create the Minimum Rancher AKSv2 Permission Role by running this command: + + ``` + cat >> rancher-azure.json << EOF + + { + "Name": "Rancher AKSv2", + "IsCustom": true, + "Description": "Everything needed by Rancher AKSv2 operator", + "Actions": [ + "Microsoft.Compute/disks/delete", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.Compute/locations/DiskOperations/read", + "Microsoft.Compute/locations/vmSizes/read", + "Microsoft.Compute/locations/operations/read", + "Microsoft.Compute/proximityPlacementGroups/write", + "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/snapshots/write", + "Microsoft.Compute/virtualMachineScaleSets/manualUpgrade/action", + "Microsoft.Compute/virtualMachineScaleSets/delete", + "Microsoft.Compute/virtualMachineScaleSets/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/ipconfigurations/publicipaddresses/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/write", + "Microsoft.Compute/virtualMachineScaleSets/write", + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/virtualMachines/write", + "Microsoft.ContainerService/managedClusters/read", + "Microsoft.ContainerService/managedClusters/write" + "Microsoft.ContainerService/managedClusters/delete", + "Microsoft.ContainerService/managedClusters/accessProfiles/listCredential/action", + "Microsoft.ContainerService/managedClusters/agentPools/read", + "Microsoft.ContainerService/managedClusters/agentPools/write", + "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action", + "Microsoft.Network/applicationGateways/read", + "Microsoft.Network/applicationGateways/write", + "Microsoft.Network/loadBalancers/write", + "Microsoft.Network/loadBalancers/backendAddressPools/join/action", + "Microsoft.Network/loadBalancers/delete", + "Microsoft.Network/loadBalancers/read", + "Microsoft.Network/networkInterfaces/join/action", + "Microsoft.Network/networkInterfaces/read", + "Microsoft.Network/networkInterfaces/write", + "Microsoft.Network/networkSecurityGroups/read", + "Microsoft.Network/networkSecurityGroups/write", + "Microsoft.Network/publicIPAddresses/delete", + "Microsoft.Network/publicIPAddresses/join/action", + "Microsoft.Network/publicIPAddresses/read", + "Microsoft.Network/publicIPAddresses/write", + "Microsoft.Network/publicIPPrefixes/join/action", + "Microsoft.Network/privatednszones/*", + "Microsoft.Network/routeTables/read", + "Microsoft.Network/routeTables/routes/delete", + "Microsoft.Network/routeTables/routes/read", + "Microsoft.Network/routeTables/routes/write", + "Microsoft.Network/routeTables/write", + "Microsoft.Network/virtualNetworks/read", + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/read", + "Microsoft.Network/virtualNetworks/joinLoadBalancer/action", + "Microsoft.OperationalInsights/workspaces/sharedkeys/read", + "Microsoft.OperationalInsights/workspaces/read", + "Microsoft.OperationsManagement/solutions/write", + "Microsoft.OperationsManagement/solutions/read", + "Microsoft.Resources/subscriptions/resourcegroups/read", + "Microsoft.Resources/subscriptions/resourcegroups/write", + "Microsoft.Storage/operations/read", + "Microsoft.Storage/storageAccounts/listKeys/action", + "Microsoft.Storage/storageAccounts/delete", + "Microsoft.Storage/storageAccounts/read", + "Microsoft.Storage/storageAccounts/write" + ], + "NotActions": [], + "DataActions": [], + "NotDataActions": [], + "AssignableScopes": [ + "/subscriptions/SUBSCRIPTION_ID" + ] + } + EOF + ``` + +2. Apply the Rancher AKSv2 Role: + + ``` + az role definition create --role-definition rancher-azure.json + ``` + +3. Verify if the Rancher AKSv2 Role was created: + + ``` + az role definition list | grep "Rancher AKSv2" + ``` + ## Syncing The AKS provisioner can synchronize the state of an AKS cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) For information on configuring the refresh interval, see [this section.](../../../../pages-for-subheaders/gke-cluster-configuration.md#configuring-the-refresh-interval) + ## Programmatically Creating AKS Clusters -The most common way to programmatically deploy AKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) \ No newline at end of file +The most common way to programmatically deploy AKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster). diff --git a/docs/pages-for-subheaders/rancher-hardening-guides.md b/docs/pages-for-subheaders/rancher-hardening-guides.md index a7ffa3ce2e1..5db6c192945 100644 --- a/docs/pages-for-subheaders/rancher-hardening-guides.md +++ b/docs/pages-for-subheaders/rancher-hardening-guides.md @@ -27,25 +27,25 @@ Each self-assessment guide is accompanied by a hardening guide. These guides wer | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | |--------------------|-----------------------|-----------------------|------------------| | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke1-hardening-guide.md) | -| Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | -| Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke1-hardening-guide.md) | ### RKE2 Guides | Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | |------|--------------------|-----------------------|-----------------------|------------------| | Rancher provisioned RKE2 | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke2-hardening-guide.md) | -| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | -| Rancher provisioned RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](rke2-hardening-guide.md) | -| Standalone RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | +| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke2-hardening-guide.md) | +| Standalone RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | ### K3s Guides | Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | |------|--------------------|-----------------------|-----------------------|------------------| | Rancher provisioned K3s cluster | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](k3s-hardening-guide.md) | -| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | -| Rancher provisioned K3s cluster | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](k3s-hardening-guide.md) | | Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment) | [Link](https://docs.k3s.io/security/hardening-guide) | ## Rancher with SELinux diff --git a/docs/pages-for-subheaders/rancher-security.md b/docs/pages-for-subheaders/rancher-security.md index 4426fb66c60..f9c30b41f29 100644 --- a/docs/pages-for-subheaders/rancher-security.md +++ b/docs/pages-for-subheaders/rancher-security.md @@ -88,6 +88,10 @@ Rancher is committed to informing the community of security issues in our produc For recommendations on securing your Kubernetes cluster, refer to the [Kubernetes Security Best Practices](../reference-guides/rancher-security/kubernetes-security-best-practices.md) guide. +### Rancher Webhook Hardening + +The Rancher webhook deploys on both the upstream Rancher cluster and all provisioned clusters. For recommendations on hardening the Rancher webhook, see the [Hardening the Rancher Webhook](../reference-guides/rancher-security/rancher-webhook-hardening.md) guide. + ### Rancher Security Best Practices -For recommendations on securing your Rancher, refer to the [Rancher Security Best Practices](../reference-guides/rancher-security/rancher-security-best-practices.md) guide. +For recommendations on securing your Rancher Manager deployments, refer to the [Rancher Security Best Practices](../reference-guides/rancher-security/rancher-security-best-practices.md) guide. diff --git a/docs/reference-guides/about-the-api/api-tokens.md b/docs/reference-guides/about-the-api/api-tokens.md index eccd4d962b1..5fe8a0eb5a4 100644 --- a/docs/reference-guides/about-the-api/api-tokens.md +++ b/docs/reference-guides/about-the-api/api-tokens.md @@ -3,7 +3,7 @@ title: API Tokens --- - + By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. diff --git a/docs/reference-guides/backup-restore-configuration/backup-configuration.md b/docs/reference-guides/backup-restore-configuration/backup-configuration.md index 313a3de1d83..2bb79d074b7 100644 --- a/docs/reference-guides/backup-restore-configuration/backup-configuration.md +++ b/docs/reference-guides/backup-restore-configuration/backup-configuration.md @@ -3,7 +3,7 @@ title: Backup Configuration --- - + The **Backup: Create** page lets you configure a schedule, enable encryption and specify the storage location for your backups. diff --git a/docs/reference-guides/backup-restore-configuration/examples.md b/docs/reference-guides/backup-restore-configuration/examples.md index 32a1eb76c6a..0afd8a359fb 100644 --- a/docs/reference-guides/backup-restore-configuration/examples.md +++ b/docs/reference-guides/backup-restore-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + This section contains examples of Backup and Restore custom resources. diff --git a/docs/reference-guides/backup-restore-configuration/restore-configuration.md b/docs/reference-guides/backup-restore-configuration/restore-configuration.md index 5e958fcd674..b162a6ef873 100644 --- a/docs/reference-guides/backup-restore-configuration/restore-configuration.md +++ b/docs/reference-guides/backup-restore-configuration/restore-configuration.md @@ -3,7 +3,7 @@ title: Restore Configuration --- - + The Restore Create page lets you provide details of the backup to restore from diff --git a/docs/reference-guides/backup-restore-configuration/storage-configuration.md b/docs/reference-guides/backup-restore-configuration/storage-configuration.md index 3ad8b94487f..b1b8bad0896 100644 --- a/docs/reference-guides/backup-restore-configuration/storage-configuration.md +++ b/docs/reference-guides/backup-restore-configuration/storage-configuration.md @@ -3,7 +3,7 @@ title: Backup Storage Location Configuration --- - + Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible object store. diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index 9fc13c9ce70..dc200ae62b1 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -3,7 +3,7 @@ title: Logging Best Practices --- - + In this guide, we recommend best practices for cluster-level logging and application logging. diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md index 9255c76a28b..a6e34b03a50 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -3,7 +3,7 @@ title: Monitoring Best Practices --- - + Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. This is not different when using Kubernetes and Rancher. Fortunately the integrated monitoring and alerting functionality makes this whole process a lot easier. diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/docs/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md index 3012223f423..9d24fb3c5db 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -3,7 +3,7 @@ title: Best Practices for Rancher Managed vSphere Clusters --- - + This guide outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/docs/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/docs/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md index 1298ff18b87..0cec0a91864 100644 --- a/docs/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md +++ b/docs/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -3,7 +3,7 @@ title: Tips for Setting Up Containers --- - + Running well-built containers can greatly impact the overall performance and security of your environment. diff --git a/docs/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/docs/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md index c19ac243d98..74243559753 100644 --- a/docs/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md +++ b/docs/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -3,7 +3,7 @@ title: Installing Rancher in a vSphere Environment --- - + This guide outlines a reference architecture for installing Rancher on an RKE Kubernetes cluster in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/docs/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/docs/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md index cc8811dc77e..99583f75579 100644 --- a/docs/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md +++ b/docs/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -3,7 +3,7 @@ title: Rancher Deployment Strategy --- - + There are two recommended deployment strategies for a Rancher instance that manages downstream Kubernetes clusters. Each one has its own pros and cons. Read more about which one would fit best for your use case. diff --git a/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index 1863ff82503..1ed05696fa2 100644 --- a/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/docs/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -3,7 +3,7 @@ title: Tips for Running Rancher --- - + This guide is geared toward use cases where Rancher is used to manage downstream Kubernetes clusters. The high-availability setup is intended to prevent losing access to downstream clusters if the Rancher server is not available. diff --git a/docs/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md b/docs/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md index 691ef18f576..e8e919bde9b 100644 --- a/docs/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md +++ b/docs/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md @@ -3,7 +3,7 @@ title: Tips for Scaling Rancher --- - + This guide aims to introduce the approaches that should be considered to scale Rancher setups, and associated challenges with doing so. As systems grow performance will naturally reduce, but there are steps we can take to minimize the load put on Rancher, as well as optimize Rancher's ability to handle these larger setups. diff --git a/docs/reference-guides/cli-with-rancher/kubectl-utility.md b/docs/reference-guides/cli-with-rancher/kubectl-utility.md index 45502753a08..fefa1a66b1b 100644 --- a/docs/reference-guides/cli-with-rancher/kubectl-utility.md +++ b/docs/reference-guides/cli-with-rancher/kubectl-utility.md @@ -3,7 +3,7 @@ title: kubectl Utility --- - + ## kubectl diff --git a/docs/reference-guides/cli-with-rancher/rancher-cli.md b/docs/reference-guides/cli-with-rancher/rancher-cli.md index 44024172658..0c9a27c1694 100644 --- a/docs/reference-guides/cli-with-rancher/rancher-cli.md +++ b/docs/reference-guides/cli-with-rancher/rancher-cli.md @@ -4,7 +4,7 @@ description: Interact with Rancher using command line interface (CLI) tools from --- - + The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md index 6043df8f1ba..4640d142697 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Machine Configuration Reference --- - + For more details about EC2 nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md index 98d299762b0..ba7e806295e 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Machine Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md index 1d62fe9d9d7..634c7f48561 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Machine Configuration --- - + For more details about DigitalOcean, Droplets, refer to the [official documentation](https://docs.digitalocean.com/products/compute/). diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md index ffe36cf7723..bd79dabbed5 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Node Template Configuration --- - + For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md index e8fa8b382ee..e747dbbb547 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Node Template Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md index 0e679ae7d13..87b5fccdcfb 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Node Template Configuration --- - + Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md index cbecb1be01b..4e44d5ceca7 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -3,7 +3,7 @@ title: Nutanix Node Template Configuration --- - + ## Account Access diff --git a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md index 0e665ba93f4..135c3edf7cc 100644 --- a/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md +++ b/docs/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -3,7 +3,7 @@ title: VSphere Node Template Configuration --- - + ## Account Access diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md index 136c71a45a2..67cd280e470 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -3,7 +3,7 @@ title: AKS Cluster Configuration Reference --- - + ## Changes in Rancher v2.6 diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md index 5bfe091028c..ba6b826b4f1 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -3,7 +3,7 @@ title: EKS Cluster Configuration Reference --- - + ### Account Access diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md index 60228c3fb97..120be47a641 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -3,7 +3,7 @@ title: Private Clusters --- - + In GKE, [private clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept) are clusters whose nodes are isolated from inbound and outbound traffic by assigning them internal IP addresses only. Private clusters in GKE have the option of exposing the control plane endpoint as a publicly accessible address or as a private address. This is different from other Kubernetes providers, which may refer to clusters with private control plane endpoints as "private clusters" but still allow traffic to and from nodes. You may want to create a cluster with private nodes, with or without a public control plane endpoint, depending on your organization's networking and security requirements. A GKE cluster provisioned from Rancher can use isolated nodes by selecting "Private Cluster" in the Cluster Options (under "Show advanced options"). The control plane endpoint can optionally be made private by selecting "Enable Private Endpoint". diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md index 94c3b2c3442..6d260b3dd68 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -127,7 +127,7 @@ ACE is available on RKE, RKE2, and K3s clusters that are provisioned or register ::: -ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters.md) on RKE2 and K3s clusters. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. +ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) on RKE2 and K3s clusters. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md index 9ccc2cfe9f0..63fa8df3511 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -3,7 +3,7 @@ title: Syncing --- - + Syncing allows Rancher to update cluster values so that they're up to date with the corresponding cluster object hosted in AKS, EKS or GKE. This enables sources other than Rancher to own a hosted cluster’s state. diff --git a/docs/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/docs/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md index a0e092fee87..3c363d053cf 100644 --- a/docs/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md +++ b/docs/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -3,7 +3,7 @@ title: Rancher Agent Options --- - + Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. diff --git a/docs/reference-guides/monitoring-v2-configuration/examples.md b/docs/reference-guides/monitoring-v2-configuration/examples.md index 7c586657eb2..c65722cfa97 100644 --- a/docs/reference-guides/monitoring-v2-configuration/examples.md +++ b/docs/reference-guides/monitoring-v2-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + ### ServiceMonitor diff --git a/docs/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/docs/reference-guides/monitoring-v2-configuration/helm-chart-options.md index fabd9fc222f..112c41c9ea0 100644 --- a/docs/reference-guides/monitoring-v2-configuration/helm-chart-options.md +++ b/docs/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -3,7 +3,7 @@ title: Helm Chart Options --- - + ## Configuring Resource Limits and Requests diff --git a/docs/reference-guides/monitoring-v2-configuration/receivers.md b/docs/reference-guides/monitoring-v2-configuration/receivers.md index eb3a029b45b..dc459f34378 100644 --- a/docs/reference-guides/monitoring-v2-configuration/receivers.md +++ b/docs/reference-guides/monitoring-v2-configuration/receivers.md @@ -3,7 +3,7 @@ title: Receiver Configuration --- - + The [Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret contains the configuration of an Alertmanager instance that sends out notifications based on alerts it receives from Prometheus. diff --git a/docs/reference-guides/monitoring-v2-configuration/routes.md b/docs/reference-guides/monitoring-v2-configuration/routes.md index 339dd2a606a..f79d1e190c6 100644 --- a/docs/reference-guides/monitoring-v2-configuration/routes.md +++ b/docs/reference-guides/monitoring-v2-configuration/routes.md @@ -3,7 +3,7 @@ title: Route Configuration --- - + The route configuration is the section of the Alertmanager custom resource that controls how the alerts fired by Prometheus are grouped and filtered before they reach the receiver. diff --git a/docs/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/docs/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md index 780d13c2673..30fccb634c8 100644 --- a/docs/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md +++ b/docs/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -3,7 +3,7 @@ title: ServiceMonitor and PodMonitor Configuration --- - + ServiceMonitors and PodMonitors are both pseudo-CRDs that map the scrape configuration of the Prometheus custom resource. diff --git a/docs/reference-guides/prometheus-federator/rbac.md b/docs/reference-guides/prometheus-federator/rbac.md index 47887415d85..8b54ce9559f 100644 --- a/docs/reference-guides/prometheus-federator/rbac.md +++ b/docs/reference-guides/prometheus-federator/rbac.md @@ -2,6 +2,10 @@ title: Role-Based Access Control --- + + + + This section describes the expectations for Role-Based Access Control (RBAC) for Prometheus Federator. As described in the section on [namespaces](../../pages-for-subheaders/prometheus-federator.md#namespaces), Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: diff --git a/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md index d68f5b9320c..1ece990ff8c 100644 --- a/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/docs/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -2,6 +2,10 @@ title: Architecture Recommendations --- + + + + If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the node running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) ## Separation of Rancher and User Clusters diff --git a/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index 0c5b83a3ffe..586b55b0db8 100644 --- a/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/docs/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -2,6 +2,10 @@ title: Communicating with Downstream User Clusters --- + + + + This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. diff --git a/docs/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/docs/reference-guides/rancher-manager-architecture/rancher-server-and-components.md index bb1cf170e7a..e9fec332622 100644 --- a/docs/reference-guides/rancher-manager-architecture/rancher-server-and-components.md +++ b/docs/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -2,6 +2,10 @@ title: Rancher Server and Components --- + + + + The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). diff --git a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..1d348069a2a --- /dev/null +++ b/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3208 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 539 1 0 Sep11 ? 00:00:09 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 45213 45195 3 Sep11 ? 00:45:14 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 47188 1 0 Sep11 ? 00:01:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 02b8fdd94b7628d575ace92f337dcc93202151d07f33858341cfcb0178fea586 -address /run/k3s/containerd/containerd.sock root 47235 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id eb0f03c4bc1125ae34e5de77c7eaeb9c2c1e46bec0bb106b99281df7353f9ded -address /run/k3s/containerd/containerd.sock root 47948 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5eece7110bd3876414f890ed1b7158eb91f60d1f14450a9f8c77a0fc62b73c4a -address /run/k3s/containerd/containerd.sock root 48047 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ef831f4bb8830a0a7b46a1e8be58197be00846ea0caa094014ea7ce3adf008e -address /run/k3s/containerd/containerd.sock root 48220 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id f8d5d3791ad2b1a8534476aa2fbdf8da1e4f0b9fdcdd26b1ae161142458a5336 -address /run/k3s/containerd/containerd.sock root 48878 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 41b93a6581e272b0852257b3d7c44ab4cadcdf465a3579a8b29b493ea75feaf7 -address /run/k3s/containerd/containerd.sock root 49870 1 0 Sep11 ? 00:00:30 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id be329cdccda4049f048c759a2b1915cd72526473ff7768e895bcfc153a66c125 -address /run/k3s/containerd/containerd.sock root 50271 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id cceb5d26eba7d7d3c474a2dc842b72495689f4e40055505ea2598e4ac6849d40 -address /run/k3s/containerd/containerd.sock root 50571 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3a26e691e0d0186165d523bef22b0af71f175c39df82845b32ce7b4e3d652d4a -address /run/k3s/containerd/containerd.sock root 96693 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9ed77e6e58410f1295f730ba4a8f8daf7aff8f4e896e24219a8fa710729c2287 -address /run/k3s/containerd/containerd.sock root 97726 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 1be4fe50e6bb29d4549f8b03102b5dc15b7669fe56b32beae3f701cf6d2e58cc -address /run/k3s/containerd/containerd.sock root 97923 1 1 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7c65c47376f36a8980a5bebad7ffdc1abe1853821cf6c4a1d45f1b6be3452062 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'600' is equal to '600' +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--event-qps' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..e7373288b35 --- /dev/null +++ b/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,3215 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 +Not Applicable. + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root +Not Applicable. + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd +Not Applicable. + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/k3s/server/tls + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... +Permissive - Enabling Pod Security Policy can cause applications to unexpectedly fail. + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log +Permissive. + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 +Permissive. + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 +Permissive. + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 +Permissive. + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s +Permissive. + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Not Applicable. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is present OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'RotateKubeletServerCertificate' is present OR 'RotateKubeletServerCertificate' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..5f3c20fb37f --- /dev/null +++ b/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3047 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 5 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 4 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:10 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..9fe9ca2045f --- /dev/null +++ b/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2863 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is greater or equal to 0 OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..ba80c0a516a --- /dev/null +++ b/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3202 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1017917 2489 83 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz root 1017945 2489 75 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/livez +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"akmMjUAq94YvsUftJpiA3b+9SClu0ESPBeckAI7KZBY="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Automated) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r ".spec.runAsUser.rule" || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r ".spec.runAsUser.rule" +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is equal to 'MustRunAsNonRoot' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is equal to 'ALL' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..b7d4a36ea4b --- /dev/null +++ b/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2967 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then find /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml -name '*etcd*' | xargs stat -c permissions=%a; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1018768 2419 99 16:17 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 +Not Applicable. + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= +Not Applicable. + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" +Not Applicable. + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= +Not Applicable. + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true +Not Applicable. + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not applicable. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present OR '{.eventRecordQPS}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.podPidsLimit}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/docs/reference-guides/rancher-security/rancher-webhook-hardening.md b/docs/reference-guides/rancher-security/rancher-webhook-hardening.md new file mode 100644 index 00000000000..1f771dfdfe5 --- /dev/null +++ b/docs/reference-guides/rancher-security/rancher-webhook-hardening.md @@ -0,0 +1,133 @@ +--- +title: Hardening the Rancher Webhook +--- + +Rancher Webhook is an important component within Rancher, playing a role in enforcing security requirements for Rancher and its workloads. To decrease its attack surface, access to it should be limited to the only valid caller it has: the Kubernetes API server. This can be done by using network policies and authentication independently or in conjunction with each other to harden the webhook against attacks. + +## Block External Traffic Using Network Policies + +The webhook is only expected to accept requests from the Kubernetes API server. By default, however, the webhook can accept traffic from any source. If you are using a CNI that supports Network Policies, you can create a policy that blocks traffic that doesn't originate from the API server. + +The built-in NetworkPolicy resource in Kubernetes can't block or admit traffic from the cluster hosts, and the `kube-apiserver` process is always running on the host network. Therefore, you must use the advanced network policy resources from the CNI in use. Examples for Calico and Cilium follow. Consult the documentation for your CNI for more details. + +### Calico + +Use the NetworkPolicy resource in the `crd.projectcalico.org/v1` API group. Use the selector `app == 'rancher-webhook'` to create a rule for the webhook, and set the CIDR of the control plane hosts as the ingress source: + +```yaml +apiVersion: crd.projectcalico.org/v1 +kind: NetworkPolicy +metadata: + name: allow-k8s + namespace: cattle-system +spec: + selector: app == 'rancher-webhook' + types: + - Ingress + ingress: + - action: Allow + protocol: TCP + source: + nets: + - 192.168.42.0/24 # CIDR of the control plane host. May list more than 1 if the hosts are in different subnets. + destination: + selector: + app == 'rancher-webhook' +``` + +### Cilium + +Use the CiliumNetworkPolicy resource in the `cilium.io/v2` API group. Add the `host` and `remote-node` keys to the `fromEntities` ingress rule. This blocks in-cluster and external traffic while allowing traffic from the hosts. + +```yaml +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-k8s + namespace: cattle-system +spec: + endpointSelector: + matchLabels: + app: rancher-webhook + ingress: + - fromEntities: + - host + - remote-node +``` + +## Require the Kubernetes API Server to Authenticate to the Webhook + +The webhook should only accept requests from the Kubernetes API server. By default, the webhook doesn't require clients to authenticate to it. It will accept any request. You can configure the webhook to require credentials so that only the API server can access it. More information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#authenticate-apiservers). + +1. Configure the API server to present a client certificate to the webhook, pointing to an AdmissionConfiguration file to configure the ValidatingAdmissionWebhook and MutatingAdmissionWebhook plugins: + + ```yaml + # /etc/rancher/admission/admission.yaml + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: ValidatingAdmissionWebhook + configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: WebhookAdmissionConfiguration + kubeConfigFile: "/etc/rancher/admission/kubeconfig" + - name: MutatingAdmissionWebhook + configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: WebhookAdmissionConfiguration + kubeConfigFile: "/etc/rancher/admission/kubeconfig" + ``` + + This is also the same config file where other admission plugins are configured, such as PodSecurity. If your distro or your setup uses additional admission plugins, configure those as well. For example, add [RKE2's PodSecurity configuration](https://docs.rke2.io/security/pod_security_standards) to this file. + +2. Create the kubeconfig file that the admission plugins refer to. Rancher Webhook only supports client certificate authentication, so generate a TLS key pair, and set the kubeconfig to use either `client-certificate` and `client-key` or `client-certificate-data` and `client-key-data`. For example: + + ```yaml + # /etc/rancher/admission/kubeconfig + apiVersion: v1 + kind: Config + users: + - name: 'rancher-webhook.cattle-system.svc' + user: + client-certificate: /path/to/client/cert + client-key: /path/to/client/key + ``` + +3. Start the kube-apiserver binary with the flag `--admission-control-config-file` pointing to your AdmissionConfiguration file. The way to do this varies by distro, and it isn't supported universally, such as in hosted Kubernetes providers. Consult the documentation for your Kubernetes distribution. + + For RKE2, `rke2-server` can be started with a config file like so: + + ```yaml + # /etc/rancher/rke2/config.yaml + kube-apiserver-arg: + - admission-control-config-file=/etc/rancher/admission/admission.yaml + kube-apiserver-extra-mount: + - /etc/rancher/admission:/etc/rancher/admission:ro + ``` + + :::danger + Some distros set this flag by default. If your distro provisions its own AdmissionConfiguration, you must include it in your custom admission control config file. For example, RKE2 installs an AdmissionConfiguration file at `/etc/rancher/rke2/rke2-pss.yaml`, which configures the PodSecurity admission plugin. Setting `admission-control-config-file` in config.yaml will override this essential security setting. To include both plugins, consult [the Default Pod Security Standards documentation](https://docs.rke2.io/security/pod_security_standards) and copy the appropriate plugin configuration to your admission.yaml. + ::: + +4. If you're using Rancher to provision your cluster using existing nodes, create these files on the node before you provision them. + + If you're using Rancher to provision your cluster on new nodes, allow the provisioning to complete, then use the provided SSH key and IP address to connect to the nodes, and place the RKE2 config file in the `/etc/rancher/rke2/config.yaml.d/` directory. + +5. After the cluster is configured with these credentials, configure the Rancher cluster agent to enable authentication in the webhook. Create a file containing these chart values: + + ```yaml + # values.yaml + auth: + clientCA: + allowedCNs: + - + - + ``` + +6. Create a configmap in the `cattle-system` namespace on the provisioned cluster with these values: + + ``` + kubectl --namespace cattle-system create configmap --from-file=rancher-webhook=values.yaml + ``` + + The webhook will restart with these values. diff --git a/docusaurus.config.js b/docusaurus.config.js index 3ab17e54bc1..f2b60198737 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -124,10 +124,23 @@ module.exports = { sidebarPath: require.resolve('./sidebars.js'), showLastUpdateTime: true, editUrl: 'https://github.com/rancher/rancher-docs/edit/main/', - lastVersion: 'current', + // lastVersion: '2.7', + includeCurrentVersion: false, versions: { - current: { - label: 'Latest' + // current: { + // label: 'Next 🚧', + // path: 'next', + // banner: 'unreleased' + // }, + latest: { + label: 'Latest', + path: '', + banner: 'none' + }, + 2.8: { + label: 'v2.8 (Preview)', + path: 'v2.8', + banner: 'unreleased' }, 2.7: { label: 'v2.7', @@ -1143,28 +1156,52 @@ module.exports = { from: '/v2.6/explanations/integrations-in-rancher/opa-gatekeeper' }, // Redirects for restructure from PR #234 (end) { - to: "/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + to: "/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + from: "/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24" + }, + { + to: "/v2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + from: "/2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24" + }, + { + to: "/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", from: "/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25" }, { - to: "/v2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + to: "/v2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", from: "/v2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25" }, { - to: "/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + to: "/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + from: "/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24" + }, + { + to: "/v2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + from: "/v2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24" + }, + { + to: "/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", from: "/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25" }, { - to: "/v2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + to: "/v2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", from: "/v2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25" }, { - to: "/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + to: "/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + from: "/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24" + }, + { + to: "/v2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + from: "/v2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24" + }, + { + to: "/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", from: "/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25" }, { - to: "/v2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + to: "/v2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", from: "/v2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25" } ], diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/backups/docker-installs/docker-installs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/backups/docker-installs/docker-installs.md new file mode 100644 index 00000000000..51c3001d777 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/backups/docker-installs/docker-installs.md @@ -0,0 +1,6 @@ +--- +title: 备份和æ¢å¤ Docker 安装的 Rancher +--- + +- [备份](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) +- [还原](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/cluster-provisioning/rke-clusters/options/options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/cluster-provisioning/rke-clusters/options/options.md new file mode 100644 index 00000000000..39c332461ce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/cluster-provisioning/rke-clusters/options/options.md @@ -0,0 +1,5 @@ +--- +title: RKE 集群é…ç½® +--- + +本文已è¿ç§»åˆ°[此处](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/contribute-to-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/contribute-to-rancher.md new file mode 100644 index 00000000000..baff7b735b9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/contribute-to-rancher.md @@ -0,0 +1,136 @@ +--- +title: å‚与 Rancher 社区贡献 +--- + +本文介ç»äº† Rancher 仓库和 Rancher 文档ã€å¦‚何构建 Rancher 仓库以åŠæ交 issue æ—¶è¦åŒ…å«å“ªäº›ä¿¡æ¯ã€‚ + +有关如何为 Rancher 项目开å‘åšå‡ºè´¡çŒ®çš„更多详细信æ¯ï¼Œè¯·å‚阅 [Rancher Developer Wiki](https://github.com/rancher/rancher/wiki)。Wiki 包å«ä»¥ä¸‹ä¸»é¢˜çš„资æºï¼š + +- 如何æ­å»º Rancher å¼€å‘环境并è¿è¡Œæµ‹è¯• +- Issue 在开å‘生命周期中的典型æµç¨‹ +- ç¼–ç æŒ‡å—和开å‘最佳实践 +- 调试和故障排除 +- å¼€å‘ Rancher API + +在 Rancher Users Slack 上，开å‘者的频é“是 **#developer**。 + +## Rancher 文档 + +如果你对此网站上的文档有建议，请在主 [Rancher 文档](https://github.com/rancher/rancher-docs)仓库中[æ交 issue](https://github.com/rancher/rancher-docs/issues/new/choose)ã€‚æ­¤ä»“åº“åŒ…å« Rancher v2.0 åŠæ›´é«˜ç‰ˆæœ¬çš„文档。 + +有关贡献和构建 Rancher v2.x 文档仓库的更多详细信æ¯ï¼Œè¯·å‚阅 [Rancher 文档 README](https://github.com/rancher/rancher-docs#readme)。 + +有关 Rancher v1.6 åŠæ›´æ—©ç‰ˆæœ¬çš„文档，请å‚阅 [Rancher 1.x docs](https://github.com/rancher/rancher.github.io) ä»“åº“ï¼Œå…¶ä¸­åŒ…å« https://rancher.com/docs/rancher/v1.6/en/ çš„æºæ–‡ä»¶ã€‚ + +## Rancher 仓库 + +所有仓库都ä½äºŽæˆ‘ä»¬çš„ä¸»è¦ GitHub 组织内。Rancher 使用了很多仓库，以下是部分主è¦ä»“库的æ述： + +| 仓库 | URL | æè¿° | +-----------|-----|------------- +| Rancher | https://github.com/rancher/rancher | Rancher 2.x 的主è¦æºç ä»“库。 | +| Types | https://github.com/rancher/types | åŒ…å« Rancher 2.x 的所有 API 类型的仓库。 | +| API Framework | https://github.com/rancher/norman | API 框架，用于构建由 Kubernetes 自定义资æºæ”¯æŒçš„ Rancher 风格的 API。 | +| User Interface | https://github.com/rancher/dashboard/ | Dashboard UI æºç ä»“库。 | +| (Rancher) Docker Machine | https://github.com/rancher/machine | 使用主机驱动时使用的 Docker Machine 二进制文件的æºç ä»“库。这是 `docker/machine` 仓库的一个 fork。 | +| machine-package | https://github.com/rancher/machine-package | 用于构建 Rancher Docker Machine 二进制文件。 | +| kontainer-engine | https://github.com/rancher/kontainer-engine | kontainer-engine çš„æºç ä»“库，它是é…置托管 Kubernetes 集群的工具。 | +| RKE repository | https://github.com/rancher/rke | Rancher Kubernetes Engine çš„æºç ä»“库，该工具å¯åœ¨ä»»ä½•ä¸»æœºä¸Šé…ç½® Kubernetes 集群。 | +| CLI | https://github.com/rancher/cli | Rancher 2.x 中使用的 Rancher CLI çš„æºç ä»“库。 | +| (Rancher) Helm repository | https://github.com/rancher/helm | 打包的 Helm 二进制文件的æºç ä»“库。这是 `helm/helm` 仓库的一个 fork。 | +| Telemetry repository | https://github.com/rancher/telemetry | Telemetry 二进制文件的æºç ä»“库。 | +| loglevel repository | https://github.com/rancher/loglevel | loglevel 二进制文件的æºç ä»“库，用于动æ€æ›´æ”¹æ—¥å¿—级别。 | + +è¦æŸ¥çœ‹ Rancher 使用的所有库/项目，请查看 `rancher/rancher` 仓库中的 [`go.mod` 文件](https://github.com/rancher/rancher/blob/master/go.mod)。 + +![Rancher diagram](/img/ranchercomponentsdiagram-2.6.svg)
+用于é…ç½®/ç®¡ç† Kubernetes 集群的 Rancher 组件。 + +### 构建 Rancher 仓库 + +æ¯ä¸ªä»“库都应该有一个 Makefile，并且å¯ä»¥ä½¿ç”¨ `make` 命令进行构建。`make` 目标基于仓库中 `/scripts` 目录中的脚本，æ¯ä¸ªç›®æ ‡éƒ½ä½¿ç”¨ [Dapper](https://github.com/rancher/dapper) 在孤立的环境中è¿è¡Œã€‚`Dockerfile.dapper` 将用于此æ“作，它包å«äº†æ‰€éœ€çš„所有构建工具。 + +默认目标是 `ci`，它将è¿è¡Œ `./scripts/validate`ã€`./scripts/build`ã€`./scripts/test ` å’Œ `./scripts/package`。生æˆçš„二进制文件将在 `./build/bin` 中，通常也打包在 Docker é•œåƒä¸­ã€‚ + +### Rancher Bugã€Issue 或疑问 + +如果你å‘现任何 bug 或问题，由于有人å¯èƒ½é‡åˆ°è¿‡åŒæ ·çš„问题，或者我们已ç»æ­£åœ¨å¯»æ‰¾è§£å†³æ–¹æ¡ˆï¼Œå› æ­¤è¯·å…ˆåœ¨[已报告 issue](https://github.com/rancher/rancher/issues) 中æœç´¢ã€‚ + +如果找ä¸åˆ°ä¸Žä½ çš„问题相关的内容，请通过[æ出 issue](https://github.com/rancher/rancher/issues/new) 与我们è”系。与 Rancher 相关的仓库有很多，但请将 issue æ交到 Rancher 仓库中，这样能确ä¿æˆ‘们能看到这些 issue。如果你想就一个用例æ出问题或询问其他用户，你å¯ä»¥åœ¨ [Rancher 论å›](https://forums.rancher.com)上å‘帖。 + +#### æ交 Issue çš„æ£€æŸ¥æ¸…å• + +æ交问题时请éµå¾ªæ­¤æ¸…å•ï¼Œä»¥ä¾¿æˆ‘们调查和解决问题。如果你能æ供更多信æ¯ï¼Œæˆ‘们就å¯ä»¥ä½¿ç”¨æ›´å¤šæ•°æ®æ¥ç¡®å®šå¯¼è‡´é—®é¢˜çš„原因或å‘现更多相关的内容。 + +:::note + +如果数æ®é‡å¾ˆå¤§ï¼Œè¯·ä½¿ç”¨ [GitHub Gist](https://gist.github.com/) 或类似工具，并在 issue 中链接你创建的资æºã€‚ + +::: + +:::note é‡è¦æ示： + +请删除所有æ•æ„Ÿæ•°æ®ã€‚ + +::: + +- **资æº**：请尽é‡è¯¦ç»†åœ°æ供所使用的资æºã€‚导致问题的原因å¯èƒ½å¾ˆå¤šï¼Œå› æ­¤è¯·å°½é‡æ供更多细节æ¥å¸®åŠ©æˆ‘们确定根本原因。下é¢æ˜¯ä¸€äº›å‚考示例： + - **主机**：主机的规格（例如 CPU/内存/ç£ç›˜ï¼‰ï¼Œè¿è¡Œåœ¨ä»€ä¹ˆäº‘厂商上，使用的 Amazon Machine Image，使用的 DigitalOcean droplet，é…置的镜åƒï¼ˆå¤çŽ°æ—¶ç”¨äºŽé‡æ–°æž„建或使用）。 + - **æ“作系统**：使用的是什么æ“作系统。在此处æ供详细信æ¯ï¼Œä¾‹å¦‚ `cat /etc/os-release` 的输出（确切的æ“作系统版本）和 `uname -r` 的输出（确切的内核）。 + - **Docker**：使用的 Docker 版本以åŠå®‰è£…的方法。Docker 的大部分详情都å¯ä»¥åœ¨ `docker version` å’Œ `docker info` 的输出中找到。 + - **环境**：是å¦ä½¿ç”¨äº†ä»£ç†ï¼Œæ˜¯å¦ä½¿ç”¨å¯ä¿¡çš„ CA/自签åè¯ä¹¦ï¼Œæ˜¯å¦ä½¿ç”¨äº†å¤–部负载å‡è¡¡å™¨ã€‚ + - **Rancher**：使用的 Rancher 版本，å¯ä»¥åœ¨ UI 左下角或者从主机è¿è¡Œçš„ image 标签中获å–。 + - **集群**：创建了什么样的集群，如何创建的，在创建时指定了什么å‚数。 +- **å¤çŽ° issue 的步骤**：尽é‡è¯¦ç»†åœ°è¯´æ˜Žä½ æ˜¯å¦‚何触å‘所报告的情况的。这有助于å¤çŽ°ä½ çš„情况。 + - æ供从创建到你报告的情况使用的手动步骤或自动化脚本。 +- **日志**：æ供使用资æºçš„æ•°æ®/日志。 + - Rancher + - Docker 安装 + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher:|rancher/rancher " | awk '{ print $1 }') + ``` + - 使用 `kubectl` çš„ Kubernetes 安装 + + :::note + + ç¡®ä¿ä½ é…置了正确的 kubeconfig（例如，如果 Rancher 安装在 Kubernetes 集群上，则 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + -l app=rancher \ + --timestamps=true + ``` + - 在 RKE 集群的æ¯ä¸ªèŠ‚点上使用 `docker` çš„ Docker 安装 + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher@|rancher_rancher" | awk '{ print $1 }') + ``` + - 使用 RKE 附加组件的 Kubernetes 安装 + + :::note + + ç¡®ä¿ä½ é…置了正确的 kubeconfig（例如，如果 Rancher Server 安装在 Kubernetes 集群上，则 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + --timestamps=true \ + -f $(kubectl --kubeconfig $KUBECONFIG get pods -n cattle-system -o json | jq -r '.items[] | select(.spec.containers[].name="cattle-server") | .metadata.name') + ``` + - 系统日志记录（å¯èƒ½ä¸å­˜åœ¨ï¼Œå–决于æ“作系统） + - `/var/log/messages` + - `/var/log/syslog` + - `/var/log/kern.log` + - Docker Daemon 日志记录（å¯èƒ½å¹¶ä¸å…¨éƒ¨å­˜åœ¨ï¼Œå–决于æ“作系统） + - `/var/log/docker.log` +- **指标**：如果你é‡åˆ°æ€§èƒ½é—®é¢˜ï¼Œè¯·æ供尽å¯èƒ½å¤šçš„指标数æ®ï¼ˆæ–‡ä»¶æˆ–å±å¹•æˆªå›¾ï¼‰æ¥å¸®åŠ©æˆ‘们确定问题。如果你é‡åˆ°ä¸»æœºç›¸å…³çš„问题，你å¯ä»¥æä¾› `top`ã€`free -m`ã€`df` 的输出，这些输出会显示进程/内存/ç£ç›˜çš„使用情况。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/container-network-interface-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/container-network-interface-providers.md new file mode 100644 index 00000000000..ad796bb5c4c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/container-network-interface-providers.md @@ -0,0 +1,204 @@ +--- +title: CNI 网络æ’件 +description: äº†è§£å®¹å™¨ç½‘ç»œæŽ¥å£ (CNI)ã€Rancher æ供的 CNI 网络æ’件ã€æ供商的功能，以åŠå¦‚何选择网络æ供商 +--- + +## 什么是 CNI？ + +CNI（容器网络接å£ï¼‰æ˜¯ä¸€ä¸ª[云原生计算基金会项目](https://cncf.io/)，它包å«äº†ä¸€äº›è§„范和库，用于编写在 Linux 容器中é…置网络接å£çš„一系列æ’件。CNI åªå…³æ³¨å®¹å™¨çš„网络连接，并在容器被删除时移除所分é…的资æºã€‚ + +Kubernetes 使用 CNI 作为网络æ供商和 Kubernetes Pod 网络之间的接å£ã€‚ + +![CNI Logo](/img/cni-logo.png) + +有关更多信æ¯ï¼Œè¯·è®¿é—® [CNI GitHub 项目](https://github.com/containernetworking/cni)。 + +## CNI 使用了哪些网络模型？ + +CNI 网络æ’件使用å°è£…网络模型（例如 Virtual Extensible Lan，缩写是 [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)）或éžå°è£…网络模型（例如 Border Gateway Protocol，缩写是 [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol)）æ¥å®žçŽ°ç½‘络结构。 + +### 什么是å°è£…网络？ + +此网络模型æ供了一个逻辑二层（L2）网络，该网络å°è£…在跨 Kubernetes 集群节点的现有三层（L3）网络拓扑上。使用此模型，你å¯ä»¥ä¸ºå®¹å™¨æ供一个隔离的 L2 网络，而无需分å‘路由。å°è£…网络带æ¥äº†å°‘é‡çš„处ç†å¼€é”€ä»¥åŠç”±äºŽè¦†ç›–å°è£…ç”Ÿæˆ IP header 造æˆçš„ IP 包大å°å¢žåŠ ã€‚å°è£…ä¿¡æ¯ç”± Kubernetes worker 之间的 UDP 端å£åˆ†å‘，交æ¢å¦‚何访问 MAC 地å€çš„网络控制平é¢ä¿¡æ¯ã€‚此类网络模型中常用的å°è£…是 VXLANã€Internet å议安全性 (IPSec) å’Œ IP-in-IP。 + +简å•æ¥è¯´ï¼Œè¿™ç§ç½‘络模型在 Kubernetes worker 之间生æˆäº†ä¸€ç§æ‰©å±•ç½‘桥，其中连接了 pod。 + +如果你åå‘使用扩展 L2 网桥，则å¯ä»¥é€‰æ‹©æ­¤ç½‘络模型。此网络模型对 Kubernetes worker çš„ L3 网络延迟很æ•æ„Ÿã€‚如果数æ®ä¸­å¿ƒä½äºŽä¸åŒçš„地ç†ä½ç½®ï¼Œè¯·ç¡®ä¿å®ƒä»¬ä¹‹é—´çš„延迟较低，以é¿å…最终的网络分段。 + +使用这ç§ç½‘络模型的 CNI 网络æ’件包括 Flannelã€Canalã€Weave å’Œ Cilium。默认情况下，Calico ä¸ä¼šä½¿ç”¨æ­¤æ¨¡åž‹ï¼Œä½†ä½ å¯ä»¥å¯¹å…¶è¿›è¡Œé…置。 + +![å°è£…网络](/img/encapsulated-network.png) + +### 什么是éžå°è£…网络？ + +该网络模型æ供了一个 L3 网络，用于在容器之间路由数æ®åŒ…。此模型ä¸ä¼šç”Ÿæˆéš”离的 L2 网络，也ä¸ä¼šäº§ç”Ÿå¼€é”€ã€‚这些好处的代价是，Kubernetes worker 必须管ç†æ‰€éœ€çš„所有路由分å‘。该网络模型ä¸ä½¿ç”¨ IP header 进行å°è£…，而是使用 Kubernetes Worker 之间的网络åè®®æ¥åˆ†å‘路由信æ¯ä»¥å®žçŽ° Pod 连接，例如 [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol)。 + +简而言之，这ç§ç½‘络模型在 Kubernetes worker 之间生æˆäº†ä¸€ç§æ‰©å±•ç½‘络路由器，æ供了如何连接 Pod çš„ä¿¡æ¯ã€‚ + +如果你åå‘使用 L3 网络，则å¯ä»¥é€‰æ‹©æ­¤ç½‘络模型。此模型在æ“作系统级别为 Kubernetes Worker 动æ€æ›´æ–°è·¯ç”±ã€‚对延迟较ä¸æ•æ„Ÿã€‚ + +使用这ç§ç½‘络模型的 CNI 网络æ’件包括 Calico å’Œ Cilium。Cilium å¯ä»¥ä½¿ç”¨æ­¤æ¨¡åž‹è¿›è¡Œé…置，å³ä½¿è¿™ä¸æ˜¯é»˜è®¤æ¨¡å¼ã€‚ + +![éžå°è£…网络](/img/unencapsulated-network.png) + +## Rancher æ供哪些 CNI æ’件？ + +### RKE Kubernetes 集群 + +Rancher 开箱å³ç”¨åœ°ä¸º RKE Kubernetes 集群æ供了几个 CNI 网络æ’件，分别是 Canalã€Flannelã€Calico å’Œ Weave。 + +如果你使用 Rancher 创建新的 Kubernetes 集群，你å¯ä»¥é€‰æ‹©ä½ çš„ CNI 网络æ’件。 + +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal 是一个 CNI 网络æ’件，它很好地结åˆäº† Flannel å’Œ Calico 的优点。它让你轻æ¾åœ°å°† Calico å’Œ Flannel 网络部署为统一的网络解决方案，将 Calico 的网络策略执行与 Calico（未å°è£…）和 Flannel（å°è£…）丰富的网络连接选项结åˆèµ·æ¥ã€‚ + +Canal 是 Rancher 默认的 CNI 网络æ’件，并采用了 Flannel å’Œ VXLAN å°è£…。 + +Kubernetes Worker 需è¦æ‰“å¼€ UDP ç«¯å£ `8472` (VXLAN) å’Œ TCP ç«¯å£ `9099`（å¥åº·æ£€æŸ¥ï¼‰ã€‚如果使用 Wireguard，则需è¦æ‰“å¼€ UDP ç«¯å£ `51820` å’Œ `51821`。有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +![](/img/canal-diagram.png) + +有关详细信æ¯ï¼Œè¯·å‚阅 [Canal GitHub 页é¢](https://github.com/projectcalico/canal)。 + +#### Flannel + +![Flannel Logo](/img/flannel-logo.png) + +Flannel 是为 Kubernetes é…ç½® L3 网络结构的简å•æ–¹æ³•ã€‚Flannel 在æ¯å°ä¸»æœºä¸Šè¿è¡Œä¸€ä¸ªå为 flanneld 的二进制 Agent，该 Agent 负责从更大的预é…置地å€ç©ºé—´ä¸­ä¸ºæ¯å°ä¸»æœºåˆ†é…å­ç½‘租约。Flannel 通过 Kubernetes API 或直接使用 etcd æ¥å­˜å‚¨ç½‘络é…ç½®ã€åˆ†é…çš„å­ç½‘ã€ä»¥åŠå…¶ä»–辅助数æ®ï¼ˆä¾‹å¦‚主机的公共 IP）。数æ®åŒ…使用æŸç§åŽç«¯æœºåˆ¶æ¥è½¬å‘，默认å°è£…为 [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)。 + +默认情况下，å°è£…çš„æµé‡æ˜¯ä¸åŠ å¯†çš„。Flannel æ供了两ç§åŠ å¯†æ–¹æ¡ˆï¼š + +* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec)：使用 [strongSwan](https://www.strongswan.org/) 在 Kubernetes worker 之间建立加密的 IPSec 隧é“。它是加密的实验性åŽç«¯ã€‚ +* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard)：比 strongSwan 更快的替代方案。 + +Kubernetes Worker 需è¦æ‰“å¼€ UDP ç«¯å£ `8472` (VXLAN)。有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +![Flannel Diagram](/img/flannel-diagram.png) + +有关详细信æ¯ï¼Œè¯·å‚阅 [Flannel GitHub 页é¢](https://github.com/flannel-io/flannel)。 + +#### Weave + +![Weave Logo](/img/weave-logo.png) + +Weave 在云上的 Kubernetes 集群中å¯ç”¨ç½‘络和网络策略。此外，它还支æŒåŠ å¯†å¯¹ç­‰èŠ‚点之间的æµé‡ã€‚ + +Kubernetes worker 需è¦æ‰“å¼€ TCP ç«¯å£ `6783`（控制端å£ï¼‰ã€UDP ç«¯å£ `6783` å’Œ UDP ç«¯å£ `6784`（数æ®ç«¯å£ï¼‰ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +有关详细信æ¯ï¼Œè¯·å‚阅以下页é¢ï¼š + +- [Weave Net 官网](https://www.weave.works/) + +### RKE2 Kubernetes 集群 + +Rancher 开箱å³ç”¨åœ°ä¸º RKE2 Kubernetes 集群æ供了几个 CNI 网络æ’件，分别是 [Canal](#canal)（è§ä¸Šä¸€èŠ‚）ã€Calico å’Œ Cilium。 + +如果你使用 Rancher 创建新的 Kubernetes 集群，你å¯ä»¥é€‰æ‹©ä½ çš„ CNI 网络æ’件。 + +#### Calico + +![Calico Logo](/img/calico-logo.png) + +Calico 在云上的 Kubernetes 集群中å¯ç”¨ç½‘络和网络策略。默认情况下，Calico 使用纯净ã€æœªå°è£…çš„ IP 网络结构和策略引擎为 Kubernetes 工作负载æ供网络。工作负载能够使用 BGP 在云上和本地进行通信。 + +Calico 还æ供了一ç§æ— çŠ¶æ€çš„ IP-in-IP 或 VXLAN å°è£…模å¼ã€‚如果需è¦ï¼Œä½ å¯ä»¥ä½¿ç”¨å®ƒã€‚Calico 还支æŒç­–略隔离，让你使用高级 ingress å’Œ egress ç­–ç•¥ä¿æŠ¤å’Œç®¡ç† Kubernetes 工作负载。 + +如果使用 BGP，Kubernetes Worker 需è¦æ‰“å¼€ TCP ç«¯å£ `179`，如果使用 VXLAN å°è£…，则需è¦æ‰“å¼€ UDP ç«¯å£ `4789`。å¦å¤–，使用 Typha æ—¶éœ€è¦ TCP ç«¯å£ `5473`。有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +:::note é‡è¦æ示： + +在 Rancher 2.6.3 中，Calico 探测到在安装 RKE2 æ—¶ Windows 节点会失败。请注æ„，此问题已在 v2.6.4 中解决。 + +- è¦è§£å†³æ­¤é—®é¢˜ï¼Œè¯·å…ˆå¯¼èˆªåˆ° `https:///v3/settings/windows-rke2-install-script`。 + +- 在那里，将当å‰è®¾ç½® `https://raw.githubusercontent.com/rancher/wins/v0.1.3/install.ps1` 更改为新设置 `https://raw.githubusercontent .com/rancher/rke2/master/windows/rke2-install.ps1`。 + +::: + +![Calico Diagram](/img/calico-diagram.svg) + +有关详细信æ¯ï¼Œè¯·å‚阅以下页é¢ï¼š + +- [Project Calico 官方网站](https://www.projectcalico.org/) +- [Calico 项目 GitHub 页é¢](https://github.com/projectcalico/calico) + +#### Cilium + +![Cilium Logo](/img/cilium-logo.png) + +Cilium 在 Kubernetes 中å¯ç”¨ç½‘络和网络策略（L3ã€L4 å’Œ L7）。默认情况下，Cilium 使用 eBPF 技术在节点内部路由数æ®åŒ…，并使用 VXLAN 将数æ®åŒ…å‘é€åˆ°å…¶ä»–节点。你也å¯ä»¥é…ç½®éžå°è£…的技术。 + +Cilium 推è大于 5.2 的内核版本，从而充分利用 eBPF 的能力。Kubernetes worker 需è¦æ‰“å¼€ TCP ç«¯å£ `8472`（VXLAN）和 TCP ç«¯å£ `4240`（å¥åº·æ£€æŸ¥ï¼‰ã€‚此外，还必须为å¥åº·æ£€æŸ¥å¯ç”¨ ICMP 8/0。有关详细信æ¯ï¼Œè¯·æŸ¥çœ‹ [Cilium 系统è¦æ±‚](https://docs.cilium.io/en/latest/operations/system_requirements/#firewall-requirements)。 + +##### Cilium 中跨节点的 Ingress 路由 +
+默认情况下，Cilium ä¸å…许 Pod 与其他节点上的 Pod 通信。è¦è§£å†³æ­¤é—®é¢˜ï¼Œè¯·å¯ç”¨ Ingress Controller 以使用 “CiliumNetworkPolicy†进行跨节点路由请求。 + +选择 Cilium CNI 并为新集群å¯ç”¨é¡¹ç›®ç½‘络隔离åŽï¼Œé…置如下： + +``` +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: hn-nodes + namespace: default +spec: + endpointSelector: {} + ingress: + - fromEntities: + - remote-node +``` + +## å„个网络æ’件的 CNI 功能 + +下表总结了 Rancher 中æ¯ä¸ª CNI 网络æ’件支æŒçš„ä¸åŒåŠŸèƒ½ï¼š + +| æ供商 | 网络模型 | è·¯çº¿åˆ†å‘ | 网络策略 | 网格 | 外部数æ®å­˜å‚¨ | 加密 | Ingress/Egress ç­–ç•¥ | +| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Canal | å°è£… (VXLAN) | å¦ | 是 | å¦ | K8s API | 是 | 是 | +| Flannel | å°è£… (VXLAN) | å¦ | å¦ | å¦ | K8s API | 是 | å¦ | +| Calico | å°è£…（VXLAN，IPIP）或未å°è£… | 是 | 是 | 是 | Etcd å’Œ K8s API | 是 | 是 | +| Weave | å°è£… | 是 | 是 | 是 | å¦ | 是 | 是 | +| Cilium | å°è£… (VXLAN) | 是 | 是 | 是 | Etcd å’Œ K8s API | 是 | 是 | + +- 网络模型：å°è£…或未å°è£…。如需更多信æ¯ï¼Œè¯·å‚阅 [CNI 中使用的网络模型](#cni-使用了哪些网络模型)。 + +- 路由分å‘：一ç§å¤–部网关å议，用于在互è”网上交æ¢è·¯ç”±å’Œå¯è¾¾æ€§ä¿¡æ¯ã€‚BGP å¯ä»¥å¸®åŠ©è¿›è¡Œè·¨é›†ç¾¤ pod 之间的网络。此功能对于未å°è£…çš„ CNI 网络æ’件是必须的，并且通常由 BGP 完æˆã€‚如果你想构建跨网段拆分的集群，路由分å‘是一个很好的功能。 + +- 网络策略：Kubernetes æ供了强制执行规则的功能，这些规则决定了哪些 service å¯ä»¥ä½¿ç”¨ç½‘络策略进行相互通信。这是从 Kubernetes 1.7 起稳定的功能，å¯ä»¥ä¸ŽæŸäº›ç½‘络æ’件一起使用。 + +- 网格：å…许在ä¸åŒçš„ Kubernetes 集群间进行 service 之间的网络通信。 + +- 外部数æ®å­˜å‚¨ï¼šå…·æœ‰æ­¤åŠŸèƒ½çš„ CNI 网络æ’件需è¦ä¸€ä¸ªå¤–部数æ®å­˜å‚¨æ¥å­˜å‚¨æ•°æ®ã€‚ + +- 加密：å…许加密和安全的网络控制和数æ®å¹³é¢ã€‚ + +- Ingress/Egress 策略：å…è®¸ä½ ç®¡ç† Kubernetes å’Œéž Kubernetes 通信的路由控制。 + + +## CNI 社区人气 + +下表总结了ä¸åŒçš„ GitHub 指标，让你了解æ¯ä¸ªé¡¹ç›®çš„å—欢迎程度和活动。数æ®æ”¶é›†äºŽ 2022 å¹´ 1 月。 + +| æ供商 | 项目 | Stars | Forks | Contributors | +| ---- | ---- | ---- | ---- | ---- | +| Canal | https://github.com/projectcalico/canal | 679 | 100 | 21 | +| Flannel | https://github.com/flannel-io/flannel | 7k | 2.5k | 185 | +| Calico | https://github.com/projectcalico/calico | 3.1k | 741 | 224 | +| Weave | https://github.com/weaveworks/weave/ | 6.2k | 635 | 84 | +| Cilium | https://github.com/cilium/cilium | 10.6k | 1.3k | 352 | + +
+ +## 使用哪个 CNI æ’件？ + +è¿™å–决于你的项目需求。å„个æ供商都有ä¸åŒçš„功能和选项。没有一个æ供商å¯ä»¥æ»¡è¶³æ‰€æœ‰ç”¨æˆ·çš„需求。 + +Canal 是默认的 CNI 网络æ’件。对于大多数用例，我们推è你使用它。它使用 Flannel 为容器æä¾›å°è£…网络，åŒæ—¶æ·»åŠ  Calico 网络策略，å¯ä»¥åœ¨ç½‘络方é¢æ供项目/命å空间隔离。 + +## 如何é…ç½® CNI 网络æ’件？ + +如需了解如何为你的集群é…置网络æ’件，请å‚阅[集群选项](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。有关更高级的é…置选项，请å‚阅有关使用[é…置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)å’Œ[网络æ’件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/)选项æ¥é…置集群的说明。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/deprecated-features-in-v2.5.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/deprecated-features-in-v2.5.md new file mode 100644 index 00000000000..ec1898663b2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/deprecated-features-in-v2.5.md @@ -0,0 +1,26 @@ +--- +title: Rancher 弃用的功能 +--- + +### Rancher 的弃用策略是什么？ + +我们在支æŒ[æœåŠ¡æ¡æ¬¾](https://rancher.com/support-maintenance-terms)中å‘布了官方弃用策略。 + +### 在哪里å¯ä»¥æ‰¾åˆ° Rancher 已弃用的功能？ + +Rancher 会在 GitHub 上的[å‘行说明](https://github.com/rancher/rancher/releases)中公布已弃用的功能。请å‚阅以下补ä¸ç‰ˆæœ¬äº†è§£å·²å¼ƒç”¨çš„功能： + +| è¡¥ä¸ç‰ˆæœ¬ | å‘布日期 | +|---------------|---------------| +| [2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) | 2021 å¹´ 8 月 31 æ—¥ | +| [2.6.1](https://github.com/rancher/rancher/releases/tag/v2.6.1) | 2021 å¹´ 10 月 11 æ—¥ | +| [2.6.2](https://github.com/rancher/rancher/releases/tag/v2.6.2) | 2021 å¹´ 10 月 19 æ—¥ | +| [2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) | 2021 å¹´ 12 月 21 æ—¥ | +| [2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | 2022 å¹´ 3 月 31 æ—¥ | +| [2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) | 2022 å¹´ 5 月 12 æ—¥ | +| [2.6.6](https://github.com/rancher/rancher/releases/tag/v2.6.6) | 2022 å¹´ 6 月 30 æ—¥ | + + +### 如果æŸä¸ªåŠŸèƒ½æ ‡è®°ä¸ºå¼ƒç”¨ï¼Œæˆ‘è¦æ€Žä¹ˆåšï¼Ÿ + +如果æŸä¸ªå‘行版将æŸåŠŸèƒ½æ ‡è®°ä¸º"Deprecated"（已弃用），该功能ä»ç„¶å¯ç”¨å¹¶å—支æŒï¼Œä»Žè€Œå…许用户按照常规æµç¨‹è¿›è¡Œå‡çº§ã€‚在å‡çº§åˆ°è¯¥åŠŸèƒ½è¢«æ ‡è®°ä¸º"已删除"çš„å‘行版å‰ï¼Œç”¨æˆ·/管ç†å‘˜åº”该计划剥离该功能。对于新部署，我们建议ä¸è¦ä½¿ç”¨å·²å¼ƒç”¨çš„功能。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/dockershim.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/dockershim.md new file mode 100644 index 00000000000..cfab0dfbaf4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/dockershim.md @@ -0,0 +1,45 @@ +--- +title: Dockershim +--- + +Dockershim 是 Kubelet å’Œ Docker Daemon 之间的 CRI 兼容层。Kubernetes 1.20 版本宣布了[移除树内 Dockershim](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/)。目å‰è®¡åˆ’在 Kubernetes 1.24 中移除。有关此移除的更多信æ¯ä»¥åŠæ—¶é—´çº¿ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 + +从 Kubernetes 1.21 开始。RKE 集群支æŒå¤–部 Dockershim，æ¥è®©ç”¨æˆ·ç»§ç»­ä½¿ç”¨ Docker 作为 CRI è¿è¡Œæ—¶ã€‚现在，我们通过使用 [Mirantis å’Œ Docker ](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) æ¥ç¡®ä¿ RKE 集群å¯ä»¥ç»§ç»­ä½¿ç”¨ Docker，从而实现上游开æºç¤¾åŒºçš„ Dockershim。 + +è¦å¯ç”¨å¤–部 Dockershim，é…置以下选项： + +``` +enable_cri_dockerd: true +``` + +如果你想使用其他容器è¿è¡Œæ—¶ï¼ŒRancher 也æ供使用 Containerd 作为默认è¿è¡Œæ—¶çš„，以边缘为中心的 K3s，和以数æ®ä¸­å¿ƒä¸ºä¸­å¿ƒçš„ RKE2 Kubernetes å‘行版。å³ä½¿åœ¨ Kubernetes 1.24 删除了树内 Dockershim 之åŽï¼Œä½ ä¹Ÿå¯ä»¥é€šè¿‡ Rancher å‡çº§å’Œç®¡ç†å¯¼å…¥çš„ RKE2 å’Œ K3s Kubernetes 集群。 + +### 常è§é—®é¢˜ + +
+ +Q. 如果è¦èŽ·å¾— Rancher 对上游 Dockershim 的支æŒï¼Œæˆ‘需è¦å‡çº§ Rancher å—？ + +对于 RKE，Dockershim 的上游支æŒä»Ž Kubernetes 1.21 开始。你需è¦ä½¿ç”¨ Rancher 2.6 或更高版本æ‰èƒ½èŽ·å–使用 Kubernetes 1.21 çš„ RKE 的支æŒã€‚详情请å‚阅我们的[支æŒçŸ©é˜µ](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/)。 + +
+ +Q. 我目å‰çš„ RKE 使用 Kubernetes 1.20。为了é¿å…出现ä¸å†æ”¯æŒ Dockershim 的情况，我是å¦éœ€è¦å°½æ—©å°† RKE å‡çº§åˆ° Kubernetes 1.21？ + +A. 在使用 Kubernetes 1.20 çš„ RKE 中，Dockershim 版本ä¾ç„¶å¯ç”¨ï¼Œè€Œä¸”在 Kubernetes 1.24 之å‰ä¸ä¼šåœ¨ä¸Šæ¸¸å¼ƒç”¨ã€‚Kubernetes 会å‘出弃用 Dockershim 的警告，而 Rancher 在使用 Kubernetes 1.21 çš„ RKE 中已ç»ç¼“解了这个问题。你å¯ä»¥æŒ‰ç…§è®¡åˆ’正常å‡çº§åˆ° Kubernetes 1.21，但也应该考虑在å‡çº§åˆ° Kubernetes 1.22 æ—¶å¯ç”¨å¤–部 Dockershim。在å‡çº§åˆ° Kubernetes 1.24 之å‰ï¼Œä½ éœ€è¦å¯ç”¨å¤–部 Dockershim，此时现有的实现都会被删除。 + +有关此移除的更多信æ¯ä»¥åŠæ—¶é—´çº¿ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 + +
+ +Q: 如果我ä¸æƒ³å†ä¾èµ– Dockershim，我还有什么选择？ + +A: ä½ å¯ä»¥ä¸º Kubernetes 使用ä¸éœ€è¦ Dockershim 支æŒçš„è¿è¡Œæ—¶ï¼Œå¦‚ Containerd。RKE2 å’Œ K3s 就是其中的两个选项。 + +
+ +Q: 如果我目å‰ä½¿ç”¨ RKE1，但想切æ¢åˆ° RKE2，我å¯ä»¥æ€Žæ ·è¿›è¡Œè¿ç§»ï¼Ÿ + +A: Rancher 也在探索就地å‡çº§è·¯å¾„çš„å¯èƒ½æ€§ã€‚此外，你始终å¯ä»¥ä½¿ç”¨ kubectl 将工作负载è¿ç§»åˆ°å¦ä¸€ä¸ªé›†ç¾¤ã€‚ + +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/general-faq.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/general-faq.md new file mode 100644 index 00000000000..45d4ae835ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/general-faq.md @@ -0,0 +1,69 @@ +--- +title: 一般常è§é—®é¢˜è§£ç­” +--- + +本文包å«äº†ç”¨æˆ·å¸¸è§çš„ Rancher 2.x 问题。 + +有关常è§æŠ€æœ¯é—®é¢˜ï¼Œè¯·å‚阅[常è§æŠ€æœ¯é—®é¢˜è§£ç­”](technical-items.md)。 + +
+ +**Rancher 2.x æ”¯æŒ Docker Swarm å’Œ Mesos 作为环境类型å—？** + +如果你在 Rancher 2.x 中创建环境，Swarm å’Œ Mesos å°†ä¸å†æ˜¯å¯é€‰çš„标准选项。但是，Swarm å’Œ Mesos 还能继续作为å¯ä»¥éƒ¨ç½²çš„商店应用程åºã€‚这是一个艰难的决定，但这是大势所趋。比如说，15,000 多个集群å¯èƒ½åªæœ‰å¤§çº¦ 200 个在è¿è¡Œ Swarm。 + +
+ +**是å¦å¯ä»¥ä½¿ç”¨ Rancher 2.x ç®¡ç† Azure Kubernetes æœåŠ¡ï¼Ÿ** + +是的。 + +
+ +**Rancher 是å¦æ”¯æŒ Windows？** + +Rancher æ”¯æŒ Windows Server 1809 容器。有关如何使用 Windows Worker 节点设置集群的详细信æ¯ï¼Œè¯·å‚阅[为 Windows é…置自定义集群](../pages-for-subheaders/use-windows-clusters.md)。 + +
+ +**Rancher 是å¦æ”¯æŒ Istio？** + +Rancher æ”¯æŒ [Istio](../pages-for-subheaders/istio.md)。 + +此外，Istio 是在我们的微型 PaaS “Rio†中实现的，它å¯ä»¥è¿è¡Œåœ¨ Rancher 2.x 以åŠä»»ä½•ç¬¦åˆ CNCF çš„ Kubernetes 集群上。详情请å‚阅[这里](https://rio.io/)。 + +
+ +**Rancher 2.x 是å¦æ”¯æŒä½¿ç”¨ Hashicorp çš„ Vault æ¥å­˜å‚¨å¯†æ–‡ï¼Ÿ** + +密文管ç†å·²åœ¨æˆ‘们的 roadmap 上，但我们尚未将该功能分é…给特定版本。 + +
+ +**Rancher 2.x 是å¦ä¹Ÿæ”¯æŒ RKT 容器？** + +ç›®å‰ï¼Œæˆ‘们åªæ”¯æŒ Docker。 + +
+ +**Rancher 2.x 是å¦æ”¯æŒå°† Calicoã€Contivã€Contrailã€Flannelã€Weave net 等网络æ’件用于嵌入和已注册的 Kubernetes？** + +Rancher 开箱å³ç”¨åœ°ä¸º Kubernetes 集群æ供了几个 CNI 网络æ’件，分别是 Canalã€Flannelã€Calico å’Œ Weave。有关官方支æŒçš„详细信æ¯ï¼Œè¯·å‚阅 [Rancher 支æŒçŸ©é˜µ](https://rancher.com/support-maintenance-terms/)。 + +
+ +**Rancher 是å¦è®¡åˆ’æ”¯æŒ Traefik？** + +ç›®å‰ï¼Œæˆ‘们ä¸æ‰“ç®—æä¾›åµŒå…¥å¼ Traefik 支æŒï¼Œä½†æˆ‘们ä»åœ¨æŽ¢ç´¢è´Ÿè½½å‡è¡¡æ–¹æ¡ˆã€‚ + +
+ +**我å¯ä»¥å°† OpenShift Kubernetes 集群导入 2.x å—？** + +我们的目标是è¿è¡Œä»»ä½•ä¸Šæ¸¸ Kubernetes 集群。因此，Rancher 2.x 应该å¯ä»¥ä¸Ž OpenShift 一起使用，但我们尚未对此进行测试。 + +
+ +**Rancher ä¼šé›†æˆ Longhorn å—？** + +是的。Longhorn 已集æˆåˆ° Rancher 2.5+ 中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/install-and-configure-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/install-and-configure-kubectl.md new file mode 100644 index 00000000000..21c301639b6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/install-and-configure-kubectl.md @@ -0,0 +1,29 @@ +--- +title: 安装和é…ç½® kubectl +--- + +`kubectl` 是一个 CLI 工具，用于è¿è¡Œ Kubernetes 集群相关的命令。Rancher 2.x 中的许多维护和管ç†ä»»åŠ¡éƒ½éœ€è¦å®ƒã€‚ + +### 安装 + +请å‚阅 [kubectl 安装](https://kubernetes.io/docs/tasks/tools/install-kubectl/)å°† kubectl 安装到你的æ“作系统上。 + +### é…ç½® + +使用 RKE 创建 Kubernetes 集群时，RKE 会在本地目录中创建一个 `kube_config_cluster.yml`，该文件包å«ä½¿ç”¨ `kubectl` 或 `helm` 等工具连接到新集群的凭è¯ã€‚ + +ä½ å¯ä»¥å°†æ­¤æ–‡ä»¶å¤åˆ¶ä¸º `$HOME/.kube/config`。如果你使用多个 Kubernetes 集群，将 `KUBECONFIG` 环境å˜é‡è®¾ç½®ä¸º `kube_config_cluster.yml` 的路径： + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +使用 `kubectl` 测试你的连接性，并查看你是å¦å¯ä»¥èŽ·å–节点列表： + +``` +kubectl get nodes + NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.10.1 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/rancher-is-no-longer-needed.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/rancher-is-no-longer-needed.md new file mode 100644 index 00000000000..ffb98927c69 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/rancher-is-no-longer-needed.md @@ -0,0 +1,61 @@ +--- +title: å¸è½½ Rancher +--- + +本文介ç»äº†å¦‚果你ä¸å†éœ€è¦ Rancherã€ä¸æƒ³å†ç”± Rancher 管ç†é›†ç¾¤ã€æˆ–想删除 Rancher Server 需è¦æ€Žä¹ˆåšã€‚ + + +### 如果 Rancher Server 被删除，下游集群中的工作负载会怎样？ + +如果 Rancher 删除了或无法æ¢å¤ï¼ŒRancher 管ç†çš„下游 Kubernetes 集群中的所有工作负载将继续正常è¿è¡Œã€‚ + +### 如果删除了 Rancher Server，该如何访问下游集群？ + +如果删除了 Rancher，访问下游集群的方å¼å–决于集群的类型和集群的创建方å¼ã€‚总而言之： + +- **注册集群**：集群ä¸å—å½±å“，你å¯ä»¥æ³¨å†Œé›†ç¾¤å‰çš„方法访问该集群。 +- **托管的 Kubernetes 集群**：如果你在 Kubernetes 云æ供商（例如 EKSã€GKE 或 AKS）中创建集群，你å¯ä»¥ç»§ç»­ä½¿ç”¨æ供商的云凭è¯æ¥ç®¡ç†é›†ç¾¤ã€‚ +- **RKE 集群**：è¦è®¿é—® [RKE 集群](../pages-for-subheaders/launch-kubernetes-with-rancher.md)，集群必须å¯ç”¨äº†[授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆauthorized cluster endpoint，ACE）](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)，而且你必须从 Rancher UI 下载了集群的 kubeconfig 文件。RKE 集群默认å¯ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹ã€‚通过使用此端点，你å¯ä»¥ç›´æŽ¥ä½¿ç”¨ kubectl 访问你的集群，而ä¸ç”¨é€šè¿‡ Rancher Server çš„[认è¯ä»£ç†](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-认è¯ä»£ç†)进行通信。有关é…ç½® kubectl 以使用授æƒé›†ç¾¤ç«¯ç‚¹çš„说明，请å‚阅[使用 kubectl å’Œ kubeconfig 文件直接访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)。这些集群将使用删除 Rancher æ—¶é…置的身份验è¯å¿«ç…§ã€‚ + +### 如果我ä¸æƒ³å†ä½¿ç”¨ Rancher 了该怎么åšï¼Ÿ + +:::note + +之å‰æŽ¨èçš„ [System Tools](../reference-guides/system-tools.md) 自 2022 å¹´ 6 月起已弃用。 + +::: + +如果你[在 Kubernetes 集群上安装了 Rancher](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)，你å¯ä»¥ä½¿ç”¨ [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) 工具删除 Rancher。 + +在高å¯ç”¨ (HA) 模å¼ä¸‹å¸è½½ Rancher 还将删除所有 `helm-operation-*` Pod 和以下应用程åºï¼š + +- fleet +- fleet-agent +- rancher-operator +- rancher-webhook + +è‡ªå®šä¹‰èµ„æº (CRD) 和自定义命å空间ä»éœ€è¦æ‰‹åŠ¨åˆ é™¤ã€‚ + +如果你在 Docker 中安装 Rancher，则å¯ä»¥é€šè¿‡åˆ é™¤è¿è¡Œ Rancher çš„å•ä¸ª Docker 容器æ¥å¸è½½ Rancher。 + +移除 Rancher ä¸ä¼šå½±å“导入的集群。有关其他集群类型，请å‚考[移除 Rancher åŽè®¿é—®ä¸‹æ¸¸é›†ç¾¤](#如果删除了-rancher-server该如何访问下游集群)。 + +### 如果我ä¸æƒ³ Rancher 管ç†æˆ‘的注册集群该怎么办？ + +如果你在 Rancher UI 中删除了已注册的集群，则该集群将与 Rancher 分离，集群ä¸ä¼šå‘生改å˜ï¼Œä½ å¯ä»¥ä½¿ç”¨æ³¨å†Œé›†ç¾¤ä¹‹å‰çš„方法访问该集群。 + +è¦åˆ†ç¦»é›†ç¾¤ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +2. 转到è¦ä¸Ž Rancher 分离的已注册集群，然åŽå•å‡» **â‹® > 删除**。 +3. å•å‡»**删除**。 + +**结果**：注册的集群已与 Rancher 分离，并在 Rancher 外正常è¿è¡Œã€‚ + +### 如果我ä¸æƒ³ Rancher 管ç†æˆ‘çš„ RKE 集群或托管的 Kubernetes 集群该怎么办？ + +ç›®å‰ï¼Œæˆ‘们没有将这些集群从 Rancher 中分离出æ¥çš„功能。在这ç§æƒ…况下，“分离â€æŒ‡çš„是将 Rancher 组件移除出集群，并独立于 Rancher 管ç†å¯¹é›†ç¾¤çš„访问。 + +[æ­¤ issue](https://github.com/rancher/rancher/issues/25234) 跟踪了在没有 Rancher 的情况下管ç†è¿™äº›é›†ç¾¤çš„功能。 + +有关如何在删除 Rancher Server åŽè®¿é—®é›†ç¾¤çš„更多信æ¯ï¼Œè¯·å‚阅[本节](#如果删除了-rancher-server该如何访问下游集群)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/security.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/security.md new file mode 100644 index 00000000000..0078c58eac7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/security.md @@ -0,0 +1,14 @@ +--- +title: 安全 + +--- + +**是å¦æœ‰å¼ºåŒ–指å—？** + +强化指å—现在ä½äºŽ[安全](../pages-for-subheaders/rancher-security.md)部分。 + +
+ +**Rancher Kubernetes 集群 CIS Benchmark 测试的结果是什么？** + +我们已ç»é’ˆå¯¹å¼ºåŒ–çš„ Rancher Kubernetes 集群è¿è¡Œäº† CIS Kubernetes Benchmark 测试。你å¯ä»¥åœ¨[安全](../pages-for-subheaders/rancher-security.md)中找到该评估的结果。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/technical-items.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/technical-items.md new file mode 100644 index 00000000000..2bc3cfb6bfc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/technical-items.md @@ -0,0 +1,176 @@ +--- +title: 技术 +--- + +### 如何é‡ç½®ç®¡ç†å‘˜å¯†ç ï¼Ÿ + +Docker 安装： +``` +$ docker exec -ti reset-password +New password for default administrator (user-xxxxx): + +``` + +Kubernetes 安装（Helm）： +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher --no-headers | head -1 | awk '{ print $1 }') -c rancher -- reset-password +New password for default administrator (user-xxxxx): + +``` + + + +### 我删除/åœç”¨äº†æœ€åŽä¸€ä¸ª admin，该如何解决？ +Docker 安装： +``` +$ docker exec -ti ensure-default-admin +New default administrator (user-xxxxx) +New password for default administrator (user-xxxxx): + +``` + +Kubernetes 安装（Helm）： +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher | grep '1/1' | head -1 | awk '{ print $1 }') -- ensure-default-admin +New password for default administrator (user-xxxxx): + +``` +### 如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录？ + +请å‚阅[故障排除：日志记录](../troubleshooting/other-troubleshooting-tips/logging.md)。 + +### 我的 ClusterIP ä¸å“应 ping，该如何解决？ + +ClusterIP 是一个虚拟 IP，ä¸ä¼šå“应 ping。è¦æµ‹è¯• ClusterIP 是å¦é…置正确，最好的方法是使用 `curl` 访问 IP 和端å£å¹¶æ£€æŸ¥å®ƒæ˜¯å¦å“应。 + +### 在哪里管ç†èŠ‚点模æ¿ï¼Ÿ + +打开你的账å·èœå•ï¼ˆå³ä¸Šè§’）并选择`节点模æ¿`。 + +### 为什么我的四层负载å‡è¡¡å™¨å¤„于 `Pending` 状æ€ï¼Ÿ + +四层负载å‡è¡¡å™¨åˆ›å»ºä¸º `type: LoadBalancer`。Kubernetes 需è¦ä¸€ä¸ªå¯ä»¥æ»¡è¶³è¿™äº›è¯·æ±‚的云æ供商或控制器，å¦åˆ™è¿™äº›è¯·æ±‚将永远处于 `Pending` 状æ€ã€‚有关更多信æ¯ï¼Œè¯·å‚阅[云æ供商](../pages-for-subheaders/set-up-cloud-providers.md)或[创建外部负载å‡è¡¡å™¨](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/)。 + +### Rancher 的状æ€å­˜å‚¨åœ¨å“ªé‡Œï¼Ÿ + +- Docker 安装：在 `rancher/rancher` å®¹å™¨çš„åµŒå…¥å¼ etcd 中，ä½äºŽ `/var/lib/rancher`。 +- Kubernetes install：在为è¿è¡Œ Rancher 而创建的 RKE 集群的 etcd 中。 + +### 支æŒçš„ Docker 版本是如何确定的？ + +我们éµå¾ªä¸Šæ¸¸ Kubernetes 版本验è¯è¿‡çš„ Docker 版本。如果需è¦èŽ·å–验è¯è¿‡çš„版本，请查看 Kubernetes 版本 CHANGELOG.md 中的 [External Dependencies](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.10.md#external-dependencies)。 + +### 如何访问 Rancher 创建的节点？ + +ä½ å¯ä»¥è½¬åˆ°**节点**视图，然åŽä¸‹è½½ç”¨äºŽè®¿é—® Rancher 创建的节点的 SSH 密钥。选择è¦è®¿é—®çš„节点并å•å‡»è¡Œå°¾ **â‹®** 按钮，然åŽé€‰æ‹©**下载密钥**，如下图所示。 + +![下载密钥](/img/downloadsshkeys.png) + +解压缩下载的 zip 文件，并使用 `id_rsa` 文件连接到你的主机。请务必使用正确的用户å（如果是 RancherOS，则使用 `rancher` 或 `docker`；如果是 Ubuntu，则使用 `ubuntu`；如果是 Amazon Linux，则使用 `ec2-user`）。 + +``` +$ ssh -i id_rsa user@ip_of_node +``` + +### 如何在 Rancher 中自动化任务 X？ + +UI ç”±é™æ€æ–‡ä»¶ç»„æˆï¼Œå¹¶æ ¹æ® API çš„å“应工作。æ¢è¨€ä¹‹ï¼ŒUI 中å¯ä»¥æ‰§è¡Œçš„æ¯ä¸ªæ“作/任务都å¯ä»¥é€šè¿‡ API 进行自动化。有两ç§æ–¹æ³•å¯ä»¥å®žçŽ°è¿™ä¸€ç‚¹ï¼š + +* 访问 `https://your_rancher_ip/v3` 并æµè§ˆ API 选项。 +* 在使用 UI æ—¶æ•èŽ· API 调用（通常使用 [Chrome å¼€å‘者工具](https://developers.google.com/web/tools/chrome-devtools/#network)，但你也å¯ä»¥ä½¿ç”¨å…¶ä»–工具）。 + +### 节点的 IP 地å€æ”¹å˜äº†ï¼Œè¯¥å¦‚何æ¢å¤ï¼Ÿ + +节点需è¦é…ç½®é™æ€ IP（或使用 DHCP ä¿ç•™çš„ IP）。如果节点的 IP 已更改，你必须在集群中删除并é‡æ–°æ·»åŠ å®ƒã€‚删除åŽï¼ŒRancher 会将集群更新为正确的状æ€ã€‚如果集群ä¸å†å¤„于 `Provisioning` 状æ€ï¼Œåˆ™å·²ä»Žé›†ç¾¤åˆ é™¤è¯¥èŠ‚点。 + +节点的 IP 地å€å‘生å˜åŒ–时，Rancher 会失去与节点的连接，因此无法正常清ç†èŠ‚点。请å‚阅[清ç†é›†ç¾¤èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md)æ¥æ¸…ç†èŠ‚点。 + +在集群中移除并清ç†èŠ‚点时，你å¯ä»¥å°†èŠ‚点é‡æ–°æ·»åŠ åˆ°é›†ç¾¤ä¸­ã€‚ + +### 如何将其他å‚æ•°/绑定/环境å˜é‡æ·»åŠ åˆ° Rancher å¯åŠ¨çš„ Kubernetes 集群的 Kubernetes 组件中？ + +ä½ å¯ä»¥ä½¿ç”¨é›†ç¾¤é€‰é¡¹ä¸­çš„[é…置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)选项æ¥æ·»åŠ å…¶ä»–å‚æ•°/​​绑定/环境å˜é‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 RKE 文档中的[其他å‚æ•°ã€ç»‘定和环境å˜é‡](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/)，或æµè§ˆ [Cluster.ymls 示例](https://rancher.com/docs/rke/latest/en/example-yamls/)。 + +### 如何检查è¯ä¹¦é“¾æ˜¯å¦æœ‰æ•ˆï¼Ÿ + +使用 `openssl verify` 命令æ¥éªŒè¯ä½ çš„è¯ä¹¦é“¾ï¼š + +:::tip + +å°† `SSL_CERT_DIR` å’Œ `SSL_CERT_FILE` é…置到虚拟ä½ç½®ï¼Œä»Žè€Œç¡®ä¿åœ¨æ‰‹åŠ¨éªŒè¯æ—¶ä¸ä½¿ç”¨æ“作系统安装的è¯ä¹¦ã€‚ + +::: + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +如果你看到 `unable to get local issuer certificate` 错误，则表示链ä¸å®Œæ•´ã€‚通常情况下，这表示你的æœåŠ¡å™¨è¯ä¹¦ç”±ä¸­é—´ CA é¢å‘。如果你已ç»æ‹¥æœ‰æ­¤è¯ä¹¦ï¼Œä½ å¯ä»¥åœ¨è¯ä¹¦çš„验è¯ä¸­ä½¿ç”¨å®ƒï¼Œå¦‚下所示： + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem -untrusted intermediate.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +如果你已æˆåŠŸéªŒè¯è¯ä¹¦é“¾ï¼Œä½ éœ€è¦åœ¨æœåŠ¡å™¨è¯ä¹¦ä¸­åŒ…å«æ‰€éœ€çš„中间 CA è¯ä¹¦ï¼Œä»Žè€Œå®Œæˆä¸Ž Rancher 连接的è¯ä¹¦é“¾ï¼ˆä¾‹å¦‚，使用 Rancher Agent）。æœåŠ¡å™¨è¯ä¹¦æ–‡ä»¶ä¸­è¯ä¹¦çš„顺åºé¦–先是æœåŠ¡å™¨è¯ä¹¦æœ¬èº«ï¼ˆ`rancher.yourdomain.com.pem` 的内容），然åŽæ˜¯ä¸­é—´ CA è¯ä¹¦ï¼ˆ`intermediate.pem` 的内容）： + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +如果在验è¯è¿‡ç¨‹ä¸­ä»ç„¶å‡ºçŽ°é”™è¯¯ï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œæ£€ç´¢æœåŠ¡å™¨è¯ä¹¦çš„主题和é¢å‘者： + +``` +openssl x509 -noout -subject -issuer -in rancher.yourdomain.com.pem +subject= /C=GB/ST=England/O=Alice Ltd/CN=rancher.yourdomain.com +issuer= /C=GB/ST=England/O=Alice Ltd/CN=Alice Intermediate CA +``` + +### 如何在æœåŠ¡å™¨è¯ä¹¦ä¸­æ£€æŸ¥ `Common Name` å’Œ `Subject Alternative Names`？ + +è™½ç„¶æŠ€æœ¯ä¸Šä»…éœ€è¦ `Subject Alternative Names` 中有一个æ¡ç›®ï¼Œä½†åœ¨ `Common Name` å’Œ `Subject Alternative Names` 中都包å«ä¸»æœºåå¯ä»¥æœ€å¤§ç¨‹åº¦åœ°æ高与旧版æµè§ˆå™¨/应用程åºçš„兼容性。 + +检查 `Common Name`： + +``` +openssl x509 -noout -subject -in cert.pem +subject= /CN=rancher.my.org +``` + +检查 `Subject Alternative Names`： + +``` +openssl x509 -noout -in cert.pem -text | grep DNS + DNS:rancher.my.org +``` + +### 为什么节点å‘生故障时é‡æ–°è°ƒåº¦ä¸€ä¸ª pod éœ€è¦ 5 分钟以上的时间？ + +这是以下默认 Kubernetes 设置的组åˆå¯¼è‡´çš„： + +* kubelet + * `node-status-update-frequency`：指定 kubelet 将节点状æ€å‘布到 master 的频率（默认 10s）。 +* kube-controller-manager + * `node-monitor-period`：在 NodeController 中åŒæ­¥ NodeStatus 的周期（默认 5s）。 + * `node-monitor-grace-period`：在将节点标记为ä¸å¥åº·ä¹‹å‰ï¼Œå…许节点无å“应的时间长度（默认 40s）。 + * `pod-eviction-timeout`：在故障节点上删除 pod 的宽é™æœŸï¼ˆé»˜è®¤ 5m0s）。 + +有关这些设置的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes：kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) å’Œ [Kubernetes：kube-controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/)。 + +Kubernetes 1.13 默认å¯ç”¨ `TaintBasedEvictions` 功能。有关详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes：基于污点的驱é€](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions)。 + +* kube-apiserver（Kubernetes 1.13 åŠæ›´é«˜ç‰ˆæœ¬ï¼‰ + * `default-not-ready-toleration-seconds`：表示 `notReady:NoExecute` 的容å¿åº¦çš„ `tolerationSeconds`，该设置默认添加到还没有该容å¿åº¦çš„ pod。 + * `default-unreachable-toleration-seconds`：表示 `unreachable:NoExecute` 的容å¿åº¦çš„ `tolerationSeconds`，该设置默认添加到还没有该容å¿åº¦çš„ pod。 + +### 我å¯ä»¥åœ¨ UI 中使用键盘快æ·é”®å—？ + +是的，你å¯ä»¥ä½¿ç”¨é”®ç›˜å¿«æ·é”®è®¿é—® UI 的大部分内容。è¦æŸ¥çœ‹å¿«æ·æ–¹å¼çš„概览，请在 UI ä»»æ„ä½ç½®æŒ‰ `?`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/telemetry.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/telemetry.md new file mode 100644 index 00000000000..400f6e839ad --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/faq/telemetry.md @@ -0,0 +1,31 @@ +--- +title: é¥æµ‹ +--- + +### 什么是é¥æµ‹ï¼Ÿ + +é¥æµ‹ï¼ˆTelemetry）收集 Rancher 安装大å°ã€ä½¿ç”¨çš„组件版本以åŠä½¿ç”¨åŠŸèƒ½çš„汇总信æ¯ã€‚Rancher Labs 会使用此信æ¯æ¥æ”¹è¿›äº§å“，我们ä¸ä¼šä¸Žç¬¬ä¸‰æ–¹å…±äº«æ­¤ä¿¡æ¯ã€‚ + +### 收集什么信æ¯ï¼Ÿ + +我们ä¸ä¼šæ”¶é›†ä»»ä½•è¯†åˆ«ä¿¡æ¯ï¼ˆå¦‚用户åã€å¯†ç æˆ–用户资æºçš„å称或地å€ï¼‰ã€‚ + +收集的主è¦å†…容包括： + +- æ¯ä¸ªé›†ç¾¤çš„节点总数（最å°ã€å¹³å‡ã€æœ€å¤§ã€æ€»æ•°ï¼‰åŠå…¶å¤§å°ï¼ˆä¾‹å¦‚ CPU 核心数和 RAM）。 +- 集群ã€é¡¹ç›®ã€å‘½å空间和 Pod 等逻辑资æºçš„èšåˆè®¡æ•°ã€‚ +- 用于部署集群和节点的驱动程åºè®¡æ•°ï¼ˆä¾‹å¦‚ GKEã€EC2ã€å¯¼å…¥ä¸Žè‡ªå®šä¹‰ï¼‰ã€‚ +- 部署在节点上的 Kubernetes 组件ã€æ“作系统和 Docker 的版本。 +- 是å¦å¯ç”¨äº†æŸäº›å¯é€‰ç»„件（例如，使用了哪些身份验è¯æ供程åºï¼‰ã€‚ +- è¿è¡Œçš„ Rancher çš„é•œåƒå称和版本。 +- 此安装的唯一éšæœºæ ‡è¯†ç¬¦ã€‚ + +### 我å¯ä»¥çœ‹åˆ°å‘é€çš„ä¿¡æ¯å—？ + +如果å¯ç”¨äº†é¥æµ‹ï¼Œä½ å¯ä»¥è½¬åˆ° `https:///v1-telemetry` 查看当å‰æ•°æ®ã€‚ + +如果未å¯ç”¨é¥æµ‹ï¼Œåˆ™æ”¶é›†æ•°æ®çš„进程未è¿è¡Œï¼Œå› æ­¤æ²¡æœ‰å¯ä¾›æŸ¥çœ‹çš„内容。 + +### 如何打开或关闭它？ + +完æˆåˆå§‹è®¾ç½®åŽï¼Œç®¡ç†å‘˜å¯ä»¥è½¬åˆ° UI `全局`中的`设置`页é¢ï¼Œå•å‡»**编辑**，然åŽå°† `telemetry-opt` 更改为 `in` 或 `out`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md new file mode 100644 index 00000000000..dee4c863bcf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md @@ -0,0 +1,89 @@ +--- +title: 在离线环境中渲染 Helm æ¨¡æ¿ +--- + +:::note + +以下说明å‡è®¾ä½ å·²ç»æŒ‰ç…§[本页](upgrades.md)çš„ Kubernetes å‡çº§è¯´æ˜Žæ“作（包括先决æ¡ä»¶ï¼‰åˆ°æ­¥éª¤ 3：å‡çº§ Rancher。 + +::: + +### Rancher Helm 模æ¿é€‰é¡¹ + +使用安装 Rancher 时选择的选项æ¥æ¸²æŸ“ Rancher 模æ¿ã€‚å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚Rancher 需è¦é…置为使用ç§æœ‰é•œåƒä»“库，以便é…置所有 Rancher å¯åŠ¨çš„ Kubernetes 集群或 Rancher 工具。 + +æ ¹æ®ä½ åœ¨å®‰è£…过程中åšå‡ºçš„选择，完æˆä»¥ä¸‹æ­¥éª¤ä¹‹ä¸€ã€‚ + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 输出压缩包的版本å·ã€‚ | +| `` | 指å‘è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 | +| `` | ä½ çš„ç§æœ‰é•œåƒä»“库的 DNS å称。 | +| `` | 在 K8s 集群上è¿è¡Œçš„ cert-manager 版本。 | + + +### 选项 A：使用默认的自签åè¯ä¹¦ + +``` +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # prevent files for Helm hooks from being generated + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +### 选项 B：使用 Kubernetes 密文从文件中获å–è¯ä¹¦ + +```plain +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # prevent files for Helm hooks from being generated + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +如果你使用的是ç§æœ‰ CA ç­¾åçš„è¯ä¹¦ï¼Œè¯·åœ¨ `--set ingress.tls.source=secret` åŽåŠ ä¸Š `--set privateCA=true`： + +```plain +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # prevent files for Helm hooks from being generated + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +### åº”ç”¨å·²æ¸²æŸ“çš„æ¨¡æ¿ + +将渲染的 manifest 目录å¤åˆ¶åˆ°å¯ä»¥è®¿é—® Rancher Server 集群的系统中，并应用渲染的模æ¿ã€‚ + +使用 `kubectl` æ¥åº”用渲染的 manifest。 + +```plain +kubectl -n cattle-system apply -R -f ./rancher +``` + +## 验è¯å‡çº§ + +登录 Rancher 以确认å‡çº§æˆåŠŸã€‚ + +:::tip + +å‡çº§åŽå‡ºçŽ°ç½‘络问题？ + +请å‚è§[æ¢å¤é›†ç¾¤ç½‘络](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md)。 + +::: + +## 已知å‡çº§é—®é¢˜ + +ä½ å¯ä»¥åœ¨ [GitHub](https://github.com/rancher/rancher/releases) å‘å¸ƒè¯´æ˜Žä»¥åŠ [Rancher 论å›](https://forums.rancher.com/c/announcements/12)中找到æ¯ä¸ª Rancher 版本的已知问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md new file mode 100644 index 00000000000..22de9e7af35 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md @@ -0,0 +1,147 @@ +--- +title: 在 Azure Kubernetes Service 上安装 Rancher +--- + +本文介ç»äº†å¦‚何在微软的 Azure Kubernetes Service (AKS) 上安装 Rancher。 + +本指å—使用命令行工具æ¥é…置一个带有 Ingress çš„ AKS 集群。如果你更喜欢使用 Azure 门户æ¥é…置集群，请å‚è§[官方文档](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal)。 + +如果你已有一个 AKS Kubernetes 集群，请直接跳到[安装 Ingress](#5-安装-ingress) 的步骤，然åŽæŒ‰ç…§[此页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。 + +## 先决æ¡ä»¶ + +:::caution + +部署到 Microsoft Azure 会产生费用。 + +::: + +- [Microsoft Azure è´¦å·](https://azure.microsoft.com/en-us/free/)：用于创建部署 Rancher å’Œ Kubernetes 的资æºã€‚ +- [Microsoft Azure 订阅](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹å¦‚何创建 Microsoft Azure 订阅。 +- [Micsoroft Azure 租户](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant)：访问此链接并å‚考教程以创建 Microsoft Azure 租户。 +- 你的订阅有足够的é…é¢ï¼Œè‡³å°‘有 2 个 vCPU。有关 Rancher Server 资æºè¦æ±‚的详情，请å‚è§[此节](../../../pages-for-subheaders/installation-requirements.md#rke-和托管-kubernetes)。 +- 在 Azure 中用 Helm 安装 Rancher 时，请使用 L7 è´Ÿè½½å‡è¡¡å™¨æ¥é¿å…网络问题。详情请å‚è§ [Azure è´Ÿè½½å‡è¡¡å™¨é™åˆ¶](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations)。 + +## 1. 准备你的工作站 + +在工作站上安装以下命令行工具： + +- **az**，Azure CLI：如需获得帮助，请å‚è§[安装步骤](https://docs.microsoft.com/en-us/cli/azure/)。 +- **kubectl**：如需获得帮助，请å‚è§[安装步骤](https://kubernetes.io/docs/tasks/tools/#kubectl)。 +- **helm**：如需获å–帮助，请å‚è§[安装步骤](https://helm.sh/docs/intro/install/)。 + +## 2. 创建资æºç»„ + +安装 CLI åŽï¼Œä½ éœ€è¦ç”¨ä½ çš„ Azure 账户登录： + +``` +az login +``` + +创建一个 [资æºç»„](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) æ¥ä¿å­˜é›†ç¾¤çš„所有相关资æºã€‚使用一个适用于你实际情况的ä½ç½®ï¼š + +``` +az group create --name rancher-rg --location eastus +``` + +## 3. 创建 AKS 集群 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ›å»ºä¸€ä¸ª AKS 集群。选择适用于你实际情况的虚拟机大å°ã€‚如需获得å¯ç”¨çš„大å°å’Œé€‰é¡¹ï¼Œè¯·å‚è§[此处](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes)。在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +:::note + +如果你è¦ä»Žæ—§çš„ Kubernetes 版本更新到 Kubernetes v1.22 或更高版本，你还需è¦[æ›´æ–°](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx。 + +::: + +``` +az aks create \ + --resource-group rancher-rg \ + --name rancher-server \ + --kubernetes-version \ + --node-count 3 \ + --node-vm-size Standard_D2_v3 +``` + +集群部署需è¦ä¸€äº›æ—¶é—´æ‰èƒ½å®Œæˆã€‚ + +## 4. 获å–è®¿é—®å‡­è¯ + +集群部署完æˆåŽï¼ŒèŽ·å–访问凭è¯ã€‚ + +``` +az aks get-credentials --resource-group rancher-rg --name rancher-server +``` + +此命令把集群的凭è¯åˆå¹¶åˆ°çŽ°æœ‰çš„ kubeconfig 中，并å…许 `kubectl` 与集群交互。 + +## 5. 安装 Ingress + +集群需è¦ä¸€ä¸ª Ingress，以从集群外部访问 Rancherã€‚è¦ Ingress，你需è¦åˆ†é…一个公共 IP 地å€ã€‚请确ä¿ä½ æœ‰è¶³å¤Ÿçš„é…é¢ï¼Œå¦åˆ™å®ƒå°†æ— æ³•åˆ†é… IP 地å€ã€‚公共 IP 地å€çš„é™åˆ¶åœ¨æ¯ä¸ªè®¢é˜…的区域级别生效。 + +为确ä¿ä½ é€‰æ‹©äº†æ­£ç¡®çš„ Ingress-NGINX Helm Chart，首先在 [Kubernetes/ingress-nginx 支æŒè¡¨](https://github.com/kubernetes/ingress-nginx#supported-versions-table)中找到与你的 Kubernetes 版本兼容的 `Ingress-NGINX 版本`。 + +然åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ—出å¯ç”¨çš„ Helm Chart： + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +`helm search` 命令的输出包å«ä¸€ä¸ª `APP VERSION` 列。此列下的版本等åŒäºŽä½ ä¹‹å‰é€‰æ‹©çš„ `Ingress-NGINX 版本`。使用应用程åºç‰ˆæœ¬ï¼Œé€‰æ‹©ä¸€ä¸ª Chart 版本，该版本打包了与你的 Kubernetes 兼容的应用程åºã€‚例如，如果使用的是 Kubernetes v1.24，则å¯ä»¥é€‰æ‹© v4.6.0 Helm Chart，因为 Ingress-NGINX v1.7.0 与该 Chart 打包在一起，而 v1.7.0 与 Kubernetes v1.24 兼容。如有疑问，请选择最新的兼容版本。 + +了解你需è¦çš„ Helm chart `版本`åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚它安装一个带有 Kubernetes è´Ÿè½½å‡è¡¡å™¨æœåŠ¡çš„ `nginx-ingress-controller`： + +``` +helm search repo ingress-nginx -l +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ + --set controller.service.externalTrafficPolicy=Local \ + --version 4.6.0 \ + --create-namespace +``` + +## 6. 获å–è´Ÿè½½å‡è¡¡å™¨çš„ IP + +è¿è¡Œä»¥ä¸‹å‘½ä»¤èŽ·å–è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€ï¼š + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +返回的结果应与以下内容类似： + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.0.116.18 40.31.180.83 80:31229/TCP,443:31050/TCP + 67s +``` + +ä¿å­˜ `EXTERNAL-IP`。 + +## 7. 设置 DNS + +到 Rancher Server 的外部æµé‡éœ€è¦é‡å®šå‘到你创建的负载å‡è¡¡å™¨ã€‚ + +创建指å‘ä½ ä¿å­˜çš„ `EXTERNAL-IP` çš„ DNS。这个 DNS 会用作 Rancher Server çš„ URL。 + +设置 DNS 的有效方法有很多。如需获å–帮助，请å‚è§ [Azure DNS 文档中心](https://docs.microsoft.com/en-us/azure/dns/)。 + +## 8. 安装 Rancher Helm Chart + +按照[本页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。任何 Kubernetes å‘行版上安装的 Rancher çš„ Helm 说明都是一样的。 + +安装 Rancher 时，使用上一步获å–çš„ DNS å称作为 Rancher Server çš„ URL。它å¯ä»¥ä½œä¸º Helm 选项传递进æ¥ã€‚例如，如果 DNS å称是 `rancher.my.org`，你需è¦ä½¿ç”¨ `--set hostname=rancher.my.org` 选项æ¥è¿è¡Œ Helm 安装命令。 + +在此设置之上安装 Rancher 时，你还需è¦å°†ä»¥ä¸‹å€¼ä¼ é€’到 Rancher Helm 安装命令，以设置与 Rancher çš„ Ingress 资æºä¸€èµ·ä½¿ç”¨çš„ Ingress Controller çš„å称： + +``` +--set ingress.ingressClassName=nginx +``` + +请å‚阅[Helm 安装命令](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher)了解你的è¯ä¹¦é€‰é¡¹ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md new file mode 100644 index 00000000000..335e3fee425 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md @@ -0,0 +1,151 @@ +--- +title: 在 Amazon EKS 上安装 Rancher +--- + +本文介ç»äº†å¦‚何在 Amazon EKS 集群上安装 Rancher。你也å¯ä»¥[通过 AWS Marketplace 安装 Rancher](../../quick-start-guides/deploy-rancher-manager/aws-marketplace.md)。 + +如果你已ç»æœ‰ä¸€ä¸ª EKS Kubernetes 集群，请直接跳转到[安装 Ingress](#5-安装-ingress)这个步骤。然åŽæŒ‰ç…§[此处](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的步骤安装 Rancher Helm Chart。 + +## 为 Rancher Server 创建 EKS 集群 + +在本节中，你将使用命令行工具安装一个带有 Ingress çš„ EKS 集群。如果你想在 EKS 上使用 Rancher 时使用较少的资æºï¼Œè¯·ä½¿ç”¨æ­¤æ–¹æ³•ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +- 已有一个 AWS 账户。 +- 建议使用 IAM 用户而ä¸æ˜¯ AWS æ ¹è´¦æˆ·ã€‚ä½ å°†éœ€è¦ IAM 用户的访问密钥 (access key) 和密文秘钥 (secret key) æ¥é…ç½® AWS 命令行界é¢ã€‚ +- IAM 用户需è¦å…·å¤‡[eksctl 文档](https://eksctl.io/usage/minimum-iam-policies/)中æ述的最低 IAM 策略。 + +::: + +### 1. 准备你的工作站 + +在工作站上安装以下命令行工具： + +- **AWS CLI v2**：如需获å–帮助，请å‚è§[安装步骤](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)。 +- **eksctl**：如需获å–帮助，请å‚è§[安装步骤](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html)。 +- **kubectl**：如需获得帮助，请å‚è§[安装步骤](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html)。 +- **helm**：如需获å–帮助，请å‚è§[安装步骤](https://helm.sh/docs/intro/install/)。 + +### 2. é…ç½® AWS CLI + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œé…ç½® AWS CLI： + +``` +aws configure +``` + +输入以下å‚数： + +| 值 | æè¿° | +|-------|-------------| +| AWS Access Key ID | 具有 EKS æƒé™çš„ IAM 用户的访问密钥凭è¯ã€‚ | +| AWS Secret Access Key | 具有 EKS æƒé™çš„ IAM 用户的密文密钥凭è¯ã€‚ | +| Default region name | 集群节点所在的 [AWS 区域](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html#Concepts.RegionsAndAvailabilityZones.Regions)。 | +| Default output format | 输入 `json`。 | + +### 3. 创建 EKS 集群 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ›å»ºä¸€ä¸ª EKS 集群。使用适用于你的用例的 AWS 区域。在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +**注æ„**：如果你è¦ä»Žæ—§çš„ Kubernetes 版本更新到 Kubernetes v1.22 或更高版本，你还需è¦[æ›´æ–°](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx。 + +``` +eksctl create cluster \ + --name rancher-server \ + --version \ + --region us-west-2 \ + --nodegroup-name ranchernodes \ + --nodes 3 \ + --nodes-min 1 \ + --nodes-max 4 \ + --managed +``` + +使用 CloudFormation 进行的集群部署å¯èƒ½éœ€è¦ä¸€äº›æ—¶é—´æ‰èƒ½å®Œæˆã€‚ + +### 4. 测试集群 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æµ‹è¯•é›†ç¾¤ï¼š + +``` +eksctl get cluster +``` + +返回的结果应与以下内容类似： + +``` +eksctl get cluster +2021-03-18 15:09:35 [ℹ] eksctl version 0.40.0 +2021-03-18 15:09:35 [ℹ] using region us-west-2 +NAME REGION EKSCTL CREATED +rancher-server-cluster us-west-2 True +``` + +### 5. 安装 Ingress + +集群需è¦ä¸€ä¸ª Ingress，以从集群外部访问 Rancher。 + +为确ä¿ä½ é€‰æ‹©äº†æ­£ç¡®çš„ Ingress-NGINX Helm Chart，首先在 [Kubernetes/ingress-nginx 支æŒè¡¨](https://github.com/kubernetes/ingress-nginx#supported-versions-table)中找到与你的 Kubernetes 版本兼容的 `Ingress-NGINX 版本`。 + +然åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ—出å¯ç”¨çš„ Helm Chart： + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +`helm search` 命令的输出包å«ä¸€ä¸ª `APP VERSION` 列。此列下的版本等åŒäºŽä½ ä¹‹å‰é€‰æ‹©çš„ `Ingress-NGINX 版本`。使用应用程åºç‰ˆæœ¬ï¼Œé€‰æ‹©ä¸€ä¸ª Chart 版本，该版本打包了与你的 Kubernetes 兼容的应用程åºã€‚例如，如果使用的是 Kubernetes v1.23，则å¯ä»¥é€‰æ‹© v4.6.0 Helm Chart，因为 Ingress-NGINX v1.7.0 与该 Chart 打包在一起，而 v1.7.0 与 Kubernetes v1.23 兼容。如有疑问，请选择最新的兼容版本。 + +了解你需è¦çš„ Helm chart `版本`åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚它安装一个带有 Kubernetes è´Ÿè½½å‡è¡¡å™¨æœåŠ¡çš„ `nginx-ingress-controller`： + +``` +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.6.0 \ + --create-namespace +``` + +### 6. 获å–è´Ÿè½½å‡è¡¡å™¨çš„ IP + +è¿è¡Œä»¥ä¸‹å‘½ä»¤èŽ·å–è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€ï¼š + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +返回的结果应与以下内容类似： + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.100.90.18 a904a952c73bf4f668a17c46ac7c56ab-962521486.us-west-2.elb.amazonaws.com 80:31229/TCP,443:31050/TCP + 27m +``` + +ä¿å­˜ `EXTERNAL-IP`。 + +### 7. 设置 DNS + +到 Rancher Server 的外部æµé‡éœ€è¦é‡å®šå‘到你创建的负载å‡è¡¡å™¨ã€‚ + +创建指å‘ä½ ä¿å­˜çš„外部 IP 地å€çš„ DNS。这个 DNS 会用作 Rancher Server çš„ URL。 + +设置 DNS 的有效方法有很多。如需获得帮助，请å‚è§ AWS 文档中心的[转å‘æµé‡åˆ° ELB è´Ÿè½½å‡è¡¡å™¨](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer.html)。 + +### 8. 安装 Rancher Helm Chart + +按照[本页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。任何 Kubernetes å‘行版上安装的 Rancher çš„ Helm 说明都是一样的。 + +安装 Rancher 时，使用上一步获å–çš„ DNS å称作为 Rancher Server çš„ URL。它å¯ä»¥ä½œä¸º Helm 选项传递进æ¥ã€‚例如，如果 DNS å称是 `rancher.my.org`，你需è¦ä½¿ç”¨ `--set hostname=rancher.my.org` 选项æ¥è¿è¡Œ Helm 安装命令。 + +在此设置之上安装 Rancher 时，你还需è¦å°†ä»¥ä¸‹å€¼ä¼ é€’到 Rancher Helm 安装命令，以设置与 Rancher çš„ Ingress 资æºä¸€èµ·ä½¿ç”¨çš„ Ingress Controller çš„å称： + +``` +--set ingress.ingressClassName=nginx +``` + +请å‚阅[Helm 安装命令](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher)了解你的è¯ä¹¦é€‰é¡¹ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md new file mode 100644 index 00000000000..90d395c0f12 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md @@ -0,0 +1,201 @@ +--- +title: 在 GKE 集群上安装 Rancher +--- + +在本节中，你将学习如何使用 GKE 安装 Rancher。 + +如果你已ç»æœ‰ä¸€ä¸ª GKE Kubernetes 集群，请直接跳转到[安装 Ingress](#7-安装-ingress)这个步骤。然åŽæŒ‰ç…§[此处](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的步骤安装 Rancher Helm Chart。 + +## 先决æ¡ä»¶ + +- 你需è¦æœ‰ä¸€ä¸ª Google è´¦å·ã€‚ +- 你需è¦æœ‰ä¸€ä¸ª Google Cloud Billing è´¦å·ã€‚ä½ å¯ä½¿ç”¨ Google Cloud Console æ¥ç®¡ç†ä½ çš„ Cloud Billing è´¦å·ã€‚有关 Cloud Console 的详情，请å‚è§ [ Console 通用指å—](https://support.google.com/cloud/answer/3465889?hl=en&ref_topic=3340599)。 +- 你需è¦è‡³å°‘一个在用的 IP 地å€å’Œè‡³å°‘ 2 个 CPU 的云é…é¢ã€‚有关 Rancher Server 的硬件è¦æ±‚，请å‚è§[本节](../../../pages-for-subheaders/installation-requirements.md#rke-和托管-kubernetes)。 + +## 1. å¯ç”¨ Kubernetes Engine API + +按照以下步骤å¯ç”¨ Kubernetes Engine API： + +1. 访问 Google Cloud Console 中的 [Kubernetes Engine 页é¢](https://console.cloud.google.com/projectselector/kubernetes?_ga=2.169595943.767329331.1617810440-856599067.1617343886)。 +1. 创建或选择一个项目。 +1. 打开项目，并为项目å¯ç”¨ Kubernetes Engine API。等待 API 和相关æœåŠ¡çš„å¯ç”¨ã€‚è¿™å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ—¶é—´ã€‚ +1. ç¡®ä¿ä¸ºä½ çš„云项目å¯ç”¨äº†è®¡è´¹ã€‚有关如何为你的项目å¯ç”¨è®¡è´¹ï¼Œè¯·å‚è§ [Google Cloud 文档中心](https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project)。 + +## 2. 打开 Cloud Shell + +Cloud Shell 是一个 shell 环境，用于管ç†æ‰˜ç®¡åœ¨ Google Cloud 上的资æºã€‚Cloud Shell 预装了 `gcloud` 命令行工具和 kubectl 命令行工具中。`gcloud` 工具为 Google Cloud æ供主è¦çš„命令行界é¢ï¼Œè€Œ`kubectl` 则æ供针对 Kubernetes 集群的主è¦å‘½ä»¤è¡Œç•Œé¢ã€‚ + +下文æ述了如何从 Google Cloud Console 或从本地工作站å¯åŠ¨ Cloud Shell。 + +### Cloud Shell + +如需从 [Google Cloud Console](https://console.cloud.google.com) å¯åŠ¨ shell，请在控制å°çš„å³ä¸Šè§’点击终端按钮。鼠标悬åœåœ¨æŒ‰é’®ä¸Šæ—¶ï¼Œå®ƒä¼šæ ‡è®°ä¸º **Activate Cloud Shell**。 + +### 本地 Shell + +执行以下步骤以安装 `gcloud` å’Œ `kubectl`： + +1. 按照[这些步骤](https://cloud.google.com/sdk/docs/install)安装 Cloud SDK。The Cloud SDK 包括 `gcloud` 命令行工具。ä¸åŒæ“作系统对应的步骤有所ä¸åŒã€‚ +1. 安装 Cloud SDK åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ä»¥å®‰è£… `kubectl` 命令行工具： + + ``` + gcloud components install kubectl + ``` + åŽé¢çš„步骤会é…ç½® `kubectl`，使其用于使用新的 GKE 集群。 +1. 如果 Helm 3 未安装的è¯ï¼Œ[安装 Helm 3](https://helm.sh/docs/intro/install/)。 +1. 使用 `HELM_EXPERIMENTAL_OCI` å˜é‡æ¥å¯ç”¨ Helm 的实验功能 [OCI é•œåƒæ”¯æŒ](https://github.com/helm/community/blob/master/hips/hip-0006.md)。把以下行添加到 `~/.bashrc` （或 macOS 中的 `~/.bash_profile`，或者你的 shell 存储环境å˜é‡çš„地方）： + + ``` + export HELM_EXPERIMENTAL_OCI=1 + ``` +1. è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åŠ è½½ä½ æ›´æ–°çš„ `.bashrc` 文件： + + ``` + source ~/.bashrc + ``` + 如果你è¿è¡Œçš„是 macOS，使用这个命令: + ``` + source ~/.bash_profile + ``` + + + +## 3. é…ç½® gcloud CLI + +选择以下方法之一é…置默认的 gcloud 设置： + +- 如果你想了解默认值，请使用 gcloud init。 +- 如需å•ç‹¬è®¾ç½®ä½ çš„项目 IDã€åœ°åŒºå’ŒåŒºåŸŸï¼Œä½¿ç”¨ gcloud config。 + + + + +1. è¿è¡Œ gcloud init 并按照指示æ“作： + + ``` + gcloud init + ``` + 如果你在远程æœåŠ¡å™¨ä¸Šä½¿ç”¨ SSH，使用 --console-only 标志，以防止该命令å¯åŠ¨æµè§ˆå™¨ã€‚ + + ``` + gcloud init --console-only + ``` +2. æŒ‰ç…§æŒ‡ç¤ºï¼Œä»¥æŽˆæƒ gcloud 使用你的 Google Cloud 账户，并选择你创建的新项目。 + + + + + + + +## 4. 确认 gcloud çš„é…置是å¦æ­£ç¡® + +è¿è¡Œï¼š + +``` +gcloud config list +``` + +返回的结果应与以下内容类似： + +``` +[compute] +region = us-west1 # Your chosen region +zone = us-west1-b # Your chosen zone +[core] +account = +disable_usage_reporting = True +project = + +Your active configuration is: [default] +``` + +## 5. 创建一个 GKE 集群 + +下é¢çš„命令创建了一个三节点的集群。 + +把 `cluster-name` 替æ¢ä¸ºä½ æ–°é›†ç¾¤çš„å称。 + +在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +è¦ä½¿ç”¨ Rancher æˆåŠŸåˆ›å»º GKE 集群，GKE 必须处于 Standard 模å¼ã€‚GKE 在创建 Kubernetes 集群时有两ç§è¿è¡Œæ¨¡å¼ï¼Œåˆ†åˆ«æ˜¯ Autopilot å’Œ Standard 模å¼ã€‚Autopilot 模å¼çš„集群é…置对编辑 kube-system 命å空间有é™åˆ¶ã€‚但是，Rancher 在安装时需è¦åœ¨ kube-system 命å空间中创建资æºã€‚因此，你将无法在以 Autopilot 模å¼åˆ›å»ºçš„ GKE 集群上安装 Rancher。如需详细了解 GKE Autopilot 模å¼å’Œ Standard 模å¼ä¹‹é—´çš„差异，请访问[比较 GKE Autopilot å’Œ Standard ](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison)。 + +**注æ„**：如果你è¦ä»Žæ—§çš„ Kubernetes 版本更新到 Kubernetes v1.22 或更高版本，你还需è¦[æ›´æ–°](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx。 + +``` +gcloud container clusters create cluster-name --num-nodes=3 --cluster-version= +``` + +## 6. 获å–验è¯å‡­è¯ + +创建集群åŽï¼Œä½ éœ€è¦èŽ·å¾—认è¯å‡­è¯æ‰èƒ½ä¸Žé›†ç¾¤äº¤äº’： + +``` +gcloud container clusters get-credentials cluster-name +``` + +此命令将 `kubectl` é…ç½®æˆä½¿ç”¨ä½ åˆ›å»ºçš„集群。 + +## 7. 安装 Ingress + +集群需è¦ä¸€ä¸ª Ingress，以从集群外部访问 Rancher。 + +以下命令安装带有 LoadBalancer æœåŠ¡çš„ `nginx-ingress-controller`： + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.0.18 \ + --create-namespace +``` + +## 8. 获å–è´Ÿè½½å‡è¡¡å™¨çš„ IP + +è¿è¡Œä»¥ä¸‹å‘½ä»¤èŽ·å–è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€ï¼š + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +返回的结果应与以下内容类似： + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +ingress-nginx-controller LoadBalancer 10.3.244.156 35.233.206.34 80:31876/TCP,443:32497/TCP 81s +``` + +ä¿å­˜ `EXTERNAL-IP`。 + +## 9. 设置 DNS + +到 Rancher Server 的外部æµé‡éœ€è¦é‡å®šå‘到你创建的负载å‡è¡¡å™¨ã€‚ + +创建指å‘ä½ ä¿å­˜çš„外部 IP 地å€çš„ DNS。这个 DNS 会用作 Rancher Server çš„ URL。 + +设置 DNS 的有效方法有很多。如需获å–帮助，请å‚è§ Google Cloud 文档中的[ç®¡ç† DNS 记录](https://cloud.google.com/dns/docs/records)部分。 + +## 10. 安装 Rancher Helm Chart + +按照[本页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。任何 Kubernetes å‘行版上安装的 Rancher çš„ Helm 说明都是一样的。 + +安装 Rancher 时，使用上一步获å–çš„ DNS å称作为 Rancher Server çš„ URL。它å¯ä»¥ä½œä¸º Helm 选项传递进æ¥ã€‚例如，如果 DNS å称是 `rancher.my.org`，你需è¦ä½¿ç”¨ `--set hostname=rancher.my.org` 选项æ¥è¿è¡Œ Helm 安装命令。 + +在此设置之上安装 Rancher 时，你还需è¦è®¾ç½®ä¸Ž Rancher çš„ Ingress 资æºä¸€èµ·ä½¿ç”¨çš„ Ingress Controller çš„å称： + +``` +--set ingress.ingressClassName=nginx +``` + +请å‚阅[Helm 安装命令](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher)了解你的è¯ä¹¦é€‰é¡¹ã€‚ + +在 Rancher v2.7.5 中，如果你打算在集群上使用默认的 GKE Ingress 而ä¸å¯ç”¨ VPC 原生的集群模å¼ï¼Œåˆ™éœ€è¦è®¾ç½®ä»¥ä¸‹æ ‡å¿—： + +``` +--set service.type=NodePort +``` + +此设置是必è¦çš„，这考虑了与 ClusterIP（`cattle-system/rancher` 的默认类型）之间的兼容性问题。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml new file mode 100644 index 00000000000..ae248cc9b7c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml @@ -0,0 +1,37 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + defaults: + audit: restricted + audit-version: latest + enforce: restricted + enforce-version: latest + warn: restricted + warn-version: latest + exemptions: + namespaces: + - ingress-nginx + - kube-system + - cattle-system + - cattle-epinio-system + - cattle-fleet-system + - longhorn-system + - cattle-neuvector-system + - cattle-monitoring-system + - rancher-alerting-drivers + - cis-operator-system + - cattle-csp-adapter-system + - cattle-externalip-system + - cattle-gatekeeper-system + - istio-system + - cattle-istio-system + - cattle-logging-system + - cattle-windows-gmsa-system + - cattle-sriov-system + - cattle-ui-plugin-system + - tigera-operator + kind: PodSecurityConfiguration + name: PodSecurity + path: "" \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md new file mode 100644 index 00000000000..c306729f752 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md @@ -0,0 +1,128 @@ +--- +title: 回滚 +--- + +## 使用 Rancher 2.6.4+ 进行回滚的其他步骤 + +Rancher v2.6.4 å°† cluster-api 模å—从 v0.4.4 å‡çº§åˆ° v1.0.2。å过æ¥ï¼Œcluster-api çš„ v1.0.2 版本将集群 API 的自定义资æºå®šä¹‰ (CRD) 从 `cluster.x-k8s.io/v1alpha4` å‡çº§åˆ° `cluster.x-k8s.io/v1beta1`。当你å°è¯•å°† Rancher v2.6.4 回滚到以å‰ç‰ˆæœ¬çš„ Rancher v2.6.x 时，CRD å‡çº§åˆ° v1beta1 会导致回滚失败。这是因为使用旧 apiVersion (v1alpha4) çš„ CRD 与 v1beta1 ä¸å…¼å®¹ã€‚ + +è¦é¿å…回滚失败，你需è¦åœ¨å°è¯•æ¢å¤æ“作或回滚**之å‰**è¿è¡Œä»¥ä¸‹ Rancher 脚本： + +* `verify.sh`：检查集群中是å¦æœ‰ä»»ä½•ä¸Ž Rancher 相关的资æºã€‚ +* `cleanup.sh`：清ç†é›†ç¾¤ã€‚ + +有关详细信æ¯å’Œæºä»£ç ï¼Œè¯·å‚阅 [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup)。 + +:::caution + +`cleanup.sh` è¿è¡Œçš„时候会有åœæœºæ—¶é—´ï¼Œè¿™æ˜¯å› ä¸ºè„šæœ¬ä¼šåˆ é™¤ Rancher 创建的资æºã€‚ + +::: + +### 从 v2.6.4+ 回滚到较低版本的 v2.6.x + +1. 按照[说明](https://github.com/rancher/rancher-cleanup/blob/main/README.md)è¿è¡Œè„šæœ¬ã€‚ +1. 按照[说明](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)在现有集群上安装 rancher-backup Helm Chart 并æ¢å¤ä¹‹å‰çš„状æ€ã€‚ + 1. çœç•¥æ­¥éª¤ 3。 + 1. 执行到步骤 4 时，在è¦å›žæ»šåˆ°çš„ local 集群上安装 Rancher 2.6.x 版本。 + +## 回滚到 Rancher 2.5.0+ + +è¦å›žæ»šåˆ° Rancher 2.5.0+，使用 **Rancher 备份**应用并通过备份æ¥æ¢å¤ Rancher。 + +回滚åŽï¼ŒRancher 必须以较低/较早的版本å¯åŠ¨ã€‚ + +还原是通过创建 Restore 自定义资æºå®žçŽ°çš„。 + +:::note é‡è¦æ示： + +* 请按照此页é¢ä¸Šçš„说明在已备份的åŒä¸€é›†ç¾¤ä¸Šè¿˜åŽŸ Rancher。è¦æŠŠ Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œè¯·å‚照步骤[è¿ç§» Rancher](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +* 在使用相åŒè®¾ç½®æ¢å¤ Rancher 时，Rancher deployment 在æ¢å¤å¼€å§‹å‰è¢«æ‰‹åŠ¨ç¼©å‡ï¼Œç„¶åŽ Operator 将在æ¢å¤å®ŒæˆåŽå°†å…¶ç¼©å›žã€‚因此，在æ¢å¤å®Œæˆä¹‹å‰ï¼ŒRancher å’Œ UI 都将ä¸å¯ç”¨ã€‚如果 UI ä¸å¯ç”¨æ—¶ï¼Œä½ å¯ä½¿ç”¨ `kubectl create -f restore.yaml`YAML æ¢å¤æ–‡ä»¶æ¥ä½¿ç”¨åˆå§‹çš„集群 kubeconfig。 + +::: + +### 创建 Restore è‡ªå®šä¹‰èµ„æº + +1. 点击 **☰ > 集群管ç†**。 +1. 找到你的本地集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Rancher 备份 > 还原**。 + :::note + + 如果 Rancher Backups 应用ä¸å¯è§ï¼Œä½ éœ€è¦åˆ° **Apps** çš„ Charts 页é¢ä¸­å®‰è£…应用。详情请å‚è§[此处](../../../pages-for-subheaders/helm-charts-in-rancher.md#charts)。 + + ::: + +1. å•å‡»**创建**。 +1. 使用表å•æˆ– YAML 创建 Restore。如需获å–使用表å•åˆ›å»º Restore 资æºçš„更多信æ¯ï¼Œè¯·å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 +1. 如需使用 YAML 编辑器，点击**创建 > 使用 YAML 文件创建**。然åŽè¾“å…¥ Restore YAML。以下是 Restore 自定义资æºç¤ºä¾‹ï¼š + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + 如需获得é…ç½® Restore 的帮助，请å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 + +1. å•å‡»**创建**。 + +**结果**：已创建备份文件并更新到目标存储ä½ç½®ã€‚资æºè¿˜åŽŸé¡ºåºå¦‚下： + +1. 自定义资æºå®šä¹‰ï¼ˆCRD） +2. é›†ç¾¤èŒƒå›´èµ„æº +3. 命åç©ºé—´èµ„æº + +如需查看还原的处ç†æ–¹å¼ï¼Œè¯·æ£€æŸ¥ Operator 的日志。按照如下步骤获å–日志： + +```yaml +kubectl get pods -n cattle-resources-system +kubectl logs -n cattle-resources-system -f +``` + +### 回滚到上一个 Rancher 版本 + +ä½ å¯ä»¥ä½¿ç”¨ Helm CLI 回滚 Rancher。è¦å›žæ»šåˆ°ä¸Šä¸€ä¸ªç‰ˆæœ¬ï¼š + +```yaml +helm rollback rancher -n cattle-system +``` + +如果你ä¸æ˜¯æƒ³å›žæ»šåˆ°ä¸Šä¸€ä¸ªç‰ˆæœ¬ï¼Œä½ ä¹Ÿå¯ä»¥æŒ‡å®šå›žæ»šçš„版本。查看部署历å²è®°å½•ï¼š + +```yaml +helm history rancher -n cattle-system +``` + +确定目标版本åŽï¼Œæ‰§è¡Œå›žæ»šã€‚此示例回滚到版本 `3`： + +```yaml +helm rollback rancher 3 -n cattle-system +``` + +## 回滚到 Rancher 2.2-2.4 + +è¦å›žæ»šåˆ° 2.5 之å‰çš„ Rancher 版本，å‚考此处的步骤[æ¢å¤å¤‡ä»½ — Kubernetes 安装](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md)。如果æ¢å¤ Rancher Server 的集群的æŸä¸ªå¿«ç…§ï¼ŒRancher 的版本以åŠçŠ¶æ€å‡ä¼šæ¢å¤å›žåˆ°å¿«ç…§æ—¶çš„版本和状æ€ã€‚ + +有关回滚 Docker 安装的 Rancher，请å‚è§[本页](../other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md)。 + +:::note + +托管集群对其状æ€å…·æœ‰æƒå¨æ€§ã€‚因此，æ¢å¤ Rancher Server ä¸ä¼šæ¢å¤å¿«ç…§åŽå¯¹æ‰˜ç®¡é›†ç¾¤è¿›è¡Œçš„工作负载部署或更改。 + +::: + +## 回滚到 Rancher 2.0-2.1 + +我们ä¸å†æ”¯æŒå›žæ»šåˆ° Rancher 2.0-2.1。回滚到这些版本的说明ä¿ç•™åœ¨[此处](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup/roll-back-to-v2.0-v2.1.md)，仅用于无法å‡çº§åˆ° v2.2 的情况。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md new file mode 100644 index 00000000000..2912d831f55 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md @@ -0,0 +1,188 @@ +--- +title: Rancher Server Kubernetes 集群的问题排查 +--- + +本文介ç»å¦‚何对安装在 Kubernetes 集群上的 Rancher 进行故障排除。 + +### 相关命å空间 + +故障排除主è¦é’ˆå¯¹ä»¥ä¸‹ 3 个命å空间中的对象： + +- `cattle-system`：`rancher` deployment å’Œ Pod。 +- `ingress-nginx`：Ingress Controller Pod å’Œ services。 +- `cert-manager`：`cert-manager` Pod。 + +### "default backend - 404" + +很多æ“作都有å¯èƒ½å¯¼è‡´ Ingress Controller 无法将æµé‡è½¬å‘到你的 Rancher 实例。但是大多数情况下都是由错误的 SSL é…置导致的。 + +检查事项： + +- [Rancher 是å¦æ­£åœ¨è¿è¡Œ](#检查-rancher-是å¦æ­£åœ¨è¿è¡Œ) +- [è¯ä¹¦çš„ Common Name（CN）是 "Kubernetes Ingress Controller Fake Certificate"](#è¯ä¹¦çš„-cn-是-kubernetes-ingress-controller-fake-certificate) + +### 检查 Rancher 是å¦æ­£åœ¨è¿è¡Œ + +使用 `kubectl` 检查 `cattle-system` 系统命å空间，并查看 Rancher Pod 的状æ€æ˜¯å¦æ˜¯ **Running**： + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +如果状æ€ä¸æ˜¯ `Running`，在 Pod 上è¿è¡Œ `describe`，并检查 **Events**： + +``` +kubectl -n cattle-system describe pod + +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 11m default-scheduler Successfully assigned rancher-784d94f59b-vgqzh to localhost + Normal SuccessfulMountVolume 11m kubelet, localhost MountVolume.SetUp succeeded for volume "rancher-token-dj4mt" + Normal Pulling 11m kubelet, localhost pulling image "rancher/rancher:v2.0.4" + Normal Pulled 11m kubelet, localhost Successfully pulled image "rancher/rancher:v2.0.4" + Normal Created 11m kubelet, localhost Created container + Normal Started 11m kubelet, localhost Started container +``` + +### 检查 Rancher 日志 + +使用 `kubectl` 列出 Pod： + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +使用 `kubectl` å’Œ Pod å称列出该 Pod 的日志： + +``` +kubectl -n cattle-system logs -f rancher-784d94f59b-vgqzh +``` + +### è¯ä¹¦çš„ CN 是 "Kubernetes Ingress Controller Fake Certificate" + +使用æµè§ˆå™¨æ£€æŸ¥è¯ä¹¦çš„详细信æ¯ã€‚如果显示 CN 是 "Kubernetes Ingress Controller Fake Certificate"，则说明读å–或é¢å‘ SSL è¯ä¹¦æ—¶å‡ºçŽ°äº†é—®é¢˜ã€‚ + +:::note + +如果你使用的是 Let's Encrypt è¯ä¹¦ï¼Œè¯ä¹¦é¢å‘的过程å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿã€‚ + +::: + +### 排查 Cert-Manager é¢å‘çš„è¯ä¹¦ï¼ˆRancher 或 Let's Encrypt 生æˆçš„）问题 + +`cert-manager` 有 3 部分： + +- `cert-manager` 命å空间中的 `cert-manager` Pod。 +- `cattle-system` 命å空间中的 `Issuer` 对象。 +- `cattle-system` 命å空间中的 `Certificate` 对象。 + +å¾€åŽæ“作，对æ¯ä¸ªå¯¹è±¡è¿è¡Œ `kubectl describe` 并检查事件。这样，你å¯ä»¥è¿½è¸ªå¯èƒ½ä¸¢å¤±çš„内容。 + +以下是 Issuer 有问题的示例： + +``` +kubectl -n cattle-system describe certificate +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning IssuerNotReady 18s (x23 over 19m) cert-manager Issuer rancher not ready +``` + +``` +kubectl -n cattle-system describe issuer +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning ErrInitIssuer 19m (x12 over 19m) cert-manager Error initializing issuer: secret "tls-rancher" not found + Warning ErrGetKeyPair 9m (x16 over 19m) cert-manager Error getting keypair for CA issuer: secret "tls-rancher" not found +``` + +### 排查你自己æ供的 SSL è¯ä¹¦é—®é¢˜ + +ä½ çš„è¯ä¹¦ç›´æŽ¥åº”用于 `cattle-system` 命å空间中的 Ingress 对象。 + +检查 Ingress 对象的状æ€ï¼Œå¹¶æŸ¥çœ‹å®ƒæ˜¯å¦å‡†å¤‡å°±ç»ªï¼š + +``` +kubectl -n cattle-system describe ingress +``` + +如果 Ingress 对象已就绪，但是 SSL ä»ç„¶æ— æ³•æ­£å¸¸å·¥ä½œï¼Œä½ çš„è¯ä¹¦æˆ–密文的格å¼å¯èƒ½ä¸æ­£ç¡®ã€‚ + +è¿™ç§æƒ…况下，请检查 nginx-ingress-controller 的日志。nginx-ingress-controller çš„ Pod 中有多个容器，因此你需è¦æŒ‡å®šå®¹å™¨çš„å称： + +``` +kubectl -n ingress-nginx logs -f nginx-ingress-controller-rfjrq nginx-ingress-controller +... +W0705 23:04:58.240571 7 backend_ssl.go:49] error obtaining PEM from secret cattle-system/tls-rancher-ingress: error retrieving secret cattle-system/tls-rancher-ingress: secret cattle-system/tls-rancher-ingress was not found +``` + +### 没有匹é…çš„ "Issuer" + +你所选的 SSL é…ç½®è¦æ±‚在安装 Rancher 之å‰å…ˆå®‰è£… Cert-Manager，å¦åˆ™ä¼šå‡ºçŽ°ä»¥ä¸‹é”™è¯¯: + +``` +Error: validation failed: unable to recognize "": no matches for kind "Issuer" in version "certmanager.k8s.io/v1alpha1" +``` + +在这ç§æƒ…况下，先安装 Cert-Manager，然åŽå†é‡æ–°å®‰è£… Rancher。 + + +### Canal Pod 显示 READY 2/3 + +此问题的最常è§åŽŸå› æ˜¯ç«¯å£ 8472/UDP 在节点之间未打开。因此，你å¯ä»¥æ£€æŸ¥ä½ çš„本地防ç«å¢™ã€ç½‘络路由或安全组。 + +解决网络问题åŽï¼Œ`canal` Pod 会超时并é‡å¯ä»¥å»ºç«‹è¿žæŽ¥ã€‚ + +### nginx-ingress-controller Pod 显示 RESTARTS + +此问题的最常è§åŽŸå› æ˜¯ `canal` pod 未能建立覆盖网络。å‚è§ [canal Pod 显示 READY `2/3`](#canal-pod-显示-ready-23) 进行排查。 + + +### Failed to dial to /var/run/docker.sock: ssh: rejected: administratively prohibited (open failed) + +此错误的原因å¯èƒ½æ˜¯ï¼š + +* 指定连接的用户无æƒè®¿é—® Docker Socket。如果是这个原因，你通过登录主机并è¿è¡Œ `docker ps` 命令æ¥æ£€æŸ¥ï¼š + +``` +$ ssh user@server +user@server$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +``` + +如果需è¦äº†è§£å¦‚何进行正确设置，请å‚è§[ä»¥éž root ç”¨æˆ·èº«ä»½ç®¡ç† Docker](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)。 + +* 你使用的æ“作系统是 RedHat 或 CentOS：由于 [Bugzilla #1527565](https://bugzilla.redhat.com/show_bug.cgi?id=1527565)，你ä¸èƒ½ä½¿ç”¨ `root` 用户连接到节点。因此，你需è¦æ·»åŠ ä¸€ä¸ªå•ç‹¬çš„用户并é…置其访问 Docker Socket。如果需è¦äº†è§£å¦‚何进行正确设置，请å‚è§[ä»¥éž root ç”¨æˆ·èº«ä»½ç®¡ç† Docker](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)。 + +* SSH æœåŠ¡å™¨ç‰ˆæœ¬ä¸æ˜¯ 6.7 或更高版本：高版本是 Socket 转å‘所必须的，用于通过 SSH 连接到 Docker Socket。你å¯ä»¥åœ¨ä½ è¦è¿žæŽ¥çš„主机上使用 `sshd -V` 或使用 netcat 进行检查： +``` +$ nc xxx.xxx.xxx.xxx 22 +SSH-2.0-OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.10 +``` + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: no key found + +`ssh_key_path` 密钥文件无法访问：请确ä¿ä½ å·²ç»æŒ‡å®šäº†ç§é’¥æ–‡ä»¶ï¼ˆä¸æ˜¯å…¬é’¥ `.pub`），而且è¿è¡Œ `rke` 命令的用户å¯ä»¥è®¿é—®è¯¥ç§é’¥æ–‡ä»¶ã€‚ + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain + +`ssh_key_path` 密钥文件ä¸æ˜¯è®¿é—®èŠ‚点的正确文件：请仔细检查，确ä¿ä½ å·²ä¸ºèŠ‚点指定了正确的 `ssh_key_path` 和连接用户。 + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: cannot decode encrypted private keys + +如需使用加密的ç§é’¥ï¼Œè¯·ä½¿ç”¨ `ssh-agent` æ¥ä½¿ç”¨å¯†ç æ¥åŠ è½½å¯†é’¥ã€‚如果在è¿è¡Œ `rke` 命令的环境中找到 `SSH_AUTH_SOCK` 环境å˜é‡ï¼Œå®ƒå°†è‡ªåŠ¨ç”¨äºŽè¿žæŽ¥åˆ°èŠ‚点。 + +### Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? + +节点无法通过é…置的 `address` å’Œ `port` 访问。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md new file mode 100644 index 00000000000..f55a95063b5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md @@ -0,0 +1,68 @@ +--- +title: 将加固的自定义/导入集群å‡çº§åˆ° Kubernetes v1.25 +--- + +Kubernetes v1.25 改å˜äº†é›†ç¾¤æ述和执行安全策略的方å¼ã€‚从这个版本开始，[Pod 安全策略 (PSP)](https://kubernetes.io/docs/concepts/security/pod-security-policy/)ä¸å†å¯ç”¨ã€‚Kubernetes v1.25 将它们替æ¢ä¸ºæ–°çš„安全对象：[Pod 安全标准 (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/) å’Œ [Pod 安全准入 (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/)。 + +如果你具有自定义或导入的加固集群，你需è¦åšå¥½å‡†å¤‡ï¼Œç¡®ä¿å°†æ—§ç‰ˆæœ¬çš„ Kubernetes 顺利å‡çº§åˆ° v1.25 或更高版本。 + +:::note + +å‡çº§åˆ° v1.25 åŽï¼Œæ·»åŠ å¿…è¦çš„ Rancher 命å空间è±å…。有关详细信æ¯ï¼Œè¯·å‚阅 [Pod 安全准入 (PSA) é…置模æ¿](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md#è±å…必须的-rancher-命å空间)。 + +::: + +## 将导入的加固集群å‡çº§åˆ° Kubernetes v1.25 或更高版本 + + + + +在集群中的æ¯ä¸ªèŠ‚点上执行以下æ“作： +1. å°† [`rancher-psact.yaml`](./rancher-psact.yaml) ä¿å­˜åˆ° `/etc/rancher/rke2` 中。 +1. 编辑 RKE2 é…置文件： + 1. å°† `profile` 字段更新为 `cis-1.23`。 + 1. 指定刚æ‰æ·»åŠ çš„é…置文件的路径：`pod-security-admission-config-file: /etc/rancher/rke2/rancher-psact.yaml`。 + + + + +在集群中的æ¯ä¸ªèŠ‚点上执行以下æ“作： + +éµå¾ª K3s [将加固集群从 v1.24.x å‡çº§åˆ° v1.25.x](https://docs.k3s.io/known-issues#hardened-125)的官方说明，但使用[自定义](./rancher-psact.yaml)Rancher PSA é…置模æ¿ï¼Œè€Œä¸æ˜¯ K3s 官方网站上æ供的é…置。 + + + +执行这些步骤åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI å‡çº§é›†ç¾¤çš„ Kubernetes 版本： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在 **Kubernetes 版本**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦ä½¿ç”¨çš„版本。 +1. å•å‡»**ä¿å­˜**。 + +## 将自定义加固集群å‡çº§åˆ° Kubernetes v1.25 或更高版本 + + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在**åŸºæœ¬ä¿¡æ¯ > 安全**下的 **CIS é…置文件**下拉èœå•ä¸­ï¼Œé€‰æ‹© `cis-1.23`。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹© `rancher-restricted`。 +1. 在 **Kubernetes 版本**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦ä½¿ç”¨çš„版本。 +1. å•å‡»**ä¿å­˜**。 + + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑 YAML**。 +1. 从 `kube-apiserver-arg.enable-admission-plugins` 中删除 `PodSecurityPolicy`。 +1. 在 `spec` 字段中，添加一行：`defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted` +1. å°† `kubernetesVersion` 更新为你选择的版本（v1.25 或更高版本）。 +1. å•å‡»**ä¿å­˜**。 + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md new file mode 100644 index 00000000000..9e5367abb2d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md @@ -0,0 +1,207 @@ +--- +title: å‡çº§ +--- + +本文介ç»å¦‚何å‡çº§ä½¿ç”¨ Helm 安装在 Kubernetes 集群上的 Rancher Server。这些步骤也适用于使用 Helm 进行的离线安装。 + +有关使用 Docker 安装的 Rancher çš„å‡çº§è¯´æ˜Žï¼Œè¯·å‚è§[本页。](../other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md) + +如需å‡çº§ Kubernetes 集群中的组件，或 [Kubernetes services](https://rancher.com/docs/rke/latest/en/config-options/services/) 或 [附加组件（add-on）](https://rancher.com/docs/rke/latest/en/config-options/add-ons/)的定义，请å‚è§ [RKE å‡çº§æ–‡æ¡£](https://rancher.com/docs/rke/latest/en/upgrades/)çš„ Rancher Kubernetes 引擎。 + + +## 先决æ¡ä»¶ + +### 访问 kubeconfig + +Helm çš„è¿è¡Œä½ç½®ï¼Œåº”该与你的 kubeconfig 文件，或你è¿è¡Œ kubectl 命令的ä½ç½®ç›¸åŒã€‚ + +如果你在安装 Kubernetes 时使用了 RKE，那么 config 将会在你è¿è¡Œ `rke up` 的目录下创建。 + +kubeconfig 也å¯ä»¥é€šè¿‡ `--kubeconfig` 标签（详情请å‚è§ https://helm.sh/docs/helm/helm/ ）æ¥æ‰‹åŠ¨æŒ‡å®šæ‰€éœ€çš„集群。 + +### 查看已知问题 + +如需查看æ¯ä¸ª Rancher 版本的已知问题，请å‚è§ [GitHub](https://github.com/rancher/rancher/releases) 中的å‘行说明，或查看 [Rancher 论å›](https://forums.rancher.com/c/announcements/12)。 + +ä¸æ”¯æŒ _å‡çº§_ 或 _å‡çº§åˆ°_ [rancher-alpha 仓库](../resources/choose-a-rancher-version.md#helm-chart-仓库)中的任何 Chart。 +### Helm 版本 + +本安装指å—å‡å®šä½ ä½¿ç”¨çš„是 Helm 3。 + +如果你使用 Helm 2，请å‚è§ [Helm 2 è¿ç§»åˆ° Helm 3 文档](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/)。如果你ä¸èƒ½å‡çº§åˆ° Helm 3，[Helm 2 å‡çº§é¡µé¢](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/helm2.md)æ供了使用 Helm 2 å‡çº§çš„æ—§å‡çº§æŒ‡å—。 + +### 离线安装：推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +[仅适用于离线安装](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)：为新的 Rancher Server 版本收集和推é€é•œåƒã€‚使用你需è¦é’ˆå¯¹ Rancher 版本å‡çº§çš„é•œåƒï¼ŒæŒ‰ç…§æ­¥éª¤[推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md)。 + +### 使用 cert-manager 0.8.0 之å‰çš„版本å‡çº§ + +[从 2019 å¹´ 11 月 1 日开始，Let's Encrypt å·²å±è”½æ—©äºŽ 0.8.0 çš„ cert-manager 实例](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753)。因此，请å‚è§[说明](../resources/upgrade-cert-manager.md)把 cert-manager å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ã€‚ + +## å‡çº§æ¦‚è¦ + +按照以下步骤å‡çº§ Rancher Server： + + +### 1. 备份è¿è¡Œ Rancher Server çš„ Kubernetes 集群 + +使用[备份应用](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)æ¥å¤‡ä»½ Rancher。 + +如果å‡çº§è¿‡ç¨‹ä¸­å‡ºçŽ°é—®é¢˜ï¼Œä½ å°†ä½¿ç”¨å¤‡ä»½ä½œä¸ºè¿˜åŽŸç‚¹ã€‚ + +### 2. æ›´æ–° Helm Chart 仓库 + +1. 更新本地 Helm 仓库缓存。 + + ``` + helm repo update + ``` + +1. 获å–你用æ¥å®‰è£… Rancher 的仓库å称。 + + 关于仓库åŠå…¶åŒºåˆ«ï¼Œè¯·å‚è§ [Helm Chart Repositories](../resources/choose-a-rancher-version.md#helm-chart-仓库)。 + + - Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + + ``` + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + + :::note + + 如果你想切æ¢åˆ°ä¸åŒçš„ Helm Chart 仓库，请按照[切æ¢ä»“库步骤](../resources/choose-a-rancher-version.md#切æ¢åˆ°ä¸åŒ-helm-chart-仓库)进行æ“作。如果你è¦åˆ‡æ¢ä»“库，请先å†æ¬¡åˆ—出仓库，å†ç»§ç»­æ‰§è¡Œæ­¥éª¤ 3，以确ä¿æ·»åŠ äº†æ­£ç¡®çš„仓库。 + + ::: + +1. 从 Helm Chart 仓库获å–最新的 Chart æ¥å®‰è£… Rancher。 + + 该命令将æå–最新的 Chart，并将其作为 `.tgz`文件ä¿å­˜åœ¨å½“å‰ç›®å½•ä¸­ã€‚ + + ```plain + helm fetch rancher-/rancher + ``` + ä½ å¯ä»¥é€šè¿‡ `--version=` 标记，æ¥æŒ‡å®šè¦å‡çº§çš„目标 Chart 版本。例如： + + ```plain + helm fetch rancher-/rancher --version=2.6.8 + ``` + +### 3. å‡çº§ Rancher + +本节介ç»äº†å¦‚何使用 Helm å‡çº§ Rancher 的一般（互è”网连接）或离线安装。 + +:::note 离线说明： + +如果你在离线环境中安装 Rancher，请跳过本页的其余部分，按照[本页](air-gapped-upgrades.md)上的说明渲染 Helm 模æ¿ã€‚ + +::: + + +从当å‰å®‰è£…çš„ Rancher Helm Chart 中获å–用 `--set`传递的值。 + +``` +helm get values rancher -n cattle-system + +hostname: rancher.my.org +``` + +:::note + +这个命令会列出更多的值。此处展示的åªæ˜¯å…¶ä¸­ä¸€ä¸ªå€¼çš„例å­ã€‚ + +::: + +:::tip + +Deployment çš„å称å¯èƒ½ä¼šæœ‰æ‰€ä¸åŒã€‚例如，如果你通过 AWS Marketplace 部署 Rancher，则 Deployment çš„å称为“rancher-stableâ€ã€‚ +因此： +``` +helm get values rancher-stable -n cattle-system + +hostname: rancher.my.org +``` + +::: + +如果è¦å°† cert-manager 从 v1.5 或更早的版本å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ï¼Œè¯·å‚阅 [cert-manager upgrade docs](../resources/upgrade-cert-manager.md#选项-cå‡çº§-15-åŠä»¥ä¸‹ç‰ˆæœ¬çš„-cert-manager) 了解如何在ä¸å¸è½½æˆ–é‡æ–°å®‰è£… Rancher 的情况下å‡çº§ cert-manager。å¦åˆ™ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹[ Rancher å‡çº§æ­¥éª¤](#rancher-å‡çº§æ­¥éª¤)进行æ“作。 + +#### Rancher å‡çº§æ­¥éª¤ + +ä¿ç•™ä½ çš„所有设置把 Rancher å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ã€‚ + +将上一步中的所有值用 `--set key=value` 追加到命令中。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` + +:::note + +以上是一个例å­ï¼Œå¯èƒ½æœ‰æ›´å¤šä¸Šä¸€æ­¥çš„值需è¦è¿½åŠ ã€‚ + +::: + +:::tip + +如果你通过 AWS Marketplace 部署 Rancher，则 Deployment çš„å称为“rancher-stableâ€ã€‚ +因此： +``` +helm upgrade rancher-stable rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` + +::: + +å¦å¤–，你也å¯ä»¥å°†å½“å‰çš„值导出到一个文件中，并在å‡çº§æ—¶å¼•ç”¨è¯¥æ–‡ä»¶ã€‚例如，如果你åªéœ€è¦æ”¹å˜ Rancher 的版本： + +1. 将当å‰å€¼å¯¼å‡ºåˆ°æ–‡ä»¶ï¼š + ``` + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` +1. åªæ›´æ–° Rancher 版本： + + 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + + ``` + helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version=2.6.8 + ``` + +### 4. 验è¯å‡çº§ + +登录 Rancher 以确认å‡çº§æˆåŠŸã€‚ + +:::tip + +å‡çº§åŽå‡ºçŽ°ç½‘络问题？ + +请å‚è§[æ¢å¤é›†ç¾¤ç½‘络](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md)。 + +::: + +## 已知å‡çº§é—®é¢˜ + +ä½ å¯ä»¥åœ¨ [GitHub](https://github.com/rancher/rancher/releases) å‘å¸ƒè¯´æ˜Žä»¥åŠ [Rancher 论å›](https://forums.rancher.com/c/announcements/12)中找到æ¯ä¸ª Rancher 版本的已知问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/feature-flags.md new file mode 100644 index 00000000000..873a0689b14 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -0,0 +1,39 @@ +--- +title: 功能开关 +--- + +使用功能开关（Feature Flag），你å¯ä»¥è¯•ç”¨å¯é€‰æˆ–实验性的功能并å¯ç”¨æ­£åœ¨é€æ­¥æ·˜æ±°çš„旧版功能。 + +è¦äº†è§£åŠŸèƒ½çš„值以åŠå¦‚何å¯ç”¨å®ƒä»¬ï¼Œè¯·å‚阅[å¯ç”¨å®žéªŒæ€§åŠŸèƒ½](../../../pages-for-subheaders/enable-experimental-features.md)。 + +:::note + +æŸäº›åŠŸèƒ½è¦æ±‚é‡æ–°å¯åŠ¨ Rancher 容器。Rancher UI 中标记了è¦æ±‚é‡å¯çš„功能。 + +::: + +以下是 Rancher 中å¯ç”¨çš„功能开关列表。如果你是从旧 Rancher 版本å‡çº§çš„，你å¯èƒ½ä¼šåœ¨ Rancher UI 中看到其他功能，例如 `proxy` 或 `dashboard`（å‡[已中断](/versioned_docs/version-2.5/reference-guides/installation-references/feature-flags.md)）： + +- `continuous-delivery`：å…许从 Fleet 中å•ç‹¬ç¦ç”¨ Fleet GitOps。有关详细信æ¯ï¼Œè¯·å‚阅[æŒç»­äº¤ä»˜](../../../how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md)。 +- `fleet`：v2.6 åŠæ›´é«˜ç‰ˆæœ¬çš„ Rancher é…ç½®æ¡†æž¶éœ€è¦ Fleet。å³ä½¿ä½ åœ¨æ—§ Rancher 版本中ç¦ç”¨äº†è¯¥æ ‡å¿—，该标志也将在å‡çº§æ—¶è‡ªåŠ¨å¯ç”¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [Fleet - GitOps at Scale](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md)。 +- `harvester`ï¼šç®¡ç† Virtualization Management 页é¢çš„访问。用户å¯ä»¥åœ¨è¯¥é¡µé¢ç›´æŽ¥å¯¼èˆªåˆ° Harvester 集群并访问 Harvester UI。有关详细信æ¯ï¼Œè¯·å‚阅 [Harvester 集æˆ](../../../integrations-in-rancher/harvester.md)。 +- `istio-virtual-service-ui`：å¯ç”¨[å¯è§†ç•Œé¢](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md)æ¥åˆ›å»ºã€è¯»å–ã€æ›´æ–°å’Œåˆ é™¤ Istio 虚拟æœåŠ¡å’Œç›®æ ‡è§„则，这些都是 Istio æµé‡ç®¡ç†åŠŸèƒ½ã€‚ +- `legacy`：å¯ç”¨ 2.5.x åŠæ›´æ—©ç‰ˆæœ¬çš„一组功能，这些功能正é€æ¸è¢«æ–°çš„实现淘汰。它们是已弃用以åŠåŽç»­å¯ç”¨äºŽæ–°ç‰ˆæœ¬çš„功能组åˆã€‚æ–°çš„ Rancher 安装会默认ç¦ç”¨æ­¤æ ‡å¿—。如果你从以å‰ç‰ˆæœ¬çš„ Rancher å‡çº§ï¼Œæ­¤æ ‡å¿—会å¯ç”¨ã€‚ +- `multi-cluster-management`：å…许é…置和管ç†å¤šä¸ª Kubernetes 集群。此标志åªèƒ½åœ¨å®‰è£…时设置。åŽç»­æ— æ³•å¯ç”¨æˆ–ç¦ç”¨å®ƒã€‚ +- `rke1-custom-node-cleanup`：清除已删除的 RKE1 自定义节点。建议你å¯ç”¨æ­¤æ ‡å¿—，以防止已删除的节点å°è¯•é‡æ–°åŠ å…¥é›†ç¾¤ã€‚ +- `rke2`：å¯ç”¨é…ç½® RKE2 集群。此标志默认å¯ç”¨ã€‚ +- `token-hashing`：å¯ç”¨ä»¤ç‰Œå“ˆå¸Œã€‚å¯ç”¨åŽï¼Œä¼šä½¿ç”¨ SHA256 算法对现有 Token 和所有新 Token 进行哈希处ç†ã€‚一旦对 Token 进行哈希处ç†ï¼Œå°±æ— æ³•æ’¤æ¶ˆæ“作。此标志在å¯ç”¨åŽæ— æ³•ç¦ç”¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [API 令牌](../../../reference-guides/about-the-api/api-tokens.md#令牌哈希)。 +- `unsupported-storage-drivers`：å…许å¯ç”¨éžé»˜è®¤å¯ç”¨çš„存储æ供程åºå’Œå·æ’件。有关详细信æ¯ï¼Œè¯·å‚阅[å…许使用ä¸å—支æŒçš„存储驱动程åº](../../../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md)。 + +下表介ç»äº† Rancher 中功能开关的å¯ç”¨æ€§å’Œé»˜è®¤å€¼ã€‚标记为“GAâ€çš„功能已普éå¯ç”¨ï¼š + +| 功能开关å称 | 默认值 | çŠ¶æ€ | å¯ç”¨äºŽ | +| ----------------------------- | ------------- | ------------ | --------------- | +| `continuous-delivery` | `true` | GA | v2.6.0 | +| `fleet` | `true` | ä¸èƒ½ç¦ç”¨ | v2.6.0 | +| `fleet` | `true` | GA | v2.5.0 | +| `harvester` | `true` | 实验功能 | v2.6.1 | +| `legacy` | 新安装：`false`ï¼›å‡çº§ï¼š`true` | GA | v2.6.0 | +| `rke1-custom-node-cleanup` | `true` | GA | v2.6.0 | +| `rke2` | `true` | 实验功能 | v2.6.0 | +| `token-hashing` | 新安装：`false`ï¼›å‡çº§ï¼š`true` | GA | v2.6.0 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md new file mode 100644 index 00000000000..bd69a89cfee --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md @@ -0,0 +1,310 @@ +--- +title: Rancher Helm Chart 选项 +keywords: [rancher helm chart, rancher helm 选项, rancher helm chart 选项, helm chart rancher, helm 选项 rancher, helm chart 选项 rancher] +--- + +本文æ供了 Rancher Helm Chart çš„é…ç½®å‚考。 + +如需选择 Helm Chart 版本，请å‚è§[本页](../../../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +了解开å¯å®žéªŒæ€§åŠŸèƒ½çš„详情，请å‚è§[本页](../../../pages-for-subheaders/enable-experimental-features.md)。 + +## 常用选项 + +| 选项 | 默认值 | æè¿° | +| ------------------------- | ------------- | ---------------------------------------------------------------------------------- | +| `bootstrapPassword` | " " | `string` - 为第一个管ç†å‘˜ç”¨æˆ·è®¾ç½®[引导密ç ](#引导密ç )。登录åŽï¼Œç®¡ç†å‘˜éœ€è¦é‡ç½®å¯†ç ã€‚如ä¸è®¾ç½®ï¼Œä¼šä½¿ç”¨éšæœºç”Ÿæˆçš„引导密ç ã€‚ | +| `hostname` | " " | `string` - ä½ çš„ Rancher Server 的完全é™å®šçš„域å（FQDN) | +| `ingress.tls.source` | "rancher" | `string` - ä»Žå“ªé‡ŒèŽ·å– Ingress çš„è¯ä¹¦- "rancher, letsEncrypt, secret" | +| `letsEncrypt.email` | " " | `string` - ä½ çš„é‚®ç®±åœ°å€ | +| `letsEncrypt.environment` | "production" | `string` - å¯é€‰é¡¹ï¼š"staging, production" | +| `privateCA` | false | `bool` - 如果你的è¯ä¹¦æ˜¯ç”±ç§æœ‰ CA ç­¾å‘的，把这个值设置为 true | + +
+ +## 高级选项 + +| 选项 | 默认值 | æè¿° | +| ------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `additionalTrustedCAs` | false | `bool` - 请å‚è§[é¢å¤–的授信 CA](#é¢å¤–的授信-ca) | +| `addLocal` | "true" | `string` - 让 Rancher 检测并导入 “local†Rancher Server 集群。_注æ„：此选项在 2.5.0 中已ä¸å¯ç”¨ã€‚ä½ å¯è€ƒè™‘使用 `restrictedAdmin` 选项，æ¥é¿å…用户修改本地集群。_ | +| `antiAffinity` | "preferred" | `string` - Rancher Pod çš„å亲和性规则 - "preferred, required" | +| `auditLog.destination` | "sidecar" | `string` - å‘é€å®¡è®¡æ—¥å¿—到 Sidecar 容器的控制å°æˆ– hostPath å· - "sidecar, hostPath" | +| `auditLog.hostPath` | "/var/log/rancher/audit" | `string` - 主机上的日志文件目标地å€ï¼ˆä»…当`auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.level` | 0 | `int` - 设置 [API 审计日志](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md)等级。0 代表关闭。[0-3] | +| `auditLog.maxAge` | 1 | `int` - 旧审计日志文件最多å¯ä¿ç•™çš„天数（仅当`auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.maxBackup` | 1 | `int` - 审计文件最大å¯ä¿ç•™çš„个数（仅当 `auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.maxSize` | 100 | `int` - 在审计日志被轮æ¢å‰çš„最大容é‡ï¼Œå•ä½æ˜¯ MB（仅当 `auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.image.repository` | "registry.suse.com/bci/bci-micro" | `string` - 用于收集审计日志的镜åƒçš„ä½ç½®ã€‚ | +| `auditLog.image.tag` | "15.4.14.3" | `string` - 用于收集审计日志的镜åƒçš„标签。 | +| `auditLog.image.pullPolicy` | "IfNotPresent" | `string` - 覆盖 auditLog é•œåƒçš„ imagePullPolicy - “Alwaysâ€ã€â€œNeverâ€ã€â€œIfNotPresentâ€ã€‚ | +| `busyboxImage` | "" | `string` - 用于收集审计日志的 busybox é•œåƒä½ç½®ã€‚_注æ„：此选项已弃用，请使用 `auditLog.image.repository` æ¥æŽ§åˆ¶å®¡è®¡ sidecar é•œåƒ_。 | +| `certmanager.version` | "" | `string` - 设置 cert-manager compatibility | +| `debug` | false | `bool` - 在 Rancher Server 设置 debug å‚æ•° | +| `extraEnv` | [] | `list` - 为 Rancher é¢å¤–设置环境å˜é‡ | +| `imagePullSecrets` | [] | `list` - ç§æœ‰é•œåƒä»“库凭è¯çš„密文å称列表 | +| `ingress.configurationSnippet` | "" | `string` - 添加é¢å¤–çš„ Nginx é…置。å¯ç”¨äºŽä»£ç†é…置。 | +| `ingress.extraAnnotations` | {} | `map` - 用于自定义 Ingress çš„é¢å¤–注释 | +| `ingress.enabled` | true | 如果值为 false，Helm ä¸ä¼šå®‰è£… Rancher Ingress。你å¯æŠŠå€¼è®¾ä¸º false 以部署你自己的 Ingress。 | +| `letsEncrypt.ingress.class` | "" | `string` - cert-manager acmesolver ingress çš„å¯é€‰ ingress 类，用于å“应 Let's Encrypt ACME 质询。选项：traefik，nginx。 | | +| `noProxy` | "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local,cattle-system.svc" | `string` - ä¸ä½¿ç”¨ä»£ç†çš„主机å或 IP 地å€çš„逗å·åˆ†éš”列表 | | +| `proxy` | "" | `string` - ç»™ Rancher é…置的 HTTP[S] ä»£ç† | +| `rancherImage` | "rancher/rancher" | `string` - Rancher é•œåƒæº | +| `rancherImagePullPolicy` | "IfNotPresent" | `string` - 覆盖 Rancher Server é•œåƒçš„ imagePullPolicy - "Always", "Never", "IfNotPresent" | +| `rancherImageTag` | å’Œ Chart 版本一致 | `string` - rancher/rancher é•œåƒæ ‡ç­¾ | +| `replicas` | 3 | `int` - Rancher Server 副本数。如果设为 -1，会根æ®é›†ç¾¤ä¸­çš„å¯ç”¨èŠ‚点数自动选择 1，2或3。 | +| `resources` | {} | `map` - Rancher Pod 资æºè¯·æ±‚å’Œé™åˆ¶ | +| `restrictedAdmin` | `false` | `bool` - 如果值为 true，åˆå§‹çš„ Rancher 用户访问本地 Kubernetes 集群会å—到é™åˆ¶ï¼Œä»¥é¿å…æƒé™å‡çº§ã€‚详情请å‚è§ [restricted-admin 角色](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#å—é™ç®¡ç†å‘˜)。 | +| `systemDefaultRegistry` | "" | `string` - 用于所有系统容器镜åƒçš„ç§æœ‰ä»“库，例如 http://registry.example.com/ | +| `tls` | "ingress" | `string` - 详情请å‚è§[外部 TLS 终止](#外部-tls-终止)。- "ingress, external" | +| `useBundledSystemChart` | `false` | `bool` - 选择 Rancher Server 打包的 system-charts。此å‚数用于离线环境安装。 | +| `global.cattle.psp.enabled` | `true` | `bool` - 使用 Rancher v2.7.2-v2.7.4 时，选择 `false` 以ç¦ç”¨ Kubernetes v1.25 åŠæ›´é«˜ç‰ˆæœ¬çš„ PSP。使用 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ—¶ï¼ŒRancher 会å°è¯•æ£€æµ‹é›†ç¾¤æ˜¯å¦è¿è¡Œä¸æ”¯æŒ PSP çš„ Kubernetes 版本，如果确定集群ä¸æ”¯æŒ PSP，则将默认 PSP 的使用设置为 false。你ä»ç„¶å¯ä»¥é€šè¿‡æ˜¾å¼æ供此值的 `true` 或 `false` æ¥æ‰‹åŠ¨è¦†ç›–æ­¤å€¼ã€‚åœ¨æ”¯æŒ PSP 的集群中（例如使用 Kubernetes v1.24 或更低版本的集群），Rancher ä»å°†é»˜è®¤ä½¿ç”¨ PSP。 | + + +### å¼•å¯¼å¯†ç  + +Rancher 首次å¯åŠ¨æ—¶ï¼Œä¼šä¸ºç¬¬ä¸€ä¸ªç®¡ç†å‘˜ç”¨æˆ·éšæœºç”Ÿæˆä¸€ä¸ªå¯†ç ã€‚当管ç†å‘˜é¦–次登录 Rancher 时，用于获å–引导密ç ï¼ˆBootstrap）的命令会在 UI 上显示。管ç†å‘˜éœ€è¦è¿è¡Œå‘½ä»¤å¹¶ä½¿ç”¨å¼•å¯¼å¯†ç ç™»å½•ã€‚ç„¶åŽ Rancher 会让管ç†å‘˜é‡ç½®å¯†ç ã€‚ + +如果你想指定引导密ç è€Œä¸ä½¿ç”¨éšæœºç”Ÿæˆçš„密ç ï¼Œè¯·å‚考以下命令设置密ç ã€‚ + +```plain +--set bootstrapPassword="rancher" +``` + +无论你是使用æ供的密ç è¿˜æ˜¯ç”Ÿæˆçš„密ç ï¼Œå¯†ç å‡å­˜å‚¨åœ¨ Kubernetes 密文中。安装 Rancher åŽï¼Œå¦‚何使用 kubectl 获å–密ç çš„说明将会在 UI 中显示： + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### API 审计日志 + +å¯ç”¨ [API 审计日志](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md)。 + +ä½ å¯ä»¥åƒæ”¶é›†å…¶ä»–容器日志一样收集此日志。在 Rancher Server 集群上为 `System` 项目å¯ç”¨ [Logging](../../../pages-for-subheaders/logging.md)。 + +```plain +--set auditLog.level=1 +``` + +默认情况下，å¯ç”¨å®¡è®¡æ—¥å¿—会在 Rancher pod 中创建一个 Sidecar 容器。这个容器（`rancher-audit-log`）会把日志æµä¼ è¾“到 `stdout`。你å¯ä»¥åƒæ”¶é›†å…¶ä»–容器日志一样收集此日志。如果你使用 Sidecar 作为审计日志的目标时， `hostPath`，`maxAge`，`maxBackups` å’Œ `maxSize` 选项ä¸ä¼šç”Ÿæ•ˆã€‚建议使用你的æ“作系统或 Docker Daemon 的日志轮æ¢åŠŸèƒ½æ¥æŽ§åˆ¶ç£ç›˜ç©ºé—´çš„使用。请为 Rancher Server 集群或 System 项目å¯ç”¨ [Logging](../../../pages-for-subheaders/logging.md)。 + +å°† `auditLog.destination` 的值设为 `hostPath`，å¯ä»¥å°†æ—¥å¿—转å‘到与主机系统共享的å·ï¼Œè€Œä¸æ˜¯ä¼ è¾“到 Sidecar 容器。如果目标设置为 `hostPath`，你å¯èƒ½éœ€è¦è°ƒæ•´å…¶ä»– auditLog å‚数以进行日志轮æ¢ã€‚ + +### é¢å¤–设置环境å˜é‡ + +ä½ å¯ä»¥ä½¿ç”¨ `extraEnv` 为 Rancher Server é¢å¤–设置环境å˜é‡ã€‚该列表以 YAML æ ¼å¼ä¼ é€’ç»™ Rancher 部署，它嵌入在 Rancher 容器的 `env` 下。你å¯ä»¥å‚考 Kubernetes 文档设置容器环境å˜é‡ã€‚`extraEnv` å¯ä»¥ä½¿ç”¨ [Define Environment Variables for a Container](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container) 中引用的任何键。 + +使用 `name` å’Œ `value` 键的示例： + +```plain +--set 'extraEnv[0].name=CATTLE_TLS_MIN_VERSION' +--set 'extraEnv[0].value=1.0' +``` + +如果将æ•æ„Ÿæ•°æ®ï¼ˆä¾‹å¦‚代ç†è®¤è¯å‡­è¯ï¼‰ä½œä¸ºçŽ¯å¢ƒå˜é‡çš„值传递，则强烈建议使用 Secret 引用。这将防止æ•æ„Ÿæ•°æ®åœ¨ Helm 或 Rancher 部署中暴露。 + +ä½ å¯ä»¥å‚考使用 `name`ã€`valueFrom.secretKeyRef.name` å’Œ `valueFrom.secretKeyRef.key` é”®çš„ç¤ºä¾‹ã€‚è¯¦è§ [HTTP 代ç†](#http-代ç†)中的示例。 + +### TLS 设置 + +当你在 Kubernetes 集群内安装 Rancher 时，TLS 会在集群的 Ingress Controller 上å¸è½½ã€‚支æŒçš„ TLS 设置å–决于使用的 Ingress Controller。 + +å‚è§ [TLS 设置](tls-settings.md)了解更多信æ¯å’Œé€‰é¡¹ã€‚ + +### 导入 `local` 集群 + +默认情况下，Rancher Server 会检测并导入其所在的 `local` 集群。有æƒè®¿é—® `local` 集群的用户对 Rancher Server 管ç†çš„所有集群具有“rootâ€è®¿é—®æƒé™ã€‚ + +:::caution + +如果你关闭 addLocal，大多数 Rancher 2.5 功能都ä¸èƒ½ä½¿ç”¨ï¼ŒåŒ…括 EKS Provisioner。 + +::: + +如果这在你的环境中是一个问题，你å¯ä»¥åœ¨åˆå§‹å®‰è£…时将此选项设置为“falseâ€ã€‚ + +此选项仅在首次安装 Rancher 时有效。详情请å‚è§ [Issue 16522](https://github.com/rancher/rancher/issues/16522)。 + +```plain +--set addLocal="false" +``` + +### 自定义 Ingress + +è¦è‡ªå®šä¹‰æˆ–使用 Rancher Server 的其他 Ingress，你å¯ä»¥è®¾ç½®è‡ªå·±çš„ Ingress 注释。 + +设置自定义è¯ä¹¦é¢å‘者的示例： + +```plain +--set ingress.extraAnnotations.'cert-manager\.io/cluster-issuer'=issuer-name +``` + +以下是使用 `ingress.configurationSnippet`设置é™æ€ä»£ç†æ ‡å¤´çš„示例。该值åƒæ¨¡æ¿ä¸€æ ·è¿›è¡Œè§£æžï¼Œå› æ­¤å¯ä»¥ä½¿ç”¨å˜é‡ã€‚ + +```plain +--set ingress.configurationSnippet='more_set_input_headers X-Forwarded-Host {{ .Values.hostname }};' +``` + +### HTTP ä»£ç† + +Rancher 的一些功能（Helm Chart）需è¦ä½¿ç”¨äº’è”网æ‰èƒ½ä½¿ç”¨ã€‚ä½ å¯ä»¥ä½¿ç”¨ `proxy` 设置代ç†æœåŠ¡å™¨ï¼Œæˆ–使用 `extraEnv` 设置 `HTTPS_PROXY` 环境å˜é‡æ¥æŒ‡å‘代ç†æœåŠ¡å™¨ã€‚ + +å°†è¦æŽ’除的 IP 使用逗å·åˆ†éš”列表添加到 `noProxy` Chart value 中。确ä¿æ·»åŠ äº†ä»¥ä¸‹å€¼ï¼š +- Pod 集群 IP 范围（默认值：`10.42.0.0/16`）。 +- Service Cluster IP 范围（默认值：`10.43.0.0/16`）。 +- 内部集群域（默认值：`.svc,.cluster.local`）。 +- 任何 Worker 集群 `controlplane` 节点。 + Rancher 支æŒåœ¨æ­¤åˆ—表中使用 CIDR 表示法æ¥è¡¨ç¤ºèŒƒå›´ã€‚ + +ä¸åŒ…括æ•æ„Ÿæ•°æ®æ—¶ï¼Œå¯ä»¥ä½¿ç”¨ `proxy` 或 `extraEnv` Chart 选项。使用 `extraEnv` 时将忽略 `noProxy` Helm 选项。因此，`NO_PROXY` 环境å˜é‡ä¹Ÿå¿…须设置为 `extraEnv`。 + +以下是使用 `extraEnv` Chart 选项设置代ç†çš„示例： + +```plain +--set proxy="http:///" +``` + +使用 `extraEnv` Chart 选项设置代ç†çš„示例： +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].value="http://:/" +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +包å«æ•æ„Ÿæ•°æ®ï¼ˆä¾‹å¦‚代ç†è®¤è¯å‡­è¯ï¼‰æ—¶ï¼Œè¯·ä½¿ç”¨ `extraEnv` 选项和 `valueFrom.secretRef` æ¥é˜²æ­¢æ•æ„Ÿæ•°æ®åœ¨ Helm 或 Rancher 部署中暴露。 + +下é¢æ˜¯ä½¿ç”¨ `extraEnv` é…置代ç†çš„示例。此示例 Secret 在 Secret çš„ `"https-proxy-url"` é”®ä¸­åŒ…å« `"http://:@:/"` 值： +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].valueFrom.secretKeyRef.name=secret-name +--set extraEnv[1].valueFrom.secretKeyRef.key=https-proxy-url +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +有关如何é…置环境å˜é‡çš„更多信æ¯ï¼Œè¯·å‚阅[为容器定义环境å˜é‡](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container)。 + +### é¢å¤–的授信 CA + +如果你有ç§æœ‰é•œåƒä»“库（registries）ã€åº”用商店（catalogs）或拦截è¯ä¹¦çš„代ç†ï¼Œåˆ™å¯èƒ½éœ€è¦å‘ Rancher 添加é¢å¤–的授信 CA。 + +```plain +--set additionalTrustedCAs=true +``` + +创建完 Rancher deployment åŽï¼Œå°† pem æ ¼å¼çš„ CA è¯ä¹¦å¤åˆ¶åˆ°ä¸€ä¸ªå为 `ca-additional.pem` 的文件中，并使用 `kubectl` 在 `cattle-system` 命å空间中创建 `tls-ca-additional` 密文。 + +```plain +kubectl -n cattle-system create secret generic tls-ca-additional --from-file=ca-additional.pem=./ca-additional.pem +``` + +### ç§æœ‰ä»“库和离线安装 + +有关使用ç§æœ‰ä»“库安装 Rancher 的详情，请å‚è§[离线安装](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)。 + +## 外部 TLS 终止 + +我们建议将负载å‡è¡¡å™¨é…置为 4 层å‡è¡¡ï¼Œå°†æ™®é€š 80/tcp å’Œ 443/tcp 转å‘到 Rancher Management 集群节点。集群上的 Ingress Controller ä¼šå°†ç«¯å£ 80 上的 HTTP æµé‡é‡å®šå‘åˆ°ç«¯å£ 443 上的 HTTPS。 + +ä½ å¯ä»¥åœ¨ Rancher 集群（Ingress）外部的 L7 è´Ÿè½½å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS。使用 `--set tls=external` 选项，将负载å‡è¡¡å™¨æŒ‡å‘所有 Rancher é›†ç¾¤èŠ‚ç‚¹ä¸Šçš„ç«¯å£ HTTP 80。这将在 HTTP ç«¯å£ 80 上暴露 Rancher 接å£ã€‚请注æ„，å…许直接连接到 Rancher 集群的客户端ä¸ä¼šè¢«åŠ å¯†ã€‚如果你选择这样åšï¼Œæˆ‘们建议你将网络级别的直接访问é™åˆ¶ä¸ºä»…ä½ çš„è´Ÿè½½å‡è¡¡å™¨ã€‚ + +:::note + +如果你使用的是ç§æœ‰ CA ç­¾åçš„è¯ä¹¦ï¼Œè¯·æ·»åŠ  `--set privateCA=true` 并å‚è§[添加 TLS 密文 - 使用ç§æœ‰ CA ç­¾åè¯ä¹¦](../../../getting-started/installation-and-upgrade/resources/add-tls-secrets.md)，为 Rancher 添加 CA è¯ä¹¦ã€‚ + +::: + +ä½ çš„è´Ÿè½½å‡è¡¡å™¨å¿…须支æŒé•¿æœŸå­˜åœ¨çš„ Websocket 连接，并且需è¦æ’入代ç†å¤´ï¼Œä»¥ä¾¿ Rancher å¯ä»¥æ­£ç¡®ä¼ é€é“¾æŽ¥ã€‚ + +### 使用 NGINX v0.25 为外部 TLS é…ç½® Ingress + +在 NGINX 0.25 中，NGINX 关于转å‘头和外部 TLS 终止的行为[已更改](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0220)。因此，如果你åŒæ—¶ä½¿ç”¨ NGINX 0.25 和外部 TLS 终止é…置，你必须编辑 `cluster.yml` æ¥ä¸º Ingress å¯ç”¨ `use-forwarded-headers` 选项。 + +```yaml +ingress: + provider: nginx + options: + use-forwarded-headers: 'true' +``` + +### 必须的 Header + +- `Host` +- `X-Forwarded-Proto` +- `X-Forwarded-Port` +- `X-Forwarded-For` + +### 建议的超时时间 + +- 读超时：`1800 seconds` +- 写超时：`1800 seconds` +- 连接超时：`30 seconds` + +### å¥åº·æ£€æŸ¥ + +Rancher 将对 `/healthz` 端点的å¥åº·æ£€æŸ¥å“应`200`。 + +### 示例 NGINX é…ç½® + +æ­¤ NGINX é…置已在 NGINX 1.14 上进行了测试。 + +:::caution + +æ­¤ NGINX é…ç½®åªæ˜¯ä¸€ä¸ªç¤ºä¾‹ï¼Œå¯èƒ½ä¸é€‚åˆä½ çš„环境。如需查阅完整文档，请å‚è§ [NGINX è´Ÿè½½å‡è¡¡ - HTTP è´Ÿè½½å‡è¡¡](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/)。 + +::: + +- å°† `IP_NODE1`，`IP_NODE2` å’Œ `IP_NODE3` 替æ¢ä¸ºä½ é›†ç¾¤ä¸­èŠ‚点的 IP 地å€ã€‚ +- 将两处的 `FQDN` å‡æ›¿æ¢ä¸º Rancher çš„ DNS å称。 +- 把 `/certs/fullchain.pem` å’Œ `/certs/privkey.pem` 分别替æ¢ä¸ºæœåŠ¡å™¨è¯ä¹¦å’ŒæœåŠ¡å™¨è¯ä¹¦å¯†é’¥çš„ä½ç½®ã€‚ + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server IP_NODE_1:80; + server IP_NODE_2:80; + server IP_NODE_3:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # 此项å…许执行的 shell 窗å£ä¿æŒå¼€å¯ï¼Œæœ€é•¿å¯è¾¾15分钟。ä¸ä½¿ç”¨æ­¤å‚æ•°çš„è¯ï¼Œé»˜è®¤1分钟åŽè‡ªåŠ¨å…³é—­ã€‚ + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/tls-settings.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/tls-settings.md new file mode 100644 index 00000000000..25f5652fe6e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-references/tls-settings.md @@ -0,0 +1,21 @@ +--- +title: TLS 设置 +--- + +更改默认 TLS 设置的方法å–决于它的安装方å¼ã€‚ + +## 在高å¯ç”¨ Kubernetes 集群中è¿è¡Œ Rancher + +当你在 Kubernetes 集群内安装 Rancher 时，TLS 会在集群的 Ingress Controller 上å¸è½½ã€‚å¯ç”¨çš„ TLS 设置å–决于使用的 Ingress Controller： + +* nginx-ingress-controller（RKE1 å’Œ RKE2 默认）：[默认的 TLS 版本和密ç ](https://kubernetes.github.io/ingress-nginx/user-guide/tls/#default-tls-version-and-ciphers)。 +* traefik（K3s 默认）：[TLS 选项](https://doc.traefik.io/traefik/https/tls/#tls-options)。 + +## 在å•ä¸ª Docker 容器中è¿è¡Œ Rancher + +默认 TLS é…ç½®ä»…æ”¯æŒ TLS 1.2 和安全的 TLS 密ç å¥—件。你å¯ä»¥é€šè¿‡è®¾ç½®ä»¥ä¸‹çŽ¯å¢ƒå˜é‡æ¥æ›´æ”¹æ­¤é…置： + +| å‚æ•° | æè¿° | 默认 | å¯ç”¨é€‰é¡¹ | +|-----|-----|-----|-----| +| `CATTLE_TLS_MIN_VERSION` | æœ€å° TLS 版本 | `1.2` | `1.0`, `1.1`, `1.2`, `1.3` | +| `CATTLE_TLS_CIPHERS` | 支æŒçš„ TLS 密ç å¥—件 | `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305`,
`TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305` | 详情请å‚è§ [Golang TLS 常é‡](https://golang.org/pkg/crypto/tls/#pkg-constants)。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/dockershim.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/dockershim.md new file mode 100644 index 00000000000..84f2383eaf7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/dockershim.md @@ -0,0 +1,47 @@ +--- +title: Dockershim +--- + +Dockershim 是 Kubelet å’Œ Docker Daemon 之间的 CRI 兼容层。Kubernetes 1.20 版本宣布了[移除树内 Dockershim](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/)。有关此移除的更多信æ¯ä»¥åŠæ—¶é—´çº¿ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 + +RKE 集群现在支æŒå¤–部 Dockershim，æ¥è®©ç”¨æˆ·ç»§ç»­ä½¿ç”¨ Docker 作为 CRI è¿è¡Œæ—¶ã€‚现在，我们通过使用 [Mirantis å’Œ Docker ](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) æ¥ç¡®ä¿ RKE 集群å¯ä»¥ç»§ç»­ä½¿ç”¨ Docker，从而实现上游开æºç¤¾åŒºçš„外部 Dockershim。 + +RKE2 å’Œ K3s 集群使用嵌入的 containerd 作为容器è¿è¡Œæ—¶ï¼Œå› æ­¤ä¸å—å½±å“。 + +è¦åœ¨ 1.24 之å‰çš„ RKE 版本中å¯ç”¨å¤–部 Dockershim，请é…置以下选项： + +``` +enable_cri_dockerd: true +``` + +从 1.24 版本开始，以上默认为 true。 + +如果你想使用其他容器è¿è¡Œæ—¶ï¼ŒRancher 也æ供使用 Containerd 作为默认è¿è¡Œæ—¶çš„，以边缘为中心的 K3s，和以数æ®ä¸­å¿ƒä¸ºä¸­å¿ƒçš„ RKE2 Kubernetes å‘行版。然åŽï¼Œä½ å°±å¯ä»¥é€šè¿‡ Rancher 对导入的 RKE2 å’Œ K3s Kubernetes 集群进行å‡çº§å’Œç®¡ç†ã€‚ + +### 常è§é—®é¢˜ + +
+ +Q：是å¦å¿…é¡»å‡çº§ Rancher æ‰èƒ½èŽ·å¾— Rancher 对上游外部 Dockershim 替æ¢çš„支æŒï¼Ÿ + +A：对于 RKE，Dockershim `cri_dockerd` 替æ¢çš„上游支æŒä»Ž Kubernetes 1.21 开始。你需è¦ä½¿ç”¨æ”¯æŒ RKE 1.21 çš„ Rancher 版本。详情请å‚è§æˆ‘们的支æŒçŸ©é˜µã€‚ + +
+ +Q：我目å‰çš„ RKE 使用 Kubernetes 1.23。如果上游最终在 1.24 中删除 Dockershim，会å‘生什么？ + +A：RKE 中带有 Kubernetes çš„ Dockershim 版本将继续工作到 1.23。有关时间线的更多信æ¯ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。从 1.24 开始，RKE 将默认å¯ç”¨ `cri_dockerd` 并在之åŽçš„版本中继续å¯ç”¨ã€‚ + +
+ +Q: 如果我ä¸æƒ³å†ä¾èµ– Dockershim 或 cri_dockerd，我还有什么选择？ + +A: ä½ å¯ä»¥ä¸º Kubernetes 使用ä¸éœ€è¦ Dockershim 支æŒçš„è¿è¡Œæ—¶ï¼Œå¦‚ Containerd。RKE2 å’Œ K3s 就是其中的两个选项。 + +
+ +Q: 如果我目å‰ä½¿ç”¨ RKE1，但想切æ¢åˆ° RKE2，我å¯ä»¥æ€Žæ ·è¿›è¡Œè¿ç§»ï¼Ÿ + +A: ä½ å¯ä»¥æž„建一个新集群，然åŽå°†å·¥ä½œè´Ÿè½½è¿ç§»åˆ°ä½¿ç”¨ Containerd çš„æ–° RKE2 集群。Rancher 也在探索就地å‡çº§è·¯å¾„çš„å¯èƒ½æ€§ã€‚ + +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/install-docker.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/install-docker.md new file mode 100644 index 00000000000..8b98ce09030 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/install-docker.md @@ -0,0 +1,23 @@ +--- +title: 安装 Docker +--- + +在使用 Helm 或 Docker 在 RKE 集群节点上安装 Rancher Server å‰ï¼Œä½ éœ€è¦å…ˆå®‰è£… Docker。RKE2 å’Œ K3s 集群ä¸è¦æ±‚使用 Docker。 + +Docker 有几个安装方法。一ç§æ–¹æ³•æ˜¯å‚è§ [Docker 官方文档](https://docs.docker.com/install/)以了解如何在 Linux 上安装 Docker。ä¸åŒ Linux å‘行版的安装步骤å¯èƒ½æœ‰æ‰€ä¸åŒã€‚ + +å¦ä¸€ç§æ–¹å¼æ˜¯ä½¿ç”¨ Rancher çš„ Docker 安装脚本，该脚本å¯ç”¨äºŽè¾ƒæ–°çš„ Docker 版本。 + +例如，此命令å¯ç”¨äºŽåœ¨ SUSE Linux Enterprise 或 Ubuntu ç­‰ä¸»è¦ Linux å‘行版上安装 Docker 20.10： + +``` +curl https://releases.rancher.com/install-docker/20.10.sh | sh +``` + +Rancher æä¾› Kubernetes 支æŒçš„所有上游 Docker 版本的安装脚本。如需了解我们是å¦æä¾›æŸä¸ª Docker 版本的安装脚本，请å‚è§åŒ…å«äº† Rancher 所有的 Docker 安装脚本的 [GitHub 仓库](https://github.com/rancher/install-docker)。 + +请注æ„，必须应用以下 sysctl 设置： + +``` +net.bridge.bridge-nf-call-iptables=1 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md new file mode 100644 index 00000000000..7dcdc48683c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md @@ -0,0 +1,343 @@ +--- +title: 端å£è¦æ±‚ +description: 了解 Rancher 正常è¿è¡Œæ‰€éœ€çš„端å£è¦æ±‚，包括 Rancher 节点和下游 Kubernetes 集群节点 +--- + +import PortsIaasNodes from '@site/src/components/PortsIaasNodes' +import PortsCustomNodes from '@site/src/components/PortsCustomNodes' +import PortsImportedHosted from '@site/src/components/PortsImportedHosted' + +为了确ä¿èƒ½æ­£å¸¸è¿è¡Œï¼ŒRancher 需è¦åœ¨ Rancher 节点和下游 Kubernetes 集群节点上开放一些端å£ã€‚ + +## Rancher 节点 + +下表列出了è¿è¡Œ Rancher Server 的节点之间需è¦å¼€æ”¾çš„端å£ã€‚ + +ä¸åŒçš„ Rancher Server 架构有ä¸åŒçš„端å£è¦æ±‚。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上。如果你的 Rancher 安装在 K3sã€RKE 或 RKE2 Kubernetes 集群上，请å‚考下é¢çš„标签页。对于其他 Kubernetes å‘行版，请å‚è§è¯¥å‘行版的文档，了解集群节点的端å£è¦æ±‚。 + +:::note 注æ„事项： + +- Rancher 节点å¯èƒ½è¦æ±‚é¢å¤–出站访问已é…置的外部验è¯æ供程åºï¼ˆå¦‚ LDAP）。 +- Kubernetes 建议节点端å£æœåŠ¡ä½¿ç”¨ TCP 30000-32767。 +- 对于防ç«å¢™ï¼Œå¯èƒ½éœ€è¦åœ¨é›†ç¾¤å’Œ Pod CIDR 内å¯ç”¨æµé‡ã€‚ +- Rancher 节点å¯èƒ½è¿˜éœ€è¦å‡ºç«™è®¿é—®ç”¨äºŽå­˜å‚¨é›†ç¾¤å¤‡ä»½ï¼ˆå¦‚ Minio）的外部 S3 上的ä½ç½®ã€‚ + +::: + +### K3s 上 Rancher Server èŠ‚ç‚¹çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +K3s server 需è¦å¼€æ”¾ç«¯å£ 6443 æ‰èƒ½ä¾›èŠ‚点访问。 + +使用 Flannel VXLAN 时，节点需è¦èƒ½å¤Ÿé€šè¿‡ UDP ç«¯å£ 8472 访问其他节点。节点ä¸åº”监å¬ä»»ä½•å…¶ä»–端å£ã€‚K3s 使用åå‘隧é“，建立节点与 Server 的出站连接，所有 kubelet æµé‡éƒ½é€šè¿‡è¯¥éš§é“进行。但是，如果你ä¸ä½¿ç”¨ Flannel，而是使用自定义的 CNI，K3s 则ä¸éœ€è¦æ‰“å¼€ 8472 端å£ã€‚ + +如果è¦ä½¿ç”¨ Metrics Server，则需è¦åœ¨æ¯ä¸ªèŠ‚ç‚¹ä¸Šæ‰“å¼€ç«¯å£ 10250。 + +:::note é‡è¦æ示： + +节点上的 VXLAN 端å£ä¼šå¼€æ”¾é›†ç¾¤ç½‘络，让任何人å‡èƒ½è®¿é—®é›†ç¾¤ã€‚因此，ä¸è¦å°† VXLAN 端å£æš´éœ²ç»™å¤–界。请使用ç¦ç”¨ 8472 端å£çš„防ç«å¢™/安全组æ¥è¿è¡ŒèŠ‚点。 + +::: + +下表æ述了入站和出站æµé‡çš„端å£è¦æ±‚： + +
Rancher Server 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 80 | 执行外部 SSL 终止的负载å‡è¡¡å™¨/ä»£ç† | 使用外部 SSL 终止时的 Rancher UI/API | +| TCP | 443 |
  • Server 节点
  • Agent 节点
  • 托管/注册的 Kubernetes
  • 任何需è¦ä½¿ç”¨ Rancher UI 或 API çš„æº
| Rancher Agent，Rancher UI/API，kubectl | +| TCP | 6443 | K3s Server 节点 | Kubernetes API | +| UDP | 8472 | K3s Server å’Œ Agent 节点 | ä»… Flannel VXLAN éœ€è¦ | +| TCP | 10250 | K3s Server å’Œ Agent 节点 | kubelet | + +
Rancher 节点的出站规则
+ +| åè®® | ç«¯å£ | 目标 | æè¿° | +| -------- | ---- | -------------------------------------------------------- | --------------------------------------------- | +| TCP | 22 | 使用 Node Driver 创建的节点的任何节点 IP | 使用 Node Driver SSH é…置节点 | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | 使用 Node Driver 创建的节点的任何节点 IP | Docker Machine 使用的 Docker daemon TLS ç«¯å£ | +| TCP | 6443 | 托管/导入的 Kubernetes API | Kubernetes API Server | + +
+ +### RKE 上 Rancher Server èŠ‚ç‚¹çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +通常情况下，Rancher 安装在三个 RKE 节点上，这些节点都有 etcdã€controlplane å’Œ worker 角色。 + + + +下表æ述了 Rancher 节点之间æµé‡çš„端å£è¦æ±‚： + +
Rancher 节点的æµé‡è§„则
+ +| åè®® | ç«¯å£ | æè¿° | +|-----|-----|----------------| +| TCP | 443 | Rancher Agents | +| TCP | 2379 | etcd 客户端请求 | +| TCP | 2380 | etcd 对等通信 | +| TCP | 6443 | Kubernetes apiserver | +| TCP | 8443 | NGINX Ingress çš„éªŒè¯ Webhook | +| UDP | 8472 | Canal/Flannel VXLAN 覆盖网络 | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 10250 | Metrics Server 与所有节点的通信 | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | + +下表æ述了入站和出站æµé‡çš„端å£è¦æ±‚： + +
Rancher 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 22 | RKE CLI | RKE 通过 SSH é…置节点 | +| TCP | 80 | è´Ÿè½½å‡è¡¡å™¨/åå‘ä»£ç† | 到 Rancher UI/API çš„ HTTP æµé‡ | +| TCP | 443 |
  • è´Ÿè½½å‡è¡¡å™¨/åå‘代ç†
  • 所有集群节点和其他 API/UI 客户端的 IP
| 到 Rancher UI/API çš„ HTTPS æµé‡ | +| TCP | 6443 | Kubernetes API 客户端 | 到 Kubernetes API çš„ HTTPS æµé‡ | + +
Rancher 节点的出站规则
+ +| åè®® | ç«¯å£ | 目标 | æè¿° | +|-----|-----|----------------|---| +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 22 | 使用 Node Driver 创建的任何节点 | Node Driver 通过 SSH é…置节点 | +| TCP | 2376 | 使用 Node Driver 创建的任何节点 | Node Driver 使用的 Docker daemon TLS ç«¯å£ | +| TCP | 6443 | 托管/导入的 Kubernetes API | Kubernetes API Server | +| TCP | æ供商ä¾èµ– | 托管集群中 Kubernetes API ç«¯ç‚¹çš„ç«¯å£ | Kubernetes API | + +
+ +### RKE2 上 Rancher Server èŠ‚ç‚¹çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +RKE2 server 需è¦å¼€æ”¾ç«¯å£ 6443 å’Œ 9345 æ‰èƒ½ä¾›é›†ç¾¤ä¸­çš„其他节点访问。 + +使用 Flannel VXLAN 时，所有节点都需è¦èƒ½å¤Ÿé€šè¿‡ UDP ç«¯å£ 8472 访问其他节点。 + +如果è¦ä½¿ç”¨ Metrics Server，则需è¦åœ¨æ¯ä¸ªèŠ‚ç‚¹ä¸Šæ‰“å¼€ç«¯å£ 10250。 + +:::note é‡è¦æ示： + +节点上的 VXLAN 端å£ä¼šå¼€æ”¾é›†ç¾¤ç½‘络，让任何人å‡èƒ½è®¿é—®é›†ç¾¤ã€‚因此，ä¸è¦å°† VXLAN 端å£æš´éœ²ç»™å¤–界。请使用ç¦ç”¨ 8472 端å£çš„防ç«å¢™/安全组æ¥è¿è¡ŒèŠ‚点。 + +::: + +
RKE2 Server 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 9345 | RKE2 Server å’Œ Agent 节点 | 节点注册。需è¦åœ¨æ‰€æœ‰ Server 节点上将端å£å¼€æ”¾ç»™é›†ç¾¤ä¸­çš„所有其他节点。 | +| TCP | 6443 | RKE2 Agent 节点 | Kubernetes API | +| UDP | 8472 | RKE2 Server å’Œ Agent 节点 | ä»… Flannel VXLAN éœ€è¦ | +| TCP | 10250 | RKE2 Server å’Œ Agent 节点 | kubelet | +| TCP | 2379 | RKE2 Server 节点 | etcd å®¢æˆ·ç«¯ç«¯å£ | +| TCP | 2380 | RKE2 Server 节点 | etcd å¯¹ç­‰ç«¯å£ | +| TCP | 30000-32767 | RKE2 Server å’Œ Agent 节点 | NodePort 端å£èŒƒå›´ã€‚å¯ä»¥ä½¿ç”¨ TCP 或 UDP。 | +| TCP | 5473 | Calico-node pod 连接到 typha pod | 使用 Calico éƒ¨ç½²æ—¶éœ€è¦ | +| HTTP | 80 | 执行外部 SSL 终止的负载å‡è¡¡å™¨/ä»£ç† | 使用外部 SSL 终止时的 Rancher UI/API | +| HTTPS | 443 |
  • 托管/注册的 Kubernetes
  • 任何需è¦ä½¿ç”¨ Rancher UI 或 API çš„æº
| Rancher Agent，Rancher UI/API，kubectl。如果负载å‡è¡¡å™¨æ‰§è¡Œ TLS 终止，则ä¸éœ€è¦ã€‚ | + +所有出站æµé‡é€šå¸¸éƒ½æ˜¯å…许的。 +
+ +### Docker 安装的 Rancher Server çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了 Rancher 节点入站和出站æµé‡çš„端å£è¦æ±‚： + +
Rancher 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 80 | 执行外部 SSL 终止的负载å‡è¡¡å™¨/ä»£ç† | 使用外部 SSL 终止时的 Rancher UI/API | +| TCP | 443 |
  • 托管/注册的 Kubernetes
  • 任何需è¦ä½¿ç”¨ Rancher UI 或 API çš„æº
| Rancher Agent，Rancher UI/API，kubectl | + +
Rancher 节点的出站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 22 | 使用 Node Driver 创建的节点的任何节点 IP | 使用 Node Driver SSH é…置节点 | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | 使用 Node Driver 创建的节点的任何节点 IP | Docker Machine 使用的 Docker daemon TLS ç«¯å£ | +| TCP | 6443 | 托管/导入的 Kubernetes API | Kubernetes API Server | + +
+ +## 下游 Kubernetes 集群节点 + +下游 Kubernetes 集群用于è¿è¡Œä½ çš„应用和æœåŠ¡ã€‚本节介ç»äº†å“ªäº›ç«¯å£éœ€è¦åœ¨ä¸‹æ¸¸é›†ç¾¤çš„节点上打开，以便 Rancher 能够与它们进行通信。 + +ä¸åŒçš„下游集群的å¯åŠ¨æ–¹å¼æœ‰ä¸åŒçš„端å£è¦æ±‚。下é¢çš„æ¯ä¸ªæ ‡ç­¾éƒ½åˆ—出了ä¸åŒ[集群类型](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)所需打开的端å£ã€‚ + +下图æ述了为æ¯ä¸ª[集群类型](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)打开的端å£ã€‚ + +
Rancher 管ç†é¢æ¿çš„端å£è¦æ±‚
+ +![基本端å£è¦æ±‚](/img/port-communications.svg) + +:::tip + +如果你对安全性的关注ä¸æ˜¯å¤ªé«˜ï¼Œè€Œä¸”也愿æ„多打开几个端å£ï¼Œä½ å¯ä»¥å‚考[常用端å£](#常用端å£)中列出的端å£ï¼Œè€Œä¸æ˜¯å‚考下方的表格。 + +::: + +### Harvester é›†ç¾¤çš„ç«¯å£ + +有关 Harvester 端å£è¦æ±‚的更多信æ¯ï¼Œè¯·å‚阅[此处](../../../integrations-in-rancher/harvester.md#端å£è¦æ±‚)。 + + +### Rancher 使用节点池å¯åŠ¨ Kubernetes é›†ç¾¤çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了节点在[云æ供商](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)中创建的情况下，[Rancher å¯åŠ¨ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 的端å£è¦æ±‚。 + +:::note + +在 AWS EC2 或 DigitalOcean 等云æ供商中创建集群期间，Rancher 会自动打开所需的端å£ã€‚ + +::: + + + +
+ +### Rancher 使用自定义节点å¯åŠ¨ Kubernetes é›†ç¾¤çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了使用[自定义节点](../../../pages-for-subheaders/use-existing-nodes.md)的情况下，[Rancher å¯åŠ¨ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 的端å£è¦æ±‚。 + + + +
+ +### 托管 Kubernetes é›†ç¾¤çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了[托管集群](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)的端å£è¦æ±‚。 + + + +
+ +### å·²æ³¨å†Œé›†ç¾¤çš„ç«¯å£ + +:::note + +在 Rancher 2.5 之å‰ï¼Œæ³¨å†Œé›†ç¾¤è¢«ç§°ä¸ºå¯¼å…¥é›†ç¾¤ã€‚ + +::: + +
+ å•å‡»å±•å¼€ + +下表æ述了[注册集群](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)的端å£è¦æ±‚。 + + + +
+ + +## 其他端å£æ³¨æ„事项 + +### å¸¸ç”¨ç«¯å£ + +无论集群是什么类型，常用端å£é€šå¸¸åœ¨ä½ çš„ Kubernetes 节点上打开。 + +import CommonPortsTable from '../../../shared-files/_common-ports-table.md'; + + + +---- + +### 本地节点æµé‡ + +上述è¦æ±‚中标记为`本地æµé‡`（例如 `9099 TCP`）的端å£ä¼šç”¨äºŽ Kubernetes å¥åº·æ£€æŸ¥ （`livenessProbe` å’Œ `readinessProbe`）。 +这些å¥åº·æ£€æŸ¥æ˜¯åœ¨èŠ‚点本身执行的。在大多数云环境中，这ç§æœ¬åœ°æµé‡æ˜¯é»˜è®¤å…许的。 + +但是，在以下情况下å¯èƒ½ä¼šé˜»æ­¢æ­¤æµé‡ï¼š + +- 你已在节点上应用了严格的主机防ç«å¢™ç­–略。 +- 你正在使用有多个接å£ï¼ˆå¤šå®¿ä¸»ï¼‰çš„节点。 + +在这些情况下，你必须在你的主机防ç«å¢™ä¸­ä¸»åŠ¨å…许这ç§æµé‡ï¼Œå¦‚果是公共/ç§æœ‰äº‘托管的主机（例如 AWS 或 OpenStack），你需è¦åœ¨ä½ çš„安全组é…置中主动å…许此æµé‡ã€‚请记ä½ï¼Œå¦‚果你在安全组中使用安全组作为æºæˆ–目标，主动开放端å£åªé€‚用于节点/实例的ç§æœ‰æŽ¥å£ã€‚ + +### Rancher AWS EC2 安全组 + +当你使用 [AWS EC2 Node Driver](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) 在 Rancher 中é…置集群节点时，你å¯ä»¥è®© Rancher 创建一个å为 `rancher-nodes` 的安全组。以下规则会自动添加到该安全组中。 + +| 类型 | åè®® | 端å£èŒƒå›´ | æº/目标 | 规则类型 | +|-----------------|:--------:|:-----------:|------------------------|:---------:| +| SSH | TCP | 22 | 0.0.0.0/0 | 入站 | +| HTTP | TCP | 80 | 0.0.0.0/0 | 入站 | +| 自定义 TCP 规则 | TCP | 443 | 0.0.0.0/0 | 入站 | +| 自定义 TCP 规则 | TCP | 2376 | 0.0.0.0/0 | 入站 | +| 自定义 TCP 规则 | TCP | 2379-2380 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 UDP 规则 | UDP | 4789 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 6443 | 0.0.0.0/0 | 入站 | +| 自定义 UDP 规则 | UDP | 8472 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 10250-10252 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 10256 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 30000-32767 | 0.0.0.0/0 | 入站 | +| 自定义 UDP 规则 | UDP | 30000-32767 | 0.0.0.0/0 | 入站 | +| 所有æµé‡ | 全部 | 全部 | 0.0.0.0/0 | 出站 | + +### 打开 SUSE Linux ç«¯å£ + +SUSE Linux å¯èƒ½æœ‰ä¸€ä¸ªé˜²ç«å¢™ï¼Œé»˜è®¤æƒ…况下会阻止所有端å£ã€‚è¦æ‰“开将主机添加到自定义集群所需的端å£ï¼š + + + + +1. SSH 进入实例。 +1. 以文本模å¼å¯åŠ¨ YaST： + ``` + sudo yast2 + ``` + +1. 导航到**安全和用户** > **防ç«å¢™** > **区域：公共** > **端å£**。è¦åœ¨ç•Œé¢å†…导航，请å‚ç…§[说明](https://doc.opensuse.org/documentation/leap/reference/html/book-reference/cha-yast-text.html#sec-yast-cli-navigate)。 +1. è¦æ‰“开所需的端å£ï¼ŒæŠŠå®ƒä»¬è¾“入到 **TCP 端å£** å’Œ **UDP 端å£** 字段。在这个例å­ä¸­ï¼Œç«¯å£ 9796 å’Œ 10250 也被打开，用于监控。由此产生的字段应类似于以下内容： + ```yaml + TCP Ports + 22, 80, 443, 2376, 2379, 2380, 6443, 9099, 9796, 10250, 10254, 30000-32767 + UDP Ports + 8472, 30000-32767 + ``` + +1. 所有必须端å£éƒ½è¾“å…¥åŽï¼Œé€‰æ‹©**接å—**。 + + + + +1. SSH 进入实例。 +1. 编辑 `/etc/sysconfig/SuSEfirewall2` 并打开所需的端å£ã€‚在这个例å­ä¸­ï¼Œç«¯å£ 9796 å’Œ 10250 也被打开，用于监控。 + ``` + FW_SERVICES_EXT_TCP="22 80 443 2376 2379 2380 6443 9099 9796 10250 10254 30000:32767" + FW_SERVICES_EXT_UDP="8472 30000:32767" + FW_ROUTE=yes + ``` +1. 用新的端å£é‡å¯é˜²ç«å¢™ï¼š + ``` + SuSEfirewall2 + ``` + + + + +**结果** ：该节点已打开添加到自定义集群所需的端å£ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md new file mode 100644 index 00000000000..a639349d8cf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md @@ -0,0 +1,149 @@ +--- +title: Docker 安装命令 +--- + +Docker 安装适用于想è¦æµ‹è¯• Rancher 的用户。 + +ä½ å¯ä»¥ä½¿ç”¨ `docker run`命令，把 Rancher Server 组件安装到å•ä¸ªèŠ‚点上，而ä¸éœ€è¦è¿è¡Œ Kubernetes 集群。由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此，如果该节点å‘生故障，由于其他节点上没有å¯ç”¨çš„ etcd æ•°æ®å‰¯æœ¬ï¼Œä½ å°†ä¸¢å¤± Rancher Server 的所有数æ®ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨å¤‡ä»½åº”用，按照[这些步骤](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)，将 Rancher Server 从 Docker 安装è¿ç§»åˆ° Kubernetes 安装。 + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +| 环境å˜é‡é”® | 环境å˜é‡å€¼ | æè¿° | +| -------------------------------- | -------------------------------- | ---- | +| `CATTLE_SYSTEM_DEFAULT_REGISTRY` | `` | å°† Rancher Server é…ç½®æˆåœ¨é…置集群时，始终从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚ | +| `CATTLE_SYSTEM_CATALOG` | `bundled` | é…ç½® Rancher Server 使用打包的 Helm System Chart 副本。[system charts](https://github.com/rancher/system-charts) 仓库包å«æ‰€æœ‰ Monitoring，Logging，告警和全局 DNS 等功能所需的应用商店项目。这些 [Helm Chart](https://github.com/rancher/system-charts) ä½äºŽ GitHub 中。但是由于你处在离线环境，因此使用 Rancher 内置的 Chart 会比设置 Git mirror 容易得多。 | + +:::note 你是å¦éœ€è¦ï¼š + +- é…置自定义 CA æ ¹è¯ä¹¦ä»¥è®¿é—®æœåŠ¡ã€‚å‚è§[自定义 CA æ ¹è¯ä¹¦](../../resources/custom-ca-root-certificates.md)。 +- 记录所有 Rancher API 的事务。å‚è§ [API 审计](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志)。 + +::: + +选择以下的选项之一： + +### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你在ä¸è€ƒè™‘身份验è¯çš„å¼€å‘或测试环境中安装 Rancher，å¯ä»¥ä½¿ç”¨ Rancher 生æˆçš„自签åè¯ä¹¦å®‰è£… Rancher。这ç§å®‰è£…æ–¹å¼é¿å…了自己生æˆè¯ä¹¦çš„麻烦。 + +登录到你的 Linux 主机，然åŽè¿è¡Œä¸‹é¢çš„安装命令。输入命令时，å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚ + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å®‰è£…çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +特æƒè®¿é—®æ˜¯[å¿…é¡»](./install-rancher-ha.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` + +
+ +### 选项 B：使用你自己的è¯ä¹¦ - 自签å + +
+ å•å‡»å±•å¼€ + +在你团队访问 Rancher Server çš„å¼€å‘或测试环境中，创建一个用于你的安装的自签åè¯ä¹¦ï¼Œä»¥ä¾¿å›¢é˜ŸéªŒè¯ä»–们对实例的连接。 + +:::note 先决æ¡ä»¶ï¼š + +从能连接到互è”网的计算机上，使用 [OpenSSL](https://www.openssl.org/) 或其他方法创建自签åè¯ä¹¦ã€‚ + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 +- 在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](../rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +::: + +创建è¯ä¹¦åŽï¼Œç™»å½• Linux 主机，然åŽè¿è¡Œä»¥ä¸‹å®‰è£…命令。输入命令时，å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚使用 `-v` 标志并æä¾›è¯ä¹¦çš„路径，以将è¯ä¹¦æŒ‚载到容器中。 + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å®‰è£…çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +特æƒè®¿é—®æ˜¯[å¿…é¡»](./install-rancher-ha.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` + +
+ +### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +在公开暴露应用的开å‘或测试环境中，请使用由å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œä»¥é¿å…用户收到è¯ä¹¦å®‰å…¨è­¦å‘Šã€‚ + +:::note 先决æ¡ä»¶ï¼š + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 + +::: + +获å–è¯ä¹¦åŽï¼Œç™»å½• Linux 主机，然åŽè¿è¡Œä»¥ä¸‹å®‰è£…命令。输入命令时，å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚因为你的è¯ä¹¦æ˜¯ç”±å¯ä¿¡çš„ CA ç­¾å的，因此你ä¸éœ€è¦å®‰è£…é¢å¤–çš„ CA è¯ä¹¦æ–‡ä»¶ã€‚ + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å®‰è£…çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +:::note + +使用 `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +::: + +特æƒè®¿é—®æ˜¯[å¿…é¡»](./install-rancher-ha.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged + /rancher/rancher: +``` + +
+ + + +:::note + +如果你ä¸æƒ³å‘é€é¥æµ‹æ•°æ®ï¼Œåœ¨é¦–次登录时退出[é¥æµ‹](../../../../faq/telemetry.md)。 + +::: + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md new file mode 100644 index 00000000000..e3e7be1a627 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md @@ -0,0 +1,193 @@ +--- +title: '1. 设置基础设施和ç§æœ‰é•œåƒä»“库' +--- + +本文介ç»å¦‚何在离线环境中，为 Rancher Management server é…置底层基础设施。你还将设置 Rancher 节点中必须å¯ç”¨çš„ç§æœ‰å®¹å™¨é•œåƒä»“库。 + +离线环境是 Rancher Server 离线安装或安装在防ç«å¢™åŽé¢çš„环境。 + +Rancher 安装在 K3s Kubernetes 集群ã€RKE Kubernetes 集群还是å•ä¸ª Docker 容器上对应的基础设施设置会有所ä¸åŒã€‚如需了解å„个安装方å¼çš„更多信æ¯ï¼Œè¯·å‚è§[本页](../../../../pages-for-subheaders/installation-and-upgrade.md)。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上。为了阅读方便，我们在下文中ä»æ供了 RKE å’Œ K3s Kubernetes 基础设施教程。 + + + + +为了实现高å¯ç”¨å®‰è£…，我们建议设置以下的基础设施： + +- **2 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商中的虚拟机。 +- **1 个外部数æ®åº“**：用于存储集群数æ®ã€‚æ”¯æŒ PostgreSQL, MySQL å’Œ etcd。 +- **1 个负载å‡è¡¡å™¨**：用于将æµé‡è½¬å‘到这两个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ +- **ç§æœ‰é•œåƒä»“库**，用于将容器镜åƒåˆ†å‘到你的主机。 + +### 1. é…ç½® Linux 节点 + +这些主机会断开互è”网链接，但需è¦èƒ½ä¸Žä½ çš„ç§æœ‰é•œåƒä»“库连接。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置外部数æ®åº“ + +K3s 与其他 Kubernetes å‘行版ä¸åŒï¼Œåœ¨äºŽå…¶æ”¯æŒä½¿ç”¨ etcd 以外的数æ®åº“æ¥è¿è¡Œ Kubernetes。该功能让 Kubernetes è¿ç»´æ›´åŠ çµæ´»ã€‚ä½ å¯ä»¥æ ¹æ®å®žé™…情况选择åˆé€‚çš„æ•°æ®åº“。 + +对于 K3s 高å¯ç”¨å®‰è£…，你需è¦é…置以下的其中一个数æ®åº“： + +* [PostgreSQL](https://www.postgresql.org/)（10.7 å’Œ 11.5 已验è¯ï¼‰ +* [MySQL](https://www.mysql.com/)（5.7 已验è¯ï¼‰ +* [etcd](https://etcd.io/)（3.3.15 已验è¯ï¼‰ + +在安装 Kubernetes 时，你需è¦ä¼ å…¥ K3s 连接数æ®åº“的详细信æ¯ã€‚ + +如需获å–é…置数æ®åº“示例，请å‚è§[在 Amazon RDS æœåŠ¡ä¸­é…ç½® MySQL æ•°æ®åº“](../../../../how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md)的教程。 + +如需获å–é…ç½® K3s 集群数æ®åº“的所有å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/datastore/)。 + +### 3. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，K3s 工具会部署一个 Traefik Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 Traefik Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚Traefik Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md)。 + +:::note é‡è¦æ示： + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 4. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 + +### 5. é…ç½®ç§æœ‰é•œåƒä»“库 + +Rancher 支æŒä½¿ç”¨ç§æœ‰é•œåƒä»“库进行离线安装。你必须有自己的ç§æœ‰é•œåƒä»“库或使用其他方å¼å°†å®¹å™¨é•œåƒåˆ†å‘到主机。 + +在åŽç»­è®¾ç½® K3s Kubernetes 集群时，你需è¦åˆ›å»ºä¸€ä¸ª[ç§æœ‰é•œåƒä»“库é…置文件](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)，其中包å«æ­¤é•œåƒä»“库的信æ¯ã€‚ + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +如需在高å¯ç”¨ RKE 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商（例如 Amazon EC2，GCE 或 vSphere）中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将å‰ç«¯æµé‡è½¬å‘到这三个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ +- **ç§æœ‰é•œåƒä»“库**，用于将容器镜åƒåˆ†å‘到你的主机。 + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸæˆ–æ•°æ®ä¸­å¿ƒã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºã€‚ + +### 为什么使用三个节点？ + +在 RKE 集群中，Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。而这个 etcd æ•°æ®åº“在这三个节点上è¿è¡Œã€‚ + +为了选举出大多数 etcd 节点认å¯çš„ etcd 集群 leader，etcd æ•°æ®åº“需è¦å¥‡æ•°ä¸ªèŠ‚点。如果 etcd æ•°æ®åº“无法选出 leader，etcd å¯èƒ½ä¼šå‡ºçŽ°[脑裂（split brain）](https://www.quora.com/What-is-split-brain-in-distributed-systems)的问题，此时你需è¦ä½¿ç”¨å¤‡ä»½æ¢å¤é›†ç¾¤ã€‚如果三个 etcd 节点之一å‘生故障，其余两个节点å¯ä»¥é€‰æ‹©ä¸€ä¸ª leader，因为它们是 etcd 节点总数的大多数部分。 + +### 1. é…ç½® Linux 节点 + +这些主机会断开互è”网链接，但需è¦èƒ½ä¸Žä½ çš„ç§æœ‰é•œåƒä»“库连接。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 3. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 + +### 4. é…ç½®ç§æœ‰é•œåƒä»“库 + +Rancher 支æŒä½¿ç”¨å®‰å…¨çš„ç§æœ‰é•œåƒä»“库进行离线安装。你必须有自己的ç§æœ‰é•œåƒä»“库或使用其他方å¼å°†å®¹å™¨é•œåƒåˆ†å‘到主机。 + +在åŽç»­è®¾ç½® RKE Kubernetes 集群时，你需è¦åˆ›å»ºä¸€ä¸ª[ç§æœ‰é•œåƒä»“库é…置文件](https://rancher.com/docs/rke/latest/en/config-options/private-registries/)，其中包å«æ­¤é•œåƒä»“库的信æ¯ã€‚ + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +:::note 注æ„事项： + +- Docker 安装适用于想è¦æµ‹è¯• Rancher 的用户。由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此如果该节点å‘生故障，你将丢失 Rancher Server 的所有数æ®ã€‚ + +- Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +::: + +### 1. é…ç½® Linux 节点 + +此主机会断开互è”网链接，但需è¦èƒ½ä¸Žä½ çš„ç§æœ‰é•œåƒä»“库连接。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器，硬件和网络](../../../../pages-for-subheaders/installation-requirements.md)的常规安装è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…ç½®ç§æœ‰ Docker é•œåƒä»“库 + +Rancher 支æŒä½¿ç”¨ç§æœ‰é•œåƒä»“库在堡垒æœåŠ¡å™¨ä¸­è¿›è¡Œç¦»çº¿å®‰è£…。你必须有自己的ç§æœ‰é•œåƒä»“库或使用其他方å¼å°†å®¹å™¨é•œåƒåˆ†å‘到主机。 + +如需获得创建ç§æœ‰é•œåƒä»“库的帮助，请å‚è§ [Docker 官方文档](https://docs.docker.com/registry/)。 + + + + +### åŽç»­æ“作 +[收集镜åƒå¹¶å‘布到你的ç§æœ‰é•œåƒä»“库](publish-images.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md new file mode 100644 index 00000000000..3fcd8a992cf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md @@ -0,0 +1,387 @@ +--- +title: '3. 安装 Kubernetes（Docker 安装请跳过）' +--- + +:::note + +如果你使用 Docker 在å•ä¸ªèŠ‚点上安装 Rancher，请跳过本节。 + +::: + +本文æè¿°äº†å¦‚ä½•æ ¹æ® [Rancher Server 环境的最佳实践](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#kubernetes-安装环境)æ¥å®‰è£… Kubernetes 集群。该集群需è¦ä¸“用于è¿è¡Œ Rancher Server。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上，包括托管的 Kubernetes。 + +在 RKEã€RKE2 或 K3s 上离线安装 Kubernetes 集群的步骤如下所示： + + + + +在本指å—中，我们å‡è®¾ä½ å·²ç»åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­åˆ›å»ºäº†èŠ‚点，并且在堡垒æœåŠ¡å™¨ä¸Šæœ‰ä¸€ä¸ªå®‰å…¨çš„ Docker ç§æœ‰é•œåƒä»“库。 + +### å®‰è£…æ¦‚è¦ + +1. [准备镜åƒç›®å½•](#1-准备镜åƒç›®å½•) +2. [创建镜åƒä»“库 YAML](#2-创建镜åƒä»“库-yaml) +3. [安装 K3s](#3-安装-k3s) +4. [ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件](#4-ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨-kubeconfig-文件) + +### 1. 准备镜åƒç›®å½• +从 [Releases](https://github.com/k3s-io/k3s/releases) 页é¢èŽ·å–è¦è¿è¡Œçš„ K3s ç‰ˆæœ¬çš„é•œåƒ tar 文件。 + +在æ¯ä¸ªèŠ‚点上å¯åŠ¨ K3s 之å‰ï¼Œå°†è¿™ä¸ª tar 文件放在 `images` 目录中，例如： + +```sh +sudo mkdir -p /var/lib/rancher/k3s/agent/images/ +sudo cp ./k3s-airgap-images-$ARCH.tar /var/lib/rancher/k3s/agent/images/ +``` + +### 2. 创建镜åƒä»“库 YAML +把 `registries.yaml` 文件创建到 `/etc/rancher/k3s/registries.yaml` 中。此文件为 K3s æ供连接到你的ç§æœ‰é•œåƒä»“库的详细信æ¯ã€‚ + +在加入必è¦ä¿¡æ¯ä¹‹å‰ï¼Œ`registries.yaml` 文件是这样的： + +```yaml +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # é•œåƒä»“库的用户å + password: xxxxxx # é•œåƒä»“åº“çš„å¯†ç  + tls: + cert_file: <é•œåƒä»“库所用的è¯ä¹¦æ–‡ä»¶è·¯å¾„> + key_file: <é•œåƒä»“库所用的密钥文件路径> + ca_file: <é•œåƒä»“库所用的 CA 文件路径> +``` + +请注æ„，目å‰ï¼ŒK3s 仅支æŒå®‰å…¨çš„é•œåƒä»“库（带有自定义 CA çš„ SSL）。 + +有关 K3s çš„ç§æœ‰é•œåƒä»“库é…置文件的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)。 + +### 3. 安装 K3s + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +从 [Releases](https://github.com/k3s-io/k3s/releases) 页é¢èŽ·å– K3s 的二进制文件，该文件è¦åŒ¹é…用于获å–离线镜åƒçš„ tar 版本。 +访问 [K3s 安装脚本](https://get.k3s.io)ä»¥èŽ·å– K3s 的安装脚本。 + +将二进制文件放到æ¯ä¸ªèŠ‚点的 `/usr/local/bin` 中。 +将安装脚本放在æ¯ä¸ªèŠ‚点的任æ„ä½ç½®ï¼Œå¹¶å°†è„šæœ¬å‘½å为 `install.sh`。 + +在æ¯ä¸ª Server 上安装 K3s： + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= ./install.sh +``` + +在æ¯ä¸ª Agent 上安装 K3s： + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= K3S_URL=https://:6443 K3S_TOKEN= ./install.sh +``` + +其中 `` 是 Server çš„ IP 或有效 DNS，`` 是å¯ä»¥åœ¨ `/var/lib/rancher/k3s/server/node-token` 中找到的 Server node-token。 + +:::note + +K3s 自动为 kubelets æä¾› `--resolv-conf` 标志，该标志å¯èƒ½å¯¹åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­é…ç½® DNS 有帮助。 + +::: + +### 4. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 K3s 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/k3s/k3s.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 +2. å¤åˆ¶ `/etc/rancher/k3s/k3s.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhost。你需è¦å°† `server` é…置为负载å‡è¡¡å™¨çš„ DNSï¼Œå¹¶æŒ‡å®šç«¯å£ 6443ï¼ˆé€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server）。以下是一个 `k3s.yaml` 示例： + +```yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ K3s 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/cluster-access/) 或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### å‡çº§æ³¨æ„事项 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹æ–¹å¼å®Œæˆç¦»çº¿çŽ¯å¢ƒçš„å‡çº§ï¼š + +1. 从 [Releases](https://github.com/k3s-io/k3s/releases) 页é¢ä¸‹è½½è¦å‡çº§çš„ K3s ç‰ˆæœ¬çš„æ–°ç¦»çº¿é•œåƒ tar 包。将 tar 文件放在æ¯ä¸ªèŠ‚点上的 `/var/lib/rancher/k3s/agent/images/` 目录中。删除旧的 tar 文件。 +2. å¤åˆ¶å¹¶æ›¿æ¢æ¯ä¸ªèŠ‚点上 `/usr/local/bin` 中的旧 K3s 二进制文件。å¤åˆ¶ [K3s 安装脚本](https://get.k3s.io)（因为脚本å¯èƒ½è‡ªä¸Šæ¬¡ç‰ˆæœ¬å‘布以æ¥å·²æ›´æ”¹ï¼‰ã€‚使用相åŒçš„环境å˜é‡å†æ¬¡è¿è¡Œè„šæœ¬ã€‚ +3. é‡å¯ K3s æœåŠ¡ï¼ˆå¦‚果安装程åºæ²¡æœ‰è‡ªåŠ¨é‡å¯ K3s çš„è¯ï¼‰ã€‚ + + + + +在本指å—中，我们å‡è®¾ä½ å·²ç»åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­åˆ›å»ºäº†èŠ‚点，并且在堡垒æœåŠ¡å™¨ä¸Šæœ‰ä¸€ä¸ªå®‰å…¨çš„ Docker ç§æœ‰é•œåƒä»“库。 + +### å®‰è£…æ¦‚è¦ + +1. [创建 RKE2 é…ç½®](#1-创建-rke2-é…ç½®) +2. [创建镜åƒä»“库 YAML](#2-创建镜åƒä»“库-yaml) +3. [安装 RKE2](#3-安装-rke2) +4. [ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件](#4-ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨-kubeconfig-文件) + +### 1. 创建 RKE2 é…ç½® +把 config.yaml 文件创建到 `/etc/rancher/rke2/config.yaml` 中。这将包å«åˆ›å»ºé«˜å¯ç”¨ RKE2 集群所需的所有é…置选项。 + +第一å°æœåŠ¡å™¨çš„最低é…置是： + +``` +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +其他æœåŠ¡å™¨çš„é…置文件应该包å«ç›¸åŒçš„令牌，并让 RKE2 知é“è¦è¿žæŽ¥åˆ°çŽ°æœ‰çš„第一å°æœåŠ¡å™¨ï¼š + +``` +server: https://ip-of-first-server:9345 +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/install/ha)。 + +:::note + +RKE2 自动为 kubelets æä¾› `resolv-conf` 选项，该标志å¯èƒ½å¯¹åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­é…ç½® DNS 有帮助。 + +::: + +### 2. 创建镜åƒä»“库 YAML +把 `registries.yaml` 文件创建到 `/etc/rancher/rke2/registries.yaml` 中。此文件为 RKE2 æ供连接到你的ç§æœ‰é•œåƒä»“库的详细信æ¯ã€‚ + +在加入必è¦ä¿¡æ¯ä¹‹å‰ï¼Œ`registries.yaml` 文件是这样的： + +``` +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # é•œåƒä»“库的用户å + password: xxxxxx # é•œåƒä»“åº“çš„å¯†ç  + tls: + cert_file: <é•œåƒä»“库所用的è¯ä¹¦æ–‡ä»¶è·¯å¾„> + key_file: <é•œåƒä»“库所用的密钥文件路径> + ca_file: <é•œåƒä»“库所用的 CA 文件路径> +``` + +有关 RKE2 çš„ç§æœ‰é•œåƒä»“库é…置文件的详情，请å‚è§ [RKE2 官方文档](https://docs.rke2.io/install/containerd_registry_configuration)。 + +### 3. 安装 RKE2 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§[支æŒç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。 + +从 Release 页é¢ä¸‹è½½å®‰è£…脚本ã€rke2ã€rke2-images å’Œ sha256sum 存档，并将它们上传到æ¯ä¸ªæœåŠ¡å™¨ä¸Šçš„目录中： + +``` +mkdir /tmp/rke2-artifacts && cd /tmp/rke2-artifacts/ +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2-images.linux-amd64.tar.zst +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2.linux-amd64.tar.gz +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/sha256sum-amd64.txt +curl -sfL https://get.rke2.io --output install.sh +``` + +接下æ¥ï¼Œä½¿ç”¨æ¯ä¸ªæœåŠ¡å™¨ä¸Šçš„目录è¿è¡Œ install.sh，如下例所示： + +``` +INSTALL_RKE2_ARTIFACT_PATH=/tmp/rke2-artifacts sh install.sh +``` + +然åŽåœ¨æ‰€æœ‰æœåŠ¡å™¨ä¸Šå¯ç”¨å¹¶å¯åŠ¨è¯¥æœåŠ¡ï¼š + +`` +systemctl enable rke2-server.service +systemctl start rke2-server.service +`` + +有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/install/airgap)。 + +### 4. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 RKE2 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/rke2/rke2.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)（Kubernetes 命令行工具）。 +2. å¤åˆ¶ `/etc/rancher/rke2/rke2.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhost。你需è¦å°† `server` é…置为负载å‡è¡¡å™¨çš„ DNSï¼Œå¹¶æŒ‡å®šç«¯å£ 6443ï¼ˆé€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server）。以下是一个 `rke2.yaml` 示例： + +``` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ RKE2 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [RKE2 官方文档](https://docs.rke2.io/cluster_access)或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### å‡çº§æ³¨æ„事项 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹æ–¹å¼å®Œæˆç¦»çº¿çŽ¯å¢ƒçš„å‡çº§ï¼š + +1. 从 [Releases](https://github.com/rancher/rke2/releases) 页é¢ä¸‹è½½æ–°çš„离线工件，并安装å‡çº§ RKE2 版本的脚本。 +2. 使用相åŒçš„环境å˜é‡å†æ¬¡è¿è¡Œè„šæœ¬ã€‚ +3. é‡å¯ RKE2 æœåŠ¡ã€‚ + + + +我们将使用 Rancher Kubernetes Engine (RKE) 创建一个 Kubernetes 集群。在å¯åŠ¨ Kubernetes 集群之å‰ï¼Œä½ éœ€è¦å®‰è£… RKE 并创建 RKE é…置文件。 + +### 1. 安装 RKE + +å‚ç…§ [RKE 官方文档](https://rancher.com/docs/rke/latest/en/installation/)的说明安装 RKE。 + +:::note + +ä½ å¯ä»¥åœ¨ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)中找到基于 Rancher 版本的 RKE 认è¯ç‰ˆæœ¬ã€‚ + +::: + +### 2. 创建 RKE é…置文件 + +在å¯è®¿é—®ä½  Linux 主机节点上的 22/TCP 端å£å’Œ 6443/TCP 端å£çš„系统上，使用以下示例创建一个å为 `rancher-cluster.yml` 的新文件。 + +该文件是 RKE é…置文件，用于é…置你è¦éƒ¨ç½² Rancher 的集群。 + +å‚考下方的 _RKE 选项_ 表格，修改代ç ç¤ºä¾‹ä¸­çš„å‚数。使用你创建的三个节点的 IP 地å€æˆ– DNS å称。 + +:::tip + +如需获å–å¯ç”¨é€‰é¡¹çš„详情，请å‚è§ RKE [é…置选项](https://rancher.com/docs/rke/latest/en/config-options/)。 + +::: + +
RKE 选项
+ +| 选项 | å¿…å¡« | æè¿° | +| ------------------ | -------------------- | --------------------------------------------------------------------------------------- | +| `address` | ✓ | 离线环境中节点的 DNS 或 IP åœ°å€ | +| `user` | ✓ | å¯è¿è¡Œ Docker 命令的用户 | +| `role` | ✓ | 分é…给节点的 Kubernetes 角色列表 | +| `internal_address` | å¯é€‰1 | 用于集群内部æµé‡çš„ DNS 或 IP åœ°å€ | +| `ssh_key_path` | | 用æ¥éªŒè¯èŠ‚点的 SSH ç§é’¥æ–‡ä»¶è·¯å¾„（默认值为 `~/.ssh/id_rsa`） | + +> 1 如果你想使用引用安全组或防ç«å¢™ï¼ŒæŸäº›æœåŠ¡ï¼ˆå¦‚ AWS EC2）è¦æ±‚设置 `internal_address`。 + +```yaml +nodes: + - address: 10.10.3.187 # 离线环境节点 IP + internal_address: 172.31.7.22 # 节点内网 IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.254 # 离线环境节点 IP + internal_address: 172.31.13.132 # 节点内网 IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.89 # 离线环境节点 IP + internal_address: 172.31.3.216 # 节点内网 IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + +private_registries: + - url: # ç§æœ‰é•œåƒä»“库 URL + user: rancher + password: '*********' + is_default: true +``` + +### 3. è¿è¡Œ RKE + +é…ç½® `rancher-cluster.yml`åŽï¼Œå¯åŠ¨ä½ çš„ Kubernetes 集群： + +``` +rke up --config ./rancher-cluster.yml +``` + +### 4. ä¿å­˜ä½ çš„文件 + +:::note é‡è¦æ示： + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +::: + +将以下文件的副本ä¿å­˜åœ¨å®‰å…¨ä½ç½®ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_cluster.yml`：集群的 [Kubeconfig 文件](https://rancher.com/docs/rke/latest/en/kubeconfig/)。该文件包å«å¯å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚ +- `rancher-cluster.rkestate`：[Kubernetes 集群状æ€æ–‡ä»¶](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state)。该文件包å«é›†ç¾¤çš„当å‰çŠ¶æ€ï¼ŒåŒ…括 RKE é…置以åŠè¯ä¹¦
。
_Kubernetes 集群状æ€æ–‡ä»¶ä»…在使用 RKE 0.2.0 或更高版本时创建。_ + + + + +:::note + +åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +::: + +### 故障排除 + +å‚è§[故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)页é¢ã€‚ + +### åŽç»­æ“作 +[安装 Rancher](install-rancher-ha.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md new file mode 100644 index 00000000000..0aa220051f7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md @@ -0,0 +1,246 @@ +--- +title: 4. 安装 Rancher +--- + +本文介ç»å¦‚何在高å¯ç”¨ Kubernetes 安装的离线环境部署 Rancher。离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +### Rancher 特æƒè®¿é—® + +当 Rancher Server 部署在 Docker 容器中时，容器内会安装一个本地 Kubernetes 集群供 Rancher 使用。为 Rancher 的很多功能都是以 deployment çš„æ–¹å¼è¿è¡Œçš„，而在容器内è¿è¡Œå®¹å™¨æ˜¯éœ€è¦ç‰¹æƒæ¨¡å¼çš„，因此你需è¦åœ¨å®‰è£… Rancher 时添加 `--privileged` 选项。 + +## Docker 说明 + +如果你想使用 Docker 命令进行离线安装，请跳过本页的剩余部分，并按照[此页](docker-install-commands.md)进行æ“作。 + +## Kubernetes 说明 + +我们建议在 Kubernetes 集群上安装 Rancher。高å¯ç”¨çš„ Kubernetes 安装的情况下，一个 Kubernetes 集群包å«ä¸‰ä¸ªè¿è¡Œ Rancher Server 组件的节点。æŒä¹…层（etcd）也在这三个节点上进行å¤åˆ¶ï¼Œä»¥ä¾¿åœ¨å…¶ä¸­ä¸€ä¸ªèŠ‚点å‘生故障时æ供冗余和数æ®å¤åˆ¶ã€‚ + +### 1. 添加 Helm Chart 仓库 + +从å¯ä»¥è®¿é—®äº’è”网的系统中，获å–最新的 Helm Chart，然åŽå°† manifest å¤åˆ¶åˆ°å¯è®¿é—® Rancher Server 集群的系统中。 + +1. 如果你还没有安装 `helm`，请在å¯è®¿é—®äº’è”网的工作站上进行本地安装。注æ„：å‚考 [Helm 版本è¦æ±‚](../../resources/helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。 + +2. 执行 `helm repo add` 命令，以添加包å«å®‰è£… Rancher çš„ Chart çš„ Helm Chart 仓库。有关如何选择仓库，以åŠå“ªä¸ªä»“库最适åˆä½ çš„用例，请å‚è§[选择 Rancher 版本](../../resources/choose-a-rancher-version.md)。 + - Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + +3. 获å–最新的 Rancher Chart。此æ“ä½œå°†èŽ·å– Chart 并将其作为 `.tgz` 文件ä¿å­˜åœ¨å½“å‰ç›®å½•ä¸­ã€‚ + ```plain + helm fetch rancher-/rancher + ``` + + 如需下载特定的 Rancher 版本，你å¯ä»¥ç”¨ Helm `--version` å‚数指定版本，如下： + ```plain + helm fetch rancher-stable/rancher --version=v2.4.8 + ``` + +### 2. 选择 SSL é…ç½® + +Rancher Server é»˜è®¤è®¾è®¡ä¸ºå®‰å…¨çš„ï¼Œå¹¶ä¸”éœ€è¦ SSL/TLS é…置。 + +如果你在离线的 Kubernetes 集群中安装 Rancher，我们建议使用以下两ç§è¯ä¹¦ç”Ÿæˆæ–¹å¼ã€‚ + +:::note + +如果你想在外部终止 SSL/TLS，请å‚è§[外部负载å‡è¡¡å™¨çš„ TLS 终止](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +::: + +| é…ç½® | Chart 选项 | æè¿° | 是å¦éœ€è¦ cert-manager | +| ------------------------------------------ | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| Rancher 生æˆçš„自签åè¯ä¹¦ | `ingress.tls.source=rancher` | 使用 Rancher 生æˆçš„ CA ç­¾å‘的自签åè¯ä¹¦ã€‚此项是**默认选项**。在渲染 Helm 模æ¿çš„时候ä¸éœ€è¦ä¼ é€’此项。 | 是 | +| 你已有的è¯ä¹¦ | `ingress.tls.source=secret` | 通过创建 Kubernetes 密文使用你自己的è¯ä¹¦æ–‡ä»¶ã€‚
在渲染 Rancher Helm 模æ¿æ—¶å¿…须传递此选项。 | å¦ | + +### 离线安装的 Helm Chart 选项 + +在é…ç½® Rancher Helm 模æ¿æ—¶ï¼ŒHelm Chart 中有几个专为离线安装设计的选项，如下表： + +| Chart 选项 | Chart 值 | æè¿° | +| ----------------------- | -------------------------------- | ---- | +| `certmanager.version` | `` | æ ¹æ®è¿è¡Œçš„ cert-manager 版本é…置适当的 Rancher TLS é¢å‘者。 | +| `systemDefaultRegistry` | `` | å°† Rancher Server é…ç½®æˆåœ¨é…置集群时，始终从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚ | +| `useBundledSystemChart` | `true` | é…ç½® Rancher Server 使用打包的 Helm System Chart 副本。[system charts](https://github.com/rancher/system-charts) 仓库包å«æ‰€æœ‰ Monitoring，Logging，告警和全局 DNS 等功能所需的应用商店项目。这些 [Helm Chart](https://github.com/rancher/system-charts) ä½äºŽ GitHub 中。但是由于你处在离线环境，因此使用 Rancher 内置的 Chart 会比设置 Git mirror 容易得多。 | + +### 3. èŽ·å– Cert-Manager Chart + +æ ¹æ®ä½ åœ¨[2：选择 SSL é…ç½®](#2-选择-ssl-é…ç½®)中的选择，完æˆä»¥ä¸‹æ­¥éª¤ä¹‹ä¸€ï¼š + +#### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +默认情况下，Rancher 会生æˆä¸€ä¸ª CA 并使用 cert-manager é¢å‘è¯ä¹¦ä»¥è®¿é—® Rancher Server ç•Œé¢ã€‚ + +:::note + +由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.11.0 çš„ cert-manager 版本，请å‚è§ [cert-manager å‡çº§æ–‡æ¡£](../../resources/upgrade-cert-manager.md)。 + +::: + +##### 1. 添加 cert-manager 仓库 + +在å¯ä»¥è¿žæŽ¥äº’è”网的系统中，将 cert-manager 仓库添加到 Helm： + +```plain +helm repo add jetstack https://charts.jetstack.io +helm repo update +``` + +##### 2. èŽ·å– cert-manager Chart + +从 [Helm Chart 仓库](https://artifacthub.io/packages/helm/cert-manager/cert-manager)中获å–最新å¯ç”¨çš„ cert-manager Chart： + +```plain +helm fetch jetstack/cert-manager --version v1.11.0 +``` + +##### 3. 检索 Cert-Manager CRD + +为 cert-manager 下载所需的 CRD 文件： +```plain +curl -L -o cert-manager-crd.yaml https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml +``` + +### 4. 安装 Rancher + +将获å–çš„ Chart å¤åˆ¶åˆ°æœ‰æƒè®¿é—® Rancher Server 集群的系统以完æˆå®‰è£…。 + +##### 1. 安装 Cert-Manager + +使用è¦ç”¨äºŽå®‰è£… Chart 的选项æ¥å®‰è£… cert-manager。记ä½è¦è®¾ç½® `image.repository` 选项，以从你的ç§æœ‰é•œåƒä»“库拉å–é•œåƒã€‚æ­¤æ“ä½œä¼šåˆ›å»ºä¸€ä¸ªåŒ…å« Kubernetes manifest 文件的 `cert-manager` 目录。 + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +
+ å•å‡»å±•å¼€ + +如果你使用自签åè¯ä¹¦ï¼Œå®‰è£… cert-manager： + +1. 为 cert-manager 创建命å空间： + + ```plain + kubectl create namespace cert-manager + ``` + +2. 创建 cert-manager CustomResourceDefinition (CRD)。 + + ```plain + kubectl apply -f cert-manager/cert-manager-crd.yaml + ``` + +3. 安装 cert-manager。 + + ```plain + helm install cert-manager ./cert-manager-v1.11.0.tgz \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller \ + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook \ + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector \ + --set startupapicheck.image.repository=/quay.io/jetstack/cert-manager-ctl + ``` + +
+ +##### 2. 安装 Rancher +首先，å‚è§[添加 TLS 密文](../../resources/add-tls-secrets.md)å‘布è¯ä¹¦æ–‡ä»¶ï¼Œä»¥ä¾¿ Rancher å’Œ Ingress Controller å¯ä»¥ä½¿ç”¨å®ƒä»¬ã€‚ + +然åŽï¼Œä½¿ç”¨ kubectl 为 Rancher 创建命å空间： + +```plain +kubectl create namespace cattle-system +``` + +然åŽå®‰è£… Rancher，并声明你选择的选项。å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚Rancher 需è¦é…置为使用ç§æœ‰é•œåƒä»“库，以便é…置所有 Rancher å¯åŠ¨çš„ Kubernetes 集群或 Rancher 工具。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 输出压缩包的版本å·ã€‚ | +| `` | 指å‘è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 | +| `` | ä½ çš„ç§æœ‰é•œåƒä»“库的 DNS å称。 | +| `` | 在 K8s 集群上è¿è¡Œçš„ cert-manager 版本。 | + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart +``` + +**å¯é€‰**：如需安装特定的 Rancher 版本，设置`rancherImageTag` 的值，例如：`--set rancherImageTag=v2.5.8` + +#### 选项 B：使用 Kubernetes 密文从文件中获å–è¯ä¹¦ + +##### 1. 创建密文 + +使用你自己的è¯ä¹¦æ¥åˆ›å»º Kubernetes 密文，以供 Rancher 使用。è¯ä¹¦çš„ common name 需è¦ä¸Žä»¥ä¸‹å‘½ä»¤ä¸­çš„ `hostname` 选项匹é…，å¦åˆ™ Ingress Controller 将无法为 Rancher é…置站点。 + +##### 2. 安装 Rancher + +安装 Rancher，并声明你选择的选项。å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚Rancher 需è¦é…置为使用ç§æœ‰é•œåƒä»“库，以便é…置所有 Rancher å¯åŠ¨çš„ Kubernetes 集群或 Rancher 工具。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------- | +| `` | 输出压缩包的版本å·ã€‚ | +| `` | 指å‘è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 | +| `` | ä½ çš„ç§æœ‰é•œåƒä»“库的 DNS å称。 | + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart +``` + +如果你使用的是ç§æœ‰ CA ç­¾åçš„è¯ä¹¦ï¼Œè¯·åœ¨ `--set ingress.tls.source=secret` åŽåŠ ä¸Š `--set privateCA=true`： + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart +``` + + +安装已完æˆã€‚ +:::caution + +如果你ä¸æƒ³å‘é€é¥æµ‹æ•°æ®ï¼Œåœ¨é¦–次登录时退出[é¥æµ‹](../../../../faq/telemetry.md)。如果在离线安装的环境中让这个功能处于 active 状æ€ï¼Œsocket å¯èƒ½æ— æ³•æ‰“开。 + +::: + +## å…¶ä»–èµ„æº + +以下资æºå¯èƒ½å¯¹å®‰è£… Rancher 有帮助： + +- [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md) +- [添加 TLS 密文](../../resources/add-tls-secrets.md) +- [Rancher Kubernetes 安装的故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md new file mode 100644 index 00000000000..c64f7132d1d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md @@ -0,0 +1,305 @@ +--- +title: '2. 收集镜åƒå¹¶å‘布到ç§æœ‰ä»“库' +--- + +本文介ç»å¦‚何é…ç½®ç§æœ‰é•œåƒä»“库，以便在安装 Rancher 时，Rancher å¯ä»¥ä»Žæ­¤ç§æœ‰é•œåƒä»“库中拉å–所需的镜åƒã€‚ + +默认情况下，所有用于[é…ç½® Kubernetes 集群](../../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)或å¯åŠ¨ Rancher 中的工具（如监控，æµæ°´çº¿ï¼Œå‘Šè­¦ç­‰ï¼‰çš„é•œåƒéƒ½æ˜¯ä»Ž Docker Hub 中拉å–的。在 Rancher 的离线安装中，你需è¦ä¸€ä¸ªç§æœ‰ä»“库，该仓库ä½äºŽä½ çš„ Rancher Server 中æŸä¸ªå¯è®¿é—®çš„ä½ç½®ã€‚然åŽï¼Œä½ å¯åŠ è½½è¯¥å­˜æœ‰æ‰€æœ‰é•œåƒçš„é•œåƒä»“库。 + +使用 Docker 安装 Rancher，和把 Rancher 安装到 Kubernetes 集群，其对应的推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库步骤是一样的。 + +你使用 Rancher é…置的下游集群是å¦æœ‰è¿è¡Œ Windows 的节点，决定了本文涉åŠçš„步骤。我们æ供的推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库步骤，是基于å‡è®¾ Rancher ä»…é…ç½®è¿è¡Œ Linux 节点的下游 Kubernetes 集群的。但是，如果你计划[在下游 Kubernetes 集群中使用 Windows 节点](../../../../pages-for-subheaders/use-windows-clusters.md)，我们有å•ç‹¬çš„文档æ¥ä»‹ç»å¦‚何为需è¦çš„é•œåƒæ供支æŒã€‚ + +:::note 先决æ¡ä»¶ï¼š + +你必须有一个å¯ç”¨çš„[ç§æœ‰é•œåƒä»“库](https://docs.docker.com/registry/deploying/#run-an-externally-accessible-registry)。 + +如果镜åƒä»“库有è¯ä¹¦ï¼Œè¯·å‚è§ [K3s 文档中心](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)了解添加ç§æœ‰é•œåƒä»“库的详情。è¯ä¹¦å’Œé•œåƒä»“库é…置文件å‡éœ€è¦æŒ‚载到 Rancher 容器中。 + +::: + + + + +如果 Rancher Server 用于é…置仅有 Linux 节点的集群，请按以下步骤将你的镜åƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库。 + +1. [找到你的 Rancher 版本所需的资æº](#1-找到你的-rancher-版本所需的资æº) +2. [收集 cert-manager é•œåƒ](#2-收集-cert-manager-é•œåƒ)（除éžä½ ä½¿ç”¨è‡ªå·±çš„è¯ä¹¦ï¼Œæˆ–在负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ TLS） +3. [把镜åƒä¿å­˜åˆ°ä½ çš„工作站](#3-将镜åƒä¿å­˜åˆ°ä½ çš„工作站中) +4. [推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库](#4-推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库) + +### 先决æ¡ä»¶ + +这些步骤è¦æ±‚你使用一个 Linux 工作站，该工作站需è¦å¯è®¿é—®äº’è”网和你的ç§æœ‰é•œåƒä»“库，且至少有 20GB çš„å¯ç”¨ç£ç›˜ç©ºé—´ã€‚ + +如果你的主机架构是 ARM64，镜åƒä»“åº“å¿…é¡»æ”¯æŒ Manifest。这是因为从 2020 å¹´ 4 月开始, Amazon Elastic Container Registry å·²ç»ä¸å†æ”¯æŒ Manifest。 + +### 1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº + +1. 访问 Rancher çš„[å‘布说明](https://github.com/rancher/rancher/releases)页é¢ï¼Œæ‰¾åˆ°ä½ éœ€è¦å®‰è£…çš„ Rancher v2.x.x 版本，然åŽç‚¹å‡» **Assets**。注æ„ä¸è¦ä½¿ç”¨å¸¦æœ‰ `rc` 或 `Pre-release` 标记的版本，因为这些版本在生产环境中ä¸å¤Ÿç¨³å®šã€‚ + +2. 从你所需版本的 **Assets** 处下载以下文件，这些文件是在离线环境中安装 Rancher 所必须的： + +| Release 文件 | æè¿° | +| ---------------- | -------------- | +| `rancher-images.txt` | 此文件包å«å®‰è£… Rancherã€é…置集群和用户 Rancher 工具所需的镜åƒã€‚ | +| `rancher-save-images.sh` | 该脚本从 Docker Hub 中拉å–在文件 `rancher-images.txt` 中记录的所有镜åƒï¼Œå¹¶æŠŠè¿™äº›é•œåƒä¿å­˜ä¸º `rancher-images.tar.gz`。 | +| `rancher-load-images.sh` | 该脚本从 `rancher-images.tar.gz` 文件中加载镜åƒï¼Œå¹¶æŠŠé•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。 | + +### 2. 收集 cert-manager é•œåƒ + +:::note + +如果你使用自己的è¯ä¹¦ï¼Œæˆ–è¦åœ¨å¤–部负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ TLS，请跳过此步骤。 + +::: + +在 Kubernetes 安装中，如果你使用的是 Rancher 默认的自签å TLS è¯ä¹¦ï¼Œåˆ™å¿…须将 [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) é•œåƒæ·»åŠ åˆ° `rancher-images.txt` 文件中。 + +1. 获å–最新的 `cert-manager` Helm Chart，并解æžæ¨¡æ¿ä»¥èŽ·å–é•œåƒçš„详情信æ¯ï¼š + + :::note + + 由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.12.0 çš„ cert-manager 版本，请å‚è§[å‡çº§æ–‡æ¡£](../../resources/upgrade-cert-manager.md)。 + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. 对镜åƒåˆ—表进行排åºå’Œå”¯ä¸€åŒ–，以去除é‡å¤çš„é•œåƒæºï¼š + + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + +### 3. 将镜åƒä¿å­˜åˆ°ä½ çš„工作站中 + +1. 为 `rancher-save-images.sh` 文件添加å¯æ‰§è¡Œæƒé™ï¼š + ``` + chmod +x rancher-save-images.sh + ``` + +1. 使用 `rancher-images.txt` é•œåƒåˆ—表执行 `rancher-save-images.sh` 脚本，以创建包å«æ‰€æœ‰æ‰€éœ€é•œåƒçš„压缩包： + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + **结果**：Docker 开始拉å–用于离线安装的镜åƒã€‚请è€å¿ƒç­‰å¾…。这个过程需è¦å‡ åˆ†é’Ÿã€‚完æˆæ—¶ï¼Œä½ çš„当å‰ç›®å½•ä¼šè¾“出å为 `rancher-images.tar.gz` 的压缩包。请确认输出文件是å¦å­˜åœ¨ã€‚ + +### 4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +下一步，你将使用脚本将 `rancher-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +使用脚本将 `rancher-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +`rancher-images.txt` 需è¦ä½äºŽå·¥ä½œç«™ä¸­è¿è¡Œ `rancher-load-images.sh` 脚本的åŒä¸€ç›®å½•ä¸­ã€‚`rancher-images.tar.gz` 也需è¦ä½äºŽåŒä¸€ç›®å½•ä¸­ã€‚ + +1. 登录到你的ç§æœ‰é•œåƒä»“库： +```plain +docker login +``` +1. 为 `rancher-load-images.sh` 添加å¯æ‰§è¡Œæƒé™ï¼š +``` +chmod +x rancher-load-images.sh +``` + +1. 使用 `rancher-load-images.sh` 脚本æ¥æå–ï¼Œæ ‡è®°å’ŒæŽ¨é€ `rancher-images.txt` åŠ `rancher-images.tar.gz` 到你的ç§æœ‰é•œåƒä»“库： +```plain +./rancher-load-images.sh --image-list ./rancher-images.txt --registry +``` + + + + +如果你的 Rancher Server 将用于é…ç½® Linux å’Œ Windows 集群，你需è¦æ‰§è¡Œä¸åŒçš„步骤，æ¥å°† Windows é•œåƒå’Œ Linux é•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。由于 Windows 集群åŒæ—¶åŒ…å« Linux å’Œ Windows 节点，因此推é€åˆ°ç§æœ‰é•œåƒä»“库的 Linux é•œåƒæ˜¯ Manifest。 + +## Windows 步骤 + +从 Windows Server å·¥ä½œç«™ä¸­æ”¶é›†å’ŒæŽ¨é€ Windows é•œåƒã€‚ + +1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº +2. 将镜åƒä¿å­˜åˆ°ä½ çš„ Windows Server 工作站 +3. 准备 Docker daemon +4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +### 先决æ¡ä»¶ + +以下步骤å‡è®¾ä½ ä½¿ç”¨ Windows Server 1809 工作站，该工作站能访问网络åŠä½ çš„ç§æœ‰é•œåƒä»“库，且至少拥有 50GB çš„ç£ç›˜ç©ºé—´ã€‚ + +工作站必须安装 Docker 18.02+ 版本以æä¾› manifest 支æŒã€‚Manifest 支æŒæ˜¯é…ç½® Windows 集群所必须的。 + +ä½ çš„é•œåƒä»“åº“å¿…é¡»æ”¯æŒ Manifest。这是因为从 2020 å¹´ 4 月开始, Amazon Elastic Container Registry å·²ç»ä¸å†æ”¯æŒ Manifest。 + + + +### 1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº + +1. 访问 Rancher çš„[å‘布说明](https://github.com/rancher/rancher/releases)页é¢ï¼Œæ‰¾åˆ°ä½ éœ€è¦å®‰è£…çš„ Rancher v2.x.x 版本。ä¸è¦ä¸‹è½½å¸¦æœ‰ `rc` 或 `Pre-release` 标记的版本，因为这些版本在生产环境中ä¸å¤Ÿç¨³å®šã€‚ + +2. 从你所需版本的 **Assets** 处下载以下文件： + +| Release 文件 | æè¿° | +|----------------------------|------------------| +| `rancher-windows-images.txt` | 此文件包å«é…ç½® Windows 集群所需的 Windows é•œåƒã€‚ | +| `rancher-save-images.ps1` | 该脚本从 Docker Hub 中拉å–在文件 `rancher-windows-images.txt` 中记录的所有镜åƒï¼Œå¹¶æŠŠè¿™äº›é•œåƒä¿å­˜ä¸º `rancher-windows-images.tar.gz`。 | +| `rancher-load-images.ps1` | 该脚本从 `rancher-windows-images.tar.gz` 文件中加载镜åƒï¼Œå¹¶æŠŠé•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。 | + + + +### 2. 将镜åƒä¿å­˜åˆ°ä½ çš„ Windows Server 工作站 + +1. 在 `powershell` 中，进入上一步下载的文件所在的目录。 + +1. è¿è¡Œ `rancher-save-images.ps1` 以创建包å«æ‰€æœ‰æ‰€éœ€é•œåƒçš„压缩包： + ```plain + ./rancher-save-images.ps1 + ``` + + **结果**：Docker 开始拉å–用于离线安装的镜åƒã€‚请è€å¿ƒç­‰å¾…。这个过程需è¦å‡ åˆ†é’Ÿã€‚完æˆæ—¶ï¼Œä½ çš„当å‰ç›®å½•ä¼šè¾“出å为 `rancher-windows-images.tar.gz` 的压缩包。请确认输出文件是å¦å­˜åœ¨ã€‚ + + + +### 3. 准备 Docker daemon + +将你的ç§æœ‰é•œåƒä»“库地å€å°¾é™„到 Docker daemon (`C:\\ProgramData\\Docker\\config\\daemon.json`) çš„ `allow-nondistributable-artifacts` é…置字段中。Windows é•œåƒçš„基础镜åƒæ˜¯ç”± `mcr.microsoft.com` é•œåƒä»“库维护的，而 Docker Hub 中缺少 Microsoft é•œåƒä»“库层，且需è¦å°†å…¶æ‹‰å…¥ç§æœ‰é•œåƒä»“库，因此这一步骤是必须的。 + +```json +{ + ... + "allow-nondistributable-artifacts": [ + ... + "" + ] + ... +} +``` + + + +### 4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +使用脚本将 `rancher-windows-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +`rancher-windows-images.txt` 需è¦ä½äºŽå·¥ä½œç«™ä¸­è¿è¡Œ `rancher-load-images.ps1` 脚本的åŒä¸€ç›®å½•ä¸­ã€‚`rancher-windows-images.tar.gz` 也需è¦ä½äºŽåŒä¸€ç›®å½•ä¸­ã€‚ + +1. 使用 `powershell` 登录到你的ç§æœ‰é•œåƒä»“库： + ```plain + docker login + ``` + +1. 在 `powershell` 中，使用 `rancher-load-images.ps1` 脚本æ¥æå–ï¼Œæ ‡è®°å’ŒæŽ¨é€ `rancher-images.tar.gz` 中的镜åƒåˆ°ä½ çš„ç§æœ‰é•œåƒä»“库： + ```plain + ./rancher-load-images.ps1 --registry + ``` + +## Linux 步骤 + +Linux é•œåƒéœ€è¦åœ¨ Linux 主机上收集和推é€ï¼Œä½†æ˜¯ä½ å¿…须先将 Windows é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库，然åŽå†æŽ¨é€ Linux é•œåƒã€‚由于被推é€çš„ Linux é•œåƒå®žé™…ä¸Šæ˜¯æ”¯æŒ Windows å’Œ Linux é•œåƒçš„ manifest，因此涉åŠçš„步骤ä¸åŒäºŽåªåŒ…å« Linux 节点的集群。 + +1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº +2. 收集所有需è¦çš„é•œåƒ +3. 将镜åƒä¿å­˜åˆ°ä½ çš„ Linux 工作站中 +4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +### 先决æ¡ä»¶ + +在将 Linux é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库之å‰ï¼Œä½ å¿…须先把 Windows é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库。如果你已ç»æŠŠ Linux é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库，则需è¦å†æ¬¡æŒ‰ç…§è¯´æ˜Žé‡æ–°æŽ¨é€ï¼Œå› ä¸ºå®ƒä»¬éœ€è¦å‘å¸ƒæ”¯æŒ Windows å’Œ Linux é•œåƒçš„ manifest。 + +这些步骤è¦æ±‚你使用一个 Linux 工作站，该工作站需è¦å¯è®¿é—®äº’è”网和你的ç§æœ‰é•œåƒä»“库，且至少有 20GB çš„å¯ç”¨ç£ç›˜ç©ºé—´ã€‚ + +工作站必须安装 Docker 18.02+ 版本以æä¾› manifest 支æŒã€‚Manifest 支æŒæ˜¯é…ç½® Windows 集群所必须的。 + + + +### 1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº + +1. 访问 Rancher çš„[å‘布说明](https://github.com/rancher/rancher/releases)页é¢ï¼Œæ‰¾åˆ°ä½ éœ€è¦å®‰è£…çš„ Rancher v2.x.x 版本。ä¸è¦ä¸‹è½½å¸¦æœ‰ `rc` 或 `Pre-release` 标记的版本，因为这些版本在生产环境中ä¸å¤Ÿç¨³å®šã€‚点击 **Assets**。 + +2. 从你所需版本的 **Assets** 处下载以下文件： + +| Release 文件 | æè¿° | +|----------------------------| -------------------------- | +| `rancher-images.txt` | 此文件包å«å®‰è£… Rancherã€é…置集群和用户 Rancher 工具所需的镜åƒã€‚ | +| `rancher-windows-images.txt` | 此文件包å«é…ç½® Windows 集群所需的镜åƒã€‚ | +| `rancher-save-images.sh` | 该脚本从 Docker Hub 中拉å–在文件 `rancher-images.txt` 中记录的所有镜åƒï¼Œå¹¶æŠŠè¿™äº›é•œåƒä¿å­˜ä¸º `rancher-images.tar.gz`。 | +| `rancher-load-images.sh` | 该脚本从 `rancher-images.tar.gz` 文件中加载镜åƒï¼Œå¹¶æŠŠé•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。 | + + + +### 2. 收集所有需è¦çš„é•œåƒ + +**在 Kubernetes 安装中，如果你使用的是 Rancher 默认的自签å TLS è¯ä¹¦**，则必须将 [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) é•œåƒæ·»åŠ åˆ° `rancher-images.txt` 文件中。如果你使用自己的è¯ä¹¦ï¼Œåˆ™å¯è·³è¿‡æ­¤æ­¥éª¤ã€‚ + + +1. 获å–最新的 `cert-manager` Helm Chart，并解æžæ¨¡æ¿ä»¥èŽ·å–é•œåƒçš„详情信æ¯ï¼š + + :::note + + 由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.12.0 çš„ cert-manager 版本，请å‚è§[å‡çº§æ–‡æ¡£](../../resources/upgrade-cert-manager.md)。 + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. 对镜åƒåˆ—表进行排åºå’Œå”¯ä¸€åŒ–，以去除é‡å¤çš„é•œåƒæºï¼š + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + + + +### 3. 将镜åƒä¿å­˜åˆ°ä½ çš„工作站中 + +1. 为 `rancher-save-images.sh` 文件添加å¯æ‰§è¡Œæƒé™ï¼š + ``` + chmod +x rancher-save-images.sh + ``` + +1. 使用 `rancher-images.txt` é•œåƒåˆ—表执行 `rancher-save-images.sh` 脚本，以创建包å«æ‰€æœ‰æ‰€éœ€é•œåƒçš„压缩包： + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + +**结果**：Docker 开始拉å–用于离线安装的镜åƒã€‚请è€å¿ƒç­‰å¾…。这个过程需è¦å‡ åˆ†é’Ÿã€‚完æˆæ—¶ï¼Œä½ çš„当å‰ç›®å½•ä¼šè¾“出å为 `rancher-images.tar.gz` 的压缩包。请确认输出文件是å¦å­˜åœ¨ã€‚ + + + +### 4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +使用 `rancher-load-images.sh script` 脚本将 `rancher-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +é•œåƒåˆ—è¡¨ï¼Œå³ `rancher-images.txt` 或 `rancher-windows-images.txt` 需è¦ä½äºŽå·¥ä½œç«™ä¸­è¿è¡Œ `rancher-load-images.sh` 脚本的åŒä¸€ç›®å½•ä¸­ã€‚`rancher-images.tar.gz` 也需è¦ä½äºŽåŒä¸€ç›®å½•ä¸­ã€‚ + +1. 登录到你的ç§æœ‰é•œåƒä»“库： + ```plain + docker login + ``` + +1. 为 `rancher-load-images.sh` 添加å¯æ‰§è¡Œæƒé™ï¼š + ``` + chmod +x rancher-load-images.sh + ``` + +1. 使用 `rancher-load-images.sh` 脚本æ¥æå–ï¼Œæ ‡è®°å’ŒæŽ¨é€ `rancher-images.tar.gz` 中的镜åƒåˆ°ä½ çš„ç§æœ‰é•œåƒä»“库： + +```plain +./rancher-load-images.sh --image-list ./rancher-images.txt \ + --windows-image-list ./rancher-windows-images.txt \ + --registry +``` + + + + +### [Kubernetes 安装的åŽç»­æ­¥éª¤ - å¯åŠ¨ Kubernetes 集群](install-kubernetes.md) + +### [Docker 安装的åŽç»­æ­¥éª¤ - 安装 Rancher](install-rancher-ha.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md new file mode 100644 index 00000000000..d3628dea33f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md @@ -0,0 +1,249 @@ +--- +title: '2. 安装 Kubernetes' +--- + +基础设施é…置好åŽï¼Œä½ å¯ä»¥è®¾ç½®ä¸€ä¸ª Kubernetes 集群æ¥å®‰è£… Rancher。 + +设置 RKEã€RKE2 或 K3s 的步骤如下所示。 + +为方便起è§ï¼Œå°†ä»£ç†çš„ IP 地å€å’Œç«¯å£å¯¼å‡ºåˆ°ä¸€ä¸ªçŽ¯å¢ƒå˜é‡ä¸­ï¼Œå¹¶åœ¨æ¯ä¸ªèŠ‚点上为你当å‰çš„ shell 设置 HTTP_PROXY å˜é‡ï¼š + +``` +export proxy_host="10.0.0.5:8888" +export HTTP_PROXY=http://${proxy_host} +export HTTPS_PROXY=http://${proxy_host} +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16 +``` + + + + +首先在 K3s systemd æœåŠ¡ä¸Šé…ç½® HTTP 代ç†è®¾ç½®ï¼Œè®© K3s çš„ containerd å¯ä»¥é€šè¿‡ä»£ç†æ‹‰å–é•œåƒï¼š + +``` +cat <<'EOF' | sudo tee /etc/default/k3s > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +在第一个节点上，创建一个新集群： +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --cluster-init +``` + +然åŽåŠ å…¥å…¶ä»–节点： +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --server https://:6443 +``` + +其中 `` 是 Server çš„ IP 或有效 DNS，`` 是å¯ä»¥åœ¨ `/var/lib/rancher/k3s/server/node-token` 中找到的 Server node-token。 + +有关安装 K3s 的更多信æ¯ï¼Œè¯·å‚阅 [K3s 安装文档](https://docs.k3s.io/installation)。 + +如需查看集群，请è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +在æ¯ä¸ªèŠ‚点上，è¿è¡Œ RKE2 安装脚本。确ä¿ä½ å®‰è£…çš„ RKE2 ç‰ˆæœ¬å— [Rancher 支æŒ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +``` +curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.xx sh - +``` + +然åŽï¼Œä½ å¿…须在 RKE2 systemd æœåŠ¡ä¸Šé…ç½® HTTP 代ç†è®¾ç½®ï¼Œè®© RKE2 çš„ containerd å¯ä»¥é€šè¿‡ä»£ç†æ‹‰å–é•œåƒï¼š + +``` +cat <<'EOF' | sudo tee /etc/default/rke2-server > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +接下æ¥ï¼ŒæŒ‰ç…§ [RKE2 高å¯ç”¨æ€§æ–‡æ¡£](https://docs.rke2.io/install/ha)在æ¯ä¸ªèŠ‚点上创建 RKE2 é…置文件。 + +之åŽå¯åŠ¨å¹¶å¯ç”¨ `rke2-server` æœåŠ¡ï¼š + +``` +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` + +有关安装 RKE2 的更多信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io)。 + +如需查看集群，请è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +alias kubectl=/var/lib/rancher/rke2/bin/kubectl +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +首先，你需è¦åœ¨æ‰€æœ‰ä¸‰ä¸ª Linux 节点上安装 Docker 并设置 HTTP 代ç†ã€‚因此，你å¯ä»¥åœ¨è¿™ä¸‰ä¸ªèŠ‚点上执行以下步骤。 + +接下æ¥é…ç½® apt 以在安装包时使用这个代ç†ã€‚如果你使用的ä¸æ˜¯ Ubuntu，请相应调整步骤。 + +``` +cat <<'EOF' | sudo tee /etc/apt/apt.conf.d/proxy.conf > /dev/null +Acquire::http::Proxy "http://${proxy_host}/"; +Acquire::https::Proxy "http://${proxy_host}/"; +EOF +``` + +安装 Docker： + +``` +curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh +``` + +然åŽï¼Œç¡®ä¿ä½ çš„当å‰ç”¨æˆ·èƒ½å¤Ÿåœ¨æ²¡æœ‰ sudo 的情况下访问 Docker Daemon： + +``` +sudo usermod -aG docker YOUR_USERNAME +``` + +é…ç½® Docker Daemon 使用代ç†æ¥æ‹‰å–é•œåƒï¼š + +``` +sudo mkdir -p /etc/systemd/system/docker.service.d +cat <<'EOF' | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf > /dev/null +[Service] +Environment="HTTP_PROXY=http://${proxy_host}" +Environment="HTTPS_PROXY=http://${proxy_host}" +Environment="NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16" +EOF +``` + +è¦åº”用é…置，请é‡æ–°å¯åŠ¨ Docker Daemon： + +``` +sudo systemctl daemon-reload +sudo systemctl restart docker +``` + +#### ç¦»çº¿ä»£ç† + +你现在å¯ä»¥åœ¨é…置的离线集群中é…置主机驱动集群，以使用代ç†è¿›è¡Œå‡ºç«™è¿žæŽ¥ã€‚ + +除了为代ç†æœåŠ¡å™¨è®¾ç½®é»˜è®¤è§„则外，你还需è¦é¢å¤–添加如下所示的规则，以从代ç†çš„ Rancher 环境中é…置主机驱动集群。 + +æ ¹æ®ä½ çš„设置é…置文件路径，例如 `/etc/apt/apt.conf.d/proxy.conf`： + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` + +### 创建 RKE 集群 + +在能通过 SSH 访问 Linux 节点的主机上，你需è¦æœ‰å‡ ä¸ªå‘½ä»¤è¡Œå·¥å…·ï¼Œæ¥åˆ›å»ºé›†ç¾¤å¹¶ä¸Žä¹‹äº¤äº’： + +* [RKE CLI binary](https://rancher.com/docs/rke/latest/en/installation/#download-the-rke-binary) + +``` +sudo curl -fsSL -o /usr/local/bin/rke https://github.com/rancher/rke/releases/download/v1.1.4/rke_linux-amd64 +sudo chmod +x /usr/local/bin/rke +``` + +* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + +``` +curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +``` + +接下æ¥ï¼Œåˆ›å»ºä¸€ä¸ªæè¿° RKE 集群的 YAML 文件。确ä¿èŠ‚点的 IP 地å€å’Œ SSH 用户å是正确的。有关集群 YAML 的详情，请å‚è§ [RKE 官方文档](https://rancher.com/docs/rke/latest/en/example-yamls/)。 + +```yml +nodes: + - address: 10.0.1.200 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.201 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.202 + user: ubuntu + role: [controlplane,worker,etcd] + +services: + etcd: + backup_config: + interval_hours: 12 + retention: 6 +``` + +之åŽï¼Œä½ å¯ä»¥é€šè¿‡è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ›å»º Kubernetes 集群： + +``` +rke up --config rancher-cluster.yaml +``` + +RKE 会创建一个å为 `rancher-cluster.rkestate` 的状æ€æ–‡ä»¶ã€‚如果你需è¦æ›´æ–°æˆ–修改集群é…置，或使用备份æ¢å¤é›†ç¾¤ï¼Œåˆ™éœ€è¦ä½¿ç”¨è¯¥æ–‡ä»¶ã€‚RKE 还会创建一个 `kube_config_cluster.yaml` 文件，你å¯ä»¥ä½¿ç”¨è¯¥æ–‡ä»¶åœ¨æœ¬åœ°ä½¿ç”¨ kubectl 或 Helm 等工具连接到远端的 Kubernetes 集群。请将这些文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ï¼Œä¾‹å¦‚版本控制系统中。 + +如需查看集群，请è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +export KUBECONFIG=kube_config_cluster.yaml +kubectl cluster-info +kubectl get pods --all-namespaces +``` + +你也å¯ä»¥éªŒè¯ä½ çš„外部负载å‡è¡¡å™¨æ˜¯å¦å·¥ä½œï¼ŒDNS æ¡ç›®æ˜¯å¦è®¾ç½®æ­£ç¡®ã€‚如果你å‘其中之一å‘é€è¯·æ±‚，你会收到æ¥è‡ª Ingress Controller çš„ HTTP 404 å“应： + +``` +$ curl 10.0.1.100 +default backend - 404 +$ curl rancher.example.com +default backend - 404 +``` + +### ä¿å­˜ä½ çš„文件 + +:::note é‡è¦æ示： + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +::: + +将以下文件的副本ä¿å­˜åœ¨å®‰å…¨ä½ç½®ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_cluster.yml`：集群的 [Kubeconfig 文件](https://rancher.com/docs/rke/latest/en/kubeconfig/)。该文件包å«å¯å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚ +- `rancher-cluster.rkestate`：[Kubernetes 集群状æ€æ–‡ä»¶](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state)。此文件包å«é›†ç¾¤çš„当å‰çŠ¶æ€ï¼ŒåŒ…括 RKE é…置和è¯ä¹¦ã€‚ + +:::note + +åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +::: + + + + +### 故障排除 + +å‚è§[故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)页é¢ã€‚ + +### åŽç»­æ“作 +[安装 Rancher](install-rancher.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md new file mode 100644 index 00000000000..da7bd9fb728 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md @@ -0,0 +1,104 @@ +--- +title: 3. 安装 Rancher +--- + +在å‰æ–‡çš„æ“作åŽï¼Œä½ å·²ç»æœ‰äº†ä¸€ä¸ªè¿è¡Œçš„ RKE 集群，现在å¯ä»¥åœ¨å…¶ä¸­å®‰è£… Rancher 了。出于安全考虑，所有到 Rancher çš„æµé‡éƒ½å¿…须使用 TLS 加密。在本教程中，你将使用 [cert-manager](https://cert-manager.io/)自动é¢å‘自签åè¯ä¹¦ã€‚在实际使用情况下，你å¯ä½¿ç”¨ Let's Encrypt 或自己的è¯ä¹¦ã€‚ + +### 安装 Helm CLI + +在具有 kubeconfig 的主机上安装 [Helm](https://helm.sh/docs/intro/install/) CLI 以访问 Kubernetes 集群： + +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod +x get_helm.sh +sudo ./get_helm.sh +``` + +### 安装 cert-manager + +添加 cert-manager Helm 仓库： + +``` +helm repo add jetstack https://charts.jetstack.io +``` + +为 cert-manager 创建命å空间： + +``` +kubectl create namespace cert-manager +``` + +安装 cert-manager çš„ CustomResourceDefinitions： + +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml +``` + +使用 Helm 安装 cert-manager。请注æ„，cert-manager 还需è¦ä½ é…置代ç†ï¼Œä»¥é˜²å®ƒéœ€è¦ä¸Ž Let's Encrypt 或其他外部è¯ä¹¦é¢å‘商进行通信： + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +``` +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --version v1.11.0 \ + --set http_proxy=http://${proxy_host} \ + --set https_proxy=http://${proxy_host} \ + --set no_proxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +等待 cert-manager 完æˆå¯åŠ¨ï¼š + +``` +kubectl rollout status deployment -n cert-manager cert-manager +kubectl rollout status deployment -n cert-manager cert-manager-webhook +``` + +### 安装 Rancher + +接下æ¥ï¼Œä½ å¯ä»¥å®‰è£… Rancher 了。首先，添加 Helm 仓库： + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +``` + +创建命å空间： + +``` +kubectl create namespace cattle-system +``` + +然åŽä½¿ç”¨ Helm 安装 Rancher：Rancher 也需è¦ä½ é…置代ç†ï¼Œä»¥ä¾¿å®ƒå¯ä»¥ä¸Žå¤–部应用商店通信，或检索 Kubernetes 版本更新元数æ®ï¼š + +``` +helm upgrade --install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.example.com \ + --set proxy=http://${proxy_host} \ + --set noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +等待部署完æˆï¼š + +``` +kubectl rollout status deployment -n cattle-system rancher +``` + +现在，你å¯ä»¥å¯¼èˆªåˆ° `https://rancher.example.com` 并开始使用 Rancher。 + +:::caution + +如果你ä¸æƒ³å‘é€é¥æµ‹æ•°æ®ï¼Œåœ¨é¦–次登录时退出[é¥æµ‹](../../../../faq/telemetry.md)。如果在离线安装的环境中让这个功能处于 active 状æ€ï¼Œsocket å¯èƒ½æ— æ³•æ‰“开。 + +::: + +### å…¶ä»–èµ„æº + +以下资æºå¯èƒ½å¯¹å®‰è£… Rancher 有帮助： + +- [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md) +- [添加 TLS 密文](../../resources/add-tls-secrets.md) +- [Rancher Kubernetes 安装的故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md new file mode 100644 index 00000000000..58414b8ed1d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md @@ -0,0 +1,64 @@ +--- +title: '1. é…置基础设施' +--- + +在本节中，你将为 Rancher Management Server é…置底层基础设施，并使其通过 HTTP 代ç†è®¿é—®äº’è”网。 + +如需在高å¯ç”¨ RKE 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商（例如 Amazon EC2，GCE 或 vSphere）中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将å‰ç«¯æµé‡è½¬å‘到这三个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸæˆ–æ•°æ®ä¸­å¿ƒã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºã€‚ + +### 为什么使用三个节点？ + +在 RKE 集群中，Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。而这个 etcd æ•°æ®åº“在这三个节点上è¿è¡Œã€‚ + +为了选举出大多数 etcd 节点认å¯çš„ etcd 集群 leader，etcd æ•°æ®åº“需è¦å¥‡æ•°ä¸ªèŠ‚点。如果 etcd æ•°æ®åº“无法选出 leader，etcd å¯èƒ½ä¼šå‡ºçŽ°[脑裂（split brain）](https://www.quora.com/What-is-split-brain-in-distributed-systems)的问题，此时你需è¦ä½¿ç”¨å¤‡ä»½æ¢å¤é›†ç¾¤ã€‚如果三个 etcd 节点之一å‘生故障，其余两个节点å¯ä»¥é€‰æ‹©ä¸€ä¸ª leader，因为它们是 etcd 节点总数的大多数部分。 + +### 1. é…ç½® Linux 节点 + +这些主机将通过 HTTP 代ç†è¿žæŽ¥åˆ°äº’è”网。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md)。 + +:::note é‡è¦æ示： + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 3. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 + + +### åŽç»­æ“作 +[é…ç½® Kubernetes 集群](install-kubernetes.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md new file mode 100644 index 00000000000..6e1ac8ee365 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md @@ -0,0 +1,90 @@ +--- +title: è¯ä¹¦æ•…障排除 +--- + +### 如何确定我的è¯ä¹¦æ ¼å¼æ˜¯å¦ä¸º PEM？ + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ç‰¹å¾è¯†åˆ« PEM æ ¼å¼ï¼š + +- 文件开始的标头： + ``` + -----BEGIN CERTIFICATE----- + ``` +- 表头åŽè·Ÿä¸€é•¿ä¸²å­—符。 +- 文件结æŸçš„页脚： + ``` + -----END CERTIFICATE----- + ``` + +PEM è¯ä¹¦ç¤ºä¾‹ï¼š + +``` +----BEGIN CERTIFICATE----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END CERTIFICATE----- +``` + +PEM è¯ä¹¦å¯†é’¥ç¤ºä¾‹ï¼š + +``` +-----BEGIN RSA PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END RSA PRIVATE KEY----- +``` + +如果你的密钥与以下示例类似，请å‚è§[å°† PKCS8 è¯ä¹¦å¯†é’¥è½¬æ¢ä¸º PKCS1](#å°†-pkcs8-è¯ä¹¦å¯†é’¥è½¬æ¢ä¸º-pkcs1)。 + +``` +-----BEGIN PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END PRIVATE KEY----- +``` + +### å°† PKCS8 è¯ä¹¦å¯†é’¥è½¬æ¢ä¸º PKCS1 + +如果你使用的是 PKCS8 è¯ä¹¦å¯†é’¥æ–‡ä»¶ï¼ŒRancher 将打å°ä»¥ä¸‹æ—¥å¿—： + +``` +ListenConfigController cli-config [listener] failed with : failed to read private key: asn1: structure error: tags don't match (2 vs {class:0 tag:16 length:13 isCompound:true}) +``` + +为了能正常工作，你需è¦è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œå°†å¯†é’¥ä»Ž PKCS8 转æ¢ä¸º PKCS1： + +``` +openssl rsa -in key.pem -out convertedkey.pem +``` + +ä½ å¯ä½¿ç”¨ `convertedkey.pem` 作为 Rancher è¯ä¹¦å¯†é’¥æ–‡ä»¶ã€‚ + +### 添加中间è¯ä¹¦çš„顺åºæ˜¯ä»€ä¹ˆï¼Ÿ + +添加è¯ä¹¦çš„顺åºå¦‚下： + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +### 如何验è¯æˆ‘çš„è¯ä¹¦é“¾ï¼Ÿ + +ä½ å¯ä½¿ç”¨ `openssl` 二进制文件æ¥éªŒè¯è¯ä¹¦é“¾ã€‚如果命令的输出以 `Verify return code: 0 (ok)` 结尾（å‚è§ä»¥ä¸‹ç¤ºä¾‹ï¼‰ï¼Œä½ çš„è¯ä¹¦é“¾æ˜¯æœ‰æ•ˆçš„。`ca.pem` 文件必须与你添加到 `rancher/rancher` 容器中的文件一致。 + +如果你使用由å¯ä¿¡çš„ CA ç­¾å‘çš„è¯ä¹¦ï¼Œå¯çœç•¥ `-CAfile` å‚数。 + +命令： + +``` +openssl s_client -CAfile ca.pem -connect rancher.yourdomain.com:443 +... + Verify return code: 0 (ok) +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md new file mode 100644 index 00000000000..32fbf2f4254 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md @@ -0,0 +1,91 @@ +--- +title: 回滚 Docker 安装的 Rancher +--- + +如果 Rancher å‡çº§æ²¡æœ‰æˆåŠŸå®Œæˆï¼Œä½ éœ€è¦å›žæ»šåˆ°ä½ åœ¨ [Docker å‡çº§](upgrade-docker-installed-rancher.md)之å‰ä½¿ç”¨çš„ Rancher 设置。回滚å¯ä»¥æ¢å¤ï¼š + +- å…ˆå‰ç‰ˆæœ¬çš„ Rancher。 +- å‡çº§å‰åˆ›å»ºçš„æ•°æ®å¤‡ä»½ã€‚ + +## åœ¨ä½ å¼€å§‹å‰ + +åœ¨å›žæ»šåˆ°å…ˆå‰ Rancher 版本的过程中，你将输入一系列命令。请按照你环境的实际情况替æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。以下是带有å ä½ç¬¦çš„命令示例： + +``` +docker pull rancher/rancher: +``` + +在此命令中，`` 是å‡çº§å¤±è´¥ä¹‹å‰è¿è¡Œçš„ Rancher 版本，如 `v2.0.5`。 + +请交å‰å‚考下方的图片和表格，了解获å–æ­¤å ä½ç¬¦æ•°æ®çš„方法。在开始以下步骤之å‰ï¼Œè¯·å…ˆè®°ä¸‹æˆ–å¤åˆ¶æ­¤ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <PRIOR_RANCHER_VERSION> å’Œ <RANCHER_CONTAINER_NAME>![å ä½ç¬¦å‚考](/img/placeholder-ref-2.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | ------------------------------------------------------- | +| `` | `v2.0.5` | å‡çº§å‰ä½¿ç”¨çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.0.5` | 备份对应的 Rancher 版本。 | +| `` | `9-27-18` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ `` 。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## 回滚 Rancher + +如果你在å‡çº§ Rancher æ—¶é‡åˆ°é—®é¢˜ï¼Œä½ å¯æ‹‰å–å…ˆå‰ä½¿ç”¨çš„é•œåƒå¹¶æ¢å¤åœ¨å‡çº§å‰æ‰€åšçš„备份，从而将 Rancher 回滚到之å‰çš„正常工作状æ€ã€‚ + +:::danger + +回滚到先å‰çš„ Rancher 版本会破å你在å‡çº§åŽå¯¹ Rancher åšå‡ºçš„所有更改。丢失的数æ®å¯èƒ½æ— æ³•æ¢å¤ã€‚ + +::: + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. 拉å–å‡çº§å‰è¿è¡Œçš„ Rancher 版本。把 `` 替æ¢ä¸ºè¯¥ç‰ˆæœ¬ã€‚ + + 例如，如果å‡çº§ä¹‹å‰è¿è¡Œçš„是 Rancher v2.0.5ï¼Œè¯·æ‹‰å– v2.0.5。 + + ``` + docker pull rancher/rancher: + ``` + +1. åœæ­¢å½“å‰è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸ºä½ çš„ Rancher 容器的å称： + + ``` + docker stop + ``` + ä½ å¯è¾“å…¥ `docker ps`èŽ·å– Rancher 容器的å称。 + +1. 将你在 [Docker å‡çº§](upgrade-docker-installed-rancher.md)时创建的备份压缩包移动到 Rancher Server。切æ¢åˆ°ä½ å°†å…¶ç§»åŠ¨åˆ°çš„目录。输入 `dir` 以确认它在该ä½ç½®ã€‚ + + 如果你éµå¾ªäº†æˆ‘们在 [Docker å‡çº§](upgrade-docker-installed-rancher.md)中推è的命åæ–¹å¼ï¼Œå®ƒçš„å称会与 `rancher-data-backup--.tar.gz` 类似。 + +1. 替æ¢å ä½ç¬¦æ¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œå°† `rancher-data` 容器中的数æ®æ›¿æ¢ä¸ºå¤‡ä»½åŽ‹ç¼©åŒ…中的数æ®ã€‚ä¸è¦å¿˜è®°å…³é—­å¼•å·ã€‚ + + ``` + docker run --volumes-from rancher-data \ + -v $PWD:/backup busybox sh -c "rm /var/lib/rancher/* -rf \ + && tar zxvf /backup/rancher-data-backup--.tar.gz" + ``` + +1. å°† `` å ä½ç¬¦æŒ‡å‘æ•°æ®å®¹å™¨ï¼Œå¯åŠ¨ä¸€ä¸ªæ–°çš„ Rancher Server 容器。 + ``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: + ``` + 特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + + :::danger + + å¯åŠ¨å›žæ»šåŽï¼Œå³ä½¿å›žæ»šè€—时比预期长，也 **_ä¸è¦_** åœæ­¢å›žæ»šã€‚如果你åœæ­¢å›žæ»šï¼Œå¯èƒ½ä¼šå¯¼è‡´ä¹‹åŽçš„å‡çº§ä¸­å‡ºçŽ°æ•°æ®åº“错误。 + + ::: + +1. 等待片刻，然åŽåœ¨æµè§ˆå™¨ä¸­æ‰“å¼€ Rancher。确认回滚æˆåŠŸå¹¶ä¸”ä½ çš„æ•°æ®å·²æ¢å¤ã€‚ + +**结果**：Rancher 回滚到å‡çº§å‰çš„版本和数æ®çŠ¶æ€ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md new file mode 100644 index 00000000000..68d23b34c33 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -0,0 +1,393 @@ +--- +title: å‡çº§ Docker 安装的 Rancher +--- + +本文介ç»å¦‚何å‡çº§é€šè¿‡ Docker 安装的 Rancher Server。 + +:::caution + +**生产环境ä¸æ”¯æŒ Docker 安装**。这些说明仅适用于测试和开å‘。如果你已ç»åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­éƒ¨ç½²äº† Docker 安装并且需è¦å‡çº§åˆ°æ–°çš„ Rancher 版本，我们建议你在å‡çº§å‰å…ˆ[è¿ç§»åˆ° Helm Chart 安装](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +::: + +## 先决æ¡ä»¶ + +- 在 Rancher 文档中**检查[已知å‡çº§é—®é¢˜](../../install-upgrade-on-a-kubernetes-cluster/upgrades.md#已知å‡çº§é—®é¢˜)**，了解å‡çº§ Rancher 时最需è¦æ³¨æ„的问题。你å¯ä»¥åœ¨ [GitHub](https://github.com/rancher/rancher/releases) å‘å¸ƒè¯´æ˜Žä»¥åŠ [Rancher 论å›](https://forums.rancher.com/c/announcements/12)中找到æ¯ä¸ª Rancher 版本的已知问题。ä¸æ”¯æŒå‡çº§æˆ–å‡çº§åˆ° [rancher-alpha 仓库](../../resources/choose-a-rancher-version.md#helm-chart-仓库)中的任何 Chart。 +- **[仅适用于离线安装](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md)：为新的 Rancher Server 版本收集和推é€é•œåƒ**。按照指å—为你想è¦å‡çº§çš„目标 Rancher 版本[推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库](../air-gapped-helm-cli-install/publish-images.md)。 + +## å ä½ç¬¦ + +在å‡çº§è¿‡ç¨‹ä¸­ï¼Œä½ å°†è¾“入一系列命令。请按照你环境的实际情况替æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。 + +以下是带有å ä½ç¬¦çš„命令**示例**： + +``` +docker stop +``` + +在此命令中，`` 是你的 Rancher 容器的å称。 + +## 获å–å‡çº§å‘½ä»¤çš„æ•°æ® + +è¦èŽ·å–替æ¢å ä½ç¬¦çš„æ•°æ®ï¼Œè¯·è¿è¡Œï¼š + +``` +docker ps +``` + +在开始å‡çº§ä¹‹å‰è®°ä¸‹æˆ–å¤åˆ¶æ­¤ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <RANCHER_CONTAINER_TAG> å’Œ <RANCHER_CONTAINER_NAME> + +![å ä½ç¬¦å‚考](/img/placeholder-ref.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.1.3` | 首次安装拉å–çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.1.3` | 你为其创建备份的 Rancher 版本。 | +| `` | `2018-12-19` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ ``。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## å‡çº§ + +在å‡çº§æœŸé—´ï¼Œä½ å¯ä»¥ä¸ºå½“å‰ Rancher 容器创建数æ®çš„副本åŠå¤‡ä»½ï¼Œä»¥ç¡®ä¿å¯ä»¥åœ¨å‡çº§å‡ºçŽ°é—®é¢˜æ—¶å¯ä»¥è¿›è¡Œå›žæ»šã€‚然åŽï¼Œä½ å¯ä½¿ç”¨çŽ°æœ‰æ•°æ®å°†æ–°ç‰ˆæœ¬çš„ Rancher 部署到新容器中。 +### 1. 创建 Rancher Server 容器的数æ®å‰¯æœ¬ + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. åœæ­¢æ­£åœ¨è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸ºä½ çš„ Rancher 容器的å称： + + ``` + docker stop + ``` + +1. è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä»Žåˆšæ‰åœæ­¢çš„ Rancher 容器创建一个数æ®å®¹å™¨ã€‚请替æ¢å‘½ä»¤ä¸­çš„å ä½ç¬¦ï¼š + + ``` + docker create --volumes-from --name rancher-data rancher/rancher: + ``` + +### 2. 创建备份压缩包 + +1. 从你刚刚创建的数æ®å®¹å™¨ï¼ˆrancher-data）中，创建一个备份 tar 包（rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz）。 + + 如果å‡çº§æœŸé—´å‡ºçŽ°é—®é¢˜ï¼Œæ­¤åŽ‹ç¼©åŒ…å¯ä»¥ç”¨ä½œå›žæ»šç‚¹ã€‚替æ¢å ä½ç¬¦æ¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚ + ``` + docker run --volumes-from rancher-data -v "$PWD:/backup" --rm busybox tar zcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **步骤结果**：你输入此命令时，会è¿è¡Œä¸€ç³»åˆ—命令。 + +1. 输入 `ls` 命令，确认备份压缩包已创建æˆåŠŸã€‚压缩包的å称格å¼ç±»ä¼¼ `rancher-data-backup--.tar.gz`。 + + ``` + [rancher@ip-10-0-0-50 ~]$ ls + rancher-data-backup-v2.1.3-20181219.tar.gz + ``` + +1. 将备份压缩包移动到 Rancher Server 外的安全ä½ç½®ã€‚ + +### 3. 拉å–æ–°çš„ Docker é•œåƒ + +拉å–你需è¦å‡çº§åˆ°çš„ Rancher 版本镜åƒã€‚ + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker pull rancher/rancher: +``` + +### 4. å¯åŠ¨æ–°çš„ Rancher Server 容器 + +使用 `rancher-data` 容器中的数æ®å¯åŠ¨ä¸€ä¸ªæ–°çš„ Rancher Server 容器。记ä½è¦ä¼ å…¥å¯åŠ¨åŽŸå§‹å®¹å™¨æ—¶ä½¿ç”¨çš„所有环境å˜é‡ã€‚ + +:::danger + +å¯åŠ¨å‡çº§åŽï¼Œå³ä½¿å‡çº§è€—时比预期长，也 **_ä¸è¦_** åœæ­¢å‡çº§ã€‚如果你åœæ­¢å‡çº§ï¼Œå¯èƒ½ä¼šå¯¼è‡´ä¹‹åŽçš„å‡çº§ä¸­å‡ºçŽ°æ•°æ®åº“è¿ç§»é”™è¯¯ã€‚ + +::: + +如果你使用代ç†ï¼Œè¯·å‚è§ [HTTP 代ç†é…ç½®](../../../../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md)。 + +如果你é…置了自定义 CA æ ¹è¯ä¹¦æ¥è®¿é—®æœåŠ¡ï¼Œè¯·å‚è§[自定义 CA æ ¹è¯ä¹¦](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#自定义-ca-è¯ä¹¦)。 + +如果你è¦è®°å½•æ‰€æœ‰ Rancher API 的事务，请å‚è§ [API 审计](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志)。 + +如需查看å¯åŠ¨æ–° Rancher Server 容器时使用的命令，从以下的选项中进行选择： + +- Docker å‡çº§ +- 离线安装的 Docker å‡çº§ + + + + +选择你安装 Rancher Server 时用的选项 + +#### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你使用 Rancher 生æˆçš„自签åè¯ä¹¦ï¼Œåˆ™å°† `--volumes-from rancher-data` 添加到你å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中。 + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +
+ +#### 选项 B：使用你自己的è¯ä¹¦ - 自签å + +
+å•å‡»å±•å¼€ + +如果你选择使用自己的自签åè¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚ + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher: +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +
+ +#### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你选择使用å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚注æ„è¦ä½¿ç”¨ `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括å¯ä¿¡ CA æ供的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](certificate-troubleshooting.md)。 + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher: \ + --no-cacerts +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ +#### 选项 D：Let's Encrypt è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +:::caution + +Let's Encrypt 对新è¯ä¹¦è¯·æ±‚有频率é™åˆ¶ã€‚因此，请é™åˆ¶åˆ›å»ºæˆ–销æ¯å®¹å™¨çš„频率。详情请å‚è§ [Let's Encrypt 官方文档 - 频率é™åˆ¶](https://letsencrypt.org/docs/rate-limits/)。 + +::: + +如果你选择使用 [Let's Encrypt](https://letsencrypt.org/) è¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`，并且æ供最åˆå®‰è£… Rancher 时使用的域å。 + +:::note è¯ä¹¦è¦æ±‚æ示： + +- 在 DNS 中创建一æ¡è®°å½•ï¼Œå°† Linux 主机 IP 地å€ç»‘定到è¦ç”¨äºŽè®¿é—® Rancher 的主机å（例如，`rancher.mydomain.com`）。 +- 在 Linux 主机上打开 `TCP/80` 端å£ã€‚Let's Encrypt çš„ HTTP-01 质询å¯ä»¥æ¥è‡ªä»»ä½•æº IP 地å€ï¼Œå› æ­¤ç«¯å£ `TCP/80` 必须开放开所有 IP 地å€ã€‚ + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | +| `` | 你最åˆä½¿ç”¨çš„域å | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: \ + --acme-domain +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +
+ + + + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +å¯åŠ¨æ–°çš„ Rancher Server 容器时，从以下的选项中进行选择： + +#### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你使用 Rancher 生æˆçš„自签åè¯ä¹¦ï¼Œåˆ™å°† `--volumes-from rancher-data` 添加到你å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中。 + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ +#### 选项 B：使用你自己的è¯ä¹¦ - 自签å + +
+ å•å‡»å±•å¼€ + +如果你选择使用自己的自签åè¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](certificate-troubleshooting.md)。 + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ +#### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你选择使用å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括å¯ä¿¡ CA æ供的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](certificate-troubleshooting.md)。 + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +:::note + +使用 `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +::: + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged + /rancher/rancher: +``` +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ + + + +**结果**：你已å‡çº§ Rancher。已å‡çº§ Server 中的数æ®å°†ä¿å­˜åœ¨ `rancher-data` 容器中，用于将æ¥çš„å‡çº§ã€‚ + +### 5. 验è¯å‡çº§ + +登录到 Rancher。通过检查æµè§ˆå™¨å·¦ä¸‹è§’的版本å·ï¼Œç¡®è®¤å‡çº§æ˜¯å¦æˆåŠŸã€‚ + +:::note å‡çº§åŽä¸‹æ¸¸é›†ç¾¤å‡ºçŽ°ç½‘络问题？ + +请å‚è§[æ¢å¤é›†ç¾¤ç½‘络](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md)。 + +::: + +### 6. 清ç†æ—§çš„ Rancher Server 容器 + +移除旧的 Rancher Server 容器。如果你仅åœæ­¢äº†æ—§çš„ Rancher Server 容器，但没有移除它，该容器还å¯èƒ½åœ¨æœåŠ¡å™¨ä¸‹æ¬¡é‡å¯åŽé‡æ–°å¯åŠ¨ã€‚ + +## 回滚 + +如果å‡çº§æ²¡æœ‰æˆåŠŸå®Œæˆï¼Œä½ å¯ä»¥å°† Rancher Server åŠå…¶æ•°æ®å›žæ»šåˆ°ä¸Šæ¬¡çš„å¥åº·çŠ¶æ€ã€‚详情请å‚è§ [Docker 回滚](roll-back-docker-installed-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/add-tls-secrets.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/add-tls-secrets.md new file mode 100644 index 00000000000..dc58f216be0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/add-tls-secrets.md @@ -0,0 +1,45 @@ +--- +title: 添加 TLS 密文 +--- + +我们使用è¯ä¹¦å’Œå¯†é’¥å°† `cattle-system` 命å空间中的 `tls-rancher-ingress` 密文é…置好åŽï¼ŒKubernetes 会为 Rancher 创建对象和æœåŠ¡ã€‚ + +å°†æœåŠ¡å™¨è¯ä¹¦å’Œæ‰€éœ€çš„所有中间è¯ä¹¦åˆå¹¶åˆ°å为 `tls.crt`的文件中。将è¯ä¹¦å¯†é’¥å¤åˆ¶åˆ°å为 `tls.key` 的文件中。 + +例如，[acme.sh](https://acme.sh) 在 `fullchain.cer` 文件中æä¾›æœåŠ¡å™¨è¯ä¹¦å’Œ CA 链。 +请将 `fullchain.cer` 命å为 `tls.crt`，将è¯ä¹¦å¯†é’¥æ–‡ä»¶å‘½å为 `tls.key`。 + +使用 `kubectl` 创建 `tls` 类型的密文。 + +``` +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +:::note + +如需替æ¢è¯ä¹¦ï¼Œä½ å¯ä»¥è¿è¡Œ `kubectl -n cattle-system delete secret tls-rancher-ingress` æ¥åˆ é™¤ `tls-rancher-ingress` 密文，然åŽè¿è¡Œä¸Šæ–¹å‘½ä»¤æ¥æ·»åŠ æ–°çš„密文。如果你使用的是ç§æœ‰ CA ç­¾åè¯ä¹¦ï¼Œä»…当新è¯ä¹¦ä¸Žå½“å‰è¯ä¹¦æ˜¯ç”±åŒä¸€ä¸ª CA ç­¾å‘的，æ‰å¯ä»¥æ›¿æ¢ã€‚ + +::: + +## 使用ç§æœ‰ CA ç­¾åè¯ä¹¦ + +如果你使用的是ç§æœ‰ CA，Rancher 需è¦ç§æœ‰ CA çš„æ ¹è¯ä¹¦æˆ–è¯ä¹¦é“¾çš„副本，Rancher Agent 使用它æ¥æ ¡éªŒä¸Ž Server 的连接。 + +创建一个å为 `cacerts.pem` 的文件，该文件仅包å«ç§æœ‰ CA çš„æ ¹ CA è¯ä¹¦æˆ–è¯ä¹¦é“¾ï¼Œå¹¶ä½¿ç”¨ `kubectl` 在 `cattle-system` 命å空间中创建 `tls-ca` Secret。 + +``` +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem=./cacerts.pem +``` + +:::note + +Rancher å¯åŠ¨æ—¶ä¼šæ£€ç´¢é…置的 `tls-ca` 密文。如果 Rancher 在è¿è¡Œä¸­ï¼Œæ›´æ–°çš„ CA 会在新的 Rancher Pod å¯åŠ¨åŽç”Ÿæ•ˆã€‚ + +::: + +## æ›´æ–°ç§æœ‰ CA è¯ä¹¦ + +按照[步骤](update-rancher-certificate.md)æ›´æ–° [Rancher 高å¯ç”¨ Kubernetes 安装](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)中的 Ingress，或从默认自签åè¯ä¹¦åˆ‡æ¢åˆ°è‡ªå®šä¹‰è¯ä¹¦ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/bootstrap-password.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/bootstrap-password.md new file mode 100644 index 00000000000..249b14115a0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/bootstrap-password.md @@ -0,0 +1,27 @@ +--- +title: å¼•å¯¼å¯†ç  +--- + +Rancher 首次å¯åŠ¨æ—¶ï¼Œä¼šä¸ºç¬¬ä¸€ä¸ªç®¡ç†å‘˜ç”¨æˆ·éšæœºç”Ÿæˆä¸€ä¸ªå¯†ç ã€‚当管ç†å‘˜é¦–次登录 Rancher 时，用于获å–引导密ç ï¼ˆBootstrap）的命令会在 UI 上显示。管ç†å‘˜éœ€è¦è¿è¡Œå‘½ä»¤å¹¶ä½¿ç”¨å¼•å¯¼å¯†ç ç™»å½•ã€‚ç„¶åŽ Rancher 会让管ç†å‘˜é‡ç½®å¯†ç ã€‚ + +如果你在安装过程中没有使用å˜é‡æ¥è®¾ç½®å¼•å¯¼å¯†ç ï¼Œåˆ™ä¼šéšæœºç”Ÿæˆå¼•å¯¼å¯†ç ã€‚如需了解使用å˜é‡è®¾ç½®å¼•å¯¼å¯†ç çš„详情，请å‚è§ä¸‹æ–‡ã€‚ + +### 在 Helm å®‰è£…ä¸­æŒ‡å®šå¼•å¯¼å¯†ç  + +Helm 安装的情况下，你å¯ä»¥ä½¿ç”¨ `.Values.bootstrapPassword` 在 Helm Chart 值中指定引导密ç å˜é‡ã€‚ + +密ç å°†å­˜å‚¨åœ¨ Kubernetes 密文中。安装 Rancher åŽï¼Œå¦‚何使用 kubectl 获å–密ç çš„说明将会在 UI 中显示： + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### 在 Docker å®‰è£…ä¸­æŒ‡å®šå¼•å¯¼å¯†ç  + +如果 Rancher 是使用 Docker 安装的，你å¯ä»¥é€šè¿‡åœ¨ Docker 安装命令中传递 `-e CATTLE_BOOTSTRAP_PASSWORD=password` æ¥æŒ‡å®šå¼•å¯¼å¯†ç ã€‚ + +密ç å°†å­˜å‚¨åœ¨ Docker 容器日志中。安装 Rancher åŽï¼Œå¦‚何使用 Docker 容器 ID 获å–密ç çš„说明将会在 UI 中显示： + +``` +docker logs container-id 2>&1 | grep "Bootstrap Password:" +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md new file mode 100644 index 00000000000..b168ecdcb05 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md @@ -0,0 +1,118 @@ +--- +title: 选择 Rancher 版本 +--- + +本节介ç»å¦‚何选择 Rancher 版本。 + +在我们推è用于生产环境的 Rancher 高å¯ç”¨å®‰è£…中，Rancher Server 是通过 Kubernetes 集群上的 **Helm Chart** 安装的。请å‚è§ [Helm 版本è¦æ±‚](helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。 + +如果你在开å‘和测试中使用 Docker æ¥å®‰è£… Rancher，你需è¦æŠŠ Rancher 作为一个 **Docker é•œåƒ**æ¥å®‰è£…。 + + + + +如果 Rancher Server 是[安装在 Kubernetes 集群上](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)的，Rancher Server 的安装，å‡çº§å’Œå›žæ»šä¸­ï¼Œéƒ½æ˜¯ä½¿ç”¨ Kubernetes 集群上的 Helm Chart æ¥å®‰è£… Rancher 的。因此，在准备安装或å‡çº§ Rancher 高å¯ç”¨æ—¶ï¼Œå¿…须添加包å«ç”¨äºŽå®‰è£… Rancher çš„ Chart çš„ Helm Chart 仓库。 + +请å‚è§ [Helm 版本è¦æ±‚](helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。 + +### Helm Chart 仓库 + +Rancher æ供几个å¯é€‰çš„ Helm Chart 仓库供你选择。最新版或稳定版的 Helm Chart 仓库与用于 Docker 安装中的 Docker 标签对应。因此，`rancher-latest` 仓库包å«æ‰€æœ‰æ ‡è®°ä¸º `rancher/rancher:latest` çš„ Rancher 版本 Chart。当 Rancher 版本å‡çº§åˆ° `rancher/rancher:stable`，它会被添加到 `rancher-stable` 仓库中。 + +| 类型 | 添加仓库的命令 | 仓库æè¿° | +| -------------- | ------------ | ----------------- | +| rancher-latest | `helm repo add rancher-latest https://releases.rancher.com/server-charts/latest` | 添加最新版本的 Rancher çš„ Helm Chart 仓库。建议使用此仓库æ¥æµ‹è¯•æ–°ç‰ˆæœ¬çš„ Rancher。 | +| rancher-stable | `helm repo add rancher-stable https://releases.rancher.com/server-charts/stable` | 添加较旧的，稳定的版本的 Rancher çš„ Helm Chart 仓库。建议在生产环境中使用此仓库。 | +| rancher-alpha | `helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha` | 添加 alpha 版本的 Rancher çš„ Helm Chart 仓库，以预览å³å°†å‘布的版本。ä¸å»ºè®®åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­ä½¿ç”¨è¿™äº›ç‰ˆæœ¬ã€‚无论是什么仓库，å‡ä¸æ”¯æŒ _å‡çº§_ 或 _å‡çº§åˆ°_ rancher-alpha 仓库中的任何 Chart。 | + +了解何时选择这些仓库，请å‚è§[切æ¢åˆ°ä¸åŒ Helm Chart 仓库](#切æ¢åˆ°ä¸åŒ-helm-chart-仓库)。 + +:::note + +`rancher-stable` 仓库中的所有 Chart 都与 `stable` 标记的 Rancher 版本对应。 + +::: + +### Helm Chart 版本 + +Rancher Helm Chart 版本与 Rancher ç‰ˆæœ¬ï¼ˆå³ `appVersion`）对应。添加仓库åŽï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤æœç´¢å¯ç”¨ç‰ˆæœ¬ï¼š
+    `helm search repo --versions` + +如果你有多个仓库，你å¯æŒ‡å®šä»“库å称，å³ï¼š`helm search repo rancher-stable/rancher --versions`
+详情请访问 https://helm.sh/docs/helm/helm_search_repo/ + +è¦èŽ·å–所选仓库的指定版本，å‚è§å¦‚下示例指定 `--version` å‚数：
+    `helm fetch rancher-stable/rancher --version=2.4.8` + +### 切æ¢åˆ°ä¸åŒ Helm Chart 仓库 + +安装 Rancher åŽï¼Œå¦‚果想修改安装 Rancher çš„ Helm Chart 仓库，按照以下步骤æ“作。 + +:::note + +由于 rancher-alpha 仓库åªåŒ…å« alpha 版本 Chart，因此ä¸æ”¯æŒä»Ž rancher alpha 仓库切æ¢åˆ° rancher-stable 或 rancher-latest 仓库以进行å‡çº§ã€‚ + +::: + +- Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + +1. åˆ—å‡ºå½“å‰ Helm Chart 仓库。 + + ```plain + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + +2. 删除包å«å®‰è£… Rancher 时用的 Chart çš„ Helm Chart 仓库。是 `rancher-stable` 或 `rancher-latest` å–决于你åˆå§‹å®‰è£…时的选择。 + + ```plain + helm repo remove rancher- + ``` + +3. 添加你è¦ç”¨äºŽå®‰è£… Rancher çš„ Helm Chart 仓库。 + + ```plain + helm repo add rancher- https://releases.rancher.com/server-charts/ + ``` + +4. 按照以下步骤，从新的 Helm Chart 仓库[å‡çº§ Rancher](../install-upgrade-on-a-kubernetes-cluster/upgrades.md)。 + + + + +在执行 [Docker 安装](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md)ã€å‡çº§æˆ–回滚时，你å¯ä»¥ä½¿ç”¨ _tags_ æ¥å®‰è£…特定版本的 Rancher。 + +### Server 标签 + +Rancher Server 以 Docker é•œåƒçš„å½¢å¼åˆ†å‘并附有标签。你å¯ä»¥åœ¨è¾“入命令部署 Rancher 时指定标签。请记ä½ï¼Œå¦‚果你指定了标签，但是没有指定版本（如 `latest` 或 `stable`），你必须显å¼æ‹‰å–该镜åƒæ ‡ç­¾çš„新版本。å¦åˆ™ï¼Œå°†ä½¿ç”¨ç¼“存在主机上的镜åƒã€‚ + +| 标签 | æè¿° | +| -------------------------- | ------ | +| `rancher/rancher:latest` | 最新的开å‘版本。这些版本通过了我们的 CI 自动化验è¯ã€‚ä¸æŽ¨è在生产环境使用这些版本。 | +| `rancher/rancher:stable` | 最新的稳定版本。推è将此标签用于生产环境。 | +| `rancher/rancher:` | ä½ å¯ä»¥ä½¿ç”¨ä»¥å‰ç‰ˆæœ¬ä¸­çš„标签æ¥æŒ‡å®šè¦å®‰è£…çš„ Rancher 版本。访问 DockerHub 查看å¯ç”¨çš„版本。 | + +:::note + +- `master` 和带有 `-rc` 或其他åŽç¼€çš„标签是供 Rancher 测试团队验è¯ç”¨çš„。这些标签ä¸å—官方支æŒï¼Œå› æ­¤è¯·ä¸è¦ä½¿ç”¨è¿™äº›æ ‡ç­¾ã€‚ +- 安装 alpha 版本进行预览：使用我们的[公告页é¢](https://forums.rancher.com/c/announcements)中列出的 alpha 标签（例如，`v2.2.0-alpha1`）进行安装。ä¸æ”¯æŒå‡çº§æˆ–å‡çº§åˆ° Alpha 版本。 + +::: + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md new file mode 100644 index 00000000000..d454197c8a5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md @@ -0,0 +1,24 @@ +--- +title: 自定义 CA æ ¹è¯ä¹¦ +--- + +如果你在内部生产环境使用 Rancher，且ä¸æ‰“算公开暴露应用，你å¯ä»¥ä½¿ç”¨ä½¿ç”¨ç§æœ‰ CA é¢å‘çš„è¯ä¹¦ã€‚ + +Rancher å¯èƒ½ä¼šè®¿é—®é…置了自定义/内部 CA æ ¹è¯ä¹¦ï¼ˆä¹Ÿç§°ä¸ºè‡ªç­¾åè¯ä¹¦ï¼‰çš„æœåŠ¡ã€‚如果 Rancher 无法验è¯æœåŠ¡çš„è¯ä¹¦ï¼Œåˆ™ä¼šæ˜¾ç¤ºé”™è¯¯ä¿¡æ¯ `x509: certificate signed by unknown authority`。 + +如需验è¯è¯ä¹¦ï¼Œä½ éœ€è¦æŠŠ CA æ ¹è¯ä¹¦æ·»åŠ åˆ° Rancher。由于 Rancher 是用 Go 编写的，我们å¯ä»¥ä½¿ç”¨çŽ¯å¢ƒå˜é‡ `SSL_CERT_DIR` 指å‘容器中 CA æ ¹è¯ä¹¦æ‰€åœ¨çš„目录。å¯åŠ¨ Rancher 容器时，å¯ä»¥ä½¿ç”¨ Docker å·é€‰é¡¹ï¼ˆ`-v host-source-directory:container-destination-directory`）æ¥æŒ‚è½½ CA æ ¹è¯ä¹¦ç›®å½•ã€‚ + +Rancher å¯ä»¥è®¿é—®çš„æœåŠ¡ç¤ºä¾‹ï¼š + +- 应用商店 +- 验è¯æä¾›ç¨‹åº +- 使用 Node Driver 访问托管/云 API + +## 使用自定义 CA è¯ä¹¦å®‰è£… + +有关å¯åŠ¨æŒ‚载了ç§æœ‰ CA è¯ä¹¦çš„ Rancher 容器的详情，请å‚è§å®‰è£…文档： + +- [Docker 安装的自定义 CA è¯ä¹¦é€‰é¡¹](../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#自定义-ca-è¯ä¹¦) + +- [Kubernetes 安装的其他å—ä¿¡ CA 选项](../installation-references/helm-chart-options.md#é¢å¤–的授信-ca) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/helm-version-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/helm-version-requirements.md new file mode 100644 index 00000000000..273711a4de0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/helm-version-requirements.md @@ -0,0 +1,12 @@ +--- +title: Helm 版本è¦æ±‚ +--- + +æœ¬æ–‡ä»‹ç» Helm çš„è¦æ±‚。Helm 是用于把 Rancher 安装在高å¯ç”¨ Kubernetes 集群上的工具。 + +> 我们已针对 Helm 3 更新了安装指å—。如果你使用 Helm 2 进行安装，请å‚è§ [Helm 2 è¿ç§»åˆ° Helm 3 文档](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/)。[本文](/versioned_docs/version-2.0-2.4/pages-for-subheaders/helm2.md)æ供了较早的使用 Helm 2 çš„ Rancher 高å¯ç”¨å®‰è£…指å—的副本。如果你如果无法å‡çº§åˆ° Helm 3，å¯ä»¥ä½¿ç”¨è¿™ä¸ªè¯´æ˜Žå®‰è£…。 + +- 如需安装或å‡çº§ Rancher 2.5，请使用 Helm 3.2.x 或更高版本。 +- Kubernetes 1.16 è¦æ±‚ Helm 2.16.0 或更高版本。如果使用的是默认 Kubernetes 版本，请å‚è§[å‘行说明](https://github.com/rancher/rke/releases)获å–所使用的 RKE 版本。 +- 请ä¸è¦ä½¿ç”¨ Helm 2.15.0，因为这个版本有转æ¢/比较数字的问题。 +- 请ä¸è¦ä½¿ç”¨ Helm 2.12.0，因为该版本有 `cert-manager` 的兼容问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/local-system-charts.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/local-system-charts.md new file mode 100644 index 00000000000..4048c5b2566 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/local-system-charts.md @@ -0,0 +1,13 @@ +--- +title: 离线安装中设置本地 System Charts +--- + +[Charts](https://github.com/rancher/charts) ä»“åº“åŒ…å« Monitoringã€Loggingã€å‘Šè­¦å’Œ Istio 等功能所需的所有 Helm 目录项。 + +在 Rancher 的离线安装中，你需è¦é…ç½® Rancher 以使用 System Charts 的本地副本。本节介ç»å¦‚何通过 CLI 标志使用本地 System Charts。 + +## 使用本地 System Charts + +`system-charts` 的一个本地副本已ç»æ‰“包到 `rancher/rancher` 容器中。为了在离线安装中使用这些功能，你需è¦ä½¿ç”¨é¢å¤–的环境å˜é‡ `CATTLE_SYSTEM_CATALOG=bundled` æ¥è¿è¡Œ Rancher 安装命令，该环境å˜é‡å‘Šè¯‰ Rancher 使用 Chart 的本地副本，而ä¸æ˜¯å°è¯•ä»Ž GitHub èŽ·å– Chart。 + +带有 `system-charts` çš„ Rancher 安装命令示例包å«åœ¨ Docker å’Œ Helm çš„[离线安装说明](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md new file mode 100644 index 00000000000..3d11417cda6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md @@ -0,0 +1,263 @@ +--- +title: æ›´æ–° Rancher è¯ä¹¦ +--- + +## æ›´æ–°ç§æœ‰ CA è¯ä¹¦ + +按照以下步骤轮æ¢[安装在 Kubernetes 集群上](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)ã€ç”± Rancher 使用的 SSL è¯ä¹¦å’Œç§æœ‰ CA，或转用由ç§æœ‰ CA ç­¾å‘çš„ SSL è¯ä¹¦ã€‚ + +步骤概述： + +1. 使用新è¯ä¹¦å’Œç§é’¥åˆ›å»ºæˆ–æ›´æ–° `tls-rancher-ingress` Kubernetes Secret 对象。 +1. 使用根 CA è¯ä¹¦åˆ›å»ºæˆ–æ›´æ–° `tls-ca` Kubernetes Secret 对象（仅在使用ç§æœ‰ CA 时需è¦ï¼‰ã€‚ +1. 使用 Helm CLI æ›´æ–° Rancher 安装。 +1. é‡æ–°é…ç½® Rancher Agent 以信任新的 CA è¯ä¹¦ã€‚ +1. 选择 Fleet 集群的强制更新，æ¥å°† fleet-agent 连接到 Rancher。 + +å„个步骤的详细说明如下。 + +### 1. 创建/æ›´æ–°è¯ä¹¦ Secret 对象 + +首先，将æœåŠ¡å™¨è¯ä¹¦å’Œæ‰€æœ‰ä¸­é—´è¯ä¹¦åˆå¹¶åˆ°å为 `tls.crt` 的文件，并在å为 `tls.key` 的文件中æ供相应的è¯ä¹¦å¯†é’¥ã€‚ + +使用以下命令在 Rancher（本地）管ç†é›†ç¾¤ä¸­åˆ›å»º `tls-rancher-ingress` Secret 对象： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +或者，更新现有的 `tls-rancher-ingress` Secret： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. 创建/æ›´æ–°è¯ä¹¦ CA Secret 对象 + +如果新è¯ä¹¦ç”±ç§æœ‰ CA ç­¾å‘的，你需è¦å°†ç›¸åº”çš„æ ¹ CA è¯ä¹¦å¤åˆ¶åˆ°å为 `cacerts.pem` 的文件中，并创建或更新 `cattle-system` 命å空间中的 `tls-ca` Secret。如果è¯ä¹¦ç”±ä¸­é—´ CA ç­¾å，则 `cacerts.pem` 必须按顺åºåŒæ—¶åŒ…å«ä¸­é—´ CA è¯ä¹¦å’Œæ ¹ CA è¯ä¹¦ã€‚ + +创建åˆå§‹ `tls-ca` Secret： + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem +``` + +è¦æ›´æ–°çŽ°æœ‰çš„ `tls-ca` Secret： + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 3. é‡æ–°é…ç½® Rancher 部署 + +如果è¯ä¹¦æºä¿æŒä¸å˜ï¼ˆä¾‹å¦‚，`secret`），请按照步骤 3a 中的步骤æ“作。 + +但是，如果è¯ä¹¦æºå‘生å˜åŒ–（例如，`letsEncrypt` 更改为 `secret`），请按照 3b 中的步骤æ“作。 + +#### 3a. é‡æ–°éƒ¨ç½² Rancher pod + +当è¯ä¹¦æºä¿æŒä¸å˜ï¼Œä½†éœ€è¦æ›´æ–° CA è¯ä¹¦æ—¶ï¼Œéœ€è¦æ‰§è¡Œæ­¤æ­¥éª¤ã€‚ + +在这ç§æƒ…况下，由于 `tls-ca` secret 在å¯åŠ¨æ—¶ç”± Rancher pod 读å–，因此你需è¦é‡æ–°éƒ¨ç½² Rancher pod。 + +ä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤é‡æ–°éƒ¨ç½² Rancher pod： +```bash +kubectl rollout restart deploy/rancher -n cattle-system +``` + +修改完æˆåŽï¼Œè®¿é—® `https:///v3/settings/cacerts`，验è¯è¯¥å€¼æ˜¯å¦ä¸Žå…ˆå‰å†™å…¥ `tls-ca` Secret 中的 CA è¯ä¹¦åŒ¹é…。在所有é‡æ–°éƒ¨ç½²çš„ Rancher pod å¯åŠ¨ä¹‹å‰ï¼ŒCA `cacerts` 值å¯èƒ½ä¸ä¼šæ›´æ–°ã€‚ + +#### 3b. æ›´æ–° Rancher çš„ Helm 值 + +如果è¯ä¹¦æºæœ‰æ›´æ”¹ï¼Œåˆ™éœ€è¦æ‰§è¡Œæ­¤æ­¥éª¤ã€‚如果你的 Rancher 之å‰ä½¿ç”¨é»˜è®¤çš„自签åè¯ä¹¦ (`ingress.tls.source=rancher`) 或 Let's Encrypt (`ingress.tls.source=letsEncrypt`) è¯ä¹¦ï¼Œå¹¶ä¸”现在正在使用由ç§æœ‰ CA (`ingress.tls.source=secret`) ç­¾åçš„è¯ä¹¦ã€‚ + +以下步骤更新了 Rancher Chart çš„ Helm 值，因此 Rancher pod å’Œ ingress 会使用在步骤 1 å’Œ 2 中创建的新ç§æœ‰ CA è¯ä¹¦ã€‚ + +1. 调整åˆå§‹å®‰è£…期间使用的值，将当å‰å€¼å­˜å‚¨ä¸ºï¼š +```bash +helm get values rancher -n cattle-system -o yaml > values.yaml +``` +1. 检索当å‰éƒ¨ç½²çš„ Rancher Chart 的版本字符串： +```bash +helm ls -n cattle-system +``` +1. æ›´æ–° `values.yaml` æ–‡ä»¶ä¸­çš„å½“å‰ Helm 值以包å«ä¸‹æ–¹å†…容： +```yaml +ingress: + tls: + source: secret +privateCA: true +``` +:::note é‡è¦ï¼š +由于è¯ä¹¦ç”±ç§æœ‰ CA ç­¾å‘，因此确ä¿åœ¨ `values.yaml` 文件中设置了 [`privateCA: true`](../installation-references/helm-chart-options.md#常用选项) 是éžå¸¸é‡è¦çš„。 +::: +1. 使用 `values.yaml` æ–‡ä»¶å’Œå½“å‰ Chart 版本å‡çº§ Helm 应用程åºå®žä¾‹ã€‚版本必须匹é…以防止 Rancher å‡çº§ã€‚ +```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version +``` + +修改完æˆåŽï¼Œè®¿é—® `https:///v3/settings/cacerts`，验è¯è¯¥å€¼æ˜¯å¦ä¸Žå…ˆå‰å†™å…¥ `tls-ca` Secret 中的 CA è¯ä¹¦åŒ¹é…。在所有 Rancher pod å¯åŠ¨ä¹‹å‰ï¼ŒCA `cacerts` 值å¯èƒ½ä¸ä¼šæ›´æ–°ã€‚ + +### 4. é‡æ–°é…ç½® Rancher Agent 以信任ç§æœ‰ CA + +本节介ç»äº†é‡æ–°é…ç½® Rancher Agent 以信任ç§æœ‰ CA 的三ç§æ–¹æ³•ã€‚如果你的实际情况符åˆä»¥ä¸‹ä»»æ„一个æ¡ä»¶ï¼Œè¯·æ‰§è¡Œæ­¤æ­¥éª¤ï¼š + +- Rancher 在先å‰çš„é…置中使用了 Rancher 自签åè¯ä¹¦ (`ingress.tls.source=rancher`) 或 Let's Encrypt è¯ä¹¦ (`ingress.tls.source=letsEncrypt`)。 +- 该è¯ä¹¦ç”±ä¸åŒçš„ç§æœ‰ CA ç­¾å‘ + +#### 为什么è¦æ‰§è¡Œè¿™ä¸€æ­¥éª¤ï¼Ÿ + +如果 Rancher é…置了ç§æœ‰ CA ç­¾åçš„è¯ä¹¦æ—¶ï¼ŒCA è¯ä¹¦é“¾å°†å—到 Rancher agent 容器的信任。Agent 会对下载è¯ä¹¦çš„æ ¡éªŒå’ŒåŠ `CATTLE_CA_CHECKSUM` 环境å˜é‡è¿›è¡Œæ¯”较。æ¢è¨€ä¹‹ï¼Œå¦‚æžœ Rancher 使用的ç§æœ‰ CA è¯ä¹¦å‘生å˜åŒ–，环境å˜é‡ `CATTLE_CA_CHECKSUM` 必须相应更新。 + +#### å¯ä½¿ç”¨çš„方法 + +- 方法 1（最简å•çš„方法）：在轮æ¢è¯ä¹¦åŽå°†æ‰€æœ‰é›†ç¾¤è¿žæŽ¥åˆ° Rancher。适用于更新或é‡æ–°éƒ¨ç½² Rancher 部署（步骤 3）åŽç«‹å³æ‰§è¡Œçš„情况。 + +- 方法 2：适用于集群与 Rancher 失去连接，但所有集群都å¯ç”¨äº† [Authorized Cluster Endpoint](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) (ACE) 的情况。 + +- 方法 3：如果方法 1 å’Œ 2 ä¸å¯è¡Œï¼Œåˆ™å¯ä½¿ç”¨æ–¹æ³• 3 进行回退。 + +#### 方法 1：强制é‡æ–°éƒ¨ç½² Rancher Agent + +对于æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤ï¼Œä½¿ç”¨ Rancher（本地）管ç†é›†ç¾¤çš„ Kubeconfig 文件è¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚ + +```bash +kubectl annotate clusters.management.cattle.io io.cattle.agent.force.deploy=true +``` + +:::note +找到下游集群的集群 ID (c-xxxxx)。你å¯ä»¥åœ¨ Rancher UI çš„**集群管ç†**中查看集群时在æµè§ˆå™¨ URL 中找到 ID。 +::: + +此命令将使 Agent 清å•é‡æ–°åº”用新è¯ä¹¦çš„校验和。 + +#### 方法二：手动更新校验和环境å˜é‡ + +å°† `CATTLE_CA_CHECKSUM` 环境å˜é‡æ›´æ–°ä¸ºåŒ¹é…æ–° CA è¯ä¹¦æ ¡éªŒå’Œçš„值，从而手动为 Agent Kubernetes 对象打上补ä¸ã€‚通过以下æ“作生æˆæ–°çš„校验和： + +```bash +curl -k -s -fL /v3/settings/cacerts | jq -r .value | sha256sum | awk '{print $1}' +``` + +为æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤ä½¿ç”¨ Kubeconfig 更新两个 Agent 部署的环境å˜é‡ã€‚如果集群å¯ç”¨äº† [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md)，你å¯ä»¥[调整 kubectl 上下文](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)，从而直接连接到下游集群。 + +```bash +kubectl edit -n cattle-system ds/cattle-node-agent +kubectl edit -n cattle-system deployment/cattle-cluster-agent +``` + +#### 方法三：手动é‡æ–°éƒ¨ç½² Rancher agent + +该方法通过在æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤çš„ control plane 节点上è¿è¡Œä¸€ç»„命令，从而é‡æ–°åº”用 Rancher agent。 + +对æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤é‡å¤ä»¥ä¸‹æ­¥éª¤ï¼š + +1. 检索 agent 注册 kubectl 命令： + 1. 找到下游集群的集群 ID (c-xxxxx)。你å¯ä»¥åœ¨ Rancher UI çš„**集群管ç†**中查看集群时在æµè§ˆå™¨ URL 中找到 ID。 + 1. å°† Rancher Server URL 和集群 ID 添加到以下 URL：`https:///v3/clusterregistrationtokens?clusterId=`。 + 1. å¤åˆ¶ `insecureCommand` 字段中的命令，使用此命令是因为未使用ç§æœ‰ CA。 + +2. 使用以下其中一ç§æ–¹æ³•ï¼Œä½¿ç”¨ kubeconfig 为下游集群è¿è¡Œä¸Šä¸€æ­¥ä¸­çš„ kubectl 命令： + 1. 如果集群å¯ç”¨äº† [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md)，你å¯ä»¥[调整上下文](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)，从而直接连接到下游集群。 + 1. 或者，SSH 到 control plane 节点： + - RKE：使用[此处文档中的步骤](https://github.com/rancherlabs/support-tools/tree/master/how-to-retrieve-kubeconfig-from-custom-cluster)ç”Ÿæˆ kubeconfig + - RKE2/K3s：使用安装时填充的 kubeconfig + +### 5. 强制更新 Fleet 集群，从而将 fleet-agent é‡æ–°è¿žæŽ¥åˆ° Rancher + +在 Rancher UI çš„[æŒç»­äº¤ä»˜](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#在-rancher-ui-中访问-fleet)中，为集群选择“强制更新â€ï¼Œæ¥å…许下游集群中的 fleet-agent æˆåŠŸè¿žæŽ¥åˆ° Rancher。 + +#### 为什么è¦æ‰§è¡Œè¿™ä¸€æ­¥éª¤ï¼Ÿ + +Rancher 管ç†çš„集群中的 Fleet agent 存储了用于连接到 Rancher çš„ kubeconfig。kubeconfig 包å«ä¸€ä¸ª `certificate-authority-data` å­—æ®µï¼Œè¯¥å­—æ®µåŒ…å« Rancher 使用的è¯ä¹¦çš„ CA。更改 CA 时，你需è¦æ›´æ–°æ­¤å—æ¥å…许 fleet-agent ä¿¡ä»» Rancher 使用的è¯ä¹¦ã€‚ + +## å°†ç§æœ‰ CA è¯ä¹¦æ›´æ”¹ä¸ºå…¬å…±è¯ä¹¦ + +按照以下步骤执行与上é¢ç›¸åçš„æ“作，将ç§æœ‰ CA é¢å‘çš„è¯ä¹¦æ›´æ”¹ä¸ºå…¬å…±æˆ–自签å CA。 + +### 1. 创建/æ›´æ–°è¯ä¹¦ Secret 对象 + +首先，将æœåŠ¡å™¨è¯ä¹¦å’Œæ‰€æœ‰ä¸­é—´è¯ä¹¦åˆå¹¶åˆ°å为 `tls.crt` 的文件，并在å为 `tls.key` 的文件中æ供相应的è¯ä¹¦å¯†é’¥ã€‚ + +使用以下命令在 Rancher（本地）管ç†é›†ç¾¤ä¸­åˆ›å»º `tls-rancher-ingress` Secret 对象： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +或者，更新现有的 `tls-rancher-ingress` Secret： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. 删除 CA è¯ä¹¦ Secret 对象 + +你需è¦åˆ é™¤ `cattle-system` 命å空间中的 `tls-ca secret`（ä¸å†éœ€è¦å®ƒï¼‰ã€‚如果需è¦ï¼Œä½ è¿˜å¯ä»¥é€‰æ‹©ä¿å­˜ `tls-ca` secret 的副本。 + +è¦ä¿å­˜çŽ°æœ‰çš„ `tls-ca` Secret： + +```bash +kubectl -n cattle-system get secret tls-ca -o yaml > tls-ca.yaml +``` + +è¦åˆ é™¤çŽ°æœ‰çš„ `tls-ca` 密文： + +```bash +kubectl -n cattle-system delete secret tls-ca +``` + +### 3. é‡æ–°é…ç½® Rancher 部署 + +如果è¯ä¹¦æºæœ‰æ›´æ”¹ï¼Œåˆ™éœ€è¦æ‰§è¡Œæ­¤æ­¥éª¤ã€‚在这ç§æƒ…况下，它å˜åŒ–的原因很å¯èƒ½æ˜¯å› ä¸º Rancher 之å‰é…置为使用默认的自签åè¯ä¹¦ (`ingress.tls.source=rancher`)。 + +以下步骤更新了 Rancher Chart çš„ Helm 值，因此 Rancher pod å’Œ Ingress 会使用在步骤 1 中创建的新è¯ä¹¦ã€‚ + +1. 调整åˆå§‹å®‰è£…期间使用的值，将当å‰å€¼å­˜å‚¨ä¸ºï¼š +```bash +helm get values rancher -n cattle-system -o yaml > values.yaml +``` +1. 获å–当å‰éƒ¨ç½²çš„ Rancher Chart 的版本字符串： +```bash +helm ls -n cattle-system +``` +1. æ›´æ–° `values.yaml` æ–‡ä»¶ä¸­çš„å½“å‰ Helm 值： + 1. 由于ä¸å†ä½¿ç”¨ç§æœ‰ CA，删除 `privateCA: true` 字段，或将其设置为 `false`。 + 1. æ ¹æ®éœ€è¦è°ƒæ•´ `ingress.tls.source` 字段。有关更多信æ¯ï¼Œè¯·å‚阅 [Chart 选项](../installation-references/helm-chart-options.md#常用选项)。以下是一些示例： + 1. 如果使用公共 CA，继续使用 `secret` + 1. 如果使用 Let's Encrypt，将值更新为 `letsEncrypt` +1. 使用 `values.yaml` 文件更新 Rancher Chart çš„ Helm å€¼ï¼Œå¹¶ä½¿ç”¨å½“å‰ Chart 版本防止å‡çº§ï¼š +```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version +``` + +### 4. 为éžç§æœ‰/通用è¯ä¹¦é‡æ–°é…ç½® Rancher Agent + +由于ä¸å†ä½¿ç”¨ç§æœ‰ CA，因此你需è¦åˆ é™¤ä¸‹æ¸¸é›†ç¾¤ agent 上的 `CATTLE_CA_CHECKSUM` 环境å˜é‡ï¼Œæˆ–将其设置为 ""（空字符串）。 + +### 5. 强制更新 Fleet 集群，从而将 fleet-agent é‡æ–°è¿žæŽ¥åˆ° Rancher + +在 Rancher UI çš„[æŒç»­äº¤ä»˜](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#在-rancher-ui-中访问-fleet)中，为集群选择“强制更新â€ï¼Œæ¥å…许下游集群中的 fleet-agent æˆåŠŸè¿žæŽ¥åˆ° Rancher。 + +#### 为什么è¦æ‰§è¡Œè¿™ä¸€æ­¥éª¤ï¼Ÿ + +Rancher 管ç†çš„集群中的 Fleet agent 存储了用于连接到 Rancher çš„ kubeconfig。kubeconfig 包å«ä¸€ä¸ª `certificate-authority-data` å­—æ®µï¼Œè¯¥å­—æ®µåŒ…å« Rancher 使用的è¯ä¹¦çš„ CA。更改 CA 时，你需è¦æ›´æ–°æ­¤å—æ¥å…许 fleet-agent ä¿¡ä»» Rancher 使用的è¯ä¹¦ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md new file mode 100644 index 00000000000..40267b47065 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md @@ -0,0 +1,281 @@ +--- +title: å‡çº§ Cert-Manager +--- + +Rancher 使用 cert-manager 为 Rancher 高å¯ç”¨éƒ¨ç½²è‡ªåŠ¨ç”Ÿæˆå’Œç»­æœŸ TLS è¯ä¹¦ã€‚从 2019 秋季开始，cert-manager å‘生了以下的三个é‡è¦å˜æ›´ã€‚如果你在此时间段å‰åˆ›å»ºäº† Rancher 高å¯ç”¨éƒ¨ç½²ï¼Œè¯·è¿›è¡Œç›¸å…³æ“作。 + +1. [从 2019 å¹´ 11 月 1 日开始，Let's Encrypt 已阻止低于 0.8.0 çš„ cert-manager 实例。](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) +1. [Cert-manager æ­£åœ¨å¼ƒç”¨å’Œæ›¿æ¢ certificate.spec.acme.solvers 字段](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/)。此更改暂时没有确切的截止日期。 +1. [Cert-manager 正在弃用 `v1alpha1` API 和替æ¢å®ƒçš„ API 组](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/)。 + +为了帮助你应对这些å˜åŒ–，本文将： + +1. æä¾›å‡çº§ cert-manager 步骤的文档。 +1. é˜è¿° cert-manager API çš„å˜æ›´ï¼Œå¹¶æä¾› cert-manager 官方文档的链接，助你实现数æ®è¿ç§»ã€‚ + +:::note é‡è¦æ示： + +如果你è¦å°† cert-manager 从早于 1.5 的版本å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹[选项 C](#选项-cå‡çº§-15-åŠä»¥ä¸‹ç‰ˆæœ¬çš„-cert-manager) 中的步骤进行æ“作。请注æ„，你无需é‡æ–°å®‰è£… Rancher å³å¯æ‰§è¡Œæ­¤å‡çº§ã€‚ + +::: + +## å‡çº§ Cert-Manager + +以下说明中使用的命å空间是由当å‰å®‰è£…了 cert-manager 的命å空间决定的。如果它在 kube-system 中，在以下说明步骤中使用。你å¯ä»¥è¿è¡Œ `kubectl get pods --all-namespaces` æ¥éªŒè¯ï¼Œå¹¶æ£€æŸ¥ cert-manager-\* pods 列在哪个命å空间中。ä¸è¦æ›´æ”¹è¿è¡Œ cert-manager 的命å空间，å¦åˆ™å¯èƒ½ä¼šå‡ºçŽ°é”™è¯¯ã€‚ + +è¦å‡çº§ cert-manager，请éµå¾ªæ­¥éª¤æ“作。 + +### 选项 A：è”网å‡çº§ cert-manager + +
+ å•å‡»å±•å¼€ + +1. [备份现有资æº](https://cert-manager.io/docs/tutorials/backup/)： + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note é‡è¦æ示： + + 如果你从低于 0.11.0 的版本å‡çº§ï¼Œè¯·å°†æ‰€æœ‰å¤‡ä»½èµ„æºä¸Šçš„ apiVersion 从 `certmanager.k8s.io/v1alpha1` å‡çº§åˆ° `cert-manager.io/v1alpha2`。如果你需è¦åœ¨å…¶ä»–资æºä¸Šä½¿ç”¨ cert-manager 注释，请对其进行更新以å映新的 API 组。详情请å‚è§[附加注释å˜æ›´](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes)。 + + ::: + +1. [å¸è½½çŽ°æœ‰éƒ¨ç½²](https://cert-manager.io/docs/installation/uninstall/kubernetes/#uninstalling-with-helm)： + + ```plain + helm uninstall cert-manager + ``` + + 使用你安装的 vX.Y.Z 版本的链接删除 CustomResourceDefinition： + + ```plain + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + +1. å•ç‹¬å®‰è£… CustomResourceDefinition 资æºï¼š + + ```plain + kubectl apply --validate=false -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + + :::note + + 如果你è¿è¡Œçš„ Kubernetes 版本是 1.15 或更低版本，你需è¦åœ¨ä»¥ä¸Šçš„ `kubectl apply` 命令中添加 `--validate=false`。å¦åˆ™ä½ å°†çœ‹åˆ° cert-manager CRD 资æºä¸­çš„ `x-kubernetes-preserve-unknown-fields` 字段校验错误æ示。这是 kubectl 执行资æºæ ¡éªŒæ–¹å¼äº§ç”Ÿçš„良性错误。 + + ::: + +1. æ ¹æ®éœ€è¦ä¸º cert-manager 创建命å空间： + + ```plain + kubectl create namespace cert-manager + ``` + +1. 添加 Jetstack Helm 仓库： + + ```plain + helm repo add jetstack https://charts.jetstack.io + ``` + +1. æ›´æ–° Helm Chart 仓库本地缓存： + + ```plain + helm repo update + ``` + +1. 安装新版本的 cert-manager： + + ```plain + helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.11.0 + ``` + +1. [æ¢å¤å¤‡ä»½èµ„æº](https://cert-manager.io/docs/tutorials/backup/#restoring-resources)： + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### 选项 B：在离线环境中å‡çº§ Cert-Manager + +
+ å•å‡»å±•å¼€ + +### 先决æ¡ä»¶ + +在执行å‡çº§ä¹‹å‰ï¼Œå…ˆå°†æ‰€éœ€çš„容器镜åƒæ·»åŠ åˆ°ç§æœ‰é•œåƒä»“库中，并下载/渲染所需的 Kubernetes manifest 文件æ¥å‡†å¤‡ç¦»çº¿çŽ¯å¢ƒã€‚ + +1. å‚è§[准备ç§æœ‰é•œåƒä»“库](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md)指å—，将å‡çº§æ‰€éœ€çš„é•œåƒæŽ¨é€åˆ°é•œåƒä»“库。 + +1. 在å¯ä»¥è¿žæŽ¥äº’è”网的系统中，将 cert-manager 仓库添加到 Helm： + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + ``` + +1. 从 [Helm Chart 仓库](https://artifacthub.io/packages/helm/cert-manager/cert-manager)中获å–最新å¯ç”¨çš„ cert-manager Chart： + + ```plain + helm fetch jetstack/cert-manager --version v1.11.0 + ``` + +1. 使用安装 Chart 的选项æ¥æ¸²æŸ“ cert-manager 模æ¿ã€‚è®°ä½è¦è®¾ç½® `image.repository` 选项，以从你的ç§æœ‰é•œåƒä»“库拉å–é•œåƒã€‚æ­¤æ“ä½œä¼šåˆ›å»ºä¸€ä¸ªåŒ…å« Kubernetes manifest 文件的 `cert-manager` 目录。 + + Helm 3 命令如下： + + ```plain + helm template cert-manager ./cert-manager-v0.12.0.tgz --output-dir . \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + + Helm 2 命令如下： + + ```plain + helm template ./cert-manager-v0.12.0.tgz --output-dir . \ + --name cert-manager --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + +1. 下载新旧版 cert-manager 所需的 CRD 文件： + + ```plain + curl -L -o cert-manager-crd.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-0.12/deploy/manifests/00-crds.yaml + curl -L -o cert-manager/cert-manager-crd-old.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-X.Y/deploy/manifests/00-crds.yaml + ``` + +### 安装 cert-manager + +1. 备份现有资æºï¼š + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note é‡è¦æ示： + + 如果你从低于 0.11.0 的版本å‡çº§ï¼Œè¯·å°†æ‰€æœ‰å¤‡ä»½èµ„æºä¸Šçš„ apiVersion 从 `certmanager.k8s.io/v1alpha1` å‡çº§åˆ° `cert-manager.io/v1alpha2`。如果你需è¦åœ¨å…¶ä»–资æºä¸Šä½¿ç”¨ cert-manager 注释，请对其进行更新以å映新的 API 组。详情请å‚è§[附加注释å˜æ›´](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes)。 + + ::: + +1. 删除现有的 cert-manager 安装包： + + ```plain + kubectl -n cert-manager \ + delete deployment,sa,clusterrole,clusterrolebinding \ + -l 'app=cert-manager' -l 'chart=cert-manager-v0.5.2' + ``` + + 使用你安装的 vX.Y 版本的链接删除 CustomResourceDefinition： + + ```plain + kubectl delete -f cert-manager/cert-manager-crd-old.yaml + ``` + +1. å•ç‹¬å®‰è£… CustomResourceDefinition 资æºï¼š + + ```plain + kubectl apply -f cert-manager/cert-manager-crd.yaml + ``` + + :::note é‡è¦æ示： + + 如果你è¿è¡Œçš„ Kubernetes 版本是 1.15 或更低版本，你需è¦åœ¨ä»¥ä¸Šçš„ `kubectl apply` 命令中添加 `--validate=false`。å¦åˆ™ä½ å°†çœ‹åˆ° cert-manager CRD 资æºä¸­çš„ `x-kubernetes-preserve-unknown-fields` 字段校验错误æ示。这是 kubectl 执行资æºæ ¡éªŒæ–¹å¼äº§ç”Ÿçš„良性错误。 + + ::: + +1. 为 cert-manager 创建命å空间： + + ```plain + kubectl create namespace cert-manager + ``` + +1. 安装 cert-manager + + ```plain + kubectl -n cert-manager apply -R -f ./cert-manager + ``` + +1. [æ¢å¤å¤‡ä»½èµ„æº](https://cert-manager.io/docs/tutorials/backup/#restoring-resources)： + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### 选项 C：å‡çº§ 1.5 åŠä»¥ä¸‹ç‰ˆæœ¬çš„ cert-manager + +
+ å•å‡»å±•å¼€ + +以å‰ï¼Œè¦å‡çº§æ—§ç‰ˆæœ¬çš„ cert-manager，我们建议å¸è½½å¹¶é‡æ–°å®‰è£… Rancher。使用以下方法，你å¯ä»¥å‡çº§ cert-manager 而无需执行此é¢å¤–步骤，从而更好地ä¿æŠ¤ä½ çš„生产环境： + +1. 按照[安装指å—](https://cert-manager.io/docs/usage/cmctl/#installation)安装 `cmctl`（cert-manager CLI 工具）。 + +1. ç¡®ä¿æ‰€æœ‰ä»¥å·²å¼ƒç”¨çš„ API 版本存储在 etcd 中的 cert-manager 自定义资æºéƒ½è¿ç§»åˆ° v1： + + ``` + cmctl upgrade migrate-api-version + ``` + 有关详细信æ¯ï¼Œè¯·å‚阅 [API 版本è¿ç§»æ–‡æ¡£](https://cert-manager.io/docs/usage/cmctl/#migrate-api-version)。å¦è¯·å‚阅[å°† 1.5 å‡çº§åˆ° 1.6](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/) å’Œ[å°† 1.6 å‡çº§åˆ°åˆ° 1.7](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/)。 + +1. 正常使用 `helm upgrade` å°† cert-manager å‡çº§åˆ° 1.7.1。如果需è¦ï¼Œä½ å¯ä»¥ç›´æŽ¥ä»Žç‰ˆæœ¬ 1.5 转到 1.7。 + +1. 按照 Helm 教程[æ›´æ–°å‘布清å•çš„ API 版本](https://helm.sh/docs/topics/kubernetes_apis/#updating-api-versions-of-a-release-manifest)。Chart å‘布å称为 `release_name=rancher`，å‘布命å空​​间为 `release_namespace=cattle-system`。 + +1. 在解ç åŽçš„文件中，æœç´¢ `cert-manager.io/v1beta1` 并将其**替æ¢**为 `cert-manager.io/v1`。 + +1. 使用 `helm upgrade` 正常å‡çº§ Rancher。 + +
+ +### 验è¯éƒ¨ç½² + +安装完 cert-manager åŽï¼Œä½ å¯ä»¥é€šè¿‡æ£€æŸ¥ kube-system 命å空间中正在è¿è¡Œçš„ Pod æ¥éªŒè¯å®ƒæ˜¯å¦å·²æ­£ç¡®éƒ¨ç½²ï¼š + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +## Cert-Manager API å˜æ›´å’Œæ•°æ®è¿ç§» + +--- + +Rancher çŽ°åœ¨æ”¯æŒ cert-manager 1.6.2 å’Œ 1.7.1。推è使用 v1.7.x，因为 v 1.6.x 将在 2022 å¹´ 3 月 30 日结æŸç”Ÿå‘½å‘¨æœŸã€‚详情请å‚è§ [cert-manager 文档](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-安装-cert-manager)。有关将 cert-manager 从 1.5 å‡çº§åˆ° 1.6 的说明，请å‚è§ä¸Šæ¸¸çš„ [cert-manager 文档](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/)。有关将 cert-manager 从 1.6 å‡çº§åˆ° 1.7 的说明，请å‚è§ä¸Šæ¸¸çš„ [cert-manager 文档](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/)。 + +--- + +Cert-manager å·²ç»å¼ƒç”¨ `certificate.spec.acme.solvers` 字段，而且会在未æ¥çš„版本中放弃对该字段的支æŒã€‚ + +æ ¹æ® cert-manager 文档，v0.8 引入了é…ç½® ACME è¯ä¹¦èµ„æºçš„æ–°æ ¼å¼ã€‚具体æ¥è¯´ï¼Œå°±æ˜¯ç§»åŠ¨äº† challenge solver 字段。v0.9 新旧格å¼å‡æ”¯æŒã€‚请知悉，之åŽå‘布的新 cert-manager 版本会放弃对旧格å¼çš„支æŒã€‚Cert-Manager 文档建议你在更新åŽï¼Œå°† ACME é¢å‘者和è¯ä¹¦èµ„æºæ›´æ–°åˆ°æ–°æ ¼å¼ã€‚ + +如需了解å˜æ›´ç»†èŠ‚以åŠè¿ç§»è¯´æ˜Žï¼Œè¯·å‚è§[å°† Cert-Manager 从 v0.7 å‡çº§åˆ° v0.8](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/)。 + +v0.11 版本标志ç€åˆ é™¤å…ˆå‰ Cert-Manager 版本中使用的 v1alpha1 API，以åŠå°† API 组从 certmanager.k8s.io 更改到 cert-manager.io。 + +此外，我们已ä¸å†æ”¯æŒ v0.8 版本中已弃用的旧é…置格å¼ã€‚æ¢è¨€ä¹‹ï¼Œåœ¨å‡çº§åˆ° v0.11 之å‰ï¼Œä½ å¿…须先为 ACME å‘行者使用新的 solver æ ·å¼é…置格å¼ä½œä¸ºè¿‡æ¸¡ã€‚详情请å‚è§[å‡çº§åˆ° v0.8](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/)。 + +如需了解å˜æ›´ç»†èŠ‚以åŠè¿ç§»è¯´æ˜Žï¼Œè¯·å‚è§[å°† Cert-Manager 从 v0.10 å‡çº§åˆ° v0.11](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/)。 + +如需获得更多信æ¯ï¼Œè¯·å‚è§ [Cert-Manager å‡çº§](https://cert-manager.io/docs/installation/upgrading/)。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md new file mode 100644 index 00000000000..aa851f3a50c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md @@ -0,0 +1,126 @@ +--- +title: å‡çº§å’Œå›žæ»š Kubernetes +--- + +å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬çš„ Rancher 之åŽï¼Œä¸‹æ¸¸ Kubernetes 集群å¯ä»¥å‡çº§ä¸º Rancher 支æŒçš„最新的 Kubernetes 版本。 + +Rancher 使用 RKE（Rancher Kubernetes Engine）æ¥é¢„置和编辑 RKE 集群。有关为 RKE 集群é…ç½®å‡çº§ç­–略的更多信æ¯ï¼Œè¯·å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/)。 + + +## ç»è¿‡æµ‹è¯•çš„ Kubernetes 版本 + +Rancher 在å‘布新版本之å‰ï¼Œä¼šå¯¹å…¶ä¸Ž Kubernetes 的最新次è¦ç‰ˆæœ¬è¿›è¡Œæµ‹è¯•ï¼Œä»¥ç¡®ä¿å…¼å®¹æ€§ã€‚有关å„个 Rancher 版本测试了哪些 Kubernetes 版本的详细信æ¯ï¼Œè¯·å‚阅[支æŒç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/)。 + +## å‡çº§çš„å·¥ä½œåŽŸç† + +RKE v1.1.0 改å˜äº†é›†ç¾¤å‡çº§çš„æ–¹å¼ã€‚ + +在 [RKE 文档](https://rancher.com/docs/rke/latest/en/upgrades/how-upgrades-work)中，你将了解编辑或å‡çº§ RKE Kubernetes 集群时会å‘生的情况。 + + +## å‡çº§çš„最佳实践 + +在å‡çº§é›†ç¾¤çš„ Kubernetes 版本时，我们建议你： + +1. æ‹ä¸€å¼ å¿«ç…§ã€‚ +1. å¯åŠ¨ Kubernetes å‡çº§ã€‚ +1. 如果å‡çº§å¤±è´¥ï¼Œè¯·å°†é›†ç¾¤æ¢å¤åˆ°å‡çº§å‰çš„ Kubernetes 版本。这å¯ä»¥é€šè¿‡é€‰æ‹©**æ¢å¤ etcd å’Œ Kubernetes 版本**选项æ¥å®žçŽ°ã€‚在æ¢å¤ etcd å¿«ç…§ 之å‰ï¼Œè¿™ä¼šå°†ä½ çš„集群æ¢å¤åˆ°å‡çº§å‰çš„ kubernetes 版本。 + +æ¢å¤æ“作将在ä¸å¤„于å¥åº·æˆ– active 状æ€çš„集群上è¿è¡Œã€‚ + +## å‡çº§ Kubernetes 版本 + +:::note 先决æ¡ä»¶ï¼š + +- 以下选项适用于 [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../pages-for-subheaders/launch-kubernetes-with-rancher.md)å’Œ[注册的 K3s Kubernetes 集群](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#已注册-rke2-å’Œ-k3s-集群的附加功能)。 +- 以下选项也适用于导入且已注册的 RKE2 集群。如果你从外部云平å°å¯¼å…¥é›†ç¾¤ä½†ä¸æ³¨å†Œï¼Œä½ å°†æ— æ³•åœ¨ Rancher 中å‡çº§ Kubernetes 版本。 +- 在å‡çº§ Kubernetes 之å‰ï¼Œå…ˆ[备份你的集群](../../pages-for-subheaders/backup-restore-and-disaster-recovery.md)。 + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è¦å‡çº§çš„集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. 从 **Kubernetes 版本** 下拉列表中，选择è¦ç”¨äºŽé›†ç¾¤çš„ Kubernetes 版本。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：已开始为集群å‡çº§ Kubernetes。 + +## 回滚 + +ä½ å¯ä»¥å°†é›†ç¾¤æ¢å¤åˆ°ä½¿ç”¨å…ˆå‰ Kubernetes 版本的备份。有关详细信æ¯ï¼Œè¯·å‚阅： + +- [备份集群](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md#快照工作原ç†) +- [使用备份æ¢å¤é›†ç¾¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md#使用快照æ¢å¤é›†ç¾¤) + +## é…ç½®å‡çº§ç­–ç•¥ + +从 RKE v1.1.0 开始，我们æ供了é¢å¤–çš„å‡çº§é€‰é¡¹ï¼Œè®©ä½ æ›´ç²¾ç»†åœ°æŽ§åˆ¶å‡çº§è¿‡ç¨‹ã€‚如果满足[æ¡ä»¶å’Œè¦æ±‚](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability)，你å¯ä»¥ä½¿ç”¨è¿™äº›é€‰é¡¹ï¼Œä»Žè€Œåœ¨é›†ç¾¤å‡çº§æœŸé—´ç»´æŒåº”用的å¯ç”¨æ€§ã€‚ + +ä½ å¯ä»¥åœ¨ Rancher UI 中é…ç½®å‡çº§ç­–略，也å¯ä»¥é€šè¿‡ç¼–辑 `cluster.yml` æ¥é…置策略。编辑 `cluster.yml` å¯ä»¥é…置更多高级选项。 + +### 在 Rancher UI 中é…置最大ä¸å¯ç”¨çš„ Worker 节点 + +ä½ å¯ä»¥åœ¨ Rancher UI 中é…ç½®ä¸å¯ç”¨ worker 节点的最大数é‡ã€‚在集群å‡çº§æœŸé—´ï¼Œworker 节点将按此大å°æ‰¹é‡å‡çº§ã€‚ + +默认情况下，ä¸å¯ç”¨ worker 节点的最大数é‡ä¸ºæ‰€æœ‰ worker 节点的 10%。此数字å¯ä»¥é…置为百分比或整数。当定义为百分比时，批大å°ä¼šè¢«å››èˆäº”入到最近的节点，最å°ä¸ºä¸€ä¸ªèŠ‚点。 + +è¦æ›´æ”¹ worker 节点的默认数é‡æˆ–百分比： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è¦å‡çº§çš„集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. 在**å‡çº§ç­–ç•¥**选项å¡ä¸­ï¼Œè¾“å…¥ **Worker 并å‘**作为固定的数字或百分比。你å¯ä»¥é€šè¿‡å°†é›†ç¾¤ä¸­çš„节点数å‡åŽ»æœ€å¤§ä¸å¯ç”¨èŠ‚点数æ¥èŽ·å–该数字。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：集群更新为使用新的å‡çº§ç­–略。 + +### 使用 Rancher UI 在å‡çº§æœŸé—´å¯ç”¨èŠ‚点清空 + +默认情况下，RKE 会在å‡çº§ä¹‹å‰[å°é”](https://kubernetes.io/docs/concepts/architecture/nodes/#manual-node-administration)æ¯ä¸ªèŠ‚点。默认情况下，[清空](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/)会在å‡çº§æœŸé—´è¢«ç¦ç”¨ã€‚如果在集群é…置中å¯ç”¨äº†æ¸…空​​，RKE 将在å‡çº§ä¹‹å‰å¯¹èŠ‚点进行å°é”和清空。 + +è¦åœ¨é›†ç¾¤å‡çº§æœŸé—´æ¸…空æ¯ä¸ªèŠ‚点： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è¦å¯ç”¨èŠ‚点清空的集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. å•å‡» **â‹® > 编辑**。 +1. 在**å‡çº§ç­–ç•¥**选项å¡ä¸­ï¼Œè½¬åˆ°**清空节点**字段并å•å‡»**是**。controlplane å’Œ worker 节点的清空是å•ç‹¬é…置的。 +1. é…置如何删除 pod 的选项。有关æ¯ä¸ªé€‰é¡¹çš„详细信æ¯ï¼Œè¯·å‚阅[本节](../../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md#激进和安全的清空选项)。 +1. （å¯é€‰ï¼‰é…置宽é™æœŸã€‚宽é™æœŸæ˜¯ç»™æ¯ä¸ª pod 进行清ç†çš„超时时间，能让 pod 有机会优雅地退出。Pod å¯èƒ½éœ€è¦å®Œæˆä»»ä½•æœªå®Œæˆçš„请求ã€å›žæ»šäº‹åŠ¡æˆ–将状æ€ä¿å­˜åˆ°æŸäº›å¤–部存储。如果该值为负数，将使用 pod 中指定的默认值。 +1. （å¯é€‰ï¼‰é…置超时，这是在清空放弃之å‰åº”该继续等待的时间。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：集群更新为使用新的å‡çº§ç­–略。 + +:::note + +ç›®å‰å­˜åœ¨ä¸€ä¸ª[已知问题](https://github.com/rancher/rancher/issues/25478)，å³ä½¿ etcd å’Œ controlplane 正在被清空， Rancher UI 也ä¸ä¼šå°†å®ƒä»¬çš„状æ€æ˜¾ç¤ºä¸º drained。 + +::: + +### 在å‡çº§æœŸé—´ç»´æŠ¤åº”用的å¯ç”¨æ€§ + +_从 RKE v1.1.0 èµ·å¯ç”¨_ + +在 [RKE 文档](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability/)中，你将了解在å‡çº§é›†ç¾¤æ—¶é˜²æ­¢åº”用åœæœºçš„è¦æ±‚。 + +### 在 cluster.yml 中é…ç½®å‡çº§ç­–ç•¥ + +你通过编辑 `cluster.yml` æ¥èŽ·å¾—更高级的å‡çº§ç­–ç•¥é…置选项。 + +有关详细信æ¯ï¼Œè¯·å‚阅 RKE 文档中的[é…ç½®å‡çº§ç­–ç•¥](https://rancher.com/docs/rke/latest/en/upgrades/configuring-strategy)。这部分还包括一个用于é…ç½®å‡çº§ç­–略的示例 `cluster.yml`。 + +## 故障排除 + +如果å‡çº§åŽèŠ‚点没有出现，`rke up` 命令会出错。 + +如果ä¸å¯ç”¨èŠ‚点的数é‡è¶…过é…置的最大值，则ä¸ä¼šè¿›è¡Œå‡çº§ã€‚ + +如果å‡çº§åœæ­¢ï¼Œä½ å¯èƒ½éœ€è¦ä¿®å¤ä¸å¯ç”¨èŠ‚点或将其从集群中删除，然åŽæ‰èƒ½ç»§ç»­å‡çº§ã€‚ + +失败的节点å¯èƒ½å¤„于许多ä¸åŒçš„状æ€ï¼š + +- 关机 +- ä¸å¯ç”¨ +- 用户在å‡çº§è¿‡ç¨‹ä¸­æ¸…空了节点，因此节点上没有 kubelet +- å‡çº§æœ¬èº«å¤±è´¥ + +如果在å‡çº§è¿‡ç¨‹ä¸­è¾¾åˆ°æœ€å¤§ä¸å¯ç”¨èŠ‚点数，Rancher 的下游集群将åœç•™åœ¨æ›´æ–°ä¸­çš„状æ€ï¼Œå¹¶ä¸”ä¸ä¼šç»§ç»­å‡çº§å…¶ä»– controlplane 节点。它将继续评估ä¸å¯ç”¨çš„节点集，以防其中一个节点å˜å¾—å¯ç”¨ã€‚如果无法修å¤èŠ‚点，则必须移除节点æ‰èƒ½ç»§ç»­å‡çº§ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md new file mode 100644 index 00000000000..2d9c0f00918 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md @@ -0,0 +1,91 @@ +--- +title: 在ä¸å‡çº§ Rancher 的情况下å‡çº§ Kubernetes +--- + +RKE 元数æ®åŠŸèƒ½å…许你在新版本 Kubernetes å‘布åŽç«‹å³ä¸ºé›†ç¾¤é…置新版本，而无需å‡çº§ Rancher。此功能对于使用 Kubernetes çš„è¡¥ä¸ç‰ˆæœ¬éžå¸¸æœ‰ç”¨ï¼Œä¾‹å¦‚ï¼Œåœ¨åŽŸæœ¬æ”¯æŒ Kubernetes v1.14.6 çš„ Rancher Server 版本中，将 Kubernetes å‡çº§åˆ° v1.14.7。 + +:::note + +Kubernetes API å¯ä»¥åœ¨æ¬¡è¦ç‰ˆæœ¬ä¹‹é—´æ›´æ”¹ã€‚因此，我们ä¸æ”¯æŒå¼•å…¥ Kubernetes 次è¦ç‰ˆæœ¬ï¼Œä¾‹å¦‚在 Rancher æ”¯æŒ v1.14 的情况下引入 v1.15。在这ç§æƒ…况下，你需è¦å‡çº§ Rancher 以添加对 Kubernetes 次è¦ç‰ˆæœ¬çš„支æŒã€‚ + +::: + +Rancher çš„ Kubernetes 元数æ®åŒ…å« Rancher 用于é…ç½® [RKE 集群](../../pages-for-subheaders/launch-kubernetes-with-rancher.md)çš„ Kubernetes 版本信æ¯ã€‚Rancher 会定期åŒæ­¥æ•°æ®å¹¶ä¸º **系统镜åƒ**ã€**æœåŠ¡é€‰é¡¹**å’Œ**æ’件模æ¿**创建自定义资æºå®šä¹‰ (CRD)。因此，当新的 Kubernetes 版本与 Rancher Server 版本兼容时，Kubernetes 元数æ®å¯ä»¥ä½¿ Rancher 使用新版本æ¥é…置集群。元数æ®æ¦‚述了 [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) 用于部署å„ç§ Kubernetes 版本的信æ¯ã€‚ + +下表æ述了å—周期性数æ®åŒæ­¥å½±å“çš„ CRD。 + +:::note + +åªæœ‰ç®¡ç†å‘˜å¯ä»¥ç¼–è¾‘å…ƒæ•°æ® CRD。除éžæ˜Žç¡®éœ€è¦ï¼Œå¦åˆ™å»ºè®®ä¸è¦æ›´æ–°çŽ°æœ‰å¯¹è±¡ã€‚ + +::: + +| èµ„æº | æè¿° | Rancher API URL | +|----------|-------------|-----------------| +| ç³»ç»Ÿé•œåƒ | 用于通过 RKE 部署 Kubernetes 集群的系统镜åƒåˆ—表。 | `/v3/rkek8ssystemimages` | +| æœåŠ¡é€‰é¡¹ | 传递给 Kubernetes 组件的默认选项，例如 `kube-api`ã€`scheduler`ã€`kubelet`ã€`kube-proxy` å’Œ `kube-controller-manager` | `/v3/rkek8sserviceoptions` | +| æ’ä»¶æ¨¡æ¿ | 用于部署æ’件组件的 YAML 定义，例如 Canalã€Calicoã€Flannelã€Weaveã€Kube-dnsã€CoreDNSã€`metrics-server`ã€`nginx-ingress` | `/v3/rkeaddons` | + +管ç†å‘˜å¯ä»¥é€šè¿‡é…ç½® RKE 元数æ®è®¾ç½®æ¥æ‰§è¡Œä»¥ä¸‹æ“作： + +- 刷新 Kubernetes 元数æ®ã€‚适用于有新的 Kubernetes è¡¥ä¸ç‰ˆæœ¬å‘布，而用户希望在ä¸å‡çº§ Rancher 的情况下为集群é…置最新版本的 Kubernetes 的情景。 +- 更改 Rancher 用于åŒæ­¥å…ƒæ•°æ®çš„ URL。适用于è¦è®© Rancher 从本地åŒæ­¥è€Œä¸æ˜¯ä¸Ž GitHub åŒæ­¥çš„情况。这在离线环境下éžå¸¸æœ‰ç”¨ã€‚ +- 防止 Rancher 自动åŒæ­¥å…ƒæ•°æ®ã€‚è¿™å¯ä»¥é˜²æ­¢åœ¨ Rancher 中使用新的/ä¸å—支æŒçš„ Kubernetes 版本。 + +### 刷新 Kubernetes å…ƒæ•°æ® + +默认情况下，管ç†å‘˜æˆ–具有**管ç†é›†ç¾¤é©±åŠ¨**[全局角色](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)的用户，å¯ä»¥åˆ·æ–° Kubernetes 元数æ®ã€‚ + +è¦å¼ºåˆ¶ Rancher 刷新 Kubernetes 元数æ®ï¼Œå¯ä»¥æ‰§è¡Œæ‰‹åŠ¨åˆ·æ–°æ“作： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 +1. å•å‡»**刷新 Kubernetes 元数æ®**。 + +ä½ å¯ä»¥å°† `refresh-interval-minutes` 设置为 `0`（è§ä¸‹æ–‡ï¼‰ï¼Œå°† Rancher é…置为仅在需è¦æ—¶åˆ·æ–°å…ƒæ•°æ®ï¼Œå¹¶åœ¨éœ€è¦æ—¶ä½¿ç”¨æ­¤æŒ‰é’®æ‰‹åŠ¨æ‰§è¡Œå…ƒæ•°æ®åˆ·æ–°ã€‚ + +### é…置元数æ®åŒæ­¥ + +:::caution + +åªæœ‰ç®¡â€‹â€‹ç†å‘˜å¯ä»¥æ›´æ”¹è¿™äº›è®¾ç½®ã€‚ + +::: + +RKE 元数æ®çš„é…置控制 Rancher åŒæ­¥å…ƒæ•°æ®çš„频率以åŠä»Žä½•å¤„下载数æ®ã€‚ä½ å¯ä»¥é€šè¿‡ Rancher UI 或通过 Rancher API 端点 `v3/settings/rke-metadata-config` é…置元数æ®ã€‚ + +元数æ®çš„é…置方å¼å–决于 Rancher 版本。 + +è¦åœ¨ Rancher 中编辑元数æ®é…置： + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. 转到 **rke-metadata-config**。å•å‡» **â‹® > 编辑设置**。 +1. ä½ å¯ä»¥é€‰æ‹©å¡«å†™ä»¥ä¸‹å‚数： + +- `refresh-interval-minutes`：Rancher 等待åŒæ­¥å…ƒæ•°æ®çš„时间。如果è¦ç¦ç”¨å®šæœŸåˆ·æ–°ï¼Œè¯·å°† `refresh-interval-minutes` 设置为 0。 +- `url`：Rancher 从中获å–æ•°æ®çš„ HTTP 路径。该路径必须是 JSON 文件的直接路径。例如，Rancher v2.4 的默认 URL 是 `https://releases.rancher.com/kontainer-driver-metadata/release-v2.4/data.json`。 +1. å•å‡»**ä¿å­˜**。 + +如果你没有离线设置，则无需指定 Rancher 获å–元数æ®çš„ URL，因为默认是从 [Rancher çš„å…ƒæ•°æ® Git 仓库获å–](https://github.com/rancher/kontainer-driver-metadata/blob/dev-v2.5/data/data.json)的。 + +但是，如果你有[离线设置](#离线设置)需求，你需è¦å°† Kubernetes 元数æ®ä»“库镜åƒåˆ° Rancher å¯ç”¨çš„ä½ç½®ã€‚然åŽï¼Œä½ éœ€è¦æ›´æ”¹ URL æ¥æŒ‡å‘ JSON 文件的新ä½ç½®ã€‚ + +### 离线设置 + +Rancher Server 会定期刷新 `rke-metadata-config` æ¥ä¸‹è½½æ–°çš„ Kubernetes 版本元数æ®ã€‚有关 Kubernetes å’Œ Rancher 版本的兼容性表，请å‚阅[æœåŠ¡æ¡æ¬¾](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.2.8/)。 + +如果你使用离线设置，则å¯èƒ½æ— æ³•ä»Ž Rancher çš„ Git 仓库自动定期刷新 Kubernetes 元数æ®ã€‚在这ç§æƒ…况下，应该ç¦ç”¨å®šæœŸåˆ·æ–°ä»¥é˜²æ­¢åœ¨æ—¥å¿—中显示相关错误。或者，你å¯ä»¥é…置元数æ®ï¼Œä»¥ä¾¿ Rancher 与本地的 RKE 元数æ®å‰¯æœ¬è¿›è¡ŒåŒæ­¥ã€‚ + +è¦å°† Rancher 与 RKE 元数æ®çš„本地镜åƒåŒæ­¥ï¼Œç®¡ç†å‘˜éœ€è¦é…ç½® `rke-metadata-config` æ¥æŒ‡å‘é•œåƒã€‚详情请å‚考[é…置元数æ®åŒæ­¥](#é…置元数æ®åŒæ­¥) + +在将新的 Kubernetes 版本加载到 Rancher Server 中之åŽï¼Œéœ€è¦æ‰§è¡Œå…¶ä»–步骤æ‰èƒ½ä½¿ç”¨å®ƒä»¬å¯åŠ¨é›†ç¾¤ã€‚Rancher 需è¦è®¿é—®æ›´æ–°çš„系统镜åƒã€‚虽然åªæœ‰ç®¡ç†å‘˜å¯ä»¥æ›´æ”¹å…ƒæ•°æ®è®¾ç½®ï¼Œä½†ä»»ä½•ç”¨æˆ·éƒ½å¯ä»¥ä¸‹è½½ Rancher 系统镜åƒå¹¶ä¸ºé•œåƒå‡†å¤‡ç§æœ‰å®¹å™¨é•œåƒä»“库。 + +è¦ä¸‹è½½ç§æœ‰é•œåƒä»“库的系统镜åƒï¼š + +1. 点击左上角的 **☰**。 +1. 点击左侧导航底部的**简介**。 +1. 下载适用于 Linux 或 Windows æ“作系统的镜åƒã€‚ +1. 下载 `rancher-images.txt`。 +1. 使用[离线环境安装](other-installation-methods/air-gapped-helm-cli-install/publish-images.md)时使用的步骤准备ç§æœ‰é•œåƒä»“库，但ä¸è¦ä½¿ç”¨å‘布页é¢ä¸­çš„ `rancher-images.txt`，而是使用上一个步骤中获å–的文件。 + +**结果**：Rancher 的离线安装现在å¯ä»¥åŒæ­¥ Kubernetes 元数æ®ã€‚如果你在å‘布新版本的 Kubernetes 时更新了ç§æœ‰é•œåƒä»“库，你å¯ä»¥ä½¿ç”¨æ–°ç‰ˆæœ¬é…置集群，而无需å‡çº§ Rancher。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/overview.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/overview.md new file mode 100644 index 00000000000..903c8c28bd2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/overview.md @@ -0,0 +1,65 @@ +--- +title: 概述 +--- + +Rancher 是一个为使用容器的公å¸æ‰“造的容器管ç†å¹³å°ã€‚Rancher 使得开å‘者å¯ä»¥éšå¤„è¿è¡Œ Kubernetes（Run Kubernetes Everywhere），满足 IT 需求规范，赋能 DevOps 团队。 + +## Run Kubernetes Everywhere + +Kubernetes å·²ç»æˆä¸ºå®¹å™¨ç¼–排标准。现在，大多数云和虚拟化æ供商都æ供容器编排æœåŠ¡ã€‚Rancher 用户å¯ä»¥é€‰æ‹©ä½¿ç”¨ Rancher Kubernetes Engine（RKE）或云 Kubernetes æœåŠ¡ï¼ˆä¾‹å¦‚ GKEã€AKS å’Œ EKS）创建 Kubernetes 集群，还å¯ä»¥å¯¼å…¥å’Œç®¡ç†ä½¿ç”¨ä»»ä½• Kubernetes å‘行版或安装程åºåˆ›å»ºçš„现有 Kubernetes 集群。 + +## 满足 IT 需求规范 + +Rancher 支æŒå¯¹å…¶æŽ§åˆ¶çš„所有 Kubernetes 集群进行集中认è¯ã€è®¿é—®æŽ§åˆ¶å’Œç›‘控。例如，你å¯ä»¥ï¼š + +- 使用你的 Active Directory 凭è¯è®¿é—®ç”±äº‘æ供商（例如 GKE）托管的 Kubernetes 集群。 +- 设置所有用户ã€ç»„ã€é¡¹ç›®ã€é›†ç¾¤å’Œäº‘æœåŠ¡çš„æƒé™æŽ§åˆ¶ç­–略和安全策略。 +- 一站å¼æŸ¥çœ‹ Kubernetes 集群的è¿è¡ŒçŠ¶å†µå’Œå®¹é‡ã€‚ + +## 赋能 DevOps 团队 + +Rancher 为 DevOps 工程师æ供简å•ç›´æŽ¥çš„用户界é¢ï¼Œä»¥ç®¡ç†å…¶åº”用负载。用户ä¸éœ€è¦å¯¹ Kubernetes 有éžå¸¸æ·±å…¥çš„了解，å³å¯ä½¿ç”¨ Rancher。Rancher 应用商店包å«ä¸€å¥—实用的 DevOps å¼€å‘工具。Rancher 获得了多ç§äº‘原生生æ€ç³»ç»Ÿäº§å“的认è¯ï¼ŒåŒ…括安全工具ã€ç›‘控系统ã€å®¹å™¨é•œåƒä»“库ã€å­˜å‚¨å’Œç½‘络驱动等。 + +下图讲述了 Rancher 在 IT 管ç†å›¢é˜Ÿå’Œ DevOps å¼€å‘团队之间扮演的角色。DevOps 团队把他们的应用部署在他们选择的公有云或ç§æœ‰äº‘上。IT 管ç†å‘˜è´Ÿè´£æŸ¥çœ‹å¹¶ç®¡ç†ç”¨æˆ·ã€é›†ç¾¤ã€äº‘æœåŠ¡çš„æƒé™ã€‚ + +![å¹³å°](/img/platform.png) + +## Rancher API Server 的功能 + +Rancher API Server æ˜¯åŸºäºŽåµŒå…¥å¼ Kubernetes API Server å’Œ etcd æ•°æ®åº“建立的，它æ供了以下功能： + +### 授æƒå’ŒåŸºäºŽè§’色的æƒé™æŽ§åˆ¶ï¼ˆRBAC） + +- **用户管ç†**：Rancher API Server 除了管ç†æœ¬åœ°ç”¨æˆ·ï¼Œè¿˜[管ç†ç”¨æˆ·ç”¨æ¥è®¿é—®å¤–部æœåŠ¡æ‰€éœ€çš„认è¯ä¿¡æ¯](../pages-for-subheaders/authentication-config.md)，如登录 Active Directory å’Œ GitHub 所需的账å·å¯†ç ã€‚ +- **授æƒ**：Rancher API Server å¯ä»¥ç®¡ç†[访问控制策略](../pages-for-subheaders/manage-role-based-access-control-rbac.md)å’Œ[安全策略](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +### 使用 Kubernetes 的功能 + +- **é…ç½® Kubernetes 集群**：Rancher API Server å¯ä»¥åœ¨å·²æœ‰èŠ‚点上[é…ç½® Kubernetes](../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)，或进行 [Kubernetes 版本å‡çº§](installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 +- **管ç†åº”用商店**：Rancher 支æŒä½¿ç”¨ [Helm Chart 应用商店](../pages-for-subheaders/helm-charts-in-rancher.md)实现轻æ¾é‡å¤éƒ¨ç½²åº”用。 +- **管ç†é¡¹ç›®**：项目由集群中多个命å空间和访问控制策略组æˆï¼Œæ˜¯ Rancher 中的一个概念，Kubernetes 中并没有这个概念。你å¯ä»¥ä½¿ç”¨é¡¹ç›®å®žçŽ°ä»¥ç»„为å•ä½ï¼Œç®¡ç†å¤šä¸ªå‘½å空间，并进行 Kubernetes 相关æ“作。Rancher UI æ供用于[项目管ç†](../pages-for-subheaders/manage-projects.md)å’Œ[项目内应用管ç†](../pages-for-subheaders/kubernetes-resources-setup.md)的功能。 +- **Fleet æŒç»­äº¤ä»˜**：在 Rancher 中，你å¯ä»¥ä½¿ç”¨ [Fleet æŒç»­äº¤ä»˜](../pages-for-subheaders/fleet-gitops-at-scale.md)将应用程åºä»Ž Git 仓库部署到目标下游 Kubernetes 集群，无需任何手动æ“作。 +- **Istio**：[Rancher 与 Istio 集æˆ](../pages-for-subheaders/istio.md)，使得管ç†å‘˜æˆ–集群所有者å¯ä»¥å°† Istio 交给开å‘者，然åŽå¼€å‘者使用 Istio 执行安全策略，排查问题，或为è“绿部署，金ä¸é›€éƒ¨ç½²ï¼Œå’Œ A/B 测试进行æµé‡ç®¡ç†ã€‚ + +### é…置云基础设施 + +- **åŒæ­¥èŠ‚点信æ¯**：Rancher API Server å¯ä»¥åŒæ­¥æ‰€æœ‰é›†ç¾¤ä¸­å…¨éƒ¨[节点](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md)çš„ä¿¡æ¯ã€‚ +- **é…置云基础设施**：如果你为 Rancher é…置了云æ供商，Rancher å¯ä»¥åœ¨äº‘端动æ€é…ç½®[新节点](../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)å’Œ[æŒä¹…化存储](../pages-for-subheaders/create-kubernetes-persistent-storage.md)。 + +### æŸ¥çœ‹é›†ç¾¤ä¿¡æ¯ + +- **日志管ç†**：Rancher å¯ä»¥ä¸Žå¤šç§ Kubernetes 集群之外的主æµæ—¥å¿—管ç†å·¥å…·é›†æˆã€‚ +- **监控**：你å¯ä»¥ä½¿ç”¨ Rancher，通过业界领先并开æºçš„ Prometheus æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚ +- **å‘Šè­¦**：为了ä¿è¯é›†ç¾¤å’Œåº”用的正常è¿è¡Œï¼Œæ高公å¸çš„生产效率，你需è¦éšæ—¶äº†è§£é›†ç¾¤å’Œé¡¹ç›®çš„计划内和éžè®¡åˆ’事件。 + +## 使用 Rancher 编辑下游集群 + +对于已有集群而言，å¯æ供的选项和设置å–决于你é…置集群的方法。例如，åªæœ‰[通过 RKE å¯åŠ¨](../pages-for-subheaders/launch-kubernetes-with-rancher.md)的集群æ‰æœ‰å¯ç¼–辑的**集群选项**。 + +使用 Rancher 创建集群åŽï¼Œé›†ç¾¤ç®¡ç†å‘˜å¯ä»¥ç®¡ç†é›†ç¾¤æˆå‘˜ï¼Œç®¡ç†èŠ‚点池，或进行[其他æ“作](../pages-for-subheaders/cluster-configuration.md)。 + +下表总结了æ¯ä¸€ç§ç±»åž‹çš„集群和对应的å¯ç¼–辑的选项和设置： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md new file mode 100644 index 00000000000..2fc1ce0bd33 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md @@ -0,0 +1,6 @@ +--- +title: Rancher AWS Marketplace 快速入门 +description: 使用 Amazon EKS 部署 Rancher Server。 +--- + +Amazon Elastic Kubernetes Service (EKS) å¯ä»¥å¿«é€Ÿ[å°† Rancher 部署到 Amazon Web Services (AWS)](https://documentation.suse.com/trd/kubernetes/single-html/gs_rancher_aws-marketplace/)。详情请å‚è§æˆ‘们的 [Amazon Marketplace 列表](https://aws.amazon.com/marketplace/pp/prodview-go7ent7goo5ae)。观看 [demo](https://youtu.be/9dznJ7Ons0M)，了解 AWS Marketplace SUSE Rancher 设置的演练。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws.md new file mode 100644 index 00000000000..cb47333ce32 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws.md @@ -0,0 +1,95 @@ +--- +title: Rancher AWS å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher AWS 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 AWS çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Amazon AWS 会产生费用。 + +::: + +- [Amazon AWS è´¦å·](https://aws.amazon.com/account/): 用于创建部署 Rancher Server å’Œ Kubernetes 的资æºã€‚ +- [Amazon AWS 访问密钥](https://docs.aws.amazon.com/general/latest/gr/managing-aws-access-keys.html)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹ç›¸å…³æŒ‡å—。 +- [已创建 IAM ç­–ç•¥](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start)：定义附加此策略的账å·æ‰€å…·æœ‰çš„æƒé™ã€‚ +- [Terraform](https://www.terraform.io/downloads.html): 用于在 Amazon AWS 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + +### IAM 策略示例 + +AWS 模å—åªåˆ›å»ºä¸€ä¸ª EC2 密钥对ã€ä¸€ä¸ª EC2 安全组和一个 EC2 实例。以下是一个简å•çš„策略： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:*", + "Resource": "*" + } + ] +} +``` + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/aws` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 AWS 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + + - `aws_access_key` - 替æ¢ä¸º Amazon AWS 访问密钥 + - `aws_secret_key` - 替æ¢ä¸º Amazon AWS Secret 密钥 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [AWS Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/aws) 了解更多信æ¯ã€‚ + 建议包括： + + - `aws_region` - Amazon AWS 区域。AWS 的默认区域 (`us-east-1`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - EC2 使用的实例规格，最å°è§„格为 `t3a.medium` 。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `t3a.large` 或 `t3a.xlarge`。 + - `add_windows_node` - 如果设为 true，一个é¢å¤–çš„ Windows worker 节点会添加到工作负载集群中。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/aws` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +##### 结果 + +两个 Kubernetes 集群已部署到你的 AWS 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +## åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/aws` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/azure.md new file mode 100644 index 00000000000..33a8d411ba0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/azure.md @@ -0,0 +1,81 @@ +--- +title: Rancher Azure å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher Azure 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 Azure çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Microsoft Azure 会产生费用。 + +::: + +- [Microsoft Azure è´¦å·](https://azure.microsoft.com/en-us/free/)：用于创建部署 Rancher å’Œ Kubernetes 的资æºã€‚ +- [Microsoft Azure 订阅](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹å¦‚何创建 Microsoft Azure 订阅。 +- [Micsoroft Azure 租户](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant)：访问此链接并å‚考教程以创建 Microsoft Azure 租户。 +- [Microsoft Azure 客户端 ID/密文](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal)：访问此链接并å‚考教程以创建 Microsoft Azure 客户端和密文。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Microsoft Azure 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/azure` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 Azure 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `azure_subscription_id` - 替æ¢ä¸º Microsoft Azure 订阅 ID。 + - `azure_client_id` - 替æ¢ä¸º Microsoft Azure 客户端 ID。 + - `azure_client_secret` - 替æ¢ä¸º Microsoft Azure 客户端密文。 + - `azure_tenant_id` - 替æ¢ä¸º Microsoft Azure 租户 ID。 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [Azure Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/azure) 了解更多信æ¯ã€‚建议包括： + - `azure_location` - Microsoft Azure 区域。Azure 的默认区域 (`East US`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - 使用的计算实例大å°ï¼Œæœ€å°è§„格为 `Standard_DS2_v2`。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `Standard_DS2_v3` 或 `Standard_DS3_v2`。 + - `add_windows_node` - 如果设为 true，一个é¢å¤–çš„ Windows worker 节点会添加到工作负载集群中。 + - `windows_admin_password` - Windows worker 节点管ç†å‘˜çš„å¯†ç  + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/azure` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 Azure 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/azure` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md new file mode 100644 index 00000000000..e87f17c26d0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md @@ -0,0 +1,74 @@ +--- +title: Rancher DigitalOcean å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher DigitalOcean 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 DigitalOcean çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 DigitalOcean 会产生费用。 + +::: + +- [DigitalOcean è´¦å·](https://www.digitalocean.com)：用于è¿è¡ŒæœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ +- [DigitalOcean 访问密钥](https://www.digitalocean.com/community/tutorials/how-to-create-a-digitalocean-space-and-api-key)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥åˆ›å»ºä¸€ä¸ªã€‚ +- [Terraform](https://www.terraform.io/downloads.html)：用于在 DigitalOcean 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/do` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 DigitalOcean 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `do_token` - 替æ¢ä¸º DigitalOcean 访问密钥 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [DO Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/do) 了解更多信æ¯ã€‚建议包括： + - `do_region` - DigitalOcean 区域。DigitalOcean 的默认区域ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `droplet_size` - 使用的计算实例规格，最å°è§„格为`s-2vcpu-4gb`。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `s-4vcpu-8gb`。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/do` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 DigitalOcean 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/do` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md new file mode 100644 index 00000000000..c7316352cc2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md @@ -0,0 +1,106 @@ +--- +title: Rancher Equinix Metal 快速入门 +--- + +## 本章节引导你： + +- é…ç½® Equinix Metal server +- 安装 Rancher 2.x +- 创建你的第一个集群 +- 部署一个 Nginx 应用 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。ä¸å»ºè®®å°† Docker 安装用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 快速入门概述 + +本指å—划分为ä¸åŒä»»åŠ¡ï¼Œä»¥ä¾¿äºŽä½¿ç”¨ã€‚ + +
+ +## 先决æ¡ä»¶ + +- [Equinix Metal è´¦å·](https://metal.equinix.com/developers/docs/accounts/users/) +- [Equinix Metal 项目](https://metal.equinix.com/developers/docs/accounts/projects/) + + +### 1. é…ç½® Equinix Metal 主机 + +开始部署 Equinix Metal 主机。你å¯ä»¥ä½¿ç”¨ Equinix Metal 控制å°ã€CLI 或 API æ¥é…ç½® Equinix Metal Server。如果你需è¦äº†è§£æ¯ç§ Deployment 类型的说明，请å‚è§ [Equinix Metal 部署](https://metal.equinix.com/developers/docs/deploy/on-demand/)ã€‚ä»¥ä¸‹é“¾æŽ¥ä»‹ç» Equinix Metal Server 的类型以åŠä»·æ ¼ï¼š +- [Equinix Metal Server 类型](https://metal.equinix.com/developers/docs/servers/about/) +- [Equinix Metal ä»·æ ¼](https://metal.equinix.com/developers/docs/servers/server-specs/) + +:::note 注æ„事项： + +- 如果使用 CLI 或 API é…置新的 Equinix Metal Server，你需è¦æ供项目 IDã€è®¡åˆ’ã€metro å’Œæ“作系统。 +- 当使用云主机的虚拟机时，你需è¦å…许 80 å’Œ 443 端å£çš„入站 TCP 通信。有关端å£é…置的信æ¯ï¼Œè¯·å‚è§ä½ çš„云主机的文档。 +- 如需了解所有端å£è¦æ±‚，请å‚è§ [Docker 安装](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 +- æ ¹æ®æˆ‘们的[è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)é…置主机。 + +::: +### 2. 安装 Rancher + +è¦åœ¨ Equinix Metal 主机上安装 Rancher，先与它连接，然åŽä½¿ç”¨ shell 进行安装。 + +1. 使用你惯用的 shell（例如 PuTTy 或远程终端）登录到你的 Equinix Metal 主机。 + +2. 在 shell 中执行以下命令： + + ``` + sudo docker run -d --restart=unless-stopped -p 80:80 -p 443:443 --privileged rancher/rancher + ``` + +**结果**：Rancher 已安装。 + +### 3. 登录 + +登录到 Rancher åŽï¼Œä½ è¿˜éœ€è¦è¿›è¡Œä¸€äº›ä¸€æ¬¡æ€§é…置。 + +1. 打开 Web æµè§ˆå™¨å¹¶è¾“入主机的 IP 地å€`https://`。 + + å°† `` 替æ¢ä¸ºä½ çš„主机 IP 地å€ã€‚ + +2. 出现æ示时，为默认 `admin` è´¦å·åˆ›å»ºå¯†ç ã€‚ + +3. 设置 **Rancher Server URL**。URL å¯ä»¥æ˜¯ IP 地å€æˆ–主机å。需è¦æ³¨æ„，添加到集群中的æ¯ä¸ªèŠ‚点都必须能够连接到此 URL。

如果你在 URL 中使用主机å，则此主机å必须在 DNS 中解æžåˆ°ä½ éœ€è¦æ·»åŠ åˆ°é›†ç¾¤çš„节点上。 + +
+ +### 4. 创建集群 + +欢迎使用 Rancherï¼çŽ°åœ¨ï¼Œä½ å¯ä»¥åˆ›å»ºä½ çš„第一个 Kubernetes 集群了。 + +在此任务中，你å¯ä»¥ä½¿ç”¨**自定义**选项。此选项å…许你把 _ä»»æ„_ Linux 主机（云虚拟机ã€æœ¬åœ°è™šæ‹Ÿæœºæˆ–裸机）添加到集群中。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. 选择**自定义**。 +1. 输入**集群å称**。 +1. 点击**下一步**。 +1. 在**节点角色**中，选择 _全部_ è§’è‰²ï¼Œå³ **etcd**，**Control** å’Œ **Worker**。 + - **å¯é€‰**：Rancher 会自动检测用于 Rancher 通信和集群通信的 IP 地å€ã€‚ä½ å¯ä»¥ä½¿ç”¨**节点地å€**处的`公有地å€`å’Œ`内网地å€`进行覆盖。 +1. 将注册命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. 使用你惯用的 shell（例如 PuTTy 或远程终端）登录到你的 Linux 主机。粘贴剪贴æ¿çš„命令并è¿è¡Œã€‚ +1. 在 Linux 主机上è¿è¡Œå®Œå‘½ä»¤åŽï¼Œå•å‡»**完æˆ**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +#### 已完æˆï¼ + +æ­å–œï¼ä½ å·²åˆ›å»ºç¬¬ä¸€ä¸ªé›†ç¾¤ã€‚ + +#### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md new file mode 100644 index 00000000000..5c113fc447c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md @@ -0,0 +1,77 @@ +--- +title: Rancher GCP å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher GCP 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 GCP çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Google GCP 会产生费用。 + +::: + +- [Google GCP Account](https://console.cloud.google.com/)：用于创建部署 Rancher å’Œ Kubernetes 的资æºã€‚ +- [Google GCP 项目](https://cloud.google.com/appengine/docs/standard/nodejs/building-app/creating-project)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹å¦‚何创建 GCP 项目。 +- [Google GCP ServiceAccount](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)：请访问此链接查看如何创建 GCP ServiceAccount å’Œ Token 文件。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Google GCP 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/gcp` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 GCP 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `gcp_account_json` - 替æ¢ä¸º GCP ServiceAccount 文件路径和文件å。 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [GCP Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/gcp) 了解更多信æ¯ã€‚ + 建议包括： + - `gcp_region` - Google GCP 区域。GCP 的默认区域 (`us-east4`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `gcp_zone` - Google GCP 区域。GCP 的默认区域 (`us-east4-a`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `machine_type` - 使用的计算实例大å°ï¼Œæœ€å°è§„格为 `n1-standard-1`。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `n1-standard-2` 或 `n1-standard-4`。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/gcp` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 GCP 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/gcp` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md new file mode 100644 index 00000000000..b3acb4f9ecc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md @@ -0,0 +1,154 @@ +--- +title: Helm CLI 快速入门 +--- + +本文æ供了快速安装 Rancher 的方法。 + +这些说明å‡è®¾ä½ æœ‰ä¸€ä¸ª Linux 虚拟机，并能从本地工作站与之通信。Rancher 将安装在 Linux 主机上。你将需è¦æ£€ç´¢è¯¥ä¸»æœºçš„ IP 地å€ï¼Œä»¥ä¾¿ä»Žæœ¬åœ°å·¥ä½œç«™è®¿é—® Rancher。Rancher æ—¨åœ¨è¿œç¨‹ç®¡ç† Kubernetes 集群，因此 Rancher 管ç†çš„任何 Kubernetes 集群也都需è¦èƒ½å¤Ÿè®¿é—®è¯¥ IP 地å€ã€‚ + +我们ä¸å»ºè®®åœ¨æœ¬åœ°å®‰è£… Rancher，因为它会产生网络问题。如果你在 localhost 上安装 Rancher，Rancher 无法与下游 Kubernetes 集群通信，因此在 localhost 上你无法测试 Rancher 的集群é…置和集群管ç†åŠŸèƒ½ã€‚ + +ä½ çš„ Linux 主机å¯ä»¥ä½äºŽä»»ä½•åœ°æ–¹ã€‚例如，它å¯ä»¥æ˜¯ Amazon EC2 实例ã€Digital Ocean Droplet 或 Azure 虚拟机。其他 Rancher 文档也ç»å¸¸ç§°å®ƒä»¬ä¸ºâ€œèŠ‚点â€ã€‚部署 Linux 主机的一ç§æ–¹æ³•æ˜¯è®¾ç½®ä¸€ä¸ª Amazon EC2 实例，如[本教程](../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)中所示。 + +完整的安装è¦æ±‚在[这里](../../../pages-for-subheaders/installation-requirements.md)。 + +## 在 Linux 上安装 K3s + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +在 Linux 主机上è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å®‰è£… K3s 集群： + +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server --cluster-init +``` + +`--cluster-init` å…许 K3s ä½¿ç”¨åµŒå…¥å¼ etcd 作为数æ®å­˜å‚¨ï¼Œå¹¶èƒ½å¤Ÿè½¬æ¢ä¸º HA 设置。请å‚阅[嵌入å¼æ•°æ®åº“的高å¯ç”¨æ€§](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/)。 + +ä¿å­˜ Linux 主机的 IP。 + +## å°† kubeconfig ä¿å­˜åˆ°ä½ çš„工作站 + +kubeconfig 文件对于访问 Kubernetes 集群éžå¸¸é‡è¦ã€‚从 Linux 主机å¤åˆ¶ `/etc/rancher/k3s/k3s.yaml` 中的文件，并将其ä¿å­˜åˆ°æœ¬åœ°å·¥ä½œç«™çš„ `~/.kube/config` 目录中。一ç§æ–¹æ³•æ˜¯ä½¿ç”¨ `scp` 工具并在本地计算机上è¿è¡Œæ­¤å‘½ä»¤ï¼š + + + + +``` +scp root@:/etc/rancher/k3s/k3s.yaml ~/.kube/config +``` + +在æŸäº›æƒ…况下，它å¯èƒ½éœ€è¦ç¡®ä¿ä½ çš„ shell 定义了环境å˜é‡ `KUBECONFIG=~/.kube/config`，例如，它å¯ä»¥åœ¨ä½ çš„é…置文件或 rc 文件中导出。 + + + + +默认情况下ä¸èƒ½è¯†åˆ«â€œscpâ€å‘½ä»¤ï¼Œæ‰€ä»¥æˆ‘们需è¦å…ˆå®‰è£…一个模å—。 + +在 Windows Powershell 中： + +``` +Find-Module Posh-SSH +Install-Module Posh-SSH + +## 获å–远程 kubeconfig 文件 +scp root@:/etc/rancher/k3s/k3s.yaml $env:USERPROFILE\.kube\config +``` + + + + +## 在 kubeconfig 中编辑 Rancher Server URL + +在 kubeconfig 文件中，你需è¦å°† `server` 字段的值更改为 `:6443`。你å¯ä»¥é€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server。你需è¦è¿›è¡Œæ­¤ç¼–辑，以便你从本地工作站è¿è¡Œ Helm 或 kubectl 命令时，能够与安装了 Rancher çš„ Kubernetes 集群进行通信。 + + + + +打开 kubeconfig 文件进行编辑的一ç§æ–¹æ³•æ˜¯ä½¿ç”¨ Vim： + +``` +vi ~/.kube/config +``` + +输入 `i` 以打开 Vim çš„æ’入模å¼ã€‚è¦ä¿å­˜ä½ çš„工作，请按 `Esc`。然åŽè¾“å…¥ `:wq` 并按 `Enter`。 + + + + + +在 Windows Powershell 中，你å¯ä»¥ä½¿ç”¨ `notepad.exe` æ¥ç¼–辑 kubeconfig 文件： + +``` +notepad.exe $env:USERPROFILE\.kube\config +``` + +编辑完æˆåŽï¼ŒæŒ‰ `ctrl+s` 或转到 `File > Save` æ¥ä¿å­˜ä½ çš„工作。 + + + + +## 使用 Helm æ¥å®‰è£… Rancher + +从本地工作站è¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚你需è¦å…ˆå®‰è£… [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) å’Œ [helm](https://helm.sh/docs/intro/install/)： + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + +kubectl create namespace cattle-system + +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +helm repo add jetstack https://charts.jetstack.io + +helm repo update + +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 + +# Windows Powershell +helm install cert-manager jetstack/cert-manager ` + --namespace cert-manager ` + --create-namespace ` + --version v1.11.0 +``` + +安装 Rancher 的最终命令如下。该命令需è¦ä¸€ä¸ªå°†æµé‡è½¬å‘到 Linux 主机的域å。为了简化本教程，你å¯ä»¥ä½¿ç”¨å‡åŸŸå。`.sslip.io` 是一个å‡åŸŸå的例å­ã€‚ + +è¦å®‰è£…特定的 Rancher 版本，请使用 `--version` 标志（例如，`--version 2.6.6`）。å¦åˆ™ï¼Œé»˜è®¤å®‰è£…最新的 Rancher。请å‚阅[选择 Rancher 版本](../../installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +请注æ„，密ç è‡³å°‘éœ€è¦ 12 个字符。 + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=.sslip.io \ + --set replicas=1 \ + --set bootstrapPassword= + +# Windows Powershell +helm install rancher rancher-latest/rancher ` + --namespace cattle-system ` + --set hostname=.sslip.io ` + --set replicas=1 ` + --set bootstrapPassword= +``` + +现在，如果你在 Web æµè§ˆå™¨ä¸­å¯¼èˆªåˆ° `.sslip.io`，你应该会看到 Rancher UI。 + +为了简化说明，我们使用了一个å‡åŸŸå和自签åè¯ä¹¦æ¥è¿›è¡Œå®‰è£…。因此，你å¯èƒ½éœ€è¦åœ¨ Web æµè§ˆå™¨ä¸­æ·»åŠ ä¸€ä¸ªå®‰å…¨ä¾‹å¤–æ¥æŸ¥çœ‹ Rancher UI。请注æ„，对于生产安装，你需è¦å…·æœ‰è´Ÿè½½å‡è¡¡å™¨ã€çœŸå®žåŸŸå和真实è¯ä¹¦çš„高å¯ç”¨æ€§è®¾ç½®ã€‚ + +这些说明还çœç•¥äº†å®Œæ•´çš„安装è¦æ±‚和其他安装选项。如果你对这些步骤有任何疑问，请å‚阅完整的 [Helm CLI 安装文档](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)。 + +è¦ä½¿ç”¨æ–°çš„ Rancher Server æ¥å¯åŠ¨æ–°çš„ Kubernetes 集群，你å¯èƒ½éœ€è¦åœ¨ Rancher 中设置云凭è¯ã€‚有关更多信æ¯ï¼Œè¯·å‚阅[使用 Rancher å¯åŠ¨ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md new file mode 100644 index 00000000000..585353441b2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md @@ -0,0 +1,76 @@ +--- +title: Rancher Hetzner Cloud å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher Hetzner Cloud 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 Hetzner Cloud çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Hetzner Cloud 会产生费用。 + +::: + +- [Hetzner Cloud è´¦å·](https://www.hetzner.com)：用于è¿è¡ŒæœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ +- [Hetzner API 访问密钥](https://docs.hetzner.cloud/#getting-started)：如果你没有的è¯ï¼Œè¯·å‚考说明创建一个。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Hetzner 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/hcloud` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 Hetzner 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `hcloud_token` - 替æ¢ä¸º Hetzner API 访问密钥。 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [Hetzner Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/hcloud) 了解更多信æ¯ã€‚ + 建议包括： + + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - 实例类型，至少需è¦æ˜¯ `cx21`。 + - `hcloud_location`- Hetzner Cloud ä½ç½®ã€‚选择最近的ä½ç½®ï¼Œè€Œä¸æ˜¯ä½¿ç”¨é»˜è®¤ä½ç½®ï¼ˆ`fsn1`）。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/hcloud` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 Hetzner 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/hcloud` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md new file mode 100644 index 00000000000..46b42901db9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md @@ -0,0 +1,76 @@ +--- +title: Rancher Outscale å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher Outscale 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 Outscale çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::note + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Outscale 会产生费用。 + +::: + +- [Outscale è´¦å·](https://en.outscale.com/)：用于è¿è¡ŒæœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ +- [Outscale 访问密钥](https://docs.outscale.com/en/userguide/About-Access-Keys.html)：如果你没有的è¯ï¼Œè¯·æŒ‰ç…§è¯´æ˜Žåˆ›å»ºä¸€ä¸ª Outscale 访问密钥。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Outscale 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/outscale` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 Outscale 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `access_key_id` - 替æ¢ä¸º Outscale 访问密钥 + - `secret_key_id` - 替æ¢ä¸º Outscale 密文密钥 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [Outscale Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/outscale) 了解更多信æ¯ã€‚ + 建议包括： + - `region` - Outscale 区域。Outscale 的默认区域ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域（`eu-west-2`）。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - 实例类型，至少需è¦æ˜¯ `tinav3.c2r4p3`。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 21 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/outscale` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 Outscale 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/outscale` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/prime.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/prime.md new file mode 100644 index 00000000000..094508ff5a6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/prime.md @@ -0,0 +1,7 @@ +--- +title: Rancher Prime +--- + +Rancher 在 v2.7 中引入了 Rancher Prime，这是 Rancher ä¼ä¸šçº§äº§å“的进化。Rancher Prime 是基于相åŒæºä»£ç æž„建的商业化ã€ä¼ä¸šçº§çš„新版本。因此，Rancher 的产å“将继续ä¿æŒ 100% å¼€æºï¼Œå¹¶é€šè¿‡å®‰å…¨ã€å»¶é•¿ç”Ÿå‘½å‘¨æœŸã€è®¿é—®é‡ç‚¹æž¶æž„å’Œ Kubernetes 公告体现é¢å¤–价值。Rancher Prime 还将æ供选项，让用户获得创新 Rancher 项目的生产支æŒã€‚使用 Rancher Prime，安装 asset 会托管在由 Rancher æŒæœ‰å’Œç®¡ç†çš„å¯ä¿¡é•œåƒä»“库中。 + +è¦å¼€å§‹ä½¿ç”¨ Rancher Prime，请[转到此页é¢](https://www.rancher.com/quick-start)并填写表格。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md new file mode 100644 index 00000000000..dcddb74cfe9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md @@ -0,0 +1,51 @@ +--- +title: Vagrant 快速入门 +--- + +ä½ å¯ä»¥å‚考以下步骤快速部署 Rancher Server，并附加一个å•èŠ‚点集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +- [Vagrant](https://www.vagrantup.com)：Vagrant æ˜¯å¿…éœ€çš„ï¼Œç”¨äºŽæ ¹æ® Vagrantfile é…置主机。 +- [Virtualbox](https://www.virtualbox.org)：需è¦æŠŠ Vagrant é…置的虚拟机é…置到 VirtualBox。 +- 至少 4GB çš„å¯ç”¨å†…存。 + +### æ³¨æ„ +- Vagrant 需è¦ä½¿ç”¨æ’件æ¥åˆ›å»º VirtualBox 虚拟机。请执行以下命令进行安装： + + `vagrant plugin install vagrant-vboxmanage` + + `vagrant plugin install vagrant-vbguest` + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/vagrant` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Vagrantfile 文件的文件夹。 + +3. **å¯é€‰**：编辑 `config.yaml` 文件： + + - æ ¹æ®éœ€è¦æ›´æ”¹èŠ‚点数和内存分é…（`node.count`, `node.cpus`, `node.memory`） + - 更改 `admin` 的密ç ä»¥ç™»å½• Rancher。(`admin_password`) + +4. 执行 `vagrant up --provider=virtualbox` 以åˆå§‹åŒ–环境。 + +5. é…置完æˆåŽï¼Œåœ¨æµè§ˆå™¨ä¸­æ‰“å¼€ `https://192.168.56.101`。默认的用户å和密ç æ˜¯ `admin/adminPassword`。 + +**结果**：Rancher Server 和你的 Kubernetes 集群已安装在 VirtualBox 上。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/vagrant` 文件夹，然åŽæ‰§è¡Œ `vagrant destroy -f`。 + +2. 等待所有资æºå·²åˆ é™¤çš„确认消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/nodeports.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/nodeports.md new file mode 100644 index 00000000000..bf2294d4c06 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/nodeports.md @@ -0,0 +1,138 @@ +--- +title: 部署带有 NodePort 的工作负载 +--- + +### 先决æ¡ä»¶ + +你已有一个正在è¿è¡Œçš„集群，且该集群中有至少一个节点。 + +### 1. 部署工作负载 + +ä½ å¯ä»¥å¼€å§‹åˆ›å»ºä½ çš„第一个 Kubernetes [工作负载](https://kubernetes.io/docs/concepts/workloads/)ã€‚å·¥ä½œè´Ÿè½½æ˜¯ä¸€ä¸ªå¯¹è±¡ï¼Œå…¶ä¸­åŒ…å« pod 以åŠéƒ¨ç½²åº”用所需的其他文件和信æ¯ã€‚ + +在本文的工作负载中，你将部署一个 Rancher Hello-World 应用。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥éœ€è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 为工作负载设置**å称**。 +1. 在**容器镜åƒ**字段中，输入 `rancher/hello-world`。注æ„区分大å°å†™ã€‚ +1. 点击**添加端å£**。 +1. 在**æœåŠ¡ç±»åž‹**下拉èœå•ä¸­ï¼Œç¡®ä¿é€‰æ‹©äº† **NodePort**。 + + ![NodePort 下拉èœå•ï¼ˆåœ¨æ¯ä¸ªæ‰€é€‰èŠ‚点处）](/img/nodeport-dropdown.png) + +1. 在**å‘布容器端å£**字段中，输入端å£`80`。 + + ![å‘布容器端å£ï¼Œç«¯å£ 80 已输入](/img/container-port-field.png) + +1. å•å‡»**创建**。 + +**结果**： + +* 工作负载已部署。此过程å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿã€‚ +* 当工作负载完æˆéƒ¨ç½²åŽï¼Œå®ƒçš„状æ€ä¼šå˜ä¸º **Active**。你å¯ä»¥ä»Žé¡¹ç›®çš„**工作负载**页é¢æŸ¥çœ‹å…¶çŠ¶æ€ã€‚ + +
+ +### 2. 查看应用 + +在**工作负载**页é¢ä¸­ï¼Œç‚¹å‡»å·¥ä½œè´Ÿè½½ä¸‹æ–¹çš„链接。如果 deployment 已完æˆï¼Œä½ çš„应用会打开。 + +### 注æ„事项 + +如果使用云虚拟机，你å¯èƒ½æ— æ³•è®¿é—®è¿è¡Œå®¹å™¨çš„端å£ã€‚è¿™ç§æƒ…况下，你å¯ä»¥ä½¿ç”¨ `Execute Shell` 在本地主机的 SSH 会è¯ä¸­æµ‹è¯• Nginx。如果å¯ç”¨çš„è¯ï¼Œä½¿ç”¨å·¥ä½œè´Ÿè½½ä¸‹æ–¹çš„链接中 `:` åŽé¢çš„端å£å·ã€‚在本例中，端å£å·ä¸º `31568`。 + +```html +gettingstarted@rancher:~$ curl http://localhost:31568 + + + + Rancher + + + + + +

Hello world!

+

My hostname is hello-world-66b4b9d88b-78bhx

+
+

k8s services found 2

+ + INGRESS_D1E1A394F61C108633C4BD37AEDDE757 tcp://10.43.203.31:80
+ + KUBERNETES tcp://10.43.0.1:443
+ +
+
+ + +
+ + +
+ + + +gettingstarted@rancher:~$ + +``` + +### 已完æˆï¼ + +æ­å–œï¼ä½ å·²æˆåŠŸé€šè¿‡ NodePort 部署工作负载。 + +#### åŽç»­æ“作 + +使用完沙盒åŽï¼Œä½ éœ€è¦æ¸…ç† Rancher Server 和集群。详情请å‚è§ï¼š + +- [Amazon AWS：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/aws.md#销æ¯çŽ¯å¢ƒ) +- [DigitalOcean：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/digitalocean.md#销æ¯çŽ¯å¢ƒ) +- [Vagrant：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/vagrant.md#销æ¯çŽ¯å¢ƒ) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md new file mode 100644 index 00000000000..86f8017f09e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md @@ -0,0 +1,72 @@ +--- +title: 部署带有 Ingress 的工作负载 +--- + +### 先决æ¡ä»¶ + +你已有一个正在è¿è¡Œçš„集群，且该集群中有至少一个节点。 + +### 1. 部署工作负载 + +ä½ å¯ä»¥å¼€å§‹åˆ›å»ºä½ çš„第一个 Kubernetes [工作负载](https://kubernetes.io/docs/concepts/workloads/)ã€‚å·¥ä½œè´Ÿè½½æ˜¯ä¸€ä¸ªå¯¹è±¡ï¼Œå…¶ä¸­åŒ…å« pod 以åŠéƒ¨ç½²åº”用所需的其他文件和信æ¯ã€‚ + +在本文的工作负载中，你将部署一个 Rancher Hello-World 应用。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 点击 **Deployment**。 +1. 为工作负载设置**å称**。 +1. 在**容器镜åƒ**字段中，输入 `rancher/hello-world`。注æ„区分大å°å†™ã€‚ +1. 在 `Service Type` 点击 **Add Port** å’Œ `Cluster IP`，并在 **Private Container Port** 字段中输入`80`。你å¯ä»¥å°† `Name` 留空或指定å称。通过添加端å£ï¼Œä½ å¯ä»¥è®¿é—®é›†ç¾¤å†…外的应用。有关详细信æ¯ï¼Œè¯·å‚阅 [Service](../../../pages-for-subheaders/workloads-and-pods.md#services)。 +1. å•å‡»**创建**。 + +**结果**： + +* 工作负载已部署。此过程å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿã€‚ +* 当工作负载完æˆéƒ¨ç½²åŽï¼Œå®ƒçš„状æ€ä¼šå˜ä¸º **Active**。你å¯ä»¥ä»Žé¡¹ç›®çš„**工作负载**页é¢æŸ¥çœ‹å…¶çŠ¶æ€ã€‚ + +### 2. 通过 Ingress 暴露应用 + +现在应用已å¯åŠ¨å¹¶è¿è¡Œï¼Œä½ éœ€è¦æš´éœ²åº”用以让其他æœåŠ¡è¿žæŽ¥åˆ°å®ƒã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 + +1. 点击**æœåŠ¡å‘现 > Ingresses**。 + +1. 点击**创建**。 + +1. 在选择**命å空间**时，你需è¦é€‰æ‹©åœ¨åˆ›å»º deployment 时使用的命å空间。å¦åˆ™ï¼Œåœ¨æ­¥éª¤8中选择**目标æœåŠ¡**时，你的 deployment 会ä¸å¯ç”¨ã€‚ + +1. 输入**å称**，例如 **hello**。 + +1. 指定**路径**，例如 `/hello`。 + +1. 在**目标æœåŠ¡**字段的下拉èœå•ä¸­ï¼Œé€‰æ‹©ä½ ä¸ºæœåŠ¡è®¾ç½®çš„å称。 + +1. 在**端å£**字段中的下拉èœå•ä¸­ï¼Œé€‰æ‹© `80`。 + +1. 点击å³ä¸‹è§’çš„**创建**。 + +**结果**：应用分é…到了一个 `sslip.io` 地å€å¹¶æš´éœ²ã€‚è¿™å¯èƒ½éœ€è¦ä¸€ä¸¤åˆ†é’Ÿã€‚ + + +### 查看应用 + +在 **Deployments** 页é¢ä¸­ï¼Œæ‰¾åˆ°ä½  deployment çš„ **endpoint** 列，然åŽå•å‡»ä¸€ä¸ª endpoint。å¯ç”¨çš„ endpoint å–决于你添加到 deployment 中的端å£é…置。如果你看ä¸åˆ°éšæœºåˆ†é…端å£çš„ endpoint，请将你在创建 Ingress 时指定的路径尾附到 IP 地å€ä¸Šã€‚例如，如果你的 endpoint 是 `xxx.xxx.xxx.xxx` 或 `https://xxx.xxx.xxx.xxx`，把它修改为 `xxx.xxx.xxx.xxx/hello` 或 `https://xxx.xxx.xxx.xxx/hello`。 + +应用将在å¦ä¸€ä¸ªçª—å£ä¸­æ‰“开。 + +#### 已完æˆï¼ + +æ­å–œï¼ä½ å·²æˆåŠŸé€šè¿‡ Ingress 部署工作负载。 + +#### åŽç»­æ“作 + +使用完沙盒åŽï¼Œä½ éœ€è¦æ¸…ç† Rancher Server 和集群。详情请å‚è§ï¼š + +- [Amazon AWS：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/aws.md#销æ¯çŽ¯å¢ƒ) +- [DigitalOcean：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/digitalocean.md#销æ¯çŽ¯å¢ƒ) +- [Vagrant：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/vagrant.md#销æ¯çŽ¯å¢ƒ) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md new file mode 100644 index 00000000000..643a7122e37 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md @@ -0,0 +1,40 @@ +--- +title: 为定时扫æé…置告警 +--- + +ä½ å¯ä»¥å®šæ—¶è¿è¡Œ ClusterScan。 + +你还å¯ä»¥ä¸ºå®šæ—¶æ‰«æ指定是å¦åœ¨æ‰«æ完æˆæ—¶å‘出告警。 + +åªæœ‰å®šæ—¶è¿è¡Œçš„扫ææ‰æ”¯æŒå‘Šè­¦ã€‚ + +CIS Benchmark 应用支æŒä¸¤ç§ç±»åž‹çš„告警： + +- 扫æ完æˆå‘Šè­¦ï¼šæ­¤å‘Šè­¦åœ¨æ‰«æè¿è¡Œå®Œæˆæ—¶å‘出。告警包括详细信æ¯ï¼ŒåŒ…括 ClusterScan çš„å称和 ClusterScanProfile çš„å称。 +- 扫æ失败告警：如果扫æ中有一些测试失败或扫æ处于 `Fail` 状æ€ï¼Œåˆ™ä¼šå‘出此告警。 + +:::note 先决æ¡ä»¶ï¼š + +为 `rancher-cis-benchmark` å¯ç”¨å‘Šè­¦ä¹‹å‰ï¼Œç¡®ä¿å®‰è£…了 `rancher-monitoring` 应用并é…置了接收器（Receiver）和路由（Route）。详情请å‚è§[本章节](../../../reference-guides/monitoring-v2-configuration/receivers.md)。 + +在为 `rancher-cis-benchmark` å‘Šè­¦é…置路由时，你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹ `job:rancher-cis-scan` æ¥æŒ‡å®šåŒ¹é…。详情请查看[路由é…置示例](../../../reference-guides/monitoring-v2-configuration/receivers.md#cis-扫æ告警的示例路由é…ç½®)。 + +::: + +è¦ä¸ºå®šæ—¶è¿è¡Œçš„扫æé…置告警： + +1. 请在 `rancher-cis-benchmark` 应用程åºä¸Šå¯ç”¨å‘Šè­¦ã€‚详情请å‚è§[本页](../../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md)。 +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. å•å‡»**创建**。 +1. 选择集群扫æé…置文件。该é…置文件确定è¦ä½¿ç”¨å“ªä¸ª CIS Benchmark 版本以åŠè¦æ‰§è¡Œå“ªäº›æµ‹è¯•ã€‚如果你选择 Default é…置文件，则 CIS Operator 将选择适用于它所在的 Kubernetes 集群类型的é…置文件。 +1. 选择**定时è¿è¡Œæ‰«æ**的选项。 +1. 在**调度**字段中输入有效的 [Cron 表达å¼](https://en.wikipedia.org/wiki/Cron#CRON_expression)。 +1. 选中**å‘Šè­¦**下告警类型æ—边的框。 +1. （å¯é€‰ï¼‰é€‰æ‹©ä¸€ä¸ª**ä¿ç•™è®¡æ•°**，表示为这个定时扫æ维护的报告数é‡ã€‚默认情况下，此计数为 3。超过此ä¿ç•™é™åˆ¶æ—¶ï¼Œæ—§æŠ¥å‘Šå°†è¢«åˆ é™¤ã€‚ +1. å•å‡»**创建**。 + +**结果**：扫æè¿è¡Œï¼Œå¹¶æ ¹æ®è®¾ç½®çš„ cron 表达å¼é‡æ–°è°ƒåº¦ã€‚如果在 `rancher-monitoring` 应用下é…置了路由和接收器，则会在扫æ完æˆæ—¶å‘出告警。 + +æ¯æ¬¡è¿è¡Œæ‰«æ都会生æˆä¸€ä»½å¸¦æœ‰æ‰«æ结果的报告。如需查看最新的结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md new file mode 100644 index 00000000000..fe8477f7b55 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md @@ -0,0 +1,9 @@ +--- +title: 为集群扫æ创建自定义 Benchmark 版本 +--- + +æŸäº› Kubernetes 集群å¯èƒ½éœ€è¦è‡ªå®šä¹‰é…ç½® Benchmark 测试。例如，Kubernetes é…置文件或è¯ä¹¦çš„路径å¯èƒ½ä¸Žä¸Šæ¸¸ CIS Benchmark 的标准ä½ç½®ä¸åŒã€‚ + +现在，你å¯ä»¥ä½¿ç”¨ `rancher-cis-benchmark` 应用æ¥åˆ›å»ºè‡ªå®šä¹‰ Benchmark 版本，从而è¿è¡Œé›†ç¾¤æ‰«æ。 + +有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/cis-scans/custom-benchmark.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md new file mode 100644 index 00000000000..55c34dcdf5c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md @@ -0,0 +1,20 @@ +--- +title: 为 Rancher CIS Benchmark å¯ç”¨å‘Šè­¦ +--- + +ä½ å¯ä»¥é…置告警，从而将告警å‘é€ç»™å®šæ—¶è¿è¡Œçš„扫æ。 + +:::note 先决æ¡ä»¶ï¼š + +为 `rancher-cis-benchmark` å¯ç”¨å‘Šè­¦ä¹‹å‰ï¼Œç¡®ä¿å®‰è£…了 `rancher-monitoring` 应用并é…置了接收器（Receiver）和路由（Route）。详情请å‚è§[本章节](../../../reference-guides/monitoring-v2-configuration/receivers.md)。 + +在为 `rancher-cis-benchmark` å‘Šè­¦é…置路由时，你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹ `job:rancher-cis-scan` æ¥æŒ‡å®šåŒ¹é…。详情请查看[路由é…置示例](../../../reference-guides/monitoring-v2-configuration/receivers.md#cis-扫æ告警的示例路由é…ç½®)。 + +::: + +在安装或å‡çº§ `rancher-cis-benchmark` Helm Chart 时，在 `values.yaml` 中将以下标志设置为 `true`： + +```yaml +alerts: + enabled: true +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md new file mode 100644 index 00000000000..814d38c5349 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md @@ -0,0 +1,17 @@ +--- +title: 安装 Rancher CIS Benchmark +--- + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å®‰è£… CIS Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Charts**。 +1. å•å‡» **CIS Benchmark**。 +1. å•å‡»**安装**。 + +**结果**：CIS 扫æ应用已ç»éƒ¨ç½²åœ¨ Kubernetes 集群上。 + +:::note + +如果你使用 Kubernetes v1.24 或更早版本，并且具有使用 [Pod 安全策略](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) (PSP) 加固的集群，则 CIS Benchmark 4.0.0 åŠæ›´é«˜ç‰ˆæœ¬ä¼šé»˜è®¤ç¦ç”¨ PSP。è¦åœ¨ PSP 加固集群上安装 CIS Benchmark，请在安装 Chart 之å‰å°† values 中的 `global.psp.enabled` 设置为 `true`。[Pod 安全准入](../../new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md) (PSA) 加固集群ä¸å—å½±å“。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md new file mode 100644 index 00000000000..d43567fbb25 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md @@ -0,0 +1,20 @@ +--- +title: 定时è¿è¡Œæ‰«æ +--- + +è¦å®šæ—¶è¿è¡Œ ClusterScan： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. 选择集群扫æé…置文件。该é…置文件确定è¦ä½¿ç”¨å“ªä¸ª CIS Benchmark 版本以åŠè¦æ‰§è¡Œå“ªäº›æµ‹è¯•ã€‚如果你选择 Default é…置文件，则 CIS Operator 将选择适用于它所在的 Kubernetes 集群类型的é…置文件。 +1. 选择**定时è¿è¡Œæ‰«æ**的选项。 +1. 将一个有效的 Cron 表达å¼å¡«å†™åˆ°**调度**字段。 +1. 选择一个**ä¿ç•™è®¡æ•°**，表示为这个定时扫æ维护的报告数é‡ã€‚默认情况下，此计数为 3。超过此ä¿ç•™é™åˆ¶æ—¶ï¼Œæ—§æŠ¥å‘Šå°†è¢«åˆ é™¤ã€‚ +1. å•å‡»**创建**。 + +**结果**：扫æè¿è¡Œï¼Œå¹¶æ ¹æ®è®¾ç½®çš„ cron 表达å¼é‡æ–°è°ƒåº¦ã€‚**下一次扫æ**的值表示下次è¿è¡Œæ­¤æ‰«æ的时间。 + +æ¯æ¬¡è¿è¡Œæ‰«æ都会生æˆä¸€ä»½å¸¦æœ‰æ‰«æ结果的报告。如需查看最新的结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 + +你还å¯ä»¥åœ¨æ‰«æ详情页é¢ä¸Šçš„**报告**下拉èœå•ä¸­æŸ¥çœ‹ä¹‹å‰çš„报告。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md new file mode 100644 index 00000000000..e658029233e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md @@ -0,0 +1,22 @@ +--- +title: è¿è¡Œæ‰«æ +--- + +创建 ClusterScan 自定义资æºåŽï¼Œå®ƒä¼šåœ¨é›†ç¾¤ä¸Šä¸ºæ‰€é€‰ ClusterScanProfile å¯åŠ¨æ–°çš„ CIS 扫æ。 + +:::note + +请注æ„，目å‰ä¸€ä¸ªé›†ç¾¤æ¯æ¬¡åªèƒ½è¿è¡Œä¸€æ¬¡ CIS 扫æ。如果你创建了多个 ClusterScan 自定义资æºï¼Œoperator åªèƒ½ä¸€ä¸ªæŽ¥ä¸€ä¸ªåœ°è¿è¡Œè¿™äº›èµ„æºã€‚一个扫æ完æˆä¹‹å‰ï¼Œå…¶ä½™ ClusterScan 自定义资æºå°†å¤„于 “Pending†状æ€ã€‚ + +::: + +è¦è¿è¡Œæ‰«æ： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. å•å‡»**创建**。 +1. 选择集群扫æé…置文件。该é…置文件确定è¦ä½¿ç”¨å“ªä¸ª CIS Benchmark 版本以åŠè¦æ‰§è¡Œå“ªäº›æµ‹è¯•ã€‚如果你选择 Default é…置文件，则 CIS Operator 将选择适用于它所在的 Kubernetes 集群类型的é…置文件。 +1. å•å‡»**创建**。 + +**结果**：已生æˆå¸¦æœ‰æ‰«æ结果的报告。如需查看结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md new file mode 100644 index 00000000000..230bb1c7696 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md @@ -0,0 +1,34 @@ +--- +title: 跳过测试 +--- + +用户å¯ä»¥åœ¨æµ‹è¯•é…置文件中自定义è¦è·³è¿‡çš„æµ‹è¯•ï¼Œç„¶åŽ CIS 扫æå¯ä»¥ä½¿ç”¨è¯¥é…置文件è¿è¡Œã€‚ + +è¦è·³è¿‡æµ‹è¯•ï¼Œä½ éœ€è¦åˆ›å»ºè‡ªå®šä¹‰ä¸€ä¸ª CIS 扫æé…置文件。é…ç½®æ–‡ä»¶åŒ…å« CIS 扫æçš„é…置，包括è¦ä½¿ç”¨çš„ Benchmark 测试版本以åŠè¦åœ¨è¯¥ Benchmark 测试中跳过的测试。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **CIS Benchmark > é…置文件**。 +1. 在这里，你å¯ä»¥ä½¿ç”¨å¤šç§æ–¹å¼æ¥åˆ›å»ºé…置文件。è¦åˆ›å»ºæ–°é…置文件，å•å‡»**创建**并在 UI 中填写表å•ã€‚è¦åŸºäºŽçŽ°æœ‰é…置文件æ¥åˆ›å»ºæ–°é…置文件，请转到现有é…置文件并å•å‡»**â‹® 克隆**。如果你在填写表å•ï¼Œè¯·ä½¿ç”¨æµ‹è¯• ID 添加è¦è·³è¿‡çš„测试，并å‚考相关的 CIS Benchmark。如果你将新的测试é…置文件创建为 YAML，你需è¦åœ¨ `skipTests` å‚数中添加è¦è·³è¿‡çš„测试的 ID。你还需è¦ä¸ºé…置文件命å： + + ```yaml + apiVersion: cis.cattle.io/v1 + kind: ClusterScanProfile + metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" + spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" + ``` +1. å•å‡»**创建**。 + +**结果**：已创建一个新的 CIS 扫æé…置文件。 + +使用此é…置文件[è¿è¡Œæ‰«æ](./run-a-scan.md)时，会跳过定义的跳过测试。跳过的测试将在生æˆçš„报告中标记为 `Skip`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md new file mode 100644 index 00000000000..83d36169627 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md @@ -0,0 +1,9 @@ +--- +title: å¸è½½ Rancher CIS Benchmark +--- + +1. 在**集群**仪表æ¿ä¸­ï¼Œå•å‡»å·¦ä¾§å¯¼èˆªçš„ **Apps > Installed Apps**。 +1. å‰å¾€ `cis-operator-system` 命å空间，并选中 `rancher-cis-benchmark-crd` å’Œ `rancher-cis-benchmark` æ—边的框。 +1. å•å‡»**删除**并确认**删除**。 + +**结果**：已å¸è½½ `rancher-cis-benchmark` 应用。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md new file mode 100644 index 00000000000..adfcce53799 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md @@ -0,0 +1,12 @@ +--- +title: 查看报告 +--- + +è¦æŸ¥çœ‹ç”Ÿæˆçš„ CIS 扫æ报告： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. **扫æ**页é¢å°†æ˜¾ç¤ºç”Ÿæˆçš„报告。è¦æŸ¥çœ‹è¯¦ç»†æŠ¥å‘Šï¼Œè¯·è½¬åˆ°æ‰«æ报告并å•å‡»æŠ¥å‘Šå称。 + +ä½ å¯ä»¥ä»Žæ‰«æ列表或扫æ详情页é¢ä¸‹è½½æŠ¥å‘Šã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md new file mode 100644 index 00000000000..b10f0b61f1b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md @@ -0,0 +1,260 @@ +--- +title: 7 层 NGINX è´Ÿè½½å‡è¡¡å™¨ä¸Šçš„ TLS 终止（Docker 安装） +--- + +如果你的开å‘或测试环境è¦æ±‚在负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ TLS/SSL，而ä¸æ˜¯åœ¨ Rancher Server 上，请部署 Rancher 并é…置负载å‡è¡¡å™¨ã€‚ + +如果è¦åœ¨åŸºç¡€è®¾æ–½ä¸­å¯¹ TLS 集中进行终止，请使用 7 层负载å‡è¡¡å™¨ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚ + +本文中的安装步骤将引导你使用å•ä¸ªå®¹å™¨éƒ¨ç½² Rancher，并æä¾› 7 层 NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例é…置。 + +## æ“作系统，Docker，硬件和网络è¦æ±‚ + +请确ä¿ä½ çš„节点满足常规的[安装è¦æ±‚](../../pages-for-subheaders/installation-requirements.md)。 + +## å®‰è£…æ¦‚è¦ + + +## 1. é…ç½® Linux 主机 + +æ ¹æ®æˆ‘们的[è¦æ±‚](../../pages-for-subheaders/installation-requirements.md)é…置一个 Linux 主机æ¥å¯åŠ¨ Rancher Server。 + +## 2. 选择一个 SSL 选项并安装 Rancher + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +:::note 你是å¦éœ€è¦ï¼š + +- 完æˆç¦»çº¿å®‰è£…。 +- 记录所有 Rancher API 的事务。 + +继续之å‰ï¼Œè¯·å‚è§[高级选项](#高级选项)。 + +::: + +选择以下的选项之一： + +
+ 选项 A：使用你自己的è¯ä¹¦ - 自签å + +如果è¦ä½¿ç”¨è‡ªç­¾åè¯ä¹¦æ¥åŠ å¯†é€šä¿¡ï¼Œä½ å¿…须在负载å‡è¡¡å™¨ï¼ˆåŽç»­æ­¥éª¤ï¼‰å’Œ Rancher 容器上安装è¯ä¹¦ã€‚è¿è¡Œ Docker 命令部署 Rancher，将 Docker 指å‘ä½ çš„è¯ä¹¦ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +创建自签åè¯ä¹¦ã€‚ + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 + +::: + +**使用自签åè¯ä¹¦å®‰è£… Rancher**： + +1. 在è¿è¡Œ Docker 命令部署 Rancher 时，将 Docker 指å‘ä½ çš„ CA è¯ä¹¦æ–‡ä»¶ã€‚ + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/your_certificate_directory/cacerts.pem:/etc/rancher/ssl/cacerts.pem \ + rancher/rancher:latest + ``` + +
+
+ 选项 B：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +如果你的集群é¢å‘公众，则最好使用由公认 CA 签署的è¯ä¹¦ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 + +::: + +**使用授信 CA ç­¾å‘çš„è¯ä¹¦å®‰è£… Rancher**： + +如果你使用授信 CA ç­¾å‘çš„è¯ä¹¦ï¼Œä½ æ— éœ€åœ¨ Rancher 容器中安装è¯ä¹¦ã€‚但是，请确ä¿ä¸è¦ç”Ÿæˆå’Œå­˜å‚¨é»˜è®¤çš„ CA è¯ä¹¦ï¼ˆä½ å¯ä»¥é€šè¿‡å°† `--no-cacerts` å‚数传递给容器æ¥å®žçŽ°ï¼‰ã€‚ + +1. 输入以下命令： + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + rancher/rancher:latest --no-cacerts + ``` + +
+ +## 3. é…置负载å‡è¡¡å™¨ + +在 Rancher 容器å‰ä½¿ç”¨è´Ÿè½½å‡è¡¡å™¨æ—¶ï¼Œå®¹å™¨æ— éœ€ä»Žç«¯å£ 80 æˆ–ç«¯å£ 443 é‡å®šå‘端å£é€šä¿¡ã€‚ä½ å¯ä»¥é€šè¿‡ä¼ é€’ `X-Forwarded-Proto: https` 标头ç¦ç”¨æ­¤é‡å®šå‘。 + +è´Ÿè½½å‡è¡¡å™¨æˆ–代ç†å¿…须支æŒä»¥ä¸‹å†…容： + +- **WebSocket** 连接 +- **SPDY** / **HTTP/2** åè®® +- 传递/设置以下标头： + + | 标头 | 值 | æè¿° | + |--------|-------|-------------| + | `Host` | 用于访问 Rancher 的主机å。 | 识别客户端所请求的æœåŠ¡å™¨ã€‚ | + | `X-Forwarded-Proto` | `https` | 识别客户端连接负载å‡è¡¡å™¨æˆ–代ç†æ—¶æ‰€ç”¨çš„å议。

**注æ„**：如果此标头存在，`rancher/rancher` ä¸ä¼šå°† HTTP é‡å®šå‘到 HTTPS。 | + | `X-Forwarded-Port` | 用于访问 Rancher 的端å£ã€‚ | 识别客户端连接到负载å‡è¡¡å™¨æˆ–代ç†æ—¶æ‰€ç”¨çš„端å£ã€‚ | + | `X-Forwarded-For` | 客户端 IP åœ°å€ | 识别客户端的原始 IP 地å€ã€‚ | +### 示例 NGINX é…ç½® + +æ­¤ NGINX é…置已在 NGINX 1.14 上进行了测试。 + +:::note + +æ­¤ NGINX é…ç½®åªæ˜¯ä¸€ä¸ªç¤ºä¾‹ï¼Œå¯èƒ½ä¸é€‚åˆä½ çš„环境。如需查阅完整文档，请å‚è§ [NGINX è´Ÿè½½å‡è¡¡ - HTTP è´Ÿè½½å‡è¡¡](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/)。 + +::: + +- å°† `rancher-server` 替æ¢ä¸ºè¿è¡Œ Rancher 容器的节点的 IP 或主机å。 +- 将两处的 `FQDN` å‡æ›¿æ¢ä¸º Rancher çš„ DNS å称。 +- 把 `/certs/fullchain.pem` å’Œ `/certs/privkey.pem` 分别替æ¢ä¸ºæœåŠ¡å™¨è¯ä¹¦å’ŒæœåŠ¡å™¨è¯ä¹¦å¯†é’¥çš„ä½ç½®ã€‚ + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server rancher-server:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # 此项å…许执行的 shell 窗å£ä¿æŒå¼€å¯ï¼Œæœ€é•¿å¯è¾¾15分钟。ä¸ä½¿ç”¨æ­¤å‚æ•°çš„è¯ï¼Œé»˜è®¤1分钟åŽè‡ªåŠ¨å…³é—­ã€‚ + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` + +
+ +## åŽç»­æ“作 + +- **推è**：检查å•èŠ‚点[备份](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md)å’Œ[æ¢å¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md)。你å¯èƒ½æš‚时没有需è¦å¤‡ä»½çš„æ•°æ®ï¼Œä½†æ˜¯æˆ‘们建议你在常规使用 Rancher åŽåˆ›å»ºå¤‡ä»½ã€‚ +- 创建 Kubernetes 集群：[é…ç½® Kubernetes 集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)。 + +
+ +## 常è§é—®é¢˜å’Œæ•…障排除 + +如果你需è¦å¯¹è¯ä¹¦è¿›è¡Œæ•…障排除，请å‚è§[此章节](../../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +## 高级选项 + +### API 审计 + +如果你需è¦è®°å½•æ‰€æœ‰ Rancher API 事务，请将以下标志添加到安装命令中，从而å¯ç”¨ [API 审计](enable-api-audit-log.md)功能。 + + -e AUDIT_LEVEL=1 \ + -e AUDIT_LOG_PATH=/var/log/auditlog/rancher-api-audit.log \ + -e AUDIT_LOG_MAXAGE=20 \ + -e AUDIT_LOG_MAXBACKUP=20 \ + -e AUDIT_LOG_MAXSIZE=100 \ + +### 离线环境 + +如果你访问此页é¢æ˜¯ä¸ºäº†å®Œæˆ[离线安装](../../pages-for-subheaders/air-gapped-helm-cli-install.md)，则在è¿è¡Œå®‰è£…命令时，先将你的ç§æœ‰é•œåƒä»“库 URL 附加到 Server 标志中。也就是说，在 `rancher/rancher:latest` å‰é¢æ·»åŠ  `` å’Œç§æœ‰é•œåƒä»“库 URL。 + +**示例**： + + /rancher/rancher:latest + +### æŒä¹…åŒ–æ•°æ® + +Rancher 使用 etcd 作为数æ®å­˜å‚¨ã€‚如果 Rancher 是使用 Docker 安装的，Rancher ä¼šä½¿ç”¨åµŒå…¥å¼ etcd。æŒä¹…化数æ®ä½äºŽå®¹å™¨çš„ `/var/lib/rancher` 路径中。 + +ä½ å¯ä»¥å°†ä¸»æœºå·æŒ‚载到该ä½ç½®ï¼Œæ¥å°†æ•°æ®ä¿ç•™åœ¨è¿è¡Œå®ƒçš„主机上： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +æ­¤æ“ä½œéœ€è¦ [privileged 访问](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)。 + +这个 7 层 NGINX é…置已ç»åœ¨ NGINX 1.13（Mainline）和 1.14（Stable）版本上进行了测试。 + +:::note + +æ­¤ NGINX é…ç½®åªæ˜¯ä¸€ä¸ªç¤ºä¾‹ï¼Œå¯èƒ½ä¸é€‚åˆä½ çš„环境。如果需è¦æŸ¥é˜…完整文档，请å‚è§ [NGINX è´Ÿè½½å‡è¡¡ - TCP å’Œ UDP è´Ÿè½½å‡è¡¡å™¨](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/)。 + +::: + +``` +upstream rancher { + server rancher-server:80; +} + +map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; +} + +server { + listen 443 ssl http2; + server_name rancher.yourdomain.com; + ssl_certificate /etc/your_certificate_directory/fullchain.pem; + ssl_certificate_key /etc/your_certificate_directory/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # 此项å…许执行的 shell 窗å£ä¿æŒå¼€å¯ï¼Œæœ€é•¿å¯è¾¾15分钟。ä¸ä½¿ç”¨æ­¤å‚æ•°çš„è¯ï¼Œé»˜è®¤1分钟åŽè‡ªåŠ¨å…³é—­ã€‚ + proxy_read_timeout 900s; + proxy_buffering off; + } +} + +server { + listen 80; + server_name rancher.yourdomain.com; + return 301 https://$server_name$request_uri; +} +``` + +
+ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log.md new file mode 100644 index 00000000000..94974cb52f7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log.md @@ -0,0 +1,558 @@ +--- +title: å¯ç”¨ API 审计日志以记录系统事件 +--- + +ä½ å¯ä»¥å¯ç”¨ API 审计日志æ¥è®°å½•å„个用户å‘起的系统事件的顺åºã€‚通过查看日志，你å¯ä»¥äº†è§£å‘生了什么事件ã€äº‹ä»¶å‘生的时间，事件å‘起人，以åŠäº‹ä»¶å½±å“的集群。å¯ç”¨æ­¤åŠŸèƒ½åŽï¼Œæ‰€æœ‰ Rancher API 的请求和å“应都会写入日志中。 + +API 审计å¯ä»¥åœ¨ Rancher 安装或å‡çº§æœŸé—´å¯ç”¨ã€‚ + +## å¯ç”¨ API 审计日志 + +ä½ å¯ä»¥å°†çŽ¯å¢ƒå˜é‡ä¼ é€’ç»™ Rancher Server 容器，从而å¯ç”¨å’Œé…置审计日志。请å‚è§ä»¥ä¸‹æ–‡æ¡£ï¼Œåœ¨å®‰è£…æ—¶å¯ç”¨è¯¥åŠŸèƒ½ï¼š + +- [Docker 安装](../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志) + +- [Kubernetes 安装](../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-审计日志) + +## API 审计日志选项 + +以下å‚数定义了审计日志的记录规则，其中包括应该记录什么内容以åŠåŒ…括什么数æ®ï¼š + +| å‚æ•° | æè¿° | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AUDIT_LEVEL` | `0` - ç¦ç”¨å®¡è®¡æ—¥å¿—（默认）
`1` - 日志事件元数æ®
`2` - 日志事件元数æ®å’Œè¯·æ±‚体
`3` - 日志事件元数æ®ï¼Œè¯·æ±‚体和å“应体。请求/å“应对的æ¯ä¸ªæ—¥å¿—事务都使用åŒä¸€ä¸ªçš„ `auditID`。
如需了解æ¯ä¸ªè®¾ç½®è®°å½•çš„日志内容，请å‚è§[审计日志级别](#审核日志级别)。 | +| `AUDIT_LOG_PATH` | Rancher Server API 的日志路径。默认路径：`/var/log/auditlog/rancher-api-audit.log`。你å¯ä»¥å°†æ—¥å¿—目录挂载到主机。

示例：`AUDIT_LOG_PATH=/my/custom/path/`
| +| `AUDIT_LOG_MAXAGE` | 旧审计日志文件å¯ä¿ç•™çš„最大天数。默认为 10 天。 | +| `AUDIT_LOG_MAXBACKUP` | ä¿ç•™çš„审计日志最大文件个数。默认值为 10。 | +| `AUDIT_LOG_MAXSIZE` | 在审计日志文件被轮æ¢å‰çš„最大容é‡ï¼Œå•ä½æ˜¯ MB。默认大å°ä¸º 100MB。 | + +
+ +### 审核日志级别 + +下表介ç»äº†æ¯ä¸ª [`AUDIT_LEVEL`](#audit-level) 记录的 API 事务： + +| `AUDIT_LEVEL` 设置 | è¯·æ±‚å…ƒæ•°æ® | 请求体 | å“åº”å…ƒæ•°æ® | å“应体 | +| --------------------- | ---------------- | ------------ | ----------------- | ------------- | +| `0` | | | | | +| `1` | ✓ | | | | +| `2` | ✓ | ✓ | | | +| `3` | ✓ | ✓ | ✓ | ✓ | + +## 查看 API 审计日志 + +### Docker 安装 + +与主机系统共享 `AUDIT_LOG_PATH` 目录（默认目录：`/var/log/auditlog`）。日志å¯ä»¥é€šè¿‡æ ‡å‡† CLI 工具进行解æžï¼Œä¹Ÿå¯ä»¥è½¬å‘到 Fluentdã€Filebeatã€Logstash 等日志收集工具。 + +### Kubernetes 安装 + +使用 Helm Chart 安装 Rancher æ—¶å¯åŠ¨ API 审计日志，会在 Rancher Pod 中创建一个 `rancher-audit-log` Sidecar 容器。该容器会将日志å‘é€åˆ°æ ‡å‡†è¾“出 (stdout)。你å¯ä»¥åƒæŸ¥çœ‹å…¶ä»–容器的日志一样查看 API 审计日志。 + +`rancher-audit-log` 容器ä½äºŽ `cattle-system` 命å空间中的 `rancher` Pod 中。 + +#### CLI + +```bash +kubectl -n cattle-system logs -f rancher-84d886bdbb-s4s69 rancher-audit-log +``` + +#### å‘é€å®¡è®¡æ—¥å¿— + +ä½ å¯ä»¥ä¸ºé›†ç¾¤å¯ç”¨ Rancher 的内置日志收集和传é€åŠŸèƒ½ï¼Œå°†å®¡è®¡æ—¥å¿—和其他æœåŠ¡æ—¥å¿—å‘é€åˆ°æ”¯æŒçš„ endpoint。详情请å‚è§ [Rancher 工具 - Logging](../../pages-for-subheaders/logging.md)。 + +## 审计日志示例 + +å¯ç”¨å®¡è®¡æ—¥å¿—åŽï¼ŒRancher 会以 JSON æ ¼å¼è®°å½•æ¯ä¸ª API 的请求和å“应。下文的代ç ç¤ºä¾‹å±•ç¤ºäº†å¦‚何查看 API 事务。 + +### 元数æ®æ—¥å¿—级别 + +如果你将 `AUDIT_LEVEL` 设置为 `1`，Rancher åªä¼šè®°å½•æ¯ä¸ª API 请求的元数æ®æ ‡å¤´ï¼Œè€Œä¸ä¼šè®°å½•è¯·æ±‚体。标头记录了 API 事务的基本信æ¯ï¼ŒåŒ…括 IDã€å‘起人ã€å‘起时间等。代ç ç¤ºä¾‹å¦‚下： + +```json +{ + "auditID": "30022177-9e2e-43d1-b0d0-06ef9d3db183", + "requestURI": "/v3/schemas", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "GET", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:22:43 +0800" +} +``` + +### 元数æ®å’Œè¯·æ±‚体日志级别 + +如果你将 `AUDIT_LEVEL` 设置为 `2`，Rancher 会记录æ¯ä¸ª API 请求的元数æ®æ ‡å¤´å’Œè¯·æ±‚体。 + +下é¢çš„代ç ç¤ºä¾‹æ述了一个 API 请求，包括它的元数æ®æ ‡å¤´å’Œæ­£æ–‡ï¼š + +```json +{ + "auditID": "ef1d249e-bfac-4fd0-a61f-cbdcad53b9bb", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:28:08 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my description", + "volumes": [] + } +} +``` + +### 元数æ®ã€è¯·æ±‚体和å“应体日志级别 + +如果你将 `AUDIT_LEVEL` 设置为 `3`，Rancher 会记录： + +- æ¯ä¸ª API 请求的元数æ®æ ‡å¤´å’Œè¯·æ±‚体。 +- æ¯ä¸ª API å“应的元数æ®æ ‡å¤´å’Œå“应体。 + +#### 请求 + +下é¢çš„代ç ç¤ºä¾‹æ述了一个 API 请求，包括它的元数æ®æ ‡å¤´å’Œæ­£æ–‡ï¼š + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my decript", + "volumes": [] + } +} +``` + +#### å“应 + +下é¢çš„代ç ç¤ºä¾‹æ述了一个 API å“应，包括它的元数æ®æ ‡å¤´å’Œæ­£æ–‡ï¼š + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "responseStatus": "200", + "stage": "ResponseComplete", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "responseBody": { + "actionLinks": { + "pause": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=pause", + "resume": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=resume", + "rollback": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=rollback" + }, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements" + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container" + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "links": { + "remove": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "revisions": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/revisions", + "self": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "update": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "yaml": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/yaml" + }, + "name": "nginx", + "namespaceId": "default", + "paused": false, + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + } + } +} +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md new file mode 100644 index 00000000000..c1f42f70a4b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md @@ -0,0 +1,13 @@ +--- +title: æŒç»­äº¤ä»˜ +--- + +Rancher 中预装的 [Fleet](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) 无法完全ç¦ç”¨ã€‚但是，你å¯ä»¥ä½¿ç”¨ `continuous-delivery` 功能开关æ¥ç¦ç”¨ GitOps æŒç»­äº¤ä»˜çš„ Fleet 功能。 + +如需å¯ç”¨æˆ–ç¦ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·å‚è§[å¯ç”¨å®žéªŒåŠŸèƒ½ä¸»é¡µ](../../../pages-for-subheaders/enable-experimental-features.md)中的说明。 + +| 环境å˜é‡é”® | 默认值 | æè¿° | +---|---|--- +| `continuous-delivery` | `true` | 此开关ç¦ç”¨ Fleet çš„ GitOps æŒç»­äº¤ä»˜åŠŸèƒ½ã€‚ | + +如果你在 Rancher 2.5.x 中ç¦ç”¨äº† Fleet，然åŽå°† Rancher å‡çº§åˆ° v2.6.x，Fleet å°†å¯ç”¨ã€‚åªæœ‰ Fleet çš„æŒç»­äº¤ä»˜åŠŸèƒ½å¯ä»¥è¢«ç¦ç”¨ã€‚当 `continuous-delivery` 被ç¦ç”¨æ—¶ï¼Œ`gitjob` deployment ä¸å†éƒ¨ç½²åˆ° Rancher Server 的本地集群中，且 `continuous-delivery` ä¸ä¼šåœ¨ Rancher UI 中显示。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md new file mode 100644 index 00000000000..35801de23b4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md @@ -0,0 +1,32 @@ +--- +title: UI ç®¡ç† Istio 虚拟æœåŠ¡å’Œç›®æ ‡è§„则 +--- + +此功能å¯å¯åŠ¨ä¸€ä¸ª UIï¼Œç”¨äºŽç®¡ç† Istio çš„æµé‡ï¼Œå…¶ä¸­åŒ…括创建ã€è¯»å–ã€æ›´æ–°å’Œåˆ é™¤è™šæ‹ŸæœåŠ¡ï¼ˆVirtual Service）和目标规则（Destination Rule）。 + +> **注æ„**：å¯ç”¨æ­¤åŠŸèƒ½å¹¶ä¸ä¼šå¯ç”¨ Istio。集群管ç†å‘˜éœ€è¦[为集群å¯ç”¨ Istio](../../../pages-for-subheaders/istio-setup-guide.md) æ‰èƒ½ä½¿ç”¨è¯¥åŠŸèƒ½ã€‚ + +如需å¯ç”¨æˆ–ç¦ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·å‚è§[å¯ç”¨å®žéªŒåŠŸèƒ½ä¸»é¡µ](../../../pages-for-subheaders/enable-experimental-features.md)中的说明。 + +| 环境å˜é‡é”® | 默认值 | çŠ¶æ€ | å¯ç”¨äºŽ | +---|---|---|--- +| `istio-virtual-service-ui` | `false` | 实验功能 | v2.3.0 | +| `istio-virtual-service-ui` | `true` | GA | v2.3.2 | + +## åŠŸèƒ½ä»‹ç» + +Istio æµé‡ç®¡ç†åŠŸèƒ½çš„主è¦ä¼˜åŠ¿æ—¶å…许动æ€è¯·æ±‚路由，这对于金ä¸é›€å‘布，è“/绿å‘布或 A/B 测试都éžå¸¸æœ‰ç”¨ã€‚ + +å¯ç”¨æ­¤åŠŸèƒ½åŽï¼Œä¸€ä¸ªé¡µé¢ä¼šæ‰“开，让你通过 Rancher UI é…ç½® Istio çš„æŸäº›æµé‡ç®¡ç†åŠŸèƒ½ã€‚如果ä¸ä½¿ç”¨æ­¤åŠŸèƒ½ï¼Œä½ å¯ä»¥é€šè¿‡ `kubectl` æ¥ä½¿ç”¨ Istio 管ç†æµé‡ã€‚ + +此功能会å¯ç”¨ä¸¤ä¸ªé€‰é¡¹å¡ï¼Œä¸€ä¸ªç”¨äºŽ**虚拟æœåŠ¡**，å¦ä¸€ä¸ªç”¨äºŽ**目标规则**。 + +- **虚拟æœåŠ¡**：拦截并将æµé‡é‡å®šå‘到你的 Kubernetes Service 上。这样，你å¯ä»¥å°†éƒ¨åˆ†è¯·æ±‚æµé‡å®šå‘到ä¸åŒçš„æœåŠ¡ä¸Šã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›æœåŠ¡æ¥å®šä¹‰ä¸€ç»„路由规则，用于主机寻å€ã€‚详情请å‚è§ [Istio 官方文档](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/)。 +- **目标规则**：作为唯一å¯ä¿¡æ¥æºï¼Œè¡¨æ˜Žå“ªäº›æœåŠ¡ç‰ˆæœ¬å¯ç”¨äºŽæŽ¥æ”¶è™šæ‹ŸæœåŠ¡çš„æµé‡ã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºæ¥å®šä¹‰ç­–略，这些策略适用于路由å‘生åŽç”¨äºŽæœåŠ¡çš„æµé‡ã€‚详情请å‚è§ [Istio 官方文档](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule)。 + +如需查看选项å¡ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 转到安装了 Istio 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio**。 +1. 你将看到 **Kiali** å’Œ **Jaeger** 的选项å¡ã€‚在左侧导航æ ä¸­ï¼Œä½ å¯æŸ¥çœ‹å’Œé…ç½®**虚拟æœåŠ¡**å’Œ**目标规则**。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md new file mode 100644 index 00000000000..16ebb15e25a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md @@ -0,0 +1,44 @@ +--- +title: "在 ARM64 上è¿è¡Œ Rancher（实验性）" +--- + +:::caution + +在使用 ARM64 架构的节点上è¿è¡Œ Rancher ç›®å‰è¿˜å¤„在实验阶段，Rancher 尚未正å¼æ”¯æŒè¯¥åŠŸèƒ½ã€‚因此，我们ä¸å»ºè®®ä½ åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­ä½¿ç”¨ ARM64 架构的节点。 + +::: + +如果你的节点使用 ARM64 架构，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹é€‰é¡¹ï¼š + +- 在 ARM64 架构的节点上è¿è¡Œ Rancher + - 此选项仅适用于 Docker 安装。请知悉，以下安装命令å–代了 [Docker 安装链接](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md)中的示例： + + ``` + # 在最åŽä¸€è¡Œ `rancher/rancher:vX.Y.Z` 中，请务必将 "X.Y.Z" 替æ¢ä¸ºåŒ…å« ARM64 版本的å‘布版本。例如，如果你的匹é…版本是 v2.5.8，请在此行填写 `rancher/rancher:v2.5.8`。 + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:vX.Y.Z + ``` + +:::note + +è¦æ£€æŸ¥ä½ çš„å‘行版本是å¦ä¸Ž ARM64 架构兼容，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹ä¸¤ç§æ–¹å¼æ‰¾åˆ°å¯¹åº”版本的å‘行说明： + +- 访问 [Rancher å‘行版本](https://github.com/rancher/rancher/releases)自行查询。 +- æ ¹æ®æ ‡ç­¾å’Œç‰ˆæœ¬å·ç›´æŽ¥æ‰¾åˆ°ä½ çš„版本。例如，你使用的版本为 2.5.8，你å¯ä»¥è®¿é—® [Rancher å‘行版本 - 2.5.8](https://github.com/rancher/rancher/releases/tag/v2.5.8)。 + +::: + +- 创建自定义集群并添加使用 ARM64 架构的节点 + - Kubernetes 集群必须为 1.12 或更高版本 + - CNI 网络æ’件必须是 [Flannel](../../../faq/container-network-interface-providers.md#flannel) +- 导入包å«ä½¿ç”¨ ARM64 架构的节点的集群 + - Kubernetes 集群必须为 1.12 或更高版本 + +如需了解如何é…置集群选项，请å‚è§[集群选项](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + +以下是未ç»æµ‹è¯•çš„功能： + +- Monitoringã€å‘Šè­¦ã€Notifiersã€æµæ°´çº¿å’Œ Logging +- 通过应用商店å‘布应用 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md new file mode 100644 index 00000000000..19b9b27b2e0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md @@ -0,0 +1,39 @@ +--- +title: 使用éžé»˜è®¤æ”¯æŒçš„存储驱动 +--- + +此功能å…许你使用ä¸æ˜¯é»˜è®¤å¯ç”¨çš„存储æ供商和å·æ’件。 + +如需å¯ç”¨æˆ–ç¦ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·å‚è§[å¯ç”¨å®žéªŒåŠŸèƒ½ä¸»é¡µ](../../../pages-for-subheaders/enable-experimental-features.md)中的说明。 + +| 环境å˜é‡é”® | 默认值 | æè¿° | +---|---|--- +| `unsupported-storage-drivers` | `false` | å¯ç”¨éžé»˜è®¤å¯ç”¨çš„存储æ供商和å·æ’件。 | + +### 默认å¯ç”¨çš„æŒä¹…å·æ’件 +下表æ述了默认å¯ç”¨çš„存储类型对应的æŒä¹…å·æ’件。å¯ç”¨æ­¤åŠŸèƒ½å¼€å…³æ—¶ï¼Œä¸åœ¨æ­¤åˆ—表中的任何æŒä¹…å·æ’件å‡è¢«è§†ä¸ºå®žéªŒåŠŸèƒ½ï¼Œä¸”ä¸å—支æŒï¼š + +| å称 | æ’件 | +--------|---------- +| Amazon EBS Disk | `aws-ebs` | +| AzureFile | `azure-file` | +| AzureDisk | `azure-disk` | +| Google Persistent Disk | `gce-pd` | +| Longhorn | `flex-volume-longhorn` | +| VMware vSphere Volume | `vsphere-volume` | +| 本地 | `local` | +| 网络文件系统 | `nfs` | +| hostPath | `host-path` | + +### 默认å¯ç”¨çš„ StorageClass +下表æ述了默认å¯ç”¨çš„ StorageClass 对应的æŒä¹…å·æ’件。å¯ç”¨æ­¤åŠŸèƒ½å¼€å…³æ—¶ï¼Œä¸åœ¨æ­¤åˆ—表中的任何æŒä¹…å·æ’件å‡è¢«è§†ä¸ºå®žéªŒåŠŸèƒ½ï¼Œä¸”ä¸å—支æŒï¼š + +| å称 | æ’件 | +--------|-------- +| Amazon EBS Disk | `aws-ebs` | +| AzureFile | `azure-file` | +| AzureDisk | `azure-disk` | +| Google Persistent Disk | `gce-pd` | +| Longhorn | `flex-volume-longhorn` | +| VMware vSphere Volume | `vsphere-volume` | +| 本地 | `local` | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md new file mode 100644 index 00000000000..39c9bdee036 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md @@ -0,0 +1,29 @@ +--- +title: 1. 在集群中å¯ç”¨ Istio +--- + +:::note 先决æ¡ä»¶ï¼š + +- åªæœ‰åˆ†é…了 `cluster-admin` [Kubernetes 默认角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)的用户å¯ä»¥åœ¨ Kubernetes 集群中é…置和安装 Istio。 +- 如果你有 pod 安全策略，则需è¦å®‰è£…å¯ç”¨äº† CNI çš„ Istio。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../integrations-in-rancher/istio/configuration-options/pod-security-policies.md)。 +- è¦åœ¨ RKE2 集群上安装 Istio，则需è¦æ‰§è¡Œé¢å¤–的步骤。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md)。 +- è¦åœ¨å¯ç”¨äº†é¡¹ç›®ç½‘络隔离的集群中安装 Istio，则需è¦æ‰§è¡Œé¢å¤–的步骤。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../integrations-in-rancher/istio/configuration-options/project-network-isolation.md)。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å¯ç”¨ Istio çš„ä½ç½®ï¼Œç„¶åŽå•å‡» **Explore**。 +1. å•å‡» **Apps**。 +1. å•å‡» **Chart**。 +1. å•å‡» **Istio**。 +1. 如果你还没有安装 Monitoring 应用，系统会æ示你安装 rancher-monitoring。你也å¯ä»¥é€‰æ‹©åœ¨ Rancher-monitoring 安装上设置选择器或抓å–é…置选项。 +1. å¯é€‰ï¼šä¸º Istio 组件é…ç½®æˆå‘˜è®¿é—®å’Œ[资æºé™åˆ¶](../../../integrations-in-rancher/istio/cpu-and-memory-allocations.md)。确ä¿ä½ çš„ Worker 节点上有足够的资æºæ¥å¯ç”¨ Istio。 +1. å¯é€‰ï¼šå¦‚果需è¦ï¼Œå¯¹ values.yaml 进行é¢å¤–çš„é…置更改。 +1. å¯é€‰ï¼šé€šè¿‡[覆盖文件](../../../pages-for-subheaders/configuration-options.md#覆盖文件)æ¥æ·»åŠ å…¶ä»–资æºæˆ–é…置。 +1. å•å‡»**安装**。 + +**结果**：已在集群级别安装 Istio。 + +## 其他é…置选项 + +有关é…ç½® Istio 的更多信æ¯ï¼Œè¯·å‚阅[é…ç½®å‚考](../../../pages-for-subheaders/configuration-options.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md new file mode 100644 index 00000000000..3651b9d77c6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md @@ -0,0 +1,53 @@ +--- +title: 2. 在命å空间中å¯ç”¨ Istio +--- + +你需è¦åœ¨éœ€è¦ç”± Istio 跟踪或控制的æ¯ä¸ªå‘½å空间中手动å¯ç”¨ Istio。在命å空间中å¯ç”¨ Istio 时，Envoy sidecar 代ç†å°†è‡ªåŠ¨æ³¨å…¥åˆ°éƒ¨ç½²åœ¨å‘½å空间中的所有新工作负载中。 + +此命å空间设置åªä¼šå½±å“命å空间中的新工作负载。之å‰çš„工作负载需è¦é‡æ–°éƒ¨ç½²æ‰èƒ½ä½¿ç”¨ sidecar 自动注入。 + +:::note 先决æ¡ä»¶ï¼š + +è¦åœ¨å‘½å空间中å¯ç”¨ Istio，集群必须安装 Istio。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦å¯ç”¨ Istio 的命å空间，然åŽå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。或者，你也å¯ä»¥å•å‡»å‘½å空间，然åŽåœ¨å‘½å空间详情页é¢ä¸Šï¼Œå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。 + +**结果**：命å空间带有了 `istio-injection=enabled` 标签。默认情况下，部署在此命å空间中的所有新工作负载都将注入 Istio sidecar。 + +### 验è¯æ˜¯å¦å¯ç”¨äº†è‡ªåŠ¨ Istio Sidecar 注入 + +è¦éªŒè¯ Istio 是å¦å·²å¯ç”¨ï¼Œè¯·åœ¨å‘½å空间中部署一个 hello-world 工作负载。转到工作负载并å•å‡» pod å称。在**容器**中，你应该能看到 `istio-proxy` 容器。 + +### 排除工作负载的 Istio Sidecar 注入 + +è¦æŽ’除 Istio sidecar 被注入æŸå·¥ä½œè´Ÿè½½ï¼Œè¯·åœ¨å·¥ä½œè´Ÿè½½ä¸Šä½¿ç”¨ä»¥ä¸‹æ³¨é‡Šï¼š + +``` +sidecar.istio.io/inject: “false†+``` + +è¦å°†æ³¨é‡Šæ·»åŠ åˆ°å·¥ä½œè´Ÿè½½ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**工作负载**。 +1. 转到ä¸éœ€è¦ sidecar 的工作负载并以 yaml 编辑。 +1. 将键值 `sidecar.istio.io/inject: false` 添加为工作负载的注释。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：Istio sidecar ä¸ä¼šè¢«æ³¨å…¥åˆ°å·¥ä½œè´Ÿè½½ä¸­ã€‚ + +:::note + +如果你é‡åˆ°éƒ¨ç½²çš„ job 未完æˆçš„问题，则需è¦ä½¿ç”¨æ供的步骤将此注释添加到 pod 中。由于 Istio Sidecars 会一直è¿è¡Œï¼Œå› æ­¤å³ä½¿ä»»åŠ¡å®Œæˆäº†ï¼Œä¹Ÿä¸èƒ½è®¤ä¸º Job 已完æˆã€‚ + +::: + + +### åŽç»­æ­¥éª¤ +[使用 Istio Sidecar 添加部署](use-istio-sidecar.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md new file mode 100644 index 00000000000..a6d77763919 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md @@ -0,0 +1,26 @@ +--- +title: 6. 生æˆå’ŒæŸ¥çœ‹æµé‡ +--- + +本文介ç»å¦‚何查看 Istio 管ç†çš„æµé‡ã€‚ + +## Kiali æµé‡å›¾ + +Istio 概览页é¢æ供了 Kiali 仪表æ¿çš„链接。在 Kiali 仪表æ¿ä¸­ï¼Œä½ å¯ä»¥æŸ¥çœ‹æ¯ä¸ªå‘½å空间的图。Kiali 图æ供了一ç§å¼ºå¤§çš„æ–¹å¼æ¥å¯è§†åŒ– Istio æœåŠ¡ç½‘格的拓扑。它显示了æœåŠ¡ä¹‹é—´ç›¸äº’通信的情况。 + +:::note 先决æ¡ä»¶ï¼š + +è¦æ˜¾ç¤ºæµé‡å›¾ï¼Œè¯·ç¡®ä¿ä½ åœ¨é›†ç¾¤ä¸­å®‰è£…了 Prometheus。Rancher-istio 安装了默认é…置的 Kiali æ¥ä¸Ž rancher-monitoring Chart 一起工作。你å¯ä»¥ä½¿ç”¨ rancher-monitoring 或安装自己的监控解决方案。你也å¯ä»¥é€šè¿‡è®¾ç½®[选择器 & 抓å–é…ç½®](../../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)选项æ¥æ›´æ”¹æ•°æ®æŠ“å–çš„é…置（å¯é€‰ï¼‰ã€‚ + +::: + +è¦æŸ¥çœ‹æµé‡å›¾ï¼š + +1. 在安装了 Istio 的集群中，点击左侧导航æ ä¸­çš„ **Istio**。 +1. å•å‡» **Kiali** 链接。 +1. å•å‡»ä¾§å¯¼èˆªä¸­çš„**图**。 +1. 在**命å空间**下拉列表中，更改命å空间以查看æ¯ä¸ªå‘½å空间的æµé‡ã€‚ + +如果你多次刷新 BookInfo 应用的 URL，你将能够在 Kiali 图上看到绿色箭头，显示 `reviews` æœåŠ¡ `v1` å’Œ `v3` çš„æµé‡ã€‚图å³ä¾§çš„控制é¢æ¿å¯ç”¨äºŽé…置详细信æ¯ï¼ŒåŒ…括应在图上显示多少分钟的最新æµé‡ã€‚ + +对于其他工具和å¯è§†åŒ–，你å¯ä»¥ä»Ž**监控** > **概览**页é¢è½¬åˆ° Grafana å’Œ Prometheus 仪表æ¿ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md new file mode 100644 index 00000000000..42f89baa8ef --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md @@ -0,0 +1,147 @@ +--- +title: 4. 设置 Istio Gateway +--- + +æ¯ä¸ªé›†ç¾¤çš„网关å¯ä»¥æœ‰è‡ªå·±çš„端å£æˆ–è´Ÿè½½å‡è¡¡å™¨ï¼Œè¿™ä¸ŽæœåŠ¡ç½‘格无关。默认情况下，æ¯ä¸ª Rancher é…置的集群都有一个 NGINX Ingress Controller æ¥å…许æµé‡è¿›å…¥é›†ç¾¤ã€‚ + +无论是å¦å®‰è£…了 Istio，你都å¯ä»¥ä½¿ç”¨ NGINX Ingress Controller。如果这是你集群的唯一网关，Istio 将能够将æµé‡ä»Žé›†ç¾¤å†…部的æœåŠ¡è·¯ç”±åˆ°é›†ç¾¤å†…部的å¦ä¸€ä¸ªæœåŠ¡ï¼Œä½† Istio 将无法接收æ¥è‡ªé›†ç¾¤å¤–部的æµé‡ã€‚ + +è¦è®© Istio 接收外部æµé‡ï¼Œä½ éœ€è¦å¯ç”¨ Istio 的网关，作为外部æµé‡çš„å—北代ç†ã€‚å¯ç”¨ Istio Gateway åŽï¼Œä½ çš„集群将有两个 Ingress。 + +你还需è¦ä¸ºä½ çš„æœåŠ¡è®¾ç½® Kubernetes 网关。此 Kubernetes 资æºæŒ‡å‘ Istio 对集群 Ingress Gateway 的实现。 + +ä½ å¯ä»¥ä½¿ç”¨è´Ÿè½½å‡è¡¡å™¨å°†æµé‡è·¯ç”±åˆ°æœåŠ¡ç½‘格中，或使用 Istio çš„ NodePort 网关。本文介ç»å¦‚何设置 NodePort 网关。 + +有关 Istio Gateway 的更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](https://istio.io/docs/reference/config/networking/v1alpha3/gateway/)。 + +![å¯ç”¨ Istio 的集群å¯ä»¥æœ‰ä¸¤ä¸ª ingress，分别是默认的 Nginx ingress 和默认的 Istio controller](/img/istio-ingress.svg) + +## å¯ç”¨ Istio Gateway + +Ingress Gateway 是一个 Kubernetes æœåŠ¡ï¼Œå°†éƒ¨ç½²åœ¨ä½ çš„集群中。Istio Gateway 支æŒæ›´å¤šè‡ªå®šä¹‰è®¾ç½®ï¼Œæ›´åŠ çµæ´»ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio > 网关**。 +1. å•å‡»**使用 YAML 文件创建**。 +1. 粘贴你的 Istio Gateway yaml，或选择**从文件读å–**。 +1. å•å‡»**创建**。 + +**结果**：已部署网关，将使用应用的规则æ¥è·¯ç”±æµé‡ã€‚ + +## Istio Gateway 示例 + +在演示工作负载示例时，我们在æœåŠ¡ä¸­æ·»åŠ  BookInfo 应用部署。接下æ¥ï¼Œæˆ‘们添加一个 Istio Gateway，以便从集群外部访问该应用。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio > 网关**。 +1. å•å‡»**使用 YAML 文件创建**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ Gateway YAML。 +1. å•å‡»**创建**。 + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: bookinfo-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +--- +``` + +然åŽï¼Œéƒ¨ç½²ä¸º Gateway æä¾›æµé‡è·¯ç”±çš„ VirtualService： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio > VirtualServices**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ VirtualService YAML。 +1. å•å‡»**创建**。 + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: bookinfo +spec: + hosts: + - "*" + gateways: + - bookinfo-gateway + http: + - match: + - uri: + exact: /productpage + - uri: + prefix: /static + - uri: + exact: /login + - uri: + exact: /logout + - uri: + prefix: /api/v1/products + route: + - destination: + host: productpage + port: + number: 9080 +``` + +**结果**：你已é…置网关资æºï¼ŒIstio 现在å¯ä»¥æŽ¥æ”¶é›†ç¾¤å¤–部的æµé‡ã€‚ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥ç¡®è®¤èµ„æºå­˜åœ¨ï¼š +``` +kubectl get gateway -A +``` + +结果应与以下内容类似： +``` +NAME AGE +bookinfo-gateway 64m +``` + +### 在 Web æµè§ˆå™¨è®¿é—® ProductPage æœåŠ¡ + +è¦æµ‹è¯• BookInfo 应用是å¦å·²æ­£ç¡®éƒ¨ç½²ï¼Œä½ å¯ä»¥ä½¿ç”¨ Istio 控制器 IP 和端å£ä»¥åŠåœ¨ Kubernetes 网关资æºä¸­æŒ‡å®šçš„请求å称，在 Web æµè§ˆå™¨ä¸­æŸ¥çœ‹è¯¥åº”用： + +`http://:/productpage` + +è¦èŽ·å– Ingress Gateway URL 和端å£ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å‘下滚动到 `istio-system` 命å空间。 +1. 在 `istio-system`中，有一个å为 `istio-ingressgateway` 的工作负载。在此工作负载的å称下，你应该会看到如 `80/tcp` 的链接。 +1. å•å‡»å…¶ä¸­ä¸€ä¸ªé“¾æŽ¥ã€‚然åŽï¼Œä½ çš„ Web æµè§ˆå™¨ä¸­ä¼šæ˜¾ç¤º Ingress Gateway çš„ URL。将 `/productpage` 尾附到 URL。 + +**结果**：你能会在 Web æµè§ˆå™¨ä¸­çœ‹åˆ° BookInfo 应用。 + +如需检查 Istio 控制器 URL 和端å£çš„帮助，请å°è¯•è¿è¡Œ [Istio 文档](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#determining-the-ingress-ip-and-ports)中的命令。 + +## 故障排除 + +[官方 Istio 文档](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#troubleshooting)建议使用 `kubectl` 命令æ¥æ£€æŸ¥å¤–部请求的正确 ingress 主机和 ingress 端å£ã€‚ + +### 确认 Kubernetes 网关与 Istio çš„ Ingress Controller åŒ¹é… + +ä½ å¯ä»¥å°è¯•æ‰§è¡Œæœ¬èŠ‚ä¸­çš„æ­¥éª¤ä»¥ç¡®ä¿ Kubernetes 网关é…置正确。 + +在网关资æºä¸­ï¼Œé€‰æ‹©å™¨é€šè¿‡æ ‡ç­¾æ¥å¼•ç”¨ Istio 的默认 Ingress Controller，其中标签的键是 `Istio`，值是 `ingressgateway`。è¦ç¡®ä¿æ ‡ç­¾é€‚用于网关，请执行以下æ“作： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å‘下滚动到 `istio-system` 命å空间。 +1. 在 `istio-system`中，有一个å为 `istio-ingressgateway` 的工作负载。å•å‡»æ­¤å·¥ä½œè´Ÿè½½çš„å称并转到**标签和注释**部分。你应该看到它具有 `istio` 键和 `ingressgateway` 值。这确认了 Gateway 资æºä¸­çš„选择器与 Istio 的默认 ingress controller 匹é…。 + +### åŽç»­æ­¥éª¤ +[设置 Istio çš„æµé‡ç®¡ç†ç»„件](set-up-traffic-management.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md new file mode 100644 index 00000000000..9782bd28938 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md @@ -0,0 +1,76 @@ +--- +title: 5. 设置 Istio çš„æµé‡ç®¡ç†ç»„件 +--- + +Istio 中æµé‡ç®¡ç†çš„一个核心优势是å…许动æ€è¯·æ±‚路由。动æ€è¯·æ±‚路由通常应用于金ä¸é›€éƒ¨ç½²å’Œè“/绿部署等。Istio æµé‡ç®¡ç†ä¸­çš„两个关键资æºæ˜¯*虚拟æœåŠ¡*å’Œ*目标规则*。 + +- [虚拟æœåŠ¡](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/)：拦截并将æµé‡é‡å®šå‘到你的 Kubernetes Service 上。这样，你å¯ä»¥å°†éƒ¨åˆ†è¯·æ±‚æµé‡åˆ†é…到ä¸åŒçš„æœåŠ¡ä¸Šã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›æœåŠ¡æ¥å®šä¹‰ä¸€ç»„路由规则，用于主机寻å€ã€‚ +- [目标规则](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule/)：作为唯一å¯ä¿¡æ¥æºï¼Œè¡¨æ˜Žå“ªäº›æœåŠ¡ç‰ˆæœ¬å¯ç”¨äºŽæŽ¥æ”¶è™šæ‹ŸæœåŠ¡çš„æµé‡ã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºæ¥å®šä¹‰ç­–略，这些策略适用于路由å‘生åŽç”¨äºŽæœåŠ¡çš„æµé‡ã€‚ + +本文介ç»å¦‚何在示例 BookInfo 应用中添加与 `reviews` å¾®æœåŠ¡å¯¹åº”的虚拟æœåŠ¡ç¤ºä¾‹ã€‚æ­¤æœåŠ¡çš„目的是在 `reviews` æœåŠ¡çš„两个版本之间划分æµé‡ã€‚ + +在这个示例中，我们将æµé‡å¸¦åˆ° `reviews` æœåŠ¡ä¸­å¹¶æ‹¦æˆªæµé‡ï¼Œè¿™æ ·ï¼Œ50% çš„æµé‡ä¼šæµå‘æœåŠ¡çš„ `v1`，å¦å¤– 50% çš„æµé‡ä¼šæµå‘ `v2 `。 + +部署这个虚拟æœåŠ¡åŽï¼Œæˆ‘们将生æˆæµé‡ï¼Œå¹¶é€šè¿‡ Kiali å¯è§†åŒ–看到æµé‡å¹³å‡è·¯ç”±åˆ°æœåŠ¡çš„两个版本中。 + +è¦ä¸º `reviews` æœåŠ¡éƒ¨ç½²è™šæ‹ŸæœåŠ¡å’Œç›®æ ‡è§„则： +1. 点击 **☰ > 集群管ç†**。 +1. 转到安装了 Istio 的集群，然åŽå•å‡» **Explore**。 +1. 在安装了 Istio 的集群中，点击左侧导航æ ä¸­çš„ **Istio > DestinationRules**。 +1. å•å‡»**创建**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ DestinationRule YAML。 +1. å•å‡»**创建**。 +1. å•å‡»**以 YAML 文件编辑**并使用此é…置： + + ```yaml + apiVersion: networking.istio.io/v1alpha3 + kind: DestinationRule + metadata: + name: reviews + spec: + host: reviews + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 + - name: v3 + labels: + version: v3 + ``` +1. å•å‡»**创建**。 + +然åŽï¼Œéƒ¨ç½²æ供利用 DestinationRule çš„æµé‡è·¯ç”±çš„ VirtualService： + +1. å•å‡»ä¾§å¯¼èˆªæ ä¸­çš„ **VirtualService**。 +1. å•å‡»**使用 YAML 文件创建**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ VirtualService YAML。 +1. å•å‡»**创建**。 + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: reviews +spec: + hosts: + - reviews + http: + - route: + - destination: + host: reviews + subset: v1 + weight: 50 + - destination: + host: reviews + subset: v3 + weight: 50 +--- +``` + +**结果**：生æˆæµåˆ°è¯¥æœåŠ¡çš„æµé‡æ—¶ï¼ˆä¾‹å¦‚，刷新 Ingress Gateway URL），你å¯ä»¥åœ¨ Kiali æµé‡å›¾ä¸­çœ‹åˆ°æµåˆ° `reviews` æœåŠ¡çš„æµé‡è¢«å¹³å‡åˆ†é…到了 `v1` å’Œ `v3`。 + +### åŽç»­æ­¥éª¤ +[生æˆå’ŒæŸ¥çœ‹æµé‡](generate-and-view-traffic.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md new file mode 100644 index 00000000000..2290de911ad --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md @@ -0,0 +1,360 @@ +--- +title: 3. 使用 Istio Sidecar 添加部署和æœåŠ¡ +--- + +:::note 先决æ¡ä»¶ï¼š + +è¦ä¸ºå·¥ä½œè´Ÿè½½å¯ç”¨ Istio，你必须先在集群和命å空间中安装 Istio 应用。 + +::: + +在命å空间中å¯ç”¨ Istio åªä¼šä¸ºæ–°å·¥ä½œè´Ÿè½½å¯ç”¨è‡ªåŠ¨ sidecar 注入。è¦ä¸ºçŽ°æœ‰å·¥ä½œè´Ÿè½½å¯ç”¨ Envoy sidecar，你需è¦æ‰‹åŠ¨ä¸ºæ¯ä¸ªå·¥ä½œè´Ÿè½½å¯ç”¨å®ƒã€‚ + +è¦åœ¨å‘½å空间中的现有工作负载上注入 Istio sidecar： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 点击**工作负载**。 +1. 转到è¦æ³¨å…¥ Istio sidecar 的工作负载，然åŽå•å‡» **â‹® > é‡æ–°éƒ¨ç½²**。é‡æ–°éƒ¨ç½²å·¥ä½œè´Ÿè½½åŽï¼Œè¯¥å·¥ä½œè´Ÿè½½ä¼šè‡ªåŠ¨æ³¨å…¥ Envoy sidecar。 + +等待几分钟，然åŽå·¥ä½œè´Ÿè½½å°†å‡çº§å¹¶å…·æœ‰ Istio sidecar。å•å‡»å®ƒå¹¶è½¬åˆ°**容器**。你应该能看到该工作负载æ—边的 `istio-proxy`。这æ„味ç€ä¸ºå·¥ä½œè´Ÿè½½å¯ç”¨äº† Istio sidecar。Istio 正在为 Sidecar Envoy åšæ‰€æœ‰çš„接线工作。如果你现在在 yaml 中å¯ç”¨å®ƒä»¬ï¼ŒIstio å¯ä»¥è‡ªåŠ¨æ‰§è¡Œæ‰€æœ‰åŠŸèƒ½ã€‚ + +### 添加部署和æœåŠ¡ + +以下是在命å空间中添加新 **Deployment** 的几ç§æ–¹æ³•ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 点击 **Deployment**。 +1. 填写表å•ï¼Œæˆ–**以 YAML 文件编辑**。 +1. å•å‡»**创建**。 + +è¦å°† **Service** 添加到你的命å空间： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**æœåŠ¡å‘现 > æœåŠ¡**。 +1. å•å‡»**创建**。 +1. 选择所需的æœåŠ¡ç±»åž‹ã€‚ +1. 填写表å•ï¼Œæˆ–**以 YAML 文件编辑**。 +1. 点击**创建**。 + +你还å¯ä»¥ä½¿ç”¨ kubectl **shell** æ¥åˆ›å»º deployment å’Œ service： + +1. 如果你的文件存储在本地集群中，è¿è¡Œ `kubectl create -f .yaml`。 +1. 或è¿è¡Œ `cat<< EOF | kubectl apply -f -`，将文件内容粘贴到终端，然åŽè¿è¡Œ `EOF` æ¥å®Œæˆå‘½ä»¤ã€‚ + +### 部署和æœåŠ¡ç¤ºä¾‹ + +接下æ¥ï¼Œæˆ‘们为 Istio 文档中的 BookInfo 应用的示例部署和æœåŠ¡æ·»åŠ  Kubernetes 资æºï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œæ‰“å¼€ kubectl shell。 +1. è¿è¡Œ `cat<< EOF | kubectl apply -f -`。 +1. 将以下资æºå¤åˆ¶åˆ° shell 中。 +1. è¿è¡Œ `EOF`。 + +这将在 Istio 的示例 BookInfo 应用中设置以下示例资æºï¼š + +Details æœåŠ¡å’Œéƒ¨ç½²ï¼š + +- 一个 `details` Service。 +- 一个 `bookinfo-details` çš„ ServiceAccount。 +- 一个 `details-v1` Deployment。 + +Ratings æœåŠ¡å’Œéƒ¨ç½²ï¼š + +- 一个 `ratings` Service。 +- 一个 `bookinfo-ratings` çš„ ServiceAccount。 +- 一个 `ratings-v1` Deployment。 + +Reviews æœåŠ¡å’Œéƒ¨ç½²ï¼ˆä¸‰ä¸ªç‰ˆæœ¬ï¼‰ï¼š + +- 一个 `reviews` Service。 +- 一个 `bookinfo-reviews` çš„ ServiceAccount。 +- 一个 `reviews-v1` Deployment。 +- 一个 `reviews-v2` Deployment。 +- 一个 `reviews-v3` Deployment。 + +Productpage æœåŠ¡å’Œéƒ¨ç½²ï¼š + +这是应用的主页，å¯ä»¥é€šè¿‡ç½‘络æµè§ˆå™¨ä¸­æŸ¥çœ‹ã€‚将从该页é¢è°ƒç”¨å…¶ä»–æœåŠ¡ã€‚ + +- 一个 `productpage` service。 +- 一个 `bookinfo-productpage` çš„ ServiceAccount。 +- 一个 `productpage-v1` Deployment。 + +### èµ„æº YAML + +```yaml +# Copyright 2017 Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################################## +# Details service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: details + labels: + app: details + service: details +spec: + ports: + - port: 9080 + name: http + selector: + app: details +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-details +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: details-v1 + labels: + app: details + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: details + version: v1 + template: + metadata: + labels: + app: details + version: v1 + spec: + serviceAccountName: bookinfo-details + containers: + - name: details + image: docker.io/istio/examples-bookinfo-details-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Ratings service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: ratings + labels: + app: ratings + service: ratings +spec: + ports: + - port: 9080 + name: http + selector: + app: ratings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-ratings +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ratings-v1 + labels: + app: ratings + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: ratings + version: v1 + template: + metadata: + labels: + app: ratings + version: v1 + spec: + serviceAccountName: bookinfo-ratings + containers: + - name: ratings + image: docker.io/istio/examples-bookinfo-ratings-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Reviews service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: reviews + labels: + app: reviews + service: reviews +spec: + ports: + - port: 9080 + name: http + selector: + app: reviews +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-reviews +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v1 + labels: + app: reviews + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v1 + template: + metadata: + labels: + app: reviews + version: v1 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v2 + labels: + app: reviews + version: v2 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v2 + template: + metadata: + labels: + app: reviews + version: v2 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v2:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v3 + labels: + app: reviews + version: v3 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v3 + template: + metadata: + labels: + app: reviews + version: v3 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v3:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Productpage services +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: productpage + labels: + app: productpage + service: productpage +spec: + ports: + - port: 9080 + name: http + selector: + app: productpage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-productpage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: productpage-v1 + labels: + app: productpage + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: productpage + version: v1 + template: + metadata: + labels: + app: productpage + version: v1 + spec: + serviceAccountName: bookinfo-productpage + containers: + - name: productpage + image: docker.io/istio/examples-bookinfo-productpage-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +``` + +### åŽç»­æ­¥éª¤ +[设置 Istio Gateway](set-up-istio-gateway.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md new file mode 100644 index 00000000000..91578148d71 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md @@ -0,0 +1,39 @@ +--- +title: Pod 安全策略 +--- + +:::note + +本文介ç»çš„集群选项仅适用于 [Rancher 已在其中å¯åŠ¨ Kubernetes 的集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +::: + +ä½ å¯ä»¥åœ¨åˆ›å»ºé¡¹ç›®çš„时候设置 Pod 安全策略（PSPï¼‰ã€‚å¦‚æžœåœ¨åˆ›å»ºé¡¹ç›®æœŸé—´æ²¡æœ‰ä¸ºé¡¹ç›®åˆ†é… PSP，你也éšæ—¶å¯ä»¥å°† PSP 分é…给现有项目。 + +### 先决æ¡ä»¶ + +- 在 Rancher 中创建 Pod 安全策略。在将默认 PSP 分é…给现有项目之å‰ï¼Œä½ å¿…须有一个å¯åˆ†é…çš„ PSP。有关说明，请å‚阅[创建 Pod 安全策略](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 +- 将默认 Pod 安全策略分é…给项目所属的集群。如果 PSP 还没有应用到集群，你无法将 PSP 分é…给项目。有关详细信æ¯ï¼Œè¯·å‚阅[å°† pod 安全策略添加到集群](../../new-user-guides/manage-clusters/add-a-pod-security-policy.md)。 + +### 应用 Pod 安全策略 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°éœ€è¦ç§»åŠ¨å‘½å空间的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦æ·»åŠ  PSP 的项目。在该项目中选择 **â‹® > 编辑é…ç½®**。 +1. 从 **Pod 安全策略**下拉列表中，选择è¦åº”用于项目的 PSP。 + å°† PSP 分é…给项目将： + +- 覆盖集群的默认 PSP。 +- å°† PSP 应用于项目。 +- å°† PSP 应用到åŽç»­æ·»åŠ åˆ°é¡¹ç›®ä¸­çš„命å空间。 + +1. å•å‡»**ä¿å­˜**。 + +**结果**：已将 PSP 应用到项目以åŠé¡¹ç›®å†…的命å空间。 + +:::note + +å¯¹äºŽåœ¨åˆ†é… PSP 之å‰å·²ç»åœ¨é›†ç¾¤æˆ–项目中è¿è¡Œå·¥ä½œè´Ÿè½½ï¼ŒRancher ä¸ä¼šæ£€æŸ¥å®ƒä»¬æ˜¯å¦ç¬¦åˆ PSP。你需è¦å…‹éš†æˆ–å‡çº§å·¥ä½œè´Ÿè½½ä»¥æŸ¥çœ‹å®ƒä»¬æ˜¯å¦é€šè¿‡ PSP。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md new file mode 100644 index 00000000000..26e87e8f31a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md @@ -0,0 +1,67 @@ +--- +title: Rancher 项目中资æºé…é¢çš„å·¥ä½œåŽŸç† +--- + +Rancher 中的资æºé…é¢åŒ…å«ä¸Ž [Kubernetes 原生版本](https://kubernetes.io/docs/concepts/policy/resource-quotas/)相åŒçš„功能。Rancher 还扩展了资æºé…é¢çš„功能，让你将资æºé…é¢åº”用于项目。 + +在标准 Kubernetes deployment 中，资æºé…é¢ä¼šåº”用于å„个命å空间。但是，你ä¸èƒ½é€šè¿‡å•æ¬¡æ“作将é…é¢åº”用到多个命å空间，而必须多次应用资æºé…é¢ã€‚ + +在下图中，Kubernetes 管ç†å‘˜è¯•å›¾åœ¨æ²¡æœ‰ Rancher 的情况下强制执行资æºé…é¢ã€‚管ç†å‘˜æƒ³è¦ä½¿ç”¨ä¸€ä¸ªèµ„æºé…é¢æ¥ä¸ºé›†ç¾¤ä¸­çš„æ¯ä¸ªå‘½å空间é…置统一的 CPU 和内存é™åˆ¶ (`Namespace 1-4`)。但是，在 Kubernetes 的基础版本中，æ¯ä¸ªå‘½å空间都需è¦å•ç‹¬è®¾ç½®èµ„æºé…é¢ã€‚因此，管ç†å‘˜å¿…须创建四个é…置相åŒè§„格的ä¸åŒèµ„æºé…é¢ï¼ˆ`Resource Quota 1-4`）并å•ç‹¬åº”用这些é…é¢ã€‚ + +Kubernetes 基础版本：æ¯ä¸ªå‘½å空间都需è¦ç‹¬ç«‹è®¾ç½®èµ„æºé…é¢ + +![原生 Kubernetes 资æºé…é¢å®žçŽ°](/img/kubernetes-resource-quota.svg) + +和原生 Kubernetes 相比，Rancher 的资æºé…é¢æœ‰ä¸åŒã€‚在 Rancher 中，你å¯ä»¥æŠŠèµ„æºé…é¢åº”用到项目层级，进而让项目的资æºé…é¢æ²¿ç”¨åˆ°é¡¹ç›®å†…çš„æ¯ä¸€ä¸ªå‘½åç©ºé—´ï¼Œç„¶åŽ Kubernetes 会使用原生的资æºé…é¢æ¥å¼ºåˆ¶æ‰§è¡Œä½ è®¾ç½®çš„é™åˆ¶ã€‚如果è¦æ›´æ”¹ç‰¹å®šå‘½å空间的é…é¢ï¼Œä½ ä¹Ÿå¯ä»¥è¦†ç›–设置。 + +项目é…é¢åŒ…括你在创建或编辑集群时设置的两个é™åˆ¶ï¼š + + +- **项目é™åˆ¶**： + + é…置了项目中所有命å空间共享的æ¯ä¸ªæŒ‡å®šèµ„æºçš„总é™åˆ¶ã€‚ + +- **命å空间默认é™åˆ¶**： + + é…置了æ¯ä¸ªå‘½å空间对æ¯ä¸ªæŒ‡å®šèµ„æºçš„默认é…é¢ã€‚ + 如果项目中的命å空间é…置没有被覆盖，那么此é™åˆ¶ä¼šè‡ªåŠ¨ç»‘定到命å空间并强制执行。 + + +在下图中，Rancher 管ç†å‘˜æƒ³ä½¿ç”¨èµ„æºé…é¢æ¥ä¸ºé¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间（`命å空间 1-4`）设置相åŒçš„ CPU 和内存é™åˆ¶ã€‚在 Rancher 中，管ç†å‘˜å¯ä»¥ä¸ºé¡¹ç›®è®¾ç½®èµ„æºé…é¢ï¼ˆ`项目资æºé…é¢`），而ä¸éœ€è¦ä¸ºå‘½å空间å•ç‹¬è¿›è¡Œè®¾ç½®ã€‚æ­¤é…é¢åŒ…括整个项目（`项目é™åˆ¶`）和å•ä¸ªå‘½å空间（`命å空间默认é™åˆ¶`）的资æºé™åˆ¶ã€‚然åŽï¼ŒRancher 会将`命å空间默认é™åˆ¶`çš„é…é¢æ²¿ç”¨åˆ°æ¯ä¸ªå‘½å空间（`命å空间资æºé…é¢`）。 + +Rancher：资æºé…é¢æ²¿ç”¨åˆ°æ¯ä¸ªå‘½å空间 + +![Rancher 资æºé…é¢å®žçŽ°](/img/rancher-resource-quota.png) + +以下介ç»åœ¨ Rancher UI **_中_** 创建的命å空间的更细微的功能。如果你删除了项目级别的资æºé…é¢ï¼Œæ— è®ºå‘½å空间层级是å¦æœ‰è‡ªå®šä¹‰çš„资æºé…é¢ï¼Œé¡¹ç›®å†…的所有命å空间也会移除这个资æºé…é¢ã€‚在项目层级修改已有的命å空间默认资æºé…é¢ï¼Œä¸ä¼šå½±å“命å空间内的资æºé…é¢ï¼Œä¿®æ”¹åŽçš„项目层级资æºé…é¢åªä¼šå¯¹ä»¥åŽæ–°å»ºçš„命å空间生效。è¦ä¿®æ”¹å¤šä¸ªçŽ°æœ‰å‘½å空间的默认é™åˆ¶ï¼Œä½ å¯ä»¥åœ¨é¡¹ç›®å±‚级删除该é™åˆ¶ï¼Œç„¶åŽå†ä½¿ç”¨æ–°çš„默认值é‡æ–°åˆ›å»ºé…é¢ã€‚è¿™ç§æ–¹å¼ä¼šå°†æ–°çš„默认值应用于项目中的所有现有命å空间。 + +在项目中创建命å空间之å‰ï¼ŒRancher 会使用默认é™åˆ¶å’Œè¦†ç›–é™åˆ¶æ¥å¯¹æ¯”项目内的å¯ç”¨èµ„æºå’Œè¯·æ±‚资æºã€‚ +如果请求的资æºè¶…过了项目中这些资æºçš„剩余容é‡ï¼ŒRancher 将为命å空间分é…该资æºçš„剩余容é‡ã€‚ + +但是，在 Rancher çš„ UI **_外_** 创建的命å空间的处ç†æ–¹æ³•åˆ™ä¸ä¸€æ ·ã€‚对于通过 `kubectl` 创建的命å空间，如果请求的资æºé‡å¤šä½™é¡¹ç›®å†…çš„ä½™é‡ï¼ŒRancher 会分é…一个数值为 **0** 的资æºé…é¢ã€‚ + +è¦ä½¿ç”¨ `kubectl` 在现有项目中创建命å空间，请使用 `field.cattle.io/projectId` 注释。è¦è¦†ç›–默认的请求é…é¢é™åˆ¶ï¼Œè¯·ä½¿ç”¨ `field.cattle.io/resourceQuota` 注释。 + +请注æ„，Rancher åªä¼šè¦†ç›–项目é…é¢ä¸Šå®šä¹‰çš„资æºé™åˆ¶ã€‚ + +``` +apiVersion: v1 +kind: Namespace +metadata: + annotations: + field.cattle.io/projectId: [your-cluster-ID]:[your-project-ID] + field.cattle.io/resourceQuota: '{"limit":{"limitsCpu":"100m", "configMaps": "50"}}' + name: my-ns +``` +在此示例中，如果项目é…é¢åœ¨å…¶èµ„æºåˆ—表中ä¸åŒ…å« configMaps，那么 Rancher 将忽略此覆盖中的 `configMaps`。 + +对于项目中未定义的资æºï¼Œå»ºè®®ä½ åœ¨å‘½å空间中创建专用的 `ResourceQuota` 对象æ¥é…置其它自定义é™åˆ¶ã€‚ +资æºé…é¢æ˜¯åŽŸç”Ÿ Kubernetes 对象，如果命å空间属于具有é…é¢çš„项目，Rancher 将忽略用户定义的é…é¢ï¼Œä»Žè€Œç»™äºˆç”¨æˆ·æ›´å¤šçš„控制æƒã€‚ + +下表对比了 Rancher å’Œ Kubernetes 资æºé…é¢çš„主è¦åŒºåˆ«ï¼š + +| Rancher 资æºé…é¢ | Kubernetes 资æºé…é¢ | +| ---------------------------------------------------------- | -------------------------------------------------------- | +| 应用于项目和命å空间。 | 仅应用于命å空间。 | +| 为项目中的所有命å空间创建资æºæ± ã€‚ | å°†é™æ€èµ„æºé™åˆ¶åº”用到å•ç‹¬çš„命å空间。 | +| 通过沿用的模å¼ï¼Œå°†èµ„æºé…é¢åº”用于å„个命å空间。 | 仅应用于指定的命å空间。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md new file mode 100644 index 00000000000..f29c3004b85 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md @@ -0,0 +1,34 @@ +--- +title: 覆盖命å空间的默认é™åˆ¶ +--- + +**命å空间默认é™åˆ¶**会在创建时从项目沿用到æ¯ä¸ªå‘½å空间。但在æŸäº›æƒ…况下，你å¯èƒ½éœ€è¦å¢žåŠ æˆ–å‡å°‘特定命å空间的é…é¢ã€‚在这ç§æƒ…况下，你å¯ä»¥é€šè¿‡ç¼–辑命å空间æ¥è¦†ç›–默认é™åˆ¶ã€‚ + +在下图中，Rancher 管ç†å‘˜çš„项目有一个已生效的资æºé…é¢ã€‚但是，管ç†å‘˜æƒ³è¦è¦†ç›– `Namespace 3` 的命å空间é™åˆ¶ï¼Œä»¥ä¾¿è®©è¯¥å‘½å空间使用更多资æºã€‚因此，管ç†å‘˜[æ高了 `Namespace 3` 的命å空间é™åˆ¶](../../../new-user-guides/manage-clusters/projects-and-namespaces.md)，以便命å空间å¯ä»¥è®¿é—®æ›´å¤šèµ„æºã€‚ + +命å空间默认é™åˆ¶è¦†ç›– + +![命å空间默认é™åˆ¶è¦†ç›–](/img/rancher-resource-quota-override.svg) + +有关详细信æ¯ï¼Œè¯·å‚阅[如何编辑命å空间资æºé…é¢](../../../new-user-guides/manage-clusters/projects-and-namespaces.md)。 + +### 编辑命å空间资æºé…é¢ + +如果你已为项目é…置了资æºé…é¢ï¼Œä½ å¯ä»¥è¦†ç›–命å空间默认é™åˆ¶ï¼Œä»Žè€Œä¸ºç‰¹å®šå‘½å空间æ供对更多（或更少）项目资æºçš„访问æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦ç¼–辑命å空间资æºé…é¢çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦ä¸ºå…¶ç¼–辑资æºé…é¢çš„命å空间。å•å‡» **â‹® > 编辑é…ç½®**。 +1. 编辑资æºé™åˆ¶ã€‚这些é™åˆ¶å†³å®šäº†å‘½å空间å¯ç”¨çš„资æºã€‚必须在项目é™åˆ¶èŒƒå›´å†…é…置这些é…é¢é™åˆ¶ã€‚ + + 有关æ¯ä¸ª**资æºç±»åž‹**的详细信æ¯ï¼Œè¯·å‚阅[类型å‚考](resource-quota-types.md)。 + + :::note + + - 如果没有为项目é…置资æºé…é¢ï¼Œè¿™äº›é€‰é¡¹å°†ä¸å¯ç”¨ã€‚ + - 如果你输入的é™åˆ¶è¶…过了é…置的项目é™åˆ¶ï¼Œä½ å°†æ— æ³•ä¿å­˜ä¿®æ”¹ã€‚ + + ::: + +**结果**：覆盖设置已ç»åº”用到命å空间的资æºé…é¢ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md new file mode 100644 index 00000000000..b0873befd7a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md @@ -0,0 +1,27 @@ +--- +title: 资æºé…é¢ç±»åž‹å‚考 +--- + +创建资æºé…é¢ç›¸å½“于é…置项目å¯ç”¨çš„资æºæ± ã€‚ä½ å¯ä»¥ä¸ºä»¥ä¸‹èµ„æºç±»åž‹è®¾ç½®èµ„æºé…é¢ï¼š + +| 资æºç±»åž‹ | æè¿° | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU é™åˆ¶\* | 分é…给项目/命å空间的最大 CPU é‡ï¼ˆä»¥[毫核](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)为å•ä½ï¼‰1 | +| CPU 预留\* | 预留给项目/命åç©ºé—´çš„æœ€å° CPU é‡ï¼ˆä»¥æ¯«æ ¸ä¸ºå•ä½ï¼‰1 | +| 内存é™åˆ¶\* | 分é…给项目/命å空间的最大内存é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰1 | +| 内存预留\* | 预留给项目/命å空间的最å°å†…å­˜é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰1 | +| 存储预留 | 预留给项目/命å空间的最å°å­˜å‚¨é‡ï¼ˆä»¥åƒå…†å­—节为å•ä½ï¼‰ | +| æœåŠ¡è´Ÿè½½å‡è¡¡å™¨ | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„è´Ÿè½½å‡è¡¡å™¨æœåŠ¡çš„æœ€å¤§æ•°é‡ | +| æœåŠ¡èŠ‚ç‚¹ç«¯å£ | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„节点端å£æœåŠ¡çš„æœ€å¤§æ•°é‡ | +| Pod | å¯ä»¥åœ¨é¡¹ç›®/命å空间中以éžç»ˆç«¯çŠ¶æ€å­˜åœ¨çš„ pod 的最大数é‡ï¼ˆå³ `.status.phase in (Failed, Succeeded)` 等于 true çš„ pod） | +| Services | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„最大 service æ•°é‡ | +| ConfigMap | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„ ConfigMap çš„æœ€å¤§æ•°é‡ | +| æŒä¹…å·å£°æ˜Ž | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„æŒä¹…å·å£°æ˜Žçš„æœ€å¤§æ•°é‡ | +| ReplicationController | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„最大 ReplicationController æ•°é‡ | +| 密文 | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„æœ€å¤§å¯†æ–‡æ•°é‡ | + +:::note ***** + +在设置资æºé…é¢æ—¶ï¼Œå¦‚果你在项目或命å空间上设置了任何与 CPU 或内存相关的内容（å³é™åˆ¶æˆ–预留），所有容器都需è¦åœ¨åˆ›å»ºæœŸé—´è®¾ç½®å„自的 CPU 或内存字段。你å¯ä»¥åŒæ—¶è®¾ç½®å®¹å™¨çš„默认资æºé™åˆ¶ï¼Œä»¥é¿å…为æ¯ä¸ªå·¥ä½œè´Ÿè½½æ˜¾å¼è®¾ç½®è¿™äº›é™åˆ¶ã€‚详情请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits)。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md new file mode 100644 index 00000000000..e56f617e7a0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md @@ -0,0 +1,40 @@ +--- +title: 设置容器默认资æºé™åˆ¶ +--- + +在设置资æºé…é¢æ—¶ï¼Œå¦‚果你在项目或命å空间上设置了任何与 CPU 或内存相关的内容（å³é™åˆ¶æˆ–预留），所有容器都需è¦åœ¨åˆ›å»ºæœŸé—´è®¾ç½®å„自的 CPU 或内存字段。详情请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits)。 + +为了é¿å…在创建工作负载期间对æ¯ä¸ªå®¹å™¨è®¾ç½®è¿™äº›é™åˆ¶ï¼Œå¯ä»¥åœ¨å‘½å空间上指定一个默认的容器资æºé™åˆ¶ã€‚ + +### 编辑容器默认资æºé™åˆ¶ + +ä½ å¯ä»¥åœ¨ä»¥ä¸‹æƒ…况下编辑容器的默认资æºé™åˆ¶ï¼š + +- 你在项目上设置了 CPU 或内存资æºé…é¢ï¼ŒçŽ°åœ¨éœ€è¦ä¸ºå®¹å™¨è®¾ç½®ç›¸åº”的默认值。 +- 你需è¦ç¼–辑容器的默认资æºé™åˆ¶ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦ç¼–辑默认资æºé™åˆ¶çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦ç¼–辑容器默认资æºé™åˆ¶çš„项目。在该项目中选择 **â‹® > 编辑é…ç½®**。 +1. 展开**容器默认资æºé™åˆ¶**并编辑对应的值。 + +### 沿用资æºé™åˆ¶ + +在项目级别设置默认容器资æºé™åˆ¶åŽï¼Œé¡¹ç›®ä¸­æ‰€æœ‰æ–°å»ºçš„命å空间都会沿用这个资æºé™åˆ¶å‚数。新设置的é™åˆ¶ä¸ä¼šå½±å“项目中现有的命å空间。你需è¦ä¸ºé¡¹ç›®ä¸­çš„现有命å空间手动设置默认容器资æºé™åˆ¶ï¼Œä»¥ä¾¿åˆ›å»ºå®¹å™¨æ—¶èƒ½åº”用该é™åˆ¶ã€‚ + +ä½ å¯ä»¥ä¸ºé¡¹ç›®è®¾ç½®å®¹å™¨çš„默认资æºé™åˆ¶å¹¶å¯åŠ¨ä»»ä½•å•†åº—应用。 + +在命å空间上é…置容器默认资æºé™åˆ¶åŽï¼Œåœ¨è¯¥å‘½å空间中创建的任何容器都会沿用该默认值。你å¯ä»¥åœ¨å·¥ä½œè´Ÿè½½åˆ›å»ºæœŸé—´è¦†ç›–这些é™åˆ¶/预留。 + +### 容器资æºé…é¢ç±»åž‹ + +å¯ä»¥é…置以下资æºé™åˆ¶ï¼š + +| 资æºç±»åž‹ | æè¿° | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU é™åˆ¶ | 分é…给容器的最大 CPU é‡ï¼ˆä»¥[毫核](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)为å•ä½ï¼‰ã€‚ | +| CPU 预留 | ä¿ç•™ç»™å®¹å™¨çš„æœ€å° CPU é‡ï¼ˆä»¥æ¯«æ ¸ä¸ºå•ä½ï¼‰ã€‚ | +| 内存é™åˆ¶ | 分é…给容器的最大内存é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰ã€‚ | +| 内存预留 | ä¿ç•™ç»™å®¹å™¨çš„最å°å†…å­˜é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰ã€‚ | +| NVIDIA GPU é™åˆ¶/预留 | 分é…给容器的 GPU æ•°é‡ã€‚GPU çš„é™åˆ¶å’Œé¢„留始终相åŒã€‚ | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md new file mode 100644 index 00000000000..e5a2deccf45 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md @@ -0,0 +1,148 @@ +--- +title: æŒä¹…化 Grafana ä»ªè¡¨æ¿ +--- + +è¦åœ¨é‡å¯ Grafana 实例åŽä¿å­˜ Grafana 仪表æ¿ï¼Œè¯·å°†ä»ªè¡¨æ¿çš„é…ç½® JSON 添加到 ConfigMap 中。ConfigMap 还支æŒä½¿ç”¨åŸºäºŽ GitOps 或 CD 的方法æ¥éƒ¨ç½²ä»ªè¡¨æ¿ï¼Œä»Žè€Œè®©ä½ å¯¹ä»ªè¡¨æ¿è¿›è¡Œç‰ˆæœ¬æŽ§åˆ¶ã€‚ + +- [创建æŒä¹…化 Grafana 仪表æ¿](#创建æŒä¹…化-grafana-仪表æ¿) +- [已知问题](#已知问题) + +## 创建æŒä¹…化 Grafana ä»ªè¡¨æ¿ + + + + +:::note 先决æ¡ä»¶ï¼š + +- 已安装 Monitoring 应用。 +- è¦åˆ›å»ºæŒä¹…化仪表æ¿ï¼Œä½ å¿…é¡»åœ¨åŒ…å« Grafana 仪表æ¿çš„项目或命å空间中至少具有**ç®¡ç† ConfigMap** çš„ Rancher RBAC æƒé™ã€‚这与 Monitoring Chart 公开的 `monitoring-dashboard-edit` 或 `monitoring-dashboard-admin` Kubernetes 原生 RBAC 角色对应。 +- è¦æŸ¥çœ‹æŒ‡å‘外部监控 UI（包括 Grafana 仪表æ¿ï¼‰çš„链接，你至少需è¦ä¸€ä¸ª [project-member 角色](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#具有-rancher-æƒé™çš„用户)。 + +::: + +### 1. 获å–è¦æŒä¹…化的仪表æ¿çš„ JSON 模型 + +è¦åˆ›å»ºæŒä¹…化仪表æ¿ï¼Œä½ éœ€è¦èŽ·å–è¦æŒä¹…化的仪表æ¿çš„ JSON 模型。你å¯ä»¥ä½¿ç”¨é¢„制仪表æ¿æˆ–自行构建仪表æ¿ã€‚ + +è¦ä½¿ç”¨é¢„制仪表æ¿ï¼Œè¯·è½¬åˆ° [https://grafana.com/grafana/dashboards](https://grafana.com/grafana/dashboards)，打开详细信æ¯é¡µé¢ï¼Œç„¶åŽå•å‡» **Download JSON** 按钮æ¥èŽ·å–下一步所需的 JSON 模型。 + +è¦ä½¿ç”¨ä½ è‡ªå·±çš„仪表æ¿ï¼š + +1. 点击链接打开 Grafana。在集群详细信æ¯é¡µé¢ä¸Šï¼Œå•å‡» **Monitoring**。 +1. 登录到 Grafana。请注æ„，Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin/prom-operator`。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + :::note + + 无论è°æ‹¥æœ‰å¯†ç ï¼Œä½ éƒ½éœ€è¦åœ¨éƒ¨ç½²äº† Rancher Monitoring 的项目中至少具有管ç†æœåŠ¡æˆ–查看监控的æƒé™æ‰èƒ½è®¿é—® Grafana 实例。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + ::: + +1. 使用 Grafana UI 创建仪表æ¿ã€‚完æˆåŽï¼Œå•å‡»é¡¶éƒ¨å¯¼èˆªèœå•ä¸­çš„齿轮图标转到仪表æ¿è®¾ç½®é¡µé¢ã€‚在左侧导航èœå•ä¸­ï¼Œå•å‡» **JSON Model**。 +1. å¤åˆ¶å‡ºçŽ°çš„ JSON æ•°æ®ç»“构。 + +### 2. 使用 Grafana JSON 模型创建 ConfigMap + +åœ¨åŒ…å« Grafana 仪表æ¿çš„命å空间中创建一个 ConfigMap（默认为 cattle-dashboards ）。 + +ConfigMap 与以下内容类似： + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards # 如果ä¸ä½¿ç”¨é»˜è®¤å‘½å空间，则修改此值 +data: + .json: |- + +``` + +默认情况下，Grafana é…置为监控 `cattle-dashboards` 命å空间中带有 `grafana_dashboard` 标签的所有 ConfigMap。 + +è¦è®© Grafana 监控所有命å空间中的 ConfigMap，请å‚阅[本节](#为-grafana-仪表æ¿-configmap-é…置命å空间)。 + +è¦åœ¨ Rancher UI 中创建 ConfigMap： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**æ›´å¤šèµ„æº > 核心 > é…置映射**。 +1. å•å‡»**创建**。 +1. 设置与上例类似的键值对。输入 `.json` 的值时，点击**从文件读å–**并上传 JSON æ•°æ®æ¨¡åž‹ã€‚ +1. å•å‡»**创建**。 + +**结果**：创建 ConfigMap åŽï¼Œå³ä½¿ Grafana pod é‡å¯äº†ï¼ŒConfigMap 也能显示在 Grafana UI 上并æŒä¹…化。 + +无法在 Grafana UI 中删除或编辑使用 ConfigMap æŒä¹…化了的仪表æ¿ã€‚ + +如果你在 Grafana UI 中删除仪表æ¿ï¼Œä½ å°†çœ‹åˆ° "Dashboard cannot be deleted because it was provisioned" 的错误消æ¯ã€‚如需删除仪表æ¿ï¼Œä½ éœ€è¦åˆ é™¤ ConfigMap。 + +### 为 Grafana ä»ªè¡¨æ¿ ConfigMap é…置命å空间 + +è¦è®© Grafana 监控所有命å空间中的 ConfigMap，请在 `rancher-monitoring` Helm chart 中指定以下值： + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +请注æ„，Monitoring Chart 用于添加 Grafana 仪表æ¿çš„ RBAC 角色仅能让用户将仪表æ¿æ·»åŠ åˆ°å®šä¹‰åœ¨ `grafana.dashboards.namespace` 中的命å空间，默认为 `cattle-dashboards`。 + + + + +:::note 先决æ¡ä»¶ï¼š + +- 已安装 Monitoring 应用。 +- 你必须具有 cluster-admin ClusterRole æƒé™ã€‚ + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦åœ¨å…¶ä¸­é…ç½® Grafana 命å空间的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Grafana**。 +1. 登录到 Grafana。请注æ„，Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin/prom-operator`。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + :::note + + 无论è°æ‹¥æœ‰å¯†ç ï¼Œéƒ½éœ€è¦ Rancher 的集群管ç†å‘˜æƒé™æ‰èƒ½è®¿é—® Grafana 实例。 + + ::: + +1. 转到è¦è¿›è¡ŒæŒä¹…化的仪表æ¿ã€‚在顶部导航èœå•ä¸­ï¼Œé€šè¿‡å•å‡»é½¿è½®å›¾æ ‡è½¬åˆ°ä»ªè¡¨æ¿è®¾ç½®ã€‚ +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡» **JSON Model**。 +1. å¤åˆ¶å‡ºçŽ°çš„ JSON æ•°æ®ç»“构。 +1. 在 `cattle-dashboards` 命å空间中创建一个 ConfigMap。ConfigMap 需è¦æœ‰ `grafana_dashboard: "1"` 标签。将 JSON 粘贴到 ConfigMap 中，格å¼å¦‚下例所示： + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards + data: + .json: |- + + ``` + +**结果**：创建 ConfigMap åŽï¼Œå³ä½¿ Grafana pod é‡å¯äº†ï¼ŒConfigMap 也能显示在 Grafana UI 上并æŒä¹…化。 + +无法在 Grafana UI 中删除使用 ConfigMap æŒä¹…化了的仪表æ¿ã€‚如果你在 Grafana UI 中删除仪表æ¿ï¼Œä½ å°†çœ‹åˆ° "Dashboard cannot be deleted because it was provisioned" 的错误消æ¯ã€‚如需删除仪表æ¿ï¼Œä½ éœ€è¦åˆ é™¤ ConfigMap。 + +为防止在å¸è½½ Monitoring v2 时删除æŒä¹…化的仪表æ¿ï¼Œè¯·å°†ä»¥ä¸‹æ³¨é‡Šæ·»åŠ åˆ° `cattle-dashboards` 命å空间： + +``` +helm.sh/resource-policy: "keep" +``` + + + + +## 已知问题 + +如果你的 Monitoring V2 版本是 v9.4.203 或更低版本，å¸è½½ Monitoring chart å°†åŒæ—¶åˆ é™¤ `cattle-dashboards` 命å空间，所有æŒä¹…化的仪表æ¿å°†è¢«åˆ é™¤ï¼ˆé™¤éžå‘½å空间带有注释 `helm.sh/resource-policy: "keep"`）。 + +Rancher 2.5.8 å‘布的新 Monitoring Chart 中默认添加了该注解，但使用早期 Rancher 版本的用户ä»éœ€æ‰‹åŠ¨åº”用该注释。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md new file mode 100644 index 00000000000..5fa126a9089 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md @@ -0,0 +1,39 @@ +--- +title: 自定义 Grafana ä»ªè¡¨æ¿ +--- + +在本文中，你将学习通过自定义 Grafana 仪表æ¿æ¥æ˜¾ç¤ºç‰¹å®šå®¹å™¨çš„指标。 + +### 先决æ¡ä»¶ + +在自定义 Grafana 仪表æ¿ä¹‹å‰ï¼Œä½ å¿…须先安装 `rancher-monitoring` 应用。 + +è¦æŸ¥çœ‹æŒ‡å‘外部监控 UI（包括 Grafana 仪表æ¿ï¼‰çš„链接，你至少需è¦ä¸€ä¸ª [project-member 角色](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#具有-rancher-æƒé™çš„用户)。 + +### 登录 Grafana + +1. 在 Rancher UI 中，转到è¦è‡ªå®šä¹‰çš„仪表æ¿çš„集群。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Grafana**。Grafana 仪表æ¿å°†åœ¨æ–°é€‰é¡¹å¡ä¸­æ‰“开。 +1. 转到左下角的登录图标，然åŽå•å‡» **Sign In**。 +1. 登录到 Grafana。Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin/prom-operator`（无论è°æ‹¥æœ‰å¯†ç ï¼Œéƒ½éœ€è¦ Rancher 的集群管ç†å‘˜æƒé™æ‰èƒ½è®¿é—® Grafana 实例）。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + +### 获å–æ”¯æŒ Grafana é¢æ¿çš„ PromQL 查询 + +对于任何é¢æ¿ï¼Œä½ å¯ä»¥å•å‡»æ ‡é¢˜å¹¶å•å‡» **Explore** 以获å–支æŒå›¾å½¢çš„ PromQL 查询。 + +例如，如果è¦èŽ·å– Alertmanager 容器的 CPU 使用率，点击 **CPU Utilization > Inspect**。 + +**Data** 选项å¡å°†åŸºç¡€æ•°æ®æ˜¾ç¤ºä¸ºæ—¶é—´åºåˆ—，第一列是时间，第二列是 PromQL 查询结果。å¤åˆ¶ PromQL 查询。 + +``` +(1 - (avg(irate({__name__=~"node_cpu_seconds_total|windows_cpu_time_total",mode="idle"}[5m])))) * 100 +``` + +然åŽï¼Œä½ å¯ä»¥åœ¨ Grafana é¢æ¿ä¸­ä¿®æ”¹æŸ¥è¯¢ï¼Œæˆ–使用该查询创建新的 Grafana é¢æ¿ã€‚ + +å‚考： + +- [编辑é¢æ¿çš„ Grafana 文档](https://grafana.com/docs/grafana/latest/panels-visualizations/configure-panel-options/#edit-a-panel) +- [å‘仪表æ¿æ·»åŠ é¢æ¿çš„ Grafana 文档](https://grafana.com/docs/grafana/latest/panels-visualizations/panel-editor-overview) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md new file mode 100644 index 00000000000..a51a45041ec --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md @@ -0,0 +1,19 @@ +--- +title: è°ƒè¯•é«˜å†…å­˜ç”¨é‡ +--- + +Prometheus 中的æ¯ä¸ªæ—¶é—´åºåˆ—都由其[指标å称](https://prometheus.io/docs/practices/naming/#metric-names)和称为[标签](https://prometheus.io/docs/practices/naming/#labels)çš„å¯é€‰é”®å€¼å¯¹å”¯ä¸€æ ‡è¯†ã€‚ + +标签å…许过滤和èšåˆæ—¶é—´åºåˆ—æ•°æ®ï¼Œä½†å®ƒä»¬ä¹Ÿä½¿ Prometheus 收集的数æ®é‡æˆå€å¢žåŠ ã€‚ + +æ¯ä¸ªæ—¶é—´åºåˆ—都有一组定义的标签，Prometheus 为所有唯一的标签组åˆç”Ÿæˆä¸€ä¸ªæ–°çš„时间åºåˆ—。如果一个指标附加了两个标签，则会为该指标生æˆä¸¤ä¸ªæ—¶é—´åºåˆ—。更改任何标签值，包括添加或删除标签，都会创建一个新的时间åºåˆ—。 + +Prometheus ç»è¿‡äº†ä¼˜åŒ–，å¯ä»¥å­˜å‚¨åŸºäºŽç´¢å¼•çš„åºåˆ—æ•°æ®ã€‚它是为相对一致的时间åºåˆ—æ•°é‡å’Œç›¸å¯¹å¤§é‡çš„样本而设计的，这些样本需è¦éšæ—¶é—´ä»Ž exporter 处收集。 + +但是，Prometheus 没有就快速å˜åŒ–的时间åºåˆ—æ•°é‡è¿›è¡Œå¯¹åº”的优化。因此，如果你在创建和销æ¯äº†å¤§é‡èµ„æºçš„集群（尤其是多租户集群）上安装 Monitoring，å¯èƒ½ä¼šå‡ºçŽ°å†…存使用é‡æ¿€å¢žçš„情况。 + +### å‡å°‘内存激增 + +为了å‡å°‘内存消耗，Prometheus å¯ä»¥é€šè¿‡æŠ“å–更少的指标或在时间åºåˆ—上添加更少的标签，从而存储更少的时间åºåˆ—。è¦æŸ¥çœ‹ä½¿ç”¨å†…存最多的åºåˆ—，你å¯ä»¥æŸ¥çœ‹ Prometheus UI 中的 TSDB（时åºæ•°æ®åº“）状æ€é¡µé¢ã€‚ + +åˆ†å¸ƒå¼ Prometheus 解决方案（例如 [Thanos](https://thanos.io/) å’Œ [Cortex](https://cortexmetrics.io/)）使用了一个替代架构，该架构部署多个å°åž‹ Prometheus 实例。如果使用 Thanos，æ¯ä¸ª Prometheus 的指标都èšåˆåˆ°é€šç”¨çš„ Thanos 部署中，然åŽå†å°†è¿™äº›æŒ‡æ ‡å¯¼å‡ºåˆ° S3 之类的æŒä¹…存储。这ç§æž¶æž„更加å¥åº·ï¼Œèƒ½é¿å…ç»™å•ä¸ª Prometheus 实例带æ¥è¿‡å¤šæ—¶é—´åºåˆ—，åŒæ—¶è¿˜èƒ½åœ¨å…¨å±€çº§åˆ«æŸ¥è¯¢æŒ‡æ ‡ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md new file mode 100644 index 00000000000..cfb2ebdf338 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md @@ -0,0 +1,75 @@ +--- +title: å¯ç”¨ Monitoring +--- + +[管ç†å‘˜](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)或[集群所有者](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥éƒ¨ç½² Prometheus，从而监控 Kubernetes 集群。 + +本文介ç»å¦‚何使用新的 monitoring 应用在集群内å¯ç”¨ç›‘控和告警。 + +ä¸è®ºæ˜¯å¦ä½¿ç”¨ SSL，你都å¯ä»¥å¯ç”¨ monitoring。 + +## è¦æ±‚ + +- 在æ¯ä¸ªèŠ‚点上å…è®¸ç«¯å£ 9796 上的æµé‡ã€‚Prometheus 将从这些端å£æŠ“å–指标。 + - 如果 [PushProx](../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#pushprox) 被ç¦ç”¨ï¼ˆ`ingressNginx.enabled` 设置为 `false`），或者你已ç»å‡çº§äº†å®‰è£…了 Monitoring V1 çš„ Rancher 版本，你å¯èƒ½è¿˜éœ€è¦ä¸ºæ¯ä¸ªèŠ‚点å…è®¸ç«¯å£ 10254 上的æµé‡ã€‚ +- ç¡®ä¿ä½ çš„集群满足资æºè¦æ±‚。集群应至少有 1950Mi å¯ç”¨å†…å­˜ã€2700m CPU å’Œ 50Gi 存储。有关资æºé™åˆ¶å’Œè¯·æ±‚的详细信æ¯ï¼Œè¯·å‚阅[é…置资æºé™åˆ¶å’Œè¯·æ±‚](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#é…置资æºé™åˆ¶å’Œè¯·æ±‚)。 +- 在使用 RancherOS 或 Flatcar Linux 节点的 RKE 集群上安装 Monitoring 时，请将 etcd 节点è¯ä¹¦ç›®å½•æ›´æ”¹ä¸º `/opt/rke/etc/kubernetes/ssl`。 +- 如果集群是使用 RKE CLI é…置的，而且地å€è®¾ç½®ä¸ºä¸»æœºå而ä¸æ˜¯ IP 地å€ï¼Œè¯·åœ¨å®‰è£…çš„ Values é…置步骤中将 `rkeEtcd.clients.useLocalhost` 设置为 `true`。例如： + +```yaml +rkeEtcd: + clients: + useLocalhost: true +``` + +:::note + +如果è¦è®¾ç½® Alertmanagerã€Grafana 或 Ingress，必须通过 Helm chart 部署上的设置æ¥å®Œæˆã€‚在部署之外创建 Ingress å¯èƒ½ä¼šäº§ç”Ÿé—®é¢˜ã€‚ + +::: + +## 设置资æºé™åˆ¶å’Œè¯·æ±‚ + +安装 `rancher-monitoring` æ—¶å¯ä»¥é…置资æºè¯·æ±‚å’Œé™åˆ¶ã€‚è¦ä»Ž Rancher UI é…ç½® Prometheus 资æºï¼Œè¯·å•å‡»å·¦ä¸Šè§’çš„ **Apps > Monitoring**。 + +有关默认é™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#é…置资æºé™åˆ¶å’Œè¯·æ±‚)。 + +## 安装 Monitoring 应用 + +### 在ä¸ä½¿ç”¨ SSL 的情况下å¯ç”¨ Monitoring + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**集群工具**（左下角）。 +1. å•å‡» Monitoring çš„**安装**。 +1. å¯é€‰ï¼šåœ¨ Values 步骤中为 Alertingã€Prometheus å’Œ Grafana 自定义请求ã€é™åˆ¶ç­‰ã€‚如需帮助，请å‚阅[é…ç½®å‚考](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md)。 + +**结果**：Monitoring 应用已部署到 `cattle-monitoring-system` 命å空间中。 + +### 在使用 SSL 的情况下å¯ç”¨ Monitoring + +1. 按照[此页é¢](../../new-user-guides/kubernetes-resources-setup/secrets.md)上的步骤创建密文，以便将 SSL 用于告警。 +- 密文应该创建到 `cattle-monitoring-system` 命å空间中。如果它ä¸å­˜åœ¨ï¼Œè¯·å…ˆåˆ›å»ºå®ƒã€‚ +- å°† `ca`ã€`cert` å’Œ `key` 文件添加到密文中。 +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯ç”¨ Monitoring 以与 SSL 一起使用的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **Monitoring**。 +1. æ ¹æ®ä½ æ˜¯å¦å·²å®‰è£… Monitoring，å•å‡»**安装**或**æ›´æ–°**。 +1. 选中**在安装å‰è‡ªå®šä¹‰ Helm 选项**，然åŽå•å‡»**下一步**。 +1. å•å‡» **Alerting**。 +1. 在**补充密文**字段中，添加之å‰åˆ›å»ºçš„密文。 + +**结果**：Monitoring 应用已部署到 `cattle-monitoring-system` 命å空间中。 + +[创建接收器](../../../reference-guides/monitoring-v2-configuration/receivers.md#在-rancher-ui-中创建接收器)时，å¯ç”¨ SSL 的接收器（例如电å­é‚®ä»¶æˆ– webhook）将具有 **SSL**ï¼Œå…¶ä¸­åŒ…å« **CA 文件路径**ã€**è¯ä¹¦æ–‡ä»¶è·¯å¾„**å’Œ**密钥文件路径**字段。使用 `ca`ã€`cert` å’Œ `key` 的路径填写这些字段。路径的格å¼ä¸º `/etc/alertmanager/secrets/name-of-file-in-secret`。 + +例如，如果你使用以下键值对创建了一个密文： + +```yaml +ca.crt=`base64-content` +cert.pem=`base64-content` +key.pfx=`base64-content` +``` + +则**è¯ä¹¦æ–‡ä»¶è·¯å¾„**需è¦è®¾ä¸º `/etc/alertmanager/secrets/cert.pem`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md new file mode 100644 index 00000000000..44d69d05f9e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md @@ -0,0 +1,7 @@ +--- +title: 自定义 Grafana ä»ªè¡¨æ¿ +--- + +无论是用于 rancher-monitoring 还是 Prometheus Federator，Grafana 仪表æ¿çš„定制方å¼éƒ½æ˜¯ç›¸åŒçš„。 + +有关说明，请å‚阅[此页é¢](../customize-grafana-dashboard.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md new file mode 100644 index 00000000000..c6bac839971 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md @@ -0,0 +1,86 @@ +--- +title: å¯ç”¨ Prometheus Federator +--- + +## è¦æ±‚ + +默认情况下，Prometheus Federator å·²é…置并旨在与 [rancher-monitoring](../../../../pages-for-subheaders/monitoring-and-alerting.md) 一起部署。rancher-monitoring åŒæ—¶éƒ¨ç½²äº† Prometheus Operator å’Œ Cluster Prometheus，æ¯ä¸ªé¡¹ç›®ç›‘控堆栈（Project Monitoring Stack）默认会è”åˆå‘½å空间范围的指标。 + +有关安装 rancher-monitoring 的说明，请å‚阅[此页é¢](../enable-monitoring.md)。 + +默认é…置与你的 rancher-monitoring 堆栈是兼容的。但是，为了æ高集群中 Prometheus Federator 的安全性和å¯ç”¨æ€§ï¼Œæˆ‘们建议对 rancher-monitoring 进行以下é¢å¤–çš„é…置： + +- [ç¡®ä¿ cattle-monitoring-system 命å空间ä½äºŽ System 项目中](#ç¡®ä¿-cattle-monitoring-system-命å空间ä½äºŽ-system-项目中或者ä½äºŽä¸€ä¸ªé”定并能访问集群中其他项目的项目中) +- [å°† rancher-monitoring é…置为仅监视 Helm Chart 创建的资æº](#å°†-rancher-monitoring-é…置为仅监视-helm-chart-创建的资æº) +- [æ高 Cluster Prometheus çš„ CPU/内存é™åˆ¶](#æ高-cluster-prometheus-çš„-cpu内存é™åˆ¶) + +### ç¡®ä¿ cattle-monitoring-system 命å空间ä½äºŽ System 项目中（或者ä½äºŽä¸€ä¸ªé”定并能访问集群中其他项目的项目中） + +![选择项目/命å空间](/img/install-in-system-project.png) + +Prometheus Operator 的安全模型有一定的è¦æ±‚，å³å¸Œæœ›éƒ¨ç½²å®ƒçš„命å空间（例如，`cattle-monitoring-system`）对除集群管ç†å‘˜ä¹‹å¤–的任何用户都åªæœ‰æœ‰é™çš„访问æƒé™ï¼Œä»Žè€Œé¿å…通过 Pod 内执行（例如正在执行的 Helm æ“作的 Job）æ¥æå‡æƒé™ã€‚此外，如果将 Prometheus Federator 和所有 Project Prometheus 堆栈都部署到 System 项目中，å³ä½¿ç½‘络策略是通过项目网络隔离定义的，æ¯ä¸ª Project Prometheus 都ä¾ç„¶èƒ½å¤Ÿåœ¨æ‰€æœ‰é¡¹ç›®ä¸­æŠ“å–工作负载。它还为项目所有者ã€é¡¹ç›®æˆå‘˜å’Œå…¶ä»–用户æ供有é™çš„访问æƒé™ï¼Œä»Žè€Œç¡®ä¿è¿™äº›ç”¨æˆ·æ— æ³•è®¿é—®ä»–们ä¸åº”访问的数æ®ï¼ˆä¾‹å¦‚，在 pod 中执行，在项目外部抓å–命å空间数æ®ç­‰ï¼‰ã€‚ + +1. 打开 `System` 项目以检查你的命å空间： + + 在 Rancher UI 中å•å‡» **Cluster > Projects/Namespaces**。这将显示 `System` 项目中的所有命å空间： + + ![选择项目/命å空间](/img/cattle-monitoring-system.png) + +1. 如果你在 `cattle-monitoring-system` 命å空间中已ç»å®‰è£…了一个 Monitoring V2，但该命å空间ä¸åœ¨ `System` 项目中，你å¯ä»¥å°† `cattle- monitoring-system` 命å空间移动到 `System` 项目或å¦ä¸€ä¸ªè®¿é—®å—é™çš„项目中。为此，你有以下两ç§æ–¹æ³•ï¼š + + - 将命å空间拖放到 `System` 项目。 + - 选择命å空间å³ä¾§çš„ **â‹®**，点击 **Move**，然åŽä»Ž **Target Project** 下拉列表中选择 `System`： + + ![移至新项目](/img/move-to-new-project.png) + +### å°† rancher-monitoring é…置为仅监视 Helm Chart åˆ›å»ºçš„èµ„æº + +æ¯ä¸ªé¡¹ç›®ç›‘控堆栈都会监视其他命å空间并收集其他自定义工作负载指标或仪表æ¿ã€‚因此，我们建议在所有选择器上é…ç½®ä»¥ä¸‹è®¾ç½®ï¼Œä»¥ç¡®ä¿ Cluster Prometheus 堆栈仅监控由 Helm Chart 创建的资æºï¼š + +``` +matchLabels: + release: "rancher-monitoring" +``` + +建议为以下选择器字段赋予此值： +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +å¯ç”¨æ­¤è®¾ç½®åŽï¼Œä½ å§‹ç»ˆå¯ä»¥é€šè¿‡å‘它们添加 `release: "rancher-monitoring"` 标签æ¥åˆ›å»ºç”± Cluster Prometheus 抓å–çš„ ServiceMonitor 或 PodMonitor。在这ç§æƒ…况下，å³ä½¿è¿™äº› ServiceMonitor 或 PodMonitor 所在的命å空间ä¸æ˜¯ System 命å空间，项目监控堆栈也会默认自动忽略它们。 + +:::note + +如果你ä¸å¸Œæœ›ç”¨æˆ·èƒ½å¤Ÿåœ¨ Project 命å空间中创建èšåˆåˆ° Cluster Prometheus 中的 ServiceMonitor å’Œ PodMonitor，你å¯ä»¥å¦å¤–å°† Chart 上的 namespaceSelectors 设置为仅目标 System 命åç©ºé—´ï¼ˆå¿…é¡»åŒ…å« `cattle-monitoring-system` å’Œ `cattle-dashboards`，默认情况下资æºé€šè¿‡ rancher-monitoring 部署到该命å空间中。你还需è¦ç›‘控 `default` 命åç©ºé—´ä»¥èŽ·å– apiserver 指标，或创建自定义 ServiceMonitor 以抓å–ä½äºŽ `default` 命å空间中的 Service çš„ apiserver 指标），从而é™åˆ¶ä½ çš„ Cluster Prometheus 获å–其他 Prometheus Operator CR。在这ç§æƒ…况下，建议设置 `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。这样，你å¯ä»¥å®šä¹‰èƒ½ä»Ž System 命åç©ºé—´ä¸­ç›‘è§†éž System 命å空间的 ServiceMonitor。 + +::: + +### æ高 Cluster Prometheus çš„ CPU/内存é™åˆ¶ + +æ ¹æ®é›†ç¾¤çš„设置，我们一般建议为 Cluster Prometheus é…置大é‡çš„专用内存，以é¿å…因内存ä¸è¶³çš„错误（OOMKilled）而é‡å¯ã€‚通常情况下，集群中创建的改动项（churn）会导致大é‡é«˜åŸºæ•°æŒ‡æ ‡åœ¨ä¸€ä¸ªæ—¶é—´å—内生æˆå¹¶è¢« Prometheus 引入，然åŽå¯¼è‡´è¿™äº›é”™è¯¯ã€‚这也是为什么默认的 Rancher Monitoring 堆栈希望能分é…到大约 4GB çš„ RAM 以在正常大å°çš„集群中è¿è¡Œçš„原因之一。但是，如果你引入å‘åŒä¸€ä¸ª Cluster Prometheus å‘é€ `/federate` 请求的项目监控堆栈，并且项目监控堆栈ä¾èµ–于 Cluster Prometheus çš„å¯åŠ¨çŠ¶æ€æ¥åœ¨å…¶å‘½å空间上è”åˆç³»ç»Ÿæ•°æ®ï¼Œé‚£ä¹ˆä½ æ›´åŠ éœ€è¦ä¸º Cluster Prometheus 分é…足够的 CPU/内存，以防止集群中的所有 Prometheus 项目出现数æ®é—´éš™çš„中断。 + +:::note + +我们没有 Cluster Prometheus 内存é…置的具体建议，因为这完全å–决于用户的设置（å³é‡åˆ°é«˜æ”¹åŠ¨çŽ‡çš„å¯èƒ½æ€§ä»¥åŠå¯èƒ½åŒæ—¶ç”Ÿæˆçš„指标的规模）。ä¸åŒçš„设置通常有ä¸åŒçš„推èå‚数。 + +::: + +## 安装 Prometheus Federator åº”ç”¨ç¨‹åº + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å®‰è£… Prometheus Federator 的集群，然åŽå•å‡» **Explore**。 +1. 点击**应用 > Charts**。 +1. å•å‡» **Prometheus Federator** Chart。 +1. å•å‡»**安装**。 +1. 在**元数æ®**页é¢ï¼Œç‚¹å‡»**下一步**。 +1. 在**命å空间** > **项目 Release 命å空间项目 ID** 字段中，`System 项目`是默认值，但你å¯ä»¥ä½¿ç”¨å…·æœ‰ç±»ä¼¼[有é™è®¿é—®æƒé™](#ç¡®ä¿-cattle-monitoring-system-命å空间ä½äºŽ-system-项目中或者ä½äºŽä¸€ä¸ªé”定并能访问集群中其他项目的项目中)çš„å¦ä¸€ä¸ªé¡¹ç›®è¦†ç›–它。你å¯ä»¥åœ¨ local 上游集群中è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ‰¾åˆ°é¡¹ç›® ID： + +```plain +kubectl get projects -A -o custom-columns="NAMESPACE":.metadata.namespace,"ID":.metadata.name,"NAME":.spec.displayName +``` + +1. å•å‡»**安装**。 + +**结果**：Prometheus Federator 应用程åºå·²éƒ¨ç½²åœ¨ `cattle-monitoring-system` 命å空间中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md new file mode 100644 index 00000000000..9dba3d6d401 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md @@ -0,0 +1,17 @@ +--- +title: 安装 Project Monitor +--- + +在è¦å¯ç”¨é¡¹ç›®ç›‘控的å„个项目中安装 **Project Monitor**。 + +1. 点击 **☰ > 集群管ç†**。 + +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦å¯ç”¨ç›‘控的集群，然åŽå•å‡» **Explore**。 + +1. å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„**监控 > Project Monitor**。然åŽï¼Œç‚¹å‡»å³ä¸Šè§’çš„**创建**。 + + ![Project Monitor](/img/project-monitors.png) + +1. 从下拉èœå•ä¸­é€‰æ‹©ä½ çš„项目，然åŽå†æ¬¡å•å‡»**创建**。 + + ![创建 Project Monitor](/img/create-project-monitors.png) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md new file mode 100644 index 00000000000..10aaf5d95b7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md @@ -0,0 +1,13 @@ +--- +title: 为工作负载设置 Prometheus Federator +--- + +### 显示工作负载的 CPU 和内存指标 + +使用 Prometheus Federator 显示 CPU 和内存指标的方å¼ä¸Žä½¿ç”¨ rancher-monitoring 相åŒã€‚有关说明，请å‚阅[此处](../set-up-monitoring-for-workloads.md#显示工作负载的-cpu-和内存指标)。 + +### 设置 CPU 和内存之外的指标 + +使用 Prometheus Federator 设置 CPU 和内存之外的指标与使用 rancher-monitoring çš„æ–¹å¼ç›¸åŒã€‚有关说明，请å‚阅[此处](../set-up-monitoring-for-workloads.md#设置-cpu-和内存之外的指标)。 + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md new file mode 100644 index 00000000000..c6b04a736bb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md @@ -0,0 +1,13 @@ +--- +title: å¸è½½ Prometheus Federator +--- + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `cattle-monitoring-system` 命å空间并选中 `rancher-monitoring-crd` å’Œ `rancher-monitoring`。 +1. å•å‡»**删除**。 +1. 确认**删除**。 + +**结果**：已å¸è½½ `prometheus-federator`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md new file mode 100644 index 00000000000..7ab5dd36f79 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md @@ -0,0 +1,27 @@ +--- +title: 为工作负载设置 Monitoring +--- + +如果你åªéœ€è¦å·¥ä½œè´Ÿè½½çš„ CPU 和内存时间åºåˆ—，则ä¸éœ€è¦éƒ¨ç½² ServiceMonitor 或 PodMonitor，因为 Monitoring 应用默认会收集资æºä½¿ç”¨æƒ…况的指标数æ®ã€‚ + +设置工作负载监控的步骤å–决于你是å¦éœ€è¦åŸºæœ¬æŒ‡æ ‡ï¼ˆä¾‹å¦‚工作负载的 CPU 和内存），或者是å¦éœ€è¦ä»Žå·¥ä½œè´Ÿè½½ä¸­æŠ“å–自定义指标。 + +如果你åªéœ€è¦å·¥ä½œè´Ÿè½½çš„ CPU 和内存时间åºåˆ—，则ä¸éœ€è¦éƒ¨ç½² ServiceMonitor 或 PodMonitor，因为 Monitoring 应用默认会收集资æºä½¿ç”¨æƒ…况的指标数æ®ã€‚资æºä½¿ç”¨çš„时间åºåˆ—æ•°æ®åœ¨ Prometheus 的本地时间åºåˆ—æ•°æ®åº“中。 + +Grafana 显示èšåˆæ•°æ®ï¼Œä½ ä¹Ÿå¯ä»¥ä½¿ç”¨ PromQL 查询æ¥æŸ¥çœ‹å•ä¸ªå·¥ä½œè´Ÿè½½çš„æ•°æ®ã€‚进行 PromQL 查询åŽï¼Œä½ å¯ä»¥åœ¨ Prometheus UI 中å•ç‹¬æ‰§è¡ŒæŸ¥è¯¢å¹¶æŸ¥çœ‹å¯è§†åŒ–的时间åºåˆ—，你也å¯ä»¥ä½¿ç”¨æŸ¥è¯¢æ¥è‡ªå®šä¹‰æ˜¾ç¤ºå·¥ä½œè´Ÿè½½æŒ‡æ ‡çš„ Grafana 仪表æ¿ã€‚有关工作负载指标的 PromQL 查询示例，请å‚阅[本节](../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md#工作负载指标)。 + +è¦ä¸ºä½ çš„工作负载设置自定义指标，你需è¦è®¾ç½®ä¸€ä¸ª Exporter 并创建一个新的 ServiceMonitor 自定义资æºï¼Œä»Žè€Œå°† Prometheus é…置为从 Exporter 中抓å–指标。 + +### 显示工作负载的 CPU 和内存指标 + +默认情况下，Monitoring åº”ç”¨ä¼šæŠ“å– CPU 和内存指标。 + +è¦èŽ·å–特定工作负载的细粒度信æ¯ï¼Œä½ å¯ä»¥è‡ªå®šä¹‰ Grafana 仪表æ¿æ¥æ˜¾ç¤ºè¯¥å·¥ä½œè´Ÿè½½çš„指标。 + +### 设置 CPU 和内存之外的指标 + +对于自定义指标，你需è¦ä½¿ç”¨ Prometheus 支æŒçš„æ ¼å¼æ¥å…¬å¼€åº”用上的指标。 + +我们建议你创建一个新的 ServiceMonitor 自定义资æºã€‚创建此资æºæ—¶ï¼ŒPrometheus 自定义资æºå°†è‡ªåŠ¨æ›´æ–°ï¼Œä»¥ä¾¿å°†æ–°çš„自定义指标端点包å«åœ¨æŠ“å–é…ç½®ä¸­ã€‚ç„¶åŽ Prometheus 会开始从端点抓å–指标。 + +你还å¯ä»¥åˆ›å»º PodMonitor æ¥å…¬å¼€è‡ªå®šä¹‰æŒ‡æ ‡ç«¯ç‚¹ï¼Œä½† ServiceMonitor 更适åˆå¤§å¤šæ•°ç”¨ä¾‹ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md new file mode 100644 index 00000000000..06845311909 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md @@ -0,0 +1,19 @@ +--- +title: å¸è½½ Monitoring +--- + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `cattle-monitoring-system` 命å空间并选中 `rancher-monitoring-crd` å’Œ `rancher-monitoring`。 +1. å•å‡»**删除**。 +1. 确认**删除**。 + +**结果**：已å¸è½½ `rancher-monitoring`。 + +:::note æŒä¹…化 Grafana 仪表æ¿ï¼š + +如果你的 Monitoring V2 版本是 v9.4.203 或更低版本，å¸è½½ Monitoring chart å°†åŒæ—¶åˆ é™¤ cattle-dashboards 命å空间，所有æŒä¹…化的仪表æ¿å°†è¢«åˆ é™¤ï¼ˆé™¤éžå‘½å空间带有注释 `helm.sh/resource-policy: "keep"`）。Monitoring V2 v14.5.100+ 会默认添加此注释。但如果你的集群上安装了旧版本的 Monitoring Chart，你å¯ä»¥åœ¨å¸è½½å®ƒä¹‹å‰æ‰‹åŠ¨å°†æ³¨é‡Šåº”用到 cattle-dashboards 命å空间。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md new file mode 100644 index 00000000000..77851a3accb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md @@ -0,0 +1,43 @@ +--- +title: Alertmanager é…ç½® +--- + +通常情况下，你ä¸éœ€è¦ç›´æŽ¥ç¼–辑 Alertmanager 自定义资æºã€‚对于大多数用例，åªéœ€è¦ç¼–辑接收器和路由å³å¯é…置通知。 + +当路由和接收器更新时，Monitoring 应用将自动更新 Alertmanager 自定义资æºæ¥ä¸Žè¿™äº›æ›´æ”¹ä¿æŒä¸€è‡´ã€‚ + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-alertmanager-的工作原ç†)。 + +::: + +## 关于 Alertmanager è‡ªå®šä¹‰èµ„æº + +默认情况下，Rancher Monitoring å°†å•ä¸ª Alertmanager 部署到使用默认 Alertmanager Config Secret 的集群上。 + +如果你想使用 Rancher UI 表å•ä¸­æœªå…¬å¼€çš„高级选项（例如创建超过两层深的路由树结构），你å¯èƒ½éœ€è¦ç¼–辑 Alertmanager 自定义资æºã€‚ + +你也å¯ä»¥åœ¨é›†ç¾¤ä¸­åˆ›å»ºå¤šä¸ª Alertmanager æ¥å®žçŽ°å‘½å空间范围的监控。在这ç§æƒ…况下，你应该使用相åŒçš„底层 Alertmanager Config Secret æ¥ç®¡ç† Alertmanager 自定义资æºã€‚ + +### 深度嵌套的路由 + +虽然 Rancher UI 仅支æŒä¸¤å±‚深度的路由树，但你å¯ä»¥é€šè¿‡ç¼–辑 Alertmanager YAML æ¥é…置更深的嵌套路由结构。 + +### 多个 Alertmanager 副本 + +作为 Chart 部署选项的一部分，你å¯ä»¥é€‰æ‹©å¢žåŠ éƒ¨ç½²åˆ°é›†ç¾¤ä¸Šçš„ Alertmanager 副本的数é‡ã€‚这些副本使用相åŒçš„底层 Alertmanager Config Secret 进行管ç†ã€‚ + +æ­¤ Secret å¯ä»¥æŒ‰ç…§ä½ çš„需求éšæ—¶æ›´æ–°æˆ–修改： + +- 添加新的通知程åºæˆ–接收器 +- 更改应该å‘é€ç»™æŒ‡å®šé€šçŸ¥ç¨‹åºæˆ–接收器的告警 +- 更改å‘出的告警组 + +默认情况下，你å¯ä»¥é€‰æ‹©æ供现有的 Alertmanager Config Secretï¼ˆå³ `cattle-monitoring-system` 命å空间中的任何 Secret），或å…许 Rancher Monitoring 将默认的 Alertmanager Config Secret 部署到你的集群上。 + +默认情况下，在å‡çº§æˆ–å¸è½½ `rancher-monitoring` Chart 时，Rancher 创建的 Alertmanager Config Secret ä¸ä¼šè¢«ä¿®æ”¹æˆ–删除。这个é™åˆ¶å¯ä»¥é˜²æ­¢ç”¨æˆ·åœ¨ Chart 上执行æ“作时丢失或覆盖他们的告警é…置。 + +有关å¯ä»¥åœ¨ Alertmanager Config Secret 中指定的字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ [Prometheus Alertmanager 文档](https://prometheus.io/docs/alerting/latest/alertmanager/)。 + +ä½ å¯ä»¥åœ¨[此处](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file)找到 Alertmanager é…置文件的完整规范åŠå…¶å†…容。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md new file mode 100644 index 00000000000..b83ad90b748 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md @@ -0,0 +1,19 @@ +--- +title: Prometheus é…ç½® +--- + +通常情况下，你ä¸éœ€è¦ç›´æŽ¥ç¼–辑 Prometheus 自定义资æºï¼Œå› ä¸º Monitoring åº”ç”¨ä¼šæ ¹æ® ServiceMonitor å’Œ PodMonitor 的更改自动更新资æºã€‚ + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + +## 关于 Prometheus è‡ªå®šä¹‰èµ„æº + +Prometheus CR 定义了所需的 Prometheus deployment。Prometheus Operator 会观察 Prometheus CR。当 CR å‘生å˜åŒ–时，Prometheus Operator 会创建 `prometheus-rancher-monitoring-prometheus`，å³æ ¹æ® CR é…置的 Prometheus deployment。 + +Prometheus CR 指定了详细信æ¯ï¼Œä¾‹å¦‚规则以åŠè¿žæŽ¥åˆ° Prometheus çš„ Alertmanager。Rancher 会为你构建这个 CR。 + +Monitoring V2 仅支æŒæ¯ä¸ªé›†ç¾¤ä¸€ä¸ª Prometheus。如果你想将监控é™åˆ¶åˆ°æŒ‡å®šå‘½å空间，你需è¦ç¼–辑 Prometheus CR。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md new file mode 100644 index 00000000000..fc12de7e488 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md @@ -0,0 +1,80 @@ +--- +title: é…ç½® PrometheusRule +--- + +PrometheusRule 定义了一组 Prometheus 告警和/或记录规则。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + +### 在 Rancher UI 中创建 PrometheusRule + +:::note 先决æ¡ä»¶ï¼š + +已安装 Monitoring 应用。 + +::: + +è¦åœ¨ Rancher UI 中创建规则组： + +1. 转到è¦åˆ›å»ºè§„则组的集群。å•å‡»**监控 > 高级选项**，然åŽå•å‡» **PrometheusRules**。 +1. å•å‡»**创建**。 +1. 输入**组å称**。 +1. é…置规则。在 Rancher çš„ UI 中，规则组需è¦åŒ…å«å‘Šè­¦è§„则或记录规则，但ä¸èƒ½åŒæ—¶åŒ…å«ä¸¤è€…。如需获å–填写表å•çš„帮助，请å‚阅下方的é…置选项。 +1. å•å‡»**创建**。 + +**结果**：告警å¯ä»¥å‘接收器å‘é€é€šçŸ¥ã€‚ + +### 关于 PrometheusRule è‡ªå®šä¹‰èµ„æº + +当你定义规则时（在 PrometheusRule 资æºçš„ RuleGroup 中声明），[规则本身的规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule)会包å«æ ‡ç­¾ï¼Œç„¶åŽ Alertmanager 会使用这些标签æ¥ç¡®å®šæŽ¥æ”¶æ­¤å‘Šè­¦çš„路由。例如，标签为 `team: front-end` 的告警将​​å‘é€åˆ°ä¸Žè¯¥æ ‡ç­¾åŒ¹é…的所有路由。 + +Prometheus 规则文件ä¿å­˜åœ¨ PrometheusRule 自定义资æºä¸­ã€‚PrometheusRule 支æŒå®šä¹‰ä¸€ä¸ªæˆ–多个 RuleGroup。æ¯ä¸ª RuleGroup 由一组 Rule 对象组æˆï¼Œæ¯ä¸ª Rule 对象å‡èƒ½è¡¨ç¤ºå‘Šè­¦æˆ–记录规则，并具有以下字段： + +- 新告警或记录的å称 +- 新告警或记录的 PromQL è¡¨è¾¾å¼ +- 用于标记告警或记录的标签（例如集群å称或严é‡æ€§ï¼‰ +- 对需è¦åœ¨å‘Šè­¦é€šçŸ¥ä¸Šæ˜¾ç¤ºçš„其他é‡è¦ä¿¡æ¯è¿›è¡Œç¼–ç çš„注释（例如摘è¦ã€æè¿°ã€æ¶ˆæ¯ã€Runbook URL 等）。记录规则ä¸éœ€è¦æ­¤å­—段。 + +有关å¯ä»¥æŒ‡å®šçš„字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ [Prometheus Operator 规范。](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#prometheusrulespec) + +ä½ å¯ä»¥ä½¿ç”¨ Prometheus 对象中的标签选择器字段 `ruleSelector` æ¥å®šä¹‰è¦æŒ‚载到 Prometheus 的规则文件。 + +如需查看示例，请å‚阅 Prometheus 文档中的[记录规则](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)å’Œ[告警规则](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)。 + +## é…ç½® + +### 规则组 + +| 字段 | æè¿° | +|-------|----------------| +| 组å称 | 组的å称。在规则文件中必须是唯一的。 | +| 覆盖组间隔 | 组中规则的评估时间间隔（å•ä½ï¼šç§’）。 | + + +### 告警规则 + +[告警规则](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)å¯ä»¥è®©ä½ æ ¹æ® PromQL（Prometheus 查询语言）表达å¼æ¥å®šä¹‰å‘Šè­¦æ¡ä»¶ï¼Œå¹¶å°†è§¦å‘告警的通知å‘é€åˆ°å¤–部æœåŠ¡ã€‚ + +| 字段 | æè¿° | +|-------|----------------| +| å‘Šè­¦å称 | 告警的å称。必须是有效的标签值。 | +| 告警触å‘等待时间 | 时长，以秒为å•ä½ã€‚当告警触å‘时间到达该指定时长时，则视为触å‘。当告警未触å‘足够长的时间，则视为待处ç†ã€‚ | +| PromQL è¡¨è¾¾å¼ | è¦è¯„ä¼°çš„ PromQL 表达å¼ã€‚Prometheus 将在æ¯ä¸ªè¯„估周期评估此 PromQL 表达å¼çš„当å‰å€¼ï¼Œå¹¶ä¸”所有生æˆçš„时间åºåˆ—都将æˆä¸ºå¾…处ç†/触å‘告警。有关详细信æ¯ï¼Œè¯·å‚阅 [Prometheus 文档](https://prometheus.io/docs/prometheus/latest/querying/basics/)或我们的 [PromQL 表达å¼ç¤ºä¾‹](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md)。 | +| Labels | 为æ¯ä¸ªå‘Šè­¦æ·»åŠ æˆ–覆盖的标签。 | +| 严é‡ç¨‹åº¦ | å¯ç”¨åŽï¼Œæ ‡ç­¾â€‹â€‹ä¼šé™„加到告警或记录中，这些标签通过严é‡ç¨‹åº¦æ¥æ ‡è¯†å‘Šè­¦/记录。 | +| 严é‡ç¨‹åº¦ Label 值 | Critical，warning 或 none | +| 注释 | 注释是一组信æ¯æ ‡ç­¾ï¼Œå¯ç”¨äºŽå­˜å‚¨æ›´é•¿çš„附加信æ¯ï¼Œä¾‹å¦‚å‘Šè­¦æ述或 Runbook 链接。[Runbook](https://en.wikipedia.org/wiki/Runbook) 是一组有关如何处ç†å‘Šè­¦çš„文档。注释值å¯ä»¥æ˜¯[模æ¿åŒ–](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#templating)的。 | + +### 记录规则 + +[记录规则](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules)å…许你预先计算常用或计算é‡å¤§çš„ PromQL（Prometheus 查询语言）表达å¼ï¼Œå¹¶å°†å…¶ç»“æžœä¿å­˜ä¸ºä¸€ç»„新的时间åºåˆ—。 + +| 字段 | æè¿° | +|-------|----------------| +| 时间åºåˆ—å称 | è¦è¾“出的时间åºåˆ—çš„å称。必须是有效的指标å称。 | +| PromQL è¡¨è¾¾å¼ | è¦è¯„ä¼°çš„ PromQL 表达å¼ã€‚Prometheus 将在æ¯ä¸ªè¯„估周期评估此 PromQL 表达å¼çš„当å‰å€¼ï¼Œå¹¶ä¸”将结果记录为一组新的时间åºåˆ—，其指标å称由“记录â€æŒ‡å®šã€‚有关表达å¼çš„更多信æ¯ï¼Œè¯·å‚阅 [Prometheus 文档](https://prometheus.io/docs/prometheus/latest/querying/basics/)或我们的 [PromQL 表达å¼ç¤ºä¾‹](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md)。 | +| Labels | 在存储结果之å‰è¦æ·»åŠ æˆ–覆盖的标签。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md new file mode 100644 index 00000000000..7e43c7234b2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md @@ -0,0 +1,107 @@ +--- +title: 使用 firewalld æ‰“å¼€ç«¯å£ +--- + +> 我们建议ç¦ç”¨ firewalld。如果你使用的是 Kubernetes 1.19 或更高版本，则必须关闭 firewalld。 + +æŸäº› [æºè‡ª RHEL](https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux#Rebuilds) çš„ Linux å‘行版（包括 Oracle Linux）的默认防ç«å¢™è§„则å¯èƒ½ä¼šé˜»æ­¢ä¸Ž Helm 的通信。 + +例如，AWS 中的一个 Oracle Linux é•œåƒå…·æœ‰ REJECT 规则，这些规则会阻止 Helm 与 Tiller 通信： + +``` +Chain INPUT (policy ACCEPT) +target prot opt source destination +ACCEPT all -- anywhere anywhere state RELATED,ESTABLISHED +ACCEPT icmp -- anywhere anywhere +ACCEPT all -- anywhere anywhere +ACCEPT tcp -- anywhere anywhere state NEW tcp dpt:ssh +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain FORWARD (policy ACCEPT) +target prot opt source destination +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain OUTPUT (policy ACCEPT) +target prot opt source destination +``` + +ä½ å¯è¿è¡Œä»¥ä¸‹å‘½ä»¤æ£€æŸ¥é»˜è®¤é˜²ç«å¢™è§„则： + +``` +sudo iptables --list +``` + +下文介ç»å¦‚何使用 `firewalld`，将[防ç«å¢™ç«¯å£è§„则](../../pages-for-subheaders/installation-requirements.md#端å£è¦æ±‚)应用到高å¯ç”¨ Rancher Server 集群中的节点。 + +## 先决æ¡ä»¶ + +安装 v7.x 或更高版本的 `firewalld`： + +``` +yum install firewalld +systemctl start firewalld +systemctl enable firewalld +``` + +## 应用防ç«å¢™ç«¯å£è§„则 + +在 Rancher 高å¯ç”¨å®‰è£…中，Rancher Server 设置在三个节点上，三个节点å‡å…·æœ‰ Kubernetes 的所有三个角色（etcdã€controlplane å’Œ worker）。如果你的 Rancher Server 节点åŒæ—¶å…·æœ‰è¿™ä¸‰ä¸ªè§’色，请在æ¯ä¸ªèŠ‚点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` +如果你的 Rancher Server 节点é…置了å•ç‹¬çš„角色，请根æ®èŠ‚点角色è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +# 在 etcd 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp + +# 在 controlplane 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp + +# 在 worker 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` + +在节点上è¿è¡Œ `firewall-cmd` 命令åŽï¼Œä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å¯ç”¨é˜²ç«å¢™è§„则： + +``` +firewall-cmd --reload +``` + +**结果**：防ç«å¢™å·²æ›´æ–°ï¼Œå› æ­¤ Helm å¯ä»¥ä¸Ž Rancher Server 节点通信了。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md new file mode 100644 index 00000000000..6cadea62ced --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md @@ -0,0 +1,39 @@ +--- +title: 为大型安装进行 etcd 调优 +--- + +当你è¿è¡Œå…·æœ‰ 15 个或更多集群的大型 Rancher 安装时，我们建议你扩大 etcd 的默认 keyspace（默认为 2GB）。你最大å¯ä»¥å°†å®ƒè®¾ç½®ä¸º 8GB。此外，请确ä¿ä¸»æœºæœ‰è¶³å¤Ÿçš„ RAM æ¥ä¿å­˜æ•´ä¸ªæ•°æ®é›†ã€‚如果需è¦å¢žåŠ è¿™ä¸ªå€¼ï¼Œä½ è¿˜éœ€è¦åŒæ­¥å¢žåŠ ä¸»æœºçš„大å°ã€‚如果你预计在垃圾回收间隔期间 Pod çš„å˜åŒ–率很高，你也å¯ä»¥åœ¨è¾ƒå°çš„安装中调整 Keyspace 大å°ã€‚ + +Kubernetes æ¯éš”äº”åˆ†é’Ÿä¼šè‡ªåŠ¨æ¸…ç† etcd æ•°æ®é›†ã€‚在æŸäº›æƒ…况下（例如å‘生部署抖动），在垃圾回收å‘生并进行清ç†ä¹‹å‰ä¼šæœ‰å¤§é‡äº‹ä»¶å†™å…¥ etcd 并删除，从而导致 Keyspace 填满。如果你在 etcd 日志或 Kubernetes API Server 日志中看到 `mvcc: database space exceeded` 错误，你å¯ä»¥åœ¨ etcd æœåŠ¡å™¨ä¸Šè®¾ç½® [quota-backend-bytes](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) æ¥å¢žåŠ  Keyspace 的大å°ã€‚ + +### 示例：此 RKE cluster.yml 文件的代ç ç‰‡æ®µå°† Keyspace 的大å°å¢žåŠ åˆ° 5GB + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + quota-backend-bytes: 5368709120 +``` + +## 扩展 etcd ç£ç›˜æ€§èƒ½ + +ä½ å¯ä»¥å‚è§ [etcd 文档](https://etcd.io/docs/v3.4.0/tuning/#disk)中的建议，了解如何调整主机上的ç£ç›˜ä¼˜å…ˆçº§ã€‚ + +此外，为了å‡å°‘ etcd ç£ç›˜ä¸Šçš„ IO 争用，你å¯ä»¥ä¸º data å’Œ wal 目录使用专用设备。etcd 最佳实践ä¸å»ºè®®é…ç½® Mirror RAID（因为 etcd 在集群中的节点之间å¤åˆ¶æ•°æ®ï¼‰ã€‚ä½ å¯ä»¥ä½¿ç”¨ striping RAID é…ç½®æ¥å¢žåŠ å¯ç”¨çš„ IOPS。 + +è¦åœ¨ RKE 集群中实现此解决方案，你需è¦åœ¨åº•å±‚主机上为 `/var/lib/etcd/data` å’Œ `/var/lib/etcd/wal` 目录挂载并格å¼åŒ–ç£ç›˜ã€‚`etcd` æœåŠ¡çš„ `extra_args` æŒ‡ä»¤ä¸­å¿…é¡»åŒ…å« `wal_dir` 目录。如果ä¸æŒ‡å®š `wal_dir`，etcd 进程会å°è¯•åœ¨æƒé™ä¸è¶³çš„情况下æ“作底层的 `wal` 挂载。 + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + data-dir: '/var/lib/rancher/etcd/data/' + wal-dir: '/var/lib/rancher/etcd/wal/wal_dir' + extra_binds: + - '/var/lib/etcd/data:/var/lib/rancher/etcd/data' + - '/var/lib/etcd/wal:/var/lib/rancher/etcd/wal' +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/add-users-to-projects.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/add-users-to-projects.md new file mode 100644 index 00000000000..4f4f6712b0e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/add-users-to-projects.md @@ -0,0 +1,62 @@ +--- +title: 添加项目æˆå‘˜ +--- + +如果你想为用户æ供集群内 _特定_ 项目和资æºçš„访问æƒé™ï¼Œè¯·ä¸ºç”¨æˆ·åˆ†é…项目æˆå‘˜èµ„格。 + +ä½ å¯ä»¥åœ¨åˆ›å»ºé¡¹ç›®æ—¶å°†æˆå‘˜æ·»åŠ åˆ°é¡¹ç›®ä¸­ï¼Œæˆ–将用户添加到现有项目中。 + +:::tip + +如果你需è¦ä¸ºç”¨æˆ·æ供对集群内 _所有_ 项目的访问æƒé™ï¼Œè¯·å‚è§[添加集群æˆå‘˜](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)。 + +::: + +### å°†æˆå‘˜æ·»åŠ åˆ°æ–°é¡¹ç›® + +ä½ å¯ä»¥åœ¨åˆ›å»ºé¡¹ç›®æ—¶å°†æˆå‘˜æ·»åŠ åˆ°é¡¹ç›®ä¸­ï¼ˆå»ºè®®ï¼‰ã€‚有关创建新项目的详细信æ¯ï¼Œè¯·å‚阅[集群管ç†](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)。 + +### å°†æˆå‘˜æ·»åŠ åˆ°çŽ°æœ‰é¡¹ç›® + +创建项目åŽï¼Œä½ å¯ä»¥å°†ç”¨æˆ·æ·»åŠ ä¸ºé¡¹ç›®æˆå‘˜ï¼Œä»¥ä¾¿ç”¨æˆ·å¯ä»¥è®¿é—®é¡¹ç›®çš„资æºï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ é¡¹ç›®æˆå‘˜çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦æ·»åŠ æˆå‘˜çš„项目。在项目å称上方的**创建命å空间**按钮æ—边，å•å‡» **☰**。选择 **编辑é…ç½®**。 +1. 在**æˆå‘˜**选项å¡ä¸­ï¼Œå•å‡»**添加**。 +1. æœç´¢è¦æ·»åŠ åˆ°é¡¹ç›®çš„用户或组。 + + 如果é…置了外部身份验è¯ï¼š + + - 在你键入时，Rancher 会从你的外部身份验è¯æºè¿”回用户。 + + - ä½ å¯ä»¥åœ¨ä¸‹æ‹‰èœå•ä¸­æ·»åŠ ç»„，而ä¸æ˜¯å•ä¸ªç”¨æˆ·ã€‚下拉列表仅会列出你（登录用户）所在的组。 + + :::note + + 如果你以本地用户身份登录，外部用户ä¸ä¼šæ˜¾ç¤ºåœ¨ä½ çš„æœç´¢ç»“果中。 + + ::: + +1. 分é…用户或组的**项目**角色。 + + [什么是项目角色？](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::note 注æ„事项： + + - 如果用户分é…到了项目的`所有者`或`æˆå‘˜`角色，用户会自动继承`命å空间创建`角色。然而，这个角色是 [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole)，这表示角色的范围会延展到集群中的所有项目。因此，对于显å¼åˆ†é…到了项目`所有者`或`æˆå‘˜`角色的用户æ¥è¯´ï¼Œå³ä½¿åªæœ‰`åªè¯»`角色，这些用户也å¯ä»¥åœ¨åˆ†é…给他们的其他项目中创建或删除命å空间。 + + - 默认情况下，Rancher çš„`项目æˆå‘˜`角色继承自 `Kubernetes-edit` 角色，而`项目所有者`角色继承自 `Kubernetes-admin` 角色。因此，`项目æˆå‘˜`å’Œ`项目所有者`角色都能管ç†å‘½å空间，包括创建和删除命å空间。 + + - 对于`自定义`角色，你å¯ä»¥ä¿®æ”¹å¯åˆ†é…的角色列表。 + + - è¦å°†è§’色添加到列表中，请[添加自定义角色](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md)。 + - è¦ä»Žåˆ—表中删除角色，请[é”定/解é”角色](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md)。 + + ::: + +**结果**：已将选中的用户添加到项目中。 + +- è¦æ’¤é”€é¡¹ç›®æˆå‘˜èµ„格，请选择用户并å•å‡»**删除**。此æ“作会删除æˆå‘˜èµ„格，而ä¸ä¼šåˆ é™¤ç”¨æˆ·ã€‚ +- è¦ä¿®æ”¹é¡¹ç›®ä¸­çš„用户角色，请将其从项目中删除，然åŽä½¿ç”¨ä¿®æ”¹åŽçš„角色é‡æ–°æ·»åŠ ç”¨æˆ·ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md new file mode 100644 index 00000000000..5f9c30227ba --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md @@ -0,0 +1,42 @@ +--- +title: 集群驱动 +--- + +集群驱动用于在[托管 Kubernetes æ供商](../../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)（例如 Google GKE）中创建集群。创建集群时å¯ä»¥æ˜¾ç¤ºçš„集群驱动，是由集群驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 集群驱动将作为创建集群的选项显示。默认情况下，Rancher 与多个现有的云æ供商集群驱动打包在一起，但你也å¯ä»¥å°†è‡ªå®šä¹‰é›†ç¾¤é©±åŠ¨æ·»åŠ åˆ° Rancher。 + +如果你ä¸æƒ³å‘用户显示特定的集群驱动，你å¯ä»¥åœ¨ Rancher 中åœç”¨è¿™äº›é›†ç¾¤é©±åŠ¨ï¼Œå®ƒä»¬å°†ä¸ä¼šä½œä¸ºåˆ›å»ºé›†ç¾¤çš„选项出现。 + +### 管ç†é›†ç¾¤é©±åŠ¨ + +:::note 先决æ¡ä»¶ï¼š + +è¦åˆ›å»ºã€ç¼–辑或删除集群驱动，你需è¦ä»¥ä¸‹æƒé™ä¸­çš„_一个_： + +- [管ç†å‘˜å…¨å±€æƒé™](../manage-role-based-access-control-rbac/global-permissions.md) +- 分é…了[管ç†é›†ç¾¤é©±åŠ¨è§’色](../manage-role-based-access-control-rbac/global-permissions.md)çš„[自定义全局æƒé™](../manage-role-based-access-control-rbac/global-permissions.md#自定义全局æƒé™)。 + +::: + +## 激活/åœç”¨é›†ç¾¤é©±åŠ¨ + +默认情况下，Rancher 仅激活主æµçš„云æ供商 Google GKEã€Amazon EKS å’Œ Azure AKS 的驱动。如果è¦æ˜¾ç¤ºæˆ–éšè—驱动，你å¯ä»¥æ›´æ”¹é©±åŠ¨çš„状æ€ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 + +2. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 + +3. 在**集群驱动**选项å¡ä¸Šï¼Œé€‰æ‹©è¦æ¿€æ´»æˆ–åœç”¨çš„驱动，然åŽå•å‡» **â‹® > 激活** 或 **â‹® > åœç”¨**。 + +## 添加自定义集群驱动 + +如果你想使用 Rancher ä¸æ”¯æŒå¼€ç®±å³ç”¨çš„集群驱动，你å¯ä»¥æ·»åŠ æ供商的驱动，从而使用该驱动æ¥åˆ›å»º _托管_ Kubernetes 集群： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 +1. 在**集群驱动**选项å¡ä¸Šï¼Œå•å‡»**添加集群驱动**。 +1. 填写**添加集群驱动**表å•ã€‚然åŽå•å‡»**创建**。 + + +### å¼€å‘自己的集群驱动 + +如果è¦å¼€å‘集群驱动并添加到 Rancher，请å‚考我们的[示例](https://github.com/rancher-plugins/kontainer-engine-driver-example)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md new file mode 100644 index 00000000000..bb49bcbe46a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md @@ -0,0 +1,41 @@ +--- +title: 主机驱动 +--- + +主机驱动用于é…置主机，Rancher 使用这些主机å¯åŠ¨å’Œç®¡ç† Kubernetes 集群。主机驱动与 [Docker Machine 驱动](https://docs.docker.com/machine/drivers/)相åŒã€‚创建主机模æ¿æ—¶å¯ä»¥æ˜¾ç¤ºçš„主机驱动，是由主机驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 主机驱动将显示为创建节点模æ¿çš„选项。默认情况下，Rancher 与许多现有的 Docker Machine 驱动打包在一起，但你也å¯ä»¥åˆ›å»ºè‡ªå®šä¹‰ä¸»æœºé©±åŠ¨å¹¶æ·»åŠ åˆ° Rancher。 + +如果你ä¸æƒ³å‘用户显示特定的主机驱动，则需è¦åœç”¨è¿™äº›ä¸»æœºé©±åŠ¨ã€‚ + +#### 管ç†ä¸»æœºé©±åŠ¨ + +:::note 先决æ¡ä»¶ï¼š + +è¦åˆ›å»ºã€ç¼–辑或删除驱动，你需è¦ä»¥ä¸‹æƒé™ä¸­çš„_一个_： + +- [管ç†å‘˜å…¨å±€æƒé™](../manage-role-based-access-control-rbac/global-permissions.md) +- 分é…了[管ç†ä¸»æœºé©±åŠ¨è§’色](../manage-role-based-access-control-rbac/global-permissions.md)çš„[自定义全局æƒé™](../manage-role-based-access-control-rbac/global-permissions.md#自定义全局æƒé™)。 + +::: + +## 激活/åœç”¨ä¸»æœºé©±åŠ¨ + +默认情况下，Rancher 仅激活主æµäº‘æ供商 Amazon EC2ã€Azureã€DigitalOcean å’Œ vSphere 的驱动。如果è¦æ˜¾ç¤ºæˆ–éšè—驱动，你å¯ä»¥æ›´æ”¹é©±åŠ¨çš„状æ€ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 + +2. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 + +2. 在**主机驱动**选项å¡ä¸Šï¼Œé€‰æ‹©è¦æ¿€æ´»æˆ–åœç”¨çš„驱动，然åŽå•å‡» **â‹® > 激活** 或 **â‹® > åœç”¨**。 + +## 添加自定义主机驱动 + +如果你想使用 Rancher ä¸æ”¯æŒå¼€ç®±å³ç”¨çš„主机驱动，你å¯ä»¥æ·»åŠ æ供商的驱动，从而使用该驱动为你的 Kubernetes 集群创建节点模æ¿å¹¶æœ€ç»ˆåˆ›å»ºèŠ‚点池： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 +1. 在**主机驱动**选项å¡ä¸Šï¼Œå•å‡»**添加主机驱动**。 +1. 填写**添加主机驱动**表å•ã€‚然åŽå•å‡»**创建**。 + +### å¼€å‘自己的主机驱动 + +主机驱动使用 [Docker Machine](https://docs.docker.com/machine/) æ¥å®žçŽ°ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md new file mode 100644 index 00000000000..947f2a9784c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md @@ -0,0 +1,64 @@ +--- +title: 访问和共享 +--- + +如果你是 RKE 模æ¿æ‰€æœ‰è€…，你å¯ä»¥å°†è¯¥æ¨¡æ¿å…±äº«ç»™ç”¨æˆ·æˆ–用户组，然åŽä»–们å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚ + +由于 RKE 模æ¿æ˜¯ä¸“门与用户和组共享的，因此所有者å¯ä»¥ä¸Žä¸åŒçš„用户共享ä¸åŒçš„ RKE 模æ¿ã€‚ + +共享模æ¿æ—¶ï¼Œæ¯ä¸ªç”¨æˆ·éƒ½å¯ä»¥æ‹¥æœ‰ä»¥ä¸‹ä¸¤ä¸ªè®¿é—®æƒé™ä¸­çš„其中一个： + +- **所有者**：å¯ä»¥æ›´æ–°ã€åˆ é™¤å’Œå…±äº«ä»–们拥有的模æ¿ã€‚所有者还å¯ä»¥ä¸Žå…¶ä»–用户共享模æ¿ã€‚ +- **用户**：å¯ä»¥ä½¿ç”¨æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚他们还å¯ä»¥å°†è¿™äº›é›†ç¾¤å‡çº§åˆ°åŒä¸€æ¨¡æ¿çš„新版本。如果你将模æ¿å…±äº«ä¸º**公开（åªè¯»ï¼‰**，你的 Rancher 设置中的所有用户都拥有该模æ¿çš„用户访问æƒé™ã€‚ + +如果你创建了一个模æ¿ï¼Œä½ å°†è‡ªåŠ¨æˆä¸ºè¯¥æ¨¡æ¿çš„所有者。 + +如果你想让其他用户更新该模æ¿ï¼Œä½ å¯ä»¥å…±äº«æ¨¡æ¿çš„所有æƒã€‚有关所有者如何修改模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅[修改模æ¿æ–‡æ¡£](manage-rke1-templates.md)。 + +共享模æ¿çš„æ–¹å¼æœ‰å¦‚下几ç§ï¼š + +- 在模æ¿åˆ›å»ºæœŸé—´å°†ç”¨æˆ·æ·»åŠ åˆ°æ–°çš„ RKE æ¨¡æ¿ +- 将用户添加到现有 RKE æ¨¡æ¿ +- 公开 RKE 模æ¿ï¼Œå¹¶ä¸Ž Rancher 设置中的所有用户共享 +- 与å—信任修改模æ¿çš„用户共享模æ¿æ‰€æœ‰æƒ + +### ä¸Žç‰¹å®šç”¨æˆ·æˆ–ç»„å…±äº«æ¨¡æ¿ + +è¦å…许用户或组使用你的模æ¿åˆ›å»ºé›†ç¾¤ï¼Œä½ å¯ä»¥ä¸ºä»–们æ供模æ¿çš„基本**用户**访问æƒé™ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦å…±äº«çš„模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 在**共享模æ¿**中，å•å‡»**添加æˆå‘˜**。 +1. 在**å称**字段中æœç´¢ä½ è¦ä¸Žä¹‹å…±äº«æ¨¡æ¿çš„用户或组。 +1. 选择**用户**访问类型。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户或组å¯ä»¥ä½¿ç”¨æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚ + +### ä¸Žæ‰€æœ‰ç”¨æˆ·å…±äº«æ¨¡æ¿ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦å…±äº«çš„模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 在**共享模æ¿**下，选中 **公开（åªè¯»ï¼‰** å¤é€‰æ¡†ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：Rancher 设置中的所有用户都å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚ + +### 共享模æ¿æ‰€æœ‰æƒ + +如果你是模æ¿çš„创建者，你å¯èƒ½å¸Œæœ›å°†ç»´æŠ¤å’Œæ›´æ–°æ¨¡æ¿çš„责任委派给其他用户或组。 + +在这ç§æƒ…况下，你å¯ä»¥ä¸ºç”¨æˆ·æä¾›**所有者**访问æƒé™ï¼Œè¯¥æƒé™å…许其他用户更新ã€åˆ é™¤æ¨¡æ¿æˆ–与其他用户共享对模æ¿çš„访问æƒé™ã€‚ + +è¦æŽˆäºˆç”¨æˆ·æˆ–组**所有者**æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦å…±äº«çš„ RKE 模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 在**共享模æ¿**下，å•å‡»**添加æˆå‘˜**并在**å称**字段中æœç´¢è¦ä¸Žä¹‹å…±äº«æ¨¡æ¿çš„用户或组。 +1. 在**访问类型**字段中，å•å‡»**所有者**。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户或组具有**所有者**访问类型，å¯ä»¥ä¿®æ”¹ã€å…±äº«æˆ–删除模æ¿ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md new file mode 100644 index 00000000000..3efac3e477e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md @@ -0,0 +1,59 @@ +--- +title: åº”ç”¨æ¨¡æ¿ +--- + +ä½ å¯ä»¥ä½¿ç”¨ä½ è‡ªå·±åˆ›å»ºçš„ RKE 模æ¿æ¥åˆ›å»ºé›†ç¾¤ï¼Œä¹Ÿå¯ä»¥ä½¿ç”¨[与你共享的模æ¿](access-or-share-templates.md)æ¥åˆ›å»ºé›†ç¾¤ã€‚ + +RKE 模æ¿å¯ä»¥åº”用于新集群。 + +ä½ å¯ä»¥[将现有集群的é…ç½®ä¿å­˜ä¸º RKE 模æ¿](#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这样，åªæœ‰æ¨¡æ¿æ›´æ–°åŽæ‰èƒ½æ›´æ”¹é›†ç¾¤çš„设置。 + +你无法将集群更改为使用ä¸åŒçš„ RKE 模æ¿ã€‚ä½ åªèƒ½å°†é›†ç¾¤æ›´æ–°ä¸ºåŒä¸€æ¨¡æ¿çš„新版本。 + + +### 使用 RKE 模æ¿åˆ›å»ºé›†ç¾¤ + +è¦ä½¿ç”¨ RKE 模æ¿æ·»åŠ [由基础设施æ供商托管](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)的集群，请按照以下步骤æ“作： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**并选择基础设施æ供商。 +1. 设置集群å称和节点模æ¿è¯¦æƒ…。 +1. è¦ä½¿ç”¨ RKE 模æ¿ï¼Œè¯·åœ¨**集群选项**下，选中**使用现有 RKE 模æ¿å’Œä¿®è®¢ç‰ˆ**å¤é€‰æ¡†ã€‚ +1. 从下拉èœå•ä¸­é€‰æ‹© RKE 模æ¿å’Œä¿®è®¢ç‰ˆã€‚ +1. å¯é€‰ï¼šä½ å¯ä»¥ç¼–辑 RKE 模æ¿æ‰€æœ‰è€…在创建模æ¿æ—¶æ ‡è®°ä¸º**å…许用户覆盖**的任何设置。如果你无法更改æŸäº›è®¾ç½®ï¼Œåˆ™éœ€è¦è”系模æ¿æ‰€æœ‰è€…以获å–模æ¿çš„新修订版。然åŽï¼Œä½ éœ€è¦ç¼–辑集群æ¥å°†å…¶å‡çº§åˆ°æ–°ç‰ˆæœ¬ã€‚ +1. å•å‡»**创建**以å¯åŠ¨é›†ç¾¤ã€‚ + +### 更新使用 RKE 模æ¿åˆ›å»ºçš„集群 + +模æ¿æ‰€æœ‰è€…创建 RKE 模æ¿æ—¶ï¼Œæ¯ä¸ªè®¾ç½®åœ¨ Rancher UI 中都有一个开关，指示用户是å¦å¯ä»¥è¦†ç›–该设置。 + +- 如果æŸä¸ªè®¾ç½®å…许用户覆盖，你å¯ä»¥é€šè¿‡[编辑集群](../../../../pages-for-subheaders/cluster-configuration.md)æ¥æ›´æ–°é›†ç¾¤ä¸­çš„设置。 +- 如果该开关处于关闭状æ€ï¼Œåˆ™é™¤éžé›†ç¾¤æ‰€æœ‰è€…创建了å…许你覆盖这些设置的模æ¿ä¿®è®¢ç‰ˆï¼Œå¦åˆ™ä½ æ— æ³•æ›´æ”¹è¿™äº›è®¾ç½®ã€‚如果你无法更改æŸäº›è®¾ç½®ï¼Œåˆ™éœ€è¦è”系模æ¿æ‰€æœ‰è€…以获å–模æ¿çš„新修订版。 + +如果集群是使用 RKE 模æ¿åˆ›å»ºçš„，你å¯ä»¥ç¼–辑集群，æ¥å°†é›†ç¾¤æ›´æ–°ä¸ºæ¨¡æ¿çš„新版本。 + +现有集群的设置å¯ä»¥[ä¿å­˜ä¸º RKE 模æ¿](#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。在这ç§æƒ…况下，你还å¯ä»¥ç¼–辑集群以将集群更新为模æ¿çš„新版本。 + +:::note + +你无法将集群更改为使用ä¸åŒçš„ RKE 模æ¿ã€‚ä½ åªèƒ½å°†é›†ç¾¤æ›´æ–°ä¸ºåŒä¸€æ¨¡æ¿çš„新版本。 + +::: + +### 将现有集群转æ¢ä¸ºä½¿ç”¨ RKE æ¨¡æ¿ + +本节介ç»å¦‚何使用现有集群创建 RKE 模æ¿ã€‚ + +除éžä½ å°†çŽ°æœ‰é›†ç¾¤çš„设置ä¿å­˜ä¸º RKE 模æ¿ï¼Œå¦åˆ™ RKE 模æ¿ä¸èƒ½åº”用于现有集群。这将把集群的设置导出为新的 RKE 模æ¿ï¼Œå¹¶ä¸”将集群绑定到该模æ¿ã€‚然åŽï¼Œåªæœ‰[更新了模æ¿](manage-rke1-templates.md#更新模æ¿)并且集群å‡çº§åˆ°**使用更新版本的模æ¿**时，集群æ‰èƒ½æ”¹å˜ã€‚ + +è¦å°†çŽ°æœ‰é›†ç¾¤è½¬æ¢ä¸ºä½¿ç”¨ RKE 模æ¿ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è½¬æ¢ä¸ºä½¿ç”¨ RKE 模æ¿çš„集群。å•å‡» **â‹® > ä¿å­˜ä¸º RKE 模æ¿**。 +1. 在出现的表å•ä¸­è¾“入模æ¿çš„å称，然åŽå•å‡»**创建**。 + +**结果**： + +- 创建了一个新的 RKE 模æ¿ã€‚ +- 将集群转æ¢ä¸ºä½¿ç”¨è¯¥æ–°æ¨¡æ¿ã€‚ +- å¯ä»¥[使用新模æ¿åˆ›å»ºæ–°é›†ç¾¤](apply-templates.md#使用-rke-模æ¿åˆ›å»ºé›†ç¾¤)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md new file mode 100644 index 00000000000..7db812c38ef --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md @@ -0,0 +1,57 @@ +--- +title: 模æ¿åˆ›å»ºè€…æƒé™ +--- + +管ç†å‘˜æœ‰åˆ›å»º RKE 模æ¿çš„æƒé™ï¼Œåªæœ‰ç®¡ç†å‘˜å¯ä»¥å°†è¯¥æƒé™æŽˆäºˆå…¶ä»–用户。 + +有关管ç†å‘˜æƒé™çš„更多信æ¯ï¼Œè¯·å‚阅[全局æƒé™æ–‡æ¡£](../manage-role-based-access-control-rbac/global-permissions.md)。 + +## 授予用户创建模æ¿çš„æƒé™ + +åªæœ‰å…·æœ‰**创建 RKE 模æ¿**全局æƒé™çš„用户æ‰èƒ½åˆ›å»ºæ¨¡æ¿ã€‚ + +管ç†å‘˜æ‹¥æœ‰åˆ›å»ºæ¨¡æ¿çš„全局æƒé™ï¼Œåªæœ‰ç®¡ç†å‘˜æ‰èƒ½å°†è¯¥æƒé™æŽˆäºˆå…¶ä»–用户。 + +有关å…许用户修改现有模æ¿çš„ä¿¡æ¯ï¼Œè¯·å‚阅[共享模æ¿](access-or-share-templates.md)。 + +管ç†å‘˜å¯ä»¥é€šè¿‡ä¸¤ç§æ–¹å¼æŽˆäºˆç”¨æˆ·åˆ›å»º RKE 模æ¿çš„æƒé™ï¼š + +- 通过编辑[å•ä¸ªç”¨æˆ·](#å…许用户创建模æ¿)çš„æƒé™ +- 通过更改[新用户的默认æƒé™](#默认å…许新用户创建模æ¿) + +### å…è®¸ç”¨æˆ·åˆ›å»ºæ¨¡æ¿ + +管ç†å‘˜å¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤å°†**创建 RKE 模æ¿**角色å•ç‹¬æŽˆäºˆç»™ä»»ä½•çŽ°æœ‰ç”¨æˆ·ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. 选择è¦ç¼–辑的用户，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**内置角色**中，选中**创建 RKE 集群模æ¿**角色以åŠç”¨æˆ·åº”具有的其他角色。你å¯èƒ½è¿˜éœ€è¦é€‰ä¸­**创建 RKE 模æ¿ä¿®è®¢ç‰ˆ**å¤é€‰æ¡†ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户拥有创建 RKE 模æ¿çš„æƒé™ã€‚ + +### 默认å…è®¸æ–°ç”¨æˆ·åˆ›å»ºæ¨¡æ¿ + +管ç†å‘˜ä¹Ÿå¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤ä¸ºæ‰€æœ‰æ–°ç”¨æˆ·æŽˆäºˆåˆ›å»º RKE 模æ¿çš„默认æƒé™ã€‚è¿™ä¸ä¼šå½±å“现有用户的æƒé™ã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 转到**创建 RKE 集群模æ¿**角色，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 选择**是：新用户的默认角色**选项。 +1. å•å‡»**ä¿å­˜**。 +1. 如果你希望新用户能够创建 RKE 模æ¿ä¿®è®¢ï¼Œè¯·å°†è¯¥è§’色设置为默认值。 + +**结果**：在此 Rancher 安装中创建的任何新用户都å¯ä»¥åˆ›å»º RKE 模æ¿ã€‚现有用户将ä¸ä¼šèŽ·å¾—æ­¤æƒé™ã€‚ + +### å–消创建模æ¿çš„æƒé™ + +管ç†å‘˜å¯ä»¥é€šè¿‡ä»¥ä¸‹æ­¥éª¤åˆ é™¤ç”¨æˆ·åˆ›å»ºæ¨¡æ¿çš„æƒé™ã€‚请注æ„，无论是å¦é€‰æ‹©äº†ç»†ç²’度æƒé™ï¼Œç®¡ç†å‘˜éƒ½å¯ä»¥å®Œå…¨æŽ§åˆ¶æ‰€æœ‰èµ„æºã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. 选择è¦ç¼–辑æƒé™çš„用户，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**内置角色**中，å–消选中**创建 RKE 模æ¿**å’Œ**创建 RKE 模æ¿ä¿®è®¢ç‰ˆ**å¤é€‰æ¡†ï¼ˆå¦‚果适用）。在此处，你å¯ä»¥å°†ç”¨æˆ·æ”¹å›žæ™®é€šç”¨æˆ·ï¼Œæˆ–授予用户一组ä¸åŒçš„æƒé™ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户无法创建 RKE 模æ¿ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md new file mode 100644 index 00000000000..38ab0f41ee8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md @@ -0,0 +1,43 @@ +--- +title: å¼ºåˆ¶ä½¿ç”¨æ¨¡æ¿ +--- + +本节介ç»æ¨¡æ¿ç®¡ç†å‘˜å¦‚何在 Rancher 中强制执行模æ¿ï¼Œä»Žè€Œé™åˆ¶ç”¨æˆ·åœ¨æ²¡æœ‰æ¨¡æ¿çš„情况下创建集群。 + +默认情况下，Rancher 中的任何普通用户都å¯ä»¥åˆ›å»ºé›†ç¾¤ã€‚但当开å¯å¼ºåˆ¶ä½¿ç”¨ RKE 模æ¿æ—¶ï¼Œæœ‰ä»¥ä¸‹çº¦æŸï¼š + +- åªæœ‰ç®¡ç†å‘˜æ‰èƒ½åœ¨æ²¡æœ‰æ¨¡æ¿çš„情况下创建集群。 +- 所有普通用户必须使用 RKE 模æ¿æ¥åˆ›å»ºæ–°é›†ç¾¤ã€‚ +- 普通用户在ä¸ä½¿ç”¨æ¨¡æ¿çš„情况下无法创建集群。 + +åªæœ‰ç®¡â€‹â€‹ç†å‘˜[授予æƒé™](creator-permissions.md#å…许用户创建模æ¿)åŽï¼Œç”¨æˆ·æ‰èƒ½åˆ›å»ºæ–°æ¨¡æ¿ã€‚ + +使用 RKE 模æ¿åˆ›å»ºé›†ç¾¤åŽï¼Œé›†ç¾¤åˆ›å»ºè€…无法编辑模æ¿ä¸­å®šä¹‰çš„设置。创建集群åŽæ›´æ”¹è¿™äº›è®¾ç½®çš„唯一方法是[将集群å‡çº§åˆ°ç›¸åŒæ¨¡æ¿çš„新修订版](apply-templates.md#更新使用-rke-模æ¿åˆ›å»ºçš„集群)。如果集群创建者想è¦æ›´æ”¹æ¨¡æ¿å®šä¹‰çš„设置，他们需è¦è”系模æ¿æ‰€æœ‰è€…以获å–模æ¿çš„新版本。有关模æ¿ä¿®è®¢å¦‚何工作的详细信æ¯ï¼Œè¯·å‚阅[修订模æ¿](manage-rke1-templates.md#更新模æ¿)。 + +## 强制新集群使用 RKE æ¨¡æ¿ + +è¦æ±‚用户创建新集群时使用模æ¿ï¼Œå¯ä»¥ç¡®ä¿[普通用户](../manage-role-based-access-control-rbac/global-permissions.md)å¯åŠ¨çš„任何集群都使用ç»è¿‡ç®¡ç†å‘˜å®¡æ ¸çš„ Kubernetes å’Œ Rancher 设置。 + +管ç†å‘˜å¯ä»¥é€šè¿‡ä»¥ä¸‹æ­¥éª¤å¯ç”¨ RKE 模æ¿å¼ºåˆ¶ï¼Œä»Žè€Œè¦æ±‚用户必须使用模æ¿åˆ›å»ºé›†ç¾¤ï¼š + +1. å•å‡» **☰ > 全局设置**。 +1. 转到 `cluster-template-enforcement` 设置。å•å‡» **â‹® > 编辑设置**。 +1. 将值设置为 **True** 并å•å‡»**ä¿å­˜**。 + + :::note é‡è¦æ示： + + 如果管ç†å‘˜å°† `cluster-template-enforcement` 设置为 True，还需è¦ä¸Žç”¨æˆ·å…±äº« `clusterTemplates`，以便用户å¯ä»¥é€‰æ‹©å…¶ä¸­ä¸€ä¸ªæ¨¡æ¿æ¥åˆ›å»ºé›†ç¾¤ã€‚ + + ::: + +**结果**：除éžåˆ›å»ºè€…是管ç†å‘˜ï¼Œå¦åˆ™ Rancher é…置的所有集群都必须使用模æ¿ã€‚ + +## ç¦ç”¨ RKE 模æ¿å¼ºåˆ¶ + +管ç†å‘˜å¯ä»¥é€šè¿‡ä»¥ä¸‹æ­¥éª¤å…³é—­ RKE 模æ¿å¼ºåˆ¶ï¼Œä»Žè€Œå…许用户在没有 RKE 模æ¿çš„情况下创建新集群： + +1. å•å‡» **☰ > 全局设置**。 +1. 转到 `cluster-template-enforcement` 设置。å•å‡» **â‹® > 编辑设置**。 +1. 将值设置为 **False** 并å•å‡»**ä¿å­˜**。 + +**结果**：在 Rancher 中创建集群时，用户ä¸éœ€è¦ä½¿ç”¨æ¨¡æ¿ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md new file mode 100644 index 00000000000..7457df20253 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md @@ -0,0 +1,70 @@ +--- +title: 示例场景 +--- + +以下示例场景æ述了组织如何使用模æ¿æ¥æ ‡å‡†åŒ–集群创建。 + +- **强制执行模æ¿**：如果希望所有 Rancher é…置的新集群都具有æŸäº›è®¾ç½®ï¼Œç®¡ç†å‘˜å¯èƒ½æƒ³è¦[为æ¯ä¸ªç”¨æˆ·å¼ºåˆ¶æ‰§è¡Œä¸€é¡¹æˆ–多项模æ¿è®¾ç½®](#强制执行模æ¿è®¾ç½®)。 +- **与ä¸åŒçš„用户共享ä¸åŒçš„模æ¿**：管ç†å‘˜å¯ä»¥ä¸º[普通用户和高级用户æä¾›ä¸åŒçš„模æ¿](#普通用户和高级用户模æ¿)。这样，普通用户会有更多é™åˆ¶é€‰é¡¹ï¼Œè€Œé«˜çº§ç”¨æˆ·åœ¨åˆ›å»ºé›†ç¾¤æ—¶å¯ä»¥ä½¿ç”¨æ›´å¤šé€‰é¡¹ã€‚ +- **更新模æ¿è®¾ç½®**：如果组织的安全和 DevOps 团队决定将最佳实践嵌入到新集群所需的设置中，这些最佳实践å¯èƒ½ä¼šéšç€æ—¶é—´è€Œæ”¹å˜ã€‚如果最佳实践å‘生å˜åŒ–，[å¯ä»¥å°†æ¨¡æ¿æ›´æ–°ä¸ºæ–°ç‰ˆæœ¬](#更新模æ¿å’Œé›†ç¾¤)。这样，使用模æ¿åˆ›å»ºçš„集群å¯ä»¥å‡çº§åˆ°æ¨¡æ¿çš„新版本。 +- **共享模æ¿çš„所有æƒ**：当模æ¿æ‰€æœ‰è€…ä¸å†æƒ³è¦ç»´æŠ¤æ¨¡æ¿æˆ–想è¦å…±äº«æ¨¡æ¿çš„所有æƒæ—¶ï¼Œæ­¤æ–¹æ¡ˆæ述了如何[授æƒæ¨¡æ¿æ‰€æœ‰æƒ](#å…许其他用户控制和共享模æ¿)。 + + +## 强制执行模æ¿è®¾ç½® + +å‡è®¾ä¸€ä¸ªç»„织的管ç†å‘˜å†³å®šç”¨ Kubernetes 版本 1.14 创建所有新集群： + +1. 首先，管ç†å‘˜åˆ›å»ºä¸€ä¸ªæ¨¡æ¿ï¼Œå°† Kubernetes 版本指定为 1.14，并将所有其他设置标记为**å…许用户覆盖**。 +1. 管ç†å‘˜å°†æ¨¡æ¿å…¬å¼€ã€‚ +1. 管ç†å‘˜æ‰“开模æ¿å¼ºåˆ¶åŠŸèƒ½ã€‚ + +**结果**： + +- 组织中的所有 Rancher 用户都å¯ä»¥è®¿é—®è¯¥æ¨¡æ¿ã€‚ +- [普通用户](../manage-role-based-access-control-rbac/global-permissions.md)使用此模æ¿åˆ›å»ºçš„所有新集群都将使用 Kubernetes 1.14，它们无法使用其它 Kubernetes 版本。默认情况下，普通用户没有创建模æ¿çš„æƒé™ã€‚因此，除éžä¸Žä»–们共享更多模æ¿ï¼Œå¦åˆ™æ­¤æ¨¡æ¿å°†æ˜¯æ™®é€šç”¨æˆ·å”¯ä¸€å¯ä»¥ä½¿ç”¨çš„模æ¿ã€‚ +- 所有普通用户都必须使用集群模æ¿æ¥åˆ›å»ºæ–°é›†ç¾¤ã€‚他们无法在ä¸ä½¿ç”¨æ¨¡æ¿çš„情况下创建集群。 + +通过这ç§æ–¹å¼ï¼Œç®¡ç†å‘˜åœ¨æ•´ä¸ªç»„织中强制执行 Kubernetes 版本，åŒæ—¶ä»ç„¶å…许最终用户é…置其他所有内容。 + +## æ™®é€šç”¨æˆ·å’Œé«˜çº§ç”¨æˆ·æ¨¡æ¿ + +å‡è®¾ä¸€ä¸ªç»„织有普通用户和高级用户。管ç†å‘˜å¸Œæœ›æ™®é€šç”¨æˆ·å¿…须使用模æ¿ï¼Œè€Œé«˜çº§ç”¨æˆ·å’Œç®¡ç†å‘˜å¯ä»¥æ ¹æ®è‡ªå·±çš„需è¦åˆ›å»ºé›†ç¾¤ã€‚ + +1. 首先，管ç†å‘˜å¼€å¯ [RKE 模æ¿å¼ºåˆ¶æ‰§è¡Œ](enforce-templates.md#强制新集群使用-rke-模æ¿)。这æ„å‘³ç€ Rancher 中的æ¯ä¸ª[普通用户](../manage-role-based-access-control-rbac/global-permissions.md)在创建集群时都需è¦ä½¿ç”¨ RKE 模æ¿ã€‚ +1. 然åŽç®¡ç†å‘˜åˆ›å»ºä¸¤ä¸ªæ¨¡æ¿ï¼š + +- 一个普通用户模æ¿ï¼Œè¯¥æ¨¡æ¿é™¤äº†è®¿é—®å¯†é’¥å¤–，几乎指定了所有选项 +- 一个高级用户模æ¿ï¼Œè¯¥æ¨¡æ¿å…·æœ‰å¤§éƒ¨åˆ†æˆ–所有已å¯ç”¨**å…许用户覆盖**的选项 + +1. 管ç†å‘˜ä»…与高级用户共享高级模æ¿ã€‚ +1. 管ç†å‘˜å°†æ™®é€šç”¨æˆ·çš„模æ¿å…¬å¼€ï¼Œå› æ­¤åœ¨ Rancher 中创建的 RKE 集群的æ¯ä¸ªäººéƒ½èƒ½é€‰æ‹©é™åˆ¶æ€§æ›´å¼ºçš„模æ¿ã€‚ + +**结果**：除管ç†å‘˜å¤–，所有 Rancher 用户在创建集群时都需è¦ä½¿ç”¨æ¨¡æ¿ã€‚æ¯ä¸ªäººéƒ½å¯ä»¥è®¿é—®é™åˆ¶æ¨¡æ¿ï¼Œä½†åªæœ‰é«˜çº§ç”¨æˆ·æœ‰æƒä½¿ç”¨æ›´å®½æ¾çš„模æ¿ã€‚普通用户会å—到更多é™åˆ¶ï¼Œè€Œé«˜çº§ç”¨æˆ·åœ¨é…ç½® Kubernetes 集群时有更多选择。 + +## 更新模æ¿å’Œé›†ç¾¤ + +å‡è®¾ä¸€ä¸ªç»„织有一个模æ¿ï¼Œè¯¥æ¨¡æ¿è¦æ±‚集群使用 Kubernetes v1.14。然而，éšç€æ—¶é—´çš„推移，管ç†å‘˜æ”¹å˜äº†ä¸»æ„。管ç†å‘˜çŽ°åœ¨å¸Œæœ›ç”¨æˆ·èƒ½å¤Ÿå‡çº§é›†ç¾¤ï¼Œä»¥ä½¿ç”¨æ›´æ–°ç‰ˆæœ¬çš„ Kubernetes。 + +åœ¨è¿™ä¸ªç»„ç»‡ä¸­ï¼Œè®¸å¤šé›†ç¾¤æ˜¯ç”¨ä¸€ä¸ªéœ€è¦ Kubernetes v1.14 的模æ¿åˆ›å»ºçš„。由于模æ¿ä¸å…许é‡å†™è¯¥è®¾ç½®ï¼Œå› æ­¤åˆ›å»ºé›†ç¾¤çš„用户无法直接编辑该设置。 + +模æ¿æ‰€æœ‰è€…å¯ä»¥æœ‰ä»¥ä¸‹å‡ ä¸ªé€‰é¡¹ï¼Œæ¥å…许集群创建者在集群上å‡çº§ Kubernetes： + +- **在模æ¿ä¸ŠæŒ‡å®š Kubernetes v1.15**：模æ¿æ‰€æœ‰è€…å¯ä»¥åˆ›å»ºæŒ‡å®š Kubernetes v1.15 的新模æ¿ä¿®è®¢ç‰ˆã€‚然åŽä½¿ç”¨è¯¥æ¨¡æ¿çš„æ¯ä¸ªé›†ç¾¤çš„所有者å¯ä»¥å°†é›†ç¾¤å‡çº§åˆ°æ¨¡æ¿çš„新版本。此模æ¿å‡çº§å…许集群创建者在集群上将 Kubernetes å‡çº§åˆ° v1.15。 +- **å…许在模æ¿ä¸Šä½¿ç”¨ä»»ä½• Kubernetes 版本**：创建模æ¿ä¿®è®¢æ—¶ï¼Œæ¨¡æ¿æ‰€æœ‰è€…还å¯ä»¥ä½¿ç”¨ Rancher UI 上该设置附近的开关，将 Kubernetes 版本标记为**å…许用户覆盖**。该设置å…许å‡çº§åˆ°æ­¤æ¨¡æ¿ç‰ˆæœ¬çš„é›†ç¾¤ä½¿ç”¨ä»»æ„ Kubernetes 的版本。 +- **å…许在模æ¿ä¸Šä½¿ç”¨æœ€æ–°çš„ Kubernetes 次è¦ç‰ˆæœ¬**：模æ¿æ‰€æœ‰è€…还å¯ä»¥åˆ›å»ºä¸€ä¸ªæ¨¡æ¿ä¿®è®¢ç‰ˆï¼Œå…¶ä¸­ Kubernetes 版本被定义为 **Latest v1.14（å…许补ä¸ç‰ˆæœ¬å‡çº§ï¼‰**。这æ„味ç€ä½¿ç”¨è¯¥ç‰ˆæœ¬çš„集群将能够进行补ä¸ç‰ˆæœ¬å‡çº§ï¼Œä½†ä¸æ”¯æŒä¸»è¦ç‰ˆæœ¬å‡çº§ã€‚ + +## å…è®¸å…¶ä»–ç”¨æˆ·æŽ§åˆ¶å’Œå…±äº«æ¨¡æ¿ + +å‡è®¾ Alice 是 Rancher 管ç†å‘˜ã€‚她拥有一个 RKE 模æ¿ï¼Œè¯¥æ¨¡æ¿å映了她的组织为创建集群而商定的最佳实践。 + +Bob 是一ä½é«˜çº§ç”¨æˆ·ï¼Œå¯ä»¥å°±é›†ç¾¤é…ç½®åšå‡ºæ˜Žæ™ºçš„决策。éšç€æœ€ä½³å®žè·µéšç€æ—¶é—´çš„推移ä¸æ–­æ›´æ–°ï¼ŒAlice 相信 Bob 会为她的模æ¿åˆ›å»ºæ–°çš„修订。因此，她决定让 Bob æˆä¸ºæ¨¡æ¿çš„所有者。 + +为了与 Bob 共享模æ¿çš„所有æƒï¼ŒAlice [å°† Bob 添加为模æ¿çš„所有者](access-or-share-templates.md#共享模æ¿æ‰€æœ‰æƒ)。 + +结果是，作为模æ¿æ‰€æœ‰è€…，Bob 负责该模æ¿çš„版本控制。Bob 现在å¯ä»¥æ‰§è¡Œä»¥ä¸‹æ‰€æœ‰æ“作： + +- 当最佳实践å‘生å˜åŒ–æ—¶[修改模æ¿](manage-rke1-templates.md#更新模æ¿) +- [ç¦ç”¨æ¨¡æ¿çš„过时修订](manage-rke1-templates.md#ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆ)，以ç¦æ­¢ä½¿ç”¨è¯¥æ¨¡æ¿æ¥åˆ›å»ºé›†ç¾¤ +- 如果组织想è¦æ”¹å˜æ–¹å‘，则[删除整个模æ¿](manage-rke1-templates.md#删除模æ¿) +- [å°†æŸä¸ªç‰ˆæœ¬è®¾ç½®ä¸ºé»˜è®¤å€¼](manage-rke1-templates.md#将模æ¿ä¿®è®¢ç‰ˆè®¾ç½®ä¸ºé»˜è®¤)，用于用户创建集群。模æ¿çš„最终用户ä»ç„¶å¯ä»¥é€‰æ‹©ä»–们想è¦ä½¿ç”¨å“ªä¸ªç‰ˆæœ¬æ¥åˆ›å»ºé›†ç¾¤ã€‚ +- [与特定用户共享模æ¿](access-or-share-templates.md)，让所有 Rancher 用户都å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿ï¼Œæˆ–与其他用户共享该模æ¿çš„所有æƒã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md new file mode 100644 index 00000000000..a3cb27d9b6b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md @@ -0,0 +1,69 @@ +--- +title: RKE 模æ¿å’ŒåŸºç¡€è®¾æ–½ +--- + +在 Rancher 中，RKE 模æ¿ç”¨äºŽé…ç½® Kubernetes 和定义 Rancher 设置，而节点模æ¿åˆ™ç”¨äºŽé…置节点。 + +因此，å³ä½¿å¼€å¯äº† RKE 模æ¿å¼ºåˆ¶ï¼Œæœ€ç»ˆç”¨æˆ·åœ¨åˆ›å»º Rancher 集群时ä»ç„¶å¯ä»¥çµæ´»é€‰æ‹©åº•å±‚硬件。RKE 模æ¿çš„最终用户ä»ç„¶å¯ä»¥é€‰æ‹©åŸºç¡€è®¾æ–½æ供商和他们想è¦ä½¿ç”¨çš„节点。 + +如果è¦æ ‡å‡†åŒ–集群中的硬件，请将 RKE 模æ¿ä¸ŽèŠ‚点模æ¿æˆ–æœåŠ¡å™¨é…置工具 (如 Terraform) 结åˆä½¿ç”¨ã€‚ + +### èŠ‚ç‚¹æ¨¡æ¿ + +[节点模æ¿](../../../../reference-guides/user-settings/manage-node-templates.md)è´Ÿè´£ Rancher 中的节点é…置和节点预é…。你å¯ä»¥åœ¨ç”¨æˆ·é…置文件中设置节点模æ¿ï¼Œä»Žè€Œå®šä¹‰åœ¨æ¯ä¸ªèŠ‚点池中使用的模æ¿ã€‚å¯ç”¨èŠ‚点池åŽï¼Œå¯ä»¥ç¡®ä¿æ¯ä¸ªèŠ‚点池中都有所需数é‡çš„节点，并确ä¿æ± ä¸­çš„所有节点都相åŒã€‚ + +### Terraform + +Terraform 是一个æœåŠ¡å™¨é…置工具。它使用基础架构å³ä»£ç ï¼Œæ”¯æŒä½¿ç”¨ Terraform é…置文件创建几乎所有的基础设施。它å¯ä»¥è‡ªåŠ¨æ‰§è¡ŒæœåŠ¡å™¨é…置，这ç§æ–¹å¼æ˜¯è‡ªæ–‡æ¡£åŒ–的，并且在版本控制中易于跟踪。 + +本节é‡ç‚¹ä»‹ç»å¦‚何将 Terraform 与 [Rancher 2 Terraform Provider](https://www.terraform.io/docs/providers/rancher2/) 一起使用，这是标准化 Kubernetes 集群硬件的推è选项。如果你使用 Rancher Terraform Provider æ¥é…置硬件，然åŽä½¿ç”¨ RKE 模æ¿åœ¨è¯¥ç¡¬ä»¶ä¸Šé…ç½® Kubernetes 集群，你å¯ä»¥å¿«é€Ÿåˆ›å»ºä¸€ä¸ªå…¨é¢çš„ã€å¯ç”¨äºŽç”Ÿäº§çš„集群。 + +Terraform 支æŒï¼š + +- 定义几乎任何类型的基础架构å³ä»£ç ï¼ŒåŒ…括æœåŠ¡å™¨ã€æ•°æ®åº“ã€è´Ÿè½½å‡è¡¡å™¨ã€ç›‘控ã€é˜²ç«å¢™è®¾ç½®å’Œ SSL è¯ä¹¦ +- 使用应用商店应用和多集群应用 +- 跨多个平å°ï¼ˆåŒ…括 Rancher 和主è¦äº‘æä¾›å•†ï¼‰å¯¹åŸºç¡€è®¾æ–½è¿›è¡Œç¼–ç  +- 将基础架构å³ä»£ç æ交到版本控制 +- è½»æ¾é‡å¤ä½¿ç”¨åŸºç¡€è®¾æ–½çš„é…置和设置 +- 将基础架构更改纳入标准开å‘实践 +- 防止由于é…ç½®å移，导致一些æœåŠ¡å™¨çš„é…置与其他æœåŠ¡å™¨ä¸åŒ + +## Terraform å·¥ä½œåŽŸç† + +Terraform 是用扩展å为 `.tf` 的文件编写的。它是用 HashiCorp é…置语言编写的。HashiCorp é…置语言是一ç§å£°æ˜Žæ€§è¯­è¨€ï¼Œæ”¯æŒå®šä¹‰é›†ç¾¤ä¸­æ‰€éœ€çš„基础设施ã€æ­£åœ¨ä½¿ç”¨çš„云æ供商以åŠæ供商的凭è¯ã€‚ç„¶åŽ Terraform å‘æ供商å‘出 API 调用，以便有效地创建基础设施。 + +è¦ä½¿ç”¨ Terraform 创建 Rancher é…置的集群，请转到你的 Terraform é…置文件并将æ供商定义为 Rancher 2。你å¯ä»¥ä½¿ç”¨ Rancher API 密钥设置你的 Rancher 2 æ供商。请注æ„，API 密钥与其关è”的用户具有相åŒçš„æƒé™å’Œè®¿é—®çº§åˆ«ã€‚ + +ç„¶åŽ Terraform 会调用 Rancher API æ¥é…置你的基础设施，而 Rancher 调用基础设施æ供商。例如，如果你想使用 Rancher 在 AWS 上预é…基础设施，你需è¦åœ¨ Terraform é…置文件或环境å˜é‡ä¸­æä¾› Rancher API 密钥和 AWS 凭è¯ï¼Œä»¥ä¾¿å®ƒä»¬ç”¨äºŽé¢„é…基础设施。 + +如果你需è¦å¯¹åŸºç¡€è®¾æ–½è¿›è¡Œæ›´æ”¹ï¼Œä½ å¯ä»¥åœ¨ Terraform é…置文件中进行更改，而ä¸æ˜¯æ‰‹åŠ¨æ›´æ–°æœåŠ¡å™¨ã€‚然åŽï¼Œå¯ä»¥å°†è¿™äº›æ–‡ä»¶æ交给版本控制ã€éªŒè¯ï¼Œå¹¶æ ¹æ®éœ€è¦è¿›è¡Œæ£€æŸ¥ã€‚然åŽï¼Œå½“ä½ è¿è¡Œ `terraform apply` 时，更改将会被部署。 + +## 使用 Terraform 的技巧 + +- [Rancher 2 æ供商文档](https://www.terraform.io/docs/providers/rancher2/)æ供了如何é…置集群大部分的示例。 + +- 在 Terraform 设置中，你å¯ä»¥ä½¿ç”¨ Docker Machine 主机驱动æ¥å®‰è£… Docker Machine。 + +- å¯ä»¥åœ¨ Terraform Provider 中修改身份验è¯ã€‚ + +- å¯ä»¥é€šè¿‡æ›´æ”¹ Rancher 中的设置，æ¥åå‘工程如何在 Terraform 中定义设置，然åŽè¿”回并检查 Terraform 状æ€æ–‡ä»¶ï¼Œä»¥æŸ¥çœ‹è¯¥æ–‡ä»¶å¦‚何映射到基础设施的当å‰çŠ¶æ€ã€‚ + +- å¦‚æžœä½ æƒ³åœ¨ä¸€ä¸ªåœ°æ–¹ç®¡ç† Kubernetes 集群设置ã€Rancher 设置和硬件设置，请使用 [Terraform 模å—](https://github.com/rancher/terraform-modules)。你å¯ä»¥å°†é›†ç¾¤é…ç½® YAML 文件或 RKE 模æ¿é…置文件传递给 Terraform 模å—，以便 Terraform 模å—创建它。在这ç§æƒ…况下，你å¯ä»¥ä½¿ç”¨åŸºç¡€æž¶æž„å³ä»£ç æ¥ç®¡ç† Kubernetes 集群åŠå…¶åº•å±‚硬件的版本控制和修订历å²ã€‚ + +## åˆ›å»ºç¬¦åˆ CIS 基准的集群的技巧 + +本节æ述了一ç§æ–¹æ³•ï¼Œå¯ä»¥ä½¿å®‰å…¨åˆè§„相关的é…置文件æˆä¸ºé›†ç¾¤çš„标准é…置文件。 + +在你创建[ç¬¦åˆ CIS 基准的集群](../../../../pages-for-subheaders/rancher-security.md)时，你有一个加密é…置文件和一个审计日志é…置文件。 + +你的基础设施预é…系统å¯ä»¥å°†è¿™äº›æ–‡ä»¶å†™å…¥ç£ç›˜ã€‚然åŽåœ¨ä½ çš„ RKE 模æ¿ä¸­ï¼Œä½ éœ€è¦æŒ‡å®šè¿™äº›æ–‡ä»¶çš„ä½ç½®ï¼Œç„¶åŽå°†ä½ çš„加密é…置文件和审计日志é…置文件作为é¢å¤–的挂载添加到 `kube-api-server`。 + +然åŽï¼Œä½ éœ€è¦ç¡®ä¿ RKE 模æ¿ä¸­çš„ `kube-api-server` æ ‡å¿—ä½¿ç”¨ç¬¦åˆ CIS çš„é…置文件。 + +通过这ç§æ–¹å¼ï¼Œä½ å¯ä»¥åˆ›å»ºç¬¦åˆ CIS 基准的标志。 + +## èµ„æº + +- [Terraform 文档](https://www.terraform.io/docs/) +- [Rancher2 Terraform Provider 文档](https://www.terraform.io/docs/providers/rancher2/) +- [The RanchCast - 第 1 集：Rancher 2 Terraform Provider](https://youtu.be/YNCq-prI8-8)：在此演示中，社区主管 Jason van Brackel 使用 Rancher 2 Terraform Provider 创建了节点并创建自定义集群。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md new file mode 100644 index 00000000000..6e4adb98239 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md @@ -0,0 +1,161 @@ +--- +title: 创建和修改 RKE æ¨¡æ¿ +--- + +本节介ç»å¦‚ä½•ç®¡ç† RKE 模æ¿å’Œä¿®è®¢ç‰ˆã€‚ä½ å¯ä»¥ä»Ž **RKE1 é…ç½® > RKE 模æ¿**下的**集群管ç†**视图创建ã€å…±äº«ã€æ›´æ–°å’Œåˆ é™¤æ¨¡æ¿ã€‚ + +模æ¿æ›´æ–°é€šè¿‡ä¿®è®¢ç³»ç»Ÿå¤„ç†ã€‚当模æ¿æ‰€æœ‰è€…想è¦æ›´æ”¹æˆ–更新模æ¿æ—¶ï¼Œä»–们会创建模æ¿çš„新版本。å•ä¸ªä¿®è®¢æ— æ³•ç¼–辑。但是，如果你想防止使用修订æ¥åˆ›å»ºæ–°é›†ç¾¤ï¼Œä½ å¯ä»¥ç¦ç”¨å®ƒã€‚ + +ä½ å¯ä»¥ä½¿ç”¨ä¸¤ç§æ–¹å¼æ¥ä½¿ç”¨æ¨¡æ¿ä¿®è®¢ï¼šåˆ›å»ºæ–°é›†ç¾¤ï¼Œæˆ–å‡çº§ä½¿ç”¨è¾ƒæ—©ç‰ˆæœ¬çš„模æ¿åˆ›å»ºçš„集群。模æ¿åˆ›å»ºè€…å¯ä»¥è®¾ç½®é»˜è®¤ä¿®è®¢ç‰ˆï¼Œä½†æ˜¯åœ¨æœ€ç»ˆç”¨æˆ·åˆ›å»ºé›†ç¾¤æ—¶ï¼Œä»–们å¯ä»¥é€‰æ‹©ä»»ä½•æ¨¡æ¿ä»¥åŠå¯ä¾›ä½¿ç”¨çš„任何模æ¿ä¿®è®¢ç‰ˆã€‚使用指定的修订版创建集群åŽï¼Œå°±æ— æ³•å°†å…¶æ›´æ”¹ä¸ºå¦ä¸€ä¸ªæ¨¡æ¿ï¼Œä½†æ˜¯å¯ä»¥å°†é›†ç¾¤å‡çº§ä¸ºåŒä¸€æ¨¡æ¿çš„较新的å¯ç”¨ä¿®è®¢ç‰ˆã€‚ + +模æ¿æ‰€æœ‰è€…对模æ¿ä¿®è®¢ç‰ˆå…·æœ‰å®Œå…¨æŽ§åˆ¶æƒï¼Œå¹¶ä¸”å¯ä»¥åˆ›å»ºæ–°çš„修订版æ¥æ›´æ–°æ¨¡æ¿ï¼Œåˆ é™¤æˆ–ç¦ç”¨ä¸åº”被用于创建集群的修订版，和设置默认的模æ¿ä¿®è®¢ç‰ˆã€‚ + + +### 先决æ¡ä»¶ + +如果你具有**创建 RKE 模æ¿**æƒé™ï¼Œåˆ™å¯ä»¥åˆ›å»º RKE 模æ¿ï¼Œè¯¥æƒé™å¯ç”±[管ç†å‘˜æŽˆäºˆ](creator-permissions.md)。 + +如果你是模æ¿çš„所有者，你å¯ä»¥ä¿®æ”¹ã€å…±äº«å’Œåˆ é™¤æ¨¡æ¿ã€‚有关如何æˆä¸ºæ¨¡æ¿æ‰€æœ‰è€…的详细信æ¯ï¼Œè¯·å‚阅[共享模æ¿æ‰€æœ‰æƒæ–‡æ¡£](access-or-share-templates.md#共享模æ¿æ‰€æœ‰æƒ)。 + +### åˆ›å»ºæ¨¡æ¿ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. 输入模æ¿çš„å称。Rancher å·²ç»ä¸ºæ¨¡æ¿çš„第一个版本自动生æˆäº†å称，该版本与该模æ¿ä¸€èµ·åˆ›å»ºã€‚ +1. å¯é€‰ï¼šé€šè¿‡å°†ç”¨æˆ·æ·»åŠ ä¸ºæˆå‘˜ï¼Œæ¥[与其他用户或组共享模æ¿](access-or-share-templates.md#与特定用户或组共享模æ¿)。你还å¯ä»¥å°†æ¨¡æ¿å…¬å¼€ï¼Œä»Žè€Œä¸Ž Rancher 中的所有人共享。 +1. 然åŽæŒ‰ç…§å±å¹•ä¸Šçš„表格将集群é…ç½®å‚æ•°ä¿å­˜ä¸ºæ¨¡æ¿ä¿®è®¢çš„一部分。å¯ä»¥å°†ä¿®è®¢æ ‡è®°ä¸ºæ­¤æ¨¡æ¿çš„默认值。 + +**结果**：é…置了具有一个修订版的 RKE 模æ¿ã€‚ä½ å¯ä»¥ç¨åŽåœ¨[é…ç½® Rancher å¯åŠ¨çš„集群](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)时使用此 RKE 模æ¿ä¿®è®¢ç‰ˆã€‚通过 RKE 模æ¿ç®¡ç†é›†ç¾¤åŽï¼Œé›†ç¾¤æ— æ³•è§£é™¤ä¸Žæ¨¡æ¿çš„绑定，并且无法å–消选中**使用现有 RKE 模æ¿å’Œä¿®è®¢ç‰ˆ**。 + +### æ›´æ–°æ¨¡æ¿ + +æ›´æ–° RKE 模æ¿ç›¸å½“于创建现有模æ¿çš„修订版。使用旧版本模æ¿åˆ›å»ºçš„集群å¯ä»¥è¿›è¡Œæ›´æ–°ï¼Œä»Žè€ŒåŒ¹é…新版本。 + +ä½ ä¸èƒ½ç¼–辑å•ä¸ªä¿®è®¢ã€‚由于你无法编辑模æ¿çš„å•ä¸ªä¿®è®¢ï¼Œä¸ºäº†é˜²æ­¢ä½¿ç”¨æŸä¸ªä¿®è®¢ï¼Œä½ å¯ä»¥[ç¦ç”¨è¯¥ä¿®è®¢ç‰ˆ](#ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆ)。 + +创建新模æ¿ä¿®è®¢æ—¶ï¼Œä½¿ç”¨æ—§æ¨¡æ¿ä¿®è®¢çš„集群ä¸å—å½±å“。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦ç¼–辑的模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 编辑所需信æ¯å¹¶å•å‡»**ä¿å­˜**。 +1. å¯é€‰ï¼šä½ å¯ä»¥æ›´æ”¹æ­¤æ¨¡æ¿çš„默认修订版，也å¯ä»¥æ›´æ”¹å…±äº«å¯¹è±¡ã€‚ + +**结果**：模æ¿å·²æ›´æ–°ã€‚è¦å°†å…¶åº”用到使用旧版本模æ¿çš„集群，请å‚阅[å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ](#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)。 + +### åˆ é™¤æ¨¡æ¿ + +当ä¸å†éœ€è¦ä¸ºä»»ä½•é›†ç¾¤ä½¿ç”¨æŸä¸ª RKE 模æ¿æ—¶ï¼Œå¯ä»¥å°†å…¶åˆ é™¤ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦åˆ é™¤çš„ RKE 模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 删除**。 +1. 确认删除。 + +**结果**：模æ¿è¢«åˆ é™¤ã€‚ + +### 基于默认版创建新修订版 + +ä½ å¯ä»¥å¤åˆ¶é»˜è®¤æ¨¡æ¿ä¿®è®¢ç‰ˆå¹¶å¿«é€Ÿæ›´æ–°å…¶è®¾ç½®ï¼Œè€Œæ— éœ€ä»Žå¤´å¼€å§‹åˆ›å»ºæ–°ä¿®è®¢ç‰ˆã€‚克隆模æ¿ä¸ºä½ çœåŽ»äº†é‡æ–°è¾“入集群创建所需的访问密钥和其他å‚数的麻烦。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦å…‹éš†çš„ RKE 模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 基于默认版创建新修订版**。 +1. 填写表å•çš„其余部分æ¥åˆ›å»ºæ–°ä¿®è®¢ã€‚ + +**结果**：克隆并é…置了 RKE 模æ¿ä¿®è®¢ç‰ˆã€‚ + +### 基于克隆版创建新修订版 + +通过用户设置创建新的 RKE 模æ¿ä¿®è®¢ç‰ˆæ—¶ï¼Œå¯ä»¥å…‹éš†çŽ°æœ‰ä¿®è®¢ç‰ˆå¹¶å¿«é€Ÿæ›´æ–°å…¶è®¾ç½®ï¼Œè€Œæ— éœ€ä»Žå¤´å¼€å§‹åˆ›å»ºæ–°çš„修订版。克隆模æ¿ä¿®è®¢çœåŽ»äº†é‡æ–°è¾“入集群å‚数的麻烦。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦å…‹éš†çš„模æ¿ä¿®è®¢ã€‚然åŽé€‰æ‹© **â‹® > 克隆修订版**。 +1. 填写表å•çš„其余部分。 + +**结果**：克隆并é…置了 RKE 模æ¿ä¿®è®¢ç‰ˆã€‚ä½ å¯ä»¥åœ¨é…置集群时使用 RKE 模æ¿ä¿®è®¢ã€‚任何使用此 RKE 模æ¿çš„现有集群都å¯ä»¥å‡çº§åˆ°æ­¤æ–°ç‰ˆæœ¬ã€‚ + +### ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆ + +当你ä¸éœ€è¦å°† RKE 模æ¿ä¿®è®¢ç‰ˆæœ¬ç”¨äºŽåˆ›å»ºæ–°é›†ç¾¤æ—¶ï¼Œå¯ä»¥ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆã€‚你也å¯ä»¥é‡æ–°å¯ç”¨ç¦ç”¨äº†çš„修订版。 + +如果没有任何集群使用该修订，你å¯ä»¥ç¦ç”¨è¯¥ä¿®è®¢ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦ç¦ç”¨çš„模æ¿ä¿®è®¢ç‰ˆã€‚然åŽé€‰æ‹© **â‹® > ç¦ç”¨**。 + +**结果**：RKE 模æ¿ä¿®è®¢ç‰ˆä¸èƒ½ç”¨äºŽåˆ›å»ºæ–°é›†ç¾¤ã€‚ + +### é‡æ–°å¯ç”¨ç¦ç”¨çš„模æ¿ä¿®è®¢ç‰ˆ + +如果è¦ä½¿ç”¨å·²ç¦ç”¨çš„ RKE 模æ¿ä¿®è®¢ç‰ˆæ¥åˆ›å»ºæ–°é›†ç¾¤ï¼Œä½ å¯ä»¥é‡æ–°å¯ç”¨è¯¥ä¿®è®¢ç‰ˆã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦é‡æ–°å¯ç”¨çš„模æ¿ä¿®è®¢ã€‚然åŽé€‰æ‹© **â‹® > å¯ç”¨**。 + +**结果**：RKE 模æ¿ä¿®è®¢ç‰ˆå¯ç”¨äºŽåˆ›å»ºæ–°é›†ç¾¤ã€‚ + +### 将模æ¿ä¿®è®¢ç‰ˆè®¾ç½®ä¸ºé»˜è®¤ + +当最终用户使用 RKE 模æ¿åˆ›å»ºé›†ç¾¤æ—¶ï¼Œä»–们å¯ä»¥é€‰æ‹©ä½¿ç”¨å“ªä¸ªç‰ˆæœ¬æ¥åˆ›å»ºé›†ç¾¤ã€‚ä½ å¯ä»¥é…置默认使用的版本。 + +è¦å°† RKE 模æ¿ä¿®è®¢ç‰ˆè®¾ç½®ä¸ºé»˜è®¤ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦è®¾ä¸ºé»˜è®¤çš„ RKE 模æ¿ä¿®è®¢ç‰ˆï¼Œç„¶åŽå•å‡» **â‹® > 设为默认é…ç½®**。 + +**结果**：使用模æ¿åˆ›å»ºé›†ç¾¤æ—¶ï¼ŒRKE 模æ¿ä¿®è®¢ç‰ˆå°†ç”¨ä½œé»˜è®¤é€‰é¡¹ã€‚ + +### 删除模æ¿ä¿®è®¢ç‰ˆ + +ä½ å¯ä»¥åˆ é™¤æ¨¡æ¿çš„所有修订（默认修订除外）。 + +è¦æ°¸ä¹…删除修订版： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦åˆ é™¤çš„ RKE 模æ¿ä¿®è®¢ç‰ˆï¼Œç„¶åŽå•å‡» **â‹® > 删除**。 + +**结果**：RKE 模æ¿ä¿®è®¢ç‰ˆè¢«åˆ é™¤ã€‚ + +### å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ + +:::note + +本部分å‡è®¾ä½ å·²ç»æœ‰ä¸€ä¸ªé›†ç¾¤ï¼Œè¯¥é›†ç¾¤[应用了 RKE 模æ¿](apply-templates.md)。 + +本部分还å‡è®¾ä½ å·²[更新了集群使用的模æ¿](#更新模æ¿)，以便å¯ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆã€‚ + +::: + +è¦å°†é›†ç¾¤å‡çº§åˆ°ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**集群选项**中，å•å‡»æ¨¡æ¿ä¿®è®¢ç‰ˆçš„下拉èœå•ï¼Œç„¶åŽé€‰æ‹©æ–°çš„模æ¿ä¿®è®¢ç‰ˆã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：集群已å‡çº§ä¸ºä½¿ç”¨æ–°æ¨¡æ¿ä¿®è®¢ç‰ˆä¸­å®šä¹‰çš„设置。 + +### 将正在è¿è¡Œçš„集群导出到新的 RKE 模æ¿å’Œä¿®è®¢ç‰ˆ + +ä½ å¯ä»¥å°†çŽ°æœ‰é›†ç¾¤çš„设置ä¿å­˜ä¸º RKE 模æ¿ã€‚ + +这将把集群的设置导出为新的 RKE 模æ¿ï¼Œå¹¶ä¸”将集群绑定到该模æ¿ã€‚然åŽï¼Œåªæœ‰[更新了模æ¿](#更新模æ¿)并且集群å‡çº§åˆ°[使用更新版本的模æ¿](#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)时，集群æ‰èƒ½æ”¹å˜ã€‚ + +è¦å°†çŽ°æœ‰é›†ç¾¤è½¬æ¢ä¸ºä½¿ç”¨ RKE 模æ¿ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到将被转æ¢ä¸ºä½¿ç”¨ RKE 模æ¿çš„é›†ç¾¤ï¼Œç„¶åŽ **â‹® > ä¿å­˜ä¸º RKE 模æ¿**。 +1. 在出现的表å•ä¸­è¾“å…¥ RKE 模æ¿çš„å称，然åŽå•å‡»**创建**。 + +**结果**： + +- 创建了一个新的 RKE 模æ¿ã€‚ +- 将集群转æ¢ä¸ºä½¿ç”¨è¯¥æ–°æ¨¡æ¿ã€‚ +- å¯ä»¥[使用新模æ¿å’Œä¿®è®¢ç‰ˆåˆ›å»ºæ–°é›†ç¾¤ã€‚](apply-templates.md#使用-rke-模æ¿åˆ›å»ºé›†ç¾¤) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md new file mode 100644 index 00000000000..11ecac6fabd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md @@ -0,0 +1,14 @@ +--- +title: 覆盖模æ¿è®¾ç½® +--- + +用户创建 RKE 模æ¿æ—¶ï¼Œæ¨¡æ¿ä¸­çš„æ¯ä¸ªè®¾ç½®åœ¨ Rancher UI 中都有一个开关，指示用户是å¦å¯ä»¥è¦†ç›–该设置。此开关将这些设置标记为**å…许用户覆盖**。打开开关表示用户å¯ä»¥ä¿®æ”¹å¯¹åº”çš„å‚数，关闭开关表示用户无æƒä¿®æ”¹å¯¹åº”çš„å‚数。 + +使用模æ¿åˆ›å»ºé›†ç¾¤åŽï¼Œé™¤éžæ¨¡æ¿æ‰€æœ‰è€…将设置标记为**å…许用户覆盖**，å¦åˆ™æœ€ç»ˆç”¨æˆ·æ— æ³•æ›´æ–°æ¨¡æ¿ä¸­å®šä¹‰çš„任何设置。但是，如果模æ¿[更新到新修订版](manage-rke1-templates.md)，且该修订版更改了设置或å…许最终用户更改设置，则集群å¯ä»¥å‡çº§åˆ°æ¨¡æ¿çš„新修订版，并且新修订版中的更改将应用于集群。 + +如果 RKE 模æ¿ä¸Šçš„任何å‚数设置为**å…许用户覆盖**，最终用户必须在集群创建期间设置这些字段，然åŽä»–们å¯ä»¥éšæ—¶ç¼–辑这些设置。 + +RKE 模æ¿çš„**å…许用户覆盖**选项的适用场景如下： + +- 管ç†å‘˜è®¤ä¸ºæŸäº›å‚数需è¦ä¿æŒçµæ´»æ€§ï¼Œä»¥ä¾¿éšæ—¶æ›´æ–°ã€‚ +- 最终用户将需è¦è¾“入他们自己的访问密钥或密文密钥，例如，云凭è¯æˆ–备份快照的凭è¯ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md new file mode 100644 index 00000000000..704c47a68c5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md @@ -0,0 +1,219 @@ +--- +title: é…ç½® Active Directory (AD) +--- + +如果你的组织使用 Microsoft Active Directory 作为中心用户仓库，你å¯ä»¥å°† Rancher é…置为与 Active Directory æœåŠ¡å™¨é€šä¿¡ï¼Œä»Žè€Œå¯¹ç”¨æˆ·è¿›è¡Œèº«ä»½éªŒè¯ã€‚这使 Rancher 管ç†å‘˜å¯ä»¥å¯¹å¤–部用户系统中的用户和组进行集群和项目的访问控制，åŒæ—¶å…许最终用户在登录 Rancher UI 时使用 Active Directory 凭è¯è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +Rancher 使用 LDAP 与 Active Directory æœåŠ¡å™¨é€šä¿¡ã€‚因此，Active Directory 与 [OpenLDAP 身份验è¯](../../../../pages-for-subheaders/configure-openldap.md)çš„æµç¨‹ç›¸åŒã€‚ + +:::note + +在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + +::: + +## 先决æ¡ä»¶ + +你需è¦åˆ›å»ºæˆ–从你的 AD 管ç†å‘˜å¤„获å–一个新的 AD 用户，用作 Rancher çš„ ServiceAccount。此用户必须具有足够的æƒé™ï¼Œæ‰èƒ½æ‰§è¡Œ LDAP æœç´¢å¹¶è¯»å–ä½ çš„ AD 域下的用户和组的属性。 + +通常å¯ä»¥ä½¿ç”¨ï¼ˆéžç®¡ç†å‘˜ï¼‰**域用户**è´¦å·ï¼Œå› ä¸ºé»˜è®¤æƒ…况下，该用户对域分区中的大多数对象具有åªè¯»ç‰¹æƒã€‚ + +但是，请注æ„，在æŸäº›é”定的 Active Directory é…置中，此默认æ“作å¯èƒ½ä¸é€‚用。在这ç§æƒ…况下，你需è¦ç¡®ä¿ ServiceAccount 用户在 Base OU（包å«ç”¨æˆ·å’Œç»„）上或全局范围内至少拥有域的 **Read** å’Œ **List Content** æƒé™ã€‚ + +:::note 使用 TLS？ + +- 如果 AD æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +- å‡çº§åˆ° v2.6.0 åŽï¼Œå¦‚æžœ AD æœåŠ¡å™¨ä¸Šçš„è¯ä¹¦ä¸æ”¯æŒ SAN 属性，则使用 TLS 通过 Rancher 对 Active Directory 进行身份验è¯å¯èƒ½ä¼šå¤±è´¥ã€‚这是 Go v1.15 中默认å¯ç”¨çš„检查。 + + - 收到"Error creating SSL connection: LDAP Result Code 200 "Network Error": x509 错误：è¯ä¹¦ä¾èµ–于旧的通用å称（Common Name）字段，使用 SAN 或临时å¯ç”¨ä¸Ž GODEBUG=x509ignoreCN=0 匹é…的通用å称。 + + - è¦è§£å†³æ­¤é”™è¯¯ï¼Œè¯·ä½¿ç”¨æ”¯æŒ SAN 属性的新è¯ä¹¦æ›´æ–°æˆ–æ›¿æ¢ AD æœåŠ¡å™¨ä¸Šçš„è¯ä¹¦ã€‚或者，将 `GODEBUG=x509ignoreCN=0` 设置为 Rancher Server 容器的环境å˜é‡æ¥å¿½ç•¥æ­¤é”™è¯¯ã€‚ + +::: + +## é…置步骤 +### 打开 Active Directory é…ç½® + +1. 使用åˆå§‹çš„本地 `admin` è´¦å·ç™»å½•åˆ° Rancher UI。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **ActiveDirectory**。然åŽä¼šæ˜¾ç¤º**验è¯æ供程åºï¼šActiveDirectory** 的表å•ã€‚ +1. 填写表å•ã€‚如果需è¦èŽ·å–帮助，请å‚è§ä¸‹æ–¹çš„é…置选项详情。 +1. 点击**å¯ç”¨**。 + +### é…ç½® Active Directory æœåŠ¡å™¨ + +在 `1. é…ç½® Active Directory æœåŠ¡å™¨` 的步骤中，使用你 Active Directory 的实际信æ¯å®Œæˆå­—段é…置。有关æ¯ä¸ªå‚数所需值的详细信æ¯ï¼Œè¯·å‚阅下表。 + +:::note + +如果你ä¸ç¡®å®šè¦åœ¨ç”¨æˆ·/组`æœç´¢åº“`字段中输入什么值，请å‚è§[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema)。 + +::: + +**表 1：AD æœåŠ¡å™¨å‚æ•°** + +| å‚æ•° | æè¿° | +|:--|:--| +| 主机å | 指定 AD æœåŠ¡å™¨çš„主机å或 IP 地å€ã€‚ | +| ç«¯å£ | 指定 AD æœåŠ¡å™¨ç›‘å¬è¿žæŽ¥çš„端å£ã€‚未加密的 LDAP 通常使用 389 的标准端å£ï¼Œè€Œ LDAPS 使用 636 端å£ã€‚ | +| TLS | 选中此框å¯å¯ç”¨ SSL/TLS 上的 LDAP（通常称为 LDAPS）。 | +| æœåŠ¡å™¨è¿žæŽ¥è¶…æ—¶ | Rancher 在认为无法访问 AD æœåŠ¡å™¨ä¹‹å‰ç­‰å¾…的时间（秒）。 | +| ServiceAccount 用户å | 输入对域分区具有åªè¯»è®¿é—®æƒé™çš„ AD è´¦å·çš„用户å（å‚è§[先决æ¡ä»¶](#先决æ¡ä»¶)）。用户åå¯ä»¥ç”¨ NetBIOS（例如 "DOMAIN\serviceaccount"）或 UPN æ ¼å¼ï¼ˆä¾‹å¦‚ "serviceaccount@domain.com"）。 | +| ServiceAccount å¯†ç  | ServiceAccount 的密ç ã€‚ | +| 默认登录域 | 如果你使用 AD 域的 NetBIOS å称é…置此字段，在绑定到 AD æœåŠ¡å™¨æ—¶ï¼Œæ²¡æœ‰åŒ…å«åŸŸçš„用户å（例如“jdoeâ€ï¼‰å°†è‡ªåŠ¨è½¬æ¢ä¸ºå¸¦æ–œæ çš„ NetBIOS 登录（例如，“LOGIN_DOMAIN\jdoeâ€ï¼‰ã€‚如果你的用户以 UPN（例如，"jdoe@acme.com"）作为用户å进行身份验è¯ï¼Œåˆ™æ­¤å­—段必须**å¿…é¡»**留空。 | +| 用户æœç´¢åº“ | 输入目录树中开始æœç´¢ç”¨æˆ·å¯¹è±¡çš„节点的标识å称（DN）。所有用户都必须是此基础标识å称的åŽä»£ã€‚例如，"ou=people,dc=acme,dc=com"。 | +| 组æœç´¢åº“ | 如果组ä½äºŽ`用户æœç´¢åº“`下é…置的节点之外的其他节点下，则需è¦åœ¨æ­¤å¤„æ供标识å称。å¦åˆ™è¯·ç•™ç©ºã€‚例如："ou=groups,dc=acme,dc=com"。 | + +--- + +### é…置用户/组 Schema + +在 `2. 自定义 Schema` 中，你必须为 Rancher æ供与目录中使用的 Schema 对应的用户和组属性的正确映射。 + +Rancher 使用 LDAP 查询æ¥æœç´¢å’Œæ£€ç´¢å…³äºŽ Active Directory 中的用户和组的信æ¯ã€‚本节中é…置的属性映射用于构造æœç´¢ç­›é€‰å™¨å’Œè§£æžç»„æˆå‘˜èº«ä»½ã€‚因此，所æ供的设置需è¦å映你 AD 域的真实情况。 + +:::note + +如果你ä¸ç†Ÿæ‚‰ Active Directory 域中使用的 Schema，请å‚è§[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema) æ¥ç¡®å®šæ­£ç¡®çš„é…置值。 + +::: + +#### 用户 Schema + +下表详细说明了用户 Schema é…置的å‚数。 + +**表 2：用户 Schema é…ç½®å‚æ•°** + +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于用户对象的对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Username Attribute | 用户属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Login Attribute | 登录属性的值与用户登录 Rancher 时输入的凭è¯çš„用户å部分匹é…。如果你的用户以他的 UPN（例如 "jdoe@acme.com"）作为用户å进行身份验è¯ï¼Œåˆ™æ­¤å­—段通常必须设置为 `userPrincipalName`。å¦åˆ™ï¼Œå¯¹äºŽæ—§çš„ NetBIOS 风格的登录å（例如 "jdoe"），则通常设为 `sAMAccountName`。 | +| User Member Attribute | 包å«ç”¨æˆ·æ‰€å±žç»„的属性。 | +| Search Attribute | 当用户输入文本以在用户界é¢ä¸­æ·»åŠ ç”¨æˆ·æˆ–组时，Rancher 会查询 AD æœåŠ¡å™¨ï¼Œå¹¶å°è¯•æ ¹æ®æ­¤è®¾ç½®ä¸­æ供的属性匹é…用户。å¯ä»¥é€šè¿‡ä½¿ç”¨ç®¡é“（“\|â€ï¼‰ç¬¦å·åˆ†éš”属性æ¥æŒ‡å®šå¤šä¸ªå±žæ€§ã€‚è¦åŒ¹é… UPN 用户å（例如 jdoe@acme.com），通常应将此字段的值设置为 `userPrincipalName`。 | +| Search Filter | 当 Rancher å°è¯•å°†ç”¨æˆ·æ·»åŠ åˆ°ç½‘站访问列表，或å°è¯•å°†æˆå‘˜æ·»åŠ åˆ°é›†ç¾¤æˆ–项目时，此筛选器将应用于æœç´¢çš„用户列表。例如，用户æœç´¢ç­›é€‰å™¨å¯èƒ½æ˜¯ (|(memberOf=CN=group1,CN=Users,DC=testad,DC=rancher,DC=io)(memberOf=CN=group2,CN=Users,DC=testad,DC=rancher,DC=io))。注æ„：如果æœç´¢ç­›é€‰å™¨æœªä½¿ç”¨[有效的 AD æœç´¢è¯­æ³•](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax)，则用户列表将为空。 | +| User Enabled Attribute | 该属性是一个整数值，代表用户账å·æ ‡å¿—的枚举。Rancher 使用此选项æ¥ç¡®å®šç”¨æˆ·è´¦å·æ˜¯å¦å·²ç¦ç”¨ã€‚通常应该将此å‚数设置为 AD 标准的 `userAccountControl`。 | +| Disabled Status Bitmask | 指定的ç¦ç”¨ç”¨æˆ·è´¦å·çš„ `User Enabled Attribute` 的值。通常，你应该将此å‚数设置为 Microsoft Active Directory Schema 中指定的默认值 2（请å‚è§[此处](https://docs.microsoft.com/en-us/windows/desktop/adschema/a-useraccountcontrol#remarks)）。 | + +--- + +#### 组 Schema + +下表详细说明了组 Schema é…置的å‚数。 + +**表 3：组 Schema é…ç½®å‚æ•°** + +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于组对象的对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Name Attribute | å称属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Group Member User Attribute | **用户属性**çš„å称。它的格å¼ä¸Ž `Group Member Mapping Attribute` 中的组æˆå‘˜åŒ¹é…。 | +| Group Member Mapping Attribute | 包å«ç»„æˆå‘˜çš„组属性的å称。 | +| Search Attribute | 在将组添加到集群或项目时，用于构造æœç´¢ç­›é€‰å™¨çš„属性。请å‚è§ç”¨æˆ· Schema çš„ `Search Attribute`。 | +| Search Filter | 当 Rancher å°è¯•å°†ç»„添加到网站访问列表，或将组添加到集群或项目时，此筛选器将应用于æœç´¢çš„组列表。例如，组æœç´¢ç­›é€‰å™¨å¯ä»¥æ˜¯ (|(cn=group1)(cn=group2))。注æ„：如果æœç´¢ç­›é€‰å™¨æœªä½¿ç”¨[有效的 AD æœç´¢è¯­æ³•](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax)，则组列表将为空。 | +| Group DN Attribute | 组属性的å称，其格å¼ä¸Žæ述用户æˆå‘˜èº«ä»½çš„用户属性中的值匹é…。å‚è§ `User Member Attribute`。 | +| Nested Group Membership | 此设置定义 Rancher 是å¦åº”解æžåµŒå¥—组æˆå‘˜èº«ä»½ã€‚仅当你的组织使用这些嵌套æˆå‘˜èº«ä»½æ—¶æ‰ä½¿ç”¨ï¼ˆå³ä½ æœ‰åŒ…å«å…¶ä»–组作为æˆå‘˜çš„组。我们建议尽é‡é¿å…使用嵌套组，从而é¿å…在存在大é‡åµŒå¥—æˆå‘˜æ—¶å‡ºçŽ°æ½œåœ¨çš„性能问题）。 | + +--- + +### æµ‹è¯•èº«ä»½éªŒè¯ + +完æˆé…ç½®åŽï¼Œè¯·**使用你的 AD 管ç†å‘˜è´¦æˆ·**测试与 AD æœåŠ¡å™¨çš„连接。如果测试æˆåŠŸï¼Œå°†å¯ç”¨é…置的 Active Directory 身份验è¯ï¼Œæµ‹è¯•æ—¶ä½¿ç”¨çš„è´¦å·ä¼šæˆä¸ºç®¡ç†å‘˜ã€‚ + +:::note + +与此步骤中输入的凭è¯ç›¸å…³çš„ AD 用户将映射到本地主体账å·ï¼Œå¹¶åœ¨ Rancher 中分é…系统管ç†å‘˜æƒé™ã€‚因此，你应该决定使用哪个 AD è´¦å·æ¥æ‰§è¡Œæ­¤æ­¥éª¤ã€‚ + +::: + +1. 输入应映射到本地主体账å·çš„ AD è´¦å·çš„**用户å**å’Œ**密ç ** 。 +2. 点击**å¯ç”¨ Active Directory 认è¯**æ¥å®Œæˆè®¾ç½®ã€‚ + +**结果**： + +- å·²å¯ç”¨ Active Directory 身份验è¯ã€‚ +- 你已使用 AD 凭è¯ä»¥ç³»ç»Ÿç®¡ç†å‘˜èº«ä»½ç™»å½•åˆ° Rancher。 + +:::note + +如果 LDAP æœåŠ¡ä¸­æ–­ï¼Œä½ ä»ç„¶å¯ä»¥ä½¿ç”¨æœ¬åœ°é…置的 `admin` è´¦å·å’Œå¯†ç ç™»å½•ã€‚ + +::: + +## 附录：使用 ldapsearch 确定æœç´¢åº“å’Œ Schema + +为了æˆåŠŸé…ç½® AD 身份验è¯ï¼Œä½ å¿…é¡»æä¾› AD æœåŠ¡å™¨çš„层次结构和 Schema 的正确é…置。 + +[`ldapsearch`](https://manpages.ubuntu.com/manpages/kinetic/en/man1/ldapsearch.1.html) 工具å…许你查询你的 AD æœåŠ¡å™¨ï¼Œä»Žè€Œäº†è§£ç”¨äºŽç”¨æˆ·å’Œç»„对象的 Schema。 + +在下é¢çš„示例命令中，我们å‡è®¾ï¼š + +- Active Directory æœåŠ¡å™¨çš„主机å是 `ad.acme.com`。 +- æœåŠ¡å™¨æ­£åœ¨ç›‘å¬ç«¯å£ `389` 上的未加密连接。 +- Active Directory 的域是 `acme`。 +- 你有一个用户å为 `jdoe`，密ç ä¸º `secret` 的有效 AD è´¦å·ã€‚ + +### 确认æœç´¢åº“ + +首先，我们将使用 `ldapsearch` æ¥æ‰¾åˆ°ç”¨æˆ·å’Œç»„的父节点的标识å称： + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "dc=acme,dc=com" -s sub "sAMAccountName=jdoe" +``` + +此命令执行 LDAP æœç´¢ï¼Œæœç´¢èµ·ç‚¹è®¾ç½®ä¸ºåŸŸæ ¹ç›®å½•ï¼ˆ`-b "dc=acme,dc=com"`），并执行针对用户账å·ï¼ˆ`sAMAccountNam=jdoe`）的筛选器，返回所述用户的属性： + +![](/img/ldapsearch-user.png) + +因为在这ç§æƒ…况下，用户的 DN 是 `CN=John Doe,CN=Users,DC=acme,DC=com` [5]，所以我们应该使用父节点 DN `CN=Users,DC=acme,DC=com` æ¥é…ç½®**用户æœç´¢åº“**。 + +åŒæ ·ï¼ŒåŸºäºŽ **memberOf** 属性 [4] 中引用的组的 DN，**组æœç´¢åº“**çš„å€¼å°†æ˜¯è¯¥å€¼çš„çˆ¶èŠ‚ç‚¹ï¼Œå³ `OU=Groups,DC=acme,DC=com`。 + +### 确定用户 Schema + +上述 `ldapsearch` 查询的输出还能用于确定在用户 Schema é…置中使用的值： + +- `Object Class`：**person** [1] +- `Username Attribute`:：**name** [2] +- `Login Attribute`：**sAMAccountName** [3] +- `User Member Attribute`：**memberOf** [4] + +:::note + +如果我们组织中的 AD 用户使用其 UPN（例如 `jdoe@acme.com`）而ä¸æ˜¯çŸ­ç™»å½•å进行身份验è¯ï¼Œåˆ™å¿…须将 `Login Attribute` 设置为 **userPrincipalName**。 + +::: + +我们还将 `Search Attribute` 数设置为 **sAMAccountName|name**。这样，用户å¯ä»¥é€šè¿‡è¾“入用户å或全å添加到 Rancher UI 中的集群/项目中。 + +### 确定组 Schema + +接下æ¥ï¼Œæˆ‘们将查询与此用户关è”的一个组，在本例中为 `CN=examplegroup,OU=Groups,DC=acme,DC=com`： + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "ou=groups,dc=acme,dc=com" \ +-s sub "CN=examplegroup" +``` + +此命令将告知我们用于组对象的属性： + +![](/img/ldapsearch-group.png) + +åŒæ ·ï¼Œè¿™èƒ½è®©æˆ‘们确定è¦åœ¨ç»„ Schema é…置中输入的值： + +- `Object Class`：**group** [1] +- `Name Attribute`：**name** [2] +- `Group Member Mapping Attribute`：**member** [3] +- `Search Attribute`：**sAMAccountName** [4] + +查看 **member** 属性的值，我们å¯ä»¥çœ‹åˆ°å®ƒåŒ…å«è¢«å¼•ç”¨ç”¨æˆ·çš„ DN。这对应我们的用户对象中的 **distinguishedName** 属性。因此，必须将 `Group Member User Attribute` å‚数的值设置为此属性。 + +åŒæ ·ï¼Œæˆ‘们å¯ä»¥çœ‹åˆ°ç”¨æˆ·å¯¹è±¡ä¸­ **memberOf** 属性中的值对应组的 **distinguishedName** [5]。因此，我们需è¦å°† `Group DN Attribute` å‚数的值设置为此属性。 + +## 附录：故障排除 + +如果在测试与 Active Directory æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·é¦–先仔细检查为 ServiceAccount 输入的凭è¯ä»¥åŠæœç´¢åº“é…置。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md new file mode 100644 index 00000000000..77d51e007bf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md @@ -0,0 +1,326 @@ +--- +title: é…ç½® Azure AD +--- + +## Microsoft Graph API + +Microsoft Graph API 现在是设置 Azure AD çš„æµç¨‹ã€‚下文将帮助[新用户](#新用户设置)使用新实例æ¥é…ç½® Azure AD，并帮助现有 Azure 应用所有者[è¿ç§»åˆ°æ–°æµç¨‹](#从-azure-ad-graph-api-è¿ç§»åˆ°-microsoft-graph-api)。 + +Rancher 中的 Microsoft Graph API æµç¨‹æ­£åœ¨ä¸æ–­å‘展。建议你使用最新的 2.7 è¡¥ä¸ç‰ˆæœ¬ï¼Œè¯¥ç‰ˆæœ¬ä»åœ¨ç§¯æžå¼€å‘中，并将æŒç»­èŽ·å¾—新功能和改进。 + +### 新用户设置 + +如果你在 Azure 中托管了一个 Active Directory（AD）实例，你å¯ä»¥å°† Rancher é…置为å…许你的用户使用 AD è´¦å·ç™»å½•ã€‚你需è¦åœ¨ Azure å’Œ Rancher 中进行 Azure AD 外部身份验è¯ã€‚ + +:::note 注æ„事项 + +- Azure AD 集æˆä»…支æŒæœåŠ¡æ供商å‘起的登录。 +- 大部分æ“作是从 [Microsoft Azure 门户](https://portal.azure.com/)执行的。 + +::: + +#### Azure Active Directory é…置概述 + +è¦å°† Rancher é…置为å…许用户使用其 Azure AD è´¦å·è¿›è¡Œèº«ä»½éªŒè¯ï¼Œä½ éœ€è¦æ‰§è¡Œå¤šä¸ªæ­¥éª¤ã€‚在你开始之å‰ï¼Œè¯·æŸ¥çœ‹ä¸‹æ–‡æ“作步骤大纲。 + +:::tip + +在开始之å‰ï¼Œæ‰“开两个æµè§ˆå™¨é€‰é¡¹å¡ï¼šä¸€ä¸ªç”¨äºŽ Rancher，å¦ä¸€ä¸ªç”¨äºŽ Azure 门户。这样，你å¯ä»¥å°†é—¨æˆ·çš„é…置值å¤åˆ¶å¹¶ç²˜è´´åˆ° Rancher 中。 + +::: + + +#### 1. 在 Azure 注册 Rancher + +在 Rancher 中å¯ç”¨ Azure AD 之å‰ï¼Œä½ å¿…é¡»å…ˆå‘ Azure 注册 Rancher。 + +1. 以管ç†ç”¨æˆ·èº«ä»½ç™»å½• [Microsoft Azure](https://portal.azure.com/)。åŽç»­é…置步骤中需è¦ç®¡ç†è®¿é—®æƒé™ã€‚ + +1. 使用æœç´¢åŠŸèƒ½æ‰“å¼€ **App registrations** æœåŠ¡ã€‚ + +1. 点击 **New registration** 并填写表å•ã€‚ + + ![New App Registration](/img/new-app-registration.png) + + 1. 输入 **Name**（例如 `Rancher`）。 + + + 1. 在 **Supported account types** 中，选择 **Accounts in this organizational directory only (AzureADTest only - Single tenant)**。这对应于旧版应用注册选项。 + + :::note + + 在更新åŽçš„ Azure 门户中，Redirect URI 与 Reply URL çš„æ„æ€ç›¸åŒã€‚为了将 Azure AD 与 Rancher 一起使用，你必须将 Rancher 列入 Azure 白åå•ï¼ˆä¹‹å‰é€šè¿‡ Reply URL 完æˆï¼‰ã€‚因此，请确ä¿ä½¿ç”¨ä½ çš„ Rancher Server URL 填写 Redirect URI，以包å«ä¸‹é¢åˆ—出的验è¯è·¯å¾„。 + + ::: + + 1. 在 [**Redirect URI**](https://docs.microsoft.com/en-us/azure/active-directory/develop/reply-url) 中，确ä¿ä»Žä¸‹æ‹‰åˆ—表中选择 **Web**，并在æ—边的文本框中输入 Rancher Server çš„ URL。Rancher Server URL åŽéœ€è¦è¿½åŠ éªŒè¯è·¯å¾„，例如 `/verify-auth-azure`。 + + :::tip + + ä½ å¯ä»¥åœ¨ Azure AD 身份验è¯é¡µé¢ï¼ˆå…¨å±€è§†å›¾ > Authentication > Web）中找到你 Rancher 中的个性化 Azure Redirect URI（Reply URL）。 + + ::: + + 1. å•å‡» **Register**。 + +:::note + +此更改å¯èƒ½éœ€è¦äº”分钟æ‰èƒ½ç”Ÿæ•ˆã€‚因此，如果在é…ç½® Azure AD 之åŽæ— æ³•ç«‹å³è¿›è¡Œèº«ä»½éªŒè¯ï¼Œè¯·ä¸è¦æƒŠæ…Œã€‚ + +::: + +#### 2. 创建客户端密文 + +在 Azure 门户中，创建一个客户端密文。Rancher å°†ä½¿ç”¨æ­¤å¯†é’¥å‘ Azure AD 进行身份验è¯ã€‚ + +1. 使用æœç´¢åŠŸèƒ½æ‰“å¼€ **App registrations** æœåŠ¡ã€‚然åŽæ‰“开你在上一个步骤中创建的 Rancher 项。 + + ![Open Rancher Registration](/img/open-rancher-app-reg.png) + +1. 在导航窗格中，å•å‡» **Certificates & secrets**。 + +1. å•å‡» **New client secret**。 + ![创建新的客户端密文](/img/new-client-secret.png) +1. 输入 **Description**（例如 `Rancher`）。 +1. 从 **Expires** 下的选项中选择æŒç»­æ—¶é—´ã€‚此下拉èœå•è®¾ç½®çš„是密钥的到期日期。日期越短则越安全，但需è¦ä½ æ›´é¢‘ç¹åœ°åˆ›å»ºæ–°å¯†é’¥ã€‚ + 请注æ„ï¼Œå¦‚æžœæ£€æµ‹åˆ°åº”ç”¨ç¨‹åº Secret 已过期，用户将无法登录 Rancher。为é¿å…此问题，请在 Azure ä¸­è½®æ¢ Secret 并在过期å‰åœ¨ Rancher 中更新它。 +1. å•å‡» **Add**（无需输入值，ä¿å­˜åŽä¼šè‡ªåŠ¨å¡«å……）。 + + +1. ç¨åŽä½ å°†åœ¨ Rancher UI 中输入此密钥作为你的 **Application Secret**。由于你将无法在 Azure UI 中å†æ¬¡è®¿é—®é”®å€¼ï¼Œå› æ­¤è¯·åœ¨å…¶ä½™è®¾ç½®è¿‡ç¨‹ä¸­ä¿æŒæ‰“开此窗å£ã€‚ + +#### 3. 设置 Rancher 所需的æƒé™ + +接下æ¥ï¼Œåœ¨ Azure 中为 Rancher 设置 API æƒé™ã€‚ + +:::caution + +ç¡®ä¿ä½ è®¾ç½®äº† Application æƒé™ï¼Œè€Œ*ä¸æ˜¯* Delegated æƒé™ã€‚å¦åˆ™ï¼Œä½ å°†æ— æ³•ç™»å½• Azure AD。 + +::: + +1. 在导航窗格中，选择 **API permissions**。 + +1. å•å‡» **Add a permission**。 + +1. 从 Microsoft Graph API 中，选择以下 **Application Permissions**: `Directory.Read.All`。 + + ![选择 API æƒé™](/img/api-permissions.png) + +1. 返回导航æ ä¸­çš„ **API permissions**。在那里，å•å‡» **Grant admin consent**。然åŽå•å‡» **Yes**。该应用程åºçš„æƒé™åº”如下所示： + +![Open Required Permissions](/img/select-req-permissions.png) + +:::note + +Rancher ä¸ä¼šéªŒè¯ä½ æŽˆäºˆ Azure 应用程åºçš„æƒé™ã€‚ä½ å¯ä»¥è‡ªç”±ä½¿ç”¨ä»»ä½•ä½ æ‰€éœ€çš„æƒé™ï¼Œåªè¦è¿™äº›æƒé™å…许 Rancher 使用 AD 用户和组。 + +具体æ¥è¯´ï¼ŒRancher 需è¦å…许以下æ“作的æƒé™ï¼š +- 获å–一个用户。 +- 列出所有用户。 +- 列出给定用户所属的组。 +- 获å–一个组。 +- 列出所有组。 + +Rancher 执行这些æ“作æ¥ç™»å½•ç”¨æˆ·æˆ–æœç´¢ç”¨æˆ·/组。请记ä½ï¼Œæƒé™å¿…须是 `Application` 类型。 + +下é¢æ˜¯å‡ ä¸ªæ»¡è¶³ Rancher 需求的æƒé™ç»„åˆç¤ºä¾‹ï¼š +- `Directory.Read.All` +- `User.Read.All` å’Œ `GroupMember.Read.All` +- `User.Read.All` å’Œ `Group.Read.All` + +::: + +#### 4. å¤åˆ¶ Azure åº”ç”¨æ•°æ® + +![Application ID](/img/app-configuration.png) + +1. 获å–ä½ çš„ Rancher **租户 ID**。 + + 1. 使用æœç´¢æ‰“å¼€ **App registrations**。 + + 1. 找到你为 Rancher 创建的项。 + + 1. å¤åˆ¶ **Directory ID** 并将其作为 **Tenant ID** 粘贴到 Rancher 中。 + +1. 获å–ä½ çš„ Rancher **Application (Client) ID**。 + + 1. 如果你还未在该ä½ç½®ï¼Œè¯·ä½¿ç”¨æœç´¢æ‰“å¼€ **App registrations**。 + + 1. 在 **Overview**中，找到你为 Rancher 创建的æ¡ç›®ã€‚ + + 1. å¤åˆ¶ **Application (Client) ID** 并将其作为 **Application ID** 粘贴到 Rancher 中。 + +1. 你的端点选项通常是 [Standard](#global) 或 [China](#中国)。对于这两个选项，你åªéœ€è¦è¾“å…¥ **Tenant ID**ã€**Application ID** å’Œ **Application Secret**。 + +![标准端点选项](/img/tenant-application-id-secret.png) + +**对于自定义端点**： + +:::caution + +Rancher 未测试也未完全支æŒè‡ªå®šä¹‰ç«¯ç‚¹ã€‚ + +::: + +你还需è¦æ‰‹åŠ¨è¾“å…¥ Graphã€Token å’Œ Auth Endpoints。 + +- 从 App registrations 中，点击 Endpoints： + +![点击端点](/img/endpoints.png) + +- 以下端点将是你的 Rancher 端点值。请使用这些端点的 v1 版本。 + - **Microsoft Graph API endpoint**（Graph 端点） + - **OAuth 2.0 token endpoint (v1)**（Token 端点） + - **OAuth 2.0 authorization endpoint (v1)** (Auth 端点) + +#### 5. 在 Rancher 中é…ç½® Azure AD + +è¦å®Œæˆé…置，请在 Rancher UI 中输入你的 AD 实例信æ¯ã€‚ + +1. 登录到 Rancher。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 + +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 + +1. å•å‡» **AzureAD**。 + +1. 使用你在[å¤åˆ¶ Azure 应用数æ®](#4-å¤åˆ¶-azure-应用数æ®)æ—¶å¤åˆ¶çš„ä¿¡æ¯ï¼Œå¡«å†™**é…ç½® Azure AD è´¦å·**的表å•ã€‚ + + :::caution + + Azure AD å¸æˆ·å°†è¢«æŽˆäºˆç®¡ç†å‘˜æƒé™ï¼Œå› ä¸ºå…¶è¯¦ç»†ä¿¡æ¯å°†æ˜ å°„到 Rancher 本地主体å¸æˆ·ã€‚在继续之å‰ç¡®ä¿æ­¤æƒé™çº§åˆ«æ˜¯é€‚当的。 + + ::: + + **对于标准或中国端点：** + + 下表介ç»äº†ä½ åœ¨ Azure 门户中å¤åˆ¶çš„值与 Rancher 中字段的映射： + + | Rancher 字段 | Azure 值 | + | ------------------ | ------------------------------------- | + | 租户 ID | Directory ID | + | Application ID | Application ID | + | 应用密文 | Key Value | + | 端点 | https://login.microsoftonline.com/ | + + **对于自定义端点**： + + 下表将你的自定义é…置值映射到 Rancher 字段： + + | Rancher 字段 | Azure 值 | + | ------------------ | ------------------------------------- | + | Graph 端点 | Microsoft Graph API Endpoint | + | Token 端点 | OAuth 2.0 Token Endpoint | + | Auth 端点 | OAuth 2.0 Authorization Endpoint | + + **é‡è¦æ示**：在自定义é…置中输入 Graph Endpoint 时，请从 URL 中删除 Tenant ID： + + https://graph.microsoft.com/abb5adde-bee8-4821-8b03-e63efdc7701c + +1. 点击**å¯ç”¨**。 + +**结果**：Azure Active Directory 身份验è¯å·²é…置。 + + +### 从 Azure AD Graph API è¿ç§»åˆ° Microsoft Graph API + +由于 [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) 已弃用并计划于 2023 å¹´ 6 月åœç”¨ï¼Œç®¡ç†å‘˜åº”更新他们的 Azure AD 应用程åºä»¥åœ¨ Rancher 中使用 [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api)。 +你需è¦åœ¨ç«¯ç‚¹å¼ƒç”¨ä¹‹å‰å®Œæˆæ“作。 +如果在åœç”¨åŽ Rancher ä»é…置为使用 Azure AD Graph API，用户å¯èƒ½æ— æ³•ä½¿ç”¨ Azure AD 登录 Rancher。 + +#### 在 Rancher UI 中更新端点 + +:::caution + +管ç†å‘˜éœ€è¦åœ¨è¿ç§»ä¸‹è¿°ç«¯ç‚¹ä¹‹å‰åˆ›å»ºä¸€ä¸ª [Rancher 备份](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)。 + +::: + +1. [æ›´æ–°](#3-设置-rancher-所需的æƒé™) Azure AD 应用程åºæ³¨å†Œçš„æƒé™ã€‚这个步骤éžå¸¸å…³é”®ã€‚ + +1. 登录到 Rancher。 + +1. 在 Rancher UI 主页中，记下å±å¹•é¡¶éƒ¨çš„横幅，该横幅建议用户更新 Azure AD 身份验è¯ã€‚å•å‡»æ供的链接以执行此æ“作。 + + ![Rancher UI 横幅](/img/rancher-ui-azure-update.png) + +1. è¦å®Œæˆæ–°çš„ Microsoft Graph API è¿ç§»ï¼Œè¯·å•å‡» **Update Endpoint**。 + + **注æ„**：在开始更新之å‰ï¼Œè¯·ç¡®ä¿ä½ çš„ Azure 应用程åºå…·æœ‰[æ–°çš„æƒé™é›†](#3-设置-rancher-所需的æƒé™)。 + + ![更新端点](/img/rancher-button-to-update.png) + +1. 在收到弹出警告消æ¯æ—¶ï¼Œå•å‡» **Update**： + + ![Azure 更新弹出窗å£](/img/azure-update-popup.png) + +1. 有关 Rancher 执行的完整端点更改，请å‚阅下é¢çš„[表格](#global)。管ç†å‘˜ä¸éœ€è¦æ‰‹åŠ¨æ‰§è¡Œæ­¤æ“作。 + +#### 离线环境 + +在离线环境中，由于 Graph Endpoint URL 正在更改，因此管ç†å‘˜éœ€è¦ç¡®ä¿å…¶ç«¯ç‚¹è¢«[列入白åå•](#3.2)。 + +#### 回滚è¿ç§» + +如果你需è¦å›žæ»šè¿ç§»ï¼Œè¯·æ³¨æ„以下事项： + +1. 如果管ç†å‘˜æƒ³è¦å›žæ»šï¼Œæˆ‘们建议他们使用正确的æ¢å¤æµç¨‹ã€‚有关å‚考信æ¯ï¼Œè¯·å‚阅[备份文档](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)ã€[æ¢å¤æ–‡æ¡£](../../../new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md)å’Œ[示例](../../../../reference-guides/backup-restore-configuration/examples.md)。 + +1. 如果 Azure 应用程åºæ‰€æœ‰è€…想è¦è½®æ¢åº”用程åºå¯†é’¥ï¼Œä»–们也需è¦åœ¨ Rancher 中进行轮æ¢ï¼ˆå› ä¸ºåœ¨ Azure 中更改应用程åºå¯†é’¥æ—¶ï¼ŒRancher ä¸ä¼šè‡ªåŠ¨æ›´æ–°åº”用程åºå¯†é’¥ï¼‰ã€‚在 Rancher 中，它存储在å为 `azureadconfig-applicationsecret` çš„ Kubernetes 密文中，该密文ä½äºŽ `cattle-global-data` 命å空间中。 + +:::caution + +如果你使用现有的 Azure AD 设置å‡çº§åˆ° Rancher v2.7.0+，并选择了ç¦ç”¨è®¤è¯æ供程åºï¼Œä½ å°†æ— æ³•æ¢å¤ä»¥å‰çš„设置。你也无法使用旧æµç¨‹è®¾ç½® Azure AD。你需è¦ä½¿ç”¨æ–°çš„认è¯æµç¨‹é‡æ–°æ³¨å†Œã€‚由于 Rancher 现在使用 Graph API，因此用户需è¦[在 Azure 门户中设置适当的æƒé™](#3-设置-rancher-所需的æƒé™)。 + +::: + + +#### Global: + +| Rancher 字段 | 已弃用的端点 | +---------------- | ------------------------------------------------------------- +| Auth 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/authorize | +| 端点 | https://login.microsoftonline.com/ | +| Graph 端点 | https://graph.windows.net/ | +| Token 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/token | + +| Rancher 字段 | 新端点 | +---------------- | ------------------------------------------------------------------ +| Auth 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/authorize | +| 端点 | https://login.microsoftonline.com/ | +| Graph 端点 | https://graph.microsoft.com | +| Token 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/token | + +#### 中国： + +| Rancher 字段 | 已弃用的端点 | +---------------- | ---------------------------------------------------------- +| Auth 端点 | https://login.chinacloudapi.cn/{tenantID}/oauth2/authorize | +| 端点 | https://login.chinacloudapi.cn/ | +| Graph 端点 | https://graph.chinacloudapi.cn/ | +| Token 端点 | https://login.chinacloudapi.cn/{tenantID}/oauth2/token | + +| Rancher 字段 | 新端点 | +---------------- | ------------------------------------------------------------------------- +| Auth 端点 | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/authorize | +| 端点 | https://login.partner.microsoftonline.cn/ | +| Graph 端点 | https://microsoftgraph.chinacloudapi.cn | +| Token 端点 | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/token | + + +## 已弃用的 Azure AD Graph API + +> **é‡è¦æ示**： +> +> - [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) 已被弃用，Microsoft 将在 2023 å¹´ 6 月 30 æ—¥åŽéšæ—¶åœç”¨å®ƒä¸”ä¸ä¼šå¦è¡Œé€šçŸ¥ã€‚我们将更新我们的文档，以便在åœç”¨æ—¶å‘社区æ供建议。Rancher 现在使用 [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api) æ¥å°† Azure AD 设置为外部身份验è¯æ供程åºã€‚ +> +> +> - 如果你是新用户或希望进行è¿ç§»ï¼Œè¯·å‚阅新的æµç¨‹è¯´æ˜Žï¼š Rancher v2.7.0+。 +> +> +> - 如果你ä¸æƒ³åœ¨ Azure AD Graph API åœç”¨åŽå‡çº§åˆ° v2.7.0+，你需è¦ï¼š +> - 使用内置的 Rancher 身份认è¯ï¼Œæˆ–者 +> - 使用å¦ä¸€ä¸ªç¬¬ä¸‰æ–¹èº«ä»½è®¤è¯ç³»ç»Ÿå¹¶åœ¨ Rancher 中进行设置。请å‚阅[身份验è¯æ–‡æ¡£](../../../../pages-for-subheaders/authentication-config.md)，了解如何é…置其他开放å¼èº«ä»½éªŒè¯æ供程åºã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md new file mode 100644 index 00000000000..6f3100e462c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md @@ -0,0 +1,60 @@ +--- +title: é…ç½® FreeIPA +--- + +如果你的组织使用 FreeIPA 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许你的用户使用 FreeIPA 凭è¯ç™»å½•ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +- ä½ å¿…é¡»é…置了 [FreeIPA æœåŠ¡å™¨](https://www.freeipa.org/)。 +- 在 FreeIPA 中创建一个具有 `read-only` 访问æƒé™çš„ ServiceAccount 。当用户使用 API​​ 密钥å‘出请求时，Rancher 使用此账å·æ¥éªŒè¯ç»„æˆå‘˜èº«ä»½ã€‚ +- å‚è§[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + +::: + +1. 使用分é…了 `administrator` è§’è‰²ï¼ˆå³ _本地主体_）的本地用户登录到 Rancher。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **FreeIPA**。 +1. 填写**é…ç½® FreeIPA æœåŠ¡å™¨**表å•ï¼Œ + + ä½ å¯èƒ½éœ€è¦ç™»å½•åˆ°åŸŸæŽ§åˆ¶å™¨ï¼Œæ¥æŸ¥æ‰¾è¡¨å•ä¸­è¯·æ±‚çš„ä¿¡æ¯ã€‚ + + :::note 使用 TLS？ + + 如果è¯ä¹¦æ˜¯è‡ªç­¾å，或者ä¸æ˜¯æ¥è‡ªå…¬è®¤çš„è¯ä¹¦é¢å‘机构的，请确ä¿æ供完整的è¯ä¹¦é“¾ã€‚Rancher 需è¦è¯¥é“¾æ¥éªŒè¯æœåŠ¡å™¨çš„è¯ä¹¦ã€‚ + + ::: + + :::note 用户æœç´¢åº“ vs. 组æœç´¢åº“ + + æœç´¢åº“使 Rancher å¯ä»¥æœç´¢ FreeIPA 中的用户和组。这些字段仅用于æœç´¢åº“，ä¸é€‚用于æœç´¢ç­›é€‰å™¨ã€‚ + + * 如果你的用户和组ä½äºŽåŒä¸€æœç´¢åº“中，则仅填写用户æœç´¢åº“。 + * 如果你的组ä½äºŽå…¶ä»–æœç´¢åº“中，则å¯ä»¥é€‰æ‹©å¡«å†™ç»„æœç´¢åº“。该字段专用于æœç´¢ç»„，但ä¸æ˜¯å¿…需的。 + + ::: + +1. 如果你的 FreeIPA ä¸åŒäºŽæ ‡å‡†çš„ AD Schema，则必须完æˆ**自定义 Schema** 部分实现匹é…。å¦åˆ™ï¼Œè°ƒè¿‡æ­¤æ­¥éª¤ã€‚ + + :::note æœç´¢å±žæ€§ + + `æœç´¢å±žæ€§`字段的默认值为三个特定值：`uid|sn|givenName`。é…ç½® FreeIPA åŽï¼Œå½“用户输入文本以添加用户或组时，Rancher 会自动查询 FreeIPA æœåŠ¡å™¨ï¼Œå¹¶å°è¯•æŒ‰ç”¨æˆ· ID，姓æ°æˆ–åå­—æ¥åŒ¹é…字段。Rancher 专门æœç´¢ä»¥åœ¨æœç´¢å­—段中输入的文本开头的用户/组。 + + 默认字段值为 `uid|sn|givenName`，但是你å¯ä»¥å°†æ­¤å­—段é…置为这些字段的å­é›†ã€‚管é“符 (`|`) 用于分隔å„个字段。 + + * `uid`：用户 ID + * `sn`：姓 + * `givenName`：å + + Rancher 使用此æœç´¢å±žæ€§ä¸ºç”¨æˆ·å’Œç»„创建æœç´¢ç­›é€‰å™¨ï¼Œä½†æ˜¯ä½ *ä¸èƒ½*在此字段中添加自己的æœç´¢ç­›é€‰å™¨ã€‚ + + ::: + +1. 在 **Authenticate with FreeIPA** 中输入你的 FreeIPA 用户å和密ç ï¼Œç¡®è®¤å·²ä¸º Rancher é…ç½® FreeIPA 身份验è¯ã€‚ +1. 点击**å¯ç”¨**。 + +**结果**： + +- FreeIPA 验è¯é…ç½®æˆåŠŸã€‚ +- 你将使用你的 FreeIPA è´¦å·ï¼ˆå³ _外部主体_）登录到 Rancher。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md new file mode 100644 index 00000000000..e903421611d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md @@ -0,0 +1,56 @@ +--- +title: é…ç½® GitHub +--- + +在使用 GitHub 的环境中，你å¯ä»¥é€šè¿‡é…ç½® Rancher å…许用户使用 GitHub 凭è¯ç™»å½•ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +å‚è§[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + +::: + +1. 使用分é…了 `administrator` è§’è‰²ï¼ˆå³ _本地主体_）的本地用户登录到 Rancher。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **GitHub**。 +1. 按照显示的说明设置 GitHub 应用。Rancher 会将你é‡å®šå‘到 GitHub 完æˆæ³¨å†Œã€‚ + + :::note 什么是授æƒå›žè°ƒ URL？ + + 授æƒå›žè°ƒ URL 是用户开始使用你的应用（å³åˆå§‹å±å¹•ï¼‰çš„ URL。 + + 使用外部身份验è¯æ—¶ï¼Œå®žé™…上ä¸ä¼šåœ¨ä½ çš„应用中进行身份验è¯ã€‚相å，身份验è¯åœ¨å¤–部进行（在本例中为 GitHub）。在外部身份验è¯æˆåŠŸå®ŒæˆåŽï¼Œç”¨æˆ·å°†é€šè¿‡æŽˆæƒå›žè°ƒ URL é‡æ–°è¿›å…¥åº”用。 + + ::: + +1. 从 GitHub å¤åˆ¶ **Client ID** å’Œ **Client Secret**。将它们粘贴到 Rancher 中。 + + :::note 在哪里å¯ä»¥æ‰¾åˆ° Client ID å’Œ Client Secret？ + + 在 GitHub 中，选择 **Settings > Developer Settings > OAuth Apps**。你å¯ä»¥åœ¨æ­¤å¤„找到 Client ID å’Œ Client Secret。 + + ::: + +1. å•å‡»**使用 GitHub 认è¯**。 + +1. 使用 **Site Access** 选项æ¥é…置用户授æƒçš„范围。 + + - **å…许任何有效用户** + + _任何_ GitHub 用户都能访问 Rancher。通常ä¸å»ºè®®ä½¿ç”¨æ­¤è®¾ç½®ã€‚ + + - **å…许集群和项目æˆå‘˜ï¼Œä»¥åŠæŽˆæƒçš„用户和组织** + + 添加为**集群æˆå‘˜**或**项目æˆå‘˜**的任何 GitHub 用户或组都å¯ä»¥ç™»å½• Rancher。此外，任何添加到**授æƒç”¨æˆ·å’Œç»„织**列表中的 GitHub 用户和组都能登录到 Rancher。 + + - **ä»…å…许授æƒç”¨æˆ·å’Œç»„织** + + åªæœ‰æ·»åŠ åˆ°**授æƒç”¨æˆ·å’Œç»„织**çš„ GitHub 用户和组能登录 Rancher。 +
+1. 点击**å¯ç”¨**。 + +**结果**： + +- GitHub 验è¯é…ç½®æˆåŠŸã€‚ +- 你将使用你的 GitHub è´¦å·ï¼ˆå³ _外部主体_）登录到 Rancher。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md new file mode 100644 index 00000000000..ca4d522ba35 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md @@ -0,0 +1,111 @@ +--- +title: é…ç½® Google OAuth +--- + +如果你的组织使用 G Suite 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许你的用户使用 G Suite 凭è¯ç™»å½•ã€‚ + +åªæœ‰ G Suite 域的管ç†å‘˜æ‰èƒ½è®¿é—® Admin SDK。因此，åªæœ‰ G Suite 管ç†å‘˜å¯ä»¥é…ç½® Rancher çš„ Google OAuth。 + +在 Rancher 中，åªæœ‰å…·æœ‰ **Manage Authentication** [全局角色](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)的管ç†å‘˜æˆ–用户æ‰èƒ½é…置身份验è¯ã€‚ + +## 先决æ¡ä»¶ + +- ä½ å¿…é¡»é…置了 [G Suite 管ç†å‘˜è´¦å·](https://admin.google.com)。 +- G Suite 需è¦ä¸€ä¸ª[顶级ç§æœ‰åŸŸ FQDN](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains) 作为授æƒåŸŸã€‚èŽ·å– FQDN 的一ç§æ–¹æ³•æ˜¯åœ¨ Route53 中为 Rancher Server 创建 A 记录。你ä¸éœ€è¦ä½¿ç”¨è¯¥è®°å½•æ›´æ–° Rancher Server çš„ URL 设置，因为å¯èƒ½æœ‰é›†ç¾¤ä½¿ç”¨è¯¥ URL。 +- ä½ çš„ G Suite 域必须å¯ç”¨äº† Admin SDK API。你å¯ä»¥æŒ‰ç…§[此页é¢](https://support.google.com/a/answer/60757?hl=en)中的步骤å¯ç”¨å®ƒã€‚ + +å¯ç”¨ Admin SDK API åŽï¼Œä½ çš„ G Suite 域的 API 页é¢åº”如下图所示： +![Enable Admin APIs](/img/Google-Enable-APIs-Screen.png) + +## 在 Rancher 中为 OAuth 设置 G Suite + +在 Rancher 中设置 Google OAuth 之å‰ï¼Œä½ éœ€è¦ç™»å½•åˆ°ä½ çš„ G Suite è´¦å·å¹¶å®Œæˆä»¥ä¸‹è®¾ç½®ï¼š + +1. [在 G Suite 中将 Rancher 添加为授æƒåŸŸ](#1-å°†-rancher-添加为授æƒåŸŸ) +1. [为 Rancher Server ç”Ÿæˆ OAuth2 凭è¯](#2-为-rancher-server-生æˆ-oauth2-凭è¯) +1. [为 Rancher Server 创建 ServiceAccount 凭è¯](#3-创建-serviceaccount-凭è¯) +1. [å°† ServiceAccount å¯†é’¥æ³¨å†Œæˆ OAuth Client](#4-å°†-serviceaccount-密钥注册æˆ-oauth-client) + +### 1. å°† Rancher 添加为授æƒåŸŸ + +1. 点击[此处](https://console.developers.google.com/apis/credentials)å‰å¾€ä½ çš„ Google 域的凭è¯é¡µé¢ã€‚ +1. 选择你的项目，然åŽç‚¹å‡» **OAuth consent screen**。 + ![OAuth Consent Screen](/img/Google-OAuth-consent-screen-tab.png) +1. å‰å¾€ **Authorized Domains**，并在列表中输入你的 Rancher Server URL 的顶级ç§æœ‰åŸŸã€‚顶级ç§æœ‰åŸŸæ˜¯æœ€å³è¾¹çš„超级域。例如，`www.foo.co.uk` 是 `foo.co.uk` 的顶级ç§æœ‰åŸŸã€‚有关顶级ç§æœ‰åŸŸçš„更多信æ¯ï¼Œè¯·å‚è§[此处](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains)。 +1. å‰å¾€ **Scopes for Google APIs**，并确ä¿å·²å¯ç”¨ **email**，**profile** å’Œ **openid**。 + +**结果**：Rancher 已被添加为 Admin SDK API 的授æƒåŸŸã€‚ + +### 2. 为 Rancher Server ç”Ÿæˆ OAuth2 å‡­è¯ + +1. å‰å¾€ Google API 控制å°ï¼Œé€‰æ‹©ä½ çš„项目并å‰å¾€ [credentials ](https://console.developers.google.com/apis/credentials)页é¢ã€‚ + ![Credentials](/img/Google-Credentials-tab.png) +1. 在 **Create Credentials** 下拉框中，选择 **OAuth client ID**。 +1. 点击 **Web application**。 +1. 输入一个å称。 +1. 填写 **Authorized JavaScript origins** å’Œ **Authorized redirect URIs**。请注æ„，设置 Google OAuth çš„ Rancher UI 页é¢ï¼ˆ**Security > Authentication > Google** 下的全局视图）为你æ供了这一步è¦è¾“入的准确链接。 +- 在 **Authorized JavaScript origins** 处，输入你的 Rancher Server URL。 +- 在 **Authorized redirect URIs** 处，输入你的 Rancher Server çš„ URL 并附加路径 `verify-auth`。例如，如果你的 URI 是 `https://rancherServer`，你需è¦è¾“å…¥ `https://rancherServer/verify-auth`。 +1. 点击 **Create**。 +1. 创建凭è¯ä¹‹åŽï¼Œä½ å°†çœ‹åˆ°ä¸€ä¸ªé¡µé¢ï¼Œå…¶ä¸­æ˜¾ç¤ºä½ çš„凭è¯åˆ—表。选择刚刚创建的凭è¯ï¼Œç„¶åŽåœ¨æœ€å³è¾¹çš„行中å•å‡» **Download JSON**。ä¿å­˜è¯¥æ–‡ä»¶ï¼Œä»¥ä¾¿å‘ Rancher æ供这些凭è¯ã€‚ + +**结果**：你已æˆåŠŸåˆ›å»º OAuth 凭è¯ã€‚ + +### 3. 创建 ServiceAccount å‡­è¯ +由于 Google Admin SDK åªå¯¹ç®¡ç†å‘˜å¯ç”¨ï¼Œæ™®é€šç”¨æˆ·ä¸èƒ½ä½¿ç”¨å®ƒæ¥æ£€ç´¢å…¶ä»–用户或其组的é…置文件。普通用户甚至ä¸èƒ½æ£€ç´¢ä»–们自己的组。 + +由于 Rancher æ供基于组的æˆå‘˜è®¿é—®ï¼Œæˆ‘们è¦æ±‚用户能够获得自己的组，并在需è¦æ—¶æŸ¥æ‰¾å…¶ä»–用户和组。 + +为了解决这个问题，G Suite 建议创建一个 ServiceAccount，并将你的 G Suite 域的æƒé™å§”托给该 ServiceAccount。 + +本节介ç»å¦‚何： + +- 创建一个 ServiceAccount +- 为 ServiceAccount 创建一个密钥并下载 JSON æ ¼å¼çš„å‡­è¯ + +1. 点击[此处](https://console.developers.google.com/iam-admin/serviceaccounts)并选择è¦ç”Ÿæˆ OAuth 凭è¯çš„项目。 +1. 点击 **Create Service Account**。 +1. 输入å称，并点击 **Create**。 + ![Service account creation Step 1](/img/Google-svc-acc-step1.png) +1. ä¸è¦åœ¨ **Service account permissions** 页é¢è®¾ç½®ä»»ä½•è§’色，然åŽå•å‡» **Continue**。 + ![Service account creation Step 2](/img/Google-svc-acc-step2.png) +1. 点击 **Create Key** 并选择 JSON 选项。下载并ä¿å­˜ JSON 文件，以便你å¯ä»¥å°†å…¶ä½œä¸º ServiceAccount 凭è¯æ供给 Rancher。 + ![Service account creation Step 3](/img/Google-svc-acc-step3-key-creation.png) + +**结果**：你的 ServiceAccount 已创建æˆåŠŸã€‚ + +### 4. å°† ServiceAccount å¯†é’¥æ³¨å†Œæˆ OAuth Client + +你需è¦ä¸ºåœ¨ä¸Šä¸€æ­¥ä¸­åˆ›å»ºçš„ ServiceAccount 授予一些æƒé™ã€‚Rancher ä»…è¦æ±‚为用户和组授予åªè¯»æƒé™ã€‚ + +使用 ServiceAccount 密钥的唯一 ID，按照以下步骤将其注册为 Oauth Client： + +1. 获å–你刚刚创建的密钥的唯一 ID。如果它没有显示在你创建的键æ—边的键列表中，你需è¦å…ˆå¯ç”¨ Unique ID 列。点击 **Unique ID** 然åŽç‚¹å‡» **OK**ã€‚è¿™å°†å‘ ServiceAccount 密钥列表中添加 **Unique ID** 列。ä¿å­˜ä½ åˆ›å»ºçš„ ServiceAccount 对应的唯一 ID。注æ„：这是一个数字 Key，ä¸è¦ä¸Žå­—æ¯æ•°å­—字段 **Key ID** 混淆。 + + ![Service account Unique ID](/img/Google-Select-UniqueID-column.png) +1. å‰å¾€ [**Domain-wide Delegation** 页é¢ã€‚](https://admin.google.com/ac/owl/domainwidedelegation) +1. 在 **Client Name** 字段中添加上一步中获得的唯一 ID。 +1. 在 **One or More API Scopes** 字段中，添加以下作用域： + ``` + openid,profile,email,https://www.googleapis.com/auth/admin.directory.user.readonly,https://www.googleapis.com/auth/admin.directory.group.readonly + ``` +1. 点击 **Authorize**。 + +**结果**：ServiceAccount 在你的 G Suite è´¦å·ä¸­å·²æ³¨å†Œä¸º OAuth 客户端。 + +## 在 Rancher 中é…ç½® Google OAuth + +1. 使用分é…了 [administrator](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) 角色的本地用户登录到 Rancher。这个用户也称为本地主体。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Google**。UI 中的说明介ç»äº†ä½¿ç”¨ Google OAuth 设置身份验è¯çš„步骤。 + 1. 管ç†å‘˜é‚®ç®±ï¼šæä¾› GSuite 设置中的管ç†å‘˜è´¦æˆ·çš„电å­é‚®ç®±ã€‚为了查找用户和组，Google API 需è¦ç®¡ç†å‘˜çš„电å­é‚®ä»¶å’Œ ServiceAccount 密钥。 + 1. 域å：æä¾›é…置了 G Suite 的域。请æ供准确的域，而ä¸æ˜¯åˆ«å。 + 1. 属于多个用户组的用户：选中此框以å¯ç”¨åµŒå¥—组æˆå‘˜å…³ç³»ã€‚Rancher 管ç†å‘˜å¯ä»¥åœ¨é…置认è¯åŽçš„任何时候ç¦ç”¨å®ƒã€‚ + - **步骤一**是将 Rancher 添加为授æƒåŸŸï¼ˆè¯¦æƒ…请å‚è§[本节](#1-å°†-rancher-添加为授æƒåŸŸ)）。 + - **步骤二**æ供你完æˆ[本节](#2-为-rancher-server-生æˆ-oauth2-凭è¯)åŽä¸‹è½½çš„ OAuth å‡­è¯ JSON。你å¯ä»¥ä¸Šä¼ æ–‡ä»¶æˆ–将内容粘贴到 **OAuth Credentials** 字段。 + - **步骤三**æ供在[本节](#3-创建-serviceaccount-凭è¯)末尾下载的 ServiceAccount å‡­è¯ JSON。仅当你æˆåŠŸ[在 G Suite è´¦å·ä¸­å°† ServiceAccount 密钥注册为 OAuth Client](#4-å°†-serviceaccount-密钥注册æˆ-oauth-client) åŽï¼Œå‡­è¯æ‰èƒ½æ­£å¸¸å·¥ä½œã€‚ +1. 点击**使用 Google 认è¯**。 +1. 点击**å¯ç”¨**。 + +**结果**：Google 验è¯é…ç½®æˆåŠŸã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md new file mode 100644 index 00000000000..5516672eb87 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md @@ -0,0 +1,145 @@ +--- +title: é…ç½® Keycloak (OIDC) +description: 创建 Keycloak OpenID Connect (OIDC) 客户端并é…ç½® Rancher 以使用 Keycloak。你的用户将能够使用他们的 Keycloak 登录å登录 Rancher。 +--- + +如果你的组织使用 [Keycloak Identity Provider (IdP)](https://www.keycloak.org) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚Rancher 支æŒä½¿ç”¨ OpenID Connect (OIDC) å议和 SAML åè®®æ¥é›†æˆ Keycloak。与 Rancher 一起使用时，这两ç§å®žçŽ°åœ¨åŠŸèƒ½ä¸Šæ˜¯ç­‰æ•ˆçš„。本文æ述了é…ç½® Rancher 以通过 OIDC å议与 Keycloak 一起使用的æµç¨‹ã€‚ + +如果你更喜欢将 Keycloak 与 SAML å议一起使用，请å‚è§[此页é¢](configure-keycloak-saml.md)。 + +如果你有使用 SAML å议的现有é…置并希望切æ¢åˆ° OIDC å议，请å‚è§[本节](#从-saml-è¿ç§»åˆ°-oidc)。 + +## 先决æ¡ä»¶ + +- 已在 Rancher 上ç¦ç”¨ Keycloak (SAML)。 +- ä½ å¿…é¡»é…置了 [Keycloak IdP æœåŠ¡å™¨](https://www.keycloak.org/docs/latest/server_installation/)。 +- 在 Keycloak 中，使用以下设置创建一个[æ–°çš„ OIDC 客户端](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients)。如需获å–帮助，请å‚è§ [Keycloak 文档](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients)。 + + | 设置 | 值 | + ------------|------------ + | `Client ID` | <CLIENT_ID> (例如 `rancher`) | + | `Name` | <CLIENT_NAME> (例如 `rancher`) | + | `Client Protocol` | `openid-connect` | + | `Access Type` | `confidential` | + | `Valid Redirect URI` | `https://yourRancherHostURL/verify-auth` | + +- 在新的 OIDC 客户端中，创建 [Mappers](https://www.keycloak.org/docs/latest/server_admin/#_protocol-mappers) æ¥å…¬å¼€ç”¨æˆ·å­—段。 + - 使用以下设置创建一个新的 "Groups Mapper"： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Groups Mapper` | + | `Mapper Type` | `Group Membership` | + | `Token Claim Name` | `groups` | + | `Add to ID token` | `OFF` | + | `Add to access token` | `OFF` | + | `Add to user info` | `ON` | + + - 使用以下设置创建一个新的 "Client Audience" ： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Client Audience` | + | `Mapper Type` | `Audience` | + | `Included Client Audience` | <CLIENT_NAME> | + | `Add to access token` | `ON` | + + - 使用以下设置创建一个新的 "Groups Path"： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Group Path` | + | `Mapper Type` | `Group Membership` | + | `Token Claim Name` | `full_group_path` | + | `Full group path` | `ON` | + | `Add to user info` | `ON` | + +## 在 Rancher 中é…ç½® Keycloak + +1. 在 Rancher UI 中，å•å‡» **☰ > 用户 & 认è¯**。 +1. å•å‡»å·¦ä¾§å¯¼èˆªæ çš„**认è¯**。 +1. 选择 **Keycloak (OIDC)**。 +1. 填写**é…ç½® Keycloak OIDC è´¦å·**表å•ã€‚有关填写表å•çš„帮助，请å‚è§[é…ç½®å‚考](#é…ç½®å‚考)。 +1. 完æˆ**é…ç½® Keycloak OIDC è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Keycloak IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Keycloak é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 OIDC å议与 Keycloak 一起工作。你的用户现在å¯ä»¥ä½¿ç”¨ Keycloak 登录å登录 Rancher。 + +## é…ç½®å‚考 + +| 字段 | æè¿° | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 客户端 ID | ä½ çš„ Keycloak 客户端的 `Client ID`。 | +| å®¢æˆ·ç«¯å¯†ç  | ä½ çš„ Keycloak 客户端生æˆçš„ `Secret`。在 Keycloak 控制å°ä¸­ï¼Œå•å‡» **Clients**，选择你创建的客户端，选择 **Credentials** 选项å¡ï¼Œç„¶åŽå¤åˆ¶ `Secret` 字段的值。 | +| ç§é’¥/è¯ä¹¦ | 在 Rancher 和你的 IdP 之间创建安全外壳（SSH）的密钥/è¯ä¹¦å¯¹ã€‚如果你的 Keycloak æœåŠ¡å™¨ä¸Šå¯ç”¨äº† HTTPS/SSL，则为必填。 | +| 端点 | 选择为 `Rancher URL`ã€`å‘行者`å’Œ `Auth 端点`字段使用生æˆçš„值，还是在ä¸æ­£ç¡®æ—¶è¿›è¡Œæ‰‹åŠ¨è¦†ç›–。 | +| Keycloak URL | ä½ çš„ Keycloak æœåŠ¡å™¨çš„ URL。 | +| Keycloak Realm | 创建 Keycloak 客户端的 Realm çš„å称。 | +| Rancher URL | Rancher Server çš„ URL。 | +| Issuer | ä½ çš„ IdP çš„ URL。 | +| Auth 端点 | é‡å®šå‘用户进行身份验è¯çš„ URL。 | + +## 从 SAML è¿ç§»åˆ° OIDC + +本节æ述了将使用 Keycloak (SAML) çš„ Rancher 过渡到 Keycloak (OIDC) 的过程。 + +### é‡æ–°é…ç½® Keycloak + +1. 将现有客户端更改为使用 OIDC å议。在 Keycloak 控制å°ä¸­ï¼Œå•å‡» **Clients**，选择è¦è¿ç§»çš„ SAML 客户端，选择 **Settings** 选项å¡ï¼Œå°† `Client Protocol` 从 `saml` 更改为 `openid-connect`，然åŽç‚¹å‡» **Save**。 + +1. éªŒè¯ `Valid Redirect URIs` 是å¦ä»ç„¶æœ‰æ•ˆã€‚ + +1. 选择 **Mappers** 选项å¡å¹¶ä½¿ç”¨ä»¥ä¸‹è®¾ç½®åˆ›å»ºä¸€ä¸ªæ–°çš„ Mapper： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Groups Mapper` | + | `Mapper Type` | `Group Membership` | + | `Token Claim Name` | `groups` | + | `Add to ID token` | `ON` | + | `Add to access token` | `ON` | + | `Add to user info` | `ON` | + +### é‡æ–°é…ç½® Rancher + +在将 Rancher é…置为使用 Keycloak (OIDC) 之å‰ï¼Œå¿…须先ç¦ç”¨ Keycloak (SAML)： + +1. 在 Rancher UI 中，å•å‡» **☰ > 用户 & 认è¯**。 +1. å•å‡»å·¦ä¾§å¯¼èˆªæ çš„**认è¯**。 +1. 选择 **Keycloak (SAML)**。 +1. å•å‡»**ç¦ç”¨**。 + +按照[本节](#在-rancher-中é…ç½®-keycloak)中的步骤将 Rancher é…置为使用 Keycloak (OIDC)。 + +:::note + +é…置完æˆåŽï¼Œç”±äºŽç”¨æˆ·æƒé™ä¸ä¼šè‡ªåŠ¨è¿ç§»ï¼Œä½ éœ€è¦é‡æ–°ç”³è¯· Rancher 用户æƒé™ã€‚ + +::: + +## 附录：故障排除 + +如果你在测试与 Keycloak æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·å…ˆæ£€æŸ¥ OIDC 客户端的é…置选项。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 + +所有与 Keycloak 相关的日志æ¡ç›®éƒ½å°†æ·»åŠ  `[generic oidc]` 或 `[keycloak oidc]`。 + +### ä¸èƒ½é‡å®šå‘到 Keycloak + +完æˆ**é…ç½® Keycloak OIDC è´¦å·**表å•å¹¶å•å‡»**å¯ç”¨**åŽï¼Œä½ æ²¡æœ‰è¢«é‡å®šå‘到你的 IdP。 + +* 验è¯ä½ çš„ Keycloak 客户端é…置。 + +### 生æˆçš„ `Issuer` å’Œ `Auth 端点`ä¸æ­£ç¡® + +* 在**é…ç½® Keycloak OIDC è´¦å·**表å•ä¸­ï¼Œå°†**端点**更改为`指定（高级设置）`并覆盖`å‘行者` å’Œ `Auth 端点`的值。è¦æŸ¥æ‰¾è¿™äº›å€¼ï¼Œå‰å¾€ Keycloak 控制å°å¹¶é€‰æ‹© **Realm Settings**，选择 **General** 选项å¡ï¼Œç„¶åŽå•å‡» **OpenID Endpoint Configuration**。JSON 输出将显示 `issuer` å’Œ `authorization_endpoint` 的值。 + +### Keycloak 错误："Invalid grant_type" + +* 在æŸäº›æƒ…况下，这æ¡é”™è¯¯æ示信æ¯å¯èƒ½æœ‰è¯¯å¯¼æ€§ï¼Œå®žé™…上造æˆé”™è¯¯çš„原因是 `Valid Redirect URI` é…置错误。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md new file mode 100644 index 00000000000..17d906c51c0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md @@ -0,0 +1,190 @@ +--- +title: é…ç½® Keycloak (SAML) +description: 创建 Keycloak SAML 客户端并é…ç½® Rancher 以使用 Keycloak。你的用户将能够使用他们的 Keycloak 登录å登录 Rancher。 +--- + +如果你的组织使用 Keycloak Identity Provider (IdP) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚ + +## 先决æ¡ä»¶ + +- ä½ å¿…é¡»é…置了 [Keycloak IdP æœåŠ¡å™¨](https://www.keycloak.org/docs/latest/server_installation/)。 +- 在 Keycloak 中，使用以下设置创建一个[æ–°çš„ SAML 客户端](https://www.keycloak.org/docs/latest/server_admin/#saml-clients)。如需获å–帮助，请å‚è§ [Keycloak 文档](https://www.keycloak.org/docs/latest/server_admin/#saml-clients)。 + + | 设置 | 值 | + ------------|------------ + | `Sign Documents` | `ON` 1 | + | `Sign Assertions` | `ON` 1 | + | 所有其他 `ON/OFF` 设置 | `OFF` | + | `Client ID` | 输入 `https://yourRancherHostURL/v1-saml/keycloak/saml/metadata`，或在 Rancher Keycloak é…ç½®2 中 `Entry ID 字段`的值。 | + | `Client Name` | (例如 `rancher`) | + | `Client Protocol` | `SAML` | + | `Valid Redirect URI` | `https://yourRancherHostURL/v1-saml/keycloak/saml/acs` | + + > 1：å¯ä»¥é€‰æ‹©å¯ç”¨è¿™äº›è®¾ç½®ä¸­çš„一个或两个。 + > 2：在é…置和ä¿å­˜ SAML 身份æ供商之å‰ï¼Œä¸ä¼šç”Ÿæˆ Rancher SAML 元数æ®ã€‚ + + ![](/img/keycloak/keycloak-saml-client-configuration.png) + +- 在新的 SAML 客户端中，创建 Mappers æ¥å…¬å¼€ç”¨æˆ·å­—段。 + - 添加所有 "Builtin Protocol Mappers" + ![](/img/keycloak/keycloak-saml-client-builtin-mappers.png) + - 创建一个 "Group list" mapper，æ¥å°†æˆå‘˜å±žæ€§æ˜ å°„到用户的组： + ![](/img/keycloak/keycloak-saml-client-group-mapper.png) + +## èŽ·å– IDP å…ƒæ•°æ® + + + + +è¦èŽ·å– IDP 元数æ®ï¼Œè¯·ä»Ž Keycloak 客户端导出 `metadata.xml` 文件。 +在**安装**选项å¡ä¸­ï¼Œé€‰æ‹©**SAML å…ƒæ•°æ® IDPSSODescriptor** æ ¼å¼é€‰é¡¹å¹¶ä¸‹è½½ä½ çš„文件。 + + + + +1. 在**é…ç½®**中，å•å‡» **Realm 设置**选项å¡ã€‚ +1. 点击**通用**选项å¡ã€‚ +1. 在**端点**字段中，å•å‡» **SAML 2.0 身份æ供者元数æ®**。 + +éªŒè¯ IDP 元数æ®æ˜¯å¦åŒ…å«ä»¥ä¸‹å±žæ€§ï¼š + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +æŸäº›æµè§ˆå™¨ï¼ˆä¾‹å¦‚ Firefox）å¯èƒ½ä¼šæ¸²æŸ“/处ç†æ–‡æ¡£ï¼Œä½¿å¾—内容看起æ¥å·²è¢«ä¿®æ”¹ï¼Œå¹¶ä¸”æŸäº›å±žæ€§çœ‹èµ·æ¥å¯èƒ½æœ‰ç¼ºå¤±ã€‚在这ç§æƒ…况下，请使用通过æµè§ˆå™¨æ‰¾åˆ°çš„原始å“应数æ®ã€‚ + +以下是 Firefox 的示例æµç¨‹ï¼Œå…¶ä»–æµè§ˆå™¨å¯èƒ½ä¼šç•¥æœ‰ä¸åŒï¼š + +1. 按下 **F12** 访问开å‘者控制å°ã€‚ +1. 点击 **Network** 选项å¡ã€‚ +1. 从表中，å•å‡»åŒ…å« `descriptor` 的行。 +1. 在 details 窗格中，å•å‡» **Response** 选项å¡ã€‚ +1. å¤åˆ¶åŽŸå§‹å“应数æ®ã€‚ + +获得的 XML 以 `EntitiesDescriptor` 作为根元素。然而，Rancher 希望根元素是 `EntityDescriptor` 而ä¸æ˜¯ `EntitiesDescriptor`。因此，在将这个 XML 传递给 Rancher 之å‰ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤è°ƒæ•´ï¼š + +1. 将所有ä¸å­˜åœ¨çš„属性从 `EntitiesDescriptor` å¤åˆ¶åˆ° `EntityDescriptor`。 +1. 删除开头的 `` 标签。 +1. 删除 xml 末尾的 ``。 + +最åŽçš„代ç ä¼šæ˜¯å¦‚下： + +``` + +.... + +``` + + + + +1. 在**é…ç½®**中，å•å‡» **Realm 设置**选项å¡ã€‚ +1. 点击**通用**选项å¡ã€‚ +1. 在**端点**字段中，å•å‡» **SAML 2.0 身份æ供者元数æ®**。 + +éªŒè¯ IDP 元数æ®æ˜¯å¦åŒ…å«ä»¥ä¸‹å±žæ€§ï¼š + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +æŸäº›æµè§ˆå™¨ï¼ˆä¾‹å¦‚ Firefox）å¯èƒ½ä¼šæ¸²æŸ“/处ç†æ–‡æ¡£ï¼Œä½¿å¾—内容看起æ¥å·²è¢«ä¿®æ”¹ï¼Œå¹¶ä¸”æŸäº›å±žæ€§çœ‹èµ·æ¥å¯èƒ½æœ‰ç¼ºå¤±ã€‚在这ç§æƒ…况下，请使用通过æµè§ˆå™¨æ‰¾åˆ°çš„原始å“应数æ®ã€‚ + +以下是 Firefox 的示例æµç¨‹ï¼Œå…¶ä»–æµè§ˆå™¨å¯èƒ½ä¼šç•¥æœ‰ä¸åŒï¼š + +1. 按下 **F12** 访问开å‘者控制å°ã€‚ +1. 点击 **Network** 选项å¡ã€‚ +1. 从表中，å•å‡»åŒ…å« `descriptor` 的行。 +1. 在 details 窗格中，å•å‡» **Response** 选项å¡ã€‚ +1. å¤åˆ¶åŽŸå§‹å“应数æ®ã€‚ + + + + +## 在 Rancher 中é…ç½® Keycloak + + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Keycloak SAML**。 +1. 填写**é…ç½® Keycloak è´¦å·**表å•ã€‚有关填写表å•çš„帮助，请å‚è§[é…ç½®å‚考](#é…ç½®å‚考)。 +1. 完æˆ**é…ç½® Keycloak è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Keycloak IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Keycloak é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 Keycloak。你的用户现在å¯ä»¥ä½¿ç”¨ Keycloak 登录å登录 Rancher。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## é…ç½®å‚考 + +| 字段 | æè¿° | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 显示å称字段 | 包å«ç”¨æˆ·æ˜¾ç¤ºå称的属性。

示例：`givenName` | +| 用户å字段 | 包å«ç”¨æˆ·å/给定å称的属性。

示例：`email` | +| UID 字段 | æ¯ä¸ªç”¨æˆ·ç‹¬æœ‰çš„属性。

示例：`email` | +| 用户组字段 | 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ã€‚

示例：`member` | +| Entity ID 字段 | Keycloak 客户端中需è¦é…置为客户端的 ID。

默认值：`https://yourRancherHostURL/v1-saml/keycloak/saml/metadata` | +| Rancher API 主机 | Rancher Server çš„ URL。 | +| ç§é’¥/è¯ä¹¦ | 在 Rancher 和你的 IdP 之间创建安全外壳（SSH）的密钥/è¯ä¹¦å¯¹ã€‚ | +| IDP å…ƒæ•°æ® | 从 IdP æœåŠ¡å™¨å¯¼å‡ºçš„ `metadata.xml` 文件。 | + +:::tip + +ä½ å¯ä»¥ä½¿ç”¨ openssl 命令生æˆä¸€ä¸ªå¯†é’¥/è¯ä¹¦å¯¹ã€‚例如： + +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.cert + +::: + +## 附录：故障排除 + +如果你在测试与 Keycloak æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·å…ˆæ£€æŸ¥ SAML 客户端的é…置选项。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 + +### ä¸èƒ½é‡å®šå‘到 Keycloak + +点击**使用 Keycloak 认è¯**时，没有é‡å®šå‘到你的 IdP。 + +* 验è¯ä½ çš„ Keycloak 客户端é…置。 +* ç¡®ä¿ `Force Post Binding` 设为 `OFF`。 + + +### IdP 登录åŽæ˜¾ç¤ºç¦æ­¢æ¶ˆæ¯ + +你已正确é‡å®šå‘到你的 IdP 登录页é¢ï¼Œå¹¶ä¸”å¯ä»¥è¾“入凭è¯ï¼Œä½†æ˜¯ä¹‹åŽæ”¶åˆ° `Forbidden` 消æ¯ã€‚ + +* 检查 Rancher 调试日志。 +* 如果日志显示 `ERROR: either the Response or Assertion must be signed`ï¼Œç¡®ä¿ `Sign Documents` 或 `Sign assertions` 在 Keycloak 客户端中设置为 `ON`。 + +### 访问 `/v1-saml/keycloak/saml/metadata` 时返回 HTTP 502 + +常è§åŽŸå› ï¼šé…ç½® SAML æ供商之å‰æœªåˆ›å»ºå…ƒæ•°æ®ã€‚ +å°è¯•é…ç½® Keycloak，并将它ä¿å­˜ä¸ºä½ çš„ SAML æ供商，然åŽè®¿é—®å…ƒæ•°æ®ã€‚ + +### Keycloak 错误："We're sorry, failed to process response" + +* 检查你的 Keycloak 日志。 +* 如果日志显示 `failed: org.keycloak.common.VerificationException: Client does not have a public key`，请在 Keycloak 客户端中将 `Encrypt Assertions` 设为 `OFF`。 + +### Keycloak 错误："We're sorry, invalid requester" + +* 检查你的 Keycloak 日志。 +* 如果日志显示 `request validation failed: org.keycloak.common.VerificationException: SigAlg was null`，请在 Keycloak 客户端中将 `Client Signature Required` 设为 `OFF`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md new file mode 100644 index 00000000000..212f7cf45d1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md @@ -0,0 +1,107 @@ +--- +title: é…ç½® Okta (SAML) +--- + +如果你的组织使用 Okta Identity Provider (IdP) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚ + +:::note + +Okta 集æˆä»…支æŒæœåŠ¡æ供商å‘起的登录。 + +::: +## 先决æ¡ä»¶ + +在 Okta 中，使用以下设置创建一个新的 SAML 应用。如需获å–帮助，请å‚è§ [Okta 文档](https://developer.okta.com/standards/SAML/setting_up_a_saml_application_in_okta)。 + +| 设置 | 值 | +------------|------------ +| `Single Sign on URL` | `https://yourRancherHostURL/v1-saml/okta/saml/acs` | +| `Audience URI (SP Entity ID)` | `https://yourRancherHostURL/v1-saml/okta/saml/metadata` | + +## 在 Rancher 中é…ç½® Okta + +ä½ å¯ä»¥å°† Okta 与 Rancher 集æˆï¼Œä»¥ä¾¿ç»è¿‡èº«ä»½è®¤è¯çš„用户通过组æƒé™è®¿é—® Rancher 资æºã€‚Okta 会返回一个对用户进行身份认è¯çš„ SAML 断言，包括用户所属的组。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Okta**。 +1. 填写**é…ç½® Okta è´¦å·**表å•ã€‚下é¢çš„示例æ述了如何将 Okta 属性从属性语å¥æ˜ å°„到 Rancher 中的字段： + + | 字段 | æè¿° | + | ------------------------- | ----------------------------------------------------------------------------- | + | 显示å称字段 | 属性语å¥ä¸­åŒ…å«ç”¨æˆ·æ˜¾ç¤ºå称的属性å称。 | + | 用户å字段 | 属性语å¥ä¸­åŒ…å«ç”¨æˆ·å/给定å称的属性å称。 | + | UID 字段 | 属性语å¥ä¸­æ¯ä¸ªç”¨æˆ·å”¯ä¸€çš„属性å称。 | + | 用户组字段 | 组属性语å¥ä¸­å…¬å¼€ä½ çš„组的属性å称。 | + | Rancher API 主机 | Rancher Server çš„ URL。 | + | ç§é’¥/è¯ä¹¦ | 密钥/è¯ä¹¦å¯¹ï¼Œç”¨äºŽæ–­è¨€åŠ å¯†ã€‚ | + | å…ƒæ•°æ® XML | 你在应用 `Sign On` 部分中找到的 `Identity Provider metadata` 文件。 | + + :::tip + + ä½ å¯ä»¥ä½¿ç”¨ openssl 命令生æˆä¸€ä¸ªå¯†é’¥/è¯ä¹¦å¯¹ã€‚例如： + + ``` + openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.crt + ``` + + ::: + + +1. 完æˆ**é…ç½® Okta è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Okta IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Okta é…置。 + + :::note + + 如果什么都没有å‘生，很å¯èƒ½æ˜¯å› ä¸ºä½ çš„æµè§ˆå™¨é˜»æ­¢äº†å¼¹å‡ºçª—å£ã€‚请在弹出窗å£é˜»æ­¢ç¨‹åºä¸­ç¦ç”¨ Rancher 域，并在其他类似扩展中将 Rancher 列入白åå•ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 Okta。你的用户现在å¯ä»¥ä½¿ç”¨ Okta 登录å登录 Rancher。 + +:::note SAML 身份æ供商注æ„事项 + +如果你在没有 OpenLDAP 的情况下é…ç½® Okta，你将无法æœç´¢æˆ–直接查找用户或组。相关的警告如下： + +- 在 Rancher 中为用户和组分é…æƒé™æ—¶å°†ä¸ä¼šéªŒè¯ç”¨æˆ·å’Œç»„。 +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## Okta 与 OpenLDAP æœç´¢ + +ä½ å¯ä»¥æ·»åŠ  OpenLDAP åŽç«¯æ¥å助用户和组æœç´¢ã€‚Rancher 将显示æ¥è‡ª OpenLDAP æœåŠ¡çš„其他用户和组。这å…许你将æƒé™åˆ†é…给用户ä¸å±žäºŽçš„组。 + +### OpenLDAP 先决æ¡ä»¶ + +如果你使用 Okta 作为 IdP，你å¯ä»¥[设置 LDAP 接å£](https://help.okta.com/en-us/Content/Topics/Directory/LDAP-interface-main.htm)以供 Rancher 使用。你还å¯ä»¥é…置外部 OpenLDAP Server。 + +你必须使用 LDAP 绑定å¸æˆ·ï¼ˆä¹Ÿç§°ä¸º ServiceAccount）æ¥é…ç½® Rancher，以便æœç´¢å’Œæ£€ç´¢åº”具有访问æƒé™çš„用户和组的 LDAP æ¡ç›®ã€‚ä¸è¦ä½¿ç”¨ç®¡ç†å‘˜å¸æˆ·æˆ–个人å¸æˆ·ä½œä¸º LDAP 绑定å¸æˆ·ã€‚在 OpenLDAP 中[创建](https://help.okta.com/en-us/Content/Topics/users-groups-profiles/usgp-add-users.htm)一个专用å¸æˆ·ï¼Œå¯¹æœç´¢åº“下的用户和组具有åªè¯»è®¿é—®æƒé™ã€‚ + +:::warning 安全注æ„事项 + +OpenLDAP ServiceAccount 用于所有æœç´¢ã€‚无论用户个人的 SAML æƒé™æ˜¯ä»€ä¹ˆï¼ŒRancher 用户都将看到 OpenLDAP ServiceAccount å¯ä»¥æŸ¥çœ‹çš„用户和组。 + +::: + + +> **使用 TLS？** +> +> 如果 OpenLDAP Server 使用的è¯ä¹¦æ˜¯è‡ªç­¾å的或æ¥è‡ªæ— æ³•è¯†åˆ«çš„è¯ä¹¦é¢å‘机构，则 Rancher éœ€è¦ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你需è¦åœ¨é…置期间æ供此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +### 在 Rancher 中é…ç½® OpenLDAP + +[é…ç½® OpenLDAP Serverã€ç»„和用户的设置](../configure-openldap/openldap-config-reference.md)。请注æ„，ä¸æ”¯æŒåµŒå¥—组æˆå‘˜ã€‚ + +> 在继续é…置之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份认è¯é…置和主è¦ç”¨æˆ·](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + +1. 使用分é…了 [administrator](https://ranchermanager.docs.rancher.com/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions) è§’è‰²ï¼ˆå³ _本地主体_）的本地用户登录到 Rancher。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Okta**，如果已é…ç½® SAML，则å•å‡»**编辑é…ç½®**。 +1. 在**用户和组æœç´¢**下，选中**é…ç½® OpenLDAP Server**。 + +如果你在测试与 OpenLDAP Server 的连接时é‡åˆ°é—®é¢˜ï¼Œè¯·ç¡®ä¿ä½ è¾“入了ServiceAccount 的凭è¯å¹¶æ­£ç¡®é…置了æœç´¢åº“。你å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žæ ¹æœ¬åŽŸå› ã€‚调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚请å‚阅[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)了解更多信æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md new file mode 100644 index 00000000000..0eebb8363c9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md @@ -0,0 +1,62 @@ +--- +title: é…ç½® PingIdentity (SAML) +--- + +如果你的组织使用 Ping Identity Provider (IdP) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚ + +> **先决æ¡ä»¶**： +> +> - ä½ å¿…é¡»é…置了 [Ping IdP æœåŠ¡å™¨](https://www.pingidentity.com/)。 +> - 以下是 Rancher Service Provider é…置所需的 URL： +> å…ƒæ•°æ® URL：`https:///v1-saml/ping/saml/metadata` +> 断言使用者æœåŠ¡ (ACS) URL：`https:///v1-saml/ping/saml/acs` +> 请注æ„，在 Rancher 中ä¿å­˜éªŒè¯é…置之å‰ï¼Œè¿™äº› URL ä¸ä¼šè¿”回有效数æ®ã€‚ +> - 从 IdP æœåŠ¡å™¨å¯¼å‡º `metadata.xml` 文件。详情请å‚è§ [PingIdentity 文档](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html)。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Ping Identity**。 +1. 填写**é…ç½® Ping è´¦å·**表å•ã€‚Ping IdP å…许你指定è¦ä½¿ç”¨çš„æ•°æ®å­˜å‚¨ã€‚ä½ å¯ä»¥æ·»åŠ æ•°æ®åº“或使用现有的 ldap æœåŠ¡å™¨ã€‚例如，如果你选择 Active Directory (AD) æœåŠ¡å™¨ï¼Œä¸‹é¢çš„示例将æ述如何将 AD 属性映射到 Rancher 中的字段： + + 1. **显示å称字段**：包å«ç”¨æˆ·æ˜¾ç¤ºå称的 AD 属性（例如：`displayName`）。 + + 1. **用户å字段**：包å«ç”¨æˆ·å/给定å称的 AD 属性（例如：`givenName`）。 + + 1. **UID 字段**：æ¯ä¸ªç”¨æˆ·å”¯ä¸€çš„ AD 属性（例如：`sAMAccountName`ã€`distinguishedName`）。 + + 1. **用户组字段**: 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ï¼ˆä¾‹å¦‚：`memberOf`）。 + + 1. **Entity ID 字段**（å¯é€‰ï¼‰ï¼šä½ çš„åˆä½œä¼™ä¼´å·²å…¬å¸ƒçš„ã€ä¾èµ–å议的ã€å”¯ä¸€çš„标识符。该 ID 将你的组织定义为将æœåŠ¡å™¨ç”¨äºŽ SAML 2.0 事务的实体。这个 ID å¯ä»¥é€šè¿‡å¸¦å¤–传输或 SAML 元数æ®æ–‡ä»¶èŽ·å¾—。 + + 1. **Rancher API 主机**：你的 Rancher Server çš„ URL。 + + 1. **ç§é’¥**å’Œ**è¯ä¹¦**：密钥/è¯ä¹¦å¯¹ï¼Œç”¨äºŽåœ¨ Rancher 和你的 IdP 之间创建一个安全外壳（SSH）。 + + ä½ å¯ä»¥ä½¿ç”¨ openssl 命令进行创建。例如： + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP 元数æ®**：[从 IdP æœåŠ¡å™¨å¯¼å‡ºçš„ `metadata.xml` 文件](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html)。 + + +1. 完æˆ**é…ç½® Ping è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Ping IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher PingIdentity é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 PingIdentity。你的用户现在å¯ä»¥ä½¿ç”¨ PingIdentity 登录å登录 Rancher。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md new file mode 100644 index 00000000000..e6589e67693 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md @@ -0,0 +1,15 @@ +--- +title: æœ¬åœ°èº«ä»½éªŒè¯ +--- + +在é…置外部验è¯æ供程åºä¹‹å‰ï¼Œä½ å°†é»˜è®¤ä½¿ç”¨æœ¬åœ°èº«ä»½éªŒè¯ã€‚Rancher 将用户å¸æˆ·ä¿¡æ¯ï¼ˆä¾‹å¦‚用户å和密ç ï¼‰å­˜å‚¨åœ¨æœ¬åœ°ã€‚默认情况下，用于首次登录 Rancher çš„ `admin` 用户就是一个本地用户。 + +## 添加本地用户 + +无论是å¦ä½¿ç”¨å¤–部身份验è¯æœåŠ¡ï¼Œä½ éƒ½åº”创建一些本地身份认è¯çš„用户，以便在外部验è¯æœåŠ¡é‡åˆ°é—®é¢˜æ—¶ç»§ç»­ä½¿ç”¨ Rancher。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**用户**。 +1. å•å‡»**创建**。 +1. 完æˆ**添加用户**的表å•ã€‚ +1. å•å‡»**创建**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md new file mode 100644 index 00000000000..b44cc440312 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md @@ -0,0 +1,77 @@ +--- +title: 用户和组 +--- + +Rancher ä¾èµ–用户和组æ¥å†³å®šå…许登录到 Rancher 的用户，以åŠä»–们å¯ä»¥è®¿é—®å“ªäº›èµ„æºã€‚ä½ é…置外部身份验è¯æ供程åºåŽï¼Œè¯¥æ供程åºçš„用户将能够登录到你的 Rancher Server。用户登录时，验è¯æ供程åºå°†å‘ä½ çš„ Rancher Server æ供该用户所属的组列表。 + +ä½ å¯ä»¥é€šè¿‡å‘资æºæ·»åŠ ç”¨æˆ·æˆ–组，æ¥æŽ§åˆ¶å…¶å¯¹é›†ç¾¤ã€é¡¹ç›®ã€å¤šé›†ç¾¤åº”用ã€å…¨å±€ DNS æ供程åºå’Œç›¸å…³èµ„æºçš„访问。将组添加到资æºæ—¶ï¼Œèº«ä»½éªŒè¯æ供程åºä¸­å±žäºŽè¯¥ç»„的所有用户都将能够使用组的æƒé™è®¿é—®è¯¥èµ„æºã€‚有关角色和æƒé™çš„更多信æ¯ï¼Œè¯·å‚è§ [RBAC](../../../../pages-for-subheaders/manage-role-based-access-control-rbac.md)。 + +## 管ç†æˆå‘˜ + +å‘资æºæ·»åŠ ç”¨æˆ·æˆ–用户组时，你å¯ä»¥é€šè¿‡è¾“入用户或组的å称æ¥æœç´¢ç”¨æˆ·æˆ–组。Rancher Server 会查询身份验è¯æ供程åºï¼Œæ¥æŸ¥æ‰¾ä¸Žä½ è¾“入的内容匹é…的用户和组。æœç´¢ä»…é™äºŽä½ ç™»å½•æ—¶ä½¿ç”¨çš„身份验è¯æ供程åºã€‚例如，如果你å¯ç”¨äº† GitHub 身份验è¯ï¼Œä½†ä½¿ç”¨[本地](create-local-users.md)用户登录，则无法æœç´¢ GitHub 用户或组。 + +ä½ å¯ä»¥æŸ¥çœ‹å’Œç®¡ç†æ‰€æœ‰ç”¨æˆ·ï¼ŒåŒ…括本地用户和æ¥è‡ªèº«ä»½éªŒè¯æ供程åºçš„用户。在左上角，å•å‡» **☰ > 用户 & 认è¯**。在左侧导航æ ä¸­å•å‡»**用户**。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## ç”¨æˆ·ä¿¡æ¯ + +Rancher 会维护通过身份验è¯æ供程åºç™»å½•çš„æ¯ä¸ªç”¨æˆ·çš„ä¿¡æ¯ï¼ŒåŒ…括用户是å¦å…许访问 Rancher Server，以åŠç”¨æˆ·æ‰€å±žçš„组的列表。Rancher ä¿ç•™æ­¤ç”¨æˆ·ä¿¡æ¯ï¼Œä»¥ä¾¿ CLIã€API å’Œ kubectl 能够准确地å映用户基于身份验è¯æ供程åºä¸­çš„组æˆå‘˜å…³ç³»çš„访问。 + +当用户使用身份验è¯æ供程åºç™»å½•åˆ° UI 时，Rancher 将自动更新该用户信æ¯ã€‚ + +### è‡ªåŠ¨åˆ·æ–°ç”¨æˆ·ä¿¡æ¯ + +Rancher 会定期刷新用户信æ¯ï¼Œç”šè‡³åœ¨ç”¨æˆ·é€šè¿‡ UI 登录之å‰ä¹Ÿæ˜¯å¦‚此。你å¯ä»¥æŽ§åˆ¶ Rancher 执行刷新的频率。 + +有两个å‚æ•°å¯ä»¥æŽ§åˆ¶è¿™ä¸ªæ“作： + +- **`auth-user-info-max-age-seconds`** + + 此设置控制用户信æ¯çš„最大è€åŒ–时间，如果超过这个时间，Rancher 就会刷新信æ¯ã€‚如果用户进行 API 调用（直接 UI 访问或通过使用 Rancher CLI 或 kubectl 调用），并且与 Rancher 上次刷新用户信æ¯çš„时间间隔大于此设置，则 Rancher 将触å‘刷新。此设置默认为 `3600` ç§’ï¼Œå³ 1 å°æ—¶ã€‚ + +- **`auth-user-info-resync-cron`** + + 此设置控制用于所有用户é‡æ–°åŒæ­¥èº«ä»½éªŒè¯æ供程åºä¿¡æ¯çš„定期任务周期。无论用户最近是å¦ç™»å½•æˆ–使用 API，自动刷新任务都会按照指定的时间间隔刷新用户信æ¯ã€‚此设置默认为 `0 0 * * *`，å³æ¯å¤©åˆå¤œè¿›è¡Œä¸€æ¬¡ã€‚有关此设置的有效值的更多信æ¯ï¼Œè¯·å‚è§ [Cron 文档](https://en.wikipedia.org/wiki/Cron)。 + +如果需è¦æ›´æ”¹è®¾ç½®ï¼š + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å‰å¾€ä½ éœ€è¦é…置的设置，并点击 **â‹® > 编辑设置**。 + +:::note + +由于 SAML ä¸æ”¯æŒç”¨æˆ·æŸ¥æ‰¾ï¼Œå› æ­¤åŸºäºŽ SAML 的身份验è¯æ供程åºä¸æ”¯æŒå®šæœŸåˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚åªæœ‰å½“用户登录到 Rancher UI 时，æ‰ä¼šåˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚ + +::: +### æ‰‹åŠ¨åˆ·æ–°ç”¨æˆ·ä¿¡æ¯ + +如果你ä¸ç¡®å®š Rancher 上一次执行用户信æ¯è‡ªåŠ¨åˆ·æ–°çš„时间，则å¯ä»¥é€šè¿‡æ‰‹åŠ¨åˆ·æ–°æ¥åˆ·æ–°æ‰€æœ‰ç”¨æˆ·çš„ä¿¡æ¯ã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在**用户**页é¢ï¼Œå•å‡»**刷新用户组æˆå‘˜åå•**。 + +**结果**：Rancher 刷新了所有用户的信æ¯ã€‚请求此刷新将更新哪些用户å¯ä»¥è®¿é—® Rancher 以åŠæ¯ä¸ªç”¨æˆ·æ‰€å±žçš„所有组。 + +:::note + +由于 SAML ä¸æ”¯æŒç”¨æˆ·æŸ¥æ‰¾ï¼Œå› æ­¤åŸºäºŽ SAML 的身份验è¯æ供程åºä¸æ”¯æŒæ‰‹åŠ¨åˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚åªæœ‰å½“用户登录到 Rancher UI 时，æ‰ä¼šåˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚ + +::: + +## 会è¯å‘¨æœŸ + +用户会è¯çš„默认生命周期（TTL）是å¯è°ƒçš„。默认的会è¯å‘¨æœŸæ˜¯ 16 å°æ—¶ã€‚ + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å‰å¾€ **`auth-user-session-ttl-minutes`** 并å•å‡»**â‹® > 编辑设置**。 +1. 输入会è¯åº”该æŒç»­çš„时间（以分钟为å•ä½ï¼‰ï¼Œç„¶åŽå•å‡»**Save**。 + +**结果**：用户的 Rancher 登录会è¯åœ¨è®¾å®šçš„分钟数åŽè‡ªåŠ¨é€€å‡ºã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md new file mode 100644 index 00000000000..858be390e9c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md @@ -0,0 +1,83 @@ +--- +title: 1. 在 Microsoft AD FS 中é…ç½® Rancher +--- + +在é…ç½® Rancher ä»¥æ”¯æŒ Active Directory Federation Service (AD FS) 之å‰ï¼Œä½ å¿…须在 AD FS 中将 Rancher 添加为 [relying party trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/technical-reference/understanding-key-ad-fs-concepts)（信赖方信任）。 + +1. 以管ç†ç”¨æˆ·èº«ä»½ç™»å½• AD æœåŠ¡å™¨ã€‚ + +1. 打开 **AD FS Management** 控制å°ã€‚在 **Actions** èœå•ä¸­é€‰æ‹© **Add Relying Party Trust...**。然åŽå•å‡» **Start**。 + + ![](/img/adfs/adfs-overview.png) + +1. 选择 **Enter data about the relying party manually** 作为获å–信赖方数æ®çš„选项。 + + ![](/img/adfs/adfs-add-rpt-2.png) + +1. 为 **Relying Party Trust** 设置**显示å称**，例如 `Rancher`。 + + ![](/img/adfs/adfs-add-rpt-3.png) + +1. 选择 **AD FS profile** 作为信赖方信任的é…置文件。 + + ![](/img/adfs/adfs-add-rpt-4.png) + +1. 留空 **optional token encryption certificate**，因为 Rancher AD FS ä¸ä¼šä½¿ç”¨å®ƒã€‚ + + ![](/img/adfs/adfs-add-rpt-5.png) + +1. 选择 **Enable support for the SAML 2.0 WebSSO protocol** 并在 Service URL 处输入 `https:///v1-saml/adfs/saml/acs`。 + + ![](/img/adfs/adfs-add-rpt-6.png) + +1. å°† `https:///v1-saml/adfs/saml/metadata` 添加为 **Relying party trust identifier**。 + + ![](/img/adfs/adfs-add-rpt-7.png) + +1. 本教程ä¸æ¶‰åŠå¤šé‡èº«ä»½éªŒè¯ã€‚如果你想é…置多é‡èº«ä»½éªŒè¯ï¼Œè¯·å‚è§ [Microsoft 文档](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/configure-additional-authentication-methods-for-ad-fs)。 + + ![](/img/adfs/adfs-add-rpt-8.png) + +1. 在 **Choose Issuance Authorization RUles** 中，你å¯ä»¥æ ¹æ®ç”¨ä¾‹é€‰æ‹©ä»»ä½•ä¸€ä¸ªå¯ç”¨é€‰é¡¹ã€‚但是考虑到本指å—的目的，请选择 **Permit all users to access this relying party**。 + + ![](/img/adfs/adfs-add-rpt-9.png) + +1. 检查所有设置åŽï¼Œé€‰æ‹© **Next** æ¥æ·»åŠ ä¿¡èµ–方信任。 + + ![](/img/adfs/adfs-add-rpt-10.png) + + +1. 选择 **Open the Edit Claim Rules...**。然åŽå•å‡» **Close**。 + + ![](/img/adfs/adfs-add-rpt-11.png) + +1. 在 **Issuance Transform Rules** 选项å¡ä¸­ï¼Œå•å‡» **Add Rule...**。 + + ![](/img/adfs/adfs-edit-cr.png) + +1. 在 **Claim rule template** 中选择 **Send LDAP Attributes as Claims**。 + + ![](/img/adfs/adfs-add-tcr-1.png) + +1. å°† **Claim rule name** 设置为所需的å称（例如 `Rancher Attributes`）并选择 **Active Directory** 作为 **Attribute store**。创建对应下表的映射： + + | LDAP 属性 | 传出声明类型 | + | -------------------------------------------- | ------------------- | + | Given-Name | Given Name | + | User-Principal-Name | UPN | + | Token-Groups - Qualified by Long Domain Name | Group | + | SAM-Account-Name | å称 | +
+ + ![](/img/adfs/adfs-add-tcr-2.png) + +1. 从 AD æœåŠ¡å™¨çš„以下ä½ç½®ä¸‹è½½ `federationmetadata.xml`： +``` +https:///federationmetadata/2007-06/federationmetadata.xml +``` + +**结果**：你已将 Rancher 添加为ä¾èµ–信任方。现在你å¯ä»¥é…ç½® Rancher æ¥ä½¿ç”¨ AD。 + +### åŽç»­æ“作 + +[在 Rancher 中é…ç½® Microsoft AD FS ](configure-rancher-for-ms-adfs.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md new file mode 100644 index 00000000000..b4a15d2bf4c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md @@ -0,0 +1,54 @@ +--- +title: 2. 在 Rancher 中é…ç½® Microsoft AD FS +--- + +完æˆ[在 Microsoft AD FS 中é…ç½® Rancher](configure-ms-adfs-for-rancher.md) åŽï¼Œå°†ä½ çš„ Active Directory Federation Service (AD FS) ä¿¡æ¯è¾“å…¥ Rancher，以便 AD FS 用户å¯ä»¥é€šè¿‡ Rancher 进行身份认è¯ã€‚ + +:::note é…ç½® ADFS æœåŠ¡å™¨çš„é‡è¦è¯´æ˜Žï¼š + +- SAML 2.0 WebSSO åè®®æœåŠ¡ URL 为：`https:///v1-saml/adfs/saml/acs` +- 信赖方信任标识符 URL 为：`https:///v1-saml/adfs/saml/metadata` +- 你必须从 AD FS æœåŠ¡å™¨å¯¼å‡º `federationmetadata.xml` 文件。你å¯ä»¥åœ¨ `https:///federationmetadata/2007-06/federationmetadata.xml` 中找到该文件。 + +::: + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **ADFS**。 +1. 填写**é…ç½® AD FS è´¦å·**表å•ã€‚Microsoft AD FS å…许你指定现有的 Active Directory (AD) æœåŠ¡å™¨ã€‚[以下é…置示例](#é…ç½®)æ述了如何将 AD 属性映射到 Rancher 中的字段。 +1. 完æˆ**é…ç½® AD FS è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 AD FS 登录页é¢ã€‚输入使用 Microsoft AD FS 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher AD FS é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° AD FS 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 AD FS。你的用户现在å¯ä»¥ä½¿ç”¨ AD FS 登录å登录 Rancher。 + +## é…ç½® + +| 字段 | æè¿° | +|---------------------------|-----------------| +| 显示å称字段 | 包å«ç”¨æˆ·æ˜¾ç¤ºå称的 AD 属性。

示例：`http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name` | +| 用户å字段 | 包å«ç”¨æˆ·å/给定å称的 AD 属性。

示例：`http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname` | +| UID 字段 | æ¯ä¸ªç”¨æˆ·ç‹¬æœ‰çš„ AD 属性。

示例：`http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn` | +| 用户组字段 | 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ã€‚

示例：`http://schemas.xmlsoap.org/claims/Group` | +| Rancher API 主机 | Rancher Server çš„ URL。 | +| ç§é’¥/è¯ä¹¦ | 在 Rancher 和你的 AD FS 之间创建安全外壳（SSH）的密钥/è¯ä¹¦å¯¹ã€‚ç¡®ä¿å°† Common Name (CN) 设置为 Rancher Server URL。

[è¯ä¹¦åˆ›å»ºå‘½ä»¤](#cert-command) | +| å…ƒæ•°æ® XML | 从 AD FS æœåŠ¡å™¨å¯¼å‡ºçš„ `federationmetadata.xml` 文件。

ä½ å¯ä»¥åœ¨ `https:///federationmetadata/2007-06/federationmetadata.xml` 找到该文件。 | + + + + +:::tip + +ä½ å¯ä»¥ä½¿ç”¨ openssl 命令生æˆè¯ä¹¦ã€‚例如： + +``` +openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" +``` + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md new file mode 100644 index 00000000000..581a85efc2c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md @@ -0,0 +1,78 @@ +--- +title: OpenLDAP é…ç½®å‚考 +--- + +有关é…ç½® OpenLDAP 认è¯çš„更多详细信æ¯ï¼Œè¯·å‚è§[官方文档](https://www.openldap.org/doc/)。 + +> 在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + + +## 背景：OpenLDAP 认è¯æµç¨‹ + +1. 当用户å°è¯•ä½¿ç”¨å…¶ LDAP 凭è¯ç™»å½•æ—¶ï¼ŒRancher 会使用具有æœç´¢ç›®å½•å’Œè¯»å–用户/组属性æƒé™çš„ ServiceAccount，创建与 LDAP æœåŠ¡å™¨çš„åˆå§‹ç»‘定。 +2. 然åŽï¼ŒRancher 使用æœç´¢ç­›é€‰å™¨æ ¹æ®ç”¨æˆ·åå’Œé…置的属性映射为用户æœç´¢ç›®å½•ã€‚ +3. 找到用户åŽï¼Œå°†ä½¿ç”¨ç”¨æˆ·çš„ DN å’Œæ供的密ç ï¼Œé€šè¿‡å¦ä¸€ä¸ª LDAP 绑定请求对用户进行身份验è¯ã€‚ +4. 身份验è¯æˆåŠŸåŽï¼ŒRancher 将基于用户对象的æˆå‘˜å±žæ€§å’Œé…置的用户映射属性执行组æœç´¢ï¼Œæ¥è§£æžç»„æˆå‘˜ã€‚ + +## OpenLDAP æœåŠ¡å™¨é…ç½® + +你将需è¦è¾“入地å€ï¼Œç«¯å£å’Œåè®®æ¥è¿žæŽ¥åˆ° OpenLDAP æœåŠ¡å™¨ã€‚ä¸å®‰å…¨æµé‡çš„标准端å£ä¸º `389`，TLS æµé‡çš„标准端å£ä¸º `636`。 + +> **使用 TLS？** +> +> 如果 OpenLDAP æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +如果你ä¸ç¡®å®šè¦åœ¨ç”¨æˆ·/组`æœç´¢åº“`字段中输入什么值，请咨询你的 LDAP 管ç†å‘˜ï¼Œæˆ–å‚è§ Active Directory 身份验è¯æ–‡æ¡£ä¸­çš„[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema) 章节。 + +
OpenLDAP æœåŠ¡å™¨å‚æ•°
+ +| å‚æ•° | æè¿° | +|:--|:--| +| 主机å | 指定 OpenLDAP æœåŠ¡å™¨çš„主机å或 IP 地å€ã€‚ | +| ç«¯å£ | 指定 OpenLDAP æœåŠ¡å™¨ç›‘å¬è¿žæŽ¥çš„端å£ã€‚未加密的 LDAP 通常使用 389 的标准端å£ï¼Œè€Œ LDAPS 使用 636 端å£ã€‚ | +| TLS | 选中此框å¯å¯ç”¨ SSL/TLS 上的 LDAP（通常称为 LDAPS）。如果æœåŠ¡å™¨ä½¿ç”¨è‡ªç­¾å/ä¼ä¸šç­¾åçš„è¯ä¹¦ï¼Œåˆ™è¿˜éœ€è¦ç²˜è´´ CA è¯ä¹¦ã€‚ | +| æœåŠ¡å™¨è¿žæŽ¥è¶…æ—¶ | Rancher 在认为无法访问æœåŠ¡å™¨ä¹‹å‰ç­‰å¾…的时间（秒）。 | +| ServiceAccount 标识å称 | 输入用于绑定，æœç´¢å’Œæ£€ç´¢ LDAP æ¡ç›®çš„用户的标识å称（DN）。 | +| ServiceAccount å¯†ç  | ServiceAccount 的密ç ã€‚ | +| 用户æœç´¢åº“ | 输入目录树中开始æœç´¢ç”¨æˆ·å¯¹è±¡çš„节点的标识å称（DN）。所有用户都必须是此基础标识å称的åŽä»£ã€‚例如，"ou=people,dc=acme,dc=com"。 | +| 组æœç´¢åº“ | 如果组ä½äºŽ`用户æœç´¢åº“`下é…置的节点之外的其他节点下，则需è¦åœ¨æ­¤å¤„æ供标识å称。å¦åˆ™ï¼Œå°†æ­¤å­—段留空。例如："ou=groups,dc=acme,dc=com"。 | + +## 用户/组 Schema é…ç½® + +如果你的 OpenLDAP 目录ä¸åŒäºŽæ ‡å‡†çš„ OpenLDAP Schema，则必须完æˆ**自定义 Schema** 部分实现匹é…。 + +请注æ„，Rancher 使用本节中é…置的属性映射æ¥æž„造æœç´¢ç­›é€‰å™¨å’Œè§£æžç»„æˆå‘˜ã€‚因此，我们建议你验è¯æ­¤å¤„çš„é…置是å¦ä¸Žä½ åœ¨ OpenLDAP 中使用的 Schema 匹é…。 + +如果你ä¸ç¡®å®š OpenLDAP æœåŠ¡å™¨ä¸­ä½¿ç”¨çš„用户/组 Schema，请咨询你的 LDAP 管ç†å‘˜ï¼Œæˆ–å‚è§ Active Directory 身份验è¯æ–‡æ¡£ä¸­çš„[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema) 章节。 + +### 用户 Schema é…ç½® + +下表详细说明了用户 Schema é…置的å‚数。 + +
用户 Schema é…ç½®å‚æ•°
+ +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于用户对象的对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Username Attribute | 用户属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Login Attribute | 登录属性的值与用户登录 Rancher 时输入的凭è¯çš„用户å部分匹é…。通常是 `uid`。 | +| User Member Attribute | 包å«ç”¨æˆ·æ‰€å±žç»„的标识å称的用户属性。通常是 `memberOf` 或 `isMemberOf`。 | +| Search Attribute | 当用户输入文本以在用户界é¢ä¸­æ·»åŠ ç”¨æˆ·æˆ–组时，Rancher 会查询 LDAP æœåŠ¡å™¨ï¼Œå¹¶å°è¯•æ ¹æ®æ­¤è®¾ç½®ä¸­æ供的属性匹é…用户。å¯ä»¥é€šè¿‡ä½¿ç”¨ç®¡é“（“\|â€ï¼‰ç¬¦å·åˆ†éš”属性æ¥æŒ‡å®šå¤šä¸ªå±žæ€§ã€‚ | +| User Enabled Attribute | 如果 OpenLDAP æœåŠ¡å™¨çš„ Schema 支æŒä½¿ç”¨ç”¨æˆ·å±žæ€§çš„值æ¥è¯„ä¼°è´¦å·æ˜¯ç¦ç”¨è¿˜æ˜¯å…³é—­ï¼Œè¯·è¾“入该属性的å称。默认的 OpenLDAP Schema ä¸æ”¯æŒæ­¤åŠŸèƒ½ï¼Œå› æ­¤æ­¤å­—段通常留空。 | +| Disabled Status Bitmask | ç¦ç”¨/é”定的用户账å·çš„值。如果 `User Enabled Attribute` 是空的，则忽略此å‚数。 | + +### 组 Schema é…ç½® + +下表详细说明了组 Schema é…置的å‚数。 + +
组 Schema é…ç½®å‚æ•°
+ +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于组æ¡ç›®çš„对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Name Attribute | å称属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Group Member User Attribute | **用户属性**çš„å称。它的格å¼ä¸Ž `Group Member Mapping Attribute` 中的组æˆå‘˜åŒ¹é…。 | +| Group Member Mapping Attribute | 包å«ç»„æˆå‘˜çš„组属性的å称。 | +| Search Attribute | 在 UI 中将组添加到集群或项目时，用于构造æœç´¢ç­›é€‰å™¨çš„属性。请å‚è§ç”¨æˆ· Schema çš„ `Search Attribute`。 | +| Group DN Attribute | 组属性的å称，其格å¼ä¸Žç”¨æˆ·çš„组æˆå‘˜å±žæ€§ä¸­çš„值匹é…。å‚è§ `User Member Attribute`。 | +| Nested Group Membership | 此设置定义 Rancher 是å¦åº”解æžåµŒå¥—组æˆå‘˜èº«ä»½ã€‚仅当你的组织使用这些嵌套æˆå‘˜èº«ä»½æ—¶æ‰ä½¿ç”¨ï¼ˆå³ä½ æœ‰åŒ…å«å…¶ä»–组作为æˆå‘˜çš„组）。如果你使用 Shibboleth，此选项会被ç¦ç”¨ã€‚ | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md new file mode 100644 index 00000000000..69155120d28 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md @@ -0,0 +1,28 @@ +--- +title: Shibboleth å’Œ OpenLDAP 的组æƒé™ +--- + +由于 Shibboleth 是 SAML æ供者，因此它ä¸æ”¯æŒæœç´¢ç”¨æˆ·ç»„的功能。虽然 Shibboleth 集æˆå¯ä»¥éªŒè¯ç”¨æˆ·å‡­è¯ï¼Œä½†æ˜¯å¦‚果没有其他é…置，Shibboleth ä¸èƒ½åœ¨ Rancher 中给用户组分é…æƒé™ã€‚ + +ä½ å¯ä»¥é€šè¿‡é…ç½® OpenLDAP 标识æ供者æ¥è§£å†³è¿™ä¸ªé—®é¢˜ã€‚如果让 Shibboleth 使用 OpenLDAP åŽç«¯ï¼Œä½ å°†èƒ½å¤Ÿåœ¨ Rancher 中æœç´¢ç»„，并从 Rancher UI 将集群ã€é¡¹ç›®æˆ–命å空间等资æºåˆ†é…给用户组。 + +### åè¯è§£é‡Š + +- **Shibboleth**：用于计算机网络和互è”网的å•ç‚¹ç™»å½•ç³»ç»Ÿã€‚它å…许用户仅使用一ç§èº«ä»½ç™»å½•åˆ°å„ç§ç³»ç»Ÿã€‚它验è¯ç”¨æˆ·å‡­è¯ï¼Œä½†ä¸å•ç‹¬å¤„ç†ç»„æˆå‘˜èº«ä»½ã€‚ +- **SAML**：安全声明标记语言（Security Assertion Markup Language），用于在身份æ供程åºå’ŒæœåŠ¡æ供商之间交æ¢èº«ä»½éªŒè¯å’ŒæŽˆæƒæ•°æ®çš„开放标准。 +- **OpenLDAP**：轻型目录访问å议（LDAP）的å…费开æºå®žçŽ°ã€‚它用于管ç†ç»„织的计算机和用户。OpenLDAP 对 Rancher 用户很有用，因为它支æŒç»„。åªè¦ç»„已存在于身份æ供程åºä¸­ï¼Œä½ å°±å¯ä»¥åœ¨ Rancher 中为组分é…æƒé™ï¼Œä»Žè€Œè®©ç»„访问资æºï¼ˆä¾‹å¦‚集群，项目或命å空间）。 +- **IdP 或 IDP**：身份æ供程åºã€‚OpenLDAP 是身份æ供程åºçš„一个例å­ã€‚ + +### å°† OpenLDAP 组æƒé™æ·»åŠ åˆ° Rancher èµ„æº + +下图说明了 OpenLDAP 组的æˆå‘˜å¦‚何访问 Rancher 中该组有æƒè®¿é—®çš„资æºã€‚ + +例如，集群所有者å¯ä»¥å°† OpenLDAP 组添加到集群，从而让组有æƒæŸ¥çœ‹å¤§å¤šæ•°é›†ç¾¤çº§åˆ«çš„资æºå¹¶åˆ›å»ºæ–°é¡¹ç›®ã€‚然åŽï¼ŒOpenLDAP 组æˆå‘˜åœ¨ç™»å½• Rancher åŽå°±å¯ä»¥è®¿é—®é›†ç¾¤ã€‚ + +在这ç§æƒ…况下，OpenLDAP å…许集群所有者在分é…æƒé™æ—¶æœç´¢ç»„。如果没有 OpenLDAP，将ä¸æ”¯æŒæœç´¢ç»„的功能。 + +当 OpenLDAP 组的æˆå‘˜ç™»å½•åˆ° Rancher 时，用户将被é‡å®šå‘到 Shibboleth 并在那里输入用户å和密ç ã€‚ + +Shibboleth 会验è¯ç”¨æˆ·çš„凭è¯ï¼Œå¹¶ä»Ž OpenLDAP 检索用户属性，其中包括用户所在的组信æ¯ã€‚ç„¶åŽ Shibboleth å°†å‘ Rancher å‘é€ä¸€ä¸ªåŒ…å«ç”¨æˆ·å±žæ€§çš„ SAML 断言。Rancher 会使用组数æ®ï¼Œä»¥ä¾¿ç”¨æˆ·å¯ä»¥è®¿é—®ä»–所在的组有æƒè®¿é—®çš„所有资æºã€‚ + +![Adding OpenLDAP Group Permissions to Rancher Resources](/img/shibboleth-with-openldap-groups.svg) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md new file mode 100644 index 00000000000..1136fdabc91 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md @@ -0,0 +1,78 @@ +--- +title: Pod 安全策略 +--- + +:::caution +Pod 安全策略仅在 Kubernetes v1.24 之å‰å¯ç”¨ã€‚[Pod 安全标准](pod-security-standards.md) 是内置的替代方案。 +::: + +[Pod 安全策略（PSP）](https://kubernetes.io/docs/concepts/security/pod-security-policy/)是用æ¥æŽ§åˆ¶å®‰å…¨æ•æ„Ÿç›¸å…³ Pod 规范（例如 root 特æƒï¼‰çš„对象。 + +如果æŸä¸ª Pod ä¸æ»¡è¶³ PSP 指定的æ¡ä»¶ï¼ŒKubernetes å°†ä¸å…许它å¯åŠ¨ï¼ŒRancher ä¸­å°†æ˜¾ç¤ºé”™è¯¯æ¶ˆæ¯ `Pod is forbidden: unable to validate...`。 + + +## PSP å·¥ä½œåŽŸç† + +ä½ å¯ä»¥åœ¨é›†ç¾¤æˆ–é¡¹ç›®çº§åˆ«åˆ†é… PSP。 + +PSP 通过继承的方å¼å·¥ä½œï¼š + +- 默认情况下，分é…给集群的 PSP 由其项目以åŠæ·»åŠ åˆ°è¿™äº›é¡¹ç›®çš„任何命å空间继承。 +- **例外**：无论 PSP 是分é…给集群还是项目，未分é…给项目的命å空间ä¸ä¼šç»§æ‰¿ PSP。因为这些命å空间没有 PSP，所以这些命å空间的工作负载 deployment 将失败，这是 Kubernetes 的默认行为。 +- ä½ å¯ä»¥é€šè¿‡å°†ä¸åŒçš„ PSP 直接分é…给项目æ¥è¦†ç›–默认 PSP。 + +åœ¨åˆ†é… PSP 之å‰å·²ç»åœ¨é›†ç¾¤æˆ–项目中è¿è¡Œçš„ä»»ä½•å·¥ä½œè´Ÿè½½å¦‚æžœç¬¦åˆ PSP，则ä¸ä¼šè¢«æ£€æŸ¥ã€‚你需è¦å…‹éš†æˆ–å‡çº§å·¥ä½œè´Ÿè½½ä»¥æŸ¥çœ‹å®ƒä»¬æ˜¯å¦é€šè¿‡ PSP。 + +在 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)中阅读有关 Pod 安全策略的更多信æ¯ã€‚ + +## 默认 PSP + +Rancher 内置了三个默认 Pod 安全策略 (PSP)，分别是 `restricted-noroot`（å—é™ noroot），`restricted`（å—é™ï¼‰å’Œ `unrestricted`（ä¸å—é™ï¼‰ç­–略。 + +### å—é™-NoRoot + +此策略基于 Kubernetes [示例å—é™ç­–ç•¥](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/policy/restricted-psp.yaml)。它æžå¤§åœ°é™åˆ¶äº†å¯ä»¥å°†å“ªäº›ç±»åž‹çš„ Pod 部署到集群或项目中。这项策略： + +- 阻止 Pod 以特æƒç”¨æˆ·èº«ä»½è¿è¡Œï¼Œå¹¶é˜²æ­¢ç‰¹æƒå‡çº§ã€‚ +- 验è¯æœåŠ¡å™¨æ‰€éœ€çš„安全机制是å¦åˆ°ä½ï¼Œä¾‹å¦‚é™åˆ¶å“ªäº›å·åªèƒ½æŒ‚载到核心å·ç±»åž‹ï¼Œå¹¶é˜²æ­¢æ·»åŠ  root 补充组。 + +### å—é™ + +该策略是宽æ¾ç‰ˆçš„ `restricted-noroot` 策略，除了å…许以特æƒç”¨æˆ·èº«ä»½è¿è¡Œå®¹å™¨å¤–，几乎所有é™åˆ¶éƒ½åˆ°ä½ã€‚ + +### ä¸å—é™ + +该策略等效于在ç¦ç”¨ PSP 控制器的情况下è¿è¡Œ Kubernetes。对于å¯ä»¥å°†å“ªäº› Pod 部署到集群或项目中，它没有任何é™åˆ¶ã€‚ + +:::note é‡è¦æ示： + +ç¦ç”¨ PSP 时，默认 PSP **ä¸ä¼š**自动从集群中删除。如果ä¸å†éœ€è¦å®ƒä»¬ï¼Œä½ å¿…须手动删除它们。 + +::: + +## 创建 PSP + +使用 Rancher，你å¯ä»¥ä½¿ç”¨æˆ‘们的 GUI 创建 Pod 安全策略，而ä¸æ˜¯åˆ›å»º YAML 文件。 + +### è¦æ±‚ + +Rancher åªèƒ½ä¸º[使用 RKE å¯åŠ¨çš„集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)åˆ†é… PSP。 + +你必须先在集群级别å¯ç”¨ PSP，然åŽæ‰èƒ½å°†å®ƒä»¬åˆ†é…给项目。这å¯ä»¥é€šè¿‡[编辑集群](../../../pages-for-subheaders/cluster-configuration.md)æ¥é…置。 + +最好的åšæ³•æ˜¯åœ¨é›†ç¾¤çº§åˆ«è®¾ç½® PSP。 + +我们建议在集群和项目创建期间添加 PSP，而ä¸æ˜¯å°†å…¶æ·»åŠ åˆ°çŽ°æœ‰çš„项目或集群中。 + +### 在 Rancher UI 中创建 PSP + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Pod 安全策略**。 +1. å•å‡»**添加策略**。 +1. 为策略命å。 +1. 填写表格的æ¯ä¸ªéƒ¨åˆ†ã€‚请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)，了解æ¯ä¸ªç­–略的作用。 +1. å•å‡»**创建**。 + +## é…ç½® + +关于 PSP çš„ Kubernetes 文档，请å‚阅[这里](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md new file mode 100644 index 00000000000..59bd02daf07 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md @@ -0,0 +1,192 @@ +--- +title: 定制å“牌 +--- + +## 先决æ¡ä»¶ + +你至少需è¦æ‹¥æœ‰é›†ç¾¤æˆå‘˜æƒé™ã€‚ + +## å“牌é…ç½® + +è¦é…ç½®å“牌设置： + +1. å•å‡»**☰ > 全局设置**。 +2. å•å‡»**å…¬å¸å“牌**。 + +### 自有å“牌公å¸å称 + +此选项将大多数出现的 “Rancher†替æ¢ä¸ºä½ æ供的值。å称中需è¦åŒ…å« Rancher 的文件，例如 `rancher-compose.yml`，ä¸ä¼šè¢«æ›´æ”¹ã€‚ + +### 支æŒé“¾æŽ¥ + +使用 URL 地å€å‘é€æ–°çš„`æ交问题`报告，而ä¸æ˜¯è®©ç”¨æˆ·å‘é€åˆ° GitHub Issue 页é¢ã€‚å¯é€‰æ‹©æ˜¾ç¤º Rancher 社区支æŒé“¾æŽ¥ã€‚ + +### Logo + +上传深色和浅色的 Logo æ¥æ›¿æ¢é¡¶å±‚导航标题中的 Rancher logo。 + +### 主颜色 + +使用自定义颜色替æ¢æ•´ä¸ª UI 中使用的主颜色。 + +### 固定横幅 + +在页眉ã€é¡µè„šæˆ–两者中显示自定义固定横幅。 + +è¦é…置横幅： + +1. å•å‡»**☰ > 全局设置**。 +2. 点击**横幅**。 + +## 自定义导航链接 + +在本节中，你将了解如何é…ç½®**集群仪表æ¿**左侧导航æ ä¸­çš„链接。è¦è®¿é—®é›†ç¾¤ä»ªè¡¨æ¿ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°ä½ æƒ³è¦è‡ªå®šä¹‰å¯¼èˆªé“¾æŽ¥çš„集群，然åŽå•å‡» **Explore**。 + +添加链接å¯ä»¥è®©ä½ å¿«é€Ÿè®¿é—®å®‰è£…在集群上的æœåŠ¡ã€‚例如，你å¯ä»¥ä¸ºå®‰è£…了 Istio çš„é›†ç¾¤æ·»åŠ æŒ‡å‘ Kiali UI 的链接，或者为安装了 Rancher Monitoring çš„é›†ç¾¤æ·»åŠ æŒ‡å‘ Grafana UI 的链接。 + +自定义链接ä¸ä¼šå½±å“å¯ä»¥è®¿é—®å„项æœåŠ¡çš„用户。 + +å¯ä»¥åœ¨é¡¶å±‚创建链接，并且å¯ä»¥å°†å¤šä¸ªé“¾æŽ¥ç»„åˆåœ¨ä¸€èµ·ã€‚ + +### 添加自定义导航链接 + +:::note 先决æ¡ä»¶ï¼š + +你至少需è¦æ‹¥æœ‰é›†ç¾¤æˆå‘˜æˆ–项目æˆå‘˜æƒé™ã€‚ + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰å¯¼èˆªé“¾æŽ¥çš„集群，然åŽå•å‡» **Explore**。 +2. 在顶部导航èœå•ä¸­ï¼Œå•å‡» **ðŸ”（资æºæœç´¢ï¼‰**。 +3. 输入 **Nav** 并点击 **Nav Links**。 +4. å•å‡»**使用 YAML 文件创建**。 +5. 创建导航链接的最简å•æ–¹æ³•æ˜¯æ·»åŠ ä»¥ä¸‹å­—段： + + name: linkname + toURL: https://example.com + + 有关设置链接（包括å¯é€‰å­—段）的更多详细信æ¯ï¼Œè¯·å‚阅[链接é…ç½®](#链接é…ç½®)。 +6. å•å‡»**创建**。 + +## 链接é…ç½® + +### `name` + +链接的显示å称。必填项。 + +### `group` + +å•å‡»æ—¶å±•å¼€çš„一组链接的å称。 + +å¯é€‰é¡¹ã€‚如果未æ供，则显示为独立链接。 + +组与独立链接分开显示，如下所示： + +![Screenshot of group and standalone link](/img/grouped-vs-standalone-links.png) + +### `iconSrc` + +Base64 æ ¼å¼çš„ Logo 图标æºã€‚ + +以下是 Base64 æ ¼å¼çš„ Grafana Logo 示例： + +``` +data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAgAElEQVR4Aey9d5xkZZn3zb/P+3mffZ9nDcDAzHTuqs49PZEhCBhBJc10nO7pHKbD9PR07p5AWlEEZM2ioph3dXGNqLgCxhVBRIYRmNy5cjqnTlWdqu/7ue5zTk+DiNIsCn4suOacrjrxvq/fle/7PofX+ycDZNT/pIAUaUUmaRBKp8FMQ1L9qL6S4+VYUikIeuBrn+LppiuJNG/BvHYd7HbhbXLBZ/dB8AkwFokCHsAH6Kb8IxTHJIyBHwgDMTBT6h7yWAmb1L3sxyEjfxkYxDDQMa0nWV0vyE2slwZ5JtUO8v5JUhgk0EkRw5TnUg9sgJm03lsezkyTIU0C2VrNJU2WsdtTPVQyAmYU4mGIhDESaULAcSG5RjwJEQ8YsxgEOZoOcSxjvXsyKM8nL24QJ4UmF5TnlL7IWLure/G/3lnn/PVu9SrdaQVApO3/CCAZAYjNSLIVBrD/VMwSnsf4/B38ofWthFu3krhhPZmaLJZuyEY7vJPIV++AwEnImJwJ66qTFcMLSswkGWKkiKqtMIICwB890zL/2lwoHGLYnCIMtMqP3Md5N7mMDRDTBkhKAURAqNBs3TOdspjWERgrmkYuJbQMEPUeKdAEIBrIuSIKEiZ+B0ShADz7W/Tjv8TMLOIjybxcJwNJOUgAkjHQSFoAkedMWn2g7rXKV/9rnfZ3DRCRjgoN0ik2A0nDLgMkbYD3aU7dPcRT7ZfhadmEVldA/IZ1RN6TRahxO8f37CTyxU/B4pySvOlYHNM0sZhQRKgwTRrRWALOZc0lWksoY30n0lPkuMBCPYCcKn/Ic632Iy/ivNvy+4k2EOAamDbJ/rIKFYEhx74AWPIWcpRhv8dyu5mQTEEkmUYTjSsaMBWDiA9mjsF/foWTH76Z33zqEL6jD5IgRkTeR64valfOQQSL6My0Ap/c32qvlNJgq339v8Z5fzcAkcayBKrFkNIBCiDOD/Lj8jG2iZWOwvGHefxQNY+3beHk7grCDS7067JhRwn6dZX4d72L3zVei/6Vz0HYB/EQJINKawRTQXVP+UfYQEjdRphITBlTNIrFqBopIsqcsZnaQYww0iv5yA1XkuJ+eRJREStJvrMFhnO8A5S09ewCDoMkCQUw67KxOMQMCBkpkok4JIIQnYGf3k/s7mlO7N3Fw7VX8J2Ot3Pi/rvBXMLMJKxbx+UR5L4CEJ0IKYUbeV0xAUXDZVSrvZIGeHXPff0DRDGE9PRZPhGec8jhBWcr30uXCWEG4Xf/wW+H38ajXVUcaS7GX+dGvyYX6jeQvL6MZO1lzFx7Mc81XkPyM3eC/xlIz5LJzBIz/bbUtUyKZaksUtPUIS06wzK/LGluM6jwrVg9wkCvECDOe51lE2kL5w2drdU+Ths5bSBbMacsVMtGtKDFug5+5Q00Iw2JFOhhmD0C3/goS6M1HKvfiqfxMo7t3MLv2i7F97nDoJ+BpG45IXLysgYRgIhvJPeRu4QVibZ7LX/+rgDiNLTT58IADjM4rPI8HyXpgYc+yXODF3G0ZyPPtZSxUFeM/9p8MrUVJK4rIX5NMfEdm1jauZ1j7Vfj/exNcPoRSB2HxIICiHI+Hb4U00mYSWkP4RAhHTKiUexggfCEPJDiUOepX/5W3tN5R9m3PpZZJ6bdWbJ+kWPkto51JyaVxbBnpYtcT35XwFDXt8Ee8cJ//wj9X6c40fF2TtZU4qspJV5bidawCU/HxWgfHoTwccuhl4srE0saRnwQwwaIZQKa+BH6B0CcfnvVttIBK8jmFId5zjKEmA+WhJTIkeoYYxY+t5/FtmJ8zeUEWzdypqmM07VFhGpK0W9wYVyXh15dQLxnG/NdF/NE00V4PzgAv/0OLB5RbyWKQD2CML5wV1KMbIlmCSgkapQEkcLCNPJ72mJAsdXl+Vb7cRh+mcnlQvKl0IomUfs2mOT28rwCaiErgiW+hXWOaBSnzRSw4/Mw/wR87zN4xht55vqNzNxQQXj3VoyWzRjX5ZPcWUigrozozU0QeMbSNAnIyA0EcaQRE1N8EPWn0hoSDxSSRnntfl73GkTMAsvXsDnCYZAXMERc2dei2i0HVnWMdpro4etYuv58orUujLatLLZsZKapkqXaYqINRZi7XWQ63ASacwh2lhPtvZwjdVs4M94ETz4M8ajFjI5TLlsJLavwsu0GCA84JpX4uEAAVHBYGHa1H3lVuZaQxXgvAIh86QDFDqu+ECDSIstS3AGWnCdmUnwRjv4Y/XMHON51OSerSwjv2kCqdRta81ZiDZWwqwjq8onWFKIfrIPQs5CIKl/ekJvZDyagFJJbWKFuiQFLePwfJtZq+//PnieNLc64lUuwTYoXMITwZowMgbSu3EIjE8NMiKMdhmd/zlzrRjJ12UQb3IQaKojUbiRUW8VSQynzjQV4mtfjbz6fSNN5hBvXsrjbzXzTZjz1V/Bs0/Xw0A9g7qRy3E3DRzARUJpKni0ZSljpEUkcCEBsQR3BYIEIC2mxw+WBV/dx3v950TP5UshpBxskqURG+cvCjtImwqyyDYZ9pDPiMKfR4hHQY5aJdPIIPPg1jrS/nZndW/E0lRJodBHY5SbYUEq4biOx2goi16+D1iLCO/PwDL0HfvdD5X9JFNwXP+vjyL2UMJDnUs8kRpzkjv4BkNX1/l9wlmiOhHL4RIbaDrA0vs0UwifSMVEkuSWJsyTxRACMIKSi8Nj3WWyphLr16PWFaPVlGDs3ou2swldXpgCy0LoWT+t5RJreqEASaMpRDGLs2E6w+u2c7mkgcM/t8IdHID2PSZAQUaJmSrkAypgXXrClaTIj5kZcRXXiKlH4ygAibXA2Yme3wUqAJDMWWDJgGEmiWgzDFL1hCRcjHkWP+kgaPkgHQUyqIz8l+fHDzLa/i0DdVrTaUmL1LsINBTZIignXVRCpK8W3cx3Jdjehnfl4970bHnsA0rpi/QWxLqUf7SiZ2pd/BBPio0kQQyVO/4LO/hsd8ro2sYQxRPYJSJSZJYwhHSBkWoJUABJRqTLRM0m08KKlPYwgiR99kZOdlRi71pOuKSS90wJIrLqK2M4yZTaEGy0N4ml9M96W89Hqs0lVu0nt2Ii+YztPv8XF8ZZ3EPrkAfj9dyF50gaJhscwEP5U0twGrZlKkEhKwDNkZ7gV27yC7ndUxYtfwkymMZMp5L7KF0pJNj2OmYiR0MMkRWsIoyY8MPsYia/fyZmxGzjdsInAjjKM6nIS1aUKJOH6UgK7LAo2FBPc5WKh9kJiYoJWF7I08G5SP7sfMrryLuZsYCjZJYlFp39EWKhIn4TBVy8gXvyN/2e/fd0DREoY4sqvsEo8lHQSCSV8Z5c8RDJJxAeRUKvmm4FEGDQ/S1+6k2NdG4g1ZWPWFJLZWUZi5wbiOypJXF9GfIcbrT6HYNNaPC0X4G2+AF0BxEVqZ7k6NvRuN96dGzjR+VZOva+LyMNfgOizYrwQJkIkY1qmhcMcSZGccTJmCF0XJ/UVAESuuRytOssY8rWjSGVf12MYsbDKyYh0z4SXyIQWIB60hIU2D7/6FjN37uVo56XMNpQSqXdj1LpIVBdj1JSi1WwgUrtBaQ4xscTc8u7O40z9hYTa3UR2uvD2vpPIA/dBJooXmLOfQwkIx7SS13X6RwIY/wDI2Y77n94TDWKVMCQtP0Q6QRpfJJRsJXyfgWhaoCFaJk5SACKM4ZnlxIemOd2+mWhTnvJB4nUlmDsqMK8vVwDRdxYr0yLSkEekIYfwrhyCjXkqmajVFlvMc0M++k43vh2lnKzfxqmpRrRvfRwWHlPVW7G0D40YyaSBGRVJLV66xSWmZPIV96yyZRQSnIy9JRDkK7m6NIES1CIYMgkykv0W3yvug+giaB7LnDrxK1LfvYeT040cqdvEUm0JdFRhtpbhr84mXF9MuL4crWYjevVG9JoKYhIKbypgsTmH0w3rCLcWkbjBTbD9SrxfuxvSfnyYLDmvpQIXKzSpoFceVJlX/9AgTjP9j2/FsYyRVKQcdWlr4QrHIzQhYUIsbSq7Py2Z8/CC0h4cO8rRw3uZb9mKvstFZJdoixISO8tJ3lCuTKxItUjSYiVNtTqXAotyVBtd6ju91gWNLpI7skheV4BevYHFhks4PnA9S/feQup3D4B+0o7WxIjrYUxdEogWs4j584o+DkCk3kku9YJIlTjikZROUsIGKfEvvJDyg+EB3wkSj3yD2dt7OTnwdk7UbySwq4pE0wYiNYUEavOJNJfiF5OqoVw55HpNOUICEKVBmgo4U7+OsBR2Xu8itvsyFu65GVIegsRUpkN1hiRNRVvY7+3UellZ9FfYBq+oAf/8ya9rE0sAItlZAclLAUTLpIkmJfQoUZoASNLrsV/x9Gg3vqaLSNcWkagtQq8rUY6nxPT9tWXKUQ/WVygJKlJUnFIHMFKSEqvPg+4K0rU5pG7IgdpSkjdUMn/NBmbarmLu1n7MR/4NPE8qkMTQCKSlhtcSnlIoqyTpn++nFz9CACKaUsiWyqI9RD5ICkLyDhF0YmYYQ1+CmJhVi7BwlNPf+zK/PtBGtPsSwlefy+w7z1P+RazrYhYbN3K6upTFunIFDvE35H0jDQVKMKg2qCtV2mWxJksFOHivi2TdxczdNQqJWaKZABKtIxO2KhZsh1we09JuVtWxFaJ/8dd7LXz7ugaINLb4FpZ/YYd5RZ07XJK0SoESaZOY5icj9dcSvZLy7Ie/y+mBBsINW0nXlJKsKVEAEfva11CCTyI0dWUogNRV2FEbkZ6lSoKKjR6pd5FoLMLcVUSqzkXyhgL09+ShS/a9fjvB7qs5PlJP8vufhugzKjEmxp4wsDyiVIorS8PGibP/oowhP4oIdrbOvi2V5VfHtLJyDhLhk5yPbmmN2Ax4j5L61bc58pGDPDTcyFNd78SoLoTq9dBUjL67grn6ck5Ul3GmpgJPQxXB+nLCDW4VwXLMSzG7YrVWW/h2ZqHX5cF1hcTrt+G5bS/EThNP+YjKmwowklEwJWJllZo4Cl4EnHqdF33h18aXr2uAqCa0TQtpaKvCVuobjGVTS3zAuCHJvCCYAZICEM2DcecowV0X2aZDKbo4ohLOFC2xTGJarKA66xjruGIideUqbyI5AUvTyPluxBwzatzqev66TXj73g2fvRWOPQpxj2KaU1qIZ2NhO5ttMbcw+LJ5btsjmYyE44QEHBKIcLZytDU2Q5KOkmqRrELELsBPZOKQDkPKC8YMnPkl+r/fydHhnfy+YRvzjdtJtl+GUVNMosaFVudW7y2ACDRUqPdR4Kh3tKZoEREK8t5WG4mppTfkE63OVu8c7ryMk/vq4fhvVeInYJorhJVIAyvULhUN8p6veXQAr2+ACCrEvLATcAlV8xNUA4WUHo+DRDWTCRnLEADTgxxDeAbz5naSNWXLwBCmf7kkibJw7VYCddvwN2zE21iKv9GlnHkV7arJg8ZKaLmUeP8NxO+cgF9+T4VUpZJ1MR1TCbuzDvXZ6FPaCYvKOzpaQrbCWYoEKOJbpfASZyEVYTERJGRKUFukdRik1mzpaXjk6yzePcLxve9ltmkbkboK0tUlZKrd6LVnGd5h/L90K8IgUZdPpCbb8km6L2W2fycc+ZXKhQTkOaV/RGWIv5XRVd9IxbAChy3cXhu64sWf4u8OIHGkFF23OiYOetQkrQoHxbySMYE6zD5FdLzOju8XK5NJpOHLJqVdJPy5Udnq3sZiPLsL8O7OIdyYrULEyVo3qcbNeOsv4Wjj2zl2yyCZn/4nhE6qkXppCSRImZZpkRPkUmkLqWdaaTeJ8xJNk4mYZMIp0mHNMhkTS6rM3Kpt8oG5CN4j8Mwj+L54JzOHezjW/HbmqrcSq5YQtovUNdkY165Hr7W1ggpGSEDiLycBiFGbpwAigsHTtZ2Z/hvg1z9WWjKsRk39AyAvDr2/xrciXYWBVmiQuCqjtgFiQDRkmSLL5obk1Z98mKW+d2PsLEHCtS8bGDaYRONIIk2rLVcOqwzTXWjJY6ElB//ubKVJQjuyiVaLlN7CfMOlHG27iuMHOkh842Nw9BcQ84IeAc2maBhiMuYiCJEgEm3j2afh6FNw5En4/eOkn3iUzOO/hsd+Bk88BI//CJ78L3juF3Div+G3P8T4z3vwfewQj7VdzTONV6roWrLxIlXGn7khH65bR2ZHNhKJezmgWHmsACReY2kQ365Cljq3MrPnGvjJf6j8SlTU4PM0iDUGRHyjf2iQvyZARHur6Ig4wFGVEFQdk4BYWPS8HCCOokhXP+Eff4XZjiuIVxe9IoCIeZLaKTa85bB7mwqYb81jti1HgUQ0iUR+gtflEb2uGKPhIoJNlzHbdDmefTtI3L4HfnUf/Pwe+OHd8B+3wudGMT7cQ/C23XhursV7Yw3ew9V4DlzP4tS1LIy/h7nRq5nd/w7m9l1FaLgeT88OllquI9xRjdm/G/qbibVdz2LNFczVXsJ8/RYWaivw1rgxGlywuwh25UN1tkoGrmT6l7MvANGr84hU5+JpzGexfSMz3e+Cb34GdC9xGaIrALESMsrekuG/UtHwD4D8DQAiDS9DPsVZd0I6CTFLRJJJNEUAEjnJwlc+yJmWi/4HAOLG3OkiVS0h3wKVPJtvKWCmLQ/Zirnlr8kjurOA5A1ukteXkthZgV5bRbhhM77GSuYb3cwJsHa7WGh2sdTsxtPqwt/qVhRocRNsLiDYlE+gMQ9/Qw6++vV469bhq87FaN5GtHoLwasriF1VBddeDDVXQt1boe5StOrNxHZVqfox784cwrXryezOh0YpmblQDTF+OaBYeawARNuZvwyQhfYK5trfCl+4EyLzmKrU3/ZDRE5lZEShjLBcAZC/Bp+8gnu8/n0Q24GV9heAWMM4ZRyG3TGSaLYdRFI+mP8tpz48ohKEEmmSTrZCti9/K5EfAUequkAlEcUOX2h2M9viZr65GE9TMUvX55KQ7HRtKZnrCkm8NwdzRyHpWpcybxJdm9G6thDr3EqkfSPR1irCzZWEdlcQ3F2Kt86Ft64An02B2gKVxJNEnqIGF776QjUSMlBfonI2oZoygjfIwK9CNfhL8jvplgqSzUWEG7Lw1a5hqfZNeOrWqPCtFcaVUO7LI2m72I48wjtzWGrMYb6tlPnWizE/cRiC4mNJlMQ2g6WfVFhXQs82QETtv8Y/r2+ASOPakRALICKh7LCW9YX6XUq9JcRI0gfHH+H0+9rxNm9RodhXChAJkUrNkoAssEtAUcrC7nI8jRX4d1UQra9QtUzmDS7YUQA78mBnNunq9aTq85h/73pmr8ln/rpCFm9w4a0uIVgrSbhyIg1lBGqKCFbLAC6poC2y/SUr6y8l+r7GXPxt+QQ6ixR5W4tZairEU5uPrzqPeEOxer5kvYtUc5Eqq5mvXcOJujczs+tCAo0FLxsYDpCk7aLXWwBZbMpisa2IxeatpP51BHzPWZM7SD8ICRgUIOz+Wf77tY2Q1z9A7Ia2+kEST45TbneMjHKNZ0gbGsTm4dff4Ll9V+HdVUFkR54aKBWuc7EakkFWMaGaIsK1JWpUnb+uEl99FYHaKjWuRKJGcSlf2VGiRt4ZNdnEa9cRq19DtH6dun+0RiqHy4jWlqhtrKYEi4rUtbWaIixyEa92Ea8pVNtoXT6epizmWtcx15JlUy4Lu3PxNeYTqi8kVluIVmNRTLROg5yTy1xLLvPNuYhzHWhwrYqkzVKitXbmEel2M9ucQ6BlI6cHr4VnfmbNpyUdY/uIFhRWRFVe29hQT/d3ARDBiAMQ9VYKNHZ9kgKIOIoJ0Bbgh59kpu9ygo1lRHfmvyKAWKCywBGqKyNUW6lAEarZpPyC2M5NaIoq0apLFLOG6yVnsA7f7jX4G9cpYMZqyrBIgCH7Jep4BYrqEhscAhKXTWcB4m/MwrN7Hb6mdch+qEEKDHMRMGi1+QpMAhABijyvgMG3y4WnsVCRf5XgkOvI9TKNlerawW4XpxsvVLVbZ/a8Ax7/nnLUpWOk9spRIhZabCH2D4C8+i3gaGoLILYaV2aX3QkSaZTaCylBic3BFw7i6d5MrEkYNl+Vi0id0epIRtcVWyPspCSjbgOxmo3Eajaj79yKXr3ZcpKlTFyNpXCrEvHFlmzm29YiWzlfEncWSb2X7MvWrv1a3p7NaIs5p0LT9S60+jwStdnKbEPqwa4VyoNrC1QBYXqHW5mSco5VmWuFtSUCJyFu+V7GdayG5Fx2b1ZmnL87n5O7zlNaa777Cvjp1yC2ZJm4diBL+mi5ImA5+/nq88grucPrXoM42kPgIPtn292ydSURJ9l09aPvGJHbWwm3lxFvLlYRHLHjpZJ3NRRqLCLYWKK0kfgL4m9odVUqShWv2YReu4loXZUayhtoKsHb7GapJZ+FtlwW2rNYaslV58t50Xo53yJ1rYYSIg0lRO1tZFcRFp191uguqX/KJ1mXT6Y6H3bkg+Q4ri+EGwphhxuzxi7ErC9S1crisCdqyzCry1TwQMbdhxrdqyI5l8aNGLtK8HbmcqrxXPRdBYR6r4AHP2dVDcuQAztXKH1k9Y/s2QLslXDvX+Hcvz+ALKsSVd+gZsuURLoCyPHH8Ey8B6PFhdkiYz0KiO0qItq4Ogo3FRFqKlYUbSwmtqsUvaGUeH05upSIN5QTbiwluLsYX0sRvhYXntYCfK15ioLNecQaXarcXkruHbK+KyDWWIDeKMesoKZcYjbJOJbw7gKCzS517aVWF/PtFkmoWL6X90vWF6mK5UxNMUIpSXDWW88qzx3aXbQqiu0qJlVTqTSIpyub2eY3k2krIrnvbfDtj1rjTaQWzh4DvwwQmfFFjc957YPkdQ0QR3sIJqSplQax2l6VlMiUm6rOT6KKaQj88gGWBq+E1nwyLUVEanPQmtzEmopXRZHdxUR2uxXJdYTijdY21uQm3Owm0FqIt80if2uh+jvcXIjWlE+8KR+tSfbPUmx3IbHd+RYJgGyKNOcRacl5PjXnWfdoKWapvZjZzmJOdbs52eNmpsvNggyFbZH3c5PY5casd5NqKCbRUEx8V4kyM+Udws2rIzFT9etLlgGy0HYedBeT2Xcl+hduBn1WFVeenWLIcRb/AZC/gvKzAOGAYxkgdhRRaq5UqbeNIhFaz/3nF/ANXKoAQoubUHWWYh6HyVezFYYWZtd259lkMXGoNQd/Ww6ejhyWOmWbR6A1TzF+QrLYdYVkJMrU5CYkz/I8soAk4LIoX4Vy/W15rKRQSz7xxmJl4lgMbwFWQCFg9LQXstRhkewLUOU3Aa7zrrK/WlJ+3LVFGA1leLuzWGw/HzpdJHsu4rRUCcwfeXGAqBGVMsJTek0lSP4q/LKam7yGNYg03AtImH0l2X9KM4udq5xASzjZVq8M2LF/MBIcue9u/AOXQ0chtOYS3HkuWnMBEdEmq6Boi4toSwFaS766jlxL9iOt+QTb8vF15CKmh5C3M5tgezZ6cy5mYz7UCxWo84JtBTyfXATank++dhe+9gLObgsIt7owmmQ8SolNRSTFZNttPVegvQBvRwGLXfksdMlWzrfuFWktUM8uzy/7kRbXi9AL26WESMsKai7FU+0i1FaqAOJrPRfackk2l3F85Fr43Q9UTZb0i5hZ0keqS6WPVg7DVUEV6UyJBzsduAI49kzyZ7veCuf/NQD2NweIemm7ASwbyW5FaShFYh8JOWMhbFTY2DH0FKFYnEBaJmawf9OTEI1DPIVKEkpnhDw8dfdBwmPXIqFW/7X/C63h/6J3FBDrKH4F5LbPLUVrq0Br20CsfQORjgrCnaUEu9wEevII9uQQ6c5C78wi1ZYDLXkgJlPTeQR3WxRqXkOweR3B3VmEmnMJiXllP1u0qwShSFcZEdnvLkXvKiXVWUK6vRizrYhUq4u4Ddjobjk/l+DuHIIt2YRa8wi35RNtK0BvdxHvcBPvLCTTX0Zqj7UfFaEhxyvKJbS7AL2t1KZy9LYK9NYNyxTs3MCZvnJO9RcR6F6P2boGGtbArlx8HZt5Yu+74blHIHpazSujy8yWUuErGJCpkJYk9C6DWVIQS9jDcmWAlcxbJgdYCRQVJl4xQbhUTJiq5k7mPZaLvXqfvz1AnBj5WfFgN4wARIAhWmAFQBylImLJljwpE8JpmXTZBoiIKl2mNMkQlepYmazg+GMcOdSJv/cdBGtz0Rr/mVjLG9A684h0Fq+Kwp3FBLtKCXQJEEoJd5YT6Sgn1lFqkwBPAOQm1ulC63Ap5jTaC0i0uZBttD2HyDIJE+cqRhZmDreKlrA0RailUDndgd2FquZLEnzeXXnonUXqurF2l2J+0QYWMCxm1zrc6nfZxtoLcY6LthUSEQ3TkoO3JQtfS5YCUrglj1hrAUabm1R7MYlWtyKjrRi9vRit3Xq3SEcp/p5iTvWXcKbfRbgzF7NlPTQK5aK1lnFqz0Wc/FAXxq+/BimZ4ySKaWhktKQFDBlyvDLEJdEUGXkY91ujEJVQtPIoAgM51NIvUq4SVmR98/cMEOHpleCQ/WU9LA6FTUqb2ClZOcYGhxxtJFLEjJTS2nYLKmUkQzp1meTTnIOf/RvHB65Fb96OUZ2P2VeAv20Nka4CIp3uVZGAwtu9AU9PBd49bvx7cgj3XECs+3z0rgswOrIsBmupING8WRUW6i1b0No2Kk0T7thAcM8lBHovw997Ef6erfh6qvB1leHvKsLfUUi8v1xRYk8ZiT0VJLorMbo2kOisIt6xAb13I9G+DUR6qwjvqSDUU0mop5xgdwXB7jKWWouQ6Ja3vQRfRzGBrnL1e3jPBoL9Vcz3livy9pap4/WOEhKtxdaUq40F0JxHujWHVJul/cI96/H3rmexfy3evvWEOwrR2oqItZURtSncXkKoo4iFnjKe6d3MiffVE/7OR+HMb0Am7ktrpE2ZsClD1ATNnsPMshIMezoima9LhuxafS7yUMAhW+vj8IYwwtH2mK0AACAASURBVKv3+ZtrkBf6FC8KDtU0KxpiBaDiCQMhmR9agUa2KUvaxJUenwf9d6S+cjNL3ZdD6zaodcPQRubb1qsSiWhXEashMXVC3WUEe0oI7ikk2JtFqG8N0T3nE+9eQ7IzC7O1ALO5HHP3ZhK7t6G1bCfcth1/5za8ndsI9V9OsO8tCij+nm0Eujfh79qAv70Uf0cx3uZ8fM35BHbnK5NHImDK92kW/8ClGFGAtJIsX8XyN8I9xaykUHcRDgX2lLDQVcRiVyn+7lJlthldZUpzpFvdpFsKSLflYrZnk+jMQutZR6j3Qnz9F7K0d41633SzG5rE9NpIoHMjiz0bWNhTgWdPsSU0ess51V7BM3vfhudTY/D7H0JK1qAKq8k2ZhI6c8kUkUzGsrClRF40iZhYQmI9KCjZUUpHOFqqxP7y7xkg8m4OwysOFyDI2zsSwop2yOwXzmErtzK6Wf5W/5j2HAEx0FOaWhIMjsLv72Pp5huItGyAzougqRRz31Zm2wsJdYttvzqA6J2FinnS7VkkOnPRugqXr6V1WXZ9ol2kbw6pVrH93Yjd7unaxkzPW5jv2U5kTxnx7gLiXUWWuSQmTGs5sdZKoi2WTyO+jfgClrlUgN6Vi96VTbxrHVrHhWida5ZJNNdKCrW8iXDrm4m0nUus43x1XLz7QoyetaR6sqC3CPaUwZ5i6Cki2eNSzxPtzifcnUeoJ5/Qntw/okhPrvJj2LUBdm0m0bJBaadZ8UkGyjk9UM7inlLlR9FZgtnmxtNejveDTfDYVyEzq2Zc8WCySEZNESTTM1n9qEqwQeYSUHPAyJgFMaDtyQGFPeRPoRVy89WAyd9egyyDwgGGAwirLFqcMGuCaqfe6vlbOdoBiClDVMW3k/GrMsWPcZzQo5/lzGc6ebarglBjIXRuhLYK4n2bWOouI9xTSqy7eFUU73JBe7aiTHuBkrzJ9kqMjg3EOyuJdxUrHyfWlaUcdHHU/T1uPD3lLPRsxttdgdaejdl+gTJhku35JDrcGJ2l6nytu4pY9xYi3VsI7dlCsG8zgf6N+Pduwje4Ed9QFWe6rNzH6Z4SzuwpZqa3jJneEoRRZ/tKWdi7gYW9FcwPVDLXX6a+l9/P7CllsdtNqrecdFcRiY5Com15hNoLCHQVEuotItxfQrC3yNrfU0S0pwi9u0iBOdlZRLqtlExzJWZLFVpHJYEeuUclswMVLPaVEepxk27NgqYLQbY9JSwObufY+xsJPHQvxI+rhT/DagWqJFEzoeYPVh0qnSrLRygUSKeqjrU6X76WiMzfP0DssJ74Fyu0howXsIBhgUN8CecIZyu2qFAsmSSeThNPmWoOWjVBmRGGM0/AL77KsX9t46n9F3O6w0203QWdpdBdTqijhEhvJdGeYtXx0vkvl2KKWYpJdhSTaSuGllKbykm3laN3lePtKWa+z83sgIv5/jy8vVmEe9aS6DxfASO9ex1mSzbptnzMrnxSfS4Sg0XEhyuIjFUxt7eS0/u3cmb87cwcvJ75W1tZ+mA/3rtGmf/INOZ370V/4F7iP7gP44efJ/ngl0j9+IuY//UV0j/5Mvz062qrP/B5fP/xUc588Q6euecmnvzIAf5wRz8nJt/J3P4tnOyt4GRnKTMdxfh6KtH3biWz/2IFVKOjklS79U7yXplWi8SJj3bnEugtZKm3ksU9W/D2bCXSWYnZng9ta6BnHVrdPyEROnPsYkKjb+NY18WEpnfCN2+HM/9F2jyhpmoVq0F8SS1h8b70tQjHsyCRCejsUYoCDvHa/741iAMQx5yytgIQmQjO+k/A8XyAOOCQBtRUOtCad1eNGpRZA71HSXzn4xw9UM+xfVfyXFsZsaHNJPZWqmiS0Vem6qBifWWEbcn4csEhx4v2CfVsINwtGqOcVHupKrVQ5RYSnu0uxttTynxfKbP9xcz3F+DvzVJOvNnxz5jt5xHtKFQOdaC3iuC+LQRGLsY/fQWLN76D2Vuvgf84AN+9HX5yLzz2HXjuUVg6DiEZHSnj1kPWeHZdVqKVCNCfIRn7LueF/OA/BY/+O/z4bmJfmWLu7g5OHtzByeGrmN/7Nvx9VxDu3I7esZVU20YyrZXQWm4JgdZiUh0u5YcsDmSz2F+EV4IEXRXEO0rItOVC6zqMhjcqX0zrkQBBKaHujST7Loaei1jq3U7gK1Okn/0OJGfUNEWJpEnAgGDaWqBKcGCFciWaac/Q6Mx9JoygzIdXw7iyrvk3NrEEIHYo9wU+h0gOAUYsqas1ti2QpNXU/VpcxzBkjiWZORD8yTAZAUZiFk79HP2rt3D64E4W979DRYciPZuI9FQR2VNBuLeEUL+bUH8h4b5CIr1uIr3Fq6JwbxmhgYsI7NtOaGgL4X0VBAeK8PVk42lfy1LzGrQ9Vpg30laI1llEvLcUY28xWl8eS31uToxdzLGb3sviv7ahf3EafvBJ+M234blfw+yzEPFDLAq6zPUlQ4nFiXVyRLIvs+OtklKylqBMYC1zZy2A/1l4+mFS3/8c/k8eYPHWDuZGriW4753oA5eS7tsCPRXQ6YaOPOjMJT5USnS/1F/lYu5dT7L3QrSutfhac5nfXajMQ61rE2ZHBbQXQ1shtBdidhYjkbRj+7ZjfGUcnvsRhE+QNmJqfi9ZWkfm+1KznCnr24CUzOIijrvARozrV//zGgGIo0EsA8oBh4BCZpaV5Q3ipoFuRLEmfLYYJGnGmUv41YqzSgL9/rtEPjPK7ORVeIYuITl2GWLHG50Wad0bCPWV4B9w49+bq8gCyWoBUkJo72Z8e7fgFZ9gpJLAeAWBURf+gWx8XRdi9rthsBIGt2Ds3YZv8BIWxt6B933VLH1sD8YPP0ryv78Gxx4C/x9AZlrXRDNEIBg5G/wXv1XW+FieVtQywxWLCAOthjIQ0ZJoyYwValeTLAgYF8H3DMz9Gn79NTJfO8zcjdfybO9G5nsr0PZvwhypwtxXQbK3HGOPi8SeCzH6ziW+91wi+9bhHShmsbeKQM+laB1vISPRw5YKCyAdORh7soj05ZIa2Yxn5C2EPjEIR36gZoGUyR7mE7AohpcpARd5P5lXQISpRjIZQktbK7682lOXvgYAskIa2supiea0oCLbNEY6QVQLocl0OCr0JwkliWzEVJ4jkzoGj99P7GNDLAxchlcy2AMlxPaVou8pJdlVCe0bSXdUofWWEtjrYnF/LovD2QT3uoj2Fa2aRFJG9pSgDVcR2l+JZ28xnoECtJEiOLABvS9XZdElJzAz9nY8HxtSfgKnnrKWbwuKiSTTg9raQGL/GV2tgmUtj+CYmxZWBCDin0oCWvalrVb7EQmsEttSki5zc6mtRAXl6iLDpdjwGOiPw6lvk3j4Q8x+dg9P3nw1j49cwrHeKlLid7WXkt5bTniomDP7cjk5uJ6FoXxi+ysxujeT7LiEZNtbSXRcjtazmUi/i+DQm4kOvgm616vk5cz+K1n8zCippx8EM6T632OkCMQzhGPidwhARAoY6EkfYdOrVl5XczKvtgH+gvP+tgBRkQp7PT+pwzkbrX0+QJKamhldqVgjjBlcsObXTcv6GsfxPPYFTt3ejm/gbZh9WzD7iwkN5OIbyEbrc5HsLoH2ckWJnlIFioXhXIReKUCMgVJCnYUsteYQ6HRhDFVhjm9BG97AQn8R0ZveQuTO64l/aQR+/jmYewIkiGAzpRIEUhKehFjcJBo3iCY04qZGAs2eBURmApGAhQWI/8mtU8Yhgb9oKkM4lSSc0YgisSUfCRaAU8AxQDTcb0g8803+8O338+Rd7cr8MnsvJdS3nfl+GW5bxtxgEYGhIuJD5eji8HdvJdJzGYHeS/AObMSzz4V3+M1E972BVOMboK8cfeRijo69g2c/PUbm2YfU9EwChlQiSTQSJ67ZdXWyTHVGpuQOqxls/r5LTQQgCiRnnS0l1VZoEGEGNQuGWndcJK3M0O6B6DzEjhH41Sc59slWZgcuJdWzFfZUoHVmsbh3HeEDLqKDOaR686FD7GY3qT3FRAfceIfcLO13E9orf6+OIgOl+PaUEeqtwOgpJdNdCj3lsHczxtTb8PzLTvj5p+H4g5A4LXljtU6JP5ViMawzGwgRNDQFCN1MIsuzyfuKNWGaGSsq5xTwKfPCWbZM5hqWBXGseiWnGVe3XVn45wRJ4qRFi2WiJNJBYkkfkZRHMWWcKBphvJlFwv6nmP/mR/D+6xBnRq9lrutSol2byfRvxezfRKyvAt9gBYv7y5gbKWVuxM3ScD6+/XmE9uUR788j05YNXW6SQ5vxT17Oczdew4l79pF54n5IzEBKSoV04ppOMmXlwtIk1eJ7ii9e5TDW316DOAARBWIXLQqTiGRVklJpFjHD7BIEWedCZicJncH87bd47Jb3MDu+lVTfRugsJ9GWh6f9As4MrcN/MJ/ovrWk+9ZCVxZ05ZDszVcACQ6KJikj8goBstTlVgDR+qvQ+jajD10Od7TC9z8Kz8oquIvq2WViackfz6QzLKUFKhLUzBCKB4kmIyQyoiXOJkOV7y02j+NbSORGrbEh64s4AJGrnHVWpSkdLfyXbOXiGbWIj5irTjLOWqJN1i6MabJMnMwSLzPoy8TYsGgmmU0nmFeGWIq0/wQceRi+fQ/6HYOE915NqGWLKtgUcza4rxDvcBaLYxewNHoBvuF1BPflERkoUwCS+jTJvxgD5SQnt7M0spWjo5exdO8gPHW/NfF2Okg6k0A3TeIpazHSjCwrZ8gqodY7v1r//u0B4jDACwCiwCHS1JnNXCIYRggML/hP4v31jzh272HmhreSEQ3R6YIOF+E9LuYHCzgxms2Z4QuI71sD/edC77nQvwZtMJfwYAnC0JIs1KSj9hatirS9LpjcgD5UxPzQJvx3NsGDn4b5p8CIKZQLvrUURAzQhcdVT6ZVLZKmiZ0vDJ5W852bmRSptEhHK/n5ohpBhIhNZ5ljZSO+jH2ZyC0WISMr28rAfclkv4gfGE0k8IdlASA7ky2HxSGyFLAEl0j5yAI8+RDpL91B+OYmYvsvJj3ottp+4J/IDP4TqcE3EB9cp9o6vHeTigDOdJYQGChH31tKRLL2XbkEhysJvO89nP54Jzz9HYgdVwBWII2nSciO4DlqC5BXCx0yu7u8qwVCJyehQgbqO8dRtjpVDnT8BZHo9okrStXVdV7498t+eOlgub5M/iYmlTT+HERmQJ8Bz+8JPvh5fvPhcZ44cAPcciUMCkByoN+NNlLG3LCL40NZzA+vUwDJDJyLUHLvGqL7cgnvKyLeV0Wytwqjv4j4gMvqtMESooNFimKDLrS9BcQHLBIQRQdLCA+WEdpnkX9ISr3LiN1VAz+5C+Z+CvE5VWQnTq/Id6nkliWRRQEobSDOpvRwRhxhXUlwicwlUjpGUkNPxRFzSyI5MuRLcmEOOf3hbJf7ZYXHptruL/1bhI+Ej6XUPGaQihsqfK4nU8iiQ/KEihclepZIk1RIF7SnIJqBeFqZPoYCV8oSCoFZePS7ZD49RnjyCuJ788kMnAf9b1JgSQ2st9p7oJLA3q3M79uOZ/92fH3l+NrWo7WdR2ZvLsbERmYPvgX/v43CyR9BJqiESzSWsCbhcBpFAdriWcWTTjvL1uFRtSti56yGtnj+zzPnOdIIEuZLRHzWRMpGkIweJRQHX8Jad0KTq0lvSLw9IYVkMjm0tVCNDGlVK4DZW8GQREaU8ycxBzt8n5Sl+WxGERvbuaSEcWUFc7VMmJQTqHU8vJCYA/0EBJ6EmZ/CE1+HH3yI1OeG8XygnpMT7+TkwEai+0tIDuaTHCwkPlRIdL+L0IiL4GghoZF8tOFc4vuzFen7c4kOW8doQyUqY53oySLZl4e+v5zo+EZi4xuIjZYQGxJgZMFQLum9uUT6i1jqK2F+oJyF4U0EDmxn7rZrmPv6nYR++6AFYDH/TA0jFkXTk6TsUv5lIbTcYbYQULkfQ6U6rVUUrVYQplT9smIrESvhCetazvmy1LSMo7BzI85WKjdtyiSd1W1F4NjHiaZQJHMi2ReWrf2gspF7Oc8hW/lbfSQULLNgJERD6giYxPSSY6TfVWjNG4Hjz8ETD+O9o5fZ4Svw7CmFsUoV2ZP2jPesx9hfgXffpfgmrsI79lY8feWkutfAwAUwlEVsbw6nRyvQ75+CpafsnE8aM2pYkb+krpZ6ULP5y4vI/QUrK7MG0lRKP69MPC+/qv1Sf3pzjrycNL6KMZuy4IosmSXrOFg2pyzMIq6gxdEyCCmo1vjLxCIktJgFBFuxpFMmYhsSF3tCA01UtzSmbkkXZxZzZ/bymAf96E+JHvkRoce/jf8XX2PpwXtY+vYdeP79MIEvj3Lqrnrmb78e7/veReCmywkf2kZ0spLoWCmx0SKL4YXpV0H6UCHJvhxM0RLDZUTHNxCcqCI4WkpoqIDI3vUw7ibRn6Wknj6xhfmRbfxh/3ZCH2uFR7+szD1JWjqfRCJBLBZD0zQljZ3vX3wrvWnZ/zJ+XiJVKxnTAYnDoMK4Vu/rloMumeWUQSadIiMqSqSVSB9RVbL5c1slzUTQ2bafLXUdPpNnEZKntLFjD2Kzw+xqQRxL0zjaRiaxFPmZ8SVhdhGO/Abzm59k7uYmTvZvwz9YRmaiBEbzVHLRN1BFYPhyAiOXEhnaBMMlMFkEY9kk+y9Q/ey9/Rq0733YWutEVqtKGKTj0ioy1WzMjmbZAHEALw8u+zZABB4yq7z4U897nxfvmOVvzxEAyMupj7Lzg6QTIaJxHa9usBCPs2RohBMBjKQPM+0V11I9WIogaWbI8BzwDKSPgnEEor+DwOPgeRROPgJP/wh+9Z8gSwR/+cNkPn4Lidsnid/aS+TA9cSm30ls+h1Ep68gOnUZkaltxKY2E5ssJzRWQHgsh/D4WiJj5xMZfxORsf9DZOz/IzL6RqIj2asChwBK2+9SmeDEUBnJkSLi48X4xzewOFbJ4nAxS4NuVQ+VGK2AkQLM0VICt1xF9L4x+O13rYnolJMr89IliMfjpFIpMlK6ncmofadpX3wrjCzhSyHLbBXmdCT4SuZUHa0Uhy3BJR8kmfCMwMoKdr7YViw6iY4JU8jvcm2HrCSb46BLyMAyQZbVuxxonWidZONOvhKRICR5KusJFCdawJRdeXjh4XgKluYxHrqfUx8Z4pnxK5kbLCYxlAWj64n2riXen0+mJ59kr5vwyDYiBy4iPOrC07de8cLMyEV4b3sP/PgOCB612wuloSWJLIsiSTDheRpkBQqsuj4Bk5AVKZRXUzLkxTtm+dtzfAYE4qBpcUjIEgGiQXyoiZ5lXTtZjEXq95NnVHUskT9A6Aj4nwLvf5M59XUyz34W44kPE3zkfSx8e5JTX+jj+CdaOHlXPSduuZaZm65h6dA1BKbeTXT83ehjV5EYfhfsv4K0lB60rrOofS10rCXTfSFmz/kkxLEeW09mYh2pqTUkp8/DOPBm9ANvInbwjcQOnE90NIfYiGt1NOzGGKtSlBgrQRsrwj9RwcJ4BXOj5cwNl+Ef30RkfBOJySrSH7gavvMvajkzlctIWhJbgBGJRBRAlltWBJiYpC/1UWgQ6b1CggtzvZBWMqpjPigT12JKh4dfbCsOdixpoKVSyq8xMk4pqMXcsqSoQUwZuSJd5Vbq/nIxYXKH5L5iItt8L5aHkPxsiWo7BJ0W/8qW5vJ4coK0kyyc+uzPCX39Vv5w8F0cE59utABz34UwKIGUtUqTL+zfyqmxS5gbrWRufwmBqYsIT28jNl1B5K6rMR7+FETFz4OIWPzqGawckWJ4555yXxsBFkAERJJ0PeuHOL+/VBedE08lEScrbUSs9fOiJyzJ/8y34NHPwoN3wPdvg/tvIfXlQ0TvGWPpQwPMvL+ThVvqWRi9FN/wJvwjm/GNbVQMFZjYiH9qE/6pjYQObyZwuArfwUoWD5QyN1nEmYkCTo/msTCUA6NlsK8Y9hXBYBH0F5Lqz1cx8uhArpLy0RE3oVE3ofEiAlPF+KdL8B8sJ3SgVEmaVwKQ1HgF8fEqgpOb8E5swDfhxjPhZmZqA3OTGwlOVOKd3Er0I+3w8/vAd8Raa1ySe7bzqurCEgkr4ma3tmiUaFTc9D/zkU50ACGdqrhcfAMxfSSDbPsJDvfLMSIsbV9PggGJjJVDEfloaRFHq5hKeyjZmZHsQep5f8tKHWFSyh6I2KFcuc3zJPELASIxhhf4RkozpXTSMlow4YOMU0Wlq+XvTPFX0mIO+sHzFNGHP8cfPt7PsQNXkh7Khv1vhulcMjeWEzq4lYWJS5mfuBzPgSuUsIrfUoV5sBD/SCGhuxvgN99S682L5SPPIs8szaIwYfu/0qwWCXjEz3PIEir2j3+mc+Ac9VKiNdIRCBwn9N9f59i9+znz/muIve8KEoc2Yx7YTHzqEiITbyE49jb8I1cSGroMfe8WGK6CoVIYKoEhqTsqJL0vj8S+XLShLEJDa/HvX4t/bC2+qfV4D67Hd2M2gZtzid5UQGK8EEaKyIyWYI6VKDMmNVZOcqxSRTK00SoiYxsJj2wiNLoF/8hWvCNb8I5sVcCMjJagjbpXTfGxMiLj5XgnqlicLMc/6cY37cJ7eAPew1uZP7Ad874BePJ7EJoDQypKlz0Ba3KOdFqZVIq3bPPKMbNeqgfOduJyb2LNdCdOthTnOSTOtQUMcYSFKcR6ERNHysPjCYmAxe0QsZSKOBGbF9864WQtIwCxAjGOLyq3UcyzUhLb9xZQOiFm59klEKFAmohjGCFMQxYp8trl60FSpoDFHs8hyU0zCAkv0T/8kvl/uw3/eBXpobXo0wXoN1eg37gR7/gmTo9eyszEJSyNFhCbyiE5uZ7wcA7+qYuJfX4M89nHl4fjOs8ijynkyBILOKI1rECI1WK2KeYIJTn5JT7npPynSIdnIBmA8Ck8D3+e37x/J8+OFmMeLoDRN8LIG0mOZhMdLyI0uYHQpER7yjAmpPAvm8DeLCKDOWj78zFGXSQnXKQni8hMuUmM5dqUTWIsG2M0i8ToOoyRtcRG1xM56CZ4qJjAwRKC08X4J4vwjxcTGCkjMFKhIkcS3ZDx0+GBzeiD2zGGLiO1/0qSIxersG5srJDVUGSsSDnkgZES/KMFeMfy8EwXEjjsJnmzm+htF6F9dRKe+xnEJaMLft3qbmlXMyVh2hSmaSpSOZuXaOwX/mR1rGPDixx27CdhfWdfutkyfWTPAYcDEHVN5b+IDyN5ColWSeTKIJOMk9AiJPUoqXjMmjBBQrLyezqpQC3Xs3wJxyKRu8i9rXs6v8tWgUcBR7SaDV47IJYwIW5CTGaXkSy8MsAkT7IEyNqQHtLxBTJSjStMLNp39jm0f59Ev+sqZqc3sTRRijblUn6nRK/OjJah7V9LYug8jLE8olOleCY3MHvr1cS+eYe1arDSThaopT3l6eV9HLJmQHkBQBzwy/bPAURlZePyItKwUcK/e4AnPtzOsclyuNUF+/8X7P9/SIydS2wyj/DBUsKHK4gfLiFxuJTgeAH+cReBCTehCTeRiSKi4y70CTfx8UIFltR4AamxfMyRPFJDOST3ZZMaXI+2fz3esRwWpnJZmipQzOmbLiI4XUJ0qhxtagPahPgIm0iObiY1shVz+CIYvkRRemSbKgpcDTjknPC4G+9IMZ4RFwHJ8I5egH86i9BNbqLv38T83deC/7dqzUDhKbF5fRlL4lpdIZ2dUqaVOOcrASKgEd/kpT/SQ1bnOSsvWdEWcTytDLbjBJ/VWfYKthkJKftAPw3h58D3NMz9Fk78itQfHiH++x8TfeKHJJ9+CPOZn6rv1e/+oxA7AfHTYMxb0UUpo3cYXkwhO6sugBVj7awTLuFKu6hSwsiqzN72n+xKY9FEEviRtXZ1Ff4JQnqRdPy0BZC0rFNoB4bk9aNPwX99kOMf2MGxye0EDlWg31RM+MZStEMu0vvXwPCFpCbdyg9ZHC9jZnIz0Q/VkfrG+6wlLdSzW0EOAbGAYzmqpt7BgYsEQ+zkohz4lwBE5a1slS0Pju7H+8h9PHf7e5mfKCA2/Ab0oX8iPHIBwSkXnuly5ibK8E+Xod20geBUGcGpCkWhyQqEIpNliqITJQjFbNLHixGSaJFFbvSJfPTJvGWKT+SzTOOFygQTMywxJtrITVJotFiR/C1AjE0Wroqik26lrbxKjWeRPHgh8Yl/JvIvJfDgYdCetMpaZGJ4ex1ykaSW0HEkvLTyS38EPLpujWER08v5SA4jri2RSPqUByDgEEhJ5wqjiQcTjmtkpP5MCjNTpyH0Gzj2Tfjlx0g/cCOBTzQS/WgN4bt34L/rOjwfeA/zt13F7C3v4Mwtb+fY9GWcOHwFc7e+E9+d16J9tJbUZ3bDfR3w5X2qnJ2nH4KFp60aNwmjSoLWtH3S9JI1mMk4qRK1GVl70IgSNNJEpTRAluU1ksrmk5GAQUAMLClVl9IaaTe1aKdjZknEzp6pRDGoVEZEn4Hf34/vi8OceP+78N6yidShtaT2/2+Sg2/CHF5HeiKPuPieN27Ff+M2jOkNykcJf+cTVhY/EcaMSvGqNf3TYjyCpoSM5YNYsEla95YucwDidMaf2J4jLyDk5DvEZDCO/YLFL45wfHoL4YlczIn1JCfy0cZF/ZUpM0tCsGJmCRhCU2VEJmxgONtlkJQRnRCywLJyG5sQoLjPgmA8n4RNFkjyiE+cBY/Yos+jyTxik/kvCQ5R2S9F/uFcQuM5hCbWEphah3nXBnigD05/H+ILymwRwSFttBwOtxX4yymWE5CIMy+RLdEulo8i/kJSMVw4GkI3pHpXACg9KEwXAP8JMid+jv6rL+K7/zBzn2rm1O3v5PTN21g4UI55axXcVAaHy0kdLCI5VUR8sgB9rIDYmGSkXepv0ebaeN7y97GRPPxjpfhueQ/zd7TgvWeU2P13wi++Bsd/CsGnWewSOQAAIABJREFUrahl4jikjoM5Axlh+6gan+PLZPALMGI6BMOkQgaiMB1eEpDMZ2TuEqvd5L2U1lUmoJPIdELWsrLvKTVoau7rh/j9wUtYGvg/MPZPcFMuifG1aENr0MfzCB3cRHi6HCZySB7exKm7u+Hx74MmGnEJUrIMuEwimCaUljCEFbZQ7SlWkoDT0RxnZdWfgAeoRKG8lLyIkPrI6Lxf3Mfs+68ndnAzTBXDaCEM5sCgC0bKYX8pZl8+KQmPThYRn7C22qSzLVLfy29/mkrQJsrRJiptKle5j9hkKdHpEsIHilRFbuigi9DBAgKH8vEfzrUpm8ChXGLTBS8JgJcChzBScjKb6NhavBPZ+O66nPSDN8HSTyB5gozhU1BQhpI0pmpQm3ntBJ/V2k7D/fFWolkCCgGIbJ19OVKYRq4dk7XSRTWpS8cgeAaO/hfmw/eyeO8w3o8147ntXSxOb2RJqgRGckiM55GeyleCy5i0BIs+nqtsdW0sh/hoLrLlcCncWAIHizEPujGnXDjHByeLmb3xLZw4fCUz0xfhnd6Cfsul8OHr4EsD8K33wZmfQeiYmi5GTC0pwg8SJYIMWJIKWystYbstjnqVOfsIJayRJSJYxMhRIJE2VOZZ1Brbk05jZsScFDEUJn3mlyx8dRrPwU1oI2tI35hN4uA64pNr0cfFhy0iNV4E09nKxD86fSVz98mIxAdAgVnK8wXESfyGGHp2vylgiI8mZDvqf9xdf/TNOVa1pl37lcpYGVnJpi88hv6Nmwh94N1KnaWH1sPgBTAsodkC2F+golWm+Bc2JScKEDImz5IwoZA2ZZEw9DJNFdmAqLSy41PlyvcIT5cSXgGQ8IECwgfzCR3KJXQo26b1hA9m/1mA6NNu/hQZ0wVwcxHBiRw8H3gLPHirFeJOLZJJBkiZcdWxwruKeUX6qQiRdKZjqyvD9I8a1vlC8iOiORz/RLYCFkVSniP952BOxpk/8wsSD3wC/z19LP3L1fimNiFCJzNZCFP5MJUH03lwoAAOFpAczyE+kWMx/VQeyelCktP5pA64SB0oIDFVsPy3hErTh4rIHHYr0Ji3lBG4uRLfjeVEDxRgjGeTHlkLYtIe2IL/pqsI3TuM/zsfRzvykLUwJ34y+EixoMaLCOOr9hHGl3yORN5kdsSkTsaQ3IP1+/OOU+UqkuiU8K+pJsEU40hpaIl0Lf0GfvAvRD70LuZG1pO8OZ/MbRXKWtCGs2HKBYfEesjDe/PFPHvL1YTvPwCen4J5EtIe9ERImaeqH6R9paHl+eTeapIQK8Ln9NOf2p6jXkYShAkdMx5VJM46yVk48QCeL/Vx5raLmBk+j8jUGzBvXoN2+Hy8h9YSuTGflHTYuE0TeTCRp+xFczJP/SYdJIwoYTztQAGxgxZFDhUgFDvgPksHC9EOCBWgH8jDmM4jMZ1DaipLkTm1HqH01FpF5lSWurZ1jnPu87f6QRcvRfFDbvzvu4zQNw/D4qNKikXSCbwJqYsS7rWljlOEqVryLwfICxteTCtHm6hybcl1BL2YR39J8PufZunTQyze9l6CUxswhBluzIcDa2HiPBg9F/avIbN/HYmRXKUlBBzxqWzVVsmDeaQOFZK+sZDMTUVkbnKp743pHLWV4+JTuep4fVLOuwB97P8lMfW/SRw4n8RNBRg3V2Hcsp34TZeh3XQ5wUOXMX/wck697z0sfWEvyUfvBd8vgTNqftyQDNu1eU/5LhLmjS9ZVdcyMExK9O2aNAGSEgiqUSypEIj6kJiXaFJxaSxTSIYzHIHffZngnVeRvHUzocMb8U4XEz6YS+qmfNKHRBMWwPu3MH9oI8++/53Ef/JBMJ6EzDwyGjMptWdyTUGpyDYFEvlHqgYkAGFVD7ywj1b+fQ4pAYc4ZGEy8TAyDkBFtERCmrPEfvkZTt1Ty7ED2YRuPI/UbRcQuvHNzNy4Dv9tJYqRxYESElA4JMBJKHo+QFaCRMAifys6mId2MG8ZGNKpFjhyMCdzSCvKgoksGLdIvhPwvRRA/n/e3gO6ruu689b62swksS1ZYhFI9F4JgBSpFpdxquM4TuI4zngmseOZzKRMkkkmK7YlkUTvAEH0QoIA2FQs23KXYjuO4xa5SLIkq8vsRH29l9+3/vvcB0KyLUfkzGCtg/vKfe/du/f+73b22UcA+GlD2biLLXcQvffvXNOAxJLNtCrQlEYT482tMq2z4T14BJZ2FISMq5tp+hMfK+4QMF7xlwzAc18i8+URLs7+Oc93/ibnD9yB/0CTaX8+tgP+7kb4nze4499tJfP3BaQ+VkHi7hpi+2sJfqyUwN2FBO8uIbS/mMiBcqItZUQPVhBrLScukLRVEGuptOc65t6PHyyEljeTPfgm4i03E2wpZf1gLWsHGlm/R9a8nszBRuJ3KdOnJQT1vNT5qyyf+Esy/zIHF75pzTKcKlHZiwJ8zXOsQUxLE/TcZZfMs/KsiYDiwJIhkgkSS0eJe3OiTq8nHG3TF+HRY8RnPsgLd+3j/D2NRLtqibeVEd9fDa31cLDMUvLP3r2Lp0beR+KJU5A556JGscbhwf2gQKLrsdIUxSeql35t/l1nlWWZhMuZJxLmOMjUKS4x5q/8kMgX+swFibdUQlsJyXuKWLm7hrXWZvwHawgdrLQROVCJRnT/lRG7pwo3aoypYuyVUUXi7ioS98ikv2rodRubz68jcXcdybvcMCG5p+Y1LUT8YA0/bSy338nlEx+DJ78IMZXSLBGO+S17pOktBZqiw4YLYb6ry17JZ47napdeIfU//kRxh4oXFY/oT2BZWVkh9NSXiB95L/RWE7pnB/67dhBuKSPRXUeso45AazXLd5Wwur+SYGsdkY5dxDp2EWmrJXSgytzOtf1lFospPlNMpqPvYBm+gxV2zL2v5/6WcgKtVQTbKgm11xDqrGWltYLltjKbvF1rKcR/sIDAgZ2EDuwkcqCQ5b/dRuxAFfTshZ7biO/fbe5w4uBt+A69h8Sj04TOPALhZ7117EpbuwIAzY3YxKJueqOCO7cAK2OrK6XJ1YQhHlGdX2xjnkeTj6aok0skHr2XlZkPstZ1p5tMPFhF9GA1mdZy+Nib4WAeq+31fL/jLTx2/G8Jq/5PYLVCtE0+3gZABA7FKWGviu3HeZZ75Tp/yN1QOu1KTsRCmbt1FeXqiUozH/88q0f+C6udd5Bsryd9sIrwXXVu7uNgDb7WSnwtVfhaagwwAk3wYJUxUYCJSfhfIfB1JO5qMGF/NUB07mZQRQWAe2Sp6ojsbyB6T8OmY50BUlozdLDcgdPOd6DRb4dbq4m0ViNwJw6WE28pJ9xaZ77rM92/R+Kxh12aUKXqcR/B0DqBVNqyesvqGu/R44olcbZapDFxlzR4SsjccM+i67H9STCSKbLWmCFmWRbOP84LDx/l8dEPsL5/O9x1HRz4t2Rb30yybQfR9lLCHbUEOxpZ72hmtXMPa+1NJgTrrVWst5ThaynCBLqjkvWOatbaq/C1ubHeVk5uBDqr0PB3VNr7662VaKy1CBiVnG2t5XxHLcudVQS7y4h17iTevo1YyxYi+7eSaCsmcHc+vr8vspQ8B13GjL8vx/+3pTx5cB/Pz3+I0LcnYPnrNuchDS3ayL5KX28EyoYT5ZVkc1RZmySuiT4VXmqZg601d+ltNbSOJNLus5EL8J1jrE7+By7cs4fQQSmJeoKysAe2kr3rBuhvYqnnTr7R8hYufL7TpY7Da+4idCGObRvunmZpsqb+XmXVc3zzjtcJDLoZVw9qmXd3aznG68sTARKPf5zHR97PSx23EOtugP0lcFcByZYqp9F0we27CHgj2F5PpK3GzHuypYy0xsEKktLoLbtsDiXS2kS8tda5AXIFNkYN8dYfH5pA0vkbx7Yqou3FRDoKCXYWE2qvInGwieSBZvt8qKOCpdZSIr01JNuLyLTuINmxgzP3lPPs4T8h/O0vuDb7tt5bvrKIdWXWWs82GGwEExLcMACozGI15NJQXgmIKKg158FU0vbCSMdyRVspiKzC9+7n8tQfcv5gFYGOmwn2bCXYcxOhru1EOgWOYosFRKf0gVrSB+pJHqwl0eJoGekoJtSZT7B7B76efNa7K1jvqmG9qwrfxqgg0PmTxuZzquwzq91VrHZX4OsuJdhVSLRjB4n2PJLteaTbdpBsyzeQyE1ztN9FtKXZSkKiLU1mXaQYz/XcwYXjf0zyiaOQesqcVFkE0UmeZUpC5imSZEa9zlQR7pSznaS5l2zcQCWrLbmU6Pm1nkWPkkvw1KcIHvvPLB/cQ/hgLQw1EW8vhJ5yaKm1xESwpZhLfXtIffEeiDwJkRDZtRgRrb7wem35jLFayeb3MiSvQsWmp7ZgShdyBSBrriGARfwuKWGB1vJTXPiHQzw19E4utdVDWxns30HmYKkJtsy+QCGA+Dp2EeioNYGNtpcZgdMtpQgoidYqoq31hFqbCbU1GbiiEvSrGe1lRDrzCXXtINBVTLCjinjLKwHi76/h/P6bCR7YQranEH9bHi9338rlz47DyiWX1dhEkNf1UAwPu10qxcdg0jWyE4OV4wqq4Zssh+q3tCXyN+9nfeqP8Xc2Q+cW6L/BgLHeU4yvu5hgZymR9jKibRUOJC1VZq1NsYh2baXEPYUQ6nRKQdbB11lLoKPGhmggxaDvsdFRahZJVkm8yA29F+oo2wBSsLOMiM7tKCTRnm/AEDiSbYXud9sqTOFJCfram1lv32v8o10ZpUJz5852N/PS7Hu4/PB+ki9/BmIvuSJFT6/YOvtE1lZPJtTAIhs0bFyxzq4qV+DQkECvqdOKiCsARV6C7x8jevQPWWm5Bd+BagLt5QQPFpFRVq+1Cva/mfWPXk94+l3wvWMQugShhLUz1QLnFa30FGpl1mJa3++h9qcwfsOC2Ps2kSJz57Wdt2rVlNXykF6Di9/g/MJf8lLb7SQ6a6GlwPLUsg7S/rIYwXYxS2C5ApB4WymJVoHDnScLIOsRad1FtK3m6sBh1qPMNJ40b7CjglC7rEu95celcQW61FA9F+/eynrrVoLdJbzcUs3K/B/Dy/9sltHSfj+FOP+ql0XfpDCgYj0tNHMzuaYBVfekVObqM2T+aZqzh3+f8wf3EO+phwFljPJItlQ469DqrlfCHegsM42+3l1qR1+XE2RZSCmieIvucZdZy2irrPEuu29peN23rLpAJeXFwWJQMH5AI3/T2GHvyVLLXdH3ihfio7JZOX4Zz9pUSLiZv/WeEqwnoc8drLCM5lJrOc8dqOCp7lu5cOrPiH57DvzPeVbatTbS3jnmlJAgquSQVUCahnbk1ky7uWAu/ltVWbspa7li6mTztLlbvpk/4kd3NxHpamTlYzutRCjTWQWtO4jdvZPVrn28NPZBOPeoldEIDz6to1e1SQ4gWuJqrsBP5/R1uhxZHPsTsy01JofNXWhErT9jEYfg+EUyXz/GhfEPsNzZRORgMdnWEmgpIdVWRqK9nGhHtfnPCgDDHdVEOyqJdZSTaK90o63aiBpra8BGey2x9uqrGx3l5jPHOguIttdZABdrq7PvT7bWWnYj2V1JrLcUf28BL7SW8MLgu+BbRyH2sosHbG4jR4DXd8zRWRv4JBPSd45msWQIdX20tjzrPyTw5RFeGHgnZ/dXE+2uhqF6/N3VXPhoIamWBrIHGu1owtZRTaizEn93OWu9Zaz1lrPWU4m/q5ZgZwOh9mairbegIDlxcB+Jlt0kWptJtjSSbGkg1VJHVu6H5joOVMKBcpsz4ECJc4v3F4GNAns9c7Bh4zs2f0/C6FhHzPhT6/FVvNX1uRHsrMXfXkegrY5Ie4XxOXCwiMsHyjjXfRsvj70P/5cOk9I2bEnFGBmLm0MpBeqebHoAkQCbrFrHlpAp+JxLJKoaLa29rEpTnod/WWB56kOsdr2V1YNVrHeWEuoqIbq/kExbPdGefTyx/y2saf1I+Ix9uRZ+ZjX772UlfxY49LPXuWpHl4qzqzKnW0hxacxQJkI8mySrzmJqjLz8FMmHB3l54Je4IC0lgBwsgBb5+CUeSCoJd3pDgOmsNALHBBIDiKuWzT2+FoAkOneS6Cgi1tpMrHU3UWNoJQJIpqXatEl2oJKLPSU83Xcrvi8M2T2QuAgpr0jz9eFi42wx1NyBTMra0rhIUAyIeKnOF7n8xT7OzPwel9oriXfsINtdQLgtn4utNSx37SHc3kiitYGcQG4+CuzR9gbCNhoJdjTj79iNr/MWfB37CLbfYvctcCRa3RBIHFAcWDIH67DRUmvAEXjcqCZ7sI7sgWYyB1T7tM+GyjcSLfre3UTbmu367Pc7aj2elhPtLCPaWUKoqxxfbyPB3iYHkLt3krpru1ksxSyXWhv5Yfcvc+nBj8JLD7vFd7h17FpLY4pZRPQeO4B4JfGeJZa7agAhSSq86rqvyN3yPQ/fu48LYx/kcvcdrPcXEuzdYUWvvv3NZHpvY7XvLTx39I9IvPAZNx8jFNr+h2tuojYHyg2O/viD61wkL6Z6pkRXrce6IOKEslpxBiGVsvojbrHQi1/iwvE/4+Xe20mq4rc1H1oLoa2YdHspSdPsFUS6qgh31diIdNYQ66wh3lFFsqOCdJsb+ny8Q69fxeisINlZaL+Xat1NsnUPkc46Yp1VpNrqoKXGViKGW4t5oaeWix//b7CkAsSwt1xWMx6yllf3J3qHbLbdBfZJdSuRo61KhMuPE/nnKX44/C6WDu0h1qfrvJHowZsIthbh67mF4ODb8HfvJty5y1K48fZ6Um31ZFtUHqIcvzumW3eRbNM5TQS6drPevZvVnj2s9zQR76izCcF0Ww2p9hq772R7nWUb9X3x9l02Yu1NvHok25rg4G44sAe07ufgXlIttxJvvZVI+62EO/YS7thDuLOJcFed8TPWVU68q5h4Vz6R7mJLgvh76o23sbsLSH30ZtifDy3FxPYXs9JWw/Pdt/PywodJPn7CGuhp8ZTcHO2zmrMcOtqfqollRWwSz4FDHR8jiaRrqCehlnyqgjd+iew3Frk4+X6ebdnGxfYtpjwCrXeQ6GgmPngrz/btZfWLH4Nz33OVy+qpFl8hop3Jchm23G//hON1Lpz0lkjqKjWsVMXlq0PeSjMtLU5Z0VAYwi8T+5djnJn9AKHOJmgthjYBpJBsezGpjmLineVEuioIddcQ6qqzISJLeAUgAcmBqcJe0+uvd8QNIKWk2itwQtREpLPeQCIhsaCtQynREi7M/hbZpxe9+EqqX9ZDSdxrAYgaFsQ2Mi7GOJWOX36O5FcmuDj2XlZ7dxEfLCbRl0eo4yYC7fnE+htI9N9GsGsfa917We/eY3TUHEe6VcBW4FvhuUflcLAcWitMGYmGEkh9r7+nhnRHPrRth/Y8aN9JuqNwE/2dggp0y6XbZUOg0ljrabLfNDAeEBh3kVWCo203sfY9BoxQp65rt513BSClxLsKSXXuJNmVz/KBbfjaipECTHU0uMLJllrMpTuQB4MV+A7czPP7izk/9154TIHzC+ojxLpfcyE5C5GTfEmpBND8IZsLUTtWvysYNqsTVY1XDlnhs6S+Ps6PRu/kTEcJgf47ifW9zVnezkp8PWVcPHwnwS90w+XHbFWj1qRoN6t/JUAkIF6K0wq6nN3R78tPVMQvXzDnM2ZDPrde/fJ3OHf/3Sz33EmqvYp0e7mBQwBJdxSTVEamq9wAEuiuQ0wSUGRVBJBsuwOTgPR6gbH5fFkk+31p0LY608ahrl2mWdNtVXCokXDvHmKf/xisP+Z8YNMUihY1HbgRgf0E/fHaL4lGIrI/rZy+Z3mjQVJfe4DlsT9iuaWRbG859OWRHMonNlxOeKCBgK6vvYlE5257LOENdeWArRKRGgeSlgpXaqJyk7ZSUh2lxEXX7lJCPToWQsdWaL8B2m+0x9mOPBNeaXk7r7uCQE+VgUmA2jzCXVVk25QidQAUvaRYBFRZDSm/QJfGrldYkGRnManOfJJdO8kOlRnoV/aXEm5vJttzJ3TeCm210FHkZeu2kurbwaWeal6eei/+r4zD0nPWTUV0Ew0tHthoLqHn8mpUzRG1OrbcBIR6GKtNlarP7XNKKCWeg0eHeHn4VzjX2kS4u5FYdx3J7nIYKuXiR29ibeJd8Ox9toZGa22UZRQ4za17DTZfl7swd3ruY1dWk+kiNPRlTpSEbK+kIPoMkfkP4ZPpb62GgXrSncpW7STdXUqsq5RQdxUCiLSejuHuCmNytiMfjVhHGeHO8qsaSgCkuhqtvit2oIB0Z4XNxay1NhLurCfZowRAJckj74FnPmlBuXzai+pkbskI3ZX5k69Bop/+lughJ031SM56aFL1q5wb/nNSg78BPU3QkU+m62bCfSWs99ey3ttMuKuZTFsDaNXigTxC7TuMVom+GpJ9TcS7mwm2NrC+v8YeJ7oaSHdVkeosJNN+M5n2G8h0/Dy0/zy03QBtN5Jpu4lMxzZSHZrr2WlaPiZXqLuEaGcR4bYdBFu2ET6wlcjBrTYRmGrdBl159p2a95C7muqrRNcR7Kpktb3ceGZ866rZcLGk/OQl6PxYdz7hnmIDeKBzL+H2O4m13UpaZSDyKto10/3/km7/OeL9hVzuv4VzY79H5stjth+I5E9S5/jgLIr60qXkQtkut5I397oBwnNyVAakoaJEsqrdet7iy5WhO4n3lZEarLbro7MADlWS6Skhcuo/WlcU/d6lgN+aWPxsgEg+7Cx9TDiVCLl8tK039laKCekK6LM57Cn1pqzCPw4RmX4vSwoauxtJ99aQ6Sol011MqrfMABHoqWG9VyCpI9RdgRgn10Aj1lFyVeAwUHVUk+7Za5mWRPsO0t3FRLt3E+y5lWDfLgK9Daz17IaH/hYuPm7VnBJoFUSL2BllM67hT59e9poMWhXrxSfJPjRIeOB3Sbfshu46aC8i0V1CoK+a9b4GfL3NRDsbnYZtKyPdmUei52biPQWEZRm6a1jvrGe5vZHLrY3Ehn+RcP8tbib8wA4C+28k1no9dL8RBncQ6akk0NNowbK/bzf+3lvw9d1CoG8vgf49BHsaCPfUkuiuId1dTqazyEBLWx60bSF04I1Eu24i2b+D1EAB0d4CAp078XUU4tOkZLfc5Arjo1zmWKcUXLl5AbJmod5iAr0CUj0BuWQdil/2uoqL9lLo2k6m5Q2k2n+BdO92/B2lXGrfRWL29+Gro1bek0jEyCTlp7j2RLYNQyJFRk0Kvepbiagstv501NmSVFecouW/wIUfwmf/gtWeUtZbtsOhCtBE4oGtZFpuIDhxJ0vfOkI4eMHm8TW/8rMBol8zJSqAyCf3AiRVYOY60lg1pkCTm573Yhah/OI34B86OT/8W5xr20usrxH6q8n2FMFgGdGeMoK9lfj6am3ocbSnhHRXgQ1ZGRH+qob83r59pHoarYgy3p1Hoq+ZyNAd+AaaOdfTxNLEb9rkkm1Kk4b1jNv5QgBJZLwNQD3Cv96DeLLkbZlH/DzZR4aJDP0WdN9maxfoqUPaP6R4oW+XgTba00CyS8CpIdtZSaq/iPhgAdGBIhM2X5c3M97TZMK+2r3bQO6XsA80ERxsIDBYjb+vlIt9jVw++iHOHP0fNs4e+2vOzf01F4/9FctH/4K1o39CYOYPCYz9Dv6+X8anyb2DSgTUgaohhhoI9exgvXc7Po2ePAI9eYR68wn3FRDtKzZeiYcacVn/rkobic5qIt3VxtP1vnoCvfVEuutJdNaT6qgl1VlNQuf2VeDvyDfQxfpLCHcVsX73zcRbS4iOvxOe/gSsv+S2V1PK3eaO0taFUzsebHg43ky7BFp013DuWYqQukSav5uBHz3I+vHfZFUKoDMP5NJ/JI9Y+82sDNTy4vT7SDz9oMm5Xw2wfwbTr7MzDEa6misWxC5MwNmwMLn3BZKUazOpb9e65pcfZuXev+OZjrey0tVEqq+ajPzjoRLiPUWEe8vw9TliCiCRnjJS3QU24t1lRLsrr2pEumuJ9ewhOdBMrCePcOcW4v01RIf3cXloN88PvoXApz8CF9R0IYlaNkW9mEoaKPKvXzfzE8mo2zf3QAHlS//A2ux/InigzhV03pVHtqeGUE+TaXRpdwEl0V1FuqvSwKGj6p/8A+VEhmoID1YT7K8k0FdBeLCO8OEm/MO3sHJoHxcHbuPs0Nu5OPluAif/mNRn/ob0l/tg+XFYfdEbL8CqxnOw8jQsfx9e/BI8fi98aZDU6f9JaPwD+Pp/g/WOt7PWuY+zHRWc7ytldagS36EKAkNlhAdLLLGQGiol0V3gjRLEK4HE8auacHc9a32NrPU3EOytJdpdQaqrhHRnCQkpvZ5KQkNNLHXXsNRZRWigllRfubl0tLyJTF8Nl+Y/TPrpT0HqsqvgCF22tqZG282pYC3BSIdMLC3es5WCScLErIe1lbLY9N2L8MMp0vO/xtpHt0KPJlQriQ/UstZfy/nOBlKf/4htnaFCSc86/ET+6kUvBtFD11XDlZyYSXHoNYBonkSi4LLSKhE25Jnd0z5350k9usjzY3/AGWVAuivIdBdAXz7JXmmiUoL9VajsQ8dIXzmJ3kKSPYXEesuJyqpczeipMdMeHdxFsDuP9Y43E+0vwD9UxdnhvTw/+3vw9AMQPKPmUWSEDrOGyj5lbYbW7uOnkudnveFlW8IvEvxcK0tDbzd3KttyI3Tnk+gpJ9DfjG/AuTvR3l0ke6vMH870FhHvreJSSx3LvXsJDu4lNCRNXMx69834+rfhGypgbayWlZk7WT/1BwQ/9zGS356H5/4Zln8EIc0uez6HbmTzkIq1Xsra3FPNHc65FPczD8PXjhH+ZA/rp/6Wx9ru4OXBO1mbejuh6bcQOiw3rdw0vcAhPmZ6dprCS/YUb/BL9W3BvgazjP7+BuOfeJru2Umydyex3mLjdXBkn9VGXehsZqWnkWg4+/l8AAAgAElEQVR/FdnefOi5AQYK+VF7E2v3/xU892mIvuB6Msd91g4sF/ua/pYFSav61lkPFxXHrdG3XLJkWP2lBR25Pefg0UGCw2/D39ZoMd/64K2s9zeR6CojPfsr8MRJW1hl2bLXYPN1OQTphzcPez2XSVCmwIYKykT5K2ZOKTerk1n6Lv7PHOTy8C+ZH20A6dlOujfP5gCkGX0DNfgHqgj1a3a7mERvsZngWF8VVzOiskrdtUSG6vF1F+Lr2kp0cAvLvTfzo7HbWfrsRyHwDKgSQK37NVQ5l3VrloNZr1r0NQj0mm8py6LdWZ/7JC9O/h5Lw7eSHa0g1v4GGMoj1ZdPuL/WQKKYINTXbPeZ7Csk1b/d/P611hJLYEQE8v4qVjuLudRZyNJABcvje0k++B/gKx+BZ4+D//uuMZs1lfO692hNkDwMrU60NqNXeCO8RCPy5dWxRn661v6sQfSitwfhN4h8vp31U/+VsyPv5kzf21jpv5XwQJNZg2iLXJSd0J1Htnsn6Z4C45sUnK412F9DpLeOWE+V8TLRt5NY/3aiA3kEB4rxidejt7M2eDuXuvey1NnEuiaQ5V733ES2P8+U6bnOvayc+jN4+XOg5hCJgDU9F0A0TOIsw5rT+HpFHo1iFK2CxeKVVDrrMlsCidrePjrKub63s9631yo/ZMEZqiMy2Ezsof8Boe+49SuvwWRvHkT9ZF13buvMbh/QBag8YNXrqKElkl4DM9PC7hJXVEem89We9NkHSN37QWKDzdBbAN0qENxOoreA4EC5EUxEE1hkVQSSWF/FVYFDgBJAZJUiI40OeAMFxIffyMXuX+D8kXfAD+bNLEuRpiVU2nBFBYQJVUzJiniW8DUI9JpvqT5t7Z9JfPYj/HDoHVwYv43wdA3BwZvIDLwJDm0l3V9gbl9gYB/rg/vwDzUQGSoiPaQg+9+Q7f6/oP/nYXArkd4iVnsaWBv5VaKn/pLUI0Nw6VFYeR4CWqUXcQrSEw8THGUU1fFEbYDMRVYFhGuCLQ6aK6ljNkMsk7BCwbSWxKpTSuwiBB+3lXvh+z/CxdH3szr4a0SGfpFkT70VKtK5Ezq3G0hkSTThGeovxT9QaYou21WMMkWyHAKGf2gn64cKWD0kt62KwKFmwkP7iA7eRrB3N6tdtaz0VLA2WEZwsNDiVK0/We65lewX7rGl3lK4ygpq2wjZhNwwS2LzI86TMWWtxtsJFc+rQthlu9RoxXVb+RHhz/xPwlNvs0VW9NWTGHkLl/puJTX7a/C9Qbdy9jWY/GMA2VjgYggV0QWQNWclcl3APTdFDFjSmgkz7apYfQ6+0odv8teJDtaS7M0j03ezaVIBISyNMqhjGbF+N+KyJgOVXM0xMlhNeLiO2FgjoUNVxEeKyYy9mbNdN/Hi8T+CpW9btaYW7jjbHCezvmSbZ+bW4hvRN/Vr3WxF7b1cHOatRReznGCKMVqWPI9v+le5MHQna6O3EhirIDqyg0D3z8Ooywzp/oKDjQQGmwgYQIpJHfoFGLoO+q9zQBnaYe9dGn03a/ffBd/9NFx+CdS+VJtYumpw85py+UYrJldvrOQFSC07PuGaiGpBqbSvRElDQMlp5Ki2184kSce1BuYyRF6Ey9+Fby5wee5veLHrnSy330aqvxm6S6Bnh41M3w5iA4UWo4QGHf/oLoLuQpL9+YSH8vENF7J6uIjVw2X4hlx5vWJEhu8ge2gfkcEmUxKrY3VGKwZ2wGCxxSO+sV8n/vVJp2yzKfwh3cUVi3gFIKKAZ02sY13MmtWFBRPNk0QduKx+68JDxD7xYWJDt8LoHQRGf8lcPsYa4BMfgOQLr/BMc/x3nlWK69zPu5jDXUAOTnpNFyEYeBOJVz7t+YLq46Qafq+Vik5dfp7ww/2cHXsHofE6ksMFMFwMh+rJ9lcbIeP9+SQGK0gMVJE4VEnsUOVVHquJDCgoryYyWkh2ugRmKjnbu5tL37qXpLZgyAm4lTXrfpRk8NYh6Fb1unZa8jSVYd1r2qbHJlXm2kZtn75lEq77i0qlg0+yNPMrpMbrjPkM7CEzWGpZqdBgCb7BcvyHqu0Y6S+0ib1wa4mluePDNxDr+39gVtm+Yla6dnFp6vdJf/soqD9yKkTCr7okL7ZQutMrstNLEnaR2/iXm3U2fgm+Wrh65aO5r9h8NF4bcVyhpTbPsS0rLr1A5GunWZn7c1b63krmUJNNtikFnBgoIDZURHyohPShMjhcTXKwjsRgnfEgNFJKcLTERvhwqfE0018HfQ3QI6+ikdSAOzc4Uk5wtIjo0M0k+7ZDbyGx/jpWFn8bnj1mnktOHjPaPiOrqUFHDyW7tMAvpcV8Of6+KkRw9xeG2LPw0oNcuv8veHbg7Vwc3kd4uBr63kx48hYuP3GvWZ71RIalQNqslegkmU/FVgSQq/9TQB+3jSmTLqsl+QtFyTz1CEsP/gXnx24lMFxKWqa0Twvsy8n2FZAa2ElS5d6DHkCGy22WWTPNr2sckqtWTWSwnPj4DtKTO2Gshosjv8nFpx4l4CmZjXy1uR6qLXOiZcLn5dklVnpVR7lfAr1ZCiX2JI3am5tLXLJmz0JT2LJ3genbYLQYBhpNAOgrJSH/e6iapUN1rIzsYn2onFh/Hpkel3ZUGjw+mkf40E0wVE5q8DbiJz4M35kDdT5Mhm22WNfvmOUlAxTzKC70wKzrdYJwdTzUd+s7AllYkfepG9YXqhXt9+8n+sm/4/LAW0iN7iNzuIZY906ivfkWZ6WGS1lu3070UB3hQw2EDlcTGiknNFJCeKSE2HAJqcFyS/nTWwc9DeiYGagmcajczgmNFhAbzSc5tMMSOqn+Utan9xH+yt/C0jddBa8tB9e+NRc9d9/F4bpMgcSuVyDRCxvE0Au6mSiZ6Hnr65V85jTPzP8hL/TfYpk6+m9k7VAVz365i0T2slUeBsIZ2yrFyuGzKbJJ37UCRH58VN2RHCNFcU3uBF+C70xzfvY9LA3X45f57Soi2VNKpr/EtGxSbtZguRHrdYFiM4g8gMgnjo/l20gdqiJ0/EPEL71o/aZMw9iEjoRIvvmrAKL3tDegJyy6BblfapBpANF0rZUFrZJkBbc7t9ImPlLfmMU3uQ9Gi2BAmrIe+svRvfkP1bA8Usf6WL2lT02w+opgaBccbiA6VoJvpIz1Q7eROPmf4bE5iD1j20/okgJagOix2dSkFfFpnY4uxpvI3SwTV4MRTQLr67z14i6DKfdFTRdegDMPE/9cK6Hp9xHpaoYele5UkBwsZKW/gKWhYqLDlabUoodL0ZDl0BBPk0OVBohsfy301ZoHkRrMnV9CdKSAzHgR6ZECkoMlluZeGmnkR3O/SeCbYxBTwzq/xZGZ5IoJbA4I4pOWnOeebxw9OrhmDHGS2hE5ex7ijxH6p27OzvwmlwcaifQVsDJUxeOL7ye7/AVXaRyJWB4jJYUopyiTuVaAgPabCKXFMKFWKk8r6Fbh4jeJfKGFy7O/wuWhSlY6dxLuKSQ9VGNDLlZ8qIzkcAXxw1c5hquI9FZZajQxXkBoZIelSnnkHtsJS6tdnYvi9Y81i/cqgJjr5dYnyJNyNNdzJ4TZHECiasK8ip+EbdiiJg+XH/gYvom9ZEeKwbRlFQxUkRysxne4hpXRWoLTjYQOK41dQLa3FA7vIjVcz9qhCi6O30Lw/j+F7y+4HrWpVeu4KBHVzyp2MJAat+ROKJvo/G9dp4RkQ2leLUD0dfZF+iX9oiaLFXsG3A6za98j85VDBMd/l8TQbWQPNRDpL7YYI3K03uK+9HA+qeFikgLGSBkx8XO4iuShGlLG7yrSQxoVHr/L7DzFjAIH46UWQ64N1bI03MgLg7dx4fR/gR/e56p/CZNKh60DpQXmuHUlWrPuGCYllxvuJUcb9R9Tlx7F0IqzHib7T21cnvkNLvXWszLSzGMje0l8tx2C33W7pwVd11VVIgmA1+hiOQaqeEyz7M5HVDcJBZbL8NKX8D30V1yafSsX+vIJDOSTGd5FZqjBxSCHSkkOlxE/fJXDA4jqnGITBQRGCvGP3Q6PTdtKPtMwopRUssBiluHVAHGvb86USPu4lpVeYki3lnAdyjUxZfe59G1emv0g/pE9pA8JIIUwUAoDNSQ1nzFax/p4DdHZBmLjtbZoS6U3HK61DNDZwTrOzv82PPMA+J51lkPdTuKwnAGVUQokuq5XWJBNbta1AiQHsmgsQcKv5MXKxv7rcl8StlOvD0JPw2NHuXzs/ZzpaSR4uIHUzC0kJ6tJH94OwzeRHd5pdDCFN1xDXMmT4ToDilOApcQPbx5lBqjE4E7S45WsjdZxabiB9dE9LA81szKhXmV/CuvfMIumWbqgJnvNgiYIxZIEtPmoaTSl7jUEkisAEX2Up0yoF0D8AiSfg0ufI/bFuzk3/m5e7N/NC4criXzyt+Hl+yB0zpltTR0pJLOJQmPA1f0zAstMW38usVN7QwRA2wpr5ZbM2+MnWP/EBzk/Usn6cB6ZkV2khxotQBcxEyPl1zAqbeIpKndtogDfZDnB+d+Cs582S2baN+eOGu2037cAolvPaU73MAcQ90xkFUu89I+2G7ZOWZeJmZZdgR/cy5nx38F/uJnkIQXaO2CokKyC1uFGgnKtJqoJTVSRnKwl0VdtFQbpkRqWBqo4N/F2/A/fDckX0W5VyZTbi1zrps8m4Ly3gaqzICLw/3oLovtTuiIUD5MIr0My6LZ1SyZtbkEC5hcvZU1SzxF+bJYfHf+PnBt5C6HRvTBZC4e3wMj1cPhmGC4iPVxpFjIx3EB0pJ7oaDWRsTIi40VEx4qIjxWRGCkznqcOlxMfLCIxVsHSeB3nDzcQHN1Damw38ZFaLkrZPTEBseeNF/6E5ntjqOmDwBHJlUIYODyQbEpQiHaK2BJqdapFbNotLfgkvPhpgp89wA8G38HFsRpWj94GT6gJ9osbKT9tuKZK9mu2IMKBJqiypu/E3iiJeNQaNlou2vc80X/q5vyRJtbH8yzjkz7UQHywhuRIHYmRSuKj5Vc19FnNzCoTFh3PJ3Cknuin/gR834JUzK7LLIdTJUbkDYAYup2PoocCiAmjnlga2+u6pzcCApTmGi7YnkwkzhD+UgerU79O4NAuEkPFZIa3wKE80oPSms0EJxrwT1WxcqiA5Fg1GZVZDNSRGG/g0kgTq/d9EF76lKVmtVgzqOF13VjKuhovPdflGJsNIF6ud1PnFfe+g/Xr/S9L6U+qC7oSxk77EtdKJm9vMz1MaxdbpTUElDMknrufyx//c/xj/x6mmmBkC4y+EQ5vM4BkDlUj/uYAEh6rJDhRQnCygOBkvgPJaAmpw5WkDlfbMThcweXxKi6N1jgXeawRFKMNlrL2iT+Giw/bfiYCSMTbFTkmoklvmdVQsC4o5LIyjpfGOq0jyRFJezqGzkLoOXj+YZYf+ihnJ27n/NSd8MQshF6EkLrVO6Wv/Mw1AySkCgDjjITIbylUlZL7c/6BOpp/7xTnjt/B6tQ2UhPSMLVunmS0kfg1AiQ2ICJXG0D8c3uIPHIXxJ81Kyba/WsAousXMe0+vM+oTseCVs1S+5VilHU8R1pBY+QF1j/xF0SPvIPAUJ2lPpPDbyZ1eItpz9jIbkKTuwhMV3OpZ5tpS1kWWc7k9G4uz7yV4MMHIKGgXLu0OsuuBgXy/kU6m7fIXVCuosEL0MVvfUYjx/vXCw6dL1Bo+zaXtXOFBrZE3wiSJOpXax53LYpMNB2npt688HFin/pzVoabLDGSHruJzIjc5zIyh2o3AaSW4EQ5vqkifNP5BKbyCU8UEM8BZLiWzESzxWMrY2WsjpVZZTCHqmC6kMxoAWfHb4UfztoqwFA8ixbwqVWpFzo6gOi6cgDxKj1yPF2JqHWQRyvpAK1n0vp47Wf/wuf5wcRv89z078Czn4DIefArRPBCh/8VABEBjY8miRIqEdxlRmVZzJu5+AOWHnwna7PbiI8VkB1zmiI62kRsrIbYePlVjcRoFYw2GUAyR8pZOnorPLvgUoK2/NW7OF2gN/t/xYJ4cYn3+gZAdK4pU1ebZmWllni/DFllRCKw8n3WTn+A6HgzsdFdJEcryIxvsZEYrSEy1oR/0lkQuRdyKZK9FWRHmgjP7GP9vt+HMw9BTFsKXJkMywm96OcEXxci5lsazfnZnqXLzYM42l8NPLzPbNDFeRf6Xl2Hl7qzZuY5QKrmyWbr0y+Q/eExzp1+PxfEg+N1ZKdrLHhnpJHMcAMRrSuZqCM0WY5/upj12QJ8MwWEpgqJjZeSHqkwDyIyupvg+C5CU8XEJgqRC8pwBRzeCqNbCY5Xc2bud2H9Sbt/TYMabHVRqo4wLSiKeRbEAOLmgXQfup/cJKnOMsKqIkTLrtWX4IXPwsuftyXSBC+5LKzR2GUyr9mC6EeNSTltbZNsTgvq4izRvvQjVj71btaObUXZJgEkeGgXIk70WgAyVgEjDUZUAWR54S2e2xIgq15KuYvTtW0IgtwFuSqvBIi0qLkZ3mdkPdTcOKNcu24ivQTZFcgmrEo2ePp3iY9XkhitMx86PbGF5MQWYmMVhMZ34RNAJmtsXiAzVkFmsJzsaCOBI3cQ+NyfweVvubZDpvlEqNyfLjYXdG4Gh3MfROsc40XfawGIZEyf1y1vFiRl8DSZakLnJThU7xTXHmuWRVsmtfYtVr8zzMr9v41vdg/B8SZSU3tgtJn0oVo313GoiNBkqQeQQtZnigkKCALIaDnJkRpCY7cYQCQX6fE8A05mRHzdbgBJjxWyMv1Wkt9dtNWAiiC1qYEtqLK9F0Q3j2YbINHzV96XQKI72pBV3ZdcrrWXwH8G1IVRnek9vidto4fAtbpYLh0qAhuVxUOvRFkmWV6rcXN9hfXPvIfgwjbSE8UwXk9guN6IGpuoJj5RcVUjOV5pAMmM1pIWQE7/GsmLX7OLyajn0WaAiB4mCDmAvLKuTEG5neEdBBYtEEvZPuhx0iq3kfOTUYDwfSKn30VqopDkWC26jtTkzSSnthKbLCM0VYt/ahcBjaEimKgETZiONeA79g4y39TWYS9dabG0YSFUfKdgfNOw9xw4JNC6vBxANhhu4vD6/+m7xCMNPXb/5SpruCSFZNC0bjpDMqJ9Gl3NV1Z7eYQeI/Hdfl4+8uucGb2T5Pw7SE40kzpUQXa0DH/fNqJTJQRmSvDNluGbrSA0XWE0yoyVkRqrJDTRbO5oZmIHjG/DvS6LXEh2fAeM7SQy2cT5U/8NVtRVft21LY2onsxZCtHFgcRztSxod5edo9VmK2LnC0O54SlPR1vJtFqnrhPj8rUBxE2oOR92g3Oe3Akg8qWNm/41Ap97D+GF7WQnq2G8Gd9IHb6pOqKTlVcFDoFKgpk9XG8WKX2kEt8nf5ek7wemDV1libeQRoTwgKvSGGvRLSJ66lcHWQsTEB1EZ7OEWmvu2h9pP1jLM4m6y08Qu/dXyU7m2TXEx6tJTu4gPrWN6HQhgekKA0hoopFAfxHYPZeQGq9hfUFroxdd5xO7rlyGKpel2lw57SYFxUn9bE7bO0Y6cXbCYRLyuv9JeLwCDvt+A6atu5CFdb+nCnJHpjjp+JrbxXajFiFkmw1d/Mzf8dzku1mefgexid3IYjJWSPzQNuKTRYSnyvBPV9sITVUSmxAAikiNlxGdqCc6UUt2PB/GbyY1XkR8ooz4RIkpICYKyEzV8OKht8D3pyFzxs2Qp5MkE25qM4dho8iGS6pXHY2kSGQhddQ92/2IcJ5M5N7XOe48RVtrpFi6doDEbPGit6xOv+RZkZxmMuD4zhL6zDuJzefB2G4Yu4P18SrWj1QSmS4nPnV1IzVRZQBhvJaUtNMX/iOpxHmb6jKOW5m3R5GfBBBHQyPYhkshCnrpdVcjKosjpyxiwaxpnaUnSdz3yzC5xZIOiYkaElNFxGfyCB/JI3CkGP9Mk9X6hAcqYKoSJnaSmq5m7eR74dIjZj0sRjNGea7Uj1kOd4GbgaHHV0AhhF39Xw4gpsjEOE3yqu5J5bCebtEuUcpxpQmTZp1Map2stxGnlWQkLpF+/iFevPeveGb47QSndsOUXKRtMJVvHkNsqsomTAPTTYSn64hPVpKdKLKhpE1istImCxFoJouIyw3zZILpMpit57JKWj75XyDwLbd5qZKLXqcTiZ0DcS5m87J9KhfxQOKx1cTxCv3c5/SegJGznTG0DEJrT66x1EQWRGJj+5YKGDmYZlxgJLNmV7T6FIGH3k78iCaVfhFGf5n16Qp8CyWEZ0qvASCVZIbrYKLBABL/yp+g7rjatsCIEPfUn2dBHCFkQVSyobUhTric0Hno1sEDSMx2kEgQsSSn61huMeHK0yTueztMv5nURDnxyRoSk6XEZnYSnNuCf26nA8jEPhKH60ECMLGV7JEqC+7xPWrmXfS5wlhdjOcimJZxVkOv/jhzJQhe4G6IvTqQODdSFlKiIffOS9/oorziA4UdcjWdIxYkk9GcTRS1EHVLYiWlZ1n/+gTPTP8WgZk9MFsGozfC7E4k9FIgKgwMTu4jPNVIfLKK7EQBTOTZMauYZLzKKZupAmIzBUSmK4lN1pCcqoIj9eZl+KZ/EZ5ZsK6Y4q+WO0sRS+w8Vjq6mKLRO1dcRd2SznG8lkA4emfScVKag9K0ne5JsZadp+RJ+NosiH4i7NUt2VoRXamG139VP2SLlC79M74H9hGd2goDvwoj78Z/pBL/qXzCs8XXBBAFhEzuMoBkvv6X5hKpKMQAoopbj9l64bUB4gmJBxB9TABxu0gkCXo70Cq2Y/kZDyBvIjVZagCRFRRjg3M34p/bbgAJTtxOdnw3jJfD2BthoYr1+z4EgadsT8L17BXmOsY5Qde163lOs+X0ziuEwFuC6kovrg4gBrLkZbfc1eusLtdYbtcGeFXMqeBVrVWTMdsXPpnN2PqSUEaWRWCKwvkv4//8X+Of20tmpgCmboLJmwwgqnaOjN+K6BGe3G0AscLSyS0wsQ3GC0iMNxKbaDQaxmbzDCCRqTqbcE3N1MNMoZXs8MW/h7Xv2e9e2HStopX9SXFYSY7WyXi1axZPuSSMc8oEeDFaMx1K3/vNauo2teWH9iYRraUYrimLJeK4qlepm9QVjnp5Z110JrgK5z6P73QzkfGtMPjvYfRdBI9UET5ZRHS2hMR0xVUNaRdNFqand6G6oPQ3/952JpIFMYFTJZ4eSOF6LpbsnWnMTVksCaRbn+y5il54ogU4mqXQ5JQ2rzQmCCArT+N78L2EjpQRmakmOlNNcrqE5MwO4nM3EDl6M6HpZqITt8FMswlJcuwGEieauPTgn0LwaaKRDD51c93kG4spulwNPdbviY25c/SaZdqkgZQ8sGXQYrbe0U16msG0gycvGy/pwjedY0pUM2JaA67da3WPrqGF6KfH9jXWKTIO4bDtcaLYTtfn9nF3v2yZvfgZeHKeS/O/juIM5gvIjr4BpnaQnKogOtVEdHIviYm9JCbrSczuID0rEG2HiWKSE7uITzYRnykheiTfaBs8Us3KWBXRiWqYyLe6r8SJ34LlLxKN+FHxT27OSBRw9/cqgJgVcTQSONxw0xHO/njrnTIpuw31IVGJl+PDNRYrOoaI6B7hRXRx1cOKbVGWUlnGAmtzjXCsGCb3kZ64g9hMA7HZSpKzVab9FUO83pE8UmHfEZlrwH/irfDUIeuWKIFywuTRzNMIrqAy559urmLdJFtecJoTTgmC7iORWkcVpbZ4LPUCgX8Z4fuTv8ZLU00Ej1TAfBEcuxlmbyQ5sZ3YaAPp0SY4vMOC+fhiHWcW38aTn/rvEH8SUhHXpdSzwB7ZbNmAkVPVCd61KMMmbaYNZ3Rebuge3TZwcicEGC91vQkLDmy50hmvXkkf3PgSCZRgqHPcy3qmx/Zn37mJxxuAE82SxBKCk1Zpxmzb7PVHOjl/5JdJzVeTmLmR+PxNxOa3E58rJj1db/FndnIvoYVCQovbSR0phekqmKqxuZTkbA2RuQr8i4WsHy/EN1tCbKqSzFgJmdlaLszW4//a39iW1NpuIncbjre6MV19LprwvAJzEV1Bbe7WdZZS+y5oFoCuKI/NeuaaLIgxRBpG6BPh9E+/5S2V1LwnqQvwgzn8RwWQQjIzDTabnJzSsYbUNQAkfrSC8NFSfAsNrJ78JXhyzNYY54TCBEgaT4v/jLEyDXK7LE11BUSbCtxyBLxCeImJAljtuXfR7X+XfpHVJz/J149+mOen9xI6Wgzz2+HYFpjZ7vzqsToYb4DRfDJT+QQWannp1Nt47KE/Bf8/u7XQlkN1gqlr1hANbWwIonxht5+eSz++2qpIGwog7p7sS/RFHr/d/UiQ9D06Ry6Tlh97aJCMXPWfmO0t9xXvEwkyT36OCyc/RPBYI/GFbYSO30R48SaSx7aTnVFscgeM324AWDu5ndhcmQHDgvHpCpIz9YTnalg7kc/ayTz8R4tIzAggFaRmarg4V8rKlz4A4ccM2LonKQ9XO6fr+UlDVBCdHaB067KQOopUr/V3bQCRiJmZ1yo9XaT3i0Z/IVZFbpccQObqYD6P9Gwt8dldJGaqScyWkzxSdtUjfrTMABKY9wDy1Ajax0TXIZJoCByvBZDNbol47ATKASr3Hc790szrZZArkT5D4uzX+N6J/86FiWYS09vJLNxIdnEr6aNlpGeqbGUjM7Wg1YbTVQSPbmflvjqee+B9ZF7+uPWINV7mfkSCugFUWQyXTFBCQWlpCYGbzPQUkWcldK9OezpQXLlx933ufrQOxgmSA5IHEr2pm77qP31BiHD4svsa9d4JvET4iy2sCiAndxJazCO8kGcAYaYURm83gAQWClk/uZXYXCnpmQr0ngDkAFJr1mP9xA6CR4vJHK0iO15OarqSpWNFnP3UO0mc+bR5C8o+ilqiwZWkb26idTNxHUAcPa7oh591+9cOEAt0AsvFCHwAACAASURBVF4a0HNQDSDy9TU7uWTNE/xz1bC4hdTRSuJHaokdKSF2tIjUXBmpo+VXNRJz5Q4gC3WsnnobPHXYNSfecE2c16FGFFcsiLp7iICe9hCFPOHMvZYja+5oK9ekb5JrrtgtcxF8T/PiQ/fgn2qG6ZuIL9xA9MR2YvMVpI7Vw9EaOFoP07eQmakjOPMGIvflc+H0W4g92g8KjqXCcppckm4AcRZDOzA5cAgoEgBPlj1gGBA8cus67R5ymkFKVOd54NHnlbJWulpAM8DbZJpnZtypV/FfQpnAH1hxIFWAHPHBE/dy+fg+Qid3ElwsIbJQQlru50wJjN1mbnZkoYDAyW3E54tMqXCkBFVDJI7WETlWR+B4CYHjRUSPlsJRxSBlpKdKWDtWxEsnbmftu4fdQigVx+aUYY5GdvOeW5h77FVK5EiU+8z/ZoCIE8p7aFLFZ8Q3ThpAVFK2Cl4M4jtWCYs3GCBiR6uIzu0kOpdH4liJA4mA8jpHYq7MCBicr2P91C/C04OWI7/iu3vh0StcLKUynWCIWBuC5Qle7rM5cNhRfXWk7lMh0kG5WW59gf8bo6Rnb4GZm2wSdP3kTgKnikicqAbFXEf2wuxe0keqic38HKkTb8Z/ooLgwx+G4DNYxzNxSmT0LkYa0c055DSj6zbzyuvU5KJLY+r6DPy6fu8ynel00uLcCvUOUBCudkculnHrXTwUXQU0ch+JRuOEoxGi6q6iOjW1Vlp5nNVP/QbLxwsILFYSPVZO5uhOmCmCsVtgcg/RhZ2ET2whPr+T9FGBxAEl7gEkuFBOcLEMeQkGkKkyMlPFBOcKeWmunkuP/A3EvmcWLCfsuv0NgdeDVw97yVkS0cVInruRn3K8RgsizrrOJwnW3Eo7/arlkj2AJNetSZdvvhxOXI+EOjJXRfjYzYTnt5GcLyZ9rPyqRvJYuQeQWnynboOne627R07IjWCeT29CJMmR9XAGJCeTzq/yAGJafJMF0mSe+i0ZktIxsjGf279Cu+I+/0k4/jY4VkDweAHLp3fiv/dm4qdK4KiAcxvZI3tIzZaRnXsTyfl/R+z4Vvwf/yV45iFYWbkCDuOsxF0mxeXvjcE5wc8hVs+lqY3usjLuE3aJuuHceXbzkhH53SmbrwqT0tq8jbkrJSDsZ3+KcPxrXg4EEpYWjWT8ZLNhL610kdS3/orLp+vwHa8nNl8FitNmS2GiCaZ3kVjIJ3piK8mFPNLz+aTnC0nOl5I4Vkd0vp7IQhXRhQrzOCw+mSknO11kivXisUouffJ9cP5+U845gOjWc5jYeJB7IXf0LInTjD9bQfwvAEiALGvE8RkjctpL5lwzkWgfusfvZW2+0gASP1ZOeK6K0PzNhBa2kFhwhBFxXvc4Vk7sSCnh+Wr8p/bC0x1uUcyGL38l4HXC4Pnem+JZ08yirATqp4FEk2WpjCtcTGsWUdYkAr7vwIPvg5O7CJ0sY+X0DgL3XU/slGKtW0hO3E5qtpH4bDGczCMy8/+ROnG9CU30kTY4q/0qvASHSaMuwAuk9VDXJR2UG7lrtDe0TZlK9zyA6PP6jM7JBV5Zl/uXACkolc3JDfUEU4pep1/1nzqze1NNccFO/bb0Q9EAnJth+aFfZvXELgeQI+UwXQGTdTBTRXp+J4njW0kubietGGUhn8R8KbH5GgNIdL6GxLEqMrOVruTkaDnM5BM7utOs0tp9byf5WDdwecPF+tcCxBFKZ2uIaD/97xoBoh/QlHzAupuIEcZMS/OK8VrXHDKA+I5Vw/EbiM9VWpYiOJ9HaH7btQFEBD1SQuRYNcFTzfB0K6TPmxrZUBgeWFyFqjfP8WqAmEBtAoho5lkefY/ejicypHNd9bJpsmqynDxD8rN/Dff/EqHT1ayf2kbw3jcSO7mV+Gw9sfFbSM40EZ0qgvsLWZ/8v0mduhHfXBmB0x+A5z/t9loxRomWHjhela7dDN7cfclF0CdeARAxW7PhueGVWogvSn1LdnNDDpyjyWsLyE8XHUejXN9oS0OnI07m1Jwv+lVWHv5Dlk7tIrpQDrM1lso1kBwpJDOfR1JJjQUHElmU2EIpUVmO+ToDSvJYFdmjFdYBhvlSOLKTxNGdxE+UEzrZSOSrfw7ZM0YH0ws/yYJ4vNywKAYIj9b/+wGiX48Ty4RYT/qNWQbnhJo56JKV+QmR+fYCsZO7SR65gfBMuQVhMd3kwg4SC8XmZsnVev2j1Fyz1Ik6/MdryHzzv9uiJk3+iiAJW5TuzK6yOJbl8LSxCb64bzNfV5ILRj9PCnPCuPlohNbn9GJqDZa/QuJz/5W1kzUkPr6D0OIbCc/dQPZkDSzuIT27l+RcvblWodNvImiZnRpiR29nae79pH74WQhdtM4dmpHOxCOkUgnDhG4jlFaoriDbyZ6oamlgZay9+Sa954ot5ZopFrgyg6wO9loguBJO4Yu7NKeyYQnNxMuGaH/FTMZGNpu1phGvCYrNb+bcVytHcXM1Ya3I04XFniHx/S7OLNYRO14Es41WxY2s6dFtJI9uIXnsJtKLWwwoscWdRI4XE1msMJAkjtWQnquCuTKYK7KJR+YLyS6UkjleRvJEGZfu/RVY/66xIpFME405axzTfSZc4bW9uZmBUj5KUGyY5ddWENdoQUzCiGcS1oI+Z0HEn1AsSkp9YONr8Ogs8ePNJKevJzpTQXxxF4lTNQTnCzyAyM0SQF7vsZjssQoyJ2os+E1/408grd1rnQCr2lMEEjDcDPoVCyJBs4kiESznkohWr9I4r6Cthwu9Zn8qsfA/D49PEPjEW/EvbiN5agepE3nEFgvJnqgnfWS3ASR0Mg/fvVtZO1VOaLEGZitIHLmdcw/8BclnP2kd/qzDeWydYGgdXzyCL6vlrqou04y+A4mu29DjTa/ruQO/nCclTLyOJBbHZAhFde+OJFpfZDeogFrd1JVlzAgkVw8Q0ctau3qxTiBqUmA9gONPHuPsqSZiJ/JhugkmG+FoPtn5LaSOaW7kJjILW8yKRI/nEzpeTOh4KZHFcmQ9MnPlFt9xbAcs7iS7WAiLVbBYAcdvZv30bbD0Vbu7dCpBQp0WBPtEBtWpWkX3qxmoC/4/CRAZCi29EFnECAFTfRuCwSDx8GXX3PnRw8QXmkhMXE9itpTMyWbS9zYRPFpCbKHYAjbFIjKzr+coUDFfSWaxitDJEmJf/U9uua2shGbPdbTyAjkUShp4KterYVRcYieKkhqmXTwUGKmvIOLVdDaQ6PxYABUfJr/631iaqyB5qhTuKyZ8bCeZE5WmBcXs4IlSVk+Vs3KqjpAYfGQrHMnn/HQTa5/9Y/jBGCz/o9sLnCTBDJyNWR5QuUDWSVviN6kYSF0y5CuZPCgF7FkOA0cOIK57pLXv9DS9aVNtkJk8A2vfIqG9XdIqQLxKgIg80jkxZ3k00+JPu6JOKZ3Ui1/k4ifuIHpyB8zsghlNFheY1UgJJPNbPIBsJ7pYSPBEsQ1ZEiVgsnPFsLATjm8nfSKP1PFi0ov1MC93fQvhU03wo4ecxUyGScmdV0ugZBrNwUrvbeZbjqWeuvCsiGmNK2+96tE1WxBrTOilUS2TIqaF427f8PQyxB6Df2kltdBAavx6kmq1eV8z3L+X4NEy4oslJBcLrmqkFmR6nUaJnCok8MhvQ+AJd/+qp/EQq5lm5W9+HCAyyR5IcsccRV9FKD3NvbX5aCDMLMOzR1i7/zfwK7g8WWL3FZ8vILOQj64zdLyW9ZMNrJ1qIHq8DI69CY69gfjCDpYWGrh477sIf+0gXPoyiG4qlMzmaqOyrKvlaTpMUn2e1FZJpR1S3RvaMGdBZEW0j4abuNVOssJTStZUiQVtDbD+T0S+P8xL2l8ktXL1ABHwFJtph9e0RSH4iFoiQG4gS4+z8vl3EjldADPVcHQXmYUiEioxOb7NgMLCNjKLO4ieKDElFzpVQPREAZn5Emc9FreRPbGFxGmXHUwu7IGFBlh8s3khsafnIbPutHJc27ElkTVRvJhbcGhu6Sb+/R8FiMmV/H1ZEYFDWw2oc7hptYtw+eMkv/6nZBY12XM96aN58EADPHALIU0aCiDH869qpBYLYEGFcaXETu1k9TO/CivfdvcvcJg/Yo1trC731QDR5JmbgZX4vwoor3j+E9DiEdw8CpWB+n5A6ls9nJ+/Df9CGdn76okc2UbmxPWkjm8lutBAaGGvAcVcjoU3wOK/IXP852w+QDHM2ifehf/Lf0/iBydh5QnH9HQuThCYZadlOhRnBLxq1Zya1A2LAa73sJ7p7KglGFQ2HHYZvsSj8OQQlz71+zzzwIcgffaaAGLOfjwqBFpdmCJRVQObcQudIfyP/4nwac2GF8HxalKL5cidSp/aTnrxBgwgSvkeLyNwqoTA6R1ET95MZqEA5nfAyTeTPn0Dkfu3E7q3jMTx22H+Fli8kezJKgL/Mux6XqmJtxoWqrIjE7e19IpBJQI5UdBjBxbRUu6F3vnfaUGkJfT9qkpVoy0pMRWvZdVHVcszn2XlsTYCj7wbTpbB9PVkj94I95fDx/cQPVZN4ngpqRMFVzXSx51Pmj1WQvz0Di4++Fay5796RUHYvcv9CFi7HgtkJTWeiyWAKA5xhHMpUWdRRLhXj80AcoDR7UtDxzQ5Jq1+8R9Z+9yHCdx3KzxYR3zuTWRP/QLpk1vMgibmbyG1WEXqxE7Sp64nffrnSJ1+I4lTNxK7txj/6V1cPvGLXLrv9wl/tRWefdAqf4mqe7sAoXuR6GkyUW6jHO1NfN6It5TCzcHFuRpypUg9C+cWWP38B7h8qonAI38AqeevHiD6cbl8aW14rmVVMes8qcXJcnYkC4lv/iXhU+Wkj26HUyU2iRo9XkH6vjxTHizkWWwRO1FJ8HQ5wfvyiNy7jezxnbCoz7yJ5H1vIPjAVgL3VxLVvNP87bBwo8nU0lfarMsMGVUmqLGGytxjqLFXPOEqEF7NSSlKDVeaYkLykzXgtXY1MZ8jnbGsogxHPCIOnQdeIBl8maUXP8sTX/gAlx/aC6eLYe4GMnM/D/fmw8ebiS/UGkASJwq4mmEAOV4Nx4qI33szZ+69lcTL/+BkW0DYAIhm+VUOc6UOX6DQXI3iEEdAN8MqojnC6VUv5bWhbXKkdmARQMICiAJmE9RlePE42X/4ANmPF8Dpf+uAcCKPrCzdXA3MVZJdLCB83434HthC7BPVhE5W2ERj/HQhiVOFRBaKiJ9qJv3J98BjnfDsKbj4bQircYRL2arzoqINSxbqXnVpnjHR01xaN6wgOp0iE3yJ1I8+ZZWwZxabid5fAI99ANJPXj1AFIOpEljzHxpErOOI1uPo+lTgmf3ufoKnakgeeyM8kE/8RD2h4/VkH1CM9iYEEHkAkRNVBE6XE7h/O5F7t5A9ngeL2+D0z5P4+C/g/8QWfA9UEznxKzD/Vpi/CY6Xcf5zH3HN4DLnXZeS/5+99wCvK7vuey+HBexEryQI9jKcrhLLlhRFUYrlOLFjRYmduDyX51jOs2PHzy+2FSdOHMe27Eh2JCeO7TiSZoYVBAESIFhBdICoLGCvYEXH7fee8sv33/scEBwNZ4ag9OaT7MtvcV/ce8655+y9/qvttdfWPimKEmn9TkZCxGrSQC4GvrICFyFI3hUgQc/qgUKNoFZkR/3x3p/5zp6nxFgJEa2nMWkG3g3w+0kPH+Rq62/Q9+bL3Nu7GjQgXyvA+4vl8GYF7H2FzNd2PBNAHGkQOcL/ew3ZXWUMv/4iXN4T2OjGQw8yYZXwJ6n7iOctQEJwSAmGAJEm0bO9Ez0CiNZsG7lu1JBWil2G/t9hatcLUFNBamcpmTdWY8D81XXwlxvwv7aO6d2rGd9bSaz6BSbf2GzylvzqCqguwH9zqZWuezbh7PsY3pGfgP4/hLuHIXnZaESbfBgkmYWKZbbVYMZPs3j3IN5L+vrr3G/+Ja7v/hRX/mIb6Vr11S+aiolzdtJNBFAMoHX00hvTJuImD2pC/aF0mL4vMfn6R0h+dSXUFJHauY2YUnH2VuK/XmCjU1/dgNUgG4jvKie1s8Q45Xy9BH9XLpm9uUxXlzG1dwupNz6K/7W/AV8thTe2cefAL8Jkn82yVgV4E8lThzikJLUDs0qjKWGouZ/3Cg6dG1FKulmPrQ7VQ4V8IR4wyzoVU5ddp/yfADPqGMMWWjgfOMNG2GoBzg0YP0i29xe5Vf0Sk3VFZGqWwe4iUDRj52rYuQHvzZdxRbu24O5eP2ealk9TvRn2V5KQ3d/4y5A6C9nbSqNDm6rIbZVE1eOFzyAnUo8a4j1sTY+GBz2xtUfpfHlbijKNKe6uPsjEYLIbzvxH4oe/l7Fdm4ntrSSzbzXe3rW4O9eTfXMLqV3Pk9i9nWT1RpLVlST3lhDfk0ty7zKy1ctwDuTi1JYQ21/JWPU27u77EMO1f5/Rpp8ideY/wJ3/CeMHIN4N0SsQGzPBEeOiyOSTuZHthZt/SLL9hxmv/zATh17l9u5XuPS1DxNv1sKtertPn32cp/9fnRbyCbIcVJrVztZLYpsgwnAz0T0/QqZ6De6BxWRqVxPbWYH7xhp4vQp2rjf84O7ahLN7A+7uKkPsqsLQ7tW4e1bj7Kkis2cD7s7n8XZugzcrcXa+yoM9Pws3DgfFHMaIE2XKi5NUiXYDYPGt5sBsJFOC0pimM/f+zo8dUYRnBiB62BAkRgJoMY11euxcQdghYg2hNGvX6MjgNQDRhu7tOKd/k9G672GqutyCozoHdufDLgFkA7z5PO7OF3F37kAdM1eAqENjr6/F27ce9pfh7NxItu5HYKIR3CtkmWQKz6h7yRITZVOniYLGasp37qQnfasu0nUFQJXFlDa1/TAJY21w7U94eOQHGa3/AGP7q5jcW0Jq7xq8vZtgz0uw+wXYtwl33xqye8rJ7Com9WYuyTdXkti5koTs75oy4nvLGNtZxv3X13D/jY2M73uFaP3HSDV+H+7Rn8A98StkW36fdMefEe38KhMtX2Hi5K8TO/rPyTZ+GK++Cu9gCfGaNdza9SpXq/8JsbN/CpnLgTnypCd8D58bgOjBh4F7xoQROyjEapK0bvUxtednSO5fS6ZuIdmDhcR2FeMKHPJLJSx3rZ8TJXe9xujuH4aLX7fzX0wHAtEnrXQgSSyFMs2CMIX5Y0E0U3wdKAMB5R1ej4d5jRgN5gTCuQFdaDaZC4ZXDxlCnSHtcRku/1eGD34vt9/YTqx6E44xr4pgT67VIm9uw3vzRZxdz+Ps3mKkhrNnPXOi3RvI7N5GRtJmf4H5reje74NbO8G5Y6I+kvCKqsxokFByCCDmed+hd97tq/BainKmPDzVtVXXCDlmYdZNkre+yoPun+PSgRe5truA2IEiOFQJ+7QKcT18bQN8bRN8XZNfWyxpnuSrG+z3e7fDni2wax3+m3Luy3Dk4O4rxzlQydiu1Yzs38BI/YuMHHmVh0df4cHhrYzUljCxbxmxNyL4e5ea4yd27uBuzQ8ycfr38cds1XSL6Hd70Hf4Xs9rQs1Wg0hzPAKIA7d6mNr706Rq1pOpW0TmUCHx3SV4byg/7dkAkt69g5E9fwuv9z9C9Kzpc5Ut0+/LjEopcBIKRFum26xR8iXcQ0Vg+PnJzxeZYX7zRpe2mmHGHwnBYa4RRnoeuSjupBbxqlTMLbzLf8ndY/+Y4dpXmTrwMun9L+Lu3o6/ewPsLYU95fi7tuLu3kF2z1Yy+zbi7t0wd9oj0+pVUm+sxd+/HKe6kLGd3w1n/hhSDw2zChgCiRk0PUPIwCbi8+SOeU/f6Fq6sCjj4afTpNJZEloKaraukFq5Rfx+Dfd7/jW3Gz/JWN1WsjUVsEtOZoExFYxW3bUZdj0PO1+ANxTnfwG+qkVmz5tiD3x9E7y5DnZXwp4y2FcK1SVk9xeTqCliqjaPsbqlTNTnEDu8AKdxHhyeD/ueg+oiMw5Te/8uifbfgtFmM4v+XqI479oPBiD6TzPzY2/RIA7c7mZq708agGTrlpA5VPw4QHYpJL5hTpTZu5XJ/a8wffJHYVg75Gr9kd3xV0ZVLJMwOLAYkJYL54qCWgUCybsCRAfp+YxtoAtYBycEiMn7MULRLuSxSxxtRXdjxEdVCmIYRvfzoOknuLhnB6N1W3GPvIpb+xLerpfwd76Av3uz6QR37zoy1etIVW80lN0ngKybGwkgNR+0E0v7c0jvX8bIzhdItfwaTN6dCecKJNZJD8KiQce8S9+8K2/Y5w9imhKdXoqEN8GoO8KIM82Yp0rAQmIC4lfg+k7iLf83o9UvMbVzMfHdEZyGlWQO5ZOpKzLmlHyV7M4qk2vkfG0L3td3wNdfwn/dkv5Of3ULib/cQPx/rYc9r8Gul3B2bia+s5LpXcXE9xXg7s+D/dLcq6H6Faj/AWj7DbhVD9mHZng10uqKZ3qpE01HWic91CAS3CbfY7iV6L4fJ1OzCad2BdlDZSR2l+FrScCbmwxPeHvWMRcSH8Xrt/Cw9ruh5/dg+oKZb1AQS+WKNNFqx143Ewh/41dbnjf3+C5MEDGDrPNNjdhAe+hPhbi1LmKmGohmozWfoOWfds2RERfJcXjQSKbvl7hT/z3c2rve2NvZui0QAuTN1/B3voK7ZyvZ6jWkasqJ799gHNTMMwFkI96+l40GcaoXk6pZzET1ZkYbfgLuXLRYN4l0EixKD1e0JeAKo0H0xzO8dLrsN8kVk+ikvpEdPMUUUSbNAiXbl7avJuBBA5z5fxk/9ho39y5j5GABD2vzeLg/l4f78pjYV0S8uoJMzQao3YovU2zvVti7w5AmIOWoOm9ux1M0aOfL8OYOM7ckXyZRXczk3gLGdxUz+vpaxt/4AG79j0L/H8Gdk6AFX1m7F59qQH3zACJPzFaqFytagCThbhPR6h8xAHHrVpE9VE5sT5kJVjwrQLL71pKuX8+DvVtIHFZ29Fdh/ApMJMzWcvIPZT2ofJMBiHwSw7wWIIGcfEcGiFgnRikLIhtINxGeIIIoiaDQWMasGNTKQaVsWDVmBv1hO5z+PA8PfZwHNZuYrN1IbH8Vmeq1ULcdf8/zsOtV/F0fxNnzItmaNaRqy4gfWE+iZiPZ/etxqqvmSBvJ7txKdvc6MvuWkqhZSrxhHXdqvheuttjZKj2AAYPmQSatsDMC4R375T1+6eE7SbxsAleFkM3MvECiPlL8TJVGAuFlR8ok8ZHpxbn9RUZO/xhjJ/8eE43fxcO6l7i/bxOj1WuJ1a3Bb6yE42tIV6/A2bfMkLdvJX51oBkOlMEBTaSthF1L8WqeI1kfYeJohAfHl3P/2HpuH/448bZfhbO74e51u1+3wKx5PUWkg4Vj7/Fh3/Yw48eZb3Rhm4elrBYjiBTRu9dAouYzFiC1BTj1FcT3lpvADLs2wt71+Puq5kTim/SBDUzv20h0/wtwSnuu1MDEPTN3qTuysA3sYIFDKiUYFv35LgqEiPHwTY2lIIw7S2uE6tKGxkIG088Gxyrdu//3SDR8L8NvbuDhntWk6jbi12+GgxvgwEb8vZsMSLzdL+PsfYF0TRWJ2jXEa6sMZfdX4eyvtAB52rZ6vdUee6pI7c0nXrOC9LE13K7+EFzYA9H4jEulaTWl/tkZ1PfQM2/LDo9/aFfraalQnKnsJAnHrs03PyqzypTCCUJd6rZQQWu2lxtAD9z8czj728Ta/hX3Gz7D7eq/yZ3qVxk9sI3pQ+tIHV5Dur6EVF0+yZpckjX5ZA8U49aVwcE1UF2Jt7+CWG0po42rGWnZzlj/p5i69C9JXP0CRLshqRBwIE7VamCVGiSz/BlfhsH0nzFd5BrbyxspqjX892tIHPh+sgc249YWBQApMwAxvocBSGUAkKdrFb1M7Nlorh3bnUviwA4499swNWhuQjETC5AgvPgNAJGn8s5WRMSIEhMKCyWt1UIWHCrXrIJpGVw/Y2qymlTd7Ah+8gZMH+b+wU+QqN1MsuZF0vt34NSsw62pxN+/FqrXQfVGQ17NetyazWRqXiB14HmSB6tIHlqDW7sW74Co6m1bt2atud7bt2vN6j1qN+HUrCVzsIL08RLuHtjCaOMvwd0ztockQJgmyR0eRm+RNqkINrbwLPyh/g75TuVJVWjBqi3lJgWaI5BYxsWzCtrwpz42zGX2c7wNYwNw6yjZ83/B9On/xFjL5xg98c+41/ApHjR8jJGGv8FYwwcZa3iNcVH9Bxk/9D1EG/4xsaM/Q6Ll10gP/BHO1Wq7tcLUdUjIaQ2KgolHdE+iQIOa33+WDtClpJ3N9SQ0fbPxpX7CAuQBXPwTotWfNP4R9atNQCG2rwRqt8C+DbB/PdSshZqqp2/3byaz60VczYMdWkaqeinj1R+Bi182m+RIu92Pxon6Lqr270xmbPqBKS3rEc3Kong3gISdFhhkOkErD2xVjRgJN246wfrwStWYAO8q7tgBJgd/gYn6DWTqCnD2b8WpfhF3/6YAHAJIJdSsgQMVeLUVZGvXk6l5idSBl0geXEvqUBlu7ZoAGCFQHm8Ftnei7G519nrky6TqqsicLOZhXQXjDZ+BK/tgUqvbNGJavz3CpDOM1k+LO73A/Jorj6jrHgHEbjFtfkyTqmLIwC80bWDs6yfDuROda8ZH32mFT3oSErfsjrfRdpg+Qvbqn+Fc+zL+tS/CtS/Atd+Ha79r6eofwd39JgeMBwMwph2SMvam9NshKAJA6Oc0vtKidjZZUcm5v3SuwZveCCkz/l7wXM4wXPg9YjUfNRVdLEAKie8vhroNVoDOAEMgeUravxktmzA+Wl0O1D5H4kAVbudPwugpM5OvqKI2/jGRaPGBOt8YQFlSbuI9AEQjpo40QFJ+ihxxrfO9h8MYWSWhqbNlQ0tAph7AZD0T5/8frh1+nujhlbgHF+PuX4e7bwf+vu341QJJJf6BEqgrwj9YhFe3Gq92A86BHWRrt5M9WIlzsNx+iNxyWgAAIABJREFUXleJ9wRyDqzmSeTWrMZTuLNmrVG1ybqNOKdKGT9cyOjBD+F1fR7GtB2b/CsFGSZJ8ZCoP4JWnRkp9wwcolOtpp2xWiy36QsxvbhnFpMaHAS40TgJINNmpl+bwqiCvNXWnjbpEYWJgEpTl6bJToEzYZfpOqopO4HvTpF1EmZDmWzaMRntM4drvPSjgeYSKFRjK8U0CcZNa1cizg0keszw8ULNFMpbA5jsFTjz6yRrP0hS0bRDFcRri+xc0EFN7lbBgSr82so5ETVKen0Z9m2DmuVwKIfsoVwmG14mPvAbJsUGd1ZIVzyuLcudJI6XfU8r8iNGus0ARPlKIzgMGzLLddQLAocy47Sr6EQP8b7Pc+Poa9xtzCd1eAkcXAZyGvfLnHoeb/8WozG8ukL8Q3kBQKrwajcFtM4C4z0ARBrmSSStRE2ZAWNszwYSBy1Aoo25jNRUMnXkH8GNg5AYMdJEjnMKpSOMkxBoxDx6vmd46XQTCg9MDRPnEEOK9JkJENif0s/ZteTKIrabKkSJMUXMGIBy6xUWNqFJreEIuU9t+F4caC9kri0MKJCmIVKuqOZfdJ7Bls4xzyetoXI/KvIQI8UkCSMs5DPpgnN76dLKhlYxa7PMOdAguj1T8Do5iN/zOdIHXzKpNn5dGfHaAkPGfzqwZk7ACAHFgXVwQDlvyu3Kx9+/FL9+KWM1hYwc/bv4V/8bZIZnKQBFZWM4fswsNdbYvNsrYkSg6SOpW0VfBJAHRnuYkpZCnVFNHkzfhPN/wp2Df4fhA4VkmwvIHsrHqyvGq8vHqy3Fq9mIe2Ajbl0x7sFc/Pp8/EPlJmTJgW1QVwEHpWLLg/dr8A8+mdzaCp5EBiC1miRcS6x6PbH6KtInS4keWcF47UpGa1+CPjltKlOpNAPVhRKDxNEaZpM79W499I7fiwMlOZTNaplY2lYRIpn+0i6PAhzKeQu9dHW4zpVWGwm09YiJss3YR8pk0DWFlpCkyeV5aofXjLaHU4DACxxRa9EZ5jSo1Y3rBFWdUXBC46qJvOngnuzl3wOPPLEHZK4pwqm0DpNmMwMQDy8zZdKO0m0/TKZ+G87+CqgtJXEw35B3sAxqy22gQcGGuVDdWji0CQ5uxtldQWpXHv6hXBIHcxmv28B002fhYYOZ95GNZarSe6rTr37ROAWq9YlPqF1u1aPGvLILi6yJpTmDQLJIa5g9z4fh0k4mjv4YD/dvJ3E4F5oEDqnIdVZLHMpDD+4cKsOpV0ivALe+HP+gkL4NDmyGulI4mA91AUgOVrwjQIxpJvPsbUiShAPrze/L6Y83VBI/WkyscQWx+kVM1FUwffKfw/B+8GQ6iv0yaJcpMYZjp7vfoXve7St1tFIVVSvLmlSKlivmod+ypL4ctyWQjFEVLJZWv2vQXG31NULWH8fxdJ1gIZRSVVRqXNmgIam0p8iohwwyxaQTQs2T9rWKzsNxHLOXeNrXdpcqxyRgKEQvsMjcnLm5Z9KgFiBBxUbFdmcAksVT5GzsKNMnvx+ncaPxQ2Vyi3mTh1aZ3DAOCiASmHMFSAXUr4H6dXj7NpLctcYIaw4Xkj5YxKiiWmd/C8aVOTBNwneY9jX6Fhi++PpdJETESixZw7bOklSmBtYkmylOrm18YsMwXM10288b2z7VsA4aC6FOam0HXs2Lxodw6nNxGueTPTKfbEMe2UOrceu24tdug9qNIMRLatQWQ22eBcrBclxFs55AzsHVPIkETmoEuk1kG9cSP1LK1KE84oeX4xxdQOpIIQ/qP0ri7Bcg8cDIApkk0pMaXLNB57t00DtDRKiQJpBmUB+K9F5kxH3wffDeJNAFzKnTgnXT0jTqcwk0Mbg2qElnYmTMfoCPrqfUEOEqBJ/RFvoj9CNVFE9FGHx5M3ESZvmrZmVmfFOrNnQ74a3q/Dm+LEDSpPwk2cCn0z2ZxVyJh/j3DzF65JM4R6qMteDVFBpwpBpW4R4shEMSls8AEAMwCdtSOLQDr+5F3AProaES6guJ7i8n3vSPyF79Y9zUJZR4orCvFLIBhgSQ1Q5P7IGIzaGRhLOTPDpZp5n13Or41Bjc3k+i++eZOPZxEkd34BxbS7ZhJXHlPx3YbgDi1FXh1OeRPTLPkgBysAq3bpsh/+BajMSQ31BbasBlNIkBSEUAkG9sLTjKA5A83kqrULsODm3AO1ZF8shqJmuKiB/Kh5OL8JpWcL9uPZMdPwsT/Qb1inzLsJEv4D1m2AfqVoB5K5lONGp2piPDQ+wH4jJBT9pEqyHU6u9gSwIdPDv5U1wUAEQ8rbQIs2RZZpmksOeazUMdXwaU9vASqWio/tnx0TjJjDNRGXn8Kd9qGhNREUDSZgY5hK05XvehC+iPbwJALJ95qJBENp0x2knmi5ZQuLHbcGcfIw0fgSMSjJX4e0vJ1JWQqi8kVZ9PtrHImN/+IQuSp20NPx1YhX+gAI6+CMc/aMx7o5GOFuDUrWKiYQuJgZ/FGzthxkQjpa4y5rUZUv1n1wBJAIVkgaP9QdwL4N03kkn9LItapL1nTOdPNOIP/hzRxh1m0so7+SLJk5t5cHIZd48uJN24Gk8MWl+J11CGd7jAkt7rs0PrLdUL1astqUPqywKqwGuowKtfPYe2DBoKoKEEt76KbP0WMod2kG1YT/bYQjInIsRPFDJxXFt3/RkkboIW9gdCdzKrFGhxipg5ULe2v6ykNe8FHKlSda3pUYMfKylDDa3P9b06LaTg+BBJT2hnO/LhIRZ04Y2Eg/Z4oqh+X8eb/3SoOXn2OXYBmO5CNHP8o0NmfWh/8an/12+q+0yKm2z7MaLcI+3fhvR1HrT/LtOHP2yiV1RvsNGmA9vM+ERPFDB9MpdsY4kZd/GG+OBpWuor4FCZvf6hqhleM583LsM9ugSvpYzx4x/GH/oSxIcgOYyTiptKPNFE0G3BpkrihTAlySwBIU3ELJF1Y6YCRMyz4FDI0Umrds8AXPk8tH836YNL4NBiOLKZ5PGtPGjK5cGp5WSOluI3rJk7HV6NP1dqLIYjK/AbC/AaNuIdegHn0Es4DRtxji0ne2IBiWPLiB/fQbrj5+DmPltWx1UMHKKeCg1I6UokBAAJOT/g70c5ajJ1JH5tp4aHGSZ9as76DjlBYAtki8cUSYZJowW3Kit0nJHWf0288RU4pIRJpZVshwPbcQ5vYPJUIZPNeZZ/5jj+NJRDQymmrV+L3yBSmk4x/tHleMeW4p0sJN30Aqn2H4crfwqxQcjGSacgqjVVZjwVvbVmaTj/p4CKQBIxYy7fL6mCW5YFHH8YEtUw/pskWz9K9nAZ1ETMRIx1qDYYcyZ1fA1O4zOAQw9jOqcc//AcSB0RAqR+qwWIwCGt1liKc6SYeH0BiRObud/wYaZO/yKkZGrFSCWSJnPK5mfJeQ0UxGMiV3+IA2ymqhWXdnLNdux3CKPP9TFkqpj6COqTSZL+/UCI3CF2/Q8Ybf00ySNbLUBq1oGobi1OYznRk8VMNRWSlYCdy9gfLofDxXA4DznlHLIpTv7hUrwjefhH8gwPZA9vxm1+jQdHNnO/9R+Zbau1dimbcEwdcw28xlK+t5YxK/iufwqLi8x6EBN5kWUQ6uJUL/7Ir+Fe/hST9ZW4tQVQuxRqta5Ys+Mb4dB6aNxgESsmD5n9ads5do7pVAOQPPzGQryGDTgNmw1gDUAa1uEcXke8roTEkUruH67iYcvfgzt/AdnL4GgpsawMJRfK1LKZy+E8hkWMNIY0jAASgkQdJdH51y8xlqpqJh2bnJlOaxJTxfmGGOn/UUabt5I5oiiTQvqiUuM8e0cKiZ8oJnailOyROQjGkGdkQTSugsY8oz28hnV4jeV44gsB5fBa3MPbcI5v537jSm4c3Ub6whdtpRhV59dgy4IOMtY1stYHF1gUrPKIRLlPQkvtLYxAD/mwlvTQDzPduo2E0tYP7oD656FuG+zfDvuVyr4eDlaCNMiRCvy5tjr3SNmcicYyOFKCf6QE92gJ7pE1uI3r8Ru24TdsIXmwlOThIlKtFUy2v8h474/C2C6zTUJoVdn5ChsMmq1AbM0s63Q+8s6kUWRu6ci/2kDRVECcaSY91X1Ut2gtfBRGD/Kg5UNMnVqFe7QIDhdBQyE05MORlfjHckkfLzOJpe4zjL/GnSOFcCQXjq3AP1qIf3ijGXsatsDh9XBsDenGQmInyhk9uZV77Z+G+18CX0JSmylZB01NCBALEssPkYdcYJrbJlpialqNNeFc+vdEOz7CZGMp2cMb4ejLcOQDcPAFqN0OhzbbUJom/BpD5g4Z/Wnb8Py5tBVwuMoA1D2+EufECpxjZThH1hvJIYBogU5aczany0h2r+P28RdJX/lVu5FmQuvtbUc8Aomteh7+bftPjrK80TCEIa2iCNU3IV/821gRaeJ1iklGmcTsMSQOmxyAi/+esRNVJE8shKO50JgPjSuNv2gY+ViuHaejFbhH5zLu4TkCSBEcXYl/IgfxgNu4EbfhRfz6HdCwEY7bbA+/ZS2pjm3cPL6O6NA/hWSdWQVrXIxZ4XaBwwQFA20SGeE8ca7hch9nopn4lV9lsvtjTDVVkjieS+p4Hl5TKeljVcTq15I6vBZOrYNTRbhHlsOxYjhaCkfL59jq3DnSkUo4vBWOrMM7uRS3aRHuiSLco2vxjmyynzeU4B/Nh5Z8Uq0FPDhRznjH98L1P4eoainZ2sICgt3yTHudaDsHK1FCoEhaKnxpTS2p578GiGbRJ5hmTCaqOEuK9eGfE2/7hNEQcpQ5mmfp+HI4sRKOFbwNz8xx/MV75nqr8E6swj1RgHusEq9RAvwFOLwJGpfhH1sEp8rJNK/l4ckiRjtewb/xKyYbnawmZAMtEqxAFEBCikS5RUZrE9whssP/i8nuTzF2tJh4YwE0lxA/lkPs5BLGGldy9+AqJo8V4Xeuga5csicW4R2f9cAhWN5rO1dghOeFAGncYDuoaRFOkzRJEd7R1SBqLIKThWQOLyJ+dAmp1jU8PPYSMc2NxNpmOkeTXjaLQLPOsRl1GwJFALKhP4FExQCCGXGZpn9FX7LTJ0iaYnFW7I7BvX/DpMyeYyU2BG8kfAHeiZW4J1fhHRMYVsORUjgSCte5AqQIjq2C46vwjhfhHi+1dGw13pEN0LgWrYOnKRfv6ApSR1eRbClnpKmKye6PwvgfgDKOBRAZA8EiwVCLCCSRTDZqioulbuxntOVzJJtfhpMlODUqCFBEtG4BY4efY+T4Ukab8phoKSTenofTtRg6l+E15eKfKMY/XvK2LSdKzfXettV3z0LHK+HIFji6BY5X4jcV4LUuNERTnr22BuJoARxfin8yl8zJKmInX2Sy6XsYPqnatNfMwiY/JSfclkBS/ELLUdVRIoHEmlryOYI5Eb0VfYcDJJlM4rrGw5gRA0plUbE59Uk438h0FG7W4PR+F7GGCN6BFXB0HWgW/dhqOzanivCPr4Nj66zFIRAdfxYeKIETeXCyAP9EFd7JKmPteKfyMON/UsK7HATKk0vh1EqyJzcRP7WFyY4S7rV/GGInbE2xVAIv7ZqJWo23lOG0r1ysWILszS7GOn6V0caPkT2sybw8qF8LTdugZQ20l5LtLCPRUUa8o5hE6yrSzYtwTi0OAFJowXHiG1uBjZOS4m/X6rOA3glIOubtvj8hLbEBjm6CYxvwmkpw2ubjtEfwmxcbyYGcRCPFVhnV7h3bSPbEVuLN6xhp+Qjc/e8Q7wHtXJuJ4SY8m2wYzLiGGkSmlmETASLAyaPZtxne+Y57k1UF+VkvgSOVSpHJZMxkm9GsWpE60YV76TdJd1bhHo9Ag6JLSknaEoxNqVmK4J3YYAEipj1WBCeeYfwNXxSYa4QA0XIHp2UlfstCaM6xADy22gLpRAH+8U2kT1UR75rHWOdquP3FIMviAXhR9LwSjgKIwB9h/C7x7v/O2PHvMrOOnFA4Ls+mghypgJPr4dQGaN0E7evw2yugvQRayuCUkFlkpIPfNJdW54RUErx/760BzbEqK6mOS3qU4LTm4LRF8Noi0LwQjss5FEgqbMqDgHSykmzLSqbaS7nb8Qmc2/8Zkn2QiJuisr7WUwWJd9bcUgasXU6qyKARnValfMdrkFnYMFojnU4jraLWJHuqH5K38O/9IdG+v028qQhOSVrLzKqAxu2g9KSmSpymciPlpe2NVDcAeYbxF0CkgY6X4mvsT5WSPbXaZJmLB/zWCDTJTy4H45NughOr8ZpXku6MMNWZS7Tvx+HO18HrM9u5+dkJ4umMqYov6yHC6HmiLb/JyPHV0LMYWjT5kgtHFkDjAlAlioPFpOvLSB8pIXN8hbH3OVYGRyugqdR2hjpkTlQMp+ZI+u3ja+DYWjhRgd9cjNu6Cqd9MW5HBNrmQdMSOFYIjZq3kaapNOrXb1tAqnMxd04UMNb1d+HG/4DoDSs2BBAzy6p1G9q9V6T3wfJSM2nyV8PECgGi7dlkakmDhCaWieIlJ01KefLSZxlrryB6LA9OFUCr+KECjm6DYy/gN63DbS6DpjI4KbOq2FLTHMdePKPxP1Fh6VQxbnMxmZZSMq35OO0L8NvnQctiOKmMi+ct6bdbF+F2Rkh0LmOi9WOkz/0bmHzd1ghQqkx6ykasDUCifcT6/h13W9cy0TmPZPNyaFkCp8VgEWhZhH8qzyDUbcrHPbUQ9+RCM/egORBOlUFLwZOpOR/eieYEqgCM6tzjZdb8kt3ZnIfTIhVbGHRQBP/UAjiuUKMiGpIgCj4shrbncDqeI9a8gPGmtaS7fgzu7oLkHe1nYBas+07UOO42/UCTR3Yy0ZhYf0V8kLCwtVqB5LGXKqmPt8Dl32SyZzvjrYtJyKIwTFgIHRqbdXBiI37TGiPANEacWgUnCi3jPuv4Gx9WwKvAbS4l07aCdMcy0vKTW/OgdbH9vaObQdSkiOYi/LbnSLevINYmf+Rv41xWTed9wEV8Z8xsnaDxjpC+Cnd3M33lJ5m88Alig99Dov9lol1FjJzIIdO5AbdtB27LDvzW9fgdi6H9OWjRepBiKykkLZ5E7wQefddcNHc6Jd9GpOvkGpC6LavJtpbjti/Ha8+xdujJXDhaaemkgCSALIT2+dC6ALelkHTLh/H6fgZG3gDnnNlsJjl536TFS3M8CveGVpU+lX0upHznvkKN8VZH3Txx5oZZJ+/2fpqp9nyi7QtJN5eQPVlmwdBRbiKhSHM0yyzPh9YlVgCfyoemkrmPvfjmlPwXgbAcmlbjtZSQ6VxKunM5mbYysq1l0Cq+yIWT5ZbEJ61LoXUlbls+XncZE80bmOz6Xrj/e+B2KlZt19QYJ13FAlLjuNNncSaPQ3QPzv0vMXH+3zLe86+Y6vgcsbafJtn8z0m3fJxM1woy3REyHRGy0jDtK6BVKvUJpE55Ij0jQKSZZO/KzhRYTpXjtpTbzmkrwW0rwD+10n5/cjmcXAUGIOqkJQFp0EqMepY0yV74LEx+GZIDZKeUOvFoD0aBRGkpZnvrmXV8Asp37is0q6RBwpc+m5qaInG/iWj3D+C0VxqJnWpbbswbmTlO8xoj0WlbDm3q65XQovcSSvOtQNOYPZOAVPBGjC8AFuC15ZLpyCfZWUC6XQCRv1EZgFPAXGjvwQjmcgMoTq8g1Z7PWPsHmTr3L2F8f1CIW7V9IaI1I1pPnFbRQVMoSTVWr4FzEbyLMNIM9w7BzTfg8m+RufgPiJ17kYnezYx3bCTd+jJOy6s4LS/itm7DbduI37YaX458Rz62g4TYgNRJLeqs3MD0eitIZLIV4bWtxGtbjteabx7EPuhqa8pJKrQKeJIOS+CUACJptBq3tYxsW4mRHnrvyR4WSKQ1TikmLpNPYNY9rIQuResK8ZoWku5YQqy3itjQ98OD/2krhodhLNXaDXK3lOCoORNVC58Jos947gJMSKEdFrLWW9vw+0fM99YjvvFvHTv7+sERb7F+vvG8Wcfp2JCMFgwnPYPU/mBdhCZHwyW11gOTxozhJm8yMdzC5MUv8PDoFtzWZdBZQKYtj3RrLm5HCdmWSpInCvDalwSaejm0CiALLZnx19iHJLBIy4iK8FpFBUbKu225hvnFC9ac17EKEgXgMD7PcmM1CCDpjiIyhgfK8Zqr7LHmd2X5iBdK4FQVXrP4NA+vp5ypzud50PV9pG79PtBrsibcdErp7rM6y3SyOkFBrqQtja9dUc1S0HG0nx3eADgdkGmD1Ammu3+HVOevkez4HIn2Hyba/kmibS8Saysn0bYMvzcHTs+DLmkbRZbmwcn5cCwHjkj654LUrbRBSwlKCXAUVj49n2xPhGRbDpm2cpyWl8k0bSPVtIx0i663AnpLwNiZYvhi/NZSQ15bMV6bfe+3hWZgqOH0d/CZAhIKNhzXdZZC53y8vgWk+0pJnP4E453K/r0KsYytjKCZI8OeWr6kPKQkqewkjjONp6WyBjBxfD+K40Uxc0yGmR9lAAdsOhMrDhfoaKJy9lA89kf4hTHnwvFREFKMHVxKuAmPe1Ib4nF2kCGtPdqDfdqVm5SeJp0cIZoeYcofNxOBYyZbz8wMmN3DmK4hdeHfMNr0UTJtimzm4bcJDCEVojHw2wrx21fhy8poywtoFbSJgvFoXmV8RxMRba7Ab60w45/pLCTVvYrk6aUkexaTOr2ITPdSnI5VZmy9lg34LeusCdW2xJjTbscSAxIDKAnW1gBIBnShQC2E5lJoroDmdbintnGvbjX3jlYy2vMhRs78E+LDvwsZ1TEYeRuAPKFzTeKjQp8qjZX2SKdc/MQUZJWuMQSJdhivgTtfxr/6a2Qv/ATZc/+QB03PM9a0jqlTFSSay0g3y4kuNSFlMTRdq/E6y030IXGqmERTKanWPLI9C/AGFpLpWoXftQE6PoDb+hLptkJSbYtwupbC6VVmYASCuRD6fWO/llrV27nQBid6l+L2rCfd9Sm48QV4cBQS9yFjCiqZ6Q8rQlwyaKGQdiHRbubSKAk8L2FLkqrqvUkPDpg3QIedtbf1qbRG3G4oGWiRsP9DZp7dmu/0gUAi1faWRV76Ssc8FkV4y0rJ8HohSJJBKSEJQbN92SiOPxqUBkrywFHdSNcsozXrPKL7cK/8IomeD5HUOLYXBmNQ8NStAY20T+uKwDKwgi3bUUyqq4Dk6RUGHMmeBaR6FpDpXozTucIAT/NzApPAkO1YOousJnHb5WMoB68YTpdCt3itFDpk2ZSYuT3a1kLXazjN20l2ljPds5oH3duZOP9ZmPg6+L08vj/II/H2De9kXphCG9peN+uS0QaJaS3JUpEBbUg/aXdl1TpqLdNNaOuvq7ao2d1quPUVvKufJzX0fxEd/D7G+j7CWM8Oome2MN27jrG29Yw2lTHZstSEX+nLh4FKaFday0Y4vRm6NuO0bTKZuYnOJWROL8KXGddeNDdqk6ottoMjqdaeCx1F0FmB172WdO9Wov0fxLn+4zD9v8A7axfOyMIxeTvSIirZM2FaUw5ADKjvQ69erfg5OEcdqzRqFQ5QqoYFR3hAwMxvZeKQmdXqOuHfBhCBRjG5YdIooU04u9Uxb0M6X1pR92h8qmEy3CLBXVOlS1eQTDC/qe2nJ0+QvfILTHa/xlR7LunORfid6rP8OZLOlXmWAxJOXTn4nctxO3NxO/Mtda00n9G5HDpWgpjemNgy11YabZPqqCDVWUqqU8DKM0I1073MaJx01xIe0VLrQ3etItOVZzQe/VXQVwI9EdLdEaY7lzLR/Tzu5Z+H+J+/O0AEjJBCgGgPakfFBVzHbHWVcFLGoTECVuMQMofpYY2N6uA8hPQgJI5A9HX86S/jTf4W0Zs/TvTqP2bq/A8x2fdJpk9vJdFdQbZD4FhrHT1FyLoK8bvKcNu3km7fQrxbEmaZ6UTaCwOAzKHVtY26z8NeZzV0VOJ1VZDpLWaqbwWT59YRv/YDMPoFSPWCmEUZJ1qVSNqYIZOoBI9vhIhhKPVByKN6L0YTQwZvw69kYs0s6Q21TQgQtSEYZrch0+ozk1EcACNYjz5jIpu5YJliYUJIYDrPmNCPcmm0Pj/JQ6a4yzSjpsCBWQKh8Uw8hFvVpPp/gWjnh4h2FpLuysE5LYZeOUdw5Btw6Xy/axm+wBEAxABOQk8CrK3IRkhbQodeJpL8xxzr/LeXmCCB07Eap6MCp7MEp6sQpyufbHcusZYc4go/ty0h2b6UlELAinLJMjGgWwYdEdxmmfMR3IEI8Z4iJrv+Jt7Fn353gMzUGnjrxlOqwOFbASTrW6TiZSFpd1UzZZ/FgEemmeeEeymo5pO83jH8dDskD8LUXnjwP+Dar+Ke/Qzprg+Sbi0j0zwfpyVCtjWC07aEbMdqMt2bSfZUkTxdjttVbBxEOiX5C3jqVuAyWiiUgrpeMV5XAU7PCtKDOUwN5jAyWMjEhe8iO/xvYbIBEhNGEIj39cyBR2Aqk5jl66GgCJk5ZPqA58OPH/Mhws42TC/zaRbjP/Ze380qHxSCT79hflzw0x3J5HvkG1ng6MZ0QmCO6TIKZHqPjtaZGipzmPacvPtnuAN/n2RrCamWZZaxThfA6aDPu/JgDmT6uLsUp6sct7PcmNq0r4Y2BWPWQnOQOd6kQMoaE8o1Gl9zKc2Knsq3CfwZ+Zptpfbc1rXQKv8kIJlSum57CX5HIZ78mI4VZsrCO6G5sgiJpgjxtgjZMxGifUsZ73wJZ+gH3h0g0haimbELNEooCbWb6Zg/kwQeBD89s/zd1uNQ7SbPTN0nHEhqP/UEJGKgTUgd1XkyVUDugXcJEi0w8jrc/E9w9V8YU8fpXkCyPUKiPUK6vYRM90bSfevJ9K3F7Q6BIXA8JWlQu1daEsA6iq0W6cjF61xKtlt+0HxSg88xPfgco70FjPZ+F6kLKox9GOIjNgYc+g1GSfhknCzxRIppVZcPlnTObi13iptnuQvSBobpW/u1AAAgAElEQVTBw0iSmFwSX+wqCqW/PhdApHmCc2bODZjecLaAIAqON5GpxwMBOl3Va6TgtUZf1T5mLC79XvYc2cu/Ded/EPrXQvcC6BDlQFe+te3nAIwQTJ7ModMFZE4X48gX7VDe3xpoXWNzAAWQpg3QpOUVa6BZ5rAm/wSMFdAus2uJpfaV0C5zW2NYZq/TVgkdG6FjA7Svx2+rxGkuI3WqhGRTEfGTinZuxu3cQra3nNRgKfGBKsb7XyN16Sdg/D+/O0A0UaS4dzijOns2VR0cDqM6Vqu8tVN2nAfEuE2Cm6S4Tdps7jhhlLhklzHZxAeu3Wdc+1QYZ1bF0hRvFnomJmCyBy7+PAy9QmZgAfFuAaSMbNdG3L5S/L58/O58mCudXgH986FP6l3Om6RMkYm6eJ0RXEXe+nLw+paT6VtJ7HQhk53rifV+DIZ+Dq5+Be6dhrG7kBizRd/MRpEJprwkDzOquWs1jPJ6JArUZ9Zx12KtoPxpaD4ZgIQH6Q91jCgAkzk3+D441jr5ykLW/Iwt56/fMBQIs3AjJNO+xWrzFVTw0wYoRvkId6lpstNv4D78Wcb7N+D0F0LfGji9Gjplii4BZXP3LA8ETC50Pz35PcvIDkQMub0r8but9raCShpB0UUBpgI6ZGYvg57n8PsieL05NkLasxB65sPpoO3VeNoxpW8x9OVCfwl+/zr8/m3Q9xp+/3dB/8dxej8DF34Xd+jLcPO/woMvkb31X0nf/EtTYhfn3nsHSAgSI/Vm/acCcyILHMFFal2p45pPuYNnViuqHcFjHOkWVQT0Xc9YWcbq0vyC4QchJli4p+IYIknqG79AamgLsf4ckt0FuKfL8HuKoEca4OkHJjxHA8Tgc/j98w3QvM5S/I4AIB3zcZXL02s1TLYzD6e7FG9gLe7AJjI9zxNt+YTdj+LqmzB2DNyztqIHWnSl1XYm99HssiugCCTCgl0LHdQoDRjdcrTtWBvynV3uZ5Yror4K+svCR9saq2qkStbY2YoQVo8BIzxvFkCM/2PGSoWwo5CagOlrZO/tYfziP+Nezxrigwtx+iK2n00EqBDkd3QrfC9aMecx8HtW4PblkO1fhNe7BL9nCfSE11WrlKcV+KdzcXsKSfWWEOtbw1R/FRP9m5ns20ayZysp0entJHu2E+/ZQaz3RaZ7X2Gq50Okzv0d0kM/gHv5x+DmL8Dd34SRL8DEV2ByP4xfIHvnEkQvgn/Njp+CTupETRTO4vU5vBUyZMTOIv09Q2FUJTQVQps4XLoqyRdIOyP9BLBAaEolhYffbyZ55WeZPqsiYItwJSVkTyoPTCZWz6q5Ue9y/L7F+P05RlPQuxxkW0ubGCqC7lXQLSCWQPdq6KrE75FEkgRbSeJUFdnOT8LQ5+D+VyDeaje1N7UxrPYI/TOjZV3XhMiduEs2ljUVFNPOFBlPJULF6KpwKVJwVaVoRCqXak2gpI8xV5Ne0JpAlN0YT6asjpMuMUVrfG2HbPfsMAWmFa71M2Qczd/ESKfDQndX7bzWw68Qv/gjTPR+gMmuzcS7V5s5oWz/KpyBHDzzzHm2j06vhNPqrzn2vcasu5xsy3YTmfT7lsPZCJyP4A3KUY4w2RkhdSFC8sJSYhe3Ebv4aaYv/Sumr/4Hpm7+B6K3fpvYtf9C4voXyd7+U3iwEyZk+nZCesguqXVGzF4hpjq+CnVoBaFK+GjORwWFJbFMirYV7D7jQSEP64c9I0AEs3BCUVPy+kFbKcIgcLbpICfG2MQCi25GIAm26TXhTlUWETPYpEBpFqPydfhkGu/OHuKX/qlxmJNKpOyQk7jeDpak/BzJ712FGRxjaklNSzqWQKfs10roEkgEkIKABMjlMBCx1CtToxS371WcMz+Ie+XX4OHXINkO/k18Rs0KxaSXJOlmTd1ps/4o0BxZL0nW1NC1GyGEhctsETNVXFHFS81oPwqKGbAEGkmgC0ldpe8eeR2BSacKHdkkmeQkWWkJV7pNY3AP6IP4Trw7nyd1/h+QHqjC61sJfUUwUIHTU066v4jk2RxSZxeSHViO17fKAuNZwBEAhK4PQOfL+D0FeAPzcM5FSJ2PED0zj9GBFSRvvED85qeJ3/4lUvf/B9mxo/iJPvDOABfsgjd3GNwR+1wh0wcaYCaiqh0lAtktv8vUT9beQmHcyFS/j5tKxyr7E/Zx5Bsmld46yfSOf+suQi9Erf7WyIcOp1UP8jmkKozvEah4fWTUh0BkTpF5oOrrsrJkuz8we1joigZX02M4t/6EyfNVTPVG8HsXwflSAwwx+JxI4JA2kLlm7NWF0KsJSGmRSujURFJpICVzQJqrV8eEtNisrNQMb6p7Gan+IjLnN+Fc+wT+7Z/CHf7PkD4JqSugOSP5zDJJg6Wd5tlCFaruMCnlvgmfy9FPOtrfS7P2tuC2ClUn3/Jv0plmyomZIngJ3xZTNf2tPhUJWWHgylTxiEL6ISRuQ6KD2J3PE7v+A8TOvkqmrwp61tgMhe4VZl5C/ldqYBWxMwXEzq4ieXYx2QEJBYEkz7TSpHMhc43+EhhcA4MbcPrKiPYuY2KwgOjlV0ne+imI1sPEdZiSbyrhK6tDGla1q4JAhfrQc021+2wmTiodJZGeJpqaCjYLst0gDn0raQysHBf/2cqKj8xVlR41vRj25tO2spPtTHCYKhEqDdMGdm94E2pnkwFJeEJQh0gAiZJkmltEGTaAMbeoJxs5TfL6DzN1tojU4AI4vwA0WP3LoF8O91O2koS9hdBTDL35oL/NZ7n2MzMLq+/0uX7nORiYB/0LoHeFOTbVtYBk7zwy/RGcs/Pwzi3COZtHZnAj8b6Pw93fgIf/G6ZaIH0zcOS1eUvW1OD1ZDNJBejBxcxhZwVDoWohUspWfEibqD69FnBpjYpMMn1jB9lEGnWNUEmbCIFCh5q3eQDuZUi2wsguYtf+iPGhX+Th4HczMbiGeO9yXK0HklaQ+dS1yKTe+P0ryAzkkzhTSvxsMalzS3DOLLL9HfbXXNv+JXjdETi3DC5WkT2zgdHTVYye+QjZe78M6Xq7kWKgIhXpVoFvdVnMc4m5jonAKZ9Q2M9mfLPSMZNNkHaiyHQN657ZfrLWibpodldbGaUIn0o8CYA2amgrKz4DQGzKxONMbwAgtRXSrJvRoiO7tkKFEe6ZTTWN3hNIBKYA4eoPYVll/RNmB4yslYKxEbMpZOrazzBxMY/xM2LK56B/SQCOubSLoE+OpsKYMtnKLEgGnoPBSHBtgUHREIFQQFF0ZAX+wDK8C4twLi4ge24h7uCSQJLmQW8xfs8asn1byJ79CM7lz+Ld/XUY/7pNy3HugqJ2GqnZYi3U7eFnihQnbAVDL+0bU0m5X9o2IasJSwFJnR5eR+CwHWjb2ChM9cCD1+H2b5C69hmmL36IyaF1TA+VkDxbQuZMAU7vIrPKLq3VmFps1rsAzhZBr8zHMpz+YgMUZyAXd2AZDEg4LXjUP6Zv1D9PQQOLyZ6O4A5GyAxFSFwoIXrxk2Tu/DokjoP/wFTz1opeJ4UpF6qa0Yp6ahMgU+ZUyHgrxxsmnFENto8MCtRZYehbroFsrVC6BBZN+L2ZdFXp0W8FQGZpCv1+KOGsWpR61IYuD5BDZMwsHSB//y1jLR4xm904U2aDeFIqqHoPRvYyeePT3DuziszZlTCwdI60BAYWghzEnnLoqYRe2d5LrcMop3FwCd5gLt5AkSEGVsLgUhhcjndmCdmhRTiXFsOlfLhQCufLYKDISGLNEDunl5DoXs5EVyljp19kcvAfkrr6y3D/yza12u8B9xw4lyGj4toPQflRiRTE3EfgERNkpU60h7N8Ny17jEImjukXTS5pdZ9209I+h8mzkFQm9p/DzX9n0vhj/a8x1VNMtH8B6XMRGFK4VH2QB2dXwJnnyPZFSPdE8Pol2eXbCSRltu0rAFH/SnMsZ2z/mD5RvzwtDS6BcxHivRHGTkeIDr0AD38dsifBVdjcagWlnaeUca6ijYGbq02BTJhTfaF+MTRr7/JQwDwGFplockQUPAoA8tj3gVkq4ATHRd5G/s9i7/DsJ7UhGnU3eh8au0Jq8LL6y3L/W9+H0k+XD7SIjb9oNyYb01fJRy+TxpeZoH23tVdf7AqJe/uYvvxLJBXbVkfPhfqXW80gp3Qgx4R8zaCfmR8AYyXeQB7uQBHuQImlwQLcMyvxzi7EPbuQZJ9MkGI4VwEXVsOFcjhTjNujXKXlMFiC119CuqeIeHehSWNI9pfinFtL8uJ2Jm/8LcZv/xDT93+S9Oj/B9NfhMROSJ+ATD8412xCaHYEU9hPrcwlkXcrAFcbJPeTnfxjEg9+hanhzzJ57buZurKVyaFyEufzyJ5bhX9mFQzkQ48icuUm58wsk+4shDOFcGkV/pUcspeso5wxgQgFLpbbYEWnzivARPsG50PQTwwoaDEHOpODfz5CvC+HidOvkLn4CzDRaKteah2OapYZ5ah6AFpeoK0lFPtXHyjJUoEGmUWWFMOTESVBrKkEV6onVABv5T39Lf4LNUjY6jOT8mN5O9Aguo2QW5+21U1Ir4tC2M4Cy+z8ovAm1M7GXPi5btpcR4FRXctmz5pGE4jZuO21dJLM5D0y9w8x1vd9JAfXkRkswT2z3A7aGZlHOVYTDIpJl+KdEc0C0oDMseXQK6koiRh5pDXOKtS43IJDgAgoa9o88zvuufkGIJwXcFS1cRVRrSfpzCGl+ZUzBTBURqYnD29A70vgSjFcWW4kt8yK6b4Ik5dzGL2Sz/iVtUxce4mpqx8nevUHiV/5KWKXfxnvzn/Bu/0l3Nt/gnvrT8ne/jMyt//MhDWzw3/M9LXfYPL6zzNx7UcYu/JJRi5t4eHFlYwMqdxAhNGBCNGBiIkOcUFh1FwYkLbcAF3roXuNiQhmOpeQ6FtI9sIS3GuL8a9H4FLE9ouWLGi+qUuRw1zokwZdaISE6degj42Qmq3N1e/qx8GVVgsP6rdX2n7vk+ZaTrwvzwQI0hc/B7e/DmMXIWHNJpM5bgAi01xb1N0K9pdX1oXdnFOTnOH+KWFrLRWBJvSPQ/4OzKgQLGL1t5LhQZkzOkfFq81LR+k1l1bnhNz+1l8L7CZdOryp2a0On/23uQd7YzN3Hn4fPokBXPgAw8Qe/jfunP8h7g++SOJSMdmhYGCv5sKFIvzBfOM0Z8+tIHt+Kd65pXBGJlng0A/KtJBJtQDOzTetf24R/tml+GeXPUbezN9Lsccstt+fWYEvmvl+1nmPfW7Ps781D//8PJyh+WSHFpAdWkxmaBmZ83lkzhaTOVNBZnAdqcGNpAY3kxrcSmpgu6HkYNhuJXlmM/Gz60meW0PyfCmpoVVkhhaTvTCf7IUIzoUI3lDESGr7fIvs8+q5B6VRVoHpg2Wmb7xzi3HOL8I7vwDf9IeOXwyDmlQNyPTXYnO8NJP61j+72PbhwCIQnVEfLsYZWk7qfB6pc6U4Z8vgrLSVwsjSTCXcaf0oYxd/Bab2WG2onXy1D6MsoWCLd+tiaApAQQlpEjnRllcfTarKyVbQSK2lGR6a4WvDYI/+C3krbB99M4OcZ5wHeeyK78Mfk2TSzTy8818YufpZoldfJHrWJpxxYQWcL8Y7U0j2bKFhnNTQMpzzGkgNYuC3nBWTv090TqCcZ+xw2eJzITGxf07MPFeyjDyXPpDAyJ7LI3s23wgII2gGF4EijGd1P4vInFtC4twK4mfzyZ4rsZpUWqx/vqm1Nn7mXzI9/HVIdmH3qlG6keVPhaslLi1ANBckTaKoXbBNYCBfv5WM920OEEmSGzjJg2Qe/g7pW59lerCS6e75NtJyvgD3bBHOuRLSQ/lkLyzHHVpow8NnckB0LgfOv1+ke3nOzB7PrdW5CnfrOnOlZ3j2czJp7YSiWbszKA2x/LE+VUg4O5hDdnAhvo6/WAgX8412muqvxB/5HfyYIlZK9VAoOmXTxGUkBJFNa5/IDxE47D6Nssol+L/Vr29zgMjXUSRMdmsjjH2BzOVPm5VhSUWmhlbinCsykit7vojs0ErcC2IoSevn4OxCOCfGegYmeaZz9du6HzH6XCiYC5ozOJ7x2c/Jj5A2CEAyWIF7Lh/3/DLcoRz8oQX4Z+bBWaslvbPPkT6zgtT5CjJXXiB54zPgVQPnbA6UP24mVJ1k1gaRAh9aGiPc0MZuSRGA4/8HhHybA0TLSpQHFofMXRvWfPgfyVz4W8QHC3AuLsW5UIBzvtSQO7QKLs7DOqvz4Px8OL8ILryPJI32rHRhIcyZnuHZBUwBQNGsc4vh/AqcoWIyFwpxLy6Ciza3yrSXrB80PbiUiaEPkb3972HyEDAUVBEZA2cSJxkjGU+YZd2O6xt/QuB45HAHfu3b+g3ffH3ybQ0Q9VEyyBq3AbQHkKiD4Z8mfqGM9JX5ZC6uMoMmkLhDeXbQNHBy5gUQMef7CpBFAUDm2M4ZGCGongEgQ4H/pPki9eeF+TgXVpG5mGcBoiiY+vpqBK5F8C+tJHbhg8Su/zKMd9mJUi3ZVl6eF8XNREmnEqb2bzqbQbvnWsfbAsR6JIFtFcaDvsVa5NseIJkw3V7uiJYxpgfgweeJXqkgcT1C5tJynKFC3POleOdnA2TeI8n9zEwWMtvTtrOYc2ixBerTtu/bvetZA2AYcFgweJciuJfmmwwDo60FjqsREzb2br8Gd34fJs7a1BpZyCYY5Zg9610nY0ubellcX4ZVOF0QtoH20DlyQtT+NUCerDbVN4pwKK0lE7VFOUwh5YdfYPL6ahK3IqQvLzKawztXDAKIBtVoEJk2YsqnZepv9vHSHIvnRkbzfbPv52muJ60R9Kf69DGah3dpHt61CPGLS5m8uJXM8Ocg3mUHLVwVocltOeOhoDM8H4ZsBQyFtNTOWkH51wB5MihmfxMCRGsjTDxQc5XOBKS/xtjtSkavRshcXYB3cSWcL7STZBpEqf4LS/AvrMC/mIN/acH7RIvs719cgj8n0r0vep/uXX22CG4sw72cg2f8uQVwcQlcXoh/KULsfITJC0VMX/t7ZLQvYKzT1uCSz+hlTI0CZTYrSjV7rtgu5HoLOKQuNOAijXdIsxniW/D+29zEUjzcIe27NpVewsZVbtJeRu+s5+H1COnr8+CSIlrSHsvh0jy4PM/Yw97lVfiXF5sB1aC+L3Rpsb2HS0ufukU5YJdz3p/7vrwQ70oO8cs5pK6thGv5cKUALmqCdjnOhRyiQ2vI3v8XpB9+hezEOUgZO9gWkvAn8bLKywusgEDGWQyEk8WB5ngrOEKQqP0Wv77NAaKYuGZYlbvl4GeVyKe850OM3N3G6M0QIDKllsIFMVMErszDubyK7JU8vCuL8a/Mf59oocl9EqMJqE/bKm/Kv6JrPAvN/dmz1+YzdT1CbHgp7s0KuFJhtfKFUvwrH8e9qYrpbZAew/fs/IWiUWaxljuFrw2LvgEgdgnFjGllEfNIY4RFML7FwAgv/20NEClnVTPMKn6ueirelF0qljnF6L0PMHYrggaRSznW11CI93IE/6qK2q8kbQAiJps7kzzrud5VK4kFjqemq88CDJ37bM+tvo3fWcz0neVMX80jeqGY5MX1cOP7YOSLEDsNWWUdWjNK/qIAYUBiFnBoCfAjDTLznQ1JfqPfEVhZOk6LpRT+tcgJ2fmb336bA8TD99K4qupoikQoy1MiqZ+Je59i4lYOzjXZxEpHj1jtEURU0tdXkL6+yiTmce053he6ugD/2iLcOZLO5apowbPRHJ9ffZu6XcbEjXxGrkcYv1lK+s6/gIk3ID5uI1WBby1RpjWAqqWmajZmnLQc4zGAyDmXnSxnUl58wP86XuCYmVlXzq6WI2vlvQb8W/f69gaIOtBRRpvWWA/bNdYSKpnrTN39DNO3i3CursLY6gLIFRuPd29ESN1YQuLmCsOc7ws4rj2Hb2gB7vUFFiRzaN8vcKjP3KsrSF5ea9Lqo3eex5n6p5B5E3zt92gDUI7WcHguGT9lSNEqJSEaVaLIVYAVtdb8CjPDNQEcHKcvtRRmJjdLK/VjM6b1tw4ez1zV5Ft5a+/h2gKIBI4q0JlU6LvBwDwgevcnid3egHe1CC4vseBQTP5GBO9WhMRwDonbS3FvLAQ58u8LzYfr8/FuiBbOoV0I156Rruse5vj8V3Nxr+yAG/8ERv4Akk1mrYakuoYl5vlmWWzKyZr14mZfu0BDSFPITJL8F+m91gLNFMkzi5ZmAWSWtlFJpYTZy16ZvZKI37rXt78Gyfi2IIJZTKNUaYFmgqm7v8T07VfIXiuxTBTM5gog7q0IyeH5hrybFjT6/BGJYcQ4Ak9Ilpm5MW/WcW85R98F5z1i+Hl4TzxH54fnBNc357/H999wXd3bbGafHwBoCVxdakkmp55p9rla+xHchwVreB2dn2OvoWMM5cD1HNRv2WtFMPxD8FD+Rp9d3aiSs6aMaYqEaoR4abP2/tEiJIFAzrnW1atVnlWoPULzSu1jDsfbahBpkb82sd5ROEj2hLULg8VVEiiph7jRL5Gd+H4SwyW4dyJwOwK3xAg5cHMZ7vB8nLsR0Hez6XYE//ZC/FvLcG8tnyH97d/KgdvPwXDE0h0da8m9GcG58RzpazmkruaQuLqI+JUFZG8vIDs8D2c4gnMngjdsSeeZ+7n5HNycPze6FcG/E8G9Z1u9N/cWXts8rwCRC0ZQlMD1fNBz6BkeO34+7u0cnGHRQtzh5+zz3iiAaysMOPxrEdwruTg38snciTA9vB7YbXaHxZ0GN2N3AJgpMiGtELzMG9lMoc7Qd/Z7e0w4saHv9X7Wef+nvXMNte266vg65+xzzj7Pe+7Nuff03pt3G1objG1ME7W9IpVQCTVSqpZUKkZMiSik2pp+CKgfzAcpVBFbNdoiFB/UQCBSIaCxRaGlEUKj1UaTe85+v9+P9V4/+c+51r773ua2+ZDmZF+yYbDW2WudNedce/zHGHOOMcfQDenqlT21pSNMVpP5e7P/eQ2Pi61BjLpo2fyFenN6twKI1yAYfgGv+wCj8gFBykAGJIe7cLhHUlw2zDVj9pTpzffFVcssxQ3C4gaRoXUiAUf/V7IMaRhMIBOVVgxDhUeb+IfbeEebuEdr+KVlw0xBeQ4gApWYV9pLx1ekZSjM0woUrqDisgG/gB6lNGN68/xlOBRAduHidRYcR1skxRU79jmA6DuBIyht4JUFEgsYLp6Bl09YgBw6hC/t4R+ewa+t06v+KPCNWVraLD3t/PE15NVjedSCA0RBbG0SJfkUQLIZn9cmGn+JsP8rDCoHeLWUoSVZ5wEiUBxtw+EpC5qjbZLChmEgaZ2M6SSZDSikPTIm1bmeV1mGSg7Ka1DKQ3ELSlvEpTxROWe0lJ5zuXRfsYx7tGo1wByDZ22a9tUHUUm0bLSeQJqRwBsW9ggLu/Z6phHNuASOdduOjuqrKAW3GY/uS7/XM/W8oLx+OUAO94y3XCBOjhyiQwmNMwT1c4waHzTlyuYBceX5sXD1a9jowgNEkaAJw0srI9IickxNvkwyfJhh5Rx+PZXyxSU4OgFHJ43Ej8s5OHwLXLzJHgsnoZg31y4zu8pLICqtXKJiDor6O2OyHJRWobgOZdEqSWWZuOIQV+fMOPWhoBCNbShsEpdXCGvWTAqrVx6XiCpLhFUdVwirK5cdo3KeqLBvSIBUW7N+q39qR6Q2pS10vWr7Y8ZeEqgtaOLSGmF5E7+Sx6+uEVbWDMiTw004WjX3xUUHX1TZJ27eRdB+xGQ3VMK7q9FryKvH8qhrACADU/4sVLmRTIOoypVqjkw/ybBy/rsBUjhBXF5CTBEfnic+vIm4cNL8PQNCYQcO9y1wdCxsg5hODJWZJjqXmXSZqSTQrBmQJBULkkQAEXOKStI8q1DYgKNdKJwkKl5HXLiOqHiSuHiCqLRDLO90OY8YWeBQfzXXENiiWnrUuUBS2jb3azy2jysWFCnzZyDIwGPGXs4ZcGZjEcAMQKpr+FVpPgFkjeRIgmDJ9F3j8CoOXv0MtD8Cg88DxauCQ6BZ9M+CA0TqYmQAIoetqvVaM0t5tP4VvN9nVLmZoGElp2XOXSjuoh87ruaIi6dJCgck5S0SmUoymaQBCqfg6ACOzkJhH4oblrkFDgMQSdVV4osOiZaOj7R8rIntBklph6SyTVzbJK6vkEiDZSSwGI0iIOVtO4XTcHTatqN2i3tQ3IGS2pQ2kgknU+6SFphpA2mfWo64sk5S3ki119y92f9Iu5W27DjLW8QpeM1zypqPbRFVdghqqwS1HFFV4F414zL3qJ26w7Tu4LbOQ+dRGP9zmsl/0WFw9f4vPEC0dq4sF0EYm4TEBiDBANxvgv8ZhtVb8RsOSU2MLem9C5LQ6Q8e1WQGrVqGETBKYsydOQYVYFKmFrPpXPMMAefoPMHhJkEhT1DcJSjvE1UPiOpvIWyeIWrv4TVX8VoOQdtS3HKImw5JMwWNeZ6kdEoFmWnSMK90FHA1n0iPpRxx3TECQMxrxqL5lsYqyjSXQCYAHp0nKZwnKe4bQOkdWNBdDSA5Qk329ZyWQ9R0GDUdJu3boPtn4B7aBGxX56+Fv7LgALGhCcqDpGTP0iIGICai97/A/3OGldssQIzUFhMKAFsWIA2HSFRPtUl5awYgM58QIPR/lzGbJvGnSApniUoHTKs7TGu7TGvXM629E7f2Ptz6B/DqP4vfuJ9h6f0My+9lXHkX49oPMandilu/Aa9+lrB+ykr8kky3V0tWc2VzHb+2wbSxjltfNeOUttSYYgkFM/fSsvSOAXRydKMBiLRmXNkgrl7SNJdpkPqy1SBVh1CTeD2n5eA3Hfoth3H73baisetjizIuPA6uOoCFB4h2nYmU7yJK4nQeotpuSjL2JOPq7YQtB2RmVVahvA2VzRlAdM1I9NoqSXUTypv2PgFDjNFO/7eYJzw8QVg4RVw5CXrZgg4AAAtoSURBVO1NktEq3bbDoJ/HG78L/Acg/AxET0P0gu1DUAXvJZh+E8ZfgcET0H2cpPVJks4DJK0fgc710DpH3NwnrO/h17bwqmu41WW8umx+B6/x3TSt5Um6dzOu30W3dJpBZQWv48DQgZEDfS3drpKUdqFyGqpnoLZPXN4lKGziyjko8NeXobpDUjtB1FgjauZIGjk7fr23ngNdh2HNodtycMf3wvg/bKzVnMviqly2wBcWHCByKcndZGM7zTZNpThX2Luv3K1fYVK7IwWImECTZwEkTyIp23TMtVAmT32JpLYC1ZRpxDiipv7OQ/mAuHQ9fuU00/oWk7ZDv+vQ7p+mN76bIHwQ+CPgqxBXLfMoM2ZGZiuEijMewvQ5m5zZ+xuYPgbex8F9CNwHYfIxkvFHiEcfJhrdz7DxfobNCwxb72XY+rHLqf4zMHiUuPfbBIMP4vVvZ9o9ybTtGLNO4Pc1udcSdG0DGinVNohKm4SFHMjUayxBfZu4sUvcXCdurUBTpPELaBlAlui0tnHHD8BEmeJTjb3AAPh+XV98gJgANoFEOkSe3DSpq+KzkmcZ1d5NIKmqH7y6DtUtqK0bE0TAkLQUGUbINI00R00T9h2oHED5RpLqzfiNc0xaJ+h3lul2HarNk/T6v8Zw9DmI/xH4lg2YNHn6bbIVw0RZaQOFsmq/ihIuKwI5Ua5ZBVkeXkEXbTkwlQTzXgTvf9KjzkWqN/J/4L4I4+fBfxaSv4LwEdz+Bbr1M3SqDoOqw7TmmFU8CQOjDVtL0MpBbQ2qcyCob84BRPdYs8ocew5xZ4lBbYtu++1Mp58CL417e1ODfD+MHd/1zDeo1V3l3FNu1kRhpAohVV3j+BvG/Ai7S9CUFM1DbRsa6ySadIrqW0SSrGIgMYWOAkhVK0L7UD5PVLkBt3aGcWufQfeAweAWBoMfptv7RfzpU/iT75AombRSYipUNTH1XAh8peyPbck1PyJRhgl5+udJ4eAaQLZMHdnVOFVDMqtyuq65laK/rzjOIgdUU8/klnoK3MeYdO6jU7uJTjXHpOnga5wyFWe0DNIS+jsDQiNP3Ngmaq0TtZcu3avrPYewk6PfOE2v89P43mchTOv4Lf5K7vdk4IXWIBlAxDeKH1X5MlOlT3HRRv0/z6R+D1FvGdrr1sSo70Azbxggbmsyu2fImFIZQBrSNntQvc7Y5V4jz6jt0Ovs0u/dyWT0q0STJ0imXzf17xJhMiviog2NgUK8lSAwxo083GiKH3oEQWTvU6oi5eFWOq8AVB5bKf7NMbT15ZXuX3Xm9RyVZ1ZMplFEc0cpJBNBYMKYhKgaRP8O7ufwex9l1Lwdt3OCsOsQSYvOAJKedx3Ivm+tkbS2iNt54s6y/V7XDUBWCDp5hs23Mug+SBT+A9A3+zNM+9+TxRb74jUEEC33XgmQbzNt/ARxfwU6eWhuQmPbAqTjkHQEkG2o71oTbAYQzTs2jV0uDeO39ph0rmc6uIA3fgS8v4egOB93Zwq8yKpTni5PlY5izYtk9qkYp6JOxzY1v8K2Vc5R6wjTtE55Visv3RCUKhS7IGeUi4LzsgC9YC7LoA0X97VXQgGwZhVPCRFeAO8JGD5E3LsLBmegl7NMn40xM7kEGoGklSNpbbwyQDrrBJ0NRq07mPQfI4m+ZnZwqp/XuALJsrsvJsqNBpltotEOs4HdU6BfTltEoheZNt9HPJREzENrywKktW5XZnoOSVvmRg5km18pYVs5osZJovZ7oKeUNV8A7zkIW2memjTwVB2RRSdGjxLCWKtq6o9KgGXFgurEtEzcWKLiLaaSZEzsTYi9MYk/MWSrsGpvvW/J7ItIH64G5klGZTRmjGtSH6kP1nyTSvoOeE9C52EY/DgMdu2YBRBjQq5BffWSpmivQnuDpLNB0l0xq1ZaudJ7STqb+J0dxq27mfY/C8l/ktAzuzmUSf1a/iy+BplJ3WzDjSoHmdJUEB/Rb96HPxRzbEB7DVoyr1ZIeg7RwCHpWk0ibWKo6xB3HWNSuK0bGFcvEDQ/AcOnIaxdmj+oqJFKHqkSkeFKa+jZdTUFUSqZRB8vruFTMRSmZecS7Z8XghPXFoFRIRjZW7Fr615okm9SqgagIhmKo8lIdpUSganZOGIYdEyZumy/t9EiMi81B4vK0Pw76PwOcfcCQec0QTN1LMr7LoAIBKIUIFq+pnNJgFjBsYHbyzPs3sN08NeQaGGhk9by07iv3c9CA0RAMOE+EmLSGjqa8lppYQma+NGniZO7jEc7C/kwXmzXIZSvYOLA2B4j18FzHSbuHsPBXXRbvwxodep/LcOKKWecqMYyu0bnGYlZdG73O1hO1n0Z6SGXwDQ7NzPwbD/qFUcB/pXIlAHQ8oTmX5bsttW0CfVV5L4Mk7/EG/wS7vBtuON1omkOvHW8wSphb90KjqY07J4xtYKuJubWl8JwnUnPYTB8D/CMWYyIgxrE3VQ4vAmQN+wbmAEkE+RGoqcOQwZMkj8k4l6C3qaVlNmk1F3DFUCCJZg6uAP5Ndbo925hNLmPIPhd4lCZxwvWrMmePzO89YVE9fFJUJk3mulYcNgkBjo3XbQYtQARSLxvEbt/QeB+FHf6TsajLYajJcbjNYLhBnRWQaZnW9p225il8UDaJQ+DTaaDJUbjnySOv27BqoDQWJrw+Mb/ejDlYmuQbJIoZsgY+DKAjHD5Y+B+kuGe9TDL6dVeIh6fYNhdI/KW8CcO49Eqo94dTFufhsHTELycMoCYQMtOWWNiCGkDgUNHXTiejwWIACESWNJIm6yrpm/qZ5r1JX4egi8RTn7DLFO3uuuM/FWCqQMCgzzm/TUQYCZ5GG9D6xR0zuIODphMfoFQBUfN+5apl6YsOZ7hvy6tLjRAZHWIKYz5MbNabLoY41tgyjT+PAk/TzLcTwGi5cwNgv4NdJtnGfZPMRnfSOz/FISPgvcv4CoaODVPDP8LfQKD3OFyjeuo744PHJY7sjiCOXCkczKZWppIJ5RT56Wck/JdfBvcv8V1H2Ls3s3QO4fnrs8BxCEZrRBNNmC8B63T0LqFYHAn/vS3CIKL1m+TCST9CNfw55oAiPmNMvEp71qWewmPSfBF4vhjBL2zYPwh20T1A/zunXRq9zDs30sw+TgEXwSesxNQ1S9XbXIpjxkOBAyFr6RfqtHjxkfaB5mZOrUaRWDRUvDYACSmSmKqwk7AV7HxLgT/DclTwOOMph/And5KMsgZEzToOfgDB3+cIx5uQ/skNG8j6X+I2PtT/KBpMi3JspL/xzT8JkDemG9ATJFVHJrZF2ZZNPVOyzoPv0wS/Tpe+3oSrWI194kb74DRzzHuChh/AsE/QfCSRYRQEcuZEZo0v+a5BggyVQQO1dhOwaHvdX5cH7UtzWnUqLqiRGpKhdMzWUOSNHtIGA8J/AmhOzXLyoQaR9XsJ/emjxO5H4bRLdBfwu87TAcOk5FDMNIqV564+TYYPAzBk3hB2zgtzWt+EyDH9cu/unYlMTVFNdktLFpsPIYWgQzTyAR5BpJH8btvNev5tG+G3gWIPwHTL0DUtNaTvG3hBCLtb1fohkgJ6dKJuJ4vEijm6dV19Qdzl/ohJpWWM1pEDskyCSUSswwbGLdJpKGFIUE0IIx7JhvlDPj+C2ZbANMPwehGgvEy3sRhqlU+1yHpO8Stm2D0KYifwY2qxtgMtYqctvuDGdwb46kLbmLNAyQtMGEcaRlA9JL/Dfg9wt7boasVmnfA+D7gD4BnLf9rSiFGk3POVDxSphTFVknKyqxKJ+kCXSaxj1t7aGjqg6ZGM0kuR+nLJLyUAiSe1d2QqIgZEtEhSnzj+Y+MM1UhKl8F/zfBvYPE3cL3HfzQIQwdkoEins/B+DHga0ySkpmeyRUj18yxatDXAUP/Dzulf7mPSc8LAAAAAElFTkSuQmCC +``` + +### `annotation` + +NavLink 自定义资æºçš„ Kubernetes 注释。 + +### `label` + +NavLink 自定义资æºçš„ Kubernetes 标签。 + +### `sideLabel` + +出现在左侧导航æ ä¸­çš„标签。 + +### `target` + +设置链接的锚标记 (``) 的目标属性。该属性（å–决于æµè§ˆå™¨ï¼‰ç¡®å®šå®ƒæ˜¯åœ¨æ–°çª—å£ä¸­æ‰“开还是在当å‰é€‰é¡¹å¡ä¸­æ‰“开。 + +默认值为 `_self`，å³åœ¨å½“å‰é€‰é¡¹å¡ä¸Šæ‰“开链接。è¦åœ¨æ–°çª—å£æˆ–选项å¡ä¸­æ‰“开链接，请将目标设置为 `_blank`。 + +有关目标属性的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://www.w3schools.com/tags/att_a_target.asp)。 + +### `toService` + +有五个字段用于创建如下所示的 URL：`https:///k8s/clusters//k8s/namespace//service/: :/proxy/` + +例如，Monitoring æœåŠ¡çš„链接å¯ä»¥è¿™æ ·è®¾ç½®ï¼š + +- å称：`rancher-monitoring-grafana` +- 命å空间：`cattle-monitoring-system` +- 路径：`proxy/?orgId=1` +- 端å£ï¼š`"80"` +- Scheme：`http` + +需è¦æä¾› `toService` 或 `toURL` 指令中的一个。 + +### `toUrl` + +å¯ä»¥æ˜¯ä»»ä½•é“¾æŽ¥ï¼Œç”šè‡³å¯ä»¥æ˜¯é›†ç¾¤å¤–部的链接。 + +需è¦æä¾› `toService` 或 `toURL` 指令中的一个。 + +## 链接示例 + +### `toUrl` 的链接示例 + +以下 NavLink YAML 显示了将 NavLink é…置到 Grafana 仪表æ¿çš„示例： + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + name: grafana +spec: + group: "Monitoring Dashboards" + toURL: https:///api/v1/namespaces/cattle-monitoring-system/services/http:rancher-monitoring-grafana:80/proxy/?orgId=1 +``` + +添加上述 YAML 会创建 Grafana 的链接，如以下截图所示： + +![Grafana 链接截图](/img/example-grafana-link.png) + +### `toService` 的链接示例 + +以下 YAML 显示了用于链接目标的 `toService` 示例： + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + annotations: + key: annotation + labels: + key: label + name: navlinkname +spec: + description: This is a description field # Optional. + group: "group1" # å¯é€‰ã€‚如果未æ供，则显示为独立链接。 + iconSrc: data:image/jpeg;base64,[icon source string is clipped for brevity] + label: This is a label # å¯é€‰ã€‚ + sideLabel: A side label. # å¯é€‰ã€‚ + target: _blank # å¯é€‰ã€‚_blank 表示在新选项å¡æˆ–窗å£ä¸­æ‰“开链接。 + toService: # 需è¦æä¾› toService 或 #toUrl。 + name: rancher-monitoring-grafana + namespace: cattle-monitoring-system + path: proxy/?orgId=1 + port: "80" + scheme: http +``` + +添加上é¢çš„ `toService` å‚数会创建 Grafana 的链接，如以下截图所示： + +![Grafana 链接截图](/img/example-service-link.png) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md new file mode 100644 index 00000000000..818ccfc025a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md @@ -0,0 +1,57 @@ +--- +title: é…置全局默认ç§æœ‰é•œåƒä»“库 +--- + +:::note +本页介ç»äº†å®‰è£… Rancher åŽå¦‚何从 Rancher UI é…置全局默认ç§æœ‰é•œåƒä»“库。 + +有关如何在 Rancher 安装期间设置ç§æœ‰é•œåƒä»“库的说明，请å‚阅[离线安装指å—](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)。 + +::: + +ç§æœ‰é•œåƒä»“库是集群中ç§æœ‰ã€ä¸€è‡´ä¸”集中的容器镜åƒæºã€‚ä½ å¯ä»¥ä½¿ç”¨ç§æœ‰å®¹å™¨é•œåƒä»“库，在组织内共享自定义基础镜åƒã€‚ + +在 Rancher 中设置ç§æœ‰é•œåƒä»“库主è¦æœ‰ä¸¤ç§æ–¹å¼ï¼š + +* 通过全局视图中的 **Settings** 选项å¡è®¾ç½®å…¨å±€é»˜è®¤é•œåƒä»“库。 +* 在集群级别设置下的高级选项中设置ç§æœ‰é•œåƒä»“库。 + +全局默认镜åƒä»“库适用于离线环境，å¯ç”¨äºŽä¸éœ€è¦å‡­è¯çš„é•œåƒä»“库。而集群级ç§æœ‰é•œåƒä»“库用于需è¦å‡­è¯çš„ç§æœ‰é•œåƒä»“库。 + +## å°†ä¸éœ€è¦å‡­è¯çš„ç§æœ‰é•œåƒä»“库设置为默认镜åƒä»“库 + +1. 登录 Rancher 并é…置默认管ç†å‘˜å¯†ç ã€‚ +1. 选择 **☰ > 全局设置**。 +1. 转到 `system-default-registry` 并选择 **â‹® > 编辑设置**。 +1. 输入你镜åƒä»“库的主机å和端å£ï¼ˆä¾‹å¦‚ `registry.yourdomain.com:port`）。ä¸è¦åœ¨æ–‡æœ¬å‰åŠ ä¸Š `http://` 或 `https://`。 + +**结果**：Rancher 会从你的ç§æœ‰é•œåƒä»“库中拉å–系统镜åƒã€‚ + +### 带 RKE2 下游集群的命å空间ç§æœ‰é•œåƒä»“库 + +默认情况下，大多数ç§æœ‰é•œåƒä»“库应该能与 RKE2 下游集群一起工作。 + +但是，如果你å°è¯•è®¾ç½® URL æ ¼å¼ä¸º `website/subdomain:portnumber` 的命å空间ç§æœ‰é•œåƒä»“库，则需è¦æ‰§è¡Œé¢å¤–的步骤： + +1. 选择 **☰ > 集群管ç†**。 +1. 在列表中找到 RKE2 集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. 从**集群é…ç½®**èœå•ä¸­ï¼Œé€‰æ‹©**é•œåƒä»“库**。 +1. 在**é•œåƒä»“库**中，选择**é…置高级 Containerd Mirror 和仓库认è¯é€‰é¡¹**选项。 +1. 在 **Mirrors** 下的文本字段中，输入**é•œåƒä»“库主机å**å’Œ **Mirror 端点**。 +1. å•å‡»**ä¿å­˜**。 +1. æ ¹æ®éœ€è¦å¯¹æ¯ä¸ªä¸‹æ¸¸ RKE2 集群é‡å¤æ“作。 + +## 创建集群时é…置使用凭è¯çš„ç§æœ‰é•œåƒä»“库 + +无法为æ¯ä¸ª Rancher é…置的集群全局设置具有授æƒè®¤è¯çš„ç§æœ‰é•œåƒä»“库。因此，如果你希望 Rancher é…置的集群从使用凭è¯çš„ç§æœ‰é•œåƒä»“库中拉å–é•œåƒï¼Œåˆ™æ¯æ¬¡åˆ›å»ºæ–°é›†ç¾¤æ—¶éƒ½å¿…须通过高级集群选项传递镜åƒä»“库凭è¯ã€‚ + +由于创建集群åŽæ— æ³•é…ç½®ç§æœ‰é•œåƒä»“库，因此你需è¦åœ¨åˆå§‹é›†ç¾¤è®¾ç½®æœŸé—´æ‰§è¡Œè¿™äº›æ­¥éª¤ã€‚ + +1. 选择 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 选择集群类型。 +1. 在**集群é…ç½®**中，转到**é•œåƒä»“库**选项å¡ï¼Œç„¶åŽé€‰æ‹©**为 Rancher 从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒ**。 +1. 输入镜åƒä»“库主机å和凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：新集群将从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md new file mode 100644 index 00000000000..09b1cec4990 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md @@ -0,0 +1,236 @@ +--- +title: 集群和项目角色 +--- + +集群和项目角色定义集群或项目内的用户授æƒã€‚ + +è¦ç®¡ç†è¿™äº›è§’色： + +1. å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**并转到**集群**或**项目或命å空间**选项å¡ã€‚ + +### æˆå‘˜èµ„æ ¼å’Œè§’è‰²åˆ†é… + +éžç®¡ç†ç”¨æˆ·å¯ä»¥è®¿é—®çš„项目和集群由 _æˆå‘˜èµ„æ ¼_ 决定。æˆå‘˜èµ„格是根æ®è¯¥é›†ç¾¤æˆ–项目中分é…的角色而有æƒè®¿é—®ç‰¹å®šé›†ç¾¤æˆ–项目的用户列表。æ¯ä¸ªé›†ç¾¤å’Œé¡¹ç›®éƒ½åŒ…å«ä¸€ä¸ªé€‰é¡¹å¡ï¼Œå…·æœ‰é€‚当æƒé™çš„用户å¯ä»¥ä½¿ç”¨è¯¥é€‰é¡¹å¡æ¥ç®¡ç†æˆå‘˜èµ„格。 + +创建集群或项目时，Rancher 会自动将创建者分é…为`所有者`。分é…了`所有者`角色的用户å¯ä»¥åœ¨é›†ç¾¤æˆ–项目中给其他用户分é…角色。 + +:::note + +默认情况下，éžç®¡ç†å‘˜ç”¨æˆ·æ— æ³•è®¿é—®ä»»ä½•çŽ°æœ‰é¡¹ç›®/集群。具有适当æƒé™çš„用户（通常是所有者）必须显å¼åˆ†é…项目和集群æˆå‘˜èµ„格。 + +::: + +### 集群角色 + +_集群角色_ 是你å¯ä»¥åˆ†é…给用户的角色，以授予他们对集群的访问æƒé™ã€‚集群的两个主è¦è§’色分别是`所有者`å’Œ`æˆå‘˜`。 + +- **集群所有者：** + + å¯ä»¥å®Œå…¨æŽ§åˆ¶é›†ç¾¤åŠå…¶ä¸­çš„所有资æºã€‚ + +- **集群æˆå‘˜ï¼š** + + å¯ä»¥æŸ¥çœ‹å¤§å¤šæ•°é›†ç¾¤çº§åˆ«çš„资æºå¹¶åˆ›å»ºæ–°é¡¹ç›®ã€‚ + +#### 自定义集群角色 + +Rancher 支æŒå°† _自定义集群角色_ 分é…给普通用户，而ä¸æ˜¯å…¸åž‹çš„`所有者`或`æˆå‘˜`角色。这些角色å¯ä»¥æ˜¯å†…置的自定义集群角色，也å¯ä»¥æ˜¯ Rancher 管ç†å‘˜å®šä¹‰çš„角色。这些角色便于为集群内的普通用户定义更å—é™æˆ–特定的访问æƒé™ã€‚有关内置自定义集群角色的列表，请å‚阅下表。 + +#### 集群角色å‚考 + +下表列出了å¯ç”¨çš„内置自定义集群角色，以åŠé»˜è®¤çš„集群级别角色`集群所有者`å’Œ`集群æˆå‘˜`是å¦åŒ…å«è¯¥æƒé™ï¼š + +| 内置集群角色 | 所有者 | æˆå‘˜ | +| ---------------------------------- | ------------- | --------------------------------- | +| 创建项目 | ✓ | ✓ | +| 管ç†é›†ç¾¤å¤‡ä»½Â Â Â Â Â Â Â Â Â Â Â Â  | ✓ | | +| 管ç†é›†ç¾¤åº”用商店 | ✓ | | +| 管ç†é›†ç¾¤æˆå‘˜ | ✓ | | +| 管ç†èŠ‚点[（è§ä¸‹è¡¨ï¼‰](#管ç†èŠ‚点æƒé™) | ✓ | | +| 管ç†å­˜å‚¨ | ✓ | | +| 查看所有项目 | ✓ | | +| 查看集群应用商店 | ✓ | ✓ | +| 查看集群æˆå‘˜ | ✓ | ✓ | +| 查看节点 | ✓ | ✓ | + +#### 管ç†èŠ‚点æƒé™ + +下表列出了 RKE å’Œ RKE2 中`管ç†èŠ‚点`角色å¯ç”¨çš„æƒé™ï¼š + +| 管ç†èŠ‚点æƒé™ | RKE | RKE2 | +|-----------------------------|-------- |--------- | +| SSH 访问 | ✓ | ✓ | +| 删除节点 | ✓ | ✓ | +| 集群的垂直扩缩容 | ✓ | * | +\***在 RKE2 中，你必须拥有编辑集群的æƒé™æ‰èƒ½å¯¹é›†ç¾¤è¿›è¡Œåž‚直扩缩容。** +
+ +如果需è¦äº†è§£å„个集群角色如何访问 Kubernetes 资æºï¼Œåœ¨ Rancher UI 中找到这些角色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. å•å‡»**集群**选项å¡ã€‚ +1. å•å‡»è§’色的å称。表格会显示角色授æƒçš„所有æ“作和资æºã€‚ + +:::note + +在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ + +::: + +### 为集群æˆå‘˜æ供自定义集群角色 + +在管ç†å‘˜[设置自定义集群角色åŽ](custom-roles.md)，集群所有者和管ç†å‘˜å¯ä»¥å°†è¿™äº›è§’色分é…给集群æˆå‘˜ã€‚ + +è¦å°†è‡ªå®šä¹‰è§’色分é…给新的集群æˆå‘˜ï¼Œä½ å¯ä»¥ä½¿ç”¨ Rancher UI。è¦ä¿®æ”¹çŽ°æœ‰æˆå‘˜çš„æƒé™ï¼Œä½ éœ€è¦ä½¿ç”¨ Rancher API 视图。 + +è¦å°†è§’色分é…给新的集群æˆå‘˜ï¼š + + + + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°†è§’色分é…ç»™æˆå‘˜çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群æˆå‘˜**。 +1. å•å‡»**添加**。 +1. 在**集群æƒé™**中，选择è¦åˆ†é…ç»™æˆå‘˜çš„自定义集群角色。 +1. å•å‡»**创建**。 + + + + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°†è§’色分é…ç»™æˆå‘˜çš„集群，然åŽå•å‡» **Explore**。 +1. 点击**集群 > 集群æˆå‘˜**。 +1. å•å‡»**添加**。 +1. 在**集群æƒé™**中，选择è¦åˆ†é…ç»™æˆå‘˜çš„自定义集群角色。 +1. å•å‡»**创建**。 + + + + +**结果**：æˆå‘˜å…·æœ‰æ‰€åˆ†é…的角色。 + +è¦å°†è‡ªå®šä¹‰è§’色分é…给现有集群æˆå‘˜ï¼š + +1. å•å‡» **☰ > 用户 & 认è¯**。 +1. 找到è¦åˆ†é…角色的æˆå‘˜ã€‚å•å‡» **â‹® > 编辑é…ç½®**。 +1. 如果你添加了自定义角色，它们将显示在**自定义**中。选择è¦åˆ†é…ç»™æˆå‘˜çš„角色。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：æˆå‘˜å…·æœ‰æ‰€åˆ†é…的角色。 + +### 项目角色 + +_项目角色_ 是用于授予用户访问项目æƒé™çš„角色。主è¦çš„项目角色分别是`所有者`ã€`æˆå‘˜`å’Œ`åªè¯»`。 + +- **项目所有者：** + + å¯ä»¥å®Œå…¨æŽ§åˆ¶é¡¹ç›®åŠå…¶ä¸­çš„所有资æºã€‚ + +- **项目æˆå‘˜ï¼š** + + å¯ä»¥ç®¡ç†é¡¹ç›®èŒƒå›´çš„资æºï¼Œå¦‚命å空间和工作负载，但ä¸èƒ½ç®¡ç†å…¶ä»–项目æˆå‘˜ã€‚ + +:::note + +默认情况下，Rancher çš„`项目æˆå‘˜`角色继承自 `Kubernetes-edit` 角色，而`项目所有者`角色继承自 `Kubernetes-admin` 角色。因此，`项目æˆå‘˜`å’Œ`项目所有者`角色都能管ç†å‘½å空间，包括创建和删除命å空间。 + +::: + +- **åªè¯»ï¼š** + + å¯ä»¥æŸ¥çœ‹é¡¹ç›®ä¸­çš„所有内容，但ä¸èƒ½åˆ›å»ºã€æ›´æ–°æˆ–删除任何内容。 + +:::danger + +如果用户分é…到了项目的`所有者`或`æˆå‘˜`角色，用户会自动继承`命å空间创建`角色。然而，这个角色是 [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole)，这表示角色的范围会延展到集群中的所有项目。因此，对于显å¼åˆ†é…到了项目`所有者`或`æˆå‘˜`角色的用户æ¥è¯´ï¼Œå³ä½¿åªæœ‰`åªè¯»`角色，这些用户也å¯ä»¥åœ¨åˆ†é…给他们的其他项目中创建命å空间。 + +::: + +#### 自定义项目角色 + +Rancher 支æŒå°† _自定义项目角色_ 分é…给普通用户，而ä¸æ˜¯å…¸åž‹çš„`所有者`ã€`æˆå‘˜`或`åªè¯»`角色。这些角色å¯ä»¥æ˜¯å†…置的自定义项目角色，也å¯ä»¥æ˜¯ Rancher 管ç†å‘˜å®šä¹‰çš„角色。这些角色便于为项目内的普通用户定义更å—é™æˆ–特定的访问æƒé™ã€‚有关内置自定义项目角色的列表，请å‚阅下表。 + +#### 项目角色å‚考 + +下表列出了 Rancher 中å¯ç”¨çš„内置自定义项目角色，以åŠè¿™äº›è§’色是å¦ç”±`所有者`,`æˆå‘˜`或`åªè¯»`角色授予的： + +| 内置项目角色 | 所有者 | æˆå‘˜ | åªè¯» | +| ---------------------------------- | ------------- | ----------------------------- | ------------- | +| 管ç†é¡¹ç›®æˆå‘˜ | ✓ | | | +| 创建命å空间 | ✓ | ✓ | | +| 管ç†é…置映射 | ✓ | ✓ | | +| ç®¡ç† Ingress | ✓ | ✓ | | +| 管ç†é¡¹ç›®åº”用商店 | ✓ | | | +| 管ç†å¯†æ–‡ | ✓ | ✓ | | +| ç®¡ç† ServiceAccount | ✓ | ✓ | | +| 管ç†æœåŠ¡ | ✓ | ✓ | | +| 管ç†å· | ✓ | ✓ | | +| 管ç†å·¥ä½œè´Ÿè½½ | ✓ | ✓ | | +| 查看密文 | ✓ | ✓ | | +| 查看é…置图 | ✓ | ✓ | ✓ | +| 查看 Ingress | ✓ | ✓ | ✓ | +| 查看项目æˆå‘˜ | ✓ | ✓ | ✓ | +| 查看项目应用商店 | ✓ | ✓ | ✓ | +| 查看 ServiceAccount | ✓ | ✓ | ✓ | +| 查看æœåŠ¡ | ✓ | ✓ | ✓ | +| æŸ¥çœ‹å· | ✓ | ✓ | ✓ | +| 查看工作负载 | ✓ | ✓ | ✓ | + +:::note 注æ„事项： + +- 上é¢åˆ—出的æ¯ä¸ªé¡¹ç›®è§’色（包括`所有者`ã€`æˆå‘˜`å’Œ`åªè¯»`）å‡ç”±å¤šä¸ªè§„则组æˆï¼Œè¿™äº›è§„则授予对å„ç§èµ„æºçš„访问æƒé™ã€‚ä½ å¯ä»¥åœ¨**全局 > 安全 > 角色**页é¢ä¸ŠæŸ¥çœ‹è§’色åŠå…¶è§„则。 +- 在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ +- `管ç†é¡¹ç›®æˆå‘˜`角色å…许项目所有者管ç†é¡¹ç›®çš„所有æˆå‘˜ï¼Œ**并**授予这些æˆå‘˜ä»»ä½•é¡¹ç›®èŒƒå›´çš„角色（ä¸è®ºä»–们是å¦æœ‰æƒè®¿é—®é¡¹ç›®èµ„æºï¼‰ã€‚å•ç‹¬åˆ†é…此角色时è¦å°å¿ƒã€‚ + +::: + +### 定义自定义角色 +如å‰æ‰€è¿°ï¼Œä½ å¯ä»¥å®šä¹‰è‡ªå®šä¹‰è§’色，并将这些角色用在集群或项目中。上下文字段定义了角色是å¦æ˜¾ç¤ºåœ¨é›†ç¾¤æˆå‘˜é¡µé¢ã€é¡¹ç›®æˆå‘˜é¡µé¢æˆ–åŒæ—¶æ˜¾ç¤ºåœ¨è¿™ä¸¤ä¸ªé¡µé¢ã€‚ + +定义自定义角色时，你å¯ä»¥æŽˆäºˆå¯¹ç‰¹å®šèµ„æºçš„访问æƒé™ï¼Œæˆ–指定自定义角色应继承的角色。自定义角色å¯ä»¥ç”±ç‰¹å®šæŽˆæƒå’Œç»§æ‰¿è§’色组æˆã€‚所有授æƒéƒ½æ˜¯ç´¯åŠ çš„。æ¢è¨€ä¹‹ï¼Œå¦‚果你为特定资æºå®šä¹‰æ›´å—é™çš„授æƒï¼Œè‡ªå®šä¹‰è§’色继承的角色中定义的更广泛的授æƒ**ä¸ä¼š**被覆盖。 + +### 默认集群和项目角色 + +默认情况下，在普通用户创建新集群或项目时，他们会自动分é…到所有者的角色，å³[集群所有者](#集群角色)或[项目所有者](#项目角色)。但是，在æŸäº›ç»„织中，这些角色å¯èƒ½ä¼šè¢«è®¤ä¸ºæœ‰è¿‡å¤šçš„管ç†è®¿é—®æƒé™ã€‚在这ç§æƒ…况下，你å¯ä»¥å°†é»˜è®¤è§’色更改为更具é™åˆ¶æ€§çš„角色，例如一组å•ç‹¬çš„角色或一个自定义角色。 + +更改默认集群/项目角色有以下两ç§æ–¹æ³•ï¼š + +- **分é…自定义角色**：为你的[集群](#自定义集群角色)或[项目](#自定义项目角色)创建一个[自定义角色](custom-roles.md)，然åŽå°†è‡ªå®šä¹‰è§’色设置为默认。 + +- **分é…å•ç‹¬çš„角色**：将多个[集群](#集群角色å‚考)/[项目](#项目角色å‚考)角色é…置为默认角色，并分é…给创建的用户。 + + 例如，你å¯ä»¥é€‰æ‹©æ··åˆä½¿ç”¨å¤šä¸ªè§’色（例如`管ç†èŠ‚点`å’Œ`管ç†å­˜å‚¨`），而ä¸æ˜¯ä½¿ç”¨ç»§æ‰¿çš„角色（例如`集群所有者`）。 + +:::note + +- 虽然你å¯ä»¥[é”定](locked-roles.md)一个默认角色，但系统ä»ä¼šå°†è¿™ä¸ªè§’色分é…给创建集群/项目的用户。 +- åªæœ‰åˆ›å»ºé›†ç¾¤/项目的用户æ‰èƒ½ç»§æ‰¿ä»–们的角色。对于之åŽæ·»åŠ ä¸ºé›†ç¾¤/项目æˆå‘˜çš„用户，你必须显å¼åˆ†é…角色。 + +::: + +### 为集群和项目创建者é…置默认角色 + +ä½ å¯ä»¥æ›´æ”¹ä¸ºåˆ›å»ºé›†ç¾¤æˆ–项目的用户自动创建的角色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. å•å‡»**集群**或**项目或命å空间**选项å¡ã€‚ +1. 找到你è¦ç”¨ä½œé»˜è®¤è§’色的自定义或å•ä¸ªè§’色。然åŽé€šè¿‡é€‰æ‹© **â‹® > 编辑é…ç½®**æ¥ç¼–辑角色。 +1. 在**集群创建者的默认角色**或**项目创建者的默认角色**中，将角色å¯ç”¨ä¸ºé»˜è®¤ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：默认角色已根æ®ä½ çš„更改é…置。分é…给集群/项目创建者的角色会在**集群创建者的默认角色/项目创建者的默认角色**列中勾选。 + +如果è¦åˆ é™¤é»˜è®¤è§’色，请编辑æƒé™ï¼Œå¹¶åœ¨é»˜è®¤è§’色选项中选择**å¦**。 + +### 撤销集群æˆå‘˜èµ„æ ¼ + +如果你撤销一个普通用户的集群æˆå‘˜èµ„格，而且该用户已显å¼åˆ†é…集群的集群 _å’Œ_ 项目的æˆå‘˜èµ„格，该普通用户将[失去集群角色](#集群角色)但[ä¿ç•™é¡¹ç›®è§’色](#项目角色)。æ¢å¥è¯è¯´ï¼Œå³ä½¿ä½ å·²ç»æ’¤é”€äº†ç”¨æˆ·è®¿é—®é›†ç¾¤å’Œå…¶ä¸­çš„节点的æƒé™ï¼Œä½†è¯¥æ™®é€šç”¨æˆ·ä»ç„¶å¯ä»¥ï¼š + +- 访问他们拥有æˆå‘˜èµ„格的项目。 +- 行使分é…给他们的任何[å•ä¸ªé¡¹ç›®è§’色](#项目角色å‚考)。 + +如果你想完全撤销用户在集群中的访问æƒé™ï¼Œè¯·åŒæ—¶æ’¤é”€ä»–们的集群和项目æˆå‘˜èµ„格。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md new file mode 100644 index 00000000000..0ab8560983b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md @@ -0,0 +1,120 @@ +--- +title: 自定义角色 +--- + +在 Rancher 中，_角色_ 决定了用户å¯ä»¥åœ¨é›†ç¾¤æˆ–项目中执行哪些æ“作。 + +请注æ„，_角色_ 与 _æƒé™_ ä¸åŒï¼Œæƒé™å†³å®šçš„是你å¯ä»¥è®¿é—®å“ªäº›é›†ç¾¤å’Œé¡¹ç›®ã€‚ + +:::danger + +自定义角色å¯ä»¥å¯ç”¨æƒé™æå‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](#æƒé™æå‡)。 + +::: + + +## 先决æ¡ä»¶ + +è¦å®Œæˆæ­¤é¡µé¢ä¸Šçš„任务，需è¦ä»¥ä¸‹æƒé™ä¹‹ä¸€ï¼š + +- [管ç†å‘˜å…¨å±€æƒé™](global-permissions.md)。 +- 分é…了[管ç†è§’色](global-permissions.md)çš„[自定义全局æƒé™](global-permissions.md#自定义全局æƒé™)。 + +## 创建自定义角色 + +虽然 Rancher æ供一组开箱å³ç”¨çš„默认用户角色，但你还å¯ä»¥åˆ›å»ºé»˜è®¤çš„自定义角色，从而在 Rancher 中为用户æ供更精细的æƒé™ã€‚ + +添加自定义角色的步骤因 Rancher 的版本而异。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 选择一个选项å¡æ¥ç¡®å®šè¦æ·»åŠ çš„角色的范围。这些选项å¡æ˜¯ï¼š + +- **全局**：仅在å…许æˆå‘˜ç®¡ç†å…¨å±€èŒƒå›´çš„资æºæ—¶ï¼Œå¯ä»¥åˆ†é…该角色。 +- **集群**：仅在å‘集群添加/管ç†æˆå‘˜æ—¶ï¼Œå¯ä»¥åˆ†é…该角色。 +- **项目或命å空间**：仅在å‘项目或命å空间添加/管ç†æˆå‘˜æ—¶ï¼Œå¯ä»¥åˆ†é…该角色。 + +1. æ ¹æ®æ‰€éœ€è¦çš„范围，å•å‡»**创建全局角色**ã€**创建集群角色**或**创建项目或命å空间的角色**。 +1. 输入角色的**å称**。 +1. å¯é€‰ï¼šé€‰æ‹©**集群创建者的默认角色/项目创建者的默认角色**选项，以将该角色分é…给集群/项目创建者。使用此功能，你å¯ä»¥æ‰©å±•æˆ–é™åˆ¶é›†ç¾¤/项目创建者的默认角色。 + + > 开箱å³ç”¨çš„**集群创建者的默认角色**å’Œ**项目创建者的默认角色**分别是`集群所有者`å’Œ`项目所有者`。 + +1. 使用**授æƒèµ„æº**选项将å„个 [Kubernetes API 端点](https://kubernetes.io/docs/reference/)分é…给角色。 + + > 在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ + + > **资æº**文本字段å¯ä»¥ç”¨æ¥æœç´¢é¢„定义的 Kubernetes API 资æºï¼Œæˆ–者为授æƒè¾“入自定义资æºå称。在此字段中输入资æºå称åŽï¼Œå¿…须从下拉列表中选择预定义或`（自定义）`资æºã€‚ + + 你还å¯ä»¥é€‰æ‹©æ¯ä¸ªåˆ†é…的端点å¯ç”¨çš„ cURL 方法（`Create`ã€`Delete`ã€`Get` 等）。 + +1. 使用 **Inherit from** 选项将å„个 Rancher 角色分é…给你的自定义角色。请注æ„，如果自定义角色从父角色继承，你需è¦å…ˆåˆ é™¤å­è§’色æ‰èƒ½åˆ é™¤çˆ¶è§’色。 + +1. å•å‡»**创建**。 + +## 创建从å¦ä¸€ä¸ªè§’色继承的自定义角色 + +如果你有一组需è¦åœ¨ Rancher 中具有相åŒè®¿é—®æƒé™çš„用户，一ç§èŠ‚çœæ—¶é—´çš„方法是创建一个新的自定义角色，而该角色的规则都是从å¦ä¸€ä¸ªè§’色（例如管ç†å‘˜è§’色）å¤åˆ¶è€Œæ¥çš„。这样，你åªéœ€è¦é…置现有角色和新角色之间ä¸åŒçš„部分。 + +然åŽï¼Œä½ å¯ä»¥å°†è‡ªå®šä¹‰è§’色分é…给用户或组。该角色在用户首次登录 Rancher 时生效。 + +è¦åŸºäºŽçŽ°æœ‰è§’色创建自定义角色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. å•å‡»**集群**或**项目或命å空间**选项å¡ã€‚æ ¹æ®æ‰€éœ€è¦çš„范围，å•å‡»**创建集群角色**或**创建项目或命å空间的角色**。请注æ„，åªæœ‰é›†ç¾¤è§’色和项目/命å空间角色å¯ä»¥ä»Žå¦ä¸€ä¸ªè§’色继承。 +1. 输入角色的å称。 +1. 在 **Inherit From** 选项å¡ä¸­ï¼Œé€‰æ‹©è‡ªå®šä¹‰è§’色需è¦ä»Žå“ªä¸ªè§’色继承æƒé™ã€‚ +1. 在**授æƒèµ„æº**选项å¡ä¸­ï¼Œé€‰æ‹©æ‹¥æœ‰è‡ªå®šä¹‰è§’色的用户è¦å¯ç”¨çš„ Kubernetes 资æºæ“作。 + + > **资æº**文本字段å¯ä»¥ç”¨æ¥æœç´¢é¢„定义的 Kubernetes API 资æºï¼Œæˆ–者为授æƒè¾“入自定义资æºå称。在此字段中输入资æºå称åŽï¼Œå¿…须从下拉列表中选择预定义或`（自定义）`资æºã€‚ +1. å¯é€‰ï¼šå°†è§’色设置为默认。 +1. å•å‡»**创建**。 + +## 删除自定义角色 + +删除自定义角色时，具有此自定义角色的所有全局角色绑定（Global Role Bindings）都将被删除。 + +如果æŸä¸ªç”¨æˆ·ä»…分é…了一个自定义全局角色，而且你删除了这个角色，该用户将ä¸èƒ½å†è®¿é—® Rancher。è¦è®©ç”¨æˆ·é‡æ–°èŽ·å¾—访问æƒé™ï¼Œç®¡ç†å‘˜éœ€è¦ç¼–辑用户并应用新的全局æƒé™ã€‚ + +自定义角色å¯ä»¥åˆ é™¤ï¼Œä½†å†…置角色ä¸èƒ½åˆ é™¤ã€‚ + +è¦åˆ é™¤è‡ªå®šä¹‰è§’色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +2. 转到è¦åˆ é™¤çš„自定义全局角色，然åŽå•å‡» **â‹® (…) > 删除**。 +3. å•å‡»**删除**。 + +## 为组分é…自定义角色 + +如果你有一组需è¦åœ¨ Rancher 中具有相åŒè®¿é—®æƒé™çš„用户，一ç§èŠ‚çœæ—¶é—´çš„方法是创建一个新的自定义角色。将角色分é…给组时，组中的用户在首次登录 Rancher 时就会拥有é…置的访问级别。 + +组中的用户登录时，他们默认获得内置的**普通用户**全局角色。他们还将获得分é…给他们的组的æƒé™ã€‚ + +如果将用户从外部身份验è¯ç³»ç»Ÿçš„组中删除，用户将失去分é…给该组的自定义全局角色的æƒé™ã€‚但是，用户ä»ä¼šæ‹¥æœ‰**普通用户**角色。 + +:::note 先决æ¡ä»¶ï¼š + +åªæœ‰åœ¨ä»¥ä¸‹æƒ…况下，你æ‰èƒ½å°†å…¨å±€è§’色分é…给组： + +* 你已设置[外部身份验è¯æ供程åº](../../../../pages-for-subheaders/authentication-config.md#外部验è¯ä¸Žæœ¬åœ°éªŒè¯)。 +* 外部身份验è¯æ供程åºæ”¯æŒ[用户组](../../authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md)。 +* 你已使用身份验è¯æ供程åºè®¾ç½®äº†è‡³å°‘一个用户组。 + +::: + +è¦å°†è‡ªå®šä¹‰è§’色分é…给组，请执行以下步骤： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**组**。 +1. 转到将分é…自定义角色的组，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 如果你已创建角色，角色将显示在**自定义**中。选择è¦åˆ†é…给组的自定义角色。 +1. å¯é€‰ï¼šåœ¨**全局æƒé™**或**内置角色**中，选择è¦åˆ†é…给该组的其他æƒé™ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：自定义角色将在组内用户登录 Rancher 时生效。 + +## æƒé™æå‡ + +`é…置应用商店`这个自定义æƒé™å¾ˆå¼ºå¤§ï¼Œåº”谨慎使用。如果管ç†å‘˜å°†`é…置应用商店`æƒé™åˆ†é…给普通用户，å¯èƒ½ä¼šå¯¼è‡´æƒé™æå‡ã€‚在这ç§æƒ…况下，用户å¯ä»¥è®©è‡ªå·±å¯¹ Rancher é…置的集群进行管ç†å‘˜è®¿é—®ã€‚因此，拥有此æƒé™çš„任何用户都应被视为管ç†å‘˜ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md new file mode 100644 index 00000000000..3fb65c5d7e0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md @@ -0,0 +1,247 @@ +--- +title: 全局æƒé™ +--- + +_æƒé™_ 是你在为用户选择自定义æƒé™æ—¶å¯ä»¥åˆ†é…的个人访问æƒé™ã€‚ + +全局æƒé™å®šä¹‰ç”¨æˆ·åœ¨ä»»ä½•ç‰¹å®šé›†ç¾¤ä¹‹å¤–的授æƒã€‚Rancher æ供四ç§å¼€ç®±å³ç”¨çš„默认全局æƒé™ï¼š`Administrator` (管ç†å‘˜)ã€`Restricted Admin` (å—é™ç®¡ç†å‘˜)ã€`Standard User` (标准用户) å’Œ `User-Base` 用户。 + +- **管ç†å‘˜**：å¯ä»¥å®Œå…¨æŽ§åˆ¶æ•´ä¸ª Rancher 系统和其中的所有集群。 + +- **å—é™ç®¡ç†å‘˜**：å¯ä»¥å®Œå…¨æŽ§åˆ¶ä¸‹æ¸¸é›†ç¾¤ï¼Œä½†ä¸èƒ½æ›´æ”¹æœ¬åœ° Kubernetes 集群。 + +- **普通用户**：å¯ä»¥åˆ›å»ºæ–°é›†ç¾¤å¹¶ä½¿ç”¨å®ƒä»¬ã€‚普通用户还å¯ä»¥åœ¨è‡ªå·±çš„集群中å‘其他用户分é…集群æƒé™ã€‚ + +- **User-Base 用户**：åªæœ‰ç™»å½•æƒé™ã€‚ + +你无法更新或删除内置的全局æƒé™ã€‚ + +## å—é™ç®¡ç†å‘˜ + +Rancher 2.5 创建了一个新的 `restricted-admin` 角色，以防止本地 Rancher Server Kubernetes 集群的æƒé™æå‡ã€‚此角色对 Rancher 管ç†çš„所有下游集群具有完全管ç†å‘˜æƒé™ï¼Œä½†æ²¡æœ‰æ›´æ”¹æœ¬åœ° Kubernetes 集群的æƒé™ã€‚ + +`restricted-admin` å¯ä»¥åˆ›å»ºå…¶ä»–具有åŒæ ·è®¿é—®æƒé™çš„ `restricted-admin` 用户。 + +Rancher 还增加了一个新设置，æ¥å°†åˆå§‹å¯åŠ¨çš„管ç†å‘˜è®¾ç½®ä¸º `restricted-admin` 角色。该设置适用于 Rancher Server 首次å¯åŠ¨æ—¶åˆ›å»ºçš„第一个用户。如果设置了这个环境å˜é‡ï¼Œåˆ™ä¸ä¼šåˆ›å»ºå…¨å±€ç®¡ç†å‘˜ï¼Œä¹Ÿå°±æ— æ³•é€šè¿‡ Rancher 创建全局管ç†å‘˜ã€‚ + +è¦ä»¥ `restricted-admin` 作为åˆå§‹ç”¨æˆ·æ¥å¯åŠ¨ Rancher，你需è¦ä½¿ç”¨ä»¥ä¸‹çŽ¯å¢ƒå˜é‡æ¥å¯åŠ¨ Rancher Server： + +``` +CATTLE_RESTRICTED_DEFAULT_ADMIN=true +``` +### `restricted-admin` çš„æƒé™åˆ—表 + +下表列出了 `restricted-admin` 与 `Administrator` å’Œ `Standard User` 角色相比应具有的æƒé™å’Œæ“作： + +| 类别 | æ“作 | 全局管ç†å‘˜ | 普通用户 | å—é™ç®¡ç†å‘˜ | å—é™ç®¡ç†å‘˜çš„注æ„事项 | +| -------- | ------ | ------------ | ------------- | ---------------- | ------------------------------- | +| 本地集群功能 | 管ç†æœ¬åœ°é›†ç¾¤ï¼ˆåˆ—出ã€ç¼–辑ã€å¯¼å…¥ä¸»æœºï¼‰ | 是 | å¦ | å¦ | | +| | 创建项目/命å空间 | 是 | å¦ | å¦ | | +| | 添加集群/项目æˆå‘˜ | 是 | å¦ | å¦ | | +| | 全局 DNS | 是 | å¦ | å¦ | | +| | 访问 CRD å’Œ CR 的管ç†é›†ç¾¤ | 是 | å¦ | 是 | | +| | å¦å­˜ä¸º RKE æ¨¡æ¿ | 是 | å¦ | å¦ | | +| 安全 | | | | | | +| å¯ç”¨èº«ä»½éªŒè¯ | é…ç½®èº«ä»½éªŒè¯ | 是 | å¦ | 是 | | +| 角色 | 创建/åˆ†é… GlobalRoles | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | è®¤è¯ Webhook å…许为已ç»å­˜åœ¨çš„æƒé™åˆ›å»º globalrole | +| | 创建/åˆ†é… ClusterRoles | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | ä¸åœ¨æœ¬åœ°é›†ç¾¤ä¸­ | +| | 创建/åˆ†é… ProjectRoles | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | ä¸åœ¨æœ¬åœ°é›†ç¾¤ä¸­ | +| 用户 | 添加用户/编辑/删除/åœç”¨ç”¨æˆ· | 是 | å¦ | 是 | | +| 组 | 将全局角色分é…给组 | 是 | å¦ | 是 | 在 Webhook å…许的范围内 | +| | 刷新组 | 是 | å¦ | 是 | | +| PSP | ç®¡ç† PSP æ¨¡æ¿ | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | 与 PSP 的全局管ç†å‘˜æƒé™ç›¸åŒ | +| 工具 | | | | | | +| | ç®¡ç† RKE æ¨¡æ¿ | 是 | å¦ | 是 | | +| | 管ç†å…¨å±€åº”用商店 | 是 | å¦ | 是 | 无法编辑/删除内置系统应用商店。å¯ä»¥ç®¡ç† Helm 库 | +| | 集群驱动 | 是 | å¦ | 是 | | +| | 主机驱动 | 是 | å¦ | 是 | | +| | GlobalDNS æ供商 | 是 | 是（自己） | 是 | | +| | GlobalDNS æ¡ç›® | 是 | 是（自己） | 是 | | +| 设置 | | | | | | +| | 管ç†è®¾ç½® | 是 | å¦ï¼ˆå¯åˆ—出） | å¦ï¼ˆå¯åˆ—出） | | +| 用户 | | | | | | +| | ç®¡ç† API 密钥 | 是（管ç†æ‰€æœ‰ï¼‰ | 是（管ç†è‡ªå·±çš„） | 是（管ç†è‡ªå·±çš„） | | +| | 管ç†èŠ‚ç‚¹æ¨¡æ¿ | 是 | 是（管ç†è‡ªå·±çš„） | 是（管ç†è‡ªå·±çš„） | åªèƒ½ç®¡ç†è‡ªå·±çš„节点模æ¿ï¼Œä¸èƒ½ç®¡ç†å…¶ä»–用户创建的节点模æ¿ã€‚ | +| | 管ç†äº‘å‡­è¯ | 是 | 是（管ç†è‡ªå·±çš„） | 是（管ç†è‡ªå·±çš„） | åªèƒ½ç®¡ç†è‡ªå·±çš„云凭è¯ï¼Œä¸èƒ½ç®¡ç†å…¶ä»–用户创建的云凭è¯ã€‚ | +| 下游集群 | 创建集群 | 是 | 是 | 是 | | +| | 编辑集群 | 是 | 是 | 是 | | +| | è½®æ¢è¯ä¹¦ | 是 | | 是 | | +| | ç«‹å³åˆ›å»ºå¿«ç…§ | 是 | | 是 | | +| | æ¢å¤å¿«ç…§ | 是 | | 是 | | +| | å¦å­˜ä¸º RKE æ¨¡æ¿ | 是 | å¦ | 是 | | +| | è¿è¡Œ CIS 扫æ | 是 | 是 | 是 | | +| | 添加æˆå‘˜ | 是 | 是 | 是 | | +| | 创建项目 | 是 | 是 | 是 | | +| 自 2.5 起的功能 Chart | | | | | | +| | 安装 Fleet | 是 | | 是 | 无法在本地集群中è¿è¡Œ Fleet | +| | 部署 EKS 集群 | 是 | 是 | 是 | | +| | 部署 GKE 集群 | 是 | 是 | 是 | | +| | 部署 AKS 集群 | 是 | 是 | 是 | | + + +### 将全局管ç†å‘˜æ›´æ”¹ä¸ºå—é™ç®¡ç†å‘˜ + +如果 Rancher å·²ç»æœ‰ä¸€ä¸ªå…¨å±€ç®¡ç†å‘˜ï¼Œåˆ™åº”该将所有全局管ç†å‘˜æ›´æ”¹ä¸ºæ–°çš„ `restricted-admin`。 + +ä½ å¯ä»¥å‰å¾€**安全 > 用户**，并将所有管ç†å‘˜è§’色转为å—é™ç®¡ç†å‘˜ã€‚ + +已登录的用户å¯ä»¥æ ¹æ®éœ€è¦å°†è‡ªå·±æ›´æ”¹ä¸º `restricted-admin`，但这应该是他们的最åŽä¸€æ­¥æ“作，å¦åˆ™ä»–们将没有进行该æ“作的æƒé™ã€‚ + +## 分é…全局æƒé™ + +本地用户的全局æƒé™åˆ†é…与使用外部身份验è¯ç™»å½• Rancher 的用户ä¸åŒã€‚ + +### 新本地用户的全局æƒé™ + +在创建新本地用户时，请在填写**添加用户**表å•æ—¶ä¸ºä»–分é…全局æƒé™ã€‚ + +如果需è¦æŸ¥çœ‹æ–°ç”¨æˆ·çš„默认æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. **角色**页é¢æœ‰æŒ‰èŒƒå›´åˆ†ç»„的角色选项å¡ã€‚æ¯ä¸ªè¡¨éƒ½åˆ—出了范围内的角色。在**全局**选项å¡çš„**新用户的默认角色**列中，默认授予新用户的æƒé™ç”¨å¤é€‰æ ‡è®°è¡¨ç¤ºã€‚ + +ä½ å¯ä»¥[更改默认全局æƒé™æ¥æ»¡è¶³ä½ çš„需è¦](#é…置默认全局æƒé™)。 + +### 使用外部身份验è¯ç¨‹åºçš„用户的全局æƒé™ + +当用户首次使用外部身份验è¯æ供程åºç™»å½• Rancher 时，他们会自动分é…到**新用户的默认角色**的全局æƒé™ã€‚默认情况下，Rancher ä¸ºæ–°ç”¨æˆ·åˆ†é… **Standard User** æƒé™ã€‚ + +如果需è¦æŸ¥çœ‹æ–°ç”¨æˆ·çš„默认æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. **角色**页é¢æœ‰æŒ‰èŒƒå›´åˆ†ç»„的角色选项å¡ã€‚æ¯ä¸ªè¡¨éƒ½åˆ—出了范围内的角色。在æ¯ä¸ªé¡µé¢çš„**新用户的默认角色**列中，默认授予新用户的æƒé™ç”¨å¤é€‰æ ‡è®°è¡¨ç¤ºã€‚ + +ä½ å¯ä»¥[更改默认æƒé™æ¥æ»¡è¶³ä½ çš„需è¦](#é…置默认全局æƒé™)。 + +æƒé™å¯ä»¥[分é…](#为å•ä¸ªç”¨æˆ·é…置全局æƒé™)ç»™å•ä¸ªç”¨æˆ·ã€‚ + +如果外部身份验è¯æ供程åºæ”¯æŒç»„，你å¯ä»¥[åŒæ—¶ä¸ºç»„中的æ¯ä¸ªæˆå‘˜åˆ†é…角色](#为组é…置全局æƒé™)。 + +## 自定义全局æƒé™ + +使用自定义æƒé™å¯ä»¥ä¸ºç”¨æˆ·é…置在 Rancher 中的更å—é™æˆ–指定的访问æƒé™ã€‚ + +当æ¥è‡ª[外部身份验è¯ç³»ç»Ÿ](../../../../pages-for-subheaders/authentication-config.md)的用户首次登录 Rancher 时，他们会自动分é…到一组全局æƒé™ï¼ˆä»¥ä¸‹ç®€ç§°æƒé™ï¼‰ã€‚默认情况下，用户第一次登录åŽä¼šè¢«åˆ›å»ºä¸ºç”¨æˆ·ï¼Œå¹¶åˆ†é…到默认的`用户`æƒé™ã€‚标准的`用户`æƒé™å…许用户登录和创建集群。 + +但是，在æŸäº›ç»„织中，这些æƒé™å¯èƒ½ä¼šè¢«è®¤ä¸ºæƒé™è¿‡å¤§ã€‚ä½ å¯ä»¥ä¸ºç”¨æˆ·åˆ†é…一组更具é™åˆ¶æ€§çš„自定义全局æƒé™ï¼Œè€Œä¸æ˜¯ä¸ºç”¨æˆ·åˆ†é… `Administrator` 或 `Standard User` 的默认全局æƒé™ã€‚ + +默认角色（管ç†å‘˜å’Œæ ‡å‡†ç”¨æˆ·ï¼‰éƒ½å†…置了多个全局æƒé™ã€‚系统管ç†å‘˜è§’色包括所有全局æƒé™ï¼Œè€Œé»˜è®¤ç”¨æˆ·è§’色包括三个全局æƒé™ï¼Œåˆ†åˆ«æ˜¯åˆ›å»ºé›†ç¾¤ï¼Œä½¿ç”¨åº”用商店模æ¿ï¼Œå’Œ User Base（登录 Rancher 的最低æƒé™ï¼‰ã€‚æ¢å¥è¯è¯´ï¼Œè‡ªå®šä¹‰å…¨å±€æƒé™æ˜¯æ¨¡å—化的，因此，如果你è¦æ›´æ”¹é»˜è®¤ç”¨æˆ·è§’色æƒé™ï¼Œä½ å¯ä»¥é€‰æ‹©éœ€è¦åŒ…括在新的默认用户角色中的全局æƒé™å­é›†ã€‚ + +管ç†å‘˜å¯ä»¥é€šè¿‡å¤šç§æ–¹å¼å¼ºåˆ¶æ‰§è¡Œè‡ªå®šä¹‰å…¨å±€æƒé™ï¼š + +- [更改新用户的默认æƒé™](#é…置默认全局æƒé™) +- [为å•ä¸ªç”¨æˆ·é…置全局æƒé™](#为å•ä¸ªç”¨æˆ·é…置全局æƒé™) +- [为组é…置全局æƒé™](#为组é…置全局æƒé™) + +### 自定义全局æƒé™å‚考 + +下表列出了æ¯ä¸ªå¯ç”¨çš„自定义全局æƒé™ï¼Œä»¥åŠè¯¥æƒé™æ˜¯å¦åŒ…å«åœ¨é»˜è®¤å…¨å±€æƒé™ `Administrator`ã€`Standard User` å’Œ `User-Base` 中： + +| 自定义全局æƒé™ | 管ç†å‘˜ | 普通用户 | User-Base | +| ---------------------------------- | ------------- | ------------- |-----------| +| 创建集群 | ✓ | ✓ | | +| 创建 RKE æ¨¡æ¿ | ✓ | ✓ | | +| 管ç†èº«ä»½éªŒè¯ | ✓ | | | +| 管ç†åº”用商店 | ✓ | | | +| 管ç†é›†ç¾¤é©±åŠ¨ | ✓ | | | +| 管ç†ä¸»æœºé©±åŠ¨ | ✓ | | | +| ç®¡ç† PodSecurityPolicy æ¨¡æ¿ | ✓ | | | +| 管ç†è§’色 | ✓ | | | +| 管ç†è®¾ç½® | ✓ | | | +| 管ç†ç”¨æˆ· | ✓ | | | +| ä½¿ç”¨åº”ç”¨å•†åº—æ¨¡æ¿ | ✓ | ✓ | | +| User-Base（基本登录访问） | ✓ | ✓ | | + +如果需è¦æŸ¥çœ‹æ¯ä¸ªå…¨å±€æƒé™å¯¹åº”哪些 Kubernetes 资æºï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 如果å•å‡»å•ä¸ªè§’色的å称，表格会显示该角色授æƒçš„所有æ“作和资æºã€‚ + +:::note 注æ„事项： + +- 上é¢åˆ—出的æ¯ä¸ªæƒé™éƒ½åŒ…å«å¤šä¸ªæœªåœ¨ Rancher UI 中列出的æƒé™ã€‚如果需è¦èŽ·å–完整æƒé™åˆ—表以åŠç»„æˆæƒé™çš„规则，请通过 `/v3/globalRoles` API 进行访问。 +- 在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ + +::: + +### é…置默认全局æƒé™ + +如果你想é™åˆ¶æ–°ç”¨æˆ·çš„默认æƒé™ï¼Œä½ å¯ä»¥åˆ é™¤ä½œä¸ºé»˜è®¤è§’色的`用户`æƒé™ï¼Œç„¶åŽåˆ†é…多个å•ç‹¬çš„æƒé™ä½œä¸ºé»˜è®¤æƒé™ã€‚你也å¯ä»¥åœ¨ä¸€ç»„其他标准æƒé™ä¹‹ä¸Šæ·»åŠ ç®¡ç†æƒé™ã€‚ + +:::note + +默认角色仅分é…给从外部身份验è¯æ供程åºæ·»åŠ çš„用户。对于本地用户，在将用户添加到 Rancher 时，必须显å¼åˆ†é…全局æƒé™ã€‚ä½ å¯ä»¥åœ¨æ·»åŠ ç”¨æˆ·æ—¶è‡ªå®šä¹‰è¿™äº›å…¨å±€æƒé™ã€‚ + +::: + +è¦æ›´æ”¹åœ¨å¤–部用户首次登录时分é…给他们的默认全局æƒé™ï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。在**角色**页é¢ä¸Šï¼Œç¡®ä¿é€‰æ‹©äº†**全局**选项å¡ã€‚ +1. 查找è¦æ·»åŠ æˆ–删除的默认æƒé™é›†ã€‚然åŽï¼Œé€šè¿‡é€‰æ‹© **â‹® > 编辑é…ç½®**æ¥ç¼–辑æƒé™ã€‚ +1. 如果è¦å°†æƒé™æ·»åŠ ä¸ºé»˜è®¤æƒé™ï¼Œè¯·é€‰æ‹©**是：新用户的默认角色**，然åŽå•å‡»**ä¿å­˜**。如果è¦åˆ é™¤é»˜è®¤æƒé™ï¼Œè¯·ç¼–辑该æƒé™å¹¶é€‰æ‹©**å¦**。 + +**结果**：默认全局æƒé™å·²æ ¹æ®ä½ çš„更改é…置。分é…给新用户的æƒé™ä¼šåœ¨**新用户的默认角色**列中显示为å¤é€‰æ ‡è®°ã€‚ + +### 为å•ä¸ªç”¨æˆ·é…置全局æƒé™ + +è¦ä¸ºå•ä¸ªç”¨æˆ·é…ç½®æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. 找到è¦æ›´æ”¹è®¿é—®çº§åˆ«çš„用户，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**全局æƒé™**å’Œ**内置角色**部分中，选中你希望用户拥有的æƒé™çš„å¤é€‰æ¡†ã€‚如果你在**角色**页é¢åˆ›å»ºäº†è§’色，这些角色将出现在**自定义**部分，你也å¯ä»¥é€‰æ‹©è¿™äº›è§’色。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户的全局æƒé™å·²æ›´æ–°ã€‚ + +### 为组é…置全局æƒé™ + +如果你有一组需è¦åœ¨ Rancher 中有相åŒè®¿é—®æƒé™çš„用户，你å¯ä»¥ä¸€æ¬¡æ€§å°†æƒé™åˆ†é…给整个组æ¥èŠ‚çœæ—¶é—´ã€‚这样，组中的用户在第一次登录 Rancher 时能拥有相应级别的访问æƒé™ã€‚ + +将自定义全局角色分é…给组åŽï¼Œè¯¥è§’色将在组中用户登录 Rancher 时分é…给用户。 + +对于现有用户，新æƒé™å°†åœ¨ç”¨æˆ·é€€å‡º Rancher 并é‡æ–°ç™»å½•æ—¶ï¼Œæˆ–当管ç†å‘˜[刷新用户组æˆå‘˜åå•](#刷新用户组æˆå‘˜åå•)时生效。 + +对于新用户，新æƒé™åœ¨ç”¨æˆ·é¦–次登录 Rancher 时生效。除了**新用户的默认角色**全局æƒé™å¤–，æ¥è‡ªè¯¥ç»„的新用户还将获得自定义全局角色的æƒé™ã€‚默认情况下，**新用户的默认角色**æƒé™ç­‰åŒäºŽ **Standard User** 全局角色，但默认æƒé™å¯ä»¥[é…置。](#é…置默认全局æƒé™) + +如果从外部身份验è¯æ供程åºç»„中删除用户，该用户将失去分é…给该组的自定义全局角色的æƒé™ã€‚他们将继续拥有分é…给他们的所有剩余角色，这通常包括标记为**新用户的默认角色**的角色。Rancher 将在用户登出或管ç†å‘˜[刷新用户组æˆå‘˜åå•](#刷新用户组æˆå‘˜åå•)时删除与组关è”çš„æƒé™ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +åªæœ‰åœ¨ä»¥ä¸‹æƒ…况下，你æ‰èƒ½å°†å…¨å±€è§’色分é…给组： + +* 你已设置[外部身份验è¯æ供程åº](../../../../pages-for-subheaders/authentication-config.md#外部验è¯ä¸Žæœ¬åœ°éªŒè¯)。 +* 外部身份验è¯æ供程åºæ”¯æŒ[用户组](../authentication-config/manage-users-and-groups.md)。 +* 你已使用身份验è¯æ供程åºè®¾ç½®äº†è‡³å°‘一个用户组。 + +::: + +è¦å°†è‡ªå®šä¹‰å…¨å±€è§’色分é…给组，请执行以下步骤： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**组**。 +1. 转到你è¦åˆ†é…自定义全局角色的组，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**全局æƒé™**，**自定义**å’Œ/或**内置角色**部分中，选择该组应具有的æƒé™ã€‚ +1. å•å‡»**创建**。 + +**结果**：自定义全局角色会在组内用户登录 Rancher 时生效。 + +### 刷新用户组æˆå‘˜åå• + +当管ç†å‘˜æ›´æ–°ç»„的全局æƒé™æ—¶ï¼Œæ›´æ”¹å°†åœ¨ç»„æˆå‘˜é€€å‡º Rancher 并é‡æ–°ç™»å½•åŽç”Ÿæ•ˆã€‚ + +如果è¦è®©æ›´æ”¹ç«‹å³ç”Ÿæ•ˆï¼Œç®¡ç†å‘˜æˆ–集群所有者å¯ä»¥åˆ·æ–°ç”¨æˆ·ç»„æˆå‘˜åå•ã€‚ + +如果用户已ç»ä»Žå¤–部身份验è¯æœåŠ¡ä¸­çš„组中删除，管ç†å‘˜ä¹Ÿå¯èƒ½éœ€è¦åˆ·æ–°ç”¨æˆ·ç»„æˆå‘˜åå•ã€‚在这ç§æƒ…况下，刷新æ“作会让 Rancher 知é“用户已从组中删除。 + +è¦åˆ·æ–°ç”¨æˆ·ç»„æˆå‘˜åå•ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. å•å‡»**刷新用户组æˆå‘˜åå•**。 + +**结果**：对组æˆå‘˜æƒé™çš„所有更改都会生效。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md new file mode 100644 index 00000000000..cb0ad71fa38 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md @@ -0,0 +1,38 @@ +--- +title: é”定角色 +--- + +ä½ å¯ä»¥å°†è§’色设置为`é”定`状æ€ã€‚é”定角色å¯é˜²æ­¢æŠŠè¿™äº›è§’色分é…给用户。 + +处于é”定状æ€çš„角色具有如下特性： + +- 无法å†åˆ†é…给当下还没有被分é…到该角色的用户。 +- 将用户添加到集群或项目时，ä¸ä¼šåœ¨**æˆå‘˜è§’色**下拉列表中列出。 +- ä¸ä¼šå½±å“在é”定该角色之å‰ï¼Œå·²ç»åˆ†é…了该角色的用户。å³ä½¿åŽæ¥é”定了该角色，这些用户ä»ç„¶ä¿ç•™è¯¥è§’色æ供的访问æƒé™ã€‚ + + **示例**：å‡è®¾ä½ çš„组织制定了一个内部策略，ç¦æ­¢æŠŠåˆ›å»ºé¡¹ç›®çš„æƒé™åˆ†é…给集群用户。这时候你需è¦æ‰§è¡Œè¿™ä¸ªç­–略。 + + 因此，在将新用户添加到集群之å‰ï¼Œä½ éœ€è¦é”定以下角色：`集群所有者`，`集群æˆå‘˜`å’Œ`创建项目`。然åŽï¼Œåˆ›å»ºä¸€ä¸ªæ–°çš„自定义角色，该角色的æƒé™ä¸Ž`集群æˆå‘˜`相åŒï¼Œä½†æ²¡æœ‰åˆ›å»ºé¡¹ç›®çš„æƒé™ã€‚然åŽï¼Œåœ¨å°†ç”¨æˆ·æ·»åŠ åˆ°é›†ç¾¤æ—¶ä½¿ç”¨è¿™ä¸ªæ–°çš„自定义角色。 + +以下用户å¯ä»¥é”定角色： + +- 任何分é…了`管ç†å‘˜`全局æƒé™çš„用户。 +- 任何分é…了带有`管ç†è§’色`æƒé™çš„`自定义用户`。 + + +## é”定/解é”角色 + +如果è¦é˜²æ­¢å°†è§’色分é…给用户，å¯ä»¥å°†å…¶è®¾ç½®ä¸º`é”定`状æ€ã€‚ + +ä½ å¯ä»¥åœ¨ä¸¤ç§æƒ…况下é”定角色： + +- [添加自定义角色](custom-roles.md)时。 +- 编辑现有角色时（è§ä¸‹æ–‡ï¼‰ã€‚ + +集群角色和项目/命å空间角色å¯ä»¥é”定，而全局角色ä¸èƒ½é”定。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 转到**集群**选项å¡æˆ–**项目或命å空间**选项å¡ã€‚ +1. 找到è¦é”定（或解é”）的角色，选择 **â‹® > 编辑é…ç½®**。 +1. 从**é”定**选项中，选择**是** 或**å¦**。然åŽç‚¹å‡»**ä¿å­˜**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md new file mode 100644 index 00000000000..1f56b62762f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md @@ -0,0 +1,126 @@ +--- +title: Pod 安全标准 (PSS) å’Œ Pod 安全准入 (PSA) +--- + +[Pod 安全标准 (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/) å’Œ [Pod 安全准入 (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) 为大é‡å·¥ä½œè´Ÿè½½å®šä¹‰äº†å®‰å…¨é™åˆ¶ã€‚ +它们在 Kubernetes v1.23 中å¯ç”¨å¹¶é»˜è®¤æ‰“开，并在 Kubernetes v1.25 åŠæ›´é«˜ç‰ˆæœ¬ä¸­æ›¿æ¢äº† [Pod Security Policies (PSP)](https://kubernetes.io/docs/concepts/security/pod-security-policy/)。 + +PSS 定义了工作负载的安全级别。PSA æ述了 Pod 安全上下文和相关字段的è¦æ±‚。PSA å‚考 PSS 级别æ¥å®šä¹‰å®‰å…¨é™åˆ¶ã€‚ + +## å‡çº§åˆ° Pod 安全标准 (PSS) + +ç¡®ä¿å°†æ‰€æœ‰ PSP 都è¿ç§»åˆ°äº†å¦ä¸€ä¸ªå·¥ä½œè´Ÿè½½å®‰å…¨æœºåˆ¶ï¼ŒåŒ…括将你当å‰çš„ PSP 映射到 Pod 安全标准，以便使用 [PSA 控制器](https://kubernetes.io/docs/concepts/security/pod-security-admission/)执行。如果 PSA 控制器ä¸èƒ½æ»¡è¶³ä¼ä¸šçš„所有需求，建议你使用策略引擎，例如 [OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper)ã€[Kubewarden](https://www.kubewarden.io/)ã€[Kyverno](https://kyverno.io/) 或 [NeuVector](https://neuvector.com/)。有关如何è¿ç§» PSP 的更多信æ¯ï¼Œè¯·å‚阅你选择的策略引擎的文档。 + +:::caution +必须在删除 PodSecurityPolicy 对象_之å‰_添加新的策略执行机制。å¦åˆ™ï¼Œä½ å¯èƒ½ä¼šä¸ºé›†ç¾¤å†…的特æƒå‡çº§æ”»å‡»åˆ›é€ æœºä¼šã€‚ +::: + +### 从 Rancher 维护的应用程åºå’Œå¸‚场工作负载中删除 PodSecurityPolicies {#remove-psp-rancher-workloads} + +Rancher v2.7.2 æ供了 Rancher 维护的 Helm Chart 的新主è¦ç‰ˆæœ¬ã€‚v102.x.y å…许你删除与以å‰çš„ Chart 版本一起安装的 PSP。这个新版本使用标准化的 `global.cattle.psp.enabled` 开关（默认关闭）替æ¢äº†éžæ ‡å‡†çš„ PSP 开关。 + +你必须在_ä»ä½¿ç”¨ Kubernetes v1.24_ 时执行以下步骤： +1. æ ¹æ®éœ€è¦é…ç½® PSA 控制器。你å¯ä»¥ä½¿ç”¨ Rancher 的内置 [PSA é…置模æ¿](#psa-config-templates)，或创建自定义模æ¿å¹¶å°†å…¶åº”用于正在è¿ç§»çš„集群。 + +1. 将活动的 PSP 映射到 Pod 安全标准： + 1. 查看集群中哪些 PSP ä»å¤„于活动状æ€ï¼š + :::caution + 此策略å¯èƒ½ä¼šé”™è¿‡å½“å‰æœªè¿è¡Œçš„工作负载，例如 CronJobsã€å½“å‰ç¼©æ”¾ä¸ºé›¶çš„工作负载或尚未推出的工作负载。 + ::: + + ```shell + kubectl get pods \ + --all-namespaces \ + --output jsonpath='{.items[*].metadata.annotations.kubernetes\.io\/psp}' \ + | tr " " "\n" | sort -u + ``` + + 1. 按照[å°† PSP 映射到 Pod 安全标准](https://kubernetes.io/docs/reference/access-authn-authz/psp-to-pod-security-standards/)çš„ Kubernetes 指å—å°† PSS 应用于ä¾èµ– PSP 的工作负载。有关详细信æ¯ï¼Œè¯·å‚阅[从 PodSecurityPolicy è¿ç§»åˆ°å†…ç½® PodSecurity Admission 控制器](https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/)。 + +1. è¦ä»Ž Rancher Chart 中删除 PSP，请在å‡çº§åˆ° Kubernetes v1.25 _之å‰_å°† Chart å‡çº§åˆ°æœ€æ–°çš„ v102.x.y ç‰ˆæœ¬ã€‚ç¡®ä¿ **Enable PodSecurityPolicies** 选项**å·²ç¦ç”¨**。这将删除与以å‰çš„ Chart 版本一起安装的所有 PSP。 + +:::info é‡è¦æ示 +如果你想将 Chart å‡çº§åˆ° v102.x.y，但ä¸æ‰“算将集群å‡çº§åˆ° Kubernetes v1.25 和弃用 PSP，请确ä¿ä¸ºæ¯ä¸ªè¦å‡çº§çš„ Chart 选择 **Enable PodSecurityPolicies** 选项。 +::: + +### 在 Kubernetes v1.25 å‡çº§åŽæ¸…ç†ç‰ˆæœ¬ + +如果你在删除 Chart çš„ PSP æ—¶é‡åˆ°é—®é¢˜ï¼Œæˆ–者 Chart ä¸åŒ…å«ç”¨äºŽåˆ é™¤ PSP 的内置机制，Chart å‡çº§æˆ–删除å¯èƒ½ä¼šå¤±è´¥å¹¶æ˜¾ç¤ºå¦‚下错误消æ¯ï¼š +```console +Error: UPGRADE FAILED: resource mapping not found for name: "" namespace: "" from "": no matches for kind "PodSecurityPolicy" in version "policy/v1beta1" +ensure CRDs are installed first +``` + +Helm å°è¯•åœ¨é›†ç¾¤ä¸­æŸ¥è¯¢å­˜å‚¨åœ¨å…ˆå‰ç‰ˆæœ¬çš„æ•°æ® blob 中的对象时，就会å‘生这ç§æƒ…况。è¦æ¸…ç†è¿™äº›ç‰ˆæœ¬å¹¶é¿å…此错误，请使用 `helm-mapkubeapis` Helm æ’件。è¦è¯¦ç»†äº†è§£ `helm-mapkubeapis`ã€å®ƒçš„工作原ç†ä»¥åŠå¦‚何针对你的用例进行微调，请å‚阅 [Helm 官方文档](https://github.com/helm/helm-mapkubeapis#readme)。 + +请注æ„，Helm æ’件安装在你è¿è¡Œå‘½ä»¤çš„机器本地。因此，请确ä¿ä»ŽåŒä¸€å°æœºå™¨è¿è¡Œå®‰è£…和清ç†ã€‚ + +#### 安装 `helm-mapkubeapis` + +1. 在打算使用 `helm-mapkubeapis` 的机器上打开你的终端并安装æ’件： + ```shell + helm plugin install https://github.com/helm/helm-mapkubeapis + ``` + + 你将看到类似于以下的输出： + ```console + Downloading and installing helm-mapkubeapis v0.4.1 ... + https://github.com/helm/helm-mapkubeapis/releases/download/v0.4.1/helm-mapkubeapis_0.4.1_darwin_amd64.tar.gz + Installed plugin: mapkubeapis + ``` + + :::info é‡è¦æ示 + ç¡®ä¿ `helm-mapkubeapis` æ’件至少为 v0.4.1，因为旧版本_ä¸_支æŒèµ„æºåˆ é™¤ã€‚ + ::: + +1. 验è¯æ’件是å¦å·²æ­£ç¡®å®‰è£…： + ```shell + helm mapkubeapis --help + ``` + + 你将看到类似于以下的输出： + ```console + Map release deprecated or removed Kubernetes APIs in-place + + Usage: + mapkubeapis [flags] RELEASE + + Flags: + --dry-run simulate a command + -h, --help help for mapkubeapis + --kube-context string name of the kubeconfig context to use + --kubeconfig string path to the kubeconfig file + --mapfile string path to the API mapping file + --namespace string namespace scope of the release + ``` + +#### 清ç†æŸå的版本 + +安装 `helm-mapkubeapis` æ’件åŽï¼Œæ¸…ç†å‡çº§åˆ° Kubernetes v1.25 åŽæŸå的版本。 + +1. 打开你的首选终端并通过è¿è¡Œ `kubectl cluster-info` ç¡®ä¿ç»ˆç«¯å·²è¿žæŽ¥åˆ°æ‰€éœ€é›†ç¾¤ã€‚ + +1. è¿è¡Œ `helm list --all-namespaces` 列出你在集群中安装的所有版本。 + +1. 通过è¿è¡Œ `helm mapkubeapis --dry-run --namespace ` 为è¦æ¸…ç†çš„æ¯ä¸ªç‰ˆæœ¬æ‰§è¡Œè¯•è¿è¡Œã€‚ä½ å¯ä»¥é€šè¿‡æ­¤å‘½ä»¤çš„结果了解è¦æ›¿æ¢æˆ–删除哪些资æºã€‚ + +1. 最åŽï¼Œåœ¨æŸ¥çœ‹æ›´æ”¹åŽï¼Œä½¿ç”¨ `helm mapkubeapis --namespace ` 执行完整è¿è¡Œã€‚ + +#### å°† Chart å‡çº§åˆ°æ”¯æŒ Kubernetes v1.25 的版本 + +清ç†äº†å…·æœ‰ PSP 的所有版本åŽï¼Œä½ å°±å¯ä»¥ç»§ç»­å‡çº§äº†ã€‚对于 Rancher 维护的工作负载，请按照本文档[从 Rancher 维护的应用程åºå’Œå¸‚场工作负载中删除 PodSecurityPolicies](#remove-psp-rancher-workloads) 部分中的步骤进行æ“作。 +如果工作负载ä¸æ˜¯ç”± Rancher 维护的，请å‚阅对应的æ供商的文档。 + +:::caution +ä¸è¦è·³è¿‡æ­¤æ­¥éª¤ã€‚与 Kubernetes v1.25 ä¸å…¼å®¹çš„应用程åºä¸èƒ½ä¿è¯åœ¨æ¸…ç†åŽæ­£å¸¸å·¥ä½œã€‚ +::: + +## Pod 安全准入é…ç½®æ¨¡æ¿ {#psa-config-templates} + +Rancher æ供了 PSA é…置模æ¿ã€‚它们是å¯ä»¥åº”用到集群的预定义安全é…置。Rancher 管ç†å‘˜ï¼ˆæˆ–具有æƒé™çš„人员）å¯ä»¥[创建ã€ç®¡ç†å’Œç¼–辑](./psa-config-templates.md) PSA 模æ¿ã€‚ + +### å— PSA é™åˆ¶çš„集群上的 Rancher + +Rancher system 命å空间也å—到 PSA 模æ¿æè¿°çš„é™åˆ¶æ€§å®‰å…¨ç­–略的影å“。你需è¦åœ¨åˆ†é…模æ¿åŽè±å… Rancher çš„ system 命å空间，å¦åˆ™é›†ç¾¤å°†æ— æ³•æ­£å¸¸è¿è¡Œã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [Pod 安全准入 (PSA) é…置模æ¿](./psa-config-templates.md#è±å…必须的-rancher-命å空间)。 + +有关è¿è¡Œ Rancher 所需的所有è±å…的完整文件，请å‚阅此[准入é…置示例](../../../reference-guides/rancher-security/psa-restricted-exemptions.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md new file mode 100644 index 00000000000..c3b4d6252f4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md @@ -0,0 +1,138 @@ +--- +title: Pod 安全准入 (PSA) é…ç½®æ¨¡æ¿ +--- + +[Pod Security admission (PSA)](./pod-security-standards.md) é…置模æ¿æ˜¯ Rancher è‡ªå®šä¹‰èµ„æº (CRD)，在 Rancher v2.7.2 åŠæ›´é«˜ç‰ˆæœ¬ä¸­å¯ç”¨ã€‚这些模æ¿æ供了å¯åº”用于集群的预定义安全é…置： + +- `rancher-privileged`：最宽æ¾çš„é…置。它ä¸é™åˆ¶ä»»ä½• Pod 行为，å…许已知的æƒé™å‡çº§ã€‚该策略没有è±å…。 +- `rancher-restricted`：严格é™åˆ¶çš„é…置，éµå¾ªå½“å‰åŠ å›º pod 的最佳实践。你必须对 Rancher 组件进行[命å空间级别è±å…](./pod-security-standards.md#å—-psa-é™åˆ¶çš„集群上的-rancher)。 + +## åˆ†é… Pod 安全准入 (PSA) é…ç½®æ¨¡æ¿ + +ä½ å¯ä»¥åœ¨åˆ›å»ºä¸‹æ¸¸é›†ç¾¤çš„åŒæ—¶åˆ†é… PSA 模æ¿ã€‚你还å¯ä»¥é€šè¿‡é…置现有集群æ¥æ·»åŠ æ¨¡æ¿ã€‚ + +### 在集群创建期间分é…æ¨¡æ¿ + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**按钮。 +1. 选择æ供商。 +1. 在**集群: 创建**页é¢ä¸Šï¼Œè½¬åˆ°**åŸºæœ¬ä¿¡æ¯ > 安全**。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**创建**。 + +### 将模æ¿åˆ†é…给现有集群 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**ä¿å­˜**。 + +### 加固集群 + +如果选择 **rancher-restricted** 模æ¿ä½†ä¸é€‰æ‹© **CIS é…置文件**，你将无法满足 CIS Benchmark。有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 加固指å—](../../../pages-for-subheaders/rke2-hardening-guide.md)。 + + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**按钮。 +1. 选择æ供商。 +1. 在**添加集群**页é¢ä¸Šçš„**集群选项**下，å•å‡» **高级选项**。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**创建**。 + +### 将模æ¿åˆ†é…给现有集群 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在**编辑集群**页é¢ä¸Šï¼Œè½¬åˆ°**集群选项 > 高级选项**。 +1. 在 **PSA é…置模æ¿**中，选择è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**ä¿å­˜**。 + + + + +## 添加或编辑 Pod 安全准入 (PSA) é…ç½®æ¨¡æ¿ + +如果你拥有管ç†å‘˜æƒé™ï¼Œåˆ™å¯ä»¥é€šè¿‡åˆ›å»ºå…¶ä»– PSA 模æ¿æˆ–编辑现有模æ¿æ¥è‡ªå®šä¹‰å®‰å…¨é™åˆ¶å’Œæƒé™ã€‚ + +:::caution +如果编辑使用中的现有 PSA 模æ¿ï¼Œæ›´æ”¹å°†åº”用​​于已分é…给该模æ¿çš„所有集群。 +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 点击**高级选项**打开下拉èœå•ã€‚ +1. 选择 **Pod 安全准入**。 +1. 找到è¦ä¿®æ”¹çš„模æ¿ï¼Œç‚¹å‡» **â‹®**。 +1. 选择**编辑é…ç½®**æ¥ç¼–辑模æ¿ã€‚ +1. 完æˆé…置编辑åŽï¼Œå•å‡»**ä¿å­˜**。 + +### å…许éžç®¡ç†å‘˜ç”¨æˆ·ç®¡ç† PSA æ¨¡æ¿ + +如果你想å…许其他用户管ç†æ¨¡æ¿ï¼Œä½ å¯ä»¥å°†è¯¥ç”¨æˆ·ç»‘定到一个角色，并为该角色授予 `management.cattle.io/podsecurityadmissionconfigurationtemplates` 上的所有æ“作 (`"*"`)。 + +:::caution +绑定到上述æƒé™çš„用户都能够更改使用该 PSA 模æ¿çš„_所有_托管集群的é™åˆ¶çº§åˆ«ï¼ŒåŒ…括用户没有æƒé™çš„集群。 +::: + +## è±å…必须的 Rancher 命å空间 + +在默认执行é™åˆ¶æ€§å®‰å…¨ç­–略的 Kubernetes 集群上è¿è¡Œ Rancher 时，你需è¦[è±å…以下命å空间](#è±å…命å空间)，å¦åˆ™è¯¥ç­–ç•¥å¯èƒ½ä¼šé˜»æ­¢ Rancher system pod 正常è¿è¡Œã€‚ + +- `calico-apiserver` +- `calico-system` +- `cattle-alerting` +- `cattle-csp-adapter-system` +- `cattle-epinio-system` +- `cattle-externalip-system` +- `cattle-fleet-local-system` +- `cattle-fleet-system` +- `cattle-gatekeeper-system` +- `cattle-global-data` +- `cattle-global-nt` +- `cattle-impersonation-system` +- `cattle-istio` +- `cattle-istio-system` +- `cattle-logging` +- `cattle-logging-system` +- `cattle-monitoring-system` +- `cattle-neuvector-system` +- `cattle-prometheus` +- `cattle-sriov-system` +- `cattle-system` +- `cattle-ui-plugin-system` +- `cattle-windows-gmsa-system` +- `cert-manager` +- `cis-operator-system` +- `fleet-default` +- `ingress-nginx` +- `istio-system` +- `kube-node-lease` +- `kube-public` +- `kube-system` +- `longhorn-system` +- `rancher-alerting-drivers` +- `security-scan` +- `tigera-operator` + +Rancherã€Rancher 拥有的一些 Chart ä»¥åŠ RKE2 å’Œ K3s å‘行版都使用这些命å空间。列出的命å空间的一个å­é›†å·²ç»åœ¨å†…置的 Rancher `rancher-restricted` 策略中被è±å…，用于下游集群。有关è¿è¡Œ Rancher 所需的所有è±å…的完整模æ¿ï¼Œè¯·å‚阅此[准入é…置示例](../../../reference-guides/rancher-security/psa-restricted-exemptions.md)。 + +## è±å…命å空间 + +如果你将 `rancher-restricted` 模æ¿åˆ†é…给集群，默认情况下，é™åˆ¶ä¼šåœ¨å‘½å空间级别应用于整个集群。è¦åœ¨æ­¤é«˜åº¦å—é™çš„策略下è±å…特定的命å空间，执行以下æ“作： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 点击**高级选项**打开下拉èœå•ã€‚ +1. 选择 **Pod 安全准入**。 +1. 找到è¦ä¿®æ”¹çš„模æ¿ï¼Œç‚¹å‡» **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 选中**è±å…**下的**命å空间**å¤é€‰æ¡†ä»¥ç¼–辑**命å空间**字段。 +1. è±å…命å空间åŽï¼Œå•å‡»**ä¿å­˜**。 + +:::note +你需è¦æ›´æ–°ç›®æ ‡é›†ç¾¤æ‰èƒ½è®©æ–°æ¨¡æ¿åœ¨é›†ç¾¤ä¸­ç”Ÿæ•ˆã€‚è¦è§¦å‘更新，在ä¸æ›´æ”¹å€¼çš„情况下编辑和ä¿å­˜é›†ç¾¤ã€‚ +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md new file mode 100644 index 00000000000..51b24396cbd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md @@ -0,0 +1,74 @@ +--- +title: 备份 Docker 安装的 Rancher +--- + +æˆåŠŸä½¿ç”¨ Docker 安装 Rancher åŽï¼Œæˆ‘们建议你定期创建备份。最近创建的备份能让你在æ„外ç¾éš¾å‘生åŽå¿«é€Ÿè¿›è¡Œæ¢å¤ã€‚ + +## åœ¨ä½ å¼€å§‹å‰ + +在创建备份的过程中，你将输入一系列命令。请使用环境中的数æ®æ›¿æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。以下是带有å ä½ç¬¦çš„命令示例： + +``` +docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher +``` + +在该命令中，`` 是数æ®å®¹å™¨å’Œå¤‡ä»½åˆ›å»ºæ—¥æœŸçš„å ä½ç¬¦ï¼ˆä¾‹å¦‚，`9-27-18`）。 + +请交å‰å‚考下方的图片和表格，了解获å–æ­¤å ä½ç¬¦æ•°æ®çš„方法。在开始[以下步骤](#创建备份)之å‰ï¼Œè¯·è®°ä¸‹æˆ–å¤åˆ¶è¿™äº›ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <RANCHER_CONTAINER_TAG> å’Œ <RANCHER_CONTAINER_NAME> + +![å ä½ç¬¦å‚考](/img/placeholder-ref.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | 首次安装拉å–çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.0.5` | 你为其创建备份的 Rancher 版本。 | +| `` | `9-27-18` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ ``。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## 创建备份 + +此步骤将创建一个备份文件。如果 Rancher é‡åˆ°ç¾éš¾æƒ…况，你å¯ä»¥ä½¿ç”¨è¯¥å¤‡ä»½æ–‡ä»¶è¿›è¡Œè¿˜åŽŸã€‚ + + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. åœæ­¢å½“å‰è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸ºä½ çš„ Rancher 容器的å称： + + ``` + docker stop + ``` +1. è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä»Žåˆšæ‰åœæ­¢çš„ Rancher 容器创建一个数æ®å®¹å™¨ã€‚请替æ¢å‘½ä»¤ä¸­çš„å ä½ç¬¦ï¼š + + ``` + docker create --volumes-from --name rancher-data- rancher/rancher: + ``` + +1. 从你刚刚创建的数æ®å®¹å™¨ï¼ˆrancher-data-<DATE>）中，创建一个备份 tar 包（rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz）。替æ¢å ä½ç¬¦æ¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ``` + docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup:z busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **步骤结果**：å±å¹•ä¸Šå°†è¿è¡Œå‘½ä»¤æµã€‚ + +1. 输入 `ls` 命令，确认备份压缩包已创建æˆåŠŸã€‚压缩包的å称格å¼ç±»ä¼¼ `rancher-data-backup--.tar.gz`。 + +1. 将备份压缩包移动到 Rancher Server 外的安全ä½ç½®ã€‚然åŽä»Ž Rancher Server 中删除 `rancher-data-` å’Œ `busybox-backup-` 容器。 + + ``` + docker rm rancher-data- + docker rm busybox-backup- + ``` + +1. é‡å¯ Rancher Server。将 `` 替æ¢ä¸º Rancher 容器的å称： + + ``` + docker start + ``` + +**结果**：创建了 Rancher Server æ•°æ®çš„备份压缩包。如果你需è¦æ¢å¤å¤‡ä»½æ•°æ®ï¼Œè¯·å‚è§[æ¢å¤å¤‡ä»½ï¼šDocker 安装](restore-docker-installed-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md new file mode 100644 index 00000000000..3ba17f6ed64 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md @@ -0,0 +1,307 @@ +--- +title: 备份集群 +--- + +在 Rancher UI 中，你å¯ä»¥è½»æ¾å¤‡ä»½å’Œæ¢å¤ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)çš„ etcd。 + +Rancher 建议为所有生产集群é…置定期 `etcd` 快照。此外，你还å¯ä»¥åˆ›å»ºå•æ¬¡å¿«ç…§ã€‚ + +etcd æ•°æ®åº“的快照会ä¿å­˜åœ¨ [etcd 节点](#本地备份目标)或 [S3 兼容目标](#s3-备份目标)上。é…ç½® S3 的好处是，如果所有 etcd 节点都丢失了，你的快照会ä¿å­˜åˆ°è¿œç«¯å¹¶èƒ½ç”¨äºŽæ¢å¤é›†ç¾¤ã€‚ + +## å¿«ç…§å·¥ä½œåŽŸç† + +### 快照组件 + + + + +Rancher 创建快照时，快照里包括三个组件： + +- etcd ä¸­çš„é›†ç¾¤æ•°æ® +- Kubernetes 版本 +- `cluster.yml` å½¢å¼çš„集群é…ç½® + +由于 Kubernetes 版本现在包å«åœ¨å¿«ç…§ä¸­ï¼Œå› æ­¤ä½ å¯ä»¥å°†é›†ç¾¤æ¢å¤åˆ°åŽŸæœ¬çš„ Kubernetes 版本。 + + + + +Rancher 将快照创建任务委托给下游 Kubernetes 引擎。Kubernetes 引擎创建快照时包括了三个组件： + +- etcd ä¸­çš„é›†ç¾¤æ•°æ® +- Kubernetes 版本 +- 集群é…ç½® + +由于 Kubernetes 版本包å«åœ¨å¿«ç…§ä¸­ï¼Œå› æ­¤ä½ å¯ä»¥å°†é›†ç¾¤è¿˜åŽŸåˆ°ä¹‹å‰çš„ Kubernetes 版本，åŒæ—¶è¿˜åŽŸ etcd 快照。 + + + + +如果你需è¦ä½¿ç”¨å¿«ç…§æ¢å¤é›†ç¾¤ï¼Œå¿«ç…§çš„多个组件å…许你选择： + +- **ä»…æ¢å¤ etcd 内容**：类似于在 Rancher v2.4.0 之å‰ç‰ˆæœ¬ä¸­çš„使用快照æ¢å¤ã€‚ +- **æ¢å¤ etcd å’Œ Kubernetes 版本**：如果 Kubernetes å‡çº§å¯¼è‡´é›†ç¾¤å¤±è´¥ï¼Œå¹¶ä¸”你没有更改任何集群é…置，则应使用此选项。 +- **æ¢å¤ etcdã€Kubernetes 版本和集群é…ç½®**：如果你在å‡çº§æ—¶åŒæ—¶æ›´æ”¹äº† Kubernetes 版本和集群é…置，则应使用此选项。 + +建议你在执行é…置更改或å‡çº§ä¹‹å‰åˆ›å»ºæ–°å¿«ç…§ã€‚ + + +### 从 etcd 节点生æˆå¿«ç…§ + + + + +集群中的æ¯ä¸ª etcd 节点都会检查 etcd 集群的å¥åº·çŠ¶å†µã€‚如果节点报告 etcd 集群是å¥åº·çš„，则会从中创建一个快照，并å¯é€‰æ‹©ä¸Šä¼ åˆ° S3。 + +快照存储在 `/opt/rke/etcd-snapshots` 中。如果该目录在节点上é…置为共享挂载，它将被覆盖。由于所有 etcd 节点都会上传快照并ä¿ç•™æœ€åŽä¸€ä¸ªï¼Œå› æ­¤ S3 上始终会ä¿ç•™æœ€åŽä¸€ä¸ªä¸Šä¼ çš„节点的快照。 + +在存在多个 etcd 节点的情况下，任何快照都是在集群å¥åº·æ£€æŸ¥é€šè¿‡åŽåˆ›å»ºçš„，因此这些快照å¯ä»¥è®¤ä¸ºæ˜¯ etcd 集群中数æ®çš„有效快照。 + + + + +快照是默认å¯åŠ¨çš„。 + +快照目录默认为 `/var/lib/rancher//server/db/snapshots`，其中 `` å¯ä»¥æ˜¯ `rke2` 或 `k3s`。 + +在 RKE2 中，快照会存储在æ¯ä¸ª etcd 节点上。如果你有多个 etcd 或 etcd + control plane 节点，你将拥有本地 etcd 快照的多个副本。 + + + + +### 快照命å规则 + + + + +快照的å称是自动生æˆçš„。在使用 RKE CLI 创建一次性快照时，你å¯ä»¥ä½¿ç”¨ `--name` 选项æ¥æŒ‡å®šå¿«ç…§çš„å称。 + +Rancher 在创建 RKE 集群的快照时，快照å称是基于快照创建类型（手动快照或定期快照）和目标（快照是ä¿å­˜åœ¨æœ¬åœ°è¿˜æ˜¯ä¸Šä¼ åˆ° S3）决定的。命å规则如下： + +- `m` 代表手动 +- `r` 代表定期 +- `l` 代表本地 +- `s` 代表 S3 + +å¿«ç…§å称示例如下： + +- c-9dmxz-rl-8b2cx +- c-9dmxz-ml-kr56m +- c-9dmxz-ms-t6bjb +- c-9dmxz-rs-8gxc8 + + + + +快照的å称是自动生æˆçš„。使用 RKE2 或 K3s CLI 创建一次性快照时，`--name` 选项å¯ç”¨äºŽè¦†ç›–快照的基本å称。 + +Rancher 在创建 RKE2 或 K3s 集群的快照时，快照å称是基于快照创建类型（手动快照或定期快照）和目标（快照是ä¿å­˜åœ¨æœ¬åœ°è¿˜æ˜¯ä¸Šä¼ åˆ° S3）决定的。命å规则如下： + +`--` + +``：`--name` 设置的基本å称，å¯ä»¥æ˜¯ä»¥ä¸‹ä¹‹ä¸€ï¼š + +- `etcd-snapshot`：ä½äºŽå®šæœŸå¿«ç…§å‰é¢ +- `on-demand`：ä½äºŽæ‰‹åŠ¨æŒ‰éœ€å¿«ç…§ä¹‹å‰ + +``：创建快照的节点的å称。 + +``：快照创建日期的 unix 时间戳。 + +å¿«ç…§å称示例如下： + +- `on-demand-my-super-rancher-k8s-node1-1652288934` +- `on-demand-my-super-rancher-k8s-node2-1652288936` +- `etcd-snapshot-my-super-rancher-k8s-node1-1652289945` +- `etcd-snapshot-my-super-rancher-k8s-node2-1652289948` + + + + +### 从快照æ¢å¤çš„å·¥ä½œåŽŸç† + + + + +在æ¢å¤æ—¶ä¼šå‘生以下过程： + +1. 如果é…置了 S3，则从 S3 检索快照。 +2. 如果快照压缩了，则将快照解压缩。 +3. 集群中的一个 etcd 节点会将该快照文件æ供给其他节点。 +4. 其他 etcd 节点会下载快照并验è¯æ ¡éªŒå’Œï¼Œä»¥ä¾¿éƒ½èƒ½ä½¿ç”¨ç›¸åŒçš„快照进行æ¢å¤ã€‚ +5. 集群已æ¢å¤ï¼Œæ¢å¤åŽçš„æ“作将在集群中完æˆã€‚ + + + + +在还原时，Rancher 会æ供几组执行还原的计划。期间将包括以下阶段： + +- Started +- Shutdown +- Restore +- RestartCluster +- Finished + +如果 etcd 快照还原失败，阶段将设置为 `Failed`。 + +1. 收到 etcd 快照还原请求åŽï¼Œæ ¹æ® `restoreRKEConfig` å调集群é…置和 Kubernetes 版本。 +1. 该阶段设置为 `Started`。 +1. 该阶段设置为 `Shutdown`，并使用è¿è¡Œ `killall.sh` 脚本的计划æ¥å…³é—­æ•´ä¸ªé›†ç¾¤ã€‚一个新的åˆå§‹èŠ‚点会被选举出æ¥ã€‚如果还原的快照是本地快照，则选择该快照所在的节点作为åˆå§‹èŠ‚点。如果使用 S3 还原快照，将使用现有的åˆå§‹èŠ‚点。 +1. 该阶段设置为 `Restore`，并且快照将还原到åˆå§‹èŠ‚点上。 +1. 该阶段设置为 `RestartCluster`，集群将é‡å¯å¹¶é‡æ–°åŠ å…¥åˆ°å…·æœ‰æ–°è¿˜åŽŸçš„快照信æ¯çš„æ–°åˆå§‹èŠ‚点。 +1. 该阶段设置为 `Finished`，集群被视为已æˆåŠŸè¿˜åŽŸã€‚`cattle-cluster-agent` å°†é‡æ–°è¿žæŽ¥ï¼Œé›†ç¾¤å°†å®Œæˆå调。 + + + + +## é…置定期快照 + + + + +选择创建定期快照的频率以åŠè¦ä¿ç•™çš„快照数é‡ã€‚时间的å•ä½æ˜¯å°æ—¶ã€‚用户å¯ä»¥ä½¿ç”¨æ—¶é—´æˆ³å¿«ç…§è¿›è¡Œæ—¶é—´ç‚¹æ¢å¤ã€‚ + +默认情况下，[Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)会é…置为创建定期快照（ä¿å­˜åˆ°æœ¬åœ°ç£ç›˜ï¼‰ã€‚为防止本地ç£ç›˜æ•…障，建议使用 [S3 目标](#s3-备份目标)或å¤åˆ¶ç£ç›˜ä¸Šçš„路径。 + +在集群é…置或编辑集群期间，å¯ä»¥åœ¨**集群选项**的高级部分中找到快照的é…置。点击**显示高级选项**。 + +在集群的**高级选项**中å¯ä»¥é…置以下选项： + +| 选项 | æè¿° | 默认值 | +| --- | ---| --- | +| etcd 快照备份目标 | 选择è¦ä¿å­˜å¿«ç…§çš„ä½ç½®ã€‚å¯ä»¥æ˜¯æœ¬åœ°æˆ– S3 | 本地 | +| å¯ç”¨å®šæœŸ etcd å¿«ç…§ | å¯ç”¨/ç¦ç”¨å®šæœŸå¿«ç…§ | 是 | +| 定期 etcd 快照的创建周期 | 定期快照之间的间隔（以å°æ—¶ä¸ºå•ä½ï¼‰ | 12 å°æ—¶ | +| 定期 etcd 快照的ä¿ç•™æ•°é‡ | è¦ä¿ç•™çš„å¿«ç…§æ•°é‡ | 6 | + + + + +设置创建定期快照的方å¼ä»¥åŠè¦ä¿ç•™çš„快照数é‡ã€‚该计划采用传统的 Cron æ ¼å¼ã€‚ä¿ç•™ç­–略规定了在æ¯ä¸ªèŠ‚点上è¦ä¿ç•™çš„匹é…å称的快照数é‡ã€‚ + +默认情况下，[Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)从凌晨 12 ç‚¹å¼€å§‹æ¯ 5 å°æ—¶åˆ›å»ºä¸€æ¬¡å®šæœŸå¿«ç…§ï¼ˆä¿å­˜åˆ°æœ¬åœ°ç£ç›˜ï¼‰ã€‚为了防止本地ç£ç›˜æ•…障，建议使用 [S3 目标](#s3-备份目标)或å¤åˆ¶ç£ç›˜ä¸Šçš„路径。 + +在集群é…置或编辑集群期间，你å¯ä»¥åœ¨**集群é…ç½®**下找到快照é…置。å•å‡» **etcd**。 + +| 选项 | æè¿° | 默认值 | +| --- | ---| --- | +| å¯ç”¨å®šæœŸ etcd å¿«ç…§ | å¯ç”¨/ç¦ç”¨å®šæœŸå¿«ç…§ | 是 | +| 定期 etcd 快照的创建周期 | 定期快照的 Cron 计划 | `0 */5 * * *` | +| 定期 etcd 快照的ä¿ç•™æ•°é‡ | è¦ä¿ç•™çš„å¿«ç…§æ•°é‡ | 5 | + + + + +## å•æ¬¡å¿«ç…§ + + + + +除了定期快照之外，你å¯èƒ½è¿˜æƒ³åˆ›å»ºâ€œä¸€æ¬¡æ€§â€å¿«ç…§ã€‚例如，在å‡çº§é›†ç¾¤çš„ Kubernetes 版本之å‰ï¼Œæœ€å¥½å¤‡ä»½é›†ç¾¤çš„状æ€ä»¥é˜²æ­¢å‡çº§å¤±è´¥ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå¯¼èˆªåˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºä¸€æ¬¡æ€§å¿«ç…§çš„集群。 +1. å•å‡» **â‹® > æ‹æ‘„å¿«ç…§**。 + + + + +除了定期快照之外，你å¯èƒ½è¿˜æƒ³åˆ›å»ºâ€œä¸€æ¬¡æ€§â€å¿«ç…§ã€‚例如，在å‡çº§é›†ç¾¤çš„ Kubernetes 版本之å‰ï¼Œæœ€å¥½å¤‡ä»½é›†ç¾¤çš„状æ€ä»¥é˜²æ­¢å‡çº§å¤±è´¥ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå¯¼èˆªåˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºä¸€æ¬¡æ€§å¿«ç…§çš„集群。 +1. 导航至`å¿«ç…§`选项å¡ï¼Œç„¶åŽå•å‡»`ç«‹å³åˆ›å»ºå¿«ç…§` + +### åˆ›å»ºä¸€æ¬¡æ€§å¿«ç…§çš„å·¥ä½œåŽŸç† + +在创建一次性快照时，Rancher 会传递几组计划æ¥æ‰§è¡Œå¿«ç…§åˆ›å»ºã€‚期间将包括以下阶段： + +- Started +- RestartCluster +- Finished + +如果 etcd 快照创建失败，阶段将设置为 `Failed`。 + +1. 收到 etcd 快照创建请求。 +1. 该阶段设置为 `Started`。集群中的所有 etcd 节点都会根æ®é›†ç¾¤é…置收到创建 etcd 快照的计划。 +1. 该阶段设置为 `RestartCluster`，并且æ¯ä¸ª etcd 节点上的计划都将é‡ç½®ä¸º etcd 节点的原始计划。 +1. 该阶段设置为 `Finished`。 + + + + +**结果**：根æ®ä½ çš„[快照备份目标](#快照备份目标)创建一次性快照，并将其ä¿å­˜åœ¨é€‰å®šçš„备份目标中。 + +## 快照备份目标 + +Rancher 支æŒä¸¤ç§ä¸åŒçš„备份目标： + +- [本地目标](#本地备份目标) +- [S3 目标](#s3-备份目标) + +### 本地备份目标 + + + + +默认情况下会选择 `local` 备份目标。此选项的好处是ä¸éœ€è¦è¿›è¡Œå¤–部é…置。快照会在本地自动ä¿å­˜åˆ° [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中 etcd 节点的 `/opt/rke/etcd-snapshots` 中。所有定期快照都是按照é…置的时间间隔创建的。使用 `local` 备份目标的缺点是，如果å‘生全é¢ç¾éš¾å¹¶ä¸”丢失 _所有_ etcd 节点时，则无法æ¢å¤é›†ç¾¤ã€‚ + + + + +默认情况下会选择 `local` 备份目标。此选项的好处是ä¸éœ€è¦è¿›è¡Œå¤–部é…置。快照会自动ä¿å­˜åˆ° [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中的本地 etcd 节点上的 `/var/lib/rancher//server/db/snapshots` 中，其中 `` å¯ä»¥æ˜¯ `k3s` 或 `rke2`。所有定期快照å‡æŒ‰ç…§ Cron 计划进行。使用 `local` 备份目标的缺点是，如果å‘生全é¢ç¾éš¾å¹¶ä¸”丢失 _所有_ etcd 节点时，则无法æ¢å¤é›†ç¾¤ã€‚ + + + + +### S3 备份目标 + +我们建议你使用 `S3` 备份目标。你å¯ä»¥å°†å¿«ç…§å­˜å‚¨åœ¨å¤–部 S3 兼容的åŽç«¯ä¸Šã€‚由于快照ä¸å­˜å‚¨åœ¨æœ¬åœ°ï¼Œå› æ­¤å³ä½¿ä¸¢å¤±æ‰€æœ‰ etcd 节点，你ä»ç„¶å¯ä»¥è¿˜åŽŸé›†ç¾¤ã€‚ + +虽然 `S3` 比本地备份具有优势，但它需è¦é¢å¤–çš„é…置。 + +:::caution + +如果你使用 S3 备份目标，请确ä¿æ¯ä¸ªé›†ç¾¤éƒ½æœ‰è‡ªå·±çš„存储桶或文件夹。Rancher 将使用集群é…置的 S3 存储桶或文件夹中的å¯ç”¨å¿«ç…§æ¥å¡«å……快照信æ¯ã€‚ + +::: + +| 选项 | æè¿° | å¿…å¡« | +|---|---|---| +| S3 存储桶å称 | 用于存储备份的 S3 存储桶å称 | * | +| S3 区域 | 备份存储桶的 S3 区域 | | +| S3 区域端点 | 备份存储桶的 S3 区域端点 | * | +| S3 访问密钥 | 有æƒè®¿é—®å¤‡ä»½å­˜å‚¨æ¡¶çš„ S3 访问密钥 | * | +| S3 密文密钥 | 有æƒè®¿é—®å¤‡ä»½å­˜å‚¨æ¡¶çš„ S3 密文密钥 | * | +| 自定义 CA è¯ä¹¦ | 用于访问ç§æœ‰ S3 åŽç«¯çš„自定义è¯ä¹¦ | + +### 为 S3 使用自定义 CA è¯ä¹¦ + +备份快照å¯ä»¥å­˜å‚¨åœ¨è‡ªå®šä¹‰ `S3` 备份中，例如 [minio](https://min.io/)。如果 S3 åŽç«¯ä½¿ç”¨è‡ªç­¾å或自定义è¯ä¹¦ï¼Œè¯·ä½¿ç”¨`自定义 CA è¯ä¹¦`选项æ¥æ供自定义è¯ä¹¦ï¼Œä»Žè€Œè¿žæŽ¥åˆ° S3 åŽç«¯ã€‚ + +### 在 S3 中存储快照的 IAM æ”¯æŒ + +除了使用 API 凭è¯ä¹‹å¤–，`S3` 备份目标还支æŒå¯¹ AWS API 使用 IAM 身份验è¯ã€‚IAM 角色会授予应用在对 S3 存储进行 API 调用时的临时æƒé™ã€‚è¦ä½¿ç”¨ IAM 身份验è¯ï¼Œå¿…须满足以下è¦æ±‚： + +- 集群 etcd 节点必须具有实例角色，该角色具有对指定备份存储桶的读/写访问æƒé™ã€‚ +- 集群 etcd 节点必须对指定的 S3 端点具有网络访问æƒé™ã€‚ +- Rancher Server worker 节点必须具有实例角色，该实例角色具有对指定备份存储桶的读/写访问æƒé™ã€‚ +- Rancher Server worker 节点必须对指定的 S3 端点具有网络访问æƒé™ã€‚ + +è¦æŽˆäºˆåº”用对 S3 的访问æƒé™ï¼Œè¯·å‚阅[使用 IAM 角色å‘在 Amazon EC2 实例上è¿è¡Œçš„应用授予æƒé™](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html)çš„ AWS 文档。 + +## 查看å¯ç”¨å¿«ç…§ + +Rancher UI 中æ供了集群所有å¯ç”¨å¿«ç…§çš„列表： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦æŸ¥çœ‹å¿«ç…§çš„集群并å•å‡»å…¶å称。 +1. å•å‡»**å¿«ç…§**选项å¡æ¥æŸ¥çœ‹å·²ä¿å­˜å¿«ç…§çš„列表。这些快照包括创建时间的时间戳。 + +## 安全时间戳（RKE） + +快照文件带有时间戳，从而简化使用外部工具和脚本处ç†æ–‡ä»¶çš„过程。但在æŸäº›ä¸Ž S3 兼容的åŽç«¯ä¸­ï¼Œè¿™äº›æ—¶é—´æˆ³æ— æ³•ä½¿ç”¨ã€‚ + +添加了选项 `safe_timestamp` 以支æŒå…¼å®¹çš„文件å。当此标志设置为 `true` 时，快照文件å时间戳中的所有特殊字符都将被替æ¢ã€‚ + +此选项ä¸èƒ½ç›´æŽ¥åœ¨ UI 中使用，åªèƒ½é€šè¿‡`以 YAML 文件编辑`使用。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md new file mode 100644 index 00000000000..cf023f56e0a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md @@ -0,0 +1,92 @@ +--- +title: 备份 Rancher +--- + +在本节中，你将学习如何备份è¿è¡Œåœ¨ä»»ä½• Kubernetes 集群上的 Rancher。è¦å¤‡ä»½é€šè¿‡ Docker 安装的 Rancher，请å‚è§[å•èŠ‚点备份](back-up-docker-installed-rancher.md)。 + +`backup-restore` operator 需è¦å®‰è£…在 local 集群上，并且åªå¯¹ Rancher 应用进行备份。备份和æ¢å¤æ“作仅在本地 Kubernetes 集群中执行。 + +请知悉，`rancher-backup` operator çš„ 2.x.x 版本用于 Rancher v2.6.x。 + +:::caution + +当把备份æ¢å¤åˆ°ä¸€ä¸ªæ–°çš„ Rancher 设置中时，新设置的版本应该与备份的版本相åŒã€‚在æ¢å¤å¤‡ä»½æ—¶è¿˜åº”考虑 Kubernetes 的版本，因为集群中支æŒçš„ apiVersion 和备份文件中的 apiVersion å¯èƒ½ä¸åŒã€‚ + +::: + +### 先决æ¡ä»¶ + +Rancher 必须是 2.5.0 或更高版本。 + +请å‚è§[此处](migrate-rancher-to-new-cluster.md#2-使用-restore-自定义资æºæ¥è¿˜åŽŸå¤‡ä»½)获å–在 Rancher 2.6.3 中将现有备份文件æ¢å¤åˆ° v1.22 集群的帮助。 + +### 1. 安装 Rancher Backup Operator + +备份存储ä½ç½®æ˜¯ operator 级别的设置，所以需è¦åœ¨å®‰è£…或å‡çº§ `rancher backup` 应用时进行é…置。 + +备份文件的格å¼æ˜¯ `.tar.gz`。这些文件å¯ä»¥æŽ¨é€åˆ° S3 或 Minio，也å¯ä»¥å­˜å‚¨åœ¨ä¸€ä¸ªæŒä¹…å·ä¸­ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。Rancher Server è¿è¡Œåœ¨ `local` 集群中。 +1. å•å‡» **Apps > Charts**。 +1. 点击 **Rancher 备份**。 +1. å•å‡»**安装**。 +1. é…置默认存储ä½ç½®ã€‚如需获å–帮助，请å‚è§[存储é…ç½®](../../../reference-guides/backup-restore-configuration/storage-configuration.md)。 +1. å•å‡»**安装**。 + +:::note + +使用 `backup-restore` operator 执行æ¢å¤åŽï¼ŒFleet 中会出现一个已知问题：用于 `clientSecretName` å’Œ `helmSecretName` 的密文ä¸åŒ…å«åœ¨ Fleet çš„ Git 仓库中。请å‚è§[此处](../deploy-apps-across-clusters/fleet.md#故障排除)获得解决方法。 + +::: + +### 2. 执行备份 + +è¦æ‰§è¡Œå¤‡ä»½ï¼Œå¿…须创建 Backup 类型的自定义资æºã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Rancher 备份 > 备份**。 +1. å•å‡»**创建**。 +1. 使用表å•æˆ– YAML 编辑器创建 Backup。 +1. è¦ä½¿ç”¨è¯¥è¡¨å•é…ç½® Backup 详细信æ¯ï¼Œè¯·å•å‡»**创建**，然åŽå‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/backup-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md#备份)进行æ“作。 +1. è¦ä½¿ç”¨ YAML 编辑器，å•å‡»**创建 > 使用 YAML 文件创建**。输入 Backup YAML。这个示例 Backup 自定义资æºå°†åœ¨ S3 中创建加密的定期备份。这个应用使用 `credentialSecretNamespace` 值æ¥ç¡®å®šåœ¨å“ªé‡Œå¯»æ‰¾ S3 备份的密文： + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Backup + metadata: + name: s3-recurring-backup + spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 + ``` + + :::note + + 使用 YAML 编辑器创建 Backup 资æºæ—¶ï¼Œ`resourceSetName` 必须设置为 `rancher-resource-set`。 + + ::: + + 如需获得é…ç½® Backup 的帮助，请å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/backup-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md#备份)。 + + :::caution + + `rancher-backup` operator ä¸ä¿å­˜ `EncryptionConfiguration` 文件。创建加密备份时，必须ä¿å­˜ `EncryptionConfiguration` 文件的内容，而且在使用备份还原时必须使用åŒä¸€ä¸ªæ–‡ä»¶ã€‚ + + ::: + +1. å•å‡»**创建**。 + +**结果**：备份文件创建在 Backup 自定义资æºä¸­é…置的存储ä½ç½®ä¸­ã€‚执行还原时使用该文件的å称。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md new file mode 100644 index 00000000000..03b4a704185 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md @@ -0,0 +1,129 @@ +--- +title: 备份æ¢å¤ä½¿ç”¨æŒ‡å— +--- + +Rancher Backups Chart 是我们的ç¾éš¾æ¢å¤å’Œè¿ç§»è§£å†³æ–¹æ¡ˆã€‚æ­¤ Chart 用于备份 Kubernetes 资æºå¹¶å°†å…¶ä¿å­˜åˆ°å„ç§æŒä¹…存储ä½ç½®ã€‚ + +这个 Chart 是一个éžå¸¸ç®€å•çš„工具，适用于 Rancher 生æ€ç³»ç»Ÿçš„许多ä¸åŒé¢†åŸŸã€‚但是，也因此出现了未记录功能的边缘用例。本文档旨在强调 Rancher Backups 正确的用法，并讨论我们é‡åˆ°çš„一些边缘情况。 + +## 功能概述 + +### 备份 + +该 Operator å°† Chart 中的 resourceSet æ•èŽ·çš„所有资æºæ”¶é›†ä¸ºå†…存中的éžç»“构化对象。收集资æºåŽï¼Œèµ„æºçš„ tar 包将ä¿å­˜ä¸º JSON æ ¼å¼çš„清å•é›†åˆï¼Œç„¶åŽä¸Šä¼ åˆ°ç”¨æˆ·å®šä¹‰çš„对象存储。该备份å¯ä»¥æŒ‰é‡å¤è®¡åˆ’进行，也å¯ä»¥è¿›è¡ŒåŠ å¯†ã€‚由于æŸäº›èµ„æºæ˜¯æ•æ„Ÿçš„，并且值以未加密的明文形å¼å­˜å‚¨ï¼Œå› æ­¤æ­¤åŠ å¯†é€‰é¡¹å¾ˆé‡è¦ã€‚ + +有关é…置备份的选项（包括加密），请å‚阅[备份é…置文档](../../../reference-guides/backup-restore-configuration/backup-configuration.md)。 + +:::note + +如[备份 Rancher 文档](./back-up-rancher.md)所述，你必须手动ä¿å­˜åŠ å¯†é…置文件的内容，因为 Operator **ä¸ä¼š**备份它。 + +::: + +### 还原 + +有两ç§ä¸»è¦çš„还原场景：还原正在è¿è¡Œ Rancher 的集群以åŠè¿˜åŽŸæ–°é›†ç¾¤ã€‚åªæœ‰å°†å¤‡ä»½è¿˜åŽŸåˆ°è¯¥å¤‡ä»½çš„æºé›†ç¾¤ï¼Œä¸”在还原过程中å¯ç”¨äº† [`prune` 选项](../../../reference-guides/backup-restore-configuration/restore-configuration.md#还原过程中修剪)时，你æ‰èƒ½è¿˜åŽŸæ­£åœ¨è¿è¡Œ Rancher 的集群。还原具有与备份类似的输入。它需è¦å¤‡ä»½æ–‡ä»¶åã€encryptionConfigSecret å称和存储ä½ç½®ã€‚ + +资æºæŒ‰ä»¥ä¸‹é¡ºåºè¿˜åŽŸï¼š + +1. 自定义资æºå®šä¹‰ï¼ˆCRD） +2. é›†ç¾¤èŒƒå›´èµ„æº +3. 命åç©ºé—´èµ„æº + +有关é…置还原的选项，请å‚阅[还原é…置文档](../../../reference-guides/backup-restore-configuration/restore-configuration.md)。 + +### 资æºé›† + +ResourceSet 确定了 backup-restore-operator 在备份中收集哪些资æºã€‚它是一组 ResourceSelector，使用键/值对匹é…ã€æ­£åˆ™è¡¨è¾¾å¼åŒ¹é…或 Kubernetes 客户端 labelSelector æ¥å®šä¹‰é€‰æ‹©è¦æ±‚。 + +以下是å¯ç”¨äºŽ resourceSelector 的字段： + +- apiVersion +- excludeKinds +- excludeResourceNameRegexp +- kinds +- kindsRegexp +- labelSelectors +- namespaceRegexp +- namespaces +- resourceNameRegexp +- resourceNames + +Rancher Backups Chart 包å«äº†ä¸€ä¸ª[默认 resourceSet](https://github.com/rancher/backup-restore-operator/tree/release/v3.0/charts/rancher-backup/files/default-resourceset-contents)，它是安装 Chart 时附加到一个大型 resourceSet çš„ YAML 文件组åˆã€‚文件顺åºå¹¶ä¸é‡è¦ã€‚ä¸åŒç‰ˆæœ¬çš„ resourceSet å¯èƒ½æœ‰æ‰€ä¸åŒã€‚ + +:::caution + +如果你希望编辑 resourceSet，请在安装 Chart 之å‰è¿›è¡Œ**编辑**。 + +::: + +## 正确使用 + +本节概述了如何根æ®ç”¨ä¾‹æ­£ç¡®ä½¿ç”¨ Rancher Backups Chart。 + +### 所有案例 + +- Rancher Backups 必须安装在 local 集群上。 + - 注æ„：Rancher Backups åªä¼šå¤„ç†å®‰è£…了它的集群。它å¯èƒ½ä¼šè¿˜åŽŸ local 集群上的集群资æºï¼Œä½†ä¸ä¼šè”系或备份下游集群。 +- è¦è¿˜åŽŸçš„ Rancher 版本必须与备份中的 Rancher 版本匹é…。 +- 由于你å¯èƒ½éœ€è¦è¿˜åŽŸå·²è¿‡æ—¶çš„资æºï¼ˆè¦è¿˜åŽŸçš„ Kubernetes 版本已弃用的资æºï¼‰ï¼Œå› æ­¤ä½ éœ€è¦è€ƒè™‘ Kubernetes 版本。 + +### 备份 + +- 用户生æˆçš„æŸäº›èµ„æºä¸ä¼šè¢«å¤‡ä»½ï¼Œé™¤éžè¿™äº›èµ„æºå¯ä»¥è¢«é»˜è®¤ resourceSet æ•èŽ·ï¼Œæˆ–者 resourceSet 已被更改为æ•èŽ·è¿™äº›èµ„æºã€‚ + - 我们æ供了一个 `resources.cattle.io/backup:true` 标签，将该标签添加到任何命å空间中的 Secret 时，命å空间将被备份。 +- 备份是ä¸å¯æ”¹å˜çš„ +- 仅备份 local 集群 + +### 还原 + +- 还原是指将备份还原到原æ¥çš„集群。å¯ä»¥åœ¨å®‰è£…了 Rancher 的情况下进行（**å¿…é¡»å¯ç”¨ prune**），也å¯ä»¥åœ¨æœªå®‰è£… Rancher 的情况下进行（无特殊说明）。 +- 还原时需è¦æ³¨æ„的一件事是，你å¯èƒ½éœ€è¦â€œæ“¦é™¤â€é›†ç¾¤ä¸­çš„所有 Rancher 资æºã€‚ä½ å¯ä»¥é€šè¿‡å°† [Rancher cleanup script](https://github.com/rancher/rancher-cleanup) 脚本作为 job 部署到集群æ¥å®Œæˆè¿™ç‚¹ã€‚这样，你å¯ä»¥å†æ¬¡å®‰è£… Rancher Backups 并还原到全新的集群。 + - ç¡®ä¿ä½¿ç”¨äº† kubectl æ¥éƒ¨ç½²è„šæœ¬ã€‚ + +### è¿ç§» + +由于我们è¦è¿˜åŽŸåˆ°ä¸åŒçš„集群，因此对应的è¿ç§»æœ‰ä¸€äº›ç»†å¾®å·®åˆ«ã€‚以下是需è¦è®°ä½ä½†åˆå®¹æ˜“被忘记的事情。 + +- è¿ç§»æ—¶ Rancher 域必须相åŒã€‚æ¢è¨€ä¹‹ï¼Œä½ æ—§é›†ç¾¤çš„域å现在必须指å‘新集群。 +- Rancher **ä¸åº”该**å·²è¿è¡Œåœ¨ä½ è¦è¿ç§»åˆ°çš„集群，这å¯èƒ½ä¼šå¯¼è‡´ Rancher 备份和æŸäº› Rancher æœåŠ¡å‡ºçŽ°è®¸å¤šé—®é¢˜ã€‚ +- **还原备份åŽ**，安装与备份**相åŒ**çš„ Rancher 版本。 +- 在其他 Kubernetes 版本上é…置新集群å¯èƒ½ä¼šå‡ºçŽ°å„ç§ä¸å—支æŒçš„情况，这是因为å¯ç”¨çš„ Kubernetes API å¯èƒ½ä¸Žä½ å¤‡ä»½çš„ API ä¸åŒã€‚è¿™å¯èƒ½ä¼šå¯¼è‡´å·²å¼ƒç”¨çš„资æºè¢«æ¢å¤ï¼Œä»Žè€Œå¯¼è‡´é—®é¢˜ã€‚ +- 在è¿ç§»æœŸé—´**ä¸è¦**执行任何å‡çº§æ“作。 + +## 边缘案例和ä¸å½“使用 + +以下是 Rancher Backups 的一些**ä¸å½“**使用示例。 + +### å‡çº§ + +- 使用 Rancher Backups æ¥å‡çº§ Rancher 版本ä¸æ˜¯ä¸€ä¸ªæœ‰æ•ˆç”¨æ³•ã€‚推èçš„åšæ³•æ˜¯ï¼šå…ˆå¤‡ä»½å½“å‰ç‰ˆæœ¬ï¼Œç„¶åŽæŒ‰ç…§[说明](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md)å‡çº§ä½ çš„ Rancher 实例，在å‡çº§å®ŒæˆåŽå†è¿›è¡Œ**å¦ä¸€ä¸ª**备份。这样，如果å‡çº§å¤±è´¥ï¼Œä½ å°±æœ‰ä¸€ä¸ªå¯ä»¥ç”¨æ¥è¿˜åŽŸçš„备份，而第二个备份将能用于还原到å‡çº§åŽçš„ Rancher 版本。 +- 使用 Rancher Backups æ¥å‡çº§ Kubernetes 版本也ä¸æ˜¯ä¸€ä¸ªæœ‰æ•ˆç”¨æ³•ã€‚由于 Kubernetes API 以åŠå¯ç”¨èµ„æºä¸Žç‰ˆæœ¬ç›¸å…³ï¼Œå› æ­¤ä½¿ç”¨å¤‡ä»½è¿˜åŽŸçš„方法æ¥è¿›è¡Œå‡çº§å¯èƒ½ä¼šå¯¼è‡´èµ„æºé›†ä¸å¯¹é½çš„问题，这些资æºå¯èƒ½å·²è¢«å¼ƒç”¨ã€ä¸å—支æŒæˆ–已更新。å‡çº§é›†ç¾¤ç‰ˆæœ¬çš„æ–¹å¼å–决于其é…置方å¼ï¼Œä½†å»ºè®®ä½¿ç”¨ä¸Šè¿°çš„æµç¨‹ï¼ˆå¤‡ä»½ã€å‡çº§ã€å¤‡ä»½ï¼‰ã€‚ + +### ResourceSet + +- 由于ä¸åŒå›¢é˜Ÿçš„资æºå’ŒæœåŠ¡ä¼šä¸æ–­å‘展，开å‘人员应è¦æ³¨æ„是å¦éœ€è¦å‘默认 resourceSet 添加或删除新资æºã€‚ +- Rancher Backups 仅备份默认 resourceSet æ•èŽ·çš„内容（除éžè¿›è¡Œç¼–辑）。我们为用户创建的 Secret 添加了特定标签，无论 Secret çš„å称是什么，无论它属于哪个命å空间，具有该标签 Secret 都会被备份（请å‚阅[备份的正确用法](#备份)）。 + +### 下游集群 + +- Rancher Backups **ä»…**备份 local 集群上的 Kubernetes 资æºã€‚æ¢è¨€ä¹‹ï¼Œé™¤äº†å­˜åœ¨äºŽ local 集群中的资æºï¼Œä¸‹æ¸¸é›†ç¾¤**ä¸ä¼š**被触åŠæˆ–备份。下游集群的更新和通信由 rancher-agent å’Œ rancher-webhook 负责。 + +### è¿˜åŽŸå·²åˆ é™¤çš„èµ„æº + +- 有些资æºä¼šäº§ç”Ÿå¤–部结果，例如会é…置下游集群。删除下游集群并还原 local 集群上的集群资æº**ä¸ä¼š**导致 Rancher é‡æ–°é…置所述集群。æŸäº›èµ„æºå¯èƒ½æ— æ³•é€šè¿‡è¿˜åŽŸå›žåˆ°å¯ç”¨çŠ¶æ€ã€‚ +- “还原已删除的集群â€**ä¸æ˜¯**å—支æŒçš„功能。涉åŠä¸‹æ¸¸é›†ç¾¤æ—¶ï¼Œæ— è®ºé›†ç¾¤æ˜¯é…置的还是导入的，删除集群都会执行一系列清ç†ä»»åŠ¡ï¼Œå¯¼è‡´æˆ‘们无法还原已删除的集群。é…置的集群节点以åŠä¸Ž Rancher 相关的é…置资æºå°†è¢«é”€æ¯ï¼Œè€Œå¯¼å…¥çš„集群的 Rancher Agent 以åŠä¸Ž local 集群注册相关的其他资æº/æœåŠ¡å¯èƒ½ä¼šè¢«é”€æ¯ã€‚ + +:::caution + +å°è¯•åˆ é™¤å’Œè¿˜åŽŸä¸‹æ¸¸é›†ç¾¤å¯èƒ½ä¼šå¯¼è‡´ Rancherã€Rancher Backupsã€rancher-webhookã€Fleet 等出现å„ç§é—®é¢˜ã€‚因此，我们ä¸å»ºè®®ä½ è¿™æ ·åšã€‚ + +::: + +### Fleetã€Harvester 和其他æœåŠ¡ + +ç”± Rancher Backups 支æŒçš„其他æœåŠ¡ä¼šç»å¸¸å‘生å˜åŒ–å’Œå‘展。å‘生这ç§æƒ…况时，他们的资æºå’Œå¤‡ä»½éœ€æ±‚也å¯èƒ½ä¼šå‘生å˜åŒ–。有些资æºå¯èƒ½æ ¹æœ¬ä¸éœ€è¦å¤‡ä»½ã€‚团队需è¦åœ¨å¼€å‘过程中考虑这一点，并评估相关 resourceSet 是å¦èƒ½æ­£ç¡®æ•èŽ·æ­£ç¡®çš„资æºé›†æ¥è¿˜åŽŸå…¶æœåŠ¡ã€‚ + +## 结论 + +Rancher Backups 是一个éžå¸¸æœ‰ç”¨çš„工具，但它的使用范围和使用目的有é™çš„。为了é¿å…出现问题，请éµå¾ªæœ¬æ–‡æ‰€è¿°çš„æµç¨‹æ¥ç¡®ä¿ Chart 能正确è¿ä½œã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md new file mode 100644 index 00000000000..5aa5ac6e523 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md @@ -0,0 +1,184 @@ +--- +title: å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ +--- + +如果你è¦å°† Rancher è¿ç§»åˆ°ä¸€ä¸ªæ–°çš„ Kubernetes 集群，先ä¸è¦åœ¨æ–°é›†ç¾¤ä¸Šå®‰è£… Rancher。这是因为如果将 Rancher 还原到已安装 Rancher 的新集群，å¯èƒ½ä¼šå¯¼è‡´é—®é¢˜ã€‚ + +### 先决æ¡ä»¶ + +以下说明å‡è®¾ä½ å·²ç»å®Œæˆ[备份创建](back-up-rancher.md)，并且已ç»å®‰è£…了用于部署 Rancher çš„æ–° Kubernetes 集群。 + +:::caution + +你需è¦ä½¿ç”¨ä¸Žç¬¬ä¸€ä¸ªé›†ç¾¤ä¸­è®¾ç½®çš„ Server URL 相åŒçš„主机å。å¦åˆ™ï¼Œä¸‹æ¸¸é›†ç¾¤ä¼šåœ¨ UI 的管ç†é¡µé¢æ˜¾ç¤ºä¸ºä¸å¯ç”¨ï¼Œå¹¶ä¸”ä½ ä¸èƒ½ç‚¹å‡»é›†ç¾¤å†…或集群的 **Explore** 按钮。 + +::: + +Rancher 必须是 2.5.0 或更高版本。 + +Rancher å¯ä»¥å®‰è£…åˆ°ä»»æ„ Kubernetes 集群上，包括托管的 Kubernetes 集群（如 Amazon EKS 集群）。如需获å–安装 Kubernetes 的帮助，请å‚è§ Kubernetes å‘行版的文档。你也å¯ä»¥ä½¿ç”¨ä»¥ä¸‹ Rancher çš„ Kubernetes å‘行版： + +- [RKE Kubernetes 安装文档](https://rancher.com/docs/rke/latest/en/installation/) +- [K3s Kubernetes 安装文档](https://rancher.com/docs/k3s/latest/en/installation/) + +### 1. 安装 rancher-backup Helm Chart +安装 [rancher-backup chart](https://github.com/rancher/backup-restore-operator/tags)，请使用 2.x.x 主è¦ç‰ˆæœ¬å†…的版本： + +1. 添加 helm 仓库： + + ```bash + helm repo add rancher-charts https://charts.rancher.io + helm repo update + ``` + +1. 使用 2.x.x rancher-backup 版本设置 `CHART_VERSION` å˜é‡ï¼š + ```bash + helm search repo --versions rancher-charts/rancher-backup + CHART_VERSION=<2.x.x> + ``` + +1. 安装 Chart： + ```bash + helm install rancher-backup-crd rancher-charts/rancher-backup-crd -n cattle-resources-system --create-namespace --version $CHART_VERSION + helm install rancher-backup rancher-charts/rancher-backup -n cattle-resources-system --version $CHART_VERSION + ``` + + :::note + + 以上å‡è®¾ä½ çš„环境具有到 Docker Hub 的出站连接。 + + 对于**离线环境**，在安装 rancher-backup Helm Chart 时，使用下é¢çš„ Helm 值从你的ç§æœ‰é•œåƒä»“åº“ä¸­æ‹‰å– `backup-restore-operator` é•œåƒã€‚ + + ```bash + --set image.repository $REGISTRY/rancher/backup-restore-operator + ``` + + ::: + +### 2. 使用 Restore 自定义资æºæ¥è¿˜åŽŸå¤‡ä»½ + +:::note é‡è¦æ示： + +Kubernetes v1.22 是 Rancher 2.6.3 的实验功能，ä¸æ”¯æŒä½¿ç”¨ apiVersion `apiextensions.k8s.io/v1beta1`æ¥è¿˜åŽŸåŒ…å« CRD 文件的备份文件。在 v1.22 中，`rancher-backup` 应用的默认 `resourceSet` åªæ”¶é›†ä½¿ç”¨ `apiextensions.k8s.io/v1` çš„ CRD。你å¯ä»¥é€šè¿‡ä¸‹é¢ä¸¤ç§æ–¹æ³•è§£å†³è¿™ä¸ªé—®é¢˜ã€‚ + +1. 使用 apiVersion v1 æ¥æ›´æ–°é»˜è®¤ `resourceSet`，从而收集 CRD。 +1. 使用 `apiextensions.k8s.io/v1` 作为替代，æ¥æ›´æ–°é»˜è®¤ `resourceSet` 和客户端，从而在内部使用新的 API。 + + :::note + + 在为 v1.22 版本制作或æ¢å¤å¤‡ä»½æ—¶ï¼ŒRancher 版本和本地集群的 Kubernetes 版本应该是一样的。由于集群中支æŒçš„ apiVersion 和备份文件中的 apiVersion å¯èƒ½ä¸åŒï¼Œå› æ­¤åœ¨è¿˜åŽŸå¤‡ä»½æ—¶è¯·è€ƒè™‘ Kubernetes 的版本。 + + ::: + +::: + +1. 在使用 S3 对象存储作为需è¦ä½¿ç”¨å‡­è¯çš„还原的备份æºæ—¶ï¼Œè¯·åœ¨æ­¤é›†ç¾¤ä¸­åˆ›å»ºä¸€ä¸ª `Secret` 对象以添加 S3 凭è¯ã€‚Secret æ•°æ®å¿…é¡»æœ‰ä¸¤ä¸ªå¯†é’¥ï¼Œåˆ†åˆ«æ˜¯åŒ…å« S3 凭è¯çš„ `accessKey` å’Œ `secretKey`。 + + ä½ å¯ä»¥åœ¨ä»»ä½•å‘½å空间中创建 Secret，本示例使用 default 命å空间。 + + ```bash + kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= + ``` + + :::note + + 在上é¢çš„命令中添加你的 Access Key å’Œ Secret Key 作为 `accessKey` å’Œ `secretKey` 的值。 + + ::: + +1. 创建一个 `Restore` 对象： + + 在è¿ç§»æœŸé—´ï¼Œ`prune` 必须设置为 `false`。请å‚è§ä¸‹é¢çš„示例： + + ```yaml + # restore-migration.yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + // highlight-next-line + prune: false + // highlight-next-line + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + :::note é‡è¦æ示： + + åªæœ‰åœ¨åˆ›å»ºå¤‡ä»½æ—¶å¯ç”¨äº†åŠ å¯†åŠŸèƒ½æ—¶ï¼Œæ‰éœ€è¦è®¾ç½® `encryptionConfigSecretName` 字段。 + + 如果适用，请æ供包å«åŠ å¯†é…置文件的 `Secret` 对象的å称。如果你åªæœ‰åŠ å¯†é…置文件，但没有在此集群中创建 Secret，请按照以下步骤创建 Secret。 + + 1. 创建[加密é…置文件](../../../reference-guides/backup-restore-configuration/backup-configuration.md#加密) + 1. 下é¢çš„命令使用一个å为 `encryption-provider-config.yaml` 的文件，使用了 `--from-file` 标志。将 `EncryptionConfiguration` ä¿å­˜åˆ°å为 `encryption-provider-config.yaml` 的文件中之åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ```bash + kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system + ``` + + ::: + +1. 应用清å•ï¼Œå¹¶ç›‘控 Restore 的状æ€ï¼š + 1. 应用 `Restore` 对象资æºï¼š + + ```bash + kubectl apply -f restore-migration.yaml + ``` + + 1. 观察 Restore 的状æ€ï¼š + ```bash + kubectl get restore + ``` + + 1. 查看æ¢å¤æ—¥å¿—： + ```bash + kubectl logs -n cattle-resources-system --tail 100 -f -l app.kubernetes.io/instance=rancher-backup + ``` + + 1. Restore 资æºçš„状æ€å˜æˆ `Completed` åŽï¼Œä½ å¯ä»¥ç»§ç»­å®‰è£… cert-manager å’Œ Rancher。 + +### 3. 安装 cert-manager + +按照在 Kubernetes 上安装 cert-manager的步骤[安装 cert-manager](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-安装-cert-manager)。 + +### 4. 使用 Helm 安装 Rancher + +使用与第一个集群上使用的相åŒç‰ˆæœ¬çš„ Helm æ¥å®‰è£… Rancher： + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +```bash +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname= \ + --version x.y.z +``` + +:::note + +如果原始的 Rancher 环境正在è¿è¡Œï¼Œä½ å¯ä»¥ä½¿ç”¨ kubeconfig 为原始环境收集当å‰å€¼ï¼š + +```bash +helm get values rancher -n cattle-system -o yaml > rancher-values.yaml +``` + +ä½ å¯ä»¥ä½¿ç”¨ `rancher-values.yaml` 文件æ¥å¤ç”¨è¿™äº›å€¼ã€‚ç¡®ä¿å°† kubeconfig 切æ¢åˆ°æ–°çš„ Rancher 环境。 + +```bash +helm install rancher rancher-latest/rancher -n cattle-system -f rancher-values.yaml --version x.y.z +``` + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md new file mode 100644 index 00000000000..b491321f2bf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md @@ -0,0 +1,72 @@ +--- +title: æ¢å¤å¤‡ä»½ - Docker 安装 +--- + +如果é‡åˆ°ç¾éš¾æƒ…况，你å¯ä»¥å°† Rancher Server æ¢å¤åˆ°æœ€æ–°çš„备份。 + +## åœ¨ä½ å¼€å§‹å‰ + +在æ¢å¤å¤‡ä»½çš„过程中，你将输入一系列命令。请使用环境中的数æ®æ›¿æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。以下是带有å ä½ç¬¦çš„命令示例： + +``` +docker run --volumes-from -v $PWD:/backup \ +busybox sh -c "rm /var/lib/rancher/* -rf && \ +tar pzxvf /backup/rancher-data-backup--" +``` + +在此命令中，`` å’Œ `-` 是用于 Rancher 部署的环境å˜é‡ã€‚ + +请交å‰å‚考下方的图片和表格，了解获å–æ­¤å ä½ç¬¦æ•°æ®çš„方法。在开始以下步骤之å‰ï¼Œè¯·å…ˆè®°ä¸‹æˆ–å¤åˆ¶æ­¤ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <RANCHER_CONTAINER_TAG> å’Œ <RANCHER_CONTAINER_NAME> + +![å ä½ç¬¦å‚考](/img/placeholder-ref.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | 首次安装拉å–çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.0.5` | Rancher 备份的版本å·ã€‚ | +| `` | `9-27-18` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ ``。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## æ¢å¤å¤‡ä»½ + +使用你之å‰åˆ›å»ºçš„[备份](back-up-docker-installed-rancher.md)，将 Rancher æ¢å¤åˆ°æœ€åŽå·²çŸ¥çš„å¥åº·çŠ¶æ€ã€‚ + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. åœæ­¢å½“å‰è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸º Rancher 容器的å称： + + ``` + docker stop + ``` +1. 将你在[创建备份 - Docker 安装](back-up-docker-installed-rancher.md)时创建的备份压缩包移动到 Rancher Server。切æ¢åˆ°ä½ å°†å…¶ç§»åŠ¨åˆ°çš„目录。输入 `dir` 以确认它在该ä½ç½®ã€‚ + + 如果你éµå¾ªäº†æˆ‘们在[创建备份 - Docker 安装](back-up-docker-installed-rancher.md)中推è的命åæ–¹å¼ï¼Œå®ƒçš„å称会与 `rancher-data-backup--.tar.gz` 类似。 + +1. 输入以下命令删除当å‰çŠ¶æ€æ•°æ®ï¼Œå¹¶å°†å…¶æ›¿æ¢ä¸ºå¤‡ä»½æ•°æ®ã€‚请替æ¢å ä½ç¬¦ã€‚ä¸è¦å¿˜è®°å…³é—­å¼•å·ã€‚ + + :::danger + + 该命令将删除 Rancher Server 容器中的所有当å‰çŠ¶æ€æ•°æ®ã€‚创建备份压缩包åŽä¿å­˜çš„任何更改都将丢失。 + + ::: + + ``` + docker run --volumes-from -v $PWD:/backup \ + busybox sh -c "rm /var/lib/rancher/* -rf && \ + tar pzxvf /backup/rancher-data-backup--.tar.gz" + ``` + + **步骤结果**：å±å¹•ä¸Šå°†è¿è¡Œå‘½ä»¤æµã€‚ + +1. é‡æ–°å¯åŠ¨ Rancher Server 容器，替æ¢å ä½ç¬¦ã€‚Rancher Server 将使用你的备份数æ®é‡æ–°å¯åŠ¨ã€‚ + + ``` + docker start + ``` + +1. 等待片刻，然åŽåœ¨æµè§ˆå™¨ä¸­æ‰“å¼€ Rancher。确认还原æˆåŠŸï¼Œå¹¶ä¸”ä½ çš„æ•°æ®å·²æ¢å¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md new file mode 100644 index 00000000000..97e23f81446 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md @@ -0,0 +1,129 @@ +--- +title: 使用备份æ¢å¤é›†ç¾¤ +--- + +ä½ å¯ä»¥è½»æ¾å¤‡ä»½å’Œæ¢å¤ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)çš„ etcd。etcd æ•°æ®åº“的快照会ä¿å­˜åœ¨ etcd 节点或 S3 兼容目标上。é…ç½® S3 的好处是，如果所有 etcd 节点都丢失了，你的快照会ä¿å­˜åˆ°è¿œç«¯å¹¶èƒ½ç”¨äºŽæ¢å¤é›†ç¾¤ã€‚ + +Rancher 建议å¯ç”¨ [etcd 定期快照的功能](back-up-rancher-launched-kubernetes-clusters.md#é…置定期快照)，但你也å¯ä»¥è½»æ¾åˆ›å»º[一次性快照](back-up-rancher-launched-kubernetes-clusters.md#å•æ¬¡å¿«ç…§)。Rancher å…许使用[ä¿å­˜çš„å¿«ç…§](#使用快照æ¢å¤é›†ç¾¤)进行æ¢å¤ã€‚如果你没有任何快照，你ä»ç„¶å¯ä»¥[æ¢å¤ etcd](#在没有快照的情况下æ¢å¤-etcdrke)。 + +集群也å¯ä»¥æ¢å¤åˆ°ä¹‹å‰çš„ Kubernetes 版本和集群é…置。 + +## 查看å¯ç”¨å¿«ç…§ + +Rancher UI 中æ供了集群所有å¯ç”¨å¿«ç…§çš„列表： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦æŸ¥çœ‹å¿«ç…§çš„集群并点击集群å称。 +1. å•å‡»**å¿«ç…§**选项å¡ã€‚列出的快照包括创建时间的时间戳。 + +## 使用快照æ¢å¤é›†ç¾¤ + +如果你的 Kubernetes 集群已æŸå，你å¯ä»¥ä½¿ç”¨å¿«ç…§æ¥æ¢å¤é›†ç¾¤ã€‚ + +快照由 etcd 中的集群数æ®ã€Kubernetes 版本和 `cluster.yml` 中的集群é…置组æˆã€‚有了这些组件，你å¯ä»¥åœ¨ä½¿ç”¨å¿«ç…§æ¢å¤é›†ç¾¤æ—¶é€‰æ‹©ï¼š + +- **ä»…æ¢å¤ etcd 内容**：类似于在 Rancher v2.4.0 之å‰ç‰ˆæœ¬ä¸­çš„使用快照æ¢å¤ã€‚ +- **æ¢å¤ etcd å’Œ Kubernetes 版本**：如果 Kubernetes å‡çº§å¯¼è‡´é›†ç¾¤å¤±è´¥ï¼Œå¹¶ä¸”你没有更改任何集群é…置，则应使用此选项。 +- **æ¢å¤ etcdã€Kubernetes 版本和集群é…ç½®**：如果你在å‡çº§æ—¶åŒæ—¶æ›´æ”¹äº† Kubernetes 版本和集群é…置，则应使用此选项。 + +回滚到之å‰çš„ Kubernetes 版本时，[å‡çº§ç­–略选项](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md#é…ç½®å‡çº§ç­–ç•¥)会被忽略。在æ¢å¤åˆ°æ—§ Kubernetes 版本之å‰ï¼ŒWorker 节点ä¸ä¼šè¢«å°é”或清空，因此å¯ä»¥æ›´å¿«åœ°å°†ä¸å¥åº·çš„集群æ¢å¤åˆ°å¥åº·çŠ¶æ€ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +è¦æ¢å¤ S3 中的快照，需è¦å°†é›†ç¾¤é…置为[在 S3 上创建定期快照](back-up-rancher-launched-kubernetes-clusters.md#é…置定期快照)。 + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦æŸ¥çœ‹å¿«ç…§çš„集群并点击集群å称。 +1. å•å‡»**å¿«ç…§**选项å¡æ¥æŸ¥çœ‹å·²ä¿å­˜å¿«ç…§çš„列表。 +1. 转到è¦æ¢å¤çš„快照，然åŽå•å‡» **â‹® > 还原**。 +1. 选择一个**还原类型**。 +1. 点击**还原**。 + +**结果**：集群将进入 `updating` 状æ€ï¼Œç„¶åŽå°†å¼€å§‹ä½¿ç”¨å¿«ç…§æ¢å¤ `etcd` 节点。集群会在返回到 `active` 状æ€åŽè¢«æ¢å¤ã€‚ + +## Control Plane/etcd 完全ä¸å¯ç”¨æ—¶ä½¿ç”¨å¿«ç…§è¿˜åŽŸé›†ç¾¤ + +在ç¾éš¾æ¢å¤åœºæ™¯ä¸­ï¼Œä¸‹æ¸¸é›†ç¾¤ä¸­ Rancher 管ç†çš„ Control Plane å’Œ etcd 节点å¯èƒ½ä¸å†å¯ç”¨æˆ–è¿è¡Œã€‚ä½ å¯ä»¥é€šè¿‡å†æ¬¡æ·»åŠ  Control Plane å’Œ etcd 节点æ¥é‡å»ºé›†ç¾¤ï¼Œç„¶åŽä½¿ç”¨å¯ç”¨å¿«ç…§æ¥è¿›è¡Œè¿˜åŽŸã€‚ + + + + +请按照 [SUSE 知识库](https://www.suse.com/support/kb/doc/?id=000020695)中æè¿°çš„æµç¨‹è¿›è¡Œæ“作。 + + + + +如果集群完全故障，则必须从集群中删除所有 etcd 节点和机器，然åŽæ‰èƒ½æ·»åŠ æ–°çš„ etcd 节点æ¥è¿›è¡Œè¿˜åŽŸã€‚ + +:::note + +由于[已知问题](https://github.com/rancher/rancher/issues/41080)ï¼Œæ­¤è¿‡ç¨‹éœ€è¦ Rancher v2.7.5 或更高版本。 + +::: + +:::note + +如果你使用[本地快照](./back-up-rancher-launched-kubernetes-clusters.md#本地备份目标)，那么请**务必**从è¦åˆ é™¤çš„ etcd 节点上的 `/var/lib/rancher//server/db/snapshots/` 文件夹中备份è¦è¿˜åŽŸçš„快照。你å¯ä»¥å°†å¿«ç…§å¤åˆ¶åˆ° `/var/lib/rancher//server/db/snapshots/` 文件夹中的新节点上。此外，如果使用本地快照并还原到新节点，目å‰è¿˜æ— æ³•é€šè¿‡ UI 进行还原。 + +::: + +1. 从集群中删除所有 etcd 节点。 + + 1. 在左上角，å•å‡» **☰ > 集群管ç†**。 + 1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦åˆ é™¤èŠ‚点的集群。 + 1. 在**主机**选项å¡ä¸­ï¼Œæ‰¾åˆ°è¦åˆ é™¤çš„æ¯ä¸ªèŠ‚点并å•å‡» **â‹® > 删除**。开始时，节点会挂在 `deleting` 状æ€ï¼Œæ‰€æœ‰ etcd 节点都被删除åŽï¼Œå®ƒä»¬å°†è¢«ä¸€èµ·åˆ é™¤ã€‚这是因为 Rancher å‘现所有 etcd 节点都在删除，并开始 "短路" etcd 安全删除逻辑。 + +1. 删除所有 etcd 节点åŽï¼Œæ·»åŠ è¦ç”¨äºŽè¿˜åŽŸçš„æ–° etcd 节点。 + + - 对于自定义集群，请转到**注册**选项å¡ï¼Œç„¶åŽåœ¨èŠ‚点上å¤åˆ¶å¹¶è¿è¡Œæ³¨å†Œå‘½ä»¤ã€‚如果该节点之å‰å·²åœ¨é›†ç¾¤ä¸­ä½¿ç”¨è¿‡ï¼Œè¯·å…ˆ[清ç†è¯¥èŠ‚点](../manage-clusters/clean-cluster-nodes.md#清ç†èŠ‚点)。 + - 对于主机驱动集群，则会自动é…置新节点。 + + 此时，Rancher 会æ示你需è¦ä½¿ç”¨ etcd 快照进行还原。 + +1. 使用 etcd 快照还原。 + + - 对于 S3 快照，使用 UI 进行还原。 + 1. å•å‡»**å¿«ç…§**选项å¡æ¥æŸ¥çœ‹å·²ä¿å­˜å¿«ç…§çš„列表。 + 1. 转到è¦æ¢å¤çš„快照，然åŽå•å‡» **â‹® > 还原**。 + 1. 选择一个**还原类型**。 + 1. 点击**还原**。 + - 对于本地快照，使用 UI 进行还原**ä¸**å¯ç”¨ã€‚ + 1. å•å‡»å³ä¸Šè§’çš„ **â‹®> 编辑 YAML**。 + 1. å°† `spec.cluster.rkeConfig.etcdSnapshotRestore.name` 定义为 `/var/lib/rancher//server/db/snapshots/` 中ç£ç›˜ä¸Šå¿«ç…§çš„文件å。 + +1. 还原æˆåŠŸåŽï¼Œä½ å¯ä»¥å°† etcd 节点扩展至所需的冗余。 + + + + +## 在没有快照的情况下æ¢å¤ etcd（RKE） + +如果 etcd 节点组失去了仲è£ï¼ˆquorum），由于没有æ“作（例如部署工作负载）å¯ä»¥åœ¨ Kubernetes 集群中执行，Kubernetes 集群将报告失败。集群需è¦æœ‰ä¸‰ä¸ª etcd 节点以防止仲è£ä¸¢å¤±ã€‚如果你想æ¢å¤ä½ çš„ etcd 节点集，请按照以下说明æ“作： + +1. 通过删除所有其他 etcd 节点，从而仅在集群中ä¿ç•™ä¸€ä¸ª etcd 节点。 + +2. 在剩余的å•ä¸ª etcd 节点上，è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ```bash + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock assaflavie/runlike etcd + ``` + + 此命令会输出 etcd è¦è¿è¡Œçš„命令，请ä¿å­˜æ­¤å‘½ä»¤ä»¥å¤‡åŽç”¨ã€‚ + +3. åœæ­¢æ­£åœ¨è¿è¡Œçš„ `etcd` 容器并将其é‡å‘½å为 `etcd-old`。 + + ```bash + docker stop etcd + docker rename etcd etcd-old + ``` + +4. 修改步骤 2 中获å–ä¿å­˜çš„命令： + + - 如果你最åˆæœ‰è¶…过 1 个 etcd 节点，则将 `--initial-cluster` 更改为仅包å«å‰©ä½™çš„å•ä¸ªèŠ‚点。 + - å°† `--force-new-cluster` 添加到命令的末尾。 + +5. è¿è¡Œä¿®æ”¹åŽçš„命令。 + +6. 在å•ä¸ªèŠ‚点å¯åŠ¨å¹¶è¿è¡ŒåŽï¼ŒRancher 建议å‘你的集群添加é¢å¤–çš„ etcd 节点。如果你有一个[自定义集群](../../../pages-for-subheaders/use-existing-nodes.md)，并且想è¦å¤ç”¨æ—§èŠ‚点，则需è¦å…ˆ[清ç†èŠ‚点](../manage-clusters/clean-cluster-nodes.md)，然åŽå†å°è¯•å°†å®ƒä»¬é‡æ–°æ·»åŠ åˆ°é›†ç¾¤ä¸­ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md new file mode 100644 index 00000000000..11760970b23 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md @@ -0,0 +1,97 @@ +--- +title: 还原 Rancher +--- + +本页概述了如何使用 Rancher 执行æ¢å¤ã€‚ + +:::note é‡è¦æ示： + +* 请按照此页é¢ä¸Šçš„说明在已备份的åŒä¸€é›†ç¾¤ä¸Šè¿˜åŽŸ Rancher。è¦æŠŠ Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œè¯·å‚照步骤[è¿ç§» Rancher](migrate-rancher-to-new-cluster.md)。 +* 在使用相åŒè®¾ç½®è¿˜åŽŸ Rancher 时，operator å°†åœ¨è¿˜åŽŸå¼€å§‹æ—¶ç¼©å‡ Rancher deployment，还原完æˆåŽåˆä¼šæ‰©å±• deployment。因此，Rancher 在还原期间将ä¸å¯ç”¨ã€‚ +* 如果你需è¦åœ¨å‡çº§åŽå°† Rancher 还原到先å‰ç‰ˆæœ¬ï¼Œè¯·å‚è§[回滚](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md)。 + +::: + +## 使用 Rancher 2.6.4+ 进行回滚的其他步骤 + +Rancher v2.6.4 å°† cluster-api 模å—从 v0.4.4 å‡çº§åˆ° v1.0.2。å过æ¥ï¼Œcluster-api çš„ v1.0.2 版本将集群 API 的自定义资æºå®šä¹‰ (CRD) 从 `cluster.x-k8s.io/v1alpha4` å‡çº§åˆ° `cluster.x-k8s.io/v1beta1`。当你å°è¯•å°† Rancher v2.6.4 回滚到以å‰ç‰ˆæœ¬çš„ Rancher v2.6.x 时，CRD å‡çº§åˆ° v1beta1 会导致回滚失败。这是因为使用旧 apiVersion (v1alpha4) çš„ CRD 与 v1beta1 ä¸å…¼å®¹ã€‚ + +è¦é¿å…回滚失败，你需è¦åœ¨å°è¯•æ¢å¤æ“作或回滚**之å‰**è¿è¡Œä»¥ä¸‹ Rancher 脚本： + +* `verify.sh`：检查集群中是å¦æœ‰ä»»ä½•ä¸Ž Rancher 相关的资æºã€‚ +* `cleanup.sh`：清ç†é›†ç¾¤ã€‚ + +有关详细信æ¯å’Œæºä»£ç ï¼Œè¯·å‚阅 [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup)。 + +:::caution + +`cleanup.sh` è¿è¡Œçš„时候会有åœæœºæ—¶é—´ï¼Œè¿™æ˜¯å› ä¸ºè„šæœ¬ä¼šåˆ é™¤ Rancher 创建的资æºã€‚ + +::: + +### 从 v2.6.4+ 回滚到较低版本的 v2.6.x + +1. 按照[说明](https://github.com/rancher/rancher-cleanup/blob/main/README.md)è¿è¡Œè„šæœ¬ã€‚ +1. 按照[说明](../backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)在现有集群上安装 rancher-backup Helm Chart 并æ¢å¤ä¹‹å‰çš„状æ€ã€‚ + 1. çœç•¥æ­¥éª¤ 3。 + 1. 执行到步骤 4 时，在è¦å›žæ»šåˆ°çš„ local 集群上安装 Rancher 2.6.x 版本。 + +### 创建 Restore è‡ªå®šä¹‰èµ„æº + +还原是通过创建 Restore 自定义资æºå®žçŽ°çš„。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。Rancher Server è¿è¡Œåœ¨ `local` 集群中。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Rancher 备份 > 还原**。 +1. å•å‡»**创建**。 +1. 使用表å•æˆ– YAML 创建 Restore。如需获å–使用表å•åˆ›å»º Restore 资æºçš„更多信æ¯ï¼Œè¯·å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 +1. è¦ä½¿ç”¨ YAML 编辑器，å•å‡»**创建 > 使用 YAML 文件创建**。输入 Restore YAML。 + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + 如需获得é…ç½® Restore 的帮助，请å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 + +1. å•å‡»**创建**。 + +**结果**：rancher-operator 在还原过程中将 Rancher deployment 缩容，并在还原完æˆåŽå°†å®ƒé‡æ–°æ‰©å®¹ã€‚资æºè¿˜åŽŸé¡ºåºå¦‚下： + +1. 自定义资æºå®šä¹‰ï¼ˆCRD） +2. é›†ç¾¤èŒƒå›´èµ„æº +3. 命åç©ºé—´èµ„æº + +### 日志 + +如需查看还原的处ç†æ–¹å¼ï¼Œè¯·æ£€æŸ¥ Operator 的日志。查看日志的命令如下： + +``` +kubectl logs -n cattle-resources-system -l app.kubernetes.io/name=rancher-backup -f +``` + +### æ¸…ç† + +如果你使用 kubectl 创建了 Restore 资æºï¼Œè¯·åˆ é™¤è¯¥èµ„æºä»¥é˜²æ­¢ä¸Žæœªæ¥çš„还原å‘生命å冲çªã€‚ + +### 已知问题 +在æŸäº›æƒ…况下，æ¢å¤å¤‡ä»½åŽï¼ŒRancher 日志会显示类似以下的错误： +``` +2021/10/05 21:30:45 [ERROR] error syncing 'c-89d82/m-4067aa68dd78': handler rke-worker-upgrader: clusters.management.cattle.io "c-89d82" not found, requeuing +``` +å‘生这ç§æƒ…况的原因是，刚刚还原的æŸä¸ªèµ„æºæœ‰ finalizer，但相关的资æºå·²ç»è¢«åˆ é™¤ï¼Œå¯¼è‡´å¤„ç†ç¨‹åºæ— æ³•æ‰¾åˆ°è¯¥èµ„æºã€‚ + +为了消除这些错误，你需è¦æ‰¾åˆ°å¹¶åˆ é™¤å¯¼è‡´é”™è¯¯çš„资æºã€‚详情请å‚è§[此处](https://github.com/rancher/rancher/issues/35050#issuecomment-937968556)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md new file mode 100644 index 00000000000..def223a8337 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md @@ -0,0 +1,67 @@ +--- +title: 使用 Feet 进行æŒç»­äº¤ä»˜ +--- + +使用 Fleet çš„æŒç»­äº¤ä»˜æ˜¯å¤§è§„模的 GitOps。你å¯ä»¥ä½¿ç”¨ Fleet 管ç†å¤šè¾¾ä¸€ç™¾ä¸‡ä¸ªé›†ç¾¤ã€‚Fleet éžå¸¸è½»é‡ï¼Œå¯ä»¥å¾ˆå¥½åœ°ç”¨äºŽ[å•ä¸ªé›†ç¾¤](https://fleet.rancher.io/installation#default-install)，但是在你达到[大规模](https://fleet.rancher.io/installation#configuration-for-multi-cluster)时，它能å‘挥更强的实力。此处的大规模指的是大é‡é›†ç¾¤ã€å¤§é‡éƒ¨ç½²ã€æˆ–组织中存在大é‡å›¢é˜Ÿçš„情况。 + +Fleet 是一个独立于 Rancher 的项目，你å¯ä»¥ä½¿ç”¨ Helm 将它安装在任何 Kubernetes 集群上。 + + +## 架构 + +有关 Fleet 工作原ç†çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/fleet-gitops-at-scale/architecture.md)。 + +## 在 Rancher UI 中访问 Fleet + +Fleet 预装在 Rancher 中，通过 Rancher UI 中的**æŒç»­äº¤ä»˜**选项管ç†ã€‚有关æŒç»­äº¤ä»˜å’Œ Fleet 故障排除技巧的更多信æ¯ï¼Œè¯·å‚阅[此处](https://fleet.rancher.io/troubleshooting)。 + +用户å¯ä»¥é€šè¿‡éµå¾ª **gitops** 的实践，利用æŒç»­äº¤ä»˜å°†åº”用部署到 git 仓库中的 Kubernetes 集群，而无需任何手动æ“作。 + +按照以下步骤在 Rancher UI 中访问æŒç»­äº¤ä»˜ï¼š + +1. å•å‡» **☰ > æŒç»­äº¤ä»˜**。 + +1. 在èœå•é¡¶éƒ¨é€‰æ‹©ä½ çš„命å空间，注æ„以下几点： + - 默认情况下会选中 `fleet-default`，其中包括注册到 Rancher 的所有下游集群。 + - ä½ å¯ä»¥åˆ‡æ¢åˆ°ä»…åŒ…å« `local` 集群的 `fleet-local`，或者创建自己的工作空间，并将集群分é…和移动到该工作空间。 + - 然åŽï¼Œä½ å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„**集群**æ¥ç®¡ç†é›†ç¾¤ã€‚ + +1. å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„ **Git 仓库**å°† git 仓库部署到当å‰å·¥ä½œç©ºé—´ä¸­çš„集群中。 + +1. 选择你的 [git 仓库](https://fleet.rancher.io/gitrepo-add)å’Œ[目标集群/集群组](https://fleet.rancher.io/gitrepo-targets)。你还å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸­çš„**集群组**在 UI 中创建集群组。 + +1. 部署 git 仓库åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 监控应用。 + +## Windows æ”¯æŒ + +有关对具有 Windows 节点的集群的支æŒçš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md)。 + + +## GitHub 仓库 + +ä½ å¯ä»¥å•å‡»æ­¤å¤„èŽ·å– [Fleet Helm Chart](https://github.com/rancher/fleet/releases/latest)。 + + +## 在代ç†åŽä½¿ç”¨ Fleet + +有关在代ç†åŽä½¿ç”¨ Fleet 的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md)。 + +## Helm Chart ä¾èµ– + +由于用户需è¦å®Œæˆä¾èµ–列表，因此为了æˆåŠŸéƒ¨ç½²å…·æœ‰ä¾èµ–项的 Helm Chart，你必须手动è¿è¡Œå‘½ä»¤ï¼ˆå¦‚下所列）。如果你ä¸è¿™æ ·åšï¼Œå¹¶ç»§ç»­å…‹éš†ä»“库并è¿è¡Œ `helm install`，由于ä¾èµ–项将丢失，因此你的安装将失败。 + +git 仓库中的 Helm Chart 必须在 Chart å­ç›®å½•ä¸­åŒ…å«å…¶ä¾èµ–项。你必须手动è¿è¡Œ `helm dependencies update $chart`，或在本地è¿è¡Œ `helm dependencies build $chart`，然åŽå°†å®Œæ•´çš„ Chart 目录æ交到你的 git 仓库。请注æ„，你需è¦ä½¿ç”¨é€‚当的å‚æ•°æ¥ä¿®æ”¹å‘½ä»¤ã€‚ + +## 故障排除 + +--- +* **已知问题**：Fleet git 仓库的 clientSecretName å’Œ helmSecretName 密文ä¸åŒ…å«åœ¨ [backup-restore-operator](../backup-restore-and-disaster-recovery/back-up-rancher.md#1-安装-rancher-backup-operator) 创建的备份或æ¢å¤ä¸­ã€‚如果我们有了永久的解决方案，我们将通知社区。 + +* **临时解决方法：**
+ 默认情况下，用户定义的密文ä¸ä¼šåœ¨ Fleet 中备份。如果执行ç¾éš¾æ¢å¤æˆ–å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œåˆ™éœ€è¦é‡æ–°åˆ›å»ºå¯†æ–‡ã€‚è¦ä¿®æ”¹ resourceSet 以包å«éœ€è¦å¤‡ä»½çš„其他资æºï¼Œè¯·å‚阅[此文档](https://github.com/rancher/backup-restore-operator#user-flow)。 + +--- + +## 文档 + +Fleet 文档链接：[https://fleet.rancher.io/](https://fleet.rancher.io/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md new file mode 100644 index 00000000000..113a30ad48b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md @@ -0,0 +1,175 @@ +--- +title: 多集群应用 +--- + +通常，大多数应用都部署在å•ä¸ª Kubernetes 集群上，但有时你å¯èƒ½éœ€è¦è·¨ä¸åŒé›†ç¾¤å’Œ/或项目部署åŒä¸€åº”用的多个副本。在 Rancher 中，_多集群应用_ 指的是使用 Helm Chart 跨多个集群部署的应用。由于能够跨多个集群部署相åŒçš„应用，因此å¯ä»¥é¿å…在æ¯ä¸ªé›†ç¾¤ä¸Šé‡å¤æ‰§è¡Œç›¸åŒçš„应用é…ç½®æ“作而引入的人为错误。使用多集群应用，你å¯ä»¥é€šè¿‡è‡ªå®šä¹‰åœ¨æ‰€æœ‰é¡¹ç›®/集群中使用相åŒçš„é…置，并根æ®ä½ çš„目标项目更改é…置。由于多集群应用被视为å•ä¸ªåº”用，因此更容易管ç†å’Œç»´æŠ¤ã€‚ + +全局应用商店中的任何 Helm Chart 都å¯ç”¨äºŽéƒ¨ç½²å’Œç®¡ç†å¤šé›†ç¾¤åº”用。 + +创建多集群应用åŽï¼Œä½ å¯ä»¥å¯¹å…¨å±€ DNS æ¡ç›®è¿›è¡Œç¼–程，以便更轻æ¾åœ°è®¿é—®åº”用。 + +## 先决æ¡ä»¶ + +### æƒé™ + +è¦åœ¨ Rancher 中创建多集群应用，你至少需è¦å…·æœ‰ä»¥ä¸‹æƒé™ä¹‹ä¸€ï¼š + +- 目标集群中的[项目æˆå‘˜è§’色](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)，能够创建ã€è¯»å–ã€æ›´æ–°å’Œåˆ é™¤å·¥ä½œè´Ÿè½½ +- 目标项目所在集群的[集群所有者角色](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色) + +### å¯ç”¨æ—§ç‰ˆåŠŸèƒ½ + +由于 Rancher 2.5 已弃用多集群应用并使用 Fleet å–代它，你需è¦ä½¿ç”¨åŠŸèƒ½å¼€å…³ä»¥å¯ç”¨å¤šé›†ç¾¤åº”用。 + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 转到 `Legacy` 功能开关并å•å‡»**激活**。 + +## å¯åŠ¨å¤šé›†ç¾¤åº”用 + +1. 在左上角，å•å‡»**☰ > 多集群应用**。 +1. 点击**å¯åŠ¨**。 +1. 找到è¦å¯åŠ¨çš„应用。 +1. （å¯é€‰ï¼‰æŸ¥çœ‹æ¥è‡ª Helm Chart `README` 的详细æ述。 +1. 在**é…置选项**下输入多集群应用的**å称**。默认情况下，此å称还用于在æ¯ä¸ª[目标项目](#目标)中为多集群应用创建一个 Kubernetes 命å空间。命å空间命å为 `-`。 +1. 选择一个**模æ¿ç‰ˆæœ¬**。 +1. 完æˆ[多集群应用é…置选项](#多集群应用é…置选项)以åŠ[应用é…置选项](#应用é…置选项)。 +1. 选择å¯ä»¥[与多集群应用交互](#æˆå‘˜)çš„**æˆå‘˜**。 +1. 添加[自定义应用é…置答案](#覆盖特定项目的应用é…置选项)，这将更改默认应用é…置答案中特定项目的é…置。 +1. 查看**预览**中的文件。确认åŽï¼Œå•å‡»**å¯åŠ¨**。 + +**结果**：应用已部署到所选的命å空间。你å¯ä»¥ä»Žé¡¹ç›®ä¸­æŸ¥çœ‹åº”用状æ€ã€‚ + +## 多集群应用é…置选项 + +Rancher 将多集群应用的é…置选项分为以下几个部分。 + +### 目标 + +在**目标**部分中，选择用于部署应用的项目。项目列表仅显示你有æƒè®¿é—®çš„项目。所选的æ¯ä¸ªé¡¹ç›®éƒ½ä¼šè¢«æ·»åŠ åˆ°åˆ—表中，其中显示了所选的集群å称和项目å称。è¦ç§»é™¤ç›®æ ‡é¡¹ç›®ï¼Œå•å‡» **-**。 + +### å‡çº§ + +在**å‡çº§**部分中，选择å‡çº§åº”用时需è¦ä½¿ç”¨çš„å‡çº§ç­–略。 + +* **滚动更新（批é‡ï¼‰**：选择此å‡çº§ç­–略时，æ¯æ¬¡å‡çº§çš„应用数é‡å–决于选择的**批é‡å¤§å°**å’Œ**é—´éš”**（多少秒åŽæ‰å¼€å§‹ä¸‹ä¸€æ‰¹æ›´æ–°ï¼‰ã€‚ + +* **åŒæ—¶å‡çº§æ‰€æœ‰åº”用**：选择此å‡çº§ç­–略时，所有项目的所有应用都将åŒæ—¶å‡çº§ã€‚ + +### 角色 + +在**角色**中，你å¯ä»¥å®šä¹‰å¤šé›†ç¾¤åº”用的角色。通常，当用户[å¯åŠ¨å•†åº—应用](../../../pages-for-subheaders/helm-charts-in-rancher.md)时，该用户的æƒé™ä¼šç”¨äºŽåˆ›å»ºåº”用所需的所有工作负载/资æºã€‚ + +多集群应用由 _系统用户_ 部署，系统用户还被指定为所有底层资æºçš„创建者。由于实际用户å¯ä»¥ä»ŽæŸä¸ªç›®æ ‡é¡¹ç›®ä¸­åˆ é™¤ï¼Œå› æ­¤ä½¿ç”¨ _系统用户_ 而ä¸æ˜¯å®žé™…用户。如果实际用户从其中一个项目中删除，则该用户将ä¸å†èƒ½å¤Ÿç®¡ç†å…¶ä»–项目的应用。 + +Rancher å…许你选择**项目**或**集群**的角色选项。Rancher å°†å…许你根æ®ç”¨æˆ·çš„æƒé™ä½¿ç”¨å…¶ä¸­ä¸€ä¸ªè§’色进行创建。 + +- **项目** - 相当于[项目æˆå‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)。如果你选择此角色，Rancher 将检查用户是å¦åœ¨æ‰€æœ‰ç›®æ ‡é¡¹ç›®ä¸­è‡³å°‘具有[项目æˆå‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)的角色。虽然用户å¯èƒ½æ²¡æœ‰è¢«æ˜Žç¡®æŽˆäºˆ _项目æˆå‘˜_ 角色，但如果用户是[管ç†å‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)ã€[集群所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)或[项目所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)，则认为该用户具有所需的æƒé™çº§åˆ«ã€‚ + +- **集群** - 相当于[集群所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)。如果你选择此角色，Rancher 将检查用户是å¦åœ¨æ‰€æœ‰ç›®æ ‡é¡¹ç›®ä¸­è‡³å°‘具有[集群所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)的角色。虽然用户å¯èƒ½æ²¡æœ‰è¢«æ˜Žç¡®æŽˆäºˆ _集群所有者_ 角色，但如果用户是[管ç†å‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)，则认为该用户具有所需的æƒé™çº§åˆ«ã€‚ + +在å¯åŠ¨åº”用时，Rancher 会在å¯åŠ¨åº”用之å‰ç¡®è®¤ä½ åœ¨ç›®æ ‡é¡¹ç›®ä¸­æ˜¯å¦æ‹¥æœ‰è¿™äº›æƒé™ã€‚ + +:::note + +æŸäº›åº”用（如 _Grafana_ 或 _Datadog_）需è¦è®¿é—®ç‰¹å®šé›†ç¾¤çº§åˆ«çš„资æºã€‚è¿™äº›åº”ç”¨å°†éœ€è¦ _集群_ 角色。如果你之åŽå‘现应用需è¦é›†ç¾¤è§’色，则å¯ä»¥å‡çº§å¤šé›†ç¾¤åº”用以更新角色。 + +::: + +## 应用é…置选项 + +对于æ¯ä¸ª Helm Chart，你需è¦è¾“入一个必须的答案列表æ‰èƒ½æˆåŠŸéƒ¨ç½² Chart。由于 Rancher 会将答案作为 `--set` 标志传递给 Helm，因此你必须按照[使用 Helm：–set çš„æ ¼å¼å’Œé™åˆ¶](https://helm.sh/docs/intro/using_helm/#the-format-and-limitations-of---set)中的语法规则æ¥æ ¼å¼åŒ–这些答案。 + +:::note 示例 + +当输入的答案包å«ç”¨é€—å·åˆ†éš”çš„ä¸¤ä¸ªå€¼ï¼ˆå³ `abc, bcd`）时，你需è¦ç”¨åŒå¼•å·å°†å€¼æ‹¬èµ·æ¥ï¼ˆå³ ``"abc, bcd" ``）。 + +::: + +### 使用 questions.yml 文件 + +如果你部署的 Helm Chart åŒ…å« `questions.yml` 文件，Rancher UI 会将此文件转æ¢æˆæ˜“于使用的 UI æ¥æ”¶é›†é—®é¢˜çš„答案。 + +### 原生 Helm Chart 的键值对 + +对于原生 Helm Chart（å³æ¥è‡ª **Helm Stable** 或 **Helm Incubator** 应用商店或自定义 Helm Chart 仓库的 Chart），答案会在 **Answers** 中以键值对的形å¼æ供。这些答案能覆盖默认值。 + +### æˆå‘˜ + +默认情况下，多集群应用åªèƒ½ç”±åº”用的创建者管ç†ã€‚ä½ å¯ä»¥åœ¨**æˆå‘˜**中添加其他用户，以便这些用户管ç†æˆ–查看多集群应用。 + +1. 在**æˆå‘˜**æœç´¢æ¡†ä¸­é”®å…¥æˆå‘˜çš„å称，查找è¦æ·»åŠ çš„用户。 + +2. 为该æˆå‘˜é€‰æ‹©**访问类型**。多集群项目有三ç§è®¿é—®ç±»åž‹ï¼Œè¯·ä»”细阅读并了解这些访问类型的å«ä¹‰ï¼Œä»¥äº†è§£å¤šé›†ç¾¤åº”用æƒé™çš„å¯ç”¨æ–¹å¼ã€‚ + + - **所有者**：此访问类型å¯ä»¥ç®¡ç†å¤šé›†ç¾¤åº”用的任何é…置，包括模æ¿ç‰ˆæœ¬ã€[多集群应用é…置选项](#多集群应用é…置选项)，[应用é…置选项](#应用é…置选项)，å¯ä»¥ä¸Žå¤šé›†ç¾¤åº”用交互的æˆå‘˜ï¼Œä»¥åŠ[自定义应用é…置答案](#覆盖特定项目的应用é…置选项)。由于多集群应用的创建使用与用户ä¸åŒçš„æƒé™é›†ï¼Œå› æ­¤å¤šé›†ç¾¤åº”用的任何 _所有者_ 都å¯ä»¥ç®¡ç†/删除[目标项目](#目标)中的应用，而ä¸éœ€è¦æ˜¾å¼æŽˆæƒè®¿é—®è¿™äº›é¡¹ç›®ã€‚请仅为å—信任的用户é…置此访问类型。 + + - **æˆå‘˜**：此访问类型åªèƒ½ä¿®æ”¹æ¨¡æ¿ç‰ˆæœ¬ã€[应用é…置选项](#应用é…置选项)å’Œ[自定义应用é…置答案](#覆盖特定项目的应用é…置选项)。由于多集群应用的创建使用与用户ä¸åŒçš„æƒé™é›†ï¼Œå› æ­¤å¤šé›†ç¾¤åº”用的任何 _æˆå‘˜_ 都å¯ä»¥ä¿®æ”¹åº”用，而ä¸éœ€è¦æ˜¾å¼æŽˆæƒè®¿é—®è¿™äº›é¡¹ç›®ã€‚请仅为å—信任的用户é…置此访问类型。 + + - **åªè¯»**：此访问类型ä¸èƒ½ä¿®æ”¹å¤šé›†ç¾¤åº”用的任何é…置选项。用户åªèƒ½æŸ¥çœ‹è¿™äº›åº”用。 + + :::caution + + 请确ä¿ä»…为å—信任的用户授予 _所有者_ 或 _æˆå‘˜_ 访问æƒé™ï¼Œå› ä¸ºè¿™äº›ç”¨æˆ·å³ä½¿æ— æ³•ç›´æŽ¥è®¿é—®é¡¹ç›®ï¼Œä¹Ÿå°†è‡ªåŠ¨èƒ½å¤Ÿç®¡ç†ä¸ºæ­¤å¤šé›†ç¾¤åº”用创建的应用。 + + ::: + +### 覆盖特定项目的应用é…置选项 + +多集群应用的主è¦ä¼˜åŠ¿ä¹‹ä¸€ï¼Œæ˜¯èƒ½å¤Ÿåœ¨å¤šä¸ªé›†ç¾¤/项目中使用相åŒé…置部署相åŒçš„应用。在æŸäº›æƒ…况下，你å¯èƒ½éœ€è¦ä¸ºæŸä¸ªç‰¹å®šé¡¹ç›®ä½¿ç”¨ç¨å¾®ä¸åŒçš„é…置选项，但你ä¾ç„¶å¸Œæœ›ç»Ÿä¸€ç®¡ç†è¯¥åº”用与其他匹é…的应用。此时，你å¯ä»¥ä¸ºè¯¥é¡¹ç›®è¦†ç›–特定的[应用é…置选项](#应用é…置选项)，而ä¸éœ€è¦åˆ›å»ºå…¨æ–°çš„应用。 + +1. 在**答案覆盖**中，å•å‡»**添加覆盖**。 + +2. 对于æ¯ä¸ªè¦†ç›–，你å¯ä»¥é€‰æ‹©ä»¥ä¸‹å†…容： + + - **范围**：在é…置选项中选择è¦è¦†ç›–哪些目标项目的答案。 + + - **问题**：选择è¦è¦†ç›–的问题。 + + - **答案**：输入è¦ä½¿ç”¨çš„答案。 + +## å‡çº§å¤šé›†ç¾¤åº”用角色和项目 + +- **在现有的多集群应用上更改角色** + 多集群应用的创建者和任何具有“所有者â€è®¿é—®ç±»åž‹çš„用户都å¯ä»¥å‡çº§å…¶**角色**。添加新角色时，我们会检查用户在所有当å‰ç›®æ ‡é¡¹ç›®ä¸­æ˜¯å¦å…·æœ‰è¯¥è§’色。Rancher ä¼šæ ¹æ® `Roles` 字段的安装部分，相应地检查用户是å¦å…·æœ‰å…¨å±€ç®¡ç†å‘˜ã€é›†ç¾¤æ‰€æœ‰è€…或项目所有者的角色。 + +- **添加/删除目标项目** +1. 多集群应用的创建者和任何具有“所有者â€è®¿é—®ç±»åž‹çš„用户都添加或移除目标项目。添加新项目时，我们检查此请求的调用者是å¦å…·æœ‰å¤šé›†ç¾¤åº”用中定义的所有角色。Rancher 会检查用户是å¦å…·æœ‰å…¨å±€ç®¡ç†å‘˜ã€é›†ç¾¤æ‰€æœ‰è€…和项目所有者的角色。 +2. 删除目标项目时，我们ä¸ä¼šè¿›è¡Œè¿™äº›æˆå‘˜èµ„格检查。这是因为调用者的æƒé™å¯èƒ½ä¸Žç›®æ ‡é¡¹ç›®æœ‰å…³ï¼Œæˆ–者由于该项目已被删除导致调用者希望将该项目从目标列表中删除。 + + +## å¤šé›†ç¾¤åº”ç”¨ç®¡ç† + +与åŒä¸€ç±»åž‹çš„多个å•ç‹¬åº”用相比，使用多集群应用的好处之一是易于管ç†ã€‚ä½ å¯ä»¥å…‹éš†ã€å‡çº§æˆ–回滚多集群应用。 + +:::note 先决æ¡ä»¶ï¼š + +`Legacy` 功能开关已å¯ç”¨ã€‚ + +::: + +1. 在左上角，å•å‡»**☰ > 多集群应用**。 + +2. 选择è¦å¯¹å…¶æ‰§è¡Œæ“作的多集群应用，然åŽå•å‡» **â‹®**。选择以下选项之一： + + * **克隆**：创建å¦ä¸€ä¸ªå…·æœ‰ç›¸åŒé…置的多集群应用。通过使用此选项，你å¯ä»¥è½»æ¾å¤åˆ¶å¤šé›†ç¾¤åº”用。 + * **å‡çº§**：å‡çº§å¤šé›†ç¾¤åº”用以更改æŸäº›é…置。在为多集群应用执行å‡çº§æ—¶ï¼Œå¦‚果你有åˆé€‚çš„[访问类型](#æˆå‘˜)，则å¯ä»¥ä¿®æ”¹[å‡çº§ç­–ç•¥](#å‡çº§)。 + * **回滚**：将你的应用回滚到特定版本。如果你的一个或多个[目标](#目标)的多集群应用在å‡çº§åŽå‡ºçŽ°é—®é¢˜ï¼Œä½ å¯ä»¥ä½¿ç”¨ Rancher 存储的多达 10 个多集群应用版本进行回滚。回滚多集群应用会æ¢å¤**所有**目标集群和项目的应用，而ä¸ä»…仅是å—å‡çº§é—®é¢˜å½±å“的目标。 + +## 删除多集群应用 + +:::note 先决æ¡ä»¶ï¼š + +`Legacy` 功能开关已å¯ç”¨ã€‚ + +::: + +1. 在左上角，å•å‡»**☰ > 多集群应用**。 + +2. 选择è¦åˆ é™¤çš„多集群应用，然åŽå•å‡»**â‹® > 删除**。删除多集群应用会删除所有目标项目中的所有应用和命å空间。 + + :::note + + ä¸èƒ½ç‹¬ç«‹åˆ é™¤åœ¨ç›®æ ‡é¡¹ç›®ä¸­ä¸ºå¤šé›†ç¾¤åº”用创建的应用。åªæœ‰åˆ é™¤å¤šé›†ç¾¤åº”用åŽæ‰èƒ½åˆ é™¤è¿™äº›åº”用。 + + ::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md new file mode 100644 index 00000000000..b6605e51965 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -0,0 +1,126 @@ +--- +title: 创建应用 +--- + +:::tip + +æœ‰å…³å¼€å‘ Chart 的完整演示，请å‚阅 Helm 官方文档中的 [Chart 模æ¿å¼€å‘者指å—](https://helm.sh/docs/chart_template_guide/)。 + +::: + +## Chart 类型 + +Rancher 支æŒä¸¤ç§ä¸åŒç±»åž‹çš„ Chart，分别是 Helm Chart å’Œ Rancher Chart。 + +### Helm Chart + +原生 Helm Chart 包括一个应用以åŠè¿è¡Œå®ƒæ‰€éœ€çš„软件。部署原生 Helm Chart 时，你å¯ä»¥åœ¨ YAML 编辑器中æä¾› Chart çš„å‚数值。 + +### Rancher Chart + +Rancher Chart 是原生 helm Chart，包å«ä¸¤ä¸ªå¯å¢žå¼ºç”¨æˆ·ä½“验的文件 `app-readme.md` å’Œ `questions.yaml`。在 [Rancher Chart 的附加文件](#rancher-chart-的附加文件)中了解更多信æ¯ã€‚ + +Rancher Chart 添加了简化的 Chart æè¿°å’Œé…置表å•ï¼Œä½¿åº”用部署å˜å¾—容易。Rancher 用户无需通读整个 Helm å˜é‡åˆ—表å³å¯äº†è§£å¦‚何å¯åŠ¨åº”用。 + +## Chart 目录结构 + +ä½ å¯ä»¥åœ¨åŸºäºŽ HTTP 的标准 Helm 仓库中æä¾› Helm Chart。有关更多信æ¯ï¼Œè¯·å‚阅 Helm 官方文档中的 [Chart 仓库指å—](https://helm.sh/docs/topics/chart_repository)。 + +或者，你å¯ä»¥åœ¨ Git 仓库中组织 Chart 并将其直接添加到应用市场。 + +下表演示了 Git 仓库的目录结构。`charts` 目录是仓库基础下的顶层目录。将仓库添加到 Rancher 将公开其中包å«çš„所有 Chart。`questions.yaml`ã€`README.md` å’Œ `requirements.yml` 文件是针对于 Rancher Chart 的，但对于自定义 Chart 是å¯é€‰çš„。 + +``` +/ + │ + ├── charts/ + │ ├── / # 这个目录å称将作为 Chart å称出现在 Rancher UI 中。 + │ │ ├── / # 这个级别的æ¯ä¸ªç›®å½•æ供了ä¸åŒçš„应用版本，å¯ä»¥åœ¨ Rancher UI çš„ Chart 中选择。 + │ │ │ ├── Chart.yaml # 必需的 Helm Chart ä¿¡æ¯æ–‡ä»¶ã€‚ + │ │ │ ├── questions.yaml # 在 Rancher UI 中显示的表å•é—®é¢˜ã€‚问题显示在é…置选项中。* + │ │ │ ├── README.md # å¯é€‰ï¼šåœ¨ Rancher UI 中显示的 Helm 自述文件。此文本显示在详细说明中。 + │ │ │ ├── requirements.yml # å¯é€‰ï¼šåˆ—出 Chart çš„ä¾èµ–项的 YAML 文件。 + │ │ │ ├── values.yml # Chart 的默认é…置值。 + │ │ │ ├── templates/ # 包å«æ¨¡æ¿çš„目录，与 values.yml ä¸€èµ·èƒ½ç”Ÿæˆ Kubernetes YAML。 +``` + +## Rancher Chart 的附加文件 + +在创建你的自定义目录之å‰ï¼Œä½ éœ€è¦å¤§è‡´äº†è§£ Rancher chart 与原生 Helm chart 的区别。Rancher Chart 的目录结构与 Helm Chart 略有ä¸åŒã€‚Rancher Chart 包å«ä¸¤ä¸ª Helm Chart 没有的文件： + +- `app-readme.md` + + 在 Chart çš„ UI 标头中æä¾›æ述性文本的文件。 + +- `questions.yml` + + 包å«è¡¨å•é—®é¢˜çš„文件。这些表å•é—®é¢˜ç®€åŒ–了 Chart 的部署。没有它，你必须使用更å¤æ‚çš„ YAML é…ç½®æ¥é…置部署。下图显示了 Rancher Chartï¼ˆåŒ…å« `questions.yml`）和原生 Helm Chart（ä¸åŒ…å«ï¼‰ä¹‹é—´çš„区别。 + +
带有 questions.yml 的 Rancher Chart（上）与 Helm Chart（下）
+ + ![questions.yml](/img/rancher-app-2.6.png) + ![values.yaml](/img/helm-app-2.6.png) + + +### Chart.yaml 注释 + +Rancher 支æŒä½ æ·»åŠ åˆ° `Chart.yaml` 文件的其他注释。这些注释å…许你定义应用ä¾èµ–项或é…置其他 UI 默认值： + +| 注解 | æè¿° | 示例 | +| --------------------------------- | ----------- | ------- | +| catalog.cattle.io/auto-install | 如果设置，将在安装此 Chart 之å‰å…ˆå®‰è£…指定 Chart 的指定版本。 | other-chart-name=1.0.0 | +| catalog.cattle.io/display-name | è¦åœ¨åº”用市场中显示的å称，而ä¸æ˜¯ Chart 本身的å称。 | Chart 的显示å称 | +| catalog.cattle.io/namespace | 用于部署 Chart 的固定命å空间。如果设置，则用户无法更改。 | fixed-namespace | +| catalog.cattle.io/release-name | Helm 安装的固定版本å称。如果设置，则用户无法更改。 | fixed-release-name | +| catalog.cattle.io/requests-cpu | 应该在集群中ä¿ç•™çš„ CPU 总é‡ã€‚如果å¯ç”¨ CPU 资æºå°‘于该值，将显示警告。 | 2000m | +| catalog.cattle.io/requests-memory | 应该在集群中ä¿ç•™çš„内存总é‡ã€‚如果å¯ç”¨å†…存资æºå°‘于该值，将显示警告。 | 2Gi | +| catalog.cattle.io/os | é™åˆ¶å¯ä»¥å®‰è£…æ­¤ Chart çš„æ“作系统。å¯ç”¨å€¼ï¼š`linux`ã€`windows`。默认：无é™åˆ¶ | linux | + +### questions.yml + +`questions.yml` 中大部分是å‘最终用户æ出的问题，但也有一部分å¯ä»¥åœ¨æ­¤æ–‡ä»¶ä¸­è®¾ç½®å­—段。 + +### 最低/最高 Rancher 版本 + +ä½ å¯ä»¥ä¸ºæ¯ä¸ª Chart 添加最低和/或最高的 Rancher 版本，这决定了该 Chart 是å¦å¯ä»¥ä»Ž Rancher 部署。 + +:::note + +Rancher 版本带有 `v` å‰ç¼€ï¼Œä½†æ˜¯ä½¿ç”¨æ­¤é€‰é¡¹æ—¶è¯·*ä¸è¦*包括å‰ç¼€ã€‚ + +::: + +``` +rancher_min_version: 2.3.0 +rancher_max_version: 2.3.99 +``` + +### Question å˜é‡å‚考 + +æ­¤å‚考包å«å¯ä»¥åµŒå¥—在 `questions:` 下的 `questions.yml` 中的å˜é‡ï¼š + +| å˜é‡ | 类型 | å¿…å¡« | æè¿° | +| ------------- | ------------- | --- |------------- | +| variable | string | true | 定义 `values.yml` 文件中指定的å˜é‡å，嵌套对象使用 `foo.bar`。 | +| label | string | true | 定义 UI 标签。 | +| description | string | false | 指定å˜é‡çš„æ述。 | +| type | string | false | 如果未指定，则默认为 `string`（支æŒçš„类型为 stringã€multilineã€booleanã€intã€enumã€passwordã€storageclassã€hostnameã€pvc å’Œ secret）。 | +| required | bool | false | 定义å˜é‡æ˜¯å¦æ˜¯å¿…须的（true \| false）。 | +| default | string | false | 指定默认值。仅在 `values.yml` 文件中没有对应值时使用。 | +| group | string | false | 按输入值对问题进行分组。 | +| min_length | int | false | 最å°å­—符长度。 | +| max_length | int | false | 最大字符长度。 | +| min | int | false | 最å°æ•´æ•°é•¿åº¦ã€‚ | +| max | int | false | 最大整数长度。 | +| options | []string | false | 为 `enum` 类型的å˜é‡æŒ‡å®šé€‰é¡¹ï¼Œä¾‹å¦‚：options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer" | +| valid_chars | string | false | 输入字符验è¯çš„正则表达å¼ã€‚ | +| invalid_chars | string | false | 无效输入字符验è¯çš„正则表达å¼ã€‚ | +| subquestions | []subquestion | false | 添加一组å­é—®é¢˜ã€‚ | +| show_if | string | false | 如果æ¡ä»¶å˜é‡ä¸º true，则显示当å‰å˜é‡ã€‚例如 `show_if: "serviceType=Nodeport"` | +| show\_subquestion_if | string | false | 如果为 true 或等于æŸä¸ªé€‰é¡¹ï¼Œåˆ™æ˜¾ç¤ºå­é—®é¢˜ã€‚例如 `show_subquestion_if: "true"` | + +:::note + +`subquestions[]` ä¸èƒ½åŒ…å« `subquestions` 或 `show_subquestions_if` 键，但支æŒä¸Šè¡¨ä¸­çš„所有其他键。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md new file mode 100644 index 00000000000..6486e4fc4ca --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md @@ -0,0 +1,175 @@ +--- +title: 设置 Amazon NLB 网络负载å‡è¡¡å™¨ +--- + +本文介ç»äº†å¦‚何在 Amazon EC2 æœåŠ¡ä¸­è®¾ç½® Amazon NLB 网络负载å‡è¡¡å™¨ï¼Œç”¨äºŽå°†æµé‡è½¬å‘到 EC2 上的多个实例中。 + +这些示例中，负载å‡è¡¡å™¨å°†æµé‡è½¬å‘到三个 Rancher Server 节点。如果 Rancher 安装在 RKE Kubernetes 集群上，则需è¦ä¸‰ä¸ªèŠ‚点。如果 Rancher 安装在 K3s Kubernetes 集群上，则åªéœ€è¦ä¸¤ä¸ªèŠ‚点。 + +本文介ç»çš„åªæ˜¯é…置负载å‡è¡¡å™¨çš„其中一ç§æ–¹å¼ã€‚其他负载å‡è¡¡å™¨å¦‚传统负载路由器（Classic Load Balancer）和应用负载路由器（Application Load Balancer），也å¯ä»¥å°†æµé‡è½¬å‘到 Rancher Server 节点。 + +Rancher 仅支æŒä½¿ç”¨ Amazon NLB 以 `TCP` 模å¼ç»ˆæ­¢ 443 端å£çš„æµé‡ï¼Œè€Œä¸æ”¯æŒ `TLS` 模å¼ã€‚这试因为在 NLB 终止时，NLB ä¸ä¼šå°†æ­£ç¡®çš„标头注入请求中。如果你想使用由 Amazon Certificate Manager (ACM) 托管的è¯ä¹¦ï¼Œè¯·ä½¿ç”¨ ALB。 + + + +## è¦æ±‚ + +你已在 EC2 中创建了 Linux 实例。此外，负载å‡è¡¡å™¨ä¼šæŠŠæµé‡è½¬å‘到这些节点。 + +## 1. 创建目标组 + +首先，为 **TCP** å议创建两个目标组，其中一个使用 TCP ç«¯å£ 443，å¦ä¸€ä¸ªä½¿ç”¨ TCP ç«¯å£ 80（用于é‡å®šå‘到 TCP ç«¯å£ 443）。然åŽï¼Œå°† Linux 节点添加到这些组中。 + +é…ç½® NLB 的第一个步骤是创建两个目标组。一般æ¥è¯´ï¼Œåªéœ€è¦ç«¯å£ 443 å°±å¯ä»¥è®¿é—® Rancherã€‚ä½†æ˜¯ï¼Œç”±äºŽç«¯å£ 80 çš„æµé‡ä¼šè¢«è‡ªåŠ¨é‡å®šå‘åˆ°ç«¯å£ 443，因此，你也å¯ä»¥ä¸ºç«¯å£ 80 也添加一个监å¬å™¨ã€‚ + +ä¸ç®¡ä½¿ç”¨çš„是 NGINX Ingress 还是 Traefik Ingress Controller，Ingress éƒ½åº”è¯¥å°†ç«¯å£ 80 çš„æµé‡é‡å®šå‘åˆ°ç«¯å£ 443。以下为æ“作步骤： + +1. 登录到 [Amazon AWS 控制å°](https://console.aws.amazon.com/ec2/)。确ä¿é€‰æ‹©çš„**区域**是你创建 EC2 实例 （Linux 节点）的区域。 +1. 选择**æœåŠ¡** > **EC2**，找到**è´Ÿè½½å‡è¡¡å™¨**并打开**目标组**。 +1. å•å‡»**创建目标组**，然åŽåˆ›å»ºç”¨äºŽ TCP ç«¯å£ 443 的第一个目标组。 + +:::note + +ä¸åŒ Ingress çš„å¥åº·æ£€æŸ¥å¤„ç†æ–¹æ³•ä¸åŒã€‚详情请å‚阅[本节](#nginx-ingress-å’Œ-traefik-ingress-çš„å¥åº·æ£€æŸ¥è·¯å¾„)。 + +::: + +#### 目标组（TCP ç«¯å£ 443） + +æ ¹æ®ä¸‹è¡¨é…置第一个目标组： + +| 选项 | 设置 | +|-------------------|-------------------| +| 目标组å称 | `rancher-tcp-443` | +| 目标类型 | `instance` | +| åè®® | `TCP` | +| ç«¯å£ | `443` | +| VPC | 选择 VPC | + +å¥åº·æ£€æŸ¥è®¾ç½®ï¼š + +| 选项 | 设置 | +|---------------------|-----------------| +| åè®® | TCP | +| ç«¯å£ | `override`,`80` | +| å¥åº·é˜ˆå€¼ | `3` | +| ä¸æ­£å¸¸é˜ˆå€¼ | `3` | +| 超时 | `6 秒` | +| é—´éš” | `10 秒` | + +å•å‡»**创建目标组**，然åŽåˆ›å»ºç”¨äºŽ TCP ç«¯å£ 80 的第二个目标组。 + +#### 目标组（TCP ç«¯å£ 80） + +æ ¹æ®ä¸‹è¡¨é…置第二个目标组： + +| 选项 | 设置 | +|-------------------|------------------| +| 目标组å称 | `rancher-tcp-80` | +| 目标类型 | `instance` | +| åè®® | `TCP` | +| ç«¯å£ | `80` | +| VPC | 选择 VPC | + + +å¥åº·æ£€æŸ¥è®¾ç½®ï¼š + +| 选项 | 设置 | +|---------------------|----------------| +| åè®® | TCP | +| ç«¯å£ | `traffic port` | +| å¥åº·é˜ˆå€¼ | `3` | +| ä¸æ­£å¸¸é˜ˆå€¼ | `3` | +| 超时 | `6 秒` | +| é—´éš” | `10 秒` | + +## 2. 注册目标 + +接下æ¥ï¼Œå°† Linux 节点添加到两个目标组中。 + +选择å为 **rancher-tcp-443** 的目标组，点击**目标**选项å¡å¹¶é€‰æ‹©**编辑**。 + +![](/img/ha/nlb/edit-targetgroup-443.png) + +选择你è¦æ·»åŠ çš„实例（Linux 节点），然åŽå•å‡»**添加到已注册**。 + +*** +**将目标添加到目标组 TCP ç«¯å£ 443**
+ +![](/img/ha/nlb/add-targets-targetgroup-443.png) + +*** +**已将目标添加到目标组 TCP ç«¯å£ 443**
+ +![](/img/ha/nlb/added-targets-targetgroup-443.png) + +添加实例åŽï¼Œå•å‡»å³ä¸‹æ–¹çš„**ä¿å­˜**。 + +å°† **rancher-tcp-443** 替æ¢ä¸º **rancher-tcp-80**，然åŽé‡å¤ä¸Šè¿°æ­¥éª¤ã€‚你需è¦å°†ç›¸åŒçš„实例作为目标添加到此目标组。 + +## 3. 创建 NLB + +使用 Amazon çš„å‘导创建网络负载å‡è¡¡å™¨ã€‚在这个过程中，你需è¦æ·»åŠ åœ¨ [1. 创建目标组](#1-创建目标组)中创建的目标组。 + +1. 在网页æµè§ˆå™¨ä¸­ï¼Œå¯¼èˆªåˆ° [Amazon EC2 控制å°](https://console.aws.amazon.com/ec2/)。 + +2. 在导航æ ä¸­ï¼Œé€‰æ‹©**è´Ÿè½½å‡è¡¡** > **è´Ÿè½½å‡è¡¡å™¨**。 + +3. å•å‡»**创建负载å‡è¡¡å™¨**。 + +4. 选择**网络负载å‡è¡¡å™¨**并å•å‡»**创建**。然åŽï¼Œå¡«å†™æ¯ä¸ªè¡¨æ ¼ã€‚ + +- [步骤 1：é…置负载å‡è¡¡å™¨](#步骤-1é…置负载å‡è¡¡å™¨) +- [步骤 2：é…置路由](#步骤-2é…置路由) +- [步骤 3：注册目标](#步骤-3注册目标) +- [步骤 4：审核](#步骤-4审核) + +### 步骤 1：é…置负载å‡è¡¡å™¨ + +在表å•ä¸­è®¾ç½®ä»¥ä¸‹å­—段： + +- **å称**：`rancher` +- **Scheme**：`internal` 或 `internet-facing`。实例和 VPC çš„é…置决定了 NLB çš„ Scheme。如果你的实例没有绑定公共 IP，或者你åªéœ€è¦é€šè¿‡å†…网访问 Rancher，请将 NLB çš„ Scheme 设置为 `internal` 而ä¸æ˜¯ `internet-facing`。 +- **监å¬å™¨**：负载å‡è¡¡å™¨å议需è¦æ˜¯ `TCP`，而且负载å‡è¡¡å™¨ç«¯å£éœ€è¦è®¾ä¸º `443`。 +- **å¯ç”¨åŒºï¼š**选择你的**VPC**å’Œ**å¯ç”¨åŒº**。 + +### 步骤 2：é…置路由 + +1. 从**目标组**下拉列表中，选择 **现有目标组**。 +1. 从**å称**下拉列表中，选择 `rancher-tcp-443`。 +1. 打开**高级å¥åº·æ£€æŸ¥è®¾ç½®**，并将**é—´éš”**设为 `10 秒`。 + +### 步骤 3：注册目标 + +由于你已ç»åœ¨å…ˆå‰æ­¥éª¤æ³¨å†Œäº†ç›®æ ‡ï¼Œå› æ­¤ä½ åªéœ€å•å‡» **下一步：审核**。 + +### 步骤 4：审核 + +检查负载å‡è¡¡å™¨ä¿¡æ¯æ— è¯¯åŽï¼Œå•å‡»**创建**。 + +AWS å®Œæˆ NLB 创建åŽï¼Œå•å‡»**关闭**。 + +## 4. 为 TCP ç«¯å£ 80 å‘ NLB 添加监å¬å™¨ + +1. 选择新创建的 NLB 并选择**监å¬å™¨**选项å¡ã€‚ + +2. å•å‡»**添加监å¬å™¨**。 + +3. 使用 `TCP`:`80` 作为**åè®®**:**端å£**。 + +4. å•å‡»**添加æ“作**并选择**转å‘到..**。 + +5. 从**转å‘到**下拉列表中，选择 `rancher-tcp-80`。 + +6. å•å‡»å³ä¸Šè§’çš„**ä¿å­˜**。 + +## NGINX Ingress å’Œ Traefik Ingress çš„å¥åº·æ£€æŸ¥è·¯å¾„ + +K3s å’Œ RKE Kubernetes 集群使用的默认 Ingress ä¸åŒï¼Œå› æ­¤å¯¹åº”çš„å¥åº·æ£€æŸ¥æ–¹å¼ä¹Ÿä¸åŒã€‚ + +RKE Kubernetes 集群默认使用 NGINX Ingress，而 K3s Kubernetes 集群默认使用 Traefik Ingress。 + +- **Traefik**：默认å¥åº·æ£€æŸ¥è·¯å¾„是 `/ping`。默认情况下，ä¸ç®¡ä¸»æœºå¦‚何，`/ping` 总是匹é…，而且 [Traefik 自身](https://docs.traefik.io/operations/ping/)总会å“应。 +- **NGINX Ingress**：NGINX Ingress Controller 的默认åŽç«¯æœ‰ä¸€ä¸ª `/healthz` 端点。默认情况下，ä¸ç®¡ä¸»æœºå¦‚何，`/healthz` 总是匹é…，而且 [`ingress-nginx` 自身](https://github.com/kubernetes/ingress-nginx/blob/0cbe783f43a9313c9c26136e888324b1ee91a72f/charts/ingress-nginx/values.yaml#L212)总会å“应。 + +想è¦ç²¾ç¡®æ¨¡æ‹Ÿå¥åº·æ£€æŸ¥ï¼Œæœ€å¥½æ˜¯ä½¿ç”¨ Host 标头（Rancher hostname）加上 `/ping` 或 `/healthz`（分别对应 K3s å’Œ RKE 集群）æ¥èŽ·å– Rancher Pod çš„å“应，而ä¸æ˜¯ Ingress çš„å“应。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md new file mode 100644 index 00000000000..277961c5dae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md @@ -0,0 +1,73 @@ +--- +title: 为高å¯ç”¨ K3s Kubernetes 集群设置基础设施 +--- + +本教程旨在帮助你为 Rancher Management Server é…置底层基础设施。 + +æˆ‘ä»¬æ ¹æ® Rancher 的安装ä½ç½®ï¼ˆK3s Kubernetes 集群ã€RKE Kubernetes 集群或å•ä¸ª Docker 容器）为专用于 Rancher çš„ Kubernetes 集群推èä¸åŒåŸºç¡€è®¾æ–½ã€‚ + +有关æ¯ä¸ªå®‰è£…选项的详情，请å‚è§[本页](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +:::note é‡è¦æ示： + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºï¼ˆæ•°æ®ä¸­å¿ƒï¼‰ã€‚ + +::: + +如需在高å¯ç”¨ K3s 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **2 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商中的虚拟机。 +- **1 个外部数æ®åº“**：用于存储集群数æ®ã€‚建议使用 MySQL。 +- **1 个负载å‡è¡¡å™¨**：用于将æµé‡è½¬å‘到这两个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +### 1. é…ç½® Linux 节点 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置外部数æ®åº“ + +K3s 与其他 Kubernetes å‘行版ä¸åŒï¼Œåœ¨äºŽå…¶æ”¯æŒä½¿ç”¨ etcd 以外的数æ®åº“æ¥è¿è¡Œ Kubernetes。该功能让 Kubernetes è¿ç»´æ›´åŠ çµæ´»ã€‚ä½ å¯ä»¥æ ¹æ®å®žé™…情况选择åˆé€‚çš„æ•°æ®åº“。 + +对于 K3s 高å¯ç”¨å®‰è£…，你需è¦é…置一个 [MySQL](https://www.mysql.com/) 外部数æ®åº“。Rancher 已在使用 MySQL 5.7 作为数æ®å­˜å‚¨çš„ K3s Kubernetes 集群上进行了测试。 + +在使用 K3s 安装脚本安装 Kubernetes 时，你需è¦ä¼ å…¥ K3s 连接数æ®åº“的详细信æ¯ã€‚ + +如需获å–é…ç½® MySQL æ•°æ®åº“示例，请å‚è§[在 Amazon RDS æœåŠ¡ä¸­é…ç½® MySQL](mysql-database-in-amazon-rds.md) 的教程。 + +如需获å–é…ç½® K3s 集群数æ®åº“的所有å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/datastore/)。 + +### 3. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，K3s 工具会部署一个 Traefik Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 Traefik Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚Traefik Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 4. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md new file mode 100644 index 00000000000..24387c1f786 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md @@ -0,0 +1,64 @@ +--- +title: 为高å¯ç”¨ RKE Kubernetes 集群设置基础设施 +--- + +本教程旨在帮助你创建一个高å¯ç”¨çš„ RKE 集群，用于安装 Rancher Server。 + +:::note é‡è¦æ示： + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºï¼ˆæ•°æ®ä¸­å¿ƒï¼‰ã€‚ + +::: + +如需在高å¯ç”¨ RKE 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商（例如 Amazon EC2，GCE 或 vSphere）中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将å‰ç«¯æµé‡è½¬å‘到这三个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸæˆ–æ•°æ®ä¸­å¿ƒã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºã€‚ + +### 为什么使用三个节点？ + +在 RKE 集群中，Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。而这个 etcd æ•°æ®åº“在这三个节点上è¿è¡Œã€‚ + +为了选举出大多数 etcd 节点认å¯çš„ etcd 集群 leader，etcd æ•°æ®åº“需è¦å¥‡æ•°ä¸ªèŠ‚点。如果 etcd æ•°æ®åº“无法选出 leader，etcd å¯èƒ½ä¼šå‡ºçŽ°[脑裂（split brain）](https://www.quora.com/What-is-split-brain-in-distributed-systems)的问题，此时你需è¦ä½¿ç”¨å¤‡ä»½æ¢å¤é›†ç¾¤ã€‚如果三个 etcd 节点之一å‘生故障，其余两个节点å¯ä»¥é€‰æ‹©ä¸€ä¸ª leader，因为它们是 etcd 节点总数的大多数部分。 + +### 1. é…ç½® Linux 节点 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到三个节点中的任æ„一个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 3. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md new file mode 100644 index 00000000000..0af8f17b3bb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md @@ -0,0 +1,58 @@ +--- +title: 为高å¯ç”¨ RKE2 Kubernetes 集群设置基础设施 +--- + +本教程旨在帮助你为 Rancher Management Server é…置底层基础设施。 + +æˆ‘ä»¬æ ¹æ® Rancher 的安装ä½ç½®ï¼ˆRKE2 Kubernetes 集群ã€RKE Kubernetes 集群或å•ä¸ª Docker 容器）为专用于 Rancher çš„ Kubernetes 集群推èä¸åŒåŸºç¡€è®¾æ–½ã€‚ + +:::note é‡è¦æ示： + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºï¼ˆæ•°æ®ä¸­å¿ƒï¼‰ã€‚ + +::: + +如需在高å¯ç”¨ RKE2 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将æµé‡è½¬å‘到这两个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +### 1. é…ç½® Linux 节点 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到所有节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE2 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 4. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md new file mode 100644 index 00000000000..d312a5f6207 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md @@ -0,0 +1,34 @@ +--- +title: 在 Amazon RDS 中创建 MySQL æ•°æ®åº“ +--- + +本教程介ç»å¦‚何在 Amazon Relational Database Service (RDS) 中创建 MySQL æ•°æ®åº“。 + +该数æ®åº“å¯ç”¨ä½œé«˜å¯ç”¨ K3s Kubernetes 集群的外部数æ®å­˜å‚¨ã€‚ + +1. 登录到 [Amazon AWS RDS 控制å°](https://console.aws.amazon.com/rds/)。确ä¿é€‰æ‹©çš„**区域**是你创建 EC2 实例 （Linux 节点）的区域。 +1. 在左侧é¢æ¿ä¸­ï¼Œç‚¹å‡»**æ•°æ®åº“**。 +1. 点击**创建数æ®åº“**。 +1. 在**引擎类型**中，点击 **MySQL**。 +1. 在**版本**中，选择 **MySQL 5.7.22**。 +1. 在**设置**部分的**凭è¯è®¾ç½®**下，输入 **admin** 主用户å的密ç ã€‚确认密ç ã€‚ +1. 展开**其它é…ç½®**。在**åˆå§‹æ•°æ®åº“å称**字段，设置数æ®åº“å称。该å称仅å¯åŒ…å«å­—æ¯ï¼Œæ•°å­—和下划线。这个å称会用于连接到数æ®åº“。 +1. 点击**创建数æ®åº“**。 + +你需è¦èŽ·å–æ–°æ•°æ®åº“的以下信æ¯ï¼Œä»¥ä¾¿ K3s Kubernetes 集群å¯ä»¥è¿žæŽ¥åˆ°è¯¥æ•°æ®åº“。 + +如需在 Amazon RDS 控制å°æŸ¥çœ‹æ­¤ä¿¡æ¯ï¼Œç‚¹å‡»**æ•°æ®åº“**，然åŽç‚¹å‡»ä½ åˆ›å»ºçš„æ•°æ®åº“çš„å称。 + +- **用户å**：使用 admin 用户å。 +- **密ç **：使用 admin 密ç ã€‚ +- **主机å**：使用**端点**作为主机å。端点å¯ä»¥åœ¨**连接性和安全性**部分找到。 +- **端å£**：默认为 3306。你å¯ä»¥åœ¨**连接性和安全性**处确认端å£ã€‚ +- **æ•°æ®åº“å称**：å‰å¾€**é…ç½®**选项å¡ç¡®è®¤æ•°æ®åº“å称。å称会在**æ•°æ®åº“å称**中列出。 + +以下方格å¼ä½¿ç”¨ä¸Šè¿°ä¿¡æ¯ï¼Œè¿žæŽ¥åˆ°æ•°æ®åº“： + +``` +mysql://username:password@tcp(hostname:3306)/database-name +``` + +有关为 K3s é…置数æ®åº“的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/datastore/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md new file mode 100644 index 00000000000..b686c28221a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md @@ -0,0 +1,102 @@ +--- +title: é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨ +--- + +å°† NGINX é…置为四层负载å‡è¡¡å™¨ï¼ˆTCP），用于将连接转å‘到 Rancher 节点。 + +在此é…置中，负载å‡è¡¡å™¨ä½äºŽèŠ‚点的å‰é¢ã€‚è´Ÿè½½å‡è¡¡å™¨å¯ä»¥æ˜¯ä»»ä½•èƒ½è¿è¡Œ NGINX 的主机。 + +:::note + +ä¸è¦ä½¿ç”¨ Rancher 节点作为负载å‡è¡¡å™¨ã€‚ + +::: + +> 这些示例中，负载å‡è¡¡å™¨å°†æµé‡è½¬å‘到三个 Rancher Server 节点。如果 Rancher 安装在 RKE Kubernetes 集群上，则需è¦ä¸‰ä¸ªèŠ‚点。如果 Rancher 安装在 K3s Kubernetes 集群上，则åªéœ€è¦ä¸¤ä¸ªèŠ‚点。 + +## 安装 NGINX + +首先，在è¦ç”¨ä½œè´Ÿè½½å‡è¡¡å™¨çš„节点上安装 NGINX。NGINX 有适用于所有已知æ“作系统的软件包。已测试的版本为 `1.14` å’Œ `1.15`。如需获得安装 NGINX 的帮助，请å‚è§[安装文档](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/)。 + +`stream` 模å—是必需的，NGINX 官方安装包包å«äº†è¯¥æ¨¡å—。请å‚è§ä½ æ“作系统的文档，了解如何在æ“作系统上安装和å¯ç”¨ NGINX çš„ `stream` 模å—。 + +## 创建 NGINX é…ç½® + +安装 NGINX åŽï¼Œä½¿ç”¨èŠ‚点的 IP 地å€æ›´æ–° NGINX é…置文件 `nginx.conf`。 + +1. 将以下的示例代ç å¤åˆ¶å¹¶ç²˜è´´åˆ°ä½ ä½¿ç”¨çš„文本编辑器中。将文件ä¿å­˜ä¸º `nginx.conf`。 + +2. 在 `nginx.conf` ä¸­ï¼Œå°†æ‰€æœ‰ï¼ˆç«¯å£ 80 å’Œç«¯å£ 443）的 ``，``å’Œ `` 替æ¢ä¸ºä½ èŠ‚点的 IP 地å€ã€‚ + + :::note + + å‚è§ [NGINX 文档：TCP å’Œ UDP è´Ÿè½½å‡è¡¡](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/)了解所有é…置选项。 + + ::: + +
示例 NGINX é…ç½®
+ + ``` + worker_processes 4; + worker_rlimit_nofile 40000; + + events { + worker_connections 8192; + } + + stream { + upstream rancher_servers_http { + least_conn; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + } + server { + listen 80; + proxy_pass rancher_servers_http; + } + + } + + http { + + upstream rancher_servers_https { + least_conn; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + } + server { + listen 443 ssl; + proxy_pass rancher_servers_https; + ssl_certificate /path/to/tls.crt; + ssl_certificate_key /path/to/key.key; + location / { + proxy_pass https://rancher_servers_https; + proxy_set_header Host ; + proxy_ssl_server_name on; + proxy_ssl_name + } + } + } + ``` + + +3. å°† `nginx.conf` ä¿å­˜åˆ°ä½ çš„è´Ÿè½½å‡è¡¡å™¨çš„ `/etc/nginx/nginx.conf` 路径上。 + +4. è¿è¡Œä»¥ä¸‹å‘½ä»¤é‡æ–°åŠ è½½ NGINX é…置： + + ``` + # nginx -s reload + ``` + +## å¯é€‰ - å°† NGINX 作为 Docker 容器è¿è¡Œ + +除了将 NGINX 作为软件包安装在æ“作系统上外，你也å¯ä»¥å°†å…¶ä½œä¸º Docker 容器è¿è¡Œã€‚将编辑åŽçš„ **NGINX é…置示例** ä¿å­˜ä¸º`/etc/nginx.conf`，并è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å¯åŠ¨ NGINX 容器： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/nginx.conf:/etc/nginx/nginx.conf \ + nginx:1.14 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md new file mode 100644 index 00000000000..21294b1016e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md @@ -0,0 +1,74 @@ +--- +title: 在 Amazon EC2 中é…置节点 +--- + +在本教程中，你将学习一ç§ä¸º Rancher Mangement Server 创建 Linux 节点的方法。这些节点将满足[æ“作系统ã€Dockerã€ç¡¬ä»¶å’Œç½‘络的è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)。 + +如果 Rancher Server 安装在 RKE Kubernetes 集群上，你需è¦é…置三个实例。 + +如果 Rancher Server 安装在 K3s Kubernetes 集群上，你åªéœ€è¦é…置两个实例。 + +如果 Rancher Server 安装在å•ä¸ª Docker 容器中，你åªéœ€è¦é…置一个实例。 + +### 1. 准备工作（å¯é€‰ï¼‰ + +- **创建 IAM 角色**：è¦å…许 Rancher æ“作 AWS 资æºï¼Œä¾‹å¦‚创建新存储或新节点，你需è¦å°† Amazon é…置为云æ供商。è¦åœ¨ EC2 上设置云æ供商，你需è¦è¿›è¡Œå‡ ä¸ªæ“作，其中包括为 Rancher Server 节点设置 IAM 角色。有关设置云æ供商的详情，请å‚è§[本页](../../../pages-for-subheaders/set-up-cloud-providers.md)。 +- **创建安全组**：我们建议为 Rancher èŠ‚ç‚¹è®¾ç½®ä¸€ä¸ªç¬¦åˆ [Rancher 节点端å£è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md#端å£è¦æ±‚)的安全组。 + +### 2. é…置实例 + +1. 登录到 [Amazon AWS EC2 控制å°](https://console.aws.amazon.com/ec2/)。由于 Rancher Management Server 的所有基础设施都需è¦ä½äºŽåŒä¸€åŒºåŸŸï¼Œå› æ­¤ï¼Œè¯·åŠ¡å¿…记下创建 EC2 实例（Linux 节点）的**区域**。 +1. 在左侧é¢æ¿ä¸­ï¼Œç‚¹å‡»**实例**。 +1. 点击**å¯åŠ¨ç¤ºä¾‹**。 +1. 在**步骤 1：选择 Amazon Machine Image (AMI)** 中，使用 `ami-0d1cd67c26f5fca19 (64-bit x86)` æ¥ä½¿ç”¨ Ubuntu 18.04 作为 Linux æ“作系统。去到 Ubuntu AMI 并点击**选择**。 +1. 在**步骤 2：选择实例类型**中，选择 `t2.medium`。 +1. 点击**下一步：é…置实例详细信æ¯**。 +1. 在**实例数é‡**字段中，输入实例数é‡ã€‚创建高å¯ç”¨ K3s 集群仅需è¦ä¸¤ä¸ªå®žä¾‹ï¼Œè€Œé«˜å¯ç”¨ RKE 集群则需è¦ä¸‰ä¸ªå®žä¾‹ã€‚ +1. å¯é€‰ï¼šå¦‚果你为 Rancher 创建了一个 IAM 角色æ¥æ“作 AWS 资æºï¼Œè¯·åœ¨ **IAM 角色**字段中选择新 IAM 角色。 +1. 分别点击**下一步：添加存储**，**下一步：添加标签**å’Œ**下一步：é…置安全组**。 +1. 在**步骤 6：é…置安全组**ä¸­ï¼Œé€‰æ‹©ä¸€ä¸ªç¬¦åˆ Rancher 节点[端å£è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md#端å£è¦æ±‚)的安全组。 +1. 点击**查看并å¯åŠ¨**。 +1. 点击**å¯åŠ¨**。 +1. 选择一个新的或现有的密钥对，用于之åŽè¿žæŽ¥åˆ°ä½ çš„实例。如果使用现有密钥对，请确ä¿ä½ æœ‰è®¿é—®ç§é’¥çš„æƒé™ã€‚ +1. 点击**å¯åŠ¨å®žä¾‹**。 + + +**结果**：你已创建满足æ“作系统ã€ç¡¬ä»¶å’Œç½‘络è¦æ±‚çš„ Rancher 节点。 + +:::note + +如果节点用于 RKE Kubernetes 集群，请在下一步æ“作中为æ¯ä¸ªèŠ‚点安装 Docker 。如果节点用于 K3s Kubernetes 集群，你å¯ä»¥å¼€å§‹åœ¨èŠ‚点上安装 K3s 了。 + +::: + +### 3. 为 RKE Kubernetes 集群节点安装 Docker 并创建用户 + +1. 在 [AWS EC2 控制å°](https://console.aws.amazon.com/ec2/)中，点击左侧é¢æ¿ä¸­çš„**实例**。 +1. 转到你想è¦å®‰è£… Docker 的实例。选择实例，并点击**æ“作 > 连接**。 +1. 按照å±å¹•ä¸Šçš„说明连接到实例。å¤åˆ¶å®žä¾‹çš„公共 DNS。SSH 进入实例的示例命令如下： +``` +sudo ssh -i [path-to-private-key] ubuntu@[public-DNS-of-instance] +``` +1. 在实例上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä½¿ç”¨ Rancher 的其中一个安装脚本æ¥å®‰è£… Docker： +``` +curl https://releases.rancher.com/install-docker/18.09.sh | sh +``` +1. 连接到实例åŽï¼Œåœ¨å®žä¾‹ä¸Šè¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ›å»ºç”¨æˆ·ï¼š +``` +sudo usermod -aG docker ubuntu +``` +1. 在æ¯ä¸ªèŠ‚点上é‡å¤ä¸Šè¿°æ­¥éª¤ï¼Œä»¥ç¡®ä¿ Docker 安装到æ¯ä¸ªç”¨äºŽè¿è¡Œ Rancher Management Server 的节点上。 + +:::tip + +è¦äº†è§£æˆ‘们是å¦æ供指定的 Docker 版本的安装脚本，请访问此 [GitHub 仓库](https://github.com/rancher/install-docker)ï¼Œè¯¥ä»“åº“åŒ…å« Rancher 的所有 Docker 安装脚本。 + +::: + +**结果**：你已é…置满足æ“作系统ã€Dockerã€ç¡¬ä»¶å’Œç½‘络è¦æ±‚çš„ Rancher Server 节点。 + +### RKE Kubernetes 集群节点的åŽç»­æ­¥éª¤ + +如需在新节点上安装 RKE 集群，请记ä½æ¯ä¸ªèŠ‚点的 **IPv4 公共 IP** å’Œ **ç§æœ‰ IP**。创建节点åŽï¼Œæ­¤ä¿¡æ¯å¯ä»¥åœ¨æ¯ä¸ªèŠ‚点的**æè¿°**选项å¡ä¸­æ‰¾åˆ°ã€‚公共和ç§æœ‰ IP 将用于设置 RKE 集群é…置文件 `rancher-cluster.yml` 中æ¯ä¸ªèŠ‚点的 `address` å’Œ `internal_address`。 + +RKE 还需è¦è®¿é—®ç§é’¥æ‰èƒ½è¿žæŽ¥åˆ°æ¯ä¸ªèŠ‚点。因此，请记ä½è¿žæŽ¥åˆ°èŠ‚点的ç§é’¥çš„路径，该路径也å¯ç”¨äºŽè®¾ç½® `rancher-cluster.yml` 中æ¯ä¸ªèŠ‚点的 `ssh_key_path`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md new file mode 100644 index 00000000000..2f28274e31b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md @@ -0,0 +1,25 @@ +--- +title: 高å¯ç”¨å®‰è£… +--- + +我们建议使用 Helm（ Kubernetes 包管ç†å™¨ ）在专用的 Kubernetes 集群上安装 Rancher。由于 Rancher è¿è¡Œåœ¨å¤šä¸ªèŠ‚点上æå‡äº†å¯ç”¨æ€§ï¼Œå› æ­¤è¿™ç§å®‰è£…æ–¹å¼å«åšé«˜å¯ç”¨ Kubernetes 安装。 + +在标准安装中，先将 Kubernetes 安装在托管在云æ供商（例如 Amazon çš„ EC2 或 Google Compute Engine）中的三个节点上。 + +然åŽä½¿ç”¨ Helm 在 Kubernetes 集群上安装 Rancher。Helm 使用 Rancher çš„ Helm Chart 在 Kubernetes 集群的三个节点中å‡å®‰è£… Rancher 的副本。我们建议使用负载å‡è¡¡å™¨å°†æµé‡è½¬å‘到集群中的æ¯ä¸ª Rancher 副本中，以æ高 Rancher çš„å¯ç”¨æ€§ã€‚ + +Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。etcd æ•°æ®åº“å¯ä»¥åœ¨æ‰€æœ‰ä¸‰ä¸ªèŠ‚点上è¿è¡Œã€‚为了选举出大多数 etcd 节点认åŒçš„ etcd 集群 leader，节点的数é‡éœ€è¦æ˜¯å¥‡æ•°ã€‚如果 etcd æ•°æ®åº“ä¸èƒ½é€‰å‡º leader，etcd å¯èƒ½ä¼šå¤±è´¥ã€‚这时候就需è¦ä½¿ç”¨å¤‡ä»½æ¥è¿˜åŽŸé›†ç¾¤ã€‚ + +有关 Rancher 如何工作的详情（与安装方法无关），请å‚è§[架构](../../../pages-for-subheaders/rancher-manager-architecture.md)。 + +### 推è架构 + +- Rancher çš„ DNS 应该解æžä¸º 4 层负载å‡è¡¡å™¨ã€‚ +- è´Ÿè½½å‡è¡¡å™¨åº”该把 TCP/80 端å£å’Œ TCP/443 端å£çš„æµé‡è½¬å‘到 Kubernetes 集群的全部 3 个节点上。 +- Ingress Controller 会把 HTTP é‡å®šå‘到 HTTPS，在 TCP/443 端å£ç»ˆç»“ SSL/TLS。 +- Ingress Controller 会把æµé‡è½¬å‘到 Rancher deployment çš„ Pod 上的 TCP/80 端å£ã€‚ + +
使用 4 层负载å‡è¡¡å™¨åœ¨ Kubernetes 集群中安装 Rancher：Ingress Controller çš„ SSL 终止：
+ +![Rancher 的高å¯ç”¨ Kubernetes 安装](/img/ha/rancher2ha.svg) +使用 4 层负载å‡è¡¡å™¨åœ¨ Kubernetes 集群中安装 Rancher：Ingress Controller çš„ SSL 终止 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md new file mode 100644 index 00000000000..4d8ca8ac24c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md @@ -0,0 +1,136 @@ +--- +title: 为 Rancher 设置高å¯ç”¨ K3s Kubernetes 集群 +--- + +本文介ç»äº†å¦‚ä½•æ ¹æ® [Rancher Server 环境的最佳实践](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#kubernetes-安装环境)安装 Kubernetes 集群。 + +如果你的系统无法直接访问互è”网，请å‚è§ç¦»çº¿å®‰è£…说明。 + +:::tip å•èŠ‚点安装æ示： + +在å•èŠ‚点 Kubernetes 集群中，Rancher Server ä¸å…·å¤‡é«˜å¯ç”¨æ€§ï¼Œè€Œé«˜å¯ç”¨æ€§å¯¹åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­è¿è¡Œ Rancher éžå¸¸é‡è¦ã€‚但是，如果你想è¦çŸ­æœŸå†…使用å•èŠ‚点节çœèµ„æºï¼ŒåŒæ—¶åˆä¿ç•™é«˜å¯ç”¨æ€§è¿ç§»è·¯å¾„，把 Rancher 安装到å•èŠ‚点集群也是åˆé€‚的。 + +è¦é…ç½®å•èŠ‚点 K3s 集群，你åªéœ€è¦åœ¨å•ä¸ªèŠ‚点上è¿è¡Œ Rancher Server 安装命令（ä¸éœ€è¦åœ¨ä¸¤ä¸ªèŠ‚点上è¿è¡Œå‘½ä»¤ï¼‰ã€‚ + +在这两ç§å•èŠ‚点设置中，Rancher å¯ä»¥ä¸Ž Helm 一起安装在 Kubernetes 集群上，安装方法与安装到其他集群上一样。 + +::: + +## 先决æ¡ä»¶ + +以下说明å‡è®¾ä½ å·²å‚è§[此章节](../infrastructure-setup/ha-k3s-kubernetes-cluster.md)é…置好两个节点，一个负载å‡è¡¡å™¨ï¼Œä¸€ä¸ª DNS 记录和一个外部 MySQL æ•°æ®åº“。 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://rancher.com/support-maintenance-terms/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +## 安装 Kubernetes + +### 1. 安装 Kubernetes 并设置 K3s Server + +在è¿è¡Œå¯åŠ¨ K3s Kubernetes API Server 的命令时，你会传入使用之å‰è®¾ç½®çš„外部数æ®å­˜å‚¨çš„选项。 + +1. 连接到你准备用于è¿è¡Œ Rancher Server 的其中一个 Linux 节点。 +1. 在 Linux 节点上，è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å¯åŠ¨ K3s Server，并将其连接到外部数æ®å­˜å‚¨ã€‚ + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" + ``` + + 其中 `` 是数æ®å­˜å‚¨çš„连接 URI。例如，如果你使用的是 MySQL，则为 `mysql://username:password@tcp(hostname:3306)/database-name`。有效的数æ®å­˜å‚¨åŒ…括 etcdã€MySQLã€PostgreSQL 或 SQLite（默认）。 + + :::note + + 你也å¯ä»¥ä½¿ç”¨ `$K3S_DATASTORE_ENDPOINT` 环境å˜é‡æ¥ä¼ é€’æ•°æ®å­˜å‚¨ç«¯ç‚¹ã€‚ + + ::: + +1. 获å–主 Server 节点令牌： + ``` + cat /var/lib/rancher/k3s/server/token + ``` + +1. 在第二个 K3s Server 节点上è¿è¡Œå‘½ä»¤ï¼š + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" \ + --token "" + ``` + +### 2. 检查 K3s 是å¦æ­£å¸¸è¿è¡Œ + +在其中一个 K3s Server 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œæ¥ç¡®è®¤ K3s 是å¦å·²ç»è®¾ç½®æˆåŠŸï¼š +``` +sudo k3s kubectl get nodes +``` + +然åŽä½ ä¼šçœ‹åˆ°ä¸¤ä¸ªå…·æœ‰ master 角色的节点。 +``` +ubuntu@ip-172-31-60-194:~$ sudo k3s kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-172-31-60-194 Ready master 44m v1.17.2+k3s1 +ip-172-31-63-88 Ready master 6m8s v1.17.2+k3s1 +``` + +测试集群 Pod çš„å¥åº·çŠ¶å†µï¼š +``` +sudo k3s kubectl get pods --all-namespaces +``` + +**结果**：你已æˆåŠŸé…ç½® K3s Kubernetes 集群。 + +### 3. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 K3s 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/k3s/k3s.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 +2. å¤åˆ¶ `/etc/rancher/k3s/k3s.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhost。你需è¦å°† `server` é…置为负载å‡è¡¡å™¨çš„ DNSï¼Œå¹¶æŒ‡å®šç«¯å£ 6443ï¼ˆé€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server）。以下是一个 `k3s.yaml` 示例： + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ K3s 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/cluster-access/) 或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### 4. 检查集群 Pod çš„å¥åº·çŠ¶å†µ + +现在你已ç»è®¾ç½®äº† `kubeconfig` 文件。你å¯ä»¥ä½¿ç”¨ `kubectl` 在本地主机访问集群。 + +检查所有需è¦çš„ Pod 和容器是å¦å¥åº·ï¼š + +``` +ubuntu@ip-172-31-60-194:~$ sudo kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system metrics-server-6d684c7b5-bw59k 1/1 Running 0 8d +kube-system local-path-provisioner-58fb86bdfd-fmkvd 1/1 Running 0 8d +kube-system coredns-d798c9dd-ljjnf 1/1 Running 0 8d +``` + +**结果**：你å¯é€šè¿‡ä½¿ç”¨ `kubectl` 访问集群，并且 K3s 集群能æˆåŠŸè¿è¡Œã€‚现在，你å¯ä»¥åœ¨é›†ç¾¤ä¸Šå®‰è£… Rancher Management Server。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md new file mode 100644 index 00000000000..397fe2aa0d0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md @@ -0,0 +1,193 @@ +--- +title: é…置高å¯ç”¨çš„ RKE Kubernetes 集群 +--- + +本文介ç»å¦‚何安装 Kubernetes 集群。该集群应专用于è¿è¡Œ Rancher Server。 + +:::note + +Rancher å¯ä»¥è¿è¡Œåœ¨ä»»ä½• Kubernetes 集群上，包括托管的 Kubernetes，例如 Amazon EKS。以下说明åªæ˜¯å®‰è£… Kubernetes 其中一ç§æ–¹å¼ã€‚ + +::: + +如果系统无法直接访问互è”网，请å‚è§[离线环境：Kubernetes 安装](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)。 + +:::tip å•èŠ‚点安装æ示： + +在å•èŠ‚点 Kubernetes 集群中，Rancher Server ä¸å…·å¤‡é«˜å¯ç”¨æ€§ï¼Œè€Œé«˜å¯ç”¨æ€§å¯¹åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­è¿è¡Œ Rancher éžå¸¸é‡è¦ã€‚但是，如果你想è¦çŸ­æœŸå†…使用å•èŠ‚点节çœèµ„æºï¼ŒåŒæ—¶åˆä¿ç•™é«˜å¯ç”¨æ€§è¿ç§»è·¯å¾„，把 Rancher 安装到å•èŠ‚点集群也是åˆé€‚的。 + +è¦è®¾ç½®å•èŠ‚点 RKE 集群，在 `cluster.yml` 中é…置一个节点。该节点需具备所有三个角色，分别是`etcd`，`controlplane`å’Œ`worker`。 + +在这两ç§å•èŠ‚点设置中，Rancher å¯ä»¥ä¸Ž Helm 一起安装在 Kubernetes 集群上，安装方法与安装到其他集群上一样。 + +::: + +## 安装 Kubernetes + +### 所需的 CLI 工具 + +安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 + +安装 [RKE](https://rancher.com/docs/rke/latest/en/installation/)（Rancher Kubernetes Engine，是一个 Kubernetes å‘行版和命令行工具）。 + +### 1. 创建集群é…置文件 + +在这部分，你将创建一个å为 `rancher-cluster.yml`çš„ Kubernetes 集群é…置文件。在åŽç»­ä½¿ç”¨ RKE 命令设置集群的步骤中，此文件会用于在节点上安装 Kubernetes。 + +使用下é¢çš„示例作为指å—，创建 `rancher-cluster.yml` 文件。将 `nodes` 列表中的 IP 地å€æ›¿æ¢ä¸ºä½ åˆ›å»ºçš„ 3 个节点的 IP 地å€æˆ– DNS å称。 + +如果你的节点有公共地å€å’Œå†…部地å€ï¼Œå»ºè®®è®¾ç½® `internal_address:` 以便 Kubernetes 使用它实现集群内部通信。如果你想使用引用安全组或防ç«å¢™ï¼ŒæŸäº›æœåŠ¡ï¼ˆå¦‚ AWS EC2）è¦æ±‚设置 `internal_address:`。 + +RKE 需è¦é€šè¿‡ SSH 连接到æ¯ä¸ªèŠ‚点，它会在 `~/.ssh/id_rsa`的默认ä½ç½®æŸ¥æ‰¾ç§é’¥ã€‚如果æŸä¸ªèŠ‚点的ç§é’¥ä¸åœ¨é»˜è®¤ä½ç½®ä¸­ï¼Œä½ è¿˜éœ€è¦ä¸ºè¯¥èŠ‚点é…ç½® `ssh_key_path` 选项。 + +在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +```yaml +nodes: + - address: 165.227.114.63 + internal_address: 172.16.22.12 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.116.167 + internal_address: 172.16.32.37 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.127.226 + internal_address: 172.16.42.73 + user: ubuntu + role: [controlplane, worker, etcd] + +services: + etcd: + snapshot: true + creation: 6h + retention: 24h + +# Required for external TLS termination with +# ingress-nginx v0.22+ +ingress: + provider: nginx + options: + use-forwarded-headers: "true" + +kubernetes_version: v1.25.6-rancher4-1 +``` + +
通用 RKE 节点选项
+ +| 选项 | å¿…å¡« | æè¿° | +| ------------------ | -------- | -------------------------------------------------------------------------------------- | +| `address` | 是 | 公共 DNS 或 IP åœ°å€ | +| `user` | 是 | å¯ä»¥è¿è¡Œ docker 命令的用户 | +| `role` | 是 | 分é…给节点的 Kubernetes 角色列表 | +| `internal_address` | å¦ | 内部集群æµé‡çš„ç§æœ‰ DNS 或 IP åœ°å€ | +| `ssh_key_path` | å¦ | 用æ¥éªŒè¯èŠ‚点的 SSH ç§é’¥æ–‡ä»¶è·¯å¾„（默认值为 `~/.ssh/id_rsa`） | + +:::note 高级é…置： + +RKE æ供大é‡é…置选项，用于针对你的环境进行自定义安装。 + +如需了解选项和功能的完整列表，请å‚è§ [RKE 官方文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +è¦ä¸ºå¤§è§„模 Rancher 安装优化 etcd 集群，请å‚è§ [etcd 设置指å—](../../advanced-user-guides/tune-etcd-for-large-installs.md)。 + +有关 Dockershim 支æŒçš„详情，请å‚è§[此页é¢](../../../getting-started/installation-and-upgrade/installation-requirements/dockershim.md)。 + +::: + +### 2. è¿è¡Œ RKE + +``` +rke up --config ./rancher-cluster.yml +``` + +完æˆåŽï¼Œç»“æŸè¡Œåº”该是：`Finished build Kubernetes cluster succeeded`。 + +### 3. 测试集群 + +本节介ç»å¦‚何设置工作区，以便你å¯ä»¥ä½¿ç”¨ `kubectl` 命令行工具与此集群进行交互。 + +如果你已安装 `kubectl`，你需è¦å°† `kubeconfig` 文件放在 `kubectl` å¯è®¿é—®çš„ä½ç½®ã€‚`kubeconfig` 文件包å«ä½¿ç”¨ `kubectl` 访问集群所需的凭è¯ã€‚ + +你在è¿è¡Œ `rke up` 时，RKE 应该已ç»åˆ›å»ºäº†ä¸€ä¸ªå为 `kube_config_cluster.yml`çš„ `kubeconfig` 文件。该文件具有 `kubectl` å’Œ `helm`的凭è¯ã€‚ + +:::note + +如果你的文件åä¸æ˜¯ `rancher-cluster.yml`，kubeconfig 文件将命å为 `kube_config_.yml`。 + +::: + +将此文件移动到 `$HOME/.kube/config`。如果你使用多个 Kubernetes 集群，将 `KUBECONFIG` 环境å˜é‡è®¾ç½®ä¸º `kube_config_cluster.yml` 的路径: + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +用 `kubectl` 测试你的连接性，并查看你的所有节点是å¦éƒ½å¤„于 `Ready` 状æ€ï¼š + +``` +kubectl get nodes + +NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.13.5 +``` + +### 4. 检查集群 Pod çš„å¥åº·çŠ¶å†µ + +检查所有需è¦çš„ Pod 和容器是å¦å¥åº·ã€‚ + +- Pod 处于 `Running` 或 `Completed` 状æ€ã€‚ +- `READY` 表示è¿è¡Œ `STATUS` 为 `Running` çš„ Pod 的所有容器（例如， `3/3`）。 +- `STATUS` 为 `Completed` çš„ Pod 是一次è¿è¡Œçš„ Job。这些 Pod `READY` 列的值应该为 `0/1`。 + +``` +kubectl get pods --all-namespaces + +NAMESPACE NAME READY STATUS RESTARTS AGE +ingress-nginx nginx-ingress-controller-tnsn4 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-tw2ht 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-v874b 1/1 Running 0 30s +kube-system canal-jp4hz 3/3 Running 0 30s +kube-system canal-z2hg8 3/3 Running 0 30s +kube-system canal-z6kpw 3/3 Running 0 30s +kube-system kube-dns-7588d5b5f5-sf4vh 3/3 Running 0 30s +kube-system kube-dns-autoscaler-5db9bbb766-jz2k6 1/1 Running 0 30s +kube-system metrics-server-97bc649d5-4rl2q 1/1 Running 0 30s +kube-system rke-ingress-controller-deploy-job-bhzgm 0/1 Completed 0 30s +kube-system rke-kubedns-addon-deploy-job-gl7t4 0/1 Completed 0 30s +kube-system rke-metrics-addon-deploy-job-7ljkc 0/1 Completed 0 30s +kube-system rke-network-plugin-deploy-job-6pbgj 0/1 Completed 0 30s +``` + +这表示你已æˆåŠŸå®‰è£…了å¯è¿è¡Œ Rancher Server çš„ Kubernetes 集群。 + +### 5. ä¿å­˜ä½ çš„文件 + +:::note é‡è¦æ示： + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +::: + +将以下文件的副本ä¿å­˜åœ¨å®‰å…¨ä½ç½®ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_cluster.yml`：集群的 [Kubeconfig 文件](https://rancher.com/docs/rke/latest/en/kubeconfig/)。该文件包å«å¯å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚ +- `rancher-cluster.rkestate`：[Kubernetes 状æ€æ–‡ä»¶](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state)。此文件包括用于完全访问集群的凭è¯ã€‚

_Kubernetes 集群状æ€æ–‡ä»¶ä»…在 RKE 版本是 0.2.0 或更高版本时生æˆã€‚_ + +:::note + +åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +::: + +### 故障排除 + +å‚è§[故障排除](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)页é¢ã€‚ + + +### åŽç»­æ“作 +[安装 Rancher](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md new file mode 100644 index 00000000000..cc61efd50e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md @@ -0,0 +1,165 @@ +--- +title: 为 Rancher 设置高å¯ç”¨çš„ RKE2 Kubernetes 集群 +--- + +本文介ç»äº†å¦‚ä½•æ ¹æ® [Rancher Server 环境的最佳实践](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#kubernetes-安装环境)安装 Kubernetes 集群。 + +## 先决æ¡ä»¶ + +以下说明å‡è®¾ä½ å·²å‚è§[此章节](../infrastructure-setup/ha-rke2-kubernetes-cluster.md)é…置好三个节点，一个负载å‡è¡¡å™¨å’Œä¸€ä¸ª DNS 记录。 + +为了让 RKE2 与负载å‡è¡¡å™¨æ­£å¸¸å·¥ä½œï¼Œä½ éœ€è¦è®¾ç½®ä¸¤ä¸ªç›‘å¬å™¨ï¼Œä¸€ä¸ªç”¨äºŽ 9345 端å£ï¼Œå¦ä¸€ä¸ªç”¨äºŽ 6443 端å£çš„ Kubernetes API。 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§[支æŒç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。如需指定 RKE2 版本，请在è¿è¡Œ RKE2 安装脚本时，使用 `INSTALL_RKE2_VERSION` 环境å˜é‡ã€‚ + +## 安装 Kubernetes + +### 1. 安装 Kubernetes 并设置 RKE2 Server + +RKE2 Server ä½¿ç”¨åµŒå…¥å¼ etcd è¿è¡Œã€‚因此你ä¸éœ€è¦è®¾ç½®å¤–部数æ®å­˜å‚¨å°±å¯ä»¥åœ¨ HA 模å¼ä¸‹è¿è¡Œã€‚ + +在第一个节点上，使用你的预共享密文作为 Token æ¥è®¾ç½®é…置文件。Token å‚æ•°å¯ä»¥åœ¨å¯åŠ¨æ—¶è®¾ç½®ã€‚ + +如果你ä¸æŒ‡å®šé¢„共享密文，RKE2 会生æˆä¸€ä¸ªé¢„共享密文并将它放在 `/var/lib/rancher/rke2/server/node-token` 中。 + +为了é¿å…固定注册地å€çš„è¯ä¹¦é”™è¯¯ï¼Œè¯·åœ¨å¯åŠ¨ Server 时设置 `tls-san` å‚数。这个选项在 Server çš„ TLS è¯ä¹¦ä¸­å¢žåŠ ä¸€ä¸ªé¢å¤–的主机å或 IP 作为 Subject Alternative Name。如果你想通过 IP 和主机å访问，你å¯ä»¥å°†å®ƒæŒ‡å®šä¸ºä¸€ä¸ªåˆ—表。 + +首先，创建用于存放 RKE2 é…置文件的目录： + +``` +mkdir -p /etc/rancher/rke2/ +``` + +然åŽï¼Œå‚è§ä»¥ä¸‹ç¤ºä¾‹åœ¨ `/etc/rancher/rke2/config.yaml` 中创建 RKE2 é…置文件： + +``` +token: my-shared-secret +tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +``` +之åŽï¼Œè¿è¡Œå®‰è£…命令并å¯ç”¨å’Œå¯åŠ¨ RKE2： + +``` +curl -sfL https://get.rke2.io | sh - +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` +1. è¦åŠ å…¥å…¶ä½™çš„节点，使用åŒä¸€ä¸ªå…±äº«æˆ–自动生æˆçš„ Token æ¥é…ç½®æ¯ä¸ªé¢å¤–的节点。以下是é…置文件的示例： + + token: my-shared-secret + server: https://:9345 + tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +è¿è¡Œå®‰è£…程åºï¼Œç„¶åŽå¯ç”¨å¹¶å¯åŠ¨ RKE2： + + curl -sfL https://get.rke2.io | sh - + systemctl enable rke2-server.service + systemctl start rke2-server.service + + +1. 在第三 RKE2 Server 节点上è¿è¡ŒåŒæ ·çš„命令。 + +### 2. 检查 RKE2 是å¦æ­£å¸¸è¿è¡Œ + +在所有 Server 节点上å¯åŠ¨äº† RKE2 Server 进程åŽï¼Œç¡®ä¿é›†ç¾¤å·²ç»æ­£å¸¸å¯åŠ¨ï¼Œè¯·è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get nodes +现在，Server 节点的状æ€åº”该是 Ready。 +``` + +测试集群 Pod çš„å¥åº·çŠ¶å†µï¼š +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get pods --all-namespaces +``` + +**结果**：你已æˆåŠŸé…ç½® RKE2 Kubernetes 集群。 + +### 3. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 RKE2 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/rke2/rke2.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 +2. å¤åˆ¶ `/etc/rancher/rke2/rke2.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhostã€‚åœ¨ç«¯å£ 6443 上将æœåŠ¡å™¨é…置为 controlplane è´Ÿè½½å‡è¡¡å™¨çš„ DNS（RKE2 Kubernetes API Server ä½¿ç”¨ç«¯å£ 6443，而 Rancher Server 将通过 NGINX Ingress åœ¨ç«¯å£ 80 å’Œ 443 上æä¾›æœåŠ¡ã€‚）以下是一个示例 `rke2.yaml`： + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ RKE2 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [RKE2 官方文档](https://docs.rke2.io/cluster_access)或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### 4. 检查集群 Pod çš„å¥åº·çŠ¶å†µ + +现在你已ç»è®¾ç½®äº† `kubeconfig` 文件。你å¯ä»¥ä½¿ç”¨ `kubectl` 在本地主机访问集群。 + +检查所有需è¦çš„ Pod 和容器是å¦å¥åº·ï¼š + +``` +/var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml get pods -A +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system cloud-controller-manager-rke2-server-1 1/1 Running 0 2m28s +kube-system cloud-controller-manager-rke2-server-2 1/1 Running 0 61s +kube-system cloud-controller-manager-rke2-server-3 1/1 Running 0 49s +kube-system etcd-rke2-server-1 1/1 Running 0 2m13s +kube-system etcd-rke2-server-2 1/1 Running 0 87s +kube-system etcd-rke2-server-3 1/1 Running 0 56s +kube-system helm-install-rke2-canal-hs6sx 0/1 Completed 0 2m17s +kube-system helm-install-rke2-coredns-xmzm8 0/1 Completed 0 2m17s +kube-system helm-install-rke2-ingress-nginx-flwnl 0/1 Completed 0 2m17s +kube-system helm-install-rke2-metrics-server-7sggn 0/1 Completed 0 2m17s +kube-system kube-apiserver-rke2-server-1 1/1 Running 0 116s +kube-system kube-apiserver-rke2-server-2 1/1 Running 0 66s +kube-system kube-apiserver-rke2-server-3 1/1 Running 0 48s +kube-system kube-controller-manager-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-controller-manager-rke2-server-2 1/1 Running 0 57s +kube-system kube-controller-manager-rke2-server-3 1/1 Running 0 42s +kube-system kube-proxy-rke2-server-1 1/1 Running 0 2m25s +kube-system kube-proxy-rke2-server-2 1/1 Running 0 59s +kube-system kube-proxy-rke2-server-3 1/1 Running 0 85s +kube-system kube-scheduler-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-scheduler-rke2-server-2 1/1 Running 0 57s +kube-system kube-scheduler-rke2-server-3 1/1 Running 0 42s +kube-system rke2-canal-b9lvm 2/2 Running 0 91s +kube-system rke2-canal-khwp2 2/2 Running 0 2m5s +kube-system rke2-canal-swfmq 2/2 Running 0 105s +kube-system rke2-coredns-rke2-coredns-547d5499cb-6tvwb 1/1 Running 0 92s +kube-system rke2-coredns-rke2-coredns-547d5499cb-rdttj 1/1 Running 0 2m8s +kube-system rke2-coredns-rke2-coredns-autoscaler-65c9bb465d-85sq5 1/1 Running 0 2m8s +kube-system rke2-ingress-nginx-controller-69qxc 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-7hprp 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-x658h 1/1 Running 0 52s +kube-system rke2-metrics-server-6564db4569-vdfkn 1/1 Running 0 66s +``` + +**结果**：你å¯é€šè¿‡ä½¿ç”¨ `kubectl` 访问集群，并且 RKE2 集群能æˆåŠŸè¿è¡Œã€‚现在，你å¯ä»¥åœ¨é›†ç¾¤ä¸Šå®‰è£… Rancher Management Server。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md new file mode 100644 index 00000000000..a0ec43f76c5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md @@ -0,0 +1,73 @@ +--- +title: 推è的集群架构 +--- + +有三个角色å¯ä»¥åˆ†é…给节点，分别是 `etcd`ã€`controlplane` å’Œ `worker`。 + +## å°† Worker 节点与具有其他角色的节点分开 + +在设计集群时，你有两ç§é€‰æ‹©ï¼š + +* 为æ¯ä¸ªè§’色使用专用节点。这确ä¿äº†ç‰¹å®šè§’色所需组件的资æºå¯ç”¨æ€§ã€‚它还根æ®[端å£è¦æ±‚](../node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)严格隔离æ¯ä¸ªè§’色之间的网络æµé‡ã€‚ +* å°† `etcd` å’Œ `controlplane` 角色分é…给相åŒçš„节点。该节点必须满足这两个角色的硬件è¦æ±‚。 + +无论在哪ç§æƒ…况下，都ä¸åº”该在具有 `etcd` 或 `controlplane` 角色的节点中使用或添加 `worker` 角色。 + +因此，æ¯ä¸ªèŠ‚点的角色都有如下几ç§é…置选择： + +* `etcd` +* `controlplane` +* `etcd` å’Œ `controlplane` +* `worker` + +## æ¯ä¸ªè§’色的推è节点数 + +集群应该有： + +- 至少拥有三个角色为 `etcd` 的节点，æ¥ç¡®ä¿å¤±åŽ»ä¸€ä¸ªèŠ‚点时ä»èƒ½å­˜æ´»ã€‚增加 etcd 节点数é‡èƒ½æ高容错率，而将 etcd 分散到ä¸åŒå¯ç”¨åŒºç”šè‡³èƒ½èŽ·å–更好的容错能力。 +- 至少两个节点具有 `controlplane` 角色，以实现主组件高å¯ç”¨æ€§ã€‚ +- 至少两个具有 `worker` 角色的节点，用于在节点故障时é‡æ–°å®‰æŽ’工作负载。 + +有关æ¯ä¸ªè§’色的用途的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 中的节点角色](roles-for-nodes-in-kubernetes.md)。 + + +### controlplane 节点数 + +添加多个具有 `controlplane` 角色的节点，使æ¯ä¸ªä¸»ç»„件都具有高å¯ç”¨æ€§ã€‚ + +### etcd 节点数 + +在ä¿æŒé›†ç¾¤å¯ç”¨æ€§çš„åŒæ—¶ï¼Œå¯ä»¥ä¸€æ¬¡ä¸¢å¤±çš„节点数由分é…了 `etcd` 角色的节点数决定。对于具有 n 个æˆå‘˜çš„集群，最å°å€¼ä¸º (n/2)+1。因此，我们建议在一个区域内的 3 个ä¸åŒå¯ç”¨åŒºä¸­å„创建一个 `etcd` 节点，以在一个å¯ç”¨åŒºä¸¢å¤±çš„情况下存活。如果你åªä½¿ç”¨ä¸¤ä¸ªåŒºåŸŸï¼Œé‚£ä¹ˆåœ¨â€œå¤šæ•°èŠ‚点â€æ‰€åœ¨çš„å¯ç”¨åŒºä¸å¯ç”¨æ—¶ï¼Œä½ å°†ä¼šä¸¢å¤± etcd 集群。 + +| 具有 `etcd` 角色的节点 | 多数节点 | 容错能力 | +|--------------|------------|-------------------| +| 1 | 1 | 0 | +| 2 | 2 | 0 | +| 3 | 2 | **1** | +| 4 | 3 | 1 | +| 5 | 3 | **2** | +| 6 | 4 | 2 | +| 7 | 4 | **3** | +| 8 | 5 | 3 | +| 9 | 5 | **4** | + +å‚考： + +* [最佳 etcd 集群大å°çš„官方 etcd 文档](https://etcd.io/docs/v3.4.0/faq/#what-is-failure-tolerance) +* [为 Kubernetes æ“作 etcd 集群的官方 Kubernetes 文档](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) + +### Worker 节点数 + +添加多个具有 `worker` 角色的节点能确ä¿ä¸€ä¸ªèŠ‚点出现故障时å¯ä»¥é‡æ–°å®‰æŽ’工作负载。 + +### 为什么 Rancher 集群和è¿è¡Œåº”用的集群的生产è¦æ±‚ä¸åŒ + +ä½ å¯èƒ½å·²ç»æ³¨æ„到我们的 [Kubernetes 安装](../../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)说明并ä¸ç¬¦åˆæˆ‘们对生产就绪集群的è¦æ±‚，这是因为 `worker` 角色没有专用节点。然而，你 Rancher 中的这个三节点集群是有效的，因为： + +* 它å…许一个 `etcd` 节点故障。 +* 它通过多个 `controlplane` 节点æ¥ç»´æŠ¤ master 组件的多个实例。 +* 此集群上没有创建除 Rancher 之外的其他工作负载。 + +## å‚考 + +* [Kubernetes：主组件](https://kubernetes.io/docs/concepts/overview/components/#master-components) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md new file mode 100644 index 00000000000..1f929eb52f9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md @@ -0,0 +1,50 @@ +--- +title: Kubernetes 中节点的角色 +--- + +æœ¬èŠ‚ä»‹ç» Kubernetes 中 etcd 节点ã€controlplane 节点和 worker 节点的角色，以åŠè¿™äº›è§’色如何在集群中ååŒå·¥ä½œã€‚ + +此图适用于 [Rancher 通过 RKE 部署的 Kubernetes 集群](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)： + +![集群图](/img/clusterdiagram.svg)
+线æ¡è¡¨ç¤ºç»„件之间的通信。而颜色纯粹用于视觉辅助。 + +## etcd + +具有 `etcd` 角色的节点è¿è¡Œ etcd，这是一个一致且高å¯ç”¨çš„键值存储，用作 Kubernetes 所有集群数æ®çš„åŽå¤‡å­˜å‚¨ã€‚etcd 将数æ®å¤åˆ¶åˆ°æ¯ä¸ªèŠ‚点。 + +:::note + +具有 `etcd` 角色的节点在 UI 中显示为`ä¸å¯è°ƒåº¦`，å³é»˜è®¤æƒ…况下ä¸ä¼šå°† Pod 调度到这些节点。 + +::: + +## controlplane + +具有 `controlplane` 角色的节点è¿è¡Œ Kubernetes 主组件（ä¸åŒ…括 `etcd`，因为它是一个å•ç‹¬çš„角色）。有关组件的详细列表，请å‚阅 [Kubernetes：主组件](https://kubernetes.io/docs/concepts/overview/components/#master-components)。 + +:::note + +具有 `controlplane` 角色的节点在 UI 中显示为`ä¸å¯è°ƒåº¦`，å³é»˜è®¤æƒ…况下ä¸ä¼šå°† Pod 调度到这些节点。 + +::: + +### kube-apiserver + +Kubernetes API Server (`kube-apiserver`) 能水平扩展。如果节点具有需è¦è®¿é—® Kubernetes API Server 的组件，则æ¯ä¸ªå…·æœ‰ `controlplane` 角色的节点都将被添加到节点上的 NGINX 代ç†ä¸­ã€‚è¿™æ„味ç€å¦‚果一个节点å˜å¾—ä¸å¯è®¿é—®ï¼Œè¯¥èŠ‚点上的本地 NGINX 代ç†ä¼šå°†è¯·æ±‚转å‘到列表中的å¦ä¸€ä¸ª Kubernetes API Server。 + +### kube-controller-manager + +Kubernetes Controller Manager 使用 Kubernetes 中的端点进行 Leader 选举。`kube-controller-manager` 的一个实例将在 Kubernetes 端点中创建一个æ¡ç›®ï¼Œå¹¶åœ¨é…置的时间间隔内更新该æ¡ç›®ã€‚其他实例将看到一个状æ€ä¸º Active çš„ Leader，并等待该æ¡ç›®è¿‡æœŸï¼ˆä¾‹å¦‚节点无å“应）。 + +### kube-scheduler + +Kubernetes 调度器使用 Kubernetes 中的端点进行 Leader 选举。`kube-scheduler` 的一个实例将在 Kubernetes 端点中创建一个æ¡ç›®ï¼Œå¹¶åœ¨é…置的时间间隔内更新该æ¡ç›®ã€‚其他实例将看到一个状æ€ä¸º Active çš„ Leader，并等待该æ¡ç›®è¿‡æœŸï¼ˆä¾‹å¦‚节点无å“应）。 + +## worker + +具有 `worker` 角色的节点è¿è¡Œ Kubernetes 节点组件。有关组件的详细列表，请å‚阅 [Kubernetes：节点组件](https://kubernetes.io/docs/concepts/overview/components/#node-components)。 + +## å‚考 + +* [Kubernetes：节点组件](https://kubernetes.io/docs/concepts/overview/components/#node-components) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md new file mode 100644 index 00000000000..85bcf4b7b11 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md @@ -0,0 +1,129 @@ +--- +title: Rancher 管ç†é›†ç¾¤çš„节点è¦æ±‚ +--- + +本页æ述了 Rancher 管ç†çš„ Kubernetes 集群的è¦æ±‚，你的应用和æœåŠ¡å°†å®‰è£…在这些集群中。这些下游集群应该与è¿è¡Œ Rancher 的三节点集群分开。 + +:::note + +如果 Rancher 安装在高å¯ç”¨çš„ Kubernetes 集群上，Rancher Server 的三节点集群和下游集群有ä¸åŒçš„è¦æ±‚。有关 Rancher 的安装è¦æ±‚，请å‚考[安装文档](../../../pages-for-subheaders/installation-requirements.md)中的节点è¦æ±‚。 + +::: + +## æ“作系统和容器è¿è¡Œæ—¶è¦æ±‚ + +Rancher 兼容当å‰æ‰€æœ‰çš„ä¸»æµ Linux å‘行版和任何通用的 Docker 版本。所有下游集群的 etcd å’Œ controlplane 节点都需è¦è¿è¡Œåœ¨ Linux 上。而 Worker 节点å¯ä»¥è¿è¡Œåœ¨ Linux 或 [Windows Server](#windows-节点) 上。 + +如需了解å„个 Rancher 版本通过了哪些æ“作系统和 Docker 版本测试，请å‚è§[支æŒå’Œç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。 + +所有支æŒçš„æ“作系统都使用 64-bit x86 架构。 + +如果你想使用 ARM64，请å‚阅[在 ARM64 上è¿è¡Œï¼ˆå®žéªŒåŠŸèƒ½ï¼‰ã€‚](../../advanced-user-guides/enable-experimental-features/rancher-on-arm64.md) + +有关如何安装 Docker çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Docker 官方文档](https://docs.docker.com/)。 + +### Oracle Linux å’Œ RHEL è¡ç”Ÿçš„ Linux 节点 + +æŸäº›æºè‡ª RHEL çš„ Linux å‘行版（包括 Oracle Linux）的默认防ç«å¢™è§„则å¯èƒ½ä¼šé˜»æ­¢ä¸Ž Helm 的通信。我们建议ç¦ç”¨ firewalld。如果你的 Kubernetes 版本是 1.19，请务必ç¦ç”¨ firewalld。 + +:::note + +在 RHEL 8.4 中，NetworkManager 上有两个é¢å¤–çš„æœåŠ¡ï¼Œå³ `nm-cloud-setup.service` å’Œ `nm-cloud-setup.timer`。这些æœåŠ¡å¢žåŠ äº†ä¸€ä¸ªè·¯ç”±è¡¨ï¼Œå¹²æ‰°äº† CNI æ’件的é…置。如果å¯ç”¨äº†è¿™äº›æœåŠ¡ï¼Œä½ å¿…须使用以下命令ç¦ç”¨å®ƒä»¬ï¼Œç„¶åŽé‡æ–°å¯åŠ¨èŠ‚点以æ¢å¤è¿žæŽ¥ï¼š + +``` +systemctl disable nm-cloud-setup.service nm-cloud-setup.timer +reboot +``` + +::: + +### SUSE Linux 节点 + +SUSE Linux å¯èƒ½æœ‰ä¸€ä¸ªé˜²ç«å¢™ï¼Œé»˜è®¤æƒ…况下会阻止所有端å£ã€‚在这ç§æƒ…况下，请按照[步骤](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#打开-suse-linux-端å£)打开将主机添加到自定义集群所需的端å£ã€‚ + +### Flatcar Container Linux 节点 + +使用 Flatcar Container Linux 节点[通过 Rancher å¯åŠ¨ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 时，需è¦åœ¨ [Cluster Config 文件](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)中使用如下é…置： + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: canal + options: + canal_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: calico + options: + calico_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +还需è¦å¯ç”¨ Docker æœåŠ¡ï¼Œä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å¯ç”¨ Docker æœåŠ¡ï¼š + +``` +systemctl enable docker.service +``` + +使用[主机驱动](../../../pages-for-subheaders/about-provisioning-drivers.md#主机驱动)时会自动å¯ç”¨ Docker æœåŠ¡ã€‚ + +### Windows 节点 + +è¿è¡Œ Windows Server 节点必须使用 Docker ä¼ä¸šç‰ˆã€‚ + +Windows 节点åªèƒ½ç”¨äºŽ Worker 节点。请å‚阅[é…ç½® Windows 自定义集群](../../../pages-for-subheaders/use-windows-clusters.md)。 + +## 硬件è¦æ±‚ + +你的工作负载决定了具有 `worker` 角色的节点的硬件è¦æ±‚。è¿è¡Œ Kubernetes 节点组件的最低è¦æ±‚是 1 个 CPU（核心）和 1GB 内存。 + +关于 CPU 和内存，建议将 Kubernetes 集群的ä¸åŒå¹³é¢ï¼ˆetcdã€controlplane å’Œ worker）托管在ä¸åŒçš„节点上，以便它们å¯ä»¥ç›¸äº’独立扩展。 + +有关大型 Kubernetes 集群的硬件建议，请å‚阅[构建大型集群](https://kubernetes.io/docs/setup/best-practices/cluster-large/)的官方 Kubernetes 文档。 + +有关生产环境中 etcd 集群的硬件建议，请å‚阅官方 [etcd 文档](https://etcd.io/docs/v3.4.0/op-guide/hardware/)。 + +## 网络è¦æ±‚ + +对于生产集群，我们建议你通过仅打开以下端å£è¦æ±‚中定义的端å£æ¥é™åˆ¶æµé‡ã€‚ + +需è¦å¼€æ”¾çš„端å£æ ¹æ®ä¸‹æ¸¸é›†ç¾¤çš„å¯åŠ¨æ–¹å¼è€Œæœ‰æ‰€ä¸åŒã€‚以下列出了需è¦ä¸ºä¸åŒ[集群创建选项](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)打开的端å£ã€‚ + +有关 Kubernetes 集群中 etcd 节点ã€controlplane 节点和 Worker 节点的端å£è¦æ±‚的详细信æ¯ï¼Œè¯·å‚阅 [Rancher Kubernetes Engine 的端å£è¦æ±‚](https://rancher.com/docs/rke/latest/en/os/#ports)。 + +在[下游集群端å£è¦æ±‚](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#下游-kubernetes-集群节点)中，你å¯ä»¥æ‰¾åˆ°åœ¨å„ç§æƒ…况下使用的端å£çš„详细信æ¯ã€‚ + +:::caution + +ä¸è¦ä½¿ç”¨ä¸ŽçŽ°æœ‰èŠ‚点相åŒçš„主机å或 IP 地å€æ¥æ³¨å†ŒèŠ‚点。这样åšä¼šå¯¼è‡´ RKE 阻止节点加入，并导致é…置挂起。主机驱动和自定义集群都å¯èƒ½å‘生这ç§æƒ…况。如果节点必须é‡ç”¨çŽ°æœ‰èŠ‚点的主机å或 IP，则必须在注册节点之å‰è®¾ç½® `hostname_override` [RKE 选项](https://rke.docs.rancher.com/config-options/nodes#overriding-the-hostname)，以便节点能够正确加入。 + +::: + +## å¯é€‰ï¼šå®‰å…¨æ³¨æ„事项 + +如果你è¦é…ç½®ç¬¦åˆ CIS（互è”网安全中心）Kubernetes 基准的 Kubernetes 集群，我们建议你在安装 Kubernetes 之å‰æŒ‰ç…§æˆ‘们的强化指å—æ¥é…置节点。 + +有关强化指å—的更多信æ¯ï¼Œä»¥åŠäº†è§£å“ªä¸ªæŒ‡å—版本对应于你的 Rancher å’Œ Kubernetes 版本，请å‚阅[安全](../../../pages-for-subheaders/rancher-security.md#rancher-强化指å—)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md new file mode 100644 index 00000000000..e3857d23024 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -0,0 +1,282 @@ +--- +title: 注册现有集群 +--- + +集群注册功能å–代了导入集群的功能。 + +Rancher 管ç†æ³¨å†Œé›†ç¾¤çš„范围å–决于集群的类型。详情请å‚è§[对注册集群的管ç†èƒ½åŠ›](#对注册集群的管ç†èƒ½åŠ›)。 + + +## 先决æ¡ä»¶ + +### Kubernetes 节点角色 + +已注册的 RKE Kubernetes 集群必须具有所有三个节点角色，分别是 etcdã€controlplane å’Œ worker。åªæœ‰ controlplane 组件的集群无法在 Rancher 中注册。 + +有关 RKE 节点角色的更多信æ¯ï¼Œè¯·å‚阅[最佳实践](../../../pages-for-subheaders/checklist-for-production-ready-clusters.md#集群架构)。 + +### æƒé™ + +è¦åœ¨ Rancher 中注册集群，你必须在该集群中拥有 `cluster-admin` æƒé™ã€‚如果没有，请è¿è¡Œä»¥ä¸‹å‘½ä»¤å°†æƒé™æŽˆäºˆç”¨æˆ·ï¼š + +```plain +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole cluster-admin \ + --user [USER_ACCOUNT] +``` + +由于默认情况下 Google Kubernetes Engine (GKE) ä¸æŽˆäºˆ `cluster-admin` 角色，因此你必须先在 GKE 集群上è¿è¡Œå‘½ä»¤ï¼Œç„¶åŽæ‰èƒ½æ³¨å†Œã€‚è¦è¯¦ç»†äº†è§£ GKE RBAC，请å‚阅 [Google 官方文档](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control)。 + +### EKSã€AKS å’Œ GKE 集群 + +è¦æˆåŠŸå°†è¿™äº›é›†ç¾¤å¯¼å…¥ Rancher 或从 Rancher 中é…置它们，Elastic Kubernetes Service (EKS)ã€Azure Kubernetes Service (AKS) å’Œ Google Kubernetes Engine (GKE) 集群必须至少有一个托管节点组。此外，ä¸æ”¯æŒ GKE Autopilot 集群。有关 GKE 模å¼ä¹‹é—´å·®å¼‚，请å‚阅[比较 GKE Autopilot å’Œ Standard](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison)。 + +## 注册集群 + +1. 点击 **☰ > 集群管ç†**。 +2. 在**集群**页é¢ä¸Šï¼Œå•å‡»**导入集群**。 +3. 选择集群类型。 +4. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +5. 如果你在 Rancher 中导入一个通用 Kubernetes 集群，请执行以下步骤：
+ a. 点击**集群选项**下的 **Agent 环境å˜é‡**，为 [rancher cluster agent](../launch-kubernetes-with-rancher/about-rancher-agents.md) 设置环境å˜é‡ã€‚ä½ å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚如果 Rancher Agent 需è¦ä½¿ç”¨ä»£ç†ä¸Ž Rancher Server 通信，则å¯ä»¥ä½¿ç”¨ Agent 环境å˜é‡è®¾ç½® `HTTP_PROXY`，`HTTPS_PROXY` å’Œ `NO_PROXY` 环境å˜é‡ã€‚
+ b. å¯ç”¨é¡¹ç›®ç½‘络隔离，确ä¿é›†ç¾¤æ”¯æŒ Kubernetes `NetworkPolicy` 资æºã€‚用户å¯ä»¥é€šè¿‡**项目网络隔离**选项下的**高级选项**下拉èœå•è¿›è¡Œæ“作。 +6. å•å‡»**创建**。 +7. 此处会显示 `cluster-admin` æƒé™çš„先决æ¡ä»¶ï¼ˆå‚è§ä¸Šæ–‡çš„**先决æ¡ä»¶**），其中包括满足先决æ¡ä»¶çš„示例命令。 +8. å°† `kubectl` 命令å¤åˆ¶åˆ°å‰ªè´´æ¿ï¼Œå¹¶åœ¨é…置了 kubeconfig 的节点上è¿è¡Œè¯¥å‘½ä»¤ï¼Œä»Žè€ŒæŒ‡å‘è¦å¯¼å…¥çš„集群。如果你ä¸ç¡®å®šé…置是å¦æ­£ç¡®ï¼Œè¯·åœ¨è¿è¡Œ Rancher 显示的命令之å‰è¿è¡Œ `kubectl get nodes` 进行验è¯ã€‚ +9. 如果你使用自签åè¯ä¹¦ï¼Œä½ å°†æ”¶åˆ° `certificate signed by unknown authority` 的消æ¯ã€‚è¦è§£å†³æ­¤éªŒè¯é—®é¢˜ï¼Œè¯·å°† Rancher 中显示的以 `curl` 开头的命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚然åŽåœ¨é…置了 kubeconfig 的节点上è¿è¡Œè¯¥å‘½ä»¤ï¼Œä»Žè€ŒæŒ‡å‘è¦å¯¼å…¥çš„集群。 +10. 在节点上è¿è¡Œå®Œå‘½ä»¤åŽï¼Œå•å‡»**完æˆ**。 + +**结果**： + +- 集群已注册并分é…了 **Pending** 状æ€ã€‚Rancher 正在部署资æºæ¥ç®¡ç†ä½ çš„集群。 +- 当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ +- **Active** 集群分é…了两个项目，分别是 `Default`（包å«å‘½å空间 `default`）和 `System`（包å«å‘½å空间 `cattle-system`ã€`ingress-nginx`ã€`kube-public` å’Œ `kube-system`）。 + + +:::note + +无法é‡æ–°æ³¨å†Œå½“å‰åœ¨ Rancher 中处于 Active 状æ€çš„集群。 + +::: + +### 使用 Terraform é…置导入的 EKSã€AKS 或 GKE 集群 + +ä½ **ä»…**需è¦å®šä¹‰ Rancher 使用 Terraform 导入 EKSã€AKS 或 GKE 集群所需的最少字段。请谨记这点，因为 Rancher 会使用用户æ供的任何é…置覆盖集群的é…置。 + +:::caution + +å³ä½¿å½“å‰é›†ç¾¤çš„é…置与用户æ供的é…置之间åªå­˜åœ¨å¾®å°å·®å¼‚，但是微å°çš„差异也有å¯èƒ½äº§ç”Ÿå¾ˆå¤§çš„å½±å“。 + +::: + +Rancher 通过 `eks_config_v2` 使用 Terraform 导入 EKS 集群所需的最少é…置字段如下： + +- cloud_credential_id +- name +- region +- imported （对于导入的集群，此字段应始终设置为 `true`） + +导入的 EKS 集群的示例 YAML é…置： + +``` +resource "rancher2_cluster" "my-eks-to-import" { + name = "my-eks-to-import" + description = "Terraform EKS Cluster" + eks_config_v2 { + cloud_credential_id = rancher2_cloud_credential.aws.id + name = var.aws_eks_name + region = var.aws_region + imported = true + } +} +``` + +ä½ å¯ä»¥åœ¨ [Rancher2 Terraform Provider 文档](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)中找到其他云æ供商的更多示例。 + +## 对注册集群的管ç†èƒ½åŠ› + +Rancher 管ç†æ³¨å†Œé›†ç¾¤çš„范围å–决于集群的类型。 + +- [所有已注册集群的功能](#所有已注册集群的功能) +- [已注册 RKE2 å’Œ K3s 集群的附加功能](#已注册-rke2-å’Œ-k3s-集群的附加功能) +- [已注册 EKS, AKS å’Œ GKE 集群的附加功能](#已注册-eksaks-å’Œ-gke-集群的附加功能) + +### 所有已注册集群的功能 + +注册集群åŽï¼Œé›†ç¾¤æ‰€æœ‰è€…å¯ä»¥ï¼š + +- 通过 RBAC [管ç†é›†ç¾¤è®¿é—®](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) +- å¯ç”¨[Monitoringã€å‘Šè­¦å’Œ Notifiers](../../../pages-for-subheaders/monitoring-and-alerting.md) +- å¯ç”¨ [Logging](../../../pages-for-subheaders/logging.md) +- å¯ç”¨ [Istio](../../../pages-for-subheaders/istio.md) +- 管ç†é¡¹ç›®å’Œå·¥ä½œè´Ÿè½½ + +### 已注册 RKE2 å’Œ K3s 集群的附加功能 + +[K3s](https://rancher.com/docs/k3s/latest/en/) 是用于边缘安装的轻é‡çº§ã€å®Œå…¨å…¼å®¹çš„ Kubernetes å‘行版。 +[RKE2](https://docs.rke2.io) 是 Rancher 用于数æ®ä¸­å¿ƒå’Œäº‘安装的下一代 Kubernetes å‘行版。 + +RKE2 或 K3s 集群注册到 Rancher åŽï¼ŒRancher 会识别它。Rancher UI 将开放[所有已注册集群](#所有已注册集群的功能)的功能，以åŠä»¥ä¸‹ç”¨äºŽç¼–辑和å‡çº§é›†ç¾¤çš„功能： + +- [å‡çº§ Kubernetes 版本](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)的能力 + :::danger + + 将集群导入 Rancher åŽï¼Œä½ éœ€è¦ä½¿ç”¨ Rancher 执行å‡çº§ã€‚**ä¸**支æŒåœ¨ Rancher 之外å‡çº§å¯¼å…¥çš„集群。 + + ::: +- é…置能åŒæ—¶å‡çº§çš„最大节点数 +- 查看集群的é…ç½®å‚数和用于å¯åŠ¨é›†ç¾¤ä¸­æ¯ä¸ªèŠ‚点的环境å˜é‡çš„åªè¯»ç‰ˆæœ¬ + +### 已注册 EKS，AKS å’Œ GKE 集群的附加功能 + +Rancher 处ç†æ³¨å†Œçš„ EKSã€AKS 或 GKE 集群的方å¼ä¸Žå¤„ç†åœ¨ Rancher 中创建的集群的方å¼ç±»ä¼¼ã€‚但是，如果你通过 Rancher UI 删除已注册的集群，Rancher ä¸ä¼šé”€æ¯è¿™äº›é›†ç¾¤ã€‚ + +如果你在 Rancher 中创建 EKSã€AKS 或 GKE 集群，然åŽå°†å…¶åˆ é™¤ï¼ŒRancher 会销æ¯è¯¥é›†ç¾¤ã€‚通过 Rancher 删除已注册的集群时，Rancher Server 会_æ–­å¼€_与集群的连接。该集群ä»ç„¶å­˜åœ¨ï¼Œåªæ˜¯å®ƒä¸å†åœ¨ Rancher 中。你ä»ç„¶å¯ä»¥åƒæ³¨å†Œå‰ä¸€æ ·è®¿é—®å·²æ³¨é”€çš„集群。 + +有关å¯ç”¨äºŽç®¡ç†å·²æ³¨å†Œé›†ç¾¤çš„功能，请å‚阅[按集群类型划分的集群管ç†åŠŸèƒ½](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)。 + +## é…ç½® RKE2 å’Œ K3s 集群å‡çº§ + +:::tip + +Kubernetes 的最佳实践是在å‡çº§ä¹‹å‰å¤‡ä»½é›†ç¾¤ã€‚使用外部数æ®åº“å‡çº§é«˜å¯ç”¨ K3s 集群时，请使用关系数æ®åº“æ供商推èçš„æ–¹å¼å¤‡ä»½æ•°æ®åº“。 + +::: + +**并å‘**是å‡çº§æœŸé—´å…许ä¸å¯ç”¨çš„最大节点数。如果ä¸å¯ç”¨èŠ‚点的数é‡å¤§äºŽ**并å‘**，å‡çº§å°†å¤±è´¥ã€‚如果å‡çº§å¤±è´¥ï¼Œä½ å¯èƒ½éœ€è¦ä¿®å¤æˆ–移除失败的节点，然åŽå‡çº§æ‰èƒ½æˆåŠŸã€‚ + +- **controlplane 并å‘**：å¯ä»¥åŒæ—¶å‡çº§çš„最大æœåŠ¡å™¨èŠ‚点数；也是最大ä¸å¯ç”¨æœåŠ¡å™¨èŠ‚点数 +- **Worker 并å‘**：å¯ä»¥åŒæ—¶å‡çº§çš„最大 worker 节点数；也是最大ä¸å¯ç”¨ worker 节点数 + +在 RKE2 å’Œ K3s 文档中，control plane 节点也称为 Server 节点。Kubernetes master 节点è¿è¡Œåœ¨è¿™äº›èŠ‚点上，用于维护集群的状æ€ã€‚默认情况下，control plane 节点默认能够让工作负载调度到节点上。 + +类似的，在 RKE2 å’Œ K3s 文档中，具有 worker 角色的节点称为 Agent 节点。默认情况下，部署在集群中的任何工作负载或 Pod 都能调度到这些节点上。 + +## 已注册 RKE2 å’Œ K3s 集群的 Logging 调试和故障排除 + +节点由è¿è¡Œåœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­çš„ `system-upgrade-controller` å‡çº§ã€‚基于集群é…置，Rancher 部署了两个[计划](https://github.com/rancher/system-upgrade-controller#example-upgrade-plan)æ¥å‡çº§èŠ‚点，分别用于å‡çº§ control plane 节点和 worker 节点。`system-upgrade-controller` 会按照计划对节点进行å‡çº§ã€‚ + +è¦åœ¨ `system-upgrade-controller` deployment 上å¯ç”¨è°ƒè¯•æ—¥å¿—记录，请编辑 [configmap](https://github.com/rancher/system-upgrade-controller/blob/50a4c8975543d75f1d76a8290001d87dc298bdb4/manifests/system-upgrade-controller.yaml#L32) 以将调试环境å˜é‡è®¾ç½®ä¸º true。然åŽé‡å¯ `system-upgrade-controller` pod。 + +ä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤æŸ¥çœ‹ `system-upgrade-controller` 创建的日志： + +``` +kubectl logs -n cattle-system system-upgrade-controller +``` + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æŸ¥çœ‹è®¡åˆ’的当å‰çŠ¶æ€ï¼š + +``` +kubectl get plans -A -o yaml +``` + +如果集群å¡åœ¨å‡çº§ä¸­ï¼Œè¯·é‡å¯ `system-upgrade-controller`。 + +为防止å‡çº§æ—¶å‡ºçŽ°é—®é¢˜ï¼Œåº”éµå¾ª [Kubernetes å‡çº§æœ€ä½³å®žè·µ](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/)进行æ“作。 + +## 对 RKE2 å’Œ K3s 集群的授æƒé›†ç¾¤ç«¯ç‚¹æ”¯æŒ + +_从 v2.6.3 èµ·å¯ç”¨_ + +授æƒé›†ç¾¤ç«¯ç‚¹ (ACE) 已支æŒæ³¨å†Œçš„ RKE2 å’Œ K3s 集群。此支æŒè¿˜åŒ…括你在下游集群上å¯ç”¨ ACE 的手动步骤。有关授æƒé›†ç¾¤ç«¯ç‚¹çš„更多信æ¯ï¼Œè¯·å•å‡»[这里](../manage-clusters/access-clusters/authorized-cluster-endpoint.md)。 + +:::note 注æ„事项： + +- åªéœ€è¦åœ¨ä¸‹æ¸¸é›†ç¾¤çš„ controlplane 节点上执行这些步骤。你必须å•ç‹¬é…ç½®æ¯ä¸ª controlplane 节点。 + +- 以下步骤适用于在 v2.6.x 中注册的 RKE2 å’Œ K3s 集群，以åŠä»Žå…ˆå‰çš„ Rancher 版本注册（或导入）并å‡çº§åˆ° v2.6.x 的集群。 + +- 这些步骤将改å˜ä¸‹æ¸¸ RKE2 å’Œ K3s 集群的é…置并部署 `kube-api-authn-webhook`。如果 ACE 的未æ¥å®žçŽ°éœ€è¦æ›´æ–° `kube-api-authn-webhook`，那么这也必须手动完æˆã€‚有关此 webhook 的更多信æ¯ï¼Œè¯·å•å‡»[此处](../manage-clusters/access-clusters/authorized-cluster-endpoint.md#关于-kube-api-auth-身份验è¯-webhook)。 + +::: + +###### **在æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤çš„ controlplane 上å¯ç”¨ ACE 的手动执行步骤**： + +1. 在 `/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml` 创建一个文件，内容如下： + ```yaml + apiVersion: v1 + kind: Config + clusters: + - name: Default + cluster: + insecure-skip-tls-verify: true + server: http://127.0.0.1:6440/v1/authenticate + users: + - name: Default + user: + insecure-skip-tls-verify: true + current-context: webhook + contexts: + - name: webhook + context: + user: Default + cluster: Default + ``` + +2. 将以下内容添加到é…置文件中（如果文件ä¸å­˜åœ¨ï¼Œåˆ™åˆ›å»ºä¸€ä¸ªï¼‰ã€‚请注æ„，默认ä½ç½®æ˜¯ `/etc/rancher/{rke2,k3s}/config.yaml`： + ```yaml + kube-apiserver-arg: + - authentication-token-webhook-config-file=/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml + ``` + +3. è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + sudo systemctl stop {rke2,k3s}-server + sudo systemctl start {rke2,k3s}-server + +4. 最åŽï¼Œä½ **å¿…é¡»**返回 Rancher UI å¹¶åœ¨é‚£é‡Œç¼–è¾‘å¯¼å…¥çš„é›†ç¾¤ï¼Œä»Žè€Œå®Œæˆ ACE å¯ç”¨ã€‚å•å‡» **â‹® > 编辑é…ç½®**，然åŽå•å‡»**集群é…ç½®**下的**网络**选项å¡ã€‚最åŽï¼Œå•å‡»**授æƒç«¯ç‚¹**çš„**å¯ç”¨**按钮。å¯ç”¨ ACE åŽï¼Œä½ å¯ä»¥è¾“入完全é™å®šçš„域å (FQDN) å’Œè¯ä¹¦ä¿¡æ¯ã€‚ + +:::note + +FQDN 字段是å¯é€‰çš„。如果指定了该字段，它应该指å‘下游集群。仅当下游集群å‰é¢æœ‰ä½¿ç”¨äº†ä¸å—ä¿¡ä»»è¯ä¹¦çš„è´Ÿè½½å‡è¡¡å™¨æ—¶æ‰éœ€è¦è¯ä¹¦ä¿¡æ¯ã€‚如果你使用的是有效è¯ä¹¦ï¼Œåˆ™ä¸éœ€è¦å¡«å†™ CA è¯ä¹¦å­—段。 + +::: + +## 注释已注册的集群 + +Rancher 没有注册的 Kubernetes 集群（除了 RKE2 å’Œ K3s Kubernetes 集群之外）如何预置或é…置集群的任何信æ¯ã€‚ + +因此，当 Rancher 注册集群时，它å‡è®¾æŸäº›åŠŸèƒ½æ˜¯é»˜è®¤ç¦ç”¨çš„。Rancher 这样åšæ˜¯ä¸ºäº†é¿å…å‘用户暴露 UI 选项（å³ä½¿æ³¨å†Œçš„集群没有å¯ç”¨è¿™äº›åŠŸèƒ½ï¼‰ã€‚ + +但是，如果集群具有æŸç§åŠŸèƒ½ï¼ˆä¾‹å¦‚使用 pod 安全策略），那么该集群的用户å¯èƒ½ä»å¸Œæœ›åœ¨ Rancher UI 中为集群选择 pod 安全策略。为此，用户需è¦æ‰‹åŠ¨è®© Rancher 知é“集群已å¯ç”¨ pod 安全策略。 + +通过对已注册的集群进行注释，你å¯ä»¥å‘ Rancher 表明集群在 Rancher 之外被赋予了 Pod 安全策略或其他功能。 + +此示例注释表示å¯ç”¨äº† pod 安全策略： + +```json +"capabilities.cattle.io/pspEnabled": "true" +``` + +以下注释表示 Ingress 功能。请注æ„，éžåŽŸå§‹å¯¹è±¡çš„值需è¦è¿›è¡Œ JSON ç¼–ç ï¼Œå¹¶è½¬ä¹‰å¼•å·ï¼š + +```json +"capabilities.cattle.io/ingressCapabilities": "[ + { + "customDefaultBackend":true, + "ingressProvider":"asdf" + } +]" +``` + +ä½ å¯ä»¥ä¸ºé›†ç¾¤æ³¨é‡Šä»¥ä¸‹åŠŸèƒ½ï¼š + +- `ingressCapabilities` +- `loadBalancerCapabilities` +- `nodePoolScalingSupported` +- `nodePortRange` +- `pspEnabled` +- `taintSupport` + +所有功能åŠå…¶ç±»åž‹å®šä¹‰éƒ½å¯ä»¥åœ¨ Rancher API 视图中查看，地å€æ˜¯ `[Rancher Server URL]/v3/schemas/capabilities`。 + +è¦æ³¨é‡Šå·²æ³¨å†Œçš„集群： + +1. 点击 **☰ > 集群管ç†**。 +2. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ³¨é‡Šçš„自定义集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +3. 展开**标签 & 注释**。 +4. å•å‡»**添加注释**。 +5. 使用 `capabilities/: ` æ ¼å¼å‘集群添加注释，其中 `value` 是è¦ä½¿ç”¨æ³¨é‡Šè¦†ç›–的集群功能。在这ç§æƒ…况下，Rancher 在你添加注释之å‰éƒ½ä¸çŸ¥é“集群的任何功能。 +6. å•å‡»**ä¿å­˜**。 + +**结果**：注释并ä¸æ˜¯ç»™é›†ç¾¤æ供功能，而是告知 Rancher 集群具有这些功能。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md new file mode 100644 index 00000000000..3c31569bf95 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md @@ -0,0 +1,160 @@ +--- +title: 设置 Amazon 云æ供商 +weight: 1 +--- + +使用 `Amazon` 云æ供商时，你å¯ä»¥åˆ©ç”¨ä»¥ä¸‹åŠŸèƒ½ï¼š + +- **è´Ÿè½½å‡è¡¡å™¨**：在 **Port Mapping** 中选择 `Layer-4 Load Balancer` 或使用 `type: LoadBalancer` å¯åŠ¨ `Service` 时，å¯åŠ¨ AWS 弹性负载å‡è¡¡å™¨ (ELB)。 +- **æŒä¹…å·**：å…许你将 AWS 弹性å—存储 (EBS) 用于æŒä¹…å·ã€‚ + +有关 Amazon 云æ供商的所有信æ¯ï¼Œè¯·å‚阅 [cloud-provider-aws 自述文件](https://kubernetes.github.io/cloud-provider-aws/)。 + +è¦è®¾ç½® Amazon 云æ供商： + +1. [创建一个 IAM 角色并附加到实例](#1-创建-iam-角色并附加到实例) +2. [é…ç½® ClusterID](#2-创建-clusterid) + +:::note é‡è¦æ示： + +从 Kubernetes 1.23 开始，你必须åœç”¨ `CSIMigrationAWS` 特性开关æ‰èƒ½ä½¿ç”¨æ ‘内 AWS 云æ供商。为此，你å¯ä»¥åœ¨é«˜çº§é›†ç¾¤é…置中将 `feature-gates=CSIMigrationAWS=false` 设置为集群 Kubeletã€Controller Managerã€API Server å’Œ Scheduler 的附加å‚数。 + +::: + +### 1. 创建 IAM 角色并附加到实例 + +添加到集群的所有节点都必须能够与 EC2 交互，以便它们å¯ä»¥åˆ›å»ºå’Œåˆ é™¤èµ„æºã€‚ä½ å¯ä»¥ä½¿ç”¨é™„加到实例的 IAM 角色æ¥å¯ç”¨äº¤äº’。请å‚阅 [Amazon 文档：创建 IAM 角色](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#create-iam-role) æ¥åˆ›å»º IAM 角色。有两个示例策略： + +* 第一个策略适用于具有 `controlplane` 角色的节点。这些节点必须能够创建/删除 EC2 资æºã€‚以下 IAM 策略是一个示例，请根æ®ä½ çš„实际用例移除ä¸éœ€è¦çš„æƒé™ã€‚ +* 第二个策略适用于具有 `etcd` 或 `worker` 角色的节点。这些节点åªéœ€èƒ½å¤Ÿä»Ž EC2 检索信æ¯ã€‚ + +在创建 [Amazon EC2 集群](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md)时，你必须在创建**节点模æ¿**时填写创建的 IAM 角色的 **IAM Instance Profile Name**（ä¸æ˜¯ ARN）。 + +创建[自定义集群](../../../../pages-for-subheaders/use-existing-nodes.md)时，你必须手动将 IAM 角色附加到实例。 + +具有 `controlplane` 角色的节点的 IAM 策略： + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } +] +} +``` + +具有 `etcd` 或 `worker` 角色的节点的 IAM 策略： + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } +] +} +``` + +### 2. 创建 ClusterID + +以下资æºéœ€è¦ä½¿ç”¨ `ClusterID` 进行标记： + +- **Nodes**：Rancher 中添加的所有主机。 +- **Subnet**：集群使用的å­ç½‘。 +- **Security Group**：用于你的集群的安全组。 + +:::note + +ä¸è¦æ ‡è®°å¤šä¸ªå®‰å…¨ç»„。创建弹性负载å‡è¡¡å™¨ (ELB) 时，标记多个组会产生错误。 + +::: + +创建 [Amazon EC2 集群](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md)时，会自动为创建的节点é…ç½® `ClusterID`。其他资æºä»ç„¶éœ€è¦æ‰‹åŠ¨æ ‡è®°ã€‚ + +使用以下标签： + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `owned` + +`CLUSTERID` å¯ä»¥æ˜¯ä»»ä½•å­—符串，åªè¦å®ƒåœ¨æ‰€æœ‰æ ‡ç­¾é›†ä¸­ç›¸åŒå³å¯ã€‚ + +将标签的值设置为 `owned` 会通知集群带有该标签的所有资æºéƒ½ç”±è¯¥é›†ç¾¤æ‹¥æœ‰å’Œç®¡ç†ã€‚如果你在集群之间共享资æºï¼Œä½ å¯ä»¥å°†æ ‡ç­¾æ›´æ”¹ä¸ºï¼š + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `shared`. + +### 使用 Amazon Elastic Container Registry (ECR) + +在将[创建 IAM 角色并附加到实例](#1-创建-iam-角色并附加到实例)中的 IAM é…置文件附加到实例时，kubelet ç»„ä»¶èƒ½å¤Ÿè‡ªåŠ¨èŽ·å– ECR 凭è¯ã€‚使用低于 v1.15.0 çš„ Kubernetes 版本时，需è¦åœ¨é›†ç¾¤ä¸­é…ç½® Amazon 云æ供商。从 Kubernetes 版本 v1.15.0 开始，kubelet 无需在集群中é…ç½® Amazon 云æ供商å³å¯èŽ·å– ECR 凭è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md new file mode 100644 index 00000000000..0b6fc4bac45 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md @@ -0,0 +1,111 @@ +--- +title: 设置 Azure 云æ供商 +--- + +使用 `Azure` 云æ供商时，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹åŠŸèƒ½ï¼š + +- **è´Ÿè½½å‡è¡¡å™¨**：在特定网络安全组中å¯åŠ¨ Azure è´Ÿè½½å‡è¡¡å™¨ã€‚ + +- **æŒä¹…å·**：支æŒå°† Azure Blob ç£ç›˜å’Œ Azure 托管ç£ç›˜ä¸Žæ ‡å‡†å’Œé«˜çº§ Storage Account 一起使用。 + +- **网络存储**：通过 CIFS æŒ‚è½½æ”¯æŒ Azure 文件。 + +Azure 订阅ä¸æ”¯æŒä»¥ä¸‹è´¦å·ç±»åž‹ï¼š + +- å•ç§Ÿæˆ·è´¦å·ï¼ˆå³æ²¡æœ‰è®¢é˜…çš„è´¦å·ï¼‰ã€‚ +- 多订阅账å·ã€‚ + +## RKE å’Œ RKE2 的先决æ¡ä»¶ + +è¦ä¸º RKE å’Œ RKE2 设置 Azure 云æ供商，你需è¦é…置以下凭è¯ï¼š + +1. [设置 Azure 租户 ID](#1-设置-azure-租户-id) +2. [设置 Azure 客户端 ID å’Œ Azure 客户端密ç ](#2-设置-azure-客户端-id-å’Œ-azure-客户端密ç ) +3. [é…置应用注册æƒé™](#3-é…置应用注册æƒé™) +4. [设置 Azure 网络安全组å称](#4-设置-azure-网络安全组å称) + +### 1. 设置 Azure 租户 ID + +访问 [Azure 门户](https://portal.azure.com)，登录并转到 **Azure Active Directory**，然åŽé€‰æ‹© **Properties**。你的 **Directory ID** 是你的 **Tenant ID** (tenantID)。 + +如果è¦ä½¿ç”¨ Azure CLI，你å¯ä»¥è¿è¡Œ `az account show` 命令æ¥èŽ·å–ä¿¡æ¯ã€‚ + +### 2. 设置 Azure 客户端 ID å’Œ Azure å®¢æˆ·ç«¯å¯†ç  + +访问 [Azure 门户](https://portal.azure.com)，登录并按照以下步骤创建 **App Registration** 和对应的 **Azure Client ID** (aadClientId) ä»¥åŠ **Azure Client Secret** (aadClientSecret)。 + +1. 选择 **Azure Active Directory**。 +1. 选择 **App registrations**。 +1. 选择 **New application registration**。 +1. 选择 **Name**，选择 `Web app/API` 作为 **Application Type**ï¼Œå¹¶é€‰æ‹©ä»»æ„ **Sign-on URL**。 +1. 选择 **Create**。 + +在 **App registrations** 视图中，你应该会看到你创建的应用注册。**APPLICATION ID** 列中显示的值是需è¦ç”¨ä½œ **Azure Client ID** 的值。 + +ä¸‹ä¸€æ­¥æ˜¯ç”Ÿæˆ **Azure Client Secret**： + +1. 打开你创建的应用注册。 +1. 在 **Settings** 视图中，打开 **Keys**。 +1. 输入 **Key description**，选择过期时间，然åŽé€‰æ‹© **Save**。 +1. **Value** 列中显示的生æˆå€¼æ˜¯éœ€è¦ç”¨ä½œ **Azure Client Secret** 的值。该值åªä¼šæ˜¾ç¤ºä¸€æ¬¡ã€‚ + +### 3. é…置应用注册æƒé™ + +最åŽï¼Œä¸ºä½ çš„应用注册分é…适当的æƒé™ï¼š + +1. å‰å¾€ **More services**，æœç´¢ **Subscriptions** 并打开它。 +1. 打开 **Access control (IAM)**。 +1. 选择 **Add**。 +1. 在 **Role** 中选择 `Contributor`。 +1. 在 **Select** 中选择你创建的应用注册的å称。 +1. 选择 **Save**。 + +### 4. 设置 Azure 网络安全组å称 + +è¦ä½¿ Azure è´Ÿè½½å‡è¡¡å™¨æ­£å¸¸å·¥ä½œï¼Œä½ éœ€è¦è‡ªå®šä¹‰ä¸€ä¸ª Azure 网络安全组 (securityGroupName)。 + +如果你使用 Rancher Machine Azure 驱动æ¥é…置主机，则需è¦æ‰‹åŠ¨ç¼–辑主机，从而将主机分é…给此网络安全组。 + +你需è¦åœ¨é…置期间将自定义主机分é…给此网络安全组。 + +åªæœ‰éœ€è¦æˆä¸ºè´Ÿè½½å‡è¡¡å™¨åŽç«¯çš„主机æ‰éœ€è¦åˆ†é…到该组。 + +## Rancher 中的 RKE2 集群设置 + +1. 在**集群é…ç½®**中的**云æ供商**下拉列表中选择 **Azure**。 + +1. + * é…置云æ供商。请注æ„，Rancher 会自动创建新的网络安全组ã€èµ„æºç»„ã€å¯ç”¨æ€§é›†ã€å­ç½‘和虚拟网络。如果你已ç»åˆ›å»ºäº†å…¶ä¸­çš„一部分或全部，则需è¦åœ¨åˆ›å»ºé›†ç¾¤ä¹‹å‰æŒ‡å®šå®ƒä»¬ã€‚ + * ä½ å¯ä»¥å•å‡»**显示高级选项**以查看更多自动生æˆçš„å称，并在需è¦çš„时候更新它们。你的云æ供商é…ç½®**å¿…é¡»**与**主机池**中的字段匹é…。如果你有多个池，它们必须使用相åŒçš„资æºç»„ã€å¯ç”¨æ€§é›†ã€å­ç½‘ã€è™šæ‹Ÿç½‘络和网络安全组。 + * 下é¢æ供了一个示例。你å¯ä»¥æ ¹æ®éœ€è¦å¯¹å…¶è¿›è¡Œä¿®æ”¹ã€‚ + +
+ 云æ供商é…置示例 + + ```yaml + { + "cloud":"AzurePublicCloud", + "tenantId": "YOUR TENANTID HERE", + "aadClientId": "YOUR AADCLIENTID HERE", + "aadClientSecret": "YOUR AADCLIENTSECRET HERE", + "subscriptionId": "YOUR SUBSCRIPTIONID HERE", + "resourceGroup": "docker-machine", + "location": "westus", + "subnetName": "docker-machine", + "securityGroupName": "rancher-managed-KA4jV9V2", + "securityGroupResourceGroup": "docker-machine", + "vnetName": "docker-machine-vnet", + "vnetResourceGroup": "docker-machine", + "primaryAvailabilitySetName": "docker-machine", + "routeTableResourceGroup": "docker-machine", + "cloudProviderBackoff": false, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true + } + ``` + +
+ +1. 在**集群é…ç½® > 高级选项**ã€ä¸­ï¼Œå•å‡»**补充的 Controller Manager å‚æ•°**下的**添加**，并添加 `--configure-cloud-routes=false` 标志。 + +1. å•å‡»**创建**按钮æ¥æ交表å•å¹¶åˆ›å»ºé›†ç¾¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md new file mode 100644 index 00000000000..17d1e4f671c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md @@ -0,0 +1,19 @@ +--- +title: é…置树内 vSphere 云æ供商 +--- + +è¦è®¾ç½®æ ‘内 vSphere 云æ供商，请在 Rancher 中创建 vSphere 集群时按照以下步骤æ“作： + +1. å°† **Cloud Provider** 选项设置为 `Custom` 或 `Custom (In-Tree)`。 +1. å•å‡»**以 YAML 文件编辑**。 +1. 将以下结构æ’入到预填充的集群 YAML。这个结构必须放在 `rancher_kubernetes_engine_config` 下。请注æ„，`name` *å¿…é¡»*设置为 `vsphere`。 + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: vsphere + vsphereCloudProvider: + [Insert provider configuration] + ``` + +Rancher 使用 RKE（Rancher Kubernetes Engine）æ¥é…ç½® Kubernetes 集群。有关 `vsphereCloudProvider` 的属性的详细信æ¯ï¼Œè¯·å‚阅 RKE 文档中的 [vSphere é…ç½®å‚考](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere/config-reference/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md new file mode 100644 index 00000000000..f0461eb27b5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md @@ -0,0 +1,64 @@ +--- +title: é…置树外 vSphere 云æ供商 +--- + +Kubernetes 正在é€æ¸ä¸åœ¨æ ‘内维护云æ供商。vSphere 有一个树外云æ供商，å¯é€šè¿‡å®‰è£… vSphere 云æ供商和云存储æ’件æ¥ä½¿ç”¨ã€‚ + +本文介ç»äº†å¦‚何在设置集群åŽå®‰è£… Cloud Provider Interface (CPI) å’Œ Cloud Storage Interface (CSI) æ’件。 + +## 先决æ¡ä»¶ + +支æŒçš„ vSphere 版本： + +* 6.7u3 +* 7.0u1 或更高版本。 + +Kubernetes 版本必须为 1.19 或更高版本。 + +树外 vSphere 云æ供商è¦æ±‚使用 Linux 节点，ä¸æ”¯æŒ Windows 节点。 + +## 安装 + +åœ¨å®‰è£…äº‘å­˜å‚¨æŽ¥å£ (CSI) 之å‰ï¼Œåº”先安装云æä¾›å•†æŽ¥å£ (CPI)。 + +### 1. 创建 vSphere 集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **VMware vSphere** 或 **自定义**。 +1. 在**集群é…ç½®**çš„**基本信æ¯**选项å¡ä¸­ï¼Œå°†**云æ供商**设置为 **vSphere**。 +1. 在**附加é…ç½®**选项å¡ä¸­ï¼Œè®¾ç½® vSphere Cloud Provider (CPI) å’Œ Storage Provider (CSI) 选项。 +1. 完æˆé›†ç¾¤åˆ›å»ºã€‚ + +### 2. 安装 CPI æ’件 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CPI æ’件的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CPI**。 +1. 填写所需的 vCenter 详细信æ¯ã€‚ +1. vSphere CPI 使用 vSphere CSI 驱动所需的 ProviderID æ¥åˆå§‹åŒ–所有节点。在使用以下命令安装 CSI 驱动之å‰ï¼Œæ£€æŸ¥æ˜¯å¦æ‰€æœ‰èŠ‚点都使用 ProviderID 进行了åˆå§‹åŒ–： + + ``` + kubectl describe nodes | grep "ProviderID" + ``` + +### 3. 安装 CSI æ’件 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CSI æ’件的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CSI**。 +1. å•å‡»**安装**。 +1. 填写所需的 vCenter 详细信æ¯ã€‚在**功能**选项å¡ä¸­ï¼Œå°†**å¯ç”¨ CSI è¿ç§»**设置为 **false**。 +3. 在**存储**选项å¡ä¸­ï¼Œå¡«å†™ StorageClass 的详细信æ¯ã€‚æ­¤ Chart 使用 `csi.vsphere.vmware.com` 作为置备程åºæ¥åˆ›å»ºä¸€ä¸ª StorageClass。 +1. å•å‡»**安装**。 + + +## 使用 CSI 驱动æ¥ç½®å¤‡å· + +CSI chart 默认创建一个 storageClass。 + +如果在å¯åŠ¨ chart 时未选择该选项，请使用 `csi.vsphere.vmware.com` 作为置备程åºæ¥åˆ›å»ºä¸€ä¸ª storageClass。 + +使用此 StorageClass 置备的所有å·éƒ½å°†ç”± CSI 驱动é…置。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md new file mode 100644 index 00000000000..f5e437e9de8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md @@ -0,0 +1,57 @@ +--- +title: 设置 Google Compute Engine 云æ供商 +--- + +在本节中，你将了解如何在 Rancher 中为自定义集群å¯ç”¨ Google Compute Engine (GCE) 云æ供商。自定义集群指的是 Rancher 在现有节点上安装 Kubernetes 的集群。 + +详情请å‚è§ [GCE 云æ供商的官方 Kubernetes 文档](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#gce)。 + +:::note 先决æ¡ä»¶ï¼š + +GCE 上的 `Identity` ServiceAccount å’Œ API 访问需è¦`Computer Admin`æƒé™ã€‚ + +::: + +如果你使用 Calico： + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è‡ªå®šä¹‰é›†ç¾¤ï¼Œç‚¹å‡» **â‹® > 编辑 YAML** 并输入如下é…置： + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + network: + options: + calico_cloud_provider: "gce" + plugin: "calico" + ``` + +如果你使用 Canal 或 Flannel： + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è‡ªå®šä¹‰é›†ç¾¤ï¼Œç‚¹å‡» **â‹® > 编辑 YAML** 并输入如下é…置： + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + services: + kube_controller: + extra_args: + configure-cloud-routes: true # we need to allow the cloud provider configure the routes for the hosts + ``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md new file mode 100644 index 00000000000..065ff2deb84 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md @@ -0,0 +1,108 @@ +--- +title: å°† vSphere 树内å·è¿ç§»åˆ°æ ‘外 +--- + +Kubernetes 正在é€æ¸ä¸åœ¨æ ‘内维护云æ供商。vSphere 有一个树外云æ供商，å¯é€šè¿‡å®‰è£… vSphere 云æ供商和云存储æ’件æ¥ä½¿ç”¨ã€‚ + +本页介ç»å¦‚何从树内 vSphere 云æ供商è¿ç§»åˆ°æ ‘外，以åŠå¦‚何在è¿ç§»åŽç®¡ç†çŽ°æœ‰è™šæ‹Ÿæœºã€‚ + +本文éµå¾ªå®˜æ–¹ [vSphere è¿ç§»æ–‡æ¡£](https://vsphere-csi-driver.sigs.k8s.io/features/vsphere_csi_migration.html)中æ供的步骤，并介ç»äº†è¦åœ¨ Rancher 中执行的步骤。 + +### Cloud-config æ ¼å¼é™åˆ¶ + +由于 vSphere CSI 中的现有错误，使用以下 cloud-config æ ¼å¼é…置的现有å·å°†ä¸ä¼šè¿ç§»ã€‚ + +如果 cloud-config 具有以下格å¼çš„ datastore 和资æºæ± è·¯å¾„，vsphere CSI 驱动将无法识别它： + +```yaml +default-datastore: /datastore/ +resourcepool-path: "/host//Resources/" +``` + +æ ¼å¼å¦‚下的使用树内æ供商预置的å·å°†èƒ½æ­£ç¡®è¿ç§»ï¼š + +```yaml +default-datastore: +resourcepool-path: "/Resources/" +``` + +上游 bug：https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/628 + +跟踪此 bug çš„ Rancher issue：https://github.com/rancher/rancher/issues/31105 + +## 先决æ¡ä»¶ + +- vSphere CSI è¿ç§»éœ€è¦ vSphere 7.0u1。为了管ç†çŽ°æœ‰çš„树内 vSphere å·ï¼Œè¯·å°† vSphere å‡çº§åˆ° 7.0u1。 +- Kubernetes 版本必须为 1.19 或更高版本。 + +## è¿ç§» + +### 1. 安装 CPI æ’件 + +在安装 CPI 之å‰ï¼Œæˆ‘们需è¦ä½¿ç”¨ `node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule` 为所有节点添加污点。 + +è¿™å¯ä»¥é€šè¿‡è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å®Œæˆï¼š + +``` +curl -O https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +``` + +或者： + +``` +wget https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +chmod +x taints.sh +./taints.sh +``` + +通过è¿è¡Œè„šæœ¬ä¸ºæ‰€æœ‰èŠ‚点添加污点åŽï¼Œå¯åŠ¨ Helm vSphere CPI Chart： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CPI chart 的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CPI**。 +1. å•å‡»**安装**。 +1. 填写所需的 vCenter 详细信æ¯ï¼Œç„¶åŽå•å‡»**安装**。 + +vSphere CPI 使用 vSphere CSI 驱动所需的 ProviderID æ¥åˆå§‹åŒ–所有节点。 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œæ£€æŸ¥æ˜¯å¦å·²ä½¿ç”¨ ProviderID åˆå§‹åŒ–了所有节点： + +``` +kubectl describe nodes | grep "ProviderID" +``` + +### 2. 安装 CSI 驱动 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CSI chart 的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CSI**。 +1. å•å‡»**安装**。 +1. 填写所需的 vCenter 详细信æ¯ï¼Œç„¶åŽå•å‡»**安装**。 +1. 选中**在安装å‰è‡ªå®šä¹‰ Helm 选项**，然åŽå•å‡»**下一步**。 +1. 在**功能**选项å¡ä¸­ï¼Œé€‰ä¸­**å¯ç”¨ CSI è¿ç§»**。 +1. 你也å¯ä»¥è½¬åˆ°**存储**选项å¡å¹¶è®¾ç½® datastore。此 Chart 使用 `csi.vsphere.vmware.com` 作为置备程åºæ¥åˆ›å»ºä¸€ä¸ª StorageClass。在创建此 StorageClass 时，你也å¯ä»¥æ供用于é…ç½® CSI å·çš„ datastore URL。通过选择 datastore 并转到**概述**选项å¡ï¼Œä½ å¯ä»¥åœ¨ vSphere 客户端中找到 datastore URL。填写 StorageClass 的详细信æ¯ã€‚ +1. å•å‡»**安装**。 + +### 3. 编辑集群以å¯ç”¨ CSI è¿ç§»çš„功能开关 + +1. 在编辑集群时，如果 Kubernetes 版本低于 1.19，请从 **Kubernetes 版本**下拉èœå•ä¸­é€‰æ‹© Kubernetes 1.19 或更高版本。 +2. è¦å¯ç”¨åŠŸèƒ½å¼€å…³ï¼Œè¯·å•å‡»**以 YAML 文件编辑**，然åŽåœ¨ kube-controller å’Œ kubelet 下添加以下内容： + + ```yaml + extra_args: + feature-gates: "CSIMigration=true,CSIMigrationvSphere=true" + ``` + +### 4. 清空 worker 节点 + +在更改 kubelet å’Œ kube-controller-manager å‚数之å‰ï¼Œå¿…须在å‡çº§æœŸé—´æ¸…空 worker 节点： + + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ¸…空 worker 节点的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**高级选项**中，将**最大ä¸å¯ç”¨çš„ Worker 节点数**字段设置为 1。 +1. è¦åœ¨å‡çº§æœŸé—´æ¸…空节点，请选择**清空节点 > 是**。 +1. å°†**强制**å’Œ**删除本地数æ®**设置为 **true**。 +1. 点击**ä¿å­˜**以å‡çº§é›†ç¾¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md new file mode 100644 index 00000000000..4369666a95d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md @@ -0,0 +1,153 @@ +--- +title: 创建 AKS 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在 Microsoft Azure Kubernetes æœåŠ¡ (AKS) 中的集群。 + + +## Microsoft Azure 中的先决æ¡ä»¶ + +:::caution + +部署到 AKS 会产生费用。 + +::: + +è‹¥è¦ä¸Ž Azure API 交互，AKS é›†ç¾¤éœ€è¦ Azure Active Directory (AD) æœåŠ¡ä¸»ä½“。æœåŠ¡ä¸»ä½“是动æ€åˆ›å»ºå’Œç®¡ç†å…¶ä»– Azure 资æºæ‰€å¿…需的，它为集群æ供凭è¯æ¥ä¸Ž AKS 通信。有关æœåŠ¡ä¸»ä½“的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/kubernetes-service-principal)。 + +在创建æœåŠ¡ä¸»ä½“之å‰ï¼Œä½ éœ€è¦ä»Ž [Microsoft Azure 门户](https://portal.azure.com)获å–以下信æ¯ï¼š + +- 订阅 ID +- 客户端 ID +- å®¢æˆ·ç«¯å¯†ç  + +以下介ç»å¦‚何使用 Azure 命令行工具或 Azure 门户进行设置，从而满足先决æ¡ä»¶ã€‚ + +### 使用 Azure 命令行工具设置æœåŠ¡ä¸»ä½“ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ›å»ºæœåŠ¡ä¸»ä½“： + +``` +az ad sp create-for-rbac --skip-assignment +``` + +结果应显示新æœåŠ¡ä¸»ä½“çš„ä¿¡æ¯ï¼š +``` +{ + "appId": "xxxx--xxx", + "displayName": "", + "name": "http://", + "password": "", + "tenant": "" +} +``` + +你还需è¦å‘æœåŠ¡ä¸»ä½“添加角色，以便它具有与 AKS API 通信的æƒé™ã€‚它还需è¦è®¿é—®æƒé™æ‰èƒ½åˆ›å»ºå’Œåˆ—出虚拟网络。 + +下é¢æ˜¯å°†å‚与者角色分é…ç»™æœåŠ¡ä¸»ä½“的示例命令。å‚与者å¯ä»¥ç®¡ç† AKS 上的任何内容，但ä¸èƒ½æŽˆäºˆå…¶ä»–人访问æƒé™ï¼š + +``` +az role assignment create \ + --assignee $appId \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +你还å¯ä»¥é€šè¿‡åˆå¹¶ä¸¤ä¸ªå‘½ä»¤ï¼Œæ¥åˆ›å»ºæœåŠ¡ä¸»ä½“并授予å‚与者æƒé™ã€‚在此命令中，`scope` 需è¦æä¾› Azure 资æºçš„完整路径： + +``` +az ad sp create-for-rbac \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +### 从 Azure 门户设置æœåŠ¡ä¸»ä½“ + +你还å¯ä»¥æŒ‰ç…§è¿™äº›è¯´æ˜Žè®¾ç½®æœåŠ¡ä¸»ä½“，并从 Azure 门户为它授予基于角色的访问æƒé™ã€‚ + +1. 转到 Microsoft Azure 门户[主页](https://portal.azure.com)。 + +1. å•å‡» **Azure Active Directory**。 +1. å•å‡» **App registrations**。 +1. å•å‡» **New registration**。 +1. 输入å称。这将是你æœåŠ¡ä¸»ä½“çš„å称。 +1. å¯é€‰ï¼šé€‰æ‹©å¯ä»¥ä½¿ç”¨æœåŠ¡ä¸»ä½“çš„è´¦å·ã€‚ +1. å•å‡» **Register**。 +1. 你现在应该能在 **Azure Active Directory > App registrations** 下看到æœåŠ¡ä¸»ä½“çš„å称。 +1. å•å‡»ä½ çš„æœåŠ¡ä¸»ä½“çš„å称。记下应用 ID（也称为应用 ID 或客户端 IDï¼‰ï¼Œä»¥ä¾¿åœ¨é¢„é… AKS 集群时使用。然åŽå•å‡» **Certificates & secrets**。 +1. å•å‡» **New client secret**。 +1. 输入简短说明，选择过期时间，然åŽå•å‡» **Add**。记下客户端密ç ï¼Œä»¥ä¾¿åœ¨é¢„é… AKS 集群时使用它。 + +**结果**：你已ç»åˆ›å»ºäº†ä¸€ä¸ªæœåŠ¡ä¸»ä½“，现在你能够在 **App registrations** 下的 **Azure Active Directory** 中找到它。你ä»ç„¶éœ€è¦å‘æœåŠ¡ä¸»ä½“授予对 AKS 的访问æƒé™ã€‚ + +è¦æŽˆäºˆå¯¹æœåŠ¡ä¸»ä½“基于角色的访问æƒé™ï¼š + +1. 点击左侧导航æ ä¸­çš„ **All Services**。然åŽå•å‡» **Subscriptions**。 +1. å•å‡»è¦ä¸Ž Kubernetes 集群关è”的订阅的å称。记下订阅 IDï¼Œä»¥ä¾¿åœ¨é¢„é… AKS 集群时使用它。 +1. å•å‡» **Access Control (IAM)**。 +1. 在 **Add role assignment** 中，å•å‡» **Add**。 +1. 在 **Role** 字段中，选择将有æƒè®¿é—® AKS 的角色。例如，你å¯ä»¥ä½¿ç”¨ **Contributor** 角色，该角色有æƒç®¡ç†é™¤æŽˆäºˆå…¶ä»–用户访问æƒé™ä¹‹å¤–的所有内容。 +1. 在 **Assign access to** 字段中，选择 **Azure AD 用户ã€ç»„或æœåŠ¡ä¸»ä½“**。 +1. 在 **Select** 字段中，选择æœåŠ¡ä¸»ä½“çš„å称，然åŽå•å‡» **Save**。 + +**结果**：你的æœåŠ¡ä¸»ä½“现在å¯ä»¥è®¿é—® AKS。 + +## 1. 创建 AKS äº‘å‡­è¯ + +1. 在 Rancher UI 中，å•å‡» **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 填写表å•ã€‚有关填写表格的帮助，请å‚阅[é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md#云凭è¯)。 +1. å•å‡»**创建**。 + +## 2. 创建 AKS 集群 + +使用 Rancher é…置你的 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**中，å•å‡»**创建**。 +1. å•å‡» **Azure AKS**。 +1. 填写表å•ã€‚有关填写表格的帮助，请å‚阅[é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md)。 +1. å•å‡»**创建**。 + +**结果**：集群已创建，并处于 **Provisioning** 状æ€ã€‚Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +## RBAC +在 Rancher UI 中é…ç½® AKS 集群时，由于 RBAC 需è¦å¯ç”¨ï¼Œå› æ­¤ RBAC ä¸å¯é…置。 + +注册或导入到 Rancher çš„ AKS é›†ç¾¤éœ€è¦ RBAC。 + +## AKS 集群é…ç½®å‚考 + +有关如何在 Rancher UI é…ç½® AKS 集群的更多信æ¯ï¼Œè¯·å‚阅[é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md)。 + +## ç§æœ‰é›†ç¾¤ + +通常情况下，无论集群是å¦ä¸ºç§æœ‰ï¼ŒAKS worker 节点都ä¸ä¼šèŽ·å¾—公共 IP。在ç§æœ‰é›†ç¾¤ä¸­ï¼Œcontrolplane 没有公共端点。 + +Rancher å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€è¿žæŽ¥åˆ°ç§æœ‰ AKS 集群。 + +第一ç§æ–¹æ³•æ˜¯ç¡®ä¿ Rancher è¿è¡Œåœ¨ä¸Ž AKS 节点相åŒçš„ [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) 上。 + +第二ç§æ–¹æ³•æ˜¯è¿è¡Œå‘½ä»¤å‘ Rancher 注册集群。é…置集群åŽï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èƒ½è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API 的地方è¿è¡Œæ˜¾ç¤ºçš„命令。é…ç½®å¯ç”¨äº†ç§æœ‰ API 端点的 AKS 集群时，此命令将显示在弹出窗å£ä¸­ã€‚ + +:::note + +注册现有 AKS 集群时，集群å¯èƒ½éœ€è¦ä¸€äº›æ—¶é—´ï¼ˆå¯èƒ½æ˜¯æ•°å°æ—¶ï¼‰æ‰ä¼šå‡ºçŽ°åœ¨ `Cluster To register` 下拉列表中。ä¸åŒåŒºåŸŸçš„结果å¯èƒ½ä¸åŒã€‚ + +::: + +有关连接到 AKS 专用集群的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster)。 + +## åŒæ­¥ + +AKS é…置者å¯ä»¥åœ¨ Rancher å’Œæ供商之间åŒæ­¥ AKS 集群的状æ€ã€‚有关其工作原ç†çš„技术说明，请å‚阅[åŒæ­¥](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md)。 + +有关é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/gke-cluster-configuration.md#é…置刷新间隔)。 + +## 以编程方å¼åˆ›å»º AKS 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² AKS 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md new file mode 100644 index 00000000000..a10200d37fa --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md @@ -0,0 +1,59 @@ +--- +title: 创建阿里云 ACK 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在阿里云 Alibaba Cloud Kubernetes (ACK) 中的集群。Rancher å·²ç»ä¸º ACK 实现并打包了针对 ACK çš„[集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md)，但是默认情况下，这个集群驱动的状æ€æ˜¯ `inactive`。为了å¯åŠ¨ ACK 集群，你需è¦[å¯ç”¨ ACK 集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#激活åœç”¨é›†ç¾¤é©±åŠ¨)。å¯ç”¨é›†ç¾¤é©±åŠ¨åŽï¼Œä½ å¯ä»¥å¼€å§‹é…ç½® ACK 集群。 + +## Rancher 之外的先决æ¡ä»¶ + +:::caution + +部署到 ACK 会产生费用。 + +::: + +1. 在阿里云中，在对应的控制å°ä¸­æ¿€æ´»ä»¥ä¸‹æœåŠ¡ã€‚ + + - [容器æœåŠ¡](https://cs.console.aliyun.com) + - [资æºç¼–排æœåŠ¡](https://ros.console.aliyun.com) + - [RAM](https://ram.console.aliyun.com) + +2. ç¡®ä¿ä½ ç”¨äºŽåˆ›å»º ACK 集群的账å·å…·æœ‰é€‚当的æƒé™ã€‚详è§é˜¿é‡Œäº‘[角色授æƒ](https://www.alibabacloud.com/help/doc-detail/86483.htm)å’Œ[使用容器æœåŠ¡æŽ§åˆ¶å°ä½œä¸º RAM 用户](https://www.alibabacloud.com/help/doc-detail/86484.htm)官方文档。 + +3. 在阿里云中，创建[访问密钥](https://www.alibabacloud.com/help/doc-detail/53045.html)。 + +4. 在阿里云中，创建 [SSH 密钥对](https://www.alibabacloud.com/help/doc-detail/51793.html)。该密钥用于访问 Kubernetes 集群中的节点。 + +## Rancher 先决æ¡ä»¶ + +你需è¦å¯ç”¨é˜¿é‡Œäº‘ ACK 集群驱动： + +1. 点击 **☰ > 集群管ç†**。 +1. 点击**驱动**。 +1. 在**集群驱动**选项å¡ä¸­ï¼Œè½¬åˆ° **Alibaba ACK** 集群驱动并å•å‡» **â‹® > 激活**。 + +集群驱动下载完æˆåŽï¼Œå°±å¯ä»¥åœ¨ Rancher 中创建阿里云 ACK 集群了。 + +## 创建 ACK 集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. å•å‡» **Alibaba ACK**。 +1. 输入**集群å称**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 为 ACK 集群é…ç½®**è´¦å·è®¿é—®**。选择è¦åœ¨å…¶ä¸­æž„建集群的地ç†åŒºåŸŸï¼Œå¹¶è¾“入在先决æ¡ä»¶æ­¥éª¤ä¸­åˆ›å»ºçš„访问密钥。 +1. 点击**下一步：é…置集群**，然åŽé€‰æ‹©é›†ç¾¤ç±»åž‹ã€Kubernetes 版本和å¯ç”¨åŒºã€‚ +1. 如果选择 **Kubernetes** 作为集群类型，请å•å‡»**下一步：é…ç½® master 节点**，然åŽå¡«å†™ **master 节点**表å•ã€‚ +1. å•å‡»**下一步：é…ç½® Worker 节点**，然åŽå¡«å†™ **Worker 节点**表å•ã€‚ +1. 检查并确认你的选项。然åŽå•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md new file mode 100644 index 00000000000..ede4d2dad7a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md @@ -0,0 +1,329 @@ +--- +title: 创建 EKS 集群 +--- + +Amazon EKS 为 Kubernetes 集群æ供托管的 controlplane。Amazon EKS 跨多个å¯ç”¨åŒºè¿è¡Œ Kubernetes controlplane 实例，以确ä¿é«˜å¯ç”¨æ€§ã€‚Rancher æ供了一个直观的用户界é¢ï¼Œç”¨äºŽç®¡ç†å’Œéƒ¨ç½²ä½ è¿è¡Œåœ¨ Amazon EKS 中的 Kubernetes 集群。通过本指å—，你将使用 Rancher 在你的 AWS 账户中快速轻æ¾åœ°å¯åŠ¨ Amazon EKS Kubernetes 集群。有关 Amazon EKS 的更多信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html)。 + + +## Amazon Web æœåŠ¡çš„先决æ¡ä»¶ + +:::caution + +部署到 Amazon AWS 会产生费用。有关详细信æ¯ï¼Œè¯·å‚阅 [EKS 定价页é¢](https://aws.amazon.com/eks/pricing/)。 + +::: + +è¦åœ¨ EKS 上设置集群，你需è¦è®¾ç½® Amazon VPC（虚拟ç§æœ‰äº‘）。你还需è¦ç¡®ä¿ç”¨äºŽåˆ›å»º EKS 集群的账å·å…·æœ‰é€‚当的[æƒé™](#最å°-eks-æƒé™)。详情请å‚阅 [Amazon EKS 先决æ¡ä»¶å®˜æ–¹æŒ‡å—](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-console.html#eks-prereqs)。 + +### Amazon VPC + +你需è¦å»ºç«‹ä¸€ä¸ª Amazon VPC æ¥å¯åŠ¨ EKS 集群。VPC 使你能够将 AWS 资æºå¯åŠ¨åˆ°ä½ å®šä¹‰çš„虚拟网络中。你å¯ä»¥è‡ªå·±è®¾ç½®ä¸€ä¸ª VPC，并在 Rancher 中创建集群时æ供它。如果你创建过程中没有æ供，Rancher 将创建一个 VPC。详情请å‚阅[教程：为你的 Amazon EKS 集群创建具有公有和ç§æœ‰å­ç½‘çš„ VPC](https://docs.aws.amazon.com/eks/latest/userguide/create-public-private-vpc.html)。 + +### IAM ç­–ç•¥ + +Rancher 需è¦è®¿é—®ä½ çš„ AWS 账户æ‰èƒ½åœ¨ Amazon EKS 中预置和管ç†ä½ çš„ Kubernetes 集群。你需è¦åœ¨ AWS 账户中为 Rancher 创建一个用户，并定义该用户å¯ä»¥è®¿é—®çš„内容。 + +1. 按照[此处](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html)的步骤创建具有编程访问æƒé™çš„用户。 + +2. 创建一个 IAM 策略，定义该用户在 AWS 账户中有æƒè®¿é—®çš„内容。请务必仅授予此用户所需的最å°è®¿é—®æƒé™ã€‚[此处](#最å°-eks-æƒé™)列出了 EKS 集群所需的最低æƒé™ã€‚请按照[此处](https://docs.aws.amazon.com/eks/latest/userguide/EKS_IAM_user_policies.html)的步骤创建 IAM 策略并将策略绑定到你的用户。 + +3. 最åŽï¼ŒæŒ‰ç…§[此处](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)的步骤为该用户创建访问密钥和密文密钥。 + +:::note é‡è¦æ示： + +定期轮æ¢è®¿é—®å¯†é’¥å’Œå¯†æ–‡å¯†é’¥éžå¸¸é‡è¦ã€‚有关详细信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#rotating_access_keys_console)。 + +::: + +有关 EKS çš„ IAM 策略的更多详细信æ¯ï¼Œè¯·å‚阅 [Amazon EKS IAM ç­–ç•¥ã€è§’色和æƒé™çš„官方文档](https://docs.aws.amazon.com/eks/latest/userguide/IAM_policies.html)。 + + +## 创建 EKS 集群 + +使用 Rancher é…置你的 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 选择 **Amazon EKS**。 +1. 输入**集群å称**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 完æˆè¡¨å•çš„其余部分。如需帮助，请å‚阅[é…ç½®å‚考](#eks-集群é…ç½®å‚考)。 +1. å•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## EKS 集群é…ç½®å‚考 + +有关 EKS 集群é…置选项的完整列表，请å‚阅[此页é¢](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md)。 + +## 架构 + +下图展示了 Rancher 2.x 的上层架构。下图中，Rancher Server 管ç†ä¸¤ä¸ª Kubernetes 集群，其中一个由 RKE 创建，å¦ä¸€ä¸ªç”± EKS 创建。 + +
通过 Rancher 的认è¯ä»£ç†ç®¡ç† Kubernetes 集群
+ +![架构](/img/rancher-architecture-rancher-api-server.svg) + +## AWS æœåŠ¡äº‹ä»¶ + +有关 AWS æœåŠ¡äº‹ä»¶çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](https://status.aws.amazon.com/)。 + +## 安全与åˆè§„ + +默认情况下，åªæœ‰åˆ›å»ºé›†ç¾¤çš„ IAM 用户或角色æ‰èƒ½è®¿é—®è¯¥é›†ç¾¤ã€‚在没有é¢å¤–é…置的情况下，使用其他用户或角色访问集群将导致错误。在 Rancher 中，这æ„味ç€ä½¿ç”¨æ˜ å°„到未用于创建集群的用户或角色的凭è¯ï¼Œå¯¼è‡´æœªç»æŽˆæƒçš„错误。除éžç”¨äºŽæ³¨å†Œé›†ç¾¤çš„凭è¯ä¸Ž EKSCtl 使用的角色或用户匹é…，å¦åˆ™ EKSCtl 集群将ä¸ä¼šæ³¨å†Œåˆ° Rancher。通过将其他用户和角色添加到 kube-system 命å空间中的 aws-auth configmap，å¯ä»¥æŽˆæƒå…¶ä»–用户和角色访问集群。如需更深入的解释和详细说明，请å‚阅此[文档](https://aws.amazon.com/premiumsupport/knowledge-center/amazon-eks-cluster-access/)。 + +有关 Amazon EKS Kubernetes 集群的安全性和åˆè§„性的更多信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/eks/latest/userguide/shared-responsibilty.html)。 + +## 教程 + +AWS å¼€æºåšå®¢ä¸Šçš„这篇[教程](https://aws.amazon.com/blogs/opensource/managing-eks-clusters-rancher/)将指导你使用 Rancher 设置一个 EKS 集群，部署一个å¯å…¬å¼€è®¿é—®çš„示例应用æ¥æµ‹è¯•é›†ç¾¤ï¼Œå¹¶éƒ¨ç½²ä¸€ä¸ªä½¿ç”¨å…¶ä»–å¼€æºè½¯ä»¶ï¼ˆå¦‚ Grafana å’Œ influxdb）æ¥å®žæ—¶ç›‘控地ç†ä¿¡æ¯çš„示例项目。 + +## æœ€å° EKS æƒé™ + +这些是访问 Rancher EKS 驱动程åºçš„全部功能所需的最低æƒé™é›†ã€‚ä½ éœ€è¦ Rancher 的其他æƒé™æ‰èƒ½é…ç½® `Service Role` å’Œ `VPC` 资æºã€‚如果你在创建集群**之å‰**创建了这些资æºï¼Œä½ åœ¨é…置集群时将å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºã€‚ + +| èµ„æº | æè¿° | +---------|------------ +| æœåŠ¡è§’色 | æä¾›å…许 Kubernetes 代表你管ç†èµ„æºçš„æƒé™ã€‚Rancher å¯ä»¥ä½¿ç”¨ä»¥ä¸‹[æœåŠ¡è§’色æƒé™](#æœåŠ¡è§’色æƒé™)æ¥åˆ›å»ºæœåŠ¡è§’色。 | +| VPC | æä¾› EKS å’Œ Worker 节点使用的隔离网络资æºã€‚Rancher 使用以下 [VPC æƒé™](#vpc-æƒé™)创建 VPC 资æºã€‚ | + + +资æºå®šä½ä½¿ç”¨ `*` 作为在 Rancher 中创建 EKS 集群之å‰ï¼Œæ— æ³•å·²çŸ¥åˆ›å»ºçš„资æºçš„å称（ARN）。 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "EC2Permissions", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:RevokeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:DescribeInstanceTypes", + "ec2:DescribeRegions", + "ec2:DescribeVpcs", + "ec2:DescribeTags", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeRouteTables", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeKeyPairs", + "ec2:DescribeInternetGateways", + "ec2:DescribeImages", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeAccountAttributes", + "ec2:DeleteTags", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteSecurityGroup", + "ec2:DeleteKeyPair", + "ec2:CreateTags", + "ec2:CreateSecurityGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateLaunchTemplate", + "ec2:CreateKeyPair", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress" + ], + "Resource": "*" + }, + { + "Sid": "CloudFormationPermissions", + "Effect": "Allow", + "Action": [ + "cloudformation:ListStacks", + "cloudformation:ListStackResources", + "cloudformation:DescribeStacks", + "cloudformation:DescribeStackResources", + "cloudformation:DescribeStackResource", + "cloudformation:DeleteStack", + "cloudformation:CreateStackSet", + "cloudformation:CreateStack" + ], + "Resource": "*" + }, + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:ListInstanceProfilesForRole", + "iam:ListInstanceProfiles", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "iam:GetInstanceProfile", + "iam:DetachRolePolicy", + "iam:DeleteRole", + "iam:CreateRole", + "iam:AttachRolePolicy" + ], + "Resource": "*" + }, + { + "Sid": "KMSPermissions", + "Effect": "Allow", + "Action": "kms:ListKeys", + "Resource": "*" + }, + { + "Sid": "EKSPermissions", + "Effect": "Allow", + "Action": [ + "eks:UpdateNodegroupVersion", + "eks:UpdateNodegroupConfig", + "eks:UpdateClusterVersion", + "eks:UpdateClusterConfig", + "eks:UntagResource", + "eks:TagResource", + "eks:ListUpdates", + "eks:ListTagsForResource", + "eks:ListNodegroups", + "eks:ListFargateProfiles", + "eks:ListClusters", + "eks:DescribeUpdate", + "eks:DescribeNodegroup", + "eks:DescribeFargateProfile", + "eks:DescribeCluster", + "eks:DeleteNodegroup", + "eks:DeleteFargateProfile", + "eks:DeleteCluster", + "eks:CreateNodegroup", + "eks:CreateFargateProfile", + "eks:CreateCluster" + ], + "Resource": "*" + } + ] +} +``` + +### æœåŠ¡è§’色æƒé™ + +这些是 EK​​S 集群创建期间所需的æƒé™ï¼Œä»¥ä¾¿ Rancher å¯ä»¥ä»£è¡¨ç”¨æˆ·åˆ›å»ºæœåŠ¡è§’色。 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreateRole", + "iam:CreateServiceLinkedRole", + "iam:DeleteInstanceProfile", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfiles", + "iam:ListInstanceProfilesForRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile" + ], + "Resource": "*" + } + ] +} +``` + +当你创建 EKS 集群时，Rancher 会创建一个具有以下信任策略的æœåŠ¡è§’色： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "eks.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} +``` + +此角色还有两个角色策略，它们具有以下策略的 ARN： + +``` +arn:aws:iam::aws:policy/AmazonEKSClusterPolicy +arn:aws:iam::aws:policy/AmazonEKSServicePolicy +``` + +### VPC æƒé™ + +这些是 Rancher 创建虚拟ç§æœ‰äº‘ (VPC) 和相关资æºæ‰€éœ€çš„æƒé™ã€‚ + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VPCPermissions", + "Effect": "Allow", + "Action": [ + "ec2:ReplaceRoute", + "ec2:ModifyVpcAttribute", + "ec2:ModifySubnetAttribute", + "ec2:DisassociateRouteTable", + "ec2:DetachInternetGateway", + "ec2:DescribeVpcs", + "ec2:DeleteVpc", + "ec2:DeleteTags", + "ec2:DeleteSubnet", + "ec2:DeleteRouteTable", + "ec2:DeleteRoute", + "ec2:DeleteInternetGateway", + "ec2:CreateVpc", + "ec2:CreateSubnet", + "ec2:CreateSecurityGroup", + "ec2:CreateRouteTable", + "ec2:CreateRoute", + "ec2:CreateInternetGateway", + "ec2:AttachInternetGateway", + "ec2:AssociateRouteTable" + ], + "Resource": "*" + } + ] +} +``` + +## åŒæ­¥ + +EKS é…置者å¯ä»¥åœ¨ Rancher å’Œæ供商之间åŒæ­¥ EKS 集群的状æ€ã€‚有关其工作原ç†çš„技术说明，请å‚阅[åŒæ­¥](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md)。 + +有关é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[本节](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md#é…置刷新间隔)。 + +## 故障排除 + +如果你的更改被覆盖，å¯èƒ½æ˜¯é›†ç¾¤æ•°æ®ä¸Ž EKS åŒæ­¥çš„æ–¹å¼å¯¼è‡´çš„。ä¸è¦åœ¨ä½¿ç”¨å…¶ä»–æºï¼ˆä¾‹å¦‚ EKS 控制å°ï¼‰å¯¹é›†ç¾¤è¿›è¡Œæ›´æ”¹åŽï¼Œåˆåœ¨äº”分钟之内在 Rancher 中进行更改。有关其工作原ç†ï¼Œä»¥åŠå¦‚何é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[åŒæ­¥](#åŒæ­¥)。 + +如果在修改或注册集群时返回未ç»æŽˆæƒçš„错误，并且集群ä¸æ˜¯ä½¿ç”¨ä½ çš„凭è¯æ‰€å±žçš„角色或用户创建的，请å‚阅[安全与åˆè§„](#安全与åˆè§„)。 + +有关 Amazon EKS Kubernetes 集群的任何问题或故障排除详细信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/eks/latest/userguide/troubleshooting.html)。 + +## 以编程方å¼åˆ›å»º EKS 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² EKS 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md new file mode 100644 index 00000000000..ee3f3397dc6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md @@ -0,0 +1,101 @@ +--- +title: 创建 GKE 集群 +--- + +## 先决æ¡ä»¶ + +你需è¦åœ¨ Google Kubernetes Engine 中进行一些设置。 + +### ServiceAccount 令牌 + +使用 [Google Kubernetes Engine](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts) 创建æœåŠ¡å¸å·ã€‚GKE 使用这个账å·æ¥æ“作你的集群。创建此账å·è¿˜ä¼šç”Ÿæˆç”¨äºŽèº«ä»½éªŒè¯çš„ç§é’¥ã€‚ + +该 ServiceAccount 需è¦ä»¥ä¸‹è§’色： + +- **Compute Viewer:** `roles/compute.viewer` +- **Project Viewer:** `roles/viewer` +- **Kubernetes Engine Admin:** `roles/container.admin` +- **Service Account User:** `roles/iam.serviceAccountUser` + +[Google 文档：创建和å¯ç”¨ ServiceAccount](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) + +æœ‰å…³èŽ·å– ServiceAccount çš„ç§é’¥çš„详细信æ¯ï¼Œè¯·å‚阅[æ­¤ Google 云文档](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys)。你需è¦å°†å¯†é’¥ä¿å­˜ä¸º JSON æ ¼å¼ã€‚ + +### Google 项目 ID + +你的集群需è¦åŒ…括到谷歌项目中。 + +è¦åˆ›å»ºæ–°é¡¹ç›®ï¼Œè¯·å‚阅[æ­¤ Google 云文档](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project)。 + +è¦èŽ·å–现有项目的项目 ID，请å‚阅[æ­¤ Google 云文档](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects)。 + +## é…ç½® GKE 集群 + +:::caution + +部署到 GKE 会产生费用。 + +::: + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. 输入你 Google 云凭è¯çš„å称。 +1. 在 **ServiceAccount** 文本框中，粘贴你的 ServiceAccount ç§é’¥ JSON，或上传 JSON 文件。 +1. å•å‡»**创建**。 + +**结果**：你已创建 Rancher 用于é…置新 GKE 集群的凭è¯ã€‚ + +### 2. 创建 GKE 集群 +使用 Rancher é…置你的 Kubernetes 集群。è¦ä½¿ç”¨ Rancher æˆåŠŸåˆ›å»º GKE 集群，GKE 必须处于 Standard 模å¼ã€‚GKE 在创建 Kubernetes 集群时有两ç§è¿è¡Œæ¨¡å¼ï¼Œåˆ†åˆ«æ˜¯ Autopilot å’Œ Standard 模å¼ã€‚Autopilot 模å¼çš„集群é…置对编辑 kube-system 命å空间有é™åˆ¶ã€‚但是，Rancher 在安装时需è¦åœ¨ kube-system 命å空间中创建资æºã€‚因此，你将无法创建 Autopilot 模å¼çš„ GKE 集群。如需详细了解 GKE Autopilot 模å¼å’Œ Standard 模å¼ä¹‹é—´çš„差异，请访问[比较 GKE Autopilot å’Œ Standard ](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison)。 + + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Google GKE**。 +1. 输入**集群å称**。 +1. å¯é€‰ï¼šä½¿ç”¨**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å¯é€‰ï¼šå°† Kubernetes [标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)或[注释](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)添加到集群。 +1. 输入你的 Google 项目 ID å’Œ Google 云凭è¯ã€‚ +1. 完æˆè¡¨å•çš„其余部分。如需帮助，请å‚阅 [GKE 集群é…ç½®å‚考](../../../../pages-for-subheaders/gke-cluster-configuration.md)。 +1. å•å‡»**创建**。 + +**结果**：你已æˆåŠŸéƒ¨ç½² GKE 集群。 + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## ç§æœ‰é›†ç¾¤ + +支æŒç§æœ‰ GKE 集群。注æ„，此高级设置å¯èƒ½æ¶‰åŠæ›´å¤šæ­¥éª¤ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md)。 + +## é…ç½®å‚考 + +有关在 Rancher 中é…ç½® GKE 集群的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../../pages-for-subheaders/gke-cluster-configuration.md)。 +## æ›´æ–° Kubernetes 版本 + +集群的 Kubernetes 版本å¯ä»¥å‡çº§åˆ° GKE 集群所在区域或地区中å¯ç”¨çš„任何版本。å‡çº§ Kubernetes 主版本ä¸ä¼šè‡ªåŠ¨å‡çº§ Worker 节点。节点å¯ä»¥ç‹¬ç«‹å‡çº§ã€‚ + +:::note + +GKE 在 1.19+ 中å–消了基本身份验è¯ã€‚è¦å°†é›†ç¾¤å‡çº§åˆ° 1.19+，必须在谷歌云中ç¦ç”¨åŸºæœ¬èº«ä»½è®¤è¯ã€‚å¦åˆ™ï¼Œåœ¨å‡çº§åˆ° 1.19+ 时，Rancher 会出现错误。你å¯ä»¥æŒ‰ç…§ [Google 文档](https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#disabling_authentication_with_a_static_password)进行æ“作。然åŽï¼Œä½ å°±å¯ä»¥ä½¿ç”¨ Rancher å°† Kubernetes 版本更新到 1.19+。 + +::: + +## åŒæ­¥ + +GKE é…置者å¯ä»¥åœ¨ Rancher å’Œæ供商之间åŒæ­¥ GKE 集群的状æ€ã€‚有关其工作原ç†çš„技术说明，请å‚阅[åŒæ­¥](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md)。 + +有关é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/gke-cluster-configuration.md#é…置刷新间隔)。 + +## 以编程方å¼åˆ›å»º GKE 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² GKE 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md new file mode 100644 index 00000000000..b6244c617d6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md @@ -0,0 +1,100 @@ +--- +title: 创建åŽä¸º CCE 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在åŽä¸ºäº‘容器引擎 (CCE) 中的集群。Rancher å·²ç»ä¸º CCE 实现并打包了针对 CCE çš„[集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md)，但是默认情况下，这个集群驱动的状æ€æ˜¯ `inactive`。为了å¯åŠ¨ CCE 集群，你需è¦[å¯ç”¨ CCE 集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#激活åœç”¨é›†ç¾¤é©±åŠ¨)。å¯ç”¨é›†ç¾¤é©±åŠ¨åŽï¼Œä½ å¯ä»¥å¼€å§‹é…ç½® CCE 集群。 + +## åŽä¸ºå…ˆå†³æ¡ä»¶ + +:::caution + +部署到 CCE 会产生费用。 + +::: + +1. 在åŽä¸º CCE 门户中找到你的项目 ID。请å‚阅 CCE 文档以了解如何[管ç†ä½ çš„项目](https://doc.hcs.huawei.com/api/cce/cce_02_0341.html#cce_02_0341__section17571944123214)。 + +2. 创建一个[访问密钥 ID 和密文访问密钥](https://support.huaweicloud.com/en-us/usermanual-iam/en-us_topic_0079477318.html)。 + +## Rancher 先决æ¡ä»¶ + +你需è¦å¯ç”¨åŽä¸º CCE 集群驱动： + +1. 点击 **☰ > 集群管ç†**。 +1. 点击**驱动**。 +1. 在**集群驱动**选项å¡ä¸­ï¼Œè½¬åˆ° **Huawei CCE** 集群驱动并å•å‡» **â‹® > 激活**。 + +集群驱动下载完æˆåŽï¼Œå°±å¯ä»¥åœ¨ Rancher 中创建åŽä¸º CCE 集群了。 + +## é™åˆ¶ + +åŽä¸º CCE æœåŠ¡ä¸æ”¯æŒé€šè¿‡å…¶ API 创建具有公共访问æƒé™çš„集群。你需è¦åœ¨è¦é…置的 CCE é›†ç¾¤çš„ç›¸åŒ VPC 中è¿è¡Œ Rancher。 + +## 创建 CCE 集群 + +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. å•å‡» **Huawei CCE**。 +1. 输入**集群å称**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 输入**项目 ID**，访问密钥 ID，**Access Key**，和密文访问密钥 **Secret Key**。然åŽç‚¹å‡»**下一步：é…置集群**。填写集群é…置。有关填写表å•çš„帮助，请å‚阅[åŽä¸º CCE é…ç½®](#åŽä¸º-cce-é…ç½®)。 +1. 填写集群的以下节点é…置。有关填写表å•çš„帮助，请å‚阅[节点é…ç½®](#节点é…ç½®)。 +1. 点击**创建**æ¥åˆ›å»º CCE 集群。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## åŽä¸º CCE é…ç½® + +| 设置 | æè¿° | +|---|---| +| 集群类型 | è¦åŒ…å«åˆ°é›†ç¾¤çš„节点类型，å¯ä»¥æ˜¯ `VirtualMachine` 或 `BareMetal`。 | +| æè¿° | 集群的æ述。 | +| 主版本 | Kubernetes 版本。 | +| 管ç†è§„æ¨¡æ•°é‡ | 集群的最大节点数。选项为 50ã€200 å’Œ 1000。规模计数越大，æˆæœ¬è¶Šé«˜ã€‚ | +| 高å¯ç”¨æ€§ | å¯ç”¨ master 节点的高å¯ç”¨æ€§ã€‚å¯ç”¨é«˜å¯ç”¨æ€§çš„集群æˆæœ¬ä¼šæ›´é«˜ã€‚ | +| å®¹å™¨ç½‘ç»œæ¨¡å¼ | 集群中使用的网络模å¼ã€‚`VirtualMachine` æ”¯æŒ `overlay_l2` å’Œ `vpc-router`，而 `BareMetal` æ”¯æŒ `underlay_ipvlan`。 | +| 容器网络 CIDR | 集群的网络 CIDR。 | +| VPC å称 | è¦éƒ¨ç½²é›†ç¾¤çš„ VPC å称。如果留空，Rancher 将创建一个。 | +| å­ç½‘å称 | è¦éƒ¨ç½²é›†ç¾¤çš„å­ç½‘å称。如果留空，Rancher 将创建一个。 | +| 外部æœåŠ¡å™¨ | 预留选项，用于通过 API å¯ç”¨ CCE 集群的公共访问。这个选项暂时是一直ç¦ç”¨çš„。 | +| 集群标签 | 集群的标签。 | +| 高速å­ç½‘ | åªæœ‰ `BareMetal` 支æŒè¯¥é€‰é¡¹ã€‚裸机è¦æ±‚选择网速高的 VPC。 | + +:::note + +如果你在 `cluster.yml` 中编辑集群，而ä¸æ˜¯ä½¿ç”¨ Rancher UI，则集群é…ç½®å‚数必须嵌套在 `cluster.yml` 中的 `rancher_kubernetes_engine_config` 中。有关详细信æ¯ï¼Œè¯·å‚阅[é…置文件结构](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rancher-中的é…置文件结构)。 + +::: + +## 节点é…ç½® + +| 设置 | æè¿° | +|---|---| +| 地区 | 部署集群节点的å¯ç”¨åŒºåŸŸã€‚ | +| è®¡è´¹æ–¹å¼ | 集群节点的计费模å¼ã€‚`VirtualMachine` ä»…æ”¯æŒ `Pay-per-use`。`BareMetal` æ”¯æŒ `Pay-per-use` 或 `Yearly/Monthly`。 | +| 有效期 | 该选项仅在 `Yearly/Monthly` 计费模å¼ä¸­æ˜¾ç¤ºã€‚表示支付集群节点费用的时间。 | +| 自动续期 | 该选项仅在 `Yearly/Monthly` 计费模å¼ä¸­æ˜¾ç¤ºã€‚表示是å¦ä¸ºé›†ç¾¤èŠ‚点自动续期 `Yearly/Monthly` 计费。 | +| æ•°æ®å·ç±»åž‹ | 集群节点的数æ®å·ç±»åž‹ã€‚å¯é€‰ `SATA`，`SSD` 或 `SAS`。 | +| æ•°æ®é‡å¤§å° | 集群节点的数æ®å·å¤§å°ã€‚ | +| æ ¹å·ç±»åž‹ | 集群节点的根å·ç±»åž‹ã€‚å¯é€‰ `SATA`，`SSD` 或 `SAS`。 | +| æ ¹å·å¤§å° | 集群节点的根å·å¤§å°ã€‚ | +| 节点风格 | 集群节点的节点风格。Rancher UI 中的风格列表å–自åŽä¸ºäº‘。其中包括所有支æŒçš„节点风格。 | +| 节点数 | 集群的节点数 | +| 节点æ“作系统 | 集群节点的æ“作系统。目å‰ä»…æ”¯æŒ `EulerOS 2.2` å’Œ `CentOS 7.4`。 | +| SSH 密钥å称 | 集群节点的 SSH 密钥 | +| EIP | 集群节点的公共 IP 选项。`å·²ç¦ç”¨`表示集群节点ä¸ä¼šç»‘定公共 IP。`创建 EIP`表示集群节点在é…ç½®åŽå°†ç»‘定一个或多个新创建的 EIP，UI 中将显示更多用æ¥åˆ›å»º EIP å‚数的选项。`选择现有 EIP` 表示节点将绑定到你选择的 EIP。 | +| EIP æ•°é‡ | 此选项仅在选择`创建 EIP`时显示。表示你è¦ä¸ºèŠ‚点创建的 EIP æ•°é‡ã€‚ | +| EIP 类型 | 此选项仅在选择`创建 EIP`时显示。å¯é€‰ `5_bgp` å’Œ `5_sbgp`。 | +| EIP 共享类型 | 此选项仅在选择`创建 EIP`时显示。仅å¯é€‰ `PER`。 | +| EIP æ”¶è´¹æ¨¡å¼ | 此选项仅在选择`创建 EIP`时显示。选择按照`带宽`或`æµé‡`计费。 | +| EIP å¸¦å®½å¤§å° | 此选项仅在选择`创建 EIP`时显示。EIP 的带宽。 | +| 身份验è¯æ¨¡å¼ | 表示å¯ç”¨ `RBAC`，或åŒæ—¶å¯ç”¨`认è¯ä»£ç†`。如果选择`认è¯ä»£ç†`，则还需è¦ç”¨äºŽéªŒè¯ä»£ç†çš„è¯ä¹¦ã€‚ | +| 节点标签 | 集群节点的标签。无效标签会阻止å‡çº§ï¼Œæˆ–阻止 Rancher å¯åŠ¨ã€‚有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md new file mode 100644 index 00000000000..13349de2c08 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md @@ -0,0 +1,99 @@ +--- +title: 创建腾讯 TKE 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在腾讯 Tencent Kubernetes Engine (TKE) 中的集群。Rancher å·²ç»ä¸º TKE 实现并打包了针对 TKE çš„[集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md)，但是默认情况下，这个集群驱动的状æ€æ˜¯ `inactive`。为了å¯åŠ¨ TKE 集群，你需è¦[å¯ç”¨ TKE 集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#激活åœç”¨é›†ç¾¤é©±åŠ¨)。å¯ç”¨é›†ç¾¤é©±åŠ¨åŽï¼Œä½ å¯ä»¥å¼€å§‹é…ç½® TKE 集群。 + +## 腾讯先决æ¡ä»¶ + +:::caution + +部署到 TKE 会产生费用。 + +::: + +1. å‚è§[云访问管ç†](https://intl.cloud.tencent.com/document/product/598/10600)文档，确ä¿ç”¨äºŽåˆ›å»º TKE 集群的账å·å…·æœ‰é€‚当的æƒé™ã€‚ + +2. 创建[云 API 密钥 ID 和密钥](https://console.cloud.tencent.com/capi)。 + +3. 在è¦éƒ¨ç½² Kubernetes 集群的区域中创建[ç§æœ‰ç½‘络和å­ç½‘](https://intl.cloud.tencent.com/document/product/215/4927)。 + +4. 创建 [SSH 密钥对](https://intl.cloud.tencent.com/document/product/213/6092)。该密钥用于访问 Kubernetes 集群中的节点。 + +## Rancher 先决æ¡ä»¶ + +你需è¦å¯ç”¨è…¾è®¯ TKE 集群驱动： + +1. 点击 **☰ > 集群管ç†**。 +1. 点击**驱动**。 +1. 在**集群驱动**选项å¡ä¸­ï¼Œè½¬åˆ° **Tencent TKE** 集群驱动并å•å‡» **â‹® > 激活**。 + +集群驱动下载完æˆåŽï¼Œå°±å¯ä»¥åœ¨ Rancher 中创建腾讯 TKE 集群了。 + +## 创建 TKE 集群 + +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 + +2. 选择 **Tencent TKE**。 + +3. 输入**集群å称**。 + +4. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ + +5. 为 TKE 集群é…ç½®**è´¦å·è®¿é—®**。使用[先决æ¡ä»¶](#腾讯先决æ¡ä»¶)中获得的信æ¯å®Œæˆæ¯ä¸ªä¸‹æ‹‰åˆ—表和字段。 + + | 选项 | æè¿° | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | 区域 | 从下拉列表中选择构建集群的地ç†åŒºåŸŸã€‚ | + | 密文 ID | 输入从腾讯云控制å°èŽ·å–的密文 ID。 | + | 密文密钥 | 输入从腾讯云控制å°èŽ·å–的密文密钥。 | + +6. 然åŽï¼Œå•å‡»`下一步：é…置集群`æ¥é…ç½® TKE 集群。 + + | 选项 | æè¿° | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Kubernetes 版本 | TKE ç›®å‰åªæ”¯æŒ Kubernetes 1.10.5。 | + | 节点数 | 输入è¦ä¸º Kubernetes 集群购买的 worker 节点数，最大值是 100。 | + | VPC | 选择在腾讯云控制å°ä¸­åˆ›å»ºçš„ VPC å称。 | + | 容器网络 CIDR | 输入 Kubernetes 集群的 CIDR 范围。你å¯ä»¥åœ¨è…¾è®¯äº‘控制å°çš„ VPC æœåŠ¡ä¸­æŸ¥çœ‹è¯¥ CIDR çš„å¯ç”¨èŒƒå›´ã€‚默认为 172.16.0.0/16。 | + + :::note + + 如果你在 `cluster.yml` 中编辑集群，而ä¸æ˜¯ä½¿ç”¨ Rancher UI，则集群é…ç½®å‚数必须嵌套在 `cluster.yml` 中的 `rancher_kubernetes_engine_config` 中。有关详细信æ¯ï¼Œè¯·å‚阅 [Rancher 2.3.0+ é…置文件结构](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rancher-中的é…置文件结构)。 + + ::: + +7. 点击`下一步：选择实例类型`，然åŽé€‰æ‹©å°†ç”¨äºŽ TKE 集群的实例类型。 + + | 选项 | æè¿° | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | å¯ç”¨åŒº | 选择 VPC 区域的å¯ç”¨åŒºã€‚ | + | å­ç½‘ | 选择你在 VPC 中创建的å­ç½‘。如果在所选å¯ç”¨åŒºä¸­æ²¡æœ‰è¯¥å­ç½‘，请添加一个新å­ç½‘。 | + | 实例类型 | 从下拉列表中选择è¦ç”¨äºŽ TKE 集群的 VM 实例类型，默认为 S2.MEDIUM4（CPU 2；内存 4 GiB）。 | + +8. 点击`下一步：é…置实例`，é…置用于 TKE 集群的 VM 实例。 + + | 选项 | æè¿° | + -------|------------ + | æ“作系统 | æ“作系统å称，目å‰æ”¯æŒ Centos7.2x86_64 或 ubuntu16.04.1 LTSx86_64。 | + | 安全组 | 安全组 ID，默认ä¸ç»‘定任何安全组。 | + | æ ¹ç£ç›˜ç±»åž‹ | 系统盘类型。系统盘类型é™åˆ¶è¯¦è§ [CVM 实例é…ç½®](https://cloud.tencent.com/document/product/213/11518)。 | + | æ ¹ç£ç›˜å¤§å° | 系统盘大å°ã€‚Linux 系统调整范围为 20-50 GB，步长为 1。 | + | æ•°æ®ç›˜ç±»åž‹ | æ•°æ®ç›˜ç±»åž‹ï¼Œé»˜è®¤ä¸º SSD 云盘。 | + | æ•°æ®ç£ç›˜å¤§å° | æ•°æ®ç›˜å¤§å°ï¼ˆGB），步长为 10。 | + | 带宽类型 | 带宽收费类型，PayByTraffic 或 PayByHour。 | + | 带宽 | 公网带宽 (Mbps) | + | 密钥对 | 密钥 ID，关è”åŽå¯ä»¥ç”¨æ¥ç™»å½• VM 节点。 | + +9. å•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md new file mode 100644 index 00000000000..3003cab8733 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md @@ -0,0 +1,41 @@ +--- +title: Azure 中存储类的é…ç½® +--- + +如果你的节点使用 Azure VM，则å¯ä»¥ä½¿ç”¨ [Azure 文件](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv)作为集群的存储类（StorageClass）。 + +为了让 Azure 创建所需的存储资æºï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤æ“作： + +1. [é…ç½® Azure 云æ供商](../../kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md)。 +1. é…ç½® `kubectl` 以连接到你的集群。 +1. å¤åˆ¶ ServiceAccount çš„ `ClusterRole` å’Œ `ClusterRoleBinding` 清å•ï¼š + ```yml + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:azure-cloud-provider + rules: + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:azure-cloud-provider + roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider + subjects: + - kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system + ``` + +1. 使用以下命令在集群中进行创建： + + ``` + # kubectl create -f + ``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md new file mode 100644 index 00000000000..3a5ac26045c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md @@ -0,0 +1,36 @@ +--- +title: Host Gateway (L2bridge) 的网络è¦æ±‚ +--- + +本节介ç»å¦‚何é…置使用 *Host Gateway (L2bridge)* 模å¼çš„自定义 Windows 集群。 + +### ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥ + +如果你使用 *Host Gateway (L2bridge)* 模å¼ï¼Œå¹¶å°†èŠ‚点托管在下é¢åˆ—出的云æœåŠ¡ä¸Šï¼Œåˆ™å¿…须在å¯åŠ¨æ—¶ç¦ç”¨ Linux 或 Windows 主机的ç§æœ‰ IP 地å€æ£€æŸ¥ã€‚è¦ä¸ºæ¯ä¸ªèŠ‚点ç¦ç”¨æ­¤æ£€æŸ¥ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹å„个云æœåŠ¡å¯¹åº”的说明进行æ“作： + +| æœåŠ¡ | ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥çš„说明 | +--------|------------------------------------------------ +| Amazon EC2 | [ç¦ç”¨æº/目标检查](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_NAT_Instance.html#EIP_Disable_SrcDestCheck) | +| Google GCE | [为实例å¯ç”¨ IP 转å‘](https://cloud.google.com/vpc/docs/using-routes#canipforward)（默认情况下，VM 无法转å‘ç”±å¦ä¸€ä¸ª VM å‘èµ·çš„æ•°æ®åŒ…） | +| Azure VM | [å¯ç”¨æˆ–ç¦ç”¨ IP 转å‘](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) | + +### 云托管虚拟机的路由é…ç½® + +如果是使用 Flannel çš„ [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) åŽç«¯ï¼ŒåŒä¸€èŠ‚点上的所有容器都属于ç§æœ‰å­ç½‘，æµé‡é€šè¿‡ä¸»æœºç½‘络从一个节点上的å­ç½‘路由到在å¦ä¸€ä¸ªèŠ‚点上的å­ç½‘。 + +- 在 AWSã€è™šæ‹ŸåŒ–集群或裸机æœåŠ¡å™¨ä¸Šé…ç½® worker 节点时，请确ä¿å®ƒä»¬å±žäºŽåŒä¸€ä¸ªç¬¬ 2 层å­ç½‘。如果节点ä¸å±žäºŽåŒä¸€ä¸ªç¬¬ 2 层å­ç½‘，`host-gw` 网络将ä¸èµ·ä½œç”¨ã€‚ + +- 在 GCE 或 Azure 上é…ç½® worker 节点时，节点ä¸åœ¨åŒä¸€ä¸ªç¬¬ 2 层å­ç½‘上。GCE å’Œ Azure 上的节点属于å¯è·¯ç”±çš„第 3 层网络。按照以下说明é…ç½® GCE å’Œ Azure，以便云网络知é“如何在æ¯ä¸ªèŠ‚点上传é€ä¸»æœºå­ç½‘。 + +è¦åœ¨ GCE 或 Azure 上é…置主机å­ç½‘路由，首先è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä»¥æ‰¾å‡ºæ¯ä¸ª worker 节点上的主机å­ç½‘： + +```bash +kubectl get nodes -o custom-columns=nodeName:.metadata.name,nodeIP:status.addresses[0].address,routeDestination:.spec.podCIDR +``` + +然åŽæŒ‰ç…§å„个云æ供商的说明，为æ¯ä¸ªèŠ‚点é…置路由规则： + +| æœåŠ¡ | 说明 | +--------|------------- +| Google GCE | 为æ¯ä¸ªèŠ‚点添加é™æ€è·¯ç”±ï¼š[添加é™æ€è·¯ç”±](https://cloud.google.com/vpc/docs/using-routes#addingroute)。 | +| Azure VM | 创建一个路由表：[自定义路由：用户定义](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview#user-defined)。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md new file mode 100644 index 00000000000..81d832577f2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md @@ -0,0 +1,42 @@ +--- +title: Windows å’Œ Linux 集群的功能奇å¶ä¸€è‡´æ€§ +--- + +Windows 集群与 Linux 集群的功能支æŒä¸ç›¸åŒã€‚ + +下图æ述了 Rancher 上 Windows å’Œ Linux 之间的功能奇å¶ä¸€è‡´æ€§ï¼š + +| **组件** | **Linux** | **Windows** | +--- | --- | --- +| **å‘行版** | | +| RKE | æ”¯æŒ | æ”¯æŒ | +| RKE2 | æ”¯æŒ | æ”¯æŒ | +| K3S | æ”¯æŒ | ä¸æ”¯æŒ | +| EKS | æ”¯æŒ | ä¸æ”¯æŒ | +| GKE | æ”¯æŒ | ä¸æ”¯æŒ | +| AKS | æ”¯æŒ | ä¸æ”¯æŒ | +| **Rancher 组件** | | +| Server | æ”¯æŒ | ä¸æ”¯æŒ | +| Agent | æ”¯æŒ | æ”¯æŒ | +| Fleet | æ”¯æŒ | æ”¯æŒ | +| EKS Operator | æ”¯æŒ | ä¸æ”¯æŒ | +| AKS Operator | ä¸æ”¯æŒ | ä¸æ”¯æŒ | +| GKE Operator | ä¸æ”¯æŒ | ä¸æ”¯æŒ | +| Monitoring/Alerting v2 | æ”¯æŒ | æ”¯æŒ | +| Logging v2 | æ”¯æŒ | æ”¯æŒ | +| Istio | æ”¯æŒ | ä¸æ”¯æŒ | +| Catalog v2 | æ”¯æŒ | ä¸æ”¯æŒ | +| OPA | æ”¯æŒ | ä¸æ”¯æŒ | +| Longhorn | æ”¯æŒ | ä¸æ”¯æŒ | +| CIS 扫æ | æ”¯æŒ | ä¸æ”¯æŒ | +| Backup/Restore Operator | æ”¯æŒ | ä¸æ”¯æŒ | +| **CNI/附加组件** | | +| Flannel | æ”¯æŒ | æ”¯æŒ | +| Canal | æ”¯æŒ | ä¸æ”¯æŒ | +| Calico | æ”¯æŒ | 支æŒï¼ˆä»… RKE2） | +| Cilium | æ”¯æŒ | ä¸æ”¯æŒ | +| Multus | æ”¯æŒ | ä¸æ”¯æŒ | +| Traefik | æ”¯æŒ | ä¸æ”¯æŒ | +| NGINX Ingress | æ”¯æŒ | ä¸æ”¯æŒ | + +有关功能支æŒçš„æ›´æ–°ä¿¡æ¯ï¼Œä½ å¯ä»¥è®¿é—® GitHub 上的 [rancher/windows](https://github.com/rancher/windows)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md new file mode 100644 index 00000000000..ce334bc5382 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md @@ -0,0 +1,303 @@ +--- +title: å°† Windows 工作负载从 RKE2 è¿ç§»åˆ° RKE2 +--- + +**注æ„**：Rancher Support çš„ SLA ä¸æ¶µç›–本文档的内容。因此，请谨慎继续。 + +本文档介ç»äº†å¦‚何将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2。 + +## RKE1 Windows 调度 + +RKE1 Windows 工作负载是基于污点和容å¿åº¦è°ƒåº¦çš„。 + +RKE1 Windows 集群中的æ¯ä¸ª Linux 节点（无论分é…了什么角色）都会有一个默认污点，除éžå·¥ä½œè´Ÿè½½é…置了容å¿åº¦ï¼Œå¦åˆ™è¯¥æ±¡ç‚¹ä¼šé˜»æ­¢å·¥ä½œè´Ÿè½½è°ƒåº¦åˆ°èŠ‚点上。这是 RKE1 Windows 集群的一个主è¦ç‰¹æ€§ï¼Œç›®çš„是为了仅è¿è¡Œ Windows 工作负载。 + +- 默认 RKE1 Linux 节点 `NoSchedule` 污点： + +```yml +apiVersion: v1 +kind: Node +spec: + ... + taints: + - effect: NoSchedule + key: cattle.io/os + value: linux +``` +
+ +- RKE1 Linux `NoSchedule` 对工作负载的容å¿åº¦ + +以下容å¿åº¦å…许将最终用户的工作负载调度到 RKE1 Windows 集群中的任何 Linux 节点上。这些容å¿åº¦ç”¨äºŽæ ¸å¿ƒçš„ Rancher æœåŠ¡å’Œå·¥ä½œè´Ÿè½½ï¼š + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` +
+ +- æ ¹æ®æœ€ä½³å®žè·µï¼Œåœ¨ Linux 节点上è¿è¡Œçš„任何最终用户工作负载都将仅调度到具有 worker 角色的节点上： + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + weight: 100 + ... +``` + +## RKE2 Windows 调度 + +æ ¹æ®å¯¹æ··åˆå·¥ä½œè´Ÿè½½æ”¯æŒçš„å馈和请求，RKE2 Windows é»˜è®¤æ”¯æŒ Linux å’Œ Windows 工作负载。RKE2 调度默认ä¾èµ–节点选择器。这是ä¸åŒäºŽ RKE1 的一个显ç€å˜åŒ–，因为 RKE2 中没有包å«æ±¡ç‚¹å’Œå®¹å¿ã€‚节点选择器是 RKE1 Windows 集群的关键部分，å¯ä»¥å¸®åŠ©ä½ è½»æ¾è¿ç§»å·¥ä½œè´Ÿè½½ã€‚ + +## 示例è¿ç§» + +### å°† Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 + +- è¿ç§»å‰çš„ RKE1 Windows Deployment： + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - linux +``` +
+ +- 使用 `nodeAffinity` è¿ç§»åŽçš„ RKE2 Windows Deployment： + +```yaml +apiVersion: apps/v1 +kind: Deployment +... +spec: + ... + template: + ... + spec: + ... + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows +``` + +### RKE1 Windows 集群的 Linux-Only Deployment + +> **é‡è¦æ示**：在利用 `nodeSelector` å’Œ `nodeAffinity` 时，请注æ„以下几点： +> +> - 如果åŒæ—¶æŒ‡å®šäº† `nodeSelector` å’Œ `nodeAffinity`，则必须满足这两个æ¡ä»¶æ‰èƒ½å°† `Pod` 调度到节点上。 +> - 如果指定了与å•ä¸ª `nodeSelectorTerms` å…³è”的多个 `matchExpressions`，则åªæœ‰å½“所有 `matchExpressions` 都符åˆè¦æ±‚æ—¶æ‰ä¼šè°ƒåº¦ `Pod`。 + +
+ +- è¿ç§»å‰çš„ RKE1 Windows 集群 Linux-only Deployment（针对 RKE1 Linux Worker 节点）： + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + +``` +
+ +- 使用 `nodeSelector` è¿ç§»åŽçš„ RKE2 æ··åˆé›†ç¾¤ Linux-only Deployment（针对 RKE2 Linux Worker 节点） + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + nodeSelector: + kubernetes.io/os: "linux" + node-role.kubernetes.io/worker: "true" +``` +
+ +- 使用 `nodeAffinity` è¿ç§»åŽçš„ RKE2 æ··åˆé›†ç¾¤ Linux-only Deployment（针对 RKE2 Linux Worker 节点） + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux +``` +## RKE1 Windows - 支æŒçš„ Windows Server 版本 + +### 长期æœåŠ¡æ¸ é“（LTSC） + +- Windows Server 2019 LTSC ✅ 将于 2024 å¹´ 1 月 9 æ—¥ 达到 Mainstream EOL 并于 2029 å¹´ 1 月 9 æ—¥ 达到 Extended EOL + +### åŠå¹´é¢‘é“ (SAC) + +- Windows Server 20H2 SAC ❌ 2022 å¹´ 8 月 9 日已达到 EOL +- Windows Server 2004 SAC ❌ 2021 å¹´ 12 月 14 日已达到 EOL +- Windows Server 1909 SAC ❌ 2021 å¹´ 5 月 11 日已达到 EOL +- Windows Server 1903 SAC ❌ 2020 å¹´ 12 月 8 日已达到 EOL +- Windows Server 1809 SAC ❌ 2020 å¹´ 11 月 10 日已达到 EOL + +## RKE2 Windows - 支æŒçš„ Windows Server 版本 + +### RKE2 中的长期æœåŠ¡é€šé“（LTSC） + +- Windows Server 2019 LTSC ✅ 将于 2024 å¹´ 1 月 9 æ—¥ 达到 Mainstream EOL 并于 2029 å¹´ 1 月 9 æ—¥ 达到 Extended EOL +- Windows Server 2022 LTSC ✅ 将于 2026 å¹´ 10 月 13 日达到 Mainstream EOL，并于 2031 å¹´ 10 月 13 日达到 Extended EOL + +> **注æ„**：RKE2 ä¸æ”¯æŒ SAC。 + + +有关详细信æ¯ï¼Œè¯·å‚阅以下å‚考资料： + +- [Windows Server SAC 生命周期](https://docs.microsoft.com/en-us/lifecycle/products/windows-server) + +- [Windows Server 2022 LTSC 生命周期](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2022) + +- [Windows Server 2019 LTSC 生命周期](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2019) + + +## Kubernetes ç‰ˆæœ¬æ”¯æŒ + +> **注æ„**ï¼šæ ¹æ® [Rancher 2.6.7 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/rancher-v2-6-7/)，下é¢åˆ—出的所有版本å‡æ”¯æŒ SLA。未列出的版本都视为为已达到 EOL 且ä¸å— SUSE SLA 支æŒã€‚ + +### 对比 Rancher 2.5 与 Rancher 2.6：Windows 集群的支æŒçŸ©é˜µ + +**对比 RKE1 与 RKE2：Windows 集群支æŒçš„ Kubernetes 版本**： + +| Kubernetes 版本 | RKE1 | RKE2 | +|-------------- |:----: |:----: | +| 1.18 | ✓ | | +| 1.19 | ✓ | | +| 1.20 | ✓ | | +| 1.21 | ✓ | | +| 1.22 | ✓ | ✓ | +| 1.23 | | ✓ | +| 1.24 | | ✓ | +| 1.25+ | | ✓ | + + +### 对比 Rancher 2.5 与 Rancher 2.6：支æŒç”¨äºŽé…ç½® RKE1 å’Œ RKE2 Windows 集群的 Kubernetes 版本 + +| Rancher 版本 | Kubernetes 版本 | RKE1 | RKE2 | +|:-----------------------: |:------------------------: |:----: |:----: | +| 2.5 - RKE1 é…ç½® | 1.18 1.19 1.20 | ✓ | | +| 2.6 - RKE1 é…ç½® | 1.18 1.19 1.20 1.21 1.22 | ✓ | | +| 2.6 - RKE2 é…ç½® | 1.22 1.23 1.24 1.25+ | | ✓ | + + +## 将工作负载è¿ç§»åˆ° RKE2 Windows çš„æŒ‡å— + +å‚考[对比 Rancher 2.5 与 Rancher 2.6：Windows 集群的支æŒçŸ©é˜µ](#对比-rancher-25-与-rancher-26windows-集群的支æŒçŸ©é˜µ)å’Œ[对比 Rancher 2.5 与 Rancher 2.6：支æŒç”¨äºŽé…ç½® RKE1 å’Œ RKE2 Windows 集群的 Kubernetes 版本](#对比-rancher-25-与-rancher-26支æŒç”¨äºŽé…ç½®-rke1-å’Œ-rke2-windows-集群的-kubernetes-版本)中的表格，你会å‘现 RKE1 å’Œ RKE2 çš„ Kubernetes 1.22 版本å‘生了é‡å ã€‚因此，当éµå¾ª Rancher 推è的方法时，这将是è¿ç§» RKE1 Windows 工作负载所需的基本版本。 + +### Rancher 2.5 的就地å‡çº§ + +1. å°† Rancher 版本å‡çº§åˆ° 2.6.5+。 +1. 使用最新的å¯ç”¨è¡¥ä¸ç‰ˆæœ¬å°† RKE1 Windows 下游集群å‡çº§åˆ° RKE1 v1.22。 +1. 通过 RKE1 Windows 集群所在的匹é…è¡¥ä¸ç‰ˆæœ¬ï¼Œä½¿ç”¨ RKE2 v1.22 é…置新的 RKE2 Windows 下游集群。 +1. 开始将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 集群。 +1. 执行验è¯æµ‹è¯•ï¼Œç¡®ä¿åœ¨å°†åº”用程åºä»Ž RKE1 è¿ç§»åˆ° RKE2 时没有丢失或更改功能。 +1. 验è¯æµ‹è¯•æˆåŠŸåŽï¼Œä½ å¯ä»¥é€‰æ‹©å°† RKE2 1.22.x 集群å‡çº§åˆ°æ–°çš„次è¦ç‰ˆæœ¬ï¼Œä¾‹å¦‚ 1.23 或 1.24。 + + +### å°† Windows 工作负载è¿ç§»åˆ°æ–°çš„ Rancher 环境 + +> **é‡è¦æ示**：è¦æ‰§è¡Œä»¥ä¸‹ä»»ä¸€é€‰é¡¹ï¼Œä½ éœ€è¦ä½¿ç”¨ Rancher 2.6.5 或更高版本。 + +**为 RKE1 å’Œ RKE2 使用匹é…çš„ Kubernetes è¡¥ä¸ç‰ˆæœ¬æ—¶**： + +1. 通过 RKE1 Windows 集群所在的匹é…è¡¥ä¸ç‰ˆæœ¬ï¼Œä½¿ç”¨ RKE2 v1.22 é…置新的 RKE2 Windows 下游集群。 +1. 开始将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 集群。 +1. 执行验è¯æµ‹è¯•ï¼Œç¡®ä¿åœ¨å°†åº”用程åºä»Ž RKE1 è¿ç§»åˆ° RKE2 时没有丢失或更改功能。 +1. 验è¯æµ‹è¯•æˆåŠŸåŽï¼Œä½ å¯ä»¥é€‰æ‹©å°† RKE2 1.22.x 集群å‡çº§åˆ°æ–°çš„次è¦ç‰ˆæœ¬ï¼Œä¾‹å¦‚ 1.23 或 1.24。 + + +**为 RKE2 使用更新的 Kubernetes è¡¥ä¸ç‰ˆæœ¬æ—¶**： + +1. 使用 RKE2 v1.23 或 v1.24 é…置新的 RKE2 Windows 下游集群。 +1. 开始将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 集群。 +1. 执行验è¯æµ‹è¯•ï¼Œç¡®ä¿åœ¨å°†åº”用程åºä»Ž RKE1 è¿ç§»åˆ° RKE2 时没有丢失或更改功能。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md new file mode 100644 index 00000000000..e66e26a3975 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md @@ -0,0 +1,35 @@ +--- +title: ConfigMap +--- + +大多数 Kubernetes 密文用于存储æ•æ„Ÿä¿¡æ¯ï¼Œä½†æ˜¯ [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) 会存储一般é…置信æ¯ï¼Œä¾‹å¦‚一组é…置文件。由于 ConfigMap ä¸å­˜å‚¨æ•æ„Ÿä¿¡æ¯ï¼Œå› æ­¤ ConfigMap å¯ä»¥è‡ªåŠ¨æ›´æ–°ï¼Œä¸éœ€è¦åœ¨æ›´æ–°åŽé‡å¯å…¶å®¹å™¨ï¼ˆå…¶ä»–类型的密文大多需è¦æ‰‹åŠ¨æ›´æ–°å’Œé‡å¯å®¹å™¨åŽæ‰èƒ½ç”Ÿæ•ˆï¼‰ã€‚ + +ConfigMap 接å—常è§å­—符串格å¼çš„键值对，例如é…置文件或 JSON Blob。上传é…置映射åŽï¼Œä»»ä½•å·¥ä½œè´Ÿè½½éƒ½å¯ä»¥å°†å…¶å¼•ç”¨ä¸ºçŽ¯å¢ƒå˜é‡æˆ–å·æŒ‚载。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到具有è¦å¼•ç”¨ ConfigMap 的工作负载的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æ›´å¤šèµ„æº > 核心 > ConfigMaps**。 +1. å•å‡»**创建**。 +1. 输入 ConfigMap çš„**å称**。 + + :::note + + Kubernetes å°† ConfigMap 归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，项目或命å空间中的密文ä¸èƒ½é‡å。因此，为了é¿å…冲çªï¼ŒConfigMap çš„å称必须与工作空间中的其他è¯ä¹¦ã€é•œåƒä»“库和密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 选择è¦æ·»åŠ  ConfigMap çš„**命å空间**。 + +1. 在**æ•°æ®**选项å¡ä¸Šï¼Œå°†é”®å€¼å¯¹æ·»åŠ åˆ°ä½ çš„ ConfigMap。你å¯ä»¥æ ¹æ®éœ€è¦æ·»åŠ ä»»æ„æ•°é‡çš„值。通过å¤åˆ¶å’Œç²˜è´´çš„æ–¹å¼å°†å¤šä¸ªé”®å€¼å¯¹æ·»åŠ åˆ° ConfigMap。你也å¯ä»¥ä½¿ç”¨**从文件读å–**æ¥æ·»åŠ æ•°æ®ã€‚请注æ„，如果è¦å­˜å‚¨æ•æ„Ÿæ•°æ®ï¼Œè¯·[使用密文](secrets.md)，ä¸è¦ä½¿ç”¨ ConfigMap。 +1. å•å‡»**创建**。 + +**结果**：已将 ConfigMap 添加到命å空间。你å¯ä»¥åœ¨ Rancher UI çš„**èµ„æº > ConfigMaps** 视图中查看它。 + +## åŽç»­æ“作 + +现在你已ç»å°† ConfigMap 添加到命å空间，你å¯ä»¥å°†å…¶æ·»åŠ åˆ°åŽŸå§‹å‘½å空间部署的工作负载中。你å¯ä»¥ä½¿ç”¨ ConfigMap 指定应用程åºä½¿ç”¨çš„ä¿¡æ¯ï¼Œä¾‹å¦‚： + +- 应用程åºçŽ¯å¢ƒå˜é‡ã€‚ +- 指定安装到工作负载的å·çš„å‚数。 + +有关将 ConfigMap 添加到工作负载的更多信æ¯ï¼Œè¯·å‚阅[部署工作负载](workloads-and-pods/deploy-workloads.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md new file mode 100644 index 00000000000..2c9be4b41c9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md @@ -0,0 +1,29 @@ +--- +title: Services +--- + +Deploymentã€StatefulSet å’Œ Daemonset ä¼šç®¡ç† Pod é…置，而 Service 使用选择器将æµé‡å¼•å¯¼åˆ° Pod。 + +æ¯ä¸ªå·¥ä½œè´Ÿè½½ï¼ˆè‡³å°‘é…置一个端å£ï¼‰éƒ½ä¼šåˆ›å»ºä¸€ä¸ªè¡¥å……çš„æœåŠ¡å‘现æ¡ç›®ã€‚æ­¤æœåŠ¡å‘现æ¡ç›®ä½¿ç”¨ä»¥ä¸‹å‘½å约定为工作负载的 pod å¯ç”¨ DNS 解æžï¼š +`..svc.cluster.local`。 + +ä½ å¯ä»¥åˆ›å»ºå…¶å®ƒ service，这样，指定的命å空间就å¯ä»¥ä½¿ç”¨ä¸€ä¸ªæˆ–多个外部 IP 地å€ã€å¤–部主机åã€å¦ä¸€ä¸ª DNS 记录的别åã€å…¶ä»–工作负载或与你创建的选择器匹é…的一组 pod æ¥è¿›è¡Œè§£æžã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ  service 的集群，然åŽå•å‡» **Explore**。 +1. 点击**æœåŠ¡å‘现 > æœåŠ¡**。 +1. å•å‡»**创建**。 +1. 选择è¦åˆ›å»ºçš„ service 类型。 +1. 从下拉列表中选择一个的**命å空间**。 +1. 输入 service çš„**å称**。此å称会用于 DNS 解æžã€‚ +1. 完æˆè¡¨å•çš„其余部分。如需帮助，请å‚阅 [service 的上游 Kubernetes 文档](https://kubernetes.io/docs/concepts/services-networking/service/)。 +1. å•å‡»**创建**。 + +**结果**：创建了一个新 service。 + +- ä½ å¯ä»¥ä»Žé¡¹ç›®çš„**æœåŠ¡å‘现**选项å¡æŸ¥çœ‹è®°å½•ã€‚ +- 访问你创建的新记录的新 DNS å称 (`..svc.cluster.local`) 时，它会解æžæ‰€é€‰çš„命å空间。 + +## 相关链接 + +- [使用 HostAliases å‘ Pod /etc/hosts 文件添加æ¡ç›®](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md new file mode 100644 index 00000000000..1c4c314708d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md @@ -0,0 +1,38 @@ +--- +title: 加密 HTTP 通信 +description: 了解如何添加 SSL è¯ä¹¦æˆ– TLS è¯ä¹¦ +--- + +在 Rancher/Kubernetes 中创建 Ingress 时，你必须æä¾›åŒ…å« TLS ç§é’¥å’Œè¯ä¹¦çš„密文，用于加密和解密通过 Ingress 的通信。你å¯ä»¥é€šè¿‡å°†è¯ä¹¦æ·»åŠ åˆ° Ingress Deployment æ¥è®© Ingress 使用è¯ä¹¦ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +你必须具有å¯ä¸Šä¼ çš„ TLS ç§é’¥å’Œè¯ä¹¦ã€‚ + +::: + +### 1. 创建一个密文 + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦éƒ¨ç½² Ingress 的集群，然åŽå•å‡»**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. å•å‡» **TLS è¯ä¹¦**。 +1. 输入密文的å称。请注æ„，密文的å称必须与项目/工作空间中的其他è¯ä¹¦ã€é•œåƒä»“库和密文ä¸ä¸€æ ·ã€‚ +1. 在**ç§é’¥**字段中，将è¯ä¹¦çš„ç§é’¥å¤åˆ¶å¹¶ç²˜è´´åˆ°æ–‡æœ¬æ¡†ä¸­ï¼ˆåŒ…括标头和页脚），或者å•å‡»**从文件读å–**选择文件系统上的ç§é’¥ã€‚如果å¯èƒ½ï¼Œæˆ‘们建议使用**从文件读å–**以å‡å°‘出错的å¯èƒ½æ€§ã€‚请注æ„，ç§é’¥æ–‡ä»¶çš„扩展å是 `.key`。 +1. 在**è¯ä¹¦**字段中，将你的è¯ä¹¦å¤åˆ¶å¹¶ç²˜è´´åˆ°æ–‡æœ¬æ¡†ä¸­ï¼ˆåŒ…括标头和页脚），或者å•å‡»**从文件读å–**选择文件系统上的è¯ä¹¦æ–‡ä»¶ã€‚如果å¯èƒ½ï¼Œæˆ‘们建议使用**从文件读å–**以å‡å°‘出错的å¯èƒ½æ€§ã€‚请注æ„，è¯ä¹¦æ–‡ä»¶çš„扩展å是 `.crt`。 +1. å•å‡»**创建**。 + +### 2. 将密文添加到 Ingress + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦éƒ¨ç½² Ingress 的集群，然åŽå•å‡»**æœåŠ¡å‘现 > Ingress**。 +1. å•å‡»**创建**。 +1. 选择 Ingress çš„**命å空间**。 +1. 输入 Ingress çš„**å称**。 +1. 在**è¯ä¹¦**选项å¡ä¸­ï¼Œé€‰æ‹©åŒ…å«ä½ çš„è¯ä¹¦å’Œç§é’¥çš„密文。 +1. å•å‡»**创建**。 + +## åŽç»­æ“作 + +现在，你å¯ä»¥åœ¨è¯¥é¡¹ç›®æˆ–命å空间中å¯åŠ¨ Ingress 时添加è¯ä¹¦ã€‚如需更多信æ¯ï¼Œè¯·å‚阅[添加 Ingress](load-balancer-and-ingress-controller/add-ingresses.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md new file mode 100644 index 00000000000..eb60cb4c97b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md @@ -0,0 +1,39 @@ +--- +title: Horizontal Pod Autoscaler ä»‹ç» +--- + +[Horizontal Pod Autoscaler（HPA）](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)是一项 Kubernetes 功能，用于将集群é…置为自动扩缩其è¿è¡Œçš„æœåŠ¡ã€‚æœ¬èŠ‚ä»‹ç» HPA 如何与 Kubernetes 一起工作。 + +## 为什么è¦ä½¿ç”¨ Horizo​​ntal Pod Autoscaler？ + +ä½ å¯ä»¥ä½¿ç”¨ HPA æ¥è‡ªåŠ¨æ‰©ç¼© ReplicationControllerã€Deployment 或 ReplicaSet 中的 pod æ•°é‡ã€‚HPA 会自动扩缩正在è¿è¡Œçš„ pod æ•°é‡ä»¥å®žçŽ°æœ€é«˜æ•ˆçŽ‡ã€‚å½±å“ pod æ•°é‡çš„因素包括： + +- å…许è¿è¡Œçš„最å°å’Œæœ€å¤§ pod 数，由用户定义。 +- 资æºæŒ‡æ ‡ä¸­æŠ¥å‘Šçš„ CPU/内存使用情况。 +- 第三方指标应用程åºï¼ˆå¦‚ Prometheusã€Datadog 等）æ供的自定义指标。 + +HPA 通过以下方å¼æ”¹è¿›ä½ çš„æœåŠ¡ï¼š + +- 释放硬件资æºï¼Œé¿å…资æºè¢«è¿‡å¤šçš„ pod 浪费。 +- 按需æ高/é™ä½Žæ€§èƒ½ä»¥è¾¾åˆ°æœåŠ¡çº§åˆ«å议。 + +## HPA çš„å·¥ä½œåŽŸç† + +![HPA 架构](/img/horizontal-pod-autoscaler.jpg) + +HPA 实现为一个控制循环，其周期由以下 `kube-controller-manager` 标志控制： + +| 标志 | 默认 | æè¿° | +---------|----------|----------| +| `--horizontal-pod-autoscaler-sync-period` | `30s` | HPA 在 deployment 中审核资æº/自定义指标的频率。 | +| `--horizontal-pod-autoscaler-downscale-delay` | `5m0s` | 完æˆç¼©å‡æ“作åŽï¼ŒHPA 必须等待多长时间æ‰èƒ½å¯åŠ¨å¦ä¸€ä¸ªç¼©å‡æ“作。 | +| `--horizontal-pod-autoscaler-upscale-delay` | `3m0s` | 完æˆæ‰©å±•æ“作åŽï¼ŒHPA 必须等待多长时间æ‰èƒ½å¯åŠ¨å¦ä¸€ä¸ªæ‰©å±•æ“作。 | + + +有关 HPA 的完整文档，请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)。 + +## Horizo​​ntal Pod Autoscaler API 对象 + +HPA 是 Kubernetes `autoscaling` API 组中的 API 资æºã€‚当å‰çš„稳定版本是 `autoscaling/v1`，它åªæ”¯æŒå¯¹ CPU 自动扩缩。è¦èŽ·å¾—内存和自定义指标的扩缩支æŒï¼Œè¯·æ”¹ç”¨ beta 版本 `autoscaling/v2beta1`。 + +有关 HPA API 对象的更多信æ¯ï¼Œè¯·å‚阅 [HPA GitHub 自述文件](https://github.com/kubernetes/design-proposals-archive/blob/main/autoscaling/horizontal-pod-autoscaler.md#horizontalpodautoscaler-object)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md new file mode 100644 index 00000000000..a8ff4dd0879 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md @@ -0,0 +1,204 @@ +--- +title: 使用 kubectl ç®¡ç† HPA +--- + +本文介ç»å¦‚何使用 `kubectl` ç®¡ç† HPA。本文对以下æ“作进行了说明： + +- 创建 HPA +- èŽ·å– HPA çš„ä¿¡æ¯ +- 删除 HPA +- é…ç½® HPA ä»¥æ ¹æ® CPU 或内存利用率进行扩缩 +- 如果你使用 Prometheus 等第三方工具的指标，é…ç½® HPA 以使用自定义指标进行扩缩 + + +ä½ å¯ä»¥åœ¨ Rancher UI 中创建ã€æŸ¥çœ‹å’Œåˆ é™¤ HPA。你还å¯ä»¥æ ¹æ® Rancher UI 中的 CPU/内存使用情况进行扩缩。有关详细信æ¯ï¼Œè¯·å‚阅[使用 Rancher UI ç®¡ç† HPA](./manage-hpas-with-ui.md)。如果è¦ä½¿ç”¨ CPU/内存以外的指标æ¥æ‰©ç¼© HPA，你需è¦ä½¿ç”¨ `kubectl`。 + +## ç®¡ç† HPA 的基本 kubectl 命令 + +如果你有 HPA 清å•æ–‡ä»¶ï¼Œåˆ™å¯ä»¥ä½¿ç”¨ `kubectl` æ¥åˆ›å»ºã€ç®¡ç†å’Œåˆ é™¤ HPA： + +- 创建 HPA + + - 有清å•ï¼š`kubectl create -f ` + + - 没有清å•ï¼ˆä»…æ”¯æŒ CPU）：`kubectl autoscale deployment hello-world --min=2 --max=5 --cpu-percent=50` + +- èŽ·å– HPA ä¿¡æ¯ + + - 基本：`kubectl get hpa hello-world` + + - 详细æ述：`kubectl describe hpa hello-world` + +- 删除 HPA + + - `kubectl delete hpa hello-world` + +## HPA 清å•å®šä¹‰ç¤ºä¾‹ + +HPA 清å•æ˜¯ç”¨äºŽä½¿ç”¨ `kubectl` ç®¡ç† HPA çš„é…置文件。 + +以下代ç ç‰‡æ®µåœ¨ HPA 清å•ä¸­ä½¿ç”¨äº†ä¸åŒçš„指令。请å‚阅示例下方的列表以了解æ¯ä¸ªæŒ‡ä»¤çš„用途。 + +```yml +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: hello-world +spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi +``` + + +| 指令 | æè¿° | +---------|----------| +| `apiVersion: autoscaling/v2beta1` | 正在使用的 Kubernetes `autoscaling` API 组的版本。此示例清å•ä½¿ç”¨ beta 版本，因此å¯ç”¨äº†æŒ‰ CPU./内存进行扩缩。 | +| `name: hello-world` | 表示 HPA 在为 `hello-word` deployment 执行自动扩缩。 | +| `minReplicas: 1` | 表示è¿è¡Œçš„最å°å‰¯æœ¬æ•°ä¸èƒ½ä½ŽäºŽ 1。 | +| `maxReplicas: 10` | 指示 deployment 中的最大副本数ä¸èƒ½è¶…过 10。 | +| `targetAverageUtilization: 50` | 表示当平å‡è¿è¡Œçš„ pod 使用超过请求 CPU çš„ 50% 时，deployment 将扩展 pod。 | +| `targetAverageValue: 100Mi` | 表示当平å‡è¿è¡Œçš„ pod 使用超过 100Mi 的内存时，deployment 将扩展 pod。 | +
+ +## 使用资æºæŒ‡æ ‡ï¼ˆCPU 和内存）é…ç½® HPA 以进行扩缩 + +在 Rancher 2.0.7 åŠæ›´é«˜ç‰ˆæœ¬ä¸­åˆ›å»ºçš„集群满足使用 Horizontal Pod Autoscaler 的所有è¦æ±‚（metrics-server å’Œ Kubernetes 集群é…置）。 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ä»¥æ£€æŸ¥ä½ çš„安装中是å¦æœ‰å¯ç”¨çš„指标： + +``` +$ kubectl top nodes +NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% +node-controlplane 196m 9% 1623Mi 42% +node-etcd 80m 4% 1090Mi 28% +node-worker 64m 3% 1146Mi 29% +$ kubectl -n kube-system top pods +NAME CPU(cores) MEMORY(bytes) +canal-pgldr 18m 46Mi +canal-vhkgr 20m 45Mi +canal-x5q5v 17m 37Mi +canal-xknnz 20m 37Mi +kube-dns-7588d5b5f5-298j2 0m 22Mi +kube-dns-autoscaler-5db9bbb766-t24hw 0m 5Mi +metrics-server-97bc649d5-jxrlt 0m 12Mi +$ kubectl -n kube-system logs -l k8s-app=metrics-server +I1002 12:55:32.172841 1 heapster.go:71] /metrics-server --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true +I1002 12:55:32.172994 1 heapster.go:72] Metrics Server version v0.2.1 +I1002 12:55:32.173378 1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default.svc" and version +I1002 12:55:32.173401 1 configs.go:62] Using kubelet port 10250 +I1002 12:55:32.173946 1 heapster.go:128] Starting with Metric Sink +I1002 12:55:32.592703 1 serving.go:308] Generated self-signed cert (apiserver.local.config/certificates/apiserver.crt, apiserver.local.config/certificates/apiserver.key) +I1002 12:55:32.925630 1 heapster.go:101] Starting Heapster API server... +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] listing is available at https:///swaggerapi +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] https:///swaggerui/ is mapped to folder /swagger-ui/ +I1002 12:55:32.928597 1 serve.go:85] Serving securely on 0.0.0.0:443 +``` + + +## é…ç½® HPA 以使用 Prometheus 自定义指标进行扩缩 + +ä½ å¯ä»¥å°† HPA é…置为根æ®ç¬¬ä¸‰æ–¹è½¯ä»¶æ供的自定义指标进行自动扩缩。使用第三方软件进行自动扩缩的最常è§ç”¨ä¾‹æ˜¯ä½¿ç”¨åº”用程åºçº§åˆ«çš„指标（å³æ¯ç§’ HTTP 请求数）。HPA 使用 `custom.metrics.k8s.io` API æ¥ä½¿ç”¨è¿™äº›æŒ‡æ ‡ã€‚通过为指标收集解决方案部署自定义指标适é…器，你å¯ä»¥å¯ç”¨æ­¤ API。 + +对于这个例å­ï¼Œæˆ‘们将使用 [Prometheus](https://prometheus.io/)。我们å‡è®¾ï¼š + +- Prometheus 部署在集群中。 +- Prometheus é…置正确，并从 podã€èŠ‚点ã€å‘½å空间等收集åˆé€‚的指标。 +- Prometheus 暴露在以下 URL 和端å£ï¼š`http://prometheus.mycompany.io:80` + +Prometheus å¯ç”¨äºŽ Rancher 2.0 应用商店中的 Deployment。如果它还没有è¿è¡Œåœ¨ä½ çš„集群中，请在 Rancher 应用商店中部署它。 + +为了让 HPA 使用æ¥è‡ª Prometheus 的自定义指标，[k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) 包需è¦åœ¨ä½ é›†ç¾¤çš„ `kube-system` 命å空间中。è¦å®‰è£… `k8s-prometheus-adapter`，使用 [banzai-charts](https://github.com/banzaicloud/banzai-charts) æ供的 Helm Chart。 + +1. 在集群中åˆå§‹åŒ– Helm。 + ``` + # kubectl -n kube-system create serviceaccount tiller + kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller + helm init --service-account tiller + ``` + +1. 从 GitHub 克隆 `banzai-charts` 仓库： + ``` + # git clone https://github.com/banzaicloud/banzai-charts + ``` + +1. 安装 `prometheus-adapter` Chart，指定 Prometheus URL 和端å£å·ï¼š + ``` + # helm install --name prometheus-adapter banzai-charts/prometheus-adapter --set prometheus.url="http://prometheus.mycompany.io",prometheus.port="80" --namespace kube-system + ``` + +1. 检查 `prometheus-adapter` 是å¦æ­£å¸¸è¿è¡Œã€‚检查æœåŠ¡ pod 并在 `kube-system` 命å空间中登录。 + + 1. 检查æœåŠ¡ pod 是å¦å¤„于 `Running` 状æ€ã€‚输入以下命令： + ``` + # kubectl get pods -n kube-system + ``` + 在输出结果中查找 `Running` 状æ€ã€‚ + ``` + NAME READY STATUS RESTARTS AGE + ... + prometheus-adapter-prometheus-adapter-568674d97f-hbzfx 1/1 Running 0 7h + ... + ``` + 1. 通过输入以下命令检查æœåŠ¡æ—¥å¿—，确ä¿æœåŠ¡æ­£å¸¸è¿è¡Œï¼š + ``` + # kubectl logs prometheus-adapter-prometheus-adapter-568674d97f-hbzfx -n kube-system + ``` + 然åŽæŸ¥çœ‹æ—¥å¿—输出以确认æœåŠ¡æ­£åœ¨è¿è¡Œã€‚ + +
+ Prometheus Adaptor 日志 + + ... + I0724 10:18:45.696679 1 round_trippers.go:436] GET https://10.43.0.1:443/api/v1/namespaces/default/pods?labelSelector=app%3Dhello-world 200 OK in 2 milliseconds + I0724 10:18:45.696695 1 round_trippers.go:442] Response Headers: + I0724 10:18:45.696699 1 round_trippers.go:445] Date: Tue, 24 Jul 2018 10:18:45 GMT + I0724 10:18:45.696703 1 round_trippers.go:445] Content-Type: application/json + I0724 10:18:45.696706 1 round_trippers.go:445] Content-Length: 2581 + I0724 10:18:45.696766 1 request.go:836] Response Body: {"kind":"PodList","apiVersion":"v1","metadata":{"selfLink":"/api/v1/namespaces/default/pods","resourceVersion":"6237"},"items":[{"metadata":{"name":"hello-world-54764dfbf8-q6l82","generateName":"hello-world-54764dfbf8-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/hello-world-54764dfbf8-q6l82","uid":"484cb929-8f29-11e8-99d2-067cac34e79c","resourceVersion":"4066","creationTimestamp":"2018-07-24T10:06:50Z","labels":{"app":"hello-world","pod-template-hash":"1032089694"},"annotations":{"cni.projectcalico.org/podIP":"10.42.0.7/32"},"ownerReferences":[{"apiVersion":"extensions/v1beta1","kind":"ReplicaSet","name":"hello-world-54764dfbf8","uid":"4849b9b1-8f29-11e8-99d2-067cac34e79c","controller":true,"blockOwnerDeletion":true}]},"spec":{"volumes":[{"name":"default-token-ncvts","secret":{"secretName":"default-token-ncvts","defaultMode":420}}],"containers":[{"name":"hello-world","image":"rancher/hello-world","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"default-token-ncvts","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"34.220.18.140","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}]},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:54Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"}],"hostIP":"34.220.18.140","podIP":"10.42.0.7","startTime":"2018-07-24T10:06:50Z","containerStatuses":[{"name":"hello-world","state":{"running":{"startedAt":"2018-07-24T10:06:54Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"rancher/hello-world:latest","imageID":"docker-pullable://rancher/hello-world@sha256:4b1559cb4b57ca36fa2b313a3c7dde774801aa3a2047930d94e11a45168bc053","containerID":"docker://cce4df5fc0408f03d4adf82c90de222f64c302bf7a04be1c82d584ec31530773"}],"qosClass":"Burstable"}}]} + I0724 10:18:45.699525 1 api.go:74] GET http://prometheus-server.prometheus.34.220.18.140.xip.io/api/v1/query?query=sum%28rate%28container_fs_read_seconds_total%7Bpod_name%3D%22hello-world-54764dfbf8-q6l82%22%2Ccontainer_name%21%3D%22POD%22%2Cnamespace%3D%22default%22%7D%5B5m%5D%29%29+by+%28pod_name%29&time=1532427525.697 200 OK + I0724 10:18:45.699620 1 api.go:93] Response Body: {"status":"success","data":{"resultType":"vector","result":[{"metric":{"pod_name":"hello-world-54764dfbf8-q6l82"},"value":[1532427525.697,"0"]}]}} + I0724 10:18:45.699939 1 wrap.go:42] GET /apis/custom.metrics.k8s.io/v1beta1/namespaces/default/pods/%2A/fs_read?labelSelector=app%3Dhello-world: (12.431262ms) 200 [[kube-controller-manager/v1.10.1 (linux/amd64) kubernetes/d4ab475/system:serviceaccount:kube-system:horizontal-pod-autoscaler] 10.42.0.0:24268] + I0724 10:18:51.727845 1 request.go:836] Request Body: {"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","metadata":{"creationTimestamp":null},"spec":{"nonResourceAttributes":{"path":"/","verb":"get"},"user":"system:anonymous","group":["system:unauthenticated"]},"status":{"allowed":false}} + ... +
+ + + +1. 检查是å¦å¯ä»¥ä»Ž kubectl 访问 metrics API。 + + - 如果你直接访问集群，请在 kubectl é…置中输入你的æœåŠ¡å™¨ URL，格å¼æ˜¯ `https://:6443`： + ``` + # kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 + ``` + 如果 API å¯è®¿é—®ï¼Œä½ åº”该会看到类似以下内容的输出： + +
+ API å“应 + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
+ + - 如果你通过 Rancher 访问集群，请在 kubectl é…置中输入你的æœåŠ¡å™¨ URL，格å¼æ˜¯ `https:///k8s/clusters/`。将 `/k8s/clusters/` åŽç¼€æ·»åŠ åˆ° API 路径： + ``` + # kubectl get --raw /k8s/clusters//apis/custom.metrics.k8s.io/v1beta1 + ``` + 如果 API å¯è®¿é—®ï¼Œä½ åº”该会看到类似以下内容的输出： + +
+ API å“应 + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md new file mode 100644 index 00000000000..42fbec1785a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md @@ -0,0 +1,51 @@ +--- +title: 使用 Rancher UI ç®¡ç† HPA +--- + +Rancher UI 支æŒåˆ›å»ºã€ç®¡ç†å’Œåˆ é™¤ HPA。你å¯ä»¥å°† CPU 或内存使用情况é…置为用于 HPA 扩缩的指标。 + +如果你想使用 CPU/内存之外的其他指标创建 HPA，请å‚阅[é…ç½® HPA 以使用 Prometheus 的自定义指标进行扩缩](./manage-hpas-with-kubectl.md#é…ç½®-hpa-以使用-prometheus-自定义指标进行扩缩)。 + +## 创建 HPA + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦åˆ›å»º HPA 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æœåŠ¡å‘现 > Horizo​​ntalPodAutoscalers**。 +1. å•å‡»**创建**。 +1. 为 HPA 选择 **命å空间**。 +1. 输入 HPA çš„**å称**。 +1. 选择**目标引用**作为 HPA 的扩缩目标。 +1. 为 HPA 指定**最å°å‰¯æœ¬æ•°**å’Œ**最大副本数**。 +1. é…ç½® HPA 的指标。你å¯ä»¥å°†å†…存或 CPU 使用率作为让 HPA 扩缩æœåŠ¡çš„指标。在**æ•°é‡**字段中，输入让 HPA 扩缩æœåŠ¡çš„工作负载内存/CPU 使用率的百分比。è¦é…置其他 HPA 指标，包括 Prometheus æ供的指标，你需è¦[使用 kubectl ç®¡ç† HPA](./manage-hpas-with-kubectl.md#é…ç½®-hpa-以使用-prometheus-自定义指标进行扩缩)。 + +1. å•å‡»**创建**以创建HPA。 + +:::note 结果： + +HPA 已被部署到选定的命å空间。你å¯ä»¥åœ¨é¡¹ç›®çš„**èµ„æº > HPA** 视图中查看 HPA 的状æ€ã€‚ + +::: + +## èŽ·å– HPA æŒ‡æ ‡å’ŒçŠ¶æ€ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到具有 HPA 的集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æœåŠ¡å‘现 > Horizo​​ntalPodAutoscalers**。**Horizo​​ntalPodAutoscalers** 页é¢æ˜¾ç¤ºå½“å‰å‰¯æœ¬çš„æ•°é‡ã€‚ + +有关特定 HPA 的更详细指标和状æ€ï¼Œè¯·å•å‡» HPA çš„å称。你会转到 HPA 的详情页é¢ã€‚ + + +## 删除 HPA + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦åˆ é™¤çš„ HPA 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æœåŠ¡å‘现 > Horizo​​ntalPodAutoscalers**。 +1. å•å‡»**èµ„æº > HPA**。 +1. 找到è¦åˆ é™¤çš„ HPA，然åŽå•å‡» **â‹® > 删除**。 +1. å•å‡»**删除**以进行确认。 + +:::note 结果： + +HPA 已从当å‰é›†ç¾¤ä¸­åˆ é™¤ã€‚ + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md new file mode 100644 index 00000000000..d48c98b3e8c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md @@ -0,0 +1,531 @@ +--- +title: 使用 kubectl 测试 HPA +--- + +本文介ç»å¦‚何在使用负载测试工具扩缩 HPA åŽæ£€æŸ¥ HPA 的状æ€ã€‚有关使用 Rancher UI（最低版本 2.3.x）检查状æ€çš„ä¿¡æ¯ï¼Œè¯·å‚阅[使用 Rancher UI ç®¡ç† HPA](./manage-hpas-with-kubectl.md)。 + +è¦è®© HPA 正常工作，æœåŠ¡éƒ¨ç½²åº”该具有容器的资æºè¯·æ±‚定义。按照以下 hello-world 示例测试 HPA 是å¦æ­£å¸¸å·¥ä½œã€‚ + +1. å°† `kubectl` 连接到你的 Kubernetes 集群。 + +1. å¤åˆ¶ä¸‹æ–¹çš„ `hello-world` 部署清å•ã€‚ + +
+ Hello World æ¸…å• + + ``` + apiVersion: apps/v1beta2 + kind: Deployment + metadata: + labels: + app: hello-world + name: hello-world + namespace: default + spec: + replicas: 1 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + resources: + requests: + cpu: 500m + memory: 64Mi + ports: + - containerPort: 80 + protocol: TCP + restartPolicy: Always + --- + apiVersion: v1 + kind: Service + metadata: + name: hello-world + namespace: default + spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-world + ``` + +
+ +1. 将其部署到你的集群。 + + ``` + # kubectl create -f + ``` + +1. æ ¹æ®ä½ ä½¿ç”¨çš„指标类型å¤åˆ¶ä»¥ä¸‹å…¶ä¸­ä¸€ä¸ª HPA： + +
+ Hello World HPA：资æºæŒ‡æ ‡ + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 1000Mi + ``` + +
+
+ Hello World HPA：自定义指标 + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi + - type: Pods + pods: + metricName: cpu_system + targetAverageValue: 20m + ``` + +
+ +1. 查看 HPA ä¿¡æ¯å’Œæ述。确认显示的指标数æ®ã€‚ + +
+ 资æºæŒ‡æ ‡ + + 1. 输入以下命令： + ``` + # kubectl get hpa + NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE + hello-world Deployment/hello-world 1253376 / 100Mi, 0% / 50% 1 10 1 6m + # kubectl describe hpa + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 20:21:16 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 1253376 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+
+ 自定义指标 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:36:28 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 3514368 / 100Mi + "cpu_system" on pods: 0 / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+ +1. 为æœåŠ¡ç”Ÿæˆè´Ÿè½½ï¼Œä»Žè€Œæµ‹è¯•ä½ çš„ pod 是å¦æŒ‰é¢„期进行了自动扩缩。你å¯ä»¥ä½¿ç”¨ä»»ä½•è´Ÿè½½æµ‹è¯•å·¥å…·ï¼ˆHeyã€Gatling 等），我们使用的是 [Hey](https://github.com/rakyll/hey)。 + +1. 测试 pod 自动扩缩是å¦æŒ‰é¢„期工作。

+ **使用资æºæŒ‡æ ‡æµ‹è¯•è‡ªåŠ¨æ‰©ç¼©ï¼š** + +
+ 扩展到两个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨ä½ çš„è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 使用情况扩展到两个 Pod。 + + 1. 查看你的 HPA。 + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10928128 / 100Mi + resource cpu on pods (as a percentage of request): 56% (280m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. 输入以下命令，确认你已扩展到两个 pod： + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-k8ph2 1/1 Running 0 1m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ 扩展到三个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨ä½ çš„è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 用é‡æ‰©å±•åˆ°ä¸‰ä¸ª Pod，并将 `horizo​​ntal-pod-autoscaler-upscale-delay` 设置为 3 分钟。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 9424896 / 100Mi + resource cpu on pods (as a percentage of request): 66% (333m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 4m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + ``` + 2. 输入以下命令，确认三个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-f46kh 0/1 Running 0 1m + hello-world-54764dfbf8-k8ph2 1/1 Running 0 5m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ 缩å‡åˆ° 1 个 Pod：所有指标å‡ä½ŽäºŽç›®æ ‡ + + 当 `horizontal-pod-autoscaler-downscale-delay` 的所有指标å‡ä½ŽäºŽç›®æ ‡ï¼ˆé»˜è®¤ä¸º 5 分钟）时，使用你的负载测试工具缩å‡åˆ° 1 个 pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10070016 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 6m horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 1s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + +
+ + **使用自定义指标测试自动缩放：** + +
+ 扩展到两个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 用é‡æ‰©å±•åˆ°ä¸¤ä¸ª Pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8159232 / 100Mi + "cpu_system" on pods: 7m / 20m + resource cpu on pods (as a percentage of request): 64% (321m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. 输入以下命令，确认两个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ 扩展到三个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + 当 cpu_system 用é‡è¾¾åˆ°ç›®æ ‡æ—¶ï¼Œä½¿ç”¨ä½ çš„负载测试工具扩展到三个 Pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3s horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + ``` + 1. 输入以下命令，确认三个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + # kubectl get pods + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ 扩展到四个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 用é‡æ‰©å±•åˆ°å››ä¸ª Pod。`horizontal-pod-autoscaler-upscale-delay` 默认设置为 3 分钟。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 4s horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + ``` + + 1. 输入以下命令，确认四个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-2p9xb 1/1 Running 0 5m + hello-world-54764dfbf8-5pfdr 1/1 Running 0 2m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ 缩å‡åˆ° 1 个 Pod：所有指标å‡ä½ŽäºŽç›®æ ‡ + + 当 `horizontal-pod-autoscaler-downscale-delay` 的所有指标å‡ä½ŽäºŽç›®æ ‡æ—¶ï¼Œä½¿ç”¨ä½ çš„负载测试工具缩å‡åˆ° 1 个 pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8101888 / 100Mi + "cpu_system" on pods: 8m / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 8m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + 1. 输入以下命令，确认å•ä¸ª pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md new file mode 100644 index 00000000000..6231ec1697a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md @@ -0,0 +1,178 @@ +--- +title: Kubernetes é•œåƒä»“库和容器镜åƒä»“库 +description: 了解容器镜åƒä»“库和 Kubernetes é•œåƒä»“库ã€å®ƒä»¬çš„用例以åŠå¦‚何在 Rancher UI 中使用ç§æœ‰é•œåƒä»“库 +--- + +é•œåƒä»“库是 Kubernetes 密文（Secret），包å«ç”¨äºŽå‘[ç§æœ‰å®¹å™¨é•œåƒä»“库](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/)进行身份验è¯çš„凭è¯ã€‚ + +“Registry†这个è¯å¯èƒ½æœ‰ä¸¤ç§æ„æ€ï¼Œå¯æŒ‡ä»£å®¹å™¨æˆ– Kubernetes é•œåƒä»“库： + +- **容器镜åƒä»“库**（以å‰ç§°ä¸ºâ€œDocker é•œåƒä»“库â€ï¼‰åŒ…å«ä½ å¯ä»¥æ‹‰å–和部署的容器镜åƒã€‚é•œåƒä»“库是一个无状æ€ã€å¯æ‰©å±•çš„æœåŠ¡å™¨ç«¯åº”用程åºï¼Œç”¨äºŽå­˜å‚¨å’Œåˆ†å‘容器镜åƒã€‚ +- **Kubernetes é•œåƒä»“库**是一个镜åƒæ‹‰å–密文，你的 deployment 使用该密文æ¥å‘容器镜åƒä»“库进行身份验è¯ã€‚ + +Deployment 使用 Kubernetes é•œåƒä»“库密文å‘ç§æœ‰å®¹å™¨é•œåƒä»“库进行身份验è¯ï¼Œç„¶åŽæ‹‰å–托管在仓库上的容器镜åƒã€‚ + +ç›®å‰ï¼Œåªæœ‰åœ¨ Rancher UI 中创建工作负载时（通过 kubectl 创建的工作负载ä¸å¯ä»¥ï¼‰ï¼ŒDeployment æ‰ä¼šè‡ªåŠ¨æ‹‰å–ç§æœ‰é•œåƒä»“库凭è¯ã€‚ + +## 在命å空间中创建镜åƒä»“库 + +:::note 先决æ¡ä»¶ï¼š + +你必须已ç»éƒ¨ç½²äº†ä¸€ä¸ªå¯ç”¨çš„ç§æœ‰é•œåƒä»“库。 + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ é•œåƒä»“库的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航中，å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. å•å‡»**é•œåƒä»“库**。 +1. 输入镜åƒä»“库的**å称**。 + + :::note + + Kubernetes 将密文ã€è¯ä¹¦å’Œé•œåƒä»“库都归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，命å空间或项目中的密文å称ä¸èƒ½é‡å¤ã€‚因此，为了é¿å…冲çªï¼Œé•œåƒä»“库的å称必须与工作空间中的其他密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 为镜åƒä»“库选择一个命å空间。 +1. 选择托管你的ç§äººé•œåƒä»“库的网站。然åŽï¼Œè¾“å…¥å‘é•œåƒä»“库进行身份验è¯çš„凭è¯ã€‚例如，如果你使用 DockerHub，请æ供你的 DockerHub 用户å和密ç ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**： + +- 已将你的密文添加到命å空间中。 +- ä½ å¯ä»¥é€šè¿‡å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**在 Rancher UI 中查看密文。 +- 如果工作负载在镜åƒä»“库的范围内，你在 Rancher UI 中创建的任何工作负载都将具有访问镜åƒä»“库的凭è¯ã€‚ + +## 在项目中创建镜åƒä»“库 + +:::note 先决æ¡ä»¶ï¼š + +你必须已ç»éƒ¨ç½²äº†ä¸€ä¸ªå¯ç”¨çš„ç§æœ‰é•œåƒä»“库。 + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +在 Rancher 2.6 之å‰ï¼Œå¯†æ–‡å¿…须创建在项目级别。现在ä¸å†éœ€è¦é¡¹ç›®çº§åˆ«ï¼Œä½ å¯ä»¥é‡‡ç”¨å‘½å空间级别。因此，Rancher UI 进行了更新以å映这一新功能。但是，你ä»ç„¶å¯ä»¥æŒ‰ç…§éœ€è¦åˆ›å»ºé¡¹ç›®çº§åˆ«çš„é•œåƒä»“库。执行以下步骤进行æ“作： + +1. 在左上角，å•å‡»ä¸‹æ‹‰èœå•ä¸­çš„ **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 转到 `Legacy` 功能开关并å•å‡»**激活**。 +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ é•œåƒä»“库的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航中，å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. å•å‡»**é•œåƒä»“库**。 +1. 在顶部导航æ ä¸­è¿›è¡Œè¿‡æ»¤ï¼Œä»Žè€Œä»…查看一个项目。 +1. 输入镜åƒä»“库的**å称**。 + + :::note + + Kubernetes 将密文ã€è¯ä¹¦å’Œé•œåƒä»“库都归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，命å空间或项目中的密文å称ä¸èƒ½é‡å¤ã€‚因此，为了é¿å…冲çªï¼Œé•œåƒä»“库的å称必须与工作空间中的其他密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 为镜åƒä»“库选择一个命å空间。 +1. 选择托管你的ç§äººé•œåƒä»“库的网站。然åŽï¼Œè¾“å…¥å‘é•œåƒä»“库进行身份验è¯çš„凭è¯ã€‚例如，如果你使用 DockerHub，请æ供你的 DockerHub 用户å和密ç ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**： + +- 密文已添加到你选择的项目中。 +- ä½ å¯ä»¥é€šè¿‡å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**在 Rancher UI 中查看密文。 +- 如果工作负载在镜åƒä»“库的范围内，你在 Rancher UI 中创建的任何工作负载都将具有访问镜åƒä»“库的凭è¯ã€‚ + +:::note + +local 集群上的项目级别镜åƒä»“库仅在选择å•ä¸ªé¡¹ç›®æ—¶å¯è§ã€‚ + +::: + +## 使用ç§æœ‰é•œåƒä»“库 + +ä½ å¯ä»¥é€šè¿‡ Rancher UI 或 `kubectl` 使用ç§æœ‰é•œåƒä»“库的镜åƒæ¥éƒ¨ç½²å·¥ä½œè´Ÿè½½ã€‚ + +### 在 Rancher UI 中使用ç§æœ‰é•œåƒä»“库 + +è¦ä½¿ç”¨ç§æœ‰é•œåƒä»“库中的镜åƒæ¥éƒ¨ç½²å·¥ä½œè´Ÿè½½ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 选择è¦åˆ›å»ºçš„工作负载类型。 +1. 输入工作负载的独特å称，并选择命å空间。 +1. 在**容器镜åƒ**字段中，输入ç§æœ‰é•œåƒä»“库中镜åƒçš„路径 URL。例如，如果你的ç§æœ‰é•œåƒä»“库ä½äºŽ Quay.io，你å¯ä»¥ä½¿ç”¨ `quay.io//`。 +1. å•å‡»**创建**。 + +**结果**：你的 deployment 能å¯åŠ¨ï¼Œèƒ½ä½¿ç”¨ä½ åœ¨ Rancher UI 中添加的ç§æœ‰é•œåƒä»“库凭è¯è¿›è¡Œèº«ä»½éªŒè¯ï¼Œå¹¶æ‹‰å–指定的容器镜åƒã€‚ + +### 通过 kubectl 使用ç§æœ‰é•œåƒä»“库 + +使用 `kubectl`创建工作负载时，你需è¦é…ç½® pod，从而使其 YAML 具有ç§æœ‰é•œåƒä»“库中镜åƒçš„路径。如果 Pod 是在 Rancher UI 中创建的，它åªä¼šè‡ªåŠ¨èŽ·å–对ç§æœ‰é•œåƒä»“库凭è¯çš„访问æƒé™ï¼Œå› æ­¤ä½ è¿˜å¿…须创建和引用镜åƒä»“库密文。 + +密文必须创建在部署工作负载的åŒä¸€å‘½å空间中。 + +下é¢æ˜¯ä¸€ä¸ªç¤ºä¾‹ `pod.yml`，它用于使用ç§æœ‰é•œåƒä»“库的镜åƒçš„工作负载。在此示例中，pod 使用æ¥è‡ª Quay.io çš„é•œåƒï¼Œè€Œä¸” .yml 指定了镜åƒçš„路径。pod 使用存储在å为 `testquay` çš„ Kubernetes 密文中的凭è¯æ¥å‘é•œåƒä»“库进行身份验è¯ï¼Œè¯¥å¯†æ–‡åœ¨ `name` 字段的 `spec.imagePullSecrets` 中指定： + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: private-reg +spec: + containers: + - name: private-reg-container + image: quay.io// + imagePullSecrets: + - name: testquay +``` + +在此示例中，å为 `testquay` 的密文ä½äºŽ Default 命å空间中。 + +ä½ å¯ä»¥é€šè¿‡ `kubectl` 使用ç§æœ‰é•œåƒä»“库凭è¯æ¥åˆ›å»ºå¯†æ–‡ã€‚此命令创建å为 `testquay` 的密文： + +``` +kubectl create secret docker-registry testquay \ + --docker-server=quay.io \ + --docker-username= \ + --docker-password= +``` + +è¦æŸ¥çœ‹å¯†æ–‡æ˜¯å¦‚何存储在 Kubernetes 中的，å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤ï¼š + +``` +kubectl get secret testquay --output="jsonpath={.data.\.dockerconfigjson}" | base64 --decode +``` + +结果如下所示： + +``` +{"auths":{"quay.io":{"username":"","password":"","auth":"c291bXlhbGo6dGVzdGFiYzEyMw=="}}} +``` + +部署工作负载åŽï¼Œä½ å¯ä»¥æ£€æŸ¥é•œåƒæ˜¯å¦å·²æ‹‰å–æˆåŠŸï¼š + +``` +kubectl get events +``` +结果应如下所示： +``` +14s Normal Scheduled Pod Successfully assigned default/private-reg2 to minikube +11s Normal Pulling Pod pulling image "quay.io//" +10s Normal Pulled Pod Successfully pulled image "quay.io//" +``` + +有关详细信æ¯ï¼Œè¯·å‚阅 Kubernetes 文档中的[创建使用你密文的 pod](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md new file mode 100644 index 00000000000..5717ca726a5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md @@ -0,0 +1,18 @@ +--- +title: 添加 Ingress +description: ä½ å¯ä»¥ä¸ºå·¥ä½œè´Ÿè½½æ·»åŠ  Ingress，从而æ供负载å‡è¡¡ã€SSL 终止和基于主机/路径的路由。了解如何添加 Rancher Ingress +--- + +ä½ å¯ä»¥ä¸ºå·¥ä½œè´Ÿè½½æ·»åŠ  Ingress，从而æ供负载å‡è¡¡ã€SSL 终止和基于主机/路径的路由。在项目中使用 Ingress 时，你å¯ä»¥é€šè¿‡è®¾ç½®å…¨å±€ DNS æ¡ç›®æ¥å°† Ingress 主机å编程到外部 DNS。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ  Ingress 的集群，然åŽå•å‡» **Explore**。 +1. 点击**æœåŠ¡å‘现 > Ingresses**。 +1. å•å‡»**创建**。 +1. 从下拉列表中选择一个现有的**命å空间**。 +1. 输入 Ingress çš„**å称**。 +1. [创建](./ingress-configuration.md) Ingress 转å‘**规则**。如果你的任何 Ingress 规则处ç†åŠ å¯†ç«¯å£çš„请求，请添加è¯ä¹¦ä»¥åŠ å¯†/解密通信。 +1. **å¯é€‰**：点击**添加规则**æ¥åˆ›å»ºå…¶ä»– Ingress 规则。例如，在创建 Ingress 规则以引导主机å请求åŽï¼Œä½ å¯èƒ½æƒ³åˆ›å»ºä¸€ä¸ªé»˜è®¤åŽç«¯æ¥å¤„ç† 404。 +1. 点击å³ä¸‹è§’çš„**创建**。 + +**结果**：已将 Ingress 添加到项目中。Ingress 会开始执行你的 Ingress 规则。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md new file mode 100644 index 00000000000..3ff047bc108 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md @@ -0,0 +1,41 @@ +--- +title: Ingress é…ç½® +description: Ingress é…ç½® +--- + +:::note + +在 Kubernetes v1.21 åŠæ›´é«˜ç‰ˆæœ¬ä¸­ï¼ŒNGINX Ingress Controller ä¸å†é»˜è®¤è¿è¡Œåœ¨ hostNetwork 中。它改为将 hostPorts ç”¨äºŽç«¯å£ 80 å’Œç«¯å£ 443，因此你å¯ä»¥å°†å‡†å…¥ Webhook é…置为åªèƒ½é€šè¿‡ ClusterIP 访问。这确ä¿äº†åªèƒ½ä»Žé›†ç¾¤å†…部访问 webhook。 + +由于 controller 的这一更改，默认 RKE1 é…ç½®ä¸å†å°† `hostNetwork` 设置为 `true`。但是，你必须将 `hostNetwork` 设置为 `true` æ‰èƒ½ä½¿åŸºäºŽ TCP å’Œ UDP çš„ Service 正常工作。为此，请[编辑](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#使用-yaml-编辑集群)集群的 YAML 并按照[官方 RKE1 文档](https://rke.docs.rancher.com/config-options/add-ons/ingress-controllers#configuring-network-options)中的步骤æ“作。 + +::: + +## 指定è¦ä½¿ç”¨çš„主机å + +如果使用此选项，Ingress 会将主机å请求路由到你指定的æœåŠ¡æˆ–工作负载。 + +1. 指定类型为 `Prefix` 的路径并指定路径，例如 `/`。 +1. 添加一个**目标æœåŠ¡**。 +1. **å¯é€‰**：如果你想在将请求å‘é€åˆ°ç‰¹å®šä¸»æœºå路径时指定工作负载或æœåŠ¡ï¼Œè¯·ä¸ºç›®æ ‡æ·»åŠ **路径**。例如，如果你希望将 `www.mysite.com/contact-us` 的请求å‘é€åˆ°ä¸Ž `www.mysite.com` ä¸åŒçš„æœåŠ¡ï¼Œåœ¨**路径**字段中输入 `/contact-us`。通常情况下，你创建的第一æ¡è§„则ä¸åŒ…å«è·¯å¾„。 +1. 输入æ¯ä¸ªç›®æ ‡æ“作的**端å£**å·ã€‚ + +## è¯ä¹¦ + +:::note + +你必须具有 SSL è¯ä¹¦ï¼ŒIngress å¯ä½¿ç”¨è¯¥è¯ä¹¦æ¥åŠ å¯†å’Œè§£å¯†é€šä¿¡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[添加 SSL è¯ä¹¦](../encrypt-http-communication.md)。 + +::: + +1. è¦åˆ›å»º Ingress Controller，å•å‡»**è¯ä¹¦**选项å¡ã€‚ +1. å•å‡»**添加è¯ä¹¦**。 +1. 从下拉列表中选择一个**è¯ä¹¦ - 密文å称**。 +1. 使用加密通信进入主机。 +1. è¦æ·»åŠ ä½¿ç”¨è¯ä¹¦çš„其他主机，请å•å‡»**添加主机**。 + +## 标签和注释 + +添加[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)å’Œ/或[注释](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)æ¥ä¸ºä½ çš„ Ingress Controller æ供元数æ®ã€‚ + +有关å¯ç”¨çš„注释列表，请å‚阅 [Nginx Ingress Controller 文档](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md new file mode 100644 index 00000000000..c13f8dbf75e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md @@ -0,0 +1,68 @@ +--- +title: "四层和七层负载å‡è¡¡" +description: "Kubernetes 支æŒå››å±‚è´Ÿè½½å‡è¡¡å’Œä¸ƒå±‚è´Ÿè½½å‡è¡¡ã€‚了解对ä¸åŒ deployment 的支æŒ" +--- + +Kubernetes 支æŒå››å±‚è´Ÿè½½å‡è¡¡å’Œä¸ƒå±‚è´Ÿè½½å‡è¡¡ã€‚ + +## 四层负载å‡è¡¡å™¨ + +四层负载å‡è¡¡å™¨ï¼ˆæˆ–外部负载å‡è¡¡å™¨ï¼‰å°†æµé‡è½¬å‘到 Nodeport。四层负载å‡è¡¡å™¨æ”¯æŒè½¬å‘ HTTP å’Œ TCP æµé‡ã€‚ + +通常情况下，四层负载å‡è¡¡å™¨ç”±åº•å±‚云æ供商支æŒï¼Œå› æ­¤ï¼Œå¦‚果你在裸金属æœåŠ¡å™¨å’Œ vSphere 集群上部署 RKE 集群，则ä¸æ”¯æŒå››å±‚è´Ÿè½½å‡è¡¡å™¨ã€‚但是，å•ä¸ª[全局管ç†çš„ config-map](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) å¯ç”¨äºŽåœ¨ NGINX 或第三方 Ingress 上公开æœåŠ¡ã€‚ + +:::note + +ä½ å¯ä»¥ä½¿ç”¨éžäº‘è´Ÿè½½å‡è¡¡å™¨ï¼ˆä¾‹å¦‚ [MetalLB](https://metallb.universe.tf/)）æ¥éƒ¨ç½²é›†ç¾¤ã€‚但是，该用例比云æ供商支æŒçš„四层负载å‡è¡¡å™¨æ›´é«˜çº§ï¼Œè€Œä¸”ä¸å¯ä»¥åœ¨ Rancher 或 RKE 中é…置。 + +::: + +### 四层负载å‡è¡¡æ”¯æŒ + +ä¸åŒåº•å±‚云æ供商对四层负载å‡è¡¡å™¨çš„支æŒæœ‰æ‰€ä¸åŒï¼š + +| 集群部署 | 四层负载å‡è¡¡å™¨æ”¯æŒ | +----------------------------------------------|-------------------------------- +| Amazon EKS | ç”± AWS 云æä¾›å•†æ”¯æŒ | +| Google GKE | ç”± GCE 云æä¾›å•†æ”¯æŒ | +| Azure AKS | ç”± Azure 云æä¾›å•†æ”¯æŒ | +| EC2 上的 RKE | ç”± AWS 云æä¾›å•†æ”¯æŒ | +| DigitalOcean 上的 RKE | å—é™çš„ NGINX 或第三方 Ingress\* | +| vSphere 上的 RKE | å—é™çš„ NGINX 或第三方 Ingress\* | +| 自定义主机上的 RKE
（例如裸金属æœåŠ¡å™¨ï¼‰ | å—é™çš„ NGINX 或第三方 Ingress\* | +| 第三方 MetalLB | å—é™çš„ NGINX 或第三方 Ingress\* | + +\* å¯ä»¥é€šè¿‡å•ä¸ª[全局管ç†çš„ config-map](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) æ¥å…¬å¼€æœåŠ¡ã€‚ + +## 七层负载å‡è¡¡å™¨ + +七层负载å‡è¡¡å™¨ï¼ˆæˆ– Ingress Controller）支æŒåŸºäºŽä¸»æœºå’Œè·¯å¾„çš„è´Ÿè½½å‡è¡¡å’Œ SSL 终止。七层负载å‡è¡¡å™¨ä»…è½¬å‘ HTTP å’Œ HTTPS æµé‡ï¼Œå› æ­¤å®ƒä»¬ä»…侦å¬ç«¯å£ 80 å’Œ 443。亚马逊和谷歌等云æ供商支æŒä¸ƒå±‚è´Ÿè½½å‡è¡¡å™¨ã€‚此外，RKE 集群部署了 Nginx Ingress Controller。 + +### 七层负载å‡è¡¡æ”¯æŒ + +ä¸åŒåº•å±‚云æ供商对七层负载å‡è¡¡å™¨çš„支æŒæœ‰æ‰€ä¸åŒï¼š + +| 集群部署 | 七层负载å‡è¡¡å™¨æ”¯æŒ | +----------------------------------------------|-------------------------------- +| Amazon EKS | ç”± AWS 云æä¾›å•†æ”¯æŒ | +| Google GKE | ç”± GKE 云æä¾›å•†æ”¯æŒ | +| Azure AKS | ä¸æ”¯æŒ | +| EC2 上的 RKE | Nginx Ingress Controller | +| DigitalOcean 上的 RKE | Nginx Ingress Controller | +| vSphere 上的 RKE | Nginx Ingress Controller | +| 自定义主机上的 RKE
（例如裸金属æœåŠ¡å™¨ï¼‰ | Nginx Ingress Controller | + +### 七层负载å‡è¡¡å™¨ä¸­çš„主机å + +一些托管在云上的七层负载å‡è¡¡å™¨ï¼ˆä¾‹å¦‚ AWS 上的 ALB Ingress Controller）会为 Ingress 规则公开 DNS 地å€ã€‚你需è¦ï¼ˆä½¿ç”¨ CNAME）将你的域å映射到七层负载å‡è¡¡å™¨ç”Ÿæˆçš„ DNS 地å€ã€‚ + +其他七层负载å‡è¡¡å™¨ï¼ˆä¾‹å¦‚ Google Load Balancer 或 Nginx Ingress Controller）会直接公开一个或多个 IP 地å€ã€‚Google Load Balancer æ供了一个å¯è·¯ç”±çš„ IP 地å€ã€‚Nginx Ingress Controller 公开了è¿è¡Œ Nginx Ingress Controller 的所有节点的外部 IP。你å¯ä»¥æ‰§è¡Œä»¥ä¸‹ä»»ä¸€æ“作： + +1. é…置你自己的 DNS，从而（使用 A 记录）将你的域å映射到七层负载å‡è¡¡å™¨å…¬å¼€çš„ IP 地å€ã€‚ +2. 让 Rancher 为你的 Ingress 规则生æˆä¸€ä¸ª xip.io 主机å。Rancher 将使用你公开的其中一个 IP（å‡è®¾æ˜¯ `a.b.c.d`）生æˆä¸€ä¸ªä¸»æœºåï¼ˆå³ `..a.b.c.d.xip.io`）。 + +使用 xip.io 的好处是你å¯ä»¥åœ¨åˆ›å»º Ingress 规则åŽç«‹å³èŽ·å¾—一个有效的入å£ç‚¹ URL。此外，如果你设置自己的域å，则需è¦é…ç½® DNS æœåŠ¡å™¨å¹¶ç­‰å¾… DNS 传播。 + +## 相关链接 + +- [创建外部负载å‡è¡¡å™¨](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md new file mode 100644 index 00000000000..d4618d910e1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md @@ -0,0 +1,78 @@ +--- +title: 密文 +--- + +[密文（secret）](https://kubernetes.io/docs/concepts/configuration/secret/#overview-of-secrets)存储æ•æ„Ÿæ•°æ®ï¼Œä¾‹å¦‚密ç ã€ä»¤ç‰Œæˆ–密钥。它们å¯èƒ½åŒ…å«ä¸€ä¸ªæˆ–多个键值对。 + +:::note + +本文介ç»å¯†æ–‡çš„概述。有关设置ç§æœ‰é•œåƒä»“库的详细信æ¯ï¼Œè¯·å‚阅[é•œåƒä»“库](kubernetes-and-docker-registries.md)。 + +::: + +é…置工作负载时，你能够选择è¦åŒ…å«çš„密文。与 ConfigMap 一样，工作负载å¯ä»¥å°†å¯†æ–‡å¼•ç”¨ä¸ºçŽ¯å¢ƒå˜é‡æˆ–å·æŒ‚载。 + +除éžä½œä¸ºå­è·¯å¾„å·æŒ‚载，å¦åˆ™æŒ‚载的密文会自动更新。有关如何传播更新的密文，请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically)。 + +## 在命å空间中创建密文 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ å¯†æ–‡çš„集群，然åŽå•å‡» **Explore**。 +1. è¦å¯¼èˆªåˆ°å¯†æ–‡ï¼Œå•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. 选择è¦åˆ›å»ºçš„密文类型。 +1. 为密文选择一个**命å空间**。 +1. 输入密文的**å称**。 + + :::note + + Kubernetes 将密文ã€è¯ä¹¦å’Œé•œåƒä»“库都归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，命å空间中的密文å称ä¸èƒ½é‡å¤ã€‚因此，为了é¿å…冲çªï¼Œå¯†æ–‡çš„å称必须与工作空间中的其他密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 在**æ•°æ®**中，å•å‡»**添加**以添加键值对。你å¯ä»¥æ ¹æ®éœ€è¦æ·»åŠ ä»»æ„æ•°é‡çš„值。 + + :::tip + + ä½ å¯ä»¥é€šè¿‡å¤åˆ¶å’Œç²˜è´´çš„æ–¹å¼å°†å¤šä¸ªé”®å€¼å¯¹æ·»åŠ åˆ°å¯†æ–‡ä¸­ã€‚ + + ::: + + ![](/img/bulk-key-values.gif) + +1. å•å‡»**ä¿å­˜**。 + +**结果**：密文已添加到你选择的命å空间中。你å¯ä»¥åœ¨ Rancher UI 中å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**æ¥æŸ¥çœ‹å¯†æ–‡ã€‚ + +除éžä½œä¸ºå­è·¯å¾„å·æŒ‚载，å¦åˆ™æŒ‚载的密文会自动更新。有关如何传播更新的密文，请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically)。 + + +## 在项目中创建密文 + +在 Rancher 2.6 之å‰ï¼Œå¯†æ–‡å¿…须创建在项目级别。现在ä¸å†éœ€è¦é¡¹ç›®çº§åˆ«ï¼Œä½ å¯ä»¥é‡‡ç”¨å‘½å空间级别。因此，Rancher UI 进行了更新以å映这一新功能。但是，你ä»ç„¶å¯ä»¥æŒ‰ç…§éœ€è¦åˆ›å»ºé¡¹ç›®çº§åˆ«çš„密文。请注æ„，你必须先å¯ç”¨`旧版`功能开关并查看å•ä¸ªé¡¹ç›®ã€‚执行以下步骤设置你的项目级别密文： + +1. 在左上角，å•å‡»ä¸‹æ‹‰èœå•ä¸­çš„ **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 转到 `Legacy` 功能开关并å•å‡»**激活**。 +1. 点击左上角下拉èœå•ä¸­çš„ **☰ > 集群管ç†**。 +1. 转到你创建的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Legacy > 项目**。 +1. 在顶部导航æ ä¸­è¿›è¡Œè¿‡æ»¤ï¼Œä»Žè€Œä»…查看一个项目。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**密文**。 +1. å•å‡»**添加密文**。 + +**结果**：密文已添加到你选择的项目中。你å¯ä»¥åœ¨ Rancher UI 中å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**æ¥æŸ¥çœ‹å¯†æ–‡ã€‚ + +由于项目范围的 Secret 是在项目级别设置的，因此在命å空间级别所åšçš„任何更改都将被覆盖。 + +:::note + +local 集群上的项目级别密文仅在选择å•ä¸ªé¡¹ç›®æ—¶å¯è§ã€‚ + +::: + +## åŽç»­æ“作 + +现在你已将密文添加到命å空间，你å¯ä»¥å°†å…¶æ·»åŠ åˆ°ä½ éƒ¨ç½²çš„工作负载中。 + +有关将密文添加到工作负载的更多信æ¯ï¼Œè¯·å‚阅[部署工作负载](workloads-and-pods/deploy-workloads.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md new file mode 100644 index 00000000000..fe840f7005f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md @@ -0,0 +1,35 @@ +--- +title: 添加 Sidecar +--- + +_sidecar_ 是一个容器，用于扩展或增强 pod 中的主容器。主容器和 Sidecar 共享一个 pod，因此共享相åŒçš„网络空间和存储。你å¯ä»¥ä½¿ç”¨**添加 Sidecar** 选项将 Sidecar 添加到现有工作负载。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ  Sidecar 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 + +1. 找到è¦æ‰©å±•çš„工作负载。选择 **â‹® > + 添加 Sidecar**。 + +1. 输入 Sidecar çš„**å称**。 + +1. 在**通用**中，选择 Sidecar 类型。此选项确定 Sidecar 容器是在主容器之å‰è¿˜æ˜¯ä¹‹åŽéƒ¨ç½²ã€‚ + + - **标准容器**： + + Sidecar 容器部署在主容器之åŽã€‚ + + - **åˆå§‹åŒ–容器**： + + Sidecar 容器部署在主容器之å‰ã€‚ + +1. 在**容器镜åƒ**字段中，输入è¦éƒ¨ç½²æ¥æ”¯æŒä¸»å®¹å™¨çš„容器镜åƒçš„å称。部署时，Rancher 会从 [Docker Hub](https://hub.docker.com/explore/) 拉å–这个镜åƒã€‚输入与 Docker Hub 上完全相åŒçš„å称。 + +1. 设置其余选项。你å¯ä»¥åœ¨[部署工作负载](deploy-workloads.md)中了解它们。 + +1. 点击**å¯åŠ¨**。 + +**结果**：已根æ®ä½ çš„å‚数部署 Sidecar。部署åŽï¼Œä½ å¯ä»¥é€‰æ‹© **â‹® > 编辑** æ¥æŸ¥çœ‹ Sidecar。 + +## 相关链接 + +- [分布å¼ç³»ç»Ÿå·¥å…·åŒ…：å¤åˆå®¹å™¨çš„模å¼](https://kubernetes.io/blog/2015/06/the-distributed-system-toolkit-patterns/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md new file mode 100644 index 00000000000..6fecebc7341 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md @@ -0,0 +1,57 @@ +--- +title: 部署工作负载 +description: 阅读此步骤指å—以部署工作负载。部署工作负载以在一个或多个容器中è¿è¡Œåº”用程åºã€‚ +--- + +部署工作负载以在一个或多个容器中è¿è¡Œåº”用程åºã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡»**创建**。 +1. 选择工作负载的类型。 +1. 选择è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„命å空间。 +1. 为工作负载设置**å称**。 + +1. 在**容器镜åƒ**字段中，输入è¦éƒ¨ç½²åˆ°é¡¹ç›®çš„ Docker é•œåƒçš„å称。你å¯ä»¥é€‰æ‹©åœ¨å‰é¢åŠ ä¸Šé•œåƒä»“库主机（例如 `quay.io`ã€`registry.gitlab.com` 等）。部署时，Rancher 从指定的公共或ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚如果没有æ供镜åƒä»“库主机，Rancher 将从 [Docker Hub](https://hub.docker.com/explore/) 拉å–é•œåƒã€‚输入与镜åƒä»“库æœåŠ¡å™¨ä¸­æ˜¾ç¤ºçš„å称完全相åŒçš„å称，任何所需的路径，也å¯ä»¥é€‰æ‹©æ‰€éœ€çš„标签（例如 `registry.gitlab.com/user/path/image:tag`）。如果没有æ供标签，将自动使用 `latest` 标签。 + +1. 选择现有命å空间，或å•å‡»**添加到新命å空间**并输入新命å空间。 + +1. 点击**添加端å£**进入端å£æ˜ å°„，这让你å¯ä»¥è®¿é—®é›†ç¾¤å†…外的应用程åºã€‚如需更多信æ¯ï¼Œè¯·å‚阅 [Service](../../../../pages-for-subheaders/workloads-and-pods.md#services)。 + +1. é…置其余选项： + + - **环境å˜é‡** + + ä½ å¯ä»¥åœ¨è¿™é‡Œä¸ºå·¥ä½œè´Ÿè½½æŒ‡å®šçŽ¯å¢ƒå˜é‡ä»¥å³æ—¶ä½¿ç”¨å®ƒä»¬ï¼Œæˆ–者从其他æ¥æºï¼ˆä¾‹å¦‚密文或 [ConfigMap](../configmaps.md)）拉å–它们。 + + - **节点调度** + - **å¥åº·æ£€æŸ¥** + - **å·** + + 在此处为你的工作负载添加存储。你å¯ä»¥æ‰‹åŠ¨æŒ‡å®šè¦æ·»åŠ çš„å·ï¼Œä½¿ç”¨æŒä¹…å·å£°æ˜Žä¸ºå·¥ä½œè´Ÿè½½åŠ¨æ€åˆ›å»ºå·ï¼Œæˆ–从 [ConfigMap](../configmaps.md) 等文件中读å–è¦ä½¿ç”¨çš„å·æ•°æ®ã€‚ + + 部署 StatefulSet 时，应在使用æŒä¹…å·æ—¶ä½¿ç”¨å·å£°æ˜Žæ¨¡æ¿ã€‚这将确ä¿åœ¨æ‰©å±• StatefulSet 时能动æ€åˆ›å»ºæŒä¹…å·ã€‚ + + - **扩展/å‡çº§ç­–ç•¥** + + :::note AWS å·æ³¨æ„事项： + + è¦æŒ‚è½½ Amazon EBS å·ï¼š + + - 在 [AWS](https://aws.amazon.com/) 中，节点必须ä½äºŽåŒä¸€å¯ç”¨åŒºä¸­å¹¶å…·æœ‰é™„加/分离å·çš„ IAM æƒé™ã€‚ + + - 集群必须使用 [AWS 云æ供商](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#aws)选项。有关å¯ç”¨æ­¤é€‰é¡¹çš„更多信æ¯ï¼Œè¯·å‚阅[创建 AWS EC2 集群](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md)或[创建自定义集群](../../../../pages-for-subheaders/use-existing-nodes.md)。 + + ::: + +1. 点击**显示高级选项**并é…置： + + - **命令** + - **网络** + - **标签 & 注释** + - **安全和主机é…ç½®** + +1. 点击**å¯åŠ¨**。 + +**结果**：工作负载已部署到选定的命å空间。你å¯ä»¥åœ¨é¡¹ç›®çš„**工作负载**视图查看工作负载的状æ€ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md new file mode 100644 index 00000000000..c12dfdd469a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md @@ -0,0 +1,14 @@ +--- +title: 回滚工作负载 +--- + +在调试或å‡çº§ä¸é¡ºåˆ©æ—¶ï¼Œä½ å¯èƒ½éœ€è¦å°†åº”用程åºå›žæ»šåˆ°å…ˆå‰çš„版本。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. 找到è¦å›žæ»šçš„工作负载，然åŽé€‰æ‹© **â‹® > 回滚**。 + +1. 选择è¦å›žæ»šåˆ°çš„修订版。å•å‡»**回滚**。 + +**结果**：你的工作负载已æ¢å¤åˆ°ä½ é€‰æ‹©çš„å…ˆå‰ç‰ˆæœ¬ã€‚等待几分钟待æ“作完æˆã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md new file mode 100644 index 00000000000..6d2313f5514 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md @@ -0,0 +1,21 @@ +--- +title: å‡çº§å·¥ä½œè´Ÿè½½ +--- + +如果 Docker Hub å‘布新版本的应用程åºé•œåƒï¼Œä½ å¯ä»¥å°†è¿è¡Œæ—§ç‰ˆåº”用程åºçš„工作负载å‡çº§åˆ°æ–°ç‰ˆæœ¬ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 + +1. 找到è¦å‡çº§çš„工作负载并选择 **â‹® > 编辑é…ç½®**。 + +1. æ›´æ–°**容器镜åƒ**å’Œè¦æ›´æ”¹çš„选项。 + +1. 查看和编辑工作负载的**扩展/å‡çº§ç­–ç•¥**。 + + 这些选项能控制å‡çº§å¦‚何在è¿è¡Œçš„容器中进行。例如，对于å¯æ‰©å±•çš„ deployment，你å¯ä»¥é€‰æ‹©æ˜¯å¦è¦åœ¨éƒ¨ç½²æ–° Pod 之å‰åœæ­¢æ—§ Pod，以åŠé€‰æ‹©å‡çº§çš„批数é‡ã€‚ + +1. å•å‡»**ä¿å­˜**。 + +**结果**：工作负载会根æ®ä½ çš„设置æ¥å‡çº§å®¹å™¨ã€‚请注æ„，扩展 deployment 或更新å‡çº§/扩展策略ä¸ä¼šå¯¼è‡´ Pod é‡æ–°åˆ›å»ºã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md new file mode 100644 index 00000000000..15263fff292 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md @@ -0,0 +1,43 @@ +--- +title: Rancher Agent +--- + +Rancher 管ç†çš„集群上部署了两ç§ä¸åŒçš„ Agent 资æºï¼š + +- [cattle-cluster-agent](#cattle-cluster-agent) +- [cattle-node-agent](#cattle-node-agent) + +有关 Rancher Server 如何é…置集群并与集群通信的概述，请å‚阅[产å“架构](../../../pages-for-subheaders/rancher-manager-architecture.md)。 + +### cattle-cluster-agent + +`cattle-cluster-agent` 用于连接 [Rancher å¯åŠ¨çš„ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群的 Kubernetes API。`cattle-cluster-agent` 使用 Deployment 资æºè¿›è¡Œéƒ¨ç½²ã€‚ + +### cattle-node-agent + +`cattle-node-agent` 用于在执行集群æ“作时与 [Rancher å¯åŠ¨çš„ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群中的节点进行交互。集群æ“作包括å‡çº§ Kubernetes 版本和创建/æ¢å¤ etcd 快照。`cattle-node-agent` 使用 DaemonSet 资æºè¿›è¡Œéƒ¨ç½²ï¼Œä»¥ç¡®ä¿èƒ½åœ¨æ¯ä¸ªèŠ‚点上è¿è¡Œã€‚当 `cattle-cluster-agent` ä¸å¯ç”¨æ—¶ï¼Œ`cattle-node-agent` å¯ä»¥ä½œä¸ºå¤‡é€‰æ–¹æ¡ˆï¼Œç”¨æ¥è¿žæŽ¥ [Rancher å¯åŠ¨çš„ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群的 Kubernetes API。 + +### 调度规则 + +`cattle-cluster-agent` 使用一组固定的容å¿åº¦ï¼Œæˆ–基于应用于 control plane 节点的污点动æ€æ·»åŠ çš„容å¿åº¦ã€‚è¿™ç§ç»“æž„å…许[基于污点进行驱é€](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions)为 `cattle-cluster-agent` 正常工作。 + +如果集群中存在 control plane 节点，则默认容å¿åº¦å°†æ›¿æ¢ä¸ºä¸Ž control plane 节点的污点匹é…的容å¿åº¦ã€‚默认的容å¿åº¦å¦‚下。 + +| 组件 | nodeAffinity nodeSelectorTerms | nodeSelector | 容å¿åº¦ | +| ---------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| `cattle-cluster-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | **注æ„**：这些是默认容å¿åº¦ï¼Œå¹¶å°†æ›¿æ¢ä¸ºä¸Ž controlplane 节点的污点匹é…的容å¿åº¦ã€‚

`effect:NoSchedule`
`key:node-role.kubernetes.io/controlplane`
`value:true`

`effect:NoSchedule`
`key:node-role.kubernetes.io/control-plane`
`operator:Exists`

`effect:NoSchedule`
`key:node-role.kubernetes.io/master`
`operator:Exists` | +| `cattle-node-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | `operator:Exists` | + +`cattle-cluster-agent` Deployment 使用 `preferredDuringSchedulingIgnoredDuringExecution` 的首选调度规则，倾å‘于在具有 `controlplane` 节点的节点上进行调度。当集群中没有å¯è§çš„ controlplane 节点时（通常是使用[æ供商托管的 Kubernetes 的集群](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)），你å¯ä»¥åœ¨èŠ‚点上添加 `cattle.io/cluster-agent=true` 标签，从而优先将 `cattle-cluster-agent` pod 调度到该节点。 + +有关调度规则的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes：将 Pod 分é…给节点](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)。 + +`preferredDuringSchedulingIgnoredDuringExecution` é…置如下表所示： + +| æƒé‡ | è¡¨è¾¾å¼ | +| ------ | ------------------------------------------------ | +| 100 | `node-role.kubernetes.io/controlplane:In:"true"` | +| 100 | `node-role.kubernetes.io/control-plane:In:"true"` | +| 100 | `node-role.kubernetes.io/master:In:"true"` | +| 1 | `cattle.io/cluster-agent:In:"true"` | + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md new file mode 100644 index 00000000000..3941f3b37c3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md @@ -0,0 +1,31 @@ +--- +title: RKE1 å’Œ RKE2 差​​异 +--- + +RKE2，也称为 RKE Government，是一个完全符åˆæ ‡å‡†çš„ Kubernetes å‘行版，它专注于安全性和åˆè§„性。它被认为是 Rancher Kubernetes Engineï¼ˆå³ RKE1）的下一个迭代。 + +RKE1 å’Œ RKE2 有一些细微的差异，本文将é‡ç‚¹ä»‹ç»è¿™äº›å·®å¼‚。 + +### controlplane 组件 + +RKE1 使用 Docker æ¥éƒ¨ç½²å’Œç®¡ç† controlplane 组件，还使用 Docker 作为 Kubernetes 的容器è¿è¡Œæ—¶ã€‚相比之下，RKE2 å°† controlplane 组件作为由 kubelet 管ç†çš„é™æ€ pod å¯åŠ¨ã€‚RKE2 的容器è¿è¡Œæ—¶æ˜¯ Containerd，它å…许 Mirror 容器镜åƒä»“库等内容。使用 Docker çš„ RKE1 ä¸å…许 Mirror。 + +### Cluster API + +RKE2/K3s é…置是基于 Cluster API (CAPI) 上游框架之上构建的，这导致 RKE2 é…置的集群的行为通常与 RKE1 é…置的集群ä¸åŒã€‚ + +如果你在 RKE2 中更改集群é…置，节点**å¯èƒ½**会é‡æ–°é…置。这是由 CAPI 控制器而ä¸æ˜¯ Rancher 控制的。请注æ„，etcd 节点ä¸é€‚用相åŒçš„行为。 + +以下是一些å¯èƒ½å¯¼è‡´ä¸Šè¿°è¡Œä¸ºçš„é…置更改示例： + +- 编辑集群并å¯ç”¨`删除å‰æ¸…空`时，会删除现有的 controlplane 节点和 worker 并创建新节点。 + +如果你是习惯于 RKE1 é…置的用户，请注æ„æ–°çš„ RKE2 行为。 + +### åè¯è§£é‡Š + +从 RKE1 到 RKE2，æŸäº›æœ¯è¯­å·²æ›´æ”¹æˆ–å·²ä¸å†ä½¿ç”¨ã€‚例如，在 RKE1中，你使用**节点模æ¿**，而在 RKE2 中，你å¯ä»¥åœ¨åˆ›å»ºæˆ–编辑集群时é…置集群节点池。å¦ä¸€ä¸ªä¾‹å­æ˜¯ RKE1 中的**节点池（node pool）** 现在在 RKE2 中称为**主机池（machine pool）**。 + + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md new file mode 100644 index 00000000000..39f8b418151 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md @@ -0,0 +1,99 @@ +--- +title: 创建 DigitalOcean 集群 +--- + +在本节中，你将学习如何使用 Rancher 在 DigitalOcean 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 DigitalOcean 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ DigitalOcean 中é…置新节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 DigitalOcean 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + + + + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **DigitalOcean**。 +1. 输入你的 Digital Ocean 凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 DigitalOcean 创建[节点模æ¿](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿) 会å…许 Rancher 在 DigitalOcean 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **DigitalOcean**。 +1. 填写 DigitalOcean 的节点模æ¿ã€‚有关填写表å•çš„帮助，请å‚阅 [DigitalOcean 节点模æ¿é…ç½®](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md)。 + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **DigitalOcean**。 +1. 输入**集群å称**。 +1. 将一个或多个节点池添加到你的集群。将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +1. 在**集群é…ç½®**中，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +### 1. åˆ›å»ºäº‘å‡­è¯ + +如果你已ç»æœ‰ä¸€ç»„å¯ç”¨çš„云凭è¯ï¼Œè¯·è·³è¿‡æ­¤éƒ¨åˆ†ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **DigitalOcean**。 +1. 输入你的 Digital Ocean 凭è¯ã€‚ +1. å•å‡»**创建**。 + +### 2. 创建你的集群 + +使用 Rancher 在 DigitalOcean 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 将开关切æ¢åˆ° **RKE2/K3s**。 +1. å•å‡» **DigitalOcean**。 +1. 选择一个**云凭è¯**。如果存在多个则需è¦é€‰æ‹©ã€‚å¦åˆ™ï¼Œå®ƒæ˜¯é¢„选的。 +1. 输入**集群å称**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个主机池。请å‚阅[最佳实践](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点角色)了解角色分é…和计数的建议。 + 1. 为æ¯ä¸ªä¸»æœºæ± å®šä¹‰ä¸»æœºé…置。有关é…置选项的信æ¯ï¼Œè¯·å‚阅 [DigitalOcean 主机é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md)。 +1. 使用**集群é…ç½®**，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。有关é…置集群的帮助，请å‚阅 [RKE2 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md new file mode 100644 index 00000000000..b2918cf475a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md @@ -0,0 +1,283 @@ +--- +title: 创建 Amazon EC2 集群 +description: 了解使用 Rancher 创建 Amazon EC2 集群所需的先决æ¡ä»¶å’Œæ­¥éª¤ +--- + +在本节中，你将学习如何使用 Rancher 在 Amazon EC2 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 EC2 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ EC2 中é…置新节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 EC2 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +### 先决æ¡ä»¶ + +- **AWS EC2 访问密钥和密文密钥**，用于创建实例。请å‚阅 [Amazon 文档：创建访问密钥](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)æ¥åˆ›å»ºè®¿é—®å¯†é’¥å’Œå¯†æ–‡å¯†é’¥ã€‚ +- **已创建 IAM ç­–ç•¥**，用于为用户添加的访问密钥和密文密钥。请å‚阅 [Amazon 文档：创建 IAM 策略（控制å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start)æ¥åˆ›å»º IAM 策略。å‚阅下é¢çš„三个示例 JSON 策略： + - [IAM 策略示例](#iam-策略示例) + - [带有 PassRole çš„ IAM 策略示例](#带有-passrole-çš„-iam-策略示例)（如果è¦ä½¿ç”¨ [Kubernetes 云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)，或将 IAM é…置文件传递给实例，则需è¦ï¼‰ + - [å…许加密 EBS å·çš„ IAM 策略示例](#å…许加密-ebs-å·çš„-iam-策略示例) +- 为用户添加 **IAM ç­–ç•¥æƒé™**。请å‚阅 [Amazon 文档：为用户添加æƒé™ï¼ˆæŽ§åˆ¶å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console)，æ¥å°†æƒé™æ·»åŠ ç»™ç”¨æˆ·ã€‚ + +## 创建 EC2 集群 + +创建集群的步骤因 Rancher 版本而异。 + + + + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯å’Œ EC2 çš„ä¿¡æ¯æ¥åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯å’Œ-ec2-çš„ä¿¡æ¯æ¥åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Amazon**。 +1. 输入云凭è¯çš„å称。 +1. 在**默认区域**字段中，选择集群节点所在的 AWS 区域。 +1. 输入 AWS EC2 **Access Key** å’Œ **Secret Key**。 +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯å’Œ EC2 çš„ä¿¡æ¯æ¥åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 EC2 创建[节点模æ¿](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 EC2 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. 填写 EC2 的节点模æ¿ã€‚有关填写表å•çš„帮助，请å‚阅 [EC2 节点模æ¿é…ç½®](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md)。 +1. å•å‡»**创建**。 + + :::note + + 如果è¦ä½¿ç”¨[åŒæ ˆ](https://kubernetes.io/docs/concepts/services-networking/dual-stack/)功能，请关注其他é¢å¤–çš„[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + + ::: + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +将一个或多个节点池添加到你的集群。有关节点池的更多信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Amazon EC2**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个节点池。为æ¯ä¸ªèŠ‚点池选择你已创建的节点模æ¿ã€‚有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +1. 点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 使用**集群选项**选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。å‚è§[选择云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)æ¥é…ç½® Kubernetes 云æ供商。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + + :::note + + 如果è¦ä½¿ç”¨[åŒæ ˆ](https://kubernetes.io/docs/concepts/services-networking/dual-stack/)功能，请关注其他é¢å¤–çš„[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + + ::: + +1. å•å‡»**创建**。 + + + + +### 1. åˆ›å»ºäº‘å‡­è¯ + +如果你已ç»æœ‰ä¸€ç»„å¯ç”¨çš„云凭è¯ï¼Œè¯·è·³è¿‡æ­¤éƒ¨åˆ†ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Amazon**。 +1. 输入云凭è¯çš„å称。 +1. 在**默认区域**字段中，选择集群节点所在的 AWS 区域。 +1. 输入 AWS EC2 **Access Key** å’Œ **Secret Key**。 +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 创建你的集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 将开关切æ¢åˆ° **RKE2/K3s**。 +1. å•å‡» **Amazon EC2**。 +1. 选择一个**云凭è¯**。如果存在多个则需è¦é€‰æ‹©ã€‚å¦åˆ™ï¼Œå®ƒæ˜¯é¢„选的。 +1. 输入**集群å称**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个主机池。请å‚阅[最佳实践](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点角色)了解角色分é…和计数的建议。 + 1. 为æ¯ä¸ªä¸»æœºæ± å®šä¹‰ä¸»æœºé…置。有关é…置选项的信æ¯ï¼Œè¯·å‚阅 [EC2 主机é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md)。 +1. 使用**集群é…ç½®**，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。有关é…置集群的帮助，请å‚阅 [RKE2 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +### å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 + +## IAM ç­–ç•¥ + +### IAM 策略示例 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` + +### 带有 PassRole çš„ IAM 策略示例 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*", + "arn:aws:iam::AWS_ACCOUNT_ID:role/YOUR_ROLE_NAME" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` +### å…许加密 EBS å·çš„ IAM 策略示例 +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:Encrypt", + "kms:DescribeKey", + "kms:CreateGrant", + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:CreateSnapshot" + ], + "Resource": [ + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:snapshot/*", + "arn:aws:kms:REGION:AWS_ACCOUNT_ID:key/KMS_KEY_ID" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots" + ], + "Resource": "*" + } + ] +} +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md new file mode 100644 index 00000000000..a3af5bca578 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md @@ -0,0 +1,141 @@ +--- +title: 创建 Azure 集群 +--- + +在本节中，你将学习如何通过 Rancher 在 Azure 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 Azure 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ Azure 中é…置新节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 Azure 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +:::caution + +当 Rancher RKE 集群在 Azure 中è¿è¡Œå¹¶ä¸”å‰é¢æœ‰ Azure è´Ÿè½½å‡è¡¡å™¨æ—¶ï¼Œå‡ºç«™æµé‡ä¼šå¤±è´¥ã€‚问题的解决方法如下： + +- 终止内部负载å‡è¡¡å™¨ä¸Šçš„ SSL/TLS +- 使用 L7 è´Ÿè½½å‡è¡¡å™¨ + +详情请å‚è§ [Azure è´Ÿè½½å‡è¡¡å™¨é™åˆ¶](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations)。 + +::: + +有关 Rancher 在 Azure 节点上安装的 Kubernetes 集群的更多信æ¯ï¼Œè¯·å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + +有关é…ç½® Azure 节点模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅 [Azure 节点模æ¿é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md)。 + +- [Azure 中的准备工作](#azure-中的准备工作) +- [创建 Azure 集群](#创建-azure-集群) + +## Azure 中的准备工作 + +在使用 Azure 等云厂商在 Rancher 中创建节点模æ¿ä¹‹å‰ï¼Œå¿…须先é…ç½® Rancher 以å…许æ“作 Azure 订阅中的资æºã€‚ + +为此，首先在 Azure çš„ **Active Directory (AD)** 中创建一个新的 Azure **service principal (æœåŠ¡ä¸»ä½“，缩写是 SP)**ï¼Œå®ƒæ˜¯å…·æœ‰ç®¡ç† Azure æƒé™çš„应用用户。 + +以下是创建æœåŠ¡ä¸»ä½“å¿…é¡»è¿è¡Œçš„ `az cli` 脚本模æ¿ï¼Œä½ å¿…须在其中输入你的æœåŠ¡ä¸»ä½“å称ã€è§’色和范围： + +``` +az ad sp create-for-rbac \ + --name="" \ + --role="Contributor" \ + --scopes="/subscriptions/" +``` + +创建æœåŠ¡ä¸»ä½“åŽä¼šè¿”回三个标识信æ¯ï¼Œåˆ†åˆ«æ˜¯*应用 ID（也称为客户端 ID）*，以åŠ*客户端密ç *。为 Azure 创建节点模æ¿æ—¶å°†ä½¿ç”¨è¿™äº›ä¿¡æ¯ã€‚ + +## 创建 Azure 集群 + + + + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 输入你的 Azure 凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 Azure 创建[节点模æ¿](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 Azure 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **Azure**。 +1. 填写 Azure 的节点模æ¿ã€‚有关填写表å•çš„帮助，请å‚阅 [Azure 节点模æ¿é…ç½®](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md)。 + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +使用 Rancher 在 Azure 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 输入**集群å称**。 +1. 将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池åŠå…¶æœ€ä½³å®žè·µçš„更多信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +1. 在**集群é…ç½®**中，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +### 1. åˆ›å»ºäº‘å‡­è¯ + +如果你已ç»æœ‰ä¸€ç»„å¯ç”¨çš„云凭è¯ï¼Œè¯·è·³è¿‡æ­¤éƒ¨åˆ†ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 输入你的 Azure 凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 创建你的集群 + +使用 Rancher 在 Azure 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 将开关切æ¢åˆ° **RKE2/K3s**。 +1. å•å‡» **Azure**。 +1. 选择一个**云凭è¯**。如果存在多个则需è¦é€‰æ‹©ã€‚å¦åˆ™ï¼Œå®ƒæ˜¯é¢„选的。 +1. 输入**集群å称**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个主机池。请å‚阅[最佳实践](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点角色)了解角色分é…和计数的建议。 + 1. 为æ¯ä¸ªä¸»æœºæ± å®šä¹‰ä¸»æœºé…置。有关é…置选项的信æ¯ï¼Œè¯·å‚阅 [Azure 主机é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md)。 +1. 使用**集群é…ç½®**，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。有关é…置集群的帮助，请å‚阅 [RKE2 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +### å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md new file mode 100644 index 00000000000..c360364aa91 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md @@ -0,0 +1,92 @@ +--- +title: 在 Nutanix AOS 中é…ç½® Kubernetes 集群 +--- + +è¦ä½¿ç”¨ Rancher 在 Nutanix AOS (AHV) 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群： + +1. 找到 Rancher 的内置 Nutanix [主机驱动并激活它](../../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#激活åœç”¨ä¸»æœºé©±åŠ¨)。 + +1. 创建一个节点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ Nutanix AOS 中é…置节点。 + +1. 在 Rancher 中创建 Nutanix AOS 集群。é…置新集群时，你需è¦ä¸ºå®ƒå®šä¹‰èŠ‚点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +有关é…ç½® Nutanix AOS 节点模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅 [Nutanix AOS 节点模æ¿é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md)。 + +有关在 Rancher 中é…ç½® RKE Kubernetes 集群的详细信æ¯ï¼Œè¯·å‚阅[集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + +- [Nutanix AOS 中的准备工作](#nutanix-aos-中的准备工作) +- [创建 Nutanix AOS 集群](#创建-nutanix-aos-集群) + +## Nutanix AOS 中的准备工作 + +下文介ç»äº†è®¾ç½® Nutanix AOS çš„è¦æ±‚，以便 Rancher å¯ä»¥é…置虚拟机和集群。 + +:::note + +节点模æ¿å·²ä½¿ç”¨ Nutanix AOS 5.20.2 å’Œ 6.0.1 版本进行记录和测试。 + +::: + +### 在 Nutanix AOS ä¸­åˆ›å»ºå‡­è¯ + +在继续创建集群之å‰ï¼Œè¯·ç¡®ä¿ä½ æ‹¥æœ‰å…·æœ‰ç®¡ç†å‘˜æƒé™çš„ [Nutanix Prism Central 用户账å·](https://portal.nutanix.com/page/documents/details?targetId=Nutanix-Security-Guide-v6_0:wc-user-create-wc-t.html)。设置节点模æ¿æ—¶ï¼Œæ¨¡æ¿å°†éœ€è¦ä½¿ç”¨è¿™äº›å‡­è¯ã€‚ + +### 网络æƒé™ + +必须确ä¿è¿è¡Œ Rancher Server 的主机能够建立以下网络连接： + +- 连接到 Nutanix Prism Central APIï¼ˆé€šå¸¸æ˜¯ç«¯å£ 9440/TCP）。 +- 连接到创建的 VM 上的 22/TCP å’Œ 2376/TCP 端å£ã€‚ + +有关在基础设施æ供商上创建节点的端å£è¦æ±‚，请å‚阅[节点网络è¦æ±‚](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +### VM-VM å亲和性策略 + +建议设置 [VM-VM å亲和性策略](https://portal.nutanix.com/page/documents/details?targetId=AHV-Admin-Guide-v6_1:ahv-vm-anti-affinity-t.html)。这些规则å…许分é…了 etcd å’Œ controlplane 角色的虚拟机在分é…ç»™ä¸åŒèŠ‚点池时，在å•ç‹¬çš„ AHV 主机上è¿è¡Œã€‚è¿™ç§åšæ³•å¯ç¡®ä¿å•ä¸ªç‰©ç†æœºçš„æ•…éšœä¸ä¼šå½±å“这些平é¢çš„å¯ç”¨æ€§ã€‚ + +## 创建 Nutanix AOS 集群 + +1. [创建节点模æ¿](#1-创建节点模æ¿) +2. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#2-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 Nutanix AOS 创建[节点模æ¿](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 Nutanix AOS 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**创建**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **Nutanix**。 +1. 填写 Nutanix AOS 的节点模æ¿ã€‚有关填写表格的帮助，请å‚阅 Nutanix AOS 节点模æ¿[é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md)。 +1. å•å‡»**创建**。 + +### 2. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +使用 Rancher 在 Nutanix AOS 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Nutanix**。 +1. 输入**集群å称**，然åŽç‚¹å‡»**继续**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 使用**集群选项**选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)。 +1. 检查并确认你的选项。然åŽå•å‡»**创建**。 + +**结果**：集群已创建，并处于 **Provisioning** 状æ€ã€‚Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ + +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md new file mode 100644 index 00000000000..6872dae6a02 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md @@ -0,0 +1,162 @@ +--- +title: 创建 vSphere è™šæ‹Ÿæœºæ¨¡æ¿ +--- + +è¦é‡å¤ä¸”å¯é åœ°åˆ›å»ºè™šæ‹Ÿæœºé€šå¸¸éžå¸¸å›°éš¾ã€‚VMware vSphere 支æŒæž„建å¯ä»¥è½¬æ¢ä¸ºæ¨¡æ¿çš„虚拟机。然åŽï¼Œä½ å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿æ¥åˆ›å»ºé…置相åŒçš„虚拟机。Rancher 会在节点池中利用此功能æ¥åˆ›å»ºç›¸åŒçš„ RKE1 å’Œ RKE2 节点。 + +为了使用模æ¿åˆ›å»ºæ–°çš„虚拟机，Rancher 有虚拟机必须预先安装的[特定è¦æ±‚](#è¦æ±‚)。根æ®è¿™äº›è¦æ±‚é…置虚拟机åŽï¼Œä½ éœ€è¦[准备虚拟机](#准备虚拟机)，然åŽå†[创建模æ¿](#创建模æ¿)。准备工作完æˆåŽï¼Œè™šæ‹Ÿæœºå¯ä»¥[转æ¢ä¸ºæ¨¡æ¿](#转æ¢ä¸ºæ¨¡æ¿)并[移动到内容库中](#移动到内容库)ï¼Œç„¶åŽ Rancher 节点池就å¯ä»¥ä½¿ç”¨å®ƒäº†ã€‚ + + +## è¦æ±‚ + +Linux å’Œ Windows 虚拟机都需è¦ç‰¹å®šå·¥å…·æ‰èƒ½ä¾› vSphere 主机驱动使用。最关键的ä¾èµ–项是 Linux çš„ [cloud-init](https://cloud-init.io/) å’Œ Windows çš„ [cloudbase-init](https://cloudbase.it/cloudbase-init/)。二者都用于通过é…置主机å和设置 SSH 访问以åŠé»˜è®¤ Rancher 用户æ¥é…置虚拟机。如果需è¦å…¶ä»–é…置，用户å¯ä»¥æ ¹æ®éœ€è¦æ·»åŠ å…¶ä»–内容。此外，下é¢åˆ—出了其他è¦æ±‚以供å‚考。 + +:::note + +如果你有任何特定的防ç«å¢™è§„则或é…置，则需è¦åœ¨åˆ›å»ºæ¨¡æ¿ä¹‹å‰å°†å…¶æ·»åŠ åˆ°è™šæ‹Ÿæœºã€‚ + +::: + +## Linux ä¾èµ–项 + +下é¢åˆ—出了需è¦åœ¨æ¨¡æ¿ä¸Šå®‰è£…的包。ä¸åŒçš„å‘行版对应的å称会略有ä¸åŒï¼Œä¾‹å¦‚，æŸäº›å‘行版会默认æ供。 + +* curl +* wget +* git +* net-tools +* unzip +* apparmor-parser +* ca-certificates +* cloud-init +* cloud-guest-utils +* cloud-image-utils +* growpart +* cloud-initramfs-growroot +* open-iscsi +* openssh-server +* [open-vm-tools](https://docs.vmware.com/en/VMware-Tools/11.3.0/com.vmware.vsphere.vmwaretools.doc/GUID-8B6EA5B7-453B-48AA-92E5-DB7F061341D1.html) + +## Windows ä¾èµ–项 + +模æ¿ä¸Šéœ€è¦å®‰è£…的包如下： + +* Windows 容器功能 +* [cloudbase-init](https://cloudbase.it/cloudbase-init/#download) +* [Docker EE](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server#install-docker) - ä»…é™ RKE1 + +:::note + +RKE1 å’Œ RKE2 对应的 Windows 模æ¿çš„é…置有所ä¸åŒï¼š + +- RKE1 使用 Docker，因此任何 RKE1 模æ¿éƒ½éœ€è¦é¢„先安装 Docker EE +- RKE2 ä¸éœ€è¦ Docker EE，因此ä¸éœ€è¦å®‰è£… + +::: + +## åˆ›å»ºæ¨¡æ¿ + +ä½ å¯ä»¥æ‰‹åŠ¨åˆ›å»ºè™šæ‹Ÿæœºï¼Œä¹Ÿå¯ä»¥ä½¿ç”¨[其他替代方法](#手动创建的替代方案)æ¥åˆ›å»ºè™šæ‹Ÿæœºã€‚ + +### 手动创建 +1. 在 VMware 中按照[这些说明](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AE8AFBF1-75D1-4172-988C-378C35C9FAF2.html)手动创建虚拟机。虚拟机è¿è¡ŒåŽï¼Œä½ å¯ä»¥æ‰‹åŠ¨å®‰è£…上é¢åˆ—出的ä¾èµ–项，以便为 vSphere 主机驱动正确é…置虚拟机。 +2. æ ¹æ®ä½ çš„特定环境和è¦æ±‚按需定制。 +3. 在创建模æ¿ä¹‹å‰è¿›è¡Œæœ€åŽçš„准备工作。 + +### 手动创建的替代方案 + +下é¢åˆ—出了创建虚拟机的其他替代选项： + +* [VMware PowerCLI](https://developer.vmware.com/powercli) +* [Packer](https://www.packer.io/) +* [SaltStack](https://saltproject.io/) +* [Ansible](https://www.ansible.com/) + +Packer 是一ç§å¸¸ç”¨çš„替代方案。有关将其与 vSphere 一起使用的示例，请å‚阅[å‚考](https://github.com/vmware-samples/packer-examples-for-vsphere)。 + +## 准备虚拟机 + +创建具有所有必需ä¾èµ–项（以åŠä»»ä½•å…¶ä»–必需项）的虚拟机åŽï¼Œä½ å¿…须执行最关键的下一个步骤，å³å‡†å¤‡å°†è™šæ‹Ÿæœºè½¬æ¢ä¸ºæ¨¡æ¿ã€‚此准备步骤会é‡ç½®å…³é”®æ•°æ®ï¼ˆä¾‹å¦‚虚拟机主机åã€IP 等）以防止这些信æ¯è¢«å¸¦å…¥æ–°è™šæ‹Ÿæœºã€‚如果你无法执行此步骤，你也å¯ä»¥åˆ›å»ºä¸€ä¸ªå…·æœ‰ç›¸åŒä¸»æœºåã€IP 地å€ç­‰çš„虚拟机。 + +请注æ„，Linux å’Œ Windows 对应的准备步骤有所ä¸åŒã€‚ + +### Linux 准备 + +以下命令将在 Linux 中é‡ç½®ä½ çš„虚拟机： + +```bash +# 清ç†æ—¥å¿—。 +if [ -f /var/log/audit/audit.log ]; then + cat /dev/null > /var/log/audit/audit.log +fi +if [ -f /var/log/wtmp ]; then + cat /dev/null > /var/log/wtmp +fi +if [ -f /var/log/lastlog ]; then + cat /dev/null > /var/log/lastlog +fi + +# æ¸…ç† udev 规则。 +if [ -f /etc/udev/rules.d/70-persistent-net.rules ]; then + rm /etc/udev/rules.d/70-persistent-net.rules +fi + +# æ¸…ç† /tmp 路径。 +rm -rf /tmp/* +rm -rf /var/tmp/* + +# æ¸…ç† SSH 主机密钥。 +rm -f /etc/ssh/ssh_host_* + +# æ¸…ç† machine-id。 +truncate -s 0 /etc/machine-id +rm /var/lib/dbus/machine-id +ln -s /etc/machine-id /var/lib/dbus/machine-id + +# æ¸…ç† shell 历å²ã€‚ +unset HISTFILE +history -cw +echo > ~/.bash_history +rm -fr /root/.bash_history + +# 截断主机åã€ä¸»æœºå’Œ resolv.conf，并将主机å设置为 localhost。 +truncate -s 0 /etc/{hostname,hosts,resolv.conf} +hostnamectl set-hostname localhost + +# æ¸…ç† cloud-init。 +cloud-init clean -s -l +``` + +### Windows 准备 + +Windows 有一个å为 [sysprep](https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/sysprep--generalize--a-windows-installation) 的实用程åºï¼Œç”¨äºŽä¸€èˆ¬åŒ–é•œåƒå¹¶é‡ç½®ä¸Šè¿° Linux 项目。命令如下： + +```PowerShell +sysprep.exe /generalize /shutdown /oobe +``` + +## 转æ¢ä¸ºæ¨¡æ¿ + +1. 关闭并åœæ­¢è™šæ‹Ÿæœºã€‚ +2. å³é”®å•å‡»æ¸…å•åˆ—表中的虚拟机，然åŽé€‰æ‹©**模æ¿**。 +3. 点击**转æ¢ä¸ºæ¨¡æ¿**。 + +**结果**：æµç¨‹å®ŒæˆåŽï¼Œå³å¯ä½¿ç”¨æ¨¡æ¿ã€‚ + +有关将虚拟机转æ¢ä¸ºæ¨¡æ¿çš„更多信æ¯ï¼Œè¯·å‚阅 [VMware 指å—](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-5B3737CC-28DB-4334-BD18-6E12011CDC9F.html)。 + +## 移动到内容库 + +Rancher 支æŒä½¿ç”¨å†…容库æ供的模æ¿ã€‚内容库在 vSphere 中存储和管ç†å†…容，还支æŒå‘布和共享该内容。 + +以下是有关内容库的一些有用链接： + +* [创建内容库](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-2A0F1C13-7336-45CE-B211-610D39A6E1F4.html) +* [将模æ¿å…‹éš†åˆ°å†…容库](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AC1545F0-F8BA-4CD2-96EB-21B3DFAA1DC1.html) + +## å…¶ä»–èµ„æº + +以下是å¯èƒ½æœ‰ç”¨çš„其他资æºåˆ—表： + +* [Linux 模æ¿åˆ›å»ºæ•™ç¨‹](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-ubuntu-template) +* [Windows 模æ¿åˆ›å»ºæ•™ç¨‹](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-windows-template) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md new file mode 100644 index 00000000000..8e1905f7ecc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md @@ -0,0 +1,45 @@ +--- +title: 在 vSphere 控制å°ä¸­åˆ›å»ºå‡­è¯ +--- + +本节介ç»å¦‚何创建 vSphere 用户å和密ç ã€‚你需è¦å°†è¿™äº› vSphere 凭è¯æ供给 Rancher，Rancher æ‰èƒ½åœ¨ vSphere 中é…置资æºã€‚ + +下表列出了 vSphere 用户账å·æ‰€éœ€çš„æƒé™ï¼š + +| 特æƒç»„ | æ“作 | +|:-------------------------|:-----------------------------------------------------------------------| +| Cns Privileges | Searchable | +| Content library | Read Storage
(Required only if deploying Template from Content Library) | +| Cryptographic operations | Direct Access | +| Datastore | AllocateSpace
Browse
FileManagement (Low level file operations)
UpdateVirtualMachineFiles
UpdateVirtualMachineMetadata | +| 全局 | Set custom attribute | +| 网络 | Assign | +| èµ„æº | AssignVMToPool | +| 虚拟机 | Config (All)
GuestOperations (All)
Interact (All)
Inventory (All)
Provisioning (All) | +| vSphere 标签 | Assign or Unassign vSphere Tag
Assign or Unassign vSphere Tag on Object | + +以下步骤创建具有所需æƒé™çš„角色，然åŽåœ¨ vSphere 控制å°ä¸­å°†è§’色分é…给新用户： + +1. 在 **vSphere** 控制å°ä¸­è½¬åˆ° **Administration** 页é¢ã€‚ + +2. 转到 **Roles** 选项å¡ã€‚ + +3. 创建一个新角色。为新角色命å，并选择上é¢æƒé™è¡¨ä¸­åˆ—出的æƒé™ã€‚ + + ![](/img/rancherroles1.png) + +4. 转到 **Users and Groups** 选项å¡ã€‚ + +5. 创建一个新用户。填写表å•ï¼Œç„¶åŽå•å‡» **OK**。请务必记下用户å和密ç ï¼Œå› ä¸ºåœ¨ Rancher 中é…置节点模æ¿æ—¶å°†éœ€è¦ç”¨åˆ°è¿™äº›ä¿¡æ¯ã€‚ + + ![](/img/rancheruser.png) + +6. 转到 **Global Permissions** 选项å¡ã€‚ + +7. 创建一个新的全局æƒé™ã€‚添加之å‰åˆ›å»ºçš„用户，并为其分é…之å‰åˆ›å»ºçš„角色。å•å‡» **OK**。 + + ![](/img/globalpermissionuser.png) + + ![](/img/globalpermissionrole.png) + +**结果**：你已拥有 Rancher 所需的æ“作 vSphere 资æºçš„凭è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md new file mode 100644 index 00000000000..806090bff7c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md @@ -0,0 +1,110 @@ +--- +title: 在 vSphere 中é…ç½® Kubernetes 集群 +--- + +在本节中，你将学习如何使用 Rancher 在 vSphere 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 vSphere 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ vSphere 中é…置节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 vSphere 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +有关é…ç½® vSphere 节点模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅 [vSphere 节点模æ¿é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)。 + +有关在 Rancher 中é…ç½® RKE Kubernetes 集群的详细信æ¯ï¼Œè¯·å‚阅[集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)。 + + +- [vSphere 中的准备工作](#vsphere-中的准备工作) +- [创建 vSphere 集群](#创建-vsphere-集群) + +## vSphere 中的准备工作 + +本节介ç»è®¾ç½® vSphere çš„è¦æ±‚，以便 Rancher å¯ä»¥é…置虚拟机和集群。 + +节点模æ¿å·²ä½¿ç”¨ vSphere Web Services API 6.5 版本进行记录和测试。 + +### 在 vSphere ä¸­åˆ›å»ºå‡­è¯ + +在继续创建集群之å‰ï¼Œç¡®ä¿ä½ çš„ vSphere 用户拥有足够的æƒé™ã€‚设置节点模æ¿æ—¶ï¼Œæ¨¡æ¿å°†éœ€è¦ä½¿ç”¨è¿™äº› vSphere 凭è¯ã€‚ + +有关如何在 vSphere 中创建具有所需æƒé™çš„用户的说明，请å‚阅此[æ“作指å—](create-credentials.md)。这些步骤会产生需è¦å‘ Rancher æ供的用户å和密ç ï¼Œä»Žè€Œå…许 Rancher 在 vSphere 中é…置资æºã€‚ + +### 网络æƒé™ + +必须确ä¿è¿è¡Œ Rancher Server 的主机能够建立以下网络连接： + +- 连接到 vCenter Server 上的 vSphere APIï¼ˆé€šå¸¸æ˜¯ç«¯å£ 443/TCP）。 +- 连接到用于实例化集群虚拟机的所有 ESXi 主机上的 Host APIï¼ˆç«¯å£ 443/TCP）（*仅在使用 ISO 创建模å¼æ—¶éœ€è¦*）。 +- 连接到创建的 VM 上的 22/TCP å’Œ 2376/TCP 端å£ã€‚ + +有关在基础设施æ供商上创建节点的端å£è¦æ±‚，请å‚阅[节点网络è¦æ±‚](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +### 用于 vSphere API 访问的有效 ESXi 许å¯è¯ + +å…费的 ESXi 许å¯è¯ä¸æ”¯æŒ API 访问。vSphere Server 必须具有有效或评估的 ESXi 许å¯è¯ã€‚ + +### 具有 DRS 的集群的 VM-VM å…³è”规则 + +如果你的集群å¯ç”¨äº† DRS，建议设置 [VM-VM å…³è”规则](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.resmgmt.doc/GUID-7297C302-378F-4AF2-9BD6-6EDB1E0A850A.html)。这些规则å…许分é…了 etcd å’Œ controlplane 角色的虚拟机在分é…ç»™ä¸åŒèŠ‚点池时，在å•ç‹¬çš„ ESXi 主机上è¿è¡Œã€‚è¿™ç§åšæ³•å¯ç¡®ä¿å•ä¸ªç‰©ç†æœºçš„æ•…éšœä¸ä¼šå½±å“这些平é¢çš„å¯ç”¨æ€§ã€‚ + +## 创建 vSphere 集群 + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **VMware vSphere**。 +1. 输入你的 vSphere 凭è¯ã€‚如需帮助，请å‚è§[节点模æ¿é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)中的**è´¦å·è®¿é—®**。 +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 vSphere 创建[节点模æ¿](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 vSphere 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**创建**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **vSphere**。 +1. 填写 vSphere 的节点模æ¿ã€‚有关填写表格的帮助，请å‚阅 vSphere 节点模æ¿[é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)。 +1. å•å‡»**创建**。 + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +使用 Rancher 在 vSphere 中创建 Kubernetes 集群。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **VMware vSphere**。 +1. 输入**集群å称**并使用你的 vSphere 云凭è¯ã€‚点击**继续**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 使用**集群选项**选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 如果你想ç¨åŽåŠ¨æ€é…ç½®æŒä¹…存储或其他基础设施，你需è¦ä¿®æ”¹é›†ç¾¤ YAML 文件æ¥å¯ç”¨ vSphere 云æ供商。有关更多信æ¯ï¼Œè¯·å‚阅[树内 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 +1. 将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)。 +1. 检查并确认你的选项。然åŽå•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 +- **é…置存储**：有关如何使用 Rancher 在 vSphere 中é…置存储的示例，请å‚阅[本节](../../../../../pages-for-subheaders/provisioning-storage-examples.md)。è¦åœ¨ vSphere 中动æ€é…置存储，你必须å¯ç”¨ vSphere 云æ供商。有关更多信æ¯ï¼Œè¯·å‚阅[树内 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md new file mode 100644 index 00000000000..aef1f3c08b4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md @@ -0,0 +1,66 @@ +--- +title: 将用户添加到集群 +--- + +如果你想为用户æ供对集群内 _所有_ 项目ã€èŠ‚点和资æºçš„访问æƒé™ï¼Œè¯·ä¸ºç”¨æˆ·åˆ†é…集群æˆå‘˜èµ„格。 + +:::tip + +如果你想为用户æ供对集群内 _特定_ 项目的访问æƒé™ï¼Œè¯·å‚è§[添加项目æˆå‘˜](../../../new-user-guides/add-users-to-projects.md)。 + +::: + +ä½ å¯ä»¥åœ¨ä¸¤ç§æƒ…况下添加集群æˆå‘˜ï¼š + +- å°†æˆå‘˜æ·»åŠ åˆ°æ–°é›†ç¾¤ + + ä½ å¯ä»¥åœ¨åˆ›å»ºé›†ç¾¤æ—¶å°†æˆå‘˜æ·»åŠ åˆ°é›†ç¾¤ï¼ˆæŽ¨è）。 + +- [å‘现有集群添加æˆå‘˜](#编辑集群æˆå‘˜) + + é…置集群åŽï¼Œä½ å§‹ç»ˆå¯ä»¥å‘集群添加æˆå‘˜ã€‚ + +## 编辑集群æˆå‘˜ + +集群管ç†å‘˜å¯ä»¥ç¼–辑集群的æˆå‘˜ï¼ŒæŽ§åˆ¶å“ªäº› Rancher 用户å¯ä»¥è®¿é—®é›†ç¾¤ä»¥åŠä»–们å¯ä»¥ä½¿ç”¨å“ªäº›åŠŸèƒ½ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ æˆå‘˜çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**æˆå‘˜è§’色**选项å¡ä¸­ï¼Œå•å‡»**添加æˆå‘˜**。 +1. æœç´¢è¦æ·»åŠ åˆ°é›†ç¾¤çš„用户或组。 + + 如果é…置了外部身份验è¯ï¼š + + - 在你键入时，Rancher 会从你的[外部身份验è¯](../../../../pages-for-subheaders/authentication-config.md)æºè¿”回用户。 + + :::note 使用 AD 但找ä¸åˆ°ä½ çš„用户？ + + ä½ çš„æœç´¢å±žæ€§é…ç½®å¯èƒ½å­˜åœ¨é—®é¢˜ã€‚请å‚阅[é…ç½® Active Directory 身份验è¯ï¼šæ­¥éª¤ 5](../../../new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md)。 + + ::: + + - ä½ å¯ä»¥åœ¨ä¸‹æ‹‰èœå•ä¸­æ·»åŠ ç»„，而ä¸æ˜¯å•ä¸ªç”¨æˆ·ã€‚下拉列表仅列出你（登录用户）所属的组。 + + :::note + + 如果你以本地用户身份登录，外部用户ä¸ä¼šæ˜¾ç¤ºåœ¨ä½ çš„æœç´¢ç»“果中。有关详细信æ¯ï¼Œè¯·å‚阅[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + + ::: + +1. 分é…用户或组的**集群**角色。 + + [什么是集群角色？](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::tip + + 对于自定义角色，你å¯ä»¥ä¿®æ”¹å¯åˆ†é…çš„å•ä¸ªè§’色列表。 + + - è¦å°†è§’色添加到列表中，请[添加自定义角色](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md)。 + - è¦ä»Žåˆ—表中删除角色，请[é”定/解é”角色](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md)。 + + ::: + +**结果**：已将选中的用户添加到集群中。 + +- è¦æ’¤é”€é›†ç¾¤æˆå‘˜èµ„格，请选择用户并å•å‡»**删除**。此æ“作会删除æˆå‘˜èµ„格，而ä¸ä¼šåˆ é™¤ç”¨æˆ·ã€‚ +- è¦ä¿®æ”¹é›†ç¾¤ä¸­çš„用户角色，请将其从集群中删除，然åŽä½¿ç”¨ä¿®æ”¹åŽçš„角色é‡æ–°æ·»åŠ ç”¨æˆ·ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md new file mode 100644 index 00000000000..364a4012eee --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md @@ -0,0 +1,49 @@ +--- +title: 授æƒé›†ç¾¤ç«¯ç‚¹çš„å·¥ä½œåŽŸç† +--- + +æœ¬æ–‡ä»‹ç» kubectl CLIã€kubeconfig 文件和授æƒé›†ç¾¤ç«¯ç‚¹å¦‚何ååŒå·¥ä½œï¼Œä½¿ä½ å¯ä»¥ç›´æŽ¥è®¿é—®ä¸‹æ¸¸ Kubernetes 集群，而无需通过 Rancher Server 进行身份验è¯ã€‚本文旨在为[设置 kubectl 以直接访问集群的说明](use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)æ供背景信æ¯å’Œä¸Šä¸‹æ–‡ã€‚ + +### Kubeconfig 文件说明 + +kubeconfig 文件是与 kubectl 命令行工具（或其他客户端）结åˆä½¿ç”¨æ—¶ç”¨äºŽé…ç½® Kubernetes 访问的文件。 + +kubeconfig 文件åŠå…¶å†…容特定于å„个集群。你å¯ä»¥ä»Ž Rancher çš„**集群**页é¢è¿›è¡Œä¸‹è½½ï¼š + +1. 点击左上角的 **☰**。 +1. 选择**集群管ç†**。 +1. 找到è¦ä¸‹è½½å…¶ kubeconfig 的集群，然åŽé€‰æ‹©è¡Œæœ«å°¾çš„ **â**。 +1. 从å­èœå•ä¸­é€‰æ‹©**下载 KubeConfig**。 + +在 Rancher 中å¯ä»¥è®¿é—®çš„æ¯ä¸ªé›†ç¾¤éƒ½éœ€è¦ä¸€ä¸ªå•ç‹¬çš„ kubeconfig 文件。 + +下载 kubeconfig 文件åŽï¼Œä½ å°†èƒ½å¤Ÿä½¿ç”¨ kubeconfig 文件åŠå…¶ Kubernetes [上下文](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#kubectl-context-and-configuration)访问下游集群。 + +如果管ç†å‘˜[关闭了 kubeconfig 令牌生æˆ](../../../../reference-guides/about-the-api/api-tokens.md#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)，则 kubeconfig 文件è¦æ±‚ [Rancher CLI](./authorized-cluster-endpoint.md) 存在于你的 PATH 中。 + +### RKE 集群的两ç§èº«ä»½éªŒè¯æ–¹æ³• + +如果集群ä¸æ˜¯ [RKE 集群](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)，kubeconfig 文件åªå…许你以一ç§æ–¹å¼è®¿é—®â€‹â€‹é›†ç¾¤ï¼Œå³é€šè¿‡ Rancher Server 进行身份验è¯ï¼Œç„¶åŽ Rancher å…许你在集群上è¿è¡Œ kubectl 命令。 + +对于 RKE 集群，kubeconfig 文件å…许你通过两ç§æ–¹å¼è¿›è¡Œèº«ä»½éªŒè¯ï¼š + +- **通过 Rancher Server 身份验è¯ä»£ç†**：Rancher 的身份验è¯ä»£ç†ä¼šéªŒè¯ä½ çš„身份，然åŽå°†ä½ è¿žæŽ¥åˆ°è¦è®¿é—®çš„下游集群。 +- **直接使用下游集群的 API Server**：RKE 集群默认å¯ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹ã€‚此端点å…许你使用 kubectl CLI å’Œ kubeconfig 文件访问下游 Kubernetes 集群，且 RKE 集群默认å¯ç”¨è¯¥ç«¯ç‚¹ã€‚在这ç§æƒ…况下，下游集群的 Kubernetes API server 通过调用 Rancher 设置的 webhook（`kube-api-auth` å¾®æœåŠ¡ï¼‰å¯¹ä½ è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +第二ç§æ–¹æ³•ï¼ˆå³ç›´æŽ¥è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API server）éžå¸¸é‡è¦ï¼Œå› ä¸ºå¦‚果你无法连接到 Rancher，这ç§æ–¹æ³•å¯ä»¥è®©ä½ è®¿é—®ä¸‹æ¸¸é›†ç¾¤ã€‚ + +è¦ä½¿ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹ï¼Œä½ éœ€è¦é…ç½® kubectl，从而使用 Rancher 在创建 RKE 集群时生æˆçš„ kubeconfig 文件中的é¢å¤– kubectl 上下文。该文件å¯ä»¥ä»Ž Rancher UI çš„**集群**视图中下载，é…ç½® kubectl 的说明在[此页é¢](use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)。 + +[架构介ç»](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md)也详细解释了这些与下游 Kubernetes 集群通信的方法，并介ç»äº† Rancher 的工作原ç†ä»¥åŠ Rancher 如何与下游集群通信的详细信æ¯ã€‚ + +### 关于 kube-api-auth èº«ä»½éªŒè¯ Webhook + +`kube-api-auth` å¾®æœåŠ¡æ˜¯ä¸º[授æƒé›†ç¾¤ç«¯ç‚¹](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)æ供用户认è¯åŠŸèƒ½è€Œéƒ¨ç½²çš„。当你使用 `kubectl` 访问下游集群时，集群的 Kubernetes API server 会使用 `kube-api-auth` æœåŠ¡ä½œä¸º webhook 对你进行身份验è¯ã€‚ + +在集群é…置期间会部署 `/etc/kubernetes/kube-api-authn-webhook.yaml` 文件，并使用 `--authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml` é…ç½® `kube-apiserver`。这会将 `kube-apiserver` é…置为通过查询 `http://127.0.0.1:6440/v1/authenticate` æ¥ç¡®å®šæŒæœ‰è€… token 的身份验è¯ã€‚ + +`kube-api-auth` 的调度规则如下： + +| 组件 | nodeAffinity nodeSelectorTerms | nodeSelector | 容å¿åº¦ | +| -------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| kube-api-auth | `beta.kubernetes.io/os:NotIn:windows`
`node-role.kubernetes.io/controlplane:In:"true"` | none | `operator:Exists` | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md new file mode 100644 index 00000000000..ac6a4a338fb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -0,0 +1,106 @@ +--- +title: "使用 Kubectl å’Œ kubeconfig 访问集群" +description: "了解如何通过 kubectl Shell 使用 kubectl，或通过 kubectl CLI å’Œ kubeconfig 文件，æ¥è®¿é—®å’Œç®¡ç† Kubernetes 集群。kubeconfig 文件用于é…置对 Kubernetes 的访问。当你使用 Rancher 创建集群时，Rancher 会自动为你的集群创建 kubeconfig。" +--- + +本文介ç»å¦‚何通过 Rancher UI 或工作站æ¥ä½¿ç”¨ kubectl æ“作下游 Kubernetes 集群。 + +有关使用 kubectl 的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档：kubectl 概述](https://kubernetes.io/docs/reference/kubectl/overview/)。 + +### 在 Rancher UI 中使用 kubectl shell 访问集群 + +ä½ å¯ä»¥é€šè¿‡ç™»å½• Rancher 并在 UI 中打开 kubectl shell æ¥è®¿é—®å’Œç®¡ç†ä½ çš„集群。你无需进一步é…置。 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦ä½¿ç”¨ kubectl 访问的集群，然åŽå•å‡» **Explore**。 +1. 在顶部导航èœå•ä¸­ï¼Œå•å‡» **Kubectl Shell** 按钮。使用打开的窗å£ä¸Žä½ çš„ Kubernetes 集群进行交互。 + +### 在工作站使用 kubectl 访问集群 + +本节介ç»å¦‚何下载集群的 kubeconfig 文件ã€ä»Žå·¥ä½œç«™å¯åŠ¨ kubectl 以åŠè®¿é—®ä¸‹æ¸¸é›†ç¾¤ã€‚ + +è¿™ç§è®¿é—®é›†ç¾¤çš„替代方法å…许你在ä¸ä½¿ç”¨ Rancher UI 的情况下通过 Rancher 进行身份验è¯å¹¶ç®¡ç†é›†ç¾¤ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +以下说明å‡è®¾ä½ å·²ç»åˆ›å»ºäº†ä¸€ä¸ª Kubernetes 集群，并且已将 kubectl 安装在工作站上。有关安装 kubectl 的帮助，请å‚阅官方 [Kubernetes 文档](https://kubernetes.io/docs/tasks/tools/install-kubectl/)。 + +::: + +1. 点击左上角的 **☰**。 +1. 选择**集群管ç†**。 +1. 找到è¦ä¸‹è½½å…¶ kubeconfig 的集群，然åŽé€‰æ‹©è¡Œæœ«å°¾çš„ **â**。 +1. 从å­èœå•ä¸­é€‰æ‹©**下载 KubeConfig**。 +1. å°† YAML 文件ä¿å­˜åœ¨æœ¬åœ°è®¡ç®—机上。将文件移动到 `~/.kube/config`。注æ„：kubectl 用于 kubeconfig 文件的默认ä½ç½®æ˜¯ `~/.kube/config`。但是你也å¯ä»¥è¿è¡Œç±»ä¼¼ä»¥ä¸‹çš„命令，使用 `--kubeconfig` 标志指定任何其他目录： +``` +kubectl --kubeconfig /custom/path/kube.config get pods +``` +1. 从工作站å¯åŠ¨ kubectl。使用它与 Kubernetes 集群进行交互。 + + +### 使用 kubectl 创建的资æºçš„注æ„事项 + +Rancher 会å‘现并显示由 `kubectl` 创建的资æºã€‚但是在å‘现资æºçš„时候，这些资æºå¯èƒ½æ²¡æœ‰åŒ…括所有必须的注释。如果资æºå·²ç»ä½¿ç”¨ Rancher UI/API 进行æ“作（例如，扩展工作负载），但是由于缺少注释，资æºçš„é‡æ–°åˆ›å»ºå¯èƒ½ä¼šè§¦å‘。åªæœ‰åœ¨é¦–次对å‘现的资æºè¿›è¡Œæ“作时，这ç§æƒ…况æ‰ä¼šå‘生。 + +## ç›´æŽ¥ä½¿ç”¨ä¸‹æ¸¸é›†ç¾¤è¿›è¡Œèº«ä»½éªŒè¯ + +本节旨在帮助你设置访问 [RKE 集群的替代方法](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +此方法仅适用于å¯ç”¨äº†[授æƒé›†ç¾¤ç«¯ç‚¹](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)çš„ RKEã€RKE2 å’Œ K3s集群。在 Rancher 创建集群时，Rancher 会生æˆä¸€ä¸ª kubeconfig 文件，其中包å«ç”¨äºŽè®¿é—®é›†ç¾¤çš„é¢å¤– kubectl 上下文。该上下文å…许你使用 kubectl 通过下游集群进行身份验è¯ï¼Œè€Œæ— éœ€é€šè¿‡ Rancher 进行身份验è¯ã€‚有关授æƒé›†ç¾¤ç«¯ç‚¹å¦‚何工作的详细说明，请å‚阅[此页é¢](authorized-cluster-endpoint.md)。 + +在 RKE2 å’Œ K3s 集群上，你需è¦[手动å¯ç”¨](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#对-rke2-å’Œ-k3s-集群的授æƒé›†ç¾¤ç«¯ç‚¹æ”¯æŒ)授æƒé›†ç¾¤ç«¯ç‚¹ã€‚ + +我们的最佳实践是使用此方法æ¥è®¿é—® RKEã€RKE2 å’Œ K3s集群。这样，万一你无法连接到 Rancher，你ä»ç„¶å¯ä»¥è®¿é—®è¯¥é›†ç¾¤ã€‚ + + +:::note 先决æ¡ä»¶ï¼š + +以下步骤å‡è®¾ä½ å·²ç»åˆ›å»ºäº†ä¸€ä¸ª Kubernetes 集群，并按照步骤[从工作站使用 kubectl 连接到集群](#在工作站使用-kubectl-访问集群)。 + +::: + +è¦åœ¨ä¸‹è½½çš„ kubeconfig 文件中查找上下文的å称，请è¿è¡Œï¼š + +``` +kubectl config get-contexts --kubeconfig /custom/path/kube.config +CURRENT NAME CLUSTER AUTHINFO NAMESPACE +* my-cluster my-cluster user-46tmn + my-cluster-controlplane-1 my-cluster-controlplane-1 user-46tmn +``` + +在此示例中，当你将 `kubectl` 与第一个上下文 `my-cluster` 一起使用时，你将通过 Rancher Server 进行身份验è¯ã€‚ + +使用第二个上下文 `my-cluster-controlplane-1`，你将使用授æƒé›†ç¾¤ç«¯ç‚¹è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç›´æŽ¥ä¸Žä¸‹æ¸¸ RKE 集群通信。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +现在你已ç»æœ‰äº†ç›´æŽ¥ä½¿ç”¨é›†ç¾¤è¿›è¡Œèº«ä»½éªŒè¯æ‰€éœ€çš„上下文å称，你å¯ä»¥åœ¨è¿è¡Œ kubectl 命令时将上下文å称作为选项传入。根æ®ä½ çš„集群是å¦å®šä¹‰äº† FQDN，这些命令会有所ä¸åŒã€‚以下几节æ供了示例。 + +当 `kubectl` 正常工作时，它确认你å¯ä»¥ç»•è¿‡ Rancher 的身份验è¯ä»£ç†è®¿é—®é›†ç¾¤ã€‚ + +### 直接连接到定义了 FQDN 的集群 + +如果集群定义了 FQDN，将会创建一个引用 FQDN 的上下文。上下文将命å为 `-fqdn`。当你想在没有 Rancher 的情况下使用 `kubectl` 访问这个集群时，你需è¦ä½¿ç”¨è¿™ä¸ªä¸Šä¸‹æ–‡ã€‚ + +å‡è®¾ kubeconfig 文件ä½äºŽ `~/.kube/config`： + +``` +kubectl --context -fqdn get nodes +``` +直接引用 kubeconfig 文件的ä½ç½®ï¼š +``` +kubectl --kubeconfig /custom/path/kube.config --context -fqdn get pods +``` + +### 直接连接到未定义 FQDN 的集群 + +如果集群没有定义 FQDN，则会创建é¢å¤–的上下文æ¥å¼•ç”¨ controlplane 中æ¯ä¸ªèŠ‚点的 IP 地å€ã€‚æ¯ä¸ªä¸Šä¸‹æ–‡å°†è¢«å‘½å为 `-`。当你想在没有 Rancher 的情况下使用 `kubectl` 访问这个集群时，你需è¦ä½¿ç”¨è¿™ä¸ªä¸Šä¸‹æ–‡ã€‚ + +å‡è®¾ kubeconfig 文件ä½äºŽ `~/.kube/config`： +``` +kubectl --context - get nodes +``` +直接引用 kubeconfig 文件的ä½ç½®ï¼š +``` +kubectl --kubeconfig /custom/path/kube.config --context - get pods +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md new file mode 100644 index 00000000000..d881d6c968e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md @@ -0,0 +1,39 @@ +--- +title: 添加 Pod 安全策略 +--- + +:::note 先决æ¡ä»¶ï¼š + +以下选项仅适用于[使用 RKE å¯åŠ¨çš„集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +::: + +当你的集群上è¿è¡Œäº†å…·æœ‰å®‰å…¨æ•æ„Ÿé…置的 pod æ—¶ï¼Œè¯·ä¸ºå…¶åˆ†é… [pod 安全策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md)，这是一组用于监控 pod 中的状æ€å’Œè®¾ç½®çš„规则。如果 pod ä¸ç¬¦åˆä½ çš„策略中指定的规则，则该策略会阻止它è¿è¡Œã€‚ + +ä½ å¯ä»¥åœ¨é…ç½®é›†ç¾¤æ—¶åˆ†é… pod 安全策略。如果你以åŽéœ€è¦æ”¾æ¾æˆ–é™åˆ¶ pod 的安全性，你å¯ä»¥åœ¨ç¼–辑集群时更新策略： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦åº”用 pod 安全策略的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在 **Pod 安全策略支æŒ**中，选择**å¯ç”¨**。 + + :::note + + 此选项仅适用于[ç”± RKE é…置的集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + + ::: + +4. 从**默认 Pod 安全策略**下拉列表中，选择è¦åº”用于集群的策略。 + + Rancher 支æŒäº†`å—é™`å’Œ`ä¸å—é™`çš„[ç­–ç•¥](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#默认-psp)，你也å¯ä»¥[创建自定义策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#创建-psp)。 + +5. å•å‡»**ä¿å­˜**。 + +**结果**：pod 安全策略应用于集群和集群内的任何项目。 + +:::note + +åœ¨åˆ†é… pod 安全策略之å‰å·²ç»è¿è¡Œçš„工作负载是ä¸å—é™åˆ¶çš„。å³ä½¿å®ƒä»¬ä¸ç¬¦åˆä½ çš„ pod 安全策略，在分é…策略之å‰è¿è¡Œçš„工作负载也会继续è¿è¡Œã€‚ + +è¦æ£€æŸ¥æ­£åœ¨è¿è¡Œçš„工作负载是å¦é€šè¿‡äº†ä½ çš„ pod 安全策略，请克隆或å‡çº§å®ƒã€‚ + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md new file mode 100644 index 00000000000..180a248025b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md @@ -0,0 +1,25 @@ +--- +title: åˆ†é… Pod 安全策略 +--- + +_Pod 安全策略_（PSP）是控制 Pod 安全的规范（如是å¦å¯ä»¥ä½¿ç”¨ root æƒé™ç­‰ï¼‰çš„对象。 + +## 添加默认 Pod 安全策略 + +使用 RKE 创建新集群时，你å¯ä»¥å°†å…¶é…置为立å³åº”用 PSP。创建集群时，使用**集群选项**æ¥å¯ç”¨ PSP。分é…给集群的 PSP 将是集群内项目的默认 PSP。 + +:::note 先决æ¡ä»¶ï¼š + +在 Rancher 中创建 Pod 安全策略。在将默认 PSP 分é…给新集群之å‰ï¼Œä½ å¿…须有一个å¯åˆ†é…çš„ PSP。有关说明，请å‚阅[创建 Pod 安全策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +::: + +:::note + +å‡ºäºŽå®‰å…¨è€ƒè™‘ï¼Œæˆ‘ä»¬å»ºè®®ä½ åœ¨åˆ›å»ºé›†ç¾¤æ—¶åˆ†é… PSP。 + +::: + +è¦å¯ç”¨é»˜è®¤ Pod 安全策略，请将 **Pod 安全策略支æŒ**选项设置为**å¯ç”¨**，然åŽä»Ž**默认 Pod 安全策略**下拉框中进行选择。 + +集群完æˆé…ç½®åŽï¼Œä½ é€‰æ‹©çš„ PSP 将应用于集群内的所有项目。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md new file mode 100644 index 00000000000..4ed2eb4d44d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -0,0 +1,376 @@ +--- +title: 从节点中移除 Kubernetes 组件 +description: 了解从 Rancher å¯åŠ¨çš„ Kubernetes 集群中删除节点时的集群清ç†è¿‡ç¨‹ã€‚了解删除了的内容，以åŠå¦‚何手动进行æ“作 +--- + +本文介ç»å¦‚何断开 Rancher 创建的 Kubernetes 集群中的一个节点，并删除该节点中的所有 Kubernetes 组件。此过程å…许你将节点用于其他用途。 + +在通过 Rancher 在基础设施æ供商的新节点上安装 Kubernetes 时，会创建资æºï¼ˆå®¹å™¨/虚拟网络接å£ï¼‰å’Œé…置项（è¯ä¹¦/é…置文件）。 + +从 Rancher å¯åŠ¨çš„ Kubernetes 集群中移除节点时（å‰æ是它们处于 `Active` 状æ€ï¼‰ï¼Œè¿™äº›èµ„æºä¼šè¢«è‡ªåŠ¨æ¸…ç†ï¼Œå”¯ä¸€éœ€è¦çš„æ“作是é‡å¯èŠ‚点。如果节点å˜å¾—ä¸å¯è®¿é—®å¹¶ä¸”无法自动清ç†è¿›ç¨‹æ—¶ï¼Œè¯·å…ˆæ‰§è¡Œæˆ‘们æ供的步骤，然åŽè¯¥èŠ‚点æ‰èƒ½é‡æ–°æ·»åŠ åˆ°é›†ç¾¤ã€‚ + +## 移除了什么？ + +清ç†ä½¿ç”¨ Rancher é…置的节点时，会根æ®è¦åˆ é™¤çš„集群节点类型移除以下组件： + +| 移除的组件 | [由基础设施æ供商托管的节点][1] | [自定义节点][2] | [托管集群][3] | [注册节点][4] | +|--------------------------------------------------------------------------------|----------------------------------------------|-------------------|---------------------|-----------------------| +| Rancher deployment 命å空间（默认：`cattle-system`） | ✓ | ✓ | ✓ | ✓ | +| ç”± Rancher 标记的 `serviceAccount`ã€`clusterRoles` å’Œ `clusterRoleBindings` | ✓ | ✓ | ✓ | ✓ | +| 标签ã€æ³¨é‡Šå’Œç»ˆç»“器 | ✓ | ✓ | ✓ | ✓ | +| Rancher Deployment | ✓ | ✓ | ✓ | | +| 主机ã€é›†ç¾¤ã€é¡¹ç›®å’Œç”¨æˆ·çš„自定义资æºå®šä¹‰ (CRD) | ✓ | ✓ | ✓ | | +| 在 `management.cattle.io` API Group ä¸‹åˆ›å»ºçš„æ‰€æœ‰èµ„æº | ✓ | ✓ | ✓ | | +| Rancher v2.x 创建的所有 CRD | ✓ | ✓ | ✓ | | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +## 通过 Rancher UI 删除集群中的节点 + +如果节点处于 `Active` 状æ€ï¼Œåˆ é™¤é›†ç¾¤ä¸­çš„节点会触å‘清ç†èŠ‚点的进程。完æˆè‡ªåŠ¨æ¸…ç†åŽè¯·é‡å¯èŠ‚点，以确ä¿æ‰€æœ‰éžæŒä¹…性数æ®å·²æ­£ç¡®åˆ é™¤ã€‚ + +**é‡å¯èŠ‚点**： + +``` +# 使用é‡å¯ +$ sudo reboot + +# 使用关机 +$ sudo shutdown -r now +``` + +## 手动移除集群中的 Rancher 组件 + +当节点ä¸å¯è¾¾å¹¶å·²ä»Žé›†ç¾¤ä¸­ç§»é™¤æ—¶ï¼Œç”±äºŽè¯¥èŠ‚点ä¸å¯è¾¾ï¼Œåˆ™æ— æ³•è§¦å‘自动清ç†è¿‡ç¨‹ã€‚请按照以下步骤手动删除 Rancher 组件。 + +:::danger + +下é¢åˆ—出的命令将删除节点中的数æ®ã€‚在执行命令之å‰ï¼Œç”±äºŽæ•°æ®å°†ä¼šä¸¢å¤±ï¼Œè¯·ç¡®ä¿ä½ å·²ç»å¤‡ä»½äº†è¦ä¿ç•™çš„文件。 + +::: + +### 移除注册集群中的 Rancher 组件 + +移除注册集群中的 Rancher 的过程略有ä¸åŒã€‚ä½ å¯ä»¥é€‰æ‹©åœ¨ Rancher UI 中简å•åœ°åˆ é™¤é›†ç¾¤ï¼Œä¹Ÿå¯ä»¥è¿è¡Œè„šæœ¬ä»ŽèŠ‚点中删除 Rancher 组件。两个选项的删除效果是一样的。 + +将注册集群分离 Rancher åŽï¼Œé›†ç¾¤çš„工作负载将ä¸å—å½±å“，你å¯ä»¥ä½¿ç”¨ä¸Žé›†ç¾¤æ³¨å†Œåˆ° Rancher 之å‰ç›¸åŒçš„方法æ¥è®¿é—®é›†ç¾¤ã€‚ + + + + +:::danger + +此过程将删除你的集群数æ®ã€‚在执行命令之å‰ï¼Œç”±äºŽæ•°æ®å°†ä¼šä¸¢å¤±ï¼Œè¯·ç¡®ä¿ä½ å·²ç»å¤‡ä»½äº†è¦ä¿ç•™çš„文件。 + +::: + +使用 Rancher UI（或 API）删除注册集群åŽï¼Œä¼šå‘生以下事件： + +1. Rancher 创建一个 `serviceAccount`，用于删除集群中的 Rancher 组件。此账å·åˆ†é…了 [clusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole) å’Œ [clusterRoleBinding](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding) æƒé™ï¼Œè¿™äº›æƒé™æ˜¯åˆ é™¤ Rancher 组件所必需的。 + +1. Rancher 会使用 `serviceAccount` 调度并è¿è¡Œä¸€ä¸ª [job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/)，以将 Rancher 组件从集群中清除。该 job 还将 `serviceAccount` åŠå…¶è§’色作为ä¾èµ–引用，因此该 job 在完æˆä¹‹å‰ä¼šåˆ é™¤å®ƒä»¬ã€‚ + +1. 已从集群中移除 Rancher。然而，集群ä»ç„¶å­˜åœ¨ï¼Œè¿è¡Œç€åŽŸç”Ÿç‰ˆæœ¬çš„ Kubernetes。 + +**结果**：已删除[移除了什么](#移除了什么)章节中为注册集群列出的所有组件。 + + + + +ä½ å¯ä»¥è¿è¡Œè„šæœ¬ï¼Œè€Œä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥æ¸…ç†å·²æ³¨å†Œçš„集群节点。 + +:::note 先决æ¡ä»¶ï¼š + +安装 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)。 + +::: + +1. 打开æµè§ˆå™¨å¹¶å¯¼èˆªåˆ° [GitHub](https://github.com/rancher/rancher/blob/master/cleanup/user-cluster.sh)，然åŽä¸‹è½½ `user-cluster.sh`。 + +1. 在 `user-cluster.sh` 所在的目录下è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä½¿è„šæœ¬å¯æ‰§è¡Œï¼š + + ``` + chmod +x user-cluster.sh + ``` + +1. **ä»…é™ç¦»çº¿çŽ¯å¢ƒ**：打开 `user-cluster.sh` 并将 `yaml_url` 替æ¢ä¸º `user-cluster.yml` 中的 URL。 + + 如果你没有使用离线环境，请跳过此步骤。 + +1. 在åŒä¸€ç›®å½•ä¸­ï¼Œè¿è¡Œè„šæœ¬å¹¶æä¾› `rancher/rancher-agent` é•œåƒç‰ˆæœ¬ï¼Œè¯¥ç‰ˆæœ¬åº”与用于管ç†é›†ç¾¤çš„ Rancher 版本相åŒï¼ˆ``）： + + :::tip + + ä½ å¯ä»¥æ·»åŠ  `-dry-run` 标志æ¥é¢„览脚本的结果，但ä¸è¿›è¡Œæ›´æ”¹ã€‚ + ``` + ./user-cluster.sh rancher/rancher-agent: + ``` + + ::: + +**结果**：脚本已è¿è¡Œã€‚已删除[移除了什么](#移除了什么)章节中为注册集群列出的所有组件。 + + + + +## 清ç†èŠ‚点 + + + + +在è¿è¡Œä»¥ä¸‹å‘½ä»¤ä¹‹å‰ï¼Œé¦–先通过 Rancher UI 删除节点。 + +è¦åˆ é™¤èŠ‚点： + +1. 点击 **☰** 并选择**集群管ç†**。 +1. 在集群表中，å•å‡»èŠ‚点所属集群的å称。 +1. 在第一个选项å¡ä¸­ï¼Œå•å‡»èŠ‚点状æ€æ—边的å¤é€‰æ¡†ã€‚ +1. å•å‡»**删除**。 + +è¦åˆ é™¤æ•´ä¸ªé›†ç¾¤è€Œä¸æ˜¯å•ä¸ªèŠ‚点，或者ä¸ä½¿ç”¨ Rancher UI é‡æ–°ç§»åŠ¨èŠ‚点，请按照以下步骤æ“作： + +1. 从节点中[删除](#docker-容器镜åƒå’Œå·) Docker 容器并[å¸è½½](#挂载)任何å·ã€‚ +1. é‡å¯èŠ‚点。 +1. [删除](#目录和文件)所有剩余的文件。 +1. 确认在é‡æ–°å¯åŠ¨åŽå·²æ­£ç¡®æ¸…ç†ç½‘络接å£å’Œ IP 表。å¦åˆ™è¯·å†é‡å¯ä¸€æ¬¡ã€‚ + +### Windows 节点 + +è¦æ¸…ç† Windows 节点，请è¿è¡Œ `c:\\etc\\rancher` 中的脚本。此脚本删除 Kubernetes 生æˆçš„资æºå’Œæ‰§è¡ŒäºŒè¿›åˆ¶æ–‡ä»¶ï¼Œè¿˜ä¼šåˆ é™¤é˜²ç«å¢™è§„则和网络设置： + +``` +pushd c:\etc\rancher +.\cleanup.ps1 +popd +``` + +è¿è¡Œæ­¤è„šæœ¬åŽï¼ŒèŠ‚点将é‡ç½®å¹¶å¯ä»¥é‡æ–°æ·»åŠ åˆ° Kubernetes 集群。 + +### Docker 容器ã€é•œåƒå’Œå· + +:::caution + +æ¸…ç† Docker 容器时è¦å°å¿ƒã€‚以下命令将删除节点上的*所有* Docker 容器ã€é•œåƒå’Œå·ï¼ŒåŒ…括与 Rancher 无关的容器： + +::: + +``` +docker rm -f $(docker ps -qa) +docker rmi -f $(docker images -q) +docker volume rm $(docker volume ls -q) +``` + +### 挂载 + +Kubernetes 组件和 secret 会留下以下挂载： + +* `/var/lib/kubelet` +* `/var/lib/rancher` +* `/var/lib/kubelet/pods/` 中的其他挂载 + +è¦å¸è½½æ‰€æœ‰æŒ‚载，请è¿è¡Œï¼š + +``` +for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done +``` + + + + +你需è¦ä»Ž Rancher æ供的 RKE2 节点中删除以下组件： + +* rancher-system-agent，用于连接 Rancher å¹¶å®‰è£…å’Œç®¡ç† RKE2。 +* RKE2 本身。 + +### 删除 rancher-system-agent + +è¦åˆ é™¤ rancher-system-agent，请è¿è¡Œ [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) 脚本： + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### 删除 RKE2 + +è¦åˆ é™¤ RKE2，请è¿è¡ŒèŠ‚点上已存在的 `rke2-uninstall` 脚本： + +``` +sudo rke2-uninstall.sh +``` + + + + +你需è¦ä»Ž Rancher æ供的 K3s 节点中删除以下组件： + +* rancher-system-agent：连接 Rancher å¹¶å®‰è£…å’Œç®¡ç† K3s。 +* K3s 本身。 + +### 删除 rancher-system-agent + +è¦åˆ é™¤ rancher-system-agent，请è¿è¡Œ [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) 脚本： + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### 删除 K3s + +è¦åˆ é™¤ K3s，请è¿è¡ŒèŠ‚点上已存在的 `k3s-uninstall` 脚本： + +``` +sudo k3s-uninstall.sh +``` + + + + +### 目录和文件 + +以下目录在将节点添加到集群时使用，应将该目录删除。你å¯ä»¥ä½¿ç”¨ `rm -rf /directory_name` æ¥åˆ é™¤ç›®å½•ã€‚ + +:::note + +节点分é…的角色决定了出现在节点上的目录。 + +::: + +| 目录 | +|------------------------------| +| `/etc/ceph` | +| `/etc/cni` | +| `/etc/kubernetes` | +| `/etc/rancher` | +| `/opt/cni` | +| `/opt/rke` | +| `/run/secrets/kubernetes.io` | +| `/run/calico` | +| `/run/flannel` | +| `/var/lib/calico` | +| `/var/lib/etcd` | +| `/var/lib/cni` | +| `/var/lib/kubelet` | +| `/var/lib/rancher` | +| `/var/log/containers` | +| `/var/log/kube-audit` | +| `/var/log/pods` | +| `/var/run/calico` | + +**清ç†ç›®å½•**： + +``` +rm -rf /etc/ceph \ + /etc/cni \ + /etc/kubernetes \ + /etc/rancher \ + /opt/cni \ + /opt/rke \ + /run/secrets/kubernetes.io \ + /run/calico \ + /run/flannel \ + /var/lib/calico \ + /var/lib/etcd \ + /var/lib/cni \ + /var/lib/kubelet \ + /var/lib/rancher\ + /var/log/containers \ + /var/log/kube-audit \ + /var/log/pods \ + /var/run/calico +``` + +### 网络接å£å’Œ iptables + +其余两个更改/é…置的组件是（虚拟）网络接å£å’Œ iptables 规则。两者都对节点ä¸æŒä¹…，这æ„味ç€å®ƒä»¬å°†åœ¨èŠ‚点é‡æ–°å¯åŠ¨åŽè¢«æ¸…除。è¦åˆ é™¤è¿™äº›ç»„件，建议é‡æ–°å¯åŠ¨å®ƒä»¬ã€‚ + +**é‡å¯èŠ‚点**： + +``` +# 使用é‡å¯ +$ sudo reboot + +# 使用关机 +$ sudo shutdown -r now +``` + +如果你想了解更多关于（虚拟）网络接å£æˆ– iptables 规则的信æ¯ï¼Œè¯·å‚阅下é¢çš„具体内容。 + +### ç½‘ç»œæŽ¥å£ + +:::note + +节点所在的集群所é…置的网络æ供商决定了节点上将出现的接å£ã€‚ + +::: + +| æŽ¥å£ | +|--------------------------------------------| +| `flannel.1` | +| `cni0` | +| `tunl0` | +| `caliXXXXXXXXXXX` (random interface names) | +| `vethXXXXXXXX` (random interface names) | + +**列出所有接å£**： + +``` +# 使用 ip +ip address show + +# 使用 ifconfig +ifconfig -a +``` + +**删除接å£**： + +``` +ip link delete interface_name +``` + +### Iptables + +:::note + +节点所在的集群所é…置的网络æ供商决定了节点上将出现的链。 + +::: + +iptables 规则用于将æµé‡ä»Žå®¹å™¨è·¯ç”±åˆ°å®¹å™¨ã€‚创建的规则ä¸æ˜¯æŒä¹…性的，因此é‡æ–°å¯åŠ¨èŠ‚点会将 iptables æ¢å¤åˆ°åŽŸå§‹çŠ¶æ€ã€‚ + +| 链 | +|--------------------------------------------------| +| `cali-failsafe-in` | +| `cali-failsafe-out` | +| `cali-fip-dnat` | +| `cali-fip-snat` | +| `cali-from-hep-forward` | +| `cali-from-host-endpoint` | +| `cali-from-wl-dispatch` | +| `cali-fw-caliXXXXXXXXXXX` (random chain names) | +| `cali-nat-outgoing` | +| `cali-pri-kns.NAMESPACE` (chain per namespace) | +| `cali-pro-kns.NAMESPACE` (chain per namespace) | +| `cali-to-hep-forward` | +| `cali-to-host-endpoint` | +| `cali-to-wl-dispatch` | +| `cali-tw-caliXXXXXXXXXXX` (random chain names) | +| `cali-wl-to-host` | +| `KUBE-EXTERNAL-SERVICES` | +| `KUBE-FIREWALL` | +| `KUBE-MARK-DROP` | +| `KUBE-MARK-MASQ` | +| `KUBE-NODEPORTS` | +| `KUBE-SEP-XXXXXXXXXXXXXXXX` (random chain names) | +| `KUBE-SERVICES` | +| `KUBE-SVC-XXXXXXXXXXXXXXXX` (random chain names) | + +**列出所有 iptables 规则**： + +``` +iptables -L -t nat +iptables -L -t mangle +iptables -L +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md new file mode 100644 index 00000000000..9a4c878d631 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md @@ -0,0 +1,37 @@ +--- +title: GlusterFS å· +--- + +:::note + +本文仅适用于 [RKE 集群](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +::: + +在将数æ®å­˜å‚¨åœ¨ GlusterFS å·ä¸Šçš„集群中，你å¯èƒ½ä¼šé‡åˆ°é‡å¯ `kubelet` åŽ pod 无法挂载å·çš„问题。`kubelet` 的日志将显示 `transport endpoint is not connected`。为了é¿å…è¿™ç§æƒ…况，你å¯ä»¥åœ¨é›†ç¾¤ä¸­å°† `systemd-run` 二进制文件挂载到 `kubelet` 容器中。在更改集群é…置之å‰æœ‰ä¸¤ä¸ªè¦æ±‚： + +- 该节点需è¦å®‰è£…了 `systemd-run` 二进制文件（å¯ä»¥é€šè¿‡åœ¨æ¯ä¸ªé›†ç¾¤èŠ‚点上è¿è¡Œ `which systemd-run` 命令æ¥æ£€æŸ¥ï¼‰ã€‚ +- `systemd-run` 二进制文件需è¦ä¸Ž hyperkube é•œåƒæ‰€åŸºäºŽçš„ Debian OS 兼容（å¯ä»¥é€šè¿‡åœ¨æ¯ä¸ªé›†ç¾¤èŠ‚点上è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ£€æŸ¥ï¼Œè¯·å°†é•œåƒæ ‡ç­¾æ›¿æ¢ä¸ºä½ æƒ³è¦çš„ Kubernetes 版本）： + +``` +docker run -v /usr/bin/systemd-run:/usr/bin/systemd-run --entrypoint /usr/bin/systemd-run rancher/hyperkube:v1.16.2-rancher1 --version +``` + +:::caution + +在更新 Kubernetes YAML 以挂载 `systemd-run` 二进制文件之å‰ï¼Œè¯·ç¡®ä¿åœ¨é›†ç¾¤èŠ‚点上安装了 `systemd` 包。如果在绑定挂载创建到你的 Kubernetes YAML _之å‰_ 未安装此包，Docker 将自动在æ¯ä¸ªèŠ‚点上创建目录和文件，并且ä¸å…许包安装æˆåŠŸã€‚ + +::: + +``` +services: + kubelet: + extra_binds: + - "/usr/bin/systemd-run:/usr/bin/systemd-run" +``` + +集群完æˆé…ç½®åŽï¼Œä½ å¯ä»¥é€šè¿‡æŸ¥æ‰¾ä»¥ä¸‹æ—¥å¿—æ¥æ£€æŸ¥ `kubelet` 容器日志记录，从而查看该功能是å¦å·²æ¿€æ´»ï¼š + +``` +Detected OS with systemd +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md new file mode 100644 index 00000000000..6e744c1295f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md @@ -0,0 +1,72 @@ +--- +title: æŒä¹…å­˜å‚¨çš„å·¥ä½œåŽŸç† +--- + +æŒä¹…å· (PV) 是 Kubernetes 集群中的一å—存储，而æŒä¹…å·å£°æ˜Ž (PVC) 是对存储的请求。 + +在 Kubernetes 中使用æŒä¹…存储有两ç§æ–¹æ³•ï¼š + +- 使用现有的æŒä¹…å· +- 动æ€é…置新的æŒä¹…å· + +è¦ä½¿ç”¨çŽ°æœ‰ PV，你的应用需è¦ä½¿ç”¨å·²ç»‘定到 PV çš„ PVC，并且 PV åº”åŒ…å« PVC 所需的最少资æºã€‚ + +对于动æ€å­˜å‚¨é…置，你的应用需è¦ä½¿ç”¨ç»‘定到存储类的 PVC。存储类包å«é…置新æŒä¹…å·çš„授æƒã€‚ + +![设置新的和现有的æŒä¹…存储](/img/rancher-storage.svg) + +有关更多信æ¯ï¼Œè¯·å‚阅[关于存储的官方 Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/volumes/)。 + + +## 关于æŒä¹…å·å£°æ˜Ž + +æŒä¹…å·å£°æ˜Ž (PVC) 是从集群请求存储资æºçš„对象，类似于你的 deployment 用于兑æ¢å­˜å‚¨è®¿é—®çš„凭è¯ã€‚PVC 以å·çš„å½¢å¼æŒ‚载到工作负载中，以便工作负载å¯ä»¥å£°æ˜Žå…¶æŒ‡å®šçš„æŒä¹…存储份é¢ã€‚ + +è¦è®¿é—®æŒä¹…存储，Pod 必须有挂载为å·çš„ PVC。此 PVC 让你的 deployment 应用将其数æ®å­˜å‚¨åœ¨å¤–部ä½ç½®ï¼Œè¿™æ ·ï¼Œå¦‚æžœ pod å‘生故障，就å¯ä»¥ç”¨æ–°çš„ pod 替æ¢å®ƒå¹¶ç»§ç»­è®¿é—®å…¶å¤–部存储的数æ®ã€‚ + +æ¯ä¸ª Rancher 项目都包å«ä½ åˆ›å»ºçš„ PVC 列表，å¯ä»Ž**èµ„æº > 工作负载 > å·**中查看。你å¯ä»¥åœ¨å°†æ¥åˆ›å»º deployment æ—¶å¤ç”¨è¿™äº› PVC。 + +### 新的和现有的æŒä¹…å­˜å‚¨éƒ½éœ€è¦ PVC + +无论工作负载打算使用现有存储，还是需è¦æ ¹æ®éœ€æ±‚动æ€é…置新存储，pod éƒ½éœ€è¦ PVC æ‰èƒ½ä½¿ç”¨æŒä¹…存储。 + +如果你为工作负载设置现有存储，则工作负载会挂载引用一个 PV çš„ PVC，该 PV 对应现有的存储基础设施。 + +如果工作负载需è¦è¯·æ±‚新的存储，则工作负载会挂载引用一个存储类的 PVC，该存储类能创建新 PV åŠå…¶åº•å±‚存储基础设施。 + +Rancher å…许你在项目中创建任æ„æ•°é‡çš„ PVC。 + +ä½ å¯ä»¥åœ¨åˆ›å»º deployment 时将 PVC 挂载到 deployment 中，也å¯ä»¥åœ¨åˆ›å»º deployment åŽåœ¨ deployment è¿è¡Œçš„时候将 PVC 挂载到 deployment 中。 + +## 使用 PVC å’Œ PV 设置现有存储 + +ä½ çš„ pod å¯ä»¥å°†æ•°æ®å­˜å‚¨åœ¨[å·](https://kubernetes.io/docs/concepts/storage/volumes/)中，但如果 pod å‘生故障，该数æ®å°±ä¼šä¸¢å¤±ã€‚为了解决这个问题，Kubernetes æ供了æŒä¹…å· (PV)。PV 是 Kubernetes 资æºï¼Œè¿™äº›èµ„æºå¯¹åº”于你的 pod å¯ä»¥è®¿é—®çš„外部存储ç£ç›˜æˆ–文件系统。如果一个 pod 崩溃，它的替代 pod å¯ä»¥è®¿é—®æŒä¹…存储中的数æ®ï¼Œè€Œä¸ä¼šä¸¢å¤±ä»»ä½•æ•°æ®ã€‚ + +PV å¯ä»¥æ˜¯ä½ æœ¬åœ°æˆ–托管的云上（Amazon EBS 或 Azure Disk）物ç†ç£ç›˜æˆ–文件系统。 + +在 Rancher 中创建æŒä¹…å·ä¸ä¼šåˆ›å»ºå­˜å‚¨å·ã€‚它åªåˆ›å»ºæ˜ å°„到现有å·çš„ Kubernetes 资æºã€‚因此，在你å¯ä»¥å°†æŒä¹…å·åˆ›å»ºä¸º Kubernetes 资æºä¹‹å‰ï¼Œä½ å¿…须先é…置存储。 + +:::note é‡è¦æ示： + +PV 是在集群级别创建的，这æ„味ç€åœ¨å¤šç§Ÿæˆ·é›†ç¾¤ä¸­ï¼Œæœ‰æƒè®¿é—®ä¸åŒå‘½å空间的团队å¯ä»¥è®¿é—®åŒä¸€ä¸ª PV。 + +::: + +### å°† PV 绑定到 PVC + +当 pod 设置为使用æŒä¹…存储时，pod 会挂载æŒä¹…å·å£°æ˜Ž (PVC)，PVC 的挂载方å¼ä¸Žå…¶ä»– Kubernetes å·ç›¸åŒã€‚在创建æ¯ä¸ª PVC 时，Kubernetes master å°† PVC 视为对存储的请求，并将其绑定到与 PVC 的最低资æºè¦æ±‚相匹é…çš„ PV。并éžæ¯ä¸ª PVC 都ä¿è¯ç»‘定到 PV。详情请å‚阅 Kubernetes [文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)。 + +> 如果没有匹é…çš„å·ï¼Œåˆ™å£°æ˜Žå°†ä¸€ç›´ä¿æŒä¸ºæœªç»‘定状æ€ã€‚当匹é…çš„å·å¯ç”¨æ—¶ï¼Œå£°æ˜Žå°†ä¼šè¢«ç»‘定。例如，é…置有许多 50Gi PV 的集群ä¸ä¼šåŒ¹é…请求 100Gi çš„ PVC。PVC å¯ä»¥åœ¨ 100Gi PV 添加到集群时进行绑定。 + +æ¢å¥è¯è¯´ï¼Œä½ å¯ä»¥åˆ›å»ºæ— é™é‡çš„ PVC，但åªæœ‰åœ¨ Kubernetes master å¯ä»¥æ‰¾åˆ°æ»¡è¶³ PVC 所需的ç£ç›˜ç©ºé—´é‡çš„ PV 时，PVC æ‰ä¼šç»‘定到 PV。 + +è¦åŠ¨æ€é…置新存储，挂载在 pod 中的 PVC 对应的必须是存储类而ä¸æ˜¯æŒä¹…å·ã€‚ + +## 使用 PVC 和存储类é…置新存储 + +存储类å…许你动æ€åˆ›å»º PV，而无需先在基础设施æ供商中创建æŒä¹…存储。 + +例如，如果工作负载绑定到 PVC 并且 PVC 引用 Amazon EBS 存储类，则存储类å¯ä»¥åŠ¨æ€åˆ›å»º EBS å·å’Œç›¸åº”çš„ PV。 + +ç„¶åŽ Kubernetes master 会将新创建的 PV 绑定到你的工作负载的 PVC，从而å…许你的工作负载使用æŒä¹…存储。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md new file mode 100644 index 00000000000..6582e5e0f50 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md @@ -0,0 +1,87 @@ +--- +title: 在 Rancher 中动æ€é…置新存储 +--- + +本文介ç»å¦‚何为 Rancher 中的工作负载é…置新的æŒä¹…存储。 + +本节å‡è®¾ä½ äº†è§£ Kubernetes 的存储类和æŒä¹…å·å£°æ˜Žçš„概念。如需更多信æ¯ï¼Œè¯·å‚阅[存储的工作原ç†](about-persistent-storage.md)。 + +新存储通常由 Amazon EBS 等云æ供商æ供。但是，新存储ä¸ä¸€å®šè¦åœ¨äº‘中。 + +如果你有一个å—存储池并且ä¸æƒ³ä½¿ç”¨äº‘æ供商，你å¯ä»¥ä½¿ç”¨ Longhorn 为 Kubernetes 集群æä¾›æŒä¹…存储。详情请å‚è§[本页é¢](../../../../../integrations-in-rancher/longhorn.md)。 + +è¦ä¸ºä½ çš„工作负载é…置新存储，请执行以下步骤： + +1. [添加一个存储类并将其é…置为使用你的存储](#1-添加一个存储类并将其é…置为使用你的存储) +2. [为使用 StatefulSet 部署的 Pod 使用存储类](#2-为使用-statefulset-部署的-pod-使用存储类) + +### 先决æ¡ä»¶ + +- 设置æŒä¹…存储需è¦`管ç†å·`çš„[角色](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色å‚考)。 +- 如果你è¦ä¸ºäº‘集群é…置存储，则存储和集群主机必须使用相åŒçš„云æ供商。 +- å¿…é¡»å¯ç”¨äº‘æ供商。有关å¯ç”¨äº‘æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../../../pages-for-subheaders/set-up-cloud-providers.md)。 +- ç¡®ä¿ä½ çš„存储å·æ’件å¯ä»¥å¯ç”¨ã€‚ + +默认情况下å¯ç”¨ä»¥ä¸‹å­˜å‚¨å·æ’件： + +| å称 | æ’件 | +--------|---------- +| Amazon EBS Disk | `aws-ebs` | +| AzureFile | `azure-file` | +| AzureDisk | `azure-disk` | +| Google Persistent Disk | `gce-pd` | +| Longhorn | `flex-volume-longhorn` | +| VMware vSphere Volume | `vsphere-volume` | +| 本地 | `local` | +| 网络文件系统 | `nfs` | +| hostPath | `host-path` | + +如果你的存储å·æ’件没有在上述列表中，你需è¦[使用功能开关æ¥å¯ç”¨ä¸å—支æŒçš„存储驱动](../../../../advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md)。 + +### 1. 添加一个存储类并将其é…置为使用你的存储 + +这些步骤æ述了如何在集群级别设置存储类： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦åŠ¨æ€é…ç½®æŒä¹…存储å·çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**存储 > 存储类**。 +1. å•å‡»**创建**。 +1. 输入存储类的å称。 +1. 从 **Provisioner** 下拉列表中，选择è¦ç”¨äºŽåŠ¨æ€é…置存储å·çš„æœåŠ¡ã€‚例如，如果你有一个 Amazon EC2 集群并且想è¦ä½¿ç”¨äº‘存储，请使用 `Amazon EBS Disk`。 +1. 在**å‚æ•°**选项å¡ä¸­ï¼Œå¡«å†™æœåŠ¡ç”¨äºŽåŠ¨æ€é…置存储å·æ‰€éœ€çš„ä¿¡æ¯ã€‚æ¯ä¸ªå·æ’件都需è¦ä¸åŒçš„ä¿¡æ¯æ¥åŠ¨æ€é…置存储å·ã€‚有关如何获å–此信æ¯çš„帮助，请å‚阅æœåŠ¡æ–‡æ¡£ã€‚ +1. å•å‡»**创建**。 + +**结果**：存储类å¯ä¾› PVC 使用。 + +有关存储类å‚数的完整信æ¯ï¼Œè¯·å‚阅官方 [Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/storage-classes/#parameters)。 + +### 2. 为使用 StatefulSet 部署的 Pod 使用存储类 + +StatefulSet ç®¡ç† Pod 的部署和扩展，åŒæ—¶ä¸ºæ¯ä¸ª Pod 维护一个粘性标识。在这个 StatefulSet 中，我们将é…置一个 VolumeClaimTemplate。StatefulSet 管ç†çš„æ¯ä¸ª Pod 都将部署一个基于此 VolumeClaimTemplate çš„ PersistentVolumeClaim。PersistentVolumeClaim 将引用我们创建的 StorageClass。因此，在部署 StatefulSet 管ç†çš„æ¯ä¸ª Pod 时，都会使用 PersistentVolumeClaim 中定义的 StorageClass æ¥ç»‘定到动æ€é…置的存储。 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°† StorageClass 用于工作负载的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡»**创建**。 +1. å•å‡» **StatefulSet**。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. 输入æŒä¹…å·çš„å称。 +1. 在*存储类*\*字段中，选择将为此 StatefulSet 管ç†çš„ pod 动æ€é…置存储的 StorageClass。 +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. 点击**å¯åŠ¨**。 + +**结果**：StatefulSet 管ç†çš„æ¯ä¸ª Pod 部署完æˆåŽï¼Œéƒ½ä¼šå‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到具有兼容 PVC çš„ Pod。 + +è¦å°† PVC 附加到现有工作负载， + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°† StorageClass 用于工作负载的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡» **â‹® > 编辑é…ç½®**，转到使用由 StorageClass é…置的存储的工作负载。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**中，å•å‡»**添加声明模æ¿**。 +1. 输入æŒä¹…å·å称。 +1. 在*存储类*\*字段中，选择将为此 StatefulSet 管ç†çš„ pod 动æ€é…置存储的 StorageClass。 +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. å•å‡»**ä¿å­˜**。 + +**结果**ï¼šå·¥ä½œè´Ÿè½½å°†å‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到 PVC。å¦åˆ™ï¼ŒRancher å°†é…置新的æŒä¹…存储。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md new file mode 100644 index 00000000000..ea7fde37298 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md @@ -0,0 +1,62 @@ +--- +title: iSCSI å· +--- + +在将数æ®å­˜å‚¨åœ¨ iSCSI å·ä¸Šçš„ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中，你å¯èƒ½ä¼šé‡åˆ° kubelet 无法自动连接 iSCSI å·çš„问题。æˆæ­¤é—®é¢˜çš„原因很å¯èƒ½æ˜¯ iSCSI å¯åŠ¨å™¨å·¥å…·ä¸å…¼å®¹ã€‚ä½ å¯ä»¥åœ¨æ¯ä¸ªé›†ç¾¤èŠ‚点上安装 iSCSI å¯åŠ¨å™¨å·¥å…·æ¥è§£å†³æ­¤é—®é¢˜ã€‚ + +将数æ®å­˜å‚¨åˆ° iSCSI å·çš„ç”± Rancher å¯åŠ¨çš„ Kubernetes 集群使用 [iSCSI å¯åŠ¨å™¨å·¥å…·](http://www.open-iscsi.com/)，该工具嵌入在 kubelet çš„ `rancher/hyperkube` Docker é•œåƒä¸­ã€‚该工具从æ¯ä¸ª kubeletï¼ˆå³ _initiator_）å‘现并å‘起与 iSCSI å·ï¼ˆå³ _target_）的会è¯ã€‚但是，在æŸäº›æƒ…况下，initiator å’Œ target 上安装的 iSCSI å¯åŠ¨å™¨å·¥å…·çš„版本å¯èƒ½ä¸åŒ¹é…，从而导致连接失败。 + +如果你é‡åˆ°æ­¤é—®é¢˜ï¼Œä½ å¯ä»¥åœ¨é›†ç¾¤ä¸­çš„æ¯ä¸ªèŠ‚点上安装å¯åŠ¨å™¨å·¥å…·æ¥è§£å†³è¯¥é—®é¢˜ã€‚ä½ å¯ä»¥é€šè¿‡ç™»å½•åˆ°é›†ç¾¤èŠ‚点并输入以下其中一个命令æ¥å®‰è£… iSCSI å¯åŠ¨å™¨å·¥å…·ï¼š + +| å¹³å° | 包å | 安装命令 | +| ------------- | ----------------------- | -------------------------------------- | +| Ubuntu/Debian | `open-iscsi` | `sudo apt install open-iscsi` | +| RHEL | `iscsi-initiator-utils` | `yum install iscsi-initiator-utils -y` | + + +在节点上安装å¯åŠ¨å™¨å·¥å…·åŽï¼Œç¼–辑集群的 YAML，编辑 kubelet é…置以挂载 iSCSI 二进制文件和é…置，如下é¢çš„示例所示。 + +:::note 注æ„事项 + +- 在更新 Kubernetes YAML 以挂载 iSCSI 二进制文件和é…置之å‰ï¼Œè¯·ç¡®ä¿å·²å°† `open-iscsi`（deb）或 `iscsi-initiator-utils`（yum）安装到你的集群节点。如果在绑定挂载创建到你的 Kubernetes YAML _之å‰_ 未安装此包，Docker 将自动在æ¯ä¸ªèŠ‚点上创建目录和文件，并且ä¸å…许包安装æˆåŠŸã€‚ + +- 下é¢çš„示例 YAML ä¸é€‚用于 K3s，仅适用于 RKE 集群。由于 K3s kubelet ä¸åœ¨å®¹å™¨ä¸­è¿è¡Œï¼Œå› æ­¤ä¸éœ€è¦æ·»åŠ é¢å¤–的绑定。但是，所有 iSCSI 工具ä»å¿…须安装在你的 K3s 节点上。 + +::: + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" +``` + +如果你使用的是 RHEL 7.9，你需è¦æŒ‚载文件 `/usr/lib64/libcrypto.so.10`，如下例所示。 + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.10:/usr/lib/libcrypto.so.10" +``` + +如果你使用的是 RHEL 8.6，你需è¦æŒ‚载文件 `/usr/lib64/libcrypto.so.1.1`，如下例所示。 + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.1.1:/usr/lib/libcrypto.so.1.1" +``` + + +:::tip + +è¦éªŒè¯ iscsiadm 是å¦æ­£å¸¸å·¥ä½œï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èŠ‚点上è¿è¡Œå‘½ä»¤ `docker exec kubelet iscsiadm --version`。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md new file mode 100644 index 00000000000..2232fc9c253 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md @@ -0,0 +1,85 @@ +--- +title: 设置现有存储 +--- + +本文介ç»å¦‚何在 Rancher 中为工作负载设置现有的æŒä¹…存储。 + +:::note + +本节å‡è®¾ä½ äº†è§£ Kubernetes çš„æŒä¹…å·å’ŒæŒä¹…å·å£°æ˜Žçš„概念。如需更多信æ¯ï¼Œè¯·å‚阅[存储的工作原ç†](about-persistent-storage.md)。 + +::: + +è¦è®¾ç½®å­˜å‚¨ï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ï¼š + +1. [设置æŒä¹…存储](#1-设置æŒä¹…存储)。 +2. [添加一个引用æŒä¹…存储的 PersistentVolume](#2-添加一个引用æŒä¹…存储的-persistentvolume)。 +3. [为使用 StatefulSet 部署的 Pod 使用存储类](#3-为使用-statefulset-部署的-pod-使用存储类) + +### 先决æ¡ä»¶ + +- è¦å°†æŒä¹…å·åˆ›å»ºä¸º Kubernetes 资æºï¼Œä½ å¿…须具有`管ç†å·`çš„[角色。](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色å‚考) +- 如果你è¦ä¸ºäº‘集群é…置存储，则存储和集群主机必须使用相åŒçš„云æ供商。 + +### 1. 设置æŒä¹…存储 + +在 Rancher 中创建æŒä¹…å·ä¸ä¼šåˆ›å»ºå­˜å‚¨å·ã€‚它åªåˆ›å»ºæ˜ å°„到现有å·çš„ Kubernetes 资æºã€‚因此，在你å¯ä»¥å°†æŒä¹…å·åˆ›å»ºä¸º Kubernetes 资æºä¹‹å‰ï¼Œä½ å¿…须先é…置存储。 + +设置æŒä¹…存储设备的步骤会因你的基础设施而异。我们æ供了使用 [vSphere](../../provisioning-storage-examples/vsphere-storage.md)ã€[NFS](../../provisioning-storage-examples/nfs-storage.md) 或 [Amazon EBS ](../../provisioning-storage-examples/persistent-storage-in-amazon-ebs.md)设置存储的示例。 + +如果你有一个å—存储池并且ä¸æƒ³ä½¿ç”¨äº‘æ供商，你å¯ä»¥ä½¿ç”¨ Longhorn 为 Kubernetes 集群æä¾›æŒä¹…存储。详情请å‚è§[本页é¢](../../../../../integrations-in-rancher/longhorn.md)。 + +### 2. 添加一个引用æŒä¹…存储的 PersistentVolume + +这些步骤æ述了如何在 Kubernetes 的集群级别设置 PersistentVolume。 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ æŒä¹…å·çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**存储 > æŒä¹…å·**。 +1. å•å‡»**创建**。 +1. 输入æŒä¹…å·çš„**å称**。 +1. 为正在使用的ç£ç›˜ç±»åž‹æˆ–æœåŠ¡é€‰æ‹©**å·æ’件**。将存储添加到由云æ供商托管的集群时，请使用云æ供商的云存储æ’件。例如，如果你有一个 Amazon EC2 集群并且想è¦ä½¿ç”¨äº‘存储，则必须使用 `Amazon EBS ç£ç›˜`å·æ’件。 +1. 输入å·çš„**容é‡**（å•ä½ï¼šGB）。 +1. 填写**æ’件é…ç½®**表å•ã€‚æ¯ä¸ªæ’件类型都需è¦ç£ç›˜ç±»åž‹ä¾›åº”商的信æ¯ã€‚有关æ¯ä¸ªæ’件的表å•å’Œæ‰€éœ€ä¿¡æ¯çš„帮助，请å‚阅æ’件的供应商文档。 +1. å¯é€‰ï¼šåœ¨**自定义**表å•ä¸­ï¼Œé…ç½®[访问模å¼](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)。此选项设置å¯ä»¥è®¿é—®å·çš„节点数é‡ï¼Œä»¥åŠèŠ‚点的读/写æƒé™ã€‚[Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)中的表格列出了æ’件支æŒçš„访问模å¼ã€‚ +1. å¯é€‰ï¼šåœ¨**自定义**表å•ä¸­ï¼Œé…ç½®[挂载选项](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options)。æ¯ä¸ªå·æ’件都å…许你在安装过程中指定其他命令行选项。请查阅æ¯ä¸ªæ’件的供应商文档以获å–å¯ç”¨çš„挂载选项。 +1. å•å‡»**创建**。 + +**结果**：已创建你的新æŒä¹…å·ã€‚ + + +### 3. 为使用 StatefulSet 部署的 Pod 使用存储类 + +StatefulSet ç®¡ç† Pod 的部署和扩展，åŒæ—¶ä¸ºæ¯ä¸ª Pod 维护一个粘性标识。在这个 StatefulSet 中，我们将é…置一个 VolumeClaimTemplate。StatefulSet 管ç†çš„æ¯ä¸ª Pod 都将部署一个基于此 VolumeClaimTemplate çš„ PersistentVolumeClaim。PersistentVolumeClaim 将引用我们创建的 PersistentVolume。因此，在部署 StatefulSet 管ç†çš„æ¯ä¸ª Pod 时，都会绑定一个 PersistentVolumeClaim 中定义的 PersistentVolume。 + +ä½ å¯ä»¥åœ¨å·¥ä½œè´Ÿè½½åˆ›å»ºæœŸé—´æˆ–之åŽä¸º StatefulSet é…置存储。 + +以下步骤æ述了如何将现有存储分é…给新的 StatefulSet： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦ä¸º StatefulSet é…置存储的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载 > StatefulSets**。 +1. å•å‡»**创建**。 +1. 选择è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„命å空间。 +1. 输入 StatefulSet çš„å称。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. å•å‡»**使用现有的æŒä¹…å·**。 +1. 在**æŒä¹…å·**字段中，选择你创建的æŒä¹…å·ã€‚ +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. 点击**å¯åŠ¨**。 + +**结果**ï¼šå½“å·¥ä½œè´Ÿè½½è¢«éƒ¨ç½²æ—¶ï¼Œå®ƒä¼šå‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到 PVC。 + +以下步骤æ述了如何将æŒä¹…存储分é…给现有工作负载： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦ä¸º StatefulSet é…置存储的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载 > StatefulSets**。 +1. 转到è¦æ·»åŠ æŒä¹…存储的工作负载。å•å‡» **â‹® > 编辑**。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. å•å‡»**使用现有的æŒä¹…å·**。 +1. 在**æŒä¹…å·**字段中，选择你创建的æŒä¹…å·ã€‚ +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. 点击**å¯åŠ¨**。 + +**结果**ï¼šå·¥ä½œè´Ÿè½½å°†å‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到 PVC。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md new file mode 100644 index 00000000000..31f13278ae2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md @@ -0,0 +1,420 @@ +--- +title: 使用外部 Ceph 驱动 +--- + +本文介ç»å¦‚何在 RKE2 集群中使用外部 Ceph 驱动。如果你使用的是 RKE，则需è¦æ‰§è¡Œå…¶ä»–步骤。详情请å‚阅[本节](#在-rke-中使用-ceph-驱动)。 + + +## è¦æ±‚ + +ç¡®ä¿ ceph-common å’Œ xfsprogs 软件包安装在 SLE worker 节点上。 + +## 在 RKE 中使用 Ceph 驱动 + +以下资æºä¸Ž RKE 集群完全兼容，但需è¦ä¸º RKE 执行é¢å¤–çš„ kubelet é…置。 + +在 RKE 集群上，kubelet 组件在 Docker 容器中è¿è¡Œï¼Œé»˜è®¤æƒ…å†µä¸‹æ— æ³•åƒ rbd å’Œ libceph 一样访问主机的内核模å—。 + +为了解决这个é™åˆ¶ï¼Œä½ å¯ä»¥åœ¨ worker 节点上è¿è¡Œ `modprobe rbd`，或者é…ç½® kubelet 容器以将主机中的 `/lib/modules` 目录自动挂载到容器中。 + +对于 kubelet é…置，在 RKE 集群é…置之å‰ï¼Œå°†ä»¥ä¸‹å†…容放入 `cluster.yml` 文件中。你ç¨åŽè¿˜å¯ä»¥åœ¨ Rancher UI 中通过å•å‡»**编辑集群 > 以 YAML 文件编辑**并é‡æ–°å¯åŠ¨ worker 节点æ¥ä¿®æ”¹ `cluster.yml`。 + +```yaml +services: + kubelet: + extra_binds: + - '/lib/modules:/lib/modules:ro' +``` + +有关 `extra_binds` 指令的更多信æ¯ï¼Œè¯·å‚阅[本节](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds)。 + +## 在 RKE2 集群上安装 ceph-csi 驱动 + +:::note + +åªæœ‰åŠ¨æ€ RBD é…置需è¦æ‰§è¡Œè¿™äº›æ­¥éª¤ã€‚ + +::: + +有关 `ceph-csi-rbd` chart 的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://github.com/ceph/ceph-csi/blob/devel/charts/ceph-csi-rbd/README.md)。 + +è¦èŽ·å–有关 SES 集群的详细信æ¯ï¼Œè¯·è¿è¡Œï¼š + +``` +ceph mon dump +``` + +查看输出： + +``` +dumped monmap epoch 3 +epoch 3 +fsid 79179d9d-98d8-4976-ab2e-58635caa7235 +last_changed 2021-02-11T10:56:42.110184+0000 +created 2021-02-11T10:56:22.913321+0000 +min_mon_release 15 (octopus) +0: [v2:10.85.8.118:3300/0,v1:10.85.8.118:6789/0] mon.a +1: [v2:10.85.8.123:3300/0,v1:10.85.8.123:6789/0] mon.b +2: [v2:10.85.8.124:3300/0,v1:10.85.8.124:6789/0] mon.c +``` + +ç¨åŽä½ å°†éœ€è¦ fsid å’Œ mon 地å€å€¼ã€‚ + +## 使用 Helm 安装 ceph-csi 驱动 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +helm repo add ceph-csi https://ceph.github.io/csi-charts +helm repo update +helm search repo ceph-csi -l +helm inspect values ceph-csi/ceph-csi-rbd > ceph-csi-rbd-values.yaml +``` + +修改 `ceph-csi-rbd-values.yaml` 文件并åªä¿ç•™æ‰€éœ€çš„更改： + +```yaml +# ceph-csi-rbd-values.yaml +csiConfig: + - clusterID: "79179d9d-98d8-4976-ab2e-58635caa7235" + monitors: + - "10.85.8.118:6789" + - "10.85.8.123:6789" + - "10.85.8.124:6789" +provisioner: + name: provisioner + replicaCount: 2 +``` + +ç¡®ä¿å¯ä»¥ä»Ž RKE2 集群访问 ceph 监视器（例如通过 ping）。 + +``` +kubectl create namespace ceph-csi-rbd +helm install --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +kubectl rollout status deployment ceph-csi-rbd-provisioner -n ceph-csi-rbd +helm status ceph-csi-rbd -n ceph-csi-rbd +``` + +如果你想直接通过 Helm 修改é…置，则å¯ä»¥ä½¿ç”¨ `ceph-csi-rbd-values.yaml` 文件并调用： + +``` +helm upgrade \ + --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +``` + +## 创建 RBD Ceph èµ„æº + +``` +# 创建一个 ceph 池： +ceph osd pool create myPool 64 64 + +# 创建一个å—设备池： +rbd pool init myPool + +# 创建一个å—设备镜åƒï¼š +rbd create -s 2G myPool/image + +# 创建一个å—设备用户并记录密钥： +ceph auth get-or-create-key client.myPoolUser mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=myPool" | tr -d '\n' | base64 +QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== + +# Base64 加密 ceph 用户 myPoolUser： +echo "myPoolUser" | tr -d '\n' | base64 +bXlQb29sVXNlcg== + +# 创建一个å—设备管ç†å‘˜å¹¶è®°å½•å¯†é’¥ï¼š +ceph auth get-or-create-key client.myPoolAdmin mds 'allow *' mgr 'allow *' mon 'allow *' osd 'allow * pool=myPool' | tr -d '\n' | base64 +QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== + +# Base64 加密 ceph 用户 myPoolAdmin： +echo "myPoolAdmin" | tr -d '\n' | base64 +bXlQb29sQWRtaW4= +``` +## é…ç½® RBD Ceph 访问密钥 + +### ç”¨æˆ·è´¦å· + +对于é™æ€ RBD é…置（ceph 池中的镜åƒå¿…须存在），è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +cat > ceph-user-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-user + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sVXNlcg== + userKey: QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== +EOF + +kubectl apply -f ceph-user-secret.yaml +``` + +### 管ç†å‘˜è´¦å· + +å¯¹äºŽåŠ¨æ€ RBD é…置（用于在给定 ceph 池中自动创建镜åƒï¼‰ï¼Œè¯·è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +cat > ceph-admin-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-admin + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sQWRtaW4= + userKey: QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== +EOF + +kubectl apply -f ceph-admin-secret.yaml +``` + +## 创建 RBD æµ‹è¯•èµ„æº + +### 在 Pod 中使用 RBD + +```yaml +# pod +cat > ceph-rbd-pod-inline.yaml << EOF +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-inline +spec: + containers: + - name: ceph-rbd-pod-inline + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +EOF + +kubectl apply -f ceph-rbd-pod-inline.yaml +kubectl get pod +kubectl exec pod/ceph-rbd-pod-inline -- df -k | grep rbd +``` + +### 在æŒä¹…å·ä¸­ä½¿ç”¨ RBD + +```yaml +# pod-pvc-pv +cat > ceph-rbd-pod-pvc-pv-allinone.yaml << EOF +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ceph-rbd-pv +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ceph-rbd-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-pvc-pv +spec: + containers: + - name: ceph-rbd-pod-pvc-pv + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + persistentVolumeClaim: + claimName: ceph-rbd-pvc +EOF + +kubectl apply -f ceph-rbd-pod-pvc-pv-allinone.yaml +kubectl get pv,pvc,pod +kubectl exec pod/ceph-rbd-pod-pvc-pv -- df -k | grep rbd +``` + +### 在存储类中使用 RBD + +此示例用于动æ€é…ç½®ã€‚éœ€è¦ ceph-csi 驱动。 + +```yaml +# pod-pvc-sc +cat > ceph-rbd-pod-pvc-sc-allinone.yaml < /root/.bashrc << EOF +export PATH=$PATH:/var/lib/rancher/rke2/bin/ +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +EOF + +cat /var/lib/rancher/rke2/server/node-token +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +``` + +### RKE2 Agent/Worker é…ç½® + +``` +mkdir -p /etc/rancher/rke2/ + +cat > /etc/rancher/rke2/config.yaml << EOF +server: https://10.100.103.23:9345 +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +EOF + +curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh - +systemctl enable --now rke2-agent.service +``` + +è¦å°†é›†ç¾¤å¯¼å…¥ Rancher，请å•å‡» **☰ > 集群管ç†**。然åŽåœ¨**集群**页é¢ä¸Šï¼Œå•å‡»**导入已有集群**。然åŽåœ¨ server/master 上è¿è¡Œæ供的 kubectl 命令。 + +## 测试版本 + +è¿è¡Œ RKE2 节点的æ“作系统：安装了 kernel-default-5.3.18-24.49 çš„ JeOS SLE15-SP2 + +``` +kubectl version +Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.4", GitCommit:"c96aede7b5205121079932896c4ad89bb93260af", GitTreeState:"clean", BuildDate:"2020-06-22T12:00:00Z", GoVersion:"go1.13.11", Compiler:"gc", Platform:"linux/amd64"} +Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7+rke2r1", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-20T01:50:52Z", GoVersion:"go1.15.5b5", Compiler:"gc", Platform:"linux/amd64"} + +helm version +version.BuildInfo{Version:"3.4.1", GitCommit:"c4e74854886b2efe3321e185578e6db9be0a6e29", GitTreeState:"clean", GoVersion:"go1.14.12"} +``` + +RKE2 集群上的 Kubernetes 版本：v1.19.7+rke2r1 + +## 故障排除 + +如果你使用的是基于 SES7 çš„ SUSE ceph-rook，你å¯ä»¥é€šè¿‡ç¼–辑 `rook-1.4.5/ceph/cluster.yaml` 并设置 `spec.network.hostNetwork=true` æ¥å…¬å¼€ hostNetwork 上的监视器。 + +如果è¦æ“作 ceph-rook 集群，则å¯ä»¥åœ¨ Kubernetes 集群上部署一个工具箱，其中 ceph-rook 通过 `kubectl apply -f rook-1.4.5/ceph/toolbox.yaml` é…置。然åŽæ‰€æœ‰ä¸Ž ceph 相关的命令都å¯ä»¥åœ¨ toolbox pod 中执行，例如，è¿è¡Œ `kubectl exec -it -n rook-ceph rook-ceph-tools-686d8b8bfb-2nvqp -- bash`。 + +ceph æ“作 - 基本命令： + +``` +ceph osd pool stats +ceph osd pool delete myPool myPool --yes-i-really-really-mean-it +rbd list -p myPool +> csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 +> image +``` + +删除镜åƒï¼š`rbd rm csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 -p myPool` + +rook 工具箱中的 CephFS 命令： + +``` +ceph -s +ceph fs ls +ceph fs fail cephfs +ceph fs rm cephfs --yes-i-really-mean-it +ceph osd pool delete cephfs_data cephfs_data --yes-i-really-really-mean-it +ceph osd pool delete cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it +``` + +è¦å‡†å¤‡ cephfs 文件系统，你å¯ä»¥åœ¨ rook 集群上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +kubectl apply -f rook-1.4.5/ceph/filesystem.yaml +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md new file mode 100644 index 00000000000..a262560e90f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md @@ -0,0 +1,575 @@ +--- +title: 通过 AWS EC2 Auto Scaling 组使用 Cluster Autoscaler +--- + +本指å—介ç»å¦‚何使用 AWS EC2 Auto Scaling 组在 Rancher 自定义集群上安装和使用 [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/)。 + +我们将安装一个 Rancher RKE 自定义集群，该集群具有固定数é‡çš„具有 etcd å’Œ controlplane 角色的节点，以åŠæ•°é‡å¯å˜çš„具有 worker 角色的节点，它们由 `cluster-autoscaler` 管ç†ã€‚ + + +## 先决æ¡ä»¶ + +本指å—è¦æ±‚： + +* Rancher Server 正常è¿è¡Œã€‚ +* ä½ çš„ AWS EC2 用户具有创建虚拟机ã€Auto Scaling ç»„ä»¥åŠ IAM é…置文件和角色的适当æƒé™ã€‚ + +### 1. 创建自定义集群 + +在 Rancher Server 上，我们需è¦åˆ›å»ºä¸€ä¸ªè‡ªå®šä¹‰çš„ K8s 集群。请å‚阅[此处](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/)以检查版本兼容性。 + +è¯·ç¡®ä¿ cloud_provider å称设置为 `amazonec2`。创建集群åŽï¼Œæˆ‘们需è¦èŽ·å¾—： + +* clusterID：`c-xxxxx` 将用于 EC2 `kubernetes.io/cluster/` 实例标签。 +* clusterName：将用于 EC2 `k8s.io/cluster-autoscaler/` 实例标签。 +* nodeCommand：将添加到 EC2 实例 `user_data` 上，以包å«é›†ç¾¤ä¸Šçš„新节点。 + + ```sh + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum + ``` + +### 2. é…置云æ供商 + +在 AWS EC2 上，我们需è¦åˆ›å»ºä¸€äº›å¯¹è±¡æ¥é…置系统。为了在 AWS 上进行é…置，我们定义了三个ä¸åŒçš„组和 IAM é…置文件。 + +1. Autoscaling 组：将加入 EC2 Auto Scaling 组 (ASG) 的节点。`cluster-autoscaler` 将使用 ASG æ¥è¿›è¡Œæ‰©ç¼©å®¹ã€‚ +* IAM é…置文件：è¿è¡Œ cluster-autoscaler çš„ K8s 节点需è¦è¯¥æ–‡ä»¶ã€‚推è用于 Kubernetes master 节点。此é…置文件称为 `K8sAutoscalerProfile`。 + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:DescribeTags", + "autoscaling:DescribeLaunchConfigurations", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + +2. Master 组：将æˆä¸º Kubernetes etcd å’Œ/或 controlplane 的节点。该组ä¸ä¼šåœ¨ ASG 中。 +* IAM é…ç½®æ–‡ä»¶ï¼šé›†æˆ Kubernetes cloud_provider 需è¦è¯¥æ–‡ä»¶ã€‚或者，你也å¯ä»¥ä½¿ç”¨ `AWS_ACCESS_KEY` å’Œ `AWS_SECRET_KEY` æ¥ä»£æ›¿ [using-aws-credentials](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)。此é…置文件称为 `K8sMasterProfile`。 + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + + * IAM 角色：`K8sMasterRole: [K8sMasterProfile,K8sAutoscalerProfile]` + * 安全组：`K8sMasterSg`。详情请å‚è§ [RKE 端å£ï¼ˆè‡ªå®šä¹‰èŠ‚点选项å¡ï¼‰](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#下游-kubernetes-集群节点) + * 标签： + `kubernetes.io/cluster/: owned` + * 用户数æ®ï¼š`K8sMasterUserData` Ubuntu 18.04(ami-0e11cbb34015ff725)，安装 Docker 并将 etcd å’Œ controlplane 节点添加到 K8s 集群。 + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--etcd --controlplane" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +3. Worker 组：将加入 K8s worker plane 的节点。Worker 节点将由 cluster-autoscaler 使用 ASG 进行扩缩容。 +* IAM é…置文件：æä¾› cloud_provider worker 集æˆã€‚ + æ­¤é…置文件称为 `K8sWorkerProfile`。 + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } + ] + } + ``` + +* IAM 角色：`K8sWorkerRole：[K8sWorkerProfile]` +* 安全组：`K8sWorkerSg`。详情请å‚è§ [RKE 端å£ï¼ˆè‡ªå®šä¹‰èŠ‚点选项å¡ï¼‰](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#下游-kubernetes-集群节点) +* 标签： + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` +* 用户数æ®ï¼š`K8sWorkerUserData` Ubuntu 18.04(ami-0e11cbb34015ff725)，安装 Docker 并将 worker 节点添加到 K8s 集群。 + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--worker" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +详情请å‚è§ [AWS 上的 RKE 集群](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md) å’Œ [AWS 上的 Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md)。 + +### 3. 部署节点 + +我们é…ç½® AWS åŽï¼Œæˆ‘们需è¦åˆ›å»ºè™šæ‹Ÿæœºæ¥å¼•å¯¼é›†ç¾¤ï¼š + +* master (etcd+controlplane)：根æ®éœ€è¦éƒ¨ç½²ä¸‰ä¸ªé€‚当大å°çš„ master 实例。详情请å‚è§[生产就绪集群的建议](../../../../pages-for-subheaders/checklist-for-production-ready-clusters.md)。 + * IAM 角色：`K8sMasterRole` + * 安全组：`K8sMasterSg` + * 标签： + * `kubernetes.io/cluster/: owned` + * 用户数æ®ï¼š`K8sMasterUserData` + +* worker：使用以下设置在 EC2 上定义 ASG： + * å称：`K8sWorkerAsg` + * IAM 角色：`K8sWorkerRole` + * 安全组：`K8sWorkerSg` + * 标签： + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` + * 用户数æ®ï¼š`K8sWorkerUserData` + * 实例： + * 最少：2 + * ç†æƒ³æƒ…况：2 + * 最大：10 + +部署 VM åŽï¼Œä½ çš„ Rancher 自定义集群应该å¯ä»¥æ­£å¸¸è¿è¡Œäº†ï¼Œå…¶ä¸­åŒ…å«ä¸‰ä¸ª master 节点和两个 worker 节点。 + +### 4. 安装 Cluster-autoscaler + +此时，我们的 Rancher 集群应该已正常è¿è¡Œã€‚æˆ‘ä»¬å°†æ ¹æ® cluster-autoscaler 的建议，在 master 节点和 `kube-system` 命å空间上安装 cluster-autoscaler。 + +#### å‚æ•° + +下表显示了用于微调的 cluster-autoscaler å‚数： + +| å‚æ•° | 默认 | æè¿° | +|---|---|---| +| cluster-name | - | 自动扩缩的集群的å称（如果å¯ç”¨ï¼‰ | +| address | :8085 | 公开 Prometheus æŒ‡æ ‡çš„åœ°å€ | +| kubernetes | - | Kubernetes master ä½ç½®ã€‚如需使用默认值，则留空 | +| kubeconfig | - | 带有授æƒå’Œ master ä½ç½®ä¿¡æ¯çš„ kubeconfig 文件的路径 | +| cloud-config | - | 云æ供商é…置文件的路径。如果没有é…置文件，则为空字符串 | +| namespace | "kube-system" | è¿è¡Œ cluster-autoscaler 的命å空间 | +| scale-down-enabled | true | CA 是å¦åº”该缩å‡é›†ç¾¤ | +| scale-down-delay-after-add | "10m" | 扩容多久åŽæ¢å¤ç¼©å®¹è¯„ä¼° | +| scale-down-delay-after-delete | 0 | 节点删除多久åŽæ¢å¤ç¼©å®¹è¯„估，默认为 scanInterval 的值 | +| scale-down-delay-after-failure | "3m" | 缩容失败多久åŽæ¢å¤ç¼©å®¹è¯„ä¼° | +| scale-down-unneeded-time | "10m" | 在能进行缩容之å‰ï¼ŒèŠ‚点需è¦ä¸è¢«ä½¿ç”¨çš„时间 | +| scale-down-unready-time | "20m" | 在能进行缩容之å‰ï¼Œéžå°±ç»ªèŠ‚点应该需è¦ä¸è¢«ä½¿ç”¨çš„时间 | +| scale-down-utilization-threshold | 0.5 | 节点上è¿è¡Œçš„所有 pod çš„ CPU 或内存之和除以节点对应的å¯åˆ†é…资æºï¼Œä½ŽäºŽè¯¥å€¼æ—¶å¯ä»¥è€ƒè™‘缩å‡ä¸€ä¸ªèŠ‚点 | +| scale-down-gpu-utilization-threshold | 0.5 | 节点上è¿è¡Œçš„所有 pod çš„ GPU 请求总和除以节点的å¯åˆ†é…资æºï¼Œä½ŽäºŽè¯¥å€¼æ—¶å¯ä»¥è€ƒè™‘缩å‡ä¸€ä¸ªèŠ‚点 | +| scale-down-non-empty-candidates-count | 30 | 在一次迭代中被认为是空节点的最大数é‡ï¼Œè¿™äº›èŠ‚点会æˆä¸ºä½¿ç”¨æ¸…空æ¥ç¼©å®¹çš„候选节点 | +| scale-down-candidates-pool-ratio | 0.1 | 当先å‰è¿­ä»£çš„æŸäº›å€™é€‰èŠ‚点失效时，被视为è¦ç¼©å®¹çš„é¢å¤–éžç©ºå€™é€‰èŠ‚点的比率 | +| scale-down-candidates-pool-min-count | 50 | 当先å‰è¿­ä»£çš„æŸäº›å€™é€‰èŠ‚点失效时，被视为è¦ç¼©å®¹çš„é¢å¤–éžç©ºå€™é€‰èŠ‚ç‚¹çš„æœ€å¤§æ•°é‡ | +| node-deletion-delay-timeout | "2m" | CA 在删除节点之å‰ï¼Œç­‰å¾…删除 delay-deletion.cluster-autoscaler.kubernetes.io/ 注释的最长时间 | +| scan-interval | "10s" | é‡æ–°è¯„估集群扩缩容的频率 | +| max-nodes-total | 0 | 所有节点组中的最大节点数。Cluster Autoscaler ä¸ä¼šè®©é›†ç¾¤å¢žé•¿åˆ°è¶…è¿‡æ­¤æ•°é‡ | +| cores-total | "0:320000" | 集群中的最å°å’Œæœ€å¤§çš„核心数，格å¼ä¸º `:`。Cluster Autoscaler 会在该范围内扩缩集群。 | +| memory-total | "0:6400000" | 集群中最å°å’Œæœ€å¤§å†…å­˜åƒå…†å­—节数，格å¼ä¸º `:`。Cluster Autoscaler 会在该范围内扩缩集群。 | +| cloud-provider | - | 云æ供商类型 | +| max-bulk-soft-taint-count | 10 | å¯ä»¥åŒæ—¶æ·»åŠ /移除 PreferNoSchedule 污点的最大节点数。设置为 0 则关闭此类污点 | +| max-bulk-soft-taint-time | "3s" | åŒæ—¶æ·»åŠ /移除 PreferNoSchedule 污点的最大æŒç»­æ—¶é—´ã€‚ | +| max-empty-bulk-delete | 10 | å¯ä»¥åŒæ—¶åˆ é™¤çš„最大空节点数 | +| max-graceful-termination-sec | 600 | å°è¯•ç¼©å‡èŠ‚点时，CA 等待 pod 终止的最大秒数 | +| max-total-unready-percentage | 45 | 集群中未就绪节点的最大百分比。超过此值åŽï¼ŒCA å°†åœæ­¢æ“作 | +| ok-total-unready-count | 3 | å…许的未就绪节点数，与 max-total-unready-percentage æ— å…³ | +| scale-up-from-zero | true | 就绪节点数等于 0 时，CA 是å¦åº”该扩容 | +| max-node-provision-time | "15m" | CA 等待节点é…置的最长时间 | +| nodes | - | 以云æ供商接å—çš„æ ¼å¼è®¾ç½®èŠ‚点组的最å°ã€æœ€å¤§å¤§å°å’Œå…¶ä»–é…置数æ®ã€‚å¯ä»¥å¤šæ¬¡ä½¿ç”¨ã€‚æ ¼å¼æ˜¯ `::`。 | +| node-group-auto-discovery | - | 节点组自动å‘现的一个或多个定义。定义表示为 `:[[=]]` | +| estimator | - | "binpacking" | 用于扩容的资æºè¯„估器类型。å¯ç”¨å€¼ï¼š["binpacking"] | +| expander | "random" | è¦åœ¨æ‰©å®¹ä¸­ä½¿ç”¨çš„节点组扩展器的类型。å¯ç”¨å€¼ï¼š`["random","most-pods","least-waste","price","priority"]` | +| ignore-daemonsets-utilization | false | CA 为了缩容而计算资æºåˆ©ç”¨çŽ‡æ—¶ï¼Œæ˜¯å¦åº”忽略 DaemonSet pod | +| ignore-mirror-pods-utilization | false | CA 为了缩容而计算资æºåˆ©ç”¨çŽ‡æ—¶ï¼Œæ˜¯å¦åº”忽略 Mirror pod | +| write-status-configmap | true | CA 是å¦åº”该将状æ€ä¿¡æ¯å†™å…¥ configmap | +| max-inactivity | "10m" | 从上次记录的 autoscaler 活动åŽï¼Œè‡ªåŠ¨é‡å¯ä¹‹å‰çš„最长时间 | +| max-failing-time | "15m" | 从上次记录的 autoscaler æˆåŠŸè¿è¡ŒåŽï¼Œè‡ªåŠ¨é‡å¯ä¹‹å‰çš„最长时间 | +| balance-similar-node-groups | false | 检测相似的节点组，并å‡è¡¡å®ƒä»¬çš„èŠ‚ç‚¹æ•°é‡ | +| node-autoprovisioning-enabled | false | CA 是å¦åº”在需è¦æ—¶è‡ªåŠ¨é…置节点组 | +| max-autoprovisioned-node-group-count | 15 | 集群中自动é…ç½®ç»„çš„æœ€å¤§æ•°é‡ | +| unremovable-node-recheck-timeout | "5m" | 在å†æ¬¡æ£€æŸ¥æ— æ³•åˆ é™¤çš„节点之å‰ï¼ŒèŠ‚点的超时时间 | +| expendable-pods-priority-cutoff | -10 | 优先级低于 cutoff çš„ Pod 将是消耗性 pod。这些 pod å¯ä»¥åœ¨ç¼©å®¹æœŸé—´ä¸åŠ è€ƒè™‘地被终止，并且ä¸ä¼šå¯¼è‡´æ‰©å®¹ã€‚优先级是 null（ç¦ç”¨ PodPriority）的 pod ä¸æ˜¯æ¶ˆè€—性的 | +| regional | false | 集群是区域性的 | +| new-pod-scale-up-delay | "0s" | 生命短于这个值的 Pod å°†ä¸è€ƒè™‘扩容 | +| ignore-taint | - | 在扩缩容节点组时，指定在节点模æ¿ä¸­è¦å¿½ç•¥çš„污点 | +| balancing-ignore-label | - | 在比较两个节点组是å¦ç›¸ä¼¼æ—¶ï¼ŒæŒ‡å®šè¦å¿½ç•¥çš„标签（基本标签集和云æ供商标签集除外） | +| aws-use-static-instance-list | false | CA 在è¿è¡Œæ—¶è¿˜æ˜¯ä½¿ç”¨é™æ€åˆ—表获å–实例类型。仅适用于 AWS | +| profiling | false | 是å¦å¯ç”¨äº† debug/pprof 端点 | + +#### 部署 + +基于 [cluster-autoscaler-run-on-control-plane.yaml](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml) 示例，我们已ç»åˆ›å»ºäº†è‡ªå·±çš„ `cluster-autoscaler-deployment.yaml` 以使用首选的 [auto-discovery 设置](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#auto-discovery-setup)，更新容å¿åº¦ã€nodeSelectorã€é•œåƒç‰ˆæœ¬å’Œå‘½ä»¤é…置： + + +```yml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/controlplane + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + containers: + - image: eu.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler: + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + +``` + +准备好清å•æ–‡ä»¶åŽï¼Œå°†è¯¥æ–‡ä»¶éƒ¨ç½²åˆ° Kubernetes 集群中（也å¯ä»¥ä½¿ç”¨ Rancher UI 进行æ“作）： + +```sh +kubectl -n kube-system apply -f cluster-autoscaler-deployment.yaml +``` + +:::note + +你也å¯ä»¥é€šè¿‡[手动é…ç½®](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#manual-configuration)æ¥è®¾ç½® Cluster-autoscaler deployment。 + +::: + +## 测试 + +此时，cluster-autoscaler 应该已ç»åœ¨ Rancher 自定义集群中å¯åŠ¨å¹¶è¿è¡Œã€‚当满足以下æ¡ä»¶ä¹‹ä¸€æ—¶ï¼Œcluster-autoscaler 需è¦ç®¡ç† `K8sWorkerAsg` ASG，以在 2 到 10 个节点之间进行扩缩容： + +* 集群中有 Pod 因资æºä¸è¶³è€Œæ— æ³•è¿è¡Œã€‚在这ç§æƒ…况下，集群被扩容。 +* 集群中有一些节点长时间未得到充分利用，而且它们的 Pod å¯ä»¥æ”¾åˆ°å…¶ä»–现有节点上。在这ç§æƒ…况下，集群被缩容。 + +### 生æˆè´Ÿè½½ + +为了在 Kubernetes 集群上产生负载并查看 cluster-autoscaler 是å¦æ­£å¸¸å·¥ä½œï¼Œæˆ‘们准备了一个 `test-deployment.yaml`。`test-deployment` 通过三个副本请求 1000m CPU å’Œ 1024Mi 内存。通过调整请求的资æºå’Œ/或副本以确ä¿è€—å°½ Kubernetes 集群资æºï¼š + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hello-world + name: hello-world +spec: + replicas: 3 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + ports: + - containerPort: 80 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1024Mi + requests: + cpu: 1000m + memory: 1024Mi +``` + +准备好 test deployment åŽï¼Œå°†å…¶éƒ¨ç½²åœ¨ Kubernetes 集群的默认命å空间中（å¯ä»¥ä½¿ç”¨ Rancher UI）： + +``` +kubectl -n default apply -f test-deployment.yaml +``` + +### 检查扩缩容 + +Kubernetes 资æºè€—å°½åŽï¼Œcluster-autoscaler 应该扩容无法调度 pod çš„ worker 节点。它应该进行扩容，直到所有 pod 都能被调度。你应该会在 ASG å’Œ Kubernetes 集群上看到新节点。检查 `kube-system` cluster-autoscaler pod 上的日志。 + +检查完扩容åŽï¼Œæˆ‘们开始检查缩容。为此，请å‡å°‘ test deployment 的副本数，直到你能释放足够的 Kubernetes 集群资æºä»¥è¿›è¡Œç¼©å®¹ã€‚你应该能看到 ASG å’Œ Kubernetes 集群上的节点消失了。检查 `kube-system` cluster-autoscaler pod 上的日志。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md new file mode 100644 index 00000000000..9cb5a041693 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md @@ -0,0 +1,130 @@ +--- +title: é›†ç¾¤æ¨¡æ¿ +--- + +集群模æ¿åŒ…å« Kubernetes é…置和节点池é…​​置，å…许å•ä¸ªæ¨¡æ¿åŒ…å« Rancher 在云æ供商中é…置新节点并在这些节点上安装 Kubernetes 所需的所有信æ¯ã€‚ + +## 概述 + +集群模æ¿ä»¥ Helm Chart çš„å½¢å¼æ供。è¦ä½¿ç”¨é›†ç¾¤æ¨¡æ¿ï¼Œä½ éœ€è¦å…‹éš†å’Œå¤åˆ»æ¨¡æ¿ï¼Œæ ¹æ®ä½ çš„用例更改模æ¿ï¼Œç„¶åŽåœ¨ Rancher 管ç†é›†ç¾¤ä¸Šå®‰è£… Helm Chart。如果 Helm Chart 安装在 Rancher 管ç†é›†ç¾¤ä¸Šï¼Œä¼šåˆ›å»ºä¸€ä¸ªæ–°çš„集群资æºï¼ŒRancher 使用它æ¥é…置新集群。 + +使用模æ¿é…置集群åŽï¼Œå¯¹æ¨¡æ¿çš„任何更改都ä¸ä¼šå½±å“集群。使用集群模æ¿åˆ›å»ºé›†ç¾¤åŽï¼Œé›†ç¾¤çš„é…置和基础设施å¯èƒ½ä¼šå‘生å˜åŒ–，因为集群模æ¿æ²¡æœ‰ä»»ä½•æ‰§è¡Œé™åˆ¶ã€‚ + +### Kubernetes å‘行版 + +集群模æ¿å¯ä»¥ä½¿ç”¨ä»»ä½• Kubernetes å‘行版。现在，我们æ供一个带有 RKE2 Kubernetes 集群的示例。将æ¥æˆ‘们å¯èƒ½ä¼šæ供更多使用其他 Kubernetes å‘行版的示例。 + +### 版本控制 + +Rancher ä¸ç®¡ç†é›†ç¾¤æ¨¡æ¿çš„版本控制。版本控制在包å«æ¨¡æ¿çš„ Helm Chart 的仓库中处ç†ã€‚ + +## RKE2 é›†ç¾¤æ¨¡æ¿ + +请å‚阅 [RKE2 集群模æ¿](https://github.com/rancher/cluster-template-examples)的示例仓库。 + +## å‘ Rancher æ·»åŠ é›†ç¾¤æ¨¡æ¿ + +在本节中，你将学习如何将集群模æ¿æ·»åŠ åˆ° `local` 集群的 Chart 仓库列表中。这样，当用户安装新的 Kubernetes 集群时，Rancher å°†æ供集群模æ¿çš„选项。 + +:::note 先决æ¡ä»¶ï¼š + +- 你需è¦æœ‰åœ¨`本地` Kubernetes 集群上安装 Helm Chart çš„æƒé™ã€‚ +- 为了使 Chart 以创建新集群的形å¼å‡ºçŽ°ï¼ŒChart 必须具有注释 `catalog.cattle.io/type: cluster-template`。 + +::: + +1. 转到集群模æ¿ç¤ºä¾‹ä»“库。Rancher 的示例ä½äºŽ[æ­¤ GitHub 仓库中](https://github.com/rancher/cluster-template-examples)。 +1. å¤åˆ»ä»“库。 +1. å¯é€‰ï¼šé€šè¿‡ç¼–辑 `values.yaml` 文件æ¥ç¼–辑集群选项。有关编辑文件的帮助，请å‚阅集群模æ¿çš„ Helm Chart 自述文件。 +1. å°† Chart 仓库添加到 Rancher。点击 **☰ > 集群管ç†**。 +1. 转到 `local` 集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Chart Repositories**。 +1. 点击**创建**。 +1. 输入集群模æ¿ä»“库的å称。 +1. 点击**åŒ…å« Helm Chart 定义的 Git 仓库**。 +1. 在 **Git 仓库 URL**字段，输入仓库的 URL，例如，`https://github.com/rancher/cluster-template-examples.git`。 +1. 在 **Git 分支**字段，输入è¦ç”¨ä½œæ¨¡æ¿æºçš„分支。Rancher 的示例仓库使用了 `main` 分支。 +1. 点击**创建**。 + +**结果**：集群模æ¿å¯ä»Ž Rancher `local` 集群中的 **Apps** 获得。现在，你å¯ä»¥ä½¿ç”¨æ¨¡æ¿æ¥éƒ¨ç½²é›†ç¾¤ã€‚ + +:::note å—é™ç®¡ç†å‘˜è®¿é—®ï¼š + +如果你是å—é™ç®¡ç†å‘˜å¹¶ä¸”æ— æƒè®¿é—® `local` 集群，你ä»ç„¶å¯ä»¥æ·»åŠ æ–°çš„ RKE2 模æ¿å¹¶ç®¡ç†é›†ç¾¤ä»“库。è¦å¯¼èˆªåˆ° Chart 仓库，请在左侧导航æ å•å‡» **☰ > é›†ç¾¤ç®¡ç† > 高级选项 > 仓库**。你将绕过上述步骤 1 - 6，然åŽç»§ç»­æŒ‰ç…§æ­¥éª¤ 7 - 12 创建集群模æ¿ã€‚ + +::: + +## 使用集群模æ¿åˆ›å»ºé›†ç¾¤ + +:::note 先决æ¡ä»¶ï¼š + +- 你需è¦èŽ·å¾—é…置新 Kubernetes 集群的æƒé™ã€‚ +- 你将需è¦äº‘凭è¯æ¥ä½¿ç”¨æ¨¡æ¿é…置基础设施。 +- 为了以创建新集群的形å¼æ˜¾ç¤ºï¼Œé›†ç¾¤æ¨¡æ¿çš„ Helm Chart 必须具有 `catalog.cattle.io/type: cluster-template` 注释。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. å•å‡»é›†ç¾¤æ¨¡æ¿çš„å称。 +1. å®Œæˆ Helm Chart 的安装。 + +**结果**：在 Rancher é…置新集群åŽï¼Œä½ å¯ä»¥åƒç®¡ç†å…¶ä»– Rancher å¯åŠ¨çš„ Kubernetes 集群一样管ç†è¯¥æ–°é›†ç¾¤ã€‚如果集群模æ¿æœ‰å¯ä¾›ç”¨æˆ·é€‰æ‹©çš„选项，你å¯ä»¥é€šè¿‡ UI é…置这些选项。 + +## 更新使用集群模æ¿åˆ›å»ºçš„集群 + +如果集群所用的模æ¿æœ‰äº†æ–°ç‰ˆæœ¬ï¼Œä½ å¯ä»¥ä½¿ç”¨ **Apps > Installed Apps** 页é¢ä¸­çš„模æ¿æ›´æ–°é›†ç¾¤ã€‚ + +## 使用 Fleet 从模æ¿éƒ¨ç½²é›†ç¾¤ + +:::note 先决æ¡ä»¶ï¼š + +- 你需è¦èŽ·å¾—é…置新 Kubernetes 集群的æƒé™ã€‚ +- 你将需è¦äº‘凭è¯æ¥ä½¿ç”¨æ¨¡æ¿é…置基础设施。 +- 为了以创建新集群的形å¼æ˜¾ç¤ºï¼Œé›†ç¾¤æ¨¡æ¿çš„ Helm Chart 必须具有 `catalog.cattle.io/type:cluster-template` 注释。 +- 为了将模æ¿ç”¨ä½œæŒç»­äº¤ä»˜/GitOps 的一部分，集群模æ¿éœ€è¦éƒ¨ç½²åœ¨`本地`集群的 `fleet-local` 命å空间中。 +- 所有值都必须在模æ¿çš„ `values.yaml` 中设置。 +- Fleet 仓库必须éµå¾ªæ­¤å¤„çš„[指å—](https://fleet.rancher.io/gitrepo-content)。对于 RKE2 集群模æ¿ï¼Œåˆ™å¿…须把 `fleet.yaml` 文件添加到仓库。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. 点击**使用模æ¿åˆ›å»ºé›†ç¾¤**。 + +**结果**：Rancher é…置好新集群åŽï¼Œé›†ç¾¤ç”± Fleet 管ç†ã€‚ + +## å¸è½½é›†ç¾¤æ¨¡æ¿ + +1. 点击 **☰ > 集群管ç†**。 +1. å‰å¾€ `local` 集群并å•å‡» **Apps > Chart Repositories**。 +1. 转到集群模æ¿çš„ Chart 仓库，然åŽå•å‡» **â‹® > 删除。** +1. 确认删除。 + +**结果**：集群模æ¿å·²å¸è½½ã€‚æ­¤æ“作ä¸ä¼šå½±å“使用集群模æ¿åˆ›å»ºçš„现有集群。 + +能够访问 `local` 集群的管ç†å‘˜è¿˜å¯ä»¥å‰å¾€ **Apps > Installed Apps** 页é¢ï¼Œé€šè¿‡é›†ç¾¤æ¨¡æ¿æ¥ç§»é™¤å·²éƒ¨ç½²çš„集群。 + +## é…置选项 + +集群模æ¿éžå¸¸çµæ´»ï¼Œå¯ç”¨äºŽé…置以下所有选项： + +- 节点é…ç½® +- 节点池 +- é¢„å…ˆæŒ‡å®šçš„äº‘å‡­è¯ +- å¯ç”¨/é…置授æƒçš„集群端点，以在ä¸ä½¿ç”¨ Rancher 作为代ç†çš„情况下获得对集群的 kubectl 访问æƒé™ +- 安装 Rancher V2 monitoring +- Kubernetes 版本 +- 分é…集群æˆå‘˜ +- é…置基础设施，例如 AWS VPC/å­ç½‘或 vSphere æ•°æ®ä¸­å¿ƒ +- 云æ供商选项 +- Pod 安全选项 +- 网络æ供商 +- Ingress Controller +- 网络安全é…ç½® +- 网络æ’件 +- ç§æœ‰é•œåƒä»“库 URL å’Œå‡­è¯ +- 附加组件 +- Kubernetes 选项，包括 kube-apiã€kube-controllerã€kubelet å’ŒæœåŠ¡ç­‰ Kubernetes 组件的é…ç½® + +有关如何é…置模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅集群模æ¿çš„ Helm Chart README 文件。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md new file mode 100644 index 00000000000..a942534d1c6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md @@ -0,0 +1,192 @@ +--- +title: 节点和节点池 +--- + +在 Rancher 中å¯åŠ¨ Kubernetes 集群åŽï¼Œä½ å¯ä»¥ä»Žé›†ç¾¤çš„**节点**选项å¡ç®¡ç†å„个节点。 + +1. 点击左上角的 **☰**。 +1. 选择**集群管ç†**。 +1. 找到è¦ç®¡ç†å…¶èŠ‚点的集群，然åŽå•å‡»è¡Œæœ«å°¾çš„**æµè§ˆ**按钮。 +1. 从左侧导航中选择**节点**。 + +ä¸åŒçš„集群é…ç½®[选项](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)对应ä¸åŒçš„å¯ç”¨èŠ‚点选项。 + +:::note + +å¦‚æžœä½ æƒ³ç®¡ç† _集群_ 而ä¸æ˜¯å•ä¸ªèŠ‚点，请å‚阅[编辑集群](../../../pages-for-subheaders/cluster-configuration.md)。 + +::: + + +## æ¯ä¸ªé›†ç¾¤åˆ›å»ºé€‰é¡¹çš„å¯ç”¨èŠ‚点选项 + +下表列出了 Rancher 中æ¯ç§é›†ç¾¤ç±»åž‹çš„å¯ç”¨èŠ‚点选项。å•å‡»**选项**列中的链接以获å–æ¯ä¸ªåŠŸèƒ½çš„更多信æ¯ã€‚ + +| 选项 | [由基础设施æ供商托管的节点][1] | [自定义节点][2] | [托管集群][3] | [注册的 EKS 节点][4] | [所有其他注册节点][5] | æè¿° | +| ------------------------------------------------ | ------------------------------------------------ | ---------------- | ------------------- | ------------------- | -------------------| ------------------------------------------------------------------ | +| [å°é”](#å°é”节点) | ✓ | ✓ | ✓ | ✓ | ✓ | 将节点标记为ä¸å¯è°ƒåº¦ã€‚ | +| [清空](#清空节点) | ✓ | ✓ | ✓ | ✓ | ✓ | 将节点标记为ä¸å¯è°ƒåº¦ _并且_ 驱é€æ‰€æœ‰ pod。 | +| [编辑](#管ç†å’Œç¼–辑å•ä¸ªèŠ‚点) | ✓ | ✓ | ✓ | ✓ | ✓ | 输入节点的自定义å称ã€æè¿°ã€æ ‡ç­¾æˆ–污点。 | +| [查看 API](#在-rancher-api-中查看节点) | ✓ | ✓ | ✓ | ✓ | ✓ | 查看 API æ•°æ®ã€‚ | +| [删除](#删除节点) | ✓ | ✓ | | * | * | 从集群中删除有缺陷的节点。 | +| [下载密钥](#通过-ssh-连接到由基础设施æ供商托管的节点) | ✓ | | | | | 下载 SSH 密钥以通过 SSH 连接到节点。 | +| [节点缩放](#扩缩节点) | ✓ | | | ✓ | | å‘上或å‘下扩展节点池中的节点数。 | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +[5]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +\* å¯é€šè¿‡ View API 访问的删除选项 + + +### 由基础设施æ供商托管的节点 + +在[托管在基础设施æ供商](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)中的节点上é…置由 Rancher å¯åŠ¨çš„ Kubernetes 集群时，你å¯ä»¥ä½¿ç”¨èŠ‚点池。 + +如果节点池被编辑，通过[节点池选项](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)é…置的集群å¯ä»¥çºµå‘扩容或缩容。 + +如果å¯ç”¨[节点自动替æ¢åŠŸèƒ½](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点自动替æ¢)，节点池还å¯ä»¥è‡ªåŠ¨ç»´æŠ¤åœ¨åˆå§‹é›†ç¾¤é…置期间设置的节点规模。该规模决定了 Rancher 为集群维护的 active 节点的数é‡ã€‚ + +Rancher 使用[节点模æ¿](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)æ¥æ›¿æ¢èŠ‚点池中的节点。æ¯ä¸ªèŠ‚点模æ¿éƒ½ä½¿ç”¨äº‘æ供商凭è¯æ¥å…许 Rancher 在基础设施æ供商中设置节点。 + +### 由托管 Kubernetes æ供商é…置的节点 + +用于管ç†[ç”± Kubernetes æ供商托管](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)的节点的选项在 Rancher 中有些é™åˆ¶ã€‚例如，你ä¸èƒ½ä½¿ç”¨ Rancher UI å‘上或å‘下缩放节点数é‡ï¼Œè€Œæ˜¯éœ€è¦ç›´æŽ¥ç¼–辑集群。 + +### 注册节点 + +虽然你å¯ä»¥ä½¿ç”¨ Rancher 将工作负载部署到[已注册集群](../../new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)，但你无法管ç†å•ä¸ªé›†ç¾¤èŠ‚点。导入的集群节点的所有管ç†éƒ½å¿…须在 Rancher 之外进行。 + +## 管ç†å’Œç¼–辑å•ä¸ªèŠ‚点 + +编辑节点å¯è®©ä½ ï¼š + +* 更改其å称 +* 更改其æè¿° +* 添加[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +* 添加/删除[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + +è¦ç®¡ç†å•ä¸ªèŠ‚点，请æµè§ˆåˆ°è¦ç®¡ç†çš„集群，然åŽä»Žä¸»èœå•ä¸­é€‰æ‹©**节点**。你å¯ä»¥é€šè¿‡å•å‡»èŠ‚点的 **â‹®** (**...**) 图标æ¥æ‰“开选项èœå•ã€‚ + +## 在 Rancher API 中查看节点 + +选择此选项以查看节点的 [API 端点](../../../pages-for-subheaders/about-the-api.md)。 + +## 删除节点 + +使用 **Delete** 从云æ供商中删除有缺陷的节点。 + +当你删除有缺陷的节点时，如果该节点在节点池中并å¯ç”¨äº†[节点自动替æ¢](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点自动替æ¢)，Rancher å¯ä»¥è‡ªåŠ¨å°†å…¶æ›¿æ¢ä¸ºå…·æœ‰ç›¸åŒé…置的节点。 + +:::tip + +如果你的集群由基础设施æ供商托管，并且你希望缩容集群，而ä¸æ˜¯åˆ é™¤æœ‰ç¼ºé™·çš„节点，请进行[缩容](#扩缩节点)而ä¸æ˜¯åˆ é™¤ã€‚ + +::: + +## 扩缩节点 + +对于由基础设施æ供商托管的节点，你å¯ä»¥ä½¿ç”¨ç¼©æ”¾æŽ§ä»¶æ¥ç¼©æ”¾æ¯ä¸ª[节点池](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)中的节点数é‡ã€‚此选项ä¸é€‚用于其他集群类型。 + +## 通过 SSH 连接到由基础设施æ供商托管的节点 + +对于[由基础设施æ供商托管的节点](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)，你å¯ä»¥é€‰æ‹©ä¸‹è½½å…¶ SSH 密钥，以便从桌é¢è¿œç¨‹è¿žæŽ¥åˆ°å®ƒã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦é€šè¿‡ SSH 连接到节点的集群，然åŽå•å‡»é›†ç¾¤å称。 +1. 在**主机池**选项å¡ä¸Šï¼Œæ‰¾åˆ°è¦è¿œç¨‹è®¿é—®çš„节点，然åŽå•å‡» **â‹® > 下载 SSH 密钥**。这将下载用于 SSH 的文件的 ZIP 包。 +1. å°† ZIP 文件解压缩到任何ä½ç½®ã€‚ +1. 打开终端。转到解压了的 ZIP 文件的ä½ç½®ã€‚ +1. 输入以下命令： + + ``` + ssh -i id_rsa root@ + ``` + +## å°é”节点 + +_å°é”_ 节点表示将节点标记为ä¸å¯è°ƒåº¦ã€‚此功能适用于在短期维护（如é‡å¯ï¼Œå‡çº§æˆ–åœç”¨ï¼‰æ—¶åœ¨èŠ‚点上执行短期任务。完æˆåŽï¼Œé‡æ–°æ‰“开电æºå¹¶é€šè¿‡å–消å°é”使节点å†æ¬¡å¯è°ƒåº¦ã€‚ + +## 清空节点 + +_清空_ 指的是先å°é”节点，然åŽé©±é€å…¶æ‰€æœ‰ pod 的过程。此功能对于执行节点维护（如内核å‡çº§æˆ–硬件维护）很有用。它å¯ä»¥é˜²æ­¢æ–°çš„ pod 部署到节点，åŒæ—¶èƒ½é‡æ–°åˆ†é…现有的 pod，从而é¿å…用户é‡åˆ°æœåŠ¡ä¸­æ–­çš„情况。 + +- 具有副本集的 pod 将被一个新的 pod 替æ¢ï¼Œè¯¥æ–° pod 将被调度到一个新节点。此外，如果 pod 是æœåŠ¡çš„一部分，则客户端将自动é‡å®šå‘到新的 pod。 + +- 对于没有副本集的 pod，你需è¦è°ƒå‡º pod 的新副本。如果该 pod ä¸æ˜¯æœåŠ¡çš„一部分，将客户端é‡å®šå‘到它。 + +ä½ å¯ä»¥æ¸…空处于 `cordoned` 或 `active` 状æ€çš„节点。清空一个节点时，该节点会被å°é”，然åŽä¼šè¯„估节点是å¦æ»¡è¶³æ¸…空的必备è¦æ±‚，如果满足è¦æ±‚，则会驱é€èŠ‚点的 pod。 + +但是，你å¯ä»¥åœ¨å¯åŠ¨æ¸…空时覆盖清空æ¡ä»¶ã€‚你还有机会设置宽é™æœŸå’Œè¶…时值。 + +### 激进和安全的清空选项 + +为集群é…ç½®å‡çº§ç­–略时，你将能够å¯ç”¨èŠ‚点清空。如果å¯ç”¨äº†èŠ‚点清空，你将能够é…置如何删除和é‡æ–°è°ƒåº¦ pod。 + +- **激进模å¼** + + 在这ç§æ¨¡å¼ä¸‹ï¼Œå³ä½¿ pod 没有控制器也ä¸ä¼šè¢«é‡æ–°è°ƒåº¦åˆ°æ–°èŠ‚点。Kubernetes 会认为你拥有自己的 pod 删除逻辑。 + + Kubernetes 还会认为你有处ç†ä½¿ç”¨ emptyDir çš„ pod 的实现。如果 Pod 使用 emptyDir 存储本地数æ®ï¼Œä½ å¯èƒ½æ— æ³•å®‰å…¨åœ°åˆ é™¤è¯¥ pod，因为一旦 Pod 从节点中删除，emptyDir 中的数æ®å°†è¢«åˆ é™¤ã€‚选择激进模å¼å°†åˆ é™¤è¿™äº› pod。 + +- **安全模å¼** + + 如果一个节点有独立的 pod 或临时数æ®ï¼Œå®ƒå°†è¢«å°é”但ä¸ä¼šè¢«æ¸…空。 +### 宽é™æœŸ + +ç»™æ¯ä¸ª pod 清ç†çš„超时时间，从而让它们优雅地退出。例如，pod å¯èƒ½éœ€è¦å®Œæˆä»»ä½•æœªå®Œæˆçš„请求ã€å›žæ»šäº‹åŠ¡æˆ–将状æ€ä¿å­˜åˆ°æŸä¸ªå¤–部存储。如果该值为负数，将使用 pod 中指定的默认值。 + +### 超时 + +在清空放弃之å‰åº”该继续等待的时间。 + +:::note Kubernetes 已知问题： + +Kubernetes 1.12 之å‰çš„版本中，在清空节点时ä¸ä¼šå¼ºåˆ¶æ‰§è¡Œ[超时设置](https://github.com/kubernetes/kubernetes/pull/64378)。 + +::: + +### 清空和å°é”çŠ¶æ€ + +如果有任何与用户输入相关的错误，节点会由于清空失败进入 `cordoned` 状æ€ã€‚ä½ å¯ä»¥æ›´æ­£è¾“入并å†æ¬¡å°è¯•æ¸…空节点，也å¯ä»¥é€šè¿‡è§£å°èŠ‚点æ¥ä¸­æ­¢ã€‚ + +如果清空没有错误，则节点会进入 `draining` 状æ€ã€‚当节点处于此状æ€æ—¶ï¼Œä½ å¯ä»¥é€‰æ‹©åœæ­¢æ¸…空，这将åœæ­¢æ¸…空过程并将节点的状æ€æ›´æ”¹ä¸º `cordoned`。 + +清空æˆåŠŸå®ŒæˆåŽï¼ŒèŠ‚点将处于 `drained` 状æ€ã€‚然åŽä½ å¯ä»¥å…³é—­æˆ–删除节点。 + +有关**å°é”和清空**的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/)。 + +## 标记 Rancher 忽略的节点 + +æŸäº›è§£å†³æ–¹æ¡ˆï¼ˆä¾‹å¦‚ F5 çš„ BIG-IP 集æˆï¼‰å¯èƒ½éœ€è¦åˆ›å»ºä¸€ä¸ªä¸ä¼šæ³¨å†Œåˆ°é›†ç¾¤çš„节点。 + +由于节点永远ä¸ä¼šå®Œæˆæ³¨å†Œï¼Œå› æ­¤å®ƒåœ¨ Rancher UI 中总是显示为ä¸å¥åº·ã€‚ + +在这ç§æƒ…况下，你å¯èƒ½å¸Œæœ›å°†èŠ‚点标记为 Rancher 忽略的节点，从而让 Rancher 仅在节点实际å‘生故障时将节点状æ€æ˜¾ç¤ºä¸ºä¸å¥åº·ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 或 `kubectl` 标记è¦å¿½ç•¥çš„节点。 + +:::note + +已知一个[未解决的问题](https://github.com/rancher/rancher/issues/24172)，å³æ ‡è®°ä¸ºè¢«å¿½ç•¥çš„节点å¯èƒ½ä¼šå¡åœ¨æ›´æ–°çŠ¶æ€ã€‚ + +::: + + +### 使用 kubectl 标记è¦å¿½ç•¥çš„节点 + +è¦æ·»åŠ  Rancher 忽略的节点，请使用 `kubectl` 创建具有以下标签的节点： + +``` +cattle.rancher.io/node-status: ignore +``` + +**结果**：如果将节点添加到集群中，Rancher å°†ä¸ä¼šå°è¯•ä¸Žè¯¥èŠ‚点åŒæ­¥ã€‚该节点ä»ç„¶å¯ä»¥æ˜¯é›†ç¾¤çš„一部分，并且å¯ä»¥ä½¿ç”¨ `kubectl` 列出。 + +如果在将节点添加到集群之å‰æ·»åŠ äº†æ ‡ç­¾ï¼Œåˆ™è¯¥èŠ‚点将ä¸ä¼šæ˜¾ç¤ºåœ¨ Rancher UI 中。 + +如果在将节点添加到 Rancher 集群åŽæ·»åŠ æ ‡ç­¾ï¼Œåˆ™ä¸ä¼šä»Ž UI 中删除该节点。 + +如果你使用 Rancher UI 或 API 从 Rancher server 中删除节点，å‡å¦‚ `nodeName` 在 Rancher API 下的 `v3/settings/ignore-node-name` 中列出，则ä¸ä¼šä»Žé›†ç¾¤ä¸­åˆ é™¤è¯¥èŠ‚点。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md new file mode 100644 index 00000000000..f8e5d94e510 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md @@ -0,0 +1,197 @@ +--- +title: 使用 Rancher 的项目和 Kubernetes 命å空间 +description: Rancher Projects å‡è½»äº†é›†ç¾¤çš„管ç†è´Ÿæ‹…并支æŒå¤šç§Ÿæˆ·ã€‚学习创建项目并将项目划分为 Kubernetes 命å空间 +--- + +命å空间是 Kubernetes 的概念，它å…许你在集群中使用虚拟集群。这对于将集群划分为å•ç‹¬çš„“虚拟集群â€å¾ˆæœ‰ç”¨ï¼Œæ¯ä¸ªâ€œè™šæ‹Ÿé›†ç¾¤â€éƒ½æœ‰è‡ªå·±çš„访问控制和资æºé…é¢ã€‚ + +项目是一组命å空间，是 Rancher 引入的一个概念。项目å…许你将多个命å空间作为一个组进行管ç†ï¼Œå¹¶åœ¨å…¶ä¸­æ‰§è¡Œ Kubernetes æ“作。你å¯ä»¥ä½¿ç”¨é¡¹ç›®æ¥æ”¯æŒå¤šç§Ÿæˆ·ï¼Œä»¥ä¾¿å›¢é˜Ÿå¯ä»¥åœ¨é›†ç¾¤å†…访问项目，而无需访问åŒä¸€é›†ç¾¤ä¸­çš„其他项目。 + +:::note + +UI 上ä¸å†å¼ºè°ƒé¡¹ç›®ï¼Œå› ä¸ºä¸éœ€è¦åœ¨é¡¹ç›®èŒƒå›´å†…创建 Kubernetes 资æºã€‚但是，如果å¯ç”¨äº†æ—§ç‰ˆåŠŸèƒ½å¼€å…³ï¼Œåˆ™ä»å¯ä»¥åœ¨é¡¹ç›®èŒƒå›´å†…创建诸如 [Secrets](../../new-user-guides/kubernetes-resources-setup/secrets.md#在项目中创建密文) 之类的资æºã€‚ + +::: + +本文介ç»é¡¹ç›®å’Œå‘½å空间如何与 Rancher 一起使用。 + +## 关于命å空间 + +命å空间是 Kubernetes 引入的一个概念。详情请å‚è§[命å空间的官方 Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)。 + +:::note + +Kubernetes 支æŒç”±åŒä¸€ä¸ªç‰©ç†é›†ç¾¤æ”¯æŒçš„多个虚拟集群。这些虚拟集群称为命å空间。命å空间适用于多用户分布在多个团队或项目中的环境。如果你的集群åªæœ‰å‡ ä¸ªåˆ°å‡ å个用户，你ä¸éœ€è¦åˆ›å»ºæˆ–考虑使用命å空间。 + +::: + +命å空间æ供以下功能： + +- **为å称æ供范围**：资æºå称在命å空间内必须是唯一的，但跨命å空间时则ä¸éœ€è¦ã€‚命å空间ä¸èƒ½ç›¸äº’嵌套，æ¯ä¸ª Kubernetes 资æºåªèƒ½ä½äºŽä¸€ä¸ªå‘½å空间中。 +- **资æºé…é¢**：命å空间æ供了在多用户之间划分集群资æºçš„方法。 + +ä½ å¯ä»¥åœ¨é¡¹ç›®çº§åˆ«åˆ†é…资æºï¼Œä»¥ä¾¿é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间都å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºã€‚你也å¯ä»¥é€šè¿‡å°†èµ„æºæ˜¾å¼åˆ†é…给命å空间æ¥ç»•è¿‡æ­¤ç»§æ‰¿æ¨¡å¼ã€‚ + +ä½ å¯ä»¥å°†ä»¥ä¸‹èµ„æºç›´æŽ¥åˆ†é…给命å空间： + +- [工作负载](../../../pages-for-subheaders/workloads-and-pods.md) +- [è´Ÿè½½å‡è¡¡å™¨/Ingress](../../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [æœåŠ¡å‘现记录](../../new-user-guides/kubernetes-resources-setup/create-services.md) +- [æŒä¹…å·å£°æ˜Ž](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [è¯ä¹¦](../../new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMap](../../new-user-guides/kubernetes-resources-setup/configmaps.md) +- [é•œåƒä»“库](../../new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [密文](../../new-user-guides/kubernetes-resources-setup/secrets.md) + +为了在 vanilla Kubernetes 集群中管ç†æƒé™ï¼Œé›†ç¾¤ç®¡ç†å‘˜è¦ä¸ºæ¯ä¸ªå‘½å空间é…置基于角色的访问策略。Rancher 在项目级别上分é…用户æƒé™ï¼Œé¡¹ç›®ä¸­çš„命å空间会自动继承项目的æƒé™ã€‚ + +有关创建和移动命å空间的更多信æ¯ï¼Œè¯·å‚阅[命å空间](../../new-user-guides/manage-namespaces.md)。 + +### 命å空间和 kubectl çš„ RBAC 问题 + +由于项目是 Rancher 引入的概念，因此 kubectl ä¸èƒ½å°†å‘½å空间的创建é™åˆ¶åœ¨åˆ›å»ºè€…å¯ä»¥è®¿é—®çš„项目中。 + +è¿™æ„味ç€å½“具有项目范围æƒé™çš„普通用户使用 `kubectl` 创建命å空间时，由于 `kubectl` ä¸éœ€è¦å°†æ–°å‘½å空间é™åˆ¶åœ¨ç‰¹å®šé¡¹ç›®ä¸­ï¼Œåˆ›å»ºçš„命å空间å¯èƒ½æ— æ³•ä½¿ç”¨ã€‚ + +如果你的æƒé™ä»…é™äºŽé¡¹ç›®çº§åˆ«ï¼Œåˆ™æœ€å¥½[通过 Rancher 创建命å空间](../../new-user-guides/manage-namespaces.md)，以确ä¿ä½ æœ‰æƒè®¿é—®è¯¥å‘½å空间。 + +如果普通用户是项目所有者，则该用户将能够在该项目中创建命å空间。Rancher UI 将阻止该用户在他们有æƒè®¿é—®çš„项目之外创建命å空间。 + +## 关于项目 + +在层次结构方é¢ï¼š + +- 集群包å«é¡¹ç›® +- 项目包å«å‘½å空间 + +ä½ å¯ä»¥ä½¿ç”¨é¡¹ç›®æ¥æ”¯æŒå¤šç§Ÿæˆ·ï¼Œä»¥ä¾¿å›¢é˜Ÿå¯ä»¥åœ¨é›†ç¾¤å†…访问项目，而无需访问åŒä¸€é›†ç¾¤ä¸­çš„其他项目。 + +在 Kubernetes 的基础版本中，RBAC 或集群资æºç­‰åŠŸèƒ½è¢«åˆ†é…给了å„个命å空间。项目å…许你让个人或团队åŒæ—¶è®¿é—®å¤šä¸ªå‘½å空间æ¥èŠ‚çœæ—¶é—´ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨é¡¹ç›®æ¥æ‰§è¡Œä»¥ä¸‹æ“作： + +- 将用户分é…到一组命å空间（å³[项目æˆå‘˜èº«ä»½](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)）。 +- 在项目中为用户分é…特定角色。角色å¯ä»¥æ˜¯æ‰€æœ‰è€…ã€æˆå‘˜ã€åªè¯»æˆ–[自定义](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md)。 +- 为项目分é…资æºã€‚ +- åˆ†é… Pod 安全策略。 + +创建集群时，会在其中自动创建两个项目： + +- [Default 项目](#集群的-default-项目) +- [System 项目](#system-项目) + +### 集群的 Default 项目 + +使用 Rancher é…置集群时，Rancher 会自动为集群创建一个 `default` 项目。你å¯ä»¥ä½¿ç”¨è¿™ä¸ªé¡¹ç›®æ¥å¼€å§‹ä½¿ç”¨é›†ç¾¤ã€‚但你始终å¯ä»¥åˆ é™¤è¯¥é¡¹ç›®ï¼Œå¹¶ç”¨å…·æœ‰æ›´å¤šæ述性å称的项目替æ¢å®ƒã€‚ + +如果你åªéœ€è¦ä½¿ç”¨ `default` 命å空间，那么你就ä¸éœ€è¦ Rancher 中 **Default** 之外的其他项目。 + +å¦‚æžœä½ éœ€è¦ **Default** 之外的其他项目，你å¯ä»¥åœ¨ Rancher 中创建更多项目æ¥éš”离命å空间ã€åº”用和资æºã€‚ + +### System 项目 + +故障排除时，你å¯ä»¥æŸ¥çœ‹ `system` 项目æ¥æ£€æŸ¥ Kubernetes 系统中é‡è¦çš„命å空间是å¦æ­£å¸¸ã€‚这是一个易于访问的项目，能让你é¿å…对系统命å空间容器进行é€ä¸ªæŽ’查。 + +è¦æ‰“开这个项目，请打开集群视图并å•å‡»**集群 > 项目/命å空间**。该视图显示了 `system` 项目中的所有命å空间。 + +`system` 项目： + +- 在你é…置集群时自动创建。 +- 列出存在于 `v3/settings/system-namespaces` 中的所有命å空间（如果存在）。 +- å…许你添加更多命å空间，或将项目内的命å空间移动到其他项目。 +- 无法删除，因为它是集群æ“作所必需的。 + +:::note + +在å¯ç”¨äº†é¡¹ç›®ç½‘络隔离的 RKE 集群中，`system` 项目会覆盖项目网络隔离选项，以便项目能与其他项目通信ã€æ”¶é›†æ—¥å¿—和检查å¥åº·çŠ¶æ€ã€‚ + +::: + +## é¡¹ç›®æŽˆæƒ + +普通用户仅在两ç§æƒ…况下会被授æƒè®¿é—®é¡¹ç›®ï¼š + +- 管ç†å‘˜ã€é›†ç¾¤æ‰€æœ‰è€…或集群æˆå‘˜å°†æ™®é€šç”¨æˆ·æ˜¾å¼æ·»åŠ åˆ°é¡¹ç›®çš„**æˆå‘˜**中。 +- 普通用户å¯ä»¥è®¿é—®ä»–们自己创建的项目。 + +## Pod 安全策略 + +Rancher 在 Kubernetes 之上进行了扩展，除了集群级别之外，还å…许在项目级别应用 [Pod 安全策略](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)。但是，最佳实践是在集群级别应用 Pod 安全策略。 + +## 创建项目 + +本节介ç»å¦‚何创建具有å称以åŠå¯é€‰ pod 安全策略ã€æˆå‘˜å’Œèµ„æºé…é¢çš„新项目。 + +1. [命å新项目](#1-命å新项目) +2. [å¯é€‰ï¼šé€‰æ‹© pod 安全策略](#2-å¯é€‰é€‰æ‹©-pod-安全策略) +3. [推è：添加项目æˆå‘˜](#3-推è添加项目æˆå‘˜) +4. [å¯é€‰ï¼šæ·»åŠ èµ„æºé…é¢](#4-å¯é€‰æ·»åŠ èµ„æºé…é¢) + +### 1. 命å新项目 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºé¡¹ç›®çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. å•å‡»**创建项目**。 +1. 输入**项目å称**。 + +### 2. å¯é€‰ï¼šé€‰æ‹© Pod 安全策略 + +此选项仅在 Pod 安全策略已创建时å¯ç”¨ã€‚有关说明，请å‚阅[创建 Pod 安全策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +å°† PSP 分é…给项目将： + +- 覆盖集群的默认 PSP。 +- å°† PSP 应用于项目。 +- å°† PSP 应用到åŽç»­æ·»åŠ åˆ°é¡¹ç›®ä¸­çš„命å空间。 + +### 3. 推è：添加项目æˆå‘˜ + +使用**æˆå‘˜**为其他用户æ供项目访问æƒé™å’Œè§’色。 + +默认情况下，你的用户会被添加为项目的 `Owner`（所有者）。 + +:::note æƒé™è¯´æ˜Žï¼š + +- 如果用户分é…到了项目的`所有者`或`æˆå‘˜`角色，用户会自动继承`命å空间创建`角色。然而，这个角色是 [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole)，这表示角色的范围会延展到集群中的所有项目。因此，对于显å¼åˆ†é…到了项目`所有者`或`æˆå‘˜`角色的用户æ¥è¯´ï¼Œå³ä½¿åªæœ‰`åªè¯»`角色，这些用户也å¯ä»¥åœ¨åˆ†é…给他们的其他项目中创建命å空间。 + +- 默认情况下，Rancher çš„`项目æˆå‘˜`角色继承自 `Kubernetes-edit` 角色，而`项目所有者`角色继承自 `Kubernetes-admin` 角色。因此，`项目æˆå‘˜`å’Œ`项目所有者`角色都能管ç†å‘½å空间，包括创建和删除命å空间。 + +- 选择`自定义`æ¥ç«‹å³åˆ›å»ºè‡ªå®šä¹‰è§’色：[自定义项目角色](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#自定义项目角色)。 + +::: + +è¦æ·»åŠ æˆå‘˜ï¼š + +1. 在**æˆå‘˜**选项å¡ä¸­ï¼Œå•å‡»**添加**。 +1. 在**选择æˆå‘˜**字段中，æœç´¢è¦åˆ†é…项目访问æƒé™çš„用户或组。请注æ„，如果你å¯ç”¨äº†å¤–部身份验è¯ï¼Œåˆ™åªèƒ½æœç´¢ç»„。 +1. 在**项目æƒé™**中选择一个角色。如需更多信æ¯ï¼Œè¯·å‚阅[项目角色文档](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)。 + +### 4. å¯é€‰ï¼šæ·»åŠ èµ„æºé…é¢ + +资æºé…é¢ç”¨äºŽé™åˆ¶é¡¹ç›®ï¼ˆåŠå…¶å‘½å空间）å¯ä»¥ä½¿ç”¨çš„资æºã€‚有关详细信æ¯ï¼Œè¯·å‚阅[资æºé…é¢](projects-and-namespaces.md)。 + +è¦æ·»åŠ èµ„æºé…é¢ï¼š + +1. 在**资æºé…é¢**选项å¡ä¸­ï¼Œå•å‡»**添加资æº**。 +1. 选择一个**资æºç±»åž‹**。有关详细信æ¯ï¼Œè¯·å‚阅[资æºé…é¢](projects-and-namespaces.md)。 +1. 输入**项目é™åˆ¶**å’Œ**命å空间默认é™åˆ¶**的值。 +1. **å¯é€‰**：指定**容器默认资æºé™åˆ¶**，这将应用于项目中å¯åŠ¨çš„所有容器。如果资æºé…é¢è®¾ç½®äº† CPU 或内存é™åˆ¶ï¼Œåˆ™å»ºè®®ä½¿ç”¨è¯¥å‚数。å¯ä»¥åœ¨å•ä¸ªå‘½å空间或容器级别上覆盖它。有关详细信æ¯ï¼Œè¯·å‚阅[容器默认资æºé™åˆ¶](../../../pages-for-subheaders/manage-project-resource-quotas.md)。 +1. å•å‡»**创建**。 + +**结果**：项目已创建。你å¯ä»¥ä»Žé›†ç¾¤çš„**项目/命å空间**视图中查看它。 + +| 字段 | æè¿° | +| ----------------------- | -------------------------------------------------------------------------------------------------------- | +| 项目é™åˆ¶ | 项目的总资æºé™åˆ¶ã€‚ | +| 命å空间默认é™åˆ¶ | æ¯ä¸ªå‘½å空间的默认资æºé™åˆ¶ã€‚æ­¤é™åˆ¶åœ¨åˆ›å»ºæ—¶ä¼šæ²¿ç”¨åˆ°é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间。项目中所有命å空间的é™åˆ¶ä¹‹å’Œä¸åº”超过项目é™åˆ¶ã€‚ | + +## 删除项目 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°é¡¹ç›®è¿žæŽ¥çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦åˆ é™¤çš„项目，点击 **â‹®**。 +1. 选择**删除**。 + +删除项目时，以å‰ä¸Žè¯¥é¡¹ç›®å…³è”的任何命å空间都将ä¿ç•™åœ¨é›†ç¾¤ä¸Šã€‚ä½ å¯ä»¥åœ¨ Rancher UI **项目/命å空间**页é¢çš„**ä¸åœ¨é¡¹ç›®å†…**选项å¡ä¸­æ‰¾åˆ°è¿™äº›å‘½å空间。你å¯ä»¥é€šè¿‡[移动](../manage-namespaces.md#将命å空间移动到å¦ä¸€ä¸ªé¡¹ç›®)æ¥å°†å®ƒä»¬é‡æ–°åˆ†é…给项目。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md new file mode 100644 index 00000000000..569a052d5fe --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md @@ -0,0 +1,75 @@ +--- +title: NFS 存储 +--- + +在将 NFS 存储å·æ’件用于 Rancher deployment 之å‰ï¼Œä½ éœ€è¦é…ç½® NFS æœåŠ¡å™¨ã€‚ + +:::note + +- 如果你已ç»æ‹¥æœ‰ NFS 共享，则无需é…置新的 NFS æœåŠ¡å™¨å³å¯åœ¨ Rancher 中使用 NFS å·æ’件。这样的è¯ï¼Œä½ å¯ä»¥è·³è¿‡æ­¤è¿‡ç¨‹çš„其余部分并直接[添加存储](../../../../pages-for-subheaders/create-kubernetes-persistent-storage.md)。 + +- 此教程演示了如何使用 Ubuntu 设置 NFS æœåŠ¡å™¨ã€‚你也应该能够将这些说明用于其他 Linux å‘行版（例如 Debianã€RHELã€Arch Linux 等）。有关如何使用å¦ä¸€ä¸ª Linux å‘行版创建 NFS æœåŠ¡å™¨çš„官方说明，请å‚阅å‘行版的文档。 + +::: + +:::note 推è： + +è¦ç®€åŒ–管ç†é˜²ç«å¢™è§„则的过程，请使用 NFSv4。 + +::: + +1. 使用远程终端连接，登录到你打算用于 NFS 存储的 Ubuntu æœåŠ¡å™¨ã€‚ + +1. 输入以下命令： + + ``` + sudo apt-get install nfs-kernel-server + ``` + +1. 输入以下命令，设置存储目录以åŠç”¨æˆ·è®¿é—®æƒé™ã€‚如果你想使用ä¸åŒçš„存储目录，请修改该命令： + + ``` + mkdir -p /nfs && chown nobody:nogroup /nfs + ``` + - `-p /nfs` å‚数在根目录下创建一个å为 `nfs` 的目录。 + - `chown nobody:nogroup /nfs` å‚æ•°å…许对存储目录的所有访问。 + +1. 创建 NFS 导出表。此表设置 NFS æœåŠ¡å™¨ä¸Šçš„目录路径，这些路径会暴露给将使用æœåŠ¡å™¨è¿›è¡Œå­˜å‚¨çš„节点。 + + 1. 使用文本编辑器打开 `/etc/exports`。 + 1. 添加你在步骤 3 中创建的 `/nfs` 文件夹的路径以åŠé›†ç¾¤èŠ‚点的 IP 地å€ã€‚为集群中的æ¯ä¸ª IP 地å€æ·»åŠ ä¸€ä¸ªæ¡ç›®ã€‚在æ¯ä¸ªåœ°å€åŠå…¶é™„带的å‚æ•°åŽé¢åŠ ä¸Šä¸€ä¸ªä½œä¸ºåˆ†éš”符的空格： + + ``` + /nfs (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) + ``` + + :::tip + + ä½ å¯ä»¥å°† IP 地å€æ›¿æ¢ä¸ºå­ç½‘，例如，`10.212.50.12/24`。 + + ::: + + 1. 通过输入以下命令更新 NFS 表： + + ``` + exportfs -ra + ``` + +1. 打开 NFS 使用的端å£ã€‚ + + 1. è¦æ‰¾å‡º NFS 正在使用的端å£ï¼Œè¯·è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ``` + rpcinfo -p | grep nfs + ``` + 2. [打开上一个命令输出的端å£](https://help.ubuntu.com/lts/serverguide/firewall.html.en)ã€‚ä¾‹å¦‚ï¼Œä»¥ä¸‹å‘½ä»¤æ‰“å¼€ç«¯å£ 2049： + + ``` + sudo ufw allow 2049 + ``` + +**结果**：已将 NFS æœåŠ¡å™¨é…置你的 Rancher 节点的存储。 + +## åŽç»­æ“作 + +在 Rancher 中，将 NFS æœåŠ¡å™¨æ·»åŠ ä¸ºå­˜å‚¨å·å’Œ/或存储类。添加æœåŠ¡å™¨åŽï¼Œä½ å¯ä»¥å°†å…¶ç”¨äºŽå­˜å‚¨ä»¥è¿›è¡Œéƒ¨ç½²ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md new file mode 100644 index 00000000000..3955afea1f2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md @@ -0,0 +1,15 @@ +--- +title: 在 Amazon EBS 中创建æŒä¹…存储 +--- + +本文介ç»å¦‚何在 EC2 中设置 Amazon çš„ Elastic Block Store。 + +1. 在 EC2 控制å°ä¸­ï¼Œè½¬åˆ°å·¦ä¾§é¢æ¿ä¸­çš„ **ELASTIC BLOCK STORE** 中，然åŽå•å‡» **Volumes**。 +1. å•å‡» **Create Volume**。 +1. å¯é€‰ï¼šé…ç½®å·çš„大å°æˆ–其他选项。你需è¦åœ¨å·è¦æŒ‚载到的实例所在的å¯ç”¨åŒºä¸­åˆ›å»ºå·ã€‚ +1. å•å‡» **Create Volume**。 +1. 点击 **Close**。 + +**结果**：已创建æŒä¹…存储。 + +有关如何在 Rancher 中设置新创建的存储的详细信æ¯ï¼Œè¯·å‚阅[设置现有存储](../../../new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md new file mode 100644 index 00000000000..1fb1c5d5e06 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md @@ -0,0 +1,72 @@ +--- +title: vSphere 存储 +--- + +è¦ä¸ºæœ‰çŠ¶æ€çš„工作负载æä¾› vSphere 存储，我们建议创建一个 vSphereVolume StorageClass。当工作负载通过 PersistentVolumeClaim 请求å·æ—¶ï¼Œè¿™ç§åšæ³•ä¼šåŠ¨æ€è°ƒé… vSphere 存储。 + +为了在 vSphere 中动æ€è°ƒé…存储，必须å¯ç”¨ vSphere æ供商。有关更多信æ¯ï¼Œè¯·å‚阅[树外 vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md) å’Œ[树内 vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)。 + + +### 先决æ¡ä»¶ + +为了在 [Rancher Kubernetes Engine (RKE)](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群中é…ç½® vSphere å·ï¼Œ[vSphere cloud provider](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere) 必须在[集群选项](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)中显å¼å¯ç”¨ã€‚ + +### 创建一个 StorageClass + +:::tip + +你也å¯ä»¥ä½¿ç”¨ `kubectl` 命令行工具æ¥æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[关于æŒä¹…å·çš„ Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 选择è¦ä½¿ç”¨ vSphere 存储的集群，然åŽå•å‡»**æµè§ˆ**。 +1. 在左侧导航æ ä¸­ï¼Œé€‰æ‹©**存储 > StorageClass**。 +1. å•å‡»**创建**。 +3. 输入存储类的**å称**。 +4. 在 **Provisioner**下，选择 **VMWare vSphere å·**。 + + ![](/img/vsphere-storage-class.png) + +5. å¯é€‰åœ°ï¼Œä½ å¯ä»¥åœ¨**å‚æ•°**下指定存储类的其他属性。有关详细信æ¯ï¼Œè¯·å‚阅 [vSphere 存储文档](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/blob/master/documentation/storageclass.md)。 +5. å•å‡»**创建**。 + +### 创建使用 vSphere å·çš„工作负载 + +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡»**创建**。 +1. å•å‡» **StatefulSet**。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. 输入æŒä¹…å·å称。 +1. 在**存储类**字段中，选择你创建的 vSphere 存储类。 +6. 输入å·æ‰€éœ€çš„**容é‡**。然åŽå•å‡»**定义**。 +7. 在**挂载点**字段中指定路径。这是å·å°†å®‰è£…在容器文件系统中的完整路径，例如 `/persistent`。 +8. å•å‡»**创建**。 + +### 验è¯å·çš„æŒä¹…性 + +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载 > Pod**。 +1. 转到你刚刚创建的工作负载，然åŽå•å‡» **â‹® > 执行命令行**。 +2. 请注æ„å·å·²æŒ‚载到的根目录下的目录（在本例中为 `/persistent`）。 +3. 通过执行命令 `touch //data.txt` 在å·ä¸­åˆ›å»ºä¸€ä¸ªæ–‡ä»¶ã€‚ +4. 关闭 shell 窗å£ã€‚ +5. å•å‡»å·¥ä½œè´Ÿè½½çš„å称以显示详细信æ¯ã€‚ +7. å•å‡» **â‹® > 删除**。 +8. 观察 pod 是å¦è¢«åˆ é™¤ã€‚为了让工作负载维æŒåœ¨æ‰€é…置的å•ä¸ªæœ‰çŠ¶æ€ pod 的规模，之åŽä¼šè°ƒåº¦ä¸€ä¸ªæ–°çš„ Pod æ¥æ›¿æ¢è¯¥ Pod。 +9. 替æ¢çš„ pod è¿è¡ŒåŽï¼Œå•å‡»**执行命令行**。 +10. 输入 `ls -l /` 以检查å·æ‰€æŒ‚载的目录的内容。请注æ„，你之å‰åˆ›å»ºçš„文件ä»ç„¶å­˜åœ¨ã€‚ + + ![workload-persistent-data](/img/workload-persistent-data.png) + +### 为什么使用 StatefulSet 替代 Deployment + +对于消耗 vSphere 存储的工作负载，你应该始终使用 [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)，因为这ç§èµ„æºç±»åž‹æ—¨åœ¨è§£å†³ VMDK å—存储警告。 + +由于 vSphere å·ç”± VMDK å—存储支æŒï¼Œå› æ­¤å®ƒä»¬ä»…æ”¯æŒ `ReadWriteOnce` çš„[访问模å¼](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)。此设置é™åˆ¶å·ï¼Œä½¿å…¶ä¸€æ¬¡åªèƒ½æŒ‚载到一个 pod（除éžä½¿ç”¨è¯¥å·çš„所有 pod ä½äºŽåŒä¸€èŠ‚点上）。如果 deployment 资æºæ¶ˆè€— vSphere å·ï¼Œåˆ™ deployment 资æºæ— æ³•ç”¨äºŽæ‰©å±•åˆ°è¶…出å•ä¸ªå‰¯æœ¬ã€‚ + +å³ä½¿ä½¿ç”¨ä»…具有å•ä¸ªå‰¯æœ¬çš„ deployment 资æºä¹Ÿå¯èƒ½åœ¨æ›´æ–° deployment 时出现死é”情况。如果更新的 pod 被调度到ä¸åŒçš„节点，由于 VMDK ä»ç„¶è¿žæŽ¥åˆ°å¦ä¸€ä¸ªèŠ‚点，因此 pod 将无法å¯åŠ¨ã€‚ + +### 相关链接 + +- [用于 Kubernetes çš„ vSphere 存储](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/tree/master/documentation) +- [Kubernetes æŒä¹…å·](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md new file mode 100644 index 00000000000..62550604077 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -0,0 +1,78 @@ +--- +title: è¯ä¹¦è½®æ¢ +--- + +:::caution + +è½®æ¢ Kubernetes è¯ä¹¦å¯èƒ½ä¼šå¯¼è‡´é›†ç¾¤åœ¨ç»„件é‡å¯æ—¶æš‚æ—¶ä¸å¯ç”¨ã€‚对于生产环境，建议在维护时段内执行此æ“作。 + +::: + +默认情况下，Kubernetes 集群需è¦è¯ä¹¦ï¼ŒRancher å¯åŠ¨çš„ Kubernetes 集群会自动为 Kubernetes 组件生æˆè¯ä¹¦ã€‚在è¯ä¹¦è¿‡æœŸä¹‹å‰æˆ–被泄露åŽè½®æ¢è¯ä¹¦éžå¸¸é‡è¦ã€‚è¯ä¹¦è½®æ¢åŽï¼ŒKubernetes 组件会自动é‡å¯ã€‚ + +å¯ä»¥ä¸ºä»¥ä¸‹æœåŠ¡è½®æ¢è¯ä¹¦ï¼š + + + + +- etcd +- kubelet（节点è¯ä¹¦ï¼‰ +- kubelet（æœåŠ¡è¯ä¹¦ï¼Œå¦‚æžœ[å¯ç”¨](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)） +- kube-apiserver +- kube-proxy +- kube-scheduler +- kube-controller-manager + + + + +- admin +- api-server +- controller-manager +- scheduler +- rke2-controller +- rke2-server +- cloud-controller +- etcd +- auth-proxy +- kubelet +- kube-proxy + + + + +:::note + +å¦‚æžœä½ æœªè½®æ¢ webhook è¯ä¹¦ï¼Œä¸”è¯ä¹¦ç”¨äº†ä¸€å¹´åŽå·²ç»è¿‡æœŸï¼Œè¯·å‚阅此[页é¢](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md)。 + +::: + +### è¯ä¹¦è½®æ¢ + +Rancher å¯åŠ¨çš„ Kubernetes 集群能够通过 UI è½®æ¢è‡ªåŠ¨ç”Ÿæˆçš„è¯ä¹¦ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è½®æ¢è¯ä¹¦çš„集群，然åŽå•å‡» **â‹® > è½®æ¢è¯ä¹¦**。 +1. 选择è¦è½®æ¢çš„è¯ä¹¦ã€‚ + + * è½®æ¢æ‰€æœ‰æœåŠ¡è¯ä¹¦ï¼ˆä½¿ç”¨ç›¸åŒçš„ CA） + * è½®æ¢å•ä¸ªæœåŠ¡ï¼Œå¹¶ä»Žä¸‹æ‹‰èœå•ä¸­é€‰æ‹©å…¶ä¸­ä¸€é¡¹æœåŠ¡ã€‚ + +1. å•å‡»**ä¿å­˜**。 + +**结果**：将轮æ¢æ‰€é€‰è¯ä¹¦ï¼Œç›¸å…³æœåŠ¡å°†é‡æ–°å¯åŠ¨ä»¥ä½¿ç”¨æ–°è¯ä¹¦ã€‚ + +### 补充说明 + + + + +虽然 RKE CLI å¯ä»¥ä¸º Kubernetes 集群组件使用自定义è¯ä¹¦ï¼Œä½† Rancher ç›®å‰ä¸å…许在 Rancher å¯åŠ¨çš„ Kubernetes 集群中上传这些è¯ä¹¦ã€‚ + + + + +在 RKE2 中，etcd å’Œ controlplane 节点都被视为相åŒçš„ `server`。因此，如果你轮æ¢å…¶ä¸­ä¸€ä¸ªç»„件的æœåŠ¡è¯ä¹¦ï¼Œåˆ™ä¸¤è€…çš„è¯ä¹¦éƒ½ä¼šè¢«è½®æ¢ã€‚è¯ä¹¦åªä¼šé’ˆå¯¹æŒ‡å®šçš„æœåŠ¡æ›´æ”¹ï¼Œä½†ä½ ä¼šçœ‹åˆ°ä¸¤ä¸ªç»„件的节点都进入更新状æ€ã€‚ä½ å¯èƒ½è¿˜ä¼šçœ‹åˆ°ä»… Worker 节点进入更新状æ€ã€‚这是在è¯ä¹¦æ›´æ”¹åŽé‡å¯ Worker，以确ä¿ä»–们获得最新的客户端è¯ä¹¦ã€‚ + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md new file mode 100644 index 00000000000..cfebc6443aa --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -0,0 +1,48 @@ +--- +title: åŠ å¯†å¯†é’¥è½®æ¢ +--- + +### RKE1 åŠ å¯†å¯†é’¥è½®æ¢ + +1. 使用以下两个选项之一æ¥å¯ç”¨åŠ å¯†å¯†é’¥è½®æ¢ï¼š + + - 在 Rancher UI 中的 **Cluster Options > Advanced Options > Secrets Encryption** 下选择 `Enabled` å•é€‰æŒ‰é’®ï¼š + + ![å¯ç”¨åŠ å¯†å¯†é’¥è½®æ¢](/img/rke1-enable-secrets-encryption.png) + + - 或者，应用以下 YAML： + + ```yaml + rancher_kubernetes_engine_config: + services: + kube_api: + secrets_encryption_config: + enabled: true + ``` + +2. 在 Rancher UI 中轮æ¢å¯†é’¥ï¼š + + 2.1. 点击 **☰ > 集群管ç†**。 + + 2.2. 在所选集群æ—边的å±å¹•æœ€å³ä¾§é€‰æ‹© **â‹® > Rotate Encryption Keys**： + + ![加密密钥轮æ¢](/img/rke1-encryption-key.png) + + + +### RKE2 åŠ å¯†å¯†é’¥è½®æ¢ + +_**v2.6.7 新功能**_ + +> **é‡è¦æ示**：加密密钥轮æ¢é»˜è®¤å¯ç”¨ï¼Œä¸èƒ½ç¦ç”¨ã€‚ + +è¦åœ¨ Rancher UI 中轮æ¢å¯†é’¥ï¼š + +1. 点击 **☰ > 集群管ç†**。 + +1. 在所选集群æ—边的å±å¹•æœ€å³ä¾§é€‰æ‹© **â‹® > Rotate Encryption Keys**： + + ![加密密钥轮æ¢](/img/rke2-encryption-key.png) + + +> **注æ„**：有关 RKE2 密文加密é…置的更多信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/security/secrets_encryption)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-namespaces.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-namespaces.md new file mode 100644 index 00000000000..1c8aeba0767 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/how-to-guides/new-user-guides/manage-namespaces.md @@ -0,0 +1,74 @@ +--- +title: 命å空间 +--- + +在 Rancher 中，你å¯ä»¥å°†é¡¹ç›®è¿›ä¸€æ­¥åˆ’分为多个[命å空间](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)，项目是由物ç†é›†ç¾¤æ”¯æŒçš„，而命å空间是项目中的虚拟集群。当项目和 `default` 命å空间ä¸èƒ½æ»¡è¶³ä½ çš„需求时，你å¯ä»¥åœ¨é¡¹ç›®ä¸­åˆ›å»ºå¤šä¸ªå‘½å空间æ¥éš”离应用和资æºã€‚ + +ä½ å¯ä»¥åœ¨é¡¹ç›®å±‚级分é…资æºï¼Œä»Žè€Œä½¿é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºï¼Œä¹Ÿå¯ä»¥å•ç‹¬ç»™æŸä¸ªå‘½å空间显å¼åˆ†é…资æºã€‚ + +å¯ä»¥ç›´æŽ¥åˆ†é…给命å空间的资æºåŒ…括： + +- [工作负载](../../pages-for-subheaders/workloads-and-pods.md) +- [è´Ÿè½½å‡è¡¡å™¨/Ingress](../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [æœåŠ¡å‘现记录](kubernetes-resources-setup/create-services.md) +- [æŒä¹…å·å£°æ˜Ž](../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [è¯ä¹¦](kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMap](kubernetes-resources-setup/configmaps.md) +- [é•œåƒä»“库](kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [密文](kubernetes-resources-setup/secrets.md) + +为了在 vanilla Kubernetes 集群中管ç†æƒé™ï¼Œé›†ç¾¤ç®¡ç†å‘˜è¦ä¸ºæ¯ä¸ªå‘½å空间é…置基于角色的访问策略。Rancher 在项目级别上分é…用户æƒé™ï¼Œé¡¹ç›®ä¸­çš„命å空间会自动继承项目的æƒé™ã€‚ + +:::note + +如果你使用 `kubectl`创建命å空间，由于 `kubectl` ä¸è¦æ±‚将新命å空间é™å®šåœ¨ä½ å¯ä»¥è®¿é—®çš„项目内，因此你å¯èƒ½æ— æ³•ä½¿ç”¨è¯¥å‘½å空间。如果你的æƒé™ä»…é™äºŽé¡¹ç›®çº§åˆ«ï¼Œåˆ™æœ€å¥½[通过 Rancher 创建命å空间](manage-namespaces.md)，以确ä¿ä½ æœ‰æƒè®¿é—®è¯¥å‘½å空间。 + +::: + +### 创建命å空间 + +创建一个新的命å空间æ¥éš”离项目中的应用和资æºã€‚ + +:::tip + +使用å¯ä»¥åˆ†é…给命å空间的项目资æºï¼ˆä¾‹å¦‚，[工作负载](kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md)ã€[è¯ä¹¦](kubernetes-resources-setup/encrypt-http-communication.md)ã€[ConfigMap](kubernetes-resources-setup/configmaps.md) 等）时，你å¯ä»¥åŠ¨æ€åˆ›å»ºå‘½å空间。 + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºå‘½å空间的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦æ·»åŠ å‘½å空间的项目，并å•å‡»**创建命å空间**。或者，你也å¯ä»¥è½¬åˆ°**ä¸åœ¨é¡¹ç›®å†…**以创建ä¸ä¸Žé¡¹ç›®å…³è”的命å空间。 + +1. **å¯é€‰**：如果你的项目具有有效的[资æºé…é¢](../../pages-for-subheaders/manage-project-resource-quotas.md)，你å¯ä»¥è¦†ç›–默认资æº**é™åˆ¶**（é™åˆ¶å‘½å空间å¯ä»¥ä½¿ç”¨çš„资æºï¼‰ã€‚ + +1. 输入**å称**，然åŽå•å‡»**创建**。 + +**结果**：已将命å空间添加到项目中。你å¯ä»¥å¼€å§‹å°†é›†ç¾¤èµ„æºåˆ†é…给命å空间。 + +### 将命å空间移动到å¦ä¸€ä¸ªé¡¹ç›® + +在æŸäº›æƒ…况下（例如希望其他团队使用该应用时），集群管ç†å‘˜å’Œæˆå‘˜å¯èƒ½éœ€è¦å°†å‘½å空间移动到å¦ä¸€ä¸ªé¡¹ç›®ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°éœ€è¦ç§»åŠ¨å‘½å空间的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦ç§»åŠ¨çš„命å空间，然åŽå•å‡» **â‹® > 移动**。 + +1. 选择è¦ç§»åŠ¨åˆ°å¦ä¸€ä¸ªé¡¹ç›®çš„命å空间。然åŽå•å‡»**移动**。你å¯ä»¥åŒæ—¶ç§»åŠ¨å¤šä¸ªå‘½å空间。 + + :::note 注æ„事项： + + - ä¸è¦ç§»åŠ¨ `System` 项目中的命å空间。移动命å空间å¯èƒ½ä¼šå¯¹é›†ç¾¤ç½‘络产生ä¸åˆ©å½±å“。 + - ä½ ä¸èƒ½å°†å‘½å空间移动到已é…ç½®[资æºé…é¢](../../pages-for-subheaders/manage-project-resource-quotas.md)的项目中。 + - 如果你将命å空间从已设置é…é¢çš„项目移动到未设置é…é¢çš„项目，则会删除该命å空间的é…é¢ã€‚ + +1. 为新命å空间选择一个新项目，然åŽå•å‡»**移动**。你也å¯ä»¥é€‰æ‹©**æ— **，从而将命å空间从所有项目中移除。 + +**结果**：你的命å空间已移至其他项目（或从所有项目中移除）。如果命å空间绑定了项目资æºï¼Œå‘½å空间会释放这些资æºï¼Œç„¶åŽç»‘定新项目的资æºã€‚ + +### 编辑命å空间资æºé…é¢ + +ä½ å¯ä»¥è¦†ç›–命å空间默认é™åˆ¶ï¼Œä»Žè€Œä¸ºç‰¹å®šå‘½å空间æ供对更多（或更少）项目资æºçš„访问æƒé™ï¼š + +有关详细信æ¯ï¼Œè¯·å‚阅[编辑命å空间资æºé…é¢](../advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/configuration-reference.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/configuration-reference.md new file mode 100644 index 00000000000..7a530783b36 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/configuration-reference.md @@ -0,0 +1,105 @@ +--- +title: é…ç½® +--- + +æ­¤é…ç½®å‚考用于帮助你管ç†ç”± `rancher-cis-benchmark` 应用创建的自定义资æºã€‚这些资æºç”¨äºŽåœ¨é›†ç¾¤ä¸Šæ‰§è¡Œ CIS 扫æã€è·³è¿‡æµ‹è¯•ã€è®¾ç½®æ‰«æ使用的测试é…置文件和其他自定义é…置。 + +è¦é…置自定义资æºï¼Œè½¬åˆ°**集群**仪表æ¿ã€‚è¦é…ç½® CIS 扫æ： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦é…ç½® CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark**。 + +### 扫æ + +扫æ是用æ¥æ ¹æ®å®šä¹‰çš„é…ç½®æ–‡ä»¶ï¼Œåœ¨é›†ç¾¤ä¸Šè§¦å‘ CIS 扫æ的。扫æ完æˆåŽä¼šåˆ›å»ºä¸€ä»½æŠ¥å‘Šã€‚ + +é…置扫æ时，你需è¦å®šä¹‰ä¸Ž `scanProfileName` å‚数一起使用的扫æé…置文件的å称。 + +下é¢æ˜¯ä¸€ä¸ª ClusterScan 自定义资æºç¤ºä¾‹ï¼š + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScan +metadata: + name: rke-cis +spec: + scanProfileName: rke-profile-hardened +``` + +### é…置文件 + +é…ç½®æ–‡ä»¶åŒ…å« CIS 扫æçš„é…置，包括è¦ä½¿ç”¨çš„ Benchmark 测试版本以åŠè¦åœ¨è¯¥ Benchmark 测试中跳过的测试。 + +:::caution + +默认情况下，一些 ClusterScanProfile 会作为 `rancher-cis-benchmark` Chart 的一部分进行安装。如果用户编辑了这些默认 Benchmark 或é…置文件，它们会在下一次 Chart 更新时被é‡ç½®ã€‚因此，建议用户ä¸è¦ç¼–辑默认的 ClusterScanProfile。 + +::: + +用户å¯ä»¥é€šè¿‡å…‹éš† ClusterScanProfile æ¥åˆ›å»ºè‡ªå®šä¹‰é…置文件。 + +跳过的测试会列在 `skipTests` å‚数下。 + +创建新é…置文件时，你还需è¦å‘½åé…置文件。 + +`ClusterScanProfile` 示例如下： + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" +spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" +``` + +### Benchmark 版本 + +Benchmark 版本是指使用 `kube-bench` è¿è¡Œçš„ Benchmark å称，以åŠè¯¥ Benchmark 的有效é…ç½®å‚数。 + +`ClusterScanBenchmark` 定义了 CIS `BenchmarkVersion` çš„å称和测试é…置。`BenchmarkVersion` å称是æ供给 `kube-bench` 工具的å‚数。 + +默认情况下，一些 `BenchmarkVersion` å称和测试é…置会作为 CIS 扫æ应用的一部分进行打包。å¯ç”¨æ­¤åŠŸèƒ½åŽï¼Œè¿™äº›é»˜è®¤ BenchmarkVersion 将自动安装，用户å¯ä»¥ä½¿ç”¨å®ƒä»¬æ¥åˆ›å»º ClusterScanProfile。 + +:::caution + +如果用户编辑了默认的 BenchmarkVersion，它们会在下一次 Chart 更新时被é‡ç½®ã€‚因此，ä¸å»ºè®®ç¼–辑默认的 ClusterScanBenchmark。 + +::: + +ClusterScanBenchmark 由以下字段组æˆï¼š + +- `ClusterProvider`：此 Benchmark 适用的集群æ供商å称，例如，RKEã€EKSã€GKE。如果此基准测试å¯ä»¥åœ¨ä»»ä½•é›†ç¾¤ç±»åž‹ä¸Šè¿è¡Œï¼Œåˆ™ç•™ç©ºã€‚ +- `MinKubernetesVersion`：集群è¿è¡Œæ­¤ Benchmark 测试所需的最低 kubernetes 版本。如果ä¸ä¾èµ–特定的 Kubernetes 版本，则留空。 +- `MaxKubernetesVersion`：集群è¿è¡Œæ­¤ Benchmark 测试所需的最高 Kubernetes 版本。如果ä¸ä¾èµ–特定的 Kubernetes 版本，则留空。 + +`ClusterScanBenchmark` 示例如下： + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + creationTimestamp: "2020-08-28T18:18:07Z" + generation: 1 + labels: + app.kubernetes.io/managed-by: Helm + name: cis-1.5 + resourceVersion: "203878" + selfLink: /apis/cis.cattle.io/v1/clusterscanbenchmarks/cis-1.5 + uid: 309e543e-9102-4091-be91-08d7af7fb7a7 +spec: + clusterProvider: "" + minKubernetesVersion: 1.15.0 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/custom-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/custom-benchmark.md new file mode 100644 index 00000000000..6818ac56118 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/custom-benchmark.md @@ -0,0 +1,78 @@ +--- +title: 为集群扫æ创建自定义 Benchmark 版本 +--- + +æ¯ä¸ª Benchmark 版本都定义了一组测试é…置文件，这些文件定义了由 kube-bench 工具è¿è¡Œçš„ CIS 测试。 +`rancher-cis-benchmark` 应用安装了一些默认的 Benchmark 测试版本，这些版本列在了 CIS Benchmark 测试应用èœå•ä¸‹ã€‚ + +但是，æŸäº› Kubernetes 集群å¯èƒ½éœ€è¦è‡ªå®šä¹‰é…ç½® Benchmark 测试。例如，Kubernetes é…置文件或è¯ä¹¦çš„路径å¯èƒ½ä¸Žä¸Šæ¸¸ CIS Benchmark 的标准ä½ç½®ä¸åŒã€‚ + +现在，你å¯ä»¥ä½¿ç”¨ `rancher-cis-benchmark` 应用æ¥åˆ›å»ºè‡ªå®šä¹‰ Benchmark 版本，从而è¿è¡Œé›†ç¾¤æ‰«æ。 + +è¿è¡Œé›†ç¾¤æ‰«æ时，你需è¦é€‰æ‹©æŒ‡å‘特定 Benchmark 版本的é…置文件。 + +按照以下所有步骤添加自定义 Benchmark 版本并使用它è¿è¡Œæ‰«æ。 + +### 1. 准备自定义 Benchmark 版本 ConfigMap + +è¦åˆ›å»ºè‡ªå®šä¹‰ Benchmark 版本，你需è¦å…ˆåˆ›å»ºä¸€ä¸ªåŒ…å« Benchmark 版本é…置文件的 ConfigMap，并将其上传到è¦è¿è¡Œæ‰«æçš„ Kubernetes 集群。 + +å‡è®¾è¦æ·»åŠ ä¸€ä¸ªå为 `foo` 的自定义 Benchmark 版本，你å¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤å‡†å¤‡è‡ªå®šä¹‰ Benchmark 版本 ConfigMap： + +1. 创建一个å为 `foo` 的目录，并在该目录中放置所有é…ç½® YAML 文件， kube-bench 工具会æœç´¢è¿™äº›æ–‡ä»¶ã€‚例如，通用 CIS 1.5 Benchmark 版本的é…ç½® YAML 文件在[此处](https://github.com/aquasecurity/kube-bench/tree/master/cfg/cis-1.5)。 +1. 放置完整的 `config.yaml` 文件，其中包括所有è¦æµ‹è¯•çš„组件。 +1. å°† Benchmark 版本å称添加到 `config.yaml` çš„ `target_mapping` 中： + + ```yaml + target_mapping: + "foo": + - "master" + - "node" + - "controlplane" + - "etcd" + - "policies" + ``` +1. 通过创建 ConfigMap 将此目录上传到 Kubernetes 集群： + + ```yaml + kubectl create configmap -n foo --from-file= + ``` + +### 2. 将自定义 Benchmark 版本添加到集群 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰ Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark > Benchmark 版本**。 +1. å•å‡»**创建**。 +1. 输入自定义 Benchmark 版本的**å称**å’Œæ述。 +1. 选择è¦åº”用 Benchmark 版本的集群æ供商。 +1. 在下拉列表中选择你已上传的 ConfigMap。 +1. 添加最低和最高 Kubernetes 版本é™åˆ¶ï¼ˆå¦‚果有）。 +1. å•å‡»**创建**。 + +### 3. 为自定义 Benchmark 版本创建新é…置文件 + +è¦ä½¿ç”¨ä½ çš„自定义 Benchmark 版本è¿è¡Œæ‰«æ，你需è¦æ·»åŠ ä¸€ä¸ªæŒ‡å‘æ­¤ Benchmark 版本的新é…置文件： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰ Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark > é…置文件**。 +1. å•å‡»**创建**。 +1. 设置**å称**å’Œæ述。在本例中，我们将其命å为 `foo-profile`。 +1. 在下拉列表中选择 Benchmark 版本。 +1. å•å‡»**创建**。 + +### 4. 使用自定义 Benchmark 版本è¿è¡Œæ‰«æ + +指å‘你的自定义 Benchmark 版本的 `foo` é…置文件创建完æˆåŽï¼Œä½ å¯ä»¥åˆ›å»ºä¸€ä¸ªæ–°çš„扫æ，从而在 Benchmark 版本中è¿è¡Œè‡ªå®šä¹‰æµ‹è¯•ã€‚ + +è¦è¿è¡Œæ‰«æ： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰ Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark > 扫æ**。 +1. å•å‡»**创建**。 +1. 选择新的集群扫æé…置文件。 +1. å•å‡»**创建**。 + +**结果**：已生æˆå¸¦æœ‰æ‰«æ结果的报告。如需查看结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md new file mode 100644 index 00000000000..6398f217c7e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md @@ -0,0 +1,46 @@ +--- +title: RBAC +--- + +本文介ç»ä½¿ç”¨ rancher-cis-benchmark 应用所需的æƒé™ã€‚ + +默认情况下，rancher-cis-benchmark 是集群管ç†å‘˜ç‹¬æœ‰çš„功能。 + +但是，`rancher-cis-benchmark` Chart 安装了这两个默认的 `ClusterRole`： + +- cis-admin +- cis-view + +在 Rancher 中，默认情况下åªæœ‰é›†ç¾¤æ‰€æœ‰è€…和全局管ç†å‘˜å…·æœ‰ `cis-admin` 访问æƒé™ã€‚ + +注æ„：如果你使用在 Rancher 2.5 中添加的 `cis-edit` 角色，该角色从 2.5.2 开始已被删除，删除的原因是该角色本质上与 `cis-admin` 相åŒã€‚如果你为 `cis-edit` 创建了任何 clusterrolebinding，请更新这些绑定以使用 `cis-admin` ClusterRole。 + +## Cluster-Admin 访问 + +默认情况下，Rancher CIS 扫æ是集群管ç†å‘˜ç‹¬æœ‰çš„功能。 +æ¢è¨€ä¹‹ï¼Œåªæœ‰ Rancher 全局管ç†å‘˜å’Œé›†ç¾¤çš„集群所有者å¯ä»¥ï¼š + +- 安装/å¸è½½ rancher-cis-benchmark 应用。 +- 查看 CIS Benchmark CRD 的导航链接 - ClusterScanBenchmarksã€ClusterScanProfilesã€ClusterScans。 +- 列出默认的 ClusterScanBenchmarks å’Œ ClusterScanProfiles。 +- 创建/编辑/删除新的 ClusterScanProfiles。 +- 创建/编辑/删除新的 ClusterScan，从而在集群上è¿è¡Œ CIS 扫æ。 +- 在 ClusterScan 完æˆåŽæŸ¥çœ‹å¹¶ä¸‹è½½ ClusterScanReport。 + + +## Kubernetes 默认角色的默认æƒé™æ‘˜è¦ + +rancher-cis-benchmark 创建了三个 `ClusterRole`，并将 CIS Benchmark CRD 访问æƒé™æ·»åŠ åˆ°ä»¥ä¸‹é»˜è®¤ K8s `ClusterRole`： + +| Chart 创建的 ClusterRole | 默认 K8s ClusterRole | 角色赋予的æƒé™ | +| ------------------------------| ---------------------------| ---------------------------| +| `cis-admin` | `admin` | 增刪查改（CRUD）clusterscanbenchmarksã€clusterscanprofilesã€clusterscansã€clusterscanreports CR | +| `cis-view` | `view ` | 列出 (R) clusterscanbenchmarksã€clusterscanprofilesã€clusterscansã€clusterscanreports CR | + + +默认情况下，åªæœ‰ cluster-owner 角色能管ç†å’Œä½¿ç”¨ `rancher-cis-benchmark` 功能。 + +默认情况下，其他 Rancher 角色（cluster-memberã€project-owner å’Œ project-member）没有管ç†å’Œä½¿ç”¨ rancher-cis-benchmark 资æºçš„æƒé™ã€‚ + +但是，如果 cluster-owner 想将访问æƒé™åˆ†é…给其他用户，他们å¯ä»¥æ‰‹åŠ¨åˆ›å»ºç›®æ ‡ç”¨æˆ·ä¸Žä¸Šè¿° CIS ClusterRole 之间的 ClusterRoleBinding，从而实现æƒé™åˆ†é…。 +`rancher-cis-benchmark` ClusterRole ä¸æ”¯æŒè‡ªåŠ¨è§’色èšåˆã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md new file mode 100644 index 00000000000..a7afe788a87 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md @@ -0,0 +1,53 @@ +--- +title: 跳过和ä¸é€‚用的测试 +--- + +本文列出了在 RKE çš„ permissive 测试é…置文件中跳过的测试。 + +> 在 v2.5 生æˆçš„报告中，此页é¢ä¸Šè¢«è·³è¿‡ä¸”ä¸é€‚用的测试将会计为ä¸é€‚用。跳过的测试计数åªä¼šæ¶‰åŠç”¨æˆ·å®šä¹‰çš„跳过测试。这样，你å¯ä»¥åŒºåˆ†ç”¨æˆ·è¦è·³è¿‡çš„测试与 RKE permissive 测试é…置文件中默认跳过的测试。 + +## CIS Benchmark v1.5 + +### CIS Benchmark v1.5 跳过的测试 + +| æ•°å­— | æè¿° | 跳过的原因 | +| ---------- | ------------- | --------- | +| 1.1.12 | ç¡®ä¿ etcd æ•°æ®ç›®å½•æ‰€æœ‰æƒè®¾ç½®ä¸º etcd:etcd（自动） | etcd æ•°æ®ç›®å½•æ‰€æœ‰æƒéœ€è¦ç³»ç»Ÿ ServiceAccount。有关如何é…置所有æƒçš„更多信æ¯ï¼Œè¯·å‚阅 Rancher 的强化指å—。 | +| 1.2.6 | ç¡®ä¿æ ¹æ®éœ€è¦è®¾ç½® --kubelet-certificate-authority å‚数（自动） | 在生æˆæœåŠ¡è¯ä¹¦æ—¶ï¼ŒåŠŸèƒ½å¯èƒ½ä¼šä¸ŽæŸäº›äº‘æ供商所需的主机å覆盖一起中断。 | +| 1.2.16 | ç¡®ä¿è®¾ç½®äº†å‡†å…¥æŽ§åˆ¶æ’件 PodSecurityPolicy（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 1.2.33 | ç¡®ä¿èƒ½æ ¹æ®éœ€è¦è®¾ç½® --encryption-provider-config å‚数（手动） | å¯ç”¨åŠ å¯†ä¼šæ”¹å˜æ¢å¤åŠ å¯†æ•°æ®çš„æ–¹å¼ã€‚ | +| 1.2.34 | ç¡®ä¿æ­£ç¡®é…置了加密æ供程åºï¼ˆæ‰‹åŠ¨ï¼‰ | å¯ç”¨åŠ å¯†ä¼šæ”¹å˜æ¢å¤åŠ å¯†æ•°æ®çš„æ–¹å¼ã€‚ | +| 4.2.6 | ç¡®ä¿ --protect-kernel-defaults å‚数设置为 true（自动） | 在é…置集群之å‰éœ€è¦ç³»ç»Ÿçº§åˆ«çš„é…置，æ‰èƒ½å°†æ­¤å‚数设置为 true。 | +| 4.2.10 | ç¡®ä¿æ ¹æ®éœ€è¦è®¾ç½® --tls-cert-file å’Œ --tls-private-key-file å‚数（自动） | 在生æˆæœåŠ¡è¯ä¹¦æ—¶ï¼ŒåŠŸèƒ½å¯èƒ½ä¼šä¸ŽæŸäº›äº‘æ供商所需的主机å覆盖一起中断。 | +| 5.1.5 | ç¡®ä¿æœªä¸»åŠ¨ä½¿ç”¨é»˜è®¤ ServiceAccount。（自动） | Kubernetes æ供了è¦ä½¿ç”¨çš„默认 ServiceAccount。 | +| 5.2.2 | 最å°åŒ–需è¦å…±äº«ä¸»æœºè¿›ç¨‹ ID 命å空间的容器准入（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.2.3 | 最å°åŒ–需è¦å…±äº«ä¸»æœº IPC 命å空间的容器准入（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.2.4 | 最å°åŒ–需è¦å…±äº«ä¸»æœºç½‘络命å空间的容器准入（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.2.5 | 使用 allowPrivilegeEscalation（自动）最å°åŒ–容器的准入 | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.3.2 | ç¡®ä¿æ‰€æœ‰å‘½å空间都定义了网络策略（自动） | å¯ç”¨ç½‘络策略å¯ä»¥é˜²æ­¢æŸäº›åº”用进行相互通信。 | +| 5.6.4 | ç¡®ä¿ä¸ä½¿ç”¨ Default 命å空间（自动） | Kubernetes æ供了一个 Default 命å空间。 | + +### CIS Benchmark v1.5 ä¸é€‚用的测试 + +| æ•°å­— | æè¿° | ä¸é€‚用的原因 | +| ---------- | ------------- | --------- | +| 1.1.1 | ç¡®ä¿ API Server pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kube-apiserver çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.2 | ç¡®ä¿ API Server pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kube-apiserver çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.3 | ç¡®ä¿ Controller Manager pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.4 | ç¡®ä¿ Controller Manager pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.5 | ç¡®ä¿ Scheduler pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.6 | ç¡®ä¿ Scheduler pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.7 | ç¡®ä¿ etcd pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 etcd çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.8 | ç¡®ä¿ etcd pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 etcd çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.13 | ç¡®ä¿ admin.conf 文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸ä¼šåœ¨èŠ‚点上存储 kubernetes 的默认 kubeconfig 凭è¯æ–‡ä»¶ã€‚ | +| 1.1.14 | ç¡®ä¿ admin.conf 文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸ä¼šåœ¨èŠ‚点上存储 kubernetes 的默认 kubeconfig 凭è¯æ–‡ä»¶ã€‚ | +| 1.1.15 | ç¡®ä¿ scheduler.conf 文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.16 | ç¡®ä¿ scheduler.conf 文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.17 | ç¡®ä¿ controller-manager.conf 文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.18 | ç¡®ä¿å°† controller-manager.conf 文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.3.6 | ç¡®ä¿ RotateKubeletServerCertificate å‚数设置为 true（自动） | RKE é…置的集群直接使用 RKE 处ç†è¯ä¹¦è½®æ¢ã€‚ | +| 4.1.1 | ç¡®ä¿ kubelet æœåŠ¡æ–‡ä»¶æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet æœåŠ¡çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.1.2 | ç¡®ä¿ kubelet æœåŠ¡æ–‡ä»¶æ‰€æœ‰æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet æœåŠ¡çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.1.9 | ç¡®ä¿ kubelet é…置文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.1.10 | ç¡®ä¿ kubelet é…置文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.2.12 | ç¡®ä¿ RotateKubeletServerCertificate å‚数设置为 true（自动） | RKE é…置的集群直接使用 RKE 处ç†è¯ä¹¦è½®æ¢ã€‚ | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md new file mode 100644 index 00000000000..8431d4fbd66 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md @@ -0,0 +1,96 @@ +--- +title: 先决æ¡ä»¶ +--- + +### 1. 设置许å¯è¯ç®¡ç†å™¨å’Œè´­ä¹°æ”¯æŒ + +首先，完æˆè®¸å¯è¯ç®¡ç†å™¨è®¾ç½®çš„[第一步](https://docs.aws.amazon.com/license-manager/latest/userguide/getting-started.html)。 +然åŽï¼Œè½¬åˆ° AWS Marketplace。找到 “Rancher Premium Support Billing Container Starter Packâ€ã€‚最åŽï¼Œè´­ä¹°è‡³å°‘一项 Entitlement。 + +如果你已使用 “Rancher Setup†AWS Marketplace 产å“安装了 Rancher，请跳至[步骤 4](#4-创建-oidc-æ供程åº)。 + +> **注æ„**：æ¯é¡¹ Entitlement 都对一定数é‡çš„节点授予访问支æŒçš„æƒé™ã€‚ä½ å¯ä»¥åŽç»­æ ¹æ®éœ€è¦è´­ä¹°æ›´å¤šè®¸å¯è¯ã€‚ + +### 2. 创建 EKS 集群 +按照 [Rancher 文档](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md)创建 EKS 集群。进行到[安装 Rancher Helm Chart](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-安装-rancher-helm-chart)（最åŽä¸€æ­¥ï¼‰æ—¶ï¼Œ**åœæ­¢å¹¶è¿”回此页é¢**。该集群需è¦æ»¡è¶³ä»¥ä¸‹è¦æ±‚： + +- EKS 1.22 版本。 +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—®åŒ…å« Rancher åŠå…¶ç›¸å…³é•œåƒçš„é•œåƒä»“库。 +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—®å­˜å‚¨ CSP Adapter çš„ ECR 仓库。 +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—®è®¸å¯è¯ç®¡ç†å™¨æœåŠ¡ã€‚ +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—® STS æœåŠ¡çš„全局端点。 + +### 3. 安装 Rancher + +除了在 [Rancher 文档](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-安装-rancher-helm-chart)中指定的 Rancher 安装选项外，你还需è¦å¯ç”¨å…¶å®ƒæŒ‡æ ‡ã€‚ +ä½ å¯ä»¥é€šè¿‡ Helm CLI 使用以下选项æ¥å®Œæˆï¼š + +```bash +--set extraEnv\[0\].name="CATTLE_PROMETHEUS_METRICS" --set-string extraEnv\[0\].value=true +``` + +你还å¯ä»¥ä½¿ç”¨ values.yaml，如下所示： + +```yaml +extraEnv: + - name: "CATTLE_PROMETHEUS_METRICS" + value: "true" +``` + +你还需è¦å®‰è£… Rancher 2.6.7 或更高版本。 + +### 4. 创建 OIDC æä¾›ç¨‹åº + +按照 [AWS 文档](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html)为上一节中指定的集群创建 OIDC æ供程åºã€‚ + +### 5. 创建 IAM 角色 + +CSP Adapter éœ€è¦ IAM 角色æ‰èƒ½ç­¾å…¥/签出 Entitlement。 + +首先，é…置如下的信任策略。你需è¦å°† `MY_AWS_ACC` 替æ¢ä¸ºä½ çš„ AWS å¸å·ï¼Œå°† `MY_AWS_REGION` 替æ¢ä¸ºä½ çš„ AWS 区域，并将 `MY_OIDC_PROVIDER` 替æ¢ä¸ºä½ çš„ OIDC æ供商 ID： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${MY_AWS_ACC}:oidc-provider/oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:sub": "system:serviceaccount:cattle-csp-adapter-system:rancher-csp-adapter", + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:aud": "sts.amazonaws.com" + } + } + } + ] +} +``` + +接下æ¥ï¼Œä¸ºå…·æœ‰ä»¥ä¸‹æƒé™çš„角色使用策略： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "RancherCSPAdapterPermissions", + "Effect": "Allow", + "Action": [ + "license-manager:ListReceivedLicenses", + "license-manager:CheckoutLicense", + "license-manager:ExtendLicenseConsumption", + "license-manager:CheckInLicense", + "license-manager:GetLicense", + "license-manager:GetLicenseUsage" + ], + "Resource": "*" + } + ] +} +``` + +ä¿å­˜è§’色的å称。ç¨åŽå®‰è£… CSP Adapter 时将需è¦å®ƒã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md new file mode 100644 index 00000000000..0e9f63f4086 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md @@ -0,0 +1,28 @@ +--- +title: 常è§é—®é¢˜ +--- + +**安装 Adapter åŽï¼ŒRancher 中出现一æ¡æ¨ªå¹…消æ¯ï¼Œä¸Šé¢å†™ç€ "AWS Marketplace Adapter: Unable to run the adapter, please check the adapter logs"** + +此错误表示 Adapter 安装到集群中时å‘生了一个错误，导致它无法正确签入/签出许å¯è¯ã€‚ + +这通常是因为 IAM 角色设置ä¸æ­£ç¡®ã€‚请查看[先决æ¡ä»¶](./adapter-requirements.md)并确认： + +- 已创建一个 OIDC æ供程åºï¼Œå¹¶ä¸”它已关è”到è¿è¡Œ Rancher 的集群。 +- IAM 角色已é…置为信任此 OIDC æ供程åºã€‚ +- IAM 角色至少具有策略中概述的æƒé™ã€‚ + +如果上述所有é…ç½®å‡å·²æ­£ç¡®é…置，请è”系支æŒå¯»æ±‚帮助。 + +**我看到一æ¡æ¨ªå¹…消æ¯ï¼Œä¸Šé¢å†™ç€ "AWS Marketplace Adapter: You have exceeded your licensed node count. At least x more license(s) are required in AWS to become compliant"** + +此消æ¯è¡¨æ˜Žä½ æ²¡æœ‰è¶³å¤Ÿçš„ Entitlement æ¥æ»¡è¶³ Rancher 当å‰ç®¡ç†çš„节点数é‡ã€‚ + +请记ä½ä»¥ä¸‹é™åˆ¶ï¼š + +- æ¯ä¸ª Entitlement 仅对一定数é‡çš„节点有效。 +- 当å‰ç”± Rancher 管ç†çš„æ¯ä¸ªèŠ‚点都计入你的总使用é‡ï¼ˆå®‰è£…了集群 Rancher 的节点除外）。 +- æ¯ä¸ª Entitlement 最多å¯ä»¥è¢«ä¸€ä¸ª Rancher 实例使用。例如，如果你的账户中有两个正在è¿è¡Œçš„ Rancher 实例（æ¯ä¸ªéƒ½å®‰è£…在å•ç‹¬çš„ EKS 集群上），那么你至少需è¦ä¸¤ä¸ª Entitlement。 + +你最近å¯èƒ½è¿˜å¸è½½/é‡æ–°å®‰è£…了 Adapter。如果 Adapter 丢失了它当å‰ç®¡ç†çš„许å¯è¯çš„跟踪，则å¯èƒ½éœ€è¦ä¸€ä¸ªå°æ—¶æ¥è§£æžè®¸å¯è¯çš„实际状æ€ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md new file mode 100644 index 00000000000..831bf952a6d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -0,0 +1,155 @@ +--- +title: 安装 Adapter +--- + +> **é‡è¦æ示**：如果你å°è¯•é‡æ–°å®‰è£… Adapter，你å¯èƒ½ä¼šåœ¨é•¿è¾¾ä¸€å°æ—¶çš„时间内收到ä¸åˆè§„的错误消æ¯ã€‚ + +### Rancher 与 Adapter 的兼容性矩阵 + +:::note é‡è¦æ示： + +ä¸åŒç‰ˆæœ¬çš„ CSP Adapter ä¾èµ–于特定 Rancher 版本的功能。 +为了æˆåŠŸéƒ¨ç½²å’Œè¿è¡Œ Adapter，你需è¦ç¡®ä¿ Adapter 版本与必è¦çš„ Rancher 版本对应。 + +::: + +| Rancher 版本 | Adapter 版本 | +|-----------------|:---------------:| +| v2.7.0 | v2.0.0 | +| v2.7.1 | v2.0.0 | +| v2.7.2 | v2.0.1 | +| v2.7.3 | v2.0.1 | +| v2.7.4 | v2.0.1 | +| v2.7.5 | v2.0.2 | + + +### 1. 获å–对 Local 集群的访问æƒé™ + +> **注æ„**：åªæœ‰ç®¡ç†å‘˜ç”¨æˆ·æ‰èƒ½è®¿é—® Local 集群。因为 CSP Adapter 必须安装在 Local 集群中，所以此安装必须由管ç†å‘˜ç”¨æˆ·æ‰§è¡Œã€‚ + +首先，å•å‡» Local 集群并下载 kubeconfig 令牌。然åŽï¼Œä½¿ç”¨ä¸‹æ–¹çš„命令é…ç½® CLI 以使用此新令牌，你需è¦å°† `$TOKEN_PATH` 替æ¢ä¸ºæ–‡ä»¶ç³»ç»Ÿä¸Šä»¤ç‰Œçš„下载路径： + +```bash +export KUBECONFIG=$TOKEN_PATH +``` + +### 2. 创建 Adapter 命å空间 + +创建è¦å®‰è£… Adapter 的命å空间： + +```bash +kubectl create ns cattle-csp-adapter-system +``` + +### 3. 创建è¯ä¹¦å¯†æ–‡ + +Adapter 需è¦è®¿é—® Rancher 用æ¥ä¸Ž Rancher Server 通信的根 CA。有关 Rancher 支æŒçš„è¯ä¹¦é€‰é¡¹çš„更多信æ¯ï¼Œè¯·å‚阅 [Chart 选项页é¢](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)。 + +如果你的 Rancher 使用由公认的è¯ä¹¦é¢å‘机构（例如 Let's Encrypt）签å‘çš„è¯ä¹¦ï¼Œä½ å¯ä»¥è·³åˆ°[步骤 4](#4-安装-chart)。 + +但是，如果你的 Rancher 使用了自定义è¯ä¹¦ï¼ˆä¾‹å¦‚ Rancher 生æˆçš„è¯ä¹¦æˆ–ç”±ç§æœ‰è¯ä¹¦é¢å‘机构签å‘çš„è¯ä¹¦ï¼‰ï¼Œä½ éœ€è¦ä¸ºè¯¥ CA æä¾› PEM ç¼–ç æ ¼å¼çš„è¯ä¹¦ä»¥ä¾¿ Adapter å¯ä»¥ä¸Ž Rancher 通信。 + +首先，检索 Rancher 正在使用的è¯ä¹¦å¹¶å°†å…¶æ”¾å…¥å为 `ca-additional.pem` 的文件中。如果你使用 Rancher 生æˆçš„è¯ä¹¦ï¼Œä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å®Œæˆæ­¤æ“作： + +```bash +kubectl get secret tls-rancher -n cattle-system -o jsonpath="{.data.tls\.crt}" | base64 -d >> ca-additional.pem +``` + +然åŽï¼Œåˆ›å»ºä¸€ä¸ªä½¿ç”¨æ­¤è¯ä¹¦çš„密文： + +```bash +kubectl -n cattle-csp-adapter-system create secret generic tls-ca-additional --from-file=ca-additional.pem +``` + +> **é‡è¦æ示**：ä¸è¦æ›´æ”¹æ–‡ä»¶å或创建的密文的å称，å¦åˆ™å¯èƒ½ä¼šå¯¼è‡´ Adapter è¿è¡Œå‡ºé”™ã€‚ + +### 4. 安装 Chart + +首先，使用以下命令添加 `rancher/charts` 仓库： + +```bash +helm repo add rancher-charts https://charts.rancher.io +``` + +接下æ¥ï¼Œå®‰è£… CSP Adapter。你必须指定多个值，其中包括账å·å·ç ä»¥åŠåœ¨å…ˆå†³æ¡ä»¶ä¸­åˆ›å»ºçš„角色的å称。 + +ç¡®ä¿ä½ çš„ CSP Adapter 版本与你正在è¿è¡Œçš„ Rancher 版本匹é…，如[上文](#rancher-与-adapter-的兼容性矩阵)所述。 + +在下方的æ“作中，将 `$MY_ACC_NUM` 替æ¢ä¸ºä½ çš„ AWS è´¦å·ï¼Œå°† `$MY_ROLE_NAME` 替æ¢ä¸ºå…ˆå†³æ¡ä»¶ä¸­åˆ›å»ºçš„角色的å称。此外，将 `$CSP_ADAPTER_VERSION` 替æ¢ä¸ºä¸Ž[版本矩阵](#rancher-与-adapter-的兼容性矩阵)中的 Rancher 版本匹é…的版本。 + +> **注æ„**：如果你使用 shell å˜é‡ï¼Œè¯·ä¸è¦ä½¿ç”¨å¼•å·ã€‚例如，MY_ACC_NUM=123456789012 å¯ç”¨ï¼Œä½† MY_ACC_NUM="123456789012" 将失败。 + +> **注æ„**：使用欧盟和英国的 AWS Marketplace 列表的账å·éœ€è¦é¢å¤–指定 `--set image.repository=rancher/rancher-csp-adapter-eu` 选项。è¦æŸ¥çœ‹ä½ çš„è´¦å·åœ¨å®‰è£… Adapter 时是å¦éœ€è¦æ­¤é€‰é¡¹ï¼Œè¯·å‚阅 Marketplace 列表的使用说明。 + + + + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --version $CSP_ADAPTER_VERSION +``` + + +你也å¯ä»¥ä½¿ç”¨ `values.yaml` 并指定以下选项： + +```yaml +aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME +``` + +> **注æ„**：账å·éœ€è¦åƒä¸Šé¢é‚£æ ·ä»¥å­—符串格å¼æŒ‡å®šï¼Œå¦åˆ™å®‰è£…会失败。 + +然åŽï¼Œä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å®‰è£… Adapter： + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION +``` + + + + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --set additionalTrustedCAs=true --version $CSP_ADAPTER_VERSION +``` + +你也å¯ä»¥ä½¿ç”¨ `values.yaml` 并指定以下选项： + +```yaml +aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME +additionalTrustedCAs: true +``` + +> **注æ„**：账å·éœ€è¦åƒä¸Šé¢é‚£æ ·ä»¥å­—符串格å¼æŒ‡å®šï¼Œå¦åˆ™å®‰è£…会失败。 + +然åŽï¼Œä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å®‰è£… Adapter： + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION +``` + + + + +### 5. 管ç†è¯ä¹¦æ›´æ–° + +如果你在[步骤 3](#3-创建è¯ä¹¦å¯†æ–‡) 中创建了一个用于存储自定义è¯ä¹¦çš„密文，则éšç€è¯ä¹¦çš„è½®æ¢ï¼Œä½ å°†éœ€è¦æ›´æ–°æ­¤å¯†æ–‡ã€‚ + +首先，使用以下命令删除 cattle-csp-adapter-system 命å空间中的原始密文： + +```bash +kubectl delete secret tls-ca-additional -n cattle-csp-adapter-system +``` + +然åŽï¼ŒæŒ‰ç…§[步骤 3](#3-创建è¯ä¹¦å¯†æ–‡) 中的安装步骤，将密文的内容替æ¢ä¸ºæ›´æ–°åŽçš„值。 + +最åŽï¼Œé‡æ–°å¯åŠ¨ rancher-csp-adapter deployment æ¥ç¡®ä¿æ›´æ–°åŽçš„值å¯ä¾› Adapter 使用： + +```bash +kubectl rollout restart deploy rancher-csp-adapter -n cattle-csp-adapter-system +``` + +> **注æ„**：有一些方法（例如 cert-manager çš„ [trust operator](https://cert-manager.io/docs/projects/trust/)）å¯ä»¥å¸®åŠ©ä½ å‡å°‘手动轮æ¢ä»»åŠ¡çš„æ•°é‡ã€‚这些选项ä¸å—官方支æŒï¼Œä½†å¯èƒ½å¯¹æƒ³è¦è‡ªåŠ¨åŒ–æŸäº›ä»»åŠ¡çš„用户有用。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md new file mode 100644 index 00000000000..f290d881ae9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md @@ -0,0 +1,21 @@ +--- +title: å¸è½½ Adapter +--- + +### 1. 使用 Helm å¸è½½ Adapter Chart： + +```bash +helm uninstall rancher-csp-adapter -n cattle-csp-adapter-system +``` + +### 2. 删除为 Adapter 创建的命å空间： + +```bash +kubectl delete ns cattle-csp-adapter-system +``` + +### 3. （å¯é€‰ï¼‰åˆ é™¤æœªå®Œæˆçš„用户通知： + +```bash +kubectl delete RancherUserNotification csp-compliance +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/supportconfig.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/supportconfig.md new file mode 100644 index 00000000000..f5f0d722487 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/cloud-marketplace/supportconfig.md @@ -0,0 +1,53 @@ +--- +title: Supportconfig Bundle +--- + +安装 CSP Adapter åŽï¼Œä½ å°†èƒ½å¤Ÿç”Ÿæˆä¸€ä¸ª Supportconfig Bundle。此 Bundle 是一个 tar 包，å¯ç”¨äºŽå¿«é€Ÿæ供支æŒä¿¡æ¯ã€‚ + +ä½ å¯ä»¥é€šè¿‡ Rancher 或通过直接访问安装 Rancher 的集群æ¥åˆ›å»ºè¿™äº› Bundle。请注æ„，建议通过 Rancher 访问。 + +> **注æ„**：无论采用何ç§æ–¹æ³•ï¼Œåªæœ‰ç®¡ç†å‘˜å¯ä»¥ç”Ÿæˆ/下载 Supportconfig Bundle。 + +### 通过 Rancher 访问 + +首先，点击汉堡èœå•ã€‚然åŽå•å‡» `Get Support` 按钮。 + +![Get Support](/img/support-help.png) + +在下一页中，å•å‡» `Generate Support Config` 按钮。 + +> **注æ„**：如果未安装 Adapter ，则ä¸ä¼šå‡ºçŽ°ç”Ÿæˆ Supportconfig Bundle 的选项。你必须安装 CSP Adapter æ‰èƒ½ç”Ÿæˆ Supportconfig Bundle。 + +![Get Support](/img/generate-support-config.png) + +### ä¸é€šè¿‡ Rancher 进行访问 + +首先，为安装 Rancher çš„é›†ç¾¤ç”Ÿæˆ kubeconfig。 + +> **注æ„**：如果 Rancher 宕机，你将无法使用 Rancher 生æˆçš„ kubeconfig 令牌访问集群。 + +é…置你的 shell 环境以使用此 kubeconfig 令牌： + +```bash +export KUBECONFIG=$MY_KUBECONFIG_PATH +``` + +建议在è¿è¡Œæ­¤å‘½ä»¤æ—¶åˆ›å»ºä¸€ä¸ªä¸´æ—¶å·¥ä½œç›®å½•ï¼Œå¦‚下所示： + +```bash +mkdir temp && cd temp +``` + +然åŽï¼Œæ£€ç´¢ Supportconfig Bundle： + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && tar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` + +这将在你的当å‰ç›®å½•ä¸­åˆ›å»ºä¸€ä¸ª `supportconfig_rancher.tar` 文件。 + +由于 gnu-tar å’Œ bsd-tar ä¸å…¼å®¹ï¼Œåœ¨ Mac 上è¿è¡Œè¿™äº›å‘½ä»¤çš„用户å¯èƒ½ä¼šé‡åˆ°é—®é¢˜ã€‚如果支æŒéƒ¨é—¨åœ¨è¯»å–你制作的 Supportconfig 时出现问题，你å¯ä»¥å…ˆå°è¯•åœ¨ä½ çš„路径上将 gnu-tar 作为 `gtar` 进行访问，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && gtar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/architecture.md new file mode 100644 index 00000000000..e039b353a31 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/architecture.md @@ -0,0 +1,8 @@ +--- +title: 架构 +--- + +Fleet å¯ä»¥ç®¡ç†æ¥è‡ª Git 的原始 Kubernetes YAMLã€Helm Chartã€Kustomize 或三者的任何组åˆçš„部署。无论æ¥æºå¦‚何，所有资æºéƒ½ä¼šåŠ¨æ€è½¬åŒ–为 Helm Chart，Helm 会用作引擎æ¥å°†æ‰€æœ‰èµ„æºéƒ¨ç½²åˆ°é›†ç¾¤ä¸­ã€‚这给了你高度的控制ã€ä¸€è‡´æ€§å’Œå¯å®¡è®¡æ€§ã€‚Fleet ä¸ä»…关注扩展能力，而且还æ供高度的控制和å¯è§æ€§ï¼Œä»Žè€Œè®©ç”¨æˆ·å‡†ç¡®äº†è§£é›†ç¾¤ä¸Šå®‰è£…的内容。 + +![架构](/img/fleet-architecture.svg) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md new file mode 100644 index 00000000000..37ba154ae39 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md @@ -0,0 +1,54 @@ +--- +title: 在代ç†åŽä½¿ç”¨ Fleet +--- + +在本节中，你将学习如何在一个设置中å¯ç”¨ Fleet，该设置有一个具有公共 IP çš„ Rancher Server，以åŠä¸€ä¸ªæ²¡æœ‰å…¬å…± IP 但é…置为使用代ç†çš„ Kubernetes 集群。 + +Rancher ä¸ä¼šä¸Žå·²æ³¨å†Œçš„下游集群建立连接。部署在下游集群上的 Rancher agent 必须能够与 Rancher 建立连接。 + +è¦è®© Fleet 在代ç†åŽå·¥ä½œï¼Œä½ éœ€è¦ä¸ºä¸‹æ¸¸é›†ç¾¤è®¾ç½® **Agent 环境å˜é‡**。以下是集群级别的é…置选项。 + +ä½ å¯ä»¥é€šè¿‡ Rancher UI 为任何集群类型（包括注册集群和自定义集群）é…置这些环境å˜é‡ã€‚å¯ä»¥åœ¨ç¼–辑现有集群或é…置新集群时添加å˜é‡ã€‚ + +对于公共下游集群，[在 Rancher UI 中设置必è¦çš„环境å˜é‡](#在-rancher-ui-中设置环境å˜é‡)就足够了。 + +对于ç§æœ‰èŠ‚点或ç§æœ‰é›†ç¾¤ï¼Œåˆ™éœ€è¦åœ¨èŠ‚点上设置环境å˜é‡ã€‚然åŽï¼Œåœ¨é…置自定义集群或注册ç§æœ‰é›†ç¾¤æ—¶ï¼Œåœ¨ Rancher UI 中é…置环境å˜é‡ã€‚有关如何在 K3s Kubernetes 集群中的 Ubuntu 节点上设置环境å˜é‡çš„示例，请å‚阅[本节](#在ç§æœ‰èŠ‚点上设置环境å˜é‡)。 + +## å¿…è¦çš„环境å˜é‡ + +为代ç†æ·»åŠ  Fleet agent 环境å˜é‡æ—¶ï¼Œå°† 替æ¢ä¸ºä½ çš„ç§æœ‰ä»£ç† IP。 + +| å˜é‡å称 | 值 | +|------------------|--------| +| `HTTP_PROXY` | http://:8888 | +| `HTTPS_PROXY` | http://:8888 | +| `NO_PROXY` | 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local | + +## 在 Rancher UI 中设置环境å˜é‡ + +è¦å°†çŽ¯å¢ƒå˜é‡æ·»åŠ åˆ°çŽ°æœ‰é›†ç¾¤ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ çŽ¯å¢ƒå˜é‡çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. å•å‡»**高级选项**。 +1. å•å‡»**添加环境å˜é‡**。 +1. 输入[å¿…è¦çš„环境å˜é‡](#å¿…è¦çš„环境å˜é‡) +1. å•å‡»**ä¿å­˜**。 + +**结果**：Fleet agent 会在代ç†åŽå·¥ä½œã€‚ + +## 在ç§æœ‰èŠ‚点上设置环境å˜é‡ + +对于ç§æœ‰èŠ‚点和ç§æœ‰é›†ç¾¤ï¼Œä»£ç†çŽ¯å¢ƒå˜é‡éœ€è¦åœ¨èŠ‚点本身上设置，以åŠä»Ž Rancher UI é…置。 + +此示例显示了如何在 K3s Kubernetes 集群中的 Ubuntu 节点上设置环境å˜é‡ï¼š + +``` +ssh -o ForwardAgent=yes ubuntu@ +ssh +export proxy_private_ip= +export HTTP_PROXY=http://${proxy_private_ip}:8888 +export HTTPS_PROXY=http://${proxy_private_ip}:8888 +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md new file mode 100644 index 00000000000..f322aac9f3a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md @@ -0,0 +1,21 @@ +--- +title: Windows æ”¯æŒ +--- + +在 Rancher 2.5.6 之å‰ï¼Œ`agent` 在具有 Windows 节点的下游集群上没有原生的 Windows 清å•ã€‚这将导致集群的 `agent` pod 失败。 + +如果你从旧版本的 Rancher å‡çº§åˆ° 2.5.6+，你å¯ä»¥åœ¨*下游集群*中部署具有以下工作æµçš„工作 `agent`： + +1. å°é”所有 Windows 节点。 +1. 对 `agent` 工作负载应用以下容å¿åº¦ã€‚ +1. å–消所有 Windows 节点的å°é”。 +1. 删除所有 `agent` pod。使用新的容å¿åº¦æ¥åˆ›å»ºæ–° pod。 +1. `agent` pod è¿è¡Œå¹¶ä¸º Fleet å¯ç”¨äº†è‡ªåŠ¨æ›´æ–°åŽï¼Œå®ƒä»¬ä¼šæ›´æ–°åˆ°ä¸Ž Windows 兼容的 `agent` 版本。 + +```yaml +tolerations: +- effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/harvester.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/harvester.md new file mode 100644 index 00000000000..81823b2999a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/harvester.md @@ -0,0 +1,48 @@ +--- +title: Harvester é›†æˆ +--- + +Harvester 是 Rancher 2.6.1 新增的功能，[Harvester](https://docs.harvesterhci.io/) 是基于 Kubernetes 构建的开æºè¶…èžåˆåŸºç¡€æž¶æž„ (HCI) 软件。Harvester 安装在裸金属æœåŠ¡å™¨ä¸Šï¼Œæ供集æˆçš„虚拟化和分布å¼å­˜å‚¨åŠŸèƒ½ã€‚虽然 Harvester 使用 Kubernetes è¿è¡Œï¼Œä½†å®ƒä¸éœ€è¦ç”¨æˆ·äº†è§£ Kubernetes 概念，因此是一个更加用户å‹å¥½çš„应用。 + +### 功能开关 + +ä½ å¯ä»¥ä½¿ç”¨ Harvester 的功能开关æ¥ç®¡ç† Harvester 在 Rancher 虚拟化管ç†é¡µé¢çš„访问，用户å¯ä»¥åœ¨è¯¥é¡µé¢ç›´æŽ¥å¯¼èˆªåˆ° Harvester 集群并访问 Harvester UI。Harvester 的功能开关是默认å¯ç”¨çš„。如需了解 Rancher 中功能开关的更多详细信æ¯ï¼Œè¯·å•å‡»[此处](../pages-for-subheaders/enable-experimental-features.md)。 + +è¦å¯¼èˆªåˆ° Harvester 集群，请å•å‡» **☰ > 虚拟化管ç†**。在 **Harvester 集群**页é¢ä¸­ï¼Œå•å‡»é›†ç¾¤ä»¥è½¬åˆ°è¯¥ Harvester 集群的视图。 + +* 如果å¯ç”¨äº† Harvester 功能开关，则会从列出 Kubernetes 集群的任何页é¢æˆ–应用（例如 Fleet 和多集群应用）中过滤掉 Harvester 集群。 + +* 如果ç¦ç”¨äº† Harvester 功能开关，并且导入了 Harvester 集群，Harvester 集群将显示在**集群管ç†**页é¢çš„ Rancher 集群列表中。仅当功能开关为关闭时，Harvester 集群æ‰ä¼šæ˜¾ç¤ºåœ¨é›†ç¾¤åˆ—表中。 + +* é›†æˆ Harvester åŽï¼Œä½ å¯ä»¥å°† Harvester 集群导入 Rancher，对应的集群类型是 `Harvester`。 + +* 用户åªèƒ½åœ¨**虚拟化管ç†**页é¢ä¸Šå¯¼å…¥ Harvester 集群。在**集群管ç†**页é¢ä¸Šå¯¼å…¥é›†ç¾¤æ˜¯ä¸æ”¯æŒçš„，而且会出现警告。建议你返回**虚拟化管ç†**页é¢æ‰§è¡Œæ­¤æ“作。 + +### Harvester 主机驱动 + +[Harvester 主机驱动](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/) 通常å¯ç”¨äºŽ Rancher 中的 RKE å’Œ RKE2 选项。无论 Harvester 功能开关是å¦å¯ç”¨ï¼Œä¸»æœºé©±åŠ¨éƒ½æ˜¯å¯ç”¨çš„。请注æ„，默认情况下主机驱动是关闭的。用户åªèƒ½é€šè¿‡**集群管ç†**页é¢åœ¨ Harvester 上创建 RKE 或 RKE2 集群。 + +Harvester å…许通过 Harvester UI 上传和显示 `.ISO` é•œåƒï¼Œä½† Rancher UI ä¸æ”¯æŒã€‚这是因为 `.ISO` é•œåƒé€šå¸¸éœ€è¦é¢å¤–的设置，这会干扰干净的部署（å³æ— éœ€ç”¨æˆ·å¹²é¢„），并且它们通常ä¸ç”¨äºŽäº‘环境。 + +如需了解 Rancher 中主机驱动的更多详细信æ¯ï¼Œè¯·å•å‡»[此处](../pages-for-subheaders/about-provisioning-drivers.md#主机驱动)。 + +### 端å£è¦æ±‚ + +å¯ä»¥åœ¨[此处](https://docs.harvesterhci.io/v1.1/install/requirements#networking)找到 Harvester 集群的端å£è¦æ±‚。 + +此外，其他网络注æ„事项如下： + +- 请务必为 VM VLAN 网络å¯ç”¨ç‰©ç†äº¤æ¢æœºçš„ VLAN 中继端å£ã€‚ +- 按照[此处](https://docs.harvesterhci.io/v1.1/networking/clusternetwork)的网络设置指å—进行æ“作。 + +对于其他集群（例如 K3s å’Œ RKE1）的其他端å£è¦æ±‚，请å‚阅[这些文档](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters)。 + +### é™åˆ¶ + +--- +**仅适用于 Rancher v2.6.1 å’Œ v2.6.2**： + +- Harvester 0.3.0 ä¸æ”¯æŒç¦»çº¿çŽ¯å¢ƒå®‰è£…。 +- ä¸æ”¯æŒå°† Harvester 0.2.0 å‡çº§åˆ° 0.3.0，也ä¸æ”¯æŒå‡çº§åˆ°æ–°çš„ 1.0.0 版本。 + +--- diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md new file mode 100644 index 00000000000..811c89b3ea8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md @@ -0,0 +1,60 @@ +--- +title: 在 RKE2 å’Œ K3s 集群上安装 Istio 的其他步骤 +--- + +通过 **Apps** 页é¢å®‰è£…或å‡çº§ Istio Helm Chart 时： + +1. 如果è¦å®‰è£… Chart，请å•å‡»**在安装å‰è‡ªå®šä¹‰ Helm 选项**，然åŽå•å‡»**下一步**。 +1. 你将看到é…ç½® Istio Helm Chart 的选项。在**组件**选项å¡ä¸Šï¼Œé€‰ä¸­**å¯ç”¨ CNI** æ—边的框。 +1. 添加一个自定义覆盖文件，该文件指定 `cniBinDir` å’Œ `cniConfDir`。有关这些选项的更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](https://istio.io/latest/docs/setup/additional-setup/cni/#helm-chart-parameters)。下方是一个示例： + + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /opt/cni/bin + cniConfDir: /etc/cni/net.d +``` + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /var/lib/rancher/k3s/data/current/bin + cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d +``` + + + +**结果**：现在你应该å¯ä»¥æ ¹æ®éœ€è¦ä½¿ç”¨ Istio，包括 Sidecar 注入和通过 Kiali 进行监控。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/pod-security-policies.md new file mode 100644 index 00000000000..2ceed9acdce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/pod-security-policies.md @@ -0,0 +1,53 @@ +--- +title: 在具有 Pod 安全策略的情况下å¯ç”¨ Istio +--- + +如果你å¯ç”¨äº†é™åˆ¶æ€§ Pod 安全策略（Pod Security Policy），由于 Istio 需è¦æŸäº›æƒé™æ‰èƒ½è‡ªè¡Œå®‰è£…å’Œç®¡ç† pod 基础设施，因此 Istio å¯èƒ½æ— æ³•æ­£å¸¸è¿è¡Œã€‚在本文中，我们将é…置一个为 Istio å¯ç”¨äº† PSP 的集群，并设置 Istio CNI æ’件。 + +Istio CNI æ’件ä¸å†è¦æ±‚æ¯ä¸ªåº”用 pod å…·æœ‰ç‰¹æƒ `NET_ADMIN` 容器。如需更多信æ¯ï¼Œè¯·å‚阅 [Istio CNI æ’件文档](https://istio.io/docs/setup/additional-setup/cni)。请注æ„，[Istio CNI æ’件处于 alpha 阶段](https://istio.io/about/feature-stages/)。 + +:::note 先决æ¡ä»¶ï¼š + +- 集群必须是 RKE Kubernetes 集群。 +- 必须使用默认 PodSecurityPolicy 创建集群。 + +è¦åœ¨ä½¿ç”¨ Rancher UI 创建 Kubernetes 集群时å¯ç”¨ Pod 安全策略支æŒï¼Œè¯·è½¬åˆ°é«˜çº§é€‰é¡¹ã€‚在 Pod 安全策略支æŒä¸­ï¼Œå•å‡»å¯ç”¨ï¼Œç„¶åŽé€‰æ‹©ä¸€ä¸ªé»˜è®¤çš„ pod 安全策略。 + +::: + +1. [å°† PodSecurityPolicy 设置为ä¸å—é™åˆ¶](#1-å°†-podsecuritypolicy-设置为ä¸å—é™åˆ¶) +2. [å¯ç”¨ CNI](#2-å¯ç”¨-cni) +3. [éªŒè¯ CNI 是å¦æ­£å¸¸å·¥ä½œ](#3-验è¯-cni-是å¦æ­£å¸¸å·¥ä½œ) + +### 1. å°† PodSecurityPolicy 设置为ä¸å—é™åˆ¶ + +ä¸å—é™åˆ¶çš„ PSP 支æŒå®‰è£… Istio。 + +在安装 Istio 的项目或计划安装 Istio 的项目中，将 PSP 设置为 `unrestricted`。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到**项目: System**，然åŽé€‰æ‹© **â‹® > 编辑é…ç½®**。 +1. å°† Pod 安全策略选项更改为ä¸å—é™åˆ¶ï¼Œç„¶åŽå•å‡»**ä¿å­˜**。 + +### 2. å¯ç”¨ CNI + +通过 **Apps** 安装或å‡çº§ Istio 时： + +1. å•å‡»**组件**。 +2. 选中**å¯ç”¨ CNI**æ—边的框。 +3. å®Œæˆ Istio 的安装或å‡çº§ã€‚ + +你也å¯ä»¥é€šè¿‡ç¼–辑 `values.yaml` æ¥å¯ç”¨ CNI： + +``` +istio_cni.enabled: true +``` + +在集群中å¯ç”¨ CNI åŽï¼ŒIstio 应该能æˆåŠŸå®‰è£…。 + +### 3. éªŒè¯ CNI 是å¦æ­£å¸¸å·¥ä½œ + +通过部署[示例应用](https://istio.io/latest/docs/examples/bookinfo/)或部署你自己的应用，æ¥éªŒè¯ CNI 是å¦æ­£å¸¸å·¥ä½œã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/project-network-isolation.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/project-network-isolation.md new file mode 100644 index 00000000000..00b16856358 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/project-network-isolation.md @@ -0,0 +1,21 @@ +--- +title: 项目网络隔离的其他步骤 +--- + +在集群中： + +- ä½ åŒæ—¶ä½¿ç”¨äº† Canal 网络æ’件与 Rancher 2.5.8 之å‰ç‰ˆæœ¬ï¼Œæˆ–者åŒæ—¶ä½¿ç”¨äº† Rancher 2.5.8+ 以åŠä»»æ„支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE 网络æ’件（例如 Canal 或 Cisco ACI æ’件）。 +- å¯ç”¨äº†é¡¹ç›®ç½‘络隔离选项。 +- 安装了 Istio Ingress 模å—。 + +默认情况下，Istio Ingress Gateway pod 无法将入å£æµé‡é‡å®šå‘到工作负载。这是因为安装了 Istio 的命å空间无法访问所有命å空间。为此你有两个选项。 + +第一个选项是在需è¦è®© Istio 控制入å£çš„æ¯ä¸ªå‘½å空间中添加一个新的网络策略。你的策略需è¦åŒ…括以下几行： + +``` +- podSelector: + matchLabels: + app: istio-ingressgateway +``` + +第二个选项是将 `istio-system` 命å空间移动到 `system` 项目中，默认情况下该项目被排除在网络隔离之外。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md new file mode 100644 index 00000000000..5a40d032482 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md @@ -0,0 +1,120 @@ +--- +title: 选择器和抓å–é…ç½® +--- + +Monitoring 应用设置了 `prometheus.prometheusSpec.ignoreNamespaceSelectors=false`，å³åœ¨é»˜è®¤æƒ…况下跨所有命å空间进行监控。 + +这样，你å¯ä»¥æŸ¥çœ‹éƒ¨ç½²åœ¨å…·æœ‰ `istio-injection=enabled` 标签的命å空间中的资æºçš„æµé‡ã€æŒ‡æ ‡å’Œå›¾ã€‚ + +如果你想将 Prometheus é™åˆ¶ä¸ºç‰¹å®šçš„命å空间，请设置 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。完æˆæ­¤æ“作åŽï¼Œä½ éœ€è¦æ·»åŠ å…¶ä»–é…ç½®æ¥ç»§ç»­ç›‘控你的资æºã€‚ + + +### 通过将 ignoreNamespaceSelectors 设置为 True æ¥é™åˆ¶å¯¹ç‰¹å®šå‘½å空间的监控 + +è¦é™åˆ¶å¯¹ç‰¹å®šå‘½å空间的监控，你需è¦ç¼–辑 `ignoreNamespaceSelectors` Helm Chart 选项。你å¯ä»¥åœ¨å®‰è£…或å‡çº§ Monitoring Helm Chart æ—¶é…置此选项： + +1. 安装或å‡çº§ Monitoring Helm Chart 时，编辑 values.yml 并设置 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。 +1. 完æˆå®‰è£…或å‡çº§ã€‚ + +**结果**：Prometheus 将仅用于特定命å空间。æ¢è¨€ä¹‹ï¼Œä½ éœ€è¦è®¾ç½®ä»¥ä¸‹é…置之一æ‰èƒ½ç»§ç»­åœ¨å„ç§ä»ªè¡¨æ¿ä¸­æŸ¥çœ‹æ•°æ®ã€‚ + +### 让 Prometheus 检测其他命åç©ºé—´ä¸­çš„èµ„æº + +如果设置了 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`，则有两ç§æ–¹æ³•è®© Prometheus 检测其他命å空间中的资æºï¼š + +- **监控特定的命å空间**：在命å空间中添加一个 ServiceMonitor 或 PodMonitor 以åŠè¦æŠ“å–的目标。 +- **跨命å空间监控**：将 `additionalScrapeConfig` 添加到你的 rancher-monitoring 实例，从而抓å–所有命å空间中的所有目标。 + +### 监控特定命å空间：创建 ServiceMonitor 或 PodMonitor + +此选项用于定义在特定命å空间中è¦ç›‘控的æœåŠ¡æˆ– pod。 + +å¯ç”¨æ€§æƒè¡¡æŒ‡çš„是，由于你无法跨命å空间进行监控，因此你必须为æ¯ä¸ªå‘½å空间创建 ServiceMonitor 或 PodMonitor。 + +:::note 先决æ¡ä»¶ï¼š + +为 `` 定义 ServiceMonitor 或 PodMonitor。下é¢æ供了一个 ServiceMonitor 示例。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œæ‰“å¼€ kubectl shell。 +1. 如果 ServiceMonitor 或 PodMonitor 文件存储在本地集群中，请è¿è¡Œ `kubectl create -f .yaml`。 +1. 如果 ServiceMonitor 或 PodMonitor 没有存储在本地，请è¿è¡Œ `cat<< EOF | kubectl apply -f -`，将文件内容粘贴到终端，然åŽè¿è¡Œ ​​`EOF` æ¥å®Œæˆå‘½ä»¤ã€‚ +1. è¿è¡Œ `kubectl label namespace istio-injection=enabled` æ¥å¯ç”¨ Envoy sidecar 注入。 + +**结果**：Prometheus å¯ä»¥æŠ“å– ``。 + +
Istio 代ç†çš„ ServiceMonitor 示例
+ +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: envoy-stats-monitor + namespace: istio-system + labels: + monitoring: istio-proxies +spec: + selector: + matchExpressions: + - {key: istio-prometheus-ignore, operator: DoesNotExist} + namespaceSelector: + any: true + jobLabel: envoy-stats + endpoints: + - path: /stats/prometheus + targetPort: 15090 + interval: 15s + relabelings: + - sourceLabels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - action: labeldrop + regex: "__meta_kubernetes_pod_label_(.+)" + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: namespace + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: pod_name +``` + +### 跨命å空间监控：将 ignoreNamespaceSelectors 设置为 False + +此设置为 Prometheus æä¾›é¢å¤–的抓å–é…ç½®æ¥å®žçŽ°è·¨å‘½å空间监控。 + +å¯ç”¨æ€§æƒè¡¡æŒ‡çš„是 Prometheus 的所有 `additionalScrapeConfigs` 都维护在一个 Secret 中。如果在安装 Istio 之å‰å·²ç»ä½¿ç”¨ additionalScrapeConfigs 部署了监控，å‡çº§å¯èƒ½ä¼šå˜å¾—困难。 + +1. 安装或å‡çº§ Monitoring Helm Chart 时，编辑 values.yml 并将 `prometheus.prometheusSpec.additionalScrapeConfigs` 数组设置为下方的**其它抓å–é…ç½®**。 +1. 完æˆå®‰è£…或å‡çº§ã€‚ + +**结果**：Promethe 会抓å–所有带有 `istio-injection=enabled` 标签的命å空间。 + +
其它抓å–é…ç½®
+ +```yaml +- job_name: 'istio/envoy-stats' + scrape_interval: 15s + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:15090 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/cpu-and-memory-allocations.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/cpu-and-memory-allocations.md new file mode 100644 index 00000000000..3a19c21dcc5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/cpu-and-memory-allocations.md @@ -0,0 +1,63 @@ +--- +title: CPU å’Œå†…å­˜åˆ†é… +--- + +本文介ç»æˆ‘们建议在集群中分é…ç»™ Istio 组件的最少计算资æºã€‚ + +æ¯ä¸ªç»„件的 CPU 和内存分é…是[å¯é…ç½®](#é…置资æºåˆ†é…)的。 + +在å¯ç”¨ Istio 之å‰ï¼Œå»ºè®®ä½ å…ˆç¡®è®¤ä½ çš„ Rancher worker 节点是å¦æœ‰è¶³å¤Ÿçš„ CPU 和内存æ¥è¿è¡Œ Istio 的所有组件。 + +:::tip + +在规模较大的部署中，我们强烈建议通过为æ¯ä¸ª Istio 组件添加节点选择器，æ¥å°†åŸºç¡€è®¾æ–½æ”¾ç½®åœ¨é›†ç¾¤ä¸­çš„专用节点上。 + +::: + +下表总结了æ¯ä¸ªæ ¸å¿ƒ Istio 组件推èé…置的 CPU 和内存的最低资æºè¯·æ±‚å’Œé™åˆ¶ã€‚ + +Kubernetes 中的资æºè¯·æ±‚指的是，除éžè¯¥èŠ‚点至少具有指定数é‡çš„å¯ç”¨å†…存和 CPU，å¦åˆ™å·¥ä½œè´Ÿè½½ä¸ä¼šéƒ¨ç½²åœ¨èŠ‚点上。如果工作负载超过 CPU 或内存的é™åˆ¶ï¼Œåˆ™å¯ä»¥å°†å…¶ä»ŽèŠ‚点中终止或驱é€ã€‚有关管ç†å®¹å™¨èµ„æºé™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)。 + +| 工作负载 | CPU - 请求 | 内存 - 请求 | CPU - é™åˆ¶ | 内存 - é™åˆ¶ | +|----------------------|---------------|------------|-----------------|-------------------| +| å…¥å£ç½‘å…³ | 100m | 128mi | 2000m | 1024mi | +| 出å£ç½‘å…³ | 100m | 128mi | 2000m | 1024mi | +| istiod | 500m | 2048mi | 没有é™åˆ¶ | 没有é™åˆ¶ | +| proxy | 10m | 10mi | 2000m | 1024mi | +| **总计：** | **710m** | **2314Mi** | **6000m** | **3072Mi** | + +## é…置资æºåˆ†é… + +ä½ å¯ä»¥ä¸ºæ¯ç§ç±»åž‹çš„ Istio 组件å•ç‹¬é…置资æºåˆ†é…。本节介ç»äº†æ¯ä¸ªç»„件默认分é…的资æºã€‚ + +为了更轻æ¾åœ°å°†å·¥ä½œè´Ÿè½½è°ƒåº¦åˆ°èŠ‚点，集群管ç†å‘˜å¯ä»¥é™ä½Žç»„件的 CPU 和内存资æºè¯·æ±‚。默认 CPU 和内存分é…是我们推è的最å°å€¼ã€‚ + +关于 Istio é…置的更多信æ¯ï¼Œè¯·å‚阅 [Istio 官方文档](https://istio.io/)。 + +è¦é…置分é…ç»™ Istio 组件的资æºï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `istio-system` 命å空间。在æŸä¸ª Istio 工作负载中（例如 `rancher-istio`），点击**â‹® > 编辑/å‡çº§**。 +1. 点击**å‡çº§**，然åŽé€šè¿‡æ›´æ”¹ values.yaml 或添加[覆盖文件](../../pages-for-subheaders/configuration-options.md#覆盖文件)æ¥ç¼–辑基本组件。有关编辑覆盖文件的更多信æ¯ï¼Œè¯·å‚阅[本节](cpu-and-memory-allocations.md#编辑覆盖文件)。 +1. 更改 CPU 或内存分é…ã€è°ƒåº¦å„个组件的节点，或节点容å¿åº¦ã€‚ +1. 点击**å‡çº§**。然åŽï¼Œæ›´æ”¹å°±èƒ½å¯ç”¨ã€‚ + +**结果**：已更新 Istio 组件的资æºåˆ†é…。 + +### 编辑覆盖文件 + +覆盖文件å¯ä»¥åŒ…å« [Istio Operator 规范](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/#IstioOperatorSpec)中的任æ„å€¼ã€‚åŒ…å« Istio 应用的覆盖文件åªæ˜¯è¦†ç›–文件潜在é…置的一个示例。 + +åªè¦æ–‡ä»¶åŒ…å« `kind: IstioOperator` 且 YAML 选项有效，文件就å¯ä»¥ç”¨ä½œè¦†ç›–。 + +在 Istio 应用æ供的示例覆盖文件中，以下部分能让你更改 Kubernetes 资æºï¼š + +``` +# k8s: +# resources: +# requests: +# cpu: 200m +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/disable-istio.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/disable-istio.md new file mode 100644 index 00000000000..66e0b1eb9b1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/disable-istio.md @@ -0,0 +1,42 @@ +--- +title: ç¦ç”¨ Istio +--- + +本文介ç»å¦‚何在集群中å¸è½½ Istio，以åŠå¦‚何在命å空间或工作负载中ç¦ç”¨ Istio。 + +## 在集群中å¸è½½ Istio + +è¦å¸è½½ Istio： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Installed Apps**。 +1. 在 `istio-system` 命å空间中，转到 `rancher-istio` 并å•å‡» **â‹® > 删除**。 +1. 删除 `rancher-istio` åŽï¼Œé€‰æ‹© `istio-system` 命å空间中所有剩余的应用，然åŽå•å‡»**删除**。 + +**结果**：已删除集群中的 `rancher-istio` 应用。Istio sidecar ä¸èƒ½éƒ¨ç½²åœ¨é›†ç¾¤ä¸­çš„任何工作负载上。 + +:::note + +ä½ ä¸èƒ½å†ç¦ç”¨å’Œé‡æ–°å¯ç”¨ä½ çš„ Istio 安装。如果你想ä¿å­˜è®¾ç½®ä»¥ä¾›å°†æ¥çš„安装使用，请查看并ä¿å­˜å„个 YAML，以便在之åŽçš„安装中å‚考/é‡å¤ä½¿ç”¨ã€‚ + +::: + +**å¸è½½ç–‘难解答**：如果你没有按照å¸è½½æ­¥éª¤æ“作，则å¯èƒ½ä¼šåœ¨å¸è½½è¿‡ç¨‹ä¸­é‡åˆ°ä»¥ä¸‹è­¦å‘Šï¼š + +`Error: uninstallation completed with 1 error(s): unable to build kubernetes objects for delete: unable to recognize "": no matches for kind "MonitoringDashboard" in version "monitoring.kiali.io/v1alpha1"` + +è¿™å¯èƒ½æ„味ç€å‡ ç§æƒ…况。第一ç§æƒ…况是你选择了 `istio-system` 命å空间中的所有应用并åŒæ—¶åˆ é™¤äº†å®ƒä»¬ï¼Œå¦ä¸€ç§æƒ…况是你在删除 `rancher-istio` Chart 之å‰åˆ é™¤äº†`rancher-istio` Chart ä¾èµ–项。由于å¸è½½æœªæ­£ç¡®å®Œæˆï¼Œä½ å°†éœ€è¦æ‰‹åŠ¨æ¸…ç† `istio-system` 命å空间中剩余的资æºã€‚如果ä¸æƒ³è¿›è¡Œæ‰‹åŠ¨æ¸…ç†ï¼Œä½ å¯ä»¥é‡æ–°å®‰è£… `rancher-istio`，然åŽæŒ‰ç…§æ­£ç¡®çš„顺åºå¸è½½å®ƒã€‚ + +## 在命å空间中ç¦ç”¨ Istio + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦å¯ç”¨ Istio 的命å空间，然åŽå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。或者，你也å¯ä»¥å•å‡»å‘½å空间，然åŽåœ¨å‘½å空间详情页é¢ä¸Šï¼Œå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。 + +**结果**：如果工作负载部署到此命å空间，它们将没有 Istio sidecar。 + +## 从工作负载中移除 Istio Sidecar + +在命å空间中ç¦ç”¨ Istio，然åŽé‡æ–°éƒ¨ç½²å…¶ä¸­çš„工作负载。这些工作负载将在没有 Istio sidecar 的情况下部署。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/rbac-for-istio.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/rbac-for-istio.md new file mode 100644 index 00000000000..ba56d0ff9b9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/istio/rbac-for-istio.md @@ -0,0 +1,43 @@ +--- +title: RBAC +--- + +本文介ç»è®¿é—® Istio 功能所需的æƒé™ã€‚ + +Rancher Istio Chart 安装了三个 `ClusterRole`。 + +## Cluster-Admin 访问 + +默认情况下，åªæœ‰å…·æœ‰ `cluster-admin` `ClusterRole` 的用户å¯ä»¥ï¼š + +- 在集群中安装 Istio 应用。 +- 为 Istio é…置资æºåˆ†é…。 + + +## Admin å’Œ Edit æƒé™ + +默认情况下，åªæœ‰ Admin å’Œ Edit 角色å¯ä»¥ï¼š + +- 为命å空间å¯ç”¨å’Œç¦ç”¨ Istio sidecar 自动注入。 +- å°† Istio sidecar 添加到工作负载。 +- 查看集群的æµé‡æŒ‡æ ‡å’Œæµé‡å›¾ã€‚ +- é…ç½® Istio 的资æºï¼ˆä¾‹å¦‚网关ã€ç›®æ ‡è§„则或虚拟æœåŠ¡ï¼‰ã€‚ + +## Kubernetes 默认角色的默认æƒé™æ‘˜è¦ + +Istio 创建了三个 `ClusterRole`，并将 Istio CRD 访问æƒé™æ·»åŠ åˆ°ä»¥ä¸‹é»˜è®¤ K8s `ClusterRole`： + +| Chart 创建的 ClusterRole | 默认 K8s ClusterRole | Rancher 角色 | + ------------------------------:| ---------------------------:|---------:| +| `istio-admin` | admin | 项目所有者 | +| `istio-edit` | edit | 项目æˆå‘˜ | +| `istio-view` | view | åªè¯» | + +Rancher 将继续使用 cluster-ownerã€cluster-memberã€project-ownerã€project-member 等作为角色å称，但会使用默认角色æ¥ç¡®å®šè®¿é—®æƒé™ã€‚æ¯ä¸ªé»˜è®¤çš„ K8s `ClusterRole` 都有ä¸åŒçš„ Istio CRD æƒé™ä»¥åŠå¯ä»¥æ‰§è¡Œçš„ K8s æ“作，包括 Create (C)，Get (G)，List (L)，Watch (W)，Update (U)，Patch (P)，Delete (D) å’Œ All (*)。 + + +| CRD | Admin | Edit | View | +|----------------------------| ------| -----| ----- +|
  • `config.istio.io`
    • `adapters`
    • `attributemanifests`
    • `handlers`
    • `httpapispecbindings`
    • `httpapispecs`
    • `instances`
    • `quotaspecbindings`
    • `quotaspecs`
    • `rules`
    • `templates`
| GLW | GLW | GLW | +|
  • `networking.istio.io`
    • `destinationrules`
    • `envoyfilters`
    • `gateways`
    • `serviceentries`
    • `sidecars`
    • `virtualservices`
    • `workloadentries`
| * | * | GLW | +|
  • `security.istio.io`
    • `authorizationpolicies`
    • `peerauthentications`
    • `requestauthentications`
| * | * | GLW | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md new file mode 100644 index 00000000000..ab7f461150a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md @@ -0,0 +1,75 @@ +--- +title: Flows å’Œ ClusterFlows +--- + +有关如何é…ç½® `Flow` å’Œ `ClusterFlow` 的完整详细信æ¯ï¼Œè¯·å‚阅 [Logging Operator 文档](https://kube-logging.github.io/docs/configuration/flow/)。 + +有关如何解决 Logging 缓冲区的内存问题，请å‚阅 [Rancher 与 Logging æœåŠ¡çš„集æˆï¼šæ•…障排除](../../../pages-for-subheaders/logging.md#日志缓冲区导致-pod-过载)。 + +## Flows + +`Flow` 定义è¦æ”¶é›†å’Œè¿‡æ»¤å“ªäº›æ—¥å¿—，以åŠå°†æ—¥å¿—å‘é€åˆ°å“ªä¸ª Output。 + +`Flow` 是一个命å空间资æºã€‚æ¢è¨€ä¹‹ï¼Œåªæœ‰éƒ¨ç½²äº†è¯¥ Flow 的命å空间日志æ‰èƒ½è¢« `Flow` 收集。 + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `Flow`。 + +有关 `Flow` 自定义资æºçš„更多详细信æ¯ï¼Œè¯·å‚阅 [FlowSpec](https://kube-logging.github.io/docs/configuration/crds/v1beta1/flow_types/)。 + +### Matches + +匹é…语å¥ç”¨äºŽé€‰æ‹©ä»Žå“ªäº›å®¹å™¨ä¸­æ‹‰å–日志。 + +ä½ å¯ä»¥æŒ‡å®š match 语å¥ï¼Œç„¶åŽæ ¹æ® Kubernetes 标签ã€å®¹å™¨å’Œä¸»æœºåæ¥é€‰æ‹©æˆ–排除日志。匹é…语å¥ä¼šæŒ‰ç…§å®šä¹‰å’Œå¤„ç†çš„顺åºè¿›è¡Œè¯„估，直到应用了第一个匹é…的选择/排除规则。 + +ä½ å¯ä»¥é€šè¿‡å¡«å†™ Rancher UI 中的 `Flow` 或 `ClusterFlow` 表å•æ¥é…置匹é…。 + +使用 match 语å¥çš„详细示例，请å‚阅[日志路由的官方文档](https://kube-logging.github.io/docs/configuration/log-routing/)。 + +### Filters + +ä½ å¯ä»¥åœ¨ `Flow` 中定义一个或多个过滤器。过滤器å¯ä»¥å¯¹æ—¥å¿—执行å„ç§æ“作，例如，添加其他数æ®ã€è½¬æ¢æ—¥å¿—或解æžè®°å½•ä¸­çš„值。`Flow` 中的过滤器会按定义的顺åºåº”用。 + +有关 Logging Operator 支æŒçš„过滤器列表，请å‚阅 [Fluentd 过滤器的官方文档](https://kube-logging.github.io/docs/configuration/plugins/filters/)。 + +过滤器需è¦åœ¨ YAML 中é…置。 + +### Outputs + +æ­¤ `Output` 会接收æ¥è‡ª `Flow` 的日志。由于 `Flow` 是一个命å空间资æºï¼Œå› æ­¤ `Output` 必须与 `Flow` ä½äºŽç›¸åŒçš„命å空间中。 + +在 Rancher UI 中填写 `Flow` 或 `ClusterFlow` 表å•æ—¶ï¼Œä½ å¯ä»¥å¼•ç”¨`Output`。 + +## ClusterFlows + +为 `ClusterFlow` é…置匹é…ã€è¿‡æ»¤å™¨å’Œ `Output` çš„æ–¹å¼ä¸Ž `Flow` çš„é…置方å¼ç›¸åŒã€‚主è¦åŒºåˆ«åœ¨äºŽ `ClusterFlow` 是集群级别的，并且å¯ä»¥è·¨æ‰€æœ‰å‘½å空间é…置日志收集。 + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `ClusterFlow`。 + +`ClusterFlow` 选择集群中所有命å空间的日志åŽï¼Œé›†ç¾¤çš„日志会被收集并记录到所选的 `ClusterOutput`。 + +## YAML 示例 + +以下示例 `Flow` 转æ¢äº†é»˜è®¤å‘½å空间的日志消æ¯ï¼Œå¹¶å°†æ—¥å¿—å‘é€åˆ° S3 `Output`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: flow-sample + namespace: default +spec: + filters: + - parser: + remove_key_name_field: true + parse: + type: nginx + - tag_normaliser: + format: ${namespace_name}.${pod_name}.${container_name} + localOutputRefs: + - s3-output + match: + - select: + labels: + app: nginx +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md new file mode 100644 index 00000000000..1ad5b65f48b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md @@ -0,0 +1,295 @@ +--- +title: Outputs å’Œ ClusterOutputs +--- + +有关如何é…ç½® `Flow` å’Œ `ClusterFlow` 的完整详细信æ¯ï¼Œè¯·å‚阅 [Logging Operator 文档](https://kube-logging.github.io/docs/configuration/flow/)。 + +有关如何解决 Logging 缓冲区的内存问题，请å‚阅 [Rancher 与 Logging æœåŠ¡çš„集æˆï¼šæ•…障排除](../../../pages-for-subheaders/logging.md#日志缓冲区导致-pod-过载)。 + +## Outputs + +`Output` 资æºå®šä¹‰äº†ä½ çš„ `Flow` å¯ä»¥å‘é€æ—¥å¿—消æ¯çš„ä½ç½®ã€‚`Output` 是 Logging `Flow` 的最åŽé˜¶æ®µã€‚ + +`Output` 是命å空间资æºï¼Œæ¢è¨€ä¹‹ï¼Œåªæœ‰åŒä¸€å‘½å空间内的 `Flow` å¯ä»¥è®¿é—®å®ƒã€‚ + +ä½ å¯ä»¥åœ¨è¿™äº›å®šä¹‰ä¸­ä½¿ç”¨å¯†æ–‡ï¼Œä½†è¿™äº›å¯†æ–‡ä¹Ÿå¿…é¡»ä½äºŽåŒä¸€å‘½å空间中。 + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `Output`。 + +有关 `Output` 自定义资æºçš„更多详细信æ¯ï¼Œè¯·å‚阅 [OutputSpec](https://kube-logging.github.io/docs/configuration/crds/v1beta1/output_types/)。 + +Rancher UI æ供了用于é…置以下类型 `Output` 的表å•ï¼š + +- Amazon ElasticSearch +- Azure Storage +- Cloudwatch +- Datadog +- Elasticsearch +- File +- Fluentd +- GCS +- Kafka +- Kinesis Stream +- LogDNA +- LogZ +- Loki +- New Relic +- Splunk +- SumoLogic +- Syslog + +Rancher UI æ供了用于é…ç½® `Output` 类型ã€ç›®æ ‡å’Œè®¿é—®å‡­è¯ï¼ˆå¦‚果适用）的表å•ã€‚ + +有关 Logging Operator 支æŒçš„日志æ’件é…置示例，请å‚阅 [Logging Operator 文档](https://kube-logging.github.io/docs/configuration/plugins/outputs/)。 + +## ClusterOutputs + +`ClusterOutput` 定义了一个没有命å空间é™åˆ¶çš„ `Output`。åªæœ‰åœ¨ä¸Ž Logging Operator 部署在åŒä¸€å‘½å空间中时，它æ‰èƒ½ç”Ÿæ•ˆã€‚ + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `ClusterOutput`。 + +有关 `ClusterOutput` 自定义资æºçš„更多详细信æ¯ï¼Œè¯·å‚阅 [ClusterOutput](https://kube-logging.github.io/docs/configuration/crds/v1beta1/clusteroutput_types/)。 + +## YAML 示例 + +安装 Logging åŽï¼Œä½ å¯ä»¥å‚考以下示例æ¥æž„建你自己的 Logging æµæ°´çº¿ï¼š + +- [ClusterOutput 设为 ElasticSearch](#clusteroutput-设为-elasticsearch) +- [Output 设为 Splunk](#output-设为-splunk) +- [Output 设为 Syslog](#output-设为-syslog) +- [ä¸æ”¯æŒçš„ Output](#ä¸æ”¯æŒçš„-output) + +### ClusterOutput 设为 ElasticSearch + +å‡è®¾ä½ æƒ³å°†é›†ç¾¤ä¸­çš„所有日志å‘é€åˆ° `elasticsearch` 集群。首先，创建一个集群 `Output`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-es" + namespace: "cattle-logging-system" +spec: + elasticsearch: + host: elasticsearch.example.com + port: 9200 + scheme: http +``` + +在与 Operator 相åŒçš„命å空间（`cattle-logging-system`）中，我们创建了这个 `ClusterOutput`（没有 elasticsearch é…置）。æ¯æ¬¡åˆ›å»º `ClusterFlow` 或 `ClusterOutput` 时，我们都必须将其放在 `cattle-logging-system` 命å空间中。 + +é…置日志的目的ä½ç½®åŽï¼Œæˆ‘们å¯ä»¥å°è¯•å°†æ‰€æœ‰æ—¥å¿—都é…置到该 `ClusterOutput`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: "cattle-logging-system" +spec: + globalOutputRefs: + - "example-es" +``` + +现在，我们应该能看到é…置的包å«æ—¥å¿—的索引。 + + +### Output 设为 Splunk + +有时候，你的应用程åºå›¢é˜Ÿå¯èƒ½åªæƒ³å°†æŸä¸ªå‘½å空间的日志å‘é€åˆ° `splunk` æœåŠ¡å™¨ã€‚对于这ç§æƒ…况，你å¯ä»¥ä½¿ç”¨å‘½å空间范围的 `Output` å’Œ `Flow`。 + +在开始之å‰ï¼Œå…ˆè®¾ç½®è¯¥å›¢é˜Ÿçš„åº”ç”¨ç¨‹åº `coolapp`。 + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: devteam +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coolapp + namespace: devteam + labels: + app: coolapp +spec: + replicas: 2 + selector: + matchLabels: + app: coolapp + template: + metadata: + labels: + app: coolapp + spec: + containers: + - name: generator + image: paynejacob/loggenerator:latest +``` + +`coolapp` è¿è¡Œæ—¶ï¼Œæˆ‘们将使用与创建 `ClusterOutput` 时类似的路径。但是，我们ä¸ä½¿ç”¨ `ClusterOutput`，而是在应用程åºçš„命å空间中创建 `Output`。 + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Output +metadata: + name: "devteam-splunk" + namespace: "devteam" +spec: + splunkHec: + hec_host: splunk.example.com + hec_port: 8088 + protocol: http +``` + +然åŽï¼Œå†æ¬¡ä¸º `Output` æ供一些日志： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: "devteam-logs" + namespace: "devteam" +spec: + localOutputRefs: + - "devteam-splunk" +``` + + +### Output 设为 Syslog + +å‡è®¾ä½ æƒ³å°†é›†ç¾¤ä¸­çš„所有日志å‘é€åˆ° `syslog` æœåŠ¡å™¨ã€‚首先，我们创建一个 `ClusterOutput`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-syslog" + namespace: "cattle-logging-system" +spec: + syslog: + buffer: + timekey: 30s + timekey_use_utc: true + timekey_wait: 10s + flush_interval: 5s + format: + type: json + app_name_field: test + host: syslog.example.com + insecure: true + port: 514 + transport: tcp +``` + +é…置日志的目的ä½ç½®åŽï¼Œæˆ‘们å¯ä»¥å°è¯•å°†æ‰€æœ‰æ—¥å¿—都é…置到该 `Output`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: cattle-logging-system +spec: + globalOutputRefs: + - "example-syslog" +``` + +### ä¸æ”¯æŒçš„ Output + +对于最åŽä¸€ä¸ªç¤ºä¾‹ï¼Œæˆ‘们创建一个 `Output` æ¥å°†æ—¥å¿—写入到ä¸æ˜¯å¼€ç®±å³ç”¨çš„目标ä½ç½®ï¼š + +:::note Syslog 注æ„事项： + +`Syslog` 是å—支æŒçš„ `Output`。但是，此示例ä»æ供了ä¸å—支æŒçš„æ’件概述。 + +::: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: syslog-config + namespace: cattle-logging-system +type: Opaque +stringData: + fluent-bit.conf: | + [INPUT] + Name forward + Port 24224 + + [OUTPUT] + Name syslog + InstanceName syslog-output + Match * + Addr syslog.example.com + Port 514 + Cluster ranchers + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentbit-syslog-forwarder + namespace: cattle-logging-system + labels: + output: syslog +spec: + selector: + matchLabels: + output: syslog + template: + metadata: + labels: + output: syslog + spec: + containers: + - name: fluentbit + image: paynejacob/fluent-bit-out-syslog:latest + ports: + - containerPort: 24224 + volumeMounts: + - mountPath: "/fluent-bit/etc/" + name: configuration + volumes: + - name: configuration + secret: + secretName: syslog-config +--- +apiVersion: v1 +kind: Service +metadata: + name: syslog-forwarder + namespace: cattle-logging-system +spec: + selector: + output: syslog + ports: + - protocol: TCP + port: 24224 + targetPort: 24224 +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: all-logs + namespace: cattle-logging-system +spec: + globalOutputRefs: + - syslog +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: syslog + namespace: cattle-logging-system +spec: + forward: + servers: + - host: "syslog-forwarder.cattle-logging-system" + require_ack_response: false + ignore_network_errors_at_startup: false +``` + +现在，我们分解这里的内容。首先，我们创建一个容器 Deployment，该容器具有é¢å¤–çš„ `syslog` æ’件并支æŒè½¬å‘自å¦ä¸€ä¸ª `fluentd` 的日志。接下æ¥ï¼Œæˆ‘们创建一个é…置为 Deployment 转å‘器的 `Output`。然åŽï¼ŒDeployment `fluentd` 会将所有日志转å‘到é…置的 `syslog` 目标。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/logging-architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/logging-architecture.md new file mode 100644 index 00000000000..c7e0510ebe9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/logging-architecture.md @@ -0,0 +1,28 @@ +--- +title: 架构 +--- + +本节介ç»äº† Rancher Logging 应用程åºçš„架构。 + +有关 Logging Operator 工作原ç†çš„更多详细信æ¯ï¼Œè¯·å‚阅[官方文档](https://kube-logging.github.io/docs/#architecture)。 + +### Logging Operator å·¥ä½œåŽŸç† + +Logging Operator 自动部署和é…ç½® Kubernetes 日志æµæ°´çº¿ã€‚它会在æ¯ä¸ªèŠ‚点上部署和é…置一个 Fluent Bit DaemonSet，从而收集节点文件系统中的容器和应用程åºæ—¥å¿—。 + +Fluent Bit 查询 Kubernetes API 并使用 pod 的元数æ®æ¥ä¸°å¯Œæ—¥å¿—，然åŽå°†æ—¥å¿—和元数æ®éƒ½ä¼ è¾“到 Fluentd。Fluentd 会接收和过滤日志并将日志传输到多个`Output`。 + +以下自定义资æºç”¨äºŽå®šä¹‰äº†å¦‚何过滤日志并将日志å‘é€åˆ° `Output`： + +- `Flow` 是一个命å空间自定义资æºï¼Œå®ƒä½¿ç”¨è¿‡æ»¤å™¨å’Œé€‰æ‹©å™¨å°†æ—¥å¿—消æ¯è·¯ç”±åˆ°å¯¹åº”çš„ `Output`。 +- `ClusterFlow` 用于路由集群级别的日志消æ¯ã€‚ +- `Output` 是一个命å空间资æºï¼Œç”¨äºŽå®šä¹‰å‘é€æ—¥å¿—消æ¯çš„ä½ç½®ã€‚ +- `ClusterOutput` 定义了一个所有 `Flow` å’Œ `ClusterFlow` 都å¯ç”¨çš„ `Output`。 + +æ¯ä¸ª `Flow` 都必须引用一个 `Output`，而æ¯ä¸ª `ClusterFlow` 都必须引用一个 `ClusterOutput`。 + +[Logging Operator 文档](https://kube-logging.github.io/docs/#architecture)中的下图显示了新的 Logging 架构： + +
Logging Operator 如何与 Fluentd 和 Fluent Bit 一起使用
+ +![Logging Operator 如何与 Fluentd 一起使用](/img/banzai-cloud-logging-operator.png) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/logging-helm-chart-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/logging-helm-chart-options.md new file mode 100644 index 00000000000..5c08fb1166a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/logging-helm-chart-options.md @@ -0,0 +1,94 @@ +--- +title: rancher-logging Helm Chart 选项 +--- + +### å¯ç”¨/ç¦ç”¨ Windows 节点 Logging + +è¦å¯ç”¨æˆ–ç¦ç”¨ Windows 节点 Logging，你å¯ä»¥åœ¨ `values.yaml` 中将 `global.cattle.windows.enabled` 设置为 `true` 或 `false`。 + +默认情况下，如果使用 Cluster Dashboard UI 在 Windows 集群上安装了 Logging 应用程åºï¼ŒWindows 节点的 Logging 就会å¯ç”¨ã€‚ + +在这ç§æƒ…况下，将 `global.cattle.windows.enabled` 设置为 `false` 会ç¦ç”¨é›†ç¾¤ä¸Šçš„ Windows 节点 Logging。 +ç¦ç”¨åŽï¼Œä»ä¼šä»Ž Windows 集群中的 Linux 节点收集日志。 + +:::note + +ç›®å‰å­˜åœ¨ä¸€ä¸ª[问题](https://github.com/rancher/rancher/issues/32325)，在 Windows 集群中ç¦ç”¨ Windows Logging åŽæ‰§è¡Œ `helm upgrade` æ—¶ä¸ä¼šåˆ é™¤ Windows nodeAgent。在这ç§æƒ…况下，如果已安装 Windows nodeAgents，用户å¯èƒ½éœ€è¦æ‰‹åŠ¨å¸è½½å®ƒä»¬ã€‚ + +::: + +### 使用自定义 Docker 根目录 + +如果使用了自定义 Docker 根目录，你å¯ä»¥åœ¨ `values.yaml` 中设置 `global.dockerRootDirectory`。 + +这将确ä¿åˆ›å»ºçš„ Logging CR 使用你指定的路径，而ä¸æ˜¯ä½¿ç”¨é»˜è®¤çš„ Docker `data-root` ä½ç½®ã€‚ + +请注æ„，这åªå½±å“ Linux 节点。 + +如果集群中有任何 Windows 节点，则更改将ä¸é€‚用于这些节点。 + +### 为自定义污点添加 NodeSelector 设置和容å¿åº¦ + +ä½ å¯ä»¥æ·»åŠ  `nodeSelector` 设置，并通过编辑 Logging Helm Chart 值æ¥æ·»åŠ å…¶ä»–`容å¿åº¦`。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](taints-and-tolerations.md)。 + +### å¯ç”¨ Logging 应用程åºä»¥ä½¿ç”¨ SELinux + +:::note è¦æ±‚： + +Logging v2 已在 RHEL/CentOS 7 å’Œ 8 上使用 SELinux 进行了测试。 + +::: + +[安全增强型 Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) 是对 Linux 的安全增强。被政府机构使用之åŽï¼ŒSELinux å·²æˆä¸ºè¡Œä¸šæ ‡å‡†ï¼Œå¹¶åœ¨ CentOS 7 å’Œ 8 上默认å¯ç”¨ã€‚ + +è¦é…åˆä½¿ç”¨ Logging V2 与 SELinux，我们建议你根æ®[此说明](../../pages-for-subheaders/selinux-rpm.md)安装 `rancher-selinux` RPM。 + +然åŽï¼Œåœ¨å®‰è£… Logging 应用程åºæ—¶ï¼Œåœ¨ `values.yaml` 中将 `global.seLinux.enabled` 更改为 `true`，使 Chart æ”¯æŒ SELinux。 + +### 其他日志æ¥æº + +默认情况下，Rancher 会收集所有类型集群的 [controlplane 组件](https://kubernetes.io/docs/concepts/overview/components/#control-plane-components)å’Œ[节点组件](https://kubernetes.io/docs/concepts/overview/components/#node-components)的日志。 + +在æŸäº›æƒ…况下，Rancher 也能收集其他的日志。 + +下表总结了æ¯ç§èŠ‚点类型å¯ä»¥æ”¶é›†çš„其他日志æ¥æºï¼š + +| 日志æ¥æº | Linux 节点（包括在 Windows 集群中） | Windows 节点 | +| --- | --- | ---| +| RKE | ✓ | ✓ | +| RKE2 | ✓ | | +| K3s | ✓ | | +| AKS | ✓ | | +| EKS | ✓ | | +| GKE | ✓ | | + +è¦å°†æ‰˜ç®¡ Kubernetes çš„æ供商作为é¢å¤–的日志æ¥æºï¼Œåœ¨å®‰è£…或å‡çº§ Logging Helm Chart 时，请å¯ç”¨ **Enable enhanced cloud provider logging** 选项。 + +å¯ç”¨åŽï¼ŒRancher 会收集æ供商开放å¯ç”¨çš„所有其他节点和 controlplane 日志，ä¸åŒæ供商å¯èƒ½æœ‰æ‰€ä¸åŒã€‚ + +如果你已ç»ä½¿ç”¨äº†äº‘æ供商的日志解决方案，例如 AWS CloudWatch 或 Google Cloud Operations Suite（以å‰ç§°ä¸º Stackdriver），由于原生解决方案å¯ä»¥ä¸å—é™åˆ¶åœ°è®¿é—®æ‰€æœ‰æ—¥å¿—，因此你无需å¯ç”¨æ­¤é€‰é¡¹ã€‚ + +### Systemd é…ç½® + +在 Rancher Logging 中，你必须为 K3s å’Œ RKE2 Kubernetes å‘行版é…ç½® `SystemdLogPath`。 + +K3s å’Œ RKE2 Kubernetes å‘行版将日志写入到 journald，它是 systemd çš„å­ç³»ç»Ÿï¼Œç”¨äºŽæ—¥å¿—记录。è¦æ”¶é›†è¿™äº›æ—¥å¿—，你需è¦å®šä¹‰ `systemdLogPath`。默认路径是 `run/log/journal`，但æŸäº› Linux å‘行版ä¸é»˜è®¤ä½¿ç”¨è¯¥è·¯å¾„。例如，Ubuntu 默认使用 `var/log/journal`。è¦ç¡®å®šä½ çš„ `systemdLogPath` é…置，请å‚阅以下步骤。 + +**Systemd é…置步骤：** + +* 在其中一个节点上è¿è¡Œ `cat /etc/systemd/journald.conf | grep -E ^\#?Storage | cut -d"=" -f2`。 +* 如果返回 `persistent`，则你的 `systemdLogPath` 是 `/var/log/journal`。 +* 如果返回 `volatile`，则你的 `systemdLogPath` 是 `/run/log/journal`。 +* 如果返回 `auto`，则检查 `/var/log/journal` 是å¦å­˜åœ¨ã€‚ + * 如果 `/var/log/journal` 存在，则使用 `/var/log/journal`。 + * 如果 `/var/log/journal` ä¸å­˜åœ¨ï¼Œåˆ™ä½¿ç”¨ `/run/log/journal`。 + +:::note 注æ„事项： + +如果返回的值ä¸åŒ…括在上述æ述中，Rancher Logging 将无法收集 controlplane 日志。è¦è§£å†³æ­¤é—®é¢˜ï¼Œä½ éœ€è¦åœ¨æ¯ä¸ª controlplane 节点上执行以下æ“作。 + +* 在 journald.conf 中设置 `Storage=volatile`。 +* é‡å¯ä¸»æœºã€‚ +* å°† `systemdLogPath` 设置为 `/run/log/journal`。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/rbac-for-logging.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/rbac-for-logging.md new file mode 100644 index 00000000000..2959a9f7946 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/rbac-for-logging.md @@ -0,0 +1,23 @@ +--- +title: Logging çš„ RBAC +--- + +Rancher Logging 有两个角色，分别是 `logging-admin` å’Œ `logging-view`。 + +- `logging-admin` å…许用户完全访问命å空间的 `Flow` å’Œ `Output`。 +- `logging-view` å…许用户*查看*命å空间的 `Flow` å’Œ `Output`ï¼Œä»¥åŠ `ClusterFlow` å’Œ `ClusterOutput`。 + +:::note 为什么选择一个角色而ä¸æ˜¯å¦ä¸€ä¸ªè§’色？ + +`ClusterFlow` å’Œ `ClusterOutput` 资æºçš„编辑æƒé™éžå¸¸å¼ºå¤§ã€‚任何拥有该æƒé™çš„用户都能编辑集群中的所有日志。 + +::: + +在 Rancher 中，集群管ç†å‘˜è§’色是唯一å¯ä»¥å®Œå…¨è®¿é—®æ‰€æœ‰ `rancher-logging` 资æºçš„角色。集群æˆå‘˜æ— æ³•ç¼–辑或读å–任何 Logging 资æºã€‚项目所有者和æˆå‘˜å…·æœ‰ä»¥ä¸‹æƒé™ï¼š + +| 项目所有者 | 项目æˆå‘˜ | +--- | --- +| 能够在其项目命å空间中创建命å空间级别的 `Flow` å’Œ `Output` | åªèƒ½æŸ¥çœ‹é¡¹ç›®å‘½å空间中的 `Flow` å’Œ `Output` | +| å¯ä»¥ä»Žé¡¹ç›®å‘½å空间中收集任何日志 | 无法在其项目命å空间中收集任何日志 | + +如果项目所有者和项目æˆå‘˜éœ€è¦ä½¿ç”¨ Logging，他们需è¦åœ¨é¡¹ç›®ä¸­è‡³å°‘有*一个*命å空间。如果没有，他们å¯èƒ½çœ‹ä¸åˆ°é¡¶éƒ¨å¯¼èˆªä¸‹æ‹‰åˆ—表中的 Logging 按钮。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/taints-and-tolerations.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/taints-and-tolerations.md new file mode 100644 index 00000000000..16a6a08fd26 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/logging/taints-and-tolerations.md @@ -0,0 +1,65 @@ +--- +title: 处ç†æ±¡ç‚¹å’Œå®¹å¿åº¦ +--- + +在 Kubernetes 节点上添加污点会导致 pod 排斥在该节点上è¿è¡Œã€‚ + +é™¤éž pod 对该节点的污点具有`容å¿åº¦`（toleration），å¦åˆ™ Pod 将在集群中的其他节点上è¿è¡Œã€‚ + +[污点和容å¿åº¦](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)å¯ä»¥ä¸Ž `PodSpec` 中的 `nodeSelector` [字段](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)一起使用，从而实现*相åçš„*污点效果。 + +`nodeSelector` å¯ä»¥ä½¿ pod 被å¸å¼•åˆ°æŸç±»èŠ‚点。 + +两者都能让 pod 选择在哪个节点上è¿è¡Œã€‚ + +- [Rancher 日志堆栈中的默认实现](#rancher-日志堆栈中的默认实现) +- [为自定义污点添加 NodeSelector 设置和容å¿åº¦](#为自定义污点添加-nodeselector-设置和容å¿åº¦) + + +### Rancher 日志堆栈中的默认实现 + +默认情况下，Rancher 使用 `cattle.io/os=linux` æ¥å°†æ±¡ç‚¹åº”用到所有 Linux 节点，而ä¸å½±å“ Windows 节点。 +日志堆栈 pod 具有针对此污点的`容å¿åº¦`，因此它们能够è¿è¡Œåœ¨ Linux 节点上。 +此外，大多数日志堆栈 pod 仅在 Linux 上è¿è¡Œï¼Œå¹¶æ·»åŠ äº† `nodeSelector` 以确ä¿å®ƒä»¬åœ¨ Linux 节点上è¿è¡Œã€‚ + +此示例 Pod YAML 文件显示了与容å¿åº¦ä¸€èµ·ä½¿ç”¨çš„ nodeSelector： + +```yaml +apiVersion: v1 +kind: Pod +# metadata... +spec: + # containers... + tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + nodeSelector: + kubernetes.io/os: linux +``` + +在上é¢çš„示例中，我们确ä¿äº†çš„ pod 仅在 Linux 节点上è¿è¡Œï¼Œå¹¶ä¸”为所有 Linux 节点上的污点添加了`容å¿åº¦`。 + +ä½ å¯ä»¥å¯¹ Rancher 现有的污点或你自己的自定义污点执行相åŒçš„æ“作。 + +### 为自定义污点添加 NodeSelector 设置和容å¿åº¦ + +如果è¦æ·»åŠ ä½ è‡ªå·±çš„ `nodeSelector` 设置，或者è¦ä¸ºå…¶ä»–污点添加 `容å¿åº¦`，你å¯ä»¥å°†ä»¥ä¸‹å†…容传递给 Chart 的值： + +```yaml +tolerations: + # insert tolerations... +nodeSelector: + # insert nodeSelector... +``` + +这些值会将这两个设置添加到 `fluentd`ã€`fluentbit`å’Œ `logging-operator` 容器中。 +本质上，这些是日志堆栈中所有 pod 的全局设置。 + +但是，如果你想*ä»…*为 `fluentbit` 容器添加容å¿åº¦ï¼Œä½ å¯ä»¥å°†ä»¥ä¸‹å†…容添加到 Chart 的值中： + +```yaml +fluentbit_tolerations: + # insert tolerations list for fluentbit containers only... +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/longhorn.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/longhorn.md new file mode 100644 index 00000000000..ea34d0938ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/longhorn.md @@ -0,0 +1,70 @@ +--- +title: Longhorn - Kubernetes 的云原生分布å¼å—存储 +--- + +[Longhorn](https://longhorn.io/) 是一个轻é‡çº§ã€å¯é ã€æ˜“用的 Kubernetes 分布å¼å—存储系统。 + +Longhorn 是å…费的开æºè½¯ä»¶ã€‚Longhorn 最åˆç”± Rancher Labs å¼€å‘，现在正在作为云原生计算基金会的沙盒项目进行开å‘。它å¯ä»¥é€šè¿‡ Helmã€kubectl 或 Rancher UI 安装在任何 Kubernetes 集群上。有关其架构的更多信æ¯ï¼Œè¯·å‚阅[此处](https://longhorn.io/docs/latest/concepts/)。 + +使用 Longhorn，你å¯ä»¥ï¼š + +- å°† Longhorn å·ç”¨ä½œ Kubernetes 集群中分布å¼æœ‰çŠ¶æ€åº”用程åºçš„æŒä¹…存储 +- 将你的å—存储分区为 Longhorn å·ï¼Œä»¥ä¾¿åœ¨æœ‰æˆ–没有云æ供商的情况下使用 Kubernetes å· +- 跨多个节点和数æ®ä¸­å¿ƒå¤åˆ¶å—存储以æ高å¯ç”¨æ€§ +- 将备份数æ®å­˜å‚¨åœ¨ NFS 或 AWS S3 等外部存储中 +- 创建跨集群ç¾éš¾æ¢å¤å·ï¼Œä»¥ä¾¿ä½¿ç”¨å¦ä¸€ä¸ª Kubernetes 集群中的备份快速æ¢å¤ä¸» Kubernetes é›†ç¾¤ä¸­çš„æ•°æ® +- 计划å·çš„定期快照，并将定期备份调度到 NFS 或兼容 S3 的辅助存储 +- 使用备份æ¥æ¢å¤å· +- 在ä¸ä¸­æ–­æŒä¹…å·çš„情况下å‡çº§ Longhorn + +
Longhorn 仪表æ¿
+ +![Longhorn 仪表æ¿](/img/longhorn-screenshot.png) + +### 使用 Rancher 安装 Longhorn + +1. 满足所有[安装è¦æ±‚](https://longhorn.io/docs/latest/deploy/install/#installation-requirements)。 +1. 转到è¦å®‰è£… Longhorn 的集群。 +1. å•å‡» **Apps**。 +1. å•å‡» **Chart**。 +1. 点击 **Longhorn**。 +1. å¯é€‰ï¼šè¦è‡ªå®šä¹‰åˆå§‹è®¾ç½®ï¼Œè¯·å•å‡» **Longhorn 默认设置**并编辑é…置。如需自定义设置的帮助，请å‚阅 [Longhorn 文档](https://longhorn.io/docs/latest/references/settings/)。 +1. å•å‡»**安装**。 + +**结果**：Longhorn 已部署到 Kubernetes 集群中。 + +### 从 Rancher UI 访问 Longhorn + +1. 转到安装了 Longhorn 的集群。在左侧导航èœå•ä¸­ï¼Œå•å‡» **Longhorn**。 +1. 在此页é¢ä¸Šï¼Œä½ å¯ä»¥ç¼–辑 Longhorn 管ç†çš„ Kubernetes 资æºã€‚è¦æŸ¥çœ‹ Longhorn UI，请å•å‡»**概述**中的 **Longhorn** 按钮。 + +**结果**：你将转到 Longhorn UI，你å¯ä»¥åœ¨é‚£é‡Œç®¡ç† Longhorn å·åŠå…¶åœ¨ Kubernetes 集群中的副本，还å¯ä»¥æŸ¥çœ‹ä½äºŽå¦ä¸€ä¸ª Kubernetes 集群或 S3 中的 Longhorn 存储辅助备份。 + +### 从 Rancher UI å¸è½½ Longhorn + +1. 转到安装了 Longhorn 的集群，然åŽå•å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `longhorn-system` 命å空间并选中 `longhorn` å’Œ `longhorn-crd` 应用程åºæ—边的框。 +1. å•å‡»**删除**并确认**删除**。 + +**结果**：Longhorn 已被å¸è½½ã€‚ + +### GitHub 仓库 + +Longhorn 项目在[此处](https://github.com/longhorn/longhorn)。 + +### 文档 + +Longhorn 文档在[此处](https://longhorn.io/docs/)。 + +### 架构 + +Longhorn 为æ¯ä¸ªå·åˆ›å»ºä¸“用的存储控制器，并在存储在多个节点上的多个副本之间åŒæ­¥å¤åˆ¶è¯¥å·ã€‚ + +存储控制器和副本本身是使用 Kubernetes 编排的。 + +有关其架构的更多信æ¯ï¼Œè¯·å‚阅[此处](https://longhorn.io/docs/latest/concepts/)。 + +
Longhorn 架构
+ +![Longhorn 架构](/img/longhorn-architecture.svg) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md new file mode 100644 index 00000000000..5dd50402e17 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md @@ -0,0 +1,117 @@ +--- +title: å†…ç½®ä»ªè¡¨æ¿ +--- + +## Grafana UI + +ä½ å¯ä»¥ä½¿ç”¨ [Grafana](https://grafana.com/grafana/) 对存储在å„个地方的指标进行查询ã€å¯è§†åŒ–ã€å‘Šè­¦å’Œäº†è§£ã€‚你能与你的团队创建ã€æŽ¢ç´¢å’Œå…±äº«ä»ªè¡¨æ¿ï¼Œå¹¶åŸ¹å…»æ•°æ®é©±åŠ¨çš„文化。 + +è¦æŸ¥çœ‹æ—¶é—´åºåˆ—æ•°æ®å¯è§†åŒ–的默认仪表æ¿ï¼Œè¯·è½¬åˆ° Grafana UI。 + +### 自定义 Grafana + +è¦æŸ¥çœ‹å’Œè‡ªå®šä¹‰ç”¨äºŽæ”¯æŒ Grafana 仪表æ¿çš„ PromQL 查询，请å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md)。 + +### æŒä¹…化 Grafana ä»ªè¡¨æ¿ + +è¦åˆ›å»ºæŒä¹…化 Grafana 仪表æ¿ï¼Œè¯·å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md)。 + +### 访问 Grafana + +有关 Grafana çš„ RBAC，请å‚阅[本节](rbac-for-monitoring.md#grafana-çš„-rbac)。 + + +## Alertmanager UI + +安装 `rancher-monitoring` åŽï¼Œä¼šéƒ¨ç½² Prometheus Alertmanager UI，从而让你查看告警以åŠå½“å‰çš„ Alertmanager é…置。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅 [Alertmanager 工作原ç†](how-monitoring-works.md#3-alertmanager-的工作原ç†)。 + +::: + +### 访问 Alertmanager UI + +Alertmanager UI å¯è®©ä½ æŸ¥çœ‹æœ€è¿‘触å‘的告警。 + +:::note 先决æ¡ä»¶ï¼š + +必须安装 `rancher-monitoring` 应用。 + +::: + +è¦æŸ¥çœ‹ Alertmanager UI： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Alertmanager UI 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Alertmanager**。 + +**结果**：在新选项å¡ä¸­æ‰“å¼€ Alertmanager UI。如需é…置帮助，请å‚阅[官方 Alertmanager 文档](https://prometheus.io/docs/alerting/latest/alertmanager/)。 + +有关在 Rancher 中é…ç½® Alertmanager 的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 + +
Alertmanager UI
+ +![Alertmanager UI](/img/alertmanager-ui.png) + + +### 查看默认告警 + +è¦æŸ¥çœ‹é»˜è®¤è§¦å‘的告警，请转到 Alertmanager UI 并å•å‡»**展开所有组**。 + + +## Prometheus UI + +默认情况下，[kube-state-metrics service](https://github.com/kubernetes/kube-state-metrics) å‘ monitoring 应用æä¾› CPU 和内存利用率的大é‡ä¿¡æ¯ã€‚这些指标涵盖了跨命å空间的 Kubernetes 资æºã€‚æ¢è¨€ä¹‹ï¼Œå¦‚果你è¦æŸ¥çœ‹æœåŠ¡çš„资æºæŒ‡æ ‡ï¼Œä½ æ— éœ€ä¸ºå…¶åˆ›å»ºæ–°çš„ ServiceMonitor。由于数æ®å·²ç»åœ¨æ—¶é—´åºåˆ—æ•°æ®åº“中，你å¯ä»¥è½¬åˆ° Prometheus UI 并è¿è¡Œ PromQL 查询æ¥èŽ·å–ä¿¡æ¯ã€‚ä½ å¯ä»¥ä½¿ç”¨ç›¸åŒçš„查询æ¥é…ç½® Grafana 仪表æ¿ï¼Œä»Žè€Œæ˜¾ç¤ºè¿™äº›æŒ‡æ ‡éšæ—¶é—´å˜åŒ–的图表。 + +è¦æŸ¥çœ‹ Prometheus UI，请安装 `rancher-monitoring`。然åŽï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Prometheus UI 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Prometheus Graph**。 + +
Prometheus Graph UI
+ +![Prometheus Graph UI](/img/prometheus-graph-ui.png) + +### 查看 Prometheus 目标 + +è¦æŸ¥çœ‹ä½ æ­£åœ¨ç›‘控的æœåŠ¡ï¼Œä½ éœ€è¦æŸ¥çœ‹ä½ çš„目标。目标是由 ServiceMonitor å’Œ PodMonitor 设置的，指的是指标抓å–çš„çš„æºã€‚你无需直接编辑目标，但 Prometheus UI å¯ä¸ºä½ æ供抓å–的所有指标æ¥æºçš„概览。 + +è¦æŸ¥çœ‹ Prometheus 目标，请安装 `rancher-monitoring`。然åŽï¼š + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Prometheus 目标的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Prometheus 目标**。 + +
Prometheus UI 中的目标
+ +![Prometheus 目标 UI](/img/prometheus-targets-ui.png) + +### 查看 PrometheusRules + +当你定义规则时（在 PrometheusRule 资æºçš„ RuleGroup 中声明），[规则本身的规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule)会包å«æ ‡ç­¾ï¼Œç„¶åŽ Alertmanager 会使用这些标签æ¥ç¡®å®šæŽ¥æ”¶å¯¹åº”告警的路由。 + +è¦æŸ¥çœ‹ PrometheusRule，请安装 `rancher-monitoring`。然åŽï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Prometheus 规则**。 + +你还å¯ä»¥åœ¨ Prometheus UI 中查看规则： + +
Prometheus UI 中的规则
+ +![PrometheusRules UI](/img/prometheus-rules-ui.png) + +有关在 Rancher 中é…ç½® PrometheusRule 的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md)。 + +## 旧版 UI + +有关在引入 `rancher-monitoring` 应用程åºä¹‹å‰ Rancher v2.2 到 v2.4 中å¯ç”¨ä»ªè¡¨æ¿çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Rancher v2.0—v2.4 文档](/versioned_docs/version-2.0-2.4/explanations/integrations-in-rancher/cluster-monitoring/viewing-metrics.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md new file mode 100644 index 00000000000..3a2ec873a29 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md @@ -0,0 +1,248 @@ +--- +title: Monitoring çš„å·¥ä½œåŽŸç† +--- + +## 1. 架构概述 + +_**下文æ述了数æ®å¦‚何æµç» Monitoring V2 应用程åºï¼š**_ + +### Prometheus Operator + +Prometheus Operator 观察正在创建的 ServiceMonitorsã€PodMonitors å’Œ PrometheusRules。在创建 Prometheus é…置资æºæ—¶ï¼ŒPrometheus Operator 会调用 Prometheus API æ¥åŒæ­¥æ–°é…置。如本节末尾的图所示，Prometheus Operator 充当 Prometheus å’Œ Kubernetes 之间的中介，调用 Prometheus API æ¥åŒæ­¥ Prometheus 与 Kubernetes 中的监控相关资æºã€‚ + +### ServiceMonitor å’Œ PodMonitor + +ServiceMonitor å’Œ PodMonitor 以声明方å¼æŒ‡å®šéœ€è¦ç›‘控的目标，例如 Service å’Œ Pod。 + +- 目标是根æ®é…置的 Prometheus 抓å–间隔定期抓å–的，而抓å–的指标存储在 Prometheus 时间åºåˆ—æ•°æ®åº“（Time Series Database，TSDB）中。 + +- 为了执行抓å–，你需è¦ä½¿ç”¨æ ‡ç­¾é€‰æ‹©å™¨æ¥å®šä¹‰ ServiceMonitor å’Œ PodMonitor，这些标签选择器用于确定è¦æŠ“å–çš„ Service 或 Pod，并能确定端点（端点能确定抓å–如何在给定目标上进行，例如在 TCP 10252 中抓å–指标，通过 IP åœ°å€ x.x.x.x 进行代ç†ï¼‰ã€‚ + +- 开箱å³ç”¨åœ°ï¼ŒMonitoring V2 附带了一些预é…置的 Exporter，这些 Exporter æ ¹æ®å…¶æ‰€åœ¨çš„ Kubernetes 集群的类型进行部署。有关详细信æ¯ï¼Œè¯·å‚阅[抓å–和公开指标](#5-抓å–和公开指标)。 + +### PushProx çš„å·¥ä½œåŽŸç† + +- æŸäº›å†…部 Kubernetes 组件是通过部署在 Monitoring V2 中å为 **PushProx** 的代ç†æ¥æŠ“å–的。通过 PushProx å‘ Prometheus 公开指标的 Kubernetes 组件分别是 `kube-controller-manager`ã€`kube-scheduler`, `etcd` å’Œ `kube-proxy`。 + +- 对于æ¯ä¸ª PushProx Exporter，我们在所有目标节点上都部署一个 PushProx 客户端。例如，将 PushProx 客户端部署到 kube-controller-manager 的所有 controlplane 节点ã€kube-etcd 的所有 etcd 节点上，和 kubelet 的所有节点上。 + +- 我们为æ¯ä¸ª Exporter 部署了一个 PushProx 代ç†ã€‚导出指标的æµç¨‹å¦‚下： + +1. PushProx 客户端与 PushProx 代ç†å»ºç«‹å‡ºç«™è¿žæŽ¥ã€‚ +1. 然åŽï¼Œå®¢æˆ·ç«¯ä¼šè½®è¯¢ä»£ç†ä»¥èŽ·å–进入代ç†çš„抓å–请求。 +1. 当代ç†æ”¶åˆ°æ¥è‡ª Prometheus 的抓å–请求时，客户端会将其视为轮询的结果。 +1. 客户端抓å–内部组件。 +1. 内部组件通过将指标推回代ç†æ¥å“应。 + + +

使用 PushProx 导出指标的过程：
+ +![使用 PushProx 导出指标的过程](/img/pushprox-process.svg) + +### PrometheusRules + +PrometheusRule 用于定义指标或时间åºåˆ—æ•°æ®åº“查询触å‘告警的规则。规则是按时间间隔评估的。 + +- **记录规则**æ ¹æ®å·²æ”¶é›†çš„现有åºåˆ—创建一个新的时间åºåˆ—。它们常用于预先计算的å¤æ‚查询。 +- **告警规则**è¿è¡Œç‰¹å®šæŸ¥è¯¢ï¼Œå¦‚果查询评估为éžé›¶å€¼ï¼Œåˆ™ä»Ž Prometheus 触å‘告警。 + +### 告警路由 + +一旦 Prometheus 确定需è¦è§¦å‘告警，就会将告警转å‘到 **Alertmanager**。 + +- 告警包å«æ¥è‡ª PromQL 查询本身的标签，以åŠæŒ‡å®šåˆå§‹ PrometheusRule 时设置的其他标签和注释。 + +- 在接收任何告警之å‰ï¼ŒAlertmanager 会用é…置中指定的**路由 (Route)** å’Œ**接收器 (Receiver)** æ¥å½¢æˆä¸€ä¸ªè·¯ç”±æ ‘，所有传入的告警都会在该路由树上进行评估。路由树的æ¯ä¸ªèŠ‚点都å¯ä»¥æ ¹æ®é™„加到 Prometheus 告警的标签æ¥æŒ‡å®šè¦è¿›è¡Œçš„其他分组ã€æ ‡ç­¾å’Œè¿‡æ»¤ã€‚路由树上的节点（通常是å¶èŠ‚点）还å¯ä»¥æŒ‡å®šåˆ°è¾¾çš„告警需è¦å‘é€åˆ°ä»€ä¹ˆæŽ¥æ”¶å™¨ï¼Œä¾‹å¦‚ Slackã€PagerDutyã€SMS 等。请注æ„，Alertmanager é¦–å…ˆå‘ **alertingDriver** å‘é€å‘Šè­¦ï¼Œç„¶åŽ alertingDriver å†å°†å‘Šè­¦å‘é€æˆ–转å‘到正确的目标ä½ç½®ã€‚ + +- 路由和接收器也通过 Alertmanager Secret 存储在 Kubernetes API 中。当 Secret 更新时，Alertmanager 也会自动更新。请注æ„，路由仅通过标签å‘生（而ä¸æ˜¯é€šè¿‡æ³¨é‡Šç­‰ï¼‰ã€‚ + +
æ•°æ®å¦‚何æµç» Monitoring 应用程åº
+ + +## 2. Prometheus çš„å·¥ä½œåŽŸç† + +### 存储时间åºåˆ—æ•°æ® + +收集 Exporter 的指标åŽï¼ŒPrometheus 将时间åºåˆ—存储在本地ç£ç›˜æ—¶é—´åºåˆ—æ•°æ®åº“中。Prometheus å¯ä»¥é€‰æ‹©ä¸Žè¿œç¨‹ç³»ç»Ÿé›†æˆï¼Œä½† `rancher-monitoring` 使用本地存储æ¥å­˜å‚¨æ—¶é—´åºåˆ—æ•°æ®åº“。 + +存储åŽï¼Œç”¨æˆ·å¯ä»¥ä½¿ç”¨ PromQL（Prometheus 的查询语言）查询此 TSDB。 + +PromQL 查询å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼è¿›è¡Œå¯è§†åŒ–： + +1. 通过在 Prometheus çš„ Graph UI 中æ供查询，UI 会显示数æ®çš„简å•å›¾å½¢è§†å›¾ã€‚ +1. é€šè¿‡åˆ›å»ºåŒ…å« PromQL 查询和其他格å¼åŒ–指令的 Grafana 仪表æ¿ï¼Œè¿™äº›æŒ‡ä»¤ç”¨äºŽæ ‡è®°è½´ã€æ·»åŠ å•ä½ã€æ›´æ”¹é¢œè‰²ã€ä½¿ç”¨æ›¿ä»£å¯è§†åŒ–等。 + +### 为 Prometheus 定义规则 + +规则定义了 Prometheus 需è¦åœ¨å¸¸è§„ `evaluationInterval` 上执行的查询，从而执行æŸäº›æ“作，例如触å‘å‘Šè­¦ï¼ˆå‘Šè­¦è§„åˆ™ï¼‰æˆ–æ ¹æ® TSDB 中存在的其他查询（记录规则）预先计算查询。这些规则编ç åœ¨ PrometheusRules 自定义资æºä¸­ã€‚创建或更新 PrometheusRule 自定义资æºåŽï¼ŒPrometheus Operator 会观察å˜åŒ–，并调用 Prometheus API æ¥å®šæœŸåŒæ­¥ Prometheus 当å‰æ­£åœ¨è¯„估的规则集。 + +PrometheusRule 支æŒå®šä¹‰ä¸€ä¸ªæˆ–多个 RuleGroup。æ¯ä¸ª RuleGroup 由一组 Rule 对象组æˆï¼Œæ¯ä¸ª Rule 对象å‡èƒ½è¡¨ç¤ºå‘Šè­¦æˆ–记录规则，并具有以下字段： + +- 新告警或记录的å称 +- 新告警或记录的 PromQL è¡¨è¾¾å¼ +- 用于标记告警或记录的标签（例如集群å称或严é‡æ€§ï¼‰ +- 对需è¦åœ¨å‘Šè­¦é€šçŸ¥ä¸Šæ˜¾ç¤ºçš„其他é‡è¦ä¿¡æ¯è¿›è¡Œç¼–ç çš„注释（例如摘è¦ã€æè¿°ã€æ¶ˆæ¯ã€Runbook URL 等）。记录规则ä¸éœ€è¦æ­¤å­—段。 + +在评估[规则](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#rule)时，Prometheus 将执行é…置的 PromQL 查询，添加其他标签（或注释 - 仅用于告警规则），并为规则执行所需的æ“作。例如，如果æŸä¸ªå‘Šè­¦è§„则将 `team:front-end` 作为标签添加到é…置的 PromQL 查询，该标签会尾附到触å‘的告警，这将å…许 Alertmanager 将告警转å‘到正确的接收器。 + +### 告警和记录规则 + +Prometheus ä¸ä¼šç»´æŠ¤å‘Šè­¦æ˜¯å¦å¤„于 active 状æ€ã€‚它在æ¯ä¸ªè¯„估间隔内é‡å¤è§¦å‘å‘Šè­¦ï¼Œå¹¶æ ¹æ® Alertmanager 对告警进行分组和过滤æˆæœ‰æ„义的通知。 + +`evaluation_interval` 常é‡å®šä¹‰äº† Prometheus æ ¹æ®æ—¶é—´åºåˆ—æ•°æ®åº“评估告警规则的频率。与 `scrape_interval` 类似，`evaluation_interval` 的默认值也是一分钟。 + +规则包å«åœ¨ä¸€ç»„规则文件中。规则文件包括告警规则和记录规则，但åªæœ‰å‘Šè­¦è§„则æ‰ä¼šåœ¨è¯„ä¼°åŽè§¦å‘告警。 + +对于记录规则，Prometheus 会è¿è¡ŒæŸ¥è¯¢ï¼Œç„¶åŽå°†å…¶å­˜å‚¨ä¸ºæ—¶é—´åºåˆ—。如果需è¦å­˜å‚¨éžå¸¸æ˜‚贵或耗时的查询的结果，这ç§åˆæˆçš„时间åºåˆ—则éžå¸¸æœ‰ç”¨ï¼Œå› æ­¤ä½ å¯ä»¥åœ¨åŽç»­æ›´å¿«åœ°è¿›è¡ŒæŸ¥è¯¢å®ƒä»¬ã€‚ + +告警规则是更常用的。æ¯å½“告警规则评估为正数时，Prometheus 都会触å‘告警。 + +在触å‘告警之å‰ï¼ŒRule 文件会根æ®å®žé™…用例将标签和注释添加到告警中： + +- 标签用于标识告警的信æ¯ï¼Œå¹¶å¯èƒ½å½±å“告警的路由。例如，如果在å‘é€æœ‰å…³æŸä¸ªå®¹å™¨çš„告警时，你å¯ä»¥ä½¿ç”¨å®¹å™¨ ID 作为标签。 + +- 注释用于表示ä¸å½±å“告警路由ä½ç½®çš„ä¿¡æ¯ï¼Œä¾‹å¦‚ Runbook 或错误消æ¯ã€‚ + +## 3. Alertmanager çš„å·¥ä½œåŽŸç† + +Alertmanager 处ç†ç”± Prometheus server 等客户端应用å‘é€çš„告警。它负责以下任务： + +- 删除é‡å¤æ•°æ®ï¼Œåˆ†ç»„，并将告警路由到正确的接收器集æˆï¼ˆä¾‹å¦‚电å­é‚®ä»¶ã€PagerDuty 或 OpsGenie） + +- é™éŸ³å’ŒæŠ‘制告警 + +- 跟踪éšæ—¶é—´è§¦å‘çš„å‘Šè­¦ + +- å‘é€å‘Šè­¦çš„状æ€ï¼Œå³å‘Šè­¦æ˜¯å¦æ­£åœ¨è§¦å‘，或者是å¦å·²ç»è§£å†³ + +### ç”± alertingDrivers 转å‘çš„å‘Šè­¦ + +安装 alertingDriver åŽä¼šæ ¹æ® alertingDriver çš„é…置创建一个 `Service`，å¯ç”¨ä½œ Teams 或 SMS 的接收器 URL。接收器中的 URL ä¼šæŒ‡å‘ alertingDrivers。因此 Alertmanager é¦–å…ˆå‘ alertingDriver å‘é€å‘Šè­¦ï¼Œç„¶åŽ alertingDriver 将告警转å‘或å‘é€åˆ°æ­£ç¡®çš„目的ä½ç½®ã€‚ + +### 将告警路由到接收器 + +Alertmanager è´Ÿè´£å调告警的å‘é€ä½ç½®ã€‚它å…许你根æ®æ ‡ç­¾å¯¹å‘Šè­¦è¿›è¡Œåˆ†ç»„，并根æ®æ ‡ç­¾åŒ¹é…情况æ¥è§¦å‘告警。一个最顶层路由会接å—所有告警。然åŽï¼ŒAlertmanager 会根æ®å‘Šè­¦æ˜¯å¦åŒ¹é…下一个路由的æ¡ä»¶ï¼Œç»§ç»­å°†å‘Šè­¦è·¯ç”±åˆ°æŽ¥æ”¶å™¨ã€‚ + +虽然 Rancher UI 表å•åªå…许编辑两层深的路由树，但你å¯ä»¥é€šè¿‡ç¼–辑 Alertmanager Secret æ¥é…置更深的嵌套路由结构。 + +### é…置多个接收器 + +ä½ å¯ä»¥ç¼–辑 Rancher UI 中的表å•æ¥è®¾ç½®ä¸€ä¸ªæŽ¥æ”¶å™¨èµ„æºï¼Œå…¶ä¸­åŒ…å« Alertmanager 将告警å‘é€åˆ°ä½ çš„通知系统所需的所有信æ¯ã€‚ + +通过在 Alertmanager 或接收器é…置中编辑自定义 YAML，你还å¯ä»¥å°†å‘Šè­¦å‘é€åˆ°å¤šä¸ªé€šçŸ¥ç³»ç»Ÿã€‚有关详细信æ¯ï¼Œè¯·å‚阅[接收器é…ç½®](../../reference-guides/monitoring-v2-configuration/receivers.md#é…置多个接收器)。 + +## 4. Monitoring V2 特定组件 + +Prometheus Operator 引入了一组[自定义资æºå®šä¹‰](https://github.com/prometheus-operator/prometheus-operator#customresourcedefinitions)，å…许用户通过在集群上创建和修改这些自定义资æºæ¥éƒ¨ç½²å’Œç®¡ç† Prometheus å’Œ Alertmanager 实例。 + +Prometheus Operator ä¼šæ ¹æ® Rancher UI 中编辑的资æºå’Œé…置选项的实时状æ€æ¥è‡ªåŠ¨æ›´æ–° Prometheus é…置。 + +### é»˜è®¤éƒ¨ç½²çš„èµ„æº + +默认情况下，由 [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) 项目策划的一组资æºä¼šä½œä¸º Rancher Monitoring 安装的一部分部署到你的集群上，用æ¥è®¾ç½®åŸºæœ¬çš„ Monitoring/Alerting 堆栈。 + +ä½ å¯ä»¥åœ¨ [`rancher-monitoring`](https://github.com/rancher/charts/tree/main/charts/rancher-monitoring) Helm Chart 中找到部署到你的集群以支æŒæ­¤è§£å†³æ–¹æ¡ˆçš„资æºï¼Œè¯¥ chart 密切跟踪由 Prometheus 社区维护的上游 [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm Chart，并在 [CHANGELOG.md](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/CHANGELOG.md) 中跟踪å˜æ›´ã€‚ + +### 默认 Exporter + +Monitoring V2 部署了三个默认 Exporter，它们为 Prometheus æä¾›é¢å¤–的指标æ¥å­˜å‚¨ï¼š + +1. `node-exporter`：公开 Linux 主机的硬件和æ“作系统指标。有关 `node-exporter` 的更多信æ¯ï¼Œè¯·å‚阅[上游文档](https://prometheus.io/docs/guides/node-exporter/)。 + +1. `windows-exporter`：公开 Windows 主机的硬件和æ“作系统指标（仅部署在 Windows 集群上）。有关 `windows-exporter` 的更多信æ¯ï¼Œè¯·å‚阅[上游文档](https://github.com/prometheus-community/windows_exporter)。 + +1. `kube-state-metrics`：公开跟踪 Kubernetes API 中包å«çš„资æºçŠ¶æ€çš„其他指标（例如，podã€å·¥ä½œè´Ÿè½½ç­‰ï¼‰ã€‚有关 `kube-state-metrics` 的更多信æ¯ï¼Œè¯·å‚阅[上游文档](https://github.com/kubernetes/kube-state-metrics/tree/master/docs)。 + +ServiceMonitor å’Œ PodMonitor 将按照[此定义](#定义è¦æŠ“å–的指标)æ¥æŠ“å–这些 Exporter。Prometheus 会存储这些指标，你å¯ä»¥é€šè¿‡ Prometheus çš„ UI 或 Grafana 查询结果。 + +有关记录规则ã€å‘Šè­¦è§„则和 Alertmanager 的更多信æ¯ï¼Œè¯·å‚阅[架构](#1-架构概述)。 + +### Rancher UI 中公开的组件 + +安装 monitoring 应用åŽï¼Œä½ å°†èƒ½å¤Ÿåœ¨ Rancher UI 中编辑以下组件： + +| 组件 | 组件类型 | 编辑的目的和常è§ç”¨ä¾‹ | +|--------------|------------------------|---------------------------| +| ServiceMonitor | è‡ªå®šä¹‰èµ„æº | 设置 Kubernetes Service æ¥èŽ·å–其自定义指标。自动更新 Prometheus 自定义资æºä¸­çš„抓å–é…置。 | +| PodMonitor | è‡ªå®šä¹‰èµ„æº | 设置 Kubernetes Pod æ¥èŽ·å–其自定义指标。自动更新 Prometheus 自定义资æºä¸­çš„抓å–é…置。 | +| 接收器 | é…ç½®å—（Alertmanager 的一部分） | 修改将告警å‘é€åˆ°ä»€ä¹ˆä½ç½®çš„ä¿¡æ¯ï¼ˆä¾‹å¦‚，Slackã€PagerDuty 等）以åŠå‘é€å‘Šè­¦çš„其他必è¦ä¿¡æ¯ï¼ˆä¾‹å¦‚，TL​​S è¯ä¹¦ã€ä»£ç† URL 等）。自动更新 Alertmanager 自定义资æºã€‚ | +| Route | é…ç½®å—（Alertmanager 的一部分） | 修改用于根æ®æ ‡ç­¾è¿‡æ»¤ã€æ ‡è®°å’Œåˆ†ç»„告警的路由树，并将告警å‘é€åˆ°æ‰€éœ€çš„接收器。自动更新 Alertmanager 自定义资æºã€‚ | +| PrometheusRule | è‡ªå®šä¹‰èµ„æº | 定义其他查询，这些查询能触å‘告警或定义 Prometheus TSDB 中现有的物化视图。自动更新 Prometheus 自定义资æºã€‚ | + +### PushProx + +PushProx å…许 Prometheus 跨网络边界抓å–指标，这样，用户就ä¸ç”¨å¿…须为 Kubernetes 集群中æ¯ä¸ªèŠ‚点上的内部 Kubernetes 组件公开指标端å£ã€‚ + +由于 Kubernetes 组件的指标通常暴露在集群中节点的主机网络上，PushProx 部署了一个客户端 DaemonSet，这些客户端ä½äºŽæ¯ä¸ªèŠ‚点的主机网络上，并与ä½äºŽ Kubernetes API 上的å•ä¸ªä»£ç†å»ºç«‹å‡ºç«™è¿žæŽ¥ã€‚然åŽï¼Œä½ å¯ä»¥è®© Prometheus 通过代ç†å°†æŠ“å–请求å‘é€åˆ°æ¯ä¸ªå®¢æˆ·ç«¯ï¼Œè¿™æ ·ï¼ŒPrometheus 能从内部 Kubernetes 组件抓å–指标，而ä¸éœ€è¦æ‰“开任何入站节点端å£ã€‚ + +有关更多信æ¯ï¼Œè¯·å‚阅[使用 PushProx 抓å–指标](#使用-pushprox-抓å–指标)。 + +## 5. 抓å–和公开指标 + +### 定义è¦æŠ“å–的指标 + +ServiceMonitor å’Œ PodMonitor 定义了 Prometheus è¦æŠ“å–的目标。[Prometheus 自定义资æº](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md#prometheus)告诉 Prometheus 应该使用哪个 ServiceMonitor 或 PodMonitor æ¥ç¡®å®šä»Žå“ªé‡ŒæŠ“å–指标。 + +Prometheus Operator 观察 ServiceMonitor å’Œ PodMonitor。当它观察到二者被创建或更新时，它会调用 Prometheus API æ¥æ›´æ–° Prometheus 自定义资æºä¸­çš„抓å–é…置，并使该é…置与 ServiceMonitor 或 PodMonitor 中的抓å–é…ç½®ä¿æŒåŒæ­¥ã€‚此抓å–é…置告诉 Prometheus 从哪些端点抓å–指标，以åŠå¦‚何标记这些端点的指标。 + +Prometheus ä¼šæ ¹æ® `scrape_interval`（默认为一分钟）æ¥æŠ“å–定义在抓å–é…置中的所有指标。 + +抓å–é…ç½®å¯ä»¥ä½œä¸º Prometheus 自定义资æºçš„一部分被查看，该资æºåœ¨ Rancher UI 中公开。 + +### Prometheus Operator å¦‚ä½•è®¾ç½®æŒ‡æ ‡æŠ“å– + +Prometheus Deployment 或 StatefulSet 能抓å–指标，而 Prometheus çš„é…置由 Prometheus 自定义资æºæŽ§åˆ¶ã€‚Prometheus Operator 会观察 Prometheus å’Œ Alertmanager 资æºï¼Œå½“它们被创建时，Prometheus Operator 使用用户定义的é…置，为 Prometheus 或 Alertmanager 创建一个 Deployment 或 StatefulSet。 + +如果 Prometheus Operator 观察到正在创建的 ServiceMonitorã€PodMonitor å’Œ PrometheusRule，它就知é“需è¦åœ¨ Prometheus 中更新抓å–é…置。首先，会通过更新 Prometheus çš„ Deployment 或 StatefulSet å·ä¸­çš„é…置和规则文件æ¥æ›´æ–° Prometheus。然åŽï¼Œå†è°ƒç”¨ Prometheus API æ¥åŒæ­¥æ–°é…置，从而将 Prometheus Deployment 或 StatefulSet 修改到ä½ã€‚ + +### 如何公开 Kubernetes 组件指标 + +Prometheus 从称为 [exporter](https://prometheus.io/docs/instrumenting/exporters/) çš„ deployment 中抓å–指标，exporter 以 Prometheus å¯ä»¥æŠ“å–çš„æ ¼å¼å¯¼å‡ºæ—¶é—´åºåˆ—æ•°æ®ã€‚在 Prometheus 中，时间åºåˆ—由属于相åŒæŒ‡æ ‡å’Œç›¸åŒæ ‡è®°ç»´åº¦é›†çš„时间戳值æµç»„æˆã€‚ + +### 使用 PushProx 抓å–指标 + +æŸäº›å†…部 Kubernetes 组件是通过部署在 Monitoring V2 中å为 PushProx 的代ç†æ¥æŠ“å–的。有关 PushProx 的详细信æ¯ï¼Œè¯·å‚阅[此处](#pushprox-的工作原ç†)和上é¢çš„[架构](#1-架构概述)部分。 + +### 抓å–指标 + +Prometheus 直接抓å–以下 Kubernetes 组件： + +- kubelet\* +- ingress-nginx\*\* +- coreDns/kubeDns +- kube-api-server + +\* ä½ å¯ä»¥é€‰æ‹©é€šè¿‡ `hardenedKubelet.enabled` æ¥ä½¿ç”¨ PushProx，但这ä¸æ˜¯é»˜è®¤è®¾ç½®ã€‚ + +\*\* RKE å’Œ RKE2 集群默认部署 ingress-nginx，并将其视为内部 Kubernetes 组件。 + + +### 基于 Kubernetes å‘行版抓å–指标 + +指标的抓å–æ–¹å¼æ ¹æ® Kubernetes å‘行版而有所ä¸åŒã€‚有关术语的帮助，请å‚阅[此处](#åè¯è§£é‡Š)。详情è§ä¸‹è¡¨ï¼š + +
指标如何暴露给 Prometheus
+ +| Kubernetes 组件 | RKE | RKE2 | KubeADM | K3s | +|-----|-----|-----|-----|-----| +| kube-controller-manager | rkeControllerManager.enabled | rke2ControllerManager.enabled | kubeAdmControllerManager.enabled | k3sServer.enabled | +| kube-scheduler | rkeScheduler.enabled | rke2Scheduler.enabled | kubeAdmScheduler.enabled | k3sServer.enabled | +| etcd | rkeEtcd.enabled | rke2Etcd.enabled | kubeAdmEtcd.enabled | ä¸å¯ç”¨ | +| kube-proxy | rkeProxy.enabled | rke2Proxy.enabled | kubeAdmProxy.enabled | k3sServer.enabled | +| kubelet | 收集 kubelet 直接公开的指标 | 收集 kubelet 直接公开的指标 | 收集 kubelet 直接公开的指标 | 收集 kubelet 直接公开的指标 | +| ingress-nginx* | 收集 kubelet 直接公开的指标，由 rkeIngressNginx.enabled 公开 | 收集 kubelet 直接公开的指标，由 rke2IngressNginx.enabled 公开 | ä¸å¯ç”¨ | ä¸å¯ç”¨ | +| coreDns/kubeDns | 收集 coreDns/kubeDns 直接公开的指标 | 收集 coreDns/kubeDns 直接公开的指标 | 收集 coreDns/kubeDns 直接公开的指标 | 收集 coreDns/kubeDns 直接公开的指标 | +| kube-api-server | 收集 kube-api-server 直接公开的指标 | 收集 kube-api-server 直接公开的指标 | 收集 kube-appi-server 直接公开的指标 | 收集 kube-api-server 直接公开的指标 | + +\* RKE å’Œ RKE2 集群默认部署 ingress-nginx，并将其视为内部 Kubernetes 组件。 + +### åè¯è§£é‡Š + +- **kube-scheduler**：内部 Kubernetes 组件，该组件使用 pod 规范中的信æ¯æ¥å†³å®šåœ¨å“ªä¸ªèŠ‚点上è¿è¡Œ pod。 +- **kube-controller-manager**：负责节点管ç†ï¼ˆæ£€æµ‹èŠ‚点是å¦å¤±è´¥ï¼‰ã€pod å¤åˆ¶ï¼Œä»¥åŠç«¯ç‚¹åˆ›å»ºçš„内部 Kubernetes 组件。 +- **etcd**：Kubernetes 内部组件，它是 Kubernetes 用于æŒä¹…存储所有集群信æ¯çš„分布å¼é”®/值存储。 +- **kube-proxy**：内部 Kubernetes 组件，用于监控 API server çš„ pod/service 更改以ä¿æŒç½‘络最新状æ€ã€‚ +- **kubelet**：内部 Kubernetes 组件，用于为 pod 监视节点上的 API server 并确ä¿è¿™äº› pod 能è¿è¡Œã€‚ +- **ingress-nginx**：用于 Kubernetes çš„ Ingress controller，使用 NGINX 作为åå‘代ç†å’Œè´Ÿè½½å‡è¡¡å™¨ã€‚ +- **coreDns/kubeDns**：负责 DNS 的内部 Kubernetes 组件。 +- **kube-api-server**：负责为其他 master 组件公开 API 的主è¦å†…部 Kubernetes 组件。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md new file mode 100644 index 00000000000..006df34697c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md @@ -0,0 +1,365 @@ +--- +title: PromQL 表达å¼å‚考 +--- + +本文档中的 PromQL 表达å¼å¯ç”¨äºŽé…置告警。 + +关于查询 Prometheus 时间åºåˆ—æ•°æ®åº“的更多信æ¯ï¼Œè¯·å‚阅 [Prometheus 官方文档](https://prometheus.io/docs/prometheus/latest/querying/basics/)。 + + +## 集群指标 + +### 集群 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance))` | +| æ‘˜è¦ | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])))` | + +### 集群平å‡è´Ÿè½½ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
| +| æ‘˜è¦ |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"})`
| + +### 集群内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `1 - sum(node_memory_MemAvailable_bytes) by (instance) / sum(node_memory_MemTotal_bytes) by (instance)` | +| æ‘˜è¦ | `1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes)` | + +### 集群ç£ç›˜åˆ©ç”¨çŽ‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `(sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance) - sum(node_filesystem_free_bytes{device!="rootfs"}) by (instance)) / sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance)` | +| æ‘˜è¦ | `(sum(node_filesystem_size_bytes{device!="rootfs"}) - sum(node_filesystem_free_bytes{device!="rootfs"})) / sum(node_filesystem_size_bytes{device!="rootfs"})` | + +### 集群ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(node_disk_read_bytes_total[5m])) by (instance)`
written`sum(rate(node_disk_written_bytes_total[5m])) by (instance)`
| +| æ‘˜è¦ |
read`sum(rate(node_disk_read_bytes_total[5m]))`
written`sum(rate(node_disk_written_bytes_total[5m]))`
| + +### 集群网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| æ‘˜è¦ |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +### 集群网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| æ‘˜è¦ |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +## 节点指标 + +### 节点 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `avg(irate(node_cpu_seconds_total{mode!="idle", instance=~"$instance"}[5m])) by (mode)` | +| æ‘˜è¦ | `1 - (avg(irate(node_cpu_seconds_total{mode="idle", instance=~"$instance"}[5m])))` | + +### 节点平å‡è´Ÿè½½ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| +| æ‘˜è¦ |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| + +### 节点内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"})` | +| æ‘˜è¦ | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"}) ` | + +### 节点ç£ç›˜åˆ©ç”¨çŽ‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"}) by (device)) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device)` | +| æ‘˜è¦ | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"})) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"})` | + +### 节点ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| +| æ‘˜è¦ |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| + +### 节点网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| æ‘˜è¦ |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +### 节点网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| æ‘˜è¦ |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +## ETCD 指标 + +### ETCD 有一个 Leader + +`max(etcd_server_has_leader)` + +### Leader æ›´æ¢æ¬¡æ•° + +`max(etcd_server_leader_changes_seen_total)` + +### 失败的 Proposal æ•°é‡ + +`sum(etcd_server_proposals_failed_total)` + +### GRPC 客户端æµé‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m])) by (instance)`
| +| æ‘˜è¦ |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m]))`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m]))`
| + +### 对等æµé‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
in`sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)`
| +| æ‘˜è¦ |
in`sum(rate(etcd_network_peer_received_bytes_total[5m]))`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m]))`
| + +### æ•°æ®åº“å¤§å° + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(etcd_debugging_mvcc_db_total_size_in_bytes) by (instance)` | +| æ‘˜è¦ | `sum(etcd_debugging_mvcc_db_total_size_in_bytes)` | + +### æ´»åŠ¨æµ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance)`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance)`
| +| æ‘˜è¦ |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})`
| + +### Raft 方案 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
applied`sum(increase(etcd_server_proposals_applied_total[5m])) by (instance)`
committed`sum(increase(etcd_server_proposals_committed_total[5m])) by (instance)`
pending`sum(increase(etcd_server_proposals_pending[5m])) by (instance)`
failed`sum(increase(etcd_server_proposals_failed_total[5m])) by (instance)`
| +| æ‘˜è¦ |
applied`sum(increase(etcd_server_proposals_applied_total[5m]))`
committed`sum(increase(etcd_server_proposals_committed_total[5m]))`
pending`sum(increase(etcd_server_proposals_pending[5m]))`
failed`sum(increase(etcd_server_proposals_failed_total[5m]))`
| + +### RPC 速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m])) by (instance)`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m])) by (instance)`
| +| æ‘˜è¦ |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m]))`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m]))`
| + +### ç£ç›˜æ“作 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m])) by (instance)`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m])) by (instance)`
| +| æ‘˜è¦ |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m]))`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m]))`
| + +### ç£ç›˜åŒæ­¥æŒç»­æ—¶é—´ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
wal`histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))`
db`histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))`
| +| æ‘˜è¦ |
wal`sum(histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)))`
db`sum(histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le)))`
| + +## Kubernetes 组件指标 + +### API Server 请求延迟 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance, verb) /1e+06` | +| æ‘˜è¦ | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance) /1e+06` | + +### API Server 请求速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(apiserver_request_count[5m])) by (instance, code)` | +| æ‘˜è¦ | `sum(rate(apiserver_request_count[5m])) by (instance)` | + +### 调度失败的 Pod + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(kube_pod_status_scheduled{condition="false"})` | +| æ‘˜è¦ | `sum(kube_pod_status_scheduled{condition="false"})` | + +### Controller Manager 队列深度 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
volumes`sum(volumes_depth) by instance`
deployment`sum(deployment_depth) by instance`
replicaset`sum(replicaset_depth) by instance`
service`sum(service_depth) by instance`
serviceaccount`sum(serviceaccount_depth) by instance`
endpoint`sum(endpoint_depth) by instance`
daemonset`sum(daemonset_depth) by instance`
statefulset`sum(statefulset_depth) by instance`
replicationmanager`sum(replicationmanager_depth) by instance`
| +| æ‘˜è¦ |
volumes`sum(volumes_depth)`
deployment`sum(deployment_depth)`
replicaset`sum(replicaset_depth)`
service`sum(service_depth)`
serviceaccount`sum(serviceaccount_depth)`
endpoint`sum(endpoint_depth)`
daemonset`sum(daemonset_depth)`
statefulset`sum(statefulset_depth)`
replicationmanager`sum(replicationmanager_depth)`
| + +### 调度器 E2E 调度延迟 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06` | +| æ‘˜è¦ | `sum(histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06)` | + +### 调度器抢å å°è¯• + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(scheduler_total_preemption_attempts[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(scheduler_total_preemption_attempts[5m]))` | + +### Ingress Controller 连接数 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"}) by (instance)`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"}) by (instance)`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"}) by (instance)`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m]))) by (instance)`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m]))) by (instance)`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m]))) by (instance)`
| +| æ‘˜è¦ |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"})`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"})`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"})`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m])))`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m])))`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m])))`
| + +### Ingress Controller 请求处ç†æ—¶é—´ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `topk(10, histogram_quantile(0.95,sum by (le, host, path)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | +| æ‘˜è¦ | `topk(10, histogram_quantile(0.95,sum by (le, host)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | + +## Rancher Logging 指标 + + +### Fluentd 缓冲区队列速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_output_status_buffer_queue_length[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(fluentd_output_status_buffer_queue_length[5m]))` | + +### Fluentd 输入速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(fluentd_input_status_num_records_total[5m]))` | + +### Fluentd 输出错误率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_output_status_num_errors[5m])) by (type)` | +| æ‘˜è¦ | `sum(rate(fluentd_output_status_num_errors[5m]))` | + +### Fluentd 输出速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_output_status_num_records_total[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(fluentd_output_status_num_records_total[5m]))` | + +## 工作负载指标 + +### 工作负载 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### 工作负载内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""}) by (pod_name)` | +| æ‘˜è¦ | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""})` | + +### 工作负载网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### 工作负载网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### 工作负载ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +## Pod 指标 + +### Pod CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
| +| æ‘˜è¦ |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
| + +### Pod 内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""}) by (container_name)` | +| æ‘˜è¦ | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""})` | + +### Pod 网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| æ‘˜è¦ |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod 网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| æ‘˜è¦ |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
| +| æ‘˜è¦ |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +## 容器指标 + +### 容器 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| cfs throttled seconds | `sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| usage seconds | `sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| system seconds | `sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| user seconds | `sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | + +### 容器内存利用率 + +`sum(container_memory_working_set_bytes{namespace="$namespace",pod_name="$podName",container_name="$containerName"})` + +### 容器ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| read | `sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| write | `sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md new file mode 100644 index 00000000000..dfeefda6f97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md @@ -0,0 +1,251 @@ +--- +title: RBAC +--- +本节æ述了 RBAC 在 Rancher Monitoring 中的作用。 + + +## 集群管ç†å‘˜ + +默认情况下，åªæœ‰å…·æœ‰ cluster-admin `ClusterRole` 的用户æ‰èƒ½ï¼š + +- å°† `rancher-monitoring` 应用安装到集群上，并在 Chart 部署上执行所有其他相关é…置。 + - 例如，是å¦åˆ›å»ºäº†é»˜è®¤ä»ªè¡¨æ¿ï¼Œè¦å°†å“ªäº› Exporter 部署到集群上以收集指标等。 +- 通过 Prometheus CR 在集群中创建/修改/删除 Prometheus deployment。 +- 通过 Alertmanager CR 在集群中创建/修改/删除 Alertmanager deployment。 +- 通过在命å空间中创建 ConfigMap æ¥ä¿ç•™æ–°çš„ Grafana 仪表æ¿æˆ–æ•°æ®æºã€‚ +- 通过 `cattle-monitoring-system` 命å空间中的 Secret å°†æŸäº› Prometheus 指标暴露给 HPA çš„ K8s Custom Metrics API。 + +## 具有基于 Kubernetes ClusterRole æƒé™çš„用户 + +`rancher-monitoring` Chart 安装了以下三个 `ClusterRole`。默认情况下，它们会èšåˆåˆ°ç›¸åº”çš„ K8s `ClusterRole` 中： + +| ClusterRole | èšåˆåˆ°é»˜è®¤çš„ K8s ClusterRole | +| ------------------------------| ---------------------------| +| `monitoring-admin` | `admin` | +| `monitoring-edit` | `edit` | +| `monitoring-view` | `view ` | + +这些 `ClusterRole` æ ¹æ®å¯æ‰§è¡Œçš„æ“作æ供对 Monitoring CRD çš„ä¸åŒè®¿é—®çº§åˆ«ï¼š + +| CRD (monitoring.coreos.com) | Admin | Edit | View | +| ------------------------------| ---------------------------| ---------------------------| ---------------------------| +|
  • `prometheuses`
  • `alertmanagers`
| Get, List, Watch | Get, List, Watch | Get, List, Watch | +|
  • `servicemonitors`
  • `podmonitors`
  • `prometheusrules`
| * | * | Get, List, Watch | + +在较高级别上，默认情况下会分é…以下æƒé™ã€‚ + +### 具有 Kubernetes 管ç†å‘˜/编辑æƒé™çš„用户 + +åªæœ‰å…·æœ‰ cluster-adminã€admin 或 edit çš„ `ClusterRole` å¯ä»¥ï¼š + +- 通过 ServiceMonitor å’Œ PodMonitor CR 修改 Prometheus deployment 的抓å–é…置。 +- 通过 PrometheusRules CR 修改 Prometheus deployment çš„å‘Šè­¦/记录规则。 + +### 具有 Kubernetes 查看æƒé™çš„用户 + +åªæœ‰å…·æœ‰ Kubernetes `ClusterRole` 的用户å¯ä»¥ï¼š + +- 查看集群内部署的 Prometheuses çš„é…置。 +- 查看集群内部署的 Alertmanagers çš„é…置。 +- 通过 ServiceMonitor å’Œ PodMonitor CR 查看 Prometheus deployment 的抓å–é…置。 +- 通过 PrometheusRules CR 查看 Prometheus deployment çš„å‘Šè­¦/记录规则。 + +### 其他监控角色 + +Monitoring 还会创建其他 `Role`，这些角色默认情况下ä¸ä¼šåˆ†é…给用户，而是在集群中创建。你å¯ä»¥éƒ¨ç½²å¼•ç”¨è§’色的 `RoleBinding` æ¥å°†è§’色绑定到命å空间。è¦ä½¿ç”¨ `kubectl` 而ä¸æ˜¯é€šè¿‡ Rancher æ¥å®šä¹‰ `RoleBinding`，请å•å‡»[此处](#使用-kubectl-分é…-role-å’Œ-clusterrole)。 + +管ç†å‘˜åº”使用这些角色为用户æ供更细粒度的访问æƒé™ï¼š + +| 角色 | 用途 | +| ------------------------------| ---------------------------| +| monitoring-config-admin | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-monitoring-system 命å空间中的 Secret å’Œ ConfigMap。修改此命å空间中的 Secret/ConfigMap å¯ä»¥å…许用户更改集群的 Alertmanager é…ç½®ã€Prometheus Adapter é…ç½®ã€å…¶ä»– Grafana æ•°æ®æºã€TLS 密钥等。 | +| monitoring-config-edit | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-monitoring-system 命å空间中的 Secret å’Œ ConfigMap。修改此命å空间中的 Secret/ConfigMap å¯ä»¥å…许用户更改集群的 Alertmanager é…ç½®ã€Prometheus Adapter é…ç½®ã€å…¶ä»– Grafana æ•°æ®æºã€TLS 密钥等。 | +| monitoring-config-view | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看 cattle-monitoring-system 命å空间中的 Secret å’Œ ConfigMap。查看此命å空间中的 Secret/ConfigMap å¯ä»¥å…许用户观察集群的 Alertmanager é…ç½®ã€Prometheus Adapter é…ç½®ã€å…¶ä»– Grafana æ•°æ®æºã€TLS 密钥等。 | +| monitoring-dashboard-admin | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-dashboards 命å空间中的 ConfigMap。此命å空间中的 ConfigMap 将对应于æŒä¹…化到集群上的 Grafana 仪表æ¿ã€‚ | +| monitoring-dashboard-edit | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-dashboards 命å空间中的 ConfigMap。此命å空间中的 ConfigMap 将对应于æŒä¹…化到集群上的 Grafana 仪表æ¿ã€‚ | +| monitoring-dashboard-view | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看 cattle-dashboards 命å空间中的 ConfigMap。此命å空间中的 ConfigMap 将对应于æŒä¹…化到集群上的 Grafana 仪表æ¿ã€‚ | + + +### 通过自定义角色分é…监控角色 + +管ç†å‘˜å¯ä»¥åœ¨ Rancher UI 中分é…自定义角色以管ç†ã€ç¼–辑和查看监控。这些“角色â€æ˜¯åœ¨å®‰è£… Monitoring 应用程åºæ—¶é»˜è®¤åˆ›å»ºçš„。此外，这些角色也会被部署到相应的 Kubernetes 角色：adminã€edit å’Œ view `ClusterRoles`。 + +:::note é‡è¦æ示： + +将用户添加到集群时，UI ä¸ä¼šæä¾› `monitoring-admin`ã€`monitoring-edit` å’Œ `monitoring-view` 选项。这些监控角色åªèƒ½é€šè¿‡æ‰‹åŠ¨åˆ›å»ºè‡ªå®šä¹‰è§’色æ¥åˆ†é…，该自定义角色继承 Project Owner å’Œ Project Monitoring View 角色。 + +::: + +1. 创建自定义角色： + + 1.1 å•å‡» **☰ > Users & Authentication > Roles**。 + + 1.2 选择适当的选项å¡ï¼Œä¾‹å¦‚ **Cluster** 角色。然åŽå•å‡» **Create Cluster Role**。 + + 1.3 在 **Name** 字段中，创建自定义角色，例如 `View Monitoring`ã€`Edit Monitoring` 或 `Admin Monitoring`。 + + 1.4 å•å‡» **Inherit From > Add Resource**，然åŽä»Žä¸‹æ‹‰åˆ—表中选择所需的 Kubernetes 角色。 + + 1.5 å•å‡» **Create**。 + + +2. 将自定义角色分é…给新用户： + + 2.1 å•å‡» **☰ > Cluster Management > Cluster Explore > Cluster > Cluster Members > Add**。 + + 2.2 从显示的 **Select Member** 中æœç´¢ä½ çš„新用户å。 + + 2.3 å°† **Cluster Permissions** 中的新自定义角色分é…给新用户。 + + 2.4 å•å‡» **Create**。 + +**结果**：新用户现在应该能够看到 monitoring 工具。 + +### 其他监控集群角色 + +Monitoring 还会创建其他 `ClusterRole`，这些角色默认情况下ä¸ä¼šåˆ†é…给用户，而是在集群中创建。默认情况下，这些角色ä¸ä¼šèšåˆï¼Œä½†ä½ å¯ä»¥éƒ¨ç½²å¼•ç”¨è§’色的 `RoleBinding` 或 `ClusterRoleBinding` æ¥å°†è§’色绑定到命å空间。è¦ä½¿ç”¨ `kubectl` 而ä¸æ˜¯é€šè¿‡ Rancher æ¥å®šä¹‰ `RoleBinding`，请å•å‡»[此处](#使用-kubectl-分é…-role-å’Œ-clusterrole)。 + +| 角色 | 用途 | +| ------------------------------| ---------------------------| +| monitoring-ui-view | _自 Monitoring v2 14.5.100+ èµ·å¯ç”¨_ æ­¤ ClusterRole å…许用户在 Rancher UI 中查看指定集群的指标图。这是通过授予对外部监控 UI çš„åªè¯»è®¿é—®æƒé™æ¥å®žçŽ°çš„。具有此角色的用户有æƒé™åˆ—出 Prometheusã€Alertmanager å’Œ Grafana 端点，并通过 Rancher 代ç†å‘ Prometheusã€Grafana å’Œ Alertmanager UI å‘出 GET 请求。 | + +### 使用 kubectl åˆ†é… Role å’Œ ClusterRole + +#### 使用 `kubectl create` + +一ç§æ–¹æ³•æ˜¯ä½¿ç”¨ `kubectl create clusterrolebinding` 或 `kubectl create rolebinding` æ¥åˆ†é…一个 `Role` 或 `ClusterRole`。如以下示例所示： + +- 分é…给特定用户： + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx --namespace=my-namespace + ``` + + + +- 分é…给所有ç»è¿‡èº«ä»½è®¤è¯çš„用户： + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated --namespace=my-namespace + ``` + + + + +#### 使用 YAML 文件 + +å¦ä¸€ç§æ–¹æ³•æ˜¯åœ¨ä½ åˆ›å»ºçš„ YAML 文件中定义绑定。你必须先使用 YAML 文件é…ç½® `RoleBinding` 或 `ClusterRoleBinding`。然åŽï¼Œé€šè¿‡è¿è¡Œ `kubectl apply` 命令æ¥åº”用é…置更改。 + +- **Role**：以下 YAML 文件示例å¯å¸®åŠ©ä½ åœ¨ Kubernetes 中é…ç½® `RoleBinding`。你需è¦åœ¨ä¸‹é¢å¡«å†™å称。 + +:::note + +å称区分大å°å†™ã€‚ + +::: + +```yaml +# monitoring-config-view-role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: monitoring-config-view + namespace: cattle-monitoring-system +roleRef: + kind: Role + name: monitoring-config-view + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: User + name: u-b4qkhsnliz # this can be found via `kubectl get users -A` + apiGroup: rbac.authorization.k8s.io +``` + +- **kubectl**：以下 `kubectl` 示例命令用于应用 YAML 文件中创建的绑定。请记ä½ç›¸åº”填写你的 YAML 文件å。 + ```plain + kubectl apply -f monitoring-config-view-role-binding.yaml + ``` + +## 具有 Rancher æƒé™çš„用户 + +Rancher éƒ¨ç½²çš„é»˜è®¤è§’è‰²ï¼ˆå³ cluster-ownerã€cluster-memberã€project-ownerã€project-member）ã€Kubernetes é»˜è®¤è§’è‰²ï¼Œä»¥åŠ rancher-monitoring Chart 部署的角色之间的关系如下： + +
默认 Rancher æƒé™å’Œå¯¹åº”çš„ Kubernetes ClusterRole
+ +| Rancher 角色 | Kubernetes 角色 | Monitoring ClusterRole/Role | ClusterRoleBinding/RoleBinding | +| --------- | --------- | --------- | --------- | +| cluster-owner | cluster-admin | N/A | ClusterRoleBinding | +| cluster-member | admin | monitoring-admin | ClusterRoleBinding | +| project-owner | admin | monitoring-admin | 项目命å空间中的 RoleBinding | +| project-member | edit | monitoring-edit | 项目命å空间中的 RoleBinding | + +除这些默认角色之外，你还å¯ä»¥å°†ä»¥ä¸‹çš„其他 Rancher 项目角色应用于集群æˆå‘˜ï¼Œä»¥æ供对 Monitoring 的其他访问。这些 Rancher 角色将与 Monitoring Chart 部署的 ClusterRole 相关è”： + +
éžé»˜è®¤çš„ Rancher æƒé™å’Œå¯¹åº”çš„ Kubernetes ClusterRole
+ +| Rancher 角色 | Kubernetes ClusterRole | å¯ç”¨ Rancher 版本 | å¯ç”¨ Monitoring V2 版本 | +|--------------------------|-------------------------------|-------|------| +| 查看 Monitoring\* | [monitoring-ui-view](#monitoring-ui-view) | 2.4.8+ | 9.4.204+ | + +\* 如果æŸä¸ªç”¨æˆ·ç»‘定了 Rancher çš„ **View Monitoring** 角色，该用户åªæœ‰åœ¨æœ‰ UI 链接时æ‰æœ‰æƒè®¿é—®å¤–部 Monitoring UI。è¦è®¿é—® Monitoring Pane 以获å–这些链接，用户必须是至少一个项目的项目æˆå‘˜ã€‚ + +### 2.5.x 中的差异 + +在 Rancher 2.5.x 中，分é…了 project-member 或 project-owner 角色的用户将无法访问 Prometheus 或 Grafana，因为我们仅在集群级别创建 Grafana 或 Prometheus。 + +此外，虽然 project-owner ä»ç„¶åªèƒ½æ·»åŠ é»˜è®¤åœ¨é¡¹ç›®å‘½å空间内抓å–资æºçš„ ServiceMonitor/PodMonitor，但 PrometheusRule 并ä¸å±€é™äºŽå•ä¸ªå‘½å空间/项目。因此，å³ä½¿ project-owner 无法查看/编辑/删除在项目命å空间之外创建的任何规则，project-owner 在其项目命å空间内创建的任何告警规则或记录规则都将应用于整个集群。 + +### 分é…其他访问æƒé™ + +如果 cluster-admin 想è¦ä¸ºä¸å…·æœ‰ rancher-monitoring chart 角色的用户æä¾› admin/edit 访问æƒé™ï¼Œåˆ™å­˜åœ¨ä¸‹è¡¨çš„潜在影å“： + +| CRD (monitoring.coreos.com) | 是å¦ä¼šåœ¨å‘½å空间/项目之外造æˆå½±å“ | å½±å“ | +|----------------------------| ------| ----------------------------| +| `prometheuses` | 是。该资æºå¯ä»¥ä»Žæ•´ä¸ªé›†ç¾¤ä¸­çš„任何目标中抓å–æŒ‡æ ‡ï¼ˆé™¤éž Operator 本身进行了é¢å¤–çš„é…置）。 | 用户将能够定义应在集群中创建的新集群级 Prometheus deployment çš„é…置。 | +| `alertmanagers` | å¦ | 用户将能够定义应在集群中创建的新集群级 Alertmanager deployment çš„é…置。注æ„：如果你åªæƒ³å…许用户é…置路由和接收器等设置，你应该åªæ供对 Alertmanager Config Secret 的访问æƒé™ã€‚ | +|
  • `servicemonitors`
  • `podmonitors`
| å¦ï¼ˆé»˜è®¤ï¼‰ã€‚å¯ä»¥é€šè¿‡ Prometheus CR 上的 `ignoreNamespaceSelectors` 进行é…置。 | 用户将能够通过 Prometheus，在他们被授予此æƒé™çš„命å空间内的 Service/Pod 暴露的端点上设置抓å–。 | +| `prometheusrules` | 是。PrometheusRules 是集群范围的。 | 用户将能够根æ®åœ¨æ•´ä¸ªé›†ç¾¤ä¸­æ”¶é›†çš„任何系列，在 Prometheus 上定义告警或记录规则。 | + +| K8s èµ„æº | 命å空间 | 是å¦ä¼šåœ¨å‘½å空间/项目之外造æˆå½±å“ | å½±å“ | +|----------------------------| ------| ------| ----------------------------| +|
  • `secrets`
  • `configmaps`
| `cattle-monitoring-system` | 是。此命å空间中的 Config å’Œ Secret 会影å“整个监控/å‘Šè­¦æµæ°´çº¿ã€‚ | 用户将能够创建或编辑 Secret/ ConfigMap，例如 Alertmanager Configã€Prometheus Adapter é…ç½®ã€TLS 密文ã€å…¶ä»– Grafana æ•°æ®æºç­‰ã€‚这会对所有集群监控/告警产生广泛影å“。 | +|
  • `secrets`
  • `configmaps`
| `cattle-dashboards` | 是。此命å空间中的 Config å’Œ Secret å¯ä»¥åˆ›å»ºä»ªè¡¨æ¿ï¼Œå¯¹åœ¨é›†ç¾¤çº§åˆ«æ”¶é›†çš„所有指标进行查询。 | 用户将能够创建仅ä¿ç•™æ–° Grafana 仪表æ¿çš„ Secret/ConfigMap。 | + +## Grafana çš„ RBAC + +对于ç»è¿‡ Kubernetes 认è¯å¹¶å¯ä»¥è®¿é—® Rancher Monitoring Chart 部署的 Grafana æœåŠ¡çš„任何用户，Rancher å…许他们通过 Rancher Dashboard UI 访问 Grafana。默认情况下，所有能够访问 Grafana 的用户都被赋予 [Viewer](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#viewer-role) 角色，能查看 Rancher 部署的任何默认仪表æ¿ã€‚ + +但是，如果有需è¦çš„è¯ï¼Œç”¨æˆ·å¯ä»¥é€‰æ‹©ä»¥ [Admin](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#admin-role) 身份登录 Grafana。Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin`/`prom-operator`，但你也å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + +è¦æŸ¥çœ‹ Grafana UI，请安装 `rancher-monitoring`。然åŽï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Grafana**。 + +
Grafana 中的集群计算资æºä»ªè¡¨æ¿
+ +![Grafana 中的集群计算资æºä»ªè¡¨æ¿](/img/cluster-compute-resources-dashboard.png) + +
Grafana 中的默认仪表æ¿
+ +![Grafana 中的默认仪表æ¿](/img/grafana-default-dashboard.png) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/windows-support.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/windows-support.md new file mode 100644 index 00000000000..c8de9476221 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/windows-support.md @@ -0,0 +1,48 @@ +--- +title: Monitoring V2 çš„ Windows é›†ç¾¤æ”¯æŒ +--- + +_从 v2.5.8 èµ·å¯ç”¨_ + +从 Monitoring V2 14.5.100（Rancher 2.5.8 的默认版本）开始，Monitoring V2 å¯ä»¥éƒ¨ç½²åœ¨ Windows 集群上，并将使用 [prometheus-community/windows_exporter](https://github.com/prometheus-community/windows_exporter)（旧å是 `wmi_exporter`）æ¥æŠ“å– Windows 节点的指标。 + +## 集群è¦æ±‚ + +Monitoring V2 for Windows åªèƒ½ä»Žæœ€ä½Žæ˜¯ `wins` v0.1.0 çš„ Windows 主机中抓å–指标。è¦å®Œå…¨éƒ¨ç½² Monitoring V2 for Windows，你的所有主机都必须满足此è¦æ±‚。 + +如果你在 Rancher 2.5.8 中é…置新的 RKE1 集群，你的集群应该已ç»æ»¡è¶³æ­¤è¦æ±‚。 + +### 将现有集群å‡çº§åˆ° wins v0.1.0 + +如果集群是在 Rancher 2.5.8 之å‰é…置的（å³ä½¿å½“å‰ Rancher 版本是 2.5.8），你将无法æˆåŠŸéƒ¨ç½² Monitoring V2 for Windows，除éžä½ å°†æ¯å°ä¸»æœºçš„ wins 版本å‡çº§åˆ° v0.1.0 或以上版本。 + +为了方便此次å‡çº§ï¼ŒRancher 2.5.8 å‘布了一个全新的 Helm Chart，å为 `rancher-wins-upgrader`。 + +1. 使用以下覆盖部署 `rancher-wins-upgrader`： + ```yaml + # 通过先å‰å·²åˆ—入白åå•çš„进程路径 + # æ¥å¼•å¯¼ win-upgrader 安装，这是因为正常安装路径 + # c:\etc\rancher\wins\wins-upgrade.exe 通常ä¸ä¼šè¢«åˆ—入白åå•ã€‚ + # 因此，我们使用 Monitoring V1 之å‰æ‰€ç”¨çš„ + # 已列入白åå•çš„进程路径。 + masquerade: + enabled: true + as: c:\\etc\wmi-exporter\wmi-exporter.exe + ``` + :::note éžé»˜è®¤ Windows å‰ç¼€è·¯å¾„的注æ„事项： + + - 如果你使用具有éžé»˜è®¤ `win_prefix_path` çš„ `cluster.yml` æ¥è®¾ç½® RKE 集群，你需è¦å°† `c:\\` 替æ¢ä¸ºä½ çš„å‰ç¼€è·¯å¾„字段的值æ¥ä¿®æ”¹ `masquerade.as`。 + + - 例如，如果你使用 `win_prefix_path: 'c:\host\opt\'`，则需è¦è®¾ç½®ä¸º `as: c:\host\opt\etc\wmi-exporter\wmi-exporter.exe`。 + + ::: + +2. æˆåŠŸå‡çº§æ‰€æœ‰ä¸»æœºåŽï¼Œè¯·å†æ¬¡ä½¿ç”¨é»˜è®¤å€¼éƒ¨ç½² Helm Chart，以é¿å…与以下设置å‘生冲çªï¼š + ```yaml + masquerade: + enabled: false + ``` + +**结果**：主机已准备好安装 Monitoring V2。你å¯ä»¥é€‰æ‹©å¸è½½ `rancher-wins-upgrader` Chart，或将其ä¿ç•™åœ¨é›†ç¾¤ä¸­ä»¥æ–¹ä¾¿å°†æ¥å‡çº§ã€‚ + +有关如何使用它的更多信æ¯ï¼Œè¯·å‚阅 Chart çš„ [README.md](https://github.com/rancher/wins/blob/master/charts/rancher-wins-upgrader/README.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/neuvector.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/neuvector.md new file mode 100644 index 00000000000..42a2e900c59 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/neuvector.md @@ -0,0 +1,199 @@ +--- +title: NeuVector é›†æˆ +--- + +### Rancher 中的 NeuVector é›†æˆ + +[NeuVector 5.x](https://open-docs.neuvector.com/) 是一个开æºçš„，以容器为中心的安全应用程åºï¼ŒRancher å·²é›†æˆ NeuVector。NeuVector 在è¿è¡Œæ—¶ä¸ºå…³é”®åº”用程åºå’Œæ•°æ®æ供实时的åˆè§„ã€å¯è§å’Œä¿æŠ¤åŠŸèƒ½ã€‚NeuVector æ供具有 CIS Benchmark å’Œæ¼æ´žæ‰«æ的防ç«å¢™ã€å®¹å™¨è¿›ç¨‹/文件系统监控和安全审计。有关 Rancher 安全性的更多信æ¯ï¼Œè¯·å‚阅[安全文档](../pages-for-subheaders/rancher-security.md)。 + +NeuVector å¯ä»¥é€šè¿‡ Helm Chart å¯ç”¨ã€‚ä½ å¯ä»¥åœ¨ **Apps** 或 Rancher UI 中的 **Cluster Tools** 中安装该 Chart。安装 Helm Chart åŽï¼Œç”¨æˆ·å¯ä»¥è½»æ¾åœ°[在 Rancher ä¸­éƒ¨ç½²å’Œç®¡ç† NeuVector 集群](https://open-docs.neuvector.com/deploying/rancher#deploy-and-manage-neuvector-through-rancher-apps-marketplace)。 + +### 使用 Rancher 安装 NeuVector + +Harvester Helm Chart ç”¨äºŽç®¡ç† Rancher 中 NeuVector UI 的访问，用户å¯ä»¥åœ¨ Rancher 中直接跳转，然åŽéƒ¨ç½²å’Œç®¡ç† NeuVector 集群。 + +**通过 "Apps" 导航并安装 NeuVector Chart**： + +1. 点击 **☰ > 集群管ç†**。 +1. 在 Clusters 页é¢ä¸Šï¼Œè½¬åˆ°è¦éƒ¨ç½² NeuVector 的集群，然åŽå•å‡» **Explore**。 +1. 转到 **Apps > Charts**，然åŽä»Ž Chart 仓库中安装 **NeuVector**。 +1. ä¸åŒçš„集群类型需è¦ä¸åŒçš„容器è¿è¡Œæ—¶ã€‚é…ç½® Helm Chart 值时，转到**容器è¿è¡Œæ—¶**，然åŽæ ¹æ®é›†ç¾¤ç±»åž‹é€‰æ‹©è¿è¡Œæ—¶ã€‚最åŽï¼Œå†æ¬¡å•å‡»**安装**。 + +以下是一些例å­ï¼š + +- RKE1：`docker` +- K3s å’Œ RKE2：`k3scontainerd` +- AKS：`containerd` 适用于 v1.19 åŠæ›´é«˜ç‰ˆæœ¬ +- EKS：`docker` 适用于 v1.22 åŠä»¥ä¸‹ç‰ˆæœ¬ï¼›`containerd` 适用于 v1.23 åŠæ›´é«˜ç‰ˆæœ¬ +- GKE：`containerd`（请å‚阅 [Google 文档](https://cloud.google.com/kubernetes-engine/docs/concepts/using-containerd)了解更多信æ¯ï¼‰ + + :::note + + 在安装过程中一次åªèƒ½é€‰æ‹©ä¸€ä¸ªå®¹å™¨è¿è¡Œæ—¶å¼•æ“Žã€‚ + + ::: + +**通过集群工具导航并安装 NeuVector Chart**： + +1. 点击 **☰ > 集群管ç†**。 +1. 在 Clusters 页é¢ä¸Šï¼Œè½¬åˆ°è¦éƒ¨ç½² NeuVector 的集群，然åŽå•å‡» **Explore**。 +1. 点击左侧导航æ åº•éƒ¨çš„**集群工具**。 +1. 按照上é¢çš„步骤 4 相应地选择你的容器è¿è¡Œæ—¶ï¼Œç„¶åŽå†æ¬¡å•å‡»**安装**。 + +### 从 Rancher UI 访问 NeuVector + +1. 导航到安装了 NeuVector 的集群的 Cluster Explorer。在左侧导航æ ä¸­ï¼Œå•å‡» **NeuVector**。 +1. å•å‡»å¤–部链接以转到 NeuVector UI。选择链接åŽï¼Œç”¨æˆ·å¿…须接å—`最终用户许å¯åè®®`æ‰èƒ½è®¿é—® NeuVector UI。 + +### 从 Rancher UI å¸è½½ NeuVector + +**通过 "Apps" å¸è½½**： + +1. 点击 **☰ > 集群管ç†**。 +1. 在 **Apps** 下，点击 **Installed Apps**。 +1. 在 `cattle-neuvector-system` 下，选择 NeuVector 应用程åºï¼ˆå¦‚果需è¦ï¼Œè¿˜å¯ä»¥é€‰æ‹©ç›¸å…³çš„ CRD），然åŽå•å‡»**删除**。 + +**通过集群工具å¸è½½**： + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»å±å¹•å·¦ä¸‹è§’çš„**集群工具**，然åŽå•å‡» NeuVector Chart 下方的垃圾桶图标。如果需è¦ï¼Œé€‰æ‹©`删除与此应用关è”çš„ CRD`，然åŽå•å‡»**删除**。 + +### GitHub 仓库 + +NeuVector 项目在[这里](https://github.com/neuvector/neuvector)。 + +### 文档 + +NeuVector 文档在[这里](https://open-docs.neuvector.com/)。 + +### 架构 + +NeuVector 安全解决方案包å«å››ç§ç±»åž‹çš„安全容器，分别是 Controllerã€Enforcerã€Manager å’Œ Scanner。它还æ供了一个称为 All-in-One 的特殊容器（主è¦ç”¨äºŽ Docker 原生部署），能将 Controllerã€Enforcer å’Œ Manager 功能组åˆåœ¨ä¸€ä¸ªå®¹å™¨ä¸­ã€‚此外，还有一个 Updater，è¿è¡Œè¯¥ç¨‹åºæ—¶ä¼šæ›´æ–° CVE æ•°æ®åº“。 + +- **Controller**ï¼šç®¡ç† NeuVector Enforcer 容器；为管ç†æŽ§åˆ¶å°æä¾› REST API。 +- **Enforcer**：执行安全策略。 +- **Manager**：æ供一个 web-UI å’Œ CLI 控制å°æ¥ç®¡ç† NeuVector å¹³å°ã€‚ +- **All-in-One**：包括 Controllerã€Enforcer å’Œ Manager。 +- **Scanner**：对镜åƒã€å®¹å™¨å’ŒèŠ‚点执行æ¼æ´žå’Œåˆè§„性扫æ。 +- **Updater**：更新 Neuvector çš„ CVE æ•°æ®åº“（è¿è¡Œçš„时候）；é‡æ–°éƒ¨ç½² scanner pod。 + +
**NeuVector 安全容器：**
+ +![NeuVector 安全容器](/img/neuvector-security-containers.png) + +
**NeuVector 架构：**
+ +![NeuVector 架构](/img/neuvector-architecture.png) + +è¦äº†è§£æœ‰å…³ NeuVector 架构的更多信æ¯ï¼Œè¯·å‚阅[此处](https://open-docs.neuvector.com/basics/overview#architecture)。 + +### CPU å’Œå†…å­˜åˆ†é… + +以下是默认 NeuVector Chart 安装部署的最低计算资æºæŽ¨è。请注æ„，未设置资æºé™åˆ¶ã€‚ + +| 容器 | CPU - 请求 | 内存 - 请求 | +|------------|--------|---------| +| Controller | 3（æ¯ä¸ªæŽ§åˆ¶å™¨éœ€è¦ 1GB 1vCPU） | * | +| Enforcer | 所有节点上 (500MB .5vCPU) | 1GB | +| Manager | 1 (500MB .5vCPU) | * | +| Scanner | 3 (100MB .5vCPU) | * | + +\* Controllerã€Manager å’Œ Scanner 容器åˆè®¡è‡³å°‘éœ€è¦ 1GB 内存。 + + +### å¼ºåŒ–é›†ç¾¤æ”¯æŒ - Calico å’Œ Canal + + + + +- 如果 PSP 设置为 true，则所有 NeuVector 组件都是å¯éƒ¨ç½²çš„。 + +你需è¦ä¸ºå¼ºåŒ–集群环境进行é¢å¤–çš„é…置，如下所示： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 安装（或å‡çº§åˆ°ï¼‰NeuVector 版本 `100.0.1+up2.2.2`。 + +- 在 **编辑选项** > **其它é…ç½®**下，选中å¤é€‰æ¡†æ¥å¯ç”¨ **Pod 安全策略**（无需其他é…置）： + + ![为 RKE1 强化集群å¯ç”¨ PSP](/img/psp-nv-rke.png) + +1. 点击å³ä¸‹è§’çš„**安装**。 + + + + + +- 如果 PSP 设置为 true，则å¯ä»¥éƒ¨ç½² NeuVector 组件 Controller å’Œ Enforcer。 + + +**仅适用于 NeuVector Chart 版本 100.0.0+up2.2.0**： + +- 对于 Managerã€Scanner å’Œ Updater 组件，需è¦è¿›è¡Œé¢å¤–çš„é…置，如下所示： + +``` +kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' +``` +
+ +你需è¦ä¸ºå¼ºåŒ–集群环境进行é¢å¤–çš„é…置。 + +> **注æ„**：你必须更新 RKE2 å’Œ K3s 强化集群中的é…置，如下所示。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 安装（或å‡çº§åˆ°ï¼‰NeuVector 版本 `100.0.1+up2.2.2`。 + +- 在 **编辑选项** > **其它é…ç½®**下，选中å¤é€‰æ¡†æ¥å¯ç”¨ **Pod 安全策略**。请注æ„，对于 `Manager runAsUser ID`ã€`Scanner runAsUser ID` å’Œ `Updater runAsUser ID`，你还必须输入大于 `0` 的值: + + ![为 RKE2 å’Œ K3s 强化集群å¯ç”¨ PSP](/img/psp-nv-rke2.png) + +1. 点击å³ä¸‹è§’çš„**安装**。 + + + + + +### å¯ç”¨ SELinux çš„é›†ç¾¤æ”¯æŒ - Calico å’Œ Canal + +è¦åœ¨ RKE2 集群上å¯ç”¨ SELinux，请执行以下步骤： + +- 如果 PSP 设置为 true，则å¯ä»¥éƒ¨ç½² NeuVector 组件 Controller å’Œ Enforcer。 + + +**仅适用于 NeuVector Chart 版本 100.0.0+up2.2.0**： + +- 对于 Managerã€Scanner å’Œ Updater 组件，需è¦è¿›è¡Œé¢å¤–çš„é…置，如下所示： + +``` +kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' +``` + +### ç¦»çº¿çŽ¯å¢ƒä¸­çš„é›†ç¾¤æ”¯æŒ + +- 所有 NeuVector 组件都å¯éƒ¨ç½²åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­çš„集群上，无需任何é¢å¤–é…置。 + + +### 支æŒé™åˆ¶ + +* ç›®å‰ä»…支æŒç®¡ç†å‘˜å’Œé›†ç¾¤æ‰€æœ‰è€…。 + +* ä¸æ”¯æŒ Fleet 多集群部署。 + +* Windows 集群ä¸æ”¯æŒ NeuVector。 + + +### 其他é™åˆ¶ + +* ç›®å‰ï¼Œå¦‚æžœ NeuVector partner Chart 已存在，则 NeuVector 功能 Chart 的安装会失败。è¦è§£å†³æ­¤é—®é¢˜ï¼Œè¯·å¸è½½ NeuVector partner Chart 并é‡æ–°å®‰è£… NeuVector 功能 Chart。 + +* Controller 未准备好时，有å¯èƒ½æ— æ³•ä»Ž Rancher UI 访问 NeuVector UI。在此期间，Controller å°†å°è¯•é‡æ–°å¯åŠ¨ï¼Œå¹¶ä¸”需è¦å‡ åˆ†é’Ÿæ‰èƒ½è¿›å…¥ active 状æ€ã€‚ + +* 安装 NeuVector Chart 时，ä¸ä¼šé’ˆå¯¹ä¸åŒçš„集群类型自动检测容器è¿è¡Œæ—¶ã€‚è¦è§£å†³æ­¤é—®é¢˜ï¼Œä½ å¯ä»¥æ‰‹åŠ¨æŒ‡å®šè¿è¡Œæ—¶ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/opa-gatekeeper.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/opa-gatekeeper.md new file mode 100644 index 00000000000..41c91274c66 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/opa-gatekeeper.md @@ -0,0 +1,111 @@ +--- +title: OPA Gatekeeper +--- + +为了确ä¿ä¸€è‡´æ€§å’Œåˆè§„性，æ¯ä¸ªç»„织都需è¦èƒ½å¤Ÿä»¥è‡ªåŠ¨åŒ–çš„æ–¹å¼åœ¨çŽ¯å¢ƒä¸­å®šä¹‰å’Œæ‰§è¡Œç­–略。[OPA（Open Policy Agent）](https://www.openpolicyagent.org/) 是一个策略引擎，用于基于策略控制云原生环境。Rancher 支æŒåœ¨ Kubernetes 集群中å¯ç”¨ OPA Gatekeeper，并且还安装了一些内置的策略定义（也称为约æŸæ¨¡æ¿ï¼‰ã€‚ + +OPA æ供了一ç§é«˜çº§å£°æ˜Žæ€§è¯­è¨€ï¼Œå¯ä»¥è®©ä½ å°†ç­–略指定为代ç ï¼Œè¿˜èƒ½æ‰©å±•ç®€å•çš„ API，从而å‡è½»ç­–略决策的负担。 + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) 是一个æä¾› OPA å’Œ Kubernetes 集æˆçš„项目。OPA Gatekeeper æ供： + +- 一个å¯æ‰©å±•çš„å‚数化策略库。 +- 用于实例化策略库的原生 Kubernetes CRD，也称为“约æŸâ€ã€‚ +- 用于扩展策略库的原生 Kubernetes CRD，也称为“约æŸæ¨¡æ¿â€ã€‚ +- 审计功能。 + +è¦äº†è§£æ›´å¤šå…³äºŽ OPA çš„ä¿¡æ¯ï¼Œè¯·å‚阅[官方文档](https://www.openpolicyagent.org/docs/latest/)。 + +## OPA Gatekeeper 集æˆçš„å·¥ä½œåŽŸç† + +Kubernetes 支æŒé€šè¿‡å‡†å…¥æŽ§åˆ¶å™¨ï¼ˆå‡†å…¥æŽ§åˆ¶å™¨ï¼‰webhook æ¥æ‰©å±• API Server 的功能，创建ã€æ›´æ–°æˆ–删除资æºæ—¶éƒ½ä¼šè°ƒç”¨è¿™äº› webhook。Gatekeeper ä½œä¸ºéªŒè¯ webhook 安装，并执行由 Kubernetes CRD（Custom Resource Definition）定义的策略。除了使用准入控制之外，Gatekeeper 还能审计 Kubernetes 集群中的现有资æºï¼Œå¹¶å¯¹è¿å当å‰ç­–略的情况进行标记。 + +OPA Gatekeeper ç”± Rancher çš„ Helm system Chart æ供，它安装在å为 `gatekeeper-system` 的命å空间中。 + +## 在集群中å¯ç”¨ OPA Gatekeeper + +:::note + +Rancher 2.5 改进了 OPA Gatekeeper 应用。无法从 Rancher 2.4 å‡çº§åˆ° Rancher 2.5 中的新版本。如果你在 Rancher 2.4 中安装了 OPA Gatekeeper，则需è¦åœ¨æ—§ UI 中å¸è½½ OPA Gatekeeper åŠå…¶ CRD，然åŽåœ¨ Rancher 2.5 中é‡æ–°å®‰è£…它。如需å¸è½½ CRD，请在 kubectl 控制å°ä¸­è¿è¡Œ `kubectl delete crd configs.config.gatekeeper.sh constrainttemplates.templates.gatekeeper.sh` 命令。 + +::: + +:::note 先决æ¡ä»¶ï¼š + +åªæœ‰ç®¡ç†å‘˜å’Œé›†ç¾¤æ‰€æœ‰è€…æ‰èƒ½å¯ç”¨ OPA Gatekeeper。 + +::: + +ä½ å¯ä»¥åœ¨ **Apps** 页é¢å®‰è£… OPA Gatekeeper Helm Chart。 + +### å¯ç”¨ OPA Gatekeeper + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦å¯ç”¨ OPA Gatekeeper 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击 **Charts** 并点击 **OPA Gatekeeper**。 +1. å•å‡»**安装**。 + +**结果**：已将 OPA Gatekeeper 部署到你的 Kubernetes 集群。 + +## 约æŸæ¨¡æ¿ + +[约æŸæ¨¡æ¿](https://github.com/open-policy-agent/gatekeeper#constraint-templates)是 Kubernetes 自定义资æºï¼Œç”¨äºŽå®šä¹‰è¦ç”± Gatekeeper 应用的 OPA 策略的架构和 Rego 逻辑。有关 Rego 策略语言的更多信æ¯ï¼Œè¯·å‚阅[官方文档](https://www.openpolicyagent.org/docs/latest/policy-language/)。 + +å¯ç”¨ OPA Gatekeeper åŽï¼ŒRancher 默认会安装一些模æ¿ã€‚ + +è¦åˆ—出集群中安装的约æŸæ¨¡æ¿ï¼Œè¯·è½¬åˆ° OPA Gatekeeper 下的左侧èœå•ï¼Œç„¶åŽå•å‡»**模æ¿**。 + +Rancher 还支æŒé€šè¿‡å¯¼å…¥ YAML 定义æ¥åˆ›å»ºä½ è‡ªå·±çš„约æŸæ¨¡æ¿ã€‚ + +## 创建和é…ç½®çº¦æŸ + +[约æŸ](https://github.com/open-policy-agent/gatekeeper#constraints)是 Kubernetes 自定义资æºï¼Œç”¨äºŽå®šä¹‰è¦åº”用约æŸæ¨¡æ¿çš„对象范围。约æŸæ¨¡æ¿å’Œçº¦æŸå…±åŒå®šä¹‰ä¸€ä¸ªå®Œæ•´çš„策略。 + +:::note 先决æ¡ä»¶ï¼š + +集群中已å¯ç”¨ OPA Gatekeeper。 + +::: + +è¦åˆ—出已安装的约æŸï¼Œè¯·è½¬åˆ° OPA Gatekeeper 下的左侧èœå•ï¼Œç„¶åŽå•å‡»**约æŸ**。 + +å¯ä»¥ä»Žçº¦æŸæ¨¡æ¿åˆ›å»ºæ–°çš„约æŸã€‚ + +Rancher 支æŒé€šè¿‡ä½¿ç”¨æ–¹ä¾¿çš„表å•æ¥åˆ›å»ºçº¦æŸï¼Œä½ å¯ä»¥åœ¨è¯¥è¡¨å•ä¸­è¾“å…¥å„ç§çº¦æŸå­—段。 + +**以 YAML 文件编辑**选项也å¯ä»¥ç”¨äºŽé…置约æŸçš„ YAML 定义。 + +### 使 Rancher çš„ System 命å空间ä¸å—çº¦æŸ + +创建约æŸæ—¶ï¼Œè¯·ç¡®ä¿è¯¥çº¦æŸä¸åº”用于任何 Rancher 或 Kubernetes System 命å空间。如果ä¸æŽ’除 System 命å空间，则å¯èƒ½ä¼šå‡ºçŽ° system 命å空间下的许多资æºè¢«æ ‡è®°ä¸ºè¿å约æŸã€‚ + +è¦è®©çº¦æŸä»…é™åˆ¶ç”¨æˆ·å‘½å空间，请在约æŸçš„**匹é…**字段下指定这些命å空间。 + +此外，该约æŸå¯èƒ½ä¼šå¹²æ‰°å…¶ä»– Rancher 功能并拒ç»éƒ¨ç½²ç³»ç»Ÿå·¥ä½œè´Ÿè½½ã€‚为é¿å…è¿™ç§æƒ…况，请从你的约æŸä¸­æŽ’除所有 Rancher 特定的命å空间。 + +## åœ¨é›†ç¾¤ä¸­å®žæ–½çº¦æŸ + +如果**执行动作**为 **Deny**，约æŸä¼šç«‹å³å¯ç”¨ï¼Œå¹¶æ‹’ç»ä»»ä½•è¿å策略的请求。默认情况下，执行的值为 **Deny**。 + +如果**执行动作** 为 **Dryrun**，è¿å策略的资æºä»…会记录在约æŸçš„状æ€å­—段中。 + +è¦å¼ºåˆ¶æ‰§è¡Œçº¦æŸï¼Œè¯·ä½¿ç”¨è¡¨å•åˆ›å»ºçº¦æŸã€‚在**执行动作**字段中，选择 **Deny**。 + +## 集群中的审计和è¿è§„ + +OPA Gatekeeper è¿è¡Œå®šæœŸå®¡è®¡ï¼Œä»¥æ£€æŸ¥çŽ°æœ‰èµ„æºæ˜¯å¦è¿å强制执行的约æŸã€‚ä½ å¯ä»¥åœ¨å®‰è£… Gatekeeper æ—¶é…置审计间隔（默认 300 秒）。 + +Gatekeeper 页é¢ä¸Šåˆ—出了è¿å已定义的约æŸçš„情况。 + +此外，你也å¯ä»¥åœ¨**约æŸ**页é¢ä¸­æ‰¾åˆ°è¿å约æŸçš„æ•°é‡ã€‚ + +æ¯ä¸ªçº¦æŸçš„详细信æ¯è§†å›¾åˆ—出了è¿å约æŸçš„资æºçš„ä¿¡æ¯ã€‚ + +## ç¦ç”¨ Gatekeeper + +1. 导航到集群的仪表æ¿è§†å›¾ã€‚ +1. 在左侧èœå•ä¸­ï¼Œå±•å¼€é›†ç¾¤èœå•å¹¶å•å‡» **OPA Gatekeeper**。 +1. å•å‡» **â‹® > ç¦ç”¨**。 + +**结果**：ç¦ç”¨ OPA Gatekeeper åŽï¼Œæ‰€æœ‰çº¦æŸæ¨¡æ¿å’Œçº¦æŸä¹Ÿå°†è¢«åˆ é™¤ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/rancher-extensions.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/rancher-extensions.md new file mode 100644 index 00000000000..0f6de8e5900 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/integrations-in-rancher/rancher-extensions.md @@ -0,0 +1,84 @@ +--- +title: Rancher 扩展 +--- + +Rancher v2.7.0 引入了**扩展（Extension）**的新功能。扩展å…许用户ã€å¼€å‘人员ã€åˆä½œä¼™ä¼´å’Œå®¢æˆ·æ‰©å±•å’Œå¢žå¼º Rancher UI。此外，用户å¯ä»¥ç‹¬ç«‹äºŽ Rancher 版本对其 UI 功能进行更改和增强。有了扩展，用户能够在 Rancher 之上进行构建，从而更好地根æ®çŽ¯å¢ƒè¿›è¡Œå®šåˆ¶ã€‚用户还å¯ä»¥æ›´æ–°åˆ°æ–°ç‰ˆæœ¬ä»¥åŠå›žæ»šåˆ°ä»¥å‰çš„版本。 + +扩展是åªèƒ½åœ¨é›†ç¾¤ä¸­å®‰è£…一次的 Helm Chart。因此，这些 Chart å·²ç»è¿‡ç®€åŒ–，并与 **Apps** 下列出的常规 Helm Chart 分开。 + +内置的 Rancher 扩展示例包括 Fleetã€Explorer å’Œ Harvester。使用å¯æ‰‹åŠ¨æ·»åŠ çš„ Extensions API 的其他扩展示例包括 Kubewarden å’Œ Elemental。 + +## 先决æ¡ä»¶ + +> 你必须以管ç†å‘˜èº«ä»½ç™»å½•æ‰èƒ½æŸ¥çœ‹æ‰©å±•ç®¡ç†é¡µé¢å¹¶ä¸Žä¹‹äº¤äº’。 + +## 安装扩展 + +1. 点击 **Configuration** 下的 **☰ > Extensions**。 + +2. 如果它尚未安装到 **Apps** 中，则你必须通过å•å‡» **Enable** 按钮å¯ç”¨æ‰©å±• operator。 + + - 如果ä¸æ˜¯ç¦»çº¿å®‰è£…环境，请å•å‡» **OK** 添加 Rancher 扩展仓库。å¦åˆ™ï¼Œå–消选中å¤é€‰æ¡†å¹¶å•å‡» **OK**。 + + ![Rancher extension repository](/img/add-rancher-extension-repo.png) + +3. 在 **Extensions** 页é¢ä¸Šï¼Œå•å‡» **Available** 选项å¡é€‰æ‹©è¦å®‰è£…的扩展。 + +:::info + +在 v2.7.0 中，**Available** 选项å¡ä¸‹ä¸ä¼šæ˜¾ç¤ºå†…置扩展。因此，你需è¦æ‰‹åŠ¨æ·»åŠ æ‰€éœ€çš„仓库以安装扩展。一旦这些扩展å¯ç”¨äº†ï¼Œæˆ‘们将åŒæ­¥ç¤¾åŒºã€‚ + +::: +
+ +4. 如果没有显示å¯ç”¨çš„扩展，你å¯ä»¥æ‰‹åŠ¨æ·»åŠ ä»“库，如下所示： + + 4.1. 在å±å¹•å³ä¸Šè§’，点击 **â‹® > Manage Repositories > Create**。 + + 4.2. 添加所需的仓库å称，确ä¿æŒ‡å®šäº† Git 仓库 URL å’Œ Git 分支。 + + 4.3. å†æ¬¡ç‚¹å‡»å³ä¸‹æ–¹çš„ **Create**。 + + ![Manage repositories](/img/manage-repos.png) + +5. 在 **Available** 选项å¡ä¸‹ï¼Œå•å‡»æ‰€éœ€æ‰©å±•å’Œç‰ˆæœ¬ä¸Šçš„ **Install** 按钮，如下例所示。请注æ„，如果扩展程åºå¯ç”¨ï¼Œ**Update** 按钮将出现在该扩展上，因此你å¯ä»¥è½»æ¾æ›´æ–°æ‰©å±•ã€‚ + + ![Install Kubewarden](/img/install-kubewarden.png) + +6. 点击 **Reload** 按钮，该按钮将在你的扩展æˆåŠŸå®‰è£…åŽå‡ºçŽ°ã€‚请注æ„，对于刚刚安装扩展的登录用户而言，**除éž**他们é‡æ–°åŠ è½½é¡µé¢ï¼Œå¦åˆ™ä»–们ä¸ä¼šçœ‹åˆ° UI çš„å˜åŒ–。 + + ![Reload button](/img/reload-button.png) + +## å¸è½½æ‰©å±• + +ä½ å¯ä»¥é€šè¿‡ä¸¤ç§æ–¹å¼å¸è½½æˆ–ç¦ç”¨æ‰©å±•ï¼š + +1. 在 **Installed** 选项å¡ä¸‹ï¼Œå•å‡»è¦åˆ é™¤çš„扩展上的 **Uninstall** 按钮。 + + ![Uninstall extensions](/img/uninstall-extension.png) + +1. 在扩展管ç†é¡µé¢ï¼Œç‚¹å‡» **â‹® > Disable Extension Support**。这将ç¦ç”¨æ‰€æœ‰å·²å®‰è£…的扩展。 + + ![Disable extensions](/img/disable-extension-support.png) + +:::caution + +你必须在ç¦ç”¨æ‰©å±•åŽé‡æ–°åŠ è½½é¡µé¢ï¼Œå¦åˆ™å¯èƒ½ä¼šå‡ºçŽ°æ˜¾ç¤ºé—®é¢˜ã€‚ + +::: + +## 回滚扩展 + +在 **Installed** 选项å¡ä¸‹ï¼Œå•å‡»è¦å›žæ»šçš„扩展上的 **Rollback** 按钮。 + +![Roll back extensions](/img/roll-back-extension.png) + +:::caution + +回滚扩展åŽå¿…é¡»é‡æ–°åŠ è½½é¡µé¢ï¼Œå¦åˆ™å¯èƒ½ä¼šå‡ºçŽ°æ˜¾ç¤ºé—®é¢˜ã€‚ + +::: + +## å¼€å‘扩展 + +è¦äº†è§£å¦‚何开å‘你自己的扩展，请å‚阅官方[入门指å—](https://rancher.github.io/dashboard/extensions/extensions-getting-started)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-provisioning-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-provisioning-drivers.md new file mode 100644 index 00000000000..65868b38bd3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-provisioning-drivers.md @@ -0,0 +1,47 @@ +--- +title: é…置驱动 +--- + +使用 Rancher 中的驱动，你å¯ä»¥ç®¡ç†å¯ä»¥ä½¿ç”¨å“ªäº›ä¾›åº”商æ¥éƒ¨ç½²[托管的 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)或[云æœåŠ¡å™¨èŠ‚点](use-new-nodes-in-an-infra-provider.md)，以å…许 Rancher éƒ¨ç½²å’Œç®¡ç† Kubernetes。 + +### Rancher 驱动 + +ä½ å¯ä»¥å¯ç”¨æˆ–ç¦ç”¨ Rancher 中内置的驱动。如果相关驱动 Rancher 尚未实现，你å¯ä»¥æ·»åŠ è‡ªå·±çš„驱动。 + +Rancher 中有两ç§ç±»åž‹çš„驱动： + +* [集群驱动](#集群驱动) +* [主机驱动](#主机驱动) + +### 集群驱动 + +集群驱动用于é…ç½®[托管的 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)，例如 GKEã€EKSã€AKS 等。创建集群时å¯ä»¥æ˜¾ç¤ºçš„集群驱动，是由集群驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 集群驱动将显示为为托管 Kubernetes 集群创建集群的选项。默认情况下，Rancher 与几个现有的集群驱动打包在一起，但你也å¯ä»¥åˆ›å»ºè‡ªå®šä¹‰é›†ç¾¤é©±åŠ¨å¹¶æ·»åŠ åˆ° Rancher。 + +默认情况下，Rancher 已激活多个托管 Kubernetes 云æ供商，包括： + +* [Amazon EKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +* [Google GKE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +* [Azure AKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) + +还有几个托管的 Kubernetes 云æ供商是默认ç¦ç”¨çš„，但也打包在 Rancher 中： + +* [Alibaba ACK](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +* [Huawei CCE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) +* [Tencent](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) + +### 主机驱动 + +主机驱动用于é…置主机，Rancher 使用这些主机å¯åŠ¨å’Œç®¡ç† Kubernetes 集群。主机驱动与 [Docker Machine 驱动](https://docs.docker.com/machine/drivers/)相åŒã€‚创建主机模æ¿æ—¶å¯ä»¥æ˜¾ç¤ºçš„主机驱动，是由主机驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 主机驱动将显示为创建节点模æ¿çš„选项。默认情况下，Rancher 与许多现有的 Docker Machine 驱动打包在一起，但你也å¯ä»¥åˆ›å»ºè‡ªå®šä¹‰ä¸»æœºé©±åŠ¨å¹¶æ·»åŠ åˆ° Rancher。 + +如果你ä¸æƒ³å‘用户显示特定的主机驱动，则需è¦åœç”¨è¿™äº›ä¸»æœºé©±åŠ¨ã€‚ + +Rancher 支æŒå‡ å®¶ä¸»è¦çš„云æ供商，但默认情况下，这些主机驱动处于 active 状æ€å¹¶å¯ä¾›éƒ¨ç½²ï¼š + +* [Amazon EC2](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) +* [Azure](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md) +* [Digital Ocean](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md) +* [vSphere](vsphere.md) + +还有其他几个默认ç¦ç”¨çš„主机驱动，但打包在 Rancher 中： + +* [Harvester](../integrations-in-rancher/harvester.md#harvester-主机驱动) - 在 Rancher 2.6.1 中å¯ç”¨ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-rke1-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-rke1-templates.md new file mode 100644 index 00000000000..3f4012ac5dd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-rke1-templates.md @@ -0,0 +1,126 @@ +--- +title: RKE æ¨¡æ¿ +--- + +RKE 模æ¿æ—¨åœ¨è®© DevOps 和安全团队标准化和简化 Kubernetes 集群创建的æµç¨‹ã€‚ + +RKE 的全称是 [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/)，它是 Rancher 用æ¥é…ç½® Kubernetes 集群的工具。 + +éšç€ Kubernetes 越æ¥è¶Šå—欢迎，管ç†æ›´å¤šå°åž‹é›†ç¾¤é€æ¸æˆä¸ºè¶‹åŠ¿ã€‚如果你想è¦åˆ›å»ºå¤§é‡é›†ç¾¤ï¼Œå¯¹é›†ç¾¤è¿›è¡Œä¸€è‡´ç®¡ç†å°¤ä¸ºé‡è¦ã€‚多集群管ç†é¢ä¸´ç€å®‰å…¨å’Œé™„件é…置执行的挑战，在将集群移交给最终用户之å‰ï¼Œè¿™äº›é…置需è¦æ ‡å‡†åŒ–。 + +RKE 模æ¿æœ‰åŠ©äºŽæ ‡å‡†åŒ–这些é…置。无论是使用 Rancher UIã€Rancher API 还是自动化æµç¨‹åˆ›å»ºçš„集群，Rancher 都将ä¿è¯ä»Ž RKE 集群模æ¿åˆ›å»ºçš„æ¯ä¸ªé›†ç¾¤åœ¨ç”Ÿæˆæ–¹å¼ä¸Šæ˜¯ä¸€è‡´çš„。 + +管ç†å‘˜å¯ä»¥æŽ§åˆ¶æœ€ç»ˆç”¨æˆ·èƒ½æ›´æ”¹çš„集群选项。RKE 模æ¿è¿˜å¯ä»¥ä¸Žç‰¹å®šçš„用户和组共享，以便管ç†å‘˜å¯ä»¥ä¸ºä¸åŒçš„用户集创建ä¸åŒçš„ RKE 模æ¿ã€‚ + +如果集群是使用 RKE 模æ¿åˆ›å»ºçš„，则ä¸èƒ½è®©é›†ç¾¤ä½¿ç”¨å¦ä¸€ä¸ª RKE 模æ¿ã€‚ä½ åªèƒ½å°†é›†ç¾¤æ›´æ–°ä¸ºåŒä¸€æ¨¡æ¿çš„新版本。 + +ä½ å¯ä»¥[将现有集群的é…ç½®ä¿å­˜ä¸º RKE 模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这样，åªæœ‰æ¨¡æ¿æ›´æ–°åŽæ‰èƒ½æ›´æ”¹é›†ç¾¤çš„设置。新模æ¿è¿˜å¯ç”¨äºŽå¯åŠ¨æ–°é›†ç¾¤ã€‚ + +RKE 模æ¿çš„核心功能å…许 DevOps 和安全团队： + +- 标准化集群é…置并确ä¿æŒ‰ç…§æœ€ä½³å®žè·µåˆ›å»º Rancher é…置的集群 +- é…置集群时，防止用户åšå‡ºä¸æ˜Žæ™ºçš„选择 +- 与ä¸åŒçš„用户和组共享ä¸åŒçš„æ¨¡æ¿ +- 将模æ¿çš„所有æƒå§”托给å—信任的用户进行更改 +- 控制哪些用户å¯ä»¥åˆ›å»ºæ¨¡æ¿ +- è¦æ±‚用户使用模æ¿æ¥åˆ›å»ºé›†ç¾¤ + +## å¯é…置的设置 + +RKE 模æ¿å¯ä»¥åœ¨ Rancher UI 中创建或以 YAML æ ¼å¼å®šä¹‰ã€‚当你使用 Rancher 从基础设施æ供商é…置自定义节点或一般节点时，它们å¯ä»¥æŒ‡å®šä¸ºç›¸åŒçš„å‚数： + +- 云æ供商选项 +- Pod 安全选项 +- 网络æ供商 +- Ingress Controller +- 网络安全é…ç½® +- 网络æ’件 +- ç§æœ‰é•œåƒä»“库 URL å’Œå‡­è¯ +- 附加组件 +- Kubernetes 选项，包括 kube-apiã€kube-controllerã€kubelet å’ŒæœåŠ¡ç­‰ Kubernetes 组件的é…ç½® + +RKE 模æ¿çš„[附加组件](#附加组件)的功能特别强大，因为它å…许多ç§è‡ªå®šä¹‰é€‰é¡¹ã€‚ + +## RKE 模æ¿çš„范围 + +Rancher é…ç½®çš„é›†ç¾¤æ”¯æŒ RKE 模æ¿ã€‚模æ¿å¯ç”¨äºŽé…置自定义集群或由基础设施æ供商å¯åŠ¨çš„集群。 + +RKE 模æ¿ç”¨äºŽå®šä¹‰ Kubernetes å’Œ Rancher 设置。节点模æ¿è´Ÿè´£é…置节点。有关如何将 RKE 模æ¿ä¸Žç¡¬ä»¶ç»“åˆä½¿ç”¨çš„å‚考，请å‚阅 [RKE 模æ¿å’Œç¡¬ä»¶](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md)。 + +å¯ä»¥ä»Žå¤´å¼€å§‹åˆ›å»º RKE 模æ¿æ¥é¢„先定义集群é…置。它们å¯ä»¥ç”¨äºŽå¯åŠ¨æ–°é›†ç¾¤ï¼Œä¹Ÿå¯ä»¥ä»ŽçŽ°æœ‰çš„ RKE 集群导出模æ¿ã€‚ + +现有集群的设置å¯ä»¥[ä¿å­˜ä¸º RKE 模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这会创建一个新模æ¿å¹¶å°†é›†ç¾¤è®¾ç½®ç»‘定到该模æ¿ã€‚这样，集群åªæœ‰åœ¨[模æ¿æ›´æ–°](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#更新模æ¿)的情况下æ‰èƒ½[使用新版本的模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)进行å‡çº§ã€‚新模æ¿ä¹Ÿå¯ä»¥ç”¨æ¥åˆ›å»ºæ–°é›†ç¾¤ã€‚ + + +## 示例场景 +如果一个组织åŒæ—¶æ‹¥æœ‰æ™®é€šå’Œé«˜çº§ Rancher 用户，管ç†å‘˜å¯èƒ½å¸Œæœ›ä¸ºé«˜çº§ç”¨æˆ·æ供更多用于集群创建的选项，并é™åˆ¶æ™®é€šç”¨æˆ·çš„选项。 + +这些[示例场景](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md)æ述组织如何使用模æ¿æ¥æ ‡å‡†åŒ–集群创建。 + +示例场景包括： + +- **强制执行模æ¿**：如果希望所有 Rancher é…置的新集群都具有æŸäº›è®¾ç½®ï¼Œç®¡ç†å‘˜å¯èƒ½æƒ³è¦[为æ¯ä¸ªç”¨æˆ·å¼ºåˆ¶æ‰§è¡Œä¸€é¡¹æˆ–多项模æ¿è®¾ç½®](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#强制执行模æ¿è®¾ç½®)。 +- **与ä¸åŒçš„用户共享ä¸åŒçš„模æ¿**：管ç†å‘˜å¯ä»¥ä¸º[普通用户和高级用户æä¾›ä¸åŒçš„模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#普通用户和高级用户模æ¿)。这样，普通用户会有更多é™åˆ¶é€‰é¡¹ï¼Œè€Œé«˜çº§ç”¨æˆ·åœ¨åˆ›å»ºé›†ç¾¤æ—¶å¯ä»¥ä½¿ç”¨æ›´å¤šé€‰é¡¹ã€‚ +- **更新模æ¿è®¾ç½®**：如果组织的安全和 DevOps 团队决定将最佳实践嵌入到新集群所需的设置中，这些最佳实践å¯èƒ½ä¼šéšç€æ—¶é—´è€Œæ”¹å˜ã€‚如果最佳实践å‘生å˜åŒ–，[å¯ä»¥å°†æ¨¡æ¿æ›´æ–°ä¸ºæ–°ç‰ˆæœ¬](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#更新模æ¿å’Œé›†ç¾¤)，这样，使用模æ¿åˆ›å»ºçš„集群å¯ä»¥[å‡çº§åˆ°æ¨¡æ¿çš„新版本](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)。 +- **共享模æ¿çš„所有æƒ**：当模æ¿æ‰€æœ‰è€…ä¸å†æƒ³è¦ç»´æŠ¤æ¨¡æ¿æˆ–想è¦å…±äº«æ¨¡æ¿çš„所有æƒæ—¶ï¼Œæ­¤æ–¹æ¡ˆæ述了如何[共享模æ¿æ‰€æœ‰æƒ](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#å…许其他用户控制和共享模æ¿)。 + +## 模æ¿ç®¡ç† + +创建 RKE 模æ¿æ—¶ï¼Œå¯ä»¥åœ¨ Rancher UI 中的**集群管ç†**下的 **RKE 模æ¿**中使用模æ¿ã€‚创建模æ¿åŽï¼Œä½ å°†æˆä¸ºæ¨¡æ¿æ‰€æœ‰è€…，这将授予你修改和共享模æ¿çš„æƒé™ã€‚ä½ å¯ä»¥ä¸Žç‰¹å®šç”¨æˆ·æˆ–组共享 RKE 模æ¿ï¼Œä¹Ÿå¯ä»¥å…¬å¼€æ¨¡æ¿ã€‚ + +管ç†å‘˜å¯ä»¥å¼€å¯æ¨¡æ¿å¼ºåˆ¶æ‰§è¡Œï¼Œè¦æ±‚用户在创建集群时始终使用 RKE 模æ¿ã€‚这使管ç†å‘˜å¯ä»¥ä¿è¯ Rancher 总是创建指定é…置的集群。 + +RKE 模æ¿æ›´æ–°é€šè¿‡ä¿®è®¢ç³»ç»Ÿå¤„ç†ã€‚如果è¦æ›´æ”¹æˆ–更新模æ¿ï¼Œè¯·åˆ›å»ºæ¨¡æ¿çš„新版本。然åŽï¼Œå¯ä»¥å°†ä½¿ç”¨æ—§ç‰ˆæœ¬æ¨¡æ¿åˆ›å»ºçš„集群å‡çº§åˆ°æ–°æ¨¡æ¿ä¿®è®¢ç‰ˆã€‚ + +在 RKE 模æ¿ä¸­ï¼Œæ¨¡æ¿æ‰€æœ‰è€…å¯ä»¥é™åˆ¶è®¾ç½®çš„内容，也å¯ä»¥æ‰“开设置以供最终用户选择值。它们的差别体现在，创建模æ¿æ—¶ï¼ŒRancher UI 中的æ¯ä¸ªè®¾ç½®ä¸Šçš„**å…许用户覆盖**标示。 + +对于无法覆盖的设置，最终用户将无法直接编辑它们。为了让用户使用这些设置的ä¸åŒé€‰é¡¹ï¼ŒRKE 模æ¿æ‰€æœ‰è€…需è¦åˆ›å»º RKE 模æ¿çš„新版本，这将å…许用户å‡çº§å’Œæ›´æ”¹è¯¥é€‰é¡¹ã€‚ + +本节中的文件解释了 RKE 模æ¿ç®¡ç†çš„细节： + +- [获å–创建模æ¿çš„æƒé™](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md) +- [创建和修改模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md) +- [强制执行模æ¿è®¾ç½®](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md#强制新集群使用-rke-模æ¿) +- [覆盖模æ¿è®¾ç½®](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md) +- [与集群创建者共享模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#与特定用户或组共享模æ¿) +- [共享模æ¿çš„所有æƒ](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#共享模æ¿æ‰€æœ‰æƒ) + +ä½ å¯ä»¥å‚è§æ­¤[模æ¿çš„示例 YAML 文件](../reference-guides/rke1-template-example-yaml.md)作为å‚考。 + +## åº”ç”¨æ¨¡æ¿ + +ä½ å¯ä»¥ä½¿ç”¨ä½ è‡ªå·±åˆ›å»ºçš„模æ¿æ¥[创建集群](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#使用-rke-模æ¿åˆ›å»ºé›†ç¾¤)，也å¯ä»¥ä½¿ç”¨[与你共享的模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md)æ¥åˆ›å»ºé›†ç¾¤ã€‚ + +如果 RKE 模æ¿æ‰€æœ‰è€…创建了模æ¿çš„新版本，你å¯ä»¥[将你的集群å‡çº§åˆ°è¯¥ç‰ˆæœ¬](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#更新使用-rke-模æ¿åˆ›å»ºçš„集群)。 + +å¯ä»¥ä»Žå¤´å¼€å§‹åˆ›å»º RKE 模æ¿æ¥é¢„先定义集群é…置。它们å¯ä»¥ç”¨äºŽå¯åŠ¨æ–°é›†ç¾¤ï¼Œä¹Ÿå¯ä»¥ä»ŽçŽ°æœ‰çš„ RKE 集群导出模æ¿ã€‚ + +ä½ å¯ä»¥[将现有集群的é…ç½®ä¿å­˜ä¸º RKE 模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这样，åªæœ‰æ¨¡æ¿æ›´æ–°åŽæ‰èƒ½æ›´æ”¹é›†ç¾¤çš„设置。 + +## 标准化硬件 + +RKE 模æ¿çš„目的是标准化 Kubernetes å’Œ Rancher 设置。如果你还想标准化你的基础设施，一个选择是将 RKE 模æ¿ä¸Ž[其他工具](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md)一起使用。 + +å¦ä¸€ç§é€‰æ‹©æ˜¯ä½¿ç”¨åŒ…å«èŠ‚点池é…置选项，但ä¸å¼ºåˆ¶æ‰§è¡Œé…置的[集群模æ¿](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md)。 + +## YAML 定制 + +如果将 RKE 模æ¿å®šä¹‰ä¸º YAML 文件，则å¯ä»¥ä¿®æ”¹æ­¤[示例 RKE æ¨¡æ¿ YAML](../reference-guides/rke1-template-example-yaml.md)。RKE 模æ¿ä¸­çš„ YAML 使用了 Rancher 在创建 RKE 集群时使用的相åŒè‡ªå®šä¹‰è®¾ç½®ã€‚但由于 YAML è¦åœ¨ Rancher é…置的集群中使用，因此需è¦å°† RKE 模æ¿è‡ªå®šä¹‰é¡¹åµŒå¥—在 YAML 中的 `rancher_kubernetes_engine_config` å‚数下。 + +RKE 文档也æä¾›[注释的](https://rancher.com/docs/rke/latest/en/example-yamls/) `cluster.yml` 文件供你å‚考。 + +有关å¯ç”¨é€‰é¡¹çš„更多信æ¯ï¼Œè¯·å‚阅[集群é…ç½®](https://rancher.com/docs/rke/latest/en/config-options/)上的 RKE 文档。 + +### 附加组件 + +RKE 模æ¿é…置文件的附加组件部分的工作方å¼ä¸Ž[集群é…置文件的附加组件部分](https://rancher.com/docs/rke/latest/en/config-options/add-ons/)相åŒã€‚ + +用户定义的附加组件指令å…许你调用和下拉 Kubernetes 清å•æˆ–将它们直接内è”。如果这些 YAML 清å•åŒ…括在 RKE 模æ¿ä¸­ï¼ŒRancher 将在集群中部署这些 YAML 文件。 + +ä½ å¯ä»¥ä½¿ç”¨é™„加组件执行以下æ“作： + +- å¯åŠ¨ Kubernetes 集群åŽï¼Œåœ¨é›†ç¾¤ä¸Šå®‰è£…应用 +- 在使用 Kubernetes Daemonset 部署的节点上安装æ’件 +- 自动设置命å空间ã€ServiceAccount 或角色绑定 + +RKE 模æ¿é…置必须嵌套在 `rancher_kubernetes_engine_config` å‚数中。è¦è®¾ç½®é™„加组件，在创建模æ¿æ—¶å•å‡»**以 YAML 文件编辑**。然åŽä½¿ç”¨ `addons` 指令添加清å•ï¼Œæˆ–使用 `addons_include` 指令设置哪些 YAML 文件å¯ç”¨äºŽé™„加组件。有关自定义附加组件的更多信æ¯ï¼Œè¯·å‚è§[用户自定义附加组件文档](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-the-api.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-the-api.md new file mode 100644 index 00000000000..db0ad233862 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/about-the-api.md @@ -0,0 +1,80 @@ +--- +title: API +--- + +## 如何使用 API + +API 有自己的用户界é¢ï¼Œä½ å¯ä»¥ä»Ž Web æµè§ˆå™¨è®¿é—®å®ƒã€‚这是查看资æºã€æ‰§è¡Œæ“作以åŠæŸ¥çœ‹ç­‰æ•ˆ cURL 或 HTTP 请求和å“应的一ç§ç®€å•çš„方法。è¦è®¿é—®å®ƒï¼š + + + + +1. å•å‡»å³ä¸Šè§’的用户头åƒã€‚ +1. å•å‡»**è´¦å· & API 密钥**。 +1. 在 **API 密钥**下，找到 **API 端点**字段并å•å‡»é“¾æŽ¥ã€‚该链接类似于 `https:///v3`，其中 `` 是 Rancher deployment 的完全é™å®šåŸŸå。 + + + + +转到ä½äºŽ `https:///v3` çš„ URL 端点，其中 `` 是你的 Rancher deployment 的完全é™å®šåŸŸå。 + + + + +## èº«ä»½éªŒè¯ + +API 请求必须包å«èº«ä»½éªŒè¯ä¿¡æ¯ã€‚身份验è¯æ˜¯é€šè¿‡ [API 密钥](../reference-guides/user-settings/api-keys.md)使用 HTTP 基本身份验è¯å®Œæˆçš„。API 密钥å¯ä»¥åˆ›å»ºæ–°é›†ç¾¤å¹¶é€šè¿‡ `/v3/clusters/` 访问多个集群。[集群和项目角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)会应用于这些键，并é™åˆ¶è´¦å·å¯ä»¥æŸ¥çœ‹çš„集群和项目以åŠå¯ä»¥æ‰§è¡Œçš„æ“作。 + +默认情况下，æŸäº›é›†ç¾¤çº§åˆ«çš„ API 令牌是使用无é™æœŸ TTL（`ttl=0`）生æˆçš„。æ¢è¨€ä¹‹ï¼Œé™¤éžä½ è®©ä»¤ç‰Œå¤±æ•ˆï¼Œå¦åˆ™ `ttl=0` çš„ API 令牌永远ä¸ä¼šè¿‡æœŸã€‚有关如何使 API 令牌失效的详细信æ¯ï¼Œè¯·å‚阅 [API 令牌](../reference-guides/about-the-api/api-tokens.md)。 + +## å‘出请求 + +该 API 通常是 RESTful 的，但是还具有多ç§åŠŸèƒ½ã€‚这些功能å¯ä»¥ä½¿å®¢æˆ·ç«¯å‘现所有内容，因此å¯ä»¥ç¼–写通用客户端，而ä¸å¿…为æ¯ç§èµ„æºç¼–写特定代ç ã€‚有关通用 API 规范的详细信æ¯ï¼Œè¯·å‚阅[此处](https://github.com/rancher/api-spec/blob/master/specification.md)。 + +- æ¯ç§ç±»åž‹éƒ½æœ‰ä¸€ä¸ª Schema，这个 Schema æ述了以下内容： + - 用于获å–此类资æºé›†åˆçš„ URL + - 资æºå¯ä»¥å…·æœ‰çš„æ¯ä¸ªå­—段åŠå…¶ç±»åž‹ã€åŸºæœ¬éªŒè¯è§„则ã€æ˜¯å¿…填还是å¯é€‰å­—段等 + - 在此类资æºä¸Šå¯ä»¥æ‰§è¡Œçš„æ¯ä¸ªæ“作，以åŠå®ƒä»¬çš„输入和输出（也作为 schema） + - å…许过滤的æ¯ä¸ªå­—段 + - 集åˆæœ¬èº«æˆ–集åˆä¸­çš„å•ä¸ªèµ„æºå¯ä»¥ä½¿ç”¨çš„ HTTP æ“作方法 + + +- 因此，你å¯ä»¥åªåŠ è½½ schema 列表并了解 API 的所有信æ¯ã€‚实际上，这是 API çš„ UI 工作方å¼ï¼Œå®ƒä¸åŒ…å«ç‰¹å®šäºŽ Rancher 本身的代ç ã€‚æ¯ä¸ª HTTP å“应中的 `X-Api-Schemas` 标头都会å‘é€èŽ·å– Schemas çš„ URL。你å¯ä»¥æŒ‰ç…§æ¯ä¸ª schema 上的 `collection` 链接了解è¦åœ¨å“ªé‡Œåˆ—出资æºï¼Œå¹¶åœ¨è¿”回资æºä¸­çš„其他 `links` 中获å–其他信æ¯ã€‚ + +- 在实践中，你å¯èƒ½åªæƒ³æž„造 URL 字符串。我们强烈建议将此é™åˆ¶ä¸ºåœ¨é¡¶å±‚åˆ—å‡ºçš„é›†åˆ (`/v3/`)，或获å–ç‰¹å®šèµ„æº (`/v3//`)。除此之外的任何内容都å¯èƒ½åœ¨å°†æ¥çš„版本中å‘生更改。 + +- 资æºä¹‹é—´ç›¸äº’之间有è”系，称为链接（links）。æ¯ä¸ªèµ„æºéƒ½åŒ…å«ä¸€ä¸ª `links` 映射，其中包å«é“¾æŽ¥å称和用于检索该信æ¯çš„ URL。åŒæ ·ï¼Œä½ åº”该 `GET` 资æºå¹¶éµå¾ª `links` 映射中的 URL，而ä¸æ˜¯è‡ªå·±æž„造这些字符串。 + +- 大多数资æºéƒ½æœ‰æ“作（action），表示å¯ä»¥æ‰§è¡ŒæŸä¸ªæ“作或改å˜èµ„æºçš„状æ€ã€‚è¦ä½¿ç”¨æ“作，请将 HTTP `POST` 请求å‘é€åˆ° `actions` 映射中你想è¦çš„æ“作的 URL。æŸäº›æ“作需è¦è¾“入或生æˆè¾“出，请å‚阅æ¯ç§ç±»åž‹çš„独立文档或 schema 以获å–具体信æ¯ã€‚ + +- è¦ç¼–辑资æºï¼Œè¯·å°† HTTP `PUT` 请求å‘é€åˆ°èµ„æºä¸Šçš„ `links.update` 链接，其中包å«è¦æ›´æ”¹çš„字段。如果链接丢失，则你无æƒæ›´æ–°èµ„æºã€‚未知字段和ä¸å¯ç¼–辑的字段将被忽略。 + +- è¦åˆ é™¤èµ„æºï¼Œè¯·å°† HTTP `DELETE` 请求å‘é€åˆ°èµ„æºä¸Šçš„ `links.remove` 链接。如果链接丢失，则你无æƒæ›´æ–°èµ„æºã€‚ + +- è¦åˆ›å»ºæ–°èµ„æºï¼ŒHTTP `POST` 到 schemaï¼ˆå³ `/v3/`ï¼‰ä¸­çš„é›†åˆ URL。 + +## 过滤 + +ä½ å¯ä»¥ä½¿ç”¨ HTTP 查询å‚数的公共字段在æœåŠ¡å™¨ç«¯è¿‡æ»¤å¤§å¤šæ•°é›†åˆã€‚`filters` 映射显示了å¯ä»¥è¿‡æ»¤çš„字段，以åŠè¿‡æ»¤åŽçš„值在你å‘起的请求中是什么。API UI 具有设置过滤和显示适当请求的控件。对于简å•çš„ "equals" 匹é…，它åªæ˜¯ `field=value`。你å¯ä»¥å°†ä¿®é¥°ç¬¦æ·»åŠ åˆ°å­—段å称，例如 `field_gt=42` 表示“字段大于 42â€ã€‚详情请å‚阅 [API 规范](https://github.com/rancher/api-spec/blob/master/specification.md#filtering)。 + +## æŽ’åº + +ä½ å¯ä»¥ä½¿ç”¨ HTTP 查询å‚数的公共字段在æœåŠ¡å™¨ç«¯æŽ’åºå¤§å¤šæ•°é›†åˆã€‚`sortLinks` 映射显示了å¯ç”¨çš„排åºï¼Œä»¥åŠç”¨äºŽèŽ·å–éµå¾ªè¯¥æŽ’åºçš„集åˆçš„ URL。它还包括当å‰å“排åºä¾æ®çš„ä¿¡æ¯ï¼ˆå¦‚果指定）。 + +## 分页 + +默认情况下，API å“应以æ¯é¡µ 100 个资æºçš„é™åˆ¶è¿›è¡Œåˆ†é¡µã€‚ä½ å¯ä»¥é€šè¿‡ `limit` 查询å‚数进行更改，最大为 1000，例如 `/v3/pods?limit=1000`。集åˆå“应中的 `pagination` 映射能让你知é“你是å¦æ‹¥æœ‰å®Œæ•´çš„结果集，如果没有，则会指å‘下一页的链接。 + +## æ•èŽ· Rancher API 调用 + +ä½ å¯ä»¥ä½¿ç”¨æµè§ˆå™¨å¼€å‘人员工具æ¥æ•èŽ· Rancher API 的调用方å¼ã€‚例如，你å¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤ä½¿ç”¨ Chrome å¼€å‘人员工具æ¥èŽ·å–用于é…ç½® RKE 集群的 API 调用： + +1. 在 Rancher UI 中，转到**集群管ç†**并å•å‡»**创建**。 +1. å•å‡»æŸä¸ªé›†ç¾¤ç±»åž‹ã€‚此示例使用 Digital Ocean。 +1. 使用集群å称和节点模æ¿å¡«å†™è¡¨å•ï¼Œä½†ä¸è¦å•å‡»**创建**。 +1. 在创建集群之å‰ï¼Œä½ éœ€è¦æ‰“开开å‘人员工具æ‰èƒ½çœ‹åˆ°æ­£åœ¨è®°å½•çš„ API 调用。è¦æ‰“开工具，å³é”®å•å‡» Rancher UI，然åŽå•å‡»**检查**。 +1. 在开å‘者工具中，å•å‡» **Network** 选项å¡ã€‚ +1. 在 **Network** 选项å¡ä¸Šï¼Œç¡®ä¿é€‰æ‹©äº† **Fetch/XHR**。 +1. 在 Rancher UI 中，å•å‡»**创建**。在开å‘者工具中，你应该会看到一个å为 `cluster?_replace=true` 的新网络请求。 +1. å³é”®å•å‡» `cluster?_replace=true` 并å•å‡»**å¤åˆ¶ > å¤åˆ¶ä¸º cURL**。 +1. 将结果粘贴到文本编辑器中。你将能够看到 POST 请求，包括被å‘é€åˆ°çš„ URLã€æ‰€æœ‰æ ‡å¤´ä»¥åŠè¯·æ±‚的完整正文。此命令å¯ç”¨äºŽä»Žå‘½ä»¤è¡Œåˆ›å»ºé›†ç¾¤ã€‚请注æ„，请求包å«å‡­è¯ï¼Œå› æ­¤è¯·å°†è¯·æ±‚存储在安全的地方。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/access-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/access-clusters.md new file mode 100644 index 00000000000..72cd215bb47 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/access-clusters.md @@ -0,0 +1,61 @@ +--- +title: 集群访问 +--- + +本节介ç»å¯ä»¥ç”¨æ¥è®¿é—® Rancher 管ç†çš„集群的工具。 + +有关如何授予用户访问集群的æƒé™çš„ä¿¡æ¯ï¼Œè¯·å‚阅[将用户添加到集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md)。 + +有关 RBAC 的更多信æ¯ï¼Œè¯·å‚阅[本节](manage-role-based-access-control-rbac.md)。 + +有关如何设置身份验è¯ç³»ç»Ÿçš„ä¿¡æ¯ï¼Œè¯·å‚阅[本节](authentication-config.md)。 + +## Rancher UI 中的集群 + +我们æ供了多ç§é€šè¿‡ Rancher UI 查看和管ç†é›†ç¾¤çš„方法。 + +### é›†ç¾¤é¡µé¢ + +从 **☰** èœå•è®¿é—®**集群**页é¢ï¼š + +1. å•å‡» **☰**。 +1. 选择**集群管ç†**。 + +你还å¯ä»¥é€šè¿‡å•å‡» Rancher UI **主页**集群表格上方的**管ç†**按钮æ¥è®¿é—®**集群**页é¢ã€‚ + +在**集群**页é¢ä¸Šï¼Œé€‰æ‹©æ¯è¡Œæœ«å°¾çš„ **â** 以查看包å«ä»¥ä¸‹é€‰é¡¹çš„å­èœå•ï¼š + +* [Kubectl Shell](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) +* 下载 KubeConfig +* å°† KubeConfig å¤åˆ¶åˆ°å‰ªåˆ‡æ¿ +* 编辑é…ç½® +* 查看 YAML +* 下载 YAML + +### é›†ç¾¤ä»ªè¡¨æ¿ + +在**集群**页é¢ä¸Šï¼Œé€‰æ‹©æ¯è¡Œæœ«å°¾çš„**æµè§ˆ**按钮查看该集群的**集群仪表æ¿**。你还å¯ä»¥é€šè¿‡å•å‡»è¡¨ä¸­é›†ç¾¤çš„å称，然åŽå•å‡»**集群**页é¢ä¸Šçš„**æµè§ˆ**按钮æ¥æŸ¥çœ‹ä»ªè¡¨æ¿ã€‚ + +也å¯ä»¥é€šè¿‡å•å‡»é›†ç¾¤å称从 Rancher UI **主页**访问**集群仪表æ¿**。 + +你还å¯ä»¥ä»Žé¡¶éƒ¨å¯¼èˆªæ ä¸­çš„ **☰** 访问**集群仪表æ¿**： + +1. å•å‡» **☰**。 +1. 从**æµè§ˆé›†ç¾¤**èœå•ä¸­é€‰æ‹©é›†ç¾¤çš„å称。 + +**集群仪表æ¿**列出了集群相关的信æ¯ï¼Œä¾‹å¦‚节点数é‡ã€å†…存使用情况ã€äº‹ä»¶å’Œèµ„æºã€‚ + +## kubectl + +ä½ å¯ä»¥ä½¿ç”¨ Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/reference/kubectl/overview/) æ¥ç®¡ç†ä½ çš„集群。使用 kubectl 有两ç§é€‰æ‹©ï¼š + +- **Rancher kubectl shell**：通过å¯åŠ¨ Rancher UI 中å¯ç”¨çš„ kubectl shell 与集群交互。此选项ä¸éœ€è¦ä½ è¿›è¡Œä»»ä½•é…ç½®æ“作。有关详细信æ¯ï¼Œè¯·å‚阅[使用 kubectl Shell 访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 +- **终端远程连接**：你也å¯ä»¥é€šè¿‡åœ¨æœ¬åœ°æ¡Œé¢ä¸Šå®‰è£… [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)，然åŽå°†é›†ç¾¤çš„ kubeconfig 文件å¤åˆ¶åˆ°æœ¬åœ° `~/.kube/config` 目录æ¥ä¸Žé›†ç¾¤äº¤äº’。有关更多信æ¯ï¼Œè¯·å‚阅[使用 kubectl å’Œ kubeconfig 文件访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 + +## Rancher CLI + +ä½ å¯ä»¥ä¸‹è½½ Rancher 自己的命令行工具 [Rancher CLI](cli-with-rancher.md) æ¥æŽ§åˆ¶ä½ çš„集群。这个 CLI 工具å¯ä»¥ç›´æŽ¥ä¸Žä¸åŒçš„集群和项目进行交互，或者å‘它们传递 `kubectl` 命令。 + +## Rancher API + +最åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher API 与集群进行交互。在使用 API 之å‰ï¼Œä½ å¿…é¡»å…ˆèŽ·å– [API 密钥](../reference-guides/user-settings/api-keys.md)。è¦æŸ¥çœ‹ API 对象的ä¸åŒèµ„æºå­—段和æ“作，请打开 API UI（API UI å¯ä»¥é€šè¿‡å•å‡» Rancher UI 对象的**在 API 中查看**访问）。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/advanced-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/advanced-configuration.md new file mode 100644 index 00000000000..cf942d3dac5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/advanced-configuration.md @@ -0,0 +1,15 @@ +--- +title: 高级é…ç½® +--- + +### Alertmanager + +有关é…ç½® Alertmanager 自定义资æºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 + +### Prometheus + +有关é…ç½® Prometheus 自定义资æºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md)。 + +### PrometheusRules + +有关é…ç½® Prometheus 自定义资æºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/advanced-user-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/advanced-user-guides.md new file mode 100644 index 00000000000..0f8f5a7df69 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/advanced-user-guides.md @@ -0,0 +1,7 @@ +--- +title: é«˜çº§ç”¨æˆ·æŒ‡å— +--- + +高级用户指å—是“问题导å‘â€çš„文档，用户å¯ä»¥ä»Žä¸­å­¦ä¹ å¦‚何解决问题。高级用于指å—与新用户指å—的主è¦åŒºåˆ«åœ¨äºŽï¼Œé«˜çº§ç”¨æˆ·æŒ‡å—é¢å‘更有ç»éªŒæˆ–更高级的用户，这些用户对文档有更多的技术需求，而且已ç»äº†è§£ Rancher åŠå…¶åŠŸèƒ½ã€‚他们知é“自己需è¦åšä»€ä¹ˆï¼Œåªæ˜¯éœ€è¦é¢å¤–的指导æ¥å®Œæˆæ›´å¤æ‚的任务。 + +应该注æ„的是，新用户指å—和高级用户指å—都没有æ供详细的解释或讨论（这些文档ä¸åŒ…括在本部分）。æ“作指å—侧é‡äºŽå¼•å¯¼ç”¨æˆ·é€šè¿‡å¯é‡å¤ã€æœ‰æ•ˆçš„步骤æ¥å­¦ä¹ æ–°æŠ€èƒ½ã€æŽŒæ¡æŸäº›æ“作或解决æŸäº›é—®é¢˜ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/air-gapped-helm-cli-install.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/air-gapped-helm-cli-install.md new file mode 100644 index 00000000000..b5b58f77e56 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/air-gapped-helm-cli-install.md @@ -0,0 +1,31 @@ +--- +title: 离线 Helm CLI 安装 +--- + +本文介ç»å¦‚何使用 Helm CLI 在离线环境中安装 Rancher Server。离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +Rancher 安装在 RKE Kubernetes 集群ã€K3s Kubernetes 集群，或å•ä¸ª Docker 容器上对应的安装步骤会有所ä¸åŒã€‚ + +如需了解å„个安装方å¼çš„更多信æ¯ï¼Œè¯·å‚è§[本页](installation-and-upgrade.md)。 + +在安装指导中，我们为ä¸åŒçš„安装选项æ供对应的 _选项å¡_ 。 + +:::note é‡è¦æ示： + +如果你按照 Docker 安装指å—安装 Rancher，你将没有把 Docker 安装转æ¢ä¸º Kubernetes 安装的å‡çº§é€”径。 + +::: + +## å®‰è£…æ¦‚è¦ + +1. [设置基础设施和ç§æœ‰é•œåƒä»“库](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) +2. [收集镜åƒåˆ°ç§æœ‰é•œåƒä»“库](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md) +3. [设置 Kubernetes 集群（如果你使用 Docker 安装，请跳过此步骤）](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md) +4. [安装 Rancher](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md) + +## å‡çº§ + +如需在离线环境中使用 Helm CLI å‡çº§ Rancher，请按照[å‡çº§æ­¥éª¤](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md)进行æ“作。 + +### åŽç»­æ“作 +[准备节点](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/authentication-config.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/authentication-config.md new file mode 100644 index 00000000000..a77cbe0b204 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/authentication-config.md @@ -0,0 +1,132 @@ +--- +title: 身份验è¯é…ç½® +weight: 10 +--- + +Rancher å‘ Kubernetes 添加的关键功能之一，就是集中å¼ç”¨æˆ·èº«ä»½éªŒè¯ã€‚此功能å…许你的用户使用一组凭è¯å¯¹ä½ çš„所有 Kubernetes 集群进行身份验è¯ã€‚ + +è¿™ç§é›†ä¸­å¼çš„用户身份验è¯æ˜¯ä½¿ç”¨ Rancher 身份验è¯ä»£ç†å®Œæˆçš„，该代ç†ä¸Ž Rancher 的其他组件一起安装。这个代ç†éªŒè¯ä½ çš„用户，并使用一个 ServiceAccount 将用户请求转å‘到你的 Kubernetes 集群。 + +## 外部验è¯ä¸Žæœ¬åœ°éªŒè¯ + +Rancher 身份验è¯ä»£ç†æ”¯æŒä¸Žä»¥ä¸‹å¤–部身份验è¯æœåŠ¡é›†æˆï¼š + +| 验è¯æœåŠ¡ | +| ------------------------------------------------------------------------------------------------ | +| [Microsoft Active Directory](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md) | +| [GitHub](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md) | +| [Microsoft Azure AD](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md) | +| [FreeIPA](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md) | +| [OpenLDAP](configure-openldap.md) | +| [Microsoft AD FS](configure-microsoft-ad-federation-service-saml.md) | +| [PingIdentity](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md) | +| [Keycloak (OIDC)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md) | +| [Keycloak (SAML)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md) | +| [Okta](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md) | +| [Google OAuth](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md) | +| [Shibboleth](configure-shibboleth-saml.md) | + +åŒæ—¶ï¼ŒRancher 也æ供了[本地身份验è¯](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md)。 + +大多数情况下，应该使用外部身份验è¯æœåŠ¡ï¼Œè€Œä¸æ˜¯æœ¬åœ°èº«ä»½éªŒè¯ï¼Œå› ä¸ºå¤–部身份验è¯å…许对用户进行集中管ç†ã€‚但是你å¯èƒ½éœ€è¦ä¸€äº›æœ¬åœ°èº«ä»½éªŒè¯ç”¨æˆ·ï¼Œä»¥ä¾¿åœ¨ç‰¹å®šçš„情况下（例如在外部身份验è¯ç³»ç»Ÿä¸å¯ç”¨æˆ–æ­£åœ¨è¿›è¡Œç»´æŠ¤æ—¶ï¼‰ç®¡ç† Rancher。 + +## 用户和组 + +Rancher ä¾èµ–用户和组æ¥å†³å®šå…许登录到 Rancher 的用户，以åŠä»–们å¯ä»¥è®¿é—®å“ªäº›èµ„æºã€‚使用外部系统进行身份验è¯æ—¶ï¼Œå°†ç”±å¤–部系统æ供用户和组。这些用户和组被赋予集群ã€é¡¹ç›®ã€å¤šé›†ç¾¤åº”用ã€å…¨å±€ DNS æ供商等资æºçš„特定角色。当你将访问æƒé™æŽˆäºˆæŸä¸ªç»„时，身份验è¯æ供程åºä¸­å±žäºŽè¯¥ç»„的所有用户都将能够使用你指定的æƒé™è®¿é—®è¯¥èµ„æºã€‚有关角色和æƒé™çš„更多信æ¯ï¼Œè¯·å‚è§ [RBAC](manage-role-based-access-control-rbac.md)。 + +:::note + +本地认è¯ä¸æ”¯æŒåˆ›å»ºæˆ–管ç†ç”¨æˆ·ç»„。 + +::: + +详情请å‚è§[用户和组](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md)。 + +## Rancher 授æƒèŒƒå›´ + +å°† Rancher é…ç½®æˆå…许使用外部验è¯æ供程åºç™»å½•åŽï¼Œä½ éœ€è¦é…ç½®å…许登录和使用 Rancher 的用户。å¯ç”¨é€‰é¡¹å¦‚下： + +| 访问级别 | æè¿° | +|----------------------------------------------|-------------| +| å…许任何有效用户 | 授æƒæœåŠ¡ä¸­çš„ _任何_ 用户都å¯ä»¥è®¿é—®Rancher。通常ä¸å»ºè®®ä½¿ç”¨æ­¤è®¾ç½®ã€‚ | +| å…许集群和项目æˆå‘˜ï¼Œä»¥åŠæŽˆæƒçš„用户和组织 | 认è¯æœåŠ¡ä¸­çš„任何用户，以åŠæ·»åŠ ä¸º**集群æˆå‘˜**或**项目æˆå‘˜**的任何组都å¯ä»¥ç™»å½•åˆ° Rancher。此外，添加到**授æƒç”¨æˆ·å’Œç»„织**列表中的身份验è¯æœåŠ¡ä¸­çš„任何用户和组都能登录到 Rancher。 | +| ä»…å…许授æƒç”¨æˆ·å’Œç»„织 | åªæœ‰æ·»åŠ åˆ°**授æƒç”¨æˆ·å’Œç»„织**的身份验è¯æœåŠ¡ä¸­çš„用户和组能登录 Rancher。 | + +è¦åœ¨æŽˆæƒæœåŠ¡ä¸­ä¸ºç”¨æˆ·è®¾ç½® Rancher 访问级别，请执行以下步骤： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. å•å‡»å·¦ä¾§å¯¼èˆªæ çš„**认è¯**。 +1. 设置好认è¯æ供程åºçš„é…ç½®åŽï¼Œä½¿ç”¨ **Site Access** 选项æ¥é…置用户的授æƒèŒƒå›´ã€‚上表说明了æ¯ä¸ªé€‰é¡¹çš„访问级别。 +1. å¯é€‰ï¼šå¦‚果你选择**å…许任何有效用户**以外的选项，你å¯ä»¥é€šè¿‡åœ¨æ˜¾ç¤ºçš„文本字段中æœç´¢ç”¨æˆ·ï¼Œå°†ç”¨æˆ·æ·»åŠ åˆ°**授æƒç”¨æˆ·å’Œç»„织**的列表中。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：Rancher 访问é…置已应用。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## 外部身份验è¯é…置和用户主体 + +é…置外部认è¯éœ€è¦ï¼š + +- 分é…了管ç†å‘˜è§’色的本地用户，以下称为 _本地主体_。 +- å¯ä»¥ä½¿ç”¨å¤–部认è¯æœåŠ¡è¿›è¡Œè®¤è¯çš„外部用户，以下称为 _外部主体_。 + +外部身份验è¯çš„é…ç½®å°†å½±å“ Rancher 中主体用户的管ç†æ–¹å¼ã€‚按照下é¢çš„列表æ¥æ›´å¥½åœ°ç†è§£è¿™äº›å½±å“。 + +1. 作为本地主体登录到 Rancher 并完æˆå¤–部身份验è¯çš„é…置。 + + ![Sign In](/img/sign-in.png) + +2. Rancher 将外部主体与本地主体相关è”。这两个用户共享本地主体的用户 ID。 + + ![Principal ID Sharing](/img/principal-ID.png) + +3. 完æˆé…ç½®åŽï¼ŒRancher 将自动退出本地主体。 + + ![Sign Out Local Principal](/img/sign-out-local.png) + +4. 然åŽï¼ŒRancher 会自动将你作为外部主体é‡æ–°ç™»å½•ã€‚ + + ![Sign In External Principal](/img/sign-in-external.png) + +5. 由于外部主体与本地主体共享一个 ID，因此**用户**页é¢ä¸ä¼šå†å•ç‹¬æ˜¾ç¤ºå¤–部主体的对象。 + + ![Sign In External Principal](/img/users-page.png) + +6. 外部主体和本地主体共享相åŒçš„访问æƒé™ã€‚ + +:::note é‡æ–°é…置以å‰è®¾ç½®çš„身份验è¯æä¾›ç¨‹åº + +如果你需è¦é‡æ–°é…置或ç¦ç”¨ä»¥å‰è®¾ç½®çš„æ供程åºç„¶åŽå†é‡æ–°å¯ç”¨å®ƒï¼Œè¯·ç¡®ä¿è¿›è¡Œæ­¤æ“作的用户使用外部用户身份登录 Rancher，而ä¸æ˜¯æœ¬åœ°ç®¡ç†å‘˜ã€‚ + +::: + +## ç¦ç”¨è®¤è¯æä¾›ç¨‹åº + +ç¦ç”¨èº«ä»½è®¤è¯æ供程åºæ—¶ï¼ŒRancher 会删除与其关è”的所有资æºï¼Œä¾‹å¦‚： +- Secrets +- 全局角色绑定。 +- 集群角色模æ¿ç»‘定。 +- 项目角色模æ¿ç»‘定。 +- 与æ供商关è”的外部用户，但是这些用户从未以本地用户身份登录到 Rancher。 + +由于此æ“作å¯èƒ½ä¼šå¯¼è‡´è®¸å¤šèµ„æºä¸¢å¤±ï¼Œå› æ­¤ä½ å¯èƒ½å¸Œæœ›åœ¨æ供程åºä¸Šæ·»åŠ ä¿æŠ¤æŽªæ–½ã€‚为确ä¿åœ¨ç¦ç”¨èº«ä»½è®¤è¯æ供程åºæ—¶ä¸ä¼šè¿è¡Œæ­¤æ¸…ç†ï¼Œè¯·å‘相应的身份认è¯é…置添加特殊注释。 + +例如，è¦ä¸º Azure AD æ供程åºæ·»åŠ å®‰å…¨æŽªæ–½ï¼Œè¯·æ³¨é‡Š `azuread` authconfig 对象： + +`kubectl annotate --overwrite authconfig azuread management.cattle.io/auth-provider-cleanup='user-locked'` + +在你将注释设置为 `unlocked` 之å‰ï¼ŒRancher ä¸ä¼šæ‰§è¡Œæ¸…ç†ã€‚ + +### 手动è¿è¡Œèµ„æºæ¸…ç† + +å³ä½¿åœ¨ä½ é…置了å¦ä¸€ä¸ªèº«ä»½è®¤è¯æ供程åºï¼ŒRancher 也å¯èƒ½ä¼šä¿ç•™ local 集群中已ç¦ç”¨çš„身份认è¯æ供程åºé…置的资æºã€‚例如，如果你使用 Provider A，然åŽç¦ç”¨äº†å®ƒå¹¶å¼€å§‹ä½¿ç”¨ Provider B，当你å‡çº§åˆ°æ–°ç‰ˆæœ¬çš„ Rancher 时，你å¯ä»¥æ‰‹åŠ¨è§¦å‘对 Provider A é…置的资æºçš„清ç†ã€‚ + +è¦ä¸ºå·²ç¦ç”¨çš„身份认è¯æ供程åºæ‰‹åŠ¨è§¦å‘清ç†ï¼Œè¯·å°†å¸¦æœ‰ `unlocked` 值的 `management.cattle.io/auth-provider-cleanup` 注释添加到 auth é…置中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/authentication-permissions-and-global-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/authentication-permissions-and-global-configuration.md new file mode 100644 index 00000000000..88ed5f38751 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/authentication-permissions-and-global-configuration.md @@ -0,0 +1,81 @@ +--- +title: 身份验è¯ã€æƒé™å’Œå…¨å±€è®¾ç½® +--- + +安装完æˆåŽï¼Œ[系统管ç†å‘˜](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)需è¦é…ç½® Rancher æ¥é…置身份验è¯ï¼Œå®‰å…¨ï¼Œé»˜è®¤è®¾å®šï¼Œå®‰å…¨ç­–略，驱动和全局 DNS æ¡ç›®ã€‚ + +## 首次登录 + +首次登录 Rancher åŽï¼ŒRancher 会æ示你输入 **Rancher Server URL**。你需è¦å°† URL 设置为 Rancher Server 的主è¦å…¥å£ç‚¹ã€‚当负载å‡è¡¡å™¨ä½äºŽ Rancher Server 集群å‰é¢æ—¶ï¼ŒURL 需è¦è®¾ç½®ä¸ºè´Ÿè½½å‡è¡¡å™¨åœ°å€ã€‚系统会自动å°è¯•ä»Žè¿è¡Œ Rancher Server 的主机的 IP 地å€æˆ–主机å推断 Rancher Server çš„URL，上述推断仅在你è¿è¡Œå•èŠ‚点 Rancher Server æ—¶æ‰æ­£ç¡®ã€‚因此，在大多数情况下，你需è¦è‡ªå·±å°† Rancher Server çš„ URL 设置为正确的值。 + +:::danger + +Rancher Server çš„ URL 在设置åŽä¸å¯å†æ›´æ–°ã€‚因此，你需è¦è°¨æ…Žè®¾ç½®è¯¥ URL。 + +::: + +## èº«ä»½éªŒè¯ + +Rancher å‘ Kubernetes 添加的关键功能之一，就是集中å¼ç”¨æˆ·èº«ä»½éªŒè¯ã€‚此功能å…许将本地用户连接到外部身份验è¯ç³»ç»Ÿï¼Œä½¿ç”¨è¯¥ç³»ç»Ÿçš„用户和组进行身份验è¯ã€‚ + +有关身份验è¯å¦‚何工作åŠå¦‚何设置外部身份认è¯ç³»ç»Ÿï¼Œè¯·å‚è§[身份验è¯](authentication-config.md)。 + +## æŽˆæƒ + +Rancher 通过 _用户_ 进行授æƒç®¡ç†ã€‚用户的 _授æƒ_ 或系统访问æƒé™ç”±ç”¨æˆ·è§’色决定。Rancher æ供了预设角色，让你轻æ¾é…置用户对资æºçš„æƒé™ï¼Œè¿˜æ供了为æ¯ä¸ª Kubernetes 资æºå®šåˆ¶è§’色的能力。 + +有关授æƒå¦‚何工作åŠå¦‚何自定义角色，请å‚è§ [RBAC](manage-role-based-access-control-rbac.md)。 + +## Pod 安全策略 + +_Pod 安全策略（PSP）_ 是用æ¥æŽ§åˆ¶å®‰å…¨æ•æ„Ÿç›¸å…³ Pod 规范（例如 root 特æƒï¼‰çš„对象。如果æŸä¸ª Pod ä¸æ»¡è¶³ PSP 指定的æ¡ä»¶ï¼ŒKubernetes å°†ä¸å…许它å¯åŠ¨ï¼Œå¹¶åœ¨ Rancher 中显示错误消æ¯ã€‚ + +有关如何创建和使用 PSP，请å‚è§ [Pod 安全策略](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +## é…置驱动 + +使用 Rancher 中的驱动，你å¯ä»¥ç®¡ç†å¯ä»¥ä½¿ç”¨å“ªäº›ä¾›åº”商æ¥é…ç½®[托管的 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)或[云æœåŠ¡å™¨èŠ‚点](use-new-nodes-in-an-infra-provider.md)，以å…许 Rancher éƒ¨ç½²å’Œç®¡ç† Kubernetes。 + +详情请å‚考[é…置驱动](about-provisioning-drivers.md)。 + +## 添加 Kubernetes 版本到 Rancher + +ä½ å¯ä»¥é€šè¿‡è¿™ä¸ªåŠŸèƒ½ï¼Œåœ¨ä¸å‡çº§ Rancher 的情况下，å‡çº§åˆ°æœ€æ–°å‘布的 Kubernetes 版本。Kubernetes 倾å‘于在次è¦ç‰ˆæœ¬åˆ é™¤æˆ–新增 API 接å£ã€‚本功能让你轻æ¾å‡çº§ Kubernetes è¡¥ä¸ç‰ˆæœ¬ï¼ˆå³ `v1.15.X`），但ä¸å‡çº§ Kubernetes 次è¦ç‰ˆæœ¬ï¼ˆå³ `v1.X.0`）。 + +Rancher 用于é…ç½® [RKE 集群](launch-kubernetes-with-rancher.md) çš„ä¿¡æ¯çŽ°åœ¨ä½äºŽ Rancher Kubernetes 元数æ®ä¸­ã€‚有关元数æ®é…置以åŠå¦‚何更改用于é…ç½® RKE 集群的 Kubernetes 版本，请å‚è§ [Rancher Kubernetes 元数æ®ã€‚](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md) + +Rancher 用于é…ç½® [RKE 集群](launch-kubernetes-with-rancher.md)çš„ Kubernetes 版本信æ¯åŒ…å«åœ¨ Rancher Kubernetes 元数æ®ä¸­ã€‚ + +有关元数æ®å¦‚何工作以åŠå¦‚何é…置元数æ®ï¼Œè¯·å‚è§ [Rancher Kubernetes 元数æ®](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md)。 + +## 全局设置 + +顶部导航æ ä¸­æ供了控制全局级别 Rancher 设置的选项。 + +点击左上角的 **☰**，然åŽé€‰æ‹©**全局设置**æ¥æŸ¥çœ‹å¹¶è¿›è¡Œé…置： + +- **设置**：å„ç§ Rancher 默认值，例如用户密ç çš„最å°é•¿åº¦ (`password-min-length`)。需è¦å°å¿ƒä¿®æ”¹è¿™äº›è®¾ç½®ï¼Œå› ä¸ºæ— æ•ˆçš„值å¯èƒ½ä¼šç ´å Rancher 安装。 +- **功能开关**：打开或关闭的 Rancher 功能。其中一些是[实验功能](#å¯ç”¨å®žéªŒåŠŸèƒ½)。 +- **横幅**：å¯ä»¥æ·»åŠ åˆ°é—¨æˆ·ä¸Šå›ºå®šä½ç½®çš„元素。例如，你å¯ä»¥ä½¿ç”¨è¿™äº›é€‰é¡¹åœ¨ç”¨æˆ·ç™»å½• Rancher æ—¶[设置自定义横幅](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md#固定横幅)。 +- **å“牌**：å¯ä»¥[自定义](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md)çš„ Rancher UI 设计元素。你å¯ä»¥æ·»åŠ è‡ªå®šä¹‰å¾½æ ‡æˆ–图标，并修改 UI 颜色。 +- **性能**：Rancher UI 的性能设置，例如增é‡èµ„æºåŠ è½½ã€‚ +- **主页链接**：Rancher UI **主页**上显示的链接。你å¯ä»¥ä¿®æ”¹é»˜è®¤é“¾æŽ¥çš„å¯è§æ€§æˆ–添加你自己的链接。 + +### å¯ç”¨å®žéªŒåŠŸèƒ½ + +Rancher 包å«ä¸€äº›å®žéªŒæ€§æˆ–默认ç¦ç”¨çš„功能。你å¯ä»¥ä½¿ç”¨åŠŸèƒ½å¼€å…³æ¥å¯ç”¨è¿™äº›åŠŸèƒ½ã€‚详情请å‚è§[功能开关](enable-experimental-features.md)的章节。 + +### 全局设置 + +除éžä½ æ¿€æ´»äº†**旧版**[功能开关](enable-experimental-features.md)，å¦åˆ™**全局é…ç½®**选项ä¸å¯è§ã€‚v2.6 åŠæ›´é«˜ç‰ˆæœ¬çš„ Rancher 默认ç¦ç”¨ **legacy** 标志。如果你从旧 Rancher 版本å‡çº§ï¼Œæˆ–者在 Rancher v2.6 åŠæ›´é«˜ç‰ˆæœ¬ä¸Šæ¿€æ´»äº† **legacy** 功能开关，则å¯ä»¥ä»Žé¡¶éƒ¨å¯¼èˆªèœå•è®¿é—®**全局设置**： + +1. 点击左上角的 **☰**。 +1. 从**旧版应用**中选择**全局设置**。 + +**全局设置**æ供了以下功能： + +- **应用商店** +- **全局 DNS æ¡ç›®** +- **全局 DNS æ供商** + +由于这些是旧版功能，因此请å‚阅有关[应用商店](/versioned_docs/version-2.0-2.4/pages-for-subheaders/helm-charts-in-rancher.md)ã€[全局 DNS æ¡ç›®](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/globaldns.md#adding-a-global-dns-entry)å’Œ[全局 DNS æ供商](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/globaldns.md#editing-a-global-dns-provider)çš„ Rancher v2.0-v2.4 文档了解更多详情。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/aws-cloud-marketplace.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/aws-cloud-marketplace.md new file mode 100644 index 00000000000..980a427cf4d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/aws-cloud-marketplace.md @@ -0,0 +1,32 @@ +--- +title: AWS Marketplace é›†æˆ +--- + +## 概述 + +Rancher æ供了与 AWS Marketplace 的集æˆï¼Œå…è®¸ç”¨æˆ·å‘ SUSE 购买支æŒã€‚此集æˆå¸®åŠ©ä½ åœ¨éœ€è¦ä½¿ç”¨æ›´å¤šé›†ç¾¤æ—¶è½»æ¾è°ƒæ•´æ”¯æŒéœ€æ±‚。 + +## é™åˆ¶ + +- 必须使用 Rancher v2.6.7 或更高版本。 +- Rancher 必须在å¯ç”¨å…¶ä»–指标的情况下进行部署。 +- Rancher 必须安装在 EKS 集群上。 +- 必须通过 AWS Marketplace 购买至少一项 Rancher 支æŒçš„ Entitlement。 +- ä½ å¯èƒ½éœ€è¦é¢å¤–的设置æ¥æ”¯æŒä»£ç†/离线用例。有关详细信æ¯ï¼Œè¯·å‚阅[先决æ¡ä»¶](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md)。 + +## 如何使用 + +1. 完æˆ[先决æ¡ä»¶æ­¥éª¤](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md)。 +2. [安装 CSP Adapter](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md)。 + +## 常è§é—®é¢˜ + +**我以åŽåŽç»­å†è´­ä¹°æ›´å¤šèŠ‚点的支æŒå—？** + +是的。你需è¦è½¬åˆ°æœ€åˆè´­ä¹°æ”¯æŒçš„ AWS Marketplace æ¡ç›®å¹¶å¢žåŠ  Entitlement çš„æ•°é‡ã€‚ + +**我å¯ä»¥åœ¨åŒä¸€ä¸ª AWS 账户中使用多个 Rancher 实例å—？** + +是的。但是，安装 Rancher çš„æ¯ä¸ªé›†ç¾¤éƒ½éœ€è¦éµå®ˆå…ˆå†³æ¡ä»¶ã€‚ + +此外，一个 Entitlement æ¯æ¬¡åªèƒ½ç”±ä¸€å° Rancher management server 使用。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/backup-restore-and-disaster-recovery.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/backup-restore-and-disaster-recovery.md new file mode 100644 index 00000000000..b7460369da2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/backup-restore-and-disaster-recovery.md @@ -0,0 +1,96 @@ +--- +title: 备份和ç¾éš¾æ¢å¤ +keywords: [rancher 备份还原, rancher 备份与æ¢å¤, 备份æ¢å¤ rancher, rancher 备份与æ¢å¤ rancher] +--- + +在本节中，你将学习如何创建 Rancher 的备份，如何从备份中æ¢å¤ Rancher，以åŠå¦‚何将 Rancher è¿ç§»åˆ°æ–°çš„ Kubernetes 集群。 + +`rancher-backup` operator å¯ä»¥ç”¨æ¥å¤‡ä»½å’Œæ¢å¤ä»»ä½• Kubernetes 集群上的 Rancher。这个应用是一个 Helm Chart，å¯ä»¥é€šè¿‡ Rancher çš„ **Apps** 页é¢æˆ–使用 Helm CLI 部署。你å¯ä»¥è®¿é—®[本页é¢](https://github.com/rancher/charts/tree/release-v2.6/charts/rancher-backup)èŽ·å– `rancher-backup` Helm Chart。 + +`backup-restore` operator 需è¦å®‰è£…在 local 集群上，并且åªå¯¹ Rancher 应用进行备份。备份和æ¢å¤æ“作仅在本地 Kubernetes 集群中执行。 + + +## 备份和æ¢å¤ Docker 安装的 Rancher + +对于使用 Docker 安装的 Rancher，请å‚è§[备份](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md)å’Œ[æ¢å¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md)对 Rancher 进行备份和æ¢å¤ã€‚ + +## 备份和æ¢å¤åŽŸç† + +`rancher-backup` operator 引入了三个自定义资æºï¼Œåˆ†åˆ«æ˜¯ Backupsã€Restores å’Œ ResourceSets。将以下集群范围的自定义资æºå®šä¹‰æ·»åŠ åˆ°é›†ç¾¤ä¸­ï¼š + +- `backups.resources.cattle.io` +- `resourcesets.resources.cattle.io` +- `restores.resources.cattle.io` + +ResourceSet 定义了需è¦å¤‡ä»½å“ªäº› Kubernetes 资æºã€‚由于备份 Rancher 所需的值是预设的，因此 ResourceSet 无法通过 Rancher UI 进行é…置。请ä¸è¦ä¿®æ”¹æ­¤ ResourceSet。 + +在创建 Backup 自定义资æºæ—¶ï¼Œ`rancher-backup` operator 调用 `kube-apiserver` æ¥èŽ·å– Backup 自定义资æºå¼•ç”¨çš„ ResourceSet（å³é¢„设的 `rancher-resource-set`）资æºã€‚ + +然åŽï¼Œoperator 以 `.tar.gz` æ ¼å¼åˆ›å»ºå¤‡ä»½æ–‡ä»¶ï¼Œå¹¶å°†å…¶å­˜å‚¨åœ¨ Backup 资æºä¸­é…置的ä½ç½®ã€‚ + +在创建 Restore 自定义资æºæ—¶ï¼Œoperator 访问 Restore 指定的 `tar.gz` 备份文件，并从该文件æ¢å¤åº”用。 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 或 `kubectl apply` æ¥åˆ›å»º Backup å’Œ Restore 自定义资æºã€‚ + +:::note + +请å‚è§[此处](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md#2-使用-restore-自定义资æºæ¥è¿˜åŽŸå¤‡ä»½)获å–在 Rancher 2.6.3 中将现有备份文件æ¢å¤åˆ° v1.22 集群的帮助。 + +::: + +## 安装 rancher-backup operator + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 或 Helm CLI æ¥å®‰è£… `rancher-backup` operator。两ç§å®‰è£…方法都将 `rancher-backup` Helm Chart 安装在è¿è¡Œ Rancher Server çš„ Kubernetes 集群上。它是集群管ç†å‘˜ç‹¬æœ‰çš„功能，仅适用于 **local** 集群。（*如果你在 Rancher UI 中没有看到 `rancher-backup`，你å¯èƒ½é€‰æ‹©äº†é”™è¯¯çš„集群。*） + +:::note + +使用 `backup-restore` operator 执行æ¢å¤åŽï¼ŒFleet 中会出现一个已知问题：用于 `clientSecretName` å’Œ `helmSecretName` 的密文ä¸åŒ…å«åœ¨ Fleet çš„ Git 仓库中。请å‚è§[此处](./fleet-gitops-at-scale.md#故障排除)获得解决方法。 + +::: + +### 使用 Rancher UI 安装 rancher-backup + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Charts**。 +1. 点击 **Rancher 备份**。 +1. å•å‡»**安装**。 +1. å¯é€‰ï¼šé…置默认存储ä½ç½®ã€‚如需获å–帮助，请å‚è§[é…ç½®](../reference-guides/backup-restore-configuration/storage-configuration.md)。 +1. å•å‡»**安装**。 + +**结果**：`rancher-backup` operator 已安装。 + +在**集群仪表æ¿**中，你å¯ä»¥çœ‹åˆ°åˆ—在 **Deployments** 下的 `rancher-backup` operator。 + +如果需è¦åœ¨ Rancher 中é…置备份应用，在左侧导航æ ä¸­å•å‡» **Rancher 备份**。 + +### RBAC + +åªæœ‰ Rancher 管ç†å‘˜å’Œæœ¬åœ°é›†ç¾¤çš„所有者å¯ä»¥ï¼š + +* 安装 Chart +* 看到 Backup å’Œ Restore CRD 的导航链接 +* 通过分别创建 Backup CR å’Œ Restore CR 执行备份和æ¢å¤ +* 列出目å‰å·²æ‰§è¡Œçš„备份和æ¢å¤æ“作 + +## 备份 Rancher + +备份是通过创建 Backup 自定义资æºå®žçŽ°çš„。如需查看教程，请å‚è§[本页é¢](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)。 + +## 还原 Rancher + +还原是通过创建 Restore 自定义资æºå®žçŽ°çš„。如需查看教程，请å‚è§[本页é¢](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md)。 + +## å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ + +ä½ å¯ä»¥æŒ‰ç…§[这些步骤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)执行è¿ç§»ã€‚ + +## 默认存储ä½ç½®é…ç½® + +é…置一个用于ä¿å­˜æ‰€æœ‰å¤‡ä»½çš„默认存储ä½ç½®ã€‚ä½ å¯ä»¥é€‰æ‹©å¯¹æ¯ä¸ªå¤‡ä»½è¿›è¡Œè¦†ç›–，但仅é™äºŽä½¿ç”¨ S3 或 Minio 对象存储。 + +如需了解å„个选项的é…置，请å‚è§[本页é¢](../reference-guides/backup-restore-configuration/storage-configuration.md)。 + +### rancher-backup Helm Chart 的示例 values.yaml + +当使用 Helm CLI 安装时，å¯ä»¥ä½¿ç”¨ç¤ºä¾‹ [values.yaml 文件](../reference-guides/backup-restore-configuration/storage-configuration.md#rancher-backup-helm-chart-的示例-valuesyaml) æ¥é…ç½® `rancher-backup` operator。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/backup-restore-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/backup-restore-configuration.md new file mode 100644 index 00000000000..b016cbe2f97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/backup-restore-configuration.md @@ -0,0 +1,8 @@ +--- +title: Rancher 备份é…ç½®å‚考 +--- + +- [备份é…ç½®](../reference-guides/backup-restore-configuration/backup-configuration.md) +- [还原é…ç½®](../reference-guides/backup-restore-configuration/restore-configuration.md) +- [存储ä½ç½®é…ç½®](../reference-guides/backup-restore-configuration/storage-configuration.md) +- [Backup å’Œ Restore 自定义资æºç¤ºä¾‹](../reference-guides/backup-restore-configuration/examples.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/best-practices.md new file mode 100644 index 00000000000..8c7c1fd9d07 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/best-practices.md @@ -0,0 +1,17 @@ +--- +title: 最佳实践 +--- + +æœ¬èŠ‚ä»‹ç» Rancher 实现的最佳实践，其中包括对 Kubernetesã€Dockerã€å®¹å™¨ç­‰æŠ€æœ¯çš„使用建议。最佳实践旨在利用 Rancher åŠå…¶å®¢æˆ·çš„è¿è¥ç»éªŒï¼ŒåŠ©ä½ æ›´å¥½åœ°å®žçŽ° Rancher。 + +如果你对用例的实际应用有任何疑问，请è”系客户æˆåŠŸç»ç†æˆ–支æŒä¸­å¿ƒã€‚ + +ä½ å¯ä»¥åœ¨å·¦ä¾§å¯¼èˆªæ å¿«é€Ÿæ‰¾åˆ°ç®¡ç†å’Œéƒ¨ç½² Rancher Server 的最佳实践。 + +如需查看更多最佳实践指å—，请å‚è§ï¼š + +- [安全类文档](rancher-security.md) +- [Rancher åšå®¢](https://www.suse.com/c/rancherblog/) +- [Rancher 论å›](https://forums.rancher.com/) +- [Rancher 用户的 Slack 群组](https://slack.rancher.io/) +- [B ç«™](https://space.bilibili.com/430496045/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/checklist-for-production-ready-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/checklist-for-production-ready-clusters.md new file mode 100644 index 00000000000..5c925f124ed --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/checklist-for-production-ready-clusters.md @@ -0,0 +1,48 @@ +--- +title: ç”Ÿäº§å°±ç»ªé›†ç¾¤æ£€æŸ¥æ¸…å• +--- + +本节将介ç»åˆ›å»ºç”Ÿäº§å°±ç»ªåž‹ Kubernetes 集群的最佳实践。这个集群å¯ç”¨äºŽè¿è¡Œä½ çš„应用和æœåŠ¡ã€‚ + +有关集群的è¦æ±‚（包括对 OS/Dockerã€ç¡¬ä»¶å’Œç½‘络的è¦æ±‚），请å‚阅[节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +本文介ç»äº†æˆ‘们推è用于所有生产集群的最佳实践的简短列表。 + +如需获å–推è的所有最佳实践的完整列表，请å‚阅[最佳实践](best-practices.md)。 + +### 节点è¦æ±‚ + +* ç¡®ä¿ä½ çš„节点满足所有[节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)，包括端å£è¦æ±‚。 + +### 备份 etcd + +* å¯ç”¨ etcd 快照。验è¯æ˜¯å¦æ­£åœ¨åˆ›å»ºå¿«ç…§ï¼Œå¹¶æ‰§è¡Œç¾éš¾æ¢å¤æ–¹æ¡ˆï¼Œä»Žè€ŒéªŒè¯å¿«ç…§æ˜¯å¦æœ‰æ•ˆã€‚etcd 是存储集群状æ€çš„ä½ç½®ï¼Œä¸¢å¤± etcd æ•°æ®æ„味ç€ä¸¢å¤±é›†ç¾¤ã€‚因此，请确ä¿ä¸ºé›†ç¾¤é…ç½® etcd 的定期快照，并确ä¿å¿«ç…§ä¹Ÿæ˜¯å­˜å‚¨åœ¨å¤–部（节点外）的。 + +### 集群架构 + +* 节点应具有以下角色é…置之一： + * `etcd` + * `controlplane` + * `etcd` å’Œ `controlplane` + * `worker`（ä¸åº”在具有 `etcd` 或 `controlplane` 角色的节点上使用或添加 `worker` 角色） +* 至少拥有三个角色为 `etcd` 的节点，æ¥ç¡®ä¿å¤±åŽ»ä¸€ä¸ªèŠ‚点时ä»èƒ½å­˜æ´»ã€‚增加 etcd 节点数é‡èƒ½æ高容错率，而将 etcd 分散到ä¸åŒå¯ç”¨åŒºç”šè‡³èƒ½èŽ·å–更好的容错能力。 +* ä¸ºä¸¤ä¸ªæˆ–æ›´å¤šèŠ‚ç‚¹åˆ†é… `controlplane` 角色，能实现主组件的高å¯ç”¨æ€§ã€‚ +* ä¸ºä¸¤ä¸ªæˆ–å¤šä¸ªèŠ‚ç‚¹åˆ†é… `worker` 角色，以便在节点故障时é‡æ–°å®‰æŽ’工作负载。 + +有关æ¯ä¸ªè§’色的用途的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 中的节点角色](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md)。 + +有关æ¯ä¸ª Kubernetes 角色的节点数的详细信æ¯ï¼Œè¯·å‚阅[推è架构](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)。 + +### Logging å’Œ Monitoring + +* 为 Kubernetes 组件（系统æœåŠ¡ï¼‰é…置告警/通知程åºã€‚ +* 为集群分æžå’Œäº‹åŽå‰–æžé…ç½® Logging。 + +### å¯é æ€§ + +* 在集群上执行负载测试，以验è¯ç¡¬ä»¶æ˜¯å¦å¯ä»¥æ”¯æŒä½ çš„工作负载。 + +### 网络 + +* 最å°åŒ–网络延迟。Rancher 建议尽é‡å‡å°‘ etcd 节点之间的延迟。`heartbeat-interval` 的默认设置是 `500`，`election-timeout` 的默认设置是 `5000`。这些 [etcd 调优设置](https://coreos.com/etcd/docs/latest/tuning.html) å…许 etcd 在大多数网络（网络延迟特别高的情况下除外）中è¿è¡Œã€‚ +* 集群节点应ä½äºŽå•ä¸ªåŒºåŸŸå†…。大多数云厂商在一个区域内æ供多个å¯ç”¨åŒºï¼Œè¿™å¯ä»¥æ高你集群的å¯ç”¨æ€§ã€‚任何角色的节点都å¯ä»¥ä½¿ç”¨å¤šä¸ªå¯ç”¨åŒºã€‚如果你使用 [Kubernetes Cloud Provider](set-up-cloud-providers.md) 资æºï¼Œè¯·æŸ¥é˜…文档以了解é™åˆ¶ï¼ˆå³åŒºåŸŸå­˜å‚¨é™åˆ¶ï¼‰ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cis-scan-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cis-scan-guides.md new file mode 100644 index 00000000000..269edaaec4a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cis-scan-guides.md @@ -0,0 +1,13 @@ +--- +title: CIS 扫ææŒ‡å— +--- + +- [安装 rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md) +- [å¸è½½ rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md) +- [è¿è¡Œæ‰«æ](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md) +- [定期è¿è¡Œæ‰«æ](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md) +- [跳过测试](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md) +- [查看报告](../how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md) +- [为 rancher-cis-benchmark å¯ç”¨å‘Šè­¦](../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) +- [为定时扫æé…置告警](../how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md) +- [创建è¦è¿è¡Œçš„自定义 Benchmark 版本](../how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cis-scans.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cis-scans.md new file mode 100644 index 00000000000..156d0ed946b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cis-scans.md @@ -0,0 +1,118 @@ +--- +title: CIS 扫æ +--- + +Rancher å¯ä»¥é€šè¿‡è¿è¡Œå®‰å…¨æ‰«ææ¥æ£€æŸ¥ Kubernetes 是å¦æŒ‰ç…§ CIS Kubernetes Benchmark 中定义的安全最佳实践进行部署。CIS 扫æå¯ä»¥è¿è¡Œåœ¨ä»»ä½• Kubernetes 集群，包括托管的 Kubernetes，例如 EKSã€AKS å’Œ GKE。 + +`rancher-cis-benchmark` 应用使用了 kube-bench ，这是 Aqua Security çš„å¼€æºå·¥å…·ï¼Œç”¨äºŽæ£€æŸ¥é›†ç¾¤æ˜¯å¦ç¬¦åˆ CIS Kubernetes Benchmark。此外，为了生æˆé›†ç¾¤çº§åˆ«çš„报告，此应用使用了 Sonobuoy æ¥èšåˆæŠ¥å‘Šã€‚ + + +## 关于 CIS Benchmark + +CIS（Center for Internet Security）是一个 501(c\)(3) éžè¥åˆ©ç»„织，æˆç«‹äºŽ 2000 å¹´ 10 月，其使命是识别ã€å¼€å‘ã€éªŒè¯ã€ä¿ƒè¿›å’Œç»´æŒç½‘络防御的最佳实践方案，并建立和指导社区，以在网络空间中è¥é€ ä¿¡ä»»çš„环境。该组织总部ä½äºŽçº½çº¦ä¸œæ ¼æž—布什，其æˆå‘˜åŒ…括大公å¸ã€æ”¿åºœæœºæž„和学术机构。 + +CIS Benchmark 是目标系统安全é…置的最佳实践。CIS Benchmark 是由安全专家ã€æŠ€æœ¯ä¾›åº”商ã€å…¬å¼€å’Œç§äººç¤¾åŒºæˆå‘˜ï¼Œä»¥åŠ CIS Benchmark å¼€å‘团队共åŒå¿—æ„¿å¼€å‘的。 + +在 CIS 网站上[注册](https://learn.cisecurity.org/benchmarks)以查看官方 Benchmark 文档。 + +## 关于生æˆçš„报告 + +æ¯æ¬¡æ‰«æ都会生æˆä¸€ä»½æŠ¥å‘Šï¼Œä½ å¯ä»¥åœ¨ Rancher UI 中查看该报告，并以 CSV æ ¼å¼ä¸‹è½½å®ƒã€‚ + +默认情况下使用 CIS Benchmark v1.6。 + +Benchmark 版本包å«åœ¨ç”Ÿæˆçš„报告中。 + +Benchmark æ供两ç§ç±»åž‹çš„建议，分别是自动（Automated）和手动（Manual）。Benchmark 中标记为 Manual 的建议ä¸åŒ…å«åœ¨ç”Ÿæˆçš„报告中。 + +一些测试会被标记为“ä¸é€‚用â€ã€‚由于 Rancher é…ç½® RKE 集群的方å¼ï¼Œè¿™äº›æµ‹è¯•ä¸ä¼šåœ¨ä»»ä½• CIS 扫æ中è¿è¡Œã€‚有关如何审核测试结果，以åŠä¸ºä»€ä¹ˆæŸäº›æµ‹è¯•ä¼šè¢«æ ‡è®°ä¸ºä¸é€‚用，请å‚阅 Rancher çš„ Kubernetes 对应版本的[自测指å—](./rancher-security.md#cis-benchmark-和自我评估)。 + +该报告包å«ä»¥ä¸‹ä¿¡æ¯ï¼š + +| 报告中的列 | æè¿° | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | CIS Benchmark çš„ ID å·ã€‚ | +| `description` | CIS Benchmark 测试的æ述。 | +| `remediation` | 为了通过测试需è¦ä¿®å¤çš„内容。 | +| `state` | 测试的状æ€ï¼Œå¯ä»¥æ˜¯é€šè¿‡ã€å¤±è´¥ã€è·³è¿‡æˆ–ä¸é€‚用。 | +| `node_type` | 节点角色，角色决定了在节点上è¿è¡Œçš„测试。主测试在 controlplane 节点上è¿è¡Œï¼Œetcd 测试在 etcd 节点上è¿è¡Œï¼ŒèŠ‚点测试在 Worker 节点上è¿è¡Œã€‚ | +| `audit` | 这是 `kube-bench` 为此测试è¿è¡Œçš„审计检查。 | +| `audit_config` | 适用于审计脚本的任何é…置。 | +| `test_info` | `kube-bench` 报告的测试相关信æ¯ï¼ˆå¦‚果存在）。 | +| `commands` | `kube-bench` 报告的测试相关的命令（如果存在）。 | +| `config_commands` | `kube-bench` 报告的测试相关的é…置数æ®ï¼ˆå¦‚果存在）。 | +| `actual_value` | 测试的实际值。如果由 `kube-bench` 报告，则会显示。 | +| `expected_result` | 测试的预期值。如果由 `kube-bench` 报告，则会显示。 | + +请å‚阅[集群加固指å—中的表格](./rancher-security.md)，以了解 Kubernetesã€Benchmarkã€Rancher 以åŠæˆ‘们的集群强化指å—的版本对应关系。å¦å¤–，请å‚阅强化指å—，以获å–ç¬¦åˆ CIS 的集群的é…置文件以åŠä¿®å¤å¤±è´¥æµ‹è¯•çš„ä¿¡æ¯ã€‚ + +## 测试é…置文件 + +以下是å¯ç”¨çš„é…置文件： + +- Generic CIS 1.6 +- Generic CIS 1.20 +- Generic CIS 1.23 +- RKE permissive 1.6 +- RKE hardened 1.6 +- RKE permissive 1.20 +- RKE hardened 1.20 +- RKE permissive 1.23 +- RKE hardened 1.23 +- RKE2 permissive 1.6 +- RKE2 hardened 1.6 +- RKE2 permissive 1.20 +- RKE2 hardened 1.20 +- RKE2 permissive 1.23 +- RKE2 hardened 1.23 +- K3s permissive 1.6 +- K3s hardened 1.6 +- K3s permissive 1.20 +- K3s hardened 1.20 +- K3s permissive 1.23 +- K3s hardened 1.23 +- AKS +- EKS +- GKE + +你还å¯ä»¥é€šè¿‡ä¿å­˜ä¸€ç»„è¦è·³è¿‡çš„测试æ¥è‡ªå®šä¹‰é…置文件。 + +所有é…置文件都会有一组ä¸é€‚用的测试，CIS 扫æ会跳过这些测试。RKE é›†ç¾¤ç®¡ç† Kubernetes çš„æ–¹å¼å¯¼è‡´è¿™äº›æµ‹è¯•è¢«è®¤ä¸ºä¸é€‚用。 + +RKE 集群扫æé…置文件有两ç§ç±»åž‹ï¼š + +- **Permissive**：此é…置文件有一组è¦è·³è¿‡çš„测试，跳过的原因是这些测试会在默认的 RKE Kubernetes 集群上失败。除了跳过的测试列表之外，é…置文件也ä¸ä¼šè¿è¡Œä¸é€‚用的测试。 +- **Hardened**：此é…置文件ä¸ä¼šè·³è¿‡ä»»ä½•æµ‹è¯•ï¼ˆä¸é€‚用的测试除外）。 + +EKS å’Œ GKE 集群扫æçš„é…置文件基于这些集群类型特定的 CIS Benchmark 版本。 + +è¦é€šè¿‡ “Hardened†é…置文件，你需è¦éµä»Ž[强化指å—](./rancher-security.md#rancher-强化指å—)并使用强化指å—中定义的 `cluster.yml` æ¥é…置一个强化集群。 + +默认é…置文件和支æŒçš„ CIS Benchmark 版本å–决于扫æ的集群类型： + +`rancher-cis-benchmark` æ”¯æŒ CIS 1.6 Benchmark 版本。 + +- RKE Kubernetes 集群默认使用 RKE Permissive 1.6 é…置文件。 +- EKS å’Œ GKE 有自己的 CIS Benchmark，由 `kube-bench` å‘布。这些集群默认使用相应的测试é…置文件。 +- RKE2 Kubernetes 集群默认使用 RKE2 Permissive 1.6 é…置文件。 +- RKEã€RKE2ã€EKS å’Œ GKE 以外的集群类型默认使用 Generic CIS 1.5 é…置文件。 + +## 跳过和ä¸é€‚用的测试 + +有关è¦è·³è¿‡å’Œä¸é€‚用的测试列表，请å‚阅[此页é¢](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md)。 + +ç›®å‰ï¼Œåªæœ‰ç”¨æˆ·å®šä¹‰çš„跳过测试会在生æˆæŠ¥å‘Šä¸­æ ‡è®°ä¸ºè·³è¿‡ã€‚ + +如果æŸä¸ªé»˜è®¤é…置文件将æŸä¸ªæµ‹è¯•å®šä¹‰ä¸ºè·³è¿‡ï¼Œåˆ™è¯¥æµ‹è¯•ä¹Ÿä¼šæ ‡è®°ä¸ºä¸é€‚用。 + +## RBAC + +有关æƒé™çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/cis-scans/rbac-for-cis-scans.md)。 + +## é…ç½® + +有关为扫æã€é…置文件和 Benchmark 版本é…置自定义资æºçš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/cis-scans/configuration-reference.md)。 + +## æ“ä½œæŒ‡å— + +è¦äº†è§£å¦‚何è¿è¡Œ CIS 扫æ，请å‚阅 [CIS 扫æ指å—](../pages-for-subheaders/cis-scan-guides.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cli-with-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cli-with-rancher.md new file mode 100644 index 00000000000..f1e68727a8d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cli-with-rancher.md @@ -0,0 +1,5 @@ +--- +title: Rancher CLI +--- + +Rancher CLI 是一个命令行工具，用于在工作站中与 Rancher 进行交互。以下文档将æè¿° [Rancher CLI](../reference-guides/cli-with-rancher/rancher-cli.md) å’Œ [kubectl Utility](../reference-guides/cli-with-rancher/kubectl-utility.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cloud-marketplace.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cloud-marketplace.md new file mode 100644 index 00000000000..1ef64aa10a8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cloud-marketplace.md @@ -0,0 +1,7 @@ +--- +title: äº‘å¸‚åœºé›†æˆ +--- + +Rancher æ供与云市场的集æˆï¼Œè®©ä½ èƒ½è½»æ¾è´­ä¹°äº‘æ供商上的安装支æŒã€‚此外，该集æˆè¿˜æ”¯æŒç”Ÿæˆ supportconfig bundle，你å¯ä»¥å°†è¯¥ bundle æ供给 Rancher。 + +此集æˆä»…æ”¯æŒ AWS。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cluster-configuration.md new file mode 100644 index 00000000000..68908371deb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/cluster-configuration.md @@ -0,0 +1,29 @@ +--- +title: 集群é…ç½® +--- + +使用 Rancher é…ç½® Kubernetes 集群åŽï¼Œä½ ä»ç„¶å¯ä»¥ç¼–辑集群的选项和设置。 + +有关编辑集群æˆå‘˜èµ„格的信æ¯ï¼Œè¯·è½¬è‡³[此页é¢](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md)。 + +### 集群é…ç½®å‚考 + +集群é…置选项å–决于 Kubernetes 集群的类型： + +- [RKE 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [GKE 集群é…ç½®](gke-cluster-configuration.md) +- [AKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) + +### ä¸åŒç±»åž‹é›†ç¾¤çš„管ç†åŠŸèƒ½ + +对于已有集群而言，å¯æ供的选项和设置å–决于你é…置集群的方法。 + +下表总结了æ¯ä¸€ç§ç±»åž‹çš„集群和对应的å¯ç¼–辑的选项和设置： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configuration-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configuration-options.md new file mode 100644 index 00000000000..f98a4990ceb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configuration-options.md @@ -0,0 +1,39 @@ +--- +title: é…置选项 +--- + +### Egress æ”¯æŒ + +默认情况下，Egress 网关是ç¦ç”¨çš„，但你å¯ä»¥åœ¨å®‰è£…或å‡çº§æ—¶ä½¿ç”¨ values.yaml 或[覆盖文件](#覆盖文件)å¯ç”¨å®ƒã€‚ + +### å¯ç”¨è‡ªåŠ¨ Sidecar 注入 + +默认情况下，自动 sidecar 注入是ç¦ç”¨çš„。è¦å¯ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·åœ¨å®‰è£…或å‡çº§æ—¶åœ¨ values.yaml 中设置 `sidecarInjectorWebhook.enableNamespacesByDefault=true`。这会自动将 Istio sidecar 注入到所有已部署的新命å空间。 + +### 覆盖文件 + +覆盖文件用于为 Istio 进行更广泛的é…置。它å…许你更改 [IstioOperator API](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/) 中å¯ç”¨çš„任何值。你å¯ä»¥è‡ªå®šä¹‰é»˜è®¤å®‰è£…以满足你的需求。 + +覆盖文件将在 Istio Chart 默认安装的基础上添加é…置。æ¢è¨€ä¹‹ï¼Œä½ ä¸éœ€è¦ä¸ºå®‰è£…中已定义的组件进行é‡æ–°å®šä¹‰ã€‚ + +有关覆盖文件的更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](https://istio.io/latest/docs/setup/install/istioctl/#configure-component-settings)。 + +### 选择器和抓å–é…ç½® + +Monitoring 应用设置了 `prometheus.prometheusSpec.ignoreNamespaceSelectors=false`，å³åœ¨é»˜è®¤æƒ…况下跨所有命å空间进行监控。这样，你å¯ä»¥æŸ¥çœ‹éƒ¨ç½²åœ¨å…·æœ‰ `istio-injection=enabled` 标签的命å空间中的资æºçš„æµé‡ã€æŒ‡æ ‡å’Œå›¾ã€‚ + +如果你想将 Prometheus é™åˆ¶ä¸ºç‰¹å®šçš„命å空间，请设置 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。完æˆæ­¤æ“作åŽï¼Œä½ éœ€è¦æ·»åŠ å…¶ä»–é…ç½®æ¥ç»§ç»­ç›‘控你的资æºã€‚ + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)。 + +### 在具有 Pod 安全策略的情况下å¯ç”¨ Istio + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/pod-security-policies.md)。 + +### 在 RKE2 集群上安装 Istio 的其他步骤 + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md)。 + +### 项目网络隔离的其他步骤 + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/project-network-isolation.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md new file mode 100644 index 00000000000..9cb1a64ca66 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md @@ -0,0 +1,38 @@ +--- +title: é…ç½® Microsoft AD FS (SAML) +--- + +如果你的组织使用 Microsoft è”åˆèº«ä»½éªŒè¯æœåŠ¡ (AD FS) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 AD FS 凭è¯ç™»å½•ã€‚ + +## 先决æ¡ä»¶ + +已安装 Rancher。 + +- 获å–ä½ çš„ Rancher Server URL。é…ç½® AD FS 时，请使用该 URL æ›¿æ¢ `` å ä½ç¬¦ã€‚ +- ä½ çš„ Rancher 必须具有全局管ç†å‘˜è´¦å·ã€‚ + +ä½ å¿…é¡»é…ç½® [Microsoft AD FS æœåŠ¡å™¨](https://docs.microsoft.com/en-us/windows-server/identity/active-directory-federation-services)。 + +- 获å–ä½ çš„ AD FS æœåŠ¡å™¨ IP/DNS å称。é…ç½® AD FS 时，请使用该 IP/DNS åç§°æ›¿æ¢ `` å ä½ç¬¦ã€‚ +- 你必须有在 AD FS æœåŠ¡å™¨ä¸Šæ·»åŠ  [Relying Party Trusts](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) çš„æƒé™ã€‚ + +## é…ç½®æ¦‚è¦ + +è¦è®© Rancher Server 使用 Microsoft AD FS，你需è¦åœ¨ Active Directory æœåŠ¡å™¨ä¸Šé…ç½® AD FS，并将 Rancher é…置为使用 AD FS æœåŠ¡å™¨ã€‚如果需è¦èŽ·å–在 Rancher 中设置 Microsoft AD FS 身份验è¯çš„指å—，请å‚è§ï¼š + +- [1. 在 Microsoft AD FS 中é…ç½® Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) +- [2. 在 Rancher 中é…ç½® Microsoft AD FS](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md) + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + + +### åŽç»­æ“作 + +[在 Microsoft AD FS 中é…ç½® Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-openldap.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-openldap.md new file mode 100644 index 00000000000..5b8f3d014d9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-openldap.md @@ -0,0 +1,52 @@ +--- +title: é…ç½® OpenLDAP +--- + +如果你的组织使用 LDAP 进行用户身份验è¯ï¼Œåˆ™å¯ä»¥é…ç½® Rancher 与 OpenLDAP æœåŠ¡å™¨é€šä¿¡ï¼Œä»Žè€Œå¯¹ç”¨æˆ·è¿›è¡Œèº«ä»½éªŒè¯ã€‚这使 Rancher 管ç†å‘˜å¯ä»¥å¯¹å¤–部用户系统中的用户和组进行集群和项目的访问控制，åŒæ—¶å…许最终用户在登录 Rancher UI 时使用 LDAP 凭è¯è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +## 先决æ¡ä»¶ + +必须为 Rancher é…ç½® LDAP 绑定账å·ï¼ˆå³ ServiceAccount），æ¥æœç´¢å’Œæ£€ç´¢åº”该具有访问æƒé™çš„用户和组的 LDAP æ¡ç›®ã€‚建议ä¸è¦ä½¿ç”¨ç®¡ç†å‘˜è´¦å·æˆ–个人账å·ï¼Œè€Œåº”在 OpenLDAP 中创建一个专用账å·ï¼Œè¯¥è´¦å·å¯¹é…置的æœç´¢åº“下的用户和组需è¦å…·æœ‰åªè¯»æƒé™ï¼ˆå‚è§ä¸‹æ–‡ï¼‰ã€‚ + +> **使用 TLS？** +> +> 如果 OpenLDAP æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +## 在 Rancher 中é…ç½® OpenLDAP + +é…ç½® OpenLDAP æœåŠ¡å™¨ï¼Œç»„和用户的设置。有关填写æ¯ä¸ªå­—段的帮助，请å‚è§[é…ç½®å‚考](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md)。 + +> 在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **OpenLDAP**。填写**é…ç½® OpenLDAP æœåŠ¡å™¨**表å•ã€‚ +1. 点击**å¯ç”¨**。 + +### æµ‹è¯•èº«ä»½éªŒè¯ + +完æˆé…ç½®åŽï¼Œè¯·æµ‹è¯•ä¸Ž OpenLDAP æœåŠ¡å™¨çš„连接。如果测试æˆåŠŸï¼Œåˆ™è¡¨æ˜Ž OpenLDAP 身份验è¯å·²å¯ç”¨ã€‚ + +:::note + +与此步骤中输入的凭è¯ç›¸å…³çš„ OpenLDAP 用户将映射到本地主体账å·ï¼Œå¹¶åœ¨ Rancher 中分é…系统管ç†å‘˜æƒé™ã€‚因此，你应该决定使用哪个 OpenLDAP è´¦å·æ¥æ‰§è¡Œæ­¤æ­¥éª¤ã€‚ + +::: + +1. 输入应映射到本地主体账å·çš„ OpenLDAP è´¦å·çš„**用户å**å’Œ**密ç ** 。 +2. 点击**å¯ç”¨ OpenLDAP 认è¯**æ¥æµ‹è¯• OpenLDAP 的连接并完æˆè®¾ç½®ã€‚ + +**结果**： + +- OpenLDAP 验è¯é…ç½®æˆåŠŸã€‚ +- 与输入凭è¯å¯¹åº”çš„ LDAP 用户被映射到本地主体（管ç†å‘˜ï¼‰è´¦å·ã€‚ + +:::note + +如果 LDAP æœåŠ¡ä¸­æ–­ï¼Œä½ ä»ç„¶å¯ä»¥ä½¿ç”¨æœ¬åœ°é…置的 `admin` è´¦å·å’Œå¯†ç ç™»å½•ã€‚ + +::: + +## 附录：故障排除 + +如果在测试与 OpenLDAP æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·é¦–先仔细检查为 ServiceAccount 输入的凭è¯ä»¥åŠæœç´¢åº“é…置。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-shibboleth-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-shibboleth-saml.md new file mode 100644 index 00000000000..49a68af19f4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/configure-shibboleth-saml.md @@ -0,0 +1,102 @@ +--- +title: é…ç½® Shibboleth (SAML) +--- + +如果你的组织使用 Shibboleth Identity Provider (IdP)) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 Shibboleth 凭è¯ç™»å½•ã€‚ + +在此é…置中，当 Rancher 用户登录时，他们将被é‡å®šå‘到 Shibboleth IdP æ¥è¾“入凭è¯ã€‚身份验è¯ç»“æŸåŽï¼Œä»–们将被é‡å®šå‘回 Rancher UI。 + +如果你将 OpenLDAP é…置为 Shibboleth çš„åŽç«¯ï¼ŒSAML 断言会返回到 Rancher，其中包括用于引用组的用户属性。然åŽï¼Œé€šè¿‡èº«ä»½éªŒè¯çš„用户将能够访问其所在的组有æƒè®¿é—®çš„ Rancher 资æºã€‚ + +> 本节å‡å®šä½ å·²äº†è§£ Rancher，Shibboleth å’Œ OpenLDAP 是如何ååŒå·¥ä½œçš„。有关工作原ç†çš„详细说明，请å‚è§[本页](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md)。 + + +## 在 Rancher 中设置 Shibboleth + +### Shibboleth 先决æ¡ä»¶ +> +> - ä½ å¿…é¡»é…置了 Shibboleth IdP æœåŠ¡å™¨ã€‚ +> - 以下是 Rancher Service Provider é…置所需的 URL： +> å…ƒæ•°æ® URL：`https:///v1-saml/shibboleth/saml/metadata` +> 断言使用者æœåŠ¡ (ACS) URL：`https:///v1-saml/shibboleth/saml/acs` +> - 从 IdP æœåŠ¡å™¨å¯¼å‡º `metadata.xml` 文件。详情请å‚è§ [Shibboleth 文档](https://wiki.shibboleth.net/confluence/display/SP3/Home)。 + +### 在 Rancher 中é…ç½® Shibboleth + +如果你的组织使用 Shibboleth 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许你的用户使用 IdP 凭è¯ç™»å½•ã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Shibboleth**。 +1. 填写**é…ç½® Shibboleth è´¦å·**表å•ã€‚Shibboleth IdP å…许你指定è¦ä½¿ç”¨çš„æ•°æ®å­˜å‚¨ã€‚ä½ å¯ä»¥æ·»åŠ æ•°æ®åº“或使用现有的 ldap æœåŠ¡å™¨ã€‚例如，如果你选择 Active Directory (AD) æœåŠ¡å™¨ï¼Œä¸‹é¢çš„示例将æ述如何将 AD 属性映射到 Rancher 中的字段： + + 1. **显示å称字段**：包å«ç”¨æˆ·æ˜¾ç¤ºå称的 AD 属性（例如：`displayName`）。 + + 1. **用户å字段**：包å«ç”¨æˆ·å/给定å称的 AD 属性（例如：`givenName`）。 + + 1. **UID 字段**：æ¯ä¸ªç”¨æˆ·å”¯ä¸€çš„ AD 属性（例如：`sAMAccountName`ã€`distinguishedName`）。 + + 1. **用户组字段**: 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ï¼ˆä¾‹å¦‚：`memberOf`）。 + + 1. **Rancher API 主机**：你的 Rancher Server çš„ URL。 + + 1. **ç§é’¥**å’Œ**è¯ä¹¦**：密钥/è¯ä¹¦å¯¹ï¼Œç”¨äºŽåœ¨ Rancher 和你的 IdP 之间创建一个安全外壳（SSH）。 + + ä½ å¯ä»¥ä½¿ç”¨ openssl 命令进行创建。例如： + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP 元数æ®**：从 IdP æœåŠ¡å™¨å¯¼å‡ºçš„ `metadata.xml` 文件。 + + +1. 完æˆ**é…ç½® Shibboleth è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Shibboleth IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Shibboleth é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 Shibboleth。你的用户现在å¯ä»¥ä½¿ç”¨ Shibboleth 登录å登录 Rancher。 + +### SAML æ供商注æ„事项 + +SAML åè®®ä¸æ”¯æŒç”¨æˆ·æˆ–用户组的æœç´¢æˆ–查找。因此，如果你没有为 Shibboleth é…ç½® OpenLDAP，则请留æ„以下警告。 + +- 在 Rancher 中为用户或组分é…æƒé™æ—¶ï¼Œä¸ä¼šå¯¹ç”¨æˆ·æˆ–组进行验è¯ã€‚ +- 添加用户时，必须正确输入准确的用户 IDï¼ˆå³ UID 字段）。在你输入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +è¦åœ¨ Rancher 中分é…æƒé™æ—¶å¯ç”¨æœç´¢ç»„，你需è¦ä¸ºæ”¯æŒç»„çš„ SAML 身份验è¯æ供商é…ç½®åŽç«¯ï¼ˆä¾‹å¦‚ OpenLDAP）。 + +## 在 Rancher 中设置 OpenLDAP + +如果你将 OpenLDAP é…置为 Shibboleth çš„åŽç«¯ï¼ŒSAML 断言会返回到 Rancher，其中包括用于引用组的用户属性。然åŽï¼Œé€šè¿‡èº«ä»½éªŒè¯çš„用户将能够访问其所在的组有æƒè®¿é—®çš„ Rancher 资æºã€‚ + +### OpenLDAP 先决æ¡ä»¶ + +必须为 Rancher é…ç½® LDAP 绑定账å·ï¼ˆå³ ServiceAccount），æ¥æœç´¢å’Œæ£€ç´¢åº”该具有访问æƒé™çš„用户和组的 LDAP æ¡ç›®ã€‚建议ä¸è¦ä½¿ç”¨ç®¡ç†å‘˜è´¦å·æˆ–个人账å·ï¼Œè€Œåº”在 OpenLDAP 中创建一个专用账å·ï¼Œè¯¥è´¦å·å¯¹é…置的æœç´¢åº“下的用户和组需è¦å…·æœ‰åªè¯»æƒé™ï¼ˆå‚è§ä¸‹æ–‡ï¼‰ã€‚ + +> **使用 TLS？** +> +> 如果 OpenLDAP æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +### 在 Rancher 中é…ç½® OpenLDAP + +é…ç½® OpenLDAP æœåŠ¡å™¨ï¼Œç»„和用户的设置。有关填写æ¯ä¸ªå­—段的帮助，请å‚è§[é…ç½®å‚考](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md)。请注æ„，嵌套组æˆå‘˜èµ„æ ¼ä¸é€‚用于 Shibboleth。 + +> 在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + +1. 使用åˆå§‹çš„本地 `admin` è´¦å·ç™»å½•åˆ° Rancher UI。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Shibboleth**，如果已é…ç½® SAML，则å•å‡»**编辑é…ç½®**。 +1. 在**用户和组æœç´¢**下，选中**é…ç½® OpenLDAP Server**。 + +## 故障排除 + +如果在测试与 OpenLDAP æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·é¦–先仔细检查为 ServiceAccount 输入的凭è¯ä»¥åŠæœç´¢åº“é…置。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/create-kubernetes-persistent-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/create-kubernetes-persistent-storage.md new file mode 100644 index 00000000000..e59d36c843d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/create-kubernetes-persistent-storage.md @@ -0,0 +1,74 @@ +--- +title: "Kubernetes æŒä¹…存储：å·å’Œå­˜å‚¨ç±»" +description: "了解在 Kubernetes 中创建æŒä¹…存储的两ç§æ–¹å¼ï¼šæŒä¹…å·å’Œå­˜å‚¨ç±»" +--- + +在部署需è¦ä¿â€‹â€‹ç•™æ•°æ®çš„应用时，你需è¦åˆ›å»ºæŒä¹…存储。æŒä¹…存储å…许你在è¿è¡Œåº”用的 pod 之外存储应用数æ®ã€‚å³ä½¿è¿è¡Œåº”用的 pod å‘生故障，这ç§å­˜å‚¨æ–¹å¼ä¹Ÿèƒ½è®©ä½ ä¿ç•™åº”用数æ®ã€‚ + +本文å‡è®¾ä½ å·²äº†è§£ Kubernetes çš„æŒä¹…å·ã€æŒä¹…å·å£°æ˜Žå’Œå­˜å‚¨ç±»çš„概念。如需更多信æ¯ï¼Œè¯·å‚阅[存储的工作原ç†](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md)。 + +### 先决æ¡ä»¶ + +设置æŒä¹…存储需è¦`管ç†å·`çš„[角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色å‚考)。 + +如果你è¦ä¸ºäº‘集群é…置存储，则存储和集群主机必须使用相åŒçš„云æ供商。 + +è¦ä½¿ç”¨ Rancher é…置新存储，则必须å¯ç”¨äº‘æ供商。有关å¯ç”¨äº‘æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../pages-for-subheaders/set-up-cloud-providers.md)。 + +如果è¦å°†çŽ°æœ‰çš„æŒä¹…存储连接到集群，则ä¸éœ€è¦å¯ç”¨äº‘æ供商。 + +### 设置现有存储 + +设置现有存储的总体æµç¨‹å¦‚下： + +1. 设置你的æŒä¹…存储。å¯ä»¥æ˜¯äº‘存储或你自己的存储。 +2. 添加引用æŒä¹…存储的æŒä¹…å· (PV)。 +3. 添加引用 PV çš„æŒä¹…å·å£°æ˜Ž (PVC)。 +4. å°† PVC 挂载为工作负载中的å·ã€‚ + +有关详细信æ¯å’Œå…ˆå†³æ¡ä»¶ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md)。 + +### 在 Rancher 中动æ€é…置新存储 + +é…置新存储的总体æµç¨‹å¦‚下： + +1. 添加一个 StorageClass 并将它é…置为使用你的存储æ供商。StorageClass å¯ä»¥å¼•ç”¨äº‘存储或你自己的存储。 +2. 添加引用存储类的æŒä¹…å·å£°æ˜Ž (PVC)。 +3. å°† PVC 挂载为工作负载的å·ã€‚ + +有关详细信æ¯å’Œå…ˆå†³æ¡ä»¶ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md)。 + +### Longhorn 存储 + +[Longhorn](https://longhorn.io/) 是一个轻é‡çº§ã€å¯é ã€æ˜“用的 Kubernetes 分布å¼å—存储系统。 + +Longhorn 是å…费的开æºè½¯ä»¶ã€‚Longhorn 最åˆç”± Rancher Labs å¼€å‘，现在正在作为云原生计算基金会的沙盒项目进行开å‘。它å¯ä»¥é€šè¿‡ Helmã€kubectl 或 Rancher UI 安装在任何 Kubernetes 集群上。 + +如果你有å—存储池，Longhorn å¯ä»¥å¸®åŠ©ä½ ä¸º Kubernetes 集群æä¾›æŒä¹…存储，而无需ä¾èµ–云æ供商。有关 Longhorn 功能的更多信æ¯ï¼Œè¯·å‚阅[文档](https://longhorn.io/docs/latest/what-is-longhorn/)。 + +Rancher v2.5 简化了在 Rancher 管ç†çš„集群上安装 Longhorn 的过程。详情请å‚è§[本页é¢](../integrations-in-rancher/longhorn.md)。 + +### é…置存储示例 + +我们æ供了如何使用 [NFSã€](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) [vSphere](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) å’Œ [Amazon EBS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) æ¥é…置存储的示例。 + +### GlusterFS å· + +在将数æ®å­˜å‚¨åœ¨ GlusterFS å·ä¸Šçš„集群中，你å¯èƒ½ä¼šé‡åˆ°é‡å¯ `kubelet` åŽ pod 无法挂载å·çš„问题。有关é¿å…此情况å‘生的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md)。 + +### iSCSI å· + +在将数æ®å­˜å‚¨åœ¨ iSCSI å·ä¸Šçš„ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../pages-for-subheaders/launch-kubernetes-with-rancher.md)中，你å¯èƒ½ä¼šé‡åˆ° kubelet 无法自动连接 iSCSI å·çš„问题。有关解决此问题的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md)。 + +### hostPath å· +在创建 hostPath å·ä¹‹å‰ï¼Œä½ éœ€è¦åœ¨é›†ç¾¤é…置中设置 [extra_bind](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds/)。这会将路径作为å·å®‰è£…在你的 kubelet 中，å¯ç”¨äºŽå·¥ä½œè´Ÿè½½ä¸­çš„ hostPath å·ã€‚ + +### å°† vSphere Cloud Provider 从树内è¿ç§»åˆ°æ ‘外 + +Kubernetes 正在é€æ¸ä¸åœ¨æ ‘内维护云æ供商。vSphere 有一个树外云æ供商，å¯é€šè¿‡å®‰è£… vSphere 云æ供商和云存储æ’件æ¥ä½¿ç”¨ã€‚ + +有关如何从树内 vSphere 云æ供商è¿ç§»åˆ°æ ‘外，以åŠå¦‚何在è¿ç§»åŽç®¡ç†çŽ°æœ‰è™šæ‹Ÿæœºï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 + +### 相关链接 + +- [Kubernetes 文档：存储](https://kubernetes.io/docs/concepts/storage/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/custom-resource-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/custom-resource-configuration.md new file mode 100644 index 00000000000..602ae0bfb63 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/custom-resource-configuration.md @@ -0,0 +1,8 @@ +--- +title: 自定义资æºé…ç½® +--- + +以下自定义资æºå®šä¹‰ï¼ˆCustom Resource Definition，CRD）用于é…ç½® Logging： + +- [Flow å’Œ ClusterFlow](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md) +- [Output å’Œ ClusterOutput](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-apps-across-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-apps-across-clusters.md new file mode 100644 index 00000000000..5a6382f71b1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-apps-across-clusters.md @@ -0,0 +1,15 @@ +--- +title: 跨集群部署应用 +--- + +### Fleet + +Rancher 2.5 引入了 Fleet，这是一ç§è·¨é›†ç¾¤éƒ¨ç½²åº”用的新方å¼ã€‚ + +使用 Fleet çš„æŒç»­äº¤ä»˜æ˜¯å¤§è§„模的 GitOps。如需更多信æ¯ï¼Œè¯·å‚阅 [Fleet](../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md)。 + +### 多集群应用 + +在 2.5 之å‰çš„ Rancher 版本中，多集群应用功能用于跨集群部署应用。我们已弃用多集群应用功能，但你ä»ç„¶å¯ä»¥åœ¨ Rancher 2.5 中使用该功能。 + +详情请å‚阅[此文档](../how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-rancher-manager.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-rancher-manager.md new file mode 100644 index 00000000000..c20dbce3552 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-rancher-manager.md @@ -0,0 +1,19 @@ +--- +title: 部署 Rancher Server +--- + +ä½ å¯ä½¿ç”¨ä»¥ä¸‹æŒ‡å—之一，在你选择的æ供商中部署和é…ç½® Rancher å’Œ Kubernetes 集群。 + +- [AWS](../getting-started/quick-start-guides/deploy-rancher-manager/aws.md)（使用 Terraform） +- [AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md)（使用 Amazon EKS） +- [Azure](../getting-started/quick-start-guides/deploy-rancher-manager/azure.md)（使用 Terraform） +- [DigitalOcean](../getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md)（使用 Terraform） +- [GCP](../getting-started/quick-start-guides/deploy-rancher-manager/gcp.md)（使用 Terraform） +- [Hetzner Cloud](../getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md)（使用 Terraform） +- [Vagrant](../getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md) +- [Equinix Metal](../getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md) +- [Outscale](../getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md)（使用 Terraform） + +如有需è¦ï¼Œä½ å¯ä»¥æŸ¥çœ‹ä»¥ä¸‹æŒ‡å—以了解分步步骤。如果你需è¦åœ¨å…¶ä»–æ供商中或本地è¿è¡Œ Rancher，或者你åªæ˜¯æƒ³çœ‹çœ‹å®ƒæ˜¯å¤šä¹ˆå®¹æ˜“上手，你å¯é˜…读以下指å—： + +- [手动安装](../getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-rancher-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-rancher-workloads.md new file mode 100644 index 00000000000..01d72560dde --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/deploy-rancher-workloads.md @@ -0,0 +1,8 @@ +--- +title: 部署工作负载 +--- + +这些指å—指导你完æˆä¸€ä¸ªåº”用的部署，包括如何将应用暴露在集群之外使用。 + +- [部署带有 Ingress 的工作负载](../getting-started/quick-start-guides/deploy-workloads/workload-ingress.md) +- [部署带有 NodePort 的工作负载](../getting-started/quick-start-guides/deploy-workloads/nodeports.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/downstream-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/downstream-cluster-configuration.md new file mode 100644 index 00000000000..3d09efe4a8e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/downstream-cluster-configuration.md @@ -0,0 +1,5 @@ +--- +title: 下游集群é…ç½® +--- + +以下文档将讨论[节点模æ¿é…ç½®](./node-template-configuration.md)å’Œ[主机é…ç½®](./machine-configuration.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/enable-experimental-features.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/enable-experimental-features.md new file mode 100644 index 00000000000..4f759f616fe --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/enable-experimental-features.md @@ -0,0 +1,125 @@ +--- +title: å¯ç”¨å®žéªŒåŠŸèƒ½ +--- + +Rancher 包å«ä¸€äº›é»˜è®¤å…³é—­çš„实验功能。在æŸäº›æƒ…况下，例如当你认为使用[ä¸æ”¯æŒçš„存储类型](../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md)的好处大于使用未ç»æµ‹è¯•çš„功能的风险时，你å¯èƒ½æƒ³è¦å¯ç”¨å®žéªŒåŠŸèƒ½ã€‚为了让你能够试用这些默认关闭的功能，我们引入了功能开关（feature flag）。 + +实验功能å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸‰ç§æ–¹å¼å¯ç”¨ï¼š + +- [使用 CLI](#å¯åŠ¨-rancher-æ—¶å¯ç”¨åŠŸèƒ½)：在使用 CLI 安装 Rancher 时，使用功能开关默认å¯ç”¨æŸä¸ªåŠŸèƒ½ã€‚ +- [使用 Rancher UI](#使用-rancher-ui-å¯ç”¨åŠŸèƒ½)：在**设置**页é¢å¯ç”¨åŠŸèƒ½ã€‚ +- [使用 Rancher API](#使用-rancher-api-å¯ç”¨åŠŸèƒ½)：安装 Rancher åŽå¯ç”¨åŠŸèƒ½ã€‚ + +æ¯ä¸ªåŠŸèƒ½å‡æœ‰ä»¥ä¸‹ä¸¤ä¸ªå€¼ï¼š + +- 默认值：å¯ä»¥é€šè¿‡åœ¨å‘½ä»¤è¡Œä½¿ç”¨æ ‡å¿—或环境å˜é‡è¿›è¡Œé…置。 +- 设置值：å¯ä»¥é€šè¿‡ Rancher API 或 UI 进行é…置。 + +如果没有设置值，Rancher 会使用默认值。 + +设置值是通过 API 设置的，而默认值是通过命令行设置。因此，如果你使用 API 或 UI å¯ç”¨æˆ–ç¦ç”¨æŸä¸ªåŠŸèƒ½ï¼Œå‘½ä»¤è¡Œä¸­è®¾ç½®çš„值将被覆盖。 + +如果你安装 Rancher åŽä½¿ç”¨ Rancher API 将功能开关设置为 true，然åŽåœ¨ä½¿ç”¨å‘½ä»¤å‡çº§ Rancher 时将功能开关设置为 false，在这ç§æƒ…况下，虽然默认值会是 false，但是该功能ä¾ç„¶ä¼šè¢«å¯ç”¨ï¼Œå› ä¸ºå®ƒæ˜¯é€šè¿‡ API 设置的。如果你éšåŽä½¿ç”¨ Rancher API 删除设置值（true）并将它设置为 NULL，则默认值（false）将生效。有关详细信æ¯ï¼Œè¯·å‚阅[功能开关页é¢](../getting-started/installation-and-upgrade/installation-references/feature-flags.md)。 + +## å¯åŠ¨ Rancher æ—¶å¯ç”¨åŠŸèƒ½ + +安装 Rancher 时，使用功能开关å¯ç”¨ä½ æ‰€éœ€çš„功能。通过å•èŠ‚点容器安装 Rancher，和在 Kubernetes 集群上安装 Rancher 对应的命令有所ä¸åŒã€‚ + +### Kubernetes 安装的情况下å¯ç”¨åŠŸèƒ½ + +:::note + +通过 Rancher API 设置的值会覆盖命令行传入的值。 + +::: + +使用 Helm Chart 安装 Rancher 时，使用 `--set` 选项。下é¢çš„示例通过传递功能开关å称（用逗å·åˆ†éš”）æ¥å¯ç”¨ä¸¤ä¸ªåŠŸèƒ½ï¼š + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +:::note + +如果你安装的是 alpha 版本，Helm è¦æ±‚你在命令中添加 `--devel` 选项。 + +::: + +### 离线安装的情况下渲染 Helm Chart + +如果你是在离线环境安装 Rancher 的，在使用 Helm 安装 Rancher 之å‰ï¼Œä½ éœ€è¦æ·»åŠ ä¸€ä¸ª Helm Chart 仓库并渲染一个 Helm 模æ¿ã€‚详情请å‚è§[离线安装文档](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md)。 + +以下是在渲染 Helm 模æ¿æ—¶ä¼ å…¥åŠŸèƒ½å¼€å…³å称的命令示例。下é¢çš„示例通过传递功能开关å称（用逗å·åˆ†éš”）æ¥å¯ç”¨ä¸¤ä¸ªåŠŸèƒ½ã€‚ + +Helm 命令如下： + +``` +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # é¿å…ç”Ÿæˆ Helm é’©å­æ–‡ä»¶ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +### Docker 安装的情况下å¯ç”¨åŠŸèƒ½ + +如果 Rancher 是使用 Docker 安装的，请使用 `--features` 选项。下é¢çš„示例通过传递功能开关å称（用逗å·åˆ†éš”）æ¥å¯ç”¨ä¸¤ä¸ªåŠŸèƒ½ï¼š + +``` +docker run -d -p 80:80 -p 443:443 \ + --restart=unless-stopped \ + rancher/rancher:rancher-latest \ + --features==true,=true +``` + + +## 使用 Rancher UI å¯ç”¨åŠŸèƒ½ + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 如需å¯ç”¨æŸä¸ªåŠŸèƒ½ï¼Œæ‰¾åˆ°è¯¥å·²ç¦ç”¨çš„功能，并点击**â‹® > 激活**。 + +**结果**：该功能已å¯ç”¨ã€‚ + +### 使用 Rancher UI ç¦ç”¨åŠŸèƒ½ + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å•å‡»**功能开关**。你将看到实验功能列表。 +1. 如需ç¦ç”¨æŸä¸ªåŠŸèƒ½ï¼Œæ‰¾åˆ°è¯¥å·²å¯ç”¨çš„功能，并点击**â‹® > åœç”¨**。 + +**结果**：该功能已ç¦ç”¨ã€‚ + +## 使用 Rancher API å¯ç”¨åŠŸèƒ½ + +1. å‰å¾€ `/v3/features`。 +1. 在 `data` 中，你会看到一个数组，该数组包å«æ‰€æœ‰èƒ½é€šè¿‡åŠŸèƒ½å¼€å…³å¯ç”¨çš„功能。功能的å称在 `id` 字段中。å•å‡»è¦å¯ç”¨çš„功能的å称。 +1. 在左上角的 **Operations** 下，点击 **Edit**。 +1. 在 **Value** 下拉èœå•ä¸­ï¼Œå•å‡» **True**。 +1. å•å‡» **Show Request**。 +1. å•å‡» **Send Request**。 +1. 点击 **Close**。 + +**结果**：该功能已å¯ç”¨ã€‚ + +### 使用 Rancher API ç¦ç”¨åŠŸèƒ½ + +1. å‰å¾€ `/v3/features`。 +1. 在 `data` 中，你会看到一个数组，该数组包å«æ‰€æœ‰èƒ½é€šè¿‡åŠŸèƒ½å¼€å…³å¯ç”¨çš„功能。功能的å称在 `id` 字段中。å•å‡»è¦å¯ç”¨çš„功能的å称。 +1. 在左上角的 **Operations** 下，点击 **Edit**。 +1. 在 **Value** 下拉èœå•ä¸­ï¼Œå•å‡» **False**。 +1. å•å‡» **Show Request**。 +1. å•å‡» **Send Request**。 +1. 点击 **Close**。 + +**结果**：该功能已ç¦ç”¨ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/fleet-gitops-at-scale.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/fleet-gitops-at-scale.md new file mode 100644 index 00000000000..c27fe57a2cd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/fleet-gitops-at-scale.md @@ -0,0 +1,64 @@ +--- +title: 使用 Feet 进行æŒç»­äº¤ä»˜ +--- + +使用 Fleet çš„æŒç»­äº¤ä»˜æ˜¯å¤§è§„模的 GitOps。你å¯ä»¥ä½¿ç”¨ Fleet 管ç†å¤šè¾¾ä¸€ç™¾ä¸‡ä¸ªé›†ç¾¤ã€‚此外，它éžå¸¸è½»é‡ï¼Œå› æ­¤ä¹Ÿéžå¸¸é€‚用于[å•ä¸ªé›†ç¾¤](https://fleet.rancher.io/installation#default-install)。但是，它在[大规模](https://fleet.rancher.io/installation#configuration-for-multi-cluster)场景下的功能更加强大。大规模指的是大é‡é›†ç¾¤ã€å¤§é‡éƒ¨ç½²æˆ–大é‡å›¢é˜Ÿã€‚ + +Fleet 是一个独立于 Rancher 的项目，你å¯ä»¥ä½¿ç”¨ Helm 将它安装在任何 Kubernetes 集群上。 + + +## 架构 + +有关 Fleet 工作原ç†çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此处](../integrations-in-rancher/fleet-gitops-at-scale/architecture.md)。 + +## 在 Rancher UI 中访问 Fleet + +Fleet 预装在 Rancher 中，å¯ä»¥é€šè¿‡ Rancher UI 中的**æŒç»­äº¤ä»˜**选项进行管ç†ã€‚有关æŒç»­äº¤ä»˜å’Œ Fleet 故障排除技巧的更多信æ¯ï¼Œè¯·å‚阅[此处](https://fleet.rancher.io/troubleshooting)。 + +用户å¯ä»¥é€šè¿‡éµå¾ª **gitops** 的实践，利用æŒç»­äº¤ä»˜å°†åº”用部署到 git 仓库中的 Kubernetes 集群，而无需任何手动æ“作。 + +按照以下步骤在 Rancher UI 中访问æŒç»­äº¤ä»˜ï¼š + +1. å•å‡» **☰ > æŒç»­äº¤ä»˜**。 + +1. 在èœå•é¡¶éƒ¨é€‰æ‹©ä½ çš„命å空间，注æ„以下几点： + + - 默认情况下会选中 **fleet-default**，其中包括注册到 Rancher 的所有下游集群。 + + - ä½ å¯ä»¥åˆ‡æ¢åˆ°ä»…åŒ…å« **local** 集群的 **fleet-local**，或者创建自己的工作空间，并将集群分é…和移动到该工作空间。 + + - 然åŽï¼Œä½ å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„**集群**æ¥ç®¡ç†é›†ç¾¤ã€‚ + +1. å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„ **Git 仓库**å°† git 仓库部署到当å‰å·¥ä½œç©ºé—´ä¸­çš„集群中。 + +1. 选择你的 [git 仓库](https://fleet.rancher.io/gitrepo-add)å’Œ[目标集群/集群组](https://fleet.rancher.io/gitrepo-targets)。你还å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸­çš„**集群组**在 UI 中创建集群组。 + +1. 部署 git 仓库åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 监控应用。 + +## Windows æ”¯æŒ + +有关对具有 Windows 节点的集群的支æŒï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md)。 + +## GitHub 仓库 + +ä½ å¯ä»¥å•å‡»æ­¤å¤„èŽ·å– [Fleet Helm Chart](https://github.com/rancher/fleet/releases)。 + +## 在代ç†åŽä½¿ç”¨ Fleet + +有关在代ç†åŽä½¿ç”¨ Fleet 的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md)。 + +## Helm Chart ä¾èµ– + +由于用户需è¦å®Œæˆä¾èµ–列表，因此为了æˆåŠŸéƒ¨ç½²å…·æœ‰ä¾èµ–项的 Helm Chart，你必须手动è¿è¡Œå‘½ä»¤ï¼ˆå¦‚下所列）。如果你ä¸è¿™æ ·åšï¼Œå¹¶ç»§ç»­å…‹éš†ä»“库并è¿è¡Œ `helm install`，由于ä¾èµ–项将丢失，因此你的安装将失败。 + +git 仓库中的 Helm Chart 必须在 Chart å­ç›®å½•ä¸­åŒ…å«å…¶ä¾èµ–项。你必须手动è¿è¡Œ `helm dependencies update $chart`，或在本地è¿è¡Œ `helm dependencies build $chart`，然åŽå°†å®Œæ•´çš„ Chart 目录æ交到你的 git 仓库。请注æ„，你需è¦ä½¿ç”¨é€‚当的å‚æ•°æ¥ä¿®æ”¹å‘½ä»¤ã€‚ + +## 故障排除 + +- **已知问题**：Fleet git 仓库的 clientSecretName å’Œ helmSecretName 密文ä¸åŒ…å«åœ¨ç”± [backup-restore-operator](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md#1-安装-rancher-backup-operator) 创建的备份或æ¢å¤ä¸­ã€‚如果我们有了永久的解决方案，我们将通知社区。 + +- **临时解决方法**：默认情况下，用户定义的密文ä¸ä¼šåœ¨ Fleet 中备份。如果执行ç¾éš¾æ¢å¤æˆ–å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œåˆ™éœ€è¦é‡æ–°åˆ›å»ºå¯†æ–‡ã€‚è¦ä¿®æ”¹ resourceSet 以包å«éœ€è¦å¤‡ä»½çš„其他资æºï¼Œè¯·å‚阅[此文档](https://github.com/rancher/backup-restore-operator#user-flow)。 + +## 文档 + +Fleet 文档链接：https://fleet.rancher.io/ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/gke-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/gke-cluster-configuration.md new file mode 100644 index 00000000000..7ae7e6292c2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/gke-cluster-configuration.md @@ -0,0 +1,321 @@ +--- +title: GKE 集群é…ç½®å‚考 +--- + +## Rancher 2.6 å˜æ›´ + +- 支æŒé¢å¤–çš„é…置选项： + - 项目网络隔离 + - 网络标签 + +## 集群ä½ç½® + +| 值 | æè¿° | +|--------|--------------| +| ä½ç½®ç±»åž‹ | 地区 (zone) 或区域 (region)。借助 GKE，你å¯ä»¥æ ¹æ®å·¥ä½œè´Ÿè½½çš„å¯ç”¨æ€§è¦æ±‚和预算创建一个é‡èº«å®šåˆ¶çš„集群。默认情况下，集群的节点在å•ä¸ªè®¡ç®—区域中è¿è¡Œã€‚选择多个区域时，集群的节点将跨越多个计算区域，而 controlplane 则åªä½äºŽå•ä¸ªåŒºåŸŸä¸­ã€‚区域集群也增加了 controlplane çš„å¯ç”¨æ€§ã€‚有关选择集群å¯ç”¨æ€§ç±»åž‹çš„帮助，请å‚阅[这些文档](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#choosing_a_regional_or_zonal_control_plane)。 | +| 地区 | 计算引擎中的æ¯ä¸ªåŒºåŸŸéƒ½åŒ…å«å¤šåœ°åŒºã€‚有关å¯ç”¨åŒºåŸŸå’Œå¯ç”¨åŒºçš„更多信æ¯ï¼Œè¯·å‚阅[这些文档](https://cloud.google.com/compute/docs/regions-zones#available)。 | +| 其他地区 | 对于地区性集群，你å¯ä»¥é€‰æ‹©å…¶ä»–地区æ¥åˆ›å»º[多地区集群](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#multi-zonal_clusters)。 | +| 区域 | 对于[区域性集群](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#regional_clusters)，你å¯ä»¥é€‰æ‹©ä¸€ä¸ªåŒºåŸŸã€‚有关å¯ç”¨åŒºåŸŸå’Œå¯ç”¨åŒºçš„更多信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/compute/docs/regions-zones#available)。地区å称的å‰é¢éƒ¨åˆ†æ˜¯åŒºåŸŸçš„å称。 | + +## 集群选项 + +### Kubernetes 版本 + +_å¯å˜ï¼šæ˜¯_ + +有关 GKE Kubernetes 版本的更多信æ¯ï¼Œè¯·å‚阅[这些文档](https://cloud.google.com/kubernetes-engine/versioning)。 + +### 容器地å€èŒƒå›´ + +_å¯å˜ï¼šå¦_ + +集群中 Pod çš„ IP 地å€èŒƒå›´ã€‚必须是有效的 CIDR 范围，例如 10.42.0.0/16。如果未指定，则会自动从 10.0.0.0/8 中选择一个éšæœºèŒƒå›´ï¼Œå¹¶æŽ’除已分é…ç»™ VMã€å…¶ä»–集群或路由的范围。自动选择的范围å¯èƒ½ä¸Žé¢„留的 IP 地å€ã€åŠ¨æ€è·¯ç”±æˆ–与集群对等的 VPC 中的路由å‘生冲çªã€‚ + +### 网络 + +_å¯å˜ï¼šå¦_ + +集群连接的 Compute Engine 网络。将使用此网络创建路由和防ç«å¢™ã€‚如果使用[共享 VPC](https://cloud.google.com/vpc/docs/shared-vpc)，与你的项目共享的 VPC 网络将显示在此处。你将å¯ä»¥åœ¨æ­¤å­—段中进行选择。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets)。 + +### 节点å­ç½‘/å­ç½‘ + +_å¯å˜ï¼šå¦_ + +集群连接到的 Compute Engine å­ç½‘。该å­ç½‘必须属于**网络**字段中指定的网络。选择一个现有的å­ç½‘，或选择“自动创建å­ç½‘â€æ¥è‡ªåŠ¨åˆ›å»ºä¸€ä¸ªå­ç½‘。如果ä¸ä½¿ç”¨çŽ°æœ‰ç½‘络，则需è¦ä½¿ç”¨**å­ç½‘å称**æ¥ç”Ÿæˆä¸€ä¸ªã€‚如果使用[共享 VPC](https://cloud.google.com/vpc/docs/shared-vpc)，与你的项目共享的 VPC å­ç½‘将显示在此处。如果使用共享 VPC 网络，则无法选择“自动创建å­ç½‘â€ã€‚如需更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets)。 + +### å­ç½‘å称 + +_å¯å˜ï¼šå¦_ + +使用æ供的å称自动创建å­ç½‘。如果为**节点å­ç½‘**或**å­ç½‘**选择了“自动创建å­ç½‘â€ï¼Œåˆ™ä¸ºå¿…填。有关å­ç½‘的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets)。 + +### IP 别å + +_å¯å˜ï¼šå¦_ + +å¯ç”¨[别å IP](https://cloud.google.com/vpc/docs/alias-ip)。这将å¯ç”¨ VPC 原生æµé‡è·¯ç”±ã€‚如果使用[共享 VPC](https://cloud.google.com/vpc/docs/shared-vpc)，则为必填。 + +### 网络策略 + +_å¯å˜ï¼šæ˜¯_ + +在集群上å¯ç”¨çš„网络策略。网络策略定义了集群中 pod å’Œ service 之间å¯ä»¥å‘生的通信级别。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/how-to/network-policy)。 + +### 项目网络隔离 + +_å¯å˜ï¼šæ˜¯_ + +选择å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´é€šä¿¡ã€‚请注æ„，如果å¯ç”¨**项目网络隔离**，则将自动å¯ç”¨**网络策略**å’Œ**网络策略é…ç½®**，å之则ä¸ç„¶ã€‚ + +### 节点 IPv4 CIDR å— + +_å¯å˜ï¼šå¦_ + +此集群中实例 IP çš„ IP 地å€èŒƒå›´ã€‚如果为**节点å­ç½‘**或**å­ç½‘**选择了“自动创建å­ç½‘â€ï¼Œåˆ™å¯ä»¥è¿›è¡Œè®¾ç½®ã€‚必须是有效的 CIDR 范围，例如 10.96.0.0/14。有关如何确定 IP 地å€èŒƒå›´çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing)。 + +### 集群次è¦èŒƒå›´å称 + +_å¯å˜ï¼šå¦_ + +Pod IP 地å€çš„现有次è¦èŒƒå›´çš„å称。如果选中，将自动填充**集群 Pod 地å€èŒƒå›´**。如果使用共享 VPC 网络，则为必填。 + +### 集群 Pod 地å€èŒƒå›´ + +_å¯å˜ï¼šå¦_ + +分é…给集群中 pod çš„ IP 地å€èŒƒå›´ã€‚必须是有效的 CIDR 范围，例如 10.96.0.0/11。如果未æ供，将自动创建。如果使用共享 VPC 网络，则必须æ供。有关如何确定 pod çš„ IP 地å€èŒƒå›´çš„更多信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_pods)。 + +### Service 次è¦èŒƒå›´å称 + +_å¯å˜ï¼šå¦_ + +Service IP 地å€çš„现有次è¦èŒƒå›´çš„å称。如果选中，将自动填充 **Service 地å€èŒƒå›´**。如果使用共享 VPC 网络，则为必填。 + +### Service 地å€èŒƒå›´ + +_å¯å˜ï¼šå¦_ + +分é…给集群中 Service 的地å€èŒƒå›´ã€‚必须是有效的 CIDR 范围，例如 10.94.0.0/18。如果未æ供，将自动创建。如果使用共享 VPC 网络，则必须æ供。有关如何确定 Service çš„ IP 地å€èŒƒå›´çš„详细信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_svcs)。 + +### ç§æœ‰é›†ç¾¤ + +_å¯å˜ï¼šå¦_ + +:::caution + +ç§æœ‰é›†ç¾¤éœ€è¦åœ¨ Rancher 之外进行é¢å¤–的规划和é…置。请å‚阅[ç§æœ‰é›†ç¾¤æŒ‡å—](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md)。 + +::: + +仅分é…节点内部 IP 地å€ã€‚除éžåœ¨ GCP 中执行了é¢å¤–çš„è”网步骤，å¦åˆ™ç§æœ‰é›†ç¾¤èŠ‚点无法访问公共互è”网。 + +### å¯ç”¨ç§æœ‰ç«¯ç‚¹ + +:::caution + +ç§æœ‰é›†ç¾¤éœ€è¦åœ¨ Rancher 之外进行é¢å¤–的规划和é…置。请å‚阅[ç§æœ‰é›†ç¾¤æŒ‡å—](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md)。 + +::: + +_å¯å˜ï¼šå¦_ + +é”定对 controlplane 端点的外部访问。仅当**ç§æœ‰é›†ç¾¤**也被选中时å¯ç”¨ã€‚如果选中，并且 Rancher 无法直接访问集群所在的虚拟ç§æœ‰äº‘网络，Rancher å°†æ供在集群上è¿è¡Œçš„注册命令，以使 Rancher 能够连接到集群。 + +### 主 IPV4 CIDR å— + +_å¯å˜ï¼šå¦_ + +controlplane VPC çš„ IP 范围。 + +### 主授æƒç½‘络 + +_å¯å˜ï¼šæ˜¯_ + +å¯ç”¨ controlplane 授æƒç½‘络，以阻止ä¸å—ä¿¡ä»»çš„éž GCP æº IP 通过 HTTPS 访问 Kubernetes master。如果选择，则å¯ä»¥æ·»åŠ é¢å¤–的授æƒç½‘络。如果集群是使用公共端点创建的，则此选项å¯ç”¨äºŽå°†å…¬å…±ç«¯ç‚¹çš„访问é”定到特定网络（例如è¿è¡Œ Rancher æœåŠ¡çš„网络）。如果集群åªæœ‰ä¸€ä¸ªç§æœ‰ç«¯ç‚¹ï¼Œåˆ™éœ€è¦æ­¤è®¾ç½®ã€‚ + +## 其他选项 + +### 集群æ’件 + +其他 Kubernetes 集群组件。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.locations.clusters#Cluster.AddonsConfig)。 + +#### æ°´å¹³ Pod 自动缩放 + +_å¯å˜ï¼šæ˜¯_ + +Horizo​​ntal Pod Autoscaler 通过自动增加或å‡å°‘ Pod çš„æ•°é‡æ¥è°ƒæ•´ Kubernetes 工作负载，从而å“应工作负载的 CPU æˆ–å†…å­˜æ¶ˆè€—ï¼Œä»¥åŠ Kubernetes 内部报告的自定义指标或集群外部设置的指标。详情请å‚è§[本页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/horizontalpodautoscaler)。 + +#### HTTP (L7) è´Ÿè½½å‡è¡¡ + +_å¯å˜ï¼šæ˜¯_ + +HTTP (L7) è´Ÿè½½å‡è¡¡å°† HTTP å’Œ HTTPS æµé‡åˆ†é…到托管在 GKE 上的åŽç«¯ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/tutorials/http-balancer)。 + +#### 网络策略é…ç½®ï¼ˆä»…é™ master） + +_å¯å˜ï¼šæ˜¯_ + +NetworkPolicy çš„é…置。仅跟踪 master 节点上是å¦å¯ç”¨äº†æ’件，ä¸è·Ÿè¸ªæ˜¯å¦ä¸ºèŠ‚点å¯ç”¨äº†ç½‘络策略。 + +### 集群特å¾ï¼ˆAlpha 功能） + +_å¯å˜ï¼šå¦_ + +打开集群的所有 Kubernetes alpha API 组和功能。å¯ç”¨åŽï¼Œé›†ç¾¤æ— æ³•å‡çº§ï¼Œå¹¶ä¸”会在 30 天åŽè‡ªåŠ¨åˆ é™¤ã€‚由于 GKE SLA æœªæ”¯æŒ alpha 集群，因此ä¸å»ºè®®å°† Alpha 集群用于生产环境。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/alpha-clusters)。 + +### Logging æœåŠ¡ + +_å¯å˜ï¼šæ˜¯_ + +集群用于写入日志的日志管ç†æœåŠ¡ã€‚è¦ä¹ˆä½¿ç”¨ [Cloud Logging](https://cloud.google.com/logging)，è¦ä¹ˆä¸ä½¿ç”¨æ—¥å¿—管ç†æœåŠ¡ï¼ˆä¸ä¼šä»Žé›†ç¾¤ä¸­å¯¼å‡ºæ—¥å¿—）。 + +### 监控æœåŠ¡ + +_å¯å˜ï¼šæ˜¯_ + +集群用于写入指标的监控æœåŠ¡ã€‚è¦ä¹ˆä½¿ç”¨ [Cloud Monitoring](https://cloud.google.com/monitoring)，è¦ä¹ˆä¸ä½¿ç”¨é›†ç¾¤ç›‘控æœåŠ¡ï¼ˆä¸ä¼šä»Žé›†ç¾¤ä¸­å¯¼å‡ºæŒ‡æ ‡ï¼‰ã€‚ + + +### ç»´æŠ¤çª—å£ + +_å¯å˜ï¼šæ˜¯_ + +设置时长 4 å°æ—¶çš„维护窗å£çš„开始时间。使用 HH:MM æ ¼å¼åœ¨ UTC 时区中指定时间。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/maintenance-windows-and-exclusions)。 + +## 节点池 + +在此部分中，输入æ述节点池中æ¯ä¸ªèŠ‚点的é…置的详细信æ¯ã€‚ + +### Kubernetes 版本 + +_å¯å˜ï¼šæ˜¯_ + +节点池中æ¯ä¸ªèŠ‚点的 Kubernetes 版本。有关 GKE Kubernetes 版本的更多信æ¯ï¼Œè¯·å‚阅[这些文档](https://cloud.google.com/kubernetes-engine/versioning)。 + +### é•œåƒç±»åž‹ + +_å¯å˜ï¼šæ˜¯_ + +节点æ“作系统镜åƒã€‚有关 GKE 为æ¯ä¸ªæ“作系统æ供的节点镜åƒé€‰é¡¹ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/node-images#available_node_images)。 + +:::note + +默认选项是 “Container-Optimized OS with Dockerâ€ã€‚GCP Container-Optimized OS 上的åªè¯»æ–‡ä»¶ç³»ç»Ÿä¸Ž Rancher 中的 [legacy logging](/versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-logging.md) 实现ä¸å…¼å®¹ã€‚如果你需è¦ä½¿ç”¨æ—§ç‰ˆæ—¥å¿—管ç†åŠŸèƒ½ï¼Œè¯·é€‰æ‹© “Ubuntu with Docker†或 “Ubuntu with Containerdâ€ã€‚[current logging feature](logging.md) 与 Container-Optimized OS é•œåƒå…¼å®¹ã€‚ + +::: + +:::note + +如果节点池镜åƒç±»åž‹é€‰æ‹© “Windows Long Term Service Channel†或 “Windows Semi-Annual Channelâ€ï¼Œè¿˜å¿…须至少添加一个 Container-Optimized OS 或 Ubuntu 节点池。 + +::: + +### 主机类型 + +_å¯å˜ï¼šå¦_ + +节点实例å¯ç”¨çš„虚拟化硬件资æºã€‚有关 Google Cloud 主机类型的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/compute/docs/machine-types#machine_types)。 + +### æ ¹ç£ç›˜ç±»åž‹ + +_å¯å˜ï¼šå¦_ + +标准永久性ç£ç›˜ç”±æ ‡å‡†ç£ç›˜é©±åŠ¨å™¨ (HDD) 支æŒï¼Œè€Œ SSD 永久性ç£ç›˜ç”±å›ºæ€ç¡¬ç›˜ (SSD) 支æŒã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/compute/docs/disks)。 + +### 本地 SSD ç£ç›˜ + +_å¯å˜ï¼šå¦_ + +é…ç½®æ¯ä¸ªèŠ‚点的本地 SSD ç£ç›˜å­˜å‚¨ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚本地 SSD 物ç†è¿žæŽ¥åˆ°æ‰˜ç®¡ä½ çš„ VM 实例的æœåŠ¡å™¨ã€‚与标准永久性ç£ç›˜æˆ– SSD 永久性ç£ç›˜ç›¸æ¯”，本地 SSD 具有更高的åžåé‡å’Œæ›´ä½Žçš„延迟。存储在本地 SSD 上的数æ®åªä¼šä¿ç•™åˆ°å®žä¾‹åœæ­¢æˆ–删除。有关详细信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/compute/docs/disks#localssds)。 + +### 抢å å¼èŠ‚点（beta） + +_å¯å˜ï¼šå¦_ + +抢å å¼èŠ‚点也称为抢å å¼è™šæ‹Ÿæœºã€‚通常是最长æŒç»­ 24 å°æ—¶çš„ Compute Engine 虚拟机实例，ä¸æä¾›å¯ç”¨æ€§ä¿è¯ã€‚详情请å‚è§[本页é¢](https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms)。 + +### 污点 + +_å¯å˜ï¼šå¦_ + +将污点应用于节点时，仅å…许容å¿è¯¥æ±¡ç‚¹çš„ Pod 在该节点上è¿è¡Œã€‚在 GKE 集群中，你å¯ä»¥å°†æ±¡ç‚¹åº”用到节点池，这会将污点应用到池中的所有节点。 + +### 节点标签 + +_å¯å˜ï¼šå¦_ + +ä½ å¯ä»¥å°†æ ‡ç­¾åº”用到节点池，这会将标签应用到池中的所有节点。 + +无效标签会阻止å‡çº§ï¼Œæˆ–阻止 Rancher å¯åŠ¨ã€‚有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 + +### 网络标签 + +_å¯å˜ï¼šå¦_ + +ä½ å¯ä»¥å°†ç½‘络标签添加到节点池以制定防ç«å¢™è§„则和å­ç½‘之间的路由。标签将应用于池中的所有节点。 + +有关标签语法和è¦æ±‚的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://cloud.google.com/vpc/docs/add-remove-network-tags)。 + +## ç»„è¯¦ç»†ä¿¡æ¯ + +在此部分中，输入æ述节点池的详细信æ¯ã€‚ + +### å称 + +_å¯å˜ï¼šå¦_ + +输入节点池的å称。 + +### åˆå§‹èŠ‚点数 + +_å¯å˜ï¼šæ˜¯_ + +节点池中åˆå§‹èŠ‚点数的整数。 + +### æ¯ä¸ªèŠ‚点的最大 Pod æ•°é‡ + +_å¯å˜ï¼šå¦_ + +GKE 的硬性é™åˆ¶æ˜¯æ¯ä¸ªèŠ‚点 110 个 Pod。有关 Kubernetes é™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#dimension_limits)。 + +### 自动缩放 + +_å¯å˜ï¼šæ˜¯_ + +节点池自动缩放会根æ®å·¥ä½œè´Ÿè½½çš„需求动æ€åˆ›å»ºæˆ–删除节点。详情请å‚è§[本页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-autoscaler)。 + +### è‡ªåŠ¨ä¿®å¤ + +_å¯å˜ï¼šæ˜¯_ + +GKE 的节点自动修å¤åŠŸèƒ½å¯å¸®åŠ©ä½ å°†é›†ç¾¤ä¸­çš„节点ä¿æŒåœ¨å¥åº·çš„è¿è¡ŒçŠ¶æ€ã€‚å¯ç”¨åŽï¼ŒGKE 会定期检查集群中æ¯ä¸ªèŠ‚点的è¿è¡ŒçŠ¶å†µã€‚如果æŸä¸ªèŠ‚点在较长时间段内连续未通过å¥åº·æ£€æŸ¥ï¼ŒGKE 会为该节点å¯åŠ¨ä¿®å¤è¿‡ç¨‹ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[自动修å¤èŠ‚点](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-repair)。 + +### 自动å‡çº§ + +_å¯å˜ï¼šæ˜¯_ + +å¯ç”¨åŽï¼Œå½“ä½ çš„ controlplane [按照你的需求更新](https://cloud.google.com/kubernetes-engine/upgrades#automatic_cp_upgrades)时，自动å‡çº§åŠŸèƒ½ä¼šä½¿é›†ç¾¤ä¸­çš„节点与集群 controlplane（master）版本ä¿æŒåŒæ­¥ã€‚有关自动å‡çº§èŠ‚点的更多信æ¯ï¼Œå‚è§[此页é¢ã€‚](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades) + +### 访问范围 + +_å¯å˜ï¼šå¦_ + +设置访问范围是为你的节点指定æƒé™çš„旧版方法。 + +- **å…许默认访问**：新集群的默认访问是 [Compute Engine 默认 ServiceAccount](https://cloud.google.com/compute/docs/access/service-accounts?hl=en_US#default_service_account)。 +- **å…许完全访问所有 Cloud API**：通常，你åªéœ€è®¾ç½®äº‘å¹³å°è®¿é—®èŒƒå›´æ¥å…许完全访问所有 Cloud API，然åŽä»…授予 ServiceAccount 相关的 IAM 角色。授予虚拟机实例的访问范围和授予 ServiceAccount çš„ IAM 角色的组åˆå†³å®šäº† ServiceAccount 对该实例的访问é‡ã€‚ +- **为æ¯ä¸ª API 设置访问æƒé™**：或者，你å¯ä»¥è®¾ç½®æœåŠ¡å°†è°ƒç”¨çš„特定 API 方法的访问范围。 + +有关详细信æ¯ï¼Œè¯·å‚阅[为 VM å¯ç”¨ ServiceAccount](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances)。 + + +### é…置刷新间隔 + +刷新间隔å¯ä»¥é€šè¿‡ “gke-refresh†æ¥é…置，它是一个代表秒的整数。 + +默认值为 300 秒。 + +ä½ å¯ä»¥é€šè¿‡è¿è¡Œ `kubectl edit setting gke-refresh` æ¥æ›´æ”¹åŒæ­¥é—´éš”。 + +刷新窗å£è¶ŠçŸ­ï¼Œäº‰ç”¨æ¡ä»¶å‘生的å¯èƒ½æ€§å°±è¶Šå°ã€‚但这确实增加了é‡åˆ° GCP API å¯èƒ½å­˜åœ¨çš„请求é™åˆ¶çš„å¯èƒ½æ€§ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/helm-charts-in-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/helm-charts-in-rancher.md new file mode 100644 index 00000000000..67223803e4d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/helm-charts-in-rancher.md @@ -0,0 +1,155 @@ +--- +title: Rancher 中的 Helm Chart +--- + +在本节中，你将学习如何在 Rancher ä¸­ç®¡ç† Helm Chart 仓库和应用。你å¯ä»¥åœ¨ **Apps** ä¸­ç®¡ç† Helm Chart 仓库。它使用类似目录的系统从仓库中导入 Chart 包，然åŽä½¿ç”¨è¿™äº› Chart æ¥éƒ¨ç½²è‡ªå®šä¹‰ Helm 应用或 Rancher 工具（例如监控和 Istio）。Rancher 工具以预加载仓库的方å¼æ供，并能部署为独立的 Helm Chart 。其他仓库åªä¼šæ·»åŠ åˆ°å½“å‰é›†ç¾¤ã€‚ + +### 版本控制方案 + +Rancher 功能 Chart 版本控制方案以 Chart 的主è¦ç‰ˆæœ¬å’Œä¸Šæ¸¸ Chart çš„ `+up` 注释（如果适用）为中心。 + +**主è¦ç‰ˆæœ¬**：Chart 的主è¦ç‰ˆæœ¬ä¸Ž Rancher 次è¦ç‰ˆæœ¬ç›¸å…³è”。当你å‡çº§åˆ°æ–°çš„ Rancher 次è¦ç‰ˆæœ¬æ—¶ï¼Œä½ åº”该确ä¿ä½ çš„所有 **Apps** Chart 也å‡çº§åˆ° Chart 的正确å‘行版本。 + +**功能 Chart**： + +| **Name** | **支æŒçš„最低版本** | **支æŒçš„最高版本** | +| ---------------- | ------------ | ------------ | +| external-ip-webhook | 100.0.0+up1.0.0 | 100.0.1+up1.0.1 | +| harvester-cloud-provider | 100.0.2+up0.1.12 | 100.0.2+up0.1.12 | +| harvester-csi-driver | 100.0.2+up0.1.11 | 100.0.2+up0.1.11 | +| neuvector | 100.0.0+up2.2.0 | 100.0.0+up2.2.0 | +| rancher-alerting-drivers | 100.0.0 | 100.0.2 | +| rancher-backup | 2.0.1 | 2.1.2 | +| rancher-cis-benchmark | 2.0.1 | 2.0.4 | +| rancher-gatekeeper | 100.0.0+up3.6.0 | 100.1.0+up3.7.1 | +| rancher-istio | 100.0.0+up1.10.4 | 100.3.0+up1.13.3 | +| rancher-logging | 100.0.0+up3.12.0 | 100.1.2+up3.17.4 | +| rancher-longhorn | 100.0.0+up1.1.2 | 100.1.2+up1.2.4 | +| rancher-monitoring | 100.0.0+up16.6.0 | 100.1.2+up19.0.3 | +| rancher-sriov (experimental) | 100.0.0+up0.1.0 | 100.0.3+up0.1.0 | +| rancher-vsphere-cpi | 100.3.0+up1.2.1 | 100.3.0+up1.2.1 | +| rancher-vsphere-csi | 100.3.0+up2.5.1-rancher1 | 100.3.0+up2.5.1-rancher1 | +| rancher-wins-upgrader | 0.0.100 | 100.0.1+up0.0.1 | + +
+ +**基于上游的 Chart**：对于基于上游的 Chart ，+up 注释用于表示 Rancher Chart 正在跟踪的上游版本。在å‡çº§æ—¶ï¼Œè¯·æ£€æŸ¥ä¸Šæ¸¸ç‰ˆæœ¬ä¸Ž Rancher 的兼容性。 + +- 例如，用于 Monitoring çš„ `100.x.x+up16.6.0` 跟踪上游 kube-prometheus-stack `16.6.0` 并添加了一些 Rancher è¡¥ä¸ã€‚ + +- 在å‡çº§æ—¶ï¼Œè¯·ç¡®ä¿ä½ æ²¡æœ‰é™çº§ä½ æ­£åœ¨ä½¿ç”¨çš„ Chart 版本。例如，如果你在 Rancher 2.5 中使用 Monitoring > `16.6.0` 版本，则ä¸åº”å‡çº§åˆ° `100.x.x+up16.6.0`。相å，你应该在下一个å‘行版中å‡çº§åˆ°é€‚当的版本。 + +### 预å‘布版本 + +预å‘布版本éµå¾ª [Semantic Versioning 2.0.0](https://semver.org/) 定义的[规范](https://semver.org/#spec-item-9)。例如，版本为 `0.1.3-dev.12ab4f` çš„ Helm chart 为预å‘布版本。默认情况下ä¸æ˜¾ç¤ºé¢„å‘布版本，必须进行é…ç½®æ‰èƒ½æ˜¾ç¤ºã€‚ + +è¦æ˜¾ç¤ºé¢„å‘布版本： + +1. å•å‡»å³ä¸Šè§’的用户头åƒã€‚ +1. å•å‡»**å好设置**。 +1. 在 **Helm Chart** 下，选择**包括预å‘布版本**。 + +### Charts + +从左上角的èœå•ä¸­é€‰æ‹© _Apps_，然åŽä½ ä¼šè½¬åˆ° Chart 页é¢ã€‚ + +Chart 页é¢åŒ…å«æ‰€æœ‰ Rancherã€Partner 和自定义 Chart 。 + +* Rancher 工具（例如 Logging 或 Monitoring）包å«åœ¨ Rancher 标签下 +* Partner Chart ä½äºŽ Partner 标签下 +* 自定义 Chart 将显示在仓库的å称下 + +所有这三ç§ç±»åž‹éƒ½ä»¥ç›¸åŒçš„æ–¹å¼éƒ¨ç½²å’Œç®¡ç†ã€‚ + +:::note + +ç”± Cluster Manager （旧版 Rancher UI 中的全局视图）管ç†çš„应用应继续仅由 Cluster Manager 管ç†ï¼Œè€Œåœ¨æ–° UI 中使用 Apps 管ç†çš„应用则仅能由 Apps 管ç†ã€‚ + +::: + +### 仓库 + +从左侧边æ ä¸­é€‰æ‹© _仓库_。 + +这些项目代表 helm 仓库，å¯ä»¥æ˜¯å…·æœ‰ index.yaml 的传统 helm 端点，也å¯ä»¥æ˜¯è¢«å…‹éš†å¹¶æŒ‡å‘特定分支的 git 仓库。è¦ä½¿ç”¨è‡ªå®šä¹‰ Chart ，åªéœ€åœ¨æ­¤å¤„添加你的仓库å³å¯ï¼Œå®ƒä»¬å°†åœ¨ä»“库å称下的 Chart 选项å¡ä¸­å¯ç”¨ã€‚ + +为 Helm Chart 仓库添加ç§æœ‰ CA： + +- **基于 HTTP çš„ Chart 仓库**：你必须将 DER æ ¼å¼çš„ CA è¯ä¹¦çš„ base64 ç¼–ç å‰¯æœ¬æ·»åŠ åˆ° Chart 仓库的 spec.caBundle 字段，例如 `openssl x509 -outform der -in ca.pem | base64 -w0`。点击 Chart 仓库的**编辑 YAML** 并进行设置，如下所示：
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + + +- **基于 Git çš„ Chart 仓库**：你必须将 DER æ ¼å¼çš„ CA è¯ä¹¦çš„ base64 ç¼–ç å‰¯æœ¬æ·»åŠ åˆ° Chart 仓库的 spec.caBundle 字段，例如 `openssl x509 -outform der -in ca.pem | base64 -w0`。点击 Chart 仓库的**编辑 YAML** 并进行设置，如下所示：
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + +:::note + +带有身份验è¯çš„ Helm Chart 仓库 + +Repo.Spec 包å«ä¸€ä¸ª `disableSameOriginCheck` 值，该值å…许用户绕过相åŒæºçš„检查，将仓库身份认è¯ä¿¡æ¯ä½œä¸ºåŸºæœ¬ Auth 标头与所有 API 调用一起å‘é€ã€‚ä¸å»ºè®®é‡‡ç”¨è¿™ç§åšæ³•ï¼Œä½†è¿™å¯ä»¥ç”¨ä½œéžæ ‡å‡† Helm Chart 仓库（例如é‡å®šå‘到ä¸åŒæº URL 的仓库）的临时解决方案。 + +è¦å°†æ­¤åŠŸèƒ½ç”¨äºŽçŽ°æœ‰ Helm Chart 仓库，请å•å‡» â‹® > 编辑 YAML。在 YAML 文件的 `spec` 部分，添加 `disableSameOriginCheck` 并将其设置为 `true`： + +```yaml +[...] +spec: + disableSameOriginCheck: true +[...] +``` + +::: + +### Helm 兼容性 + +ä»…æ”¯æŒ Helm 3 兼容 Chart 。 + + +### 部署和å‡çº§ + +从 _Chart_ 选项å¡ä¸­é€‰æ‹©è¦å®‰è£…çš„ Chart 。Rancher å’Œ Partner Chart å¯èƒ½é€šè¿‡è‡ªå®šä¹‰é¡µé¢æˆ– questions.yaml 文件进行é¢å¤–çš„é…置，但所有 Chart 安装都å¯ä»¥ä¿®æ”¹ values.yaml 和其他基本设置。å•å‡»å®‰è£…åŽï¼Œå°†éƒ¨ç½²ä¸€ä¸ª Helm æ“作作业，并显示该作业的控制å°ã€‚ + +è¦æŸ¥çœ‹æ‰€æœ‰æœ€è¿‘的更改，请转到 _最近的æ“作_ 选项å¡ã€‚ä½ å¯ä»¥æŸ¥çœ‹å·²è¿›è¡Œçš„调用ã€æ¡ä»¶ã€äº‹ä»¶å’Œæ—¥å¿—。 + +安装 Chart åŽï¼Œä½ å¯ä»¥åœ¨ _已安装的应用_ 选项å¡ä¸­æ‰¾åˆ°è¯¥ Chart。在本节中，你å¯ä»¥å‡çº§æˆ–删除安装，并查看更多详细信æ¯ã€‚选择å‡çº§æ—¶ï¼Œå‘ˆçŽ°çš„å½¢å¼å’Œæ•°å€¼ä¸Žå®‰è£…相åŒã€‚ + +大多数 Rancher 工具在 _Apps_ 下方的工具æ ä¸­éƒ½æœ‰é¢å¤–的页é¢ï¼Œä»¥å¸®åŠ©ä½ ç®¡ç†å’Œä½¿ç”¨è¿™äº›åŠŸèƒ½ã€‚这些页é¢åŒ…括指å‘仪表æ¿çš„链接ã€å¯è½»æ¾æ·»åŠ è‡ªå®šä¹‰èµ„æºçš„表å•ä»¥åŠå…¶ä»–ä¿¡æ¯ã€‚ + +:::caution + +如果你使用 _在å‡çº§å‰è‡ªå®šä¹‰ Helm 选项_ æ¥å‡çº§ Chart，如果你的 Chart 有ä¸å¯æ›´æ”¹çš„字段，使用 _--force_ 选项å¯èƒ½ä¼šå¯¼è‡´é”™è¯¯ã€‚这是因为 Kubernetes 中的æŸäº›å¯¹è±¡ä¸€æ—¦åˆ›å»ºå°±æ— æ³•æ›´æ”¹ã€‚è¦é¿å…该错误，你å¯ä»¥ï¼š + +* 使用默认å‡çº§é€‰é¡¹ï¼ˆå³ä¸è¦ä½¿ç”¨ _--force_ 选项） +* å¸è½½çŽ°æœ‰ Chart 并安装å‡çº§åŽçš„ Chart +* 在执行 _--force_ å‡çº§ä¹‹å‰åˆ é™¤é›†ç¾¤ä¸­å…·æœ‰ä¸å¯æ›´æ”¹å­—æ®µçš„èµ„æº + +::: + +#### 旧版应用 + +**Apps > Installed Apps** 页é¢ä¸­ï¼Œæ—§ç‰ˆåº”用的å‡çº§æŒ‰é’®å·²è¢«ç§»é™¤ã€‚ + +如果你安装了旧版应用并想è¦å‡çº§å®ƒï¼š + +- 必须开å¯æ—§ç‰ˆ[功能开关](enable-experimental-features.md)（如果在å‡çº§å‰æœ‰æ—§ç‰ˆåº”用导致该开关未自动开å¯ï¼‰ +- ä½ å¯ä»¥ä»Ž cluster explorer å‡çº§åº”用，从左侧导航部分选择**旧版 > 项目 > 应用** +- 对于多集群应用，你å¯ä»¥è½¬åˆ° **≡ > 多集群应用**并在那里å‡çº§åº”用 + +### é™åˆ¶ + +Rancher CLI **ä¸èƒ½**用于安装[仪表æ¿åº”用程åºæˆ– Rancher 功能 Chart](helm-charts-in-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/horizontal-pod-autoscaler.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/horizontal-pod-autoscaler.md new file mode 100644 index 00000000000..eeee4305c97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/horizontal-pod-autoscaler.md @@ -0,0 +1,26 @@ +--- +title: Horizontal Pod Autoscaler +description: 了解 Pod 水平自动扩缩 (HPA)ã€‚å¦‚ä½•ç®¡ç† HPA 以åŠå¦‚何使用æœåŠ¡éƒ¨ç½²æ¥è¿›è¡Œæµ‹è¯• +--- + +[Horizontal Pod Autoscaler（HPA）](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)是一项 Kubernetes 功能，用于将集群é…置为自动扩缩其è¿è¡Œçš„æœåŠ¡ã€‚ + +Rancher æ供了一些附加功能æ¥å¸®åŠ©ç®¡ç† HPA，具体å–决于 Rancher 的版本。 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 创建ã€ç®¡ç†å’Œåˆ é™¤ HPA。仅在 `autoscaling/v2beta2` API ä¸­æ”¯æŒ HPA。 + +## ç®¡ç† HPA + +ç®¡ç† HPA çš„æ–¹å¼å› ä½ çš„ Kubernetes API 版本而异： + +- **Kubernetes API 版本 autoscaling/V2beta1**：å…许根æ®åº”用程åºçš„ CPU 和内存利用率自动扩缩 pod。 +- **Kubernetes API 版本 autoscaling/V2beta2**：å…è®¸æ ¹æ® CPU 和内存利用率以åŠè‡ªå®šä¹‰æŒ‡æ ‡è‡ªåŠ¨æ‰©ç¼© pod。 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 创建ã€ç®¡ç†å’Œåˆ é™¤ HPA。在 Rancher UI 中，你å¯ä»¥å°† HPA é…ç½®ä¸ºæ ¹æ® CPU 和内存利用率进行扩缩。有关详细信æ¯ï¼Œè¯·å‚阅[使用 Rancher UI ç®¡ç† HPA](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md)。如需根æ®è‡ªå®šä¹‰æŒ‡æ ‡è¿›è¡Œ HPA，你ä»ç„¶éœ€è¦ä½¿ç”¨ `kubectl`。有关详细信æ¯ï¼Œè¯·å‚阅[é…ç½® HPA 以使用 Prometheus 自定义指标进行扩缩](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md#é…ç½®-hpa-以使用-prometheus-自定义指标进行扩缩)。 + +在 Rancher 2.0.7 åŠæ›´é«˜ç‰ˆæœ¬ä¸­åˆ›å»ºçš„集群自动满足使用 HPA 的所有è¦æ±‚（metrics-server å’Œ Kubernetes 集群é…置）。 +## 使用æœåŠ¡éƒ¨ç½²æµ‹è¯• HPA + +ä½ å¯ä»¥è½¬åˆ°ä½ çš„项目并å•å‡»**èµ„æº > HPA**æ¥æŸ¥çœ‹â€‹â€‹ HPA 当å‰çš„副本数。有关详细信æ¯ï¼Œè¯·å‚阅[èŽ·å– HPA 指标和状æ€](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md)。 + +你还å¯ä»¥ä½¿ç”¨ `kubectl` æ¥èŽ·å–你使用负载测试工具测试的 HPA 的状æ€ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[使用 kubectl 测试 HPA](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/infrastructure-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/infrastructure-setup.md new file mode 100644 index 00000000000..16b3b3d4d97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/infrastructure-setup.md @@ -0,0 +1,8 @@ +--- +title: Kubernetes 集群基础设施 +--- + +è¦ä¸ºå…·æœ‰å¤–部数æ®åº“的高å¯ç”¨ K3s Kubernetes 集群设置基础设施，请å‚è§[本页é¢](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md)。 + + +è¦ä¸ºé«˜å¯ç”¨ RKE Kubernetes 集群设置基础设施，请å‚è§[本页é¢](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/install-cluster-autoscaler.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/install-cluster-autoscaler.md new file mode 100644 index 00000000000..87620b84498 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/install-cluster-autoscaler.md @@ -0,0 +1,24 @@ +--- +title: Cluster Autoscaler +--- + +在本文中，你将学习如何使用 AWS EC2 Auto Scaling 组在 Rancher 自定义集群上安装和使用 [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/)。 + +Cluster Autoscaler 是一个自动调整 Kubernetes 集群大å°çš„工具。该工具在满足以下æ¡ä»¶ä¹‹ä¸€æ—¶èƒ½è‡ªåŠ¨è°ƒæ•´é›†ç¾¤å¤§å°ï¼š + +* 集群中有 Pod 因资æºä¸è¶³è€Œæ— æ³•è¿è¡Œã€‚ +* 集群中有一些节点长时间未得到充分利用，而且它们的 Pod å¯ä»¥æ”¾åˆ°å…¶ä»–现有节点上。 + +为防止你的 pod 被驱é€ï¼Œè¯·åœ¨ä½ çš„ pod 规范中设置 `priorityClassName: system-cluster-critical` 属性。 + +Cluster Autoscaler è¿è¡Œåœ¨ Kubernetes master 节点上。它å¯ä»¥åœ¨ `kube-system` 命å空间中è¿è¡Œã€‚Cluster Autoscaler ä¸ä¼šç¼©å‡è¿è¡Œéžé•œåƒ `kube-system` pod 的节点。 + +ä½ å¯ä»¥åœ¨ worker 节点上è¿è¡Œ Cluster Autoscaler 的自定义 deployment，但需è¦å°å¿ƒä»¥ä¿è¯ Cluster Autoscaler 能正常è¿è¡Œã€‚ + +## 云æ供商 + +Cluster Autoscaler 为ä¸åŒçš„云æ供商æ供支æŒã€‚有关详细信æ¯ï¼Œè¯·å‚è§ [Cluster Autoscaler 支æŒçš„云æ供商](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#deployment)。 + +### 在 Amazon 上设置 Cluster Autoscaler + +有关在 Amazon 上è¿è¡Œ Cluster Autoscaler 的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md new file mode 100644 index 00000000000..b189e6d7a35 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md @@ -0,0 +1,352 @@ +--- +title: 在 Kubernetes 集群上安装/å‡çº§ Rancher +description: 了解如何在开å‘和生产环境中安装 Rancher。了解å•èŠ‚点和高å¯ç”¨å®‰è£… +--- + +在本节中，你将学习如何使用 Helm CLI 在 Kubernetes 集群上部署 Rancher。 + + +## 先决æ¡ä»¶ + +- [Kubernetes 集群](#kubernetes-集群) +- [Ingress Controller](#ingress-controller) +- [CLI 工具](#cli-工具) + +### Kubernetes 集群 + +设置 Rancher Server 的本地 Kubernetes 集群。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上。这个集群å¯ä»¥ä½¿ç”¨ä¸Šæ¸¸ Kubernetes，也å¯ä»¥ä½¿ç”¨ Rancher çš„ Kubernetes å‘行版之一，也å¯ä»¥æ˜¯æ¥è‡ª Amazon EKS ç­‰æ供商的托管 Kubernetes 集群。 + +ä½ å¯å‚考以下教程，以获得设置 Kubernetes 集群的帮助： + +- **RKE**：[安装 RKE Kubernetes 集群的教程](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md)ï¼›[为高å¯ç”¨ RKE 集群设置基础设施的教程](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md)。 +- **K3s**：[安装 K3s Kubernetes 集群的教程](../how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md)ï¼›[设置高å¯ç”¨ K3s 集群的基础设施的教程](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md)。 +- **RKE2:** ：[安装 RKE2 Kubernetes 集群的教程](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md)ï¼›[设置高å¯ç”¨ RKE2 集群的基础设施的教程](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md)。 +- **Amazon EKS**：[在 Amazon EKS 上安装 Rancher 以åŠå¦‚何安装 Ingress Controller 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md)。 +- **AKS**：[使用 Azure Kubernetes æœåŠ¡å®‰è£… Rancher 以åŠå¦‚何安装 Ingress Controller 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md)。 +- **GKE**：有关如何使用 GKE 安装 Rancher，包括如何安装 Ingress Controller 以便å¯ä»¥è®¿é—® Rancher Server，请å‚阅[此页é¢](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md)。GKE 创建 Kubernetes 集群时有两ç§è¿è¡Œæ¨¡å¼ï¼Œåˆ†åˆ«æ˜¯ Autopilot å’Œ Standard 模å¼ã€‚Autopilot 模å¼çš„集群é…置对编辑 kube-system 命å空间有é™åˆ¶ã€‚但是，Rancher 在安装时需è¦åœ¨ kube-system 命å空间中创建资æºã€‚因此，你将无法在以 Autopilot 模å¼åˆ›å»ºçš„ GKE 集群上安装 Rancher。 + + +### Ingress Controller + +Rancher UI å’Œ API 通过 Ingress 公开。æ¢è¨€ä¹‹ï¼Œå®‰è£… Rancher çš„ Kubernetes 集群必须包å«ä¸€ä¸ª Ingress Controller。 + +对于 RKEã€RKE2 å’Œ K3s，你ä¸éœ€è¦æ‰‹åŠ¨å®‰è£… Ingress Controller，因为它是默认安装的。 + +对于默认ä¸åŒ…å« Ingress Controller çš„å‘行版（例如 EKSã€GKE 或 AKS 等托管 Kubernetes 集群），你必须先部署 Ingress Controller。请注æ„，Rancher Helm Chart 默认情况下ä¸ä¼šåœ¨ Ingress 上设置 `ingressClassName`。因此，你必须将 Ingress Controller é…置为在没有 `ingressClassName` 的情况下也å¯ä»¥ç›‘视 Ingress。 + +上é¢çš„ **Amazon EKS**ã€**AKS** å’Œ **GKE** 教程中包å«äº†ç¤ºä¾‹ã€‚ + +### CLI 工具 + +设置 Kubernetes 集群需è¦ä»¥ä¸‹ CLI 工具。请确ä¿è¿™äº›å·¥å…·å·²å®‰è£…并在你的 `$PATH` 中å¯ç”¨ã€‚ + +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) - Kubernetes 命令行工具。 +- [Helm](https://docs.helm.sh/using_helm/#installing-helm) - Kubernetes 的包管ç†å™¨ã€‚请å‚è§ [Helm 版本è¦æ±‚](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。请为你的具体平å°å‚è§ [Helm 项目æ供的说明](https://helm.sh/docs/intro/install/)。 + +## 安装 Rancher Helm Chart + +Rancher 是使用 Kubernetes çš„ [Helm](https://helm.sh/) 包管ç†å™¨å®‰è£…的。Helm Chart 为 Kubernetes YAML 清å•æ–‡ä»¶æ供了模æ¿è¯­æ³•ã€‚通过 Helm，用户å¯ä»¥åˆ›å»ºå¯é…置的 deployment，而ä¸ä»…ä»…åªèƒ½ä½¿ç”¨é™æ€æ–‡ä»¶ã€‚ + +如果系统无法直接访问互è”网，请å‚è§[离线环境：Kubernetes 安装](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md)。 + +如果è¦æŒ‡å®šå®‰è£…çš„ Rancher 版本，请å‚è§[选择 Rancher 版本](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +如果è¦æŒ‡å®šç”¨äºŽå®‰è£… Rancher çš„ Helm 版本，请å‚è§[Helm 版本è¦æ±‚](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md)。 + +:::note + +本安装指å—å‡å®šä½ ä½¿ç”¨çš„是 Helm 3。 + +::: + +è¦è®¾ç½® Rancher： + +1. [添加 Helm Chart 仓库](#1-添加-helm-chart-仓库) +2. [为 Rancher 创建命å空间](#2-为-rancher-创建命å空间) +3. [选择 SSL é…ç½®](#3-选择-ssl-é…ç½®) +4. [安装 cert-manager](#4-安装-cert-manager)（除éžä½ è‡ªå¸¦è¯ä¹¦ï¼Œå¦åˆ™ TLS 将在负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ï¼‰ +5. [使用 Helm 和你选择的è¯ä¹¦é€‰é¡¹å®‰è£… Rancher](#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher) +6. [éªŒè¯ Rancher Server 是å¦éƒ¨ç½²æˆåŠŸ](#6-验è¯-rancher-server-是å¦éƒ¨ç½²æˆåŠŸ) +7. [ä¿å­˜é€‰é¡¹](#7-ä¿å­˜é€‰é¡¹) + +### 1. 添加 Helm Chart 仓库 + +执行 `helm repo add` 命令，以添加包å«å®‰è£… Rancher çš„ Chart çš„ Helm Chart 仓库。有关如何选择仓库，以åŠå“ªä¸ªä»“库最适åˆä½ çš„用例，请å‚è§[选择 Rancher 版本](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +- Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + +### 2. 为 Rancher 创建命å空间 + +你需è¦å®šä¹‰ä¸€ä¸ª Kubernetes 命å空间，用于安装由 Chart 创建的资æºã€‚这个命å空间的å称为 `cattle-system`： + +``` +kubectl create namespace cattle-system +``` + +### 3. 选择 SSL é…ç½® + +Rancher Management Server é»˜è®¤éœ€è¦ SSL/TLS é…ç½®æ¥ä¿è¯è®¿é—®çš„安全性。 + +:::note + +如果你想在外部终止 SSL/TLS，请å‚è§[外部负载å‡è¡¡å™¨çš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +::: + +ä½ å¯ä»¥ä»Žä»¥ä¸‹ä¸‰ç§è¯ä¹¦æ¥æºä¸­é€‰æ‹©ä¸€ç§ï¼Œç”¨äºŽåœ¨ Rancher Server 中终止 TLS： + +- **Rancher 生æˆçš„ TLS è¯ä¹¦**：è¦æ±‚你在集群中安装 `cert-manager`。Rancher 使用 `cert-manager` ç­¾å‘并维护è¯ä¹¦ã€‚Rancher 会生æˆè‡ªå·±çš„ CA è¯ä¹¦ï¼Œå¹¶ä½¿ç”¨è¯¥ CA 签署è¯ä¹¦ã€‚ç„¶åŽ `cert-manager`负责管ç†è¯¥è¯ä¹¦ã€‚ +- **Let's Encrypt**：Let's Encrypt 选项也需è¦ä½¿ç”¨ `cert-manager`。但是，在这ç§æƒ…况下，cert-manager 与 Let's Encrypt 的特殊é¢å‘者相结åˆï¼Œè¯¥é¢å‘è€…æ‰§è¡ŒèŽ·å– Let's Encrypt é¢å‘çš„è¯ä¹¦æ‰€éœ€çš„所有æ“作（包括请求和验è¯ï¼‰ã€‚æ­¤é…置使用 HTTP 验è¯ï¼ˆ`HTTP-01`），因此负载å‡è¡¡å™¨å¿…须具有å¯ä»¥ä»Žäº’è”网访问的公共 DNS 记录。 +- **你已有的è¯ä¹¦**：使用已有的 CA é¢å‘的公有或ç§æœ‰è¯ä¹¦ã€‚Rancher 将使用该è¯ä¹¦æ¥ä¿æŠ¤ WebSocket å’Œ HTTPS æµé‡ã€‚在这ç§æƒ…况下，你必须上传å称分别为 `tls.crt` å’Œ `tls.key`çš„ PEM æ ¼å¼çš„è¯ä¹¦ä»¥åŠç›¸å…³çš„密钥。如果你使用ç§æœ‰ CA，则还必须上传该 CA è¯ä¹¦ã€‚这是由于你的节点å¯èƒ½ä¸ä¿¡ä»»æ­¤ç§æœ‰ CA。Rancher 将获å–该 CA è¯ä¹¦ï¼Œå¹¶ä»Žä¸­ç”Ÿæˆä¸€ä¸ªæ ¡éªŒå’Œï¼Œå„ç§ Rancher 组件将使用该校验和æ¥éªŒè¯å…¶ä¸Ž Rancher 的连接。 + + +| é…ç½® | Helm Chart 选项 | 是å¦éœ€è¦ cert-manager | +| ------------------------------ | ----------------------- | ------------------------------------- | +| Rancher 生æˆçš„è¯ä¹¦ï¼ˆé»˜è®¤ï¼‰ | `ingress.tls.source=rancher` | [是](#4-安装-cert-manager) | +| Let’s Encrypt | `ingress.tls.source=letsEncrypt` | [是](#4-安装-cert-manager) | +| 你已有的è¯ä¹¦ | `ingress.tls.source=secret` | å¦ | + +### 4. 安装 cert-manager + +> 如果你使用自己的è¯ä¹¦æ–‡ä»¶ï¼ˆ`ingress.tls.source=secret`）或使用[外部负载å‡è¡¡å™¨çš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)，你å¯ä»¥è·³è¿‡æ­¤æ­¥éª¤ã€‚ + +仅在使用 Rancher 生æˆçš„è¯ä¹¦ï¼ˆ`ingress.tls.source=rancher`）或 Let's Encrypt é¢å‘çš„è¯ä¹¦ï¼ˆ`ingress.tls.source=letsEncrypt`）时，æ‰éœ€è¦å®‰è£… cert-manager。 + +
+ å•å‡»å±•å¼€ + +:::note é‡è¦æ示： + +由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.11.0 çš„ cert-manager 版本，请å‚è§[å‡çº§æ–‡æ¡£](../getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md)。 + +::: + +这些说明æ¥è‡ª [cert-manager 官方文档](https://cert-manager.io/docs/installation/kubernetes/#installing-with-helm)。 + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +``` +# 如果你手动安装了CRD，而ä¸æ˜¯åœ¨ Helm 安装命令中添加了 `--set installCRDs=true` 选项，你应该在å‡çº§ Helm Chart 之å‰å‡çº§ CRD 资æºã€‚ +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +# 添加 Jetstack Helm 仓库 +helm repo add jetstack https://charts.jetstack.io + +# 更新本地 Helm Chart 仓库缓存 +helm repo update + +# 安装 cert-manager Helm Chart +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 +``` + +安装完 cert-manager åŽï¼Œä½ å¯ä»¥é€šè¿‡æ£€æŸ¥ cert-manager 命å空间中正在è¿è¡Œçš„ Pod æ¥éªŒè¯å®ƒæ˜¯å¦å·²æ­£ç¡®éƒ¨ç½²ï¼š + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +
+ +### 5. æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹ï¼Œé€šè¿‡ Helm 安装 Rancher + +ä¸åŒçš„è¯ä¹¦é…置需è¦ä½¿ç”¨ä¸åŒçš„ Rancher 安装命令。 + +但是，无论è¯ä¹¦å¦‚何é…置，Rancher 在 `cattle-system` 命å空间中的安装å称应该总是 `rancher`。 + +:::tip 测试和开å‘： + +这个安装 Rancher 的最终命令需è¦ä¸€ä¸ªå°†æµé‡è½¬å‘到 Rancher 的域å。如果你使用 Helm CLI 设置概念è¯æ˜Žï¼Œåˆ™å¯ä»¥åœ¨ä¼ å…¥ `hostname` 选项时使用伪域å。伪域å的一个例å­æ˜¯ `.sslip.io`，这会把 Rancher 暴露在它è¿è¡Œçš„ IP 上。生产安装中è¦æ±‚填写真实的域å。 + +::: + + + + +默认情况是使用 Rancher ç”Ÿæˆ CA，并使用 `cert-manager` é¢å‘用于访问 Rancher Server 接å£çš„è¯ä¹¦ã€‚ + +由于 `rancher` 是 `ingress.tls.source` 的默认选项，因此在执行 `helm install` 命令时，我们ä¸éœ€è¦æŒ‡å®š `ingress.tls.source`。 + +- å°† `hostname` 设置为解æžåˆ°ä½ çš„è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 +- å°† `bootstrapPassword` 设置为 `admin` 用户独有的值。 +- 如果你需è¦å®‰è£…指定的 Rancher 版本，使用 `--version` 标志，例如 `--version 2.7.0`。 +- 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin +``` + +如果你安装的是 alpha 版本，Helm 会è¦æ±‚你在安装命令中添加 `--devel` 选项： + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +等待 Rancher è¿è¡Œï¼š + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + + +此选项使用 `cert-manager` æ¥è‡ªåŠ¨è¯·æ±‚和续订 [Let's Encrypt](https://letsencrypt.org/) è¯ä¹¦ã€‚Let's Encrypt 是å…费的，而且是å—ä¿¡çš„ CA，因此å¯ä»¥ä¸ºä½ æ供有效的è¯ä¹¦ã€‚ + +:::note + +由于 HTTP-01 质询åªèƒ½åœ¨ç«¯å£ 80 上完æˆï¼Œå› æ­¤ä½ éœ€è¦æ‰“å¼€ç«¯å£ 80。 + +::: + +在以下命令中， + +- å°† `hostname` 设置为公有 DNS 记录。 +- å°† `bootstrapPassword` 设置为 `admin` 用户独有的值。 +- å°† `ingress.tls.source` 设置为 `letsEncrypt`。 +- å°† `letsEncrypt.email` 设置为å¯é€šè®¯çš„电å­é‚®ä»¶åœ°å€ï¼Œç”¨äºŽå‘é€é€šçŸ¥ï¼ˆä¾‹å¦‚è¯ä¹¦åˆ°æœŸçš„通知）。 +- å°† `letsEncrypt.ingress.class` 设为你的 Ingress Controller（例如 `traefik`，`nginx`，`haproxy`） +- 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=letsEncrypt \ + --set letsEncrypt.email=me@example.org \ + --set letsEncrypt.ingress.class=nginx +``` + +如果你安装的是 alpha 版本，Helm 会è¦æ±‚你在安装命令中添加 `--devel` 选项： + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +等待 Rancher è¿è¡Œï¼š + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + +在此选项中，你使用你自己的è¯ä¹¦æ¥åˆ›å»º Kubernetes 密文，以供 Rancher 使用。 + +è¿è¡Œè¿™ä¸ªå‘½ä»¤æ—¶ï¼Œ`hostname` 选项必须与æœåŠ¡å™¨è¯ä¹¦ä¸­çš„ `Common Name` 或 `Subject Alternative Names` æ¡ç›®åŒ¹é…，å¦åˆ™ Ingress controller 将无法正确é…置。 + +è™½ç„¶æŠ€æœ¯ä¸Šä»…éœ€è¦ `Subject Alternative Names` 中有一个æ¡ç›®ï¼Œä½†æ˜¯æ‹¥æœ‰ä¸€ä¸ªåŒ¹é…çš„ `Common Name` å¯ä»¥æœ€å¤§ç¨‹åº¦åœ°æ高与旧版æµè§ˆå™¨/应用的兼容性。 + +:::note + +如果你想检查è¯ä¹¦æ˜¯å¦æ­£ç¡®ï¼Œè¯·æŸ¥çœ‹[如何在æœåŠ¡å™¨è¯ä¹¦ä¸­æ£€æŸ¥ Common Name å’Œ Subject Alternative Names](../faq/technical-items.md#如何在æœåŠ¡å™¨è¯ä¹¦ä¸­æ£€æŸ¥-common-name-å’Œ-subject-alternative-names)。 + +::: + +- 设置 `hostname`。 +- å°† `bootstrapPassword` 设置为 `admin` 用户独有的值。 +- å°† `ingress.tls.source` 设置为 `secret`。 +- 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret +``` +如果你安装的是 alpha 版本，Helm 会è¦æ±‚你在安装命令中添加 `--devel` 选项： + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +如果你使用的是ç§æœ‰ CA è¯ä¹¦ï¼Œè¯·åœ¨å‘½ä»¤ä¸­å¢žåŠ  `--set privateCA=true`。 + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret \ + --set privateCA=true +``` + +**添加 TLS 密文（åƒä¸‡ä¸è¦é—æ¼æ­¤æ­¥éª¤ï¼‰**：现在 Rancher å·²ç»å®Œæˆéƒ¨ç½²ï¼Œä½ è¿˜éœ€è¦å‚考[添加 TLS 密文](../getting-started/installation-and-upgrade/resources/add-tls-secrets.md)å‘布è¯ä¹¦æ–‡ä»¶ï¼Œä»¥ä¾¿ Rancher å’Œ Ingress Controller å¯ä»¥ä½¿ç”¨å®ƒä»¬ã€‚ + + + + +Rancher Chart 有许多选项，用于为你的具体环境自定义安装。以下是一些常è§çš„高级方案： + +- [HTTP 代ç†](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#http-代ç†) +- [ç§æœ‰å®¹å™¨é•œåƒä»“库](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#ç§æœ‰ä»“库和离线安装) +- [外部负载å‡è¡¡å™¨ä¸Šçš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止) + +如需获å–完整的选项列表，请å‚è§ [Chart 选项](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)。 + + +### 6. éªŒè¯ Rancher Server 是å¦éƒ¨ç½²æˆåŠŸ + +添加密文åŽï¼Œæ£€æŸ¥ Rancher 是å¦å·²æˆåŠŸè¿è¡Œï¼š + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + +如果你看到 `error: deployment "rancher" exceeded its progress deadline` 这个错误，å¯è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ£€æŸ¥ deployment 的状æ€ï¼š + +``` +kubectl -n cattle-system get deploy rancher +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +rancher 3 3 3 3 3m +``` + +`DESIRED` å’Œ `AVAILABLE`的个数应该相åŒã€‚ + +### 7. ä¿å­˜é€‰é¡¹ + +请ä¿å­˜ä½ ä½¿ç”¨çš„ `--set` 选项。使用 Helm å‡çº§ Rancher 到新版本时，你将需è¦ä½¿ç”¨ç›¸åŒçš„选项。 + +### å®‰è£…å®Œæˆ + +安装已完æˆã€‚现在 Rancher Server 应该已ç»å¯ä»¥æ­£å¸¸è¿è¡Œäº†ã€‚ + +使用æµè§ˆå™¨æ‰“开把æµé‡è½¬å‘到你的负载å‡è¡¡å™¨çš„ DNS 域å。然åŽï¼Œä½ å°±ä¼šçœ‹åˆ°ä¸€ä¸ªæ¼‚亮的登录页é¢äº†ã€‚ + +如果é‡åˆ°ä»»ä½•é—®é¢˜ï¼Œè¯·å‚è§[故障排除](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-and-upgrade.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-and-upgrade.md new file mode 100644 index 00000000000..d57549034f6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-and-upgrade.md @@ -0,0 +1,94 @@ +--- +title: 安装/å‡çº§ Rancher +description: 了解如何在开å‘和生产环境中安装 Rancher。了解å•èŠ‚点和高å¯ç”¨å®‰è£… +--- + +本节介ç»äº† Rancher å„ç§å®‰è£…æ–¹å¼ä»¥åŠæ¯ä¸ªå®‰è£…æ–¹å¼çš„优点。 + +## åè¯è§£é‡Š + +本章节涉åŠä»¥ä¸‹åè¯ï¼š + +- **Rancher Server**：用于管ç†å’Œé…ç½® Kubernetes 集群。你å¯ä»¥é€šè¿‡ Rancher Server çš„ UI 与下游 Kubernetes 集群进行交互。Rancher Management Server å¯ä»¥å®‰è£…åˆ°ä»»æ„ Kubernetes 集群上，包括托管的集群，如 Amazon EKS 集群。 +- **RKE（Rancher Kubernetes Engine）**：是ç»è¿‡è®¤è¯çš„ Kubernetes å‘è¡Œç‰ˆï¼Œä¹Ÿæ˜¯ç”¨äºŽåˆ›å»ºå’Œç®¡ç† Kubernetes 集群的 CLI 工具和库。 +- **K3s（轻é‡çº§ Kubernetes）**：也是ç»è¿‡è®¤è¯çš„ Kubernetes å‘行版。它比 RKE 更新，更易用且更轻é‡ï¼Œå…¶æ‰€æœ‰ç»„件都在一个å°äºŽ 100 MB 的二进制文件中。 +- **RKE2**：一个完全åˆè§„çš„ Kubernetes å‘行版，专注于安全和åˆè§„性。 + +`restrictedAdmin` Helm Chart 选项在 **Rancher Server** å¯ç”¨ã€‚如果该选项设置为 true，åˆå§‹çš„ Rancher 用户访问本地 Kubernetes 集群会å—到é™åˆ¶ï¼Œä»¥é¿å…æƒé™å‡çº§ã€‚详情请å‚è§ [restricted-admin 角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#å—é™ç®¡ç†å‘˜)。 + +## 安装方å¼æ¦‚è¿° + +Rancher å¯ä»¥å®‰è£…在以下主è¦æž¶æž„上： + +### 使用 Helm CLI 安装的高å¯ç”¨ Kubernetes + +我们建议使用 Kubernetes 包管ç†å™¨ Helm 在专用的 Kubernetes 集群上安装 Rancher。在 RKE 集群中，需è¦ä½¿ç”¨ä¸‰ä¸ªèŠ‚点æ‰èƒ½å®žçŽ°é«˜å¯ç”¨é›†ç¾¤ã€‚在 K3s 集群中，åªéœ€è¦ä¸¤ä¸ªèŠ‚点å³å¯ã€‚ + +### 通过 AWS Marketplace 在 EKS 上安装 Rancher + +ä½ å¯ä»¥[通过 AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md) å°† Rancher 安装到 Amazon Elastic Kubernetes Service (EKS) 上。部署的 EKS 集群已生产就绪，并éµå¾ª AWS 最佳实践。 + +### å•èŠ‚点 Kubernetes 安装 + +Rancher å¯ä»¥å®‰è£…在å•èŠ‚点 Kubernetes 集群上。但是，在å•èŠ‚点安装的情况下，Rancher Server 没有高å¯ç”¨æ€§ã€‚而高å¯ç”¨æ€§å¯¹åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­è¿è¡Œ Rancher éžå¸¸é‡è¦ã€‚ + +但是，如果你想è¦çŸ­æœŸå†…使用å•èŠ‚点节çœèµ„æºï¼ŒåŒæ—¶åˆä¿ç•™é«˜å¯ç”¨æ€§è¿ç§»è·¯å¾„，那么å•èŠ‚点 Kubernetes 安装也是åˆé€‚的。你也å¯ä»¥ä¹‹åŽå‘集群中添加节点，获得高å¯ç”¨çš„ Rancher Server。 + +### Docker 安装 + +如果你的目的是测试或演示，你å¯ä»¥ä½¿ç”¨ Docker 把 Rancher 安装到å•ä¸ªèŠ‚点中。本地 Kubernetes 集群是安装到å•ä¸ª Docker 容器中的，而 Rancher 是安装到本地集群中的。 + +Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +### å…¶ä»–æ–¹å¼ + +如果你需è¦åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­æˆ–使用 HTTP 代ç†å®‰è£… Rancher，请å‚è§ä»¥ä¸‹ç‹¬ç«‹çš„说明文档： + +| ç½‘ç»œè®¿é—®æ–¹å¼ | 基于 Kubernetes 安装（推è） | 基于 Docker 安装 | +| ---------------------------------- | ------------------------------ | ---------- | +| å¯ç›´æŽ¥è®¿é—®äº’è”网 | [文档](install-upgrade-on-a-kubernetes-cluster.md) | [文档](rancher-on-a-single-node-with-docker.md) | +| 使用 HTTP ä»£ç† | [文档](rancher-behind-an-http-proxy.md) | [文档](rancher-on-a-single-node-with-docker.md)åŠ[é…ç½®](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) | +| 离线环境 | [文档](air-gapped-helm-cli-install.md) | [文档](air-gapped-helm-cli-install.md) | + +我们建议在 Kubernetes 集群上安装 Rancher，因为在多节点集群中，Rancher Server å¯ä»¥å®žçŽ°é«˜å¯ç”¨ã€‚高å¯ç”¨é…ç½®å¯ä»¥æå‡ Rancher 访问其管ç†çš„下游 Kubernetes 集群的稳定性。 + +因此，我们建议在生产级别的架构中，设置一个高å¯ç”¨çš„ Kubernetes 集群，然åŽåœ¨è¿™ä¸ªé›†ç¾¤ä¸Šå®‰è£… Rancher。安装 Rancher åŽï¼Œä½ å¯ä»¥ä½¿ç”¨ Rancher éƒ¨ç½²å’Œç®¡ç† Kubernetes 集群。 + +如果你的目的是测试或演示，你å¯ä»¥å°† Rancher 安装到å•ä¸ª Docker 容器中。Docker 安装å¯ä»¥è®©ä½ å®žçŽ°å¼€ç®±å³ç”¨ï¼Œä»¥ä½¿ç”¨ Rancher 设置 Kubernetes 集群。Docker 安装主è¦æ˜¯ç”¨äºŽæŽ¢ç´¢ Rancher Server 的功能，åªé€‚用于开å‘和测试。 + +[在 Kubernetes 上安装 Rancher 的说明](install-upgrade-on-a-kubernetes-cluster.md)介ç»äº†å¦‚何首先使用 K3s 或 RKE åˆ›å»ºå’Œç®¡ç† Kubernetes 集群，然åŽå†å°† Rancher 安装到该集群上。 + +如果 Kubernetes 集群中的节点正在è¿è¡Œä¸”满足[节点è¦æ±‚](installation-requirements.md)，你å¯ä»¥ä½¿ç”¨ Helm å°† Rancher 部署到 Kubernetes 上。Helm 使用 Rancher çš„ Helm Chart 在 Kubernetes 集群的æ¯ä¸ªèŠ‚点上安装 Rancher 的副本。我们建议使用负载å‡è¡¡å™¨å°†æµé‡å®šå‘到集群中的æ¯ä¸ª Rancher 副本上。 + +如需进一步了解 Rancher 架构，请å‚è§[架构概述](rancher-manager-architecture.md)，[生产级别架构推è](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)或[最佳实践指å—](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md)。 + +## 先决æ¡ä»¶ + +安装 Rancher 之å‰ï¼Œè¯·ç¡®ä¿ä½ çš„节点满足所有[安装è¦æ±‚](installation-requirements.md)。 + +## 架构建议 + +为了达到最佳性能和安全性，我们建议你为 Rancher Management Server 使用å•ç‹¬çš„专用 Kubernetes 集群。ä¸å»ºè®®åœ¨æ­¤é›†ç¾¤ä¸Šè¿è¡Œç”¨æˆ·å·¥ä½œè´Ÿè½½ã€‚部署 Rancher åŽï¼Œä½ å¯ä»¥[创建或导入集群](kubernetes-clusters-in-rancher-setup.md)æ¥è¿è¡Œä½ çš„工作负载。 + +详情请å‚è§[架构推è](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)。 + +### 在 Kubernetes 上安装 Rancher 的更多选项 + +å‚è§ [Helm Chart 选项](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)以了解在 Kubernetes 集群上安装 Rancher 的其他é…置，包括： + +- [å¼€å¯ API 审计日志æ¥è®°å½•æ‰€æœ‰äº‹åŠ¡](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-审计日志) +- [è´Ÿè½½å‡è¡¡å™¨ä¸Šçš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止) +- [自定义 Ingress](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#自定义-ingress) + +在 Rancher 的安装指å—中，我们推è使用 K3s 或 RKE æ¥é…ç½® Kubernetes 集群，然åŽå†åœ¨è¿™ä¸ªé›†ç¾¤ä¸­å®‰è£… Rancher。K3s å’Œ RKE å‡æ供许多é…置选项，用于为你的具体环境自定义 Kubernetes 集群。有关选项和功能的完整列表，请å‚è§ï¼š + +- [RKE é…置选项](https://rancher.com/docs/rke/latest/en/config-options/) +- [K3s é…置选项](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +### 在 Docker 上安装 Rancher 的更多选项 + +å‚è§ [Docker 安装选项](rancher-on-a-single-node-with-docker.md)了解其他é…置，包括： + +- [å¼€å¯ API 审计日志æ¥è®°å½•æ‰€æœ‰äº‹åŠ¡](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志) +- [外部负载å‡è¡¡å™¨](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md) +- [æŒä¹…化数æ®å­˜å‚¨](../reference-guides/single-node-rancher-in-docker/advanced-options.md#æŒä¹…化数æ®) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-references.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-references.md new file mode 100644 index 00000000000..e7a5b5ac802 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-references.md @@ -0,0 +1,5 @@ +--- +title: 安装å‚考 +--- + +有关其他安装资æºï¼Œè¯·å‚阅以下å‚考指å—：[Rancher Helm Chart 选项](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)ã€[TLS 设置](../getting-started/installation-and-upgrade/installation-references/tls-settings.md)å’Œ[功能开关](../getting-started/installation-and-upgrade/installation-references/feature-flags.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-requirements.md new file mode 100644 index 00000000000..0328552e0e8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/installation-requirements.md @@ -0,0 +1,155 @@ +--- +title: 安装è¦æ±‚ +description: 如果 Rancher é…置在 Docker 或 Kubernetes 中è¿è¡Œæ—¶ï¼Œäº†è§£è¿è¡Œ Rancher Server çš„æ¯ä¸ªèŠ‚点的节点è¦æ±‚ +--- + +本文æ述了对需è¦å®‰è£… Rancher Server 的节点的软件ã€ç¡¬ä»¶å’Œç½‘络è¦æ±‚。Rancher Server å¯ä»¥å®‰è£…在å•ä¸ªèŠ‚点或高å¯ç”¨çš„ Kubernetes 集群上。 + +:::note é‡è¦æ示： + +如果你需è¦åœ¨ Kubernetes 集群上安装 Rancher，该节点的è¦æ±‚与用于è¿è¡Œåº”用和æœåŠ¡çš„[下游集群的节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)ä¸åŒã€‚ + +::: + +Rancher UI 在基于 Firefox 或 Chromium çš„æµè§ˆå™¨ï¼ˆChromeã€Edgeã€Operaã€Brave）中效果最佳。 + +查看我们的[最佳实践](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md)页é¢ï¼ŒèŽ·å–在生产环境中è¿è¡Œ Rancher Server 的建议。 + +## Kubernetes 与 Rancher 的兼容性 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。请查阅 [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)，确ä¿ä½ çš„ Kubernetes 版本å—支æŒã€‚ + +## æ“作系统和容器è¿è¡Œæ—¶è¦æ±‚ + +所有支æŒçš„æ“作系统都使用 64-bit x86 架构。Rancher 兼容当å‰æ‰€æœ‰çš„ä¸»æµ Linux å‘行版。 + +[Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)列出了æ¯ä¸ª Rancher 版本测试过的æ“作系统和 Docker 版本。 + +è¿è¡Œ RKE 集群的节点需è¦å®‰è£… Docker。RKE2 或 K3s 集群ä¸éœ€è¦å®ƒã€‚ + +请安装 `ntp`（Network Time Protocol），以防止在客户端和æœåŠ¡å™¨ä¹‹é—´ç”±äºŽæ—¶é—´ä¸åŒæ­¥é€ æˆçš„è¯ä¹¦éªŒè¯é”™è¯¯ã€‚ + +æŸäº› Linux å‘行版的默认防ç«å¢™è§„则å¯èƒ½ä¼šé˜»æ­¢ Kubernetes 集群内的通信。从 Kubernetes v1.19 开始，你必须关闭 firewalld，因为它与 Kubernetes 网络æ’件冲çªã€‚ + +如果你ä¸å¤ªæƒ³è¿™æ ·åšçš„è¯ï¼Œä½ å¯ä»¥æŸ¥çœ‹[相关问题](https://github.com/rancher/rancher/issues/28840)中的建议。æŸäº›ç”¨æˆ·å·²èƒ½æˆåŠŸ[使用 ACCEPT ç­–ç•¥ 为 Pod CIDR 创建一个独立的 firewalld 区域](https://github.com/rancher/rancher/issues/28840#issuecomment-787404822)。 + +如果你需è¦åœ¨ ARM64 上使用 Rancher，请å‚è§[在 ARM64（实验功能）上è¿è¡Œ Rancher](../how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md)。 + +### RKE è¦æ±‚ + +容器è¿è¡Œæ—¶æ–¹é¢ï¼ŒRKE å¯ä»¥å…¼å®¹å½“å‰çš„所有 Docker 版本。 + +有关详细信æ¯ï¼Œè¯·å‚阅[安装 Docker](../getting-started/installation-and-upgrade/installation-requirements/install-docker.md)。 + +### K3s è¦æ±‚ + +对于容器è¿è¡Œæ—¶ï¼ŒK3s 默认附带了自己的 containerd。你也å¯ä»¥å°† K3s é…置为使用已安装的 Docker è¿è¡Œæ—¶ã€‚有关在 Docker 中使用 K3s 的更多信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://docs.k3s.io/advanced#using-docker-as-the-container-runtime)。 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)。如需指定 K3s 版本，在è¿è¡Œ K3s 安装脚本时，使用 `INSTALL_K3S_VERSION` 环境å˜é‡ã€‚ + +如果你使用 **Raspbian Buster** 在 K3s 集群上安装 Rancher，请按照[这些步骤](https://rancher.com/docs/k3s/latest/en/advanced/#enabling-legacy-iptables-on-raspbian-buster)切æ¢åˆ°æ—§ç‰ˆ iptables。 + +如果你使用 Alpine Linux çš„ K3s 集群上安装 Rancher，请按照[这些步骤](https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-alpine-linux-setup) 进行其他设置。 + +### RKE2 è¦æ±‚ + +对于容器è¿è¡Œæ—¶ï¼ŒRKE2 附带了自己的 containerd。RKE2 安装ä¸éœ€è¦ Docker。 + +如需了解 RKE2 通过了哪些æ“作系统版本的测试，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)。 + +## 硬件è¦æ±‚ + +本节æ述安装 Rancher Server 的节点的 CPUã€å†…存和ç£ç›˜è¦æ±‚。 + +## CPU 和内存 + +硬件è¦æ±‚æ ¹æ®ä½ çš„ Rancher 部署规模而定。请根æ®è¦æ±‚é…ç½®æ¯ä¸ªèŠ‚点。通过å•èŠ‚点容器安装 Rancher，和在 Kubernetes 集群上安装 Rancher çš„è¦æ±‚有所ä¸åŒã€‚ + +### RKE 和托管 Kubernetes + +这些 CPU 和内存è¦æ±‚适用于æ¯ä¸ªå®‰è£… Rancher Server çš„ Kubernetes 集群中的主机。 + +这些è¦æ±‚适用于 RKE Kubernetes 集群以åŠæ‰˜ç®¡çš„ Kubernetes 集群，例如 EKS。 + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | +| --------------- | ---------- | ------------ | -------| ------- | +| å° | 最多 150 个 | 最多 1500 个 | 2 | 8 GB | +| 中 | 最多 300 个 | 最多 3,000 个 | 4 | 16 GB | +| 大 | 最多 500 个 | 最多 5,000 个 | 8 | 32 GB | +| 特大 | 最多 1,000 个 | 最多 10,000 个 | 16 | 64 GB | +| 超大 | 最多 2,000 个 | 最多 20,000 个 | 32 | 128 GB | + +æ¯ä¸ªç”¨ä¾‹å’ŒçŽ¯å¢ƒéƒ½æ˜¯ä¸åŒçš„。请[è”ç³» Rancher](https://rancher.com/contact/) æ¥å®¡æ ¸ä½ çš„情况。 + +### K3s Kubernetes + +这些 CPU 和内存è¦æ±‚适用于æ¯ä¸ª[安装 Rancher Server çš„ Kubernetes 集群](install-upgrade-on-a-kubernetes-cluster.md)中的主机。 + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | æ•°æ®åº“å¤§å° | +| --------------- | ---------- | ------------ | -------| ---------| ------------------------- | +| å° | 最多 150 个 | 最多 1500 个 | 2 | 8 GB | 2 核，4 GB + 1,000 IOPS | +| 中 | 最多 300 个 | 最多 3,000 个 | 4 | 16 GB | 2 核，4 GB + 1,000 IOPS | +| 大 | 最多 500 个 | 最多 5,000 个 | 8 | 32 GB | 2 核，4 GB + 1,000 IOPS | +| 特大 | 最多 1,000 个 | 最多 10,000 个 | 16 | 64 GB | 2 核，4 GB + 1,000 IOPS | +| 超大 | 最多 2,000 个 | 最多 20,000 个 | 32 | 128 GB | 2 核，4 GB + 1,000 IOPS | + +æ¯ä¸ªç”¨ä¾‹å’ŒçŽ¯å¢ƒéƒ½æ˜¯ä¸åŒçš„。请[è”ç³» Rancher](https://rancher.com/contact/) æ¥å®¡æ ¸ä½ çš„情况。 + + +### RKE2 Kubernetes + +这些 CPU 和内存è¦æ±‚适用于安装了 RKE2 çš„æ¯ä¸ªå®žä¾‹ã€‚最低é…ç½®è¦æ±‚如下： + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | +| --------------- | -------- | --------- | ----- | ---- | +| å° | 最多 5 个 | 最多 50 个 | 2 | 5 GB | +| 中 | 最多 15 个 | 最多 200 个 | 3 | 9 GB | + +### Docker + +这些 CPU 和内存è¦æ±‚适用于[å•èŠ‚点](rancher-on-a-single-node-with-docker.md)安装 Rancher 的主机。 + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | +| --------------- | -------- | --------- | ----- | ---- | +| å° | 最多 5 个 | 最多 50 个 | 1 | 4 GB | +| 中 | 最多 15 个 | 最多 200 个 | 2 | 8 GB | + +## Ingress + +安装 Rancher çš„ Kubernetes 集群中的æ¯ä¸ªèŠ‚点都应该è¿è¡Œä¸€ä¸ª Ingress。 + +Ingress 需è¦éƒ¨ç½²ä¸º DaemonSet 以确ä¿è´Ÿè½½å‡è¡¡å™¨èƒ½æˆåŠŸæŠŠæµé‡è½¬å‘到å„个节点。 + +如果是 RKE，RKE2 å’Œ K3s 安装，你ä¸éœ€è¦æ‰‹åŠ¨å®‰è£… Ingress，因为它是默认安装的。 + +对于托管的 Kubernetes 集群（EKSã€GKEã€AKS），你需è¦è®¾ç½® Ingress。 + +- **Amazon EKS**：[在 Amazon EKS 上安装 Rancher 以åŠå¦‚何安装 Ingress 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md)。 +- **AKS**：[使用 Azure Kubernetes æœåŠ¡å®‰è£… Rancher 以åŠå¦‚何安装 Ingress 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md)。 +- **GKE**：[使用 GKE 安装 Rancher 以åŠå¦‚何安装 Ingress 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md)。 + +## ç£ç›˜ + +etcd 在集群中的性能决定了 Rancher 的性能。因此，为了获得最佳速度，我们建议使用 SSD ç£ç›˜æ¥æ”¯æŒ Rancher 管ç†çš„ Kubernetes 集群。在云æ供商上，你还需使用能获得最大 IOPS 的最å°å¤§å°ã€‚在较大的集群中，请考虑使用专用存储设备存储 etcd æ•°æ®å’Œ wal 目录。 + +## 网络è¦æ±‚ + +本节æ述了安装 Rancher Server 的节点的网络è¦æ±‚。 + +:::caution + +å¦‚æžœåŒ…å« Rancher çš„æœåŠ¡å™¨å¸¦æœ‰ `X-Frame-Options=DENY` 标头，在å‡çº§æ—§ç‰ˆ UI 之åŽï¼ŒRancher UI 中的æŸäº›é¡µé¢å¯èƒ½æ— æ³•æ¸²æŸ“。这是因为æŸäº›æ—§ç‰ˆé¡µé¢åœ¨æ–° UI 中是以 iFrames 模å¼åµŒå…¥çš„。 + +::: + +### 节点 IP åœ°å€ + +无论你是在å•ä¸ªèŠ‚点还是高å¯ç”¨é›†ç¾¤ä¸Šå®‰è£… Rancher，æ¯ä¸ªèŠ‚点都应é…置一个é™æ€ IP。如果使用 DHCP，则æ¯ä¸ªèŠ‚点都应该有一个 DHCP 预留，以确ä¿èŠ‚点分é…到相åŒçš„ IP 地å€ã€‚ + +### 端å£è¦æ±‚ + +为了确ä¿èƒ½æ­£å¸¸è¿è¡Œï¼ŒRancher 需è¦åœ¨ Rancher 节点和下游 Kubernetes 集群节点上开放一些端å£ã€‚ä¸åŒé›†ç¾¤ç±»åž‹çš„ Rancher 和下游集群的所有必è¦ç«¯å£ï¼Œè¯·å‚è§[端å£è¦æ±‚](../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md)。 + +## Dockershim æ”¯æŒ + +有关 Dockershim 支æŒçš„详情，请å‚è§[此页é¢](../getting-started/installation-and-upgrade/installation-requirements/dockershim.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/istio-setup-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/istio-setup-guide.md new file mode 100644 index 00000000000..4ed26979729 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/istio-setup-guide.md @@ -0,0 +1,30 @@ +--- +title: è®¾ç½®æŒ‡å— +--- + +本文介ç»å¦‚何å¯ç”¨ Istio 并在你的项目中使用它。 + +如果你使用 Istio 进行æµé‡ç®¡ç†ï¼Œåˆ™éœ€è¦å…许外部æµé‡è¿›å…¥é›†ç¾¤ã€‚在这ç§æƒ…况下，你将需è¦æ‰§è¡Œä»¥ä¸‹æ‰€æœ‰æ­¥éª¤ã€‚ + +## 先决æ¡ä»¶ + +本指å—å‡è®¾ä½ å·²ç»[安装 Rancher](installation-and-upgrade.md)，且已ç»[é…置了一个å•ç‹¬çš„ Kubernetes 集群](kubernetes-clusters-in-rancher-setup.md)并è¦åœ¨è¯¥é›†ç¾¤ä¸Šå®‰è£… Istio。 + +集群中的节点必须满足 [CPU 和内存è¦æ±‚](../integrations-in-rancher/istio/cpu-and-memory-allocations.md)。 + +Istio 控制的工作负载和æœåŠ¡å¿…须满足 [Istio è¦æ±‚](https://istio.io/docs/setup/additional-setup/requirements/)。 + +## 安装 + +:::tip 快速设置æ示： + +如果你ä¸éœ€è¦å¤–部æµé‡åˆ°è¾¾ Istio，而åªæƒ³è®¾ç½® Istio 以监控和跟踪集群内的æµé‡ï¼Œè¯·è·³è¿‡[设置 Istio Gateway](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) å’Œ[设置 Istio çš„æµé‡ç®¡ç†ç»„件](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md)步骤。 + +::: + +1. [在集群中å¯ç”¨ Istio](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md) +1. [在è¦ä½¿ç”¨ Istio 的所有命å空间中å¯ç”¨ Istio](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md) +1. [添加注入了 Istio sidecar 的部署和æœåŠ¡](../how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md) +1. [设置 Istio Gateway](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) +1. [设置 Istio çš„æµé‡ç®¡ç†ç»„件](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md) +1. [生æˆæµé‡å¹¶æŸ¥çœ‹ Istio çš„è¿è¡Œæƒ…况](../how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/istio.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/istio.md new file mode 100644 index 00000000000..a485651a4cb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/istio.md @@ -0,0 +1,137 @@ +--- +title: Istio +--- + +[Istio](https://istio.io/) 是一ç§å¼€æºå·¥å…·ï¼Œå¯ä»¥è®© DevOps 团队更轻æ¾åœ°è§‚察ã€æŽ§åˆ¶ã€æŽ’查并ä¿æŠ¤å¤æ‚çš„å¾®æœåŠ¡ç½‘络中的æµé‡ã€‚ + +éšç€å¾®æœåŠ¡ç½‘络的å˜åŒ–和增长，微æœåŠ¡ç½‘络之间的交互å˜å¾—越æ¥è¶Šéš¾ä»¥ç®¡ç†å’Œç†è§£ã€‚在这ç§æƒ…况下，将æœåŠ¡ç½‘格作为å•ç‹¬çš„基础设施层是éžå¸¸æœ‰ç”¨çš„。Istio çš„æœåŠ¡ç½‘æ ¼å¯ä»¥è®©ä½ åœ¨ä¸ç›´æŽ¥æ›´æ”¹å¾®æœåŠ¡çš„情况下控制微æœåŠ¡ä¹‹é—´çš„æµé‡ã€‚ + +Rancher 与 Istio 集æˆï¼Œä½¿å¾—管ç†å‘˜æˆ–集群所有者å¯ä»¥å°† Istio 交给开å‘者团队，然åŽå¼€å‘者使用 Istio 执行安全策略，排查问题，或为è“绿部署，金ä¸é›€éƒ¨ç½²ï¼Œå’Œ A/B 测试进行æµé‡ç®¡ç†ã€‚ + +此核心æœåŠ¡ç½‘格支æŒä½†ä¸é™äºŽä»¥ä¸‹åŠŸèƒ½ï¼š + +- **管ç†æµé‡**：例如入å£å’Œå‡ºå£è·¯ç”±ã€æ–­è·¯ã€é•œåƒã€‚ +- **安全**：具有用于验è¯å’ŒæŽˆæƒæµé‡å’Œç”¨æˆ·çš„资æºï¼ŒåŒ…括 mTLS。 +- **å¯è§‚察性**：观察日志ã€æŒ‡æ ‡å’Œåˆ†å¸ƒå¼æµé‡ã€‚ + +[设置 Istio](istio-setup-guide.md) åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UIã€`kubectl` 或 ` Istioctl` æ¥ä½¿ç”¨ Istio çš„ controlplane 功能。 + +Istio 需è¦ç”± `cluster-admin` 设置åŽæ‰èƒ½åœ¨é¡¹ç›®ä¸­ä½¿ç”¨ã€‚ + + +## Rancher 2.5 的新功能 + +Istio å·²ç®€åŒ–äº†æ•´ä½“æž¶æž„ã€‚ç»“åˆ Pilotã€Citadelã€Galley å’Œ sidecar injector 创建了一个å•ç‹¬çš„组件 Istiod。Node Agent 功能也已åˆå¹¶åˆ° istio-agent 中。 + +以å‰ç”± Istio 安装的æ’件（cert-managerã€Grafanaã€Jaegerã€Kialiã€Prometheusã€Zipkin）现在需è¦å•ç‹¬å®‰è£…。Istio 支æŒå®‰è£…æ¥è‡ª Istio 项目的集æˆï¼Œå¹¶ä¿æŒä¸Žéž Istio 项目的兼容性。 + +ä½ ä»ç„¶å¯ä»¥é€šè¿‡å®‰è£… [Rancher Monitoring](monitoring-and-alerting.md) 或安装你自己的 Prometheus operator æ¥ä½¿ç”¨ Prometheus 集æˆã€‚Rancher çš„ Istio chart 还默认安装 Kiali，确ä¿ä½ å¯ä»¥å¼€ç®±å³ç”¨åœ°å…¨é¢äº†è§£å¾®æœåŠ¡ã€‚ + +Istio å·²ç»è„±ç¦»äº†ä½¿ç”¨ Helm 安装的方å¼ï¼ŒçŽ°åœ¨é€šè¿‡ Istioctl 二进制文件或 Istio Operator 进行安装。为了使用最简å•çš„æ–¹å¼ä¸Ž Istio 交互，Rancher çš„ Istio 会维护一个 Helm Chart，该 Chart 使用 Istioctl 二进制文件æ¥ç®¡ç†ä½ çš„ Istio 安装。 + +æ­¤ Helm Chart 将在 UI çš„**应用 & 市场市场**中æ供。有æƒè®¿é—® Rancher Chart 应用商店的用户需è¦å…ˆè®¾ç½® Istio，然åŽæ‰èƒ½åœ¨é¡¹ç›®ä¸­ä½¿ç”¨å®ƒã€‚ + +## Istio 附带的工具 + +我们的 [Istio](https://istio.io/) 安装程åºå°† istioctl 二进制命令包装在一个 Helm chart 中，其中包括一个覆盖文件的选项，用æ¥æ”¯æŒå¤æ‚的自定义é…置。 + +它还包括以下内容。 + +### Kiali + +Kiali 是一个全é¢çš„å¯è§†åŒ–辅助工具，用于绘制整个æœåŠ¡ç½‘格中的æµé‡å›¾ã€‚它å…许你查看它们的连接方å¼ï¼ŒåŒ…括它们之间的æµé‡é€ŸçŽ‡å’Œå»¶è¿Ÿã€‚ + +ä½ å¯ä»¥æ£€æŸ¥æœåŠ¡ç½‘格的è¿è¡ŒçŠ¶å†µï¼Œæˆ–深入查看å•ä¸ªç»„件的传入和传出请求。 + +### Jaeger + +Jaeger 是用于跟踪分布å¼ç³»ç»Ÿçš„工具。我们的 Istio 安装程åºåŒ…括能快速å¯åŠ¨çš„一体化 [Jaeger](https://www.jaegertracing.io/) 安装。 + +请注æ„，这ä¸æ˜¯ç¬¦åˆ Jaeger 生产è¦æ±‚的部署。此部署使用在内存中的存储组件，而 Jaeger 推è在生产环境中使用æŒä¹…存储组件。有关你所需的部署策略的更多信æ¯ï¼Œè¯·å‚阅 [Jaeger 文档](https://www.jaegertracing.io/docs/latest/operator/#production-strategy)。 + +## 先决æ¡ä»¶ + +在å¯ç”¨ Istio 之å‰ï¼Œå»ºè®®ä½ å…ˆç¡®è®¤ä½ çš„ Rancher worker 节点是å¦æœ‰è¶³å¤Ÿçš„ [CPU 和内存](../integrations-in-rancher/istio/cpu-and-memory-allocations.md)æ¥è¿è¡Œ Istio 的所有组件。 + +如果è¦åœ¨ RKE2 集群上安装 Istio，则需è¦æ‰§è¡Œä¸€äº›é¢å¤–的步骤。有关详细信æ¯ï¼Œè¯·å‚阅[本节](#在-rke2-集群上安装-istio-的其他步骤)。 + +## è®¾ç½®æŒ‡å— + +如需了解如何设置 Istio 并在项目中使用它，请å‚阅[设置指å—](istio-setup-guide.md)。 + +## å¸è½½ Istio + +è¦ä»Žé›†ç¾¤ã€å‘½å空间或工作负载中删除 Istio 组件，请å‚阅[å¸è½½ Istio](../integrations-in-rancher/istio/disable-istio.md)。 + +## 访问å¯è§†åŒ– + +> 默认情况下，åªæœ‰ cluster-admin å¯ä»¥è®¿é—® Kiali。有关如何å…许具有管ç†å‘˜ã€ç¼–辑或查看æƒé™çš„角色访问它们的说明，请å‚阅[本节](../integrations-in-rancher/istio/rbac-for-istio.md)。 + +在集群中设置 Istio åŽï¼Œä½ å¯ä»¥åœ¨ Rancher UI 中使用 Grafanaã€Prometheus å’Œ Kiali。 + +è¦è®¿é—® Grafana å’Œ Prometheus å¯è§†åŒ–： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Grafana** 或任何其他仪表æ¿ã€‚ + +è¦è®¿é—® Kiali å¯è§†åŒ–： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Kiali 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio**。 +1. å•å‡» **Kiali**。从这里，你å¯ä»¥è®¿é—®**æµé‡å›¾**或**æµé‡æŒ‡æ ‡**选项å¡ï¼Œä»Žè€Œå¯è§†åŒ–网络指标。 + +默认情况下，prometheus 会拾å–所有命å空间，并将数æ®ç”¨äºŽ Kiali 图。如果你想使用ä¸åŒçš„é…置进行 prometheus æ•°æ®æŠ“å–，请å‚阅[选择器/抓å–é…ç½®](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)。 + +你的角色决定了你对å¯è§†åŒ–的访问。åªæœ‰ `cluster-admin` 角色å¯ä»¥ä½¿ç”¨ Grafana å’Œ Prometheus。默认情况下，åªæœ‰ `cluster-admin` å¯ä»¥ä½¿ç”¨ Kiali UI，但是 `cluster-admin` å¯ä»¥é€šè¿‡ç¼–辑 Istio values.yaml æ¥å…许其他角色进行访问。 + +## 架构 + +Istio 安装了一个æœåŠ¡ç½‘格，它使用 [Envoy](https://www.envoyproxy.io) Sidecar 代ç†æ¥æ‹¦æˆªåˆ°æ¯ä¸ªå·¥ä½œè´Ÿè½½çš„æµé‡ã€‚这些 sidecar 拦截并管ç†æœåŠ¡ä¹‹é—´çš„通信，从而实现精细化观察并控制集群内的æµé‡ã€‚ + +åªæœ‰æ³¨å…¥äº† Istio sidecar 的工作负载å¯ä»¥é€šè¿‡ Istio 进行跟踪和控制。 + +如果命å空间å¯ç”¨äº† Istio，部署到命å空间的新工作负载会自动具有 Istio sidecar。你需è¦ä¸ºä¹‹å‰çš„工作负载手动å¯ç”¨ Istio。 + +有关 Istio sidecar 的更多信æ¯ï¼Œè¯·å‚阅 [Istio sidecare-injection 文档](https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/)。有关 Istio 架构的更多信æ¯ï¼Œè¯·å‚阅 [Istio 架构文档](https://istio.io/latest/docs/ops/deployment/architecture/)。 + +### 多个 Ingress + +默认情况下，æ¯ä¸ª Rancher é…置的集群都有一个 NGINX Ingress Controller æ¥å…许æµé‡è¿›å…¥é›†ç¾¤ã€‚Istio 还在 `istio-system` 命å空间中默认安装一个 Ingress Gateway。因此，你的集群将有两个 ingress。 + +![å¯ç”¨ Istio 的集群å¯ä»¥æœ‰ä¸¤ä¸ª ingress，分别是默认的 Nginx ingress 和默认的 Istio controller](/img/istio-ingress.svg) + +å¯ä»¥é€šè¿‡[覆盖文件](configuration-options.md#覆盖文件)æ¥å¯ç”¨å…¶ä»– Istio Ingress Gateway。 + +### Egress æ”¯æŒ + +默认情况下，Egress 网关是ç¦ç”¨çš„，但你å¯ä»¥åœ¨å®‰è£…或å‡çº§æ—¶ä½¿ç”¨ values.yaml 或[覆盖文件](configuration-options.md#覆盖文件)å¯ç”¨å®ƒã€‚ + +## 在 RKE2 集群上安装 Istio 的其他步骤 + +è¦åœ¨ RKE2 集群上安装 Istio，请按照[步骤](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md)进行æ“作。 + +## 在离线环境中å‡çº§ Istio + +现在，Istio Pod 安全策略默认å¯ç”¨ã€‚新值 `installer.releaseMirror.enabled` 已添加到 rancher-istio Chart 中，以å¯ç”¨å’Œç¦ç”¨æ”¯æŒç¦»çº¿å‡çº§çš„ Server。请注æ„，`installer.releaseMirror.enabled` 默认设置为 `false`。你å¯ä»¥åœ¨å®‰è£…或å‡çº§æ—¶æ ¹æ®éœ€è¦è®¾ç½®è¯¥å€¼ã€‚按照以下步骤执行： + +1. 在 Rancher UI 中é…置离线 Rancher 实例和离线自定义集群。 +2. 在集群中安装 Monitoring：**Cluster Explorer > Apps & Marketplace > Charts > Monitoring**。 +3. å°† Istio 所需的所有镜åƒæ‹‰å…¥åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­ä½¿ç”¨çš„ç§æœ‰é•œåƒä»“库。 +4. 在集群中安装 Istio：**Cluster Explorer > Apps & Marketplace > Charts > Istio**。 + +:::note + +ä½ å¯ä»¥åœ¨æ–°å®‰è£…çš„ Istio 上å¯ç”¨ [Jaeger](https://www.jaegertracing.io/) å’Œ [Kiali](https://kiali.io/)ã€‚ä¸ºç¡®ä¿ Jaeger å’Œ Kiali 正常工作，请在安装期间将 `values.yaml` 中的 `installer.releaseMirror.enabled` 设置为 `true`。 + +::: + +5. å‡çº§ Istio。 + +:::caution + +如果你还没有执行æ“作，请设置 `installer.releaseMirror.enabled=true` 以å‡çº§ Istio。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/k3s-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/k3s-hardening-guide.md new file mode 100644 index 00000000000..fc356dfda17 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/k3s-hardening-guide.md @@ -0,0 +1,728 @@ +--- +title: K3s Hardening Guide +--- + +This document provides prescriptive guidance for how to harden a K3s cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for K3s clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 up to v1.25 | + +:::note +At the time of writing, the upstream CIS Kubernetes v1.25 benchmark is not yet available in Rancher. At this time Rancher is using the CIS v1.23 benchmark when scanning Kubernetes v1.25 clusters. +::: + +For more details on how to evaluate a hardened K3s cluster against the official CIS benchmark, refer to the K3s self-assessment guides for specific Kubernetes and CIS benchmark versions. + +K3s passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. K3s does not modify the host operating system. Any host-level modifications need to be done manually. +2. Certain CIS policy controls for `NetworkPolicies` and `PodSecurityStandards` (`PodSecurityPolicies` on v1.24 and older) restrict cluster functionality. + You must opt into having K3s configure these policies. Add the appropriate options to your command-line flags or configuration file (enable admission plugins), and manually apply the appropriate policies. + See further for more details. + +The first section (1.1) of the CIS Benchmark primarily focuses on pod manifest permissions and ownership. Since everything in the distribution is packaged in a single binary, this section does not apply to the core components of K3s. + +## Host-level Requirements + +### Ensure `protect-kernel-defaults` is set + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + protect-kernel-defaults: true +``` + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +This configuration needs to be done before setting the kubelet flag, otherwise K3s will fail to start. + +## Kubernetes Runtime Requirements + +The CIS Benchmark runtime requirements center around pod security (via PSP or PSA), network policies and API Server auditing logs. + +By default, K3s does not include any pod security or network policies. However, K3s ships with a controller that enforces any network policies you create. By default, K3s enables both the `PodSecurity` and `NodeRestriction` admission controllers, among others. + +### Pod Security + + + + +K3s v1.25 and newer support [Pod Security admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) for controlling pod security. + +You can specify the PSA configuration by setting the `defaultPodSecurityAdmissionConfigurationTemplateName` field in the cluster configuration in Rancher: + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted +``` + +The `rancher-restricted` template is provided by Rancher to enforce the highly-restrictive Kubernetes upstream [`Restricted`](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile with best practices for pod hardening. + + + + +K3s v1.24 and older support [Pod Security Policy (PSP)](https://v1-24.docs.kubernetes.io/docs/concepts/security/pod-security-policy/) for controlling pod security. + +You can enable PSPs by passing the following flags in the cluster configuration in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount +``` + +This maintains the `NodeRestriction` plugin and enables the `PodSecurityPolicy`. + +Once you enable PSPs, you can apply a policy to satisfy the necessary controls described in section 5.2 of the CIS Benchmark. + +:::note +These are manual checks in the CIS Benchmark. The CIS scan flags the results as `warning`, because manual inspection is necessary by the cluster operator. +::: + +Here is an example of a compliant PSP: + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false # CIS - 5.2.1 + allowPrivilegeEscalation: false # CIS - 5.2.5 + requiredDropCapabilities: # CIS - 5.2.7/8/9 + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false # CIS - 5.2.4 + hostIPC: false # CIS - 5.2.3 + hostPID: false # CIS - 5.2.2 + runAsUser: + rule: 'MustRunAsNonRoot' # CIS - 5.2.6 + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +``` + +For the example PSP to be effective, we need to create a `ClusterRole` and a `ClusterRoleBinding`. We also need to include a "system unrestricted policy" for system-level pods that require additional privileges, and an additional policy that allows the necessary sysctls for full functionality of ServiceLB. + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: system-unrestricted-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + hostIPC: true + hostNetwork: true + hostPID: true + hostPorts: + - max: 65535 + min: 0 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: svclb-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + allowedUnsafeSysctls: + - net.ipv4.ip_forward + - net.ipv6.conf.all.forwarding + fsGroup: + rule: RunAsAny + hostPorts: + - max: 65535 + min: 0 + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - restricted-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:system-unrestricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svclb-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - svclb-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-local-path-provisioner-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-coredns-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-cis-operator-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-psp +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-unrestricted-node-psp-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-unrestricted-svc-acct-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svclb-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svclb-psp +subjects: +- kind: ServiceAccount + name: svclb +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-local-path-provisioner-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-local-path-provisioner-psp +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-coredns-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-coredns-psp +subjects: +- kind: ServiceAccount + name: coredns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-cis-operator-psp-rolebinding + namespace: cis-operator-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-cis-operator-psp +subjects: +- kind: ServiceAccount + name: cis-operator-serviceaccount +``` + +The policies presented above can be placed in a file named `policy.yaml` in the `/var/lib/rancher/k3s/server/manifests` directory. Both the policy file and the its directory hierarchy must be created before starting K3s. A restrictive access permission is recommended to avoid leaking potential sensitive information. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +:::note +The critical Kubernetes additions such as CNI, DNS, and Ingress are run as pods in the `kube-system` namespace. Therefore, this namespace has a less restrictive policy, so that these components can run properly. +::: + + + + +### Network Policies + +CIS requires that all namespaces apply a network policy that reasonably limits traffic into namespaces and pods. + +:::note +This is a manual check in the CIS Benchmark. The CIS scan flags the result as a `warning`, because manual inspection is necessary by the cluster operator. +::: + +The network policies can be placed in the `policy.yaml` file in `/var/lib/rancher/k3s/server/manifests` directory. If the directory was not created as part of the PSP (as described above), it must be created first. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +Here is an example of a compliant network policy: + +```yaml +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-system +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-system +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: default +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: default +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-public +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-public +``` + +The active restrictions block DNS unless purposely allowed. Below is a network policy that allows DNS-related traffic: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-network-dns-policy + namespace: +spec: + ingress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Ingress +``` + +The metrics-server and Traefik ingress controller are blocked by default if network policies are not created to allow access. + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-metrics-server + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: metrics-server + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-svclbtraefik-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + svccontroller.k3s.cattle.io/svcname: traefik + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-traefik-v121-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ingress: + - {} + policyTypes: + - Ingress +``` + +:::note +You must manage network policies as normal for any additional namespaces you create. +::: + +### API Server audit configuration + +CIS requirements 1.2.22 to 1.2.25 are related to configuring audit logs for the API Server. K3s does not create by default the log directory and audit policy, as auditing requirements are specific to each user's policies and environment. + +If you need a log directory, it must be created before you start K3s. We recommend a restrictive access permission to avoid leaking sensitive information. + +```bash +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/logs +``` + +The following is a starter audit policy to log request metadata. This policy should be written to a file named `audit.yaml` in the `/var/lib/rancher/k3s/server` directory. Detailed information about policy configuration for the API server can be found in the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + +```yaml +--- +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata +``` + +Further configurations are also needed to pass CIS checks. These are not configured by default in K3s, because they vary based on your environment and needs: + +- Ensure that the `--audit-log-path` argument is set. +- Ensure that the `--audit-log-maxage` argument is set to 30 or as appropriate. +- Ensure that the `--audit-log-maxbackup` argument is set to 10 or as appropriate. +- Ensure that the `--audit-log-maxsize` argument is set to 100 or as appropriate. + +Combined, to enable and configure audit logs, add the following lines to the K3s cluster configuration file in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 +``` + +### Controller Manager Requirements + +CIS requirement 1.3.1 checks for garbage collection settings in the Controller Manager. Garbage collection is important to ensure sufficient resource availability and avoid degraded performance and availability. Based on your system resources and tests, choose an appropriate threshold value to activate garbage collection. + +This can be remediated by setting the following configuration in the K3s cluster file in Rancher. The value below is only an example. The appropriate threshold value is specific to each user's environment. + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 +``` + +### Configure `default` Service Account + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. + +For CIS requirement 5.1.5 the `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +This can be remediated by updating the `automountServiceAccountToken` field to `false` for the `default` service account in each namespace. + +For `default` service accounts in the built-in namespaces (`kube-system`, `kube-public`, `kube-node-lease`, and `default)`, K3s does not automatically do this. + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```shell +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Run the script every time a new service account is added to your cluster. + +## Reference Hardened K3s Template Configuration + +The following reference template configuration is used in Rancher to create a hardened K3s custom cluster based on each CIS control in this guide. This reference does not include other required **cluster configuration** directives, which vary based on your environment. + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,ServiceAccount # CIS 1.2.15, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount # CIS 1.2.15, 5.2, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 +``` + + + + +## Conclusion + +If you have followed this guide, your K3s custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our K3s self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-cluster-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-cluster-setup.md new file mode 100644 index 00000000000..67a402ff972 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-cluster-setup.md @@ -0,0 +1,7 @@ +--- +title: "Kubernetes 使用教程 " +--- + +本章节介ç»å¦‚何安装 Kubernetes 集群，使得 Rancher Server å¯ä»¥å®‰è£…在该集群上。 + +Rancher å¯ä»¥åœ¨ä»»ä½• Kubernetes 集群上è¿è¡Œã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md new file mode 100644 index 00000000000..702c3542c7e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md @@ -0,0 +1,76 @@ +--- +title: 在 Rancher 中设置 Kubernetes 集群 +description: é…ç½® Kubernetes 集群 +--- + +Rancher å…许你通过 Rancher UI æ¥åˆ›å»ºé›†ç¾¤ï¼Œä»Žè€Œç®€åŒ–了集群的创建æµç¨‹ã€‚Rancher æ供了多ç§å¯åŠ¨é›†ç¾¤çš„选项。你å¯ä»¥é€‰æ‹©æœ€é€‚åˆä½ çš„用例的选项。 + +本节默认你已对 Docker å’Œ Kubernetes 有一定的了解。如果你需è¦äº†è§£ Kubernetes 组件如何å作，请å‚è§ [Kubernetes 概念](../reference-guides/kubernetes-concepts.md)。 + +有关 Rancher Server é…置集群的方å¼ï¼Œä»¥åŠä½¿ç”¨ä»€ä¹ˆå·¥å…·æ¥åˆ›å»ºé›†ç¾¤çš„详细信æ¯ï¼Œè¯·å‚阅[产å“架构](rancher-manager-architecture.md)。 + + + +### ä¸åŒç±»åž‹é›†ç¾¤çš„管ç†åŠŸèƒ½ + +下表总结了æ¯ä¸€ç§ç±»åž‹çš„集群和对应的å¯ç¼–辑的选项和设置： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## 在托管的 Kubernetes æ供商中设置集群 + +在这ç§æƒ…况下，Rancher ä¸ä¼šé…ç½® Kubernetes，因为它是由 Google Kubernetes Engine (GKE)ã€Amazon Elastic Container Service for Kubernetes 或 Azure Kubernetes Service ç­‰æ供商安装的。 + +如果你使用 Kubernetes æ供商，例如 Google GKE，Rancher 将与对应的云 API 集æˆï¼Œå…许你从 Rancher UI ä¸ºæ‰˜ç®¡é›†ç¾¤åˆ›å»ºå’Œç®¡ç† RBAC。 + +详情请å‚阅[托管 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)。 + +## 使用 Rancher å¯åŠ¨ Kubernetes + +在你自己的节点上é…ç½® Kubernetes 时，Rancher 使用 [Rancher Kubernetes Engine (RKE)](https://rancher.com/docs/rke/latest/en/) 作为库。RKE 是 Rancher 自己的轻é‡çº§ Kubernetes 安装程åºã€‚ + +在 RKE 集群中，Rancher ç®¡ç† Kubernetes 的部署。这些集群å¯ä»¥éƒ¨ç½²åœ¨ä»»ä½•è£¸æœºæœåŠ¡å™¨ã€äº‘æ供商或虚拟化平å°ä¸Šã€‚ + +这些节点å¯ä»¥é€šè¿‡ Rancher çš„ UI 动æ€é…置，该 UI 调用 [Docker Machine](https://docs.docker.com/machine/) 在å„ç§äº‘æ供商上å¯åŠ¨èŠ‚点。 + +如果你已ç»æœ‰ä¸€ä¸ªæƒ³è¦æ·»åŠ åˆ° RKE 集群的节点，你å¯ä»¥é€šè¿‡åœ¨èŠ‚点上è¿è¡Œ Rancher Agent 容器将节点添加到集群中。 + +有关详细信æ¯ï¼Œè¯·å‚阅 [RKE 集群](../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +### 在基础设施æ供商中å¯åŠ¨ Kubernetes 并é…置节点 + +Rancher å¯ä»¥åœ¨ Amazon EC2ã€DigitalOceanã€Azure 或 vSphere 等基础设施æ供商中动æ€é…置节点，然åŽåœ¨èŠ‚点上安装 Kubernetes。 + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)创建节点池。此模æ¿å®šä¹‰äº†è¦åœ¨äº‘æ供商中å¯åŠ¨çš„节点的å‚数。 + +使用由基础设施æ供商托管的节点的一个好处是，如果一个节点与集群失去连接，Rancher å¯ä»¥è‡ªåŠ¨æ›¿æ¢å®ƒï¼Œä»Žè€Œç»´æŠ¤é›†ç¾¤é…置。 + +Rancher UI 中状æ€ä¸º Active çš„[主机驱动](use-new-nodes-in-an-infra-provider.md#主机驱动)决定了å¯ç”¨äºŽåˆ›å»ºèŠ‚点模æ¿çš„云æ供商。 + +如需更多信æ¯ï¼Œè¯·å‚阅[基础设施æ供商托管的节点](use-new-nodes-in-an-infra-provider.md)。 + +### 在现有自定义节点上å¯åŠ¨ Kubernetes + +在设置这ç§ç±»åž‹çš„集群时，Rancher 会在现有的[自定义节点](use-existing-nodes.md)上安装 Kubernetes，从而创建一个自定义集群。 + +ä½ å¯ä»¥ä½¿ç”¨ä»»ä½•èŠ‚点，在 Rancher 中创建一个集群。 + +这些节点包括本地裸机æœåŠ¡å™¨ã€äº‘托管虚拟机或本地虚拟机。 + +## 注册现有集群 + +集群注册功能å–代了导入集群的功能。 + +注册 EKS 集群的优点更多。在大多数情况下，注册的 EKS 集群和在 Rancher 中创建的 EKS 集群在 Rancher UI 中的处ç†æ–¹å¼ç›¸åŒï¼ˆé™¤äº†åˆ é™¤ï¼‰ã€‚ + +删除在 Rancher 中创建的 EKS 集群åŽï¼Œè¯¥é›†ç¾¤å°†è¢«é”€æ¯ã€‚删除在 Rancher 中注册的 EKS 集群时，它与 Rancher Server 会断开连接，但它ä»ç„¶å­˜åœ¨ã€‚ä½ ä»ç„¶å¯ä»¥åƒåœ¨ Rancher 中注册之å‰ä¸€æ ·è®¿é—®å®ƒã€‚ + +详情请å‚è§[本页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)。 + +## 以编程方å¼åˆ›å»ºé›†ç¾¤ + +通过 Rancher 以编程方å¼éƒ¨ç½² Kubernetes 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 + +ä½ å¯ä»¥ä½¿ç”¨ Terraform 创建或导入 EKSã€GKEã€AKS 集群和 RKE 集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-components.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-components.md new file mode 100644 index 00000000000..7deb6057b8a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-components.md @@ -0,0 +1,17 @@ +--- +title: Kubernetes 组件 +--- + +本文列出的命令和步骤适用于 [Rancher å¯åŠ¨çš„ Kubernetes](../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群上的核心 Kubernetes 组件。 + +本文包括以下类别的故障排除æ示： + +- [etcd 节点故障排除](../troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md) +- [Controlplane 节点故障排除](../troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md) +- [nginx-proxy 节点故障排除](../troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md) +- [Worker 节点和通用组件故障排除](../troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md) + +## Kubernetes 组件图 + +![集群图](/img/clusterdiagram.svg)
+线æ¡è¡¨ç¤ºç»„件之间的通信。而颜色纯粹用于视觉辅助。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-resources-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-resources-setup.md new file mode 100644 index 00000000000..146521b2bc5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/kubernetes-resources-setup.md @@ -0,0 +1,63 @@ +--- +title: Kubernetes èµ„æº +--- + +ä½ å¯ä»¥åœ¨ Rancher UI 中查看和æ“作 Kubernetes 集群中的所有自定义资æºå’Œ CRD。 + +## 工作负载 + +使用[工作负载](workloads-and-pods.md)将应用部署到集群节点，工作负载是包å«ç”¨äºŽè¿è¡Œåº”用的 pod 的对象，以åŠä¸ºéƒ¨ç½²è¡Œä¸ºè®¾ç½®è§„则的元数æ®ã€‚工作负载å¯ä»¥éƒ¨ç½²åœ¨é›†ç¾¤èŒƒå›´å†…，也å¯ä»¥éƒ¨ç½²åœ¨ä¸€ä¸ªå‘½å空间内。 + +部署工作负载时，你å¯ä»¥ä½¿ç”¨ä»»ä½•é•œåƒè¿›è¡Œéƒ¨ç½²ã€‚å¯ä¾›é€‰æ‹©çš„[工作负载类型](workloads-and-pods.md#工作负载类型)有多ç§ï¼Œå·¥ä½œè´Ÿè½½ç±»åž‹å†³å®šäº†ä½ çš„应用程åºçš„è¿è¡Œæ–¹å¼ã€‚ + +在工作负载部署之åŽï¼Œä½ å¯ä»¥ç»§ç»­ä½¿ç”¨å®ƒã€‚ä½ å¯ä»¥ï¼š + +- 将工作负载[å‡çº§](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md)到它è¿è¡Œçš„应用的更新版本。 +- 如果å‡çº§å‡ºçŽ°é—®é¢˜ï¼Œå°†å·¥ä½œè´Ÿè½½[回滚](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md)到以å‰çš„版本。 +- [添加一个 sidecar](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md)，这是一个支æŒä¸»è¦å·¥ä½œè´Ÿè½½çš„工作负载。 + +## è´Ÿè½½å‡è¡¡å’Œ Ingress + +### è´Ÿè½½å‡è¡¡å™¨ + +å¯åŠ¨åº”用程åºåŽï¼Œå®ƒä»…在集群中å¯ç”¨ã€‚无法从外部访问它。 + +如果你希望你的应用程åºå¯ä»¥ä»Žå¤–部访问，则必须å‘集群添加负载å‡è¡¡å™¨ã€‚如果用户知é“è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€å’Œåº”用的端å£å·ï¼Œè´Ÿè½½å‡è¡¡å™¨å¯ä»¥ä¸ºå¤–部连接创建一个访问集群的网关。 + +Rancher 支æŒä¸¤ç§ç±»åž‹çš„è´Ÿè½½å‡è¡¡å™¨ï¼š + +- [Layer-4 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#四层负载å‡è¡¡å™¨) +- [Layer-7 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#七层负载å‡è¡¡å™¨) + +有关详细信æ¯ï¼Œè¯·å‚阅[è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md)。 + +#### Ingress + +è´Ÿè½½å‡è¡¡å™¨åªèƒ½å¤„ç†æ¯ä¸ª service 的一个 IP 地å€ã€‚æ¢è¨€ä¹‹ï¼Œå¦‚果你在集群中è¿è¡Œäº†å¤šä¸ª service，则必须为æ¯ä¸ª service é…备一个负载å‡è¡¡å™¨ã€‚è¿è¡Œå¤šä¸ªè´Ÿè½½å‡è¡¡å™¨çš„花费å¯èƒ½éžå¸¸é«˜æ˜‚。因此，你å¯ä»¥ä½¿ç”¨ Ingress æ¥è§£å†³æ­¤é—®é¢˜ã€‚ + +Ingress 是一组充当负载å‡è¡¡å™¨çš„规则。Ingress 与一个或多个 Ingress Controller 一起动æ€è·¯ç”± service 的请求。Ingress 收到请求时，集群中的 Ingress Controller 会对负载å‡è¡¡å™¨è¿›è¡Œé…置，从而根æ®ä½ é…置的 service å­åŸŸæˆ–路径规则将请求定å‘到正确的 service。 + +有关详细信æ¯ï¼Œè¯·å‚阅 [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md)。 + +在项目中使用 Ingress 时，你å¯ä»¥é€šè¿‡è®¾ç½®å…¨å±€ DNS æ¡ç›®æ¥å°† Ingress 主机å编程到外部 DNS。 + +## æœåŠ¡å‘现 + +使用负载å‡è¡¡å™¨å’Œ/或 Ingress 将集群公开给外部请求åŽï¼Œä½ åªèƒ½é€šè¿‡ IP 地å€è®¿é—®é›†ç¾¤ã€‚è¦åˆ›å»ºå¯è§£æžçš„主机å，你必须创建æœåŠ¡è®°å½•ï¼Œè¯¥è®°å½•å°† IP 地å€ã€å¤–部主机åã€DNS 记录别åã€å·¥ä½œè´Ÿè½½æˆ–标记的 pod 映射到特定主机å。 + +有关详细信æ¯ï¼Œè¯·å‚阅[æœåŠ¡å‘现](../how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md)。 + +## åº”ç”¨ç¨‹åº + +除了å¯åŠ¨åº”用程åºçš„å„个组件外，你还å¯ä»¥ä½¿ç”¨ Rancher 应用商店æ¥å¯åŠ¨åº”ç”¨ï¼Œå³ Helm Chart。 + +## Kubernetes èµ„æº + +在 Rancher 项目或命å空间的上下文中，_资æº_ æ˜¯æ”¯æŒ Pod æ“作的文件和数æ®ã€‚在 Rancher 中，è¯ä¹¦ã€é•œåƒä»“库和密文都被视为资æºã€‚但是，Kubernetes 将资æºåˆ’分为ä¸åŒç±»åž‹çš„[密文（secret）](https://kubernetes.io/docs/concepts/configuration/secret/)。因此，在å•ä¸ªé¡¹ç›®æˆ–命å空间中，å„个资æºå¿…须具有唯一的å称以é¿å…冲çªã€‚资æºä¸»è¦ç”¨äºŽæ‰¿è½½æ•æ„Ÿä¿¡æ¯ï¼Œä½†ä¹Ÿæœ‰å…¶ä»–用途。 + +资æºåŒ…括： + +- [è¯ä¹¦](../how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md)：用于加密/解密进入或离开集群的数æ®çš„文件。 +- [ConfigMap](../how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md)：存储一般é…置信æ¯çš„文件，例如一组é…置文件。 +- [密文](../how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md)：存储密ç ã€token 或密钥等æ•æ„Ÿæ•°æ®çš„文件。 +- [é•œåƒä»“库](../how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md)：æºå¸¦ç”¨äºŽéªŒè¯ç§æœ‰é•œåƒä»“库的凭è¯çš„文件。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/launch-kubernetes-with-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/launch-kubernetes-with-rancher.md new file mode 100644 index 00000000000..966e88886a4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/launch-kubernetes-with-rancher.md @@ -0,0 +1,80 @@ +--- +title: 使用 Rancher å¯åŠ¨ Kubernetes +--- + +Rancher å¯ä»¥ä½¿ç”¨ä»»æ„节点å¯åŠ¨ Kubernetes 集群。在 Rancher 中将 Kubernetes 部署到这些节点上时，你å¯ä»¥é€‰æ‹© [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) 或 [RKE2](https://docs.rke2.io) å‘行版。Rancher å¯ä»¥åœ¨ä»»ä½•è®¡ç®—机上å¯åŠ¨ Kubernetes，包括： + +- 裸金属æœåŠ¡å™¨ +- 本地虚拟机 +- 由云厂商托管的虚拟机 + +Rancher å¯ä»¥åœ¨çŽ°æœ‰èŠ‚点上安装 Kubernetes，也å¯ä»¥åœ¨äº‘厂商中动æ€é…置节点并安装 Kubernetes。 + +Rancher 还å¯ä»¥åˆ›å»ºèŠ‚点池。在托管在云厂商的节点池上安装 Kubernetes 的一个好处是，如果一个节点与集群断开连接，Rancher å¯ä»¥è‡ªåŠ¨åˆ›å»ºå¦ä¸€ä¸ªèŠ‚点并将其加入集群，从而确ä¿èŠ‚点池的数é‡ç¬¦åˆè¦æ±‚。 + +## RKE + +### è¦æ±‚ + +如果你使用 RKE 建立集群，节点必须满足下游集群的[节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +### 在云厂商的新节点上å¯åŠ¨ Kubernetes + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)创建节点池。此节点模æ¿å®šä¹‰äº†è¦ç”¨äºŽåœ¨äº‘厂商中å¯åŠ¨èŠ‚点的å‚数。 + +在托管在云厂商的节点池上安装 Kubernetes 的一个好处是，如果一个节点与集群断开连接，Rancher å¯ä»¥è‡ªåŠ¨åˆ›å»ºå¦ä¸€ä¸ªèŠ‚点并将其加入集群，从而确ä¿èŠ‚点池的数é‡ç¬¦åˆè¦æ±‚。 + +有关详细信æ¯ï¼Œè¯·å‚阅[在新节点上å¯åŠ¨ Kubernetes](use-new-nodes-in-an-infra-provider.md)。 + +### 在现有自定义节点上å¯åŠ¨ Kubernetes + +在这ç§æƒ…况下，你希望将 Kubernetes 安装到裸机æœåŠ¡å™¨ã€æœ¬åœ°è™šæ‹Ÿæœºæˆ–云厂商中已存在的虚拟机上。使用此选项，你将在主机上è¿è¡Œ Rancher Agent Docker 容器。 + +如果è¦é‡å¤ä½¿ç”¨ä¹‹å‰çš„自定义集群中的节点，请在å¤ç”¨ä¹‹å‰[清ç†èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md)。如果你é‡å¤ä½¿ç”¨å°šæœªæ¸…ç†çš„节点，则集群é…ç½®å¯èƒ½ä¼šå¤±è´¥ã€‚ + +有关详细信æ¯ï¼Œè¯·å‚阅[自定义节点](use-existing-nodes.md)。 + +### 以编程方å¼åˆ›å»º RKE 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² RKE 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 + +## RKE2 + +Rancher 2.6 支æŒç›´æŽ¥ä½¿ç”¨ Rancher UI é…ç½® [RKE2](https://docs.rke2.io/) 集群。RKE2，也称为 RKE Government，是一个完全符åˆæ ‡å‡†çš„ Kubernetes å‘行版，它专注于安全性和åˆè§„性。在 Rancher 2.6.4 åŠæ›´æ—©ç‰ˆæœ¬ä¸­ï¼ŒRKE2 é…置还处于技术预览阶段。 + +在 Rancher 2.6.5 中，RKE2 å·²ç» GA。 + +### è¦æ±‚ + +如果你使用 RKE2 建立集群，节点必须满足下游集群的[节点è¦æ±‚](https://docs.rke2.io/install/requirements)。 + +### 在云厂商的新节点上å¯åŠ¨ Kubernetes + +RKE2 基于使用上游[集群 API](https://github.com/kubernetes-sigs/cluster-api) 项目的新é…置框架。这个新é…置框架支æŒï¼š + +- å°† RKE2 集群é…置到 Rancher 具有主机驱动的任何æ供商上 +- 完全在 Rancher 中é…ç½® RKE2 集群 +- 除了 Canal 之外，还å¯ä»¥é€‰æ‹© CNI 选项， Calicoã€Cilium å’Œ Multus + +RKE2 é…置还包括在具有 Windows 节点的集群上安装 RKE2。 + +RKE2 çš„ Windows 功能包括： + +- Windows æ”¯æŒ vSphere 主机驱动 +- 用于 Windows RKE2 自定义集群的 Calico CNI +- Calico 的项目网络隔离 (PNI) +- ç”± containerd æ供支æŒçš„使用 RKE2 çš„ Windows 容器 +- 通过 Terraform é…ç½® Windows RKE2 集群 +- 直接从 Rancher UI é…ç½® Windows RKE2 自定义集群 + +è¦ä½¿ Windows æ”¯æŒ RKE2 自定义集群，请选择 Calico 作为 CNI。 + +### 在现有自定义节点上å¯åŠ¨ Kubernetes + +RKE2 还支æŒåœ¨é¢„é…置的虚拟机或裸机节点上安装自定义集群。 + +如果è¦é‡å¤ä½¿ç”¨ä¹‹å‰çš„自定义集群中的节点，请在å¤ç”¨ä¹‹å‰æ¸…ç†èŠ‚点。如果你é‡å¤ä½¿ç”¨å°šæœªæ¸…ç†çš„节点，则集群é…ç½®å¯èƒ½ä¼šå¤±è´¥ã€‚ + +### 以编程方å¼åˆ›å»º RKE2 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² RKE2 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster_v2)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/load-balancer-and-ingress-controller.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/load-balancer-and-ingress-controller.md new file mode 100644 index 00000000000..e88352bc8b3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/load-balancer-and-ingress-controller.md @@ -0,0 +1,61 @@ +--- +title: 在 Rancher 中设置负载å‡è¡¡å™¨å’Œ Ingress Controller +description: 了解如何设置负载å‡è¡¡å™¨å’Œ Ingress Controller 以在 Rancher 中é‡å®šå‘æœåŠ¡è¯·æ±‚，并了解负载å‡è¡¡å™¨çš„é™åˆ¶ +--- + +在 Rancher 中，你å¯ä»¥é€šè¿‡è®¾ç½®è´Ÿè½½å‡è¡¡å™¨å’Œ Ingress Controller æ¥é‡å®šå‘æœåŠ¡è¯·æ±‚。 + +## è´Ÿè½½å‡è¡¡å™¨ + +å¯åŠ¨åº”用程åºåŽï¼Œè¯¥åº”用程åºä»…在集群内å¯ç”¨ã€‚你无法从集群外部访问它。 + +如果你希望从外部访问应用程åºï¼Œåˆ™å¿…é¡»å‘集群添加负载å‡è¡¡å™¨æˆ– Ingress。如果用户知é“è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€å’Œåº”用的端å£å·ï¼Œè´Ÿè½½å‡è¡¡å™¨å¯ä»¥ä¸ºå¤–部连接创建一个访问集群的网关。 + +Rancher 支æŒä¸¤ç§ç±»åž‹çš„è´Ÿè½½å‡è¡¡å™¨ï¼š + +- [Layer-4 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#四层负载å‡è¡¡å™¨) +- [Layer-7 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#七层负载å‡è¡¡å™¨) + +有关详细信æ¯ï¼Œè¯·å‚阅[è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md)。 + +### è´Ÿè½½å‡è¡¡å™¨é™åˆ¶ + +è´Ÿè½½å‡è¡¡å™¨æœ‰å‡ ä¸ªéœ€è¦æ³¨æ„çš„é™åˆ¶ï¼š + +- è´Ÿè½½å‡è¡¡å™¨åªèƒ½å¤„ç†æ¯ä¸ª service 的一个 IP 地å€ã€‚æ¢è¨€ä¹‹ï¼Œå¦‚果你在集群中è¿è¡Œäº†å¤šä¸ª service，则必须为æ¯ä¸ª service é…备一个负载å‡è¡¡å™¨ã€‚è¿è¡Œå¤šä¸ªè´Ÿè½½å‡è¡¡å™¨çš„花费å¯èƒ½éžå¸¸é«˜æ˜‚。 + +- 如果你想将负载å‡è¡¡å™¨ä¸Žæ‰˜ç®¡çš„ Kubernetes 集群（å³æ‰˜ç®¡åœ¨ GKEã€EKS 或 AKS 中的集群）一起使用，则负载å‡è¡¡å™¨å¿…é¡»è¿è¡Œåœ¨è¯¥äº‘æ供商的基础设施上。请根æ®ä½ é…置集群的方å¼æŸ¥çœ‹è´Ÿè½½å‡è¡¡å™¨çš„兼容列表： + +- [æ”¯æŒ Layer-4 è´Ÿè½½å‡è¡¡](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#四层负载å‡è¡¡æ”¯æŒ) + +- [æ”¯æŒ Layer-7 è´Ÿè½½å‡è¡¡](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#七层负载å‡è¡¡æ”¯æŒ) + +## Ingress + +如上所述，使用负载å‡è¡¡å™¨çš„缺点是： + +- æ¯ä¸ªæœåŠ¡è´Ÿè½½å‡è¡¡å™¨åªèƒ½å¤„ç†ä¸€ä¸ª IP 地å€ã€‚ +- 如果你在集群中è¿è¡Œå¤šä¸ªæœåŠ¡ï¼Œåˆ™æ¯ä¸ªæœåŠ¡éƒ½å¿…é¡»é…备一个负载å‡è¡¡å™¨ã€‚ +- 为æ¯ä¸ªæœåŠ¡é…备负载å‡è¡¡å™¨çš„花费å¯èƒ½éžå¸¸é«˜æ˜‚。 + +相å，如果将æŸä¸ª Ingress 用作集群的入å£ç‚¹ï¼ŒIngress å¯ä»¥æ›´çµæ´»åœ°å°†æµé‡è·¯ç”±åˆ°å¤šä¸ª service。它å¯ä»¥å°†å¤šä¸ª HTTP 请求映射到 service，而无需为æ¯ä¸ª service æä¾›å•ç‹¬çš„ IP 地å€ã€‚ + +因此，如果你需è¦ä½¿ç”¨ç›¸åŒçš„ IP 地å€ã€Layer 7 å议或特æƒèŠ‚点端å£ï¼ˆ80 å’Œ 443）æ¥å…¬å¼€å¤šä¸ª service，你å¯ä»¥ä½¿ç”¨ä¸€ä¸ª Ingress。 + +Ingress 与一个或多个 Ingress Controller 一起动æ€è·¯ç”± service 的请求。Ingress 收到请求时，集群中的 Ingress Controller 会根æ®ä½ é…置的 service å­åŸŸæˆ–路径规则将请求定å‘到正确的 service。 + +æ¯ä¸ª Kubernetes Ingress 资æºéƒ½å¯¹åº”一个 `/etc/nginx/sites-available/` 中的文件，其中包å«ä¸€ä¸ªé…置对特定文件和文件夹的请求的 `server{}` é…ç½®å—。 + +Ingress 能为你的集群创建一个入å£ç«¯å£ï¼ˆä¸Žè´Ÿè½½å‡è¡¡å™¨ç±»ä¼¼ï¼‰ï¼Œå¯ä»¥ä½äºŽé›†ç¾¤çš„内部或外部。RKE å¯åŠ¨çš„集群中的 Ingress å’Œ Ingress Controller ç”± [Nginx](https://www.nginx.com/) æ供支æŒã€‚ + +Ingress 还支æŒå…¶ä»–功能，例如 SSL 终止ã€åŸºäºŽå称的虚拟主机等。 + +:::note 在高å¯ç”¨æ€§é…置中使用 Rancher： + +请é¿å…å°† Ingress 添加到 `local` 集群。Rancher å°† Nginx Ingress Controller 作为 Rancher 管ç†çš„ _所有_ 集群的全局入å£ç‚¹ï¼Œå…¶ä¸­åŒ…括 `local` 集群。因此，当用户å°è¯•è®¿é—®åº”用程åºæ—¶ï¼ŒRancher å¯èƒ½ä¼šç”±äºŽé‡æ–°åŠ è½½ Nginx é…置而断开连接。è¦è§£å†³è¿™ä¸ªé—®é¢˜ï¼Œæˆ‘们建议你仅在通过 Rancher å¯åŠ¨çš„集群中部署应用程åºã€‚ + +::: + +- 有关如何在 Rancher 中设置 Ingress 的更多信æ¯ï¼Œè¯·å‚阅 [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md)。 +- 有关 Ingress å’Œ Ingress Controller 的完整信æ¯ï¼Œè¯·å‚阅 [Kubernetes Ingress 文档](https://kubernetes.io/docs/concepts/services-networking/ingress/)。 +- 在项目中使用 Ingress 时，你å¯ä»¥é€šè¿‡è®¾ç½®å…¨å±€ DNS æ¡ç›®æ¥å°† Ingress 主机å编程到外部 DNS。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/logging.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/logging.md new file mode 100644 index 00000000000..a6f81844170 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/logging.md @@ -0,0 +1,117 @@ +--- +title: Rancher Logging é›†æˆ +description: Rancher 集æˆäº†ä¸»æµçš„日志æœåŠ¡ã€‚了解集æˆæ—¥å¿—æœåŠ¡çš„è¦æ±‚和优势，并在你的集群上å¯ç”¨ Logging。 +--- + +现在，Rancher 的日志管ç†ç”± [Logging operator](https://kube-logging.github.io/docs/) æ供支æŒï¼Œå®ƒå–代了以å‰çš„内部解决方案。 + +## å¯ç”¨ Logging + +ä½ å¯ä»¥è½¬åˆ°**应用**页é¢å¹¶å®‰è£… Logging 应用程åºï¼Œä»Žè€Œä¸º Rancher 管ç†çš„集群å¯ç”¨ Logging： + +1. 转到è¦å®‰è£… Logging 的集群，然åŽå•å‡» **Apps**。 +1. 点击 **Logging** 应用。 +1. 滚动到 Helm Chart README 的底部，然åŽå•å‡»**安装**。 + +**结果**：Logging 应用已部署到 `cattle-logging-system` 命å空间中。 + +## å¸è½½ Logging + +1. 转到è¦å®‰è£… Logging 的集群，然åŽå•å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `cattle-logging-system` 命å空间并选中 `rancher-logging` å’Œ `rancher-logging-crd` 框。 +1. å•å‡»**删除**。 +1. 确认**删除**。 + +**结果**：已å¸è½½ `rancher-logging`。 + +## 架构 + +有关 Logging 应用程åºå·¥ä½œåŽŸç†çš„更多信æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-architecture.md)。 + + + +## RBAC + +Rancher Logging 有两个角色，分别是 `logging-admin` å’Œ `logging-view`。有关如何以åŠä½•æ—¶ä½¿ç”¨è¿™äº›è§’色的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/logging/rbac-for-logging.md)。 + +## é…ç½® Logging è‡ªå®šä¹‰èµ„æº + +è¦ç®¡ç† `Flows`ã€`ClusterFlows`ã€`Outputs` å’Œ `ClusterOutputs`： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦é…ç½® Logging 自定义资æºçš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Logging**。 + +### Flows å’Œ ClusterFlows + +有关é…ç½® `Flows` å’Œ `ClusterFlows` 的帮助，请å‚阅[此页é¢](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md)。 + +### Outputs å’Œ ClusterOutputs + +有关é…ç½® `Outputs` å’Œ `ClusterOutputs` 的帮助，请å‚阅[此页é¢](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md)。 + +## é…ç½® Logging Helm Chart + +有关在安装或å‡çº§ Logging 应用程åºæ—¶å¯é…置的选项，请å‚阅[此页é¢](../integrations-in-rancher/logging/logging-helm-chart-options.md)。 + +### Windows æ”¯æŒ + +ä½ å¯ä»¥ä»Ž Windows 节点[å¯ç”¨ Logging](../integrations-in-rancher/logging/logging-helm-chart-options.md#å¯ç”¨ç¦ç”¨-windows-节点-logging)。 + + +### 使用自定义 Docker 根目录 + +有关使用自定义 Docker 根目录的详细信æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-helm-chart-options.md#使用自定义-docker-根目录)。 + + +### 处ç†æ±¡ç‚¹å’Œå®¹å¿åº¦ + +有关如何在 Logging 应用程åºä¸­ä½¿ç”¨æ±¡ç‚¹å’Œå®¹å¿åº¦çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/logging/taints-and-tolerations.md)。 + + +### 在 SELinux 上使用 Logging V2 + +有关在å¯ç”¨äº† SELinux 的节点上使用 Logging 应用程åºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-helm-chart-options.md#å¯ç”¨-logging-应用程åºä»¥ä½¿ç”¨-selinux)。 + +### 其他日志æ¥æº + +默认情况下，Rancher 会收集所有类型集群的 controlplane 组件和节点组件的日志。在æŸäº›æƒ…况下，也会收集其他日志。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-helm-chart-options.md#其他日志æ¥æº)。 + + +## 故障排除 + +### 日志缓冲区导致 Pod 过载 + +æ ¹æ®ä½ çš„é…置，默认缓冲区大å°å¯èƒ½å¤ªå¤§å¹¶å¯¼è‡´ Pod 故障。å‡å°‘负载的一ç§æ–¹æ³•æ˜¯é™ä½Žè®°å½•å™¨çš„刷新间隔。这å¯ä»¥é˜²æ­¢æ—¥å¿—溢出缓冲区。你还å¯ä»¥æ·»åŠ æ›´å¤šåˆ·æ–°çº¿ç¨‹æ¥å¤„ç†å¤§é‡æ—¥å¿—试图åŒæ—¶å¡«å……缓冲区的情况。 + +有关如何é…置日志缓冲区æ¥æ»¡è¶³ä¼ä¸šéœ€æ±‚的更完整说明，请å‚阅[缓冲区](https://kube-logging.github.io/docs/configuration/plugins/outputs/buffer/)å’Œ [Fluentd é…ç½®](https://kube-logging.github.io/docs/logging-infrastructure/fluentd/)的官方 Logging Operator 文档。 + +### `cattle-logging` 命å空间正在é‡æ–°åˆ›å»º + +如果你的集群之å‰åœ¨æ—§ç‰ˆ Rancher UI 的全局视图中部署了 Logging，`cattle-logging` 命å空间å¯èƒ½ä¼šä¸æ–­è¢«é‡æ–°åˆ›å»ºã€‚ + +è¦è§£å†³è¿™ä¸ªé—®é¢˜ï¼Œä½ å¯ä»¥å°†æ‰€æœ‰ `clusterloggings.management.cattle.io` å’Œ `projectloggings.management.cattle.io` 自定义资æºä»Žç®¡ç†é›†ç¾¤ä¸­é’ˆå¯¹è¯¥é›†ç¾¤çš„命å空间中删除。 +这些自定义资æºä¼šå¯¼è‡´ Rancher 在下游集群中创建 `cattle-logging` 命å空间（如果ä¸å­˜åœ¨ï¼‰ã€‚ + +集群命å空间与集群 ID 匹é…，因此我们需è¦æ‰¾åˆ°æ¯ä¸ªé›†ç¾¤çš„集群 ID。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦èŽ·å– ID 的集群，然åŽå•å‡» **Explore**。 +2. 从以下其中一个 URL 中å¤åˆ¶ `` 的内容。`` 是集群命å空间å称。 + +```bash +# Cluster Management UI +https:///c// + +# Cluster Dashboard +https:///dashboard/c// +``` + +现在我们有了 `` 命å空间，我们å¯ä»¥åˆ é™¤å¯¼è‡´ `cattle-logging` ä¸æ–­é‡æ–°åˆ›å»ºçš„自定义资æºã€‚ +*警告*：请当å‰æœªä½¿ç”¨ç¡®ä¿ Logging（从旧版 Rancher UI 全局视图中安装的版本）。 + +```bash +kubectl delete crd clusterloggings.management.cattle.io -n +kubectl delete crd projectloggings.management.cattle.io -n +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/machine-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/machine-configuration.md new file mode 100644 index 00000000000..f7878f968b7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/machine-configuration.md @@ -0,0 +1,5 @@ +--- +title: 主机é…ç½® +--- + +主机é…置指的是如何将资æºåˆ†é…给虚拟机。请å‚阅 [Amazon EC2](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md)ã€[DigitalOcean](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md) å’Œ [Azure](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) 的文档以了解更多信æ¯ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-clusters.md new file mode 100644 index 00000000000..9e327a6138b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-clusters.md @@ -0,0 +1,32 @@ +--- +title: é›†ç¾¤ç®¡ç† +--- + +在 Rancher 中é…置集群åŽï¼Œä½ å¯ä»¥å¼€å§‹ä½¿ç”¨å¼ºå¤§çš„ Kubernetes 功能在开å‘ã€æµ‹è¯•æˆ–生产环境中部署和扩展容器化应用。 + +:::note + +本节默认你已对 Docker å’Œ Kubernetes 有一定的了解。如果你需è¦äº†è§£ Kubernetes 组件如何å作，请å‚è§ [Kubernetes 概念](../reference-guides/kubernetes-concepts.md)。 + +::: + +## 在 Rancher 中管ç†é›†ç¾¤ + +将集群[é…置到 Rancher](kubernetes-clusters-in-rancher-setup.md) 之åŽï¼Œ[集群所有者](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)需è¦ç®¡ç†è¿™äº›é›†ç¾¤ã€‚管ç†é›†ç¾¤çš„选项如下： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## é…置工具 + +Rancher åŒ…å« Kubernetes 中未包å«çš„å„ç§å·¥å…·æ¥å助你进行 DevOps æ“作。Rancher å¯ä»¥ä¸Žå¤–部æœåŠ¡é›†æˆï¼Œè®©ä½ çš„集群更高效地è¿è¡Œã€‚工具分为以下几类： + +- å‘Šè­¦ +- Notifiers +- Logging +- Monitoring +- Istio æœåŠ¡ç½‘æ ¼ +- OPA Gatekeeper + +ä½ å¯ä»¥é€šè¿‡ **Apps** æ¥å®‰è£…工具。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-project-resource-quotas.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-project-resource-quotas.md new file mode 100644 index 00000000000..89c8a760f5e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-project-resource-quotas.md @@ -0,0 +1,46 @@ +--- +title: 项目资æºé…é¢ +--- + +如果多个团队共享一个集群，æŸä¸ªå›¢é˜Ÿå¯èƒ½ä¼šä½¿ç”¨è¿‡å¤šçš„å¯ç”¨èµ„æºï¼Œä¾‹å¦‚ CPUã€å†…å­˜ã€å­˜å‚¨ã€æœåŠ¡ã€Kubernetes 对象（如 Pod 或 Secret）等。你å¯ä»¥åº”用 _资æºé…é¢_ æ¥é˜²æ­¢è¿‡åº¦æ¶ˆè€—资æºã€‚资æºé…é¢æ˜¯ Rancher 用æ¥é™åˆ¶é¡¹ç›®æˆ–命å空间å¯ç”¨èµ„æºçš„功能。 + +本文介ç»å¦‚何在现有项目中创建资æºé…é¢ã€‚ + +你也å¯ä»¥åœ¨åˆ›å»ºæ–°é¡¹ç›®æ—¶è®¾ç½®èµ„æºé…é¢ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[创建新项目](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md#创建项目)。 + +Rancher 中的资æºé…é¢åŒ…å«ä¸Ž [Kubernetes 原生版本](https://kubernetes.io/docs/concepts/policy/resource-quotas/)相åŒçš„功能。Rancher 还扩展了资æºé…é¢çš„功能，从而让你将资æºé…é¢åº”用于项目。有关资æºé…é¢å¦‚何与 Rancher 中的项目一起使用的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md)。 + +### 将资æºé…é¢åº”用于现有项目 + +修改资æºé…é¢çš„使用场景如下： + +- é™åˆ¶æŸä¸ªé¡¹ç›®å’Œé¡¹ç›®ä¸‹çš„命åç©ºé—´èƒ½ä½¿ç”¨çš„èµ„æº +- 在资æºé…é¢å·²ç”Ÿæ•ˆçš„情况下，对项目å¯ç”¨çš„资æºè¿›è¡Œæ‰©å®¹æˆ–缩容 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œè¿›å…¥è¦åº”用资æºé…é¢çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. ç¡®ä¿ **Projects/Namespaces** 页é¢å¤„于 **Group by Project** 视图模å¼ã€‚ + ![Screenshot highlighting the "Group by Project" icon, above the list of projects. It resembles a folder.](/img/edit-project-config-for-resource-quotas-group-by-project.png) + +1. 找到è¦æ·»åŠ èµ„æºé…é¢çš„项目，选择与项目å称åŒè¡Œçš„ **â‹®**。 + ![Screenshot highlighting triple dots icon at the end of the same row as the project name.](/img/edit-project-config-for-resource-quotas-dots.png) + +1. 选择**编辑é…ç½®**。 + +1. 展开**资æºé™é¢**并å•å‡»**添加资æº**。你也å¯ä»¥ç¼–辑现有é…é¢ã€‚ + +1. 选择资æºç±»åž‹ã€‚有关类型的更多信æ¯ï¼Œè¯·å‚阅[é…é¢ç±»åž‹å‚考](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md)。 + +1. 输入**项目é™åˆ¶**å’Œ**命å空间默认é™åˆ¶**的值。 + + | 字段 | æè¿° | + | ----------------------- | -------------------------------------------------------------------------------------------------------- | + | 项目é™åˆ¶ | 项目的总资æºé™åˆ¶ã€‚ | + | 命å空间默认é™åˆ¶ | æ¯ä¸ªå‘½å空间的默认资æºé™åˆ¶ã€‚æ­¤é™åˆ¶ä¼šæ²¿ç”¨åˆ°é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间。项目中所有命å空间的é™åˆ¶ä¹‹å’Œä¸åº”超过项目é™åˆ¶ã€‚ | + +1. **å¯é€‰**：添加更多é…é¢ã€‚ + +1. å•å‡»**创建**。 + +**结果**：资æºé…é¢å·²åº”用到你的项目和命å空间。如果你åŽç»­éœ€è¦æ·»åŠ æ›´å¤šå‘½å空间，Rancher 会验è¯é¡¹ç›®æ˜¯å¦å¯ä»¥å®¹çº³è¯¥å‘½å空间。如果项目无法分é…资æºï¼Œä½ ä»ç„¶å¯ä»¥åˆ›å»ºå‘½å空间，但命å空间将获得的资æºé…é¢ä¸º 0ã€‚ç„¶åŽ Rancher å°†ä¸å…许你创建任何å—æ­¤é…é¢é™åˆ¶çš„资æºã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-projects.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-projects.md new file mode 100644 index 00000000000..341878779ef --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-projects.md @@ -0,0 +1,37 @@ +--- +title: é¡¹ç›®ç®¡ç† +--- + +_项目_ 是 Rancher 中引入的对象，å¯å¸®åŠ©ä½ æ›´æœ‰ç»„ç»‡åœ°ç®¡ç† Kubernetes 集群中的命å空间。你å¯ä»¥ä½¿ç”¨é¡¹ç›®åˆ›å»ºå¤šç§Ÿæˆ·é›†ç¾¤ï¼Œè¿™ç§é›†ç¾¤å…许一组用户共享相åŒçš„底层资æºæ¥åˆ›å»ºåº”用，而应用之间ä¸ä¼šç›¸äº’å½±å“。 + +在层次结构方é¢ï¼š + +- 集群包å«é¡¹ç›® +- 项目包å«å‘½å空间 + +在 Rancher 中，你å¯ä»¥ä½¿ç”¨é¡¹ç›®å°†å¤šä¸ªå‘½å空间作为一个实体进行管ç†ã€‚在原生 Kubernetes（没有项目这个概念）中，RBAC 或集群资æºç­‰åŠŸèƒ½è¢«åˆ†é…给了å„个命å空间。如果集群中的多个命å空间需è¦åˆ†é…åŒæ ·çš„访问æƒé™ï¼Œåˆ†é…æƒé™ä¼šå˜å¾—éžå¸¸ç¹ç。å³ä½¿æ‰€æœ‰å‘½å空间都需è¦ç›¸åŒçš„æƒé™ï¼Œä½†ä¹Ÿæ— æ³•ä½¿ç”¨ä¸€ä¸ªæ“作中将这些æƒé™åº”用于所有命å空间。你必须é‡å¤åœ°å°†è¿™äº›æƒé™åˆ†é…ç»™æ¯ä¸ªå‘½å空间。 + +而 Rancher 通过引入项目的概念，通过å…许你在项目级别应用资æºå’Œè®¿é—®æƒé™ã€‚然åŽï¼Œé¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间都会继承这些资æºå’Œç­–略。因此你åªéœ€å°†èµ„æºå’Œç­–略分é…给项目å³å¯ï¼Œä¸éœ€è¦å°†å®ƒä»¬åˆ†é…ç»™æ¯ä¸ªå•ç‹¬çš„命å空间。 + +ä½ å¯ä»¥ä½¿ç”¨é¡¹ç›®æ‰§è¡Œä»¥ä¸‹æ“作： + +- [为用户分é…一组命å空间的访问æƒé™](../how-to-guides/new-user-guides/add-users-to-projects.md) +- 为用户分é…[项目中的特定角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)。角色å¯ä»¥æ˜¯æ‰€æœ‰è€…ã€æˆå‘˜ã€åªè¯»æˆ–[自定义](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md) +- [设置资æºé…é¢](manage-project-resource-quotas.md) +- [管ç†å‘½å空间](../how-to-guides/new-user-guides/manage-namespaces.md) +- [é…置工具](../reference-guides/rancher-project-tools.md) +- [é…ç½® Pod 安全策略](../how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md) + +### æŽˆæƒ + +éžç®¡ç†è€…用户åªæœ‰åœ¨[管ç†å‘˜](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)ã€[集群所有者或æˆå‘˜](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)或[项目所有者](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)å°†éžç®¡ç†å‘˜ç”¨æˆ·æ·»åŠ åˆ°é¡¹ç›®çš„**æˆå‘˜**选项å¡åŽï¼Œæ‰èƒ½èŽ·å–项目的访问æƒé™ã€‚ + +创建项目的人自动æˆä¸º[项目所有者](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)。 + +## åœ¨é¡¹ç›®ä¹‹é—´åˆ‡æ¢ + +è¦åœ¨é¡¹ç›®ä¹‹é—´åˆ‡æ¢ï¼Œè¯·ä½¿ç”¨å¯¼èˆªæ ä¸­çš„下拉èœå•ã€‚你也å¯ä»¥ç›´æŽ¥åœ¨å¯¼èˆªæ ä¸­åˆ‡æ¢é¡¹ç›®ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œè¿›å…¥è¦åˆ‡æ¢é¡¹ç›®çš„集群然åŽç‚¹å‡» **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œé€‰æ‹©è¦æ‰“开的项目。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-role-based-access-control-rbac.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-role-based-access-control-rbac.md new file mode 100644 index 00000000000..a4cf2425ecb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/manage-role-based-access-control-rbac.md @@ -0,0 +1,25 @@ +--- +title: RBAC +--- + +Rancher 通过 _用户_ 进行授æƒç®¡ç†ã€‚如[身份验è¯](authentication-config.md)中所述，用户å¯ä»¥æ˜¯æœ¬åœ°ç”¨æˆ·ï¼Œä¹Ÿå¯ä»¥æ˜¯å¤–部用户。 + +é…置外部身份验è¯åŽï¼Œ**用户**页é¢ä¸Šæ˜¾ç¤ºçš„用户会å‘生å˜åŒ–。 + +- 如果你以本地用户身份登录，则仅显示本地用户。 + +- 如果你以外部用户身份登录，则会åŒæ—¶æ˜¾ç¤ºå¤–部用户和本地用户。 + +## 用户和角色 + +一旦用户登录到 Rancher，他们的 _授æƒ_，也就是他们在系统中的访问æƒé™ï¼Œå°†ç”± _全局æƒé™_ å’Œ _集群和项目角色_ 决定。 + +- [全局æƒé™](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)： + + 定义用户在任何特定集群之外的授æƒã€‚ + +- [集群和项目角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)： + + 定义用户在分é…了角色的特定集群或项目中的授æƒã€‚ + +全局æƒé™ä»¥åŠé›†ç¾¤å’Œé¡¹ç›®è§’色都是基于 [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) 实现的。因此，æƒé™å’Œè§’色的底层实现是由 Kubernetes 完æˆçš„。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-alerting-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-alerting-guides.md new file mode 100644 index 00000000000..7769d685ae4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-alerting-guides.md @@ -0,0 +1,10 @@ +--- +title: Monitoring æŒ‡å— +--- + +- [å¯ç”¨ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [å¸è½½ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring 工作负载](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [自定义 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [æŒä¹…化 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [调试高内存使用率](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-and-alerting.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-and-alerting.md new file mode 100644 index 00000000000..84d29ee6ac0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-and-alerting.md @@ -0,0 +1,95 @@ +--- +title: 监控和告警 +description: Prometheus å…许你查看æ¥è‡ªä¸åŒ Rancher å’Œ Kubernetes 对象的指标。了解监控范围以åŠå¦‚何å¯ç”¨é›†ç¾¤ç›‘控 +--- + +`rancher-monitoring` 应用å¯ä»¥å¿«é€Ÿå°†é¢†å…ˆçš„å¼€æºç›‘控和告警解决方案部署到你的集群上。 + +该应用程åºåœ¨ Rancher v2.5 中引入，由 [Prometheus](https://prometheus.io/)ã€[Grafana](https://grafana.com/grafana/)ã€[Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)ã€[Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) å’Œ [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) æ供支æŒã€‚ + +有关 Rancher v2.2 至 v2.4 中 V1 monitoring å’Œ alerting çš„ä¿¡æ¯ï¼Œè¯·å‚阅有关[集群监控](/versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-monitoring.md)ã€[å‘Šè­¦](/versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-alerts.md)ã€[notifiers](/versioned_docs/version-2.0-2.4/explanations/integrations-in-rancher/notifiers.md) 和其他[工具](/versioned_docs/version-2.0-2.4/pages-for-subheaders/project-tools.md)çš„ Rancher v2.0-v2.4 文档。 + +ä½ å¯ä»¥ä½¿ç”¨ `rancher-monitoring` 应用，将业界领先的开æºç›‘控和告警解决方案快速部署到你的集群中。 + +### 功能 + +Prometheus 支æŒæŸ¥çœ‹ Rancher å’Œ Kubernetes 对象的指标。通过使用时间戳，Prometheus 能让你通过 Rancher UI 或 Grafana（与 Prometheus 一起部署的分æžæŸ¥çœ‹å¹³å°ï¼‰ä»¥æ›´å®¹æ˜“阅读的图表和视觉形å¼æ¥æŸ¥è¯¢å’ŒæŸ¥çœ‹è¿™äº›æŒ‡æ ‡ã€‚ + +通过查看 Prometheus 从集群的 controlplaneã€èŠ‚点和 deployment 中抓å–çš„æ•°æ®ï¼Œä½ å¯ä»¥éšæ—¶äº†è§£é›†ç¾¤ä¸­å‘生的所有事件。然åŽï¼Œä½ å¯ä»¥ä½¿ç”¨è¿™äº›åˆ†æžæ¥æ›´å¥½åœ°è¿è¡Œä½ çš„环境，例如在系统紧急情况å‘生之å‰é˜»æ­¢å®ƒä»¬ã€åˆ¶å®šç»´æŠ¤ç­–略，或æ¢å¤å´©æºƒçš„æœåŠ¡å™¨ã€‚ + +Monitoring 应用： + +- 监控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚ +- æ ¹æ® Prometheus 收集的指标定义告警。 +- 创建自定义 Grafana 仪表æ¿ã€‚ +- 使用 Prometheus Alertmanager 通过电å­é‚®ä»¶ã€Slackã€PagerDuty ç­‰é…置告警通知。 +- æ ¹æ® Prometheus 收集的指标，将预先计算的ã€ç»å¸¸éœ€è¦çš„，或计算æˆæœ¬é«˜çš„表达å¼å®šä¹‰ä¸ºæ–°çš„时间åºåˆ—。 +- 通过 Prometheus Adapter，将从 Prometheus 收集的指标公开给 Kubernetes Custom Metrics API，以便在 HPA 中使用。 + +有关监控组件如何ååŒå·¥ä½œçš„说明，请å‚阅 [Monitoring 工作原ç†](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +## 默认组件和部署 + +### å†…ç½®ä»ªè¡¨æ¿ + +默认情况下，监控应用将 Grafana 仪表æ¿ï¼ˆç”± [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) 项目策划）部署到集群上。 + +它害部署一个 Alertmanager UI 和一个 Prometheus UI。有关这些工具的更多信æ¯ï¼Œè¯·å‚è§[内置仪表æ¿](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md)。 +### 默认指标 Exporter + +默认情况下，Rancher Monitoring 会部署 Exporter（例如 [node-exporter](https://github.com/prometheus/node_exporter) å’Œ [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics)）。 + +这些默认 Exporter 会自动从 Kubernetes é›†ç¾¤çš„æ‰€æœ‰ç»„ä»¶ï¼ˆåŒ…æ‹¬å·¥ä½œè´Ÿè½½ï¼‰ä¸­æŠ“å– CPU 和内存的指标。 + +### 默认告警 + +Monitoring 应用会默认部署一些告警。è¦æŸ¥çœ‹é»˜è®¤å‘Šè­¦ï¼Œè¯·è½¬åˆ° [Alertmanager UI](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md#alertmanager-ui) 并å•å‡»**展开所有组**。 + +### Rancher UI 中公开的组件 + +有关 Rancher UI 中公开的监控组件列表，以åŠç¼–辑它们的常è§ç”¨ä¾‹ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#rancher-ui-中公开的组件)。 + +## RBAC + +有关é…ç½® monitoring 访问æƒé™çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md)。 + +## æŒ‡å— + +- [å¯ç”¨ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [å¸è½½ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring 工作负载](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [自定义 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [æŒä¹…化 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [调试高内存使用率](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) + +## é…ç½® + +### 在 Rancher 中é…ç½® Monitoring èµ„æº + +此处的é…ç½®å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚如需更多信æ¯ï¼Œè¯·å‚阅 [monitoring 的工作原ç†](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +- [ServiceMonitor å’Œ PodMonitor](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [接收器](../reference-guides/monitoring-v2-configuration/receivers.md) +- [路由](../reference-guides/monitoring-v2-configuration/routes.md) +- [PrometheusRule](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) +- [Prometheus](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) +- [Alertmanager](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### é…ç½® Helm Chart 选项 + +有关 `rancher-monitoring` Chart 选项的更多信æ¯ï¼ŒåŒ…括设置资æºé™åˆ¶å’Œè¯·æ±‚的选项，请å‚阅 [Helm Chart 选项](../reference-guides/monitoring-v2-configuration/helm-chart-options.md)。 + +## Windows é›†ç¾¤æ”¯æŒ + +如果 Monitoring 部署到 RKE1 Windows 集群，Monitoring V2 将自动部署 [windows-exporter](https://github.com/prometheus-community/windows_exporter) DaemonSet 并设置 ServiceMonitor，以从æ¯ä¸ªéƒ¨ç½²çš„ Pod 中收集指标。这将使用 `windows_` 指标填充 Prometheus，这些指标与 [node_exporter](https://github.com/prometheus/node_exporter) 为 Linux 主机导出的 `node_` 指标类似。 + +为了能够为 Windows 完全部署 Monitoring V2，你的所有 Windows 主机都必须至少具有 v0.1.0 çš„ [wins](https://github.com/rancher/wins) 版本。 + +有关如何在现有 Windows 主机上å‡çº§ wins 版本的更多信æ¯ï¼Œè¯·å‚阅 [Windows 集群对 Monitoring V2 的支æŒ](../integrations-in-rancher/monitoring-and-alerting/windows-support.md)。 + + +## 已知问题 + +有一个[已知问题](https://github.com/rancher/rancher/issues/28787#issuecomment-693611821)ï¼Œå³ K3s 集群需è¦çš„内存超过分é…的默认内存。如果你在 K3s 集群上å¯ç”¨ Monitoring，将 `prometheus.prometheusSpec.resources.memory.limit` 设置为 2500 Mi，并将 `prometheus.prometheusSpec.resources.memory.request` 设置为 1750 Mi。 + +如需获å–æ„è§å’Œå»ºè®®ï¼Œè¯·å‚阅[调试高内存使用情况](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration-guides.md new file mode 100644 index 00000000000..cdc63d55f53 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration-guides.md @@ -0,0 +1,51 @@ +--- +title: é…ç½® +--- + +本文介ç»åœ¨ Rancher UI 中é…ç½® Monitoring V2 的一些最é‡è¦é€‰é¡¹ã€‚ + +有关为 Prometheus é…置自定义抓å–目标和规则的信æ¯ï¼Œè¯·å‚阅 [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 的上游文档。Prometheus Operator [设计文档](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md)中解释了一些最é‡è¦çš„自定义资æºã€‚Prometheus Operator 文档还å¯ä»¥å¸®åŠ©ä½ è®¾ç½® RBACã€Thanos 或进行自定义é…置。 + +## 设置资æºé™åˆ¶å’Œè¯·æ±‚ + +安装 `rancher-monitoring` æ—¶å¯ä»¥é…ç½® Monitoring 应用的资æºè¯·æ±‚å’Œé™åˆ¶ã€‚有关默认é™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/helm-chart-options.md#é…置资æºé™åˆ¶å’Œè¯·æ±‚)。 + +:::tip + +在空闲集群上，Monitoring å¯èƒ½ä¼šå ç”¨å¾ˆå¤š CPU 资æºã€‚è¦æ高性能，请关闭 Prometheus Adapter。 + +::: + +## Prometheus é…ç½® + +通常ä¸éœ€è¦ç›´æŽ¥ç¼–辑 Prometheus 自定义资æºã€‚ + +相å，è¦è®© Prometheus 抓å–自定义指标，你åªéœ€åˆ›å»ºä¸€ä¸ªæ–°çš„ ServiceMonitor 或 PodMonitor æ¥å°† Prometheus é…置为抓å–其他指标。 + + +### ServiceMonitor å’Œ PodMonitor é…ç½® + +有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md)。 + +### 高级 Prometheus é…ç½® + +有关直接编辑 Prometheus 自定义资æºï¼ˆå¯¹é«˜çº§ç”¨ä¾‹å¯èƒ½æœ‰å¸®åŠ©ï¼‰çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md)。 + +## Alertmanager é…ç½® + +Alertmanager 自定义资æºé€šå¸¸ä¸éœ€è¦ç›´æŽ¥ç¼–辑。在常è§ç”¨ä¾‹ä¸­ï¼Œä½ å¯ä»¥é€šè¿‡æ›´æ–°è·¯ç”±å’ŒæŽ¥æ”¶å™¨æ¥ç®¡ç†å‘Šè­¦ã€‚ + +路由和接收器是 Alertmanager 自定义资æºé…置的一部分。在 Rancher UI 中，路由（Route）和接收器（Receiver）并ä¸æ˜¯çœŸæ­£çš„自定义资æºï¼Œè€Œæ˜¯ Prometheus Operator 用æ¥å°†ä½ çš„é…置与 Alertmanager 自定义资æºåŒæ­¥çš„伪自定义资æºã€‚当路由和接收器更新时，Monitoring 应用将自动更新 Alertmanager æ¥å映这些更改。 + +对于一些高级用例，你å¯èƒ½éœ€è¦ç›´æŽ¥é…ç½® Alertmanager。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 + +### 接收器 + +接收器（Receiver）用于设置通知。有关如何é…置接收器的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/receivers.md)。 +### 路由 + +路由（Route）在通知到达接收器之å‰è¿‡æ»¤å®ƒä»¬ã€‚æ¯æ¡è·¯ç”±éƒ½éœ€è¦å¼•ç”¨ä¸€ä¸ªå·²ç»é…置好的接收器。有关如何é…置路由的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/routes.md)。 + +### 高级é…ç½® + +有关直接编辑 Alertmanager 自定义资æºï¼ˆå¯¹é«˜çº§ç”¨ä¾‹å¯èƒ½æœ‰å¸®åŠ©ï¼‰çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration.md new file mode 100644 index 00000000000..6cd47ac5d10 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration.md @@ -0,0 +1,11 @@ +--- +title: Monitoring V2 é…ç½® +--- + +本文介ç»äº†åœ¨ Rancher 中é…ç½® Monitoring V2 çš„å¿…è¦é€‰é¡¹ï¼š + +- [接收器é…ç½®](../reference-guides/monitoring-v2-configuration/receivers.md) +- [路由é…ç½®](../reference-guides/monitoring-v2-configuration/routes.md) +- [ServiceMonitor å’Œ PodMonitor é…ç½®](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [Helm Chart 选项](../reference-guides/monitoring-v2-configuration/helm-chart-options.md) +- [示例](../reference-guides/monitoring-v2-configuration/examples.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/new-user-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/new-user-guides.md new file mode 100644 index 00000000000..7e46d5b69a8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/new-user-guides.md @@ -0,0 +1,9 @@ +--- +title: æ–°ç”¨æˆ·æŒ‡å— +--- + +新用户指å—（也称为**教程**）æ述了æŸäº›æ“作的实际步骤。这些文档是“学习导å‘â€çš„，也就是说用户通过“æ“作â€æ¥å­¦ä¹ ã€‚ + +新用户指å—旨在引导åˆå­¦è€…或 Rancher 的日常用户通过一系列步骤æ¥å­¦ä¹ å¦‚何进行æŸäº›æ“作。这些文档旨在帮助用户通过使用易于éµå¾ªã€æœ‰æ„义且å¯é‡å¤çš„æ“作æ¥äº†è§£å¦‚何完æˆä»»åŠ¡ã€‚这些指å—将帮助用户完æˆå·¥ä½œï¼Œå¹¶èƒ½ç«‹åˆ»çœ‹åˆ°æ•ˆæžœã€‚ + +正常æ¥è¯´ï¼Œæ™®é€š Rancher 用户的技术水平高于“åˆå­¦è€…â€ï¼Œä½†æ˜¯ï¼Œæˆ‘们的新用户指å—为åˆå­¦è€…以åŠç»éªŒä¸°å¯Œçš„ Rancher 用户æ供相åŒçš„指导。我们结åˆä½¿ç”¨äº†é«˜çº§è¯­è¨€å’ŒæŠ€æœ¯è¯­è¨€æ¥ä»‹ç»å„ä¸ªä¸»é¢˜ï¼Œå¹¶æŒ‡å¯¼ç”¨æˆ·å®Œæˆ Rancher 用户需è¦äº†è§£çš„通用任务。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/node-template-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/node-template-configuration.md new file mode 100644 index 00000000000..489dbbafa13 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/node-template-configuration.md @@ -0,0 +1,5 @@ +--- +title: 节点模æ¿é…ç½® +--- + +è¦äº†è§£èŠ‚点模æ¿é…置，请å‚阅[EC2 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md)ã€[DigitalOcean 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md)ã€[Azure 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md)ã€[vSphere 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)å’Œ [Nutanix 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/nutanix.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/nutanix.md new file mode 100644 index 00000000000..a3e28a1182e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/nutanix.md @@ -0,0 +1,17 @@ +--- +title: 创建 Nutanix AOS 集群 +description: 使用 Rancher 创建 Nutanix AOS (AHV) 集群。集群å¯èƒ½åŒ…括具有ä¸åŒå±žæ€§çš„ VM 组，这些属性å¯ç”¨äºŽç»†ç²’度控制节点的大å°ã€‚ +--- + +[Nutanix Acropolis æ“作系统](https://www.nutanix.com/products/acropolis) (Nutanix AOS) 是适用于 Nutanix 超èžåˆåŸºç¡€æž¶æž„å¹³å°çš„æ“作系统。AOS 带有一个å为 [Acropolis Hypervisor（AHV）](https://www.nutanix.com/products/ahv)的内置虚拟机监控程åºã€‚ä½ å¯ä»¥ç»“åˆä½¿ç”¨ Rancher 与 Nutanix AOS (AHV)，从而在本地体验云环境的æ“作。 + +Rancher å¯ä»¥åœ¨ AOS (AHV) 中é…置节点并在其上安装 Kubernetes。在 AOS 中创建 Kubernetes 集群时，Rancher 首先与 Prism Central API 通信æ¥é…置指定数é‡çš„虚拟机。然åŽåœ¨è™šæ‹Ÿæœºä¸Šå®‰è£… Kubernetes。 + +Nutanix 集群å¯èƒ½ç”±å¤šç»„具有ä¸åŒå±žæ€§ï¼ˆä¾‹å¦‚内存或 vCPU æ•°é‡ï¼‰çš„ VM 组æˆã€‚è¿™ç§åˆ†ç»„å…许对æ¯ä¸ª Kubernetes 角色的节点大å°è¿›è¡Œç»†ç²’度控制。 + +- [创建 Nutanix 集群](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md#创建-nutanix-aos-集群) +- [é…置存储](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md) + +## 创建 Nutanix 集群 + +在[本节](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md)中，你将学习如何使用 Rancher 在 Nutanix AOS 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/other-installation-methods.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/other-installation-methods.md new file mode 100644 index 00000000000..f56c0cbb179 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/other-installation-methods.md @@ -0,0 +1,19 @@ +--- +title: å…¶ä»–å®‰è£…æ–¹å¼ +--- + +### 离线安装 + +按照[以下步骤](air-gapped-helm-cli-install.md)在离线环境中安装 Rancher Server。 + +离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +### Docker 安装 + +[å•èŠ‚点 Docker 安装](rancher-on-a-single-node-with-docker.md)适用于想è¦æµ‹è¯• Rancher 的用户。你无需使用 Helm 在 Kubernetes 集群上è¿è¡Œ Rancher，你å¯ä»¥ä½¿ç”¨ `docker run` 命令，把 Rancher Server 组件安装到å•ä¸ªèŠ‚点上。 + +Docker 安装仅用于开å‘和测试环境。 + +由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此，如果该节点å‘生故障，由于其他节点上没有å¯ç”¨çš„ etcd æ•°æ®å‰¯æœ¬ï¼Œä½ å°†ä¸¢å¤± Rancher Server 的所有数æ®ã€‚ + +Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/prometheus-federator-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/prometheus-federator-guides.md new file mode 100644 index 00000000000..742fd54b6fe --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/prometheus-federator-guides.md @@ -0,0 +1,8 @@ +--- +title: Prometheus Federator æŒ‡å— +--- + +- [å¯ç”¨ Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md) +- [å¸è½½ Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md) +- [自定义 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md) +- [设置工作负载](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/prometheus-federator.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/prometheus-federator.md new file mode 100644 index 00000000000..f78b8d9543f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/prometheus-federator.md @@ -0,0 +1,104 @@ +--- +title: Prometheus Federator +--- + +Prometheus Federator（也称为 Project Monitoring V2）基于 [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator) 部署一个 Helm Project Operator。该 Operator ç®¡ç† Helm Chart 的部署，æ¯ä¸ª Operator 都包å«ä¸€ä¸ª Project Monitoring Stack，而æ¯ä¸ªå †æ ˆéƒ½åŒ…å«ï¼š + +- [Prometheus](https://prometheus.io/)（由 [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 在外部管ç†ï¼‰ +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)（由 [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 在外部管ç†ï¼‰ +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana)ï¼ˆé€šè¿‡åµŒå…¥å¼ Helm Chart 部署） +- 基于 [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) 社区策划资æºé›†åˆçš„默认 PrometheusRules å’Œ Grafana ä»ªè¡¨æ¿ +- 监视已部署资æºçš„默认 ServiceMonitor + +:::note é‡è¦æ示： + +Prometheus Federator 适åˆåœ¨å·²å®‰è£… Prometheus Operator CRD 的集群中与现有的 Prometheus Operator Deployment 一起部署。 + +::: + +## Operator å·¥ä½œåŽŸç† + +1. 在部署此 Chart 时，用户å¯ä»¥åˆ›å»º ProjectHelmCharts CR，并在**项目 Registration 命å空间 (`cattle-project-`)** 中将 `spec.helmApiVersion` 设置为 `monitoring.cattle.io/v1alpha1`（在 Rancher UI 中也称为“项目监控â€ï¼‰ã€‚ +2. 在看到æ¯ä¸ª ProjectHelmChartCR 时，Operator 会代表项目所有者在**项目 Release 命å空间 (`cattle-project--monitoring`)** 中自动部署一个 Project Prometheus 堆栈（基于 ProjectHelmChart 控制器在 **Operator / System Namespace** 中创建的 HelmChart CR å’Œ HelmRelease CR）。 +3. RBAC 将自动分é…到项目 Release 命å空间中，从而å…许用户查看 Prometheusã€Alertmanager 以åŠå·²éƒ¨ç½²çš„ Project Monitoring Stack çš„ Grafana UI（基于在项目 Registration 命å空间上针对[é¢å‘用户的默认 Kubernetes 角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)定义的 RBAC）。有关详细信æ¯ï¼Œè¯·å‚阅[é…ç½® RBAC](../reference-guides/prometheus-federator/rbac.md)。 + +### 什么是项目？ + +在 Prometheus Federator 中，项目是一组å¯ä»¥ç”± `metav1.LabelSelector` 标识的命å空间。默认情况下，用于标识项目的标签是 `field.cattle.io/projectId`，该标签用于标识给定 Rancher 项目中包å«çš„命å空间。 + +### é…置由 ProjectHelmChart 创建的 Helm 版本 + +æ­¤ ProjectHelmChart 资æºçš„ `spec.values` 对应于为底层 Helm Chart é…置的 `values.yaml` 覆盖，该 Helm Chart 是 Operator 代表用户部署的。è¦æŸ¥çœ‹åº•å±‚ Chart çš„ `values.yaml` 规范，你å¯ä»¥é€‰æ‹©ä»¥ä¸‹å…¶ä¸­ä¸€ç§æ–¹å¼ï¼š + +- 查看ä½äºŽ [`charts/rancher-project-monitoring` 中的 `rancher/prometheus-federator`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) çš„ Chart 定义（Chart 版本会绑定到 Operator 版本）。 +- 查找在æ¯ä¸ªé¡¹ç›® Registration 命å空间中自动创建的å为 `monitoring.cattle.io.v1alpha1` çš„ ConfigMap，其中包å«ç”¨äºŽé…ç½® Chart（直接嵌入到 `prometheus-federator` 二进制文件中）的`values.yaml` å’Œ `questions.yaml`。 + +### 命å空间 + +Prometheus Federator 是基于 [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator) çš„ Project Operator，Prometheus Federator æ供了三类命å空间供 Operator 查找： + +1. **Operator / System 命å空间**：部署 Operator 的命å空间（例如 `cattle-monitoring-system`）。此命å空间将包å«è¯¥ Operator 监视的所有 ProjectHelmChart 的所有 HelmChart å’Œ HelmRelease。**åªæœ‰é›†ç¾¤ç®¡ç†å‘˜æ‰èƒ½è®¿é—®æ­¤å‘½å空间**。 + +2. **项目 Registration 命å空间 (`cattle-project-`)**：Operator 在这些命å空间中监视 ProjectHelmChart。对于在项目å‘布命å空​​间中创建的自动分é…çš„ RBAC，应用于此命å空间的 RoleBinding å’Œ ClusterRoleBinding 也会作为 RBAC 的真实æ¥æºã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [RBAC 页é¢](../reference-guides/prometheus-federator/rbac.md)。**项目所有者（admin）ã€é¡¹ç›®æˆå‘˜ï¼ˆedit）和åªè¯»æˆå‘˜ï¼ˆview）应该有æƒè®¿é—®æ­¤å‘½å空间。** + + :::note 注æ„事项： + + - 如果æ供了 `.Values.global.cattle.projectLabel`（默认设置为 `field.cattle.io/projectId`），则 Operator 会自动生æˆé¡¹ç›®æ³¨å†Œå‘½å空间，并将它导入到命å空间绑定的项目中。æ¢è¨€ä¹‹ï¼Œå¦‚果观察到至少一个带有该标签的命å空间，则 Operator 会创建一个项目 Registration 命å空间。除éžå‡ºçŽ°ä»¥ä¸‹ä¸¤ç§æƒ…况，å¦åˆ™ Operator ä¸ä¼šè®©è¿™äº›å‘½å空间被删除。第一ç§æƒ…况是带有该标签的所有命å空间都消失了（例如，这是该项目中的最åŽä¸€ä¸ªå‘½å空间，在这ç§æƒ…况下，命å空间将标有标签 `"helm.cattle.io/helm-project-operator-orphaned": "true"`，表示å¯ä»¥åˆ é™¤ï¼‰ã€‚第二ç§æƒ…况是由于项目 ID 是在 `.Values.helmProjectOperator.otherSystemProjectLabelValues` 下æ供的（用作项目的拒ç»åå•ï¼‰ï¼Œå¯¼è‡´ Operator ä¸å†ç›‘视该项目。这些命å空间ä¸ä¼šè¢«è‡ªåŠ¨åˆ é™¤ï¼Œè¿™æ ·èƒ½é¿å…ç ´å用户数æ®ã€‚如果需è¦ï¼Œå»ºè®®ç”¨æˆ·åœ¨åˆ›å»ºæˆ–删除项目时手动清ç†è¿™äº›å‘½å空间。 + + - 如果未æä¾› `.Values.global.cattle.projectLabel`，则 Operator / System 命å空间也是项目注册命å空间。 + + ::: + +3. **项目å‘布命å空​​间（`cattle-project--monitoring`）**：Operator 代表 ProjectHelmChart 在其中部署项目监控堆栈的命å空间集。Operator 还将根æ®åœ¨é¡¹ç›® Registration 命å空间中找到的绑定，自动为项目监控堆栈在此命åç©ºé—´ä¸­åˆ›å»ºçš„è§’è‰²åˆ†é… RBAC。**åªæœ‰é›†ç¾¤ç®¡ç†å‘˜æ‰èƒ½è®¿é—®è¿™ä¸ªå‘½å空间。部署的 Helm Chart å’Œ Prometheus Federator 将为项目所有者（admin）ã€é¡¹ç›®æˆå‘˜ï¼ˆedit）和åªè¯»æˆå‘˜ï¼ˆview）分é…该命å空间的有é™è®¿é—®æƒé™ã€‚** + + :::note 注æ„事项： + + - 项目å‘布命å空间会自动部署并导入到 ID 在 `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` 下指定的项目中，如果未指定，且项目注册命å空间中指定了 ProjectHelmChart，则默认为 `.Values.global.cattle.systemProjectId` 的值。 + + - 项目å‘布命å空​​间的孤立约定与项目注册命å空间的相åŒï¼ˆå‚è§ä¸Šé¢çš„注释）。 + + - 如果 `.Values.projectReleaseNamespaces.enabled` 为 false，则项目å‘布命å空​​间与项目注册命å空间是相åŒçš„。 + + ::: + +### Helm 资æºï¼ˆHelmChartã€HelmRelease） + +在部署 ProjectHelmChart 时，Prometheus Federator 将自动创建和管ç†ä¸¤ä¸ªå­è‡ªå®šä¹‰èµ„æºï¼Œå®ƒä»¬ä¾æ¬¡ç®¡ç†ä»¥ä¸‹åº•å±‚ Helm 资æºï¼š + +- HelmChart CR（通过 Operator ä¸­çš„åµŒå…¥å¼ [k3s-io​​/helm-contoller](https://github.com/k3s-io/helm-controller) 管ç†ï¼‰ï¼šæ­¤è‡ªå®šä¹‰èµ„æºä¼šæ ¹æ®åº”用到 HelmChart CR çš„å˜æ›´ï¼Œåœ¨è§¦å‘ `helm install`ã€`helm upgrade` 或 `helm uninstall` çš„åŒä¸€å‘½å空间中自动创建一个 Job。此 CR ä¼šæ ¹æ® ProjectHelmChart 的更改（例如，修改 `values.yaml`）或底层项目定义的更改（例如，从项目中添加或删除命å空间）自动更新。 + +:::note é‡è¦æ示： + +如果 ProjectHelmChart 没有部署或更新底层项目监控堆栈，你å¯ä»¥å…ˆä½¿ç”¨æ­¤èµ„æºåœ¨ Operator / System 命å空间中创建的 Job æ¥æ£€æŸ¥ Helm æ“作是å¦æœ‰é—®é¢˜ã€‚通常åªèƒ½ç”±**集群管ç†å‘˜è®¿é—®**。 + +::: + +- HelmRelease CR（通过 Operator ä¸­çš„åµŒå…¥å¼ [rancher/helm-locker](https://github.com/rancher/helm-locker) 管ç†ï¼‰ï¼šæ­¤è‡ªå®šä¹‰èµ„æºä¼šè‡ªåŠ¨é”定已部署的 Helm 版本并自动覆盖对底层资æºçš„更新，除éžæ›´æ”¹æ˜¯ Helm æ“作导致的（`helm install`ã€`helm upgrade` 或 `helm uninstall` ç”± HelmChart CR 执行）。 + +:::note + +HelmRelease CR 会å‘出 Kubernetes 事件，用于检测底层 Helm 版本修改并将其é”定回原ä½ã€‚è¦æŸ¥çœ‹è¿™äº›äº‹ä»¶ï¼Œä½ å¯ä»¥ä½¿ç”¨ `kubectl describe helmrelease -n `。你还å¯ä»¥æŸ¥çœ‹æ­¤ Operator 的日志，了解检测到更改的时间以åŠå“ªäº›èµ„æºè¢«å°è¯•æ›´æ”¹ã€‚ + +::: + +这两ç§èµ„æºéƒ½æ˜¯ä¸º Operator / System 命å空间中的所有 Helm Chart 创建的，用于é¿å…低æƒé™ç”¨æˆ·çš„æƒé™å‡çº§ã€‚ + +### 高级 Helm Project Operator é…ç½® + +有关高级é…置的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://github.com/rancher/prometheus-federator/blob/main/charts/prometheus-federator/0.0.1/README.md#advanced-helm-project-operator-configuration)。 + + + +### Local 集群上的 Prometheus Federator + +Prometheus Federator 是一个资æºå¯†é›†åž‹åº”用程åºã€‚ä½ å¯ä»¥å°†å…¶å®‰è£…到 Local 集群（**ä¸æŽ¨è**）。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/provisioning-storage-examples.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/provisioning-storage-examples.md new file mode 100644 index 00000000000..e13500f2aaf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/provisioning-storage-examples.md @@ -0,0 +1,11 @@ +--- +title: é…置存储示例 +--- + +Rancher 通过å„ç§å·æ’件æ¥æ”¯æŒæŒä¹…存储。但是，在使用这些æ’件将æŒä¹…存储绑定到工作负载之å‰ï¼Œæ— è®ºæ˜¯ä½¿ç”¨äº‘解决方案还是你自己管ç†çš„本地解决方案，你都必须先é…置存储本身。 + +为了你的方便，Rancher æ供了é…置主æµå­˜å‚¨çš„å‚考文档： + +- [NFS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) +- [vSphere](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) +- [EBS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/quick-start-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/quick-start-guides.md new file mode 100644 index 00000000000..fd0013e0777 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/quick-start-guides.md @@ -0,0 +1,17 @@ +--- +title: Rancher éƒ¨ç½²å¿«é€Ÿå…¥é—¨æŒ‡å— +--- + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](installation-and-upgrade.md)。 + +::: + +ä½ å¯ä»¥é˜…读本章节，以快速开始部署和测试 Rancher 2.xã€‚æœ¬ç« èŠ‚åŒ…å« Rancher 的简å•è®¾ç½®å’Œä¸€äº›å¸¸è§ç”¨ä¾‹çš„说明。未æ¥ï¼Œæˆ‘们会在本章节中添加更多内容。 + +我们æ供以下快速入门指å—： + +- [部署 Rancher Server](deploy-rancher-manager.md)：使用最方便的方å¼è¿è¡Œ Rancher。 + +- [部署工作负载](deploy-rancher-workloads.md)：部署一个简å•çš„[工作负载](https://kubernetes.io/docs/concepts/workloads/)并公暴露工作负载，以从集群外部访问工作负载。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-behind-an-http-proxy.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-behind-an-http-proxy.md new file mode 100644 index 00000000000..e468db53dd5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-behind-an-http-proxy.md @@ -0,0 +1,13 @@ +--- +title: 使用 HTTP 代ç†å®‰è£… Rancher +--- + +很多ä¼ä¸šæœ¬åœ°è¿è¡Œçš„æœåŠ¡å™¨æˆ–虚拟机ä¸èƒ½ç›´æŽ¥è®¿é—®äº’è”网，但是出于安全考虑，他们必须通过 HTTP(S) 代ç†è¿žæŽ¥åˆ°å¤–部æœåŠ¡ã€‚本教程将分步介ç»å¦‚何在这样的环境中进行高å¯ç”¨çš„ Rancher 安装。 + +å¦å¤–，用户也å¯ä»¥åœ¨æ²¡æœ‰ä»»ä½•äº’è”网访问的情况下离线设置 Rancher。详情请å‚è§ [Rancher 官方文档](air-gapped-helm-cli-install.md)。 + +## å®‰è£…æ¦‚è¦ + +1. [é…置基础设施](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md) +2. [é…ç½® Kubernetes 集群](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md) +3. [安装 Rancher](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-hardening-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-hardening-guides.md new file mode 100644 index 00000000000..fcbbf594054 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-hardening-guides.md @@ -0,0 +1,52 @@ +--- +title: Self-Assessment and Hardening Guides for Rancher +--- + +Rancher provides specific security hardening guides for each supported Rancher version's Kubernetes distributions. + +## Rancher Kubernetes Distributions + +Rancher uses the following Kubernetes distributions: + +- [**RKE**](https://rancher.com/docs/rke/latest/en/), Rancher Kubernetes Engine, is a CNCF-certified Kubernetes distribution that runs entirely within Docker containers. +- [**RKE2**](https://docs.rke2.io/) is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. +- [**K3s**](https://docs.k3s.io/) is a fully conformant, lightweight Kubernetes distribution. It is easy to install, with half the memory requirement of upstream Kubernetes, all in a binary of less than 100 MB. + +To harden a Kubernetes cluster that's running a distribution other than those listed, refer to your Kubernetes provider docs. + +## Hardening Guides and Benchmark Versions + +Each self-assessment guide is accompanied by a hardening guide. These guides were tested alongside the listed Rancher releases. Each self-assessment guides was tested on a specific Kubernetes version and CIS benchmark version. If a CIS benchmark has not been validated for your Kubernetes version, you can use the existing guides until a guide for your version is added. + +### RKE Guides + +| Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|--------------------|-----------------------|-----------------------|------------------| +| Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](rke1-hardening-guide.md) | + +### RKE2 Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned RKE2 | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](rke2-hardening-guide.md) | +| Standalone RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | + +### K3s Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned K3s cluster | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](k3s-hardening-guide.md) | +| Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment) | [Link](https://docs.k3s.io/security/hardening-guide) | + +## Rancher with SELinux + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a kernel module that adds extra access controls and security tools to Linux. Historically used by government agencies, SELinux is now industry-standard. SELinux is enabled by default on RHEL and CentOS. + +To use Rancher with SELinux, we recommend [installing](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) the `rancher-selinux` RPM. + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-managed-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-managed-clusters.md new file mode 100644 index 00000000000..15d2194487a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-managed-clusters.md @@ -0,0 +1,19 @@ +--- +title: Rancher 管ç†é›†ç¾¤çš„最佳实践 +--- + +### Logging + +有关集群级别日志和应用日志的建议，请å‚è§ [Logging 最佳实践](../reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md)。 + +### Monitoring + +é…ç½®åˆç†çš„监控和告警规则对于安全ã€å¯é åœ°è¿è¡Œç”Ÿäº§çŽ¯å¢ƒä¸­çš„工作负载至关é‡è¦ã€‚有关更多建议，请å‚阅[最佳实践](../reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md)。 + +### 设置容器的技巧 + +é…置良好的容器å¯ä»¥æžå¤§åœ°æ高环境的整体性能和安全性。有关容器设置的建议，请å‚è§[设置容器的技巧](../reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md)。 + +### Rancher ç®¡ç† vSphere 集群的最佳实践 + +[Rancher ç®¡ç† vSphere 集群的最佳实践](../reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md)概述了在 vSphere 环境中é…置下游 Rancher 集群的å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-manager-architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-manager-architecture.md new file mode 100644 index 00000000000..ad6a3ea462f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-manager-architecture.md @@ -0,0 +1,17 @@ +--- +title: 架构 +--- + +本章节é‡ç‚¹ä»‹ç» [Rancher Server åŠå…¶ç»„件](../reference-guides/rancher-manager-architecture/rancher-server-and-components.md) ä»¥åŠ [Rancher 如何与下游 Kubernetes 集群通信](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md)。 + +有关安装 Rancher çš„ä¸åŒæ–¹å¼çš„ä¿¡æ¯ï¼Œè¯·å‚è§[安装选项概述](installation-and-upgrade.md#安装方å¼æ¦‚è¿°)。 + +有关 Rancher API Server 的主è¦åŠŸèƒ½ï¼Œè¯·å‚è§[概述](../getting-started/overview.md#rancher-api-server-的功能)。 + +有关如何为 Rancher Server 设置底层基础架构，请å‚è§[架构推è](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)。 + +:::note + +本节默认你已对 Docker å’Œ Kubernetes 有一定的了解。如果你需è¦äº†è§£ Kubernetes 组件如何å作，请å‚è§ [Kubernetes 概念](../reference-guides/kubernetes-concepts.md)。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker.md new file mode 100644 index 00000000000..b8875d47d51 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker.md @@ -0,0 +1,207 @@ +--- +title: 使用 Docker å°† Rancher 安装到å•ä¸ªèŠ‚点中 +description: 在开å‘和测试环境中，你å¯ä»¥ä½¿ç”¨ Docker 安装。在å•ä¸ª Linux 主机上安装 Docker，然åŽä½¿ç”¨ä¸€ä¸ª Docker 容器部署 Rancher。 +--- + +Rancher å¯ä»¥é€šè¿‡è¿è¡Œå•ä¸ª Docker 容器进行安装。 + +在这ç§å®‰è£…方案中，你需è¦å°† Docker 安装到å•ä¸ª Linux 主机，然åŽä½¿ç”¨å•ä¸ª Docker 容器将 Rancher 部署到主机中。 + +:::note 想è¦ä½¿ç”¨å¤–部负载å‡è¡¡å™¨ï¼Ÿ + +请å‚阅[使用外部负载å‡è¡¡å™¨çš„ Docker 安装](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md)。 + +::: + +Rancher çš„ Docker 安装仅推è用于开å‘和测试环境中。Rancher 版本决定了能å¦å°† Rancher è¿ç§»åˆ°é«˜å¯ç”¨é›†ç¾¤ã€‚ + +Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +## Rancher 特æƒè®¿é—® + +当 Rancher Server 部署在 Docker 容器中时，容器内会安装一个本地 Kubernetes 集群供 Rancher 使用。为 Rancher 的很多功能都是以 deployment çš„æ–¹å¼è¿è¡Œçš„，而在容器内è¿è¡Œå®¹å™¨æ˜¯éœ€è¦ç‰¹æƒæ¨¡å¼çš„，因此你需è¦åœ¨å®‰è£… Rancher 时添加 `--privileged` 选项。 + +## æ“作系统，Docker，硬件和网络è¦æ±‚ + +请确ä¿ä½ çš„节点满足常规的[安装è¦æ±‚](installation-requirements.md)。 + +## 1. é…ç½® Linux 主机 + +按照[è¦æ±‚](installation-requirements.md)é…置一个 Linux 主机，用于è¿è¡Œ Rancher Server。 + +## 2. 选择一个 SSL 选项并安装 Rancher + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +:::tip 你是å¦éœ€è¦ï¼š + +- 使用代ç†ã€‚å‚è§ [HTTP 代ç†é…ç½®](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md)。 +- é…置自定义 CA æ ¹è¯ä¹¦ä»¥è®¿é—®æœåŠ¡ã€‚å‚è§[自定义 CA æ ¹è¯ä¹¦](../reference-guides/single-node-rancher-in-docker/advanced-options.md#自定义-ca-è¯ä¹¦)。 +- 完æˆç¦»çº¿å®‰è£…。å‚è§ [离线：Docker 安装](air-gapped-helm-cli-install.md)。 +- 记录所有 Rancher API 的事务。å‚加 [API 审计](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志)。 + +::: + +选择以下的选项之一： + +- [选项 A：使用 Rancher 生æˆçš„默认自签åè¯ä¹¦](#选项-a使用-rancher-生æˆçš„默认自签åè¯ä¹¦) +- [选项 B：使用你自己的è¯ä¹¦ - 自签å](#选项-b使用你自己的è¯ä¹¦---自签å) +- [选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦](#选项-c使用你自己的è¯ä¹¦---å¯ä¿¡-ca-ç­¾åçš„è¯ä¹¦) +- [选项 D：Let's Encrypt è¯ä¹¦](#选项-dlets-encrypt-è¯ä¹¦) +- [选项 E：Localhost 隧é“，ä¸ä½¿ç”¨è¯ä¹¦](#选项-elocalhost-隧é“ä¸ä½¿ç”¨è¯ä¹¦) + +### 选项 A：使用 Rancher 生æˆçš„默认自签åè¯ä¹¦ + +如果你在ä¸è€ƒè™‘身份验è¯çš„å¼€å‘或测试环境中安装 Rancher，å¯ä»¥ä½¿ç”¨ Rancher 生æˆçš„自签åè¯ä¹¦å®‰è£… Rancher。这ç§å®‰è£…æ–¹å¼é¿å…了自己生æˆè¯ä¹¦çš„麻烦。 + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest +``` + +### 选项 B：使用你自己的è¯ä¹¦ - 自签å +在你团队访问 Rancher Server çš„å¼€å‘或测试环境中，创建一个用于你的安装的自签åè¯ä¹¦ï¼Œä»¥ä¾¿å›¢é˜ŸéªŒè¯ä»–们对实例的连接。 + +:::note 先决æ¡ä»¶ï¼š + +使用 [OpenSSL](https://www.openssl.org/) 或其他方法创建自签åè¯ä¹¦ã€‚ + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 +- 在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +::: + +创建è¯ä¹¦åŽï¼Œè¿è¡Œä»¥ä¸‹ Docker 命令以安装 Rancher。使用 `-v` 标志并æä¾›è¯ä¹¦çš„路径，以将è¯ä¹¦æŒ‚载到容器中。 + +| å ä½ç¬¦ | æè¿° | +| ------------------- | --------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher:latest +``` + +### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +在公开暴露应用的生产环境中，请使用由å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œä»¥é¿å…用户收到è¯ä¹¦å®‰å…¨è­¦å‘Šã€‚ + +ä¸å»ºè®®å°† Docker 安装用于生产环境。这些说明仅适用于测试和开å‘。 + +:::note 先决æ¡ä»¶ï¼š + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 +- 在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括å¯ä¿¡ CA æ供的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +::: + +获å–è¯ä¹¦åŽï¼Œè¿è¡Œä»¥ä¸‹ Docker 命令。 + +- 使用 `-v` 标志并æä¾›è¯ä¹¦çš„路径，以将è¯ä¹¦æŒ‚载到容器中。因为你的è¯ä¹¦æ˜¯ç”±å¯ä¿¡çš„ CA ç­¾å的，因此你ä¸éœ€è¦å®‰è£…é¢å¤–çš„ CA è¯ä¹¦æ–‡ä»¶ã€‚ +- 使用 `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +| å ä½ç¬¦ | æè¿° | +| ------------------- | ----------------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +### 选项 D：Let's Encrypt è¯ä¹¦ + +:::caution + +Let's Encrypt 对新è¯ä¹¦è¯·æ±‚有频率é™åˆ¶ã€‚因此，请é™åˆ¶åˆ›å»ºæˆ–销æ¯å®¹å™¨çš„频率。详情请å‚è§ [Let's Encrypt 官方文档 - 频率é™åˆ¶](https://letsencrypt.org/docs/rate-limits/)。 + +::: + +你也å¯ä»¥åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­ä½¿ç”¨ [Let's Encrypt](https://letsencrypt.org/) è¯ä¹¦ã€‚Let's Encrypt 使用 HTTP-01 质询æ¥éªŒè¯ä½ å¯¹åŸŸå的控制æƒã€‚如果è¦ç¡®è®¤ä½ å¯¹è¯¥åŸŸå有控制æƒï¼Œä½ å¯å°†ç”¨äºŽè®¿é—® Rancher 的主机å（例如 `rancher.mydomain.com`）指å‘è¿è¡Œçš„主机的 IP。你å¯é€šè¿‡åœ¨ DNS 中创建 A 记录，以将主机å绑定到 IP 地å€ã€‚ + +ä¸å»ºè®®å°† Docker 安装用于生产环境。这些说明仅适用于测试和开å‘。 + +:::note 先决æ¡ä»¶ï¼š + +- Let's Encrypt 是è”网æœåŠ¡ã€‚因此，在内网和离线环境中ä¸èƒ½ä½¿ç”¨ã€‚ +- 在 DNS 中创建一æ¡è®°å½•ï¼Œå°† Linux 主机 IP 地å€ç»‘定到è¦ç”¨äºŽè®¿é—® Rancher 的主机å（例如，`rancher.mydomain.com`）。 +- 在 Linux 主机上打开 `TCP/80` 端å£ã€‚Let's Encrypt çš„ HTTP-01 质询å¯ä»¥æ¥è‡ªä»»ä½•æº IP 地å€ï¼Œå› æ­¤ç«¯å£ `TCP/80` 必须开放开所有 IP 地å€ã€‚ + +::: + +满足先决æ¡ä»¶åŽï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ä½¿ç”¨ Let's Encrypt è¯ä¹¦å®‰è£… Rancher。 + +| å ä½ç¬¦ | æè¿° | +| ----------------- | ------------------- | +| `` | 你的域ååœ°å€ | + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --acme-domain +``` + +### 选项 E：Localhost 隧é“，ä¸ä½¿ç”¨è¯ä¹¦ + +如果你在开å‘或测试环境中安装 Rancher，且环境中有è¿è¡Œçš„ localhost 隧é“解决方案（如 [ngrok](https://ngrok.com/)），ä¸è¦ç”Ÿæˆè¯ä¹¦ã€‚此安装选项ä¸éœ€è¦è¯ä¹¦ã€‚ + +- 使用 `--no-cacerts` 作为å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +## 高级选项 + +使用 Docker å°† Rancher 安装到å•ä¸ªèŠ‚点时，有如下几个å¯å¼€å¯çš„高级选项： + +- 自定义 CA è¯ä¹¦ +- API 审计日志 +- TLS 设置 +- 离线环境 +- æŒä¹…åŒ–æ•°æ® +- 在åŒä¸€ä¸ªèŠ‚点中è¿è¡Œ `rancher/rancher` å’Œ `rancher/rancher-agent` + +详情请å‚è§[本页](../reference-guides/single-node-rancher-in-docker/advanced-options.md)。 + +## 故障排除 + +如需了解常è§é—®é¢˜åŠæ•…障排除æ示，请å‚è§[本页](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +## åŽç»­æ“作 + +- **推è**：检查å•èŠ‚点[备份](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md)å’Œ[æ¢å¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md)。你å¯èƒ½æš‚时没有需è¦å¤‡ä»½çš„æ•°æ®ï¼Œä½†æ˜¯æˆ‘们建议你在常规使用 Rancher åŽåˆ›å»ºå¤‡ä»½ã€‚ +- 创建 Kubernetes 集群：[é…ç½® Kubernetes 集群](kubernetes-clusters-in-rancher-setup.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-security.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-security.md new file mode 100644 index 00000000000..15677f4a9a7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-security.md @@ -0,0 +1,85 @@ +--- +title: 安全 +--- + + + + + + + +
+

安全策略

+

Rancher Labs 会负责任地披露问题，并致力于在åˆç†çš„时间内解决所有问题。

+		 +

报告过程

+

请将安全问题å‘é€è‡³ security-rancher@suse.com。

+		 +

公告

+

订阅 Rancher 公告论å›ä»¥èŽ·å–版本更新。

+
+ +安全是 Rancher 全部功能的基础。Rancher 集æˆäº†å…¨éƒ¨ä¸»æµèº«ä»½éªŒè¯å·¥å…·å’ŒæœåŠ¡ï¼Œå¹¶æ供了ä¼ä¸šçº§çš„ [RBAC 功能](manage-role-based-access-control-rbac.md)，让你的 Kubernetes 集群更加安全。 + +本文介ç»äº†å®‰å…¨ç›¸å…³çš„文档以åŠèµ„æºï¼Œè®©ä½ çš„ Rancher 安装和下游 Kubernetes 集群更加安全。 + +### NeuVector 与 Rancher çš„é›†æˆ + +NeuVector 是一个开æºçš„ã€ä»¥å®¹å™¨ä¸ºä¸­å¿ƒçš„安全应用程åºï¼ŒçŽ°å·²é›†æˆåˆ° Rancher 中。NeuVector æ供生产安全ã€DevOps æ¼æ´žä¿æŠ¤å’Œå®¹å™¨é˜²ç«å¢™ç­‰åŠŸèƒ½ã€‚请å‚阅 [Rancher 文档](../integrations-in-rancher/neuvector.md)å’Œ [NeuVector 文档](https://open-docs.neuvector.com/)了解更多信æ¯ã€‚ + +### 在 Kubernetes 集群上è¿è¡Œ CIS 安全扫æ + +Rancher 使用 [kube-bench](https://github.com/aquasecurity/kube-bench) æ¥è¿è¡Œå®‰å…¨æ‰«æ，从而检查 Kubernetes 是å¦æŒ‰ç…§ [CIS](https://www.cisecurity.org/cis-benchmarks/)（Center for Internet Security，互è”网安全中心）Kubernetes Benchmark 中定义的安全最佳实践进行部署。 + +CIS Kubernetes Benchmark 是一个å‚考文档，用于为 Kubernetes 建立安全é…置基线。 + +CIS 是一个 501(c\)(3) éžè¥åˆ©ç»„织，æˆç«‹äºŽ 2000 å¹´ 10 月，其使命是识别ã€å¼€å‘ã€éªŒè¯ã€ä¿ƒè¿›å’Œç»´æŒç½‘络防御的最佳实践方案，并建立和指导社区，以在网络空间中è¥é€ ä¿¡ä»»çš„环境。 + +CIS Benchmark 是目标系统安全é…置的最佳实践。CIS Benchmark 是由安全专家ã€æŠ€æœ¯ä¾›åº”商ã€å…¬å¼€å’Œç§äººç¤¾åŒºæˆå‘˜ï¼Œä»¥åŠ CIS Benchmark å¼€å‘团队共åŒå¿—æ„¿å¼€å‘的。 + +Benchmark æ供两ç§ç±»åž‹çš„建议，分别是自动（Automated）和手动（Manual）。我们åªè¿è¡Œ Automated 相关的测试。 + +Rancher 在集群上è¿è¡Œ CIS 安全扫æ时会生æˆä¸€ä»½æŠ¥å‘Šï¼Œè¯¥æŠ¥å‘Šä¼šæ˜¾ç¤ºæ¯ä¸ªæµ‹è¯•çš„结果，包括测试概è¦ä»¥åŠ `passed`ã€`skipped` å’Œ `failed` 的测试数é‡ã€‚报告还包括失败测试的修正步骤。 + +有关详细信æ¯ï¼Œè¯·å‚阅[安全扫æ](cis-scan-guides.md)。 + +### SELinux RPM + +[安全增强型 Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) 是对 Linux 的安全增强。被政府机构使用之åŽï¼ŒSELinux å·²æˆä¸ºè¡Œä¸šæ ‡å‡†ï¼Œå¹¶åœ¨ CentOS 7 å’Œ 8 上默认å¯ç”¨ã€‚ + +我们æ供了 `rancher-selinux` å’Œ `rke2-selinux` 两个 RPM（Red Hat 软件包），让 Rancher 产å“能够在 SELinux 主机上正常è¿è¡Œã€‚有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](selinux-rpm.md)。 + +### Rancher å¼ºåŒ–æŒ‡å— + +Rancher 强化指å—基于 CIS Kubernetes Benchmark。 + +强化指å—为强化 Rancher 的生产安装æ供了说明性指导。有关安全管控的完整列表，请å‚阅 Rancher çš„ [CIS Kubernetes Benchmark自我评估指å—](#cis-benchmark-和自我评估)。 + +> 强化指å—æ述了如何ä¿æŠ¤é›†ç¾¤ä¸­çš„节点，建议在安装 Kubernetes 之å‰å‚考强化指å—中的步骤。 + +æ¯ä¸ªå¼ºåŒ–指å—版本都针对特定的 CIS Kubernetes Benchmarkã€Kubernetes å’Œ Rancher 版本。 + +### CIS Benchmark 和自我评估 + +Benchmark 自我评估是 Rancher 安全强化指å—的辅助。强化指å—展示了如何强化集群，而 Benchmark 指å—旨在帮助你评估强化集群的安全级别。 + +由于 Rancher å’Œ RKE å°† Kubernetes æœåŠ¡å®‰è£…为 Docker 容器，因此 CIS Kubernetes Benchmark 中的许多管控验è¯æ£€æŸ¥éƒ½ä¸é€‚用。本指å—将介ç»å„ç§ç®¡æŽ§ï¼Œå¹¶æ供更新的示例命令æ¥å®¡æ ¸ Rancher 创建的集群的åˆè§„性。你å¯ä»¥å‰å¾€ [CIS 网站](https://www.cisecurity.org/benchmark/kubernetes/)下载原始的 Benchmark 文档。 + +Rancher æ¯ä¸ªç‰ˆæœ¬çš„自我评估指å—都对应特定的强化指å—ã€Rancherã€Kubernetes å’Œ CIS Benchmark 版本。 + +### 第三方渗é€æµ‹è¯•æŠ¥å‘Š + +Rancher 会定期è˜è¯·ç¬¬ä¸‰æ–¹å¯¹ Rancher 2.x 软件栈进行安全审核和渗é€æµ‹è¯•ã€‚测试环境会éµå¾ª Rancher æ供的强化指å—。旧的渗é€æµ‹è¯•æŠ¥å‘Šå¦‚下。 + +结果： + +- [Cure53 渗é€æµ‹è¯• - 2019 å¹´ 7 月](https://releases.rancher.com/documents/security/pen-tests/2019/RAN-01-cure53-report.final.pdf) +- [Untamed Theory 渗é€æµ‹è¯• - 2019 å¹´ 3 月](https://releases.rancher.com/documents/security/pen-tests/2019/UntamedTheory-Rancher_SecurityAssessment-20190712_v5.pdf) + +### Rancher 安全公告和 CVE + +Rancher 致力于å‘社区披露我们产å“的安全问题。有关我们已解决问题的 CVE（Common Vulnerabilities and Exposures，通用æ¼æ´žæŠ«éœ²ï¼‰åˆ—表，请å‚阅[此页é¢](../reference-guides/rancher-security/security-advisories-and-cves.md)。 + +### Kubernetes 安全最佳实践 + +有关ä¿æŠ¤ Kubernetes 集群的建议，请å‚阅 [Kubernetes 安全最佳实践指å—](../reference-guides/rancher-security/kubernetes-security-best-practices.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-server-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-server-configuration.md new file mode 100644 index 00000000000..a0b6e3850b1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-server-configuration.md @@ -0,0 +1,12 @@ +--- +title: Rancher Server é…ç½® +--- + +- [RKE1 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [AKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) +- [GKE 集群é…ç½®](../pages-for-subheaders/gke-cluster-configuration.md) +- [使用现有节点](../pages-for-subheaders/use-existing-nodes.md) +- [åŒæ­¥é›†ç¾¤](../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-server.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-server.md new file mode 100644 index 00000000000..0d632b6def6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rancher-server.md @@ -0,0 +1,17 @@ +--- +title: Rancher Server 的最佳实践 +--- + +本指å—介ç»äº†è®© Rancher 管ç†ä¸‹æ¸¸ Kubernetes 集群的 Rancher Server è¿è¡Œå»ºè®®ã€‚ + +### 推è的架构和基础设施 + +有关在高å¯ç”¨ Kubernetes 集群上设置 Rancher Server 的通用建议，请å‚è§[本指å—](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md)。 + +### 部署策略 + +[本指å—](../reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md)旨在帮助你选择部署策略（区域部署/中心è¾å°„型部署），æ¥è®© Rancher Server 更好地管ç†ä¸‹æ¸¸ Kubernetes 集群。 + +### 在 vSphere 环境中安装 Rancher + +[本指å—](../reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md)介ç»äº†åœ¨ vSphere 环境中安装 Rancher çš„å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/resources.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/resources.md new file mode 100644 index 00000000000..45aa544fa7e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/resources.md @@ -0,0 +1,25 @@ +--- +title: èµ„æº +--- + +### Docker 安装 + +[å•èŠ‚点 Docker 安装](rancher-on-a-single-node-with-docker.md)适用于想è¦æµ‹è¯• Rancher 的用户。你无需使用 Helm 在 Kubernetes 集群上è¿è¡Œ Rancher，你å¯ä»¥ä½¿ç”¨ `docker run` 命令，把 Rancher Server 组件安装到å•ä¸ªèŠ‚点上。 + +由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此，如果该节点å‘生故障，由于其他节点上没有å¯ç”¨çš„ etcd æ•°æ®å‰¯æœ¬ï¼Œä½ å°†ä¸¢å¤± Rancher Server 的所有数æ®ã€‚ + +### 离线安装 + +按照[以下步骤](air-gapped-helm-cli-install.md)在离线环境中安装 Rancher Server。 + +离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +### 高级选项 + +安装 Rancher 时，有如下几个å¯å¼€å¯çš„高级选项：æ¯ä¸ªå®‰è£…指å—中都æ供了对应的选项。了解选项详情： + +- [自定义 CA è¯ä¹¦](../getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md) +- [API 审计日志](../how-to-guides/advanced-user-guides/enable-api-audit-log.md) +- [TLS 设置](../getting-started/installation-and-upgrade/installation-references/tls-settings.md) +- [etcd é…ç½®](../how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md) +- [离线安装 Local System Chart](../getting-started/installation-and-upgrade/resources/local-system-charts.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rke1-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rke1-hardening-guide.md new file mode 100644 index 00000000000..6a187df3273 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rke1-hardening-guide.md @@ -0,0 +1,504 @@ +--- +title: RKE Hardening Guide +--- + +This document provides prescriptive guidance for how to harden an RKE cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 up to v1.25 | + +:::note +At the time of writing, the upstream CIS Kubernetes v1.25 benchmark is not yet available in Rancher. At this time Rancher is using the CIS v1.23 benchmark when scanning Kubernetes v1.25 clusters. Due to that, the CIS checks 5.2.3, 5.2.4, 5.2.5 and 5.2.6 might fail. +::: + +For more details on how to evaluate a hardened RKE cluster against the official CIS benchmark, refer to the RKE self-assessment guides for specific Kubernetes and CIS benchmark versions. + +## Host-level requirements + +### Configure Kernel Runtime Parameters + +The following `sysctl` configuration is recommended for all nodes types in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.overcommit_memory=1 +vm.panic_on_oom=0 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Configure `etcd` user and group + +A user account and group for the **etcd** service is required to be set up before installing RKE. + +#### Create `etcd` user and group + +To create the **etcd** user and group run the following console commands. +The commands below use `52034` for **uid** and **gid** for example purposes. +Any valid unused **uid** or **gid** could also be used in lieu of `52034`. + +```bash +groupadd --gid 52034 etcd +useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin +``` + +When deploying RKE through its cluster configuration `config.yml` file, update the `uid` and `gid` of the `etcd` user: + +```yaml +services: + etcd: + gid: 52034 + uid: 52034 +``` + +## Kubernetes runtime requirements + +### Configure `default` Service Account + +#### Set `automountServiceAccountToken` to `false` for `default` service accounts + +Kubernetes provides a default service account which is used by cluster workloads where no specific service account is assigned to the pod. +Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. +The default service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. +Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### Configure Network Policy + +#### Ensure that all Namespaces have Network Policies defined + +Running different applications on the same Kubernetes cluster creates a risk of one compromised application attacking a neighboring application. Network segmentation is important to ensure that containers can communicate only with those they are supposed to. A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. + +Network Policies are namespace scoped. When a network policy is introduced to a given namespace, all traffic not allowed by the policy is denied. However, if there are no network policies in a namespace all traffic will be allowed into and out of the pods in that namespace. To enforce network policies, a container network interface (CNI) plugin must be enabled. This guide uses [Canal](https://github.com/projectcalico/canal) to provide the policy enforcement. Additional information about CNI providers can be found [here](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/). + +Once a CNI provider is enabled on a cluster a default network policy can be applied. For reference purposes a **permissive** example is provided below. If you want to allow all traffic to all pods in a namespace (even if policies are added that cause some pods to be treated as “isolatedâ€), you can create a policy that explicitly allows all traffic in that namespace. Save the following configuration as `default-allow-all.yaml`. Additional [documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) about network policies can be found on the Kubernetes site. + +:::caution +This network policy is just an example and is not recommended for production use. +::: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +``` + +Create a bash script file called `apply_networkPolicy_to_all_ns.sh`. Be sure to `chmod +x apply_networkPolicy_to_all_ns.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl apply -f default-allow-all.yaml -n ${namespace} +done +``` + +Execute this script to apply the `default-allow-all.yaml` configuration with the **permissive** `NetworkPolicy` to all namespaces. + +## Known Limitations + +- Rancher **exec shell** and **view logs** for pods are **not** functional in a hardened setup when only a public IP is provided when registering custom nodes. This functionality requires a private IP to be provided when registering the custom nodes. +- When setting `default_pod_security_policy_template_id:` to `restricted` or `restricted-noroot`, based on the pod security policies (PSP) [provided](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) by Rancher, Rancher creates `RoleBindings` and `ClusterRoleBindings` on the `default` service accounts. The CIS check 5.1.5 requires that the `default` service accounts have no roles or cluster roles bound to it apart from the defaults. In addition, the `default` service accounts should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +## Reference Hardened RKE `cluster.yml` Configuration + +The reference `cluster.yml` is used by the RKE CLI that provides the configuration needed to achieve a hardened installation of RKE. RKE [documentation](https://rancher.com/docs/rke/latest/en/installation/) provides additional details about the configuration items. This reference `cluster.yml` does not include the required `nodes` directive which will vary depending on your environment. Documentation for node configuration in RKE can be found [here](https://rancher.com/docs/rke/latest/en/config-options/nodes/). + +The example `cluster.yml` configuration file contains an Admission Configuration policy in the `services.kube-api.admission_configuration` field. This [sample](../reference-guides/rancher-security/psa-restricted-exemptions.md) policy contains the namespace exemptions necessary for an imported RKE cluster to run properly in Rancher, similar to Rancher's pre-defined [`rancher-restricted`](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) policy. + +If you prefer to use RKE's default `restricted` policy, then leave the `services.kube-api.admission_configuration` field empty and set `services.pod_security_configuration` to `restricted`. See [the RKE docs](https://rke.docs.rancher.com/config-options/services/pod-security-admission) for more information. + + + + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + # Leave `pod_security_configuration` out if you are setting a + # custom policy in `admission_configuration`. Otherwise set + # it to `restricted` to use RKE's pre-defined restricted policy, + # and remove everything inside `admission_configuration` field. + # + # pod_security_configuration: restricted + # + admission_configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [ calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator ] + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: "true" + generate_serving_certificate: true +addons: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + generate_serving_certificate: true +addons: | + # Upstream Kubernetes restricted PSP policy + # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: restricted-noroot + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: psp:restricted-noroot + rules: + - apiGroups: + - extensions + resourceNames: + - restricted-noroot + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: psp:restricted-noroot + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-noroot + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +## Reference Hardened RKE Cluster Template Configuration + +The reference RKE cluster template provides the minimum required configuration to achieve a hardened installation of Kubernetes. RKE templates are used to provision Kubernetes and define Rancher settings. Follow the Rancher [documentation](installation-and-upgrade.md) for additional information about installing RKE and its template details. + + + + +```yaml +# +# Cluster Config +# +default_pod_security_admission_configuration_template_name: rancher-restricted +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: false + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +```yaml +# +# Cluster Config +# +default_pod_security_policy_template_id: restricted-noroot +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +## Conclusion + +If you have followed this guide, your RKE custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rke2-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rke2-hardening-guide.md new file mode 100644 index 00000000000..962462c7f3f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/rke2-hardening-guide.md @@ -0,0 +1,259 @@ +--- +title: RKE2 Hardening Guide +--- + +This document provides prescriptive guidance for how to harden an RKE2 cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE2 clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 up to v1.25 | + +:::note +At the time of writing, the upstream CIS Kubernetes v1.25 benchmark is not yet available in Rancher. At this time Rancher is using the CIS v1.23 benchmark when scanning Kubernetes v1.25 clusters. Due to that, the CIS checks 5.2.2, 5.2.3, 5.2.5, 5.2.6, 5.2.7 and 5.2.8 might fail. +::: + +For more details on how to evaluate a hardened RKE2 cluster against the official CIS benchmark, refer to the RKE2 self-assessment guides for specific Kubernetes and CIS benchmark versions. + +RKE2 passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. RKE2 will not modify the host operating system. Therefore, you, the operator, must make a few host-level modifications. +2. Certain CIS controls for Network Policies and Pod Security Standards (or Pod Security Policies (PSP) on RKE2 versions prior to v1.25) will restrict the functionality of the cluster. You must opt into having RKE2 configure these for you. To help ensure these requirements are met, RKE2 can be started with the profile flag set to `cis-1.23` for v1.25 and newer or `cis-1.6` for v1.24 and older. + +## Host-level requirements + +There are two areas of host-level requirements: kernel parameters and etcd process/directory configuration. These are outlined in this section. + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Ensure etcd is configured properly + +The CIS Benchmark requires that the etcd data directory be owned by the `etcd` user and group. This implicitly requires the etcd process run as the host-level `etcd` user. To achieve this, RKE2 takes several steps when started with a valid `cis-1.xx` profile: + +1. Check that the `etcd` user and group exists on the host. If they don't, exit with an error. +2. Create etcd's data directory with `etcd` as the user and group owner. +3. Ensure the etcd process is ran as the `etcd` user and group by setting the etcd static pod's `SecurityContext` appropriately. + +To meet the above requirements, you must: + +#### Create the etcd user + +On some Linux distributions, the `useradd` command will not create a group. The `-U` flag is included below to account for that. This flag tells `useradd` to create a group with the same name as the user. + +```bash +sudo useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U +``` + +## Kubernetes runtime requirements + +The runtime requirements to pass the CIS Benchmark are centered around pod security, network policies and kernel parameters. Most of this is automatically handled by RKE2 when using a valid `cis-1.xx` profile, but some additional operator intervention is required. These are outlined in this section. + +### Ensure `protect-kernel-defaults` is set + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +Both `protect-kernel-defaults` and `profile` flags can be set in the RKE2 template configuration file. +When the `profile` flag is set, RKE2 will set the flag to `true` if it is unset. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + profile: # use cis-1.23 or cis-1.6 + protect-kernel-defaults: true +``` + +### PodSecurity + +RKE2 always runs with some amount of pod security. + + + + +On v1.25 and newer, [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) are used for pod security. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 +``` + +When both the `defaultPodSecurityAdmissionConfigurationTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures the Pod Security Admission Controller with the PSA configuration template `rancher-restricted`, to enforce restricted mode in all namespaces, except the ones in the template's exemption list. + These namespaces are exempted to allow system pods to run without restrictions, which is required for proper operation of the cluster. + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + + + + + +On Kubernetes v1.24 and older, the `PodSecurityPolicy` admission controller is always enabled. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. + +:::note +In the following example the profile is set to `cis-1.6` which is the value defined in the upstream RKE2, but the cluster is actually configured to pass the CIS v1.23 hardened profile +::: + +```yaml +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 +``` + + +When both the `defaultPodSecurityPolicyTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures runtime pod security policies that allow the cluster to pass associated controls. + + + + +:::note +The Kubernetes control plane components and critical additions such as CNI, DNS, and Ingress are ran as pods in the `kube-system` namespace. Therefore, this namespace will have a policy that is less restrictive so that these components can run properly. +::: + +### NetworkPolicies + +When ran with a valid `cis-1.xx` profile, RKE2 will put `NetworkPolicies` in place that passes the CIS Benchmark for Kubernetes' built-in namespaces. These namespaces are: `kube-system`, `kube-public`, `kube-node-lease`, and `default`. + +The `NetworkPolicy` used will only allow pods within the same namespace to talk to each other. The notable exception to this is that it allows DNS requests to be resolved. + +:::note +Operators must manage network policies as normal for additional namespaces that are created. +::: + +### Configure `default` service account + +**Set `automountServiceAccountToken` to `false` for `default` service accounts** + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. The `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE2 install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +For namespaces created by the cluster operator, the following script and configuration file can be used to configure the `default` service account. + +The configuration bellow must be saved to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `sudo chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + echo -n "Patching namespace $namespace - " + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### API Server audit configuration + +CIS requirements 1.2.19 to 1.2.22 are related to configuring audit logs for the API Server. When RKE2 is started with the `profile` flag set, it will automatically configure hardened `--audit-log-` parameters in the API Server to pass those CIS checks. + +RKE2's default audit policy is configured to not log requests in the API Server. This is done to allow cluster operators flexibility to customize an audit policy that suits their auditing requirements and needs, as these are specific to each users' environment and policies. + +A default audit policy is created by RKE2 when started with the `profile` flag set. The policy is defined in `/etc/rancher/rke2/audit-policy.yaml`. + +```yaml +apiVersion: audit.k8s.io/v1 +kind: Policy +metadata: + creationTimestamp: null +rules: +- level: None +``` + +## Reference Hardened RKE2 Template Configuration + +The reference template configuration is used in Rancher to create a hardened RKE2 custom cluster. This reference does not include other required **cluster configuration** directives which will vary depending on your environment. + + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 + protect-kernel-defaults: true +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 + protect-kernel-defaults: true +``` + + + + +## Conclusion + +If you have followed this guide, your RKE2 custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE2 self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/selinux-rpm.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/selinux-rpm.md new file mode 100644 index 00000000000..6dd562f95e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/selinux-rpm.md @@ -0,0 +1,16 @@ +--- +title: SELinux RPM +--- + +[安全增强型 Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) 是对 Linux 的安全增强。 + +它由 Red Hat å¼€å‘，是 Linux 上 MAC（mandatory access controls，强制访问控制）的实现。系统管ç†å‘˜å¯ä»¥ä½¿ç”¨ MAC 设置应用程åºå’Œç”¨æˆ·æ˜¯å¦‚何访问ä¸åŒèµ„æºçš„，例如文件ã€è®¾å¤‡ã€ç½‘络和进程间的通信。SELinux 还通过默认é™åˆ¶æ“作系统æ¥å¢žå¼ºå®‰å…¨æ€§ã€‚ + +被政府机构使用之åŽï¼ŒSELinux å·²æˆä¸ºè¡Œä¸šæ ‡å‡†ï¼Œå¹¶åœ¨ CentOS 7 å’Œ 8 上默认å¯ç”¨ã€‚è¦æ£€æŸ¥ SELinux 是å¦åœ¨ä½ çš„系统上å¯ç”¨å’Œæ‰§è¡Œï¼Œè¯·ä½¿ç”¨ `getenforce`： + +``` +# getenforce +Enforcing +``` + +我们æ供了 [`rancher-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) å’Œ [`rke2-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md) 两个 RPM（Red Hat 软件包），让 Rancher 产å“能够在 SELinux 主机上正常è¿è¡Œã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/set-up-cloud-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/set-up-cloud-providers.md new file mode 100644 index 00000000000..a35d5a22313 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/set-up-cloud-providers.md @@ -0,0 +1,43 @@ +--- +title: 设置 Cloud Provider +--- + +_cloud provider_ 是 Kubernetes 中的一个模å—，它æ供了一个用于管ç†èŠ‚点ã€è´Ÿè½½å‡è¡¡å™¨å’Œç½‘络路由的接å£ã€‚ + +在 Rancher 中设置 cloud provider 时，如果你使用的云æ供商支æŒè‡ªåŠ¨åŒ–，Rancher Server å¯ä»¥åœ¨å¯åŠ¨ Kubernetes 定义时自动é…置新节点ã€è´Ÿè½½å‡è¡¡å™¨æˆ–æŒä¹…存储设备。 + +如果你é…置的节点云æ供商集群ä¸æ»¡è¶³å…ˆå†³æ¡ä»¶ï¼Œé›†ç¾¤å°†æ— æ³•æ­£ç¡®é…置。 + +**Cloud Provider** 选项默认设置为 `None`。 + +å¯ä»¥å¯ç”¨çš„云æ供商包括： + +* Amazon +* Azure +* GCE (Google Compute Engine) +* vSphere + +### 设置 Amazon 云æ供商 + +有关å¯ç”¨ Amazon 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md)。 + +### 设置 Azure 云æ供商 + +有关å¯ç”¨ Azure 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md)。 + +### 设置 GCE 云æ供商 + +有关å¯ç”¨ Google Compute Engine 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md)。 + +### 设置 vSphere 云æ供商 + +有关å¯ç”¨ vSphere 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[树内 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 + +### 设置自定义云æ供商 + +任何 Kubernetes Cloud Provider 都å¯ä»¥é€šè¿‡`自定义`云æ供商进行é…置。 + +对于自定义云æ供商选项，你å¯ä»¥å‚考 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/)，了解如何为你的云æ供商编辑 yaml 文件。特定云æ供商的详细é…置说明如下： + +* [vSphere](https://rke.docs.rancher.com/config-options/cloud-providers/vsphere) +* [OpenStack](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/openstack/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md new file mode 100644 index 00000000000..d93fda6f3a6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md @@ -0,0 +1,29 @@ +--- +title: 通过托管 Kubernetes æ供商设置集群 +--- + +在这ç§æƒ…况下，Rancher ä¸ä¼šé…ç½® Kubernetes，因为它是由 Google Kubernetes Engine (GKE)ã€Amazon Elastic Container Service for Kubernetes 或 Azure Kubernetes Service ç­‰æ供商安装的。 + +如果你使用 Kubernetes æ供商，例如 Google GKE，Rancher 将与对应的云 API 集æˆï¼Œå…许你从 Rancher UI ä¸ºæ‰˜ç®¡é›†ç¾¤åˆ›å»ºå’Œç®¡ç† RBAC。 + +在这个用例中，Rancher 使用æ供商的 API å‘托管æ供商å‘é€è¯·æ±‚。然åŽï¼Œæ供商会为你é…置和托管集群。集群创建æˆåŠŸåŽï¼Œä½ å¯ä»¥åƒç®¡ç†æœ¬åœ°é›†ç¾¤æˆ–云上集群一样，通过 Rancher UI 对集群进行管ç†ã€‚ + +Rancher 支æŒä»¥ä¸‹ Kubernetes æ供商： + +- [Google GKE (Google Kubernetes Engine)](https://cloud.google.com/kubernetes-engine/) +- [Amazon EKS (Amazon Elastic Container Service for Kubernetes)](https://aws.amazon.com/eks/) +- [Microsoft AKS (Azure Kubernetes Service)](https://azure.microsoft.com/en-us/services/kubernetes-service/) +- [Alibaba ACK (Alibaba Cloud Container Service for Kubernetes)](https://www.alibabacloud.com/product/kubernetes) +- [Tencent TKE (Tencent Kubernetes Engine)](https://intl.cloud.tencent.com/product/tke) +- [Huawei CCE (Huawei Cloud Container Engine)](https://www.huaweicloud.com/en-us/product/cce.html) + +## 托管 Kubernetes æä¾›å•†çš„èº«ä»½éªŒè¯ + +使用 Rancher 创建由æ供商托管的集群时，你需è¦è¾“入身份验è¯ä¿¡æ¯ã€‚Rancher 会使用验è¯ä¿¡æ¯æ¥è®¿é—®äº‘厂商的 API。有关如何获å–此信æ¯çš„详情，请å‚阅： + +- [创建 GKE 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +- [创建 EKS 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +- [创建 AKS 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) +- [创建 ACK 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +- [创建 TKE 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) +- [创建 CCE 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/single-node-rancher-in-docker.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/single-node-rancher-in-docker.md new file mode 100644 index 00000000000..ec2524a165f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/single-node-rancher-in-docker.md @@ -0,0 +1,5 @@ +--- +title: Docker 中的å•èŠ‚点 Rancher +--- + +以下文档将讨论 Docker 安装的 [HTTP 代ç†é…ç½®](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md)å’Œ[高级选项](../reference-guides/single-node-rancher-in-docker/advanced-options.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-existing-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-existing-nodes.md new file mode 100644 index 00000000000..d54ae37d50b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-existing-nodes.md @@ -0,0 +1,137 @@ +--- +title: 在现有自定义节点上å¯åŠ¨ Kubernetes +description: è¦åˆ›å»ºå…·æœ‰è‡ªå®šä¹‰èŠ‚点的集群，你需è¦è®¿é—®é›†ç¾¤ä¸­çš„æœåŠ¡å™¨ï¼Œå¹¶æ ¹æ® Rancher çš„è¦æ±‚é…ç½®æœåŠ¡å™¨ã€‚ +--- + +创建自定义集群时，Rancher 使用 RKE（Rancher Kubernetes Engine）在本地裸机æœåŠ¡å™¨ã€æœ¬åœ°è™šæ‹Ÿæœºæˆ–云æœåŠ¡å™¨èŠ‚点中创建 Kubernetes 集群。 + +è¦ä½¿ç”¨æ­¤é€‰é¡¹ï¼Œä½ éœ€è¦è®¿é—®è¦åœ¨ Kubernetes 集群中使用的æœåŠ¡å™¨ã€‚请根æ®[è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)é…ç½®æ¯å°æœåŠ¡å™¨ï¼Œå…¶ä¸­åŒ…括硬件è¦æ±‚å’Œ Docker è¦æ±‚。在æ¯å°æœåŠ¡å™¨ä¸Šå®‰è£… Docker åŽï¼Œä½ è¿˜éœ€è¦åœ¨æ¯å°æœåŠ¡å™¨ä¸Šè¿è¡Œ Rancher UI 中æ供的命令，从而将æ¯å°æœåŠ¡å™¨è½¬æ¢ä¸º Kubernetes 节点。 + +本节介ç»å¦‚何设置自定义集群。 + +## 使用自定义节点创建集群 + +:::note 使用 Windows 主机作为 Kubernetes Worker 节点？ + +在开始之å‰ï¼Œè¯·å‚阅[é…ç½® Windows 自定义集群](use-windows-clusters.md)。 + +::: + +### 1. é…ç½® Linux 主机 + +ä½ å¯ä»¥é€šè¿‡é…ç½® Linux 主机，æ¥åˆ›å»ºè‡ªå®šä¹‰é›†ç¾¤ã€‚你的主机å¯ä»¥æ˜¯ï¼š + +- 云虚拟机 +- 本地虚拟机 +- 裸机æœåŠ¡å™¨ + +如果è¦é‡å¤ä½¿ç”¨ä¹‹å‰çš„自定义集群中的节点，请在å¤ç”¨ä¹‹å‰[清ç†èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md)。如果你é‡å¤ä½¿ç”¨å°šæœªæ¸…ç†çš„节点，则集群é…ç½®å¯èƒ½ä¼šå¤±è´¥ã€‚ + +æ ¹æ®[安装è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)å’Œ[生产就绪集群的检查清å•](checklist-for-production-ready-clusters.md)é…置主机。 + +如果你使用 Amazon EC2 作为主机，并希望使用[åŒæ ˆ (dual-stack)](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) 功能，则需è¦æ»¡è¶³é…置主机的其他[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + +### 2. 创建自定义集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡»**自定义**。 +1. 输入**集群å称**。 +1. 在**集群é…ç½®**中，选择 Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。 + + :::note 你使用 Windows 主机作为 Kubernetes Worker 节点？ + + - 请å‚阅[å¯ç”¨ Windows 支æŒé€‰é¡¹](use-windows-clusters.md)。 + - æ”¯æŒ Windows 集群的唯一网络æ’件是 Flannel。 + + ::: + + :::note Amazon EC2 上的åŒæ ˆï¼š + + 如果你使用 Amazon EC2 作为主机，并希望使用[åŒæ ˆ (dual-stack)](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) 功能，则需è¦æ»¡è¶³é…ç½® RKE 的其他[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + + ::: + +6. 点击**下一步**。 + +4. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ + +7. 从**节点角色**中，选择è¦ç”±é›†ç¾¤èŠ‚点充当的角色。你必须为 `etcd`ã€`worker` å’Œ `controlplane` 角色é…置至少一个节点。自定义集群需è¦æ‰€æœ‰ä¸‰ä¸ªè§’色æ‰èƒ½å®Œæˆé…置。有关角色的详细信æ¯ï¼Œè¯·å‚阅[本节](../reference-guides/kubernetes-concepts.md#kubernetes-集群中节点的角色)。 + +:::note + +- 使用 Windows 主机作为 Kubernetes Worker 节点？请å‚阅[本节](use-windows-clusters.md)。 +- 裸机æœåŠ¡å™¨æ醒：如果你想将裸机æœåŠ¡å™¨ä¸“用于æ¯ä¸ªè§’色，则必须为æ¯ä¸ªè§’色é…置一个裸机æœåŠ¡å™¨ï¼ˆå³é…置多个裸机æœåŠ¡å™¨ï¼‰ã€‚ + +::: + +8. **å¯é€‰**：点击[显示高级选项](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md)æ¥æŒ‡å®šæ³¨å†ŒèŠ‚点时使用的 IP 地å€ï¼Œè¦†ç›–节点的主机å，或将[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)或[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)添加到节点。 + +9. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ + +10. 使用你惯用的 shell（例如 PuTTy 或远程终端）登录到你的 Linux 主机。粘贴剪贴æ¿çš„命令并è¿è¡Œã€‚ + +:::note + +如果è¦å°†ç‰¹å®šä¸»æœºä¸“用于特定节点角色，请é‡å¤æ­¥éª¤ 7-10。根æ®éœ€è¦å¤šæ¬¡é‡å¤è¿™äº›æ­¥éª¤ã€‚ + +::: + +11. 在 Linux 主机上è¿è¡Œå®Œå‘½ä»¤åŽï¼Œå•å‡»**完æˆ**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +### 3. ä»…é™äºšé©¬é€Šï¼šæ ‡ç­¾èµ„æº + +如果你已将集群é…置为使用 Amazon 作为**云æ供商**，请使用集群 ID 标记你的 AWS 资æºã€‚ + +[Amazon 文档：标记你的 Amazon EC2 资æº](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html) + +:::note + +ä½ å¯ä»¥ä½¿ç”¨ Amazon EC2 实例，而无需在 Kubernetes 中é…置云æ供商。如果你想使用特定的 Kubernetes 云æ供商功能，é…置云æ供商å³å¯ã€‚如需更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 云æ供商](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/)。 + +::: + +以下资æºéœ€è¦ä½¿ç”¨ `ClusterID` 进行标记： + +- **Nodes**：Rancher 中添加的所有主机。 +- **Subnet**：集群使用的å­ç½‘。 +- **Security Group**：用于你的集群的安全组。 + +:::note + +ä¸è¦æ ‡è®°å¤šä¸ªå®‰å…¨ç»„。创建 Elastic Load Balancer 时，标记多个组会导致错误。 + +::: + +应该使用的标签是： + +``` +Key=kubernetes.io/cluster/, Value=owned +``` + +`` å¯ä»¥æ˜¯ä½ é€‰æ‹©çš„任何字符串。但是，必须在你标记的æ¯ä¸ªèµ„æºä¸Šä½¿ç”¨ç›¸åŒçš„字符串。将值设置为 `owned` 会通知集群所有带有 `` 标记的资æºéƒ½ç”±è¯¥é›†ç¾¤æ‹¥æœ‰å’Œç®¡ç†ã€‚ + +如果你在集群之间共享资æºï¼Œä½ å¯ä»¥å°†æ ‡ç­¾æ›´æ”¹ä¸ºï¼š + +``` +Key=kubernetes.io/cluster/CLUSTERID, Value=shared +``` + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md new file mode 100644 index 00000000000..4811fca8fd9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md @@ -0,0 +1,152 @@ +--- +title: 在云厂商的新节点上å¯åŠ¨ Kubernetes +--- + +在 Rancher 中使用节点模æ¿æ¥åˆ›å»º RKE 或 RKE2 集群时，æ¯ä¸ªç”Ÿæˆçš„节点池都会显示在新的**主机池**选项å¡ä¸­ã€‚ä½ å¯ä»¥é€šè¿‡æ‰§è¡Œä»¥ä¸‹æ“作æ¥æŸ¥çœ‹ä¸»æœºæ± ï¼š + +1. 点击**☰ > 集群管ç†**。 +1. å•å‡» RKE 或 RKE2 集群的å称。 + +## RKE 集群 + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)创建节点池。此节点模æ¿å®šä¹‰äº†è¦ç”¨äºŽåœ¨åŸºç¡€è®¾æ–½æ供商或云厂商中å¯åŠ¨èŠ‚点的å‚数。 + +在托管在云厂商的节点池上安装 Kubernetes 的一个好处是，如果一个节点与集群断开连接，Rancher å¯ä»¥è‡ªåŠ¨åˆ›å»ºå¦ä¸€ä¸ªèŠ‚点并将其加入集群，从而确ä¿èŠ‚点池的数é‡ç¬¦åˆè¦æ±‚。 + +å¯ç”¨äºŽåˆ›å»ºèŠ‚点模æ¿çš„云æ供商是由[主机驱动](use-new-nodes-in-an-infra-provider.md#主机驱动)决定的。 + +### èŠ‚ç‚¹æ¨¡æ¿ + +节点模æ¿ä¿å­˜äº†ç”¨äºŽåœ¨ç‰¹å®šäº‘æ供商中é…置节点时è¦ä½¿ç”¨çš„å‚数。这些节点å¯ä»¥ä»Ž UI å¯åŠ¨ã€‚Rancher 使用 [Docker Machine](https://docs.docker.com/machine/) æ¥é…置这些节点。å¯ç”¨äºŽåˆ›å»ºèŠ‚点模æ¿çš„云æ供商å–决于 Rancher 中状æ€æ˜¯ Active 的主机驱动。 + +在 Rancher 中创建节点模æ¿åŽï¼Œæ¨¡æ¿ä¼šè¢«ä¿å­˜ï¼Œä»¥ä¾¿ä½ å¯ä»¥å†æ¬¡ä½¿ç”¨è¯¥æ¨¡æ¿æ¥åˆ›å»ºèŠ‚点池。节点模æ¿ç»‘定到你的登录å。添加模æ¿åŽï¼Œä½ å¯ä»¥å°†å…¶ä»Žç”¨æˆ·é…置文件中删除。 + +#### 节点标签 + +ä½ å¯ä»¥ä¸ºæ¯ä¸ªèŠ‚点模æ¿æ·»åŠ [标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)，这样，使用节点模æ¿åˆ›å»ºçš„节点都会自动带有这些标签。 + +无效标签会阻止å‡çº§ï¼Œæˆ–阻止 Rancher å¯åŠ¨ã€‚有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 + +#### 节点污点 + +ä½ å¯ä»¥ä¸ºæ¯ä¸ªèŠ‚点模æ¿æ·»åŠ [污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)，这样，使用节点模æ¿åˆ›å»ºçš„节点都会自动带有这些污点。 + +由于污点å¯ä»¥åŒæ—¶æ·»åŠ åˆ°èŠ‚点模æ¿å’ŒèŠ‚点池中，因此如果添加了相åŒé”®çš„污点效果没有冲çªï¼Œåˆ™æ‰€æœ‰æ±¡ç‚¹éƒ½å°†æ·»åŠ åˆ°èŠ‚点中。如果存在具有相åŒé”®ä½†ä¸åŒæ•ˆæžœçš„污点，则节点池中的污点将覆盖节点模æ¿ä¸­çš„污点。 + +#### 节点模æ¿çš„管ç†å‘˜æŽ§åˆ¶ + +管ç†å‘˜å¯ä»¥æŽ§åˆ¶æ‰€æœ‰èŠ‚点模æ¿ã€‚现在，管ç†å‘˜å¯ä»¥ç»´æŠ¤ Rancher 中的所有节点模æ¿ã€‚当节点模æ¿æ‰€æœ‰è€…ä¸å†ä½¿ç”¨ Rancher 时，他们创建的节点模æ¿å¯ä»¥ç”±ç®¡ç†å‘˜ç®¡ç†ï¼Œä»¥ä¾¿ç»§ç»­æ›´æ–°å’Œç»´æŠ¤é›†ç¾¤ã€‚ + +è¦è®¿é—®æ‰€æœ‰èŠ‚点模æ¿ï¼Œç®¡ç†å‘˜éœ€è¦æ‰§è¡Œä»¥ä¸‹æ“作： + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 + +**结果**：列出所有节点模æ¿ã€‚ä½ å¯ä»¥é€šè¿‡å•å‡» **â‹®** æ¥ç¼–辑或克隆模æ¿ã€‚ + +### 节点池 + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](#节点模æ¿)创建节点池。 + +节点模æ¿å®šä¹‰äº†èŠ‚点的é…置，例如è¦ä½¿ç”¨çš„æ“作系统ã€CPU æ•°é‡å’Œå†…å­˜é‡ã€‚ + +使用节点池的好处是，如果一个节点被销æ¯æˆ–删除，你å¯ä»¥å¢žåŠ  Active 节点的数é‡æ¥è¡¥å¿ä¸¢å¤±çš„节点。节点池å¯ä»¥å¸®åŠ©ä½ ç¡®ä¿èŠ‚点池的计数符åˆè¦æ±‚。 + +æ¯ä¸ªèŠ‚点池必须分é…一个或多个节点角色。 + +æ¯ä¸ªèŠ‚ç‚¹è§’è‰²ï¼ˆå³ etcdã€controlplane å’Œ worker）都应分é…ç»™ä¸åŒçš„节点池。虽然你å¯ä»¥å°†å¤šä¸ªèŠ‚点角色分é…ç»™åŒä¸€ä¸ªèŠ‚点池，但ä¸è¦åœ¨ç”Ÿäº§é›†ç¾¤ä¸­æ‰§è¡Œæ­¤æ“作。 + +推è的设置： + +- 具有 etcd 角色且计数为 3 的节点池 +- 具有 controlplane 角色且计数至少为 2 的节点池 +- 具有 worker 角色且计数至少为 2 的节点池 + +**离线环境中的 RKE1 下游集群节点**： + +默认情况下，在é…ç½® RKE1 下游集群节点时（例如在 vSphere 中），Rancher 会å°è¯•è¿è¡Œ Docker 安装脚本。但是，Rancher Docker 安装脚本在离线环境中会è¿è¡Œå¤±è´¥ã€‚è¦è§£å†³æ­¤é—®é¢˜ï¼Œå¦‚æžœ Docker 已预安装到 VM é•œåƒä¸Šï¼Œä½ å¯ä»¥é€‰æ‹©åœ¨åˆ›å»ºèŠ‚点模æ¿æ—¶è·³è¿‡å®‰è£… Docker。为此，你å¯ä»¥åœ¨ Rancher UI **引擎选项**下的 `Docker 安装 URL` 下拉列表中选择 **æ— **。 + +
**引擎选项下拉列表**
+ +![引擎选项下拉列表](/img/node-template-engine-options-rke1.png) + +#### 节点池污点 + +如果你没有在节点模æ¿ä¸Šå®šä¹‰[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)，则å¯ä»¥ä¸ºæ¯ä¸ªèŠ‚点池添加污点。将污点添加到节点池的好处是你å¯ä»¥æ›´æ”¹èŠ‚点模æ¿ï¼Œè€Œä¸éœ€è¦å…ˆç¡®ä¿æ±¡ç‚¹å­˜åœ¨äºŽæ–°æ¨¡æ¿ä¸­ã€‚ + +æ¯ä¸ªæ±¡ç‚¹éƒ½å°†è‡ªåŠ¨æ·»åŠ åˆ°èŠ‚点池中已创建的节点。因此，如果你在已有节点的节点池中添加污点，污点ä¸ä¼šåº”用到已有的节点，但是添加到该节点池中的新节点都将获得该污点。 + +如果污点åŒæ—¶æ·»åŠ åˆ°èŠ‚点模æ¿å’ŒèŠ‚点池中，且添加了相åŒé”®çš„污点效果没有冲çªï¼Œåˆ™æ‰€æœ‰æ±¡ç‚¹éƒ½å°†æ·»åŠ åˆ°èŠ‚点中。如果存在具有相åŒé”®ä½†ä¸åŒæ•ˆæžœçš„污点，则节点池中的污点将覆盖节点模æ¿ä¸­çš„污点。 + +#### èŠ‚ç‚¹è‡ªåŠ¨æ›¿æ¢ + +Rancher å¯ä»¥è‡ªåŠ¨æ›¿æ¢èŠ‚点池中无法访问的节点。如果节点在指定的时间中处于 Inactive 状æ€ï¼ŒRancher 将使用该节点池的节点模æ¿æ¥é‡æ–°åˆ›å»ºèŠ‚点。 + +:::caution + +自我修å¤èŠ‚点池的功能帮助你替æ¢æ— çŠ¶æ€åº”用的 worker 节点。ä¸å»ºè®®åœ¨ master 节点或连接了æŒä¹…å·çš„节点的节点池上å¯ç”¨èŠ‚点自动替æ¢ï¼Œå› ä¸ºè™šæ‹Ÿæœºä¼šè¢«ä¸´æ—¶å¤„ç†ã€‚节点池中的节点与集群断开连接时，其æŒä¹…å·å°†è¢«ç ´å，从而导致有状æ€åº”用的数æ®ä¸¢å¤±ã€‚ + +::: + +节点自动替æ¢åŸºäºŽ Kubernetes 节点控制器工作。节点控制器定期检查所有节点的状æ€ï¼ˆå¯é€šè¿‡ `kube-controller` çš„ `--node-monitor-period` 标志é…置）。一个节点ä¸å¯è®¿é—®æ—¶ï¼ŒèŠ‚点控制器将污染该节点。å‘生这ç§æƒ…况时，Rancher 将开始其删除倒计时。你å¯ä»¥é…ç½® Rancher 等待删除节点的时间。如果在删除倒计时结æŸå‰æ±¡ç‚¹æ²¡æœ‰è¢«åˆ é™¤ï¼ŒRancher 将继续删除该节点。Rancher 会根æ®èŠ‚点池设置的数é‡æ¥åˆ›å»ºæ–°çš„节点。 + +#### å¯ç”¨èŠ‚ç‚¹è‡ªåŠ¨æ›¿æ¢ + +创建节点池时，你å¯ä»¥æŒ‡å®š Rancher 替æ¢æ— å“应节点的等待时间（以分钟为å•ä½ï¼‰ã€‚ + +1. 在创建或编辑集群的表å•ä¸­ï¼Œè½¬åˆ°**节点池**。 +1. 转到è¦å¯ç”¨èŠ‚点自动替æ¢çš„节点池。在 **Recreate Unreachable After** 字段中，输入 Rancher 在替æ¢èŠ‚点之å‰åº”该等待节点å“应的分钟数。 +1. 填写表å•çš„其余部分以创建或编辑集群。 + +**结果** ：已为节点池å¯ç”¨èŠ‚点自动替æ¢ã€‚ + +#### ç¦ç”¨èŠ‚ç‚¹è‡ªåŠ¨æ›¿æ¢ + +ä½ å¯ä»¥æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ä»Ž Rancher UI ç¦ç”¨èŠ‚点自动替æ¢ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦ç¦ç”¨èŠ‚点自动替æ¢çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**节点池**部分中，转到è¦å¯ç”¨èŠ‚点自动替æ¢çš„节点池。在 **Recreate Unreachable After** 字段中，输入 0。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：已ç¦ç”¨èŠ‚点池的节点自动替æ¢ã€‚ + +### äº‘å‡­è¯ + +节点模æ¿å¯ä»¥ä½¿ç”¨äº‘凭è¯ï¼Œæ¥å­˜å‚¨ç”¨äºŽåœ¨äº‘æ供商中å¯åŠ¨èŠ‚点的凭è¯ï¼Œå…¶ä¼˜ç‚¹æ˜¯ï¼š + +- 凭è¯ä¼šå­˜å‚¨ä¸ºæ›´å®‰å…¨çš„ Kubernetes 密文，而且你无需æ¯æ¬¡éƒ½è¾“入凭è¯ä¾¿å¯ç¼–辑节点模æ¿ã€‚ + +- 创建云凭è¯åŽï¼Œä½ å¯ä»¥é‡æ–°ä½¿ç”¨è¯¥å‡­è¯æ¥åˆ›å»ºå…¶ä»–节点模æ¿ã€‚ + +- 多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯æ¥åˆ›å»ºèŠ‚点池。如果你的密钥被泄露或过期，则å¯ä»¥åœ¨ä¸€ä¸ªä½ç½®æ›´æ–°äº‘凭è¯ï¼Œä»Žè€Œä¸€æ¬¡æ›´æ–°æ‰€æœ‰ä½¿ç”¨è¯¥å‡­è¯çš„节点模æ¿ã€‚ + +创建云凭è¯åŽï¼Œç”¨æˆ·å¯ä»¥[管ç†åˆ›å»ºçš„云凭è¯](../reference-guides/user-settings/manage-cloud-credentials.md)。 + +### 主机驱动 + +如果你找ä¸åˆ°æƒ³è¦çš„主机驱动，你å¯ä»¥åœ¨ Rancher çš„[内置主机驱动](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#激活åœç”¨ä¸»æœºé©±åŠ¨)中查看并激活它，也å¯ä»¥[添加自定义主机驱动](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#添加自定义主机驱动)。 + +## RKE2 集群 + +Rancher 2.6 支æŒç›´æŽ¥ä½¿ç”¨ Rancher UI é…ç½® [RKE2](https://docs.rke2.io/) 集群。RKE2，也称为 RKE Government，是一个完全符åˆæ ‡å‡†çš„ Kubernetes å‘行版，它专注于安全性和åˆè§„性。 + +:::note + +对于 RKE2 集群模æ¿ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md#rke2-集群模æ¿)了解更多信æ¯ã€‚ + +::: + +### 节点角色 + +RKE2 CLI 公开了 `server` å’Œ `agent` 两个角色，它们分别代表 Kubernetes 节点角色 `etcd` + `controlplane` å’Œ `worker`。通过 Rancher 2.6 中的 RKE2 集æˆï¼ŒRKE2 节点池å¯ä»¥åˆ†é…更细粒度的角色，例如 `etcd` å’Œ `controlplane`。 + +ä½ å¯ä»¥åœ¨ RKE2 CLI 中使用标志和节点污染，æ¥æŽ§åˆ¶è°ƒåº¦å·¥ä½œè´Ÿè½½å’Œ Kubernetes master 节点的ä½ç½®ï¼Œä»Žè€Œä½¿ç”¨ `etcd`，`controlplane` å’Œ `worker` 节点功能。这些角色没有在 RKE2 CLI 中实现为第一级角色的原因是，RKE2 被概念化为一组原始构建å—，使用 Rancher 等编排系统得到最佳利用。 + +在 Rancher 中实现这三个节点角色，表示 Rancher 管ç†çš„ RKE2 集群能够轻æ¾ä½¿ç”¨ä¸º RKE 集群推è的相åŒæž¶æž„的所有最佳实践。 + +在[推è的集群架构](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md)中，我们概述了æ¯ä¸ªè§’色集群应该有多少节点： + +- 至少拥有三个角色为 etcd 的节点，æ¥ç¡®ä¿å¤±åŽ»ä¸€ä¸ªèŠ‚点时ä»èƒ½å­˜æ´»ã€‚ +- 至少两个节点具有 controlplane 角色，以实现主组件高å¯ç”¨æ€§ã€‚ +- 至少两个具有 worker 角色的节点，用于在节点故障时é‡æ–°å®‰æŽ’工作负载。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-windows-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-windows-clusters.md new file mode 100644 index 00000000000..59d893a6cc8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/use-windows-clusters.md @@ -0,0 +1,277 @@ +--- +title: 在 Windows 集群上å¯åŠ¨ Kubernetes +--- + +使用 Rancher é…ç½®[自定义集群](use-existing-nodes.md)时，Rancher 通过 RKE（Rancher Kubernetes Engine）在现有节点上安装 Kubernetes。 + +在使用 Rancher é…置的 Windows 集群中，集群必须åŒæ—¶åŒ…å« Linux å’Œ Windows 节点。Kubernetes controlplane åªèƒ½è¿è¡Œåœ¨ Linux 节点上，Windows 节点åªèƒ½æœ‰ Worker 角色。Windows 节点åªèƒ½ç”¨äºŽéƒ¨ç½²å·¥ä½œè´Ÿè½½ã€‚ + +Windows 集群的其他è¦æ±‚如下： + +- åªæœ‰åœ¨åˆ›å»ºé›†ç¾¤æ—¶å¯ç”¨äº† Windows 支æŒçš„集群æ‰èƒ½æ·»åŠ  Windows 节点。无法为现有集群å¯ç”¨ Windows 支æŒã€‚ +- éœ€è¦ Kubernetes 1.15+。 +- 必须使用 Flannel 网络æ供商。 +- Windows 节点必须有 50 GB çš„ç£ç›˜ç©ºé—´ã€‚ + +有关完整的è¦æ±‚列表，请å‚阅[本节](#windows-集群的è¦æ±‚)。 + +æœ‰å…³æ”¯æŒ Windows çš„ Kubernetes 功能摘è¦ï¼Œè¯·å‚阅[在 Windows 中使用 Kubernetes 支æŒçš„功能和é™åˆ¶](https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#supported-functionality-and-limitations)çš„ Kubernetes 文档，或[在 Kubernetes 中调度 Windows 容器的指å—](https://kubernetes.io/docs/setup/production-environment/windows/user-guide-windows-containers/)。 + +### RKE2 Windows + +RKE2 é…置功能还包括在 Windows 集群上安装 RKE2。RKE2 çš„ Windows 功能包括： + +- ç”± containerd æ供支æŒçš„使用 RKE2 çš„ Windows 容器 +- 直接从 Rancher UI é…ç½® Windows RKE2 自定义集群 +- 用于 Windows RKE2 自定义集群的 Calico CNI +- 技术预览包å«äº† Windows Server çš„ SAC 版本（2004 å’Œ 20H2） + +è¦ä½¿ Windows æ”¯æŒ RKE2 自定义集群，请选择 Calico 作为 CNI。 + +:::note + +默认情况下，Rancher å…许 Windows 工作负载 pod 部署在 Windows å’Œ Linux Worker 节点上。在 RKE2 中创建混åˆé›†ç¾¤æ—¶ï¼Œä½ å¿…须编辑 Chart 中的 `nodeSelector`，从而将 Pod 放置到兼容的 Windows 节点上。有关如何使用 `nodeSelector` å°† pod 分é…给节点的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)。 + +::: + +- Kubernetes v1.24.1 åŠæ›´é«˜ç‰ˆæœ¬æ”¯æŒ Windows RKE2 中的 HostProcess 容器。有关详细信æ¯ï¼Œè¯·å‚阅[上游文档](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/)。 + +## Windows 集群的è¦æ±‚ + +网络ã€æ“作系统和 Docker 的一般节点è¦æ±‚与 [Rancher 安装](installation-requirements.md)的节点è¦æ±‚相åŒã€‚ + +### æ“作系统和 Docker è¦æ±‚ + +我们对 Windows Server å’Œ Windows 容器的支æŒä¸Ž LTSC（长期æœåŠ¡æ¸ é“）和 SAC（åŠå¹´æ¸ é“）的 Microsoft 官方生命周期相匹é…。 + +有关 Windows Server 的支æŒç”Ÿå‘½å‘¨æœŸçš„日期，请å‚阅 [Microsoft 文档](https://docs.microsoft.com/en-us/windows-server/get-started/windows-server-release-info)。 + +### Kubernetes 版本 + +éœ€è¦ Kubernetes v1.15+。 + +如果你在 Windows Server 20H2 Standard Core 上使用 Kubernetes v1.21，则必须在节点上安装补ä¸â€œ2019-08 Servicing Stack Update for Windows Serverâ€ã€‚ + +### 节点è¦æ±‚ + +集群中的主机至少需è¦ï¼š + +- 2 æ ¸ CPU +- 5 GB 内存 +- 50 GB ç£ç›˜ç©ºé—´ + +Rancher ä¸ä¼šé…ç½®ä¸æ»¡è¶³è¦æ±‚的节点。 + +### 网络è¦æ±‚ + +在é…置新集群之å‰ï¼Œè¯·ç¡®ä¿ä½ å·²ç»åœ¨æŽ¥æ”¶å…¥ç«™ç½‘络æµé‡çš„设备上安装了 Rancher。这是集群节点与 Rancher 通信所必需的。如果你尚未安装 Rancher，请在继续阅读本指å—之å‰å…ˆå‚阅[安装文档](installation-and-upgrade.md)进行安装。 + +Rancher 仅支æŒä½¿ç”¨ Flannel 作为网络æ供商的 Windows。 + +有两个网络选项：[**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) å’Œ [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan)。默认选项是 **VXLAN (Overlay)** 模å¼ã€‚ + +对于 **Host Gateway (L2bridge)** 网络，最好为所有节点使用相åŒçš„第 2 层网络。å¦åˆ™ï¼Œä½ éœ€è¦ä¸ºå®ƒä»¬é…置路由规则。有关详细信æ¯ï¼Œè¯·å‚阅[é…置云托管 VM 路由的文档](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#云托管虚拟机的路由é…ç½®)。如果你使用的是 Amazon EC2ã€Google GCE 或 Azure 虚拟机，你需è¦[ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#ç¦ç”¨ç§æœ‰-ip-地å€æ£€æŸ¥)。 + +对于 **VXLAN (Overlay)** 网络，你必须安装 [KB4489899](https://support.microsoft.com/en-us/help/4489899) 修补程åºã€‚大多数云托管的 VM å·²ç»å…·æœ‰æ­¤ä¿®è¡¥ç¨‹åºã€‚ + +如果你在为 AWS 虚拟ç§æœ‰äº‘é…ç½® DHCP 选项集，请注æ„，你åªèƒ½åœ¨ `domain-name` 选项字段中指定一个域å。详情请å‚è§ [DHCP 选项文档](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_DHCP_Options.html)。 + +:::note + +一些 Linux æ“作系统支æŒä»¥ç©ºæ ¼åˆ†éš”的多个域å。但是，其他 Linux æ“作系统和 Windows 将该值视为å•ä¸ªåŸŸå，从而导致æ„外错误。如果你的 DHCP 选项集与具有多个æ“作系统实例的 VPC 相关è”，请仅指定一个域å。 + +::: + +### 带有 ESXi 6.7u2 åŠæ›´é«˜ç‰ˆæœ¬çš„ vSphere 上的 Rancher + +如果你在带有 ESXi 6.7u2 或更高版本的 VMware vSphere 上使用 Rancher，并使用 Red Hat Enterprise Linux 8.3ã€CentOS 8.3 或 SUSE Enterprise Linux 15 SP2 或更高版本，你需è¦ç¦ç”¨ `vmxnet3` 虚拟网络适é…器硬件å¸è½½åŠŸèƒ½ã€‚å¦åˆ™ï¼Œä¸åŒé›†ç¾¤èŠ‚点上的 pod 之间的所有网络连接会因为超时错误而失败。从 Windows pod 到在 Linux 节点上è¿è¡Œçš„关键æœåŠ¡ï¼ˆä¾‹å¦‚ CoreDNS）的所有连接也将失败。外部连接也å¯èƒ½å¤±è´¥ã€‚出现这个问题的原因是 Linux å‘行版在 `vmxnet3` 中å¯ç”¨äº†ç¡¬ä»¶å¸è½½åŠŸèƒ½ï¼Œè€Œä¸” `vmxnet3` 硬件å¸è½½åŠŸèƒ½ä¸­å­˜åœ¨ä¸€ä¸ªä¼šä¸¢å¼ƒå®¢æˆ·è¦†ç›–æµé‡çš„æ•°æ®åŒ…çš„ bug。è¦è§£å†³æ­¤é—®é¢˜ï¼Œå¿…é¡»ç¦ç”¨ `vmxnet3` 硬件å¸è½½åŠŸèƒ½ã€‚此设置ä¸ä¼šåœ¨é‡å¯åŽç»§ç»­ç”Ÿæ•ˆï¼Œå› æ­¤éœ€è¦åœ¨æ¯æ¬¡å¯åŠ¨æ—¶ç¦ç”¨ã€‚推èçš„åšæ³•æ˜¯åœ¨ `/etc/systemd/system/disable_hw_offloading.service` 中创建一个 systemd å•å…ƒæ–‡ä»¶ï¼Œè¿™ä¼šåœ¨å¯åŠ¨æ—¶ç¦ç”¨ `vmxnet3` 硬件å¸è½½åŠŸèƒ½ã€‚ç¦ç”¨ `vmxnet3` 硬件å¸è½½åŠŸèƒ½çš„示例 systemd å•å…ƒæ–‡ä»¶å¦‚下所示。注æ„，`` 必须自定义为主机的 `vmxnet3` 网络接å£ï¼Œå¦‚ `ens192`： + +``` +[Unit] +Description=Disable vmxnet3 hardware offloading feature + +[Service] +Type=oneshot +ExecStart=ethtool -K tx-udp_tnl-segmentation off +ExecStart=ethtool -K tx-udp_tnl-csum-segmentation off +StandardOutput=journal + +[Install] +WantedBy=multi-user.target +``` +然åŽåœ¨ systemd å•å…ƒæ–‡ä»¶ä¸Šè®¾ç½®é€‚当的æƒé™ï¼š +``` +chmod 0644 /etc/systemd/system/disable_hw_offloading.service +``` +最åŽï¼Œå¯ç”¨ systemd æœåŠ¡ï¼š +``` +systemctl enable disable_hw_offloading.service +``` + +### 架构è¦æ±‚ + +Kubernetes 集群管ç†èŠ‚点（`etcd` å’Œ `controlplane`）必须è¿è¡Œåœ¨ Linux 节点上。 + +部署工作负载的 `worker` 节点通常是 Windows 节点，但必须至少有一个 `worker` 节点è¿è¡Œåœ¨ Linux 上，æ‰èƒ½æŒ‰é¡ºåºè¿è¡Œ Rancher Cluster Agentã€DNSã€Metrics Server å’Œ Ingress 相关容器。 + +#### 推è架构 + +我们推è下表中列出的三节点架构，但你始终å¯ä»¥æ·»åŠ é¢å¤–çš„ Linux å’Œ Windows worker 节点æ¥æ‰©å±•é›†ç¾¤ï¼Œä»Žè€Œå®žçŽ°å†—余： + +| 节点 | æ“作系统 | Kubernetes 集群角色 | 用途 | +| ------ | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| 节点 1 | Linux（推è Ubuntu Server 18.04） | controlplane, etcd, worker | ç®¡ç† Kubernetes 集群 | +| 节点 2 | Linux（推è Ubuntu Server 18.04） | Worker | 支æŒé›†ç¾¤çš„ Rancher Cluster Agentã€Metrics Serverã€DNS å’Œ Ingress | +| 节点 3 | Windows（Windows Server 核心版本 1809 或更高版本） | Worker | è¿è¡Œ Windows 容器 | + +### 容器è¦æ±‚ + +Windows è¦æ±‚容器的版本必须与部署容器的 Windows Server 的版本一致。因此，你必须在 Windows Server 核心版本 1809 或更高版本上构建容器。如果你已ç»ä½¿ç”¨æ—©æœŸçš„ Windows Server 核心版本构建了容器，则必须使用 Windows Server 核心版本 1809 或更高版本é‡æ–°æž„建容器。 + +### 云æ供商è¦æ±‚ + +如果你在集群中设置了 Kubernetes 云æ供商，则需è¦è¿›è¡Œä¸€äº›é¢å¤–çš„æ“作。如果你想使用云æ供商的功能，例如为集群自动é…置存储ã€è´Ÿè½½å‡è¡¡å™¨æˆ–其他基础设施，你å¯èƒ½éœ€è¦è®¾ç½®äº‘æ供商。有关如何é…置满足æ¡ä»¶çš„云æ供商集群节点，请å‚阅[此页é¢](../pages-for-subheaders/set-up-cloud-providers.md)。 + +如果你的云æ供商是 GCE（Google Compute Engine），则必须执行以下æ“作： + +- 按照[步骤](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md) 在`cluster.yml` 中å¯ç”¨ GCE 云æ供商。 +- 在 Rancher 中é…置集群时，在 Rancher UI 中选择**自定义云æ供商**作为云æ供商。 + +## æ•™ç¨‹ï¼šå¦‚ä½•åˆ›å»ºæ”¯æŒ Windows 的集群 + +本教程æ述了如何使用[推è架构](#推è架构)中的三个节点创建由 Rancher é…置的集群。 + +在现有节点上使用 Rancher é…置集群时，你需è¦åœ¨æ¯ä¸ªèŠ‚点上安装 [Rancher Agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) æ¥å°†èŠ‚点添加到集群中。在 Rancher UI 中创建或编辑集群时，你会看到一个**自定义节点è¿è¡Œå‘½ä»¤**，你å¯ä»¥åœ¨æ¯å°æœåŠ¡å™¨ä¸Šè¿è¡Œè¯¥å‘½ä»¤ï¼Œä»Žè€Œå°†æœåŠ¡å™¨æ·»åŠ åˆ°é›†ç¾¤ä¸­ã€‚ + +è¦è®¾ç½®æ”¯æŒ Windows 节点和容器的集群，你需è¦å®Œæˆä»¥ä¸‹ä»»åŠ¡ï¼š + + +### 1. é…置主机 + +è¦åœ¨å…·æœ‰ Windows 支æŒçš„现有节点上é…置集群，请准备好你的主机。 + +主机å¯ä»¥æ˜¯ï¼š + +- 云托管的虚拟机 +- 虚拟化集群中的虚拟机 +- 裸金属æœåŠ¡å™¨ + +ä½ å°†é…置三个节点： + +- 一个 Linux èŠ‚ç‚¹ï¼Œç”¨äºŽç®¡ç† Kubernetes controlplane 并存储你的 `etcd`。 +- 第二个 Linux 节点，它将作为 worker 节点。 +- Windows 节点，它将作为 worker 节点è¿è¡Œ Windows 容器。 + +| 节点 | æ“作系统 | +| ------ | ------------------------------------------------------------ | +| 节点 1 | Linux（推è Ubuntu Server 18.04） | +| 节点 2 | Linux（推è Ubuntu Server 18.04） | +| 节点 3 | Windows（Windows Server 核心版本 1809 或更高版本） | + +如果你的节点托管在**云æ供商**上，并且你需è¦è‡ªåŠ¨åŒ–支æŒï¼ˆä¾‹å¦‚è´Ÿè½½å‡è¡¡å™¨æˆ–æŒä¹…存储设备），你的节点还需è¦æ»¡è¶³é¢å¤–çš„é…ç½®è¦æ±‚。详情请å‚è§[选择云æ供商](../pages-for-subheaders/set-up-cloud-providers.md)。 + +### 2. 在现有节点上创建集群 + +在现有节点上创建 Windows 集群的说明与一般[创建自定义集群的说明](use-existing-nodes.md)éžå¸¸ç›¸ä¼¼ï¼Œä½†æœ‰ä¸€äº›ç‰¹å®šäºŽ Windows çš„è¦æ±‚。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡»**自定义**。 +1. 在**集群å称**字段中输入集群的å称。 +1. 在 **Kubernetes 版本**下拉èœå•ä¸­ï¼Œé€‰æ‹© v1.19 或更高版本。 +1. 在**网络æ供商**字段中，选择 **Flannel**。 +1. 在 **Windows 支æŒ**中，å•å‡»**å¯ç”¨**。 +1. å¯é€‰ï¼šå¯ç”¨ Windows 支æŒåŽï¼Œä½ å°†èƒ½å¤Ÿé€‰æ‹© Flannel åŽç«¯æ¨¡å¼ã€‚有两个网络选项：[**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) å’Œ [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan)。默认选项是 **VXLAN (Overlay)** 模å¼ã€‚ +1. 点击**下一步**。 + +:::note é‡è¦æ示： + +对于 Host Gateway (L2bridge) 网络，最好为所有节点使用相åŒçš„第 2 层网络。å¦åˆ™ï¼Œä½ éœ€è¦ä¸ºå®ƒä»¬é…置路由规则。有关详细信æ¯ï¼Œè¯·å‚阅[é…置云托管 VM 路由的文档](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#云托管虚拟机的路由é…ç½®)。如果你使用的是 Amazon EC2ã€Google GCE 或 Azure 虚拟机，你需è¦[ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#ç¦ç”¨ç§æœ‰-ip-地å€æ£€æŸ¥)。 + +::: + +### 3. 将节点添加到集群 + +本节介ç»å¦‚何将 Linux å’Œ Worker 节点注册到集群。你将在æ¯ä¸ªèŠ‚点上è¿è¡Œä¸€ä¸ªå‘½ä»¤ï¼Œè¯¥å‘½ä»¤å°†å®‰è£… Rancher Agent 并å…许 Rancher 管ç†æ¯ä¸ªèŠ‚点。 + +#### 添加 Linux master 节点 + +在本节中，你需è¦åœ¨ Rancher UI 上填写表å•ä»¥èŽ·å–自定义命令，从而在 Linux master 节点上安装 Rancher Agent。然åŽï¼Œå¤åˆ¶è¯¥å‘½ä»¤å¹¶åœ¨ Linux master 节点上è¿è¡Œå‘½ä»¤ï¼Œä»Žè€Œåœ¨é›†ç¾¤ä¸­æ³¨å†Œè¯¥èŠ‚点。 + +集群中的第一个节点应该是具有 **controlplane** å’Œ **etcd** 角色的 Linux 主机。至少必须为此节点å¯ç”¨è¿™ä¸¤ä¸ªè§’色，并且必须先将此节点添加到集群中，然åŽæ‰èƒ½æ·»åŠ  Windows 主机。 + +1. 在**节点æ“作系统**中，å•å‡» **Linux**。 +1. 在**节点角色**中，至少选择 **etcd** å’Œ **controlplane**。推è选择所有的三个角色。 +1. å¯é€‰ï¼šå¦‚果点击**显示高级选项**，你å¯ä»¥è‡ªå®šä¹‰ [Rancher Agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) å’Œ[节点标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)的设置。 +1. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. SSH 到你的 Linux 主机，然åŽè¿è¡Œå¤åˆ¶åˆ°å‰ªè´´æ¿çš„命令。 +1. 完æˆé…ç½® Linux 节点åŽï¼Œé€‰æ‹©**完æˆ**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +节点å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ‰èƒ½æ³¨å†Œåˆ°é›†ç¾¤ä¸­ã€‚ + +#### 添加 Linux Worker 节点 + +在本节中，我们通过è¿è¡Œå‘½ä»¤å°† Linux Worker 节点注册到集群中。 + +在åˆå§‹é…置集群之åŽï¼Œä½ çš„集群åªæœ‰ä¸€ä¸ª Linux 主机。接下æ¥ï¼Œæˆ‘们添加å¦ä¸€ä¸ª Linux `worker` 主机，用于支æŒé›†ç¾¤çš„ _Rancher Cluster Agent_ã€_Metrics Server_ã€_DNS_ å’Œ _Ingress_。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到你创建的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. å‘下滚动到**节点æ“作系统**。选择 **Linux**。 +1. 在**自定义节点è¿è¡Œå‘½ä»¤**中，转到**节点选项**并选择 **Worker** 角色。 +1. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. 使用远程终端连接登录到你的 Linux 主机。粘贴剪贴æ¿çš„命令并è¿è¡Œã€‚ +1. 在 **Rancher**中，å•å‡»**ä¿å­˜**。 + +**结果**：**Worker** 角色已安装在你的 Linux ä¸»æœºä¸Šï¼Œå¹¶ä¸”èŠ‚ç‚¹ä¼šå‘ Rancher 注册。节点å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ‰èƒ½æ³¨å†Œåˆ°é›†ç¾¤ä¸­ã€‚ + +:::note + +Linux Worker 节点上的污点 + +以下污点将添加集群中的 Linux Worker 节点中。将此污点添加到 Linux Worker 节点åŽï¼Œæ·»åŠ åˆ° Windows 集群的任何工作负载都将自动调度到 Windows Worker 节点。如果想将工作负载专门调度到 Linux Worker 节点上，则需è¦ä¸ºè¿™äº›å·¥ä½œè´Ÿè½½æ·»åŠ å®¹å¿åº¦ã€‚ + +| 污点键 | 污点值 | 污点效果 | +| -------------- | ----------- | ------------ | +| `cattle.io/os` | `linux` | `NoSchedule` | + +::: + +#### 添加 Windows Worker 节点 + +在本节中，我们通过è¿è¡Œå‘½ä»¤å°† Windows Worker 节点注册到集群中。 + +ä½ å¯ä»¥é€šè¿‡ç¼–辑集群并选择 **Windows** 选项，从而将 Windows 主机添加到集群中。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到你创建的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. å‘下滚动到**节点æ“作系统**。选择 **Windows**。注æ„：你将看到 **worker** 角色是唯一å¯ç”¨çš„角色。 +1. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. 使用你喜欢的工具（例如 [Microsoft 远程桌é¢](https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-clients)）登录到 Windows 主机。在 **Command Prompt (CMD)** 中è¿è¡Œå¤åˆ¶åˆ°å‰ªè´´æ¿çš„命令。 +1. 在 Rancher 中，å•å‡»**ä¿å­˜**。 +1. å¯é€‰ï¼šå¦‚æžœè¦å‘集群添加更多 Windows 节点，请é‡å¤è¿™äº›æ“作。 + +**结果**：**Worker** 角色已安装在你的 Windows ä¸»æœºä¸Šï¼Œå¹¶ä¸”èŠ‚ç‚¹ä¼šå‘ Rancher 注册。节点å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ‰èƒ½æ³¨å†Œåˆ°é›†ç¾¤ä¸­ã€‚你现在已拥有一个 Windows Kubernetes 集群。 + +### å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher Server 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 + +## Azure 中存储类的é…ç½® + +如果你的节点使用 Azure VM，则å¯ä»¥ä½¿ç”¨ [Azure 文件](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv)作为集群的存储类（StorageClass）。详情请å‚è§[此部分](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/user-settings.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/user-settings.md new file mode 100644 index 00000000000..5ed627b7b07 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/user-settings.md @@ -0,0 +1,15 @@ +--- +title: 用户设置 +--- + +在 Rancher 中，æ¯ä¸ªç”¨æˆ·éƒ½æœ‰å¾ˆå¤šä¸Žç™»å½•ç›¸å…³çš„设置，例如个人å好ã€API 密钥等。你å¯ä»¥ä»Ž**用户设置**èœå•ä¸­é…置这些设置。你å¯ä»¥å•å‡»ä¸»èœå•ä¸­çš„头åƒæ¥æ‰“开此èœå•ã€‚ + +![用户设置èœå•](/img/user-settings.png) + +å¯ç”¨çš„用户设置包括： + +- [API & 密钥](../reference-guides/user-settings/api-keys.md)：如果你想以编程方å¼ä¸Ž Rancher 交互，你需è¦ä¸€ä¸ª API 密钥。你å¯ä»¥æŒ‰ç…§æœ¬èŠ‚中的说明获å–密钥。 +- [云凭è¯](../reference-guides/user-settings/manage-cloud-credentials.md)：管ç†[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)使用的云凭è¯ï¼Œä»Žè€Œ[为集群é…置节点](../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 +- [节点模æ¿](../reference-guides/user-settings/manage-node-templates.md)ï¼šç®¡ç† [Rancher 用æ¥ä¸ºé›†ç¾¤é…置节点](../pages-for-subheaders/launch-kubernetes-with-rancher.md)的模æ¿ã€‚ +- [å好设置](../reference-guides/user-settings/user-preferences.md)：设置 Rancher UI 的表é¢é¦–选项。 +- 登出：结æŸä½ çš„用户会è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/vsphere.md new file mode 100644 index 00000000000..db206b47f96 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/vsphere.md @@ -0,0 +1,56 @@ +--- +title: 创建 vSphere 集群 +description: 使用 Rancher 创建 vSphere 集群。集群å¯èƒ½åŒ…括具有ä¸åŒå±žæ€§çš„ VM 组，这些属性å¯ç”¨äºŽç»†ç²’度控制节点的大å°ã€‚ +--- + +import YouTube from '@site/src/components/YouTube' + +ä½ å¯ä»¥ç»“åˆä½¿ç”¨ Rancher 与 vSphere，从而在本地体验云环境的æ“作。 + +Rancher å¯ä»¥åœ¨ vSphere 中é…置节点并在其上安装 Kubernetes。在 vSphere 中创建 Kubernetes 集群时，Rancher 首先与 vCenter API 通信æ¥é…置指定数é‡çš„虚拟机。然åŽåœ¨å®ƒä»¬ä¹‹ä¸Šå®‰è£… Kubernetes。 + +vSphere 集群å¯èƒ½ç”±å¤šç»„具有ä¸åŒå±žæ€§ï¼ˆä¾‹å¦‚内存或 vCPU æ•°é‡ï¼‰çš„ VM 组æˆã€‚è¿™ç§åˆ†ç»„å…许对æ¯ä¸ª Kubernetes 角色的节点大å°è¿›è¡Œç»†ç²’度控制。 + +## Rancher 2.3 中的 vSphere 增强功能 + +我们更新了 vSphere 节点模æ¿ï¼Œä½¿ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹å¢žå¼ºåŠŸèƒ½åœ¨æœ¬åœ°ä½“验云æ“作： + +### 自我修å¤çš„节点池 + +使用 Rancher é…ç½® vSphere 节点的最大优势之一，是å…许你在本地集群中使用 Rancher 的自我修å¤èŠ‚点池（也称为[节点自动替æ¢åŠŸèƒ½](use-new-nodes-in-an-infra-provider.md#节点自动替æ¢)）。自我修å¤èŠ‚点池的功能帮助你替æ¢æ— çŠ¶æ€åº”用的 worker 节点。当 Rancher 使用节点模æ¿é…置节点时，Rancher å¯ä»¥è‡ªåŠ¨æ›¿æ¢æ— æ³•è®¿é—®çš„节点。 + +:::caution + +ä¸å»ºè®®åœ¨ master 节点或连接了æŒä¹…å·çš„节点的节点池上å¯ç”¨èŠ‚点自动替æ¢ï¼Œå› ä¸ºè™šæ‹Ÿæœºä¼šè¢«ä¸´æ—¶å¤„ç†ã€‚节点池中的节点与集群断开连接时，其æŒä¹…å·å°†è¢«ç ´å，从而导致有状æ€åº”用的数æ®ä¸¢å¤±ã€‚ + +::: + +### 实例和调度的动æ€å¡«å……选项 + +vSphere 的节点模æ¿å·²æ›´æ–°ã€‚当你使用 vSphere 凭è¯åˆ›å»ºèŠ‚点模æ¿æ—¶ï¼Œè¯¥æ¨¡æ¿ä¼šè‡ªåŠ¨å¡«å……你在 vSphere 控制å°ä¸­å¯ä»¥è®¿é—®çš„相åŒçš„虚拟机é…置选项。 + +è¦å¡«å……的字段设置需è¦æ»¡è¶³[先决æ¡ä»¶](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md#vsphere-中的准备工作)。 + +### 更多支æŒçš„æ“作系统 + +ä½ å¯ä»¥ä½¿ç”¨ä»»ä½•æ”¯æŒ `cloud-init` çš„æ“作系统æ¥é…ç½® VM。[cloud config](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) ä»…æ”¯æŒ YAML æ ¼å¼ã€‚ + +### 2.3.3 节点模æ¿åŠŸèƒ½çš„è§†é¢‘ä»‹ç» + +在这段 YouTube 视频中，我们演示了如何使用新的节点模æ¿åœ¨æœ¬åœ°çŽ¯å¢ƒä½“验云环境一样的æ“作。 + + + +## 创建 vSphere 集群 + +在[本节](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md)中，你将学习如何使用 Rancher 在 vSphere 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +## é…置存储 + +有关如何使用 Rancher 在 vSphere 中é…置存储的示例，请å‚阅[本节](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md)。è¦åœ¨ vSphere 中动æ€é…置存储，你必须å¯ç”¨ vSphere 云æ供商。请å‚阅[树内 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 + +## å¯ç”¨ vSphere 云æ供商 + +在 Rancher 中设置云æ供商时，Rancher Server å¯ä»¥è‡ªåŠ¨ä¸ºé›†ç¾¤é…置新的基础设施，包括新节点或æŒä¹…存储设备。 + +有关å¯ç”¨ vSphere 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[树内 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/workloads-and-pods.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/workloads-and-pods.md new file mode 100644 index 00000000000..d2f6bba4258 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/pages-for-subheaders/workloads-and-pods.md @@ -0,0 +1,78 @@ +--- +title: "Kubernetes 工作负载和 Pod" +description: "了解在 Kubernetes 中构建å¤æ‚容器化应用程åºçš„两ç§ç»“构：Kubernetes 工作负载和 Pod" +--- + +ä½ å¯ä»¥ä½¿ç”¨ä¸¤ç§åŸºæœ¬ç»“构（pod 和工作负载）在 Kubernetes 中构建å¤æ‚的容器化应用程åºã€‚构建应用程åºåŽï¼Œä½ å¯ä»¥ä½¿ç”¨ç¬¬ä¸‰ç§ç»“构（service）在集群中或互è”网上公开应用程åºã€‚ + +### Pod + +[_Pod_](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) 是一个或多个共享网络命å空间和存储å·çš„容器。大多数 pod åªæœ‰ä¸€ä¸ªå®¹å™¨ã€‚因此，我们讨论的 _pod_ 通常等åŒäºŽ _容器_。扩展 pod çš„æ–¹å¼ä¸Žæ‰©å±•å®¹å™¨çš„æ–¹å¼ç›¸åŒï¼Œå³é…置实现æœåŠ¡çš„åŒä¸€ pod 的多个实例。通常，Pod 会根æ®å·¥ä½œè´Ÿè½½è¿›è¡Œæ‰©å±•å’Œç®¡ç†ã€‚ + +### 工作负载 + +_工作负载_ 是为 pod 设置部署规则的对象。Kubernetes 基于这些规则执行部署，并根æ®åº”用程åºçš„当å‰çŠ¶æ€æ¥æ›´æ–°å·¥ä½œè´Ÿè½½ã€‚ +工作负载让你å¯ä»¥å®šä¹‰åº”用程åºè°ƒåº¦ã€æ‰©å±•å’Œå‡çº§çš„规则。 + +#### 工作负载类型 + +Kubernetes 将工作负载分为ä¸åŒçš„类型。Kubernetes 支æŒçš„最æµè¡Œçš„类型是： + +- [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) + + _Deployment_ 最适åˆç”¨äºŽæ— çŠ¶æ€åº”用程åºï¼ˆå³ä¸éœ€è¦ç»´æŠ¤å·¥ä½œè´Ÿè½½çš„状æ€ï¼‰ã€‚ç”± Deployment 类型工作负载管ç†çš„ Pod 是独立且一次性的。如果 pod 中断了，Kubernetes 会删除该 pod 然åŽé‡æ–°åˆ›å»ºå®ƒã€‚一个示例应用程åºæ˜¯ Nginx Web æœåŠ¡å™¨ã€‚ + +- [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + + 与 Deployment 相比，_StatefulSet_ 最适åˆåœ¨éœ€è¦ç»´æŠ¤èº«ä»½å’Œå­˜å‚¨æ•°æ®çš„应用程åºä¸­ä½¿ç”¨ã€‚适用的应用程åºç±»ä¼¼äºŽ Zookeeper（一个需è¦æ•°æ®åº“进行存储的应用程åºï¼‰ã€‚ + +- [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) + + _Daemonset_ ç¡®ä¿é›†ç¾¤ä¸­çš„æ¯ä¸ªèŠ‚点都è¿è¡Œ pod 的副本。如果你需è¦æ”¶é›†æ—¥å¿—或监控节点性能，这ç§ç±»ä¼¼ daemon 的工作负载效果是最好的。 + +- [Job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) + + _Job_ å¯åŠ¨ä¸€ä¸ªæˆ–多个 Pod 并确ä¿æŒ‡å®šæ•°é‡çš„ Pod 能æˆåŠŸç»ˆæ­¢ã€‚Job 最好用于è¿è¡Œæœ‰é™ä»»åŠ¡è‡³å®ŒæˆçŠ¶æ€ï¼Œè€Œä¸æ˜¯ç®¡ç†æ­£åœ¨è¿›è¡Œçš„应用程åºçš„所需状æ€ã€‚ + +- [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) + + _CronJobs_ 与 Job 类似。但是，CronJob 会基于 cron 的计划è¿è¡Œåˆ°å®ŒæˆçŠ¶æ€ã€‚ + +### Services + +在许多用例中，工作负载必须： + +- 由集群中的其他工作负载访问。 +- 暴露给外部。 + +ä½ å¯ä»¥é€šè¿‡åˆ›å»ºä¸€ä¸ª _Service_ 实现这些目的。Service 使用[选择器/标签（查看代ç ç¤ºä¾‹ï¼‰](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#service-and-replicationcontroller)æ¥æ˜ å°„到底层工作负载的 pod。Rancher UI 使用你选择的æœåŠ¡ç«¯å£å’Œç±»åž‹æ¥è‡ªåŠ¨åˆ›å»º service 以åŠå·¥ä½œè´Ÿè½½ï¼Œä»Žè€Œç®€åŒ–此映射过程。 + +#### Service 类型 + +Rancher 中有几ç§å¯ç”¨çš„ Service 类型。以下æè¿°æ¥è‡ª [Kubernetes 文档](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types)。 + +- **ClusterIP** + + > 在集群内部 IP 上公开 Service。如果你选择此值，Service åªèƒ½ä»Žé›†ç¾¤å†…访问。这是默认的 `ServiceType`。 + +- **NodePort** + + > 在æ¯ä¸ªèŠ‚点 IP 上的é™æ€ç«¯å£ï¼ˆ`NodePort`）上暴露 Service。`ClusterIP` service 是自动创建的，而 `NodePort` service 会路由到 ClusterIP service。你å¯ä»¥é€šè¿‡è¯·æ±‚ `:` 在集群外部è”ç³» `NodePort` service。 + +- **LoadBalancer** + + > 使用云æ供商的负载å‡è¡¡å™¨å‘外部公开æœåŠ¡ã€‚`NodePort` å’Œ `ClusterIP` service 是自动创建的，外部负载å‡è¡¡å™¨ä¼šè·¯ç”±åˆ°è¿™äº› service。 + +## 工作负载选项 + +以下文档介ç»äº†å¦‚何部署工作负载和使用工作负载选项。 + +- [部署工作负载](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md) +- [å‡çº§å·¥ä½œè´Ÿè½½](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md) +- [回滚工作负载](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md) + +## 相关链接 + +### 外部链接 + +- [Service](https://kubernetes.io/docs/concepts/services-networking/service/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/rancher-manager.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/rancher-manager.md new file mode 100644 index 00000000000..7df74298395 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/rancher-manager.md @@ -0,0 +1,15 @@ +--- +slug: / +title: "什么是 Rancher？" +sidebar_label: 什么是 Rancher？ +description: "Rancher 在 Kubernetes 之上添加了新的功能，让你一站å¼ç®¡ç†ä¸Šç™¾ä¸ªé›†ç¾¤ï¼Œç»Ÿä¸€ RBAC，开å¯ç›‘控和告警。了解更多。" +--- +Rancher 是一个 Kubernetes 管ç†å·¥å…·ï¼Œè®©ä½ èƒ½åœ¨ä»»ä½•åœ°æ–¹å’Œä»»ä½•æ供商上部署和è¿è¡Œé›†ç¾¤ã€‚ + +Rancher å¯ä»¥åˆ›å»ºæ¥è‡ª Kubernetes 托管æœåŠ¡æ供商的集群，创建节点并安装 Kubernetes，或者导入在任何地方è¿è¡Œçš„现有 Kubernetes 集群。 + +Rancher 基于 Kubernetes 添加了新的功能，包括统一所有集群的身份验è¯å’Œ RBAC，让系统管ç†å‘˜ä»Žä¸€ä¸ªä½ç½®æŽ§åˆ¶å…¨éƒ¨é›†ç¾¤çš„访问。 + +此外，Rancher å¯ä»¥ä¸ºé›†ç¾¤å’Œèµ„æºæ供更精细的监控和告警，将日志å‘é€åˆ°å¤–部æ供商，并通过应用商店（Application Catalogï¼‰ç›´æŽ¥é›†æˆ Helm。如果你拥有外部 CI/CD 系统，你å¯ä»¥å°†å…¶ä¸Ž Rancher 对接。没有的è¯ï¼Œä½ ä¹Ÿå¯ä»¥ä½¿ç”¨ Rancher æ供的 Fleet 自动部署和å‡çº§å·¥ä½œè´Ÿè½½ã€‚ + +Rancher 是一个 _全栈å¼_ çš„ Kubernetes 容器管ç†å¹³å°ï¼Œä¸ºä½ æ供在任何地方都能æˆåŠŸè¿è¡Œ Kubernetes 的工具。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/about-the-api/api-tokens.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/about-the-api/api-tokens.md new file mode 100644 index 00000000000..59b103fa476 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/about-the-api/api-tokens.md @@ -0,0 +1,86 @@ +--- +title: API 令牌 +--- + +默认情况下，æŸäº›é›†ç¾¤çº§åˆ«çš„ API 令牌是使用无é™æœŸ TTL（`ttl=0`）生æˆçš„。æ¢è¨€ä¹‹ï¼Œé™¤éžä½ è®©ä»¤ç‰Œå¤±æ•ˆï¼Œå¦åˆ™ `ttl=0` çš„ API 令牌永远ä¸ä¼šè¿‡æœŸã€‚令牌ä¸ä¼šå› ä¸ºæ›´æ”¹å¯†ç è€Œå¤±æ•ˆã€‚ + +è¦åœç”¨ API 令牌，你å¯ä»¥åˆ é™¤ä»¤ç‰Œæˆ–åœç”¨ç”¨æˆ·è´¦å·ã€‚ + +### 删除令牌 +è¦åˆ é™¤ä»¤ç‰Œï¼š + +1. 转到 `https:///v3/tokens`，在 Rancher API 视图中查看包å«æ‰€æœ‰ä»¤ç‰Œçš„列表。 + +1. 通过 ID 访问è¦åˆ é™¤çš„令牌。例如，`https:///v3/tokens/kubectl-shell-user-vqkqt`。 + +1. å•å‡»**删除**。 + +以下是使用 `ttl=0` 生æˆçš„完整令牌列表： + +| 令牌 | æè¿° | +| ----------------- | -------------------------------------------------------------------------------------- | +| `kubeconfig-*` | Kubeconfig 令牌 | +| `kubectl-shell-*` | 在æµè§ˆå™¨ä¸­è®¿é—® `kubectl` shell | +| `agent-*` | Agent deployment 令牌 | +| `compose-token-*` | compose 令牌 | +| `helm-token-*` | Helm Chart deployment 令牌 | +| `telemetry-*` | é¥æµ‹ä»¤ç‰Œ | +| `drain-node-*` | 用于清空的令牌（由于没有原生 Kubernetes API，我们使用 `kubectl` æ¥æ¸…空） | + + +### 在 Kubeconfig 令牌上设置 TTL + +管ç†å‘˜å¯ä»¥åœ¨ Kubeconfig 令牌上设置全局存活时间 (time-to-live，TTL)。如需更改默认 kubeconfig TTL，你å¯ä»¥å¯¼èˆªåˆ°å…¨å±€è®¾ç½®å¹¶å°† [`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) 设置为所需的æŒç»­æ—¶é—´ï¼ˆå•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚[`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) 的默认值为 0，表示令牌永ä¸è¿‡æœŸã€‚ + +:::note + +除了由 CLI 创建的用于[ç”Ÿæˆ kubeconfig 令牌](#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)的令牌之外，所有 kubeconfig 令牌都使用此设置。 + +::: + +### 在生æˆçš„ Kubeconfig 中ç¦ç”¨ä»¤ç‰Œ + +1. å°† `kubeconfig-generate-token` 设置为 `false`。此设置让 Rancher ä¸å†åœ¨ç”¨æˆ·å•å‡»ä¸‹è½½ kubeconfig 文件时自动生æˆä»¤ç‰Œã€‚如果åœç”¨æ­¤è®¾ç½®ï¼Œç”Ÿæˆçš„ kubeconfig 将引用 [Rancher CLI](../cli-with-rancher/kubectl-utility.md#使用-kubectl-å’Œ-kubeconfig-令牌进行-ttl-认è¯) æ¥æ£€ç´¢é›†ç¾¤çš„短期令牌。当这个 kubeconfig 在客户端（例如 `kubectl`）中使用时，你需è¦å®‰è£… Rancher CLI æ¥å®Œæˆç™»å½•è¯·æ±‚。 + +2. å°† `kubeconfig-token-ttl-minutes` 设置为所需的时长（å•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚`kubeconfig-token-ttl-minutes` 默认设置为 960ï¼ˆå³ 16 å°æ—¶ï¼‰ã€‚ + +### 令牌哈希 + +ä½ å¯ä»¥å¯ç”¨ä»¤ç‰Œå“ˆå¸Œï¼Œä»¤ç‰Œå°†ä½¿ç”¨ SHA256 算法进行å•å‘哈希。这是一个ä¸å¯é€†çš„æ“作，一旦å¯ç”¨ï¼Œæ­¤åŠŸèƒ½å°†æ— æ³•ç¦ç”¨ã€‚在å¯ç”¨åŠŸèƒ½æˆ–在测试环境中评估之å‰ï¼Œå»ºè®®ä½ å…ˆè¿›è¡Œå¤‡ä»½ã€‚ + +è¦å¯ç”¨ä»¤ç‰Œå“ˆå¸Œï¼Œè¯·å‚阅[本节](../../pages-for-subheaders/enable-experimental-features.md)。 + +此功能将影å“所有令牌，包括但ä¸é™äºŽä»¥ä¸‹å†…容： + +- Kubeconfig 令牌 +- æŒæœ‰è€…令牌 API 密钥/调用 +- 内部æ“作使用的令牌 + +### 令牌设置 +ä»¥ä¸‹å…¨å±€è®¾ç½®ä¼šå½±å“ Rancher 令牌的行为： + +| 设置 | æè¿° | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes) | 用户认è¯ä¼šè¯ä»¤ç‰Œçš„ TTL（å•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚ | +| [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes) | 默认 TTL，应用于所有 kubeconfig 令牌（除了[ç”± Rancher CLI 生æˆçš„令牌](#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)）。**此设置从 2.6.6 版本开始引入。** | +| [`kubeconfig-token-ttl-minutes`](#kubeconfig-token-ttl-minutes) | 在 CLI 中生æˆçš„令牌 TTL。**自 2.6.6 起已弃用，并将在 2.8.0 中删除**。请知悉，`kubeconfig-default-token-TTL-minutes` 将用于所有 kubeconfig 令牌。 | +| [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) | 除了由 [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes) 控制的令牌外，所有令牌的最大 TTL。 | +| [`kubeconfig-generate-token`](#kubeconfig-generate-token) | 如果为 true，则在用户下载 kubeconfig 时自动生æˆä»¤ç‰Œã€‚ | + +#### auth-user-session-ttl-minutes +存活时间（TTL）（å•ä½ï¼šåˆ†é’Ÿï¼‰ï¼Œç”¨äºŽç¡®å®šç”¨æˆ·èº«ä»½éªŒè¯ä¼šè¯ä»¤ç‰Œçš„到期时间。过期åŽï¼Œç”¨æˆ·å°†éœ€è¦ç™»å½•å¹¶èŽ·å–新令牌。此设置ä¸å— [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) çš„å½±å“。会è¯ä»¤ç‰Œæ˜¯åœ¨ç”¨æˆ·ç™»å½• Rancher 时创建的。 + +#### kubeconfig-default-token-TTL-minutes +存活时间（TTL）（å•ä½ï¼šåˆ†é’Ÿï¼‰ï¼Œç”¨äºŽç¡®å®š kubeconfig 令牌的到期时间。令牌过期åŽï¼ŒAPI 将拒ç»ä»¤ç‰Œã€‚此设置的值ä¸èƒ½å¤§äºŽ [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) 的值。此设置适用于在请求的 kubeconfig 文件中生æˆçš„令牌，ä¸åŒ…括[ç”± Rancher CLI 生æˆçš„](#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)令牌。 +**此设置从 2.6.6 版本开始引入**。 + +#### kubeconfig-token-ttl-minutes +存活时间（TTL）（å•ä½ï¼šåˆ†é’Ÿï¼‰ï¼Œç”¨äºŽç¡®å®šç”± CLI 生æˆçš„ kubeconfig 令牌的到期时间。当 [`kubeconfig-generate-token`](#kubeconfig-generate-token) 设为 false 时，则由 CLI 生æˆä»¤ç‰Œã€‚令牌过期åŽï¼ŒAPI 将拒ç»ä»¤ç‰Œã€‚此设置的值ä¸èƒ½å¤§äºŽ [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) 的值。 +**自版本 2.6.6 起已弃用，并将在 2.8.0 中删除。请知悉，此设置将被 [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes) 的值替æ¢**。 + +#### auth-token-max-ttl-minutes +身份验è¯ä»¤ç‰Œçš„最大生存时间 (TTL)（å•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚如果用户å°è¯•åˆ›å»ºä¸€ä¸ª TTL 大于 `auth-token-max-ttl-minutes` 的令牌，Rancher 会将令牌 TTL 设置为 `auth-token-max-ttl-minutes` 的值。身份验è¯ä»¤ç‰Œæ˜¯ä¸ºéªŒè¯ API 请求而创建的。 +**2.6.6 版本更改：适用于所有 kubeconfig 令牌和 API 令牌。** + +#### kubeconfig-generate-token +如果设置为 true，则通过 UI 请求的 kubeconfig 将包å«ä¸€ä¸ªæœ‰æ•ˆçš„令牌。如果设置为 false，kubeconfig 将包å«ä¸€ä¸ªä½¿ç”¨ Rancher CLI æ示用户登录的命令。然åŽï¼Œ[CLI 将为用户检索和缓存令牌](../cli-with-rancher/kubectl-utility.md#使用-kubectl-å’Œ-kubeconfig-令牌进行-ttl-认è¯)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/backup-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/backup-configuration.md new file mode 100644 index 00000000000..005e93110aa --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/backup-configuration.md @@ -0,0 +1,188 @@ +--- +title: 备份é…ç½® +--- + +ä½ å¯ä»¥åœ¨å¤‡ä»½åˆ›å»ºé¡µé¢é…置调度，å¯ç”¨åŠ å¯†ï¼Œå¹¶æŒ‡å®šå¤‡ä»½çš„存储ä½ç½®ã€‚ + + +## 调度 + +选择第一个选项æ¥æ‰§è¡Œä¸€æ¬¡æ€§å¤‡ä»½ï¼Œæˆ–选择第二个选项æ¥å®‰æŽ’定期备份。选择**定期备份**åŽï¼Œä½ å¯ä»¥é…置以下两个字段： + +- **调度**ï¼šè¯¥å­—æ®µæ”¯æŒ + - 标准 [cron 表达å¼](https://en.wikipedia.org/wiki/Cron)，例如 `"0 * * * *"` + - æ述符，例如 `"@midnight"` 或 `"@every 1h30m"` +- **ä¿ç•™æ•°é‡**：指定必须ä¿ç•™çš„备份文件数é‡ã€‚如果文件数é‡è¶…过 `retentionCount` 设置的值，最旧的文件将被删除。默认值为 10。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `schedule` | æ供用于调度定期备份的 cron 字符串。 | +| `retentionCount` | æä¾›è¦ä¿ç•™çš„备份文件数é‡ã€‚ | + +## 加密 + +rancher-backup 通过调用 kube-apiserver æ¥æ”¶é›†èµ„æºã€‚apiserver 返回的对象会被解密，所以å³ä½¿å¯ç”¨äº†[é™æ€åŠ å¯†](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/)，备份收集的加密对象也会是明文。 + +为é¿å…以明文形å¼å­˜å‚¨èµ„æºï¼Œä½ å¯ä»¥ä½¿ç”¨ä¸Žé™æ€åŠ å¯†ç›¸åŒçš„ `EncryptionConfiguration` 文件æ¥åŠ å¯†å¤‡ä»½ä¸­çš„特定资æºã€‚ + +:::note é‡è¦æ示： + +rancher-backup Operator ä¸ä¼šä¿å­˜ `EncryptionConfiguration`，因此在备份中加密对象时，你必须自行ä¿å­˜è¯¥æ–‡ä»¶ã€‚ + +例如，[å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)时，该文件å¯ç”¨äºŽåœ¨æ–°é›†ç¾¤â€‹â€‹ä¸­é‡æ–°åˆ›å»º secret。 + +::: + +Operator å°† `EncryptionConfiguration` 用作 `cattle-resources-system` 命å空间中的 Kubernetes Secret，在 Secret æ•°æ®ä¸­ï¼Œå®ƒä½äºŽå为 `encryption-provider-config.yaml` çš„ key 下é¢ã€‚ + +对于 `EncryptionConfiguration`，你å¯ä»¥ä½¿ç”¨ [Kubernetes 文档中æ供的示例](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#understanding-the-encryption-at-rest-configuration)。 + +为确ä¿åœ¨ secret 中使用正确的密钥，加密é…置文件必须命å为 `encryption-provider-config.yaml`。下é¢çš„命令使用 `--from-file` 标志æ¥åˆ›å»ºå…·æœ‰æ­£ç¡®å¯†é’¥å称的 Secret。 + +å°† `EncryptionConfiguration` ä¿å­˜åˆ°å为 `encryption-provider-config.yaml` 的文件中，并è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +```bash +kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system +``` + +这将确ä¿å¯†æ–‡åŒ…å«ä¸€ä¸ªå为 `encryption-provider-config.yaml` çš„ key，而且 operator 会使用该 key æ¥èŽ·å–加密é…置。 + +`Encryption Config Secret` 下拉èœå•å°†è¿‡æ»¤å¹¶ä»…列出拥有这个 key 的密文。 + +![](/img/backup_restore/backup/encryption.png) + +在上é¢çš„示例命令中，`encryptionconfig` 这个å称å¯ä»¥ä»»æ„修改。 + + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | æä¾› `cattle-resources-system` 命å空间中包å«åŠ å¯†é…置文件的密文的å称。 | + +## 存储ä½ç½® + +如果在备份中指定了 `StorageLocation`，operator 将从该特定的 S3 存储桶中检索备份ä½ç½®ã€‚如果没有指定，operator å°†å°è¯•åœ¨é»˜è®¤ operator 级别的 S3 存储和 operator 级别的 PVC 存储中查找这个文件。默认存储ä½ç½®æ˜¯åœ¨éƒ¨ç½² `rancher-backup` operator æ—¶é…置的。 + +如果你选择第一个选项，备份将存储在安装 rancher-backup Chart æ—¶é…置的存储ä½ç½®ä¸­ã€‚如果你选择第二个选项，你å¯ä»¥é…ç½®ä¸åŒçš„兼容 S3 的对象存储作为备份存储ä½ç½®ã€‚ + +### S3 + +:::caution + +如果你使用 S3 备份目标，请确ä¿æ¯ä¸ªé›†ç¾¤éƒ½æœ‰è‡ªå·±çš„存储桶或文件夹。Rancher 将使用集群é…置的 S3 存储桶或文件夹中的å¯ç”¨å¿«ç…§æ¥å¡«å……快照信æ¯ã€‚ + +::: + +S3 存储ä½ç½®åŒ…å«ä»¥ä¸‹é…置字段： + +1. **凭è¯å¯†æ–‡**（å¯é€‰ï¼‰ï¼šå¦‚果你需è¦ä½¿ç”¨ AWS 访问密钥（access key）和密文密钥（secret key）æ¥è®¿é—® S3 存储桶，请使用带有密钥和指令 `accessKey` å’Œ `secretKey` 的凭è¯æ¥åˆ›å»ºå¯†æ–‡ã€‚它å¯ä»¥æ˜¯ä»»æ„一个命å空间。你å¯ä»¥ç‚¹å‡»[此处](#credentialsecret-示例)查看示例密文。如果è¿è¡Œ operator 的节点在 EC2 中，并且设置了å…许它们访问 S3 çš„ IAM æƒé™ï¼Œåˆ™æ­¤æŒ‡ä»¤æ˜¯ä¸å¿…è¦çš„（如[本节](#ec2-节点访问-s3-çš„-iam-æƒé™)所述）。凭è¯å¯†æ–‡ä¸‹æ‹‰èœå•åˆ—出了所有命å空间的密文。 +1. **存储桶å称**：存储备份文件的 S3 存储桶的å称。 +1. **区域**（å¯é€‰ï¼‰ï¼šS3 存储桶所在的 AWS [区域](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/)。é…ç½® MinIO æ—¶ä¸éœ€è¦è¯¥å­—段。 +1. **文件夹**（å¯é€‰ï¼‰ï¼šS3 存储桶中存储备份文件的文件夹å称。ä¸æ”¯æŒåµŒå¥—文件夹（例如， `rancher/cluster1`）。如果此字段留空，则默认将备份文件存储在 S3 存储桶的根文件夹中。 +1. **端点**：用于访问存储桶区域中的 S3 çš„[端点](https://docs.aws.amazon.com/general/latest/gr/s3.html)。 +1. **端点 CA**（å¯é€‰ï¼‰ï¼šBase64 ç¼–ç çš„ CA è¯ä¹¦ã€‚如需获å–示例，请å‚è§ [S3 兼容é…置示例](#s3-存储é…置示例)。 +1. **跳过 TLS 验è¯**（å¯é€‰ï¼‰ï¼šå¦‚果你ä¸ä½¿ç”¨ TLS，则设置为 `true`。 + + +| YAML 指令å称 | æè¿° | å¿…å¡« | +| ---------------- | ---------------- | ------------ | +| `credentialSecretName` | 如果你需è¦ä½¿ç”¨ AWS 访问密钥（access key）和密文密钥（secret key）æ¥è®¿é—® S3 存储桶，请使用带有密钥和指令 `accessKey` å’Œ `secretKey` 的凭è¯æ¥åˆ›å»ºå¯†æ–‡ã€‚它å¯ä»¥åœ¨ä»»ä½•å‘½å空间中，åªè¦ä½ åœ¨ `credentialSecretNamespace` 中æ供该命å空间。点击[这里](#credentialsecret-示例)查看示例密文。如果è¿è¡Œ operator 的节点在 EC2 中，并且设置了å…许它们访问 S3 çš„ IAM æƒé™ï¼Œåˆ™æ­¤æŒ‡ä»¤æ˜¯ä¸å¿…è¦çš„（如[本节](#ec2-节点访问-s3-çš„-iam-æƒé™)所述）。 | | +| `credentialSecretNamespace` | 包å«è®¿é—® S3 的凭è¯çš„密文的命å空间。如果è¿è¡Œ operator 的节点在 EC2 中，并且设置了å…许它们访问 S3 çš„ IAM æƒé™ï¼Œåˆ™æ­¤æŒ‡ä»¤æ˜¯ä¸å¿…è¦çš„（如[本节](#ec2-节点访问-s3-çš„-iam-æƒé™)所述）。 | | +| `bucketName` | 存储备份文件的 S3 存储桶的å称。 | ✓ | +| `folder` | S3 存储桶中存储备份文件的文件夹å称。ä¸æ”¯æŒåµŒå¥—文件夹（例如， `rancher/cluster1`）。如果此字段留空，则默认将备份文件存储在 S3 存储桶的根文件夹中。 | | +| `region` | S3 存储桶所在的 AWS [区域](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/)。 | ✓ | +| `endpoint` | 用于访问存储桶区域中的 S3 çš„[端点](https://docs.aws.amazon.com/general/latest/gr/s3.html)。 | ✓ | +| `endpointCA` | Base64 ç¼–ç çš„ CA è¯ä¹¦ã€‚如需获å–示例，请å‚è§ [S3 兼容é…置示例](#s3-存储é…置示例)。 | | +| `insecureTLSSkipVerify` | 如果你ä¸ä½¿ç”¨ TLS，则设置为 `true`。 | | + +### S3 存储é…置示例 + +```yaml +s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### MinIO é…置示例 + +```yaml +s3: + credentialSecretName: minio-creds + bucketName: rancherbackups + endpoint: minio.35.202.130.254.xip.io + endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURHakNDQWdLZ0F3SUJBZ0lKQUtpWFZpNEpBb0J5TUEwR0NTcUdTSWIzRFFFQkN3VUFNQkl4RURBT0JnTlYKQkFNTUIzUmxjM1F0WTJFd0hoY05NakF3T0RNd01UZ3lOVFE1V2hjTk1qQXhNREk1TVRneU5UUTVXakFTTVJBdwpEZ1lEVlFRRERBZDBaWE4wTFdOaE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBCjA4dnV3Q2Y0SEhtR2Q2azVNTmozRW5NOG00T2RpS3czSGszd1NlOUlXQkwyVzY5WDZxenBhN2I2M3U2L05mMnkKSnZWNDVqeXplRFB6bFJycjlpbEpWaVZ1NFNqWlFjdG9jWmFCaVNsL0xDbEFDdkFaUlYvKzN0TFVTZSs1ZDY0QQpWcUhDQlZObU5xM3E3aVY0TE1aSVpRc3N6K0FxaU1Sd0pOMVVKQTZ6V0tUc2Yzc3ByQ0J2dWxJWmZsVXVETVAyCnRCTCt6cXZEc0pDdWlhNEEvU2JNT29tVmM2WnNtTGkwMjdub3dGRld3MnRpSkM5d0xMRE14NnJoVHQ4a3VvVHYKQXJpUjB4WktiRU45L1Uzb011eUVKbHZyck9YS2ZuUDUwbk8ycGNaQnZCb3pUTStYZnRvQ1d5UnhKUmI5cFNTRApKQjlmUEFtLzNZcFpMMGRKY2sxR1h3SURBUUFCbzNNd2NUQWRCZ05WSFE0RUZnUVU5NHU4WXlMdmE2MTJnT1pyCm44QnlFQ2NucVFjd1FnWURWUjBqQkRzd09ZQVU5NHU4WXlMdmE2MTJnT1pybjhCeUVDY25xUWVoRnFRVU1CSXgKRURBT0JnTlZCQU1NQjNSbGMzUXRZMkdDQ1FDb2wxWXVDUUtBY2pBTUJnTlZIUk1FQlRBREFRSC9NQTBHQ1NxRwpTSWIzRFFFQkN3VUFBNElCQVFER1JRZ1RtdzdVNXRQRHA5Q2psOXlLRW9Vd2pYWWM2UlAwdm1GSHpubXJ3dUVLCjFrTkVJNzhBTUw1MEpuS29CY0ljVDNEeGQ3TGdIbTNCRE5mVVh2anArNnZqaXhJYXR2UWhsSFNVaWIyZjJsSTkKVEMxNzVyNCtROFkzelc1RlFXSDdLK08vY3pJTGh5ei93aHRDUlFkQ29lS1dXZkFiby8wd0VSejZzNkhkVFJzNwpHcWlGNWZtWGp6S0lOcTBjMHRyZ0xtalNKd1hwSnU0ZnNGOEcyZUh4b2pOKzdJQ1FuSkg5cGRIRVpUQUtOL2ppCnIvem04RlZtd1kvdTBndEZneWVQY1ZWbXBqRm03Y0ZOSkc4Y2ZYd0QzcEFwVjhVOGNocTZGeFBHTkVvWFZnclMKY1VRMklaU0RJd1FFY3FvSzFKSGdCUWw2RXBaUVpWMW1DRklrdFBwSQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t +``` +### credentialSecret 示例 + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +:::note + +为é¿å…ç¼–ç é—®é¢˜ï¼Œä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤åˆ›å»º `credentialSecret`，更新 `accessKey` å’Œ `secretKey` 的值。 + +```bash +kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= +``` + +::: + +### EC2 节点访问 S3 çš„ IAM æƒé™ + +有两ç§æ–¹æ³•å¯ä»¥è®¾ç½® `rancher-backup` operator æ¥ä½¿ç”¨ S3 作为备份存储ä½ç½®ã€‚ + +一ç§æ–¹æ³•æ˜¯åœ¨ Backup 自定义资æºä¸­é…ç½® `credentialSecretName`（å³å¯ä»¥è®¿é—® S3 çš„ AWS 凭è¯ï¼‰ã€‚ + +如果集群节点在 Amazon EC2 中，你也å¯ä»¥ç»™ EC2 èŠ‚ç‚¹åˆ†é… IAM æƒé™ï¼Œä½¿å…¶å¯ä»¥è®¿é—® S3。 + +è¦å…许节点访问 S3，请按照 [AWS 文档](https://aws.amazon.com/premiumsupport/knowledge-center/ec2-instance-access-s3-bucket/)中的说明，为 EC2 创建一个 IAM 角色。在å‘角色添加自定义策略时，添加以下æƒé™ï¼Œå¹¶å°† `Resource` 替æ¢ä¸ºä½ çš„存储桶å称： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups/*" + ] + } + ] +} +``` + +在创建角色并将相应的实例é…置文件附加 EC2 实例åŽï¼ŒBackup 自定义资æºä¸­çš„ `credentialSecretName` 指令å¯ä»¥ç•™ç©ºã€‚ + +## 示例 + +å¦‚éœ€èŽ·å– Backup 自定义资æºçš„示例，请å‚è§[本页](examples.md#备份)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/examples.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/examples.md new file mode 100644 index 00000000000..ca1ce934d84 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/examples.md @@ -0,0 +1,293 @@ +--- +title: 示例 +--- + +æœ¬èŠ‚åŒ…å« Backup å’Œ Restore 自定义资æºçš„示例。 + +默认的备份存储ä½ç½®æ˜¯åœ¨å®‰è£…或å‡çº§ `rancher-backup` operator æ—¶é…置的。 + +åªæœ‰ Restore 自定义资æºä½¿ç”¨åˆ›å»ºå¤‡ä»½æ—¶ä½¿ç”¨çš„加密é…置密文时，æ‰èƒ½è¿˜åŽŸåŠ å¯†çš„备份。 + +## 备份 + +æœ¬èŠ‚åŒ…å« Backup 自定义资æºçš„示例。 + +> **注æ„**：有关é…置以下选项的更多信æ¯ï¼Œè¯·å‚阅[备份é…ç½®å‚考页é¢](./backup-configuration.md)。 + +### 在默认ä½ç½®è¿›è¡ŒåŠ å¯†å¤‡ä»½ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-encrypted-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### 在默认ä½ç½®è¿›è¡Œå®šæœŸå¤‡ä»½ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-recurring-backup +spec: + resourceSetName: rancher-resource-set + schedule: "@every 1h" + retentionCount: 10 +``` + +### 在默认ä½ç½®è¿›è¡ŒåŠ å¯†çš„定期备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-enc-recurring-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 3 +``` + +### Minio 中的加密备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: minio-backup +spec: + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### 使用 AWS 凭è¯å¯†æ–‡åœ¨ S3 中备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### 使用 AWS 凭è¯å¯†æ–‡åœ¨ S3 中进行定期备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-recurring-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 +``` + +### 从具有访问 S3 çš„ IAM æƒé™çš„ EC2 节点进行备份 + +这个例å­è¡¨æ˜Žï¼Œå¦‚æžœè¿è¡Œ `rancher-backup` 的节点拥有这些[访问 S3 çš„æƒé™](backup-configuration.md#ec2-节点访问-s3-çš„-iam-æƒé™)，就ä¸å¿…æä¾› AWS 的凭è¯å¯†æ–‡æ¥åˆ›å»ºå¤‡ä»½ã€‚ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-iam-backup +spec: + storageLocation: + s3: + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +## 还原 + +æœ¬èŠ‚åŒ…å« Restore 自定义资æºçš„示例。 + +> **注æ„**：有关é…置以下选项的更多信æ¯ï¼Œè¯·å‚阅[æ¢å¤é…ç½®å‚考页é¢](./restore-configuration.md)。 + +### 使用默认备份文件ä½ç½®è¿˜åŽŸ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-default +spec: + backupFilename: default-location-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-29-54-07-00.tar.gz +# encryptionConfigSecretName: test-encryptionconfig +``` + +### 为 Rancher è¿ç§»è¿›è¡Œè¿˜åŽŸ +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-migration +spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + prune: false + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### 使用加密的备份还原 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-encrypted +spec: + backupFilename: default-test-s3-def-backup-c583d8f2-6daf-4648-8ead-ed826c591471-2020-08-24T20-47-05Z.tar.gz + encryptionConfigSecretName: encryptionconfig +``` + +### 从 Minio 还原加密的备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-minio +spec: + backupFilename: default-minio-backup-demo-aa5c04b7-4dba-4c48-9ac4-ab7916812eaa-2020-08-30T13-18-17-07-00.tar.gz + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + encryptionConfigSecretName: test-encryptionconfig +``` + +### 使用 AWS 凭è¯å¯†æ–‡è®¿é—® S3 从备份中还原 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: test-s3-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-49-34-07-00.tar.gz.enc + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +### 从具有访问 S3 çš„ IAM æƒé™çš„ EC2 节点进行还原 + +这个例å­è¡¨æ˜Žï¼Œå¦‚æžœè¿è¡Œ `rancher-backup` 的节点拥有这些[访问 S3 çš„æƒé™](backup-configuration.md#ec2-节点访问-s3-çš„-iam-æƒé™)，就ä¸å¿…æä¾› AWS 的凭è¯å¯†æ–‡æ¥ä»Žå¤‡ä»½ä¸­è¿˜åŽŸã€‚ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: default-test-s3-recurring-backup-84bf8dd8-0ef3-4240-8ad1-fc7ec308e216-2020-08-24T10#52#44-07#00.tar.gz + storageLocation: + s3: + bucketName: rajashree-backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +## 在 S3 中存储备份的凭è¯å¯†æ–‡ç¤ºä¾‹ + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +## EncryptionConfiguration 示例 + +以下代ç ç‰‡æ®µæ¼”示了两ç§ä¸åŒç±»åž‹çš„密文åŠå…¶ä¸Žè‡ªå®šä¹‰èµ„æºçš„备份和还原的相关性。 + +第一个示例是用于加密备份文件的密钥。在这ç§æƒ…况下，Backup operator 将无法读å–密文加密文件。它åªä½¿ç”¨å¯†æ–‡çš„内容。 + +第二个示例是 Kubernetes 密文加密é…置文件，用于加密存储在 etcd 中的密文。**备份 etcd æ•°æ®å­˜å‚¨æ—¶ï¼Œè¯·åŠ¡å¿…åŒæ—¶å¤‡ä»½ EncryptionConfiguration**。如果你没有这样åšï¼Œè€Œä¸”备份数æ®æ—¶æ­£åœ¨ä½¿ç”¨å¯†æ–‡åŠ å¯†ï¼Œä½ å°†æ— æ³•ä½¿ç”¨æ¢å¤çš„æ•°æ®ã€‚ + + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: + - secrets + providers: + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - secretbox: + keys: + - name: key1 + secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY= +``` + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/restore-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/restore-configuration.md new file mode 100644 index 00000000000..5931f1f82d8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/restore-configuration.md @@ -0,0 +1,81 @@ +--- +title: 还原é…ç½® +--- + +ä½ å¯ä»¥åœ¨è¿˜åŽŸåˆ›å»ºé¡µé¢æ供还原备份的详细信æ¯ã€‚ + +![](/img/backup_restore/restore/restore.png) + + +## å¤‡ä»½æº +你需è¦æ供备份文件和备份文件存储ä½ç½®çš„详细信æ¯ï¼Œoperator 会使用这些信æ¯æ‰§è¡Œè¿˜åŽŸã€‚选择以下其中一个选项æ¥æ供这些详细信æ¯ã€‚ + + + + +### 使用已有的备份é…ç½®æ¢å¤ + +如果你选择这个选项，**目标备份**下拉èœå•ä¸­ä¼šå‡ºçŽ°é›†ç¾¤ä¸­å¯ç”¨çš„备份。从下拉èœå•ä¸­é€‰æ‹© Backup，选择åŽ**备份文件å**字段会自动填写，而且所选 Backup çš„ä¿¡æ¯ä¼šä¼ é€’ç»™ operator + +![](/img/backup_restore/restore/existing.png) + +如果 Backup 自定义资æºåœ¨é›†ç¾¤ä¸­ä¸å­˜åœ¨ï¼Œä½ éœ€è¦èŽ·å–准确的文件å，并使用默认存储目标或与 S3 兼容的对象存储æ¥æ供备份æºçš„详细信æ¯ã€‚ + + +### 使用默认存储目标æ¢å¤ + +如果你è¦ä»Ž operator 级别é…置的默认存储ä½ç½®ä¸­çš„备份文件进行æ¢å¤ï¼Œè¯·é€‰æ‹©æ­¤é€‰é¡¹ã€‚operator 级别的é…置是指安装或å‡çº§ `rancher-backup` operator æ—¶é…置的存储ä½ç½®ã€‚在**备份文件å**字段中æ供准确的文件å。 + +![](/img/backup_restore/restore/default.png) + +### 使用与 S3 兼容的对象存储æ¢å¤ + +如果在 operator 级别没有é…置默认存储ä½ç½®ï¼Œæˆ–者备份文件所在的存储桶与é…置的默认存储ä½ç½®ä¸ä¸€æ ·æ—¶ï¼Œè¯·é€‰æ‹©æ­¤é€‰é¡¹ã€‚在**备份文件å**字段中æ供准确的文件å。请å‚è§[本节](#从-s3-获å–备份文件å)了解从 S3 获å–备份文件å的具体步骤。填写 S3 兼容对象存储的所有详细信æ¯ã€‚它的字段与 [Backup 自定义资æº](backup-configuration.md#存储ä½ç½®)中 `backup.StorageLocation` é…置的字段一样。 + +![](/img/backup_restore/restore/s3store.png) + +## 加密 + +如果备份是在å¯ç”¨åŠ å¯†çš„情况下创建的，备份文件的åŽç¼€ä¸º `.enc`。如果你选择这类的 Backup，或者æä¾›åŽç¼€ä¸º `.enc` 的备份文件å，则会显示å¦ä¸€ä¸ªå为 **Encryption Config Secret** 的下拉èœå•ã€‚ + +![](/img/backup_restore/restore/encryption.png) + +从该下拉èœå•ä¸­é€‰æ‹©çš„密文必须与执行备份时用于 Backup 自定义资æºçš„密文内容相åŒã€‚如果加密é…ç½®ä¸åŒ¹é…，还原将会失败。 + +`Encryption Config Secret` 下拉èœå•å°†è¿‡æ»¤å¹¶ä»…列出拥有这个 key 的密文。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | æä¾› `cattle-resources-system` 命å空间中包å«åŠ å¯†é…置文件的密文的å称。 | + +:::note é‡è¦æ示： + +此字段仅在备份创建的过程中å¯ç”¨äº†åŠ å¯†åŠŸèƒ½æ—¶æ‰éœ€è¦è®¾ç½®ã€‚æ供错误的加密é…置将导致还原失败。 + +::: + +## 还原过程中修剪 + +* **Prune**：为了从备份中完全æ¢å¤ Rancher，并回到备份时的确切状æ€ï¼Œæˆ‘们需è¦åˆ é™¤ Rancher 在备份åŽåˆ›å»ºçš„所有é¢å¤–资æºã€‚如果 **Prune** 标志被å¯ç”¨ï¼Œoperator 就会删除这些资æºã€‚Prune 是默认å¯ç”¨çš„，建议ä¿æŒå¯ç”¨çŠ¶æ€ã€‚ +* **删除超时**：在删除资æºçš„时候，operator 编辑资æºä»¥åˆ é™¤ Finalizers，并试图å†æ¬¡åˆ é™¤å‰å°†ç­‰å¾…的时间。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `prune` | 删除备份中ä¸å­˜åœ¨çš„ç”± Rancher 管ç†çš„资æºï¼ˆæŽ¨è）。 | +| `deleteTimeoutSeconds` | 在删除资æºçš„时候，operator 编辑资æºä»¥åˆ é™¤ Finalizers，并试图å†æ¬¡åˆ é™¤å‰å°†ç­‰å¾…的时间。 | + +## 从 S3 获å–备份文件å + +这是 `rancher-backup` operator 用æ¥æ‰§è¡Œè¿˜åŽŸçš„备份文件的å称。 + +è¦ä»Ž S3 获å–这个文件å，请进入你的 S3 存储桶（或在执行备份时指定的文件夹）。 + +å¤åˆ¶æ–‡ä»¶å并将其存储在你的 Restore 自定义资æºä¸­ã€‚å‡è®¾ä½ çš„备份文件的å字是 `backupfile`： + +- 如果你的存储桶å称是 `s3bucket`，而且你没有指定文件夹，那么è¦ä½¿ç”¨çš„ `backupFilename` 会是 `backupfile`。 +- 如果你的存储桶å称是 `s3bucket`，而且基础文件夹是 `s3folder`，那么è¦ä½¿ç”¨çš„ `backupFilename` 会是 `backupfile`。 +- 如果 `s3Folder` 中有一个å为 `s3sub` çš„å­æ–‡ä»¶å¤¹ï¼Œè€Œä¸”你的备份文件存放在该文件夹中，那么è¦ä½¿ç”¨çš„`backupFilename` 会是 `s3sub/backupfile`。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `backupFilename` | 这是 `rancher-backup` operator 用æ¥æ‰§è¡Œè¿˜åŽŸçš„备份文件的å称。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/storage-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/storage-configuration.md new file mode 100644 index 00000000000..2db4ed74ef5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/backup-restore-configuration/storage-configuration.md @@ -0,0 +1,56 @@ +--- +title: 备份存储ä½ç½®é…ç½® +--- + +é…置一个用于ä¿å­˜æ‰€æœ‰å¤‡ä»½çš„默认存储ä½ç½®ã€‚ä½ å¯ä»¥é€‰æ‹©å¯¹æ¯ä¸ªå¤‡ä»½è¿›è¡Œè¦†ç›–，但仅é™äºŽä½¿ç”¨ S3 对象存储。 + +在 operator 级别åªèƒ½é…置一个存储ä½ç½®ã€‚ + + +## 存储ä½ç½®é…ç½® + +### 无默认存储ä½ç½® + +ä½ å¯ä»¥é€‰æ‹©ä¸é…ç½® operator 级别的存储ä½ç½®ã€‚如果选择此选项，你必须é…置一个与 S3 兼容的对象存储作为æ¯ä¸ªå¤‡ä»½çš„存储ä½ç½®ã€‚ + +### S3 兼容的对象存储 + +| å‚æ•° | æè¿° | +| -------------- | -------------- | +| 凭è¯å¯†æ–‡ | 从 Rancher 的密文中选择 S3 的凭è¯ã€‚[示例](examples.md#在-s3-中存储备份的凭è¯å¯†æ–‡ç¤ºä¾‹)。 | +| 存储桶å称 | 存储备份的 [S3 存储桶](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html)çš„å称。默认值：`rancherbackups`。 | +| 区域 | S3 存储桶所在的 [AWS 区域](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/)。 | +| 文件夹 | 存储备份的 [S3 存储桶中的文件夹](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html)。如果此字段留空，则默认将备份文件存储在 S3 存储桶的根文件夹中。 | +| 端点 | [S3 端点](https://docs.aws.amazon.com/general/latest/gr/s3.html)，例如 `s3.us-west-2.amazonaws.com`。 | +| Endpoint CA | 用于 S3 端点的 CA è¯ä¹¦ã€‚默认值：base64 ç¼–ç çš„ CA è¯ä¹¦ã€‚ | +| insecureTLSSkipVerify | 如果你ä¸ä½¿ç”¨ TLS，则设置为 `true`。 | + +### 使用现有的 StorageClass + +如果你通过选择 StorageClass 选项æ¥å®‰è£… `rancher-backup` Chart，将会创建一个æŒä¹…å·è¯´æ˜Žï¼ˆPersistent Volume Claim，PVC），而且 Kubernetes 会动æ€é…置一个æŒä¹…å·ï¼ˆPersistent Volume，PV），所有备份都会默认ä¿å­˜åˆ°è¯¥æŒä¹…å·ä¸­ã€‚ + +关于创建存储类的信æ¯ï¼Œè¯·å‚è§[本章节](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md)。 + +:::note é‡è¦æ示： + +强烈建议使用回收策略为 "Retain" çš„ StorageClass。å¦åˆ™ï¼Œå¦‚æžœ `rancher-backup` Chart 创建的 PVC 在应用å‡çº§æœŸé—´æˆ–æ„外被删除åŽï¼ŒPV 也会被删除，也就是说所有ä¿å­˜åœ¨å…¶ä¸­çš„备份都会被删除。 +如果没有这样的 StorageClass å¯ç”¨ï¼Œåœ¨é…ç½® PV åŽï¼Œè¯·ç¡®ä¿å…ˆå°†å…¶å›žæ”¶ç­–略设置为 "Retain"，然åŽå†å­˜å‚¨å¤‡ä»½ã€‚ + +::: + +### 使用现有的æŒä¹…å· + +选择一个用于存储备份的现有æŒä¹…å·ã€‚有关在 Rancher 中创建 PersistentVolumes 的更多信æ¯ï¼Œè¯·å‚è§[本节](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md#2-添加一个引用æŒä¹…存储的-persistentvolume)。 + +:::note é‡è¦æ示： + +强烈建议使用回收策略为 "Retain" çš„ Persistent Volume。å¦åˆ™ï¼Œå¦‚æžœ `rancher-backup` Chart 创建的 PVC 在应用å‡çº§æœŸé—´æˆ–æ„外被删除åŽï¼ŒPV 也会被删除，也就是说所有ä¿å­˜åœ¨å…¶ä¸­çš„备份都会被删除。 + +::: + +## rancher-backup Helm Chart 的示例 values.yaml + +使用 Helm CLI 时，å¯ç”¨äºŽé…ç½® `rancher-backup` operator çš„ `values.yaml` å¯ä»¥åœ¨ [backup-restore-operator](https://github.com/rancher/backup-restore-operator/blob/master/charts/rancher-backup/values.yaml) 仓库中找到。 + +有关 `values.yaml` 文件和在安装时é…ç½® Helm Charts 的更多信æ¯ï¼Œè¯·å‚è§ [Helm 文档](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing)。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md new file mode 100644 index 00000000000..afc1247854f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -0,0 +1,88 @@ +--- +title: Logging 最佳实践 +--- + +本指å—介ç»äº†é›†ç¾¤çº§åˆ«æ—¥å¿—和应用日志的最佳实践。 + +- [集群级别日志](#集群级别日志) +- [应用日志](#应用日志) +- [通用最佳实践](#通用最佳实践) + +在 Rancher 2.5 之å‰ï¼ŒRancher çš„ Logging 是一个é™æ€é›†æˆã€‚å¯ä¾›é€‰æ‹©çš„èšåˆå™¨æ˜¯å›ºå®šçš„（包括 ElasticSearchã€Splunkã€Kafkaã€Fluentd å’Œ Syslog），而且åªæœ‰ä¸¤ä¸ªé…置级别å¯ä¾›é€‰æ‹©ï¼ˆé›†ç¾¤çº§åˆ«å’Œé¡¹ç›®çº§åˆ«ï¼‰ã€‚ + +现在，Rancher 的日志èšåˆæ›´åŠ çµæ´»ã€‚通过新的 Logging 的功能，管ç†å‘˜å’Œç”¨æˆ·éƒ½å¯ä»¥éƒ¨ç½²ç¬¦åˆç»†ç²’度收集标准的日志记录，åŒæ—¶æ供更多的目标和é…置选项。 + +Rancher Logging 使用的是 [Logging Operator](https://github.com/kube-logging/logging-operator)。我们让你å¯ä»¥ç®¡ç†è¿™ä¸ª operator åŠå…¶èµ„æºï¼Œå¹¶å°†å®ƒçš„管ç†åŠŸèƒ½å’Œ Rancher 集群管ç†è”系起æ¥ã€‚ + +## 集群级别日志 + +### 抓å–集群内日志 + +æŸäº›ç”¨æˆ·å¯èƒ½æƒ³ä»Žé›†ç¾¤ä¸­è¿è¡Œçš„æ¯ä¸ªå®¹å™¨ä¸­æŠ“å–日志。但是你的安全团队å¯èƒ½è¦æ±‚你从所有执行点收集所有日志。 + +在这ç§æƒ…况下，我们建议你至少创建两个 _ClusterOutput_ 对象 - 一个用于安全团队（如果需è¦ï¼‰ï¼Œå¦ä¸€ä¸ªç”¨äºŽä½ è‡ªå·±ï¼Œå³é›†ç¾¤ç®¡ç†å‘˜ã€‚创建这些对象时，请选择一个å¯ä»¥å¤„ç†æ•´ä¸ªé›†ç¾¤çš„大é‡æ—¥å¿—的输出端点。此外，你还需è¦é€‰æ‹©åˆé€‚的索引æ¥æŽ¥æ”¶è¿™äº›æ—¥å¿—。 + +创建这些 _ClusterOutput_ 对象åŽï¼Œåˆ›å»ºä¸€ä¸ª _ClusterFlow_ æ¥æ”¶é›†æ‰€æœ‰æ—¥å¿—。ä¸è¦åœ¨æ­¤ flow 上定义任何 _Include_ 或 _Exclude_ 规则。这å¯ä»¥ç¡®ä¿ä½ èƒ½æ”¶é›†æ•´ä¸ªé›†ç¾¤çš„所有日志。如果你有两个 _ClusterOutputs_，请确ä¿å®ƒä»¬éƒ½èƒ½æ”¶åˆ°æ—¥å¿—。 + +### Kubernetes 组件 + +_ClusterFlows_ 能够收集 Kubernetes 集群中所有主机上所有容器的日志。如果这些容器包å«åœ¨ Kubernetes Pod 中，这个方法是适用的。但是，RKE 容器ä¸å­˜åœ¨äºŽ Kubernetes 内。 + +ç›®å‰ï¼ŒRancher 能æœé›† RKE 容器的日志，但ä¸èƒ½è½»æ˜“过滤。这是因为这些日志ä¸åŒ…å«æºå®¹å™¨çš„ä¿¡æ¯ï¼ˆä¾‹å¦‚ `etcd` 或 `kube-apiserver`）。 + +Rancher 的未æ¥ç‰ˆæœ¬å°†åŒ…å«æºå®¹å™¨å称，æ¥æ”¯æŒè¿‡æ»¤è¿™äº›ç»„件的日志。该功能实现之åŽï¼Œä½ å°†èƒ½å¤Ÿè‡ªå®šä¹‰ _ClusterFlow_ æ¥**ä»…**检索 Kubernetes 组件日志，并将日志å‘é€åˆ°é€‚当的输出ä½ç½®ã€‚ + +## 应用日志 + +对于 Kubernetes 和所有基于容器的应用而言，最佳实践是将应用日志引导到 `stdout`/`stderr`。容器è¿è¡Œæ—¶å°†æ•èŽ·è¿™äº›æ—¥å¿—并用它们进行**æŸäº›æ“作** - 通常是将它们写入文件。根æ®å®¹å™¨è¿è¡Œæ—¶ï¼ˆåŠå…¶é…置），这些日志å¯ä»¥æ”¾ç½®åœ¨ä»»æ„æ•°é‡çš„ä½ç½®ã€‚ + +在将日志写入文件的情况下，Kubernetes 通过在æ¯ä¸ªä¸»æœºä¸Šåˆ›å»ºä¸€ä¸ª `/var/log/containers` 目录æ¥æ供帮助。这个目录将日志文件符å·é“¾æŽ¥åˆ°å®ƒä»¬çš„实际目的地（å¯èƒ½å› ä¸ºé…置或容器è¿è¡Œæ—¶è€Œæœ‰æ‰€ä¸åŒï¼‰ã€‚ + +Rancher çš„ Logging å°†è¯»å– `/var/log/containers` 中的所有日志æ¡ç›®ï¼Œç¡®ä¿æ‰€æœ‰å®¹å™¨çš„所有日志æ¡ç›®ï¼ˆå‡è®¾ä½¿ç”¨é»˜è®¤é…置）å‡èƒ½è¢«æ”¶é›†å’Œå¤„ç†ã€‚ + +### 特定日志文件 + +日志收集仅从 Kubernetes 中的 Pod 中检索 `stdout`/`stderr` 日志。但是，我们也å¯èƒ½æƒ³ä»Žåº”用生æˆçš„其他文件中收集日志。在这ç§æƒ…况下，你å¯ä»¥ä½¿ç”¨ä¸€ä¸ªï¼ˆæˆ–ä¸¤ä¸ªï¼‰æ—¥å¿—æµ Sidecar。 + +è®¾ç½®æ—¥å¿—æµ Sidecar 的目的是获å–写入ç£ç›˜çš„日志文件，并将其内容传输到 `stdout`。这样一æ¥ï¼ŒLogging Operator å°±å¯ä»¥æŽ¥æ”¶è¿™äº›æ—¥å¿—，并把日志å‘é€åˆ°ç›®æ ‡è¾“出ä½ç½®ã€‚ + +è¦è¿›è¡Œè®¾ç½®ï¼Œç¼–辑你的工作负载资æºï¼ˆä¾‹å¦‚ Deployment）并添加以下 Sidecar 定义： + +```yaml +... +containers: +- args: + - -F + - /path/to/your/log/file.log + command: + - tail + image: busybox + name: stream-log-file-[name] + volumeMounts: + - mountPath: /path/to/your/log + name: mounted-log +... +``` + +这将添加一个容器到你的工作负载定义中，用于将 `/path/to/your/log/file.log` 的内容（在本示例中）传输到 `stdout`。 + +然åŽå°†æ ¹æ®ä½ è®¾ç½®çš„ _Flows_ 或 _ClusterFlows_ 自动收集该日志æµã€‚你还å¯ä»¥é€šè¿‡ä½¿ç”¨å®¹å™¨çš„å称，专门为该日志文件创建一个 _Flow_。示例如下： + +```yaml +... +spec: + match: + - select: + container_names: + - stream-log-file-name +... +``` + + +## 通用最佳实践 + +- å°½é‡è¾“出结构化日志æ¡ç›®ï¼ˆä¾‹å¦‚ `syslog`ã€JSON）。这些格å¼å·²ç»æœ‰äº†è§£æžå™¨ï¼Œå› æ­¤ä½ å¯ä»¥æ›´è½»æ¾åœ°å¤„ç†æ—¥å¿—æ¡ç›®ã€‚ +- å°½é‡åœ¨æ—¥å¿—æ¡ç›®å†…æ供创建该日志æ¡ç›®çš„应用的å称。这å¯ä»¥ä½¿æ•…障排除更容易。这是因为 Kubernetes 并ä¸æ€»æ˜¯å°†åº”用的å称作为对象å称。例如，æŸä¸ª Pod ID å¯èƒ½æ˜¯ `myapp-098kjhsdf098sdf98`，从这个 ID 中我们ä¸èƒ½èŽ·å–è¿è¡Œåœ¨å®¹å™¨å†…的应用的太多信æ¯ã€‚ +- 除了在集群范围内收集所有日志的情况外，尽é‡ä¸¥æ ¼é™å®š _Flow_ å’Œ _ClusterFlow_ 对象的范围。这使得在出现问题时更容易进行故障排除，并且还有助于确ä¿ä¸ç›¸å…³çš„日志æ¡ç›®ä¸ä¼šå‡ºçŽ°åœ¨ä½ çš„èšåˆå™¨ä¸­ã€‚严格é™å®šèŒƒå›´çš„一个例å­æ˜¯å°† _Flow_ é™åˆ¶åœ¨å‘½å空间中的å•ä¸ª _Deployment_，甚至是 _Pod_ 中的å•ä¸ªå®¹å™¨ã€‚ +- 除éžè¦è¿›è¡Œæ•…障排除，å¦åˆ™ä¸è¦è®©æ—¥å¿—太详细。太详细的日志会带æ¥è®¸å¤šé—®é¢˜ï¼Œå…¶ä¸­æœ€ä¸»è¦çš„是**带æ¥å¹²æ‰°**，å³é‡è¦äº‹ä»¶å¯èƒ½ä¼šæ·¹æ²¡åœ¨æµ·é‡ `DEBUG` ä¿¡æ¯ä¸­ã€‚ä½ å¯ä»¥é€šè¿‡ä½¿ç”¨è‡ªåŠ¨å‘Šè­¦å’Œè„šæœ¬æ¥ç¼“解这ç§é—®é¢˜ï¼Œä½†å¤ªè¯¦ç»†çš„日志ä»ç„¶ç»™æ—¥å¿—管ç†åŸºç¡€è®¾æ–½å¸¦æ¥è¿‡å¤§çš„压力。 +- 如果å¯èƒ½ï¼Œå°½é‡åœ¨æ—¥å¿—æ¡ç›®ä¸­æ供事务或请求 ID。这å¯ä»¥ä½¿è·¨æ—¥å¿—æºè¿½è¸ªåº”用活动å˜å¾—更容易，尤其是在处ç†åˆ†å¸ƒå¼åº”用时。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md new file mode 100644 index 00000000000..c48a1d04995 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -0,0 +1,111 @@ +--- +title: 监控最佳实践 +--- + +é…ç½®åˆç†çš„监控和告警规则对于安全ã€å¯é åœ°è¿è¡Œç”Ÿäº§çŽ¯å¢ƒä¸­çš„工作负载至关é‡è¦ã€‚在使用 Kubernetes å’Œ Rancher 时也是如此。幸è¿çš„是，你å¯ä»¥ä½¿ç”¨é›†æˆçš„监控和告警功能æ¥ç®€åŒ–整个过程。 + +[Rancher 监控文档](../../../pages-for-subheaders/monitoring-and-alerting.md)æ述了如何设置完整的 Prometheus å’Œ Grafana。这是开箱å³ç”¨çš„功能，它将从集群中的所有系统和 Kubernetes 组件中抓å–监控数æ®ï¼Œå¹¶æä¾›åˆç†çš„仪表æ¿å’Œå‘Šè­¦ã€‚但为了实现å¯é çš„设置，你还需è¦ç›‘控你的工作负载并使 Prometheus å’Œ Grafana 适应你的特定用例和集群规模。本文档将为你æ供这方é¢çš„最佳实践。 + +## 监控内容 + +Kubernetes 本身以åŠè¿è¡Œåœ¨å…¶å†…部的应用构æˆäº†ä¸€ä¸ªåˆ†å¸ƒå¼ç³»ç»Ÿï¼Œä¸åŒçš„组件之间能够进行交互。对于整个系统和æ¯ä¸ªå•ç‹¬çš„组件，你必须确ä¿å…¶æ€§èƒ½ã€å¯ç”¨æ€§ã€å¯é æ€§å’Œå¯æ‰©å±•æ€§ã€‚详情请å‚è§ Google å…费的 [Site Reliability Engineering Book](https://sre.google/sre-book/table-of-contents/)，尤其是关于 [Monitoring distributed systems](https://sre.google/sre-book/monitoring-distributed-systems/) 的章节。 + +## é…ç½® Prometheus 资æºä½¿ç”¨ + +在安装集æˆç›‘控时，Rancher å…许进行一些é…置，这些é…ç½®å–决于你的集群规模和其中è¿è¡Œçš„工作负载。本章将更详细地介ç»è¿™äº›å†…容。 + +### 存储和数æ®ä¿ç•™ + +Prometheus 所需的存储空间å–决于你存储的时间åºåˆ—和标签的数é‡ï¼Œä»¥åŠä½ é…置的数æ®ä¿ç•™é‡ã€‚需è¦æ³¨æ„的是，Prometheus 并ä¸æ˜¯ç”¨æ¥é•¿æœŸå­˜å‚¨æŒ‡æ ‡çš„。数æ®é€šå¸¸åªä¿ç•™å‡ å¤©ï¼Œè€Œä¸æ˜¯å‡ å‘¨æˆ–几个月。这是因为 Prometheus ä¸ä¼šå¯¹å­˜å‚¨çš„指标进行任何èšåˆã€‚ä¸èšåˆçš„好处是é¿å…稀释数æ®ï¼Œä½†è¿™ä¹Ÿæ„味ç€ä¸è®¾ç½®ä¿ç•™æ—¶é•¿çš„è¯ï¼Œæ‰€éœ€çš„存储空间éšç€æ—¶é—´çš„推移线性增长。 + +以下是计算所需存储空间的一ç§æ–¹æ³•ã€‚首先，通过以下查询æ¥æŸ¥çœ‹ Prometheus 中存储å—çš„å¹³å‡å¤§å°ï¼š + +``` +rate(prometheus_tsdb_compaction_chunk_size_bytes_sum[1h]) / rate(prometheus_tsdb_compaction_chunk_samples_sum[1h]) +``` + +接下æ¥ï¼Œæ‰¾å‡ºæ¯ç§’çš„æ•°æ®å¼•å…¥é€ŸçŽ‡ï¼š + +``` +rate(prometheus_tsdb_head_samples_appended_total[1h]) +``` + +然åŽä¸Žä¿ç•™æ—¶é—´ç›¸ä¹˜ï¼Œå¹¶æ·»åŠ å‡ ä¸ªç™¾åˆ†ç‚¹ä½œä¸ºç¼“冲区： + +``` +å¹³å‡å—大å°ï¼ˆä»¥å­—节为å•ä½ï¼‰x æ¯ç§’çš„æ•°æ®å¼•å…¥é€ŸçŽ‡ x ä¿ç•™æ—¶é—´ï¼ˆä»¥ç§’为å•ä½ï¼‰x 1.1 = 必需的存储空间（以字节为å•ä½ï¼‰ +``` + +ä½ å¯ä»¥è®¿é—®[这篇åšå®¢](https://www.robustperception.io/how-much-disk-space-do-prometheus-blocks-use)了解关于如何计算必须的存储空间的更多信æ¯ã€‚ + +ä½ å¯ä»¥å‚è§ [Prometheus 官方文档](https://prometheus.io/docs/prometheus/latest/storage)æ¥è¿›ä¸€æ­¥äº†è§£ Prometheus 的存储概念。 + +### CPU 和内存的请求和é™åˆ¶ + +在较大的 Kubernetes 集群中，Prometheus 会消耗大é‡å†…存。Prometheus 需è¦çš„内存与它存储的时间åºåˆ—和标签的数é‡ä»¥åŠæŠ“å–间隔有关。 + +ä½ å¯ä»¥è®¿é—®[这篇åšå®¢](https://www.robustperception.io/how-much-ram-does-prometheus-2-x-need-for-cardinality-and-ingestion)了解关于如何计算必须的内存的更多信æ¯ã€‚ + +所需的 CPU æ•°é‡ä¸Žä½ æ­£åœ¨æ‰§è¡Œçš„查询数é‡ç›¸å…³ã€‚ + +### 长期储存 + +Prometheus ä¸æ˜¯ç”¨äºŽé•¿æœŸå­˜å‚¨æŒ‡æ ‡çš„，它åªç”¨äºŽçŸ­æœŸå­˜å‚¨ã€‚ + +如果想è¦é•¿æ—¶é—´å­˜å‚¨éƒ¨åˆ†æˆ–全部指标，你å¯ä»¥åˆ©ç”¨ Prometheus çš„[远程读/写](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)功能将其连接到 [Thanos](https://thanos.io/)，[InfluxDB](https://www.influxdata.com/) 或 [M3DB](https://www.m3db.io/) 等存储系统。你å¯ä»¥è®¿é—®[这篇åšå®¢](https://rancher.com/blog/2020/prometheus-metric-federation)找到设置示例。 + +## 抓å–自定义工作负载 + +虽然集æˆçš„ Rancher Monitoring å·²ç»å¯ä»¥ä»Žé›†ç¾¤çš„节点和系统组件中抓å–系统指标，但你也需è¦ä¸ºéƒ¨ç½²åœ¨ Kubernetes 上的自定义工作负载抓å–æ•°æ®ã€‚为此，你å¯ä»¥é…ç½® Prometheus，让它在一定的时间间隔内对你应用的端点进行 HTTP 请求。然åŽï¼Œè¿™äº›ç«¯ç‚¹ä¼šä»¥ Prometheus æ ¼å¼è¿”回指标。 + +一般æ¥è¯´ï¼Œä½ ä¼šä»Žé›†ç¾¤ä¸­è¿è¡Œçš„所有工作负载中抓å–æ•°æ®ï¼Œç„¶åŽå°†æ•°æ®ç”¨äºŽå‘Šè­¦æˆ–调试问题。一般情况下，你åªæœ‰åœ¨å…·ä½“事件å‘生åŽæ‰éœ€è¦æŸäº›æŒ‡æ ‡æ•°æ®ã€‚如果数æ®å·²ç»è¢«æŠ“å–并存储了，那就好办了。由于 Prometheus åªæ˜¯çŸ­æœŸå­˜å‚¨æŒ‡æ ‡ï¼Œå› æ­¤æŠ“å–和存储大é‡æ•°æ®çš„æˆæœ¬å¹¶ä¸æ˜¯é‚£ä¹ˆé«˜ã€‚如果你使用的是 Prometheus 的长期存储方案，那么你也å¯ä»¥å†³å®šæŒä¹…存储哪些数æ®ã€‚ + +### 关于 Prometheus Exporters + +许多第三方工作负载（如数æ®åº“ã€é˜Ÿåˆ—或 Web æœåŠ¡å™¨ï¼‰è¦ä¹ˆå·²ç»æ”¯æŒä»¥ Prometheus æ ¼å¼å…¬å¼€æŒ‡æ ‡ï¼Œè¦ä¹ˆæœ‰æ‰€è°“çš„ exporter，æ¥å¯¹å·¥å…·çš„指标格å¼å’Œ Prometheus ç†è§£çš„指标格å¼è¿›è¡Œè½¬æ¢ã€‚通常，你å¯ä»¥å°†è¿™äº› exporter 作为é¢å¤–çš„ Sidecar 容器添加到工作负载的 Pod 中。很多 Helm Chart å·²ç»åŒ…å«äº†éƒ¨ç½² exporter 的选项。此外，你还å¯ä»¥åœ¨ [promcat.io](https://promcat.io/) å’Œ [ExporterHub](https://exporterhub.io/) 上找到一个由 SysDig 策划的 exporter 列表。 + +### Prometheus çš„ç¼–ç¨‹è¯­è¨€å’Œæ¡†æž¶æ”¯æŒ + +è¦æƒ³æŠŠä½ çš„自定义应用指标放到 Prometheus 中，你必须直接从你的应用代ç ä¸­æ”¶é›†å’Œæš´éœ²è¿™äº›æŒ‡æ ‡ã€‚幸è¿çš„是，对于大多数æµè¡Œçš„编程语言和框架æ¥è¯´ï¼Œå·²ç»æœ‰ä¸€äº›åº“和集æˆæ¥å¸®åŠ©è§£å†³è¿™ä¸ªé—®é¢˜ã€‚其中一个例å­æ˜¯ [Spring 框架](https://docs.spring.io/spring-metrics/docs/current/public/prometheus)中的 Prometheus 支æŒã€‚ + +### ServiceMonitor å’Œ PodMonitor + +一旦所有工作负载都以 Prometheus æ ¼å¼å…¬å¼€äº†æŒ‡æ ‡åŽï¼Œä½ å¿…é¡»é…ç½® Prometheus æ¥æŠ“å–æ•°æ®ã€‚Rancher 使用 [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator)，这使得使用 ServiceMonitors å’Œ PodMonitors æ¥æ·»åŠ å…¶ä»–抓å–目标å˜å¾—容易。很多 Helm Chart å·²ç»åŒ…å«ç›´æŽ¥åˆ›å»ºè¿™äº›ç›‘控器的选项。你也å¯ä»¥åœ¨ Rancher 文档中找到更多信æ¯ã€‚ + +### Prometheus 推é€ç½‘关（Pushgateway） + +有些工作负载的指标很难被 Prometheus 抓å–ï¼Œå°±åƒ Jobs å’Œ CronJobs 这样的短期工作负载，或者是ä¸å…许在å•ä¸ªå¤„ç†çš„传入请求之间共享数æ®çš„应用，如 PHP 应用。 + +è¦æƒ³èŽ·å¾—这些用例的指标，你å¯ä»¥è®¾ç½® [prometheus-pushgateways](https://github.com/prometheus/pushgateway)。CronJob 或 PHP 应用将把指标更新推é€åˆ° pushgateway。pushgateway 汇总并通过 HTTP 端点暴露它们，然åŽå¯ä»¥ç”± Prometheus 抓å–。 + +### Prometheus Blackbox 监控 + +有时，你å¯èƒ½éœ€è¦ä»Žå¤–部监控工作负载。为此，你å¯ä»¥ä½¿ç”¨ [Prometheus blackbox-exporter](https://github.com/prometheus/blackbox_exporter)，它å…许通过 HTTPã€HTTPSã€DNSã€TCP å’Œ ICMP 探测任何类型的端点。 + +## 在（微）æœåŠ¡æž¶æž„中进行监控 + +如果你有一个（微）æœåŠ¡æž¶æž„，在该架构中集群的多个å•ç‹¬çš„工作负载相互通信，那么拥有这些æµé‡çš„详细指标和跟踪是éžå¸¸é‡è¦çš„，因为这å¯ä»¥å¸®åŠ©ä½ äº†è§£æ‰€æœ‰è¿™äº›å·¥ä½œè´Ÿè½½ä¹‹é—´çš„通信方å¼ï¼Œä»¥åŠé—®é¢˜æˆ–瓶颈å¯èƒ½å‡ºçŽ°çš„地方。 + +当然，你å¯ä»¥ç›‘控所有工作负载中的所有内部æµé‡ï¼Œå¹¶å°†è¿™äº›æŒ‡æ ‡æš´éœ²ç»™ Prometheusï¼Œä½†è¿™ç›¸å½“è€—è´¹ç²¾åŠ›ã€‚åƒ Istio 这样的æœåŠ¡ç½‘格（å¯ä»¥é€šè¿‡[å•å‡»](../../../pages-for-subheaders/istio.md)在 Rancher 中安装）å¯ä»¥è‡ªåŠ¨å®Œæˆè¿™é¡¹å·¥ä½œï¼Œå¹¶æ供所有 Service 之间æµé‡çš„丰富的é¥æµ‹æ•°æ®ã€‚ + +## 真实用户监控 + +监控所有内部工作负载的å¯ç”¨æ€§å’Œæ€§èƒ½å¯¹äºŽç¨³å®šã€å¯é å’Œå¿«é€Ÿåœ°è¿è¡Œåº”用至关é‡è¦ã€‚但这些指标åªèƒ½å‘你展示部分情况。è¦æƒ³èŽ·å¾—一个完整的视图，还必须知é“你的最终用户是如何实际感知的。为此，你å¯ä»¥ç ”究å„ç§[真实用户监控解决方案](https://en.wikipedia.org/wiki/Real_user_monitoring)。 + +## 安全监控 + +除了通过监控工作负载æ¥æ£€æµ‹æ€§èƒ½ã€å¯ç”¨æ€§æˆ–å¯æ‰©å±•æ€§ä¹‹å¤–，你还应该监控集群和è¿è¡Œåœ¨é›†ç¾¤ä¸­çš„工作负载，æ¥å‘现潜在的安全问题。一个好的åšæ³•æ˜¯ç»å¸¸è¿è¡Œ [CIS 扫æ](../../../pages-for-subheaders/cis-scan-guides.md)并å‘出告警，æ¥æ£€æŸ¥é›†ç¾¤æ˜¯å¦æŒ‰ç…§å®‰å…¨æœ€ä½³å®žè·µè¿›è¡Œé…置。 + +对于工作负载，你å¯ä»¥æŸ¥çœ‹ Kubernetes å’Œ Container 安全解决方案，例如 [NeuVector](https://www.suse.com/products/neuvector/)ã€[Falco](https://falco.org/)ã€[Aqua Kubernetes Security](https://www.aquasec.com/solutions/kubernetes-container-security/) å’Œ [SysDig](https://sysdig.com/)。 + +## 设置告警 + +将所有的指标纳入监控系统并在仪表æ¿ä¸­å¯è§†åŒ–是很好的åšæ³•ï¼Œä½†ä½ ä¹Ÿå¸Œæœ›åœ¨å‡ºçŽ°é—®é¢˜æ—¶èƒ½ä¸»åŠ¨æ”¶åˆ°æ醒。 + +集æˆçš„ Rancher 监控已ç»é…置了一套åˆç†çš„告警，这些告警在任何 Kubernetes 集群中都是å¯ç”¨çš„。你å¯ä»¥æ‰©å±•å‘Šè­¦ï¼Œæ¥è¦†ç›–特定的工作负载和用例。 + +在设置告警时，你需è¦ä¸ºå¯¹ä½ åº”用éžå¸¸å…³é”®çš„工作负载é…置告警，但åŒæ—¶ä¹Ÿè¦ç¡®ä¿å‘Šè­¦ä¸ä¼šå¤ªé¢‘ç¹ã€‚ç†æƒ³æƒ…况下，你收到的æ¯ä¸€ä¸ªå‘Šè­¦éƒ½åº”该是一个你需è¦å…³æ³¨å¹¶è§£å†³çš„问题。如果你一直收到ä¸å¤ªå…³é”®çš„告警，你就有å¯èƒ½å¼€å§‹å®Œå…¨å¿½ç•¥å‘Šè­¦ä¿¡æ¯ï¼Œç„¶åŽé”™è¿‡çœŸæ­£é‡è¦çš„告警。因此，少é‡çš„å‘Šè­¦å¯èƒ½ä¼šæ›´å¥½ã€‚首先，你å¯ä»¥å…³æ³¨çœŸæ­£é‡è¦çš„指标，例如应用离线等。之åŽï¼Œè§£å†³å‡ºçŽ°çš„所有问题，然åŽå†åˆ›å»ºæ›´è¯¦ç»†çš„告警。 + +如果告警开始å‘é€ï¼Œä½†ä½ æš‚时无法处ç†ï¼Œä½ ä¹Ÿå¯ä»¥å°†å‘Šè­¦é™é»˜ä¸€å®šæ—¶é—´ï¼Œä»¥ä¾¿ä»¥åŽæŸ¥çœ‹ã€‚ + +如果需è¦äº†è§£æ›´å¤šå…³äºŽå¦‚何设置告警和通知通é“çš„ä¿¡æ¯ï¼Œè¯·è®¿é—® [Rancher 文档中心](../../../pages-for-subheaders/monitoring-and-alerting.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md new file mode 100644 index 00000000000..de11f24cedd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -0,0 +1,58 @@ +--- +title: Rancher ç®¡ç† vSphere 集群的最佳实践 +--- + +本指å—概述了在 vSphere 环境中é…置下游 Rancher 集群的å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 + +- [1. 虚拟机注æ„事项](#1-虚拟机注æ„事项) +- [2. 网络注æ„事项](#2-网络注æ„事项) +- [3. 存储注æ„事项](#3-存储注æ„事项) +- [4. 备份和ç¾éš¾æ¢å¤](#4-备份和ç¾éš¾æ¢å¤) + +
解决方案概述
+ +![解决方案概述](/img/solution_overview.drawio.svg) + +## 1. 虚拟机注æ„事项 + +### 利用虚拟机模æ¿æ¥æž„建环境 + +为了ä¿è¯è·¨çŽ¯å¢ƒéƒ¨ç½²çš„虚拟机的一致性，你å¯ä»¥è€ƒè™‘使用虚拟机模æ¿å½¢å¼çš„黄金镜åƒï¼ˆgolden image）。你å¯ä»¥ä½¿ç”¨ Packer æ¥å®žçŽ°ï¼Œä»Žè€Œå¢žåŠ æ›´å¤šè‡ªå®šä¹‰é€‰é¡¹ã€‚ + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ ESXi 主机上分离下游集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚ç‚¹è™šæ‹Ÿæœºåˆ†å¸ƒåœ¨å¤šå° ESXi 主机上，从而防止主机级别的å•ç‚¹æ•…障。 + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ Datastore 上分离下游集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚点虚拟机分布在多个 Datastore 上，从而防止 Datastore 级别的å•ç‚¹æ•…障。 + +### 为 Kubernetes é…ç½®åˆé€‚的虚拟机 + +在部署节点时，请éµå¾ª K8s å’Œ etcd 的最佳实践，其中包括ç¦ç”¨ swap，检查集群中的所有主机之间是å¦æœ‰è‰¯å¥½çš„网络连接，为æ¯ä¸ªèŠ‚点使用唯一的主机åã€MAC 地å€å’Œ `product_uuids`。 + +## 2. 网络注æ„事项 + +### 利用 ETCD 节点之间的低延迟和高带宽连接 + +å°½å¯èƒ½åœ¨å•ä¸ªæ•°æ®ä¸­å¿ƒå†…部署 etcd æˆå‘˜ï¼Œæ¥é¿å…延迟开销并å‡å°‘网络分区的å¯èƒ½æ€§ã€‚大多数情况下，1Gb 的连接就足够了。对于大型集群，10Gb 的连接å¯ä»¥ç¼©çŸ­æ¢å¤å¤‡ä»½æ‰€éœ€çš„时间。 + +### 为虚拟机æ供固定的 IP åœ°å€ + +ä½ å¯ä»¥ä¸ºä½¿ç”¨çš„所有节点都é…置一个é™æ€ IP。如果使用 DHCP，则æ¯ä¸ªèŠ‚点都应该有一个 DHCP 预留，以确ä¿èŠ‚点分é…到相åŒçš„ IP 地å€ã€‚ + +## 3. 存储注æ„事项 + +### 在 ETCD 节点上使用 SSD ç£ç›˜ + +ETCD 对写入延迟éžå¸¸æ•æ„Ÿã€‚因此，你å¯ä»¥å°½é‡ä½¿ç”¨ SSD ç£ç›˜æ¥æ高写入性能。 + +## 4. 备份和ç¾éš¾æ¢å¤ + +### 定期备份下游集群 + +Kubernetes 使用 etcd æ¥å­˜å‚¨å…¶æ‰€æœ‰æ•°æ®ï¼ŒåŒ…括é…ç½®ã€çŠ¶æ€å’Œå…ƒæ•°æ®ã€‚在ç¾éš¾æ¢å¤çš„情况下，备份这些数æ®æ˜¯è‡³å…³é‡è¦çš„。 + +### 备份下游节点虚拟机 + +å°† Rancher 下游节点的虚拟机纳入标准的虚拟机备份策略中。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md new file mode 100644 index 00000000000..d48c0cf2af2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -0,0 +1,52 @@ +--- +title: 设置容器的技巧 +--- + +é…置良好的容器å¯ä»¥æžå¤§åœ°æ高环境的整体性能和安全性。 + +本文介ç»ä¸€äº›è®¾ç½®å®¹å™¨çš„技巧。 + +如果你需è¦äº†è§£å®¹å™¨å®‰å…¨çš„详细信æ¯ï¼Œä¹Ÿå¯ä»¥å‚è§ Rancher çš„[容器安全指å—](https://rancher.com/complete-guide-container-security)。 + +### 使用通用容器æ“作系统 + +在å¯èƒ½çš„情况下，你应该尽é‡åœ¨é€šç”¨çš„容器基础æ“作系统上进行标准化。 + +Alpine å’Œ BusyBox 等较å°çš„å‘行版å‡å°‘了容器镜åƒçš„大å°ï¼Œå¹¶ä¸”通常具有较å°çš„æ¼æ´žã€‚ + +æµè¡Œçš„å‘行版如 Ubuntuã€Fedora å’Œ CentOS 等都ç»è¿‡äº†å¤§é‡çš„测试，并æ供了更多的功能。 + +### 使用 From scratch 容器 +如果你的微æœåŠ¡æ˜¯ä¸€ä¸ªç‹¬ç«‹çš„é™æ€äºŒè¿›åˆ¶ï¼Œä½ åº”该使用 `From scratch` 容器。 + +`FROM scratch` 容器是一个[官方 Docker é•œåƒ](https://hub.docker.com/_/scratch)，它是空的，这样你就å¯ä»¥ç”¨å®ƒæ¥è®¾è®¡æœ€å°çš„é•œåƒã€‚ + +这个镜åƒè¿™å°†å…·æœ‰æœ€å°çš„攻击层和最å°çš„é•œåƒå¤§å°ã€‚ + +### 以éžç‰¹æƒæ–¹å¼è¿è¡Œå®¹å™¨è¿›ç¨‹ +在å¯èƒ½çš„情况下，在容器内è¿è¡Œè¿›ç¨‹æ—¶ä½¿ç”¨éžç‰¹æƒç”¨æˆ·ã€‚虽然容器è¿è¡Œæ—¶æ供了隔离，但ä»ç„¶å¯èƒ½å­˜åœ¨æ¼æ´žå’Œæ”»å‡»ã€‚如果容器以 root 身份è¿è¡Œï¼Œæ— æ„中或æ„外的主机挂载也会å—到影å“。有关为 Pod 或容器é…置安全上下文的详细信æ¯ï¼Œè¯·å‚è§ [Kubernetes 文档](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)。 + +### 定义资æºé™åˆ¶ +你应该将 CPU 和内存é™åˆ¶åº”用到你的 Pod 上。这å¯ä»¥å¸®åŠ©ç®¡ç† worker 节点上的资æºï¼Œå¹¶é¿å…å‘生故障的微æœåŠ¡å½±å“其他微æœåŠ¡ã€‚ + +在标准 Kubernetes 中，你å¯ä»¥è®¾ç½®å‘½å空间级别的资æºé™åˆ¶ã€‚在 Rancher 中，你å¯ä»¥è®¾ç½®é¡¹ç›®çº§åˆ«çš„资æºé™åˆ¶ï¼Œé¡¹ç›®å†…的所有命å空间都会继承这些é™åˆ¶ã€‚详情请å‚è§ Rancher 官方文档。 + +在设置资æºé…é¢æ—¶ï¼Œå¦‚果你在项目或命å空间上设置了任何与 CPU 或内存相关的内容（å³é™åˆ¶æˆ–预留），所有容器都需è¦åœ¨åˆ›å»ºæœŸé—´è®¾ç½®å„自的 CPU 或内存字段。为了é¿å…在创建工作负载期间对æ¯ä¸ªå®¹å™¨è®¾ç½®è¿™äº›é™åˆ¶ï¼Œå¯ä»¥åœ¨å‘½å空间上指定一个默认的容器资æºé™åˆ¶ã€‚ + +有关如何在[容器级别](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container)和命å空间级别设置资æºé™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚è§ Kubernetes 文档。 + +### 定义资æºéœ€æ±‚ +你应该将 CPU 和内存è¦æ±‚应用到你的 Pod 上。这对于通知调度器需è¦å°†ä½ çš„ pod 放置在哪ç§ç±»åž‹çš„计算节点上，并确ä¿å®ƒä¸ä¼šè¿‡åº¦é…置该节点资æºè‡³å…³é‡è¦ã€‚在 Kubernetes 中，你å¯ä»¥é€šè¿‡åœ¨ pod 的容器规范的资æºè¯·æ±‚字段中定义 `resources.requests` æ¥è®¾ç½®èµ„æºéœ€æ±‚。详情请å‚è§ [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container)。 + +:::note + +如果你为 pod 所部署的命å空间设置了资æºé™åˆ¶ï¼Œè€Œå®¹å™¨æ²¡æœ‰ç‰¹å®šçš„资æºè¯·æ±‚，则ä¸å…许å¯åŠ¨ pod。为了é¿å…在创建工作负载期间对æ¯ä¸ªå®¹å™¨è®¾ç½®è¿™äº›å­—段，å¯ä»¥åœ¨å‘½å空间上指定一个默认的容器资æºé™åˆ¶ã€‚ + +::: + +建议在容器级别上定义资æºéœ€æ±‚，å¦åˆ™ï¼Œè°ƒåº¦å™¨ä¼šè®¤ä¸ºé›†ç¾¤åŠ è½½å¯¹ä½ çš„应用没有帮助。 + +### é…置存活和就绪探测器 +ä½ å¯ä»¥ä¸ºä½ çš„容器é…置存活探测器和就绪探测器。如果你的容器ä¸æ˜¯å®Œå…¨å´©æºƒï¼ŒKubernetes 是ä¸ä¼šçŸ¥é“它是ä¸å¥åº·çš„，除éžä½ åˆ›å»ºä¸€ä¸ªå¯ä»¥æŠ¥å‘Šå®¹å™¨çŠ¶æ€çš„端点或机制。或者，确ä¿ä½ çš„容器在ä¸å¥åº·çš„情况下åœæ­¢å¹¶å´©æºƒã€‚ + +Kubernetes 文档展示了如何[为容器é…置存活和就绪探测器](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md new file mode 100644 index 00000000000..8ffe650d009 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -0,0 +1,84 @@ +--- +title: 在 vSphere 环境中安装 Rancher +--- + +本指å—概述了在 vSphere 环境中在 RKE Kubernetes 集群上安装 Rancher çš„å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 + + +
解决方案概述
+ +![解决方案概述](/img/rancher-on-prem-vsphere.svg) + +## 1. è´Ÿè½½å‡è¡¡å™¨æ³¨æ„事项 + +你需è¦ä½¿ç”¨ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨å°†æµé‡è½¬å‘到 RKE 节点上的 Rancher 工作负载。 + +### 利用容错和高å¯ç”¨æ€§ + +请充分利用具有继承高å¯ç”¨åŠŸèƒ½çš„外部（硬件或软件）负载å‡è¡¡å™¨ï¼ˆå¦‚：F5ã€NSX-Tã€Keepalived 等）。 + +### 备份负载å‡è¡¡å™¨é…ç½® + +在ç¾éš¾æ¢å¤æ—¶ï¼Œå¯ç”¨çš„è´Ÿè½½å‡è¡¡å™¨é…ç½®å¯ä»¥åŠ å¿«æ¢å¤è¿‡ç¨‹ã€‚ + +### é…ç½®å¥åº·æ£€æŸ¥ + +让负载å‡è¡¡å™¨åœ¨å¥åº·æ£€æŸ¥å¤±è´¥æ—¶è‡ªåŠ¨å°†èŠ‚点标记为ä¸å¯ç”¨ã€‚例如，NGINX å¯ä»¥é€šè¿‡ä»¥ä¸‹é…ç½®æ¥å®žçŽ°è¿™ä¸€åŠŸèƒ½ï¼š + +`max_fails=3 fail_timeout=5s` + +### 利用外部负载å‡è¡¡å™¨ + +é¿å…在管ç†é›†ç¾¤å†…使用软件负载å‡è¡¡å™¨ã€‚ + +### 安全访问 Rancher + +将防ç«å¢™/ACL 规则é…置为åªå…许 Rancher 访问。 + +## 2. 虚拟机注æ„事项 + +### æ ¹æ® Rancher æ–‡æ¡£ç¡®å®šè™šæ‹Ÿæœºçš„å¤§å° + +请å‚阅[安装è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)。 + +### 利用虚拟机模æ¿æ¥æž„建环境 + +为了ä¿è¯è·¨çŽ¯å¢ƒéƒ¨ç½²çš„虚拟机的一致性，你å¯ä»¥è€ƒè™‘使用虚拟机模æ¿å½¢å¼çš„黄金镜åƒï¼ˆgolden image）。你å¯ä»¥ä½¿ç”¨ Packer æ¥å®žçŽ°ï¼Œä»Žè€Œå¢žåŠ æ›´å¤šè‡ªå®šä¹‰é€‰é¡¹ã€‚ + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ ESXi 主机上分离 Rancher 集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚ç‚¹è™šæ‹Ÿæœºåˆ†å¸ƒåœ¨å¤šå° ESXi 主机上，从而防止主机级别的å•ç‚¹æ•…障。 + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ Datastore 上分离 Rancher 集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚点虚拟机分布在多个 Datastore 上，从而防止 Datastore 级别的å•ç‚¹æ•…障。 + +### 为 Kubernetes é…ç½®åˆé€‚的虚拟机 + +在部署节点时，请éµå¾ª K8s å’Œ etcd 的最佳实践，其中包括ç¦ç”¨ swap，检查集群中的所有主机之间是å¦æœ‰è‰¯å¥½çš„网络连接，为æ¯ä¸ªèŠ‚点使用唯一的主机åã€MAC 地å€å’Œ `product_uuids`。 + +## 3. 网络注æ„事项 + +### 利用 ETCD 节点之间的低延迟和高带宽连接 + +å°½å¯èƒ½åœ¨å•ä¸ªæ•°æ®ä¸­å¿ƒå†…部署 etcd æˆå‘˜ï¼Œæ¥é¿å…延迟开销并å‡å°‘网络分区的å¯èƒ½æ€§ã€‚大多数情况下，1Gb 的连接就足够了。对于大型集群，10Gb 的连接å¯ä»¥ç¼©çŸ­æ¢å¤å¤‡ä»½æ‰€éœ€çš„时间。 + +### 为虚拟机æ供固定的 IP åœ°å€ + +ä½ å¯ä»¥ä¸ºä½¿ç”¨çš„所有节点都é…置一个é™æ€ IP。如果使用 DHCP，则æ¯ä¸ªèŠ‚点都应该有一个 DHCP 预留，以确ä¿èŠ‚点分é…到相åŒçš„ IP 地å€ã€‚ + +## 4. 存储注æ„事项 + +### 在 ETCD 节点上使用 SSD ç£ç›˜ + +ETCD 对写入延迟éžå¸¸æ•æ„Ÿã€‚因此，你å¯ä»¥å°½é‡ä½¿ç”¨ SSD ç£ç›˜æ¥æ高写入性能。 + +## 5. 备份和ç¾éš¾æ¢å¤ + +### 定期备份管ç†é›†ç¾¤ + +Rancher 将数æ®å­˜å‚¨åœ¨å®ƒæ‰€åœ¨çš„ Kubernetes 集群的 ETCD datastore 中。与其它 Kubernetes 集群一样，你需è¦å¯¹è¯¥é›†ç¾¤è¿›è¡Œé¢‘ç¹ä¸”ç»è¿‡æµ‹è¯•çš„备份。 + +### 备份 Rancher 集群节点虚拟机 + +å°† Rancher 管ç†èŠ‚点的虚拟机纳入标准的虚拟机备份策略中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md new file mode 100644 index 00000000000..8c4d1541296 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -0,0 +1,36 @@ +--- +title: Rancher 部署策略 +--- + +本文æä¾› Rancher 实例的两ç§éƒ¨ç½²ç­–略，用于管ç†ä¸‹æ¸¸ Kubernetes 集群。æ¯ç§ç­–略都有优缺点。请按照你的实际情况选择最适åˆçš„部署策略。 + +## 中心è¾å°„åž‹ç­–ç•¥ +--- + +在中心è¾å°„型部署中，一个 Rancher 实例就å¯ä»¥ç®¡ç†é布全çƒçš„ Kubernetes 集群。Rancher 实例è¿è¡Œåœ¨é«˜å¯ç”¨ Kubernetes 集群上，并且会å—延迟影å“。 + +### 优点 + +* å¯ä»¥é€šè¿‡ä¸€ä¸ª controlplane ç•Œé¢æŸ¥çœ‹æ‰€æœ‰åŒºåŸŸå’ŒçŽ¯å¢ƒã€‚ +* Kubernetes ä¸éœ€è¦ Rancher 进行æ“作，并且å¯ä»¥å®¹å¿ä¸Ž Rancher 实例断开连接。 + +### 缺点 + +* å—é™äºŽç½‘络延迟。 +* 如果 Rancher 出现故障，在æ¢å¤ä¹‹å‰ä¸å¯ä»¥åœ¨å…¨çƒèŒƒå›´å†…创建新æœåŠ¡ã€‚但是æ¯ä¸ª Kubernetes 集群都å¯ä»¥ç»§ç»­å•ç‹¬ç®¡ç†ã€‚ + +## 区域型策略 +--- +在区域型部署模型中，Rancher 实例部署在é è¿‘下游 Kubernetes 集群的ä½ç½®ã€‚ + +### 优点 + +* 如果æŸä¸ªåŒºåŸŸçš„ Rancher 实例出现故障，其他区域内的 Rancher 功能ä»ç„¶å¯ä»¥ä¿æŒæ­£å¸¸ã€‚ +* Rancher 和下游集群之间的网络延迟大大é™ä½Žï¼Œæ高了 Rancher 的性能。 +* ä½ å¯ä»¥åœ¨æ¯ä¸ªåŒºåŸŸå†…独立å‡çº§ Rancher。 + +### 缺点 + +* 管ç†å¤šä¸ª Rancher 安装的开销较大。 +* 需è¦åœ¨å¤šä¸ªç•Œé¢ä¸­æŸ¥çœ‹ä¸åŒåŒºåŸŸçš„ Kubernetes 集群。 +* 在 Rancher 中部署多集群应用时，需è¦åœ¨æ¯ä¸ª Rancher Server 中é‡å¤éƒ¨ç½²æ­¥éª¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md new file mode 100644 index 00000000000..0c47b75fa06 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -0,0 +1,36 @@ +--- +title: Rancher è¿è¡ŒæŠ€å·§ +--- + +本指å—适用于使用 Rancher 管ç†ä¸‹æ¸¸ Kubernetes 集群的用例。高å¯ç”¨è®¾ç½®å¯ä»¥é˜²æ­¢ Rancher Server ä¸å¯ç”¨æ—¶æ— æ³•è®¿é—®ä¸‹æ¸¸é›†ç¾¤ã€‚ + +高å¯ç”¨ Rancher 安装指将 Rancher 安装到至少有三个节点的 Kubernetes 集群上，适用于所有生产环境以åŠé‡è¦çš„安装场景。在多个节点上è¿è¡Œå¤šä¸ª Rancher 实例能够实现å•èŠ‚点安装无法æ供的高å¯ç”¨æ€§ã€‚ + +如果你在 vSphere 环境中安装 Rancher，请å‚è§[最佳实践文档](on-premises-rancher-in-vsphere.md)。 + +在设置高å¯ç”¨ Rancher 安装时，请考虑以下事项。 + +### 在å•ç‹¬çš„集群上è¿è¡Œ Rancher +ä¸è¦åœ¨å®‰è£…了 Rancher çš„ Kubernetes 集群上è¿è¡Œå…¶ä»–工作负载或微æœåŠ¡ã€‚ + +### ç¡®ä¿ Kubernetes 节点é…置正确 +在部署节点时，请éµå¾ª K8s å’Œ etcd 的最佳实践，其中包括ç¦ç”¨ swap，检查集群中的所有主机之间是å¦æœ‰è‰¯å¥½çš„网络连接，为æ¯ä¸ªèŠ‚点使用唯一的主机åã€MAC 地å€å’Œ `product_uuids`，检查所需端å£æ˜¯å¦å·²ç»æ‰“开，并使用é…ç½® SSD çš„ etcd 进行部署。详情请å‚è§ [kubernetes 官方文档](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin)å’Œ [etcd 性能æ“作指å—](https://etcd.io/docs/v3.4/op-guide/performance/)。 + +### 使用 RKE 时：备份状æ€æ–‡ä»¶ï¼ˆStatefile） +RKE 将集群状æ€è®°å½•åœ¨ä¸€ä¸ªå为 `cluster.rkestate` 的文件中，该文件对集群的æ¢å¤å’Œ/或通过 RKE 维护集群éžå¸¸é‡è¦ã€‚由于这个文件包å«è¯ä¹¦æ料，我们强烈建议在备份å‰å¯¹è¯¥æ–‡ä»¶è¿›è¡ŒåŠ å¯†ã€‚请在æ¯æ¬¡è¿è¡Œ `rke up` åŽå¤‡ä»½çŠ¶æ€æ–‡ä»¶ã€‚ + +### 在åŒä¸€ä¸ªæ•°æ®ä¸­å¿ƒè¿è¡Œé›†ç¾¤ä¸­çš„所有节点 +为达到最佳性能，请在åŒä¸€åœ°ç†æ•°æ®ä¸­å¿ƒè¿è¡Œæ‰€æœ‰ä¸‰ä¸ªèŠ‚点。如果你在云（如 AWS）上è¿è¡ŒèŠ‚点，请在ä¸åŒçš„å¯ç”¨åŒºï¼ˆAZ）中è¿è¡Œè¿™ä¸‰ä¸ªèŠ‚点。例如，在 us-west-2a 中è¿è¡ŒèŠ‚点 1，在 us-west-2b 中è¿è¡ŒèŠ‚点 2，在 us-west-2c 中è¿è¡ŒèŠ‚点 3。 + +### ä¿è¯å¼€å‘和生产环境的相似性 +强烈建议为è¿è¡Œ Rancher çš„ Kubernetes 集群é…备 “staging†或 “pre-production†环境。这个环境的软件和硬件é…置应该尽å¯èƒ½æŽ¥è¿‘你的生产环境。 + +### ç›‘æŽ§é›†ç¾¤ä»¥è§„åˆ’å®¹é‡ +Rancher Server çš„ Kubernetes 集群应该尽å¯èƒ½æ»¡è¶³[系统和硬件è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)。越å离系统和硬件è¦æ±‚，你å¯èƒ½é¢ä¸´çš„风险就越大。 + +但是，已å‘布的è¦æ±‚å·²ç»è€ƒè™‘了å„ç§å·¥ä½œè´Ÿè½½ç±»åž‹ï¼Œå› æ­¤ï¼ŒåŸºäºŽæŒ‡æ ‡æ¥è§„划容é‡åº”该是扩展 Rancher 的最佳实践。 + +ä½ å¯ä»¥å°† Rancher 集æˆä¸šç•Œé¢†å…ˆçš„å¼€æºç›‘控解决方案 Prometheus 以åŠèƒ½å¯è§†åŒ– Prometheus 指标的 Grafana，æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿‡ç¨‹ã€‚ + +在集群中[å¯ç”¨ç›‘控](../../../pages-for-subheaders/monitoring-and-alerting.md)åŽï¼Œä½ å¯ä»¥é€šè¿‡è®¾ç½®å‘Šè­¦é€šçŸ¥ï¼Œæ¥äº†è§£é›†ç¾¤å®¹é‡çš„使用情况。你还å¯ä»¥ä½¿ç”¨ Prometheus å’Œ Grafana 监控框架，在你扩容时建立关键指标的基线。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md new file mode 100644 index 00000000000..8d5fc725d36 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md @@ -0,0 +1,61 @@ +--- +title: Rancher 扩展技巧 +--- + +本指å—介ç»äº†æ‰©å±• Rancher 时应该考虑的方法，以åŠè¿™æ ·åšçš„难度。éšç€ç³»ç»Ÿçš„增长，性能自然会é™ä½Žï¼Œä½†æˆ‘们å¯ä»¥é‡‡å–一些措施æ¥æœ€å¤§é™åº¦åœ°å‡å°‘ Rancher 的负载，并优化 Rancher 处ç†è¿™äº›å¤§åž‹è®¾ç½®çš„能力。 + +## 优化 Rancher 性能的技巧 +* 建议åŠæ—¶æ›´æ–° Rancher çš„è¡¥ä¸ç‰ˆæœ¬ã€‚在次è¦ç‰ˆæœ¬çš„整个生命周期中，我们都会进行性能改进和错误修å¤ã€‚ä½ å¯ä»¥æŸ¥çœ‹å‘行说明决定是å¦éœ€è¦å‡çº§ï¼Œä½†åœ¨å¤§å¤šæ•°æƒ…况下，我们建议你使用最新版本。 + +* æ€§èƒ½ä¼šå— Rancher 的基础设施和下游集群的基础设施之间的延迟影å“（例如，地ç†è·ç¦»ï¼‰ã€‚如果用户需è¦é布全çƒæˆ–分布在许多地区的集群/节点，最好使用多个 Rancher。 + +* 请始终å°è¯•é€æ¸æ‰©å¤§è§„模，åŒæ—¶ç›‘控和观察å˜åŒ–。通常情况下，性能问题越早解决越好，而且在其他问题出现之å‰æ›´å®¹æ˜“解决。 + +## 最å°åŒ–本地集群的负载 +扩展 Rancher 最大的瓶颈是本地 Kubernetes 集群中的资æºå¢žé•¿ã€‚local 集群包å«æ‰€æœ‰ä¸‹æ¸¸é›†ç¾¤çš„ä¿¡æ¯ã€‚下游集群的许多æ“作将在 local 集群中创建新对象，并需è¦ä½¿ç”¨ local 集群中è¿è¡Œçš„处ç†ç¨‹åºè¿›è¡Œè®¡ç®—。 + +### 管ç†å¯¹è±¡æ•°é‡ +ETCD 最终会é‡åˆ°å­˜å‚¨çš„ Kubernetes 资æºæ•°é‡è¶…过é™åˆ¶çš„问题。暂时没有这些数字的确切记录。根æ®å†…部观察，一旦å•ä¸ªèµ„æºç±»åž‹çš„对象数é‡è¶…过 60k（通常是 Rolebindings），就很有å¯èƒ½é‡åˆ°æ€§èƒ½é—®é¢˜ã€‚ + +Rolebindings 是在 local 集群中创建的，是许多æ“作的结果。 + +å°è¯•å‡å°‘ local 集群中的 rolebindings 时需è¦æ³¨æ„： +* 仅在必è¦æ—¶å°†ç”¨æˆ·æ·»åŠ åˆ°é›†ç¾¤å’Œé¡¹ç›®ä¸­ +* 删除ä¸éœ€è¦çš„集群和项目 +* 仅在必è¦æ—¶ä½¿ç”¨è‡ªå®šä¹‰è§’色 +* 在自定义角色中使用尽å¯èƒ½å°‘的规则 +* 考虑给用户添加角色是å¦å¤šä½™ +* 使用更少但更强大的集群å¯èƒ½æ›´é«˜æ•ˆ +* 检查是å¦èƒ½é€šè¿‡åˆ›å»ºæ–°é¡¹ç›®æˆ–新集群为你的用例å‡å°‘ rolebindings。 + +### 使用新版应用程åºè€Œä¸æ˜¯æ—§ç‰ˆåº”ç”¨ç¨‹åº +Rancher ä½¿ç”¨äº†ä¸¤ä¸ªåº”ç”¨ç¨‹åº kubernetes 资æºï¼Œåˆ†åˆ«æ˜¯ apps.projects.cattle.io å’Œ apps.cattle.cattle.ioã€‚æ—§ç‰ˆåº”ç”¨ç¨‹åº apps.projects.cattle.io 最åˆæ˜¯åœ¨ Cluster Manager 中引入的，现在已ç»è¿‡æ—¶ã€‚æ–°ç‰ˆåº”ç”¨ç¨‹åº apps.catalog.cattle.io å¯åœ¨å…¶å„自集群的 Cluster Explorer 中找到。新版应用程åºæ›´å—欢迎，因为它们ä½äºŽä¸‹æ¸¸é›†ç¾¤ä¸­ï¼Œè€Œæ—§ç‰ˆåº”用程åºä½äºŽ local 集群中。 + +我们建议删除 Cluster Manager 中出现的应用程åºï¼Œå¦‚有必è¦ï¼Œå¯ä»¥ä¸ºç›®æ ‡é›†ç¾¤æ›¿æ¢ä¸º Cluster Explorer 中的应用程åºï¼Œå¹¶ä»…在集群的 Cluster Explorer 中创建åŽç»­çš„应用程åºã€‚ + +### 使用授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE） +Rancher é…置的 RKE1ã€RKE2 å’Œ K3s 集群有一个 _授æƒé›†ç¾¤ç«¯ç‚¹_ 选项。å¯ç”¨åŽï¼Œä¼šåœ¨ä¸ºé›†ç¾¤ç”Ÿæˆçš„ kubeconfigs 中添加一个上下文，该上下文使用一个直接到集群的端点，并绕过 Rancher。但是，仅å¯ç”¨æ­¤é€‰é¡¹æ˜¯ä¸å¤Ÿçš„。Kubeconfig 的用户需è¦ä½¿ç”¨ `kubectl use-context ` æ‰èƒ½å¼€å§‹ä½¿ç”¨å®ƒã€‚ + +如果ä¸ä½¿ç”¨ ACE，所有 kubeconfig 请求都会先通过 Rancher 进行路由。 + +### 实验功能：å‡å°‘事件处ç†ç¨‹åºæ‰§è¡Œçš„选项 +Rancher 的大部分逻辑都å‘生在事件处ç†ç¨‹åºä¸Šã€‚æ¯å½“更新对象以åŠå¯åŠ¨ Rancher 时，这些事件处ç†ç¨‹åºéƒ½ä¼šåœ¨å¯¹è±¡ä¸Šè¿è¡Œã€‚此外，当缓存åŒæ­¥æ—¶ï¼Œå®ƒä»¬æ¯ 15 å°æ—¶ä¼šè¿è¡Œä¸€æ¬¡ã€‚在缩放设置中，由于æ¯ä¸ªå¤„ç†ç¨‹åºéƒ½ä¼šåœ¨æ¯ä¸ªé€‚用对象上è¿è¡Œï¼Œå› æ­¤è¿è¡Œè¿™äº›è®¡åˆ’会消耗巨大的性能。但是，你å¯ä»¥ä½¿ç”¨ `CATTLE_SYNC_ONLY_CHANGED_OBJECTS` 环境å˜é‡æ¥ç¦ç”¨å¤„ç†ç¨‹åºçš„这些计划执行。如果在大约 15 å°æ—¶çš„间隔内看到资æºåˆ†é…峰值，则此设置å¯èƒ½æœ‰å¸®åŠ©ã€‚ + +环境å˜é‡çš„值å¯ä»¥æ˜¯ä»¥ä¸‹é€‰é¡¹çš„逗å·åˆ†éš”列表。这些值指的是控制器的类型（包å«å’Œè¿è¡Œå¤„ç†ç¨‹åºçš„ struct）åŠå…¶å¤„ç†ç¨‹åºã€‚如果将控制器类型添加到å˜é‡ä¸­ï¼Œé‚£ä¹ˆè¯¥ç»„控制器将无法在é‡æ–°åŒæ­¥ç¼“存时è¿è¡Œå…¶å¤„ç†ç¨‹åºã€‚ + +* `mgmt` 指的是åªè¿è¡Œåœ¨ä¸€ä¸ª Rancher 节点上的管ç†æŽ§åˆ¶å™¨ã€‚ +* `user` 指的是为æ¯ä¸ªé›†ç¾¤è¿è¡Œçš„用户控制器。其中一些è¿è¡Œåœ¨ä¸Žç®¡ç†æŽ§åˆ¶å™¨ç›¸åŒçš„节点上，而其他则è¿è¡Œåœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­ã€‚该选项针对å‰è€…。 +* `scaled` 指的是在æ¯ä¸ª Rancher 节点上è¿è¡Œçš„缩放控制器。由于缩放处ç†ç¨‹åºè´Ÿè´£çš„关键功能，因此ä¸å»ºè®®è®¾ç½®ã€‚ + +简而言之，如果你å‘现 CPU ä½¿ç”¨çŽ‡æ¯ 15 å°æ—¶å‡ºçŽ°ä¸€æ¬¡å³°å€¼ï¼Œè¯·å°† `CATTLE_SYNC_ONLY_CHANGED_OBJECTS` 环境å˜é‡æ·»åŠ åˆ°ä½ çš„ Rancher 部署中，其值为 `mgmt,user`。 + +## Rancher 之外的优化 +å½±å“性能的一个é‡è¦å› ç´ æ˜¯ local 集群åŠå…¶é…置方å¼ã€‚该集群å¯èƒ½ä¼šåœ¨ Rancher è¿è¡Œä¹‹å‰å¼•å…¥ç“¶é¢ˆã€‚当 Rancher 节点出现资æºå ç”¨è¿‡é«˜çš„情况时，你å¯ä»¥ä½¿ç”¨ top 命令æ¥åˆ¤æ–­æ˜¯ Rancher 还是 Kubernetes 组件在超é¢æ¶ˆè€—资æºã€‚ + +### ä¿æŒä½¿ç”¨æœ€æ–°ç‰ˆæœ¬çš„ Kubernetes +与 Rancher 版本类似，我们建议让你的 kubernetes 集群ä¿æŒä½¿ç”¨æœ€æ–°ç‰ˆæœ¬ã€‚这将确ä¿ä½ çš„集群能包å«å¯ç”¨çš„性能增强或错误修å¤ã€‚ + +### 优化 ETCD +[ETCD 性能](https://etcd.io/docs/v3.4/op-guide/performance/)的两个主è¦ç“¶é¢ˆæ˜¯ç£ç›˜é€Ÿåº¦å’Œç½‘络速度。对任何一个进行优化都应该能æ高性能。有关 ETCD 性能的信æ¯ï¼Œè¯·å‚阅 [etcd 性能慢（性能测试和优化）](https://www.suse.com/support/kb/doc/?id=000020100)å’Œ[为大型安装调优 etcd](https://docs.ranchermanager.rancher.io/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs)。有关ç£ç›˜çš„ä¿¡æ¯ï¼Œä½ ä¹Ÿå¯ä»¥å‚阅[我们的文档](https://docs.Ranchermanager.Rancher.io/v2.5/pages-for-subheaders/installation-requirements#disks)。 + +ç†è®ºä¸Šï¼ŒETCD 集群中的节点越多，由于å¤åˆ¶è¦æ±‚ [source](https://etcd.io/docs/v3.3/faq)，它就会越慢。这å¯èƒ½ä¸Žå¸¸è§çš„缩放方法相悖。我们还å¯ä»¥æŽ¨æ–­ï¼ŒETCD 的性能将å—到节点间è·ç¦»çš„åé¢å½±å“，因为这将å‡æ…¢ç½‘络通信。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cli-with-rancher/kubectl-utility.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cli-with-rancher/kubectl-utility.md new file mode 100644 index 00000000000..62377d846c6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cli-with-rancher/kubectl-utility.md @@ -0,0 +1,32 @@ +--- +title: kubectl å®žç”¨ç¨‹åº +--- + +## kubectl + +kubectl 用于与 Rancher 进行交互。 + +### kubectl å®žç”¨ç¨‹åº + +安装 `kubectl`。详情请å‚è§[安装 kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)。 + +è¦é…ç½® kubectl，通过 Rancher Web UI 访问你的集群，å•å‡» `Kubeconfig`，然åŽå¤åˆ¶å†…容并将其粘贴到你的 `~/.kube/config` 文件中。 + +检查是å¦å¯ä»¥æˆåŠŸè¿è¡Œ `kubectl cluster-info` 或 `kubectl get pods` 命令。 + +### 使用 kubectl å’Œ kubeconfig 令牌进行 TTL è®¤è¯ + +_è¦æ±‚_ + +如果管ç†å‘˜[关闭了 kubeconfig 令牌生æˆ](../about-the-api/api-tokens.md#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)，当你è¿è¡Œ `kubectl` 时，kubeconfig æ–‡ä»¶éœ€è¦ [Rancher CLI](./rancher-cli.md) 存在于你的 PATH 中。å¦åˆ™ï¼Œä½ ä¼šçœ‹åˆ°è¿™æ ·çš„错误信æ¯ï¼š +`Unable to connect to the server: getting credentials: exec: exec: "rancher": executable file not found in $PATH`。 + +该功能å¯ä»¥è®© kubectl 与 Rancher Server 进行身份验è¯ï¼Œå¹¶åœ¨éœ€è¦æ—¶èŽ·å¾—æ–°çš„ kubeconfig token。目å‰æ”¯æŒä»¥ä¸‹éªŒè¯æ供程åºï¼š + +1. 本地 +2. Active Directory (ä»…é™ LDAP) +3. FreeIPA +4. OpenLDAP +5. SAML 身份æ供商：Ping，Okta，ADFS，Keycloak å’Œ Shibboleth + +如果你是第一次è¿è¡Œ kubectl（例如，`kubectl get pods`），它会è¦æ±‚你选择一个验è¯æ供程åºå¹¶ä½¿ç”¨ Rancher Server 登录。kubeconfig token 会被缓存到 `./.cache/token` 下你è¿è¡Œ kubectl 的路径中。该 Token 在[过期](../about-the-api/api-tokens.md#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)或[从 Rancher Server 删除](../about-the-api/api-tokens.md#删除令牌)之å‰éƒ½æ˜¯æœ‰æ•ˆçš„。过期åŽï¼Œä¸‹ä¸€ä¸ª `kubectl get pods` 命令会è¦æ±‚ä½ å†æ¬¡ä½¿ç”¨ Rancher Server 登录。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cli-with-rancher/rancher-cli.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cli-with-rancher/rancher-cli.md new file mode 100644 index 00000000000..4cf44238004 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cli-with-rancher/rancher-cli.md @@ -0,0 +1,87 @@ +--- +title: Rancher CLI +description: Rancher CLI 是一个命令行工具，用于在工作站中与 Rancher 进行交互。 +--- + +Rancher CLI（命令行界é¢ï¼‰æ˜¯ä¸€ä¸ªå‘½ä»¤è¡Œå·¥å…·ï¼Œå¯ç”¨äºŽä¸Ž Rancher 进行交互。使用此工具，你å¯ä»¥ä½¿ç”¨å‘½ä»¤è¡Œè€Œä¸ç”¨é€šè¿‡ GUI æ¥æ“作 Rancher。 + +### 下载 Rancher CLI + +ä½ å¯ä»¥ç›´æŽ¥ UI 下载二进制文件。 + +1. 点击左上角的 **☰**。 +1. 在导航侧边æ èœå•åº•éƒ¨ï¼Œå•å‡»**简介**。 +1. 在 **CLI 下载**中，有 Windowsã€Mac å’Œ Linux 的二进制文件下载链接。你还å¯ä»¥è®¿é—®æˆ‘们的 CLI [å‘布页é¢](https://github.com/rancher/cli/releases)直接下载二进制文件。 + +### è¦æ±‚ + +下载 Rancher CLI åŽï¼Œä½ éœ€è¦è¿›è¡Œä¸€äº›é…置。Rancher CLI 需è¦ï¼š + +- ä½ çš„ Rancher Server URL，用于连接到 Rancher Server。 +- API æŒæœ‰è€…令牌（Bearer Tokenï¼‰ï¼Œç”¨äºŽå‘ Rancher 进行身份验è¯ã€‚有关获å–æŒæœ‰è€…令牌的更多信æ¯ï¼Œè¯·å‚阅[创建 API 密钥](../user-settings/api-keys.md)。 + +### CLI èº«ä»½éªŒè¯ + +在使用 Rancher CLI 控制你的 Rancher Server 之å‰ï¼Œä½ å¿…须使用 API æŒæœ‰è€…令牌进行身份验è¯ã€‚è¿è¡Œä»¥ä¸‹å‘½ä»¤è¿›è¡Œç™»å½•ï¼ˆå°† `` å’Œ `` 替æ¢ä¸ºä½ çš„实际信æ¯ï¼‰ï¼š + +```bash +$ ./rancher login https:// --token +``` + +如果 Rancher Server 使用自签åè¯ä¹¦ï¼ŒRancher CLI 会æ示你继续连接。 + +### 项目选择 + +在执行命令之å‰ï¼Œä½ å¿…须先选择一个 Rancher 项目æ¥æ‰§è¡Œè¿™äº›å‘½ä»¤ã€‚è¦é€‰æ‹©[项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)，请è¿è¡Œ `./rancher context switch` 命令。输入此命令åŽï¼Œä¼šæ˜¾ç¤ºå¯ç”¨é¡¹ç›®çš„列表。输入一个数字以选择项目。 + +**示例：`./rancher context switch` 输出** +``` +User:rancher-cli-directory user$ ./rancher context switch +NUMBER CLUSTER NAME PROJECT ID PROJECT NAME +1 cluster-2 c-7q96s:p-h4tmb project-2 +2 cluster-2 c-7q96s:project-j6z6d Default +3 cluster-1 c-lchzv:p-xbpdt project-1 +4 cluster-1 c-lchzv:project-s2mch Default +Select a Project: +``` + +输入数字åŽï¼ŒæŽ§åˆ¶å°ä¼šæ˜¾ç¤ºä½ æ‰€é€‰é¡¹ç›®çš„消æ¯ã€‚ + +``` +INFO[0005] Setting new context to project project-1 +INFO[0005] Saving config to /Users/markbishop/.ranchcli2.json +``` + +请确ä¿ä½ å¯ä»¥æˆåŠŸè¿è¡Œ `rancher kubectl get pods`。 + +### 命令 + +以下命令å¯ç”¨äºŽ Rancher CLI： + +| 命令 | 结果 | +|---|---| +| `apps, [app]` | 对商店应用（å³å•ä¸ª [Helm Chart](https://docs.helm.sh/developing_charts/)）或 Rancher Chart 执行æ“作。 | +| `catalog` | 对[应用商店](../../pages-for-subheaders/helm-charts-in-rancher.md)执行æ“作。 | +| `clusters, [cluster]` | 对[集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)执行æ“作。 | +| `context` | 在 Rancher [项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)之间切æ¢ã€‚有关示例，请å‚阅[项目选择](#项目选择)。 | +| `inspect [OPTIONS] [RESOURCEID RESOURCENAME]` | 显示 [Kubernetes 资æº](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#resource-types)或 Rancher 资æºï¼ˆå³[项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)å’Œ[工作负载](../../pages-for-subheaders/workloads-and-pods.md)）的详细信æ¯ã€‚按å称或 ID 指定资æºã€‚ | +| `kubectl` | è¿è¡Œ [kubectl 命令](https://kubernetes.io/docs/reference/kubectl/overview/#operations)。 | +| `login, [l]` | 登录 Rancher Server。有关示例，请å‚阅 [CLI 身份验è¯](#cli-身份验è¯)。 | +| `namespaces, [namespace]` | 执行命å空间æ“作。 | +| `nodes, [node]` | 执行节点空间æ“作。 | +| `projects, [project]` | 执行[项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)æ“作。 | +| `ps` | 显示项目中的[工作负载](../../pages-for-subheaders/workloads-and-pods.md)。 | +| `settings, [setting]` | 显示 Rancher Server 的当å‰è®¾ç½®ã€‚ | +| `ssh` | 使用 SSH å议连接到你的æŸä¸ªé›†ç¾¤èŠ‚点。 | +| `help, [h]` | 显示命令列表或æŸä¸ªå‘½ä»¤çš„帮助。 | + + +### Rancher CLI 帮助 + +使用 CLI 登录 Rancher Server åŽï¼Œè¾“å…¥ `./rancher --help` 以获å–命令列表。 + +æ‰€æœ‰å‘½ä»¤éƒ½æ”¯æŒ `--help` 标志，该标志解释了æ¯ä¸ªå‘½ä»¤çš„用法。 + +### é™åˆ¶ + +Rancher CLI **ä¸èƒ½**用于安装[仪表æ¿åº”用程åºæˆ– Rancher 功能 Chart](../../pages-for-subheaders/helm-charts-in-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md new file mode 100644 index 00000000000..98e6b500737 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -0,0 +1,78 @@ +--- +title: EC2 主机é…ç½®å‚考 +--- + +有关 EC2 和节点的更多详细信æ¯ï¼Œè¯·å‚阅 [EC2 管ç†æŽ§åˆ¶å°](https://aws.amazon.com/ec2)的官方文档。 + +### 区域 + +构建集群的地ç†[区域](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html)。 + +### 地区 + +[地区](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones)，一个区域内用于构建集群的隔离ä½ç½®ã€‚ + +### 实例类型 + +[实例类型](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html)用于é…置集群，能确定硬件特性。 + +### æ ¹ç£ç›˜å¤§å° + +é…ç½®[根设备](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/RootDeviceStorage.html)的大å°ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚ + +### VPC/å­ç½‘ + +[VPC](https://docs.aws.amazon.com/vpc/latest/userguide/configure-your-vpc.html) 或特定的[å­ç½‘](https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html)（VPC 中的一个 IP 范围），用于添加你的资æºã€‚ + +### IAM 实例é…置文件å称 + +示例é…置文件的å称，用于将 IAM 角色传递给 EC2 实例。 + +## 高级选项 + +### AMI ID + +用于集群中节点的 [Amazon Machine Image（AMI）](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html)。 + +### 用于 AMI çš„ SSH 用户å + +用于连接到你å¯åŠ¨çš„实例的用户å。有关选定 AMI 的默认用户å，请å‚阅[此处](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connection-prereqs.html)。对于未列出的 AMI，请咨询 AMI æ供商。 + +### 安全组 + +选择默认安全组或é…置安全组。 + +请å‚考[使用主机驱动时的 Amazon EC2 安全组](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-安全组)，了解 `rancher-nodes` 安全组中创建的规则。 + +### EBS æ ¹å·ç±»åž‹ + +用于根设备的 [EBS å·ç±»åž‹](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html)。 + +### 加密 EBS å· + +å¯ç”¨ [Amazon EBS 加密](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html)。 + +### 请求 Spot 实例 + +å¼€å¯[请求 Spot 实例](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html)选项，并指定你愿æ„支付的最高实例价格（æ¯å°æ—¶ï¼‰ã€‚ + +### 仅使用ç§æœ‰åœ°å€ + +å¯ç”¨ä»…使用[ç§äººåœ°å€](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html)的选项。 + +### EBS 优化实例 + +使用 [EBS 优化实例](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html)。 + +### å…许访问 EC2 å…ƒæ•°æ® + +å¯ç”¨å¯¹ [EC2 元数æ®](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)的访问。 + +### 为元数æ®ä½¿ç”¨ Token + +使用 [Instance Metadata Service Version 2 (IMDSv2)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html)，å³åŸºäºŽä»¤ç‰Œè®¿é—®å…ƒæ•°æ®çš„方法。 + +### 添加标签 + +使用[标签](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html)添加元数æ®ï¼Œä»Žè€Œå¯¹èµ„æºè¿›è¡Œåˆ†ç±»ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md new file mode 100644 index 00000000000..6ea2549ccf8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -0,0 +1,120 @@ +--- +title: Azure 主机é…ç½® +--- + +有关 Azure 的更多信æ¯ï¼Œè¯·å‚阅官方 [Azure 文档](https://docs.microsoft.com/en-us/azure/?product=featured)。 + +### 环境 + +Microsoft æ供了多个[云](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest)æ¥æ»¡è¶³åœ°åŒºæ³•å¾‹çš„è¦æ±‚： + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +### ä½ç½® + +é…置集群和节点的[ä½ç½®](https://docs.microsoft.com/en-us/azure/virtual-machines/regions)。 + +### 资æºç»„ + +资æºç»„æ˜¯ä¸€ä¸ªå®¹å™¨ï¼Œå…¶ä¸­åŒ…å« Azure 解决方案的相关资æºã€‚资æºç»„å¯ä»¥åŒ…括解决方案的所有资æºï¼Œæˆ–者仅包括你希望作为一个组æ¥ç®¡ç†çš„资æºã€‚ä½ å¯ä»¥æ ¹æ®ç»„织的需求æ¥å†³å®šå¦‚何将资æºåˆ†é…给资æºç»„。通常情况下，你å¯ä»¥å°†ç”Ÿå‘½å‘¨æœŸç›¸åŒçš„资æºæ·»åŠ åˆ°åŒä¸€ä¸ªèµ„æºç»„，以便轻æ¾åœ°æŒ‰ç»„进行部署ã€æ›´æ–°å’Œåˆ é™¤ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰èµ„æºç»„，也å¯ä»¥è¾“入资æºç»„çš„å称，然åŽç³»ç»Ÿå°†ä¸ºä½ åˆ›å»ºä¸€ä¸ªèµ„æºç»„。 + +有关管ç†èµ„æºç»„çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal)。 + +### å¯ç”¨æ€§é›†ï¼ˆéžæ‰˜ç®¡ï¼‰ + +è¦æ·»åŠ  VM 的现有[å¯ç”¨æ€§é›†](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview)çš„å称或 ID。 + +### é•œåƒ + +作为 ARM 资æºæ ‡è¯†ç¬¦æ供的æ“作系统镜åƒçš„å称。需è¦ä½¿ç”¨æ‰˜ç®¡ç£ç›˜ã€‚ + +### è™šæ‹Ÿæœºå¤§å° + +为节点池中的æ¯ä¸ª VM 选择一个大å°ã€‚有关æ¯ä¸ª VM 大å°çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/)。 + +## 高级选项 + +### å®¹é”™åŸŸæ•°é‡ + +容错域定义了共享公共电æºå’Œç½‘络交æ¢æœºçš„虚拟机组。如果å¯ç”¨æ€§é›†å·²åˆ›å»ºï¼Œå°†å¿½ç•¥å®¹é”™åŸŸæ•°é‡ã€‚ + +有关容错域的更多信æ¯ï¼Œè¯·å‚阅[此处](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work)。 + +### æ›´æ–°åŸŸæ•°é‡ + +更新域表示å¯ä»¥åŒæ—¶é‡æ–°å¯åŠ¨çš„虚拟机组和底层物ç†ç¡¬ä»¶ã€‚如果å¯ç”¨æ€§é›†å·²åˆ›å»ºï¼Œå°†å¿½ç•¥æ›´æ–°åŸŸæ•°é‡ã€‚ + +有关更新域的更多信æ¯ï¼Œè¯·å‚阅[此处](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work)。 + +### 购买计划 + +Azure 市场中的æŸäº› VM é•œåƒéœ€è¦è´­ä¹°è®¡åˆ’。如果适用，请为你选择的镜åƒé€‰æ‹©æ ¼å¼ä¸º `publisher:product:plan` 的购买计划。 + +### å­ç½‘ + +创建新 VNet 或引用现有 VNet 时的å­ç½‘å称。 + +默认值：`docker-machine` + +### å­ç½‘å‰ç¼€ + +创建 CIDR æ ¼å¼çš„æ–° VNet æ—¶è¦ä½¿ç”¨çš„å­ç½‘ IP 地å€å‰ç¼€ã€‚ + +默认值：`192.168.0.0/16` + +### 虚拟网络 + +è¦ä½¿ç”¨æˆ–创建的[虚拟网络](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)（如果ä¸å­˜åœ¨ï¼‰ã€‚æ ¼å¼ä¸º `[resourcegroup:]name`。 + +### 公共 IP 选项 + +#### 没有公共 IP + +ä¸è¦åˆ†é…公共 IP 地å€ã€‚ + +#### é™æ€å…¬å…± IP + +分é…é™æ€å…¬å…± IP 地å€ã€‚ + +### 使用ç§æœ‰ IP + +使用é™æ€ç§æœ‰ IP 地å€ã€‚ + +### ç§æœ‰ IP åœ°å€ + +é…ç½®è¦ä½¿ç”¨çš„é™æ€ç§æœ‰ IP 地å€ã€‚ + +### 网络安全组 + +è¦ä½¿ç”¨çš„[网络安全组（NSG）](https://docs.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview)。使用此模æ¿çš„所有节点都将使用é…置的网络安全组。如果没有é…置网络安全组，则为æ¯ä¸ªèŠ‚点创建一个新的网络安全组。 + +### DNS 标签 + +公共 IP 地å€çš„唯一 DNS å称标签。 + +### 储存类型 + +用于 VM çš„[存储账å·](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview)类型。å¯é€‰é¡¹åŒ…括 Standard LRSã€Standard ZRSã€Standard GRSã€Standard RAGRS å’Œ Premium LRS。 + +### 使用托管ç£ç›˜ + +[Azure 托管ç£ç›˜](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview)是由 Azure 管ç†å¹¶ä¸Ž Azure Virtual Machines 一起使用的å—级存储å·ã€‚托管ç£ç›˜æ—¨åœ¨å®žçŽ° 99.999% 的高å¯ç”¨æ€§ã€‚托管ç£ç›˜é€šè¿‡æ供三个数æ®å‰¯æœ¬æ¥å®žçŽ°é«˜å¯ç”¨æ€§å’Œé«˜æŒç»­æ€§ã€‚ + +### 托管ç£ç›˜å¤§å° + +æ¯ä¸ªèŠ‚点的ç£ç›˜å¤§å°ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚ + +### SSH 用户å + +用于 SSH 连接到节点的用户å。 + +### å¼€æ”¾ç«¯å£ + +打开指定端å£ä¸Šçš„入站æµé‡ã€‚如果你使用现有的网络安全组，将忽略开放端å£ã€‚ + +默认值：`2379/tcp, 2380/tcp, 6443/tcp, 9796/tcp, 10250/tcp, 10251/tcp, 10252/tcp, 10256/tcp` å’Œ `8472/udp, 4789/udp` \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md new file mode 100644 index 00000000000..6fdddebab84 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -0,0 +1,33 @@ +--- +title: DigitalOcean 主机é…ç½® +--- + +有关 DigitalOceanã€Droplet 的更多详细信æ¯ï¼Œè¯·å‚阅[官方文档](https://docs.digitalocean.com/products/compute/)。 + +### 区域 + +é…置创建 Droplet çš„[区域](https://docs.digitalocean.com/glossary/region/)。 + +### å¤§å° + +é…ç½® Droplet çš„[大å°](https://docs.digitalocean.com/products/droplets/resources/choose-plan/)。 + +### æ“ä½œç³»ç»Ÿé•œåƒ + +é…置用于创建 Droplet çš„æ“作系统[é•œåƒ](https://docs.digitalocean.com/products/images/)。 + +### Monitoring + +å¯ç”¨ DigitalOcean 代ç†ä»¥è¿›è¡Œé¢å¤–çš„[监控](https://docs.digitalocean.com/products/monitoring/)。 + +### IPv6 + +为 Droplet å¯ç”¨ IPv6。 + +### ç§æœ‰ç½‘络 + +为 Droplet å¯ç”¨ç§æœ‰ç½‘络。 + +### Droplet 标签 + +将标签（tag, label）应用于 Droplet。标签åªèƒ½åŒ…å«å­—æ¯ã€æ•°å­—ã€å†’å·ã€ç ´æŠ˜å·å’Œä¸‹åˆ’线。例如，`my_server`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md new file mode 100644 index 00000000000..56082ed497a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -0,0 +1,47 @@ +--- +title: EC2 节点模æ¿é…ç½® +--- + +有关 EC2 和节点的更多详细信æ¯ï¼Œè¯·å‚阅 [EC2 管ç†æŽ§åˆ¶å°](https://aws.amazon.com/ec2)的官方文档。 + +### 区域 + +在**区域**字段中，选择创建云凭è¯æ—¶ä½¿ç”¨çš„åŒä¸€åŒºåŸŸã€‚ + +### äº‘å‡­è¯ + +ä½ çš„ AWS 账户访问信æ¯ï¼Œå­˜å‚¨åœ¨[云凭è¯](../../../user-settings/manage-cloud-credentials.md)中。 + +请å‚阅 [Amazon 文档：创建访问密钥](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)æ¥åˆ›å»ºè®¿é—®å¯†é’¥å’Œå¯†æ–‡å¯†é’¥ã€‚ + +请å‚阅 [Amazon 文档：创建 IAM 策略（控制å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start)æ¥åˆ›å»º IAM 策略。 + +请å‚阅 [Amazon 文档：为用户添加æƒé™ï¼ˆæŽ§åˆ¶å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console)了解如何绑定 IAM。 + +å‚阅下é¢çš„三个示例 JSON 策略： + +- [IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#iam-策略示例) +- [带有 PassRole çš„ IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#带有-passrole-çš„-iam-策略示例)（如果è¦ä½¿ç”¨ [Kubernetes 云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)，或将 IAM é…置文件传递给实例，则需è¦ï¼‰ +- [å…许用户加密 EBS å·çš„ IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#å…许加密-ebs-å·çš„-iam-策略示例) + +### 验è¯å’Œé…置节点 + +为集群选择å¯ç”¨åŒºå’Œç½‘络设置。 + +### 安全组 + +选择默认安全组或é…置安全组。 + +请å‚考[使用主机驱动时的 Amazon EC2 安全组](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-安全组)，了解 `rancher-nodes` 安全组中创建的规则。 + +如果你自行为 EC2 实例æ供安全组，Rancher ä¸ä¼šå¯¹å…¶è¿›è¡Œä¿®æ”¹ã€‚因此，你需è¦è®©ä½ çš„安全组å…许 [Rancher é…置实例所需的端å£](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rke-上-rancher-server-节点的端å£)。有关使用安全组控制 EC2 实例的入站和出站æµé‡çš„更多信æ¯ï¼Œè¯·å‚阅[这里](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#WorkingWithSecurityGroups)。 + +### 实例选项 + +é…ç½®è¦åˆ›å»ºçš„实例。确ä¿ä¸º AMI é…置正确的 **SSH 用户**。所选的区域å¯èƒ½ä¸æ”¯æŒé»˜è®¤å®žä¾‹ç±»åž‹ã€‚在这ç§æƒ…况下，你必须选择一个确实存在的实例类型。å¦åˆ™å°†å‡ºçŽ°é”™è¯¯ï¼Œè¡¨ç¤ºè¯·æ±‚çš„é…ç½®ä¸å—支æŒã€‚ + +如果需è¦ä¼ é€’ **IAM 示例é…ç½®å称**（ä¸æ˜¯ ARN），例如è¦ä½¿ç”¨ [Kubernetes 云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)时，策略则需è¦å…¶ä»–æƒé™ã€‚有关示例策略，请å‚阅[带有 PassRole çš„ IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#带有-passrole-çš„-iam-策略示例)。 + +### 引擎选项 + +在节点模æ¿çš„**引擎选项**中，你å¯ä»¥é…置容器 daemon。你å¯èƒ½éœ€è¦æŒ‡å®šå®¹å™¨ç‰ˆæœ¬æˆ–容器镜åƒä»“库 Mirror。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md new file mode 100644 index 00000000000..25531ce9a1c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -0,0 +1,33 @@ +--- +title: Azure 节点模æ¿é…ç½® +--- + +有关 Azure 的更多信æ¯ï¼Œè¯·å‚阅官方 [Azure 文档](https://docs.microsoft.com/en-us/azure/?product=featured)。 + +账户访问信æ¯å­˜å‚¨åœ¨äº‘凭è¯ä¸­ã€‚云凭è¯å­˜å‚¨åœ¨ Kubernetes 密文中。多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯ã€‚ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰çš„云凭è¯æˆ–创建新的凭è¯ã€‚ + +- **Placement** 设置托管集群的地ç†åŒºåŸŸä»¥åŠå…¶ä»–ä½ç½®å…ƒæ•°æ®ã€‚ +- **Network** é…置集群中使用的网络。 +- **Instance** 自定义你的 VM é…置。 + + :::note + + 如果在与 VM ä¸åŒçš„资æºç»„中使用 VNet，则 VNet å称需è¦ä»¥èµ„æºç»„å称作为å‰ç¼€ã€‚例如，`:`。 + + ::: + +如果你使用 Docker，[Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) é…置选项包括： + +- **标签**：有关标签的信æ¯ï¼Œè¯·å‚阅 [Docker 对象标签文档](https://docs.docker.com/config/labels-custom-metadata/)。 +- **Docker 引擎安装 URL**：确定è¦åœ¨å®žä¾‹ä¸Šå®‰è£…çš„ Docker 版本。 + + :::note + + 如果è¦é…ç½® Red Hat Enterprise Linux (RHEL) 或 CentOS 节点，请将 **Docker Install URL** 字段ä¿ç•™ä¸ºé»˜è®¤å€¼ï¼Œæˆ–选择 **none**。由于 Docker å·²ç»å®‰è£…在这些节点上，因此将绕过 Docker 安装检查。 + + 如果没有将 **Docker Install URL** 设置为默认值或 **none**，你å¯èƒ½ä¼šçœ‹åˆ°é”™è¯¯æ¶ˆæ¯ï¼š`Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories`。 + + ::: + +- **é•œåƒä»“库 mirror**：Docker daemon 使用的 Docker é•œåƒä»“库镜åƒã€‚ +- **其他高级选项**：å‚è§ [Docker daemon 选项å‚考](https://docs.docker.com/engine/reference/commandline/dockerd/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md new file mode 100644 index 00000000000..4c00d381982 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -0,0 +1,18 @@ +--- +title: DigitalOcean 节点模æ¿é…ç½® +--- + +账户访问信æ¯å­˜å‚¨åœ¨äº‘凭è¯ä¸­ã€‚云凭è¯å­˜å‚¨åœ¨ Kubernetes 密文中。多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯ã€‚ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰çš„云凭è¯æˆ–创建新的凭è¯ã€‚ + +### Droplet 选项 + +**Droplet 选项**用于é…置集群的地ç†åŒºåŸŸå’Œè§„范。 + +### Docker Daemon + +如果你使用 Docker，[Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) é…置选项包括： + +- **标签**：有关标签的信æ¯ï¼Œè¯·å‚阅 [Docker 对象标签文档](https://docs.docker.com/config/labels-custom-metadata/)。 +- **Docker 引擎安装 URL**：确定è¦åœ¨å®žä¾‹ä¸Šå®‰è£…çš„ Docker 版本。 +- **é•œåƒä»“库 mirror**：Docker daemon 使用的 Docker é•œåƒä»“库镜åƒã€‚ +- **其他高级选项**：å‚è§ [Docker daemon 选项å‚考](https://docs.docker.com/engine/reference/commandline/dockerd/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md new file mode 100644 index 00000000000..21f03714a21 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -0,0 +1,66 @@ +--- +title: Nutanix 节点模æ¿é…ç½® +--- + +## è´¦å·è®¿é—® + +| å‚æ•° | å¿…å¡« | æè¿° | 默认 | +|:-----------------------------|:--------:|:-----------------------------------------------------------------|:----- +| 管ç†ç«¯ç‚¹ | ✓ | Prism Central 的主机å/IP åœ°å€ | +| 用户å | ✓ | Prism Central 用户的用户å | +| å¯†ç  | ✓ | Prism Central ç”¨æˆ·çš„å¯†ç  | +| å…许ä¸å®‰å…¨çš„通信 | | 设置为 true 以å…许与 Prism Central 进行ä¸å®‰å…¨çš„ SSL 通信 | False | + +## 调度 + +选择虚拟机è¦è°ƒåº¦åˆ°å“ªä¸ª Nutanix 集群。 + +| å‚æ•° | å¿…å¡« | æè¿° | +|:----------|:--------:|:---------------------------------------------------------------------------- +| 集群 | ✓ | 部署虚拟机的 Nutanix 集群的å称（区分大å°å†™ï¼‰ | + +## 实例选项 + +在**实例å‚æ•°**中，é…置此模æ¿åˆ›å»ºçš„ VM çš„ vCPU æ•°é‡ã€å†…存和ç£ç›˜å¤§å°ã€‚ + +| å‚æ•° | å¿…å¡« | æè¿° | 默认 | +|:---------------------|:--------:|:--------------------------------------------------------------------------------------------|:------- +| CPU | | 分é…ç»™ VM çš„ vCPU æ•°é‡ï¼ˆæ ¸å¿ƒï¼‰ | 2 | +| 内存 | | 分é…ç»™ VM çš„ RAM é‡ (MB) | 2 GB | +| Template Image | ✓ | è¦ä½œä¸ºè™šæ‹Ÿæœºä¸»ç£ç›˜è¿›è¡Œå…‹éš†çš„ç£ç›˜é•œåƒæ¨¡æ¿çš„åç§°ï¼ˆå¿…é¡»æ”¯æŒ cloud-init） | +| 虚拟机ç£ç›˜å¤§å° | | 虚拟机主ç£ç›˜çš„新大å°ï¼ˆä»¥ GiB 为å•ä½ï¼‰ | +| 其他ç£ç›˜å¤§å° | | è¦æ·»åŠ åˆ°è™šæ‹Ÿæœºçš„其他ç£ç›˜çš„大å°ï¼ˆä»¥ GiB 为å•ä½ï¼‰ | +| 储存容器 | | è¦é…置其他ç£ç›˜çš„存储容器 _UUID_ | +| Cloud Config YAML | | è¦æ供给虚拟机的 cloud-init（将使用 Rancher root 用户修补） | +| 网络 | ✓ | è¦é™„加到虚拟机的网络的å称 | +| 虚拟机类别 | | è¦åº”用于虚拟机的类别å称 | + +虚拟机支æŒé€šè¿‡ [Config Drive v2 datasource](https://cloudinit.readthedocs.io/en/latest/reference/datasources/configdrive.html) æ¥æ”¯æŒ [cloud-init](https://cloudinit.readthedocs.io/en/latest/) 的任何现代 Linux æ“作系统。 + +## 网络 + +节点模æ¿å…许你为虚拟机é…置多个网络。在**网络**字段中，你å¯ä»¥å•å‡»**添加**，然åŽåœ¨ AOS 中添加任何å¯ç”¨çš„网络。 + +## 虚拟机类别 + +类别用于将实体分组æˆé”®å€¼å¯¹ã€‚通常会根æ®æŸäº›æ ‡å‡†ä¸ºè™šæ‹Ÿæœºåˆ†é…一个类别。然åŽï¼Œä½ å¯ä»¥å°†ç­–略绑定到分é…（分组）了特定类别值的实体。 + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) å…许你在首次å¯åŠ¨æ—¶åº”用é…置，从而åˆå§‹åŒ–节点。这å¯èƒ½æ¶‰åŠåˆ›å»ºç”¨æˆ·æˆ–æŽˆæƒ SSH 密钥之类的æ“作。 + +è¦ä½¿ç”¨ cloud-init åˆå§‹åŒ–，请将使用有效 YAML 语法的 cloud config 粘贴到 **Cloud Config YAML** 字段中。è¦èŽ·å–支æŒçš„ cloud config 指令的注释示例集，请å‚阅 [cloud-init 文档](https://cloudinit.readthedocs.io/en/latest/topics/examples.html)。 + +ä¸å»ºè®®ä½¿ç”¨åŸºäºŽ cloud-init 的网络é…置，仅支æŒä½¿ç”¨ç”¨æˆ·æ•°æ® `runcmd`，ä¸æ”¯æŒ NoCloud 或其他网络é…置数æ®æºã€‚ + +建议使用 Nutanix IP Address Management（IPAM) 或其他 DHCP æœåŠ¡ã€‚ + +## 引擎选项 + +在节点模æ¿çš„**引擎选项**中，你å¯ä»¥é…置容器 daemon。你å¯èƒ½éœ€è¦æŒ‡å®šå®¹å™¨ç‰ˆæœ¬æˆ–容器镜åƒä»“库 Mirror。 + +:::note +如果è¦é…ç½® Red Hat Enterprise Linux (RHEL) 或 CentOS 节点，请将 **Docker Install URL** 字段ä¿ç•™ä¸ºé»˜è®¤å€¼ï¼Œæˆ–选择 **none**。由于 Docker å·²ç»å®‰è£…在这些节点上，因此将绕过 Docker 安装检查。 + +如果没有将 **Docker Install URL** 设置为默认值或 **none**，你å¯èƒ½ä¼šçœ‹åˆ°é”™è¯¯æ¶ˆæ¯ï¼š`Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories`。 +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md new file mode 100644 index 00000000000..a8956d548d5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -0,0 +1,95 @@ +--- +title: vSphere 节点模æ¿é…ç½® +--- + +## è´¦å·è®¿é—® + +| å‚æ•° | å¿…å¡« | æè¿° | +|:----------------------|:--------:|:-----| +| äº‘å‡­è¯ | * | ä½ çš„ vSphere è´¦å·è®¿é—®ä¿¡æ¯ï¼Œå­˜å‚¨åœ¨[云凭è¯](../../../user-settings/manage-cloud-credentials.md)中。 | + +你的云凭è¯å…·æœ‰ä»¥ä¸‹å­—段： + +| 凭è¯å­—段 | æè¿° | +|-----------------|--------------| +| vCenter 或 ESXi Server | 输入 vCenter 或 ESXi 主机å/IP。ESXi 是你创建和è¿è¡Œè™šæ‹Ÿæœºå’Œè™šæ‹Ÿè®¾å¤‡çš„虚拟化平å°ã€‚ä½ å¯ä»¥é€šè¿‡ vCenter Server æœåŠ¡æ¥ç®¡ç†ç½‘络中连接的多个主机并池化主机资æºã€‚ | +| ç«¯å£ | å¯é€‰ï¼šé…ç½® vCenter 或 ESXi Server 的端å£ã€‚ | +| 用户åå’Œå¯†ç  | ä½ çš„ vSphere 登录用户å和密ç ã€‚ | + +## 调度 + +选择虚拟机è¦è°ƒåº¦åˆ°å“ªä¸ªè™šæ‹Ÿæœºç›‘控程åºã€‚ + +**调度**中的字段应使用 vSphere 中å¯ç”¨çš„æ•°æ®ä¸­å¿ƒå’Œå…¶ä»–计划选项自动填充。 + +| 字段 | å¿…å¡« | 解释 | +|---------|---------------|-----------| +| æ•°æ®ä¸­å¿ƒ | * | 选择è¦è°ƒåº¦ VM çš„æ•°æ®ä¸­å¿ƒçš„å称/路径。 | +| 资æºæ±  | | è¦åœ¨å…¶ä¸­è°ƒåº¦ VM 的资æºæ± å称。资æºæ± å¯å¯¹ç‹¬ç«‹ä¸»æœºæˆ–集群的å¯ç”¨ CPU 和内存资æºè¿›è¡Œåˆ†åŒºï¼Œä¹Ÿå¯ä»¥åµŒå¥—使用。如果是独立 ESXi，请留空。如果未指定，则使用默认资æºæ± ã€‚ | +| æ•°æ®å­˜å‚¨ | * | 如果你有数æ®å­˜å‚¨é›†ç¾¤ï¼Œåˆ™å¯ä»¥æ‰“å¼€**æ•°æ®å­˜å‚¨**字段。此字段å…许你选择è¦å°† VM 调度到哪个数æ®å­˜å‚¨é›†ç¾¤ã€‚如果该字段未打开，你å¯ä»¥é€‰æ‹©å•ä¸ªç£ç›˜ã€‚ | +| 文件夹 | | æ•°æ®ä¸­å¿ƒä¸­ç”¨äºŽåˆ›å»º VM 的文件夹的å称。必须已ç»å­˜åœ¨ã€‚此下拉èœå•ä¸­çš„ VM 文件夹直接对应于 vSphere 中的 VM 文件夹。在 vSphere é…置文件中，文件夹å称应以 `vm/` 开头。 | +| 主机 | | 用于调度 VM 的主机系统的 IP。如果是独立 ESXi 或具有 DRS（分布å¼èµ„æºè°ƒåº¦å™¨ï¼‰çš„集群，将此字段留空。如果指定，将使用主机系统的池，而忽略**资æºæ± **å‚数。 | + +## 实例选项 + +在**实例å‚æ•°**中，é…置此模æ¿åˆ›å»ºçš„ VM çš„ vCPU æ•°é‡ã€å†…存和ç£ç›˜å¤§å°ã€‚ + +| å‚æ•° | å¿…å¡« | æè¿° | +|:----------------|:--------:|:-----------| +| CPU | * | è¦åˆ†é…ç»™ VM çš„ vCPU æ•°é‡ã€‚ | +| 内存 | * | è¦åˆ†é…ç»™ VM 的内存é‡ã€‚ | +| ç£ç›˜ | * | è¦æŒ‚载到 VM çš„ç£ç›˜å¤§å°ï¼ˆä»¥ MB 为å•ä½ï¼‰ã€‚ | +| 创建方法 | * | 在节点上设置æ“作系统的方法。å¯ä»¥ä½¿ç”¨ ISO 或 VM 模æ¿å®‰è£…æ“作系统。根æ®åˆ›å»ºæ–¹æ³•ï¼Œä½ è¿˜å¿…须指定 VM 模æ¿ã€å†…容库ã€çŽ°æœ‰ VM 或 ISO。有关创建方法的详细信æ¯ï¼Œè¯·å‚阅 [VM 创建方法](#vm-创建方法)。 | +| Cloud Init | | `cloud-config.yml` 文件的 URL 或用于é…ç½® VM çš„ URL。此文件å…许进一步定制æ“作系统，例如网络é…ç½®ã€DNS æœåŠ¡å™¨æˆ–系统守护程åºã€‚æ“ä½œç³»ç»Ÿå¿…é¡»æ”¯æŒ `cloud-init`。 | +| 网络 | | è¦æŒ‚è½½ VM 的网络的å称。 | +| guestinfo é…ç½®å‚æ•° | | VM 的其他é…ç½®å‚数。这些å‚数对应 vSphere 控制å°ä¸­çš„[高级设置](https://kb.vmware.com/s/article/1016098)。示例用例包括æä¾› RancherOS [guestinfo](https://rancher.com/docs/os/v1.x/en/installation/cloud/vmware-esxi/#vmware-guestinfo) å‚数，或为 VM å¯ç”¨ç£ç›˜ UUID (`disk.EnableUUID=TRUE`)。 | + + +### VM 创建方法 + +在**创建方法**字段中，é…置用于在 vSphere 中é…ç½® VM 的方法。å¯ç”¨çš„选项包括创建从 RancherOS ISO å¯åŠ¨çš„ VM，或通过从现有虚拟机或 [VM 模æ¿](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-F7BF0E6B-7C4F-4E46-8BBF-76229AEA7220.html)克隆æ¥åˆ›å»º VM。 + +现有 VM 或模æ¿å¯ä»¥ä½¿ç”¨ä»»ä½•çŽ°ä»£ Linux æ“作系统，该æ“作系统é…置为使用 [NoCloud æ•°æ®æº](https://canonical-cloud-init.readthedocs-hosted.com/en/latest/reference/datasources/nocloud.html)æ¥æ”¯æŒ [cloud-init](https://cloudinit.readthedocs.io/en/latest/)。 + +选择创建 VM çš„æ–¹å¼ï¼š + +- **使用模æ¿éƒ¨ç½²ï¼šæ•°æ®ä¸­å¿ƒ**：选择存在于所选数æ®ä¸­å¿ƒçš„ VM 模æ¿ã€‚ +- **使用模æ¿éƒ¨ç½²ï¼šå†…容库**：首先选择包å«ä½ çš„模æ¿çš„[内容库](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-254B2CE8-20A8-43F0-90E8-3F6776C2C896.html)，然åŽä»Žå¡«å……列表**库模æ¿**中选择模æ¿ã€‚ +- **克隆现有的虚拟机**：在**虚拟机**字段中选择一个现有虚拟机，新虚拟机将克隆自该虚拟机。 +- **使用 boot2docker ISO 安装**ï¼šç¡®ä¿ **OS ISO URL** å­—æ®µåŒ…å« RancherOS çš„ VMware ISO 版本的 URL (`rancheros-vmware.iso`)。请注æ„，è¿è¡Œ Rancher Server 节点必须能访问该 URL。 + +## 网络 + +节点模æ¿å…许你为虚拟机é…置多个网络。在**网络**字段中，你å¯ä»¥å•å‡»**添加网络**æ¥æ·»åŠ  vSphere 中å¯ç”¨çš„任何网络。 + +## 节点标签和自定义属性 + +æ ‡ç­¾ç”¨äºŽå‘ vSphere 对象清å•ä¸­çš„对象添加元数æ®ï¼Œä»¥ä¾¿å¯¹å¯¹è±¡è¿›è¡ŒæŽ’åºå’Œæœç´¢ã€‚ + +你的所有 vSphere 标签都将显示为节点模æ¿ä¸­å¯ä¾›é€‰æ‹©çš„选项。 + +在自定义属性中，Rancher 会让你选择你已ç»åœ¨ vSphere 中设置的所有自定义属性。自定义属性是键，你å¯ä»¥ä¸ºæ¯ä¸ªå±žæ€§è¾“入值。 + +:::note + +自定义属性是一项旧版功能，最终将从 vSphere 中删除。 + +::: + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) å…许你在首次å¯åŠ¨æ—¶åº”用é…置，从而åˆå§‹åŒ–节点。这å¯èƒ½æ¶‰åŠåˆ›å»ºç”¨æˆ·ã€æŽˆæƒ SSH 密钥或设置网络之类的æ“作。 + +è¦ä½¿ç”¨ cloud-init åˆå§‹åŒ–，请使用有效的 YAML 语法创建一个 cloud config 文件，并将文件内容粘贴到 **Cloud Init** 字段中。è¦èŽ·å–支æŒçš„ cloud config 指令的注释示例集，请å‚阅 [cloud-init 文档](https://cloudinit.readthedocs.io/en/latest/topics/examples.html)。 + +请注æ„，使用 ISO 创建方法时ä¸æ”¯æŒ cloud-init。 + +## 引擎选项 + +在节点模æ¿çš„**引擎选项**中，你å¯ä»¥é…置容器 daemon。你å¯èƒ½éœ€è¦æŒ‡å®šå®¹å™¨ç‰ˆæœ¬æˆ–容器镜åƒä»“库 Mirror。 + +:::note +如果è¦é…ç½® Red Hat Enterprise Linux (RHEL) 或 CentOS 节点，请将 **Docker Install URL** 字段ä¿ç•™ä¸ºé»˜è®¤å€¼ï¼Œæˆ–选择 **none**。由于 Docker å·²ç»å®‰è£…在这些节点上，因此将绕过 Docker 安装检查。 + +如果没有将 **Docker Install URL** 设置为默认值或 **none**，你å¯èƒ½ä¼šçœ‹åˆ°é”™è¯¯æ¶ˆæ¯ï¼š`Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories`。 +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md new file mode 100644 index 00000000000..a296187be6b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -0,0 +1,224 @@ +--- +title: AKS 集群é…ç½®å‚考 +--- + +## Rancher 2.6 å˜æ›´ + +- 支æŒæ·»åŠ å¤šä¸ªèŠ‚点池 +- 支æŒç§æœ‰é›†ç¾¤ +- å¯ç”¨è‡ªåŠ¨ç¼©æ”¾èŠ‚点池 +- 在云凭è¯ä¸­é…ç½® AKS æƒé™ + +## RBAC + +在 Rancher UI 中é…ç½® AKS 集群时，无法ç¦ç”¨ RBAC。如果在 AKS 中为集群ç¦ç”¨äº† RBAC，则无法在 Rancher 中注册或导入集群。 + +Rancher å¯ä»¥ä½¿ç”¨ä¸Žå…¶ä»–集群一样的方å¼ä¸º AKS 集群é…ç½®æˆå‘˜è§’色。有关详细信æ¯ï¼Œè¯·å‚阅 [RBAC](../../../pages-for-subheaders/manage-role-based-access-control-rbac.md)。 + +## äº‘å‡­è¯ + +:::note + +本节中的é…置信æ¯å‡è®¾ä½ å·²ç»ä¸º Rancher 设置了æœåŠ¡ä¸»ä½“。有关如何设置æœåŠ¡ä¸»ä½“的分步说明，请å‚阅[本节](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md#microsoft-azure-中的先决æ¡ä»¶)。 + +::: + +### 订阅 ID + +è¦èŽ·å–订阅 ID，请å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸­çš„ **All Services**。然åŽå•å‡» **Subscriptions**。转到è¦ä¸Ž Kubernetes 集群关è”的订阅的å称，然åŽå¤åˆ¶ **Subscription ID**。 + +### 客户端 ID + +è¦èŽ·å–客户端 ID，请转到 Azure 门户，然åŽå•å‡» **Azure Active Directory**，å•å‡» **App registrations**，然åŽå•å‡»æœåŠ¡ä¸»ä½“çš„å称。客户端 ID 在 app registration 详细信æ¯é¡µé¢ä¸Šåˆ—为 **Application (client) ID**。 + +### å®¢æˆ·ç«¯å¯†ç  + +在创建客户端密ç å€¼åŽï¼Œä½ æ— æ³•å†èŽ·å–它的值。因此如果你还没有客户端密ç å€¼ï¼Œåˆ™éœ€è¦åˆ›å»ºä¸€ä¸ªæ–°çš„客户端密ç ã€‚ + +è¦èŽ·å–新的客户端密ç ï¼Œè¯·è½¬åˆ° Azure 门户，然åŽå•å‡» **Azure Active Directory**，å•å‡» **App registrations**，然åŽå•å‡»æœåŠ¡ä¸»ä½“çš„å称。 + +然åŽç‚¹å‡» **Certificates & secrets** 并点击 **New client secret**。å•å‡» **Add**。然åŽå¤åˆ¶æ–°å®¢æˆ·ç«¯å¯†ç çš„ **Value**。 + +### 环境 + +Microsoft æ供了多个[云](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest)æ¥æ»¡è¶³åœ°åŒºæ³•å¾‹çš„è¦æ±‚： + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +## è´¦å·è®¿é—® + +在本部分中，你需è¦é€‰æ‹©çŽ°æœ‰çš„ Azure 云凭è¯æˆ–创建一个新凭è¯ã€‚ + +有关é…ç½® Azure 云凭è¯çš„帮助，请å‚阅[本部分](#云凭è¯)。 + +## 集群ä½ç½® + +é…置集群和节点ä½ç½®ã€‚有关 AKS å¯ç”¨åŒºçš„详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/availability-zones)。 + +高å¯ç”¨æ€§ä½ç½®åŒ…括多个å¯ç”¨åŒºã€‚ + +## 集群选项 + +### Kubernetes 版本 + +å¯ç”¨çš„ Kubernetes 版本是从 Azure API 动æ€èŽ·å–的。 + +### 集群资æºç»„ + +资æºç»„æ˜¯ä¸€ä¸ªå®¹å™¨ï¼Œå…¶ä¸­åŒ…å« Azure 解决方案的相关资æºã€‚资æºç»„å¯ä»¥åŒ…括解决方案的所有资æºï¼Œæˆ–者仅包括你希望作为一个组æ¥ç®¡ç†çš„资æºã€‚ä½ å¯ä»¥æ ¹æ®ç»„织的需求æ¥å†³å®šå¦‚何将资æºåˆ†é…给资æºç»„。通常情况下，你å¯ä»¥å°†ç”Ÿå‘½å‘¨æœŸç›¸åŒçš„资æºæ·»åŠ åˆ°åŒä¸€ä¸ªèµ„æºç»„，以便轻æ¾åœ°æŒ‰ç»„进行部署ã€æ›´æ–°å’Œåˆ é™¤ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰èµ„æºç»„，也å¯ä»¥è¾“入资æºç»„çš„å称，然åŽç³»ç»Ÿå°†ä¸ºä½ åˆ›å»ºä¸€ä¸ªèµ„æºç»„。 + +使用包å«çŽ°æœ‰ AKS 集群的资æºç»„将会创建一个新资æºç»„。Azure AKS ä»…å…许æ¯ä¸ªèµ„æºç»„对应一个 AKS 集群。 + +有关管ç†èµ„æºç»„çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal)。 + +### Linux 管ç†å‘˜ç”¨æˆ·å + +用于创建到 Linux 节点的 SSH 连接的用户å。 + +AKS 节点的默认用户å是 `azureuser`。 + +### SSH 公钥 + +用于创建到 Linux 节点的 SSH 连接的密钥。 + +### Tags + +如果你的组织使用标签（Tag）æ¥ç®¡ç†å¤šä¸ª Azure æœåŠ¡çš„资æºï¼Œé‚£ä¹ˆé›†ç¾¤æ ‡ç­¾åˆ™éžå¸¸æœ‰ç”¨ã€‚这些标签ä¸é€‚用于集群内的资æºã€‚ + +## 网络选项 + +### è´Ÿè½½å‡è¡¡å™¨ SKU + +Azure è´Ÿè½½å‡è¡¡å™¨æ”¯æŒ standard å’Œ basic SKU（stock keeping units）。 + +有关 standard è´Ÿè½½å‡è¡¡å™¨å’Œ basic è´Ÿè½½å‡è¡¡å™¨çš„对比，请å‚阅官方 [Azure 文档](https://docs.microsoft.com/en-us/azure/load-balancer/skus#skus)。Microsoft 建议使用 standard è´Ÿè½½å‡è¡¡å™¨ã€‚ + +如果你选择了一个或多个å¯ç”¨åŒºï¼Œæˆ–è€…ä½ æœ‰å¤šä¸ªèŠ‚ç‚¹æ± ï¼Œåˆ™éœ€è¦ Standard è´Ÿè½½å‡è¡¡å™¨ã€‚ + +### 网络策略 + +默认情况下，AKS 集群中的所有 Pod 都å¯ä»¥æ— é™åˆ¶åœ°å‘é€å’ŒæŽ¥æ”¶æµé‡ã€‚为了æ高安全性，你å¯ä»¥å®šä¹‰æŽ§åˆ¶æµé‡çš„规则。Kubernetes 中的网络策略功能å…许你定义集群中 pod 之间的入å£å’Œå‡ºå£æµé‡è§„则。 + +Azure æ供了两ç§å®žçŽ°ç½‘络策略的方法。创建 AKS 集群时会选择网络策略选项。创建集群åŽæ— æ³•æ›´æ”¹ç­–略选项： + +- Azure 自己的实现，称为 Azure 网络策略。Azure ç½‘ç»œç­–ç•¥éœ€è¦ Azure CNI。 +- Calico Network Policies，一个由 [Tigera](https://www.tigera.io/) 创立的开æºç½‘络和网络安全解决方案。 + +你也å¯ä»¥é€‰æ‹©ä¸ä½¿ç”¨ç½‘络策略。 + +有关 Azure å’Œ Calico 网络策略åŠå…¶åŠŸèƒ½ä¹‹é—´çš„差异，请å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/use-network-policies#differences-between-azure-and-calico-policies-and-their-capabilities)。 + +### DNS å‰ç¼€ +为集群的 Kubernetes API server FQDN 输入唯一的 DNS å‰ç¼€ã€‚ + +### 网络æ’件 +有两个网络æ’件，分别是 kubenet å’Œ Azure CNI。 + +[kubenet](https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet) Kubernetes æ’件是 AKS 创建的集群的默认é…置。使用 kubenet 时，集群中的æ¯ä¸ªèŠ‚点都会收到一个å¯è·¯ç”±çš„ IP 地å€ã€‚Pod 使用 NAT 与 AKS 集群外部的其他资æºè¿›è¡Œé€šä¿¡ã€‚è¿™ç§æ–¹æ³•å‡å°‘了需è¦åœ¨ç½‘络空间中ä¿ç•™ä»¥ä¾› Pod 使用的 IP 地å€æ•°é‡ã€‚ + +如果使用 Azure CNI（高级）网络æ’件，Pod å¯ä»¥ä½¿ç”¨å®Œæ•´çš„虚拟网络连接，并且å¯ä»¥ä»Žè¿žæŽ¥çš„网络中通过 pod çš„ç§æœ‰ IP 地å€ç›´æŽ¥è®¿é—®ã€‚这个æ’件需è¦æ›´å¤šçš„ IP 地å€ç©ºé—´ã€‚ + +有关 kubenet å’Œ Azure CNI 之间差异的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/concepts-network#compare-network-models)。 + +### HTTP 应用路由 + +å¯ç”¨åŽï¼ŒHTTP 应用路由附加组件å¯ä»¥æ›´è½»æ¾åœ°è®¿é—®éƒ¨ç½²åˆ° AKS 集群的应用。它部署了两个组件，分别是 [Kubernetes Ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress/) å’Œ [External-DNS](https://github.com/kubernetes-incubator/external-dns) controller。 + +有关详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/http-application-routing)。 + +### è®¾ç½®æŽˆæƒ IP 范围 + +ä½ å¯ä»¥ä½¿ç”¨[授æƒçš„ IP 地å€èŒƒå›´](https://docs.microsoft.com/en-us/azure/aks/api-server-authorized-ip-ranges#overview-of-api-server-authorized-ip-ranges)æ¥ä¿æŠ¤å¯¹ Kubernetes API server 的访问。 + +Kubernetes API server 公开 Kubernetes API。该组件æ供管ç†å·¥å…·ï¼ˆä¾‹å¦‚ kubectl）的交互。AKS æ供带有专用 API server çš„å•ç§Ÿæˆ·é›†ç¾¤ controlplane。默认情况下，API server 分é…了一个公共 IP 地å€ï¼Œä½ åº”该使用基于 Kubernetes 或 Azure çš„ RBAC æ¥æŽ§åˆ¶å¯¹ API server 的访问。 + +è¦ä¿æŠ¤å¯¹å…¶ä»–å¯å…¬å¼€çš„ AKS controlplane å’Œ API server 的访问，你å¯ä»¥å¯ç”¨å¹¶ä½¿ç”¨æŽˆæƒçš„ IP 范围。这些授æƒçš„ IP 范围åªå…许定义的 IP 地å€èŒƒå›´ä¸Ž API server 通信。 + +但是，å³ä½¿ä½ ä½¿ç”¨äº†æŽˆæƒçš„ IP 地å€èŒƒå›´ï¼Œä½ ä»åº”使用 Kubernetes RBAC 或 Azure RBAC æ¥æŽˆæƒç”¨æˆ·åŠå…¶è¯·æ±‚çš„æ“作。 + +### 容器监控 + +容器监控使用 Metrics API 从 Kubernetes 中å¯ç”¨çš„控制器ã€èŠ‚点和容器中收集内存和处ç†å™¨æŒ‡æ ‡ï¼Œä»Žè€Œä¸ºä½ å¯è§†åŒ–性能数æ®ã€‚容器日志也能被收集。å¯ç”¨ç›‘控åŽï¼Œç³»ç»Ÿä¼šé€šè¿‡ Linux çš„ Log Analytics 代ç†çš„容器化版本自动为你收集指标和日志。指标会被写入指标存储，而日志数æ®ä¼šè¢«å†™å…¥ä¸Žä½ çš„ [Log Analytics](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview) 工作区关è”的日志存储。 + +### Log Analytics 工作区资æºç»„ + +[资æºç»„](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups)åŒ…å« Log Analytics 工作区。你必须至少创建一个工作区æ‰èƒ½ä½¿ç”¨ Azure Monitor Logs。 + +### Log Analytics 工作区å称 + +Azure Monitor Logs 收集的数æ®å­˜å‚¨åœ¨ä¸€ä¸ªæˆ–多个 [Log Analytics 工作区中](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/design-logs-deployment)。工作区定义了数æ®çš„地ç†ä½ç½®ã€è®¿é—®æƒé™ï¼ˆå®šä¹‰äº†å“ªäº›ç”¨æˆ·å¯ä»¥è®¿é—®æ•°æ®ï¼‰ä»¥åŠé…置设置（定价层和数æ®ä¿ç•™ç­‰ï¼‰ã€‚ + +你必须至少创建一个工作区æ‰èƒ½ä½¿ç”¨ Azure Monitor Logs。一个工作区å¯èƒ½å°±è¶³ä»¥æ»¡è¶³ä½ çš„所有监控数æ®ã€‚你也å¯ä»¥æ ¹æ®éœ€æ±‚创建多个工作区。例如，你å¯èƒ½æœ‰ä¸€ä¸ªå·¥ä½œåŒºç”¨äºŽç”Ÿäº§æ•°æ®ï¼Œå¦ä¸€ä¸ªå·¥ä½œåŒºç”¨äºŽæµ‹è¯•ã€‚ + +有关 Azure Monitor Logs 的详细信æ¯ï¼Œè¯·å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-platform-logs)。 + +### 支æŒç§æœ‰ Kubernetes æœåŠ¡ + +通常情况下，无论集群是å¦ä¸ºç§æœ‰ï¼ŒAKS worker 节点都ä¸ä¼šèŽ·å¾—公共 IP。在ç§æœ‰é›†ç¾¤ä¸­ï¼Œcontrolplane 没有公共端点。 + +Rancher å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€è¿žæŽ¥åˆ°ç§æœ‰ AKS 集群。 + +第一ç§æ–¹æ³•æ˜¯ç¡®ä¿ Rancher è¿è¡Œåœ¨ä¸Ž AKS 节点相åŒçš„ [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) 上。 + +第二ç§æ–¹æ³•æ˜¯è¿è¡Œå‘½ä»¤å‘ Rancher 注册集群。é…置集群åŽï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èƒ½è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API 的地方è¿è¡Œæ˜¾ç¤ºçš„命令。é…ç½®å¯ç”¨äº†ç§æœ‰ API 端点的 AKS 集群时，此命令将显示在弹出窗å£ä¸­ã€‚ + +:::note + +注册现有 AKS 集群时，集群å¯èƒ½éœ€è¦ä¸€äº›æ—¶é—´ï¼ˆå¯èƒ½æ˜¯æ•°å°æ—¶ï¼‰æ‰ä¼šå‡ºçŽ°åœ¨ `Cluster To register` 下拉列表中。ä¸åŒåŒºåŸŸçš„结果å¯èƒ½ä¸åŒã€‚ + +::: + +有关连接到 AKS 专用集群的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster)。 + +## 节点池 + +### æ¨¡å¼ + +在 Azure ç•Œé¢ä¸­ï¼Œç”¨æˆ·èƒ½å¤ŸæŒ‡å®šä¸»è¦èŠ‚点池（Primary Node Pool）ä¾èµ–于 `system`（通常用于 controlplane）还是 `user`（Rancher 最常用的）。 + +ä½ å¯ä»¥æŒ‡å®šä¸»è¦èŠ‚点池（Primary Node Pool）的模å¼ã€æ“作系统ã€æ•°é‡å’Œå¤§å°ã€‚ + +`system` 节点池总是需è¦è¿è¡ŒèŠ‚点，因此它们ä¸èƒ½ç¼©å®¹åˆ°ä¸€ä¸ªèŠ‚点以下。至少需è¦ä¸€ä¸ª `system` 节点池。 + +对于åŽç»­çš„节点池，Rancher UI 强制使用默认的 `user`。`user` 节点池å…许缩容到零节点。`user` 节点池ä¸è¿è¡Œ Kubernetes controlplane 的任何部分。 + +AKS ä¸ä¼šå…¬å¼€è¿è¡Œ Kubernetes controlplane 组件的节点。 + +### å¯ç”¨åŒº + +[å¯ç”¨åŒº](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview)是区域内的唯一物ç†ä½ç½®ã€‚æ¯ä¸ªå¯ç”¨åŒºç”±ä¸€ä¸ªæˆ–多个é…备独立电æºã€å†·å´ç³»ç»Ÿå’Œç½‘络的数æ®ä¸­å¿ƒç»„æˆã€‚ + +并éžæ‰€æœ‰åŒºåŸŸéƒ½æ”¯æŒå¯ç”¨åŒºã€‚有关具有å¯ç”¨åŒºçš„ Azure 区域列表，请å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/availability-zones/az-region#azure-regions-with-availability-zones)。 + +### è™šæ‹Ÿæœºå¤§å° + +为节点池中的æ¯ä¸ª VM 选择一个大å°ã€‚有关æ¯ä¸ª VM 大å°çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/)。 + +### æ“作系统ç£ç›˜ç±»åž‹ + +节点池中的节点å¯ä»¥ä½¿ç”¨æ‰˜ç®¡ç£ç›˜æˆ–临时ç£ç›˜ã€‚ + +[临时 OS ç£ç›˜](https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks)在本地虚拟机存储上创建，并ä¸ä¼šä¿å­˜åˆ°è¿œç¨‹ Azure 存储。临时 OS ç£ç›˜é€‚用于无状æ€å·¥ä½œè´Ÿè½½ï¼Œå…¶ä¸­çš„应用å¯ä»¥å®¹å¿å•ä¸ª VM æ•…éšœï¼Œä½†æ›´å®¹æ˜“å— VM 部署时间或é‡ç½®å•ä¸ªè™šæ‹Ÿæœºå®žä¾‹é•œåƒçš„å½±å“。使用临时 OS ç£ç›˜ï¼Œä½ å¯ä»¥ä½“验更低的 OS ç£ç›˜è¯»/写延迟和更快的 VM é‡ç½®é•œåƒè¿‡ç¨‹ã€‚ + +[Azure 托管ç£ç›˜](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview)是由 Azure 管ç†å¹¶ä¸Ž Azure Virtual Machines 一起使用的å—级存储å·ã€‚托管ç£ç›˜æ—¨åœ¨å®žçŽ° 99.999% 的高å¯ç”¨æ€§ã€‚托管ç£ç›˜é€šè¿‡æ供三个数æ®å‰¯æœ¬æ¥å®žçŽ°é«˜å¯ç”¨æ€§å’Œé«˜æŒç»­æ€§ã€‚ + +### æ“作系统ç£ç›˜å¤§å° + +æ¯ä¸ªèŠ‚点的ç£ç›˜å¤§å°ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚ + +### 节点数 +节点池中的节点数。[Azure 订阅](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits)å¯èƒ½ä¼šé™åˆ¶æœ€å¤§èŠ‚点数。 + +### æ¯ä¸ªèŠ‚点的最大 Pod æ•°é‡ +æ¯ä¸ªèŠ‚点的最大 Pod æ•°é‡é»˜è®¤ä¸º 110，最大为 250。 + +### å¯ç”¨è‡ªåŠ¨æ‰©ç¼©å®¹ + +å¯ç”¨è‡ªåŠ¨æ‰©ç¼©å®¹åŽï¼Œä½ éœ€è¦è¾“入最å°å’Œæœ€å¤§èŠ‚点数。 + +å¯ç”¨ Auto Scaling åŽï¼Œä½ å°†æ— æ³•æ‰‹åŠ¨å¯¹èŠ‚点池进行扩缩容。扩缩容由 AKS autoscaler 控制。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md new file mode 100644 index 00000000000..5de68eb41ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -0,0 +1,149 @@ +--- +title: EKS 集群é…ç½®å‚考 +--- + +### è´¦å·è®¿é—® + +使用获å–çš„ä¿¡æ¯ä¸º IAM 策略填写æ¯ä¸ªä¸‹æ‹‰åˆ—表和字段： + +| 设置 | æè¿° | +| ---------- | -------------------------------------------------------------------------------------------------------------------- | +| 区域 | 从下拉列表中选择构建集群的地ç†åŒºåŸŸã€‚ | +| äº‘å‡­è¯ | 选择为 IAM 策略创建的云凭è¯ã€‚有关在 Rancher 中创建云凭è¯çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../user-settings/manage-cloud-credentials.md)。 | + +### æœåŠ¡è§’色 + +选择一个[æœåŠ¡è§’色](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html)。 + +| æœåŠ¡è§’色 | æè¿° | +-------------|--------------------------- +| Standard：Rancher 生æˆçš„æœåŠ¡è§’色 | 如果选择此角色，Rancher 会自动添加一个æœåŠ¡è§’色以供集群使用。 | +| 自定义：从现有的æœåŠ¡è§’色中选择 | 如果选择此角色，Rancher å°†å…许你从已在 AWS 中创建的æœåŠ¡è§’色中进行选择。有关在 AWS 中创建自定义æœåŠ¡è§’色的更多信æ¯ï¼Œè¯·å‚阅 [Amazon 文档](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html#create-service-linked-role)。 | + +### 密文加密 + +å¯é€‰ï¼šè¦åŠ å¯†å¯†æ–‡ï¼Œè¯·é€‰æ‹©æˆ–输入在 [AWS 密钥管ç†æœåŠ¡ (KMS)](https://docs.aws.amazon.com/kms/latest/developerguide/overview.html) 中创建的密钥。 + +### API Server 端点访问 + +é…置公共/ç§æœ‰ API 访问是一个高级用例。有关详细信æ¯ï¼Œè¯·å‚阅 [EKS 集群端点访问控制文档](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html)。 + +### 专用 API 端点 + +如果你在创建集群时å¯ç”¨äº†ç§æœ‰ API 端点访问，并ç¦ç”¨äº†å…¬å…± API 端点访问，那么你必须进行é¢å¤–的步骤æ‰èƒ½ä½¿ Rancher æˆåŠŸè¿žæŽ¥åˆ°é›†ç¾¤ã€‚在这ç§æƒ…况下，一个弹窗将会显示，其中包å«éœ€è¦åœ¨è¦æ³¨å†Œåˆ° Rancher 的集群上è¿è¡Œçš„命令。é…置集群åŽï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èƒ½è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API 的地方è¿è¡Œæ˜¾ç¤ºçš„命令。 + +以下两ç§æ–¹æ³•èƒ½é¿å…这个é¢å¤–的手动步骤： +- 在创建集群时，创建具有ç§æœ‰å’Œå…¬å…± API 端点访问æƒé™çš„集群。在集群创建并处于 active 状æ€åŽï¼Œä½ å¯ä»¥ç¦ç”¨å…¬å…±è®¿é—®ï¼ŒRancher 将能继续与 EKS 集群通信。 +- ç¡®ä¿ Rancher 与 EKS 集群共享åŒä¸€ä¸ªå­ç½‘。然åŽï¼Œä½ å¯ä»¥ä½¿ç”¨å®‰å…¨ç»„使 Rancher 能够与集群的 API 端点进行通信。在这ç§æƒ…况下，你ä¸éœ€è¦è¿è¡Œæ³¨å†Œé›†ç¾¤çš„命令，Rancher 就能够与你的集群通信。有关é…置安全组的更多信æ¯ï¼Œè¯·å‚阅[安全组文档](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html)。 + +### 公共访问端点 + +你也å¯ä»¥é€‰æ‹©é€šè¿‡æ˜¾å¼ CIDR å—æ¥é™åˆ¶å¯¹å…¬å…±ç«¯ç‚¹çš„访问。 + +如果你é™åˆ¶å¯¹ç‰¹å®š CIDR å—的访问，那么建议你也å¯ç”¨ç§æœ‰è®¿é—®ï¼Œä»¥é¿å…丢失与集群的网络通信。 + +å¯ç”¨ç§æœ‰è®¿é—®éœ€è¦æ»¡è¶³ä»¥ä¸‹æ¡ä»¶ä¹‹ä¸€ï¼š +- Rancher çš„ IP 必须是å…许的 CIDR å—的一部分。 +- 应该å¯ç”¨äº†ç§æœ‰è®¿é—®ã€‚此外，Rancher 必须和集群共享åŒä¸€ä¸ªå­ç½‘，并对集群有网络访问æƒé™ã€‚ä½ å¯ä»¥é€šè¿‡å®‰å…¨ç»„æ¥è¿›è¡Œé…置。 + +有关对集群端点的公共和ç§æœ‰è®¿é—®çš„更多信æ¯ï¼Œè¯·å‚阅 [Amazon EKS 文档](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html)。 + +### å­ç½‘ + +| 选项 | æè¿° | +| ------- | ------------ | +| Standard：Rancher 生æˆçš„ VPC å’Œå­ç½‘ | 在é…置集群时，Rancher 会生æˆä¸€ä¸ªå…·æœ‰ 3 个公有å­ç½‘çš„æ–° VPC。 | +| 自定义：从现有的 VPC å’Œå­ç½‘中选择 | 在é…置集群时，Rancher 将你的 controlplane 和节点é…置为使用你已ç»[在 AWS 中创建](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html)çš„ VPC å’Œå­ç½‘。 | + +有关更多信æ¯ï¼Œè¯·å‚阅 AWS 文档以了解[集群 VPC 注æ„事项](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html)。根æ®ä½ åœ¨ä¸Šä¸€æ­¥ä¸­çš„选择，按照以下说明进行æ“作。 + +- [什么是 Amazon VPC？](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) +- [VPC å’Œå­ç½‘](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) + +### 安全组 + +Amazon 文档： + +- [集群安全组注æ„事项](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) +- [VPC 的安全组](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) +- [创建安全组](https://docs.aws.amazon.com/vpc/latest/userguide/getting-started-ipv4.html#getting-started-create-security-group) + +### Logging + +å°† controlplane 日志é…置为å‘é€åˆ° Amazon CloudWatch。如果你将集群日志å‘é€åˆ° CloudWatch Logs，你需è¦æŒ‰ç…§ standard CloudWatch Logs 支付数æ®å¼•å…¥å’Œå­˜å‚¨è´¹ç”¨ã€‚ + +æ¯ä¸ªæ—¥å¿—类型å‡å¯¹åº”一个 Kubernetes controlplane 组件。有关这些组件的更多信æ¯ï¼Œè¯·å‚阅 Kubernetes 文档中的 [Kubernetes 组件](https://kubernetes.io/docs/concepts/overview/components/)。 + +有关 EKS controlplane 日志管ç†çš„更多信æ¯ï¼Œè¯·å‚阅[官方文档](https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)。 + +### 托管节点组 + +Amazon EKS 托管的节点组自动为 Amazon EKS Kubernetes 集群的节点（Amazon EC2 实例）进行预置和生命周期管ç†ã€‚ + +有关节点组如何工作以åŠå¦‚何é…置的更多信æ¯ï¼Œè¯·å‚阅 [EKS 文档](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html)。 + +#### 使用你自己的å¯åŠ¨æ¨¡æ¿ + +ä½ å¯ä»¥æä¾›å¯åŠ¨æ¨¡æ¿ ID 和版本，以便轻æ¾é…置节点组中的 EC2 实例。如果你æ供了å¯åŠ¨æ¨¡æ¿ï¼Œåˆ™ä»¥ä¸‹è®¾ç½®éƒ½æ— æ³•åœ¨ Rancher 中进行é…置。因此，如果你使用å¯åŠ¨æ¨¡æ¿ï¼Œåˆ™éœ€è¦åœ¨å¯åŠ¨æ¨¡æ¿ä¸­æŒ‡å®šä»¥ä¸‹åˆ—表中的所有必须和所需的设置。å¦è¯·æ³¨æ„，如果æ供了å¯åŠ¨æ¨¡æ¿ ID 和版本，则åªèƒ½æ›´æ–°æ¨¡æ¿ç‰ˆæœ¬ã€‚如果è¦ä½¿ç”¨æ–°æ¨¡æ¿ ID，则需è¦åˆ›å»ºæ–°çš„托管节点组。 + +| 选项 | æè¿° | å¿…å¡«/选填 | +| ------ | ----------- | ----------------- | +| 实例类型 | 为è¦é…置的实例选择[硬件规格](https://aws.amazon.com/ec2/instance-types/)。 | å¿…å¡« | +| é•œåƒ ID | 为节点指定自定义 AMI。与 EKS 一起使用的自定义 AMI å¿…é¡»[正确é…ç½®](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/)。 | 选填 | +| 节点å·å¤§å° | å¯åŠ¨æ¨¡æ¿å¿…须指定具有所需大å°çš„ EBS å·ã€‚ | å¿…å¡« | +| SSH 密钥 | è¦æ·»åŠ åˆ°å®žä¾‹ä»¥å¯¹èŠ‚点进行 SSH 访问的密钥。 | 选填 | +| ç”¨æˆ·æ•°æ® | [MIME 多部分格å¼](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data)çš„ Cloud init 脚本。 | 选填 | +| 实例资æºæ ‡ç­¾ | 标记节点组中的æ¯ä¸ª EC2 实例。 | 选填 | + +#### Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿ + +如果你ä¸æŒ‡å®šå¯åŠ¨æ¨¡æ¿ï¼Œä½ å°†èƒ½å¤Ÿåœ¨ Rancher UI 中é…置上述选项，并且å¯ä»¥åœ¨åˆ›å»ºåŽæ›´æ–°æ‰€æœ‰è¿™äº›é€‰é¡¹ã€‚为了利用所有这些选项，Rancher 将为你创建和管ç†å¯åŠ¨æ¨¡æ¿ã€‚Rancher 中的所有集群都将有一个 Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿ã€‚此外，æ¯ä¸ªæ²¡æœ‰æŒ‡å®šå¯åŠ¨æ¨¡æ¿çš„托管节点组都将具有一个管ç†çš„å¯åŠ¨æ¨¡æ¿ç‰ˆæœ¬ã€‚æ­¤å¯åŠ¨æ¨¡æ¿çš„å称将具有 “rancher-managed-lt-†å‰ç¼€ï¼ŒåŽé¢æ˜¯é›†ç¾¤çš„显示å称。此外，Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿å°†ä½¿ç”¨ “rancher-managed-template†键和 “do-not-modify-or-delete†值æ¥è¿›è¡Œæ ‡è®°ï¼Œä»¥å°†å…¶è¯†åˆ«ä¸º Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿ã€‚请注æ„，ä¸è¦ä¿®æ”¹æˆ–删除此å¯åŠ¨æ¨¡æ¿ï¼Œæˆ–将此å¯åŠ¨æ¨¡æ¿ä¸Žå…¶ä»–集群或托管节点组一起使用。因为这å¯èƒ½ä¼šä½¿ä½ çš„节点组“é™çº§â€å¹¶éœ€è¦é”€æ¯å’Œé‡æ–°åˆ›å»ºã€‚ + +#### 自定义 AMI + +如果你在å¯åŠ¨æ¨¡æ¿æˆ– Rancher 中指定了自定义 AMI，则必须[正确é…ç½®](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/)é•œåƒï¼Œå¹¶ä¸”å¿…é¡»æ供用户数æ®ä»¥[引导节点](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami)。这是一个高级用例，因此你必须è¦äº†è§£å…¶è¦æ±‚。 + +如果你指定了ä¸åŒ…å«è‡ªå®šä¹‰ AMI çš„å¯åŠ¨æ¨¡æ¿ï¼Œåˆ™ Amazon 将为 Kubernetes 版本和所选区域使用 [EKS 优化的 AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html)。你还å¯ä»¥ä¸ºèƒ½ä»Žä¸­å—益的工作负载选择[å¯ç”¨ GPU 的实例](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html#gpu-ami)。 + +:::note + +如果你在下拉èœå•æˆ–å¯åŠ¨æ¨¡æ¿ä¸­æ供了自定义 AMI，则会忽略 Rancher 中设置的å¯ç”¨ GPU 的实例。 + +::: + +#### Spot 实例 + +Spot 实例现在[å— EKS 支æŒ](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types-spot)。如果你指定了å¯åŠ¨æ¨¡æ¿ï¼ŒAmazon 建议ä¸è¦åœ¨æ¨¡æ¿ä¸­æ供实例类型。相å，Amazon 建议æ供多ç§å®žä¾‹ç±»åž‹ã€‚如果你为节点组å¯ç”¨äº†â€œè¯·æ±‚ Spot 实例â€å¤é€‰æ¡†ï¼Œé‚£ä¹ˆä½ å°†æœ‰æœºä¼šæ供多ç§å®žä¾‹ç±»åž‹ã€‚ + +:::note + +在这ç§æƒ…况下，你在实例类型下拉列表中所选的选项都将被忽略，你必须在“Spot 实例类型â€ä¸­è‡³å°‘指定一ç§å®žä¾‹ç±»åž‹ã€‚此外，与 EKS 一起使用的å¯åŠ¨æ¨¡æ¿æ— æ³•è¯·æ±‚ Spot 实例。请求 Spot 实例必须是 EKS é…置的一部分。 + +::: + +#### 节点组设置 + +以下设置也是å¯é…置的。在创建节点组åŽï¼Œé™¤â€œèŠ‚点组å称â€å¤–的所有选项都是å¯ç¼–辑的。 + +| 选项 | æè¿° | +| ------- | ------------ | +| 节点组å称 | 节点组的å称。 | +| 期望 ASG å¤§å° | 期望的实例数é‡ã€‚ | +| 最大 ASG å¤§å° | 最大的实例数é‡ã€‚在安装 [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) 之å‰ï¼Œæ­¤è®¾ç½®ä¸ä¼šç”Ÿæ•ˆã€‚ | +| æœ€å° ASG å¤§å° | 最å°çš„实例数é‡ã€‚在安装 [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) 之å‰ï¼Œæ­¤è®¾ç½®ä¸ä¼šç”Ÿæ•ˆã€‚ | +| Labels | 应用于管ç†çš„节点组中节点的 Kubernetes 标签。 | +| Tags | 管ç†çš„节点组的标签，这些标签ä¸ä¼šä¼ æ’­åˆ°ä»»ä½•ç›¸å…³èµ„æºã€‚ | + + +### é…置刷新间隔 + +`eks-refresh-cron` 设置已弃用。它已è¿ç§»åˆ° `eks-refresh` 设置，这是一个表示秒的整数。 + +默认值为 300 秒。 + +ä½ å¯ä»¥é€šè¿‡è¿è¡Œ `kubectl edit setting eks-refresh` æ¥æ›´æ”¹åŒæ­¥é—´éš”。 + +如果之å‰è®¾ç½®äº† `eks-refresh-cron` 设置，è¿ç§»å°†è‡ªåŠ¨è¿›è¡Œã€‚ + +刷新窗å£è¶ŠçŸ­ï¼Œäº‰ç”¨æ¡ä»¶å‘生的å¯èƒ½æ€§å°±è¶Šå°ã€‚但这确实增加了é‡åˆ° AWS API å¯èƒ½å­˜åœ¨çš„请求é™åˆ¶çš„å¯èƒ½æ€§ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md new file mode 100644 index 00000000000..fa562ae9cee --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -0,0 +1,49 @@ +--- +title: ç§æœ‰é›†ç¾¤ +--- + +在 GKE 中，[ç§æœ‰é›†ç¾¤](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept)是一ç§é›†ç¾¤ï¼Œå…¶èŠ‚点仅通过分é…内部 IP 地å€ä¸Žå…¥ç«™å’Œå‡ºç«™æµé‡ç›¸éš”离。GKE 中的ç§æœ‰é›†ç¾¤å¯ä»¥é€‰æ‹©å°† controlplane 端点作为公开访问的地å€æˆ–作为ç§æœ‰åœ°å€ã€‚这与其他 Kubernetes æ供商ä¸åŒï¼ŒåŽè€…å¯èƒ½å°†å…·æœ‰ç§æœ‰ controlplane 端点的集群称为“ç§æœ‰é›†ç¾¤â€ï¼Œä½†ä»å…许进出节点的æµé‡ã€‚基于你的组织的网络和安全è¦æ±‚，你å¯èƒ½æƒ³åˆ›å»ºä¸€ä¸ªæœ‰ç§æœ‰èŠ‚点的集群，其中有或没有公共 controlplane 端点。从 Rancher é…置的 GKE 集群å¯ä»¥é€šè¿‡åœ¨**集群选项**中选择**ç§æœ‰é›†ç¾¤**（在**显示高级选项**下）æ¥ä½¿ç”¨éš”离的节点。通过选择**å¯ç”¨ç§æœ‰ç«¯ç‚¹**，å¯ä»¥é€‰æ‹©å°† controlplane 端点设为ç§æœ‰ã€‚ + +### ç§æœ‰èŠ‚点 + +由于ç§æœ‰é›†ç¾¤ä¸­çš„节点åªæœ‰å†…部 IP 地å€ï¼Œå®ƒä»¬å°†æ— æ³•å®‰è£… cluster agent，Rancher 将无法完全管ç†é›†ç¾¤ã€‚è¿™å¯ä»¥é€šè¿‡å‡ ç§æ–¹å¼æ¥å¤„ç†ã€‚ + +#### Cloud NAT + +:::caution + +Cloud NAT å°†[产生费用](https://cloud.google.com/nat/pricing)。 + +::: + +如果é™åˆ¶å¤–出的互è”网访问对你的组织æ¥è¯´ä¸æ˜¯ä¸€ä¸ªé—®é¢˜ï¼Œå¯ä»¥ä½¿ç”¨ Google çš„ [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) æœåŠ¡æ¥å…许ç§æœ‰ç½‘络中的节点访问互è”网，使它们能够从 Dockerhub 下载所需的镜åƒå¹¶ä¸Ž Rancher management server 通信。这是最简å•çš„解决方案。 + +#### ç§æœ‰é•œåƒä»“库 + +:::caution + +此方案ä¸å—官方支æŒã€‚如果 Cloud NAT æœåŠ¡ä¸è¶³ä»¥æ»¡è¶³ä½ çš„需求，则å¯ä»¥å‚考此方案。 + +::: + +如果è¦æ±‚é™åˆ¶èŠ‚点的传入和传出æµé‡ï¼Œè¯·æŒ‰ç…§ç¦»çº¿å®‰è£…说明，在集群所在的 VPC 上设置一个ç§æœ‰å®¹å™¨[é•œåƒä»“库](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md)，从而å…许集群节点访问和下载è¿è¡Œ cluster agent 所需的镜åƒã€‚如果 controlplane 端点也是ç§æœ‰çš„，Rancher 将需è¦[直接访问](#直接访问)它。 + +### ç§æœ‰ controlplane 端点 + +如果集群暴露了公共端点，Rancher 将能够访问集群，且无需执行é¢å¤–çš„æ­¥éª¤ã€‚ä½†æ˜¯ï¼Œå¦‚æžœé›†ç¾¤æ²¡æœ‰å…¬å…±ç«¯ç‚¹ï¼Œåˆ™å¿…é¡»ç¡®ä¿ Rancher å¯ä»¥è®¿é—®é›†ç¾¤ã€‚ + +#### Cloud NAT + +:::caution + +Cloud NAT å°†[产生费用](https://cloud.google.com/nat/pricing)。 + +::: + +如上所述，如果ä¸è€ƒè™‘é™åˆ¶å¯¹èŠ‚点的传出互è”网访问，则å¯ä»¥ä½¿ç”¨ Google çš„ [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) æœåŠ¡æ¥å…许节点访问互è”网。当集群进行é…置时，Rancher å°†æ供一个在集群上è¿è¡Œçš„注册命令。下载新集群的 [kubeconfig](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) 并在集群上è¿è¡Œæ供的 kubectl 命令。如果è¦é€šè¿‡èŽ·å–集群访问æƒæ¥è¿è¡Œæ­¤å‘½ä»¤ï¼Œä½ å¯ä»¥åˆ›å»ºä¸´æ—¶èŠ‚点或使用 VPC 中的现有节点，或者登录到æŸä¸ªé›†ç¾¤èŠ‚点或使用æŸä¸ªé›†ç¾¤èŠ‚点创建 SSH 隧é“。 + +#### 直接访问 + +如果 Rancher server 与集群的 controlplane è¿è¡Œåœ¨åŒä¸€ VPC 上，它将直接访问 controlplane çš„ç§æœ‰ç«¯ç‚¹ã€‚集群节点将需è¦è®¿é—®[ç§æœ‰é•œåƒä»“库](#ç§æœ‰é•œåƒä»“库)以下载上述的镜åƒã€‚ + +你还å¯ä»¥ä½¿ç”¨ Google çš„æœåŠ¡ï¼ˆä¾‹å¦‚ [Cloud VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/overview) 或 [Cloud Interconnect VLAN](https://cloud.google.com/network-connectivity/docs/interconnect)）æ¥ä¿ƒè¿›ç»„织网络与 Google VPC 之间的连接。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md new file mode 100644 index 00000000000..507f93870bc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md @@ -0,0 +1,151 @@ +--- +title: K3s 集群é…ç½®å‚考 +--- + +æœ¬æ–‡ä»‹ç» Rancher 中å¯ç”¨äºŽæ–°çš„或现有的 K3s Kubernetes 集群的é…置选项。 + +## 概述 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€æ¥é…ç½® Kubernetes 选项： + +- [Rancher UI](#rancher-ui-中的é…置选项)：使用 Rancher UI æ¥é€‰æ‹©è®¾ç½® Kubernetes 集群时常用的自定义选项。 +- [集群é…置文件](#集群é…置文件)：高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª K3s é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å…许你设置 K3s 安装中å¯ç”¨çš„任何[选项](https://rancher.com/docs/k3s/latest/en/installation/install-options/)。 + +## Rancher UI 中的é…置选项 + +:::tip + +一些高级é…置选项没有在 Rancher UI 表å•ä¸­å¼€æ”¾ï¼Œä½†ä½ å¯ä»¥é€šè¿‡åœ¨ YAML 中编辑 K3s 集群é…置文件æ¥å¯ç”¨è¿™äº›é€‰é¡¹ã€‚有关 YAML 中 K3s 集群的å¯é…置选项的完整å‚考，请å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/installation/install-options/)。 + +::: + +### åŸºæœ¬ä¿¡æ¯ +#### Kubernetes 版本 + +这指的是集群节点上安装的 Kubernetes 版本。Rancher 基于 [hyperkube](https://github.com/rancher/hyperkube) 打包了自己的 Kubernetes 版本。 + +有关更多详细信æ¯ï¼Œè¯·å‚阅[å‡çº§ Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 + +#### 加密密文 + +å¯ç”¨æˆ–ç¦ç”¨å¯†æ–‡åŠ å¯†çš„选项。å¯ç”¨åŽï¼Œå¯†æ–‡å°†ä½¿ç”¨ AES-CBC 密钥进行加密。如果ç¦ç”¨ï¼Œåˆ™åœ¨å†æ¬¡å¯ç”¨åŠ å¯†ä¹‹å‰æ— æ³•è¯»å–任何以å‰çš„密文。有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/advanced/#secrets-encryption-config-experimental)。 + +#### 项目网络隔离 + +如果你的网络æ供商å…许项目网络隔离，你å¯ä»¥é€‰æ‹©å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´çš„通信。 + +#### SELinux + +å¯ç”¨æˆ–ç¦ç”¨ [SELinux](https://rancher.com/docs/k3s/latest/en/advanced/#selinux-support) 支æŒçš„选项。 + +#### CoreDNS + +默认情况下，[CoreDNS](https://coredns.io/) 会安装为默认 DNS æ供程åºã€‚如果未安装 CoreDNS，则必须自己安装备用 DNS æ供程åºã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/networking/#coredns)。 + +#### Klipper Service LB + +å¯ç”¨æˆ–ç¦ç”¨ [Klipper](https://github.com/rancher/klipper-lb) æœåŠ¡è´Ÿè½½å‡è¡¡å™¨çš„选项。有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer)。 + +#### Traefik Ingress + +å¯ç”¨æˆ–ç¦ç”¨ [Traefik](https://traefik.io/) HTTP åå‘代ç†å’Œè´Ÿè½½å‡è¡¡å™¨çš„选项。有关更多详细信æ¯å’Œé…置选项，请å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/networking/#traefik-ingress-controller)。 + +#### Local Storage + +在节点上å¯ç”¨æˆ–ç¦ç”¨ [Local Storage](https://rancher.com/docs/k3s/latest/en/storage/) 的选项。 + +#### Metrics Server + +å¯ç”¨æˆ–ç¦ç”¨ [metrics server](https://github.com/kubernetes-incubator/metrics-server) 的选项。如果å¯ç”¨ï¼Œè¯·ç¡®ä¿ä¸ºå…¥ç«™ TCP æµé‡æ‰“å¼€ 10250 端å£ã€‚ + +### 附加é…ç½® + +集群å¯åŠ¨æ—¶å°†åº”用的其他 Kubernetes 清å•ï¼Œä¼šä½œä¸º[附加组件](https://kubernetes.io/docs/concepts/cluster-administration/addons/)æ¥ç®¡ç†ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/helm/#automatically-deploying-manifests-and-helm-charts)。 + +### Agent 环境å˜é‡ + +为 [K3s agent](https://rancher.com/docs/k3s/latest/en/architecture/) 设置环境å˜é‡çš„选项。你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/)。 + +### etcd + +#### 自动快照 + +å¯ç”¨æˆ–ç¦ç”¨å®šæœŸ etcd 快照的选项。如果å¯ç”¨ï¼Œç”¨æˆ·å¯ä»¥é…置快照的频率。有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/backup-restore/#creating-snapshots)。 + +#### 指标 + +选择å‘公众公开或仅在集群内公开 etcd 指标的选项。 + +### 网络 + +#### 集群 CIDR + +用于 pod IP çš„ IPv4/IPv6 网络 CIDR（默认：10.42.0.0/16）。 + +#### Service CIDR + +用于 Service IP çš„ IPv4/IPv6 网络 CIDR（默认：10.43.0.0/16）。 + +#### 集群 DNS + +用于 coredns æœåŠ¡çš„ IPv4 集群 IP。应该在你的 service-cidr 范围内（默认：10.43.0.10）。 + +#### 集群域å + +选择集群的域。默认值为 `cluster.local`。 + +#### NodePort æœåŠ¡ç«¯å£èŒƒå›´ + +更改å¯ç”¨äºŽ [NodePort æœåŠ¡](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)的端å£èŒƒå›´çš„选项。默认值为 `30000-32767`。 + +#### 截断主机å + +将主机å截断为 15 个或更少字符。该字段åªèƒ½åœ¨é›†ç¾¤åˆå§‹åˆ›å»ºæ—¶è®¾ç½®ã€‚创建集群åŽï¼Œä½ æ— æ³•å†å¯ç”¨æˆ–ç¦ç”¨ 15 个字符的é™åˆ¶ã€‚ + +此设置仅影å“é…置了机器的集群。由于自定义集群在创建节点时设置主机å（å‘生在 Rancher 之外），因此该字段ä¸é™åˆ¶è‡ªå®šä¹‰é›†ç¾¤çš„主机å长度。 + +截断集群中的主机åå¯æ高与基于 Windows 的系统的兼容性。虽然 Kubernetes å…许最长的主机å长度为 63 个字符，但使用 NetBIOS 的系统将主机åé™åˆ¶ä¸º 15 个字符之内。 + +#### TLS å¯é€‰å称 + +在æœåŠ¡å™¨ TLS è¯ä¹¦ä¸Šæ·»åŠ å…¶ä»–主机å或 IPv4/IPv6 地å€ä½œä¸º Subject Alternative Name。 + +#### 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯ç”¨äºŽç›´æŽ¥è®¿é—® Kubernetes API server，而无需通过 Rancher 进行通信。 + +有关授æƒé›†ç¾¤ç«¯ç‚¹çš„工作原ç†ä»¥åŠä½¿ç”¨çš„原因，请å‚阅[架构介ç»](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +### é•œåƒä»“库 + +选择è¦ä»Žä¸­æ‹‰å– Rancher é•œåƒçš„é•œåƒä»“库。有关更多详细信æ¯å’Œé…置选项，请å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)。 + +### å‡çº§ç­–ç•¥ + +#### controlplane å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### Worker å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### 清空节点（controlplane） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +#### 清空节点（worker 节点） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +### 高级é…ç½® + +为ä¸åŒèŠ‚点设置 kubelet 选项。有关å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)。 + +## 集群é…置文件 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª K3s é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI 表å•æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å…许你设置 K3s 安装中å¯ç”¨çš„任何[选项](https://rancher.com/docs/k3s/latest/en/installation/install-options/)。 + +è¦ç›´æŽ¥ä»Ž Rancher UI 编辑 K3s é…置文件，å•å‡»**以 YAML 文件编辑**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md new file mode 100644 index 00000000000..7cdc1ec7c64 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -0,0 +1,361 @@ +--- +title: RKE 集群é…ç½®å‚考 +--- + +Rancher 安装 Kubernetes 时，它使用 [RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 或 [RKE2](https://docs.rke2.io/) 作为 Kubernetes å‘行版。 + +æœ¬æ–‡ä»‹ç» Rancher 中å¯ç”¨äºŽæ–°çš„或现有的 RKE Kubernetes 集群的é…置选项。 + + +## 概述 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€æ¥é…ç½® Kubernetes 选项： + +- [Rancher UI](#rancher-ui-中的é…置选项)：使用 Rancher UI æ¥é€‰æ‹©è®¾ç½® Kubernetes 集群时常用的自定义选项。 +- [集群é…置文件](#rke-集群é…置文件å‚考)：高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ ä½¿ç”¨ YAML æ¥æŒ‡å®š RKE 安装中å¯ç”¨çš„任何选项（除了 system_images é…置）。 + +RKE 集群é…置选项嵌套在 `rancher_kubernetes_engine_config` å‚数下。有关详细信æ¯ï¼Œè¯·å‚阅[集群é…置文件](#rke-集群é…置文件å‚考)。 + +在 [RKE å¯åŠ¨çš„集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中，你å¯ä»¥ç¼–辑任何åŽç»­å‰©ä½™çš„选项。 + +有关 RKE é…置文件语法的示例，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/example-yamls/)。 + +Rancher UI 中的表å•ä¸åŒ…括é…ç½® RKE 的所有高级选项。有关 YAML 中 RKE Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +## 在 Rancher UI 中使用表å•ç¼–辑集群 + +è¦ç¼–辑你的集群： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦é…置的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 + + +## 使用 YAML 编辑集群 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ ä½¿ç”¨ YAML æ¥æŒ‡å®š RKE 安装中å¯ç”¨çš„任何选项（除了 system_images é…置）。 + +RKE 集群（也称为 RKE1 集群）的编辑方å¼ä¸Ž RKE2 å’Œ K3s 集群ä¸åŒã€‚ + +è¦ç›´æŽ¥ä»Ž Rancher UI 编辑 RKE é…置文件： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦é…置的 RKE 集群。å•å‡»å¹¶å•å‡» **â‹® > 编辑é…ç½®**。你将会转到 RKE é…置表å•ã€‚请注æ„，由于集群é…置在 Rancher 2.6 中å‘生了å˜æ›´ï¼Œ**â‹® > 以 YAML 文件编辑**å¯ç”¨äºŽé…ç½® RKE2 集群，但ä¸èƒ½ç”¨äºŽç¼–辑 RKE1 é…置。 +1. 在é…置表å•ä¸­ï¼Œå‘下滚动并å•å‡»**以 YAML 文件编辑**。 +1. 编辑 `rancher_kubernetes_engine_config` å‚数下的 RKE 选项。 + +## Rancher UI 中的é…置选项 + +:::tip + +一些高级é…置选项没有在 Rancher UI 表å•ä¸­å¼€æ”¾ï¼Œä½†ä½ å¯ä»¥é€šè¿‡åœ¨ YAML 中编辑 RKE 集群é…置文件æ¥å¯ç”¨è¿™äº›é€‰é¡¹ã€‚有关 YAML 中 RKE Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +::: + +### Kubernetes 版本 + +这指的是集群节点上安装的 Kubernetes 版本。Rancher 基于 [hyperkube](https://github.com/rancher/hyperkube) 打包了自己的 Kubernetes 版本。 + +有关更多详细信æ¯ï¼Œè¯·å‚阅[å‡çº§ Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 + +### 网络æ供商 + +这指的是集群使用的[网络æ供商](https://kubernetes.io/docs/concepts/cluster-administration/networking/)。有关ä¸åŒç½‘络æ供商的更多详细信æ¯ï¼Œè¯·æŸ¥çœ‹æˆ‘们的[网络常è§é—®é¢˜è§£ç­”](../../../faq/container-network-interface-providers.md)。 + +:::caution + +å¯åŠ¨é›†ç¾¤åŽï¼Œä½ æ— æ³•æ›´æ”¹ç½‘络æ供商。由于 Kubernetes ä¸å…许在网络æ供商之间切æ¢ï¼Œå› æ­¤ï¼Œè¯·è°¨æ…Žé€‰æ‹©è¦ä½¿ç”¨çš„网络æ供商。使用网络æ供商创建集群åŽï¼Œå¦‚果你需è¦æ›´æ”¹ç½‘络æ供商，你将需è¦æ‹†é™¤æ•´ä¸ªé›†ç¾¤ä»¥åŠå…¶ä¸­çš„所有应用。 + +::: + +Rancher 与以下开箱å³ç”¨çš„网络æ供商兼容： + +- [Canal](https://github.com/projectcalico/canal) +- [Flannel](https://github.com/coreos/flannel#flannel) +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Weave](https://github.com/weaveworks/weave) + +:::note Weave 注æ„事项： + +选择 Weave 作为网络æ供商时，Rancher 将通过生æˆéšæœºå¯†ç æ¥è‡ªåŠ¨å¯ç”¨åŠ å¯†ã€‚如果你想手动指定密ç ï¼Œè¯·å‚阅使用[é…置文件](#rke-集群é…置文件å‚考)å’Œ [Weave 网络æ’件选项](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/#weave-network-plug-in-options)æ¥é…置集群。 + +::: + +### 项目网络隔离 + +如果你的网络æ供商å…许项目网络隔离，你å¯ä»¥é€‰æ‹©å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´çš„通信。 + +如果你使用支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE 网络æ’件（例如 Canal 或 Cisco ACI æ’件），则å¯ä»¥ä½¿ç”¨é¡¹ç›®ç½‘络隔离。 + +### Kubernetes 云æ供商 + +ä½ å¯ä»¥é…ç½® [Kubernetes 云æ供商](../../../pages-for-subheaders/set-up-cloud-providers.md)。如果你想在 Kubernetes 中使用动æ€é…置的[å·å’Œå­˜å‚¨](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md)，你通常需è¦é€‰æ‹©ç‰¹å®šçš„云æ供商。例如，如果你想使用 Amazon EBS，则需è¦é€‰æ‹© `aws` 云æ供商。 + +:::note + +如果你è¦ä½¿ç”¨çš„云æ供商未作为选项列出，你需è¦ä½¿ç”¨[é…置文件选项](#rke-集群é…置文件å‚考)æ¥é…置云æ供商。请å‚考 [RKE 云æ供商文档](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/)æ¥äº†è§£å¦‚何é…置云æ供商。 + +::: + +### ç§æœ‰é•œåƒä»“库 + +集群级别的ç§æœ‰é•œåƒä»“库é…置仅能用于é…置集群。 + +在 Rancher 中设置ç§æœ‰é•œåƒä»“库的主è¦æ–¹æ³•æœ‰ä¸¤ç§ï¼šé€šè¿‡[全局默认镜åƒä»“库](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md)中的**设置**选项å¡è®¾ç½®å…¨å±€é»˜è®¤é•œåƒä»“库，以åŠåœ¨é›†ç¾¤çº§åˆ«è®¾ç½®çš„高级选项中设置ç§æœ‰é•œåƒä»“库。全局默认镜åƒä»“库å¯ä»¥ç”¨äºŽç¦»çº¿è®¾ç½®ï¼Œä¸éœ€è¦å‡­è¯çš„é•œåƒä»“库。而集群级ç§æœ‰é•œåƒä»“库用于所有需è¦å‡­è¯çš„ç§æœ‰é•œåƒä»“库。 + +如果你的ç§æœ‰é•œåƒä»“库需è¦å‡­è¯ï¼Œä¸ºäº†å°†å‡­è¯ä¼ é€’ç»™ Rancher，你需è¦ç¼–辑æ¯ä¸ªéœ€è¦ä»Žä»“库中拉å–é•œåƒçš„集群的集群选项。 + +ç§æœ‰é•œåƒä»“库的é…置选项能让 Rancher 知é“è¦ä»Žå“ªé‡Œæ‹‰å–用于集群的[系统镜åƒ](https://rancher.com/docs/rke/latest/en/config-options/system-images/)或[附加组件镜åƒ](https://rancher.com/docs/rke/latest/en/config-options/add-ons/)。 + +- **系统镜åƒ**是维护 Kubernetes 集群所需的组件。 +- **附加组件**用于部署多个集群组件，包括网络æ’件ã€ingress controllerã€DNS æ供商或 metrics server。 + +有关为集群é…置期间应用的组件设置ç§æœ‰é•œåƒä»“库的更多信æ¯ï¼Œè¯·å‚阅[ç§æœ‰é•œåƒä»“库的 RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/private-registries/)。 + +Rancher v2.6 引入了[为 RKE 集群é…ç½® ECR é•œåƒä»“库](https://rancher.com/docs/rke/latest/en/config-options/private-registries/#amazon-elastic-container-registry-ecr-private-registry-setup)的功能。 + +### 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯ç”¨äºŽç›´æŽ¥è®¿é—® Kubernetes API server，而无需通过 Rancher 进行通信。 + +:::note + +授æƒé›†ç¾¤ç«¯ç‚¹ä»…适用于 Rancher å¯åŠ¨çš„ Kubernetes 集群，å³åªé€‚用于 Rancher [使用 RKE](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#é…ç½®-kubernetes-集群的工具) æ¥é…置的集群。它ä¸é€‚用于托管在 Kubernetes æ供商中的集群，例如 Amazon çš„ EKS。 + +::: + +在 Rancher å¯åŠ¨çš„ Kubernetes 集群中，它默认å¯ç”¨ï¼Œä½¿ç”¨å…·æœ‰ `controlplane` 角色的节点的 IP 和默认的 Kubernetes 自签åè¯ä¹¦ã€‚ + +有关授æƒé›†ç¾¤ç«¯ç‚¹çš„工作原ç†ä»¥åŠä½¿ç”¨çš„原因，请å‚阅[架构介ç»](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +### 节点池 + +有关使用 Rancher UI 在 RKE 集群中设置节点池的信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 + +### NGINX Ingress + +如果你想使用高å¯ç”¨æ€§é…ç½®æ¥å‘布应用，并且你使用没有原生负载å‡è¡¡åŠŸèƒ½çš„云æ供商æ¥æ‰˜ç®¡ä¸»æœºï¼Œè¯·å¯ç”¨æ­¤é€‰é¡¹ä»¥åœ¨é›†ç¾¤ä¸­ä½¿ç”¨ NGINX Ingress。 + +### Metrics Server 监控 + +这是å¯ç”¨æˆ–ç¦ç”¨ [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/) 的选项。 + +æ¯ä¸ªèƒ½å¤Ÿä½¿ç”¨ RKE å¯åŠ¨é›†ç¾¤çš„云æ供商都å¯ä»¥æ”¶é›†æŒ‡æ ‡å¹¶ç›‘控你的集群节点。如果å¯ç”¨æ­¤é€‰é¡¹ï¼Œä½ å¯ä»¥ä»Žä½ çš„云æ供商门户查看你的节点指标。 + +### Pod å®‰å…¨ç­–ç•¥æ”¯æŒ + +为集群å¯ç”¨ [pod 安全策略](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。å¯ç”¨æ­¤é€‰é¡¹åŽï¼Œä½¿ç”¨**默认 Pod 安全策略**下拉èœå•é€‰æ‹©ä¸€ä¸ªç­–略。 + +你必须有已é…置的 Pod 安全策略æ‰èƒ½ä½¿ç”¨æ­¤é€‰é¡¹ã€‚ + +### 节点上的 Docker 版本 + +表示是å¦å…许节点è¿è¡Œ Rancher ä¸æ­£å¼æ”¯æŒçš„ Docker 版本。 + +如果你选择使用支æŒçš„ Docker 版本，Rancher 会ç¦æ­¢ pod è¿è¡Œåœ¨å®‰è£…了ä¸æ”¯æŒçš„ Docker 版本的节点上。 + +如需了解å„个 Rancher 版本通过了哪些 Docker 版本测试，请å‚è§[支æŒå’Œç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。 + +### Docker 根目录 + +如果è¦æ·»åŠ åˆ°é›†ç¾¤çš„节点为 Docker é…置了éžé»˜è®¤ Docker 根目录（默认为 `/var/lib/docker`），请在此选项中指定正确的 Docker 根目录。 + +### 默认 Pod 安全策略 + +如果你å¯ç”¨äº† **Pod 安全策略支æŒ**，请使用此下拉èœå•é€‰æ‹©åº”用于集群的 pod 安全策略。 + +### 节点端å£èŒƒå›´ + +更改å¯ç”¨äºŽ [NodePort æœåŠ¡](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)的端å£èŒƒå›´çš„选项。默认为 `30000-32767`。 + +### 定期 etcd å¿«ç…§ + +å¯ç”¨æˆ–ç¦ç”¨[定期 etcd å¿«ç…§](https://rancher.com/docs/rke/latest/en/etcd-snapshots/#etcd-recurring-snapshots)的选项。 + +### Agent 环境å˜é‡ + +为 [rancher agent](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md) 设置环境å˜é‡çš„选项。你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚如果 Rancher Agent 需è¦ä½¿ç”¨ä»£ç†ä¸Ž Rancher Server 通信，则å¯ä»¥ä½¿ç”¨ Agent 环境å˜é‡è®¾ç½® `HTTP_PROXY`，`HTTPS_PROXY` å’Œ `NO_PROXY` 环境å˜é‡ã€‚ + +### æ›´æ–° ingress-nginx + +使用 Kubernetes 1.16 之å‰ç‰ˆæœ¬åˆ›å»ºçš„集群将具有 `OnDelete`çš„ `ingress-nginx` `updateStrategy`。使用 Kubernetes 1.16 或更高版本创建的集群将具有 `RollingUpdate`。 + +如果 `ingress-nginx` çš„ `updateStrategy` 是 `OnDelete`，则需è¦åˆ é™¤è¿™äº› pod 以获得 deployment 正确的版本。 + +### Cluster Agent é…置和 Fleet Agent é…ç½® + +ä½ å¯ä»¥ä¸º Cluster Agent 和集群的 Fleet Agent é…置调度字段和资æºé™åˆ¶ã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›å­—段æ¥è‡ªå®šä¹‰å®¹å¿åº¦ã€äº²å’Œæ€§è§„则和资æºè¦æ±‚。其他容å¿åº¦ä¼šè¢«å°¾é™„到默认容å¿åº¦å’Œ Control Plane 节点污点的列表中。如果你定义了自定义亲和性规则，它们将覆盖全局默认亲和性设置。定义资æºè¦æ±‚会在以å‰æ²¡æœ‰çš„地方设置请求或é™åˆ¶ã€‚ + +:::note + +有了这个选项，你å¯ä»¥è¦†ç›–或删除è¿è¡Œé›†ç¾¤æ‰€éœ€çš„规则。我们强烈建议你ä¸è¦åˆ é™¤æˆ–覆盖这些规则和其他亲和性规则，因为这å¯èƒ½ä¼šå¯¼è‡´ä¸å¿…è¦çš„å½±å“： + +- `affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution` 用于 `cattle-cluster-agent` +- `cluster-agent-default-affinity` 用于 `cattle-cluster-agent` +- `fleet-agent-default-affinity` 用于 `fleet-agent` + +::: + +如果将 Rancher é™çº§åˆ° v2.7.4 或更低版本，你的更改将丢失，而且 Agent 将在没有你的自定义设置的情况下é‡æ–°éƒ¨ç½²ã€‚é‡æ–°éƒ¨ç½²æ—¶ï¼ŒFleet Agent 将回退到使用内置默认值。如果é™çº§æœŸé—´ Fleet 版本没有更改，则ä¸ä¼šç«‹å³é‡æ–°éƒ¨ç½²ã€‚ + + +## RKE 集群é…置文件å‚考 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ åœ¨ RKE 安装中设置任何[å¯ç”¨é€‰é¡¹](https://rancher.com/docs/rke/latest/en/config-options/)（`system_images` é…置除外）。使用 Rancher UI 或 API 创建集群时，ä¸æ”¯æŒ `system_images` 选项。 + +有关 YAML 中 RKE Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +### Rancher 中的é…置文件结构 + +RKE（Rancher Kubernetes Engine）是 Rancher 用æ¥é…ç½® Kubernetes 集群的工具。过去，Rancher 的集群é…置文件与 [RKE é…置文件](https://rancher.com/docs/rke/latest/en/example-yamls/)的结构是一致的。但由于 Rancher 文件结构å‘生了å˜åŒ–，因此在 Rancher 中，RKE 集群é…ç½®é¡¹ä¸Žéž RKE é…置项是分开的。所以，你的集群é…置需è¦åµŒå¥—在集群é…置文件中的 `rancher_kubernetes_engine_config` å‚数下。使用早期版本的 Rancher 创建的集群é…置文件需è¦é’ˆå¯¹è¿™ç§æ ¼å¼è¿›è¡Œæ›´æ–°ã€‚以下是一个集群é…置文件示例： + +
+ 集群é…置文件示例 + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker +enable_cluster_alerting: false +enable_cluster_monitoring: false +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # ç›®å‰ä»…æ”¯æŒ Nginx ingress provider +# # è¦ç¦ç”¨ Ingress controller，设置 `provider: none` +# # è¦åœ¨æŒ‡å®šèŠ‚点上ç¦ç”¨ Ingress，使用 node_selector，例如： +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` +
+ +### 默认 DNS æ供商 + +下表显示了默认部署的 DNS æ供商。有关如何é…ç½®ä¸åŒ DNS æ供商的更多信æ¯ï¼Œè¯·å‚阅 [DNS æ供商相关的 RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/add-ons/dns/)。CoreDNS åªèƒ½åœ¨ Kubernetes v1.12.0 åŠæ›´é«˜ç‰ˆæœ¬ä¸Šä½¿ç”¨ã€‚ + +| Rancher 版本 | Kubernetes 版本 | 默认 DNS æ供商 | +|-------------|--------------------|----------------------| +| v2.2.5 åŠæ›´é«˜ç‰ˆæœ¬ | v1.14.0 åŠæ›´é«˜ç‰ˆæœ¬ | CoreDNS | +| v2.2.5 åŠæ›´é«˜ç‰ˆæœ¬ | v1.13.x åŠæ›´ä½Žç‰ˆæœ¬ | kube-dns | +| v2.2.4 åŠæ›´ä½Žç‰ˆæœ¬ | ä»»æ„ | kube-dns | + +## YAML 中的 Rancher 特定å‚æ•° + +除了 RKE é…置文件选项外，还有å¯ä»¥åœ¨é…置文件 (YAML) 中é…置的 Rancher 特定设置如下。 + +### docker_root_dir + +请å‚阅 [Docker 根目录](#docker-根目录)。 + +### enable_cluster_monitoring + +å¯ç”¨æˆ–ç¦ç”¨[集群监控](../../../pages-for-subheaders/monitoring-and-alerting.md)的选项。 + +### enable_network_policy + +å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®ç½‘络隔离的选项。 + +如果你使用支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE 网络æ’件（例如 Canal 或 Cisco ACI æ’件），则å¯ä»¥ä½¿ç”¨é¡¹ç›®ç½‘络隔离。 + +### local_cluster_auth_endpoint + +请å‚阅[授æƒé›†ç¾¤ç«¯ç‚¹](#授æƒé›†ç¾¤ç«¯ç‚¹)。 + +示例： + +```yaml +local_cluster_auth_endpoint: + enabled: true + fqdn: "FQDN" + ca_certs: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +### 自定义网络æ’件 + +ä½ å¯ä»¥ä½¿ç”¨ RKE çš„[用户定义的附加组件功能](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/)æ¥æ·»åŠ è‡ªå®šä¹‰ç½‘络æ’件。部署 Kubernetes 集群之åŽï¼Œä½ å¯ä»¥å®šä¹‰è¦éƒ¨ç½²çš„任何附加组件。 + +有两ç§æ–¹æ³•å¯ä»¥æŒ‡å®šé™„加组件： + +- [内嵌附加组件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#in-line-add-ons) +- [为附加组件引用 YAML 文件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#referencing-yaml-files-for-add-ons) + +有关如何通过编辑 `cluster.yml` æ¥é…置自定义网络æ’件的示例，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/custom-network-plugin-example)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md new file mode 100644 index 00000000000..3f4328512ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md @@ -0,0 +1,360 @@ +--- +title: RKE2 集群é…ç½®å‚考 +--- + +æœ¬æ–‡ä»‹ç» Rancher 中å¯ç”¨äºŽæ–°çš„或现有的 RKE2 Kubernetes 集群的é…置选项。 + +## 概述 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€æ¥é…ç½® Kubernetes 选项： + +- [Rancher UI](#rancher-ui-中的é…置选项)：使用 Rancher UI æ¥é€‰æ‹©è®¾ç½® Kubernetes 集群时常用的自定义选项。 +- [集群é…置文件](#集群é…置文件å‚考)：高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE2 é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件让你能设置更多å¯ç”¨äºŽ RKE2 的其他[安装选项](https://docs.rke2.io/install/configuration)。 + +## 在 Rancher UI 中使用表å•ç¼–辑集群 + +è¦ç¼–辑你的集群： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦é…置的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 + +## 使用 YAML 编辑集群 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE2 é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ ä½¿ç”¨ YAML æ¥æŒ‡å®š RKE2 安装中å¯ç”¨çš„任何选项。 + +è¦ç›´æŽ¥ä»Ž Rancher UI 中编辑 RKE2 é…置文件： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到你è¦é…置的集群，然åŽå•å‡» **â‹® > 以 YAML 文件编辑**。 +1. 编辑 `rkeConfig` å‚数下的 RKE 选项。 + +## Rancher UI 中的é…置选项 + +:::tip + +一些高级é…置选项没有在 Rancher UI 表å•ä¸­å¼€æ”¾ï¼Œä½†ä½ å¯ä»¥é€šè¿‡åœ¨ YAML 中编辑 RKE2 集群é…置文件æ¥å¯ç”¨è¿™äº›é€‰é¡¹ã€‚有关 YAML 中 RKE2 Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE2 文档](https://docs.rke2.io/install/configuration)。 + +::: + +## 主机池 + +本å°èŠ‚介ç»äº†é€šç”¨çš„主机池é…置。对于针对特定基础设施æ供商的é…置，请å‚阅以下页é¢ï¼š + +- [Azure](../downstream-cluster-configuration/machine-configuration/azure.md) +- [DigitalOcean](../downstream-cluster-configuration/machine-configuration/digitalocean.md) +- [EC2](../downstream-cluster-configuration/machine-configuration/amazon-ec2.md) + +### æ± å称 + +主机池的å称。 + +### ä¸»æœºæ•°é‡ + +池中的主机数。 + +### 角色 + +å°† etcdã€controlplane å’Œ worker 角色分é…给节点的选项。 + +### 高级é…ç½® + +#### è‡ªåŠ¨æ›¿æ¢ + +如果节点无法访问的æŒç»­æ—¶é—´è¾¾åˆ°è¯¥å€¼ï¼Œåˆ™ä¼šè¢«è‡ªåŠ¨åˆ é™¤å’Œæ›¿æ¢ã€‚ + +#### 删除å‰æ¸…空 + +通过在删除节点之å‰é©±é€æ‰€æœ‰ pod æ¥æ¸…空节点。 + +#### Kubernetes 节点标签 + +å°†[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)（label）添加到节点，让对象选择更加简便。 + +有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 + +#### 污点 + +å°†[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)（taint）添加到节点，防止 pod è¢«è°ƒåº¦æˆ–åœ¨èŠ‚ç‚¹ä¸Šæ‰§è¡Œï¼ˆé™¤éž pod 具有匹é…的容å¿åº¦ï¼‰ã€‚ + +## 集群é…ç½® + +### åŸºæœ¬ä¿¡æ¯ +#### Kubernetes 版本 + +这指的是集群节点上安装的 Kubernetes 版本。Rancher 基于 [hyperkube](https://github.com/rancher/hyperkube) 打包了自己的 Kubernetes 版本。 + +有关更多详细信æ¯ï¼Œè¯·å‚阅[å‡çº§ Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 + +#### 容器网络æ供商 + +这指的是集群使用的[网络æ供商](https://kubernetes.io/docs/concepts/cluster-administration/networking/)。 + +:::caution + +å¯åŠ¨é›†ç¾¤åŽï¼Œä½ æ— æ³•æ›´æ”¹ç½‘络æ供商。由于 Kubernetes ä¸å…许在网络æ供商之间切æ¢ï¼Œå› æ­¤ï¼Œè¯·è°¨æ…Žé€‰æ‹©è¦ä½¿ç”¨çš„网络æ供商。使用网络æ供商创建集群åŽï¼Œå¦‚果你需è¦æ›´æ”¹ç½‘络æ供商，你将需è¦æ‹†é™¤æ•´ä¸ªé›†ç¾¤ä»¥åŠå…¶ä¸­çš„所有应用。 + +::: + +Rancher 与以下开箱å³ç”¨çš„网络æ供商兼容： + +- [Canal](https://github.com/projectcalico/canal) +- [Cilium](https://cilium.io/)\* +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Multus](https://github.com/k8snetworkplumbingwg/multus-cni) + +\* 在 [Cilium CNI](../../../faq/container-network-interface-providers.md#cilium) 中使用[项目网络隔离](#项目网络隔离)时，你å¯ä»¥å¼€å¯è·¨èŠ‚点入å£è·¯ç”±ã€‚详情请å‚è§ [CNI æ供商文档](../../../faq/container-network-interface-providers.md#cilium-中跨节点的-ingress-路由)。 + +有关ä¸åŒç½‘络æ供商以åŠå¦‚何é…置它们的更多详细信æ¯ï¼Œè¯·æŸ¥é˜… [RKE2 文档](https://docs.rke2.io/install/network_options)。 + +##### åŒæ ˆç½‘络 + +所有 CNI 网络æ’件都支æŒ[åŒæ ˆ](https://docs.rke2.io/install/network_options#dual-stack-configuration)网络。è¦åœ¨åŒæ ˆæ¨¡å¼ä¸‹é…ç½® RKE2，请为你的[集群 CIDR](#集群-cidr) å’Œ/或 [Service CIDR](#service-cidr) 设置有效的 IPv4/IPv6 CIDR。 + +###### é¢å¤–é…ç½® {#dual-stack-additional-config} + +使用 `cilium` 或 `multus,cilium` 作为容器网络接å£æ供商时，请确ä¿**å¯ç”¨ IPv6 支æŒ**选项。 + +#### 云æ供商 + +ä½ å¯ä»¥é…ç½® [Kubernetes 云æ供商](../../../pages-for-subheaders/set-up-cloud-providers.md)。如果你想在 Kubernetes 中使用动æ€é…置的[å·å’Œå­˜å‚¨](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md)，你通常需è¦é€‰æ‹©ç‰¹å®šçš„云æ供商。例如，如果你想使用 Amazon EBS，则需è¦é€‰æ‹© `aws` 云æ供商。 + +:::note + +如果你è¦ä½¿ç”¨çš„云æ供商未作为选项列出，你需è¦ä½¿ç”¨[é…置文件选项](#集群é…置文件å‚考)æ¥é…置云æ供商。请å‚考[本文档](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/)æ¥äº†è§£å¦‚何é…置云æ供商。 + +::: + +#### 默认 Pod 安全策略 + +为集群选择默认的 [pod 安全策略](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。请å‚阅 [RKE2 文档](https://docs.rke2.io/security/pod_security_policies)æ¥äº†è§£æ¯ä¸ªå¯ç”¨ç­–略的规范。 + +#### Worker CIS é…置文件 + +选择一个 [CIS benchmark](../../../pages-for-subheaders/cis-scan-guides.md) æ¥éªŒè¯ç³»ç»Ÿé…置。 + +#### 项目网络隔离 + +如果你的网络æ供商å…许项目网络隔离，你å¯ä»¥é€‰æ‹©å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´çš„通信。 + +如果你使用支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE2 网络æ’件（例如 Canal），则å¯ä»¥ä½¿ç”¨é¡¹ç›®ç½‘络隔离。 + +#### CoreDNS + +默认情况下，[CoreDNS](https://coredns.io/) 会安装为默认 DNS æ供程åºã€‚如果未安装 CoreDNS，则必须自己安装备用 DNS æ供程åºã€‚有关其他 CoreDNS é…置，请å‚阅 [RKE2 文档](https://docs.rke2.io/networking#coredns)。 + +#### NGINX Ingress + +如果你想使用高å¯ç”¨æ€§é…ç½®æ¥å‘布应用，并且你使用没有原生负载å‡è¡¡åŠŸèƒ½çš„云æ供商æ¥æ‰˜ç®¡ä¸»æœºï¼Œè¯·å¯ç”¨æ­¤é€‰é¡¹ä»¥åœ¨é›†ç¾¤ä¸­ä½¿ç”¨ NGINX Ingress。有关其他é…置选项，请å‚阅 [RKE2 文档](https://docs.rke2.io/networking#nginx-ingress-controller)。 + +有关其他é…置选项，请å‚阅 [RKE2 文档](https://docs.rke2.io/networking#nginx-ingress-controller)。 + +#### Metrics Server + +这是å¯ç”¨æˆ–ç¦ç”¨ [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/) 的选项。 + +æ¯ä¸ªèƒ½å¤Ÿä½¿ç”¨ RKE2 å¯åŠ¨é›†ç¾¤çš„云æ供商都å¯ä»¥æ”¶é›†æŒ‡æ ‡å¹¶ç›‘控你的集群节点。如果å¯ç”¨æ­¤é€‰é¡¹ï¼Œä½ å¯ä»¥ä»Žä½ çš„云æ供商门户查看你的节点指标。 + +### 附加é…ç½® + +集群å¯åŠ¨æ—¶å°†åº”用的其他 Kubernetes 清å•ï¼Œä¼šä½œä¸º[附加组件](https://kubernetes.io/docs/concepts/cluster-administration/addons/)æ¥ç®¡ç†ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/helm#automatically-deploying-manifests-and-helm-charts)。 + +### Agent 环境å˜é‡ + +为 [Rancher agent](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md) 设置环境å˜é‡çš„选项。你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/reference/linux_agent_config)。 + +### etcd + +#### 自动快照 + +å¯ç”¨æˆ–ç¦ç”¨å®šæœŸ etcd 快照的选项。如果å¯ç”¨ï¼Œç”¨æˆ·å¯ä»¥é…置快照的频率。有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/backup_restore#creating-snapshots)。请注æ„，如果使用 RKE2，快照会存储在æ¯ä¸ª etcd 节点上，这与 RKE1 ä¸åŒï¼ˆRKE1 æ¯ä¸ªé›†ç¾¤åªå­˜å‚¨ä¸€ä¸ªå¿«ç…§ï¼‰ã€‚ + +#### 指标 + +选择å‘公众公开或仅在集群内公开 etcd 指标的选项。 + +### 网络 + +#### 集群 CIDR + +用于 pod IP çš„ IPv4 å’Œ/或 IPv6 网络 CIDR（默认：10.42.0.0/16）。 + +##### åŒæ ˆç½‘络 + +è¦é…ç½®[åŒæ ˆ](https://docs.rke2.io/install/network_options#dual-stack-configuration)模å¼ï¼Œè¯·è¾“入有效的 IPv4/IPv6 CIDR。例如 `10.42.0.0/16,2001:cafe:42:0::/56`。 + +使用 `cilium` 或 `multus,cilium` 作为[容器网络](#容器网络æ供商)接å£æ供商时，你需è¦è¿›è¡Œ[附加é…ç½®](#dual-stack-additional-config)。 + +#### Service CIDR + +用于 Service IP çš„ IPv4/IPv6 网络 CIDR（默认：10.43.0.0/16）。 + +##### åŒæ ˆç½‘络 + +è¦é…ç½®[åŒæ ˆ](https://docs.rke2.io/install/network_options#dual-stack-configuration)模å¼ï¼Œè¯·è¾“入有效的 IPv4/IPv6 CIDR。例如 `10.42.0.0/16,2001:cafe:42:0::/56`。 + +使用 `cilium` 或 `multus,cilium` 作为[容器网络](#容器网络æ供商)接å£æ供商时，你需è¦è¿›è¡Œ[附加é…ç½®](#dual-stack-additional-config)。 + +#### 集群 DNS + +用于 coredns æœåŠ¡çš„ IPv4 集群 IP。应该在你的 service-cidr 范围内（默认：10.43.0.10）。 + +#### 集群域å + +选择集群的域。默认值为 `cluster.local`。 + +#### NodePort æœåŠ¡ç«¯å£èŒƒå›´ + +更改å¯ç”¨äºŽ [NodePort æœåŠ¡](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)的端å£èŒƒå›´çš„选项。默认值为 `30000-32767`。 + +#### 截断主机å + +将主机å截断为 15 个或更少字符。该字段åªèƒ½åœ¨é›†ç¾¤åˆå§‹åˆ›å»ºæ—¶è®¾ç½®ã€‚创建集群åŽï¼Œä½ æ— æ³•å†å¯ç”¨æˆ–ç¦ç”¨ 15 个字符的é™åˆ¶ã€‚ + +此设置仅影å“é…置了机器的集群。由于自定义集群在创建节点时设置主机å（å‘生在 Rancher 之外），因此该字段ä¸é™åˆ¶è‡ªå®šä¹‰é›†ç¾¤çš„主机å长度。 + +截断集群中的主机åå¯æ高与基于 Windows 的系统的兼容性。虽然 Kubernetes å…许最长的主机å长度为 63 个字符，但使用 NetBIOS 的系统将主机åé™åˆ¶ä¸º 15 个字符之内。 + +#### TLS å¯é€‰å称 + +在æœåŠ¡å™¨ TLS è¯ä¹¦ä¸Šæ·»åŠ å…¶ä»–主机å或 IPv4/IPv6 地å€ä½œä¸º Subject Alternative Name。 + +#### 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯ç”¨äºŽç›´æŽ¥è®¿é—® Kubernetes API server，而无需通过 Rancher 进行通信。 + +在 Rancher å¯åŠ¨çš„ Kubernetes 集群中，它默认å¯ç”¨ï¼Œä½¿ç”¨å…·æœ‰ `controlplane` 角色的节点的 IP 和默认的 Kubernetes 自签åè¯ä¹¦ã€‚ + +有关授æƒé›†ç¾¤ç«¯ç‚¹çš„工作原ç†ä»¥åŠä½¿ç”¨çš„原因，请å‚阅[架构介ç»](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +### é•œåƒä»“库 + +选择è¦ä»Žä¸­æ‹‰å– Rancher é•œåƒçš„é•œåƒä»“库。有关更多详细信æ¯å’Œé…置选项，请å‚阅 [RKE2 文档](https://docs.rke2.io/install/containerd_registry_configuration)。 + +### å‡çº§ç­–ç•¥ + +#### controlplane å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### Worker å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### 清空节点（controlplane） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +#### 清空节点（worker 节点） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +### 高级é…ç½® + +为ä¸åŒèŠ‚点设置 kubelet 选项。有关å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)。 + +## 集群é…置文件å‚考 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ªé…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å…许你为 RKE2 设置[å¯ç”¨çš„选项](https://docs.rke2.io/install/configuration)，其中包括已ç»åœ¨ [Rancher UI é…置选项](#rancher-ui-中的é…置选项)ä¸­åˆ—å‡ºçš„é€‰é¡¹ä»¥åŠ Rancher 特定的å‚数。 + +
+ + 集群é…置文件片段示例 + + +```yaml +spec: + cloudCredentialSecretName: cattle-global-data:cc-s879v + kubernetesVersion: v1.23.6+rke2r2 + localClusterAuthEndpoint: {} + rkeConfig: + chartValues: + rke2-calico: {} + etcd: + snapshotRetention: 5 + snapshotScheduleCron: 0 */5 * * * + machineGlobalConfig: + cni: calico + disable-kube-proxy: false + etcd-expose-metrics: false + profile: null + machinePools: + - controlPlaneRole: true + etcdRole: true + machineConfigRef: + kind: Amazonec2Config + name: nc-test-pool1-pwl5h + name: pool1 + quantity: 1 + unhealthyNodeTimeout: 0s + workerRole: true + machineSelectorConfig: + - config: + protect-kernel-defaults: false + registries: {} + upgradeStrategy: + controlPlaneConcurrency: "1" + controlPlaneDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 + workerConcurrency: "1" + workerDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 +``` +
+ +### chartValues + +此选项用于为 RKE2/K3s 安装的 System Chart 指定值。 + +示例： + +```yaml +chartValues: + chart-name: + key: value +``` +### machineGlobalConfig + +RKE2/K3s é…置嵌套在 `machineGlobalConfig` å‚数下。在这里所åšçš„任何é…置更改都将应用到æ¯ä¸ªèŠ‚点。你å¯ä»¥åœ¨æ­¤å¤„应用[RKE2 的独立版本](https://docs.rke2.io/reference/server_config)中å¯ç”¨çš„é…置选项。 + +示例： + +```yaml +machineGlobalConfig: + etcd-arg: + - key1=value1 + - key2=value2 +``` + +### machineSelectorConfig + +æ­¤å‚数与 [`machineGlobalConfig`](#machineglobalconfig) 相åŒï¼Œåªæ˜¯å¯ä»¥åœ¨é…置中指定 [label](#kubernetes-节点标签) 选择器。该é…置仅应用于与标签选择器匹é…的节点。 + +å…许多个 `config` æ¡ç›®ï¼Œå¯ä»¥ä¸ºæ¯ä¸ªæ¡ç›®æŒ‡å®šå„自的 `machineLabelSelector`。用户å¯ä»¥æŒ‡å®š `matchExpressions`ã€`matchLabels`ã€æŒ‡å®šäºŒè€…或都ä¸æŒ‡å®šã€‚如果你çœç•¥äº† `machineLabelSelector`，则与将 config 放入 `machineGlobalConfig` 的效果相åŒã€‚ + +示例： + +```yaml +machineSelectorConfig + - config: + config-key: config-value + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # 有效的è¿ç®—符：Inã€NotInã€Exists å’Œ DoesNotExist + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md new file mode 100644 index 00000000000..823aaa2de81 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -0,0 +1,41 @@ +--- +title: åŒæ­¥ +--- + +åŒæ­¥å…许 Rancher 更新集群值，以便与托管在 AKSã€EKS 或 GKE 中的集群对象ä¿æŒåŒæ­¥ã€‚这使得 Rancher 以外的æ¥æºèƒ½å¤ŸèŽ·å–托管集群的状æ€ã€‚这是 UI 中显示的内容。 + +:::caution +如果你åŒæ—¶å¤„ç†æ¥è‡ªå¦ä¸€ä¸ªæ¥æºçš„更新，你å¯èƒ½ä¼šä¸å°å¿ƒè¦†ç›–一个æ¥æºçš„状æ€ã€‚如果你在完æˆä¸€ä¸ªæ¥æºçš„æ›´æ–°åŽ 5 分钟内处ç†å¦ä¸€ä¸ªæ¥æºçš„更新，也å¯èƒ½ä¼šå‘生这ç§æƒ…况。 +::: + +### å·¥ä½œåŽŸç† + +è¦ç†è§£åŒæ­¥æ˜¯å¦‚何工作的，则必须ç†è§£ Rancher Cluster 对象上的两个字段： + +1. 集群的 config 对象，ä½äºŽé›†ç¾¤çš„规范上： + + * 对于 AKS，该字段称为 AKSConfig + * 对于 EKS，该字段称为 EKSConfig + * 对于 GKE，该字段称为 GKEConfig + +2. UpstreamSpec 对象 + + * 对于 AKS，它ä½äºŽç¾¤é›†çŠ¶æ€çš„ AKSStatus 字段中。 + * 对于 EKS，它ä½äºŽé›†ç¾¤çŠ¶æ€çš„ EKSStatus 字段中。 + * 对于 GKE，它ä½äºŽé›†ç¾¤çŠ¶æ€çš„ GKEStatus 字段中。 + +定义这些对象的结构类型å¯ä»¥åœ¨å®ƒä»¬å¯¹åº”çš„ operator 项目中找到： + +* [aks-operator](https://github.com/rancher/aks-operator/blob/master/pkg/apis/aks.cattle.io/v1/types.go) +* [eks-operator](https://github.com/rancher/eks-operator/blob/master/pkg/apis/eks.cattle.io/v1/types.go) +* [gke-operator](https://github.com/rancher/gke-operator/blob/master/pkg/apis/gke.cattle.io/v1/types.go) + +除集群å称ã€ä½ç½®ï¼ˆåŒºåŸŸæˆ–地区）ã€å¯¼å…¥å’Œäº‘凭è¯å¼•ç”¨å¤–，所有字段å‡å¯ä¸ºç©ºã€‚ + +AKSConfigã€EKSConfig 或 GKEConfig 表示所需的状æ€ã€‚零值会被忽略。é…置对象中éžé›¶çš„字段å¯ä»¥è¢«è®¤ä¸ºæ˜¯â€œç®¡ç†çš„â€ã€‚在 Rancher 中创建集群时，所有字段都是éžé›¶çš„，因此都是“管ç†â€çš„。在把一个已存在的集群注册到 Rancher 时，所有å¯ä¸ºç©ºå­—段都是 nil 并且ä¸æ˜¯â€œç®¡ç†â€çš„。一旦 Rancher 更改了这些字段的值，这些字段就会被管ç†ã€‚ + +UpstreamSpec 代表集群在托管的 Kubernetes æä¾›å•†ä¸­çš„æƒ…å†µã€‚å®ƒæ¯ 5 分钟刷新一次。刷新 UpstreamSpec åŽï¼ŒRancher 会检查集群是å¦æ­£åœ¨è¿›è¡Œæ›´æ–°ã€‚如果它正在更新，则ä¸åšä»»ä½•è¿›ä¸€æ­¥å¤„ç†ã€‚如果它目å‰æ²¡æœ‰æ›´æ–°ï¼ŒAKSConfigã€EKSConfig 或 GKEConfig 上的任何 "管ç†" 字段都会被最近更新的 UpstreamSpec 上的相应值覆盖。 + +有效的期望状æ€å¯ä»¥è¢«è®¤ä¸ºæ˜¯ UpstreamSpec，加上 AKSConfigã€EKSConfig 或 GKEConfig 中的所有éžé›¶å­—段。这是 UI 中显示的内容。 + +如果 Rancher å’Œå¦ä¸€ä¸ªæ¥æºè¯•å›¾åœ¨åŒä¸€æ—¶é—´æˆ–在更新完æˆåŽçš„ 5 分钟å°è¯•æ›´æ–°é›†ç¾¤ï¼Œä»»ä½•ç®¡ç†å­—段都å¯èƒ½é™·å…¥ç«žäº‰çŠ¶æ€ã€‚以 EKS 为例，集群å¯èƒ½å°† PrivateAccess 作为管ç†å­—段。如果 PrivateAccess 为 false，在 11:01 在 EKS 控制å°ä¸­å¯ç”¨ï¼Œç„¶åŽ Rancher 在 11:05 之å‰æ›´æ–°æ ‡ç­¾ï¼Œé‚£ä¹ˆè¯¥å€¼å¾ˆå¯èƒ½è¢«è¦†ç›–。如果在集群处ç†æ›´æ–°æ—¶æ›´æ–°äº†æ ‡ç­¾ï¼Œä¹Ÿä¼šå‘生这ç§æƒ…况。如果集群已注册并且 PrivateAccess 字段为 nil，则ä¸åº”å‘生此示例中æ述的问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md new file mode 100644 index 00000000000..c301e408d1f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -0,0 +1,53 @@ +--- +title: Rancher Agent 选项 +--- + +Rancher 在æ¯ä¸ªèŠ‚点上部署一个 Agent æ¥ä¸ŽèŠ‚点通信。本文æ述了å¯ä»¥ä¼ é€’ç»™ Agent 的选项。è¦ä½¿ç”¨è¿™äº›é€‰é¡¹ï¼Œä½ éœ€è¦[使用自定义节点创建集群](../../../../pages-for-subheaders/use-existing-nodes.md)，并在添加节点时将选项添加到生æˆçš„ `docker run` 命令。 + +有关 Rancher 如何使用 Node Agent 与下游集群通信的概述，请å‚阅[产å“架构](../../../rancher-manager-architecture/communicating-with-downstream-user-clusters.md#3-node-agents)。 + +## 通用选项 + +| å‚æ•° | 环境å˜é‡ | æè¿° | +| ---------- | -------------------- | ----------- | +| `--server` | `CATTLE_SERVER` | å·²é…置的 Rancher `server-url`，Agent 将通过这个地å€è¿žæŽ¥ Server。 | +| `--token` | `CATTLE_TOKEN` | 在 Rancher 中注册节点所需的 Token。 | +| `--ca-checksum` | `CATTLE_CA_CHECKSUM` | 使用已é…置的 Rancher `cacerts` 进行 SHA256 校验和验è¯ã€‚ | +| `--node-name` | `CATTLE_NODE_NAME` | 覆盖用于注册节点的主机å（默认为 `hostname -s`）。 | +| `--label` | `CATTLE_NODE_LABEL` | å‘节点添加节点标签。对于多个标签，请传递é¢å¤–çš„ `--label` 选项。(`--label key=value`) | +| `--taints` | `CATTLE_NODE_TAINTS` | 将节点污点添加到节点。对于多个污点，请传递é¢å¤–çš„ `--taints` 选项。(`--taints key=value:effect`) | + +## 角色选项 + +| å‚æ•° | 环境å˜é‡ | æè¿° | +| ---------- | -------------------- | ----------- | +| `--all-roles` | `ALL=true` | 将所有角色（`etcd`ã€`controlplane`ã€`worker`）应用到节点。 | +| `--etcd` | `ETCD=true` | 将角色 `etcd` 应用到节点。 | +| `--controlplane` | `CONTROL=true` | 将角色 `controlplane` 应用到节点。 | +| `--worker` | `WORKER=true` | 将角色 `worker` 应用到节点。 | + +## IP 地å€é€‰é¡¹ + +| å‚æ•° | 环境å˜é‡ | æè¿° | +| ---------- | -------------------- | ----------- | +| `--address` | `CATTLE_ADDRESS` | 节点将注册的 IP 地å€ï¼ˆé»˜è®¤ä¸ºç”¨äºŽè¿žæŽ¥ `8.8.8.8`çš„ IP）。 | +| `--internal-address` | `CATTLE_INTERNAL_ADDRESS` | ç§æœ‰ç½‘络上用于主机间通信的 IP 地å€ã€‚ | + +### åŠ¨æ€ IP 地å€é€‰é¡¹ + +出于自动化目的，你ä¸èƒ½åœ¨å‘½ä»¤ä¸­ä½¿ç”¨ç‰¹å®šçš„ IP 地å€ï¼Œå› ä¸ºå®ƒå¿…须是通用的æ‰èƒ½ç”¨äºŽæ¯ä¸ªèŠ‚点。为此，我们æä¾›äº†åŠ¨æ€ IP 地å€çš„选项。它们用作现有 IP 地å€é€‰é¡¹çš„å€¼ã€‚æ”¯æŒ `--address` å’Œ `--internal-address`。 + +| 值 | 示例 | æè¿° | +| ---------- | -------------------- | ----------- | +| 接å£å称 | `--address eth0` | 将从给定的接å£ä¸­æ£€ç´¢ç¬¬ä¸€ä¸ªé…置的 IP 地å€ã€‚ | +| `ipify` | `--address ipify` | 将使用从 `https://api.ipify.org` 检索到的值。 | +| `awslocal` | `--address awslocal` | 将使用从 `http://169.254.169.254/latest/meta-data/local-ipv4` 检索到的值。 | +| `awspublic` | `--address awspublic` | 将使用从 `http://169.254.169.254/latest/meta-data/public-ipv4` 检索到的值。 | +| `doprivate` | `--address doprivate` | 将使用从 `http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address` 检索到的值。 | +| `dopublic` | `--address dopublic` | 将使用从 `http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address` 检索到的值。 | +| `azprivate` | `--address azprivate` | 将使用从 `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text` 检索到的值。 | +| `azpublic` | `--address azpublic` | 将使用从 `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-08-01&format=text` 检索到的值。 | +| `gceinternal` | `--address gceinternal` | 将使用从 `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip` 检索到的值。 | +| `gceexternal` | `--address gceexternal` | 将使用从 `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip` 检索到的值。 | +| `packetlocal` | `--address packetlocal` | 将使用从 `https://metadata.packet.net/2009-04-04/meta-data/local-ipv4` 检索到的值。 | +| `packetpublic` | `--address packetlocal` | 将使用从 `https://metadata.packet.net/2009-04-04/meta-data/public-ipv4` 检索到的值。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/kubernetes-concepts.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/kubernetes-concepts.md new file mode 100644 index 00000000000..17a1bc2b969 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/kubernetes-concepts.md @@ -0,0 +1,64 @@ +--- +title: Kubernetes 概念 +--- + +本文解释了 Kubernetes 的相关概念，以便让你更好地了解 Rancher çš„è¿è¡Œæœºåˆ¶ã€‚本文仅对 Kubernetes 组件进行了简å•çš„æ述。如需了解更多详情，请å‚è§ [Kubernetes 组件的官方文档](https://kubernetes.io/docs/concepts/overview/components/)。 + +## 关于 Docker + +Docker 是容器打包和è¿è¡Œæ—¶ç³»ç»Ÿçš„标准。开å‘者在 Dockerfiles 中构建容器镜åƒï¼Œå¹¶åœ¨ Docker é•œåƒä»“库中分å‘容器镜åƒã€‚[Docker Hub](https://hub.docker.com) 是市é¢ä¸Šä¸»æµçš„公有镜åƒä»“库。许多ä¼ä¸šè¿˜åˆ›å»ºç§æœ‰ Docker é•œåƒä»“库。Docker 主è¦ç”¨äºŽç®¡ç†å•ä¸ªèŠ‚点上的容器。 + +:::note + +由于 Kubernetes å·²ç»æˆä¸ºäº†å®¹å™¨ç®¡ç†çš„主æµå·¥å…·ï¼Œæ‰€ä»¥ä»Ž Rancher 2.x 版本开始，我们ä¸å†æ”¯æŒ Docker Swarm。如果你有使用 Docker 管ç†å®¹å™¨çš„需求，å¯ä»¥å®‰è£… Rancher 1.6 进行æ“作。 + +::: + +## 关于 Kubernetes + +Kubernetes 是容器和集群管ç†çš„标准。YAML 文件规定了组æˆä¸€ä¸ªåº”用所需的容器和其他资æºã€‚Kubernetes æ供了调度ã€ä¼¸ç¼©ã€æœåŠ¡å‘现ã€å¥åº·æ£€æŸ¥ã€å¯†æ–‡ç®¡ç†å’Œé…置管ç†ç­‰åŠŸèƒ½ã€‚ + +## Kubernetes 集群是什么 + +集群是å¯ä½œä¸ºä¸€ä¸ªç³»ç»ŸååŒå·¥ä½œçš„一组计算机。 + +_Kubernetes 集群_ 是使用 [Kubernetes 容器编排系统](https://kubernetes.io/)æ¥éƒ¨ç½²ã€è¿ç»´å’Œä¼¸ç¼© Docker 容器的集群，让你的组织实现应用自动化è¿ç»´ã€‚ + +## Kubernetes 集群中节点的角色 + +Kubernetes 集群中的æ¯ä¸ªè®¡ç®—资æºç§°ä¸ºä¸€ä¸ª _节点_ 。节点å¯ä»¥æ˜¯è£¸é‡‘属æœåŠ¡å™¨æˆ–虚拟机。Kubernetes 将节点分为 _etcd_ 节点ã€_controlplane_ 节点和 _worker_ 节点。 + +一个 Kubernetes 集群至少包å«ä¸€ä¸ª etcd 节点，一个 controlplane 节点和一个 worker 节点。 + +### etcd 节点 + +Rancher 在å•èŠ‚点和高å¯ç”¨å®‰è£…中å‡ä½¿ç”¨ etcd 作为数æ®å­˜å‚¨ã€‚在 Kubernetes 中，etcd 也是存储集群状æ€çš„节点的角色。 + +Kubernetes 集群的状æ€ä¿å­˜åœ¨ [etcd](https://kubernetes.io/docs/concepts/overview/components/#etcd) 中。etcd 节点è¿è¡Œ etcd æ•°æ®åº“。 + +etcd æ•°æ®åº“组件是一个分布å¼çš„键值对存储系统，用于存储所有 Kubernetes 的集群数æ®ï¼Œä¾‹å¦‚集群å作和状æ€ç®¡ç†ç›¸å…³çš„æ•°æ®ã€‚建议在多个节点上è¿è¡Œ etcd，以ä¿è¯åœ¨å‘生故障时有å¯ç”¨çš„备份数æ®ã€‚ + +å³ä½¿ä½ å¯ä»¥ä»…在一个节点上è¿è¡Œ etcd，但 etcd 需è¦å¤§å¤šæ•°èŠ‚ç‚¹ï¼ˆå³ quorum）的åŒæ„æ‰èƒ½æ›´æ–°é›†ç¾¤çŠ¶æ€ã€‚集群需è¦åŒ…å«è¶³å¤Ÿæ•°é‡çš„å¥åº· etcd èŠ‚ç‚¹ï¼Œä»¥å½¢æˆ quorum。å‡è®¾é›†ç¾¤ä¸­æœ‰ n 个节点，quorum çš„æ•°é‡åˆ™éœ€è¦æ˜¯ (n/2)+1。如果集群节点数é‡æ˜¯å¥‡æ•°ï¼Œæ¯æ–°å¢žä¸€ä¸ªèŠ‚点，都会增加 quorum 所需节点数。 + +一般情况下，集群中åªè¦é…置三个 etcd 节点就能满足å°åž‹é›†ç¾¤çš„需求，五个 etcd 节点能满足大型集群的需求。 + +### controlplane 节点 + +controlplane 节点上è¿è¡Œ Kubernetes API serverã€scheduler å’Œ Controller Manager。这些节点执行日常任务，以确ä¿é›†ç¾¤çŠ¶æ€å’Œä½ çš„é…置一致。因为 etcd 节点ä¿å­˜äº†é›†ç¾¤çš„全部数æ®ï¼Œæ‰€ä»¥ controlplane 节点是无状æ€çš„。虽然你å¯ä»¥åœ¨å•ä¸ªèŠ‚点上è¿è¡Œ controlplane，但是我们建议在两个或以上的节点上è¿è¡Œ controlplane，以ä¿è¯å†—余性。å¦å¤–，一个节点å¯ä»¥æ—¢æ˜¯ controlplane 节点，åˆæ˜¯ etcd 节点。 + +### Worker 节点 + +æ¯ä¸ª [worker 节点](https://kubernetes.io/docs/concepts/architecture/nodes/)都能è¿è¡Œï¼š + +- **Kubelets**：监控节点状æ€çš„ Agent，确ä¿ä½ çš„容器处于å¥åº·çŠ¶æ€ã€‚ +- **工作负载**：承载应用和其他 deployment 的容器和 Pod。 + +Worker 节点也è¿è¡Œå­˜å‚¨å’Œç½‘络驱动，有必è¦æ—¶ä¹Ÿä¼šè¿è¡Œ Ingress Controller。你å¯ä»¥æ ¹æ®éœ€è¦ï¼Œåˆ›å»ºå°½å¯èƒ½å¤šçš„ worker 节点æ¥è¿è¡Œä½ çš„[工作负载](../pages-for-subheaders/workloads-and-pods.md)。 + +## 关于 Helm + +在 Rancher 高å¯ç”¨å®‰è£…的场景下，你å¯ä»¥ä½¿ç”¨ Helm 工具，把 Rancher 安装到 Kubernetes 集群上。 + +Helm 是 Kubernetes 的包管ç†å·¥å…·ã€‚Helm Chart 为 Kubernetes YAML 清å•æ–‡ä»¶æ供了模æ¿è¯­æ³•ã€‚通过 Helm，用户å¯ä»¥åˆ›å»ºå¯é…置的 deployment，而ä¸ä»…ä»…åªèƒ½ä½¿ç”¨é™æ€æ–‡ä»¶ã€‚如果你需è¦åˆ›å»ºè‡ªå·±çš„ deployment 商店应用，请å‚è§ [https://helm.sh/](https://helm.sh) 上的文档。 + +有关 ServiceAccount å’Œ Cluster Role Binding 的更多信æ¯ï¼Œè¯·å‚è§ [Kubernetes 官方文档](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/examples.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/examples.md new file mode 100644 index 00000000000..98169f63376 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/examples.md @@ -0,0 +1,23 @@ +--- +title: 示例 +--- + +### ServiceMonitor + +ä½ å¯ä»¥åœ¨[此处](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml)找到 ServiceMonitor 自定义资æºçš„示例。 + +### PodMonitor + +ä½ å¯ä»¥åœ¨[此处](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/example-app-pod-monitor.yaml)找到 PodMonitor 示例，还å¯ä»¥åœ¨[此处](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/prometheus-pod-monitor.yaml)找到引用它的 Prometheus 资æºç¤ºä¾‹ã€‚ + +### PrometheusRule + +PrometheusRule 包å«ä½ é€šå¸¸æ”¾ç½®åœ¨ [Prometheus 规则文件](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)中的告警和记录规则。 + +è¦åœ¨é›†ç¾¤ä¸­æ›´ç»†ç²’度地应用 PrometheusRules，你å¯ä»¥ä½¿ç”¨ Prometheus 资æºçš„ ruleSelector 字段，从而根æ®æ·»åŠ åˆ° PrometheusRules 资æºçš„标签æ¥é€‰æ‹©è¦åŠ è½½åˆ° Prometheus 上的 PrometheusRule。 + +ä½ å¯ä»¥åœ¨[此页é¢](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/alerting.md)找到 PrometheusRule 示例。 + +### Alertmanager é…ç½® + +有关示例é…置，请å‚阅[本节](./receivers.md#alertmanager-é…置示例)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/helm-chart-options.md new file mode 100644 index 00000000000..476af17585c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -0,0 +1,68 @@ +--- +title: Helm Chart 选项 +--- + +## é…置资æºé™åˆ¶å’Œè¯·æ±‚ + +安装 `rancher-monitoring` æ—¶å¯ä»¥é…置资æºè¯·æ±‚å’Œé™åˆ¶ã€‚ + +默认值在 `rancher-monitoring` Helm Chart çš„ [values.yaml](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/values.yaml) 中。 + +下表中的默认值是所需的最低资æºé™åˆ¶å’Œè¯·æ±‚： + +| 资æºå称 | 内存é™åˆ¶ | CPU é™åˆ¶ | 内存请求 | CPU 请求 | +| ------------- | ------------ | ----------- | ---------------- | ------------------ | +| alertmanager | 500Mi | 1000m | 100Mi | 100m | +| grafana | 200Mi | 200m | 100Mi | 100m | +| kube-state-metrics subchart | 200Mi | 100m | 130Mi | 100m | +| prometheus-node-exporter subchart | 50Mi | 200m | 30Mi | 100m | +| prometheusOperator | 500Mi | 200m | 100Mi | 100m | +| prometheus | 2500Mi | 1000m | 1750Mi | 750m | +| **总计** | **3950Mi** | **2700m** | **2210Mi** | **1250m** | + +å»ºè®®è‡³å°‘é… 50Gi 存储。 + + +## Notifiers çš„å¯ä¿¡ CA + +如果你需è¦å°†å—信任的 CA 添加到 Notifiers，请执行以下步骤： + +1. 创建 `cattle-monitoring-system` 命å空间。 +1. 将你信任的 CA 密文添加到 `cattle-monitoring-system` 命å空间。 +1. 部署或å‡çº§ `rancher-monitoring` Helm Chart。在 Chart 选项中，引用**å‘Šè­¦ > 补充密文**中的密钥。 + +**结果**：默认的 Alertmanager 自定义资æºå°†æœ‰æƒè®¿é—®ä½ ä¿¡ä»»çš„ CA。 + + +## 其它抓å–é…ç½® + +如果无法通过 ServiceMonitor 或 PodMonitor 指定你想è¦çš„抓å–é…置，你å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ `rancher-monitoring` æ—¶æä¾› `additionalScrapeConfigSecret`。 + +[scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) 指定一组目标以åŠæŠ“å–这些目标的å‚数。在一般情况下，一个抓å–é…置指定一个 job。 + +Istio 就是一个å¯èƒ½ç”¨åˆ°è¿™ä¸ªé…置的例å­ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)。 + + +## é…置打包在 Monitoring V2 中的应用 + +我们使用 Monitoring V2 æ¥éƒ¨ç½² kube-state-metrics å’Œ node-exporter。node-exporter 部署为 DaemonSet。对于 Monitoring V2 helm Chart，values.yaml ä¸­çš„æ‰€æœ‰ä¸œè¥¿éƒ½éƒ¨ç½²ä¸ºå­ Chart。 + +我们还部署了ä¸ç”± prometheus 管ç†çš„ Grafana。 + +如果你查看 Helm Chart 在 kube-state-metrics 中的功能，则还有很多å¯ä»¥è®¾ç½®çš„值没有在顶层 Chart 中显示。 + +但是，你å¯ä»¥åœ¨é¡¶å±‚ Chart ä¸­æ·»åŠ è¦†ç›–å­ Chart 的值。 + +### 增加 Alertmanager 的副本 + +作为 Chart 部署选项的一部分，你å¯ä»¥é€‰æ‹©å¢žåŠ éƒ¨ç½²åˆ°é›†ç¾¤ä¸Šçš„ Alertmanager 副本的数é‡ã€‚这些副本使用相åŒçš„底层 Alertmanager Config Secret 进行管ç†ã€‚有关 Alertmanager Config Secret 的更多信æ¯ï¼Œè¯·å‚阅[本节](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md#多个-alertmanager-副本)。 + +### 为æŒä¹…化 Grafana 仪表æ¿é…置命å空间 + +è¦è®© Grafana 监控所有命å空间中的 ConfigMap，请在 `rancher-monitoring` Helm chart 中指定以下值： + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +请注æ„，Monitoring Chart 用于添加 Grafana 仪表æ¿çš„ RBAC 角色仅能让用户将仪表æ¿æ·»åŠ åˆ°å®šä¹‰åœ¨ `grafana.dashboards.namespace` 中的命å空间，默认为 `cattle-dashboards`。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/receivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/receivers.md new file mode 100644 index 00000000000..e961ec75d77 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/receivers.md @@ -0,0 +1,299 @@ +--- +title: 接收器é…ç½® +--- + +[Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret åŒ…å« Alertmanager 实例的é…ç½®ï¼Œè¯¥å®žä¾‹æ ¹æ® Prometheus å‘出的告警å‘é€é€šçŸ¥ã€‚ + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-alertmanager-的工作原ç†)。 + +::: + + +## 在 Rancher UI 中创建接收器 + +:::note 先决æ¡ä»¶ï¼š + +- 已安装 Monitoring 应用。 +- 如果你使用现有的 Alertmanager Secret é…ç½® Monitoring，则它必须具有 Rancher çš„ UI 支æŒçš„æ ¼å¼ã€‚å¦åˆ™ï¼Œä½ å°†åªèƒ½ç›´æŽ¥ä¿®æ”¹ Alertmanager Secret æ¥è¿›è¡Œæ›´æ”¹ã€‚请注æ„，对于通过使用路由和接收器 UI 支æŒçš„ Alertmanager é…置类型，我们会继续进行强化。因此如果你有增强功能的请求，请[æ交 issue](https://github.com/rancher/rancher/issues/new)。 + +::: + +è¦åœ¨ Rancher UI 中创建通知接收器： + + + + +1. 转到è¦åˆ›å»ºæŽ¥æ”¶å™¨çš„集群。å•å‡» **监控 -> å‘Šè­¦ -> AlertManagerConfigs**。 +1. å•å‡»**创建**。 +1. 点击**添加接收器**。 +1. 输入接收器的**å称**。 +1. 为接收器é…置一个或多个æ供程åºã€‚如需获å–填写表å•çš„帮助，请å‚阅下方的é…置选项。 +1. å•å‡»**创建**。 + + + + +1. 转到è¦åˆ›å»ºæŽ¥æ”¶å™¨çš„集群。å•å‡»**监控**，然åŽå•å‡»**接收器**。 +2. 输入接收器的å称。 +3. 为接收器é…置一个或多个æ供程åºã€‚如需获å–填写表å•çš„帮助，请å‚阅下方的é…置选项。 +4. å•å‡»**创建**。 + + + + +**结果**：告警å¯ä»¥å‘接收器å‘é€é€šçŸ¥ã€‚ + +## 接收器é…ç½® + +通知集æˆæ˜¯é€šè¿‡ `receiver` é…置的，[Prometheus](https://prometheus.io/docs/alerting/latest/configuration/#receiver) 文档对此进行了说明。 + +### 原生和éžåŽŸç”ŸæŽ¥æ”¶å™¨ + +默认情况下，AlertManager æ供与一些接收器的原生集æˆï¼Œè¿™äº›æŽ¥æ”¶å™¨åœ¨[本节](https://prometheus.io/docs/alerting/latest/configuration/#receiver)中列出。所有原生支æŒçš„接收器都å¯ä»¥é€šè¿‡ Rancher UI 进行é…置。 + +对于 AlertManager ä¸æ供原生支æŒçš„通知机制，å¯ä½¿ç”¨ [webhook 接收器](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config)实现集æˆã€‚ä½ å¯ä»¥åœ¨[此处](https://prometheus.io/docs/operating/integrations/#alertmanager-webhook-receiver)找到æ供此类集æˆçš„第三方驱动程åºåˆ—表。Alerting Drivers 应用能让你访问这些驱动程åºï¼Œä»¥åŠå®ƒä»¬ç›¸å…³çš„集æˆã€‚å¯ç”¨åŽï¼Œä½ å°†å¯ä»¥åœ¨ Rancher UI 中é…ç½®éžåŽŸç”Ÿçš„接收器。 + +ç›®å‰ Rancher Alerting Drivers 应用支æŒè®¿é—®ä»¥ä¸‹é›†æˆï¼š +- Microsoft Teams，基于 [prom2teams](https://github.com/idealista/prom2teams) é©±åŠ¨ç¨‹åº +- SMS，基于 [Sachet](https://github.com/messagebird/sachet) é©±åŠ¨ç¨‹åº + +ä½ å¯ä»¥åœ¨ Rancher UI 中å¯ä»¥é…置以下类型的接收器： + +- Slack +- 电å­é‚®ä»¶ +- PagerDuty +- Opsgenie +- Webhook +- 自定义 +- Teams +- SMS + +ä½ å¯ä»¥åœ¨ YAML 中使用自定义接收器选项，从而é…置无法通过 Rancher UI 表å•é…置的接收器。 + +## Slack + +| 字段 | 类型 | æè¿° | +|------|--------------|------| +| URL | String | 输入你的 Slack webhook URL。有关创建 Slack webhook 的说明，请å‚阅 [Slack 文档](https://get.slack.help/hc/en-us/articles/115005265063-Incoming-WebHooks-for-Slack)。 | +| é»˜è®¤é¢‘é“ | String | 输入è¦å‘é€å‘Šè­¦é€šçŸ¥çš„频é“å称。格å¼ï¼š`#`。 | +| ä»£ç† URL | String | webhook 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | Bool | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +## 电å­é‚®ä»¶ + +| 字段 | 类型 | æè¿° | +|------|--------------|------| +| é»˜è®¤æ”¶ä»¶äººåœ°å€ | String | 接收通知的电å­é‚®ä»¶åœ°å€ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | Bool | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +SMTP 选项： + +| 字段 | 类型 | æè¿° | +|------|--------------|------| +| å‘件人 | String | ä½ çš„ SMTP 邮件æœåŠ¡å™¨ä¸Šå¯ç”¨çš„电å­é‚®ä»¶åœ°å€ï¼Œç”¨äºŽå‘é€é€šçŸ¥ã€‚ | +| 主机 | String | SMTP æœåŠ¡å™¨çš„ IP 地å€æˆ–主机å。示例：`smtp.email.com`。 | +| 使用 TLS | Bool | 使用 TLS 进行加密。 | +| 用户å | String | 用户å，用于通过 SMTP æœåŠ¡å™¨è¿›è¡Œèº«ä»½éªŒè¯ã€‚ | +| å¯†ç  | String | 密ç ï¼Œç”¨äºŽé€šè¿‡ SMTP æœåŠ¡å™¨è¿›è¡Œèº«ä»½éªŒè¯ã€‚ | + +## PagerDuty + +| 字段 | 类型 | æè¿° | +|------|------|-------| +| 集æˆç±»åž‹ | String | `Events API v2` 或 `Prometheus`。 | +| 默认集æˆå¯†é’¥ | String | 有关获å–集æˆå¯†é’¥çš„说明，请å‚阅 [PagerDuty 文档](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/)。 | +| ä»£ç† URL | String | PagerDuty 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | Bool | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +## Opsgenie + +| 字段 | æè¿° | +|------|-------------| +| API 密钥 | æœ‰å…³èŽ·å– API 密钥的说明，请å‚阅 [Opsgenie 文档](https://docs.opsgenie.com/docs/api-key-management)。 | +| ä»£ç† URL | Opsgenie 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +Opsgenie å“应者： + +| 字段 | 类型 | æè¿° | +|-------|------|--------| +| 类型 | String | 计划程åºã€å›¢é˜Ÿã€ç”¨æˆ·æˆ–å‡çº§ã€‚有关告警å“应者的更多信æ¯ï¼Œè¯·å‚阅 [Opsgenie 文档](https://docs.opsgenie.com/docs/alert-recipients-and-teams)。 | +| å‘é€è‡³ | String | Opsgenie 收件人的 IDã€å称或用户å。 | + +## Webhook + +| 字段 | æè¿° | +|-------|--------------| +| URL | 你所选的应用的 Webhook URL。 | +| ä»£ç† URL | webhook 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + + + +## 自定义 + +此处æ供的 YAML 将直接附加到 Alertmanager Config Secret 的接收器中。 + +## Teams + +### 为 Rancher 管ç†çš„集群å¯ç”¨ Teams 接收器 + +Teams 接收器ä¸æ˜¯åŽŸç”ŸæŽ¥æ”¶å™¨ï¼Œå› æ­¤éœ€è¦å¯ç”¨åŽæ‰èƒ½ä½¿ç”¨ã€‚ä½ å¯ä»¥é€šè¿‡è½¬åˆ°åº”用页é¢ï¼Œå®‰è£… rancher-alerting-drivers 应用，然åŽé€‰æ‹© Teams 选项，从而为 Rancher 管ç†çš„集群å¯ç”¨ Teams 接收器。 + +1. 在 Rancher UI 中，转到è¦å®‰è£… rancher-alerting-drivers 的集群，然åŽå•å‡» **Apps**。 +1. 点击 **Alerting Drivers** 应用。 +1. å•å‡» **Helm 部署选项**选项å¡ã€‚ +1. 选择 **Teams** 并å•å‡»**安装**。 +1. 记下使用的命å空间，åŽç»­æ­¥éª¤ä¸­å°†éœ€è¦è¯¥å‘½å空间。 + +### é…ç½® Teams 接收器 + +å¯ä»¥é€šè¿‡æ›´æ–° ConfigMap æ¥é…ç½® Teams 接收器。例如，以下是最å°çš„ Teams 接收器é…置： + +```yaml +[Microsoft Teams] +teams-instance-1: https://your-teams-webhook-url +``` + +é…置完æˆåŽï¼ŒæŒ‰ç…§[本节](#在-rancher-ui-中创建接收器)中的步骤添加接收器。 + +使用以下示例作为 URL，其中： + +- å°† `ns-1` 替æ¢ä¸ºå®‰è£… `rancher-alerting-drivers` 应用的命å空间。 + +```yaml +url: http://rancher-alerting-drivers-prom2teams.ns-1.svc:8089/v2/teams-instance-1 +``` + + + +## SMS + +### 为 Rancher 管ç†çš„集群å¯ç”¨ SMS 接收器 + +SMS 接收器ä¸æ˜¯åŽŸç”ŸæŽ¥æ”¶å™¨ï¼Œå› æ­¤éœ€è¦å¯ç”¨åŽæ‰èƒ½ä½¿ç”¨ã€‚ä½ å¯ä»¥é€šè¿‡è½¬åˆ°åº”用页é¢ï¼Œå®‰è£… rancher-alerting-drivers 应用，然åŽé€‰æ‹© SMS 选项，从而为 Rancher 管ç†çš„集群å¯ç”¨ SMS 接收器。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å®‰è£… `rancher-alerting-drivers` 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**应用 & 应用市场**。 +1. 点击 **Alerting Drivers** 应用。 +1. å•å‡» **Helm 部署选项**选项å¡ã€‚ +1. 选择 **SMS** 并å•å‡»**安装**。 +1. 记下使用的命å空间，åŽç»­æ­¥éª¤ä¸­å°†éœ€è¦è¯¥å‘½å空间。 + +### é…ç½® SMS 接收器 + +å¯ä»¥é€šè¿‡æ›´æ–° ConfigMap æ¥é…ç½® SMS 接收器。例如，以下是最å°çš„ SMS 接收器é…置： + +```yaml +providers: + telegram: + token: 'your-token-from-telegram' + +receivers: +- name: 'telegram-receiver-1' + provider: 'telegram' + to: + - '123456789' +``` + +é…置完æˆåŽï¼ŒæŒ‰ç…§[本节](#在-rancher-ui-中创建接收器)中的步骤添加接收器。 + +使用以下示例作为å称和 URL，其中： + +- 分é…给接收器的å称（例如 `telegram-receiver-1`）必须与 ConfigMap 中 `receivers.name` 字段中的å称（例如 `telegram-receiver-1`）匹é…。 +- å°† URL 中的 `ns-1` 替æ¢ä¸ºå®‰è£… `rancher-alerting-drivers` 应用的命å空间。 + +```yaml +name: telegram-receiver-1 +url http://rancher-alerting-drivers-sachet.ns-1.svc:9876/alert +``` + + + + +## é…置多个接收器 + +ä½ å¯ä»¥ç¼–辑 Rancher UI 中的表å•æ¥è®¾ç½®ä¸€ä¸ªæŽ¥æ”¶å™¨èµ„æºï¼Œå…¶ä¸­åŒ…å« Alertmanager 将告警å‘é€åˆ°ä½ çš„通知系统所需的所有信æ¯ã€‚ + +也å¯ä»¥å‘多个通知系统å‘é€å‘Šè­¦ã€‚一ç§æ–¹æ³•æ˜¯ä½¿ç”¨è‡ªå®šä¹‰ YAML æ¥é…置接收器。如果你需è¦è®©ä¸¤ä¸ªç³»ç»ŸæŽ¥æ”¶ç›¸åŒçš„消æ¯ï¼Œåˆ™å¯ä»¥ä¸ºå¤šä¸ªé€šçŸ¥ç³»ç»Ÿæ·»åŠ é…置。 + +你还å¯ä»¥é€šè¿‡ä½¿ç”¨è·¯ç”±çš„ `continue` 选项æ¥è®¾ç½®å¤šä¸ªæŽ¥æ”¶å™¨ã€‚这样，å‘é€åˆ°æŽ¥æ”¶å™¨çš„告警会在路由树（å¯èƒ½åŒ…å«å¦ä¸€ä¸ªæŽ¥æ”¶å™¨ï¼‰çš„下一级进行评估。 + + +## Alertmanager é…置示例 + +### Slack +è¦é€šè¿‡ Slack 设置通知，你å¯ä»¥å°†ä»¥ä¸‹ Alertmanager Config YAML 放入 Alertmanager Config Secret çš„ `alertmanager.yaml` 键中，你需è¦æ›´æ–° `api_url` æ¥ä½¿ç”¨æ¥è‡ª Slack çš„ Webhook URL： + +```yaml +route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + receiver: 'slack-notifications' +receivers: +- name: 'slack-notifications' + slack_configs: + - send_resolved: true + text: '{{ template "slack.rancher.text" . }}' + api_url: +templates: +- /etc/alertmanager/config/*.tmpl +``` + +### PagerDuty +è¦é€šè¿‡ PagerDuty 设置通知，请使用 [PagerDuty 文档](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/) 中的以下示例作为指导。此示例设置了一个路由，该路由æ•èŽ·æ•°æ®åº“æœåŠ¡çš„告警，并将告警å‘é€åˆ°é“¾æŽ¥åˆ°æœåŠ¡çš„接收器，该æœåŠ¡å°†ç›´æŽ¥é€šçŸ¥ PagerDuty 中的 DBA，而其他告警将被定å‘到具有ä¸åŒ PagerDuty 集æˆå¯†é’¥çš„默认接收器。 + +ä½ å¯ä»¥å°†ä»¥ä¸‹ Alertmanager Config YAML 放入 Alertmanager Config Secret çš„ `alertmanager.yaml` 键中。你需è¦å°† `service_key` 更新为使用你的 PagerDuty 集æˆå¯†é’¥ï¼Œå¯ä»¥æ ¹æ® PagerDuty 文档的 "Integrating with Global Event Routing" 找到该密钥。有关é…置选项的完整列表，请å‚阅 [Prometheus 文档](https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config)。 + +```yaml +route: + group_by: [cluster] + receiver: 'pagerduty-notifications' + group_interval: 5m + routes: + - match: + service: database + receiver: 'database-notifcations' + +receivers: +- name: 'pagerduty-notifications' + pagerduty_configs: + - service_key: 'primary-integration-key' + +- name: 'database-notifcations' + pagerduty_configs: + - service_key: 'database-integration-key' +``` + +## CIS 扫æ告警的示例路由é…ç½® + +在为 `rancher-cis-benchmark` å‘Šè­¦é…置路由时，你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹ `job:rancher-cis-scan` æ¥æŒ‡å®šåŒ¹é…。 + +例如，以下路由é…置示例å¯ä»¥ä¸Žå为 `test-cis` çš„ Slack 接收器一起使用： + +```yaml +spec: + receiver: test-cis + group_by: +# - string + group_wait: 30s + group_interval: 30s + repeat_interval: 30s + match: + job: rancher-cis-scan +# key: string + match_re: + {} +# key: string +``` + +有关为 `rancher-cis-benchmark` å¯ç”¨å‘Šè­¦çš„更多信æ¯ï¼Œè¯·å‚阅[本节](../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md)。 + +## Notifiers çš„å¯ä¿¡ CA + +如果你需è¦å°†å—信任的 CA 添加到 Notifiers，请按照[本节](helm-chart-options.md#notifiers-çš„å¯ä¿¡-ca)中的步骤æ“作。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/routes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/routes.md new file mode 100644 index 00000000000..0c39e576670 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/routes.md @@ -0,0 +1,64 @@ +--- +title: 路由é…ç½® +--- + +路由（Route）é…置是 Alertmanager 自定义资æºçš„一部分，用于控制 Prometheus 触å‘的告警在到达接收器之å‰çš„分组和过滤方å¼ã€‚ + +当路由更改时，Prometheus Operator 会é‡æ–°ç”Ÿæˆ Alertmanager 自定义资æºä»¥å映更改。 + +有关é…置路由的更多信æ¯ï¼Œè¯·å‚阅[官方 Alertmanager 文档](https://www.prometheus.io/docs/alerting/latest/configuration/#route)。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + + + +## 路由é™åˆ¶ + +Alertmanager æ ¹æ®æŽ¥æ”¶å™¨å’Œè·¯ç”±æ ‘æ¥ä»£ç† Prometheus 的告警，该路由树根æ®æ ‡ç­¾å°†å‘Šè­¦è¿‡æ»¤åˆ°æŒ‡å®šæŽ¥æ”¶å™¨ã€‚ + +Alerting Drivers 为 Alertmanager 将告警代ç†åˆ°éžåŽŸç”ŸæŽ¥æ”¶å™¨ï¼Œä¾‹å¦‚ Microsoft Teams å’Œ SMS。 + +在é…置路由和接收器的 Rancher UI 中，你å¯ä»¥é…置有一个根的路由树，然åŽå†é…置一个深度，这样的树就有两个深度。但是如果在直接é…ç½® Alertmanager 时使用 `continue` 路由，你å¯ä»¥è®©æ ‘更深。 + +æ¯ä¸ªæŽ¥æ”¶å™¨ç”¨äºŽä¸€ä¸ªæˆ–多个通知æ供商。因此，如果你需è¦å°†å‘é€åˆ° Slack çš„æ¯ä¸ªå‘Šè­¦ä¹Ÿå‘é€åˆ° PagerDuty，你å¯ä»¥åœ¨åŒä¸€ä¸ªæŽ¥æ”¶å™¨ä¸­é…置两者。 + +## 路由é…ç½® + +### 标签和注释的注æ„事项 + +标签用于识别å¯èƒ½å½±å“通知路由的信æ¯ã€‚告警的标识信æ¯å¯èƒ½åŒ…括容器å称，或应接收通知的团队的å称。 + +注释用于标识ä¸å½±å“告警接收者的信æ¯ï¼Œä¾‹å¦‚ Runbook URL 或错误消æ¯ã€‚ + + +### 接收器 +路由需è¦å¼•ç”¨ä¸€ä¸ªå·²ç»é…置好的[接收器](./receivers.md)。 + +### 分组 + +| 字段 | 默认 | æè¿° | +|-------|--------------|---------| +| 分组ä¾æ® | N/A | 用于分组的标签列表。所有标签都必须是唯一的。如果æ供了特殊标签“...â€ï¼ˆç”±æ‰€æœ‰å¯èƒ½çš„标签èšåˆï¼‰ï¼Œæ ‡ç­¾å¿…须在列表中是唯一的元素。接å—字符串列表。有关详细信æ¯ï¼Œè¯·å‚阅[上游文档](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#route)。 | +| 组等待时长 | 30s | 在å‘é€ä¹‹å‰ï¼Œç¼“冲åŒä¸€ç»„告警的等待时间。 | +| 组间隔 | 5m | 等待多长时间æ‰å‘é€å·²æ·»åŠ åˆ°å‘Šè­¦ç»„的告警，其中该告警组的åˆæ¬¡é€šçŸ¥å·²è¢«å‘é€ã€‚ | +| é‡å¤é—´éš” | 4h | 等待多长时间åŽï¼Œæ‰é‡æ–°å‘é€å·²å‘é€çš„告警。 | + +### åŒ¹é… + +**Match** 字段指一组相等匹é…器，用于根æ®å‘Šè­¦ä¸Šå®šä¹‰çš„标签æ¥è¯†åˆ«è¦å‘é€åˆ°æŒ‡å®šè·¯ç”±çš„告警。在 Rancher UI 中添加键值对时，它们对应于以下格å¼çš„ YAML： + +```yaml +match: + [ : , ... ] +``` + +**Match Regex** 字段指一组正则表达å¼åŒ¹é…器，用于根æ®åœ¨è¯¥å‘Šè­¦ä¸Šå®šä¹‰çš„标签æ¥è¯†åˆ«è¦å‘é€åˆ°æŒ‡å®šè·¯ç”±çš„告警。在 Rancher UI 中添加键值对时，它们对应于以下格å¼çš„ YAML： + +```yaml +match_re: + [ : , ... ] +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md new file mode 100644 index 00000000000..e514d60a928 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -0,0 +1,33 @@ +--- +title: ServiceMonitor å’Œ PodMonitor é…ç½® +--- + +ServiceMonitor å’Œ PodMonitor 都是伪 CRD，它们映射 Prometheus 自定义资æºçš„抓å–é…置。 + +这些é…置对象以声明方å¼æŒ‡å®š Prometheus 抓å–指标的端点。 + +ServiceMonitor 比 PodMonitor 更常用，推è用于大多数用例。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + +### ServiceMonitor + +这个伪 CRD 映射到 Prometheus 自定义资æºé…置的一部分。它以声明方å¼æŒ‡å®šåº”如何监控 Kubernetes æœåŠ¡ç»„。 + +创建 ServiceMonitor 时，Prometheus Operator 会更新 Prometheus 抓å–é…ç½®ï¼Œä»Žè€ŒåŒ…å« ServiceMonitor é…ç½®ã€‚ç„¶åŽ Prometheus 开始从 ServiceMonitor 中定义的端点抓å–指标。 + +如果集群中的任何 Service 与 ServiceMonitor `selector` 字段中的标签匹é…ï¼Œåˆ™ä¼šæ ¹æ® ServiceMonitor 指定的 `endpoints` 进行监控。有关å¯ä»¥æŒ‡å®šçš„字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ Prometheus Operator çš„[规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor)。 + +有关 ServiceMonitor 工作原ç†çš„更多信æ¯ï¼Œè¯·å‚阅 [Prometheus Operator 文档](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/running-exporters.md)。 + +### PodMonitor + +这个伪 CRD 映射到 Prometheus 自定义资æºé…置的一部分。它以声明方å¼æŒ‡å®šåº”如何监控 Pod 组。 + +创建 PodMonitor 时，Prometheus Operator 会更新 Prometheus 抓å–é…ç½®ï¼Œä»Žè€ŒåŒ…å« PodMonitor é…ç½®ã€‚ç„¶åŽ Prometheus 开始从 PodMonitor 中定义的端点抓å–指标。 + +如果集群中的任何 Pod 与 PodMonitor `selector` 字段中的标签匹é…ï¼Œåˆ™ä¼šæ ¹æ® PodMonitor 指定的 `podMetricsEndpoints` 进行监控。有关å¯ä»¥æŒ‡å®šçš„字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ Prometheus Operator çš„[规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#podmonitorspec)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/prometheus-federator/rbac.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/prometheus-federator/rbac.md new file mode 100644 index 00000000000..a519b5a8cbc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/prometheus-federator/rbac.md @@ -0,0 +1,31 @@ +--- +title: RBAC +--- + +æœ¬æ–‡ä»‹ç» Prometheus Federator RBAC。 + +如[命å空间](../../pages-for-subheaders/prometheus-federator.md#命å空间)部分所述，Prometheus Federator 期望集群中具有项目级别æƒé™ï¼ˆä¾‹å¦‚，具有由å•ä¸ªæ ‡ç­¾é€‰æ‹©å™¨ç¡®å®šçš„命å空间组的æƒé™ï¼‰çš„项目所有者ã€é¡¹ç›®æˆå‘˜å’Œå…¶ä»–用户，除了项目 Registration 命å空间（默认导入到项目中）和那些已ç»åŒ…å«å…¶é¡¹ç›®çš„命å空间之外，在任何其他命å空间中都åªæœ‰æœ€ä½Žæƒé™ã€‚因此，为了让项目所有者将特定 Chart æƒé™åˆ†é…给其项目命å空间中的其他用户，Helm Project Operator 将自动监视以下绑定： + +- ClusterRoleBindings +- 项目å‘布命å空​​间中的 RoleBindings + +如果 Helm Project Operator 观察到其中一ç§ç»‘定的更改，Helm Project Operator 会检查绑定指å‘çš„ `roleRef` 是å¦ä¸Žå…·æœ‰ä»¥ä¸‹å称的 ClusterRole 匹é…： + +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view` + +默认情况下，这些 roleRef 分别对应 `admin`ã€`edit` å’Œ `view`，它们都是 [Kubernetes é¢å‘用户的默认角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)。 + +:::note + +对于 Rancher RBAC 用户，这些 [Kubernetes é¢å‘用户的默认角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)与`项目所有者`ã€`项目æˆå‘˜`å’Œ`åªè¯»`默认项目角色模æ¿ç›´æŽ¥å¯¹åº”。 + +::: + +如果 `roleRef` 匹é…，Helm Project Operator 将为所有用户和组过滤绑定的 `subjects`，并使用它为项目 Release 命å空间中的æ¯ä¸ªè§’色自动构造一个 RoleBinding，该 RoleBinding çš„å称与角色相åŒå¹¶å¸¦æœ‰ä»¥ä¸‹æ ‡ç­¾ï¼š + +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +默认情况下，Prometheus Federator 部署的底层 Chart `rancher-project-monitoring` 会为æ¯ä¸ªé¡¹ç›®å‘布命åç©ºâ€‹â€‹é—´åˆ›å»ºä¸‰ä¸ªé»˜è®¤è§’è‰²ï¼Œè¿™äº›è§’è‰²èƒ½æŽˆæƒ `admin`ã€`edit` å’Œ `view` 用户查看项目监控堆栈的 Prometheusã€Alertmanager å’Œ Grafana UI，从而æ供最低æƒé™ã€‚如果集群管ç†å‘˜æƒ³è¦ä¸ºæŸäº›ç”¨æˆ·åˆ†é…é¢å¤–çš„æƒé™ï¼Œä¸€ç§åšæ³•æ˜¯ç›´æŽ¥å°†é¡¹ç›® Release 命å空间中的 RoleBinding 分é…ç»™æŸäº›ç”¨æˆ·ã€‚å¦ä¸€ç§åšæ³•æ˜¯åˆ›å»ºå¸¦æœ‰ä¸Šè¿°ä¸¤ä¸ªæ ‡ç­¾çš„角色，然åŽï¼Œé¡¹ç›®æ‰€æœ‰è€…å¯ä»¥æŽ§åˆ¶åœ¨é¡¹ç›® Registration 命å空间中分é…这些 RBAC 角色的用户。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-cluster-tools.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-cluster-tools.md new file mode 100644 index 00000000000..7f1ed5206c7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-cluster-tools.md @@ -0,0 +1,48 @@ +--- +title: 集群工具：Logging，Monitoring å’Œå¯è§†åŒ– +--- + +Rancher åŒ…å« Kubernetes 中未包å«çš„å„ç§å·¥å…·æ¥å助你进行 DevOps æ“作。Rancher å¯ä»¥ä¸Žå¤–部æœåŠ¡é›†æˆï¼Œè®©ä½ çš„集群更高效地è¿è¡Œã€‚工具分为以下几类： + + +## Logging + +Logging 支æŒï¼š + +- 获å–和分æžé›†ç¾¤çš„çŠ¶æ€ +- 在你的环境中å‘现趋势 +- 将日志ä¿å­˜åˆ°é›†ç¾¤å¤–部的安全ä½ç½® +- éšæ—¶äº†è§£å®¹å™¨å´©æºƒã€pod 驱é€æˆ–节点死亡等事件 +- æ›´è½»æ¾åœ°è°ƒè¯•å’Œè§£å†³é—®é¢˜ + +Rancher å¯ä»¥ä¸Ž Elasticsearchã€splunkã€kafkaã€syslog å’Œ fluentd 集æˆã€‚ + +有关详细信æ¯ï¼Œè¯·å‚阅 [Logging 文档](../pages-for-subheaders/logging.md)。 +## 监控和告警 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher，通过业界领先并开æºçš„ [Prometheus](https://prometheus.io/) æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚ + +å¯ç”¨ç›‘控åŽï¼Œä½ å¯ä»¥é€šè¿‡è®¾ç½®å‘Šè­¦å’Œé€šçŸ¥å™¨æ¥é…置接收告警的机制。 + +通知器是通知你告警事件的æœåŠ¡ã€‚ä½ å¯ä»¥é€šè¿‡é…置通知器æ¥å‘最适åˆé‡‡å–纠正措施的员工å‘é€å‘Šè­¦é€šçŸ¥ã€‚支æŒä½¿ç”¨ Slackã€ç”µå­é‚®ä»¶ã€PagerDutyã€å¾®ä¿¡å’Œ webhook å‘é€é€šçŸ¥ã€‚ + +告警是触å‘这些通知的规则。在接收告警之å‰ï¼Œä½ å¿…须在 Rancher 中é…置一个或多个通知器。你å¯ä»¥åœ¨é›†ç¾¤æˆ–项目级别设置告警范围。 + +如需更多信æ¯ï¼Œè¯·å‚阅[监控文档](../pages-for-subheaders/monitoring-and-alerting.md)。 + +## Istio + +[Istio](https://istio.io/) 是一ç§å¼€æºå·¥å…·ï¼Œå¯ä»¥è®© DevOps 团队更轻æ¾åœ°è§‚察ã€æŽ§åˆ¶ã€æŽ’查并ä¿æŠ¤å¤æ‚çš„å¾®æœåŠ¡ç½‘络中的æµé‡ã€‚ + +Rancher v2.5 改进了与 Istio 的集æˆã€‚ + +如需更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](../pages-for-subheaders/istio.md)。 +## OPA Gatekeeper + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) 是一个开æºé¡¹ç›®ï¼Œå®ƒå¯¹ OPA å’Œ Kubernetes 进行了集æˆï¼Œä»¥é€šè¿‡è®¸å¯æŽ§åˆ¶å™¨ Webhook æ供策略控制。有关如何在 Rancher 中å¯ç”¨ Gatekeeper 的详细信æ¯ï¼Œè¯·å‚阅 [OPA Gatekeeper](../integrations-in-rancher/opa-gatekeeper.md)。 + +## CIS 扫æ + +Rancher å¯ä»¥é€šè¿‡è¿è¡Œå®‰å…¨æ‰«ææ¥æ£€æŸ¥ Kubernetes 是å¦æŒ‰ç…§ CIS Kubernetes Benchmark 中定义的安全最佳实践进行部署。 + +如需更多信æ¯ï¼Œè¯·å‚阅 [CIS 扫æ文档](../pages-for-subheaders/cis-scan-guides.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/architecture-recommendations.md new file mode 100644 index 00000000000..10ae10563b0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -0,0 +1,106 @@ +--- +title: 架构推è +--- + +如果你准备在å•ä¸ªèŠ‚点上安装 Rancher，我们推èä½ [分开部署 Rancher 与下游集群](#分开部署-rancher-与下游集群)。 + +## 分开部署 Rancher 与下游集群 + +下游集群，是è¿è¡Œä½ è‡ªå·±çš„应用和æœåŠ¡çš„下游 Kubernetes 集群。 + +如果你通过 Docker 安装了 Rancher，è¿è¡Œ Rancher Server 的节点应该与你的下游集群分开。 + +如果你需è¦ä½¿ç”¨ Rancher 管ç†ä¸‹æ¸¸ Kubernetes 集群，那么è¿è¡Œ Rancher Server çš„ Kubernetes 集群也应该与下游集群分开。 + +![分开部署 Rancher Server 与下游集群](/img/rancher-architecture-separation-of-rancher-server.svg) + +## 为什么高å¯ç”¨ï¼ˆHA）更适åˆç”Ÿäº§çŽ¯å¢ƒä¸­çš„ Rancher + +我们建议在高å¯ç”¨ Kubernetes 集群上安装 Rancher Server，以ä¿æŠ¤ Rancher Server çš„æ•°æ®ã€‚在高å¯ç”¨å®‰è£…中，负载å‡è¡¡å™¨å……当客户端的å•ç‚¹å…¥å£ï¼Œå¹¶åœ¨é›†ç¾¤ä¸­çš„多å°æœåŠ¡å™¨ä¹‹é—´åˆ†é…网络æµé‡ï¼Œè¿™æœ‰åŠ©äºŽé˜²æ­¢ä»»ä½•ä¸€å°æœåŠ¡å™¨æˆä¸ºå•ç‚¹æ•…障。 + +我们ä¸å»ºè®®åœ¨å•ä¸ª Docker 容器中安装 Rancher，因为如果该节点å‘生故障，则其他节点上将没有å¯ç”¨çš„集群数æ®å‰¯æœ¬ï¼Œå¹¶ä¸”ä½ å¯èƒ½ä¼šä¸¢å¤± Rancher Server 上的数æ®ã€‚ + +### K3s Kubernetes 集群安装 + +底层 Kubernetes 集群的一ç§é€‰æ‹©æ˜¯ä½¿ç”¨ K3s Kubernetes。K3s 是 Rancher CNCF 认è¯çš„ Kubernetes å‘行版。K3s æ˜“äºŽå®‰è£…ï¼Œä»…éœ€è¦ Kubernetes 内存的一åŠï¼Œæ‰€æœ‰ç»„件都在一个å°äºŽ 100 MB 的二进制文件中。K3s çš„å¦ä¸€ä¸ªä¼˜ç‚¹æ˜¯å…许外部 Datastore ä¿å­˜é›†ç¾¤æ•°æ®ï¼Œå› æ­¤å¯ä»¥æŠŠ K3s æœåŠ¡å™¨èŠ‚点视为无状æ€ã€‚ + +
è¿è¡Œ Rancher Management Server çš„ K3s Kubernetes 集群的架构
+ +![è¿è¡Œ Rancher Management Server çš„ K3s Kubernetes 集群的架构](/img/k3s-server-storage.svg) + +### RKE Kubernetes 集群安装 + +在 RKE 安装中，集群数æ®åœ¨é›†ç¾¤ä¸­çš„三个 etcd 节点上å¤åˆ¶ï¼Œä»¥åœ¨æŸä¸ªèŠ‚点å‘生故障时æ供冗余和进行数æ®å¤åˆ¶ã€‚ + +
è¿è¡Œ Rancher Management Server çš„ RKE Kubernetes 集群的架构
+ +![è¿è¡Œ Rancher Management Server çš„ RKE Kubernetes 集群的架构](/img/rke-server-storage.svg) + +## Kubernetes 安装的负载å‡è¡¡å™¨æŽ¨èé…ç½® + +我们建议你为负载å‡è¡¡å™¨å’Œ Ingress Controller 使用以下é…置： + +* 把 Rancher çš„ DNS 解æžåˆ°å››å±‚è´Ÿè½½å‡è¡¡å™¨ä¸Šã€‚ +* è´Ÿè½½å‡è¡¡å™¨åº”该把 TCP/80 端å£å’Œ TCP/443 端å£çš„æµé‡è½¬å‘到 Kubernetes 集群的全部 3 个节点上。 +* Ingress Controller 会把 HTTP é‡å®šå‘到 HTTPS，在 TCP/443 端å£ç»ˆç»“ SSL/TLS。 +* Ingress Controller 会把æµé‡è½¬å‘到 Rancher deployment çš„ Pod 上的 TCP/80 端å£ã€‚ + +
在 Kubernetes 集群中安装 Rancher，并使用四层负载å‡è¡¡å™¨ï¼ŒSSL 终止在 Ingress Controller 中
+ +![Rancher HA](/img/ha/rancher2ha.svg) + +## Kubernetes 安装环境 + +我们强烈建议你把 Rancher 安装到托管在云æ供商（如 AWS EC2 å’Œ Google Compute Engine（GCE）等）上的 Kubernetes 集群上。 + +为了达到最佳性能和安全性，我们建议你为 Rancher Management Server 创建一个专用的 Kubernetes 集群。ä¸å»ºè®®åœ¨æ­¤é›†ç¾¤ä¸Šè¿è¡Œç”¨æˆ·å·¥ä½œè´Ÿè½½ã€‚部署 Rancher åŽï¼Œä½ å¯ä»¥[创建或导入集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)æ¥è¿è¡Œä½ çš„工作负载。 + +## Kubernetes 安装的推è节点角色 + +如果 Rancher 安装在 K3s Kubernetes 或 RKE Kubernetes 集群上，以下建议适用。 + +### K3s 集群角色 + +在 K3s 集群中有两ç§ç±»åž‹çš„节点，分别是 Server 节点和 Agent 节点。你å¯ä»¥æŠŠå·¥ä½œè´Ÿè½½è°ƒåº¦åˆ° Server 节点和 Agent 节点上。Server 节点è¿è¡Œ Kubernetes master。 + +对于è¿è¡Œ Rancher Management Server 的集群，我们建议使用两个 server 节点。ä¸éœ€è¦ Agent 节点。 + +### RKE 集群角色 + +如果 Rancher 安装在 RKE Kubernetes 集群上，该集群应具有三个节点，并且æ¯ä¸ªèŠ‚点都应具有所有三个 Kubernetes 角色，分别是 etcd，controlplane å’Œ worker。 + +### Rancher Server 和下游 Kubernetes 集群的 RKE 集群架构对比 + +我们对 Rancher Server 集群上 RKE 节点角色建议，与对è¿è¡Œä½ çš„应用和æœåŠ¡çš„下游集群的建议相å。 + +在é…置下游 Kubernetes 集群时，Rancher 使用 RKE 作为创建下游 Kubernetes 集群的工具。注æ„：Rancher 将在未æ¥çš„版本中添加é…置下游 K3s 集群的功能。 + +我们建议下游 Kubernetes 集群中的æ¯ä¸ªèŠ‚点都åªåˆ†é…一个角色，以确ä¿ç¨³å®šæ€§å’Œå¯æ‰©å±•æ€§ã€‚ + +![Rancher Server 集群中和下游集群中节点的 Kubernetes 角色对比](/img/rancher-architecture-node-roles.svg) + +RKE æ¯ä¸ªè§’色至少需è¦ä¸€ä¸ªèŠ‚点，但并ä¸å¼ºåˆ¶æ¯ä¸ªèŠ‚点åªèƒ½æœ‰ä¸€ä¸ªè§’色。但是，我们建议为è¿è¡Œåº”用的集群中的æ¯ä¸ªèŠ‚点，使用å•ç‹¬çš„角色，以ä¿è¯åœ¨æœåŠ¡æ‹“展时，worker 节点上的工作负载ä¸å½±å“ Kubernetes master 或集群的数æ®ã€‚ + +以下是我们对下游集群的最低é…置建议： + +- **三个仅使用 etcd 角色的节点** ，以在三个节点中其中一个å‘生故障时，ä»èƒ½ä¿éšœé›†ç¾¤çš„高å¯ç”¨æ€§ã€‚ +- **两个åªæœ‰ controlplane 角色的节点** ，以ä¿è¯ master 组件的高å¯ç”¨æ€§ã€‚ +- **一个或多个åªæœ‰ worker 角色的节点**，用于è¿è¡Œ Kubernetes 节点组件，以åŠä½ éƒ¨ç½²çš„æœåŠ¡æˆ–应用的工作负载。 + +在设置 Rancher Server 时，在三个节点上使用全部这三个角色也是安全的，因为： + +* 它å…许一个 `etcd` 节点故障。 +* 它通过多个 `controlplane` 节点æ¥ç»´æŠ¤ master 组件的多个实例。 +* 此集群上没有创建除 Rancher 之外的其他工作负载。 + +由于 Rancher Server 集群中没有部署其他工作负载，因此在大多数情况下，这个集群都ä¸éœ€è¦ä½¿ç”¨æˆ‘们出于å¯æ‰©å±•æ€§å’Œå¯ç”¨æ€§çš„考虑，而为下游集群推è的架构。 + +有关下游集群的最佳实践，请查看[生产环境清å•](../../pages-for-subheaders/checklist-for-production-ready-clusters.md)或[最佳实践](../../pages-for-subheaders/best-practices.md)。 + +## 授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„ + +如果你使用[授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)，我们建议你创建一个指å‘è´Ÿè½½å‡è¡¡å™¨çš„ FQDN，这个负载å‡è¡¡å™¨æŠŠæµé‡è½¬åˆ°æ‰€æœ‰è§’色为 `controlplane` 的节点。 + +如果你在负载å‡è¡¡å™¨ä¸Šä½¿ç”¨äº†ç§æœ‰ CA ç­¾å‘çš„è¯ä¹¦ï¼Œä½ éœ€è¦æä¾› CA è¯ä¹¦ï¼Œè¿™ä¸ªè¯ä¹¦ä¼šåŒ…å«åœ¨ç”Ÿæˆçš„ kubeconfig 文件中，以校验è¯ä¹¦é“¾ã€‚详情请å‚è§ [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)å’Œ [API 密钥](../user-settings/api-keys.md#创建-api-密钥)的相关文档。 + +注册的 RKE2 å’Œ K3s 集群å¯ä»¥ä½¿ç”¨ ACE 支æŒã€‚点击[这里](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#对-rke2-å’Œ-k3s-集群的授æƒé›†ç¾¤ç«¯ç‚¹æ”¯æŒ)äº†è§£åœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­å¼€å¯ ACE 的步骤。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md new file mode 100644 index 00000000000..56800763102 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -0,0 +1,132 @@ +--- +title: 与下游集群通信 +--- + +æœ¬èŠ‚ä»‹ç» Rancher 如何é…置和管ç†è¿è¡Œåº”用和æœåŠ¡çš„下游集群。 + +下图显示了 Cluster Controllerã€Cluster Agent å’Œ Node Agent 让 Rancher 控制下游集群的。 + +
与下游集群通信
+ +![Rancher 组件](/img/rancher-architecture-cluster-controller.svg) + +以下æ述对应于上图中的数字： + +1. [认è¯ä»£ç†](#1-认è¯ä»£ç†) +2. [Cluster Controller å’Œ Cluster Agent](#2-cluster-controller-å’Œ-cluster-agent) +3. [Node Agents](#3-node-agents) +4. [授æƒé›†ç¾¤ç«¯ç‚¹](#4-授æƒé›†ç¾¤ç«¯ç‚¹) + +### 1. 认è¯ä»£ç† + +在此图中，å为 Bob 的用户希望查看在å为 User Cluster 1 的下游集群上è¿è¡Œçš„所有 Pod。在 Rancher 中，他å¯ä»¥è¿è¡Œ `kubectl` 命令æ¥æŸ¥çœ‹ +Pod。Bob 通过 Rancher 的认è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +认è¯ä»£ç†å°†æ‰€æœ‰ Kubernetes API 调用转å‘到下游集群。它集æˆäº†æœ¬åœ°èº«ä»½éªŒè¯ã€Active Directory å’Œ GitHub 等身份验è¯æ–¹å¼ã€‚在æ¯ä¸ª Kubernetes API 调用请求时，认è¯ä»£ç†ä¼šéªŒè¯è¯·æ±‚方的身份，并在转å‘ç»™ Kubernetes master 节点之å‰ï¼Œè®¾ç½®æ­£ç¡®çš„ Kubernetes 消æ¯å¤´ã€‚ + +Rancher 使用 [ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 与 Kubernetes 集群通信，该 ServiceAccount 为在 Pod 中è¿è¡Œçš„进程æ供身份。 + +默认情况下，Rancher 生æˆä¸€ä¸ª [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)，文件包å«å‡­è¯ä¿¡æ¯ï¼Œç”¨äºŽä¸º Rancher Server 连接下游集群的 Kubernetes API Server 的代ç†ã€‚kubeconfig 文件 (`kube_config_rancher-cluster.yml`) 包å«å¯¹é›†ç¾¤çš„完全访问æƒé™ã€‚ + +### 2. Cluster Controller å’Œ Cluster Agent + +æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤éƒ½æœ‰ä¸€ä¸ª Cluster Agent，用于打开与 Rancher Server 中对应的 Cluster Controller 之间的通é“。 + +æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤æœ‰ä¸€ä¸ª Cluster Controller 和一个 Cluster Agent。æ¯ä¸ª Cluster Controller 都能： + +- 检测下游集群中的资æºå˜åŒ– +- 将下游集群的当å‰çŠ¶æ€å˜æ›´åˆ°ç›®æ ‡çŠ¶æ€ +- é…置集群和项目的访问控制策略 +- 通过调用所需的 Docker Machine 驱动和 Kubernetes 引擎（例如 RKE å’Œ GKE）æ¥é…置集群 + +默认情况下，Cluster Controller 连接到 Cluster Agent，Rancher æ‰èƒ½ä¸Žä¸‹æ¸¸é›†ç¾¤é€šä¿¡ã€‚如果 Cluster Agent ä¸å¯ç”¨ï¼ŒCluster Controller å¯ä»¥è¿žæŽ¥åˆ° [Node Agent](#3-node-agents)。 + +Cluster Agent，也å«åš `cattle-cluster-agent`，是è¿è¡Œåœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­çš„组件。它具有以下功能： + +- 连接 Rancher å¯åŠ¨çš„ Kubernetes 集群中的 Kubernetes API。 +- 管ç†é›†ç¾¤å†…的工作负载，pod 创建和部署。 +- æ ¹æ®æ¯ä¸ªé›†ç¾¤çš„全局策略，应用定义的角色和绑定。 +- 通过与 Cluster Controller 之间的通é“，实现集群和 Rancher Server 之间的通信，包括事件，统计数æ®ï¼ŒèŠ‚点信æ¯å’Œå¥åº·çŠ¶å†µã€‚ + +### 3. Node Agents + +如果 Cluster Agent（也称为 `cattle-cluster-agent`）ä¸å¯ç”¨ï¼Œå…¶ä¸­ä¸€ä¸ª Node Agent 会创建一个连接到 Cluster Controller 的通é“与 Rancher 通信。 + +`cattle-node-agent` 使用 [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) 资æºè¿›è¡Œéƒ¨ç½²ï¼Œä»¥ç¡®ä¿å®ƒèƒ½åœ¨ Rancher å¯åŠ¨çš„ Kubernetes 集群中的æ¯ä¸ªèŠ‚点上è¿è¡Œï¼Œç”¨äºŽåœ¨æ‰§è¡Œé›†ç¾¤æ“作时与节点交互。集群æ“作的包括å‡çº§ Kubernetes 版本，创建或æ¢å¤ etcd 快照等。 + +### 4. 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯è¿žæŽ¥åˆ°ä¸‹æ¸¸é›†ç¾¤çš„ Kubernetes API Server，而ä¸ç”¨é€šè¿‡ Rancher 认è¯ä»£ç†è°ƒåº¦è¯·æ±‚。 + +> 授æƒé›†ç¾¤ç«¯ç‚¹ä»…适用于 Rancher å¯åŠ¨çš„ Kubernetes 集群，å³åªé€‚用于 Rancher [使用 RKE](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) æ¥é…置的集群。它ä¸é€‚用于导入的集群，也ä¸é€‚用于托管在 Kubernetes æ供商中的集群（例如 Amazon çš„ EKS）。 + +授æƒé›†ç¾¤ç«¯ç‚¹çš„主è¦ç”¨é€”： + +- 在 Rancher ä¸å¯ç”¨æ—¶è®¿é—®ä¸‹æ¸¸é›†ç¾¤ +- 在 Rancher Server 和与下游集群之间相è·ç”šè¿œæ—¶é™ä½Žå»¶è¿Ÿ + +`kube-api-auth` å¾®æœåŠ¡ä¸ºæŽˆæƒé›†ç¾¤ç«¯ç‚¹æ供用户验è¯åŠŸèƒ½ã€‚当使用 `kubectl`访问下游集群时，集群的 Kubernetes API Server 使用 `kube-api-auth` æœåŠ¡ä½œä¸º webhook 对用户进行身份验è¯ã€‚ + +与授æƒé›†ç¾¤ç«¯ç‚¹ä¸€æ ·ï¼Œ`kube-api-auth` 的身份验è¯åŠŸèƒ½ä¹Ÿä»…适用于 Rancher å¯åŠ¨çš„ Kubernetes 集群。 + +> **示例场景：** å‡è®¾ Rancher Server ä½äºŽç¾Žå›½ï¼ŒUser Cluster 1 与用户 Alice å‡ä½äºŽæ¾³å¤§åˆ©äºšã€‚Alice å¯ä»¥ä½¿ç”¨ Rancher UI æ“作 User Cluster 1 中的资æºï¼Œä½†å¥¹çš„请求必须从澳大利亚å‘é€åˆ°ç¾Žå›½çš„ Rancher Server，然åŽé€šè¿‡ä»£ç†è¿”回澳大利亚，å³ä¸‹æ¸¸é›†ç¾¤æ‰€åœ¨çš„ä½ç½®ã€‚地ç†è·ç¦»å¯èƒ½å¯¼è‡´æ˜Žæ˜¾å»¶è¿Ÿï¼Œå› æ­¤ï¼ŒAlice å¯ä»¥ä½¿ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹æ¥é™ä½Žå»¶è¿Ÿã€‚ + +为下游集群å¯ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹åŽï¼ŒRancher 会在 kubeconfig 文件中é¢å¤–生æˆä¸€æ®µ Kubernetes 上下文，用于直连到集群。该文件具有 `kubectl` å’Œ `helm`的凭è¯ã€‚ + +如果 Rancher 出现问题，你需è¦ä½¿ç”¨æ­¤ kubeconfig 文件中定义的上下文æ¥è®¿é—®é›†ç¾¤ã€‚因此，我们建议你导出 kubeconfig 文件，以便在 Rancher 出现问题时，ä»èƒ½ä½¿ç”¨æ–‡ä»¶ä¸­çš„凭è¯è®¿é—®é›†ç¾¤ã€‚详情请å‚è§ä½¿ç”¨ [kubectl å’Œ kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)访问集群的章节。 + +## é‡è¦æ–‡ä»¶ + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_rancher-cluster.yml`：集群的 Kubeconfig 文件，包å«å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚如果 Rancher 出现问题时，你å¯ä»¥ä½¿ç”¨æ­¤æ–‡ä»¶è®¤è¯ç”± Rancher å¯åŠ¨çš„ Kubernetes 集群。 +- `rancher-cluster.rkestate`：Kubernetes 集群状æ€æ–‡ä»¶ï¼Œæ–‡ä»¶åŒ…å«ç”¨äºŽå®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚注æ„：仅在使用 RKE v0.2.0 或更高版本时，æ‰ä¼šåˆ›å»ºæ­¤è¯¥æ–‡ä»¶ã€‚ + +> **注æ„**：åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +有关在没有 Rancher 认è¯ä»£ç†å’Œå…¶ä»–é…置选项的情况下连接到集群的更多信æ¯ï¼Œè¯·å‚è§ [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 + +## é…ç½® Kubernetes 集群的工具 + +Rancher 使用什么工具é…置下游集群，å–决于集群的类型。 + +### Rancher 为托管在云æ供商中的节点å¯åŠ¨ Kubernetes + +Rancher å¯ä»¥åŠ¨æ€å¯åŠ¨äº‘上（如 Amazon EC2ã€DigitalOceanã€Azure 或 vSphere 等）的节点，然åŽåœ¨èŠ‚点上安装 Kubernetes。 + +Rancher 使用 [RKE](https://github.com/rancher/rke) å’Œ [docker-machine](https://github.com/rancher/machine) æ¥é…置这类型的集群。 + +### Rancher 为自定义节点å¯åŠ¨ Kubernetes + +在é…置此类集群时，Rancher 会在现有节点上安装 Kubernetes，从而创建自定义集群。 + +Rancher 使用 [RKE](https://github.com/rancher/rke) æ¥å¯åŠ¨æ­¤ç±»é›†ç¾¤ã€‚ + +### 托管的 Kubernetes æ供商 + +é…置此类集群时，Kubernetes 由云æ供商安装，如 GKEã€ECS 或 AKS 等。 + +Rancher 使用 [kontainer-engine](https://github.com/rancher/kontainer-engine) é…置此类型的集群。 + +### 导入的 Kubernetes 集群 + +è¿™ç§æƒ…况下，Rancher 需è¦è¿žæŽ¥åˆ°ä¸€ä¸ªè®¾ç½®å¥½çš„ Kubernetes 集群。因此，Rancher ä¸æä¾› Kubernetes，åªè®¾ç½® Rancher Agent 实现与集群通信。 + +## Rancher Server 组件和æºç  + +下图展示了 Rancher Server 的组件： + +![Rancher 组件](/img/rancher-architecture-rancher-components.svg) + +Rancher çš„ GitHub 代ç ä»“库如下： + +- [Rancher Server 主代ç åº“](https://github.com/rancher/rancher) +- [Rancher UI](https://github.com/rancher/ui) +- [Rancher API UI](https://github.com/rancher/api-ui) +- [Norman](https://github.com/rancher/norman)（Rancher çš„ API 框架） +- [类型](https://github.com/rancher/types) +- [Rancher CLI](https://github.com/rancher/cli) +- [应用商店](https://github.com/rancher/helm) + +以上仅列出部分 Rancher 最é‡è¦çš„仓库。详情请å‚è§[å‚与 Rancher å¼€æºè´¡çŒ®](../../contribute-to-rancher.md#rancher-仓库)ã€‚å¦‚éœ€èŽ·å– Rancher 使用的所有库和项目，请å‚è§ `rancher/rancher` 仓库中的 [`go.mod` 文件](https://github.com/rancher/rancher/blob/master/go.mod)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/rancher-server-and-components.md new file mode 100644 index 00000000000..b715f4be1e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -0,0 +1,35 @@ +--- +title: Rancher Server å’Œ Components +--- + +大多数 Rancher 2.x 软件å‡è¿è¡Œåœ¨ Rancher Server 上。Rancher Server 包括用于管ç†æ•´ä¸ª Rancher 部署的所有软件组件。 + +下图展示了 Rancher 2.x 的上层架构。下图中，Rancher Server 管ç†ä¸¤ä¸ªä¸‹æ¸¸ Kubernetes 集群，其中一个由 RKE 创建，å¦ä¸€ä¸ªç”± Amazon EKS 创建。 + +为了达到最佳性能和安全性，我们建议你为 Rancher Management Server 创建一个专用的 Kubernetes 集群。ä¸å»ºè®®åœ¨æ­¤é›†ç¾¤ä¸Šè¿è¡Œç”¨æˆ·å·¥ä½œè´Ÿè½½ã€‚部署 Rancher åŽï¼Œä½ å¯ä»¥[创建或导入集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)æ¥è¿è¡Œä½ çš„工作负载。 + +下图介ç»äº†ç”¨æˆ·å¦‚何通过 Rancher 的认è¯ä»£ç†ç®¡ç† [Rancher å¯åŠ¨çš„ Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群和[托管的 Kubernetes](../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md) 集群： + +
通过 Rancher 的认è¯ä»£ç†ç®¡ç† Kubernetes 集群
+ +![架构](/img/rancher-architecture-rancher-api-server.svg) + +ä½ å¯ä»¥æŠŠ Rancher 安装到å•ä¸ªèŠ‚点或高å¯ç”¨ Kubernetes 集群上。 + +在生产环境中，建议安装到高å¯ç”¨ Kubernetes 集群。 + +Rancher çš„ Docker 安装仅推è用于开å‘和测试环境中。Rancher 版本决定了能å¦å°† Rancher è¿ç§»åˆ°é«˜å¯ç”¨é›†ç¾¤ã€‚ + +è¦æŸ¥çœ‹å·²éƒ¨ç½²çš„资æºï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +```bash +kubectl get all -n +``` +如果你具有管ç†å‘˜æƒé™ï¼Œä½ è¿˜å¯ä»¥åœ¨ Rancher UI 中看到列出的资æºï¼š + +1. å•å‡» **☰** 并选择一个集群。 +1. 从侧é¢å¯¼èˆªèœå•ä¸­é€‰æ‹©**更多资æº**，从而按类型查看已部署的资æºã€‚ +1. 从侧é¢å¯¼èˆªèœå•ä¸­é€‰æ‹©**集群** > **项目/命å空间**，然åŽé€‰æ‹©ä¸€ä¸ªå‘½å空间，从而按命å空间查看已部署的资æºã€‚ + Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。有关详细信æ¯ï¼Œè¯·å‚阅[å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +ä¸ç®¡ Rancher Server 是如何安装的，它都应该è¿è¡Œåœ¨ä¸Žå…¶ç®¡ç†çš„下游集群ä¸åŒèŠ‚点上。如果 Rancher 安装在高å¯ç”¨çš„ Kubernetes 集群上，它需è¦è¿è¡Œåœ¨ä¸Žå…¶ç®¡ç†çš„集群ä¸åŒçš„集群上。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-project-tools.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-project-tools.md new file mode 100644 index 00000000000..ad05d4166e0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-project-tools.md @@ -0,0 +1,32 @@ +--- +title: 项目工具：Logging，Monitoring å’Œå¯è§†åŒ– +--- + +Rancher åŒ…å« Kubernetes 中未包å«çš„å„ç§å·¥å…·æ¥å助你进行 DevOps æ“作。Rancher å¯ä»¥ä¸Žå¤–部æœåŠ¡é›†æˆï¼Œè®©ä½ çš„集群更高效地è¿è¡Œã€‚ + + +## Notifiers 和告警 + +通知器和告警是两个ååŒå·¥ä½œçš„功能，它们å¯ä»¥å°† Rancher 系统中的事件告知你。在å¯ç”¨å®ƒä»¬ä¹‹å‰ï¼Œä½ å¿…须先安装监控应用。 + +通知器是通知你告警事件的æœåŠ¡ã€‚ä½ å¯ä»¥é€šè¿‡é…置通知器æ¥å‘最适åˆé‡‡å–纠正措施的员工å‘é€å‘Šè­¦é€šçŸ¥ã€‚支æŒä½¿ç”¨ Slackã€ç”µå­é‚®ä»¶ã€PagerDutyã€å¾®ä¿¡å’Œ webhook å‘é€é€šçŸ¥ã€‚ + +告警是触å‘这些通知的规则。在接收告警之å‰ï¼Œä½ å¿…须在 Rancher 中é…置一个或多个通知器。你å¯ä»¥åœ¨é›†ç¾¤æˆ–项目级别设置告警范围。 + +## Logging + +Logging 支æŒï¼š + +- 获å–和分æžé›†ç¾¤çš„çŠ¶æ€ +- 在你的环境中å‘现趋势 +- 将日志ä¿å­˜åˆ°é›†ç¾¤å¤–部的安全ä½ç½® +- éšæ—¶äº†è§£å®¹å™¨å´©æºƒã€pod 驱é€æˆ–节点死亡等事件 +- æ›´è½»æ¾åœ°è°ƒè¯•å’Œè§£å†³é—®é¢˜ + +Rancher å¯ä»¥ä¸Ž Elasticsearchã€splunkã€kafkaã€syslog å’Œ fluentd 集æˆã€‚ + +有关详细信æ¯ï¼Œè¯·å‚阅 [Logging](../pages-for-subheaders/logging.md)。 + +## Monitoring + +ä½ å¯ä»¥ä½¿ç”¨ Rancher，通过业界领先并开æºçš„ [Prometheus](https://prometheus.io/) æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [Monitoring](../pages-for-subheaders/monitoring-and-alerting.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..ddf9809440a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3148 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 518 1 0 Feb26 ? 00:03:06 /usr/bin/containerd root 794 1 0 Feb26 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 94093 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7927fe0998264b8fa49789e8e2793a21ea8f22001cb580de46e1fcc01b1d1455 -address /run/k3s/containerd/containerd.sock root 94107 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ccec5bdaf56b4aad4018e8fefabfa103122905ecf4f908dbddaf9fbb985bc5c -address /run/k3s/containerd/containerd.sock root 94188 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d0f840cefd5b5ba804b2795420963e2c43a0c8b7b6b2c796e859c57ba0d38b3f -address /run/k3s/containerd/containerd.sock root 95349 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3849ee071d3f5a8e8a63ee1c4d590f88e0d21bc5fc25e20bbecb1e104b2835d0 -address /run/k3s/containerd/containerd.sock root 95421 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id a608e22cc1520272c91a2aca6a029e662a5d13defc7cf67de6a59594b859c550 -address /run/k3s/containerd/containerd.sock root 96569 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id f46f18b427a026156a41a54daad5c55a7d04bee1d112ae32e9c3d07c8cfc5a4e -address /run/k3s/containerd/containerd.sock root 97820 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id aa5731bfdbd6ff9c87043ccd4e30adc1c384eb8653b97169629598852931375d -address /run/k3s/containerd/containerd.sock root 97868 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 99f90b82253554453d0d1e57a23e770f000cf8972ca751387fd5682e9e1607f2 -address /run/k3s/containerd/containerd.sock root 100124 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 632f8fcf8174c12048354376e1f22e87ac95baa26a46e77f4cb8c1549f00fc4f -address /run/k3s/containerd/containerd.sock root 101049 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 05cc13a10b18601034393b12dcb0b140799cef2f3c7f712de671054cb65a4cff -address /run/k3s/containerd/containerd.sock root 105484 105441 2 21:41 ? 00:00:10 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 107618 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d7baeb7e8d463b1afd831ca4367d4c552c8e87afc29f7cae7cb5bf987c2e4b1f -address /run/k3s/containerd/containerd.sock root 107751 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 8fd7e8824474bb98e13d39dca0e130444b7b6fb917b7dc5e662c5a5f669fe584 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'644' is equal to '644' OR '640' is present OR '600' is present OR '444' is present OR '440' is present OR '400' is present OR '000' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md similarity index 99% rename from versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md index 3e7cd1e1e06..8944de04c27 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md @@ -2,10 +2,6 @@ title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.24 --- - - - - This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md similarity index 99% rename from versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md index a75c59a31ab..1f5b7dfd37b 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md @@ -1,22 +1,18 @@ --- -title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25 +title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.25 --- - - - - This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: | Rancher Version | CIS Benchmark Version | Kubernetes Version | |-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 | +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.25 | This document is for Rancher operators, security teams, auditors and decision makers. -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). ## Testing Methodology diff --git a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md similarity index 99% rename from docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md index a75c59a31ab..16614a7e520 100644 --- a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md @@ -2,10 +2,6 @@ title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25 --- - - - - This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..34bb63f89f0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3083 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/!(*key).pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 5 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 2 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): replicationcontrollers is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicationcontrollers" in API group "" in the namespace "default" Error from server (Forbidden): services is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "services" in API group "" in the namespace "default" Error from server (Forbidden): daemonsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "daemonsets" in API group "apps" in the namespace "default" Error from server (Forbidden): deployments.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "deployments" in API group "apps" in the namespace "default" Error from server (Forbidden): replicasets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicasets" in API group "apps" in the namespace "default" Error from server (Forbidden): statefulsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "statefulsets" in API group "apps" in the namespace "default" Error from server (Forbidden): horizontalpodautoscalers.autoscaling is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "horizontalpodautoscalers" in API group "autoscaling" in the namespace "default" Error from server (Forbidden): cronjobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "cronjobs" in API group "batch" in the namespace "default" Error from server (Forbidden): jobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "jobs" in API group "batch" in the namespace "default" true +``` diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md similarity index 99% rename from versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md index 3d12d66fb51..25108b848b5 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md @@ -2,10 +2,6 @@ title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.24 --- - - - - This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md similarity index 99% rename from versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md index a45fa89c682..0cec041061f 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md @@ -1,11 +1,7 @@ --- -title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25 +title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.25 --- - - - - This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. @@ -13,13 +9,13 @@ This guide corresponds to the following versions of Rancher, CIS Benchmarks, and | Rancher Version | CIS Benchmark Version | Kubernetes Version | |-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 | +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.25 | This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. This document is for Rancher operators, security teams, auditors and decision makers. -For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). ## Testing Methodology diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md similarity index 99% rename from docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md index a45fa89c682..084568cc402 100644 --- a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md @@ -2,10 +2,6 @@ title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25 --- - - - - This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..83cac7d6e59 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3196 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"zF6yWmUNDvLZW5QnHG580wVBYOwq9S+VXJ3G4JjDaEk="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md similarity index 99% rename from versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md rename to i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md index 548d3e92d47..ffb7755d7ad 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md @@ -2,10 +2,6 @@ title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.24 --- - - - - This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md new file mode 100644 index 00000000000..4d524d3fab4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md @@ -0,0 +1,3196 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.25 +--- + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.25 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 15378 3910 99 23:32 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"TSpBkJhIU0sRx+84IZuBZ1qO+eaRdW31C7QCnF3+n8s="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4126 4014 0 23:27 ? 00:00:02 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4126 4014 0 23:27 ? 00:00:02 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** fail + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** true + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md new file mode 100644 index 00000000000..252accbb79d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md @@ -0,0 +1,3196 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25 +--- + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 15378 3910 99 23:32 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"TSpBkJhIU0sRx+84IZuBZ1qO+eaRdW31C7QCnF3+n8s="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4126 4014 0 23:27 ? 00:00:02 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4126 4014 0 23:27 ? 00:00:02 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** fail + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** true + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/kubernetes-security-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/kubernetes-security-best-practices.md new file mode 100644 index 00000000000..af9746e899f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/kubernetes-security-best-practices.md @@ -0,0 +1,11 @@ +--- +title: Kubernetes 安全最佳实践 +--- + +### é™åˆ¶äº‘å…ƒæ•°æ® API 访问 + +AWSã€Azureã€DigitalOcean 或 GCP 等云æ供商通常会在本地å‘实例公开元数æ®æœåŠ¡ã€‚默认情况下，此端点å¯è¢«è¿è¡Œåœ¨äº‘实例上的 pod 访问，包括在托管的 Kubernetes（如 EKSã€AKSã€DigitalOcean Kubernetes 或 GKE）中的 pod，并且å¯ä»¥åŒ…å«è¯¥èŠ‚点的云凭è¯ã€é…置数æ®ï¼ˆå¦‚ kubelet 凭è¯ï¼‰ä»¥åŠå…¶ä»–æ•æ„Ÿæ•°æ®ã€‚为了é™ä½Žåœ¨äº‘å¹³å°ä¸Šè¿è¡Œçš„è¿™ç§é£Žé™©ï¼Œè¯·éµå¾ª [Kubernetes 安全建议](https://kubernetes.io/docs/tasks/administer-cluster/securing-a-cluster/#restricting-cloud-metadata-api-access)，å³é™åˆ¶æŽˆäºˆå®žä¾‹å‡­è¯çš„æƒé™ï¼Œä½¿ç”¨ç½‘络策略é™åˆ¶ pod å¯¹å…ƒæ•°æ® API 的访问，并é¿å…使用é…置数æ®æ¥ä¼ é€’密文。 + +建议你å‚阅你使用的云æ供商的安全最佳实践，获å–é™åˆ¶å¯¹äº‘å®žä¾‹å…ƒæ•°æ® API 访问的建议和详情。 + +è¦èŽ·å–更多å‚考资料，请å‚阅 MITRE ATT&CK 知识库 - [ä¸å®‰å…¨å‡­è¯ï¼šäº‘å®žä¾‹å…ƒæ•°æ® API](https://attack.mitre.org/techniques/T1552/005/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/psa-restricted-exemptions.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/psa-restricted-exemptions.md new file mode 100644 index 00000000000..d623ef71dfb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/psa-restricted-exemptions.md @@ -0,0 +1,62 @@ +--- +title: PodSecurityConfiguration 示例 +--- + +以下 PodSecurityConfiguration 包å«äº† `rancher-restricted` 集群正常è¿è¡Œæ‰€éœ€çš„ Rancher 命å空间è±å…。 + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-elemental-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-resources-system, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator] +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/security-advisories-and-cves.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/security-advisories-and-cves.md new file mode 100644 index 00000000000..c2baac88dcf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/security-advisories-and-cves.md @@ -0,0 +1,41 @@ +--- +title: 安全公告和 CVE +--- + +Rancher 致力于å‘社区披露我们产å“的安全问题。我们会针对已解决的问题å‘布安全公告和 CVE（Common Vulnerabilities and Exposures，通用æ¼æ´žæŠ«éœ²ï¼‰ã€‚Rancher GitHub 上的[安全页é¢](https://github.com/rancher/rancher/security/advisories)也会å‘布新的安全公告。 + +| ID | æè¿° | 日期 | 解决 | +|----|-------------|------|------------| +| [CVE-2023-22648](https://github.com/rancher/rancher/security/advisories/GHSA-vf6j-6739-78m8) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。在用户注销并é‡æ–°ç™»å½•åˆ° Rancher UI 之å‰ï¼ŒAzure AD 中的æƒé™æ›´æ”¹ä¸ä¼šå映给用户。 | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2022-43760](https://github.com/rancher/rancher/security/advisories/GHSA-46v3-ggjg-qq3x) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。攻击者å¯ä»¥é€šè¿‡ Rancher UI 利用多个跨站脚本 (XSS) æ¼æ´žã€‚ | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2020-10676](https://github.com/rancher/rancher/security/advisories/GHSA-8vhc-hwhc-cpj4) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。具有更新命å空间æƒé™çš„用户å¯ä»¥å°†è¯¥å‘½å空间移动到他们无æƒè®¿é—®çš„项目中。 | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22647](https://github.com/rancher/rancher/security/advisories/GHSA-p976-h52c-26p6) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。Standard åŠä»¥ä¸Šç”¨æˆ·èƒ½å¤Ÿå°†ä»–们的æƒé™æå‡ä¸º local 集群中的管ç†å‘˜ã€‚ | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22651](https://github.com/rancher/rancher/security/advisories/GHSA-6m9f-pj6w-w87g) | 由于 webhook 的更新逻辑失败，Rancher 准入 webhook å¯èƒ½ä¼šé…置错误。准入 webhook 在资æºå…许进入 Kubernetes 集群之å‰ä¼šå¼ºåˆ¶æ‰§è¡ŒéªŒè¯è§„则和安全检查。webhook 在é™çº§çŠ¶æ€ä¸‹è¿è¡Œæ—¶å°†ä¸å†éªŒè¯ä»»ä½•èµ„æºï¼Œè¿™å¯èƒ½å¯¼è‡´ä¸¥é‡çš„æƒé™æå‡å’Œæ•°æ®æŸå。 | 2023 å¹´ 4 月 24 æ—¥ | Rancher [v2.7.3](https://github.com/rancher/rancher/releases/tag/v2.7.3) | +| [CVE-2022-43758](https://github.com/rancher/rancher/security/advisories/GHSA-34p5-jp77-fcrc) | 在 Rancher 2.5.0 至 2.5.16ã€2.6.0 至 2.6.9 å’Œ 2.7.0 版本中å‘现了一个问题，Rancher Git 包中存在命令注入æ¼æ´žã€‚这个包使用 Rancher 容器镜åƒä¸­å¯ç”¨çš„底层 Git 二进制文件æ¥æ‰§è¡Œ Git æ“作。特制的命令如果没有消除歧义，å¯èƒ½ä¼šåœ¨é€šè¿‡ Git 执行时造æˆæ··æ·†ï¼Œå¯¼è‡´åœ¨åº•å±‚ Rancher 主机中进行命令注入。 | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43757](https://github.com/rancher/rancher/security/advisories/GHSA-cq4p-vp5q-4522) | æ­¤é—®é¢˜å½±å“ Rancher 2.5.0 到 2.5.16，2.6.0 至 2.6.9 å’Œ 2.7.0。我们å‘现 Rancher 之å‰å‘布的安全公告 [CVE-2021-36782](https://github.com/advisories/GHSA-g7j7-h4q8-8w2f) 没有解决æŸäº›æ•æ„Ÿå­—段ã€Secret Tokenã€åŠ å¯†å¯†é’¥å’Œ SSH 密钥，这些字段ä»ç„¶ä»¥æ˜Žæ–‡å½¢å¼ç›´æŽ¥å­˜å‚¨åœ¨ Kubernetes 上 `Clusters` 之类的对象。在 Rancher 中，集群中已认è¯çš„ `Cluster Owners`ã€`Cluster Members`ã€`Project Owners` å’Œ `Project Members` å¯ä»¥çœ‹åˆ°å…¬å¼€çš„凭è¯ã€‚ | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43755](https://github.com/rancher/rancher/security/advisories/GHSA-8c69-r38j-rpfj) | 在 Rancher 2.6.9 å’Œ 2.7.0 åŠä¹‹å‰çš„版本中å‘现了一个问题。`cattle-cluster-agent` 使用的 `cattle-token` Secret 是å¯é¢„测的。é‡æ–°ç”Ÿæˆ Token 之åŽï¼ŒToken 的值ä¾ç„¶ç›¸åŒã€‚如果 Token 被泄露并且出于安全目的需è¦é‡æ–°åˆ›å»ºï¼Œè¿™å¯èƒ½ä¼šé€ æˆä¸¥é‡çš„问题。Rancher çš„ `cattle-cluster-agent` 使用 `cattle-token` æ¥è¿žæŽ¥åˆ° Rancher é…置的下游集群 Kubernetes API。 | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1) å’Œ [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) | +| [CVE-2022-21953](https://github.com/rancher/rancher/security/advisories/GHSA-g25r-gvq3-wrq7) | 在 Rancher 2.5.16ã€2.6.9 å’Œ 2.7.0 åŠä¹‹å‰çš„版本中å‘现了一个问题。由于授æƒé€»è¾‘缺陷，任何下游集群上ç»è¿‡èº«ä»½è®¤è¯çš„用户都能在 Rancher `local` 集群中打开一个 shell pod (1)，而且对 kubectl 具有有é™çš„访问æƒé™ (2)。预期的行为是：除éžæ˜Žç¡®æŽˆäºˆæƒé™ï¼Œå¦åˆ™ç”¨æˆ·åœ¨ Rancher `local` 集群中没有这样的访问æƒé™ã€‚ | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [GHSA-c45c-39f6-6gw9](https://github.com/rancher/rancher/security/advisories/GHSA-c45c-39f6-6gw9) | æ­¤é—®é¢˜å½±å“ Rancher 2.5.0 到 2.5.16，2.6.0 至 2.6.9 å’Œ 2.7.0。åªä¼šå½±å“é…置了或é…置过外部身份认è¯æ供程åºçš„ Rancher 设置。我们å‘现，当在 Rancher 中é…置外部身份认è¯æ供程åºç„¶åŽå°†å…¶ç¦ç”¨æ—¶ï¼ŒRancher 生æˆçš„ Token 如果关è”了通过现已ç¦ç”¨çš„身份认è¯æ供程åºæŽˆäºˆè®¿é—®æƒé™çš„用户，那么 Token ä¸ä¼šè¢«æ’¤é”€ã€‚ | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-31247](https://github.com/rancher/rancher/security/advisories/GHSA-6x34-89p7-95wg) | 在 Rancher 2.5.15 å’Œ 2.6.6 åŠä¹‹å‰çš„版本中å‘现了一个问题。授æƒé€»è¾‘缺陷å…许在下游集群中通过集群角色模æ¿ç»‘定 (CRTB) 和项目角色模æ¿ç»‘定 (PRTB) æ¥æå‡æƒé™ã€‚任何有æƒé™åˆ›å»º/编辑 CRTB 或 PRTB 的用户（例如 `cluster-owner`ã€`manage cluster members`ã€`project-owner` å’Œ `manage project members`）都å¯ä»¥åˆ©ç”¨è¯¥æ¼æ´žï¼Œåœ¨åŒä¸€é›†ç¾¤çš„å¦ä¸€ä¸ªé¡¹ç›®æˆ–ä¸åŒä¸‹æ¸¸é›†ç¾¤çš„å¦ä¸€ä¸ªé¡¹ç›®ä¸­èŽ·å¾—所有者æƒé™ã€‚ | 2022 å¹´ 8 月 18 æ—¥ | [Rancher 2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) å’Œ [Rancher 2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36783](https://github.com/rancher/rancher/security/advisories/GHSA-8w87-58w6-hfv8) | 2.5.12 到 2.6.3 çš„ Rancher 版本无法正确清ç†é›†ç¾¤æ¨¡æ¿ answer 中的凭è¯ã€‚此错误å¯èƒ½ä¼šå¯¼è‡´æ˜Žæ–‡å­˜å‚¨ä»¥åŠå‡­è¯ã€å¯†ç å’Œ API 令牌被暴露。在 Rancher 中，已认è¯çš„ `Cluster Owner`ã€`Cluster Member`ã€`Project Owner` å’Œ `Project Member` å¯ä»¥åœ¨ `/v1/management.cattle.io.clusters`ã€`/v3/clusters` å’Œ `/k8s/clusters/local/apis/management.cattle.io/v3/clusters` 端点上看到暴露的凭è¯ã€‚ | 2022 å¹´ 8 月 18 æ—¥ | [Rancher 2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) å’Œ [Rancher 2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36782](https://github.com/rancher/rancher/security/advisories/GHSA-g7j7-h4q8-8w2f) | 在 2.5.15 到 2.6.6 çš„ Rancher 版本中å‘现了一个问题，其中密ç ã€API 密钥和 Rancher çš„ ServiceAccount 令牌（用于é…置集群）等æ•æ„Ÿå­—段直接以明文形å¼å­˜å‚¨åœ¨ `Cluster` ç­‰ Kubernetes 对象上（例如，`cluster.management.cattle.io`ï¼‰ã€‚ä»»ä½•èƒ½å¤Ÿè¯»å– Kubernetes API 中的对象的用户都å¯ä»¥æ£€ç´¢è¿™äº›æ•æ„Ÿæ•°æ®çš„明文版本。该问题由 Florian Struck（æ¥è‡ª [Continum AG](https://www.continum.net/)）和 [Marco Stuurman](https://github.com/fe-ax)（æ¥è‡ª [Shock Media B.V.](https://www.shockmedia.nl/)）å‘现并报告。 | 2022 å¹´ 8 月 18 æ—¥ | [Rancher 2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) å’Œ [Rancher 2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2022-21951](https://github.com/rancher/rancher/security/advisories/GHSA-vrph-m5jj-c46c) | æ­¤æ¼æ´žä»…å½±å“通过 [RKE 模æ¿](../../pages-for-subheaders/about-rke1-templates.md)é…ç½® [Weave](../../faq/container-network-interface-providers.md#weave) å®¹å™¨ç½‘ç»œæŽ¥å£ (CNI) 的客户。在 Rancher 2.5.0 到 2.5.13 å’Œ Rancher 2.6.0 到 2.6.4 版本中å‘现了一个æ¼æ´žã€‚如果将 CNI 选为 Weave，RKE 模æ¿çš„ç”¨æˆ·ç•Œé¢ (UI) ä¸åŒ…括 Weave 密ç çš„值。如果基于上述模æ¿åˆ›å»ºé›†ç¾¤ï¼Œå¹¶ä¸”å°† Weave é…置为 CNI，则 Weave 中ä¸ä¼šä¸º[网络加密](https://www.weave.works/docs/net/latest/tasks/manage/security-untrusted-networks/)创建密ç ã€‚因此，集群中的网络æµé‡å°†ä¸åŠ å¯†å‘é€ã€‚ | 2022 å¹´ 5 月 24 æ—¥ | [Rancher 2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) å’Œ [Rancher 2.5.14](https://github.com/rancher/rancher/releases/tag/v2.5.14) | +| [CVE-2021-36784](https://github.com/rancher/rancher/security/advisories/GHSA-jwvr-vv7p-gpwq) | 在 Rancher 2.5.0 到 2.5.12 å’Œ Rancher 2.6.0 到 2.6.3 中å‘现了一个æ¼æ´žï¼Œè¯¥æ¼æ´žå…许能创建或更新[全局角色](../../pages-for-subheaders/manage-role-based-access-control-rbac.md)的用户将他们或其他用户å‡çº§ä¸ºç®¡ç†å‘˜ã€‚全局角色能授予用户 Rancher 级别的æƒé™ï¼Œä¾‹å¦‚能创建集群。在已识别的 Rancher 版本中，如果用户被授予了编辑或创建全局角色的æƒé™ï¼Œä»–们ä¸ä»…仅能授予他们已ç»æ‹¥æœ‰çš„æƒé™ã€‚æ­¤æ¼æ´žå½±å“使用能够创建或编辑全局角色的éžç®¡ç†å‘˜ç”¨æˆ·çš„客户。此场景最常è§çš„用例是 `restricted-admin` 角色。 | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) å’Œ [Rancher 2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-4200](https://github.com/rancher/rancher/security/advisories/GHSA-hx8w-ghh8-r4xf) | æ­¤æ¼æ´žä»…å½±å“在 Rancher 中使用 `restricted-admin` 角色的客户。在 Rancher 2.5.0 到 2.5.12 å’Œ 2.6.0 到 2.6.3 中å‘现了一个æ¼æ´žï¼Œå…¶ä¸­ `cattle-global-data` 命å空间中的 `global-data` 角色授予了应用商店的写æƒé™ã€‚由于具有任何级别的应用商店访问æƒé™çš„用户都会绑定到 `global-data` è§’è‰²ï¼Œå› æ­¤è¿™äº›ç”¨æˆ·éƒ½èƒ½å†™å…¥æ¨¡æ¿ `CatalogTemplates`) 和模æ¿ç‰ˆæœ¬ (`CatalogTemplateVersions`)。在 Rancher 中创建的新用户默认分é…到 `user` 角色（普通用户），该角色本ä¸è¯¥å…·æœ‰å†™å…¥åº”用商店的æƒé™ã€‚æ­¤æ¼æ´žæå‡äº†èƒ½å†™å…¥åº”用商店模æ¿å’Œåº”用商店模æ¿ç‰ˆæœ¬èµ„æºçš„用户的æƒé™ã€‚ | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) å’Œ [Rancher 2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [GHSA-wm2r-rp98-8pmh](https://github.com/rancher/rancher/security/advisories/GHSA-wm2r-rp98-8pmh) | æ­¤æ¼æ´žä»…å½±å“使用ç»è¿‡è®¤è¯çš„ Git å’Œ/或 Helm 仓库通过 [Fleet](../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) 进行æŒç»­äº¤ä»˜çš„客户。在 [`v1.5.11`](https://github.com/hashicorp/go-getter/releases/tag/v1.5.11) 之å‰ç‰ˆæœ¬ä¸­çš„ `go-getter` 库中å‘现了一个问题，错误消æ¯ä¸­æ²¡æœ‰åˆ é™¤ Base64 ç¼–ç çš„ SSH ç§é’¥ï¼Œå¯¼è‡´è¯¥ä¿¡æ¯æš´éœ²ã€‚Rancher 中 [`v0.3.9`](https://github.com/rancher/fleet/releases/tag/v0.3.9) 之å‰çš„ Fleet 版本使用了该库的æ¼æ´žç‰ˆæœ¬ã€‚æ­¤é—®é¢˜å½±å“ Rancher 2.5.0 到 2.5.12（包括 2.5.12ï¼‰ä»¥åŠ 2.6.0 到 2.6.3（包括 2.6.3）。该问题由 Raft Engineering çš„ Dagan Henderson å‘现并报告。 | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) å’Œ [Rancher 2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-36778](https://github.com/rancher/rancher/security/advisories/GHSA-4fc7-hc63-7fjg) | 在 Rancher 2.5.0 到 2.5.11 å’Œ Rancher 2.6.0 到 2.6.2 中å‘现了一个æ¼æ´žï¼Œå½“从é…置的ç§æœ‰ä»“库下载 Helm Chart 时，对åŒæºç­–略的检查ä¸è¶³å¯èƒ½å¯¼è‡´ä»“库凭è¯æš´éœ²ç»™ç¬¬ä¸‰æ–¹æ供商。仅当用户在 Rancher çš„`应用 & 应用市场 > 仓库`中é…ç½®ç§æœ‰ä»“库的访问凭è¯æ—¶æ‰ä¼šå‡ºçŽ°æ­¤é—®é¢˜ã€‚该问题由 Martin Andreas Ullrich å‘现并报告。 | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) å’Œ [Rancher 2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) | +| [GHSA-hwm2-4ph6-w6m5](https://github.com/rancher/rancher/security/advisories/GHSA-hwm2-4ph6-w6m5) | 在 Rancher 2.0 到 2.6.3 中å‘现了一个æ¼æ´žã€‚Rancher æ供的 `restricted` Pod 安全策略（PSP）与 Kubernetes æ供的上游 `restricted` 策略有差别，因此 Rancher çš„ PSP å°† `runAsUser` 设置为 `runAsAny`，而上游将 `runAsUser` 设置为 `MustRunAsNonRoot`。因此，å³ä½¿ Rancher çš„ `restricted` 策略是在项目或集群级别上强制执行的，容器也å¯ä»¥ä»¥ä»»ä½•ç”¨æˆ·èº«ä»½è¿è¡Œï¼ŒåŒ…括特æƒç”¨æˆ· (`root`)。 | 2022 å¹´ 3 月 31 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | +| [CVE-2021-36775](https://github.com/rancher/rancher/security/advisories/GHSA-28g7-896h-695v) | 在 Rancher 2.4.17ã€2.5.11 å’Œ 2.6.2 以åŠæ›´é«˜çš„版本中å‘现了一个æ¼æ´žã€‚从项目中删除与æŸä¸ªç»„å…³è”çš„`项目角色`åŽï¼Œèƒ½è®©è¿™äº›ä½¿ç”¨è€…访问集群级别资æºçš„绑定（Binding）ä¸ä¼šè¢«åˆ é™¤ã€‚导致问题的原因是ä¸å®Œæ•´çš„授æƒé€»è¾‘检查。如果用户是å—å½±å“组中的æˆå‘˜ï¼Œä¸”能对 Rancher 进行认è¯è®¿é—®ï¼Œé‚£ä¹ˆç”¨æˆ·å¯ä»¥åˆ©ç”¨æ­¤æ¼æ´žè®¿é—®ä»–们ä¸åº”该能访问的资æºã€‚暴露级别å–决于å—å½±å“项目角色的原始æƒé™çº§åˆ«ã€‚æ­¤æ¼æ´žä»…å½±å“在 Rancher 中基于组进行身份验è¯çš„客户。 | 2022 å¹´ 3 月 31 æ—¥ | [Rancher 2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3)ã€[Rancher 2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) å’Œ [Rancher 2.4.18](https://github.com/rancher/rancher/releases/tag/v2.4.18) | +| [CVE-2021-36776](https://github.com/rancher/rancher/security/advisories/GHSA-gvh9-xgrq-r8hw) | 在 Rancher 2.5.0 到 2.5.9 中å‘现了一个æ¼æ´žï¼Œè¯¥æ¼æ´žå…许ç»è¿‡è®¤è¯ç”¨æˆ·é€šè¿‡ API 代ç†æ¨¡æ‹Ÿé›†ç¾¤ä¸Šçš„任何用户，而无需知é“被模拟用户的凭è¯ã€‚问题的原因是 API 代ç†åœ¨å°†è¯·æ±‚å‘é€åˆ° Kubernetes API 之å‰æœªåˆ é™¤æ¨¡æ‹Ÿæ ‡å¤´ã€‚能认è¯è®¿é—® Rancher çš„æ¶æ„用户å¯ä»¥å†’å……å¦ä¸€ä¸ªåœ¨ Rancher 认è¯ç”¨æˆ·ï¼Œä»Žè€Œå¯¹é›†ç¾¤è¿›è¡Œç®¡ç†å‘˜çº§åˆ«çš„访问。 | 2022 å¹´ 3 月 31 æ—¥ | [Rancher 2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) å’Œ [Rancher 2.5.10](https://github.com/rancher/rancher/releases/tag/v2.5.10) | +| [CVE-2021-25318](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25318) | Rancher 2.0 çš„ä¸å¯ç¼–辑版本å‘现了一个æ¼æ´žï¼Œåœ¨è¯¥ç‰ˆæœ¬ä¸­ï¼Œæ— è®ºèµ„æºçš„ API 组如何，用户都å¯ä»¥è®¿é—®èµ„æºã€‚例如，Rancher 应该å…许用户访问 `apps.catalog.cattle.io`，但å´é”™è¯¯åœ°æŽˆäºˆäº†å¯¹ `apps.*` 的访问æƒé™ã€‚ä½ å¯ä»¥åœ¨[这里](https://github.com/rancher/rancher/security/advisories/GHSA-f9xf-jq4j-vqw4)找到**下游集群**å’Œ **Rancher 管ç†é›†ç¾¤**中å—å½±å“的资æºã€‚除了å‡çº§åˆ°æ‰“了补ä¸çš„ Rancher 版本之外，暂时没有直接的缓解措施。 | 2021 å¹´ 7 月 14 æ—¥ | [Rancher 2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) å’Œ [Rancher 2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-31999](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-31999) | Rancher 2.0.0 çš„è¡¥ä¸ç‰ˆæœ¬å‘现了一个æ¼æ´žï¼Œæ¶æ„çš„ Rancher 用户å¯ä»¥é’ˆå¯¹æ‰˜ç®¡é›†ç¾¤çš„ Kubernetes API 的代ç†å‘起一个 API 请求，以获å–他们无æƒè®¿é—®çš„ä¿¡æ¯ã€‚这是通过在 Connection 标头中传递 “Impersonate-User†或 “Impersonate-Group†标头æ¥å®žçŽ°çš„，然åŽä»£ç†ä¼šåˆ é™¤è¯¥æ ‡å¤´ã€‚此时，请求ä¸ä¼šæ¨¡æ‹Ÿç”¨æˆ·åŠå…¶æƒé™ï¼Œè€Œæ˜¯ä¼šç±»ä¼¼ Rancher management server 的请求，并错误地返回信æ¯ã€‚该æ¼æ´žä»…å½±å“对集群具有一定级别æƒé™çš„ Rancher 用户。除了å‡çº§åˆ°æ‰“了补ä¸çš„ Rancher 版本之外，暂时没有直接的缓解措施。 | 2021 å¹´ 7 月 14 æ—¥ | [Rancher 2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) å’Œ [Rancher 2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25320](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25320) | Rancher 2.2.0 çš„è¡¥ä¸ç‰ˆæœ¬å‘现了一个æ¼æ´žï¼Œäº‘凭è¯æ²¡æœ‰æ­£ç¡®é€šè¿‡ Rancher API 验è¯ã€‚具体地说，是通过用于与云æ供商通信的代ç†ã€‚任何登录并具有有效云æä¾›å•†äº‘å‡­è¯ ID çš„ Rancher 用户都å¯ä»¥é€šè¿‡ä»£ç† API 调用该云æ供商的 API，并且云凭è¯ä¼šè¢«ç»‘定。该æ¼æ´žä»…å½±å“有效的 Rancher 用户。除了å‡çº§åˆ°æ‰“了补ä¸çš„ Rancher 版本之外，暂时没有直接的缓解措施。 | 2021 å¹´ 7 月 14 æ—¥ | [Rancher 2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) å’Œ [Rancher 2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25313](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25313) | 所有 Rancher 2 版本上都å‘现了一个安全æ¼æ´žã€‚使用æµè§ˆå™¨è®¿é—® Rancher API 时，URL 没有正确转义，导致它容易å—到 XSS 攻击。这些 API 端点的特制 URL å¯èƒ½åŒ…括嵌入页é¢å¹¶åœ¨æµè§ˆå™¨ä¸­æ‰§è¡Œçš„ JavaScript。暂时没有直接的缓解措施。请ä¸è¦å•å‡»æŒ‡å‘ Rancher Server çš„ä¸å—信任链接。 | 2021 å¹´ 3 月 2 æ—¥ | [Rancher v2.5.6](https://github.com/rancher/rancher/releases/tag/v2.5.6)ã€[Rancher v2.4.14](https://github.com/rancher/rancher/releases/tag/v2.4.14) å’Œ [Rancher v2.3.11](https://github.com/rancher/rancher/releases/tag/v2.3.11) | +| [CVE-2019-14435](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14435) | æ­¤æ¼æ´žè®©å·²éªŒè¯çš„用户å¯ä»¥é€šè¿‡ Rancher 使用的系统æœåŠ¡å®¹å™¨å¯è®¿é—®çš„ IP æå–ç§æœ‰æ•°æ®ã€‚这包括但ä¸é™äºŽäº‘æ供商元数æ®æœåŠ¡ç­‰æœåŠ¡ã€‚虽然 Rancher å…许用户为系统æœåŠ¡é…置白åå•åŸŸï¼Œä½†è¿™ä¸ªæ¼æ´žä»ç„¶å¯ä»¥è¢«ç²¾å¿ƒè®¾è®¡çš„ HTTP 请求利用。该问题由 Workiva çš„ Matt Belisle å’Œ Alex Stevenson å‘现并报告。 | 2019 å¹´ 8 月 5 æ—¥ | [Rancher 2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) å’Œ [Rancher 2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-14436](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14436) | 该æ¼æ´žå…许有æƒç¼–辑角色绑定的项目æˆå‘˜ä¸ºè‡ªå·±æˆ–其他用户分é…集群级别的角色，从而授予他们对该集群的管ç†å‘˜è®¿é—®æƒé™ã€‚该问题由 Nokia çš„ Michal Lipinski å‘现并报告。 | 2019 å¹´ 8 月 5 æ—¥ | [Rancher 2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) å’Œ [Rancher 2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-13209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-13209) | 该æ¼æ´žè¢«ç§°ä¸º[跨网页 Websocket 劫æŒæ”»å‡»](https://www.christian-schneider.net/CrossSiteWebSocketHijacking.html)。该攻击å…许攻击者以å—害用户的角色/æƒé™è®¿é—®ç”± Rancher 管ç†çš„集群。它让å—害用户登录到 Rancher Server，然åŽè®¿é—®ç”±æ”»å‡»è€…托管的第三方站点。完æˆåŽï¼Œæ”»å‡»è€…å°±å¯ä»¥ä½¿ç”¨å—害用户的æƒé™å’Œèº«ä»½å¯¹ Kubernetes API 执行命令。该问题由 Workiva çš„ Matt Belisle å’Œ Alex Stevenson 报告。 | 2019 å¹´ 7 月 15 æ—¥ | [Rancher 2.2.5](https://github.com/rancher/rancher/releases/tag/v2.2.5)ã€[Rancher 2.1.11](https://github.com/rancher/rancher/releases/tag/v2.1.11) å’Œ [Rancher 2.0.16](https://github.com/rancher/rancher/releases/tag/v2.0.16) | +| [CVE-2019-12303](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12303) | 项目所有者å¯ä»¥æ³¨å…¥é¢å¤–çš„ fluentd 日志é…置，从而在 fluentd 容器内读å–文件或执行任æ„命令。该问题由 Untamed Theory çš„ Tyler Welton 报告。 | 2019 å¹´ 6 月 5 æ—¥ | [Rancher 2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4)ã€[Rancher 2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) å’Œ [Rancher 2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-12274](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12274) | 如果节点使用的内置主机驱动使用了文件路径选项，则节点å¯ä»¥è¯»å– Rancher Server 容器内的任æ„文件，包括æ•æ„Ÿæ–‡ä»¶ã€‚ | 2019 å¹´ 6 月 5 æ—¥ | [Rancher 2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4)ã€[Rancher 2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) å’Œ [Rancher 2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-11202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11202) | å³ä½¿å·²è¢«æ˜¾å¼åˆ é™¤ï¼ŒRancher 的默认管ç†å‘˜ä¼šåœ¨ Rancher é‡å¯æ—¶é‡æ–°åˆ›å»ºã€‚ | 2019 å¹´ 4 月 16 æ—¥ | [Rancher 2.2.2](https://github.com/rancher/rancher/releases/tag/v2.2.2)ã€[Rancher 2.1.9](https://github.com/rancher/rancher/releases/tag/v2.1.9) å’Œ [Rancher 2.0.14](https://github.com/rancher/rancher/releases/tag/v2.0.14) | +| [CVE-2019-6287](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6287) | 如果将项目æˆå‘˜æ·»åŠ åˆ°å¤šä¸ªé¡¹ç›®ä¸­ï¼Œåˆ™æˆå‘˜è¿˜èƒ½ç»§ç»­è®¿é—®è¢«åˆ é™¤çš„项目中的命å空间。 | 2019 å¹´ 1 月 29 æ—¥ | [Rancher 2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) å’Œ [Rancher 2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) | +| [CVE-2018-20321](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-20321) | 任何有æƒè®¿é—® `default` 命å空间的项目æˆå‘˜éƒ½å¯ä»¥åœ¨ pod 中挂载 `netes-default` ServiceAccount，然åŽä½¿ç”¨è¯¥ pod 对 Kubernetes 集群执行管ç†ç‰¹æƒå‘½ä»¤ã€‚ | 2019 å¹´ 1 月 29 æ—¥ | [Rancher 2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) å’Œ [Rancher 2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) - 对于这些版本或更高版本，我们有对应的[回滚说明](../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md)。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md new file mode 100644 index 00000000000..5146e3aeb0c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md @@ -0,0 +1,66 @@ +--- +title: 关于 rancher-selinux +--- + +è¦è®© Rancher 使用 SELinux，你必须手动为 SELinux 节点å¯ç”¨ä¸€äº›åŠŸèƒ½ã€‚为了解决这个问题，Rancher æ供了一个 SELinux RPM。 + +`rancher-selinux` RPM ä»…åŒ…å« [rancher-logging 应用程åº](https://github.com/rancher/charts/tree/dev-v2.5/charts/rancher-logging)的策略。 + +`rancher-selinux` çš„ GitHub 仓库在[这里](https://github.com/rancher/rancher-selinux)。 + +## 安装 rancher-selinux RPM + +:::note è¦æ±‚ + +rancher-selinux RPM 已在 CentOS 7 å’Œ 8 上进行了测试。 + +::: + +### 1. 设置 yum 仓库 + +设置 yum 仓库，从而直接在集群中的所有主机上安装 `rancher-selinux`。 + +è¦ä½¿ç”¨ RPM 仓库，在 CentOS 7 或 RHEL 7 系统上è¿è¡Œä»¥ä¸‹ bash 代ç ç‰‡æ®µï¼š + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/7/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` + +è¦ä½¿ç”¨ RPM 仓库，在 CentOS 8 或 RHEL 8 系统上è¿è¡Œä»¥ä¸‹ bash 代ç ç‰‡æ®µï¼š + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/8/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` +### 2. 安装 RPM + +安装 RPM： + +``` +yum -y install rancher-selinux +``` + +## é…ç½® Logging 应用程åºä»¥ä½¿ç”¨ SELinux + +:::note è¦æ±‚ + +Logging v2 已在 RHEL/CentOS 7 å’Œ 8 上使用 SELinux 进行了测试。 + +::: + +在主机上安装 `rancher-selinux` RPM åŽï¼Œåº”用程åºä¸ä¼šè‡ªåŠ¨è¿è¡Œã€‚它们需è¦åœ¨ RPM æ供的å…许的 SELinux 容器域中è¿è¡Œã€‚ + +è¦å°† `rancher-logging` Chart é…ç½®ä¸ºæ”¯æŒ SELinux，请在安装 Chart 时将 `values.yaml` 中的 `global.seLinux.enabled` 更改为 true。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md new file mode 100644 index 00000000000..b5e4a22cad9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md @@ -0,0 +1,9 @@ +--- +title: 关于 rke2-selinux +--- + +`rke2-selinux` 为 RKE2 æ供策略。如果 RKE2 安装程åºè„šæœ¬æ£€æµ‹åˆ°å®ƒè¿è¡Œåœ¨åŸºäºŽ RPM çš„å‘行版上，它会自动安装。 + +`rke2-selinux` çš„ GitHub 仓库在[这里](https://github.com/rancher/rke2-selinux)。 + +有关在å¯ç”¨ SELinux 的主机上安装 RKE2 的更多信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/install/methods#rpm)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-webhook.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-webhook.md new file mode 100644 index 00000000000..caa0335ca2a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rancher-webhook.md @@ -0,0 +1,36 @@ +--- +title: Rancher Webhook +--- + +Rancher-Webhook 是 Rancher çš„é‡è¦ç»„件，它与 Kubernetes 结åˆä½¿ç”¨ï¼Œç”¨äºŽå¢žå¼ºå®‰å…¨æ€§å¹¶ä¸º Rancher 管ç†çš„集群å¯ç”¨å…³é”®åŠŸèƒ½ã€‚ + +如 [Kubernetes 文档](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)中所述，它与 Kubernetes çš„å¯æ‰©å±•å‡†å…¥æŽ§åˆ¶å™¨é›†æˆï¼Œå…许 Rancher-Webhook 检查å‘é€åˆ° Kubernetes API Server 的特定请求，添加自定义和 Rancher 相关的验è¯ï¼Œä»¥åŠ Rancher 相关请求的å˜åŒ–。Rancher-Webhook 使用 `rancher.cattle.io` `ValidatingWebhookConfiguration` å’Œ `rancher.cattle.io` `MutatingWebhookConfiguration` 管ç†è¦éªŒè¯çš„资æºï¼Œå¹¶è¦†ç›–任何手动编辑。 +Rancher å°† Rancher-Webhook 作为å•ç‹¬çš„ deployment å’ŒæœåŠ¡éƒ¨ç½²åœ¨ local 和下游集群中。Rancher 使用 Helm ç®¡ç† Rancher-Webhook。需è¦æ³¨æ„的是，Rancher å¯èƒ½ä¼šè¦†ç›–用户对 Helm 版本所åšçš„修改。 + +## 为什么我们需è¦å®ƒï¼Ÿ + +Rancher-Webhook 对于让 Rancher ä¿æŠ¤é›†ç¾¤å…å—æ¶æ„攻击并å¯ç”¨å„ç§åŠŸèƒ½è‡³å…³é‡è¦ã€‚ +Rancher ä¾èµ– Rancher-Webhook 作为其功能的组æˆéƒ¨åˆ†ã€‚如果没有 Webhook，Rancher å°†ä¸æ˜¯ä¸€ä¸ªå®Œæ•´çš„产å“。 +它为 Rancher 管ç†çš„集群æ供了必è¦çš„ä¿æŠ¤ï¼Œé˜²æ­¢å®‰å…¨æ¼æ´žå¹¶ç¡®ä¿é›†ç¾¤çš„一致性和稳定性。 + +## 常è§é—®é¢˜ + +### 带有 Calico CNI çš„ EKS 集群 + +当 Kubernetes API Server å°è¯•è”ç³» Rancher-Webhook 时，使用 Calico CNI è¿è¡Œ EKS 集群的用户å¯èƒ½ä¼šé‡åˆ°é”™è¯¯ã€‚ +æ ¹æ® [Calico 文档](https://docs.tigera.io/calico/latest/getting-started/kubernetes/managed-public-cloud/eks#install-eks-with-calico-networking)，此问题的解决方法是为 Webhook 部署设置 `hostNetwork=true`。用户å¯ä»¥åœ¨å—å½±å“的集群上使用下é¢çš„ Helm 命令更改此设置。 + +```bash +helm repo add rancher-charts https://charts.rancher.io +helm upgrade --reuse-values rancher-webhook rancher-chart/rancher-webhook -n cattle-system --set global.hostNetwork=true +``` +**注æ„**：这个临时解决方法å¯èƒ½ä¼šè¿å环境的安全策略。此解决方法还è¦æ±‚ä¸»æœºç½‘ç»œä¸Šæœªä½¿ç”¨ç«¯å£ 9443。 + +### ç§æœ‰ GKE 集群 + +使用ç§æœ‰ GKE 集群时å¯èƒ½ä¼šå‘生错误，导致 Kubernetes API Server 无法与 Webhook 通信。以下错误消æ¯å¯èƒ½ä¼šå‡ºçŽ°ï¼š + +``` +Internal error occurred: failed calling webhook "rancher.cattle.io.namespaces.create-non-kubesystem": failed to call webhook: Post "https://rancher-webhook.cattle-system.svc:443/v1/webhook/validation/namespaces?timeout=10s": context deadline exceeded +``` +出现此问题的原因是防ç«å¢™è§„则é™åˆ¶äº† API Server 与ç§æœ‰é›†ç¾¤ä¹‹é—´çš„通信。è¦è§£å†³æ­¤é€šä¿¡é—®é¢˜ï¼Œä½ å¿…须通过添加防ç«å¢™è§„则æ¥å…许 GKE Control Plane é€šè¿‡ç«¯å£ 9443 与 Rancher-Webhook 进行通信。请å‚阅 [GKE 文档](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules)，了解更新防ç«å¢™è§„则的详细信æ¯å’Œæ­¥éª¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rke1-template-example-yaml.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rke1-template-example-yaml.md new file mode 100644 index 00000000000..5efdc493a95 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/rke1-template-example-yaml.md @@ -0,0 +1,111 @@ +--- +title: RKE1 示例 YAML +--- + +以下是一个供å‚考的 RKE 模æ¿é…置文件示例。 + +RKE 模æ¿ä¸­çš„ YAML 使用与创建 RKE 集群时相åŒçš„自定义项。但是，由于 YAML ä½äºŽ Rancher é…置的 RKE 集群的上下文中，因此 RKE 文档中的自定义项需è¦åµŒå¥—在 `rancher_kubernetes_engine` 指令下。 + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker + +enable_cluster_alerting: false +# This setting is not enforced. Clusters +# created with this sample template +# would have alerting turned off by default, +# but end users could still turn alerting +# on or off. + +enable_cluster_monitoring: true +# This setting is not enforced. Clusters +# created with this sample template +# would have monitoring turned on +# by default, but end users could still +# turn monitoring on or off. + +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # ç›®å‰ä»…æ”¯æŒ Nginx ingress provider +# # è¦ç¦ç”¨ Ingress controller，设置 `provider: none` +# # è¦åœ¨æŒ‡å®šèŠ‚点上ç¦ç”¨ Ingress，使用 node_selector，例如： +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/single-node-rancher-in-docker/advanced-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/single-node-rancher-in-docker/advanced-options.md new file mode 100644 index 00000000000..3eae5197503 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/single-node-rancher-in-docker/advanced-options.md @@ -0,0 +1,105 @@ +--- +title: Docker 安装高级选项 +--- + +### 自定义 CA è¯ä¹¦ + +如需 Rancher 在验è¯æœåŠ¡æ—¶ä½¿ç”¨ CA æ ¹è¯ä¹¦ï¼Œè¯·åœ¨å¯åŠ¨ Rancher å®¹å™¨æ—¶å…±äº«åŒ…å« CA æ ¹è¯ä¹¦çš„目录。 + +使用命令示例æ¥å¯åŠ¨æŒ‚载了ç§æœ‰ CA è¯ä¹¦çš„ Rancher 容器。 + +- å·æ ‡å¿— (`-v`) åº”æŒ‡å®šåŒ…å« CA æ ¹è¯ä¹¦çš„主机目录。 +- 环境å˜é‡æ ‡å¿— (`-e`) 与 `SSL_CERT_DIR` 和目录共åŒå£°æ˜Žä¸€ä¸ªçŽ¯å¢ƒå˜é‡ï¼Œè¯¥å˜é‡æŒ‡å®šå®¹å™¨å†…挂载了 CA æ ¹è¯ä¹¦çš„目录ä½ç½®ã€‚ +- ä½ å¯ä½¿ç”¨ `-e KEY=VALUE` 或 `--env KEY=VALUE`将环境å˜é‡ä¼ ç»™ Rancher 容器。 +- ä½ å¯ä½¿ç”¨ `-v host-source-directory:container-destination-directory` 或 `--volume host-source-directory:container-destination-directory`在容器内挂载主机目录。 + +在以下示例将ä½äºŽ `/host/certs` 主机目录中的 CA è¯ä¹¦ï¼ŒæŒ‚载到 Rancher 容器内的 `/container/certs` 上。 + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /host/certs:/container/certs \ + -e SSL_CERT_DIR="/container/certs" \ + --privileged \ + rancher/rancher:latest +``` + +### API 审计日志 + +API 审计日志记录了通过 Rancher Server 进行的所有用户和系统事务。 + +默认情况下，API 审计日志会写入到 Rancher 容器内的 `/var/log/auditlog`。你å¯å°†æ­¤ç›®å½•å…±äº«ä¸ºå·ï¼Œå¹¶è®¾ç½® `AUDIT_LEVEL` 以å¯ç”¨æ—¥å¿—。 + +有关更多信æ¯å’Œé€‰é¡¹ï¼Œè¯·å‚è§ [API 审计日志](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md)。 + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /var/log/rancher/auditlog:/var/log/auditlog \ + -e AUDIT_LEVEL=1 \ + --privileged \ + rancher/rancher:latest +``` + +### TLS 设置 + +如需使用ä¸åŒçš„ TLS é…置，你å¯ä½¿ç”¨ `CATTLE_TLS_MIN_VERSION` å’Œ `CATTLE_TLS_CIPHERS` 环境å˜é‡ã€‚例如，将 TLS 1.0 设为å¯æŽ¥å—的最低 TLS 版本，如下： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_TLS_MIN_VERSION="1.0" \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +å‚è§ [TLS 设置](../../getting-started/installation-and-upgrade/installation-references/tls-settings.md)了解更多信æ¯å’Œé€‰é¡¹ã€‚ + +### 离线环境 + +如果è¦è®¿é—®æ­¤é¡µé¢ä»¥å®Œæˆç¦»çº¿å®‰è£…，在è¿è¡Œå®‰è£…命令时，你必须把ç§æœ‰é•œåƒä»“库的 URL 添加到 Server 标志å‰é¢ã€‚例如，将带有ç§æœ‰é•œåƒä»“库 URL çš„ `` 添加到 `rancher/rancher:latest` å‰é¢ã€‚ + +**示例**： + + /rancher/rancher:latest + +### æŒä¹…åŒ–æ•°æ® + +Rancher 使用 etcd 作为数æ®å­˜å‚¨ã€‚如果 Rancher 是使用 Docker 安装的，Rancher ä¼šä½¿ç”¨åµŒå…¥å¼ etcd。æŒä¹…化数æ®ä½äºŽå®¹å™¨çš„ `/var/lib/rancher` 路径中。 + +ä½ å¯ä»¥å°†ä¸»æœºå·æŒ‚载到该ä½ç½®ï¼Œæ¥å°†æ•°æ®ä¿ç•™åœ¨è¿è¡Œå®ƒçš„主机上： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +### 在åŒä¸€ä¸ªèŠ‚点中è¿è¡Œ `rancher/rancher` å’Œ `rancher/rancher-agent` + +如需使用å•ä¸ªèŠ‚点è¿è¡Œ Rancher 并将åŒä¸€ä¸ªèŠ‚点添加到集群，你必须调整映射给 `rancher/rancher` 容器的主机端å£ã€‚ + +å¦‚æžœå°†èŠ‚ç‚¹æ·»åŠ åˆ°é›†ç¾¤ä¸­ï¼ŒèŠ‚ç‚¹å°†éƒ¨ç½²ä½¿ç”¨ç«¯å£ 80 å’Œ 443 çš„ NGINX Ingress Controller。而这将与我们建议用于暴露 `rancher/rancher` 容器的默认端å£å†²çªã€‚ + +请知悉我们ä¸å»ºè®®å°†æ­¤è®¾ç½®ç”¨äºŽç”Ÿäº§çŽ¯å¢ƒï¼Œè¿™ç§æ–¹å¼ä»…用æ¥æ–¹ä¾¿è¿›è¡Œå¼€å‘/演示。 + +如需更改主机端å£æ˜ å°„，将 `-p 80:80 -p 443:443` 替æ¢ä¸º `-p 8080:80 -p 8443:443`： + +``` +docker run -d --restart=unless-stopped \ + -p 8080:80 -p 8443:443 \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md new file mode 100644 index 00000000000..7d7405a255b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md @@ -0,0 +1,61 @@ +--- +title: HTTP 代ç†é…ç½® +--- + +如果你通过代ç†æ¥æ“作 Rancher，并想è¦é€šè¿‡ä»£ç†è®¿é—®æœåŠ¡ï¼ˆä¾‹å¦‚拉å–应用商店），你需è¦æä¾› Rancher 代ç†çš„ä¿¡æ¯ã€‚由于 Rancher 是用 Go 编写的，Rancher 使用如下常è§çš„代ç†çŽ¯å¢ƒå˜é‡ã€‚ + +è¯·ç¡®ä¿ `NO_PROXY` 包å«ä¸ä½¿ç”¨ä»£ç†çš„网络地å€ï¼Œç½‘络地å€èŒƒå›´å’ŒåŸŸå。 + +| 环境å˜é‡ | 用途 | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| HTTP_PROXY | å‘èµ· HTTP 连接的代ç†åœ°å€ | +| HTTPS_PROXY | å‘èµ· HTTPS 连接的代ç†åœ°å€ | +| NO_PROXY | å‘起连接时ä¸ä½¿ç”¨ä»£ç†çš„网络地å€ï¼Œç½‘络地å€èŒƒå›´å’ŒåŸŸå | + +:::note é‡è¦æ示： + +`NO_PROXY` 必须大写æ‰èƒ½ä½¿ç”¨ç½‘络范围 CIDR 表示法。 + +::: + +## 基于 Docker 安装 + +ä½ å¯ä½¿ç”¨ `-e KEY=VALUE` 或 `--env KEY=VALUE`将环境å˜é‡ä¼ ç»™ Rancher 容器。在 [Docker 安装](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md)中，`NO_PROXY` 必须的值为： + +- `localhost` +- `127.0.0.1` +- `0.0.0.0` +- `10.0.0.0/8` +- `cattle-system.svc` +- `.svc` +- `.cluster.local` + +以下示例中，代ç†æœåŠ¡å™¨å¯ä»¥é€šè¿‡ `http://192.168.0.1:3128` 访问。此外，在访问 `192.168.10.0/24` ç½‘ç»œèŒƒå›´ä»¥åŠ `example.com` 域å下的æ¯ä¸ªä¸»æœºåæ—¶å‡ä¸ä½¿ç”¨ä»£ç†æœåŠ¡å™¨ã€‚ + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e HTTP_PROXY="http://192.168.10.1:3128" \ + -e HTTPS_PROXY="http://192.168.10.1:3128" \ + -e NO_PROXY="localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,cattle-system.svc,192.168.10.0/24,.svc,.cluster.local,example.com" \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +### 离线代ç†é…ç½® + +你现在å¯ä»¥åœ¨é…置的离线集群中é…置主机驱动集群，以使用代ç†è¿›è¡Œå‡ºç«™è¿žæŽ¥ã€‚ + +除了如上为代ç†æœåŠ¡å™¨è®¾ç½®é»˜è®¤è§„则外，你还需è¦é¢å¤–添加如下所示的规则，以从代ç†çš„ Rancher 环境中é…置主机驱动集群。 + +æ ¹æ®ä½ çš„设置é…置文件路径，例如 `/etc/apt/apt.conf.d/proxy.conf`： + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/system-tools.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/system-tools.md new file mode 100644 index 00000000000..ce21d6c41fd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/system-tools.md @@ -0,0 +1,31 @@ +--- +title: 系统工具 +--- + +:::note + +系统工具（System Tools）自 2022 å¹´ 6 月起已弃用。 + +::: + +## 日志 + +请使用 [logs-collector](https://github.com/rancherlabs/support-tools/tree/master/collection/rancher/v2.x/logs-collector) æ¥æ”¶é›†ä½ çš„集群日志。 + +## Stats + +如果è¦å¤åˆ¶ stats 命令，你å¯ä»¥åœ¨é›†ç¾¤èŠ‚点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +:::note + +以下命令需è¦é›†ç¾¤èŠ‚点上的 `sysstat` 包。 + +::: + +``` +/usr/bin/sar -u -r -F 1 1 +``` + +## Remove + +请使用 [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) 工具。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/api-keys.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/api-keys.md new file mode 100644 index 00000000000..287b71feaf0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/api-keys.md @@ -0,0 +1,59 @@ +--- +title: API 密钥 +--- + +## API å¯†é’¥å’Œç”¨æˆ·èº«ä»½éªŒè¯ + +如果你想通过外部应用程åºæ¥è®¿é—® Rancher 集群ã€é¡¹ç›®æˆ–其他对象，你å¯ä»¥ä½¿ç”¨ Rancher API。但是，在你的应用程åºå¯ä»¥è®¿é—® API 之å‰ï¼Œä½ å¿…须为应用程åºæä¾›ç”¨äºŽå‘ Rancher 进行身份验è¯çš„密钥。你å¯ä»¥é€šè¿‡ Rancher UI 获å–密钥。 + +使用 Rancher CLI ä¹Ÿéœ€è¦ API 密钥。 + +API 密钥由四个组件组æˆï¼š + +- **端点**：其他应用程åºç”¨æ¥å‘ Rancher API å‘é€è¯·æ±‚çš„ IP 地å€å’Œè·¯å¾„。 +- **访问密钥**：Token 的用户å。 +- **密文密钥**：Token 的密ç ã€‚如果应用程åºæ示你输入两个ä¸åŒçš„字符串进行 API 身份验è¯ï¼Œä½ é€šå¸¸éœ€è¦å°†ä¸¤ä¸ªå¯†é’¥ä¸€èµ·è¾“入。 +- **æŒæœ‰è€…令牌**：连接在一起的令牌用户å和密ç ã€‚如果应用程åºæ示你输入一个身份验è¯å­—符串，则使用此字符串。 + +:::note + +用户å¯ä»¥é€‰æ‹©å¯ç”¨[令牌哈希（token hashing）](../about-the-api/api-tokens.md)。 + +::: + +## 创建 API 密钥 + +1. 从å³ä¸Šè§’选择**ç”¨æˆ·å¤´åƒ > è´¦å· & API 密钥**。 + +2. å•å‡»**创建 API 密钥**。 + +3. **å¯é€‰**：输入 API 密钥的æ述并选择有效期或范围。我们建议设置到期日期。 + + API 密钥过期åŽå°†å¤±æ•ˆã€‚有效期短一点会更安全。 + + æœ‰æ•ˆæœŸå°†å— `v3/settings/auth-token-max-ttl-minutes` 约æŸã€‚如果超过 max-ttl，则会以 max-ttl 为有效期创建 API 密钥。 + + 范围将é™åˆ¶ API 密钥，使其仅适用于指定集群的 Kubernetes API。如果集群é…置了授æƒé›†ç¾¤ç«¯ç‚¹ï¼Œä½ å°†èƒ½å¤Ÿç›´æŽ¥é’ˆå¯¹é›†ç¾¤çš„ API 使用设定了范围的令牌，而无需通过 Rancher Server 进行代ç†ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[授æƒé›†ç¾¤ç«¯ç‚¹](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +4. å•å‡»**创建**。 + + **步骤结果**：已创建 API 密钥。将会显示你的 API **端点**ã€**访问密钥**ã€**密文密钥**å’Œ**æŒæœ‰è€…令牌**。 + + 使用**æŒæœ‰è€…令牌**通过 Rancher CLI 进行身份验è¯ã€‚ + +5. 将显示的信æ¯å¤åˆ¶åˆ°å®‰å…¨ä½ç½®ã€‚此信æ¯ä»…显示一次，因此如果你丢失了密钥，则必须制作一个新的。 + +## åŽç»­æ“作 + +- å°† API 密钥信æ¯è¾“å…¥åˆ°å°†å‘ Rancher API å‘é€è¯·æ±‚的应用程åºä¸­ã€‚ +- è¦äº†è§£ Rancher 端点和å‚数的更多信æ¯ï¼Œä½ å¯ä»¥ä¸º Rancher UI 中的对象选择**在 API 中查看**。 +- API 密钥å¯ç”¨äºŽ API 调用和 [Rancher CLI](../../pages-for-subheaders/cli-with-rancher.md)。 + +## 删除 API 密钥 + +如果è¦æ’¤é”€ API 密钥，请将其删除。如果出现以下情况，你需è¦åˆ é™¤ API 密钥： + +- API 密钥å¯èƒ½å·²ç»æ³„露。 +- API 密钥已过期。 + +è¦åˆ é™¤ API，选择密钥并å•å‡»**删除**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/manage-cloud-credentials.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/manage-cloud-credentials.md new file mode 100644 index 00000000000..b4f7f71c543 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/manage-cloud-credentials.md @@ -0,0 +1,51 @@ +--- +title: 管ç†äº‘å‡­è¯ +--- + +如果è¦åˆ›å»º[由基础设施æ供商托管](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)的集群，则å¯ä»¥ä½¿ç”¨[节点模æ¿](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)æ¥é…置集群节点。这些模æ¿ä½¿ç”¨ Docker Machine é…置选项æ¥å®šä¹‰èŠ‚点的æ“作系统镜åƒä»¥åŠè®¾ç½®/å‚数。 + +节点模æ¿å¯ä»¥ä½¿ç”¨äº‘凭è¯æ¥è®¿é—®åœ¨åŸºç¡€è®¾æ–½æ供商中é…置节点所需的凭è¯ä¿¡æ¯ã€‚多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯ã€‚云凭è¯çœåŽ»äº†ä¸ºåŒä¸€äº‘æ供商é‡æ–°è¾“入访问密钥的麻烦。云凭è¯å­˜å‚¨åœ¨ Kubernetes 密文中。 + +åªæœ‰å­˜åœ¨æ ‡è®°ä¸º `password` 的字段时，节点模æ¿æ‰ä¼šä½¿ç”¨äº‘凭è¯ã€‚默认的 `active` 主机驱动将其账å·è®¿é—®å­—段标记为 `password`，但å¯èƒ½æœ‰ä¸€äº› `inactive` 主机驱动没有使用它们。这些主机驱动ä¸ä¼šä½¿ç”¨äº‘凭è¯ã€‚ + +ä½ å¯ä»¥åœ¨ä¸¤ç§æƒ…况下创建云凭è¯ï¼š + +- [在为集群创建节点模æ¿æœŸé—´](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)。 +- 在**用户设置**中。 + +所有云凭è¯éƒ½ç»‘定到创建者的用户é…置文件。**ä¸èƒ½**与其他用户共享。 + +## ä½¿ç”¨ç”¨æˆ·è®¾ç½®åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡»äº‘凭è¯ç±»åž‹ã€‚下拉列表中的选项å–决于 Rancher 中的 `active` [主机驱动](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md)。 +1. 输入云凭è¯çš„å称。 +1. æ ¹æ®æ‰€é€‰çš„云凭è¯ç±»åž‹è¾“入所需的值，从而å‘基础设施æ供商进行身份验è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建云凭è¯ï¼Œå®ƒå¯ä»¥ç«‹å³ç”¨äºŽ[创建节点模æ¿](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)。 + +## æ›´æ–°äº‘å‡­è¯ + +如果访问凭è¯æ›´æ”¹æˆ–泄露了，你å¯ä»¥é€šè¿‡æ›´æ–°äº‘凭è¯æ¥è¿›è¡Œè½®æ¢ï¼ŒåŒæ—¶ä¿ç•™ç›¸åŒçš„节点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. 选择è¦ç¼–辑的云凭è¯ï¼Œç„¶åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 更新凭è¯ä¿¡æ¯å¹¶å•å‡»**ä¿å­˜**。 + +**结果**：已使用新的访问凭è¯æ›´æ–°äº‘凭è¯ã€‚[添加新节点](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)时，所有使用此云凭è¯çš„现有节点模æ¿éƒ½ä¼šè‡ªåŠ¨ä½¿ç”¨æ›´æ–°çš„ä¿¡æ¯ã€‚ + +## åˆ é™¤äº‘å‡­è¯ + +如果è¦åˆ é™¤äº‘凭è¯ï¼Œäº‘凭è¯ä¸èƒ½æœ‰å…³è”的节点模æ¿ã€‚如果你无法删除云凭è¯ï¼Œè¯·[删除与该云凭è¯å…³è”的所有节点模æ¿](manage-node-templates.md#删除节点模æ¿)。 + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. ä½ å¯ä»¥åˆ é™¤å•ä¸ªäº‘凭è¯æˆ–进行批é‡åˆ é™¤ã€‚ + + - è¦å•ç‹¬åˆ é™¤ä¸€ä¸ªäº‘凭è¯ï¼Œè¯·é€‰æ‹©ä½ è¦ç¼–辑的云凭è¯ï¼Œç„¶åŽå•å‡» **â‹® > 删除**。 + - è¦æ‰¹é‡åˆ é™¤äº‘凭è¯ï¼Œè¯·ä»Žåˆ—表中选择一个或多个云凭è¯ã€‚å•å‡»**删除**。 +1. 确认你需è¦åˆ é™¤è¿™äº›äº‘凭è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/manage-node-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/manage-node-templates.md new file mode 100644 index 00000000000..5fbb8845fce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/manage-node-templates.md @@ -0,0 +1,54 @@ +--- +title: 管ç†èŠ‚ç‚¹æ¨¡æ¿ +--- + +如果è¦é…ç½®[由基础设施æ供商托管](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)的集群，则å¯ä»¥ä½¿ç”¨[节点模æ¿](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)æ¥é…置集群节点。这些模æ¿ä½¿ç”¨ Docker Machine é…置选项æ¥å®šä¹‰èŠ‚点的æ“作系统镜åƒä»¥åŠè®¾ç½®/å‚数。你å¯ä»¥åœ¨ä¸¤ç§æƒ…况下创建节点模æ¿ï¼š + +- [é…置节点池集群](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +- 在任何时间使用[用户设置](../../pages-for-subheaders/user-settings.md)。 + +创建节点模æ¿æ—¶ï¼Œå®ƒä¼šç»‘定到你的用户é…置文件。节点模æ¿ä¸èƒ½åœ¨ç”¨æˆ·ä¹‹é—´å…±äº«ã€‚ä½ å¯ä»¥ä»Žç”¨æˆ·è®¾ç½®ä¸­åˆ é™¤ä¸å†ä½¿ç”¨çš„旧节点模æ¿ã€‚ + +## åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. 选择一个å¯ç”¨çš„云æ供商。然åŽæŒ‰ç…§å±å¹•ä¸Šçš„说明é…置模æ¿ã€‚ + +**结果**：模æ¿å·²é…置。你å¯ä»¥ç¨åŽåœ¨[é…置节点池集群](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)时使用该模æ¿ã€‚ + +## æ›´æ–°èŠ‚ç‚¹æ¨¡æ¿ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. 选择è¦ç¼–辑的节点模æ¿å¹¶å•å‡» **â‹® > 编辑**。 + + :::note + + 默认的 `active` [主机驱动](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md)和任何标记了 `password` 字段的主机驱动都需è¦ä½¿ç”¨[云凭è¯](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#云凭è¯)。 + + ::: + +1. 编辑所需信æ¯å¹¶å•å‡»**ä¿å­˜**。 + +**结果**：节点模æ¿å·²æ›´æ–°ã€‚添加新节点时，使用此节点模æ¿çš„所有节点池都会自动使用更新的信æ¯ã€‚ + +## å…‹éš†èŠ‚ç‚¹æ¨¡æ¿ + +通过用户设置创建新的节点模æ¿æ—¶ï¼Œå¯ä»¥å…‹éš†çŽ°æœ‰æ¨¡æ¿å¹¶å¿«é€Ÿæ›´æ–°å…¶è®¾ç½®ï¼Œè€Œæ— éœ€ä»Žå¤´å¼€å§‹åˆ›å»ºæ–°çš„模æ¿ã€‚克隆模æ¿é¿å…了为云æ供商é‡æ–°è¾“入访问密钥的麻烦。 + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. 找到è¦å…‹éš†çš„模æ¿ã€‚然åŽé€‰æ‹© **â‹® > 克隆**。 +1. 填写表å•çš„其余部分。 + +**结果**：已克隆和é…置模æ¿ã€‚ä½ å¯ä»¥ç¨åŽåœ¨[é…置节点池集群](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)时使用该模æ¿ã€‚ + +## åˆ é™¤èŠ‚ç‚¹æ¨¡æ¿ + +ä¸å†ä½¿ç”¨èŠ‚点模æ¿æ—¶ï¼Œä½ å¯ä»¥å°†å…¶ä»Žç”¨æˆ·è®¾ç½®ä¸­åˆ é™¤ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. 从列表中选择一个或多个模æ¿ã€‚然åŽç‚¹å‡»**删除**。出现æ示时确认删除。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/user-preferences.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/user-preferences.md new file mode 100644 index 00000000000..96650669366 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/reference-guides/user-settings/user-preferences.md @@ -0,0 +1,62 @@ +--- +title: 用户å好 +--- + +ä½ å¯ä»¥é€šè¿‡å好设置æ¥ä¸ªæ€§åŒ–ä½ çš„ Rancher 体验。è¦æ›´æ”¹å好设置： + +1. å•å‡»å³ä¸Šè§’的用户头åƒã€‚ +1. å•å‡»**å好设置**。 + +## 语言 + +选择 Rancher UI 显示的语言。选项包括： + +- English +- 简体中文 + +## 主题 + +选择 Rancher UI 的背景颜色。如果选择**自动**ï¼ŒèƒŒæ™¯é¢œè‰²ä¼šåœ¨ä¸‹åˆ 6 点从浅色å˜ä¸ºæ·±è‰²ï¼Œç„¶åŽåœ¨æ—©ä¸Š 6 点å˜å›žæµ…色。 + +## ç™»å½•é¡µé¢ + +选择登录åŽæ˜¾ç¤ºçš„页é¢ã€‚选项包括： + +- 主页。 +- 上次访问的页é¢ã€‚ +- 你选择的特定集群。 + +## 显示设置 + +选择信æ¯çš„显示方å¼ï¼š + +- æ—¥æœŸæ ¼å¼ +- æ—¶é—´æ ¼å¼ +- æ¯é¡µè¡Œæ•° +- 在侧边æ æ˜¾ç¤ºçš„é›†ç¾¤æ•°é‡ + +## 确认设置 + +_从 v2.7.2 èµ·å¯ç”¨_ + +选择缩å‡èŠ‚点池时是å¦è¦æ±‚确认。 + +## 高级功能 + +- å¯ç”¨â€œåœ¨ API 中查看â€ã€‚ +- 显示由 Rancher 管ç†çš„ system 命å空间（ä¸è¦ç¼–辑或删除）。 +- 使用快æ·é”®ï¼ˆshift+T）æ¥åˆ‡æ¢æ·±è‰²/浅色主题。 +- éšè—所有类型æ述。 +- å¯ç”¨æ‰©å±•å¼€å‘者功能。 + +![](/img/user-preferences-2-7.png) + +## YAML 编辑器 + +- 默认 +- Emacs +- Vim + +## Helm Chart + +选择仅显示已å‘布的 Helm Chart 还是åŒæ—¶åŒ…å«é¢„å‘å¸ƒç‰ˆæœ¬ã€‚å¦‚æžœç‰ˆæœ¬ç¬¦åˆ [Semantic Versioning 2.0.0](https://semver.org/) 定义的[规范](https://semver.org/#spec-item-9)，则那么该版本为预å‘布版本。例如，è¦æ˜¾ç¤ºç‰ˆæœ¬ä¸º `0.1.3-dev.12ab4f` çš„ Helm chart，你需è¦å…ˆé€‰æ‹©`包括预å‘布版本`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/security/security-scan/security-scan.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/security/security-scan/security-scan.md new file mode 100644 index 00000000000..32cf9e17a4c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/security/security-scan/security-scan.md @@ -0,0 +1,5 @@ +--- +title: 安全扫æ +--- + +CIS 安全扫æ的文档已移至[此处](../../pages-for-subheaders/cis-scan-guides.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/shared-files/_cluster-capabilities-table.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/shared-files/_cluster-capabilities-table.md new file mode 100644 index 00000000000..c418e1eaf54 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/shared-files/_cluster-capabilities-table.md @@ -0,0 +1,23 @@ +| æ“作 | Rancher å¯åŠ¨çš„ Kubernetes 集群 | EKSã€GKE å’Œ AKS 集群1 | 其他托管的 Kubernetes 集群 | éž EKS 或 GKE 注册集群 | +| --- | --- | ---| ---|----| +| [使用 kubectl å’Œ kubeconfig 文件æ¥è®¿é—®é›†ç¾¤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) | ✓ | ✓ | ✓ | ✓ | +| [管ç†é›†ç¾¤æˆå‘˜](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) | ✓ | ✓ | ✓ | ✓ | +| [编辑和å‡çº§é›†ç¾¤](../pages-for-subheaders/cluster-configuration.md) | ✓ | ✓ | ✓ | ✓2 | +| [管ç†èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md) | ✓ | ✓ | ✓ | ✓3 | +| [管ç†æŒä¹…å·å’Œå­˜å‚¨ç±»](../pages-for-subheaders/create-kubernetes-persistent-storage.md) | ✓ | ✓ | ✓ | ✓ | +| [管ç†é¡¹ç›®ã€å‘½å空间和工作负载](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) | ✓ | ✓ | ✓ | ✓ | +| [使用应用目录](../pages-for-subheaders/helm-charts-in-rancher.md) | ✓ | ✓ | ✓ | ✓ | +| é…置工具（[Alertsã€Notifiersã€Monitoring](../pages-for-subheaders/monitoring-and-alerting.md)ã€[Logging](../pages-for-subheaders/logging.md) å’Œ [Istio](../pages-for-subheaders/istio.md)） | ✓ | ✓ | ✓ | ✓ | +| [è¿è¡Œå®‰å…¨æ‰«æ](../pages-for-subheaders/cis-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | +| [è½®æ¢è¯ä¹¦](../how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md) | ✓ | ✓ | | | +| [备份](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md)å’Œ[æ¢å¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Rancher å¯åŠ¨çš„集群 | ✓ | ✓ | | ✓4 | +| [在 Rancher æ— æ³•è®¿é—®é›†ç¾¤æ—¶æ¸…ç† Kubernetes 组件](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) | ✓ | | | | +| [é…ç½® Pod 安全策略](../how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md) | ✓ | ✓ | | + +1. 注册的 EKSã€GKE å’Œ AKS 集群与从 Rancher UI 创建的 EKSã€GKE å’Œ AKS 集群的å¯ç”¨é€‰é¡¹ä¸€è‡´ã€‚ä¸åŒä¹‹å¤„是，从 Rancher UI 中删除已注册的集群åŽï¼Œé›†ç¾¤ä¸ä¼šè¢«é”€æ¯ã€‚ + +2. 无法编辑已注册的集群的集群é…置选项（[K3s å’Œ RKE2 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)除外）。 + +3. Rancher UI 为已注册的集群节点æ供了å°é”ã€æ¸…空和编辑节点的功能。 + +4. 对于使用 etcd 作为 controlplane 的注册集群，必须在 Rancher UI 之外手动创建快照以用于备份和æ¢å¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/shared-files/_common-ports-table.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/shared-files/_common-ports-table.md new file mode 100644 index 00000000000..b37d40dc276 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/shared-files/_common-ports-table.md @@ -0,0 +1,19 @@ +| åè®® | ç«¯å£ | æè¿° | +|:--------: |:----------------: |---------------------------------------------------------------------------------- | +| TCP | 22 | Node Driver SSH é…ç½® | +| TCP | 179 | Calico BGP ç«¯å£ | +| TCP | 2376 | Node Driver Docker daemon TLS ç«¯å£ | +| TCP | 2379 | etcd 客户端请求 | +| TCP | 2380 | etcd 对等通信 | +| UDP | 8472 | Canal/Flannel VXLAN 覆盖网络 | +| UDP | 4789 | Windows 集群中的 Flannel VXLAN 覆盖网络 | +| TCP | 8443 | Rancher webhook | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 9100 | Monitoring 从 Linux node-exporter 中抓å–æŒ‡æ ‡æ‰€éœ€çš„é»˜è®¤ç«¯å£ | +| TCP | 9443 | Rancher webhook | +| TCP | 9796 | Monitoring 从 Windows node-exporter 中抓å–æŒ‡æ ‡æ‰€éœ€çš„é»˜è®¤ç«¯å£ | +| TCP | 6783 | Weave ç«¯å£ | +| UDP | 6783-6784 | Weave UDP ç«¯å£ | +| TCP | 10250 | Metrics Server 与所有节点 API 的通信 | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | +| TCP/UDP | 30000-32767 | NodePort 端å£èŒƒå›´ | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/general-troubleshooting.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/general-troubleshooting.md new file mode 100644 index 00000000000..01d4356c9ca --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/general-troubleshooting.md @@ -0,0 +1,44 @@ +--- +title: 一般故障排除 +--- + +本文用于帮助你解决使用 Rancher æ—¶é‡åˆ°çš„问题。 + +- [Kubernetes 组件](../pages-for-subheaders/kubernetes-components.md) + + 对以下核心 Kubernetes 集群组件进行故障排除： + * `etcd` + * `kube-apiserver` + * `kube-controller-manager` + * `kube-scheduler` + * `kubelet` + * `kube-proxy` + * `nginx-proxy` + +- [Kubernetes resources](other-troubleshooting-tips/kubernetes-resources.md) + + 本节介ç»äº†å¯¹èŠ‚点ã€Ingress Controller å’Œ Rancher Agent ç­‰ Kubernetes 资æºè¿›è¡Œæ•…障排除的选项。 + +- [网络](other-troubleshooting-tips/networking.md) + + 介ç»äº†è§£å†³ç½‘络问题的步骤。 + +- [DNS](other-troubleshooting-tips/dns.md) + + 解决集群的å称解æžé—®é¢˜ã€‚ + +- [对安装在 Kubernetes 上的 Rancher 进行故障排除](other-troubleshooting-tips/rancher-ha.md) + + 解决[安装在 Kubernetes 上的 Rancher Server](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) 的问题。 + +- [Logging](other-troubleshooting-tips/logging.md) + + 了解å¯ä»¥é…置哪些日志级别，以åŠå¦‚何é…置日志级别。 + +- [审计日志中的用户 ID 跟踪](other-troubleshooting-tips/user-id-tracking-in-audit-logs.md) + + 了解 Rancher 管ç†å‘˜å¦‚何通过外部身份æ供程åºç”¨æˆ·å从 Rancher 审计日志和 Kubernetes 审计日志中跟踪事件。 + +- [过期的 Webhook è¯ä¹¦](other-troubleshooting-tips/expired-webhook-certificate-rotation.md) + + 了解如何在æ¯å¹´åˆ°æœŸåŽè½®æ¢ Rancher webhook è¯ä¹¦å¯†é’¥ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md new file mode 100644 index 00000000000..6c9a8d5291a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md @@ -0,0 +1,51 @@ +--- +title: Controlplane 节点故障排除 +--- + +本文适用于具有 `controlplane` 角色的节点。 + +## 检查 controlplane 容器是å¦æ­£åœ¨è¿è¡Œ + +具有 `controlplane` 角色的节点上å¯åŠ¨äº†ä¸‰ä¸ªå®¹å™¨ï¼š + +* `kube-apiserver` +* `kube-controller-manager` +* `kube-scheduler` + +容器的状æ€åº”该是 **Up**。**Up** åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name='kube-apiserver|kube-controller-manager|kube-scheduler' +``` + +输出示例： +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +26c7159abbcc rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-apiserver +f3d287ca4549 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-scheduler +bdf3898b8063 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-controller-manager +``` + +## controlplane 容器日志记录 + +:::note + +如果你添加了多个具有 `controlplane` 角色的节点，`kube-controller-manager` å’Œ `kube-scheduler` 会通过 leader 选举æ¥ç¡®å®š leader。åªæœ‰å½“å‰ leader 会记录执行的æ“作。有关检索当å‰çš„ leader 的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes leader 选举](../other-troubleshooting-tips/kubernetes-resources.md#kubernetes-leader-选举)。 + +::: + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs kube-apiserver +docker logs kube-controller-manager +docker logs kube-scheduler +``` + +## RKE2 Server 日志 + +如果 Rancher é…置的 RKE2 集群无法与 Rancher 通信，你å¯ä»¥åœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­çš„ Server 节点上è¿è¡Œæ­¤å‘½ä»¤ï¼Œä»Žè€ŒèŽ·å– RKE2 Server 日志： + +``` +journalctl -u rke2-server -f +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md new file mode 100644 index 00000000000..72d6b470bfc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md @@ -0,0 +1,365 @@ +--- +title: etcd 节点故障排除 +--- + +本文介ç»äº†å¯¹å…·æœ‰ `etcd` 角色的节点进行故障排除的命令和æ示。 + + +## 检查 etcd 容器是å¦æ­£åœ¨è¿è¡Œ + +etcd 容器的状æ€åº”该是 **Up**。**Up** åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name=etcd$ +``` + +输出示例： +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +605a124503b9 rancher/coreos-etcd:v3.2.18 "/usr/local/bin/et..." 2 hours ago Up 2 hours etcd +``` + +## etcd 容器日志记录 + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs etcd +``` +| 日志 | 解释 | +|-----|------------------| +| `health check for peer xxx could not connect: dial tcp IP:2380: getsockopt: connection refused` | æ— æ³•è¿žæŽ¥åˆ°ç«¯å£ 2380 上显示的地å€ã€‚检查 etcd 容器是å¦åœ¨æ˜¾ç¤ºåœ°å€çš„主机上è¿è¡Œã€‚ | +| `xxx is starting a new election at term x` | etcd 集群失去了集群仲è£æ•°é‡ï¼Œå¹¶æ­£åœ¨å°è¯•å»ºç«‹ä¸€ä¸ªæ–°çš„ leader。è¿è¡Œ etcd 的大多数节点关闭/无法访问时，å¯èƒ½ä¼šå‘生这ç§æƒ…况。 | +| `connection error: desc = "transport: Error while dialing dial tcp 0.0.0.0:2379: i/o timeout"; Reconnecting to {0.0.0.0:2379 0 }` | 主机防ç«å¢™æ­£åœ¨é˜»æ­¢ç½‘络通信。 | +| `rafthttp: request cluster ID mismatch` | 具有 etcd 实例日志 `rafthttp: request cluster ID mismatch` 的节点正在å°è¯•åŠ å…¥å·²ç»æ·»åŠ å¦ä¸€ä¸ªå¯¹ç­‰èŠ‚点（peer）的集群。你需è¦ä»Žé›†ç¾¤ä¸­åˆ é™¤è¯¥èŠ‚点，然åŽå†é‡æ–°æ·»åŠ ã€‚ | +| `rafthttp: failed to find member` | 集群状æ€ï¼ˆ`/var/lib/etcd`）包å«åŠ å…¥é›†ç¾¤çš„错误信æ¯ã€‚你需è¦ä»Žé›†ç¾¤ä¸­åˆ é™¤è¯¥èŠ‚点，清ç†çŠ¶æ€ç›®å½•ï¼Œç„¶åŽå†é‡æ–°æ·»åŠ ã€‚ | + +## etcd 集群和连接检查 + +è¿è¡Œ etcd 的主机的地å€é…置决定了 etcd 监å¬çš„地å€ã€‚如果为è¿è¡Œ etcd 的主机é…置了内部地å€ï¼Œåˆ™éœ€è¦æ˜¾å¼æŒ‡å®š `etcdctl` 的端点。如果任何命令的å“应是 `Error: context deadline exceeded`，则 etcd 实例ä¸å¥åº·ï¼ˆä»²è£ä¸¢å¤±æˆ–实例未正确加入集群）。 + +### 检查所有节点上的 etcd æˆå‘˜ + +输出应包å«å…·æœ‰ `etcd` 角色的所有节点，而且所有节点上的输出应该是相åŒçš„。 + +命令： +``` +docker exec etcd etcdctl member list +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list" +``` + +输出示例： +``` +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +``` + +### æ£€æŸ¥ç«¯ç‚¹çŠ¶æ€ + +`RAFT TERM` 的值应该是相等的，而且 `RAFT INDEX` 相差ä¸èƒ½å¤ªå¤§ã€‚ + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table +``` + +输出示例： +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | 333ef673fc4add56 | 3.2.18 | 24 MB | false | 72 | 66887 | +| https://IP:2379 | 5feed52d940ce4cf | 3.2.18 | 24 MB | true | 72 | 66887 | +| https://IP:2379 | db6b3bdb559a848d | 3.2.18 | 25 MB | false | 72 | 66887 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### 检查端点å¥åº· + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint health +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl endpoint health --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") +``` + +输出示例： +``` +https://IP:2379 is healthy: successfully committed proposal: took = 2.113189ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.649963ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.451201ms +``` + +### æ£€æŸ¥ç«¯å£ TCP/2379 上的连接 + +命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health" + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +输出示例： +``` +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +``` + +### æ£€æŸ¥ç«¯å£ TCP/2380 上的连接 + +命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +输出示例： +``` +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +``` + +## etcd å‘Šè­¦ + +etcd 会触å‘告警（例如空间ä¸è¶³æ—¶ï¼‰ã€‚ + +命令： +``` +docker exec etcd etcdctl alarm list +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +è§¦å‘ NOSPACE 告警的输出示例： +``` +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +``` + +## etcd 空间错误 + +相关的错误消æ¯æ˜¯ `etcdserver: mvcc: database space exceeded` 或 `applying raft message exceeded backend quota`。告警 `NOSPACE` 会被触å‘。 + +解决： + +- [压缩键空间](#压缩键空间) +- [对所有 etcd æˆå‘˜è¿›è¡Œç¢Žç‰‡æ•´ç†](#对所有-etcd-æˆå‘˜è¿›è¡Œç¢Žç‰‡æ•´ç†) +- [检查端点状æ€](#检查端点状æ€) +- [解除告警](#解除告警) + +### 压缩键空间 + +命令： +``` +rev=$(docker exec etcd etcdctl endpoint status --write-out json | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*') +docker exec etcd etcdctl compact "$rev" +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +rev=$(docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT endpoint status --write-out json | egrep -o '\"revision\":[0-9]*' | egrep -o '[0-9]*'") +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT compact \"$rev\"" +``` + +输出示例： +``` +compacted revision xxx +``` + +### 对所有 etcd æˆå‘˜è¿›è¡Œç¢Žç‰‡æ•´ç† + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl defrag +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl defrag --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','")" +``` + +输出示例： +``` +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +``` + +### æ£€æŸ¥ç«¯ç‚¹çŠ¶æ€ + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table" +``` + +输出示例： +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | e973e4419737125 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | 4a509c997b26c206 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | b217e736575e9dd3 | 3.2.18 | 553 kB | true | 32 | 2449410 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### 解除告警 + +如果压缩和整ç†ç¢Žç‰‡åŽç¡®å®šæ•°æ®åº“大å°ä¸‹é™äº†ï¼Œåˆ™éœ€è¦è§£é™¤å‘Šè­¦æ¥å…许 etcd å†æ¬¡å†™å…¥ã€‚ + +命令： +``` +docker exec etcd etcdctl alarm list +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm disarm" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +输出示例： +``` +docker exec etcd etcdctl alarm list +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +## é…置日志级别 + +:::note + +你无法å†åŠ¨æ€æ›´æ”¹ etcd v3.5 或更高版本中的日志级别。 + +::: + +### etcd v3.5 åŠæ›´é«˜ç‰ˆæœ¬ + +è¦é…ç½® etcd 的日志级别，请编辑集群 YAML： + +``` +services: + etcd: + extra_args: + log-level: "debug" +``` + +### etcd v3.4 åŠæ›´æ—©ç‰ˆæœ¬ + +在早期的 etcd 版本中，你å¯ä»¥ä½¿ç”¨ API 动æ€æ›´æ”¹æ—¥å¿—级别。使用以下命令æ¥é…置调试日志： + +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +è¦å°†æ—¥å¿—级别é‡ç½®å›žé»˜è®¤å€¼ (`INFO`)，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤ã€‚ + +命令： +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +## etcd 内容 + +如果è¦æŸ¥çœ‹ etcd 的内容，你å¯ä»¥æŸ¥çœ‹æµäº‹ä»¶ï¼Œä¹Ÿå¯ä»¥ç›´æŽ¥æŸ¥è¯¢ etcd。详情请å‚阅以下示例。 + +### 查看æµäº‹ä»¶ + +命令： +``` +docker exec etcd etcdctl watch --prefix /registry +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT watch --prefix /registry +``` + +如果你åªæƒ³æŸ¥çœ‹å—å½±å“的键（而ä¸æ˜¯äºŒè¿›åˆ¶æ•°æ®ï¼‰ï¼Œä½ å¯ä»¥å°† `| grep -a ^/registry` 尾附到该命令æ¥è¿‡æ»¤é”®ã€‚ + +### 直接查询 etcd + +命令： +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT get /registry --prefix=true --keys-only +``` + +ä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤æ¥å¤„ç†æ•°æ®ï¼Œä»Žè€ŒèŽ·å–æ¯ä¸ªé”®çš„计数摘è¦ï¼š + +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only | grep -v ^$ | awk -F'/' '{ if ($3 ~ /cattle.io/) {h[$3"/"$4]++} else { h[$3]++ }} END { for(k in h) print h[k], k }' | sort -nr +``` + +## æ›´æ¢ä¸å¥åº·çš„ etcd 节点 + +如果你 etcd 集群中的æŸä¸ªèŠ‚点å˜å¾—ä¸å¥åº·ï¼Œåœ¨å°†æ–°çš„ etcd 节点添加到集群之å‰ï¼Œæˆ‘们建议你修å¤æˆ–删除故障/ä¸å¥åº·çš„节点。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md new file mode 100644 index 00000000000..c2ee756591f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md @@ -0,0 +1,68 @@ +--- +title: nginx-proxy 故障排除 +--- + +æ¯ä¸ªæ²¡æœ‰ `controlplane` 角色的节点上都部署了 `nginx-proxy` 容器。`nginx-proxy` 基于具有 `controlplane` 角色的å¯ç”¨èŠ‚点æ¥åŠ¨æ€ç”Ÿæˆ NGINX é…置，从而æ供对这些 `controlplane` 角色节点的访问。 + +## 检查容器是å¦æ­£åœ¨è¿è¡Œ + +该容器称为 `nginx-proxy`，它的状æ€åº”该是 `Up`。`Up` åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name=nginx-proxy +``` + +输出示例： + +``` +docker ps -a -f=name=nginx-proxy +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +c3e933687c0e rancher/rke-tools:v0.1.15 "nginx-proxy CP_HO..." 3 hours ago Up 3 hours nginx-proxy +``` + +## 检查生æˆçš„ NGINX é…ç½® + +生æˆçš„é…置包括了具有 `controlplane` 角色的节点的 IP 地å€ã€‚ä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤æ¥æ£€æŸ¥é…置： + +``` +docker exec nginx-proxy cat /etc/nginx/nginx.conf +``` + +输出示例： +``` +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + + server ip_of_controlplane_node1:6443; + + server ip_of_controlplane_node2:6443; + + } + + server { + listen 6443; + proxy_pass kube_apiserver; + proxy_timeout 30; + proxy_connect_timeout 2s; + + } + +} +``` + +## nginx-proxy 容器日志记录 + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs nginx-proxy +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md new file mode 100644 index 00000000000..ae6b283c056 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md @@ -0,0 +1,34 @@ +--- +title: Worker 节点和通用组件故障排除 +--- + +本文包括了è¿è¡Œåœ¨æ‰€æœ‰è§’色节点上的组件，因此适用于æ¯ä¸ªèŠ‚点。 + +## 检查容器是å¦æ­£åœ¨è¿è¡Œ + +具有 `worker` 角色的节点上å¯åŠ¨äº†ä¸¤ä¸ªå®¹å™¨ï¼š + +* kubelet +* kube-proxy + +容器的状æ€åº”该是 `Up`。`Up` åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name='kubelet|kube-proxy' +``` + +输出示例： +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +158d0dcc33a5 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-proxy +a30717ecfb55 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kubelet +``` + +## 容器日志记录 + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs kubelet +docker logs kube-proxy +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/dns.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/dns.md new file mode 100644 index 00000000000..c5ae0ff8872 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/dns.md @@ -0,0 +1,220 @@ +--- +title: DNS +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥é›†ç¾¤ä¸­çš„å称解æžé—®é¢˜ã€‚ + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，为 Rancher HA é…置了 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + +在è¿è¡Œ DNS 检查之å‰ï¼Œè¯·æ£€æŸ¥é›†ç¾¤çš„[默认 DNS æ供商](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#默认-dns-æ供商)，并确ä¿[覆盖网络正常è¿è¡Œ](networking.md#检查覆盖网络是å¦æ­£å¸¸è¿è¡Œ)，因为这也å¯èƒ½å¯¼è‡´ DNS 解æžï¼ˆéƒ¨åˆ†ï¼‰å¤±è´¥ã€‚ + +### 检查 DNS pod 是å¦æ­£åœ¨è¿è¡Œ + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns +``` + +使用 CoreDNS 时的示例输出： +``` +NAME READY STATUS RESTARTS AGE +coredns-799dffd9c4-6jhlz 1/1 Running 0 76m +``` + +使用 kube-dns 时的示例输出： +``` +NAME READY STATUS RESTARTS AGE +kube-dns-5fd74c7488-h6f7n 3/3 Running 0 4m13s +``` + +### 检查 DNS æœåŠ¡æ˜¯å¦æ˜¾ç¤ºæ­£ç¡®çš„ cluster-ip + +``` +kubectl -n kube-system get svc -l k8s-app=kube-dns +``` + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/kube-dns ClusterIP 10.43.0.10 53/UDP,53/TCP 4m13s +``` + +### 检查是å¦æ­£åœ¨è§£æžåŸŸå + +检查是å¦æ­£åœ¨è§£æžå†…部集群å称（在本例中为 `kubernetes.default`），`Server:` åŽé¢æ˜¾ç¤ºçš„ IP 应与 `kube-dns` æœåŠ¡çš„ `CLUSTER-IP` 一致。 + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup kubernetes.default +``` + +输出示例： +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: kubernetes.default +Address 1: 10.43.0.1 kubernetes.default.svc.cluster.local +pod "busybox" deleted +``` + +检查是å¦æ­£åœ¨è§£æžå¤–部å称（在本例中为 `www.google.com`）： + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup www.google.com +``` + +输出示例： +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: www.google.com +Address 1: 2a00:1450:4009:80b::2004 lhr35s04-in-x04.1e100.net +Address 2: 216.58.211.100 ams15s32-in-f4.1e100.net +pod "busybox" deleted +``` + +如果è¦æ£€æŸ¥æ‰€æœ‰ä¸»æœºçš„域å解æžï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ï¼š + +1. 将以下文件å¦å­˜ä¸º `ds-dnstest.yml`： + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: dnstest + spec: + selector: + matchLabels: + name: dnstest + template: + metadata: + labels: + name: dnstest + spec: + tolerations: + - operator: Exists + containers: + - image: busybox:1.28 + imagePullPolicy: Always + name: alpine + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + ``` + +2. 通过è¿è¡Œ `kubectl create -f ds-dnstest.yml` å¯åŠ¨å®ƒã€‚ +3. 等待 `kubectl rollout status ds/dnstest -w` 返回 `daemon set "dnstest" successfully rolled out`。 +4. 将环境å˜é‡ `DOMAIN` é…置为主机能够解æžçš„完全é™å®šåŸŸå（Fully Qualified Domain Name，FQDN），例如 `www.google.com`，并è¿è¡Œä»¥ä¸‹å‘½ä»¤è®©æ¯ä¸ªä¸»æœºä¸Šçš„å„个容器解æžé…置的域å（它是å•è¡Œå‘½ä»¤ï¼‰ï¼š + + ``` + export DOMAIN=www.google.com; echo "=> Start DNS resolve test"; kubectl get pods -l name=dnstest --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do kubectl exec $pod -- /bin/sh -c "nslookup $DOMAIN > /dev/null 2>&1"; RC=$?; if [ $RC -ne 0 ]; then echo $host cannot resolve $DOMAIN; fi; done; echo "=> End DNS resolve test" + ``` + +5. 完æˆæ­¤å‘½ä»¤åŽï¼Œå¦‚果一切正确，则输出如下： + + ``` + => Start DNS resolve test + => End DNS resolve test + ``` + +如果你在输出中看到错误，则表示主机无法解æžç»™å®šçš„ FQDN。 + +在以下错误输出示例中，IP 为 209.97.182.150 的主机阻止了 UDP 端å£ï¼š + +``` +=> Start DNS resolve test +command terminated with exit code 1 +209.97.182.150 cannot resolve www.google.com +=> End DNS resolve test +``` + +è¿è¡Œ `kubectl delete ds/dnstest` æ¸…ç† alpine DaemonSet。 + +### CoreDNS 相关 + +#### 检查 CoreDNS 日志记录 + +``` +kubectl -n kube-system logs -l k8s-app=kube-dns +``` + +#### 检查é…ç½® + +CoreDNS é…置存储在 `kube-system` 命å空间中 configmap çš„ `coredns` 中。 + +``` +kubectl -n kube-system get configmap coredns -o go-template={{.data.Corefile}} +``` + +#### 检查 resolv.conf 中的上游å称æœåŠ¡å™¨ + +默认情况下，é…置在主机（在 `/etc/resolv.conf` 里）上的å称æœåŠ¡å™¨ä¼šç”¨ä½œ CoreDNS 的上游å称æœåŠ¡å™¨ã€‚ä½ å¯ä»¥åœ¨ä¸»æœºä¸Šæ£€æŸ¥æ­¤æ–‡ä»¶ï¼Œæˆ–å°† `dnsPolicy` 设置为 `Default`（将继承其主机的 `/etc/resolv.conf`）并è¿è¡Œä»¥ä¸‹ Pod： + +``` +kubectl run -i --restart=Never --rm test-${RANDOM} --image=ubuntu --overrides='{"kind":"Pod", "apiVersion":"v1", "spec": {"dnsPolicy":"Default"}}' -- sh -c 'cat /etc/resolv.conf' +``` + +#### å¯ç”¨æ—¥å¿—查询 + +ä½ å¯ä»¥é€šè¿‡åœ¨ configmap `coredns` çš„ Corefile é…置中å¯ç”¨ [log plugin](https://coredns.io/plugins/log/) æ¥å¯ç”¨æ—¥å¿—查询。为此，你å¯ä»¥ä½¿ç”¨ `kubectl -n kube-system edit configmap coredns`，或è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ›¿æ¢é…置： + +``` +kubectl get configmap -n kube-system coredns -o json | sed -e 's_loadbalance_log\\n loadbalance_g' | kubectl apply -f - +``` + +这样，所有查询都会记入日志，并且å¯ä»¥ä½¿ç”¨[检查 CoreDNS 日志记录](#检查-coredns-日志记录)中的命令进行检查。 + +### kube-dns 相关 + +#### 检查 kubedns 容器中的上游å称æœåŠ¡å™¨ + +默认情况下，é…置在主机（在 `/etc/resolv.conf` 里）上的å称æœåŠ¡å™¨ä¼šç”¨ä½œ kube-dns 的上游å称æœåŠ¡å™¨ã€‚有时，主机会è¿è¡Œæœ¬åœ°ç¼“å­˜ DNS å称æœåŠ¡å™¨ï¼Œè¿™æ„å‘³ç€ `/etc/resolv.conf` 中的地å€å°†æŒ‡å‘ Loopback 范围（`127.0.0.0/8`）内的地å€ï¼Œè€Œå®¹å™¨å°†æ— æ³•è®¿é—®è¯¥èŒƒå›´ã€‚对于 Ubuntu 18.04，这是由 `systemd-resolved` 进行的。我们会检测 `systemd-resolved` 是å¦æ­£åœ¨è¿è¡Œï¼Œå¹¶è‡ªåŠ¨ä½¿ç”¨å…·æœ‰æ­£ç¡®ä¸Šæ¸¸å称æœåŠ¡å™¨çš„ `/etc/resolv.conf` 文件（ä½äºŽ `/run /systemd/resolve/resolv.conf`）。 + +使用以下命令检查 kubedns 容器使用的上游å称æœåŠ¡å™¨ï¼š + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do echo "Pod ${pod} on host ${host}"; kubectl -n kube-system exec $pod -c kubedns cat /etc/resolv.conf; done +``` + +输出示例： +``` +Pod kube-dns-667c7cb9dd-z4dsf on host x.x.x.x +nameserver 1.1.1.1 +nameserver 8.8.4.4 +``` + +如果输出显示 Loopback 范围（`127.0.0.0/8`ï¼‰å†…çš„åœ°å€ ï¼Œä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼è§£å†³æ­¤é—®é¢˜ï¼š + +* ç¡®ä¿åœ¨é›†ç¾¤èŠ‚点上的 `/etc/resolv.conf` 列出了正确的å称æœåŠ¡å™¨ã€‚如果需è¦äº†è§£å¦‚何进行æ“作，请å‚阅你的æ“作系统文档。请确ä¿ä½ åœ¨é…置集群之å‰æ‰§è¡Œæ­¤æ“作，或在修改åŽé‡å¯èŠ‚点。 +* 通过é…ç½® `kubelet` æ¥ä½¿ç”¨ä¸åŒçš„文件进行å称解æžï¼Œä½ å¯ä»¥ä½¿ç”¨å¦‚下的 `extra_args`（其中 `/run/resolvconf/resolv.conf` 是具有正确å称æœåŠ¡å™¨çš„文件）： + +``` +services: + kubelet: + extra_args: + resolv-conf: "/run/resolvconf/resolv.conf" +``` + +:::note + +由于 `kubelet` 在容器内è¿è¡Œï¼Œå› æ­¤ `/etc` å’Œ `/usr` 中文件的路径ä½äºŽ `kubelet` 容器内的 `/host/etc` å’Œ `/host/usr` 中。 + +::: + +请å‚阅[使用 YAML 编辑集群](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#使用-yaml-编辑集群)了解如何应用此修改。集群é…置完æˆåŽï¼Œä½ å¿…须删除 kube-dns pod 以激活 pod 中的新设置： + +``` +kubectl delete pods -n kube-system -l k8s-app=kube-dns +pod "kube-dns-5fd74c7488-6pwsf" deleted +``` + +ä½ å¯ä»¥[检查是å¦æ­£åœ¨è§£æžåŸŸå](#检查是å¦æ­£åœ¨è§£æžåŸŸå)æ¥å°è¯•å†æ¬¡è§£æžå称。 + +如果è¦æ£€æŸ¥é›†ç¾¤ä¸­çš„ kube-dns é…置（例如，检查是å¦é…置了ä¸åŒçš„上游å称æœåŠ¡å™¨ï¼‰ï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ—出 kube-dns é…置： + +``` +kubectl -n kube-system get configmap kube-dns -o go-template='{{range $key, $value := .data}}{{ $key }}{{":"}}{{ $value }}{{"\n"}}{{end}}' +``` + +输出示例： +``` +upstreamNameservers:["1.1.1.1"] +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md new file mode 100644 index 00000000000..8cb5fd34104 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md @@ -0,0 +1,28 @@ +--- +title: 过期 Webhook è¯ä¹¦è½®æ¢ +--- + +如果你的 Rancher 版本安装了 `rancher-webhook`，æŸäº›ç‰ˆæœ¬åˆ›å»ºçš„è¯ä¹¦å°†åœ¨ä¸€å¹´åŽè¿‡æœŸã€‚如果è¯ä¹¦æœªç»­è®¢ï¼Œä½ éœ€è¦è½®æ¢ä½ çš„ webhook è¯ä¹¦ã€‚ + +在 Rancher v2.6.3 åŠæ›´é«˜ç‰ˆæœ¬ä¸­ï¼Œrancher-webhook deployments å°†åœ¨åˆ°æœŸæ—¥æœŸåŽ 30 天或更短的时间内自动更新其 TLS è¯ä¹¦ã€‚如果你使用的是 v2.6.2 或更低版本，你å¯ä»¥é€šè¿‡ä¸‹é¢ä¸¤ç§æ–¹æ³•æ¥è§£å†³è¿™ä¸ªé—®é¢˜ã€‚ + +##### 1. 如果用户具有集群访问æƒé™ï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +``` +kubectl delete secret -n cattle-system cattle-webhook-tls +kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io --ignore-not-found=true rancher.cattle.io +kubectl delete pod -n cattle-system -l app=rancher-webhook +``` + +##### 2. 如果用户没有集群访问æƒé™ï¼Œä½¿ç”¨ `kubectl`： + +1. 删除 local 集群 `cattle-system` 命å空间中的 `cattle-webhook-tls` 密文。 + +2. 删除 `rancher.cattle.io` mutating webhook。 + +3. 删除 local 集群 `cattle-system` 命å空间中的 `rancher-webhook` pod。 + +:::note + +webhook è¯ä¹¦è¿‡æœŸé—®é¢˜ä¸æ­¢ç¤ºä¾‹ä¸­åˆ—出的 `cattle-webhook-tls`。你需è¦ç›¸åº”地填写你过期的è¯ä¹¦å¯†æ–‡ã€‚ + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md new file mode 100644 index 00000000000..33e7a22738d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md @@ -0,0 +1,255 @@ +--- +title: Kubernetes èµ„æº +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥æœ€é‡è¦çš„ Kubernetes 资æºï¼Œå¹¶åº”用于 [Rancher å¯åŠ¨çš„ Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，为 Rancher HA é…置了 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + + +## 节点 + +### 获å–节点 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤å¹¶æ£€æŸ¥ä»¥ä¸‹å†…容： + +- 集群中的所有节点都已列出，没有缺失的节点。 +- 所有节点的状æ€éƒ½æ˜¯ **Ready**（如果未处于 **Ready** 状æ€ï¼Œè¯·è¿è¡Œ `docker logs kubelet` 检查该节点上的 `kubelet` 容器日志) +- 检查所有节点是å¦æŠ¥å‘Šäº†æ­£ç¡®çš„版本。 +- 检查 OS/Kernel/Docker 值是å¦æŒ‰é¢„期显示（此问题å¯èƒ½ä¸Žå‡çº§ OS/Kernel/Docker 相关）。 + + +``` +kubectl get nodes -o wide +``` + +输出示例： + +``` +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +controlplane-0 Ready controlplane 31m v1.13.5 138.68.188.91 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +etcd-0 Ready etcd 31m v1.13.5 138.68.180.33 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +worker-0 Ready worker 30m v1.13.5 139.59.179.88 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +``` + +### 获å–节点状况 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ—出具有 [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) 的节点： + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' +``` + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œåˆ—出具有 active [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) 的节点，这些节点å¯èƒ½é˜»æ­¢æ­£å¸¸æ“作： + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{if ne .type "Ready"}}{{if eq .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{else}}{{if ne .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{": "}}{{.status}}{{"\n"}}{{end}}{{end}}{{end}}{{end}}' +``` + +输出示例： + +``` +worker-0: DiskPressure:True +``` + +## Kubernetes leader 选举 + +### Kubernetes Controller 管ç†å™¨ leader + +Leader ç”± Leader 选举确定。确定 Leader åŽï¼ŒLeader（`holderIdentity`）会ä¿å­˜åœ¨ `kube-controller-manager` 端点中（在本例中为 `controlplane-0`）。 + +``` +kubectl -n kube-system get endpoints kube-controller-manager -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +### Kubernetes 调度器 Leader + +Leader ç”± Leader 选举确定。确定 Leader åŽï¼ŒLeader（`holderIdentity`）会ä¿å­˜åœ¨ `kube-scheduler` 端点中（在本例中为 `controlplane-0`）。 + +``` +kubectl -n kube-system get endpoints kube-scheduler -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +## Ingress Controller + +默认的 Ingress Controller 是 NGINX，作为 DaemonSet 部署在 `ingress-nginx` 命å空间中。Pod 仅会调度到具有 `worker` 角色的节点。 + +检查 pod 是å¦è¿è¡Œåœ¨æ‰€æœ‰èŠ‚点上： + +``` +kubectl -n ingress-nginx get pods -o wide +``` + +输出示例： + +``` +kubectl -n ingress-nginx get pods -o wide +NAME READY STATUS RESTARTS AGE IP NODE +default-http-backend-797c5bc547-kwwlq 1/1 Running 0 17m x.x.x.x worker-1 +nginx-ingress-controller-4qd64 1/1 Running 0 14m x.x.x.x worker-1 +nginx-ingress-controller-8wxhm 1/1 Running 0 13m x.x.x.x worker-0 +``` + +如果 pod 无法è¿è¡Œï¼ˆå³çŠ¶æ€ä¸æ˜¯ **Running**，Ready 状æ€æœªæ˜¾ç¤º `1/1`ï¼Œæˆ–è€…æœ‰å¤§é‡ Restarts），请检查 pod 详细信æ¯ï¼Œæ—¥å¿—和命å空间事件。 + +### Pod è¯¦ç»†ä¿¡æ¯ + +``` +kubectl -n ingress-nginx describe pods -l app=ingress-nginx +``` + +### Pod 容器日志 + +下é¢çš„命令å¯ä»¥æ˜¾ç¤ºæ‰€æœ‰æ ‡è®°ä¸º “app=ingress-nginx†的 pod 的日志，但是由于 `kubectl logs` 命令的é™åˆ¶ï¼Œå®ƒåªä¼šæ˜¾ç¤º 10 行日志。有关详细信æ¯ï¼Œè¯·å‚阅 `kubectl logs -h` çš„ `--tail`。 + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +如果需è¦æŸ¥çœ‹å®Œæ•´çš„æ—¥å¿—ï¼Œè¯·åœ¨å‘½ä»¤ä¸­å°¾éš pod å称： + +``` +kubectl -n ingress-nginx logs +``` + +### 命å空间事件 + +``` +kubectl -n ingress-nginx get events +``` + +### 调试日志 + +è¦å¯ç”¨è°ƒè¯•æ—¥å¿—： + +``` +kubectl -n ingress-nginx patch ds nginx-ingress-controller --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--v=5"}]' +``` + +### 检查é…ç½® + +在æ¯ä¸ª pod 中检索生æˆçš„é…置： + +``` +kubectl -n ingress-nginx get pods -l app=ingress-nginx --no-headers -o custom-columns=.NAME:.metadata.name | while read pod; do kubectl -n ingress-nginx exec $pod -- cat /etc/nginx/nginx.conf; done +``` + +## Rancher Agents + +Rancher Agent 用于实现与集群的通信（通过 `cattle-cluster-agent` çš„ Kubernetes API）和与节点的通信（通过 `cattle-node-agent` 的集群é…置）。 + +#### cattle-node-agent + +检查æ¯ä¸ªèŠ‚点上是å¦å­˜åœ¨ cattle-node-agent pod，状æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +检查特定或所有 cattle-node-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +检查 cattle-cluster-agent pod 是å¦å­˜åœ¨äºŽé›†ç¾¤ä¸­ï¼ŒçŠ¶æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +检查 cattle-cluster-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` + +## Jobs å’Œ Pods + +### 检查 Pod 或 Job 的状æ€æ˜¯å¦ä¸º **Running**/**Completed** + +è¿è¡Œä»¥ä¸‹å‘½ä»¤è¿›è¡Œæ£€æŸ¥ï¼š + +``` +kubectl get pods --all-namespaces +``` + +如果 Pod 的状æ€ä¸æ˜¯ **Running**，你å¯ä»¥é€šè¿‡è¿è¡Œå‘½ä»¤æ¥æ‰¾åˆ°æ ¹æœ¬åŽŸå› ã€‚ + +### æè¿° Pod + +``` +kubectl describe pod POD_NAME -n NAMESPACE +``` + +### Pod 容器日志 + +``` +kubectl logs POD_NAME -n NAMESPACE +``` + +如果 Job 的状æ€ä¸æ˜¯ **Completed**，你å¯ä»¥é€šè¿‡è¿è¡Œå‘½ä»¤æ¥æ‰¾åˆ°æ ¹æœ¬åŽŸå› ã€‚ + +### æè¿° Job + +``` +kubectl describe job JOB_NAME -n NAMESPACE +``` + +### Job Pod 容器的日志 + +``` +kubectl logs -l job-name=JOB_NAME -n NAMESPACE +``` + +### 驱é€çš„ Pod + +å¯ä»¥æ ¹æ® [eviction ä¿¡å·](https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/#eviction-policy)æ¥é©±é€ Pod。 + +检索被驱é€çš„ Pod 列表（podname 和命å空间）： + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' +``` + +è¦åˆ é™¤æ‰€æœ‰è¢«é©±é€çš„ pod： + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace delete pod $epod; done +``` + +检索被驱é€çš„ pod 列表ã€è°ƒåº¦èŠ‚点以åŠåŽŸå› ï¼š + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace get pod $epod -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName,MSG:.status.message; done +``` + +### Job æœªå®Œæˆ + +如果你å¯ç”¨äº† Istio 而且你部署的 Job 未完æˆï¼Œä½ éœ€è¦æŒ‰ç…§[这些步骤](../../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md)将注释添加到 Pod 中。 + +由于 Istio Sidecars 会一直è¿è¡Œï¼Œå› æ­¤å³ä½¿ä»»åŠ¡å®Œæˆäº†ï¼Œä¹Ÿä¸èƒ½è®¤ä¸º Job 已完æˆã€‚这是一个临时的解决方法，它ç¦æ­¢äº† Istio 和添加了注释的 Pod 之间的通信。如果你使用了这个方法，由于这个 Job 无法访问æœåŠ¡ç½‘格，因此你将ä¸èƒ½ç»§ç»­ä½¿ç”¨ Job 进行集æˆæµ‹è¯•ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/logging.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/logging.md new file mode 100644 index 00000000000..7b74521fbb9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/logging.md @@ -0,0 +1,72 @@ +--- +title: Logging +--- + +## 日志级别 + +Rancher 使用了以下日志级别： + +| å称 | æè¿° | +|---------|-------------| +| `info` | 记录信æ¯æ€§æ¶ˆæ¯ã€‚这是默认的日志级别。 | +| `debug` | 记录å¯ç”¨äºŽè°ƒè¯•çš„更详细消æ¯ã€‚ | +| `trace` | 记录内部功能的éžå¸¸è¯¦ç»†çš„消æ¯ã€‚éžå¸¸å†—长，并且å¯èƒ½åŒ…å«æ•æ„Ÿä¿¡æ¯ã€‚ | + +### 如何é…置日志级别 + +#### Kubernetes 安装 + +* é…ç½® debug 日志级别 + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set debug; done +OK +OK +OK +$ kubectl -n cattle-system logs -l app=rancher -c rancher +``` + +* é…ç½® info 日志级别 + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set info; done +OK +OK +OK +``` + +#### Docker 安装 + +* é…ç½® debug 日志级别 + +``` +$ docker exec -ti loglevel --set debug +OK +$ docker logs -f +``` + +* é…ç½® info 日志级别 + +``` +$ docker exec -ti loglevel --set info +OK +``` + +## Rancher 主机调试日志 +如果需è¦è§£å†³åœ¨åŸºç¡€è®¾æ–½æ供商中创建对象的问题，你å¯ä»¥ä½¿ç”¨ `rancher-machine` 调试日志。 + +ä½ å¯ä»¥åœ¨å¯åŠ¨ Rancher 时设置环境å˜é‡æ¥ä¸º `rancher-machine` å¯ç”¨è°ƒè¯•æ—¥å¿—。 + +`CATTLE_WHITELIST_ENVVARS` 环境å˜é‡å…许用户将特定环境å˜é‡åˆ—入白åå•ï¼Œè®©å®ƒä»¬èƒ½åœ¨é…置期间传递给 `rancher-machine`。 + +`MACHINE_DEBUG` å˜é‡åœ¨ `rancher-machine` 中å¯ç”¨è°ƒè¯•æ—¥å¿—。 + +因此，通过设置 `MACHINE_DEBUG=true` 并将 `MACHINE_DEBUG` 添加到 `CATTLE_WHITELIST_ENVVARS` 中的å˜é‡é»˜è®¤åˆ—表（例如 `CATTLE_WHITELIST_ENVVARS=HTTP_PROXY,HTTPS_PROXY,NO_PROXY,MACHINE_DEBUG`），你å¯ä»¥åœ¨é…ç½® RKE1ã€RKE2 å’Œ K3s 集群时在 `rancher-machine` 中å¯ç”¨è°ƒè¯•æ—¥å¿—。 + +:::caution + +åƒä¸Šé¢çš„ `trace` 日志级别一样，`rancher-machine` 调试日志å¯ä»¥åŒ…å«æ•æ„Ÿä¿¡æ¯ã€‚ + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/networking.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/networking.md new file mode 100644 index 00000000000..47c806b0278 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/networking.md @@ -0,0 +1,121 @@ +--- +title: 网络 +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥é›†ç¾¤ä¸­çš„网络问题。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，为 Rancher HA é…置了 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + +### 仔细检查你的（主机）防ç«å¢™ä¸­æ˜¯å¦æ‰“å¼€äº†æ‰€æœ‰å¿…éœ€çš„ç«¯å£ + +仔细检查你的（主机）防ç«å¢™ä¸­æ˜¯å¦æ‰“开了所有[必需的端å£](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。其他所需的端å£éƒ½ä½¿ç”¨ TCP，而覆盖网络使用 UDP。 +### 检查覆盖网络是å¦æ­£å¸¸è¿è¡Œ + +ä½ å¯ä»¥å°† Pod 调度到集群中的任何主机，但是 NGINX Ingress Controller 需è¦èƒ½å¤Ÿå°† `NODE_1` 请求路由到 `NODE_2`。这会在覆盖网络上进行。如果覆盖网络无法正常工作，由于 NGINX Ingress Controller 无法路由到 pod，因此 TCP/HTTP 连接会间歇性失败。 + +è¦æµ‹è¯•è¦†ç›–网络，你å¯ä»¥å¯åŠ¨ä»¥ä¸‹ `DaemonSet` 定义。这样，æ¯å°ä¸»æœºä¸Šéƒ½ä¼šè¿è¡Œä¸€ä¸ª `swiss-army-knife` 容器（镜åƒç”± Rancher 工程师开å‘，å¯åœ¨[此处](https://github.com/rancherlabs/swiss-army-knife)找到），我们使用该镜åƒåœ¨æ‰€æœ‰ä¸»æœºä¸Šçš„容器之间è¿è¡Œ `ping` 测试。 + +:::note + +此容器[ä¸æ”¯æŒ ARM 节点](https://github.com/leodotcloud/swiss-army-knife/issues/18)（例如 Raspberry Pi）。Pod 日志会显示为 `exec user process caused: exec format error`。 + +::: + +1. 将以下内容å¦å­˜ä¸º `overlaytest.yml`： + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: overlaytest + spec: + selector: + matchLabels: + name: overlaytest + template: + metadata: + labels: + name: overlaytest + spec: + tolerations: + - operator: Exists + containers: + - image: rancherlabs/swiss-army-knife + imagePullPolicy: Always + name: overlaytest + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + + ``` + +2. 使用 `kubectl create -f overlaytest.yml` å¯åŠ¨å®ƒã€‚ +3. 等待 `kubectl rollout status ds/overlaytest -w` 返回 `daemon set "overlaytest" successfully rolled out`。 +4. 从åŒä¸€ä½ç½®è¿è¡Œä»¥ä¸‹è„šæœ¬ã€‚它会让æ¯ä¸ªä¸»æœºä¸Šçš„所有 `overlaytest` 容器相互 ping 通： + ``` + #!/bin/bash + echo "=> Start network overlay test" + kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.metadata.name}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read spod shost + do kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.status.podIP}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read tip thost + do kubectl --request-timeout='10s' exec $spod -c overlaytest -- /bin/sh -c "ping -c2 $tip > /dev/null 2>&1" + RC=$? + if [ $RC -ne 0 ] + then echo FAIL: $spod on $shost cannot reach pod IP $tip on $thost + else echo $shost can reach $thost + fi + done + done + echo "=> End network overlay test" + ``` + +5. 命令è¿è¡Œå®ŒæˆåŽï¼Œå®ƒä¼šè¾“出æ¯æ¡è·¯ç”±çš„状æ€ï¼š + + ``` + => Start network overlay test + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.7.3 on wk2 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.0.5 on cp1 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.2.12 on wk1 + command terminated with exit code 1 + FAIL: overlaytest-v4qkl on cp1 cannot reach pod IP 10.42.7.3 on wk2 + cp1 can reach cp1 + cp1 can reach wk1 + command terminated with exit code 1 + FAIL: overlaytest-xpxwp on wk1 cannot reach pod IP 10.42.7.3 on wk2 + wk1 can reach cp1 + wk1 can reach wk1 + => End network overlay test + ``` + 如果你在输出中看到错误，则说明两å°ä¸»æœºä¸Šçš„ Pod 路由存在问题。在上é¢çš„输出中，节点 `wk2` 在覆盖网络上没有连接。原因å¯èƒ½æ˜¯æ²¡æœ‰ä¸º `wk2` 打开覆盖网络的[必需端å£](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 +6. 你现在å¯ä»¥é€šè¿‡è¿è¡Œ `kubectl delete ds/overlaytest` æ¥æ¸…ç† DaemonSet。 + + +### 检查主机和对等/隧é“设备/设备上的 MTU 是å¦æ­£ç¡®é…ç½® + +如果 MTU 在è¿è¡Œ Rancher 的主机ã€åˆ›å»º/导入集群中的节点或两者之间的设备上é…ç½®ä¸æ­£ç¡®ï¼ŒRancher å’Œ Agent 会记录类似以下的错误信æ¯ï¼š + +* `websocket: bad handshake` +* `Failed to connect to proxy` +* `read tcp: i/o timeout` + +有关在 Rancher 和集群节点之间使用 Google Cloud VPN 时如何正确é…ç½® MTU 的示例，请å‚阅 [Google Cloud VPN：MTU 注æ„事项](https://cloud.google.com/vpn/docs/concepts/mtu-considerations#gateway_mtu_vs_system_mtu)。 + +### 已解决的问题 + +#### 由于缺少节点注释，使用 Canal/Flannel 时覆盖网络中断 + +| | | +|------------|------------| +| GitHub issue | [#13644](https://github.com/rancher/rancher/issues/13644) | +| 解决于 | v2.1.2 | + +è¦æ£€æŸ¥ä½ çš„集群是å¦å—到影å“，è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ—出æŸå的节点（此命令è¦æ±‚安装 `jq`）： + +``` +kubectl get nodes -o json | jq '.items[].metadata | select(.annotations["flannel.alpha.coreos.com/public-ip"] == null or .annotations["flannel.alpha.coreos.com/kube-subnet-manager"] == null or .annotations["flannel.alpha.coreos.com/backend-type"] == null or .annotations["flannel.alpha.coreos.com/backend-data"] == null) | .name' +``` + +如果没有输出，则集群没有影å“。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/rancher-ha.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/rancher-ha.md new file mode 100644 index 00000000000..630f4f13de6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/rancher-ha.md @@ -0,0 +1,79 @@ +--- +title: Rancher HA +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥ä½ çš„ Rancher Kubernetes 安装。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，`export KUBECONFIG=$PWD/kube_config_cluster.yml`）。 + +### 检查 Rancher Pod + +Rancher pod 会部署为 `cattle-system` 命å空间中的一个 Deployment。 + +检查 pod 是å¦è¿è¡Œåœ¨æ‰€æœ‰èŠ‚点上： + +``` +kubectl -n cattle-system get pods -l app=rancher -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +rancher-7dbd7875f7-n6t5t 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qbj5k 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qw7wb 1/1 Running 0 8m x.x.x.x x.x.x.x +``` + +如果 pod 无法è¿è¡Œï¼ˆå³çŠ¶æ€ä¸æ˜¯ **Running**，Ready 状æ€æœªæ˜¾ç¤º `1/1`ï¼Œæˆ–è€…æœ‰å¤§é‡ Restarts），请检查 pod 详细信æ¯ï¼Œæ—¥å¿—和命å空间事件。 + +#### Pod è¯¦ç»†ä¿¡æ¯ + +``` +kubectl -n cattle-system describe pods -l app=rancher +``` + +#### Pod 容器日志 + +``` +kubectl -n cattle-system logs -l app=rancher +``` + +#### 命å空间事件 + +``` +kubectl -n cattle-system get events +``` + +### 检查 Ingress + +Ingress 应该具有正确的 `HOSTS`（显示é…置的 FQDN）和 `ADDRESS`（将被路由到该主机地å€ï¼‰ï¼š + +``` +kubectl -n cattle-system get ingress +``` + +输出示例： + +``` +NAME HOSTS ADDRESS PORTS AGE +rancher rancher.yourdomain.com x.x.x.x,x.x.x.x,x.x.x.x 80, 443 2m +``` + +### 检查 Ingress Controller 日志 + +如果访问你é…置的 Rancher FQDN 时没有显示 UI，请检查 Ingress Controller 日志以查看å°è¯•è®¿é—® Rancher æ—¶å‘生了什么： + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +### Leader 选举 + +Leader ç”± Leader 选举确定。确定 Leader åŽï¼ŒLeader（`holderIdentity`）会ä¿å­˜åœ¨ `cattle-controllers` ConfigMap 中（在本例中为 `rancher-7dbd7875f7-qbj5k`）： + +``` +kubectl -n kube-system get configmap cattle-controllers -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"rancher-7dbd7875f7-qbj5k","leaseDurationSeconds":45,"acquireTime":"2019-04-04T11:53:12Z","renewTime":"2019-04-04T12:24:08Z","leaderTransitions":0} +``` + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/registered-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/registered-clusters.md new file mode 100644 index 00000000000..58eff6a97ed --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/registered-clusters.md @@ -0,0 +1,67 @@ +--- +title: 注册集群 +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥ä½ æ­£åœ¨æ³¨å†Œæˆ–已在 Rancher 中注册的集群。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，`export KUBECONFIG=$PWD/kubeconfig_from_imported_cluster.yml`）。 + +### Rancher Agents + +Rancher Agent 用于实现与集群的通信（通过 cattle-cluster-agent çš„ Kubernetes API）和与节点的通信。 + +如果 cattle-cluster-agent 无法连接到é…置的 `server-url`，集群将ä¿æŒåœ¨ **Pending** 状æ€å¹¶æ˜¾ç¤º `Waiting for full cluster configuration`。 + +#### cattle-node-agent + +:::note + +cattle-node-agents åªä¼šåœ¨ä½¿ç”¨ RKE 在 Rancher 创建的集群中显示。 + +::: + +检查æ¯ä¸ªèŠ‚点上是å¦å­˜åœ¨ cattle-node-agent pod，状æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +检查特定或所有 cattle-node-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +检查 cattle-cluster-agent pod 是å¦å­˜åœ¨äºŽé›†ç¾¤ä¸­ï¼ŒçŠ¶æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +检查 cattle-cluster-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md new file mode 100644 index 00000000000..bdbac691333 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md @@ -0,0 +1,22 @@ +--- +title: 审计日志中的用户 ID 跟踪 +--- + +Rancher 使用以下审计日志æ¥è·Ÿè¸ªæœ¬åœ°å’Œä¸‹æ¸¸é›†ç¾¤ä¸­å‘生的事件： + +* [Kubernetes 审计日志](https://rancher.com/docs/rke/latest/en/config-options/audit-log/) +* [Rancher API 审核日志](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) + +Rancher 2.6 增强了审计日志，在 Rancher 和下游 Kubernetes 审计日志中都包å«äº†å¤–部身份æ供程åºå称（å³å¤–部身份æ供程åºä¸­ç”¨æˆ·çš„通用å称）。 + +在 Rancher 2.6 之å‰ï¼Œå¦‚果管ç†å‘˜ä¸çŸ¥é“外部身份æ供程åºä¸­çš„用户åå’Œ Rancher 中的 userId (`u-xXXX`) 之间的映射，则无法通过 Rancher 审计日志和 Kubernetes 审计日志æ¥è¿½è¸ªäº‹ä»¶ã€‚ +è¦çŸ¥é“这个映射，集群管ç†å‘˜éœ€è¦èƒ½å¤Ÿè®¿é—® Rancher APIã€UI å’Œ local 管ç†é›†ç¾¤ã€‚ + +有了这个功能之åŽï¼Œä¸‹æ¸¸é›†ç¾¤ç®¡ç†å‘˜å°±èƒ½å¤ŸæŸ¥çœ‹ Kubernetes 审计日志，并且å¯ä»¥ä¸åœ¨ Rancher 中查看任何内容就能知é“哪个外部身份æä¾›ç¨‹åº (IDP) 用户执行了æŸé¡¹æ“作。 +如果集群传出了审计日志，日志系统的用户就能够识别外部身份æ供程åºä¸­çš„用户。 +Rancher 管ç†å‘˜çŽ°åœ¨èƒ½å¤ŸæŸ¥çœ‹ Rancher 审计日志，并使用外部身份æ供程åºç”¨æˆ·åæ¥è·Ÿè¸ª Kubernetes 审计日志。 + +### 功能æè¿° + +- 在下游集群å¯ç”¨äº† Kubernetes 审计日志åŽï¼Œä¼šåœ¨â€œå…ƒæ•°æ®â€çº§åˆ«ä¸ºæ¯ä¸ªè¯·æ±‚记录外部身份æ供程åºçš„用户å。 +- 在 Rancher å¯ç”¨ Rancher API 审计日志åŽï¼Œæ¯ä¸ªåˆ°è¾¾ Rancher API Server 的请求记录（包括登录请求）对应的外部身份æ供程åºç”¨æˆ·å也会记录在 `auditLog.level=1` 中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-2.8/v2.7.md b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/v2.7.md new file mode 100644 index 00000000000..4e36ad8730b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-2.8/v2.7.md @@ -0,0 +1,16 @@ +--- +title: v2.7 +description: Dummy file used to redirect to the base url +--- + + + +import {Redirect} from '@docusaurus/router'; + +const Home = () => { +return ; +}; \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/backups/docker-installs/docker-installs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/backups/docker-installs/docker-installs.md new file mode 100644 index 00000000000..51c3001d777 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/backups/docker-installs/docker-installs.md @@ -0,0 +1,6 @@ +--- +title: 备份和æ¢å¤ Docker 安装的 Rancher +--- + +- [备份](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) +- [还原](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/cluster-provisioning/rke-clusters/options/options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/cluster-provisioning/rke-clusters/options/options.md new file mode 100644 index 00000000000..39c332461ce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/cluster-provisioning/rke-clusters/options/options.md @@ -0,0 +1,5 @@ +--- +title: RKE 集群é…ç½® +--- + +本文已è¿ç§»åˆ°[此处](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/contribute-to-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/contribute-to-rancher.md new file mode 100644 index 00000000000..baff7b735b9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/contribute-to-rancher.md @@ -0,0 +1,136 @@ +--- +title: å‚与 Rancher 社区贡献 +--- + +本文介ç»äº† Rancher 仓库和 Rancher 文档ã€å¦‚何构建 Rancher 仓库以åŠæ交 issue æ—¶è¦åŒ…å«å“ªäº›ä¿¡æ¯ã€‚ + +有关如何为 Rancher 项目开å‘åšå‡ºè´¡çŒ®çš„更多详细信æ¯ï¼Œè¯·å‚阅 [Rancher Developer Wiki](https://github.com/rancher/rancher/wiki)。Wiki 包å«ä»¥ä¸‹ä¸»é¢˜çš„资æºï¼š + +- 如何æ­å»º Rancher å¼€å‘环境并è¿è¡Œæµ‹è¯• +- Issue 在开å‘生命周期中的典型æµç¨‹ +- ç¼–ç æŒ‡å—和开å‘最佳实践 +- 调试和故障排除 +- å¼€å‘ Rancher API + +在 Rancher Users Slack 上，开å‘者的频é“是 **#developer**。 + +## Rancher 文档 + +如果你对此网站上的文档有建议，请在主 [Rancher 文档](https://github.com/rancher/rancher-docs)仓库中[æ交 issue](https://github.com/rancher/rancher-docs/issues/new/choose)ã€‚æ­¤ä»“åº“åŒ…å« Rancher v2.0 åŠæ›´é«˜ç‰ˆæœ¬çš„文档。 + +有关贡献和构建 Rancher v2.x 文档仓库的更多详细信æ¯ï¼Œè¯·å‚阅 [Rancher 文档 README](https://github.com/rancher/rancher-docs#readme)。 + +有关 Rancher v1.6 åŠæ›´æ—©ç‰ˆæœ¬çš„文档，请å‚阅 [Rancher 1.x docs](https://github.com/rancher/rancher.github.io) ä»“åº“ï¼Œå…¶ä¸­åŒ…å« https://rancher.com/docs/rancher/v1.6/en/ çš„æºæ–‡ä»¶ã€‚ + +## Rancher 仓库 + +所有仓库都ä½äºŽæˆ‘ä»¬çš„ä¸»è¦ GitHub 组织内。Rancher 使用了很多仓库，以下是部分主è¦ä»“库的æ述： + +| 仓库 | URL | æè¿° | +-----------|-----|------------- +| Rancher | https://github.com/rancher/rancher | Rancher 2.x 的主è¦æºç ä»“库。 | +| Types | https://github.com/rancher/types | åŒ…å« Rancher 2.x 的所有 API 类型的仓库。 | +| API Framework | https://github.com/rancher/norman | API 框架，用于构建由 Kubernetes 自定义资æºæ”¯æŒçš„ Rancher 风格的 API。 | +| User Interface | https://github.com/rancher/dashboard/ | Dashboard UI æºç ä»“库。 | +| (Rancher) Docker Machine | https://github.com/rancher/machine | 使用主机驱动时使用的 Docker Machine 二进制文件的æºç ä»“库。这是 `docker/machine` 仓库的一个 fork。 | +| machine-package | https://github.com/rancher/machine-package | 用于构建 Rancher Docker Machine 二进制文件。 | +| kontainer-engine | https://github.com/rancher/kontainer-engine | kontainer-engine çš„æºç ä»“库，它是é…置托管 Kubernetes 集群的工具。 | +| RKE repository | https://github.com/rancher/rke | Rancher Kubernetes Engine çš„æºç ä»“库，该工具å¯åœ¨ä»»ä½•ä¸»æœºä¸Šé…ç½® Kubernetes 集群。 | +| CLI | https://github.com/rancher/cli | Rancher 2.x 中使用的 Rancher CLI çš„æºç ä»“库。 | +| (Rancher) Helm repository | https://github.com/rancher/helm | 打包的 Helm 二进制文件的æºç ä»“库。这是 `helm/helm` 仓库的一个 fork。 | +| Telemetry repository | https://github.com/rancher/telemetry | Telemetry 二进制文件的æºç ä»“库。 | +| loglevel repository | https://github.com/rancher/loglevel | loglevel 二进制文件的æºç ä»“库，用于动æ€æ›´æ”¹æ—¥å¿—级别。 | + +è¦æŸ¥çœ‹ Rancher 使用的所有库/项目，请查看 `rancher/rancher` 仓库中的 [`go.mod` 文件](https://github.com/rancher/rancher/blob/master/go.mod)。 + +![Rancher diagram](/img/ranchercomponentsdiagram-2.6.svg)
+用于é…ç½®/ç®¡ç† Kubernetes 集群的 Rancher 组件。 + +### 构建 Rancher 仓库 + +æ¯ä¸ªä»“库都应该有一个 Makefile，并且å¯ä»¥ä½¿ç”¨ `make` 命令进行构建。`make` 目标基于仓库中 `/scripts` 目录中的脚本，æ¯ä¸ªç›®æ ‡éƒ½ä½¿ç”¨ [Dapper](https://github.com/rancher/dapper) 在孤立的环境中è¿è¡Œã€‚`Dockerfile.dapper` 将用于此æ“作，它包å«äº†æ‰€éœ€çš„所有构建工具。 + +默认目标是 `ci`，它将è¿è¡Œ `./scripts/validate`ã€`./scripts/build`ã€`./scripts/test ` å’Œ `./scripts/package`。生æˆçš„二进制文件将在 `./build/bin` 中，通常也打包在 Docker é•œåƒä¸­ã€‚ + +### Rancher Bugã€Issue 或疑问 + +如果你å‘现任何 bug 或问题，由于有人å¯èƒ½é‡åˆ°è¿‡åŒæ ·çš„问题，或者我们已ç»æ­£åœ¨å¯»æ‰¾è§£å†³æ–¹æ¡ˆï¼Œå› æ­¤è¯·å…ˆåœ¨[已报告 issue](https://github.com/rancher/rancher/issues) 中æœç´¢ã€‚ + +如果找ä¸åˆ°ä¸Žä½ çš„问题相关的内容，请通过[æ出 issue](https://github.com/rancher/rancher/issues/new) 与我们è”系。与 Rancher 相关的仓库有很多，但请将 issue æ交到 Rancher 仓库中，这样能确ä¿æˆ‘们能看到这些 issue。如果你想就一个用例æ出问题或询问其他用户，你å¯ä»¥åœ¨ [Rancher 论å›](https://forums.rancher.com)上å‘帖。 + +#### æ交 Issue çš„æ£€æŸ¥æ¸…å• + +æ交问题时请éµå¾ªæ­¤æ¸…å•ï¼Œä»¥ä¾¿æˆ‘们调查和解决问题。如果你能æ供更多信æ¯ï¼Œæˆ‘们就å¯ä»¥ä½¿ç”¨æ›´å¤šæ•°æ®æ¥ç¡®å®šå¯¼è‡´é—®é¢˜çš„原因或å‘现更多相关的内容。 + +:::note + +如果数æ®é‡å¾ˆå¤§ï¼Œè¯·ä½¿ç”¨ [GitHub Gist](https://gist.github.com/) 或类似工具，并在 issue 中链接你创建的资æºã€‚ + +::: + +:::note é‡è¦æ示： + +请删除所有æ•æ„Ÿæ•°æ®ã€‚ + +::: + +- **资æº**：请尽é‡è¯¦ç»†åœ°æ供所使用的资æºã€‚导致问题的原因å¯èƒ½å¾ˆå¤šï¼Œå› æ­¤è¯·å°½é‡æ供更多细节æ¥å¸®åŠ©æˆ‘们确定根本原因。下é¢æ˜¯ä¸€äº›å‚考示例： + - **主机**：主机的规格（例如 CPU/内存/ç£ç›˜ï¼‰ï¼Œè¿è¡Œåœ¨ä»€ä¹ˆäº‘厂商上，使用的 Amazon Machine Image，使用的 DigitalOcean droplet，é…置的镜åƒï¼ˆå¤çŽ°æ—¶ç”¨äºŽé‡æ–°æž„建或使用）。 + - **æ“作系统**：使用的是什么æ“作系统。在此处æ供详细信æ¯ï¼Œä¾‹å¦‚ `cat /etc/os-release` 的输出（确切的æ“作系统版本）和 `uname -r` 的输出（确切的内核）。 + - **Docker**：使用的 Docker 版本以åŠå®‰è£…的方法。Docker 的大部分详情都å¯ä»¥åœ¨ `docker version` å’Œ `docker info` 的输出中找到。 + - **环境**：是å¦ä½¿ç”¨äº†ä»£ç†ï¼Œæ˜¯å¦ä½¿ç”¨å¯ä¿¡çš„ CA/自签åè¯ä¹¦ï¼Œæ˜¯å¦ä½¿ç”¨äº†å¤–部负载å‡è¡¡å™¨ã€‚ + - **Rancher**：使用的 Rancher 版本，å¯ä»¥åœ¨ UI 左下角或者从主机è¿è¡Œçš„ image 标签中获å–。 + - **集群**：创建了什么样的集群，如何创建的，在创建时指定了什么å‚数。 +- **å¤çŽ° issue 的步骤**：尽é‡è¯¦ç»†åœ°è¯´æ˜Žä½ æ˜¯å¦‚何触å‘所报告的情况的。这有助于å¤çŽ°ä½ çš„情况。 + - æ供从创建到你报告的情况使用的手动步骤或自动化脚本。 +- **日志**：æ供使用资æºçš„æ•°æ®/日志。 + - Rancher + - Docker 安装 + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher:|rancher/rancher " | awk '{ print $1 }') + ``` + - 使用 `kubectl` çš„ Kubernetes 安装 + + :::note + + ç¡®ä¿ä½ é…置了正确的 kubeconfig（例如，如果 Rancher 安装在 Kubernetes 集群上，则 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + -l app=rancher \ + --timestamps=true + ``` + - 在 RKE 集群的æ¯ä¸ªèŠ‚点上使用 `docker` çš„ Docker 安装 + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher@|rancher_rancher" | awk '{ print $1 }') + ``` + - 使用 RKE 附加组件的 Kubernetes 安装 + + :::note + + ç¡®ä¿ä½ é…置了正确的 kubeconfig（例如，如果 Rancher Server 安装在 Kubernetes 集群上，则 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + --timestamps=true \ + -f $(kubectl --kubeconfig $KUBECONFIG get pods -n cattle-system -o json | jq -r '.items[] | select(.spec.containers[].name="cattle-server") | .metadata.name') + ``` + - 系统日志记录（å¯èƒ½ä¸å­˜åœ¨ï¼Œå–决于æ“作系统） + - `/var/log/messages` + - `/var/log/syslog` + - `/var/log/kern.log` + - Docker Daemon 日志记录（å¯èƒ½å¹¶ä¸å…¨éƒ¨å­˜åœ¨ï¼Œå–决于æ“作系统） + - `/var/log/docker.log` +- **指标**：如果你é‡åˆ°æ€§èƒ½é—®é¢˜ï¼Œè¯·æ供尽å¯èƒ½å¤šçš„指标数æ®ï¼ˆæ–‡ä»¶æˆ–å±å¹•æˆªå›¾ï¼‰æ¥å¸®åŠ©æˆ‘们确定问题。如果你é‡åˆ°ä¸»æœºç›¸å…³çš„问题，你å¯ä»¥æä¾› `top`ã€`free -m`ã€`df` 的输出，这些输出会显示进程/内存/ç£ç›˜çš„使用情况。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/container-network-interface-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/container-network-interface-providers.md new file mode 100644 index 00000000000..ad796bb5c4c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/container-network-interface-providers.md @@ -0,0 +1,204 @@ +--- +title: CNI 网络æ’件 +description: äº†è§£å®¹å™¨ç½‘ç»œæŽ¥å£ (CNI)ã€Rancher æ供的 CNI 网络æ’件ã€æ供商的功能，以åŠå¦‚何选择网络æ供商 +--- + +## 什么是 CNI？ + +CNI（容器网络接å£ï¼‰æ˜¯ä¸€ä¸ª[云原生计算基金会项目](https://cncf.io/)，它包å«äº†ä¸€äº›è§„范和库，用于编写在 Linux 容器中é…置网络接å£çš„一系列æ’件。CNI åªå…³æ³¨å®¹å™¨çš„网络连接，并在容器被删除时移除所分é…的资æºã€‚ + +Kubernetes 使用 CNI 作为网络æ供商和 Kubernetes Pod 网络之间的接å£ã€‚ + +![CNI Logo](/img/cni-logo.png) + +有关更多信æ¯ï¼Œè¯·è®¿é—® [CNI GitHub 项目](https://github.com/containernetworking/cni)。 + +## CNI 使用了哪些网络模型？ + +CNI 网络æ’件使用å°è£…网络模型（例如 Virtual Extensible Lan，缩写是 [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)）或éžå°è£…网络模型（例如 Border Gateway Protocol，缩写是 [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol)）æ¥å®žçŽ°ç½‘络结构。 + +### 什么是å°è£…网络？ + +此网络模型æ供了一个逻辑二层（L2）网络，该网络å°è£…在跨 Kubernetes 集群节点的现有三层（L3）网络拓扑上。使用此模型，你å¯ä»¥ä¸ºå®¹å™¨æ供一个隔离的 L2 网络，而无需分å‘路由。å°è£…网络带æ¥äº†å°‘é‡çš„处ç†å¼€é”€ä»¥åŠç”±äºŽè¦†ç›–å°è£…ç”Ÿæˆ IP header 造æˆçš„ IP 包大å°å¢žåŠ ã€‚å°è£…ä¿¡æ¯ç”± Kubernetes worker 之间的 UDP 端å£åˆ†å‘，交æ¢å¦‚何访问 MAC 地å€çš„网络控制平é¢ä¿¡æ¯ã€‚此类网络模型中常用的å°è£…是 VXLANã€Internet å议安全性 (IPSec) å’Œ IP-in-IP。 + +简å•æ¥è¯´ï¼Œè¿™ç§ç½‘络模型在 Kubernetes worker 之间生æˆäº†ä¸€ç§æ‰©å±•ç½‘桥，其中连接了 pod。 + +如果你åå‘使用扩展 L2 网桥，则å¯ä»¥é€‰æ‹©æ­¤ç½‘络模型。此网络模型对 Kubernetes worker çš„ L3 网络延迟很æ•æ„Ÿã€‚如果数æ®ä¸­å¿ƒä½äºŽä¸åŒçš„地ç†ä½ç½®ï¼Œè¯·ç¡®ä¿å®ƒä»¬ä¹‹é—´çš„延迟较低，以é¿å…最终的网络分段。 + +使用这ç§ç½‘络模型的 CNI 网络æ’件包括 Flannelã€Canalã€Weave å’Œ Cilium。默认情况下，Calico ä¸ä¼šä½¿ç”¨æ­¤æ¨¡åž‹ï¼Œä½†ä½ å¯ä»¥å¯¹å…¶è¿›è¡Œé…置。 + +![å°è£…网络](/img/encapsulated-network.png) + +### 什么是éžå°è£…网络？ + +该网络模型æ供了一个 L3 网络，用于在容器之间路由数æ®åŒ…。此模型ä¸ä¼šç”Ÿæˆéš”离的 L2 网络，也ä¸ä¼šäº§ç”Ÿå¼€é”€ã€‚这些好处的代价是，Kubernetes worker 必须管ç†æ‰€éœ€çš„所有路由分å‘。该网络模型ä¸ä½¿ç”¨ IP header 进行å°è£…，而是使用 Kubernetes Worker 之间的网络åè®®æ¥åˆ†å‘路由信æ¯ä»¥å®žçŽ° Pod 连接，例如 [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol)。 + +简而言之，这ç§ç½‘络模型在 Kubernetes worker 之间生æˆäº†ä¸€ç§æ‰©å±•ç½‘络路由器，æ供了如何连接 Pod çš„ä¿¡æ¯ã€‚ + +如果你åå‘使用 L3 网络，则å¯ä»¥é€‰æ‹©æ­¤ç½‘络模型。此模型在æ“作系统级别为 Kubernetes Worker 动æ€æ›´æ–°è·¯ç”±ã€‚对延迟较ä¸æ•æ„Ÿã€‚ + +使用这ç§ç½‘络模型的 CNI 网络æ’件包括 Calico å’Œ Cilium。Cilium å¯ä»¥ä½¿ç”¨æ­¤æ¨¡åž‹è¿›è¡Œé…置，å³ä½¿è¿™ä¸æ˜¯é»˜è®¤æ¨¡å¼ã€‚ + +![éžå°è£…网络](/img/unencapsulated-network.png) + +## Rancher æ供哪些 CNI æ’件？ + +### RKE Kubernetes 集群 + +Rancher 开箱å³ç”¨åœ°ä¸º RKE Kubernetes 集群æ供了几个 CNI 网络æ’件，分别是 Canalã€Flannelã€Calico å’Œ Weave。 + +如果你使用 Rancher 创建新的 Kubernetes 集群，你å¯ä»¥é€‰æ‹©ä½ çš„ CNI 网络æ’件。 + +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal 是一个 CNI 网络æ’件，它很好地结åˆäº† Flannel å’Œ Calico 的优点。它让你轻æ¾åœ°å°† Calico å’Œ Flannel 网络部署为统一的网络解决方案，将 Calico 的网络策略执行与 Calico（未å°è£…）和 Flannel（å°è£…）丰富的网络连接选项结åˆèµ·æ¥ã€‚ + +Canal 是 Rancher 默认的 CNI 网络æ’件，并采用了 Flannel å’Œ VXLAN å°è£…。 + +Kubernetes Worker 需è¦æ‰“å¼€ UDP ç«¯å£ `8472` (VXLAN) å’Œ TCP ç«¯å£ `9099`（å¥åº·æ£€æŸ¥ï¼‰ã€‚如果使用 Wireguard，则需è¦æ‰“å¼€ UDP ç«¯å£ `51820` å’Œ `51821`。有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +![](/img/canal-diagram.png) + +有关详细信æ¯ï¼Œè¯·å‚阅 [Canal GitHub 页é¢](https://github.com/projectcalico/canal)。 + +#### Flannel + +![Flannel Logo](/img/flannel-logo.png) + +Flannel 是为 Kubernetes é…ç½® L3 网络结构的简å•æ–¹æ³•ã€‚Flannel 在æ¯å°ä¸»æœºä¸Šè¿è¡Œä¸€ä¸ªå为 flanneld 的二进制 Agent，该 Agent 负责从更大的预é…置地å€ç©ºé—´ä¸­ä¸ºæ¯å°ä¸»æœºåˆ†é…å­ç½‘租约。Flannel 通过 Kubernetes API 或直接使用 etcd æ¥å­˜å‚¨ç½‘络é…ç½®ã€åˆ†é…çš„å­ç½‘ã€ä»¥åŠå…¶ä»–辅助数æ®ï¼ˆä¾‹å¦‚主机的公共 IP）。数æ®åŒ…使用æŸç§åŽç«¯æœºåˆ¶æ¥è½¬å‘，默认å°è£…为 [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)。 + +默认情况下，å°è£…çš„æµé‡æ˜¯ä¸åŠ å¯†çš„。Flannel æ供了两ç§åŠ å¯†æ–¹æ¡ˆï¼š + +* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec)：使用 [strongSwan](https://www.strongswan.org/) 在 Kubernetes worker 之间建立加密的 IPSec 隧é“。它是加密的实验性åŽç«¯ã€‚ +* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard)：比 strongSwan 更快的替代方案。 + +Kubernetes Worker 需è¦æ‰“å¼€ UDP ç«¯å£ `8472` (VXLAN)。有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +![Flannel Diagram](/img/flannel-diagram.png) + +有关详细信æ¯ï¼Œè¯·å‚阅 [Flannel GitHub 页é¢](https://github.com/flannel-io/flannel)。 + +#### Weave + +![Weave Logo](/img/weave-logo.png) + +Weave 在云上的 Kubernetes 集群中å¯ç”¨ç½‘络和网络策略。此外，它还支æŒåŠ å¯†å¯¹ç­‰èŠ‚点之间的æµé‡ã€‚ + +Kubernetes worker 需è¦æ‰“å¼€ TCP ç«¯å£ `6783`（控制端å£ï¼‰ã€UDP ç«¯å£ `6783` å’Œ UDP ç«¯å£ `6784`（数æ®ç«¯å£ï¼‰ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +有关详细信æ¯ï¼Œè¯·å‚阅以下页é¢ï¼š + +- [Weave Net 官网](https://www.weave.works/) + +### RKE2 Kubernetes 集群 + +Rancher 开箱å³ç”¨åœ°ä¸º RKE2 Kubernetes 集群æ供了几个 CNI 网络æ’件，分别是 [Canal](#canal)（è§ä¸Šä¸€èŠ‚）ã€Calico å’Œ Cilium。 + +如果你使用 Rancher 创建新的 Kubernetes 集群，你å¯ä»¥é€‰æ‹©ä½ çš„ CNI 网络æ’件。 + +#### Calico + +![Calico Logo](/img/calico-logo.png) + +Calico 在云上的 Kubernetes 集群中å¯ç”¨ç½‘络和网络策略。默认情况下，Calico 使用纯净ã€æœªå°è£…çš„ IP 网络结构和策略引擎为 Kubernetes 工作负载æ供网络。工作负载能够使用 BGP 在云上和本地进行通信。 + +Calico 还æ供了一ç§æ— çŠ¶æ€çš„ IP-in-IP 或 VXLAN å°è£…模å¼ã€‚如果需è¦ï¼Œä½ å¯ä»¥ä½¿ç”¨å®ƒã€‚Calico 还支æŒç­–略隔离，让你使用高级 ingress å’Œ egress ç­–ç•¥ä¿æŠ¤å’Œç®¡ç† Kubernetes 工作负载。 + +如果使用 BGP，Kubernetes Worker 需è¦æ‰“å¼€ TCP ç«¯å£ `179`，如果使用 VXLAN å°è£…，则需è¦æ‰“å¼€ UDP ç«¯å£ `4789`。å¦å¤–，使用 Typha æ—¶éœ€è¦ TCP ç«¯å£ `5473`。有关详细信æ¯ï¼Œè¯·å‚阅[下游集群的端å£è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +:::note é‡è¦æ示： + +在 Rancher 2.6.3 中，Calico 探测到在安装 RKE2 æ—¶ Windows 节点会失败。请注æ„，此问题已在 v2.6.4 中解决。 + +- è¦è§£å†³æ­¤é—®é¢˜ï¼Œè¯·å…ˆå¯¼èˆªåˆ° `https:///v3/settings/windows-rke2-install-script`。 + +- 在那里，将当å‰è®¾ç½® `https://raw.githubusercontent.com/rancher/wins/v0.1.3/install.ps1` 更改为新设置 `https://raw.githubusercontent .com/rancher/rke2/master/windows/rke2-install.ps1`。 + +::: + +![Calico Diagram](/img/calico-diagram.svg) + +有关详细信æ¯ï¼Œè¯·å‚阅以下页é¢ï¼š + +- [Project Calico 官方网站](https://www.projectcalico.org/) +- [Calico 项目 GitHub 页é¢](https://github.com/projectcalico/calico) + +#### Cilium + +![Cilium Logo](/img/cilium-logo.png) + +Cilium 在 Kubernetes 中å¯ç”¨ç½‘络和网络策略（L3ã€L4 å’Œ L7）。默认情况下，Cilium 使用 eBPF 技术在节点内部路由数æ®åŒ…，并使用 VXLAN 将数æ®åŒ…å‘é€åˆ°å…¶ä»–节点。你也å¯ä»¥é…ç½®éžå°è£…的技术。 + +Cilium 推è大于 5.2 的内核版本，从而充分利用 eBPF 的能力。Kubernetes worker 需è¦æ‰“å¼€ TCP ç«¯å£ `8472`（VXLAN）和 TCP ç«¯å£ `4240`（å¥åº·æ£€æŸ¥ï¼‰ã€‚此外，还必须为å¥åº·æ£€æŸ¥å¯ç”¨ ICMP 8/0。有关详细信æ¯ï¼Œè¯·æŸ¥çœ‹ [Cilium 系统è¦æ±‚](https://docs.cilium.io/en/latest/operations/system_requirements/#firewall-requirements)。 + +##### Cilium 中跨节点的 Ingress 路由 +
+默认情况下，Cilium ä¸å…许 Pod 与其他节点上的 Pod 通信。è¦è§£å†³æ­¤é—®é¢˜ï¼Œè¯·å¯ç”¨ Ingress Controller 以使用 “CiliumNetworkPolicy†进行跨节点路由请求。 + +选择 Cilium CNI 并为新集群å¯ç”¨é¡¹ç›®ç½‘络隔离åŽï¼Œé…置如下： + +``` +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: hn-nodes + namespace: default +spec: + endpointSelector: {} + ingress: + - fromEntities: + - remote-node +``` + +## å„个网络æ’件的 CNI 功能 + +下表总结了 Rancher 中æ¯ä¸ª CNI 网络æ’件支æŒçš„ä¸åŒåŠŸèƒ½ï¼š + +| æ供商 | 网络模型 | è·¯çº¿åˆ†å‘ | 网络策略 | 网格 | 外部数æ®å­˜å‚¨ | 加密 | Ingress/Egress ç­–ç•¥ | +| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Canal | å°è£… (VXLAN) | å¦ | 是 | å¦ | K8s API | 是 | 是 | +| Flannel | å°è£… (VXLAN) | å¦ | å¦ | å¦ | K8s API | 是 | å¦ | +| Calico | å°è£…（VXLAN，IPIP）或未å°è£… | 是 | 是 | 是 | Etcd å’Œ K8s API | 是 | 是 | +| Weave | å°è£… | 是 | 是 | 是 | å¦ | 是 | 是 | +| Cilium | å°è£… (VXLAN) | 是 | 是 | 是 | Etcd å’Œ K8s API | 是 | 是 | + +- 网络模型：å°è£…或未å°è£…。如需更多信æ¯ï¼Œè¯·å‚阅 [CNI 中使用的网络模型](#cni-使用了哪些网络模型)。 + +- 路由分å‘：一ç§å¤–部网关å议，用于在互è”网上交æ¢è·¯ç”±å’Œå¯è¾¾æ€§ä¿¡æ¯ã€‚BGP å¯ä»¥å¸®åŠ©è¿›è¡Œè·¨é›†ç¾¤ pod 之间的网络。此功能对于未å°è£…çš„ CNI 网络æ’件是必须的，并且通常由 BGP 完æˆã€‚如果你想构建跨网段拆分的集群，路由分å‘是一个很好的功能。 + +- 网络策略：Kubernetes æ供了强制执行规则的功能，这些规则决定了哪些 service å¯ä»¥ä½¿ç”¨ç½‘络策略进行相互通信。这是从 Kubernetes 1.7 起稳定的功能，å¯ä»¥ä¸ŽæŸäº›ç½‘络æ’件一起使用。 + +- 网格：å…许在ä¸åŒçš„ Kubernetes 集群间进行 service 之间的网络通信。 + +- 外部数æ®å­˜å‚¨ï¼šå…·æœ‰æ­¤åŠŸèƒ½çš„ CNI 网络æ’件需è¦ä¸€ä¸ªå¤–部数æ®å­˜å‚¨æ¥å­˜å‚¨æ•°æ®ã€‚ + +- 加密：å…许加密和安全的网络控制和数æ®å¹³é¢ã€‚ + +- Ingress/Egress 策略：å…è®¸ä½ ç®¡ç† Kubernetes å’Œéž Kubernetes 通信的路由控制。 + + +## CNI 社区人气 + +下表总结了ä¸åŒçš„ GitHub 指标，让你了解æ¯ä¸ªé¡¹ç›®çš„å—欢迎程度和活动。数æ®æ”¶é›†äºŽ 2022 å¹´ 1 月。 + +| æ供商 | 项目 | Stars | Forks | Contributors | +| ---- | ---- | ---- | ---- | ---- | +| Canal | https://github.com/projectcalico/canal | 679 | 100 | 21 | +| Flannel | https://github.com/flannel-io/flannel | 7k | 2.5k | 185 | +| Calico | https://github.com/projectcalico/calico | 3.1k | 741 | 224 | +| Weave | https://github.com/weaveworks/weave/ | 6.2k | 635 | 84 | +| Cilium | https://github.com/cilium/cilium | 10.6k | 1.3k | 352 | + +
+ +## 使用哪个 CNI æ’件？ + +è¿™å–决于你的项目需求。å„个æ供商都有ä¸åŒçš„功能和选项。没有一个æ供商å¯ä»¥æ»¡è¶³æ‰€æœ‰ç”¨æˆ·çš„需求。 + +Canal 是默认的 CNI 网络æ’件。对于大多数用例，我们推è你使用它。它使用 Flannel 为容器æä¾›å°è£…网络，åŒæ—¶æ·»åŠ  Calico 网络策略，å¯ä»¥åœ¨ç½‘络方é¢æ供项目/命å空间隔离。 + +## 如何é…ç½® CNI 网络æ’件？ + +如需了解如何为你的集群é…置网络æ’件，请å‚阅[集群选项](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。有关更高级的é…置选项，请å‚阅有关使用[é…置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)å’Œ[网络æ’件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/)选项æ¥é…置集群的说明。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/deprecated-features-in-v2.5.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/deprecated-features-in-v2.5.md new file mode 100644 index 00000000000..ec1898663b2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/deprecated-features-in-v2.5.md @@ -0,0 +1,26 @@ +--- +title: Rancher 弃用的功能 +--- + +### Rancher 的弃用策略是什么？ + +我们在支æŒ[æœåŠ¡æ¡æ¬¾](https://rancher.com/support-maintenance-terms)中å‘布了官方弃用策略。 + +### 在哪里å¯ä»¥æ‰¾åˆ° Rancher 已弃用的功能？ + +Rancher 会在 GitHub 上的[å‘行说明](https://github.com/rancher/rancher/releases)中公布已弃用的功能。请å‚阅以下补ä¸ç‰ˆæœ¬äº†è§£å·²å¼ƒç”¨çš„功能： + +| è¡¥ä¸ç‰ˆæœ¬ | å‘布日期 | +|---------------|---------------| +| [2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) | 2021 å¹´ 8 月 31 æ—¥ | +| [2.6.1](https://github.com/rancher/rancher/releases/tag/v2.6.1) | 2021 å¹´ 10 月 11 æ—¥ | +| [2.6.2](https://github.com/rancher/rancher/releases/tag/v2.6.2) | 2021 å¹´ 10 月 19 æ—¥ | +| [2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) | 2021 å¹´ 12 月 21 æ—¥ | +| [2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | 2022 å¹´ 3 月 31 æ—¥ | +| [2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) | 2022 å¹´ 5 月 12 æ—¥ | +| [2.6.6](https://github.com/rancher/rancher/releases/tag/v2.6.6) | 2022 å¹´ 6 月 30 æ—¥ | + + +### 如果æŸä¸ªåŠŸèƒ½æ ‡è®°ä¸ºå¼ƒç”¨ï¼Œæˆ‘è¦æ€Žä¹ˆåšï¼Ÿ + +如果æŸä¸ªå‘行版将æŸåŠŸèƒ½æ ‡è®°ä¸º"Deprecated"（已弃用），该功能ä»ç„¶å¯ç”¨å¹¶å—支æŒï¼Œä»Žè€Œå…许用户按照常规æµç¨‹è¿›è¡Œå‡çº§ã€‚在å‡çº§åˆ°è¯¥åŠŸèƒ½è¢«æ ‡è®°ä¸º"已删除"çš„å‘行版å‰ï¼Œç”¨æˆ·/管ç†å‘˜åº”该计划剥离该功能。对于新部署，我们建议ä¸è¦ä½¿ç”¨å·²å¼ƒç”¨çš„功能。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/dockershim.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/dockershim.md new file mode 100644 index 00000000000..cfab0dfbaf4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/dockershim.md @@ -0,0 +1,45 @@ +--- +title: Dockershim +--- + +Dockershim 是 Kubelet å’Œ Docker Daemon 之间的 CRI 兼容层。Kubernetes 1.20 版本宣布了[移除树内 Dockershim](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/)。目å‰è®¡åˆ’在 Kubernetes 1.24 中移除。有关此移除的更多信æ¯ä»¥åŠæ—¶é—´çº¿ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 + +从 Kubernetes 1.21 开始。RKE 集群支æŒå¤–部 Dockershim，æ¥è®©ç”¨æˆ·ç»§ç»­ä½¿ç”¨ Docker 作为 CRI è¿è¡Œæ—¶ã€‚现在，我们通过使用 [Mirantis å’Œ Docker ](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) æ¥ç¡®ä¿ RKE 集群å¯ä»¥ç»§ç»­ä½¿ç”¨ Docker，从而实现上游开æºç¤¾åŒºçš„ Dockershim。 + +è¦å¯ç”¨å¤–部 Dockershim，é…置以下选项： + +``` +enable_cri_dockerd: true +``` + +如果你想使用其他容器è¿è¡Œæ—¶ï¼ŒRancher 也æ供使用 Containerd 作为默认è¿è¡Œæ—¶çš„，以边缘为中心的 K3s，和以数æ®ä¸­å¿ƒä¸ºä¸­å¿ƒçš„ RKE2 Kubernetes å‘行版。å³ä½¿åœ¨ Kubernetes 1.24 删除了树内 Dockershim 之åŽï¼Œä½ ä¹Ÿå¯ä»¥é€šè¿‡ Rancher å‡çº§å’Œç®¡ç†å¯¼å…¥çš„ RKE2 å’Œ K3s Kubernetes 集群。 + +### 常è§é—®é¢˜ + +
+ +Q. 如果è¦èŽ·å¾— Rancher 对上游 Dockershim 的支æŒï¼Œæˆ‘需è¦å‡çº§ Rancher å—？ + +对于 RKE，Dockershim 的上游支æŒä»Ž Kubernetes 1.21 开始。你需è¦ä½¿ç”¨ Rancher 2.6 或更高版本æ‰èƒ½èŽ·å–使用 Kubernetes 1.21 çš„ RKE 的支æŒã€‚详情请å‚阅我们的[支æŒçŸ©é˜µ](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/)。 + +
+ +Q. 我目å‰çš„ RKE 使用 Kubernetes 1.20。为了é¿å…出现ä¸å†æ”¯æŒ Dockershim 的情况，我是å¦éœ€è¦å°½æ—©å°† RKE å‡çº§åˆ° Kubernetes 1.21？ + +A. 在使用 Kubernetes 1.20 çš„ RKE 中，Dockershim 版本ä¾ç„¶å¯ç”¨ï¼Œè€Œä¸”在 Kubernetes 1.24 之å‰ä¸ä¼šåœ¨ä¸Šæ¸¸å¼ƒç”¨ã€‚Kubernetes 会å‘出弃用 Dockershim 的警告，而 Rancher 在使用 Kubernetes 1.21 çš„ RKE 中已ç»ç¼“解了这个问题。你å¯ä»¥æŒ‰ç…§è®¡åˆ’正常å‡çº§åˆ° Kubernetes 1.21，但也应该考虑在å‡çº§åˆ° Kubernetes 1.22 æ—¶å¯ç”¨å¤–部 Dockershim。在å‡çº§åˆ° Kubernetes 1.24 之å‰ï¼Œä½ éœ€è¦å¯ç”¨å¤–部 Dockershim，此时现有的实现都会被删除。 + +有关此移除的更多信æ¯ä»¥åŠæ—¶é—´çº¿ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 + +
+ +Q: 如果我ä¸æƒ³å†ä¾èµ– Dockershim，我还有什么选择？ + +A: ä½ å¯ä»¥ä¸º Kubernetes 使用ä¸éœ€è¦ Dockershim 支æŒçš„è¿è¡Œæ—¶ï¼Œå¦‚ Containerd。RKE2 å’Œ K3s 就是其中的两个选项。 + +
+ +Q: 如果我目å‰ä½¿ç”¨ RKE1，但想切æ¢åˆ° RKE2，我å¯ä»¥æ€Žæ ·è¿›è¡Œè¿ç§»ï¼Ÿ + +A: Rancher 也在探索就地å‡çº§è·¯å¾„çš„å¯èƒ½æ€§ã€‚此外，你始终å¯ä»¥ä½¿ç”¨ kubectl 将工作负载è¿ç§»åˆ°å¦ä¸€ä¸ªé›†ç¾¤ã€‚ + +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/general-faq.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/general-faq.md new file mode 100644 index 00000000000..45d4ae835ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/general-faq.md @@ -0,0 +1,69 @@ +--- +title: 一般常è§é—®é¢˜è§£ç­” +--- + +本文包å«äº†ç”¨æˆ·å¸¸è§çš„ Rancher 2.x 问题。 + +有关常è§æŠ€æœ¯é—®é¢˜ï¼Œè¯·å‚阅[常è§æŠ€æœ¯é—®é¢˜è§£ç­”](technical-items.md)。 + +
+ +**Rancher 2.x æ”¯æŒ Docker Swarm å’Œ Mesos 作为环境类型å—？** + +如果你在 Rancher 2.x 中创建环境，Swarm å’Œ Mesos å°†ä¸å†æ˜¯å¯é€‰çš„标准选项。但是，Swarm å’Œ Mesos 还能继续作为å¯ä»¥éƒ¨ç½²çš„商店应用程åºã€‚这是一个艰难的决定，但这是大势所趋。比如说，15,000 多个集群å¯èƒ½åªæœ‰å¤§çº¦ 200 个在è¿è¡Œ Swarm。 + +
+ +**是å¦å¯ä»¥ä½¿ç”¨ Rancher 2.x ç®¡ç† Azure Kubernetes æœåŠ¡ï¼Ÿ** + +是的。 + +
+ +**Rancher 是å¦æ”¯æŒ Windows？** + +Rancher æ”¯æŒ Windows Server 1809 容器。有关如何使用 Windows Worker 节点设置集群的详细信æ¯ï¼Œè¯·å‚阅[为 Windows é…置自定义集群](../pages-for-subheaders/use-windows-clusters.md)。 + +
+ +**Rancher 是å¦æ”¯æŒ Istio？** + +Rancher æ”¯æŒ [Istio](../pages-for-subheaders/istio.md)。 + +此外，Istio 是在我们的微型 PaaS “Rio†中实现的，它å¯ä»¥è¿è¡Œåœ¨ Rancher 2.x 以åŠä»»ä½•ç¬¦åˆ CNCF çš„ Kubernetes 集群上。详情请å‚阅[这里](https://rio.io/)。 + +
+ +**Rancher 2.x 是å¦æ”¯æŒä½¿ç”¨ Hashicorp çš„ Vault æ¥å­˜å‚¨å¯†æ–‡ï¼Ÿ** + +密文管ç†å·²åœ¨æˆ‘们的 roadmap 上，但我们尚未将该功能分é…给特定版本。 + +
+ +**Rancher 2.x 是å¦ä¹Ÿæ”¯æŒ RKT 容器？** + +ç›®å‰ï¼Œæˆ‘们åªæ”¯æŒ Docker。 + +
+ +**Rancher 2.x 是å¦æ”¯æŒå°† Calicoã€Contivã€Contrailã€Flannelã€Weave net 等网络æ’件用于嵌入和已注册的 Kubernetes？** + +Rancher 开箱å³ç”¨åœ°ä¸º Kubernetes 集群æ供了几个 CNI 网络æ’件，分别是 Canalã€Flannelã€Calico å’Œ Weave。有关官方支æŒçš„详细信æ¯ï¼Œè¯·å‚阅 [Rancher 支æŒçŸ©é˜µ](https://rancher.com/support-maintenance-terms/)。 + +
+ +**Rancher 是å¦è®¡åˆ’æ”¯æŒ Traefik？** + +ç›®å‰ï¼Œæˆ‘们ä¸æ‰“ç®—æä¾›åµŒå…¥å¼ Traefik 支æŒï¼Œä½†æˆ‘们ä»åœ¨æŽ¢ç´¢è´Ÿè½½å‡è¡¡æ–¹æ¡ˆã€‚ + +
+ +**我å¯ä»¥å°† OpenShift Kubernetes 集群导入 2.x å—？** + +我们的目标是è¿è¡Œä»»ä½•ä¸Šæ¸¸ Kubernetes 集群。因此，Rancher 2.x 应该å¯ä»¥ä¸Ž OpenShift 一起使用，但我们尚未对此进行测试。 + +
+ +**Rancher ä¼šé›†æˆ Longhorn å—？** + +是的。Longhorn 已集æˆåˆ° Rancher 2.5+ 中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/install-and-configure-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/install-and-configure-kubectl.md new file mode 100644 index 00000000000..21c301639b6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/install-and-configure-kubectl.md @@ -0,0 +1,29 @@ +--- +title: 安装和é…ç½® kubectl +--- + +`kubectl` 是一个 CLI 工具，用于è¿è¡Œ Kubernetes 集群相关的命令。Rancher 2.x 中的许多维护和管ç†ä»»åŠ¡éƒ½éœ€è¦å®ƒã€‚ + +### 安装 + +请å‚阅 [kubectl 安装](https://kubernetes.io/docs/tasks/tools/install-kubectl/)å°† kubectl 安装到你的æ“作系统上。 + +### é…ç½® + +使用 RKE 创建 Kubernetes 集群时，RKE 会在本地目录中创建一个 `kube_config_cluster.yml`，该文件包å«ä½¿ç”¨ `kubectl` 或 `helm` 等工具连接到新集群的凭è¯ã€‚ + +ä½ å¯ä»¥å°†æ­¤æ–‡ä»¶å¤åˆ¶ä¸º `$HOME/.kube/config`。如果你使用多个 Kubernetes 集群，将 `KUBECONFIG` 环境å˜é‡è®¾ç½®ä¸º `kube_config_cluster.yml` 的路径： + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +使用 `kubectl` 测试你的连接性，并查看你是å¦å¯ä»¥èŽ·å–节点列表： + +``` +kubectl get nodes + NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.10.1 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/rancher-is-no-longer-needed.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/rancher-is-no-longer-needed.md new file mode 100644 index 00000000000..ffb98927c69 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/rancher-is-no-longer-needed.md @@ -0,0 +1,61 @@ +--- +title: å¸è½½ Rancher +--- + +本文介ç»äº†å¦‚果你ä¸å†éœ€è¦ Rancherã€ä¸æƒ³å†ç”± Rancher 管ç†é›†ç¾¤ã€æˆ–想删除 Rancher Server 需è¦æ€Žä¹ˆåšã€‚ + + +### 如果 Rancher Server 被删除，下游集群中的工作负载会怎样？ + +如果 Rancher 删除了或无法æ¢å¤ï¼ŒRancher 管ç†çš„下游 Kubernetes 集群中的所有工作负载将继续正常è¿è¡Œã€‚ + +### 如果删除了 Rancher Server，该如何访问下游集群？ + +如果删除了 Rancher，访问下游集群的方å¼å–决于集群的类型和集群的创建方å¼ã€‚总而言之： + +- **注册集群**：集群ä¸å—å½±å“，你å¯ä»¥æ³¨å†Œé›†ç¾¤å‰çš„方法访问该集群。 +- **托管的 Kubernetes 集群**：如果你在 Kubernetes 云æ供商（例如 EKSã€GKE 或 AKS）中创建集群，你å¯ä»¥ç»§ç»­ä½¿ç”¨æ供商的云凭è¯æ¥ç®¡ç†é›†ç¾¤ã€‚ +- **RKE 集群**：è¦è®¿é—® [RKE 集群](../pages-for-subheaders/launch-kubernetes-with-rancher.md)，集群必须å¯ç”¨äº†[授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆauthorized cluster endpoint，ACE）](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)，而且你必须从 Rancher UI 下载了集群的 kubeconfig 文件。RKE 集群默认å¯ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹ã€‚通过使用此端点，你å¯ä»¥ç›´æŽ¥ä½¿ç”¨ kubectl 访问你的集群，而ä¸ç”¨é€šè¿‡ Rancher Server çš„[认è¯ä»£ç†](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-认è¯ä»£ç†)进行通信。有关é…ç½® kubectl 以使用授æƒé›†ç¾¤ç«¯ç‚¹çš„说明，请å‚阅[使用 kubectl å’Œ kubeconfig 文件直接访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)。这些集群将使用删除 Rancher æ—¶é…置的身份验è¯å¿«ç…§ã€‚ + +### 如果我ä¸æƒ³å†ä½¿ç”¨ Rancher 了该怎么åšï¼Ÿ + +:::note + +之å‰æŽ¨èçš„ [System Tools](../reference-guides/system-tools.md) 自 2022 å¹´ 6 月起已弃用。 + +::: + +如果你[在 Kubernetes 集群上安装了 Rancher](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)，你å¯ä»¥ä½¿ç”¨ [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) 工具删除 Rancher。 + +在高å¯ç”¨ (HA) 模å¼ä¸‹å¸è½½ Rancher 还将删除所有 `helm-operation-*` Pod 和以下应用程åºï¼š + +- fleet +- fleet-agent +- rancher-operator +- rancher-webhook + +è‡ªå®šä¹‰èµ„æº (CRD) 和自定义命å空间ä»éœ€è¦æ‰‹åŠ¨åˆ é™¤ã€‚ + +如果你在 Docker 中安装 Rancher，则å¯ä»¥é€šè¿‡åˆ é™¤è¿è¡Œ Rancher çš„å•ä¸ª Docker 容器æ¥å¸è½½ Rancher。 + +移除 Rancher ä¸ä¼šå½±å“导入的集群。有关其他集群类型，请å‚考[移除 Rancher åŽè®¿é—®ä¸‹æ¸¸é›†ç¾¤](#如果删除了-rancher-server该如何访问下游集群)。 + +### 如果我ä¸æƒ³ Rancher 管ç†æˆ‘的注册集群该怎么办？ + +如果你在 Rancher UI 中删除了已注册的集群，则该集群将与 Rancher 分离，集群ä¸ä¼šå‘生改å˜ï¼Œä½ å¯ä»¥ä½¿ç”¨æ³¨å†Œé›†ç¾¤ä¹‹å‰çš„方法访问该集群。 + +è¦åˆ†ç¦»é›†ç¾¤ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +2. 转到è¦ä¸Ž Rancher 分离的已注册集群，然åŽå•å‡» **â‹® > 删除**。 +3. å•å‡»**删除**。 + +**结果**：注册的集群已与 Rancher 分离，并在 Rancher 外正常è¿è¡Œã€‚ + +### 如果我ä¸æƒ³ Rancher 管ç†æˆ‘çš„ RKE 集群或托管的 Kubernetes 集群该怎么办？ + +ç›®å‰ï¼Œæˆ‘们没有将这些集群从 Rancher 中分离出æ¥çš„功能。在这ç§æƒ…况下，“分离â€æŒ‡çš„是将 Rancher 组件移除出集群，并独立于 Rancher 管ç†å¯¹é›†ç¾¤çš„访问。 + +[æ­¤ issue](https://github.com/rancher/rancher/issues/25234) 跟踪了在没有 Rancher 的情况下管ç†è¿™äº›é›†ç¾¤çš„功能。 + +有关如何在删除 Rancher Server åŽè®¿é—®é›†ç¾¤çš„更多信æ¯ï¼Œè¯·å‚阅[本节](#如果删除了-rancher-server该如何访问下游集群)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/security.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/security.md new file mode 100644 index 00000000000..0078c58eac7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/security.md @@ -0,0 +1,14 @@ +--- +title: 安全 + +--- + +**是å¦æœ‰å¼ºåŒ–指å—？** + +强化指å—现在ä½äºŽ[安全](../pages-for-subheaders/rancher-security.md)部分。 + +
+ +**Rancher Kubernetes 集群 CIS Benchmark 测试的结果是什么？** + +我们已ç»é’ˆå¯¹å¼ºåŒ–çš„ Rancher Kubernetes 集群è¿è¡Œäº† CIS Kubernetes Benchmark 测试。你å¯ä»¥åœ¨[安全](../pages-for-subheaders/rancher-security.md)中找到该评估的结果。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/technical-items.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/technical-items.md new file mode 100644 index 00000000000..2bc3cfb6bfc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/technical-items.md @@ -0,0 +1,176 @@ +--- +title: 技术 +--- + +### 如何é‡ç½®ç®¡ç†å‘˜å¯†ç ï¼Ÿ + +Docker 安装： +``` +$ docker exec -ti reset-password +New password for default administrator (user-xxxxx): + +``` + +Kubernetes 安装（Helm）： +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher --no-headers | head -1 | awk '{ print $1 }') -c rancher -- reset-password +New password for default administrator (user-xxxxx): + +``` + + + +### 我删除/åœç”¨äº†æœ€åŽä¸€ä¸ª admin，该如何解决？ +Docker 安装： +``` +$ docker exec -ti ensure-default-admin +New default administrator (user-xxxxx) +New password for default administrator (user-xxxxx): + +``` + +Kubernetes 安装（Helm）： +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher | grep '1/1' | head -1 | awk '{ print $1 }') -- ensure-default-admin +New password for default administrator (user-xxxxx): + +``` +### 如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录？ + +请å‚阅[故障排除：日志记录](../troubleshooting/other-troubleshooting-tips/logging.md)。 + +### 我的 ClusterIP ä¸å“应 ping，该如何解决？ + +ClusterIP 是一个虚拟 IP，ä¸ä¼šå“应 ping。è¦æµ‹è¯• ClusterIP 是å¦é…置正确，最好的方法是使用 `curl` 访问 IP 和端å£å¹¶æ£€æŸ¥å®ƒæ˜¯å¦å“应。 + +### 在哪里管ç†èŠ‚点模æ¿ï¼Ÿ + +打开你的账å·èœå•ï¼ˆå³ä¸Šè§’）并选择`节点模æ¿`。 + +### 为什么我的四层负载å‡è¡¡å™¨å¤„于 `Pending` 状æ€ï¼Ÿ + +四层负载å‡è¡¡å™¨åˆ›å»ºä¸º `type: LoadBalancer`。Kubernetes 需è¦ä¸€ä¸ªå¯ä»¥æ»¡è¶³è¿™äº›è¯·æ±‚的云æ供商或控制器，å¦åˆ™è¿™äº›è¯·æ±‚将永远处于 `Pending` 状æ€ã€‚有关更多信æ¯ï¼Œè¯·å‚阅[云æ供商](../pages-for-subheaders/set-up-cloud-providers.md)或[创建外部负载å‡è¡¡å™¨](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/)。 + +### Rancher 的状æ€å­˜å‚¨åœ¨å“ªé‡Œï¼Ÿ + +- Docker 安装：在 `rancher/rancher` å®¹å™¨çš„åµŒå…¥å¼ etcd 中，ä½äºŽ `/var/lib/rancher`。 +- Kubernetes install：在为è¿è¡Œ Rancher 而创建的 RKE 集群的 etcd 中。 + +### 支æŒçš„ Docker 版本是如何确定的？ + +我们éµå¾ªä¸Šæ¸¸ Kubernetes 版本验è¯è¿‡çš„ Docker 版本。如果需è¦èŽ·å–验è¯è¿‡çš„版本，请查看 Kubernetes 版本 CHANGELOG.md 中的 [External Dependencies](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.10.md#external-dependencies)。 + +### 如何访问 Rancher 创建的节点？ + +ä½ å¯ä»¥è½¬åˆ°**节点**视图，然åŽä¸‹è½½ç”¨äºŽè®¿é—® Rancher 创建的节点的 SSH 密钥。选择è¦è®¿é—®çš„节点并å•å‡»è¡Œå°¾ **â‹®** 按钮，然åŽé€‰æ‹©**下载密钥**，如下图所示。 + +![下载密钥](/img/downloadsshkeys.png) + +解压缩下载的 zip 文件，并使用 `id_rsa` 文件连接到你的主机。请务必使用正确的用户å（如果是 RancherOS，则使用 `rancher` 或 `docker`；如果是 Ubuntu，则使用 `ubuntu`；如果是 Amazon Linux，则使用 `ec2-user`）。 + +``` +$ ssh -i id_rsa user@ip_of_node +``` + +### 如何在 Rancher 中自动化任务 X？ + +UI ç”±é™æ€æ–‡ä»¶ç»„æˆï¼Œå¹¶æ ¹æ® API çš„å“应工作。æ¢è¨€ä¹‹ï¼ŒUI 中å¯ä»¥æ‰§è¡Œçš„æ¯ä¸ªæ“作/任务都å¯ä»¥é€šè¿‡ API 进行自动化。有两ç§æ–¹æ³•å¯ä»¥å®žçŽ°è¿™ä¸€ç‚¹ï¼š + +* 访问 `https://your_rancher_ip/v3` 并æµè§ˆ API 选项。 +* 在使用 UI æ—¶æ•èŽ· API 调用（通常使用 [Chrome å¼€å‘者工具](https://developers.google.com/web/tools/chrome-devtools/#network)，但你也å¯ä»¥ä½¿ç”¨å…¶ä»–工具）。 + +### 节点的 IP 地å€æ”¹å˜äº†ï¼Œè¯¥å¦‚何æ¢å¤ï¼Ÿ + +节点需è¦é…ç½®é™æ€ IP（或使用 DHCP ä¿ç•™çš„ IP）。如果节点的 IP 已更改，你必须在集群中删除并é‡æ–°æ·»åŠ å®ƒã€‚删除åŽï¼ŒRancher 会将集群更新为正确的状æ€ã€‚如果集群ä¸å†å¤„于 `Provisioning` 状æ€ï¼Œåˆ™å·²ä»Žé›†ç¾¤åˆ é™¤è¯¥èŠ‚点。 + +节点的 IP 地å€å‘生å˜åŒ–时，Rancher 会失去与节点的连接，因此无法正常清ç†èŠ‚点。请å‚阅[清ç†é›†ç¾¤èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md)æ¥æ¸…ç†èŠ‚点。 + +在集群中移除并清ç†èŠ‚点时，你å¯ä»¥å°†èŠ‚点é‡æ–°æ·»åŠ åˆ°é›†ç¾¤ä¸­ã€‚ + +### 如何将其他å‚æ•°/绑定/环境å˜é‡æ·»åŠ åˆ° Rancher å¯åŠ¨çš„ Kubernetes 集群的 Kubernetes 组件中？ + +ä½ å¯ä»¥ä½¿ç”¨é›†ç¾¤é€‰é¡¹ä¸­çš„[é…置文件](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)选项æ¥æ·»åŠ å…¶ä»–å‚æ•°/​​绑定/环境å˜é‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 RKE 文档中的[其他å‚æ•°ã€ç»‘定和环境å˜é‡](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/)，或æµè§ˆ [Cluster.ymls 示例](https://rancher.com/docs/rke/latest/en/example-yamls/)。 + +### 如何检查è¯ä¹¦é“¾æ˜¯å¦æœ‰æ•ˆï¼Ÿ + +使用 `openssl verify` 命令æ¥éªŒè¯ä½ çš„è¯ä¹¦é“¾ï¼š + +:::tip + +å°† `SSL_CERT_DIR` å’Œ `SSL_CERT_FILE` é…置到虚拟ä½ç½®ï¼Œä»Žè€Œç¡®ä¿åœ¨æ‰‹åŠ¨éªŒè¯æ—¶ä¸ä½¿ç”¨æ“作系统安装的è¯ä¹¦ã€‚ + +::: + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +如果你看到 `unable to get local issuer certificate` 错误，则表示链ä¸å®Œæ•´ã€‚通常情况下，这表示你的æœåŠ¡å™¨è¯ä¹¦ç”±ä¸­é—´ CA é¢å‘。如果你已ç»æ‹¥æœ‰æ­¤è¯ä¹¦ï¼Œä½ å¯ä»¥åœ¨è¯ä¹¦çš„验è¯ä¸­ä½¿ç”¨å®ƒï¼Œå¦‚下所示： + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem -untrusted intermediate.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +如果你已æˆåŠŸéªŒè¯è¯ä¹¦é“¾ï¼Œä½ éœ€è¦åœ¨æœåŠ¡å™¨è¯ä¹¦ä¸­åŒ…å«æ‰€éœ€çš„中间 CA è¯ä¹¦ï¼Œä»Žè€Œå®Œæˆä¸Ž Rancher 连接的è¯ä¹¦é“¾ï¼ˆä¾‹å¦‚，使用 Rancher Agent）。æœåŠ¡å™¨è¯ä¹¦æ–‡ä»¶ä¸­è¯ä¹¦çš„顺åºé¦–先是æœåŠ¡å™¨è¯ä¹¦æœ¬èº«ï¼ˆ`rancher.yourdomain.com.pem` 的内容），然åŽæ˜¯ä¸­é—´ CA è¯ä¹¦ï¼ˆ`intermediate.pem` 的内容）： + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +如果在验è¯è¿‡ç¨‹ä¸­ä»ç„¶å‡ºçŽ°é”™è¯¯ï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œæ£€ç´¢æœåŠ¡å™¨è¯ä¹¦çš„主题和é¢å‘者： + +``` +openssl x509 -noout -subject -issuer -in rancher.yourdomain.com.pem +subject= /C=GB/ST=England/O=Alice Ltd/CN=rancher.yourdomain.com +issuer= /C=GB/ST=England/O=Alice Ltd/CN=Alice Intermediate CA +``` + +### 如何在æœåŠ¡å™¨è¯ä¹¦ä¸­æ£€æŸ¥ `Common Name` å’Œ `Subject Alternative Names`？ + +è™½ç„¶æŠ€æœ¯ä¸Šä»…éœ€è¦ `Subject Alternative Names` 中有一个æ¡ç›®ï¼Œä½†åœ¨ `Common Name` å’Œ `Subject Alternative Names` 中都包å«ä¸»æœºåå¯ä»¥æœ€å¤§ç¨‹åº¦åœ°æ高与旧版æµè§ˆå™¨/应用程åºçš„兼容性。 + +检查 `Common Name`： + +``` +openssl x509 -noout -subject -in cert.pem +subject= /CN=rancher.my.org +``` + +检查 `Subject Alternative Names`： + +``` +openssl x509 -noout -in cert.pem -text | grep DNS + DNS:rancher.my.org +``` + +### 为什么节点å‘生故障时é‡æ–°è°ƒåº¦ä¸€ä¸ª pod éœ€è¦ 5 分钟以上的时间？ + +这是以下默认 Kubernetes 设置的组åˆå¯¼è‡´çš„： + +* kubelet + * `node-status-update-frequency`：指定 kubelet 将节点状æ€å‘布到 master 的频率（默认 10s）。 +* kube-controller-manager + * `node-monitor-period`：在 NodeController 中åŒæ­¥ NodeStatus 的周期（默认 5s）。 + * `node-monitor-grace-period`：在将节点标记为ä¸å¥åº·ä¹‹å‰ï¼Œå…许节点无å“应的时间长度（默认 40s）。 + * `pod-eviction-timeout`：在故障节点上删除 pod 的宽é™æœŸï¼ˆé»˜è®¤ 5m0s）。 + +有关这些设置的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes：kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) å’Œ [Kubernetes：kube-controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/)。 + +Kubernetes 1.13 默认å¯ç”¨ `TaintBasedEvictions` 功能。有关详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes：基于污点的驱é€](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions)。 + +* kube-apiserver（Kubernetes 1.13 åŠæ›´é«˜ç‰ˆæœ¬ï¼‰ + * `default-not-ready-toleration-seconds`：表示 `notReady:NoExecute` 的容å¿åº¦çš„ `tolerationSeconds`，该设置默认添加到还没有该容å¿åº¦çš„ pod。 + * `default-unreachable-toleration-seconds`：表示 `unreachable:NoExecute` 的容å¿åº¦çš„ `tolerationSeconds`，该设置默认添加到还没有该容å¿åº¦çš„ pod。 + +### 我å¯ä»¥åœ¨ UI 中使用键盘快æ·é”®å—？ + +是的，你å¯ä»¥ä½¿ç”¨é”®ç›˜å¿«æ·é”®è®¿é—® UI 的大部分内容。è¦æŸ¥çœ‹å¿«æ·æ–¹å¼çš„概览，请在 UI ä»»æ„ä½ç½®æŒ‰ `?`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/telemetry.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/telemetry.md new file mode 100644 index 00000000000..400f6e839ad --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/faq/telemetry.md @@ -0,0 +1,31 @@ +--- +title: é¥æµ‹ +--- + +### 什么是é¥æµ‹ï¼Ÿ + +é¥æµ‹ï¼ˆTelemetry）收集 Rancher 安装大å°ã€ä½¿ç”¨çš„组件版本以åŠä½¿ç”¨åŠŸèƒ½çš„汇总信æ¯ã€‚Rancher Labs 会使用此信æ¯æ¥æ”¹è¿›äº§å“，我们ä¸ä¼šä¸Žç¬¬ä¸‰æ–¹å…±äº«æ­¤ä¿¡æ¯ã€‚ + +### 收集什么信æ¯ï¼Ÿ + +我们ä¸ä¼šæ”¶é›†ä»»ä½•è¯†åˆ«ä¿¡æ¯ï¼ˆå¦‚用户åã€å¯†ç æˆ–用户资æºçš„å称或地å€ï¼‰ã€‚ + +收集的主è¦å†…容包括： + +- æ¯ä¸ªé›†ç¾¤çš„节点总数（最å°ã€å¹³å‡ã€æœ€å¤§ã€æ€»æ•°ï¼‰åŠå…¶å¤§å°ï¼ˆä¾‹å¦‚ CPU 核心数和 RAM）。 +- 集群ã€é¡¹ç›®ã€å‘½å空间和 Pod 等逻辑资æºçš„èšåˆè®¡æ•°ã€‚ +- 用于部署集群和节点的驱动程åºè®¡æ•°ï¼ˆä¾‹å¦‚ GKEã€EC2ã€å¯¼å…¥ä¸Žè‡ªå®šä¹‰ï¼‰ã€‚ +- 部署在节点上的 Kubernetes 组件ã€æ“作系统和 Docker 的版本。 +- 是å¦å¯ç”¨äº†æŸäº›å¯é€‰ç»„件（例如，使用了哪些身份验è¯æ供程åºï¼‰ã€‚ +- è¿è¡Œçš„ Rancher çš„é•œåƒå称和版本。 +- 此安装的唯一éšæœºæ ‡è¯†ç¬¦ã€‚ + +### 我å¯ä»¥çœ‹åˆ°å‘é€çš„ä¿¡æ¯å—？ + +如果å¯ç”¨äº†é¥æµ‹ï¼Œä½ å¯ä»¥è½¬åˆ° `https:///v1-telemetry` 查看当å‰æ•°æ®ã€‚ + +如果未å¯ç”¨é¥æµ‹ï¼Œåˆ™æ”¶é›†æ•°æ®çš„进程未è¿è¡Œï¼Œå› æ­¤æ²¡æœ‰å¯ä¾›æŸ¥çœ‹çš„内容。 + +### 如何打开或关闭它？ + +完æˆåˆå§‹è®¾ç½®åŽï¼Œç®¡ç†å‘˜å¯ä»¥è½¬åˆ° UI `全局`中的`设置`页é¢ï¼Œå•å‡»**编辑**，然åŽå°† `telemetry-opt` 更改为 `in` 或 `out`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md new file mode 100644 index 00000000000..dee4c863bcf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md @@ -0,0 +1,89 @@ +--- +title: 在离线环境中渲染 Helm æ¨¡æ¿ +--- + +:::note + +以下说明å‡è®¾ä½ å·²ç»æŒ‰ç…§[本页](upgrades.md)çš„ Kubernetes å‡çº§è¯´æ˜Žæ“作（包括先决æ¡ä»¶ï¼‰åˆ°æ­¥éª¤ 3：å‡çº§ Rancher。 + +::: + +### Rancher Helm 模æ¿é€‰é¡¹ + +使用安装 Rancher 时选择的选项æ¥æ¸²æŸ“ Rancher 模æ¿ã€‚å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚Rancher 需è¦é…置为使用ç§æœ‰é•œåƒä»“库，以便é…置所有 Rancher å¯åŠ¨çš„ Kubernetes 集群或 Rancher 工具。 + +æ ¹æ®ä½ åœ¨å®‰è£…过程中åšå‡ºçš„选择，完æˆä»¥ä¸‹æ­¥éª¤ä¹‹ä¸€ã€‚ + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 输出压缩包的版本å·ã€‚ | +| `` | 指å‘è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 | +| `` | ä½ çš„ç§æœ‰é•œåƒä»“库的 DNS å称。 | +| `` | 在 K8s 集群上è¿è¡Œçš„ cert-manager 版本。 | + + +### 选项 A：使用默认的自签åè¯ä¹¦ + +``` +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # prevent files for Helm hooks from being generated + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +### 选项 B：使用 Kubernetes 密文从文件中获å–è¯ä¹¦ + +```plain +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # prevent files for Helm hooks from being generated + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +如果你使用的是ç§æœ‰ CA ç­¾åçš„è¯ä¹¦ï¼Œè¯·åœ¨ `--set ingress.tls.source=secret` åŽåŠ ä¸Š `--set privateCA=true`： + +```plain +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # prevent files for Helm hooks from being generated + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +### åº”ç”¨å·²æ¸²æŸ“çš„æ¨¡æ¿ + +将渲染的 manifest 目录å¤åˆ¶åˆ°å¯ä»¥è®¿é—® Rancher Server 集群的系统中，并应用渲染的模æ¿ã€‚ + +使用 `kubectl` æ¥åº”用渲染的 manifest。 + +```plain +kubectl -n cattle-system apply -R -f ./rancher +``` + +## 验è¯å‡çº§ + +登录 Rancher 以确认å‡çº§æˆåŠŸã€‚ + +:::tip + +å‡çº§åŽå‡ºçŽ°ç½‘络问题？ + +请å‚è§[æ¢å¤é›†ç¾¤ç½‘络](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md)。 + +::: + +## 已知å‡çº§é—®é¢˜ + +ä½ å¯ä»¥åœ¨ [GitHub](https://github.com/rancher/rancher/releases) å‘å¸ƒè¯´æ˜Žä»¥åŠ [Rancher 论å›](https://forums.rancher.com/c/announcements/12)中找到æ¯ä¸ª Rancher 版本的已知问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md new file mode 100644 index 00000000000..22de9e7af35 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md @@ -0,0 +1,147 @@ +--- +title: 在 Azure Kubernetes Service 上安装 Rancher +--- + +本文介ç»äº†å¦‚何在微软的 Azure Kubernetes Service (AKS) 上安装 Rancher。 + +本指å—使用命令行工具æ¥é…置一个带有 Ingress çš„ AKS 集群。如果你更喜欢使用 Azure 门户æ¥é…置集群，请å‚è§[官方文档](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal)。 + +如果你已有一个 AKS Kubernetes 集群，请直接跳到[安装 Ingress](#5-安装-ingress) 的步骤，然åŽæŒ‰ç…§[此页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。 + +## 先决æ¡ä»¶ + +:::caution + +部署到 Microsoft Azure 会产生费用。 + +::: + +- [Microsoft Azure è´¦å·](https://azure.microsoft.com/en-us/free/)：用于创建部署 Rancher å’Œ Kubernetes 的资æºã€‚ +- [Microsoft Azure 订阅](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹å¦‚何创建 Microsoft Azure 订阅。 +- [Micsoroft Azure 租户](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant)：访问此链接并å‚考教程以创建 Microsoft Azure 租户。 +- 你的订阅有足够的é…é¢ï¼Œè‡³å°‘有 2 个 vCPU。有关 Rancher Server 资æºè¦æ±‚的详情，请å‚è§[此节](../../../pages-for-subheaders/installation-requirements.md#rke-和托管-kubernetes)。 +- 在 Azure 中用 Helm 安装 Rancher 时，请使用 L7 è´Ÿè½½å‡è¡¡å™¨æ¥é¿å…网络问题。详情请å‚è§ [Azure è´Ÿè½½å‡è¡¡å™¨é™åˆ¶](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations)。 + +## 1. 准备你的工作站 + +在工作站上安装以下命令行工具： + +- **az**，Azure CLI：如需获得帮助，请å‚è§[安装步骤](https://docs.microsoft.com/en-us/cli/azure/)。 +- **kubectl**：如需获得帮助，请å‚è§[安装步骤](https://kubernetes.io/docs/tasks/tools/#kubectl)。 +- **helm**：如需获å–帮助，请å‚è§[安装步骤](https://helm.sh/docs/intro/install/)。 + +## 2. 创建资æºç»„ + +安装 CLI åŽï¼Œä½ éœ€è¦ç”¨ä½ çš„ Azure 账户登录： + +``` +az login +``` + +创建一个 [资æºç»„](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) æ¥ä¿å­˜é›†ç¾¤çš„所有相关资æºã€‚使用一个适用于你实际情况的ä½ç½®ï¼š + +``` +az group create --name rancher-rg --location eastus +``` + +## 3. 创建 AKS 集群 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ›å»ºä¸€ä¸ª AKS 集群。选择适用于你实际情况的虚拟机大å°ã€‚如需获得å¯ç”¨çš„大å°å’Œé€‰é¡¹ï¼Œè¯·å‚è§[此处](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes)。在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +:::note + +如果你è¦ä»Žæ—§çš„ Kubernetes 版本更新到 Kubernetes v1.22 或更高版本，你还需è¦[æ›´æ–°](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx。 + +::: + +``` +az aks create \ + --resource-group rancher-rg \ + --name rancher-server \ + --kubernetes-version \ + --node-count 3 \ + --node-vm-size Standard_D2_v3 +``` + +集群部署需è¦ä¸€äº›æ—¶é—´æ‰èƒ½å®Œæˆã€‚ + +## 4. 获å–è®¿é—®å‡­è¯ + +集群部署完æˆåŽï¼ŒèŽ·å–访问凭è¯ã€‚ + +``` +az aks get-credentials --resource-group rancher-rg --name rancher-server +``` + +此命令把集群的凭è¯åˆå¹¶åˆ°çŽ°æœ‰çš„ kubeconfig 中，并å…许 `kubectl` 与集群交互。 + +## 5. 安装 Ingress + +集群需è¦ä¸€ä¸ª Ingress，以从集群外部访问 Rancherã€‚è¦ Ingress，你需è¦åˆ†é…一个公共 IP 地å€ã€‚请确ä¿ä½ æœ‰è¶³å¤Ÿçš„é…é¢ï¼Œå¦åˆ™å®ƒå°†æ— æ³•åˆ†é… IP 地å€ã€‚公共 IP 地å€çš„é™åˆ¶åœ¨æ¯ä¸ªè®¢é˜…的区域级别生效。 + +为确ä¿ä½ é€‰æ‹©äº†æ­£ç¡®çš„ Ingress-NGINX Helm Chart，首先在 [Kubernetes/ingress-nginx 支æŒè¡¨](https://github.com/kubernetes/ingress-nginx#supported-versions-table)中找到与你的 Kubernetes 版本兼容的 `Ingress-NGINX 版本`。 + +然åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ—出å¯ç”¨çš„ Helm Chart： + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +`helm search` 命令的输出包å«ä¸€ä¸ª `APP VERSION` 列。此列下的版本等åŒäºŽä½ ä¹‹å‰é€‰æ‹©çš„ `Ingress-NGINX 版本`。使用应用程åºç‰ˆæœ¬ï¼Œé€‰æ‹©ä¸€ä¸ª Chart 版本，该版本打包了与你的 Kubernetes 兼容的应用程åºã€‚例如，如果使用的是 Kubernetes v1.24，则å¯ä»¥é€‰æ‹© v4.6.0 Helm Chart，因为 Ingress-NGINX v1.7.0 与该 Chart 打包在一起，而 v1.7.0 与 Kubernetes v1.24 兼容。如有疑问，请选择最新的兼容版本。 + +了解你需è¦çš„ Helm chart `版本`åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚它安装一个带有 Kubernetes è´Ÿè½½å‡è¡¡å™¨æœåŠ¡çš„ `nginx-ingress-controller`： + +``` +helm search repo ingress-nginx -l +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ + --set controller.service.externalTrafficPolicy=Local \ + --version 4.6.0 \ + --create-namespace +``` + +## 6. 获å–è´Ÿè½½å‡è¡¡å™¨çš„ IP + +è¿è¡Œä»¥ä¸‹å‘½ä»¤èŽ·å–è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€ï¼š + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +返回的结果应与以下内容类似： + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.0.116.18 40.31.180.83 80:31229/TCP,443:31050/TCP + 67s +``` + +ä¿å­˜ `EXTERNAL-IP`。 + +## 7. 设置 DNS + +到 Rancher Server 的外部æµé‡éœ€è¦é‡å®šå‘到你创建的负载å‡è¡¡å™¨ã€‚ + +创建指å‘ä½ ä¿å­˜çš„ `EXTERNAL-IP` çš„ DNS。这个 DNS 会用作 Rancher Server çš„ URL。 + +设置 DNS 的有效方法有很多。如需获å–帮助，请å‚è§ [Azure DNS 文档中心](https://docs.microsoft.com/en-us/azure/dns/)。 + +## 8. 安装 Rancher Helm Chart + +按照[本页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。任何 Kubernetes å‘行版上安装的 Rancher çš„ Helm 说明都是一样的。 + +安装 Rancher 时，使用上一步获å–çš„ DNS å称作为 Rancher Server çš„ URL。它å¯ä»¥ä½œä¸º Helm 选项传递进æ¥ã€‚例如，如果 DNS å称是 `rancher.my.org`，你需è¦ä½¿ç”¨ `--set hostname=rancher.my.org` 选项æ¥è¿è¡Œ Helm 安装命令。 + +在此设置之上安装 Rancher 时，你还需è¦å°†ä»¥ä¸‹å€¼ä¼ é€’到 Rancher Helm 安装命令，以设置与 Rancher çš„ Ingress 资æºä¸€èµ·ä½¿ç”¨çš„ Ingress Controller çš„å称： + +``` +--set ingress.ingressClassName=nginx +``` + +请å‚阅[Helm 安装命令](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher)了解你的è¯ä¹¦é€‰é¡¹ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md new file mode 100644 index 00000000000..335e3fee425 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md @@ -0,0 +1,151 @@ +--- +title: 在 Amazon EKS 上安装 Rancher +--- + +本文介ç»äº†å¦‚何在 Amazon EKS 集群上安装 Rancher。你也å¯ä»¥[通过 AWS Marketplace 安装 Rancher](../../quick-start-guides/deploy-rancher-manager/aws-marketplace.md)。 + +如果你已ç»æœ‰ä¸€ä¸ª EKS Kubernetes 集群，请直接跳转到[安装 Ingress](#5-安装-ingress)这个步骤。然åŽæŒ‰ç…§[此处](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的步骤安装 Rancher Helm Chart。 + +## 为 Rancher Server 创建 EKS 集群 + +在本节中，你将使用命令行工具安装一个带有 Ingress çš„ EKS 集群。如果你想在 EKS 上使用 Rancher 时使用较少的资æºï¼Œè¯·ä½¿ç”¨æ­¤æ–¹æ³•ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +- 已有一个 AWS 账户。 +- 建议使用 IAM 用户而ä¸æ˜¯ AWS æ ¹è´¦æˆ·ã€‚ä½ å°†éœ€è¦ IAM 用户的访问密钥 (access key) 和密文秘钥 (secret key) æ¥é…ç½® AWS 命令行界é¢ã€‚ +- IAM 用户需è¦å…·å¤‡[eksctl 文档](https://eksctl.io/usage/minimum-iam-policies/)中æ述的最低 IAM 策略。 + +::: + +### 1. 准备你的工作站 + +在工作站上安装以下命令行工具： + +- **AWS CLI v2**：如需获å–帮助，请å‚è§[安装步骤](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)。 +- **eksctl**：如需获å–帮助，请å‚è§[安装步骤](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html)。 +- **kubectl**：如需获得帮助，请å‚è§[安装步骤](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html)。 +- **helm**：如需获å–帮助，请å‚è§[安装步骤](https://helm.sh/docs/intro/install/)。 + +### 2. é…ç½® AWS CLI + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œé…ç½® AWS CLI： + +``` +aws configure +``` + +输入以下å‚数： + +| 值 | æè¿° | +|-------|-------------| +| AWS Access Key ID | 具有 EKS æƒé™çš„ IAM 用户的访问密钥凭è¯ã€‚ | +| AWS Secret Access Key | 具有 EKS æƒé™çš„ IAM 用户的密文密钥凭è¯ã€‚ | +| Default region name | 集群节点所在的 [AWS 区域](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html#Concepts.RegionsAndAvailabilityZones.Regions)。 | +| Default output format | 输入 `json`。 | + +### 3. 创建 EKS 集群 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ›å»ºä¸€ä¸ª EKS 集群。使用适用于你的用例的 AWS 区域。在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +**注æ„**：如果你è¦ä»Žæ—§çš„ Kubernetes 版本更新到 Kubernetes v1.22 或更高版本，你还需è¦[æ›´æ–°](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx。 + +``` +eksctl create cluster \ + --name rancher-server \ + --version \ + --region us-west-2 \ + --nodegroup-name ranchernodes \ + --nodes 3 \ + --nodes-min 1 \ + --nodes-max 4 \ + --managed +``` + +使用 CloudFormation 进行的集群部署å¯èƒ½éœ€è¦ä¸€äº›æ—¶é—´æ‰èƒ½å®Œæˆã€‚ + +### 4. 测试集群 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æµ‹è¯•é›†ç¾¤ï¼š + +``` +eksctl get cluster +``` + +返回的结果应与以下内容类似： + +``` +eksctl get cluster +2021-03-18 15:09:35 [ℹ] eksctl version 0.40.0 +2021-03-18 15:09:35 [ℹ] using region us-west-2 +NAME REGION EKSCTL CREATED +rancher-server-cluster us-west-2 True +``` + +### 5. 安装 Ingress + +集群需è¦ä¸€ä¸ª Ingress，以从集群外部访问 Rancher。 + +为确ä¿ä½ é€‰æ‹©äº†æ­£ç¡®çš„ Ingress-NGINX Helm Chart，首先在 [Kubernetes/ingress-nginx 支æŒè¡¨](https://github.com/kubernetes/ingress-nginx#supported-versions-table)中找到与你的 Kubernetes 版本兼容的 `Ingress-NGINX 版本`。 + +然åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ—出å¯ç”¨çš„ Helm Chart： + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +`helm search` 命令的输出包å«ä¸€ä¸ª `APP VERSION` 列。此列下的版本等åŒäºŽä½ ä¹‹å‰é€‰æ‹©çš„ `Ingress-NGINX 版本`。使用应用程åºç‰ˆæœ¬ï¼Œé€‰æ‹©ä¸€ä¸ª Chart 版本，该版本打包了与你的 Kubernetes 兼容的应用程åºã€‚例如，如果使用的是 Kubernetes v1.23，则å¯ä»¥é€‰æ‹© v4.6.0 Helm Chart，因为 Ingress-NGINX v1.7.0 与该 Chart 打包在一起，而 v1.7.0 与 Kubernetes v1.23 兼容。如有疑问，请选择最新的兼容版本。 + +了解你需è¦çš„ Helm chart `版本`åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚它安装一个带有 Kubernetes è´Ÿè½½å‡è¡¡å™¨æœåŠ¡çš„ `nginx-ingress-controller`： + +``` +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.6.0 \ + --create-namespace +``` + +### 6. 获å–è´Ÿè½½å‡è¡¡å™¨çš„ IP + +è¿è¡Œä»¥ä¸‹å‘½ä»¤èŽ·å–è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€ï¼š + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +返回的结果应与以下内容类似： + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.100.90.18 a904a952c73bf4f668a17c46ac7c56ab-962521486.us-west-2.elb.amazonaws.com 80:31229/TCP,443:31050/TCP + 27m +``` + +ä¿å­˜ `EXTERNAL-IP`。 + +### 7. 设置 DNS + +到 Rancher Server 的外部æµé‡éœ€è¦é‡å®šå‘到你创建的负载å‡è¡¡å™¨ã€‚ + +创建指å‘ä½ ä¿å­˜çš„外部 IP 地å€çš„ DNS。这个 DNS 会用作 Rancher Server çš„ URL。 + +设置 DNS 的有效方法有很多。如需获得帮助，请å‚è§ AWS 文档中心的[转å‘æµé‡åˆ° ELB è´Ÿè½½å‡è¡¡å™¨](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer.html)。 + +### 8. 安装 Rancher Helm Chart + +按照[本页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。任何 Kubernetes å‘行版上安装的 Rancher çš„ Helm 说明都是一样的。 + +安装 Rancher 时，使用上一步获å–çš„ DNS å称作为 Rancher Server çš„ URL。它å¯ä»¥ä½œä¸º Helm 选项传递进æ¥ã€‚例如，如果 DNS å称是 `rancher.my.org`，你需è¦ä½¿ç”¨ `--set hostname=rancher.my.org` 选项æ¥è¿è¡Œ Helm 安装命令。 + +在此设置之上安装 Rancher 时，你还需è¦å°†ä»¥ä¸‹å€¼ä¼ é€’到 Rancher Helm 安装命令，以设置与 Rancher çš„ Ingress 资æºä¸€èµ·ä½¿ç”¨çš„ Ingress Controller çš„å称： + +``` +--set ingress.ingressClassName=nginx +``` + +请å‚阅[Helm 安装命令](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher)了解你的è¯ä¹¦é€‰é¡¹ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md new file mode 100644 index 00000000000..ac40b92087d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md @@ -0,0 +1,201 @@ +--- +title: 在 GKE 集群上安装 Rancher +--- + +在本节中，你将学习如何使用 GKE 安装 Rancher。 + +如果你已ç»æœ‰ä¸€ä¸ª GKE Kubernetes 集群，请直接跳转到[安装 Ingress](#7-安装-ingress)这个步骤。然åŽæŒ‰ç…§[此处](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的步骤安装 Rancher Helm Chart。 + +## 先决æ¡ä»¶ + +- 你需è¦æœ‰ä¸€ä¸ª Google è´¦å·ã€‚ +- 你需è¦æœ‰ä¸€ä¸ª Google Cloud Billing è´¦å·ã€‚ä½ å¯ä½¿ç”¨ Google Cloud Console æ¥ç®¡ç†ä½ çš„ Cloud Billing è´¦å·ã€‚有关 Cloud Console 的详情，请å‚è§ [ Console 通用指å—](https://support.google.com/cloud/answer/3465889?hl=en&ref_topic=3340599)。 +- 你需è¦è‡³å°‘一个在用的 IP 地å€å’Œè‡³å°‘ 2 个 CPU 的云é…é¢ã€‚有关 Rancher Server 的硬件è¦æ±‚，请å‚è§[本节](../../../pages-for-subheaders/installation-requirements.md#rke-和托管-kubernetes)。 + +## 1. å¯ç”¨ Kubernetes Engine API + +按照以下步骤å¯ç”¨ Kubernetes Engine API： + +1. 访问 Google Cloud Console 中的 [Kubernetes Engine 页é¢](https://console.cloud.google.com/projectselector/kubernetes?_ga=2.169595943.767329331.1617810440-856599067.1617343886)。 +1. 创建或选择一个项目。 +1. 打开项目，并为项目å¯ç”¨ Kubernetes Engine API。等待 API 和相关æœåŠ¡çš„å¯ç”¨ã€‚è¿™å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ—¶é—´ã€‚ +1. ç¡®ä¿ä¸ºä½ çš„云项目å¯ç”¨äº†è®¡è´¹ã€‚有关如何为你的项目å¯ç”¨è®¡è´¹ï¼Œè¯·å‚è§ [Google Cloud 文档中心](https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project)。 + +## 2. 打开 Cloud Shell + +Cloud Shell 是一个 shell 环境，用于管ç†æ‰˜ç®¡åœ¨ Google Cloud 上的资æºã€‚Cloud Shell 预装了 `gcloud` 命令行工具和 kubectl 命令行工具中。`gcloud` 工具为 Google Cloud æ供主è¦çš„命令行界é¢ï¼Œè€Œ`kubectl` 则æ供针对 Kubernetes 集群的主è¦å‘½ä»¤è¡Œç•Œé¢ã€‚ + +下文æ述了如何从 Google Cloud Console 或从本地工作站å¯åŠ¨ Cloud Shell。 + +### Cloud Shell + +如需从 [Google Cloud Console](https://console.cloud.google.com) å¯åŠ¨ shell，请在控制å°çš„å³ä¸Šè§’点击终端按钮。鼠标悬åœåœ¨æŒ‰é’®ä¸Šæ—¶ï¼Œå®ƒä¼šæ ‡è®°ä¸º **Activate Cloud Shell**。 + +### 本地 Shell + +执行以下步骤以安装 `gcloud` å’Œ `kubectl`： + +1. 按照[这些步骤](https://cloud.google.com/sdk/docs/install)安装 Cloud SDK。The Cloud SDK 包括 `gcloud` 命令行工具。ä¸åŒæ“作系统对应的步骤有所ä¸åŒã€‚ +1. 安装 Cloud SDK åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ä»¥å®‰è£… `kubectl` 命令行工具： + + ``` + gcloud components install kubectl + ``` + åŽé¢çš„步骤会é…ç½® `kubectl`，使其用于使用新的 GKE 集群。 +1. 如果 Helm 3 未安装的è¯ï¼Œ[安装 Helm 3](https://helm.sh/docs/intro/install/)。 +1. 使用 `HELM_EXPERIMENTAL_OCI` å˜é‡æ¥å¯ç”¨ Helm 的实验功能 [OCI é•œåƒæ”¯æŒ](https://github.com/helm/community/blob/master/hips/hip-0006.md)。把以下行添加到 `~/.bashrc` （或 macOS 中的 `~/.bash_profile`，或者你的 shell 存储环境å˜é‡çš„地方）： + + ``` + export HELM_EXPERIMENTAL_OCI=1 + ``` +1. è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åŠ è½½ä½ æ›´æ–°çš„ `.bashrc` 文件： + + ``` + source ~/.bashrc + ``` + 如果你è¿è¡Œçš„是 macOS，使用这个命令: + ``` + source ~/.bash_profile + ``` + + + +## 3. é…ç½® gcloud CLI + +选择以下方法之一é…置默认的 gcloud 设置： + +- 如果你想了解默认值，请使用 gcloud init。 +- 如需å•ç‹¬è®¾ç½®ä½ çš„项目 IDã€åœ°åŒºå’ŒåŒºåŸŸï¼Œä½¿ç”¨ gcloud config。 + + + + +1. è¿è¡Œ gcloud init 并按照指示æ“作： + + ``` + gcloud init + ``` + 如果你在远程æœåŠ¡å™¨ä¸Šä½¿ç”¨ SSH，使用 --console-only 标志，以防止该命令å¯åŠ¨æµè§ˆå™¨ã€‚ + + ``` + gcloud init --console-only + ``` +2. æŒ‰ç…§æŒ‡ç¤ºï¼Œä»¥æŽˆæƒ gcloud 使用你的 Google Cloud 账户，并选择你创建的新项目。 + + + + + + + +## 4. 确认 gcloud çš„é…置是å¦æ­£ç¡® + +è¿è¡Œï¼š + +``` +gcloud config list +``` + +返回的结果应与以下内容类似： + +``` +[compute] +region = us-west1 # Your chosen region +zone = us-west1-b # Your chosen zone +[core] +account = +disable_usage_reporting = True +project = + +Your active configuration is: [default] +``` + +## 5. 创建一个 GKE 集群 + +下é¢çš„命令创建了一个三节点的集群。 + +把 `cluster-name` 替æ¢ä¸ºä½ æ–°é›†ç¾¤çš„å称。 + +在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +è¦ä½¿ç”¨ Rancher æˆåŠŸåˆ›å»º GKE 集群，GKE 必须处于 Standard 模å¼ã€‚GKE 在创建 Kubernetes 集群时有两ç§è¿è¡Œæ¨¡å¼ï¼Œåˆ†åˆ«æ˜¯ Autopilot å’Œ Standard 模å¼ã€‚Autopilot 模å¼çš„集群é…置对编辑 kube-system 命å空间有é™åˆ¶ã€‚但是，Rancher 在安装时需è¦åœ¨ kube-system 命å空间中创建资æºã€‚因此，你将无法在以 Autopilot 模å¼åˆ›å»ºçš„ GKE 集群上安装 Rancher。如需详细了解 GKE Autopilot 模å¼å’Œ Standard 模å¼ä¹‹é—´çš„差异，请访问[比较 GKE Autopilot å’Œ Standard ](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison)。 + +**注æ„**：如果你è¦ä»Žæ—§çš„ Kubernetes 版本更新到 Kubernetes v1.22 或更高版本，你还需è¦[æ›´æ–°](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx。 + +``` +gcloud container clusters create cluster-name --num-nodes=3 --cluster-version= +``` + +## 6. 获å–验è¯å‡­è¯ + +创建集群åŽï¼Œä½ éœ€è¦èŽ·å¾—认è¯å‡­è¯æ‰èƒ½ä¸Žé›†ç¾¤äº¤äº’： + +``` +gcloud container clusters get-credentials cluster-name +``` + +此命令将 `kubectl` é…ç½®æˆä½¿ç”¨ä½ åˆ›å»ºçš„集群。 + +## 7. 安装 Ingress + +集群需è¦ä¸€ä¸ª Ingress，以从集群外部访问 Rancher。 + +以下命令安装带有 LoadBalancer æœåŠ¡çš„ `nginx-ingress-controller`： + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.0.18 \ + --create-namespace +``` + +## 8. 获å–è´Ÿè½½å‡è¡¡å™¨çš„ IP + +è¿è¡Œä»¥ä¸‹å‘½ä»¤èŽ·å–è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€ï¼š + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +返回的结果应与以下内容类似： + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +ingress-nginx-controller LoadBalancer 10.3.244.156 35.233.206.34 80:31876/TCP,443:32497/TCP 81s +``` + +ä¿å­˜ `EXTERNAL-IP`。 + +## 9. 设置 DNS + +到 Rancher Server 的外部æµé‡éœ€è¦é‡å®šå‘到你创建的负载å‡è¡¡å™¨ã€‚ + +创建指å‘ä½ ä¿å­˜çš„外部 IP 地å€çš„ DNS。这个 DNS 会用作 Rancher Server çš„ URL。 + +设置 DNS 的有效方法有很多。如需获å–帮助，请å‚è§ Google Cloud 文档中的[ç®¡ç† DNS 记录](https://cloud.google.com/dns/docs/records)部分。 + +## 10. 安装 Rancher Helm Chart + +按照[本页](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#安装-rancher-helm-chart)的说明安装 Rancher Helm Chart。任何 Kubernetes å‘行版上安装的 Rancher çš„ Helm 说明都是一样的。 + +安装 Rancher 时，使用上一步获å–çš„ DNS å称作为 Rancher Server çš„ URL。它å¯ä»¥ä½œä¸º Helm 选项传递进æ¥ã€‚例如，如果 DNS å称是 `rancher.my.org`，你需è¦ä½¿ç”¨ `--set hostname=rancher.my.org` 选项æ¥è¿è¡Œ Helm 安装命令。 + +在此设置之上安装 Rancher 时，你还需è¦è®¾ç½®ä¸Ž Rancher çš„ Ingress 资æºä¸€èµ·ä½¿ç”¨çš„ Ingress Controller çš„å称： + +``` +--set ingress.ingressClassName=nginx +``` + +请å‚阅[Helm 安装命令](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher)了解你的è¯ä¹¦é€‰é¡¹ã€‚ + +在 Rancher v2.7.5 中，如果你打算在集群上使用默认的 GKE Ingress 而ä¸å¯ç”¨ VPC 原生的集群模å¼ï¼Œåˆ™éœ€è¦è®¾ç½®ä»¥ä¸‹æ ‡å¿—： + +``` +--set service.type=NodePort +``` + +此设置是必è¦çš„，这考虑了与 ClusterIP（`cattle-system/rancher` 的默认类型）之间的兼容性问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml new file mode 100644 index 00000000000..ae248cc9b7c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml @@ -0,0 +1,37 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + defaults: + audit: restricted + audit-version: latest + enforce: restricted + enforce-version: latest + warn: restricted + warn-version: latest + exemptions: + namespaces: + - ingress-nginx + - kube-system + - cattle-system + - cattle-epinio-system + - cattle-fleet-system + - longhorn-system + - cattle-neuvector-system + - cattle-monitoring-system + - rancher-alerting-drivers + - cis-operator-system + - cattle-csp-adapter-system + - cattle-externalip-system + - cattle-gatekeeper-system + - istio-system + - cattle-istio-system + - cattle-logging-system + - cattle-windows-gmsa-system + - cattle-sriov-system + - cattle-ui-plugin-system + - tigera-operator + kind: PodSecurityConfiguration + name: PodSecurity + path: "" \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md new file mode 100644 index 00000000000..5570d3c4753 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md @@ -0,0 +1,128 @@ +--- +title: 回滚 +--- + +## 使用 Rancher 2.6.4+ 进行回滚的其他步骤 + +Rancher v2.6.4 å°† cluster-api 模å—从 v0.4.4 å‡çº§åˆ° v1.0.2。å过æ¥ï¼Œcluster-api çš„ v1.0.2 版本将集群 API 的自定义资æºå®šä¹‰ (CRD) 从 `cluster.x-k8s.io/v1alpha4` å‡çº§åˆ° `cluster.x-k8s.io/v1beta1`。当你å°è¯•å°† Rancher v2.6.4 回滚到以å‰ç‰ˆæœ¬çš„ Rancher v2.6.x 时，CRD å‡çº§åˆ° v1beta1 会导致回滚失败。这是因为使用旧 apiVersion (v1alpha4) çš„ CRD 与 v1beta1 ä¸å…¼å®¹ã€‚ + +è¦é¿å…回滚失败，你需è¦åœ¨å°è¯•æ¢å¤æ“作或回滚**之å‰**è¿è¡Œä»¥ä¸‹ Rancher 脚本： + +* `verify.sh`：检查集群中是å¦æœ‰ä»»ä½•ä¸Ž Rancher 相关的资æºã€‚ +* `cleanup.sh`：清ç†é›†ç¾¤ã€‚ + +有关详细信æ¯å’Œæºä»£ç ï¼Œè¯·å‚阅 [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup)。 + +:::caution + +`cleanup.sh` è¿è¡Œçš„时候会有åœæœºæ—¶é—´ï¼Œè¿™æ˜¯å› ä¸ºè„šæœ¬ä¼šåˆ é™¤ Rancher 创建的资æºã€‚ + +::: + +### 从 v2.6.4+ 回滚到较低版本的 v2.6.x + +1. 按照[说明](https://github.com/rancher/rancher-cleanup/blob/main/README.md)è¿è¡Œè„šæœ¬ã€‚ +1. 按照[说明](https://rancher.com/docs/rancher/v2.6/en/backups/migrating-rancher/)在现有集群上安装 rancher-backup Helm Chart 并æ¢å¤ä¹‹å‰çš„状æ€ã€‚ + 1. çœç•¥æ­¥éª¤ 3。 + 1. 执行到步骤 4 时，在è¦å›žæ»šåˆ°çš„ local 集群上安装 Rancher 2.6.x 版本。 + +## 回滚到 Rancher 2.5.0+ + +è¦å›žæ»šåˆ° Rancher 2.5.0+，使用 **Rancher 备份**应用并通过备份æ¥æ¢å¤ Rancher。 + +回滚åŽï¼ŒRancher 必须以较低/较早的版本å¯åŠ¨ã€‚ + +还原是通过创建 Restore 自定义资æºå®žçŽ°çš„。 + +:::note é‡è¦æ示： + +* 请按照此页é¢ä¸Šçš„说明在已备份的åŒä¸€é›†ç¾¤ä¸Šè¿˜åŽŸ Rancher。è¦æŠŠ Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œè¯·å‚照步骤[è¿ç§» Rancher](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +* 在使用相åŒè®¾ç½®æ¢å¤ Rancher 时，Rancher deployment 在æ¢å¤å¼€å§‹å‰è¢«æ‰‹åŠ¨ç¼©å‡ï¼Œç„¶åŽ Operator 将在æ¢å¤å®ŒæˆåŽå°†å…¶ç¼©å›žã€‚因此，在æ¢å¤å®Œæˆä¹‹å‰ï¼ŒRancher å’Œ UI 都将ä¸å¯ç”¨ã€‚如果 UI ä¸å¯ç”¨æ—¶ï¼Œä½ å¯ä½¿ç”¨ `kubectl create -f restore.yaml`YAML æ¢å¤æ–‡ä»¶æ¥ä½¿ç”¨åˆå§‹çš„集群 kubeconfig。 + +::: + +### 创建 Restore è‡ªå®šä¹‰èµ„æº + +1. 点击 **☰ > 集群管ç†**。 +1. 找到你的本地集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Rancher 备份 > 还原**。 + :::note + + 如果 Rancher Backups 应用ä¸å¯è§ï¼Œä½ éœ€è¦åˆ° **Apps** çš„ Charts 页é¢ä¸­å®‰è£…应用。详情请å‚è§[此处](../../../pages-for-subheaders/helm-charts-in-rancher.md#charts)。 + + ::: + +1. å•å‡»**创建**。 +1. 使用表å•æˆ– YAML 创建 Restore。如需获å–使用表å•åˆ›å»º Restore 资æºçš„更多信æ¯ï¼Œè¯·å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 +1. 如需使用 YAML 编辑器，点击**创建 > 使用 YAML 文件创建**。然åŽè¾“å…¥ Restore YAML。以下是 Restore 自定义资æºç¤ºä¾‹ï¼š + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + 如需获得é…ç½® Restore 的帮助，请å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 + +1. å•å‡»**创建**。 + +**结果**：已创建备份文件并更新到目标存储ä½ç½®ã€‚资æºè¿˜åŽŸé¡ºåºå¦‚下： + +1. 自定义资æºå®šä¹‰ï¼ˆCRD） +2. é›†ç¾¤èŒƒå›´èµ„æº +3. 命åç©ºé—´èµ„æº + +如需查看还原的处ç†æ–¹å¼ï¼Œè¯·æ£€æŸ¥ Operator 的日志。按照如下步骤获å–日志： + +```yaml +kubectl get pods -n cattle-resources-system +kubectl logs -n cattle-resources-system -f +``` + +### 回滚到上一个 Rancher 版本 + +ä½ å¯ä»¥ä½¿ç”¨ Helm CLI 回滚 Rancher。è¦å›žæ»šåˆ°ä¸Šä¸€ä¸ªç‰ˆæœ¬ï¼š + +```yaml +helm rollback rancher -n cattle-system +``` + +如果你ä¸æ˜¯æƒ³å›žæ»šåˆ°ä¸Šä¸€ä¸ªç‰ˆæœ¬ï¼Œä½ ä¹Ÿå¯ä»¥æŒ‡å®šå›žæ»šçš„版本。查看部署历å²è®°å½•ï¼š + +```yaml +helm history rancher -n cattle-system +``` + +确定目标版本åŽï¼Œæ‰§è¡Œå›žæ»šã€‚此示例回滚到版本 `3`： + +```yaml +helm rollback rancher 3 -n cattle-system +``` + +## 回滚到 Rancher 2.2-2.4 + +è¦å›žæ»šåˆ° 2.5 之å‰çš„ Rancher 版本，å‚考此处的步骤[æ¢å¤å¤‡ä»½ — Kubernetes 安装](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md)。如果æ¢å¤ Rancher Server 的集群的æŸä¸ªå¿«ç…§ï¼ŒRancher 的版本以åŠçŠ¶æ€å‡ä¼šæ¢å¤å›žåˆ°å¿«ç…§æ—¶çš„版本和状æ€ã€‚ + +有关回滚 Docker 安装的 Rancher，请å‚è§[本页](../other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md)。 + +:::note + +托管集群对其状æ€å…·æœ‰æƒå¨æ€§ã€‚因此，æ¢å¤ Rancher Server ä¸ä¼šæ¢å¤å¿«ç…§åŽå¯¹æ‰˜ç®¡é›†ç¾¤è¿›è¡Œçš„工作负载部署或更改。 + +::: + +## 回滚到 Rancher 2.0-2.1 + +我们ä¸å†æ”¯æŒå›žæ»šåˆ° Rancher 2.0-2.1。回滚到这些版本的说明ä¿ç•™åœ¨[此处](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup/roll-back-to-v2.0-v2.1.md)，仅用于无法å‡çº§åˆ° v2.2 的情况。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md new file mode 100644 index 00000000000..2912d831f55 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md @@ -0,0 +1,188 @@ +--- +title: Rancher Server Kubernetes 集群的问题排查 +--- + +本文介ç»å¦‚何对安装在 Kubernetes 集群上的 Rancher 进行故障排除。 + +### 相关命å空间 + +故障排除主è¦é’ˆå¯¹ä»¥ä¸‹ 3 个命å空间中的对象： + +- `cattle-system`：`rancher` deployment å’Œ Pod。 +- `ingress-nginx`：Ingress Controller Pod å’Œ services。 +- `cert-manager`：`cert-manager` Pod。 + +### "default backend - 404" + +很多æ“作都有å¯èƒ½å¯¼è‡´ Ingress Controller 无法将æµé‡è½¬å‘到你的 Rancher 实例。但是大多数情况下都是由错误的 SSL é…置导致的。 + +检查事项： + +- [Rancher 是å¦æ­£åœ¨è¿è¡Œ](#检查-rancher-是å¦æ­£åœ¨è¿è¡Œ) +- [è¯ä¹¦çš„ Common Name（CN）是 "Kubernetes Ingress Controller Fake Certificate"](#è¯ä¹¦çš„-cn-是-kubernetes-ingress-controller-fake-certificate) + +### 检查 Rancher 是å¦æ­£åœ¨è¿è¡Œ + +使用 `kubectl` 检查 `cattle-system` 系统命å空间，并查看 Rancher Pod 的状æ€æ˜¯å¦æ˜¯ **Running**： + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +如果状æ€ä¸æ˜¯ `Running`，在 Pod 上è¿è¡Œ `describe`，并检查 **Events**： + +``` +kubectl -n cattle-system describe pod + +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 11m default-scheduler Successfully assigned rancher-784d94f59b-vgqzh to localhost + Normal SuccessfulMountVolume 11m kubelet, localhost MountVolume.SetUp succeeded for volume "rancher-token-dj4mt" + Normal Pulling 11m kubelet, localhost pulling image "rancher/rancher:v2.0.4" + Normal Pulled 11m kubelet, localhost Successfully pulled image "rancher/rancher:v2.0.4" + Normal Created 11m kubelet, localhost Created container + Normal Started 11m kubelet, localhost Started container +``` + +### 检查 Rancher 日志 + +使用 `kubectl` 列出 Pod： + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +使用 `kubectl` å’Œ Pod å称列出该 Pod 的日志： + +``` +kubectl -n cattle-system logs -f rancher-784d94f59b-vgqzh +``` + +### è¯ä¹¦çš„ CN 是 "Kubernetes Ingress Controller Fake Certificate" + +使用æµè§ˆå™¨æ£€æŸ¥è¯ä¹¦çš„详细信æ¯ã€‚如果显示 CN 是 "Kubernetes Ingress Controller Fake Certificate"，则说明读å–或é¢å‘ SSL è¯ä¹¦æ—¶å‡ºçŽ°äº†é—®é¢˜ã€‚ + +:::note + +如果你使用的是 Let's Encrypt è¯ä¹¦ï¼Œè¯ä¹¦é¢å‘的过程å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿã€‚ + +::: + +### 排查 Cert-Manager é¢å‘çš„è¯ä¹¦ï¼ˆRancher 或 Let's Encrypt 生æˆçš„）问题 + +`cert-manager` 有 3 部分： + +- `cert-manager` 命å空间中的 `cert-manager` Pod。 +- `cattle-system` 命å空间中的 `Issuer` 对象。 +- `cattle-system` 命å空间中的 `Certificate` 对象。 + +å¾€åŽæ“作，对æ¯ä¸ªå¯¹è±¡è¿è¡Œ `kubectl describe` 并检查事件。这样，你å¯ä»¥è¿½è¸ªå¯èƒ½ä¸¢å¤±çš„内容。 + +以下是 Issuer 有问题的示例： + +``` +kubectl -n cattle-system describe certificate +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning IssuerNotReady 18s (x23 over 19m) cert-manager Issuer rancher not ready +``` + +``` +kubectl -n cattle-system describe issuer +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning ErrInitIssuer 19m (x12 over 19m) cert-manager Error initializing issuer: secret "tls-rancher" not found + Warning ErrGetKeyPair 9m (x16 over 19m) cert-manager Error getting keypair for CA issuer: secret "tls-rancher" not found +``` + +### 排查你自己æ供的 SSL è¯ä¹¦é—®é¢˜ + +ä½ çš„è¯ä¹¦ç›´æŽ¥åº”用于 `cattle-system` 命å空间中的 Ingress 对象。 + +检查 Ingress 对象的状æ€ï¼Œå¹¶æŸ¥çœ‹å®ƒæ˜¯å¦å‡†å¤‡å°±ç»ªï¼š + +``` +kubectl -n cattle-system describe ingress +``` + +如果 Ingress 对象已就绪，但是 SSL ä»ç„¶æ— æ³•æ­£å¸¸å·¥ä½œï¼Œä½ çš„è¯ä¹¦æˆ–密文的格å¼å¯èƒ½ä¸æ­£ç¡®ã€‚ + +è¿™ç§æƒ…况下，请检查 nginx-ingress-controller 的日志。nginx-ingress-controller çš„ Pod 中有多个容器，因此你需è¦æŒ‡å®šå®¹å™¨çš„å称： + +``` +kubectl -n ingress-nginx logs -f nginx-ingress-controller-rfjrq nginx-ingress-controller +... +W0705 23:04:58.240571 7 backend_ssl.go:49] error obtaining PEM from secret cattle-system/tls-rancher-ingress: error retrieving secret cattle-system/tls-rancher-ingress: secret cattle-system/tls-rancher-ingress was not found +``` + +### 没有匹é…çš„ "Issuer" + +你所选的 SSL é…ç½®è¦æ±‚在安装 Rancher 之å‰å…ˆå®‰è£… Cert-Manager，å¦åˆ™ä¼šå‡ºçŽ°ä»¥ä¸‹é”™è¯¯: + +``` +Error: validation failed: unable to recognize "": no matches for kind "Issuer" in version "certmanager.k8s.io/v1alpha1" +``` + +在这ç§æƒ…况下，先安装 Cert-Manager，然åŽå†é‡æ–°å®‰è£… Rancher。 + + +### Canal Pod 显示 READY 2/3 + +此问题的最常è§åŽŸå› æ˜¯ç«¯å£ 8472/UDP 在节点之间未打开。因此，你å¯ä»¥æ£€æŸ¥ä½ çš„本地防ç«å¢™ã€ç½‘络路由或安全组。 + +解决网络问题åŽï¼Œ`canal` Pod 会超时并é‡å¯ä»¥å»ºç«‹è¿žæŽ¥ã€‚ + +### nginx-ingress-controller Pod 显示 RESTARTS + +此问题的最常è§åŽŸå› æ˜¯ `canal` pod 未能建立覆盖网络。å‚è§ [canal Pod 显示 READY `2/3`](#canal-pod-显示-ready-23) 进行排查。 + + +### Failed to dial to /var/run/docker.sock: ssh: rejected: administratively prohibited (open failed) + +此错误的原因å¯èƒ½æ˜¯ï¼š + +* 指定连接的用户无æƒè®¿é—® Docker Socket。如果是这个原因，你通过登录主机并è¿è¡Œ `docker ps` 命令æ¥æ£€æŸ¥ï¼š + +``` +$ ssh user@server +user@server$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +``` + +如果需è¦äº†è§£å¦‚何进行正确设置，请å‚è§[ä»¥éž root ç”¨æˆ·èº«ä»½ç®¡ç† Docker](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)。 + +* 你使用的æ“作系统是 RedHat 或 CentOS：由于 [Bugzilla #1527565](https://bugzilla.redhat.com/show_bug.cgi?id=1527565)，你ä¸èƒ½ä½¿ç”¨ `root` 用户连接到节点。因此，你需è¦æ·»åŠ ä¸€ä¸ªå•ç‹¬çš„用户并é…置其访问 Docker Socket。如果需è¦äº†è§£å¦‚何进行正确设置，请å‚è§[ä»¥éž root ç”¨æˆ·èº«ä»½ç®¡ç† Docker](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user)。 + +* SSH æœåŠ¡å™¨ç‰ˆæœ¬ä¸æ˜¯ 6.7 或更高版本：高版本是 Socket 转å‘所必须的，用于通过 SSH 连接到 Docker Socket。你å¯ä»¥åœ¨ä½ è¦è¿žæŽ¥çš„主机上使用 `sshd -V` 或使用 netcat 进行检查： +``` +$ nc xxx.xxx.xxx.xxx 22 +SSH-2.0-OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.10 +``` + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: no key found + +`ssh_key_path` 密钥文件无法访问：请确ä¿ä½ å·²ç»æŒ‡å®šäº†ç§é’¥æ–‡ä»¶ï¼ˆä¸æ˜¯å…¬é’¥ `.pub`），而且è¿è¡Œ `rke` 命令的用户å¯ä»¥è®¿é—®è¯¥ç§é’¥æ–‡ä»¶ã€‚ + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain + +`ssh_key_path` 密钥文件ä¸æ˜¯è®¿é—®èŠ‚点的正确文件：请仔细检查，确ä¿ä½ å·²ä¸ºèŠ‚点指定了正确的 `ssh_key_path` 和连接用户。 + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: cannot decode encrypted private keys + +如需使用加密的ç§é’¥ï¼Œè¯·ä½¿ç”¨ `ssh-agent` æ¥ä½¿ç”¨å¯†ç æ¥åŠ è½½å¯†é’¥ã€‚如果在è¿è¡Œ `rke` 命令的环境中找到 `SSH_AUTH_SOCK` 环境å˜é‡ï¼Œå®ƒå°†è‡ªåŠ¨ç”¨äºŽè¿žæŽ¥åˆ°èŠ‚点。 + +### Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? + +节点无法通过é…置的 `address` å’Œ `port` 访问。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md new file mode 100644 index 00000000000..f55a95063b5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md @@ -0,0 +1,68 @@ +--- +title: 将加固的自定义/导入集群å‡çº§åˆ° Kubernetes v1.25 +--- + +Kubernetes v1.25 改å˜äº†é›†ç¾¤æ述和执行安全策略的方å¼ã€‚从这个版本开始，[Pod 安全策略 (PSP)](https://kubernetes.io/docs/concepts/security/pod-security-policy/)ä¸å†å¯ç”¨ã€‚Kubernetes v1.25 将它们替æ¢ä¸ºæ–°çš„安全对象：[Pod 安全标准 (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/) å’Œ [Pod 安全准入 (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/)。 + +如果你具有自定义或导入的加固集群，你需è¦åšå¥½å‡†å¤‡ï¼Œç¡®ä¿å°†æ—§ç‰ˆæœ¬çš„ Kubernetes 顺利å‡çº§åˆ° v1.25 或更高版本。 + +:::note + +å‡çº§åˆ° v1.25 åŽï¼Œæ·»åŠ å¿…è¦çš„ Rancher 命å空间è±å…。有关详细信æ¯ï¼Œè¯·å‚阅 [Pod 安全准入 (PSA) é…置模æ¿](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md#è±å…必须的-rancher-命å空间)。 + +::: + +## 将导入的加固集群å‡çº§åˆ° Kubernetes v1.25 或更高版本 + + + + +在集群中的æ¯ä¸ªèŠ‚点上执行以下æ“作： +1. å°† [`rancher-psact.yaml`](./rancher-psact.yaml) ä¿å­˜åˆ° `/etc/rancher/rke2` 中。 +1. 编辑 RKE2 é…置文件： + 1. å°† `profile` 字段更新为 `cis-1.23`。 + 1. 指定刚æ‰æ·»åŠ çš„é…置文件的路径：`pod-security-admission-config-file: /etc/rancher/rke2/rancher-psact.yaml`。 + + + + +在集群中的æ¯ä¸ªèŠ‚点上执行以下æ“作： + +éµå¾ª K3s [将加固集群从 v1.24.x å‡çº§åˆ° v1.25.x](https://docs.k3s.io/known-issues#hardened-125)的官方说明，但使用[自定义](./rancher-psact.yaml)Rancher PSA é…置模æ¿ï¼Œè€Œä¸æ˜¯ K3s 官方网站上æ供的é…置。 + + + +执行这些步骤åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI å‡çº§é›†ç¾¤çš„ Kubernetes 版本： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在 **Kubernetes 版本**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦ä½¿ç”¨çš„版本。 +1. å•å‡»**ä¿å­˜**。 + +## 将自定义加固集群å‡çº§åˆ° Kubernetes v1.25 或更高版本 + + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在**åŸºæœ¬ä¿¡æ¯ > 安全**下的 **CIS é…置文件**下拉èœå•ä¸­ï¼Œé€‰æ‹© `cis-1.23`。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹© `rancher-restricted`。 +1. 在 **Kubernetes 版本**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦ä½¿ç”¨çš„版本。 +1. å•å‡»**ä¿å­˜**。 + + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑 YAML**。 +1. 从 `kube-apiserver-arg.enable-admission-plugins` 中删除 `PodSecurityPolicy`。 +1. 在 `spec` 字段中，添加一行：`defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted` +1. å°† `kubernetesVersion` 更新为你选择的版本（v1.25 或更高版本）。 +1. å•å‡»**ä¿å­˜**。 + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md new file mode 100644 index 00000000000..711077b6635 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md @@ -0,0 +1,203 @@ +--- +title: å‡çº§ +--- + +本文介ç»å¦‚何å‡çº§ä½¿ç”¨ Helm 安装在 Kubernetes 集群上的 Rancher Server。这些步骤也适用于使用 Helm 进行的离线安装。 + +有关使用 Docker 安装的 Rancher çš„å‡çº§è¯´æ˜Žï¼Œè¯·å‚è§[本页。](../other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md) + +如需å‡çº§ Kubernetes 集群中的组件，或 [Kubernetes services](https://rancher.com/docs/rke/latest/en/config-options/services/) 或 [附加组件（add-on）](https://rancher.com/docs/rke/latest/en/config-options/add-ons/)的定义，请å‚è§ [RKE å‡çº§æ–‡æ¡£](https://rancher.com/docs/rke/latest/en/upgrades/)çš„ Rancher Kubernetes 引擎。 + + +## 先决æ¡ä»¶ + +### 访问 kubeconfig + +Helm çš„è¿è¡Œä½ç½®ï¼Œåº”该与你的 kubeconfig 文件，或你è¿è¡Œ kubectl 命令的ä½ç½®ç›¸åŒã€‚ + +如果你在安装 Kubernetes 时使用了 RKE，那么 config 将会在你è¿è¡Œ `rke up` 的目录下创建。 + +kubeconfig 也å¯ä»¥é€šè¿‡ `--kubeconfig` 标签（详情请å‚è§ https://helm.sh/docs/helm/helm/ ）æ¥æ‰‹åŠ¨æŒ‡å®šæ‰€éœ€çš„集群。 + +### 查看已知问题 + +如需查看æ¯ä¸ª Rancher 版本的已知问题，请å‚è§ [GitHub](https://github.com/rancher/rancher/releases) 中的å‘行说明，或查看 [Rancher 论å›](https://forums.rancher.com/c/announcements/12)。 + +ä¸æ”¯æŒ _å‡çº§_ 或 _å‡çº§åˆ°_ [rancher-alpha 仓库](../resources/choose-a-rancher-version.md#helm-chart-仓库)中的任何 Chart。 +### Helm 版本 + +本安装指å—å‡å®šä½ ä½¿ç”¨çš„是 Helm 3。 + +如果你使用 Helm 2，请å‚è§ [Helm 2 è¿ç§»åˆ° Helm 3 文档](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/)。如果你ä¸èƒ½å‡çº§åˆ° Helm 3，[Helm 2 å‡çº§é¡µé¢](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/helm2.md)æ供了使用 Helm 2 å‡çº§çš„æ—§å‡çº§æŒ‡å—。 + +### 离线安装：推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +[仅适用于离线安装](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)：为新的 Rancher Server 版本收集和推é€é•œåƒã€‚使用你需è¦é’ˆå¯¹ Rancher 版本å‡çº§çš„é•œåƒï¼ŒæŒ‰ç…§æ­¥éª¤[推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md)。 + +### 使用 cert-manager 0.8.0 之å‰çš„版本å‡çº§ + +[从 2019 å¹´ 11 月 1 日开始，Let's Encrypt å·²å±è”½æ—©äºŽ 0.8.0 çš„ cert-manager 实例](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753)。因此，请å‚è§[说明](../resources/upgrade-cert-manager.md)把 cert-manager å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ã€‚ + +## å‡çº§æ¦‚è¦ + +按照以下步骤å‡çº§ Rancher Server： + + +### 1. 备份è¿è¡Œ Rancher Server çš„ Kubernetes 集群 + +使用[备份应用](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)æ¥å¤‡ä»½ Rancher。 + +如果å‡çº§è¿‡ç¨‹ä¸­å‡ºçŽ°é—®é¢˜ï¼Œä½ å°†ä½¿ç”¨å¤‡ä»½ä½œä¸ºè¿˜åŽŸç‚¹ã€‚ + +### 2. æ›´æ–° Helm Chart 仓库 + +1. 更新本地 Helm 仓库缓存。 + + ``` + helm repo update + ``` + +1. 获å–你用æ¥å®‰è£… Rancher 的仓库å称。 + + 关于仓库åŠå…¶åŒºåˆ«ï¼Œè¯·å‚è§ [Helm Chart Repositories](../resources/choose-a-rancher-version.md#helm-chart-仓库)。 + + - Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + + ``` + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + + :::note + + 如果你想切æ¢åˆ°ä¸åŒçš„ Helm Chart 仓库，请按照[切æ¢ä»“库步骤](../resources/choose-a-rancher-version.md#切æ¢åˆ°ä¸åŒ-helm-chart-仓库)进行æ“作。如果你è¦åˆ‡æ¢ä»“库，请先å†æ¬¡åˆ—出仓库，å†ç»§ç»­æ‰§è¡Œæ­¥éª¤ 3，以确ä¿æ·»åŠ äº†æ­£ç¡®çš„仓库。 + + ::: + +1. 从 Helm Chart 仓库获å–最新的 Chart æ¥å®‰è£… Rancher。 + + 该命令将æå–最新的 Chart，并将其作为 `.tgz`文件ä¿å­˜åœ¨å½“å‰ç›®å½•ä¸­ã€‚ + + ```plain + helm fetch rancher-/rancher + ``` + ä½ å¯ä»¥é€šè¿‡ `--version=` 标记，æ¥æŒ‡å®šè¦å‡çº§çš„目标 Chart 版本。例如： + + ```plain + helm fetch rancher-/rancher --version=2.6.8 + ``` + +### 3. å‡çº§ Rancher + +本节介ç»äº†å¦‚何使用 Helm å‡çº§ Rancher 的一般（互è”网连接）或离线安装。 + +:::note 离线说明： + +如果你在离线环境中安装 Rancher，请跳过本页的其余部分，按照[本页](air-gapped-upgrades.md)上的说明渲染 Helm 模æ¿ã€‚ + +::: + + +从当å‰å®‰è£…çš„ Rancher Helm Chart 中获å–用 `--set`传递的值。 + +``` +helm get values rancher -n cattle-system + +hostname: rancher.my.org +``` + +:::note + +这个命令会列出更多的值。此处展示的åªæ˜¯å…¶ä¸­ä¸€ä¸ªå€¼çš„例å­ã€‚ + +::: + +:::tip +Deployment çš„å称å¯èƒ½ä¼šæœ‰æ‰€ä¸åŒã€‚例如，如果你通过 AWS Marketplace 部署 Rancher，则 Deployment çš„å称为“rancher-stableâ€ã€‚ +因此： +``` +helm get values rancher-stable -n cattle-system + +hostname: rancher.my.org +``` +::: + +如果è¦å°† cert-manager 从 v1.5 或更早的版本å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ï¼Œè¯·å‚阅 [cert-manager upgrade docs](../resources/upgrade-cert-manager.md#选项-cå‡çº§-15-åŠä»¥ä¸‹ç‰ˆæœ¬çš„-cert-manager) 了解如何在ä¸å¸è½½æˆ–é‡æ–°å®‰è£… Rancher 的情况下å‡çº§ cert-manager。å¦åˆ™ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹[ Rancher å‡çº§æ­¥éª¤](#rancher-å‡çº§æ­¥éª¤)进行æ“作。 + +#### Rancher å‡çº§æ­¥éª¤ + +ä¿ç•™ä½ çš„所有设置把 Rancher å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ã€‚ + +将上一步中的所有值用 `--set key=value` 追加到命令中。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` + +:::note + +以上是一个例å­ï¼Œå¯èƒ½æœ‰æ›´å¤šä¸Šä¸€æ­¥çš„值需è¦è¿½åŠ ã€‚ + +::: + +:::tip +如果你通过 AWS Marketplace 部署 Rancher，则 Deployment çš„å称为“rancher-stableâ€ã€‚ +因此： +``` +helm upgrade rancher-stable rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` +::: + +å¦å¤–，你也å¯ä»¥å°†å½“å‰çš„值导出到一个文件中，并在å‡çº§æ—¶å¼•ç”¨è¯¥æ–‡ä»¶ã€‚例如，如果你åªéœ€è¦æ”¹å˜ Rancher 的版本： + +1. 将当å‰å€¼å¯¼å‡ºåˆ°æ–‡ä»¶ï¼š + ``` + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` +1. åªæ›´æ–° Rancher 版本： + + 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + + ``` + helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version=2.6.8 + ``` + +### 4. 验è¯å‡çº§ + +登录 Rancher 以确认å‡çº§æˆåŠŸã€‚ + +:::tip + +å‡çº§åŽå‡ºçŽ°ç½‘络问题？ + +请å‚è§[æ¢å¤é›†ç¾¤ç½‘络](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md)。 + +::: + +## 已知å‡çº§é—®é¢˜ + +ä½ å¯ä»¥åœ¨ [GitHub](https://github.com/rancher/rancher/releases) å‘å¸ƒè¯´æ˜Žä»¥åŠ [Rancher 论å›](https://forums.rancher.com/c/announcements/12)中找到æ¯ä¸ª Rancher 版本的已知问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/feature-flags.md new file mode 100644 index 00000000000..873a0689b14 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -0,0 +1,39 @@ +--- +title: 功能开关 +--- + +使用功能开关（Feature Flag），你å¯ä»¥è¯•ç”¨å¯é€‰æˆ–实验性的功能并å¯ç”¨æ­£åœ¨é€æ­¥æ·˜æ±°çš„旧版功能。 + +è¦äº†è§£åŠŸèƒ½çš„值以åŠå¦‚何å¯ç”¨å®ƒä»¬ï¼Œè¯·å‚阅[å¯ç”¨å®žéªŒæ€§åŠŸèƒ½](../../../pages-for-subheaders/enable-experimental-features.md)。 + +:::note + +æŸäº›åŠŸèƒ½è¦æ±‚é‡æ–°å¯åŠ¨ Rancher 容器。Rancher UI 中标记了è¦æ±‚é‡å¯çš„功能。 + +::: + +以下是 Rancher 中å¯ç”¨çš„功能开关列表。如果你是从旧 Rancher 版本å‡çº§çš„，你å¯èƒ½ä¼šåœ¨ Rancher UI 中看到其他功能，例如 `proxy` 或 `dashboard`（å‡[已中断](/versioned_docs/version-2.5/reference-guides/installation-references/feature-flags.md)）： + +- `continuous-delivery`：å…许从 Fleet 中å•ç‹¬ç¦ç”¨ Fleet GitOps。有关详细信æ¯ï¼Œè¯·å‚阅[æŒç»­äº¤ä»˜](../../../how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md)。 +- `fleet`：v2.6 åŠæ›´é«˜ç‰ˆæœ¬çš„ Rancher é…ç½®æ¡†æž¶éœ€è¦ Fleet。å³ä½¿ä½ åœ¨æ—§ Rancher 版本中ç¦ç”¨äº†è¯¥æ ‡å¿—，该标志也将在å‡çº§æ—¶è‡ªåŠ¨å¯ç”¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [Fleet - GitOps at Scale](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md)。 +- `harvester`ï¼šç®¡ç† Virtualization Management 页é¢çš„访问。用户å¯ä»¥åœ¨è¯¥é¡µé¢ç›´æŽ¥å¯¼èˆªåˆ° Harvester 集群并访问 Harvester UI。有关详细信æ¯ï¼Œè¯·å‚阅 [Harvester 集æˆ](../../../integrations-in-rancher/harvester.md)。 +- `istio-virtual-service-ui`：å¯ç”¨[å¯è§†ç•Œé¢](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md)æ¥åˆ›å»ºã€è¯»å–ã€æ›´æ–°å’Œåˆ é™¤ Istio 虚拟æœåŠ¡å’Œç›®æ ‡è§„则，这些都是 Istio æµé‡ç®¡ç†åŠŸèƒ½ã€‚ +- `legacy`：å¯ç”¨ 2.5.x åŠæ›´æ—©ç‰ˆæœ¬çš„一组功能，这些功能正é€æ¸è¢«æ–°çš„实现淘汰。它们是已弃用以åŠåŽç»­å¯ç”¨äºŽæ–°ç‰ˆæœ¬çš„功能组åˆã€‚æ–°çš„ Rancher 安装会默认ç¦ç”¨æ­¤æ ‡å¿—。如果你从以å‰ç‰ˆæœ¬çš„ Rancher å‡çº§ï¼Œæ­¤æ ‡å¿—会å¯ç”¨ã€‚ +- `multi-cluster-management`：å…许é…置和管ç†å¤šä¸ª Kubernetes 集群。此标志åªèƒ½åœ¨å®‰è£…时设置。åŽç»­æ— æ³•å¯ç”¨æˆ–ç¦ç”¨å®ƒã€‚ +- `rke1-custom-node-cleanup`：清除已删除的 RKE1 自定义节点。建议你å¯ç”¨æ­¤æ ‡å¿—，以防止已删除的节点å°è¯•é‡æ–°åŠ å…¥é›†ç¾¤ã€‚ +- `rke2`：å¯ç”¨é…ç½® RKE2 集群。此标志默认å¯ç”¨ã€‚ +- `token-hashing`：å¯ç”¨ä»¤ç‰Œå“ˆå¸Œã€‚å¯ç”¨åŽï¼Œä¼šä½¿ç”¨ SHA256 算法对现有 Token 和所有新 Token 进行哈希处ç†ã€‚一旦对 Token 进行哈希处ç†ï¼Œå°±æ— æ³•æ’¤æ¶ˆæ“作。此标志在å¯ç”¨åŽæ— æ³•ç¦ç”¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [API 令牌](../../../reference-guides/about-the-api/api-tokens.md#令牌哈希)。 +- `unsupported-storage-drivers`：å…许å¯ç”¨éžé»˜è®¤å¯ç”¨çš„存储æ供程åºå’Œå·æ’件。有关详细信æ¯ï¼Œè¯·å‚阅[å…许使用ä¸å—支æŒçš„存储驱动程åº](../../../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md)。 + +下表介ç»äº† Rancher 中功能开关的å¯ç”¨æ€§å’Œé»˜è®¤å€¼ã€‚标记为“GAâ€çš„功能已普éå¯ç”¨ï¼š + +| 功能开关å称 | 默认值 | çŠ¶æ€ | å¯ç”¨äºŽ | +| ----------------------------- | ------------- | ------------ | --------------- | +| `continuous-delivery` | `true` | GA | v2.6.0 | +| `fleet` | `true` | ä¸èƒ½ç¦ç”¨ | v2.6.0 | +| `fleet` | `true` | GA | v2.5.0 | +| `harvester` | `true` | 实验功能 | v2.6.1 | +| `legacy` | 新安装：`false`ï¼›å‡çº§ï¼š`true` | GA | v2.6.0 | +| `rke1-custom-node-cleanup` | `true` | GA | v2.6.0 | +| `rke2` | `true` | 实验功能 | v2.6.0 | +| `token-hashing` | 新安装：`false`ï¼›å‡çº§ï¼š`true` | GA | v2.6.0 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md new file mode 100644 index 00000000000..bd69a89cfee --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md @@ -0,0 +1,310 @@ +--- +title: Rancher Helm Chart 选项 +keywords: [rancher helm chart, rancher helm 选项, rancher helm chart 选项, helm chart rancher, helm 选项 rancher, helm chart 选项 rancher] +--- + +本文æ供了 Rancher Helm Chart çš„é…ç½®å‚考。 + +如需选择 Helm Chart 版本，请å‚è§[本页](../../../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +了解开å¯å®žéªŒæ€§åŠŸèƒ½çš„详情，请å‚è§[本页](../../../pages-for-subheaders/enable-experimental-features.md)。 + +## 常用选项 + +| 选项 | 默认值 | æè¿° | +| ------------------------- | ------------- | ---------------------------------------------------------------------------------- | +| `bootstrapPassword` | " " | `string` - 为第一个管ç†å‘˜ç”¨æˆ·è®¾ç½®[引导密ç ](#引导密ç )。登录åŽï¼Œç®¡ç†å‘˜éœ€è¦é‡ç½®å¯†ç ã€‚如ä¸è®¾ç½®ï¼Œä¼šä½¿ç”¨éšæœºç”Ÿæˆçš„引导密ç ã€‚ | +| `hostname` | " " | `string` - ä½ çš„ Rancher Server 的完全é™å®šçš„域å（FQDN) | +| `ingress.tls.source` | "rancher" | `string` - ä»Žå“ªé‡ŒèŽ·å– Ingress çš„è¯ä¹¦- "rancher, letsEncrypt, secret" | +| `letsEncrypt.email` | " " | `string` - ä½ çš„é‚®ç®±åœ°å€ | +| `letsEncrypt.environment` | "production" | `string` - å¯é€‰é¡¹ï¼š"staging, production" | +| `privateCA` | false | `bool` - 如果你的è¯ä¹¦æ˜¯ç”±ç§æœ‰ CA ç­¾å‘的，把这个值设置为 true | + +
+ +## 高级选项 + +| 选项 | 默认值 | æè¿° | +| ------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `additionalTrustedCAs` | false | `bool` - 请å‚è§[é¢å¤–的授信 CA](#é¢å¤–的授信-ca) | +| `addLocal` | "true" | `string` - 让 Rancher 检测并导入 “local†Rancher Server 集群。_注æ„：此选项在 2.5.0 中已ä¸å¯ç”¨ã€‚ä½ å¯è€ƒè™‘使用 `restrictedAdmin` 选项，æ¥é¿å…用户修改本地集群。_ | +| `antiAffinity` | "preferred" | `string` - Rancher Pod çš„å亲和性规则 - "preferred, required" | +| `auditLog.destination` | "sidecar" | `string` - å‘é€å®¡è®¡æ—¥å¿—到 Sidecar 容器的控制å°æˆ– hostPath å· - "sidecar, hostPath" | +| `auditLog.hostPath` | "/var/log/rancher/audit" | `string` - 主机上的日志文件目标地å€ï¼ˆä»…当`auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.level` | 0 | `int` - 设置 [API 审计日志](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md)等级。0 代表关闭。[0-3] | +| `auditLog.maxAge` | 1 | `int` - 旧审计日志文件最多å¯ä¿ç•™çš„天数（仅当`auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.maxBackup` | 1 | `int` - 审计文件最大å¯ä¿ç•™çš„个数（仅当 `auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.maxSize` | 100 | `int` - 在审计日志被轮æ¢å‰çš„最大容é‡ï¼Œå•ä½æ˜¯ MB（仅当 `auditLog.destination` 的值是 `hostPath` 时生效） | +| `auditLog.image.repository` | "registry.suse.com/bci/bci-micro" | `string` - 用于收集审计日志的镜åƒçš„ä½ç½®ã€‚ | +| `auditLog.image.tag` | "15.4.14.3" | `string` - 用于收集审计日志的镜åƒçš„标签。 | +| `auditLog.image.pullPolicy` | "IfNotPresent" | `string` - 覆盖 auditLog é•œåƒçš„ imagePullPolicy - “Alwaysâ€ã€â€œNeverâ€ã€â€œIfNotPresentâ€ã€‚ | +| `busyboxImage` | "" | `string` - 用于收集审计日志的 busybox é•œåƒä½ç½®ã€‚_注æ„：此选项已弃用，请使用 `auditLog.image.repository` æ¥æŽ§åˆ¶å®¡è®¡ sidecar é•œåƒ_。 | +| `certmanager.version` | "" | `string` - 设置 cert-manager compatibility | +| `debug` | false | `bool` - 在 Rancher Server 设置 debug å‚æ•° | +| `extraEnv` | [] | `list` - 为 Rancher é¢å¤–设置环境å˜é‡ | +| `imagePullSecrets` | [] | `list` - ç§æœ‰é•œåƒä»“库凭è¯çš„密文å称列表 | +| `ingress.configurationSnippet` | "" | `string` - 添加é¢å¤–çš„ Nginx é…置。å¯ç”¨äºŽä»£ç†é…置。 | +| `ingress.extraAnnotations` | {} | `map` - 用于自定义 Ingress çš„é¢å¤–注释 | +| `ingress.enabled` | true | 如果值为 false，Helm ä¸ä¼šå®‰è£… Rancher Ingress。你å¯æŠŠå€¼è®¾ä¸º false 以部署你自己的 Ingress。 | +| `letsEncrypt.ingress.class` | "" | `string` - cert-manager acmesolver ingress çš„å¯é€‰ ingress 类，用于å“应 Let's Encrypt ACME 质询。选项：traefik，nginx。 | | +| `noProxy` | "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local,cattle-system.svc" | `string` - ä¸ä½¿ç”¨ä»£ç†çš„主机å或 IP 地å€çš„逗å·åˆ†éš”列表 | | +| `proxy` | "" | `string` - ç»™ Rancher é…置的 HTTP[S] ä»£ç† | +| `rancherImage` | "rancher/rancher" | `string` - Rancher é•œåƒæº | +| `rancherImagePullPolicy` | "IfNotPresent" | `string` - 覆盖 Rancher Server é•œåƒçš„ imagePullPolicy - "Always", "Never", "IfNotPresent" | +| `rancherImageTag` | å’Œ Chart 版本一致 | `string` - rancher/rancher é•œåƒæ ‡ç­¾ | +| `replicas` | 3 | `int` - Rancher Server 副本数。如果设为 -1，会根æ®é›†ç¾¤ä¸­çš„å¯ç”¨èŠ‚点数自动选择 1，2或3。 | +| `resources` | {} | `map` - Rancher Pod 资æºè¯·æ±‚å’Œé™åˆ¶ | +| `restrictedAdmin` | `false` | `bool` - 如果值为 true，åˆå§‹çš„ Rancher 用户访问本地 Kubernetes 集群会å—到é™åˆ¶ï¼Œä»¥é¿å…æƒé™å‡çº§ã€‚详情请å‚è§ [restricted-admin 角色](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#å—é™ç®¡ç†å‘˜)。 | +| `systemDefaultRegistry` | "" | `string` - 用于所有系统容器镜åƒçš„ç§æœ‰ä»“库，例如 http://registry.example.com/ | +| `tls` | "ingress" | `string` - 详情请å‚è§[外部 TLS 终止](#外部-tls-终止)。- "ingress, external" | +| `useBundledSystemChart` | `false` | `bool` - 选择 Rancher Server 打包的 system-charts。此å‚数用于离线环境安装。 | +| `global.cattle.psp.enabled` | `true` | `bool` - 使用 Rancher v2.7.2-v2.7.4 时，选择 `false` 以ç¦ç”¨ Kubernetes v1.25 åŠæ›´é«˜ç‰ˆæœ¬çš„ PSP。使用 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ—¶ï¼ŒRancher 会å°è¯•æ£€æµ‹é›†ç¾¤æ˜¯å¦è¿è¡Œä¸æ”¯æŒ PSP çš„ Kubernetes 版本，如果确定集群ä¸æ”¯æŒ PSP，则将默认 PSP 的使用设置为 false。你ä»ç„¶å¯ä»¥é€šè¿‡æ˜¾å¼æ供此值的 `true` 或 `false` æ¥æ‰‹åŠ¨è¦†ç›–æ­¤å€¼ã€‚åœ¨æ”¯æŒ PSP 的集群中（例如使用 Kubernetes v1.24 或更低版本的集群），Rancher ä»å°†é»˜è®¤ä½¿ç”¨ PSP。 | + + +### å¼•å¯¼å¯†ç  + +Rancher 首次å¯åŠ¨æ—¶ï¼Œä¼šä¸ºç¬¬ä¸€ä¸ªç®¡ç†å‘˜ç”¨æˆ·éšæœºç”Ÿæˆä¸€ä¸ªå¯†ç ã€‚当管ç†å‘˜é¦–次登录 Rancher 时，用于获å–引导密ç ï¼ˆBootstrap）的命令会在 UI 上显示。管ç†å‘˜éœ€è¦è¿è¡Œå‘½ä»¤å¹¶ä½¿ç”¨å¼•å¯¼å¯†ç ç™»å½•ã€‚ç„¶åŽ Rancher 会让管ç†å‘˜é‡ç½®å¯†ç ã€‚ + +如果你想指定引导密ç è€Œä¸ä½¿ç”¨éšæœºç”Ÿæˆçš„密ç ï¼Œè¯·å‚考以下命令设置密ç ã€‚ + +```plain +--set bootstrapPassword="rancher" +``` + +无论你是使用æ供的密ç è¿˜æ˜¯ç”Ÿæˆçš„密ç ï¼Œå¯†ç å‡å­˜å‚¨åœ¨ Kubernetes 密文中。安装 Rancher åŽï¼Œå¦‚何使用 kubectl 获å–密ç çš„说明将会在 UI 中显示： + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### API 审计日志 + +å¯ç”¨ [API 审计日志](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md)。 + +ä½ å¯ä»¥åƒæ”¶é›†å…¶ä»–容器日志一样收集此日志。在 Rancher Server 集群上为 `System` 项目å¯ç”¨ [Logging](../../../pages-for-subheaders/logging.md)。 + +```plain +--set auditLog.level=1 +``` + +默认情况下，å¯ç”¨å®¡è®¡æ—¥å¿—会在 Rancher pod 中创建一个 Sidecar 容器。这个容器（`rancher-audit-log`）会把日志æµä¼ è¾“到 `stdout`。你å¯ä»¥åƒæ”¶é›†å…¶ä»–容器日志一样收集此日志。如果你使用 Sidecar 作为审计日志的目标时， `hostPath`，`maxAge`，`maxBackups` å’Œ `maxSize` 选项ä¸ä¼šç”Ÿæ•ˆã€‚建议使用你的æ“作系统或 Docker Daemon 的日志轮æ¢åŠŸèƒ½æ¥æŽ§åˆ¶ç£ç›˜ç©ºé—´çš„使用。请为 Rancher Server 集群或 System 项目å¯ç”¨ [Logging](../../../pages-for-subheaders/logging.md)。 + +å°† `auditLog.destination` 的值设为 `hostPath`，å¯ä»¥å°†æ—¥å¿—转å‘到与主机系统共享的å·ï¼Œè€Œä¸æ˜¯ä¼ è¾“到 Sidecar 容器。如果目标设置为 `hostPath`，你å¯èƒ½éœ€è¦è°ƒæ•´å…¶ä»– auditLog å‚数以进行日志轮æ¢ã€‚ + +### é¢å¤–设置环境å˜é‡ + +ä½ å¯ä»¥ä½¿ç”¨ `extraEnv` 为 Rancher Server é¢å¤–设置环境å˜é‡ã€‚该列表以 YAML æ ¼å¼ä¼ é€’ç»™ Rancher 部署，它嵌入在 Rancher 容器的 `env` 下。你å¯ä»¥å‚考 Kubernetes 文档设置容器环境å˜é‡ã€‚`extraEnv` å¯ä»¥ä½¿ç”¨ [Define Environment Variables for a Container](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container) 中引用的任何键。 + +使用 `name` å’Œ `value` 键的示例： + +```plain +--set 'extraEnv[0].name=CATTLE_TLS_MIN_VERSION' +--set 'extraEnv[0].value=1.0' +``` + +如果将æ•æ„Ÿæ•°æ®ï¼ˆä¾‹å¦‚代ç†è®¤è¯å‡­è¯ï¼‰ä½œä¸ºçŽ¯å¢ƒå˜é‡çš„值传递，则强烈建议使用 Secret 引用。这将防止æ•æ„Ÿæ•°æ®åœ¨ Helm 或 Rancher 部署中暴露。 + +ä½ å¯ä»¥å‚考使用 `name`ã€`valueFrom.secretKeyRef.name` å’Œ `valueFrom.secretKeyRef.key` é”®çš„ç¤ºä¾‹ã€‚è¯¦è§ [HTTP 代ç†](#http-代ç†)中的示例。 + +### TLS 设置 + +当你在 Kubernetes 集群内安装 Rancher 时，TLS 会在集群的 Ingress Controller 上å¸è½½ã€‚支æŒçš„ TLS 设置å–决于使用的 Ingress Controller。 + +å‚è§ [TLS 设置](tls-settings.md)了解更多信æ¯å’Œé€‰é¡¹ã€‚ + +### 导入 `local` 集群 + +默认情况下，Rancher Server 会检测并导入其所在的 `local` 集群。有æƒè®¿é—® `local` 集群的用户对 Rancher Server 管ç†çš„所有集群具有“rootâ€è®¿é—®æƒé™ã€‚ + +:::caution + +如果你关闭 addLocal，大多数 Rancher 2.5 功能都ä¸èƒ½ä½¿ç”¨ï¼ŒåŒ…括 EKS Provisioner。 + +::: + +如果这在你的环境中是一个问题，你å¯ä»¥åœ¨åˆå§‹å®‰è£…时将此选项设置为“falseâ€ã€‚ + +此选项仅在首次安装 Rancher 时有效。详情请å‚è§ [Issue 16522](https://github.com/rancher/rancher/issues/16522)。 + +```plain +--set addLocal="false" +``` + +### 自定义 Ingress + +è¦è‡ªå®šä¹‰æˆ–使用 Rancher Server 的其他 Ingress，你å¯ä»¥è®¾ç½®è‡ªå·±çš„ Ingress 注释。 + +设置自定义è¯ä¹¦é¢å‘者的示例： + +```plain +--set ingress.extraAnnotations.'cert-manager\.io/cluster-issuer'=issuer-name +``` + +以下是使用 `ingress.configurationSnippet`设置é™æ€ä»£ç†æ ‡å¤´çš„示例。该值åƒæ¨¡æ¿ä¸€æ ·è¿›è¡Œè§£æžï¼Œå› æ­¤å¯ä»¥ä½¿ç”¨å˜é‡ã€‚ + +```plain +--set ingress.configurationSnippet='more_set_input_headers X-Forwarded-Host {{ .Values.hostname }};' +``` + +### HTTP ä»£ç† + +Rancher 的一些功能（Helm Chart）需è¦ä½¿ç”¨äº’è”网æ‰èƒ½ä½¿ç”¨ã€‚ä½ å¯ä»¥ä½¿ç”¨ `proxy` 设置代ç†æœåŠ¡å™¨ï¼Œæˆ–使用 `extraEnv` 设置 `HTTPS_PROXY` 环境å˜é‡æ¥æŒ‡å‘代ç†æœåŠ¡å™¨ã€‚ + +å°†è¦æŽ’除的 IP 使用逗å·åˆ†éš”列表添加到 `noProxy` Chart value 中。确ä¿æ·»åŠ äº†ä»¥ä¸‹å€¼ï¼š +- Pod 集群 IP 范围（默认值：`10.42.0.0/16`）。 +- Service Cluster IP 范围（默认值：`10.43.0.0/16`）。 +- 内部集群域（默认值：`.svc,.cluster.local`）。 +- 任何 Worker 集群 `controlplane` 节点。 + Rancher 支æŒåœ¨æ­¤åˆ—表中使用 CIDR 表示法æ¥è¡¨ç¤ºèŒƒå›´ã€‚ + +ä¸åŒ…括æ•æ„Ÿæ•°æ®æ—¶ï¼Œå¯ä»¥ä½¿ç”¨ `proxy` 或 `extraEnv` Chart 选项。使用 `extraEnv` 时将忽略 `noProxy` Helm 选项。因此，`NO_PROXY` 环境å˜é‡ä¹Ÿå¿…须设置为 `extraEnv`。 + +以下是使用 `extraEnv` Chart 选项设置代ç†çš„示例： + +```plain +--set proxy="http:///" +``` + +使用 `extraEnv` Chart 选项设置代ç†çš„示例： +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].value="http://:/" +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +包å«æ•æ„Ÿæ•°æ®ï¼ˆä¾‹å¦‚代ç†è®¤è¯å‡­è¯ï¼‰æ—¶ï¼Œè¯·ä½¿ç”¨ `extraEnv` 选项和 `valueFrom.secretRef` æ¥é˜²æ­¢æ•æ„Ÿæ•°æ®åœ¨ Helm 或 Rancher 部署中暴露。 + +下é¢æ˜¯ä½¿ç”¨ `extraEnv` é…置代ç†çš„示例。此示例 Secret 在 Secret çš„ `"https-proxy-url"` é”®ä¸­åŒ…å« `"http://:@:/"` 值： +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].valueFrom.secretKeyRef.name=secret-name +--set extraEnv[1].valueFrom.secretKeyRef.key=https-proxy-url +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +有关如何é…置环境å˜é‡çš„更多信æ¯ï¼Œè¯·å‚阅[为容器定义环境å˜é‡](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container)。 + +### é¢å¤–的授信 CA + +如果你有ç§æœ‰é•œåƒä»“库（registries）ã€åº”用商店（catalogs）或拦截è¯ä¹¦çš„代ç†ï¼Œåˆ™å¯èƒ½éœ€è¦å‘ Rancher 添加é¢å¤–的授信 CA。 + +```plain +--set additionalTrustedCAs=true +``` + +创建完 Rancher deployment åŽï¼Œå°† pem æ ¼å¼çš„ CA è¯ä¹¦å¤åˆ¶åˆ°ä¸€ä¸ªå为 `ca-additional.pem` 的文件中，并使用 `kubectl` 在 `cattle-system` 命å空间中创建 `tls-ca-additional` 密文。 + +```plain +kubectl -n cattle-system create secret generic tls-ca-additional --from-file=ca-additional.pem=./ca-additional.pem +``` + +### ç§æœ‰ä»“库和离线安装 + +有关使用ç§æœ‰ä»“库安装 Rancher 的详情，请å‚è§[离线安装](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)。 + +## 外部 TLS 终止 + +我们建议将负载å‡è¡¡å™¨é…置为 4 层å‡è¡¡ï¼Œå°†æ™®é€š 80/tcp å’Œ 443/tcp 转å‘到 Rancher Management 集群节点。集群上的 Ingress Controller ä¼šå°†ç«¯å£ 80 上的 HTTP æµé‡é‡å®šå‘åˆ°ç«¯å£ 443 上的 HTTPS。 + +ä½ å¯ä»¥åœ¨ Rancher 集群（Ingress）外部的 L7 è´Ÿè½½å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS。使用 `--set tls=external` 选项，将负载å‡è¡¡å™¨æŒ‡å‘所有 Rancher é›†ç¾¤èŠ‚ç‚¹ä¸Šçš„ç«¯å£ HTTP 80。这将在 HTTP ç«¯å£ 80 上暴露 Rancher 接å£ã€‚请注æ„，å…许直接连接到 Rancher 集群的客户端ä¸ä¼šè¢«åŠ å¯†ã€‚如果你选择这样åšï¼Œæˆ‘们建议你将网络级别的直接访问é™åˆ¶ä¸ºä»…ä½ çš„è´Ÿè½½å‡è¡¡å™¨ã€‚ + +:::note + +如果你使用的是ç§æœ‰ CA ç­¾åçš„è¯ä¹¦ï¼Œè¯·æ·»åŠ  `--set privateCA=true` 并å‚è§[添加 TLS 密文 - 使用ç§æœ‰ CA ç­¾åè¯ä¹¦](../../../getting-started/installation-and-upgrade/resources/add-tls-secrets.md)，为 Rancher 添加 CA è¯ä¹¦ã€‚ + +::: + +ä½ çš„è´Ÿè½½å‡è¡¡å™¨å¿…须支æŒé•¿æœŸå­˜åœ¨çš„ Websocket 连接，并且需è¦æ’入代ç†å¤´ï¼Œä»¥ä¾¿ Rancher å¯ä»¥æ­£ç¡®ä¼ é€é“¾æŽ¥ã€‚ + +### 使用 NGINX v0.25 为外部 TLS é…ç½® Ingress + +在 NGINX 0.25 中，NGINX 关于转å‘头和外部 TLS 终止的行为[已更改](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0220)。因此，如果你åŒæ—¶ä½¿ç”¨ NGINX 0.25 和外部 TLS 终止é…置，你必须编辑 `cluster.yml` æ¥ä¸º Ingress å¯ç”¨ `use-forwarded-headers` 选项。 + +```yaml +ingress: + provider: nginx + options: + use-forwarded-headers: 'true' +``` + +### 必须的 Header + +- `Host` +- `X-Forwarded-Proto` +- `X-Forwarded-Port` +- `X-Forwarded-For` + +### 建议的超时时间 + +- 读超时：`1800 seconds` +- 写超时：`1800 seconds` +- 连接超时：`30 seconds` + +### å¥åº·æ£€æŸ¥ + +Rancher 将对 `/healthz` 端点的å¥åº·æ£€æŸ¥å“应`200`。 + +### 示例 NGINX é…ç½® + +æ­¤ NGINX é…置已在 NGINX 1.14 上进行了测试。 + +:::caution + +æ­¤ NGINX é…ç½®åªæ˜¯ä¸€ä¸ªç¤ºä¾‹ï¼Œå¯èƒ½ä¸é€‚åˆä½ çš„环境。如需查阅完整文档，请å‚è§ [NGINX è´Ÿè½½å‡è¡¡ - HTTP è´Ÿè½½å‡è¡¡](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/)。 + +::: + +- å°† `IP_NODE1`，`IP_NODE2` å’Œ `IP_NODE3` 替æ¢ä¸ºä½ é›†ç¾¤ä¸­èŠ‚点的 IP 地å€ã€‚ +- 将两处的 `FQDN` å‡æ›¿æ¢ä¸º Rancher çš„ DNS å称。 +- 把 `/certs/fullchain.pem` å’Œ `/certs/privkey.pem` 分别替æ¢ä¸ºæœåŠ¡å™¨è¯ä¹¦å’ŒæœåŠ¡å™¨è¯ä¹¦å¯†é’¥çš„ä½ç½®ã€‚ + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server IP_NODE_1:80; + server IP_NODE_2:80; + server IP_NODE_3:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # 此项å…许执行的 shell 窗å£ä¿æŒå¼€å¯ï¼Œæœ€é•¿å¯è¾¾15分钟。ä¸ä½¿ç”¨æ­¤å‚æ•°çš„è¯ï¼Œé»˜è®¤1分钟åŽè‡ªåŠ¨å…³é—­ã€‚ + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/tls-settings.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/tls-settings.md new file mode 100644 index 00000000000..25f5652fe6e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-references/tls-settings.md @@ -0,0 +1,21 @@ +--- +title: TLS 设置 +--- + +更改默认 TLS 设置的方法å–决于它的安装方å¼ã€‚ + +## 在高å¯ç”¨ Kubernetes 集群中è¿è¡Œ Rancher + +当你在 Kubernetes 集群内安装 Rancher 时，TLS 会在集群的 Ingress Controller 上å¸è½½ã€‚å¯ç”¨çš„ TLS 设置å–决于使用的 Ingress Controller： + +* nginx-ingress-controller（RKE1 å’Œ RKE2 默认）：[默认的 TLS 版本和密ç ](https://kubernetes.github.io/ingress-nginx/user-guide/tls/#default-tls-version-and-ciphers)。 +* traefik（K3s 默认）：[TLS 选项](https://doc.traefik.io/traefik/https/tls/#tls-options)。 + +## 在å•ä¸ª Docker 容器中è¿è¡Œ Rancher + +默认 TLS é…ç½®ä»…æ”¯æŒ TLS 1.2 和安全的 TLS 密ç å¥—件。你å¯ä»¥é€šè¿‡è®¾ç½®ä»¥ä¸‹çŽ¯å¢ƒå˜é‡æ¥æ›´æ”¹æ­¤é…置： + +| å‚æ•° | æè¿° | 默认 | å¯ç”¨é€‰é¡¹ | +|-----|-----|-----|-----| +| `CATTLE_TLS_MIN_VERSION` | æœ€å° TLS 版本 | `1.2` | `1.0`, `1.1`, `1.2`, `1.3` | +| `CATTLE_TLS_CIPHERS` | 支æŒçš„ TLS 密ç å¥—件 | `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305`,
`TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305` | 详情请å‚è§ [Golang TLS 常é‡](https://golang.org/pkg/crypto/tls/#pkg-constants)。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/dockershim.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/dockershim.md new file mode 100644 index 00000000000..84f2383eaf7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/dockershim.md @@ -0,0 +1,47 @@ +--- +title: Dockershim +--- + +Dockershim 是 Kubelet å’Œ Docker Daemon 之间的 CRI 兼容层。Kubernetes 1.20 版本宣布了[移除树内 Dockershim](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/)。有关此移除的更多信æ¯ä»¥åŠæ—¶é—´çº¿ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。 + +RKE 集群现在支æŒå¤–部 Dockershim，æ¥è®©ç”¨æˆ·ç»§ç»­ä½¿ç”¨ Docker 作为 CRI è¿è¡Œæ—¶ã€‚现在，我们通过使用 [Mirantis å’Œ Docker ](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) æ¥ç¡®ä¿ RKE 集群å¯ä»¥ç»§ç»­ä½¿ç”¨ Docker，从而实现上游开æºç¤¾åŒºçš„外部 Dockershim。 + +RKE2 å’Œ K3s 集群使用嵌入的 containerd 作为容器è¿è¡Œæ—¶ï¼Œå› æ­¤ä¸å—å½±å“。 + +è¦åœ¨ 1.24 之å‰çš„ RKE 版本中å¯ç”¨å¤–部 Dockershim，请é…置以下选项： + +``` +enable_cri_dockerd: true +``` + +从 1.24 版本开始，以上默认为 true。 + +如果你想使用其他容器è¿è¡Œæ—¶ï¼ŒRancher 也æ供使用 Containerd 作为默认è¿è¡Œæ—¶çš„，以边缘为中心的 K3s，和以数æ®ä¸­å¿ƒä¸ºä¸­å¿ƒçš„ RKE2 Kubernetes å‘行版。然åŽï¼Œä½ å°±å¯ä»¥é€šè¿‡ Rancher 对导入的 RKE2 å’Œ K3s Kubernetes 集群进行å‡çº§å’Œç®¡ç†ã€‚ + +### 常è§é—®é¢˜ + +
+ +Q：是å¦å¿…é¡»å‡çº§ Rancher æ‰èƒ½èŽ·å¾— Rancher 对上游外部 Dockershim 替æ¢çš„支æŒï¼Ÿ + +A：对于 RKE，Dockershim `cri_dockerd` 替æ¢çš„上游支æŒä»Ž Kubernetes 1.21 开始。你需è¦ä½¿ç”¨æ”¯æŒ RKE 1.21 çš„ Rancher 版本。详情请å‚è§æˆ‘们的支æŒçŸ©é˜µã€‚ + +
+ +Q：我目å‰çš„ RKE 使用 Kubernetes 1.23。如果上游最终在 1.24 中删除 Dockershim，会å‘生什么？ + +A：RKE 中带有 Kubernetes çš„ Dockershim 版本将继续工作到 1.23。有关时间线的更多信æ¯ï¼Œè¯·å‚è§ [Kubernetes Dockershim 弃用相关的常è§é—®é¢˜](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed)。从 1.24 开始，RKE 将默认å¯ç”¨ `cri_dockerd` 并在之åŽçš„版本中继续å¯ç”¨ã€‚ + +
+ +Q: 如果我ä¸æƒ³å†ä¾èµ– Dockershim 或 cri_dockerd，我还有什么选择？ + +A: ä½ å¯ä»¥ä¸º Kubernetes 使用ä¸éœ€è¦ Dockershim 支æŒçš„è¿è¡Œæ—¶ï¼Œå¦‚ Containerd。RKE2 å’Œ K3s 就是其中的两个选项。 + +
+ +Q: 如果我目å‰ä½¿ç”¨ RKE1，但想切æ¢åˆ° RKE2，我å¯ä»¥æ€Žæ ·è¿›è¡Œè¿ç§»ï¼Ÿ + +A: ä½ å¯ä»¥æž„建一个新集群，然åŽå°†å·¥ä½œè´Ÿè½½è¿ç§»åˆ°ä½¿ç”¨ Containerd çš„æ–° RKE2 集群。Rancher 也在探索就地å‡çº§è·¯å¾„çš„å¯èƒ½æ€§ã€‚ + +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/install-docker.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/install-docker.md new file mode 100644 index 00000000000..8b98ce09030 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/install-docker.md @@ -0,0 +1,23 @@ +--- +title: 安装 Docker +--- + +在使用 Helm 或 Docker 在 RKE 集群节点上安装 Rancher Server å‰ï¼Œä½ éœ€è¦å…ˆå®‰è£… Docker。RKE2 å’Œ K3s 集群ä¸è¦æ±‚使用 Docker。 + +Docker 有几个安装方法。一ç§æ–¹æ³•æ˜¯å‚è§ [Docker 官方文档](https://docs.docker.com/install/)以了解如何在 Linux 上安装 Docker。ä¸åŒ Linux å‘行版的安装步骤å¯èƒ½æœ‰æ‰€ä¸åŒã€‚ + +å¦ä¸€ç§æ–¹å¼æ˜¯ä½¿ç”¨ Rancher çš„ Docker 安装脚本，该脚本å¯ç”¨äºŽè¾ƒæ–°çš„ Docker 版本。 + +例如，此命令å¯ç”¨äºŽåœ¨ SUSE Linux Enterprise 或 Ubuntu ç­‰ä¸»è¦ Linux å‘行版上安装 Docker 20.10： + +``` +curl https://releases.rancher.com/install-docker/20.10.sh | sh +``` + +Rancher æä¾› Kubernetes 支æŒçš„所有上游 Docker 版本的安装脚本。如需了解我们是å¦æä¾›æŸä¸ª Docker 版本的安装脚本，请å‚è§åŒ…å«äº† Rancher 所有的 Docker 安装脚本的 [GitHub 仓库](https://github.com/rancher/install-docker)。 + +请注æ„，必须应用以下 sysctl 设置： + +``` +net.bridge.bridge-nf-call-iptables=1 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md new file mode 100644 index 00000000000..7dcdc48683c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md @@ -0,0 +1,343 @@ +--- +title: 端å£è¦æ±‚ +description: 了解 Rancher 正常è¿è¡Œæ‰€éœ€çš„端å£è¦æ±‚，包括 Rancher 节点和下游 Kubernetes 集群节点 +--- + +import PortsIaasNodes from '@site/src/components/PortsIaasNodes' +import PortsCustomNodes from '@site/src/components/PortsCustomNodes' +import PortsImportedHosted from '@site/src/components/PortsImportedHosted' + +为了确ä¿èƒ½æ­£å¸¸è¿è¡Œï¼ŒRancher 需è¦åœ¨ Rancher 节点和下游 Kubernetes 集群节点上开放一些端å£ã€‚ + +## Rancher 节点 + +下表列出了è¿è¡Œ Rancher Server 的节点之间需è¦å¼€æ”¾çš„端å£ã€‚ + +ä¸åŒçš„ Rancher Server 架构有ä¸åŒçš„端å£è¦æ±‚。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上。如果你的 Rancher 安装在 K3sã€RKE 或 RKE2 Kubernetes 集群上，请å‚考下é¢çš„标签页。对于其他 Kubernetes å‘行版，请å‚è§è¯¥å‘行版的文档，了解集群节点的端å£è¦æ±‚。 + +:::note 注æ„事项： + +- Rancher 节点å¯èƒ½è¦æ±‚é¢å¤–出站访问已é…置的外部验è¯æ供程åºï¼ˆå¦‚ LDAP）。 +- Kubernetes 建议节点端å£æœåŠ¡ä½¿ç”¨ TCP 30000-32767。 +- 对于防ç«å¢™ï¼Œå¯èƒ½éœ€è¦åœ¨é›†ç¾¤å’Œ Pod CIDR 内å¯ç”¨æµé‡ã€‚ +- Rancher 节点å¯èƒ½è¿˜éœ€è¦å‡ºç«™è®¿é—®ç”¨äºŽå­˜å‚¨é›†ç¾¤å¤‡ä»½ï¼ˆå¦‚ Minio）的外部 S3 上的ä½ç½®ã€‚ + +::: + +### K3s 上 Rancher Server èŠ‚ç‚¹çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +K3s server 需è¦å¼€æ”¾ç«¯å£ 6443 æ‰èƒ½ä¾›èŠ‚点访问。 + +使用 Flannel VXLAN 时，节点需è¦èƒ½å¤Ÿé€šè¿‡ UDP ç«¯å£ 8472 访问其他节点。节点ä¸åº”监å¬ä»»ä½•å…¶ä»–端å£ã€‚K3s 使用åå‘隧é“，建立节点与 Server 的出站连接，所有 kubelet æµé‡éƒ½é€šè¿‡è¯¥éš§é“进行。但是，如果你ä¸ä½¿ç”¨ Flannel，而是使用自定义的 CNI，K3s 则ä¸éœ€è¦æ‰“å¼€ 8472 端å£ã€‚ + +如果è¦ä½¿ç”¨ Metrics Server，则需è¦åœ¨æ¯ä¸ªèŠ‚ç‚¹ä¸Šæ‰“å¼€ç«¯å£ 10250。 + +:::note é‡è¦æ示： + +节点上的 VXLAN 端å£ä¼šå¼€æ”¾é›†ç¾¤ç½‘络，让任何人å‡èƒ½è®¿é—®é›†ç¾¤ã€‚因此，ä¸è¦å°† VXLAN 端å£æš´éœ²ç»™å¤–界。请使用ç¦ç”¨ 8472 端å£çš„防ç«å¢™/安全组æ¥è¿è¡ŒèŠ‚点。 + +::: + +下表æ述了入站和出站æµé‡çš„端å£è¦æ±‚： + +
Rancher Server 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 80 | 执行外部 SSL 终止的负载å‡è¡¡å™¨/ä»£ç† | 使用外部 SSL 终止时的 Rancher UI/API | +| TCP | 443 |
  • Server 节点
  • Agent 节点
  • 托管/注册的 Kubernetes
  • 任何需è¦ä½¿ç”¨ Rancher UI 或 API çš„æº
| Rancher Agent，Rancher UI/API，kubectl | +| TCP | 6443 | K3s Server 节点 | Kubernetes API | +| UDP | 8472 | K3s Server å’Œ Agent 节点 | ä»… Flannel VXLAN éœ€è¦ | +| TCP | 10250 | K3s Server å’Œ Agent 节点 | kubelet | + +
Rancher 节点的出站规则
+ +| åè®® | ç«¯å£ | 目标 | æè¿° | +| -------- | ---- | -------------------------------------------------------- | --------------------------------------------- | +| TCP | 22 | 使用 Node Driver 创建的节点的任何节点 IP | 使用 Node Driver SSH é…置节点 | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | 使用 Node Driver 创建的节点的任何节点 IP | Docker Machine 使用的 Docker daemon TLS ç«¯å£ | +| TCP | 6443 | 托管/导入的 Kubernetes API | Kubernetes API Server | + +
+ +### RKE 上 Rancher Server èŠ‚ç‚¹çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +通常情况下，Rancher 安装在三个 RKE 节点上，这些节点都有 etcdã€controlplane å’Œ worker 角色。 + + + +下表æ述了 Rancher 节点之间æµé‡çš„端å£è¦æ±‚： + +
Rancher 节点的æµé‡è§„则
+ +| åè®® | ç«¯å£ | æè¿° | +|-----|-----|----------------| +| TCP | 443 | Rancher Agents | +| TCP | 2379 | etcd 客户端请求 | +| TCP | 2380 | etcd 对等通信 | +| TCP | 6443 | Kubernetes apiserver | +| TCP | 8443 | NGINX Ingress çš„éªŒè¯ Webhook | +| UDP | 8472 | Canal/Flannel VXLAN 覆盖网络 | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 10250 | Metrics Server 与所有节点的通信 | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | + +下表æ述了入站和出站æµé‡çš„端å£è¦æ±‚： + +
Rancher 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 22 | RKE CLI | RKE 通过 SSH é…置节点 | +| TCP | 80 | è´Ÿè½½å‡è¡¡å™¨/åå‘ä»£ç† | 到 Rancher UI/API çš„ HTTP æµé‡ | +| TCP | 443 |
  • è´Ÿè½½å‡è¡¡å™¨/åå‘代ç†
  • 所有集群节点和其他 API/UI 客户端的 IP
| 到 Rancher UI/API çš„ HTTPS æµé‡ | +| TCP | 6443 | Kubernetes API 客户端 | 到 Kubernetes API çš„ HTTPS æµé‡ | + +
Rancher 节点的出站规则
+ +| åè®® | ç«¯å£ | 目标 | æè¿° | +|-----|-----|----------------|---| +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 22 | 使用 Node Driver 创建的任何节点 | Node Driver 通过 SSH é…置节点 | +| TCP | 2376 | 使用 Node Driver 创建的任何节点 | Node Driver 使用的 Docker daemon TLS ç«¯å£ | +| TCP | 6443 | 托管/导入的 Kubernetes API | Kubernetes API Server | +| TCP | æ供商ä¾èµ– | 托管集群中 Kubernetes API ç«¯ç‚¹çš„ç«¯å£ | Kubernetes API | + +
+ +### RKE2 上 Rancher Server èŠ‚ç‚¹çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +RKE2 server 需è¦å¼€æ”¾ç«¯å£ 6443 å’Œ 9345 æ‰èƒ½ä¾›é›†ç¾¤ä¸­çš„其他节点访问。 + +使用 Flannel VXLAN 时，所有节点都需è¦èƒ½å¤Ÿé€šè¿‡ UDP ç«¯å£ 8472 访问其他节点。 + +如果è¦ä½¿ç”¨ Metrics Server，则需è¦åœ¨æ¯ä¸ªèŠ‚ç‚¹ä¸Šæ‰“å¼€ç«¯å£ 10250。 + +:::note é‡è¦æ示： + +节点上的 VXLAN 端å£ä¼šå¼€æ”¾é›†ç¾¤ç½‘络，让任何人å‡èƒ½è®¿é—®é›†ç¾¤ã€‚因此，ä¸è¦å°† VXLAN 端å£æš´éœ²ç»™å¤–界。请使用ç¦ç”¨ 8472 端å£çš„防ç«å¢™/安全组æ¥è¿è¡ŒèŠ‚点。 + +::: + +
RKE2 Server 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 9345 | RKE2 Server å’Œ Agent 节点 | 节点注册。需è¦åœ¨æ‰€æœ‰ Server 节点上将端å£å¼€æ”¾ç»™é›†ç¾¤ä¸­çš„所有其他节点。 | +| TCP | 6443 | RKE2 Agent 节点 | Kubernetes API | +| UDP | 8472 | RKE2 Server å’Œ Agent 节点 | ä»… Flannel VXLAN éœ€è¦ | +| TCP | 10250 | RKE2 Server å’Œ Agent 节点 | kubelet | +| TCP | 2379 | RKE2 Server 节点 | etcd å®¢æˆ·ç«¯ç«¯å£ | +| TCP | 2380 | RKE2 Server 节点 | etcd å¯¹ç­‰ç«¯å£ | +| TCP | 30000-32767 | RKE2 Server å’Œ Agent 节点 | NodePort 端å£èŒƒå›´ã€‚å¯ä»¥ä½¿ç”¨ TCP 或 UDP。 | +| TCP | 5473 | Calico-node pod 连接到 typha pod | 使用 Calico éƒ¨ç½²æ—¶éœ€è¦ | +| HTTP | 80 | 执行外部 SSL 终止的负载å‡è¡¡å™¨/ä»£ç† | 使用外部 SSL 终止时的 Rancher UI/API | +| HTTPS | 443 |
  • 托管/注册的 Kubernetes
  • 任何需è¦ä½¿ç”¨ Rancher UI 或 API çš„æº
| Rancher Agent，Rancher UI/API，kubectl。如果负载å‡è¡¡å™¨æ‰§è¡Œ TLS 终止，则ä¸éœ€è¦ã€‚ | + +所有出站æµé‡é€šå¸¸éƒ½æ˜¯å…许的。 +
+ +### Docker 安装的 Rancher Server çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了 Rancher 节点入站和出站æµé‡çš„端å£è¦æ±‚： + +
Rancher 节点的入站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 80 | 执行外部 SSL 终止的负载å‡è¡¡å™¨/ä»£ç† | 使用外部 SSL 终止时的 Rancher UI/API | +| TCP | 443 |
  • 托管/注册的 Kubernetes
  • 任何需è¦ä½¿ç”¨ Rancher UI 或 API çš„æº
| Rancher Agent，Rancher UI/API，kubectl | + +
Rancher 节点的出站规则
+ +| åè®® | ç«¯å£ | æº | æè¿° | +|-----|-----|----------------|---| +| TCP | 22 | 使用 Node Driver 创建的节点的任何节点 IP | 使用 Node Driver SSH é…置节点 | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | 使用 Node Driver 创建的节点的任何节点 IP | Docker Machine 使用的 Docker daemon TLS ç«¯å£ | +| TCP | 6443 | 托管/导入的 Kubernetes API | Kubernetes API Server | + +
+ +## 下游 Kubernetes 集群节点 + +下游 Kubernetes 集群用于è¿è¡Œä½ çš„应用和æœåŠ¡ã€‚本节介ç»äº†å“ªäº›ç«¯å£éœ€è¦åœ¨ä¸‹æ¸¸é›†ç¾¤çš„节点上打开，以便 Rancher 能够与它们进行通信。 + +ä¸åŒçš„下游集群的å¯åŠ¨æ–¹å¼æœ‰ä¸åŒçš„端å£è¦æ±‚。下é¢çš„æ¯ä¸ªæ ‡ç­¾éƒ½åˆ—出了ä¸åŒ[集群类型](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)所需打开的端å£ã€‚ + +下图æ述了为æ¯ä¸ª[集群类型](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)打开的端å£ã€‚ + +
Rancher 管ç†é¢æ¿çš„端å£è¦æ±‚
+ +![基本端å£è¦æ±‚](/img/port-communications.svg) + +:::tip + +如果你对安全性的关注ä¸æ˜¯å¤ªé«˜ï¼Œè€Œä¸”也愿æ„多打开几个端å£ï¼Œä½ å¯ä»¥å‚考[常用端å£](#常用端å£)中列出的端å£ï¼Œè€Œä¸æ˜¯å‚考下方的表格。 + +::: + +### Harvester é›†ç¾¤çš„ç«¯å£ + +有关 Harvester 端å£è¦æ±‚的更多信æ¯ï¼Œè¯·å‚阅[此处](../../../integrations-in-rancher/harvester.md#端å£è¦æ±‚)。 + + +### Rancher 使用节点池å¯åŠ¨ Kubernetes é›†ç¾¤çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了节点在[云æ供商](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)中创建的情况下，[Rancher å¯åŠ¨ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 的端å£è¦æ±‚。 + +:::note + +在 AWS EC2 或 DigitalOcean 等云æ供商中创建集群期间，Rancher 会自动打开所需的端å£ã€‚ + +::: + + + +
+ +### Rancher 使用自定义节点å¯åŠ¨ Kubernetes é›†ç¾¤çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了使用[自定义节点](../../../pages-for-subheaders/use-existing-nodes.md)的情况下，[Rancher å¯åŠ¨ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 的端å£è¦æ±‚。 + + + +
+ +### 托管 Kubernetes é›†ç¾¤çš„ç«¯å£ + +
+ å•å‡»å±•å¼€ + +下表æ述了[托管集群](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)的端å£è¦æ±‚。 + + + +
+ +### å·²æ³¨å†Œé›†ç¾¤çš„ç«¯å£ + +:::note + +在 Rancher 2.5 之å‰ï¼Œæ³¨å†Œé›†ç¾¤è¢«ç§°ä¸ºå¯¼å…¥é›†ç¾¤ã€‚ + +::: + +
+ å•å‡»å±•å¼€ + +下表æ述了[注册集群](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)的端å£è¦æ±‚。 + + + +
+ + +## 其他端å£æ³¨æ„事项 + +### å¸¸ç”¨ç«¯å£ + +无论集群是什么类型，常用端å£é€šå¸¸åœ¨ä½ çš„ Kubernetes 节点上打开。 + +import CommonPortsTable from '../../../shared-files/_common-ports-table.md'; + + + +---- + +### 本地节点æµé‡ + +上述è¦æ±‚中标记为`本地æµé‡`（例如 `9099 TCP`）的端å£ä¼šç”¨äºŽ Kubernetes å¥åº·æ£€æŸ¥ （`livenessProbe` å’Œ `readinessProbe`）。 +这些å¥åº·æ£€æŸ¥æ˜¯åœ¨èŠ‚点本身执行的。在大多数云环境中，这ç§æœ¬åœ°æµé‡æ˜¯é»˜è®¤å…许的。 + +但是，在以下情况下å¯èƒ½ä¼šé˜»æ­¢æ­¤æµé‡ï¼š + +- 你已在节点上应用了严格的主机防ç«å¢™ç­–略。 +- 你正在使用有多个接å£ï¼ˆå¤šå®¿ä¸»ï¼‰çš„节点。 + +在这些情况下，你必须在你的主机防ç«å¢™ä¸­ä¸»åŠ¨å…许这ç§æµé‡ï¼Œå¦‚果是公共/ç§æœ‰äº‘托管的主机（例如 AWS 或 OpenStack），你需è¦åœ¨ä½ çš„安全组é…置中主动å…许此æµé‡ã€‚请记ä½ï¼Œå¦‚果你在安全组中使用安全组作为æºæˆ–目标，主动开放端å£åªé€‚用于节点/实例的ç§æœ‰æŽ¥å£ã€‚ + +### Rancher AWS EC2 安全组 + +当你使用 [AWS EC2 Node Driver](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) 在 Rancher 中é…置集群节点时，你å¯ä»¥è®© Rancher 创建一个å为 `rancher-nodes` 的安全组。以下规则会自动添加到该安全组中。 + +| 类型 | åè®® | 端å£èŒƒå›´ | æº/目标 | 规则类型 | +|-----------------|:--------:|:-----------:|------------------------|:---------:| +| SSH | TCP | 22 | 0.0.0.0/0 | 入站 | +| HTTP | TCP | 80 | 0.0.0.0/0 | 入站 | +| 自定义 TCP 规则 | TCP | 443 | 0.0.0.0/0 | 入站 | +| 自定义 TCP 规则 | TCP | 2376 | 0.0.0.0/0 | 入站 | +| 自定义 TCP 规则 | TCP | 2379-2380 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 UDP 规则 | UDP | 4789 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 6443 | 0.0.0.0/0 | 入站 | +| 自定义 UDP 规则 | UDP | 8472 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 10250-10252 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 10256 | sg-xxx (rancher-nodes) | 入站 | +| 自定义 TCP 规则 | TCP | 30000-32767 | 0.0.0.0/0 | 入站 | +| 自定义 UDP 规则 | UDP | 30000-32767 | 0.0.0.0/0 | 入站 | +| 所有æµé‡ | 全部 | 全部 | 0.0.0.0/0 | 出站 | + +### 打开 SUSE Linux ç«¯å£ + +SUSE Linux å¯èƒ½æœ‰ä¸€ä¸ªé˜²ç«å¢™ï¼Œé»˜è®¤æƒ…况下会阻止所有端å£ã€‚è¦æ‰“开将主机添加到自定义集群所需的端å£ï¼š + + + + +1. SSH 进入实例。 +1. 以文本模å¼å¯åŠ¨ YaST： + ``` + sudo yast2 + ``` + +1. 导航到**安全和用户** > **防ç«å¢™** > **区域：公共** > **端å£**。è¦åœ¨ç•Œé¢å†…导航，请å‚ç…§[说明](https://doc.opensuse.org/documentation/leap/reference/html/book-reference/cha-yast-text.html#sec-yast-cli-navigate)。 +1. è¦æ‰“开所需的端å£ï¼ŒæŠŠå®ƒä»¬è¾“入到 **TCP 端å£** å’Œ **UDP 端å£** 字段。在这个例å­ä¸­ï¼Œç«¯å£ 9796 å’Œ 10250 也被打开，用于监控。由此产生的字段应类似于以下内容： + ```yaml + TCP Ports + 22, 80, 443, 2376, 2379, 2380, 6443, 9099, 9796, 10250, 10254, 30000-32767 + UDP Ports + 8472, 30000-32767 + ``` + +1. 所有必须端å£éƒ½è¾“å…¥åŽï¼Œé€‰æ‹©**接å—**。 + + + + +1. SSH 进入实例。 +1. 编辑 `/etc/sysconfig/SuSEfirewall2` 并打开所需的端å£ã€‚在这个例å­ä¸­ï¼Œç«¯å£ 9796 å’Œ 10250 也被打开，用于监控。 + ``` + FW_SERVICES_EXT_TCP="22 80 443 2376 2379 2380 6443 9099 9796 10250 10254 30000:32767" + FW_SERVICES_EXT_UDP="8472 30000:32767" + FW_ROUTE=yes + ``` +1. 用新的端å£é‡å¯é˜²ç«å¢™ï¼š + ``` + SuSEfirewall2 + ``` + + + + +**结果** ：该节点已打开添加到自定义集群所需的端å£ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md new file mode 100644 index 00000000000..a639349d8cf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md @@ -0,0 +1,149 @@ +--- +title: Docker 安装命令 +--- + +Docker 安装适用于想è¦æµ‹è¯• Rancher 的用户。 + +ä½ å¯ä»¥ä½¿ç”¨ `docker run`命令，把 Rancher Server 组件安装到å•ä¸ªèŠ‚点上，而ä¸éœ€è¦è¿è¡Œ Kubernetes 集群。由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此，如果该节点å‘生故障，由于其他节点上没有å¯ç”¨çš„ etcd æ•°æ®å‰¯æœ¬ï¼Œä½ å°†ä¸¢å¤± Rancher Server 的所有数æ®ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨å¤‡ä»½åº”用，按照[这些步骤](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)，将 Rancher Server 从 Docker 安装è¿ç§»åˆ° Kubernetes 安装。 + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +| 环境å˜é‡é”® | 环境å˜é‡å€¼ | æè¿° | +| -------------------------------- | -------------------------------- | ---- | +| `CATTLE_SYSTEM_DEFAULT_REGISTRY` | `` | å°† Rancher Server é…ç½®æˆåœ¨é…置集群时，始终从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚ | +| `CATTLE_SYSTEM_CATALOG` | `bundled` | é…ç½® Rancher Server 使用打包的 Helm System Chart 副本。[system charts](https://github.com/rancher/system-charts) 仓库包å«æ‰€æœ‰ Monitoring，Logging，告警和全局 DNS 等功能所需的应用商店项目。这些 [Helm Chart](https://github.com/rancher/system-charts) ä½äºŽ GitHub 中。但是由于你处在离线环境，因此使用 Rancher 内置的 Chart 会比设置 Git mirror 容易得多。 | + +:::note 你是å¦éœ€è¦ï¼š + +- é…置自定义 CA æ ¹è¯ä¹¦ä»¥è®¿é—®æœåŠ¡ã€‚å‚è§[自定义 CA æ ¹è¯ä¹¦](../../resources/custom-ca-root-certificates.md)。 +- 记录所有 Rancher API 的事务。å‚è§ [API 审计](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志)。 + +::: + +选择以下的选项之一： + +### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你在ä¸è€ƒè™‘身份验è¯çš„å¼€å‘或测试环境中安装 Rancher，å¯ä»¥ä½¿ç”¨ Rancher 生æˆçš„自签åè¯ä¹¦å®‰è£… Rancher。这ç§å®‰è£…æ–¹å¼é¿å…了自己生æˆè¯ä¹¦çš„麻烦。 + +登录到你的 Linux 主机，然åŽè¿è¡Œä¸‹é¢çš„安装命令。输入命令时，å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚ + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å®‰è£…çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +特æƒè®¿é—®æ˜¯[å¿…é¡»](./install-rancher-ha.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` + +
+ +### 选项 B：使用你自己的è¯ä¹¦ - 自签å + +
+ å•å‡»å±•å¼€ + +在你团队访问 Rancher Server çš„å¼€å‘或测试环境中，创建一个用于你的安装的自签åè¯ä¹¦ï¼Œä»¥ä¾¿å›¢é˜ŸéªŒè¯ä»–们对实例的连接。 + +:::note 先决æ¡ä»¶ï¼š + +从能连接到互è”网的计算机上，使用 [OpenSSL](https://www.openssl.org/) 或其他方法创建自签åè¯ä¹¦ã€‚ + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 +- 在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](../rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +::: + +创建è¯ä¹¦åŽï¼Œç™»å½• Linux 主机，然åŽè¿è¡Œä»¥ä¸‹å®‰è£…命令。输入命令时，å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚使用 `-v` 标志并æä¾›è¯ä¹¦çš„路径，以将è¯ä¹¦æŒ‚载到容器中。 + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å®‰è£…çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +特æƒè®¿é—®æ˜¯[å¿…é¡»](./install-rancher-ha.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` + +
+ +### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +在公开暴露应用的开å‘或测试环境中，请使用由å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œä»¥é¿å…用户收到è¯ä¹¦å®‰å…¨è­¦å‘Šã€‚ + +:::note 先决æ¡ä»¶ï¼š + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 + +::: + +获å–è¯ä¹¦åŽï¼Œç™»å½• Linux 主机，然åŽè¿è¡Œä»¥ä¸‹å®‰è£…命令。输入命令时，å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚因为你的è¯ä¹¦æ˜¯ç”±å¯ä¿¡çš„ CA ç­¾å的，因此你ä¸éœ€è¦å®‰è£…é¢å¤–çš„ CA è¯ä¹¦æ–‡ä»¶ã€‚ + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å®‰è£…çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +:::note + +使用 `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +::: + +特æƒè®¿é—®æ˜¯[å¿…é¡»](./install-rancher-ha.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged + /rancher/rancher: +``` + +
+ + + +:::note + +如果你ä¸æƒ³å‘é€é¥æµ‹æ•°æ®ï¼Œåœ¨é¦–次登录时退出[é¥æµ‹](../../../../faq/telemetry.md)。 + +::: + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md new file mode 100644 index 00000000000..e3e7be1a627 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md @@ -0,0 +1,193 @@ +--- +title: '1. 设置基础设施和ç§æœ‰é•œåƒä»“库' +--- + +本文介ç»å¦‚何在离线环境中，为 Rancher Management server é…置底层基础设施。你还将设置 Rancher 节点中必须å¯ç”¨çš„ç§æœ‰å®¹å™¨é•œåƒä»“库。 + +离线环境是 Rancher Server 离线安装或安装在防ç«å¢™åŽé¢çš„环境。 + +Rancher 安装在 K3s Kubernetes 集群ã€RKE Kubernetes 集群还是å•ä¸ª Docker 容器上对应的基础设施设置会有所ä¸åŒã€‚如需了解å„个安装方å¼çš„更多信æ¯ï¼Œè¯·å‚è§[本页](../../../../pages-for-subheaders/installation-and-upgrade.md)。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上。为了阅读方便，我们在下文中ä»æ供了 RKE å’Œ K3s Kubernetes 基础设施教程。 + + + + +为了实现高å¯ç”¨å®‰è£…，我们建议设置以下的基础设施： + +- **2 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商中的虚拟机。 +- **1 个外部数æ®åº“**：用于存储集群数æ®ã€‚æ”¯æŒ PostgreSQL, MySQL å’Œ etcd。 +- **1 个负载å‡è¡¡å™¨**：用于将æµé‡è½¬å‘到这两个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ +- **ç§æœ‰é•œåƒä»“库**，用于将容器镜åƒåˆ†å‘到你的主机。 + +### 1. é…ç½® Linux 节点 + +这些主机会断开互è”网链接，但需è¦èƒ½ä¸Žä½ çš„ç§æœ‰é•œåƒä»“库连接。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置外部数æ®åº“ + +K3s 与其他 Kubernetes å‘行版ä¸åŒï¼Œåœ¨äºŽå…¶æ”¯æŒä½¿ç”¨ etcd 以外的数æ®åº“æ¥è¿è¡Œ Kubernetes。该功能让 Kubernetes è¿ç»´æ›´åŠ çµæ´»ã€‚ä½ å¯ä»¥æ ¹æ®å®žé™…情况选择åˆé€‚çš„æ•°æ®åº“。 + +对于 K3s 高å¯ç”¨å®‰è£…，你需è¦é…置以下的其中一个数æ®åº“： + +* [PostgreSQL](https://www.postgresql.org/)（10.7 å’Œ 11.5 已验è¯ï¼‰ +* [MySQL](https://www.mysql.com/)（5.7 已验è¯ï¼‰ +* [etcd](https://etcd.io/)（3.3.15 已验è¯ï¼‰ + +在安装 Kubernetes 时，你需è¦ä¼ å…¥ K3s 连接数æ®åº“的详细信æ¯ã€‚ + +如需获å–é…置数æ®åº“示例，请å‚è§[在 Amazon RDS æœåŠ¡ä¸­é…ç½® MySQL æ•°æ®åº“](../../../../how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md)的教程。 + +如需获å–é…ç½® K3s 集群数æ®åº“的所有å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/datastore/)。 + +### 3. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，K3s 工具会部署一个 Traefik Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 Traefik Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚Traefik Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md)。 + +:::note é‡è¦æ示： + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 4. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 + +### 5. é…ç½®ç§æœ‰é•œåƒä»“库 + +Rancher 支æŒä½¿ç”¨ç§æœ‰é•œåƒä»“库进行离线安装。你必须有自己的ç§æœ‰é•œåƒä»“库或使用其他方å¼å°†å®¹å™¨é•œåƒåˆ†å‘到主机。 + +在åŽç»­è®¾ç½® K3s Kubernetes 集群时，你需è¦åˆ›å»ºä¸€ä¸ª[ç§æœ‰é•œåƒä»“库é…置文件](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)，其中包å«æ­¤é•œåƒä»“库的信æ¯ã€‚ + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +如需在高å¯ç”¨ RKE 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商（例如 Amazon EC2，GCE 或 vSphere）中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将å‰ç«¯æµé‡è½¬å‘到这三个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ +- **ç§æœ‰é•œåƒä»“库**，用于将容器镜åƒåˆ†å‘到你的主机。 + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸæˆ–æ•°æ®ä¸­å¿ƒã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºã€‚ + +### 为什么使用三个节点？ + +在 RKE 集群中，Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。而这个 etcd æ•°æ®åº“在这三个节点上è¿è¡Œã€‚ + +为了选举出大多数 etcd 节点认å¯çš„ etcd 集群 leader，etcd æ•°æ®åº“需è¦å¥‡æ•°ä¸ªèŠ‚点。如果 etcd æ•°æ®åº“无法选出 leader，etcd å¯èƒ½ä¼šå‡ºçŽ°[脑裂（split brain）](https://www.quora.com/What-is-split-brain-in-distributed-systems)的问题，此时你需è¦ä½¿ç”¨å¤‡ä»½æ¢å¤é›†ç¾¤ã€‚如果三个 etcd 节点之一å‘生故障，其余两个节点å¯ä»¥é€‰æ‹©ä¸€ä¸ª leader，因为它们是 etcd 节点总数的大多数部分。 + +### 1. é…ç½® Linux 节点 + +这些主机会断开互è”网链接，但需è¦èƒ½ä¸Žä½ çš„ç§æœ‰é•œåƒä»“库连接。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 3. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 + +### 4. é…ç½®ç§æœ‰é•œåƒä»“库 + +Rancher 支æŒä½¿ç”¨å®‰å…¨çš„ç§æœ‰é•œåƒä»“库进行离线安装。你必须有自己的ç§æœ‰é•œåƒä»“库或使用其他方å¼å°†å®¹å™¨é•œåƒåˆ†å‘到主机。 + +在åŽç»­è®¾ç½® RKE Kubernetes 集群时，你需è¦åˆ›å»ºä¸€ä¸ª[ç§æœ‰é•œåƒä»“库é…置文件](https://rancher.com/docs/rke/latest/en/config-options/private-registries/)，其中包å«æ­¤é•œåƒä»“库的信æ¯ã€‚ + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +:::note 注æ„事项： + +- Docker 安装适用于想è¦æµ‹è¯• Rancher 的用户。由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此如果该节点å‘生故障，你将丢失 Rancher Server 的所有数æ®ã€‚ + +- Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +::: + +### 1. é…ç½® Linux 节点 + +此主机会断开互è”网链接，但需è¦èƒ½ä¸Žä½ çš„ç§æœ‰é•œåƒä»“库连接。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器，硬件和网络](../../../../pages-for-subheaders/installation-requirements.md)的常规安装è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…ç½®ç§æœ‰ Docker é•œåƒä»“库 + +Rancher 支æŒä½¿ç”¨ç§æœ‰é•œåƒä»“库在堡垒æœåŠ¡å™¨ä¸­è¿›è¡Œç¦»çº¿å®‰è£…。你必须有自己的ç§æœ‰é•œåƒä»“库或使用其他方å¼å°†å®¹å™¨é•œåƒåˆ†å‘到主机。 + +如需获得创建ç§æœ‰é•œåƒä»“库的帮助，请å‚è§ [Docker 官方文档](https://docs.docker.com/registry/)。 + + + + +### åŽç»­æ“作 +[收集镜åƒå¹¶å‘布到你的ç§æœ‰é•œåƒä»“库](publish-images.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md new file mode 100644 index 00000000000..3fcd8a992cf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md @@ -0,0 +1,387 @@ +--- +title: '3. 安装 Kubernetes（Docker 安装请跳过）' +--- + +:::note + +如果你使用 Docker 在å•ä¸ªèŠ‚点上安装 Rancher，请跳过本节。 + +::: + +本文æè¿°äº†å¦‚ä½•æ ¹æ® [Rancher Server 环境的最佳实践](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#kubernetes-安装环境)æ¥å®‰è£… Kubernetes 集群。该集群需è¦ä¸“用于è¿è¡Œ Rancher Server。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上，包括托管的 Kubernetes。 + +在 RKEã€RKE2 或 K3s 上离线安装 Kubernetes 集群的步骤如下所示： + + + + +在本指å—中，我们å‡è®¾ä½ å·²ç»åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­åˆ›å»ºäº†èŠ‚点，并且在堡垒æœåŠ¡å™¨ä¸Šæœ‰ä¸€ä¸ªå®‰å…¨çš„ Docker ç§æœ‰é•œåƒä»“库。 + +### å®‰è£…æ¦‚è¦ + +1. [准备镜åƒç›®å½•](#1-准备镜åƒç›®å½•) +2. [创建镜åƒä»“库 YAML](#2-创建镜åƒä»“库-yaml) +3. [安装 K3s](#3-安装-k3s) +4. [ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件](#4-ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨-kubeconfig-文件) + +### 1. 准备镜åƒç›®å½• +从 [Releases](https://github.com/k3s-io/k3s/releases) 页é¢èŽ·å–è¦è¿è¡Œçš„ K3s ç‰ˆæœ¬çš„é•œåƒ tar 文件。 + +在æ¯ä¸ªèŠ‚点上å¯åŠ¨ K3s 之å‰ï¼Œå°†è¿™ä¸ª tar 文件放在 `images` 目录中，例如： + +```sh +sudo mkdir -p /var/lib/rancher/k3s/agent/images/ +sudo cp ./k3s-airgap-images-$ARCH.tar /var/lib/rancher/k3s/agent/images/ +``` + +### 2. 创建镜åƒä»“库 YAML +把 `registries.yaml` 文件创建到 `/etc/rancher/k3s/registries.yaml` 中。此文件为 K3s æ供连接到你的ç§æœ‰é•œåƒä»“库的详细信æ¯ã€‚ + +在加入必è¦ä¿¡æ¯ä¹‹å‰ï¼Œ`registries.yaml` 文件是这样的： + +```yaml +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # é•œåƒä»“库的用户å + password: xxxxxx # é•œåƒä»“åº“çš„å¯†ç  + tls: + cert_file: <é•œåƒä»“库所用的è¯ä¹¦æ–‡ä»¶è·¯å¾„> + key_file: <é•œåƒä»“库所用的密钥文件路径> + ca_file: <é•œåƒä»“库所用的 CA 文件路径> +``` + +请注æ„，目å‰ï¼ŒK3s 仅支æŒå®‰å…¨çš„é•œåƒä»“库（带有自定义 CA çš„ SSL）。 + +有关 K3s çš„ç§æœ‰é•œåƒä»“库é…置文件的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)。 + +### 3. 安装 K3s + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +从 [Releases](https://github.com/k3s-io/k3s/releases) 页é¢èŽ·å– K3s 的二进制文件，该文件è¦åŒ¹é…用于获å–离线镜åƒçš„ tar 版本。 +访问 [K3s 安装脚本](https://get.k3s.io)ä»¥èŽ·å– K3s 的安装脚本。 + +将二进制文件放到æ¯ä¸ªèŠ‚点的 `/usr/local/bin` 中。 +将安装脚本放在æ¯ä¸ªèŠ‚点的任æ„ä½ç½®ï¼Œå¹¶å°†è„šæœ¬å‘½å为 `install.sh`。 + +在æ¯ä¸ª Server 上安装 K3s： + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= ./install.sh +``` + +在æ¯ä¸ª Agent 上安装 K3s： + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= K3S_URL=https://:6443 K3S_TOKEN= ./install.sh +``` + +其中 `` 是 Server çš„ IP 或有效 DNS，`` 是å¯ä»¥åœ¨ `/var/lib/rancher/k3s/server/node-token` 中找到的 Server node-token。 + +:::note + +K3s 自动为 kubelets æä¾› `--resolv-conf` 标志，该标志å¯èƒ½å¯¹åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­é…ç½® DNS 有帮助。 + +::: + +### 4. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 K3s 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/k3s/k3s.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 +2. å¤åˆ¶ `/etc/rancher/k3s/k3s.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhost。你需è¦å°† `server` é…置为负载å‡è¡¡å™¨çš„ DNSï¼Œå¹¶æŒ‡å®šç«¯å£ 6443ï¼ˆé€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server）。以下是一个 `k3s.yaml` 示例： + +```yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ K3s 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/cluster-access/) 或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### å‡çº§æ³¨æ„事项 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹æ–¹å¼å®Œæˆç¦»çº¿çŽ¯å¢ƒçš„å‡çº§ï¼š + +1. 从 [Releases](https://github.com/k3s-io/k3s/releases) 页é¢ä¸‹è½½è¦å‡çº§çš„ K3s ç‰ˆæœ¬çš„æ–°ç¦»çº¿é•œåƒ tar 包。将 tar 文件放在æ¯ä¸ªèŠ‚点上的 `/var/lib/rancher/k3s/agent/images/` 目录中。删除旧的 tar 文件。 +2. å¤åˆ¶å¹¶æ›¿æ¢æ¯ä¸ªèŠ‚点上 `/usr/local/bin` 中的旧 K3s 二进制文件。å¤åˆ¶ [K3s 安装脚本](https://get.k3s.io)（因为脚本å¯èƒ½è‡ªä¸Šæ¬¡ç‰ˆæœ¬å‘布以æ¥å·²æ›´æ”¹ï¼‰ã€‚使用相åŒçš„环境å˜é‡å†æ¬¡è¿è¡Œè„šæœ¬ã€‚ +3. é‡å¯ K3s æœåŠ¡ï¼ˆå¦‚果安装程åºæ²¡æœ‰è‡ªåŠ¨é‡å¯ K3s çš„è¯ï¼‰ã€‚ + + + + +在本指å—中，我们å‡è®¾ä½ å·²ç»åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­åˆ›å»ºäº†èŠ‚点，并且在堡垒æœåŠ¡å™¨ä¸Šæœ‰ä¸€ä¸ªå®‰å…¨çš„ Docker ç§æœ‰é•œåƒä»“库。 + +### å®‰è£…æ¦‚è¦ + +1. [创建 RKE2 é…ç½®](#1-创建-rke2-é…ç½®) +2. [创建镜åƒä»“库 YAML](#2-创建镜åƒä»“库-yaml) +3. [安装 RKE2](#3-安装-rke2) +4. [ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件](#4-ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨-kubeconfig-文件) + +### 1. 创建 RKE2 é…ç½® +把 config.yaml 文件创建到 `/etc/rancher/rke2/config.yaml` 中。这将包å«åˆ›å»ºé«˜å¯ç”¨ RKE2 集群所需的所有é…置选项。 + +第一å°æœåŠ¡å™¨çš„最低é…置是： + +``` +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +其他æœåŠ¡å™¨çš„é…置文件应该包å«ç›¸åŒçš„令牌，并让 RKE2 知é“è¦è¿žæŽ¥åˆ°çŽ°æœ‰çš„第一å°æœåŠ¡å™¨ï¼š + +``` +server: https://ip-of-first-server:9345 +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/install/ha)。 + +:::note + +RKE2 自动为 kubelets æä¾› `resolv-conf` 选项，该标志å¯èƒ½å¯¹åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­é…ç½® DNS 有帮助。 + +::: + +### 2. 创建镜åƒä»“库 YAML +把 `registries.yaml` 文件创建到 `/etc/rancher/rke2/registries.yaml` 中。此文件为 RKE2 æ供连接到你的ç§æœ‰é•œåƒä»“库的详细信æ¯ã€‚ + +在加入必è¦ä¿¡æ¯ä¹‹å‰ï¼Œ`registries.yaml` 文件是这样的： + +``` +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # é•œåƒä»“库的用户å + password: xxxxxx # é•œåƒä»“åº“çš„å¯†ç  + tls: + cert_file: <é•œåƒä»“库所用的è¯ä¹¦æ–‡ä»¶è·¯å¾„> + key_file: <é•œåƒä»“库所用的密钥文件路径> + ca_file: <é•œåƒä»“库所用的 CA 文件路径> +``` + +有关 RKE2 çš„ç§æœ‰é•œåƒä»“库é…置文件的详情，请å‚è§ [RKE2 官方文档](https://docs.rke2.io/install/containerd_registry_configuration)。 + +### 3. 安装 RKE2 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§[支æŒç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。 + +从 Release 页é¢ä¸‹è½½å®‰è£…脚本ã€rke2ã€rke2-images å’Œ sha256sum 存档，并将它们上传到æ¯ä¸ªæœåŠ¡å™¨ä¸Šçš„目录中： + +``` +mkdir /tmp/rke2-artifacts && cd /tmp/rke2-artifacts/ +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2-images.linux-amd64.tar.zst +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2.linux-amd64.tar.gz +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/sha256sum-amd64.txt +curl -sfL https://get.rke2.io --output install.sh +``` + +接下æ¥ï¼Œä½¿ç”¨æ¯ä¸ªæœåŠ¡å™¨ä¸Šçš„目录è¿è¡Œ install.sh，如下例所示： + +``` +INSTALL_RKE2_ARTIFACT_PATH=/tmp/rke2-artifacts sh install.sh +``` + +然åŽåœ¨æ‰€æœ‰æœåŠ¡å™¨ä¸Šå¯ç”¨å¹¶å¯åŠ¨è¯¥æœåŠ¡ï¼š + +`` +systemctl enable rke2-server.service +systemctl start rke2-server.service +`` + +有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/install/airgap)。 + +### 4. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 RKE2 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/rke2/rke2.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)（Kubernetes 命令行工具）。 +2. å¤åˆ¶ `/etc/rancher/rke2/rke2.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhost。你需è¦å°† `server` é…置为负载å‡è¡¡å™¨çš„ DNSï¼Œå¹¶æŒ‡å®šç«¯å£ 6443ï¼ˆé€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server）。以下是一个 `rke2.yaml` 示例： + +``` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ RKE2 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [RKE2 官方文档](https://docs.rke2.io/cluster_access)或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### å‡çº§æ³¨æ„事项 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹æ–¹å¼å®Œæˆç¦»çº¿çŽ¯å¢ƒçš„å‡çº§ï¼š + +1. 从 [Releases](https://github.com/rancher/rke2/releases) 页é¢ä¸‹è½½æ–°çš„离线工件，并安装å‡çº§ RKE2 版本的脚本。 +2. 使用相åŒçš„环境å˜é‡å†æ¬¡è¿è¡Œè„šæœ¬ã€‚ +3. é‡å¯ RKE2 æœåŠ¡ã€‚ + + + +我们将使用 Rancher Kubernetes Engine (RKE) 创建一个 Kubernetes 集群。在å¯åŠ¨ Kubernetes 集群之å‰ï¼Œä½ éœ€è¦å®‰è£… RKE 并创建 RKE é…置文件。 + +### 1. 安装 RKE + +å‚ç…§ [RKE 官方文档](https://rancher.com/docs/rke/latest/en/installation/)的说明安装 RKE。 + +:::note + +ä½ å¯ä»¥åœ¨ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)中找到基于 Rancher 版本的 RKE 认è¯ç‰ˆæœ¬ã€‚ + +::: + +### 2. 创建 RKE é…置文件 + +在å¯è®¿é—®ä½  Linux 主机节点上的 22/TCP 端å£å’Œ 6443/TCP 端å£çš„系统上，使用以下示例创建一个å为 `rancher-cluster.yml` 的新文件。 + +该文件是 RKE é…置文件，用于é…置你è¦éƒ¨ç½² Rancher 的集群。 + +å‚考下方的 _RKE 选项_ 表格，修改代ç ç¤ºä¾‹ä¸­çš„å‚数。使用你创建的三个节点的 IP 地å€æˆ– DNS å称。 + +:::tip + +如需获å–å¯ç”¨é€‰é¡¹çš„详情，请å‚è§ RKE [é…置选项](https://rancher.com/docs/rke/latest/en/config-options/)。 + +::: + +
RKE 选项
+ +| 选项 | å¿…å¡« | æè¿° | +| ------------------ | -------------------- | --------------------------------------------------------------------------------------- | +| `address` | ✓ | 离线环境中节点的 DNS 或 IP åœ°å€ | +| `user` | ✓ | å¯è¿è¡Œ Docker 命令的用户 | +| `role` | ✓ | 分é…给节点的 Kubernetes 角色列表 | +| `internal_address` | å¯é€‰1 | 用于集群内部æµé‡çš„ DNS 或 IP åœ°å€ | +| `ssh_key_path` | | 用æ¥éªŒè¯èŠ‚点的 SSH ç§é’¥æ–‡ä»¶è·¯å¾„（默认值为 `~/.ssh/id_rsa`） | + +> 1 如果你想使用引用安全组或防ç«å¢™ï¼ŒæŸäº›æœåŠ¡ï¼ˆå¦‚ AWS EC2）è¦æ±‚设置 `internal_address`。 + +```yaml +nodes: + - address: 10.10.3.187 # 离线环境节点 IP + internal_address: 172.31.7.22 # 节点内网 IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.254 # 离线环境节点 IP + internal_address: 172.31.13.132 # 节点内网 IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.89 # 离线环境节点 IP + internal_address: 172.31.3.216 # 节点内网 IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + +private_registries: + - url: # ç§æœ‰é•œåƒä»“库 URL + user: rancher + password: '*********' + is_default: true +``` + +### 3. è¿è¡Œ RKE + +é…ç½® `rancher-cluster.yml`åŽï¼Œå¯åŠ¨ä½ çš„ Kubernetes 集群： + +``` +rke up --config ./rancher-cluster.yml +``` + +### 4. ä¿å­˜ä½ çš„文件 + +:::note é‡è¦æ示： + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +::: + +将以下文件的副本ä¿å­˜åœ¨å®‰å…¨ä½ç½®ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_cluster.yml`：集群的 [Kubeconfig 文件](https://rancher.com/docs/rke/latest/en/kubeconfig/)。该文件包å«å¯å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚ +- `rancher-cluster.rkestate`：[Kubernetes 集群状æ€æ–‡ä»¶](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state)。该文件包å«é›†ç¾¤çš„当å‰çŠ¶æ€ï¼ŒåŒ…括 RKE é…置以åŠè¯ä¹¦
。
_Kubernetes 集群状æ€æ–‡ä»¶ä»…在使用 RKE 0.2.0 或更高版本时创建。_ + + + + +:::note + +åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +::: + +### 故障排除 + +å‚è§[故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)页é¢ã€‚ + +### åŽç»­æ“作 +[安装 Rancher](install-rancher-ha.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md new file mode 100644 index 00000000000..0aa220051f7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md @@ -0,0 +1,246 @@ +--- +title: 4. 安装 Rancher +--- + +本文介ç»å¦‚何在高å¯ç”¨ Kubernetes 安装的离线环境部署 Rancher。离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +### Rancher 特æƒè®¿é—® + +当 Rancher Server 部署在 Docker 容器中时，容器内会安装一个本地 Kubernetes 集群供 Rancher 使用。为 Rancher 的很多功能都是以 deployment çš„æ–¹å¼è¿è¡Œçš„，而在容器内è¿è¡Œå®¹å™¨æ˜¯éœ€è¦ç‰¹æƒæ¨¡å¼çš„，因此你需è¦åœ¨å®‰è£… Rancher 时添加 `--privileged` 选项。 + +## Docker 说明 + +如果你想使用 Docker 命令进行离线安装，请跳过本页的剩余部分，并按照[此页](docker-install-commands.md)进行æ“作。 + +## Kubernetes 说明 + +我们建议在 Kubernetes 集群上安装 Rancher。高å¯ç”¨çš„ Kubernetes 安装的情况下，一个 Kubernetes 集群包å«ä¸‰ä¸ªè¿è¡Œ Rancher Server 组件的节点。æŒä¹…层（etcd）也在这三个节点上进行å¤åˆ¶ï¼Œä»¥ä¾¿åœ¨å…¶ä¸­ä¸€ä¸ªèŠ‚点å‘生故障时æ供冗余和数æ®å¤åˆ¶ã€‚ + +### 1. 添加 Helm Chart 仓库 + +从å¯ä»¥è®¿é—®äº’è”网的系统中，获å–最新的 Helm Chart，然åŽå°† manifest å¤åˆ¶åˆ°å¯è®¿é—® Rancher Server 集群的系统中。 + +1. 如果你还没有安装 `helm`，请在å¯è®¿é—®äº’è”网的工作站上进行本地安装。注æ„：å‚考 [Helm 版本è¦æ±‚](../../resources/helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。 + +2. 执行 `helm repo add` 命令，以添加包å«å®‰è£… Rancher çš„ Chart çš„ Helm Chart 仓库。有关如何选择仓库，以åŠå“ªä¸ªä»“库最适åˆä½ çš„用例，请å‚è§[选择 Rancher 版本](../../resources/choose-a-rancher-version.md)。 + - Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + +3. 获å–最新的 Rancher Chart。此æ“ä½œå°†èŽ·å– Chart 并将其作为 `.tgz` 文件ä¿å­˜åœ¨å½“å‰ç›®å½•ä¸­ã€‚ + ```plain + helm fetch rancher-/rancher + ``` + + 如需下载特定的 Rancher 版本，你å¯ä»¥ç”¨ Helm `--version` å‚数指定版本，如下： + ```plain + helm fetch rancher-stable/rancher --version=v2.4.8 + ``` + +### 2. 选择 SSL é…ç½® + +Rancher Server é»˜è®¤è®¾è®¡ä¸ºå®‰å…¨çš„ï¼Œå¹¶ä¸”éœ€è¦ SSL/TLS é…置。 + +如果你在离线的 Kubernetes 集群中安装 Rancher，我们建议使用以下两ç§è¯ä¹¦ç”Ÿæˆæ–¹å¼ã€‚ + +:::note + +如果你想在外部终止 SSL/TLS，请å‚è§[外部负载å‡è¡¡å™¨çš„ TLS 终止](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +::: + +| é…ç½® | Chart 选项 | æè¿° | 是å¦éœ€è¦ cert-manager | +| ------------------------------------------ | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| Rancher 生æˆçš„自签åè¯ä¹¦ | `ingress.tls.source=rancher` | 使用 Rancher 生æˆçš„ CA ç­¾å‘的自签åè¯ä¹¦ã€‚此项是**默认选项**。在渲染 Helm 模æ¿çš„时候ä¸éœ€è¦ä¼ é€’此项。 | 是 | +| 你已有的è¯ä¹¦ | `ingress.tls.source=secret` | 通过创建 Kubernetes 密文使用你自己的è¯ä¹¦æ–‡ä»¶ã€‚
在渲染 Rancher Helm 模æ¿æ—¶å¿…须传递此选项。 | å¦ | + +### 离线安装的 Helm Chart 选项 + +在é…ç½® Rancher Helm 模æ¿æ—¶ï¼ŒHelm Chart 中有几个专为离线安装设计的选项，如下表： + +| Chart 选项 | Chart 值 | æè¿° | +| ----------------------- | -------------------------------- | ---- | +| `certmanager.version` | `` | æ ¹æ®è¿è¡Œçš„ cert-manager 版本é…置适当的 Rancher TLS é¢å‘者。 | +| `systemDefaultRegistry` | `` | å°† Rancher Server é…ç½®æˆåœ¨é…置集群时，始终从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚ | +| `useBundledSystemChart` | `true` | é…ç½® Rancher Server 使用打包的 Helm System Chart 副本。[system charts](https://github.com/rancher/system-charts) 仓库包å«æ‰€æœ‰ Monitoring，Logging，告警和全局 DNS 等功能所需的应用商店项目。这些 [Helm Chart](https://github.com/rancher/system-charts) ä½äºŽ GitHub 中。但是由于你处在离线环境，因此使用 Rancher 内置的 Chart 会比设置 Git mirror 容易得多。 | + +### 3. èŽ·å– Cert-Manager Chart + +æ ¹æ®ä½ åœ¨[2：选择 SSL é…ç½®](#2-选择-ssl-é…ç½®)中的选择，完æˆä»¥ä¸‹æ­¥éª¤ä¹‹ä¸€ï¼š + +#### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +默认情况下，Rancher 会生æˆä¸€ä¸ª CA 并使用 cert-manager é¢å‘è¯ä¹¦ä»¥è®¿é—® Rancher Server ç•Œé¢ã€‚ + +:::note + +由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.11.0 çš„ cert-manager 版本，请å‚è§ [cert-manager å‡çº§æ–‡æ¡£](../../resources/upgrade-cert-manager.md)。 + +::: + +##### 1. 添加 cert-manager 仓库 + +在å¯ä»¥è¿žæŽ¥äº’è”网的系统中，将 cert-manager 仓库添加到 Helm： + +```plain +helm repo add jetstack https://charts.jetstack.io +helm repo update +``` + +##### 2. èŽ·å– cert-manager Chart + +从 [Helm Chart 仓库](https://artifacthub.io/packages/helm/cert-manager/cert-manager)中获å–最新å¯ç”¨çš„ cert-manager Chart： + +```plain +helm fetch jetstack/cert-manager --version v1.11.0 +``` + +##### 3. 检索 Cert-Manager CRD + +为 cert-manager 下载所需的 CRD 文件： +```plain +curl -L -o cert-manager-crd.yaml https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml +``` + +### 4. 安装 Rancher + +将获å–çš„ Chart å¤åˆ¶åˆ°æœ‰æƒè®¿é—® Rancher Server 集群的系统以完æˆå®‰è£…。 + +##### 1. 安装 Cert-Manager + +使用è¦ç”¨äºŽå®‰è£… Chart 的选项æ¥å®‰è£… cert-manager。记ä½è¦è®¾ç½® `image.repository` 选项，以从你的ç§æœ‰é•œåƒä»“库拉å–é•œåƒã€‚æ­¤æ“ä½œä¼šåˆ›å»ºä¸€ä¸ªåŒ…å« Kubernetes manifest 文件的 `cert-manager` 目录。 + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +
+ å•å‡»å±•å¼€ + +如果你使用自签åè¯ä¹¦ï¼Œå®‰è£… cert-manager： + +1. 为 cert-manager 创建命å空间： + + ```plain + kubectl create namespace cert-manager + ``` + +2. 创建 cert-manager CustomResourceDefinition (CRD)。 + + ```plain + kubectl apply -f cert-manager/cert-manager-crd.yaml + ``` + +3. 安装 cert-manager。 + + ```plain + helm install cert-manager ./cert-manager-v1.11.0.tgz \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller \ + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook \ + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector \ + --set startupapicheck.image.repository=/quay.io/jetstack/cert-manager-ctl + ``` + +
+ +##### 2. 安装 Rancher +首先，å‚è§[添加 TLS 密文](../../resources/add-tls-secrets.md)å‘布è¯ä¹¦æ–‡ä»¶ï¼Œä»¥ä¾¿ Rancher å’Œ Ingress Controller å¯ä»¥ä½¿ç”¨å®ƒä»¬ã€‚ + +然åŽï¼Œä½¿ç”¨ kubectl 为 Rancher 创建命å空间： + +```plain +kubectl create namespace cattle-system +``` + +然åŽå®‰è£… Rancher，并声明你选择的选项。å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚Rancher 需è¦é…置为使用ç§æœ‰é•œåƒä»“库，以便é…置所有 Rancher å¯åŠ¨çš„ Kubernetes 集群或 Rancher 工具。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 输出压缩包的版本å·ã€‚ | +| `` | 指å‘è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 | +| `` | ä½ çš„ç§æœ‰é•œåƒä»“库的 DNS å称。 | +| `` | 在 K8s 集群上è¿è¡Œçš„ cert-manager 版本。 | + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart +``` + +**å¯é€‰**：如需安装特定的 Rancher 版本，设置`rancherImageTag` 的值，例如：`--set rancherImageTag=v2.5.8` + +#### 选项 B：使用 Kubernetes 密文从文件中获å–è¯ä¹¦ + +##### 1. 创建密文 + +使用你自己的è¯ä¹¦æ¥åˆ›å»º Kubernetes 密文，以供 Rancher 使用。è¯ä¹¦çš„ common name 需è¦ä¸Žä»¥ä¸‹å‘½ä»¤ä¸­çš„ `hostname` 选项匹é…，å¦åˆ™ Ingress Controller 将无法为 Rancher é…置站点。 + +##### 2. 安装 Rancher + +安装 Rancher，并声明你选择的选项。å‚考下表æ¥æ›¿æ¢æ¯ä¸ªå ä½ç¬¦ã€‚Rancher 需è¦é…置为使用ç§æœ‰é•œåƒä»“库，以便é…置所有 Rancher å¯åŠ¨çš„ Kubernetes 集群或 Rancher 工具。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +| å ä½ç¬¦ | æè¿° | +| -------------------------------- | ----------------------------------------------- | +| `` | 输出压缩包的版本å·ã€‚ | +| `` | 指å‘è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 | +| `` | ä½ çš„ç§æœ‰é•œåƒä»“库的 DNS å称。 | + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart +``` + +如果你使用的是ç§æœ‰ CA ç­¾åçš„è¯ä¹¦ï¼Œè¯·åœ¨ `--set ingress.tls.source=secret` åŽåŠ ä¸Š `--set privateCA=true`： + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart +``` + + +安装已完æˆã€‚ +:::caution + +如果你ä¸æƒ³å‘é€é¥æµ‹æ•°æ®ï¼Œåœ¨é¦–次登录时退出[é¥æµ‹](../../../../faq/telemetry.md)。如果在离线安装的环境中让这个功能处于 active 状æ€ï¼Œsocket å¯èƒ½æ— æ³•æ‰“开。 + +::: + +## å…¶ä»–èµ„æº + +以下资æºå¯èƒ½å¯¹å®‰è£… Rancher 有帮助： + +- [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md) +- [添加 TLS 密文](../../resources/add-tls-secrets.md) +- [Rancher Kubernetes 安装的故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md new file mode 100644 index 00000000000..c64f7132d1d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md @@ -0,0 +1,305 @@ +--- +title: '2. 收集镜åƒå¹¶å‘布到ç§æœ‰ä»“库' +--- + +本文介ç»å¦‚何é…ç½®ç§æœ‰é•œåƒä»“库，以便在安装 Rancher 时，Rancher å¯ä»¥ä»Žæ­¤ç§æœ‰é•œåƒä»“库中拉å–所需的镜åƒã€‚ + +默认情况下，所有用于[é…ç½® Kubernetes 集群](../../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)或å¯åŠ¨ Rancher 中的工具（如监控，æµæ°´çº¿ï¼Œå‘Šè­¦ç­‰ï¼‰çš„é•œåƒéƒ½æ˜¯ä»Ž Docker Hub 中拉å–的。在 Rancher 的离线安装中，你需è¦ä¸€ä¸ªç§æœ‰ä»“库，该仓库ä½äºŽä½ çš„ Rancher Server 中æŸä¸ªå¯è®¿é—®çš„ä½ç½®ã€‚然åŽï¼Œä½ å¯åŠ è½½è¯¥å­˜æœ‰æ‰€æœ‰é•œåƒçš„é•œåƒä»“库。 + +使用 Docker 安装 Rancher，和把 Rancher 安装到 Kubernetes 集群，其对应的推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库步骤是一样的。 + +你使用 Rancher é…置的下游集群是å¦æœ‰è¿è¡Œ Windows 的节点，决定了本文涉åŠçš„步骤。我们æ供的推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库步骤，是基于å‡è®¾ Rancher ä»…é…ç½®è¿è¡Œ Linux 节点的下游 Kubernetes 集群的。但是，如果你计划[在下游 Kubernetes 集群中使用 Windows 节点](../../../../pages-for-subheaders/use-windows-clusters.md)，我们有å•ç‹¬çš„文档æ¥ä»‹ç»å¦‚何为需è¦çš„é•œåƒæ供支æŒã€‚ + +:::note 先决æ¡ä»¶ï¼š + +你必须有一个å¯ç”¨çš„[ç§æœ‰é•œåƒä»“库](https://docs.docker.com/registry/deploying/#run-an-externally-accessible-registry)。 + +如果镜åƒä»“库有è¯ä¹¦ï¼Œè¯·å‚è§ [K3s 文档中心](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)了解添加ç§æœ‰é•œåƒä»“库的详情。è¯ä¹¦å’Œé•œåƒä»“库é…置文件å‡éœ€è¦æŒ‚载到 Rancher 容器中。 + +::: + + + + +如果 Rancher Server 用于é…置仅有 Linux 节点的集群，请按以下步骤将你的镜åƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库。 + +1. [找到你的 Rancher 版本所需的资æº](#1-找到你的-rancher-版本所需的资æº) +2. [收集 cert-manager é•œåƒ](#2-收集-cert-manager-é•œåƒ)（除éžä½ ä½¿ç”¨è‡ªå·±çš„è¯ä¹¦ï¼Œæˆ–在负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ TLS） +3. [把镜åƒä¿å­˜åˆ°ä½ çš„工作站](#3-将镜åƒä¿å­˜åˆ°ä½ çš„工作站中) +4. [推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库](#4-推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库) + +### 先决æ¡ä»¶ + +这些步骤è¦æ±‚你使用一个 Linux 工作站，该工作站需è¦å¯è®¿é—®äº’è”网和你的ç§æœ‰é•œåƒä»“库，且至少有 20GB çš„å¯ç”¨ç£ç›˜ç©ºé—´ã€‚ + +如果你的主机架构是 ARM64，镜åƒä»“åº“å¿…é¡»æ”¯æŒ Manifest。这是因为从 2020 å¹´ 4 月开始, Amazon Elastic Container Registry å·²ç»ä¸å†æ”¯æŒ Manifest。 + +### 1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº + +1. 访问 Rancher çš„[å‘布说明](https://github.com/rancher/rancher/releases)页é¢ï¼Œæ‰¾åˆ°ä½ éœ€è¦å®‰è£…çš„ Rancher v2.x.x 版本，然åŽç‚¹å‡» **Assets**。注æ„ä¸è¦ä½¿ç”¨å¸¦æœ‰ `rc` 或 `Pre-release` 标记的版本，因为这些版本在生产环境中ä¸å¤Ÿç¨³å®šã€‚ + +2. 从你所需版本的 **Assets** 处下载以下文件，这些文件是在离线环境中安装 Rancher 所必须的： + +| Release 文件 | æè¿° | +| ---------------- | -------------- | +| `rancher-images.txt` | 此文件包å«å®‰è£… Rancherã€é…置集群和用户 Rancher 工具所需的镜åƒã€‚ | +| `rancher-save-images.sh` | 该脚本从 Docker Hub 中拉å–在文件 `rancher-images.txt` 中记录的所有镜åƒï¼Œå¹¶æŠŠè¿™äº›é•œåƒä¿å­˜ä¸º `rancher-images.tar.gz`。 | +| `rancher-load-images.sh` | 该脚本从 `rancher-images.tar.gz` 文件中加载镜åƒï¼Œå¹¶æŠŠé•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。 | + +### 2. 收集 cert-manager é•œåƒ + +:::note + +如果你使用自己的è¯ä¹¦ï¼Œæˆ–è¦åœ¨å¤–部负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ TLS，请跳过此步骤。 + +::: + +在 Kubernetes 安装中，如果你使用的是 Rancher 默认的自签å TLS è¯ä¹¦ï¼Œåˆ™å¿…须将 [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) é•œåƒæ·»åŠ åˆ° `rancher-images.txt` 文件中。 + +1. 获å–最新的 `cert-manager` Helm Chart，并解æžæ¨¡æ¿ä»¥èŽ·å–é•œåƒçš„详情信æ¯ï¼š + + :::note + + 由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.12.0 çš„ cert-manager 版本，请å‚è§[å‡çº§æ–‡æ¡£](../../resources/upgrade-cert-manager.md)。 + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. 对镜åƒåˆ—表进行排åºå’Œå”¯ä¸€åŒ–，以去除é‡å¤çš„é•œåƒæºï¼š + + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + +### 3. 将镜åƒä¿å­˜åˆ°ä½ çš„工作站中 + +1. 为 `rancher-save-images.sh` 文件添加å¯æ‰§è¡Œæƒé™ï¼š + ``` + chmod +x rancher-save-images.sh + ``` + +1. 使用 `rancher-images.txt` é•œåƒåˆ—表执行 `rancher-save-images.sh` 脚本，以创建包å«æ‰€æœ‰æ‰€éœ€é•œåƒçš„压缩包： + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + **结果**：Docker 开始拉å–用于离线安装的镜åƒã€‚请è€å¿ƒç­‰å¾…。这个过程需è¦å‡ åˆ†é’Ÿã€‚完æˆæ—¶ï¼Œä½ çš„当å‰ç›®å½•ä¼šè¾“出å为 `rancher-images.tar.gz` 的压缩包。请确认输出文件是å¦å­˜åœ¨ã€‚ + +### 4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +下一步，你将使用脚本将 `rancher-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +使用脚本将 `rancher-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +`rancher-images.txt` 需è¦ä½äºŽå·¥ä½œç«™ä¸­è¿è¡Œ `rancher-load-images.sh` 脚本的åŒä¸€ç›®å½•ä¸­ã€‚`rancher-images.tar.gz` 也需è¦ä½äºŽåŒä¸€ç›®å½•ä¸­ã€‚ + +1. 登录到你的ç§æœ‰é•œåƒä»“库： +```plain +docker login +``` +1. 为 `rancher-load-images.sh` 添加å¯æ‰§è¡Œæƒé™ï¼š +``` +chmod +x rancher-load-images.sh +``` + +1. 使用 `rancher-load-images.sh` 脚本æ¥æå–ï¼Œæ ‡è®°å’ŒæŽ¨é€ `rancher-images.txt` åŠ `rancher-images.tar.gz` 到你的ç§æœ‰é•œåƒä»“库： +```plain +./rancher-load-images.sh --image-list ./rancher-images.txt --registry +``` + + + + +如果你的 Rancher Server 将用于é…ç½® Linux å’Œ Windows 集群，你需è¦æ‰§è¡Œä¸åŒçš„步骤，æ¥å°† Windows é•œåƒå’Œ Linux é•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。由于 Windows 集群åŒæ—¶åŒ…å« Linux å’Œ Windows 节点，因此推é€åˆ°ç§æœ‰é•œåƒä»“库的 Linux é•œåƒæ˜¯ Manifest。 + +## Windows 步骤 + +从 Windows Server å·¥ä½œç«™ä¸­æ”¶é›†å’ŒæŽ¨é€ Windows é•œåƒã€‚ + +1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº +2. 将镜åƒä¿å­˜åˆ°ä½ çš„ Windows Server 工作站 +3. 准备 Docker daemon +4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +### 先决æ¡ä»¶ + +以下步骤å‡è®¾ä½ ä½¿ç”¨ Windows Server 1809 工作站，该工作站能访问网络åŠä½ çš„ç§æœ‰é•œåƒä»“库，且至少拥有 50GB çš„ç£ç›˜ç©ºé—´ã€‚ + +工作站必须安装 Docker 18.02+ 版本以æä¾› manifest 支æŒã€‚Manifest 支æŒæ˜¯é…ç½® Windows 集群所必须的。 + +ä½ çš„é•œåƒä»“åº“å¿…é¡»æ”¯æŒ Manifest。这是因为从 2020 å¹´ 4 月开始, Amazon Elastic Container Registry å·²ç»ä¸å†æ”¯æŒ Manifest。 + + + +### 1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº + +1. 访问 Rancher çš„[å‘布说明](https://github.com/rancher/rancher/releases)页é¢ï¼Œæ‰¾åˆ°ä½ éœ€è¦å®‰è£…çš„ Rancher v2.x.x 版本。ä¸è¦ä¸‹è½½å¸¦æœ‰ `rc` 或 `Pre-release` 标记的版本，因为这些版本在生产环境中ä¸å¤Ÿç¨³å®šã€‚ + +2. 从你所需版本的 **Assets** 处下载以下文件： + +| Release 文件 | æè¿° | +|----------------------------|------------------| +| `rancher-windows-images.txt` | 此文件包å«é…ç½® Windows 集群所需的 Windows é•œåƒã€‚ | +| `rancher-save-images.ps1` | 该脚本从 Docker Hub 中拉å–在文件 `rancher-windows-images.txt` 中记录的所有镜åƒï¼Œå¹¶æŠŠè¿™äº›é•œåƒä¿å­˜ä¸º `rancher-windows-images.tar.gz`。 | +| `rancher-load-images.ps1` | 该脚本从 `rancher-windows-images.tar.gz` 文件中加载镜åƒï¼Œå¹¶æŠŠé•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。 | + + + +### 2. 将镜åƒä¿å­˜åˆ°ä½ çš„ Windows Server 工作站 + +1. 在 `powershell` 中，进入上一步下载的文件所在的目录。 + +1. è¿è¡Œ `rancher-save-images.ps1` 以创建包å«æ‰€æœ‰æ‰€éœ€é•œåƒçš„压缩包： + ```plain + ./rancher-save-images.ps1 + ``` + + **结果**：Docker 开始拉å–用于离线安装的镜åƒã€‚请è€å¿ƒç­‰å¾…。这个过程需è¦å‡ åˆ†é’Ÿã€‚完æˆæ—¶ï¼Œä½ çš„当å‰ç›®å½•ä¼šè¾“出å为 `rancher-windows-images.tar.gz` 的压缩包。请确认输出文件是å¦å­˜åœ¨ã€‚ + + + +### 3. 准备 Docker daemon + +将你的ç§æœ‰é•œåƒä»“库地å€å°¾é™„到 Docker daemon (`C:\\ProgramData\\Docker\\config\\daemon.json`) çš„ `allow-nondistributable-artifacts` é…置字段中。Windows é•œåƒçš„基础镜åƒæ˜¯ç”± `mcr.microsoft.com` é•œåƒä»“库维护的，而 Docker Hub 中缺少 Microsoft é•œåƒä»“库层，且需è¦å°†å…¶æ‹‰å…¥ç§æœ‰é•œåƒä»“库，因此这一步骤是必须的。 + +```json +{ + ... + "allow-nondistributable-artifacts": [ + ... + "" + ] + ... +} +``` + + + +### 4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +使用脚本将 `rancher-windows-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +`rancher-windows-images.txt` 需è¦ä½äºŽå·¥ä½œç«™ä¸­è¿è¡Œ `rancher-load-images.ps1` 脚本的åŒä¸€ç›®å½•ä¸­ã€‚`rancher-windows-images.tar.gz` 也需è¦ä½äºŽåŒä¸€ç›®å½•ä¸­ã€‚ + +1. 使用 `powershell` 登录到你的ç§æœ‰é•œåƒä»“库： + ```plain + docker login + ``` + +1. 在 `powershell` 中，使用 `rancher-load-images.ps1` 脚本æ¥æå–ï¼Œæ ‡è®°å’ŒæŽ¨é€ `rancher-images.tar.gz` 中的镜åƒåˆ°ä½ çš„ç§æœ‰é•œåƒä»“库： + ```plain + ./rancher-load-images.ps1 --registry + ``` + +## Linux 步骤 + +Linux é•œåƒéœ€è¦åœ¨ Linux 主机上收集和推é€ï¼Œä½†æ˜¯ä½ å¿…须先将 Windows é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库，然åŽå†æŽ¨é€ Linux é•œåƒã€‚由于被推é€çš„ Linux é•œåƒå®žé™…ä¸Šæ˜¯æ”¯æŒ Windows å’Œ Linux é•œåƒçš„ manifest，因此涉åŠçš„步骤ä¸åŒäºŽåªåŒ…å« Linux 节点的集群。 + +1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº +2. 收集所有需è¦çš„é•œåƒ +3. 将镜åƒä¿å­˜åˆ°ä½ çš„ Linux 工作站中 +4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +### 先决æ¡ä»¶ + +在将 Linux é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库之å‰ï¼Œä½ å¿…须先把 Windows é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库。如果你已ç»æŠŠ Linux é•œåƒæŽ¨é€åˆ°ç§æœ‰é•œåƒä»“库，则需è¦å†æ¬¡æŒ‰ç…§è¯´æ˜Žé‡æ–°æŽ¨é€ï¼Œå› ä¸ºå®ƒä»¬éœ€è¦å‘å¸ƒæ”¯æŒ Windows å’Œ Linux é•œåƒçš„ manifest。 + +这些步骤è¦æ±‚你使用一个 Linux 工作站，该工作站需è¦å¯è®¿é—®äº’è”网和你的ç§æœ‰é•œåƒä»“库，且至少有 20GB çš„å¯ç”¨ç£ç›˜ç©ºé—´ã€‚ + +工作站必须安装 Docker 18.02+ 版本以æä¾› manifest 支æŒã€‚Manifest 支æŒæ˜¯é…ç½® Windows 集群所必须的。 + + + +### 1. 找到你的 Rancher ç‰ˆæœ¬æ‰€éœ€çš„èµ„æº + +1. 访问 Rancher çš„[å‘布说明](https://github.com/rancher/rancher/releases)页é¢ï¼Œæ‰¾åˆ°ä½ éœ€è¦å®‰è£…çš„ Rancher v2.x.x 版本。ä¸è¦ä¸‹è½½å¸¦æœ‰ `rc` 或 `Pre-release` 标记的版本，因为这些版本在生产环境中ä¸å¤Ÿç¨³å®šã€‚点击 **Assets**。 + +2. 从你所需版本的 **Assets** 处下载以下文件： + +| Release 文件 | æè¿° | +|----------------------------| -------------------------- | +| `rancher-images.txt` | 此文件包å«å®‰è£… Rancherã€é…置集群和用户 Rancher 工具所需的镜åƒã€‚ | +| `rancher-windows-images.txt` | 此文件包å«é…ç½® Windows 集群所需的镜åƒã€‚ | +| `rancher-save-images.sh` | 该脚本从 Docker Hub 中拉å–在文件 `rancher-images.txt` 中记录的所有镜åƒï¼Œå¹¶æŠŠè¿™äº›é•œåƒä¿å­˜ä¸º `rancher-images.tar.gz`。 | +| `rancher-load-images.sh` | 该脚本从 `rancher-images.tar.gz` 文件中加载镜åƒï¼Œå¹¶æŠŠé•œåƒæŽ¨é€åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库。 | + + + +### 2. 收集所有需è¦çš„é•œåƒ + +**在 Kubernetes 安装中，如果你使用的是 Rancher 默认的自签å TLS è¯ä¹¦**，则必须将 [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) é•œåƒæ·»åŠ åˆ° `rancher-images.txt` 文件中。如果你使用自己的è¯ä¹¦ï¼Œåˆ™å¯è·³è¿‡æ­¤æ­¥éª¤ã€‚ + + +1. 获å–最新的 `cert-manager` Helm Chart，并解æžæ¨¡æ¿ä»¥èŽ·å–é•œåƒçš„详情信æ¯ï¼š + + :::note + + 由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.12.0 çš„ cert-manager 版本，请å‚è§[å‡çº§æ–‡æ¡£](../../resources/upgrade-cert-manager.md)。 + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. 对镜åƒåˆ—表进行排åºå’Œå”¯ä¸€åŒ–，以去除é‡å¤çš„é•œåƒæºï¼š + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + + + +### 3. 将镜åƒä¿å­˜åˆ°ä½ çš„工作站中 + +1. 为 `rancher-save-images.sh` 文件添加å¯æ‰§è¡Œæƒé™ï¼š + ``` + chmod +x rancher-save-images.sh + ``` + +1. 使用 `rancher-images.txt` é•œåƒåˆ—表执行 `rancher-save-images.sh` 脚本，以创建包å«æ‰€æœ‰æ‰€éœ€é•œåƒçš„压缩包： + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + +**结果**：Docker 开始拉å–用于离线安装的镜åƒã€‚请è€å¿ƒç­‰å¾…。这个过程需è¦å‡ åˆ†é’Ÿã€‚完æˆæ—¶ï¼Œä½ çš„当å‰ç›®å½•ä¼šè¾“出å为 `rancher-images.tar.gz` 的压缩包。请确认输出文件是å¦å­˜åœ¨ã€‚ + + + +### 4. 推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库 + +使用 `rancher-load-images.sh script` 脚本将 `rancher-images.tar.gz` 中的镜åƒç§»åŠ¨åˆ°ä½ çš„ç§æœ‰é•œåƒä»“库，以方便加载镜åƒã€‚ + +é•œåƒåˆ—è¡¨ï¼Œå³ `rancher-images.txt` 或 `rancher-windows-images.txt` 需è¦ä½äºŽå·¥ä½œç«™ä¸­è¿è¡Œ `rancher-load-images.sh` 脚本的åŒä¸€ç›®å½•ä¸­ã€‚`rancher-images.tar.gz` 也需è¦ä½äºŽåŒä¸€ç›®å½•ä¸­ã€‚ + +1. 登录到你的ç§æœ‰é•œåƒä»“库： + ```plain + docker login + ``` + +1. 为 `rancher-load-images.sh` 添加å¯æ‰§è¡Œæƒé™ï¼š + ``` + chmod +x rancher-load-images.sh + ``` + +1. 使用 `rancher-load-images.sh` 脚本æ¥æå–ï¼Œæ ‡è®°å’ŒæŽ¨é€ `rancher-images.tar.gz` 中的镜åƒåˆ°ä½ çš„ç§æœ‰é•œåƒä»“库： + +```plain +./rancher-load-images.sh --image-list ./rancher-images.txt \ + --windows-image-list ./rancher-windows-images.txt \ + --registry +``` + + + + +### [Kubernetes 安装的åŽç»­æ­¥éª¤ - å¯åŠ¨ Kubernetes 集群](install-kubernetes.md) + +### [Docker 安装的åŽç»­æ­¥éª¤ - 安装 Rancher](install-rancher-ha.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md new file mode 100644 index 00000000000..d3628dea33f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md @@ -0,0 +1,249 @@ +--- +title: '2. 安装 Kubernetes' +--- + +基础设施é…置好åŽï¼Œä½ å¯ä»¥è®¾ç½®ä¸€ä¸ª Kubernetes 集群æ¥å®‰è£… Rancher。 + +设置 RKEã€RKE2 或 K3s 的步骤如下所示。 + +为方便起è§ï¼Œå°†ä»£ç†çš„ IP 地å€å’Œç«¯å£å¯¼å‡ºåˆ°ä¸€ä¸ªçŽ¯å¢ƒå˜é‡ä¸­ï¼Œå¹¶åœ¨æ¯ä¸ªèŠ‚点上为你当å‰çš„ shell 设置 HTTP_PROXY å˜é‡ï¼š + +``` +export proxy_host="10.0.0.5:8888" +export HTTP_PROXY=http://${proxy_host} +export HTTPS_PROXY=http://${proxy_host} +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16 +``` + + + + +首先在 K3s systemd æœåŠ¡ä¸Šé…ç½® HTTP 代ç†è®¾ç½®ï¼Œè®© K3s çš„ containerd å¯ä»¥é€šè¿‡ä»£ç†æ‹‰å–é•œåƒï¼š + +``` +cat <<'EOF' | sudo tee /etc/default/k3s > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +在第一个节点上，创建一个新集群： +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --cluster-init +``` + +然åŽåŠ å…¥å…¶ä»–节点： +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --server https://:6443 +``` + +其中 `` 是 Server çš„ IP 或有效 DNS，`` 是å¯ä»¥åœ¨ `/var/lib/rancher/k3s/server/node-token` 中找到的 Server node-token。 + +有关安装 K3s 的更多信æ¯ï¼Œè¯·å‚阅 [K3s 安装文档](https://docs.k3s.io/installation)。 + +如需查看集群，请è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +在æ¯ä¸ªèŠ‚点上，è¿è¡Œ RKE2 安装脚本。确ä¿ä½ å®‰è£…çš„ RKE2 ç‰ˆæœ¬å— [Rancher 支æŒ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +``` +curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.xx sh - +``` + +然åŽï¼Œä½ å¿…须在 RKE2 systemd æœåŠ¡ä¸Šé…ç½® HTTP 代ç†è®¾ç½®ï¼Œè®© RKE2 çš„ containerd å¯ä»¥é€šè¿‡ä»£ç†æ‹‰å–é•œåƒï¼š + +``` +cat <<'EOF' | sudo tee /etc/default/rke2-server > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +接下æ¥ï¼ŒæŒ‰ç…§ [RKE2 高å¯ç”¨æ€§æ–‡æ¡£](https://docs.rke2.io/install/ha)在æ¯ä¸ªèŠ‚点上创建 RKE2 é…置文件。 + +之åŽå¯åŠ¨å¹¶å¯ç”¨ `rke2-server` æœåŠ¡ï¼š + +``` +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` + +有关安装 RKE2 的更多信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io)。 + +如需查看集群，请è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +alias kubectl=/var/lib/rancher/rke2/bin/kubectl +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +首先，你需è¦åœ¨æ‰€æœ‰ä¸‰ä¸ª Linux 节点上安装 Docker 并设置 HTTP 代ç†ã€‚因此，你å¯ä»¥åœ¨è¿™ä¸‰ä¸ªèŠ‚点上执行以下步骤。 + +接下æ¥é…ç½® apt 以在安装包时使用这个代ç†ã€‚如果你使用的ä¸æ˜¯ Ubuntu，请相应调整步骤。 + +``` +cat <<'EOF' | sudo tee /etc/apt/apt.conf.d/proxy.conf > /dev/null +Acquire::http::Proxy "http://${proxy_host}/"; +Acquire::https::Proxy "http://${proxy_host}/"; +EOF +``` + +安装 Docker： + +``` +curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh +``` + +然åŽï¼Œç¡®ä¿ä½ çš„当å‰ç”¨æˆ·èƒ½å¤Ÿåœ¨æ²¡æœ‰ sudo 的情况下访问 Docker Daemon： + +``` +sudo usermod -aG docker YOUR_USERNAME +``` + +é…ç½® Docker Daemon 使用代ç†æ¥æ‹‰å–é•œåƒï¼š + +``` +sudo mkdir -p /etc/systemd/system/docker.service.d +cat <<'EOF' | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf > /dev/null +[Service] +Environment="HTTP_PROXY=http://${proxy_host}" +Environment="HTTPS_PROXY=http://${proxy_host}" +Environment="NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16" +EOF +``` + +è¦åº”用é…置，请é‡æ–°å¯åŠ¨ Docker Daemon： + +``` +sudo systemctl daemon-reload +sudo systemctl restart docker +``` + +#### ç¦»çº¿ä»£ç† + +你现在å¯ä»¥åœ¨é…置的离线集群中é…置主机驱动集群，以使用代ç†è¿›è¡Œå‡ºç«™è¿žæŽ¥ã€‚ + +除了为代ç†æœåŠ¡å™¨è®¾ç½®é»˜è®¤è§„则外，你还需è¦é¢å¤–添加如下所示的规则，以从代ç†çš„ Rancher 环境中é…置主机驱动集群。 + +æ ¹æ®ä½ çš„设置é…置文件路径，例如 `/etc/apt/apt.conf.d/proxy.conf`： + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` + +### 创建 RKE 集群 + +在能通过 SSH 访问 Linux 节点的主机上，你需è¦æœ‰å‡ ä¸ªå‘½ä»¤è¡Œå·¥å…·ï¼Œæ¥åˆ›å»ºé›†ç¾¤å¹¶ä¸Žä¹‹äº¤äº’： + +* [RKE CLI binary](https://rancher.com/docs/rke/latest/en/installation/#download-the-rke-binary) + +``` +sudo curl -fsSL -o /usr/local/bin/rke https://github.com/rancher/rke/releases/download/v1.1.4/rke_linux-amd64 +sudo chmod +x /usr/local/bin/rke +``` + +* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + +``` +curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +``` + +接下æ¥ï¼Œåˆ›å»ºä¸€ä¸ªæè¿° RKE 集群的 YAML 文件。确ä¿èŠ‚点的 IP 地å€å’Œ SSH 用户å是正确的。有关集群 YAML 的详情，请å‚è§ [RKE 官方文档](https://rancher.com/docs/rke/latest/en/example-yamls/)。 + +```yml +nodes: + - address: 10.0.1.200 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.201 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.202 + user: ubuntu + role: [controlplane,worker,etcd] + +services: + etcd: + backup_config: + interval_hours: 12 + retention: 6 +``` + +之åŽï¼Œä½ å¯ä»¥é€šè¿‡è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ›å»º Kubernetes 集群： + +``` +rke up --config rancher-cluster.yaml +``` + +RKE 会创建一个å为 `rancher-cluster.rkestate` 的状æ€æ–‡ä»¶ã€‚如果你需è¦æ›´æ–°æˆ–修改集群é…置，或使用备份æ¢å¤é›†ç¾¤ï¼Œåˆ™éœ€è¦ä½¿ç”¨è¯¥æ–‡ä»¶ã€‚RKE 还会创建一个 `kube_config_cluster.yaml` 文件，你å¯ä»¥ä½¿ç”¨è¯¥æ–‡ä»¶åœ¨æœ¬åœ°ä½¿ç”¨ kubectl 或 Helm 等工具连接到远端的 Kubernetes 集群。请将这些文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ï¼Œä¾‹å¦‚版本控制系统中。 + +如需查看集群，请è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +export KUBECONFIG=kube_config_cluster.yaml +kubectl cluster-info +kubectl get pods --all-namespaces +``` + +你也å¯ä»¥éªŒè¯ä½ çš„外部负载å‡è¡¡å™¨æ˜¯å¦å·¥ä½œï¼ŒDNS æ¡ç›®æ˜¯å¦è®¾ç½®æ­£ç¡®ã€‚如果你å‘其中之一å‘é€è¯·æ±‚，你会收到æ¥è‡ª Ingress Controller çš„ HTTP 404 å“应： + +``` +$ curl 10.0.1.100 +default backend - 404 +$ curl rancher.example.com +default backend - 404 +``` + +### ä¿å­˜ä½ çš„文件 + +:::note é‡è¦æ示： + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +::: + +将以下文件的副本ä¿å­˜åœ¨å®‰å…¨ä½ç½®ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_cluster.yml`：集群的 [Kubeconfig 文件](https://rancher.com/docs/rke/latest/en/kubeconfig/)。该文件包å«å¯å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚ +- `rancher-cluster.rkestate`：[Kubernetes 集群状æ€æ–‡ä»¶](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state)。此文件包å«é›†ç¾¤çš„当å‰çŠ¶æ€ï¼ŒåŒ…括 RKE é…置和è¯ä¹¦ã€‚ + +:::note + +åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +::: + + + + +### 故障排除 + +å‚è§[故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)页é¢ã€‚ + +### åŽç»­æ“作 +[安装 Rancher](install-rancher.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md new file mode 100644 index 00000000000..da7bd9fb728 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md @@ -0,0 +1,104 @@ +--- +title: 3. 安装 Rancher +--- + +在å‰æ–‡çš„æ“作åŽï¼Œä½ å·²ç»æœ‰äº†ä¸€ä¸ªè¿è¡Œçš„ RKE 集群，现在å¯ä»¥åœ¨å…¶ä¸­å®‰è£… Rancher 了。出于安全考虑，所有到 Rancher çš„æµé‡éƒ½å¿…须使用 TLS 加密。在本教程中，你将使用 [cert-manager](https://cert-manager.io/)自动é¢å‘自签åè¯ä¹¦ã€‚在实际使用情况下，你å¯ä½¿ç”¨ Let's Encrypt 或自己的è¯ä¹¦ã€‚ + +### 安装 Helm CLI + +在具有 kubeconfig 的主机上安装 [Helm](https://helm.sh/docs/intro/install/) CLI 以访问 Kubernetes 集群： + +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod +x get_helm.sh +sudo ./get_helm.sh +``` + +### 安装 cert-manager + +添加 cert-manager Helm 仓库： + +``` +helm repo add jetstack https://charts.jetstack.io +``` + +为 cert-manager 创建命å空间： + +``` +kubectl create namespace cert-manager +``` + +安装 cert-manager çš„ CustomResourceDefinitions： + +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml +``` + +使用 Helm 安装 cert-manager。请注æ„，cert-manager 还需è¦ä½ é…置代ç†ï¼Œä»¥é˜²å®ƒéœ€è¦ä¸Ž Let's Encrypt 或其他外部è¯ä¹¦é¢å‘商进行通信： + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +``` +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --version v1.11.0 \ + --set http_proxy=http://${proxy_host} \ + --set https_proxy=http://${proxy_host} \ + --set no_proxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +等待 cert-manager 完æˆå¯åŠ¨ï¼š + +``` +kubectl rollout status deployment -n cert-manager cert-manager +kubectl rollout status deployment -n cert-manager cert-manager-webhook +``` + +### 安装 Rancher + +接下æ¥ï¼Œä½ å¯ä»¥å®‰è£… Rancher 了。首先，添加 Helm 仓库： + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +``` + +创建命å空间： + +``` +kubectl create namespace cattle-system +``` + +然åŽä½¿ç”¨ Helm 安装 Rancher：Rancher 也需è¦ä½ é…置代ç†ï¼Œä»¥ä¾¿å®ƒå¯ä»¥ä¸Žå¤–部应用商店通信，或检索 Kubernetes 版本更新元数æ®ï¼š + +``` +helm upgrade --install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.example.com \ + --set proxy=http://${proxy_host} \ + --set noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +等待部署完æˆï¼š + +``` +kubectl rollout status deployment -n cattle-system rancher +``` + +现在，你å¯ä»¥å¯¼èˆªåˆ° `https://rancher.example.com` 并开始使用 Rancher。 + +:::caution + +如果你ä¸æƒ³å‘é€é¥æµ‹æ•°æ®ï¼Œåœ¨é¦–次登录时退出[é¥æµ‹](../../../../faq/telemetry.md)。如果在离线安装的环境中让这个功能处于 active 状æ€ï¼Œsocket å¯èƒ½æ— æ³•æ‰“开。 + +::: + +### å…¶ä»–èµ„æº + +以下资æºå¯èƒ½å¯¹å®‰è£… Rancher 有帮助： + +- [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md) +- [添加 TLS 密文](../../resources/add-tls-secrets.md) +- [Rancher Kubernetes 安装的故障排除](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md new file mode 100644 index 00000000000..58414b8ed1d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md @@ -0,0 +1,64 @@ +--- +title: '1. é…置基础设施' +--- + +在本节中，你将为 Rancher Management Server é…置底层基础设施，并使其通过 HTTP 代ç†è®¿é—®äº’è”网。 + +如需在高å¯ç”¨ RKE 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商（例如 Amazon EC2，GCE 或 vSphere）中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将å‰ç«¯æµé‡è½¬å‘到这三个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸæˆ–æ•°æ®ä¸­å¿ƒã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºã€‚ + +### 为什么使用三个节点？ + +在 RKE 集群中，Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。而这个 etcd æ•°æ®åº“在这三个节点上è¿è¡Œã€‚ + +为了选举出大多数 etcd 节点认å¯çš„ etcd 集群 leader，etcd æ•°æ®åº“需è¦å¥‡æ•°ä¸ªèŠ‚点。如果 etcd æ•°æ®åº“无法选出 leader，etcd å¯èƒ½ä¼šå‡ºçŽ°[脑裂（split brain）](https://www.quora.com/What-is-split-brain-in-distributed-systems)的问题，此时你需è¦ä½¿ç”¨å¤‡ä»½æ¢å¤é›†ç¾¤ã€‚如果三个 etcd 节点之一å‘生故障，其余两个节点å¯ä»¥é€‰æ‹©ä¸€ä¸ª leader，因为它们是 etcd 节点总数的大多数部分。 + +### 1. é…ç½® Linux 节点 + +这些主机将通过 HTTP 代ç†è¿žæŽ¥åˆ°äº’è”网。 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md)。 + +:::note é‡è¦æ示： + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 3. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 + + +### åŽç»­æ“作 +[é…ç½® Kubernetes 集群](install-kubernetes.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md new file mode 100644 index 00000000000..6e1ac8ee365 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md @@ -0,0 +1,90 @@ +--- +title: è¯ä¹¦æ•…障排除 +--- + +### 如何确定我的è¯ä¹¦æ ¼å¼æ˜¯å¦ä¸º PEM？ + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ç‰¹å¾è¯†åˆ« PEM æ ¼å¼ï¼š + +- 文件开始的标头： + ``` + -----BEGIN CERTIFICATE----- + ``` +- 表头åŽè·Ÿä¸€é•¿ä¸²å­—符。 +- 文件结æŸçš„页脚： + ``` + -----END CERTIFICATE----- + ``` + +PEM è¯ä¹¦ç¤ºä¾‹ï¼š + +``` +----BEGIN CERTIFICATE----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END CERTIFICATE----- +``` + +PEM è¯ä¹¦å¯†é’¥ç¤ºä¾‹ï¼š + +``` +-----BEGIN RSA PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END RSA PRIVATE KEY----- +``` + +如果你的密钥与以下示例类似，请å‚è§[å°† PKCS8 è¯ä¹¦å¯†é’¥è½¬æ¢ä¸º PKCS1](#å°†-pkcs8-è¯ä¹¦å¯†é’¥è½¬æ¢ä¸º-pkcs1)。 + +``` +-----BEGIN PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END PRIVATE KEY----- +``` + +### å°† PKCS8 è¯ä¹¦å¯†é’¥è½¬æ¢ä¸º PKCS1 + +如果你使用的是 PKCS8 è¯ä¹¦å¯†é’¥æ–‡ä»¶ï¼ŒRancher 将打å°ä»¥ä¸‹æ—¥å¿—： + +``` +ListenConfigController cli-config [listener] failed with : failed to read private key: asn1: structure error: tags don't match (2 vs {class:0 tag:16 length:13 isCompound:true}) +``` + +为了能正常工作，你需è¦è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œå°†å¯†é’¥ä»Ž PKCS8 转æ¢ä¸º PKCS1： + +``` +openssl rsa -in key.pem -out convertedkey.pem +``` + +ä½ å¯ä½¿ç”¨ `convertedkey.pem` 作为 Rancher è¯ä¹¦å¯†é’¥æ–‡ä»¶ã€‚ + +### 添加中间è¯ä¹¦çš„顺åºæ˜¯ä»€ä¹ˆï¼Ÿ + +添加è¯ä¹¦çš„顺åºå¦‚下： + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +### 如何验è¯æˆ‘çš„è¯ä¹¦é“¾ï¼Ÿ + +ä½ å¯ä½¿ç”¨ `openssl` 二进制文件æ¥éªŒè¯è¯ä¹¦é“¾ã€‚如果命令的输出以 `Verify return code: 0 (ok)` 结尾（å‚è§ä»¥ä¸‹ç¤ºä¾‹ï¼‰ï¼Œä½ çš„è¯ä¹¦é“¾æ˜¯æœ‰æ•ˆçš„。`ca.pem` 文件必须与你添加到 `rancher/rancher` 容器中的文件一致。 + +如果你使用由å¯ä¿¡çš„ CA ç­¾å‘çš„è¯ä¹¦ï¼Œå¯çœç•¥ `-CAfile` å‚数。 + +命令： + +``` +openssl s_client -CAfile ca.pem -connect rancher.yourdomain.com:443 +... + Verify return code: 0 (ok) +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md new file mode 100644 index 00000000000..32fbf2f4254 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md @@ -0,0 +1,91 @@ +--- +title: 回滚 Docker 安装的 Rancher +--- + +如果 Rancher å‡çº§æ²¡æœ‰æˆåŠŸå®Œæˆï¼Œä½ éœ€è¦å›žæ»šåˆ°ä½ åœ¨ [Docker å‡çº§](upgrade-docker-installed-rancher.md)之å‰ä½¿ç”¨çš„ Rancher 设置。回滚å¯ä»¥æ¢å¤ï¼š + +- å…ˆå‰ç‰ˆæœ¬çš„ Rancher。 +- å‡çº§å‰åˆ›å»ºçš„æ•°æ®å¤‡ä»½ã€‚ + +## åœ¨ä½ å¼€å§‹å‰ + +åœ¨å›žæ»šåˆ°å…ˆå‰ Rancher 版本的过程中，你将输入一系列命令。请按照你环境的实际情况替æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。以下是带有å ä½ç¬¦çš„命令示例： + +``` +docker pull rancher/rancher: +``` + +在此命令中，`` 是å‡çº§å¤±è´¥ä¹‹å‰è¿è¡Œçš„ Rancher 版本，如 `v2.0.5`。 + +请交å‰å‚考下方的图片和表格，了解获å–æ­¤å ä½ç¬¦æ•°æ®çš„方法。在开始以下步骤之å‰ï¼Œè¯·å…ˆè®°ä¸‹æˆ–å¤åˆ¶æ­¤ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <PRIOR_RANCHER_VERSION> å’Œ <RANCHER_CONTAINER_NAME>![å ä½ç¬¦å‚考](/img/placeholder-ref-2.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | ------------------------------------------------------- | +| `` | `v2.0.5` | å‡çº§å‰ä½¿ç”¨çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.0.5` | 备份对应的 Rancher 版本。 | +| `` | `9-27-18` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ `` 。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## 回滚 Rancher + +如果你在å‡çº§ Rancher æ—¶é‡åˆ°é—®é¢˜ï¼Œä½ å¯æ‹‰å–å…ˆå‰ä½¿ç”¨çš„é•œåƒå¹¶æ¢å¤åœ¨å‡çº§å‰æ‰€åšçš„备份，从而将 Rancher 回滚到之å‰çš„正常工作状æ€ã€‚ + +:::danger + +回滚到先å‰çš„ Rancher 版本会破å你在å‡çº§åŽå¯¹ Rancher åšå‡ºçš„所有更改。丢失的数æ®å¯èƒ½æ— æ³•æ¢å¤ã€‚ + +::: + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. 拉å–å‡çº§å‰è¿è¡Œçš„ Rancher 版本。把 `` 替æ¢ä¸ºè¯¥ç‰ˆæœ¬ã€‚ + + 例如，如果å‡çº§ä¹‹å‰è¿è¡Œçš„是 Rancher v2.0.5ï¼Œè¯·æ‹‰å– v2.0.5。 + + ``` + docker pull rancher/rancher: + ``` + +1. åœæ­¢å½“å‰è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸ºä½ çš„ Rancher 容器的å称： + + ``` + docker stop + ``` + ä½ å¯è¾“å…¥ `docker ps`èŽ·å– Rancher 容器的å称。 + +1. 将你在 [Docker å‡çº§](upgrade-docker-installed-rancher.md)时创建的备份压缩包移动到 Rancher Server。切æ¢åˆ°ä½ å°†å…¶ç§»åŠ¨åˆ°çš„目录。输入 `dir` 以确认它在该ä½ç½®ã€‚ + + 如果你éµå¾ªäº†æˆ‘们在 [Docker å‡çº§](upgrade-docker-installed-rancher.md)中推è的命åæ–¹å¼ï¼Œå®ƒçš„å称会与 `rancher-data-backup--.tar.gz` 类似。 + +1. 替æ¢å ä½ç¬¦æ¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œå°† `rancher-data` 容器中的数æ®æ›¿æ¢ä¸ºå¤‡ä»½åŽ‹ç¼©åŒ…中的数æ®ã€‚ä¸è¦å¿˜è®°å…³é—­å¼•å·ã€‚ + + ``` + docker run --volumes-from rancher-data \ + -v $PWD:/backup busybox sh -c "rm /var/lib/rancher/* -rf \ + && tar zxvf /backup/rancher-data-backup--.tar.gz" + ``` + +1. å°† `` å ä½ç¬¦æŒ‡å‘æ•°æ®å®¹å™¨ï¼Œå¯åŠ¨ä¸€ä¸ªæ–°çš„ Rancher Server 容器。 + ``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: + ``` + 特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + + :::danger + + å¯åŠ¨å›žæ»šåŽï¼Œå³ä½¿å›žæ»šè€—时比预期长，也 **_ä¸è¦_** åœæ­¢å›žæ»šã€‚如果你åœæ­¢å›žæ»šï¼Œå¯èƒ½ä¼šå¯¼è‡´ä¹‹åŽçš„å‡çº§ä¸­å‡ºçŽ°æ•°æ®åº“错误。 + + ::: + +1. 等待片刻，然åŽåœ¨æµè§ˆå™¨ä¸­æ‰“å¼€ Rancher。确认回滚æˆåŠŸå¹¶ä¸”ä½ çš„æ•°æ®å·²æ¢å¤ã€‚ + +**结果**：Rancher 回滚到å‡çº§å‰çš„版本和数æ®çŠ¶æ€ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md new file mode 100644 index 00000000000..68d23b34c33 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -0,0 +1,393 @@ +--- +title: å‡çº§ Docker 安装的 Rancher +--- + +本文介ç»å¦‚何å‡çº§é€šè¿‡ Docker 安装的 Rancher Server。 + +:::caution + +**生产环境ä¸æ”¯æŒ Docker 安装**。这些说明仅适用于测试和开å‘。如果你已ç»åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­éƒ¨ç½²äº† Docker 安装并且需è¦å‡çº§åˆ°æ–°çš„ Rancher 版本，我们建议你在å‡çº§å‰å…ˆ[è¿ç§»åˆ° Helm Chart 安装](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +::: + +## 先决æ¡ä»¶ + +- 在 Rancher 文档中**检查[已知å‡çº§é—®é¢˜](../../install-upgrade-on-a-kubernetes-cluster/upgrades.md#已知å‡çº§é—®é¢˜)**，了解å‡çº§ Rancher 时最需è¦æ³¨æ„的问题。你å¯ä»¥åœ¨ [GitHub](https://github.com/rancher/rancher/releases) å‘å¸ƒè¯´æ˜Žä»¥åŠ [Rancher 论å›](https://forums.rancher.com/c/announcements/12)中找到æ¯ä¸ª Rancher 版本的已知问题。ä¸æ”¯æŒå‡çº§æˆ–å‡çº§åˆ° [rancher-alpha 仓库](../../resources/choose-a-rancher-version.md#helm-chart-仓库)中的任何 Chart。 +- **[仅适用于离线安装](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md)：为新的 Rancher Server 版本收集和推é€é•œåƒ**。按照指å—为你想è¦å‡çº§çš„目标 Rancher 版本[推é€é•œåƒåˆ°ç§æœ‰é•œåƒä»“库](../air-gapped-helm-cli-install/publish-images.md)。 + +## å ä½ç¬¦ + +在å‡çº§è¿‡ç¨‹ä¸­ï¼Œä½ å°†è¾“入一系列命令。请按照你环境的实际情况替æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。 + +以下是带有å ä½ç¬¦çš„命令**示例**： + +``` +docker stop +``` + +在此命令中，`` 是你的 Rancher 容器的å称。 + +## 获å–å‡çº§å‘½ä»¤çš„æ•°æ® + +è¦èŽ·å–替æ¢å ä½ç¬¦çš„æ•°æ®ï¼Œè¯·è¿è¡Œï¼š + +``` +docker ps +``` + +在开始å‡çº§ä¹‹å‰è®°ä¸‹æˆ–å¤åˆ¶æ­¤ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <RANCHER_CONTAINER_TAG> å’Œ <RANCHER_CONTAINER_NAME> + +![å ä½ç¬¦å‚考](/img/placeholder-ref.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.1.3` | 首次安装拉å–çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.1.3` | 你为其创建备份的 Rancher 版本。 | +| `` | `2018-12-19` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ ``。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## å‡çº§ + +在å‡çº§æœŸé—´ï¼Œä½ å¯ä»¥ä¸ºå½“å‰ Rancher 容器创建数æ®çš„副本åŠå¤‡ä»½ï¼Œä»¥ç¡®ä¿å¯ä»¥åœ¨å‡çº§å‡ºçŽ°é—®é¢˜æ—¶å¯ä»¥è¿›è¡Œå›žæ»šã€‚然åŽï¼Œä½ å¯ä½¿ç”¨çŽ°æœ‰æ•°æ®å°†æ–°ç‰ˆæœ¬çš„ Rancher 部署到新容器中。 +### 1. 创建 Rancher Server 容器的数æ®å‰¯æœ¬ + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. åœæ­¢æ­£åœ¨è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸ºä½ çš„ Rancher 容器的å称： + + ``` + docker stop + ``` + +1. è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä»Žåˆšæ‰åœæ­¢çš„ Rancher 容器创建一个数æ®å®¹å™¨ã€‚请替æ¢å‘½ä»¤ä¸­çš„å ä½ç¬¦ï¼š + + ``` + docker create --volumes-from --name rancher-data rancher/rancher: + ``` + +### 2. 创建备份压缩包 + +1. 从你刚刚创建的数æ®å®¹å™¨ï¼ˆrancher-data）中，创建一个备份 tar 包（rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz）。 + + 如果å‡çº§æœŸé—´å‡ºçŽ°é—®é¢˜ï¼Œæ­¤åŽ‹ç¼©åŒ…å¯ä»¥ç”¨ä½œå›žæ»šç‚¹ã€‚替æ¢å ä½ç¬¦æ¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚ + ``` + docker run --volumes-from rancher-data -v "$PWD:/backup" --rm busybox tar zcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **步骤结果**：你输入此命令时，会è¿è¡Œä¸€ç³»åˆ—命令。 + +1. 输入 `ls` 命令，确认备份压缩包已创建æˆåŠŸã€‚压缩包的å称格å¼ç±»ä¼¼ `rancher-data-backup--.tar.gz`。 + + ``` + [rancher@ip-10-0-0-50 ~]$ ls + rancher-data-backup-v2.1.3-20181219.tar.gz + ``` + +1. 将备份压缩包移动到 Rancher Server 外的安全ä½ç½®ã€‚ + +### 3. 拉å–æ–°çš„ Docker é•œåƒ + +拉å–你需è¦å‡çº§åˆ°çš„ Rancher 版本镜åƒã€‚ + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker pull rancher/rancher: +``` + +### 4. å¯åŠ¨æ–°çš„ Rancher Server 容器 + +使用 `rancher-data` 容器中的数æ®å¯åŠ¨ä¸€ä¸ªæ–°çš„ Rancher Server 容器。记ä½è¦ä¼ å…¥å¯åŠ¨åŽŸå§‹å®¹å™¨æ—¶ä½¿ç”¨çš„所有环境å˜é‡ã€‚ + +:::danger + +å¯åŠ¨å‡çº§åŽï¼Œå³ä½¿å‡çº§è€—时比预期长，也 **_ä¸è¦_** åœæ­¢å‡çº§ã€‚如果你åœæ­¢å‡çº§ï¼Œå¯èƒ½ä¼šå¯¼è‡´ä¹‹åŽçš„å‡çº§ä¸­å‡ºçŽ°æ•°æ®åº“è¿ç§»é”™è¯¯ã€‚ + +::: + +如果你使用代ç†ï¼Œè¯·å‚è§ [HTTP 代ç†é…ç½®](../../../../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md)。 + +如果你é…置了自定义 CA æ ¹è¯ä¹¦æ¥è®¿é—®æœåŠ¡ï¼Œè¯·å‚è§[自定义 CA æ ¹è¯ä¹¦](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#自定义-ca-è¯ä¹¦)。 + +如果你è¦è®°å½•æ‰€æœ‰ Rancher API 的事务，请å‚è§ [API 审计](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志)。 + +如需查看å¯åŠ¨æ–° Rancher Server 容器时使用的命令，从以下的选项中进行选择： + +- Docker å‡çº§ +- 离线安装的 Docker å‡çº§ + + + + +选择你安装 Rancher Server 时用的选项 + +#### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你使用 Rancher 生æˆçš„自签åè¯ä¹¦ï¼Œåˆ™å°† `--volumes-from rancher-data` 添加到你å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中。 + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +
+ +#### 选项 B：使用你自己的è¯ä¹¦ - 自签å + +
+å•å‡»å±•å¼€ + +如果你选择使用自己的自签åè¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚ + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher: +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +
+ +#### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你选择使用å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚注æ„è¦ä½¿ç”¨ `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括å¯ä¿¡ CA æ供的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](certificate-troubleshooting.md)。 + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher: \ + --no-cacerts +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ +#### 选项 D：Let's Encrypt è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +:::caution + +Let's Encrypt 对新è¯ä¹¦è¯·æ±‚有频率é™åˆ¶ã€‚因此，请é™åˆ¶åˆ›å»ºæˆ–销æ¯å®¹å™¨çš„频率。详情请å‚è§ [Let's Encrypt 官方文档 - 频率é™åˆ¶](https://letsencrypt.org/docs/rate-limits/)。 + +::: + +如果你选择使用 [Let's Encrypt](https://letsencrypt.org/) è¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`，并且æ供最åˆå®‰è£… Rancher 时使用的域å。 + +:::note è¯ä¹¦è¦æ±‚æ示： + +- 在 DNS 中创建一æ¡è®°å½•ï¼Œå°† Linux 主机 IP 地å€ç»‘定到è¦ç”¨äºŽè®¿é—® Rancher 的主机å（例如，`rancher.mydomain.com`）。 +- 在 Linux 主机上打开 `TCP/80` 端å£ã€‚Let's Encrypt çš„ HTTP-01 质询å¯ä»¥æ¥è‡ªä»»ä½•æº IP 地å€ï¼Œå› æ­¤ç«¯å£ `TCP/80` 必须开放开所有 IP 地å€ã€‚ + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | +| `` | 你最åˆä½¿ç”¨çš„域å | + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: \ + --acme-domain +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +
+ + + + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +å¯åŠ¨æ–°çš„ Rancher Server 容器时，从以下的选项中进行选择： + +#### 选项 A：使用 Rancher 默认的自签åè¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你使用 Rancher 生æˆçš„自签åè¯ä¹¦ï¼Œåˆ™å°† `--volumes-from rancher-data` 添加到你å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中。 + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ +#### 选项 B：使用你自己的è¯ä¹¦ - 自签å + +
+ å•å‡»å±•å¼€ + +如果你选择使用自己的自签åè¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](certificate-troubleshooting.md)。 + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged \ + /rancher/rancher: +``` +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ +#### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +
+ å•å‡»å±•å¼€ + +如果你选择使用å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œåˆ™åœ¨å¯åŠ¨åŽŸå§‹ Rancher Server 容器的命令中添加 `--volumes-from rancher-data`。此外，你需è¦èƒ½å¤Ÿè®¿é—®ä½ åŽŸå§‹å®‰è£…时使用的è¯ä¹¦ã€‚ + +:::note è¯ä¹¦è¦æ±‚æ示： + +è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括å¯ä¿¡ CA æ供的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](certificate-troubleshooting.md)。 + +::: + +| å ä½ç¬¦ | æè¿° | +------------|------------- +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | ç§æœ‰é•œåƒä»“库的 URL 和端å£ã€‚ | +| `` | 你想è¦å‡çº§åˆ°çš„ [Rancher 版本](../../installation-references/helm-chart-options.md)的版本标签。 | + +:::note + +使用 `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +::: + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + -e CATTLE_SYSTEM_CATALOG=bundled \ # 使用打包的 Rancher System Chart + --privileged + /rancher/rancher: +``` +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 +
+ + + + +**结果**：你已å‡çº§ Rancher。已å‡çº§ Server 中的数æ®å°†ä¿å­˜åœ¨ `rancher-data` 容器中，用于将æ¥çš„å‡çº§ã€‚ + +### 5. 验è¯å‡çº§ + +登录到 Rancher。通过检查æµè§ˆå™¨å·¦ä¸‹è§’的版本å·ï¼Œç¡®è®¤å‡çº§æ˜¯å¦æˆåŠŸã€‚ + +:::note å‡çº§åŽä¸‹æ¸¸é›†ç¾¤å‡ºçŽ°ç½‘络问题？ + +请å‚è§[æ¢å¤é›†ç¾¤ç½‘络](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md)。 + +::: + +### 6. 清ç†æ—§çš„ Rancher Server 容器 + +移除旧的 Rancher Server 容器。如果你仅åœæ­¢äº†æ—§çš„ Rancher Server 容器，但没有移除它，该容器还å¯èƒ½åœ¨æœåŠ¡å™¨ä¸‹æ¬¡é‡å¯åŽé‡æ–°å¯åŠ¨ã€‚ + +## 回滚 + +如果å‡çº§æ²¡æœ‰æˆåŠŸå®Œæˆï¼Œä½ å¯ä»¥å°† Rancher Server åŠå…¶æ•°æ®å›žæ»šåˆ°ä¸Šæ¬¡çš„å¥åº·çŠ¶æ€ã€‚详情请å‚è§ [Docker 回滚](roll-back-docker-installed-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/add-tls-secrets.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/add-tls-secrets.md new file mode 100644 index 00000000000..dc58f216be0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/add-tls-secrets.md @@ -0,0 +1,45 @@ +--- +title: 添加 TLS 密文 +--- + +我们使用è¯ä¹¦å’Œå¯†é’¥å°† `cattle-system` 命å空间中的 `tls-rancher-ingress` 密文é…置好åŽï¼ŒKubernetes 会为 Rancher 创建对象和æœåŠ¡ã€‚ + +å°†æœåŠ¡å™¨è¯ä¹¦å’Œæ‰€éœ€çš„所有中间è¯ä¹¦åˆå¹¶åˆ°å为 `tls.crt`的文件中。将è¯ä¹¦å¯†é’¥å¤åˆ¶åˆ°å为 `tls.key` 的文件中。 + +例如，[acme.sh](https://acme.sh) 在 `fullchain.cer` 文件中æä¾›æœåŠ¡å™¨è¯ä¹¦å’Œ CA 链。 +请将 `fullchain.cer` 命å为 `tls.crt`，将è¯ä¹¦å¯†é’¥æ–‡ä»¶å‘½å为 `tls.key`。 + +使用 `kubectl` 创建 `tls` 类型的密文。 + +``` +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +:::note + +如需替æ¢è¯ä¹¦ï¼Œä½ å¯ä»¥è¿è¡Œ `kubectl -n cattle-system delete secret tls-rancher-ingress` æ¥åˆ é™¤ `tls-rancher-ingress` 密文，然åŽè¿è¡Œä¸Šæ–¹å‘½ä»¤æ¥æ·»åŠ æ–°çš„密文。如果你使用的是ç§æœ‰ CA ç­¾åè¯ä¹¦ï¼Œä»…当新è¯ä¹¦ä¸Žå½“å‰è¯ä¹¦æ˜¯ç”±åŒä¸€ä¸ª CA ç­¾å‘的，æ‰å¯ä»¥æ›¿æ¢ã€‚ + +::: + +## 使用ç§æœ‰ CA ç­¾åè¯ä¹¦ + +如果你使用的是ç§æœ‰ CA，Rancher 需è¦ç§æœ‰ CA çš„æ ¹è¯ä¹¦æˆ–è¯ä¹¦é“¾çš„副本，Rancher Agent 使用它æ¥æ ¡éªŒä¸Ž Server 的连接。 + +创建一个å为 `cacerts.pem` 的文件，该文件仅包å«ç§æœ‰ CA çš„æ ¹ CA è¯ä¹¦æˆ–è¯ä¹¦é“¾ï¼Œå¹¶ä½¿ç”¨ `kubectl` 在 `cattle-system` 命å空间中创建 `tls-ca` Secret。 + +``` +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem=./cacerts.pem +``` + +:::note + +Rancher å¯åŠ¨æ—¶ä¼šæ£€ç´¢é…置的 `tls-ca` 密文。如果 Rancher 在è¿è¡Œä¸­ï¼Œæ›´æ–°çš„ CA 会在新的 Rancher Pod å¯åŠ¨åŽç”Ÿæ•ˆã€‚ + +::: + +## æ›´æ–°ç§æœ‰ CA è¯ä¹¦ + +按照[步骤](update-rancher-certificate.md)æ›´æ–° [Rancher 高å¯ç”¨ Kubernetes 安装](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)中的 Ingress，或从默认自签åè¯ä¹¦åˆ‡æ¢åˆ°è‡ªå®šä¹‰è¯ä¹¦ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/bootstrap-password.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/bootstrap-password.md new file mode 100644 index 00000000000..249b14115a0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/bootstrap-password.md @@ -0,0 +1,27 @@ +--- +title: å¼•å¯¼å¯†ç  +--- + +Rancher 首次å¯åŠ¨æ—¶ï¼Œä¼šä¸ºç¬¬ä¸€ä¸ªç®¡ç†å‘˜ç”¨æˆ·éšæœºç”Ÿæˆä¸€ä¸ªå¯†ç ã€‚当管ç†å‘˜é¦–次登录 Rancher 时，用于获å–引导密ç ï¼ˆBootstrap）的命令会在 UI 上显示。管ç†å‘˜éœ€è¦è¿è¡Œå‘½ä»¤å¹¶ä½¿ç”¨å¼•å¯¼å¯†ç ç™»å½•ã€‚ç„¶åŽ Rancher 会让管ç†å‘˜é‡ç½®å¯†ç ã€‚ + +如果你在安装过程中没有使用å˜é‡æ¥è®¾ç½®å¼•å¯¼å¯†ç ï¼Œåˆ™ä¼šéšæœºç”Ÿæˆå¼•å¯¼å¯†ç ã€‚如需了解使用å˜é‡è®¾ç½®å¼•å¯¼å¯†ç çš„详情，请å‚è§ä¸‹æ–‡ã€‚ + +### 在 Helm å®‰è£…ä¸­æŒ‡å®šå¼•å¯¼å¯†ç  + +Helm 安装的情况下，你å¯ä»¥ä½¿ç”¨ `.Values.bootstrapPassword` 在 Helm Chart 值中指定引导密ç å˜é‡ã€‚ + +密ç å°†å­˜å‚¨åœ¨ Kubernetes 密文中。安装 Rancher åŽï¼Œå¦‚何使用 kubectl 获å–密ç çš„说明将会在 UI 中显示： + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### 在 Docker å®‰è£…ä¸­æŒ‡å®šå¼•å¯¼å¯†ç  + +如果 Rancher 是使用 Docker 安装的，你å¯ä»¥é€šè¿‡åœ¨ Docker 安装命令中传递 `-e CATTLE_BOOTSTRAP_PASSWORD=password` æ¥æŒ‡å®šå¼•å¯¼å¯†ç ã€‚ + +密ç å°†å­˜å‚¨åœ¨ Docker 容器日志中。安装 Rancher åŽï¼Œå¦‚何使用 Docker 容器 ID 获å–密ç çš„说明将会在 UI 中显示： + +``` +docker logs container-id 2>&1 | grep "Bootstrap Password:" +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md new file mode 100644 index 00000000000..b168ecdcb05 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md @@ -0,0 +1,118 @@ +--- +title: 选择 Rancher 版本 +--- + +本节介ç»å¦‚何选择 Rancher 版本。 + +在我们推è用于生产环境的 Rancher 高å¯ç”¨å®‰è£…中，Rancher Server 是通过 Kubernetes 集群上的 **Helm Chart** 安装的。请å‚è§ [Helm 版本è¦æ±‚](helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。 + +如果你在开å‘和测试中使用 Docker æ¥å®‰è£… Rancher，你需è¦æŠŠ Rancher 作为一个 **Docker é•œåƒ**æ¥å®‰è£…。 + + + + +如果 Rancher Server 是[安装在 Kubernetes 集群上](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)的，Rancher Server 的安装，å‡çº§å’Œå›žæ»šä¸­ï¼Œéƒ½æ˜¯ä½¿ç”¨ Kubernetes 集群上的 Helm Chart æ¥å®‰è£… Rancher 的。因此，在准备安装或å‡çº§ Rancher 高å¯ç”¨æ—¶ï¼Œå¿…须添加包å«ç”¨äºŽå®‰è£… Rancher çš„ Chart çš„ Helm Chart 仓库。 + +请å‚è§ [Helm 版本è¦æ±‚](helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。 + +### Helm Chart 仓库 + +Rancher æ供几个å¯é€‰çš„ Helm Chart 仓库供你选择。最新版或稳定版的 Helm Chart 仓库与用于 Docker 安装中的 Docker 标签对应。因此，`rancher-latest` 仓库包å«æ‰€æœ‰æ ‡è®°ä¸º `rancher/rancher:latest` çš„ Rancher 版本 Chart。当 Rancher 版本å‡çº§åˆ° `rancher/rancher:stable`，它会被添加到 `rancher-stable` 仓库中。 + +| 类型 | 添加仓库的命令 | 仓库æè¿° | +| -------------- | ------------ | ----------------- | +| rancher-latest | `helm repo add rancher-latest https://releases.rancher.com/server-charts/latest` | 添加最新版本的 Rancher çš„ Helm Chart 仓库。建议使用此仓库æ¥æµ‹è¯•æ–°ç‰ˆæœ¬çš„ Rancher。 | +| rancher-stable | `helm repo add rancher-stable https://releases.rancher.com/server-charts/stable` | 添加较旧的，稳定的版本的 Rancher çš„ Helm Chart 仓库。建议在生产环境中使用此仓库。 | +| rancher-alpha | `helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha` | 添加 alpha 版本的 Rancher çš„ Helm Chart 仓库，以预览å³å°†å‘布的版本。ä¸å»ºè®®åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­ä½¿ç”¨è¿™äº›ç‰ˆæœ¬ã€‚无论是什么仓库，å‡ä¸æ”¯æŒ _å‡çº§_ 或 _å‡çº§åˆ°_ rancher-alpha 仓库中的任何 Chart。 | + +了解何时选择这些仓库，请å‚è§[切æ¢åˆ°ä¸åŒ Helm Chart 仓库](#切æ¢åˆ°ä¸åŒ-helm-chart-仓库)。 + +:::note + +`rancher-stable` 仓库中的所有 Chart 都与 `stable` 标记的 Rancher 版本对应。 + +::: + +### Helm Chart 版本 + +Rancher Helm Chart 版本与 Rancher ç‰ˆæœ¬ï¼ˆå³ `appVersion`）对应。添加仓库åŽï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤æœç´¢å¯ç”¨ç‰ˆæœ¬ï¼š
+    `helm search repo --versions` + +如果你有多个仓库，你å¯æŒ‡å®šä»“库å称，å³ï¼š`helm search repo rancher-stable/rancher --versions`
+详情请访问 https://helm.sh/docs/helm/helm_search_repo/ + +è¦èŽ·å–所选仓库的指定版本，å‚è§å¦‚下示例指定 `--version` å‚数：
+    `helm fetch rancher-stable/rancher --version=2.4.8` + +### 切æ¢åˆ°ä¸åŒ Helm Chart 仓库 + +安装 Rancher åŽï¼Œå¦‚果想修改安装 Rancher çš„ Helm Chart 仓库，按照以下步骤æ“作。 + +:::note + +由于 rancher-alpha 仓库åªåŒ…å« alpha 版本 Chart，因此ä¸æ”¯æŒä»Ž rancher alpha 仓库切æ¢åˆ° rancher-stable 或 rancher-latest 仓库以进行å‡çº§ã€‚ + +::: + +- Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + +1. åˆ—å‡ºå½“å‰ Helm Chart 仓库。 + + ```plain + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + +2. 删除包å«å®‰è£… Rancher 时用的 Chart çš„ Helm Chart 仓库。是 `rancher-stable` 或 `rancher-latest` å–决于你åˆå§‹å®‰è£…时的选择。 + + ```plain + helm repo remove rancher- + ``` + +3. 添加你è¦ç”¨äºŽå®‰è£… Rancher çš„ Helm Chart 仓库。 + + ```plain + helm repo add rancher- https://releases.rancher.com/server-charts/ + ``` + +4. 按照以下步骤，从新的 Helm Chart 仓库[å‡çº§ Rancher](../install-upgrade-on-a-kubernetes-cluster/upgrades.md)。 + + + + +在执行 [Docker 安装](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md)ã€å‡çº§æˆ–回滚时，你å¯ä»¥ä½¿ç”¨ _tags_ æ¥å®‰è£…特定版本的 Rancher。 + +### Server 标签 + +Rancher Server 以 Docker é•œåƒçš„å½¢å¼åˆ†å‘并附有标签。你å¯ä»¥åœ¨è¾“入命令部署 Rancher 时指定标签。请记ä½ï¼Œå¦‚果你指定了标签，但是没有指定版本（如 `latest` 或 `stable`），你必须显å¼æ‹‰å–该镜åƒæ ‡ç­¾çš„新版本。å¦åˆ™ï¼Œå°†ä½¿ç”¨ç¼“存在主机上的镜åƒã€‚ + +| 标签 | æè¿° | +| -------------------------- | ------ | +| `rancher/rancher:latest` | 最新的开å‘版本。这些版本通过了我们的 CI 自动化验è¯ã€‚ä¸æŽ¨è在生产环境使用这些版本。 | +| `rancher/rancher:stable` | 最新的稳定版本。推è将此标签用于生产环境。 | +| `rancher/rancher:` | ä½ å¯ä»¥ä½¿ç”¨ä»¥å‰ç‰ˆæœ¬ä¸­çš„标签æ¥æŒ‡å®šè¦å®‰è£…çš„ Rancher 版本。访问 DockerHub 查看å¯ç”¨çš„版本。 | + +:::note + +- `master` 和带有 `-rc` 或其他åŽç¼€çš„标签是供 Rancher 测试团队验è¯ç”¨çš„。这些标签ä¸å—官方支æŒï¼Œå› æ­¤è¯·ä¸è¦ä½¿ç”¨è¿™äº›æ ‡ç­¾ã€‚ +- 安装 alpha 版本进行预览：使用我们的[公告页é¢](https://forums.rancher.com/c/announcements)中列出的 alpha 标签（例如，`v2.2.0-alpha1`）进行安装。ä¸æ”¯æŒå‡çº§æˆ–å‡çº§åˆ° Alpha 版本。 + +::: + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md new file mode 100644 index 00000000000..d454197c8a5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md @@ -0,0 +1,24 @@ +--- +title: 自定义 CA æ ¹è¯ä¹¦ +--- + +如果你在内部生产环境使用 Rancher，且ä¸æ‰“算公开暴露应用，你å¯ä»¥ä½¿ç”¨ä½¿ç”¨ç§æœ‰ CA é¢å‘çš„è¯ä¹¦ã€‚ + +Rancher å¯èƒ½ä¼šè®¿é—®é…置了自定义/内部 CA æ ¹è¯ä¹¦ï¼ˆä¹Ÿç§°ä¸ºè‡ªç­¾åè¯ä¹¦ï¼‰çš„æœåŠ¡ã€‚如果 Rancher 无法验è¯æœåŠ¡çš„è¯ä¹¦ï¼Œåˆ™ä¼šæ˜¾ç¤ºé”™è¯¯ä¿¡æ¯ `x509: certificate signed by unknown authority`。 + +如需验è¯è¯ä¹¦ï¼Œä½ éœ€è¦æŠŠ CA æ ¹è¯ä¹¦æ·»åŠ åˆ° Rancher。由于 Rancher 是用 Go 编写的，我们å¯ä»¥ä½¿ç”¨çŽ¯å¢ƒå˜é‡ `SSL_CERT_DIR` 指å‘容器中 CA æ ¹è¯ä¹¦æ‰€åœ¨çš„目录。å¯åŠ¨ Rancher 容器时，å¯ä»¥ä½¿ç”¨ Docker å·é€‰é¡¹ï¼ˆ`-v host-source-directory:container-destination-directory`）æ¥æŒ‚è½½ CA æ ¹è¯ä¹¦ç›®å½•ã€‚ + +Rancher å¯ä»¥è®¿é—®çš„æœåŠ¡ç¤ºä¾‹ï¼š + +- 应用商店 +- 验è¯æä¾›ç¨‹åº +- 使用 Node Driver 访问托管/云 API + +## 使用自定义 CA è¯ä¹¦å®‰è£… + +有关å¯åŠ¨æŒ‚载了ç§æœ‰ CA è¯ä¹¦çš„ Rancher 容器的详情，请å‚è§å®‰è£…文档： + +- [Docker 安装的自定义 CA è¯ä¹¦é€‰é¡¹](../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#自定义-ca-è¯ä¹¦) + +- [Kubernetes 安装的其他å—ä¿¡ CA 选项](../installation-references/helm-chart-options.md#é¢å¤–的授信-ca) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/helm-version-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/helm-version-requirements.md new file mode 100644 index 00000000000..273711a4de0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/helm-version-requirements.md @@ -0,0 +1,12 @@ +--- +title: Helm 版本è¦æ±‚ +--- + +æœ¬æ–‡ä»‹ç» Helm çš„è¦æ±‚。Helm 是用于把 Rancher 安装在高å¯ç”¨ Kubernetes 集群上的工具。 + +> 我们已针对 Helm 3 更新了安装指å—。如果你使用 Helm 2 进行安装，请å‚è§ [Helm 2 è¿ç§»åˆ° Helm 3 文档](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/)。[本文](/versioned_docs/version-2.0-2.4/pages-for-subheaders/helm2.md)æ供了较早的使用 Helm 2 çš„ Rancher 高å¯ç”¨å®‰è£…指å—的副本。如果你如果无法å‡çº§åˆ° Helm 3，å¯ä»¥ä½¿ç”¨è¿™ä¸ªè¯´æ˜Žå®‰è£…。 + +- 如需安装或å‡çº§ Rancher 2.5，请使用 Helm 3.2.x 或更高版本。 +- Kubernetes 1.16 è¦æ±‚ Helm 2.16.0 或更高版本。如果使用的是默认 Kubernetes 版本，请å‚è§[å‘行说明](https://github.com/rancher/rke/releases)获å–所使用的 RKE 版本。 +- 请ä¸è¦ä½¿ç”¨ Helm 2.15.0，因为这个版本有转æ¢/比较数字的问题。 +- 请ä¸è¦ä½¿ç”¨ Helm 2.12.0，因为该版本有 `cert-manager` 的兼容问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/local-system-charts.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/local-system-charts.md new file mode 100644 index 00000000000..4048c5b2566 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/local-system-charts.md @@ -0,0 +1,13 @@ +--- +title: 离线安装中设置本地 System Charts +--- + +[Charts](https://github.com/rancher/charts) ä»“åº“åŒ…å« Monitoringã€Loggingã€å‘Šè­¦å’Œ Istio 等功能所需的所有 Helm 目录项。 + +在 Rancher 的离线安装中，你需è¦é…ç½® Rancher 以使用 System Charts 的本地副本。本节介ç»å¦‚何通过 CLI 标志使用本地 System Charts。 + +## 使用本地 System Charts + +`system-charts` 的一个本地副本已ç»æ‰“包到 `rancher/rancher` 容器中。为了在离线安装中使用这些功能，你需è¦ä½¿ç”¨é¢å¤–的环境å˜é‡ `CATTLE_SYSTEM_CATALOG=bundled` æ¥è¿è¡Œ Rancher 安装命令，该环境å˜é‡å‘Šè¯‰ Rancher 使用 Chart 的本地副本，而ä¸æ˜¯å°è¯•ä»Ž GitHub èŽ·å– Chart。 + +带有 `system-charts` çš„ Rancher 安装命令示例包å«åœ¨ Docker å’Œ Helm çš„[离线安装说明](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md new file mode 100644 index 00000000000..3d11417cda6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md @@ -0,0 +1,263 @@ +--- +title: æ›´æ–° Rancher è¯ä¹¦ +--- + +## æ›´æ–°ç§æœ‰ CA è¯ä¹¦ + +按照以下步骤轮æ¢[安装在 Kubernetes 集群上](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)ã€ç”± Rancher 使用的 SSL è¯ä¹¦å’Œç§æœ‰ CA，或转用由ç§æœ‰ CA ç­¾å‘çš„ SSL è¯ä¹¦ã€‚ + +步骤概述： + +1. 使用新è¯ä¹¦å’Œç§é’¥åˆ›å»ºæˆ–æ›´æ–° `tls-rancher-ingress` Kubernetes Secret 对象。 +1. 使用根 CA è¯ä¹¦åˆ›å»ºæˆ–æ›´æ–° `tls-ca` Kubernetes Secret 对象（仅在使用ç§æœ‰ CA 时需è¦ï¼‰ã€‚ +1. 使用 Helm CLI æ›´æ–° Rancher 安装。 +1. é‡æ–°é…ç½® Rancher Agent 以信任新的 CA è¯ä¹¦ã€‚ +1. 选择 Fleet 集群的强制更新，æ¥å°† fleet-agent 连接到 Rancher。 + +å„个步骤的详细说明如下。 + +### 1. 创建/æ›´æ–°è¯ä¹¦ Secret 对象 + +首先，将æœåŠ¡å™¨è¯ä¹¦å’Œæ‰€æœ‰ä¸­é—´è¯ä¹¦åˆå¹¶åˆ°å为 `tls.crt` 的文件，并在å为 `tls.key` 的文件中æ供相应的è¯ä¹¦å¯†é’¥ã€‚ + +使用以下命令在 Rancher（本地）管ç†é›†ç¾¤ä¸­åˆ›å»º `tls-rancher-ingress` Secret 对象： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +或者，更新现有的 `tls-rancher-ingress` Secret： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. 创建/æ›´æ–°è¯ä¹¦ CA Secret 对象 + +如果新è¯ä¹¦ç”±ç§æœ‰ CA ç­¾å‘的，你需è¦å°†ç›¸åº”çš„æ ¹ CA è¯ä¹¦å¤åˆ¶åˆ°å为 `cacerts.pem` 的文件中，并创建或更新 `cattle-system` 命å空间中的 `tls-ca` Secret。如果è¯ä¹¦ç”±ä¸­é—´ CA ç­¾å，则 `cacerts.pem` 必须按顺åºåŒæ—¶åŒ…å«ä¸­é—´ CA è¯ä¹¦å’Œæ ¹ CA è¯ä¹¦ã€‚ + +创建åˆå§‹ `tls-ca` Secret： + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem +``` + +è¦æ›´æ–°çŽ°æœ‰çš„ `tls-ca` Secret： + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 3. é‡æ–°é…ç½® Rancher 部署 + +如果è¯ä¹¦æºä¿æŒä¸å˜ï¼ˆä¾‹å¦‚，`secret`），请按照步骤 3a 中的步骤æ“作。 + +但是，如果è¯ä¹¦æºå‘生å˜åŒ–（例如，`letsEncrypt` 更改为 `secret`），请按照 3b 中的步骤æ“作。 + +#### 3a. é‡æ–°éƒ¨ç½² Rancher pod + +当è¯ä¹¦æºä¿æŒä¸å˜ï¼Œä½†éœ€è¦æ›´æ–° CA è¯ä¹¦æ—¶ï¼Œéœ€è¦æ‰§è¡Œæ­¤æ­¥éª¤ã€‚ + +在这ç§æƒ…况下，由于 `tls-ca` secret 在å¯åŠ¨æ—¶ç”± Rancher pod 读å–，因此你需è¦é‡æ–°éƒ¨ç½² Rancher pod。 + +ä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤é‡æ–°éƒ¨ç½² Rancher pod： +```bash +kubectl rollout restart deploy/rancher -n cattle-system +``` + +修改完æˆåŽï¼Œè®¿é—® `https:///v3/settings/cacerts`，验è¯è¯¥å€¼æ˜¯å¦ä¸Žå…ˆå‰å†™å…¥ `tls-ca` Secret 中的 CA è¯ä¹¦åŒ¹é…。在所有é‡æ–°éƒ¨ç½²çš„ Rancher pod å¯åŠ¨ä¹‹å‰ï¼ŒCA `cacerts` 值å¯èƒ½ä¸ä¼šæ›´æ–°ã€‚ + +#### 3b. æ›´æ–° Rancher çš„ Helm 值 + +如果è¯ä¹¦æºæœ‰æ›´æ”¹ï¼Œåˆ™éœ€è¦æ‰§è¡Œæ­¤æ­¥éª¤ã€‚如果你的 Rancher 之å‰ä½¿ç”¨é»˜è®¤çš„自签åè¯ä¹¦ (`ingress.tls.source=rancher`) 或 Let's Encrypt (`ingress.tls.source=letsEncrypt`) è¯ä¹¦ï¼Œå¹¶ä¸”现在正在使用由ç§æœ‰ CA (`ingress.tls.source=secret`) ç­¾åçš„è¯ä¹¦ã€‚ + +以下步骤更新了 Rancher Chart çš„ Helm 值，因此 Rancher pod å’Œ ingress 会使用在步骤 1 å’Œ 2 中创建的新ç§æœ‰ CA è¯ä¹¦ã€‚ + +1. 调整åˆå§‹å®‰è£…期间使用的值，将当å‰å€¼å­˜å‚¨ä¸ºï¼š +```bash +helm get values rancher -n cattle-system -o yaml > values.yaml +``` +1. 检索当å‰éƒ¨ç½²çš„ Rancher Chart 的版本字符串： +```bash +helm ls -n cattle-system +``` +1. æ›´æ–° `values.yaml` æ–‡ä»¶ä¸­çš„å½“å‰ Helm 值以包å«ä¸‹æ–¹å†…容： +```yaml +ingress: + tls: + source: secret +privateCA: true +``` +:::note é‡è¦ï¼š +由于è¯ä¹¦ç”±ç§æœ‰ CA ç­¾å‘，因此确ä¿åœ¨ `values.yaml` 文件中设置了 [`privateCA: true`](../installation-references/helm-chart-options.md#常用选项) 是éžå¸¸é‡è¦çš„。 +::: +1. 使用 `values.yaml` æ–‡ä»¶å’Œå½“å‰ Chart 版本å‡çº§ Helm 应用程åºå®žä¾‹ã€‚版本必须匹é…以防止 Rancher å‡çº§ã€‚ +```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version +``` + +修改完æˆåŽï¼Œè®¿é—® `https:///v3/settings/cacerts`，验è¯è¯¥å€¼æ˜¯å¦ä¸Žå…ˆå‰å†™å…¥ `tls-ca` Secret 中的 CA è¯ä¹¦åŒ¹é…。在所有 Rancher pod å¯åŠ¨ä¹‹å‰ï¼ŒCA `cacerts` 值å¯èƒ½ä¸ä¼šæ›´æ–°ã€‚ + +### 4. é‡æ–°é…ç½® Rancher Agent 以信任ç§æœ‰ CA + +本节介ç»äº†é‡æ–°é…ç½® Rancher Agent 以信任ç§æœ‰ CA 的三ç§æ–¹æ³•ã€‚如果你的实际情况符åˆä»¥ä¸‹ä»»æ„一个æ¡ä»¶ï¼Œè¯·æ‰§è¡Œæ­¤æ­¥éª¤ï¼š + +- Rancher 在先å‰çš„é…置中使用了 Rancher 自签åè¯ä¹¦ (`ingress.tls.source=rancher`) 或 Let's Encrypt è¯ä¹¦ (`ingress.tls.source=letsEncrypt`)。 +- 该è¯ä¹¦ç”±ä¸åŒçš„ç§æœ‰ CA ç­¾å‘ + +#### 为什么è¦æ‰§è¡Œè¿™ä¸€æ­¥éª¤ï¼Ÿ + +如果 Rancher é…置了ç§æœ‰ CA ç­¾åçš„è¯ä¹¦æ—¶ï¼ŒCA è¯ä¹¦é“¾å°†å—到 Rancher agent 容器的信任。Agent 会对下载è¯ä¹¦çš„æ ¡éªŒå’ŒåŠ `CATTLE_CA_CHECKSUM` 环境å˜é‡è¿›è¡Œæ¯”较。æ¢è¨€ä¹‹ï¼Œå¦‚æžœ Rancher 使用的ç§æœ‰ CA è¯ä¹¦å‘生å˜åŒ–，环境å˜é‡ `CATTLE_CA_CHECKSUM` 必须相应更新。 + +#### å¯ä½¿ç”¨çš„方法 + +- 方法 1（最简å•çš„方法）：在轮æ¢è¯ä¹¦åŽå°†æ‰€æœ‰é›†ç¾¤è¿žæŽ¥åˆ° Rancher。适用于更新或é‡æ–°éƒ¨ç½² Rancher 部署（步骤 3）åŽç«‹å³æ‰§è¡Œçš„情况。 + +- 方法 2：适用于集群与 Rancher 失去连接，但所有集群都å¯ç”¨äº† [Authorized Cluster Endpoint](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) (ACE) 的情况。 + +- 方法 3：如果方法 1 å’Œ 2 ä¸å¯è¡Œï¼Œåˆ™å¯ä½¿ç”¨æ–¹æ³• 3 进行回退。 + +#### 方法 1：强制é‡æ–°éƒ¨ç½² Rancher Agent + +对于æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤ï¼Œä½¿ç”¨ Rancher（本地）管ç†é›†ç¾¤çš„ Kubeconfig 文件è¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚ + +```bash +kubectl annotate clusters.management.cattle.io io.cattle.agent.force.deploy=true +``` + +:::note +找到下游集群的集群 ID (c-xxxxx)。你å¯ä»¥åœ¨ Rancher UI çš„**集群管ç†**中查看集群时在æµè§ˆå™¨ URL 中找到 ID。 +::: + +此命令将使 Agent 清å•é‡æ–°åº”用新è¯ä¹¦çš„校验和。 + +#### 方法二：手动更新校验和环境å˜é‡ + +å°† `CATTLE_CA_CHECKSUM` 环境å˜é‡æ›´æ–°ä¸ºåŒ¹é…æ–° CA è¯ä¹¦æ ¡éªŒå’Œçš„值，从而手动为 Agent Kubernetes 对象打上补ä¸ã€‚通过以下æ“作生æˆæ–°çš„校验和： + +```bash +curl -k -s -fL /v3/settings/cacerts | jq -r .value | sha256sum | awk '{print $1}' +``` + +为æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤ä½¿ç”¨ Kubeconfig 更新两个 Agent 部署的环境å˜é‡ã€‚如果集群å¯ç”¨äº† [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md)，你å¯ä»¥[调整 kubectl 上下文](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)，从而直接连接到下游集群。 + +```bash +kubectl edit -n cattle-system ds/cattle-node-agent +kubectl edit -n cattle-system deployment/cattle-cluster-agent +``` + +#### 方法三：手动é‡æ–°éƒ¨ç½² Rancher agent + +该方法通过在æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤çš„ control plane 节点上è¿è¡Œä¸€ç»„命令，从而é‡æ–°åº”用 Rancher agent。 + +对æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤é‡å¤ä»¥ä¸‹æ­¥éª¤ï¼š + +1. 检索 agent 注册 kubectl 命令： + 1. 找到下游集群的集群 ID (c-xxxxx)。你å¯ä»¥åœ¨ Rancher UI çš„**集群管ç†**中查看集群时在æµè§ˆå™¨ URL 中找到 ID。 + 1. å°† Rancher Server URL 和集群 ID 添加到以下 URL：`https:///v3/clusterregistrationtokens?clusterId=`。 + 1. å¤åˆ¶ `insecureCommand` 字段中的命令，使用此命令是因为未使用ç§æœ‰ CA。 + +2. 使用以下其中一ç§æ–¹æ³•ï¼Œä½¿ç”¨ kubeconfig 为下游集群è¿è¡Œä¸Šä¸€æ­¥ä¸­çš„ kubectl 命令： + 1. 如果集群å¯ç”¨äº† [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md)，你å¯ä»¥[调整上下文](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)，从而直接连接到下游集群。 + 1. 或者，SSH 到 control plane 节点： + - RKE：使用[此处文档中的步骤](https://github.com/rancherlabs/support-tools/tree/master/how-to-retrieve-kubeconfig-from-custom-cluster)ç”Ÿæˆ kubeconfig + - RKE2/K3s：使用安装时填充的 kubeconfig + +### 5. 强制更新 Fleet 集群，从而将 fleet-agent é‡æ–°è¿žæŽ¥åˆ° Rancher + +在 Rancher UI çš„[æŒç»­äº¤ä»˜](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#在-rancher-ui-中访问-fleet)中，为集群选择“强制更新â€ï¼Œæ¥å…许下游集群中的 fleet-agent æˆåŠŸè¿žæŽ¥åˆ° Rancher。 + +#### 为什么è¦æ‰§è¡Œè¿™ä¸€æ­¥éª¤ï¼Ÿ + +Rancher 管ç†çš„集群中的 Fleet agent 存储了用于连接到 Rancher çš„ kubeconfig。kubeconfig 包å«ä¸€ä¸ª `certificate-authority-data` å­—æ®µï¼Œè¯¥å­—æ®µåŒ…å« Rancher 使用的è¯ä¹¦çš„ CA。更改 CA 时，你需è¦æ›´æ–°æ­¤å—æ¥å…许 fleet-agent ä¿¡ä»» Rancher 使用的è¯ä¹¦ã€‚ + +## å°†ç§æœ‰ CA è¯ä¹¦æ›´æ”¹ä¸ºå…¬å…±è¯ä¹¦ + +按照以下步骤执行与上é¢ç›¸åçš„æ“作，将ç§æœ‰ CA é¢å‘çš„è¯ä¹¦æ›´æ”¹ä¸ºå…¬å…±æˆ–自签å CA。 + +### 1. 创建/æ›´æ–°è¯ä¹¦ Secret 对象 + +首先，将æœåŠ¡å™¨è¯ä¹¦å’Œæ‰€æœ‰ä¸­é—´è¯ä¹¦åˆå¹¶åˆ°å为 `tls.crt` 的文件，并在å为 `tls.key` 的文件中æ供相应的è¯ä¹¦å¯†é’¥ã€‚ + +使用以下命令在 Rancher（本地）管ç†é›†ç¾¤ä¸­åˆ›å»º `tls-rancher-ingress` Secret 对象： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +或者，更新现有的 `tls-rancher-ingress` Secret： + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. 删除 CA è¯ä¹¦ Secret 对象 + +你需è¦åˆ é™¤ `cattle-system` 命å空间中的 `tls-ca secret`（ä¸å†éœ€è¦å®ƒï¼‰ã€‚如果需è¦ï¼Œä½ è¿˜å¯ä»¥é€‰æ‹©ä¿å­˜ `tls-ca` secret 的副本。 + +è¦ä¿å­˜çŽ°æœ‰çš„ `tls-ca` Secret： + +```bash +kubectl -n cattle-system get secret tls-ca -o yaml > tls-ca.yaml +``` + +è¦åˆ é™¤çŽ°æœ‰çš„ `tls-ca` 密文： + +```bash +kubectl -n cattle-system delete secret tls-ca +``` + +### 3. é‡æ–°é…ç½® Rancher 部署 + +如果è¯ä¹¦æºæœ‰æ›´æ”¹ï¼Œåˆ™éœ€è¦æ‰§è¡Œæ­¤æ­¥éª¤ã€‚在这ç§æƒ…况下，它å˜åŒ–的原因很å¯èƒ½æ˜¯å› ä¸º Rancher 之å‰é…置为使用默认的自签åè¯ä¹¦ (`ingress.tls.source=rancher`)。 + +以下步骤更新了 Rancher Chart çš„ Helm 值，因此 Rancher pod å’Œ Ingress 会使用在步骤 1 中创建的新è¯ä¹¦ã€‚ + +1. 调整åˆå§‹å®‰è£…期间使用的值，将当å‰å€¼å­˜å‚¨ä¸ºï¼š +```bash +helm get values rancher -n cattle-system -o yaml > values.yaml +``` +1. 获å–当å‰éƒ¨ç½²çš„ Rancher Chart 的版本字符串： +```bash +helm ls -n cattle-system +``` +1. æ›´æ–° `values.yaml` æ–‡ä»¶ä¸­çš„å½“å‰ Helm 值： + 1. 由于ä¸å†ä½¿ç”¨ç§æœ‰ CA，删除 `privateCA: true` 字段，或将其设置为 `false`。 + 1. æ ¹æ®éœ€è¦è°ƒæ•´ `ingress.tls.source` 字段。有关更多信æ¯ï¼Œè¯·å‚阅 [Chart 选项](../installation-references/helm-chart-options.md#常用选项)。以下是一些示例： + 1. 如果使用公共 CA，继续使用 `secret` + 1. 如果使用 Let's Encrypt，将值更新为 `letsEncrypt` +1. 使用 `values.yaml` 文件更新 Rancher Chart çš„ Helm å€¼ï¼Œå¹¶ä½¿ç”¨å½“å‰ Chart 版本防止å‡çº§ï¼š +```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version +``` + +### 4. 为éžç§æœ‰/通用è¯ä¹¦é‡æ–°é…ç½® Rancher Agent + +由于ä¸å†ä½¿ç”¨ç§æœ‰ CA，因此你需è¦åˆ é™¤ä¸‹æ¸¸é›†ç¾¤ agent 上的 `CATTLE_CA_CHECKSUM` 环境å˜é‡ï¼Œæˆ–将其设置为 ""（空字符串）。 + +### 5. 强制更新 Fleet 集群，从而将 fleet-agent é‡æ–°è¿žæŽ¥åˆ° Rancher + +在 Rancher UI çš„[æŒç»­äº¤ä»˜](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#在-rancher-ui-中访问-fleet)中，为集群选择“强制更新â€ï¼Œæ¥å…许下游集群中的 fleet-agent æˆåŠŸè¿žæŽ¥åˆ° Rancher。 + +#### 为什么è¦æ‰§è¡Œè¿™ä¸€æ­¥éª¤ï¼Ÿ + +Rancher 管ç†çš„集群中的 Fleet agent 存储了用于连接到 Rancher çš„ kubeconfig。kubeconfig 包å«ä¸€ä¸ª `certificate-authority-data` å­—æ®µï¼Œè¯¥å­—æ®µåŒ…å« Rancher 使用的è¯ä¹¦çš„ CA。更改 CA 时，你需è¦æ›´æ–°æ­¤å—æ¥å…许 fleet-agent ä¿¡ä»» Rancher 使用的è¯ä¹¦ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md new file mode 100644 index 00000000000..40267b47065 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md @@ -0,0 +1,281 @@ +--- +title: å‡çº§ Cert-Manager +--- + +Rancher 使用 cert-manager 为 Rancher 高å¯ç”¨éƒ¨ç½²è‡ªåŠ¨ç”Ÿæˆå’Œç»­æœŸ TLS è¯ä¹¦ã€‚从 2019 秋季开始，cert-manager å‘生了以下的三个é‡è¦å˜æ›´ã€‚如果你在此时间段å‰åˆ›å»ºäº† Rancher 高å¯ç”¨éƒ¨ç½²ï¼Œè¯·è¿›è¡Œç›¸å…³æ“作。 + +1. [从 2019 å¹´ 11 月 1 日开始，Let's Encrypt 已阻止低于 0.8.0 çš„ cert-manager 实例。](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) +1. [Cert-manager æ­£åœ¨å¼ƒç”¨å’Œæ›¿æ¢ certificate.spec.acme.solvers 字段](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/)。此更改暂时没有确切的截止日期。 +1. [Cert-manager 正在弃用 `v1alpha1` API 和替æ¢å®ƒçš„ API 组](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/)。 + +为了帮助你应对这些å˜åŒ–，本文将： + +1. æä¾›å‡çº§ cert-manager 步骤的文档。 +1. é˜è¿° cert-manager API çš„å˜æ›´ï¼Œå¹¶æä¾› cert-manager 官方文档的链接，助你实现数æ®è¿ç§»ã€‚ + +:::note é‡è¦æ示： + +如果你è¦å°† cert-manager 从早于 1.5 的版本å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹[选项 C](#选项-cå‡çº§-15-åŠä»¥ä¸‹ç‰ˆæœ¬çš„-cert-manager) 中的步骤进行æ“作。请注æ„，你无需é‡æ–°å®‰è£… Rancher å³å¯æ‰§è¡Œæ­¤å‡çº§ã€‚ + +::: + +## å‡çº§ Cert-Manager + +以下说明中使用的命å空间是由当å‰å®‰è£…了 cert-manager 的命å空间决定的。如果它在 kube-system 中，在以下说明步骤中使用。你å¯ä»¥è¿è¡Œ `kubectl get pods --all-namespaces` æ¥éªŒè¯ï¼Œå¹¶æ£€æŸ¥ cert-manager-\* pods 列在哪个命å空间中。ä¸è¦æ›´æ”¹è¿è¡Œ cert-manager 的命å空间，å¦åˆ™å¯èƒ½ä¼šå‡ºçŽ°é”™è¯¯ã€‚ + +è¦å‡çº§ cert-manager，请éµå¾ªæ­¥éª¤æ“作。 + +### 选项 A：è”网å‡çº§ cert-manager + +
+ å•å‡»å±•å¼€ + +1. [备份现有资æº](https://cert-manager.io/docs/tutorials/backup/)： + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note é‡è¦æ示： + + 如果你从低于 0.11.0 的版本å‡çº§ï¼Œè¯·å°†æ‰€æœ‰å¤‡ä»½èµ„æºä¸Šçš„ apiVersion 从 `certmanager.k8s.io/v1alpha1` å‡çº§åˆ° `cert-manager.io/v1alpha2`。如果你需è¦åœ¨å…¶ä»–资æºä¸Šä½¿ç”¨ cert-manager 注释，请对其进行更新以å映新的 API 组。详情请å‚è§[附加注释å˜æ›´](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes)。 + + ::: + +1. [å¸è½½çŽ°æœ‰éƒ¨ç½²](https://cert-manager.io/docs/installation/uninstall/kubernetes/#uninstalling-with-helm)： + + ```plain + helm uninstall cert-manager + ``` + + 使用你安装的 vX.Y.Z 版本的链接删除 CustomResourceDefinition： + + ```plain + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + +1. å•ç‹¬å®‰è£… CustomResourceDefinition 资æºï¼š + + ```plain + kubectl apply --validate=false -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + + :::note + + 如果你è¿è¡Œçš„ Kubernetes 版本是 1.15 或更低版本，你需è¦åœ¨ä»¥ä¸Šçš„ `kubectl apply` 命令中添加 `--validate=false`。å¦åˆ™ä½ å°†çœ‹åˆ° cert-manager CRD 资æºä¸­çš„ `x-kubernetes-preserve-unknown-fields` 字段校验错误æ示。这是 kubectl 执行资æºæ ¡éªŒæ–¹å¼äº§ç”Ÿçš„良性错误。 + + ::: + +1. æ ¹æ®éœ€è¦ä¸º cert-manager 创建命å空间： + + ```plain + kubectl create namespace cert-manager + ``` + +1. 添加 Jetstack Helm 仓库： + + ```plain + helm repo add jetstack https://charts.jetstack.io + ``` + +1. æ›´æ–° Helm Chart 仓库本地缓存： + + ```plain + helm repo update + ``` + +1. 安装新版本的 cert-manager： + + ```plain + helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.11.0 + ``` + +1. [æ¢å¤å¤‡ä»½èµ„æº](https://cert-manager.io/docs/tutorials/backup/#restoring-resources)： + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### 选项 B：在离线环境中å‡çº§ Cert-Manager + +
+ å•å‡»å±•å¼€ + +### 先决æ¡ä»¶ + +在执行å‡çº§ä¹‹å‰ï¼Œå…ˆå°†æ‰€éœ€çš„容器镜åƒæ·»åŠ åˆ°ç§æœ‰é•œåƒä»“库中，并下载/渲染所需的 Kubernetes manifest 文件æ¥å‡†å¤‡ç¦»çº¿çŽ¯å¢ƒã€‚ + +1. å‚è§[准备ç§æœ‰é•œåƒä»“库](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md)指å—，将å‡çº§æ‰€éœ€çš„é•œåƒæŽ¨é€åˆ°é•œåƒä»“库。 + +1. 在å¯ä»¥è¿žæŽ¥äº’è”网的系统中，将 cert-manager 仓库添加到 Helm： + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + ``` + +1. 从 [Helm Chart 仓库](https://artifacthub.io/packages/helm/cert-manager/cert-manager)中获å–最新å¯ç”¨çš„ cert-manager Chart： + + ```plain + helm fetch jetstack/cert-manager --version v1.11.0 + ``` + +1. 使用安装 Chart 的选项æ¥æ¸²æŸ“ cert-manager 模æ¿ã€‚è®°ä½è¦è®¾ç½® `image.repository` 选项，以从你的ç§æœ‰é•œåƒä»“库拉å–é•œåƒã€‚æ­¤æ“ä½œä¼šåˆ›å»ºä¸€ä¸ªåŒ…å« Kubernetes manifest 文件的 `cert-manager` 目录。 + + Helm 3 命令如下： + + ```plain + helm template cert-manager ./cert-manager-v0.12.0.tgz --output-dir . \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + + Helm 2 命令如下： + + ```plain + helm template ./cert-manager-v0.12.0.tgz --output-dir . \ + --name cert-manager --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + +1. 下载新旧版 cert-manager 所需的 CRD 文件： + + ```plain + curl -L -o cert-manager-crd.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-0.12/deploy/manifests/00-crds.yaml + curl -L -o cert-manager/cert-manager-crd-old.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-X.Y/deploy/manifests/00-crds.yaml + ``` + +### 安装 cert-manager + +1. 备份现有资æºï¼š + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note é‡è¦æ示： + + 如果你从低于 0.11.0 的版本å‡çº§ï¼Œè¯·å°†æ‰€æœ‰å¤‡ä»½èµ„æºä¸Šçš„ apiVersion 从 `certmanager.k8s.io/v1alpha1` å‡çº§åˆ° `cert-manager.io/v1alpha2`。如果你需è¦åœ¨å…¶ä»–资æºä¸Šä½¿ç”¨ cert-manager 注释，请对其进行更新以å映新的 API 组。详情请å‚è§[附加注释å˜æ›´](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes)。 + + ::: + +1. 删除现有的 cert-manager 安装包： + + ```plain + kubectl -n cert-manager \ + delete deployment,sa,clusterrole,clusterrolebinding \ + -l 'app=cert-manager' -l 'chart=cert-manager-v0.5.2' + ``` + + 使用你安装的 vX.Y 版本的链接删除 CustomResourceDefinition： + + ```plain + kubectl delete -f cert-manager/cert-manager-crd-old.yaml + ``` + +1. å•ç‹¬å®‰è£… CustomResourceDefinition 资æºï¼š + + ```plain + kubectl apply -f cert-manager/cert-manager-crd.yaml + ``` + + :::note é‡è¦æ示： + + 如果你è¿è¡Œçš„ Kubernetes 版本是 1.15 或更低版本，你需è¦åœ¨ä»¥ä¸Šçš„ `kubectl apply` 命令中添加 `--validate=false`。å¦åˆ™ä½ å°†çœ‹åˆ° cert-manager CRD 资æºä¸­çš„ `x-kubernetes-preserve-unknown-fields` 字段校验错误æ示。这是 kubectl 执行资æºæ ¡éªŒæ–¹å¼äº§ç”Ÿçš„良性错误。 + + ::: + +1. 为 cert-manager 创建命å空间： + + ```plain + kubectl create namespace cert-manager + ``` + +1. 安装 cert-manager + + ```plain + kubectl -n cert-manager apply -R -f ./cert-manager + ``` + +1. [æ¢å¤å¤‡ä»½èµ„æº](https://cert-manager.io/docs/tutorials/backup/#restoring-resources)： + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### 选项 C：å‡çº§ 1.5 åŠä»¥ä¸‹ç‰ˆæœ¬çš„ cert-manager + +
+ å•å‡»å±•å¼€ + +以å‰ï¼Œè¦å‡çº§æ—§ç‰ˆæœ¬çš„ cert-manager，我们建议å¸è½½å¹¶é‡æ–°å®‰è£… Rancher。使用以下方法，你å¯ä»¥å‡çº§ cert-manager 而无需执行此é¢å¤–步骤，从而更好地ä¿æŠ¤ä½ çš„生产环境： + +1. 按照[安装指å—](https://cert-manager.io/docs/usage/cmctl/#installation)安装 `cmctl`（cert-manager CLI 工具）。 + +1. ç¡®ä¿æ‰€æœ‰ä»¥å·²å¼ƒç”¨çš„ API 版本存储在 etcd 中的 cert-manager 自定义资æºéƒ½è¿ç§»åˆ° v1： + + ``` + cmctl upgrade migrate-api-version + ``` + 有关详细信æ¯ï¼Œè¯·å‚阅 [API 版本è¿ç§»æ–‡æ¡£](https://cert-manager.io/docs/usage/cmctl/#migrate-api-version)。å¦è¯·å‚阅[å°† 1.5 å‡çº§åˆ° 1.6](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/) å’Œ[å°† 1.6 å‡çº§åˆ°åˆ° 1.7](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/)。 + +1. 正常使用 `helm upgrade` å°† cert-manager å‡çº§åˆ° 1.7.1。如果需è¦ï¼Œä½ å¯ä»¥ç›´æŽ¥ä»Žç‰ˆæœ¬ 1.5 转到 1.7。 + +1. 按照 Helm 教程[æ›´æ–°å‘布清å•çš„ API 版本](https://helm.sh/docs/topics/kubernetes_apis/#updating-api-versions-of-a-release-manifest)。Chart å‘布å称为 `release_name=rancher`，å‘布命å空​​间为 `release_namespace=cattle-system`。 + +1. 在解ç åŽçš„文件中，æœç´¢ `cert-manager.io/v1beta1` 并将其**替æ¢**为 `cert-manager.io/v1`。 + +1. 使用 `helm upgrade` 正常å‡çº§ Rancher。 + +
+ +### 验è¯éƒ¨ç½² + +安装完 cert-manager åŽï¼Œä½ å¯ä»¥é€šè¿‡æ£€æŸ¥ kube-system 命å空间中正在è¿è¡Œçš„ Pod æ¥éªŒè¯å®ƒæ˜¯å¦å·²æ­£ç¡®éƒ¨ç½²ï¼š + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +## Cert-Manager API å˜æ›´å’Œæ•°æ®è¿ç§» + +--- + +Rancher çŽ°åœ¨æ”¯æŒ cert-manager 1.6.2 å’Œ 1.7.1。推è使用 v1.7.x，因为 v 1.6.x 将在 2022 å¹´ 3 月 30 日结æŸç”Ÿå‘½å‘¨æœŸã€‚详情请å‚è§ [cert-manager 文档](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-安装-cert-manager)。有关将 cert-manager 从 1.5 å‡çº§åˆ° 1.6 的说明，请å‚è§ä¸Šæ¸¸çš„ [cert-manager 文档](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/)。有关将 cert-manager 从 1.6 å‡çº§åˆ° 1.7 的说明，请å‚è§ä¸Šæ¸¸çš„ [cert-manager 文档](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/)。 + +--- + +Cert-manager å·²ç»å¼ƒç”¨ `certificate.spec.acme.solvers` 字段，而且会在未æ¥çš„版本中放弃对该字段的支æŒã€‚ + +æ ¹æ® cert-manager 文档，v0.8 引入了é…ç½® ACME è¯ä¹¦èµ„æºçš„æ–°æ ¼å¼ã€‚具体æ¥è¯´ï¼Œå°±æ˜¯ç§»åŠ¨äº† challenge solver 字段。v0.9 新旧格å¼å‡æ”¯æŒã€‚请知悉，之åŽå‘布的新 cert-manager 版本会放弃对旧格å¼çš„支æŒã€‚Cert-Manager 文档建议你在更新åŽï¼Œå°† ACME é¢å‘者和è¯ä¹¦èµ„æºæ›´æ–°åˆ°æ–°æ ¼å¼ã€‚ + +如需了解å˜æ›´ç»†èŠ‚以åŠè¿ç§»è¯´æ˜Žï¼Œè¯·å‚è§[å°† Cert-Manager 从 v0.7 å‡çº§åˆ° v0.8](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/)。 + +v0.11 版本标志ç€åˆ é™¤å…ˆå‰ Cert-Manager 版本中使用的 v1alpha1 API，以åŠå°† API 组从 certmanager.k8s.io 更改到 cert-manager.io。 + +此外，我们已ä¸å†æ”¯æŒ v0.8 版本中已弃用的旧é…置格å¼ã€‚æ¢è¨€ä¹‹ï¼Œåœ¨å‡çº§åˆ° v0.11 之å‰ï¼Œä½ å¿…须先为 ACME å‘行者使用新的 solver æ ·å¼é…置格å¼ä½œä¸ºè¿‡æ¸¡ã€‚详情请å‚è§[å‡çº§åˆ° v0.8](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/)。 + +如需了解å˜æ›´ç»†èŠ‚以åŠè¿ç§»è¯´æ˜Žï¼Œè¯·å‚è§[å°† Cert-Manager 从 v0.10 å‡çº§åˆ° v0.11](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/)。 + +如需获得更多信æ¯ï¼Œè¯·å‚è§ [Cert-Manager å‡çº§](https://cert-manager.io/docs/installation/upgrading/)。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md new file mode 100644 index 00000000000..aa851f3a50c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md @@ -0,0 +1,126 @@ +--- +title: å‡çº§å’Œå›žæ»š Kubernetes +--- + +å‡çº§åˆ°æœ€æ–°ç‰ˆæœ¬çš„ Rancher 之åŽï¼Œä¸‹æ¸¸ Kubernetes 集群å¯ä»¥å‡çº§ä¸º Rancher 支æŒçš„最新的 Kubernetes 版本。 + +Rancher 使用 RKE（Rancher Kubernetes Engine）æ¥é¢„置和编辑 RKE 集群。有关为 RKE 集群é…ç½®å‡çº§ç­–略的更多信æ¯ï¼Œè¯·å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/)。 + + +## ç»è¿‡æµ‹è¯•çš„ Kubernetes 版本 + +Rancher 在å‘布新版本之å‰ï¼Œä¼šå¯¹å…¶ä¸Ž Kubernetes 的最新次è¦ç‰ˆæœ¬è¿›è¡Œæµ‹è¯•ï¼Œä»¥ç¡®ä¿å…¼å®¹æ€§ã€‚有关å„个 Rancher 版本测试了哪些 Kubernetes 版本的详细信æ¯ï¼Œè¯·å‚阅[支æŒç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/)。 + +## å‡çº§çš„å·¥ä½œåŽŸç† + +RKE v1.1.0 改å˜äº†é›†ç¾¤å‡çº§çš„æ–¹å¼ã€‚ + +在 [RKE 文档](https://rancher.com/docs/rke/latest/en/upgrades/how-upgrades-work)中，你将了解编辑或å‡çº§ RKE Kubernetes 集群时会å‘生的情况。 + + +## å‡çº§çš„最佳实践 + +在å‡çº§é›†ç¾¤çš„ Kubernetes 版本时，我们建议你： + +1. æ‹ä¸€å¼ å¿«ç…§ã€‚ +1. å¯åŠ¨ Kubernetes å‡çº§ã€‚ +1. 如果å‡çº§å¤±è´¥ï¼Œè¯·å°†é›†ç¾¤æ¢å¤åˆ°å‡çº§å‰çš„ Kubernetes 版本。这å¯ä»¥é€šè¿‡é€‰æ‹©**æ¢å¤ etcd å’Œ Kubernetes 版本**选项æ¥å®žçŽ°ã€‚在æ¢å¤ etcd å¿«ç…§ 之å‰ï¼Œè¿™ä¼šå°†ä½ çš„集群æ¢å¤åˆ°å‡çº§å‰çš„ kubernetes 版本。 + +æ¢å¤æ“作将在ä¸å¤„于å¥åº·æˆ– active 状æ€çš„集群上è¿è¡Œã€‚ + +## å‡çº§ Kubernetes 版本 + +:::note 先决æ¡ä»¶ï¼š + +- 以下选项适用于 [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../pages-for-subheaders/launch-kubernetes-with-rancher.md)å’Œ[注册的 K3s Kubernetes 集群](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#已注册-rke2-å’Œ-k3s-集群的附加功能)。 +- 以下选项也适用于导入且已注册的 RKE2 集群。如果你从外部云平å°å¯¼å…¥é›†ç¾¤ä½†ä¸æ³¨å†Œï¼Œä½ å°†æ— æ³•åœ¨ Rancher 中å‡çº§ Kubernetes 版本。 +- 在å‡çº§ Kubernetes 之å‰ï¼Œå…ˆ[备份你的集群](../../pages-for-subheaders/backup-restore-and-disaster-recovery.md)。 + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è¦å‡çº§çš„集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. 从 **Kubernetes 版本** 下拉列表中，选择è¦ç”¨äºŽé›†ç¾¤çš„ Kubernetes 版本。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：已开始为集群å‡çº§ Kubernetes。 + +## 回滚 + +ä½ å¯ä»¥å°†é›†ç¾¤æ¢å¤åˆ°ä½¿ç”¨å…ˆå‰ Kubernetes 版本的备份。有关详细信æ¯ï¼Œè¯·å‚阅： + +- [备份集群](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md#快照工作原ç†) +- [使用备份æ¢å¤é›†ç¾¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md#使用快照æ¢å¤é›†ç¾¤) + +## é…ç½®å‡çº§ç­–ç•¥ + +从 RKE v1.1.0 开始，我们æ供了é¢å¤–çš„å‡çº§é€‰é¡¹ï¼Œè®©ä½ æ›´ç²¾ç»†åœ°æŽ§åˆ¶å‡çº§è¿‡ç¨‹ã€‚如果满足[æ¡ä»¶å’Œè¦æ±‚](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability)，你å¯ä»¥ä½¿ç”¨è¿™äº›é€‰é¡¹ï¼Œä»Žè€Œåœ¨é›†ç¾¤å‡çº§æœŸé—´ç»´æŒåº”用的å¯ç”¨æ€§ã€‚ + +ä½ å¯ä»¥åœ¨ Rancher UI 中é…ç½®å‡çº§ç­–略，也å¯ä»¥é€šè¿‡ç¼–辑 `cluster.yml` æ¥é…置策略。编辑 `cluster.yml` å¯ä»¥é…置更多高级选项。 + +### 在 Rancher UI 中é…置最大ä¸å¯ç”¨çš„ Worker 节点 + +ä½ å¯ä»¥åœ¨ Rancher UI 中é…ç½®ä¸å¯ç”¨ worker 节点的最大数é‡ã€‚在集群å‡çº§æœŸé—´ï¼Œworker 节点将按此大å°æ‰¹é‡å‡çº§ã€‚ + +默认情况下，ä¸å¯ç”¨ worker 节点的最大数é‡ä¸ºæ‰€æœ‰ worker 节点的 10%。此数字å¯ä»¥é…置为百分比或整数。当定义为百分比时，批大å°ä¼šè¢«å››èˆäº”入到最近的节点，最å°ä¸ºä¸€ä¸ªèŠ‚点。 + +è¦æ›´æ”¹ worker 节点的默认数é‡æˆ–百分比： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è¦å‡çº§çš„集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. 在**å‡çº§ç­–ç•¥**选项å¡ä¸­ï¼Œè¾“å…¥ **Worker 并å‘**作为固定的数字或百分比。你å¯ä»¥é€šè¿‡å°†é›†ç¾¤ä¸­çš„节点数å‡åŽ»æœ€å¤§ä¸å¯ç”¨èŠ‚点数æ¥èŽ·å–该数字。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：集群更新为使用新的å‡çº§ç­–略。 + +### 使用 Rancher UI 在å‡çº§æœŸé—´å¯ç”¨èŠ‚点清空 + +默认情况下，RKE 会在å‡çº§ä¹‹å‰[å°é”](https://kubernetes.io/docs/concepts/architecture/nodes/#manual-node-administration)æ¯ä¸ªèŠ‚点。默认情况下，[清空](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/)会在å‡çº§æœŸé—´è¢«ç¦ç”¨ã€‚如果在集群é…置中å¯ç”¨äº†æ¸…空​​，RKE 将在å‡çº§ä¹‹å‰å¯¹èŠ‚点进行å°é”和清空。 + +è¦åœ¨é›†ç¾¤å‡çº§æœŸé—´æ¸…空æ¯ä¸ªèŠ‚点： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è¦å¯ç”¨èŠ‚点清空的集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. å•å‡» **â‹® > 编辑**。 +1. 在**å‡çº§ç­–ç•¥**选项å¡ä¸­ï¼Œè½¬åˆ°**清空节点**字段并å•å‡»**是**。controlplane å’Œ worker 节点的清空是å•ç‹¬é…置的。 +1. é…置如何删除 pod 的选项。有关æ¯ä¸ªé€‰é¡¹çš„详细信æ¯ï¼Œè¯·å‚阅[本节](../../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md#激进和安全的清空选项)。 +1. （å¯é€‰ï¼‰é…置宽é™æœŸã€‚宽é™æœŸæ˜¯ç»™æ¯ä¸ª pod 进行清ç†çš„超时时间，能让 pod 有机会优雅地退出。Pod å¯èƒ½éœ€è¦å®Œæˆä»»ä½•æœªå®Œæˆçš„请求ã€å›žæ»šäº‹åŠ¡æˆ–将状æ€ä¿å­˜åˆ°æŸäº›å¤–部存储。如果该值为负数，将使用 pod 中指定的默认值。 +1. （å¯é€‰ï¼‰é…置超时，这是在清空放弃之å‰åº”该继续等待的时间。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：集群更新为使用新的å‡çº§ç­–略。 + +:::note + +ç›®å‰å­˜åœ¨ä¸€ä¸ª[已知问题](https://github.com/rancher/rancher/issues/25478)，å³ä½¿ etcd å’Œ controlplane 正在被清空， Rancher UI 也ä¸ä¼šå°†å®ƒä»¬çš„状æ€æ˜¾ç¤ºä¸º drained。 + +::: + +### 在å‡çº§æœŸé—´ç»´æŠ¤åº”用的å¯ç”¨æ€§ + +_从 RKE v1.1.0 èµ·å¯ç”¨_ + +在 [RKE 文档](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability/)中，你将了解在å‡çº§é›†ç¾¤æ—¶é˜²æ­¢åº”用åœæœºçš„è¦æ±‚。 + +### 在 cluster.yml 中é…ç½®å‡çº§ç­–ç•¥ + +你通过编辑 `cluster.yml` æ¥èŽ·å¾—更高级的å‡çº§ç­–ç•¥é…置选项。 + +有关详细信æ¯ï¼Œè¯·å‚阅 RKE 文档中的[é…ç½®å‡çº§ç­–ç•¥](https://rancher.com/docs/rke/latest/en/upgrades/configuring-strategy)。这部分还包括一个用于é…ç½®å‡çº§ç­–略的示例 `cluster.yml`。 + +## 故障排除 + +如果å‡çº§åŽèŠ‚点没有出现，`rke up` 命令会出错。 + +如果ä¸å¯ç”¨èŠ‚点的数é‡è¶…过é…置的最大值，则ä¸ä¼šè¿›è¡Œå‡çº§ã€‚ + +如果å‡çº§åœæ­¢ï¼Œä½ å¯èƒ½éœ€è¦ä¿®å¤ä¸å¯ç”¨èŠ‚点或将其从集群中删除，然åŽæ‰èƒ½ç»§ç»­å‡çº§ã€‚ + +失败的节点å¯èƒ½å¤„于许多ä¸åŒçš„状æ€ï¼š + +- 关机 +- ä¸å¯ç”¨ +- 用户在å‡çº§è¿‡ç¨‹ä¸­æ¸…空了节点，因此节点上没有 kubelet +- å‡çº§æœ¬èº«å¤±è´¥ + +如果在å‡çº§è¿‡ç¨‹ä¸­è¾¾åˆ°æœ€å¤§ä¸å¯ç”¨èŠ‚点数，Rancher 的下游集群将åœç•™åœ¨æ›´æ–°ä¸­çš„状æ€ï¼Œå¹¶ä¸”ä¸ä¼šç»§ç»­å‡çº§å…¶ä»– controlplane 节点。它将继续评估ä¸å¯ç”¨çš„节点集，以防其中一个节点å˜å¾—å¯ç”¨ã€‚如果无法修å¤èŠ‚点，则必须移除节点æ‰èƒ½ç»§ç»­å‡çº§ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md new file mode 100644 index 00000000000..d8021a77f4f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md @@ -0,0 +1,88 @@ +--- +title: 在ä¸å‡çº§ Rancher 的情况下å‡çº§ Kubernetes +--- + +RKE 元数æ®åŠŸèƒ½å…许你在新版本 Kubernetes å‘布åŽç«‹å³ä¸ºé›†ç¾¤é…置新版本，而无需å‡çº§ Rancher。此功能对于使用 Kubernetes çš„è¡¥ä¸ç‰ˆæœ¬éžå¸¸æœ‰ç”¨ï¼Œä¾‹å¦‚ï¼Œåœ¨åŽŸæœ¬æ”¯æŒ Kubernetes v1.14.6 çš„ Rancher Server 版本中，将 Kubernetes å‡çº§åˆ° v1.14.7。 + +:::note + +Kubernetes API å¯ä»¥åœ¨æ¬¡è¦ç‰ˆæœ¬ä¹‹é—´æ›´æ”¹ã€‚因此，我们ä¸æ”¯æŒå¼•å…¥ Kubernetes 次è¦ç‰ˆæœ¬ï¼Œä¾‹å¦‚在 Rancher æ”¯æŒ v1.14 的情况下引入 v1.15。在这ç§æƒ…况下，你需è¦å‡çº§ Rancher 以添加对 Kubernetes 次è¦ç‰ˆæœ¬çš„支æŒã€‚ + +::: + +Rancher çš„ Kubernetes 元数æ®åŒ…å« Rancher 用于é…ç½® [RKE 集群](../../pages-for-subheaders/launch-kubernetes-with-rancher.md)çš„ Kubernetes 版本信æ¯ã€‚Rancher 会定期åŒæ­¥æ•°æ®å¹¶ä¸º **系统镜åƒ**ã€**æœåŠ¡é€‰é¡¹**å’Œ**æ’件模æ¿**创建自定义资æºå®šä¹‰ (CRD)。因此，当新的 Kubernetes 版本与 Rancher Server 版本兼容时，Kubernetes 元数æ®å¯ä»¥ä½¿ Rancher 使用新版本æ¥é…置集群。元数æ®æ¦‚述了 [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) 用于部署å„ç§ Kubernetes 版本的信æ¯ã€‚ + +下表æ述了å—周期性数æ®åŒæ­¥å½±å“çš„ CRD。 + +:::note + +åªæœ‰ç®¡ç†å‘˜å¯ä»¥ç¼–è¾‘å…ƒæ•°æ® CRD。除éžæ˜Žç¡®éœ€è¦ï¼Œå¦åˆ™å»ºè®®ä¸è¦æ›´æ–°çŽ°æœ‰å¯¹è±¡ã€‚ + +::: + +| èµ„æº | æè¿° | Rancher API URL | +|----------|-------------|-----------------| +| ç³»ç»Ÿé•œåƒ | 用于通过 RKE 部署 Kubernetes 集群的系统镜åƒåˆ—表。 | `/v3/rkek8ssystemimages` | +| æœåŠ¡é€‰é¡¹ | 传递给 Kubernetes 组件的默认选项，例如 `kube-api`ã€`scheduler`ã€`kubelet`ã€`kube-proxy` å’Œ `kube-controller-manager` | `/v3/rkek8sserviceoptions` | +| æ’ä»¶æ¨¡æ¿ | 用于部署æ’件组件的 YAML 定义，例如 Canalã€Calicoã€Flannelã€Weaveã€Kube-dnsã€CoreDNSã€`metrics-server`ã€`nginx-ingress` | `/v3/rkeaddons` | + +管ç†å‘˜å¯ä»¥é€šè¿‡é…ç½® RKE 元数æ®è®¾ç½®æ¥æ‰§è¡Œä»¥ä¸‹æ“作： + +- 刷新 Kubernetes 元数æ®ã€‚适用于有新的 Kubernetes è¡¥ä¸ç‰ˆæœ¬å‘布，而用户希望在ä¸å‡çº§ Rancher 的情况下为集群é…置最新版本的 Kubernetes 的情景。 +- 更改 Rancher 用于åŒæ­¥å…ƒæ•°æ®çš„ URL。适用于è¦è®© Rancher 从本地åŒæ­¥è€Œä¸æ˜¯ä¸Ž GitHub åŒæ­¥çš„情况。这在离线环境下éžå¸¸æœ‰ç”¨ã€‚ +- 防止 Rancher 自动åŒæ­¥å…ƒæ•°æ®ã€‚è¿™å¯ä»¥é˜²æ­¢åœ¨ Rancher 中使用新的/ä¸å—支æŒçš„ Kubernetes 版本。 + +### 刷新 Kubernetes å…ƒæ•°æ® + +默认情况下，管ç†å‘˜æˆ–具有**管ç†é›†ç¾¤é©±åŠ¨**[全局角色](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)的用户，å¯ä»¥åˆ·æ–° Kubernetes 元数æ®ã€‚ + +è¦å¼ºåˆ¶ Rancher 刷新 Kubernetes 元数æ®ï¼Œå¯ä»¥æ‰§è¡Œæ‰‹åŠ¨åˆ·æ–°æ“作： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 +1. å•å‡»**刷新 Kubernetes 元数æ®**。 + +ä½ å¯ä»¥å°† `refresh-interval-minutes` 设置为 `0`（è§ä¸‹æ–‡ï¼‰ï¼Œå°† Rancher é…置为仅在需è¦æ—¶åˆ·æ–°å…ƒæ•°æ®ï¼Œå¹¶åœ¨éœ€è¦æ—¶ä½¿ç”¨æ­¤æŒ‰é’®æ‰‹åŠ¨æ‰§è¡Œå…ƒæ•°æ®åˆ·æ–°ã€‚ + +### é…置元数æ®åŒæ­¥ + +:::caution + +åªæœ‰ç®¡â€‹â€‹ç†å‘˜å¯ä»¥æ›´æ”¹è¿™äº›è®¾ç½®ã€‚ + +::: + +RKE 元数æ®çš„é…置控制 Rancher åŒæ­¥å…ƒæ•°æ®çš„频率以åŠä»Žä½•å¤„下载数æ®ã€‚ä½ å¯ä»¥é€šè¿‡ Rancher UI 或通过 Rancher API 端点 `v3/settings/rke-metadata-config` é…置元数æ®ã€‚ + +元数æ®çš„é…置方å¼å–决于 Rancher 版本。 + +è¦åœ¨ Rancher 中编辑元数æ®é…置： + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. 转到 **rke-metadata-config**。å•å‡» **â‹® > 编辑设置**。 +1. ä½ å¯ä»¥é€‰æ‹©å¡«å†™ä»¥ä¸‹å‚数： + +- `refresh-interval-minutes`：Rancher 等待åŒæ­¥å…ƒæ•°æ®çš„时间。如果è¦ç¦ç”¨å®šæœŸåˆ·æ–°ï¼Œè¯·å°† `refresh-interval-minutes` 设置为 0。 +- `url`：Rancher 从中获å–æ•°æ®çš„ HTTP 路径。该路径必须是 JSON 文件的直接路径。例如，Rancher v2.4 的默认 URL 是 `https://releases.rancher.com/kontainer-driver-metadata/release-v2.4/data.json`。 +1. å•å‡»**ä¿å­˜**。 + +如果你没有离线设置，则无需指定 Rancher 获å–元数æ®çš„ URL，因为默认是从 [Rancher çš„å…ƒæ•°æ® Git 仓库获å–](https://github.com/rancher/kontainer-driver-metadata/blob/dev-v2.5/data/data.json)的。 + +但是，如果你有[离线设置](#离线设置)需求，你需è¦å°† Kubernetes 元数æ®ä»“库镜åƒåˆ° Rancher å¯ç”¨çš„ä½ç½®ã€‚然åŽï¼Œä½ éœ€è¦æ›´æ”¹ URL æ¥æŒ‡å‘ JSON 文件的新ä½ç½®ã€‚ + +### 离线设置 + +Rancher Server 会定期刷新 `rke-metadata-config` æ¥ä¸‹è½½æ–°çš„ Kubernetes 版本元数æ®ã€‚有关 Kubernetes å’Œ Rancher 版本的兼容性表，请å‚阅[æœåŠ¡æ¡æ¬¾](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.2.8/)。 + +如果你使用离线设置，则å¯èƒ½æ— æ³•ä»Ž Rancher çš„ Git 仓库自动定期刷新 Kubernetes 元数æ®ã€‚在这ç§æƒ…况下，应该ç¦ç”¨å®šæœŸåˆ·æ–°ä»¥é˜²æ­¢åœ¨æ—¥å¿—中显示相关错误。或者，你å¯ä»¥é…置元数æ®ï¼Œä»¥ä¾¿ Rancher 与本地的 RKE 元数æ®å‰¯æœ¬è¿›è¡ŒåŒæ­¥ã€‚ + +è¦å°† Rancher 与 RKE 元数æ®çš„本地镜åƒåŒæ­¥ï¼Œç®¡ç†å‘˜éœ€è¦é…ç½® `rke-metadata-config` æ¥æŒ‡å‘é•œåƒã€‚详情请å‚考[é…置元数æ®åŒæ­¥](#é…置元数æ®åŒæ­¥) + +在将新的 Kubernetes 版本加载到 Rancher Server 中之åŽï¼Œéœ€è¦æ‰§è¡Œå…¶ä»–步骤æ‰èƒ½ä½¿ç”¨å®ƒä»¬å¯åŠ¨é›†ç¾¤ã€‚Rancher 需è¦è®¿é—®æ›´æ–°çš„系统镜åƒã€‚虽然åªæœ‰ç®¡ç†å‘˜å¯ä»¥æ›´æ”¹å…ƒæ•°æ®è®¾ç½®ï¼Œä½†ä»»ä½•ç”¨æˆ·éƒ½å¯ä»¥ä¸‹è½½ Rancher 系统镜åƒå¹¶ä¸ºé•œåƒå‡†å¤‡ç§æœ‰å®¹å™¨é•œåƒä»“库。 + +1. è¦æŠŠç³»ç»Ÿé•œåƒä¸‹è½½åˆ°ç§æœ‰é•œåƒä»“库，请å•å‡» Rancher UI 左下角的 Rancher Server 版本。 +1. 下载适用于 Linux 或 Windows æ“作系统的镜åƒã€‚ +1. 下载 `rancher-images.txt`。 +1. 使用[离线环境安装](other-installation-methods/air-gapped-helm-cli-install/publish-images.md)时使用的步骤准备ç§æœ‰é•œåƒä»“库，但ä¸è¦ä½¿ç”¨å‘布页é¢ä¸­çš„ `rancher-images.txt`，而是使用上一个步骤中获å–的文件。 + +**结果**：Rancher 的离线安装现在å¯ä»¥åŒæ­¥ Kubernetes 元数æ®ã€‚如果你在å‘布新版本的 Kubernetes 时更新了ç§æœ‰é•œåƒä»“库，你å¯ä»¥ä½¿ç”¨æ–°ç‰ˆæœ¬é…置集群，而无需å‡çº§ Rancher。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/overview.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/overview.md new file mode 100644 index 00000000000..903c8c28bd2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/overview.md @@ -0,0 +1,65 @@ +--- +title: 概述 +--- + +Rancher 是一个为使用容器的公å¸æ‰“造的容器管ç†å¹³å°ã€‚Rancher 使得开å‘者å¯ä»¥éšå¤„è¿è¡Œ Kubernetes（Run Kubernetes Everywhere），满足 IT 需求规范，赋能 DevOps 团队。 + +## Run Kubernetes Everywhere + +Kubernetes å·²ç»æˆä¸ºå®¹å™¨ç¼–排标准。现在，大多数云和虚拟化æ供商都æ供容器编排æœåŠ¡ã€‚Rancher 用户å¯ä»¥é€‰æ‹©ä½¿ç”¨ Rancher Kubernetes Engine（RKE）或云 Kubernetes æœåŠ¡ï¼ˆä¾‹å¦‚ GKEã€AKS å’Œ EKS）创建 Kubernetes 集群，还å¯ä»¥å¯¼å…¥å’Œç®¡ç†ä½¿ç”¨ä»»ä½• Kubernetes å‘行版或安装程åºåˆ›å»ºçš„现有 Kubernetes 集群。 + +## 满足 IT 需求规范 + +Rancher 支æŒå¯¹å…¶æŽ§åˆ¶çš„所有 Kubernetes 集群进行集中认è¯ã€è®¿é—®æŽ§åˆ¶å’Œç›‘控。例如，你å¯ä»¥ï¼š + +- 使用你的 Active Directory 凭è¯è®¿é—®ç”±äº‘æ供商（例如 GKE）托管的 Kubernetes 集群。 +- 设置所有用户ã€ç»„ã€é¡¹ç›®ã€é›†ç¾¤å’Œäº‘æœåŠ¡çš„æƒé™æŽ§åˆ¶ç­–略和安全策略。 +- 一站å¼æŸ¥çœ‹ Kubernetes 集群的è¿è¡ŒçŠ¶å†µå’Œå®¹é‡ã€‚ + +## 赋能 DevOps 团队 + +Rancher 为 DevOps 工程师æ供简å•ç›´æŽ¥çš„用户界é¢ï¼Œä»¥ç®¡ç†å…¶åº”用负载。用户ä¸éœ€è¦å¯¹ Kubernetes 有éžå¸¸æ·±å…¥çš„了解，å³å¯ä½¿ç”¨ Rancher。Rancher 应用商店包å«ä¸€å¥—实用的 DevOps å¼€å‘工具。Rancher 获得了多ç§äº‘原生生æ€ç³»ç»Ÿäº§å“的认è¯ï¼ŒåŒ…括安全工具ã€ç›‘控系统ã€å®¹å™¨é•œåƒä»“库ã€å­˜å‚¨å’Œç½‘络驱动等。 + +下图讲述了 Rancher 在 IT 管ç†å›¢é˜Ÿå’Œ DevOps å¼€å‘团队之间扮演的角色。DevOps 团队把他们的应用部署在他们选择的公有云或ç§æœ‰äº‘上。IT 管ç†å‘˜è´Ÿè´£æŸ¥çœ‹å¹¶ç®¡ç†ç”¨æˆ·ã€é›†ç¾¤ã€äº‘æœåŠ¡çš„æƒé™ã€‚ + +![å¹³å°](/img/platform.png) + +## Rancher API Server 的功能 + +Rancher API Server æ˜¯åŸºäºŽåµŒå…¥å¼ Kubernetes API Server å’Œ etcd æ•°æ®åº“建立的，它æ供了以下功能： + +### 授æƒå’ŒåŸºäºŽè§’色的æƒé™æŽ§åˆ¶ï¼ˆRBAC） + +- **用户管ç†**：Rancher API Server 除了管ç†æœ¬åœ°ç”¨æˆ·ï¼Œè¿˜[管ç†ç”¨æˆ·ç”¨æ¥è®¿é—®å¤–部æœåŠ¡æ‰€éœ€çš„认è¯ä¿¡æ¯](../pages-for-subheaders/authentication-config.md)，如登录 Active Directory å’Œ GitHub 所需的账å·å¯†ç ã€‚ +- **授æƒ**：Rancher API Server å¯ä»¥ç®¡ç†[访问控制策略](../pages-for-subheaders/manage-role-based-access-control-rbac.md)å’Œ[安全策略](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +### 使用 Kubernetes 的功能 + +- **é…ç½® Kubernetes 集群**：Rancher API Server å¯ä»¥åœ¨å·²æœ‰èŠ‚点上[é…ç½® Kubernetes](../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)，或进行 [Kubernetes 版本å‡çº§](installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 +- **管ç†åº”用商店**：Rancher 支æŒä½¿ç”¨ [Helm Chart 应用商店](../pages-for-subheaders/helm-charts-in-rancher.md)实现轻æ¾é‡å¤éƒ¨ç½²åº”用。 +- **管ç†é¡¹ç›®**：项目由集群中多个命å空间和访问控制策略组æˆï¼Œæ˜¯ Rancher 中的一个概念，Kubernetes 中并没有这个概念。你å¯ä»¥ä½¿ç”¨é¡¹ç›®å®žçŽ°ä»¥ç»„为å•ä½ï¼Œç®¡ç†å¤šä¸ªå‘½å空间，并进行 Kubernetes 相关æ“作。Rancher UI æ供用于[项目管ç†](../pages-for-subheaders/manage-projects.md)å’Œ[项目内应用管ç†](../pages-for-subheaders/kubernetes-resources-setup.md)的功能。 +- **Fleet æŒç»­äº¤ä»˜**：在 Rancher 中，你å¯ä»¥ä½¿ç”¨ [Fleet æŒç»­äº¤ä»˜](../pages-for-subheaders/fleet-gitops-at-scale.md)将应用程åºä»Ž Git 仓库部署到目标下游 Kubernetes 集群，无需任何手动æ“作。 +- **Istio**：[Rancher 与 Istio 集æˆ](../pages-for-subheaders/istio.md)，使得管ç†å‘˜æˆ–集群所有者å¯ä»¥å°† Istio 交给开å‘者，然åŽå¼€å‘者使用 Istio 执行安全策略，排查问题，或为è“绿部署，金ä¸é›€éƒ¨ç½²ï¼Œå’Œ A/B 测试进行æµé‡ç®¡ç†ã€‚ + +### é…置云基础设施 + +- **åŒæ­¥èŠ‚点信æ¯**：Rancher API Server å¯ä»¥åŒæ­¥æ‰€æœ‰é›†ç¾¤ä¸­å…¨éƒ¨[节点](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md)çš„ä¿¡æ¯ã€‚ +- **é…置云基础设施**：如果你为 Rancher é…置了云æ供商，Rancher å¯ä»¥åœ¨äº‘端动æ€é…ç½®[新节点](../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)å’Œ[æŒä¹…化存储](../pages-for-subheaders/create-kubernetes-persistent-storage.md)。 + +### æŸ¥çœ‹é›†ç¾¤ä¿¡æ¯ + +- **日志管ç†**：Rancher å¯ä»¥ä¸Žå¤šç§ Kubernetes 集群之外的主æµæ—¥å¿—管ç†å·¥å…·é›†æˆã€‚ +- **监控**：你å¯ä»¥ä½¿ç”¨ Rancher，通过业界领先并开æºçš„ Prometheus æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚ +- **å‘Šè­¦**：为了ä¿è¯é›†ç¾¤å’Œåº”用的正常è¿è¡Œï¼Œæ高公å¸çš„生产效率，你需è¦éšæ—¶äº†è§£é›†ç¾¤å’Œé¡¹ç›®çš„计划内和éžè®¡åˆ’事件。 + +## 使用 Rancher 编辑下游集群 + +对于已有集群而言，å¯æ供的选项和设置å–决于你é…置集群的方法。例如，åªæœ‰[通过 RKE å¯åŠ¨](../pages-for-subheaders/launch-kubernetes-with-rancher.md)的集群æ‰æœ‰å¯ç¼–辑的**集群选项**。 + +使用 Rancher 创建集群åŽï¼Œé›†ç¾¤ç®¡ç†å‘˜å¯ä»¥ç®¡ç†é›†ç¾¤æˆå‘˜ï¼Œç®¡ç†èŠ‚点池，或进行[其他æ“作](../pages-for-subheaders/cluster-configuration.md)。 + +下表总结了æ¯ä¸€ç§ç±»åž‹çš„集群和对应的å¯ç¼–辑的选项和设置： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md new file mode 100644 index 00000000000..2fc1ce0bd33 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md @@ -0,0 +1,6 @@ +--- +title: Rancher AWS Marketplace 快速入门 +description: 使用 Amazon EKS 部署 Rancher Server。 +--- + +Amazon Elastic Kubernetes Service (EKS) å¯ä»¥å¿«é€Ÿ[å°† Rancher 部署到 Amazon Web Services (AWS)](https://documentation.suse.com/trd/kubernetes/single-html/gs_rancher_aws-marketplace/)。详情请å‚è§æˆ‘们的 [Amazon Marketplace 列表](https://aws.amazon.com/marketplace/pp/prodview-go7ent7goo5ae)。观看 [demo](https://youtu.be/9dznJ7Ons0M)，了解 AWS Marketplace SUSE Rancher 设置的演练。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws.md new file mode 100644 index 00000000000..cb47333ce32 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws.md @@ -0,0 +1,95 @@ +--- +title: Rancher AWS å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher AWS 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 AWS çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Amazon AWS 会产生费用。 + +::: + +- [Amazon AWS è´¦å·](https://aws.amazon.com/account/): 用于创建部署 Rancher Server å’Œ Kubernetes 的资æºã€‚ +- [Amazon AWS 访问密钥](https://docs.aws.amazon.com/general/latest/gr/managing-aws-access-keys.html)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹ç›¸å…³æŒ‡å—。 +- [已创建 IAM ç­–ç•¥](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start)：定义附加此策略的账å·æ‰€å…·æœ‰çš„æƒé™ã€‚ +- [Terraform](https://www.terraform.io/downloads.html): 用于在 Amazon AWS 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + +### IAM 策略示例 + +AWS 模å—åªåˆ›å»ºä¸€ä¸ª EC2 密钥对ã€ä¸€ä¸ª EC2 安全组和一个 EC2 实例。以下是一个简å•çš„策略： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:*", + "Resource": "*" + } + ] +} +``` + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/aws` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 AWS 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + + - `aws_access_key` - 替æ¢ä¸º Amazon AWS 访问密钥 + - `aws_secret_key` - 替æ¢ä¸º Amazon AWS Secret 密钥 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [AWS Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/aws) 了解更多信æ¯ã€‚ + 建议包括： + + - `aws_region` - Amazon AWS 区域。AWS 的默认区域 (`us-east-1`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - EC2 使用的实例规格，最å°è§„格为 `t3a.medium` 。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `t3a.large` 或 `t3a.xlarge`。 + - `add_windows_node` - 如果设为 true，一个é¢å¤–çš„ Windows worker 节点会添加到工作负载集群中。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/aws` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +##### 结果 + +两个 Kubernetes 集群已部署到你的 AWS 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +## åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/aws` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/azure.md new file mode 100644 index 00000000000..33a8d411ba0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/azure.md @@ -0,0 +1,81 @@ +--- +title: Rancher Azure å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher Azure 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 Azure çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Microsoft Azure 会产生费用。 + +::: + +- [Microsoft Azure è´¦å·](https://azure.microsoft.com/en-us/free/)：用于创建部署 Rancher å’Œ Kubernetes 的资æºã€‚ +- [Microsoft Azure 订阅](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹å¦‚何创建 Microsoft Azure 订阅。 +- [Micsoroft Azure 租户](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant)：访问此链接并å‚考教程以创建 Microsoft Azure 租户。 +- [Microsoft Azure 客户端 ID/密文](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal)：访问此链接并å‚考教程以创建 Microsoft Azure 客户端和密文。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Microsoft Azure 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/azure` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 Azure 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `azure_subscription_id` - 替æ¢ä¸º Microsoft Azure 订阅 ID。 + - `azure_client_id` - 替æ¢ä¸º Microsoft Azure 客户端 ID。 + - `azure_client_secret` - 替æ¢ä¸º Microsoft Azure 客户端密文。 + - `azure_tenant_id` - 替æ¢ä¸º Microsoft Azure 租户 ID。 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [Azure Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/azure) 了解更多信æ¯ã€‚建议包括： + - `azure_location` - Microsoft Azure 区域。Azure 的默认区域 (`East US`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - 使用的计算实例大å°ï¼Œæœ€å°è§„格为 `Standard_DS2_v2`。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `Standard_DS2_v3` 或 `Standard_DS3_v2`。 + - `add_windows_node` - 如果设为 true，一个é¢å¤–çš„ Windows worker 节点会添加到工作负载集群中。 + - `windows_admin_password` - Windows worker 节点管ç†å‘˜çš„å¯†ç  + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/azure` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 Azure 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/azure` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md new file mode 100644 index 00000000000..e87f17c26d0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md @@ -0,0 +1,74 @@ +--- +title: Rancher DigitalOcean å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher DigitalOcean 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 DigitalOcean çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 DigitalOcean 会产生费用。 + +::: + +- [DigitalOcean è´¦å·](https://www.digitalocean.com)：用于è¿è¡ŒæœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ +- [DigitalOcean 访问密钥](https://www.digitalocean.com/community/tutorials/how-to-create-a-digitalocean-space-and-api-key)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥åˆ›å»ºä¸€ä¸ªã€‚ +- [Terraform](https://www.terraform.io/downloads.html)：用于在 DigitalOcean 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/do` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 DigitalOcean 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `do_token` - 替æ¢ä¸º DigitalOcean 访问密钥 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [DO Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/do) 了解更多信æ¯ã€‚建议包括： + - `do_region` - DigitalOcean 区域。DigitalOcean 的默认区域ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `droplet_size` - 使用的计算实例规格，最å°è§„格为`s-2vcpu-4gb`。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `s-4vcpu-8gb`。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/do` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 DigitalOcean 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/do` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md new file mode 100644 index 00000000000..b22ff8d5ff3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md @@ -0,0 +1,106 @@ +--- +title: Rancher Equinix Metal 快速入门 +--- + +## 本章节引导你： + +- é…ç½® Equinix Metal server +- 安装 Rancher 2.x +- 创建你的第一个集群 +- 部署一个 Nginx 应用 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。ä¸å»ºè®®å°† Docker 安装用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 快速入门概述 + +本指å—划分为ä¸åŒä»»åŠ¡ï¼Œä»¥ä¾¿äºŽä½¿ç”¨ã€‚ + +
+ +## 先决æ¡ä»¶ + +- [Equinix Metal è´¦å·](https://metal.equinix.com/developers/docs/accounts/users/) +- [Equinix Metal 项目](https://metal.equinix.com/developers/docs/accounts/projects/) + + +### 1. é…ç½® Equinix Metal 主机 + +开始部署 Equinix Metal 主机。你å¯ä»¥ä½¿ç”¨ Equinix Metal 控制å°ï¼ŒAPI 或 CLI æ¥é…ç½® Equinix Metal Server。如果你需è¦äº†è§£æ¯ç§ Deployment 类型的说明，请å‚è§ [Equinix Metal 部署](https://metal.equinix.com/developers/docs/deploy/on-demand/)ã€‚ä»¥ä¸‹é“¾æŽ¥ä»‹ç» Equinix Metal Server 的类型以åŠä»·æ ¼ï¼š +- [Equinix Metal Server 类型](https://metal.equinix.com/developers/docs/servers/about/) +- [Equinix Metal ä»·æ ¼](https://metal.equinix.com/developers/docs/servers/server-specs/) + +:::note 注æ„事项： + +- 如果使用 CLI 或 API é…置新的 Equinix Metal Server，你需è¦æ供项目 IDã€è®¡åˆ’ã€metro å’Œæ“作系统。 +- 当使用云主机的虚拟机时，你需è¦å…许 80 å’Œ 443 端å£çš„入站 TCP 通信。有关端å£é…置的信æ¯ï¼Œè¯·å‚è§ä½ çš„云主机的文档。 +- 如需了解所有端å£è¦æ±‚，请å‚è§ [Docker 安装](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 +- æ ¹æ®æˆ‘们的[è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)é…置主机。 + +::: +### 2. 安装 Rancher + +è¦åœ¨ Equinix Metal 主机上安装 Rancher，先与它连接，然åŽä½¿ç”¨ shell 进行安装。 + +1. 使用你惯用的 shell（例如 PuTTy 或远程终端）登录到你的 Equinix Metal 主机。 + +2. 在 shell 中执行以下命令： + + ``` + sudo docker run -d --restart=unless-stopped -p 80:80 -p 443:443 --privileged rancher/rancher + ``` + +**结果**：Rancher 已安装。 + +### 3. 登录 + +登录到 Rancher åŽï¼Œä½ è¿˜éœ€è¦è¿›è¡Œä¸€äº›ä¸€æ¬¡æ€§é…置。 + +1. 打开 Web æµè§ˆå™¨å¹¶è¾“入主机的 IP 地å€`https://`。 + + å°† `` 替æ¢ä¸ºä½ çš„主机 IP 地å€ã€‚ + +2. 出现æ示时，为默认 `admin` è´¦å·åˆ›å»ºå¯†ç ã€‚ + +3. 设置 **Rancher Server URL**。URL å¯ä»¥æ˜¯ IP 地å€æˆ–主机å。需è¦æ³¨æ„，添加到集群中的æ¯ä¸ªèŠ‚点都必须能够连接到此 URL。

如果你在 URL 中使用主机å，则此主机å必须在 DNS 中解æžåˆ°ä½ éœ€è¦æ·»åŠ åˆ°é›†ç¾¤çš„节点上。 + +
+ +### 4. 创建集群 + +欢迎使用 Rancherï¼çŽ°åœ¨ï¼Œä½ å¯ä»¥åˆ›å»ºä½ çš„第一个 Kubernetes 集群了。 + +在此任务中，你å¯ä»¥ä½¿ç”¨**自定义**选项。此选项å…许你把 _ä»»æ„_ Linux 主机（云虚拟机ã€æœ¬åœ°è™šæ‹Ÿæœºæˆ–裸机）添加到集群中。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. 选择**自定义**。 +1. 输入**集群å称**。 +1. 点击**下一步**。 +1. 在**节点角色**中，选择 _全部_ è§’è‰²ï¼Œå³ **etcd**，**Control** å’Œ **Worker**。 + - **å¯é€‰**：Rancher 会自动检测用于 Rancher 通信和集群通信的 IP 地å€ã€‚ä½ å¯ä»¥ä½¿ç”¨**节点地å€**处的`公有地å€`å’Œ`内网地å€`进行覆盖。 +1. 将注册命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. 使用你惯用的 shell（例如 PuTTy 或远程终端）登录到你的 Linux 主机。粘贴剪贴æ¿çš„命令并è¿è¡Œã€‚ +1. 在 Linux 主机上è¿è¡Œå®Œå‘½ä»¤åŽï¼Œå•å‡»**完æˆ**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +#### 已完æˆï¼ + +æ­å–œï¼ä½ å·²åˆ›å»ºç¬¬ä¸€ä¸ªé›†ç¾¤ã€‚ + +#### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md new file mode 100644 index 00000000000..5c113fc447c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md @@ -0,0 +1,77 @@ +--- +title: Rancher GCP å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher GCP 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 GCP çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Google GCP 会产生费用。 + +::: + +- [Google GCP Account](https://console.cloud.google.com/)：用于创建部署 Rancher å’Œ Kubernetes 的资æºã€‚ +- [Google GCP 项目](https://cloud.google.com/appengine/docs/standard/nodejs/building-app/creating-project)：如果你没有的è¯ï¼Œè¯·è®¿é—®æ­¤é“¾æŽ¥æŸ¥çœ‹å¦‚何创建 GCP 项目。 +- [Google GCP ServiceAccount](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)：请访问此链接查看如何创建 GCP ServiceAccount å’Œ Token 文件。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Google GCP 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/gcp` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 GCP 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `gcp_account_json` - 替æ¢ä¸º GCP ServiceAccount 文件路径和文件å。 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [GCP Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/gcp) 了解更多信æ¯ã€‚ + 建议包括： + - `gcp_region` - Google GCP 区域。GCP 的默认区域 (`us-east4`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `gcp_zone` - Google GCP 区域。GCP 的默认区域 (`us-east4-a`) ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `machine_type` - 使用的计算实例大å°ï¼Œæœ€å°è§„格为 `n1-standard-1`。如果在预算范围内，å¯ä»¥ä½¿ç”¨ `n1-standard-2` 或 `n1-standard-4`。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/gcp` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 GCP 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/gcp` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md new file mode 100644 index 00000000000..b3acb4f9ecc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md @@ -0,0 +1,154 @@ +--- +title: Helm CLI 快速入门 +--- + +本文æ供了快速安装 Rancher 的方法。 + +这些说明å‡è®¾ä½ æœ‰ä¸€ä¸ª Linux 虚拟机，并能从本地工作站与之通信。Rancher 将安装在 Linux 主机上。你将需è¦æ£€ç´¢è¯¥ä¸»æœºçš„ IP 地å€ï¼Œä»¥ä¾¿ä»Žæœ¬åœ°å·¥ä½œç«™è®¿é—® Rancher。Rancher æ—¨åœ¨è¿œç¨‹ç®¡ç† Kubernetes 集群，因此 Rancher 管ç†çš„任何 Kubernetes 集群也都需è¦èƒ½å¤Ÿè®¿é—®è¯¥ IP 地å€ã€‚ + +我们ä¸å»ºè®®åœ¨æœ¬åœ°å®‰è£… Rancher，因为它会产生网络问题。如果你在 localhost 上安装 Rancher，Rancher 无法与下游 Kubernetes 集群通信，因此在 localhost 上你无法测试 Rancher 的集群é…置和集群管ç†åŠŸèƒ½ã€‚ + +ä½ çš„ Linux 主机å¯ä»¥ä½äºŽä»»ä½•åœ°æ–¹ã€‚例如，它å¯ä»¥æ˜¯ Amazon EC2 实例ã€Digital Ocean Droplet 或 Azure 虚拟机。其他 Rancher 文档也ç»å¸¸ç§°å®ƒä»¬ä¸ºâ€œèŠ‚点â€ã€‚部署 Linux 主机的一ç§æ–¹æ³•æ˜¯è®¾ç½®ä¸€ä¸ª Amazon EC2 实例，如[本教程](../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md)中所示。 + +完整的安装è¦æ±‚在[这里](../../../pages-for-subheaders/installation-requirements.md)。 + +## 在 Linux 上安装 K3s + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +在 Linux 主机上è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å®‰è£… K3s 集群： + +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server --cluster-init +``` + +`--cluster-init` å…许 K3s ä½¿ç”¨åµŒå…¥å¼ etcd 作为数æ®å­˜å‚¨ï¼Œå¹¶èƒ½å¤Ÿè½¬æ¢ä¸º HA 设置。请å‚阅[嵌入å¼æ•°æ®åº“的高å¯ç”¨æ€§](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/)。 + +ä¿å­˜ Linux 主机的 IP。 + +## å°† kubeconfig ä¿å­˜åˆ°ä½ çš„工作站 + +kubeconfig 文件对于访问 Kubernetes 集群éžå¸¸é‡è¦ã€‚从 Linux 主机å¤åˆ¶ `/etc/rancher/k3s/k3s.yaml` 中的文件，并将其ä¿å­˜åˆ°æœ¬åœ°å·¥ä½œç«™çš„ `~/.kube/config` 目录中。一ç§æ–¹æ³•æ˜¯ä½¿ç”¨ `scp` 工具并在本地计算机上è¿è¡Œæ­¤å‘½ä»¤ï¼š + + + + +``` +scp root@:/etc/rancher/k3s/k3s.yaml ~/.kube/config +``` + +在æŸäº›æƒ…况下，它å¯èƒ½éœ€è¦ç¡®ä¿ä½ çš„ shell 定义了环境å˜é‡ `KUBECONFIG=~/.kube/config`，例如，它å¯ä»¥åœ¨ä½ çš„é…置文件或 rc 文件中导出。 + + + + +默认情况下ä¸èƒ½è¯†åˆ«â€œscpâ€å‘½ä»¤ï¼Œæ‰€ä»¥æˆ‘们需è¦å…ˆå®‰è£…一个模å—。 + +在 Windows Powershell 中： + +``` +Find-Module Posh-SSH +Install-Module Posh-SSH + +## 获å–远程 kubeconfig 文件 +scp root@:/etc/rancher/k3s/k3s.yaml $env:USERPROFILE\.kube\config +``` + + + + +## 在 kubeconfig 中编辑 Rancher Server URL + +在 kubeconfig 文件中，你需è¦å°† `server` 字段的值更改为 `:6443`。你å¯ä»¥é€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server。你需è¦è¿›è¡Œæ­¤ç¼–辑，以便你从本地工作站è¿è¡Œ Helm 或 kubectl 命令时，能够与安装了 Rancher çš„ Kubernetes 集群进行通信。 + + + + +打开 kubeconfig 文件进行编辑的一ç§æ–¹æ³•æ˜¯ä½¿ç”¨ Vim： + +``` +vi ~/.kube/config +``` + +输入 `i` 以打开 Vim çš„æ’入模å¼ã€‚è¦ä¿å­˜ä½ çš„工作，请按 `Esc`。然åŽè¾“å…¥ `:wq` 并按 `Enter`。 + + + + + +在 Windows Powershell 中，你å¯ä»¥ä½¿ç”¨ `notepad.exe` æ¥ç¼–辑 kubeconfig 文件： + +``` +notepad.exe $env:USERPROFILE\.kube\config +``` + +编辑完æˆåŽï¼ŒæŒ‰ `ctrl+s` 或转到 `File > Save` æ¥ä¿å­˜ä½ çš„工作。 + + + + +## 使用 Helm æ¥å®‰è£… Rancher + +从本地工作站è¿è¡Œä»¥ä¸‹å‘½ä»¤ã€‚你需è¦å…ˆå®‰è£… [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) å’Œ [helm](https://helm.sh/docs/intro/install/)： + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + +kubectl create namespace cattle-system + +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +helm repo add jetstack https://charts.jetstack.io + +helm repo update + +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 + +# Windows Powershell +helm install cert-manager jetstack/cert-manager ` + --namespace cert-manager ` + --create-namespace ` + --version v1.11.0 +``` + +安装 Rancher 的最终命令如下。该命令需è¦ä¸€ä¸ªå°†æµé‡è½¬å‘到 Linux 主机的域å。为了简化本教程，你å¯ä»¥ä½¿ç”¨å‡åŸŸå。`.sslip.io` 是一个å‡åŸŸå的例å­ã€‚ + +è¦å®‰è£…特定的 Rancher 版本，请使用 `--version` 标志（例如，`--version 2.6.6`）。å¦åˆ™ï¼Œé»˜è®¤å®‰è£…最新的 Rancher。请å‚阅[选择 Rancher 版本](../../installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +请注æ„，密ç è‡³å°‘éœ€è¦ 12 个字符。 + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=.sslip.io \ + --set replicas=1 \ + --set bootstrapPassword= + +# Windows Powershell +helm install rancher rancher-latest/rancher ` + --namespace cattle-system ` + --set hostname=.sslip.io ` + --set replicas=1 ` + --set bootstrapPassword= +``` + +现在，如果你在 Web æµè§ˆå™¨ä¸­å¯¼èˆªåˆ° `.sslip.io`，你应该会看到 Rancher UI。 + +为了简化说明，我们使用了一个å‡åŸŸå和自签åè¯ä¹¦æ¥è¿›è¡Œå®‰è£…。因此，你å¯èƒ½éœ€è¦åœ¨ Web æµè§ˆå™¨ä¸­æ·»åŠ ä¸€ä¸ªå®‰å…¨ä¾‹å¤–æ¥æŸ¥çœ‹ Rancher UI。请注æ„，对于生产安装，你需è¦å…·æœ‰è´Ÿè½½å‡è¡¡å™¨ã€çœŸå®žåŸŸå和真实è¯ä¹¦çš„高å¯ç”¨æ€§è®¾ç½®ã€‚ + +这些说明还çœç•¥äº†å®Œæ•´çš„安装è¦æ±‚和其他安装选项。如果你对这些步骤有任何疑问，请å‚阅完整的 [Helm CLI 安装文档](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)。 + +è¦ä½¿ç”¨æ–°çš„ Rancher Server æ¥å¯åŠ¨æ–°çš„ Kubernetes 集群，你å¯èƒ½éœ€è¦åœ¨ Rancher 中设置云凭è¯ã€‚有关更多信æ¯ï¼Œè¯·å‚阅[使用 Rancher å¯åŠ¨ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md new file mode 100644 index 00000000000..585353441b2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md @@ -0,0 +1,76 @@ +--- +title: Rancher Hetzner Cloud å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher Hetzner Cloud 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 Hetzner Cloud çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Hetzner Cloud 会产生费用。 + +::: + +- [Hetzner Cloud è´¦å·](https://www.hetzner.com)：用于è¿è¡ŒæœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ +- [Hetzner API 访问密钥](https://docs.hetzner.cloud/#getting-started)：如果你没有的è¯ï¼Œè¯·å‚考说明创建一个。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Hetzner 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/hcloud` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 Hetzner 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `hcloud_token` - 替æ¢ä¸º Hetzner API 访问密钥。 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [Hetzner Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/hcloud) 了解更多信æ¯ã€‚ + 建议包括： + + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - 实例类型，至少需è¦æ˜¯ `cx21`。 + - `hcloud_location`- Hetzner Cloud ä½ç½®ã€‚选择最近的ä½ç½®ï¼Œè€Œä¸æ˜¯ä½¿ç”¨é»˜è®¤ä½ç½®ï¼ˆ`fsn1`）。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/hcloud` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 Hetzner 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/hcloud` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md new file mode 100644 index 00000000000..46b42901db9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md @@ -0,0 +1,76 @@ +--- +title: Rancher Outscale å¿«é€Ÿå…¥é—¨æŒ‡å— +description: 阅读此分步 Rancher Outscale 指å—，以快速部署带有å•èŠ‚点下游 Kubernetes 集群的 Rancher Server。 +--- + +ä½ å¯ä»¥å‚考以下步骤，在 Outscale çš„å•èŠ‚点 K3s Kubernetes 集群中快速部署 Rancher Server，并附加一个å•èŠ‚点下游 Kubernetes 集群。 + +:::note + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +:::caution + +部署到 Outscale 会产生费用。 + +::: + +- [Outscale è´¦å·](https://en.outscale.com/)：用于è¿è¡ŒæœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ +- [Outscale 访问密钥](https://docs.outscale.com/en/userguide/About-Access-Keys.html)：如果你没有的è¯ï¼Œè¯·æŒ‰ç…§è¯´æ˜Žåˆ›å»ºä¸€ä¸ª Outscale 访问密钥。 +- [Terraform](https://www.terraform.io/downloads.html)：用于在 Outscale 中é…ç½®æœåŠ¡å™¨å’Œé›†ç¾¤ã€‚ + + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/outscale` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Terraform 文件的 Outscale 文件夹。 + +3. 把 `terraform.tfvars.example` 文件é‡å‘½å为 `terraform.tfvars`。 + +4. 编辑 `terraform.tfvars` 文件，并替æ¢ä»¥ä¸‹å˜é‡ï¼š + - `access_key_id` - 替æ¢ä¸º Outscale 访问密钥 + - `secret_key_id` - 替æ¢ä¸º Outscale 密文密钥 + - `rancher_server_admin_password` - 替æ¢ä¸ºåˆ›å»º Rancher Server çš„ admin è´¦å·çš„密ç ï¼ˆæœ€å°‘ 12 字符） + +5. **å¯é€‰**：修改 `terraform.tfvars` 中的å¯é€‰å‚数。 + å‚è§ [Quickstart Readme](https://github.com/rancher/quickstart) ä»¥åŠ [Outscale Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/outscale) 了解更多信æ¯ã€‚ + 建议包括： + - `region` - Outscale 区域。Outscale 的默认区域ä¸ä¸€å®šæ˜¯è·ç¦»ä½ æœ€è¿‘的区域。建议修改为è·ç¦»ä½ æœ€è¿‘的区域（`eu-west-2`）。 + - `prefix` - 所有创建资æºçš„å‰ç¼€ + - `instance_type` - 实例类型，至少需è¦æ˜¯ `tinav3.c2r4p3`。 + +6. 执行 `terraform init`。 + +7. 执行 `terraform apply --auto-approve` 以åˆå§‹åŒ–环境。然åŽï¼Œç­‰å¾…命令行工具返回以下信æ¯ï¼š + + ``` + Apply complete! Resources: 21 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. 将以上输出中的 `rancher_server_url` 粘贴到æµè§ˆå™¨ä¸­ã€‚在登录页é¢ä¸­ç™»å½•ï¼ˆé»˜è®¤ç”¨æˆ·å为 `admin`，密ç ä¸ºåœ¨ `rancher_server_admin_password` 中设置的密ç ï¼‰ã€‚ +9. 使用 `quickstart/rancher/outscale` 中生æˆçš„ `id_rsa` 密钥 SSH 到 Rancher Server。 + +#### 结果 + +两个 Kubernetes 集群已部署到你的 Outscale 账户中，一个è¿è¡Œ Rancher Server，å¦ä¸€ä¸ªä¸ºå®žéªŒéƒ¨ç½²åšå¥½å‡†å¤‡ã€‚请注æ„，虽然这ç§è®¾ç½®æ˜¯æŽ¢ç´¢ Rancher 功能的好方法，但在生产环境中，应éµå¾ªæˆ‘们的高å¯ç”¨è®¾ç½®æŒ‡å—。用于虚拟机的 SSH 密钥是自动生æˆçš„，存储在模å—目录中。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/outscale` 文件夹，然åŽæ‰§è¡Œ `terraform destroy --auto-approve`。 + +2. 等待命令行界é¢æ˜¾ç¤ºèµ„æºå·²åˆ é™¤çš„消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/prime.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/prime.md new file mode 100644 index 00000000000..094508ff5a6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/prime.md @@ -0,0 +1,7 @@ +--- +title: Rancher Prime +--- + +Rancher 在 v2.7 中引入了 Rancher Prime，这是 Rancher ä¼ä¸šçº§äº§å“的进化。Rancher Prime 是基于相åŒæºä»£ç æž„建的商业化ã€ä¼ä¸šçº§çš„新版本。因此，Rancher 的产å“将继续ä¿æŒ 100% å¼€æºï¼Œå¹¶é€šè¿‡å®‰å…¨ã€å»¶é•¿ç”Ÿå‘½å‘¨æœŸã€è®¿é—®é‡ç‚¹æž¶æž„å’Œ Kubernetes 公告体现é¢å¤–价值。Rancher Prime 还将æ供选项，让用户获得创新 Rancher 项目的生产支æŒã€‚使用 Rancher Prime，安装 asset 会托管在由 Rancher æŒæœ‰å’Œç®¡ç†çš„å¯ä¿¡é•œåƒä»“库中。 + +è¦å¼€å§‹ä½¿ç”¨ Rancher Prime，请[转到此页é¢](https://www.rancher.com/quick-start)并填写表格。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md new file mode 100644 index 00000000000..dcddb74cfe9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md @@ -0,0 +1,51 @@ +--- +title: Vagrant 快速入门 +--- + +ä½ å¯ä»¥å‚考以下步骤快速部署 Rancher Server，并附加一个å•èŠ‚点集群。 + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +::: + +## 先决æ¡ä»¶ + +- [Vagrant](https://www.vagrantup.com)：Vagrant æ˜¯å¿…éœ€çš„ï¼Œç”¨äºŽæ ¹æ® Vagrantfile é…置主机。 +- [Virtualbox](https://www.virtualbox.org)：需è¦æŠŠ Vagrant é…置的虚拟机é…置到 VirtualBox。 +- 至少 4GB çš„å¯ç”¨å†…存。 + +### æ³¨æ„ +- Vagrant 需è¦ä½¿ç”¨æ’件æ¥åˆ›å»º VirtualBox 虚拟机。请执行以下命令进行安装： + + `vagrant plugin install vagrant-vboxmanage` + + `vagrant plugin install vagrant-vbguest` + +## 开始使用 + +1. 使用命令行工具，执行 `git clone https://github.com/rancher/quickstart` 把 [Rancher Quickstart](https://github.com/rancher/quickstart) 克隆到本地。 + +2. 执行 `cd quickstart/rancher/vagrant` å‘½ä»¤ï¼Œè¿›å…¥åŒ…å« Vagrantfile 文件的文件夹。 + +3. **å¯é€‰**：编辑 `config.yaml` 文件： + + - æ ¹æ®éœ€è¦æ›´æ”¹èŠ‚点数和内存分é…（`node.count`, `node.cpus`, `node.memory`） + - 更改 `admin` 的密ç ä»¥ç™»å½• Rancher。(`admin_password`) + +4. 执行 `vagrant up --provider=virtualbox` 以åˆå§‹åŒ–环境。 + +5. é…置完æˆåŽï¼Œåœ¨æµè§ˆå™¨ä¸­æ‰“å¼€ `https://192.168.56.101`。默认的用户å和密ç æ˜¯ `admin/adminPassword`。 + +**结果**：Rancher Server 和你的 Kubernetes 集群已安装在 VirtualBox 上。 + +### åŽç»­æ“作 + +使用 Rancher 创建 deployment。详情请å‚è§[创建 Deployment](../../../pages-for-subheaders/deploy-rancher-workloads.md)。 + +## 销æ¯çŽ¯å¢ƒ + +1. 进入 `quickstart/rancher/vagrant` 文件夹，然åŽæ‰§è¡Œ `vagrant destroy -f`。 + +2. 等待所有资æºå·²åˆ é™¤çš„确认消æ¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-workloads/nodeports.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-workloads/nodeports.md new file mode 100644 index 00000000000..bf2294d4c06 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-workloads/nodeports.md @@ -0,0 +1,138 @@ +--- +title: 部署带有 NodePort 的工作负载 +--- + +### 先决æ¡ä»¶ + +你已有一个正在è¿è¡Œçš„集群，且该集群中有至少一个节点。 + +### 1. 部署工作负载 + +ä½ å¯ä»¥å¼€å§‹åˆ›å»ºä½ çš„第一个 Kubernetes [工作负载](https://kubernetes.io/docs/concepts/workloads/)ã€‚å·¥ä½œè´Ÿè½½æ˜¯ä¸€ä¸ªå¯¹è±¡ï¼Œå…¶ä¸­åŒ…å« pod 以åŠéƒ¨ç½²åº”用所需的其他文件和信æ¯ã€‚ + +在本文的工作负载中，你将部署一个 Rancher Hello-World 应用。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥éœ€è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 为工作负载设置**å称**。 +1. 在**容器镜åƒ**字段中，输入 `rancher/hello-world`。注æ„区分大å°å†™ã€‚ +1. 点击**添加端å£**。 +1. 在**æœåŠ¡ç±»åž‹**下拉èœå•ä¸­ï¼Œç¡®ä¿é€‰æ‹©äº† **NodePort**。 + + ![NodePort 下拉èœå•ï¼ˆåœ¨æ¯ä¸ªæ‰€é€‰èŠ‚点处）](/img/nodeport-dropdown.png) + +1. 在**å‘布容器端å£**字段中，输入端å£`80`。 + + ![å‘布容器端å£ï¼Œç«¯å£ 80 已输入](/img/container-port-field.png) + +1. å•å‡»**创建**。 + +**结果**： + +* 工作负载已部署。此过程å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿã€‚ +* 当工作负载完æˆéƒ¨ç½²åŽï¼Œå®ƒçš„状æ€ä¼šå˜ä¸º **Active**。你å¯ä»¥ä»Žé¡¹ç›®çš„**工作负载**页é¢æŸ¥çœ‹å…¶çŠ¶æ€ã€‚ + +
+ +### 2. 查看应用 + +在**工作负载**页é¢ä¸­ï¼Œç‚¹å‡»å·¥ä½œè´Ÿè½½ä¸‹æ–¹çš„链接。如果 deployment 已完æˆï¼Œä½ çš„应用会打开。 + +### 注æ„事项 + +如果使用云虚拟机，你å¯èƒ½æ— æ³•è®¿é—®è¿è¡Œå®¹å™¨çš„端å£ã€‚è¿™ç§æƒ…况下，你å¯ä»¥ä½¿ç”¨ `Execute Shell` 在本地主机的 SSH 会è¯ä¸­æµ‹è¯• Nginx。如果å¯ç”¨çš„è¯ï¼Œä½¿ç”¨å·¥ä½œè´Ÿè½½ä¸‹æ–¹çš„链接中 `:` åŽé¢çš„端å£å·ã€‚在本例中，端å£å·ä¸º `31568`。 + +```html +gettingstarted@rancher:~$ curl http://localhost:31568 + + + + Rancher + + + + + +

Hello world!

+

My hostname is hello-world-66b4b9d88b-78bhx

+
+

k8s services found 2

+ + INGRESS_D1E1A394F61C108633C4BD37AEDDE757 tcp://10.43.203.31:80
+ + KUBERNETES tcp://10.43.0.1:443
+ +
+
+ + +
+ + +
+ + + +gettingstarted@rancher:~$ + +``` + +### 已完æˆï¼ + +æ­å–œï¼ä½ å·²æˆåŠŸé€šè¿‡ NodePort 部署工作负载。 + +#### åŽç»­æ“作 + +使用完沙盒åŽï¼Œä½ éœ€è¦æ¸…ç† Rancher Server 和集群。详情请å‚è§ï¼š + +- [Amazon AWS：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/aws.md#销æ¯çŽ¯å¢ƒ) +- [DigitalOcean：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/digitalocean.md#销æ¯çŽ¯å¢ƒ) +- [Vagrant：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/vagrant.md#销æ¯çŽ¯å¢ƒ) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md new file mode 100644 index 00000000000..86f8017f09e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md @@ -0,0 +1,72 @@ +--- +title: 部署带有 Ingress 的工作负载 +--- + +### 先决æ¡ä»¶ + +你已有一个正在è¿è¡Œçš„集群，且该集群中有至少一个节点。 + +### 1. 部署工作负载 + +ä½ å¯ä»¥å¼€å§‹åˆ›å»ºä½ çš„第一个 Kubernetes [工作负载](https://kubernetes.io/docs/concepts/workloads/)ã€‚å·¥ä½œè´Ÿè½½æ˜¯ä¸€ä¸ªå¯¹è±¡ï¼Œå…¶ä¸­åŒ…å« pod 以åŠéƒ¨ç½²åº”用所需的其他文件和信æ¯ã€‚ + +在本文的工作负载中，你将部署一个 Rancher Hello-World 应用。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 点击 **Deployment**。 +1. 为工作负载设置**å称**。 +1. 在**容器镜åƒ**字段中，输入 `rancher/hello-world`。注æ„区分大å°å†™ã€‚ +1. 在 `Service Type` 点击 **Add Port** å’Œ `Cluster IP`，并在 **Private Container Port** 字段中输入`80`。你å¯ä»¥å°† `Name` 留空或指定å称。通过添加端å£ï¼Œä½ å¯ä»¥è®¿é—®é›†ç¾¤å†…外的应用。有关详细信æ¯ï¼Œè¯·å‚阅 [Service](../../../pages-for-subheaders/workloads-and-pods.md#services)。 +1. å•å‡»**创建**。 + +**结果**： + +* 工作负载已部署。此过程å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿã€‚ +* 当工作负载完æˆéƒ¨ç½²åŽï¼Œå®ƒçš„状æ€ä¼šå˜ä¸º **Active**。你å¯ä»¥ä»Žé¡¹ç›®çš„**工作负载**页é¢æŸ¥çœ‹å…¶çŠ¶æ€ã€‚ + +### 2. 通过 Ingress 暴露应用 + +现在应用已å¯åŠ¨å¹¶è¿è¡Œï¼Œä½ éœ€è¦æš´éœ²åº”用以让其他æœåŠ¡è¿žæŽ¥åˆ°å®ƒã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 + +1. 点击**æœåŠ¡å‘现 > Ingresses**。 + +1. 点击**创建**。 + +1. 在选择**命å空间**时，你需è¦é€‰æ‹©åœ¨åˆ›å»º deployment 时使用的命å空间。å¦åˆ™ï¼Œåœ¨æ­¥éª¤8中选择**目标æœåŠ¡**时，你的 deployment 会ä¸å¯ç”¨ã€‚ + +1. 输入**å称**，例如 **hello**。 + +1. 指定**路径**，例如 `/hello`。 + +1. 在**目标æœåŠ¡**字段的下拉èœå•ä¸­ï¼Œé€‰æ‹©ä½ ä¸ºæœåŠ¡è®¾ç½®çš„å称。 + +1. 在**端å£**字段中的下拉èœå•ä¸­ï¼Œé€‰æ‹© `80`。 + +1. 点击å³ä¸‹è§’çš„**创建**。 + +**结果**：应用分é…到了一个 `sslip.io` 地å€å¹¶æš´éœ²ã€‚è¿™å¯èƒ½éœ€è¦ä¸€ä¸¤åˆ†é’Ÿã€‚ + + +### 查看应用 + +在 **Deployments** 页é¢ä¸­ï¼Œæ‰¾åˆ°ä½  deployment çš„ **endpoint** 列，然åŽå•å‡»ä¸€ä¸ª endpoint。å¯ç”¨çš„ endpoint å–决于你添加到 deployment 中的端å£é…置。如果你看ä¸åˆ°éšæœºåˆ†é…端å£çš„ endpoint，请将你在创建 Ingress 时指定的路径尾附到 IP 地å€ä¸Šã€‚例如，如果你的 endpoint 是 `xxx.xxx.xxx.xxx` 或 `https://xxx.xxx.xxx.xxx`，把它修改为 `xxx.xxx.xxx.xxx/hello` 或 `https://xxx.xxx.xxx.xxx/hello`。 + +应用将在å¦ä¸€ä¸ªçª—å£ä¸­æ‰“开。 + +#### 已完æˆï¼ + +æ­å–œï¼ä½ å·²æˆåŠŸé€šè¿‡ Ingress 部署工作负载。 + +#### åŽç»­æ“作 + +使用完沙盒åŽï¼Œä½ éœ€è¦æ¸…ç† Rancher Server 和集群。详情请å‚è§ï¼š + +- [Amazon AWS：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/aws.md#销æ¯çŽ¯å¢ƒ) +- [DigitalOcean：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/digitalocean.md#销æ¯çŽ¯å¢ƒ) +- [Vagrant：销æ¯çŽ¯å¢ƒ](../deploy-rancher-manager/vagrant.md#销æ¯çŽ¯å¢ƒ) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md new file mode 100644 index 00000000000..643a7122e37 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md @@ -0,0 +1,40 @@ +--- +title: 为定时扫æé…置告警 +--- + +ä½ å¯ä»¥å®šæ—¶è¿è¡Œ ClusterScan。 + +你还å¯ä»¥ä¸ºå®šæ—¶æ‰«æ指定是å¦åœ¨æ‰«æ完æˆæ—¶å‘出告警。 + +åªæœ‰å®šæ—¶è¿è¡Œçš„扫ææ‰æ”¯æŒå‘Šè­¦ã€‚ + +CIS Benchmark 应用支æŒä¸¤ç§ç±»åž‹çš„告警： + +- 扫æ完æˆå‘Šè­¦ï¼šæ­¤å‘Šè­¦åœ¨æ‰«æè¿è¡Œå®Œæˆæ—¶å‘出。告警包括详细信æ¯ï¼ŒåŒ…括 ClusterScan çš„å称和 ClusterScanProfile çš„å称。 +- 扫æ失败告警：如果扫æ中有一些测试失败或扫æ处于 `Fail` 状æ€ï¼Œåˆ™ä¼šå‘出此告警。 + +:::note 先决æ¡ä»¶ï¼š + +为 `rancher-cis-benchmark` å¯ç”¨å‘Šè­¦ä¹‹å‰ï¼Œç¡®ä¿å®‰è£…了 `rancher-monitoring` 应用并é…置了接收器（Receiver）和路由（Route）。详情请å‚è§[本章节](../../../reference-guides/monitoring-v2-configuration/receivers.md)。 + +在为 `rancher-cis-benchmark` å‘Šè­¦é…置路由时，你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹ `job:rancher-cis-scan` æ¥æŒ‡å®šåŒ¹é…。详情请查看[路由é…置示例](../../../reference-guides/monitoring-v2-configuration/receivers.md#cis-扫æ告警的示例路由é…ç½®)。 + +::: + +è¦ä¸ºå®šæ—¶è¿è¡Œçš„扫æé…置告警： + +1. 请在 `rancher-cis-benchmark` 应用程åºä¸Šå¯ç”¨å‘Šè­¦ã€‚详情请å‚è§[本页](../../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md)。 +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. å•å‡»**创建**。 +1. 选择集群扫æé…置文件。该é…置文件确定è¦ä½¿ç”¨å“ªä¸ª CIS Benchmark 版本以åŠè¦æ‰§è¡Œå“ªäº›æµ‹è¯•ã€‚如果你选择 Default é…置文件，则 CIS Operator 将选择适用于它所在的 Kubernetes 集群类型的é…置文件。 +1. 选择**定时è¿è¡Œæ‰«æ**的选项。 +1. 在**调度**字段中输入有效的 [Cron 表达å¼](https://en.wikipedia.org/wiki/Cron#CRON_expression)。 +1. 选中**å‘Šè­¦**下告警类型æ—边的框。 +1. （å¯é€‰ï¼‰é€‰æ‹©ä¸€ä¸ª**ä¿ç•™è®¡æ•°**，表示为这个定时扫æ维护的报告数é‡ã€‚默认情况下，此计数为 3。超过此ä¿ç•™é™åˆ¶æ—¶ï¼Œæ—§æŠ¥å‘Šå°†è¢«åˆ é™¤ã€‚ +1. å•å‡»**创建**。 + +**结果**：扫æè¿è¡Œï¼Œå¹¶æ ¹æ®è®¾ç½®çš„ cron 表达å¼é‡æ–°è°ƒåº¦ã€‚如果在 `rancher-monitoring` 应用下é…置了路由和接收器，则会在扫æ完æˆæ—¶å‘出告警。 + +æ¯æ¬¡è¿è¡Œæ‰«æ都会生æˆä¸€ä»½å¸¦æœ‰æ‰«æ结果的报告。如需查看最新的结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md new file mode 100644 index 00000000000..fe8477f7b55 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md @@ -0,0 +1,9 @@ +--- +title: 为集群扫æ创建自定义 Benchmark 版本 +--- + +æŸäº› Kubernetes 集群å¯èƒ½éœ€è¦è‡ªå®šä¹‰é…ç½® Benchmark 测试。例如，Kubernetes é…置文件或è¯ä¹¦çš„路径å¯èƒ½ä¸Žä¸Šæ¸¸ CIS Benchmark 的标准ä½ç½®ä¸åŒã€‚ + +现在，你å¯ä»¥ä½¿ç”¨ `rancher-cis-benchmark` 应用æ¥åˆ›å»ºè‡ªå®šä¹‰ Benchmark 版本，从而è¿è¡Œé›†ç¾¤æ‰«æ。 + +有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/cis-scans/custom-benchmark.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md new file mode 100644 index 00000000000..55c34dcdf5c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md @@ -0,0 +1,20 @@ +--- +title: 为 Rancher CIS Benchmark å¯ç”¨å‘Šè­¦ +--- + +ä½ å¯ä»¥é…置告警，从而将告警å‘é€ç»™å®šæ—¶è¿è¡Œçš„扫æ。 + +:::note 先决æ¡ä»¶ï¼š + +为 `rancher-cis-benchmark` å¯ç”¨å‘Šè­¦ä¹‹å‰ï¼Œç¡®ä¿å®‰è£…了 `rancher-monitoring` 应用并é…置了接收器（Receiver）和路由（Route）。详情请å‚è§[本章节](../../../reference-guides/monitoring-v2-configuration/receivers.md)。 + +在为 `rancher-cis-benchmark` å‘Šè­¦é…置路由时，你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹ `job:rancher-cis-scan` æ¥æŒ‡å®šåŒ¹é…。详情请查看[路由é…置示例](../../../reference-guides/monitoring-v2-configuration/receivers.md#cis-扫æ告警的示例路由é…ç½®)。 + +::: + +在安装或å‡çº§ `rancher-cis-benchmark` Helm Chart 时，在 `values.yaml` 中将以下标志设置为 `true`： + +```yaml +alerts: + enabled: true +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md new file mode 100644 index 00000000000..814d38c5349 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md @@ -0,0 +1,17 @@ +--- +title: 安装 Rancher CIS Benchmark +--- + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å®‰è£… CIS Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Charts**。 +1. å•å‡» **CIS Benchmark**。 +1. å•å‡»**安装**。 + +**结果**：CIS 扫æ应用已ç»éƒ¨ç½²åœ¨ Kubernetes 集群上。 + +:::note + +如果你使用 Kubernetes v1.24 或更早版本，并且具有使用 [Pod 安全策略](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) (PSP) 加固的集群，则 CIS Benchmark 4.0.0 åŠæ›´é«˜ç‰ˆæœ¬ä¼šé»˜è®¤ç¦ç”¨ PSP。è¦åœ¨ PSP 加固集群上安装 CIS Benchmark，请在安装 Chart 之å‰å°† values 中的 `global.psp.enabled` 设置为 `true`。[Pod 安全准入](../../new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md) (PSA) 加固集群ä¸å—å½±å“。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md new file mode 100644 index 00000000000..d43567fbb25 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md @@ -0,0 +1,20 @@ +--- +title: 定时è¿è¡Œæ‰«æ +--- + +è¦å®šæ—¶è¿è¡Œ ClusterScan： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. 选择集群扫æé…置文件。该é…置文件确定è¦ä½¿ç”¨å“ªä¸ª CIS Benchmark 版本以åŠè¦æ‰§è¡Œå“ªäº›æµ‹è¯•ã€‚如果你选择 Default é…置文件，则 CIS Operator 将选择适用于它所在的 Kubernetes 集群类型的é…置文件。 +1. 选择**定时è¿è¡Œæ‰«æ**的选项。 +1. 将一个有效的 Cron 表达å¼å¡«å†™åˆ°**调度**字段。 +1. 选择一个**ä¿ç•™è®¡æ•°**，表示为这个定时扫æ维护的报告数é‡ã€‚默认情况下，此计数为 3。超过此ä¿ç•™é™åˆ¶æ—¶ï¼Œæ—§æŠ¥å‘Šå°†è¢«åˆ é™¤ã€‚ +1. å•å‡»**创建**。 + +**结果**：扫æè¿è¡Œï¼Œå¹¶æ ¹æ®è®¾ç½®çš„ cron 表达å¼é‡æ–°è°ƒåº¦ã€‚**下一次扫æ**的值表示下次è¿è¡Œæ­¤æ‰«æ的时间。 + +æ¯æ¬¡è¿è¡Œæ‰«æ都会生æˆä¸€ä»½å¸¦æœ‰æ‰«æ结果的报告。如需查看最新的结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 + +你还å¯ä»¥åœ¨æ‰«æ详情页é¢ä¸Šçš„**报告**下拉èœå•ä¸­æŸ¥çœ‹ä¹‹å‰çš„报告。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md new file mode 100644 index 00000000000..e658029233e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md @@ -0,0 +1,22 @@ +--- +title: è¿è¡Œæ‰«æ +--- + +创建 ClusterScan 自定义资æºåŽï¼Œå®ƒä¼šåœ¨é›†ç¾¤ä¸Šä¸ºæ‰€é€‰ ClusterScanProfile å¯åŠ¨æ–°çš„ CIS 扫æ。 + +:::note + +请注æ„，目å‰ä¸€ä¸ªé›†ç¾¤æ¯æ¬¡åªèƒ½è¿è¡Œä¸€æ¬¡ CIS 扫æ。如果你创建了多个 ClusterScan 自定义资æºï¼Œoperator åªèƒ½ä¸€ä¸ªæŽ¥ä¸€ä¸ªåœ°è¿è¡Œè¿™äº›èµ„æºã€‚一个扫æ完æˆä¹‹å‰ï¼Œå…¶ä½™ ClusterScan 自定义资æºå°†å¤„于 “Pending†状æ€ã€‚ + +::: + +è¦è¿è¡Œæ‰«æ： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. å•å‡»**创建**。 +1. 选择集群扫æé…置文件。该é…置文件确定è¦ä½¿ç”¨å“ªä¸ª CIS Benchmark 版本以åŠè¦æ‰§è¡Œå“ªäº›æµ‹è¯•ã€‚如果你选择 Default é…置文件，则 CIS Operator 将选择适用于它所在的 Kubernetes 集群类型的é…置文件。 +1. å•å‡»**创建**。 + +**结果**：已生æˆå¸¦æœ‰æ‰«æ结果的报告。如需查看结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md new file mode 100644 index 00000000000..230bb1c7696 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md @@ -0,0 +1,34 @@ +--- +title: 跳过测试 +--- + +用户å¯ä»¥åœ¨æµ‹è¯•é…置文件中自定义è¦è·³è¿‡çš„æµ‹è¯•ï¼Œç„¶åŽ CIS 扫æå¯ä»¥ä½¿ç”¨è¯¥é…置文件è¿è¡Œã€‚ + +è¦è·³è¿‡æµ‹è¯•ï¼Œä½ éœ€è¦åˆ›å»ºè‡ªå®šä¹‰ä¸€ä¸ª CIS 扫æé…置文件。é…ç½®æ–‡ä»¶åŒ…å« CIS 扫æçš„é…置，包括è¦ä½¿ç”¨çš„ Benchmark 测试版本以åŠè¦åœ¨è¯¥ Benchmark 测试中跳过的测试。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **CIS Benchmark > é…置文件**。 +1. 在这里，你å¯ä»¥ä½¿ç”¨å¤šç§æ–¹å¼æ¥åˆ›å»ºé…置文件。è¦åˆ›å»ºæ–°é…置文件，å•å‡»**创建**并在 UI 中填写表å•ã€‚è¦åŸºäºŽçŽ°æœ‰é…置文件æ¥åˆ›å»ºæ–°é…置文件，请转到现有é…置文件并å•å‡»**â‹® 克隆**。如果你在填写表å•ï¼Œè¯·ä½¿ç”¨æµ‹è¯• ID 添加è¦è·³è¿‡çš„测试，并å‚考相关的 CIS Benchmark。如果你将新的测试é…置文件创建为 YAML，你需è¦åœ¨ `skipTests` å‚数中添加è¦è·³è¿‡çš„测试的 ID。你还需è¦ä¸ºé…置文件命å： + + ```yaml + apiVersion: cis.cattle.io/v1 + kind: ClusterScanProfile + metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" + spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" + ``` +1. å•å‡»**创建**。 + +**结果**：已创建一个新的 CIS 扫æé…置文件。 + +使用此é…置文件[è¿è¡Œæ‰«æ](./run-a-scan.md)时，会跳过定义的跳过测试。跳过的测试将在生æˆçš„报告中标记为 `Skip`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md new file mode 100644 index 00000000000..83d36169627 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md @@ -0,0 +1,9 @@ +--- +title: å¸è½½ Rancher CIS Benchmark +--- + +1. 在**集群**仪表æ¿ä¸­ï¼Œå•å‡»å·¦ä¾§å¯¼èˆªçš„ **Apps > Installed Apps**。 +1. å‰å¾€ `cis-operator-system` 命å空间，并选中 `rancher-cis-benchmark-crd` å’Œ `rancher-cis-benchmark` æ—边的框。 +1. å•å‡»**删除**并确认**删除**。 + +**结果**：已å¸è½½ `rancher-cis-benchmark` 应用。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md new file mode 100644 index 00000000000..adfcce53799 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md @@ -0,0 +1,12 @@ +--- +title: 查看报告 +--- + +è¦æŸ¥çœ‹ç”Ÿæˆçš„ CIS 扫æ报告： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è¿è¡Œ CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 点击 **CIS Benchmark > 扫æ**。 +1. **扫æ**页é¢å°†æ˜¾ç¤ºç”Ÿæˆçš„报告。è¦æŸ¥çœ‹è¯¦ç»†æŠ¥å‘Šï¼Œè¯·è½¬åˆ°æ‰«æ报告并å•å‡»æŠ¥å‘Šå称。 + +ä½ å¯ä»¥ä»Žæ‰«æ列表或扫æ详情页é¢ä¸‹è½½æŠ¥å‘Šã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md new file mode 100644 index 00000000000..b10f0b61f1b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md @@ -0,0 +1,260 @@ +--- +title: 7 层 NGINX è´Ÿè½½å‡è¡¡å™¨ä¸Šçš„ TLS 终止（Docker 安装） +--- + +如果你的开å‘或测试环境è¦æ±‚在负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ TLS/SSL，而ä¸æ˜¯åœ¨ Rancher Server 上，请部署 Rancher 并é…置负载å‡è¡¡å™¨ã€‚ + +如果è¦åœ¨åŸºç¡€è®¾æ–½ä¸­å¯¹ TLS 集中进行终止，请使用 7 层负载å‡è¡¡å™¨ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚ + +本文中的安装步骤将引导你使用å•ä¸ªå®¹å™¨éƒ¨ç½² Rancher，并æä¾› 7 层 NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例é…置。 + +## æ“作系统，Docker，硬件和网络è¦æ±‚ + +请确ä¿ä½ çš„节点满足常规的[安装è¦æ±‚](../../pages-for-subheaders/installation-requirements.md)。 + +## å®‰è£…æ¦‚è¦ + + +## 1. é…ç½® Linux 主机 + +æ ¹æ®æˆ‘们的[è¦æ±‚](../../pages-for-subheaders/installation-requirements.md)é…置一个 Linux 主机æ¥å¯åŠ¨ Rancher Server。 + +## 2. 选择一个 SSL 选项并安装 Rancher + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +:::note 你是å¦éœ€è¦ï¼š + +- 完æˆç¦»çº¿å®‰è£…。 +- 记录所有 Rancher API 的事务。 + +继续之å‰ï¼Œè¯·å‚è§[高级选项](#高级选项)。 + +::: + +选择以下的选项之一： + +
+ 选项 A：使用你自己的è¯ä¹¦ - 自签å + +如果è¦ä½¿ç”¨è‡ªç­¾åè¯ä¹¦æ¥åŠ å¯†é€šä¿¡ï¼Œä½ å¿…须在负载å‡è¡¡å™¨ï¼ˆåŽç»­æ­¥éª¤ï¼‰å’Œ Rancher 容器上安装è¯ä¹¦ã€‚è¿è¡Œ Docker 命令部署 Rancher，将 Docker 指å‘ä½ çš„è¯ä¹¦ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +创建自签åè¯ä¹¦ã€‚ + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 + +::: + +**使用自签åè¯ä¹¦å®‰è£… Rancher**： + +1. 在è¿è¡Œ Docker 命令部署 Rancher 时，将 Docker 指å‘ä½ çš„ CA è¯ä¹¦æ–‡ä»¶ã€‚ + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/your_certificate_directory/cacerts.pem:/etc/rancher/ssl/cacerts.pem \ + rancher/rancher:latest + ``` + +
+
+ 选项 B：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +如果你的集群é¢å‘公众，则最好使用由公认 CA 签署的è¯ä¹¦ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 + +::: + +**使用授信 CA ç­¾å‘çš„è¯ä¹¦å®‰è£… Rancher**： + +如果你使用授信 CA ç­¾å‘çš„è¯ä¹¦ï¼Œä½ æ— éœ€åœ¨ Rancher 容器中安装è¯ä¹¦ã€‚但是，请确ä¿ä¸è¦ç”Ÿæˆå’Œå­˜å‚¨é»˜è®¤çš„ CA è¯ä¹¦ï¼ˆä½ å¯ä»¥é€šè¿‡å°† `--no-cacerts` å‚数传递给容器æ¥å®žçŽ°ï¼‰ã€‚ + +1. 输入以下命令： + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + rancher/rancher:latest --no-cacerts + ``` + +
+ +## 3. é…置负载å‡è¡¡å™¨ + +在 Rancher 容器å‰ä½¿ç”¨è´Ÿè½½å‡è¡¡å™¨æ—¶ï¼Œå®¹å™¨æ— éœ€ä»Žç«¯å£ 80 æˆ–ç«¯å£ 443 é‡å®šå‘端å£é€šä¿¡ã€‚ä½ å¯ä»¥é€šè¿‡ä¼ é€’ `X-Forwarded-Proto: https` 标头ç¦ç”¨æ­¤é‡å®šå‘。 + +è´Ÿè½½å‡è¡¡å™¨æˆ–代ç†å¿…须支æŒä»¥ä¸‹å†…容： + +- **WebSocket** 连接 +- **SPDY** / **HTTP/2** åè®® +- 传递/设置以下标头： + + | 标头 | 值 | æè¿° | + |--------|-------|-------------| + | `Host` | 用于访问 Rancher 的主机å。 | 识别客户端所请求的æœåŠ¡å™¨ã€‚ | + | `X-Forwarded-Proto` | `https` | 识别客户端连接负载å‡è¡¡å™¨æˆ–代ç†æ—¶æ‰€ç”¨çš„å议。

**注æ„**：如果此标头存在，`rancher/rancher` ä¸ä¼šå°† HTTP é‡å®šå‘到 HTTPS。 | + | `X-Forwarded-Port` | 用于访问 Rancher 的端å£ã€‚ | 识别客户端连接到负载å‡è¡¡å™¨æˆ–代ç†æ—¶æ‰€ç”¨çš„端å£ã€‚ | + | `X-Forwarded-For` | 客户端 IP åœ°å€ | 识别客户端的原始 IP 地å€ã€‚ | +### 示例 NGINX é…ç½® + +æ­¤ NGINX é…置已在 NGINX 1.14 上进行了测试。 + +:::note + +æ­¤ NGINX é…ç½®åªæ˜¯ä¸€ä¸ªç¤ºä¾‹ï¼Œå¯èƒ½ä¸é€‚åˆä½ çš„环境。如需查阅完整文档，请å‚è§ [NGINX è´Ÿè½½å‡è¡¡ - HTTP è´Ÿè½½å‡è¡¡](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/)。 + +::: + +- å°† `rancher-server` 替æ¢ä¸ºè¿è¡Œ Rancher 容器的节点的 IP 或主机å。 +- 将两处的 `FQDN` å‡æ›¿æ¢ä¸º Rancher çš„ DNS å称。 +- 把 `/certs/fullchain.pem` å’Œ `/certs/privkey.pem` 分别替æ¢ä¸ºæœåŠ¡å™¨è¯ä¹¦å’ŒæœåŠ¡å™¨è¯ä¹¦å¯†é’¥çš„ä½ç½®ã€‚ + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server rancher-server:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # 此项å…许执行的 shell 窗å£ä¿æŒå¼€å¯ï¼Œæœ€é•¿å¯è¾¾15分钟。ä¸ä½¿ç”¨æ­¤å‚æ•°çš„è¯ï¼Œé»˜è®¤1分钟åŽè‡ªåŠ¨å…³é—­ã€‚ + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` + +
+ +## åŽç»­æ“作 + +- **推è**：检查å•èŠ‚点[备份](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md)å’Œ[æ¢å¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md)。你å¯èƒ½æš‚时没有需è¦å¤‡ä»½çš„æ•°æ®ï¼Œä½†æ˜¯æˆ‘们建议你在常规使用 Rancher åŽåˆ›å»ºå¤‡ä»½ã€‚ +- 创建 Kubernetes 集群：[é…ç½® Kubernetes 集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)。 + +
+ +## 常è§é—®é¢˜å’Œæ•…障排除 + +如果你需è¦å¯¹è¯ä¹¦è¿›è¡Œæ•…障排除，请å‚è§[此章节](../../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +## 高级选项 + +### API 审计 + +如果你需è¦è®°å½•æ‰€æœ‰ Rancher API 事务，请将以下标志添加到安装命令中，从而å¯ç”¨ [API 审计](enable-api-audit-log.md)功能。 + + -e AUDIT_LEVEL=1 \ + -e AUDIT_LOG_PATH=/var/log/auditlog/rancher-api-audit.log \ + -e AUDIT_LOG_MAXAGE=20 \ + -e AUDIT_LOG_MAXBACKUP=20 \ + -e AUDIT_LOG_MAXSIZE=100 \ + +### 离线环境 + +如果你访问此页é¢æ˜¯ä¸ºäº†å®Œæˆ[离线安装](../../pages-for-subheaders/air-gapped-helm-cli-install.md)，则在è¿è¡Œå®‰è£…命令时，先将你的ç§æœ‰é•œåƒä»“库 URL 附加到 Server 标志中。也就是说，在 `rancher/rancher:latest` å‰é¢æ·»åŠ  `` å’Œç§æœ‰é•œåƒä»“库 URL。 + +**示例**： + + /rancher/rancher:latest + +### æŒä¹…åŒ–æ•°æ® + +Rancher 使用 etcd 作为数æ®å­˜å‚¨ã€‚如果 Rancher 是使用 Docker 安装的，Rancher ä¼šä½¿ç”¨åµŒå…¥å¼ etcd。æŒä¹…化数æ®ä½äºŽå®¹å™¨çš„ `/var/lib/rancher` 路径中。 + +ä½ å¯ä»¥å°†ä¸»æœºå·æŒ‚载到该ä½ç½®ï¼Œæ¥å°†æ•°æ®ä¿ç•™åœ¨è¿è¡Œå®ƒçš„主机上： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +æ­¤æ“ä½œéœ€è¦ [privileged 访问](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)。 + +这个 7 层 NGINX é…置已ç»åœ¨ NGINX 1.13（Mainline）和 1.14（Stable）版本上进行了测试。 + +:::note + +æ­¤ NGINX é…ç½®åªæ˜¯ä¸€ä¸ªç¤ºä¾‹ï¼Œå¯èƒ½ä¸é€‚åˆä½ çš„环境。如果需è¦æŸ¥é˜…完整文档，请å‚è§ [NGINX è´Ÿè½½å‡è¡¡ - TCP å’Œ UDP è´Ÿè½½å‡è¡¡å™¨](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/)。 + +::: + +``` +upstream rancher { + server rancher-server:80; +} + +map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; +} + +server { + listen 443 ssl http2; + server_name rancher.yourdomain.com; + ssl_certificate /etc/your_certificate_directory/fullchain.pem; + ssl_certificate_key /etc/your_certificate_directory/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # 此项å…许执行的 shell 窗å£ä¿æŒå¼€å¯ï¼Œæœ€é•¿å¯è¾¾15分钟。ä¸ä½¿ç”¨æ­¤å‚æ•°çš„è¯ï¼Œé»˜è®¤1分钟åŽè‡ªåŠ¨å…³é—­ã€‚ + proxy_read_timeout 900s; + proxy_buffering off; + } +} + +server { + listen 80; + server_name rancher.yourdomain.com; + return 301 https://$server_name$request_uri; +} +``` + +
+ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log.md new file mode 100644 index 00000000000..94974cb52f7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log.md @@ -0,0 +1,558 @@ +--- +title: å¯ç”¨ API 审计日志以记录系统事件 +--- + +ä½ å¯ä»¥å¯ç”¨ API 审计日志æ¥è®°å½•å„个用户å‘起的系统事件的顺åºã€‚通过查看日志，你å¯ä»¥äº†è§£å‘生了什么事件ã€äº‹ä»¶å‘生的时间，事件å‘起人，以åŠäº‹ä»¶å½±å“的集群。å¯ç”¨æ­¤åŠŸèƒ½åŽï¼Œæ‰€æœ‰ Rancher API 的请求和å“应都会写入日志中。 + +API 审计å¯ä»¥åœ¨ Rancher 安装或å‡çº§æœŸé—´å¯ç”¨ã€‚ + +## å¯ç”¨ API 审计日志 + +ä½ å¯ä»¥å°†çŽ¯å¢ƒå˜é‡ä¼ é€’ç»™ Rancher Server 容器，从而å¯ç”¨å’Œé…置审计日志。请å‚è§ä»¥ä¸‹æ–‡æ¡£ï¼Œåœ¨å®‰è£…æ—¶å¯ç”¨è¯¥åŠŸèƒ½ï¼š + +- [Docker 安装](../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志) + +- [Kubernetes 安装](../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-审计日志) + +## API 审计日志选项 + +以下å‚数定义了审计日志的记录规则，其中包括应该记录什么内容以åŠåŒ…括什么数æ®ï¼š + +| å‚æ•° | æè¿° | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AUDIT_LEVEL` | `0` - ç¦ç”¨å®¡è®¡æ—¥å¿—（默认）
`1` - 日志事件元数æ®
`2` - 日志事件元数æ®å’Œè¯·æ±‚体
`3` - 日志事件元数æ®ï¼Œè¯·æ±‚体和å“应体。请求/å“应对的æ¯ä¸ªæ—¥å¿—事务都使用åŒä¸€ä¸ªçš„ `auditID`。
如需了解æ¯ä¸ªè®¾ç½®è®°å½•çš„日志内容，请å‚è§[审计日志级别](#审核日志级别)。 | +| `AUDIT_LOG_PATH` | Rancher Server API 的日志路径。默认路径：`/var/log/auditlog/rancher-api-audit.log`。你å¯ä»¥å°†æ—¥å¿—目录挂载到主机。

示例：`AUDIT_LOG_PATH=/my/custom/path/`
| +| `AUDIT_LOG_MAXAGE` | 旧审计日志文件å¯ä¿ç•™çš„最大天数。默认为 10 天。 | +| `AUDIT_LOG_MAXBACKUP` | ä¿ç•™çš„审计日志最大文件个数。默认值为 10。 | +| `AUDIT_LOG_MAXSIZE` | 在审计日志文件被轮æ¢å‰çš„最大容é‡ï¼Œå•ä½æ˜¯ MB。默认大å°ä¸º 100MB。 | + +
+ +### 审核日志级别 + +下表介ç»äº†æ¯ä¸ª [`AUDIT_LEVEL`](#audit-level) 记录的 API 事务： + +| `AUDIT_LEVEL` 设置 | è¯·æ±‚å…ƒæ•°æ® | 请求体 | å“åº”å…ƒæ•°æ® | å“应体 | +| --------------------- | ---------------- | ------------ | ----------------- | ------------- | +| `0` | | | | | +| `1` | ✓ | | | | +| `2` | ✓ | ✓ | | | +| `3` | ✓ | ✓ | ✓ | ✓ | + +## 查看 API 审计日志 + +### Docker 安装 + +与主机系统共享 `AUDIT_LOG_PATH` 目录（默认目录：`/var/log/auditlog`）。日志å¯ä»¥é€šè¿‡æ ‡å‡† CLI 工具进行解æžï¼Œä¹Ÿå¯ä»¥è½¬å‘到 Fluentdã€Filebeatã€Logstash 等日志收集工具。 + +### Kubernetes 安装 + +使用 Helm Chart 安装 Rancher æ—¶å¯åŠ¨ API 审计日志，会在 Rancher Pod 中创建一个 `rancher-audit-log` Sidecar 容器。该容器会将日志å‘é€åˆ°æ ‡å‡†è¾“出 (stdout)。你å¯ä»¥åƒæŸ¥çœ‹å…¶ä»–容器的日志一样查看 API 审计日志。 + +`rancher-audit-log` 容器ä½äºŽ `cattle-system` 命å空间中的 `rancher` Pod 中。 + +#### CLI + +```bash +kubectl -n cattle-system logs -f rancher-84d886bdbb-s4s69 rancher-audit-log +``` + +#### å‘é€å®¡è®¡æ—¥å¿— + +ä½ å¯ä»¥ä¸ºé›†ç¾¤å¯ç”¨ Rancher 的内置日志收集和传é€åŠŸèƒ½ï¼Œå°†å®¡è®¡æ—¥å¿—和其他æœåŠ¡æ—¥å¿—å‘é€åˆ°æ”¯æŒçš„ endpoint。详情请å‚è§ [Rancher 工具 - Logging](../../pages-for-subheaders/logging.md)。 + +## 审计日志示例 + +å¯ç”¨å®¡è®¡æ—¥å¿—åŽï¼ŒRancher 会以 JSON æ ¼å¼è®°å½•æ¯ä¸ª API 的请求和å“应。下文的代ç ç¤ºä¾‹å±•ç¤ºäº†å¦‚何查看 API 事务。 + +### 元数æ®æ—¥å¿—级别 + +如果你将 `AUDIT_LEVEL` 设置为 `1`，Rancher åªä¼šè®°å½•æ¯ä¸ª API 请求的元数æ®æ ‡å¤´ï¼Œè€Œä¸ä¼šè®°å½•è¯·æ±‚体。标头记录了 API 事务的基本信æ¯ï¼ŒåŒ…括 IDã€å‘起人ã€å‘起时间等。代ç ç¤ºä¾‹å¦‚下： + +```json +{ + "auditID": "30022177-9e2e-43d1-b0d0-06ef9d3db183", + "requestURI": "/v3/schemas", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "GET", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:22:43 +0800" +} +``` + +### 元数æ®å’Œè¯·æ±‚体日志级别 + +如果你将 `AUDIT_LEVEL` 设置为 `2`，Rancher 会记录æ¯ä¸ª API 请求的元数æ®æ ‡å¤´å’Œè¯·æ±‚体。 + +下é¢çš„代ç ç¤ºä¾‹æ述了一个 API 请求，包括它的元数æ®æ ‡å¤´å’Œæ­£æ–‡ï¼š + +```json +{ + "auditID": "ef1d249e-bfac-4fd0-a61f-cbdcad53b9bb", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:28:08 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my description", + "volumes": [] + } +} +``` + +### 元数æ®ã€è¯·æ±‚体和å“应体日志级别 + +如果你将 `AUDIT_LEVEL` 设置为 `3`，Rancher 会记录： + +- æ¯ä¸ª API 请求的元数æ®æ ‡å¤´å’Œè¯·æ±‚体。 +- æ¯ä¸ª API å“应的元数æ®æ ‡å¤´å’Œå“应体。 + +#### 请求 + +下é¢çš„代ç ç¤ºä¾‹æ述了一个 API 请求，包括它的元数æ®æ ‡å¤´å’Œæ­£æ–‡ï¼š + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my decript", + "volumes": [] + } +} +``` + +#### å“应 + +下é¢çš„代ç ç¤ºä¾‹æ述了一个 API å“应，包括它的元数æ®æ ‡å¤´å’Œæ­£æ–‡ï¼š + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "responseStatus": "200", + "stage": "ResponseComplete", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "responseBody": { + "actionLinks": { + "pause": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=pause", + "resume": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=resume", + "rollback": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=rollback" + }, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements" + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container" + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "links": { + "remove": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "revisions": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/revisions", + "self": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "update": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "yaml": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/yaml" + }, + "name": "nginx", + "namespaceId": "default", + "paused": false, + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + } + } +} +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md new file mode 100644 index 00000000000..c1f42f70a4b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md @@ -0,0 +1,13 @@ +--- +title: æŒç»­äº¤ä»˜ +--- + +Rancher 中预装的 [Fleet](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) 无法完全ç¦ç”¨ã€‚但是，你å¯ä»¥ä½¿ç”¨ `continuous-delivery` 功能开关æ¥ç¦ç”¨ GitOps æŒç»­äº¤ä»˜çš„ Fleet 功能。 + +如需å¯ç”¨æˆ–ç¦ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·å‚è§[å¯ç”¨å®žéªŒåŠŸèƒ½ä¸»é¡µ](../../../pages-for-subheaders/enable-experimental-features.md)中的说明。 + +| 环境å˜é‡é”® | 默认值 | æè¿° | +---|---|--- +| `continuous-delivery` | `true` | 此开关ç¦ç”¨ Fleet çš„ GitOps æŒç»­äº¤ä»˜åŠŸèƒ½ã€‚ | + +如果你在 Rancher 2.5.x 中ç¦ç”¨äº† Fleet，然åŽå°† Rancher å‡çº§åˆ° v2.6.x，Fleet å°†å¯ç”¨ã€‚åªæœ‰ Fleet çš„æŒç»­äº¤ä»˜åŠŸèƒ½å¯ä»¥è¢«ç¦ç”¨ã€‚当 `continuous-delivery` 被ç¦ç”¨æ—¶ï¼Œ`gitjob` deployment ä¸å†éƒ¨ç½²åˆ° Rancher Server 的本地集群中，且 `continuous-delivery` ä¸ä¼šåœ¨ Rancher UI 中显示。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md new file mode 100644 index 00000000000..35801de23b4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md @@ -0,0 +1,32 @@ +--- +title: UI ç®¡ç† Istio 虚拟æœåŠ¡å’Œç›®æ ‡è§„则 +--- + +此功能å¯å¯åŠ¨ä¸€ä¸ª UIï¼Œç”¨äºŽç®¡ç† Istio çš„æµé‡ï¼Œå…¶ä¸­åŒ…括创建ã€è¯»å–ã€æ›´æ–°å’Œåˆ é™¤è™šæ‹ŸæœåŠ¡ï¼ˆVirtual Service）和目标规则（Destination Rule）。 + +> **注æ„**：å¯ç”¨æ­¤åŠŸèƒ½å¹¶ä¸ä¼šå¯ç”¨ Istio。集群管ç†å‘˜éœ€è¦[为集群å¯ç”¨ Istio](../../../pages-for-subheaders/istio-setup-guide.md) æ‰èƒ½ä½¿ç”¨è¯¥åŠŸèƒ½ã€‚ + +如需å¯ç”¨æˆ–ç¦ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·å‚è§[å¯ç”¨å®žéªŒåŠŸèƒ½ä¸»é¡µ](../../../pages-for-subheaders/enable-experimental-features.md)中的说明。 + +| 环境å˜é‡é”® | 默认值 | çŠ¶æ€ | å¯ç”¨äºŽ | +---|---|---|--- +| `istio-virtual-service-ui` | `false` | 实验功能 | v2.3.0 | +| `istio-virtual-service-ui` | `true` | GA | v2.3.2 | + +## åŠŸèƒ½ä»‹ç» + +Istio æµé‡ç®¡ç†åŠŸèƒ½çš„主è¦ä¼˜åŠ¿æ—¶å…许动æ€è¯·æ±‚路由，这对于金ä¸é›€å‘布，è“/绿å‘布或 A/B 测试都éžå¸¸æœ‰ç”¨ã€‚ + +å¯ç”¨æ­¤åŠŸèƒ½åŽï¼Œä¸€ä¸ªé¡µé¢ä¼šæ‰“开，让你通过 Rancher UI é…ç½® Istio çš„æŸäº›æµé‡ç®¡ç†åŠŸèƒ½ã€‚如果ä¸ä½¿ç”¨æ­¤åŠŸèƒ½ï¼Œä½ å¯ä»¥é€šè¿‡ `kubectl` æ¥ä½¿ç”¨ Istio 管ç†æµé‡ã€‚ + +此功能会å¯ç”¨ä¸¤ä¸ªé€‰é¡¹å¡ï¼Œä¸€ä¸ªç”¨äºŽ**虚拟æœåŠ¡**，å¦ä¸€ä¸ªç”¨äºŽ**目标规则**。 + +- **虚拟æœåŠ¡**：拦截并将æµé‡é‡å®šå‘到你的 Kubernetes Service 上。这样，你å¯ä»¥å°†éƒ¨åˆ†è¯·æ±‚æµé‡å®šå‘到ä¸åŒçš„æœåŠ¡ä¸Šã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›æœåŠ¡æ¥å®šä¹‰ä¸€ç»„路由规则，用于主机寻å€ã€‚详情请å‚è§ [Istio 官方文档](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/)。 +- **目标规则**：作为唯一å¯ä¿¡æ¥æºï¼Œè¡¨æ˜Žå“ªäº›æœåŠ¡ç‰ˆæœ¬å¯ç”¨äºŽæŽ¥æ”¶è™šæ‹ŸæœåŠ¡çš„æµé‡ã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºæ¥å®šä¹‰ç­–略，这些策略适用于路由å‘生åŽç”¨äºŽæœåŠ¡çš„æµé‡ã€‚详情请å‚è§ [Istio 官方文档](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule)。 + +如需查看选项å¡ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 转到安装了 Istio 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio**。 +1. 你将看到 **Kiali** å’Œ **Jaeger** 的选项å¡ã€‚在左侧导航æ ä¸­ï¼Œä½ å¯æŸ¥çœ‹å’Œé…ç½®**虚拟æœåŠ¡**å’Œ**目标规则**。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md new file mode 100644 index 00000000000..16ebb15e25a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md @@ -0,0 +1,44 @@ +--- +title: "在 ARM64 上è¿è¡Œ Rancher（实验性）" +--- + +:::caution + +在使用 ARM64 架构的节点上è¿è¡Œ Rancher ç›®å‰è¿˜å¤„在实验阶段，Rancher 尚未正å¼æ”¯æŒè¯¥åŠŸèƒ½ã€‚因此，我们ä¸å»ºè®®ä½ åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­ä½¿ç”¨ ARM64 架构的节点。 + +::: + +如果你的节点使用 ARM64 架构，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹é€‰é¡¹ï¼š + +- 在 ARM64 架构的节点上è¿è¡Œ Rancher + - 此选项仅适用于 Docker 安装。请知悉，以下安装命令å–代了 [Docker 安装链接](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md)中的示例： + + ``` + # 在最åŽä¸€è¡Œ `rancher/rancher:vX.Y.Z` 中，请务必将 "X.Y.Z" 替æ¢ä¸ºåŒ…å« ARM64 版本的å‘布版本。例如，如果你的匹é…版本是 v2.5.8，请在此行填写 `rancher/rancher:v2.5.8`。 + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:vX.Y.Z + ``` + +:::note + +è¦æ£€æŸ¥ä½ çš„å‘行版本是å¦ä¸Ž ARM64 架构兼容，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹ä¸¤ç§æ–¹å¼æ‰¾åˆ°å¯¹åº”版本的å‘行说明： + +- 访问 [Rancher å‘行版本](https://github.com/rancher/rancher/releases)自行查询。 +- æ ¹æ®æ ‡ç­¾å’Œç‰ˆæœ¬å·ç›´æŽ¥æ‰¾åˆ°ä½ çš„版本。例如，你使用的版本为 2.5.8，你å¯ä»¥è®¿é—® [Rancher å‘行版本 - 2.5.8](https://github.com/rancher/rancher/releases/tag/v2.5.8)。 + +::: + +- 创建自定义集群并添加使用 ARM64 架构的节点 + - Kubernetes 集群必须为 1.12 或更高版本 + - CNI 网络æ’件必须是 [Flannel](../../../faq/container-network-interface-providers.md#flannel) +- 导入包å«ä½¿ç”¨ ARM64 架构的节点的集群 + - Kubernetes 集群必须为 1.12 或更高版本 + +如需了解如何é…置集群选项，请å‚è§[集群选项](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + +以下是未ç»æµ‹è¯•çš„功能： + +- Monitoringã€å‘Šè­¦ã€Notifiersã€æµæ°´çº¿å’Œ Logging +- 通过应用商店å‘布应用 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md new file mode 100644 index 00000000000..19b9b27b2e0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md @@ -0,0 +1,39 @@ +--- +title: 使用éžé»˜è®¤æ”¯æŒçš„存储驱动 +--- + +此功能å…许你使用ä¸æ˜¯é»˜è®¤å¯ç”¨çš„存储æ供商和å·æ’件。 + +如需å¯ç”¨æˆ–ç¦ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·å‚è§[å¯ç”¨å®žéªŒåŠŸèƒ½ä¸»é¡µ](../../../pages-for-subheaders/enable-experimental-features.md)中的说明。 + +| 环境å˜é‡é”® | 默认值 | æè¿° | +---|---|--- +| `unsupported-storage-drivers` | `false` | å¯ç”¨éžé»˜è®¤å¯ç”¨çš„存储æ供商和å·æ’件。 | + +### 默认å¯ç”¨çš„æŒä¹…å·æ’件 +下表æ述了默认å¯ç”¨çš„存储类型对应的æŒä¹…å·æ’件。å¯ç”¨æ­¤åŠŸèƒ½å¼€å…³æ—¶ï¼Œä¸åœ¨æ­¤åˆ—表中的任何æŒä¹…å·æ’件å‡è¢«è§†ä¸ºå®žéªŒåŠŸèƒ½ï¼Œä¸”ä¸å—支æŒï¼š + +| å称 | æ’件 | +--------|---------- +| Amazon EBS Disk | `aws-ebs` | +| AzureFile | `azure-file` | +| AzureDisk | `azure-disk` | +| Google Persistent Disk | `gce-pd` | +| Longhorn | `flex-volume-longhorn` | +| VMware vSphere Volume | `vsphere-volume` | +| 本地 | `local` | +| 网络文件系统 | `nfs` | +| hostPath | `host-path` | + +### 默认å¯ç”¨çš„ StorageClass +下表æ述了默认å¯ç”¨çš„ StorageClass 对应的æŒä¹…å·æ’件。å¯ç”¨æ­¤åŠŸèƒ½å¼€å…³æ—¶ï¼Œä¸åœ¨æ­¤åˆ—表中的任何æŒä¹…å·æ’件å‡è¢«è§†ä¸ºå®žéªŒåŠŸèƒ½ï¼Œä¸”ä¸å—支æŒï¼š + +| å称 | æ’件 | +--------|-------- +| Amazon EBS Disk | `aws-ebs` | +| AzureFile | `azure-file` | +| AzureDisk | `azure-disk` | +| Google Persistent Disk | `gce-pd` | +| Longhorn | `flex-volume-longhorn` | +| VMware vSphere Volume | `vsphere-volume` | +| 本地 | `local` | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md new file mode 100644 index 00000000000..39c9bdee036 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md @@ -0,0 +1,29 @@ +--- +title: 1. 在集群中å¯ç”¨ Istio +--- + +:::note 先决æ¡ä»¶ï¼š + +- åªæœ‰åˆ†é…了 `cluster-admin` [Kubernetes 默认角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)的用户å¯ä»¥åœ¨ Kubernetes 集群中é…置和安装 Istio。 +- 如果你有 pod 安全策略，则需è¦å®‰è£…å¯ç”¨äº† CNI çš„ Istio。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../integrations-in-rancher/istio/configuration-options/pod-security-policies.md)。 +- è¦åœ¨ RKE2 集群上安装 Istio，则需è¦æ‰§è¡Œé¢å¤–的步骤。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md)。 +- è¦åœ¨å¯ç”¨äº†é¡¹ç›®ç½‘络隔离的集群中安装 Istio，则需è¦æ‰§è¡Œé¢å¤–的步骤。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../integrations-in-rancher/istio/configuration-options/project-network-isolation.md)。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å¯ç”¨ Istio çš„ä½ç½®ï¼Œç„¶åŽå•å‡» **Explore**。 +1. å•å‡» **Apps**。 +1. å•å‡» **Chart**。 +1. å•å‡» **Istio**。 +1. 如果你还没有安装 Monitoring 应用，系统会æ示你安装 rancher-monitoring。你也å¯ä»¥é€‰æ‹©åœ¨ Rancher-monitoring 安装上设置选择器或抓å–é…置选项。 +1. å¯é€‰ï¼šä¸º Istio 组件é…ç½®æˆå‘˜è®¿é—®å’Œ[资æºé™åˆ¶](../../../integrations-in-rancher/istio/cpu-and-memory-allocations.md)。确ä¿ä½ çš„ Worker 节点上有足够的资æºæ¥å¯ç”¨ Istio。 +1. å¯é€‰ï¼šå¦‚果需è¦ï¼Œå¯¹ values.yaml 进行é¢å¤–çš„é…置更改。 +1. å¯é€‰ï¼šé€šè¿‡[覆盖文件](../../../pages-for-subheaders/configuration-options.md#覆盖文件)æ¥æ·»åŠ å…¶ä»–资æºæˆ–é…置。 +1. å•å‡»**安装**。 + +**结果**：已在集群级别安装 Istio。 + +## 其他é…置选项 + +有关é…ç½® Istio 的更多信æ¯ï¼Œè¯·å‚阅[é…ç½®å‚考](../../../pages-for-subheaders/configuration-options.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md new file mode 100644 index 00000000000..3651b9d77c6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md @@ -0,0 +1,53 @@ +--- +title: 2. 在命å空间中å¯ç”¨ Istio +--- + +你需è¦åœ¨éœ€è¦ç”± Istio 跟踪或控制的æ¯ä¸ªå‘½å空间中手动å¯ç”¨ Istio。在命å空间中å¯ç”¨ Istio 时，Envoy sidecar 代ç†å°†è‡ªåŠ¨æ³¨å…¥åˆ°éƒ¨ç½²åœ¨å‘½å空间中的所有新工作负载中。 + +此命å空间设置åªä¼šå½±å“命å空间中的新工作负载。之å‰çš„工作负载需è¦é‡æ–°éƒ¨ç½²æ‰èƒ½ä½¿ç”¨ sidecar 自动注入。 + +:::note 先决æ¡ä»¶ï¼š + +è¦åœ¨å‘½å空间中å¯ç”¨ Istio，集群必须安装 Istio。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦å¯ç”¨ Istio 的命å空间，然åŽå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。或者，你也å¯ä»¥å•å‡»å‘½å空间，然åŽåœ¨å‘½å空间详情页é¢ä¸Šï¼Œå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。 + +**结果**：命å空间带有了 `istio-injection=enabled` 标签。默认情况下，部署在此命å空间中的所有新工作负载都将注入 Istio sidecar。 + +### 验è¯æ˜¯å¦å¯ç”¨äº†è‡ªåŠ¨ Istio Sidecar 注入 + +è¦éªŒè¯ Istio 是å¦å·²å¯ç”¨ï¼Œè¯·åœ¨å‘½å空间中部署一个 hello-world 工作负载。转到工作负载并å•å‡» pod å称。在**容器**中，你应该能看到 `istio-proxy` 容器。 + +### 排除工作负载的 Istio Sidecar 注入 + +è¦æŽ’除 Istio sidecar 被注入æŸå·¥ä½œè´Ÿè½½ï¼Œè¯·åœ¨å·¥ä½œè´Ÿè½½ä¸Šä½¿ç”¨ä»¥ä¸‹æ³¨é‡Šï¼š + +``` +sidecar.istio.io/inject: “false†+``` + +è¦å°†æ³¨é‡Šæ·»åŠ åˆ°å·¥ä½œè´Ÿè½½ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**工作负载**。 +1. 转到ä¸éœ€è¦ sidecar 的工作负载并以 yaml 编辑。 +1. 将键值 `sidecar.istio.io/inject: false` 添加为工作负载的注释。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：Istio sidecar ä¸ä¼šè¢«æ³¨å…¥åˆ°å·¥ä½œè´Ÿè½½ä¸­ã€‚ + +:::note + +如果你é‡åˆ°éƒ¨ç½²çš„ job 未完æˆçš„问题，则需è¦ä½¿ç”¨æ供的步骤将此注释添加到 pod 中。由于 Istio Sidecars 会一直è¿è¡Œï¼Œå› æ­¤å³ä½¿ä»»åŠ¡å®Œæˆäº†ï¼Œä¹Ÿä¸èƒ½è®¤ä¸º Job 已完æˆã€‚ + +::: + + +### åŽç»­æ­¥éª¤ +[使用 Istio Sidecar 添加部署](use-istio-sidecar.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md new file mode 100644 index 00000000000..a6d77763919 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md @@ -0,0 +1,26 @@ +--- +title: 6. 生æˆå’ŒæŸ¥çœ‹æµé‡ +--- + +本文介ç»å¦‚何查看 Istio 管ç†çš„æµé‡ã€‚ + +## Kiali æµé‡å›¾ + +Istio 概览页é¢æ供了 Kiali 仪表æ¿çš„链接。在 Kiali 仪表æ¿ä¸­ï¼Œä½ å¯ä»¥æŸ¥çœ‹æ¯ä¸ªå‘½å空间的图。Kiali 图æ供了一ç§å¼ºå¤§çš„æ–¹å¼æ¥å¯è§†åŒ– Istio æœåŠ¡ç½‘格的拓扑。它显示了æœåŠ¡ä¹‹é—´ç›¸äº’通信的情况。 + +:::note 先决æ¡ä»¶ï¼š + +è¦æ˜¾ç¤ºæµé‡å›¾ï¼Œè¯·ç¡®ä¿ä½ åœ¨é›†ç¾¤ä¸­å®‰è£…了 Prometheus。Rancher-istio 安装了默认é…置的 Kiali æ¥ä¸Ž rancher-monitoring Chart 一起工作。你å¯ä»¥ä½¿ç”¨ rancher-monitoring 或安装自己的监控解决方案。你也å¯ä»¥é€šè¿‡è®¾ç½®[选择器 & 抓å–é…ç½®](../../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)选项æ¥æ›´æ”¹æ•°æ®æŠ“å–çš„é…置（å¯é€‰ï¼‰ã€‚ + +::: + +è¦æŸ¥çœ‹æµé‡å›¾ï¼š + +1. 在安装了 Istio 的集群中，点击左侧导航æ ä¸­çš„ **Istio**。 +1. å•å‡» **Kiali** 链接。 +1. å•å‡»ä¾§å¯¼èˆªä¸­çš„**图**。 +1. 在**命å空间**下拉列表中，更改命å空间以查看æ¯ä¸ªå‘½å空间的æµé‡ã€‚ + +如果你多次刷新 BookInfo 应用的 URL，你将能够在 Kiali 图上看到绿色箭头，显示 `reviews` æœåŠ¡ `v1` å’Œ `v3` çš„æµé‡ã€‚图å³ä¾§çš„控制é¢æ¿å¯ç”¨äºŽé…置详细信æ¯ï¼ŒåŒ…括应在图上显示多少分钟的最新æµé‡ã€‚ + +对于其他工具和å¯è§†åŒ–，你å¯ä»¥ä»Ž**监控** > **概览**页é¢è½¬åˆ° Grafana å’Œ Prometheus 仪表æ¿ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md new file mode 100644 index 00000000000..42f89baa8ef --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md @@ -0,0 +1,147 @@ +--- +title: 4. 设置 Istio Gateway +--- + +æ¯ä¸ªé›†ç¾¤çš„网关å¯ä»¥æœ‰è‡ªå·±çš„端å£æˆ–è´Ÿè½½å‡è¡¡å™¨ï¼Œè¿™ä¸ŽæœåŠ¡ç½‘格无关。默认情况下，æ¯ä¸ª Rancher é…置的集群都有一个 NGINX Ingress Controller æ¥å…许æµé‡è¿›å…¥é›†ç¾¤ã€‚ + +无论是å¦å®‰è£…了 Istio，你都å¯ä»¥ä½¿ç”¨ NGINX Ingress Controller。如果这是你集群的唯一网关，Istio 将能够将æµé‡ä»Žé›†ç¾¤å†…部的æœåŠ¡è·¯ç”±åˆ°é›†ç¾¤å†…部的å¦ä¸€ä¸ªæœåŠ¡ï¼Œä½† Istio 将无法接收æ¥è‡ªé›†ç¾¤å¤–部的æµé‡ã€‚ + +è¦è®© Istio 接收外部æµé‡ï¼Œä½ éœ€è¦å¯ç”¨ Istio 的网关，作为外部æµé‡çš„å—北代ç†ã€‚å¯ç”¨ Istio Gateway åŽï¼Œä½ çš„集群将有两个 Ingress。 + +你还需è¦ä¸ºä½ çš„æœåŠ¡è®¾ç½® Kubernetes 网关。此 Kubernetes 资æºæŒ‡å‘ Istio 对集群 Ingress Gateway 的实现。 + +ä½ å¯ä»¥ä½¿ç”¨è´Ÿè½½å‡è¡¡å™¨å°†æµé‡è·¯ç”±åˆ°æœåŠ¡ç½‘格中，或使用 Istio çš„ NodePort 网关。本文介ç»å¦‚何设置 NodePort 网关。 + +有关 Istio Gateway 的更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](https://istio.io/docs/reference/config/networking/v1alpha3/gateway/)。 + +![å¯ç”¨ Istio 的集群å¯ä»¥æœ‰ä¸¤ä¸ª ingress，分别是默认的 Nginx ingress 和默认的 Istio controller](/img/istio-ingress.svg) + +## å¯ç”¨ Istio Gateway + +Ingress Gateway 是一个 Kubernetes æœåŠ¡ï¼Œå°†éƒ¨ç½²åœ¨ä½ çš„集群中。Istio Gateway 支æŒæ›´å¤šè‡ªå®šä¹‰è®¾ç½®ï¼Œæ›´åŠ çµæ´»ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio > 网关**。 +1. å•å‡»**使用 YAML 文件创建**。 +1. 粘贴你的 Istio Gateway yaml，或选择**从文件读å–**。 +1. å•å‡»**创建**。 + +**结果**：已部署网关，将使用应用的规则æ¥è·¯ç”±æµé‡ã€‚ + +## Istio Gateway 示例 + +在演示工作负载示例时，我们在æœåŠ¡ä¸­æ·»åŠ  BookInfo 应用部署。接下æ¥ï¼Œæˆ‘们添加一个 Istio Gateway，以便从集群外部访问该应用。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio > 网关**。 +1. å•å‡»**使用 YAML 文件创建**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ Gateway YAML。 +1. å•å‡»**创建**。 + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: bookinfo-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +--- +``` + +然åŽï¼Œéƒ¨ç½²ä¸º Gateway æä¾›æµé‡è·¯ç”±çš„ VirtualService： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio > VirtualServices**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ VirtualService YAML。 +1. å•å‡»**创建**。 + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: bookinfo +spec: + hosts: + - "*" + gateways: + - bookinfo-gateway + http: + - match: + - uri: + exact: /productpage + - uri: + prefix: /static + - uri: + exact: /login + - uri: + exact: /logout + - uri: + prefix: /api/v1/products + route: + - destination: + host: productpage + port: + number: 9080 +``` + +**结果**：你已é…置网关资æºï¼ŒIstio 现在å¯ä»¥æŽ¥æ”¶é›†ç¾¤å¤–部的æµé‡ã€‚ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥ç¡®è®¤èµ„æºå­˜åœ¨ï¼š +``` +kubectl get gateway -A +``` + +结果应与以下内容类似： +``` +NAME AGE +bookinfo-gateway 64m +``` + +### 在 Web æµè§ˆå™¨è®¿é—® ProductPage æœåŠ¡ + +è¦æµ‹è¯• BookInfo 应用是å¦å·²æ­£ç¡®éƒ¨ç½²ï¼Œä½ å¯ä»¥ä½¿ç”¨ Istio 控制器 IP 和端å£ä»¥åŠåœ¨ Kubernetes 网关资æºä¸­æŒ‡å®šçš„请求å称，在 Web æµè§ˆå™¨ä¸­æŸ¥çœ‹è¯¥åº”用： + +`http://:/productpage` + +è¦èŽ·å– Ingress Gateway URL 和端å£ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å‘下滚动到 `istio-system` 命å空间。 +1. 在 `istio-system`中，有一个å为 `istio-ingressgateway` 的工作负载。在此工作负载的å称下，你应该会看到如 `80/tcp` 的链接。 +1. å•å‡»å…¶ä¸­ä¸€ä¸ªé“¾æŽ¥ã€‚然åŽï¼Œä½ çš„ Web æµè§ˆå™¨ä¸­ä¼šæ˜¾ç¤º Ingress Gateway çš„ URL。将 `/productpage` 尾附到 URL。 + +**结果**：你能会在 Web æµè§ˆå™¨ä¸­çœ‹åˆ° BookInfo 应用。 + +如需检查 Istio 控制器 URL 和端å£çš„帮助，请å°è¯•è¿è¡Œ [Istio 文档](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#determining-the-ingress-ip-and-ports)中的命令。 + +## 故障排除 + +[官方 Istio 文档](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#troubleshooting)建议使用 `kubectl` 命令æ¥æ£€æŸ¥å¤–部请求的正确 ingress 主机和 ingress 端å£ã€‚ + +### 确认 Kubernetes 网关与 Istio çš„ Ingress Controller åŒ¹é… + +ä½ å¯ä»¥å°è¯•æ‰§è¡Œæœ¬èŠ‚ä¸­çš„æ­¥éª¤ä»¥ç¡®ä¿ Kubernetes 网关é…置正确。 + +在网关资æºä¸­ï¼Œé€‰æ‹©å™¨é€šè¿‡æ ‡ç­¾æ¥å¼•ç”¨ Istio 的默认 Ingress Controller，其中标签的键是 `Istio`，值是 `ingressgateway`。è¦ç¡®ä¿æ ‡ç­¾é€‚用于网关，请执行以下æ“作： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å‘下滚动到 `istio-system` 命å空间。 +1. 在 `istio-system`中，有一个å为 `istio-ingressgateway` 的工作负载。å•å‡»æ­¤å·¥ä½œè´Ÿè½½çš„å称并转到**标签和注释**部分。你应该看到它具有 `istio` 键和 `ingressgateway` 值。这确认了 Gateway 资æºä¸­çš„选择器与 Istio 的默认 ingress controller 匹é…。 + +### åŽç»­æ­¥éª¤ +[设置 Istio çš„æµé‡ç®¡ç†ç»„件](set-up-traffic-management.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md new file mode 100644 index 00000000000..9782bd28938 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md @@ -0,0 +1,76 @@ +--- +title: 5. 设置 Istio çš„æµé‡ç®¡ç†ç»„件 +--- + +Istio 中æµé‡ç®¡ç†çš„一个核心优势是å…许动æ€è¯·æ±‚路由。动æ€è¯·æ±‚路由通常应用于金ä¸é›€éƒ¨ç½²å’Œè“/绿部署等。Istio æµé‡ç®¡ç†ä¸­çš„两个关键资æºæ˜¯*虚拟æœåŠ¡*å’Œ*目标规则*。 + +- [虚拟æœåŠ¡](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/)：拦截并将æµé‡é‡å®šå‘到你的 Kubernetes Service 上。这样，你å¯ä»¥å°†éƒ¨åˆ†è¯·æ±‚æµé‡åˆ†é…到ä¸åŒçš„æœåŠ¡ä¸Šã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›æœåŠ¡æ¥å®šä¹‰ä¸€ç»„路由规则，用于主机寻å€ã€‚ +- [目标规则](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule/)：作为唯一å¯ä¿¡æ¥æºï¼Œè¡¨æ˜Žå“ªäº›æœåŠ¡ç‰ˆæœ¬å¯ç”¨äºŽæŽ¥æ”¶è™šæ‹ŸæœåŠ¡çš„æµé‡ã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºæ¥å®šä¹‰ç­–略，这些策略适用于路由å‘生åŽç”¨äºŽæœåŠ¡çš„æµé‡ã€‚ + +本文介ç»å¦‚何在示例 BookInfo 应用中添加与 `reviews` å¾®æœåŠ¡å¯¹åº”的虚拟æœåŠ¡ç¤ºä¾‹ã€‚æ­¤æœåŠ¡çš„目的是在 `reviews` æœåŠ¡çš„两个版本之间划分æµé‡ã€‚ + +在这个示例中，我们将æµé‡å¸¦åˆ° `reviews` æœåŠ¡ä¸­å¹¶æ‹¦æˆªæµé‡ï¼Œè¿™æ ·ï¼Œ50% çš„æµé‡ä¼šæµå‘æœåŠ¡çš„ `v1`，å¦å¤– 50% çš„æµé‡ä¼šæµå‘ `v2 `。 + +部署这个虚拟æœåŠ¡åŽï¼Œæˆ‘们将生æˆæµé‡ï¼Œå¹¶é€šè¿‡ Kiali å¯è§†åŒ–看到æµé‡å¹³å‡è·¯ç”±åˆ°æœåŠ¡çš„两个版本中。 + +è¦ä¸º `reviews` æœåŠ¡éƒ¨ç½²è™šæ‹ŸæœåŠ¡å’Œç›®æ ‡è§„则： +1. 点击 **☰ > 集群管ç†**。 +1. 转到安装了 Istio 的集群，然åŽå•å‡» **Explore**。 +1. 在安装了 Istio 的集群中，点击左侧导航æ ä¸­çš„ **Istio > DestinationRules**。 +1. å•å‡»**创建**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ DestinationRule YAML。 +1. å•å‡»**创建**。 +1. å•å‡»**以 YAML 文件编辑**并使用此é…置： + + ```yaml + apiVersion: networking.istio.io/v1alpha3 + kind: DestinationRule + metadata: + name: reviews + spec: + host: reviews + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 + - name: v3 + labels: + version: v3 + ``` +1. å•å‡»**创建**。 + +然åŽï¼Œéƒ¨ç½²æ供利用 DestinationRule çš„æµé‡è·¯ç”±çš„ VirtualService： + +1. å•å‡»ä¾§å¯¼èˆªæ ä¸­çš„ **VirtualService**。 +1. å•å‡»**使用 YAML 文件创建**。 +1. å¤åˆ¶å¹¶ç²˜è´´ä¸‹é¢çš„ VirtualService YAML。 +1. å•å‡»**创建**。 + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: reviews +spec: + hosts: + - reviews + http: + - route: + - destination: + host: reviews + subset: v1 + weight: 50 + - destination: + host: reviews + subset: v3 + weight: 50 +--- +``` + +**结果**：生æˆæµåˆ°è¯¥æœåŠ¡çš„æµé‡æ—¶ï¼ˆä¾‹å¦‚，刷新 Ingress Gateway URL），你å¯ä»¥åœ¨ Kiali æµé‡å›¾ä¸­çœ‹åˆ°æµåˆ° `reviews` æœåŠ¡çš„æµé‡è¢«å¹³å‡åˆ†é…到了 `v1` å’Œ `v3`。 + +### åŽç»­æ­¥éª¤ +[生æˆå’ŒæŸ¥çœ‹æµé‡](generate-and-view-traffic.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md new file mode 100644 index 00000000000..2290de911ad --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md @@ -0,0 +1,360 @@ +--- +title: 3. 使用 Istio Sidecar 添加部署和æœåŠ¡ +--- + +:::note 先决æ¡ä»¶ï¼š + +è¦ä¸ºå·¥ä½œè´Ÿè½½å¯ç”¨ Istio，你必须先在集群和命å空间中安装 Istio 应用。 + +::: + +在命å空间中å¯ç”¨ Istio åªä¼šä¸ºæ–°å·¥ä½œè´Ÿè½½å¯ç”¨è‡ªåŠ¨ sidecar 注入。è¦ä¸ºçŽ°æœ‰å·¥ä½œè´Ÿè½½å¯ç”¨ Envoy sidecar，你需è¦æ‰‹åŠ¨ä¸ºæ¯ä¸ªå·¥ä½œè´Ÿè½½å¯ç”¨å®ƒã€‚ + +è¦åœ¨å‘½å空间中的现有工作负载上注入 Istio sidecar： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 点击**工作负载**。 +1. 转到è¦æ³¨å…¥ Istio sidecar 的工作负载，然åŽå•å‡» **â‹® > é‡æ–°éƒ¨ç½²**。é‡æ–°éƒ¨ç½²å·¥ä½œè´Ÿè½½åŽï¼Œè¯¥å·¥ä½œè´Ÿè½½ä¼šè‡ªåŠ¨æ³¨å…¥ Envoy sidecar。 + +等待几分钟，然åŽå·¥ä½œè´Ÿè½½å°†å‡çº§å¹¶å…·æœ‰ Istio sidecar。å•å‡»å®ƒå¹¶è½¬åˆ°**容器**。你应该能看到该工作负载æ—边的 `istio-proxy`。这æ„味ç€ä¸ºå·¥ä½œè´Ÿè½½å¯ç”¨äº† Istio sidecar。Istio 正在为 Sidecar Envoy åšæ‰€æœ‰çš„接线工作。如果你现在在 yaml 中å¯ç”¨å®ƒä»¬ï¼ŒIstio å¯ä»¥è‡ªåŠ¨æ‰§è¡Œæ‰€æœ‰åŠŸèƒ½ã€‚ + +### 添加部署和æœåŠ¡ + +以下是在命å空间中添加新 **Deployment** 的几ç§æ–¹æ³•ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 点击 **Deployment**。 +1. 填写表å•ï¼Œæˆ–**以 YAML 文件编辑**。 +1. å•å‡»**创建**。 + +è¦å°† **Service** 添加到你的命å空间： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**æœåŠ¡å‘现 > æœåŠ¡**。 +1. å•å‡»**创建**。 +1. 选择所需的æœåŠ¡ç±»åž‹ã€‚ +1. 填写表å•ï¼Œæˆ–**以 YAML 文件编辑**。 +1. 点击**创建**。 + +你还å¯ä»¥ä½¿ç”¨ kubectl **shell** æ¥åˆ›å»º deployment å’Œ service： + +1. 如果你的文件存储在本地集群中，è¿è¡Œ `kubectl create -f .yaml`。 +1. 或è¿è¡Œ `cat<< EOF | kubectl apply -f -`，将文件内容粘贴到终端，然åŽè¿è¡Œ `EOF` æ¥å®Œæˆå‘½ä»¤ã€‚ + +### 部署和æœåŠ¡ç¤ºä¾‹ + +接下æ¥ï¼Œæˆ‘们为 Istio 文档中的 BookInfo 应用的示例部署和æœåŠ¡æ·»åŠ  Kubernetes 资æºï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œæ‰“å¼€ kubectl shell。 +1. è¿è¡Œ `cat<< EOF | kubectl apply -f -`。 +1. 将以下资æºå¤åˆ¶åˆ° shell 中。 +1. è¿è¡Œ `EOF`。 + +这将在 Istio 的示例 BookInfo 应用中设置以下示例资æºï¼š + +Details æœåŠ¡å’Œéƒ¨ç½²ï¼š + +- 一个 `details` Service。 +- 一个 `bookinfo-details` çš„ ServiceAccount。 +- 一个 `details-v1` Deployment。 + +Ratings æœåŠ¡å’Œéƒ¨ç½²ï¼š + +- 一个 `ratings` Service。 +- 一个 `bookinfo-ratings` çš„ ServiceAccount。 +- 一个 `ratings-v1` Deployment。 + +Reviews æœåŠ¡å’Œéƒ¨ç½²ï¼ˆä¸‰ä¸ªç‰ˆæœ¬ï¼‰ï¼š + +- 一个 `reviews` Service。 +- 一个 `bookinfo-reviews` çš„ ServiceAccount。 +- 一个 `reviews-v1` Deployment。 +- 一个 `reviews-v2` Deployment。 +- 一个 `reviews-v3` Deployment。 + +Productpage æœåŠ¡å’Œéƒ¨ç½²ï¼š + +这是应用的主页，å¯ä»¥é€šè¿‡ç½‘络æµè§ˆå™¨ä¸­æŸ¥çœ‹ã€‚将从该页é¢è°ƒç”¨å…¶ä»–æœåŠ¡ã€‚ + +- 一个 `productpage` service。 +- 一个 `bookinfo-productpage` çš„ ServiceAccount。 +- 一个 `productpage-v1` Deployment。 + +### èµ„æº YAML + +```yaml +# Copyright 2017 Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################################## +# Details service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: details + labels: + app: details + service: details +spec: + ports: + - port: 9080 + name: http + selector: + app: details +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-details +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: details-v1 + labels: + app: details + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: details + version: v1 + template: + metadata: + labels: + app: details + version: v1 + spec: + serviceAccountName: bookinfo-details + containers: + - name: details + image: docker.io/istio/examples-bookinfo-details-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Ratings service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: ratings + labels: + app: ratings + service: ratings +spec: + ports: + - port: 9080 + name: http + selector: + app: ratings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-ratings +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ratings-v1 + labels: + app: ratings + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: ratings + version: v1 + template: + metadata: + labels: + app: ratings + version: v1 + spec: + serviceAccountName: bookinfo-ratings + containers: + - name: ratings + image: docker.io/istio/examples-bookinfo-ratings-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Reviews service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: reviews + labels: + app: reviews + service: reviews +spec: + ports: + - port: 9080 + name: http + selector: + app: reviews +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-reviews +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v1 + labels: + app: reviews + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v1 + template: + metadata: + labels: + app: reviews + version: v1 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v2 + labels: + app: reviews + version: v2 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v2 + template: + metadata: + labels: + app: reviews + version: v2 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v2:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v3 + labels: + app: reviews + version: v3 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v3 + template: + metadata: + labels: + app: reviews + version: v3 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v3:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Productpage services +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: productpage + labels: + app: productpage + service: productpage +spec: + ports: + - port: 9080 + name: http + selector: + app: productpage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-productpage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: productpage-v1 + labels: + app: productpage + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: productpage + version: v1 + template: + metadata: + labels: + app: productpage + version: v1 + spec: + serviceAccountName: bookinfo-productpage + containers: + - name: productpage + image: docker.io/istio/examples-bookinfo-productpage-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +``` + +### åŽç»­æ­¥éª¤ +[设置 Istio Gateway](set-up-istio-gateway.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md new file mode 100644 index 00000000000..91578148d71 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md @@ -0,0 +1,39 @@ +--- +title: Pod 安全策略 +--- + +:::note + +本文介ç»çš„集群选项仅适用于 [Rancher 已在其中å¯åŠ¨ Kubernetes 的集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +::: + +ä½ å¯ä»¥åœ¨åˆ›å»ºé¡¹ç›®çš„时候设置 Pod 安全策略（PSPï¼‰ã€‚å¦‚æžœåœ¨åˆ›å»ºé¡¹ç›®æœŸé—´æ²¡æœ‰ä¸ºé¡¹ç›®åˆ†é… PSP，你也éšæ—¶å¯ä»¥å°† PSP 分é…给现有项目。 + +### 先决æ¡ä»¶ + +- 在 Rancher 中创建 Pod 安全策略。在将默认 PSP 分é…给现有项目之å‰ï¼Œä½ å¿…须有一个å¯åˆ†é…çš„ PSP。有关说明，请å‚阅[创建 Pod 安全策略](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 +- 将默认 Pod 安全策略分é…给项目所属的集群。如果 PSP 还没有应用到集群，你无法将 PSP 分é…给项目。有关详细信æ¯ï¼Œè¯·å‚阅[å°† pod 安全策略添加到集群](../../new-user-guides/manage-clusters/add-a-pod-security-policy.md)。 + +### 应用 Pod 安全策略 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°éœ€è¦ç§»åŠ¨å‘½å空间的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦æ·»åŠ  PSP 的项目。在该项目中选择 **â‹® > 编辑é…ç½®**。 +1. 从 **Pod 安全策略**下拉列表中，选择è¦åº”用于项目的 PSP。 + å°† PSP 分é…给项目将： + +- 覆盖集群的默认 PSP。 +- å°† PSP 应用于项目。 +- å°† PSP 应用到åŽç»­æ·»åŠ åˆ°é¡¹ç›®ä¸­çš„命å空间。 + +1. å•å‡»**ä¿å­˜**。 + +**结果**：已将 PSP 应用到项目以åŠé¡¹ç›®å†…的命å空间。 + +:::note + +å¯¹äºŽåœ¨åˆ†é… PSP 之å‰å·²ç»åœ¨é›†ç¾¤æˆ–项目中è¿è¡Œå·¥ä½œè´Ÿè½½ï¼ŒRancher ä¸ä¼šæ£€æŸ¥å®ƒä»¬æ˜¯å¦ç¬¦åˆ PSP。你需è¦å…‹éš†æˆ–å‡çº§å·¥ä½œè´Ÿè½½ä»¥æŸ¥çœ‹å®ƒä»¬æ˜¯å¦é€šè¿‡ PSP。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md new file mode 100644 index 00000000000..26e87e8f31a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md @@ -0,0 +1,67 @@ +--- +title: Rancher 项目中资æºé…é¢çš„å·¥ä½œåŽŸç† +--- + +Rancher 中的资æºé…é¢åŒ…å«ä¸Ž [Kubernetes 原生版本](https://kubernetes.io/docs/concepts/policy/resource-quotas/)相åŒçš„功能。Rancher 还扩展了资æºé…é¢çš„功能，让你将资æºé…é¢åº”用于项目。 + +在标准 Kubernetes deployment 中，资æºé…é¢ä¼šåº”用于å„个命å空间。但是，你ä¸èƒ½é€šè¿‡å•æ¬¡æ“作将é…é¢åº”用到多个命å空间，而必须多次应用资æºé…é¢ã€‚ + +在下图中，Kubernetes 管ç†å‘˜è¯•å›¾åœ¨æ²¡æœ‰ Rancher 的情况下强制执行资æºé…é¢ã€‚管ç†å‘˜æƒ³è¦ä½¿ç”¨ä¸€ä¸ªèµ„æºé…é¢æ¥ä¸ºé›†ç¾¤ä¸­çš„æ¯ä¸ªå‘½å空间é…置统一的 CPU 和内存é™åˆ¶ (`Namespace 1-4`)。但是，在 Kubernetes 的基础版本中，æ¯ä¸ªå‘½å空间都需è¦å•ç‹¬è®¾ç½®èµ„æºé…é¢ã€‚因此，管ç†å‘˜å¿…须创建四个é…置相åŒè§„格的ä¸åŒèµ„æºé…é¢ï¼ˆ`Resource Quota 1-4`）并å•ç‹¬åº”用这些é…é¢ã€‚ + +Kubernetes 基础版本：æ¯ä¸ªå‘½å空间都需è¦ç‹¬ç«‹è®¾ç½®èµ„æºé…é¢ + +![原生 Kubernetes 资æºé…é¢å®žçŽ°](/img/kubernetes-resource-quota.svg) + +和原生 Kubernetes 相比，Rancher 的资æºé…é¢æœ‰ä¸åŒã€‚在 Rancher 中，你å¯ä»¥æŠŠèµ„æºé…é¢åº”用到项目层级，进而让项目的资æºé…é¢æ²¿ç”¨åˆ°é¡¹ç›®å†…çš„æ¯ä¸€ä¸ªå‘½åç©ºé—´ï¼Œç„¶åŽ Kubernetes 会使用原生的资æºé…é¢æ¥å¼ºåˆ¶æ‰§è¡Œä½ è®¾ç½®çš„é™åˆ¶ã€‚如果è¦æ›´æ”¹ç‰¹å®šå‘½å空间的é…é¢ï¼Œä½ ä¹Ÿå¯ä»¥è¦†ç›–设置。 + +项目é…é¢åŒ…括你在创建或编辑集群时设置的两个é™åˆ¶ï¼š + + +- **项目é™åˆ¶**： + + é…置了项目中所有命å空间共享的æ¯ä¸ªæŒ‡å®šèµ„æºçš„总é™åˆ¶ã€‚ + +- **命å空间默认é™åˆ¶**： + + é…置了æ¯ä¸ªå‘½å空间对æ¯ä¸ªæŒ‡å®šèµ„æºçš„默认é…é¢ã€‚ + 如果项目中的命å空间é…置没有被覆盖，那么此é™åˆ¶ä¼šè‡ªåŠ¨ç»‘定到命å空间并强制执行。 + + +在下图中，Rancher 管ç†å‘˜æƒ³ä½¿ç”¨èµ„æºé…é¢æ¥ä¸ºé¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间（`命å空间 1-4`）设置相åŒçš„ CPU 和内存é™åˆ¶ã€‚在 Rancher 中，管ç†å‘˜å¯ä»¥ä¸ºé¡¹ç›®è®¾ç½®èµ„æºé…é¢ï¼ˆ`项目资æºé…é¢`），而ä¸éœ€è¦ä¸ºå‘½å空间å•ç‹¬è¿›è¡Œè®¾ç½®ã€‚æ­¤é…é¢åŒ…括整个项目（`项目é™åˆ¶`）和å•ä¸ªå‘½å空间（`命å空间默认é™åˆ¶`）的资æºé™åˆ¶ã€‚然åŽï¼ŒRancher 会将`命å空间默认é™åˆ¶`çš„é…é¢æ²¿ç”¨åˆ°æ¯ä¸ªå‘½å空间（`命å空间资æºé…é¢`）。 + +Rancher：资æºé…é¢æ²¿ç”¨åˆ°æ¯ä¸ªå‘½å空间 + +![Rancher 资æºé…é¢å®žçŽ°](/img/rancher-resource-quota.png) + +以下介ç»åœ¨ Rancher UI **_中_** 创建的命å空间的更细微的功能。如果你删除了项目级别的资æºé…é¢ï¼Œæ— è®ºå‘½å空间层级是å¦æœ‰è‡ªå®šä¹‰çš„资æºé…é¢ï¼Œé¡¹ç›®å†…的所有命å空间也会移除这个资æºé…é¢ã€‚在项目层级修改已有的命å空间默认资æºé…é¢ï¼Œä¸ä¼šå½±å“命å空间内的资æºé…é¢ï¼Œä¿®æ”¹åŽçš„项目层级资æºé…é¢åªä¼šå¯¹ä»¥åŽæ–°å»ºçš„命å空间生效。è¦ä¿®æ”¹å¤šä¸ªçŽ°æœ‰å‘½å空间的默认é™åˆ¶ï¼Œä½ å¯ä»¥åœ¨é¡¹ç›®å±‚级删除该é™åˆ¶ï¼Œç„¶åŽå†ä½¿ç”¨æ–°çš„默认值é‡æ–°åˆ›å»ºé…é¢ã€‚è¿™ç§æ–¹å¼ä¼šå°†æ–°çš„默认值应用于项目中的所有现有命å空间。 + +在项目中创建命å空间之å‰ï¼ŒRancher 会使用默认é™åˆ¶å’Œè¦†ç›–é™åˆ¶æ¥å¯¹æ¯”项目内的å¯ç”¨èµ„æºå’Œè¯·æ±‚资æºã€‚ +如果请求的资æºè¶…过了项目中这些资æºçš„剩余容é‡ï¼ŒRancher 将为命å空间分é…该资æºçš„剩余容é‡ã€‚ + +但是，在 Rancher çš„ UI **_外_** 创建的命å空间的处ç†æ–¹æ³•åˆ™ä¸ä¸€æ ·ã€‚对于通过 `kubectl` 创建的命å空间，如果请求的资æºé‡å¤šä½™é¡¹ç›®å†…çš„ä½™é‡ï¼ŒRancher 会分é…一个数值为 **0** 的资æºé…é¢ã€‚ + +è¦ä½¿ç”¨ `kubectl` 在现有项目中创建命å空间，请使用 `field.cattle.io/projectId` 注释。è¦è¦†ç›–默认的请求é…é¢é™åˆ¶ï¼Œè¯·ä½¿ç”¨ `field.cattle.io/resourceQuota` 注释。 + +请注æ„，Rancher åªä¼šè¦†ç›–项目é…é¢ä¸Šå®šä¹‰çš„资æºé™åˆ¶ã€‚ + +``` +apiVersion: v1 +kind: Namespace +metadata: + annotations: + field.cattle.io/projectId: [your-cluster-ID]:[your-project-ID] + field.cattle.io/resourceQuota: '{"limit":{"limitsCpu":"100m", "configMaps": "50"}}' + name: my-ns +``` +在此示例中，如果项目é…é¢åœ¨å…¶èµ„æºåˆ—表中ä¸åŒ…å« configMaps，那么 Rancher 将忽略此覆盖中的 `configMaps`。 + +对于项目中未定义的资æºï¼Œå»ºè®®ä½ åœ¨å‘½å空间中创建专用的 `ResourceQuota` 对象æ¥é…置其它自定义é™åˆ¶ã€‚ +资æºé…é¢æ˜¯åŽŸç”Ÿ Kubernetes 对象，如果命å空间属于具有é…é¢çš„项目，Rancher 将忽略用户定义的é…é¢ï¼Œä»Žè€Œç»™äºˆç”¨æˆ·æ›´å¤šçš„控制æƒã€‚ + +下表对比了 Rancher å’Œ Kubernetes 资æºé…é¢çš„主è¦åŒºåˆ«ï¼š + +| Rancher 资æºé…é¢ | Kubernetes 资æºé…é¢ | +| ---------------------------------------------------------- | -------------------------------------------------------- | +| 应用于项目和命å空间。 | 仅应用于命å空间。 | +| 为项目中的所有命å空间创建资æºæ± ã€‚ | å°†é™æ€èµ„æºé™åˆ¶åº”用到å•ç‹¬çš„命å空间。 | +| 通过沿用的模å¼ï¼Œå°†èµ„æºé…é¢åº”用于å„个命å空间。 | 仅应用于指定的命å空间。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md new file mode 100644 index 00000000000..f29c3004b85 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md @@ -0,0 +1,34 @@ +--- +title: 覆盖命å空间的默认é™åˆ¶ +--- + +**命å空间默认é™åˆ¶**会在创建时从项目沿用到æ¯ä¸ªå‘½å空间。但在æŸäº›æƒ…况下，你å¯èƒ½éœ€è¦å¢žåŠ æˆ–å‡å°‘特定命å空间的é…é¢ã€‚在这ç§æƒ…况下，你å¯ä»¥é€šè¿‡ç¼–辑命å空间æ¥è¦†ç›–默认é™åˆ¶ã€‚ + +在下图中，Rancher 管ç†å‘˜çš„项目有一个已生效的资æºé…é¢ã€‚但是，管ç†å‘˜æƒ³è¦è¦†ç›– `Namespace 3` 的命å空间é™åˆ¶ï¼Œä»¥ä¾¿è®©è¯¥å‘½å空间使用更多资æºã€‚因此，管ç†å‘˜[æ高了 `Namespace 3` 的命å空间é™åˆ¶](../../../new-user-guides/manage-clusters/projects-and-namespaces.md)，以便命å空间å¯ä»¥è®¿é—®æ›´å¤šèµ„æºã€‚ + +命å空间默认é™åˆ¶è¦†ç›– + +![命å空间默认é™åˆ¶è¦†ç›–](/img/rancher-resource-quota-override.svg) + +有关详细信æ¯ï¼Œè¯·å‚阅[如何编辑命å空间资æºé…é¢](../../../new-user-guides/manage-clusters/projects-and-namespaces.md)。 + +### 编辑命å空间资æºé…é¢ + +如果你已为项目é…置了资æºé…é¢ï¼Œä½ å¯ä»¥è¦†ç›–命å空间默认é™åˆ¶ï¼Œä»Žè€Œä¸ºç‰¹å®šå‘½å空间æ供对更多（或更少）项目资æºçš„访问æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦ç¼–辑命å空间资æºé…é¢çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦ä¸ºå…¶ç¼–辑资æºé…é¢çš„命å空间。å•å‡» **â‹® > 编辑é…ç½®**。 +1. 编辑资æºé™åˆ¶ã€‚这些é™åˆ¶å†³å®šäº†å‘½å空间å¯ç”¨çš„资æºã€‚必须在项目é™åˆ¶èŒƒå›´å†…é…置这些é…é¢é™åˆ¶ã€‚ + + 有关æ¯ä¸ª**资æºç±»åž‹**的详细信æ¯ï¼Œè¯·å‚阅[类型å‚考](resource-quota-types.md)。 + + :::note + + - 如果没有为项目é…置资æºé…é¢ï¼Œè¿™äº›é€‰é¡¹å°†ä¸å¯ç”¨ã€‚ + - 如果你输入的é™åˆ¶è¶…过了é…置的项目é™åˆ¶ï¼Œä½ å°†æ— æ³•ä¿å­˜ä¿®æ”¹ã€‚ + + ::: + +**结果**：覆盖设置已ç»åº”用到命å空间的资æºé…é¢ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md new file mode 100644 index 00000000000..b0873befd7a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md @@ -0,0 +1,27 @@ +--- +title: 资æºé…é¢ç±»åž‹å‚考 +--- + +创建资æºé…é¢ç›¸å½“于é…置项目å¯ç”¨çš„资æºæ± ã€‚ä½ å¯ä»¥ä¸ºä»¥ä¸‹èµ„æºç±»åž‹è®¾ç½®èµ„æºé…é¢ï¼š + +| 资æºç±»åž‹ | æè¿° | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU é™åˆ¶\* | 分é…给项目/命å空间的最大 CPU é‡ï¼ˆä»¥[毫核](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)为å•ä½ï¼‰1 | +| CPU 预留\* | 预留给项目/命åç©ºé—´çš„æœ€å° CPU é‡ï¼ˆä»¥æ¯«æ ¸ä¸ºå•ä½ï¼‰1 | +| 内存é™åˆ¶\* | 分é…给项目/命å空间的最大内存é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰1 | +| 内存预留\* | 预留给项目/命å空间的最å°å†…å­˜é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰1 | +| 存储预留 | 预留给项目/命å空间的最å°å­˜å‚¨é‡ï¼ˆä»¥åƒå…†å­—节为å•ä½ï¼‰ | +| æœåŠ¡è´Ÿè½½å‡è¡¡å™¨ | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„è´Ÿè½½å‡è¡¡å™¨æœåŠ¡çš„æœ€å¤§æ•°é‡ | +| æœåŠ¡èŠ‚ç‚¹ç«¯å£ | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„节点端å£æœåŠ¡çš„æœ€å¤§æ•°é‡ | +| Pod | å¯ä»¥åœ¨é¡¹ç›®/命å空间中以éžç»ˆç«¯çŠ¶æ€å­˜åœ¨çš„ pod 的最大数é‡ï¼ˆå³ `.status.phase in (Failed, Succeeded)` 等于 true çš„ pod） | +| Services | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„最大 service æ•°é‡ | +| ConfigMap | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„ ConfigMap çš„æœ€å¤§æ•°é‡ | +| æŒä¹…å·å£°æ˜Ž | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„æŒä¹…å·å£°æ˜Žçš„æœ€å¤§æ•°é‡ | +| ReplicationController | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„最大 ReplicationController æ•°é‡ | +| 密文 | 项目/命å空间中å¯ä»¥å­˜åœ¨çš„æœ€å¤§å¯†æ–‡æ•°é‡ | + +:::note ***** + +在设置资æºé…é¢æ—¶ï¼Œå¦‚果你在项目或命å空间上设置了任何与 CPU 或内存相关的内容（å³é™åˆ¶æˆ–预留），所有容器都需è¦åœ¨åˆ›å»ºæœŸé—´è®¾ç½®å„自的 CPU 或内存字段。你å¯ä»¥åŒæ—¶è®¾ç½®å®¹å™¨çš„默认资æºé™åˆ¶ï¼Œä»¥é¿å…为æ¯ä¸ªå·¥ä½œè´Ÿè½½æ˜¾å¼è®¾ç½®è¿™äº›é™åˆ¶ã€‚详情请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits)。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md new file mode 100644 index 00000000000..e56f617e7a0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md @@ -0,0 +1,40 @@ +--- +title: 设置容器默认资æºé™åˆ¶ +--- + +在设置资æºé…é¢æ—¶ï¼Œå¦‚果你在项目或命å空间上设置了任何与 CPU 或内存相关的内容（å³é™åˆ¶æˆ–预留），所有容器都需è¦åœ¨åˆ›å»ºæœŸé—´è®¾ç½®å„自的 CPU 或内存字段。详情请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits)。 + +为了é¿å…在创建工作负载期间对æ¯ä¸ªå®¹å™¨è®¾ç½®è¿™äº›é™åˆ¶ï¼Œå¯ä»¥åœ¨å‘½å空间上指定一个默认的容器资æºé™åˆ¶ã€‚ + +### 编辑容器默认资æºé™åˆ¶ + +ä½ å¯ä»¥åœ¨ä»¥ä¸‹æƒ…况下编辑容器的默认资æºé™åˆ¶ï¼š + +- 你在项目上设置了 CPU 或内存资æºé…é¢ï¼ŒçŽ°åœ¨éœ€è¦ä¸ºå®¹å™¨è®¾ç½®ç›¸åº”的默认值。 +- 你需è¦ç¼–辑容器的默认资æºé™åˆ¶ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦ç¼–辑默认资æºé™åˆ¶çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦ç¼–辑容器默认资æºé™åˆ¶çš„项目。在该项目中选择 **â‹® > 编辑é…ç½®**。 +1. 展开**容器默认资æºé™åˆ¶**并编辑对应的值。 + +### 沿用资æºé™åˆ¶ + +在项目级别设置默认容器资æºé™åˆ¶åŽï¼Œé¡¹ç›®ä¸­æ‰€æœ‰æ–°å»ºçš„命å空间都会沿用这个资æºé™åˆ¶å‚数。新设置的é™åˆ¶ä¸ä¼šå½±å“项目中现有的命å空间。你需è¦ä¸ºé¡¹ç›®ä¸­çš„现有命å空间手动设置默认容器资æºé™åˆ¶ï¼Œä»¥ä¾¿åˆ›å»ºå®¹å™¨æ—¶èƒ½åº”用该é™åˆ¶ã€‚ + +ä½ å¯ä»¥ä¸ºé¡¹ç›®è®¾ç½®å®¹å™¨çš„默认资æºé™åˆ¶å¹¶å¯åŠ¨ä»»ä½•å•†åº—应用。 + +在命å空间上é…置容器默认资æºé™åˆ¶åŽï¼Œåœ¨è¯¥å‘½å空间中创建的任何容器都会沿用该默认值。你å¯ä»¥åœ¨å·¥ä½œè´Ÿè½½åˆ›å»ºæœŸé—´è¦†ç›–这些é™åˆ¶/预留。 + +### 容器资æºé…é¢ç±»åž‹ + +å¯ä»¥é…置以下资æºé™åˆ¶ï¼š + +| 资æºç±»åž‹ | æè¿° | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU é™åˆ¶ | 分é…给容器的最大 CPU é‡ï¼ˆä»¥[毫核](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)为å•ä½ï¼‰ã€‚ | +| CPU 预留 | ä¿ç•™ç»™å®¹å™¨çš„æœ€å° CPU é‡ï¼ˆä»¥æ¯«æ ¸ä¸ºå•ä½ï¼‰ã€‚ | +| 内存é™åˆ¶ | 分é…给容器的最大内存é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰ã€‚ | +| 内存预留 | ä¿ç•™ç»™å®¹å™¨çš„最å°å†…å­˜é‡ï¼ˆä»¥å­—节为å•ä½ï¼‰ã€‚ | +| NVIDIA GPU é™åˆ¶/预留 | 分é…给容器的 GPU æ•°é‡ã€‚GPU çš„é™åˆ¶å’Œé¢„留始终相åŒã€‚ | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md new file mode 100644 index 00000000000..e5a2deccf45 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md @@ -0,0 +1,148 @@ +--- +title: æŒä¹…化 Grafana ä»ªè¡¨æ¿ +--- + +è¦åœ¨é‡å¯ Grafana 实例åŽä¿å­˜ Grafana 仪表æ¿ï¼Œè¯·å°†ä»ªè¡¨æ¿çš„é…ç½® JSON 添加到 ConfigMap 中。ConfigMap 还支æŒä½¿ç”¨åŸºäºŽ GitOps 或 CD 的方法æ¥éƒ¨ç½²ä»ªè¡¨æ¿ï¼Œä»Žè€Œè®©ä½ å¯¹ä»ªè¡¨æ¿è¿›è¡Œç‰ˆæœ¬æŽ§åˆ¶ã€‚ + +- [创建æŒä¹…化 Grafana 仪表æ¿](#创建æŒä¹…化-grafana-仪表æ¿) +- [已知问题](#已知问题) + +## 创建æŒä¹…化 Grafana ä»ªè¡¨æ¿ + + + + +:::note 先决æ¡ä»¶ï¼š + +- 已安装 Monitoring 应用。 +- è¦åˆ›å»ºæŒä¹…化仪表æ¿ï¼Œä½ å¿…é¡»åœ¨åŒ…å« Grafana 仪表æ¿çš„项目或命å空间中至少具有**ç®¡ç† ConfigMap** çš„ Rancher RBAC æƒé™ã€‚这与 Monitoring Chart 公开的 `monitoring-dashboard-edit` 或 `monitoring-dashboard-admin` Kubernetes 原生 RBAC 角色对应。 +- è¦æŸ¥çœ‹æŒ‡å‘外部监控 UI（包括 Grafana 仪表æ¿ï¼‰çš„链接，你至少需è¦ä¸€ä¸ª [project-member 角色](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#具有-rancher-æƒé™çš„用户)。 + +::: + +### 1. 获å–è¦æŒä¹…化的仪表æ¿çš„ JSON 模型 + +è¦åˆ›å»ºæŒä¹…化仪表æ¿ï¼Œä½ éœ€è¦èŽ·å–è¦æŒä¹…化的仪表æ¿çš„ JSON 模型。你å¯ä»¥ä½¿ç”¨é¢„制仪表æ¿æˆ–自行构建仪表æ¿ã€‚ + +è¦ä½¿ç”¨é¢„制仪表æ¿ï¼Œè¯·è½¬åˆ° [https://grafana.com/grafana/dashboards](https://grafana.com/grafana/dashboards)，打开详细信æ¯é¡µé¢ï¼Œç„¶åŽå•å‡» **Download JSON** 按钮æ¥èŽ·å–下一步所需的 JSON 模型。 + +è¦ä½¿ç”¨ä½ è‡ªå·±çš„仪表æ¿ï¼š + +1. 点击链接打开 Grafana。在集群详细信æ¯é¡µé¢ä¸Šï¼Œå•å‡» **Monitoring**。 +1. 登录到 Grafana。请注æ„，Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin/prom-operator`。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + :::note + + 无论è°æ‹¥æœ‰å¯†ç ï¼Œä½ éƒ½éœ€è¦åœ¨éƒ¨ç½²äº† Rancher Monitoring 的项目中至少具有管ç†æœåŠ¡æˆ–查看监控的æƒé™æ‰èƒ½è®¿é—® Grafana 实例。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + ::: + +1. 使用 Grafana UI 创建仪表æ¿ã€‚完æˆåŽï¼Œå•å‡»é¡¶éƒ¨å¯¼èˆªèœå•ä¸­çš„齿轮图标转到仪表æ¿è®¾ç½®é¡µé¢ã€‚在左侧导航èœå•ä¸­ï¼Œå•å‡» **JSON Model**。 +1. å¤åˆ¶å‡ºçŽ°çš„ JSON æ•°æ®ç»“构。 + +### 2. 使用 Grafana JSON 模型创建 ConfigMap + +åœ¨åŒ…å« Grafana 仪表æ¿çš„命å空间中创建一个 ConfigMap（默认为 cattle-dashboards ）。 + +ConfigMap 与以下内容类似： + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards # 如果ä¸ä½¿ç”¨é»˜è®¤å‘½å空间，则修改此值 +data: + .json: |- + +``` + +默认情况下，Grafana é…置为监控 `cattle-dashboards` 命å空间中带有 `grafana_dashboard` 标签的所有 ConfigMap。 + +è¦è®© Grafana 监控所有命å空间中的 ConfigMap，请å‚阅[本节](#为-grafana-仪表æ¿-configmap-é…置命å空间)。 + +è¦åœ¨ Rancher UI 中创建 ConfigMap： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**æ›´å¤šèµ„æº > 核心 > é…置映射**。 +1. å•å‡»**创建**。 +1. 设置与上例类似的键值对。输入 `.json` 的值时，点击**从文件读å–**并上传 JSON æ•°æ®æ¨¡åž‹ã€‚ +1. å•å‡»**创建**。 + +**结果**：创建 ConfigMap åŽï¼Œå³ä½¿ Grafana pod é‡å¯äº†ï¼ŒConfigMap 也能显示在 Grafana UI 上并æŒä¹…化。 + +无法在 Grafana UI 中删除或编辑使用 ConfigMap æŒä¹…化了的仪表æ¿ã€‚ + +如果你在 Grafana UI 中删除仪表æ¿ï¼Œä½ å°†çœ‹åˆ° "Dashboard cannot be deleted because it was provisioned" 的错误消æ¯ã€‚如需删除仪表æ¿ï¼Œä½ éœ€è¦åˆ é™¤ ConfigMap。 + +### 为 Grafana ä»ªè¡¨æ¿ ConfigMap é…置命å空间 + +è¦è®© Grafana 监控所有命å空间中的 ConfigMap，请在 `rancher-monitoring` Helm chart 中指定以下值： + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +请注æ„，Monitoring Chart 用于添加 Grafana 仪表æ¿çš„ RBAC 角色仅能让用户将仪表æ¿æ·»åŠ åˆ°å®šä¹‰åœ¨ `grafana.dashboards.namespace` 中的命å空间，默认为 `cattle-dashboards`。 + + + + +:::note 先决æ¡ä»¶ï¼š + +- 已安装 Monitoring 应用。 +- 你必须具有 cluster-admin ClusterRole æƒé™ã€‚ + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦åœ¨å…¶ä¸­é…ç½® Grafana 命å空间的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Grafana**。 +1. 登录到 Grafana。请注æ„，Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin/prom-operator`。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + :::note + + 无论è°æ‹¥æœ‰å¯†ç ï¼Œéƒ½éœ€è¦ Rancher 的集群管ç†å‘˜æƒé™æ‰èƒ½è®¿é—® Grafana 实例。 + + ::: + +1. 转到è¦è¿›è¡ŒæŒä¹…化的仪表æ¿ã€‚在顶部导航èœå•ä¸­ï¼Œé€šè¿‡å•å‡»é½¿è½®å›¾æ ‡è½¬åˆ°ä»ªè¡¨æ¿è®¾ç½®ã€‚ +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡» **JSON Model**。 +1. å¤åˆ¶å‡ºçŽ°çš„ JSON æ•°æ®ç»“构。 +1. 在 `cattle-dashboards` 命å空间中创建一个 ConfigMap。ConfigMap 需è¦æœ‰ `grafana_dashboard: "1"` 标签。将 JSON 粘贴到 ConfigMap 中，格å¼å¦‚下例所示： + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards + data: + .json: |- + + ``` + +**结果**：创建 ConfigMap åŽï¼Œå³ä½¿ Grafana pod é‡å¯äº†ï¼ŒConfigMap 也能显示在 Grafana UI 上并æŒä¹…化。 + +无法在 Grafana UI 中删除使用 ConfigMap æŒä¹…化了的仪表æ¿ã€‚如果你在 Grafana UI 中删除仪表æ¿ï¼Œä½ å°†çœ‹åˆ° "Dashboard cannot be deleted because it was provisioned" 的错误消æ¯ã€‚如需删除仪表æ¿ï¼Œä½ éœ€è¦åˆ é™¤ ConfigMap。 + +为防止在å¸è½½ Monitoring v2 时删除æŒä¹…化的仪表æ¿ï¼Œè¯·å°†ä»¥ä¸‹æ³¨é‡Šæ·»åŠ åˆ° `cattle-dashboards` 命å空间： + +``` +helm.sh/resource-policy: "keep" +``` + + + + +## 已知问题 + +如果你的 Monitoring V2 版本是 v9.4.203 或更低版本，å¸è½½ Monitoring chart å°†åŒæ—¶åˆ é™¤ `cattle-dashboards` 命å空间，所有æŒä¹…化的仪表æ¿å°†è¢«åˆ é™¤ï¼ˆé™¤éžå‘½å空间带有注释 `helm.sh/resource-policy: "keep"`）。 + +Rancher 2.5.8 å‘布的新 Monitoring Chart 中默认添加了该注解，但使用早期 Rancher 版本的用户ä»éœ€æ‰‹åŠ¨åº”用该注释。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md new file mode 100644 index 00000000000..5fa126a9089 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md @@ -0,0 +1,39 @@ +--- +title: 自定义 Grafana ä»ªè¡¨æ¿ +--- + +在本文中，你将学习通过自定义 Grafana 仪表æ¿æ¥æ˜¾ç¤ºç‰¹å®šå®¹å™¨çš„指标。 + +### 先决æ¡ä»¶ + +在自定义 Grafana 仪表æ¿ä¹‹å‰ï¼Œä½ å¿…须先安装 `rancher-monitoring` 应用。 + +è¦æŸ¥çœ‹æŒ‡å‘外部监控 UI（包括 Grafana 仪表æ¿ï¼‰çš„链接，你至少需è¦ä¸€ä¸ª [project-member 角色](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#具有-rancher-æƒé™çš„用户)。 + +### 登录 Grafana + +1. 在 Rancher UI 中，转到è¦è‡ªå®šä¹‰çš„仪表æ¿çš„集群。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Grafana**。Grafana 仪表æ¿å°†åœ¨æ–°é€‰é¡¹å¡ä¸­æ‰“开。 +1. 转到左下角的登录图标，然åŽå•å‡» **Sign In**。 +1. 登录到 Grafana。Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin/prom-operator`（无论è°æ‹¥æœ‰å¯†ç ï¼Œéƒ½éœ€è¦ Rancher 的集群管ç†å‘˜æƒé™æ‰èƒ½è®¿é—® Grafana 实例）。你还å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + + +### 获å–æ”¯æŒ Grafana é¢æ¿çš„ PromQL 查询 + +对于任何é¢æ¿ï¼Œä½ å¯ä»¥å•å‡»æ ‡é¢˜å¹¶å•å‡» **Explore** 以获å–支æŒå›¾å½¢çš„ PromQL 查询。 + +例如，如果è¦èŽ·å– Alertmanager 容器的 CPU 使用率，点击 **CPU Utilization > Inspect**。 + +**Data** 选项å¡å°†åŸºç¡€æ•°æ®æ˜¾ç¤ºä¸ºæ—¶é—´åºåˆ—，第一列是时间，第二列是 PromQL 查询结果。å¤åˆ¶ PromQL 查询。 + +``` +(1 - (avg(irate({__name__=~"node_cpu_seconds_total|windows_cpu_time_total",mode="idle"}[5m])))) * 100 +``` + +然åŽï¼Œä½ å¯ä»¥åœ¨ Grafana é¢æ¿ä¸­ä¿®æ”¹æŸ¥è¯¢ï¼Œæˆ–使用该查询创建新的 Grafana é¢æ¿ã€‚ + +å‚考： + +- [编辑é¢æ¿çš„ Grafana 文档](https://grafana.com/docs/grafana/latest/panels-visualizations/configure-panel-options/#edit-a-panel) +- [å‘仪表æ¿æ·»åŠ é¢æ¿çš„ Grafana 文档](https://grafana.com/docs/grafana/latest/panels-visualizations/panel-editor-overview) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md new file mode 100644 index 00000000000..a51a45041ec --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md @@ -0,0 +1,19 @@ +--- +title: è°ƒè¯•é«˜å†…å­˜ç”¨é‡ +--- + +Prometheus 中的æ¯ä¸ªæ—¶é—´åºåˆ—都由其[指标å称](https://prometheus.io/docs/practices/naming/#metric-names)和称为[标签](https://prometheus.io/docs/practices/naming/#labels)çš„å¯é€‰é”®å€¼å¯¹å”¯ä¸€æ ‡è¯†ã€‚ + +标签å…许过滤和èšåˆæ—¶é—´åºåˆ—æ•°æ®ï¼Œä½†å®ƒä»¬ä¹Ÿä½¿ Prometheus 收集的数æ®é‡æˆå€å¢žåŠ ã€‚ + +æ¯ä¸ªæ—¶é—´åºåˆ—都有一组定义的标签，Prometheus 为所有唯一的标签组åˆç”Ÿæˆä¸€ä¸ªæ–°çš„时间åºåˆ—。如果一个指标附加了两个标签，则会为该指标生æˆä¸¤ä¸ªæ—¶é—´åºåˆ—。更改任何标签值，包括添加或删除标签，都会创建一个新的时间åºåˆ—。 + +Prometheus ç»è¿‡äº†ä¼˜åŒ–，å¯ä»¥å­˜å‚¨åŸºäºŽç´¢å¼•çš„åºåˆ—æ•°æ®ã€‚它是为相对一致的时间åºåˆ—æ•°é‡å’Œç›¸å¯¹å¤§é‡çš„样本而设计的，这些样本需è¦éšæ—¶é—´ä»Ž exporter 处收集。 + +但是，Prometheus 没有就快速å˜åŒ–的时间åºåˆ—æ•°é‡è¿›è¡Œå¯¹åº”的优化。因此，如果你在创建和销æ¯äº†å¤§é‡èµ„æºçš„集群（尤其是多租户集群）上安装 Monitoring，å¯èƒ½ä¼šå‡ºçŽ°å†…存使用é‡æ¿€å¢žçš„情况。 + +### å‡å°‘内存激增 + +为了å‡å°‘内存消耗，Prometheus å¯ä»¥é€šè¿‡æŠ“å–更少的指标或在时间åºåˆ—上添加更少的标签，从而存储更少的时间åºåˆ—。è¦æŸ¥çœ‹ä½¿ç”¨å†…存最多的åºåˆ—，你å¯ä»¥æŸ¥çœ‹ Prometheus UI 中的 TSDB（时åºæ•°æ®åº“）状æ€é¡µé¢ã€‚ + +åˆ†å¸ƒå¼ Prometheus 解决方案（例如 [Thanos](https://thanos.io/) å’Œ [Cortex](https://cortexmetrics.io/)）使用了一个替代架构，该架构部署多个å°åž‹ Prometheus 实例。如果使用 Thanos，æ¯ä¸ª Prometheus 的指标都èšåˆåˆ°é€šç”¨çš„ Thanos 部署中，然åŽå†å°†è¿™äº›æŒ‡æ ‡å¯¼å‡ºåˆ° S3 之类的æŒä¹…存储。这ç§æž¶æž„更加å¥åº·ï¼Œèƒ½é¿å…ç»™å•ä¸ª Prometheus 实例带æ¥è¿‡å¤šæ—¶é—´åºåˆ—，åŒæ—¶è¿˜èƒ½åœ¨å…¨å±€çº§åˆ«æŸ¥è¯¢æŒ‡æ ‡ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md new file mode 100644 index 00000000000..cfb2ebdf338 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md @@ -0,0 +1,75 @@ +--- +title: å¯ç”¨ Monitoring +--- + +[管ç†å‘˜](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)或[集群所有者](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥éƒ¨ç½² Prometheus，从而监控 Kubernetes 集群。 + +本文介ç»å¦‚何使用新的 monitoring 应用在集群内å¯ç”¨ç›‘控和告警。 + +ä¸è®ºæ˜¯å¦ä½¿ç”¨ SSL，你都å¯ä»¥å¯ç”¨ monitoring。 + +## è¦æ±‚ + +- 在æ¯ä¸ªèŠ‚点上å…è®¸ç«¯å£ 9796 上的æµé‡ã€‚Prometheus 将从这些端å£æŠ“å–指标。 + - 如果 [PushProx](../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#pushprox) 被ç¦ç”¨ï¼ˆ`ingressNginx.enabled` 设置为 `false`），或者你已ç»å‡çº§äº†å®‰è£…了 Monitoring V1 çš„ Rancher 版本，你å¯èƒ½è¿˜éœ€è¦ä¸ºæ¯ä¸ªèŠ‚点å…è®¸ç«¯å£ 10254 上的æµé‡ã€‚ +- ç¡®ä¿ä½ çš„集群满足资æºè¦æ±‚。集群应至少有 1950Mi å¯ç”¨å†…å­˜ã€2700m CPU å’Œ 50Gi 存储。有关资æºé™åˆ¶å’Œè¯·æ±‚的详细信æ¯ï¼Œè¯·å‚阅[é…置资æºé™åˆ¶å’Œè¯·æ±‚](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#é…置资æºé™åˆ¶å’Œè¯·æ±‚)。 +- 在使用 RancherOS 或 Flatcar Linux 节点的 RKE 集群上安装 Monitoring 时，请将 etcd 节点è¯ä¹¦ç›®å½•æ›´æ”¹ä¸º `/opt/rke/etc/kubernetes/ssl`。 +- 如果集群是使用 RKE CLI é…置的，而且地å€è®¾ç½®ä¸ºä¸»æœºå而ä¸æ˜¯ IP 地å€ï¼Œè¯·åœ¨å®‰è£…çš„ Values é…置步骤中将 `rkeEtcd.clients.useLocalhost` 设置为 `true`。例如： + +```yaml +rkeEtcd: + clients: + useLocalhost: true +``` + +:::note + +如果è¦è®¾ç½® Alertmanagerã€Grafana 或 Ingress，必须通过 Helm chart 部署上的设置æ¥å®Œæˆã€‚在部署之外创建 Ingress å¯èƒ½ä¼šäº§ç”Ÿé—®é¢˜ã€‚ + +::: + +## 设置资æºé™åˆ¶å’Œè¯·æ±‚ + +安装 `rancher-monitoring` æ—¶å¯ä»¥é…置资æºè¯·æ±‚å’Œé™åˆ¶ã€‚è¦ä»Ž Rancher UI é…ç½® Prometheus 资æºï¼Œè¯·å•å‡»å·¦ä¸Šè§’çš„ **Apps > Monitoring**。 + +有关默认é™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#é…置资æºé™åˆ¶å’Œè¯·æ±‚)。 + +## 安装 Monitoring 应用 + +### 在ä¸ä½¿ç”¨ SSL 的情况下å¯ç”¨ Monitoring + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 点击**集群工具**（左下角）。 +1. å•å‡» Monitoring çš„**安装**。 +1. å¯é€‰ï¼šåœ¨ Values 步骤中为 Alertingã€Prometheus å’Œ Grafana 自定义请求ã€é™åˆ¶ç­‰ã€‚如需帮助，请å‚阅[é…ç½®å‚考](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md)。 + +**结果**：Monitoring 应用已部署到 `cattle-monitoring-system` 命å空间中。 + +### 在使用 SSL 的情况下å¯ç”¨ Monitoring + +1. 按照[此页é¢](../../new-user-guides/kubernetes-resources-setup/secrets.md)上的步骤创建密文，以便将 SSL 用于告警。 +- 密文应该创建到 `cattle-monitoring-system` 命å空间中。如果它ä¸å­˜åœ¨ï¼Œè¯·å…ˆåˆ›å»ºå®ƒã€‚ +- å°† `ca`ã€`cert` å’Œ `key` 文件添加到密文中。 +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯ç”¨ Monitoring 以与 SSL 一起使用的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **Monitoring**。 +1. æ ¹æ®ä½ æ˜¯å¦å·²å®‰è£… Monitoring，å•å‡»**安装**或**æ›´æ–°**。 +1. 选中**在安装å‰è‡ªå®šä¹‰ Helm 选项**，然åŽå•å‡»**下一步**。 +1. å•å‡» **Alerting**。 +1. 在**补充密文**字段中，添加之å‰åˆ›å»ºçš„密文。 + +**结果**：Monitoring 应用已部署到 `cattle-monitoring-system` 命å空间中。 + +[创建接收器](../../../reference-guides/monitoring-v2-configuration/receivers.md#在-rancher-ui-中创建接收器)时，å¯ç”¨ SSL 的接收器（例如电å­é‚®ä»¶æˆ– webhook）将具有 **SSL**ï¼Œå…¶ä¸­åŒ…å« **CA 文件路径**ã€**è¯ä¹¦æ–‡ä»¶è·¯å¾„**å’Œ**密钥文件路径**字段。使用 `ca`ã€`cert` å’Œ `key` 的路径填写这些字段。路径的格å¼ä¸º `/etc/alertmanager/secrets/name-of-file-in-secret`。 + +例如，如果你使用以下键值对创建了一个密文： + +```yaml +ca.crt=`base64-content` +cert.pem=`base64-content` +key.pfx=`base64-content` +``` + +则**è¯ä¹¦æ–‡ä»¶è·¯å¾„**需è¦è®¾ä¸º `/etc/alertmanager/secrets/cert.pem`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md new file mode 100644 index 00000000000..44d69d05f9e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md @@ -0,0 +1,7 @@ +--- +title: 自定义 Grafana ä»ªè¡¨æ¿ +--- + +无论是用于 rancher-monitoring 还是 Prometheus Federator，Grafana 仪表æ¿çš„定制方å¼éƒ½æ˜¯ç›¸åŒçš„。 + +有关说明，请å‚阅[此页é¢](../customize-grafana-dashboard.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md new file mode 100644 index 00000000000..bf3b78cac72 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md @@ -0,0 +1,86 @@ +--- +title: å¯ç”¨ Prometheus Federator +--- + +## è¦æ±‚ + +默认情况下，Prometheus Federator å·²é…置并旨在与 [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) 一起部署。rancher-monitoring åŒæ—¶éƒ¨ç½²äº† Prometheus Operator å’Œ Cluster Prometheus，æ¯ä¸ªé¡¹ç›®ç›‘控堆栈（Project Monitoring Stack）默认会è”åˆå‘½å空间范围的指标。 + +有关安装 rancher-monitoring 的说明，请å‚阅[此页é¢](../enable-monitoring.md)。 + +默认é…置与你的 rancher-monitoring 堆栈是兼容的。但是，为了æ高集群中 Prometheus Federator 的安全性和å¯ç”¨æ€§ï¼Œæˆ‘们建议对 rancher-monitoring 进行以下é¢å¤–çš„é…置： + +- [ç¡®ä¿ cattle-monitoring-system 命å空间ä½äºŽ System 项目中](#ç¡®ä¿-cattle-monitoring-system-命å空间ä½äºŽ-system-项目中或者ä½äºŽä¸€ä¸ªé”定并能访问集群中其他项目的项目中) +- [å°† rancher-monitoring é…置为仅监视 Helm Chart 创建的资æº](#å°†-rancher-monitoring-é…置为仅监视-helm-chart-创建的资æº) +- [æ高 Cluster Prometheus çš„ CPU/内存é™åˆ¶](#æ高-cluster-prometheus-çš„-cpu内存é™åˆ¶) + +### ç¡®ä¿ cattle-monitoring-system 命å空间ä½äºŽ System 项目中（或者ä½äºŽä¸€ä¸ªé”定并能访问集群中其他项目的项目中） + +![选择项目/命å空间](/img/install-in-system-project.png) + +Prometheus Operator 的安全模型有一定的è¦æ±‚，å³å¸Œæœ›éƒ¨ç½²å®ƒçš„命å空间（例如，`cattle-monitoring-system`）对除集群管ç†å‘˜ä¹‹å¤–的任何用户都åªæœ‰æœ‰é™çš„访问æƒé™ï¼Œä»Žè€Œé¿å…通过 Pod 内执行（例如正在执行的 Helm æ“作的 Job）æ¥æå‡æƒé™ã€‚此外，如果将 Prometheus Federator 和所有 Project Prometheus 堆栈都部署到 System 项目中，å³ä½¿ç½‘络策略是通过项目网络隔离定义的，æ¯ä¸ª Project Prometheus 都ä¾ç„¶èƒ½å¤Ÿåœ¨æ‰€æœ‰é¡¹ç›®ä¸­æŠ“å–工作负载。它还为项目所有者ã€é¡¹ç›®æˆå‘˜å’Œå…¶ä»–用户æ供有é™çš„访问æƒé™ï¼Œä»Žè€Œç¡®ä¿è¿™äº›ç”¨æˆ·æ— æ³•è®¿é—®ä»–们ä¸åº”访问的数æ®ï¼ˆä¾‹å¦‚，在 pod 中执行，在项目外部抓å–命å空间数æ®ç­‰ï¼‰ã€‚ + +1. 打开 `System` 项目以检查你的命å空间： + + 在 Rancher UI 中å•å‡» **Cluster > Projects/Namespaces**。这将显示 `System` 项目中的所有命å空间： + + ![选择项目/命å空间](/img/cattle-monitoring-system.png) + +1. 如果你在 `cattle-monitoring-system` 命å空间中已ç»å®‰è£…了一个 Monitoring V2，但该命å空间ä¸åœ¨ `System` 项目中，你å¯ä»¥å°† `cattle- monitoring-system` 命å空间移动到 `System` 项目或å¦ä¸€ä¸ªè®¿é—®å—é™çš„项目中。为此，你有以下两ç§æ–¹æ³•ï¼š + + - 将命å空间拖放到 `System` 项目。 + - 选择命å空间å³ä¾§çš„ **â‹®**，点击 **Move**，然åŽä»Ž **Target Project** 下拉列表中选择 `System`： + + ![移至新项目](/img/move-to-new-project.png) + +### å°† rancher-monitoring é…置为仅监视 Helm Chart åˆ›å»ºçš„èµ„æº + +æ¯ä¸ªé¡¹ç›®ç›‘控堆栈都会监视其他命å空间并收集其他自定义工作负载指标或仪表æ¿ã€‚因此，我们建议在所有选择器上é…ç½®ä»¥ä¸‹è®¾ç½®ï¼Œä»¥ç¡®ä¿ Cluster Prometheus 堆栈仅监控由 Helm Chart 创建的资æºï¼š + +``` +matchLabels: + release: "rancher-monitoring" +``` + +建议为以下选择器字段赋予此值： +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +å¯ç”¨æ­¤è®¾ç½®åŽï¼Œä½ å§‹ç»ˆå¯ä»¥é€šè¿‡å‘它们添加 `release: "rancher-monitoring"` 标签æ¥åˆ›å»ºç”± Cluster Prometheus 抓å–çš„ ServiceMonitor 或 PodMonitor。在这ç§æƒ…况下，å³ä½¿è¿™äº› ServiceMonitor 或 PodMonitor 所在的命å空间ä¸æ˜¯ System 命å空间，项目监控堆栈也会默认自动忽略它们。 + +:::note + +如果你ä¸å¸Œæœ›ç”¨æˆ·èƒ½å¤Ÿåœ¨ Project 命å空间中创建èšåˆåˆ° Cluster Prometheus 中的 ServiceMonitor å’Œ PodMonitor，你å¯ä»¥å¦å¤–å°† Chart 上的 namespaceSelectors 设置为仅目标 System 命åç©ºé—´ï¼ˆå¿…é¡»åŒ…å« `cattle-monitoring-system` å’Œ `cattle-dashboards`，默认情况下资æºé€šè¿‡ rancher-monitoring 部署到该命å空间中。你还需è¦ç›‘控 `default` 命åç©ºé—´ä»¥èŽ·å– apiserver 指标，或创建自定义 ServiceMonitor 以抓å–ä½äºŽ `default` 命å空间中的 Service çš„ apiserver 指标），从而é™åˆ¶ä½ çš„ Cluster Prometheus 获å–其他 Prometheus Operator CR。在这ç§æƒ…况下，建议设置 `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。这样，你å¯ä»¥å®šä¹‰èƒ½ä»Ž System 命åç©ºé—´ä¸­ç›‘è§†éž System 命å空间的 ServiceMonitor。 + +::: + +### æ高 Cluster Prometheus çš„ CPU/内存é™åˆ¶ + +æ ¹æ®é›†ç¾¤çš„设置，我们一般建议为 Cluster Prometheus é…置大é‡çš„专用内存，以é¿å…因内存ä¸è¶³çš„错误（OOMKilled）而é‡å¯ã€‚通常情况下，集群中创建的改动项（churn）会导致大é‡é«˜åŸºæ•°æŒ‡æ ‡åœ¨ä¸€ä¸ªæ—¶é—´å—内生æˆå¹¶è¢« Prometheus 引入，然åŽå¯¼è‡´è¿™äº›é”™è¯¯ã€‚这也是为什么默认的 Rancher Monitoring 堆栈希望能分é…到大约 4GB çš„ RAM 以在正常大å°çš„集群中è¿è¡Œçš„原因之一。但是，如果你引入å‘åŒä¸€ä¸ª Cluster Prometheus å‘é€ `/federate` 请求的项目监控堆栈，并且项目监控堆栈ä¾èµ–于 Cluster Prometheus çš„å¯åŠ¨çŠ¶æ€æ¥åœ¨å…¶å‘½å空间上è”åˆç³»ç»Ÿæ•°æ®ï¼Œé‚£ä¹ˆä½ æ›´åŠ éœ€è¦ä¸º Cluster Prometheus 分é…足够的 CPU/内存，以防止集群中的所有 Prometheus 项目出现数æ®é—´éš™çš„中断。 + +:::note + +我们没有 Cluster Prometheus 内存é…置的具体建议，因为这完全å–决于用户的设置（å³é‡åˆ°é«˜æ”¹åŠ¨çŽ‡çš„å¯èƒ½æ€§ä»¥åŠå¯èƒ½åŒæ—¶ç”Ÿæˆçš„指标的规模）。ä¸åŒçš„设置通常有ä¸åŒçš„推èå‚数。 + +::: + +## 安装 Prometheus Federator åº”ç”¨ç¨‹åº + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å®‰è£… Prometheus Federator 的集群，然åŽå•å‡» **Explore**。 +1. 点击**应用 > Charts**。 +1. å•å‡» **Prometheus Federator** Chart。 +1. å•å‡»**安装**。 +1. 在**元数æ®**页é¢ï¼Œç‚¹å‡»**下一步**。 +1. 在**项目 Release 命å空间项目 ID** 字段中，`System 项目`是默认值，但你å¯ä»¥ä½¿ç”¨å…·æœ‰ç±»ä¼¼[有é™è®¿é—®æƒé™](#ç¡®ä¿-cattle-monitoring-system-命å空间ä½äºŽ-system-项目中或者ä½äºŽä¸€ä¸ªé”定并能访问集群中其他项目的项目中)çš„å¦ä¸€ä¸ªé¡¹ç›®è¦†ç›–它。你å¯ä»¥åœ¨ local 上游集群中è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ‰¾åˆ°é¡¹ç›® ID： + +```plain +kubectl get projects -A -o custom-columns="NAMESPACE":.metadata.namespace,"ID":.metadata.name,"NAME":.spec.displayName +``` + +1. å•å‡»**安装**。 + +**结果**：Prometheus Federator 应用程åºå·²éƒ¨ç½²åœ¨ `cattle-monitoring-system` 命å空间中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md new file mode 100644 index 00000000000..9dba3d6d401 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md @@ -0,0 +1,17 @@ +--- +title: 安装 Project Monitor +--- + +在è¦å¯ç”¨é¡¹ç›®ç›‘控的å„个项目中安装 **Project Monitor**。 + +1. 点击 **☰ > 集群管ç†**。 + +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦å¯ç”¨ç›‘控的集群，然åŽå•å‡» **Explore**。 + +1. å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„**监控 > Project Monitor**。然åŽï¼Œç‚¹å‡»å³ä¸Šè§’çš„**创建**。 + + ![Project Monitor](/img/project-monitors.png) + +1. 从下拉èœå•ä¸­é€‰æ‹©ä½ çš„项目，然åŽå†æ¬¡å•å‡»**创建**。 + + ![创建 Project Monitor](/img/create-project-monitors.png) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md new file mode 100644 index 00000000000..10aaf5d95b7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md @@ -0,0 +1,13 @@ +--- +title: 为工作负载设置 Prometheus Federator +--- + +### 显示工作负载的 CPU 和内存指标 + +使用 Prometheus Federator 显示 CPU 和内存指标的方å¼ä¸Žä½¿ç”¨ rancher-monitoring 相åŒã€‚有关说明，请å‚阅[此处](../set-up-monitoring-for-workloads.md#显示工作负载的-cpu-和内存指标)。 + +### 设置 CPU 和内存之外的指标 + +使用 Prometheus Federator 设置 CPU 和内存之外的指标与使用 rancher-monitoring çš„æ–¹å¼ç›¸åŒã€‚有关说明，请å‚阅[此处](../set-up-monitoring-for-workloads.md#设置-cpu-和内存之外的指标)。 + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md new file mode 100644 index 00000000000..c6b04a736bb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md @@ -0,0 +1,13 @@ +--- +title: å¸è½½ Prometheus Federator +--- + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `cattle-monitoring-system` 命å空间并选中 `rancher-monitoring-crd` å’Œ `rancher-monitoring`。 +1. å•å‡»**删除**。 +1. 确认**删除**。 + +**结果**：已å¸è½½ `prometheus-federator`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md new file mode 100644 index 00000000000..9b8fd1582c8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md @@ -0,0 +1,27 @@ +--- +title: 为工作负载设置 Monitoring +--- + +如果你åªéœ€è¦å·¥ä½œè´Ÿè½½çš„ CPU 和内存时间åºåˆ—，则ä¸éœ€è¦éƒ¨ç½² ServiceMonitor 或 PodMonitor，因为 Monitoring 应用默认会收集资æºä½¿ç”¨æƒ…况的指标数æ®ã€‚ + +设置工作负载监控的步骤å–决于你是å¦éœ€è¦åŸºæœ¬æŒ‡æ ‡ï¼ˆä¾‹å¦‚工作负载的 CPU 和内存），或者是å¦éœ€è¦ä»Žå·¥ä½œè´Ÿè½½ä¸­æŠ“å–自定义指标。 + +如果你åªéœ€è¦å·¥ä½œè´Ÿè½½çš„ CPU 和内存时间åºåˆ—，则ä¸éœ€è¦éƒ¨ç½² ServiceMonitor 或 PodMonitor，因为 Monitoring 应用默认会收集资æºä½¿ç”¨æƒ…况的指标数æ®ã€‚资æºä½¿ç”¨çš„时间åºåˆ—æ•°æ®åœ¨ Prometheus 的本地时间åºåˆ—æ•°æ®åº“中。 + +Grafana 显示èšåˆæ•°æ®ï¼Œä½ ä¹Ÿå¯ä»¥ä½¿ç”¨ PromQL 查询æ¥æŸ¥çœ‹å•ä¸ªå·¥ä½œè´Ÿè½½çš„æ•°æ®ã€‚进行 PromQL 查询åŽï¼Œä½ å¯ä»¥åœ¨ Prometheus UI 中å•ç‹¬æ‰§è¡ŒæŸ¥è¯¢å¹¶æŸ¥çœ‹å¯è§†åŒ–的时间åºåˆ—，你也å¯ä»¥ä½¿ç”¨æŸ¥è¯¢æ¥è‡ªå®šä¹‰æ˜¾ç¤ºå·¥ä½œè´Ÿè½½æŒ‡æ ‡çš„ Grafana 仪表æ¿ã€‚有关工作负载指标的 PromQL 查询示例，请å‚阅[本节](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/expression/#workload-metrics)。 + +è¦ä¸ºä½ çš„工作负载设置自定义指标，你需è¦è®¾ç½®ä¸€ä¸ª Exporter 并创建一个新的 ServiceMonitor 自定义资æºï¼Œä»Žè€Œå°† Prometheus é…置为从 Exporter 中抓å–指标。 + +### 显示工作负载的 CPU 和内存指标 + +默认情况下，Monitoring åº”ç”¨ä¼šæŠ“å– CPU 和内存指标。 + +è¦èŽ·å–特定工作负载的细粒度信æ¯ï¼Œä½ å¯ä»¥è‡ªå®šä¹‰ Grafana 仪表æ¿æ¥æ˜¾ç¤ºè¯¥å·¥ä½œè´Ÿè½½çš„指标。 + +### 设置 CPU 和内存之外的指标 + +对于自定义指标，你需è¦ä½¿ç”¨ Prometheus 支æŒçš„æ ¼å¼æ¥å…¬å¼€åº”用上的指标。 + +我们建议你创建一个新的 ServiceMonitor 自定义资æºã€‚创建此资æºæ—¶ï¼ŒPrometheus 自定义资æºå°†è‡ªåŠ¨æ›´æ–°ï¼Œä»¥ä¾¿å°†æ–°çš„自定义指标端点包å«åœ¨æŠ“å–é…ç½®ä¸­ã€‚ç„¶åŽ Prometheus 会开始从端点抓å–指标。 + +你还å¯ä»¥åˆ›å»º PodMonitor æ¥å…¬å¼€è‡ªå®šä¹‰æŒ‡æ ‡ç«¯ç‚¹ï¼Œä½† ServiceMonitor 更适åˆå¤§å¤šæ•°ç”¨ä¾‹ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md new file mode 100644 index 00000000000..06845311909 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md @@ -0,0 +1,19 @@ +--- +title: å¸è½½ Monitoring +--- + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `cattle-monitoring-system` 命å空间并选中 `rancher-monitoring-crd` å’Œ `rancher-monitoring`。 +1. å•å‡»**删除**。 +1. 确认**删除**。 + +**结果**：已å¸è½½ `rancher-monitoring`。 + +:::note æŒä¹…化 Grafana 仪表æ¿ï¼š + +如果你的 Monitoring V2 版本是 v9.4.203 或更低版本，å¸è½½ Monitoring chart å°†åŒæ—¶åˆ é™¤ cattle-dashboards 命å空间，所有æŒä¹…化的仪表æ¿å°†è¢«åˆ é™¤ï¼ˆé™¤éžå‘½å空间带有注释 `helm.sh/resource-policy: "keep"`）。Monitoring V2 v14.5.100+ 会默认添加此注释。但如果你的集群上安装了旧版本的 Monitoring Chart，你å¯ä»¥åœ¨å¸è½½å®ƒä¹‹å‰æ‰‹åŠ¨å°†æ³¨é‡Šåº”用到 cattle-dashboards 命å空间。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md new file mode 100644 index 00000000000..77851a3accb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md @@ -0,0 +1,43 @@ +--- +title: Alertmanager é…ç½® +--- + +通常情况下，你ä¸éœ€è¦ç›´æŽ¥ç¼–辑 Alertmanager 自定义资æºã€‚对于大多数用例，åªéœ€è¦ç¼–辑接收器和路由å³å¯é…置通知。 + +当路由和接收器更新时，Monitoring 应用将自动更新 Alertmanager 自定义资æºæ¥ä¸Žè¿™äº›æ›´æ”¹ä¿æŒä¸€è‡´ã€‚ + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-alertmanager-的工作原ç†)。 + +::: + +## 关于 Alertmanager è‡ªå®šä¹‰èµ„æº + +默认情况下，Rancher Monitoring å°†å•ä¸ª Alertmanager 部署到使用默认 Alertmanager Config Secret 的集群上。 + +如果你想使用 Rancher UI 表å•ä¸­æœªå…¬å¼€çš„高级选项（例如创建超过两层深的路由树结构），你å¯èƒ½éœ€è¦ç¼–辑 Alertmanager 自定义资æºã€‚ + +你也å¯ä»¥åœ¨é›†ç¾¤ä¸­åˆ›å»ºå¤šä¸ª Alertmanager æ¥å®žçŽ°å‘½å空间范围的监控。在这ç§æƒ…况下，你应该使用相åŒçš„底层 Alertmanager Config Secret æ¥ç®¡ç† Alertmanager 自定义资æºã€‚ + +### 深度嵌套的路由 + +虽然 Rancher UI 仅支æŒä¸¤å±‚深度的路由树，但你å¯ä»¥é€šè¿‡ç¼–辑 Alertmanager YAML æ¥é…置更深的嵌套路由结构。 + +### 多个 Alertmanager 副本 + +作为 Chart 部署选项的一部分，你å¯ä»¥é€‰æ‹©å¢žåŠ éƒ¨ç½²åˆ°é›†ç¾¤ä¸Šçš„ Alertmanager 副本的数é‡ã€‚这些副本使用相åŒçš„底层 Alertmanager Config Secret 进行管ç†ã€‚ + +æ­¤ Secret å¯ä»¥æŒ‰ç…§ä½ çš„需求éšæ—¶æ›´æ–°æˆ–修改： + +- 添加新的通知程åºæˆ–接收器 +- 更改应该å‘é€ç»™æŒ‡å®šé€šçŸ¥ç¨‹åºæˆ–接收器的告警 +- 更改å‘出的告警组 + +默认情况下，你å¯ä»¥é€‰æ‹©æ供现有的 Alertmanager Config Secretï¼ˆå³ `cattle-monitoring-system` 命å空间中的任何 Secret），或å…许 Rancher Monitoring 将默认的 Alertmanager Config Secret 部署到你的集群上。 + +默认情况下，在å‡çº§æˆ–å¸è½½ `rancher-monitoring` Chart 时，Rancher 创建的 Alertmanager Config Secret ä¸ä¼šè¢«ä¿®æ”¹æˆ–删除。这个é™åˆ¶å¯ä»¥é˜²æ­¢ç”¨æˆ·åœ¨ Chart 上执行æ“作时丢失或覆盖他们的告警é…置。 + +有关å¯ä»¥åœ¨ Alertmanager Config Secret 中指定的字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ [Prometheus Alertmanager 文档](https://prometheus.io/docs/alerting/latest/alertmanager/)。 + +ä½ å¯ä»¥åœ¨[此处](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file)找到 Alertmanager é…置文件的完整规范åŠå…¶å†…容。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md new file mode 100644 index 00000000000..b83ad90b748 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md @@ -0,0 +1,19 @@ +--- +title: Prometheus é…ç½® +--- + +通常情况下，你ä¸éœ€è¦ç›´æŽ¥ç¼–辑 Prometheus 自定义资æºï¼Œå› ä¸º Monitoring åº”ç”¨ä¼šæ ¹æ® ServiceMonitor å’Œ PodMonitor 的更改自动更新资æºã€‚ + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + +## 关于 Prometheus è‡ªå®šä¹‰èµ„æº + +Prometheus CR 定义了所需的 Prometheus deployment。Prometheus Operator 会观察 Prometheus CR。当 CR å‘生å˜åŒ–时，Prometheus Operator 会创建 `prometheus-rancher-monitoring-prometheus`，å³æ ¹æ® CR é…置的 Prometheus deployment。 + +Prometheus CR 指定了详细信æ¯ï¼Œä¾‹å¦‚规则以åŠè¿žæŽ¥åˆ° Prometheus çš„ Alertmanager。Rancher 会为你构建这个 CR。 + +Monitoring V2 仅支æŒæ¯ä¸ªé›†ç¾¤ä¸€ä¸ª Prometheus。如果你想将监控é™åˆ¶åˆ°æŒ‡å®šå‘½å空间，你需è¦ç¼–辑 Prometheus CR。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md new file mode 100644 index 00000000000..fc12de7e488 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md @@ -0,0 +1,80 @@ +--- +title: é…ç½® PrometheusRule +--- + +PrometheusRule 定义了一组 Prometheus 告警和/或记录规则。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + +### 在 Rancher UI 中创建 PrometheusRule + +:::note 先决æ¡ä»¶ï¼š + +已安装 Monitoring 应用。 + +::: + +è¦åœ¨ Rancher UI 中创建规则组： + +1. 转到è¦åˆ›å»ºè§„则组的集群。å•å‡»**监控 > 高级选项**，然åŽå•å‡» **PrometheusRules**。 +1. å•å‡»**创建**。 +1. 输入**组å称**。 +1. é…置规则。在 Rancher çš„ UI 中，规则组需è¦åŒ…å«å‘Šè­¦è§„则或记录规则，但ä¸èƒ½åŒæ—¶åŒ…å«ä¸¤è€…。如需获å–填写表å•çš„帮助，请å‚阅下方的é…置选项。 +1. å•å‡»**创建**。 + +**结果**：告警å¯ä»¥å‘接收器å‘é€é€šçŸ¥ã€‚ + +### 关于 PrometheusRule è‡ªå®šä¹‰èµ„æº + +当你定义规则时（在 PrometheusRule 资æºçš„ RuleGroup 中声明），[规则本身的规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule)会包å«æ ‡ç­¾ï¼Œç„¶åŽ Alertmanager 会使用这些标签æ¥ç¡®å®šæŽ¥æ”¶æ­¤å‘Šè­¦çš„路由。例如，标签为 `team: front-end` 的告警将​​å‘é€åˆ°ä¸Žè¯¥æ ‡ç­¾åŒ¹é…的所有路由。 + +Prometheus 规则文件ä¿å­˜åœ¨ PrometheusRule 自定义资æºä¸­ã€‚PrometheusRule 支æŒå®šä¹‰ä¸€ä¸ªæˆ–多个 RuleGroup。æ¯ä¸ª RuleGroup 由一组 Rule 对象组æˆï¼Œæ¯ä¸ª Rule 对象å‡èƒ½è¡¨ç¤ºå‘Šè­¦æˆ–记录规则，并具有以下字段： + +- 新告警或记录的å称 +- 新告警或记录的 PromQL è¡¨è¾¾å¼ +- 用于标记告警或记录的标签（例如集群å称或严é‡æ€§ï¼‰ +- 对需è¦åœ¨å‘Šè­¦é€šçŸ¥ä¸Šæ˜¾ç¤ºçš„其他é‡è¦ä¿¡æ¯è¿›è¡Œç¼–ç çš„注释（例如摘è¦ã€æè¿°ã€æ¶ˆæ¯ã€Runbook URL 等）。记录规则ä¸éœ€è¦æ­¤å­—段。 + +有关å¯ä»¥æŒ‡å®šçš„字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ [Prometheus Operator 规范。](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#prometheusrulespec) + +ä½ å¯ä»¥ä½¿ç”¨ Prometheus 对象中的标签选择器字段 `ruleSelector` æ¥å®šä¹‰è¦æŒ‚载到 Prometheus 的规则文件。 + +如需查看示例，请å‚阅 Prometheus 文档中的[记录规则](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)å’Œ[告警规则](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)。 + +## é…ç½® + +### 规则组 + +| 字段 | æè¿° | +|-------|----------------| +| 组å称 | 组的å称。在规则文件中必须是唯一的。 | +| 覆盖组间隔 | 组中规则的评估时间间隔（å•ä½ï¼šç§’）。 | + + +### 告警规则 + +[告警规则](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/)å¯ä»¥è®©ä½ æ ¹æ® PromQL（Prometheus 查询语言）表达å¼æ¥å®šä¹‰å‘Šè­¦æ¡ä»¶ï¼Œå¹¶å°†è§¦å‘告警的通知å‘é€åˆ°å¤–部æœåŠ¡ã€‚ + +| 字段 | æè¿° | +|-------|----------------| +| å‘Šè­¦å称 | 告警的å称。必须是有效的标签值。 | +| 告警触å‘等待时间 | 时长，以秒为å•ä½ã€‚当告警触å‘时间到达该指定时长时，则视为触å‘。当告警未触å‘足够长的时间，则视为待处ç†ã€‚ | +| PromQL è¡¨è¾¾å¼ | è¦è¯„ä¼°çš„ PromQL 表达å¼ã€‚Prometheus 将在æ¯ä¸ªè¯„估周期评估此 PromQL 表达å¼çš„当å‰å€¼ï¼Œå¹¶ä¸”所有生æˆçš„时间åºåˆ—都将æˆä¸ºå¾…处ç†/触å‘告警。有关详细信æ¯ï¼Œè¯·å‚阅 [Prometheus 文档](https://prometheus.io/docs/prometheus/latest/querying/basics/)或我们的 [PromQL 表达å¼ç¤ºä¾‹](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md)。 | +| Labels | 为æ¯ä¸ªå‘Šè­¦æ·»åŠ æˆ–覆盖的标签。 | +| 严é‡ç¨‹åº¦ | å¯ç”¨åŽï¼Œæ ‡ç­¾â€‹â€‹ä¼šé™„加到告警或记录中，这些标签通过严é‡ç¨‹åº¦æ¥æ ‡è¯†å‘Šè­¦/记录。 | +| 严é‡ç¨‹åº¦ Label 值 | Critical，warning 或 none | +| 注释 | 注释是一组信æ¯æ ‡ç­¾ï¼Œå¯ç”¨äºŽå­˜å‚¨æ›´é•¿çš„附加信æ¯ï¼Œä¾‹å¦‚å‘Šè­¦æ述或 Runbook 链接。[Runbook](https://en.wikipedia.org/wiki/Runbook) 是一组有关如何处ç†å‘Šè­¦çš„文档。注释值å¯ä»¥æ˜¯[模æ¿åŒ–](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#templating)的。 | + +### 记录规则 + +[记录规则](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules)å…许你预先计算常用或计算é‡å¤§çš„ PromQL（Prometheus 查询语言）表达å¼ï¼Œå¹¶å°†å…¶ç»“æžœä¿å­˜ä¸ºä¸€ç»„新的时间åºåˆ—。 + +| 字段 | æè¿° | +|-------|----------------| +| 时间åºåˆ—å称 | è¦è¾“出的时间åºåˆ—çš„å称。必须是有效的指标å称。 | +| PromQL è¡¨è¾¾å¼ | è¦è¯„ä¼°çš„ PromQL 表达å¼ã€‚Prometheus 将在æ¯ä¸ªè¯„估周期评估此 PromQL 表达å¼çš„当å‰å€¼ï¼Œå¹¶ä¸”将结果记录为一组新的时间åºåˆ—，其指标å称由“记录â€æŒ‡å®šã€‚有关表达å¼çš„更多信æ¯ï¼Œè¯·å‚阅 [Prometheus 文档](https://prometheus.io/docs/prometheus/latest/querying/basics/)或我们的 [PromQL 表达å¼ç¤ºä¾‹](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md)。 | +| Labels | 在存储结果之å‰è¦æ·»åŠ æˆ–覆盖的标签。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md new file mode 100644 index 00000000000..7e43c7234b2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md @@ -0,0 +1,107 @@ +--- +title: 使用 firewalld æ‰“å¼€ç«¯å£ +--- + +> 我们建议ç¦ç”¨ firewalld。如果你使用的是 Kubernetes 1.19 或更高版本，则必须关闭 firewalld。 + +æŸäº› [æºè‡ª RHEL](https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux#Rebuilds) çš„ Linux å‘行版（包括 Oracle Linux）的默认防ç«å¢™è§„则å¯èƒ½ä¼šé˜»æ­¢ä¸Ž Helm 的通信。 + +例如，AWS 中的一个 Oracle Linux é•œåƒå…·æœ‰ REJECT 规则，这些规则会阻止 Helm 与 Tiller 通信： + +``` +Chain INPUT (policy ACCEPT) +target prot opt source destination +ACCEPT all -- anywhere anywhere state RELATED,ESTABLISHED +ACCEPT icmp -- anywhere anywhere +ACCEPT all -- anywhere anywhere +ACCEPT tcp -- anywhere anywhere state NEW tcp dpt:ssh +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain FORWARD (policy ACCEPT) +target prot opt source destination +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain OUTPUT (policy ACCEPT) +target prot opt source destination +``` + +ä½ å¯è¿è¡Œä»¥ä¸‹å‘½ä»¤æ£€æŸ¥é»˜è®¤é˜²ç«å¢™è§„则： + +``` +sudo iptables --list +``` + +下文介ç»å¦‚何使用 `firewalld`，将[防ç«å¢™ç«¯å£è§„则](../../pages-for-subheaders/installation-requirements.md#端å£è¦æ±‚)应用到高å¯ç”¨ Rancher Server 集群中的节点。 + +## 先决æ¡ä»¶ + +安装 v7.x 或更高版本的 `firewalld`： + +``` +yum install firewalld +systemctl start firewalld +systemctl enable firewalld +``` + +## 应用防ç«å¢™ç«¯å£è§„则 + +在 Rancher 高å¯ç”¨å®‰è£…中，Rancher Server 设置在三个节点上，三个节点å‡å…·æœ‰ Kubernetes 的所有三个角色（etcdã€controlplane å’Œ worker）。如果你的 Rancher Server 节点åŒæ—¶å…·æœ‰è¿™ä¸‰ä¸ªè§’色，请在æ¯ä¸ªèŠ‚点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` +如果你的 Rancher Server 节点é…置了å•ç‹¬çš„角色，请根æ®èŠ‚点角色è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +# 在 etcd 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp + +# 在 controlplane 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp + +# 在 worker 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` + +在节点上è¿è¡Œ `firewall-cmd` 命令åŽï¼Œä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å¯ç”¨é˜²ç«å¢™è§„则： + +``` +firewall-cmd --reload +``` + +**结果**：防ç«å¢™å·²æ›´æ–°ï¼Œå› æ­¤ Helm å¯ä»¥ä¸Ž Rancher Server 节点通信了。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md new file mode 100644 index 00000000000..6cadea62ced --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md @@ -0,0 +1,39 @@ +--- +title: 为大型安装进行 etcd 调优 +--- + +当你è¿è¡Œå…·æœ‰ 15 个或更多集群的大型 Rancher 安装时，我们建议你扩大 etcd 的默认 keyspace（默认为 2GB）。你最大å¯ä»¥å°†å®ƒè®¾ç½®ä¸º 8GB。此外，请确ä¿ä¸»æœºæœ‰è¶³å¤Ÿçš„ RAM æ¥ä¿å­˜æ•´ä¸ªæ•°æ®é›†ã€‚如果需è¦å¢žåŠ è¿™ä¸ªå€¼ï¼Œä½ è¿˜éœ€è¦åŒæ­¥å¢žåŠ ä¸»æœºçš„大å°ã€‚如果你预计在垃圾回收间隔期间 Pod çš„å˜åŒ–率很高，你也å¯ä»¥åœ¨è¾ƒå°çš„安装中调整 Keyspace 大å°ã€‚ + +Kubernetes æ¯éš”äº”åˆ†é’Ÿä¼šè‡ªåŠ¨æ¸…ç† etcd æ•°æ®é›†ã€‚在æŸäº›æƒ…况下（例如å‘生部署抖动），在垃圾回收å‘生并进行清ç†ä¹‹å‰ä¼šæœ‰å¤§é‡äº‹ä»¶å†™å…¥ etcd 并删除，从而导致 Keyspace 填满。如果你在 etcd 日志或 Kubernetes API Server 日志中看到 `mvcc: database space exceeded` 错误，你å¯ä»¥åœ¨ etcd æœåŠ¡å™¨ä¸Šè®¾ç½® [quota-backend-bytes](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) æ¥å¢žåŠ  Keyspace 的大å°ã€‚ + +### 示例：此 RKE cluster.yml 文件的代ç ç‰‡æ®µå°† Keyspace 的大å°å¢žåŠ åˆ° 5GB + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + quota-backend-bytes: 5368709120 +``` + +## 扩展 etcd ç£ç›˜æ€§èƒ½ + +ä½ å¯ä»¥å‚è§ [etcd 文档](https://etcd.io/docs/v3.4.0/tuning/#disk)中的建议，了解如何调整主机上的ç£ç›˜ä¼˜å…ˆçº§ã€‚ + +此外，为了å‡å°‘ etcd ç£ç›˜ä¸Šçš„ IO 争用，你å¯ä»¥ä¸º data å’Œ wal 目录使用专用设备。etcd 最佳实践ä¸å»ºè®®é…ç½® Mirror RAID（因为 etcd 在集群中的节点之间å¤åˆ¶æ•°æ®ï¼‰ã€‚ä½ å¯ä»¥ä½¿ç”¨ striping RAID é…ç½®æ¥å¢žåŠ å¯ç”¨çš„ IOPS。 + +è¦åœ¨ RKE 集群中实现此解决方案，你需è¦åœ¨åº•å±‚主机上为 `/var/lib/etcd/data` å’Œ `/var/lib/etcd/wal` 目录挂载并格å¼åŒ–ç£ç›˜ã€‚`etcd` æœåŠ¡çš„ `extra_args` æŒ‡ä»¤ä¸­å¿…é¡»åŒ…å« `wal_dir` 目录。如果ä¸æŒ‡å®š `wal_dir`，etcd 进程会å°è¯•åœ¨æƒé™ä¸è¶³çš„情况下æ“作底层的 `wal` 挂载。 + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + data-dir: '/var/lib/rancher/etcd/data/' + wal-dir: '/var/lib/rancher/etcd/wal/wal_dir' + extra_binds: + - '/var/lib/etcd/data:/var/lib/rancher/etcd/data' + - '/var/lib/etcd/wal:/var/lib/rancher/etcd/wal' +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/add-users-to-projects.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/add-users-to-projects.md new file mode 100644 index 00000000000..4f4f6712b0e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/add-users-to-projects.md @@ -0,0 +1,62 @@ +--- +title: 添加项目æˆå‘˜ +--- + +如果你想为用户æ供集群内 _特定_ 项目和资æºçš„访问æƒé™ï¼Œè¯·ä¸ºç”¨æˆ·åˆ†é…项目æˆå‘˜èµ„格。 + +ä½ å¯ä»¥åœ¨åˆ›å»ºé¡¹ç›®æ—¶å°†æˆå‘˜æ·»åŠ åˆ°é¡¹ç›®ä¸­ï¼Œæˆ–将用户添加到现有项目中。 + +:::tip + +如果你需è¦ä¸ºç”¨æˆ·æ供对集群内 _所有_ 项目的访问æƒé™ï¼Œè¯·å‚è§[添加集群æˆå‘˜](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)。 + +::: + +### å°†æˆå‘˜æ·»åŠ åˆ°æ–°é¡¹ç›® + +ä½ å¯ä»¥åœ¨åˆ›å»ºé¡¹ç›®æ—¶å°†æˆå‘˜æ·»åŠ åˆ°é¡¹ç›®ä¸­ï¼ˆå»ºè®®ï¼‰ã€‚有关创建新项目的详细信æ¯ï¼Œè¯·å‚阅[集群管ç†](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)。 + +### å°†æˆå‘˜æ·»åŠ åˆ°çŽ°æœ‰é¡¹ç›® + +创建项目åŽï¼Œä½ å¯ä»¥å°†ç”¨æˆ·æ·»åŠ ä¸ºé¡¹ç›®æˆå‘˜ï¼Œä»¥ä¾¿ç”¨æˆ·å¯ä»¥è®¿é—®é¡¹ç›®çš„资æºï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ é¡¹ç›®æˆå‘˜çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦æ·»åŠ æˆå‘˜çš„项目。在项目å称上方的**创建命å空间**按钮æ—边，å•å‡» **☰**。选择 **编辑é…ç½®**。 +1. 在**æˆå‘˜**选项å¡ä¸­ï¼Œå•å‡»**添加**。 +1. æœç´¢è¦æ·»åŠ åˆ°é¡¹ç›®çš„用户或组。 + + 如果é…置了外部身份验è¯ï¼š + + - 在你键入时，Rancher 会从你的外部身份验è¯æºè¿”回用户。 + + - ä½ å¯ä»¥åœ¨ä¸‹æ‹‰èœå•ä¸­æ·»åŠ ç»„，而ä¸æ˜¯å•ä¸ªç”¨æˆ·ã€‚下拉列表仅会列出你（登录用户）所在的组。 + + :::note + + 如果你以本地用户身份登录，外部用户ä¸ä¼šæ˜¾ç¤ºåœ¨ä½ çš„æœç´¢ç»“果中。 + + ::: + +1. 分é…用户或组的**项目**角色。 + + [什么是项目角色？](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::note 注æ„事项： + + - 如果用户分é…到了项目的`所有者`或`æˆå‘˜`角色，用户会自动继承`命å空间创建`角色。然而，这个角色是 [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole)，这表示角色的范围会延展到集群中的所有项目。因此，对于显å¼åˆ†é…到了项目`所有者`或`æˆå‘˜`角色的用户æ¥è¯´ï¼Œå³ä½¿åªæœ‰`åªè¯»`角色，这些用户也å¯ä»¥åœ¨åˆ†é…给他们的其他项目中创建或删除命å空间。 + + - 默认情况下，Rancher çš„`项目æˆå‘˜`角色继承自 `Kubernetes-edit` 角色，而`项目所有者`角色继承自 `Kubernetes-admin` 角色。因此，`项目æˆå‘˜`å’Œ`项目所有者`角色都能管ç†å‘½å空间，包括创建和删除命å空间。 + + - 对于`自定义`角色，你å¯ä»¥ä¿®æ”¹å¯åˆ†é…的角色列表。 + + - è¦å°†è§’色添加到列表中，请[添加自定义角色](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md)。 + - è¦ä»Žåˆ—表中删除角色，请[é”定/解é”角色](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md)。 + + ::: + +**结果**：已将选中的用户添加到项目中。 + +- è¦æ’¤é”€é¡¹ç›®æˆå‘˜èµ„格，请选择用户并å•å‡»**删除**。此æ“作会删除æˆå‘˜èµ„格，而ä¸ä¼šåˆ é™¤ç”¨æˆ·ã€‚ +- è¦ä¿®æ”¹é¡¹ç›®ä¸­çš„用户角色，请将其从项目中删除，然åŽä½¿ç”¨ä¿®æ”¹åŽçš„角色é‡æ–°æ·»åŠ ç”¨æˆ·ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md new file mode 100644 index 00000000000..5f9c30227ba --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md @@ -0,0 +1,42 @@ +--- +title: 集群驱动 +--- + +集群驱动用于在[托管 Kubernetes æ供商](../../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)（例如 Google GKE）中创建集群。创建集群时å¯ä»¥æ˜¾ç¤ºçš„集群驱动，是由集群驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 集群驱动将作为创建集群的选项显示。默认情况下，Rancher 与多个现有的云æ供商集群驱动打包在一起，但你也å¯ä»¥å°†è‡ªå®šä¹‰é›†ç¾¤é©±åŠ¨æ·»åŠ åˆ° Rancher。 + +如果你ä¸æƒ³å‘用户显示特定的集群驱动，你å¯ä»¥åœ¨ Rancher 中åœç”¨è¿™äº›é›†ç¾¤é©±åŠ¨ï¼Œå®ƒä»¬å°†ä¸ä¼šä½œä¸ºåˆ›å»ºé›†ç¾¤çš„选项出现。 + +### 管ç†é›†ç¾¤é©±åŠ¨ + +:::note 先决æ¡ä»¶ï¼š + +è¦åˆ›å»ºã€ç¼–辑或删除集群驱动，你需è¦ä»¥ä¸‹æƒé™ä¸­çš„_一个_： + +- [管ç†å‘˜å…¨å±€æƒé™](../manage-role-based-access-control-rbac/global-permissions.md) +- 分é…了[管ç†é›†ç¾¤é©±åŠ¨è§’色](../manage-role-based-access-control-rbac/global-permissions.md)çš„[自定义全局æƒé™](../manage-role-based-access-control-rbac/global-permissions.md#自定义全局æƒé™)。 + +::: + +## 激活/åœç”¨é›†ç¾¤é©±åŠ¨ + +默认情况下，Rancher 仅激活主æµçš„云æ供商 Google GKEã€Amazon EKS å’Œ Azure AKS 的驱动。如果è¦æ˜¾ç¤ºæˆ–éšè—驱动，你å¯ä»¥æ›´æ”¹é©±åŠ¨çš„状æ€ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 + +2. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 + +3. 在**集群驱动**选项å¡ä¸Šï¼Œé€‰æ‹©è¦æ¿€æ´»æˆ–åœç”¨çš„驱动，然åŽå•å‡» **â‹® > 激活** 或 **â‹® > åœç”¨**。 + +## 添加自定义集群驱动 + +如果你想使用 Rancher ä¸æ”¯æŒå¼€ç®±å³ç”¨çš„集群驱动，你å¯ä»¥æ·»åŠ æ供商的驱动，从而使用该驱动æ¥åˆ›å»º _托管_ Kubernetes 集群： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 +1. 在**集群驱动**选项å¡ä¸Šï¼Œå•å‡»**添加集群驱动**。 +1. 填写**添加集群驱动**表å•ã€‚然åŽå•å‡»**创建**。 + + +### å¼€å‘自己的集群驱动 + +如果è¦å¼€å‘集群驱动并添加到 Rancher，请å‚考我们的[示例](https://github.com/rancher-plugins/kontainer-engine-driver-example)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md new file mode 100644 index 00000000000..bb49bcbe46a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md @@ -0,0 +1,41 @@ +--- +title: 主机驱动 +--- + +主机驱动用于é…置主机，Rancher 使用这些主机å¯åŠ¨å’Œç®¡ç† Kubernetes 集群。主机驱动与 [Docker Machine 驱动](https://docs.docker.com/machine/drivers/)相åŒã€‚创建主机模æ¿æ—¶å¯ä»¥æ˜¾ç¤ºçš„主机驱动，是由主机驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 主机驱动将显示为创建节点模æ¿çš„选项。默认情况下，Rancher 与许多现有的 Docker Machine 驱动打包在一起，但你也å¯ä»¥åˆ›å»ºè‡ªå®šä¹‰ä¸»æœºé©±åŠ¨å¹¶æ·»åŠ åˆ° Rancher。 + +如果你ä¸æƒ³å‘用户显示特定的主机驱动，则需è¦åœç”¨è¿™äº›ä¸»æœºé©±åŠ¨ã€‚ + +#### 管ç†ä¸»æœºé©±åŠ¨ + +:::note 先决æ¡ä»¶ï¼š + +è¦åˆ›å»ºã€ç¼–辑或删除驱动，你需è¦ä»¥ä¸‹æƒé™ä¸­çš„_一个_： + +- [管ç†å‘˜å…¨å±€æƒé™](../manage-role-based-access-control-rbac/global-permissions.md) +- 分é…了[管ç†ä¸»æœºé©±åŠ¨è§’色](../manage-role-based-access-control-rbac/global-permissions.md)çš„[自定义全局æƒé™](../manage-role-based-access-control-rbac/global-permissions.md#自定义全局æƒé™)。 + +::: + +## 激活/åœç”¨ä¸»æœºé©±åŠ¨ + +默认情况下，Rancher 仅激活主æµäº‘æ供商 Amazon EC2ã€Azureã€DigitalOcean å’Œ vSphere 的驱动。如果è¦æ˜¾ç¤ºæˆ–éšè—驱动，你å¯ä»¥æ›´æ”¹é©±åŠ¨çš„状æ€ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 + +2. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 + +2. 在**主机驱动**选项å¡ä¸Šï¼Œé€‰æ‹©è¦æ¿€æ´»æˆ–åœç”¨çš„驱动，然åŽå•å‡» **â‹® > 激活** 或 **â‹® > åœç”¨**。 + +## 添加自定义主机驱动 + +如果你想使用 Rancher ä¸æ”¯æŒå¼€ç®±å³ç”¨çš„主机驱动，你å¯ä»¥æ·»åŠ æ供商的驱动，从而使用该驱动为你的 Kubernetes 集群创建节点模æ¿å¹¶æœ€ç»ˆåˆ›å»ºèŠ‚点池： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航èœå•ä¸­ï¼Œå•å‡»**驱动**。 +1. 在**主机驱动**选项å¡ä¸Šï¼Œå•å‡»**添加主机驱动**。 +1. 填写**添加主机驱动**表å•ã€‚然åŽå•å‡»**创建**。 + +### å¼€å‘自己的主机驱动 + +主机驱动使用 [Docker Machine](https://docs.docker.com/machine/) æ¥å®žçŽ°ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md new file mode 100644 index 00000000000..947f2a9784c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md @@ -0,0 +1,64 @@ +--- +title: 访问和共享 +--- + +如果你是 RKE 模æ¿æ‰€æœ‰è€…，你å¯ä»¥å°†è¯¥æ¨¡æ¿å…±äº«ç»™ç”¨æˆ·æˆ–用户组，然åŽä»–们å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚ + +由于 RKE 模æ¿æ˜¯ä¸“门与用户和组共享的，因此所有者å¯ä»¥ä¸Žä¸åŒçš„用户共享ä¸åŒçš„ RKE 模æ¿ã€‚ + +共享模æ¿æ—¶ï¼Œæ¯ä¸ªç”¨æˆ·éƒ½å¯ä»¥æ‹¥æœ‰ä»¥ä¸‹ä¸¤ä¸ªè®¿é—®æƒé™ä¸­çš„其中一个： + +- **所有者**：å¯ä»¥æ›´æ–°ã€åˆ é™¤å’Œå…±äº«ä»–们拥有的模æ¿ã€‚所有者还å¯ä»¥ä¸Žå…¶ä»–用户共享模æ¿ã€‚ +- **用户**：å¯ä»¥ä½¿ç”¨æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚他们还å¯ä»¥å°†è¿™äº›é›†ç¾¤å‡çº§åˆ°åŒä¸€æ¨¡æ¿çš„新版本。如果你将模æ¿å…±äº«ä¸º**公开（åªè¯»ï¼‰**，你的 Rancher 设置中的所有用户都拥有该模æ¿çš„用户访问æƒé™ã€‚ + +如果你创建了一个模æ¿ï¼Œä½ å°†è‡ªåŠ¨æˆä¸ºè¯¥æ¨¡æ¿çš„所有者。 + +如果你想让其他用户更新该模æ¿ï¼Œä½ å¯ä»¥å…±äº«æ¨¡æ¿çš„所有æƒã€‚有关所有者如何修改模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅[修改模æ¿æ–‡æ¡£](manage-rke1-templates.md)。 + +共享模æ¿çš„æ–¹å¼æœ‰å¦‚下几ç§ï¼š + +- 在模æ¿åˆ›å»ºæœŸé—´å°†ç”¨æˆ·æ·»åŠ åˆ°æ–°çš„ RKE æ¨¡æ¿ +- 将用户添加到现有 RKE æ¨¡æ¿ +- 公开 RKE 模æ¿ï¼Œå¹¶ä¸Ž Rancher 设置中的所有用户共享 +- 与å—信任修改模æ¿çš„用户共享模æ¿æ‰€æœ‰æƒ + +### ä¸Žç‰¹å®šç”¨æˆ·æˆ–ç»„å…±äº«æ¨¡æ¿ + +è¦å…许用户或组使用你的模æ¿åˆ›å»ºé›†ç¾¤ï¼Œä½ å¯ä»¥ä¸ºä»–们æ供模æ¿çš„基本**用户**访问æƒé™ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦å…±äº«çš„模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 在**共享模æ¿**中，å•å‡»**添加æˆå‘˜**。 +1. 在**å称**字段中æœç´¢ä½ è¦ä¸Žä¹‹å…±äº«æ¨¡æ¿çš„用户或组。 +1. 选择**用户**访问类型。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户或组å¯ä»¥ä½¿ç”¨æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚ + +### ä¸Žæ‰€æœ‰ç”¨æˆ·å…±äº«æ¨¡æ¿ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦å…±äº«çš„模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 在**共享模æ¿**下，选中 **公开（åªè¯»ï¼‰** å¤é€‰æ¡†ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：Rancher 设置中的所有用户都å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿åˆ›å»ºé›†ç¾¤ã€‚ + +### 共享模æ¿æ‰€æœ‰æƒ + +如果你是模æ¿çš„创建者，你å¯èƒ½å¸Œæœ›å°†ç»´æŠ¤å’Œæ›´æ–°æ¨¡æ¿çš„责任委派给其他用户或组。 + +在这ç§æƒ…况下，你å¯ä»¥ä¸ºç”¨æˆ·æä¾›**所有者**访问æƒé™ï¼Œè¯¥æƒé™å…许其他用户更新ã€åˆ é™¤æ¨¡æ¿æˆ–与其他用户共享对模æ¿çš„访问æƒé™ã€‚ + +è¦æŽˆäºˆç”¨æˆ·æˆ–组**所有者**æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦å…±äº«çš„ RKE 模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 在**共享模æ¿**下，å•å‡»**添加æˆå‘˜**并在**å称**字段中æœç´¢è¦ä¸Žä¹‹å…±äº«æ¨¡æ¿çš„用户或组。 +1. 在**访问类型**字段中，å•å‡»**所有者**。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户或组具有**所有者**访问类型，å¯ä»¥ä¿®æ”¹ã€å…±äº«æˆ–删除模æ¿ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md new file mode 100644 index 00000000000..3efac3e477e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md @@ -0,0 +1,59 @@ +--- +title: åº”ç”¨æ¨¡æ¿ +--- + +ä½ å¯ä»¥ä½¿ç”¨ä½ è‡ªå·±åˆ›å»ºçš„ RKE 模æ¿æ¥åˆ›å»ºé›†ç¾¤ï¼Œä¹Ÿå¯ä»¥ä½¿ç”¨[与你共享的模æ¿](access-or-share-templates.md)æ¥åˆ›å»ºé›†ç¾¤ã€‚ + +RKE 模æ¿å¯ä»¥åº”用于新集群。 + +ä½ å¯ä»¥[将现有集群的é…ç½®ä¿å­˜ä¸º RKE 模æ¿](#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这样，åªæœ‰æ¨¡æ¿æ›´æ–°åŽæ‰èƒ½æ›´æ”¹é›†ç¾¤çš„设置。 + +你无法将集群更改为使用ä¸åŒçš„ RKE 模æ¿ã€‚ä½ åªèƒ½å°†é›†ç¾¤æ›´æ–°ä¸ºåŒä¸€æ¨¡æ¿çš„新版本。 + + +### 使用 RKE 模æ¿åˆ›å»ºé›†ç¾¤ + +è¦ä½¿ç”¨ RKE 模æ¿æ·»åŠ [由基础设施æ供商托管](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)的集群，请按照以下步骤æ“作： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**并选择基础设施æ供商。 +1. 设置集群å称和节点模æ¿è¯¦æƒ…。 +1. è¦ä½¿ç”¨ RKE 模æ¿ï¼Œè¯·åœ¨**集群选项**下，选中**使用现有 RKE 模æ¿å’Œä¿®è®¢ç‰ˆ**å¤é€‰æ¡†ã€‚ +1. 从下拉èœå•ä¸­é€‰æ‹© RKE 模æ¿å’Œä¿®è®¢ç‰ˆã€‚ +1. å¯é€‰ï¼šä½ å¯ä»¥ç¼–辑 RKE 模æ¿æ‰€æœ‰è€…在创建模æ¿æ—¶æ ‡è®°ä¸º**å…许用户覆盖**的任何设置。如果你无法更改æŸäº›è®¾ç½®ï¼Œåˆ™éœ€è¦è”系模æ¿æ‰€æœ‰è€…以获å–模æ¿çš„新修订版。然åŽï¼Œä½ éœ€è¦ç¼–辑集群æ¥å°†å…¶å‡çº§åˆ°æ–°ç‰ˆæœ¬ã€‚ +1. å•å‡»**创建**以å¯åŠ¨é›†ç¾¤ã€‚ + +### 更新使用 RKE 模æ¿åˆ›å»ºçš„集群 + +模æ¿æ‰€æœ‰è€…创建 RKE 模æ¿æ—¶ï¼Œæ¯ä¸ªè®¾ç½®åœ¨ Rancher UI 中都有一个开关，指示用户是å¦å¯ä»¥è¦†ç›–该设置。 + +- 如果æŸä¸ªè®¾ç½®å…许用户覆盖，你å¯ä»¥é€šè¿‡[编辑集群](../../../../pages-for-subheaders/cluster-configuration.md)æ¥æ›´æ–°é›†ç¾¤ä¸­çš„设置。 +- 如果该开关处于关闭状æ€ï¼Œåˆ™é™¤éžé›†ç¾¤æ‰€æœ‰è€…创建了å…许你覆盖这些设置的模æ¿ä¿®è®¢ç‰ˆï¼Œå¦åˆ™ä½ æ— æ³•æ›´æ”¹è¿™äº›è®¾ç½®ã€‚如果你无法更改æŸäº›è®¾ç½®ï¼Œåˆ™éœ€è¦è”系模æ¿æ‰€æœ‰è€…以获å–模æ¿çš„新修订版。 + +如果集群是使用 RKE 模æ¿åˆ›å»ºçš„，你å¯ä»¥ç¼–辑集群，æ¥å°†é›†ç¾¤æ›´æ–°ä¸ºæ¨¡æ¿çš„新版本。 + +现有集群的设置å¯ä»¥[ä¿å­˜ä¸º RKE 模æ¿](#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。在这ç§æƒ…况下，你还å¯ä»¥ç¼–辑集群以将集群更新为模æ¿çš„新版本。 + +:::note + +你无法将集群更改为使用ä¸åŒçš„ RKE 模æ¿ã€‚ä½ åªèƒ½å°†é›†ç¾¤æ›´æ–°ä¸ºåŒä¸€æ¨¡æ¿çš„新版本。 + +::: + +### 将现有集群转æ¢ä¸ºä½¿ç”¨ RKE æ¨¡æ¿ + +本节介ç»å¦‚何使用现有集群创建 RKE 模æ¿ã€‚ + +除éžä½ å°†çŽ°æœ‰é›†ç¾¤çš„设置ä¿å­˜ä¸º RKE 模æ¿ï¼Œå¦åˆ™ RKE 模æ¿ä¸èƒ½åº”用于现有集群。这将把集群的设置导出为新的 RKE 模æ¿ï¼Œå¹¶ä¸”将集群绑定到该模æ¿ã€‚然åŽï¼Œåªæœ‰[更新了模æ¿](manage-rke1-templates.md#更新模æ¿)并且集群å‡çº§åˆ°**使用更新版本的模æ¿**时，集群æ‰èƒ½æ”¹å˜ã€‚ + +è¦å°†çŽ°æœ‰é›†ç¾¤è½¬æ¢ä¸ºä½¿ç”¨ RKE 模æ¿ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è½¬æ¢ä¸ºä½¿ç”¨ RKE 模æ¿çš„集群。å•å‡» **â‹® > ä¿å­˜ä¸º RKE 模æ¿**。 +1. 在出现的表å•ä¸­è¾“入模æ¿çš„å称，然åŽå•å‡»**创建**。 + +**结果**： + +- 创建了一个新的 RKE 模æ¿ã€‚ +- 将集群转æ¢ä¸ºä½¿ç”¨è¯¥æ–°æ¨¡æ¿ã€‚ +- å¯ä»¥[使用新模æ¿åˆ›å»ºæ–°é›†ç¾¤](apply-templates.md#使用-rke-模æ¿åˆ›å»ºé›†ç¾¤)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md new file mode 100644 index 00000000000..7db812c38ef --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md @@ -0,0 +1,57 @@ +--- +title: 模æ¿åˆ›å»ºè€…æƒé™ +--- + +管ç†å‘˜æœ‰åˆ›å»º RKE 模æ¿çš„æƒé™ï¼Œåªæœ‰ç®¡ç†å‘˜å¯ä»¥å°†è¯¥æƒé™æŽˆäºˆå…¶ä»–用户。 + +有关管ç†å‘˜æƒé™çš„更多信æ¯ï¼Œè¯·å‚阅[全局æƒé™æ–‡æ¡£](../manage-role-based-access-control-rbac/global-permissions.md)。 + +## 授予用户创建模æ¿çš„æƒé™ + +åªæœ‰å…·æœ‰**创建 RKE 模æ¿**全局æƒé™çš„用户æ‰èƒ½åˆ›å»ºæ¨¡æ¿ã€‚ + +管ç†å‘˜æ‹¥æœ‰åˆ›å»ºæ¨¡æ¿çš„全局æƒé™ï¼Œåªæœ‰ç®¡ç†å‘˜æ‰èƒ½å°†è¯¥æƒé™æŽˆäºˆå…¶ä»–用户。 + +有关å…许用户修改现有模æ¿çš„ä¿¡æ¯ï¼Œè¯·å‚阅[共享模æ¿](access-or-share-templates.md)。 + +管ç†å‘˜å¯ä»¥é€šè¿‡ä¸¤ç§æ–¹å¼æŽˆäºˆç”¨æˆ·åˆ›å»º RKE 模æ¿çš„æƒé™ï¼š + +- 通过编辑[å•ä¸ªç”¨æˆ·](#å…许用户创建模æ¿)çš„æƒé™ +- 通过更改[新用户的默认æƒé™](#默认å…许新用户创建模æ¿) + +### å…è®¸ç”¨æˆ·åˆ›å»ºæ¨¡æ¿ + +管ç†å‘˜å¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤å°†**创建 RKE 模æ¿**角色å•ç‹¬æŽˆäºˆç»™ä»»ä½•çŽ°æœ‰ç”¨æˆ·ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. 选择è¦ç¼–辑的用户，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**内置角色**中，选中**创建 RKE 集群模æ¿**角色以åŠç”¨æˆ·åº”具有的其他角色。你å¯èƒ½è¿˜éœ€è¦é€‰ä¸­**创建 RKE 模æ¿ä¿®è®¢ç‰ˆ**å¤é€‰æ¡†ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户拥有创建 RKE 模æ¿çš„æƒé™ã€‚ + +### 默认å…è®¸æ–°ç”¨æˆ·åˆ›å»ºæ¨¡æ¿ + +管ç†å‘˜ä¹Ÿå¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤ä¸ºæ‰€æœ‰æ–°ç”¨æˆ·æŽˆäºˆåˆ›å»º RKE 模æ¿çš„默认æƒé™ã€‚è¿™ä¸ä¼šå½±å“现有用户的æƒé™ã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 转到**创建 RKE 集群模æ¿**角色，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 选择**是：新用户的默认角色**选项。 +1. å•å‡»**ä¿å­˜**。 +1. 如果你希望新用户能够创建 RKE 模æ¿ä¿®è®¢ï¼Œè¯·å°†è¯¥è§’色设置为默认值。 + +**结果**：在此 Rancher 安装中创建的任何新用户都å¯ä»¥åˆ›å»º RKE 模æ¿ã€‚现有用户将ä¸ä¼šèŽ·å¾—æ­¤æƒé™ã€‚ + +### å–消创建模æ¿çš„æƒé™ + +管ç†å‘˜å¯ä»¥é€šè¿‡ä»¥ä¸‹æ­¥éª¤åˆ é™¤ç”¨æˆ·åˆ›å»ºæ¨¡æ¿çš„æƒé™ã€‚请注æ„，无论是å¦é€‰æ‹©äº†ç»†ç²’度æƒé™ï¼Œç®¡ç†å‘˜éƒ½å¯ä»¥å®Œå…¨æŽ§åˆ¶æ‰€æœ‰èµ„æºã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. 选择è¦ç¼–辑æƒé™çš„用户，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**内置角色**中，å–消选中**创建 RKE 模æ¿**å’Œ**创建 RKE 模æ¿ä¿®è®¢ç‰ˆ**å¤é€‰æ¡†ï¼ˆå¦‚果适用）。在此处，你å¯ä»¥å°†ç”¨æˆ·æ”¹å›žæ™®é€šç”¨æˆ·ï¼Œæˆ–授予用户一组ä¸åŒçš„æƒé™ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户无法创建 RKE 模æ¿ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md new file mode 100644 index 00000000000..38ab0f41ee8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md @@ -0,0 +1,43 @@ +--- +title: å¼ºåˆ¶ä½¿ç”¨æ¨¡æ¿ +--- + +本节介ç»æ¨¡æ¿ç®¡ç†å‘˜å¦‚何在 Rancher 中强制执行模æ¿ï¼Œä»Žè€Œé™åˆ¶ç”¨æˆ·åœ¨æ²¡æœ‰æ¨¡æ¿çš„情况下创建集群。 + +默认情况下，Rancher 中的任何普通用户都å¯ä»¥åˆ›å»ºé›†ç¾¤ã€‚但当开å¯å¼ºåˆ¶ä½¿ç”¨ RKE 模æ¿æ—¶ï¼Œæœ‰ä»¥ä¸‹çº¦æŸï¼š + +- åªæœ‰ç®¡ç†å‘˜æ‰èƒ½åœ¨æ²¡æœ‰æ¨¡æ¿çš„情况下创建集群。 +- 所有普通用户必须使用 RKE 模æ¿æ¥åˆ›å»ºæ–°é›†ç¾¤ã€‚ +- 普通用户在ä¸ä½¿ç”¨æ¨¡æ¿çš„情况下无法创建集群。 + +åªæœ‰ç®¡â€‹â€‹ç†å‘˜[授予æƒé™](creator-permissions.md#å…许用户创建模æ¿)åŽï¼Œç”¨æˆ·æ‰èƒ½åˆ›å»ºæ–°æ¨¡æ¿ã€‚ + +使用 RKE 模æ¿åˆ›å»ºé›†ç¾¤åŽï¼Œé›†ç¾¤åˆ›å»ºè€…无法编辑模æ¿ä¸­å®šä¹‰çš„设置。创建集群åŽæ›´æ”¹è¿™äº›è®¾ç½®çš„唯一方法是[将集群å‡çº§åˆ°ç›¸åŒæ¨¡æ¿çš„新修订版](apply-templates.md#更新使用-rke-模æ¿åˆ›å»ºçš„集群)。如果集群创建者想è¦æ›´æ”¹æ¨¡æ¿å®šä¹‰çš„设置，他们需è¦è”系模æ¿æ‰€æœ‰è€…以获å–模æ¿çš„新版本。有关模æ¿ä¿®è®¢å¦‚何工作的详细信æ¯ï¼Œè¯·å‚阅[修订模æ¿](manage-rke1-templates.md#更新模æ¿)。 + +## 强制新集群使用 RKE æ¨¡æ¿ + +è¦æ±‚用户创建新集群时使用模æ¿ï¼Œå¯ä»¥ç¡®ä¿[普通用户](../manage-role-based-access-control-rbac/global-permissions.md)å¯åŠ¨çš„任何集群都使用ç»è¿‡ç®¡ç†å‘˜å®¡æ ¸çš„ Kubernetes å’Œ Rancher 设置。 + +管ç†å‘˜å¯ä»¥é€šè¿‡ä»¥ä¸‹æ­¥éª¤å¯ç”¨ RKE 模æ¿å¼ºåˆ¶ï¼Œä»Žè€Œè¦æ±‚用户必须使用模æ¿åˆ›å»ºé›†ç¾¤ï¼š + +1. å•å‡» **☰ > 全局设置**。 +1. 转到 `cluster-template-enforcement` 设置。å•å‡» **â‹® > 编辑设置**。 +1. 将值设置为 **True** 并å•å‡»**ä¿å­˜**。 + + :::note é‡è¦æ示： + + 如果管ç†å‘˜å°† `cluster-template-enforcement` 设置为 True，还需è¦ä¸Žç”¨æˆ·å…±äº« `clusterTemplates`，以便用户å¯ä»¥é€‰æ‹©å…¶ä¸­ä¸€ä¸ªæ¨¡æ¿æ¥åˆ›å»ºé›†ç¾¤ã€‚ + + ::: + +**结果**：除éžåˆ›å»ºè€…是管ç†å‘˜ï¼Œå¦åˆ™ Rancher é…置的所有集群都必须使用模æ¿ã€‚ + +## ç¦ç”¨ RKE 模æ¿å¼ºåˆ¶ + +管ç†å‘˜å¯ä»¥é€šè¿‡ä»¥ä¸‹æ­¥éª¤å…³é—­ RKE 模æ¿å¼ºåˆ¶ï¼Œä»Žè€Œå…许用户在没有 RKE 模æ¿çš„情况下创建新集群： + +1. å•å‡» **☰ > 全局设置**。 +1. 转到 `cluster-template-enforcement` 设置。å•å‡» **â‹® > 编辑设置**。 +1. 将值设置为 **False** 并å•å‡»**ä¿å­˜**。 + +**结果**：在 Rancher 中创建集群时，用户ä¸éœ€è¦ä½¿ç”¨æ¨¡æ¿ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md new file mode 100644 index 00000000000..7457df20253 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md @@ -0,0 +1,70 @@ +--- +title: 示例场景 +--- + +以下示例场景æ述了组织如何使用模æ¿æ¥æ ‡å‡†åŒ–集群创建。 + +- **强制执行模æ¿**：如果希望所有 Rancher é…置的新集群都具有æŸäº›è®¾ç½®ï¼Œç®¡ç†å‘˜å¯èƒ½æƒ³è¦[为æ¯ä¸ªç”¨æˆ·å¼ºåˆ¶æ‰§è¡Œä¸€é¡¹æˆ–多项模æ¿è®¾ç½®](#强制执行模æ¿è®¾ç½®)。 +- **与ä¸åŒçš„用户共享ä¸åŒçš„模æ¿**：管ç†å‘˜å¯ä»¥ä¸º[普通用户和高级用户æä¾›ä¸åŒçš„模æ¿](#普通用户和高级用户模æ¿)。这样，普通用户会有更多é™åˆ¶é€‰é¡¹ï¼Œè€Œé«˜çº§ç”¨æˆ·åœ¨åˆ›å»ºé›†ç¾¤æ—¶å¯ä»¥ä½¿ç”¨æ›´å¤šé€‰é¡¹ã€‚ +- **更新模æ¿è®¾ç½®**：如果组织的安全和 DevOps 团队决定将最佳实践嵌入到新集群所需的设置中，这些最佳实践å¯èƒ½ä¼šéšç€æ—¶é—´è€Œæ”¹å˜ã€‚如果最佳实践å‘生å˜åŒ–，[å¯ä»¥å°†æ¨¡æ¿æ›´æ–°ä¸ºæ–°ç‰ˆæœ¬](#更新模æ¿å’Œé›†ç¾¤)。这样，使用模æ¿åˆ›å»ºçš„集群å¯ä»¥å‡çº§åˆ°æ¨¡æ¿çš„新版本。 +- **共享模æ¿çš„所有æƒ**：当模æ¿æ‰€æœ‰è€…ä¸å†æƒ³è¦ç»´æŠ¤æ¨¡æ¿æˆ–想è¦å…±äº«æ¨¡æ¿çš„所有æƒæ—¶ï¼Œæ­¤æ–¹æ¡ˆæ述了如何[授æƒæ¨¡æ¿æ‰€æœ‰æƒ](#å…许其他用户控制和共享模æ¿)。 + + +## 强制执行模æ¿è®¾ç½® + +å‡è®¾ä¸€ä¸ªç»„织的管ç†å‘˜å†³å®šç”¨ Kubernetes 版本 1.14 创建所有新集群： + +1. 首先，管ç†å‘˜åˆ›å»ºä¸€ä¸ªæ¨¡æ¿ï¼Œå°† Kubernetes 版本指定为 1.14，并将所有其他设置标记为**å…许用户覆盖**。 +1. 管ç†å‘˜å°†æ¨¡æ¿å…¬å¼€ã€‚ +1. 管ç†å‘˜æ‰“开模æ¿å¼ºåˆ¶åŠŸèƒ½ã€‚ + +**结果**： + +- 组织中的所有 Rancher 用户都å¯ä»¥è®¿é—®è¯¥æ¨¡æ¿ã€‚ +- [普通用户](../manage-role-based-access-control-rbac/global-permissions.md)使用此模æ¿åˆ›å»ºçš„所有新集群都将使用 Kubernetes 1.14，它们无法使用其它 Kubernetes 版本。默认情况下，普通用户没有创建模æ¿çš„æƒé™ã€‚因此，除éžä¸Žä»–们共享更多模æ¿ï¼Œå¦åˆ™æ­¤æ¨¡æ¿å°†æ˜¯æ™®é€šç”¨æˆ·å”¯ä¸€å¯ä»¥ä½¿ç”¨çš„模æ¿ã€‚ +- 所有普通用户都必须使用集群模æ¿æ¥åˆ›å»ºæ–°é›†ç¾¤ã€‚他们无法在ä¸ä½¿ç”¨æ¨¡æ¿çš„情况下创建集群。 + +通过这ç§æ–¹å¼ï¼Œç®¡ç†å‘˜åœ¨æ•´ä¸ªç»„织中强制执行 Kubernetes 版本，åŒæ—¶ä»ç„¶å…许最终用户é…置其他所有内容。 + +## æ™®é€šç”¨æˆ·å’Œé«˜çº§ç”¨æˆ·æ¨¡æ¿ + +å‡è®¾ä¸€ä¸ªç»„织有普通用户和高级用户。管ç†å‘˜å¸Œæœ›æ™®é€šç”¨æˆ·å¿…须使用模æ¿ï¼Œè€Œé«˜çº§ç”¨æˆ·å’Œç®¡ç†å‘˜å¯ä»¥æ ¹æ®è‡ªå·±çš„需è¦åˆ›å»ºé›†ç¾¤ã€‚ + +1. 首先，管ç†å‘˜å¼€å¯ [RKE 模æ¿å¼ºåˆ¶æ‰§è¡Œ](enforce-templates.md#强制新集群使用-rke-模æ¿)。这æ„å‘³ç€ Rancher 中的æ¯ä¸ª[普通用户](../manage-role-based-access-control-rbac/global-permissions.md)在创建集群时都需è¦ä½¿ç”¨ RKE 模æ¿ã€‚ +1. 然åŽç®¡ç†å‘˜åˆ›å»ºä¸¤ä¸ªæ¨¡æ¿ï¼š + +- 一个普通用户模æ¿ï¼Œè¯¥æ¨¡æ¿é™¤äº†è®¿é—®å¯†é’¥å¤–，几乎指定了所有选项 +- 一个高级用户模æ¿ï¼Œè¯¥æ¨¡æ¿å…·æœ‰å¤§éƒ¨åˆ†æˆ–所有已å¯ç”¨**å…许用户覆盖**的选项 + +1. 管ç†å‘˜ä»…与高级用户共享高级模æ¿ã€‚ +1. 管ç†å‘˜å°†æ™®é€šç”¨æˆ·çš„模æ¿å…¬å¼€ï¼Œå› æ­¤åœ¨ Rancher 中创建的 RKE 集群的æ¯ä¸ªäººéƒ½èƒ½é€‰æ‹©é™åˆ¶æ€§æ›´å¼ºçš„模æ¿ã€‚ + +**结果**：除管ç†å‘˜å¤–，所有 Rancher 用户在创建集群时都需è¦ä½¿ç”¨æ¨¡æ¿ã€‚æ¯ä¸ªäººéƒ½å¯ä»¥è®¿é—®é™åˆ¶æ¨¡æ¿ï¼Œä½†åªæœ‰é«˜çº§ç”¨æˆ·æœ‰æƒä½¿ç”¨æ›´å®½æ¾çš„模æ¿ã€‚普通用户会å—到更多é™åˆ¶ï¼Œè€Œé«˜çº§ç”¨æˆ·åœ¨é…ç½® Kubernetes 集群时有更多选择。 + +## 更新模æ¿å’Œé›†ç¾¤ + +å‡è®¾ä¸€ä¸ªç»„织有一个模æ¿ï¼Œè¯¥æ¨¡æ¿è¦æ±‚集群使用 Kubernetes v1.14。然而，éšç€æ—¶é—´çš„推移，管ç†å‘˜æ”¹å˜äº†ä¸»æ„。管ç†å‘˜çŽ°åœ¨å¸Œæœ›ç”¨æˆ·èƒ½å¤Ÿå‡çº§é›†ç¾¤ï¼Œä»¥ä½¿ç”¨æ›´æ–°ç‰ˆæœ¬çš„ Kubernetes。 + +åœ¨è¿™ä¸ªç»„ç»‡ä¸­ï¼Œè®¸å¤šé›†ç¾¤æ˜¯ç”¨ä¸€ä¸ªéœ€è¦ Kubernetes v1.14 的模æ¿åˆ›å»ºçš„。由于模æ¿ä¸å…许é‡å†™è¯¥è®¾ç½®ï¼Œå› æ­¤åˆ›å»ºé›†ç¾¤çš„用户无法直接编辑该设置。 + +模æ¿æ‰€æœ‰è€…å¯ä»¥æœ‰ä»¥ä¸‹å‡ ä¸ªé€‰é¡¹ï¼Œæ¥å…许集群创建者在集群上å‡çº§ Kubernetes： + +- **在模æ¿ä¸ŠæŒ‡å®š Kubernetes v1.15**：模æ¿æ‰€æœ‰è€…å¯ä»¥åˆ›å»ºæŒ‡å®š Kubernetes v1.15 的新模æ¿ä¿®è®¢ç‰ˆã€‚然åŽä½¿ç”¨è¯¥æ¨¡æ¿çš„æ¯ä¸ªé›†ç¾¤çš„所有者å¯ä»¥å°†é›†ç¾¤å‡çº§åˆ°æ¨¡æ¿çš„新版本。此模æ¿å‡çº§å…许集群创建者在集群上将 Kubernetes å‡çº§åˆ° v1.15。 +- **å…许在模æ¿ä¸Šä½¿ç”¨ä»»ä½• Kubernetes 版本**：创建模æ¿ä¿®è®¢æ—¶ï¼Œæ¨¡æ¿æ‰€æœ‰è€…还å¯ä»¥ä½¿ç”¨ Rancher UI 上该设置附近的开关，将 Kubernetes 版本标记为**å…许用户覆盖**。该设置å…许å‡çº§åˆ°æ­¤æ¨¡æ¿ç‰ˆæœ¬çš„é›†ç¾¤ä½¿ç”¨ä»»æ„ Kubernetes 的版本。 +- **å…许在模æ¿ä¸Šä½¿ç”¨æœ€æ–°çš„ Kubernetes 次è¦ç‰ˆæœ¬**：模æ¿æ‰€æœ‰è€…还å¯ä»¥åˆ›å»ºä¸€ä¸ªæ¨¡æ¿ä¿®è®¢ç‰ˆï¼Œå…¶ä¸­ Kubernetes 版本被定义为 **Latest v1.14（å…许补ä¸ç‰ˆæœ¬å‡çº§ï¼‰**。这æ„味ç€ä½¿ç”¨è¯¥ç‰ˆæœ¬çš„集群将能够进行补ä¸ç‰ˆæœ¬å‡çº§ï¼Œä½†ä¸æ”¯æŒä¸»è¦ç‰ˆæœ¬å‡çº§ã€‚ + +## å…è®¸å…¶ä»–ç”¨æˆ·æŽ§åˆ¶å’Œå…±äº«æ¨¡æ¿ + +å‡è®¾ Alice 是 Rancher 管ç†å‘˜ã€‚她拥有一个 RKE 模æ¿ï¼Œè¯¥æ¨¡æ¿å映了她的组织为创建集群而商定的最佳实践。 + +Bob 是一ä½é«˜çº§ç”¨æˆ·ï¼Œå¯ä»¥å°±é›†ç¾¤é…ç½®åšå‡ºæ˜Žæ™ºçš„决策。éšç€æœ€ä½³å®žè·µéšç€æ—¶é—´çš„推移ä¸æ–­æ›´æ–°ï¼ŒAlice 相信 Bob 会为她的模æ¿åˆ›å»ºæ–°çš„修订。因此，她决定让 Bob æˆä¸ºæ¨¡æ¿çš„所有者。 + +为了与 Bob 共享模æ¿çš„所有æƒï¼ŒAlice [å°† Bob 添加为模æ¿çš„所有者](access-or-share-templates.md#共享模æ¿æ‰€æœ‰æƒ)。 + +结果是，作为模æ¿æ‰€æœ‰è€…，Bob 负责该模æ¿çš„版本控制。Bob 现在å¯ä»¥æ‰§è¡Œä»¥ä¸‹æ‰€æœ‰æ“作： + +- 当最佳实践å‘生å˜åŒ–æ—¶[修改模æ¿](manage-rke1-templates.md#更新模æ¿) +- [ç¦ç”¨æ¨¡æ¿çš„过时修订](manage-rke1-templates.md#ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆ)，以ç¦æ­¢ä½¿ç”¨è¯¥æ¨¡æ¿æ¥åˆ›å»ºé›†ç¾¤ +- 如果组织想è¦æ”¹å˜æ–¹å‘，则[删除整个模æ¿](manage-rke1-templates.md#删除模æ¿) +- [å°†æŸä¸ªç‰ˆæœ¬è®¾ç½®ä¸ºé»˜è®¤å€¼](manage-rke1-templates.md#将模æ¿ä¿®è®¢ç‰ˆè®¾ç½®ä¸ºé»˜è®¤)，用于用户创建集群。模æ¿çš„最终用户ä»ç„¶å¯ä»¥é€‰æ‹©ä»–们想è¦ä½¿ç”¨å“ªä¸ªç‰ˆæœ¬æ¥åˆ›å»ºé›†ç¾¤ã€‚ +- [与特定用户共享模æ¿](access-or-share-templates.md)，让所有 Rancher 用户都å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿ï¼Œæˆ–与其他用户共享该模æ¿çš„所有æƒã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md new file mode 100644 index 00000000000..a3cb27d9b6b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md @@ -0,0 +1,69 @@ +--- +title: RKE 模æ¿å’ŒåŸºç¡€è®¾æ–½ +--- + +在 Rancher 中，RKE 模æ¿ç”¨äºŽé…ç½® Kubernetes 和定义 Rancher 设置，而节点模æ¿åˆ™ç”¨äºŽé…置节点。 + +因此，å³ä½¿å¼€å¯äº† RKE 模æ¿å¼ºåˆ¶ï¼Œæœ€ç»ˆç”¨æˆ·åœ¨åˆ›å»º Rancher 集群时ä»ç„¶å¯ä»¥çµæ´»é€‰æ‹©åº•å±‚硬件。RKE 模æ¿çš„最终用户ä»ç„¶å¯ä»¥é€‰æ‹©åŸºç¡€è®¾æ–½æ供商和他们想è¦ä½¿ç”¨çš„节点。 + +如果è¦æ ‡å‡†åŒ–集群中的硬件，请将 RKE 模æ¿ä¸ŽèŠ‚点模æ¿æˆ–æœåŠ¡å™¨é…置工具 (如 Terraform) 结åˆä½¿ç”¨ã€‚ + +### èŠ‚ç‚¹æ¨¡æ¿ + +[节点模æ¿](../../../../reference-guides/user-settings/manage-node-templates.md)è´Ÿè´£ Rancher 中的节点é…置和节点预é…。你å¯ä»¥åœ¨ç”¨æˆ·é…置文件中设置节点模æ¿ï¼Œä»Žè€Œå®šä¹‰åœ¨æ¯ä¸ªèŠ‚点池中使用的模æ¿ã€‚å¯ç”¨èŠ‚点池åŽï¼Œå¯ä»¥ç¡®ä¿æ¯ä¸ªèŠ‚点池中都有所需数é‡çš„节点，并确ä¿æ± ä¸­çš„所有节点都相åŒã€‚ + +### Terraform + +Terraform 是一个æœåŠ¡å™¨é…置工具。它使用基础架构å³ä»£ç ï¼Œæ”¯æŒä½¿ç”¨ Terraform é…置文件创建几乎所有的基础设施。它å¯ä»¥è‡ªåŠ¨æ‰§è¡ŒæœåŠ¡å™¨é…置，这ç§æ–¹å¼æ˜¯è‡ªæ–‡æ¡£åŒ–的，并且在版本控制中易于跟踪。 + +本节é‡ç‚¹ä»‹ç»å¦‚何将 Terraform 与 [Rancher 2 Terraform Provider](https://www.terraform.io/docs/providers/rancher2/) 一起使用，这是标准化 Kubernetes 集群硬件的推è选项。如果你使用 Rancher Terraform Provider æ¥é…置硬件，然åŽä½¿ç”¨ RKE 模æ¿åœ¨è¯¥ç¡¬ä»¶ä¸Šé…ç½® Kubernetes 集群，你å¯ä»¥å¿«é€Ÿåˆ›å»ºä¸€ä¸ªå…¨é¢çš„ã€å¯ç”¨äºŽç”Ÿäº§çš„集群。 + +Terraform 支æŒï¼š + +- 定义几乎任何类型的基础架构å³ä»£ç ï¼ŒåŒ…括æœåŠ¡å™¨ã€æ•°æ®åº“ã€è´Ÿè½½å‡è¡¡å™¨ã€ç›‘控ã€é˜²ç«å¢™è®¾ç½®å’Œ SSL è¯ä¹¦ +- 使用应用商店应用和多集群应用 +- 跨多个平å°ï¼ˆåŒ…括 Rancher 和主è¦äº‘æä¾›å•†ï¼‰å¯¹åŸºç¡€è®¾æ–½è¿›è¡Œç¼–ç  +- 将基础架构å³ä»£ç æ交到版本控制 +- è½»æ¾é‡å¤ä½¿ç”¨åŸºç¡€è®¾æ–½çš„é…置和设置 +- 将基础架构更改纳入标准开å‘实践 +- 防止由于é…ç½®å移，导致一些æœåŠ¡å™¨çš„é…置与其他æœåŠ¡å™¨ä¸åŒ + +## Terraform å·¥ä½œåŽŸç† + +Terraform 是用扩展å为 `.tf` 的文件编写的。它是用 HashiCorp é…置语言编写的。HashiCorp é…置语言是一ç§å£°æ˜Žæ€§è¯­è¨€ï¼Œæ”¯æŒå®šä¹‰é›†ç¾¤ä¸­æ‰€éœ€çš„基础设施ã€æ­£åœ¨ä½¿ç”¨çš„云æ供商以åŠæ供商的凭è¯ã€‚ç„¶åŽ Terraform å‘æ供商å‘出 API 调用，以便有效地创建基础设施。 + +è¦ä½¿ç”¨ Terraform 创建 Rancher é…置的集群，请转到你的 Terraform é…置文件并将æ供商定义为 Rancher 2。你å¯ä»¥ä½¿ç”¨ Rancher API 密钥设置你的 Rancher 2 æ供商。请注æ„，API 密钥与其关è”的用户具有相åŒçš„æƒé™å’Œè®¿é—®çº§åˆ«ã€‚ + +ç„¶åŽ Terraform 会调用 Rancher API æ¥é…置你的基础设施，而 Rancher 调用基础设施æ供商。例如，如果你想使用 Rancher 在 AWS 上预é…基础设施，你需è¦åœ¨ Terraform é…置文件或环境å˜é‡ä¸­æä¾› Rancher API 密钥和 AWS 凭è¯ï¼Œä»¥ä¾¿å®ƒä»¬ç”¨äºŽé¢„é…基础设施。 + +如果你需è¦å¯¹åŸºç¡€è®¾æ–½è¿›è¡Œæ›´æ”¹ï¼Œä½ å¯ä»¥åœ¨ Terraform é…置文件中进行更改，而ä¸æ˜¯æ‰‹åŠ¨æ›´æ–°æœåŠ¡å™¨ã€‚然åŽï¼Œå¯ä»¥å°†è¿™äº›æ–‡ä»¶æ交给版本控制ã€éªŒè¯ï¼Œå¹¶æ ¹æ®éœ€è¦è¿›è¡Œæ£€æŸ¥ã€‚然åŽï¼Œå½“ä½ è¿è¡Œ `terraform apply` 时，更改将会被部署。 + +## 使用 Terraform 的技巧 + +- [Rancher 2 æ供商文档](https://www.terraform.io/docs/providers/rancher2/)æ供了如何é…置集群大部分的示例。 + +- 在 Terraform 设置中，你å¯ä»¥ä½¿ç”¨ Docker Machine 主机驱动æ¥å®‰è£… Docker Machine。 + +- å¯ä»¥åœ¨ Terraform Provider 中修改身份验è¯ã€‚ + +- å¯ä»¥é€šè¿‡æ›´æ”¹ Rancher 中的设置，æ¥åå‘工程如何在 Terraform 中定义设置，然åŽè¿”回并检查 Terraform 状æ€æ–‡ä»¶ï¼Œä»¥æŸ¥çœ‹è¯¥æ–‡ä»¶å¦‚何映射到基础设施的当å‰çŠ¶æ€ã€‚ + +- å¦‚æžœä½ æƒ³åœ¨ä¸€ä¸ªåœ°æ–¹ç®¡ç† Kubernetes 集群设置ã€Rancher 设置和硬件设置，请使用 [Terraform 模å—](https://github.com/rancher/terraform-modules)。你å¯ä»¥å°†é›†ç¾¤é…ç½® YAML 文件或 RKE 模æ¿é…置文件传递给 Terraform 模å—，以便 Terraform 模å—创建它。在这ç§æƒ…况下，你å¯ä»¥ä½¿ç”¨åŸºç¡€æž¶æž„å³ä»£ç æ¥ç®¡ç† Kubernetes 集群åŠå…¶åº•å±‚硬件的版本控制和修订历å²ã€‚ + +## åˆ›å»ºç¬¦åˆ CIS 基准的集群的技巧 + +本节æ述了一ç§æ–¹æ³•ï¼Œå¯ä»¥ä½¿å®‰å…¨åˆè§„相关的é…置文件æˆä¸ºé›†ç¾¤çš„标准é…置文件。 + +在你创建[ç¬¦åˆ CIS 基准的集群](../../../../pages-for-subheaders/rancher-security.md)时，你有一个加密é…置文件和一个审计日志é…置文件。 + +你的基础设施预é…系统å¯ä»¥å°†è¿™äº›æ–‡ä»¶å†™å…¥ç£ç›˜ã€‚然åŽåœ¨ä½ çš„ RKE 模æ¿ä¸­ï¼Œä½ éœ€è¦æŒ‡å®šè¿™äº›æ–‡ä»¶çš„ä½ç½®ï¼Œç„¶åŽå°†ä½ çš„加密é…置文件和审计日志é…置文件作为é¢å¤–的挂载添加到 `kube-api-server`。 + +然åŽï¼Œä½ éœ€è¦ç¡®ä¿ RKE 模æ¿ä¸­çš„ `kube-api-server` æ ‡å¿—ä½¿ç”¨ç¬¦åˆ CIS çš„é…置文件。 + +通过这ç§æ–¹å¼ï¼Œä½ å¯ä»¥åˆ›å»ºç¬¦åˆ CIS 基准的标志。 + +## èµ„æº + +- [Terraform 文档](https://www.terraform.io/docs/) +- [Rancher2 Terraform Provider 文档](https://www.terraform.io/docs/providers/rancher2/) +- [The RanchCast - 第 1 集：Rancher 2 Terraform Provider](https://youtu.be/YNCq-prI8-8)：在此演示中，社区主管 Jason van Brackel 使用 Rancher 2 Terraform Provider 创建了节点并创建自定义集群。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md new file mode 100644 index 00000000000..6e4adb98239 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md @@ -0,0 +1,161 @@ +--- +title: 创建和修改 RKE æ¨¡æ¿ +--- + +本节介ç»å¦‚ä½•ç®¡ç† RKE 模æ¿å’Œä¿®è®¢ç‰ˆã€‚ä½ å¯ä»¥ä»Ž **RKE1 é…ç½® > RKE 模æ¿**下的**集群管ç†**视图创建ã€å…±äº«ã€æ›´æ–°å’Œåˆ é™¤æ¨¡æ¿ã€‚ + +模æ¿æ›´æ–°é€šè¿‡ä¿®è®¢ç³»ç»Ÿå¤„ç†ã€‚当模æ¿æ‰€æœ‰è€…想è¦æ›´æ”¹æˆ–更新模æ¿æ—¶ï¼Œä»–们会创建模æ¿çš„新版本。å•ä¸ªä¿®è®¢æ— æ³•ç¼–辑。但是，如果你想防止使用修订æ¥åˆ›å»ºæ–°é›†ç¾¤ï¼Œä½ å¯ä»¥ç¦ç”¨å®ƒã€‚ + +ä½ å¯ä»¥ä½¿ç”¨ä¸¤ç§æ–¹å¼æ¥ä½¿ç”¨æ¨¡æ¿ä¿®è®¢ï¼šåˆ›å»ºæ–°é›†ç¾¤ï¼Œæˆ–å‡çº§ä½¿ç”¨è¾ƒæ—©ç‰ˆæœ¬çš„模æ¿åˆ›å»ºçš„集群。模æ¿åˆ›å»ºè€…å¯ä»¥è®¾ç½®é»˜è®¤ä¿®è®¢ç‰ˆï¼Œä½†æ˜¯åœ¨æœ€ç»ˆç”¨æˆ·åˆ›å»ºé›†ç¾¤æ—¶ï¼Œä»–们å¯ä»¥é€‰æ‹©ä»»ä½•æ¨¡æ¿ä»¥åŠå¯ä¾›ä½¿ç”¨çš„任何模æ¿ä¿®è®¢ç‰ˆã€‚使用指定的修订版创建集群åŽï¼Œå°±æ— æ³•å°†å…¶æ›´æ”¹ä¸ºå¦ä¸€ä¸ªæ¨¡æ¿ï¼Œä½†æ˜¯å¯ä»¥å°†é›†ç¾¤å‡çº§ä¸ºåŒä¸€æ¨¡æ¿çš„较新的å¯ç”¨ä¿®è®¢ç‰ˆã€‚ + +模æ¿æ‰€æœ‰è€…对模æ¿ä¿®è®¢ç‰ˆå…·æœ‰å®Œå…¨æŽ§åˆ¶æƒï¼Œå¹¶ä¸”å¯ä»¥åˆ›å»ºæ–°çš„修订版æ¥æ›´æ–°æ¨¡æ¿ï¼Œåˆ é™¤æˆ–ç¦ç”¨ä¸åº”被用于创建集群的修订版，和设置默认的模æ¿ä¿®è®¢ç‰ˆã€‚ + + +### 先决æ¡ä»¶ + +如果你具有**创建 RKE 模æ¿**æƒé™ï¼Œåˆ™å¯ä»¥åˆ›å»º RKE 模æ¿ï¼Œè¯¥æƒé™å¯ç”±[管ç†å‘˜æŽˆäºˆ](creator-permissions.md)。 + +如果你是模æ¿çš„所有者，你å¯ä»¥ä¿®æ”¹ã€å…±äº«å’Œåˆ é™¤æ¨¡æ¿ã€‚有关如何æˆä¸ºæ¨¡æ¿æ‰€æœ‰è€…的详细信æ¯ï¼Œè¯·å‚阅[共享模æ¿æ‰€æœ‰æƒæ–‡æ¡£](access-or-share-templates.md#共享模æ¿æ‰€æœ‰æƒ)。 + +### åˆ›å»ºæ¨¡æ¿ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. 输入模æ¿çš„å称。Rancher å·²ç»ä¸ºæ¨¡æ¿çš„第一个版本自动生æˆäº†å称，该版本与该模æ¿ä¸€èµ·åˆ›å»ºã€‚ +1. å¯é€‰ï¼šé€šè¿‡å°†ç”¨æˆ·æ·»åŠ ä¸ºæˆå‘˜ï¼Œæ¥[与其他用户或组共享模æ¿](access-or-share-templates.md#与特定用户或组共享模æ¿)。你还å¯ä»¥å°†æ¨¡æ¿å…¬å¼€ï¼Œä»Žè€Œä¸Ž Rancher 中的所有人共享。 +1. 然åŽæŒ‰ç…§å±å¹•ä¸Šçš„表格将集群é…ç½®å‚æ•°ä¿å­˜ä¸ºæ¨¡æ¿ä¿®è®¢çš„一部分。å¯ä»¥å°†ä¿®è®¢æ ‡è®°ä¸ºæ­¤æ¨¡æ¿çš„默认值。 + +**结果**：é…置了具有一个修订版的 RKE 模æ¿ã€‚ä½ å¯ä»¥ç¨åŽåœ¨[é…ç½® Rancher å¯åŠ¨çš„集群](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)时使用此 RKE 模æ¿ä¿®è®¢ç‰ˆã€‚通过 RKE 模æ¿ç®¡ç†é›†ç¾¤åŽï¼Œé›†ç¾¤æ— æ³•è§£é™¤ä¸Žæ¨¡æ¿çš„绑定，并且无法å–消选中**使用现有 RKE 模æ¿å’Œä¿®è®¢ç‰ˆ**。 + +### æ›´æ–°æ¨¡æ¿ + +æ›´æ–° RKE 模æ¿ç›¸å½“于创建现有模æ¿çš„修订版。使用旧版本模æ¿åˆ›å»ºçš„集群å¯ä»¥è¿›è¡Œæ›´æ–°ï¼Œä»Žè€ŒåŒ¹é…新版本。 + +ä½ ä¸èƒ½ç¼–辑å•ä¸ªä¿®è®¢ã€‚由于你无法编辑模æ¿çš„å•ä¸ªä¿®è®¢ï¼Œä¸ºäº†é˜²æ­¢ä½¿ç”¨æŸä¸ªä¿®è®¢ï¼Œä½ å¯ä»¥[ç¦ç”¨è¯¥ä¿®è®¢ç‰ˆ](#ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆ)。 + +创建新模æ¿ä¿®è®¢æ—¶ï¼Œä½¿ç”¨æ—§æ¨¡æ¿ä¿®è®¢çš„集群ä¸å—å½±å“。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦ç¼–辑的模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 编辑**。 +1. 编辑所需信æ¯å¹¶å•å‡»**ä¿å­˜**。 +1. å¯é€‰ï¼šä½ å¯ä»¥æ›´æ”¹æ­¤æ¨¡æ¿çš„默认修订版，也å¯ä»¥æ›´æ”¹å…±äº«å¯¹è±¡ã€‚ + +**结果**：模æ¿å·²æ›´æ–°ã€‚è¦å°†å…¶åº”用到使用旧版本模æ¿çš„集群，请å‚阅[å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ](#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)。 + +### åˆ é™¤æ¨¡æ¿ + +当ä¸å†éœ€è¦ä¸ºä»»ä½•é›†ç¾¤ä½¿ç”¨æŸä¸ª RKE 模æ¿æ—¶ï¼Œå¯ä»¥å°†å…¶åˆ é™¤ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦åˆ é™¤çš„ RKE 模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 删除**。 +1. 确认删除。 + +**结果**：模æ¿è¢«åˆ é™¤ã€‚ + +### 基于默认版创建新修订版 + +ä½ å¯ä»¥å¤åˆ¶é»˜è®¤æ¨¡æ¿ä¿®è®¢ç‰ˆå¹¶å¿«é€Ÿæ›´æ–°å…¶è®¾ç½®ï¼Œè€Œæ— éœ€ä»Žå¤´å¼€å§‹åˆ›å»ºæ–°ä¿®è®¢ç‰ˆã€‚克隆模æ¿ä¸ºä½ çœåŽ»äº†é‡æ–°è¾“入集群创建所需的访问密钥和其他å‚数的麻烦。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦å…‹éš†çš„ RKE 模æ¿ï¼Œç„¶åŽå•å‡» **â‹® > 基于默认版创建新修订版**。 +1. 填写表å•çš„其余部分æ¥åˆ›å»ºæ–°ä¿®è®¢ã€‚ + +**结果**：克隆并é…置了 RKE 模æ¿ä¿®è®¢ç‰ˆã€‚ + +### 基于克隆版创建新修订版 + +通过用户设置创建新的 RKE 模æ¿ä¿®è®¢ç‰ˆæ—¶ï¼Œå¯ä»¥å…‹éš†çŽ°æœ‰ä¿®è®¢ç‰ˆå¹¶å¿«é€Ÿæ›´æ–°å…¶è®¾ç½®ï¼Œè€Œæ— éœ€ä»Žå¤´å¼€å§‹åˆ›å»ºæ–°çš„修订版。克隆模æ¿ä¿®è®¢çœåŽ»äº†é‡æ–°è¾“入集群å‚数的麻烦。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦å…‹éš†çš„模æ¿ä¿®è®¢ã€‚然åŽé€‰æ‹© **â‹® > 克隆修订版**。 +1. 填写表å•çš„其余部分。 + +**结果**：克隆并é…置了 RKE 模æ¿ä¿®è®¢ç‰ˆã€‚ä½ å¯ä»¥åœ¨é…置集群时使用 RKE 模æ¿ä¿®è®¢ã€‚任何使用此 RKE 模æ¿çš„现有集群都å¯ä»¥å‡çº§åˆ°æ­¤æ–°ç‰ˆæœ¬ã€‚ + +### ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆ + +当你ä¸éœ€è¦å°† RKE 模æ¿ä¿®è®¢ç‰ˆæœ¬ç”¨äºŽåˆ›å»ºæ–°é›†ç¾¤æ—¶ï¼Œå¯ä»¥ç¦ç”¨æ¨¡æ¿ä¿®è®¢ç‰ˆã€‚你也å¯ä»¥é‡æ–°å¯ç”¨ç¦ç”¨äº†çš„修订版。 + +如果没有任何集群使用该修订，你å¯ä»¥ç¦ç”¨è¯¥ä¿®è®¢ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦ç¦ç”¨çš„模æ¿ä¿®è®¢ç‰ˆã€‚然åŽé€‰æ‹© **â‹® > ç¦ç”¨**。 + +**结果**：RKE 模æ¿ä¿®è®¢ç‰ˆä¸èƒ½ç”¨äºŽåˆ›å»ºæ–°é›†ç¾¤ã€‚ + +### é‡æ–°å¯ç”¨ç¦ç”¨çš„模æ¿ä¿®è®¢ç‰ˆ + +如果è¦ä½¿ç”¨å·²ç¦ç”¨çš„ RKE 模æ¿ä¿®è®¢ç‰ˆæ¥åˆ›å»ºæ–°é›†ç¾¤ï¼Œä½ å¯ä»¥é‡æ–°å¯ç”¨è¯¥ä¿®è®¢ç‰ˆã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在 **RKE1 é…ç½®**下，å•å‡» **RKE 模æ¿**。 +1. 转到è¦é‡æ–°å¯ç”¨çš„模æ¿ä¿®è®¢ã€‚然åŽé€‰æ‹© **â‹® > å¯ç”¨**。 + +**结果**：RKE 模æ¿ä¿®è®¢ç‰ˆå¯ç”¨äºŽåˆ›å»ºæ–°é›†ç¾¤ã€‚ + +### 将模æ¿ä¿®è®¢ç‰ˆè®¾ç½®ä¸ºé»˜è®¤ + +当最终用户使用 RKE 模æ¿åˆ›å»ºé›†ç¾¤æ—¶ï¼Œä»–们å¯ä»¥é€‰æ‹©ä½¿ç”¨å“ªä¸ªç‰ˆæœ¬æ¥åˆ›å»ºé›†ç¾¤ã€‚ä½ å¯ä»¥é…置默认使用的版本。 + +è¦å°† RKE 模æ¿ä¿®è®¢ç‰ˆè®¾ç½®ä¸ºé»˜è®¤ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦è®¾ä¸ºé»˜è®¤çš„ RKE 模æ¿ä¿®è®¢ç‰ˆï¼Œç„¶åŽå•å‡» **â‹® > 设为默认é…ç½®**。 + +**结果**：使用模æ¿åˆ›å»ºé›†ç¾¤æ—¶ï¼ŒRKE 模æ¿ä¿®è®¢ç‰ˆå°†ç”¨ä½œé»˜è®¤é€‰é¡¹ã€‚ + +### 删除模æ¿ä¿®è®¢ç‰ˆ + +ä½ å¯ä»¥åˆ é™¤æ¨¡æ¿çš„所有修订（默认修订除外）。 + +è¦æ°¸ä¹…删除修订版： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ï¼Œå•å‡» **RKE1 é…ç½® > RKE 模æ¿**。 +1. 转到è¦åˆ é™¤çš„ RKE 模æ¿ä¿®è®¢ç‰ˆï¼Œç„¶åŽå•å‡» **â‹® > 删除**。 + +**结果**：RKE 模æ¿ä¿®è®¢ç‰ˆè¢«åˆ é™¤ã€‚ + +### å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ + +:::note + +本部分å‡è®¾ä½ å·²ç»æœ‰ä¸€ä¸ªé›†ç¾¤ï¼Œè¯¥é›†ç¾¤[应用了 RKE 模æ¿](apply-templates.md)。 + +本部分还å‡è®¾ä½ å·²[更新了集群使用的模æ¿](#更新模æ¿)，以便å¯ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆã€‚ + +::: + +è¦å°†é›†ç¾¤å‡çº§åˆ°ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**集群选项**中，å•å‡»æ¨¡æ¿ä¿®è®¢ç‰ˆçš„下拉èœå•ï¼Œç„¶åŽé€‰æ‹©æ–°çš„模æ¿ä¿®è®¢ç‰ˆã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：集群已å‡çº§ä¸ºä½¿ç”¨æ–°æ¨¡æ¿ä¿®è®¢ç‰ˆä¸­å®šä¹‰çš„设置。 + +### 将正在è¿è¡Œçš„集群导出到新的 RKE 模æ¿å’Œä¿®è®¢ç‰ˆ + +ä½ å¯ä»¥å°†çŽ°æœ‰é›†ç¾¤çš„设置ä¿å­˜ä¸º RKE 模æ¿ã€‚ + +这将把集群的设置导出为新的 RKE 模æ¿ï¼Œå¹¶ä¸”将集群绑定到该模æ¿ã€‚然åŽï¼Œåªæœ‰[更新了模æ¿](#更新模æ¿)并且集群å‡çº§åˆ°[使用更新版本的模æ¿](#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)时，集群æ‰èƒ½æ”¹å˜ã€‚ + +è¦å°†çŽ°æœ‰é›†ç¾¤è½¬æ¢ä¸ºä½¿ç”¨ RKE 模æ¿ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到将被转æ¢ä¸ºä½¿ç”¨ RKE 模æ¿çš„é›†ç¾¤ï¼Œç„¶åŽ **â‹® > ä¿å­˜ä¸º RKE 模æ¿**。 +1. 在出现的表å•ä¸­è¾“å…¥ RKE 模æ¿çš„å称，然åŽå•å‡»**创建**。 + +**结果**： + +- 创建了一个新的 RKE 模æ¿ã€‚ +- 将集群转æ¢ä¸ºä½¿ç”¨è¯¥æ–°æ¨¡æ¿ã€‚ +- å¯ä»¥[使用新模æ¿å’Œä¿®è®¢ç‰ˆåˆ›å»ºæ–°é›†ç¾¤ã€‚](apply-templates.md#使用-rke-模æ¿åˆ›å»ºé›†ç¾¤) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md new file mode 100644 index 00000000000..11ecac6fabd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md @@ -0,0 +1,14 @@ +--- +title: 覆盖模æ¿è®¾ç½® +--- + +用户创建 RKE 模æ¿æ—¶ï¼Œæ¨¡æ¿ä¸­çš„æ¯ä¸ªè®¾ç½®åœ¨ Rancher UI 中都有一个开关，指示用户是å¦å¯ä»¥è¦†ç›–该设置。此开关将这些设置标记为**å…许用户覆盖**。打开开关表示用户å¯ä»¥ä¿®æ”¹å¯¹åº”çš„å‚数，关闭开关表示用户无æƒä¿®æ”¹å¯¹åº”çš„å‚数。 + +使用模æ¿åˆ›å»ºé›†ç¾¤åŽï¼Œé™¤éžæ¨¡æ¿æ‰€æœ‰è€…将设置标记为**å…许用户覆盖**，å¦åˆ™æœ€ç»ˆç”¨æˆ·æ— æ³•æ›´æ–°æ¨¡æ¿ä¸­å®šä¹‰çš„任何设置。但是，如果模æ¿[更新到新修订版](manage-rke1-templates.md)，且该修订版更改了设置或å…许最终用户更改设置，则集群å¯ä»¥å‡çº§åˆ°æ¨¡æ¿çš„新修订版，并且新修订版中的更改将应用于集群。 + +如果 RKE 模æ¿ä¸Šçš„任何å‚数设置为**å…许用户覆盖**，最终用户必须在集群创建期间设置这些字段，然åŽä»–们å¯ä»¥éšæ—¶ç¼–辑这些设置。 + +RKE 模æ¿çš„**å…许用户覆盖**选项的适用场景如下： + +- 管ç†å‘˜è®¤ä¸ºæŸäº›å‚数需è¦ä¿æŒçµæ´»æ€§ï¼Œä»¥ä¾¿éšæ—¶æ›´æ–°ã€‚ +- 最终用户将需è¦è¾“入他们自己的访问密钥或密文密钥，例如，云凭è¯æˆ–备份快照的凭è¯ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md new file mode 100644 index 00000000000..704c47a68c5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md @@ -0,0 +1,219 @@ +--- +title: é…ç½® Active Directory (AD) +--- + +如果你的组织使用 Microsoft Active Directory 作为中心用户仓库，你å¯ä»¥å°† Rancher é…置为与 Active Directory æœåŠ¡å™¨é€šä¿¡ï¼Œä»Žè€Œå¯¹ç”¨æˆ·è¿›è¡Œèº«ä»½éªŒè¯ã€‚这使 Rancher 管ç†å‘˜å¯ä»¥å¯¹å¤–部用户系统中的用户和组进行集群和项目的访问控制，åŒæ—¶å…许最终用户在登录 Rancher UI 时使用 Active Directory 凭è¯è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +Rancher 使用 LDAP 与 Active Directory æœåŠ¡å™¨é€šä¿¡ã€‚因此，Active Directory 与 [OpenLDAP 身份验è¯](../../../../pages-for-subheaders/configure-openldap.md)çš„æµç¨‹ç›¸åŒã€‚ + +:::note + +在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + +::: + +## 先决æ¡ä»¶ + +你需è¦åˆ›å»ºæˆ–从你的 AD 管ç†å‘˜å¤„获å–一个新的 AD 用户，用作 Rancher çš„ ServiceAccount。此用户必须具有足够的æƒé™ï¼Œæ‰èƒ½æ‰§è¡Œ LDAP æœç´¢å¹¶è¯»å–ä½ çš„ AD 域下的用户和组的属性。 + +通常å¯ä»¥ä½¿ç”¨ï¼ˆéžç®¡ç†å‘˜ï¼‰**域用户**è´¦å·ï¼Œå› ä¸ºé»˜è®¤æƒ…况下，该用户对域分区中的大多数对象具有åªè¯»ç‰¹æƒã€‚ + +但是，请注æ„，在æŸäº›é”定的 Active Directory é…置中，此默认æ“作å¯èƒ½ä¸é€‚用。在这ç§æƒ…况下，你需è¦ç¡®ä¿ ServiceAccount 用户在 Base OU（包å«ç”¨æˆ·å’Œç»„）上或全局范围内至少拥有域的 **Read** å’Œ **List Content** æƒé™ã€‚ + +:::note 使用 TLS？ + +- 如果 AD æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +- å‡çº§åˆ° v2.6.0 åŽï¼Œå¦‚æžœ AD æœåŠ¡å™¨ä¸Šçš„è¯ä¹¦ä¸æ”¯æŒ SAN 属性，则使用 TLS 通过 Rancher 对 Active Directory 进行身份验è¯å¯èƒ½ä¼šå¤±è´¥ã€‚这是 Go v1.15 中默认å¯ç”¨çš„检查。 + + - 收到"Error creating SSL connection: LDAP Result Code 200 "Network Error": x509 错误：è¯ä¹¦ä¾èµ–于旧的通用å称（Common Name）字段，使用 SAN 或临时å¯ç”¨ä¸Ž GODEBUG=x509ignoreCN=0 匹é…的通用å称。 + + - è¦è§£å†³æ­¤é”™è¯¯ï¼Œè¯·ä½¿ç”¨æ”¯æŒ SAN 属性的新è¯ä¹¦æ›´æ–°æˆ–æ›¿æ¢ AD æœåŠ¡å™¨ä¸Šçš„è¯ä¹¦ã€‚或者，将 `GODEBUG=x509ignoreCN=0` 设置为 Rancher Server 容器的环境å˜é‡æ¥å¿½ç•¥æ­¤é”™è¯¯ã€‚ + +::: + +## é…置步骤 +### 打开 Active Directory é…ç½® + +1. 使用åˆå§‹çš„本地 `admin` è´¦å·ç™»å½•åˆ° Rancher UI。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **ActiveDirectory**。然åŽä¼šæ˜¾ç¤º**验è¯æ供程åºï¼šActiveDirectory** 的表å•ã€‚ +1. 填写表å•ã€‚如果需è¦èŽ·å–帮助，请å‚è§ä¸‹æ–¹çš„é…置选项详情。 +1. 点击**å¯ç”¨**。 + +### é…ç½® Active Directory æœåŠ¡å™¨ + +在 `1. é…ç½® Active Directory æœåŠ¡å™¨` 的步骤中，使用你 Active Directory 的实际信æ¯å®Œæˆå­—段é…置。有关æ¯ä¸ªå‚数所需值的详细信æ¯ï¼Œè¯·å‚阅下表。 + +:::note + +如果你ä¸ç¡®å®šè¦åœ¨ç”¨æˆ·/组`æœç´¢åº“`字段中输入什么值，请å‚è§[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema)。 + +::: + +**表 1：AD æœåŠ¡å™¨å‚æ•°** + +| å‚æ•° | æè¿° | +|:--|:--| +| 主机å | 指定 AD æœåŠ¡å™¨çš„主机å或 IP 地å€ã€‚ | +| ç«¯å£ | 指定 AD æœåŠ¡å™¨ç›‘å¬è¿žæŽ¥çš„端å£ã€‚未加密的 LDAP 通常使用 389 的标准端å£ï¼Œè€Œ LDAPS 使用 636 端å£ã€‚ | +| TLS | 选中此框å¯å¯ç”¨ SSL/TLS 上的 LDAP（通常称为 LDAPS）。 | +| æœåŠ¡å™¨è¿žæŽ¥è¶…æ—¶ | Rancher 在认为无法访问 AD æœåŠ¡å™¨ä¹‹å‰ç­‰å¾…的时间（秒）。 | +| ServiceAccount 用户å | 输入对域分区具有åªè¯»è®¿é—®æƒé™çš„ AD è´¦å·çš„用户å（å‚è§[先决æ¡ä»¶](#先决æ¡ä»¶)）。用户åå¯ä»¥ç”¨ NetBIOS（例如 "DOMAIN\serviceaccount"）或 UPN æ ¼å¼ï¼ˆä¾‹å¦‚ "serviceaccount@domain.com"）。 | +| ServiceAccount å¯†ç  | ServiceAccount 的密ç ã€‚ | +| 默认登录域 | 如果你使用 AD 域的 NetBIOS å称é…置此字段，在绑定到 AD æœåŠ¡å™¨æ—¶ï¼Œæ²¡æœ‰åŒ…å«åŸŸçš„用户å（例如“jdoeâ€ï¼‰å°†è‡ªåŠ¨è½¬æ¢ä¸ºå¸¦æ–œæ çš„ NetBIOS 登录（例如，“LOGIN_DOMAIN\jdoeâ€ï¼‰ã€‚如果你的用户以 UPN（例如，"jdoe@acme.com"）作为用户å进行身份验è¯ï¼Œåˆ™æ­¤å­—段必须**å¿…é¡»**留空。 | +| 用户æœç´¢åº“ | 输入目录树中开始æœç´¢ç”¨æˆ·å¯¹è±¡çš„节点的标识å称（DN）。所有用户都必须是此基础标识å称的åŽä»£ã€‚例如，"ou=people,dc=acme,dc=com"。 | +| 组æœç´¢åº“ | 如果组ä½äºŽ`用户æœç´¢åº“`下é…置的节点之外的其他节点下，则需è¦åœ¨æ­¤å¤„æ供标识å称。å¦åˆ™è¯·ç•™ç©ºã€‚例如："ou=groups,dc=acme,dc=com"。 | + +--- + +### é…置用户/组 Schema + +在 `2. 自定义 Schema` 中，你必须为 Rancher æ供与目录中使用的 Schema 对应的用户和组属性的正确映射。 + +Rancher 使用 LDAP 查询æ¥æœç´¢å’Œæ£€ç´¢å…³äºŽ Active Directory 中的用户和组的信æ¯ã€‚本节中é…置的属性映射用于构造æœç´¢ç­›é€‰å™¨å’Œè§£æžç»„æˆå‘˜èº«ä»½ã€‚因此，所æ供的设置需è¦å映你 AD 域的真实情况。 + +:::note + +如果你ä¸ç†Ÿæ‚‰ Active Directory 域中使用的 Schema，请å‚è§[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema) æ¥ç¡®å®šæ­£ç¡®çš„é…置值。 + +::: + +#### 用户 Schema + +下表详细说明了用户 Schema é…置的å‚数。 + +**表 2：用户 Schema é…ç½®å‚æ•°** + +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于用户对象的对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Username Attribute | 用户属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Login Attribute | 登录属性的值与用户登录 Rancher 时输入的凭è¯çš„用户å部分匹é…。如果你的用户以他的 UPN（例如 "jdoe@acme.com"）作为用户å进行身份验è¯ï¼Œåˆ™æ­¤å­—段通常必须设置为 `userPrincipalName`。å¦åˆ™ï¼Œå¯¹äºŽæ—§çš„ NetBIOS 风格的登录å（例如 "jdoe"），则通常设为 `sAMAccountName`。 | +| User Member Attribute | 包å«ç”¨æˆ·æ‰€å±žç»„的属性。 | +| Search Attribute | 当用户输入文本以在用户界é¢ä¸­æ·»åŠ ç”¨æˆ·æˆ–组时，Rancher 会查询 AD æœåŠ¡å™¨ï¼Œå¹¶å°è¯•æ ¹æ®æ­¤è®¾ç½®ä¸­æ供的属性匹é…用户。å¯ä»¥é€šè¿‡ä½¿ç”¨ç®¡é“（“\|â€ï¼‰ç¬¦å·åˆ†éš”属性æ¥æŒ‡å®šå¤šä¸ªå±žæ€§ã€‚è¦åŒ¹é… UPN 用户å（例如 jdoe@acme.com），通常应将此字段的值设置为 `userPrincipalName`。 | +| Search Filter | 当 Rancher å°è¯•å°†ç”¨æˆ·æ·»åŠ åˆ°ç½‘站访问列表，或å°è¯•å°†æˆå‘˜æ·»åŠ åˆ°é›†ç¾¤æˆ–项目时，此筛选器将应用于æœç´¢çš„用户列表。例如，用户æœç´¢ç­›é€‰å™¨å¯èƒ½æ˜¯ (|(memberOf=CN=group1,CN=Users,DC=testad,DC=rancher,DC=io)(memberOf=CN=group2,CN=Users,DC=testad,DC=rancher,DC=io))。注æ„：如果æœç´¢ç­›é€‰å™¨æœªä½¿ç”¨[有效的 AD æœç´¢è¯­æ³•](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax)，则用户列表将为空。 | +| User Enabled Attribute | 该属性是一个整数值，代表用户账å·æ ‡å¿—的枚举。Rancher 使用此选项æ¥ç¡®å®šç”¨æˆ·è´¦å·æ˜¯å¦å·²ç¦ç”¨ã€‚通常应该将此å‚数设置为 AD 标准的 `userAccountControl`。 | +| Disabled Status Bitmask | 指定的ç¦ç”¨ç”¨æˆ·è´¦å·çš„ `User Enabled Attribute` 的值。通常，你应该将此å‚数设置为 Microsoft Active Directory Schema 中指定的默认值 2（请å‚è§[此处](https://docs.microsoft.com/en-us/windows/desktop/adschema/a-useraccountcontrol#remarks)）。 | + +--- + +#### 组 Schema + +下表详细说明了组 Schema é…置的å‚数。 + +**表 3：组 Schema é…ç½®å‚æ•°** + +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于组对象的对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Name Attribute | å称属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Group Member User Attribute | **用户属性**çš„å称。它的格å¼ä¸Ž `Group Member Mapping Attribute` 中的组æˆå‘˜åŒ¹é…。 | +| Group Member Mapping Attribute | 包å«ç»„æˆå‘˜çš„组属性的å称。 | +| Search Attribute | 在将组添加到集群或项目时，用于构造æœç´¢ç­›é€‰å™¨çš„属性。请å‚è§ç”¨æˆ· Schema çš„ `Search Attribute`。 | +| Search Filter | 当 Rancher å°è¯•å°†ç»„添加到网站访问列表，或将组添加到集群或项目时，此筛选器将应用于æœç´¢çš„组列表。例如，组æœç´¢ç­›é€‰å™¨å¯ä»¥æ˜¯ (|(cn=group1)(cn=group2))。注æ„：如果æœç´¢ç­›é€‰å™¨æœªä½¿ç”¨[有效的 AD æœç´¢è¯­æ³•](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax)，则组列表将为空。 | +| Group DN Attribute | 组属性的å称，其格å¼ä¸Žæ述用户æˆå‘˜èº«ä»½çš„用户属性中的值匹é…。å‚è§ `User Member Attribute`。 | +| Nested Group Membership | 此设置定义 Rancher 是å¦åº”解æžåµŒå¥—组æˆå‘˜èº«ä»½ã€‚仅当你的组织使用这些嵌套æˆå‘˜èº«ä»½æ—¶æ‰ä½¿ç”¨ï¼ˆå³ä½ æœ‰åŒ…å«å…¶ä»–组作为æˆå‘˜çš„组。我们建议尽é‡é¿å…使用嵌套组，从而é¿å…在存在大é‡åµŒå¥—æˆå‘˜æ—¶å‡ºçŽ°æ½œåœ¨çš„性能问题）。 | + +--- + +### æµ‹è¯•èº«ä»½éªŒè¯ + +完æˆé…ç½®åŽï¼Œè¯·**使用你的 AD 管ç†å‘˜è´¦æˆ·**测试与 AD æœåŠ¡å™¨çš„连接。如果测试æˆåŠŸï¼Œå°†å¯ç”¨é…置的 Active Directory 身份验è¯ï¼Œæµ‹è¯•æ—¶ä½¿ç”¨çš„è´¦å·ä¼šæˆä¸ºç®¡ç†å‘˜ã€‚ + +:::note + +与此步骤中输入的凭è¯ç›¸å…³çš„ AD 用户将映射到本地主体账å·ï¼Œå¹¶åœ¨ Rancher 中分é…系统管ç†å‘˜æƒé™ã€‚因此，你应该决定使用哪个 AD è´¦å·æ¥æ‰§è¡Œæ­¤æ­¥éª¤ã€‚ + +::: + +1. 输入应映射到本地主体账å·çš„ AD è´¦å·çš„**用户å**å’Œ**密ç ** 。 +2. 点击**å¯ç”¨ Active Directory 认è¯**æ¥å®Œæˆè®¾ç½®ã€‚ + +**结果**： + +- å·²å¯ç”¨ Active Directory 身份验è¯ã€‚ +- 你已使用 AD 凭è¯ä»¥ç³»ç»Ÿç®¡ç†å‘˜èº«ä»½ç™»å½•åˆ° Rancher。 + +:::note + +如果 LDAP æœåŠ¡ä¸­æ–­ï¼Œä½ ä»ç„¶å¯ä»¥ä½¿ç”¨æœ¬åœ°é…置的 `admin` è´¦å·å’Œå¯†ç ç™»å½•ã€‚ + +::: + +## 附录：使用 ldapsearch 确定æœç´¢åº“å’Œ Schema + +为了æˆåŠŸé…ç½® AD 身份验è¯ï¼Œä½ å¿…é¡»æä¾› AD æœåŠ¡å™¨çš„层次结构和 Schema 的正确é…置。 + +[`ldapsearch`](https://manpages.ubuntu.com/manpages/kinetic/en/man1/ldapsearch.1.html) 工具å…许你查询你的 AD æœåŠ¡å™¨ï¼Œä»Žè€Œäº†è§£ç”¨äºŽç”¨æˆ·å’Œç»„对象的 Schema。 + +在下é¢çš„示例命令中，我们å‡è®¾ï¼š + +- Active Directory æœåŠ¡å™¨çš„主机å是 `ad.acme.com`。 +- æœåŠ¡å™¨æ­£åœ¨ç›‘å¬ç«¯å£ `389` 上的未加密连接。 +- Active Directory 的域是 `acme`。 +- 你有一个用户å为 `jdoe`，密ç ä¸º `secret` 的有效 AD è´¦å·ã€‚ + +### 确认æœç´¢åº“ + +首先，我们将使用 `ldapsearch` æ¥æ‰¾åˆ°ç”¨æˆ·å’Œç»„的父节点的标识å称： + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "dc=acme,dc=com" -s sub "sAMAccountName=jdoe" +``` + +此命令执行 LDAP æœç´¢ï¼Œæœç´¢èµ·ç‚¹è®¾ç½®ä¸ºåŸŸæ ¹ç›®å½•ï¼ˆ`-b "dc=acme,dc=com"`），并执行针对用户账å·ï¼ˆ`sAMAccountNam=jdoe`）的筛选器，返回所述用户的属性： + +![](/img/ldapsearch-user.png) + +因为在这ç§æƒ…况下，用户的 DN 是 `CN=John Doe,CN=Users,DC=acme,DC=com` [5]，所以我们应该使用父节点 DN `CN=Users,DC=acme,DC=com` æ¥é…ç½®**用户æœç´¢åº“**。 + +åŒæ ·ï¼ŒåŸºäºŽ **memberOf** 属性 [4] 中引用的组的 DN，**组æœç´¢åº“**çš„å€¼å°†æ˜¯è¯¥å€¼çš„çˆ¶èŠ‚ç‚¹ï¼Œå³ `OU=Groups,DC=acme,DC=com`。 + +### 确定用户 Schema + +上述 `ldapsearch` 查询的输出还能用于确定在用户 Schema é…置中使用的值： + +- `Object Class`：**person** [1] +- `Username Attribute`:：**name** [2] +- `Login Attribute`：**sAMAccountName** [3] +- `User Member Attribute`：**memberOf** [4] + +:::note + +如果我们组织中的 AD 用户使用其 UPN（例如 `jdoe@acme.com`）而ä¸æ˜¯çŸ­ç™»å½•å进行身份验è¯ï¼Œåˆ™å¿…须将 `Login Attribute` 设置为 **userPrincipalName**。 + +::: + +我们还将 `Search Attribute` 数设置为 **sAMAccountName|name**。这样，用户å¯ä»¥é€šè¿‡è¾“入用户å或全å添加到 Rancher UI 中的集群/项目中。 + +### 确定组 Schema + +接下æ¥ï¼Œæˆ‘们将查询与此用户关è”的一个组，在本例中为 `CN=examplegroup,OU=Groups,DC=acme,DC=com`： + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "ou=groups,dc=acme,dc=com" \ +-s sub "CN=examplegroup" +``` + +此命令将告知我们用于组对象的属性： + +![](/img/ldapsearch-group.png) + +åŒæ ·ï¼Œè¿™èƒ½è®©æˆ‘们确定è¦åœ¨ç»„ Schema é…置中输入的值： + +- `Object Class`：**group** [1] +- `Name Attribute`：**name** [2] +- `Group Member Mapping Attribute`：**member** [3] +- `Search Attribute`：**sAMAccountName** [4] + +查看 **member** 属性的值，我们å¯ä»¥çœ‹åˆ°å®ƒåŒ…å«è¢«å¼•ç”¨ç”¨æˆ·çš„ DN。这对应我们的用户对象中的 **distinguishedName** 属性。因此，必须将 `Group Member User Attribute` å‚数的值设置为此属性。 + +åŒæ ·ï¼Œæˆ‘们å¯ä»¥çœ‹åˆ°ç”¨æˆ·å¯¹è±¡ä¸­ **memberOf** 属性中的值对应组的 **distinguishedName** [5]。因此，我们需è¦å°† `Group DN Attribute` å‚数的值设置为此属性。 + +## 附录：故障排除 + +如果在测试与 Active Directory æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·é¦–先仔细检查为 ServiceAccount 输入的凭è¯ä»¥åŠæœç´¢åº“é…置。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md new file mode 100644 index 00000000000..77d51e007bf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md @@ -0,0 +1,326 @@ +--- +title: é…ç½® Azure AD +--- + +## Microsoft Graph API + +Microsoft Graph API 现在是设置 Azure AD çš„æµç¨‹ã€‚下文将帮助[新用户](#新用户设置)使用新实例æ¥é…ç½® Azure AD，并帮助现有 Azure 应用所有者[è¿ç§»åˆ°æ–°æµç¨‹](#从-azure-ad-graph-api-è¿ç§»åˆ°-microsoft-graph-api)。 + +Rancher 中的 Microsoft Graph API æµç¨‹æ­£åœ¨ä¸æ–­å‘展。建议你使用最新的 2.7 è¡¥ä¸ç‰ˆæœ¬ï¼Œè¯¥ç‰ˆæœ¬ä»åœ¨ç§¯æžå¼€å‘中，并将æŒç»­èŽ·å¾—新功能和改进。 + +### 新用户设置 + +如果你在 Azure 中托管了一个 Active Directory（AD）实例，你å¯ä»¥å°† Rancher é…置为å…许你的用户使用 AD è´¦å·ç™»å½•ã€‚你需è¦åœ¨ Azure å’Œ Rancher 中进行 Azure AD 外部身份验è¯ã€‚ + +:::note 注æ„事项 + +- Azure AD 集æˆä»…支æŒæœåŠ¡æ供商å‘起的登录。 +- 大部分æ“作是从 [Microsoft Azure 门户](https://portal.azure.com/)执行的。 + +::: + +#### Azure Active Directory é…置概述 + +è¦å°† Rancher é…置为å…许用户使用其 Azure AD è´¦å·è¿›è¡Œèº«ä»½éªŒè¯ï¼Œä½ éœ€è¦æ‰§è¡Œå¤šä¸ªæ­¥éª¤ã€‚在你开始之å‰ï¼Œè¯·æŸ¥çœ‹ä¸‹æ–‡æ“作步骤大纲。 + +:::tip + +在开始之å‰ï¼Œæ‰“开两个æµè§ˆå™¨é€‰é¡¹å¡ï¼šä¸€ä¸ªç”¨äºŽ Rancher，å¦ä¸€ä¸ªç”¨äºŽ Azure 门户。这样，你å¯ä»¥å°†é—¨æˆ·çš„é…置值å¤åˆ¶å¹¶ç²˜è´´åˆ° Rancher 中。 + +::: + + +#### 1. 在 Azure 注册 Rancher + +在 Rancher 中å¯ç”¨ Azure AD 之å‰ï¼Œä½ å¿…é¡»å…ˆå‘ Azure 注册 Rancher。 + +1. 以管ç†ç”¨æˆ·èº«ä»½ç™»å½• [Microsoft Azure](https://portal.azure.com/)。åŽç»­é…置步骤中需è¦ç®¡ç†è®¿é—®æƒé™ã€‚ + +1. 使用æœç´¢åŠŸèƒ½æ‰“å¼€ **App registrations** æœåŠ¡ã€‚ + +1. 点击 **New registration** 并填写表å•ã€‚ + + ![New App Registration](/img/new-app-registration.png) + + 1. 输入 **Name**（例如 `Rancher`）。 + + + 1. 在 **Supported account types** 中，选择 **Accounts in this organizational directory only (AzureADTest only - Single tenant)**。这对应于旧版应用注册选项。 + + :::note + + 在更新åŽçš„ Azure 门户中，Redirect URI 与 Reply URL çš„æ„æ€ç›¸åŒã€‚为了将 Azure AD 与 Rancher 一起使用，你必须将 Rancher 列入 Azure 白åå•ï¼ˆä¹‹å‰é€šè¿‡ Reply URL 完æˆï¼‰ã€‚因此，请确ä¿ä½¿ç”¨ä½ çš„ Rancher Server URL 填写 Redirect URI，以包å«ä¸‹é¢åˆ—出的验è¯è·¯å¾„。 + + ::: + + 1. 在 [**Redirect URI**](https://docs.microsoft.com/en-us/azure/active-directory/develop/reply-url) 中，确ä¿ä»Žä¸‹æ‹‰åˆ—表中选择 **Web**，并在æ—边的文本框中输入 Rancher Server çš„ URL。Rancher Server URL åŽéœ€è¦è¿½åŠ éªŒè¯è·¯å¾„，例如 `/verify-auth-azure`。 + + :::tip + + ä½ å¯ä»¥åœ¨ Azure AD 身份验è¯é¡µé¢ï¼ˆå…¨å±€è§†å›¾ > Authentication > Web）中找到你 Rancher 中的个性化 Azure Redirect URI（Reply URL）。 + + ::: + + 1. å•å‡» **Register**。 + +:::note + +此更改å¯èƒ½éœ€è¦äº”分钟æ‰èƒ½ç”Ÿæ•ˆã€‚因此，如果在é…ç½® Azure AD 之åŽæ— æ³•ç«‹å³è¿›è¡Œèº«ä»½éªŒè¯ï¼Œè¯·ä¸è¦æƒŠæ…Œã€‚ + +::: + +#### 2. 创建客户端密文 + +在 Azure 门户中，创建一个客户端密文。Rancher å°†ä½¿ç”¨æ­¤å¯†é’¥å‘ Azure AD 进行身份验è¯ã€‚ + +1. 使用æœç´¢åŠŸèƒ½æ‰“å¼€ **App registrations** æœåŠ¡ã€‚然åŽæ‰“开你在上一个步骤中创建的 Rancher 项。 + + ![Open Rancher Registration](/img/open-rancher-app-reg.png) + +1. 在导航窗格中，å•å‡» **Certificates & secrets**。 + +1. å•å‡» **New client secret**。 + ![创建新的客户端密文](/img/new-client-secret.png) +1. 输入 **Description**（例如 `Rancher`）。 +1. 从 **Expires** 下的选项中选择æŒç»­æ—¶é—´ã€‚此下拉èœå•è®¾ç½®çš„是密钥的到期日期。日期越短则越安全，但需è¦ä½ æ›´é¢‘ç¹åœ°åˆ›å»ºæ–°å¯†é’¥ã€‚ + 请注æ„ï¼Œå¦‚æžœæ£€æµ‹åˆ°åº”ç”¨ç¨‹åº Secret 已过期，用户将无法登录 Rancher。为é¿å…此问题，请在 Azure ä¸­è½®æ¢ Secret 并在过期å‰åœ¨ Rancher 中更新它。 +1. å•å‡» **Add**（无需输入值，ä¿å­˜åŽä¼šè‡ªåŠ¨å¡«å……）。 + + +1. ç¨åŽä½ å°†åœ¨ Rancher UI 中输入此密钥作为你的 **Application Secret**。由于你将无法在 Azure UI 中å†æ¬¡è®¿é—®é”®å€¼ï¼Œå› æ­¤è¯·åœ¨å…¶ä½™è®¾ç½®è¿‡ç¨‹ä¸­ä¿æŒæ‰“开此窗å£ã€‚ + +#### 3. 设置 Rancher 所需的æƒé™ + +接下æ¥ï¼Œåœ¨ Azure 中为 Rancher 设置 API æƒé™ã€‚ + +:::caution + +ç¡®ä¿ä½ è®¾ç½®äº† Application æƒé™ï¼Œè€Œ*ä¸æ˜¯* Delegated æƒé™ã€‚å¦åˆ™ï¼Œä½ å°†æ— æ³•ç™»å½• Azure AD。 + +::: + +1. 在导航窗格中，选择 **API permissions**。 + +1. å•å‡» **Add a permission**。 + +1. 从 Microsoft Graph API 中，选择以下 **Application Permissions**: `Directory.Read.All`。 + + ![选择 API æƒé™](/img/api-permissions.png) + +1. 返回导航æ ä¸­çš„ **API permissions**。在那里，å•å‡» **Grant admin consent**。然åŽå•å‡» **Yes**。该应用程åºçš„æƒé™åº”如下所示： + +![Open Required Permissions](/img/select-req-permissions.png) + +:::note + +Rancher ä¸ä¼šéªŒè¯ä½ æŽˆäºˆ Azure 应用程åºçš„æƒé™ã€‚ä½ å¯ä»¥è‡ªç”±ä½¿ç”¨ä»»ä½•ä½ æ‰€éœ€çš„æƒé™ï¼Œåªè¦è¿™äº›æƒé™å…许 Rancher 使用 AD 用户和组。 + +具体æ¥è¯´ï¼ŒRancher 需è¦å…许以下æ“作的æƒé™ï¼š +- 获å–一个用户。 +- 列出所有用户。 +- 列出给定用户所属的组。 +- 获å–一个组。 +- 列出所有组。 + +Rancher 执行这些æ“作æ¥ç™»å½•ç”¨æˆ·æˆ–æœç´¢ç”¨æˆ·/组。请记ä½ï¼Œæƒé™å¿…须是 `Application` 类型。 + +下é¢æ˜¯å‡ ä¸ªæ»¡è¶³ Rancher 需求的æƒé™ç»„åˆç¤ºä¾‹ï¼š +- `Directory.Read.All` +- `User.Read.All` å’Œ `GroupMember.Read.All` +- `User.Read.All` å’Œ `Group.Read.All` + +::: + +#### 4. å¤åˆ¶ Azure åº”ç”¨æ•°æ® + +![Application ID](/img/app-configuration.png) + +1. 获å–ä½ çš„ Rancher **租户 ID**。 + + 1. 使用æœç´¢æ‰“å¼€ **App registrations**。 + + 1. 找到你为 Rancher 创建的项。 + + 1. å¤åˆ¶ **Directory ID** 并将其作为 **Tenant ID** 粘贴到 Rancher 中。 + +1. 获å–ä½ çš„ Rancher **Application (Client) ID**。 + + 1. 如果你还未在该ä½ç½®ï¼Œè¯·ä½¿ç”¨æœç´¢æ‰“å¼€ **App registrations**。 + + 1. 在 **Overview**中，找到你为 Rancher 创建的æ¡ç›®ã€‚ + + 1. å¤åˆ¶ **Application (Client) ID** 并将其作为 **Application ID** 粘贴到 Rancher 中。 + +1. 你的端点选项通常是 [Standard](#global) 或 [China](#中国)。对于这两个选项，你åªéœ€è¦è¾“å…¥ **Tenant ID**ã€**Application ID** å’Œ **Application Secret**。 + +![标准端点选项](/img/tenant-application-id-secret.png) + +**对于自定义端点**： + +:::caution + +Rancher 未测试也未完全支æŒè‡ªå®šä¹‰ç«¯ç‚¹ã€‚ + +::: + +你还需è¦æ‰‹åŠ¨è¾“å…¥ Graphã€Token å’Œ Auth Endpoints。 + +- 从 App registrations 中，点击 Endpoints： + +![点击端点](/img/endpoints.png) + +- 以下端点将是你的 Rancher 端点值。请使用这些端点的 v1 版本。 + - **Microsoft Graph API endpoint**（Graph 端点） + - **OAuth 2.0 token endpoint (v1)**（Token 端点） + - **OAuth 2.0 authorization endpoint (v1)** (Auth 端点) + +#### 5. 在 Rancher 中é…ç½® Azure AD + +è¦å®Œæˆé…置，请在 Rancher UI 中输入你的 AD 实例信æ¯ã€‚ + +1. 登录到 Rancher。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 + +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 + +1. å•å‡» **AzureAD**。 + +1. 使用你在[å¤åˆ¶ Azure 应用数æ®](#4-å¤åˆ¶-azure-应用数æ®)æ—¶å¤åˆ¶çš„ä¿¡æ¯ï¼Œå¡«å†™**é…ç½® Azure AD è´¦å·**的表å•ã€‚ + + :::caution + + Azure AD å¸æˆ·å°†è¢«æŽˆäºˆç®¡ç†å‘˜æƒé™ï¼Œå› ä¸ºå…¶è¯¦ç»†ä¿¡æ¯å°†æ˜ å°„到 Rancher 本地主体å¸æˆ·ã€‚在继续之å‰ç¡®ä¿æ­¤æƒé™çº§åˆ«æ˜¯é€‚当的。 + + ::: + + **对于标准或中国端点：** + + 下表介ç»äº†ä½ åœ¨ Azure 门户中å¤åˆ¶çš„值与 Rancher 中字段的映射： + + | Rancher 字段 | Azure 值 | + | ------------------ | ------------------------------------- | + | 租户 ID | Directory ID | + | Application ID | Application ID | + | 应用密文 | Key Value | + | 端点 | https://login.microsoftonline.com/ | + + **对于自定义端点**： + + 下表将你的自定义é…置值映射到 Rancher 字段： + + | Rancher 字段 | Azure 值 | + | ------------------ | ------------------------------------- | + | Graph 端点 | Microsoft Graph API Endpoint | + | Token 端点 | OAuth 2.0 Token Endpoint | + | Auth 端点 | OAuth 2.0 Authorization Endpoint | + + **é‡è¦æ示**：在自定义é…置中输入 Graph Endpoint 时，请从 URL 中删除 Tenant ID： + + https://graph.microsoft.com/abb5adde-bee8-4821-8b03-e63efdc7701c + +1. 点击**å¯ç”¨**。 + +**结果**：Azure Active Directory 身份验è¯å·²é…置。 + + +### 从 Azure AD Graph API è¿ç§»åˆ° Microsoft Graph API + +由于 [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) 已弃用并计划于 2023 å¹´ 6 月åœç”¨ï¼Œç®¡ç†å‘˜åº”更新他们的 Azure AD 应用程åºä»¥åœ¨ Rancher 中使用 [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api)。 +你需è¦åœ¨ç«¯ç‚¹å¼ƒç”¨ä¹‹å‰å®Œæˆæ“作。 +如果在åœç”¨åŽ Rancher ä»é…置为使用 Azure AD Graph API，用户å¯èƒ½æ— æ³•ä½¿ç”¨ Azure AD 登录 Rancher。 + +#### 在 Rancher UI 中更新端点 + +:::caution + +管ç†å‘˜éœ€è¦åœ¨è¿ç§»ä¸‹è¿°ç«¯ç‚¹ä¹‹å‰åˆ›å»ºä¸€ä¸ª [Rancher 备份](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)。 + +::: + +1. [æ›´æ–°](#3-设置-rancher-所需的æƒé™) Azure AD 应用程åºæ³¨å†Œçš„æƒé™ã€‚这个步骤éžå¸¸å…³é”®ã€‚ + +1. 登录到 Rancher。 + +1. 在 Rancher UI 主页中，记下å±å¹•é¡¶éƒ¨çš„横幅，该横幅建议用户更新 Azure AD 身份验è¯ã€‚å•å‡»æ供的链接以执行此æ“作。 + + ![Rancher UI 横幅](/img/rancher-ui-azure-update.png) + +1. è¦å®Œæˆæ–°çš„ Microsoft Graph API è¿ç§»ï¼Œè¯·å•å‡» **Update Endpoint**。 + + **注æ„**：在开始更新之å‰ï¼Œè¯·ç¡®ä¿ä½ çš„ Azure 应用程åºå…·æœ‰[æ–°çš„æƒé™é›†](#3-设置-rancher-所需的æƒé™)。 + + ![更新端点](/img/rancher-button-to-update.png) + +1. 在收到弹出警告消æ¯æ—¶ï¼Œå•å‡» **Update**： + + ![Azure 更新弹出窗å£](/img/azure-update-popup.png) + +1. 有关 Rancher 执行的完整端点更改，请å‚阅下é¢çš„[表格](#global)。管ç†å‘˜ä¸éœ€è¦æ‰‹åŠ¨æ‰§è¡Œæ­¤æ“作。 + +#### 离线环境 + +在离线环境中，由于 Graph Endpoint URL 正在更改，因此管ç†å‘˜éœ€è¦ç¡®ä¿å…¶ç«¯ç‚¹è¢«[列入白åå•](#3.2)。 + +#### 回滚è¿ç§» + +如果你需è¦å›žæ»šè¿ç§»ï¼Œè¯·æ³¨æ„以下事项： + +1. 如果管ç†å‘˜æƒ³è¦å›žæ»šï¼Œæˆ‘们建议他们使用正确的æ¢å¤æµç¨‹ã€‚有关å‚考信æ¯ï¼Œè¯·å‚阅[备份文档](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)ã€[æ¢å¤æ–‡æ¡£](../../../new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md)å’Œ[示例](../../../../reference-guides/backup-restore-configuration/examples.md)。 + +1. 如果 Azure 应用程åºæ‰€æœ‰è€…想è¦è½®æ¢åº”用程åºå¯†é’¥ï¼Œä»–们也需è¦åœ¨ Rancher 中进行轮æ¢ï¼ˆå› ä¸ºåœ¨ Azure 中更改应用程åºå¯†é’¥æ—¶ï¼ŒRancher ä¸ä¼šè‡ªåŠ¨æ›´æ–°åº”用程åºå¯†é’¥ï¼‰ã€‚在 Rancher 中，它存储在å为 `azureadconfig-applicationsecret` çš„ Kubernetes 密文中，该密文ä½äºŽ `cattle-global-data` 命å空间中。 + +:::caution + +如果你使用现有的 Azure AD 设置å‡çº§åˆ° Rancher v2.7.0+，并选择了ç¦ç”¨è®¤è¯æ供程åºï¼Œä½ å°†æ— æ³•æ¢å¤ä»¥å‰çš„设置。你也无法使用旧æµç¨‹è®¾ç½® Azure AD。你需è¦ä½¿ç”¨æ–°çš„认è¯æµç¨‹é‡æ–°æ³¨å†Œã€‚由于 Rancher 现在使用 Graph API，因此用户需è¦[在 Azure 门户中设置适当的æƒé™](#3-设置-rancher-所需的æƒé™)。 + +::: + + +#### Global: + +| Rancher 字段 | 已弃用的端点 | +---------------- | ------------------------------------------------------------- +| Auth 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/authorize | +| 端点 | https://login.microsoftonline.com/ | +| Graph 端点 | https://graph.windows.net/ | +| Token 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/token | + +| Rancher 字段 | 新端点 | +---------------- | ------------------------------------------------------------------ +| Auth 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/authorize | +| 端点 | https://login.microsoftonline.com/ | +| Graph 端点 | https://graph.microsoft.com | +| Token 端点 | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/token | + +#### 中国： + +| Rancher 字段 | 已弃用的端点 | +---------------- | ---------------------------------------------------------- +| Auth 端点 | https://login.chinacloudapi.cn/{tenantID}/oauth2/authorize | +| 端点 | https://login.chinacloudapi.cn/ | +| Graph 端点 | https://graph.chinacloudapi.cn/ | +| Token 端点 | https://login.chinacloudapi.cn/{tenantID}/oauth2/token | + +| Rancher 字段 | 新端点 | +---------------- | ------------------------------------------------------------------------- +| Auth 端点 | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/authorize | +| 端点 | https://login.partner.microsoftonline.cn/ | +| Graph 端点 | https://microsoftgraph.chinacloudapi.cn | +| Token 端点 | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/token | + + +## 已弃用的 Azure AD Graph API + +> **é‡è¦æ示**： +> +> - [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) 已被弃用，Microsoft 将在 2023 å¹´ 6 月 30 æ—¥åŽéšæ—¶åœç”¨å®ƒä¸”ä¸ä¼šå¦è¡Œé€šçŸ¥ã€‚我们将更新我们的文档，以便在åœç”¨æ—¶å‘社区æ供建议。Rancher 现在使用 [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api) æ¥å°† Azure AD 设置为外部身份验è¯æ供程åºã€‚ +> +> +> - 如果你是新用户或希望进行è¿ç§»ï¼Œè¯·å‚阅新的æµç¨‹è¯´æ˜Žï¼š Rancher v2.7.0+。 +> +> +> - 如果你ä¸æƒ³åœ¨ Azure AD Graph API åœç”¨åŽå‡çº§åˆ° v2.7.0+，你需è¦ï¼š +> - 使用内置的 Rancher 身份认è¯ï¼Œæˆ–者 +> - 使用å¦ä¸€ä¸ªç¬¬ä¸‰æ–¹èº«ä»½è®¤è¯ç³»ç»Ÿå¹¶åœ¨ Rancher 中进行设置。请å‚阅[身份验è¯æ–‡æ¡£](../../../../pages-for-subheaders/authentication-config.md)，了解如何é…置其他开放å¼èº«ä»½éªŒè¯æ供程åºã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md new file mode 100644 index 00000000000..6f3100e462c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md @@ -0,0 +1,60 @@ +--- +title: é…ç½® FreeIPA +--- + +如果你的组织使用 FreeIPA 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许你的用户使用 FreeIPA 凭è¯ç™»å½•ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +- ä½ å¿…é¡»é…置了 [FreeIPA æœåŠ¡å™¨](https://www.freeipa.org/)。 +- 在 FreeIPA 中创建一个具有 `read-only` 访问æƒé™çš„ ServiceAccount 。当用户使用 API​​ 密钥å‘出请求时，Rancher 使用此账å·æ¥éªŒè¯ç»„æˆå‘˜èº«ä»½ã€‚ +- å‚è§[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + +::: + +1. 使用分é…了 `administrator` è§’è‰²ï¼ˆå³ _本地主体_）的本地用户登录到 Rancher。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **FreeIPA**。 +1. 填写**é…ç½® FreeIPA æœåŠ¡å™¨**表å•ï¼Œ + + ä½ å¯èƒ½éœ€è¦ç™»å½•åˆ°åŸŸæŽ§åˆ¶å™¨ï¼Œæ¥æŸ¥æ‰¾è¡¨å•ä¸­è¯·æ±‚çš„ä¿¡æ¯ã€‚ + + :::note 使用 TLS？ + + 如果è¯ä¹¦æ˜¯è‡ªç­¾å，或者ä¸æ˜¯æ¥è‡ªå…¬è®¤çš„è¯ä¹¦é¢å‘机构的，请确ä¿æ供完整的è¯ä¹¦é“¾ã€‚Rancher 需è¦è¯¥é“¾æ¥éªŒè¯æœåŠ¡å™¨çš„è¯ä¹¦ã€‚ + + ::: + + :::note 用户æœç´¢åº“ vs. 组æœç´¢åº“ + + æœç´¢åº“使 Rancher å¯ä»¥æœç´¢ FreeIPA 中的用户和组。这些字段仅用于æœç´¢åº“，ä¸é€‚用于æœç´¢ç­›é€‰å™¨ã€‚ + + * 如果你的用户和组ä½äºŽåŒä¸€æœç´¢åº“中，则仅填写用户æœç´¢åº“。 + * 如果你的组ä½äºŽå…¶ä»–æœç´¢åº“中，则å¯ä»¥é€‰æ‹©å¡«å†™ç»„æœç´¢åº“。该字段专用于æœç´¢ç»„，但ä¸æ˜¯å¿…需的。 + + ::: + +1. 如果你的 FreeIPA ä¸åŒäºŽæ ‡å‡†çš„ AD Schema，则必须完æˆ**自定义 Schema** 部分实现匹é…。å¦åˆ™ï¼Œè°ƒè¿‡æ­¤æ­¥éª¤ã€‚ + + :::note æœç´¢å±žæ€§ + + `æœç´¢å±žæ€§`字段的默认值为三个特定值：`uid|sn|givenName`。é…ç½® FreeIPA åŽï¼Œå½“用户输入文本以添加用户或组时，Rancher 会自动查询 FreeIPA æœåŠ¡å™¨ï¼Œå¹¶å°è¯•æŒ‰ç”¨æˆ· ID，姓æ°æˆ–åå­—æ¥åŒ¹é…字段。Rancher 专门æœç´¢ä»¥åœ¨æœç´¢å­—段中输入的文本开头的用户/组。 + + 默认字段值为 `uid|sn|givenName`，但是你å¯ä»¥å°†æ­¤å­—段é…置为这些字段的å­é›†ã€‚管é“符 (`|`) 用于分隔å„个字段。 + + * `uid`：用户 ID + * `sn`：姓 + * `givenName`：å + + Rancher 使用此æœç´¢å±žæ€§ä¸ºç”¨æˆ·å’Œç»„创建æœç´¢ç­›é€‰å™¨ï¼Œä½†æ˜¯ä½ *ä¸èƒ½*在此字段中添加自己的æœç´¢ç­›é€‰å™¨ã€‚ + + ::: + +1. 在 **Authenticate with FreeIPA** 中输入你的 FreeIPA 用户å和密ç ï¼Œç¡®è®¤å·²ä¸º Rancher é…ç½® FreeIPA 身份验è¯ã€‚ +1. 点击**å¯ç”¨**。 + +**结果**： + +- FreeIPA 验è¯é…ç½®æˆåŠŸã€‚ +- 你将使用你的 FreeIPA è´¦å·ï¼ˆå³ _外部主体_）登录到 Rancher。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md new file mode 100644 index 00000000000..e903421611d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md @@ -0,0 +1,56 @@ +--- +title: é…ç½® GitHub +--- + +在使用 GitHub 的环境中，你å¯ä»¥é€šè¿‡é…ç½® Rancher å…许用户使用 GitHub 凭è¯ç™»å½•ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +å‚è§[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + +::: + +1. 使用分é…了 `administrator` è§’è‰²ï¼ˆå³ _本地主体_）的本地用户登录到 Rancher。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **GitHub**。 +1. 按照显示的说明设置 GitHub 应用。Rancher 会将你é‡å®šå‘到 GitHub 完æˆæ³¨å†Œã€‚ + + :::note 什么是授æƒå›žè°ƒ URL？ + + 授æƒå›žè°ƒ URL 是用户开始使用你的应用（å³åˆå§‹å±å¹•ï¼‰çš„ URL。 + + 使用外部身份验è¯æ—¶ï¼Œå®žé™…上ä¸ä¼šåœ¨ä½ çš„应用中进行身份验è¯ã€‚相å，身份验è¯åœ¨å¤–部进行（在本例中为 GitHub）。在外部身份验è¯æˆåŠŸå®ŒæˆåŽï¼Œç”¨æˆ·å°†é€šè¿‡æŽˆæƒå›žè°ƒ URL é‡æ–°è¿›å…¥åº”用。 + + ::: + +1. 从 GitHub å¤åˆ¶ **Client ID** å’Œ **Client Secret**。将它们粘贴到 Rancher 中。 + + :::note 在哪里å¯ä»¥æ‰¾åˆ° Client ID å’Œ Client Secret？ + + 在 GitHub 中，选择 **Settings > Developer Settings > OAuth Apps**。你å¯ä»¥åœ¨æ­¤å¤„找到 Client ID å’Œ Client Secret。 + + ::: + +1. å•å‡»**使用 GitHub 认è¯**。 + +1. 使用 **Site Access** 选项æ¥é…置用户授æƒçš„范围。 + + - **å…许任何有效用户** + + _任何_ GitHub 用户都能访问 Rancher。通常ä¸å»ºè®®ä½¿ç”¨æ­¤è®¾ç½®ã€‚ + + - **å…许集群和项目æˆå‘˜ï¼Œä»¥åŠæŽˆæƒçš„用户和组织** + + 添加为**集群æˆå‘˜**或**项目æˆå‘˜**的任何 GitHub 用户或组都å¯ä»¥ç™»å½• Rancher。此外，任何添加到**授æƒç”¨æˆ·å’Œç»„织**列表中的 GitHub 用户和组都能登录到 Rancher。 + + - **ä»…å…许授æƒç”¨æˆ·å’Œç»„织** + + åªæœ‰æ·»åŠ åˆ°**授æƒç”¨æˆ·å’Œç»„织**çš„ GitHub 用户和组能登录 Rancher。 +
+1. 点击**å¯ç”¨**。 + +**结果**： + +- GitHub 验è¯é…ç½®æˆåŠŸã€‚ +- 你将使用你的 GitHub è´¦å·ï¼ˆå³ _外部主体_）登录到 Rancher。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md new file mode 100644 index 00000000000..ca4d522ba35 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md @@ -0,0 +1,111 @@ +--- +title: é…ç½® Google OAuth +--- + +如果你的组织使用 G Suite 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许你的用户使用 G Suite 凭è¯ç™»å½•ã€‚ + +åªæœ‰ G Suite 域的管ç†å‘˜æ‰èƒ½è®¿é—® Admin SDK。因此，åªæœ‰ G Suite 管ç†å‘˜å¯ä»¥é…ç½® Rancher çš„ Google OAuth。 + +在 Rancher 中，åªæœ‰å…·æœ‰ **Manage Authentication** [全局角色](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)的管ç†å‘˜æˆ–用户æ‰èƒ½é…置身份验è¯ã€‚ + +## 先决æ¡ä»¶ + +- ä½ å¿…é¡»é…置了 [G Suite 管ç†å‘˜è´¦å·](https://admin.google.com)。 +- G Suite 需è¦ä¸€ä¸ª[顶级ç§æœ‰åŸŸ FQDN](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains) 作为授æƒåŸŸã€‚èŽ·å– FQDN 的一ç§æ–¹æ³•æ˜¯åœ¨ Route53 中为 Rancher Server 创建 A 记录。你ä¸éœ€è¦ä½¿ç”¨è¯¥è®°å½•æ›´æ–° Rancher Server çš„ URL 设置，因为å¯èƒ½æœ‰é›†ç¾¤ä½¿ç”¨è¯¥ URL。 +- ä½ çš„ G Suite 域必须å¯ç”¨äº† Admin SDK API。你å¯ä»¥æŒ‰ç…§[此页é¢](https://support.google.com/a/answer/60757?hl=en)中的步骤å¯ç”¨å®ƒã€‚ + +å¯ç”¨ Admin SDK API åŽï¼Œä½ çš„ G Suite 域的 API 页é¢åº”如下图所示： +![Enable Admin APIs](/img/Google-Enable-APIs-Screen.png) + +## 在 Rancher 中为 OAuth 设置 G Suite + +在 Rancher 中设置 Google OAuth 之å‰ï¼Œä½ éœ€è¦ç™»å½•åˆ°ä½ çš„ G Suite è´¦å·å¹¶å®Œæˆä»¥ä¸‹è®¾ç½®ï¼š + +1. [在 G Suite 中将 Rancher 添加为授æƒåŸŸ](#1-å°†-rancher-添加为授æƒåŸŸ) +1. [为 Rancher Server ç”Ÿæˆ OAuth2 凭è¯](#2-为-rancher-server-生æˆ-oauth2-凭è¯) +1. [为 Rancher Server 创建 ServiceAccount 凭è¯](#3-创建-serviceaccount-凭è¯) +1. [å°† ServiceAccount å¯†é’¥æ³¨å†Œæˆ OAuth Client](#4-å°†-serviceaccount-密钥注册æˆ-oauth-client) + +### 1. å°† Rancher 添加为授æƒåŸŸ + +1. 点击[此处](https://console.developers.google.com/apis/credentials)å‰å¾€ä½ çš„ Google 域的凭è¯é¡µé¢ã€‚ +1. 选择你的项目，然åŽç‚¹å‡» **OAuth consent screen**。 + ![OAuth Consent Screen](/img/Google-OAuth-consent-screen-tab.png) +1. å‰å¾€ **Authorized Domains**，并在列表中输入你的 Rancher Server URL 的顶级ç§æœ‰åŸŸã€‚顶级ç§æœ‰åŸŸæ˜¯æœ€å³è¾¹çš„超级域。例如，`www.foo.co.uk` 是 `foo.co.uk` 的顶级ç§æœ‰åŸŸã€‚有关顶级ç§æœ‰åŸŸçš„更多信æ¯ï¼Œè¯·å‚è§[此处](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains)。 +1. å‰å¾€ **Scopes for Google APIs**，并确ä¿å·²å¯ç”¨ **email**，**profile** å’Œ **openid**。 + +**结果**：Rancher 已被添加为 Admin SDK API 的授æƒåŸŸã€‚ + +### 2. 为 Rancher Server ç”Ÿæˆ OAuth2 å‡­è¯ + +1. å‰å¾€ Google API 控制å°ï¼Œé€‰æ‹©ä½ çš„项目并å‰å¾€ [credentials ](https://console.developers.google.com/apis/credentials)页é¢ã€‚ + ![Credentials](/img/Google-Credentials-tab.png) +1. 在 **Create Credentials** 下拉框中，选择 **OAuth client ID**。 +1. 点击 **Web application**。 +1. 输入一个å称。 +1. 填写 **Authorized JavaScript origins** å’Œ **Authorized redirect URIs**。请注æ„，设置 Google OAuth çš„ Rancher UI 页é¢ï¼ˆ**Security > Authentication > Google** 下的全局视图）为你æ供了这一步è¦è¾“入的准确链接。 +- 在 **Authorized JavaScript origins** 处，输入你的 Rancher Server URL。 +- 在 **Authorized redirect URIs** 处，输入你的 Rancher Server çš„ URL 并附加路径 `verify-auth`。例如，如果你的 URI 是 `https://rancherServer`，你需è¦è¾“å…¥ `https://rancherServer/verify-auth`。 +1. 点击 **Create**。 +1. 创建凭è¯ä¹‹åŽï¼Œä½ å°†çœ‹åˆ°ä¸€ä¸ªé¡µé¢ï¼Œå…¶ä¸­æ˜¾ç¤ºä½ çš„凭è¯åˆ—表。选择刚刚创建的凭è¯ï¼Œç„¶åŽåœ¨æœ€å³è¾¹çš„行中å•å‡» **Download JSON**。ä¿å­˜è¯¥æ–‡ä»¶ï¼Œä»¥ä¾¿å‘ Rancher æ供这些凭è¯ã€‚ + +**结果**：你已æˆåŠŸåˆ›å»º OAuth 凭è¯ã€‚ + +### 3. 创建 ServiceAccount å‡­è¯ +由于 Google Admin SDK åªå¯¹ç®¡ç†å‘˜å¯ç”¨ï¼Œæ™®é€šç”¨æˆ·ä¸èƒ½ä½¿ç”¨å®ƒæ¥æ£€ç´¢å…¶ä»–用户或其组的é…置文件。普通用户甚至ä¸èƒ½æ£€ç´¢ä»–们自己的组。 + +由于 Rancher æ供基于组的æˆå‘˜è®¿é—®ï¼Œæˆ‘们è¦æ±‚用户能够获得自己的组，并在需è¦æ—¶æŸ¥æ‰¾å…¶ä»–用户和组。 + +为了解决这个问题，G Suite 建议创建一个 ServiceAccount，并将你的 G Suite 域的æƒé™å§”托给该 ServiceAccount。 + +本节介ç»å¦‚何： + +- 创建一个 ServiceAccount +- 为 ServiceAccount 创建一个密钥并下载 JSON æ ¼å¼çš„å‡­è¯ + +1. 点击[此处](https://console.developers.google.com/iam-admin/serviceaccounts)并选择è¦ç”Ÿæˆ OAuth 凭è¯çš„项目。 +1. 点击 **Create Service Account**。 +1. 输入å称，并点击 **Create**。 + ![Service account creation Step 1](/img/Google-svc-acc-step1.png) +1. ä¸è¦åœ¨ **Service account permissions** 页é¢è®¾ç½®ä»»ä½•è§’色，然åŽå•å‡» **Continue**。 + ![Service account creation Step 2](/img/Google-svc-acc-step2.png) +1. 点击 **Create Key** 并选择 JSON 选项。下载并ä¿å­˜ JSON 文件，以便你å¯ä»¥å°†å…¶ä½œä¸º ServiceAccount 凭è¯æ供给 Rancher。 + ![Service account creation Step 3](/img/Google-svc-acc-step3-key-creation.png) + +**结果**：你的 ServiceAccount 已创建æˆåŠŸã€‚ + +### 4. å°† ServiceAccount å¯†é’¥æ³¨å†Œæˆ OAuth Client + +你需è¦ä¸ºåœ¨ä¸Šä¸€æ­¥ä¸­åˆ›å»ºçš„ ServiceAccount 授予一些æƒé™ã€‚Rancher ä»…è¦æ±‚为用户和组授予åªè¯»æƒé™ã€‚ + +使用 ServiceAccount 密钥的唯一 ID，按照以下步骤将其注册为 Oauth Client： + +1. 获å–你刚刚创建的密钥的唯一 ID。如果它没有显示在你创建的键æ—边的键列表中，你需è¦å…ˆå¯ç”¨ Unique ID 列。点击 **Unique ID** 然åŽç‚¹å‡» **OK**ã€‚è¿™å°†å‘ ServiceAccount 密钥列表中添加 **Unique ID** 列。ä¿å­˜ä½ åˆ›å»ºçš„ ServiceAccount 对应的唯一 ID。注æ„：这是一个数字 Key，ä¸è¦ä¸Žå­—æ¯æ•°å­—字段 **Key ID** 混淆。 + + ![Service account Unique ID](/img/Google-Select-UniqueID-column.png) +1. å‰å¾€ [**Domain-wide Delegation** 页é¢ã€‚](https://admin.google.com/ac/owl/domainwidedelegation) +1. 在 **Client Name** 字段中添加上一步中获得的唯一 ID。 +1. 在 **One or More API Scopes** 字段中，添加以下作用域： + ``` + openid,profile,email,https://www.googleapis.com/auth/admin.directory.user.readonly,https://www.googleapis.com/auth/admin.directory.group.readonly + ``` +1. 点击 **Authorize**。 + +**结果**：ServiceAccount 在你的 G Suite è´¦å·ä¸­å·²æ³¨å†Œä¸º OAuth 客户端。 + +## 在 Rancher 中é…ç½® Google OAuth + +1. 使用分é…了 [administrator](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) 角色的本地用户登录到 Rancher。这个用户也称为本地主体。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Google**。UI 中的说明介ç»äº†ä½¿ç”¨ Google OAuth 设置身份验è¯çš„步骤。 + 1. 管ç†å‘˜é‚®ç®±ï¼šæä¾› GSuite 设置中的管ç†å‘˜è´¦æˆ·çš„电å­é‚®ç®±ã€‚为了查找用户和组，Google API 需è¦ç®¡ç†å‘˜çš„电å­é‚®ä»¶å’Œ ServiceAccount 密钥。 + 1. 域å：æä¾›é…置了 G Suite 的域。请æ供准确的域，而ä¸æ˜¯åˆ«å。 + 1. 属于多个用户组的用户：选中此框以å¯ç”¨åµŒå¥—组æˆå‘˜å…³ç³»ã€‚Rancher 管ç†å‘˜å¯ä»¥åœ¨é…置认è¯åŽçš„任何时候ç¦ç”¨å®ƒã€‚ + - **步骤一**是将 Rancher 添加为授æƒåŸŸï¼ˆè¯¦æƒ…请å‚è§[本节](#1-å°†-rancher-添加为授æƒåŸŸ)）。 + - **步骤二**æ供你完æˆ[本节](#2-为-rancher-server-生æˆ-oauth2-凭è¯)åŽä¸‹è½½çš„ OAuth å‡­è¯ JSON。你å¯ä»¥ä¸Šä¼ æ–‡ä»¶æˆ–将内容粘贴到 **OAuth Credentials** 字段。 + - **步骤三**æ供在[本节](#3-创建-serviceaccount-凭è¯)末尾下载的 ServiceAccount å‡­è¯ JSON。仅当你æˆåŠŸ[在 G Suite è´¦å·ä¸­å°† ServiceAccount 密钥注册为 OAuth Client](#4-å°†-serviceaccount-密钥注册æˆ-oauth-client) åŽï¼Œå‡­è¯æ‰èƒ½æ­£å¸¸å·¥ä½œã€‚ +1. 点击**使用 Google 认è¯**。 +1. 点击**å¯ç”¨**。 + +**结果**：Google 验è¯é…ç½®æˆåŠŸã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md new file mode 100644 index 00000000000..5516672eb87 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md @@ -0,0 +1,145 @@ +--- +title: é…ç½® Keycloak (OIDC) +description: 创建 Keycloak OpenID Connect (OIDC) 客户端并é…ç½® Rancher 以使用 Keycloak。你的用户将能够使用他们的 Keycloak 登录å登录 Rancher。 +--- + +如果你的组织使用 [Keycloak Identity Provider (IdP)](https://www.keycloak.org) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚Rancher 支æŒä½¿ç”¨ OpenID Connect (OIDC) å议和 SAML åè®®æ¥é›†æˆ Keycloak。与 Rancher 一起使用时，这两ç§å®žçŽ°åœ¨åŠŸèƒ½ä¸Šæ˜¯ç­‰æ•ˆçš„。本文æ述了é…ç½® Rancher 以通过 OIDC å议与 Keycloak 一起使用的æµç¨‹ã€‚ + +如果你更喜欢将 Keycloak 与 SAML å议一起使用，请å‚è§[此页é¢](configure-keycloak-saml.md)。 + +如果你有使用 SAML å议的现有é…置并希望切æ¢åˆ° OIDC å议，请å‚è§[本节](#从-saml-è¿ç§»åˆ°-oidc)。 + +## 先决æ¡ä»¶ + +- 已在 Rancher 上ç¦ç”¨ Keycloak (SAML)。 +- ä½ å¿…é¡»é…置了 [Keycloak IdP æœåŠ¡å™¨](https://www.keycloak.org/docs/latest/server_installation/)。 +- 在 Keycloak 中，使用以下设置创建一个[æ–°çš„ OIDC 客户端](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients)。如需获å–帮助，请å‚è§ [Keycloak 文档](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients)。 + + | 设置 | 值 | + ------------|------------ + | `Client ID` | <CLIENT_ID> (例如 `rancher`) | + | `Name` | <CLIENT_NAME> (例如 `rancher`) | + | `Client Protocol` | `openid-connect` | + | `Access Type` | `confidential` | + | `Valid Redirect URI` | `https://yourRancherHostURL/verify-auth` | + +- 在新的 OIDC 客户端中，创建 [Mappers](https://www.keycloak.org/docs/latest/server_admin/#_protocol-mappers) æ¥å…¬å¼€ç”¨æˆ·å­—段。 + - 使用以下设置创建一个新的 "Groups Mapper"： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Groups Mapper` | + | `Mapper Type` | `Group Membership` | + | `Token Claim Name` | `groups` | + | `Add to ID token` | `OFF` | + | `Add to access token` | `OFF` | + | `Add to user info` | `ON` | + + - 使用以下设置创建一个新的 "Client Audience" ： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Client Audience` | + | `Mapper Type` | `Audience` | + | `Included Client Audience` | <CLIENT_NAME> | + | `Add to access token` | `ON` | + + - 使用以下设置创建一个新的 "Groups Path"： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Group Path` | + | `Mapper Type` | `Group Membership` | + | `Token Claim Name` | `full_group_path` | + | `Full group path` | `ON` | + | `Add to user info` | `ON` | + +## 在 Rancher 中é…ç½® Keycloak + +1. 在 Rancher UI 中，å•å‡» **☰ > 用户 & 认è¯**。 +1. å•å‡»å·¦ä¾§å¯¼èˆªæ çš„**认è¯**。 +1. 选择 **Keycloak (OIDC)**。 +1. 填写**é…ç½® Keycloak OIDC è´¦å·**表å•ã€‚有关填写表å•çš„帮助，请å‚è§[é…ç½®å‚考](#é…ç½®å‚考)。 +1. 完æˆ**é…ç½® Keycloak OIDC è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Keycloak IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Keycloak é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 OIDC å议与 Keycloak 一起工作。你的用户现在å¯ä»¥ä½¿ç”¨ Keycloak 登录å登录 Rancher。 + +## é…ç½®å‚考 + +| 字段 | æè¿° | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 客户端 ID | ä½ çš„ Keycloak 客户端的 `Client ID`。 | +| å®¢æˆ·ç«¯å¯†ç  | ä½ çš„ Keycloak 客户端生æˆçš„ `Secret`。在 Keycloak 控制å°ä¸­ï¼Œå•å‡» **Clients**，选择你创建的客户端，选择 **Credentials** 选项å¡ï¼Œç„¶åŽå¤åˆ¶ `Secret` 字段的值。 | +| ç§é’¥/è¯ä¹¦ | 在 Rancher 和你的 IdP 之间创建安全外壳（SSH）的密钥/è¯ä¹¦å¯¹ã€‚如果你的 Keycloak æœåŠ¡å™¨ä¸Šå¯ç”¨äº† HTTPS/SSL，则为必填。 | +| 端点 | 选择为 `Rancher URL`ã€`å‘行者`å’Œ `Auth 端点`字段使用生æˆçš„值，还是在ä¸æ­£ç¡®æ—¶è¿›è¡Œæ‰‹åŠ¨è¦†ç›–。 | +| Keycloak URL | ä½ çš„ Keycloak æœåŠ¡å™¨çš„ URL。 | +| Keycloak Realm | 创建 Keycloak 客户端的 Realm çš„å称。 | +| Rancher URL | Rancher Server çš„ URL。 | +| Issuer | ä½ çš„ IdP çš„ URL。 | +| Auth 端点 | é‡å®šå‘用户进行身份验è¯çš„ URL。 | + +## 从 SAML è¿ç§»åˆ° OIDC + +本节æ述了将使用 Keycloak (SAML) çš„ Rancher 过渡到 Keycloak (OIDC) 的过程。 + +### é‡æ–°é…ç½® Keycloak + +1. 将现有客户端更改为使用 OIDC å议。在 Keycloak 控制å°ä¸­ï¼Œå•å‡» **Clients**，选择è¦è¿ç§»çš„ SAML 客户端，选择 **Settings** 选项å¡ï¼Œå°† `Client Protocol` 从 `saml` 更改为 `openid-connect`，然åŽç‚¹å‡» **Save**。 + +1. éªŒè¯ `Valid Redirect URIs` 是å¦ä»ç„¶æœ‰æ•ˆã€‚ + +1. 选择 **Mappers** 选项å¡å¹¶ä½¿ç”¨ä»¥ä¸‹è®¾ç½®åˆ›å»ºä¸€ä¸ªæ–°çš„ Mapper： + + | 设置 | 值 | + ------------|------------ + | `Name` | `Groups Mapper` | + | `Mapper Type` | `Group Membership` | + | `Token Claim Name` | `groups` | + | `Add to ID token` | `ON` | + | `Add to access token` | `ON` | + | `Add to user info` | `ON` | + +### é‡æ–°é…ç½® Rancher + +在将 Rancher é…置为使用 Keycloak (OIDC) 之å‰ï¼Œå¿…须先ç¦ç”¨ Keycloak (SAML)： + +1. 在 Rancher UI 中，å•å‡» **☰ > 用户 & 认è¯**。 +1. å•å‡»å·¦ä¾§å¯¼èˆªæ çš„**认è¯**。 +1. 选择 **Keycloak (SAML)**。 +1. å•å‡»**ç¦ç”¨**。 + +按照[本节](#在-rancher-中é…ç½®-keycloak)中的步骤将 Rancher é…置为使用 Keycloak (OIDC)。 + +:::note + +é…置完æˆåŽï¼Œç”±äºŽç”¨æˆ·æƒé™ä¸ä¼šè‡ªåŠ¨è¿ç§»ï¼Œä½ éœ€è¦é‡æ–°ç”³è¯· Rancher 用户æƒé™ã€‚ + +::: + +## 附录：故障排除 + +如果你在测试与 Keycloak æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·å…ˆæ£€æŸ¥ OIDC 客户端的é…置选项。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 + +所有与 Keycloak 相关的日志æ¡ç›®éƒ½å°†æ·»åŠ  `[generic oidc]` 或 `[keycloak oidc]`。 + +### ä¸èƒ½é‡å®šå‘到 Keycloak + +完æˆ**é…ç½® Keycloak OIDC è´¦å·**表å•å¹¶å•å‡»**å¯ç”¨**åŽï¼Œä½ æ²¡æœ‰è¢«é‡å®šå‘到你的 IdP。 + +* 验è¯ä½ çš„ Keycloak 客户端é…置。 + +### 生æˆçš„ `Issuer` å’Œ `Auth 端点`ä¸æ­£ç¡® + +* 在**é…ç½® Keycloak OIDC è´¦å·**表å•ä¸­ï¼Œå°†**端点**更改为`指定（高级设置）`并覆盖`å‘行者` å’Œ `Auth 端点`的值。è¦æŸ¥æ‰¾è¿™äº›å€¼ï¼Œå‰å¾€ Keycloak 控制å°å¹¶é€‰æ‹© **Realm Settings**，选择 **General** 选项å¡ï¼Œç„¶åŽå•å‡» **OpenID Endpoint Configuration**。JSON 输出将显示 `issuer` å’Œ `authorization_endpoint` 的值。 + +### Keycloak 错误："Invalid grant_type" + +* 在æŸäº›æƒ…况下，这æ¡é”™è¯¯æ示信æ¯å¯èƒ½æœ‰è¯¯å¯¼æ€§ï¼Œå®žé™…上造æˆé”™è¯¯çš„原因是 `Valid Redirect URI` é…置错误。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md new file mode 100644 index 00000000000..17d906c51c0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md @@ -0,0 +1,190 @@ +--- +title: é…ç½® Keycloak (SAML) +description: 创建 Keycloak SAML 客户端并é…ç½® Rancher 以使用 Keycloak。你的用户将能够使用他们的 Keycloak 登录å登录 Rancher。 +--- + +如果你的组织使用 Keycloak Identity Provider (IdP) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚ + +## 先决æ¡ä»¶ + +- ä½ å¿…é¡»é…置了 [Keycloak IdP æœåŠ¡å™¨](https://www.keycloak.org/docs/latest/server_installation/)。 +- 在 Keycloak 中，使用以下设置创建一个[æ–°çš„ SAML 客户端](https://www.keycloak.org/docs/latest/server_admin/#saml-clients)。如需获å–帮助，请å‚è§ [Keycloak 文档](https://www.keycloak.org/docs/latest/server_admin/#saml-clients)。 + + | 设置 | 值 | + ------------|------------ + | `Sign Documents` | `ON` 1 | + | `Sign Assertions` | `ON` 1 | + | 所有其他 `ON/OFF` 设置 | `OFF` | + | `Client ID` | 输入 `https://yourRancherHostURL/v1-saml/keycloak/saml/metadata`，或在 Rancher Keycloak é…ç½®2 中 `Entry ID 字段`的值。 | + | `Client Name` | (例如 `rancher`) | + | `Client Protocol` | `SAML` | + | `Valid Redirect URI` | `https://yourRancherHostURL/v1-saml/keycloak/saml/acs` | + + > 1：å¯ä»¥é€‰æ‹©å¯ç”¨è¿™äº›è®¾ç½®ä¸­çš„一个或两个。 + > 2：在é…置和ä¿å­˜ SAML 身份æ供商之å‰ï¼Œä¸ä¼šç”Ÿæˆ Rancher SAML 元数æ®ã€‚ + + ![](/img/keycloak/keycloak-saml-client-configuration.png) + +- 在新的 SAML 客户端中，创建 Mappers æ¥å…¬å¼€ç”¨æˆ·å­—段。 + - 添加所有 "Builtin Protocol Mappers" + ![](/img/keycloak/keycloak-saml-client-builtin-mappers.png) + - 创建一个 "Group list" mapper，æ¥å°†æˆå‘˜å±žæ€§æ˜ å°„到用户的组： + ![](/img/keycloak/keycloak-saml-client-group-mapper.png) + +## èŽ·å– IDP å…ƒæ•°æ® + + + + +è¦èŽ·å– IDP 元数æ®ï¼Œè¯·ä»Ž Keycloak 客户端导出 `metadata.xml` 文件。 +在**安装**选项å¡ä¸­ï¼Œé€‰æ‹©**SAML å…ƒæ•°æ® IDPSSODescriptor** æ ¼å¼é€‰é¡¹å¹¶ä¸‹è½½ä½ çš„文件。 + + + + +1. 在**é…ç½®**中，å•å‡» **Realm 设置**选项å¡ã€‚ +1. 点击**通用**选项å¡ã€‚ +1. 在**端点**字段中，å•å‡» **SAML 2.0 身份æ供者元数æ®**。 + +éªŒè¯ IDP 元数æ®æ˜¯å¦åŒ…å«ä»¥ä¸‹å±žæ€§ï¼š + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +æŸäº›æµè§ˆå™¨ï¼ˆä¾‹å¦‚ Firefox）å¯èƒ½ä¼šæ¸²æŸ“/处ç†æ–‡æ¡£ï¼Œä½¿å¾—内容看起æ¥å·²è¢«ä¿®æ”¹ï¼Œå¹¶ä¸”æŸäº›å±žæ€§çœ‹èµ·æ¥å¯èƒ½æœ‰ç¼ºå¤±ã€‚在这ç§æƒ…况下，请使用通过æµè§ˆå™¨æ‰¾åˆ°çš„原始å“应数æ®ã€‚ + +以下是 Firefox 的示例æµç¨‹ï¼Œå…¶ä»–æµè§ˆå™¨å¯èƒ½ä¼šç•¥æœ‰ä¸åŒï¼š + +1. 按下 **F12** 访问开å‘者控制å°ã€‚ +1. 点击 **Network** 选项å¡ã€‚ +1. 从表中，å•å‡»åŒ…å« `descriptor` 的行。 +1. 在 details 窗格中，å•å‡» **Response** 选项å¡ã€‚ +1. å¤åˆ¶åŽŸå§‹å“应数æ®ã€‚ + +获得的 XML 以 `EntitiesDescriptor` 作为根元素。然而，Rancher 希望根元素是 `EntityDescriptor` 而ä¸æ˜¯ `EntitiesDescriptor`。因此，在将这个 XML 传递给 Rancher 之å‰ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤è°ƒæ•´ï¼š + +1. 将所有ä¸å­˜åœ¨çš„属性从 `EntitiesDescriptor` å¤åˆ¶åˆ° `EntityDescriptor`。 +1. 删除开头的 `` 标签。 +1. 删除 xml 末尾的 ``。 + +最åŽçš„代ç ä¼šæ˜¯å¦‚下： + +``` + +.... + +``` + + + + +1. 在**é…ç½®**中，å•å‡» **Realm 设置**选项å¡ã€‚ +1. 点击**通用**选项å¡ã€‚ +1. 在**端点**字段中，å•å‡» **SAML 2.0 身份æ供者元数æ®**。 + +éªŒè¯ IDP 元数æ®æ˜¯å¦åŒ…å«ä»¥ä¸‹å±žæ€§ï¼š + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +æŸäº›æµè§ˆå™¨ï¼ˆä¾‹å¦‚ Firefox）å¯èƒ½ä¼šæ¸²æŸ“/处ç†æ–‡æ¡£ï¼Œä½¿å¾—内容看起æ¥å·²è¢«ä¿®æ”¹ï¼Œå¹¶ä¸”æŸäº›å±žæ€§çœ‹èµ·æ¥å¯èƒ½æœ‰ç¼ºå¤±ã€‚在这ç§æƒ…况下，请使用通过æµè§ˆå™¨æ‰¾åˆ°çš„原始å“应数æ®ã€‚ + +以下是 Firefox 的示例æµç¨‹ï¼Œå…¶ä»–æµè§ˆå™¨å¯èƒ½ä¼šç•¥æœ‰ä¸åŒï¼š + +1. 按下 **F12** 访问开å‘者控制å°ã€‚ +1. 点击 **Network** 选项å¡ã€‚ +1. 从表中，å•å‡»åŒ…å« `descriptor` 的行。 +1. 在 details 窗格中，å•å‡» **Response** 选项å¡ã€‚ +1. å¤åˆ¶åŽŸå§‹å“应数æ®ã€‚ + + + + +## 在 Rancher 中é…ç½® Keycloak + + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Keycloak SAML**。 +1. 填写**é…ç½® Keycloak è´¦å·**表å•ã€‚有关填写表å•çš„帮助，请å‚è§[é…ç½®å‚考](#é…ç½®å‚考)。 +1. 完æˆ**é…ç½® Keycloak è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Keycloak IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Keycloak é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 Keycloak。你的用户现在å¯ä»¥ä½¿ç”¨ Keycloak 登录å登录 Rancher。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## é…ç½®å‚考 + +| 字段 | æè¿° | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 显示å称字段 | 包å«ç”¨æˆ·æ˜¾ç¤ºå称的属性。

示例：`givenName` | +| 用户å字段 | 包å«ç”¨æˆ·å/给定å称的属性。

示例：`email` | +| UID 字段 | æ¯ä¸ªç”¨æˆ·ç‹¬æœ‰çš„属性。

示例：`email` | +| 用户组字段 | 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ã€‚

示例：`member` | +| Entity ID 字段 | Keycloak 客户端中需è¦é…置为客户端的 ID。

默认值：`https://yourRancherHostURL/v1-saml/keycloak/saml/metadata` | +| Rancher API 主机 | Rancher Server çš„ URL。 | +| ç§é’¥/è¯ä¹¦ | 在 Rancher 和你的 IdP 之间创建安全外壳（SSH）的密钥/è¯ä¹¦å¯¹ã€‚ | +| IDP å…ƒæ•°æ® | 从 IdP æœåŠ¡å™¨å¯¼å‡ºçš„ `metadata.xml` 文件。 | + +:::tip + +ä½ å¯ä»¥ä½¿ç”¨ openssl 命令生æˆä¸€ä¸ªå¯†é’¥/è¯ä¹¦å¯¹ã€‚例如： + +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.cert + +::: + +## 附录：故障排除 + +如果你在测试与 Keycloak æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·å…ˆæ£€æŸ¥ SAML 客户端的é…置选项。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 + +### ä¸èƒ½é‡å®šå‘到 Keycloak + +点击**使用 Keycloak 认è¯**时，没有é‡å®šå‘到你的 IdP。 + +* 验è¯ä½ çš„ Keycloak 客户端é…置。 +* ç¡®ä¿ `Force Post Binding` 设为 `OFF`。 + + +### IdP 登录åŽæ˜¾ç¤ºç¦æ­¢æ¶ˆæ¯ + +你已正确é‡å®šå‘到你的 IdP 登录页é¢ï¼Œå¹¶ä¸”å¯ä»¥è¾“入凭è¯ï¼Œä½†æ˜¯ä¹‹åŽæ”¶åˆ° `Forbidden` 消æ¯ã€‚ + +* 检查 Rancher 调试日志。 +* 如果日志显示 `ERROR: either the Response or Assertion must be signed`ï¼Œç¡®ä¿ `Sign Documents` 或 `Sign assertions` 在 Keycloak 客户端中设置为 `ON`。 + +### 访问 `/v1-saml/keycloak/saml/metadata` 时返回 HTTP 502 + +常è§åŽŸå› ï¼šé…ç½® SAML æ供商之å‰æœªåˆ›å»ºå…ƒæ•°æ®ã€‚ +å°è¯•é…ç½® Keycloak，并将它ä¿å­˜ä¸ºä½ çš„ SAML æ供商，然åŽè®¿é—®å…ƒæ•°æ®ã€‚ + +### Keycloak 错误："We're sorry, failed to process response" + +* 检查你的 Keycloak 日志。 +* 如果日志显示 `failed: org.keycloak.common.VerificationException: Client does not have a public key`，请在 Keycloak 客户端中将 `Encrypt Assertions` 设为 `OFF`。 + +### Keycloak 错误："We're sorry, invalid requester" + +* 检查你的 Keycloak 日志。 +* 如果日志显示 `request validation failed: org.keycloak.common.VerificationException: SigAlg was null`，请在 Keycloak 客户端中将 `Client Signature Required` 设为 `OFF`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md new file mode 100644 index 00000000000..57cbaf95418 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md @@ -0,0 +1,107 @@ +--- +title: é…ç½® Okta (SAML) +--- + +如果你的组织使用 Okta Identity Provider (IdP) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚ + +:::note + +Okta 集æˆä»…支æŒæœåŠ¡æ供商å‘起的登录。 + +::: +## 先决æ¡ä»¶ + +在 Okta 中，使用以下设置创建一个新的 SAML 应用。如需获å–帮助，请å‚è§ [Okta 文档](https://developer.okta.com/standards/SAML/setting_up_a_saml_application_in_okta)。 + +| 设置 | 值 | +------------|------------ +| `Single Sign on URL` | `https://yourRancherHostURL/v1-saml/okta/saml/acs` | +| `Audience URI (SP Entity ID)` | `https://yourRancherHostURL/v1-saml/okta/saml/metadata` | + +## 在 Rancher 中é…ç½® Okta + +ä½ å¯ä»¥å°† Okta 与 Rancher 集æˆï¼Œä»¥ä¾¿ç»è¿‡èº«ä»½è®¤è¯çš„用户通过组æƒé™è®¿é—® Rancher 资æºã€‚Okta 会返回一个对用户进行身份认è¯çš„ SAML 断言，包括用户所属的组。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Okta**。 +1. 填写**é…ç½® Okta è´¦å·**表å•ã€‚下é¢çš„示例æ述了如何将 Okta 属性从属性语å¥æ˜ å°„到 Rancher 中的字段： + + | 字段 | æè¿° | + | ------------------------- | ----------------------------------------------------------------------------- | + | 显示å称字段 | 属性语å¥ä¸­åŒ…å«ç”¨æˆ·æ˜¾ç¤ºå称的属性å称。 | + | 用户å字段 | 属性语å¥ä¸­åŒ…å«ç”¨æˆ·å/给定å称的属性å称。 | + | UID 字段 | 属性语å¥ä¸­æ¯ä¸ªç”¨æˆ·å”¯ä¸€çš„属性å称。 | + | 用户组字段 | 组属性语å¥ä¸­å…¬å¼€ä½ çš„组的属性å称。 | + | Rancher API 主机 | Rancher Server çš„ URL。 | + | ç§é’¥/è¯ä¹¦ | 密钥/è¯ä¹¦å¯¹ï¼Œç”¨äºŽæ–­è¨€åŠ å¯†ã€‚ | + | å…ƒæ•°æ® XML | 你在应用 `Sign On` 部分中找到的 `Identity Provider metadata` 文件。 | + + :::tip + + ä½ å¯ä»¥ä½¿ç”¨ openssl 命令生æˆä¸€ä¸ªå¯†é’¥/è¯ä¹¦å¯¹ã€‚例如： + + ``` + openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.crt + ``` + + ::: + + +1. 完æˆ**é…ç½® Okta è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Okta IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Okta é…置。 + + :::note + + 如果什么都没有å‘生，很å¯èƒ½æ˜¯å› ä¸ºä½ çš„æµè§ˆå™¨é˜»æ­¢äº†å¼¹å‡ºçª—å£ã€‚请在弹出窗å£é˜»æ­¢ç¨‹åºä¸­ç¦ç”¨ Rancher 域，并在其他类似扩展中将 Rancher 列入白åå•ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 Okta。你的用户现在å¯ä»¥ä½¿ç”¨ Okta 登录å登录 Rancher。 + +:::note SAML 身份æ供商注æ„事项 + +如果你在没有 OpenLDAP 的情况下é…ç½® Okta，你将无法æœç´¢æˆ–直接查找用户或组。相关的警告如下： + +- 在 Rancher 中为用户和组分é…æƒé™æ—¶å°†ä¸ä¼šéªŒè¯ç”¨æˆ·å’Œç»„。 +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## Okta 与 OpenLDAP æœç´¢ + +ä½ å¯ä»¥æ·»åŠ  OpenLDAP åŽç«¯æ¥å助用户和组æœç´¢ã€‚Rancher 将显示æ¥è‡ª OpenLDAP æœåŠ¡çš„其他用户和组。这å…许你将æƒé™åˆ†é…给用户ä¸å±žäºŽçš„组。 + +### OpenLDAP 先决æ¡ä»¶ + +如果你使用 Okta 作为 IdP，你å¯ä»¥[设置 LDAP 接å£](https://help.okta.com/en-us/Content/Topics/Directory/LDAP-interface-main.htm)以供 Rancher 使用。你还å¯ä»¥é…置外部 OpenLDAP Server。 + +你必须使用 LDAP 绑定å¸æˆ·ï¼ˆä¹Ÿç§°ä¸º ServiceAccount）æ¥é…ç½® Rancher，以便æœç´¢å’Œæ£€ç´¢åº”具有访问æƒé™çš„用户和组的 LDAP æ¡ç›®ã€‚ä¸è¦ä½¿ç”¨ç®¡ç†å‘˜å¸æˆ·æˆ–个人å¸æˆ·ä½œä¸º LDAP 绑定å¸æˆ·ã€‚在 OpenLDAP 中[创建](https://help.okta.com/en-us/Content/Topics/users-groups-profiles/usgp-add-users.htm)一个专用å¸æˆ·ï¼Œå¯¹æœç´¢åº“下的用户和组具有åªè¯»è®¿é—®æƒé™ã€‚ + +:::warning 安全注æ„事项 + +OpenLDAP ServiceAccount 用于所有æœç´¢ã€‚无论用户个人的 SAML æƒé™æ˜¯ä»€ä¹ˆï¼ŒRancher 用户都将看到 OpenLDAP ServiceAccount å¯ä»¥æŸ¥çœ‹çš„用户和组。 + +::: + + +> **使用 TLS？** +> +> 如果 OpenLDAP Server 使用的è¯ä¹¦æ˜¯è‡ªç­¾å的或æ¥è‡ªæ— æ³•è¯†åˆ«çš„è¯ä¹¦é¢å‘机构，则 Rancher éœ€è¦ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你需è¦åœ¨é…置期间æ供此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +### 在 Rancher 中é…ç½® OpenLDAP + +[é…ç½® OpenLDAP Serverã€ç»„和用户的设置](../configure-openldap/openldap-config-reference.md)。请注æ„，ä¸æ”¯æŒåµŒå¥—组æˆå‘˜ã€‚ + +> 在继续é…置之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份认è¯é…置和主è¦ç”¨æˆ·](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + +1. 使用分é…了 [administrator](https://ranchermanager.docs.rancher.com/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions) è§’è‰²ï¼ˆå³ _本地主体_）的本地用户登录到 Rancher。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Okta**，如果已é…ç½® SAML，则å•å‡»**编辑é…ç½®**。 +1. 在**用户和组æœç´¢**下，选中**é…ç½® OpenLDAP Server**。 + +如果你在测试与 OpenLDAP Server 的连接时é‡åˆ°é—®é¢˜ï¼Œè¯·ç¡®ä¿ä½ è¾“入了ServiceAccount 的凭è¯å¹¶æ­£ç¡®é…置了æœç´¢åº“。你å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žæ ¹æœ¬åŽŸå› ã€‚调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚请å‚阅[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../../../../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)了解更多信æ¯ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md new file mode 100644 index 00000000000..0eebb8363c9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md @@ -0,0 +1,62 @@ +--- +title: é…ç½® PingIdentity (SAML) +--- + +如果你的组织使用 Ping Identity Provider (IdP) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 IdP 凭è¯ç™»å½•ã€‚ + +> **先决æ¡ä»¶**： +> +> - ä½ å¿…é¡»é…置了 [Ping IdP æœåŠ¡å™¨](https://www.pingidentity.com/)。 +> - 以下是 Rancher Service Provider é…置所需的 URL： +> å…ƒæ•°æ® URL：`https:///v1-saml/ping/saml/metadata` +> 断言使用者æœåŠ¡ (ACS) URL：`https:///v1-saml/ping/saml/acs` +> 请注æ„，在 Rancher 中ä¿å­˜éªŒè¯é…置之å‰ï¼Œè¿™äº› URL ä¸ä¼šè¿”回有效数æ®ã€‚ +> - 从 IdP æœåŠ¡å™¨å¯¼å‡º `metadata.xml` 文件。详情请å‚è§ [PingIdentity 文档](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html)。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Ping Identity**。 +1. 填写**é…ç½® Ping è´¦å·**表å•ã€‚Ping IdP å…许你指定è¦ä½¿ç”¨çš„æ•°æ®å­˜å‚¨ã€‚ä½ å¯ä»¥æ·»åŠ æ•°æ®åº“或使用现有的 ldap æœåŠ¡å™¨ã€‚例如，如果你选择 Active Directory (AD) æœåŠ¡å™¨ï¼Œä¸‹é¢çš„示例将æ述如何将 AD 属性映射到 Rancher 中的字段： + + 1. **显示å称字段**：包å«ç”¨æˆ·æ˜¾ç¤ºå称的 AD 属性（例如：`displayName`）。 + + 1. **用户å字段**：包å«ç”¨æˆ·å/给定å称的 AD 属性（例如：`givenName`）。 + + 1. **UID 字段**：æ¯ä¸ªç”¨æˆ·å”¯ä¸€çš„ AD 属性（例如：`sAMAccountName`ã€`distinguishedName`）。 + + 1. **用户组字段**: 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ï¼ˆä¾‹å¦‚：`memberOf`）。 + + 1. **Entity ID 字段**（å¯é€‰ï¼‰ï¼šä½ çš„åˆä½œä¼™ä¼´å·²å…¬å¸ƒçš„ã€ä¾èµ–å议的ã€å”¯ä¸€çš„标识符。该 ID 将你的组织定义为将æœåŠ¡å™¨ç”¨äºŽ SAML 2.0 事务的实体。这个 ID å¯ä»¥é€šè¿‡å¸¦å¤–传输或 SAML 元数æ®æ–‡ä»¶èŽ·å¾—。 + + 1. **Rancher API 主机**：你的 Rancher Server çš„ URL。 + + 1. **ç§é’¥**å’Œ**è¯ä¹¦**：密钥/è¯ä¹¦å¯¹ï¼Œç”¨äºŽåœ¨ Rancher 和你的 IdP 之间创建一个安全外壳（SSH）。 + + ä½ å¯ä»¥ä½¿ç”¨ openssl 命令进行创建。例如： + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP 元数æ®**：[从 IdP æœåŠ¡å™¨å¯¼å‡ºçš„ `metadata.xml` 文件](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html)。 + + +1. 完æˆ**é…ç½® Ping è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Ping IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher PingIdentity é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 PingIdentity。你的用户现在å¯ä»¥ä½¿ç”¨ PingIdentity 登录å登录 Rancher。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md new file mode 100644 index 00000000000..e6589e67693 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md @@ -0,0 +1,15 @@ +--- +title: æœ¬åœ°èº«ä»½éªŒè¯ +--- + +在é…置外部验è¯æ供程åºä¹‹å‰ï¼Œä½ å°†é»˜è®¤ä½¿ç”¨æœ¬åœ°èº«ä»½éªŒè¯ã€‚Rancher 将用户å¸æˆ·ä¿¡æ¯ï¼ˆä¾‹å¦‚用户å和密ç ï¼‰å­˜å‚¨åœ¨æœ¬åœ°ã€‚默认情况下，用于首次登录 Rancher çš„ `admin` 用户就是一个本地用户。 + +## 添加本地用户 + +无论是å¦ä½¿ç”¨å¤–部身份验è¯æœåŠ¡ï¼Œä½ éƒ½åº”创建一些本地身份认è¯çš„用户，以便在外部验è¯æœåŠ¡é‡åˆ°é—®é¢˜æ—¶ç»§ç»­ä½¿ç”¨ Rancher。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**用户**。 +1. å•å‡»**创建**。 +1. 完æˆ**添加用户**的表å•ã€‚ +1. å•å‡»**创建**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md new file mode 100644 index 00000000000..b44cc440312 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md @@ -0,0 +1,77 @@ +--- +title: 用户和组 +--- + +Rancher ä¾èµ–用户和组æ¥å†³å®šå…许登录到 Rancher 的用户，以åŠä»–们å¯ä»¥è®¿é—®å“ªäº›èµ„æºã€‚ä½ é…置外部身份验è¯æ供程åºåŽï¼Œè¯¥æ供程åºçš„用户将能够登录到你的 Rancher Server。用户登录时，验è¯æ供程åºå°†å‘ä½ çš„ Rancher Server æ供该用户所属的组列表。 + +ä½ å¯ä»¥é€šè¿‡å‘资æºæ·»åŠ ç”¨æˆ·æˆ–组，æ¥æŽ§åˆ¶å…¶å¯¹é›†ç¾¤ã€é¡¹ç›®ã€å¤šé›†ç¾¤åº”用ã€å…¨å±€ DNS æ供程åºå’Œç›¸å…³èµ„æºçš„访问。将组添加到资æºæ—¶ï¼Œèº«ä»½éªŒè¯æ供程åºä¸­å±žäºŽè¯¥ç»„的所有用户都将能够使用组的æƒé™è®¿é—®è¯¥èµ„æºã€‚有关角色和æƒé™çš„更多信æ¯ï¼Œè¯·å‚è§ [RBAC](../../../../pages-for-subheaders/manage-role-based-access-control-rbac.md)。 + +## 管ç†æˆå‘˜ + +å‘资æºæ·»åŠ ç”¨æˆ·æˆ–用户组时，你å¯ä»¥é€šè¿‡è¾“入用户或组的å称æ¥æœç´¢ç”¨æˆ·æˆ–组。Rancher Server 会查询身份验è¯æ供程åºï¼Œæ¥æŸ¥æ‰¾ä¸Žä½ è¾“入的内容匹é…的用户和组。æœç´¢ä»…é™äºŽä½ ç™»å½•æ—¶ä½¿ç”¨çš„身份验è¯æ供程åºã€‚例如，如果你å¯ç”¨äº† GitHub 身份验è¯ï¼Œä½†ä½¿ç”¨[本地](create-local-users.md)用户登录，则无法æœç´¢ GitHub 用户或组。 + +ä½ å¯ä»¥æŸ¥çœ‹å’Œç®¡ç†æ‰€æœ‰ç”¨æˆ·ï¼ŒåŒ…括本地用户和æ¥è‡ªèº«ä»½éªŒè¯æ供程åºçš„用户。在左上角，å•å‡» **☰ > 用户 & 认è¯**。在左侧导航æ ä¸­å•å‡»**用户**。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## ç”¨æˆ·ä¿¡æ¯ + +Rancher 会维护通过身份验è¯æ供程åºç™»å½•çš„æ¯ä¸ªç”¨æˆ·çš„ä¿¡æ¯ï¼ŒåŒ…括用户是å¦å…许访问 Rancher Server，以åŠç”¨æˆ·æ‰€å±žçš„组的列表。Rancher ä¿ç•™æ­¤ç”¨æˆ·ä¿¡æ¯ï¼Œä»¥ä¾¿ CLIã€API å’Œ kubectl 能够准确地å映用户基于身份验è¯æ供程åºä¸­çš„组æˆå‘˜å…³ç³»çš„访问。 + +当用户使用身份验è¯æ供程åºç™»å½•åˆ° UI 时，Rancher 将自动更新该用户信æ¯ã€‚ + +### è‡ªåŠ¨åˆ·æ–°ç”¨æˆ·ä¿¡æ¯ + +Rancher 会定期刷新用户信æ¯ï¼Œç”šè‡³åœ¨ç”¨æˆ·é€šè¿‡ UI 登录之å‰ä¹Ÿæ˜¯å¦‚此。你å¯ä»¥æŽ§åˆ¶ Rancher 执行刷新的频率。 + +有两个å‚æ•°å¯ä»¥æŽ§åˆ¶è¿™ä¸ªæ“作： + +- **`auth-user-info-max-age-seconds`** + + 此设置控制用户信æ¯çš„最大è€åŒ–时间，如果超过这个时间，Rancher 就会刷新信æ¯ã€‚如果用户进行 API 调用（直接 UI 访问或通过使用 Rancher CLI 或 kubectl 调用），并且与 Rancher 上次刷新用户信æ¯çš„时间间隔大于此设置，则 Rancher 将触å‘刷新。此设置默认为 `3600` ç§’ï¼Œå³ 1 å°æ—¶ã€‚ + +- **`auth-user-info-resync-cron`** + + 此设置控制用于所有用户é‡æ–°åŒæ­¥èº«ä»½éªŒè¯æ供程åºä¿¡æ¯çš„定期任务周期。无论用户最近是å¦ç™»å½•æˆ–使用 API，自动刷新任务都会按照指定的时间间隔刷新用户信æ¯ã€‚此设置默认为 `0 0 * * *`，å³æ¯å¤©åˆå¤œè¿›è¡Œä¸€æ¬¡ã€‚有关此设置的有效值的更多信æ¯ï¼Œè¯·å‚è§ [Cron 文档](https://en.wikipedia.org/wiki/Cron)。 + +如果需è¦æ›´æ”¹è®¾ç½®ï¼š + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å‰å¾€ä½ éœ€è¦é…置的设置，并点击 **â‹® > 编辑设置**。 + +:::note + +由于 SAML ä¸æ”¯æŒç”¨æˆ·æŸ¥æ‰¾ï¼Œå› æ­¤åŸºäºŽ SAML 的身份验è¯æ供程åºä¸æ”¯æŒå®šæœŸåˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚åªæœ‰å½“用户登录到 Rancher UI 时，æ‰ä¼šåˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚ + +::: +### æ‰‹åŠ¨åˆ·æ–°ç”¨æˆ·ä¿¡æ¯ + +如果你ä¸ç¡®å®š Rancher 上一次执行用户信æ¯è‡ªåŠ¨åˆ·æ–°çš„时间，则å¯ä»¥é€šè¿‡æ‰‹åŠ¨åˆ·æ–°æ¥åˆ·æ–°æ‰€æœ‰ç”¨æˆ·çš„ä¿¡æ¯ã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在**用户**页é¢ï¼Œå•å‡»**刷新用户组æˆå‘˜åå•**。 + +**结果**：Rancher 刷新了所有用户的信æ¯ã€‚请求此刷新将更新哪些用户å¯ä»¥è®¿é—® Rancher 以åŠæ¯ä¸ªç”¨æˆ·æ‰€å±žçš„所有组。 + +:::note + +由于 SAML ä¸æ”¯æŒç”¨æˆ·æŸ¥æ‰¾ï¼Œå› æ­¤åŸºäºŽ SAML 的身份验è¯æ供程åºä¸æ”¯æŒæ‰‹åŠ¨åˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚åªæœ‰å½“用户登录到 Rancher UI 时，æ‰ä¼šåˆ·æ–°ç”¨æˆ·ä¿¡æ¯ã€‚ + +::: + +## 会è¯å‘¨æœŸ + +用户会è¯çš„默认生命周期（TTL）是å¯è°ƒçš„。默认的会è¯å‘¨æœŸæ˜¯ 16 å°æ—¶ã€‚ + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å‰å¾€ **`auth-user-session-ttl-minutes`** 并å•å‡»**â‹® > 编辑设置**。 +1. 输入会è¯åº”该æŒç»­çš„时间（以分钟为å•ä½ï¼‰ï¼Œç„¶åŽå•å‡»**Save**。 + +**结果**：用户的 Rancher 登录会è¯åœ¨è®¾å®šçš„分钟数åŽè‡ªåŠ¨é€€å‡ºã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md new file mode 100644 index 00000000000..1bcf056acd8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md @@ -0,0 +1,83 @@ +--- +title: 1. 在 Microsoft AD FS 中é…ç½® Rancher +--- + +在é…ç½® Rancher ä»¥æ”¯æŒ AD FS 用户之å‰ï¼Œä½ å¿…须在 AD FS 中将 Rancher 添加为 [relying party trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/technical-reference/understanding-key-ad-fs-concepts)（信赖方信任）。 + +1. 以管ç†ç”¨æˆ·èº«ä»½ç™»å½• AD æœåŠ¡å™¨ã€‚ + +1. 打开 **AD FS Management** 控制å°ã€‚在 **Actions** èœå•ä¸­é€‰æ‹© **Add Relying Party Trust...**。然åŽå•å‡» **Start**。 + + ![](/img/adfs/adfs-overview.png) + +1. 选择 **Enter data about the relying party manually** 作为获å–信赖方数æ®çš„选项。 + + ![](/img/adfs/adfs-add-rpt-2.png) + +1. 为 **Relying Party Trust** 设置**显示å称**，例如 `Rancher`。 + + ![](/img/adfs/adfs-add-rpt-3.png) + +1. 选择 **AD FS profile** 作为信赖方信任的é…置文件。 + + ![](/img/adfs/adfs-add-rpt-4.png) + +1. 留空 **optional token encryption certificate**，因为 Rancher AD FS ä¸ä¼šä½¿ç”¨å®ƒã€‚ + + ![](/img/adfs/adfs-add-rpt-5.png) + +1. 选择 **Enable support for the SAML 2.0 WebSSO protocol** 并在 Service URL 处输入 `https:///v1-saml/adfs/saml/acs`。 + + ![](/img/adfs/adfs-add-rpt-6.png) + +1. å°† `https:///v1-saml/adfs/saml/metadata` 添加为 **Relying party trust identifier**。 + + ![](/img/adfs/adfs-add-rpt-7.png) + +1. 本教程ä¸æ¶‰åŠå¤šé‡èº«ä»½éªŒè¯ã€‚如果你想é…置多é‡èº«ä»½éªŒè¯ï¼Œè¯·å‚è§ [Microsoft 文档](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/configure-additional-authentication-methods-for-ad-fs)。 + + ![](/img/adfs/adfs-add-rpt-8.png) + +1. 在 **Choose Issuance Authorization RUles** 中，你å¯ä»¥æ ¹æ®ç”¨ä¾‹é€‰æ‹©ä»»ä½•ä¸€ä¸ªå¯ç”¨é€‰é¡¹ã€‚但是考虑到本指å—的目的，请选择 **Permit all users to access this relying party**。 + + ![](/img/adfs/adfs-add-rpt-9.png) + +1. 检查所有设置åŽï¼Œé€‰æ‹© **Next** æ¥æ·»åŠ ä¿¡èµ–方信任。 + + ![](/img/adfs/adfs-add-rpt-10.png) + + +1. 选择 **Open the Edit Claim Rules...**。然åŽå•å‡» **Close**。 + + ![](/img/adfs/adfs-add-rpt-11.png) + +1. 在 **Issuance Transform Rules** 选项å¡ä¸­ï¼Œå•å‡» **Add Rule...**。 + + ![](/img/adfs/adfs-edit-cr.png) + +1. 在 **Claim rule template** 中选择 **Send LDAP Attributes as Claims**。 + + ![](/img/adfs/adfs-add-tcr-1.png) + +1. å°† **Claim rule name** 设置为所需的å称（例如 `Rancher Attributes`）并选择 **Active Directory** 作为 **Attribute store**。创建对应下表的映射： + + | LDAP 属性 | 传出声明类型 | + | -------------------------------------------- | ------------------- | + | Given-Name | Given Name | + | User-Principal-Name | UPN | + | Token-Groups - Qualified by Long Domain Name | Group | + | SAM-Account-Name | å称 | +
+ + ![](/img/adfs/adfs-add-tcr-2.png) + +1. 从 AD æœåŠ¡å™¨çš„以下ä½ç½®ä¸‹è½½ `federationmetadata.xml`： +``` +https:///federationmetadata/2007-06/federationmetadata.xml +``` + +**结果**：你已将 Rancher 添加为ä¾èµ–信任方。现在你å¯ä»¥é…ç½® Rancher æ¥ä½¿ç”¨ AD。 + +### åŽç»­æ“作 + +[在 Rancher 中é…ç½® Microsoft AD FS ](configure-rancher-for-ms-adfs.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md new file mode 100644 index 00000000000..b4a15d2bf4c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md @@ -0,0 +1,54 @@ +--- +title: 2. 在 Rancher 中é…ç½® Microsoft AD FS +--- + +完æˆ[在 Microsoft AD FS 中é…ç½® Rancher](configure-ms-adfs-for-rancher.md) åŽï¼Œå°†ä½ çš„ Active Directory Federation Service (AD FS) ä¿¡æ¯è¾“å…¥ Rancher，以便 AD FS 用户å¯ä»¥é€šè¿‡ Rancher 进行身份认è¯ã€‚ + +:::note é…ç½® ADFS æœåŠ¡å™¨çš„é‡è¦è¯´æ˜Žï¼š + +- SAML 2.0 WebSSO åè®®æœåŠ¡ URL 为：`https:///v1-saml/adfs/saml/acs` +- 信赖方信任标识符 URL 为：`https:///v1-saml/adfs/saml/metadata` +- 你必须从 AD FS æœåŠ¡å™¨å¯¼å‡º `federationmetadata.xml` 文件。你å¯ä»¥åœ¨ `https:///federationmetadata/2007-06/federationmetadata.xml` 中找到该文件。 + +::: + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **ADFS**。 +1. 填写**é…ç½® AD FS è´¦å·**表å•ã€‚Microsoft AD FS å…许你指定现有的 Active Directory (AD) æœåŠ¡å™¨ã€‚[以下é…置示例](#é…ç½®)æ述了如何将 AD 属性映射到 Rancher 中的字段。 +1. 完æˆ**é…ç½® AD FS è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 AD FS 登录页é¢ã€‚输入使用 Microsoft AD FS 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher AD FS é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° AD FS 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 AD FS。你的用户现在å¯ä»¥ä½¿ç”¨ AD FS 登录å登录 Rancher。 + +## é…ç½® + +| 字段 | æè¿° | +|---------------------------|-----------------| +| 显示å称字段 | 包å«ç”¨æˆ·æ˜¾ç¤ºå称的 AD 属性。

示例：`http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name` | +| 用户å字段 | 包å«ç”¨æˆ·å/给定å称的 AD 属性。

示例：`http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname` | +| UID 字段 | æ¯ä¸ªç”¨æˆ·ç‹¬æœ‰çš„ AD 属性。

示例：`http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn` | +| 用户组字段 | 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ã€‚

示例：`http://schemas.xmlsoap.org/claims/Group` | +| Rancher API 主机 | Rancher Server çš„ URL。 | +| ç§é’¥/è¯ä¹¦ | 在 Rancher 和你的 AD FS 之间创建安全外壳（SSH）的密钥/è¯ä¹¦å¯¹ã€‚ç¡®ä¿å°† Common Name (CN) 设置为 Rancher Server URL。

[è¯ä¹¦åˆ›å»ºå‘½ä»¤](#cert-command) | +| å…ƒæ•°æ® XML | 从 AD FS æœåŠ¡å™¨å¯¼å‡ºçš„ `federationmetadata.xml` 文件。

ä½ å¯ä»¥åœ¨ `https:///federationmetadata/2007-06/federationmetadata.xml` 找到该文件。 | + + + + +:::tip + +ä½ å¯ä»¥ä½¿ç”¨ openssl 命令生æˆè¯ä¹¦ã€‚例如： + +``` +openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" +``` + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md new file mode 100644 index 00000000000..581a85efc2c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md @@ -0,0 +1,78 @@ +--- +title: OpenLDAP é…ç½®å‚考 +--- + +有关é…ç½® OpenLDAP 认è¯çš„更多详细信æ¯ï¼Œè¯·å‚è§[官方文档](https://www.openldap.org/doc/)。 + +> 在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + + +## 背景：OpenLDAP 认è¯æµç¨‹ + +1. 当用户å°è¯•ä½¿ç”¨å…¶ LDAP 凭è¯ç™»å½•æ—¶ï¼ŒRancher 会使用具有æœç´¢ç›®å½•å’Œè¯»å–用户/组属性æƒé™çš„ ServiceAccount，创建与 LDAP æœåŠ¡å™¨çš„åˆå§‹ç»‘定。 +2. 然åŽï¼ŒRancher 使用æœç´¢ç­›é€‰å™¨æ ¹æ®ç”¨æˆ·åå’Œé…置的属性映射为用户æœç´¢ç›®å½•ã€‚ +3. 找到用户åŽï¼Œå°†ä½¿ç”¨ç”¨æˆ·çš„ DN å’Œæ供的密ç ï¼Œé€šè¿‡å¦ä¸€ä¸ª LDAP 绑定请求对用户进行身份验è¯ã€‚ +4. 身份验è¯æˆåŠŸåŽï¼ŒRancher 将基于用户对象的æˆå‘˜å±žæ€§å’Œé…置的用户映射属性执行组æœç´¢ï¼Œæ¥è§£æžç»„æˆå‘˜ã€‚ + +## OpenLDAP æœåŠ¡å™¨é…ç½® + +你将需è¦è¾“入地å€ï¼Œç«¯å£å’Œåè®®æ¥è¿žæŽ¥åˆ° OpenLDAP æœåŠ¡å™¨ã€‚ä¸å®‰å…¨æµé‡çš„标准端å£ä¸º `389`，TLS æµé‡çš„标准端å£ä¸º `636`。 + +> **使用 TLS？** +> +> 如果 OpenLDAP æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +如果你ä¸ç¡®å®šè¦åœ¨ç”¨æˆ·/组`æœç´¢åº“`字段中输入什么值，请咨询你的 LDAP 管ç†å‘˜ï¼Œæˆ–å‚è§ Active Directory 身份验è¯æ–‡æ¡£ä¸­çš„[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema) 章节。 + +
OpenLDAP æœåŠ¡å™¨å‚æ•°
+ +| å‚æ•° | æè¿° | +|:--|:--| +| 主机å | 指定 OpenLDAP æœåŠ¡å™¨çš„主机å或 IP 地å€ã€‚ | +| ç«¯å£ | 指定 OpenLDAP æœåŠ¡å™¨ç›‘å¬è¿žæŽ¥çš„端å£ã€‚未加密的 LDAP 通常使用 389 的标准端å£ï¼Œè€Œ LDAPS 使用 636 端å£ã€‚ | +| TLS | 选中此框å¯å¯ç”¨ SSL/TLS 上的 LDAP（通常称为 LDAPS）。如果æœåŠ¡å™¨ä½¿ç”¨è‡ªç­¾å/ä¼ä¸šç­¾åçš„è¯ä¹¦ï¼Œåˆ™è¿˜éœ€è¦ç²˜è´´ CA è¯ä¹¦ã€‚ | +| æœåŠ¡å™¨è¿žæŽ¥è¶…æ—¶ | Rancher 在认为无法访问æœåŠ¡å™¨ä¹‹å‰ç­‰å¾…的时间（秒）。 | +| ServiceAccount 标识å称 | 输入用于绑定，æœç´¢å’Œæ£€ç´¢ LDAP æ¡ç›®çš„用户的标识å称（DN）。 | +| ServiceAccount å¯†ç  | ServiceAccount 的密ç ã€‚ | +| 用户æœç´¢åº“ | 输入目录树中开始æœç´¢ç”¨æˆ·å¯¹è±¡çš„节点的标识å称（DN）。所有用户都必须是此基础标识å称的åŽä»£ã€‚例如，"ou=people,dc=acme,dc=com"。 | +| 组æœç´¢åº“ | 如果组ä½äºŽ`用户æœç´¢åº“`下é…置的节点之外的其他节点下，则需è¦åœ¨æ­¤å¤„æ供标识å称。å¦åˆ™ï¼Œå°†æ­¤å­—段留空。例如："ou=groups,dc=acme,dc=com"。 | + +## 用户/组 Schema é…ç½® + +如果你的 OpenLDAP 目录ä¸åŒäºŽæ ‡å‡†çš„ OpenLDAP Schema，则必须完æˆ**自定义 Schema** 部分实现匹é…。 + +请注æ„，Rancher 使用本节中é…置的属性映射æ¥æž„造æœç´¢ç­›é€‰å™¨å’Œè§£æžç»„æˆå‘˜ã€‚因此，我们建议你验è¯æ­¤å¤„çš„é…置是å¦ä¸Žä½ åœ¨ OpenLDAP 中使用的 Schema 匹é…。 + +如果你ä¸ç¡®å®š OpenLDAP æœåŠ¡å™¨ä¸­ä½¿ç”¨çš„用户/组 Schema，请咨询你的 LDAP 管ç†å‘˜ï¼Œæˆ–å‚è§ Active Directory 身份验è¯æ–‡æ¡£ä¸­çš„[使用 ldapsearch 确定æœç´¢åº“å’Œ Schema](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#附录使用-ldapsearch-确定æœç´¢åº“å’Œ-schema) 章节。 + +### 用户 Schema é…ç½® + +下表详细说明了用户 Schema é…置的å‚数。 + +
用户 Schema é…ç½®å‚æ•°
+ +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于用户对象的对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Username Attribute | 用户属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Login Attribute | 登录属性的值与用户登录 Rancher 时输入的凭è¯çš„用户å部分匹é…。通常是 `uid`。 | +| User Member Attribute | 包å«ç”¨æˆ·æ‰€å±žç»„的标识å称的用户属性。通常是 `memberOf` 或 `isMemberOf`。 | +| Search Attribute | 当用户输入文本以在用户界é¢ä¸­æ·»åŠ ç”¨æˆ·æˆ–组时，Rancher 会查询 LDAP æœåŠ¡å™¨ï¼Œå¹¶å°è¯•æ ¹æ®æ­¤è®¾ç½®ä¸­æ供的属性匹é…用户。å¯ä»¥é€šè¿‡ä½¿ç”¨ç®¡é“（“\|â€ï¼‰ç¬¦å·åˆ†éš”属性æ¥æŒ‡å®šå¤šä¸ªå±žæ€§ã€‚ | +| User Enabled Attribute | 如果 OpenLDAP æœåŠ¡å™¨çš„ Schema 支æŒä½¿ç”¨ç”¨æˆ·å±žæ€§çš„值æ¥è¯„ä¼°è´¦å·æ˜¯ç¦ç”¨è¿˜æ˜¯å…³é—­ï¼Œè¯·è¾“入该属性的å称。默认的 OpenLDAP Schema ä¸æ”¯æŒæ­¤åŠŸèƒ½ï¼Œå› æ­¤æ­¤å­—段通常留空。 | +| Disabled Status Bitmask | ç¦ç”¨/é”定的用户账å·çš„值。如果 `User Enabled Attribute` 是空的，则忽略此å‚数。 | + +### 组 Schema é…ç½® + +下表详细说明了组 Schema é…置的å‚数。 + +
组 Schema é…ç½®å‚æ•°
+ +| å‚æ•° | æè¿° | +|:--|:--| +| Object Class | 域中用于组æ¡ç›®çš„对象类别å称。如果定义了此å‚数，则仅指定对象类别的å称 - *请勿*将其放在 LDAP 包装器中，例如 `&(objectClass=xxxx)`。 | +| Name Attribute | å称属性的值适åˆä½œä¸ºæ˜¾ç¤ºå称。 | +| Group Member User Attribute | **用户属性**çš„å称。它的格å¼ä¸Ž `Group Member Mapping Attribute` 中的组æˆå‘˜åŒ¹é…。 | +| Group Member Mapping Attribute | 包å«ç»„æˆå‘˜çš„组属性的å称。 | +| Search Attribute | 在 UI 中将组添加到集群或项目时，用于构造æœç´¢ç­›é€‰å™¨çš„属性。请å‚è§ç”¨æˆ· Schema çš„ `Search Attribute`。 | +| Group DN Attribute | 组属性的å称，其格å¼ä¸Žç”¨æˆ·çš„组æˆå‘˜å±žæ€§ä¸­çš„值匹é…。å‚è§ `User Member Attribute`。 | +| Nested Group Membership | 此设置定义 Rancher 是å¦åº”解æžåµŒå¥—组æˆå‘˜èº«ä»½ã€‚仅当你的组织使用这些嵌套æˆå‘˜èº«ä»½æ—¶æ‰ä½¿ç”¨ï¼ˆå³ä½ æœ‰åŒ…å«å…¶ä»–组作为æˆå‘˜çš„组）。如果你使用 Shibboleth，此选项会被ç¦ç”¨ã€‚ | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md new file mode 100644 index 00000000000..69155120d28 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md @@ -0,0 +1,28 @@ +--- +title: Shibboleth å’Œ OpenLDAP 的组æƒé™ +--- + +由于 Shibboleth 是 SAML æ供者，因此它ä¸æ”¯æŒæœç´¢ç”¨æˆ·ç»„的功能。虽然 Shibboleth 集æˆå¯ä»¥éªŒè¯ç”¨æˆ·å‡­è¯ï¼Œä½†æ˜¯å¦‚果没有其他é…置，Shibboleth ä¸èƒ½åœ¨ Rancher 中给用户组分é…æƒé™ã€‚ + +ä½ å¯ä»¥é€šè¿‡é…ç½® OpenLDAP 标识æ供者æ¥è§£å†³è¿™ä¸ªé—®é¢˜ã€‚如果让 Shibboleth 使用 OpenLDAP åŽç«¯ï¼Œä½ å°†èƒ½å¤Ÿåœ¨ Rancher 中æœç´¢ç»„，并从 Rancher UI 将集群ã€é¡¹ç›®æˆ–命å空间等资æºåˆ†é…给用户组。 + +### åè¯è§£é‡Š + +- **Shibboleth**：用于计算机网络和互è”网的å•ç‚¹ç™»å½•ç³»ç»Ÿã€‚它å…许用户仅使用一ç§èº«ä»½ç™»å½•åˆ°å„ç§ç³»ç»Ÿã€‚它验è¯ç”¨æˆ·å‡­è¯ï¼Œä½†ä¸å•ç‹¬å¤„ç†ç»„æˆå‘˜èº«ä»½ã€‚ +- **SAML**：安全声明标记语言（Security Assertion Markup Language），用于在身份æ供程åºå’ŒæœåŠ¡æ供商之间交æ¢èº«ä»½éªŒè¯å’ŒæŽˆæƒæ•°æ®çš„开放标准。 +- **OpenLDAP**：轻型目录访问å议（LDAP）的å…费开æºå®žçŽ°ã€‚它用于管ç†ç»„织的计算机和用户。OpenLDAP 对 Rancher 用户很有用，因为它支æŒç»„。åªè¦ç»„已存在于身份æ供程åºä¸­ï¼Œä½ å°±å¯ä»¥åœ¨ Rancher 中为组分é…æƒé™ï¼Œä»Žè€Œè®©ç»„访问资æºï¼ˆä¾‹å¦‚集群，项目或命å空间）。 +- **IdP 或 IDP**：身份æ供程åºã€‚OpenLDAP 是身份æ供程åºçš„一个例å­ã€‚ + +### å°† OpenLDAP 组æƒé™æ·»åŠ åˆ° Rancher èµ„æº + +下图说明了 OpenLDAP 组的æˆå‘˜å¦‚何访问 Rancher 中该组有æƒè®¿é—®çš„资æºã€‚ + +例如，集群所有者å¯ä»¥å°† OpenLDAP 组添加到集群，从而让组有æƒæŸ¥çœ‹å¤§å¤šæ•°é›†ç¾¤çº§åˆ«çš„资æºå¹¶åˆ›å»ºæ–°é¡¹ç›®ã€‚然åŽï¼ŒOpenLDAP 组æˆå‘˜åœ¨ç™»å½• Rancher åŽå°±å¯ä»¥è®¿é—®é›†ç¾¤ã€‚ + +在这ç§æƒ…况下，OpenLDAP å…许集群所有者在分é…æƒé™æ—¶æœç´¢ç»„。如果没有 OpenLDAP，将ä¸æ”¯æŒæœç´¢ç»„的功能。 + +当 OpenLDAP 组的æˆå‘˜ç™»å½•åˆ° Rancher 时，用户将被é‡å®šå‘到 Shibboleth 并在那里输入用户å和密ç ã€‚ + +Shibboleth 会验è¯ç”¨æˆ·çš„凭è¯ï¼Œå¹¶ä»Ž OpenLDAP 检索用户属性，其中包括用户所在的组信æ¯ã€‚ç„¶åŽ Shibboleth å°†å‘ Rancher å‘é€ä¸€ä¸ªåŒ…å«ç”¨æˆ·å±žæ€§çš„ SAML 断言。Rancher 会使用组数æ®ï¼Œä»¥ä¾¿ç”¨æˆ·å¯ä»¥è®¿é—®ä»–所在的组有æƒè®¿é—®çš„所有资æºã€‚ + +![Adding OpenLDAP Group Permissions to Rancher Resources](/img/shibboleth-with-openldap-groups.svg) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md new file mode 100644 index 00000000000..1136fdabc91 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md @@ -0,0 +1,78 @@ +--- +title: Pod 安全策略 +--- + +:::caution +Pod 安全策略仅在 Kubernetes v1.24 之å‰å¯ç”¨ã€‚[Pod 安全标准](pod-security-standards.md) 是内置的替代方案。 +::: + +[Pod 安全策略（PSP）](https://kubernetes.io/docs/concepts/security/pod-security-policy/)是用æ¥æŽ§åˆ¶å®‰å…¨æ•æ„Ÿç›¸å…³ Pod 规范（例如 root 特æƒï¼‰çš„对象。 + +如果æŸä¸ª Pod ä¸æ»¡è¶³ PSP 指定的æ¡ä»¶ï¼ŒKubernetes å°†ä¸å…许它å¯åŠ¨ï¼ŒRancher ä¸­å°†æ˜¾ç¤ºé”™è¯¯æ¶ˆæ¯ `Pod is forbidden: unable to validate...`。 + + +## PSP å·¥ä½œåŽŸç† + +ä½ å¯ä»¥åœ¨é›†ç¾¤æˆ–é¡¹ç›®çº§åˆ«åˆ†é… PSP。 + +PSP 通过继承的方å¼å·¥ä½œï¼š + +- 默认情况下，分é…给集群的 PSP 由其项目以åŠæ·»åŠ åˆ°è¿™äº›é¡¹ç›®çš„任何命å空间继承。 +- **例外**：无论 PSP 是分é…给集群还是项目，未分é…给项目的命å空间ä¸ä¼šç»§æ‰¿ PSP。因为这些命å空间没有 PSP，所以这些命å空间的工作负载 deployment 将失败，这是 Kubernetes 的默认行为。 +- ä½ å¯ä»¥é€šè¿‡å°†ä¸åŒçš„ PSP 直接分é…给项目æ¥è¦†ç›–默认 PSP。 + +åœ¨åˆ†é… PSP 之å‰å·²ç»åœ¨é›†ç¾¤æˆ–项目中è¿è¡Œçš„ä»»ä½•å·¥ä½œè´Ÿè½½å¦‚æžœç¬¦åˆ PSP，则ä¸ä¼šè¢«æ£€æŸ¥ã€‚你需è¦å…‹éš†æˆ–å‡çº§å·¥ä½œè´Ÿè½½ä»¥æŸ¥çœ‹å®ƒä»¬æ˜¯å¦é€šè¿‡ PSP。 + +在 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)中阅读有关 Pod 安全策略的更多信æ¯ã€‚ + +## 默认 PSP + +Rancher 内置了三个默认 Pod 安全策略 (PSP)，分别是 `restricted-noroot`（å—é™ noroot），`restricted`（å—é™ï¼‰å’Œ `unrestricted`（ä¸å—é™ï¼‰ç­–略。 + +### å—é™-NoRoot + +此策略基于 Kubernetes [示例å—é™ç­–ç•¥](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/policy/restricted-psp.yaml)。它æžå¤§åœ°é™åˆ¶äº†å¯ä»¥å°†å“ªäº›ç±»åž‹çš„ Pod 部署到集群或项目中。这项策略： + +- 阻止 Pod 以特æƒç”¨æˆ·èº«ä»½è¿è¡Œï¼Œå¹¶é˜²æ­¢ç‰¹æƒå‡çº§ã€‚ +- 验è¯æœåŠ¡å™¨æ‰€éœ€çš„安全机制是å¦åˆ°ä½ï¼Œä¾‹å¦‚é™åˆ¶å“ªäº›å·åªèƒ½æŒ‚载到核心å·ç±»åž‹ï¼Œå¹¶é˜²æ­¢æ·»åŠ  root 补充组。 + +### å—é™ + +该策略是宽æ¾ç‰ˆçš„ `restricted-noroot` 策略，除了å…许以特æƒç”¨æˆ·èº«ä»½è¿è¡Œå®¹å™¨å¤–，几乎所有é™åˆ¶éƒ½åˆ°ä½ã€‚ + +### ä¸å—é™ + +该策略等效于在ç¦ç”¨ PSP 控制器的情况下è¿è¡Œ Kubernetes。对于å¯ä»¥å°†å“ªäº› Pod 部署到集群或项目中，它没有任何é™åˆ¶ã€‚ + +:::note é‡è¦æ示： + +ç¦ç”¨ PSP 时，默认 PSP **ä¸ä¼š**自动从集群中删除。如果ä¸å†éœ€è¦å®ƒä»¬ï¼Œä½ å¿…须手动删除它们。 + +::: + +## 创建 PSP + +使用 Rancher，你å¯ä»¥ä½¿ç”¨æˆ‘们的 GUI 创建 Pod 安全策略，而ä¸æ˜¯åˆ›å»º YAML 文件。 + +### è¦æ±‚ + +Rancher åªèƒ½ä¸º[使用 RKE å¯åŠ¨çš„集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)åˆ†é… PSP。 + +你必须先在集群级别å¯ç”¨ PSP，然åŽæ‰èƒ½å°†å®ƒä»¬åˆ†é…给项目。这å¯ä»¥é€šè¿‡[编辑集群](../../../pages-for-subheaders/cluster-configuration.md)æ¥é…置。 + +最好的åšæ³•æ˜¯åœ¨é›†ç¾¤çº§åˆ«è®¾ç½® PSP。 + +我们建议在集群和项目创建期间添加 PSP，而ä¸æ˜¯å°†å…¶æ·»åŠ åˆ°çŽ°æœ‰çš„项目或集群中。 + +### 在 Rancher UI 中创建 PSP + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Pod 安全策略**。 +1. å•å‡»**添加策略**。 +1. 为策略命å。 +1. 填写表格的æ¯ä¸ªéƒ¨åˆ†ã€‚请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)，了解æ¯ä¸ªç­–略的作用。 +1. å•å‡»**创建**。 + +## é…ç½® + +关于 PSP çš„ Kubernetes 文档，请å‚阅[这里](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md new file mode 100644 index 00000000000..59bd02daf07 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md @@ -0,0 +1,192 @@ +--- +title: 定制å“牌 +--- + +## 先决æ¡ä»¶ + +你至少需è¦æ‹¥æœ‰é›†ç¾¤æˆå‘˜æƒé™ã€‚ + +## å“牌é…ç½® + +è¦é…ç½®å“牌设置： + +1. å•å‡»**☰ > 全局设置**。 +2. å•å‡»**å…¬å¸å“牌**。 + +### 自有å“牌公å¸å称 + +此选项将大多数出现的 “Rancher†替æ¢ä¸ºä½ æ供的值。å称中需è¦åŒ…å« Rancher 的文件，例如 `rancher-compose.yml`，ä¸ä¼šè¢«æ›´æ”¹ã€‚ + +### 支æŒé“¾æŽ¥ + +使用 URL 地å€å‘é€æ–°çš„`æ交问题`报告，而ä¸æ˜¯è®©ç”¨æˆ·å‘é€åˆ° GitHub Issue 页é¢ã€‚å¯é€‰æ‹©æ˜¾ç¤º Rancher 社区支æŒé“¾æŽ¥ã€‚ + +### Logo + +上传深色和浅色的 Logo æ¥æ›¿æ¢é¡¶å±‚导航标题中的 Rancher logo。 + +### 主颜色 + +使用自定义颜色替æ¢æ•´ä¸ª UI 中使用的主颜色。 + +### 固定横幅 + +在页眉ã€é¡µè„šæˆ–两者中显示自定义固定横幅。 + +è¦é…置横幅： + +1. å•å‡»**☰ > 全局设置**。 +2. 点击**横幅**。 + +## 自定义导航链接 + +在本节中，你将了解如何é…ç½®**集群仪表æ¿**左侧导航æ ä¸­çš„链接。è¦è®¿é—®é›†ç¾¤ä»ªè¡¨æ¿ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°ä½ æƒ³è¦è‡ªå®šä¹‰å¯¼èˆªé“¾æŽ¥çš„集群，然åŽå•å‡» **Explore**。 + +添加链接å¯ä»¥è®©ä½ å¿«é€Ÿè®¿é—®å®‰è£…在集群上的æœåŠ¡ã€‚例如，你å¯ä»¥ä¸ºå®‰è£…了 Istio çš„é›†ç¾¤æ·»åŠ æŒ‡å‘ Kiali UI 的链接，或者为安装了 Rancher Monitoring çš„é›†ç¾¤æ·»åŠ æŒ‡å‘ Grafana UI 的链接。 + +自定义链接ä¸ä¼šå½±å“å¯ä»¥è®¿é—®å„项æœåŠ¡çš„用户。 + +å¯ä»¥åœ¨é¡¶å±‚创建链接，并且å¯ä»¥å°†å¤šä¸ªé“¾æŽ¥ç»„åˆåœ¨ä¸€èµ·ã€‚ + +### 添加自定义导航链接 + +:::note 先决æ¡ä»¶ï¼š + +你至少需è¦æ‹¥æœ‰é›†ç¾¤æˆå‘˜æˆ–项目æˆå‘˜æƒé™ã€‚ + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰å¯¼èˆªé“¾æŽ¥çš„集群，然åŽå•å‡» **Explore**。 +2. 在顶部导航èœå•ä¸­ï¼Œå•å‡» **ðŸ”（资æºæœç´¢ï¼‰**。 +3. 输入 **Nav** 并点击 **Nav Links**。 +4. å•å‡»**使用 YAML 文件创建**。 +5. 创建导航链接的最简å•æ–¹æ³•æ˜¯æ·»åŠ ä»¥ä¸‹å­—段： + + name: linkname + toURL: https://example.com + + 有关设置链接（包括å¯é€‰å­—段）的更多详细信æ¯ï¼Œè¯·å‚阅[链接é…ç½®](#链接é…ç½®)。 +6. å•å‡»**创建**。 + +## 链接é…ç½® + +### `name` + +链接的显示å称。必填项。 + +### `group` + +å•å‡»æ—¶å±•å¼€çš„一组链接的å称。 + +å¯é€‰é¡¹ã€‚如果未æ供，则显示为独立链接。 + +组与独立链接分开显示，如下所示： + +![Screenshot of group and standalone link](/img/grouped-vs-standalone-links.png) + +### `iconSrc` + +Base64 æ ¼å¼çš„ Logo 图标æºã€‚ + +以下是 Base64 æ ¼å¼çš„ Grafana Logo 示例： + +``` +data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAgAElEQVR4Aey9d5xkZZn3zb/P+3mffZ9nDcDAzHTuqs49PZEhCBhBJc10nO7pHKbD9PR07p5AWlEEZM2ioph3dXGNqLgCxhVBRIYRmNy5cjqnTlWdqu/7ue5zTk+DiNIsCn4suOacrjrxvq/fle/7PofX+ycDZNT/pIAUaUUmaRBKp8FMQ1L9qL6S4+VYUikIeuBrn+LppiuJNG/BvHYd7HbhbXLBZ/dB8AkwFokCHsAH6Kb8IxTHJIyBHwgDMTBT6h7yWAmb1L3sxyEjfxkYxDDQMa0nWV0vyE2slwZ5JtUO8v5JUhgk0EkRw5TnUg9sgJm03lsezkyTIU0C2VrNJU2WsdtTPVQyAmYU4mGIhDESaULAcSG5RjwJEQ8YsxgEOZoOcSxjvXsyKM8nL24QJ4UmF5TnlL7IWLure/G/3lnn/PVu9SrdaQVApO3/CCAZAYjNSLIVBrD/VMwSnsf4/B38ofWthFu3krhhPZmaLJZuyEY7vJPIV++AwEnImJwJ66qTFcMLSswkGWKkiKqtMIICwB890zL/2lwoHGLYnCIMtMqP3Md5N7mMDRDTBkhKAURAqNBs3TOdspjWERgrmkYuJbQMEPUeKdAEIBrIuSIKEiZ+B0ShADz7W/Tjv8TMLOIjybxcJwNJOUgAkjHQSFoAkedMWn2g7rXKV/9rnfZ3DRCRjgoN0ik2A0nDLgMkbYD3aU7dPcRT7ZfhadmEVldA/IZ1RN6TRahxO8f37CTyxU/B4pySvOlYHNM0sZhQRKgwTRrRWALOZc0lWksoY30n0lPkuMBCPYCcKn/Ic632Iy/ivNvy+4k2EOAamDbJ/rIKFYEhx74AWPIWcpRhv8dyu5mQTEEkmUYTjSsaMBWDiA9mjsF/foWTH76Z33zqEL6jD5IgRkTeR64valfOQQSL6My0Ap/c32qvlNJgq339v8Z5fzcAkcayBKrFkNIBCiDOD/Lj8jG2iZWOwvGHefxQNY+3beHk7grCDS7067JhRwn6dZX4d72L3zVei/6Vz0HYB/EQJINKawRTQXVP+UfYQEjdRphITBlTNIrFqBopIsqcsZnaQYww0iv5yA1XkuJ+eRJREStJvrMFhnO8A5S09ewCDoMkCQUw67KxOMQMCBkpkok4JIIQnYGf3k/s7mlO7N3Fw7VX8J2Ot3Pi/rvBXMLMJKxbx+UR5L4CEJ0IKYUbeV0xAUXDZVSrvZIGeHXPff0DRDGE9PRZPhGec8jhBWcr30uXCWEG4Xf/wW+H38ajXVUcaS7GX+dGvyYX6jeQvL6MZO1lzFx7Mc81XkPyM3eC/xlIz5LJzBIz/bbUtUyKZaksUtPUIS06wzK/LGluM6jwrVg9wkCvECDOe51lE2kL5w2drdU+Ths5bSBbMacsVMtGtKDFug5+5Q00Iw2JFOhhmD0C3/goS6M1HKvfiqfxMo7t3MLv2i7F97nDoJ+BpG45IXLysgYRgIhvJPeRu4QVibZ7LX/+rgDiNLTT58IADjM4rPI8HyXpgYc+yXODF3G0ZyPPtZSxUFeM/9p8MrUVJK4rIX5NMfEdm1jauZ1j7Vfj/exNcPoRSB2HxIICiHI+Hb4U00mYSWkP4RAhHTKiUexggfCEPJDiUOepX/5W3tN5R9m3PpZZJ6bdWbJ+kWPkto51JyaVxbBnpYtcT35XwFDXt8Ee8cJ//wj9X6c40fF2TtZU4qspJV5bidawCU/HxWgfHoTwccuhl4srE0saRnwQwwaIZQKa+BH6B0CcfnvVttIBK8jmFId5zjKEmA+WhJTIkeoYYxY+t5/FtmJ8zeUEWzdypqmM07VFhGpK0W9wYVyXh15dQLxnG/NdF/NE00V4PzgAv/0OLB5RbyWKQD2CML5wV1KMbIlmCSgkapQEkcLCNPJ72mJAsdXl+Vb7cRh+mcnlQvKl0IomUfs2mOT28rwCaiErgiW+hXWOaBSnzRSw4/Mw/wR87zN4xht55vqNzNxQQXj3VoyWzRjX5ZPcWUigrozozU0QeMbSNAnIyA0EcaQRE1N8EPWn0hoSDxSSRnntfl73GkTMAsvXsDnCYZAXMERc2dei2i0HVnWMdpro4etYuv58orUujLatLLZsZKapkqXaYqINRZi7XWQ63ASacwh2lhPtvZwjdVs4M94ETz4M8ajFjI5TLlsJLavwsu0GCA84JpX4uEAAVHBYGHa1H3lVuZaQxXgvAIh86QDFDqu+ECDSIstS3AGWnCdmUnwRjv4Y/XMHON51OSerSwjv2kCqdRta81ZiDZWwqwjq8onWFKIfrIPQs5CIKl/ekJvZDyagFJJbWKFuiQFLePwfJtZq+//PnieNLc64lUuwTYoXMITwZowMgbSu3EIjE8NMiKMdhmd/zlzrRjJ12UQb3IQaKojUbiRUW8VSQynzjQV4mtfjbz6fSNN5hBvXsrjbzXzTZjz1V/Bs0/Xw0A9g7qRy3E3DRzARUJpKni0ZSljpEUkcCEBsQR3BYIEIC2mxw+WBV/dx3v950TP5UshpBxskqURG+cvCjtImwqyyDYZ9pDPiMKfR4hHQY5aJdPIIPPg1jrS/nZndW/E0lRJodBHY5SbYUEq4biOx2goi16+D1iLCO/PwDL0HfvdD5X9JFNwXP+vjyL2UMJDnUs8kRpzkjv4BkNX1/l9wlmiOhHL4RIbaDrA0vs0UwifSMVEkuSWJsyTxRACMIKSi8Nj3WWyphLr16PWFaPVlGDs3ou2swldXpgCy0LoWT+t5RJreqEASaMpRDGLs2E6w+u2c7mkgcM/t8IdHID2PSZAQUaJmSrkAypgXXrClaTIj5kZcRXXiKlH4ygAibXA2Yme3wUqAJDMWWDJgGEmiWgzDFL1hCRcjHkWP+kgaPkgHQUyqIz8l+fHDzLa/i0DdVrTaUmL1LsINBTZIignXVRCpK8W3cx3Jdjehnfl4970bHnsA0rpi/QWxLqUf7SiZ2pd/BBPio0kQQyVO/4LO/hsd8ro2sYQxRPYJSJSZJYwhHSBkWoJUABJRqTLRM0m08KKlPYwgiR99kZOdlRi71pOuKSS90wJIrLqK2M4yZTaEGy0N4ml9M96W89Hqs0lVu0nt2Ii+YztPv8XF8ZZ3EPrkAfj9dyF50gaJhscwEP5U0twGrZlKkEhKwDNkZ7gV27yC7ndUxYtfwkymMZMp5L7KF0pJNj2OmYiR0MMkRWsIoyY8MPsYia/fyZmxGzjdsInAjjKM6nIS1aUKJOH6UgK7LAo2FBPc5WKh9kJiYoJWF7I08G5SP7sfMrryLuZsYCjZJYlFp39EWKhIn4TBVy8gXvyN/2e/fd0DREoY4sqvsEo8lHQSCSV8Z5c8RDJJxAeRUKvmm4FEGDQ/S1+6k2NdG4g1ZWPWFJLZWUZi5wbiOypJXF9GfIcbrT6HYNNaPC0X4G2+AF0BxEVqZ7k6NvRuN96dGzjR+VZOva+LyMNfgOizYrwQJkIkY1qmhcMcSZGccTJmCF0XJ/UVAESuuRytOssY8rWjSGVf12MYsbDKyYh0z4SXyIQWIB60hIU2D7/6FjN37uVo56XMNpQSqXdj1LpIVBdj1JSi1WwgUrtBaQ4xscTc8u7O40z9hYTa3UR2uvD2vpPIA/dBJooXmLOfQwkIx7SS13X6RwIY/wDI2Y77n94TDWKVMCQtP0Q6QRpfJJRsJXyfgWhaoCFaJk5SACKM4ZnlxIemOd2+mWhTnvJB4nUlmDsqMK8vVwDRdxYr0yLSkEekIYfwrhyCjXkqmajVFlvMc0M++k43vh2lnKzfxqmpRrRvfRwWHlPVW7G0D40YyaSBGRVJLV66xSWmZPIV96yyZRQSnIy9JRDkK7m6NIES1CIYMgkykv0W3yvug+giaB7LnDrxK1LfvYeT040cqdvEUm0JdFRhtpbhr84mXF9MuL4crWYjevVG9JoKYhIKbypgsTmH0w3rCLcWkbjBTbD9SrxfuxvSfnyYLDmvpQIXKzSpoFceVJlX/9AgTjP9j2/FsYyRVKQcdWlr4QrHIzQhYUIsbSq7Py2Z8/CC0h4cO8rRw3uZb9mKvstFZJdoixISO8tJ3lCuTKxItUjSYiVNtTqXAotyVBtd6ju91gWNLpI7skheV4BevYHFhks4PnA9S/feQup3D4B+0o7WxIjrYUxdEogWs4j584o+DkCk3kku9YJIlTjikZROUsIGKfEvvJDyg+EB3wkSj3yD2dt7OTnwdk7UbySwq4pE0wYiNYUEavOJNJfiF5OqoVw55HpNOUICEKVBmgo4U7+OsBR2Xu8itvsyFu65GVIegsRUpkN1hiRNRVvY7+3UellZ9FfYBq+oAf/8ya9rE0sAItlZAclLAUTLpIkmJfQoUZoASNLrsV/x9Gg3vqaLSNcWkagtQq8rUY6nxPT9tWXKUQ/WVygJKlJUnFIHMFKSEqvPg+4K0rU5pG7IgdpSkjdUMn/NBmbarmLu1n7MR/4NPE8qkMTQCKSlhtcSnlIoqyTpn++nFz9CACKaUsiWyqI9RD5ICkLyDhF0YmYYQ1+CmJhVi7BwlNPf+zK/PtBGtPsSwlefy+w7z1P+RazrYhYbN3K6upTFunIFDvE35H0jDQVKMKg2qCtV2mWxJksFOHivi2TdxczdNQqJWaKZABKtIxO2KhZsh1we09JuVtWxFaJ/8dd7LXz7ugaINLb4FpZ/YYd5RZ07XJK0SoESaZOY5icj9dcSvZLy7Ie/y+mBBsINW0nXlJKsKVEAEfva11CCTyI0dWUogNRV2FEbkZ6lSoKKjR6pd5FoLMLcVUSqzkXyhgL09+ShS/a9fjvB7qs5PlJP8vufhugzKjEmxp4wsDyiVIorS8PGibP/oowhP4oIdrbOvi2V5VfHtLJyDhLhk5yPbmmN2Ax4j5L61bc58pGDPDTcyFNd78SoLoTq9dBUjL67grn6ck5Ul3GmpgJPQxXB+nLCDW4VwXLMSzG7YrVWW/h2ZqHX5cF1hcTrt+G5bS/EThNP+YjKmwowklEwJWJllZo4Cl4EnHqdF33h18aXr2uAqCa0TQtpaKvCVuobjGVTS3zAuCHJvCCYAZICEM2DcecowV0X2aZDKbo4ohLOFC2xTGJarKA66xjruGIideUqbyI5AUvTyPluxBwzatzqev66TXj73g2fvRWOPQpxj2KaU1qIZ2NhO5ttMbcw+LJ5btsjmYyE44QEHBKIcLZytDU2Q5KOkmqRrELELsBPZOKQDkPKC8YMnPkl+r/fydHhnfy+YRvzjdtJtl+GUVNMosaFVudW7y2ACDRUqPdR4Kh3tKZoEREK8t5WG4mppTfkE63OVu8c7ryMk/vq4fhvVeInYJorhJVIAyvULhUN8p6veXQAr2+ACCrEvLATcAlV8xNUA4WUHo+DRDWTCRnLEADTgxxDeAbz5naSNWXLwBCmf7kkibJw7VYCddvwN2zE21iKv9GlnHkV7arJg8ZKaLmUeP8NxO+cgF9+T4VUpZJ1MR1TCbuzDvXZ6FPaCYvKOzpaQrbCWYoEKOJbpfASZyEVYTERJGRKUFukdRik1mzpaXjk6yzePcLxve9ltmkbkboK0tUlZKrd6LVnGd5h/L90K8IgUZdPpCbb8km6L2W2fycc+ZXKhQTkOaV/RGWIv5XRVd9IxbAChy3cXhu64sWf4u8OIHGkFF23OiYOetQkrQoHxbySMYE6zD5FdLzOju8XK5NJpOHLJqVdJPy5Udnq3sZiPLsL8O7OIdyYrULEyVo3qcbNeOsv4Wjj2zl2yyCZn/4nhE6qkXppCSRImZZpkRPkUmkLqWdaaTeJ8xJNk4mYZMIp0mHNMhkTS6rM3Kpt8oG5CN4j8Mwj+L54JzOHezjW/HbmqrcSq5YQtovUNdkY165Hr7W1ggpGSEDiLycBiFGbpwAigsHTtZ2Z/hvg1z9WWjKsRk39AyAvDr2/xrciXYWBVmiQuCqjtgFiQDRkmSLL5obk1Z98mKW+d2PsLEHCtS8bGDaYRONIIk2rLVcOqwzTXWjJY6ElB//ubKVJQjuyiVaLlN7CfMOlHG27iuMHOkh842Nw9BcQ84IeAc2maBhiMuYiCJEgEm3j2afh6FNw5En4/eOkn3iUzOO/hsd+Bk88BI//CJ78L3juF3Div+G3P8T4z3vwfewQj7VdzTONV6roWrLxIlXGn7khH65bR2ZHNhKJezmgWHmsACReY2kQ365Cljq3MrPnGvjJf6j8SlTU4PM0iDUGRHyjf2iQvyZARHur6Ig4wFGVEFQdk4BYWPS8HCCOokhXP+Eff4XZjiuIVxe9IoCIeZLaKTa85bB7mwqYb81jti1HgUQ0iUR+gtflEb2uGKPhIoJNlzHbdDmefTtI3L4HfnUf/Pwe+OHd8B+3wudGMT7cQ/C23XhursV7Yw3ew9V4DlzP4tS1LIy/h7nRq5nd/w7m9l1FaLgeT88OllquI9xRjdm/G/qbibVdz2LNFczVXsJ8/RYWaivw1rgxGlywuwh25UN1tkoGrmT6l7MvANGr84hU5+JpzGexfSMz3e+Cb34GdC9xGaIrALESMsrekuG/UtHwD4D8DQAiDS9DPsVZd0I6CTFLRJJJNEUAEjnJwlc+yJmWi/4HAOLG3OkiVS0h3wKVPJtvKWCmLQ/Zirnlr8kjurOA5A1ukteXkthZgV5bRbhhM77GSuYb3cwJsHa7WGh2sdTsxtPqwt/qVhRocRNsLiDYlE+gMQ9/Qw6++vV469bhq87FaN5GtHoLwasriF1VBddeDDVXQt1boe5StOrNxHZVqfox784cwrXryezOh0YpmblQDTF+OaBYeawARNuZvwyQhfYK5trfCl+4EyLzmKrU3/ZDRE5lZEShjLBcAZC/Bp+8gnu8/n0Q24GV9heAWMM4ZRyG3TGSaLYdRFI+mP8tpz48ohKEEmmSTrZCti9/K5EfAUequkAlEcUOX2h2M9viZr65GE9TMUvX55KQ7HRtKZnrCkm8NwdzRyHpWpcybxJdm9G6thDr3EqkfSPR1irCzZWEdlcQ3F2Kt86Ft64An02B2gKVxJNEnqIGF776QjUSMlBfonI2oZoygjfIwK9CNfhL8jvplgqSzUWEG7Lw1a5hqfZNeOrWqPCtFcaVUO7LI2m72I48wjtzWGrMYb6tlPnWizE/cRiC4mNJlMQ2g6WfVFhXQs82QETtv8Y/r2+ASOPakRALICKh7LCW9YX6XUq9JcRI0gfHH+H0+9rxNm9RodhXChAJkUrNkoAssEtAUcrC7nI8jRX4d1UQra9QtUzmDS7YUQA78mBnNunq9aTq85h/73pmr8ln/rpCFm9w4a0uIVgrSbhyIg1lBGqKCFbLAC6poC2y/SUr6y8l+r7GXPxt+QQ6ixR5W4tZairEU5uPrzqPeEOxer5kvYtUc5Eqq5mvXcOJujczs+tCAo0FLxsYDpCk7aLXWwBZbMpisa2IxeatpP51BHzPWZM7SD8ICRgUIOz+Wf77tY2Q1z9A7Ia2+kEST45TbneMjHKNZ0gbGsTm4dff4Ll9V+HdVUFkR54aKBWuc7EakkFWMaGaIsK1JWpUnb+uEl99FYHaKjWuRKJGcSlf2VGiRt4ZNdnEa9cRq19DtH6dun+0RiqHy4jWlqhtrKYEi4rUtbWaIixyEa92Ea8pVNtoXT6epizmWtcx15JlUy4Lu3PxNeYTqi8kVluIVmNRTLROg5yTy1xLLvPNuYhzHWhwrYqkzVKitXbmEel2M9ucQ6BlI6cHr4VnfmbNpyUdY/uIFhRWRFVe29hQT/d3ARDBiAMQ9VYKNHZ9kgKIOIoJ0Bbgh59kpu9ygo1lRHfmvyKAWKCywBGqKyNUW6lAEarZpPyC2M5NaIoq0apLFLOG6yVnsA7f7jX4G9cpYMZqyrBIgCH7Jep4BYrqEhscAhKXTWcB4m/MwrN7Hb6mdch+qEEKDHMRMGi1+QpMAhABijyvgMG3y4WnsVCRf5XgkOvI9TKNlerawW4XpxsvVLVbZ/a8Ax7/nnLUpWOk9spRIhZabCH2D4C8+i3gaGoLILYaV2aX3QkSaZTaCylBic3BFw7i6d5MrEkYNl+Vi0id0epIRtcVWyPspCSjbgOxmo3Eajaj79yKXr3ZcpKlTFyNpXCrEvHFlmzm29YiWzlfEncWSb2X7MvWrv1a3p7NaIs5p0LT9S60+jwStdnKbEPqwa4VyoNrC1QBYXqHW5mSco5VmWuFtSUCJyFu+V7GdayG5Fx2b1ZmnL87n5O7zlNaa777Cvjp1yC2ZJm4diBL+mi5ImA5+/nq88grucPrXoM42kPgIPtn292ydSURJ9l09aPvGJHbWwm3lxFvLlYRHLHjpZJ3NRRqLCLYWKK0kfgL4m9odVUqShWv2YReu4loXZUayhtoKsHb7GapJZ+FtlwW2rNYaslV58t50Xo53yJ1rYYSIg0lRO1tZFcRFp191uguqX/KJ1mXT6Y6H3bkg+Q4ri+EGwphhxuzxi7ErC9S1crisCdqyzCry1TwQMbdhxrdqyI5l8aNGLtK8HbmcqrxXPRdBYR6r4AHP2dVDcuQAztXKH1k9Y/s2QLslXDvX+Hcvz+ALKsSVd+gZsuURLoCyPHH8Ey8B6PFhdkiYz0KiO0qItq4Ogo3FRFqKlYUbSwmtqsUvaGUeH05upSIN5QTbiwluLsYX0sRvhYXntYCfK15ioLNecQaXarcXkruHbK+KyDWWIDeKMesoKZcYjbJOJbw7gKCzS517aVWF/PtFkmoWL6X90vWF6mK5UxNMUIpSXDWW88qzx3aXbQqiu0qJlVTqTSIpyub2eY3k2krIrnvbfDtj1rjTaQWzh4DvwwQmfFFjc957YPkdQ0QR3sIJqSplQax2l6VlMiUm6rOT6KKaQj88gGWBq+E1nwyLUVEanPQmtzEmopXRZHdxUR2uxXJdYTijdY21uQm3Owm0FqIt80if2uh+jvcXIjWlE+8KR+tSfbPUmx3IbHd+RYJgGyKNOcRacl5PjXnWfdoKWapvZjZzmJOdbs52eNmpsvNggyFbZH3c5PY5casd5NqKCbRUEx8V4kyM+Udws2rIzFT9etLlgGy0HYedBeT2Xcl+hduBn1WFVeenWLIcRb/AZC/gvKzAOGAYxkgdhRRaq5UqbeNIhFaz/3nF/ANXKoAQoubUHWWYh6HyVezFYYWZtd259lkMXGoNQd/Ww6ejhyWOmWbR6A1TzF+QrLYdYVkJMrU5CYkz/I8soAk4LIoX4Vy/W15rKRQSz7xxmJl4lgMbwFWQCFg9LQXstRhkewLUOU3Aa7zrrK/WlJ+3LVFGA1leLuzWGw/HzpdJHsu4rRUCcwfeXGAqBGVMsJTek0lSP4q/LKam7yGNYg03AtImH0l2X9KM4udq5xASzjZVq8M2LF/MBIcue9u/AOXQ0chtOYS3HkuWnMBEdEmq6Boi4toSwFaS766jlxL9iOt+QTb8vF15CKmh5C3M5tgezZ6cy5mYz7UCxWo84JtBTyfXATank++dhe+9gLObgsIt7owmmQ8SolNRSTFZNttPVegvQBvRwGLXfksdMlWzrfuFWktUM8uzy/7kRbXi9AL26WESMsKai7FU+0i1FaqAOJrPRfackk2l3F85Fr43Q9UTZb0i5hZ0keqS6WPVg7DVUEV6UyJBzsduAI49kzyZ7veCuf/NQD2NweIemm7ASwbyW5FaShFYh8JOWMhbFTY2DH0FKFYnEBaJmawf9OTEI1DPIVKEkpnhDw8dfdBwmPXIqFW/7X/C63h/6J3FBDrKH4F5LbPLUVrq0Br20CsfQORjgrCnaUEu9wEevII9uQQ6c5C78wi1ZYDLXkgJlPTeQR3WxRqXkOweR3B3VmEmnMJiXllP1u0qwShSFcZEdnvLkXvKiXVWUK6vRizrYhUq4u4Ddjobjk/l+DuHIIt2YRa8wi35RNtK0BvdxHvcBPvLCTTX0Zqj7UfFaEhxyvKJbS7AL2t1KZy9LYK9NYNyxTs3MCZvnJO9RcR6F6P2boGGtbArlx8HZt5Yu+74blHIHpazSujy8yWUuErGJCpkJYk9C6DWVIQS9jDcmWAlcxbJgdYCRQVJl4xQbhUTJiq5k7mPZaLvXqfvz1AnBj5WfFgN4wARIAhWmAFQBylImLJljwpE8JpmXTZBoiIKl2mNMkQlepYmazg+GMcOdSJv/cdBGtz0Rr/mVjLG9A684h0Fq+Kwp3FBLtKCXQJEEoJd5YT6Sgn1lFqkwBPAOQm1ulC63Ap5jTaC0i0uZBttD2HyDIJE+cqRhZmDreKlrA0RailUDndgd2FquZLEnzeXXnonUXqurF2l2J+0QYWMCxm1zrc6nfZxtoLcY6LthUSEQ3TkoO3JQtfS5YCUrglj1hrAUabm1R7MYlWtyKjrRi9vRit3Xq3SEcp/p5iTvWXcKbfRbgzF7NlPTQK5aK1lnFqz0Wc/FAXxq+/BimZ4ySKaWhktKQFDBlyvDLEJdEUGXkY91ujEJVQtPIoAgM51NIvUq4SVmR98/cMEOHpleCQ/WU9LA6FTUqb2ClZOcYGhxxtJFLEjJTS2nYLKmUkQzp1meTTnIOf/RvHB65Fb96OUZ2P2VeAv20Nka4CIp3uVZGAwtu9AU9PBd49bvx7cgj3XECs+3z0rgswOrIsBmupING8WRUW6i1b0No2Kk0T7thAcM8lBHovw997Ef6erfh6qvB1leHvKsLfUUi8v1xRYk8ZiT0VJLorMbo2kOisIt6xAb13I9G+DUR6qwjvqSDUU0mop5xgdwXB7jKWWouQ6Ja3vQRfRzGBrnL1e3jPBoL9Vcz3livy9pap4/WOEhKtxdaUq40F0JxHujWHVJul/cI96/H3rmexfy3evvWEOwrR2oqItZURtSncXkKoo4iFnjKe6d3MiffVE/7OR+HMb0Am7ktrpE2ZsClD1ATNnsPMshIMezoima9LhuxafS7yUMAhW+vj8IYwwtH2mK0AACAASURBVKv3+ZtrkBf6FC8KDtU0KxpiBaDiCQMhmR9agUa2KUvaxJUenwf9d6S+cjNL3ZdD6zaodcPQRubb1qsSiWhXEashMXVC3WUEe0oI7ikk2JtFqG8N0T3nE+9eQ7IzC7O1ALO5HHP3ZhK7t6G1bCfcth1/5za8ndsI9V9OsO8tCij+nm0Eujfh79qAv70Uf0cx3uZ8fM35BHbnK5NHImDK92kW/8ClGFGAtJIsX8XyN8I9xaykUHcRDgX2lLDQVcRiVyn+7lJlthldZUpzpFvdpFsKSLflYrZnk+jMQutZR6j3Qnz9F7K0d41633SzG5rE9NpIoHMjiz0bWNhTgWdPsSU0ess51V7BM3vfhudTY/D7H0JK1qAKq8k2ZhI6c8kUkUzGsrClRF40iZhYQmI9KCjZUUpHOFqqxP7y7xkg8m4OwysOFyDI2zsSwop2yOwXzmErtzK6Wf5W/5j2HAEx0FOaWhIMjsLv72Pp5huItGyAzougqRRz31Zm2wsJdYttvzqA6J2FinnS7VkkOnPRugqXr6V1WXZ9ol2kbw6pVrH93Yjd7unaxkzPW5jv2U5kTxnx7gLiXUWWuSQmTGs5sdZKoi2WTyO+jfgClrlUgN6Vi96VTbxrHVrHhWida5ZJNNdKCrW8iXDrm4m0nUus43x1XLz7QoyetaR6sqC3CPaUwZ5i6Cki2eNSzxPtzifcnUeoJ5/Qntw/okhPrvJj2LUBdm0m0bJBaadZ8UkGyjk9UM7inlLlR9FZgtnmxtNejveDTfDYVyEzq2Zc8WCySEZNESTTM1n9qEqwQeYSUHPAyJgFMaDtyQGFPeRPoRVy89WAyd9egyyDwgGGAwirLFqcMGuCaqfe6vlbOdoBiClDVMW3k/GrMsWPcZzQo5/lzGc6ebarglBjIXRuhLYK4n2bWOouI9xTSqy7eFUU73JBe7aiTHuBkrzJ9kqMjg3EOyuJdxUrHyfWlaUcdHHU/T1uPD3lLPRsxttdgdaejdl+gTJhku35JDrcGJ2l6nytu4pY9xYi3VsI7dlCsG8zgf6N+Pduwje4Ed9QFWe6rNzH6Z4SzuwpZqa3jJneEoRRZ/tKWdi7gYW9FcwPVDLXX6a+l9/P7CllsdtNqrecdFcRiY5Com15hNoLCHQVEuotItxfQrC3yNrfU0S0pwi9u0iBOdlZRLqtlExzJWZLFVpHJYEeuUclswMVLPaVEepxk27NgqYLQbY9JSwObufY+xsJPHQvxI+rhT/DagWqJFEzoeYPVh0qnSrLRygUSKeqjrU6X76WiMzfP0DssJ74Fyu0howXsIBhgUN8CecIZyu2qFAsmSSeThNPmWoOWjVBmRGGM0/AL77KsX9t46n9F3O6w0203QWdpdBdTqijhEhvJdGeYtXx0vkvl2KKWYpJdhSTaSuGllKbykm3laN3lePtKWa+z83sgIv5/jy8vVmEe9aS6DxfASO9ex1mSzbptnzMrnxSfS4Sg0XEhyuIjFUxt7eS0/u3cmb87cwcvJ75W1tZ+mA/3rtGmf/INOZ370V/4F7iP7gP44efJ/ngl0j9+IuY//UV0j/5Mvz062qrP/B5fP/xUc588Q6euecmnvzIAf5wRz8nJt/J3P4tnOyt4GRnKTMdxfh6KtH3biWz/2IFVKOjklS79U7yXplWi8SJj3bnEugtZKm3ksU9W/D2bCXSWYnZng9ta6BnHVrdPyEROnPsYkKjb+NY18WEpnfCN2+HM/9F2jyhpmoVq0F8SS1h8b70tQjHsyCRCejsUYoCDvHa/741iAMQx5yytgIQmQjO+k/A8XyAOOCQBtRUOtCad1eNGpRZA71HSXzn4xw9UM+xfVfyXFsZsaHNJPZWqmiS0Vem6qBifWWEbcn4csEhx4v2CfVsINwtGqOcVHupKrVQ5RYSnu0uxttTynxfKbP9xcz3F+DvzVJOvNnxz5jt5xHtKFQOdaC3iuC+LQRGLsY/fQWLN76D2Vuvgf84AN+9HX5yLzz2HXjuUVg6DiEZHSnj1kPWeHZdVqKVCNCfIRn7LueF/OA/BY/+O/z4bmJfmWLu7g5OHtzByeGrmN/7Nvx9VxDu3I7esZVU20YyrZXQWm4JgdZiUh0u5YcsDmSz2F+EV4IEXRXEO0rItOVC6zqMhjcqX0zrkQBBKaHujST7Loaei1jq3U7gK1Okn/0OJGfUNEWJpEnAgGDaWqBKcGCFciWaac/Q6Mx9JoygzIdXw7iyrvk3NrEEIHYo9wU+h0gOAUYsqas1ti2QpNXU/VpcxzBkjiWZORD8yTAZAUZiFk79HP2rt3D64E4W979DRYciPZuI9FQR2VNBuLeEUL+bUH8h4b5CIr1uIr3Fq6JwbxmhgYsI7NtOaGgL4X0VBAeK8PVk42lfy1LzGrQ9Vpg30laI1llEvLcUY28xWl8eS31uToxdzLGb3sviv7ahf3EafvBJ+M234blfw+yzEPFDLAq6zPUlQ4nFiXVyRLIvs+OtklKylqBMYC1zZy2A/1l4+mFS3/8c/k8eYPHWDuZGriW4753oA5eS7tsCPRXQ6YaOPOjMJT5USnS/1F/lYu5dT7L3QrSutfhac5nfXajMQ61rE2ZHBbQXQ1shtBdidhYjkbRj+7ZjfGUcnvsRhE+QNmJqfi9ZWkfm+1KznCnr24CUzOIijrvARozrV//zGgGIo0EsA8oBh4BCZpaV5Q3ipoFuRLEmfLYYJGnGmUv41YqzSgL9/rtEPjPK7ORVeIYuITl2GWLHG50Wad0bCPWV4B9w49+bq8gCyWoBUkJo72Z8e7fgFZ9gpJLAeAWBURf+gWx8XRdi9rthsBIGt2Ds3YZv8BIWxt6B933VLH1sD8YPP0ryv78Gxx4C/x9AZlrXRDNEIBg5G/wXv1XW+FieVtQywxWLCAOthjIQ0ZJoyYwValeTLAgYF8H3DMz9Gn79NTJfO8zcjdfybO9G5nsr0PZvwhypwtxXQbK3HGOPi8SeCzH6ziW+91wi+9bhHShmsbeKQM+laB1vISPRw5YKCyAdORh7soj05ZIa2Yxn5C2EPjEIR36gZoGUyR7mE7AohpcpARd5P5lXQISpRjIZQktbK7682lOXvgYAskIa2supiea0oCLbNEY6QVQLocl0OCr0JwkliWzEVJ4jkzoGj99P7GNDLAxchlcy2AMlxPaVou8pJdlVCe0bSXdUofWWEtjrYnF/LovD2QT3uoj2Fa2aRFJG9pSgDVcR2l+JZ28xnoECtJEiOLABvS9XZdElJzAz9nY8HxtSfgKnnrKWbwuKiSTTg9raQGL/GV2tgmUtj+CYmxZWBCDin0oCWvalrVb7EQmsEttSki5zc6mtRAXl6iLDpdjwGOiPw6lvk3j4Q8x+dg9P3nw1j49cwrHeKlLid7WXkt5bTniomDP7cjk5uJ6FoXxi+ysxujeT7LiEZNtbSXRcjtazmUi/i+DQm4kOvgm616vk5cz+K1n8zCippx8EM6T632OkCMQzhGPidwhARAoY6EkfYdOrVl5XczKvtgH+gvP+tgBRkQp7PT+pwzkbrX0+QJKamhldqVgjjBlcsObXTcv6GsfxPPYFTt3ejm/gbZh9WzD7iwkN5OIbyEbrc5HsLoH2ckWJnlIFioXhXIReKUCMgVJCnYUsteYQ6HRhDFVhjm9BG97AQn8R0ZveQuTO64l/aQR+/jmYewIkiGAzpRIEUhKehFjcJBo3iCY04qZGAs2eBURmApGAhQWI/8mtU8Yhgb9oKkM4lSSc0YgisSUfCRaAU8AxQDTcb0g8803+8O338+Rd7cr8MnsvJdS3nfl+GW5bxtxgEYGhIuJD5eji8HdvJdJzGYHeS/AObMSzz4V3+M1E972BVOMboK8cfeRijo69g2c/PUbm2YfU9EwChlQiSTQSJ67ZdXWyTHVGpuQOqxls/r5LTQQgCiRnnS0l1VZoEGEGNQuGWndcJK3M0O6B6DzEjhH41Sc59slWZgcuJdWzFfZUoHVmsbh3HeEDLqKDOaR686FD7GY3qT3FRAfceIfcLO13E9orf6+OIgOl+PaUEeqtwOgpJdNdCj3lsHczxtTb8PzLTvj5p+H4g5A4LXljtU6JP5ViMawzGwgRNDQFCN1MIsuzyfuKNWGaGSsq5xTwKfPCWbZM5hqWBXGseiWnGVe3XVn45wRJ4qRFi2WiJNJBYkkfkZRHMWWcKBphvJlFwv6nmP/mR/D+6xBnRq9lrutSol2byfRvxezfRKyvAt9gBYv7y5gbKWVuxM3ScD6+/XmE9uUR788j05YNXW6SQ5vxT17Oczdew4l79pF54n5IzEBKSoV04ppOMmXlwtIk1eJ7ii9e5TDW316DOAARBWIXLQqTiGRVklJpFjHD7BIEWedCZicJncH87bd47Jb3MDu+lVTfRugsJ9GWh6f9As4MrcN/MJ/ovrWk+9ZCVxZ05ZDszVcACQ6KJikj8goBstTlVgDR+qvQ+jajD10Od7TC9z8Kz8oquIvq2WViackfz6QzLKUFKhLUzBCKB4kmIyQyoiXOJkOV7y02j+NbSORGrbEh64s4AJGrnHVWpSkdLfyXbOXiGbWIj5irTjLOWqJN1i6MabJMnMwSLzPoy8TYsGgmmU0nmFeGWIq0/wQceRi+fQ/6HYOE915NqGWLKtgUcza4rxDvcBaLYxewNHoBvuF1BPflERkoUwCS+jTJvxgD5SQnt7M0spWjo5exdO8gPHW/NfF2Okg6k0A3TeIpazHSjCwrZ8gqodY7v1r//u0B4jDACwCiwCHS1JnNXCIYRggML/hP4v31jzh272HmhreSEQ3R6YIOF+E9LuYHCzgxms2Z4QuI71sD/edC77nQvwZtMJfwYAnC0JIs1KSj9hatirS9LpjcgD5UxPzQJvx3NsGDn4b5p8CIKZQLvrUURAzQhcdVT6ZVLZKmiZ0vDJ5W852bmRSptEhHK/n5ohpBhIhNZ5ljZSO+jH2ZyC0WISMr28rAfclkv4gfGE0k8IdlASA7ky2HxSGyFLAEl0j5yAI8+RDpL91B+OYmYvsvJj3ottp+4J/IDP4TqcE3EB9cp9o6vHeTigDOdJYQGChH31tKRLL2XbkEhysJvO89nP54Jzz9HYgdVwBWII2nSciO4DlqC5BXCx0yu7u8qwVCJyehQgbqO8dRtjpVDnT8BZHo9okrStXVdV7498t+eOlgub5M/iYmlTT+HERmQJ8Bz+8JPvh5fvPhcZ44cAPcciUMCkByoN+NNlLG3LCL40NZzA+vUwDJDJyLUHLvGqL7cgnvKyLeV0Wytwqjv4j4gMvqtMESooNFimKDLrS9BcQHLBIQRQdLCA+WEdpnkX9ISr3LiN1VAz+5C+Z+CvE5VWQnTq/Id6nkliWRRQEobSDOpvRwRhxhXUlwicwlUjpGUkNPxRFzSyI5MuRLcmEOOf3hbJf7ZYXHptruL/1bhI+Ej6XUPGaQihsqfK4nU8iiQ/KEihclepZIk1RIF7SnIJqBeFqZPoYCV8oSCoFZePS7ZD49RnjyCuJ788kMnAf9b1JgSQ2st9p7oJLA3q3M79uOZ/92fH3l+NrWo7WdR2ZvLsbERmYPvgX/v43CyR9BJqiESzSWsCbhcBpFAdriWcWTTjvL1uFRtSti56yGtnj+zzPnOdIIEuZLRHzWRMpGkIweJRQHX8Jad0KTq0lvSLw9IYVkMjm0tVCNDGlVK4DZW8GQREaU8ycxBzt8n5Sl+WxGERvbuaSEcWUFc7VMmJQTqHU8vJCYA/0EBJ6EmZ/CE1+HH3yI1OeG8XygnpMT7+TkwEai+0tIDuaTHCwkPlRIdL+L0IiL4GghoZF8tOFc4vuzFen7c4kOW8doQyUqY53oySLZl4e+v5zo+EZi4xuIjZYQGxJgZMFQLum9uUT6i1jqK2F+oJyF4U0EDmxn7rZrmPv6nYR++6AFYDH/TA0jFkXTk6TsUv5lIbTcYbYQULkfQ6U6rVUUrVYQplT9smIrESvhCetazvmy1LSMo7BzI85WKjdtyiSd1W1F4NjHiaZQJHMi2ReWrf2gspF7Oc8hW/lbfSQULLNgJERD6giYxPSSY6TfVWjNG4Hjz8ETD+O9o5fZ4Svw7CmFsUoV2ZP2jPesx9hfgXffpfgmrsI79lY8feWkutfAwAUwlEVsbw6nRyvQ75+CpafsnE8aM2pYkb+krpZ6ULP5y4vI/QUrK7MG0lRKP69MPC+/qv1Sf3pzjrycNL6KMZuy4IosmSXrOFg2pyzMIq6gxdEyCCmo1vjLxCIktJgFBFuxpFMmYhsSF3tCA01UtzSmbkkXZxZzZ/bymAf96E+JHvkRoce/jf8XX2PpwXtY+vYdeP79MIEvj3Lqrnrmb78e7/veReCmywkf2kZ0spLoWCmx0SKL4YXpV0H6UCHJvhxM0RLDZUTHNxCcqCI4WkpoqIDI3vUw7ibRn6Wknj6xhfmRbfxh/3ZCH2uFR7+szD1JWjqfRCJBLBZD0zQljZ3vX3wrvWnZ/zJ+XiJVKxnTAYnDoMK4Vu/rloMumeWUQSadIiMqSqSVSB9RVbL5c1slzUTQ2bafLXUdPpNnEZKntLFjD2Kzw+xqQRxL0zjaRiaxFPmZ8SVhdhGO/Abzm59k7uYmTvZvwz9YRmaiBEbzVHLRN1BFYPhyAiOXEhnaBMMlMFkEY9kk+y9Q/ey9/Rq0733YWutEVqtKGKTj0ioy1WzMjmbZAHEALw8u+zZABB4yq7z4U897nxfvmOVvzxEAyMupj7Lzg6QTIaJxHa9usBCPs2RohBMBjKQPM+0V11I9WIogaWbI8BzwDKSPgnEEor+DwOPgeRROPgJP/wh+9Z8gSwR/+cNkPn4Lidsnid/aS+TA9cSm30ls+h1Ep68gOnUZkaltxKY2E5ssJzRWQHgsh/D4WiJj5xMZfxORsf9DZOz/IzL6RqIj2asChwBK2+9SmeDEUBnJkSLi48X4xzewOFbJ4nAxS4NuVQ+VGK2AkQLM0VICt1xF9L4x+O13rYnolJMr89IliMfjpFIpMlK6ncmofadpX3wrjCzhSyHLbBXmdCT4SuZUHa0Uhy3BJR8kmfCMwMoKdr7YViw6iY4JU8jvcm2HrCSb46BLyMAyQZbVuxxonWidZONOvhKRICR5KusJFCdawJRdeXjh4XgKluYxHrqfUx8Z4pnxK5kbLCYxlAWj64n2riXen0+mJ59kr5vwyDYiBy4iPOrC07de8cLMyEV4b3sP/PgOCB612wuloSWJLIsiSTDheRpkBQqsuj4Bk5AVKZRXUzLkxTtm+dtzfAYE4qBpcUjIEgGiQXyoiZ5lXTtZjEXq95NnVHUskT9A6Aj4nwLvf5M59XUyz34W44kPE3zkfSx8e5JTX+jj+CdaOHlXPSduuZaZm65h6dA1BKbeTXT83ehjV5EYfhfsv4K0lB60rrOofS10rCXTfSFmz/kkxLEeW09mYh2pqTUkp8/DOPBm9ANvInbwjcQOnE90NIfYiGt1NOzGGKtSlBgrQRsrwj9RwcJ4BXOj5cwNl+Ef30RkfBOJySrSH7gavvMvajkzlctIWhJbgBGJRBRAlltWBJiYpC/1UWgQ6b1CggtzvZBWMqpjPigT12JKh4dfbCsOdixpoKVSyq8xMk4pqMXcsqSoQUwZuSJd5Vbq/nIxYXKH5L5iItt8L5aHkPxsiWo7BJ0W/8qW5vJ4coK0kyyc+uzPCX39Vv5w8F0cE59utABz34UwKIGUtUqTL+zfyqmxS5gbrWRufwmBqYsIT28jNl1B5K6rMR7+FETFz4OIWPzqGawckWJ4555yXxsBFkAERJJ0PeuHOL+/VBedE08lEScrbUSs9fOiJyzJ/8y34NHPwoN3wPdvg/tvIfXlQ0TvGWPpQwPMvL+ThVvqWRi9FN/wJvwjm/GNbVQMFZjYiH9qE/6pjYQObyZwuArfwUoWD5QyN1nEmYkCTo/msTCUA6NlsK8Y9hXBYBH0F5Lqz1cx8uhArpLy0RE3oVE3ofEiAlPF+KdL8B8sJ3SgVEmaVwKQ1HgF8fEqgpOb8E5swDfhxjPhZmZqA3OTGwlOVOKd3Er0I+3w8/vAd8Raa1ySe7bzqurCEgkr4ma3tmiUaFTc9D/zkU50ACGdqrhcfAMxfSSDbPsJDvfLMSIsbV9PggGJjJVDEfloaRFHq5hKeyjZmZHsQep5f8tKHWFSyh6I2KFcuc3zJPELASIxhhf4RkozpXTSMlow4YOMU0Wlq+XvTPFX0mIO+sHzFNGHP8cfPt7PsQNXkh7Khv1vhulcMjeWEzq4lYWJS5mfuBzPgSuUsIrfUoV5sBD/SCGhuxvgN99S682L5SPPIs8szaIwYfu/0qwWCXjEz3PIEir2j3+mc+Ac9VKiNdIRCBwn9N9f59i9+znz/muIve8KEoc2Yx7YTHzqEiITbyE49jb8I1cSGroMfe8WGK6CoVIYKoEhqTsqJL0vj8S+XLShLEJDa/HvX4t/bC2+qfV4D67Hd2M2gZtzid5UQGK8EEaKyIyWYI6VKDMmNVZOcqxSRTK00SoiYxsJj2wiNLoF/8hWvCNb8I5sVcCMjJagjbpXTfGxMiLj5XgnqlicLMc/6cY37cJ7eAPew1uZP7Ad874BePJ7EJoDQypKlz0Ba3KOdFqZVIq3bPPKMbNeqgfOduJyb2LNdCdOthTnOSTOtQUMcYSFKcR6ERNHysPjCYmAxe0QsZSKOBGbF9864WQtIwCxAjGOLyq3UcyzUhLb9xZQOiFm59klEKFAmohjGCFMQxYp8trl60FSpoDFHs8hyU0zCAkv0T/8kvl/uw3/eBXpobXo0wXoN1eg37gR7/gmTo9eyszEJSyNFhCbyiE5uZ7wcA7+qYuJfX4M89nHl4fjOs8ijynkyBILOKI1rECI1WK2KeYIJTn5JT7npPynSIdnIBmA8Ck8D3+e37x/J8+OFmMeLoDRN8LIG0mOZhMdLyI0uYHQpER7yjAmpPAvm8DeLCKDOWj78zFGXSQnXKQni8hMuUmM5dqUTWIsG2M0i8ToOoyRtcRG1xM56CZ4qJjAwRKC08X4J4vwjxcTGCkjMFKhIkcS3ZDx0+GBzeiD2zGGLiO1/0qSIxersG5srJDVUGSsSDnkgZES/KMFeMfy8EwXEjjsJnmzm+htF6F9dRKe+xnEJaMLft3qbmlXMyVh2hSmaSpSOZuXaOwX/mR1rGPDixx27CdhfWdfutkyfWTPAYcDEHVN5b+IDyN5ColWSeTKIJOMk9AiJPUoqXjMmjBBQrLyezqpQC3Xs3wJxyKRu8i9rXs6v8tWgUcBR7SaDV47IJYwIW5CTGaXkSy8MsAkT7IEyNqQHtLxBTJSjStMLNp39jm0f59Ev+sqZqc3sTRRijblUn6nRK/OjJah7V9LYug8jLE8olOleCY3MHvr1cS+eYe1arDSThaopT3l6eV9HLJmQHkBQBzwy/bPAURlZePyItKwUcK/e4AnPtzOsclyuNUF+/8X7P9/SIydS2wyj/DBUsKHK4gfLiFxuJTgeAH+cReBCTehCTeRiSKi4y70CTfx8UIFltR4AamxfMyRPFJDOST3ZZMaXI+2fz3esRwWpnJZmipQzOmbLiI4XUJ0qhxtagPahPgIm0iObiY1shVz+CIYvkRRemSbKgpcDTjknPC4G+9IMZ4RFwHJ8I5egH86i9BNbqLv38T83deC/7dqzUDhKbF5fRlL4lpdIZ2dUqaVOOcrASKgEd/kpT/SQ1bnOSsvWdEWcTytDLbjBJ/VWfYKthkJKftAPw3h58D3NMz9Fk78itQfHiH++x8TfeKHJJ9+CPOZn6rv1e/+oxA7AfHTYMxb0UUpo3cYXkwhO6sugBVj7awTLuFKu6hSwsiqzN72n+xKY9FEEviRtXZ1Ff4JQnqRdPy0BZC0rFNoB4bk9aNPwX99kOMf2MGxye0EDlWg31RM+MZStEMu0vvXwPCFpCbdyg9ZHC9jZnIz0Q/VkfrG+6wlLdSzW0EOAbGAYzmqpt7BgYsEQ+zkohz4lwBE5a1slS0Pju7H+8h9PHf7e5mfKCA2/Ab0oX8iPHIBwSkXnuly5ibK8E+Xod20geBUGcGpCkWhyQqEIpNliqITJQjFbNLHixGSaJFFbvSJfPTJvGWKT+SzTOOFygQTMywxJtrITVJotFiR/C1AjE0Wroqik26lrbxKjWeRPHgh8Yl/JvIvJfDgYdCetMpaZGJ4ex1ykaSW0HEkvLTyS38EPLpujWER08v5SA4jri2RSPqUByDgEEhJ5wqjiQcTjmtkpP5MCjNTpyH0Gzj2Tfjlx0g/cCOBTzQS/WgN4bt34L/rOjwfeA/zt13F7C3v4Mwtb+fY9GWcOHwFc7e+E9+d16J9tJbUZ3bDfR3w5X2qnJ2nH4KFp60aNwmjSoLWtH3S9JI1mMk4qRK1GVl70IgSNNJEpTRAluU1ksrmk5GAQUAMLClVl9IaaTe1aKdjZknEzp6pRDGoVEZEn4Hf34/vi8OceP+78N6yidShtaT2/2+Sg2/CHF5HeiKPuPieN27Ff+M2jOkNykcJf+cTVhY/EcaMSvGqNf3TYjyCpoSM5YNYsEla95YucwDidMaf2J4jLyDk5DvEZDCO/YLFL45wfHoL4YlczIn1JCfy0cZF/ZUpM0tCsGJmCRhCU2VEJmxgONtlkJQRnRCywLJyG5sQoLjPgmA8n4RNFkjyiE+cBY/Yos+jyTxik/kvCQ5R2S9F/uFcQuM5hCbWEphah3nXBnigD05/H+ILymwRwSFttBwOtxX4yymWE5CIMy+RLdEulo8i/kJSMVw4GkI3pHpXACg9KEwXAP8JMid+jv6rL+K7/zBzn2rm1O3v5PTN21g4UI55axXcVAaHy0kdLCI5VUR8sgB9rIDYmGSkXepv0ebaeN7y97GRPPxjpfhueQ/zd7TgvWeU2P13wi++Bsd/CsGnWewSOQAAIABJREFUrahl4jikjoM5Axlh+6gan+PLZPALMGI6BMOkQgaiMB1eEpDMZ2TuEqvd5L2U1lUmoJPIdELWsrLvKTVoau7rh/j9wUtYGvg/MPZPcFMuifG1aENr0MfzCB3cRHi6HCZySB7exKm7u+Hx74MmGnEJUrIMuEwimCaUljCEFbZQ7SlWkoDT0RxnZdWfgAeoRKG8lLyIkPrI6Lxf3Mfs+68ndnAzTBXDaCEM5sCgC0bKYX8pZl8+KQmPThYRn7C22qSzLVLfy29/mkrQJsrRJiptKle5j9hkKdHpEsIHilRFbuigi9DBAgKH8vEfzrUpm8ChXGLTBS8JgJcChzBScjKb6NhavBPZ+O66nPSDN8HSTyB5gozhU1BQhpI0pmpQm3ntBJ/V2k7D/fFWolkCCgGIbJ19OVKYRq4dk7XSRTWpS8cgeAaO/hfmw/eyeO8w3o8147ntXSxOb2RJqgRGckiM55GeyleCy5i0BIs+nqtsdW0sh/hoLrLlcCncWAIHizEPujGnXDjHByeLmb3xLZw4fCUz0xfhnd6Cfsul8OHr4EsD8K33wZmfQeiYmi5GTC0pwg8SJYIMWJIKWystYbstjnqVOfsIJayRJSJYxMhRIJE2VOZZ1Brbk05jZsScFDEUJn3mlyx8dRrPwU1oI2tI35hN4uA64pNr0cfFhy0iNV4E09nKxD86fSVz98mIxAdAgVnK8wXESfyGGHp2vylgiI8mZDvqf9xdf/TNOVa1pl37lcpYGVnJpi88hv6Nmwh94N1KnaWH1sPgBTAsodkC2F+golWm+Bc2JScKEDImz5IwoZA2ZZEw9DJNFdmAqLSy41PlyvcIT5cSXgGQ8IECwgfzCR3KJXQo26b1hA9m/1mA6NNu/hQZ0wVwcxHBiRw8H3gLPHirFeJOLZJJBkiZcdWxwruKeUX6qQiRdKZjqyvD9I8a1vlC8iOiORz/RLYCFkVSniP952BOxpk/8wsSD3wC/z19LP3L1fimNiFCJzNZCFP5MJUH03lwoAAOFpAczyE+kWMx/VQeyelCktP5pA64SB0oIDFVsPy3hErTh4rIHHYr0Ji3lBG4uRLfjeVEDxRgjGeTHlkLYtIe2IL/pqsI3TuM/zsfRzvykLUwJ34y+EixoMaLCOOr9hHGl3yORN5kdsSkTsaQ3IP1+/OOU+UqkuiU8K+pJsEU40hpaIl0Lf0GfvAvRD70LuZG1pO8OZ/MbRXKWtCGs2HKBYfEesjDe/PFPHvL1YTvPwCen4J5EtIe9ERImaeqH6R9paHl+eTeapIQK8Ln9NOf2p6jXkYShAkdMx5VJM46yVk48QCeL/Vx5raLmBk+j8jUGzBvXoN2+Hy8h9YSuTGflHTYuE0TeTCRp+xFczJP/SYdJIwoYTztQAGxgxZFDhUgFDvgPksHC9EOCBWgH8jDmM4jMZ1DaipLkTm1HqH01FpF5lSWurZ1jnPu87f6QRcvRfFDbvzvu4zQNw/D4qNKikXSCbwJqYsS7rWljlOEqVryLwfICxteTCtHm6hybcl1BL2YR39J8PufZunTQyze9l6CUxswhBluzIcDa2HiPBg9F/avIbN/HYmRXKUlBBzxqWzVVsmDeaQOFZK+sZDMTUVkbnKp743pHLWV4+JTuep4fVLOuwB97P8lMfW/SRw4n8RNBRg3V2Hcsp34TZeh3XQ5wUOXMX/wck697z0sfWEvyUfvBd8vgTNqftyQDNu1eU/5LhLmjS9ZVdcyMExK9O2aNAGSEgiqUSypEIj6kJiXaFJxaSxTSIYzHIHffZngnVeRvHUzocMb8U4XEz6YS+qmfNKHRBMWwPu3MH9oI8++/53Ef/JBMJ6EzDwyGjMptWdyTUGpyDYFEvlHqgYkAGFVD7ywj1b+fQ4pAYc4ZGEy8TAyDkBFtERCmrPEfvkZTt1Ty7ED2YRuPI/UbRcQuvHNzNy4Dv9tJYqRxYESElA4JMBJKHo+QFaCRMAifys6mId2MG8ZGNKpFjhyMCdzSCvKgoksGLdIvhPwvRRA/n/e3gO6ruu689b62swksS1ZYhFI9F4JgBSpFpdxquM4TuI4zngmseOZzKRMkkkmK7YlkUTvAEH0QoIA2FQs23KXYjuO4xa5SLIkq8vsRH29l9+3/vvcB0KyLUfkzGCtg/vKfe/du/f+73b22UcA+GlD2biLLXcQvffvXNOAxJLNtCrQlEYT482tMq2z4T14BJZ2FISMq5tp+hMfK+4QMF7xlwzAc18i8+URLs7+Oc93/ibnD9yB/0CTaX8+tgP+7kb4nze4499tJfP3BaQ+VkHi7hpi+2sJfqyUwN2FBO8uIbS/mMiBcqItZUQPVhBrLScukLRVEGuptOc65t6PHyyEljeTPfgm4i03E2wpZf1gLWsHGlm/R9a8nszBRuJ3KdOnJQT1vNT5qyyf+Esy/zIHF75pzTKcKlHZiwJ8zXOsQUxLE/TcZZfMs/KsiYDiwJIhkgkSS0eJe3OiTq8nHG3TF+HRY8RnPsgLd+3j/D2NRLtqibeVEd9fDa31cLDMUvLP3r2Lp0beR+KJU5A556JGscbhwf2gQKLrsdIUxSeql35t/l1nlWWZhMuZJxLmOMjUKS4x5q/8kMgX+swFibdUQlsJyXuKWLm7hrXWZvwHawgdrLQROVCJRnT/lRG7pwo3aoypYuyVUUXi7ioS98ikv2rodRubz68jcXcdybvcMCG5p+Y1LUT8YA0/bSy338nlEx+DJ78IMZXSLBGO+S17pOktBZqiw4YLYb6ry17JZ47napdeIfU//kRxh4oXFY/oT2BZWVkh9NSXiB95L/RWE7pnB/67dhBuKSPRXUeso45AazXLd5Wwur+SYGsdkY5dxDp2EWmrJXSgytzOtf1lFospPlNMpqPvYBm+gxV2zL2v5/6WcgKtVQTbKgm11xDqrGWltYLltjKbvF1rKcR/sIDAgZ2EDuwkcqCQ5b/dRuxAFfTshZ7biO/fbe5w4uBt+A69h8Sj04TOPALhZ7117EpbuwIAzY3YxKJueqOCO7cAK2OrK6XJ1YQhHlGdX2xjnkeTj6aok0skHr2XlZkPstZ1p5tMPFhF9GA1mdZy+Nib4WAeq+31fL/jLTx2/G8Jq/5PYLVCtE0+3gZABA7FKWGviu3HeZZ75Tp/yN1QOu1KTsRCmbt1FeXqiUozH/88q0f+C6udd5Bsryd9sIrwXXVu7uNgDb7WSnwtVfhaagwwAk3wYJUxUYCJSfhfIfB1JO5qMGF/NUB07mZQRQWAe2Sp6ojsbyB6T8OmY50BUlozdLDcgdPOd6DRb4dbq4m0ViNwJw6WE28pJ9xaZ77rM92/R+Kxh12aUKXqcR/B0DqBVNqyesvqGu/R44olcbZapDFxlzR4SsjccM+i67H9STCSKbLWmCFmWRbOP84LDx/l8dEPsL5/O9x1HRz4t2Rb30yybQfR9lLCHbUEOxpZ72hmtXMPa+1NJgTrrVWst5ThaynCBLqjkvWOatbaq/C1ubHeVk5uBDqr0PB3VNr7662VaKy1CBiVnG2t5XxHLcudVQS7y4h17iTevo1YyxYi+7eSaCsmcHc+vr8vspQ8B13GjL8vx/+3pTx5cB/Pz3+I0LcnYPnrNuchDS3ayL5KX28EyoYT5ZVkc1RZmySuiT4VXmqZg601d+ltNbSOJNLus5EL8J1jrE7+By7cs4fQQSmJeoKysAe2kr3rBuhvYqnnTr7R8hYufL7TpY7Da+4idCGObRvunmZpsqb+XmXVc3zzjtcJDLoZVw9qmXd3aznG68sTARKPf5zHR97PSx23EOtugP0lcFcByZYqp9F0we27CHgj2F5PpK3GzHuypYy0xsEKktLoLbtsDiXS2kS8tda5AXIFNkYN8dYfH5pA0vkbx7Yqou3FRDoKCXYWE2qvInGwieSBZvt8qKOCpdZSIr01JNuLyLTuINmxgzP3lPPs4T8h/O0vuDb7tt5bvrKIdWXWWs82GGwEExLcMACozGI15NJQXgmIKKg158FU0vbCSMdyRVspiKzC9+7n8tQfcv5gFYGOmwn2bCXYcxOhru1EOgWOYosFRKf0gVrSB+pJHqwl0eJoGekoJtSZT7B7B76efNa7K1jvqmG9qwrfxqgg0PmTxuZzquwzq91VrHZX4OsuJdhVSLRjB4n2PJLteaTbdpBsyzeQyE1ztN9FtKXZSkKiLU1mXaQYz/XcwYXjf0zyiaOQesqcVFkE0UmeZUpC5imSZEa9zlQR7pSznaS5l2zcQCWrLbmU6Pm1nkWPkkvw1KcIHvvPLB/cQ/hgLQw1EW8vhJ5yaKm1xESwpZhLfXtIffEeiDwJkRDZtRgRrb7wem35jLFayeb3MiSvQsWmp7ZgShdyBSBrriGARfwuKWGB1vJTXPiHQzw19E4utdVDWxns30HmYKkJtsy+QCGA+Dp2EeioNYGNtpcZgdMtpQgoidYqoq31hFqbCbU1GbiiEvSrGe1lRDrzCXXtINBVTLCjinjLKwHi76/h/P6bCR7YQranEH9bHi9338rlz47DyiWX1dhEkNf1UAwPu10qxcdg0jWyE4OV4wqq4Zssh+q3tCXyN+9nfeqP8Xc2Q+cW6L/BgLHeU4yvu5hgZymR9jKibRUOJC1VZq1NsYh2baXEPYUQ6nRKQdbB11lLoKPGhmggxaDvsdFRahZJVkm8yA29F+oo2wBSsLOMiM7tKCTRnm/AEDiSbYXud9sqTOFJCfram1lv32v8o10ZpUJz5852N/PS7Hu4/PB+ki9/BmIvuSJFT6/YOvtE1lZPJtTAIhs0bFyxzq4qV+DQkECvqdOKiCsARV6C7x8jevQPWWm5Bd+BagLt5QQPFpFRVq+1Cva/mfWPXk94+l3wvWMQugShhLUz1QLnFa30FGpl1mJa3++h9qcwfsOC2Ps2kSJz57Wdt2rVlNXykF6Di9/g/MJf8lLb7SQ6a6GlwPLUsg7S/rIYwXYxS2C5ApB4WymJVoHDnScLIOsRad1FtK3m6sBh1qPMNJ40b7CjglC7rEu95celcQW61FA9F+/eynrrVoLdJbzcUs3K/B/Dy/9sltHSfj+FOP+ql0XfpDCgYj0tNHMzuaYBVfekVObqM2T+aZqzh3+f8wf3EO+phwFljPJItlQ469DqrlfCHegsM42+3l1qR1+XE2RZSCmieIvucZdZy2irrPEuu29peN23rLpAJeXFwWJQMH5AI3/T2GHvyVLLXdH3ihfio7JZOX4Zz9pUSLiZv/WeEqwnoc8drLCM5lJrOc8dqOCp7lu5cOrPiH57DvzPeVbatTbS3jnmlJAgquSQVUCahnbk1ky7uWAu/ltVWbspa7li6mTztLlbvpk/4kd3NxHpamTlYzutRCjTWQWtO4jdvZPVrn28NPZBOPeoldEIDz6to1e1SQ4gWuJqrsBP5/R1uhxZHPsTsy01JofNXWhErT9jEYfg+EUyXz/GhfEPsNzZRORgMdnWEmgpIdVWRqK9nGhHtfnPCgDDHdVEOyqJdZSTaK90o63aiBpra8BGey2x9uqrGx3l5jPHOguIttdZABdrq7PvT7bWWnYj2V1JrLcUf28BL7SW8MLgu+BbRyH2sosHbG4jR4DXd8zRWRv4JBPSd45msWQIdX20tjzrPyTw5RFeGHgnZ/dXE+2uhqF6/N3VXPhoIamWBrIHGu1owtZRTaizEn93OWu9Zaz1lrPWU4m/q5ZgZwOh9mairbegIDlxcB+Jlt0kWptJtjSSbGkg1VJHVu6H5joOVMKBcpsz4ECJc4v3F4GNAns9c7Bh4zs2f0/C6FhHzPhT6/FVvNX1uRHsrMXfXkegrY5Ie4XxOXCwiMsHyjjXfRsvj70P/5cOk9I2bEnFGBmLm0MpBeqebHoAkQCbrFrHlpAp+JxLJKoaLa29rEpTnod/WWB56kOsdr2V1YNVrHeWEuoqIbq/kExbPdGefTyx/y2saf1I+Ix9uRZ+ZjX772UlfxY49LPXuWpHl4qzqzKnW0hxacxQJkI8mySrzmJqjLz8FMmHB3l54Je4IC0lgBwsgBb5+CUeSCoJd3pDgOmsNALHBBIDiKuWzT2+FoAkOneS6Cgi1tpMrHU3UWNoJQJIpqXatEl2oJKLPSU83Xcrvi8M2T2QuAgpr0jz9eFi42wx1NyBTMra0rhIUAyIeKnOF7n8xT7OzPwel9oriXfsINtdQLgtn4utNSx37SHc3kiitYGcQG4+CuzR9gbCNhoJdjTj79iNr/MWfB37CLbfYvctcCRa3RBIHFAcWDIH67DRUmvAEXjcqCZ7sI7sgWYyB1T7tM+GyjcSLfre3UTbmu367Pc7aj2elhPtLCPaWUKoqxxfbyPB3iYHkLt3krpru1ksxSyXWhv5Yfcvc+nBj8JLD7vFd7h17FpLY4pZRPQeO4B4JfGeJZa7agAhSSq86rqvyN3yPQ/fu48LYx/kcvcdrPcXEuzdYUWvvv3NZHpvY7XvLTx39I9IvPAZNx8jFNr+h2tuojYHyg2O/viD61wkL6Z6pkRXrce6IOKEslpxBiGVsvojbrHQi1/iwvE/4+Xe20mq4rc1H1oLoa2YdHspSdPsFUS6qgh31diIdNYQ66wh3lFFsqOCdJsb+ny8Q69fxeisINlZaL+Xat1NsnUPkc46Yp1VpNrqoKXGViKGW4t5oaeWix//b7CkAsSwt1xWMx6yllf3J3qHbLbdBfZJdSuRo61KhMuPE/nnKX44/C6WDu0h1qfrvJHowZsIthbh67mF4ODb8HfvJty5y1K48fZ6Um31ZFtUHqIcvzumW3eRbNM5TQS6drPevZvVnj2s9zQR76izCcF0Ww2p9hq772R7nWUb9X3x9l02Yu1NvHok25rg4G44sAe07ufgXlIttxJvvZVI+62EO/YS7thDuLOJcFed8TPWVU68q5h4Vz6R7mJLgvh76o23sbsLSH30ZtifDy3FxPYXs9JWw/Pdt/PywodJPn7CGuhp8ZTcHO2zmrMcOtqfqollRWwSz4FDHR8jiaRrqCehlnyqgjd+iew3Frk4+X6ebdnGxfYtpjwCrXeQ6GgmPngrz/btZfWLH4Nz33OVy+qpFl8hop3Jchm23G//hON1Lpz0lkjqKjWsVMXlq0PeSjMtLU5Z0VAYwi8T+5djnJn9AKHOJmgthjYBpJBsezGpjmLineVEuioIddcQ6qqzISJLeAUgAcmBqcJe0+uvd8QNIKWk2itwQtREpLPeQCIhsaCtQynREi7M/hbZpxe9+EqqX9ZDSdxrAYgaFsQ2Mi7GOJWOX36O5FcmuDj2XlZ7dxEfLCbRl0eo4yYC7fnE+htI9N9GsGsfa917We/eY3TUHEe6VcBW4FvhuUflcLAcWitMGYmGEkh9r7+nhnRHPrRth/Y8aN9JuqNwE/2dggp0y6XbZUOg0ljrabLfNDAeEBh3kVWCo203sfY9BoxQp65rt513BSClxLsKSXXuJNmVz/KBbfjaipECTHU0uMLJllrMpTuQB4MV+A7czPP7izk/9154TIHzC+ojxLpfcyE5C5GTfEmpBND8IZsLUTtWvysYNqsTVY1XDlnhs6S+Ps6PRu/kTEcJgf47ifW9zVnezkp8PWVcPHwnwS90w+XHbFWj1qRoN6t/JUAkIF6K0wq6nN3R78tPVMQvXzDnM2ZDPrde/fJ3OHf/3Sz33EmqvYp0e7mBQwBJdxSTVEamq9wAEuiuQ0wSUGRVBJBsuwOTgPR6gbH5fFkk+31p0LY608ahrl2mWdNtVXCokXDvHmKf/xisP+Z8YNMUihY1HbgRgf0E/fHaL4lGIrI/rZy+Z3mjQVJfe4DlsT9iuaWRbG859OWRHMonNlxOeKCBgK6vvYlE5257LOENdeWArRKRGgeSlgpXaqJyk7ZSUh2lxEXX7lJCPToWQsdWaL8B2m+0x9mOPBNeaXk7r7uCQE+VgUmA2jzCXVVk25QidQAUvaRYBFRZDSm/QJfGrldYkGRnManOfJJdO8kOlRnoV/aXEm5vJttzJ3TeCm210FHkZeu2kurbwaWeal6eei/+r4zD0nPWTUV0Ew0tHthoLqHn8mpUzRG1OrbcBIR6GKtNlarP7XNKKCWeg0eHeHn4VzjX2kS4u5FYdx3J7nIYKuXiR29ibeJd8Ox9toZGa22UZRQ4za17DTZfl7swd3ruY1dWk+kiNPRlTpSEbK+kIPoMkfkP4ZPpb62GgXrSncpW7STdXUqsq5RQdxUCiLSejuHuCmNytiMfjVhHGeHO8qsaSgCkuhqtvit2oIB0Z4XNxay1NhLurCfZowRAJckj74FnPmlBuXzai+pkbskI3ZX5k69Bop/+lughJ031SM56aFL1q5wb/nNSg78BPU3QkU+m62bCfSWs99ey3ttMuKuZTFsDaNXigTxC7TuMVom+GpJ9TcS7mwm2NrC+v8YeJ7oaSHdVkeosJNN+M5n2G8h0/Dy0/zy03QBtN5Jpu4lMxzZSHZrr2WlaPiZXqLuEaGcR4bYdBFu2ET6wlcjBrTYRmGrdBl159p2a95C7muqrRNcR7Kpktb3ceGZ866rZcLGk/OQl6PxYdz7hnmIDeKBzL+H2O4m13UpaZSDyKto10/3/km7/OeL9hVzuv4VzY79H5stjth+I5E9S5/jgLIr60qXkQtkut5I397oBwnNyVAakoaJEsqrdet7iy5WhO4n3lZEarLbro7MADlWS6Skhcuo/WlcU/d6lgN+aWPxsgEg+7Cx9TDiVCLl8tK039laKCekK6LM57Cn1pqzCPw4RmX4vSwoauxtJ99aQ6Sol011MqrfMABHoqWG9VyCpI9RdgRgn10Aj1lFyVeAwUHVUk+7Za5mWRPsO0t3FRLt3E+y5lWDfLgK9Daz17IaH/hYuPm7VnBJoFUSL2BllM67hT59e9poMWhXrxSfJPjRIeOB3Sbfshu46aC8i0V1CoK+a9b4GfL3NRDsbnYZtKyPdmUei52biPQWEZRm6a1jvrGe5vZHLrY3Ehn+RcP8tbib8wA4C+28k1no9dL8RBncQ6akk0NNowbK/bzf+3lvw9d1CoG8vgf49BHsaCPfUkuiuId1dTqazyEBLWx60bSF04I1Eu24i2b+D1EAB0d4CAp078XUU4tOkZLfc5Arjo1zmWKcUXLl5AbJmod5iAr0CUj0BuWQdil/2uoqL9lLo2k6m5Q2k2n+BdO92/B2lXGrfRWL29+Gro1bek0jEyCTlp7j2RLYNQyJFRk0Kvepbiagstv501NmSVFecouW/wIUfwmf/gtWeUtZbtsOhCtBE4oGtZFpuIDhxJ0vfOkI4eMHm8TW/8rMBol8zJSqAyCf3AiRVYOY60lg1pkCTm573Yhah/OI34B86OT/8W5xr20usrxH6q8n2FMFgGdGeMoK9lfj6am3ocbSnhHRXgQ1ZGRH+qob83r59pHoarYgy3p1Hoq+ZyNAd+AaaOdfTxNLEb9rkkm1Kk4b1jNv5QgBJZLwNQD3Cv96DeLLkbZlH/DzZR4aJDP0WdN9maxfoqUPaP6R4oW+XgTba00CyS8CpIdtZSaq/iPhgAdGBIhM2X5c3M97TZMK+2r3bQO6XsA80ERxsIDBYjb+vlIt9jVw++iHOHP0fNs4e+2vOzf01F4/9FctH/4K1o39CYOYPCYz9Dv6+X8anyb2DSgTUgaohhhoI9exgvXc7Po2ePAI9eYR68wn3FRDtKzZeiYcacVn/rkobic5qIt3VxtP1vnoCvfVEuutJdNaT6qgl1VlNQuf2VeDvyDfQxfpLCHcVsX73zcRbS4iOvxOe/gSsv+S2V1PK3eaO0taFUzsebHg43ky7BFp013DuWYqQukSav5uBHz3I+vHfZFUKoDMP5NJ/JI9Y+82sDNTy4vT7SDz9oMm5Xw2wfwbTr7MzDEa6misWxC5MwNmwMLn3BZKUazOpb9e65pcfZuXev+OZjrey0tVEqq+ajPzjoRLiPUWEe8vw9TliCiCRnjJS3QU24t1lRLsrr2pEumuJ9ewhOdBMrCePcOcW4v01RIf3cXloN88PvoXApz8CF9R0IYlaNkW9mEoaKPKvXzfzE8mo2zf3QAHlS//A2ux/InigzhV03pVHtqeGUE+TaXRpdwEl0V1FuqvSwKGj6p/8A+VEhmoID1YT7K8k0FdBeLCO8OEm/MO3sHJoHxcHbuPs0Nu5OPluAif/mNRn/ob0l/tg+XFYfdEbL8CqxnOw8jQsfx9e/BI8fi98aZDU6f9JaPwD+Pp/g/WOt7PWuY+zHRWc7ytldagS36EKAkNlhAdLLLGQGiol0V3gjRLEK4HE8auacHc9a32NrPU3EOytJdpdQaqrhHRnCQkpvZ5KQkNNLHXXsNRZRWigllRfubl0tLyJTF8Nl+Y/TPrpT0HqsqvgCF22tqZG282pYC3BSIdMLC3es5WCScLErIe1lbLY9N2L8MMp0vO/xtpHt0KPJlQriQ/UstZfy/nOBlKf/4htnaFCSc86/ET+6kUvBtFD11XDlZyYSXHoNYBonkSi4LLSKhE25Jnd0z5350k9usjzY3/AGWVAuivIdBdAXz7JXmmiUoL9VajsQ8dIXzmJ3kKSPYXEesuJyqpczeipMdMeHdxFsDuP9Y43E+0vwD9UxdnhvTw/+3vw9AMQPKPmUWSEDrOGyj5lbYbW7uOnkudnveFlW8IvEvxcK0tDbzd3KttyI3Tnk+gpJ9DfjG/AuTvR3l0ke6vMH870FhHvreJSSx3LvXsJDu4lNCRNXMx69834+rfhGypgbayWlZk7WT/1BwQ/9zGS356H5/4Zln8EIc0uez6HbmTzkIq1Xsra3FPNHc65FPczD8PXjhH+ZA/rp/6Wx9ru4OXBO1mbejuh6bcQOiw3rdw0vcAhPmZ6dprCS/YUb/BL9W3BvgazjP7+BuOfeJru2Umydyex3mLjdXBkn9VGXehsZqWnkWg4+/l8AAAgAElEQVR/FdnefOi5AQYK+VF7E2v3/xU892mIvuB6Msd91g4sF/ua/pYFSav61lkPFxXHrdG3XLJkWP2lBR25Pefg0UGCw2/D39ZoMd/64K2s9zeR6CojPfsr8MRJW1hl2bLXYPN1OQTphzcPez2XSVCmwIYKykT5K2ZOKTerk1n6Lv7PHOTy8C+ZH20A6dlOujfP5gCkGX0DNfgHqgj1a3a7mERvsZngWF8VVzOiskrdtUSG6vF1F+Lr2kp0cAvLvTfzo7HbWfrsRyHwDKgSQK37NVQ5l3VrloNZr1r0NQj0mm8py6LdWZ/7JC9O/h5Lw7eSHa0g1v4GGMoj1ZdPuL/WQKKYINTXbPeZ7Csk1b/d/P611hJLYEQE8v4qVjuLudRZyNJABcvje0k++B/gKx+BZ4+D//uuMZs1lfO692hNkDwMrU60NqNXeCO8RCPy5dWxRn661v6sQfSitwfhN4h8vp31U/+VsyPv5kzf21jpv5XwQJNZg2iLXJSd0J1Htnsn6Z4C45sUnK412F9DpLeOWE+V8TLRt5NY/3aiA3kEB4rxidejt7M2eDuXuvey1NnEuiaQ5V733ES2P8+U6bnOvayc+jN4+XOg5hCJgDU9F0A0TOIsw5rT+HpFHo1iFK2CxeKVVDrrMlsCidrePjrKub63s9631yo/ZMEZqiMy2Ezsof8Boe+49SuvwWRvHkT9ZF13buvMbh/QBag8YNXrqKElkl4DM9PC7hJXVEem89We9NkHSN37QWKDzdBbAN0qENxOoreA4EC5EUxEE1hkVQSSWF/FVYFDgBJAZJUiI40OeAMFxIffyMXuX+D8kXfAD+bNLEuRpiVU2nBFBYQJVUzJiniW8DUI9JpvqT5t7Z9JfPYj/HDoHVwYv43wdA3BwZvIDLwJDm0l3V9gbl9gYB/rg/vwDzUQGSoiPaQg+9+Q7f6/oP/nYXArkd4iVnsaWBv5VaKn/pLUI0Nw6VFYeR4CWqUXcQrSEw8THGUU1fFEbYDMRVYFhGuCLQ6aK6ljNkMsk7BCwbSWxKpTSuwiBB+3lXvh+z/CxdH3szr4a0SGfpFkT70VKtK5Ezq3G0hkSTThGeovxT9QaYou21WMMkWyHAKGf2gn64cKWD0kt62KwKFmwkP7iA7eRrB3N6tdtaz0VLA2WEZwsNDiVK0/We65lewX7rGl3lK4ygpq2wjZhNwwS2LzI86TMWWtxtsJFc+rQthlu9RoxXVb+RHhz/xPwlNvs0VW9NWTGHkLl/puJTX7a/C9Qbdy9jWY/GMA2VjgYggV0QWQNWclcl3APTdFDFjSmgkz7apYfQ6+0odv8teJDtaS7M0j03ezaVIBISyNMqhjGbF+N+KyJgOVXM0xMlhNeLiO2FgjoUNVxEeKyYy9mbNdN/Hi8T+CpW9btaYW7jjbHCezvmSbZ+bW4hvRN/Vr3WxF7b1cHOatRReznGCKMVqWPI9v+le5MHQna6O3EhirIDqyg0D3z8Ooywzp/oKDjQQGmwgYQIpJHfoFGLoO+q9zQBnaYe9dGn03a/ffBd/9NFx+CdS+VJtYumpw85py+UYrJldvrOQFSC07PuGaiGpBqbSvRElDQMlp5Ki2184kSce1BuYyRF6Ey9+Fby5wee5veLHrnSy330aqvxm6S6Bnh41M3w5iA4UWo4QGHf/oLoLuQpL9+YSH8vENF7J6uIjVw2X4hlx5vWJEhu8ge2gfkcEmUxKrY3VGKwZ2wGCxxSO+sV8n/vVJp2yzKfwh3cUVi3gFIKKAZ02sY13MmtWFBRPNk0QduKx+68JDxD7xYWJDt8LoHQRGf8lcPsYa4BMfgOQLr/BMc/x3nlWK69zPu5jDXUAOTnpNFyEYeBOJVz7t+YLq46Qafq+Vik5dfp7ww/2cHXsHofE6ksMFMFwMh+rJ9lcbIeP9+SQGK0gMVJE4VEnsUOVVHquJDCgoryYyWkh2ugRmKjnbu5tL37qXpLZgyAm4lTXrfpRk8NYh6Fb1unZa8jSVYd1r2qbHJlXm2kZtn75lEq77i0qlg0+yNPMrpMbrjPkM7CEzWGpZqdBgCb7BcvyHqu0Y6S+0ib1wa4mluePDNxDr+39gVtm+Yla6dnFp6vdJf/soqD9yKkTCr7okL7ZQutMrstNLEnaR2/iXm3U2fgm+Wrh65aO5r9h8NF4bcVyhpTbPsS0rLr1A5GunWZn7c1b63krmUJNNtikFnBgoIDZURHyohPShMjhcTXKwjsRgnfEgNFJKcLTERvhwqfE0018HfQ3QI6+ikdSAOzc4Uk5wtIjo0M0k+7ZDbyGx/jpWFn8bnj1mnktOHjPaPiOrqUFHDyW7tMAvpcV8Of6+KkRw9xeG2LPw0oNcuv8veHbg7Vwc3kd4uBr63kx48hYuP3GvWZ71RIalQNqslegkmU/FVgSQq/9TQB+3jSmTLqsl+QtFyTz1CEsP/gXnx24lMFxKWqa0Twvsy8n2FZAa2ElS5d6DHkCGy22WWTPNr2sckqtWTWSwnPj4DtKTO2Gshosjv8nFpx4l4CmZjXy1uR6qLXOiZcLn5dklVnpVR7lfAr1ZCiX2JI3am5tLXLJmz0JT2LJ3genbYLQYBhpNAOgrJSH/e6iapUN1rIzsYn2onFh/Hpkel3ZUGjw+mkf40E0wVE5q8DbiJz4M35kDdT5Mhm22WNfvmOUlAxTzKC70wKzrdYJwdTzUd+s7AllYkfepG9YXqhXt9+8n+sm/4/LAW0iN7iNzuIZY906ivfkWZ6WGS1lu3070UB3hQw2EDlcTGiknNFJCeKSE2HAJqcFyS/nTWwc9DeiYGagmcajczgmNFhAbzSc5tMMSOqn+Utan9xH+yt/C0jddBa8tB9e+NRc9d9/F4bpMgcSuVyDRCxvE0Au6mSiZ6Hnr65V85jTPzP8hL/TfYpk6+m9k7VAVz365i0T2slUeBsIZ2yrFyuGzKbJJ37UCRH58VN2RHCNFcU3uBF+C70xzfvY9LA3X45f57Soi2VNKpr/EtGxSbtZguRHrdYFiM4g8gMgnjo/l20gdqiJ0/EPEL71o/aZMw9iEjoRIvvmrAKL3tDegJyy6BblfapBpANF0rZUFrZJkBbc7t9ImPlLfmMU3uQ9Gi2BAmrIe+svRvfkP1bA8Usf6WL2lT02w+opgaBccbiA6VoJvpIz1Q7eROPmf4bE5iD1j20/okgJagOix2dSkFfFpnY4uxpvI3SwTV4MRTQLr67z14i6DKfdFTRdegDMPE/9cK6Hp9xHpaoYele5UkBwsZKW/gKWhYqLDlabUoodL0ZDl0BBPk0OVBohsfy301ZoHkRrMnV9CdKSAzHgR6ZECkoMlluZeGmnkR3O/SeCbYxBTwzq/xZGZ5IoJbA4I4pOWnOeebxw9OrhmDHGS2hE5ex7ijxH6p27OzvwmlwcaifQVsDJUxeOL7ye7/AVXaRyJWB4jJYUopyiTuVaAgPabCKXFMKFWKk8r6Fbh4jeJfKGFy7O/wuWhSlY6dxLuKSQ9VGNDLlZ8qIzkcAXxw1c5hquI9FZZajQxXkBoZIelSnnkHtsJS6tdnYvi9Y81i/cqgJjr5dYnyJNyNNdzJ4TZHECiasK8ip+EbdiiJg+XH/gYvom9ZEeKwbRlFQxUkRysxne4hpXRWoLTjYQOK41dQLa3FA7vIjVcz9qhCi6O30Lw/j+F7y+4HrWpVeu4KBHVzyp2MJAat+ROKJvo/G9dp4RkQ2leLUD0dfZF+iX9oiaLFXsG3A6za98j85VDBMd/l8TQbWQPNRDpL7YYI3K03uK+9HA+qeFikgLGSBkx8XO4iuShGlLG7yrSQxoVHr/L7DzFjAIH46UWQ64N1bI03MgLg7dx4fR/gR/e56p/CZNKh60DpQXmuHUlWrPuGCYllxvuJUcb9R9Tlx7F0IqzHib7T21cnvkNLvXWszLSzGMje0l8tx2C33W7pwVd11VVIgmA1+hiOQaqeEyz7M5HVDcJBZbL8NKX8D30V1yafSsX+vIJDOSTGd5FZqjBxSCHSkkOlxE/fJXDA4jqnGITBQRGCvGP3Q6PTdtKPtMwopRUssBiluHVAHGvb86USPu4lpVeYki3lnAdyjUxZfe59G1emv0g/pE9pA8JIIUwUAoDNSQ1nzFax/p4DdHZBmLjtbZoS6U3HK61DNDZwTrOzv82PPMA+J51lkPdTuKwnAGVUQokuq5XWJBNbta1AiQHsmgsQcKv5MXKxv7rcl8StlOvD0JPw2NHuXzs/ZzpaSR4uIHUzC0kJ6tJH94OwzeRHd5pdDCFN1xDXMmT4ToDilOApcQPbx5lBqjE4E7S45WsjdZxabiB9dE9LA81szKhXmV/CuvfMIumWbqgJnvNgiYIxZIEtPmoaTSl7jUEkisAEX2Up0yoF0D8AiSfg0ufI/bFuzk3/m5e7N/NC4criXzyt+Hl+yB0zpltTR0pJLOJQmPA1f0zAstMW38usVN7QwRA2wpr5ZbM2+MnWP/EBzk/Usn6cB6ZkV2khxotQBcxEyPl1zAqbeIpKndtogDfZDnB+d+Cs582S2baN+eOGu2037cAolvPaU73MAcQ90xkFUu89I+2G7ZOWZeJmZZdgR/cy5nx38F/uJnkIQXaO2CokKyC1uFGgnKtJqoJTVSRnKwl0VdtFQbpkRqWBqo4N/F2/A/fDckX0W5VyZTbi1zrps8m4Ly3gaqzICLw/3oLovtTuiIUD5MIr0My6LZ1SyZtbkEC5hcvZU1SzxF+bJYfHf+PnBt5C6HRvTBZC4e3wMj1cPhmGC4iPVxpFjIx3EB0pJ7oaDWRsTIi40VEx4qIjxWRGCkznqcOlxMfLCIxVsHSeB3nDzcQHN1Damw38ZFaLkrZPTEBseeNF/6E5ntjqOmDwBHJlUIYODyQbEpQiHaK2BJqdapFbNotLfgkvPhpgp89wA8G38HFsRpWj94GT6gJ9osbKT9tuKZK9mu2IMKBJqiypu/E3iiJeNQaNlou2vc80X/q5vyRJtbH8yzjkz7UQHywhuRIHYmRSuKj5Vc19FnNzCoTFh3PJ3Cknuin/gR834JUzK7LLIdTJUbkDYAYup2PoocCiAmjnlga2+u6pzcCApTmGi7YnkwkzhD+UgerU79O4NAuEkPFZIa3wKE80oPSms0EJxrwT1WxcqiA5Fg1GZVZDNSRGG/g0kgTq/d9EF76lKVmtVgzqOF13VjKuhovPdflGJsNIF6ud1PnFfe+g/Xr/S9L6U+qC7oSxk77EtdKJm9vMz1MaxdbpTUElDMknrufyx//c/xj/x6mmmBkC4y+EQ5vM4BkDlUj/uYAEh6rJDhRQnCygOBkvgPJaAmpw5WkDlfbMThcweXxKi6N1jgXeawRFKMNlrL2iT+Giw/bfiYCSMTbFTkmoklvmdVQsC4o5LIyjpfGOq0jyRFJezqGzkLoOXj+YZYf+ihnJ27n/NSd8MQshF6EkLrVO6Wv/Mw1AySkCgDjjITIbylUlZL7c/6BOpp/7xTnjt/B6tQ2UhPSMLVunmS0kfg1AiQ2ICJXG0D8c3uIPHIXxJ81Kyba/WsAousXMe0+vM+oTseCVs1S+5VilHU8R1pBY+QF1j/xF0SPvIPAUJ2lPpPDbyZ1eItpz9jIbkKTuwhMV3OpZ5tpS1kWWc7k9G4uz7yV4MMHIKGgXLu0OsuuBgXy/kU6m7fIXVCuosEL0MVvfUYjx/vXCw6dL1Bo+zaXtXOFBrZE3wiSJOpXax53LYpMNB2npt688HFin/pzVoabLDGSHruJzIjc5zIyh2o3AaSW4EQ5vqkifNP5BKbyCU8UEM8BZLiWzESzxWMrY2WsjpVZZTCHqmC6kMxoAWfHb4UfztoqwFA8ixbwqVWpFzo6gOi6cgDxKj1yPF2JqHWQRyvpAK1n0vp47Wf/wuf5wcRv89z078Czn4DIefArRPBCh/8VABEBjY8miRIqEdxlRmVZzJu5+AOWHnwna7PbiI8VkB1zmiI62kRsrIbYePlVjcRoFYw2GUAyR8pZOnorPLvgUoK2/NW7OF2gN/t/xYJ4cYn3+gZAdK4pU1ebZmWllni/DFllRCKw8n3WTn+A6HgzsdFdJEcryIxvsZEYrSEy1oR/0lkQuRdyKZK9FWRHmgjP7GP9vt+HMw9BTFsKXJkMywm96OcEXxci5lsazfnZnqXLzYM42l8NPLzPbNDFeRf6Xl2Hl7qzZuY5QKrmyWbr0y+Q/eExzp1+PxfEg+N1ZKdrLHhnpJHMcAMRrSuZqCM0WY5/upj12QJ8MwWEpgqJjZeSHqkwDyIyupvg+C5CU8XEJgqRC8pwBRzeCqNbCY5Xc2bud2H9Sbt/TYMabHVRqo4wLSiKeRbEAOLmgXQfup/cJKnOMsKqIkTLrtWX4IXPwsuftyXSBC+5LKzR2GUyr9mC6EeNSTltbZNsTgvq4izRvvQjVj71btaObUXZJgEkeGgXIk70WgAyVgEjDUZUAWR54S2e2xIgq15KuYvTtW0IgtwFuSqvBIi0qLkZ3mdkPdTcOKNcu24ivQTZFcgmrEo2ePp3iY9XkhitMx86PbGF5MQWYmMVhMZ34RNAJmtsXiAzVkFmsJzsaCOBI3cQ+NyfweVvubZDpvlEqNyfLjYXdG4Gh3MfROsc40XfawGIZEyf1y1vFiRl8DSZakLnJThU7xTXHmuWRVsmtfYtVr8zzMr9v41vdg/B8SZSU3tgtJn0oVo313GoiNBkqQeQQtZnigkKCALIaDnJkRpCY7cYQCQX6fE8A05mRHzdbgBJjxWyMv1Wkt9dtNWAiiC1qYEtqLK9F0Q3j2YbINHzV96XQKI72pBV3ZdcrrWXwH8G1IVRnek9vidto4fAtbpYLh0qAhuVxUOvRFkmWV6rcXN9hfXPvIfgwjbSE8UwXk9guN6IGpuoJj5RcVUjOV5pAMmM1pIWQE7/GsmLX7OLyajn0WaAiB4mCDmAvLKuTEG5neEdBBYtEEvZPuhx0iq3kfOTUYDwfSKn30VqopDkWC26jtTkzSSnthKbLCM0VYt/ahcBjaEimKgETZiONeA79g4y39TWYS9dabG0YSFUfKdgfNOw9xw4JNC6vBxANhhu4vD6/+m7xCMNPXb/5SpruCSFZNC0bjpDMqJ9Gl3NV1Z7eYQeI/Hdfl4+8uucGb2T5Pw7SE40kzpUQXa0DH/fNqJTJQRmSvDNluGbrSA0XWE0yoyVkRqrJDTRbO5oZmIHjG/DvS6LXEh2fAeM7SQy2cT5U/8NVtRVft21LY2onsxZCtHFgcRztSxod5edo9VmK2LnC0O54SlPR1vJtFqnrhPj8rUBxE2oOR92g3Oe3Akg8qWNm/41Ap97D+GF7WQnq2G8Gd9IHb6pOqKTlVcFDoFKgpk9XG8WKX2kEt8nf5ek7wemDV1libeQRoTwgKvSGGvRLSJ66lcHWQsTEB1EZ7OEWmvu2h9pP1jLM4m6y08Qu/dXyU7m2TXEx6tJTu4gPrWN6HQhgekKA0hoopFAfxHYPZeQGq9hfUFroxdd5xO7rlyGKpel2lw57SYFxUn9bE7bO0Y6cXbCYRLyuv9JeLwCDvt+A6atu5CFdb+nCnJHpjjp+JrbxXajFiFkmw1d/Mzf8dzku1mefgexid3IYjJWSPzQNuKTRYSnyvBPV9sITVUSmxAAikiNlxGdqCc6UUt2PB/GbyY1XkR8ooz4RIkpICYKyEzV8OKht8D3pyFzxs2Qp5MkE25qM4dho8iGS6pXHY2kSGQhddQ92/2IcJ5M5N7XOe48RVtrpFi6doDEbPGit6xOv+RZkZxmMuD4zhL6zDuJzefB2G4Yu4P18SrWj1QSmS4nPnV1IzVRZQBhvJaUtNMX/iOpxHmb6jKOW5m3R5GfBBBHQyPYhkshCnrpdVcjKosjpyxiwaxpnaUnSdz3yzC5xZIOiYkaElNFxGfyCB/JI3CkGP9Mk9X6hAcqYKoSJnaSmq5m7eR74dIjZj0sRjNGea7Uj1kOd4GbgaHHV0AhhF39Xw4gpsjEOE3yqu5J5bCebtEuUcpxpQmTZp1Map2stxGnlWQkLpF+/iFevPeveGb47QSndsOUXKRtMJVvHkNsqsomTAPTTYSn64hPVpKdKLKhpE1istImCxFoJouIyw3zZILpMpit57JKWj75XyDwLbd5qZKLXqcTiZ0DcS5m87J9KhfxQOKx1cTxCv3c5/SegJGznTG0DEJrT66x1EQWRGJj+5YKGDmYZlxgJLNmV7T6FIGH3k78iCaVfhFGf5n16Qp8CyWEZ0qvASCVZIbrYKLBABL/yp+g7rjatsCIEPfUn2dBHCFkQVSyobUhTric0Hno1sEDSMx2kEgQsSSn61huMeHK0yTueztMv5nURDnxyRoSk6XEZnYSnNuCf26nA8jEPhKH60ECMLGV7JEqC+7xPWrmXfS5wlhdjOcimJZxVkOv/jhzJQhe4G6IvTqQODdSFlKiIffOS9/oorziA4UdcjWdIxYkk9GcTRS1EHVLYiWlZ1n/+gTPTP8WgZk9MFsGozfC7E4k9FIgKgwMTu4jPNVIfLKK7EQBTOTZMauYZLzKKZupAmIzBUSmK4lN1pCcqoIj9eZl+KZ/EZ5ZsK6Y4q+WO0sRS+w8Vjq6mKLRO1dcRd2SznG8lkA4emfScVKag9K0ne5JsZadp+RJ+NosiH4i7NUt2VoRXamG139VP2SLlC79M74H9hGd2goDvwoj78Z/pBL/qXzCs8XXBBAFhEzuMoBkvv6X5hKpKMQAoopbj9l64bUB4gmJBxB9TABxu0gkCXo70Cq2Y/kZDyBvIjVZagCRFRRjg3M34p/bbgAJTtxOdnw3jJfD2BthoYr1+z4EgadsT8L17BXmOsY5Qde163lOs+X0ziuEwFuC6kovrg4gBrLkZbfc1eusLtdYbtcGeFXMqeBVrVWTMdsXPpnN2PqSUEaWRWCKwvkv4//8X+Of20tmpgCmboLJmwwgqnaOjN+K6BGe3G0AscLSyS0wsQ3GC0iMNxKbaDQaxmbzDCCRqTqbcE3N1MNMoZXs8MW/h7Xv2e9e2HStopX9SXFYSY7WyXi1axZPuSSMc8oEeDFaMx1K3/vNauo2teWH9iYRraUYrimLJeK4qlepm9QVjnp5Z110JrgK5z6P73QzkfGtMPjvYfRdBI9UET5ZRHS2hMR0xVUNaRdNFqand6G6oPQ3/952JpIFMYFTJZ4eSOF6LpbsnWnMTVksCaRbn+y5il54ogU4mqXQ5JQ2rzQmCCArT+N78L2EjpQRmakmOlNNcrqE5MwO4nM3EDl6M6HpZqITt8FMswlJcuwGEieauPTgn0LwaaKRDD51c93kG4spulwNPdbviY25c/SaZdqkgZQ8sGXQYrbe0U16msG0gycvGy/pwjedY0pUM2JaA67da3WPrqGF6KfH9jXWKTIO4bDtcaLYTtfn9nF3v2yZvfgZeHKeS/O/juIM5gvIjr4BpnaQnKogOtVEdHIviYm9JCbrSczuID0rEG2HiWKSE7uITzYRnykheiTfaBs8Us3KWBXRiWqYyLe6r8SJ34LlLxKN+FHxT27OSBRw9/cqgJgVcTQSONxw0xHO/njrnTIpuw31IVGJl+PDNRYrOoaI6B7hRXRx1cOKbVGWUlnGAmtzjXCsGCb3kZ64g9hMA7HZSpKzVab9FUO83pE8UmHfEZlrwH/irfDUIeuWKIFywuTRzNMIrqAy559urmLdJFtecJoTTgmC7iORWkcVpbZ4LPUCgX8Z4fuTv8ZLU00Ej1TAfBEcuxlmbyQ5sZ3YaAPp0SY4vMOC+fhiHWcW38aTn/rvEH8SUhHXpdSzwB7ZbNmAkVPVCd61KMMmbaYNZ3Rebuge3TZwcicEGC91vQkLDmy50hmvXkkf3PgSCZRgqHPcy3qmx/Zn37mJxxuAE82SxBKCk1Zpxmzb7PVHOjl/5JdJzVeTmLmR+PxNxOa3E58rJj1db/FndnIvoYVCQovbSR0phekqmKqxuZTkbA2RuQr8i4WsHy/EN1tCbKqSzFgJmdlaLszW4//a39iW1NpuIncbjre6MV19LprwvAJzEV1Bbe7WdZZS+y5oFoCuKI/NeuaaLIgxRBpG6BPh9E+/5S2V1LwnqQvwgzn8RwWQQjIzDTabnJzSsYbUNQAkfrSC8NFSfAsNrJ78JXhyzNYY54TCBEgaT4v/jLEyDXK7LE11BUSbCtxyBLxCeImJAljtuXfR7X+XfpHVJz/J149+mOen9xI6Wgzz2+HYFpjZ7vzqsToYb4DRfDJT+QQWannp1Nt47KE/Bf8/u7XQlkN1gqlr1hANbWwIonxht5+eSz++2qpIGwog7p7sS/RFHr/d/UiQ9D06Ry6Tlh97aJCMXPWfmO0t9xXvEwkyT36OCyc/RPBYI/GFbYSO30R48SaSx7aTnVFscgeM324AWDu5ndhcmQHDgvHpCpIz9YTnalg7kc/ayTz8R4tIzAggFaRmarg4V8rKlz4A4ccM2LonKQ9XO6fr+UlDVBCdHaB067KQOopUr/V3bQCRiJmZ1yo9XaT3i0Z/IVZFbpccQObqYD6P9Gwt8dldJGaqScyWkzxSdtUjfrTMABKY9wDy1Ajax0TXIZJoCByvBZDNbol47ATKASr3Hc790szrZZArkT5D4uzX+N6J/86FiWYS09vJLNxIdnEr6aNlpGeqbGUjM7Wg1YbTVQSPbmflvjqee+B9ZF7+uPWINV7mfkSCugFUWQyXTFBCQWlpCYGbzPQUkWcldK9OezpQXLlx933ufrQOxgmSA5IHEr2pm77qP31BiHD4svsa9d4JvET4iy2sCiAndxJazCO8kGcAYaYURm83gAQWClk/uZXYXCnpmQr0ngDkAFJr1mP9xA6CR4vJHK0iO15OarqSpWNFnP3UO0mc+bR5C8o+ilqiwZWkb26idTNxHUAcPa7oh591+9cOEAt0AsvFCHwAACAASURBVF4a0HNQDSDy9TU7uWTNE/xz1bC4hdTRSuJHaokdKSF2tIjUXBmpo+VXNRJz5Q4gC3WsnnobPHXYNSfecE2c16FGFFcsiLp7iICe9hCFPOHMvZYja+5oK9ekb5JrrtgtcxF8T/PiQ/fgn2qG6ZuIL9xA9MR2YvMVpI7Vw9EaOFoP07eQmakjOPMGIvflc+H0W4g92g8KjqXCcppckm4AcRZDOzA5cAgoEgBPlj1gGBA8cus67R5ymkFKVOd54NHnlbJWulpAM8DbZJpnZtypV/FfQpnAH1hxIFWAHPHBE/dy+fg+Qid3ElwsIbJQQlru50wJjN1mbnZkoYDAyW3E54tMqXCkBFVDJI7WETlWR+B4CYHjRUSPlsJRxSBlpKdKWDtWxEsnbmftu4fdQigVx+aUYY5GdvOeW5h77FVK5EiU+8z/ZoCIE8p7aFLFZ8Q3ThpAVFK2Cl4M4jtWCYs3GCBiR6uIzu0kOpdH4liJA4mA8jpHYq7MCBicr2P91C/C04OWI7/iu3vh0StcLKUynWCIWBuC5Qle7rM5cNhRfXWk7lMh0kG5WW59gf8bo6Rnb4GZm2wSdP3kTgKnikicqAbFXEf2wuxe0keqic38HKkTb8Z/ooLgwx+G4DNYxzNxSmT0LkYa0c055DSj6zbzyuvU5KJLY+r6DPy6fu8ynel00uLcCvUOUBCudkculnHrXTwUXQU0ch+JRuOEoxGi6q6iOjW1Vlp5nNVP/QbLxwsILFYSPVZO5uhOmCmCsVtgcg/RhZ2ET2whPr+T9FGBxAEl7gEkuFBOcLEMeQkGkKkyMlPFBOcKeWmunkuP/A3EvmcWLCfsuv0NgdeDVw97yVkS0cVInruRn3K8RgsizrrOJwnW3Eo7/arlkj2AJNetSZdvvhxOXI+EOjJXRfjYzYTnt5GcLyZ9rPyqRvJYuQeQWnynboOne627R07IjWCeT29CJMmR9XAGJCeTzq/yAGJafJMF0mSe+i0ZktIxsjGf279Cu+I+/0k4/jY4VkDweAHLp3fiv/dm4qdK4KiAcxvZI3tIzZaRnXsTyfl/R+z4Vvwf/yV45iFYWbkCDuOsxF0mxeXvjcE5wc8hVs+lqY3usjLuE3aJuuHceXbzkhH53SmbrwqT0tq8jbkrJSDsZ3+KcPxrXg4EEpYWjWT8ZLNhL610kdS3/orLp+vwHa8nNl8FitNmS2GiCaZ3kVjIJ3piK8mFPNLz+aTnC0nOl5I4Vkd0vp7IQhXRhQrzOCw+mSknO11kivXisUouffJ9cP5+U845gOjWc5jYeJB7IXf0LInTjD9bQfwvAEiALGvE8RkjctpL5lwzkWgfusfvZW2+0gASP1ZOeK6K0PzNhBa2kFhwhBFxXvc4Vk7sSCnh+Wr8p/bC0x1uUcyGL38l4HXC4Pnem+JZ08yirATqp4FEk2WpjCtcTGsWUdYkAr7vwIPvg5O7CJ0sY+X0DgL3XU/slGKtW0hO3E5qtpH4bDGczCMy8/+ROnG9CU30kTY4q/0qvASHSaMuwAuk9VDXJR2UG7lrtDe0TZlK9zyA6PP6jM7JBV5Zl/uXACkolc3JDfUEU4pep1/1nzqze1NNccFO/bb0Q9EAnJth+aFfZvXELgeQI+UwXQGTdTBTRXp+J4njW0kubietGGUhn8R8KbH5GgNIdL6GxLEqMrOVruTkaDnM5BM7utOs0tp9byf5WDdwecPF+tcCxBFKZ2uIaD/97xoBoh/QlHzAupuIEcZMS/OK8VrXHDKA+I5Vw/EbiM9VWpYiOJ9HaH7btQFEBD1SQuRYNcFTzfB0K6TPmxrZUBgeWFyFqjfP8WqAmEBtAoho5lkefY/ejicypHNd9bJpsmqynDxD8rN/Dff/EqHT1ayf2kbw3jcSO7mV+Gw9sfFbSM40EZ0qgvsLWZ/8v0mduhHfXBmB0x+A5z/t9loxRomWHjhela7dDN7cfclF0CdeARAxW7PhueGVWogvSn1LdnNDDpyjyWsLyE8XHUejXN9oS0OnI07m1Jwv+lVWHv5Dlk7tIrpQDrM1lso1kBwpJDOfR1JJjQUHElmU2EIpUVmO+ToDSvJYFdmjFdYBhvlSOLKTxNGdxE+UEzrZSOSrfw7ZM0YH0ws/yYJ4vNywKAYIj9b/+wGiX48Ty4RYT/qNWQbnhJo56JKV+QmR+fYCsZO7SR65gfBMuQVhMd3kwg4SC8XmZsnVev2j1Fyz1Ik6/MdryHzzv9uiJk3+iiAJW5TuzK6yOJbl8LSxCb64bzNfV5ILRj9PCnPCuPlohNbn9GJqDZa/QuJz/5W1kzUkPr6D0OIbCc/dQPZkDSzuIT27l+RcvblWodNvImiZnRpiR29nae79pH74WQhdtM4dmpHOxCOkUgnDhG4jlFaoriDbyZ6oamlgZay9+Sa954ot5ZopFrgyg6wO9loguBJO4Yu7NKeyYQnNxMuGaH/FTMZGNpu1phGvCYrNb+bcVytHcXM1Ya3I04XFniHx/S7OLNYRO14Es41WxY2s6dFtJI9uIXnsJtKLWwwoscWdRI4XE1msMJAkjtWQnquCuTKYK7KJR+YLyS6UkjleRvJEGZfu/RVY/66xIpFME405axzTfSZc4bW9uZmBUj5KUGyY5ddWENdoQUzCiGcS1oI+Z0HEn1AsSkp9YONr8Ogs8ePNJKevJzpTQXxxF4lTNQTnCzyAyM0SQF7vsZjssQoyJ2os+E1/408grd1rnQCr2lMEEjDcDPoVCyJBs4kiESznkohWr9I4r6Cthwu9Zn8qsfA/D49PEPjEW/EvbiN5agepE3nEFgvJnqgnfWS3ASR0Mg/fvVtZO1VOaLEGZitIHLmdcw/8BclnP2kd/qzDeWydYGgdXzyCL6vlrqou04y+A4mu29DjTa/ruQO/nCclTLyOJBbHZAhFde+OJFpfZDeogFrd1JVlzAgkVw8Q0ctau3qxTiBqUmA9gONPHuPsqSZiJ/JhugkmG+FoPtn5LaSOaW7kJjILW8yKRI/nEzpeTOh4KZHFcmQ9MnPlFt9xbAcs7iS7WAiLVbBYAcdvZv30bbD0Vbu7dCpBQp0WBPtEBtWpWkX3qxmoC/4/CRAZCi29EFnECAFTfRuCwSDx8GXX3PnRw8QXmkhMXE9itpTMyWbS9zYRPFpCbKHYAjbFIjKzr+coUDFfSWaxitDJEmJf/U9uua2shGbPdbTyAjkUShp4KterYVRcYieKkhqmXTwUGKmvIOLVdDaQ6PxYABUfJr/631iaqyB5qhTuKyZ8bCeZE5WmBcXs4IlSVk+Vs3KqjpAYfGQrHMnn/HQTa5/9Y/jBGCz/o9sLnCTBDJyNWR5QuUDWSVviN6kYSF0y5CuZPCgF7FkOA0cOIK57pLXv9DS9aVNtkJk8A2vfIqG9XdIqQLxKgIg80jkxZ3k00+JPu6JOKZ3Ui1/k4ifuIHpyB8zsghlNFheY1UgJJPNbPIBsJ7pYSPBEsQ1ZEiVgsnPFsLATjm8nfSKP1PFi0ov1MC93fQvhU03wo4ecxUyGScmdV0ugZBrNwUrvbeZbjqWeuvCsiGmNK2+96tE1WxBrTOilUS2TIqaF427f8PQyxB6Df2kltdBAavx6kmq1eV8z3L+X4NEy4oslJBcLrmqkFmR6nUaJnCok8MhvQ+AJd/+qp/EQq5lm5W9+HCAyyR5IcsccRV9FKD3NvbX5aCDMLMOzR1i7/zfwK7g8WWL3FZ8vILOQj64zdLyW9ZMNrJ1qIHq8DI69CY69gfjCDpYWGrh477sIf+0gXPoyiG4qlMzmaqOyrKvlaTpMUn2e1FZJpR1S3RvaMGdBZEW0j4abuNVOssJTStZUiQVtDbD+T0S+P8xL2l8ktXL1ABHwFJtph9e0RSH4iFoiQG4gS4+z8vl3EjldADPVcHQXmYUiEioxOb7NgMLCNjKLO4ieKDElFzpVQPREAZn5Emc9FreRPbGFxGmXHUwu7IGFBlh8s3khsafnIbPutHJc27ElkTVRvJhbcGhu6Sb+/R8FiMmV/H1ZEYFDWw2oc7hptYtw+eMkv/6nZBY12XM96aN58EADPHALIU0aCiDH869qpBYLYEGFcaXETu1k9TO/CivfdvcvcJg/Yo1trC731QDR5JmbgZX4vwoor3j+E9DiEdw8CpWB+n5A6ls9nJ+/Df9CGdn76okc2UbmxPWkjm8lutBAaGGvAcVcjoU3wOK/IXP852w+QDHM2ifehf/Lf0/iBydh5QnH9HQuThCYZadlOhRnBLxq1Zya1A2LAa73sJ7p7KglGFQ2HHYZvsSj8OQQlz71+zzzwIcgffaaAGLOfjwqBFpdmCJRVQObcQudIfyP/4nwac2GF8HxalKL5cidSp/aTnrxBgwgSvkeLyNwqoTA6R1ET95MZqEA5nfAyTeTPn0Dkfu3E7q3jMTx22H+Fli8kezJKgL/Mux6XqmJtxoWqrIjE7e19IpBJQI5UdBjBxbRUu6F3vnfaUGkJfT9qkpVoy0pMRWvZdVHVcszn2XlsTYCj7wbTpbB9PVkj94I95fDx/cQPVZN4ngpqRMFVzXSx51Pmj1WQvz0Di4++Fay5796RUHYvcv9CFi7HgtkJTWeiyWAKA5xhHMpUWdRRLhXj80AcoDR7UtDxzQ5Jq1+8R9Z+9yHCdx3KzxYR3zuTWRP/QLpk1vMgibmbyG1WEXqxE7Sp64nffrnSJ1+I4lTNxK7txj/6V1cPvGLXLrv9wl/tRWefdAqf4mqe7sAoXuR6GkyUW6jHO1NfN6It5TCzcHFuRpypUg9C+cWWP38B7h8qonAI38AqeevHiD6cbl8aW14rmVVMes8qcXJcnYkC4lv/iXhU+Wkj26HUyU2iRo9XkH6vjxTHizkWWwRO1FJ8HQ5wfvyiNy7jezxnbCoz7yJ5H1vIPjAVgL3VxLVvNP87bBwo8nU0lfarMsMGVUmqLGGytxjqLFXPOEqEF7NSSlKDVeaYkLykzXgtXY1MZ8jnbGsogxHPCIOnQdeIBl8maUXP8sTX/gAlx/aC6eLYe4GMnM/D/fmw8ebiS/UGkASJwq4mmEAOV4Nx4qI33szZ+69lcTL/+BkW0DYAIhm+VUOc6UOX6DQXI3iEEdAN8MqojnC6VUv5bWhbXKkdmARQMICiAJmE9RlePE42X/4ANmPF8Dpf+uAcCKPrCzdXA3MVZJdLCB83434HthC7BPVhE5W2ERj/HQhiVOFRBaKiJ9qJv3J98BjnfDsKbj4bQircYRL2arzoqINSxbqXnVpnjHR01xaN6wgOp0iE3yJ1I8+ZZWwZxabid5fAI99ANJPXj1AFIOpEljzHxpErOOI1uPo+lTgmf3ufoKnakgeeyM8kE/8RD2h4/VkH1CM9iYEEHkAkRNVBE6XE7h/O5F7t5A9ngeL2+D0z5P4+C/g/8QWfA9UEznxKzD/Vpi/CY6Xcf5zH3HN4DLnXZeS/5+99wCvK7vuey+HBexEryQI9jKcrhLLlhRFUYrlOLFjRYmduDyX51jOs2PHzy+2FSdOHMe27Eh2JCeO7TiSZoYVBAESIFhBdICoLGCvYEXH7fee8sv33/scEBwNZ4ag9OaT7MtvcV/ce8655+y9/qvttdfWPimKEmn9TkZCxGrSQC4GvrICFyFI3hUgQc/qgUKNoFZkR/3x3p/5zp6nxFgJEa2nMWkG3g3w+0kPH+Rq62/Q9+bL3Nu7GjQgXyvA+4vl8GYF7H2FzNd2PBNAHGkQOcL/ew3ZXWUMv/4iXN4T2OjGQw8yYZXwJ6n7iOctQEJwSAmGAJEm0bO9Ez0CiNZsG7lu1JBWil2G/t9hatcLUFNBamcpmTdWY8D81XXwlxvwv7aO6d2rGd9bSaz6BSbf2GzylvzqCqguwH9zqZWuezbh7PsY3pGfgP4/hLuHIXnZaESbfBgkmYWKZbbVYMZPs3j3IN5L+vrr3G/+Ja7v/hRX/mIb6Vr11S+aiolzdtJNBFAMoHX00hvTJuImD2pC/aF0mL4vMfn6R0h+dSXUFJHauY2YUnH2VuK/XmCjU1/dgNUgG4jvKie1s8Q45Xy9BH9XLpm9uUxXlzG1dwupNz6K/7W/AV8thTe2cefAL8Jkn82yVgV4E8lThzikJLUDs0qjKWGouZ/3Cg6dG1FKulmPrQ7VQ4V8IR4wyzoVU5ddp/yfADPqGMMWWjgfOMNG2GoBzg0YP0i29xe5Vf0Sk3VFZGqWwe4iUDRj52rYuQHvzZdxRbu24O5eP2ealk9TvRn2V5KQ3d/4y5A6C9nbSqNDm6rIbZVE1eOFzyAnUo8a4j1sTY+GBz2xtUfpfHlbijKNKe6uPsjEYLIbzvxH4oe/l7Fdm4ntrSSzbzXe3rW4O9eTfXMLqV3Pk9i9nWT1RpLVlST3lhDfk0ty7zKy1ctwDuTi1JYQ21/JWPU27u77EMO1f5/Rpp8ideY/wJ3/CeMHIN4N0SsQGzPBEeOiyOSTuZHthZt/SLL9hxmv/zATh17l9u5XuPS1DxNv1sKtertPn32cp/9fnRbyCbIcVJrVztZLYpsgwnAz0T0/QqZ6De6BxWRqVxPbWYH7xhp4vQp2rjf84O7ahLN7A+7uKkPsqsLQ7tW4e1bj7Kkis2cD7s7n8XZugzcrcXa+yoM9Pws3DgfFHMaIE2XKi5NUiXYDYPGt5sBsJFOC0pimM/f+zo8dUYRnBiB62BAkRgJoMY11euxcQdghYg2hNGvX6MjgNQDRhu7tOKd/k9G672GqutyCozoHdufDLgFkA7z5PO7OF3F37kAdM1eAqENjr6/F27ce9pfh7NxItu5HYKIR3CtkmWQKz6h7yRITZVOniYLGasp37qQnfasu0nUFQJXFlDa1/TAJY21w7U94eOQHGa3/AGP7q5jcW0Jq7xq8vZtgz0uw+wXYtwl33xqye8rJ7Com9WYuyTdXkti5koTs75oy4nvLGNtZxv3X13D/jY2M73uFaP3HSDV+H+7Rn8A98StkW36fdMefEe38KhMtX2Hi5K8TO/rPyTZ+GK++Cu9gCfGaNdza9SpXq/8JsbN/CpnLgTnypCd8D58bgOjBh4F7xoQROyjEapK0bvUxtednSO5fS6ZuIdmDhcR2FeMKHPJLJSx3rZ8TJXe9xujuH4aLX7fzX0wHAtEnrXQgSSyFMs2CMIX5Y0E0U3wdKAMB5R1ej4d5jRgN5gTCuQFdaDaZC4ZXDxlCnSHtcRku/1eGD34vt9/YTqx6E44xr4pgT67VIm9uw3vzRZxdz+Ps3mKkhrNnPXOi3RvI7N5GRtJmf4H5reje74NbO8G5Y6I+kvCKqsxokFByCCDmed+hd97tq/BainKmPDzVtVXXCDlmYdZNkre+yoPun+PSgRe5truA2IEiOFQJ+7QKcT18bQN8bRN8XZNfWyxpnuSrG+z3e7fDni2wax3+m3Luy3Dk4O4rxzlQydiu1Yzs38BI/YuMHHmVh0df4cHhrYzUljCxbxmxNyL4e5ea4yd27uBuzQ8ycfr38cds1XSL6Hd70Hf4Xs9rQs1Wg0hzPAKIA7d6mNr706Rq1pOpW0TmUCHx3SV4byg/7dkAkt69g5E9fwuv9z9C9Kzpc5Ut0+/LjEopcBIKRFum26xR8iXcQ0Vg+PnJzxeZYX7zRpe2mmHGHwnBYa4RRnoeuSjupBbxqlTMLbzLf8ndY/+Y4dpXmTrwMun9L+Lu3o6/ewPsLYU95fi7tuLu3kF2z1Yy+zbi7t0wd9oj0+pVUm+sxd+/HKe6kLGd3w1n/hhSDw2zChgCiRk0PUPIwCbi8+SOeU/f6Fq6sCjj4afTpNJZEloKaraukFq5Rfx+Dfd7/jW3Gz/JWN1WsjUVsEtOZoExFYxW3bUZdj0PO1+ANxTnfwG+qkVmz5tiD3x9E7y5DnZXwp4y2FcK1SVk9xeTqCliqjaPsbqlTNTnEDu8AKdxHhyeD/ueg+oiMw5Te/8uifbfgtFmM4v+XqI479oPBiD6TzPzY2/RIA7c7mZq708agGTrlpA5VPw4QHYpJL5hTpTZu5XJ/a8wffJHYVg75Gr9kd3xV0ZVLJMwOLAYkJYL54qCWgUCybsCRAfp+YxtoAtYBycEiMn7MULRLuSxSxxtRXdjxEdVCmIYRvfzoOknuLhnB6N1W3GPvIpb+xLerpfwd76Av3uz6QR37zoy1etIVW80lN0ngKybGwkgNR+0E0v7c0jvX8bIzhdItfwaTN6dCecKJNZJD8KiQce8S9+8K2/Y5w9imhKdXoqEN8GoO8KIM82Yp0rAQmIC4lfg+k7iLf83o9UvMbVzMfHdEZyGlWQO5ZOpKzLmlHyV7M4qk2vkfG0L3td3wNdfwn/dkv5Of3ULib/cQPx/rYc9r8Gul3B2bia+s5LpXcXE9xXg7s+D/dLcq6H6Faj/AWj7DbhVD9mHZng10uqKZ3qpE01HWic91CAS3CbfY7iV6L4fJ1OzCad2BdlDZSR2l+FrScCbmwxPeHvWMRcSH8Xrt/Cw9ruh5/dg+oKZb1AQS+WKNNFqx143Ewh/41dbnjf3+C5MEDGDrPNNjdhAe+hPhbi1LmKmGohmozWfoOWfds2RERfJcXjQSKbvl7hT/z3c2rve2NvZui0QAuTN1/B3voK7ZyvZ6jWkasqJ799gHNTMMwFkI96+l40GcaoXk6pZzET1ZkYbfgLuXLRYN4l0EixKD1e0JeAKo0H0xzO8dLrsN8kVk+ikvpEdPMUUUSbNAiXbl7avJuBBA5z5fxk/9ho39y5j5GABD2vzeLg/l4f78pjYV0S8uoJMzQao3YovU2zvVti7w5AmIOWoOm9ux1M0aOfL8OYOM7ckXyZRXczk3gLGdxUz+vpaxt/4AG79j0L/H8Gdk6AFX1m7F59qQH3zACJPzFaqFytagCThbhPR6h8xAHHrVpE9VE5sT5kJVjwrQLL71pKuX8+DvVtIHFZ29Fdh/ApMJMzWcvIPZT2ofJMBiHwSw7wWIIGcfEcGiFgnRikLIhtINxGeIIIoiaDQWMasGNTKQaVsWDVmBv1hO5z+PA8PfZwHNZuYrN1IbH8Vmeq1ULcdf8/zsOtV/F0fxNnzItmaNaRqy4gfWE+iZiPZ/etxqqvmSBvJ7txKdvc6MvuWkqhZSrxhHXdqvheuttjZKj2AAYPmQSatsDMC4R375T1+6eE7SbxsAleFkM3MvECiPlL8TJVGAuFlR8ok8ZHpxbn9RUZO/xhjJ/8eE43fxcO6l7i/bxOj1WuJ1a3Bb6yE42tIV6/A2bfMkLdvJX51oBkOlMEBTaSthF1L8WqeI1kfYeJohAfHl3P/2HpuH/448bZfhbO74e51u1+3wKx5PUWkg4Vj7/Fh3/Yw48eZb3Rhm4elrBYjiBTRu9dAouYzFiC1BTj1FcT3lpvADLs2wt71+Puq5kTim/SBDUzv20h0/wtwSnuu1MDEPTN3qTuysA3sYIFDKiUYFv35LgqEiPHwTY2lIIw7S2uE6tKGxkIG088Gxyrdu//3SDR8L8NvbuDhntWk6jbi12+GgxvgwEb8vZsMSLzdL+PsfYF0TRWJ2jXEa6sMZfdX4eyvtAB52rZ6vdUee6pI7c0nXrOC9LE13K7+EFzYA9H4jEulaTWl/tkZ1PfQM2/LDo9/aFfraalQnKnsJAnHrs03PyqzypTCCUJd6rZQQWu2lxtAD9z8czj728Ta/hX3Gz7D7eq/yZ3qVxk9sI3pQ+tIHV5Dur6EVF0+yZpckjX5ZA8U49aVwcE1UF2Jt7+CWG0po42rGWnZzlj/p5i69C9JXP0CRLshqRBwIE7VamCVGiSz/BlfhsH0nzFd5BrbyxspqjX892tIHPh+sgc249YWBQApMwAxvocBSGUAkKdrFb1M7Nlorh3bnUviwA4499swNWhuQjETC5AgvPgNAJGn8s5WRMSIEhMKCyWt1UIWHCrXrIJpGVw/Y2qymlTd7Ah+8gZMH+b+wU+QqN1MsuZF0vt34NSsw62pxN+/FqrXQfVGQ17NetyazWRqXiB14HmSB6tIHlqDW7sW74Co6m1bt2atud7bt2vN6j1qN+HUrCVzsIL08RLuHtjCaOMvwd0ztockQJgmyR0eRm+RNqkINrbwLPyh/g75TuVJVWjBqi3lJgWaI5BYxsWzCtrwpz42zGX2c7wNYwNw6yjZ83/B9On/xFjL5xg98c+41/ApHjR8jJGGv8FYwwcZa3iNcVH9Bxk/9D1EG/4xsaM/Q6Ll10gP/BHO1Wq7tcLUdUjIaQ2KgolHdE+iQIOa33+WDtClpJ3N9SQ0fbPxpX7CAuQBXPwTotWfNP4R9atNQCG2rwRqt8C+DbB/PdSshZqqp2/3byaz60VczYMdWkaqeinj1R+Bi182m+RIu92Pxon6Lqr270xmbPqBKS3rEc3Kong3gISdFhhkOkErD2xVjRgJN246wfrwStWYAO8q7tgBJgd/gYn6DWTqCnD2b8WpfhF3/6YAHAJIJdSsgQMVeLUVZGvXk6l5idSBl0geXEvqUBlu7ZoAGCFQHm8Ftnei7G519nrky6TqqsicLOZhXQXjDZ+BK/tgUqvbNGJavz3CpDOM1k+LO73A/Jorj6jrHgHEbjFtfkyTqmLIwC80bWDs6yfDuROda8ZH32mFT3oSErfsjrfRdpg+Qvbqn+Fc+zL+tS/CtS/Atd+Ha79r6eofwd39JgeMBwMwph2SMvam9NshKAJA6Oc0vtKidjZZUcm5v3SuwZveCCkz/l7wXM4wXPg9YjUfNRVdLEAKie8vhroNVoDOAEMgeUravxktmzA+Wl0O1D5H4kAVbudPwugpM5OvqKI2/jGRaPGBOt8YQFlSbuI9AEQjpo40QFJ+ihxxrfO9h8MYWSWhqbNlQ0tAph7AZD0T5/8frh1+nujhlbgHF+PuX4e7bwf+vu341QJJJf6BEqgrwj9YhFe3Gq92A86BHWRrt5M9WIlzsNx+iNxyWgAAIABJREFUXleJ9wRyDqzmSeTWrMZTuLNmrVG1ybqNOKdKGT9cyOjBD+F1fR7GtB2b/CsFGSZJ8ZCoP4JWnRkp9wwcolOtpp2xWiy36QsxvbhnFpMaHAS40TgJINNmpl+bwqiCvNXWnjbpEYWJgEpTl6bJToEzYZfpOqopO4HvTpF1EmZDmWzaMRntM4drvPSjgeYSKFRjK8U0CcZNa1cizg0keszw8ULNFMpbA5jsFTjz6yRrP0hS0bRDFcRri+xc0EFN7lbBgSr82so5ETVKen0Z9m2DmuVwKIfsoVwmG14mPvAbJsUGd1ZIVzyuLcudJI6XfU8r8iNGus0ARPlKIzgMGzLLddQLAocy47Sr6EQP8b7Pc+Poa9xtzCd1eAkcXAZyGvfLnHoeb/8WozG8ukL8Q3kBQKrwajcFtM4C4z0ARBrmSSStRE2ZAWNszwYSBy1Aoo25jNRUMnXkH8GNg5AYMdJEjnMKpSOMkxBoxDx6vmd46XQTCg9MDRPnEEOK9JkJENif0s/ZteTKIrabKkSJMUXMGIBy6xUWNqFJreEIuU9t+F4caC9kri0MKJCmIVKuqOZfdJ7Bls4xzyetoXI/KvIQI8UkCSMs5DPpgnN76dLKhlYxa7PMOdAguj1T8Do5iN/zOdIHXzKpNn5dGfHaAkPGfzqwZk7ACAHFgXVwQDlvyu3Kx9+/FL9+KWM1hYwc/bv4V/8bZIZnKQBFZWM4fswsNdbYvNsrYkSg6SOpW0VfBJAHRnuYkpZCnVFNHkzfhPN/wp2Df4fhA4VkmwvIHsrHqyvGq8vHqy3Fq9mIe2Ajbl0x7sFc/Pp8/EPlJmTJgW1QVwEHpWLLg/dr8A8+mdzaCp5EBiC1miRcS6x6PbH6KtInS4keWcF47UpGa1+CPjltKlOpNAPVhRKDxNEaZpM79W499I7fiwMlOZTNaplY2lYRIpn+0i6PAhzKeQu9dHW4zpVWGwm09YiJss3YR8pk0DWFlpCkyeV5aofXjLaHU4DACxxRa9EZ5jSo1Y3rBFWdUXBC46qJvOngnuzl3wOPPLEHZK4pwqm0DpNmMwMQDy8zZdKO0m0/TKZ+G87+CqgtJXEw35B3sAxqy22gQcGGuVDdWji0CQ5uxtldQWpXHv6hXBIHcxmv28B002fhYYOZ95GNZarSe6rTr37ROAWq9YlPqF1u1aPGvLILi6yJpTmDQLJIa5g9z4fh0k4mjv4YD/dvJ3E4F5oEDqnIdVZLHMpDD+4cKsOpV0ivALe+HP+gkL4NDmyGulI4mA91AUgOVrwjQIxpJvPsbUiShAPrze/L6Y83VBI/WkyscQWx+kVM1FUwffKfw/B+8GQ6iv0yaJcpMYZjp7vfoXve7St1tFIVVSvLmlSKlivmod+ypL4ctyWQjFEVLJZWv2vQXG31NULWH8fxdJ1gIZRSVVRqXNmgIam0p8iohwwyxaQTQs2T9rWKzsNxHLOXeNrXdpcqxyRgKEQvsMjcnLm5Z9KgFiBBxUbFdmcAksVT5GzsKNMnvx+ncaPxQ2Vyi3mTh1aZ3DAOCiASmHMFSAXUr4H6dXj7NpLctcYIaw4Xkj5YxKiiWmd/C8aVOTBNwneY9jX6Fhi++PpdJETESixZw7bOklSmBtYkmylOrm18YsMwXM10288b2z7VsA4aC6FOam0HXs2Lxodw6nNxGueTPTKfbEMe2UOrceu24tdug9qNIMRLatQWQ22eBcrBclxFs55AzsHVPIkETmoEuk1kG9cSP1LK1KE84oeX4xxdQOpIIQ/qP0ri7Bcg8cDIApkk0pMaXLNB57t00DtDRKiQJpBmUB+K9F5kxH3wffDeJNAFzKnTgnXT0jTqcwk0Mbg2qElnYmTMfoCPrqfUEOEqBJ/RFvoj9CNVFE9FGHx5M3ESZvmrZmVmfFOrNnQ74a3q/Dm+LEDSpPwk2cCn0z2ZxVyJh/j3DzF65JM4R6qMteDVFBpwpBpW4R4shEMSls8AEAMwCdtSOLQDr+5F3AProaES6guJ7i8n3vSPyF79Y9zUJZR4orCvFLIBhgSQ1Q5P7IGIzaGRhLOTPDpZp5n13Or41Bjc3k+i++eZOPZxEkd34BxbS7ZhJXHlPx3YbgDi1FXh1OeRPTLPkgBysAq3bpsh/+BajMSQ31BbasBlNIkBSEUAkG9sLTjKA5A83kqrULsODm3AO1ZF8shqJmuKiB/Kh5OL8JpWcL9uPZMdPwsT/Qb1inzLsJEv4D1m2AfqVoB5K5lONGp2piPDQ+wH4jJBT9pEqyHU6u9gSwIdPDv5U1wUAEQ8rbQIs2RZZpmksOeazUMdXwaU9vASqWio/tnx0TjJjDNRGXn8Kd9qGhNREUDSZgY5hK05XvehC+iPbwJALJ95qJBENp0x2knmi5ZQuLHbcGcfIw0fgSMSjJX4e0vJ1JWQqi8kVZ9PtrHImN/+IQuSp20NPx1YhX+gAI6+CMc/aMx7o5GOFuDUrWKiYQuJgZ/FGzthxkQjpa4y5rUZUv1n1wBJAIVkgaP9QdwL4N03kkn9LItapL1nTOdPNOIP/hzRxh1m0so7+SLJk5t5cHIZd48uJN24Gk8MWl+J11CGd7jAkt7rs0PrLdUL1astqUPqywKqwGuowKtfPYe2DBoKoKEEt76KbP0WMod2kG1YT/bYQjInIsRPFDJxXFt3/RkkboIW9gdCdzKrFGhxipg5ULe2v6ykNe8FHKlSda3pUYMfKylDDa3P9b06LaTg+BBJT2hnO/LhIRZ04Y2Eg/Z4oqh+X8eb/3SoOXn2OXYBmO5CNHP8o0NmfWh/8an/12+q+0yKm2z7MaLcI+3fhvR1HrT/LtOHP2yiV1RvsNGmA9vM+ERPFDB9MpdsY4kZd/GG+OBpWuor4FCZvf6hqhleM583LsM9ugSvpYzx4x/GH/oSxIcgOYyTiptKPNFE0G3BpkrihTAlySwBIU3ELJF1Y6YCRMyz4FDI0Umrds8AXPk8tH836YNL4NBiOLKZ5PGtPGjK5cGp5WSOluI3rJk7HV6NP1dqLIYjK/AbC/AaNuIdegHn0Es4DRtxji0ne2IBiWPLiB/fQbrj5+DmPltWx1UMHKKeCg1I6UokBAAJOT/g70c5ajJ1JH5tp4aHGSZ9as76DjlBYAtki8cUSYZJowW3Kit0nJHWf0288RU4pIRJpZVshwPbcQ5vYPJUIZPNeZZ/5jj+NJRDQymmrV+L3yBSmk4x/tHleMeW4p0sJN30Aqn2H4crfwqxQcjGSacgqjVVZjwVvbVmaTj/p4CKQBIxYy7fL6mCW5YFHH8YEtUw/pskWz9K9nAZ1ETMRIx1qDYYcyZ1fA1O4zOAQw9jOqcc//AcSB0RAqR+qwWIwCGt1liKc6SYeH0BiRObud/wYaZO/yKkZGrFSCWSJnPK5mfJeQ0UxGMiV3+IA2ymqhWXdnLNdux3CKPP9TFkqpj6COqTSZL+/UCI3CF2/Q8Ybf00ySNbLUBq1oGobi1OYznRk8VMNRWSlYCdy9gfLofDxXA4DznlHLIpTv7hUrwjefhH8gwPZA9vxm1+jQdHNnO/9R+Zbau1dimbcEwdcw28xlK+t5YxK/iufwqLi8x6EBN5kWUQ6uJUL/7Ir+Fe/hST9ZW4tQVQuxRqta5Ys+Mb4dB6aNxgESsmD5n9ads5do7pVAOQPPzGQryGDTgNmw1gDUAa1uEcXke8roTEkUruH67iYcvfgzt/AdnL4GgpsawMJRfK1LKZy+E8hkWMNIY0jAASgkQdJdH51y8xlqpqJh2bnJlOaxJTxfmGGOn/UUabt5I5oiiTQvqiUuM8e0cKiZ8oJnailOyROQjGkGdkQTSugsY8oz28hnV4jeV44gsB5fBa3MPbcI5v537jSm4c3Ub6whdtpRhV59dgy4IOMtY1stYHF1gUrPKIRLlPQkvtLYxAD/mwlvTQDzPduo2E0tYP7oD656FuG+zfDvuVyr4eDlaCNMiRCvy5tjr3SNmcicYyOFKCf6QE92gJ7pE1uI3r8Ru24TdsIXmwlOThIlKtFUy2v8h474/C2C6zTUJoVdn5ChsMmq1AbM0s63Q+8s6kUWRu6ci/2kDRVECcaSY91X1Ut2gtfBRGD/Kg5UNMnVqFe7QIDhdBQyE05MORlfjHckkfLzOJpe4zjL/GnSOFcCQXjq3AP1qIf3ijGXsatsDh9XBsDenGQmInyhk9uZV77Z+G+18CX0JSmylZB01NCBALEssPkYdcYJrbJlpialqNNeFc+vdEOz7CZGMp2cMb4ejLcOQDcPAFqN0OhzbbUJom/BpD5g4Z/Wnb8Py5tBVwuMoA1D2+EufECpxjZThH1hvJIYBogU5aczany0h2r+P28RdJX/lVu5FmQuvtbUc8Aomteh7+bftPjrK80TCEIa2iCNU3IV/821gRaeJ1iklGmcTsMSQOmxyAi/+esRNVJE8shKO50JgPjSuNv2gY+ViuHaejFbhH5zLu4TkCSBEcXYl/IgfxgNu4EbfhRfz6HdCwEY7bbA+/ZS2pjm3cPL6O6NA/hWSdWQVrXIxZ4XaBwwQFA20SGeE8ca7hch9nopn4lV9lsvtjTDVVkjieS+p4Hl5TKeljVcTq15I6vBZOrYNTRbhHlsOxYjhaCkfL59jq3DnSkUo4vBWOrMM7uRS3aRHuiSLco2vxjmyynzeU4B/Nh5Z8Uq0FPDhRznjH98L1P4eoainZ2sICgt3yTHudaDsHK1FCoEhaKnxpTS2p578GiGbRJ5hmTCaqOEuK9eGfE2/7hNEQcpQ5mmfp+HI4sRKOFbwNz8xx/MV75nqr8E6swj1RgHusEq9RAvwFOLwJGpfhH1sEp8rJNK/l4ckiRjtewb/xKyYbnawmZAMtEqxAFEBCikS5RUZrE9whssP/i8nuTzF2tJh4YwE0lxA/lkPs5BLGGldy9+AqJo8V4Xeuga5csicW4R2f9cAhWN5rO1dghOeFAGncYDuoaRFOkzRJEd7R1SBqLIKThWQOLyJ+dAmp1jU8PPYSMc2NxNpmOkeTXjaLQLPOsRl1GwJFALKhP4FExQCCGXGZpn9FX7LTJ0iaYnFW7I7BvX/DpMyeYyU2BG8kfAHeiZW4J1fhHRMYVsORUjgSCte5AqQIjq2C46vwjhfhHi+1dGw13pEN0LgWrYOnKRfv6ApSR1eRbClnpKmKye6PwvgfgDKOBRAZA8EiwVCLCCSRTDZqioulbuxntOVzJJtfhpMlODUqCFBEtG4BY4efY+T4Ukab8phoKSTenofTtRg6l+E15eKfKMY/XvK2LSdKzfXettV3z0LHK+HIFji6BY5X4jcV4LUuNERTnr22BuJoARxfin8yl8zJKmInX2Sy6XsYPqnatNfMwiY/JSfclkBS/ELLUdVRIoHEmlryOYI5Eb0VfYcDJJlM4rrGw5gRA0plUbE59Uk438h0FG7W4PR+F7GGCN6BFXB0HWgW/dhqOzanivCPr4Nj66zFIRAdfxYeKIETeXCyAP9EFd7JKmPteKfyMON/UsK7HATKk0vh1EqyJzcRP7WFyY4S7rV/GGInbE2xVAIv7ZqJWo23lOG0r1ysWILszS7GOn6V0caPkT2sybw8qF8LTdugZQ20l5LtLCPRUUa8o5hE6yrSzYtwTi0OAFJowXHiG1uBjZOS4m/X6rOA3glIOubtvj8hLbEBjm6CYxvwmkpw2ubjtEfwmxcbyYGcRCPFVhnV7h3bSPbEVuLN6xhp+Qjc/e8Q7wHtXJuJ4SY8m2wYzLiGGkSmlmETASLAyaPZtxne+Y57k1UF+VkvgSOVSpHJZMxkm9GsWpE60YV76TdJd1bhHo9Ag6JLSknaEoxNqVmK4J3YYAEipj1WBCeeYfwNXxSYa4QA0XIHp2UlfstCaM6xADy22gLpRAH+8U2kT1UR75rHWOdquP3FIMviAXhR9LwSjgKIwB9h/C7x7v/O2PHvMrOOnFA4Ls+mghypgJPr4dQGaN0E7evw2yugvQRayuCUkFlkpIPfNJdW54RUErx/760BzbEqK6mOS3qU4LTm4LRF8Noi0LwQjss5FEgqbMqDgHSykmzLSqbaS7nb8Qmc2/8Zkn2QiJuisr7WUwWJd9bcUgasXU6qyKARnValfMdrkFnYMFojnU4jraLWJHuqH5K38O/9IdG+v028qQhOSVrLzKqAxu2g9KSmSpymciPlpe2NVDcAeYbxF0CkgY6X4mvsT5WSPbXaZJmLB/zWCDTJTy4H45NughOr8ZpXku6MMNWZS7Tvx+HO18HrM9u5+dkJ4umMqYov6yHC6HmiLb/JyPHV0LMYWjT5kgtHFkDjAlAlioPFpOvLSB8pIXN8hbH3OVYGRyugqdR2hjpkTlQMp+ZI+u3ja+DYWjhRgd9cjNu6Cqd9MW5HBNrmQdMSOFYIjZq3kaapNOrXb1tAqnMxd04UMNb1d+HG/4DoDSs2BBAzy6p1G9q9V6T3wfJSM2nyV8PECgGi7dlkakmDhCaWieIlJ01KefLSZxlrryB6LA9OFUCr+KECjm6DYy/gN63DbS6DpjI4KbOq2FLTHMdePKPxP1Fh6VQxbnMxmZZSMq35OO0L8NvnQctiOKmMi+ct6bdbF+F2Rkh0LmOi9WOkz/0bmHzd1ghQqkx6ykasDUCifcT6/h13W9cy0TmPZPNyaFkCp8VgEWhZhH8qzyDUbcrHPbUQ9+RCM/egORBOlUFLwZOpOR/eieYEqgCM6tzjZdb8kt3ZnIfTIhVbGHRQBP/UAjiuUKMiGpIgCj4shrbncDqeI9a8gPGmtaS7fgzu7oLkHe1nYBas+07UOO42/UCTR3Yy0ZhYf0V8kLCwtVqB5LGXKqmPt8Dl32SyZzvjrYtJyKIwTFgIHRqbdXBiI37TGiPANEacWgUnCi3jPuv4Gx9WwKvAbS4l07aCdMcy0vKTW/OgdbH9vaObQdSkiOYi/LbnSLevINYmf+Rv41xWTed9wEV8Z8xsnaDxjpC+Cnd3M33lJ5m88Alig99Dov9lol1FjJzIIdO5AbdtB27LDvzW9fgdi6H9OWjRepBiKykkLZ5E7wQefddcNHc6Jd9GpOvkGpC6LavJtpbjti/Ha8+xdujJXDhaaemkgCSALIT2+dC6ALelkHTLh/H6fgZG3gDnnNlsJjl536TFS3M8CveGVpU+lX0upHznvkKN8VZH3Txx5oZZJ+/2fpqp9nyi7QtJN5eQPVlmwdBRbiKhSHM0yyzPh9YlVgCfyoemkrmPvfjmlPwXgbAcmlbjtZSQ6VxKunM5mbYysq1l0Cq+yIWT5ZbEJ61LoXUlbls+XncZE80bmOz6Xrj/e+B2KlZt19QYJ13FAlLjuNNncSaPQ3QPzv0vMXH+3zLe86+Y6vgcsbafJtn8z0m3fJxM1woy3REyHRGy0jDtK6BVKvUJpE55Ij0jQKSZZO/KzhRYTpXjtpTbzmkrwW0rwD+10n5/cjmcXAUGIOqkJQFp0EqMepY0yV74LEx+GZIDZKeUOvFoD0aBRGkpZnvrmXV8Asp37is0q6RBwpc+m5qaInG/iWj3D+C0VxqJnWpbbswbmTlO8xoj0WlbDm3q65XQovcSSvOtQNOYPZOAVPBGjC8AFuC15ZLpyCfZWUC6XQCRv1EZgFPAXGjvwQjmcgMoTq8g1Z7PWPsHmTr3L2F8f1CIW7V9IaI1I1pPnFbRQVMoSTVWr4FzEbyLMNIM9w7BzTfg8m+RufgPiJ17kYnezYx3bCTd+jJOy6s4LS/itm7DbduI37YaX458Rz62g4TYgNRJLeqs3MD0eitIZLIV4bWtxGtbjteabx7EPuhqa8pJKrQKeJIOS+CUACJptBq3tYxsW4mRHnrvyR4WSKQ1TikmLpNPYNY9rIQuResK8ZoWku5YQqy3itjQ98OD/2krhodhLNXaDXK3lOCoORNVC58Jos947gJMSKEdFrLWW9vw+0fM99YjvvFvHTv7+sERb7F+vvG8Wcfp2JCMFgwnPYPU/mBdhCZHwyW11gOTxozhJm8yMdzC5MUv8PDoFtzWZdBZQKYtj3RrLm5HCdmWSpInCvDalwSaejm0CiALLZnx19iHJLBIy4iK8FpFBUbKu225hvnFC9ac17EKEgXgMD7PcmM1CCDpjiIyhgfK8Zqr7LHmd2X5iBdK4FQVXrP4NA+vp5ypzud50PV9pG79PtBrsibcdErp7rM6y3SyOkFBrqQtja9dUc1S0HG0nx3eADgdkGmD1Ammu3+HVOevkez4HIn2Hyba/kmibS8Saysn0bYMvzcHTs+DLmkbRZbmwcn5cCwHjkj654LUrbRBSwlKCXAUVj49n2xPhGRbDpm2cpyWl8k0bSPVtIx0i663AnpLwNiZYvhi/NZSQ15bMV6bfe+3hWZgqOH0d/CZAhIKNhzXdZZC53y8vgWk+0pJnP4E453K/r0KsYytjKCZI8OeWr6kPKQkqewkjjONp6WyBjBxfD+K40Uxc0yGmR9lAAdsOhMrDhfoaKJy9lA89kf4hTHnwvFREFKMHVxKuAmPe1Ib4nF2kCGtPdqDfdqVm5SeJp0cIZoeYcofNxOBYyZbz8wMmN3DmK4hdeHfMNr0UTJtimzm4bcJDCEVojHw2wrx21fhy8poywtoFbSJgvFoXmV8RxMRba7Ab60w45/pLCTVvYrk6aUkexaTOr2ITPdSnI5VZmy9lg34LeusCdW2xJjTbscSAxIDKAnW1gBIBnShQC2E5lJoroDmdbintnGvbjX3jlYy2vMhRs78E+LDvwsZ1TEYeRuAPKFzTeKjQp8qjZX2SKdc/MQUZJWuMQSJdhivgTtfxr/6a2Qv/ATZc/+QB03PM9a0jqlTFSSay0g3y4kuNSFlMTRdq/E6y030IXGqmERTKanWPLI9C/AGFpLpWoXftQE6PoDb+hLptkJSbYtwupbC6VVmYASCuRD6fWO/llrV27nQBid6l+L2rCfd9Sm48QV4cBQS9yFjCiqZ6Q8rQlwyaKGQdiHRbubSKAk8L2FLkqrqvUkPDpg3QIedtbf1qbRG3G4oGWiRsP9DZp7dmu/0gUAi1faWRV76Ssc8FkV4y0rJ8HohSJJBKSEJQbN92SiOPxqUBkrywFHdSNcsozXrPKL7cK/8IomeD5HUOLYXBmNQ8NStAY20T+uKwDKwgi3bUUyqq4Dk6RUGHMmeBaR6FpDpXozTucIAT/NzApPAkO1YOousJnHb5WMoB68YTpdCt3itFDpk2ZSYuT3a1kLXazjN20l2ljPds5oH3duZOP9ZmPg6+L08vj/II/H2De9kXphCG9peN+uS0QaJaS3JUpEBbUg/aXdl1TpqLdNNaOuvq7ao2d1quPUVvKufJzX0fxEd/D7G+j7CWM8Oome2MN27jrG29Yw2lTHZstSEX+nLh4FKaFday0Y4vRm6NuO0bTKZuYnOJWROL8KXGddeNDdqk6ottoMjqdaeCx1F0FmB172WdO9Wov0fxLn+4zD9v8A7axfOyMIxeTvSIirZM2FaUw5ADKjvQ69erfg5OEcdqzRqFQ5QqoYFR3hAwMxvZeKQmdXqOuHfBhCBRjG5YdIooU04u9Uxb0M6X1pR92h8qmEy3CLBXVOlS1eQTDC/qe2nJ0+QvfILTHa/xlR7LunORfid6rP8OZLOlXmWAxJOXTn4nctxO3NxO/Mtda00n9G5HDpWgpjemNgy11YabZPqqCDVWUqqU8DKM0I1073MaJx01xIe0VLrQ3etItOVZzQe/VXQVwI9EdLdEaY7lzLR/Tzu5Z+H+J+/O0AEjJBCgGgPakfFBVzHbHWVcFLGoTECVuMQMofpYY2N6uA8hPQgJI5A9HX86S/jTf4W0Zs/TvTqP2bq/A8x2fdJpk9vJdFdQbZD4FhrHT1FyLoK8bvKcNu3km7fQrxbEmaZ6UTaCwOAzKHVtY26z8NeZzV0VOJ1VZDpLWaqbwWT59YRv/YDMPoFSPWCmEUZJ1qVSNqYIZOoBI9vhIhhKPVByKN6L0YTQwZvw69kYs0s6Q21TQgQtSEYZrch0+ozk1EcACNYjz5jIpu5YJliYUJIYDrPmNCPcmm0Pj/JQ6a4yzSjpsCBWQKh8Uw8hFvVpPp/gWjnh4h2FpLuysE5LYZeOUdw5Btw6Xy/axm+wBEAxABOQk8CrK3IRkhbQodeJpL8xxzr/LeXmCCB07Eap6MCp7MEp6sQpyufbHcusZYc4go/ty0h2b6UlELAinLJMjGgWwYdEdxmmfMR3IEI8Z4iJrv+Jt7Fn353gMzUGnjrxlOqwOFbASTrW6TiZSFpd1UzZZ/FgEemmeeEeymo5pO83jH8dDskD8LUXnjwP+Dar+Ke/Qzprg+Sbi0j0zwfpyVCtjWC07aEbMdqMt2bSfZUkTxdjttVbBxEOiX5C3jqVuAyWiiUgrpeMV5XAU7PCtKDOUwN5jAyWMjEhe8iO/xvYbIBEhNGEIj39cyBR2Aqk5jl66GgCJk5ZPqA58OPH/Mhws42TC/zaRbjP/Ze380qHxSCT79hflzw0x3J5HvkG1ng6MZ0QmCO6TIKZHqPjtaZGipzmPacvPtnuAN/n2RrCamWZZaxThfA6aDPu/JgDmT6uLsUp6sct7PcmNq0r4Y2BWPWQnOQOd6kQMoaE8o1Gl9zKc2Knsq3CfwZ+Zptpfbc1rXQKv8kIJlSum57CX5HIZ78mI4VZsrCO6G5sgiJpgjxtgjZMxGifUsZ73wJZ+gH3h0g0haimbELNEooCbWb6Zg/kwQeBD89s/zd1uNQ7SbPTN0nHEhqP/UEJGKgTUgd1XkyVUDugXcJEi0w8jrc/E9w9V8YU8fpXkCyPUKiPUK6vYRM90bSfevJ9K3F7Q6BIXA8JWlQu1daEsA6iq0W6cjF61xKtlt+0HxSg88xPfgco70FjPZ+F6kLKox9GOIjNgYc+g1GSfhknCzxRIppVZcPlnTObi13iptnuQvSBobpW/u1AAAgAElEQVTBw0iSmFwSX+wqCqW/PhdApHmCc2bODZjecLaAIAqON5GpxwMBOl3Va6TgtUZf1T5mLC79XvYc2cu/Ded/EPrXQvcC6BDlQFe+te3nAIwQTJ7ModMFZE4X48gX7VDe3xpoXWNzAAWQpg3QpOUVa6BZ5rAm/wSMFdAus2uJpfaV0C5zW2NYZq/TVgkdG6FjA7Svx2+rxGkuI3WqhGRTEfGTinZuxu3cQra3nNRgKfGBKsb7XyN16Sdg/D+/O0A0UaS4dzijOns2VR0cDqM6Vqu8tVN2nAfEuE2Cm6S4Tdps7jhhlLhklzHZxAeu3Wdc+1QYZ1bF0hRvFnomJmCyBy7+PAy9QmZgAfFuAaSMbNdG3L5S/L58/O58mCudXgH986FP6l3Om6RMkYm6eJ0RXEXe+nLw+paT6VtJ7HQhk53rifV+DIZ+Dq5+Be6dhrG7kBizRd/MRpEJprwkDzOquWs1jPJ6JArUZ9Zx12KtoPxpaD4ZgIQH6Q91jCgAkzk3+D441jr5ykLW/Iwt56/fMBQIs3AjJNO+xWrzFVTw0wYoRvkId6lpstNv4D78Wcb7N+D0F0LfGji9Gjplii4BZXP3LA8ETC50Pz35PcvIDkQMub0r8but9raCShpB0UUBpgI6ZGYvg57n8PsieL05NkLasxB65sPpoO3VeNoxpW8x9OVCfwl+/zr8/m3Q9xp+/3dB/8dxej8DF34Xd+jLcPO/woMvkb31X0nf/EtTYhfn3nsHSAgSI/Vm/acCcyILHMFFal2p45pPuYNnViuqHcFjHOkWVQT0Xc9YWcbq0vyC4QchJli4p+IYIknqG79AamgLsf4ckt0FuKfL8HuKoEca4OkHJjxHA8Tgc/j98w3QvM5S/I4AIB3zcZXL02s1TLYzD6e7FG9gLe7AJjI9zxNt+YTdj+LqmzB2DNyztqIHWnSl1XYm99HssiugCCTCgl0LHdQoDRjdcrTtWBvynV3uZ5Yror4K+svCR9saq2qkStbY2YoQVo8BIzxvFkCM/2PGSoWwo5CagOlrZO/tYfziP+Nezxrigwtx+iK2n00EqBDkd3QrfC9aMecx8HtW4PblkO1fhNe7BL9nCfSE11WrlKcV+KdzcXsKSfWWEOtbw1R/FRP9m5ns20ayZysp0entJHu2E+/ZQaz3RaZ7X2Gq50Okzv0d0kM/gHv5x+DmL8Dd34SRL8DEV2ByP4xfIHvnEkQvgn/Njp+CTupETRTO4vU5vBUyZMTOIv09Q2FUJTQVQps4XLoqyRdIOyP9BLBAaEolhYffbyZ55WeZPqsiYItwJSVkTyoPTCZWz6q5Ue9y/L7F+P05RlPQuxxkW0ubGCqC7lXQLSCWQPdq6KrE75FEkgRbSeJUFdnOT8LQ5+D+VyDeaje1N7UxrPYI/TOjZV3XhMiduEs2ljUVFNPOFBlPJULF6KpwKVJwVaVoRCqXak2gpI8xV5Ne0JpAlN0YT6asjpMuMUVrfG2HbPfsMAWmFa71M2Qczd/ESKfDQndX7bzWw68Qv/gjTPR+gMmuzcS7V5s5oWz/KpyBHDzzzHm2j06vhNPqrzn2vcasu5xsy3YTmfT7lsPZCJyP4A3KUY4w2RkhdSFC8sJSYhe3Ebv4aaYv/Sumr/4Hpm7+B6K3fpvYtf9C4voXyd7+U3iwEyZk+nZCesguqXVGzF4hpjq+CnVoBaFK+GjORwWFJbFMirYV7D7jQSEP64c9I0AEs3BCUVPy+kFbKcIgcLbpICfG2MQCi25GIAm26TXhTlUWETPYpEBpFqPydfhkGu/OHuKX/qlxmJNKpOyQk7jeDpak/BzJ712FGRxjaklNSzqWQKfs10roEkgEkIKABMjlMBCx1CtToxS371WcMz+Ie+XX4OHXINkO/k18Rs0KxaSXJOlmTd1ps/4o0BxZL0nW1NC1GyGEhctsETNVXFHFS81oPwqKGbAEGkmgC0ldpe8eeR2BSacKHdkkmeQkWWkJV7pNY3AP6IP4Trw7nyd1/h+QHqjC61sJfUUwUIHTU066v4jk2RxSZxeSHViO17fKAuNZwBEAhK4PQOfL+D0FeAPzcM5FSJ2PED0zj9GBFSRvvED85qeJ3/4lUvf/B9mxo/iJPvDOABfsgjd3GNwR+1wh0wcaYCaiqh0lAtktv8vUT9beQmHcyFS/j5tKxyr7E/Zx5Bsmld46yfSOf+suQi9Erf7WyIcOp1UP8jmkKozvEah4fWTUh0BkTpF5oOrrsrJkuz8we1joigZX02M4t/6EyfNVTPVG8HsXwflSAwwx+JxI4JA2kLlm7NWF0KsJSGmRSujURFJpICVzQJqrV8eEtNisrNQMb6p7Gan+IjLnN+Fc+wT+7Z/CHf7PkD4JqSugOSP5zDJJg6Wd5tlCFaruMCnlvgmfy9FPOtrfS7P2tuC2ClUn3/Jv0plmyomZIngJ3xZTNf2tPhUJWWHgylTxiEL6ISRuQ6KD2J3PE7v+A8TOvkqmrwp61tgMhe4VZl5C/ldqYBWxMwXEzq4ieXYx2QEJBYEkz7TSpHMhc43+EhhcA4MbcPrKiPYuY2KwgOjlV0ne+imI1sPEdZiSbyrhK6tDGla1q4JAhfrQc021+2wmTiodJZGeJpqaCjYLst0gDn0raQysHBf/2cqKj8xVlR41vRj25tO2spPtTHCYKhEqDdMGdm94E2pnkwFJeEJQh0gAiZJkmltEGTaAMbeoJxs5TfL6DzN1tojU4AI4vwA0WP3LoF8O91O2koS9hdBTDL35oL/NZ7n2MzMLq+/0uX7nORiYB/0LoHeFOTbVtYBk7zwy/RGcs/Pwzi3COZtHZnAj8b6Pw93fgIf/G6ZaIH0zcOS1eUvW1OD1ZDNJBejBxcxhZwVDoWohUspWfEibqD69FnBpjYpMMn1jB9lEGnWNUEmbCIFCh5q3eQDuZUi2wsguYtf+iPGhX+Th4HczMbiGeO9yXK0HklaQ+dS1yKTe+P0ryAzkkzhTSvxsMalzS3DOLLL9HfbXXNv+JXjdETi3DC5WkT2zgdHTVYye+QjZe78M6Xq7kWKgIhXpVoFvdVnMc4m5jonAKZ9Q2M9mfLPSMZNNkHaiyHQN657ZfrLWibpodldbGaUIn0o8CYA2amgrKz4DQGzKxONMbwAgtRXSrJvRoiO7tkKFEe6ZTTWN3hNIBKYA4eoPYVll/RNmB4yslYKxEbMpZOrazzBxMY/xM2LK56B/SQCOubSLoE+OpsKYMtnKLEgGnoPBSHBtgUHREIFQQFF0ZAX+wDK8C4twLi4ge24h7uCSQJLmQW8xfs8asn1byJ79CM7lz+Ld/XUY/7pNy3HugqJ2GqnZYi3U7eFnihQnbAVDL+0bU0m5X9o2IasJSwFJnR5eR+CwHWjb2ChM9cCD1+H2b5C69hmmL36IyaF1TA+VkDxbQuZMAU7vIrPKLq3VmFps1rsAzhZBr8zHMpz+YgMUZyAXd2AZDEg4LXjUP6Zv1D9PQQOLyZ6O4A5GyAxFSFwoIXrxk2Tu/DokjoP/wFTz1opeJ4UpF6qa0Yp6ahMgU+ZUyHgrxxsmnFENto8MCtRZYehbroFsrVC6BBZN+L2ZdFXp0W8FQGZpCv1+KOGsWpR61IYuD5BDZMwsHSB//y1jLR4xm904U2aDeFIqqHoPRvYyeePT3DuziszZlTCwdI60BAYWghzEnnLoqYRe2d5LrcMop3FwCd5gLt5AkSEGVsLgUhhcjndmCdmhRTiXFsOlfLhQCufLYKDISGLNEDunl5DoXs5EVyljp19kcvAfkrr6y3D/yza12u8B9xw4lyGj4toPQflRiRTE3EfgERNkpU60h7N8Ny17jEImjukXTS5pdZ9209I+h8mzkFQm9p/DzX9n0vhj/a8x1VNMtH8B6XMRGFK4VH2QB2dXwJnnyPZFSPdE8Pol2eXbCSRltu0rAFH/SnMsZ2z/mD5RvzwtDS6BcxHivRHGTkeIDr0AD38dsifBVdjcagWlnaeUca6ijYGbq02BTJhTfaF+MTRr7/JQwDwGFplockQUPAoA8tj3gVkq4ATHRd5G/s9i7/DsJ7UhGnU3eh8au0Jq8LL6y3L/W9+H0k+XD7SIjb9oNyYb01fJRy+TxpeZoH23tVdf7AqJe/uYvvxLJBXbVkfPhfqXW80gp3Qgx4R8zaCfmR8AYyXeQB7uQBHuQImlwQLcMyvxzi7EPbuQZJ9MkGI4VwEXVsOFcjhTjNujXKXlMFiC119CuqeIeHehSWNI9pfinFtL8uJ2Jm/8LcZv/xDT93+S9Oj/B9NfhMROSJ+ATD8412xCaHYEU9hPrcwlkXcrAFcbJPeTnfxjEg9+hanhzzJ57buZurKVyaFyEufzyJ5bhX9mFQzkQ48icuUm58wsk+4shDOFcGkV/pUcspeso5wxgQgFLpbbYEWnzivARPsG50PQTwwoaDEHOpODfz5CvC+HidOvkLn4CzDRaKteah2OapYZ5ah6AFpeoK0lFPtXHyjJUoEGmUWWFMOTESVBrKkEV6onVABv5T39Lf4LNUjY6jOT8mN5O9Aguo2QW5+21U1Ir4tC2M4Cy+z8ovAm1M7GXPi5btpcR4FRXctmz5pGE4jZuO21dJLM5D0y9w8x1vd9JAfXkRkswT2z3A7aGZlHOVYTDIpJl+KdEc0C0oDMseXQK6koiRh5pDXOKtS43IJDgAgoa9o88zvuufkGIJwXcFS1cRVRrSfpzCGl+ZUzBTBURqYnD29A70vgSjFcWW4kt8yK6b4Ik5dzGL2Sz/iVtUxce4mpqx8nevUHiV/5KWKXfxnvzn/Bu/0l3Nt/gnvrT8ne/jMyt//MhDWzw3/M9LXfYPL6zzNx7UcYu/JJRi5t4eHFlYwMqdxAhNGBCNGBiIkOcUFh1FwYkLbcAF3roXuNiQhmOpeQ6FtI9sIS3GuL8a9H4FLE9ouWLGi+qUuRw1zokwZdaISE6degj42Qmq3N1e/qx8GVVgsP6rdX2n7vk+ZaTrwvzwQI0hc/B7e/DmMXIWHNJpM5bgAi01xb1N0K9pdX1oXdnFOTnOH+KWFrLRWBJvSPQ/4OzKgQLGL1t5LhQZkzOkfFq81LR+k1l1bnhNz+1l8L7CZdOryp2a0On/23uQd7YzN3Hn4fPokBXPgAw8Qe/jfunP8h7g++SOJSMdmhYGCv5sKFIvzBfOM0Z8+tIHt+Kd65pXBGJlng0A/KtJBJtQDOzTetf24R/tml+GeXPUbezN9Lsccstt+fWYEvmvl+1nmPfW7Ps781D//8PJyh+WSHFpAdWkxmaBmZ83lkzhaTOVNBZnAdqcGNpAY3kxrcSmpgu6HkYNhuJXlmM/Gz60meW0PyfCmpoVVkhhaTvTCf7IUIzoUI3lDESGr7fIvs8+q5B6VRVoHpg2Wmb7xzi3HOL8I7vwDf9IeOXwyDmlQNyPTXYnO8NJP61j+72PbhwCIQnVEfLsYZWk7qfB6pc6U4Z8vgrLSVwsjSTCXcaf0oYxd/Bab2WG2onXy1D6MsoWCLd+tiaApAQQlpEjnRllcfTarKyVbQSK2lGR6a4WvDYI/+C3krbB99M4OcZ5wHeeyK78Mfk2TSzTy8818YufpZoldfJHrWJpxxYQWcL8Y7U0j2bKFhnNTQMpzzGkgNYuC3nBWTv090TqCcZ+xw2eJzITGxf07MPFeyjDyXPpDAyJ7LI3s23wgII2gGF4EijGd1P4vInFtC4twK4mfzyZ4rsZpUWqx/vqm1Nn7mXzI9/HVIdmH3qlG6keVPhaslLi1ANBckTaKoXbBNYCBfv5WM920OEEmSGzjJg2Qe/g7pW59lerCS6e75NtJyvgD3bBHOuRLSQ/lkLyzHHVpow8NnckB0LgfOv1+ke3nOzB7PrdW5CnfrOnOlZ3j2czJp7YSiWbszKA2x/LE+VUg4O5hDdnAhvo6/WAgX8412muqvxB/5HfyYIlZK9VAoOmXTxGUkBJFNa5/IDxE47D6Nssol+L/Vr29zgMjXUSRMdmsjjH2BzOVPm5VhSUWmhlbinCsykit7vojs0ErcC2IoSevn4OxCOCfGegYmeaZz9du6HzH6XCiYC5ozOJ7x2c/Jj5A2CEAyWIF7Lh/3/DLcoRz8oQX4Z+bBWaslvbPPkT6zgtT5CjJXXiB54zPgVQPnbA6UP24mVJ1k1gaRAh9aGiPc0MZuSRGA4/8HhHybA0TLSpQHFofMXRvWfPgfyVz4W8QHC3AuLsW5UIBzvtSQO7QKLs7DOqvz4Px8OL8ILryPJI32rHRhIcyZnuHZBUwBQNGsc4vh/AqcoWIyFwpxLy6Ciza3yrSXrB80PbiUiaEPkb3972HyEDAUVBEZA2cSJxkjGU+YZd2O6xt/QuB45HAHfu3b+g3ffH3ybQ0Q9VEyyBq3AbQHkKiD4Z8mfqGM9JX5ZC6uMoMmkLhDeXbQNHBy5gUQMef7CpBFAUDm2M4ZGCGongEgQ4H/pPki9eeF+TgXVpG5mGcBoiiY+vpqBK5F8C+tJHbhg8Su/zKMd9mJUi3ZVl6eF8XNREmnEqb2bzqbQbvnWsfbAsR6JIFtFcaDvsVa5NseIJkw3V7uiJYxpgfgweeJXqkgcT1C5tJynKFC3POleOdnA2TeI8n9zEwWMtvTtrOYc2ixBerTtu/bvetZA2AYcFgweJciuJfmmwwDo60FjqsREzb2br8Gd34fJs7a1BpZyCYY5Zg9610nY0ubellcX4ZVOF0QtoH20DlyQtT+NUCerDbVN4pwKK0lE7VFOUwh5YdfYPL6ahK3IqQvLzKawztXDAKIBtVoEJk2YsqnZepv9vHSHIvnRkbzfbPv52muJ60R9Kf69DGah3dpHt61CPGLS5m8uJXM8Ocg3mUHLVwVocltOeOhoDM8H4ZsBQyFtNTOWkH51wB5MihmfxMCRGsjTDxQc5XOBKS/xtjtSkavRshcXYB3cSWcL7STZBpEqf4LS/AvrMC/mIN/acH7RIvs719cgj8n0r0vep/uXX22CG4sw72cg2f8uQVwcQlcXoh/KULsfITJC0VMX/t7ZLQvYKzT1uCSz+hlTI0CZTYrSjV7rtgu5HoLOKQuNOAijXdIsxniW/D+29zEUjzcIe27NpVewsZVbtJeRu+s5+H1COnr8+CSIlrSHsvh0jy4PM/Yw97lVfiXF5sB1aC+L3Rpsb2HS0ufukU5YJdz3p/7vrwQ70oO8cs5pK6thGv5cKUALmqCdjnOhRyiQ2vI3v8XpB9+hezEOUgZO9gWkvAn8bLKywusgEDGWQyEk8WB5ngrOEKQqP0Wv77NAaKYuGZYlbvl4GeVyKe850OM3N3G6M0QIDKllsIFMVMErszDubyK7JU8vCuL8a/Mf59oocl9EqMJqE/bKm/Kv6JrPAvN/dmz1+YzdT1CbHgp7s0KuFJhtfKFUvwrH8e9qYrpbZAew/fs/IWiUWaxljuFrw2LvgEgdgnFjGllEfNIY4RFML7FwAgv/20NEClnVTPMKn6ueirelF0qljnF6L0PMHYrggaRSznW11CI93IE/6qK2q8kbQAiJps7kzzrud5VK4kFjqemq88CDJ37bM+tvo3fWcz0neVMX80jeqGY5MX1cOP7YOSLEDsNWWUdWjNK/qIAYUBiFnBoCfAjDTLznQ1JfqPfEVhZOk6LpRT+tcgJ2fmb336bA8TD99K4qupoikQoy1MiqZ+Je59i4lYOzjXZxEpHj1jtEURU0tdXkL6+yiTmce053he6ugD/2iLcOZLO5apowbPRHJ9ffZu6XcbEjXxGrkcYv1lK+s6/gIk3ID5uI1WBby1RpjWAqqWmajZmnLQc4zGAyDmXnSxnUl58wP86XuCYmVlXzq6WI2vlvQb8W/f69gaIOtBRRpvWWA/bNdYSKpnrTN39DNO3i3CursLY6gLIFRuPd29ESN1YQuLmCsOc7ws4rj2Hb2gB7vUFFiRzaN8vcKjP3KsrSF5ea9Lqo3eex5n6p5B5E3zt92gDUI7WcHguGT9lSNEqJSEaVaLIVYAVtdb8CjPDNQEcHKcvtRRmJjdLK/VjM6b1tw4ez1zV5Ft5a+/h2gKIBI4q0JlU6LvBwDwgevcnid3egHe1CC4vseBQTP5GBO9WhMRwDonbS3FvLAQ58u8LzYfr8/FuiBbOoV0I156Rruse5vj8V3Nxr+yAG/8ERv4Akk1mrYakuoYl5vlmWWzKyZr14mZfu0BDSFPITJL8F+m91gLNFMkzi5ZmAWSWtlFJpYTZy16ZvZKI37rXt78Gyfi2IIJZTKNUaYFmgqm7v8T07VfIXiuxTBTM5gog7q0IyeH5hrybFjT6/BGJYcQ4Ak9Ilpm5MW/WcW85R98F5z1i+Hl4TzxH54fnBNc357/H999wXd3bbGafHwBoCVxdakkmp55p9rla+xHchwVreB2dn2OvoWMM5cD1HNRv2WtFMPxD8FD+Rp9d3aiSs6aMaYqEaoR4abP2/tEiJIFAzrnW1atVnlWoPULzSu1jDsfbahBpkb82sd5ROEj2hLULg8VVEiiph7jRL5Gd+H4SwyW4dyJwOwK3xAg5cHMZ7vB8nLsR0Hez6XYE//ZC/FvLcG8tnyH97d/KgdvPwXDE0h0da8m9GcG58RzpazmkruaQuLqI+JUFZG8vIDs8D2c4gnMngjdsSeeZ+7n5HNycPze6FcG/E8G9Z1u9N/cWXts8rwCRC0ZQlMD1fNBz6BkeO34+7u0cnGHRQtzh5+zz3iiAaysMOPxrEdwruTg38snciTA9vB7YbXaHxZ0GN2N3AJgpMiGtELzMG9lMoc7Qd/Z7e0w4saHv9X7Wef+nvXMNte266vg65+xzzj7Pe+7Nuff03pt3G1objG1ME7W9IpVQCTVSqpZUKkZMiSik2pp+CKgfzAcpVBFbNdoiFB/UQCBSIaCxRaGlEUKj1UaTe85+v9+P9V4/+c+51r773ua2+ZDmZF+yYbDW2WudNedce/zHGHOOMcfQDenqlT21pSNMVpP5e7P/eQ2Pi61BjLpo2fyFenN6twKI1yAYfgGv+wCj8gFBykAGJIe7cLhHUlw2zDVj9pTpzffFVcssxQ3C4gaRoXUiAUf/V7IMaRhMIBOVVgxDhUeb+IfbeEebuEdr+KVlw0xBeQ4gApWYV9pLx1ekZSjM0woUrqDisgG/gB6lNGN68/xlOBRAduHidRYcR1skxRU79jmA6DuBIyht4JUFEgsYLp6Bl09YgBw6hC/t4R+ewa+t06v+KPCNWVraLD3t/PE15NVjedSCA0RBbG0SJfkUQLIZn9cmGn+JsP8rDCoHeLWUoSVZ5wEiUBxtw+EpC5qjbZLChmEgaZ2M6SSZDSikPTIm1bmeV1mGSg7Ka1DKQ3ELSlvEpTxROWe0lJ5zuXRfsYx7tGo1wByDZ22a9tUHUUm0bLSeQJqRwBsW9ggLu/Z6phHNuASOdduOjuqrKAW3GY/uS7/XM/W8oLx+OUAO94y3XCBOjhyiQwmNMwT1c4waHzTlyuYBceX5sXD1a9jowgNEkaAJw0srI9IickxNvkwyfJhh5Rx+PZXyxSU4OgFHJ43Ej8s5OHwLXLzJHgsnoZg31y4zu8pLICqtXKJiDor6O2OyHJRWobgOZdEqSWWZuOIQV+fMOPWhoBCNbShsEpdXCGvWTAqrVx6XiCpLhFUdVwirK5cdo3KeqLBvSIBUW7N+q39qR6Q2pS10vWr7Y8ZeEqgtaOLSGmF5E7+Sx6+uEVbWDMiTw004WjX3xUUHX1TZJ27eRdB+xGQ3VMK7q9FryKvH8qhrACADU/4sVLmRTIOoypVqjkw/ybBy/rsBUjhBXF5CTBEfnic+vIm4cNL8PQNCYQcO9y1wdCxsg5hODJWZJjqXmXSZqSTQrBmQJBULkkQAEXOKStI8q1DYgKNdKJwkKl5HXLiOqHiSuHiCqLRDLO90OY8YWeBQfzXXENiiWnrUuUBS2jb3azy2jysWFCnzZyDIwGPGXs4ZcGZjEcAMQKpr+FVpPgFkjeRIgmDJ9F3j8CoOXv0MtD8Cg88DxauCQ6BZ9M+CA0TqYmQAIoetqvVaM0t5tP4VvN9nVLmZoGElp2XOXSjuoh87ruaIi6dJCgck5S0SmUoymaQBCqfg6ACOzkJhH4oblrkFDgMQSdVV4osOiZaOj7R8rIntBklph6SyTVzbJK6vkEiDZSSwGI0iIOVtO4XTcHTatqN2i3tQ3IGS2pQ2kgknU+6SFphpA2mfWo64sk5S3ki119y92f9Iu5W27DjLW8QpeM1zypqPbRFVdghqqwS1HFFV4F414zL3qJ26w7Tu4LbOQ+dRGP9zmsl/0WFw9f4vPEC0dq4sF0EYm4TEBiDBANxvgv8ZhtVb8RsOSU2MLem9C5LQ6Q8e1WQGrVqGETBKYsydOQYVYFKmFrPpXPMMAefoPMHhJkEhT1DcJSjvE1UPiOpvIWyeIWrv4TVX8VoOQdtS3HKImw5JMwWNeZ6kdEoFmWnSMK90FHA1n0iPpRxx3TECQMxrxqL5lsYqyjSXQCYAHp0nKZwnKe4bQOkdWNBdDSA5Qk329ZyWQ9R0GDUdJu3boPtn4B7aBGxX56+Fv7LgALGhCcqDpGTP0iIGICai97/A/3OGldssQIzUFhMKAFsWIA2HSFRPtUl5awYgM58QIPR/lzGbJvGnSApniUoHTKs7TGu7TGvXM629E7f2Ptz6B/DqP4vfuJ9h6f0My+9lXHkX49oPMandilu/Aa9+lrB+ykr8kky3V0tWc2VzHb+2wbSxjltfNeOUttSYYgkFM/fSsvSOAXRydKMBiLRmXNkgrl7SNJdpkPqy1SBVh1CTeD2n5eA3Hfoth3H73baisetjizIuPA6uOoCFB4h2nYmU7yJK4nQeotpuSjL2JOPq7YQtB2RmVVahvA2VzRlAdM1I9NoqSXUTypv2PgFDjNFO/7eYJzw8QVg4RVw5CXrZgg4AAAtoSURBVO1NktEq3bbDoJ/HG78L/Acg/AxET0P0gu1DUAXvJZh+E8ZfgcET0H2cpPVJks4DJK0fgc710DpH3NwnrO/h17bwqmu41WW8umx+B6/x3TSt5Um6dzOu30W3dJpBZQWv48DQgZEDfS3drpKUdqFyGqpnoLZPXN4lKGziyjko8NeXobpDUjtB1FgjauZIGjk7fr23ngNdh2HNodtycMf3wvg/bKzVnMviqly2wBcWHCByKcndZGM7zTZNpThX2Luv3K1fYVK7IwWImECTZwEkTyIp23TMtVAmT32JpLYC1ZRpxDiipv7OQ/mAuHQ9fuU00/oWk7ZDv+vQ7p+mN76bIHwQ+CPgqxBXLfMoM2ZGZiuEijMewvQ5m5zZ+xuYPgbex8F9CNwHYfIxkvFHiEcfJhrdz7DxfobNCwxb72XY+rHLqf4zMHiUuPfbBIMP4vVvZ9o9ybTtGLNO4Pc1udcSdG0DGinVNohKm4SFHMjUayxBfZu4sUvcXCdurUBTpPELaBlAlui0tnHHD8BEmeJTjb3AAPh+XV98gJgANoFEOkSe3DSpq+KzkmcZ1d5NIKmqH7y6DtUtqK0bE0TAkLQUGUbINI00R00T9h2oHED5RpLqzfiNc0xaJ+h3lul2HarNk/T6v8Zw9DmI/xH4lg2YNHn6bbIVw0RZaQOFsmq/ihIuKwI5Ua5ZBVkeXkEXbTkwlQTzXgTvf9KjzkWqN/J/4L4I4+fBfxaSv4LwEdz+Bbr1M3SqDoOqw7TmmFU8CQOjDVtL0MpBbQ2qcyCob84BRPdYs8ocew5xZ4lBbYtu++1Mp58CL417e1ODfD+MHd/1zDeo1V3l3FNu1kRhpAohVV3j+BvG/Ai7S9CUFM1DbRsa6ySadIrqW0SSrGIgMYWOAkhVK0L7UD5PVLkBt3aGcWufQfeAweAWBoMfptv7RfzpU/iT75AombRSYipUNTH1XAh8peyPbck1PyJRhgl5+udJ4eAaQLZMHdnVOFVDMqtyuq65laK/rzjOIgdUU8/klnoK3MeYdO6jU7uJTjXHpOnga5wyFWe0DNIS+jsDQiNP3Ngmaq0TtZcu3avrPYewk6PfOE2v89P43mchTOv4Lf5K7vdk4IXWIBlAxDeKH1X5MlOlT3HRRv0/z6R+D1FvGdrr1sSo70Azbxggbmsyu2fImFIZQBrSNntQvc7Y5V4jz6jt0Ovs0u/dyWT0q0STJ0imXzf17xJhMiviog2NgUK8lSAwxo083GiKH3oEQWTvU6oi5eFWOq8AVB5bKf7NMbT15ZXuX3Xm9RyVZ1ZMplFEc0cpJBNBYMKYhKgaRP8O7ufwex9l1Lwdt3OCsOsQSYvOAJKedx3Ivm+tkbS2iNt54s6y/V7XDUBWCDp5hs23Mug+SBT+A9A3+zNM+9+TxRb74jUEEC33XgmQbzNt/ARxfwU6eWhuQmPbAqTjkHQEkG2o71oTbAYQzTs2jV0uDeO39ph0rmc6uIA3fgS8v4egOB93Zwq8yKpTni5PlY5izYtk9qkYp6JOxzY1v8K2Vc5R6wjTtE55Visv3RCUKhS7IGeUi4LzsgC9YC7LoA0X97VXQgGwZhVPCRFeAO8JGD5E3LsLBmegl7NMn40xM7kEGoGklSNpbbwyQDrrBJ0NRq07mPQfI4m+ZnZwqp/XuALJsrsvJsqNBpltotEOs4HdU6BfTltEoheZNt9HPJREzENrywKktW5XZnoOSVvmRg5km18pYVs5osZJovZ7oKeUNV8A7zkIW2memjTwVB2RRSdGjxLCWKtq6o9KgGXFgurEtEzcWKLiLaaSZEzsTYi9MYk/MWSrsGpvvW/J7ItIH64G5klGZTRmjGtSH6kP1nyTSvoOeE9C52EY/DgMdu2YBRBjQq5BffWSpmivQnuDpLNB0l0xq1ZaudJ7STqb+J0dxq27mfY/C8l/ktAzuzmUSf1a/iy+BplJ3WzDjSoHmdJUEB/Rb96HPxRzbEB7DVoyr1ZIeg7RwCHpWk0ibWKo6xB3HWNSuK0bGFcvEDQ/AcOnIaxdmj+oqJFKHqkSkeFKa+jZdTUFUSqZRB8vruFTMRSmZecS7Z8XghPXFoFRIRjZW7Fr615okm9SqgagIhmKo8lIdpUSganZOGIYdEyZumy/t9EiMi81B4vK0Pw76PwOcfcCQec0QTN1LMr7LoAIBKIUIFq+pnNJgFjBsYHbyzPs3sN08NeQaGGhk9by07iv3c9CA0RAMOE+EmLSGjqa8lppYQma+NGniZO7jEc7C/kwXmzXIZSvYOLA2B4j18FzHSbuHsPBXXRbvwxodep/LcOKKWecqMYyu0bnGYlZdG73O1hO1n0Z6SGXwDQ7NzPwbD/qFUcB/pXIlAHQ8oTmX5bsttW0CfVV5L4Mk7/EG/wS7vBtuON1omkOvHW8wSphb90KjqY07J4xtYKuJubWl8JwnUnPYTB8D/CMWYyIgxrE3VQ4vAmQN+wbmAEkE+RGoqcOQwZMkj8k4l6C3qaVlNmk1F3DFUCCJZg6uAP5Ndbo925hNLmPIPhd4lCZxwvWrMmePzO89YVE9fFJUJk3mulYcNgkBjo3XbQYtQARSLxvEbt/QeB+FHf6TsajLYajJcbjNYLhBnRWQaZnW9p225il8UDaJQ+DTaaDJUbjnySOv27BqoDQWJrw+Mb/ejDlYmuQbJIoZsgY+DKAjHD5Y+B+kuGe9TDL6dVeIh6fYNhdI/KW8CcO49Eqo94dTFufhsHTELycMoCYQMtOWWNiCGkDgUNHXTiejwWIACESWNJIm6yrpm/qZ5r1JX4egi8RTn7DLFO3uuuM/FWCqQMCgzzm/TUQYCZ5GG9D6xR0zuIODphMfoFQBUfN+5apl6YsOZ7hvy6tLjRAZHWIKYz5MbNabLoY41tgyjT+PAk/TzLcTwGi5cwNgv4NdJtnGfZPMRnfSOz/FISPgvcv4CoaODVPDP8LfQKD3OFyjeuo744PHJY7sjiCOXCkczKZWppIJ5RT56Wck/JdfBvcv8V1H2Ls3s3QO4fnrs8BxCEZrRBNNmC8B63T0LqFYHAn/vS3CIKL1m+TCST9CNfw55oAiPmNMvEp71qWewmPSfBF4vhjBL2zYPwh20T1A/zunXRq9zDs30sw+TgEXwSesxNQ1S9XbXIpjxkOBAyFr6RfqtHjxkfaB5mZOrUaRWDRUvDYACSmSmKqwk7AV7HxLgT/DclTwOOMph/And5KMsgZEzToOfgDB3+cIx5uQ/skNG8j6X+I2PtT/KBpMi3JspL/xzT8JkDemG9ATJFVHJrZF2ZZNPVOyzoPv0wS/Tpe+3oSrWI194kb74DRzzHuChh/AsE/QfCSRYRQEcuZEZo0v+a5BggyVQQO1dhOwaHvdX5cH7UtzWnUqLqiRGpKhdMzWUOSNHtIGA8J/AmhOzXLyoQaR9XsJ/emjxO5H4bRLdBfwu87TAcOk5FDMNIqV564+TYYPAzBk3hB2zgtzWt+EyDH9cu/unYlMTVFNdktLFpsPIYWgQzTyAR5BpJH8btvNev5tG+G3gWIPwHTL0DUtNaTvG3hBCLtb1fohkgJ6dKJuJ4vEijm6dV19Qdzl/ohJpWWM1pEDskyCSUSswwbGLdJpKGFIUE0IIx7JhvlDPj+C2ZbANMPwehGgvEy3sRhqlU+1yHpO8Stm2D0KYifwY2qxtgMtYqctvuDGdwb46kLbmLNAyQtMGEcaRlA9JL/Dfg9wt7boasVmnfA+D7gD4BnLf9rSiFGk3POVDxSphTFVknKyqxKJ+kCXSaxj1t7aGjqg6ZGM0kuR+nLJLyUAiSe1d2QqIgZEtEhSnzj+Y+MM1UhKl8F/zfBvYPE3cL3HfzQIQwdkoEins/B+DHga0ySkpmeyRUj18yxatDXAUP/Dzulf7mPSc8LAAAAAElFTkSuQmCC +``` + +### `annotation` + +NavLink 自定义资æºçš„ Kubernetes 注释。 + +### `label` + +NavLink 自定义资æºçš„ Kubernetes 标签。 + +### `sideLabel` + +出现在左侧导航æ ä¸­çš„标签。 + +### `target` + +设置链接的锚标记 (``) 的目标属性。该属性（å–决于æµè§ˆå™¨ï¼‰ç¡®å®šå®ƒæ˜¯åœ¨æ–°çª—å£ä¸­æ‰“开还是在当å‰é€‰é¡¹å¡ä¸­æ‰“开。 + +默认值为 `_self`，å³åœ¨å½“å‰é€‰é¡¹å¡ä¸Šæ‰“开链接。è¦åœ¨æ–°çª—å£æˆ–选项å¡ä¸­æ‰“开链接，请将目标设置为 `_blank`。 + +有关目标属性的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://www.w3schools.com/tags/att_a_target.asp)。 + +### `toService` + +有五个字段用于创建如下所示的 URL：`https:///k8s/clusters//k8s/namespace//service/: :/proxy/` + +例如，Monitoring æœåŠ¡çš„链接å¯ä»¥è¿™æ ·è®¾ç½®ï¼š + +- å称：`rancher-monitoring-grafana` +- 命å空间：`cattle-monitoring-system` +- 路径：`proxy/?orgId=1` +- 端å£ï¼š`"80"` +- Scheme：`http` + +需è¦æä¾› `toService` 或 `toURL` 指令中的一个。 + +### `toUrl` + +å¯ä»¥æ˜¯ä»»ä½•é“¾æŽ¥ï¼Œç”šè‡³å¯ä»¥æ˜¯é›†ç¾¤å¤–部的链接。 + +需è¦æä¾› `toService` 或 `toURL` 指令中的一个。 + +## 链接示例 + +### `toUrl` 的链接示例 + +以下 NavLink YAML 显示了将 NavLink é…置到 Grafana 仪表æ¿çš„示例： + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + name: grafana +spec: + group: "Monitoring Dashboards" + toURL: https:///api/v1/namespaces/cattle-monitoring-system/services/http:rancher-monitoring-grafana:80/proxy/?orgId=1 +``` + +添加上述 YAML 会创建 Grafana 的链接，如以下截图所示： + +![Grafana 链接截图](/img/example-grafana-link.png) + +### `toService` 的链接示例 + +以下 YAML 显示了用于链接目标的 `toService` 示例： + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + annotations: + key: annotation + labels: + key: label + name: navlinkname +spec: + description: This is a description field # Optional. + group: "group1" # å¯é€‰ã€‚如果未æ供，则显示为独立链接。 + iconSrc: data:image/jpeg;base64,[icon source string is clipped for brevity] + label: This is a label # å¯é€‰ã€‚ + sideLabel: A side label. # å¯é€‰ã€‚ + target: _blank # å¯é€‰ã€‚_blank 表示在新选项å¡æˆ–窗å£ä¸­æ‰“开链接。 + toService: # 需è¦æä¾› toService 或 #toUrl。 + name: rancher-monitoring-grafana + namespace: cattle-monitoring-system + path: proxy/?orgId=1 + port: "80" + scheme: http +``` + +添加上é¢çš„ `toService` å‚数会创建 Grafana 的链接，如以下截图所示： + +![Grafana 链接截图](/img/example-service-link.png) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md new file mode 100644 index 00000000000..818ccfc025a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md @@ -0,0 +1,57 @@ +--- +title: é…置全局默认ç§æœ‰é•œåƒä»“库 +--- + +:::note +本页介ç»äº†å®‰è£… Rancher åŽå¦‚何从 Rancher UI é…置全局默认ç§æœ‰é•œåƒä»“库。 + +有关如何在 Rancher 安装期间设置ç§æœ‰é•œåƒä»“库的说明，请å‚阅[离线安装指å—](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)。 + +::: + +ç§æœ‰é•œåƒä»“库是集群中ç§æœ‰ã€ä¸€è‡´ä¸”集中的容器镜åƒæºã€‚ä½ å¯ä»¥ä½¿ç”¨ç§æœ‰å®¹å™¨é•œåƒä»“库，在组织内共享自定义基础镜åƒã€‚ + +在 Rancher 中设置ç§æœ‰é•œåƒä»“库主è¦æœ‰ä¸¤ç§æ–¹å¼ï¼š + +* 通过全局视图中的 **Settings** 选项å¡è®¾ç½®å…¨å±€é»˜è®¤é•œåƒä»“库。 +* 在集群级别设置下的高级选项中设置ç§æœ‰é•œåƒä»“库。 + +全局默认镜åƒä»“库适用于离线环境，å¯ç”¨äºŽä¸éœ€è¦å‡­è¯çš„é•œåƒä»“库。而集群级ç§æœ‰é•œåƒä»“库用于需è¦å‡­è¯çš„ç§æœ‰é•œåƒä»“库。 + +## å°†ä¸éœ€è¦å‡­è¯çš„ç§æœ‰é•œåƒä»“库设置为默认镜åƒä»“库 + +1. 登录 Rancher 并é…置默认管ç†å‘˜å¯†ç ã€‚ +1. 选择 **☰ > 全局设置**。 +1. 转到 `system-default-registry` 并选择 **â‹® > 编辑设置**。 +1. 输入你镜åƒä»“库的主机å和端å£ï¼ˆä¾‹å¦‚ `registry.yourdomain.com:port`）。ä¸è¦åœ¨æ–‡æœ¬å‰åŠ ä¸Š `http://` 或 `https://`。 + +**结果**：Rancher 会从你的ç§æœ‰é•œåƒä»“库中拉å–系统镜åƒã€‚ + +### 带 RKE2 下游集群的命å空间ç§æœ‰é•œåƒä»“库 + +默认情况下，大多数ç§æœ‰é•œåƒä»“库应该能与 RKE2 下游集群一起工作。 + +但是，如果你å°è¯•è®¾ç½® URL æ ¼å¼ä¸º `website/subdomain:portnumber` 的命å空间ç§æœ‰é•œåƒä»“库，则需è¦æ‰§è¡Œé¢å¤–的步骤： + +1. 选择 **☰ > 集群管ç†**。 +1. 在列表中找到 RKE2 集群，然åŽç‚¹å‡» **â‹® > 编辑é…ç½®**。 +1. 从**集群é…ç½®**èœå•ä¸­ï¼Œé€‰æ‹©**é•œåƒä»“库**。 +1. 在**é•œåƒä»“库**中，选择**é…置高级 Containerd Mirror 和仓库认è¯é€‰é¡¹**选项。 +1. 在 **Mirrors** 下的文本字段中，输入**é•œåƒä»“库主机å**å’Œ **Mirror 端点**。 +1. å•å‡»**ä¿å­˜**。 +1. æ ¹æ®éœ€è¦å¯¹æ¯ä¸ªä¸‹æ¸¸ RKE2 集群é‡å¤æ“作。 + +## 创建集群时é…置使用凭è¯çš„ç§æœ‰é•œåƒä»“库 + +无法为æ¯ä¸ª Rancher é…置的集群全局设置具有授æƒè®¤è¯çš„ç§æœ‰é•œåƒä»“库。因此，如果你希望 Rancher é…置的集群从使用凭è¯çš„ç§æœ‰é•œåƒä»“库中拉å–é•œåƒï¼Œåˆ™æ¯æ¬¡åˆ›å»ºæ–°é›†ç¾¤æ—¶éƒ½å¿…须通过高级集群选项传递镜åƒä»“库凭è¯ã€‚ + +由于创建集群åŽæ— æ³•é…ç½®ç§æœ‰é•œåƒä»“库，因此你需è¦åœ¨åˆå§‹é›†ç¾¤è®¾ç½®æœŸé—´æ‰§è¡Œè¿™äº›æ­¥éª¤ã€‚ + +1. 选择 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 选择集群类型。 +1. 在**集群é…ç½®**中，转到**é•œåƒä»“库**选项å¡ï¼Œç„¶åŽé€‰æ‹©**为 Rancher 从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒ**。 +1. 输入镜åƒä»“库主机å和凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：新集群将从ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md new file mode 100644 index 00000000000..09b1cec4990 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md @@ -0,0 +1,236 @@ +--- +title: 集群和项目角色 +--- + +集群和项目角色定义集群或项目内的用户授æƒã€‚ + +è¦ç®¡ç†è¿™äº›è§’色： + +1. å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**并转到**集群**或**项目或命å空间**选项å¡ã€‚ + +### æˆå‘˜èµ„æ ¼å’Œè§’è‰²åˆ†é… + +éžç®¡ç†ç”¨æˆ·å¯ä»¥è®¿é—®çš„项目和集群由 _æˆå‘˜èµ„æ ¼_ 决定。æˆå‘˜èµ„格是根æ®è¯¥é›†ç¾¤æˆ–项目中分é…的角色而有æƒè®¿é—®ç‰¹å®šé›†ç¾¤æˆ–项目的用户列表。æ¯ä¸ªé›†ç¾¤å’Œé¡¹ç›®éƒ½åŒ…å«ä¸€ä¸ªé€‰é¡¹å¡ï¼Œå…·æœ‰é€‚当æƒé™çš„用户å¯ä»¥ä½¿ç”¨è¯¥é€‰é¡¹å¡æ¥ç®¡ç†æˆå‘˜èµ„格。 + +创建集群或项目时，Rancher 会自动将创建者分é…为`所有者`。分é…了`所有者`角色的用户å¯ä»¥åœ¨é›†ç¾¤æˆ–项目中给其他用户分é…角色。 + +:::note + +默认情况下，éžç®¡ç†å‘˜ç”¨æˆ·æ— æ³•è®¿é—®ä»»ä½•çŽ°æœ‰é¡¹ç›®/集群。具有适当æƒé™çš„用户（通常是所有者）必须显å¼åˆ†é…项目和集群æˆå‘˜èµ„格。 + +::: + +### 集群角色 + +_集群角色_ 是你å¯ä»¥åˆ†é…给用户的角色，以授予他们对集群的访问æƒé™ã€‚集群的两个主è¦è§’色分别是`所有者`å’Œ`æˆå‘˜`。 + +- **集群所有者：** + + å¯ä»¥å®Œå…¨æŽ§åˆ¶é›†ç¾¤åŠå…¶ä¸­çš„所有资æºã€‚ + +- **集群æˆå‘˜ï¼š** + + å¯ä»¥æŸ¥çœ‹å¤§å¤šæ•°é›†ç¾¤çº§åˆ«çš„资æºå¹¶åˆ›å»ºæ–°é¡¹ç›®ã€‚ + +#### 自定义集群角色 + +Rancher 支æŒå°† _自定义集群角色_ 分é…给普通用户，而ä¸æ˜¯å…¸åž‹çš„`所有者`或`æˆå‘˜`角色。这些角色å¯ä»¥æ˜¯å†…置的自定义集群角色，也å¯ä»¥æ˜¯ Rancher 管ç†å‘˜å®šä¹‰çš„角色。这些角色便于为集群内的普通用户定义更å—é™æˆ–特定的访问æƒé™ã€‚有关内置自定义集群角色的列表，请å‚阅下表。 + +#### 集群角色å‚考 + +下表列出了å¯ç”¨çš„内置自定义集群角色，以åŠé»˜è®¤çš„集群级别角色`集群所有者`å’Œ`集群æˆå‘˜`是å¦åŒ…å«è¯¥æƒé™ï¼š + +| 内置集群角色 | 所有者 | æˆå‘˜ | +| ---------------------------------- | ------------- | --------------------------------- | +| 创建项目 | ✓ | ✓ | +| 管ç†é›†ç¾¤å¤‡ä»½Â Â Â Â Â Â Â Â Â Â Â Â  | ✓ | | +| 管ç†é›†ç¾¤åº”用商店 | ✓ | | +| 管ç†é›†ç¾¤æˆå‘˜ | ✓ | | +| 管ç†èŠ‚点[（è§ä¸‹è¡¨ï¼‰](#管ç†èŠ‚点æƒé™) | ✓ | | +| 管ç†å­˜å‚¨ | ✓ | | +| 查看所有项目 | ✓ | | +| 查看集群应用商店 | ✓ | ✓ | +| 查看集群æˆå‘˜ | ✓ | ✓ | +| 查看节点 | ✓ | ✓ | + +#### 管ç†èŠ‚点æƒé™ + +下表列出了 RKE å’Œ RKE2 中`管ç†èŠ‚点`角色å¯ç”¨çš„æƒé™ï¼š + +| 管ç†èŠ‚点æƒé™ | RKE | RKE2 | +|-----------------------------|-------- |--------- | +| SSH 访问 | ✓ | ✓ | +| 删除节点 | ✓ | ✓ | +| 集群的垂直扩缩容 | ✓ | * | +\***在 RKE2 中，你必须拥有编辑集群的æƒé™æ‰èƒ½å¯¹é›†ç¾¤è¿›è¡Œåž‚直扩缩容。** +
+ +如果需è¦äº†è§£å„个集群角色如何访问 Kubernetes 资æºï¼Œåœ¨ Rancher UI 中找到这些角色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. å•å‡»**集群**选项å¡ã€‚ +1. å•å‡»è§’色的å称。表格会显示角色授æƒçš„所有æ“作和资æºã€‚ + +:::note + +在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ + +::: + +### 为集群æˆå‘˜æ供自定义集群角色 + +在管ç†å‘˜[设置自定义集群角色åŽ](custom-roles.md)，集群所有者和管ç†å‘˜å¯ä»¥å°†è¿™äº›è§’色分é…给集群æˆå‘˜ã€‚ + +è¦å°†è‡ªå®šä¹‰è§’色分é…给新的集群æˆå‘˜ï¼Œä½ å¯ä»¥ä½¿ç”¨ Rancher UI。è¦ä¿®æ”¹çŽ°æœ‰æˆå‘˜çš„æƒé™ï¼Œä½ éœ€è¦ä½¿ç”¨ Rancher API 视图。 + +è¦å°†è§’色分é…给新的集群æˆå‘˜ï¼š + + + + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°†è§’色分é…ç»™æˆå‘˜çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群æˆå‘˜**。 +1. å•å‡»**添加**。 +1. 在**集群æƒé™**中，选择è¦åˆ†é…ç»™æˆå‘˜çš„自定义集群角色。 +1. å•å‡»**创建**。 + + + + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°†è§’色分é…ç»™æˆå‘˜çš„集群，然åŽå•å‡» **Explore**。 +1. 点击**集群 > 集群æˆå‘˜**。 +1. å•å‡»**添加**。 +1. 在**集群æƒé™**中，选择è¦åˆ†é…ç»™æˆå‘˜çš„自定义集群角色。 +1. å•å‡»**创建**。 + + + + +**结果**：æˆå‘˜å…·æœ‰æ‰€åˆ†é…的角色。 + +è¦å°†è‡ªå®šä¹‰è§’色分é…给现有集群æˆå‘˜ï¼š + +1. å•å‡» **☰ > 用户 & 认è¯**。 +1. 找到è¦åˆ†é…角色的æˆå‘˜ã€‚å•å‡» **â‹® > 编辑é…ç½®**。 +1. 如果你添加了自定义角色，它们将显示在**自定义**中。选择è¦åˆ†é…ç»™æˆå‘˜çš„角色。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：æˆå‘˜å…·æœ‰æ‰€åˆ†é…的角色。 + +### 项目角色 + +_项目角色_ 是用于授予用户访问项目æƒé™çš„角色。主è¦çš„项目角色分别是`所有者`ã€`æˆå‘˜`å’Œ`åªè¯»`。 + +- **项目所有者：** + + å¯ä»¥å®Œå…¨æŽ§åˆ¶é¡¹ç›®åŠå…¶ä¸­çš„所有资æºã€‚ + +- **项目æˆå‘˜ï¼š** + + å¯ä»¥ç®¡ç†é¡¹ç›®èŒƒå›´çš„资æºï¼Œå¦‚命å空间和工作负载，但ä¸èƒ½ç®¡ç†å…¶ä»–项目æˆå‘˜ã€‚ + +:::note + +默认情况下，Rancher çš„`项目æˆå‘˜`角色继承自 `Kubernetes-edit` 角色，而`项目所有者`角色继承自 `Kubernetes-admin` 角色。因此，`项目æˆå‘˜`å’Œ`项目所有者`角色都能管ç†å‘½å空间，包括创建和删除命å空间。 + +::: + +- **åªè¯»ï¼š** + + å¯ä»¥æŸ¥çœ‹é¡¹ç›®ä¸­çš„所有内容，但ä¸èƒ½åˆ›å»ºã€æ›´æ–°æˆ–删除任何内容。 + +:::danger + +如果用户分é…到了项目的`所有者`或`æˆå‘˜`角色，用户会自动继承`命å空间创建`角色。然而，这个角色是 [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole)，这表示角色的范围会延展到集群中的所有项目。因此，对于显å¼åˆ†é…到了项目`所有者`或`æˆå‘˜`角色的用户æ¥è¯´ï¼Œå³ä½¿åªæœ‰`åªè¯»`角色，这些用户也å¯ä»¥åœ¨åˆ†é…给他们的其他项目中创建命å空间。 + +::: + +#### 自定义项目角色 + +Rancher 支æŒå°† _自定义项目角色_ 分é…给普通用户，而ä¸æ˜¯å…¸åž‹çš„`所有者`ã€`æˆå‘˜`或`åªè¯»`角色。这些角色å¯ä»¥æ˜¯å†…置的自定义项目角色，也å¯ä»¥æ˜¯ Rancher 管ç†å‘˜å®šä¹‰çš„角色。这些角色便于为项目内的普通用户定义更å—é™æˆ–特定的访问æƒé™ã€‚有关内置自定义项目角色的列表，请å‚阅下表。 + +#### 项目角色å‚考 + +下表列出了 Rancher 中å¯ç”¨çš„内置自定义项目角色，以åŠè¿™äº›è§’色是å¦ç”±`所有者`,`æˆå‘˜`或`åªè¯»`角色授予的： + +| 内置项目角色 | 所有者 | æˆå‘˜ | åªè¯» | +| ---------------------------------- | ------------- | ----------------------------- | ------------- | +| 管ç†é¡¹ç›®æˆå‘˜ | ✓ | | | +| 创建命å空间 | ✓ | ✓ | | +| 管ç†é…置映射 | ✓ | ✓ | | +| ç®¡ç† Ingress | ✓ | ✓ | | +| 管ç†é¡¹ç›®åº”用商店 | ✓ | | | +| 管ç†å¯†æ–‡ | ✓ | ✓ | | +| ç®¡ç† ServiceAccount | ✓ | ✓ | | +| 管ç†æœåŠ¡ | ✓ | ✓ | | +| 管ç†å· | ✓ | ✓ | | +| 管ç†å·¥ä½œè´Ÿè½½ | ✓ | ✓ | | +| 查看密文 | ✓ | ✓ | | +| 查看é…置图 | ✓ | ✓ | ✓ | +| 查看 Ingress | ✓ | ✓ | ✓ | +| 查看项目æˆå‘˜ | ✓ | ✓ | ✓ | +| 查看项目应用商店 | ✓ | ✓ | ✓ | +| 查看 ServiceAccount | ✓ | ✓ | ✓ | +| 查看æœåŠ¡ | ✓ | ✓ | ✓ | +| æŸ¥çœ‹å· | ✓ | ✓ | ✓ | +| 查看工作负载 | ✓ | ✓ | ✓ | + +:::note 注æ„事项： + +- 上é¢åˆ—出的æ¯ä¸ªé¡¹ç›®è§’色（包括`所有者`ã€`æˆå‘˜`å’Œ`åªè¯»`）å‡ç”±å¤šä¸ªè§„则组æˆï¼Œè¿™äº›è§„则授予对å„ç§èµ„æºçš„访问æƒé™ã€‚ä½ å¯ä»¥åœ¨**全局 > 安全 > 角色**页é¢ä¸ŠæŸ¥çœ‹è§’色åŠå…¶è§„则。 +- 在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ +- `管ç†é¡¹ç›®æˆå‘˜`角色å…许项目所有者管ç†é¡¹ç›®çš„所有æˆå‘˜ï¼Œ**并**授予这些æˆå‘˜ä»»ä½•é¡¹ç›®èŒƒå›´çš„角色（ä¸è®ºä»–们是å¦æœ‰æƒè®¿é—®é¡¹ç›®èµ„æºï¼‰ã€‚å•ç‹¬åˆ†é…此角色时è¦å°å¿ƒã€‚ + +::: + +### 定义自定义角色 +如å‰æ‰€è¿°ï¼Œä½ å¯ä»¥å®šä¹‰è‡ªå®šä¹‰è§’色，并将这些角色用在集群或项目中。上下文字段定义了角色是å¦æ˜¾ç¤ºåœ¨é›†ç¾¤æˆå‘˜é¡µé¢ã€é¡¹ç›®æˆå‘˜é¡µé¢æˆ–åŒæ—¶æ˜¾ç¤ºåœ¨è¿™ä¸¤ä¸ªé¡µé¢ã€‚ + +定义自定义角色时，你å¯ä»¥æŽˆäºˆå¯¹ç‰¹å®šèµ„æºçš„访问æƒé™ï¼Œæˆ–指定自定义角色应继承的角色。自定义角色å¯ä»¥ç”±ç‰¹å®šæŽˆæƒå’Œç»§æ‰¿è§’色组æˆã€‚所有授æƒéƒ½æ˜¯ç´¯åŠ çš„。æ¢è¨€ä¹‹ï¼Œå¦‚果你为特定资æºå®šä¹‰æ›´å—é™çš„授æƒï¼Œè‡ªå®šä¹‰è§’色继承的角色中定义的更广泛的授æƒ**ä¸ä¼š**被覆盖。 + +### 默认集群和项目角色 + +默认情况下，在普通用户创建新集群或项目时，他们会自动分é…到所有者的角色，å³[集群所有者](#集群角色)或[项目所有者](#项目角色)。但是，在æŸäº›ç»„织中，这些角色å¯èƒ½ä¼šè¢«è®¤ä¸ºæœ‰è¿‡å¤šçš„管ç†è®¿é—®æƒé™ã€‚在这ç§æƒ…况下，你å¯ä»¥å°†é»˜è®¤è§’色更改为更具é™åˆ¶æ€§çš„角色，例如一组å•ç‹¬çš„角色或一个自定义角色。 + +更改默认集群/项目角色有以下两ç§æ–¹æ³•ï¼š + +- **分é…自定义角色**：为你的[集群](#自定义集群角色)或[项目](#自定义项目角色)创建一个[自定义角色](custom-roles.md)，然åŽå°†è‡ªå®šä¹‰è§’色设置为默认。 + +- **分é…å•ç‹¬çš„角色**：将多个[集群](#集群角色å‚考)/[项目](#项目角色å‚考)角色é…置为默认角色，并分é…给创建的用户。 + + 例如，你å¯ä»¥é€‰æ‹©æ··åˆä½¿ç”¨å¤šä¸ªè§’色（例如`管ç†èŠ‚点`å’Œ`管ç†å­˜å‚¨`），而ä¸æ˜¯ä½¿ç”¨ç»§æ‰¿çš„角色（例如`集群所有者`）。 + +:::note + +- 虽然你å¯ä»¥[é”定](locked-roles.md)一个默认角色，但系统ä»ä¼šå°†è¿™ä¸ªè§’色分é…给创建集群/项目的用户。 +- åªæœ‰åˆ›å»ºé›†ç¾¤/项目的用户æ‰èƒ½ç»§æ‰¿ä»–们的角色。对于之åŽæ·»åŠ ä¸ºé›†ç¾¤/项目æˆå‘˜çš„用户，你必须显å¼åˆ†é…角色。 + +::: + +### 为集群和项目创建者é…置默认角色 + +ä½ å¯ä»¥æ›´æ”¹ä¸ºåˆ›å»ºé›†ç¾¤æˆ–项目的用户自动创建的角色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. å•å‡»**集群**或**项目或命å空间**选项å¡ã€‚ +1. 找到你è¦ç”¨ä½œé»˜è®¤è§’色的自定义或å•ä¸ªè§’色。然åŽé€šè¿‡é€‰æ‹© **â‹® > 编辑é…ç½®**æ¥ç¼–辑角色。 +1. 在**集群创建者的默认角色**或**项目创建者的默认角色**中，将角色å¯ç”¨ä¸ºé»˜è®¤ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：默认角色已根æ®ä½ çš„更改é…置。分é…给集群/项目创建者的角色会在**集群创建者的默认角色/项目创建者的默认角色**列中勾选。 + +如果è¦åˆ é™¤é»˜è®¤è§’色，请编辑æƒé™ï¼Œå¹¶åœ¨é»˜è®¤è§’色选项中选择**å¦**。 + +### 撤销集群æˆå‘˜èµ„æ ¼ + +如果你撤销一个普通用户的集群æˆå‘˜èµ„格，而且该用户已显å¼åˆ†é…集群的集群 _å’Œ_ 项目的æˆå‘˜èµ„格，该普通用户将[失去集群角色](#集群角色)但[ä¿ç•™é¡¹ç›®è§’色](#项目角色)。æ¢å¥è¯è¯´ï¼Œå³ä½¿ä½ å·²ç»æ’¤é”€äº†ç”¨æˆ·è®¿é—®é›†ç¾¤å’Œå…¶ä¸­çš„节点的æƒé™ï¼Œä½†è¯¥æ™®é€šç”¨æˆ·ä»ç„¶å¯ä»¥ï¼š + +- 访问他们拥有æˆå‘˜èµ„格的项目。 +- 行使分é…给他们的任何[å•ä¸ªé¡¹ç›®è§’色](#项目角色å‚考)。 + +如果你想完全撤销用户在集群中的访问æƒé™ï¼Œè¯·åŒæ—¶æ’¤é”€ä»–们的集群和项目æˆå‘˜èµ„格。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md new file mode 100644 index 00000000000..0ab8560983b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md @@ -0,0 +1,120 @@ +--- +title: 自定义角色 +--- + +在 Rancher 中，_角色_ 决定了用户å¯ä»¥åœ¨é›†ç¾¤æˆ–项目中执行哪些æ“作。 + +请注æ„，_角色_ 与 _æƒé™_ ä¸åŒï¼Œæƒé™å†³å®šçš„是你å¯ä»¥è®¿é—®å“ªäº›é›†ç¾¤å’Œé¡¹ç›®ã€‚ + +:::danger + +自定义角色å¯ä»¥å¯ç”¨æƒé™æå‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](#æƒé™æå‡)。 + +::: + + +## 先决æ¡ä»¶ + +è¦å®Œæˆæ­¤é¡µé¢ä¸Šçš„任务，需è¦ä»¥ä¸‹æƒé™ä¹‹ä¸€ï¼š + +- [管ç†å‘˜å…¨å±€æƒé™](global-permissions.md)。 +- 分é…了[管ç†è§’色](global-permissions.md)çš„[自定义全局æƒé™](global-permissions.md#自定义全局æƒé™)。 + +## 创建自定义角色 + +虽然 Rancher æ供一组开箱å³ç”¨çš„默认用户角色，但你还å¯ä»¥åˆ›å»ºé»˜è®¤çš„自定义角色，从而在 Rancher 中为用户æ供更精细的æƒé™ã€‚ + +添加自定义角色的步骤因 Rancher 的版本而异。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 选择一个选项å¡æ¥ç¡®å®šè¦æ·»åŠ çš„角色的范围。这些选项å¡æ˜¯ï¼š + +- **全局**：仅在å…许æˆå‘˜ç®¡ç†å…¨å±€èŒƒå›´çš„资æºæ—¶ï¼Œå¯ä»¥åˆ†é…该角色。 +- **集群**：仅在å‘集群添加/管ç†æˆå‘˜æ—¶ï¼Œå¯ä»¥åˆ†é…该角色。 +- **项目或命å空间**：仅在å‘项目或命å空间添加/管ç†æˆå‘˜æ—¶ï¼Œå¯ä»¥åˆ†é…该角色。 + +1. æ ¹æ®æ‰€éœ€è¦çš„范围，å•å‡»**创建全局角色**ã€**创建集群角色**或**创建项目或命å空间的角色**。 +1. 输入角色的**å称**。 +1. å¯é€‰ï¼šé€‰æ‹©**集群创建者的默认角色/项目创建者的默认角色**选项，以将该角色分é…给集群/项目创建者。使用此功能，你å¯ä»¥æ‰©å±•æˆ–é™åˆ¶é›†ç¾¤/项目创建者的默认角色。 + + > 开箱å³ç”¨çš„**集群创建者的默认角色**å’Œ**项目创建者的默认角色**分别是`集群所有者`å’Œ`项目所有者`。 + +1. 使用**授æƒèµ„æº**选项将å„个 [Kubernetes API 端点](https://kubernetes.io/docs/reference/)分é…给角色。 + + > 在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ + + > **资æº**文本字段å¯ä»¥ç”¨æ¥æœç´¢é¢„定义的 Kubernetes API 资æºï¼Œæˆ–者为授æƒè¾“入自定义资æºå称。在此字段中输入资æºå称åŽï¼Œå¿…须从下拉列表中选择预定义或`（自定义）`资æºã€‚ + + 你还å¯ä»¥é€‰æ‹©æ¯ä¸ªåˆ†é…的端点å¯ç”¨çš„ cURL 方法（`Create`ã€`Delete`ã€`Get` 等）。 + +1. 使用 **Inherit from** 选项将å„个 Rancher 角色分é…给你的自定义角色。请注æ„，如果自定义角色从父角色继承，你需è¦å…ˆåˆ é™¤å­è§’色æ‰èƒ½åˆ é™¤çˆ¶è§’色。 + +1. å•å‡»**创建**。 + +## 创建从å¦ä¸€ä¸ªè§’色继承的自定义角色 + +如果你有一组需è¦åœ¨ Rancher 中具有相åŒè®¿é—®æƒé™çš„用户，一ç§èŠ‚çœæ—¶é—´çš„方法是创建一个新的自定义角色，而该角色的规则都是从å¦ä¸€ä¸ªè§’色（例如管ç†å‘˜è§’色）å¤åˆ¶è€Œæ¥çš„。这样，你åªéœ€è¦é…置现有角色和新角色之间ä¸åŒçš„部分。 + +然åŽï¼Œä½ å¯ä»¥å°†è‡ªå®šä¹‰è§’色分é…给用户或组。该角色在用户首次登录 Rancher 时生效。 + +è¦åŸºäºŽçŽ°æœ‰è§’色创建自定义角色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. å•å‡»**集群**或**项目或命å空间**选项å¡ã€‚æ ¹æ®æ‰€éœ€è¦çš„范围，å•å‡»**创建集群角色**或**创建项目或命å空间的角色**。请注æ„，åªæœ‰é›†ç¾¤è§’色和项目/命å空间角色å¯ä»¥ä»Žå¦ä¸€ä¸ªè§’色继承。 +1. 输入角色的å称。 +1. 在 **Inherit From** 选项å¡ä¸­ï¼Œé€‰æ‹©è‡ªå®šä¹‰è§’色需è¦ä»Žå“ªä¸ªè§’色继承æƒé™ã€‚ +1. 在**授æƒèµ„æº**选项å¡ä¸­ï¼Œé€‰æ‹©æ‹¥æœ‰è‡ªå®šä¹‰è§’色的用户è¦å¯ç”¨çš„ Kubernetes 资æºæ“作。 + + > **资æº**文本字段å¯ä»¥ç”¨æ¥æœç´¢é¢„定义的 Kubernetes API 资æºï¼Œæˆ–者为授æƒè¾“入自定义资æºå称。在此字段中输入资æºå称åŽï¼Œå¿…须从下拉列表中选择预定义或`（自定义）`资æºã€‚ +1. å¯é€‰ï¼šå°†è§’色设置为默认。 +1. å•å‡»**创建**。 + +## 删除自定义角色 + +删除自定义角色时，具有此自定义角色的所有全局角色绑定（Global Role Bindings）都将被删除。 + +如果æŸä¸ªç”¨æˆ·ä»…分é…了一个自定义全局角色，而且你删除了这个角色，该用户将ä¸èƒ½å†è®¿é—® Rancher。è¦è®©ç”¨æˆ·é‡æ–°èŽ·å¾—访问æƒé™ï¼Œç®¡ç†å‘˜éœ€è¦ç¼–辑用户并应用新的全局æƒé™ã€‚ + +自定义角色å¯ä»¥åˆ é™¤ï¼Œä½†å†…置角色ä¸èƒ½åˆ é™¤ã€‚ + +è¦åˆ é™¤è‡ªå®šä¹‰è§’色： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +2. 转到è¦åˆ é™¤çš„自定义全局角色，然åŽå•å‡» **â‹® (…) > 删除**。 +3. å•å‡»**删除**。 + +## 为组分é…自定义角色 + +如果你有一组需è¦åœ¨ Rancher 中具有相åŒè®¿é—®æƒé™çš„用户，一ç§èŠ‚çœæ—¶é—´çš„方法是创建一个新的自定义角色。将角色分é…给组时，组中的用户在首次登录 Rancher 时就会拥有é…置的访问级别。 + +组中的用户登录时，他们默认获得内置的**普通用户**全局角色。他们还将获得分é…给他们的组的æƒé™ã€‚ + +如果将用户从外部身份验è¯ç³»ç»Ÿçš„组中删除，用户将失去分é…给该组的自定义全局角色的æƒé™ã€‚但是，用户ä»ä¼šæ‹¥æœ‰**普通用户**角色。 + +:::note 先决æ¡ä»¶ï¼š + +åªæœ‰åœ¨ä»¥ä¸‹æƒ…况下，你æ‰èƒ½å°†å…¨å±€è§’色分é…给组： + +* 你已设置[外部身份验è¯æ供程åº](../../../../pages-for-subheaders/authentication-config.md#外部验è¯ä¸Žæœ¬åœ°éªŒè¯)。 +* 外部身份验è¯æ供程åºæ”¯æŒ[用户组](../../authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md)。 +* 你已使用身份验è¯æ供程åºè®¾ç½®äº†è‡³å°‘一个用户组。 + +::: + +è¦å°†è‡ªå®šä¹‰è§’色分é…给组，请执行以下步骤： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**组**。 +1. 转到将分é…自定义角色的组，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 如果你已创建角色，角色将显示在**自定义**中。选择è¦åˆ†é…给组的自定义角色。 +1. å¯é€‰ï¼šåœ¨**全局æƒé™**或**内置角色**中，选择è¦åˆ†é…给该组的其他æƒé™ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**：自定义角色将在组内用户登录 Rancher 时生效。 + +## æƒé™æå‡ + +`é…置应用商店`这个自定义æƒé™å¾ˆå¼ºå¤§ï¼Œåº”谨慎使用。如果管ç†å‘˜å°†`é…置应用商店`æƒé™åˆ†é…给普通用户，å¯èƒ½ä¼šå¯¼è‡´æƒé™æå‡ã€‚在这ç§æƒ…况下，用户å¯ä»¥è®©è‡ªå·±å¯¹ Rancher é…置的集群进行管ç†å‘˜è®¿é—®ã€‚因此，拥有此æƒé™çš„任何用户都应被视为管ç†å‘˜ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md new file mode 100644 index 00000000000..3fb65c5d7e0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md @@ -0,0 +1,247 @@ +--- +title: 全局æƒé™ +--- + +_æƒé™_ 是你在为用户选择自定义æƒé™æ—¶å¯ä»¥åˆ†é…的个人访问æƒé™ã€‚ + +全局æƒé™å®šä¹‰ç”¨æˆ·åœ¨ä»»ä½•ç‰¹å®šé›†ç¾¤ä¹‹å¤–的授æƒã€‚Rancher æ供四ç§å¼€ç®±å³ç”¨çš„默认全局æƒé™ï¼š`Administrator` (管ç†å‘˜)ã€`Restricted Admin` (å—é™ç®¡ç†å‘˜)ã€`Standard User` (标准用户) å’Œ `User-Base` 用户。 + +- **管ç†å‘˜**：å¯ä»¥å®Œå…¨æŽ§åˆ¶æ•´ä¸ª Rancher 系统和其中的所有集群。 + +- **å—é™ç®¡ç†å‘˜**：å¯ä»¥å®Œå…¨æŽ§åˆ¶ä¸‹æ¸¸é›†ç¾¤ï¼Œä½†ä¸èƒ½æ›´æ”¹æœ¬åœ° Kubernetes 集群。 + +- **普通用户**：å¯ä»¥åˆ›å»ºæ–°é›†ç¾¤å¹¶ä½¿ç”¨å®ƒä»¬ã€‚普通用户还å¯ä»¥åœ¨è‡ªå·±çš„集群中å‘其他用户分é…集群æƒé™ã€‚ + +- **User-Base 用户**：åªæœ‰ç™»å½•æƒé™ã€‚ + +你无法更新或删除内置的全局æƒé™ã€‚ + +## å—é™ç®¡ç†å‘˜ + +Rancher 2.5 创建了一个新的 `restricted-admin` 角色，以防止本地 Rancher Server Kubernetes 集群的æƒé™æå‡ã€‚此角色对 Rancher 管ç†çš„所有下游集群具有完全管ç†å‘˜æƒé™ï¼Œä½†æ²¡æœ‰æ›´æ”¹æœ¬åœ° Kubernetes 集群的æƒé™ã€‚ + +`restricted-admin` å¯ä»¥åˆ›å»ºå…¶ä»–具有åŒæ ·è®¿é—®æƒé™çš„ `restricted-admin` 用户。 + +Rancher 还增加了一个新设置，æ¥å°†åˆå§‹å¯åŠ¨çš„管ç†å‘˜è®¾ç½®ä¸º `restricted-admin` 角色。该设置适用于 Rancher Server 首次å¯åŠ¨æ—¶åˆ›å»ºçš„第一个用户。如果设置了这个环境å˜é‡ï¼Œåˆ™ä¸ä¼šåˆ›å»ºå…¨å±€ç®¡ç†å‘˜ï¼Œä¹Ÿå°±æ— æ³•é€šè¿‡ Rancher 创建全局管ç†å‘˜ã€‚ + +è¦ä»¥ `restricted-admin` 作为åˆå§‹ç”¨æˆ·æ¥å¯åŠ¨ Rancher，你需è¦ä½¿ç”¨ä»¥ä¸‹çŽ¯å¢ƒå˜é‡æ¥å¯åŠ¨ Rancher Server： + +``` +CATTLE_RESTRICTED_DEFAULT_ADMIN=true +``` +### `restricted-admin` çš„æƒé™åˆ—表 + +下表列出了 `restricted-admin` 与 `Administrator` å’Œ `Standard User` 角色相比应具有的æƒé™å’Œæ“作： + +| 类别 | æ“作 | 全局管ç†å‘˜ | 普通用户 | å—é™ç®¡ç†å‘˜ | å—é™ç®¡ç†å‘˜çš„注æ„事项 | +| -------- | ------ | ------------ | ------------- | ---------------- | ------------------------------- | +| 本地集群功能 | 管ç†æœ¬åœ°é›†ç¾¤ï¼ˆåˆ—出ã€ç¼–辑ã€å¯¼å…¥ä¸»æœºï¼‰ | 是 | å¦ | å¦ | | +| | 创建项目/命å空间 | 是 | å¦ | å¦ | | +| | 添加集群/项目æˆå‘˜ | 是 | å¦ | å¦ | | +| | 全局 DNS | 是 | å¦ | å¦ | | +| | 访问 CRD å’Œ CR 的管ç†é›†ç¾¤ | 是 | å¦ | 是 | | +| | å¦å­˜ä¸º RKE æ¨¡æ¿ | 是 | å¦ | å¦ | | +| 安全 | | | | | | +| å¯ç”¨èº«ä»½éªŒè¯ | é…ç½®èº«ä»½éªŒè¯ | 是 | å¦ | 是 | | +| 角色 | 创建/åˆ†é… GlobalRoles | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | è®¤è¯ Webhook å…许为已ç»å­˜åœ¨çš„æƒé™åˆ›å»º globalrole | +| | 创建/åˆ†é… ClusterRoles | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | ä¸åœ¨æœ¬åœ°é›†ç¾¤ä¸­ | +| | 创建/åˆ†é… ProjectRoles | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | ä¸åœ¨æœ¬åœ°é›†ç¾¤ä¸­ | +| 用户 | 添加用户/编辑/删除/åœç”¨ç”¨æˆ· | 是 | å¦ | 是 | | +| 组 | 将全局角色分é…给组 | 是 | å¦ | 是 | 在 Webhook å…许的范围内 | +| | 刷新组 | 是 | å¦ | 是 | | +| PSP | ç®¡ç† PSP æ¨¡æ¿ | 是 | å¦ï¼ˆå¯åˆ—出） | 是 | 与 PSP 的全局管ç†å‘˜æƒé™ç›¸åŒ | +| 工具 | | | | | | +| | ç®¡ç† RKE æ¨¡æ¿ | 是 | å¦ | 是 | | +| | 管ç†å…¨å±€åº”用商店 | 是 | å¦ | 是 | 无法编辑/删除内置系统应用商店。å¯ä»¥ç®¡ç† Helm 库 | +| | 集群驱动 | 是 | å¦ | 是 | | +| | 主机驱动 | 是 | å¦ | 是 | | +| | GlobalDNS æ供商 | 是 | 是（自己） | 是 | | +| | GlobalDNS æ¡ç›® | 是 | 是（自己） | 是 | | +| 设置 | | | | | | +| | 管ç†è®¾ç½® | 是 | å¦ï¼ˆå¯åˆ—出） | å¦ï¼ˆå¯åˆ—出） | | +| 用户 | | | | | | +| | ç®¡ç† API 密钥 | 是（管ç†æ‰€æœ‰ï¼‰ | 是（管ç†è‡ªå·±çš„） | 是（管ç†è‡ªå·±çš„） | | +| | 管ç†èŠ‚ç‚¹æ¨¡æ¿ | 是 | 是（管ç†è‡ªå·±çš„） | 是（管ç†è‡ªå·±çš„） | åªèƒ½ç®¡ç†è‡ªå·±çš„节点模æ¿ï¼Œä¸èƒ½ç®¡ç†å…¶ä»–用户创建的节点模æ¿ã€‚ | +| | 管ç†äº‘å‡­è¯ | 是 | 是（管ç†è‡ªå·±çš„） | 是（管ç†è‡ªå·±çš„） | åªèƒ½ç®¡ç†è‡ªå·±çš„云凭è¯ï¼Œä¸èƒ½ç®¡ç†å…¶ä»–用户创建的云凭è¯ã€‚ | +| 下游集群 | 创建集群 | 是 | 是 | 是 | | +| | 编辑集群 | 是 | 是 | 是 | | +| | è½®æ¢è¯ä¹¦ | 是 | | 是 | | +| | ç«‹å³åˆ›å»ºå¿«ç…§ | 是 | | 是 | | +| | æ¢å¤å¿«ç…§ | 是 | | 是 | | +| | å¦å­˜ä¸º RKE æ¨¡æ¿ | 是 | å¦ | 是 | | +| | è¿è¡Œ CIS 扫æ | 是 | 是 | 是 | | +| | 添加æˆå‘˜ | 是 | 是 | 是 | | +| | 创建项目 | 是 | 是 | 是 | | +| 自 2.5 起的功能 Chart | | | | | | +| | 安装 Fleet | 是 | | 是 | 无法在本地集群中è¿è¡Œ Fleet | +| | 部署 EKS 集群 | 是 | 是 | 是 | | +| | 部署 GKE 集群 | 是 | 是 | 是 | | +| | 部署 AKS 集群 | 是 | 是 | 是 | | + + +### 将全局管ç†å‘˜æ›´æ”¹ä¸ºå—é™ç®¡ç†å‘˜ + +如果 Rancher å·²ç»æœ‰ä¸€ä¸ªå…¨å±€ç®¡ç†å‘˜ï¼Œåˆ™åº”该将所有全局管ç†å‘˜æ›´æ”¹ä¸ºæ–°çš„ `restricted-admin`。 + +ä½ å¯ä»¥å‰å¾€**安全 > 用户**，并将所有管ç†å‘˜è§’色转为å—é™ç®¡ç†å‘˜ã€‚ + +已登录的用户å¯ä»¥æ ¹æ®éœ€è¦å°†è‡ªå·±æ›´æ”¹ä¸º `restricted-admin`，但这应该是他们的最åŽä¸€æ­¥æ“作，å¦åˆ™ä»–们将没有进行该æ“作的æƒé™ã€‚ + +## 分é…全局æƒé™ + +本地用户的全局æƒé™åˆ†é…与使用外部身份验è¯ç™»å½• Rancher 的用户ä¸åŒã€‚ + +### 新本地用户的全局æƒé™ + +在创建新本地用户时，请在填写**添加用户**表å•æ—¶ä¸ºä»–分é…全局æƒé™ã€‚ + +如果需è¦æŸ¥çœ‹æ–°ç”¨æˆ·çš„默认æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. **角色**页é¢æœ‰æŒ‰èŒƒå›´åˆ†ç»„的角色选项å¡ã€‚æ¯ä¸ªè¡¨éƒ½åˆ—出了范围内的角色。在**全局**选项å¡çš„**新用户的默认角色**列中，默认授予新用户的æƒé™ç”¨å¤é€‰æ ‡è®°è¡¨ç¤ºã€‚ + +ä½ å¯ä»¥[更改默认全局æƒé™æ¥æ»¡è¶³ä½ çš„需è¦](#é…置默认全局æƒé™)。 + +### 使用外部身份验è¯ç¨‹åºçš„用户的全局æƒé™ + +当用户首次使用外部身份验è¯æ供程åºç™»å½• Rancher 时，他们会自动分é…到**新用户的默认角色**的全局æƒé™ã€‚默认情况下，Rancher ä¸ºæ–°ç”¨æˆ·åˆ†é… **Standard User** æƒé™ã€‚ + +如果需è¦æŸ¥çœ‹æ–°ç”¨æˆ·çš„默认æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. **角色**页é¢æœ‰æŒ‰èŒƒå›´åˆ†ç»„的角色选项å¡ã€‚æ¯ä¸ªè¡¨éƒ½åˆ—出了范围内的角色。在æ¯ä¸ªé¡µé¢çš„**新用户的默认角色**列中，默认授予新用户的æƒé™ç”¨å¤é€‰æ ‡è®°è¡¨ç¤ºã€‚ + +ä½ å¯ä»¥[更改默认æƒé™æ¥æ»¡è¶³ä½ çš„需è¦](#é…置默认全局æƒé™)。 + +æƒé™å¯ä»¥[分é…](#为å•ä¸ªç”¨æˆ·é…置全局æƒé™)ç»™å•ä¸ªç”¨æˆ·ã€‚ + +如果外部身份验è¯æ供程åºæ”¯æŒç»„，你å¯ä»¥[åŒæ—¶ä¸ºç»„中的æ¯ä¸ªæˆå‘˜åˆ†é…角色](#为组é…置全局æƒé™)。 + +## 自定义全局æƒé™ + +使用自定义æƒé™å¯ä»¥ä¸ºç”¨æˆ·é…置在 Rancher 中的更å—é™æˆ–指定的访问æƒé™ã€‚ + +当æ¥è‡ª[外部身份验è¯ç³»ç»Ÿ](../../../../pages-for-subheaders/authentication-config.md)的用户首次登录 Rancher 时，他们会自动分é…到一组全局æƒé™ï¼ˆä»¥ä¸‹ç®€ç§°æƒé™ï¼‰ã€‚默认情况下，用户第一次登录åŽä¼šè¢«åˆ›å»ºä¸ºç”¨æˆ·ï¼Œå¹¶åˆ†é…到默认的`用户`æƒé™ã€‚标准的`用户`æƒé™å…许用户登录和创建集群。 + +但是，在æŸäº›ç»„织中，这些æƒé™å¯èƒ½ä¼šè¢«è®¤ä¸ºæƒé™è¿‡å¤§ã€‚ä½ å¯ä»¥ä¸ºç”¨æˆ·åˆ†é…一组更具é™åˆ¶æ€§çš„自定义全局æƒé™ï¼Œè€Œä¸æ˜¯ä¸ºç”¨æˆ·åˆ†é… `Administrator` 或 `Standard User` 的默认全局æƒé™ã€‚ + +默认角色（管ç†å‘˜å’Œæ ‡å‡†ç”¨æˆ·ï¼‰éƒ½å†…置了多个全局æƒé™ã€‚系统管ç†å‘˜è§’色包括所有全局æƒé™ï¼Œè€Œé»˜è®¤ç”¨æˆ·è§’色包括三个全局æƒé™ï¼Œåˆ†åˆ«æ˜¯åˆ›å»ºé›†ç¾¤ï¼Œä½¿ç”¨åº”用商店模æ¿ï¼Œå’Œ User Base（登录 Rancher 的最低æƒé™ï¼‰ã€‚æ¢å¥è¯è¯´ï¼Œè‡ªå®šä¹‰å…¨å±€æƒé™æ˜¯æ¨¡å—化的，因此，如果你è¦æ›´æ”¹é»˜è®¤ç”¨æˆ·è§’色æƒé™ï¼Œä½ å¯ä»¥é€‰æ‹©éœ€è¦åŒ…括在新的默认用户角色中的全局æƒé™å­é›†ã€‚ + +管ç†å‘˜å¯ä»¥é€šè¿‡å¤šç§æ–¹å¼å¼ºåˆ¶æ‰§è¡Œè‡ªå®šä¹‰å…¨å±€æƒé™ï¼š + +- [更改新用户的默认æƒé™](#é…置默认全局æƒé™) +- [为å•ä¸ªç”¨æˆ·é…置全局æƒé™](#为å•ä¸ªç”¨æˆ·é…置全局æƒé™) +- [为组é…置全局æƒé™](#为组é…置全局æƒé™) + +### 自定义全局æƒé™å‚考 + +下表列出了æ¯ä¸ªå¯ç”¨çš„自定义全局æƒé™ï¼Œä»¥åŠè¯¥æƒé™æ˜¯å¦åŒ…å«åœ¨é»˜è®¤å…¨å±€æƒé™ `Administrator`ã€`Standard User` å’Œ `User-Base` 中： + +| 自定义全局æƒé™ | 管ç†å‘˜ | 普通用户 | User-Base | +| ---------------------------------- | ------------- | ------------- |-----------| +| 创建集群 | ✓ | ✓ | | +| 创建 RKE æ¨¡æ¿ | ✓ | ✓ | | +| 管ç†èº«ä»½éªŒè¯ | ✓ | | | +| 管ç†åº”用商店 | ✓ | | | +| 管ç†é›†ç¾¤é©±åŠ¨ | ✓ | | | +| 管ç†ä¸»æœºé©±åŠ¨ | ✓ | | | +| ç®¡ç† PodSecurityPolicy æ¨¡æ¿ | ✓ | | | +| 管ç†è§’色 | ✓ | | | +| 管ç†è®¾ç½® | ✓ | | | +| 管ç†ç”¨æˆ· | ✓ | | | +| ä½¿ç”¨åº”ç”¨å•†åº—æ¨¡æ¿ | ✓ | ✓ | | +| User-Base（基本登录访问） | ✓ | ✓ | | + +如果需è¦æŸ¥çœ‹æ¯ä¸ªå…¨å±€æƒé™å¯¹åº”哪些 Kubernetes 资æºï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 如果å•å‡»å•ä¸ªè§’色的å称，表格会显示该角色授æƒçš„所有æ“作和资æºã€‚ + +:::note 注æ„事项： + +- 上é¢åˆ—出的æ¯ä¸ªæƒé™éƒ½åŒ…å«å¤šä¸ªæœªåœ¨ Rancher UI 中列出的æƒé™ã€‚如果需è¦èŽ·å–完整æƒé™åˆ—表以åŠç»„æˆæƒé™çš„规则，请通过 `/v3/globalRoles` API 进行访问。 +- 在查看 Rancher 创建的默认角色关è”的资æºæ—¶ï¼Œå¦‚果在一行上有多个 Kubernetes API 资æºï¼Œåˆ™è¯¥èµ„æºå°†å¸¦æœ‰ `(Custom)` 标识。这ä¸ä»£è¡¨è¿™ä¸ªèµ„æºæ˜¯è‡ªå®šä¹‰èµ„æºï¼Œè€Œåªæ˜¯è¡¨æ˜Žå¤šä¸ª Kubernetes API 资æºä½œä¸ºä¸€ä¸ªèµ„æºã€‚ + +::: + +### é…置默认全局æƒé™ + +如果你想é™åˆ¶æ–°ç”¨æˆ·çš„默认æƒé™ï¼Œä½ å¯ä»¥åˆ é™¤ä½œä¸ºé»˜è®¤è§’色的`用户`æƒé™ï¼Œç„¶åŽåˆ†é…多个å•ç‹¬çš„æƒé™ä½œä¸ºé»˜è®¤æƒé™ã€‚你也å¯ä»¥åœ¨ä¸€ç»„其他标准æƒé™ä¹‹ä¸Šæ·»åŠ ç®¡ç†æƒé™ã€‚ + +:::note + +默认角色仅分é…给从外部身份验è¯æ供程åºæ·»åŠ çš„用户。对于本地用户，在将用户添加到 Rancher 时，必须显å¼åˆ†é…全局æƒé™ã€‚ä½ å¯ä»¥åœ¨æ·»åŠ ç”¨æˆ·æ—¶è‡ªå®šä¹‰è¿™äº›å…¨å±€æƒé™ã€‚ + +::: + +è¦æ›´æ”¹åœ¨å¤–部用户首次登录时分é…给他们的默认全局æƒé™ï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。在**角色**页é¢ä¸Šï¼Œç¡®ä¿é€‰æ‹©äº†**全局**选项å¡ã€‚ +1. 查找è¦æ·»åŠ æˆ–删除的默认æƒé™é›†ã€‚然åŽï¼Œé€šè¿‡é€‰æ‹© **â‹® > 编辑é…ç½®**æ¥ç¼–辑æƒé™ã€‚ +1. 如果è¦å°†æƒé™æ·»åŠ ä¸ºé»˜è®¤æƒé™ï¼Œè¯·é€‰æ‹©**是：新用户的默认角色**，然åŽå•å‡»**ä¿å­˜**。如果è¦åˆ é™¤é»˜è®¤æƒé™ï¼Œè¯·ç¼–辑该æƒé™å¹¶é€‰æ‹©**å¦**。 + +**结果**：默认全局æƒé™å·²æ ¹æ®ä½ çš„更改é…置。分é…给新用户的æƒé™ä¼šåœ¨**新用户的默认角色**列中显示为å¤é€‰æ ‡è®°ã€‚ + +### 为å•ä¸ªç”¨æˆ·é…置全局æƒé™ + +è¦ä¸ºå•ä¸ªç”¨æˆ·é…ç½®æƒé™ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. 找到è¦æ›´æ”¹è®¿é—®çº§åˆ«çš„用户，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**全局æƒé™**å’Œ**内置角色**部分中，选中你希望用户拥有的æƒé™çš„å¤é€‰æ¡†ã€‚如果你在**角色**页é¢åˆ›å»ºäº†è§’色，这些角色将出现在**自定义**部分，你也å¯ä»¥é€‰æ‹©è¿™äº›è§’色。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：用户的全局æƒé™å·²æ›´æ–°ã€‚ + +### 为组é…置全局æƒé™ + +如果你有一组需è¦åœ¨ Rancher 中有相åŒè®¿é—®æƒé™çš„用户，你å¯ä»¥ä¸€æ¬¡æ€§å°†æƒé™åˆ†é…给整个组æ¥èŠ‚çœæ—¶é—´ã€‚这样，组中的用户在第一次登录 Rancher 时能拥有相应级别的访问æƒé™ã€‚ + +将自定义全局角色分é…给组åŽï¼Œè¯¥è§’色将在组中用户登录 Rancher 时分é…给用户。 + +对于现有用户，新æƒé™å°†åœ¨ç”¨æˆ·é€€å‡º Rancher 并é‡æ–°ç™»å½•æ—¶ï¼Œæˆ–当管ç†å‘˜[刷新用户组æˆå‘˜åå•](#刷新用户组æˆå‘˜åå•)时生效。 + +对于新用户，新æƒé™åœ¨ç”¨æˆ·é¦–次登录 Rancher 时生效。除了**新用户的默认角色**全局æƒé™å¤–，æ¥è‡ªè¯¥ç»„的新用户还将获得自定义全局角色的æƒé™ã€‚默认情况下，**新用户的默认角色**æƒé™ç­‰åŒäºŽ **Standard User** 全局角色，但默认æƒé™å¯ä»¥[é…置。](#é…置默认全局æƒé™) + +如果从外部身份验è¯æ供程åºç»„中删除用户，该用户将失去分é…给该组的自定义全局角色的æƒé™ã€‚他们将继续拥有分é…给他们的所有剩余角色，这通常包括标记为**新用户的默认角色**的角色。Rancher 将在用户登出或管ç†å‘˜[刷新用户组æˆå‘˜åå•](#刷新用户组æˆå‘˜åå•)时删除与组关è”çš„æƒé™ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +åªæœ‰åœ¨ä»¥ä¸‹æƒ…况下，你æ‰èƒ½å°†å…¨å±€è§’色分é…给组： + +* 你已设置[外部身份验è¯æ供程åº](../../../../pages-for-subheaders/authentication-config.md#外部验è¯ä¸Žæœ¬åœ°éªŒè¯)。 +* 外部身份验è¯æ供程åºæ”¯æŒ[用户组](../authentication-config/manage-users-and-groups.md)。 +* 你已使用身份验è¯æ供程åºè®¾ç½®äº†è‡³å°‘一个用户组。 + +::: + +è¦å°†è‡ªå®šä¹‰å…¨å±€è§’色分é…给组，请执行以下步骤： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**组**。 +1. 转到你è¦åˆ†é…自定义全局角色的组，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**全局æƒé™**，**自定义**å’Œ/或**内置角色**部分中，选择该组应具有的æƒé™ã€‚ +1. å•å‡»**创建**。 + +**结果**：自定义全局角色会在组内用户登录 Rancher 时生效。 + +### 刷新用户组æˆå‘˜åå• + +当管ç†å‘˜æ›´æ–°ç»„的全局æƒé™æ—¶ï¼Œæ›´æ”¹å°†åœ¨ç»„æˆå‘˜é€€å‡º Rancher 并é‡æ–°ç™»å½•åŽç”Ÿæ•ˆã€‚ + +如果è¦è®©æ›´æ”¹ç«‹å³ç”Ÿæ•ˆï¼Œç®¡ç†å‘˜æˆ–集群所有者å¯ä»¥åˆ·æ–°ç”¨æˆ·ç»„æˆå‘˜åå•ã€‚ + +如果用户已ç»ä»Žå¤–部身份验è¯æœåŠ¡ä¸­çš„组中删除，管ç†å‘˜ä¹Ÿå¯èƒ½éœ€è¦åˆ·æ–°ç”¨æˆ·ç»„æˆå‘˜åå•ã€‚在这ç§æƒ…况下，刷新æ“作会让 Rancher 知é“用户已从组中删除。 + +è¦åˆ·æ–°ç”¨æˆ·ç»„æˆå‘˜åå•ï¼š + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**用户**。 +1. å•å‡»**刷新用户组æˆå‘˜åå•**。 + +**结果**：对组æˆå‘˜æƒé™çš„所有更改都会生效。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md new file mode 100644 index 00000000000..cb0ad71fa38 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md @@ -0,0 +1,38 @@ +--- +title: é”定角色 +--- + +ä½ å¯ä»¥å°†è§’色设置为`é”定`状æ€ã€‚é”定角色å¯é˜²æ­¢æŠŠè¿™äº›è§’色分é…给用户。 + +处于é”定状æ€çš„角色具有如下特性： + +- 无法å†åˆ†é…给当下还没有被分é…到该角色的用户。 +- 将用户添加到集群或项目时，ä¸ä¼šåœ¨**æˆå‘˜è§’色**下拉列表中列出。 +- ä¸ä¼šå½±å“在é”定该角色之å‰ï¼Œå·²ç»åˆ†é…了该角色的用户。å³ä½¿åŽæ¥é”定了该角色，这些用户ä»ç„¶ä¿ç•™è¯¥è§’色æ供的访问æƒé™ã€‚ + + **示例**：å‡è®¾ä½ çš„组织制定了一个内部策略，ç¦æ­¢æŠŠåˆ›å»ºé¡¹ç›®çš„æƒé™åˆ†é…给集群用户。这时候你需è¦æ‰§è¡Œè¿™ä¸ªç­–略。 + + 因此，在将新用户添加到集群之å‰ï¼Œä½ éœ€è¦é”定以下角色：`集群所有者`，`集群æˆå‘˜`å’Œ`创建项目`。然åŽï¼Œåˆ›å»ºä¸€ä¸ªæ–°çš„自定义角色，该角色的æƒé™ä¸Ž`集群æˆå‘˜`相åŒï¼Œä½†æ²¡æœ‰åˆ›å»ºé¡¹ç›®çš„æƒé™ã€‚然åŽï¼Œåœ¨å°†ç”¨æˆ·æ·»åŠ åˆ°é›†ç¾¤æ—¶ä½¿ç”¨è¿™ä¸ªæ–°çš„自定义角色。 + +以下用户å¯ä»¥é”定角色： + +- 任何分é…了`管ç†å‘˜`全局æƒé™çš„用户。 +- 任何分é…了带有`管ç†è§’色`æƒé™çš„`自定义用户`。 + + +## é”定/解é”角色 + +如果è¦é˜²æ­¢å°†è§’色分é…给用户，å¯ä»¥å°†å…¶è®¾ç½®ä¸º`é”定`状æ€ã€‚ + +ä½ å¯ä»¥åœ¨ä¸¤ç§æƒ…况下é”定角色： + +- [添加自定义角色](custom-roles.md)时。 +- 编辑现有角色时（è§ä¸‹æ–‡ï¼‰ã€‚ + +集群角色和项目/命å空间角色å¯ä»¥é”定，而全局角色ä¸èƒ½é”定。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**角色**。 +1. 转到**集群**选项å¡æˆ–**项目或命å空间**选项å¡ã€‚ +1. 找到è¦é”定（或解é”）的角色，选择 **â‹® > 编辑é…ç½®**。 +1. 从**é”定**选项中，选择**是** 或**å¦**。然åŽç‚¹å‡»**ä¿å­˜**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md new file mode 100644 index 00000000000..1f56b62762f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md @@ -0,0 +1,126 @@ +--- +title: Pod 安全标准 (PSS) å’Œ Pod 安全准入 (PSA) +--- + +[Pod 安全标准 (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/) å’Œ [Pod 安全准入 (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) 为大é‡å·¥ä½œè´Ÿè½½å®šä¹‰äº†å®‰å…¨é™åˆ¶ã€‚ +它们在 Kubernetes v1.23 中å¯ç”¨å¹¶é»˜è®¤æ‰“开，并在 Kubernetes v1.25 åŠæ›´é«˜ç‰ˆæœ¬ä¸­æ›¿æ¢äº† [Pod Security Policies (PSP)](https://kubernetes.io/docs/concepts/security/pod-security-policy/)。 + +PSS 定义了工作负载的安全级别。PSA æ述了 Pod 安全上下文和相关字段的è¦æ±‚。PSA å‚考 PSS 级别æ¥å®šä¹‰å®‰å…¨é™åˆ¶ã€‚ + +## å‡çº§åˆ° Pod 安全标准 (PSS) + +ç¡®ä¿å°†æ‰€æœ‰ PSP 都è¿ç§»åˆ°äº†å¦ä¸€ä¸ªå·¥ä½œè´Ÿè½½å®‰å…¨æœºåˆ¶ï¼ŒåŒ…括将你当å‰çš„ PSP 映射到 Pod 安全标准，以便使用 [PSA 控制器](https://kubernetes.io/docs/concepts/security/pod-security-admission/)执行。如果 PSA 控制器ä¸èƒ½æ»¡è¶³ä¼ä¸šçš„所有需求，建议你使用策略引擎，例如 [OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper)ã€[Kubewarden](https://www.kubewarden.io/)ã€[Kyverno](https://kyverno.io/) 或 [NeuVector](https://neuvector.com/)。有关如何è¿ç§» PSP 的更多信æ¯ï¼Œè¯·å‚阅你选择的策略引擎的文档。 + +:::caution +必须在删除 PodSecurityPolicy 对象_之å‰_添加新的策略执行机制。å¦åˆ™ï¼Œä½ å¯èƒ½ä¼šä¸ºé›†ç¾¤å†…的特æƒå‡çº§æ”»å‡»åˆ›é€ æœºä¼šã€‚ +::: + +### 从 Rancher 维护的应用程åºå’Œå¸‚场工作负载中删除 PodSecurityPolicies {#remove-psp-rancher-workloads} + +Rancher v2.7.2 æ供了 Rancher 维护的 Helm Chart 的新主è¦ç‰ˆæœ¬ã€‚v102.x.y å…许你删除与以å‰çš„ Chart 版本一起安装的 PSP。这个新版本使用标准化的 `global.cattle.psp.enabled` 开关（默认关闭）替æ¢äº†éžæ ‡å‡†çš„ PSP 开关。 + +你必须在_ä»ä½¿ç”¨ Kubernetes v1.24_ 时执行以下步骤： +1. æ ¹æ®éœ€è¦é…ç½® PSA 控制器。你å¯ä»¥ä½¿ç”¨ Rancher 的内置 [PSA é…置模æ¿](#psa-config-templates)，或创建自定义模æ¿å¹¶å°†å…¶åº”用于正在è¿ç§»çš„集群。 + +1. 将活动的 PSP 映射到 Pod 安全标准： + 1. 查看集群中哪些 PSP ä»å¤„于活动状æ€ï¼š + :::caution + 此策略å¯èƒ½ä¼šé”™è¿‡å½“å‰æœªè¿è¡Œçš„工作负载，例如 CronJobsã€å½“å‰ç¼©æ”¾ä¸ºé›¶çš„工作负载或尚未推出的工作负载。 + ::: + + ```shell + kubectl get pods \ + --all-namespaces \ + --output jsonpath='{.items[*].metadata.annotations.kubernetes\.io\/psp}' \ + | tr " " "\n" | sort -u + ``` + + 1. 按照[å°† PSP 映射到 Pod 安全标准](https://kubernetes.io/docs/reference/access-authn-authz/psp-to-pod-security-standards/)çš„ Kubernetes 指å—å°† PSS 应用于ä¾èµ– PSP 的工作负载。有关详细信æ¯ï¼Œè¯·å‚阅[从 PodSecurityPolicy è¿ç§»åˆ°å†…ç½® PodSecurity Admission 控制器](https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/)。 + +1. è¦ä»Ž Rancher Chart 中删除 PSP，请在å‡çº§åˆ° Kubernetes v1.25 _之å‰_å°† Chart å‡çº§åˆ°æœ€æ–°çš„ v102.x.y ç‰ˆæœ¬ã€‚ç¡®ä¿ **Enable PodSecurityPolicies** 选项**å·²ç¦ç”¨**。这将删除与以å‰çš„ Chart 版本一起安装的所有 PSP。 + +:::info é‡è¦æ示 +如果你想将 Chart å‡çº§åˆ° v102.x.y，但ä¸æ‰“算将集群å‡çº§åˆ° Kubernetes v1.25 和弃用 PSP，请确ä¿ä¸ºæ¯ä¸ªè¦å‡çº§çš„ Chart 选择 **Enable PodSecurityPolicies** 选项。 +::: + +### 在 Kubernetes v1.25 å‡çº§åŽæ¸…ç†ç‰ˆæœ¬ + +如果你在删除 Chart çš„ PSP æ—¶é‡åˆ°é—®é¢˜ï¼Œæˆ–者 Chart ä¸åŒ…å«ç”¨äºŽåˆ é™¤ PSP 的内置机制，Chart å‡çº§æˆ–删除å¯èƒ½ä¼šå¤±è´¥å¹¶æ˜¾ç¤ºå¦‚下错误消æ¯ï¼š +```console +Error: UPGRADE FAILED: resource mapping not found for name: "" namespace: "" from "": no matches for kind "PodSecurityPolicy" in version "policy/v1beta1" +ensure CRDs are installed first +``` + +Helm å°è¯•åœ¨é›†ç¾¤ä¸­æŸ¥è¯¢å­˜å‚¨åœ¨å…ˆå‰ç‰ˆæœ¬çš„æ•°æ® blob 中的对象时，就会å‘生这ç§æƒ…况。è¦æ¸…ç†è¿™äº›ç‰ˆæœ¬å¹¶é¿å…此错误，请使用 `helm-mapkubeapis` Helm æ’件。è¦è¯¦ç»†äº†è§£ `helm-mapkubeapis`ã€å®ƒçš„工作原ç†ä»¥åŠå¦‚何针对你的用例进行微调，请å‚阅 [Helm 官方文档](https://github.com/helm/helm-mapkubeapis#readme)。 + +请注æ„，Helm æ’件安装在你è¿è¡Œå‘½ä»¤çš„机器本地。因此，请确ä¿ä»ŽåŒä¸€å°æœºå™¨è¿è¡Œå®‰è£…和清ç†ã€‚ + +#### 安装 `helm-mapkubeapis` + +1. 在打算使用 `helm-mapkubeapis` 的机器上打开你的终端并安装æ’件： + ```shell + helm plugin install https://github.com/helm/helm-mapkubeapis + ``` + + 你将看到类似于以下的输出： + ```console + Downloading and installing helm-mapkubeapis v0.4.1 ... + https://github.com/helm/helm-mapkubeapis/releases/download/v0.4.1/helm-mapkubeapis_0.4.1_darwin_amd64.tar.gz + Installed plugin: mapkubeapis + ``` + + :::info é‡è¦æ示 + ç¡®ä¿ `helm-mapkubeapis` æ’件至少为 v0.4.1，因为旧版本_ä¸_支æŒèµ„æºåˆ é™¤ã€‚ + ::: + +1. 验è¯æ’件是å¦å·²æ­£ç¡®å®‰è£…： + ```shell + helm mapkubeapis --help + ``` + + 你将看到类似于以下的输出： + ```console + Map release deprecated or removed Kubernetes APIs in-place + + Usage: + mapkubeapis [flags] RELEASE + + Flags: + --dry-run simulate a command + -h, --help help for mapkubeapis + --kube-context string name of the kubeconfig context to use + --kubeconfig string path to the kubeconfig file + --mapfile string path to the API mapping file + --namespace string namespace scope of the release + ``` + +#### 清ç†æŸå的版本 + +安装 `helm-mapkubeapis` æ’件åŽï¼Œæ¸…ç†å‡çº§åˆ° Kubernetes v1.25 åŽæŸå的版本。 + +1. 打开你的首选终端并通过è¿è¡Œ `kubectl cluster-info` ç¡®ä¿ç»ˆç«¯å·²è¿žæŽ¥åˆ°æ‰€éœ€é›†ç¾¤ã€‚ + +1. è¿è¡Œ `helm list --all-namespaces` 列出你在集群中安装的所有版本。 + +1. 通过è¿è¡Œ `helm mapkubeapis --dry-run --namespace ` 为è¦æ¸…ç†çš„æ¯ä¸ªç‰ˆæœ¬æ‰§è¡Œè¯•è¿è¡Œã€‚ä½ å¯ä»¥é€šè¿‡æ­¤å‘½ä»¤çš„结果了解è¦æ›¿æ¢æˆ–删除哪些资æºã€‚ + +1. 最åŽï¼Œåœ¨æŸ¥çœ‹æ›´æ”¹åŽï¼Œä½¿ç”¨ `helm mapkubeapis --namespace ` 执行完整è¿è¡Œã€‚ + +#### å°† Chart å‡çº§åˆ°æ”¯æŒ Kubernetes v1.25 的版本 + +清ç†äº†å…·æœ‰ PSP 的所有版本åŽï¼Œä½ å°±å¯ä»¥ç»§ç»­å‡çº§äº†ã€‚对于 Rancher 维护的工作负载，请按照本文档[从 Rancher 维护的应用程åºå’Œå¸‚场工作负载中删除 PodSecurityPolicies](#remove-psp-rancher-workloads) 部分中的步骤进行æ“作。 +如果工作负载ä¸æ˜¯ç”± Rancher 维护的，请å‚阅对应的æ供商的文档。 + +:::caution +ä¸è¦è·³è¿‡æ­¤æ­¥éª¤ã€‚与 Kubernetes v1.25 ä¸å…¼å®¹çš„应用程åºä¸èƒ½ä¿è¯åœ¨æ¸…ç†åŽæ­£å¸¸å·¥ä½œã€‚ +::: + +## Pod 安全准入é…ç½®æ¨¡æ¿ {#psa-config-templates} + +Rancher æ供了 PSA é…置模æ¿ã€‚它们是å¯ä»¥åº”用到集群的预定义安全é…置。Rancher 管ç†å‘˜ï¼ˆæˆ–具有æƒé™çš„人员）å¯ä»¥[创建ã€ç®¡ç†å’Œç¼–辑](./psa-config-templates.md) PSA 模æ¿ã€‚ + +### å— PSA é™åˆ¶çš„集群上的 Rancher + +Rancher system 命å空间也å—到 PSA 模æ¿æè¿°çš„é™åˆ¶æ€§å®‰å…¨ç­–略的影å“。你需è¦åœ¨åˆ†é…模æ¿åŽè±å… Rancher çš„ system 命å空间，å¦åˆ™é›†ç¾¤å°†æ— æ³•æ­£å¸¸è¿è¡Œã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [Pod 安全准入 (PSA) é…置模æ¿](./psa-config-templates.md#è±å…必须的-rancher-命å空间)。 + +有关è¿è¡Œ Rancher 所需的所有è±å…的完整文件，请å‚阅此[准入é…置示例](../../../reference-guides/rancher-security/psa-restricted-exemptions.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md new file mode 100644 index 00000000000..c3b4d6252f4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md @@ -0,0 +1,138 @@ +--- +title: Pod 安全准入 (PSA) é…ç½®æ¨¡æ¿ +--- + +[Pod Security admission (PSA)](./pod-security-standards.md) é…置模æ¿æ˜¯ Rancher è‡ªå®šä¹‰èµ„æº (CRD)，在 Rancher v2.7.2 åŠæ›´é«˜ç‰ˆæœ¬ä¸­å¯ç”¨ã€‚这些模æ¿æ供了å¯åº”用于集群的预定义安全é…置： + +- `rancher-privileged`：最宽æ¾çš„é…置。它ä¸é™åˆ¶ä»»ä½• Pod 行为，å…许已知的æƒé™å‡çº§ã€‚该策略没有è±å…。 +- `rancher-restricted`：严格é™åˆ¶çš„é…置，éµå¾ªå½“å‰åŠ å›º pod 的最佳实践。你必须对 Rancher 组件进行[命å空间级别è±å…](./pod-security-standards.md#å—-psa-é™åˆ¶çš„集群上的-rancher)。 + +## åˆ†é… Pod 安全准入 (PSA) é…ç½®æ¨¡æ¿ + +ä½ å¯ä»¥åœ¨åˆ›å»ºä¸‹æ¸¸é›†ç¾¤çš„åŒæ—¶åˆ†é… PSA 模æ¿ã€‚你还å¯ä»¥é€šè¿‡é…置现有集群æ¥æ·»åŠ æ¨¡æ¿ã€‚ + +### 在集群创建期间分é…æ¨¡æ¿ + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**按钮。 +1. 选择æ供商。 +1. 在**集群: 创建**页é¢ä¸Šï¼Œè½¬åˆ°**åŸºæœ¬ä¿¡æ¯ > 安全**。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**创建**。 + +### 将模æ¿åˆ†é…给现有集群 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**ä¿å­˜**。 + +### 加固集群 + +如果选择 **rancher-restricted** 模æ¿ä½†ä¸é€‰æ‹© **CIS é…置文件**，你将无法满足 CIS Benchmark。有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 加固指å—](../../../pages-for-subheaders/rke2-hardening-guide.md)。 + + + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**按钮。 +1. 选择æ供商。 +1. 在**添加集群**页é¢ä¸Šçš„**集群选项**下，å•å‡» **高级选项**。 +1. 在 **PSA é…置模æ¿**下拉èœå•ä¸­ï¼Œé€‰æ‹©è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**创建**。 + +### 将模æ¿åˆ†é…给现有集群 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**表中找到è¦æ›´æ–°çš„集群，点击 **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 在**编辑集群**页é¢ä¸Šï¼Œè½¬åˆ°**集群选项 > 高级选项**。 +1. 在 **PSA é…置模æ¿**中，选择è¦åˆ†é…的模æ¿ã€‚ +1. å•å‡»**ä¿å­˜**。 + + + + +## 添加或编辑 Pod 安全准入 (PSA) é…ç½®æ¨¡æ¿ + +如果你拥有管ç†å‘˜æƒé™ï¼Œåˆ™å¯ä»¥é€šè¿‡åˆ›å»ºå…¶ä»– PSA 模æ¿æˆ–编辑现有模æ¿æ¥è‡ªå®šä¹‰å®‰å…¨é™åˆ¶å’Œæƒé™ã€‚ + +:::caution +如果编辑使用中的现有 PSA 模æ¿ï¼Œæ›´æ”¹å°†åº”用​​于已分é…给该模æ¿çš„所有集群。 +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 点击**高级选项**打开下拉èœå•ã€‚ +1. 选择 **Pod 安全准入**。 +1. 找到è¦ä¿®æ”¹çš„模æ¿ï¼Œç‚¹å‡» **â‹®**。 +1. 选择**编辑é…ç½®**æ¥ç¼–辑模æ¿ã€‚ +1. 完æˆé…置编辑åŽï¼Œå•å‡»**ä¿å­˜**。 + +### å…许éžç®¡ç†å‘˜ç”¨æˆ·ç®¡ç† PSA æ¨¡æ¿ + +如果你想å…许其他用户管ç†æ¨¡æ¿ï¼Œä½ å¯ä»¥å°†è¯¥ç”¨æˆ·ç»‘定到一个角色，并为该角色授予 `management.cattle.io/podsecurityadmissionconfigurationtemplates` 上的所有æ“作 (`"*"`)。 + +:::caution +绑定到上述æƒé™çš„用户都能够更改使用该 PSA 模æ¿çš„_所有_托管集群的é™åˆ¶çº§åˆ«ï¼ŒåŒ…括用户没有æƒé™çš„集群。 +::: + +## è±å…必须的 Rancher 命å空间 + +在默认执行é™åˆ¶æ€§å®‰å…¨ç­–略的 Kubernetes 集群上è¿è¡Œ Rancher 时，你需è¦[è±å…以下命å空间](#è±å…命å空间)，å¦åˆ™è¯¥ç­–ç•¥å¯èƒ½ä¼šé˜»æ­¢ Rancher system pod 正常è¿è¡Œã€‚ + +- `calico-apiserver` +- `calico-system` +- `cattle-alerting` +- `cattle-csp-adapter-system` +- `cattle-epinio-system` +- `cattle-externalip-system` +- `cattle-fleet-local-system` +- `cattle-fleet-system` +- `cattle-gatekeeper-system` +- `cattle-global-data` +- `cattle-global-nt` +- `cattle-impersonation-system` +- `cattle-istio` +- `cattle-istio-system` +- `cattle-logging` +- `cattle-logging-system` +- `cattle-monitoring-system` +- `cattle-neuvector-system` +- `cattle-prometheus` +- `cattle-sriov-system` +- `cattle-system` +- `cattle-ui-plugin-system` +- `cattle-windows-gmsa-system` +- `cert-manager` +- `cis-operator-system` +- `fleet-default` +- `ingress-nginx` +- `istio-system` +- `kube-node-lease` +- `kube-public` +- `kube-system` +- `longhorn-system` +- `rancher-alerting-drivers` +- `security-scan` +- `tigera-operator` + +Rancherã€Rancher 拥有的一些 Chart ä»¥åŠ RKE2 å’Œ K3s å‘行版都使用这些命å空间。列出的命å空间的一个å­é›†å·²ç»åœ¨å†…置的 Rancher `rancher-restricted` 策略中被è±å…，用于下游集群。有关è¿è¡Œ Rancher 所需的所有è±å…的完整模æ¿ï¼Œè¯·å‚阅此[准入é…置示例](../../../reference-guides/rancher-security/psa-restricted-exemptions.md)。 + +## è±å…命å空间 + +如果你将 `rancher-restricted` 模æ¿åˆ†é…给集群，默认情况下，é™åˆ¶ä¼šåœ¨å‘½å空间级别应用于整个集群。è¦åœ¨æ­¤é«˜åº¦å—é™çš„策略下è±å…特定的命å空间，执行以下æ“作： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 点击**高级选项**打开下拉èœå•ã€‚ +1. 选择 **Pod 安全准入**。 +1. 找到è¦ä¿®æ”¹çš„模æ¿ï¼Œç‚¹å‡» **â‹®**。 +1. 选择**编辑é…ç½®**。 +1. 选中**è±å…**下的**命å空间**å¤é€‰æ¡†ä»¥ç¼–辑**命å空间**字段。 +1. è±å…命å空间åŽï¼Œå•å‡»**ä¿å­˜**。 + +:::note +你需è¦æ›´æ–°ç›®æ ‡é›†ç¾¤æ‰èƒ½è®©æ–°æ¨¡æ¿åœ¨é›†ç¾¤ä¸­ç”Ÿæ•ˆã€‚è¦è§¦å‘更新，在ä¸æ›´æ”¹å€¼çš„情况下编辑和ä¿å­˜é›†ç¾¤ã€‚ +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md new file mode 100644 index 00000000000..51b24396cbd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md @@ -0,0 +1,74 @@ +--- +title: 备份 Docker 安装的 Rancher +--- + +æˆåŠŸä½¿ç”¨ Docker 安装 Rancher åŽï¼Œæˆ‘们建议你定期创建备份。最近创建的备份能让你在æ„外ç¾éš¾å‘生åŽå¿«é€Ÿè¿›è¡Œæ¢å¤ã€‚ + +## åœ¨ä½ å¼€å§‹å‰ + +在创建备份的过程中，你将输入一系列命令。请使用环境中的数æ®æ›¿æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。以下是带有å ä½ç¬¦çš„命令示例： + +``` +docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher +``` + +在该命令中，`` 是数æ®å®¹å™¨å’Œå¤‡ä»½åˆ›å»ºæ—¥æœŸçš„å ä½ç¬¦ï¼ˆä¾‹å¦‚，`9-27-18`）。 + +请交å‰å‚考下方的图片和表格，了解获å–æ­¤å ä½ç¬¦æ•°æ®çš„方法。在开始[以下步骤](#创建备份)之å‰ï¼Œè¯·è®°ä¸‹æˆ–å¤åˆ¶è¿™äº›ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <RANCHER_CONTAINER_TAG> å’Œ <RANCHER_CONTAINER_NAME> + +![å ä½ç¬¦å‚考](/img/placeholder-ref.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | 首次安装拉å–çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.0.5` | 你为其创建备份的 Rancher 版本。 | +| `` | `9-27-18` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ ``。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## 创建备份 + +此步骤将创建一个备份文件。如果 Rancher é‡åˆ°ç¾éš¾æƒ…况，你å¯ä»¥ä½¿ç”¨è¯¥å¤‡ä»½æ–‡ä»¶è¿›è¡Œè¿˜åŽŸã€‚ + + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. åœæ­¢å½“å‰è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸ºä½ çš„ Rancher 容器的å称： + + ``` + docker stop + ``` +1. è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä»Žåˆšæ‰åœæ­¢çš„ Rancher 容器创建一个数æ®å®¹å™¨ã€‚请替æ¢å‘½ä»¤ä¸­çš„å ä½ç¬¦ï¼š + + ``` + docker create --volumes-from --name rancher-data- rancher/rancher: + ``` + +1. 从你刚刚创建的数æ®å®¹å™¨ï¼ˆrancher-data-<DATE>）中，创建一个备份 tar 包（rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz）。替æ¢å ä½ç¬¦æ¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ``` + docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup:z busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **步骤结果**：å±å¹•ä¸Šå°†è¿è¡Œå‘½ä»¤æµã€‚ + +1. 输入 `ls` 命令，确认备份压缩包已创建æˆåŠŸã€‚压缩包的å称格å¼ç±»ä¼¼ `rancher-data-backup--.tar.gz`。 + +1. 将备份压缩包移动到 Rancher Server 外的安全ä½ç½®ã€‚然åŽä»Ž Rancher Server 中删除 `rancher-data-` å’Œ `busybox-backup-` 容器。 + + ``` + docker rm rancher-data- + docker rm busybox-backup- + ``` + +1. é‡å¯ Rancher Server。将 `` 替æ¢ä¸º Rancher 容器的å称： + + ``` + docker start + ``` + +**结果**：创建了 Rancher Server æ•°æ®çš„备份压缩包。如果你需è¦æ¢å¤å¤‡ä»½æ•°æ®ï¼Œè¯·å‚è§[æ¢å¤å¤‡ä»½ï¼šDocker 安装](restore-docker-installed-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md new file mode 100644 index 00000000000..11b0dc2a9fb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md @@ -0,0 +1,299 @@ +--- +title: 备份集群 +--- + +在 Rancher UI 中，你å¯ä»¥è½»æ¾å¤‡ä»½å’Œæ¢å¤ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)çš„ etcd。 + +Rancher 建议为所有生产集群é…置定期 `etcd` 快照。此外，你还å¯ä»¥åˆ›å»ºå•æ¬¡å¿«ç…§ã€‚ + +etcd æ•°æ®åº“的快照会ä¿å­˜åœ¨ [etcd 节点](#本地备份目标)或 [S3 兼容目标](#s3-备份目标)上。é…ç½® S3 的好处是，如果所有 etcd 节点都丢失了，你的快照会ä¿å­˜åˆ°è¿œç«¯å¹¶èƒ½ç”¨äºŽæ¢å¤é›†ç¾¤ã€‚ + +## å¿«ç…§å·¥ä½œåŽŸç† + +### 快照组件 + + + + +Rancher 创建快照时，快照里包括三个组件： + +- etcd ä¸­çš„é›†ç¾¤æ•°æ® +- Kubernetes 版本 +- `cluster.yml` å½¢å¼çš„集群é…ç½® + +由于 Kubernetes 版本现在包å«åœ¨å¿«ç…§ä¸­ï¼Œå› æ­¤ä½ å¯ä»¥å°†é›†ç¾¤æ¢å¤åˆ°åŽŸæœ¬çš„ Kubernetes 版本。 + + + + +Rancher 将快照创建任务委托给下游 Kubernetes 引擎。Kubernetes 引擎创建快照时包括了三个组件： + +- etcd ä¸­çš„é›†ç¾¤æ•°æ® +- Kubernetes 版本 +- 集群é…ç½® + +由于 Kubernetes 版本包å«åœ¨å¿«ç…§ä¸­ï¼Œå› æ­¤ä½ å¯ä»¥å°†é›†ç¾¤è¿˜åŽŸåˆ°ä¹‹å‰çš„ Kubernetes 版本，åŒæ—¶è¿˜åŽŸ etcd 快照。 + + + + +如果你需è¦ä½¿ç”¨å¿«ç…§æ¢å¤é›†ç¾¤ï¼Œå¿«ç…§çš„多个组件å…许你选择： + +- **ä»…æ¢å¤ etcd 内容**：类似于在 Rancher v2.4.0 之å‰ç‰ˆæœ¬ä¸­çš„使用快照æ¢å¤ã€‚ +- **æ¢å¤ etcd å’Œ Kubernetes 版本**：如果 Kubernetes å‡çº§å¯¼è‡´é›†ç¾¤å¤±è´¥ï¼Œå¹¶ä¸”你没有更改任何集群é…置，则应使用此选项。 +- **æ¢å¤ etcdã€Kubernetes 版本和集群é…ç½®**：如果你在å‡çº§æ—¶åŒæ—¶æ›´æ”¹äº† Kubernetes 版本和集群é…置，则应使用此选项。 + +建议你在执行é…置更改或å‡çº§ä¹‹å‰åˆ›å»ºæ–°å¿«ç…§ã€‚ + + +### 从 etcd 节点生æˆå¿«ç…§ + + + + +集群中的æ¯ä¸ª etcd 节点都会检查 etcd 集群的å¥åº·çŠ¶å†µã€‚如果节点报告 etcd 集群是å¥åº·çš„，则会从中创建一个快照，并å¯é€‰æ‹©ä¸Šä¼ åˆ° S3。 + +快照存储在 `/opt/rke/etcd-snapshots` 中。如果该目录在节点上é…置为共享挂载，它将被覆盖。由于所有 etcd 节点都会上传快照并ä¿ç•™æœ€åŽä¸€ä¸ªï¼Œå› æ­¤ S3 上始终会ä¿ç•™æœ€åŽä¸€ä¸ªä¸Šä¼ çš„节点的快照。 + +在存在多个 etcd 节点的情况下，任何快照都是在集群å¥åº·æ£€æŸ¥é€šè¿‡åŽåˆ›å»ºçš„，因此这些快照å¯ä»¥è®¤ä¸ºæ˜¯ etcd 集群中数æ®çš„有效快照。 + + + + +快照是默认å¯åŠ¨çš„。 + +快照目录默认为 `/var/lib/rancher//server/db/snapshots`，其中 `` å¯ä»¥æ˜¯ `rke2` 或 `k3s`。 + +在 RKE2 中，快照会存储在æ¯ä¸ª etcd 节点上。如果你有多个 etcd 或 etcd + control plane 节点，你将拥有本地 etcd 快照的多个副本。 + + + + +### 快照命å规则 + + + + +快照的å称是自动生æˆçš„。在使用 RKE CLI 创建一次性快照时，你å¯ä»¥ä½¿ç”¨ `--name` 选项æ¥æŒ‡å®šå¿«ç…§çš„å称。 + +Rancher 在创建 RKE 集群的快照时，快照å称是基于快照创建类型（手动快照或定期快照）和目标（快照是ä¿å­˜åœ¨æœ¬åœ°è¿˜æ˜¯ä¸Šä¼ åˆ° S3）决定的。命å规则如下： + +- `m` 代表手动 +- `r` 代表定期 +- `l` 代表本地 +- `s` 代表 S3 + +å¿«ç…§å称示例如下： + +- c-9dmxz-rl-8b2cx +- c-9dmxz-ml-kr56m +- c-9dmxz-ms-t6bjb +- c-9dmxz-rs-8gxc8 + + + + +快照的å称是自动生æˆçš„。使用 RKE2 或 K3s CLI 创建一次性快照时，`--name` 选项å¯ç”¨äºŽè¦†ç›–快照的基本å称。 + +Rancher 在创建 RKE2 或 K3s 集群的快照时，快照å称是基于快照创建类型（手动快照或定期快照）和目标（快照是ä¿å­˜åœ¨æœ¬åœ°è¿˜æ˜¯ä¸Šä¼ åˆ° S3）决定的。命å规则如下： + +`--` + +``：`--name` 设置的基本å称，å¯ä»¥æ˜¯ä»¥ä¸‹ä¹‹ä¸€ï¼š + +- `etcd-snapshot`：ä½äºŽå®šæœŸå¿«ç…§å‰é¢ +- `on-demand`：ä½äºŽæ‰‹åŠ¨æŒ‰éœ€å¿«ç…§ä¹‹å‰ + +``：创建快照的节点的å称。 + +``：快照创建日期的 unix 时间戳。 + +å¿«ç…§å称示例如下： + +- `on-demand-my-super-rancher-k8s-node1-1652288934` +- `on-demand-my-super-rancher-k8s-node2-1652288936` +- `etcd-snapshot-my-super-rancher-k8s-node1-1652289945` +- `etcd-snapshot-my-super-rancher-k8s-node2-1652289948` + + + + +### 从快照æ¢å¤çš„å·¥ä½œåŽŸç† + + + + +在æ¢å¤æ—¶ä¼šå‘生以下过程： + +1. 如果é…置了 S3，则从 S3 检索快照。 +2. 如果快照压缩了，则将快照解压缩。 +3. 集群中的一个 etcd 节点会将该快照文件æ供给其他节点。 +4. 其他 etcd 节点会下载快照并验è¯æ ¡éªŒå’Œï¼Œä»¥ä¾¿éƒ½èƒ½ä½¿ç”¨ç›¸åŒçš„快照进行æ¢å¤ã€‚ +5. 集群已æ¢å¤ï¼Œæ¢å¤åŽçš„æ“作将在集群中完æˆã€‚ + + + + +在还原时，Rancher 会æ供几组执行还原的计划。期间将包括以下阶段： + +- Started +- Shutdown +- Restore +- RestartCluster +- Finished + +如果 etcd 快照还原失败，阶段将设置为 `Failed`。 + +1. 收到 etcd 快照还原请求åŽï¼Œæ ¹æ® `restoreRKEConfig` å调集群é…置和 Kubernetes 版本。 +1. 该阶段设置为 `Started`。 +1. 该阶段设置为 `Shutdown`，并使用è¿è¡Œ `killall.sh` 脚本的计划æ¥å…³é—­æ•´ä¸ªé›†ç¾¤ã€‚一个新的åˆå§‹èŠ‚点会被选举出æ¥ã€‚如果还原的快照是本地快照，则选择该快照所在的节点作为åˆå§‹èŠ‚点。如果使用 S3 还原快照，将使用现有的åˆå§‹èŠ‚点。 +1. 该阶段设置为 `Restore`，并且快照将还原到åˆå§‹èŠ‚点上。 +1. 该阶段设置为 `RestartCluster`，集群将é‡å¯å¹¶é‡æ–°åŠ å…¥åˆ°å…·æœ‰æ–°è¿˜åŽŸçš„快照信æ¯çš„æ–°åˆå§‹èŠ‚点。 +1. 该阶段设置为 `Finished`，集群被视为已æˆåŠŸè¿˜åŽŸã€‚`cattle-cluster-agent` å°†é‡æ–°è¿žæŽ¥ï¼Œé›†ç¾¤å°†å®Œæˆå调。 + + + + +## é…置定期快照 + + + + +选择创建定期快照的频率以åŠè¦ä¿ç•™çš„快照数é‡ã€‚时间的å•ä½æ˜¯å°æ—¶ã€‚用户å¯ä»¥ä½¿ç”¨æ—¶é—´æˆ³å¿«ç…§è¿›è¡Œæ—¶é—´ç‚¹æ¢å¤ã€‚ + +默认情况下，[Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)会é…置为创建定期快照（ä¿å­˜åˆ°æœ¬åœ°ç£ç›˜ï¼‰ã€‚为防止本地ç£ç›˜æ•…障，建议使用 [S3 目标](#s3-备份目标)或å¤åˆ¶ç£ç›˜ä¸Šçš„路径。 + +在集群é…置或编辑集群期间，å¯ä»¥åœ¨**集群选项**的高级部分中找到快照的é…置。点击**显示高级选项**。 + +在集群的**高级选项**中å¯ä»¥é…置以下选项： + +| 选项 | æè¿° | 默认值 | +| --- | ---| --- | +| etcd 快照备份目标 | 选择è¦ä¿å­˜å¿«ç…§çš„ä½ç½®ã€‚å¯ä»¥æ˜¯æœ¬åœ°æˆ– S3 | 本地 | +| å¯ç”¨å®šæœŸ etcd å¿«ç…§ | å¯ç”¨/ç¦ç”¨å®šæœŸå¿«ç…§ | 是 | +| 定期 etcd 快照的创建周期 | 定期快照之间的间隔（以å°æ—¶ä¸ºå•ä½ï¼‰ | 12 å°æ—¶ | +| 定期 etcd 快照的ä¿ç•™æ•°é‡ | è¦ä¿ç•™çš„å¿«ç…§æ•°é‡ | 6 | + + + + +设置创建定期快照的方å¼ä»¥åŠè¦ä¿ç•™çš„快照数é‡ã€‚该计划采用传统的 Cron æ ¼å¼ã€‚ä¿ç•™ç­–略规定了在æ¯ä¸ªèŠ‚点上è¦ä¿ç•™çš„匹é…å称的快照数é‡ã€‚ + +默认情况下，[Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)从凌晨 12 ç‚¹å¼€å§‹æ¯ 5 å°æ—¶åˆ›å»ºä¸€æ¬¡å®šæœŸå¿«ç…§ï¼ˆä¿å­˜åˆ°æœ¬åœ°ç£ç›˜ï¼‰ã€‚为了防止本地ç£ç›˜æ•…障，建议使用 [S3 目标](#s3-备份目标)或å¤åˆ¶ç£ç›˜ä¸Šçš„路径。 + +在集群é…置或编辑集群期间，你å¯ä»¥åœ¨**集群é…ç½®**下找到快照é…置。å•å‡» **etcd**。 + +| 选项 | æè¿° | 默认值 | +| --- | ---| --- | +| å¯ç”¨å®šæœŸ etcd å¿«ç…§ | å¯ç”¨/ç¦ç”¨å®šæœŸå¿«ç…§ | 是 | +| 定期 etcd 快照的创建周期 | 定期快照的 Cron 计划 | `0 */5 * * *` | +| 定期 etcd 快照的ä¿ç•™æ•°é‡ | è¦ä¿ç•™çš„å¿«ç…§æ•°é‡ | 5 | + + + + +## å•æ¬¡å¿«ç…§ + + + + +除了定期快照之外，你å¯èƒ½è¿˜æƒ³åˆ›å»ºâ€œä¸€æ¬¡æ€§â€å¿«ç…§ã€‚例如，在å‡çº§é›†ç¾¤çš„ Kubernetes 版本之å‰ï¼Œæœ€å¥½å¤‡ä»½é›†ç¾¤çš„状æ€ä»¥é˜²æ­¢å‡çº§å¤±è´¥ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå¯¼èˆªåˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºä¸€æ¬¡æ€§å¿«ç…§çš„集群。 +1. å•å‡» **â‹® > æ‹æ‘„å¿«ç…§**。 + + + + +除了定期快照之外，你å¯èƒ½è¿˜æƒ³åˆ›å»ºâ€œä¸€æ¬¡æ€§â€å¿«ç…§ã€‚例如，在å‡çº§é›†ç¾¤çš„ Kubernetes 版本之å‰ï¼Œæœ€å¥½å¤‡ä»½é›†ç¾¤çš„状æ€ä»¥é˜²æ­¢å‡çº§å¤±è´¥ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå¯¼èˆªåˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºä¸€æ¬¡æ€§å¿«ç…§çš„集群。 +1. 导航至`å¿«ç…§`选项å¡ï¼Œç„¶åŽå•å‡»`ç«‹å³åˆ›å»ºå¿«ç…§` + +### åˆ›å»ºä¸€æ¬¡æ€§å¿«ç…§çš„å·¥ä½œåŽŸç† + +在创建一次性快照时，Rancher 会传递几组计划æ¥æ‰§è¡Œå¿«ç…§åˆ›å»ºã€‚期间将包括以下阶段： + +- Started +- RestartCluster +- Finished + +如果 etcd 快照创建失败，阶段将设置为 `Failed`。 + +1. 收到 etcd 快照创建请求。 +1. 该阶段设置为 `Started`。集群中的所有 etcd 节点都会根æ®é›†ç¾¤é…置收到创建 etcd 快照的计划。 +1. 该阶段设置为 `RestartCluster`，并且æ¯ä¸ª etcd 节点上的计划都将é‡ç½®ä¸º etcd 节点的原始计划。 +1. 该阶段设置为 `Finished`。 + + + + +**结果**：根æ®ä½ çš„[快照备份目标](#快照备份目标)创建一次性快照，并将其ä¿å­˜åœ¨é€‰å®šçš„备份目标中。 + +## 快照备份目标 + +Rancher 支æŒä¸¤ç§ä¸åŒçš„备份目标： + +- [本地目标](#本地备份目标) +- [S3 目标](#s3-备份目标) + +### 本地备份目标 + + + + +默认情况下会选择 `local` 备份目标。此选项的好处是ä¸éœ€è¦è¿›è¡Œå¤–部é…置。快照会在本地自动ä¿å­˜åˆ° [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中 etcd 节点的 `/opt/rke/etcd-snapshots` 中。所有定期快照都是按照é…置的时间间隔创建的。使用 `local` 备份目标的缺点是，如果å‘生全é¢ç¾éš¾å¹¶ä¸”丢失 _所有_ etcd 节点时，则无法æ¢å¤é›†ç¾¤ã€‚ + + + + +默认情况下会选择 `local` 备份目标。此选项的好处是ä¸éœ€è¦è¿›è¡Œå¤–部é…置。快照会自动ä¿å­˜åˆ° [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中的本地 etcd 节点上的 `/var/lib/rancher//server/db/snapshots` 中，其中 `` å¯ä»¥æ˜¯ `k3s` 或 `rke2`。所有定期快照å‡æŒ‰ç…§ Cron 计划进行。使用 `local` 备份目标的缺点是，如果å‘生全é¢ç¾éš¾å¹¶ä¸”丢失 _所有_ etcd 节点时，则无法æ¢å¤é›†ç¾¤ã€‚ + + + + +### S3 备份目标 + +`S3` 备份目标å…许用户é…置与 S3 兼容的åŽç«¯æ¥å­˜å‚¨å¿«ç…§ã€‚此选项的主è¦å¥½å¤„是，如果集群丢失了所有 etcd 节点，由于快照存储在外部，集群ä»ç„¶å¯ä»¥æ¢å¤ã€‚Rancher 推è使用 `S3` 备份这类外部目标。但是它的é…ç½®è¦æ±‚进行é¢å¤–çš„æ“作，这也应该被考虑在其中。由于 Rancher 会为群集é…置的 S3 存储桶/文件夹中列出的任何å¯ç”¨å¿«ç…§å¡«å……快照信æ¯ï¼Œå› æ­¤å»ºè®®ä½ ç¡®ä¿æ¯ä¸ªé›†ç¾¤éƒ½å…·æœ‰å”¯ä¸€çš„存储桶或文件夹。 + +| 选项 | æè¿° | å¿…å¡« | +|---|---|---| +| S3 存储桶å称 | 将存储备份的 S3 存储桶å称 | * | +| S3 区域 | 备份存储桶的 S3 区域 | | +| S3 区域端点 | 备份存储桶的 S3 区域端点 | * | +| S3 访问密钥 | 有æƒè®¿é—®å¤‡ä»½å­˜å‚¨æ¡¶çš„ S3 访问密钥 | * | +| S3 密文密钥 | 有æƒè®¿é—®å¤‡ä»½å­˜å‚¨æ¡¶çš„ S3 密文密钥 | * | +| 自定义 CA è¯ä¹¦ | 用于访问ç§æœ‰ S3 åŽç«¯çš„自定义è¯ä¹¦ | + +### 为 S3 使用自定义 CA è¯ä¹¦ + +备份快照å¯ä»¥å­˜å‚¨åœ¨è‡ªå®šä¹‰ `S3` 备份中，例如 [minio](https://min.io/)。如果 S3 åŽç«¯ä½¿ç”¨è‡ªç­¾å或自定义è¯ä¹¦ï¼Œè¯·ä½¿ç”¨`自定义 CA è¯ä¹¦`选项æ¥æ供自定义è¯ä¹¦ï¼Œä»Žè€Œè¿žæŽ¥åˆ° S3 åŽç«¯ã€‚ + +### 在 S3 中存储快照的 IAM æ”¯æŒ + +除了使用 API 凭è¯ä¹‹å¤–，`S3` 备份目标还支æŒå¯¹ AWS API 使用 IAM 身份验è¯ã€‚IAM 角色会授予应用在对 S3 存储进行 API 调用时的临时æƒé™ã€‚è¦ä½¿ç”¨ IAM 身份验è¯ï¼Œå¿…须满足以下è¦æ±‚： + +- 集群 etcd 节点必须具有实例角色，该角色具有对指定备份存储桶的读/写访问æƒé™ã€‚ +- 集群 etcd 节点必须对指定的 S3 端点具有网络访问æƒé™ã€‚ +- Rancher Server worker 节点必须具有实例角色，该实例角色具有对指定备份存储桶的读/写访问æƒé™ã€‚ +- Rancher Server worker 节点必须对指定的 S3 端点具有网络访问æƒé™ã€‚ + +è¦æŽˆäºˆåº”用对 S3 的访问æƒé™ï¼Œè¯·å‚阅[使用 IAM 角色å‘在 Amazon EC2 实例上è¿è¡Œçš„应用授予æƒé™](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html)çš„ AWS 文档。 + +## 查看å¯ç”¨å¿«ç…§ + +Rancher UI 中æ供了集群所有å¯ç”¨å¿«ç…§çš„列表： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦æŸ¥çœ‹å¿«ç…§çš„集群并å•å‡»å…¶å称。 +1. å•å‡»**å¿«ç…§**选项å¡æ¥æŸ¥çœ‹å·²ä¿å­˜å¿«ç…§çš„列表。这些快照包括创建时间的时间戳。 + +## 安全时间戳（RKE） + +快照文件带有时间戳，从而简化使用外部工具和脚本处ç†æ–‡ä»¶çš„过程。但在æŸäº›ä¸Ž S3 兼容的åŽç«¯ä¸­ï¼Œè¿™äº›æ—¶é—´æˆ³æ— æ³•ä½¿ç”¨ã€‚ + +添加了选项 `safe_timestamp` 以支æŒå…¼å®¹çš„文件å。当此标志设置为 `true` 时，快照文件å时间戳中的所有特殊字符都将被替æ¢ã€‚ + +此选项ä¸èƒ½ç›´æŽ¥åœ¨ UI 中使用，åªèƒ½é€šè¿‡`以 YAML 文件编辑`使用。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md new file mode 100644 index 00000000000..cf023f56e0a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md @@ -0,0 +1,92 @@ +--- +title: 备份 Rancher +--- + +在本节中，你将学习如何备份è¿è¡Œåœ¨ä»»ä½• Kubernetes 集群上的 Rancher。è¦å¤‡ä»½é€šè¿‡ Docker 安装的 Rancher，请å‚è§[å•èŠ‚点备份](back-up-docker-installed-rancher.md)。 + +`backup-restore` operator 需è¦å®‰è£…在 local 集群上，并且åªå¯¹ Rancher 应用进行备份。备份和æ¢å¤æ“作仅在本地 Kubernetes 集群中执行。 + +请知悉，`rancher-backup` operator çš„ 2.x.x 版本用于 Rancher v2.6.x。 + +:::caution + +当把备份æ¢å¤åˆ°ä¸€ä¸ªæ–°çš„ Rancher 设置中时，新设置的版本应该与备份的版本相åŒã€‚在æ¢å¤å¤‡ä»½æ—¶è¿˜åº”考虑 Kubernetes 的版本，因为集群中支æŒçš„ apiVersion 和备份文件中的 apiVersion å¯èƒ½ä¸åŒã€‚ + +::: + +### 先决æ¡ä»¶ + +Rancher 必须是 2.5.0 或更高版本。 + +请å‚è§[此处](migrate-rancher-to-new-cluster.md#2-使用-restore-自定义资æºæ¥è¿˜åŽŸå¤‡ä»½)获å–在 Rancher 2.6.3 中将现有备份文件æ¢å¤åˆ° v1.22 集群的帮助。 + +### 1. 安装 Rancher Backup Operator + +备份存储ä½ç½®æ˜¯ operator 级别的设置，所以需è¦åœ¨å®‰è£…或å‡çº§ `rancher backup` 应用时进行é…置。 + +备份文件的格å¼æ˜¯ `.tar.gz`。这些文件å¯ä»¥æŽ¨é€åˆ° S3 或 Minio，也å¯ä»¥å­˜å‚¨åœ¨ä¸€ä¸ªæŒä¹…å·ä¸­ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。Rancher Server è¿è¡Œåœ¨ `local` 集群中。 +1. å•å‡» **Apps > Charts**。 +1. 点击 **Rancher 备份**。 +1. å•å‡»**安装**。 +1. é…置默认存储ä½ç½®ã€‚如需获å–帮助，请å‚è§[存储é…ç½®](../../../reference-guides/backup-restore-configuration/storage-configuration.md)。 +1. å•å‡»**安装**。 + +:::note + +使用 `backup-restore` operator 执行æ¢å¤åŽï¼ŒFleet 中会出现一个已知问题：用于 `clientSecretName` å’Œ `helmSecretName` 的密文ä¸åŒ…å«åœ¨ Fleet çš„ Git 仓库中。请å‚è§[此处](../deploy-apps-across-clusters/fleet.md#故障排除)获得解决方法。 + +::: + +### 2. 执行备份 + +è¦æ‰§è¡Œå¤‡ä»½ï¼Œå¿…须创建 Backup 类型的自定义资æºã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Rancher 备份 > 备份**。 +1. å•å‡»**创建**。 +1. 使用表å•æˆ– YAML 编辑器创建 Backup。 +1. è¦ä½¿ç”¨è¯¥è¡¨å•é…ç½® Backup 详细信æ¯ï¼Œè¯·å•å‡»**创建**，然åŽå‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/backup-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md#备份)进行æ“作。 +1. è¦ä½¿ç”¨ YAML 编辑器，å•å‡»**创建 > 使用 YAML 文件创建**。输入 Backup YAML。这个示例 Backup 自定义资æºå°†åœ¨ S3 中创建加密的定期备份。这个应用使用 `credentialSecretNamespace` 值æ¥ç¡®å®šåœ¨å“ªé‡Œå¯»æ‰¾ S3 备份的密文： + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Backup + metadata: + name: s3-recurring-backup + spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 + ``` + + :::note + + 使用 YAML 编辑器创建 Backup 资æºæ—¶ï¼Œ`resourceSetName` 必须设置为 `rancher-resource-set`。 + + ::: + + 如需获得é…ç½® Backup 的帮助，请å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/backup-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md#备份)。 + + :::caution + + `rancher-backup` operator ä¸ä¿å­˜ `EncryptionConfiguration` 文件。创建加密备份时，必须ä¿å­˜ `EncryptionConfiguration` 文件的内容，而且在使用备份还原时必须使用åŒä¸€ä¸ªæ–‡ä»¶ã€‚ + + ::: + +1. å•å‡»**创建**。 + +**结果**：备份文件创建在 Backup 自定义资æºä¸­é…置的存储ä½ç½®ä¸­ã€‚执行还原时使用该文件的å称。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md new file mode 100644 index 00000000000..03b4a704185 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md @@ -0,0 +1,129 @@ +--- +title: 备份æ¢å¤ä½¿ç”¨æŒ‡å— +--- + +Rancher Backups Chart 是我们的ç¾éš¾æ¢å¤å’Œè¿ç§»è§£å†³æ–¹æ¡ˆã€‚æ­¤ Chart 用于备份 Kubernetes 资æºå¹¶å°†å…¶ä¿å­˜åˆ°å„ç§æŒä¹…存储ä½ç½®ã€‚ + +这个 Chart 是一个éžå¸¸ç®€å•çš„工具，适用于 Rancher 生æ€ç³»ç»Ÿçš„许多ä¸åŒé¢†åŸŸã€‚但是，也因此出现了未记录功能的边缘用例。本文档旨在强调 Rancher Backups 正确的用法，并讨论我们é‡åˆ°çš„一些边缘情况。 + +## 功能概述 + +### 备份 + +该 Operator å°† Chart 中的 resourceSet æ•èŽ·çš„所有资æºæ”¶é›†ä¸ºå†…存中的éžç»“构化对象。收集资æºåŽï¼Œèµ„æºçš„ tar 包将ä¿å­˜ä¸º JSON æ ¼å¼çš„清å•é›†åˆï¼Œç„¶åŽä¸Šä¼ åˆ°ç”¨æˆ·å®šä¹‰çš„对象存储。该备份å¯ä»¥æŒ‰é‡å¤è®¡åˆ’进行，也å¯ä»¥è¿›è¡ŒåŠ å¯†ã€‚由于æŸäº›èµ„æºæ˜¯æ•æ„Ÿçš„，并且值以未加密的明文形å¼å­˜å‚¨ï¼Œå› æ­¤æ­¤åŠ å¯†é€‰é¡¹å¾ˆé‡è¦ã€‚ + +有关é…置备份的选项（包括加密），请å‚阅[备份é…置文档](../../../reference-guides/backup-restore-configuration/backup-configuration.md)。 + +:::note + +如[备份 Rancher 文档](./back-up-rancher.md)所述，你必须手动ä¿å­˜åŠ å¯†é…置文件的内容，因为 Operator **ä¸ä¼š**备份它。 + +::: + +### 还原 + +有两ç§ä¸»è¦çš„还原场景：还原正在è¿è¡Œ Rancher 的集群以åŠè¿˜åŽŸæ–°é›†ç¾¤ã€‚åªæœ‰å°†å¤‡ä»½è¿˜åŽŸåˆ°è¯¥å¤‡ä»½çš„æºé›†ç¾¤ï¼Œä¸”在还原过程中å¯ç”¨äº† [`prune` 选项](../../../reference-guides/backup-restore-configuration/restore-configuration.md#还原过程中修剪)时，你æ‰èƒ½è¿˜åŽŸæ­£åœ¨è¿è¡Œ Rancher 的集群。还原具有与备份类似的输入。它需è¦å¤‡ä»½æ–‡ä»¶åã€encryptionConfigSecret å称和存储ä½ç½®ã€‚ + +资æºæŒ‰ä»¥ä¸‹é¡ºåºè¿˜åŽŸï¼š + +1. 自定义资æºå®šä¹‰ï¼ˆCRD） +2. é›†ç¾¤èŒƒå›´èµ„æº +3. 命åç©ºé—´èµ„æº + +有关é…置还原的选项，请å‚阅[还原é…置文档](../../../reference-guides/backup-restore-configuration/restore-configuration.md)。 + +### 资æºé›† + +ResourceSet 确定了 backup-restore-operator 在备份中收集哪些资æºã€‚它是一组 ResourceSelector，使用键/值对匹é…ã€æ­£åˆ™è¡¨è¾¾å¼åŒ¹é…或 Kubernetes 客户端 labelSelector æ¥å®šä¹‰é€‰æ‹©è¦æ±‚。 + +以下是å¯ç”¨äºŽ resourceSelector 的字段： + +- apiVersion +- excludeKinds +- excludeResourceNameRegexp +- kinds +- kindsRegexp +- labelSelectors +- namespaceRegexp +- namespaces +- resourceNameRegexp +- resourceNames + +Rancher Backups Chart 包å«äº†ä¸€ä¸ª[默认 resourceSet](https://github.com/rancher/backup-restore-operator/tree/release/v3.0/charts/rancher-backup/files/default-resourceset-contents)，它是安装 Chart 时附加到一个大型 resourceSet çš„ YAML 文件组åˆã€‚文件顺åºå¹¶ä¸é‡è¦ã€‚ä¸åŒç‰ˆæœ¬çš„ resourceSet å¯èƒ½æœ‰æ‰€ä¸åŒã€‚ + +:::caution + +如果你希望编辑 resourceSet，请在安装 Chart 之å‰è¿›è¡Œ**编辑**。 + +::: + +## 正确使用 + +本节概述了如何根æ®ç”¨ä¾‹æ­£ç¡®ä½¿ç”¨ Rancher Backups Chart。 + +### 所有案例 + +- Rancher Backups 必须安装在 local 集群上。 + - 注æ„：Rancher Backups åªä¼šå¤„ç†å®‰è£…了它的集群。它å¯èƒ½ä¼šè¿˜åŽŸ local 集群上的集群资æºï¼Œä½†ä¸ä¼šè”系或备份下游集群。 +- è¦è¿˜åŽŸçš„ Rancher 版本必须与备份中的 Rancher 版本匹é…。 +- 由于你å¯èƒ½éœ€è¦è¿˜åŽŸå·²è¿‡æ—¶çš„资æºï¼ˆè¦è¿˜åŽŸçš„ Kubernetes 版本已弃用的资æºï¼‰ï¼Œå› æ­¤ä½ éœ€è¦è€ƒè™‘ Kubernetes 版本。 + +### 备份 + +- 用户生æˆçš„æŸäº›èµ„æºä¸ä¼šè¢«å¤‡ä»½ï¼Œé™¤éžè¿™äº›èµ„æºå¯ä»¥è¢«é»˜è®¤ resourceSet æ•èŽ·ï¼Œæˆ–者 resourceSet 已被更改为æ•èŽ·è¿™äº›èµ„æºã€‚ + - 我们æ供了一个 `resources.cattle.io/backup:true` 标签，将该标签添加到任何命å空间中的 Secret 时，命å空间将被备份。 +- 备份是ä¸å¯æ”¹å˜çš„ +- 仅备份 local 集群 + +### 还原 + +- 还原是指将备份还原到原æ¥çš„集群。å¯ä»¥åœ¨å®‰è£…了 Rancher 的情况下进行（**å¿…é¡»å¯ç”¨ prune**），也å¯ä»¥åœ¨æœªå®‰è£… Rancher 的情况下进行（无特殊说明）。 +- 还原时需è¦æ³¨æ„的一件事是，你å¯èƒ½éœ€è¦â€œæ“¦é™¤â€é›†ç¾¤ä¸­çš„所有 Rancher 资æºã€‚ä½ å¯ä»¥é€šè¿‡å°† [Rancher cleanup script](https://github.com/rancher/rancher-cleanup) 脚本作为 job 部署到集群æ¥å®Œæˆè¿™ç‚¹ã€‚这样，你å¯ä»¥å†æ¬¡å®‰è£… Rancher Backups 并还原到全新的集群。 + - ç¡®ä¿ä½¿ç”¨äº† kubectl æ¥éƒ¨ç½²è„šæœ¬ã€‚ + +### è¿ç§» + +由于我们è¦è¿˜åŽŸåˆ°ä¸åŒçš„集群，因此对应的è¿ç§»æœ‰ä¸€äº›ç»†å¾®å·®åˆ«ã€‚以下是需è¦è®°ä½ä½†åˆå®¹æ˜“被忘记的事情。 + +- è¿ç§»æ—¶ Rancher 域必须相åŒã€‚æ¢è¨€ä¹‹ï¼Œä½ æ—§é›†ç¾¤çš„域å现在必须指å‘新集群。 +- Rancher **ä¸åº”该**å·²è¿è¡Œåœ¨ä½ è¦è¿ç§»åˆ°çš„集群，这å¯èƒ½ä¼šå¯¼è‡´ Rancher 备份和æŸäº› Rancher æœåŠ¡å‡ºçŽ°è®¸å¤šé—®é¢˜ã€‚ +- **还原备份åŽ**，安装与备份**相åŒ**çš„ Rancher 版本。 +- 在其他 Kubernetes 版本上é…置新集群å¯èƒ½ä¼šå‡ºçŽ°å„ç§ä¸å—支æŒçš„情况，这是因为å¯ç”¨çš„ Kubernetes API å¯èƒ½ä¸Žä½ å¤‡ä»½çš„ API ä¸åŒã€‚è¿™å¯èƒ½ä¼šå¯¼è‡´å·²å¼ƒç”¨çš„资æºè¢«æ¢å¤ï¼Œä»Žè€Œå¯¼è‡´é—®é¢˜ã€‚ +- 在è¿ç§»æœŸé—´**ä¸è¦**执行任何å‡çº§æ“作。 + +## 边缘案例和ä¸å½“使用 + +以下是 Rancher Backups 的一些**ä¸å½“**使用示例。 + +### å‡çº§ + +- 使用 Rancher Backups æ¥å‡çº§ Rancher 版本ä¸æ˜¯ä¸€ä¸ªæœ‰æ•ˆç”¨æ³•ã€‚推èçš„åšæ³•æ˜¯ï¼šå…ˆå¤‡ä»½å½“å‰ç‰ˆæœ¬ï¼Œç„¶åŽæŒ‰ç…§[说明](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md)å‡çº§ä½ çš„ Rancher 实例，在å‡çº§å®ŒæˆåŽå†è¿›è¡Œ**å¦ä¸€ä¸ª**备份。这样，如果å‡çº§å¤±è´¥ï¼Œä½ å°±æœ‰ä¸€ä¸ªå¯ä»¥ç”¨æ¥è¿˜åŽŸçš„备份，而第二个备份将能用于还原到å‡çº§åŽçš„ Rancher 版本。 +- 使用 Rancher Backups æ¥å‡çº§ Kubernetes 版本也ä¸æ˜¯ä¸€ä¸ªæœ‰æ•ˆç”¨æ³•ã€‚由于 Kubernetes API 以åŠå¯ç”¨èµ„æºä¸Žç‰ˆæœ¬ç›¸å…³ï¼Œå› æ­¤ä½¿ç”¨å¤‡ä»½è¿˜åŽŸçš„方法æ¥è¿›è¡Œå‡çº§å¯èƒ½ä¼šå¯¼è‡´èµ„æºé›†ä¸å¯¹é½çš„问题，这些资æºå¯èƒ½å·²è¢«å¼ƒç”¨ã€ä¸å—支æŒæˆ–已更新。å‡çº§é›†ç¾¤ç‰ˆæœ¬çš„æ–¹å¼å–决于其é…置方å¼ï¼Œä½†å»ºè®®ä½¿ç”¨ä¸Šè¿°çš„æµç¨‹ï¼ˆå¤‡ä»½ã€å‡çº§ã€å¤‡ä»½ï¼‰ã€‚ + +### ResourceSet + +- 由于ä¸åŒå›¢é˜Ÿçš„资æºå’ŒæœåŠ¡ä¼šä¸æ–­å‘展，开å‘人员应è¦æ³¨æ„是å¦éœ€è¦å‘默认 resourceSet 添加或删除新资æºã€‚ +- Rancher Backups 仅备份默认 resourceSet æ•èŽ·çš„内容（除éžè¿›è¡Œç¼–辑）。我们为用户创建的 Secret 添加了特定标签，无论 Secret çš„å称是什么，无论它属于哪个命å空间，具有该标签 Secret 都会被备份（请å‚阅[备份的正确用法](#备份)）。 + +### 下游集群 + +- Rancher Backups **ä»…**备份 local 集群上的 Kubernetes 资æºã€‚æ¢è¨€ä¹‹ï¼Œé™¤äº†å­˜åœ¨äºŽ local 集群中的资æºï¼Œä¸‹æ¸¸é›†ç¾¤**ä¸ä¼š**被触åŠæˆ–备份。下游集群的更新和通信由 rancher-agent å’Œ rancher-webhook 负责。 + +### è¿˜åŽŸå·²åˆ é™¤çš„èµ„æº + +- 有些资æºä¼šäº§ç”Ÿå¤–部结果，例如会é…置下游集群。删除下游集群并还原 local 集群上的集群资æº**ä¸ä¼š**导致 Rancher é‡æ–°é…置所述集群。æŸäº›èµ„æºå¯èƒ½æ— æ³•é€šè¿‡è¿˜åŽŸå›žåˆ°å¯ç”¨çŠ¶æ€ã€‚ +- “还原已删除的集群â€**ä¸æ˜¯**å—支æŒçš„功能。涉åŠä¸‹æ¸¸é›†ç¾¤æ—¶ï¼Œæ— è®ºé›†ç¾¤æ˜¯é…置的还是导入的，删除集群都会执行一系列清ç†ä»»åŠ¡ï¼Œå¯¼è‡´æˆ‘们无法还原已删除的集群。é…置的集群节点以åŠä¸Ž Rancher 相关的é…置资æºå°†è¢«é”€æ¯ï¼Œè€Œå¯¼å…¥çš„集群的 Rancher Agent 以åŠä¸Ž local 集群注册相关的其他资æº/æœåŠ¡å¯èƒ½ä¼šè¢«é”€æ¯ã€‚ + +:::caution + +å°è¯•åˆ é™¤å’Œè¿˜åŽŸä¸‹æ¸¸é›†ç¾¤å¯èƒ½ä¼šå¯¼è‡´ Rancherã€Rancher Backupsã€rancher-webhookã€Fleet 等出现å„ç§é—®é¢˜ã€‚因此，我们ä¸å»ºè®®ä½ è¿™æ ·åšã€‚ + +::: + +### Fleetã€Harvester 和其他æœåŠ¡ + +ç”± Rancher Backups 支æŒçš„其他æœåŠ¡ä¼šç»å¸¸å‘生å˜åŒ–å’Œå‘展。å‘生这ç§æƒ…况时，他们的资æºå’Œå¤‡ä»½éœ€æ±‚也å¯èƒ½ä¼šå‘生å˜åŒ–。有些资æºå¯èƒ½æ ¹æœ¬ä¸éœ€è¦å¤‡ä»½ã€‚团队需è¦åœ¨å¼€å‘过程中考虑这一点，并评估相关 resourceSet 是å¦èƒ½æ­£ç¡®æ•èŽ·æ­£ç¡®çš„资æºé›†æ¥è¿˜åŽŸå…¶æœåŠ¡ã€‚ + +## 结论 + +Rancher Backups 是一个éžå¸¸æœ‰ç”¨çš„工具，但它的使用范围和使用目的有é™çš„。为了é¿å…出现问题，请éµå¾ªæœ¬æ–‡æ‰€è¿°çš„æµç¨‹æ¥ç¡®ä¿ Chart 能正确è¿ä½œã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md new file mode 100644 index 00000000000..5aa5ac6e523 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md @@ -0,0 +1,184 @@ +--- +title: å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ +--- + +如果你è¦å°† Rancher è¿ç§»åˆ°ä¸€ä¸ªæ–°çš„ Kubernetes 集群，先ä¸è¦åœ¨æ–°é›†ç¾¤ä¸Šå®‰è£… Rancher。这是因为如果将 Rancher 还原到已安装 Rancher 的新集群，å¯èƒ½ä¼šå¯¼è‡´é—®é¢˜ã€‚ + +### 先决æ¡ä»¶ + +以下说明å‡è®¾ä½ å·²ç»å®Œæˆ[备份创建](back-up-rancher.md)，并且已ç»å®‰è£…了用于部署 Rancher çš„æ–° Kubernetes 集群。 + +:::caution + +你需è¦ä½¿ç”¨ä¸Žç¬¬ä¸€ä¸ªé›†ç¾¤ä¸­è®¾ç½®çš„ Server URL 相åŒçš„主机å。å¦åˆ™ï¼Œä¸‹æ¸¸é›†ç¾¤ä¼šåœ¨ UI 的管ç†é¡µé¢æ˜¾ç¤ºä¸ºä¸å¯ç”¨ï¼Œå¹¶ä¸”ä½ ä¸èƒ½ç‚¹å‡»é›†ç¾¤å†…或集群的 **Explore** 按钮。 + +::: + +Rancher 必须是 2.5.0 或更高版本。 + +Rancher å¯ä»¥å®‰è£…åˆ°ä»»æ„ Kubernetes 集群上，包括托管的 Kubernetes 集群（如 Amazon EKS 集群）。如需获å–安装 Kubernetes 的帮助，请å‚è§ Kubernetes å‘行版的文档。你也å¯ä»¥ä½¿ç”¨ä»¥ä¸‹ Rancher çš„ Kubernetes å‘行版： + +- [RKE Kubernetes 安装文档](https://rancher.com/docs/rke/latest/en/installation/) +- [K3s Kubernetes 安装文档](https://rancher.com/docs/k3s/latest/en/installation/) + +### 1. 安装 rancher-backup Helm Chart +安装 [rancher-backup chart](https://github.com/rancher/backup-restore-operator/tags)，请使用 2.x.x 主è¦ç‰ˆæœ¬å†…的版本： + +1. 添加 helm 仓库： + + ```bash + helm repo add rancher-charts https://charts.rancher.io + helm repo update + ``` + +1. 使用 2.x.x rancher-backup 版本设置 `CHART_VERSION` å˜é‡ï¼š + ```bash + helm search repo --versions rancher-charts/rancher-backup + CHART_VERSION=<2.x.x> + ``` + +1. 安装 Chart： + ```bash + helm install rancher-backup-crd rancher-charts/rancher-backup-crd -n cattle-resources-system --create-namespace --version $CHART_VERSION + helm install rancher-backup rancher-charts/rancher-backup -n cattle-resources-system --version $CHART_VERSION + ``` + + :::note + + 以上å‡è®¾ä½ çš„环境具有到 Docker Hub 的出站连接。 + + 对于**离线环境**，在安装 rancher-backup Helm Chart 时，使用下é¢çš„ Helm 值从你的ç§æœ‰é•œåƒä»“åº“ä¸­æ‹‰å– `backup-restore-operator` é•œåƒã€‚ + + ```bash + --set image.repository $REGISTRY/rancher/backup-restore-operator + ``` + + ::: + +### 2. 使用 Restore 自定义资æºæ¥è¿˜åŽŸå¤‡ä»½ + +:::note é‡è¦æ示： + +Kubernetes v1.22 是 Rancher 2.6.3 的实验功能，ä¸æ”¯æŒä½¿ç”¨ apiVersion `apiextensions.k8s.io/v1beta1`æ¥è¿˜åŽŸåŒ…å« CRD 文件的备份文件。在 v1.22 中，`rancher-backup` 应用的默认 `resourceSet` åªæ”¶é›†ä½¿ç”¨ `apiextensions.k8s.io/v1` çš„ CRD。你å¯ä»¥é€šè¿‡ä¸‹é¢ä¸¤ç§æ–¹æ³•è§£å†³è¿™ä¸ªé—®é¢˜ã€‚ + +1. 使用 apiVersion v1 æ¥æ›´æ–°é»˜è®¤ `resourceSet`，从而收集 CRD。 +1. 使用 `apiextensions.k8s.io/v1` 作为替代，æ¥æ›´æ–°é»˜è®¤ `resourceSet` 和客户端，从而在内部使用新的 API。 + + :::note + + 在为 v1.22 版本制作或æ¢å¤å¤‡ä»½æ—¶ï¼ŒRancher 版本和本地集群的 Kubernetes 版本应该是一样的。由于集群中支æŒçš„ apiVersion 和备份文件中的 apiVersion å¯èƒ½ä¸åŒï¼Œå› æ­¤åœ¨è¿˜åŽŸå¤‡ä»½æ—¶è¯·è€ƒè™‘ Kubernetes 的版本。 + + ::: + +::: + +1. 在使用 S3 对象存储作为需è¦ä½¿ç”¨å‡­è¯çš„还原的备份æºæ—¶ï¼Œè¯·åœ¨æ­¤é›†ç¾¤ä¸­åˆ›å»ºä¸€ä¸ª `Secret` 对象以添加 S3 凭è¯ã€‚Secret æ•°æ®å¿…é¡»æœ‰ä¸¤ä¸ªå¯†é’¥ï¼Œåˆ†åˆ«æ˜¯åŒ…å« S3 凭è¯çš„ `accessKey` å’Œ `secretKey`。 + + ä½ å¯ä»¥åœ¨ä»»ä½•å‘½å空间中创建 Secret，本示例使用 default 命å空间。 + + ```bash + kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= + ``` + + :::note + + 在上é¢çš„命令中添加你的 Access Key å’Œ Secret Key 作为 `accessKey` å’Œ `secretKey` 的值。 + + ::: + +1. 创建一个 `Restore` 对象： + + 在è¿ç§»æœŸé—´ï¼Œ`prune` 必须设置为 `false`。请å‚è§ä¸‹é¢çš„示例： + + ```yaml + # restore-migration.yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + // highlight-next-line + prune: false + // highlight-next-line + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + :::note é‡è¦æ示： + + åªæœ‰åœ¨åˆ›å»ºå¤‡ä»½æ—¶å¯ç”¨äº†åŠ å¯†åŠŸèƒ½æ—¶ï¼Œæ‰éœ€è¦è®¾ç½® `encryptionConfigSecretName` 字段。 + + 如果适用，请æ供包å«åŠ å¯†é…置文件的 `Secret` 对象的å称。如果你åªæœ‰åŠ å¯†é…置文件，但没有在此集群中创建 Secret，请按照以下步骤创建 Secret。 + + 1. 创建[加密é…置文件](../../../reference-guides/backup-restore-configuration/backup-configuration.md#加密) + 1. 下é¢çš„命令使用一个å为 `encryption-provider-config.yaml` 的文件，使用了 `--from-file` 标志。将 `EncryptionConfiguration` ä¿å­˜åˆ°å为 `encryption-provider-config.yaml` 的文件中之åŽï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ```bash + kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system + ``` + + ::: + +1. 应用清å•ï¼Œå¹¶ç›‘控 Restore 的状æ€ï¼š + 1. 应用 `Restore` 对象资æºï¼š + + ```bash + kubectl apply -f restore-migration.yaml + ``` + + 1. 观察 Restore 的状æ€ï¼š + ```bash + kubectl get restore + ``` + + 1. 查看æ¢å¤æ—¥å¿—： + ```bash + kubectl logs -n cattle-resources-system --tail 100 -f -l app.kubernetes.io/instance=rancher-backup + ``` + + 1. Restore 资æºçš„状æ€å˜æˆ `Completed` åŽï¼Œä½ å¯ä»¥ç»§ç»­å®‰è£… cert-manager å’Œ Rancher。 + +### 3. 安装 cert-manager + +按照在 Kubernetes 上安装 cert-manager的步骤[安装 cert-manager](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-安装-cert-manager)。 + +### 4. 使用 Helm 安装 Rancher + +使用与第一个集群上使用的相åŒç‰ˆæœ¬çš„ Helm æ¥å®‰è£… Rancher： + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +```bash +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname= \ + --version x.y.z +``` + +:::note + +如果原始的 Rancher 环境正在è¿è¡Œï¼Œä½ å¯ä»¥ä½¿ç”¨ kubeconfig 为原始环境收集当å‰å€¼ï¼š + +```bash +helm get values rancher -n cattle-system -o yaml > rancher-values.yaml +``` + +ä½ å¯ä»¥ä½¿ç”¨ `rancher-values.yaml` 文件æ¥å¤ç”¨è¿™äº›å€¼ã€‚ç¡®ä¿å°† kubeconfig 切æ¢åˆ°æ–°çš„ Rancher 环境。 + +```bash +helm install rancher rancher-latest/rancher -n cattle-system -f rancher-values.yaml --version x.y.z +``` + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md new file mode 100644 index 00000000000..b491321f2bf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md @@ -0,0 +1,72 @@ +--- +title: æ¢å¤å¤‡ä»½ - Docker 安装 +--- + +如果é‡åˆ°ç¾éš¾æƒ…况，你å¯ä»¥å°† Rancher Server æ¢å¤åˆ°æœ€æ–°çš„备份。 + +## åœ¨ä½ å¼€å§‹å‰ + +在æ¢å¤å¤‡ä»½çš„过程中，你将输入一系列命令。请使用环境中的数æ®æ›¿æ¢å ä½ç¬¦ã€‚å ä½ç¬¦ç”¨å°–括å·å’Œå¤§å†™å­—æ¯ï¼ˆå¦‚ ``）表示。以下是带有å ä½ç¬¦çš„命令示例： + +``` +docker run --volumes-from -v $PWD:/backup \ +busybox sh -c "rm /var/lib/rancher/* -rf && \ +tar pzxvf /backup/rancher-data-backup--" +``` + +在此命令中，`` å’Œ `-` 是用于 Rancher 部署的环境å˜é‡ã€‚ + +请交å‰å‚考下方的图片和表格，了解获å–æ­¤å ä½ç¬¦æ•°æ®çš„方法。在开始以下步骤之å‰ï¼Œè¯·å…ˆè®°ä¸‹æˆ–å¤åˆ¶æ­¤ä¿¡æ¯ã€‚ + +终端 docker ps 命令，显示如何找到 <RANCHER_CONTAINER_TAG> å’Œ <RANCHER_CONTAINER_NAME> + +![å ä½ç¬¦å‚考](/img/placeholder-ref.png) + +| å ä½ç¬¦ | 示例 | æè¿° | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | 首次安装拉å–çš„ rancher/rancher é•œåƒã€‚ | +| `` | `festive_mestorf` | ä½ çš„ Rancher 容器的å称。 | +| `` | `v2.0.5` | Rancher 备份的版本å·ã€‚ | +| `` | `9-27-18` | æ•°æ®å®¹å™¨æˆ–备份的创建日期。 | +
+ +å¯ä»¥é€šè¿‡è¿œç¨‹è¿žæŽ¥ç™»å½•åˆ° Rancher Server 所在的主机并输入命令 `docker ps` 以查看正在è¿è¡Œçš„容器，从而获得 `` å’Œ ``。你还å¯ä»¥è¿è¡Œ `docker ps -a` 命令查看åœæ­¢äº†çš„容器。在创建备份期间，你éšæ—¶å¯ä»¥è¿è¡Œè¿™äº›å‘½ä»¤æ¥èŽ·å¾—帮助。 + +## æ¢å¤å¤‡ä»½ + +使用你之å‰åˆ›å»ºçš„[备份](back-up-docker-installed-rancher.md)，将 Rancher æ¢å¤åˆ°æœ€åŽå·²çŸ¥çš„å¥åº·çŠ¶æ€ã€‚ + +1. 使用远程终端连接，登录到è¿è¡Œ Rancher Server 的节点。 + +1. åœæ­¢å½“å‰è¿è¡Œ Rancher Server 的容器。将 `` 替æ¢ä¸º Rancher 容器的å称： + + ``` + docker stop + ``` +1. 将你在[创建备份 - Docker 安装](back-up-docker-installed-rancher.md)时创建的备份压缩包移动到 Rancher Server。切æ¢åˆ°ä½ å°†å…¶ç§»åŠ¨åˆ°çš„目录。输入 `dir` 以确认它在该ä½ç½®ã€‚ + + 如果你éµå¾ªäº†æˆ‘们在[创建备份 - Docker 安装](back-up-docker-installed-rancher.md)中推è的命åæ–¹å¼ï¼Œå®ƒçš„å称会与 `rancher-data-backup--.tar.gz` 类似。 + +1. 输入以下命令删除当å‰çŠ¶æ€æ•°æ®ï¼Œå¹¶å°†å…¶æ›¿æ¢ä¸ºå¤‡ä»½æ•°æ®ã€‚请替æ¢å ä½ç¬¦ã€‚ä¸è¦å¿˜è®°å…³é—­å¼•å·ã€‚ + + :::danger + + 该命令将删除 Rancher Server 容器中的所有当å‰çŠ¶æ€æ•°æ®ã€‚创建备份压缩包åŽä¿å­˜çš„任何更改都将丢失。 + + ::: + + ``` + docker run --volumes-from -v $PWD:/backup \ + busybox sh -c "rm /var/lib/rancher/* -rf && \ + tar pzxvf /backup/rancher-data-backup--.tar.gz" + ``` + + **步骤结果**：å±å¹•ä¸Šå°†è¿è¡Œå‘½ä»¤æµã€‚ + +1. é‡æ–°å¯åŠ¨ Rancher Server 容器，替æ¢å ä½ç¬¦ã€‚Rancher Server 将使用你的备份数æ®é‡æ–°å¯åŠ¨ã€‚ + + ``` + docker start + ``` + +1. 等待片刻，然åŽåœ¨æµè§ˆå™¨ä¸­æ‰“å¼€ Rancher。确认还原æˆåŠŸï¼Œå¹¶ä¸”ä½ çš„æ•°æ®å·²æ¢å¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md new file mode 100644 index 00000000000..97e23f81446 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md @@ -0,0 +1,129 @@ +--- +title: 使用备份æ¢å¤é›†ç¾¤ +--- + +ä½ å¯ä»¥è½»æ¾å¤‡ä»½å’Œæ¢å¤ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)çš„ etcd。etcd æ•°æ®åº“的快照会ä¿å­˜åœ¨ etcd 节点或 S3 兼容目标上。é…ç½® S3 的好处是，如果所有 etcd 节点都丢失了，你的快照会ä¿å­˜åˆ°è¿œç«¯å¹¶èƒ½ç”¨äºŽæ¢å¤é›†ç¾¤ã€‚ + +Rancher 建议å¯ç”¨ [etcd 定期快照的功能](back-up-rancher-launched-kubernetes-clusters.md#é…置定期快照)，但你也å¯ä»¥è½»æ¾åˆ›å»º[一次性快照](back-up-rancher-launched-kubernetes-clusters.md#å•æ¬¡å¿«ç…§)。Rancher å…许使用[ä¿å­˜çš„å¿«ç…§](#使用快照æ¢å¤é›†ç¾¤)进行æ¢å¤ã€‚如果你没有任何快照，你ä»ç„¶å¯ä»¥[æ¢å¤ etcd](#在没有快照的情况下æ¢å¤-etcdrke)。 + +集群也å¯ä»¥æ¢å¤åˆ°ä¹‹å‰çš„ Kubernetes 版本和集群é…置。 + +## 查看å¯ç”¨å¿«ç…§ + +Rancher UI 中æ供了集群所有å¯ç”¨å¿«ç…§çš„列表： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦æŸ¥çœ‹å¿«ç…§çš„集群并点击集群å称。 +1. å•å‡»**å¿«ç…§**选项å¡ã€‚列出的快照包括创建时间的时间戳。 + +## 使用快照æ¢å¤é›†ç¾¤ + +如果你的 Kubernetes 集群已æŸå，你å¯ä»¥ä½¿ç”¨å¿«ç…§æ¥æ¢å¤é›†ç¾¤ã€‚ + +快照由 etcd 中的集群数æ®ã€Kubernetes 版本和 `cluster.yml` 中的集群é…置组æˆã€‚有了这些组件，你å¯ä»¥åœ¨ä½¿ç”¨å¿«ç…§æ¢å¤é›†ç¾¤æ—¶é€‰æ‹©ï¼š + +- **ä»…æ¢å¤ etcd 内容**：类似于在 Rancher v2.4.0 之å‰ç‰ˆæœ¬ä¸­çš„使用快照æ¢å¤ã€‚ +- **æ¢å¤ etcd å’Œ Kubernetes 版本**：如果 Kubernetes å‡çº§å¯¼è‡´é›†ç¾¤å¤±è´¥ï¼Œå¹¶ä¸”你没有更改任何集群é…置，则应使用此选项。 +- **æ¢å¤ etcdã€Kubernetes 版本和集群é…ç½®**：如果你在å‡çº§æ—¶åŒæ—¶æ›´æ”¹äº† Kubernetes 版本和集群é…置，则应使用此选项。 + +回滚到之å‰çš„ Kubernetes 版本时，[å‡çº§ç­–略选项](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md#é…ç½®å‡çº§ç­–ç•¥)会被忽略。在æ¢å¤åˆ°æ—§ Kubernetes 版本之å‰ï¼ŒWorker 节点ä¸ä¼šè¢«å°é”或清空，因此å¯ä»¥æ›´å¿«åœ°å°†ä¸å¥åº·çš„集群æ¢å¤åˆ°å¥åº·çŠ¶æ€ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +è¦æ¢å¤ S3 中的快照，需è¦å°†é›†ç¾¤é…置为[在 S3 上创建定期快照](back-up-rancher-launched-kubernetes-clusters.md#é…置定期快照)。 + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦æŸ¥çœ‹å¿«ç…§çš„集群并点击集群å称。 +1. å•å‡»**å¿«ç…§**选项å¡æ¥æŸ¥çœ‹å·²ä¿å­˜å¿«ç…§çš„列表。 +1. 转到è¦æ¢å¤çš„快照，然åŽå•å‡» **â‹® > 还原**。 +1. 选择一个**还原类型**。 +1. 点击**还原**。 + +**结果**：集群将进入 `updating` 状æ€ï¼Œç„¶åŽå°†å¼€å§‹ä½¿ç”¨å¿«ç…§æ¢å¤ `etcd` 节点。集群会在返回到 `active` 状æ€åŽè¢«æ¢å¤ã€‚ + +## Control Plane/etcd 完全ä¸å¯ç”¨æ—¶ä½¿ç”¨å¿«ç…§è¿˜åŽŸé›†ç¾¤ + +在ç¾éš¾æ¢å¤åœºæ™¯ä¸­ï¼Œä¸‹æ¸¸é›†ç¾¤ä¸­ Rancher 管ç†çš„ Control Plane å’Œ etcd 节点å¯èƒ½ä¸å†å¯ç”¨æˆ–è¿è¡Œã€‚ä½ å¯ä»¥é€šè¿‡å†æ¬¡æ·»åŠ  Control Plane å’Œ etcd 节点æ¥é‡å»ºé›†ç¾¤ï¼Œç„¶åŽä½¿ç”¨å¯ç”¨å¿«ç…§æ¥è¿›è¡Œè¿˜åŽŸã€‚ + + + + +请按照 [SUSE 知识库](https://www.suse.com/support/kb/doc/?id=000020695)中æè¿°çš„æµç¨‹è¿›è¡Œæ“作。 + + + + +如果集群完全故障，则必须从集群中删除所有 etcd 节点和机器，然åŽæ‰èƒ½æ·»åŠ æ–°çš„ etcd 节点æ¥è¿›è¡Œè¿˜åŽŸã€‚ + +:::note + +由于[已知问题](https://github.com/rancher/rancher/issues/41080)ï¼Œæ­¤è¿‡ç¨‹éœ€è¦ Rancher v2.7.5 或更高版本。 + +::: + +:::note + +如果你使用[本地快照](./back-up-rancher-launched-kubernetes-clusters.md#本地备份目标)，那么请**务必**从è¦åˆ é™¤çš„ etcd 节点上的 `/var/lib/rancher//server/db/snapshots/` 文件夹中备份è¦è¿˜åŽŸçš„快照。你å¯ä»¥å°†å¿«ç…§å¤åˆ¶åˆ° `/var/lib/rancher//server/db/snapshots/` 文件夹中的新节点上。此外，如果使用本地快照并还原到新节点，目å‰è¿˜æ— æ³•é€šè¿‡ UI 进行还原。 + +::: + +1. 从集群中删除所有 etcd 节点。 + + 1. 在左上角，å•å‡» **☰ > 集群管ç†**。 + 1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦åˆ é™¤èŠ‚点的集群。 + 1. 在**主机**选项å¡ä¸­ï¼Œæ‰¾åˆ°è¦åˆ é™¤çš„æ¯ä¸ªèŠ‚点并å•å‡» **â‹® > 删除**。开始时，节点会挂在 `deleting` 状æ€ï¼Œæ‰€æœ‰ etcd 节点都被删除åŽï¼Œå®ƒä»¬å°†è¢«ä¸€èµ·åˆ é™¤ã€‚这是因为 Rancher å‘现所有 etcd 节点都在删除，并开始 "短路" etcd 安全删除逻辑。 + +1. 删除所有 etcd 节点åŽï¼Œæ·»åŠ è¦ç”¨äºŽè¿˜åŽŸçš„æ–° etcd 节点。 + + - 对于自定义集群，请转到**注册**选项å¡ï¼Œç„¶åŽåœ¨èŠ‚点上å¤åˆ¶å¹¶è¿è¡Œæ³¨å†Œå‘½ä»¤ã€‚如果该节点之å‰å·²åœ¨é›†ç¾¤ä¸­ä½¿ç”¨è¿‡ï¼Œè¯·å…ˆ[清ç†è¯¥èŠ‚点](../manage-clusters/clean-cluster-nodes.md#清ç†èŠ‚点)。 + - 对于主机驱动集群，则会自动é…置新节点。 + + 此时，Rancher 会æ示你需è¦ä½¿ç”¨ etcd 快照进行还原。 + +1. 使用 etcd 快照还原。 + + - 对于 S3 快照，使用 UI 进行还原。 + 1. å•å‡»**å¿«ç…§**选项å¡æ¥æŸ¥çœ‹å·²ä¿å­˜å¿«ç…§çš„列表。 + 1. 转到è¦æ¢å¤çš„快照，然åŽå•å‡» **â‹® > 还原**。 + 1. 选择一个**还原类型**。 + 1. 点击**还原**。 + - 对于本地快照，使用 UI 进行还原**ä¸**å¯ç”¨ã€‚ + 1. å•å‡»å³ä¸Šè§’çš„ **â‹®> 编辑 YAML**。 + 1. å°† `spec.cluster.rkeConfig.etcdSnapshotRestore.name` 定义为 `/var/lib/rancher//server/db/snapshots/` 中ç£ç›˜ä¸Šå¿«ç…§çš„文件å。 + +1. 还原æˆåŠŸåŽï¼Œä½ å¯ä»¥å°† etcd 节点扩展至所需的冗余。 + + + + +## 在没有快照的情况下æ¢å¤ etcd（RKE） + +如果 etcd 节点组失去了仲è£ï¼ˆquorum），由于没有æ“作（例如部署工作负载）å¯ä»¥åœ¨ Kubernetes 集群中执行，Kubernetes 集群将报告失败。集群需è¦æœ‰ä¸‰ä¸ª etcd 节点以防止仲è£ä¸¢å¤±ã€‚如果你想æ¢å¤ä½ çš„ etcd 节点集，请按照以下说明æ“作： + +1. 通过删除所有其他 etcd 节点，从而仅在集群中ä¿ç•™ä¸€ä¸ª etcd 节点。 + +2. 在剩余的å•ä¸ª etcd 节点上，è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ```bash + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock assaflavie/runlike etcd + ``` + + 此命令会输出 etcd è¦è¿è¡Œçš„命令，请ä¿å­˜æ­¤å‘½ä»¤ä»¥å¤‡åŽç”¨ã€‚ + +3. åœæ­¢æ­£åœ¨è¿è¡Œçš„ `etcd` 容器并将其é‡å‘½å为 `etcd-old`。 + + ```bash + docker stop etcd + docker rename etcd etcd-old + ``` + +4. 修改步骤 2 中获å–ä¿å­˜çš„命令： + + - 如果你最åˆæœ‰è¶…过 1 个 etcd 节点，则将 `--initial-cluster` 更改为仅包å«å‰©ä½™çš„å•ä¸ªèŠ‚点。 + - å°† `--force-new-cluster` 添加到命令的末尾。 + +5. è¿è¡Œä¿®æ”¹åŽçš„命令。 + +6. 在å•ä¸ªèŠ‚点å¯åŠ¨å¹¶è¿è¡ŒåŽï¼ŒRancher 建议å‘你的集群添加é¢å¤–çš„ etcd 节点。如果你有一个[自定义集群](../../../pages-for-subheaders/use-existing-nodes.md)，并且想è¦å¤ç”¨æ—§èŠ‚点，则需è¦å…ˆ[清ç†èŠ‚点](../manage-clusters/clean-cluster-nodes.md)，然åŽå†å°è¯•å°†å®ƒä»¬é‡æ–°æ·»åŠ åˆ°é›†ç¾¤ä¸­ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md new file mode 100644 index 00000000000..ea3dafca61e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md @@ -0,0 +1,97 @@ +--- +title: 还原 Rancher +--- + +本页概述了如何使用 Rancher 执行æ¢å¤ã€‚ + +:::note é‡è¦æ示： + +* 请按照此页é¢ä¸Šçš„说明在已备份的åŒä¸€é›†ç¾¤ä¸Šè¿˜åŽŸ Rancher。è¦æŠŠ Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œè¯·å‚照步骤[è¿ç§» Rancher](migrate-rancher-to-new-cluster.md)。 +* 在使用相åŒè®¾ç½®è¿˜åŽŸ Rancher 时，operator å°†åœ¨è¿˜åŽŸå¼€å§‹æ—¶ç¼©å‡ Rancher deployment，还原完æˆåŽåˆä¼šæ‰©å±• deployment。因此，Rancher 在还原期间将ä¸å¯ç”¨ã€‚ +* 如果你需è¦åœ¨å‡çº§åŽå°† Rancher 还原到先å‰ç‰ˆæœ¬ï¼Œè¯·å‚è§[回滚](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md)。 + +::: + +## 使用 Rancher 2.6.4+ 进行回滚的其他步骤 + +Rancher v2.6.4 å°† cluster-api 模å—从 v0.4.4 å‡çº§åˆ° v1.0.2。å过æ¥ï¼Œcluster-api çš„ v1.0.2 版本将集群 API 的自定义资æºå®šä¹‰ (CRD) 从 `cluster.x-k8s.io/v1alpha4` å‡çº§åˆ° `cluster.x-k8s.io/v1beta1`。当你å°è¯•å°† Rancher v2.6.4 回滚到以å‰ç‰ˆæœ¬çš„ Rancher v2.6.x 时，CRD å‡çº§åˆ° v1beta1 会导致回滚失败。这是因为使用旧 apiVersion (v1alpha4) çš„ CRD 与 v1beta1 ä¸å…¼å®¹ã€‚ + +è¦é¿å…回滚失败，你需è¦åœ¨å°è¯•æ¢å¤æ“作或回滚**之å‰**è¿è¡Œä»¥ä¸‹ Rancher 脚本： + +* `verify.sh`：检查集群中是å¦æœ‰ä»»ä½•ä¸Ž Rancher 相关的资æºã€‚ +* `cleanup.sh`：清ç†é›†ç¾¤ã€‚ + +有关详细信æ¯å’Œæºä»£ç ï¼Œè¯·å‚阅 [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup)。 + +:::caution + +`cleanup.sh` è¿è¡Œçš„时候会有åœæœºæ—¶é—´ï¼Œè¿™æ˜¯å› ä¸ºè„šæœ¬ä¼šåˆ é™¤ Rancher 创建的资æºã€‚ + +::: + +### 从 v2.6.4+ 回滚到较低版本的 v2.6.x + +1. 按照[说明](https://github.com/rancher/rancher-cleanup/blob/main/README.md)è¿è¡Œè„šæœ¬ã€‚ +1. 按照[说明](https://rancher.com/docs/rancher/v2.6/en/backups/migrating-rancher/)在现有集群上安装 rancher-backup Helm Chart 并æ¢å¤ä¹‹å‰çš„状æ€ã€‚ + 1. çœç•¥æ­¥éª¤ 3。 + 1. 执行到步骤 4 时，在è¦å›žæ»šåˆ°çš„ local 集群上安装 Rancher 2.6.x 版本。 + +### 创建 Restore è‡ªå®šä¹‰èµ„æº + +还原是通过创建 Restore 自定义资æºå®žçŽ°çš„。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。Rancher Server è¿è¡Œåœ¨ `local` 集群中。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Rancher 备份 > 还原**。 +1. å•å‡»**创建**。 +1. 使用表å•æˆ– YAML 创建 Restore。如需获å–使用表å•åˆ›å»º Restore 资æºçš„更多信æ¯ï¼Œè¯·å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 +1. è¦ä½¿ç”¨ YAML 编辑器，å•å‡»**创建 > 使用 YAML 文件创建**。输入 Restore YAML。 + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + 如需获得é…ç½® Restore 的帮助，请å‚è§[é…ç½®å‚考](../../../reference-guides/backup-restore-configuration/restore-configuration.md)å’Œ[示例](../../../reference-guides/backup-restore-configuration/examples.md)。 + +1. å•å‡»**创建**。 + +**结果**：rancher-operator 在还原过程中将 Rancher deployment 缩容，并在还原完æˆåŽå°†å®ƒé‡æ–°æ‰©å®¹ã€‚资æºè¿˜åŽŸé¡ºåºå¦‚下： + +1. 自定义资æºå®šä¹‰ï¼ˆCRD） +2. é›†ç¾¤èŒƒå›´èµ„æº +3. 命åç©ºé—´èµ„æº + +### 日志 + +如需查看还原的处ç†æ–¹å¼ï¼Œè¯·æ£€æŸ¥ Operator 的日志。查看日志的命令如下： + +``` +kubectl logs -n cattle-resources-system -l app.kubernetes.io/name=rancher-backup -f +``` + +### æ¸…ç† + +如果你使用 kubectl 创建了 Restore 资æºï¼Œè¯·åˆ é™¤è¯¥èµ„æºä»¥é˜²æ­¢ä¸Žæœªæ¥çš„还原å‘生命å冲çªã€‚ + +### 已知问题 +在æŸäº›æƒ…况下，æ¢å¤å¤‡ä»½åŽï¼ŒRancher 日志会显示类似以下的错误： +``` +2021/10/05 21:30:45 [ERROR] error syncing 'c-89d82/m-4067aa68dd78': handler rke-worker-upgrader: clusters.management.cattle.io "c-89d82" not found, requeuing +``` +å‘生这ç§æƒ…况的原因是，刚刚还原的æŸä¸ªèµ„æºæœ‰ finalizer，但相关的资æºå·²ç»è¢«åˆ é™¤ï¼Œå¯¼è‡´å¤„ç†ç¨‹åºæ— æ³•æ‰¾åˆ°è¯¥èµ„æºã€‚ + +为了消除这些错误，你需è¦æ‰¾åˆ°å¹¶åˆ é™¤å¯¼è‡´é”™è¯¯çš„资æºã€‚详情请å‚è§[此处](https://github.com/rancher/rancher/issues/35050#issuecomment-937968556)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md new file mode 100644 index 00000000000..def223a8337 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md @@ -0,0 +1,67 @@ +--- +title: 使用 Feet 进行æŒç»­äº¤ä»˜ +--- + +使用 Fleet çš„æŒç»­äº¤ä»˜æ˜¯å¤§è§„模的 GitOps。你å¯ä»¥ä½¿ç”¨ Fleet 管ç†å¤šè¾¾ä¸€ç™¾ä¸‡ä¸ªé›†ç¾¤ã€‚Fleet éžå¸¸è½»é‡ï¼Œå¯ä»¥å¾ˆå¥½åœ°ç”¨äºŽ[å•ä¸ªé›†ç¾¤](https://fleet.rancher.io/installation#default-install)，但是在你达到[大规模](https://fleet.rancher.io/installation#configuration-for-multi-cluster)时，它能å‘挥更强的实力。此处的大规模指的是大é‡é›†ç¾¤ã€å¤§é‡éƒ¨ç½²ã€æˆ–组织中存在大é‡å›¢é˜Ÿçš„情况。 + +Fleet 是一个独立于 Rancher 的项目，你å¯ä»¥ä½¿ç”¨ Helm 将它安装在任何 Kubernetes 集群上。 + + +## 架构 + +有关 Fleet 工作原ç†çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/fleet-gitops-at-scale/architecture.md)。 + +## 在 Rancher UI 中访问 Fleet + +Fleet 预装在 Rancher 中，通过 Rancher UI 中的**æŒç»­äº¤ä»˜**选项管ç†ã€‚有关æŒç»­äº¤ä»˜å’Œ Fleet 故障排除技巧的更多信æ¯ï¼Œè¯·å‚阅[此处](https://fleet.rancher.io/troubleshooting)。 + +用户å¯ä»¥é€šè¿‡éµå¾ª **gitops** 的实践，利用æŒç»­äº¤ä»˜å°†åº”用部署到 git 仓库中的 Kubernetes 集群，而无需任何手动æ“作。 + +按照以下步骤在 Rancher UI 中访问æŒç»­äº¤ä»˜ï¼š + +1. å•å‡» **☰ > æŒç»­äº¤ä»˜**。 + +1. 在èœå•é¡¶éƒ¨é€‰æ‹©ä½ çš„命å空间，注æ„以下几点： + - 默认情况下会选中 `fleet-default`，其中包括注册到 Rancher 的所有下游集群。 + - ä½ å¯ä»¥åˆ‡æ¢åˆ°ä»…åŒ…å« `local` 集群的 `fleet-local`，或者创建自己的工作空间，并将集群分é…和移动到该工作空间。 + - 然åŽï¼Œä½ å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„**集群**æ¥ç®¡ç†é›†ç¾¤ã€‚ + +1. å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„ **Git 仓库**å°† git 仓库部署到当å‰å·¥ä½œç©ºé—´ä¸­çš„集群中。 + +1. 选择你的 [git 仓库](https://fleet.rancher.io/gitrepo-add)å’Œ[目标集群/集群组](https://fleet.rancher.io/gitrepo-targets)。你还å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸­çš„**集群组**在 UI 中创建集群组。 + +1. 部署 git 仓库åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 监控应用。 + +## Windows æ”¯æŒ + +有关对具有 Windows 节点的集群的支æŒçš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md)。 + + +## GitHub 仓库 + +ä½ å¯ä»¥å•å‡»æ­¤å¤„èŽ·å– [Fleet Helm Chart](https://github.com/rancher/fleet/releases/latest)。 + + +## 在代ç†åŽä½¿ç”¨ Fleet + +有关在代ç†åŽä½¿ç”¨ Fleet 的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md)。 + +## Helm Chart ä¾èµ– + +由于用户需è¦å®Œæˆä¾èµ–列表，因此为了æˆåŠŸéƒ¨ç½²å…·æœ‰ä¾èµ–项的 Helm Chart，你必须手动è¿è¡Œå‘½ä»¤ï¼ˆå¦‚下所列）。如果你ä¸è¿™æ ·åšï¼Œå¹¶ç»§ç»­å…‹éš†ä»“库并è¿è¡Œ `helm install`，由于ä¾èµ–项将丢失，因此你的安装将失败。 + +git 仓库中的 Helm Chart 必须在 Chart å­ç›®å½•ä¸­åŒ…å«å…¶ä¾èµ–项。你必须手动è¿è¡Œ `helm dependencies update $chart`，或在本地è¿è¡Œ `helm dependencies build $chart`，然åŽå°†å®Œæ•´çš„ Chart 目录æ交到你的 git 仓库。请注æ„，你需è¦ä½¿ç”¨é€‚当的å‚æ•°æ¥ä¿®æ”¹å‘½ä»¤ã€‚ + +## 故障排除 + +--- +* **已知问题**：Fleet git 仓库的 clientSecretName å’Œ helmSecretName 密文ä¸åŒ…å«åœ¨ [backup-restore-operator](../backup-restore-and-disaster-recovery/back-up-rancher.md#1-安装-rancher-backup-operator) 创建的备份或æ¢å¤ä¸­ã€‚如果我们有了永久的解决方案，我们将通知社区。 + +* **临时解决方法：**
+ 默认情况下，用户定义的密文ä¸ä¼šåœ¨ Fleet 中备份。如果执行ç¾éš¾æ¢å¤æˆ–å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œåˆ™éœ€è¦é‡æ–°åˆ›å»ºå¯†æ–‡ã€‚è¦ä¿®æ”¹ resourceSet 以包å«éœ€è¦å¤‡ä»½çš„其他资æºï¼Œè¯·å‚阅[此文档](https://github.com/rancher/backup-restore-operator#user-flow)。 + +--- + +## 文档 + +Fleet 文档链接：[https://fleet.rancher.io/](https://fleet.rancher.io/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md new file mode 100644 index 00000000000..113a30ad48b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md @@ -0,0 +1,175 @@ +--- +title: 多集群应用 +--- + +通常，大多数应用都部署在å•ä¸ª Kubernetes 集群上，但有时你å¯èƒ½éœ€è¦è·¨ä¸åŒé›†ç¾¤å’Œ/或项目部署åŒä¸€åº”用的多个副本。在 Rancher 中，_多集群应用_ 指的是使用 Helm Chart 跨多个集群部署的应用。由于能够跨多个集群部署相åŒçš„应用，因此å¯ä»¥é¿å…在æ¯ä¸ªé›†ç¾¤ä¸Šé‡å¤æ‰§è¡Œç›¸åŒçš„应用é…ç½®æ“作而引入的人为错误。使用多集群应用，你å¯ä»¥é€šè¿‡è‡ªå®šä¹‰åœ¨æ‰€æœ‰é¡¹ç›®/集群中使用相åŒçš„é…置，并根æ®ä½ çš„目标项目更改é…置。由于多集群应用被视为å•ä¸ªåº”用，因此更容易管ç†å’Œç»´æŠ¤ã€‚ + +全局应用商店中的任何 Helm Chart 都å¯ç”¨äºŽéƒ¨ç½²å’Œç®¡ç†å¤šé›†ç¾¤åº”用。 + +创建多集群应用åŽï¼Œä½ å¯ä»¥å¯¹å…¨å±€ DNS æ¡ç›®è¿›è¡Œç¼–程，以便更轻æ¾åœ°è®¿é—®åº”用。 + +## 先决æ¡ä»¶ + +### æƒé™ + +è¦åœ¨ Rancher 中创建多集群应用，你至少需è¦å…·æœ‰ä»¥ä¸‹æƒé™ä¹‹ä¸€ï¼š + +- 目标集群中的[项目æˆå‘˜è§’色](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)，能够创建ã€è¯»å–ã€æ›´æ–°å’Œåˆ é™¤å·¥ä½œè´Ÿè½½ +- 目标项目所在集群的[集群所有者角色](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色) + +### å¯ç”¨æ—§ç‰ˆåŠŸèƒ½ + +由于 Rancher 2.5 已弃用多集群应用并使用 Fleet å–代它，你需è¦ä½¿ç”¨åŠŸèƒ½å¼€å…³ä»¥å¯ç”¨å¤šé›†ç¾¤åº”用。 + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 转到 `Legacy` 功能开关并å•å‡»**激活**。 + +## å¯åŠ¨å¤šé›†ç¾¤åº”用 + +1. 在左上角，å•å‡»**☰ > 多集群应用**。 +1. 点击**å¯åŠ¨**。 +1. 找到è¦å¯åŠ¨çš„应用。 +1. （å¯é€‰ï¼‰æŸ¥çœ‹æ¥è‡ª Helm Chart `README` 的详细æ述。 +1. 在**é…置选项**下输入多集群应用的**å称**。默认情况下，此å称还用于在æ¯ä¸ª[目标项目](#目标)中为多集群应用创建一个 Kubernetes 命å空间。命å空间命å为 `-`。 +1. 选择一个**模æ¿ç‰ˆæœ¬**。 +1. 完æˆ[多集群应用é…置选项](#多集群应用é…置选项)以åŠ[应用é…置选项](#应用é…置选项)。 +1. 选择å¯ä»¥[与多集群应用交互](#æˆå‘˜)çš„**æˆå‘˜**。 +1. 添加[自定义应用é…置答案](#覆盖特定项目的应用é…置选项)，这将更改默认应用é…置答案中特定项目的é…置。 +1. 查看**预览**中的文件。确认åŽï¼Œå•å‡»**å¯åŠ¨**。 + +**结果**：应用已部署到所选的命å空间。你å¯ä»¥ä»Žé¡¹ç›®ä¸­æŸ¥çœ‹åº”用状æ€ã€‚ + +## 多集群应用é…置选项 + +Rancher 将多集群应用的é…置选项分为以下几个部分。 + +### 目标 + +在**目标**部分中，选择用于部署应用的项目。项目列表仅显示你有æƒè®¿é—®çš„项目。所选的æ¯ä¸ªé¡¹ç›®éƒ½ä¼šè¢«æ·»åŠ åˆ°åˆ—表中，其中显示了所选的集群å称和项目å称。è¦ç§»é™¤ç›®æ ‡é¡¹ç›®ï¼Œå•å‡» **-**。 + +### å‡çº§ + +在**å‡çº§**部分中，选择å‡çº§åº”用时需è¦ä½¿ç”¨çš„å‡çº§ç­–略。 + +* **滚动更新（批é‡ï¼‰**：选择此å‡çº§ç­–略时，æ¯æ¬¡å‡çº§çš„应用数é‡å–决于选择的**批é‡å¤§å°**å’Œ**é—´éš”**（多少秒åŽæ‰å¼€å§‹ä¸‹ä¸€æ‰¹æ›´æ–°ï¼‰ã€‚ + +* **åŒæ—¶å‡çº§æ‰€æœ‰åº”用**：选择此å‡çº§ç­–略时，所有项目的所有应用都将åŒæ—¶å‡çº§ã€‚ + +### 角色 + +在**角色**中，你å¯ä»¥å®šä¹‰å¤šé›†ç¾¤åº”用的角色。通常，当用户[å¯åŠ¨å•†åº—应用](../../../pages-for-subheaders/helm-charts-in-rancher.md)时，该用户的æƒé™ä¼šç”¨äºŽåˆ›å»ºåº”用所需的所有工作负载/资æºã€‚ + +多集群应用由 _系统用户_ 部署，系统用户还被指定为所有底层资æºçš„创建者。由于实际用户å¯ä»¥ä»ŽæŸä¸ªç›®æ ‡é¡¹ç›®ä¸­åˆ é™¤ï¼Œå› æ­¤ä½¿ç”¨ _系统用户_ 而ä¸æ˜¯å®žé™…用户。如果实际用户从其中一个项目中删除，则该用户将ä¸å†èƒ½å¤Ÿç®¡ç†å…¶ä»–项目的应用。 + +Rancher å…许你选择**项目**或**集群**的角色选项。Rancher å°†å…许你根æ®ç”¨æˆ·çš„æƒé™ä½¿ç”¨å…¶ä¸­ä¸€ä¸ªè§’色进行创建。 + +- **项目** - 相当于[项目æˆå‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)。如果你选择此角色，Rancher 将检查用户是å¦åœ¨æ‰€æœ‰ç›®æ ‡é¡¹ç›®ä¸­è‡³å°‘具有[项目æˆå‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)的角色。虽然用户å¯èƒ½æ²¡æœ‰è¢«æ˜Žç¡®æŽˆäºˆ _项目æˆå‘˜_ 角色，但如果用户是[管ç†å‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)ã€[集群所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)或[项目所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)，则认为该用户具有所需的æƒé™çº§åˆ«ã€‚ + +- **集群** - 相当于[集群所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)。如果你选择此角色，Rancher 将检查用户是å¦åœ¨æ‰€æœ‰ç›®æ ‡é¡¹ç›®ä¸­è‡³å°‘具有[集群所有者](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)的角色。虽然用户å¯èƒ½æ²¡æœ‰è¢«æ˜Žç¡®æŽˆäºˆ _集群所有者_ 角色，但如果用户是[管ç†å‘˜](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)，则认为该用户具有所需的æƒé™çº§åˆ«ã€‚ + +在å¯åŠ¨åº”用时，Rancher 会在å¯åŠ¨åº”用之å‰ç¡®è®¤ä½ åœ¨ç›®æ ‡é¡¹ç›®ä¸­æ˜¯å¦æ‹¥æœ‰è¿™äº›æƒé™ã€‚ + +:::note + +æŸäº›åº”用（如 _Grafana_ 或 _Datadog_）需è¦è®¿é—®ç‰¹å®šé›†ç¾¤çº§åˆ«çš„资æºã€‚è¿™äº›åº”ç”¨å°†éœ€è¦ _集群_ 角色。如果你之åŽå‘现应用需è¦é›†ç¾¤è§’色，则å¯ä»¥å‡çº§å¤šé›†ç¾¤åº”用以更新角色。 + +::: + +## 应用é…置选项 + +对于æ¯ä¸ª Helm Chart，你需è¦è¾“入一个必须的答案列表æ‰èƒ½æˆåŠŸéƒ¨ç½² Chart。由于 Rancher 会将答案作为 `--set` 标志传递给 Helm，因此你必须按照[使用 Helm：–set çš„æ ¼å¼å’Œé™åˆ¶](https://helm.sh/docs/intro/using_helm/#the-format-and-limitations-of---set)中的语法规则æ¥æ ¼å¼åŒ–这些答案。 + +:::note 示例 + +当输入的答案包å«ç”¨é€—å·åˆ†éš”çš„ä¸¤ä¸ªå€¼ï¼ˆå³ `abc, bcd`）时，你需è¦ç”¨åŒå¼•å·å°†å€¼æ‹¬èµ·æ¥ï¼ˆå³ ``"abc, bcd" ``）。 + +::: + +### 使用 questions.yml 文件 + +如果你部署的 Helm Chart åŒ…å« `questions.yml` 文件，Rancher UI 会将此文件转æ¢æˆæ˜“于使用的 UI æ¥æ”¶é›†é—®é¢˜çš„答案。 + +### 原生 Helm Chart 的键值对 + +对于原生 Helm Chart（å³æ¥è‡ª **Helm Stable** 或 **Helm Incubator** 应用商店或自定义 Helm Chart 仓库的 Chart），答案会在 **Answers** 中以键值对的形å¼æ供。这些答案能覆盖默认值。 + +### æˆå‘˜ + +默认情况下，多集群应用åªèƒ½ç”±åº”用的创建者管ç†ã€‚ä½ å¯ä»¥åœ¨**æˆå‘˜**中添加其他用户，以便这些用户管ç†æˆ–查看多集群应用。 + +1. 在**æˆå‘˜**æœç´¢æ¡†ä¸­é”®å…¥æˆå‘˜çš„å称，查找è¦æ·»åŠ çš„用户。 + +2. 为该æˆå‘˜é€‰æ‹©**访问类型**。多集群项目有三ç§è®¿é—®ç±»åž‹ï¼Œè¯·ä»”细阅读并了解这些访问类型的å«ä¹‰ï¼Œä»¥äº†è§£å¤šé›†ç¾¤åº”用æƒé™çš„å¯ç”¨æ–¹å¼ã€‚ + + - **所有者**：此访问类型å¯ä»¥ç®¡ç†å¤šé›†ç¾¤åº”用的任何é…置，包括模æ¿ç‰ˆæœ¬ã€[多集群应用é…置选项](#多集群应用é…置选项)，[应用é…置选项](#应用é…置选项)，å¯ä»¥ä¸Žå¤šé›†ç¾¤åº”用交互的æˆå‘˜ï¼Œä»¥åŠ[自定义应用é…置答案](#覆盖特定项目的应用é…置选项)。由于多集群应用的创建使用与用户ä¸åŒçš„æƒé™é›†ï¼Œå› æ­¤å¤šé›†ç¾¤åº”用的任何 _所有者_ 都å¯ä»¥ç®¡ç†/删除[目标项目](#目标)中的应用，而ä¸éœ€è¦æ˜¾å¼æŽˆæƒè®¿é—®è¿™äº›é¡¹ç›®ã€‚请仅为å—信任的用户é…置此访问类型。 + + - **æˆå‘˜**：此访问类型åªèƒ½ä¿®æ”¹æ¨¡æ¿ç‰ˆæœ¬ã€[应用é…置选项](#应用é…置选项)å’Œ[自定义应用é…置答案](#覆盖特定项目的应用é…置选项)。由于多集群应用的创建使用与用户ä¸åŒçš„æƒé™é›†ï¼Œå› æ­¤å¤šé›†ç¾¤åº”用的任何 _æˆå‘˜_ 都å¯ä»¥ä¿®æ”¹åº”用，而ä¸éœ€è¦æ˜¾å¼æŽˆæƒè®¿é—®è¿™äº›é¡¹ç›®ã€‚请仅为å—信任的用户é…置此访问类型。 + + - **åªè¯»**：此访问类型ä¸èƒ½ä¿®æ”¹å¤šé›†ç¾¤åº”用的任何é…置选项。用户åªèƒ½æŸ¥çœ‹è¿™äº›åº”用。 + + :::caution + + 请确ä¿ä»…为å—信任的用户授予 _所有者_ 或 _æˆå‘˜_ 访问æƒé™ï¼Œå› ä¸ºè¿™äº›ç”¨æˆ·å³ä½¿æ— æ³•ç›´æŽ¥è®¿é—®é¡¹ç›®ï¼Œä¹Ÿå°†è‡ªåŠ¨èƒ½å¤Ÿç®¡ç†ä¸ºæ­¤å¤šé›†ç¾¤åº”用创建的应用。 + + ::: + +### 覆盖特定项目的应用é…置选项 + +多集群应用的主è¦ä¼˜åŠ¿ä¹‹ä¸€ï¼Œæ˜¯èƒ½å¤Ÿåœ¨å¤šä¸ªé›†ç¾¤/项目中使用相åŒé…置部署相åŒçš„应用。在æŸäº›æƒ…况下，你å¯èƒ½éœ€è¦ä¸ºæŸä¸ªç‰¹å®šé¡¹ç›®ä½¿ç”¨ç¨å¾®ä¸åŒçš„é…置选项，但你ä¾ç„¶å¸Œæœ›ç»Ÿä¸€ç®¡ç†è¯¥åº”用与其他匹é…的应用。此时，你å¯ä»¥ä¸ºè¯¥é¡¹ç›®è¦†ç›–特定的[应用é…置选项](#应用é…置选项)，而ä¸éœ€è¦åˆ›å»ºå…¨æ–°çš„应用。 + +1. 在**答案覆盖**中，å•å‡»**添加覆盖**。 + +2. 对于æ¯ä¸ªè¦†ç›–，你å¯ä»¥é€‰æ‹©ä»¥ä¸‹å†…容： + + - **范围**：在é…置选项中选择è¦è¦†ç›–哪些目标项目的答案。 + + - **问题**：选择è¦è¦†ç›–的问题。 + + - **答案**：输入è¦ä½¿ç”¨çš„答案。 + +## å‡çº§å¤šé›†ç¾¤åº”用角色和项目 + +- **在现有的多集群应用上更改角色** + 多集群应用的创建者和任何具有“所有者â€è®¿é—®ç±»åž‹çš„用户都å¯ä»¥å‡çº§å…¶**角色**。添加新角色时，我们会检查用户在所有当å‰ç›®æ ‡é¡¹ç›®ä¸­æ˜¯å¦å…·æœ‰è¯¥è§’色。Rancher ä¼šæ ¹æ® `Roles` 字段的安装部分，相应地检查用户是å¦å…·æœ‰å…¨å±€ç®¡ç†å‘˜ã€é›†ç¾¤æ‰€æœ‰è€…或项目所有者的角色。 + +- **添加/删除目标项目** +1. 多集群应用的创建者和任何具有“所有者â€è®¿é—®ç±»åž‹çš„用户都添加或移除目标项目。添加新项目时，我们检查此请求的调用者是å¦å…·æœ‰å¤šé›†ç¾¤åº”用中定义的所有角色。Rancher 会检查用户是å¦å…·æœ‰å…¨å±€ç®¡ç†å‘˜ã€é›†ç¾¤æ‰€æœ‰è€…和项目所有者的角色。 +2. 删除目标项目时，我们ä¸ä¼šè¿›è¡Œè¿™äº›æˆå‘˜èµ„格检查。这是因为调用者的æƒé™å¯èƒ½ä¸Žç›®æ ‡é¡¹ç›®æœ‰å…³ï¼Œæˆ–者由于该项目已被删除导致调用者希望将该项目从目标列表中删除。 + + +## å¤šé›†ç¾¤åº”ç”¨ç®¡ç† + +与åŒä¸€ç±»åž‹çš„多个å•ç‹¬åº”用相比，使用多集群应用的好处之一是易于管ç†ã€‚ä½ å¯ä»¥å…‹éš†ã€å‡çº§æˆ–回滚多集群应用。 + +:::note 先决æ¡ä»¶ï¼š + +`Legacy` 功能开关已å¯ç”¨ã€‚ + +::: + +1. 在左上角，å•å‡»**☰ > 多集群应用**。 + +2. 选择è¦å¯¹å…¶æ‰§è¡Œæ“作的多集群应用，然åŽå•å‡» **â‹®**。选择以下选项之一： + + * **克隆**：创建å¦ä¸€ä¸ªå…·æœ‰ç›¸åŒé…置的多集群应用。通过使用此选项，你å¯ä»¥è½»æ¾å¤åˆ¶å¤šé›†ç¾¤åº”用。 + * **å‡çº§**：å‡çº§å¤šé›†ç¾¤åº”用以更改æŸäº›é…置。在为多集群应用执行å‡çº§æ—¶ï¼Œå¦‚果你有åˆé€‚çš„[访问类型](#æˆå‘˜)，则å¯ä»¥ä¿®æ”¹[å‡çº§ç­–ç•¥](#å‡çº§)。 + * **回滚**：将你的应用回滚到特定版本。如果你的一个或多个[目标](#目标)的多集群应用在å‡çº§åŽå‡ºçŽ°é—®é¢˜ï¼Œä½ å¯ä»¥ä½¿ç”¨ Rancher 存储的多达 10 个多集群应用版本进行回滚。回滚多集群应用会æ¢å¤**所有**目标集群和项目的应用，而ä¸ä»…仅是å—å‡çº§é—®é¢˜å½±å“的目标。 + +## 删除多集群应用 + +:::note 先决æ¡ä»¶ï¼š + +`Legacy` 功能开关已å¯ç”¨ã€‚ + +::: + +1. 在左上角，å•å‡»**☰ > 多集群应用**。 + +2. 选择è¦åˆ é™¤çš„多集群应用，然åŽå•å‡»**â‹® > 删除**。删除多集群应用会删除所有目标项目中的所有应用和命å空间。 + + :::note + + ä¸èƒ½ç‹¬ç«‹åˆ é™¤åœ¨ç›®æ ‡é¡¹ç›®ä¸­ä¸ºå¤šé›†ç¾¤åº”用创建的应用。åªæœ‰åˆ é™¤å¤šé›†ç¾¤åº”用åŽæ‰èƒ½åˆ é™¤è¿™äº›åº”用。 + + ::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md new file mode 100644 index 00000000000..554725a2e8e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -0,0 +1,130 @@ +--- +title: 创建应用 +--- + +Rancher 的应用市场基于 Helm 仓库和 Helm Chart。你å¯ä»¥æ·»åŠ åŸºäºŽ HTTP 的标准 Helm 仓库以åŠä»»ä½•åŒ…å« Chart çš„ Git 仓库。 + +:::tip + +æœ‰å…³å¼€å‘ Chart 的完整演示，请å‚阅 Helm 官方文档中的 [Chart 模æ¿å¼€å‘者指å—](https://helm.sh/docs/chart_template_guide/)。 + +::: + + + +## Chart 类型 + +Rancher 支æŒä¸¤ç§ä¸åŒç±»åž‹çš„ Chart，分别是 Helm Chart å’Œ Rancher Chart。 + +### Helm Chart + +原生 Helm Chart 包括一个应用以åŠè¿è¡Œå®ƒæ‰€éœ€çš„软件。部署原生 Helm Chart 时，你å¯ä»¥åœ¨ YAML 编辑器中æä¾› Chart çš„å‚数值。 + +### Rancher Chart + +Rancher Chart 是原生 helm Chart，包å«ä¸¤ä¸ªå¯å¢žå¼ºç”¨æˆ·ä½“验的文件 `app-readme.md` å’Œ `questions.yaml`。在 [Rancher Chart 的附加文件](#rancher-chart-的附加文件)中了解更多信æ¯ã€‚ + +Rancher Chart 添加了简化的 Chart æè¿°å’Œé…置表å•ï¼Œä½¿åº”用部署å˜å¾—容易。Rancher 用户无需通读整个 Helm å˜é‡åˆ—表å³å¯äº†è§£å¦‚何å¯åŠ¨åº”用。 + +## Chart 目录结构 + +ä½ å¯ä»¥åœ¨åŸºäºŽ HTTP 的标准 Helm 仓库中æä¾› Helm Chart。有关更多信æ¯ï¼Œè¯·å‚阅 Helm 官方文档中的 [Chart 仓库指å—](https://helm.sh/docs/topics/chart_repository)。 + +或者，你å¯ä»¥åœ¨ Git 仓库中组织 Chart 并将其直接添加到应用市场。 + +下表演示了 Git 仓库的目录结构。`charts` 目录是仓库基础下的顶层目录。将仓库添加到 Rancher 将公开其中包å«çš„所有 Chart。`questions.yaml`ã€`README.md` å’Œ `requirements.yml` 文件是针对于 Rancher Chart 的，但对于自定义 Chart 是å¯é€‰çš„。 + +``` +/ + │ + ├── charts/ + │ ├── / # 这个目录å称将作为 Chart å称出现在 Rancher UI 中。 + │ │ ├── / # 这个级别的æ¯ä¸ªç›®å½•æ供了ä¸åŒçš„应用版本，å¯ä»¥åœ¨ Rancher UI çš„ Chart 中选择。 + │ │ │ ├── Chart.yaml # 必需的 Helm Chart ä¿¡æ¯æ–‡ä»¶ã€‚ + │ │ │ ├── questions.yaml # 在 Rancher UI 中显示的表å•é—®é¢˜ã€‚问题显示在é…置选项中。* + │ │ │ ├── README.md # å¯é€‰ï¼šåœ¨ Rancher UI 中显示的 Helm 自述文件。此文本显示在详细说明中。 + │ │ │ ├── requirements.yml # å¯é€‰ï¼šåˆ—出 Chart çš„ä¾èµ–项的 YAML 文件。 + │ │ │ ├── values.yml # Chart 的默认é…置值。 + │ │ │ ├── templates/ # 包å«æ¨¡æ¿çš„目录，与 values.yml ä¸€èµ·èƒ½ç”Ÿæˆ Kubernetes YAML。 +``` + +## Rancher Chart 的附加文件 + +在创建你的自定义目录之å‰ï¼Œä½ éœ€è¦å¤§è‡´äº†è§£ Rancher chart 与原生 Helm chart 的区别。Rancher Chart 的目录结构与 Helm Chart 略有ä¸åŒã€‚Rancher Chart 包å«ä¸¤ä¸ª Helm Chart 没有的文件： + +- `app-readme.md` + + 在 Chart çš„ UI 标头中æä¾›æ述性文本的文件。 + +- `questions.yml` + + 包å«è¡¨å•é—®é¢˜çš„文件。这些表å•é—®é¢˜ç®€åŒ–了 Chart 的部署。没有它，你必须使用更å¤æ‚çš„ YAML é…ç½®æ¥é…置部署。下图显示了 Rancher Chartï¼ˆåŒ…å« `questions.yml`）和原生 Helm Chart（ä¸åŒ…å«ï¼‰ä¹‹é—´çš„区别。 + +
带有 questions.yml 的 Rancher Chart（上）与 Helm Chart（下）
+ + ![questions.yml](/img/rancher-app-2.6.png) + ![values.yaml](/img/helm-app-2.6.png) + + +### Chart.yaml 注释 + +Rancher 支æŒä½ æ·»åŠ åˆ° `Chart.yaml` 文件的其他注释。这些注释å…许你定义应用ä¾èµ–项或é…置其他 UI 默认值： + +| 注解 | æè¿° | 示例 | +| --------------------------------- | ----------- | ------- | +| catalog.cattle.io/auto-install | 如果设置，将在安装此 Chart 之å‰å…ˆå®‰è£…指定 Chart 的指定版本。 | other-chart-name=1.0.0 | +| catalog.cattle.io/display-name | è¦åœ¨åº”用市场中显示的å称，而ä¸æ˜¯ Chart 本身的å称。 | Chart 的显示å称 | +| catalog.cattle.io/namespace | 用于部署 Chart 的固定命å空间。如果设置，则用户无法更改。 | fixed-namespace | +| catalog.cattle.io/release-name | Helm 安装的固定版本å称。如果设置，则用户无法更改。 | fixed-release-name | +| catalog.cattle.io/requests-cpu | 应该在集群中ä¿ç•™çš„ CPU 总é‡ã€‚如果å¯ç”¨ CPU 资æºå°‘于该值，将显示警告。 | 2000m | +| catalog.cattle.io/requests-memory | 应该在集群中ä¿ç•™çš„内存总é‡ã€‚如果å¯ç”¨å†…存资æºå°‘于该值，将显示警告。 | 2Gi | +| catalog.cattle.io/os | é™åˆ¶å¯ä»¥å®‰è£…æ­¤ Chart çš„æ“作系统。å¯ç”¨å€¼ï¼š`linux`ã€`windows`。默认：无é™åˆ¶ | linux | + +### questions.yml + +`questions.yml` 中大部分是å‘最终用户æ出的问题，但也有一部分å¯ä»¥åœ¨æ­¤æ–‡ä»¶ä¸­è®¾ç½®å­—段。 + +### 最低/最高 Rancher 版本 + +ä½ å¯ä»¥ä¸ºæ¯ä¸ª Chart 添加最低和/或最高的 Rancher 版本，这决定了该 Chart 是å¦å¯ä»¥ä»Ž Rancher 部署。 + +:::note + +Rancher 版本带有 `v` å‰ç¼€ï¼Œä½†æ˜¯ä½¿ç”¨æ­¤é€‰é¡¹æ—¶è¯·*ä¸è¦*包括å‰ç¼€ã€‚ + +::: + +``` +rancher_min_version: 2.3.0 +rancher_max_version: 2.3.99 +``` + +### Question å˜é‡å‚考 + +æ­¤å‚考包å«å¯ä»¥åµŒå¥—在 `questions:` 下的 `questions.yml` 中的å˜é‡ï¼š + +| å˜é‡ | 类型 | å¿…å¡« | æè¿° | +| ------------- | ------------- | --- |------------- | +| variable | string | true | 定义 `values.yml` 文件中指定的å˜é‡å，嵌套对象使用 `foo.bar`。 | +| label | string | true | 定义 UI 标签。 | +| description | string | false | 指定å˜é‡çš„æ述。 | +| type | string | false | 如果未指定，则默认为 `string`（支æŒçš„类型为 stringã€multilineã€booleanã€intã€enumã€passwordã€storageclassã€hostnameã€pvc å’Œ secret）。 | +| required | bool | false | 定义å˜é‡æ˜¯å¦æ˜¯å¿…须的（true \| false）。 | +| default | string | false | 指定默认值。仅在 `values.yml` 文件中没有对应值时使用。 | +| group | string | false | 按输入值对问题进行分组。 | +| min_length | int | false | 最å°å­—符长度。 | +| max_length | int | false | 最大字符长度。 | +| min | int | false | 最å°æ•´æ•°é•¿åº¦ã€‚ | +| max | int | false | 最大整数长度。 | +| options | []string | false | 为 `enum` 类型的å˜é‡æŒ‡å®šé€‰é¡¹ï¼Œä¾‹å¦‚：options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer" | +| valid_chars | string | false | 输入字符验è¯çš„正则表达å¼ã€‚ | +| invalid_chars | string | false | 无效输入字符验è¯çš„正则表达å¼ã€‚ | +| subquestions | []subquestion | false | 添加一组å­é—®é¢˜ã€‚ | +| show_if | string | false | 如果æ¡ä»¶å˜é‡ä¸º true，则显示当å‰å˜é‡ã€‚例如 `show_if: "serviceType=Nodeport"` | +| show\_subquestion_if | string | false | 如果为 true 或等于æŸä¸ªé€‰é¡¹ï¼Œåˆ™æ˜¾ç¤ºå­é—®é¢˜ã€‚例如 `show_subquestion_if: "true"` | + +:::note + +`subquestions[]` ä¸èƒ½åŒ…å« `subquestions` 或 `show_subquestions_if` 键，但支æŒä¸Šè¡¨ä¸­çš„所有其他键。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md new file mode 100644 index 00000000000..6486e4fc4ca --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md @@ -0,0 +1,175 @@ +--- +title: 设置 Amazon NLB 网络负载å‡è¡¡å™¨ +--- + +本文介ç»äº†å¦‚何在 Amazon EC2 æœåŠ¡ä¸­è®¾ç½® Amazon NLB 网络负载å‡è¡¡å™¨ï¼Œç”¨äºŽå°†æµé‡è½¬å‘到 EC2 上的多个实例中。 + +这些示例中，负载å‡è¡¡å™¨å°†æµé‡è½¬å‘到三个 Rancher Server 节点。如果 Rancher 安装在 RKE Kubernetes 集群上，则需è¦ä¸‰ä¸ªèŠ‚点。如果 Rancher 安装在 K3s Kubernetes 集群上，则åªéœ€è¦ä¸¤ä¸ªèŠ‚点。 + +本文介ç»çš„åªæ˜¯é…置负载å‡è¡¡å™¨çš„其中一ç§æ–¹å¼ã€‚其他负载å‡è¡¡å™¨å¦‚传统负载路由器（Classic Load Balancer）和应用负载路由器（Application Load Balancer），也å¯ä»¥å°†æµé‡è½¬å‘到 Rancher Server 节点。 + +Rancher 仅支æŒä½¿ç”¨ Amazon NLB 以 `TCP` 模å¼ç»ˆæ­¢ 443 端å£çš„æµé‡ï¼Œè€Œä¸æ”¯æŒ `TLS` 模å¼ã€‚这试因为在 NLB 终止时，NLB ä¸ä¼šå°†æ­£ç¡®çš„标头注入请求中。如果你想使用由 Amazon Certificate Manager (ACM) 托管的è¯ä¹¦ï¼Œè¯·ä½¿ç”¨ ALB。 + + + +## è¦æ±‚ + +你已在 EC2 中创建了 Linux 实例。此外，负载å‡è¡¡å™¨ä¼šæŠŠæµé‡è½¬å‘到这些节点。 + +## 1. 创建目标组 + +首先，为 **TCP** å议创建两个目标组，其中一个使用 TCP ç«¯å£ 443，å¦ä¸€ä¸ªä½¿ç”¨ TCP ç«¯å£ 80（用于é‡å®šå‘到 TCP ç«¯å£ 443）。然åŽï¼Œå°† Linux 节点添加到这些组中。 + +é…ç½® NLB 的第一个步骤是创建两个目标组。一般æ¥è¯´ï¼Œåªéœ€è¦ç«¯å£ 443 å°±å¯ä»¥è®¿é—® Rancherã€‚ä½†æ˜¯ï¼Œç”±äºŽç«¯å£ 80 çš„æµé‡ä¼šè¢«è‡ªåŠ¨é‡å®šå‘åˆ°ç«¯å£ 443，因此，你也å¯ä»¥ä¸ºç«¯å£ 80 也添加一个监å¬å™¨ã€‚ + +ä¸ç®¡ä½¿ç”¨çš„是 NGINX Ingress 还是 Traefik Ingress Controller，Ingress éƒ½åº”è¯¥å°†ç«¯å£ 80 çš„æµé‡é‡å®šå‘åˆ°ç«¯å£ 443。以下为æ“作步骤： + +1. 登录到 [Amazon AWS 控制å°](https://console.aws.amazon.com/ec2/)。确ä¿é€‰æ‹©çš„**区域**是你创建 EC2 实例 （Linux 节点）的区域。 +1. 选择**æœåŠ¡** > **EC2**，找到**è´Ÿè½½å‡è¡¡å™¨**并打开**目标组**。 +1. å•å‡»**创建目标组**，然åŽåˆ›å»ºç”¨äºŽ TCP ç«¯å£ 443 的第一个目标组。 + +:::note + +ä¸åŒ Ingress çš„å¥åº·æ£€æŸ¥å¤„ç†æ–¹æ³•ä¸åŒã€‚详情请å‚阅[本节](#nginx-ingress-å’Œ-traefik-ingress-çš„å¥åº·æ£€æŸ¥è·¯å¾„)。 + +::: + +#### 目标组（TCP ç«¯å£ 443） + +æ ¹æ®ä¸‹è¡¨é…置第一个目标组： + +| 选项 | 设置 | +|-------------------|-------------------| +| 目标组å称 | `rancher-tcp-443` | +| 目标类型 | `instance` | +| åè®® | `TCP` | +| ç«¯å£ | `443` | +| VPC | 选择 VPC | + +å¥åº·æ£€æŸ¥è®¾ç½®ï¼š + +| 选项 | 设置 | +|---------------------|-----------------| +| åè®® | TCP | +| ç«¯å£ | `override`,`80` | +| å¥åº·é˜ˆå€¼ | `3` | +| ä¸æ­£å¸¸é˜ˆå€¼ | `3` | +| 超时 | `6 秒` | +| é—´éš” | `10 秒` | + +å•å‡»**创建目标组**，然åŽåˆ›å»ºç”¨äºŽ TCP ç«¯å£ 80 的第二个目标组。 + +#### 目标组（TCP ç«¯å£ 80） + +æ ¹æ®ä¸‹è¡¨é…置第二个目标组： + +| 选项 | 设置 | +|-------------------|------------------| +| 目标组å称 | `rancher-tcp-80` | +| 目标类型 | `instance` | +| åè®® | `TCP` | +| ç«¯å£ | `80` | +| VPC | 选择 VPC | + + +å¥åº·æ£€æŸ¥è®¾ç½®ï¼š + +| 选项 | 设置 | +|---------------------|----------------| +| åè®® | TCP | +| ç«¯å£ | `traffic port` | +| å¥åº·é˜ˆå€¼ | `3` | +| ä¸æ­£å¸¸é˜ˆå€¼ | `3` | +| 超时 | `6 秒` | +| é—´éš” | `10 秒` | + +## 2. 注册目标 + +接下æ¥ï¼Œå°† Linux 节点添加到两个目标组中。 + +选择å为 **rancher-tcp-443** 的目标组，点击**目标**选项å¡å¹¶é€‰æ‹©**编辑**。 + +![](/img/ha/nlb/edit-targetgroup-443.png) + +选择你è¦æ·»åŠ çš„实例（Linux 节点），然åŽå•å‡»**添加到已注册**。 + +*** +**将目标添加到目标组 TCP ç«¯å£ 443**
+ +![](/img/ha/nlb/add-targets-targetgroup-443.png) + +*** +**已将目标添加到目标组 TCP ç«¯å£ 443**
+ +![](/img/ha/nlb/added-targets-targetgroup-443.png) + +添加实例åŽï¼Œå•å‡»å³ä¸‹æ–¹çš„**ä¿å­˜**。 + +å°† **rancher-tcp-443** 替æ¢ä¸º **rancher-tcp-80**，然åŽé‡å¤ä¸Šè¿°æ­¥éª¤ã€‚你需è¦å°†ç›¸åŒçš„实例作为目标添加到此目标组。 + +## 3. 创建 NLB + +使用 Amazon çš„å‘导创建网络负载å‡è¡¡å™¨ã€‚在这个过程中，你需è¦æ·»åŠ åœ¨ [1. 创建目标组](#1-创建目标组)中创建的目标组。 + +1. 在网页æµè§ˆå™¨ä¸­ï¼Œå¯¼èˆªåˆ° [Amazon EC2 控制å°](https://console.aws.amazon.com/ec2/)。 + +2. 在导航æ ä¸­ï¼Œé€‰æ‹©**è´Ÿè½½å‡è¡¡** > **è´Ÿè½½å‡è¡¡å™¨**。 + +3. å•å‡»**创建负载å‡è¡¡å™¨**。 + +4. 选择**网络负载å‡è¡¡å™¨**并å•å‡»**创建**。然åŽï¼Œå¡«å†™æ¯ä¸ªè¡¨æ ¼ã€‚ + +- [步骤 1：é…置负载å‡è¡¡å™¨](#步骤-1é…置负载å‡è¡¡å™¨) +- [步骤 2：é…置路由](#步骤-2é…置路由) +- [步骤 3：注册目标](#步骤-3注册目标) +- [步骤 4：审核](#步骤-4审核) + +### 步骤 1：é…置负载å‡è¡¡å™¨ + +在表å•ä¸­è®¾ç½®ä»¥ä¸‹å­—段： + +- **å称**：`rancher` +- **Scheme**：`internal` 或 `internet-facing`。实例和 VPC çš„é…置决定了 NLB çš„ Scheme。如果你的实例没有绑定公共 IP，或者你åªéœ€è¦é€šè¿‡å†…网访问 Rancher，请将 NLB çš„ Scheme 设置为 `internal` 而ä¸æ˜¯ `internet-facing`。 +- **监å¬å™¨**：负载å‡è¡¡å™¨å议需è¦æ˜¯ `TCP`，而且负载å‡è¡¡å™¨ç«¯å£éœ€è¦è®¾ä¸º `443`。 +- **å¯ç”¨åŒºï¼š**选择你的**VPC**å’Œ**å¯ç”¨åŒº**。 + +### 步骤 2：é…置路由 + +1. 从**目标组**下拉列表中，选择 **现有目标组**。 +1. 从**å称**下拉列表中，选择 `rancher-tcp-443`。 +1. 打开**高级å¥åº·æ£€æŸ¥è®¾ç½®**，并将**é—´éš”**设为 `10 秒`。 + +### 步骤 3：注册目标 + +由于你已ç»åœ¨å…ˆå‰æ­¥éª¤æ³¨å†Œäº†ç›®æ ‡ï¼Œå› æ­¤ä½ åªéœ€å•å‡» **下一步：审核**。 + +### 步骤 4：审核 + +检查负载å‡è¡¡å™¨ä¿¡æ¯æ— è¯¯åŽï¼Œå•å‡»**创建**。 + +AWS å®Œæˆ NLB 创建åŽï¼Œå•å‡»**关闭**。 + +## 4. 为 TCP ç«¯å£ 80 å‘ NLB 添加监å¬å™¨ + +1. 选择新创建的 NLB 并选择**监å¬å™¨**选项å¡ã€‚ + +2. å•å‡»**添加监å¬å™¨**。 + +3. 使用 `TCP`:`80` 作为**åè®®**:**端å£**。 + +4. å•å‡»**添加æ“作**并选择**转å‘到..**。 + +5. 从**转å‘到**下拉列表中，选择 `rancher-tcp-80`。 + +6. å•å‡»å³ä¸Šè§’çš„**ä¿å­˜**。 + +## NGINX Ingress å’Œ Traefik Ingress çš„å¥åº·æ£€æŸ¥è·¯å¾„ + +K3s å’Œ RKE Kubernetes 集群使用的默认 Ingress ä¸åŒï¼Œå› æ­¤å¯¹åº”çš„å¥åº·æ£€æŸ¥æ–¹å¼ä¹Ÿä¸åŒã€‚ + +RKE Kubernetes 集群默认使用 NGINX Ingress，而 K3s Kubernetes 集群默认使用 Traefik Ingress。 + +- **Traefik**：默认å¥åº·æ£€æŸ¥è·¯å¾„是 `/ping`。默认情况下，ä¸ç®¡ä¸»æœºå¦‚何，`/ping` 总是匹é…，而且 [Traefik 自身](https://docs.traefik.io/operations/ping/)总会å“应。 +- **NGINX Ingress**：NGINX Ingress Controller 的默认åŽç«¯æœ‰ä¸€ä¸ª `/healthz` 端点。默认情况下，ä¸ç®¡ä¸»æœºå¦‚何，`/healthz` 总是匹é…，而且 [`ingress-nginx` 自身](https://github.com/kubernetes/ingress-nginx/blob/0cbe783f43a9313c9c26136e888324b1ee91a72f/charts/ingress-nginx/values.yaml#L212)总会å“应。 + +想è¦ç²¾ç¡®æ¨¡æ‹Ÿå¥åº·æ£€æŸ¥ï¼Œæœ€å¥½æ˜¯ä½¿ç”¨ Host 标头（Rancher hostname）加上 `/ping` 或 `/healthz`（分别对应 K3s å’Œ RKE 集群）æ¥èŽ·å– Rancher Pod çš„å“应，而ä¸æ˜¯ Ingress çš„å“应。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md new file mode 100644 index 00000000000..277961c5dae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md @@ -0,0 +1,73 @@ +--- +title: 为高å¯ç”¨ K3s Kubernetes 集群设置基础设施 +--- + +本教程旨在帮助你为 Rancher Management Server é…置底层基础设施。 + +æˆ‘ä»¬æ ¹æ® Rancher 的安装ä½ç½®ï¼ˆK3s Kubernetes 集群ã€RKE Kubernetes 集群或å•ä¸ª Docker 容器）为专用于 Rancher çš„ Kubernetes 集群推èä¸åŒåŸºç¡€è®¾æ–½ã€‚ + +有关æ¯ä¸ªå®‰è£…选项的详情，请å‚è§[本页](../../../pages-for-subheaders/installation-and-upgrade.md)。 + +:::note é‡è¦æ示： + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºï¼ˆæ•°æ®ä¸­å¿ƒï¼‰ã€‚ + +::: + +如需在高å¯ç”¨ K3s 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **2 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商中的虚拟机。 +- **1 个外部数æ®åº“**：用于存储集群数æ®ã€‚建议使用 MySQL。 +- **1 个负载å‡è¡¡å™¨**：用于将æµé‡è½¬å‘到这两个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +### 1. é…ç½® Linux 节点 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置外部数æ®åº“ + +K3s 与其他 Kubernetes å‘行版ä¸åŒï¼Œåœ¨äºŽå…¶æ”¯æŒä½¿ç”¨ etcd 以外的数æ®åº“æ¥è¿è¡Œ Kubernetes。该功能让 Kubernetes è¿ç»´æ›´åŠ çµæ´»ã€‚ä½ å¯ä»¥æ ¹æ®å®žé™…情况选择åˆé€‚çš„æ•°æ®åº“。 + +对于 K3s 高å¯ç”¨å®‰è£…，你需è¦é…置一个 [MySQL](https://www.mysql.com/) 外部数æ®åº“。Rancher 已在使用 MySQL 5.7 作为数æ®å­˜å‚¨çš„ K3s Kubernetes 集群上进行了测试。 + +在使用 K3s 安装脚本安装 Kubernetes 时，你需è¦ä¼ å…¥ K3s 连接数æ®åº“的详细信æ¯ã€‚ + +如需获å–é…ç½® MySQL æ•°æ®åº“示例，请å‚è§[在 Amazon RDS æœåŠ¡ä¸­é…ç½® MySQL](mysql-database-in-amazon-rds.md) 的教程。 + +如需获å–é…ç½® K3s 集群数æ®åº“的所有å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/datastore/)。 + +### 3. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到两个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，K3s 工具会部署一个 Traefik Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 Traefik Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚Traefik Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 4. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md new file mode 100644 index 00000000000..24387c1f786 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md @@ -0,0 +1,64 @@ +--- +title: 为高å¯ç”¨ RKE Kubernetes 集群设置基础设施 +--- + +本教程旨在帮助你创建一个高å¯ç”¨çš„ RKE 集群，用于安装 Rancher Server。 + +:::note é‡è¦æ示： + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºï¼ˆæ•°æ®ä¸­å¿ƒï¼‰ã€‚ + +::: + +如需在高å¯ç”¨ RKE 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商（例如 Amazon EC2，GCE 或 vSphere）中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将å‰ç«¯æµé‡è½¬å‘到这三个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸæˆ–æ•°æ®ä¸­å¿ƒã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºã€‚ + +### 为什么使用三个节点？ + +在 RKE 集群中，Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。而这个 etcd æ•°æ®åº“在这三个节点上è¿è¡Œã€‚ + +为了选举出大多数 etcd 节点认å¯çš„ etcd 集群 leader，etcd æ•°æ®åº“需è¦å¥‡æ•°ä¸ªèŠ‚点。如果 etcd æ•°æ®åº“无法选出 leader，etcd å¯èƒ½ä¼šå‡ºçŽ°[脑裂（split brain）](https://www.quora.com/What-is-split-brain-in-distributed-systems)的问题，此时你需è¦ä½¿ç”¨å¤‡ä»½æ¢å¤é›†ç¾¤ã€‚如果三个 etcd 节点之一å‘生故障，其余两个节点å¯ä»¥é€‰æ‹©ä¸€ä¸ª leader，因为它们是 etcd 节点总数的大多数部分。 + +### 1. é…ç½® Linux 节点 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到三个节点中的任æ„一个节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 3. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md new file mode 100644 index 00000000000..0af8f17b3bb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md @@ -0,0 +1,58 @@ +--- +title: 为高å¯ç”¨ RKE2 Kubernetes 集群设置基础设施 +--- + +本教程旨在帮助你为 Rancher Management Server é…置底层基础设施。 + +æˆ‘ä»¬æ ¹æ® Rancher 的安装ä½ç½®ï¼ˆRKE2 Kubernetes 集群ã€RKE Kubernetes 集群或å•ä¸ª Docker 容器）为专用于 Rancher çš„ Kubernetes 集群推èä¸åŒåŸºç¡€è®¾æ–½ã€‚ + +:::note é‡è¦æ示： + +这些节点必须ä½äºŽåŒä¸€ä¸ªåŒºåŸŸã€‚但是你å¯ä»¥æŠŠè¿™äº›æœåŠ¡å™¨æ”¾åœ¨ä¸åŒçš„å¯ç”¨åŒºï¼ˆæ•°æ®ä¸­å¿ƒï¼‰ã€‚ + +::: + +如需在高å¯ç”¨ RKE2 集群中安装 Rancher Management Server，我们建议é…置以下基础设施： + +- **3 个 Linux 节点**：å¯ä»¥æ˜¯ä½ çš„云æ供商中的虚拟机。 +- **1 个负载å‡è¡¡å™¨**：用于将æµé‡è½¬å‘到这两个节点中。 +- **1 个 DNS 记录**：用于将 URL 映射到负载å‡è¡¡å™¨ã€‚æ­¤ DNS 记录将æˆä¸º Rancher Server çš„ URL，下游集群需è¦å¯ä»¥è®¿é—®åˆ°è¿™ä¸ªåœ°å€ã€‚ + +### 1. é…ç½® Linux 节点 + +请确ä¿ä½ çš„节点满足[æ“作系统，容器è¿è¡Œæ—¶ï¼Œç¡¬ä»¶å’Œç½‘络](../../../pages-for-subheaders/installation-requirements.md)的常规è¦æ±‚。 + +如需获å–é…ç½® Linux 节点的示例，请å‚è§[在 Amazon EC2 中é…置节点](nodes-in-amazon-ec2.md)的教程。 + +### 2. é…置负载å‡è¡¡å™¨ + +你还需è¦è®¾ç½®ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨ï¼Œæ¥å°†æµé‡é‡å®šå‘到所有节点上的 Rancher 副本。é…ç½®åŽï¼Œå½“å•ä¸ªèŠ‚点ä¸å¯ç”¨æ—¶ï¼Œç»§ç»­ä¿éšœä¸Ž Rancher Management Server 的通信。 + +在åŽç»­æ­¥éª¤ä¸­é…ç½® Kubernetes 时，RKE2 工具会部署一个 NGINX Ingress Controller。该 Controller å°†ä¾¦å¬ worker 节点的 80 端å£å’Œ 443 端å£ï¼Œä»¥å“应å‘é€ç»™ç‰¹å®šä¸»æœºåçš„æµé‡ã€‚ + +在安装 Rancher åŽï¼ˆä¹Ÿæ˜¯åœ¨åŽç»­æ­¥éª¤ä¸­ï¼‰ï¼ŒRancher 系统将创建一个 Ingress 资æºã€‚该 Ingress 通知 NGINX Ingress Controller 监å¬å‘å¾€ Rancher 主机åçš„æµé‡ã€‚NGINX Ingress Controller 在收到å‘å¾€ Rancher 主机åçš„æµé‡æ—¶ï¼Œä¼šå°†å…¶è½¬å‘到集群中正在è¿è¡Œçš„ Rancher Server Pod。 + +在你的实现中，你å¯ä»¥è€ƒè™‘是å¦éœ€è¦ä½¿ç”¨ 4 层或 7 层的负载å‡è¡¡å™¨ï¼š + +- **4 层负载å‡è¡¡å™¨**：两ç§é€‰æ‹©ä¸­è¾ƒä¸ºç®€å•çš„一ç§ï¼Œå®ƒå°† TCP æµé‡è½¬å‘到你的节点中。我们建议使用 4 层负载å‡è¡¡å™¨ï¼Œå°†æµé‡ä»Ž TCP/80 端å£å’Œ TCP/443 端å£è½¬å‘到 Rancher Management 集群节点上。集群上的 Ingress Controller 会将 HTTP æµé‡é‡å®šå‘到 HTTPS，并在 TCP/443 端å£ä¸Šç»ˆæ­¢ SSL/TLS。Ingress Controller 会将æµé‡è½¬å‘到 Rancher deployment 中 Ingress Pod çš„ TCP/80 端å£ã€‚ +- **7 层负载å‡è¡¡å™¨**：相对比较å¤æ‚，但功能更全é¢ã€‚例如，与 Rancher 本身进行 TLS 终止相å，7 层负载å‡è¡¡å™¨èƒ½å¤Ÿåœ¨è´Ÿè½½å‡è¡¡å™¨å¤„å¤„ç† TLS 终止。如果你需è¦é›†ä¸­åœ¨åŸºç¡€è®¾æ–½ä¸­è¿›è¡Œ TLS 终止，7 层负载å‡è¡¡å¯èƒ½ä¼šå¾ˆé€‚åˆä½ ã€‚7 层负载å‡è¡¡è¿˜èƒ½è®©ä½ çš„è´Ÿè½½å‡è¡¡å™¨åŸºäºŽ HTTP 属性（例如 cookie 等）åšå‡ºå†³ç­–，而 4 层负载å‡è¡¡å™¨åˆ™ä¸èƒ½ã€‚如果你选择在 7 层负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ SSL/TLS æµé‡ï¼Œåˆ™åœ¨å®‰è£… Rancher 时（åŽç»­æ­¥éª¤ï¼‰éœ€è¦ä½¿ç”¨ `--set tls=external` 选项。详情请å‚è§ [Rancher Helm Chart 选项](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +如需获å–é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨çš„示例，请å‚è§[本页](nginx-load-balancer.md)。 + +如需获å–如何é…ç½® Amazon ELB 网络负载å‡è¡¡å™¨çš„指å—，请å‚è§[本页](amazon-elb-load-balancer.md)。 + +:::caution + +安装åŽï¼Œè¯·å‹¿å°†æ­¤è´Ÿè½½å‡è¡¡ï¼ˆä¾‹å¦‚ `local` 集群 Ingress）用于 Rancher 以外的应用。如果此 Ingress 与其他应用共享，在其他应用的 Ingress é…ç½®é‡æ–°åŠ è½½åŽï¼Œå¯èƒ½å¯¼è‡´ Rancher 出现 websocket 错误。我们建议把 `local` 集群专用给 Rancher，ä¸è¦åœ¨é›†ç¾¤å†…部署其他应用。 + +::: + +### 4. é…ç½® DNS 记录 + +é…置完负载å‡è¡¡å™¨åŽï¼Œä½ å°†éœ€è¦åˆ›å»º DNS 记录，以将æµé‡å‘é€åˆ°è¯¥è´Ÿè½½å‡è¡¡å™¨ã€‚ + +æ ¹æ®ä½ çš„环境，DNS 记录å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ IP çš„ A 记录，也å¯ä»¥æ˜¯æŒ‡å‘è´Ÿè½½å‡è¡¡å™¨ä¸»æœºåçš„ CNAME。无论是哪ç§æƒ…况，请确ä¿è¯¥è®°å½•æ˜¯ä½ è¦ Rancher 进行å“应的主机å。 + +在安装 Rancher 时（åŽç»­æ­¥éª¤ï¼‰ï¼Œä½ éœ€è¦æŒ‡å®šæ­¤ä¸»æœºå。请知悉，此主机å无法修改。请确ä¿ä½ è®¾ç½®çš„主机å是你想è¦çš„。 + +有关设置 DNS 记录以将域æµé‡è½¬å‘到 Amazon ELB è´Ÿè½½å‡è¡¡å™¨çš„指å—，请å‚è§ [AWS 官方文档](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md new file mode 100644 index 00000000000..d312a5f6207 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md @@ -0,0 +1,34 @@ +--- +title: 在 Amazon RDS 中创建 MySQL æ•°æ®åº“ +--- + +本教程介ç»å¦‚何在 Amazon Relational Database Service (RDS) 中创建 MySQL æ•°æ®åº“。 + +该数æ®åº“å¯ç”¨ä½œé«˜å¯ç”¨ K3s Kubernetes 集群的外部数æ®å­˜å‚¨ã€‚ + +1. 登录到 [Amazon AWS RDS 控制å°](https://console.aws.amazon.com/rds/)。确ä¿é€‰æ‹©çš„**区域**是你创建 EC2 实例 （Linux 节点）的区域。 +1. 在左侧é¢æ¿ä¸­ï¼Œç‚¹å‡»**æ•°æ®åº“**。 +1. 点击**创建数æ®åº“**。 +1. 在**引擎类型**中，点击 **MySQL**。 +1. 在**版本**中，选择 **MySQL 5.7.22**。 +1. 在**设置**部分的**凭è¯è®¾ç½®**下，输入 **admin** 主用户å的密ç ã€‚确认密ç ã€‚ +1. 展开**其它é…ç½®**。在**åˆå§‹æ•°æ®åº“å称**字段，设置数æ®åº“å称。该å称仅å¯åŒ…å«å­—æ¯ï¼Œæ•°å­—和下划线。这个å称会用于连接到数æ®åº“。 +1. 点击**创建数æ®åº“**。 + +你需è¦èŽ·å–æ–°æ•°æ®åº“的以下信æ¯ï¼Œä»¥ä¾¿ K3s Kubernetes 集群å¯ä»¥è¿žæŽ¥åˆ°è¯¥æ•°æ®åº“。 + +如需在 Amazon RDS 控制å°æŸ¥çœ‹æ­¤ä¿¡æ¯ï¼Œç‚¹å‡»**æ•°æ®åº“**，然åŽç‚¹å‡»ä½ åˆ›å»ºçš„æ•°æ®åº“çš„å称。 + +- **用户å**：使用 admin 用户å。 +- **密ç **：使用 admin 密ç ã€‚ +- **主机å**：使用**端点**作为主机å。端点å¯ä»¥åœ¨**连接性和安全性**部分找到。 +- **端å£**：默认为 3306。你å¯ä»¥åœ¨**连接性和安全性**处确认端å£ã€‚ +- **æ•°æ®åº“å称**：å‰å¾€**é…ç½®**选项å¡ç¡®è®¤æ•°æ®åº“å称。å称会在**æ•°æ®åº“å称**中列出。 + +以下方格å¼ä½¿ç”¨ä¸Šè¿°ä¿¡æ¯ï¼Œè¿žæŽ¥åˆ°æ•°æ®åº“： + +``` +mysql://username:password@tcp(hostname:3306)/database-name +``` + +有关为 K3s é…置数æ®åº“的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/installation/datastore/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md new file mode 100644 index 00000000000..b686c28221a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md @@ -0,0 +1,102 @@ +--- +title: é…ç½® NGINX è´Ÿè½½å‡è¡¡å™¨ +--- + +å°† NGINX é…置为四层负载å‡è¡¡å™¨ï¼ˆTCP），用于将连接转å‘到 Rancher 节点。 + +在此é…置中，负载å‡è¡¡å™¨ä½äºŽèŠ‚点的å‰é¢ã€‚è´Ÿè½½å‡è¡¡å™¨å¯ä»¥æ˜¯ä»»ä½•èƒ½è¿è¡Œ NGINX 的主机。 + +:::note + +ä¸è¦ä½¿ç”¨ Rancher 节点作为负载å‡è¡¡å™¨ã€‚ + +::: + +> 这些示例中，负载å‡è¡¡å™¨å°†æµé‡è½¬å‘到三个 Rancher Server 节点。如果 Rancher 安装在 RKE Kubernetes 集群上，则需è¦ä¸‰ä¸ªèŠ‚点。如果 Rancher 安装在 K3s Kubernetes 集群上，则åªéœ€è¦ä¸¤ä¸ªèŠ‚点。 + +## 安装 NGINX + +首先，在è¦ç”¨ä½œè´Ÿè½½å‡è¡¡å™¨çš„节点上安装 NGINX。NGINX 有适用于所有已知æ“作系统的软件包。已测试的版本为 `1.14` å’Œ `1.15`。如需获得安装 NGINX 的帮助，请å‚è§[安装文档](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/)。 + +`stream` 模å—是必需的，NGINX 官方安装包包å«äº†è¯¥æ¨¡å—。请å‚è§ä½ æ“作系统的文档，了解如何在æ“作系统上安装和å¯ç”¨ NGINX çš„ `stream` 模å—。 + +## 创建 NGINX é…ç½® + +安装 NGINX åŽï¼Œä½¿ç”¨èŠ‚点的 IP 地å€æ›´æ–° NGINX é…置文件 `nginx.conf`。 + +1. 将以下的示例代ç å¤åˆ¶å¹¶ç²˜è´´åˆ°ä½ ä½¿ç”¨çš„文本编辑器中。将文件ä¿å­˜ä¸º `nginx.conf`。 + +2. 在 `nginx.conf` ä¸­ï¼Œå°†æ‰€æœ‰ï¼ˆç«¯å£ 80 å’Œç«¯å£ 443）的 ``，``å’Œ `` 替æ¢ä¸ºä½ èŠ‚点的 IP 地å€ã€‚ + + :::note + + å‚è§ [NGINX 文档：TCP å’Œ UDP è´Ÿè½½å‡è¡¡](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/)了解所有é…置选项。 + + ::: + +
示例 NGINX é…ç½®
+ + ``` + worker_processes 4; + worker_rlimit_nofile 40000; + + events { + worker_connections 8192; + } + + stream { + upstream rancher_servers_http { + least_conn; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + } + server { + listen 80; + proxy_pass rancher_servers_http; + } + + } + + http { + + upstream rancher_servers_https { + least_conn; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + } + server { + listen 443 ssl; + proxy_pass rancher_servers_https; + ssl_certificate /path/to/tls.crt; + ssl_certificate_key /path/to/key.key; + location / { + proxy_pass https://rancher_servers_https; + proxy_set_header Host ; + proxy_ssl_server_name on; + proxy_ssl_name + } + } + } + ``` + + +3. å°† `nginx.conf` ä¿å­˜åˆ°ä½ çš„è´Ÿè½½å‡è¡¡å™¨çš„ `/etc/nginx/nginx.conf` 路径上。 + +4. è¿è¡Œä»¥ä¸‹å‘½ä»¤é‡æ–°åŠ è½½ NGINX é…置： + + ``` + # nginx -s reload + ``` + +## å¯é€‰ - å°† NGINX 作为 Docker 容器è¿è¡Œ + +除了将 NGINX 作为软件包安装在æ“作系统上外，你也å¯ä»¥å°†å…¶ä½œä¸º Docker 容器è¿è¡Œã€‚将编辑åŽçš„ **NGINX é…置示例** ä¿å­˜ä¸º`/etc/nginx.conf`，并è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å¯åŠ¨ NGINX 容器： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/nginx.conf:/etc/nginx/nginx.conf \ + nginx:1.14 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md new file mode 100644 index 00000000000..21294b1016e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md @@ -0,0 +1,74 @@ +--- +title: 在 Amazon EC2 中é…置节点 +--- + +在本教程中，你将学习一ç§ä¸º Rancher Mangement Server 创建 Linux 节点的方法。这些节点将满足[æ“作系统ã€Dockerã€ç¡¬ä»¶å’Œç½‘络的è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)。 + +如果 Rancher Server 安装在 RKE Kubernetes 集群上，你需è¦é…置三个实例。 + +如果 Rancher Server 安装在 K3s Kubernetes 集群上，你åªéœ€è¦é…置两个实例。 + +如果 Rancher Server 安装在å•ä¸ª Docker 容器中，你åªéœ€è¦é…置一个实例。 + +### 1. 准备工作（å¯é€‰ï¼‰ + +- **创建 IAM 角色**：è¦å…许 Rancher æ“作 AWS 资æºï¼Œä¾‹å¦‚创建新存储或新节点，你需è¦å°† Amazon é…置为云æ供商。è¦åœ¨ EC2 上设置云æ供商，你需è¦è¿›è¡Œå‡ ä¸ªæ“作，其中包括为 Rancher Server 节点设置 IAM 角色。有关设置云æ供商的详情，请å‚è§[本页](../../../pages-for-subheaders/set-up-cloud-providers.md)。 +- **创建安全组**：我们建议为 Rancher èŠ‚ç‚¹è®¾ç½®ä¸€ä¸ªç¬¦åˆ [Rancher 节点端å£è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md#端å£è¦æ±‚)的安全组。 + +### 2. é…置实例 + +1. 登录到 [Amazon AWS EC2 控制å°](https://console.aws.amazon.com/ec2/)。由于 Rancher Management Server 的所有基础设施都需è¦ä½äºŽåŒä¸€åŒºåŸŸï¼Œå› æ­¤ï¼Œè¯·åŠ¡å¿…记下创建 EC2 实例（Linux 节点）的**区域**。 +1. 在左侧é¢æ¿ä¸­ï¼Œç‚¹å‡»**实例**。 +1. 点击**å¯åŠ¨ç¤ºä¾‹**。 +1. 在**步骤 1：选择 Amazon Machine Image (AMI)** 中，使用 `ami-0d1cd67c26f5fca19 (64-bit x86)` æ¥ä½¿ç”¨ Ubuntu 18.04 作为 Linux æ“作系统。去到 Ubuntu AMI 并点击**选择**。 +1. 在**步骤 2：选择实例类型**中，选择 `t2.medium`。 +1. 点击**下一步：é…置实例详细信æ¯**。 +1. 在**实例数é‡**字段中，输入实例数é‡ã€‚创建高å¯ç”¨ K3s 集群仅需è¦ä¸¤ä¸ªå®žä¾‹ï¼Œè€Œé«˜å¯ç”¨ RKE 集群则需è¦ä¸‰ä¸ªå®žä¾‹ã€‚ +1. å¯é€‰ï¼šå¦‚果你为 Rancher 创建了一个 IAM 角色æ¥æ“作 AWS 资æºï¼Œè¯·åœ¨ **IAM 角色**字段中选择新 IAM 角色。 +1. 分别点击**下一步：添加存储**，**下一步：添加标签**å’Œ**下一步：é…置安全组**。 +1. 在**步骤 6：é…置安全组**ä¸­ï¼Œé€‰æ‹©ä¸€ä¸ªç¬¦åˆ Rancher 节点[端å£è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md#端å£è¦æ±‚)的安全组。 +1. 点击**查看并å¯åŠ¨**。 +1. 点击**å¯åŠ¨**。 +1. 选择一个新的或现有的密钥对，用于之åŽè¿žæŽ¥åˆ°ä½ çš„实例。如果使用现有密钥对，请确ä¿ä½ æœ‰è®¿é—®ç§é’¥çš„æƒé™ã€‚ +1. 点击**å¯åŠ¨å®žä¾‹**。 + + +**结果**：你已创建满足æ“作系统ã€ç¡¬ä»¶å’Œç½‘络è¦æ±‚çš„ Rancher 节点。 + +:::note + +如果节点用于 RKE Kubernetes 集群，请在下一步æ“作中为æ¯ä¸ªèŠ‚点安装 Docker 。如果节点用于 K3s Kubernetes 集群，你å¯ä»¥å¼€å§‹åœ¨èŠ‚点上安装 K3s 了。 + +::: + +### 3. 为 RKE Kubernetes 集群节点安装 Docker 并创建用户 + +1. 在 [AWS EC2 控制å°](https://console.aws.amazon.com/ec2/)中，点击左侧é¢æ¿ä¸­çš„**实例**。 +1. 转到你想è¦å®‰è£… Docker 的实例。选择实例，并点击**æ“作 > 连接**。 +1. 按照å±å¹•ä¸Šçš„说明连接到实例。å¤åˆ¶å®žä¾‹çš„公共 DNS。SSH 进入实例的示例命令如下： +``` +sudo ssh -i [path-to-private-key] ubuntu@[public-DNS-of-instance] +``` +1. 在实例上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä½¿ç”¨ Rancher 的其中一个安装脚本æ¥å®‰è£… Docker： +``` +curl https://releases.rancher.com/install-docker/18.09.sh | sh +``` +1. 连接到实例åŽï¼Œåœ¨å®žä¾‹ä¸Šè¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ›å»ºç”¨æˆ·ï¼š +``` +sudo usermod -aG docker ubuntu +``` +1. 在æ¯ä¸ªèŠ‚点上é‡å¤ä¸Šè¿°æ­¥éª¤ï¼Œä»¥ç¡®ä¿ Docker 安装到æ¯ä¸ªç”¨äºŽè¿è¡Œ Rancher Management Server 的节点上。 + +:::tip + +è¦äº†è§£æˆ‘们是å¦æ供指定的 Docker 版本的安装脚本，请访问此 [GitHub 仓库](https://github.com/rancher/install-docker)ï¼Œè¯¥ä»“åº“åŒ…å« Rancher 的所有 Docker 安装脚本。 + +::: + +**结果**：你已é…置满足æ“作系统ã€Dockerã€ç¡¬ä»¶å’Œç½‘络è¦æ±‚çš„ Rancher Server 节点。 + +### RKE Kubernetes 集群节点的åŽç»­æ­¥éª¤ + +如需在新节点上安装 RKE 集群，请记ä½æ¯ä¸ªèŠ‚点的 **IPv4 公共 IP** å’Œ **ç§æœ‰ IP**。创建节点åŽï¼Œæ­¤ä¿¡æ¯å¯ä»¥åœ¨æ¯ä¸ªèŠ‚点的**æè¿°**选项å¡ä¸­æ‰¾åˆ°ã€‚公共和ç§æœ‰ IP 将用于设置 RKE 集群é…置文件 `rancher-cluster.yml` 中æ¯ä¸ªèŠ‚点的 `address` å’Œ `internal_address`。 + +RKE 还需è¦è®¿é—®ç§é’¥æ‰èƒ½è¿žæŽ¥åˆ°æ¯ä¸ªèŠ‚点。因此，请记ä½è¿žæŽ¥åˆ°èŠ‚点的ç§é’¥çš„路径，该路径也å¯ç”¨äºŽè®¾ç½® `rancher-cluster.yml` 中æ¯ä¸ªèŠ‚点的 `ssh_key_path`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md new file mode 100644 index 00000000000..2f28274e31b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md @@ -0,0 +1,25 @@ +--- +title: 高å¯ç”¨å®‰è£… +--- + +我们建议使用 Helm（ Kubernetes 包管ç†å™¨ ）在专用的 Kubernetes 集群上安装 Rancher。由于 Rancher è¿è¡Œåœ¨å¤šä¸ªèŠ‚点上æå‡äº†å¯ç”¨æ€§ï¼Œå› æ­¤è¿™ç§å®‰è£…æ–¹å¼å«åšé«˜å¯ç”¨ Kubernetes 安装。 + +在标准安装中，先将 Kubernetes 安装在托管在云æ供商（例如 Amazon çš„ EC2 或 Google Compute Engine）中的三个节点上。 + +然åŽä½¿ç”¨ Helm 在 Kubernetes 集群上安装 Rancher。Helm 使用 Rancher çš„ Helm Chart 在 Kubernetes 集群的三个节点中å‡å®‰è£… Rancher 的副本。我们建议使用负载å‡è¡¡å™¨å°†æµé‡è½¬å‘到集群中的æ¯ä¸ª Rancher 副本中，以æ高 Rancher çš„å¯ç”¨æ€§ã€‚ + +Rancher Server çš„æ•°æ®å­˜å‚¨åœ¨ etcd 中。etcd æ•°æ®åº“å¯ä»¥åœ¨æ‰€æœ‰ä¸‰ä¸ªèŠ‚点上è¿è¡Œã€‚为了选举出大多数 etcd 节点认åŒçš„ etcd 集群 leader，节点的数é‡éœ€è¦æ˜¯å¥‡æ•°ã€‚如果 etcd æ•°æ®åº“ä¸èƒ½é€‰å‡º leader，etcd å¯èƒ½ä¼šå¤±è´¥ã€‚这时候就需è¦ä½¿ç”¨å¤‡ä»½æ¥è¿˜åŽŸé›†ç¾¤ã€‚ + +有关 Rancher 如何工作的详情（与安装方法无关），请å‚è§[架构](../../../pages-for-subheaders/rancher-manager-architecture.md)。 + +### 推è架构 + +- Rancher çš„ DNS 应该解æžä¸º 4 层负载å‡è¡¡å™¨ã€‚ +- è´Ÿè½½å‡è¡¡å™¨åº”该把 TCP/80 端å£å’Œ TCP/443 端å£çš„æµé‡è½¬å‘到 Kubernetes 集群的全部 3 个节点上。 +- Ingress Controller 会把 HTTP é‡å®šå‘到 HTTPS，在 TCP/443 端å£ç»ˆç»“ SSL/TLS。 +- Ingress Controller 会把æµé‡è½¬å‘到 Rancher deployment çš„ Pod 上的 TCP/80 端å£ã€‚ + +
使用 4 层负载å‡è¡¡å™¨åœ¨ Kubernetes 集群中安装 Rancher：Ingress Controller çš„ SSL 终止：
+ +![Rancher 的高å¯ç”¨ Kubernetes 安装](/img/ha/rancher2ha.svg) +使用 4 层负载å‡è¡¡å™¨åœ¨ Kubernetes 集群中安装 Rancher：Ingress Controller çš„ SSL 终止 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md new file mode 100644 index 00000000000..4d8ca8ac24c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md @@ -0,0 +1,136 @@ +--- +title: 为 Rancher 设置高å¯ç”¨ K3s Kubernetes 集群 +--- + +本文介ç»äº†å¦‚ä½•æ ¹æ® [Rancher Server 环境的最佳实践](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#kubernetes-安装环境)安装 Kubernetes 集群。 + +如果你的系统无法直接访问互è”网，请å‚è§ç¦»çº¿å®‰è£…说明。 + +:::tip å•èŠ‚点安装æ示： + +在å•èŠ‚点 Kubernetes 集群中，Rancher Server ä¸å…·å¤‡é«˜å¯ç”¨æ€§ï¼Œè€Œé«˜å¯ç”¨æ€§å¯¹åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­è¿è¡Œ Rancher éžå¸¸é‡è¦ã€‚但是，如果你想è¦çŸ­æœŸå†…使用å•èŠ‚点节çœèµ„æºï¼ŒåŒæ—¶åˆä¿ç•™é«˜å¯ç”¨æ€§è¿ç§»è·¯å¾„，把 Rancher 安装到å•èŠ‚点集群也是åˆé€‚的。 + +è¦é…ç½®å•èŠ‚点 K3s 集群，你åªéœ€è¦åœ¨å•ä¸ªèŠ‚点上è¿è¡Œ Rancher Server 安装命令（ä¸éœ€è¦åœ¨ä¸¤ä¸ªèŠ‚点上è¿è¡Œå‘½ä»¤ï¼‰ã€‚ + +在这两ç§å•èŠ‚点设置中，Rancher å¯ä»¥ä¸Ž Helm 一起安装在 Kubernetes 集群上，安装方法与安装到其他集群上一样。 + +::: + +## 先决æ¡ä»¶ + +以下说明å‡è®¾ä½ å·²å‚è§[此章节](../infrastructure-setup/ha-k3s-kubernetes-cluster.md)é…置好两个节点，一个负载å‡è¡¡å™¨ï¼Œä¸€ä¸ª DNS 记录和一个外部 MySQL æ•°æ®åº“。 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://rancher.com/support-maintenance-terms/)。 + +如需指定 K3s（Kubernetes）版本，在è¿è¡Œ K3s 安装脚本时使用 `INSTALL_K3S_VERSION` 环境å˜é‡ï¼ˆä¾‹å¦‚ `INSTALL_K3S_VERSION="v1.24.10+k3s1"`）。 + +## 安装 Kubernetes + +### 1. 安装 Kubernetes 并设置 K3s Server + +在è¿è¡Œå¯åŠ¨ K3s Kubernetes API Server 的命令时，你会传入使用之å‰è®¾ç½®çš„外部数æ®å­˜å‚¨çš„选项。 + +1. 连接到你准备用于è¿è¡Œ Rancher Server 的其中一个 Linux 节点。 +1. 在 Linux 节点上，è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å¯åŠ¨ K3s Server，并将其连接到外部数æ®å­˜å‚¨ã€‚ + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" + ``` + + 其中 `` 是数æ®å­˜å‚¨çš„连接 URI。例如，如果你使用的是 MySQL，则为 `mysql://username:password@tcp(hostname:3306)/database-name`。有效的数æ®å­˜å‚¨åŒ…括 etcdã€MySQLã€PostgreSQL 或 SQLite（默认）。 + + :::note + + 你也å¯ä»¥ä½¿ç”¨ `$K3S_DATASTORE_ENDPOINT` 环境å˜é‡æ¥ä¼ é€’æ•°æ®å­˜å‚¨ç«¯ç‚¹ã€‚ + + ::: + +1. 获å–主 Server 节点令牌： + ``` + cat /var/lib/rancher/k3s/server/token + ``` + +1. 在第二个 K3s Server 节点上è¿è¡Œå‘½ä»¤ï¼š + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" \ + --token "" + ``` + +### 2. 检查 K3s 是å¦æ­£å¸¸è¿è¡Œ + +在其中一个 K3s Server 节点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œæ¥ç¡®è®¤ K3s 是å¦å·²ç»è®¾ç½®æˆåŠŸï¼š +``` +sudo k3s kubectl get nodes +``` + +然åŽä½ ä¼šçœ‹åˆ°ä¸¤ä¸ªå…·æœ‰ master 角色的节点。 +``` +ubuntu@ip-172-31-60-194:~$ sudo k3s kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-172-31-60-194 Ready master 44m v1.17.2+k3s1 +ip-172-31-63-88 Ready master 6m8s v1.17.2+k3s1 +``` + +测试集群 Pod çš„å¥åº·çŠ¶å†µï¼š +``` +sudo k3s kubectl get pods --all-namespaces +``` + +**结果**：你已æˆåŠŸé…ç½® K3s Kubernetes 集群。 + +### 3. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 K3s 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/k3s/k3s.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 +2. å¤åˆ¶ `/etc/rancher/k3s/k3s.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhost。你需è¦å°† `server` é…置为负载å‡è¡¡å™¨çš„ DNSï¼Œå¹¶æŒ‡å®šç«¯å£ 6443ï¼ˆé€šè¿‡ç«¯å£ 6443 访问 Kubernetes API Serverï¼Œé€šè¿‡ç«¯å£ 80 å’Œ 443 访问 Rancher Server）。以下是一个 `k3s.yaml` 示例： + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ K3s 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [K3s 官方文档](https://rancher.com/docs/k3s/latest/en/cluster-access/) 或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### 4. 检查集群 Pod çš„å¥åº·çŠ¶å†µ + +现在你已ç»è®¾ç½®äº† `kubeconfig` 文件。你å¯ä»¥ä½¿ç”¨ `kubectl` 在本地主机访问集群。 + +检查所有需è¦çš„ Pod 和容器是å¦å¥åº·ï¼š + +``` +ubuntu@ip-172-31-60-194:~$ sudo kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system metrics-server-6d684c7b5-bw59k 1/1 Running 0 8d +kube-system local-path-provisioner-58fb86bdfd-fmkvd 1/1 Running 0 8d +kube-system coredns-d798c9dd-ljjnf 1/1 Running 0 8d +``` + +**结果**：你å¯é€šè¿‡ä½¿ç”¨ `kubectl` 访问集群，并且 K3s 集群能æˆåŠŸè¿è¡Œã€‚现在，你å¯ä»¥åœ¨é›†ç¾¤ä¸Šå®‰è£… Rancher Management Server。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md new file mode 100644 index 00000000000..397fe2aa0d0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md @@ -0,0 +1,193 @@ +--- +title: é…置高å¯ç”¨çš„ RKE Kubernetes 集群 +--- + +本文介ç»å¦‚何安装 Kubernetes 集群。该集群应专用于è¿è¡Œ Rancher Server。 + +:::note + +Rancher å¯ä»¥è¿è¡Œåœ¨ä»»ä½• Kubernetes 集群上，包括托管的 Kubernetes，例如 Amazon EKS。以下说明åªæ˜¯å®‰è£… Kubernetes 其中一ç§æ–¹å¼ã€‚ + +::: + +如果系统无法直接访问互è”网，请å‚è§[离线环境：Kubernetes 安装](../../../pages-for-subheaders/air-gapped-helm-cli-install.md)。 + +:::tip å•èŠ‚点安装æ示： + +在å•èŠ‚点 Kubernetes 集群中，Rancher Server ä¸å…·å¤‡é«˜å¯ç”¨æ€§ï¼Œè€Œé«˜å¯ç”¨æ€§å¯¹åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­è¿è¡Œ Rancher éžå¸¸é‡è¦ã€‚但是，如果你想è¦çŸ­æœŸå†…使用å•èŠ‚点节çœèµ„æºï¼ŒåŒæ—¶åˆä¿ç•™é«˜å¯ç”¨æ€§è¿ç§»è·¯å¾„，把 Rancher 安装到å•èŠ‚点集群也是åˆé€‚的。 + +è¦è®¾ç½®å•èŠ‚点 RKE 集群，在 `cluster.yml` 中é…置一个节点。该节点需具备所有三个角色，分别是`etcd`，`controlplane`å’Œ`worker`。 + +在这两ç§å•èŠ‚点设置中，Rancher å¯ä»¥ä¸Ž Helm 一起安装在 Kubernetes 集群上，安装方法与安装到其他集群上一样。 + +::: + +## 安装 Kubernetes + +### 所需的 CLI 工具 + +安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 + +安装 [RKE](https://rancher.com/docs/rke/latest/en/installation/)（Rancher Kubernetes Engine，是一个 Kubernetes å‘行版和命令行工具）。 + +### 1. 创建集群é…置文件 + +在这部分，你将创建一个å为 `rancher-cluster.yml`çš„ Kubernetes 集群é…置文件。在åŽç»­ä½¿ç”¨ RKE 命令设置集群的步骤中，此文件会用于在节点上安装 Kubernetes。 + +使用下é¢çš„示例作为指å—，创建 `rancher-cluster.yml` 文件。将 `nodes` 列表中的 IP 地å€æ›¿æ¢ä¸ºä½ åˆ›å»ºçš„ 3 个节点的 IP 地å€æˆ– DNS å称。 + +如果你的节点有公共地å€å’Œå†…部地å€ï¼Œå»ºè®®è®¾ç½® `internal_address:` 以便 Kubernetes 使用它实现集群内部通信。如果你想使用引用安全组或防ç«å¢™ï¼ŒæŸäº›æœåŠ¡ï¼ˆå¦‚ AWS EC2）è¦æ±‚设置 `internal_address:`。 + +RKE 需è¦é€šè¿‡ SSH 连接到æ¯ä¸ªèŠ‚点，它会在 `~/.ssh/id_rsa`的默认ä½ç½®æŸ¥æ‰¾ç§é’¥ã€‚如果æŸä¸ªèŠ‚点的ç§é’¥ä¸åœ¨é»˜è®¤ä½ç½®ä¸­ï¼Œä½ è¿˜éœ€è¦ä¸ºè¯¥èŠ‚点é…ç½® `ssh_key_path` 选项。 + +在选择 Kubernetes 版本时，请务必先查阅[支æŒçŸ©é˜µ](https://rancher.com/support-matrix/)，以找出已针对你的 Rancher 版本验è¯çš„最新 Kubernetes 版本。 + +```yaml +nodes: + - address: 165.227.114.63 + internal_address: 172.16.22.12 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.116.167 + internal_address: 172.16.32.37 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.127.226 + internal_address: 172.16.42.73 + user: ubuntu + role: [controlplane, worker, etcd] + +services: + etcd: + snapshot: true + creation: 6h + retention: 24h + +# Required for external TLS termination with +# ingress-nginx v0.22+ +ingress: + provider: nginx + options: + use-forwarded-headers: "true" + +kubernetes_version: v1.25.6-rancher4-1 +``` + +
通用 RKE 节点选项
+ +| 选项 | å¿…å¡« | æè¿° | +| ------------------ | -------- | -------------------------------------------------------------------------------------- | +| `address` | 是 | 公共 DNS 或 IP åœ°å€ | +| `user` | 是 | å¯ä»¥è¿è¡Œ docker 命令的用户 | +| `role` | 是 | 分é…给节点的 Kubernetes 角色列表 | +| `internal_address` | å¦ | 内部集群æµé‡çš„ç§æœ‰ DNS 或 IP åœ°å€ | +| `ssh_key_path` | å¦ | 用æ¥éªŒè¯èŠ‚点的 SSH ç§é’¥æ–‡ä»¶è·¯å¾„（默认值为 `~/.ssh/id_rsa`） | + +:::note 高级é…置： + +RKE æ供大é‡é…置选项，用于针对你的环境进行自定义安装。 + +如需了解选项和功能的完整列表，请å‚è§ [RKE 官方文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +è¦ä¸ºå¤§è§„模 Rancher 安装优化 etcd 集群，请å‚è§ [etcd 设置指å—](../../advanced-user-guides/tune-etcd-for-large-installs.md)。 + +有关 Dockershim 支æŒçš„详情，请å‚è§[此页é¢](../../../getting-started/installation-and-upgrade/installation-requirements/dockershim.md)。 + +::: + +### 2. è¿è¡Œ RKE + +``` +rke up --config ./rancher-cluster.yml +``` + +完æˆåŽï¼Œç»“æŸè¡Œåº”该是：`Finished build Kubernetes cluster succeeded`。 + +### 3. 测试集群 + +本节介ç»å¦‚何设置工作区，以便你å¯ä»¥ä½¿ç”¨ `kubectl` 命令行工具与此集群进行交互。 + +如果你已安装 `kubectl`，你需è¦å°† `kubeconfig` 文件放在 `kubectl` å¯è®¿é—®çš„ä½ç½®ã€‚`kubeconfig` 文件包å«ä½¿ç”¨ `kubectl` 访问集群所需的凭è¯ã€‚ + +你在è¿è¡Œ `rke up` 时，RKE 应该已ç»åˆ›å»ºäº†ä¸€ä¸ªå为 `kube_config_cluster.yml`çš„ `kubeconfig` 文件。该文件具有 `kubectl` å’Œ `helm`的凭è¯ã€‚ + +:::note + +如果你的文件åä¸æ˜¯ `rancher-cluster.yml`，kubeconfig 文件将命å为 `kube_config_.yml`。 + +::: + +将此文件移动到 `$HOME/.kube/config`。如果你使用多个 Kubernetes 集群，将 `KUBECONFIG` 环境å˜é‡è®¾ç½®ä¸º `kube_config_cluster.yml` 的路径: + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +用 `kubectl` 测试你的连接性，并查看你的所有节点是å¦éƒ½å¤„于 `Ready` 状æ€ï¼š + +``` +kubectl get nodes + +NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.13.5 +``` + +### 4. 检查集群 Pod çš„å¥åº·çŠ¶å†µ + +检查所有需è¦çš„ Pod 和容器是å¦å¥åº·ã€‚ + +- Pod 处于 `Running` 或 `Completed` 状æ€ã€‚ +- `READY` 表示è¿è¡Œ `STATUS` 为 `Running` çš„ Pod 的所有容器（例如， `3/3`）。 +- `STATUS` 为 `Completed` çš„ Pod 是一次è¿è¡Œçš„ Job。这些 Pod `READY` 列的值应该为 `0/1`。 + +``` +kubectl get pods --all-namespaces + +NAMESPACE NAME READY STATUS RESTARTS AGE +ingress-nginx nginx-ingress-controller-tnsn4 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-tw2ht 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-v874b 1/1 Running 0 30s +kube-system canal-jp4hz 3/3 Running 0 30s +kube-system canal-z2hg8 3/3 Running 0 30s +kube-system canal-z6kpw 3/3 Running 0 30s +kube-system kube-dns-7588d5b5f5-sf4vh 3/3 Running 0 30s +kube-system kube-dns-autoscaler-5db9bbb766-jz2k6 1/1 Running 0 30s +kube-system metrics-server-97bc649d5-4rl2q 1/1 Running 0 30s +kube-system rke-ingress-controller-deploy-job-bhzgm 0/1 Completed 0 30s +kube-system rke-kubedns-addon-deploy-job-gl7t4 0/1 Completed 0 30s +kube-system rke-metrics-addon-deploy-job-7ljkc 0/1 Completed 0 30s +kube-system rke-network-plugin-deploy-job-6pbgj 0/1 Completed 0 30s +``` + +这表示你已æˆåŠŸå®‰è£…了å¯è¿è¡Œ Rancher Server çš„ Kubernetes 集群。 + +### 5. ä¿å­˜ä½ çš„文件 + +:::note é‡è¦æ示： + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +::: + +将以下文件的副本ä¿å­˜åœ¨å®‰å…¨ä½ç½®ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_cluster.yml`：集群的 [Kubeconfig 文件](https://rancher.com/docs/rke/latest/en/kubeconfig/)。该文件包å«å¯å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚ +- `rancher-cluster.rkestate`：[Kubernetes 状æ€æ–‡ä»¶](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state)。此文件包括用于完全访问集群的凭è¯ã€‚

_Kubernetes 集群状æ€æ–‡ä»¶ä»…在 RKE 版本是 0.2.0 或更高版本时生æˆã€‚_ + +:::note + +åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +::: + +### 故障排除 + +å‚è§[故障排除](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)页é¢ã€‚ + + +### åŽç»­æ“作 +[安装 Rancher](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md new file mode 100644 index 00000000000..cc61efd50e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md @@ -0,0 +1,165 @@ +--- +title: 为 Rancher 设置高å¯ç”¨çš„ RKE2 Kubernetes 集群 +--- + +本文介ç»äº†å¦‚ä½•æ ¹æ® [Rancher Server 环境的最佳实践](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#kubernetes-安装环境)安装 Kubernetes 集群。 + +## 先决æ¡ä»¶ + +以下说明å‡è®¾ä½ å·²å‚è§[此章节](../infrastructure-setup/ha-rke2-kubernetes-cluster.md)é…置好三个节点，一个负载å‡è¡¡å™¨å’Œä¸€ä¸ª DNS 记录。 + +为了让 RKE2 与负载å‡è¡¡å™¨æ­£å¸¸å·¥ä½œï¼Œä½ éœ€è¦è®¾ç½®ä¸¤ä¸ªç›‘å¬å™¨ï¼Œä¸€ä¸ªç”¨äºŽ 9345 端å£ï¼Œå¦ä¸€ä¸ªç”¨äºŽ 6443 端å£çš„ Kubernetes API。 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§[支æŒç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。如需指定 RKE2 版本，请在è¿è¡Œ RKE2 安装脚本时，使用 `INSTALL_RKE2_VERSION` 环境å˜é‡ã€‚ + +## 安装 Kubernetes + +### 1. 安装 Kubernetes 并设置 RKE2 Server + +RKE2 Server ä½¿ç”¨åµŒå…¥å¼ etcd è¿è¡Œã€‚因此你ä¸éœ€è¦è®¾ç½®å¤–部数æ®å­˜å‚¨å°±å¯ä»¥åœ¨ HA 模å¼ä¸‹è¿è¡Œã€‚ + +在第一个节点上，使用你的预共享密文作为 Token æ¥è®¾ç½®é…置文件。Token å‚æ•°å¯ä»¥åœ¨å¯åŠ¨æ—¶è®¾ç½®ã€‚ + +如果你ä¸æŒ‡å®šé¢„共享密文，RKE2 会生æˆä¸€ä¸ªé¢„共享密文并将它放在 `/var/lib/rancher/rke2/server/node-token` 中。 + +为了é¿å…固定注册地å€çš„è¯ä¹¦é”™è¯¯ï¼Œè¯·åœ¨å¯åŠ¨ Server 时设置 `tls-san` å‚数。这个选项在 Server çš„ TLS è¯ä¹¦ä¸­å¢žåŠ ä¸€ä¸ªé¢å¤–的主机å或 IP 作为 Subject Alternative Name。如果你想通过 IP 和主机å访问，你å¯ä»¥å°†å®ƒæŒ‡å®šä¸ºä¸€ä¸ªåˆ—表。 + +首先，创建用于存放 RKE2 é…置文件的目录： + +``` +mkdir -p /etc/rancher/rke2/ +``` + +然åŽï¼Œå‚è§ä»¥ä¸‹ç¤ºä¾‹åœ¨ `/etc/rancher/rke2/config.yaml` 中创建 RKE2 é…置文件： + +``` +token: my-shared-secret +tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +``` +之åŽï¼Œè¿è¡Œå®‰è£…命令并å¯ç”¨å’Œå¯åŠ¨ RKE2： + +``` +curl -sfL https://get.rke2.io | sh - +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` +1. è¦åŠ å…¥å…¶ä½™çš„节点，使用åŒä¸€ä¸ªå…±äº«æˆ–自动生æˆçš„ Token æ¥é…ç½®æ¯ä¸ªé¢å¤–的节点。以下是é…置文件的示例： + + token: my-shared-secret + server: https://:9345 + tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +è¿è¡Œå®‰è£…程åºï¼Œç„¶åŽå¯ç”¨å¹¶å¯åŠ¨ RKE2： + + curl -sfL https://get.rke2.io | sh - + systemctl enable rke2-server.service + systemctl start rke2-server.service + + +1. 在第三 RKE2 Server 节点上è¿è¡ŒåŒæ ·çš„命令。 + +### 2. 检查 RKE2 是å¦æ­£å¸¸è¿è¡Œ + +在所有 Server 节点上å¯åŠ¨äº† RKE2 Server 进程åŽï¼Œç¡®ä¿é›†ç¾¤å·²ç»æ­£å¸¸å¯åŠ¨ï¼Œè¯·è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get nodes +现在，Server 节点的状æ€åº”该是 Ready。 +``` + +测试集群 Pod çš„å¥åº·çŠ¶å†µï¼š +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get pods --all-namespaces +``` + +**结果**：你已æˆåŠŸé…ç½® RKE2 Kubernetes 集群。 + +### 3. ä¿å­˜å¹¶å¼€å§‹ä½¿ç”¨ kubeconfig 文件 + +在æ¯ä¸ª Rancher Server 节点安装 RKE2 时，会在æ¯ä¸ªèŠ‚点的 `/etc/rancher/rke2/rke2.yaml` 中生æˆä¸€ä¸ª `kubeconfig` 文件。该文件包å«è®¿é—®é›†ç¾¤çš„凭è¯ã€‚请将该文件ä¿å­˜åœ¨å®‰å…¨çš„ä½ç½®ã€‚ + +如è¦ä½¿ç”¨è¯¥ `kubeconfig` 文件： + +1. 安装 Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl)。 +2. å¤åˆ¶ `/etc/rancher/rke2/rke2.yaml` 文件并ä¿å­˜åˆ°æœ¬åœ°ä¸»æœºçš„ `~/.kube/config` 目录上。 +3. 在 kubeconfig 文件中，`server` çš„å‚数为 localhostã€‚åœ¨ç«¯å£ 6443 上将æœåŠ¡å™¨é…置为 controlplane è´Ÿè½½å‡è¡¡å™¨çš„ DNS（RKE2 Kubernetes API Server ä½¿ç”¨ç«¯å£ 6443，而 Rancher Server 将通过 NGINX Ingress åœ¨ç«¯å£ 80 å’Œ 443 上æä¾›æœåŠ¡ã€‚）以下是一个示例 `rke2.yaml`： + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # 编辑此行 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**结果**：你å¯ä»¥å¼€å§‹ä½¿ç”¨ `kubectl` æ¥ç®¡ç†ä½ çš„ RKE2 集群。如果你有多个 `kubeconfig` 文件，在使用 `kubectl` 时，你å¯ä»¥ä¼ å…¥æ–‡ä»¶è·¯å¾„æ¥æŒ‡å®šè¦ä½¿ç”¨çš„ `kubeconfig` 文件： + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +有关 `kubeconfig` 文件的详情，请å‚è§ [RKE2 官方文档](https://docs.rke2.io/cluster_access)或 [ Kubernetes 官方文档](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/)中关于使用 `kubeconfig` 文件管ç†é›†ç¾¤è®¿é—®çš„部分。 + +### 4. 检查集群 Pod çš„å¥åº·çŠ¶å†µ + +现在你已ç»è®¾ç½®äº† `kubeconfig` 文件。你å¯ä»¥ä½¿ç”¨ `kubectl` 在本地主机访问集群。 + +检查所有需è¦çš„ Pod 和容器是å¦å¥åº·ï¼š + +``` +/var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml get pods -A +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system cloud-controller-manager-rke2-server-1 1/1 Running 0 2m28s +kube-system cloud-controller-manager-rke2-server-2 1/1 Running 0 61s +kube-system cloud-controller-manager-rke2-server-3 1/1 Running 0 49s +kube-system etcd-rke2-server-1 1/1 Running 0 2m13s +kube-system etcd-rke2-server-2 1/1 Running 0 87s +kube-system etcd-rke2-server-3 1/1 Running 0 56s +kube-system helm-install-rke2-canal-hs6sx 0/1 Completed 0 2m17s +kube-system helm-install-rke2-coredns-xmzm8 0/1 Completed 0 2m17s +kube-system helm-install-rke2-ingress-nginx-flwnl 0/1 Completed 0 2m17s +kube-system helm-install-rke2-metrics-server-7sggn 0/1 Completed 0 2m17s +kube-system kube-apiserver-rke2-server-1 1/1 Running 0 116s +kube-system kube-apiserver-rke2-server-2 1/1 Running 0 66s +kube-system kube-apiserver-rke2-server-3 1/1 Running 0 48s +kube-system kube-controller-manager-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-controller-manager-rke2-server-2 1/1 Running 0 57s +kube-system kube-controller-manager-rke2-server-3 1/1 Running 0 42s +kube-system kube-proxy-rke2-server-1 1/1 Running 0 2m25s +kube-system kube-proxy-rke2-server-2 1/1 Running 0 59s +kube-system kube-proxy-rke2-server-3 1/1 Running 0 85s +kube-system kube-scheduler-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-scheduler-rke2-server-2 1/1 Running 0 57s +kube-system kube-scheduler-rke2-server-3 1/1 Running 0 42s +kube-system rke2-canal-b9lvm 2/2 Running 0 91s +kube-system rke2-canal-khwp2 2/2 Running 0 2m5s +kube-system rke2-canal-swfmq 2/2 Running 0 105s +kube-system rke2-coredns-rke2-coredns-547d5499cb-6tvwb 1/1 Running 0 92s +kube-system rke2-coredns-rke2-coredns-547d5499cb-rdttj 1/1 Running 0 2m8s +kube-system rke2-coredns-rke2-coredns-autoscaler-65c9bb465d-85sq5 1/1 Running 0 2m8s +kube-system rke2-ingress-nginx-controller-69qxc 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-7hprp 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-x658h 1/1 Running 0 52s +kube-system rke2-metrics-server-6564db4569-vdfkn 1/1 Running 0 66s +``` + +**结果**：你å¯é€šè¿‡ä½¿ç”¨ `kubectl` 访问集群，并且 RKE2 集群能æˆåŠŸè¿è¡Œã€‚现在，你å¯ä»¥åœ¨é›†ç¾¤ä¸Šå®‰è£… Rancher Management Server。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md new file mode 100644 index 00000000000..a0ec43f76c5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md @@ -0,0 +1,73 @@ +--- +title: 推è的集群架构 +--- + +有三个角色å¯ä»¥åˆ†é…给节点，分别是 `etcd`ã€`controlplane` å’Œ `worker`。 + +## å°† Worker 节点与具有其他角色的节点分开 + +在设计集群时，你有两ç§é€‰æ‹©ï¼š + +* 为æ¯ä¸ªè§’色使用专用节点。这确ä¿äº†ç‰¹å®šè§’色所需组件的资æºå¯ç”¨æ€§ã€‚它还根æ®[端å£è¦æ±‚](../node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)严格隔离æ¯ä¸ªè§’色之间的网络æµé‡ã€‚ +* å°† `etcd` å’Œ `controlplane` 角色分é…给相åŒçš„节点。该节点必须满足这两个角色的硬件è¦æ±‚。 + +无论在哪ç§æƒ…况下，都ä¸åº”该在具有 `etcd` 或 `controlplane` 角色的节点中使用或添加 `worker` 角色。 + +因此，æ¯ä¸ªèŠ‚点的角色都有如下几ç§é…置选择： + +* `etcd` +* `controlplane` +* `etcd` å’Œ `controlplane` +* `worker` + +## æ¯ä¸ªè§’色的推è节点数 + +集群应该有： + +- 至少拥有三个角色为 `etcd` 的节点，æ¥ç¡®ä¿å¤±åŽ»ä¸€ä¸ªèŠ‚点时ä»èƒ½å­˜æ´»ã€‚增加 etcd 节点数é‡èƒ½æ高容错率，而将 etcd 分散到ä¸åŒå¯ç”¨åŒºç”šè‡³èƒ½èŽ·å–更好的容错能力。 +- 至少两个节点具有 `controlplane` 角色，以实现主组件高å¯ç”¨æ€§ã€‚ +- 至少两个具有 `worker` 角色的节点，用于在节点故障时é‡æ–°å®‰æŽ’工作负载。 + +有关æ¯ä¸ªè§’色的用途的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 中的节点角色](roles-for-nodes-in-kubernetes.md)。 + + +### controlplane 节点数 + +添加多个具有 `controlplane` 角色的节点，使æ¯ä¸ªä¸»ç»„件都具有高å¯ç”¨æ€§ã€‚ + +### etcd 节点数 + +在ä¿æŒé›†ç¾¤å¯ç”¨æ€§çš„åŒæ—¶ï¼Œå¯ä»¥ä¸€æ¬¡ä¸¢å¤±çš„节点数由分é…了 `etcd` 角色的节点数决定。对于具有 n 个æˆå‘˜çš„集群，最å°å€¼ä¸º (n/2)+1。因此，我们建议在一个区域内的 3 个ä¸åŒå¯ç”¨åŒºä¸­å„创建一个 `etcd` 节点，以在一个å¯ç”¨åŒºä¸¢å¤±çš„情况下存活。如果你åªä½¿ç”¨ä¸¤ä¸ªåŒºåŸŸï¼Œé‚£ä¹ˆåœ¨â€œå¤šæ•°èŠ‚点â€æ‰€åœ¨çš„å¯ç”¨åŒºä¸å¯ç”¨æ—¶ï¼Œä½ å°†ä¼šä¸¢å¤± etcd 集群。 + +| 具有 `etcd` 角色的节点 | 多数节点 | 容错能力 | +|--------------|------------|-------------------| +| 1 | 1 | 0 | +| 2 | 2 | 0 | +| 3 | 2 | **1** | +| 4 | 3 | 1 | +| 5 | 3 | **2** | +| 6 | 4 | 2 | +| 7 | 4 | **3** | +| 8 | 5 | 3 | +| 9 | 5 | **4** | + +å‚考： + +* [最佳 etcd 集群大å°çš„官方 etcd 文档](https://etcd.io/docs/v3.4.0/faq/#what-is-failure-tolerance) +* [为 Kubernetes æ“作 etcd 集群的官方 Kubernetes 文档](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) + +### Worker 节点数 + +添加多个具有 `worker` 角色的节点能确ä¿ä¸€ä¸ªèŠ‚点出现故障时å¯ä»¥é‡æ–°å®‰æŽ’工作负载。 + +### 为什么 Rancher 集群和è¿è¡Œåº”用的集群的生产è¦æ±‚ä¸åŒ + +ä½ å¯èƒ½å·²ç»æ³¨æ„到我们的 [Kubernetes 安装](../../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md)说明并ä¸ç¬¦åˆæˆ‘们对生产就绪集群的è¦æ±‚，这是因为 `worker` 角色没有专用节点。然而，你 Rancher 中的这个三节点集群是有效的，因为： + +* 它å…许一个 `etcd` 节点故障。 +* 它通过多个 `controlplane` 节点æ¥ç»´æŠ¤ master 组件的多个实例。 +* 此集群上没有创建除 Rancher 之外的其他工作负载。 + +## å‚考 + +* [Kubernetes：主组件](https://kubernetes.io/docs/concepts/overview/components/#master-components) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md new file mode 100644 index 00000000000..1f929eb52f9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md @@ -0,0 +1,50 @@ +--- +title: Kubernetes 中节点的角色 +--- + +æœ¬èŠ‚ä»‹ç» Kubernetes 中 etcd 节点ã€controlplane 节点和 worker 节点的角色，以åŠè¿™äº›è§’色如何在集群中ååŒå·¥ä½œã€‚ + +此图适用于 [Rancher 通过 RKE 部署的 Kubernetes 集群](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)： + +![集群图](/img/clusterdiagram.svg)
+线æ¡è¡¨ç¤ºç»„件之间的通信。而颜色纯粹用于视觉辅助。 + +## etcd + +具有 `etcd` 角色的节点è¿è¡Œ etcd，这是一个一致且高å¯ç”¨çš„键值存储，用作 Kubernetes 所有集群数æ®çš„åŽå¤‡å­˜å‚¨ã€‚etcd 将数æ®å¤åˆ¶åˆ°æ¯ä¸ªèŠ‚点。 + +:::note + +具有 `etcd` 角色的节点在 UI 中显示为`ä¸å¯è°ƒåº¦`，å³é»˜è®¤æƒ…况下ä¸ä¼šå°† Pod 调度到这些节点。 + +::: + +## controlplane + +具有 `controlplane` 角色的节点è¿è¡Œ Kubernetes 主组件（ä¸åŒ…括 `etcd`，因为它是一个å•ç‹¬çš„角色）。有关组件的详细列表，请å‚阅 [Kubernetes：主组件](https://kubernetes.io/docs/concepts/overview/components/#master-components)。 + +:::note + +具有 `controlplane` 角色的节点在 UI 中显示为`ä¸å¯è°ƒåº¦`，å³é»˜è®¤æƒ…况下ä¸ä¼šå°† Pod 调度到这些节点。 + +::: + +### kube-apiserver + +Kubernetes API Server (`kube-apiserver`) 能水平扩展。如果节点具有需è¦è®¿é—® Kubernetes API Server 的组件，则æ¯ä¸ªå…·æœ‰ `controlplane` 角色的节点都将被添加到节点上的 NGINX 代ç†ä¸­ã€‚è¿™æ„味ç€å¦‚果一个节点å˜å¾—ä¸å¯è®¿é—®ï¼Œè¯¥èŠ‚点上的本地 NGINX 代ç†ä¼šå°†è¯·æ±‚转å‘到列表中的å¦ä¸€ä¸ª Kubernetes API Server。 + +### kube-controller-manager + +Kubernetes Controller Manager 使用 Kubernetes 中的端点进行 Leader 选举。`kube-controller-manager` 的一个实例将在 Kubernetes 端点中创建一个æ¡ç›®ï¼Œå¹¶åœ¨é…置的时间间隔内更新该æ¡ç›®ã€‚其他实例将看到一个状æ€ä¸º Active çš„ Leader，并等待该æ¡ç›®è¿‡æœŸï¼ˆä¾‹å¦‚节点无å“应）。 + +### kube-scheduler + +Kubernetes 调度器使用 Kubernetes 中的端点进行 Leader 选举。`kube-scheduler` 的一个实例将在 Kubernetes 端点中创建一个æ¡ç›®ï¼Œå¹¶åœ¨é…置的时间间隔内更新该æ¡ç›®ã€‚其他实例将看到一个状æ€ä¸º Active çš„ Leader，并等待该æ¡ç›®è¿‡æœŸï¼ˆä¾‹å¦‚节点无å“应）。 + +## worker + +具有 `worker` 角色的节点è¿è¡Œ Kubernetes 节点组件。有关组件的详细列表，请å‚阅 [Kubernetes：节点组件](https://kubernetes.io/docs/concepts/overview/components/#node-components)。 + +## å‚考 + +* [Kubernetes：节点组件](https://kubernetes.io/docs/concepts/overview/components/#node-components) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md new file mode 100644 index 00000000000..85bcf4b7b11 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md @@ -0,0 +1,129 @@ +--- +title: Rancher 管ç†é›†ç¾¤çš„节点è¦æ±‚ +--- + +本页æ述了 Rancher 管ç†çš„ Kubernetes 集群的è¦æ±‚，你的应用和æœåŠ¡å°†å®‰è£…在这些集群中。这些下游集群应该与è¿è¡Œ Rancher 的三节点集群分开。 + +:::note + +如果 Rancher 安装在高å¯ç”¨çš„ Kubernetes 集群上，Rancher Server 的三节点集群和下游集群有ä¸åŒçš„è¦æ±‚。有关 Rancher 的安装è¦æ±‚，请å‚考[安装文档](../../../pages-for-subheaders/installation-requirements.md)中的节点è¦æ±‚。 + +::: + +## æ“作系统和容器è¿è¡Œæ—¶è¦æ±‚ + +Rancher 兼容当å‰æ‰€æœ‰çš„ä¸»æµ Linux å‘行版和任何通用的 Docker 版本。所有下游集群的 etcd å’Œ controlplane 节点都需è¦è¿è¡Œåœ¨ Linux 上。而 Worker 节点å¯ä»¥è¿è¡Œåœ¨ Linux 或 [Windows Server](#windows-节点) 上。 + +如需了解å„个 Rancher 版本通过了哪些æ“作系统和 Docker 版本测试，请å‚è§[支æŒå’Œç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。 + +所有支æŒçš„æ“作系统都使用 64-bit x86 架构。 + +如果你想使用 ARM64，请å‚阅[在 ARM64 上è¿è¡Œï¼ˆå®žéªŒåŠŸèƒ½ï¼‰ã€‚](../../advanced-user-guides/enable-experimental-features/rancher-on-arm64.md) + +有关如何安装 Docker çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Docker 官方文档](https://docs.docker.com/)。 + +### Oracle Linux å’Œ RHEL è¡ç”Ÿçš„ Linux 节点 + +æŸäº›æºè‡ª RHEL çš„ Linux å‘行版（包括 Oracle Linux）的默认防ç«å¢™è§„则å¯èƒ½ä¼šé˜»æ­¢ä¸Ž Helm 的通信。我们建议ç¦ç”¨ firewalld。如果你的 Kubernetes 版本是 1.19，请务必ç¦ç”¨ firewalld。 + +:::note + +在 RHEL 8.4 中，NetworkManager 上有两个é¢å¤–çš„æœåŠ¡ï¼Œå³ `nm-cloud-setup.service` å’Œ `nm-cloud-setup.timer`。这些æœåŠ¡å¢žåŠ äº†ä¸€ä¸ªè·¯ç”±è¡¨ï¼Œå¹²æ‰°äº† CNI æ’件的é…置。如果å¯ç”¨äº†è¿™äº›æœåŠ¡ï¼Œä½ å¿…须使用以下命令ç¦ç”¨å®ƒä»¬ï¼Œç„¶åŽé‡æ–°å¯åŠ¨èŠ‚点以æ¢å¤è¿žæŽ¥ï¼š + +``` +systemctl disable nm-cloud-setup.service nm-cloud-setup.timer +reboot +``` + +::: + +### SUSE Linux 节点 + +SUSE Linux å¯èƒ½æœ‰ä¸€ä¸ªé˜²ç«å¢™ï¼Œé»˜è®¤æƒ…况下会阻止所有端å£ã€‚在这ç§æƒ…况下，请按照[步骤](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#打开-suse-linux-端å£)打开将主机添加到自定义集群所需的端å£ã€‚ + +### Flatcar Container Linux 节点 + +使用 Flatcar Container Linux 节点[通过 Rancher å¯åŠ¨ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 时，需è¦åœ¨ [Cluster Config 文件](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)中使用如下é…置： + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: canal + options: + canal_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: calico + options: + calico_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +还需è¦å¯ç”¨ Docker æœåŠ¡ï¼Œä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å¯ç”¨ Docker æœåŠ¡ï¼š + +``` +systemctl enable docker.service +``` + +使用[主机驱动](../../../pages-for-subheaders/about-provisioning-drivers.md#主机驱动)时会自动å¯ç”¨ Docker æœåŠ¡ã€‚ + +### Windows 节点 + +è¿è¡Œ Windows Server 节点必须使用 Docker ä¼ä¸šç‰ˆã€‚ + +Windows 节点åªèƒ½ç”¨äºŽ Worker 节点。请å‚阅[é…ç½® Windows 自定义集群](../../../pages-for-subheaders/use-windows-clusters.md)。 + +## 硬件è¦æ±‚ + +你的工作负载决定了具有 `worker` 角色的节点的硬件è¦æ±‚。è¿è¡Œ Kubernetes 节点组件的最低è¦æ±‚是 1 个 CPU（核心）和 1GB 内存。 + +关于 CPU 和内存，建议将 Kubernetes 集群的ä¸åŒå¹³é¢ï¼ˆetcdã€controlplane å’Œ worker）托管在ä¸åŒçš„节点上，以便它们å¯ä»¥ç›¸äº’独立扩展。 + +有关大型 Kubernetes 集群的硬件建议，请å‚阅[构建大型集群](https://kubernetes.io/docs/setup/best-practices/cluster-large/)的官方 Kubernetes 文档。 + +有关生产环境中 etcd 集群的硬件建议，请å‚阅官方 [etcd 文档](https://etcd.io/docs/v3.4.0/op-guide/hardware/)。 + +## 网络è¦æ±‚ + +对于生产集群，我们建议你通过仅打开以下端å£è¦æ±‚中定义的端å£æ¥é™åˆ¶æµé‡ã€‚ + +需è¦å¼€æ”¾çš„端å£æ ¹æ®ä¸‹æ¸¸é›†ç¾¤çš„å¯åŠ¨æ–¹å¼è€Œæœ‰æ‰€ä¸åŒã€‚以下列出了需è¦ä¸ºä¸åŒ[集群创建选项](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)打开的端å£ã€‚ + +有关 Kubernetes 集群中 etcd 节点ã€controlplane 节点和 Worker 节点的端å£è¦æ±‚的详细信æ¯ï¼Œè¯·å‚阅 [Rancher Kubernetes Engine 的端å£è¦æ±‚](https://rancher.com/docs/rke/latest/en/os/#ports)。 + +在[下游集群端å£è¦æ±‚](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#下游-kubernetes-集群节点)中，你å¯ä»¥æ‰¾åˆ°åœ¨å„ç§æƒ…况下使用的端å£çš„详细信æ¯ã€‚ + +:::caution + +ä¸è¦ä½¿ç”¨ä¸ŽçŽ°æœ‰èŠ‚点相åŒçš„主机å或 IP 地å€æ¥æ³¨å†ŒèŠ‚点。这样åšä¼šå¯¼è‡´ RKE 阻止节点加入，并导致é…置挂起。主机驱动和自定义集群都å¯èƒ½å‘生这ç§æƒ…况。如果节点必须é‡ç”¨çŽ°æœ‰èŠ‚点的主机å或 IP，则必须在注册节点之å‰è®¾ç½® `hostname_override` [RKE 选项](https://rke.docs.rancher.com/config-options/nodes#overriding-the-hostname)，以便节点能够正确加入。 + +::: + +## å¯é€‰ï¼šå®‰å…¨æ³¨æ„事项 + +如果你è¦é…ç½®ç¬¦åˆ CIS（互è”网安全中心）Kubernetes 基准的 Kubernetes 集群，我们建议你在安装 Kubernetes 之å‰æŒ‰ç…§æˆ‘们的强化指å—æ¥é…置节点。 + +有关强化指å—的更多信æ¯ï¼Œä»¥åŠäº†è§£å“ªä¸ªæŒ‡å—版本对应于你的 Rancher å’Œ Kubernetes 版本，请å‚阅[安全](../../../pages-for-subheaders/rancher-security.md#rancher-强化指å—)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md new file mode 100644 index 00000000000..e3857d23024 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -0,0 +1,282 @@ +--- +title: 注册现有集群 +--- + +集群注册功能å–代了导入集群的功能。 + +Rancher 管ç†æ³¨å†Œé›†ç¾¤çš„范围å–决于集群的类型。详情请å‚è§[对注册集群的管ç†èƒ½åŠ›](#对注册集群的管ç†èƒ½åŠ›)。 + + +## 先决æ¡ä»¶ + +### Kubernetes 节点角色 + +已注册的 RKE Kubernetes 集群必须具有所有三个节点角色，分别是 etcdã€controlplane å’Œ worker。åªæœ‰ controlplane 组件的集群无法在 Rancher 中注册。 + +有关 RKE 节点角色的更多信æ¯ï¼Œè¯·å‚阅[最佳实践](../../../pages-for-subheaders/checklist-for-production-ready-clusters.md#集群架构)。 + +### æƒé™ + +è¦åœ¨ Rancher 中注册集群，你必须在该集群中拥有 `cluster-admin` æƒé™ã€‚如果没有，请è¿è¡Œä»¥ä¸‹å‘½ä»¤å°†æƒé™æŽˆäºˆç”¨æˆ·ï¼š + +```plain +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole cluster-admin \ + --user [USER_ACCOUNT] +``` + +由于默认情况下 Google Kubernetes Engine (GKE) ä¸æŽˆäºˆ `cluster-admin` 角色，因此你必须先在 GKE 集群上è¿è¡Œå‘½ä»¤ï¼Œç„¶åŽæ‰èƒ½æ³¨å†Œã€‚è¦è¯¦ç»†äº†è§£ GKE RBAC，请å‚阅 [Google 官方文档](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control)。 + +### EKSã€AKS å’Œ GKE 集群 + +è¦æˆåŠŸå°†è¿™äº›é›†ç¾¤å¯¼å…¥ Rancher 或从 Rancher 中é…置它们，Elastic Kubernetes Service (EKS)ã€Azure Kubernetes Service (AKS) å’Œ Google Kubernetes Engine (GKE) 集群必须至少有一个托管节点组。此外，ä¸æ”¯æŒ GKE Autopilot 集群。有关 GKE 模å¼ä¹‹é—´å·®å¼‚，请å‚阅[比较 GKE Autopilot å’Œ Standard](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison)。 + +## 注册集群 + +1. 点击 **☰ > 集群管ç†**。 +2. 在**集群**页é¢ä¸Šï¼Œå•å‡»**导入集群**。 +3. 选择集群类型。 +4. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +5. 如果你在 Rancher 中导入一个通用 Kubernetes 集群，请执行以下步骤：
+ a. 点击**集群选项**下的 **Agent 环境å˜é‡**，为 [rancher cluster agent](../launch-kubernetes-with-rancher/about-rancher-agents.md) 设置环境å˜é‡ã€‚ä½ å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚如果 Rancher Agent 需è¦ä½¿ç”¨ä»£ç†ä¸Ž Rancher Server 通信，则å¯ä»¥ä½¿ç”¨ Agent 环境å˜é‡è®¾ç½® `HTTP_PROXY`，`HTTPS_PROXY` å’Œ `NO_PROXY` 环境å˜é‡ã€‚
+ b. å¯ç”¨é¡¹ç›®ç½‘络隔离，确ä¿é›†ç¾¤æ”¯æŒ Kubernetes `NetworkPolicy` 资æºã€‚用户å¯ä»¥é€šè¿‡**项目网络隔离**选项下的**高级选项**下拉èœå•è¿›è¡Œæ“作。 +6. å•å‡»**创建**。 +7. 此处会显示 `cluster-admin` æƒé™çš„先决æ¡ä»¶ï¼ˆå‚è§ä¸Šæ–‡çš„**先决æ¡ä»¶**），其中包括满足先决æ¡ä»¶çš„示例命令。 +8. å°† `kubectl` 命令å¤åˆ¶åˆ°å‰ªè´´æ¿ï¼Œå¹¶åœ¨é…置了 kubeconfig 的节点上è¿è¡Œè¯¥å‘½ä»¤ï¼Œä»Žè€ŒæŒ‡å‘è¦å¯¼å…¥çš„集群。如果你ä¸ç¡®å®šé…置是å¦æ­£ç¡®ï¼Œè¯·åœ¨è¿è¡Œ Rancher 显示的命令之å‰è¿è¡Œ `kubectl get nodes` 进行验è¯ã€‚ +9. 如果你使用自签åè¯ä¹¦ï¼Œä½ å°†æ”¶åˆ° `certificate signed by unknown authority` 的消æ¯ã€‚è¦è§£å†³æ­¤éªŒè¯é—®é¢˜ï¼Œè¯·å°† Rancher 中显示的以 `curl` 开头的命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚然åŽåœ¨é…置了 kubeconfig 的节点上è¿è¡Œè¯¥å‘½ä»¤ï¼Œä»Žè€ŒæŒ‡å‘è¦å¯¼å…¥çš„集群。 +10. 在节点上è¿è¡Œå®Œå‘½ä»¤åŽï¼Œå•å‡»**完æˆ**。 + +**结果**： + +- 集群已注册并分é…了 **Pending** 状æ€ã€‚Rancher 正在部署资æºæ¥ç®¡ç†ä½ çš„集群。 +- 当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ +- **Active** 集群分é…了两个项目，分别是 `Default`（包å«å‘½å空间 `default`）和 `System`（包å«å‘½å空间 `cattle-system`ã€`ingress-nginx`ã€`kube-public` å’Œ `kube-system`）。 + + +:::note + +无法é‡æ–°æ³¨å†Œå½“å‰åœ¨ Rancher 中处于 Active 状æ€çš„集群。 + +::: + +### 使用 Terraform é…置导入的 EKSã€AKS 或 GKE 集群 + +ä½ **ä»…**需è¦å®šä¹‰ Rancher 使用 Terraform 导入 EKSã€AKS 或 GKE 集群所需的最少字段。请谨记这点，因为 Rancher 会使用用户æ供的任何é…置覆盖集群的é…置。 + +:::caution + +å³ä½¿å½“å‰é›†ç¾¤çš„é…置与用户æ供的é…置之间åªå­˜åœ¨å¾®å°å·®å¼‚，但是微å°çš„差异也有å¯èƒ½äº§ç”Ÿå¾ˆå¤§çš„å½±å“。 + +::: + +Rancher 通过 `eks_config_v2` 使用 Terraform 导入 EKS 集群所需的最少é…置字段如下： + +- cloud_credential_id +- name +- region +- imported （对于导入的集群，此字段应始终设置为 `true`） + +导入的 EKS 集群的示例 YAML é…置： + +``` +resource "rancher2_cluster" "my-eks-to-import" { + name = "my-eks-to-import" + description = "Terraform EKS Cluster" + eks_config_v2 { + cloud_credential_id = rancher2_cloud_credential.aws.id + name = var.aws_eks_name + region = var.aws_region + imported = true + } +} +``` + +ä½ å¯ä»¥åœ¨ [Rancher2 Terraform Provider 文档](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)中找到其他云æ供商的更多示例。 + +## 对注册集群的管ç†èƒ½åŠ› + +Rancher 管ç†æ³¨å†Œé›†ç¾¤çš„范围å–决于集群的类型。 + +- [所有已注册集群的功能](#所有已注册集群的功能) +- [已注册 RKE2 å’Œ K3s 集群的附加功能](#已注册-rke2-å’Œ-k3s-集群的附加功能) +- [已注册 EKS, AKS å’Œ GKE 集群的附加功能](#已注册-eksaks-å’Œ-gke-集群的附加功能) + +### 所有已注册集群的功能 + +注册集群åŽï¼Œé›†ç¾¤æ‰€æœ‰è€…å¯ä»¥ï¼š + +- 通过 RBAC [管ç†é›†ç¾¤è®¿é—®](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) +- å¯ç”¨[Monitoringã€å‘Šè­¦å’Œ Notifiers](../../../pages-for-subheaders/monitoring-and-alerting.md) +- å¯ç”¨ [Logging](../../../pages-for-subheaders/logging.md) +- å¯ç”¨ [Istio](../../../pages-for-subheaders/istio.md) +- 管ç†é¡¹ç›®å’Œå·¥ä½œè´Ÿè½½ + +### 已注册 RKE2 å’Œ K3s 集群的附加功能 + +[K3s](https://rancher.com/docs/k3s/latest/en/) 是用于边缘安装的轻é‡çº§ã€å®Œå…¨å…¼å®¹çš„ Kubernetes å‘行版。 +[RKE2](https://docs.rke2.io) 是 Rancher 用于数æ®ä¸­å¿ƒå’Œäº‘安装的下一代 Kubernetes å‘行版。 + +RKE2 或 K3s 集群注册到 Rancher åŽï¼ŒRancher 会识别它。Rancher UI 将开放[所有已注册集群](#所有已注册集群的功能)的功能，以åŠä»¥ä¸‹ç”¨äºŽç¼–辑和å‡çº§é›†ç¾¤çš„功能： + +- [å‡çº§ Kubernetes 版本](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)的能力 + :::danger + + 将集群导入 Rancher åŽï¼Œä½ éœ€è¦ä½¿ç”¨ Rancher 执行å‡çº§ã€‚**ä¸**支æŒåœ¨ Rancher 之外å‡çº§å¯¼å…¥çš„集群。 + + ::: +- é…置能åŒæ—¶å‡çº§çš„最大节点数 +- 查看集群的é…ç½®å‚数和用于å¯åŠ¨é›†ç¾¤ä¸­æ¯ä¸ªèŠ‚点的环境å˜é‡çš„åªè¯»ç‰ˆæœ¬ + +### 已注册 EKS，AKS å’Œ GKE 集群的附加功能 + +Rancher 处ç†æ³¨å†Œçš„ EKSã€AKS 或 GKE 集群的方å¼ä¸Žå¤„ç†åœ¨ Rancher 中创建的集群的方å¼ç±»ä¼¼ã€‚但是，如果你通过 Rancher UI 删除已注册的集群，Rancher ä¸ä¼šé”€æ¯è¿™äº›é›†ç¾¤ã€‚ + +如果你在 Rancher 中创建 EKSã€AKS 或 GKE 集群，然åŽå°†å…¶åˆ é™¤ï¼ŒRancher 会销æ¯è¯¥é›†ç¾¤ã€‚通过 Rancher 删除已注册的集群时，Rancher Server 会_æ–­å¼€_与集群的连接。该集群ä»ç„¶å­˜åœ¨ï¼Œåªæ˜¯å®ƒä¸å†åœ¨ Rancher 中。你ä»ç„¶å¯ä»¥åƒæ³¨å†Œå‰ä¸€æ ·è®¿é—®å·²æ³¨é”€çš„集群。 + +有关å¯ç”¨äºŽç®¡ç†å·²æ³¨å†Œé›†ç¾¤çš„功能，请å‚阅[按集群类型划分的集群管ç†åŠŸèƒ½](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)。 + +## é…ç½® RKE2 å’Œ K3s 集群å‡çº§ + +:::tip + +Kubernetes 的最佳实践是在å‡çº§ä¹‹å‰å¤‡ä»½é›†ç¾¤ã€‚使用外部数æ®åº“å‡çº§é«˜å¯ç”¨ K3s 集群时，请使用关系数æ®åº“æ供商推èçš„æ–¹å¼å¤‡ä»½æ•°æ®åº“。 + +::: + +**并å‘**是å‡çº§æœŸé—´å…许ä¸å¯ç”¨çš„最大节点数。如果ä¸å¯ç”¨èŠ‚点的数é‡å¤§äºŽ**并å‘**，å‡çº§å°†å¤±è´¥ã€‚如果å‡çº§å¤±è´¥ï¼Œä½ å¯èƒ½éœ€è¦ä¿®å¤æˆ–移除失败的节点，然åŽå‡çº§æ‰èƒ½æˆåŠŸã€‚ + +- **controlplane 并å‘**：å¯ä»¥åŒæ—¶å‡çº§çš„最大æœåŠ¡å™¨èŠ‚点数；也是最大ä¸å¯ç”¨æœåŠ¡å™¨èŠ‚点数 +- **Worker 并å‘**：å¯ä»¥åŒæ—¶å‡çº§çš„最大 worker 节点数；也是最大ä¸å¯ç”¨ worker 节点数 + +在 RKE2 å’Œ K3s 文档中，control plane 节点也称为 Server 节点。Kubernetes master 节点è¿è¡Œåœ¨è¿™äº›èŠ‚点上，用于维护集群的状æ€ã€‚默认情况下，control plane 节点默认能够让工作负载调度到节点上。 + +类似的，在 RKE2 å’Œ K3s 文档中，具有 worker 角色的节点称为 Agent 节点。默认情况下，部署在集群中的任何工作负载或 Pod 都能调度到这些节点上。 + +## 已注册 RKE2 å’Œ K3s 集群的 Logging 调试和故障排除 + +节点由è¿è¡Œåœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­çš„ `system-upgrade-controller` å‡çº§ã€‚基于集群é…置，Rancher 部署了两个[计划](https://github.com/rancher/system-upgrade-controller#example-upgrade-plan)æ¥å‡çº§èŠ‚点，分别用于å‡çº§ control plane 节点和 worker 节点。`system-upgrade-controller` 会按照计划对节点进行å‡çº§ã€‚ + +è¦åœ¨ `system-upgrade-controller` deployment 上å¯ç”¨è°ƒè¯•æ—¥å¿—记录，请编辑 [configmap](https://github.com/rancher/system-upgrade-controller/blob/50a4c8975543d75f1d76a8290001d87dc298bdb4/manifests/system-upgrade-controller.yaml#L32) 以将调试环境å˜é‡è®¾ç½®ä¸º true。然åŽé‡å¯ `system-upgrade-controller` pod。 + +ä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤æŸ¥çœ‹ `system-upgrade-controller` 创建的日志： + +``` +kubectl logs -n cattle-system system-upgrade-controller +``` + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æŸ¥çœ‹è®¡åˆ’的当å‰çŠ¶æ€ï¼š + +``` +kubectl get plans -A -o yaml +``` + +如果集群å¡åœ¨å‡çº§ä¸­ï¼Œè¯·é‡å¯ `system-upgrade-controller`。 + +为防止å‡çº§æ—¶å‡ºçŽ°é—®é¢˜ï¼Œåº”éµå¾ª [Kubernetes å‡çº§æœ€ä½³å®žè·µ](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/)进行æ“作。 + +## 对 RKE2 å’Œ K3s 集群的授æƒé›†ç¾¤ç«¯ç‚¹æ”¯æŒ + +_从 v2.6.3 èµ·å¯ç”¨_ + +授æƒé›†ç¾¤ç«¯ç‚¹ (ACE) 已支æŒæ³¨å†Œçš„ RKE2 å’Œ K3s 集群。此支æŒè¿˜åŒ…括你在下游集群上å¯ç”¨ ACE 的手动步骤。有关授æƒé›†ç¾¤ç«¯ç‚¹çš„更多信æ¯ï¼Œè¯·å•å‡»[这里](../manage-clusters/access-clusters/authorized-cluster-endpoint.md)。 + +:::note 注æ„事项： + +- åªéœ€è¦åœ¨ä¸‹æ¸¸é›†ç¾¤çš„ controlplane 节点上执行这些步骤。你必须å•ç‹¬é…ç½®æ¯ä¸ª controlplane 节点。 + +- 以下步骤适用于在 v2.6.x 中注册的 RKE2 å’Œ K3s 集群，以åŠä»Žå…ˆå‰çš„ Rancher 版本注册（或导入）并å‡çº§åˆ° v2.6.x 的集群。 + +- 这些步骤将改å˜ä¸‹æ¸¸ RKE2 å’Œ K3s 集群的é…置并部署 `kube-api-authn-webhook`。如果 ACE 的未æ¥å®žçŽ°éœ€è¦æ›´æ–° `kube-api-authn-webhook`，那么这也必须手动完æˆã€‚有关此 webhook 的更多信æ¯ï¼Œè¯·å•å‡»[此处](../manage-clusters/access-clusters/authorized-cluster-endpoint.md#关于-kube-api-auth-身份验è¯-webhook)。 + +::: + +###### **在æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤çš„ controlplane 上å¯ç”¨ ACE 的手动执行步骤**： + +1. 在 `/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml` 创建一个文件，内容如下： + ```yaml + apiVersion: v1 + kind: Config + clusters: + - name: Default + cluster: + insecure-skip-tls-verify: true + server: http://127.0.0.1:6440/v1/authenticate + users: + - name: Default + user: + insecure-skip-tls-verify: true + current-context: webhook + contexts: + - name: webhook + context: + user: Default + cluster: Default + ``` + +2. 将以下内容添加到é…置文件中（如果文件ä¸å­˜åœ¨ï¼Œåˆ™åˆ›å»ºä¸€ä¸ªï¼‰ã€‚请注æ„，默认ä½ç½®æ˜¯ `/etc/rancher/{rke2,k3s}/config.yaml`： + ```yaml + kube-apiserver-arg: + - authentication-token-webhook-config-file=/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml + ``` + +3. è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + sudo systemctl stop {rke2,k3s}-server + sudo systemctl start {rke2,k3s}-server + +4. 最åŽï¼Œä½ **å¿…é¡»**返回 Rancher UI å¹¶åœ¨é‚£é‡Œç¼–è¾‘å¯¼å…¥çš„é›†ç¾¤ï¼Œä»Žè€Œå®Œæˆ ACE å¯ç”¨ã€‚å•å‡» **â‹® > 编辑é…ç½®**，然åŽå•å‡»**集群é…ç½®**下的**网络**选项å¡ã€‚最åŽï¼Œå•å‡»**授æƒç«¯ç‚¹**çš„**å¯ç”¨**按钮。å¯ç”¨ ACE åŽï¼Œä½ å¯ä»¥è¾“入完全é™å®šçš„域å (FQDN) å’Œè¯ä¹¦ä¿¡æ¯ã€‚ + +:::note + +FQDN 字段是å¯é€‰çš„。如果指定了该字段，它应该指å‘下游集群。仅当下游集群å‰é¢æœ‰ä½¿ç”¨äº†ä¸å—ä¿¡ä»»è¯ä¹¦çš„è´Ÿè½½å‡è¡¡å™¨æ—¶æ‰éœ€è¦è¯ä¹¦ä¿¡æ¯ã€‚如果你使用的是有效è¯ä¹¦ï¼Œåˆ™ä¸éœ€è¦å¡«å†™ CA è¯ä¹¦å­—段。 + +::: + +## 注释已注册的集群 + +Rancher 没有注册的 Kubernetes 集群（除了 RKE2 å’Œ K3s Kubernetes 集群之外）如何预置或é…置集群的任何信æ¯ã€‚ + +因此，当 Rancher 注册集群时，它å‡è®¾æŸäº›åŠŸèƒ½æ˜¯é»˜è®¤ç¦ç”¨çš„。Rancher 这样åšæ˜¯ä¸ºäº†é¿å…å‘用户暴露 UI 选项（å³ä½¿æ³¨å†Œçš„集群没有å¯ç”¨è¿™äº›åŠŸèƒ½ï¼‰ã€‚ + +但是，如果集群具有æŸç§åŠŸèƒ½ï¼ˆä¾‹å¦‚使用 pod 安全策略），那么该集群的用户å¯èƒ½ä»å¸Œæœ›åœ¨ Rancher UI 中为集群选择 pod 安全策略。为此，用户需è¦æ‰‹åŠ¨è®© Rancher 知é“集群已å¯ç”¨ pod 安全策略。 + +通过对已注册的集群进行注释，你å¯ä»¥å‘ Rancher 表明集群在 Rancher 之外被赋予了 Pod 安全策略或其他功能。 + +此示例注释表示å¯ç”¨äº† pod 安全策略： + +```json +"capabilities.cattle.io/pspEnabled": "true" +``` + +以下注释表示 Ingress 功能。请注æ„，éžåŽŸå§‹å¯¹è±¡çš„值需è¦è¿›è¡Œ JSON ç¼–ç ï¼Œå¹¶è½¬ä¹‰å¼•å·ï¼š + +```json +"capabilities.cattle.io/ingressCapabilities": "[ + { + "customDefaultBackend":true, + "ingressProvider":"asdf" + } +]" +``` + +ä½ å¯ä»¥ä¸ºé›†ç¾¤æ³¨é‡Šä»¥ä¸‹åŠŸèƒ½ï¼š + +- `ingressCapabilities` +- `loadBalancerCapabilities` +- `nodePoolScalingSupported` +- `nodePortRange` +- `pspEnabled` +- `taintSupport` + +所有功能åŠå…¶ç±»åž‹å®šä¹‰éƒ½å¯ä»¥åœ¨ Rancher API 视图中查看，地å€æ˜¯ `[Rancher Server URL]/v3/schemas/capabilities`。 + +è¦æ³¨é‡Šå·²æ³¨å†Œçš„集群： + +1. 点击 **☰ > 集群管ç†**。 +2. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ³¨é‡Šçš„自定义集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +3. 展开**标签 & 注释**。 +4. å•å‡»**添加注释**。 +5. 使用 `capabilities/: ` æ ¼å¼å‘集群添加注释，其中 `value` 是è¦ä½¿ç”¨æ³¨é‡Šè¦†ç›–的集群功能。在这ç§æƒ…况下，Rancher 在你添加注释之å‰éƒ½ä¸çŸ¥é“集群的任何功能。 +6. å•å‡»**ä¿å­˜**。 + +**结果**：注释并ä¸æ˜¯ç»™é›†ç¾¤æ供功能，而是告知 Rancher 集群具有这些功能。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md new file mode 100644 index 00000000000..3c31569bf95 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md @@ -0,0 +1,160 @@ +--- +title: 设置 Amazon 云æ供商 +weight: 1 +--- + +使用 `Amazon` 云æ供商时，你å¯ä»¥åˆ©ç”¨ä»¥ä¸‹åŠŸèƒ½ï¼š + +- **è´Ÿè½½å‡è¡¡å™¨**：在 **Port Mapping** 中选择 `Layer-4 Load Balancer` 或使用 `type: LoadBalancer` å¯åŠ¨ `Service` 时，å¯åŠ¨ AWS 弹性负载å‡è¡¡å™¨ (ELB)。 +- **æŒä¹…å·**：å…许你将 AWS 弹性å—存储 (EBS) 用于æŒä¹…å·ã€‚ + +有关 Amazon 云æ供商的所有信æ¯ï¼Œè¯·å‚阅 [cloud-provider-aws 自述文件](https://kubernetes.github.io/cloud-provider-aws/)。 + +è¦è®¾ç½® Amazon 云æ供商： + +1. [创建一个 IAM 角色并附加到实例](#1-创建-iam-角色并附加到实例) +2. [é…ç½® ClusterID](#2-创建-clusterid) + +:::note é‡è¦æ示： + +从 Kubernetes 1.23 开始，你必须åœç”¨ `CSIMigrationAWS` 特性开关æ‰èƒ½ä½¿ç”¨æ ‘内 AWS 云æ供商。为此，你å¯ä»¥åœ¨é«˜çº§é›†ç¾¤é…置中将 `feature-gates=CSIMigrationAWS=false` 设置为集群 Kubeletã€Controller Managerã€API Server å’Œ Scheduler 的附加å‚数。 + +::: + +### 1. 创建 IAM 角色并附加到实例 + +添加到集群的所有节点都必须能够与 EC2 交互，以便它们å¯ä»¥åˆ›å»ºå’Œåˆ é™¤èµ„æºã€‚ä½ å¯ä»¥ä½¿ç”¨é™„加到实例的 IAM 角色æ¥å¯ç”¨äº¤äº’。请å‚阅 [Amazon 文档：创建 IAM 角色](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#create-iam-role) æ¥åˆ›å»º IAM 角色。有两个示例策略： + +* 第一个策略适用于具有 `controlplane` 角色的节点。这些节点必须能够创建/删除 EC2 资æºã€‚以下 IAM 策略是一个示例，请根æ®ä½ çš„实际用例移除ä¸éœ€è¦çš„æƒé™ã€‚ +* 第二个策略适用于具有 `etcd` 或 `worker` 角色的节点。这些节点åªéœ€èƒ½å¤Ÿä»Ž EC2 检索信æ¯ã€‚ + +在创建 [Amazon EC2 集群](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md)时，你必须在创建**节点模æ¿**时填写创建的 IAM 角色的 **IAM Instance Profile Name**（ä¸æ˜¯ ARN）。 + +创建[自定义集群](../../../../pages-for-subheaders/use-existing-nodes.md)时，你必须手动将 IAM 角色附加到实例。 + +具有 `controlplane` 角色的节点的 IAM 策略： + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } +] +} +``` + +具有 `etcd` 或 `worker` 角色的节点的 IAM 策略： + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } +] +} +``` + +### 2. 创建 ClusterID + +以下资æºéœ€è¦ä½¿ç”¨ `ClusterID` 进行标记： + +- **Nodes**：Rancher 中添加的所有主机。 +- **Subnet**：集群使用的å­ç½‘。 +- **Security Group**：用于你的集群的安全组。 + +:::note + +ä¸è¦æ ‡è®°å¤šä¸ªå®‰å…¨ç»„。创建弹性负载å‡è¡¡å™¨ (ELB) 时，标记多个组会产生错误。 + +::: + +创建 [Amazon EC2 集群](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md)时，会自动为创建的节点é…ç½® `ClusterID`。其他资æºä»ç„¶éœ€è¦æ‰‹åŠ¨æ ‡è®°ã€‚ + +使用以下标签： + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `owned` + +`CLUSTERID` å¯ä»¥æ˜¯ä»»ä½•å­—符串，åªè¦å®ƒåœ¨æ‰€æœ‰æ ‡ç­¾é›†ä¸­ç›¸åŒå³å¯ã€‚ + +将标签的值设置为 `owned` 会通知集群带有该标签的所有资æºéƒ½ç”±è¯¥é›†ç¾¤æ‹¥æœ‰å’Œç®¡ç†ã€‚如果你在集群之间共享资æºï¼Œä½ å¯ä»¥å°†æ ‡ç­¾æ›´æ”¹ä¸ºï¼š + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `shared`. + +### 使用 Amazon Elastic Container Registry (ECR) + +在将[创建 IAM 角色并附加到实例](#1-创建-iam-角色并附加到实例)中的 IAM é…置文件附加到实例时，kubelet ç»„ä»¶èƒ½å¤Ÿè‡ªåŠ¨èŽ·å– ECR 凭è¯ã€‚使用低于 v1.15.0 çš„ Kubernetes 版本时，需è¦åœ¨é›†ç¾¤ä¸­é…ç½® Amazon 云æ供商。从 Kubernetes 版本 v1.15.0 开始，kubelet 无需在集群中é…ç½® Amazon 云æ供商å³å¯èŽ·å– ECR 凭è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md new file mode 100644 index 00000000000..0b6fc4bac45 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md @@ -0,0 +1,111 @@ +--- +title: 设置 Azure 云æ供商 +--- + +使用 `Azure` 云æ供商时，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹åŠŸèƒ½ï¼š + +- **è´Ÿè½½å‡è¡¡å™¨**：在特定网络安全组中å¯åŠ¨ Azure è´Ÿè½½å‡è¡¡å™¨ã€‚ + +- **æŒä¹…å·**：支æŒå°† Azure Blob ç£ç›˜å’Œ Azure 托管ç£ç›˜ä¸Žæ ‡å‡†å’Œé«˜çº§ Storage Account 一起使用。 + +- **网络存储**：通过 CIFS æŒ‚è½½æ”¯æŒ Azure 文件。 + +Azure 订阅ä¸æ”¯æŒä»¥ä¸‹è´¦å·ç±»åž‹ï¼š + +- å•ç§Ÿæˆ·è´¦å·ï¼ˆå³æ²¡æœ‰è®¢é˜…çš„è´¦å·ï¼‰ã€‚ +- 多订阅账å·ã€‚ + +## RKE å’Œ RKE2 的先决æ¡ä»¶ + +è¦ä¸º RKE å’Œ RKE2 设置 Azure 云æ供商，你需è¦é…置以下凭è¯ï¼š + +1. [设置 Azure 租户 ID](#1-设置-azure-租户-id) +2. [设置 Azure 客户端 ID å’Œ Azure 客户端密ç ](#2-设置-azure-客户端-id-å’Œ-azure-客户端密ç ) +3. [é…置应用注册æƒé™](#3-é…置应用注册æƒé™) +4. [设置 Azure 网络安全组å称](#4-设置-azure-网络安全组å称) + +### 1. 设置 Azure 租户 ID + +访问 [Azure 门户](https://portal.azure.com)，登录并转到 **Azure Active Directory**，然åŽé€‰æ‹© **Properties**。你的 **Directory ID** 是你的 **Tenant ID** (tenantID)。 + +如果è¦ä½¿ç”¨ Azure CLI，你å¯ä»¥è¿è¡Œ `az account show` 命令æ¥èŽ·å–ä¿¡æ¯ã€‚ + +### 2. 设置 Azure 客户端 ID å’Œ Azure å®¢æˆ·ç«¯å¯†ç  + +访问 [Azure 门户](https://portal.azure.com)，登录并按照以下步骤创建 **App Registration** 和对应的 **Azure Client ID** (aadClientId) ä»¥åŠ **Azure Client Secret** (aadClientSecret)。 + +1. 选择 **Azure Active Directory**。 +1. 选择 **App registrations**。 +1. 选择 **New application registration**。 +1. 选择 **Name**，选择 `Web app/API` 作为 **Application Type**ï¼Œå¹¶é€‰æ‹©ä»»æ„ **Sign-on URL**。 +1. 选择 **Create**。 + +在 **App registrations** 视图中，你应该会看到你创建的应用注册。**APPLICATION ID** 列中显示的值是需è¦ç”¨ä½œ **Azure Client ID** 的值。 + +ä¸‹ä¸€æ­¥æ˜¯ç”Ÿæˆ **Azure Client Secret**： + +1. 打开你创建的应用注册。 +1. 在 **Settings** 视图中，打开 **Keys**。 +1. 输入 **Key description**，选择过期时间，然åŽé€‰æ‹© **Save**。 +1. **Value** 列中显示的生æˆå€¼æ˜¯éœ€è¦ç”¨ä½œ **Azure Client Secret** 的值。该值åªä¼šæ˜¾ç¤ºä¸€æ¬¡ã€‚ + +### 3. é…置应用注册æƒé™ + +最åŽï¼Œä¸ºä½ çš„应用注册分é…适当的æƒé™ï¼š + +1. å‰å¾€ **More services**，æœç´¢ **Subscriptions** 并打开它。 +1. 打开 **Access control (IAM)**。 +1. 选择 **Add**。 +1. 在 **Role** 中选择 `Contributor`。 +1. 在 **Select** 中选择你创建的应用注册的å称。 +1. 选择 **Save**。 + +### 4. 设置 Azure 网络安全组å称 + +è¦ä½¿ Azure è´Ÿè½½å‡è¡¡å™¨æ­£å¸¸å·¥ä½œï¼Œä½ éœ€è¦è‡ªå®šä¹‰ä¸€ä¸ª Azure 网络安全组 (securityGroupName)。 + +如果你使用 Rancher Machine Azure 驱动æ¥é…置主机，则需è¦æ‰‹åŠ¨ç¼–辑主机，从而将主机分é…给此网络安全组。 + +你需è¦åœ¨é…置期间将自定义主机分é…给此网络安全组。 + +åªæœ‰éœ€è¦æˆä¸ºè´Ÿè½½å‡è¡¡å™¨åŽç«¯çš„主机æ‰éœ€è¦åˆ†é…到该组。 + +## Rancher 中的 RKE2 集群设置 + +1. 在**集群é…ç½®**中的**云æ供商**下拉列表中选择 **Azure**。 + +1. + * é…置云æ供商。请注æ„，Rancher 会自动创建新的网络安全组ã€èµ„æºç»„ã€å¯ç”¨æ€§é›†ã€å­ç½‘和虚拟网络。如果你已ç»åˆ›å»ºäº†å…¶ä¸­çš„一部分或全部，则需è¦åœ¨åˆ›å»ºé›†ç¾¤ä¹‹å‰æŒ‡å®šå®ƒä»¬ã€‚ + * ä½ å¯ä»¥å•å‡»**显示高级选项**以查看更多自动生æˆçš„å称，并在需è¦çš„时候更新它们。你的云æ供商é…ç½®**å¿…é¡»**与**主机池**中的字段匹é…。如果你有多个池，它们必须使用相åŒçš„资æºç»„ã€å¯ç”¨æ€§é›†ã€å­ç½‘ã€è™šæ‹Ÿç½‘络和网络安全组。 + * 下é¢æ供了一个示例。你å¯ä»¥æ ¹æ®éœ€è¦å¯¹å…¶è¿›è¡Œä¿®æ”¹ã€‚ + +
+ 云æ供商é…置示例 + + ```yaml + { + "cloud":"AzurePublicCloud", + "tenantId": "YOUR TENANTID HERE", + "aadClientId": "YOUR AADCLIENTID HERE", + "aadClientSecret": "YOUR AADCLIENTSECRET HERE", + "subscriptionId": "YOUR SUBSCRIPTIONID HERE", + "resourceGroup": "docker-machine", + "location": "westus", + "subnetName": "docker-machine", + "securityGroupName": "rancher-managed-KA4jV9V2", + "securityGroupResourceGroup": "docker-machine", + "vnetName": "docker-machine-vnet", + "vnetResourceGroup": "docker-machine", + "primaryAvailabilitySetName": "docker-machine", + "routeTableResourceGroup": "docker-machine", + "cloudProviderBackoff": false, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true + } + ``` + +
+ +1. 在**集群é…ç½® > 高级选项**ã€ä¸­ï¼Œå•å‡»**补充的 Controller Manager å‚æ•°**下的**添加**，并添加 `--configure-cloud-routes=false` 标志。 + +1. å•å‡»**创建**按钮æ¥æ交表å•å¹¶åˆ›å»ºé›†ç¾¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md new file mode 100644 index 00000000000..17d1e4f671c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md @@ -0,0 +1,19 @@ +--- +title: é…置树内 vSphere 云æ供商 +--- + +è¦è®¾ç½®æ ‘内 vSphere 云æ供商，请在 Rancher 中创建 vSphere 集群时按照以下步骤æ“作： + +1. å°† **Cloud Provider** 选项设置为 `Custom` 或 `Custom (In-Tree)`。 +1. å•å‡»**以 YAML 文件编辑**。 +1. 将以下结构æ’入到预填充的集群 YAML。这个结构必须放在 `rancher_kubernetes_engine_config` 下。请注æ„，`name` *å¿…é¡»*设置为 `vsphere`。 + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: vsphere + vsphereCloudProvider: + [Insert provider configuration] + ``` + +Rancher 使用 RKE（Rancher Kubernetes Engine）æ¥é…ç½® Kubernetes 集群。有关 `vsphereCloudProvider` 的属性的详细信æ¯ï¼Œè¯·å‚阅 RKE 文档中的 [vSphere é…ç½®å‚考](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere/config-reference/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md new file mode 100644 index 00000000000..f0461eb27b5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md @@ -0,0 +1,64 @@ +--- +title: é…置树外 vSphere 云æ供商 +--- + +Kubernetes 正在é€æ¸ä¸åœ¨æ ‘内维护云æ供商。vSphere 有一个树外云æ供商，å¯é€šè¿‡å®‰è£… vSphere 云æ供商和云存储æ’件æ¥ä½¿ç”¨ã€‚ + +本文介ç»äº†å¦‚何在设置集群åŽå®‰è£… Cloud Provider Interface (CPI) å’Œ Cloud Storage Interface (CSI) æ’件。 + +## 先决æ¡ä»¶ + +支æŒçš„ vSphere 版本： + +* 6.7u3 +* 7.0u1 或更高版本。 + +Kubernetes 版本必须为 1.19 或更高版本。 + +树外 vSphere 云æ供商è¦æ±‚使用 Linux 节点，ä¸æ”¯æŒ Windows 节点。 + +## 安装 + +åœ¨å®‰è£…äº‘å­˜å‚¨æŽ¥å£ (CSI) 之å‰ï¼Œåº”先安装云æä¾›å•†æŽ¥å£ (CPI)。 + +### 1. 创建 vSphere 集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **VMware vSphere** 或 **自定义**。 +1. 在**集群é…ç½®**çš„**基本信æ¯**选项å¡ä¸­ï¼Œå°†**云æ供商**设置为 **vSphere**。 +1. 在**附加é…ç½®**选项å¡ä¸­ï¼Œè®¾ç½® vSphere Cloud Provider (CPI) å’Œ Storage Provider (CSI) 选项。 +1. 完æˆé›†ç¾¤åˆ›å»ºã€‚ + +### 2. 安装 CPI æ’件 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CPI æ’件的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CPI**。 +1. 填写所需的 vCenter 详细信æ¯ã€‚ +1. vSphere CPI 使用 vSphere CSI 驱动所需的 ProviderID æ¥åˆå§‹åŒ–所有节点。在使用以下命令安装 CSI 驱动之å‰ï¼Œæ£€æŸ¥æ˜¯å¦æ‰€æœ‰èŠ‚点都使用 ProviderID 进行了åˆå§‹åŒ–： + + ``` + kubectl describe nodes | grep "ProviderID" + ``` + +### 3. 安装 CSI æ’件 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CSI æ’件的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CSI**。 +1. å•å‡»**安装**。 +1. 填写所需的 vCenter 详细信æ¯ã€‚在**功能**选项å¡ä¸­ï¼Œå°†**å¯ç”¨ CSI è¿ç§»**设置为 **false**。 +3. 在**存储**选项å¡ä¸­ï¼Œå¡«å†™ StorageClass 的详细信æ¯ã€‚æ­¤ Chart 使用 `csi.vsphere.vmware.com` 作为置备程åºæ¥åˆ›å»ºä¸€ä¸ª StorageClass。 +1. å•å‡»**安装**。 + + +## 使用 CSI 驱动æ¥ç½®å¤‡å· + +CSI chart 默认创建一个 storageClass。 + +如果在å¯åŠ¨ chart 时未选择该选项，请使用 `csi.vsphere.vmware.com` 作为置备程åºæ¥åˆ›å»ºä¸€ä¸ª storageClass。 + +使用此 StorageClass 置备的所有å·éƒ½å°†ç”± CSI 驱动é…置。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md new file mode 100644 index 00000000000..f5e437e9de8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md @@ -0,0 +1,57 @@ +--- +title: 设置 Google Compute Engine 云æ供商 +--- + +在本节中，你将了解如何在 Rancher 中为自定义集群å¯ç”¨ Google Compute Engine (GCE) 云æ供商。自定义集群指的是 Rancher 在现有节点上安装 Kubernetes 的集群。 + +详情请å‚è§ [GCE 云æ供商的官方 Kubernetes 文档](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#gce)。 + +:::note 先决æ¡ä»¶ï¼š + +GCE 上的 `Identity` ServiceAccount å’Œ API 访问需è¦`Computer Admin`æƒé™ã€‚ + +::: + +如果你使用 Calico： + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è‡ªå®šä¹‰é›†ç¾¤ï¼Œç‚¹å‡» **â‹® > 编辑 YAML** 并输入如下é…置： + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + network: + options: + calico_cloud_provider: "gce" + plugin: "calico" + ``` + +如果你使用 Canal 或 Flannel： + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè¿›å…¥è‡ªå®šä¹‰é›†ç¾¤ï¼Œç‚¹å‡» **â‹® > 编辑 YAML** 并输入如下é…置： + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + services: + kube_controller: + extra_args: + configure-cloud-routes: true # we need to allow the cloud provider configure the routes for the hosts + ``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md new file mode 100644 index 00000000000..065ff2deb84 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md @@ -0,0 +1,108 @@ +--- +title: å°† vSphere 树内å·è¿ç§»åˆ°æ ‘外 +--- + +Kubernetes 正在é€æ¸ä¸åœ¨æ ‘内维护云æ供商。vSphere 有一个树外云æ供商，å¯é€šè¿‡å®‰è£… vSphere 云æ供商和云存储æ’件æ¥ä½¿ç”¨ã€‚ + +本页介ç»å¦‚何从树内 vSphere 云æ供商è¿ç§»åˆ°æ ‘外，以åŠå¦‚何在è¿ç§»åŽç®¡ç†çŽ°æœ‰è™šæ‹Ÿæœºã€‚ + +本文éµå¾ªå®˜æ–¹ [vSphere è¿ç§»æ–‡æ¡£](https://vsphere-csi-driver.sigs.k8s.io/features/vsphere_csi_migration.html)中æ供的步骤，并介ç»äº†è¦åœ¨ Rancher 中执行的步骤。 + +### Cloud-config æ ¼å¼é™åˆ¶ + +由于 vSphere CSI 中的现有错误，使用以下 cloud-config æ ¼å¼é…置的现有å·å°†ä¸ä¼šè¿ç§»ã€‚ + +如果 cloud-config 具有以下格å¼çš„ datastore 和资æºæ± è·¯å¾„，vsphere CSI 驱动将无法识别它： + +```yaml +default-datastore: /datastore/ +resourcepool-path: "/host//Resources/" +``` + +æ ¼å¼å¦‚下的使用树内æ供商预置的å·å°†èƒ½æ­£ç¡®è¿ç§»ï¼š + +```yaml +default-datastore: +resourcepool-path: "/Resources/" +``` + +上游 bug：https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/628 + +跟踪此 bug çš„ Rancher issue：https://github.com/rancher/rancher/issues/31105 + +## 先决æ¡ä»¶ + +- vSphere CSI è¿ç§»éœ€è¦ vSphere 7.0u1。为了管ç†çŽ°æœ‰çš„树内 vSphere å·ï¼Œè¯·å°† vSphere å‡çº§åˆ° 7.0u1。 +- Kubernetes 版本必须为 1.19 或更高版本。 + +## è¿ç§» + +### 1. 安装 CPI æ’件 + +在安装 CPI 之å‰ï¼Œæˆ‘们需è¦ä½¿ç”¨ `node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule` 为所有节点添加污点。 + +è¿™å¯ä»¥é€šè¿‡è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥å®Œæˆï¼š + +``` +curl -O https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +``` + +或者： + +``` +wget https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +chmod +x taints.sh +./taints.sh +``` + +通过è¿è¡Œè„šæœ¬ä¸ºæ‰€æœ‰èŠ‚点添加污点åŽï¼Œå¯åŠ¨ Helm vSphere CPI Chart： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CPI chart 的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CPI**。 +1. å•å‡»**安装**。 +1. 填写所需的 vCenter 详细信æ¯ï¼Œç„¶åŽå•å‡»**安装**。 + +vSphere CPI 使用 vSphere CSI 驱动所需的 ProviderID æ¥åˆå§‹åŒ–所有节点。 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œæ£€æŸ¥æ˜¯å¦å·²ä½¿ç”¨ ProviderID åˆå§‹åŒ–了所有节点： + +``` +kubectl describe nodes | grep "ProviderID" +``` + +### 2. 安装 CSI 驱动 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到将安装 vSphere CSI chart 的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Apps > Charts**。 +1. å•å‡» **vSphere CSI**。 +1. å•å‡»**安装**。 +1. 填写所需的 vCenter 详细信æ¯ï¼Œç„¶åŽå•å‡»**安装**。 +1. 选中**在安装å‰è‡ªå®šä¹‰ Helm 选项**，然åŽå•å‡»**下一步**。 +1. 在**功能**选项å¡ä¸­ï¼Œé€‰ä¸­**å¯ç”¨ CSI è¿ç§»**。 +1. 你也å¯ä»¥è½¬åˆ°**存储**选项å¡å¹¶è®¾ç½® datastore。此 Chart 使用 `csi.vsphere.vmware.com` 作为置备程åºæ¥åˆ›å»ºä¸€ä¸ª StorageClass。在创建此 StorageClass 时，你也å¯ä»¥æ供用于é…ç½® CSI å·çš„ datastore URL。通过选择 datastore 并转到**概述**选项å¡ï¼Œä½ å¯ä»¥åœ¨ vSphere 客户端中找到 datastore URL。填写 StorageClass 的详细信æ¯ã€‚ +1. å•å‡»**安装**。 + +### 3. 编辑集群以å¯ç”¨ CSI è¿ç§»çš„功能开关 + +1. 在编辑集群时，如果 Kubernetes 版本低于 1.19，请从 **Kubernetes 版本**下拉èœå•ä¸­é€‰æ‹© Kubernetes 1.19 或更高版本。 +2. è¦å¯ç”¨åŠŸèƒ½å¼€å…³ï¼Œè¯·å•å‡»**以 YAML 文件编辑**，然åŽåœ¨ kube-controller å’Œ kubelet 下添加以下内容： + + ```yaml + extra_args: + feature-gates: "CSIMigration=true,CSIMigrationvSphere=true" + ``` + +### 4. 清空 worker 节点 + +在更改 kubelet å’Œ kube-controller-manager å‚数之å‰ï¼Œå¿…须在å‡çº§æœŸé—´æ¸…空 worker 节点： + + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ¸…空 worker 节点的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**高级选项**中，将**最大ä¸å¯ç”¨çš„ Worker 节点数**字段设置为 1。 +1. è¦åœ¨å‡çº§æœŸé—´æ¸…空节点，请选择**清空节点 > 是**。 +1. å°†**强制**å’Œ**删除本地数æ®**设置为 **true**。 +1. 点击**ä¿å­˜**以å‡çº§é›†ç¾¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md new file mode 100644 index 00000000000..4369666a95d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md @@ -0,0 +1,153 @@ +--- +title: 创建 AKS 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在 Microsoft Azure Kubernetes æœåŠ¡ (AKS) 中的集群。 + + +## Microsoft Azure 中的先决æ¡ä»¶ + +:::caution + +部署到 AKS 会产生费用。 + +::: + +è‹¥è¦ä¸Ž Azure API 交互，AKS é›†ç¾¤éœ€è¦ Azure Active Directory (AD) æœåŠ¡ä¸»ä½“。æœåŠ¡ä¸»ä½“是动æ€åˆ›å»ºå’Œç®¡ç†å…¶ä»– Azure 资æºæ‰€å¿…需的，它为集群æ供凭è¯æ¥ä¸Ž AKS 通信。有关æœåŠ¡ä¸»ä½“的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/kubernetes-service-principal)。 + +在创建æœåŠ¡ä¸»ä½“之å‰ï¼Œä½ éœ€è¦ä»Ž [Microsoft Azure 门户](https://portal.azure.com)获å–以下信æ¯ï¼š + +- 订阅 ID +- 客户端 ID +- å®¢æˆ·ç«¯å¯†ç  + +以下介ç»å¦‚何使用 Azure 命令行工具或 Azure 门户进行设置，从而满足先决æ¡ä»¶ã€‚ + +### 使用 Azure 命令行工具设置æœåŠ¡ä¸»ä½“ + +è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ›å»ºæœåŠ¡ä¸»ä½“： + +``` +az ad sp create-for-rbac --skip-assignment +``` + +结果应显示新æœåŠ¡ä¸»ä½“çš„ä¿¡æ¯ï¼š +``` +{ + "appId": "xxxx--xxx", + "displayName": "", + "name": "http://", + "password": "", + "tenant": "" +} +``` + +你还需è¦å‘æœåŠ¡ä¸»ä½“添加角色，以便它具有与 AKS API 通信的æƒé™ã€‚它还需è¦è®¿é—®æƒé™æ‰èƒ½åˆ›å»ºå’Œåˆ—出虚拟网络。 + +下é¢æ˜¯å°†å‚与者角色分é…ç»™æœåŠ¡ä¸»ä½“的示例命令。å‚与者å¯ä»¥ç®¡ç† AKS 上的任何内容，但ä¸èƒ½æŽˆäºˆå…¶ä»–人访问æƒé™ï¼š + +``` +az role assignment create \ + --assignee $appId \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +你还å¯ä»¥é€šè¿‡åˆå¹¶ä¸¤ä¸ªå‘½ä»¤ï¼Œæ¥åˆ›å»ºæœåŠ¡ä¸»ä½“并授予å‚与者æƒé™ã€‚在此命令中，`scope` 需è¦æä¾› Azure 资æºçš„完整路径： + +``` +az ad sp create-for-rbac \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +### 从 Azure 门户设置æœåŠ¡ä¸»ä½“ + +你还å¯ä»¥æŒ‰ç…§è¿™äº›è¯´æ˜Žè®¾ç½®æœåŠ¡ä¸»ä½“，并从 Azure 门户为它授予基于角色的访问æƒé™ã€‚ + +1. 转到 Microsoft Azure 门户[主页](https://portal.azure.com)。 + +1. å•å‡» **Azure Active Directory**。 +1. å•å‡» **App registrations**。 +1. å•å‡» **New registration**。 +1. 输入å称。这将是你æœåŠ¡ä¸»ä½“çš„å称。 +1. å¯é€‰ï¼šé€‰æ‹©å¯ä»¥ä½¿ç”¨æœåŠ¡ä¸»ä½“çš„è´¦å·ã€‚ +1. å•å‡» **Register**。 +1. 你现在应该能在 **Azure Active Directory > App registrations** 下看到æœåŠ¡ä¸»ä½“çš„å称。 +1. å•å‡»ä½ çš„æœåŠ¡ä¸»ä½“çš„å称。记下应用 ID（也称为应用 ID 或客户端 IDï¼‰ï¼Œä»¥ä¾¿åœ¨é¢„é… AKS 集群时使用。然åŽå•å‡» **Certificates & secrets**。 +1. å•å‡» **New client secret**。 +1. 输入简短说明，选择过期时间，然åŽå•å‡» **Add**。记下客户端密ç ï¼Œä»¥ä¾¿åœ¨é¢„é… AKS 集群时使用它。 + +**结果**：你已ç»åˆ›å»ºäº†ä¸€ä¸ªæœåŠ¡ä¸»ä½“，现在你能够在 **App registrations** 下的 **Azure Active Directory** 中找到它。你ä»ç„¶éœ€è¦å‘æœåŠ¡ä¸»ä½“授予对 AKS 的访问æƒé™ã€‚ + +è¦æŽˆäºˆå¯¹æœåŠ¡ä¸»ä½“基于角色的访问æƒé™ï¼š + +1. 点击左侧导航æ ä¸­çš„ **All Services**。然åŽå•å‡» **Subscriptions**。 +1. å•å‡»è¦ä¸Ž Kubernetes 集群关è”的订阅的å称。记下订阅 IDï¼Œä»¥ä¾¿åœ¨é¢„é… AKS 集群时使用它。 +1. å•å‡» **Access Control (IAM)**。 +1. 在 **Add role assignment** 中，å•å‡» **Add**。 +1. 在 **Role** 字段中，选择将有æƒè®¿é—® AKS 的角色。例如，你å¯ä»¥ä½¿ç”¨ **Contributor** 角色，该角色有æƒç®¡ç†é™¤æŽˆäºˆå…¶ä»–用户访问æƒé™ä¹‹å¤–的所有内容。 +1. 在 **Assign access to** 字段中，选择 **Azure AD 用户ã€ç»„或æœåŠ¡ä¸»ä½“**。 +1. 在 **Select** 字段中，选择æœåŠ¡ä¸»ä½“çš„å称，然åŽå•å‡» **Save**。 + +**结果**：你的æœåŠ¡ä¸»ä½“现在å¯ä»¥è®¿é—® AKS。 + +## 1. 创建 AKS äº‘å‡­è¯ + +1. 在 Rancher UI 中，å•å‡» **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 填写表å•ã€‚有关填写表格的帮助，请å‚阅[é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md#云凭è¯)。 +1. å•å‡»**创建**。 + +## 2. 创建 AKS 集群 + +使用 Rancher é…置你的 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**中，å•å‡»**创建**。 +1. å•å‡» **Azure AKS**。 +1. 填写表å•ã€‚有关填写表格的帮助，请å‚阅[é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md)。 +1. å•å‡»**创建**。 + +**结果**：集群已创建，并处于 **Provisioning** 状æ€ã€‚Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +## RBAC +在 Rancher UI 中é…ç½® AKS 集群时，由于 RBAC 需è¦å¯ç”¨ï¼Œå› æ­¤ RBAC ä¸å¯é…置。 + +注册或导入到 Rancher çš„ AKS é›†ç¾¤éœ€è¦ RBAC。 + +## AKS 集群é…ç½®å‚考 + +有关如何在 Rancher UI é…ç½® AKS 集群的更多信æ¯ï¼Œè¯·å‚阅[é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md)。 + +## ç§æœ‰é›†ç¾¤ + +通常情况下，无论集群是å¦ä¸ºç§æœ‰ï¼ŒAKS worker 节点都ä¸ä¼šèŽ·å¾—公共 IP。在ç§æœ‰é›†ç¾¤ä¸­ï¼Œcontrolplane 没有公共端点。 + +Rancher å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€è¿žæŽ¥åˆ°ç§æœ‰ AKS 集群。 + +第一ç§æ–¹æ³•æ˜¯ç¡®ä¿ Rancher è¿è¡Œåœ¨ä¸Ž AKS 节点相åŒçš„ [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) 上。 + +第二ç§æ–¹æ³•æ˜¯è¿è¡Œå‘½ä»¤å‘ Rancher 注册集群。é…置集群åŽï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èƒ½è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API 的地方è¿è¡Œæ˜¾ç¤ºçš„命令。é…ç½®å¯ç”¨äº†ç§æœ‰ API 端点的 AKS 集群时，此命令将显示在弹出窗å£ä¸­ã€‚ + +:::note + +注册现有 AKS 集群时，集群å¯èƒ½éœ€è¦ä¸€äº›æ—¶é—´ï¼ˆå¯èƒ½æ˜¯æ•°å°æ—¶ï¼‰æ‰ä¼šå‡ºçŽ°åœ¨ `Cluster To register` 下拉列表中。ä¸åŒåŒºåŸŸçš„结果å¯èƒ½ä¸åŒã€‚ + +::: + +有关连接到 AKS 专用集群的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster)。 + +## åŒæ­¥ + +AKS é…置者å¯ä»¥åœ¨ Rancher å’Œæ供商之间åŒæ­¥ AKS 集群的状æ€ã€‚有关其工作原ç†çš„技术说明，请å‚阅[åŒæ­¥](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md)。 + +有关é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/gke-cluster-configuration.md#é…置刷新间隔)。 + +## 以编程方å¼åˆ›å»º AKS 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² AKS 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md new file mode 100644 index 00000000000..a10200d37fa --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md @@ -0,0 +1,59 @@ +--- +title: 创建阿里云 ACK 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在阿里云 Alibaba Cloud Kubernetes (ACK) 中的集群。Rancher å·²ç»ä¸º ACK 实现并打包了针对 ACK çš„[集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md)，但是默认情况下，这个集群驱动的状æ€æ˜¯ `inactive`。为了å¯åŠ¨ ACK 集群，你需è¦[å¯ç”¨ ACK 集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#激活åœç”¨é›†ç¾¤é©±åŠ¨)。å¯ç”¨é›†ç¾¤é©±åŠ¨åŽï¼Œä½ å¯ä»¥å¼€å§‹é…ç½® ACK 集群。 + +## Rancher 之外的先决æ¡ä»¶ + +:::caution + +部署到 ACK 会产生费用。 + +::: + +1. 在阿里云中，在对应的控制å°ä¸­æ¿€æ´»ä»¥ä¸‹æœåŠ¡ã€‚ + + - [容器æœåŠ¡](https://cs.console.aliyun.com) + - [资æºç¼–排æœåŠ¡](https://ros.console.aliyun.com) + - [RAM](https://ram.console.aliyun.com) + +2. ç¡®ä¿ä½ ç”¨äºŽåˆ›å»º ACK 集群的账å·å…·æœ‰é€‚当的æƒé™ã€‚详è§é˜¿é‡Œäº‘[角色授æƒ](https://www.alibabacloud.com/help/doc-detail/86483.htm)å’Œ[使用容器æœåŠ¡æŽ§åˆ¶å°ä½œä¸º RAM 用户](https://www.alibabacloud.com/help/doc-detail/86484.htm)官方文档。 + +3. 在阿里云中，创建[访问密钥](https://www.alibabacloud.com/help/doc-detail/53045.html)。 + +4. 在阿里云中，创建 [SSH 密钥对](https://www.alibabacloud.com/help/doc-detail/51793.html)。该密钥用于访问 Kubernetes 集群中的节点。 + +## Rancher 先决æ¡ä»¶ + +你需è¦å¯ç”¨é˜¿é‡Œäº‘ ACK 集群驱动： + +1. 点击 **☰ > 集群管ç†**。 +1. 点击**驱动**。 +1. 在**集群驱动**选项å¡ä¸­ï¼Œè½¬åˆ° **Alibaba ACK** 集群驱动并å•å‡» **â‹® > 激活**。 + +集群驱动下载完æˆåŽï¼Œå°±å¯ä»¥åœ¨ Rancher 中创建阿里云 ACK 集群了。 + +## 创建 ACK 集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. å•å‡» **Alibaba ACK**。 +1. 输入**集群å称**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 为 ACK 集群é…ç½®**è´¦å·è®¿é—®**。选择è¦åœ¨å…¶ä¸­æž„建集群的地ç†åŒºåŸŸï¼Œå¹¶è¾“入在先决æ¡ä»¶æ­¥éª¤ä¸­åˆ›å»ºçš„访问密钥。 +1. 点击**下一步：é…置集群**，然åŽé€‰æ‹©é›†ç¾¤ç±»åž‹ã€Kubernetes 版本和å¯ç”¨åŒºã€‚ +1. 如果选择 **Kubernetes** 作为集群类型，请å•å‡»**下一步：é…ç½® master 节点**，然åŽå¡«å†™ **master 节点**表å•ã€‚ +1. å•å‡»**下一步：é…ç½® Worker 节点**，然åŽå¡«å†™ **Worker 节点**表å•ã€‚ +1. 检查并确认你的选项。然åŽå•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md new file mode 100644 index 00000000000..ede4d2dad7a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md @@ -0,0 +1,329 @@ +--- +title: 创建 EKS 集群 +--- + +Amazon EKS 为 Kubernetes 集群æ供托管的 controlplane。Amazon EKS 跨多个å¯ç”¨åŒºè¿è¡Œ Kubernetes controlplane 实例，以确ä¿é«˜å¯ç”¨æ€§ã€‚Rancher æ供了一个直观的用户界é¢ï¼Œç”¨äºŽç®¡ç†å’Œéƒ¨ç½²ä½ è¿è¡Œåœ¨ Amazon EKS 中的 Kubernetes 集群。通过本指å—，你将使用 Rancher 在你的 AWS 账户中快速轻æ¾åœ°å¯åŠ¨ Amazon EKS Kubernetes 集群。有关 Amazon EKS 的更多信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html)。 + + +## Amazon Web æœåŠ¡çš„先决æ¡ä»¶ + +:::caution + +部署到 Amazon AWS 会产生费用。有关详细信æ¯ï¼Œè¯·å‚阅 [EKS 定价页é¢](https://aws.amazon.com/eks/pricing/)。 + +::: + +è¦åœ¨ EKS 上设置集群，你需è¦è®¾ç½® Amazon VPC（虚拟ç§æœ‰äº‘）。你还需è¦ç¡®ä¿ç”¨äºŽåˆ›å»º EKS 集群的账å·å…·æœ‰é€‚当的[æƒé™](#最å°-eks-æƒé™)。详情请å‚阅 [Amazon EKS 先决æ¡ä»¶å®˜æ–¹æŒ‡å—](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-console.html#eks-prereqs)。 + +### Amazon VPC + +你需è¦å»ºç«‹ä¸€ä¸ª Amazon VPC æ¥å¯åŠ¨ EKS 集群。VPC 使你能够将 AWS 资æºå¯åŠ¨åˆ°ä½ å®šä¹‰çš„虚拟网络中。你å¯ä»¥è‡ªå·±è®¾ç½®ä¸€ä¸ª VPC，并在 Rancher 中创建集群时æ供它。如果你创建过程中没有æ供，Rancher 将创建一个 VPC。详情请å‚阅[教程：为你的 Amazon EKS 集群创建具有公有和ç§æœ‰å­ç½‘çš„ VPC](https://docs.aws.amazon.com/eks/latest/userguide/create-public-private-vpc.html)。 + +### IAM ç­–ç•¥ + +Rancher 需è¦è®¿é—®ä½ çš„ AWS 账户æ‰èƒ½åœ¨ Amazon EKS 中预置和管ç†ä½ çš„ Kubernetes 集群。你需è¦åœ¨ AWS 账户中为 Rancher 创建一个用户，并定义该用户å¯ä»¥è®¿é—®çš„内容。 + +1. 按照[此处](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html)的步骤创建具有编程访问æƒé™çš„用户。 + +2. 创建一个 IAM 策略，定义该用户在 AWS 账户中有æƒè®¿é—®çš„内容。请务必仅授予此用户所需的最å°è®¿é—®æƒé™ã€‚[此处](#最å°-eks-æƒé™)列出了 EKS 集群所需的最低æƒé™ã€‚请按照[此处](https://docs.aws.amazon.com/eks/latest/userguide/EKS_IAM_user_policies.html)的步骤创建 IAM 策略并将策略绑定到你的用户。 + +3. 最åŽï¼ŒæŒ‰ç…§[此处](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)的步骤为该用户创建访问密钥和密文密钥。 + +:::note é‡è¦æ示： + +定期轮æ¢è®¿é—®å¯†é’¥å’Œå¯†æ–‡å¯†é’¥éžå¸¸é‡è¦ã€‚有关详细信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#rotating_access_keys_console)。 + +::: + +有关 EKS çš„ IAM 策略的更多详细信æ¯ï¼Œè¯·å‚阅 [Amazon EKS IAM ç­–ç•¥ã€è§’色和æƒé™çš„官方文档](https://docs.aws.amazon.com/eks/latest/userguide/IAM_policies.html)。 + + +## 创建 EKS 集群 + +使用 Rancher é…置你的 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 选择 **Amazon EKS**。 +1. 输入**集群å称**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 完æˆè¡¨å•çš„其余部分。如需帮助，请å‚阅[é…ç½®å‚考](#eks-集群é…ç½®å‚考)。 +1. å•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## EKS 集群é…ç½®å‚考 + +有关 EKS 集群é…置选项的完整列表，请å‚阅[此页é¢](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md)。 + +## 架构 + +下图展示了 Rancher 2.x 的上层架构。下图中，Rancher Server 管ç†ä¸¤ä¸ª Kubernetes 集群，其中一个由 RKE 创建，å¦ä¸€ä¸ªç”± EKS 创建。 + +
通过 Rancher 的认è¯ä»£ç†ç®¡ç† Kubernetes 集群
+ +![架构](/img/rancher-architecture-rancher-api-server.svg) + +## AWS æœåŠ¡äº‹ä»¶ + +有关 AWS æœåŠ¡äº‹ä»¶çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](https://status.aws.amazon.com/)。 + +## 安全与åˆè§„ + +默认情况下，åªæœ‰åˆ›å»ºé›†ç¾¤çš„ IAM 用户或角色æ‰èƒ½è®¿é—®è¯¥é›†ç¾¤ã€‚在没有é¢å¤–é…置的情况下，使用其他用户或角色访问集群将导致错误。在 Rancher 中，这æ„味ç€ä½¿ç”¨æ˜ å°„到未用于创建集群的用户或角色的凭è¯ï¼Œå¯¼è‡´æœªç»æŽˆæƒçš„错误。除éžç”¨äºŽæ³¨å†Œé›†ç¾¤çš„凭è¯ä¸Ž EKSCtl 使用的角色或用户匹é…，å¦åˆ™ EKSCtl 集群将ä¸ä¼šæ³¨å†Œåˆ° Rancher。通过将其他用户和角色添加到 kube-system 命å空间中的 aws-auth configmap，å¯ä»¥æŽˆæƒå…¶ä»–用户和角色访问集群。如需更深入的解释和详细说明，请å‚阅此[文档](https://aws.amazon.com/premiumsupport/knowledge-center/amazon-eks-cluster-access/)。 + +有关 Amazon EKS Kubernetes 集群的安全性和åˆè§„性的更多信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/eks/latest/userguide/shared-responsibilty.html)。 + +## 教程 + +AWS å¼€æºåšå®¢ä¸Šçš„这篇[教程](https://aws.amazon.com/blogs/opensource/managing-eks-clusters-rancher/)将指导你使用 Rancher 设置一个 EKS 集群，部署一个å¯å…¬å¼€è®¿é—®çš„示例应用æ¥æµ‹è¯•é›†ç¾¤ï¼Œå¹¶éƒ¨ç½²ä¸€ä¸ªä½¿ç”¨å…¶ä»–å¼€æºè½¯ä»¶ï¼ˆå¦‚ Grafana å’Œ influxdb）æ¥å®žæ—¶ç›‘控地ç†ä¿¡æ¯çš„示例项目。 + +## æœ€å° EKS æƒé™ + +这些是访问 Rancher EKS 驱动程åºçš„全部功能所需的最低æƒé™é›†ã€‚ä½ éœ€è¦ Rancher 的其他æƒé™æ‰èƒ½é…ç½® `Service Role` å’Œ `VPC` 资æºã€‚如果你在创建集群**之å‰**创建了这些资æºï¼Œä½ åœ¨é…置集群时将å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºã€‚ + +| èµ„æº | æè¿° | +---------|------------ +| æœåŠ¡è§’色 | æä¾›å…许 Kubernetes 代表你管ç†èµ„æºçš„æƒé™ã€‚Rancher å¯ä»¥ä½¿ç”¨ä»¥ä¸‹[æœåŠ¡è§’色æƒé™](#æœåŠ¡è§’色æƒé™)æ¥åˆ›å»ºæœåŠ¡è§’色。 | +| VPC | æä¾› EKS å’Œ Worker 节点使用的隔离网络资æºã€‚Rancher 使用以下 [VPC æƒé™](#vpc-æƒé™)创建 VPC 资æºã€‚ | + + +资æºå®šä½ä½¿ç”¨ `*` 作为在 Rancher 中创建 EKS 集群之å‰ï¼Œæ— æ³•å·²çŸ¥åˆ›å»ºçš„资æºçš„å称（ARN）。 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "EC2Permissions", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:RevokeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:DescribeInstanceTypes", + "ec2:DescribeRegions", + "ec2:DescribeVpcs", + "ec2:DescribeTags", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeRouteTables", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeKeyPairs", + "ec2:DescribeInternetGateways", + "ec2:DescribeImages", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeAccountAttributes", + "ec2:DeleteTags", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteSecurityGroup", + "ec2:DeleteKeyPair", + "ec2:CreateTags", + "ec2:CreateSecurityGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateLaunchTemplate", + "ec2:CreateKeyPair", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress" + ], + "Resource": "*" + }, + { + "Sid": "CloudFormationPermissions", + "Effect": "Allow", + "Action": [ + "cloudformation:ListStacks", + "cloudformation:ListStackResources", + "cloudformation:DescribeStacks", + "cloudformation:DescribeStackResources", + "cloudformation:DescribeStackResource", + "cloudformation:DeleteStack", + "cloudformation:CreateStackSet", + "cloudformation:CreateStack" + ], + "Resource": "*" + }, + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:ListInstanceProfilesForRole", + "iam:ListInstanceProfiles", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "iam:GetInstanceProfile", + "iam:DetachRolePolicy", + "iam:DeleteRole", + "iam:CreateRole", + "iam:AttachRolePolicy" + ], + "Resource": "*" + }, + { + "Sid": "KMSPermissions", + "Effect": "Allow", + "Action": "kms:ListKeys", + "Resource": "*" + }, + { + "Sid": "EKSPermissions", + "Effect": "Allow", + "Action": [ + "eks:UpdateNodegroupVersion", + "eks:UpdateNodegroupConfig", + "eks:UpdateClusterVersion", + "eks:UpdateClusterConfig", + "eks:UntagResource", + "eks:TagResource", + "eks:ListUpdates", + "eks:ListTagsForResource", + "eks:ListNodegroups", + "eks:ListFargateProfiles", + "eks:ListClusters", + "eks:DescribeUpdate", + "eks:DescribeNodegroup", + "eks:DescribeFargateProfile", + "eks:DescribeCluster", + "eks:DeleteNodegroup", + "eks:DeleteFargateProfile", + "eks:DeleteCluster", + "eks:CreateNodegroup", + "eks:CreateFargateProfile", + "eks:CreateCluster" + ], + "Resource": "*" + } + ] +} +``` + +### æœåŠ¡è§’色æƒé™ + +这些是 EK​​S 集群创建期间所需的æƒé™ï¼Œä»¥ä¾¿ Rancher å¯ä»¥ä»£è¡¨ç”¨æˆ·åˆ›å»ºæœåŠ¡è§’色。 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreateRole", + "iam:CreateServiceLinkedRole", + "iam:DeleteInstanceProfile", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfiles", + "iam:ListInstanceProfilesForRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile" + ], + "Resource": "*" + } + ] +} +``` + +当你创建 EKS 集群时，Rancher 会创建一个具有以下信任策略的æœåŠ¡è§’色： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "eks.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} +``` + +此角色还有两个角色策略，它们具有以下策略的 ARN： + +``` +arn:aws:iam::aws:policy/AmazonEKSClusterPolicy +arn:aws:iam::aws:policy/AmazonEKSServicePolicy +``` + +### VPC æƒé™ + +这些是 Rancher 创建虚拟ç§æœ‰äº‘ (VPC) 和相关资æºæ‰€éœ€çš„æƒé™ã€‚ + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VPCPermissions", + "Effect": "Allow", + "Action": [ + "ec2:ReplaceRoute", + "ec2:ModifyVpcAttribute", + "ec2:ModifySubnetAttribute", + "ec2:DisassociateRouteTable", + "ec2:DetachInternetGateway", + "ec2:DescribeVpcs", + "ec2:DeleteVpc", + "ec2:DeleteTags", + "ec2:DeleteSubnet", + "ec2:DeleteRouteTable", + "ec2:DeleteRoute", + "ec2:DeleteInternetGateway", + "ec2:CreateVpc", + "ec2:CreateSubnet", + "ec2:CreateSecurityGroup", + "ec2:CreateRouteTable", + "ec2:CreateRoute", + "ec2:CreateInternetGateway", + "ec2:AttachInternetGateway", + "ec2:AssociateRouteTable" + ], + "Resource": "*" + } + ] +} +``` + +## åŒæ­¥ + +EKS é…置者å¯ä»¥åœ¨ Rancher å’Œæ供商之间åŒæ­¥ EKS 集群的状æ€ã€‚有关其工作原ç†çš„技术说明，请å‚阅[åŒæ­¥](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md)。 + +有关é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[本节](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md#é…置刷新间隔)。 + +## 故障排除 + +如果你的更改被覆盖，å¯èƒ½æ˜¯é›†ç¾¤æ•°æ®ä¸Ž EKS åŒæ­¥çš„æ–¹å¼å¯¼è‡´çš„。ä¸è¦åœ¨ä½¿ç”¨å…¶ä»–æºï¼ˆä¾‹å¦‚ EKS 控制å°ï¼‰å¯¹é›†ç¾¤è¿›è¡Œæ›´æ”¹åŽï¼Œåˆåœ¨äº”分钟之内在 Rancher 中进行更改。有关其工作原ç†ï¼Œä»¥åŠå¦‚何é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[åŒæ­¥](#åŒæ­¥)。 + +如果在修改或注册集群时返回未ç»æŽˆæƒçš„错误，并且集群ä¸æ˜¯ä½¿ç”¨ä½ çš„凭è¯æ‰€å±žçš„角色或用户创建的，请å‚阅[安全与åˆè§„](#安全与åˆè§„)。 + +有关 Amazon EKS Kubernetes 集群的任何问题或故障排除详细信æ¯ï¼Œè¯·å‚阅此[文档](https://docs.aws.amazon.com/eks/latest/userguide/troubleshooting.html)。 + +## 以编程方å¼åˆ›å»º EKS 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² EKS 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md new file mode 100644 index 00000000000..ee3f3397dc6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md @@ -0,0 +1,101 @@ +--- +title: 创建 GKE 集群 +--- + +## 先决æ¡ä»¶ + +你需è¦åœ¨ Google Kubernetes Engine 中进行一些设置。 + +### ServiceAccount 令牌 + +使用 [Google Kubernetes Engine](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts) 创建æœåŠ¡å¸å·ã€‚GKE 使用这个账å·æ¥æ“作你的集群。创建此账å·è¿˜ä¼šç”Ÿæˆç”¨äºŽèº«ä»½éªŒè¯çš„ç§é’¥ã€‚ + +该 ServiceAccount 需è¦ä»¥ä¸‹è§’色： + +- **Compute Viewer:** `roles/compute.viewer` +- **Project Viewer:** `roles/viewer` +- **Kubernetes Engine Admin:** `roles/container.admin` +- **Service Account User:** `roles/iam.serviceAccountUser` + +[Google 文档：创建和å¯ç”¨ ServiceAccount](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) + +æœ‰å…³èŽ·å– ServiceAccount çš„ç§é’¥çš„详细信æ¯ï¼Œè¯·å‚阅[æ­¤ Google 云文档](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys)。你需è¦å°†å¯†é’¥ä¿å­˜ä¸º JSON æ ¼å¼ã€‚ + +### Google 项目 ID + +你的集群需è¦åŒ…括到谷歌项目中。 + +è¦åˆ›å»ºæ–°é¡¹ç›®ï¼Œè¯·å‚阅[æ­¤ Google 云文档](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project)。 + +è¦èŽ·å–现有项目的项目 ID，请å‚阅[æ­¤ Google 云文档](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects)。 + +## é…ç½® GKE 集群 + +:::caution + +部署到 GKE 会产生费用。 + +::: + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. 输入你 Google 云凭è¯çš„å称。 +1. 在 **ServiceAccount** 文本框中，粘贴你的 ServiceAccount ç§é’¥ JSON，或上传 JSON 文件。 +1. å•å‡»**创建**。 + +**结果**：你已创建 Rancher 用于é…置新 GKE 集群的凭è¯ã€‚ + +### 2. 创建 GKE 集群 +使用 Rancher é…置你的 Kubernetes 集群。è¦ä½¿ç”¨ Rancher æˆåŠŸåˆ›å»º GKE 集群，GKE 必须处于 Standard 模å¼ã€‚GKE 在创建 Kubernetes 集群时有两ç§è¿è¡Œæ¨¡å¼ï¼Œåˆ†åˆ«æ˜¯ Autopilot å’Œ Standard 模å¼ã€‚Autopilot 模å¼çš„集群é…置对编辑 kube-system 命å空间有é™åˆ¶ã€‚但是，Rancher 在安装时需è¦åœ¨ kube-system 命å空间中创建资æºã€‚因此，你将无法创建 Autopilot 模å¼çš„ GKE 集群。如需详细了解 GKE Autopilot 模å¼å’Œ Standard 模å¼ä¹‹é—´çš„差异，请访问[比较 GKE Autopilot å’Œ Standard ](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison)。 + + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Google GKE**。 +1. 输入**集群å称**。 +1. å¯é€‰ï¼šä½¿ç”¨**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å¯é€‰ï¼šå°† Kubernetes [标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)或[注释](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)添加到集群。 +1. 输入你的 Google 项目 ID å’Œ Google 云凭è¯ã€‚ +1. 完æˆè¡¨å•çš„其余部分。如需帮助，请å‚阅 [GKE 集群é…ç½®å‚考](../../../../pages-for-subheaders/gke-cluster-configuration.md)。 +1. å•å‡»**创建**。 + +**结果**：你已æˆåŠŸéƒ¨ç½² GKE 集群。 + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## ç§æœ‰é›†ç¾¤ + +支æŒç§æœ‰ GKE 集群。注æ„，此高级设置å¯èƒ½æ¶‰åŠæ›´å¤šæ­¥éª¤ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../../../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md)。 + +## é…ç½®å‚考 + +有关在 Rancher 中é…ç½® GKE 集群的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../../pages-for-subheaders/gke-cluster-configuration.md)。 +## æ›´æ–° Kubernetes 版本 + +集群的 Kubernetes 版本å¯ä»¥å‡çº§åˆ° GKE 集群所在区域或地区中å¯ç”¨çš„任何版本。å‡çº§ Kubernetes 主版本ä¸ä¼šè‡ªåŠ¨å‡çº§ Worker 节点。节点å¯ä»¥ç‹¬ç«‹å‡çº§ã€‚ + +:::note + +GKE 在 1.19+ 中å–消了基本身份验è¯ã€‚è¦å°†é›†ç¾¤å‡çº§åˆ° 1.19+，必须在谷歌云中ç¦ç”¨åŸºæœ¬èº«ä»½è®¤è¯ã€‚å¦åˆ™ï¼Œåœ¨å‡çº§åˆ° 1.19+ 时，Rancher 会出现错误。你å¯ä»¥æŒ‰ç…§ [Google 文档](https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#disabling_authentication_with_a_static_password)进行æ“作。然åŽï¼Œä½ å°±å¯ä»¥ä½¿ç”¨ Rancher å°† Kubernetes 版本更新到 1.19+。 + +::: + +## åŒæ­¥ + +GKE é…置者å¯ä»¥åœ¨ Rancher å’Œæ供商之间åŒæ­¥ GKE 集群的状æ€ã€‚有关其工作原ç†çš„技术说明，请å‚阅[åŒæ­¥](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md)。 + +有关é…置刷新间隔的信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/gke-cluster-configuration.md#é…置刷新间隔)。 + +## 以编程方å¼åˆ›å»º GKE 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² GKE 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md new file mode 100644 index 00000000000..b6244c617d6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md @@ -0,0 +1,100 @@ +--- +title: 创建åŽä¸º CCE 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在åŽä¸ºäº‘容器引擎 (CCE) 中的集群。Rancher å·²ç»ä¸º CCE 实现并打包了针对 CCE çš„[集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md)，但是默认情况下，这个集群驱动的状æ€æ˜¯ `inactive`。为了å¯åŠ¨ CCE 集群，你需è¦[å¯ç”¨ CCE 集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#激活åœç”¨é›†ç¾¤é©±åŠ¨)。å¯ç”¨é›†ç¾¤é©±åŠ¨åŽï¼Œä½ å¯ä»¥å¼€å§‹é…ç½® CCE 集群。 + +## åŽä¸ºå…ˆå†³æ¡ä»¶ + +:::caution + +部署到 CCE 会产生费用。 + +::: + +1. 在åŽä¸º CCE 门户中找到你的项目 ID。请å‚阅 CCE 文档以了解如何[管ç†ä½ çš„项目](https://doc.hcs.huawei.com/api/cce/cce_02_0341.html#cce_02_0341__section17571944123214)。 + +2. 创建一个[访问密钥 ID 和密文访问密钥](https://support.huaweicloud.com/en-us/usermanual-iam/en-us_topic_0079477318.html)。 + +## Rancher 先决æ¡ä»¶ + +你需è¦å¯ç”¨åŽä¸º CCE 集群驱动： + +1. 点击 **☰ > 集群管ç†**。 +1. 点击**驱动**。 +1. 在**集群驱动**选项å¡ä¸­ï¼Œè½¬åˆ° **Huawei CCE** 集群驱动并å•å‡» **â‹® > 激活**。 + +集群驱动下载完æˆåŽï¼Œå°±å¯ä»¥åœ¨ Rancher 中创建åŽä¸º CCE 集群了。 + +## é™åˆ¶ + +åŽä¸º CCE æœåŠ¡ä¸æ”¯æŒé€šè¿‡å…¶ API 创建具有公共访问æƒé™çš„集群。你需è¦åœ¨è¦é…置的 CCE é›†ç¾¤çš„ç›¸åŒ VPC 中è¿è¡Œ Rancher。 + +## 创建 CCE 集群 + +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. å•å‡» **Huawei CCE**。 +1. 输入**集群å称**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 输入**项目 ID**，访问密钥 ID，**Access Key**，和密文访问密钥 **Secret Key**。然åŽç‚¹å‡»**下一步：é…置集群**。填写集群é…置。有关填写表å•çš„帮助，请å‚阅[åŽä¸º CCE é…ç½®](#åŽä¸º-cce-é…ç½®)。 +1. 填写集群的以下节点é…置。有关填写表å•çš„帮助，请å‚阅[节点é…ç½®](#节点é…ç½®)。 +1. 点击**创建**æ¥åˆ›å»º CCE 集群。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## åŽä¸º CCE é…ç½® + +| 设置 | æè¿° | +|---|---| +| 集群类型 | è¦åŒ…å«åˆ°é›†ç¾¤çš„节点类型，å¯ä»¥æ˜¯ `VirtualMachine` 或 `BareMetal`。 | +| æè¿° | 集群的æ述。 | +| 主版本 | Kubernetes 版本。 | +| 管ç†è§„æ¨¡æ•°é‡ | 集群的最大节点数。选项为 50ã€200 å’Œ 1000。规模计数越大，æˆæœ¬è¶Šé«˜ã€‚ | +| 高å¯ç”¨æ€§ | å¯ç”¨ master 节点的高å¯ç”¨æ€§ã€‚å¯ç”¨é«˜å¯ç”¨æ€§çš„集群æˆæœ¬ä¼šæ›´é«˜ã€‚ | +| å®¹å™¨ç½‘ç»œæ¨¡å¼ | 集群中使用的网络模å¼ã€‚`VirtualMachine` æ”¯æŒ `overlay_l2` å’Œ `vpc-router`，而 `BareMetal` æ”¯æŒ `underlay_ipvlan`。 | +| 容器网络 CIDR | 集群的网络 CIDR。 | +| VPC å称 | è¦éƒ¨ç½²é›†ç¾¤çš„ VPC å称。如果留空，Rancher 将创建一个。 | +| å­ç½‘å称 | è¦éƒ¨ç½²é›†ç¾¤çš„å­ç½‘å称。如果留空，Rancher 将创建一个。 | +| 外部æœåŠ¡å™¨ | 预留选项，用于通过 API å¯ç”¨ CCE 集群的公共访问。这个选项暂时是一直ç¦ç”¨çš„。 | +| 集群标签 | 集群的标签。 | +| 高速å­ç½‘ | åªæœ‰ `BareMetal` 支æŒè¯¥é€‰é¡¹ã€‚裸机è¦æ±‚选择网速高的 VPC。 | + +:::note + +如果你在 `cluster.yml` 中编辑集群，而ä¸æ˜¯ä½¿ç”¨ Rancher UI，则集群é…ç½®å‚数必须嵌套在 `cluster.yml` 中的 `rancher_kubernetes_engine_config` 中。有关详细信æ¯ï¼Œè¯·å‚阅[é…置文件结构](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rancher-中的é…置文件结构)。 + +::: + +## 节点é…ç½® + +| 设置 | æè¿° | +|---|---| +| 地区 | 部署集群节点的å¯ç”¨åŒºåŸŸã€‚ | +| è®¡è´¹æ–¹å¼ | 集群节点的计费模å¼ã€‚`VirtualMachine` ä»…æ”¯æŒ `Pay-per-use`。`BareMetal` æ”¯æŒ `Pay-per-use` 或 `Yearly/Monthly`。 | +| 有效期 | 该选项仅在 `Yearly/Monthly` 计费模å¼ä¸­æ˜¾ç¤ºã€‚表示支付集群节点费用的时间。 | +| 自动续期 | 该选项仅在 `Yearly/Monthly` 计费模å¼ä¸­æ˜¾ç¤ºã€‚表示是å¦ä¸ºé›†ç¾¤èŠ‚点自动续期 `Yearly/Monthly` 计费。 | +| æ•°æ®å·ç±»åž‹ | 集群节点的数æ®å·ç±»åž‹ã€‚å¯é€‰ `SATA`，`SSD` 或 `SAS`。 | +| æ•°æ®é‡å¤§å° | 集群节点的数æ®å·å¤§å°ã€‚ | +| æ ¹å·ç±»åž‹ | 集群节点的根å·ç±»åž‹ã€‚å¯é€‰ `SATA`，`SSD` 或 `SAS`。 | +| æ ¹å·å¤§å° | 集群节点的根å·å¤§å°ã€‚ | +| 节点风格 | 集群节点的节点风格。Rancher UI 中的风格列表å–自åŽä¸ºäº‘。其中包括所有支æŒçš„节点风格。 | +| 节点数 | 集群的节点数 | +| 节点æ“作系统 | 集群节点的æ“作系统。目å‰ä»…æ”¯æŒ `EulerOS 2.2` å’Œ `CentOS 7.4`。 | +| SSH 密钥å称 | 集群节点的 SSH 密钥 | +| EIP | 集群节点的公共 IP 选项。`å·²ç¦ç”¨`表示集群节点ä¸ä¼šç»‘定公共 IP。`创建 EIP`表示集群节点在é…ç½®åŽå°†ç»‘定一个或多个新创建的 EIP，UI 中将显示更多用æ¥åˆ›å»º EIP å‚数的选项。`选择现有 EIP` 表示节点将绑定到你选择的 EIP。 | +| EIP æ•°é‡ | 此选项仅在选择`创建 EIP`时显示。表示你è¦ä¸ºèŠ‚点创建的 EIP æ•°é‡ã€‚ | +| EIP 类型 | 此选项仅在选择`创建 EIP`时显示。å¯é€‰ `5_bgp` å’Œ `5_sbgp`。 | +| EIP 共享类型 | 此选项仅在选择`创建 EIP`时显示。仅å¯é€‰ `PER`。 | +| EIP æ”¶è´¹æ¨¡å¼ | 此选项仅在选择`创建 EIP`时显示。选择按照`带宽`或`æµé‡`计费。 | +| EIP å¸¦å®½å¤§å° | 此选项仅在选择`创建 EIP`时显示。EIP 的带宽。 | +| 身份验è¯æ¨¡å¼ | 表示å¯ç”¨ `RBAC`，或åŒæ—¶å¯ç”¨`认è¯ä»£ç†`。如果选择`认è¯ä»£ç†`，则还需è¦ç”¨äºŽéªŒè¯ä»£ç†çš„è¯ä¹¦ã€‚ | +| 节点标签 | 集群节点的标签。无效标签会阻止å‡çº§ï¼Œæˆ–阻止 Rancher å¯åŠ¨ã€‚有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md new file mode 100644 index 00000000000..13349de2c08 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md @@ -0,0 +1,99 @@ +--- +title: 创建腾讯 TKE 集群 +--- + +ä½ å¯ä»¥ä½¿ç”¨ Rancher 创建托管在腾讯 Tencent Kubernetes Engine (TKE) 中的集群。Rancher å·²ç»ä¸º TKE 实现并打包了针对 TKE çš„[集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md)，但是默认情况下，这个集群驱动的状æ€æ˜¯ `inactive`。为了å¯åŠ¨ TKE 集群，你需è¦[å¯ç”¨ TKE 集群驱动](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#激活åœç”¨é›†ç¾¤é©±åŠ¨)。å¯ç”¨é›†ç¾¤é©±åŠ¨åŽï¼Œä½ å¯ä»¥å¼€å§‹é…ç½® TKE 集群。 + +## 腾讯先决æ¡ä»¶ + +:::caution + +部署到 TKE 会产生费用。 + +::: + +1. å‚è§[云访问管ç†](https://intl.cloud.tencent.com/document/product/598/10600)文档，确ä¿ç”¨äºŽåˆ›å»º TKE 集群的账å·å…·æœ‰é€‚当的æƒé™ã€‚ + +2. 创建[云 API 密钥 ID 和密钥](https://console.cloud.tencent.com/capi)。 + +3. 在è¦éƒ¨ç½² Kubernetes 集群的区域中创建[ç§æœ‰ç½‘络和å­ç½‘](https://intl.cloud.tencent.com/document/product/215/4927)。 + +4. 创建 [SSH 密钥对](https://intl.cloud.tencent.com/document/product/213/6092)。该密钥用于访问 Kubernetes 集群中的节点。 + +## Rancher 先决æ¡ä»¶ + +你需è¦å¯ç”¨è…¾è®¯ TKE 集群驱动： + +1. 点击 **☰ > 集群管ç†**。 +1. 点击**驱动**。 +1. 在**集群驱动**选项å¡ä¸­ï¼Œè½¬åˆ° **Tencent TKE** 集群驱动并å•å‡» **â‹® > 激活**。 + +集群驱动下载完æˆåŽï¼Œå°±å¯ä»¥åœ¨ Rancher 中创建腾讯 TKE 集群了。 + +## 创建 TKE 集群 + +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 + +2. 选择 **Tencent TKE**。 + +3. 输入**集群å称**。 + +4. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ + +5. 为 TKE 集群é…ç½®**è´¦å·è®¿é—®**。使用[先决æ¡ä»¶](#腾讯先决æ¡ä»¶)中获得的信æ¯å®Œæˆæ¯ä¸ªä¸‹æ‹‰åˆ—表和字段。 + + | 选项 | æè¿° | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | 区域 | 从下拉列表中选择构建集群的地ç†åŒºåŸŸã€‚ | + | 密文 ID | 输入从腾讯云控制å°èŽ·å–的密文 ID。 | + | 密文密钥 | 输入从腾讯云控制å°èŽ·å–的密文密钥。 | + +6. 然åŽï¼Œå•å‡»`下一步：é…置集群`æ¥é…ç½® TKE 集群。 + + | 选项 | æè¿° | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Kubernetes 版本 | TKE ç›®å‰åªæ”¯æŒ Kubernetes 1.10.5。 | + | 节点数 | 输入è¦ä¸º Kubernetes 集群购买的 worker 节点数，最大值是 100。 | + | VPC | 选择在腾讯云控制å°ä¸­åˆ›å»ºçš„ VPC å称。 | + | 容器网络 CIDR | 输入 Kubernetes 集群的 CIDR 范围。你å¯ä»¥åœ¨è…¾è®¯äº‘控制å°çš„ VPC æœåŠ¡ä¸­æŸ¥çœ‹è¯¥ CIDR çš„å¯ç”¨èŒƒå›´ã€‚默认为 172.16.0.0/16。 | + + :::note + + 如果你在 `cluster.yml` 中编辑集群，而ä¸æ˜¯ä½¿ç”¨ Rancher UI，则集群é…ç½®å‚数必须嵌套在 `cluster.yml` 中的 `rancher_kubernetes_engine_config` 中。有关详细信æ¯ï¼Œè¯·å‚阅 [Rancher 2.3.0+ é…置文件结构](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rancher-中的é…置文件结构)。 + + ::: + +7. 点击`下一步：选择实例类型`，然åŽé€‰æ‹©å°†ç”¨äºŽ TKE 集群的实例类型。 + + | 选项 | æè¿° | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | å¯ç”¨åŒº | 选择 VPC 区域的å¯ç”¨åŒºã€‚ | + | å­ç½‘ | 选择你在 VPC 中创建的å­ç½‘。如果在所选å¯ç”¨åŒºä¸­æ²¡æœ‰è¯¥å­ç½‘，请添加一个新å­ç½‘。 | + | 实例类型 | 从下拉列表中选择è¦ç”¨äºŽ TKE 集群的 VM 实例类型，默认为 S2.MEDIUM4（CPU 2；内存 4 GiB）。 | + +8. 点击`下一步：é…置实例`，é…置用于 TKE 集群的 VM 实例。 + + | 选项 | æè¿° | + -------|------------ + | æ“作系统 | æ“作系统å称，目å‰æ”¯æŒ Centos7.2x86_64 或 ubuntu16.04.1 LTSx86_64。 | + | 安全组 | 安全组 ID，默认ä¸ç»‘定任何安全组。 | + | æ ¹ç£ç›˜ç±»åž‹ | 系统盘类型。系统盘类型é™åˆ¶è¯¦è§ [CVM 实例é…ç½®](https://cloud.tencent.com/document/product/213/11518)。 | + | æ ¹ç£ç›˜å¤§å° | 系统盘大å°ã€‚Linux 系统调整范围为 20-50 GB，步长为 1。 | + | æ•°æ®ç›˜ç±»åž‹ | æ•°æ®ç›˜ç±»åž‹ï¼Œé»˜è®¤ä¸º SSD 云盘。 | + | æ•°æ®ç£ç›˜å¤§å° | æ•°æ®ç›˜å¤§å°ï¼ˆGB），步长为 10。 | + | 带宽类型 | 带宽收费类型，PayByTraffic 或 PayByHour。 | + | 带宽 | 公网带宽 (Mbps) | + | 密钥对 | 密钥 ID，关è”åŽå¯ä»¥ç”¨æ¥ç™»å½• VM 节点。 | + +9. å•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md new file mode 100644 index 00000000000..3003cab8733 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md @@ -0,0 +1,41 @@ +--- +title: Azure 中存储类的é…ç½® +--- + +如果你的节点使用 Azure VM，则å¯ä»¥ä½¿ç”¨ [Azure 文件](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv)作为集群的存储类（StorageClass）。 + +为了让 Azure 创建所需的存储资æºï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤æ“作： + +1. [é…ç½® Azure 云æ供商](../../kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md)。 +1. é…ç½® `kubectl` 以连接到你的集群。 +1. å¤åˆ¶ ServiceAccount çš„ `ClusterRole` å’Œ `ClusterRoleBinding` 清å•ï¼š + ```yml + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:azure-cloud-provider + rules: + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:azure-cloud-provider + roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider + subjects: + - kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system + ``` + +1. 使用以下命令在集群中进行创建： + + ``` + # kubectl create -f + ``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md new file mode 100644 index 00000000000..3a5ac26045c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md @@ -0,0 +1,36 @@ +--- +title: Host Gateway (L2bridge) 的网络è¦æ±‚ +--- + +本节介ç»å¦‚何é…置使用 *Host Gateway (L2bridge)* 模å¼çš„自定义 Windows 集群。 + +### ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥ + +如果你使用 *Host Gateway (L2bridge)* 模å¼ï¼Œå¹¶å°†èŠ‚点托管在下é¢åˆ—出的云æœåŠ¡ä¸Šï¼Œåˆ™å¿…须在å¯åŠ¨æ—¶ç¦ç”¨ Linux 或 Windows 主机的ç§æœ‰ IP 地å€æ£€æŸ¥ã€‚è¦ä¸ºæ¯ä¸ªèŠ‚点ç¦ç”¨æ­¤æ£€æŸ¥ï¼Œè¯·æŒ‰ç…§ä»¥ä¸‹å„个云æœåŠ¡å¯¹åº”的说明进行æ“作： + +| æœåŠ¡ | ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥çš„说明 | +--------|------------------------------------------------ +| Amazon EC2 | [ç¦ç”¨æº/目标检查](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_NAT_Instance.html#EIP_Disable_SrcDestCheck) | +| Google GCE | [为实例å¯ç”¨ IP 转å‘](https://cloud.google.com/vpc/docs/using-routes#canipforward)（默认情况下，VM 无法转å‘ç”±å¦ä¸€ä¸ª VM å‘èµ·çš„æ•°æ®åŒ…） | +| Azure VM | [å¯ç”¨æˆ–ç¦ç”¨ IP 转å‘](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) | + +### 云托管虚拟机的路由é…ç½® + +如果是使用 Flannel çš„ [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) åŽç«¯ï¼ŒåŒä¸€èŠ‚点上的所有容器都属于ç§æœ‰å­ç½‘，æµé‡é€šè¿‡ä¸»æœºç½‘络从一个节点上的å­ç½‘路由到在å¦ä¸€ä¸ªèŠ‚点上的å­ç½‘。 + +- 在 AWSã€è™šæ‹ŸåŒ–集群或裸机æœåŠ¡å™¨ä¸Šé…ç½® worker 节点时，请确ä¿å®ƒä»¬å±žäºŽåŒä¸€ä¸ªç¬¬ 2 层å­ç½‘。如果节点ä¸å±žäºŽåŒä¸€ä¸ªç¬¬ 2 层å­ç½‘，`host-gw` 网络将ä¸èµ·ä½œç”¨ã€‚ + +- 在 GCE 或 Azure 上é…ç½® worker 节点时，节点ä¸åœ¨åŒä¸€ä¸ªç¬¬ 2 层å­ç½‘上。GCE å’Œ Azure 上的节点属于å¯è·¯ç”±çš„第 3 层网络。按照以下说明é…ç½® GCE å’Œ Azure，以便云网络知é“如何在æ¯ä¸ªèŠ‚点上传é€ä¸»æœºå­ç½‘。 + +è¦åœ¨ GCE 或 Azure 上é…置主机å­ç½‘路由，首先è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä»¥æ‰¾å‡ºæ¯ä¸ª worker 节点上的主机å­ç½‘： + +```bash +kubectl get nodes -o custom-columns=nodeName:.metadata.name,nodeIP:status.addresses[0].address,routeDestination:.spec.podCIDR +``` + +然åŽæŒ‰ç…§å„个云æ供商的说明，为æ¯ä¸ªèŠ‚点é…置路由规则： + +| æœåŠ¡ | 说明 | +--------|------------- +| Google GCE | 为æ¯ä¸ªèŠ‚点添加é™æ€è·¯ç”±ï¼š[添加é™æ€è·¯ç”±](https://cloud.google.com/vpc/docs/using-routes#addingroute)。 | +| Azure VM | 创建一个路由表：[自定义路由：用户定义](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview#user-defined)。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md new file mode 100644 index 00000000000..81d832577f2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md @@ -0,0 +1,42 @@ +--- +title: Windows å’Œ Linux 集群的功能奇å¶ä¸€è‡´æ€§ +--- + +Windows 集群与 Linux 集群的功能支æŒä¸ç›¸åŒã€‚ + +下图æ述了 Rancher 上 Windows å’Œ Linux 之间的功能奇å¶ä¸€è‡´æ€§ï¼š + +| **组件** | **Linux** | **Windows** | +--- | --- | --- +| **å‘行版** | | +| RKE | æ”¯æŒ | æ”¯æŒ | +| RKE2 | æ”¯æŒ | æ”¯æŒ | +| K3S | æ”¯æŒ | ä¸æ”¯æŒ | +| EKS | æ”¯æŒ | ä¸æ”¯æŒ | +| GKE | æ”¯æŒ | ä¸æ”¯æŒ | +| AKS | æ”¯æŒ | ä¸æ”¯æŒ | +| **Rancher 组件** | | +| Server | æ”¯æŒ | ä¸æ”¯æŒ | +| Agent | æ”¯æŒ | æ”¯æŒ | +| Fleet | æ”¯æŒ | æ”¯æŒ | +| EKS Operator | æ”¯æŒ | ä¸æ”¯æŒ | +| AKS Operator | ä¸æ”¯æŒ | ä¸æ”¯æŒ | +| GKE Operator | ä¸æ”¯æŒ | ä¸æ”¯æŒ | +| Monitoring/Alerting v2 | æ”¯æŒ | æ”¯æŒ | +| Logging v2 | æ”¯æŒ | æ”¯æŒ | +| Istio | æ”¯æŒ | ä¸æ”¯æŒ | +| Catalog v2 | æ”¯æŒ | ä¸æ”¯æŒ | +| OPA | æ”¯æŒ | ä¸æ”¯æŒ | +| Longhorn | æ”¯æŒ | ä¸æ”¯æŒ | +| CIS 扫æ | æ”¯æŒ | ä¸æ”¯æŒ | +| Backup/Restore Operator | æ”¯æŒ | ä¸æ”¯æŒ | +| **CNI/附加组件** | | +| Flannel | æ”¯æŒ | æ”¯æŒ | +| Canal | æ”¯æŒ | ä¸æ”¯æŒ | +| Calico | æ”¯æŒ | 支æŒï¼ˆä»… RKE2） | +| Cilium | æ”¯æŒ | ä¸æ”¯æŒ | +| Multus | æ”¯æŒ | ä¸æ”¯æŒ | +| Traefik | æ”¯æŒ | ä¸æ”¯æŒ | +| NGINX Ingress | æ”¯æŒ | ä¸æ”¯æŒ | + +有关功能支æŒçš„æ›´æ–°ä¿¡æ¯ï¼Œä½ å¯ä»¥è®¿é—® GitHub 上的 [rancher/windows](https://github.com/rancher/windows)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md new file mode 100644 index 00000000000..ce334bc5382 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md @@ -0,0 +1,303 @@ +--- +title: å°† Windows 工作负载从 RKE2 è¿ç§»åˆ° RKE2 +--- + +**注æ„**：Rancher Support çš„ SLA ä¸æ¶µç›–本文档的内容。因此，请谨慎继续。 + +本文档介ç»äº†å¦‚何将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2。 + +## RKE1 Windows 调度 + +RKE1 Windows 工作负载是基于污点和容å¿åº¦è°ƒåº¦çš„。 + +RKE1 Windows 集群中的æ¯ä¸ª Linux 节点（无论分é…了什么角色）都会有一个默认污点，除éžå·¥ä½œè´Ÿè½½é…置了容å¿åº¦ï¼Œå¦åˆ™è¯¥æ±¡ç‚¹ä¼šé˜»æ­¢å·¥ä½œè´Ÿè½½è°ƒåº¦åˆ°èŠ‚点上。这是 RKE1 Windows 集群的一个主è¦ç‰¹æ€§ï¼Œç›®çš„是为了仅è¿è¡Œ Windows 工作负载。 + +- 默认 RKE1 Linux 节点 `NoSchedule` 污点： + +```yml +apiVersion: v1 +kind: Node +spec: + ... + taints: + - effect: NoSchedule + key: cattle.io/os + value: linux +``` +
+ +- RKE1 Linux `NoSchedule` 对工作负载的容å¿åº¦ + +以下容å¿åº¦å…许将最终用户的工作负载调度到 RKE1 Windows 集群中的任何 Linux 节点上。这些容å¿åº¦ç”¨äºŽæ ¸å¿ƒçš„ Rancher æœåŠ¡å’Œå·¥ä½œè´Ÿè½½ï¼š + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` +
+ +- æ ¹æ®æœ€ä½³å®žè·µï¼Œåœ¨ Linux 节点上è¿è¡Œçš„任何最终用户工作负载都将仅调度到具有 worker 角色的节点上： + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + weight: 100 + ... +``` + +## RKE2 Windows 调度 + +æ ¹æ®å¯¹æ··åˆå·¥ä½œè´Ÿè½½æ”¯æŒçš„å馈和请求，RKE2 Windows é»˜è®¤æ”¯æŒ Linux å’Œ Windows 工作负载。RKE2 调度默认ä¾èµ–节点选择器。这是ä¸åŒäºŽ RKE1 的一个显ç€å˜åŒ–，因为 RKE2 中没有包å«æ±¡ç‚¹å’Œå®¹å¿ã€‚节点选择器是 RKE1 Windows 集群的关键部分，å¯ä»¥å¸®åŠ©ä½ è½»æ¾è¿ç§»å·¥ä½œè´Ÿè½½ã€‚ + +## 示例è¿ç§» + +### å°† Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 + +- è¿ç§»å‰çš„ RKE1 Windows Deployment： + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - linux +``` +
+ +- 使用 `nodeAffinity` è¿ç§»åŽçš„ RKE2 Windows Deployment： + +```yaml +apiVersion: apps/v1 +kind: Deployment +... +spec: + ... + template: + ... + spec: + ... + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows +``` + +### RKE1 Windows 集群的 Linux-Only Deployment + +> **é‡è¦æ示**：在利用 `nodeSelector` å’Œ `nodeAffinity` 时，请注æ„以下几点： +> +> - 如果åŒæ—¶æŒ‡å®šäº† `nodeSelector` å’Œ `nodeAffinity`，则必须满足这两个æ¡ä»¶æ‰èƒ½å°† `Pod` 调度到节点上。 +> - 如果指定了与å•ä¸ª `nodeSelectorTerms` å…³è”的多个 `matchExpressions`，则åªæœ‰å½“所有 `matchExpressions` 都符åˆè¦æ±‚æ—¶æ‰ä¼šè°ƒåº¦ `Pod`。 + +
+ +- è¿ç§»å‰çš„ RKE1 Windows 集群 Linux-only Deployment（针对 RKE1 Linux Worker 节点）： + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + +``` +
+ +- 使用 `nodeSelector` è¿ç§»åŽçš„ RKE2 æ··åˆé›†ç¾¤ Linux-only Deployment（针对 RKE2 Linux Worker 节点） + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + nodeSelector: + kubernetes.io/os: "linux" + node-role.kubernetes.io/worker: "true" +``` +
+ +- 使用 `nodeAffinity` è¿ç§»åŽçš„ RKE2 æ··åˆé›†ç¾¤ Linux-only Deployment（针对 RKE2 Linux Worker 节点） + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux +``` +## RKE1 Windows - 支æŒçš„ Windows Server 版本 + +### 长期æœåŠ¡æ¸ é“（LTSC） + +- Windows Server 2019 LTSC ✅ 将于 2024 å¹´ 1 月 9 æ—¥ 达到 Mainstream EOL 并于 2029 å¹´ 1 月 9 æ—¥ 达到 Extended EOL + +### åŠå¹´é¢‘é“ (SAC) + +- Windows Server 20H2 SAC ❌ 2022 å¹´ 8 月 9 日已达到 EOL +- Windows Server 2004 SAC ❌ 2021 å¹´ 12 月 14 日已达到 EOL +- Windows Server 1909 SAC ❌ 2021 å¹´ 5 月 11 日已达到 EOL +- Windows Server 1903 SAC ❌ 2020 å¹´ 12 月 8 日已达到 EOL +- Windows Server 1809 SAC ❌ 2020 å¹´ 11 月 10 日已达到 EOL + +## RKE2 Windows - 支æŒçš„ Windows Server 版本 + +### RKE2 中的长期æœåŠ¡é€šé“（LTSC） + +- Windows Server 2019 LTSC ✅ 将于 2024 å¹´ 1 月 9 æ—¥ 达到 Mainstream EOL 并于 2029 å¹´ 1 月 9 æ—¥ 达到 Extended EOL +- Windows Server 2022 LTSC ✅ 将于 2026 å¹´ 10 月 13 日达到 Mainstream EOL，并于 2031 å¹´ 10 月 13 日达到 Extended EOL + +> **注æ„**：RKE2 ä¸æ”¯æŒ SAC。 + + +有关详细信æ¯ï¼Œè¯·å‚阅以下å‚考资料： + +- [Windows Server SAC 生命周期](https://docs.microsoft.com/en-us/lifecycle/products/windows-server) + +- [Windows Server 2022 LTSC 生命周期](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2022) + +- [Windows Server 2019 LTSC 生命周期](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2019) + + +## Kubernetes ç‰ˆæœ¬æ”¯æŒ + +> **注æ„**ï¼šæ ¹æ® [Rancher 2.6.7 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/rancher-v2-6-7/)，下é¢åˆ—出的所有版本å‡æ”¯æŒ SLA。未列出的版本都视为为已达到 EOL 且ä¸å— SUSE SLA 支æŒã€‚ + +### 对比 Rancher 2.5 与 Rancher 2.6：Windows 集群的支æŒçŸ©é˜µ + +**对比 RKE1 与 RKE2：Windows 集群支æŒçš„ Kubernetes 版本**： + +| Kubernetes 版本 | RKE1 | RKE2 | +|-------------- |:----: |:----: | +| 1.18 | ✓ | | +| 1.19 | ✓ | | +| 1.20 | ✓ | | +| 1.21 | ✓ | | +| 1.22 | ✓ | ✓ | +| 1.23 | | ✓ | +| 1.24 | | ✓ | +| 1.25+ | | ✓ | + + +### 对比 Rancher 2.5 与 Rancher 2.6：支æŒç”¨äºŽé…ç½® RKE1 å’Œ RKE2 Windows 集群的 Kubernetes 版本 + +| Rancher 版本 | Kubernetes 版本 | RKE1 | RKE2 | +|:-----------------------: |:------------------------: |:----: |:----: | +| 2.5 - RKE1 é…ç½® | 1.18 1.19 1.20 | ✓ | | +| 2.6 - RKE1 é…ç½® | 1.18 1.19 1.20 1.21 1.22 | ✓ | | +| 2.6 - RKE2 é…ç½® | 1.22 1.23 1.24 1.25+ | | ✓ | + + +## 将工作负载è¿ç§»åˆ° RKE2 Windows çš„æŒ‡å— + +å‚考[对比 Rancher 2.5 与 Rancher 2.6：Windows 集群的支æŒçŸ©é˜µ](#对比-rancher-25-与-rancher-26windows-集群的支æŒçŸ©é˜µ)å’Œ[对比 Rancher 2.5 与 Rancher 2.6：支æŒç”¨äºŽé…ç½® RKE1 å’Œ RKE2 Windows 集群的 Kubernetes 版本](#对比-rancher-25-与-rancher-26支æŒç”¨äºŽé…ç½®-rke1-å’Œ-rke2-windows-集群的-kubernetes-版本)中的表格，你会å‘现 RKE1 å’Œ RKE2 çš„ Kubernetes 1.22 版本å‘生了é‡å ã€‚因此，当éµå¾ª Rancher 推è的方法时，这将是è¿ç§» RKE1 Windows 工作负载所需的基本版本。 + +### Rancher 2.5 的就地å‡çº§ + +1. å°† Rancher 版本å‡çº§åˆ° 2.6.5+。 +1. 使用最新的å¯ç”¨è¡¥ä¸ç‰ˆæœ¬å°† RKE1 Windows 下游集群å‡çº§åˆ° RKE1 v1.22。 +1. 通过 RKE1 Windows 集群所在的匹é…è¡¥ä¸ç‰ˆæœ¬ï¼Œä½¿ç”¨ RKE2 v1.22 é…置新的 RKE2 Windows 下游集群。 +1. 开始将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 集群。 +1. 执行验è¯æµ‹è¯•ï¼Œç¡®ä¿åœ¨å°†åº”用程åºä»Ž RKE1 è¿ç§»åˆ° RKE2 时没有丢失或更改功能。 +1. 验è¯æµ‹è¯•æˆåŠŸåŽï¼Œä½ å¯ä»¥é€‰æ‹©å°† RKE2 1.22.x 集群å‡çº§åˆ°æ–°çš„次è¦ç‰ˆæœ¬ï¼Œä¾‹å¦‚ 1.23 或 1.24。 + + +### å°† Windows 工作负载è¿ç§»åˆ°æ–°çš„ Rancher 环境 + +> **é‡è¦æ示**：è¦æ‰§è¡Œä»¥ä¸‹ä»»ä¸€é€‰é¡¹ï¼Œä½ éœ€è¦ä½¿ç”¨ Rancher 2.6.5 或更高版本。 + +**为 RKE1 å’Œ RKE2 使用匹é…çš„ Kubernetes è¡¥ä¸ç‰ˆæœ¬æ—¶**： + +1. 通过 RKE1 Windows 集群所在的匹é…è¡¥ä¸ç‰ˆæœ¬ï¼Œä½¿ç”¨ RKE2 v1.22 é…置新的 RKE2 Windows 下游集群。 +1. 开始将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 集群。 +1. 执行验è¯æµ‹è¯•ï¼Œç¡®ä¿åœ¨å°†åº”用程åºä»Ž RKE1 è¿ç§»åˆ° RKE2 时没有丢失或更改功能。 +1. 验è¯æµ‹è¯•æˆåŠŸåŽï¼Œä½ å¯ä»¥é€‰æ‹©å°† RKE2 1.22.x 集群å‡çº§åˆ°æ–°çš„次è¦ç‰ˆæœ¬ï¼Œä¾‹å¦‚ 1.23 或 1.24。 + + +**为 RKE2 使用更新的 Kubernetes è¡¥ä¸ç‰ˆæœ¬æ—¶**： + +1. 使用 RKE2 v1.23 或 v1.24 é…置新的 RKE2 Windows 下游集群。 +1. 开始将 Windows 工作负载从 RKE1 è¿ç§»åˆ° RKE2 集群。 +1. 执行验è¯æµ‹è¯•ï¼Œç¡®ä¿åœ¨å°†åº”用程åºä»Ž RKE1 è¿ç§»åˆ° RKE2 时没有丢失或更改功能。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md new file mode 100644 index 00000000000..e66e26a3975 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md @@ -0,0 +1,35 @@ +--- +title: ConfigMap +--- + +大多数 Kubernetes 密文用于存储æ•æ„Ÿä¿¡æ¯ï¼Œä½†æ˜¯ [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) 会存储一般é…置信æ¯ï¼Œä¾‹å¦‚一组é…置文件。由于 ConfigMap ä¸å­˜å‚¨æ•æ„Ÿä¿¡æ¯ï¼Œå› æ­¤ ConfigMap å¯ä»¥è‡ªåŠ¨æ›´æ–°ï¼Œä¸éœ€è¦åœ¨æ›´æ–°åŽé‡å¯å…¶å®¹å™¨ï¼ˆå…¶ä»–类型的密文大多需è¦æ‰‹åŠ¨æ›´æ–°å’Œé‡å¯å®¹å™¨åŽæ‰èƒ½ç”Ÿæ•ˆï¼‰ã€‚ + +ConfigMap 接å—常è§å­—符串格å¼çš„键值对，例如é…置文件或 JSON Blob。上传é…置映射åŽï¼Œä»»ä½•å·¥ä½œè´Ÿè½½éƒ½å¯ä»¥å°†å…¶å¼•ç”¨ä¸ºçŽ¯å¢ƒå˜é‡æˆ–å·æŒ‚载。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到具有è¦å¼•ç”¨ ConfigMap 的工作负载的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æ›´å¤šèµ„æº > 核心 > ConfigMaps**。 +1. å•å‡»**创建**。 +1. 输入 ConfigMap çš„**å称**。 + + :::note + + Kubernetes å°† ConfigMap 归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，项目或命å空间中的密文ä¸èƒ½é‡å。因此，为了é¿å…冲çªï¼ŒConfigMap çš„å称必须与工作空间中的其他è¯ä¹¦ã€é•œåƒä»“库和密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 选择è¦æ·»åŠ  ConfigMap çš„**命å空间**。 + +1. 在**æ•°æ®**选项å¡ä¸Šï¼Œå°†é”®å€¼å¯¹æ·»åŠ åˆ°ä½ çš„ ConfigMap。你å¯ä»¥æ ¹æ®éœ€è¦æ·»åŠ ä»»æ„æ•°é‡çš„值。通过å¤åˆ¶å’Œç²˜è´´çš„æ–¹å¼å°†å¤šä¸ªé”®å€¼å¯¹æ·»åŠ åˆ° ConfigMap。你也å¯ä»¥ä½¿ç”¨**从文件读å–**æ¥æ·»åŠ æ•°æ®ã€‚请注æ„，如果è¦å­˜å‚¨æ•æ„Ÿæ•°æ®ï¼Œè¯·[使用密文](secrets.md)，ä¸è¦ä½¿ç”¨ ConfigMap。 +1. å•å‡»**创建**。 + +**结果**：已将 ConfigMap 添加到命å空间。你å¯ä»¥åœ¨ Rancher UI çš„**èµ„æº > ConfigMaps** 视图中查看它。 + +## åŽç»­æ“作 + +现在你已ç»å°† ConfigMap 添加到命å空间，你å¯ä»¥å°†å…¶æ·»åŠ åˆ°åŽŸå§‹å‘½å空间部署的工作负载中。你å¯ä»¥ä½¿ç”¨ ConfigMap 指定应用程åºä½¿ç”¨çš„ä¿¡æ¯ï¼Œä¾‹å¦‚： + +- 应用程åºçŽ¯å¢ƒå˜é‡ã€‚ +- 指定安装到工作负载的å·çš„å‚数。 + +有关将 ConfigMap 添加到工作负载的更多信æ¯ï¼Œè¯·å‚阅[部署工作负载](workloads-and-pods/deploy-workloads.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md new file mode 100644 index 00000000000..2c9be4b41c9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md @@ -0,0 +1,29 @@ +--- +title: Services +--- + +Deploymentã€StatefulSet å’Œ Daemonset ä¼šç®¡ç† Pod é…置，而 Service 使用选择器将æµé‡å¼•å¯¼åˆ° Pod。 + +æ¯ä¸ªå·¥ä½œè´Ÿè½½ï¼ˆè‡³å°‘é…置一个端å£ï¼‰éƒ½ä¼šåˆ›å»ºä¸€ä¸ªè¡¥å……çš„æœåŠ¡å‘现æ¡ç›®ã€‚æ­¤æœåŠ¡å‘现æ¡ç›®ä½¿ç”¨ä»¥ä¸‹å‘½å约定为工作负载的 pod å¯ç”¨ DNS 解æžï¼š +`..svc.cluster.local`。 + +ä½ å¯ä»¥åˆ›å»ºå…¶å®ƒ service，这样，指定的命å空间就å¯ä»¥ä½¿ç”¨ä¸€ä¸ªæˆ–多个外部 IP 地å€ã€å¤–部主机åã€å¦ä¸€ä¸ª DNS 记录的别åã€å…¶ä»–工作负载或与你创建的选择器匹é…的一组 pod æ¥è¿›è¡Œè§£æžã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ  service 的集群，然åŽå•å‡» **Explore**。 +1. 点击**æœåŠ¡å‘现 > æœåŠ¡**。 +1. å•å‡»**创建**。 +1. 选择è¦åˆ›å»ºçš„ service 类型。 +1. 从下拉列表中选择一个的**命å空间**。 +1. 输入 service çš„**å称**。此å称会用于 DNS 解æžã€‚ +1. 完æˆè¡¨å•çš„其余部分。如需帮助，请å‚阅 [service 的上游 Kubernetes 文档](https://kubernetes.io/docs/concepts/services-networking/service/)。 +1. å•å‡»**创建**。 + +**结果**：创建了一个新 service。 + +- ä½ å¯ä»¥ä»Žé¡¹ç›®çš„**æœåŠ¡å‘现**选项å¡æŸ¥çœ‹è®°å½•ã€‚ +- 访问你创建的新记录的新 DNS å称 (`..svc.cluster.local`) 时，它会解æžæ‰€é€‰çš„命å空间。 + +## 相关链接 + +- [使用 HostAliases å‘ Pod /etc/hosts 文件添加æ¡ç›®](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md new file mode 100644 index 00000000000..1c4c314708d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md @@ -0,0 +1,38 @@ +--- +title: 加密 HTTP 通信 +description: 了解如何添加 SSL è¯ä¹¦æˆ– TLS è¯ä¹¦ +--- + +在 Rancher/Kubernetes 中创建 Ingress 时，你必须æä¾›åŒ…å« TLS ç§é’¥å’Œè¯ä¹¦çš„密文，用于加密和解密通过 Ingress 的通信。你å¯ä»¥é€šè¿‡å°†è¯ä¹¦æ·»åŠ åˆ° Ingress Deployment æ¥è®© Ingress 使用è¯ä¹¦ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +你必须具有å¯ä¸Šä¼ çš„ TLS ç§é’¥å’Œè¯ä¹¦ã€‚ + +::: + +### 1. 创建一个密文 + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦éƒ¨ç½² Ingress 的集群，然åŽå•å‡»**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. å•å‡» **TLS è¯ä¹¦**。 +1. 输入密文的å称。请注æ„，密文的å称必须与项目/工作空间中的其他è¯ä¹¦ã€é•œåƒä»“库和密文ä¸ä¸€æ ·ã€‚ +1. 在**ç§é’¥**字段中，将è¯ä¹¦çš„ç§é’¥å¤åˆ¶å¹¶ç²˜è´´åˆ°æ–‡æœ¬æ¡†ä¸­ï¼ˆåŒ…括标头和页脚），或者å•å‡»**从文件读å–**选择文件系统上的ç§é’¥ã€‚如果å¯èƒ½ï¼Œæˆ‘们建议使用**从文件读å–**以å‡å°‘出错的å¯èƒ½æ€§ã€‚请注æ„，ç§é’¥æ–‡ä»¶çš„扩展å是 `.key`。 +1. 在**è¯ä¹¦**字段中，将你的è¯ä¹¦å¤åˆ¶å¹¶ç²˜è´´åˆ°æ–‡æœ¬æ¡†ä¸­ï¼ˆåŒ…括标头和页脚），或者å•å‡»**从文件读å–**选择文件系统上的è¯ä¹¦æ–‡ä»¶ã€‚如果å¯èƒ½ï¼Œæˆ‘们建议使用**从文件读å–**以å‡å°‘出错的å¯èƒ½æ€§ã€‚请注æ„，è¯ä¹¦æ–‡ä»¶çš„扩展å是 `.crt`。 +1. å•å‡»**创建**。 + +### 2. 将密文添加到 Ingress + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦éƒ¨ç½² Ingress 的集群，然åŽå•å‡»**æœåŠ¡å‘现 > Ingress**。 +1. å•å‡»**创建**。 +1. 选择 Ingress çš„**命å空间**。 +1. 输入 Ingress çš„**å称**。 +1. 在**è¯ä¹¦**选项å¡ä¸­ï¼Œé€‰æ‹©åŒ…å«ä½ çš„è¯ä¹¦å’Œç§é’¥çš„密文。 +1. å•å‡»**创建**。 + +## åŽç»­æ“作 + +现在，你å¯ä»¥åœ¨è¯¥é¡¹ç›®æˆ–命å空间中å¯åŠ¨ Ingress 时添加è¯ä¹¦ã€‚如需更多信æ¯ï¼Œè¯·å‚阅[添加 Ingress](load-balancer-and-ingress-controller/add-ingresses.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md new file mode 100644 index 00000000000..eb60cb4c97b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md @@ -0,0 +1,39 @@ +--- +title: Horizontal Pod Autoscaler ä»‹ç» +--- + +[Horizontal Pod Autoscaler（HPA）](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)是一项 Kubernetes 功能，用于将集群é…置为自动扩缩其è¿è¡Œçš„æœåŠ¡ã€‚æœ¬èŠ‚ä»‹ç» HPA 如何与 Kubernetes 一起工作。 + +## 为什么è¦ä½¿ç”¨ Horizo​​ntal Pod Autoscaler？ + +ä½ å¯ä»¥ä½¿ç”¨ HPA æ¥è‡ªåŠ¨æ‰©ç¼© ReplicationControllerã€Deployment 或 ReplicaSet 中的 pod æ•°é‡ã€‚HPA 会自动扩缩正在è¿è¡Œçš„ pod æ•°é‡ä»¥å®žçŽ°æœ€é«˜æ•ˆçŽ‡ã€‚å½±å“ pod æ•°é‡çš„因素包括： + +- å…许è¿è¡Œçš„最å°å’Œæœ€å¤§ pod 数，由用户定义。 +- 资æºæŒ‡æ ‡ä¸­æŠ¥å‘Šçš„ CPU/内存使用情况。 +- 第三方指标应用程åºï¼ˆå¦‚ Prometheusã€Datadog 等）æ供的自定义指标。 + +HPA 通过以下方å¼æ”¹è¿›ä½ çš„æœåŠ¡ï¼š + +- 释放硬件资æºï¼Œé¿å…资æºè¢«è¿‡å¤šçš„ pod 浪费。 +- 按需æ高/é™ä½Žæ€§èƒ½ä»¥è¾¾åˆ°æœåŠ¡çº§åˆ«å议。 + +## HPA çš„å·¥ä½œåŽŸç† + +![HPA 架构](/img/horizontal-pod-autoscaler.jpg) + +HPA 实现为一个控制循环，其周期由以下 `kube-controller-manager` 标志控制： + +| 标志 | 默认 | æè¿° | +---------|----------|----------| +| `--horizontal-pod-autoscaler-sync-period` | `30s` | HPA 在 deployment 中审核资æº/自定义指标的频率。 | +| `--horizontal-pod-autoscaler-downscale-delay` | `5m0s` | 完æˆç¼©å‡æ“作åŽï¼ŒHPA 必须等待多长时间æ‰èƒ½å¯åŠ¨å¦ä¸€ä¸ªç¼©å‡æ“作。 | +| `--horizontal-pod-autoscaler-upscale-delay` | `3m0s` | 完æˆæ‰©å±•æ“作åŽï¼ŒHPA 必须等待多长时间æ‰èƒ½å¯åŠ¨å¦ä¸€ä¸ªæ‰©å±•æ“作。 | + + +有关 HPA 的完整文档，请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)。 + +## Horizo​​ntal Pod Autoscaler API 对象 + +HPA 是 Kubernetes `autoscaling` API 组中的 API 资æºã€‚当å‰çš„稳定版本是 `autoscaling/v1`，它åªæ”¯æŒå¯¹ CPU 自动扩缩。è¦èŽ·å¾—内存和自定义指标的扩缩支æŒï¼Œè¯·æ”¹ç”¨ beta 版本 `autoscaling/v2beta1`。 + +有关 HPA API 对象的更多信æ¯ï¼Œè¯·å‚阅 [HPA GitHub 自述文件](https://github.com/kubernetes/design-proposals-archive/blob/main/autoscaling/horizontal-pod-autoscaler.md#horizontalpodautoscaler-object)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md new file mode 100644 index 00000000000..a8ff4dd0879 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md @@ -0,0 +1,204 @@ +--- +title: 使用 kubectl ç®¡ç† HPA +--- + +本文介ç»å¦‚何使用 `kubectl` ç®¡ç† HPA。本文对以下æ“作进行了说明： + +- 创建 HPA +- èŽ·å– HPA çš„ä¿¡æ¯ +- 删除 HPA +- é…ç½® HPA ä»¥æ ¹æ® CPU 或内存利用率进行扩缩 +- 如果你使用 Prometheus 等第三方工具的指标，é…ç½® HPA 以使用自定义指标进行扩缩 + + +ä½ å¯ä»¥åœ¨ Rancher UI 中创建ã€æŸ¥çœ‹å’Œåˆ é™¤ HPA。你还å¯ä»¥æ ¹æ® Rancher UI 中的 CPU/内存使用情况进行扩缩。有关详细信æ¯ï¼Œè¯·å‚阅[使用 Rancher UI ç®¡ç† HPA](./manage-hpas-with-ui.md)。如果è¦ä½¿ç”¨ CPU/内存以外的指标æ¥æ‰©ç¼© HPA，你需è¦ä½¿ç”¨ `kubectl`。 + +## ç®¡ç† HPA 的基本 kubectl 命令 + +如果你有 HPA 清å•æ–‡ä»¶ï¼Œåˆ™å¯ä»¥ä½¿ç”¨ `kubectl` æ¥åˆ›å»ºã€ç®¡ç†å’Œåˆ é™¤ HPA： + +- 创建 HPA + + - 有清å•ï¼š`kubectl create -f ` + + - 没有清å•ï¼ˆä»…æ”¯æŒ CPU）：`kubectl autoscale deployment hello-world --min=2 --max=5 --cpu-percent=50` + +- èŽ·å– HPA ä¿¡æ¯ + + - 基本：`kubectl get hpa hello-world` + + - 详细æ述：`kubectl describe hpa hello-world` + +- 删除 HPA + + - `kubectl delete hpa hello-world` + +## HPA 清å•å®šä¹‰ç¤ºä¾‹ + +HPA 清å•æ˜¯ç”¨äºŽä½¿ç”¨ `kubectl` ç®¡ç† HPA çš„é…置文件。 + +以下代ç ç‰‡æ®µåœ¨ HPA 清å•ä¸­ä½¿ç”¨äº†ä¸åŒçš„指令。请å‚阅示例下方的列表以了解æ¯ä¸ªæŒ‡ä»¤çš„用途。 + +```yml +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: hello-world +spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi +``` + + +| 指令 | æè¿° | +---------|----------| +| `apiVersion: autoscaling/v2beta1` | 正在使用的 Kubernetes `autoscaling` API 组的版本。此示例清å•ä½¿ç”¨ beta 版本，因此å¯ç”¨äº†æŒ‰ CPU./内存进行扩缩。 | +| `name: hello-world` | 表示 HPA 在为 `hello-word` deployment 执行自动扩缩。 | +| `minReplicas: 1` | 表示è¿è¡Œçš„最å°å‰¯æœ¬æ•°ä¸èƒ½ä½ŽäºŽ 1。 | +| `maxReplicas: 10` | 指示 deployment 中的最大副本数ä¸èƒ½è¶…过 10。 | +| `targetAverageUtilization: 50` | 表示当平å‡è¿è¡Œçš„ pod 使用超过请求 CPU çš„ 50% 时，deployment 将扩展 pod。 | +| `targetAverageValue: 100Mi` | 表示当平å‡è¿è¡Œçš„ pod 使用超过 100Mi 的内存时，deployment 将扩展 pod。 | +
+ +## 使用资æºæŒ‡æ ‡ï¼ˆCPU 和内存）é…ç½® HPA 以进行扩缩 + +在 Rancher 2.0.7 åŠæ›´é«˜ç‰ˆæœ¬ä¸­åˆ›å»ºçš„集群满足使用 Horizontal Pod Autoscaler 的所有è¦æ±‚（metrics-server å’Œ Kubernetes 集群é…置）。 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ä»¥æ£€æŸ¥ä½ çš„安装中是å¦æœ‰å¯ç”¨çš„指标： + +``` +$ kubectl top nodes +NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% +node-controlplane 196m 9% 1623Mi 42% +node-etcd 80m 4% 1090Mi 28% +node-worker 64m 3% 1146Mi 29% +$ kubectl -n kube-system top pods +NAME CPU(cores) MEMORY(bytes) +canal-pgldr 18m 46Mi +canal-vhkgr 20m 45Mi +canal-x5q5v 17m 37Mi +canal-xknnz 20m 37Mi +kube-dns-7588d5b5f5-298j2 0m 22Mi +kube-dns-autoscaler-5db9bbb766-t24hw 0m 5Mi +metrics-server-97bc649d5-jxrlt 0m 12Mi +$ kubectl -n kube-system logs -l k8s-app=metrics-server +I1002 12:55:32.172841 1 heapster.go:71] /metrics-server --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true +I1002 12:55:32.172994 1 heapster.go:72] Metrics Server version v0.2.1 +I1002 12:55:32.173378 1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default.svc" and version +I1002 12:55:32.173401 1 configs.go:62] Using kubelet port 10250 +I1002 12:55:32.173946 1 heapster.go:128] Starting with Metric Sink +I1002 12:55:32.592703 1 serving.go:308] Generated self-signed cert (apiserver.local.config/certificates/apiserver.crt, apiserver.local.config/certificates/apiserver.key) +I1002 12:55:32.925630 1 heapster.go:101] Starting Heapster API server... +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] listing is available at https:///swaggerapi +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] https:///swaggerui/ is mapped to folder /swagger-ui/ +I1002 12:55:32.928597 1 serve.go:85] Serving securely on 0.0.0.0:443 +``` + + +## é…ç½® HPA 以使用 Prometheus 自定义指标进行扩缩 + +ä½ å¯ä»¥å°† HPA é…置为根æ®ç¬¬ä¸‰æ–¹è½¯ä»¶æ供的自定义指标进行自动扩缩。使用第三方软件进行自动扩缩的最常è§ç”¨ä¾‹æ˜¯ä½¿ç”¨åº”用程åºçº§åˆ«çš„指标（å³æ¯ç§’ HTTP 请求数）。HPA 使用 `custom.metrics.k8s.io` API æ¥ä½¿ç”¨è¿™äº›æŒ‡æ ‡ã€‚通过为指标收集解决方案部署自定义指标适é…器，你å¯ä»¥å¯ç”¨æ­¤ API。 + +对于这个例å­ï¼Œæˆ‘们将使用 [Prometheus](https://prometheus.io/)。我们å‡è®¾ï¼š + +- Prometheus 部署在集群中。 +- Prometheus é…置正确，并从 podã€èŠ‚点ã€å‘½å空间等收集åˆé€‚的指标。 +- Prometheus 暴露在以下 URL 和端å£ï¼š`http://prometheus.mycompany.io:80` + +Prometheus å¯ç”¨äºŽ Rancher 2.0 应用商店中的 Deployment。如果它还没有è¿è¡Œåœ¨ä½ çš„集群中，请在 Rancher 应用商店中部署它。 + +为了让 HPA 使用æ¥è‡ª Prometheus 的自定义指标，[k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) 包需è¦åœ¨ä½ é›†ç¾¤çš„ `kube-system` 命å空间中。è¦å®‰è£… `k8s-prometheus-adapter`，使用 [banzai-charts](https://github.com/banzaicloud/banzai-charts) æ供的 Helm Chart。 + +1. 在集群中åˆå§‹åŒ– Helm。 + ``` + # kubectl -n kube-system create serviceaccount tiller + kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller + helm init --service-account tiller + ``` + +1. 从 GitHub 克隆 `banzai-charts` 仓库： + ``` + # git clone https://github.com/banzaicloud/banzai-charts + ``` + +1. 安装 `prometheus-adapter` Chart，指定 Prometheus URL 和端å£å·ï¼š + ``` + # helm install --name prometheus-adapter banzai-charts/prometheus-adapter --set prometheus.url="http://prometheus.mycompany.io",prometheus.port="80" --namespace kube-system + ``` + +1. 检查 `prometheus-adapter` 是å¦æ­£å¸¸è¿è¡Œã€‚检查æœåŠ¡ pod 并在 `kube-system` 命å空间中登录。 + + 1. 检查æœåŠ¡ pod 是å¦å¤„于 `Running` 状æ€ã€‚输入以下命令： + ``` + # kubectl get pods -n kube-system + ``` + 在输出结果中查找 `Running` 状æ€ã€‚ + ``` + NAME READY STATUS RESTARTS AGE + ... + prometheus-adapter-prometheus-adapter-568674d97f-hbzfx 1/1 Running 0 7h + ... + ``` + 1. 通过输入以下命令检查æœåŠ¡æ—¥å¿—，确ä¿æœåŠ¡æ­£å¸¸è¿è¡Œï¼š + ``` + # kubectl logs prometheus-adapter-prometheus-adapter-568674d97f-hbzfx -n kube-system + ``` + 然åŽæŸ¥çœ‹æ—¥å¿—输出以确认æœåŠ¡æ­£åœ¨è¿è¡Œã€‚ + +
+ Prometheus Adaptor 日志 + + ... + I0724 10:18:45.696679 1 round_trippers.go:436] GET https://10.43.0.1:443/api/v1/namespaces/default/pods?labelSelector=app%3Dhello-world 200 OK in 2 milliseconds + I0724 10:18:45.696695 1 round_trippers.go:442] Response Headers: + I0724 10:18:45.696699 1 round_trippers.go:445] Date: Tue, 24 Jul 2018 10:18:45 GMT + I0724 10:18:45.696703 1 round_trippers.go:445] Content-Type: application/json + I0724 10:18:45.696706 1 round_trippers.go:445] Content-Length: 2581 + I0724 10:18:45.696766 1 request.go:836] Response Body: {"kind":"PodList","apiVersion":"v1","metadata":{"selfLink":"/api/v1/namespaces/default/pods","resourceVersion":"6237"},"items":[{"metadata":{"name":"hello-world-54764dfbf8-q6l82","generateName":"hello-world-54764dfbf8-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/hello-world-54764dfbf8-q6l82","uid":"484cb929-8f29-11e8-99d2-067cac34e79c","resourceVersion":"4066","creationTimestamp":"2018-07-24T10:06:50Z","labels":{"app":"hello-world","pod-template-hash":"1032089694"},"annotations":{"cni.projectcalico.org/podIP":"10.42.0.7/32"},"ownerReferences":[{"apiVersion":"extensions/v1beta1","kind":"ReplicaSet","name":"hello-world-54764dfbf8","uid":"4849b9b1-8f29-11e8-99d2-067cac34e79c","controller":true,"blockOwnerDeletion":true}]},"spec":{"volumes":[{"name":"default-token-ncvts","secret":{"secretName":"default-token-ncvts","defaultMode":420}}],"containers":[{"name":"hello-world","image":"rancher/hello-world","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"default-token-ncvts","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"34.220.18.140","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}]},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:54Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"}],"hostIP":"34.220.18.140","podIP":"10.42.0.7","startTime":"2018-07-24T10:06:50Z","containerStatuses":[{"name":"hello-world","state":{"running":{"startedAt":"2018-07-24T10:06:54Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"rancher/hello-world:latest","imageID":"docker-pullable://rancher/hello-world@sha256:4b1559cb4b57ca36fa2b313a3c7dde774801aa3a2047930d94e11a45168bc053","containerID":"docker://cce4df5fc0408f03d4adf82c90de222f64c302bf7a04be1c82d584ec31530773"}],"qosClass":"Burstable"}}]} + I0724 10:18:45.699525 1 api.go:74] GET http://prometheus-server.prometheus.34.220.18.140.xip.io/api/v1/query?query=sum%28rate%28container_fs_read_seconds_total%7Bpod_name%3D%22hello-world-54764dfbf8-q6l82%22%2Ccontainer_name%21%3D%22POD%22%2Cnamespace%3D%22default%22%7D%5B5m%5D%29%29+by+%28pod_name%29&time=1532427525.697 200 OK + I0724 10:18:45.699620 1 api.go:93] Response Body: {"status":"success","data":{"resultType":"vector","result":[{"metric":{"pod_name":"hello-world-54764dfbf8-q6l82"},"value":[1532427525.697,"0"]}]}} + I0724 10:18:45.699939 1 wrap.go:42] GET /apis/custom.metrics.k8s.io/v1beta1/namespaces/default/pods/%2A/fs_read?labelSelector=app%3Dhello-world: (12.431262ms) 200 [[kube-controller-manager/v1.10.1 (linux/amd64) kubernetes/d4ab475/system:serviceaccount:kube-system:horizontal-pod-autoscaler] 10.42.0.0:24268] + I0724 10:18:51.727845 1 request.go:836] Request Body: {"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","metadata":{"creationTimestamp":null},"spec":{"nonResourceAttributes":{"path":"/","verb":"get"},"user":"system:anonymous","group":["system:unauthenticated"]},"status":{"allowed":false}} + ... +
+ + + +1. 检查是å¦å¯ä»¥ä»Ž kubectl 访问 metrics API。 + + - 如果你直接访问集群，请在 kubectl é…置中输入你的æœåŠ¡å™¨ URL，格å¼æ˜¯ `https://:6443`： + ``` + # kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 + ``` + 如果 API å¯è®¿é—®ï¼Œä½ åº”该会看到类似以下内容的输出： + +
+ API å“应 + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
+ + - 如果你通过 Rancher 访问集群，请在 kubectl é…置中输入你的æœåŠ¡å™¨ URL，格å¼æ˜¯ `https:///k8s/clusters/`。将 `/k8s/clusters/` åŽç¼€æ·»åŠ åˆ° API 路径： + ``` + # kubectl get --raw /k8s/clusters//apis/custom.metrics.k8s.io/v1beta1 + ``` + 如果 API å¯è®¿é—®ï¼Œä½ åº”该会看到类似以下内容的输出： + +
+ API å“应 + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md new file mode 100644 index 00000000000..42fbec1785a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md @@ -0,0 +1,51 @@ +--- +title: 使用 Rancher UI ç®¡ç† HPA +--- + +Rancher UI 支æŒåˆ›å»ºã€ç®¡ç†å’Œåˆ é™¤ HPA。你å¯ä»¥å°† CPU 或内存使用情况é…置为用于 HPA 扩缩的指标。 + +如果你想使用 CPU/内存之外的其他指标创建 HPA，请å‚阅[é…ç½® HPA 以使用 Prometheus 的自定义指标进行扩缩](./manage-hpas-with-kubectl.md#é…ç½®-hpa-以使用-prometheus-自定义指标进行扩缩)。 + +## 创建 HPA + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦åˆ›å»º HPA 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æœåŠ¡å‘现 > Horizo​​ntalPodAutoscalers**。 +1. å•å‡»**创建**。 +1. 为 HPA 选择 **命å空间**。 +1. 输入 HPA çš„**å称**。 +1. 选择**目标引用**作为 HPA 的扩缩目标。 +1. 为 HPA 指定**最å°å‰¯æœ¬æ•°**å’Œ**最大副本数**。 +1. é…ç½® HPA 的指标。你å¯ä»¥å°†å†…存或 CPU 使用率作为让 HPA 扩缩æœåŠ¡çš„指标。在**æ•°é‡**字段中，输入让 HPA 扩缩æœåŠ¡çš„工作负载内存/CPU 使用率的百分比。è¦é…置其他 HPA 指标，包括 Prometheus æ供的指标，你需è¦[使用 kubectl ç®¡ç† HPA](./manage-hpas-with-kubectl.md#é…ç½®-hpa-以使用-prometheus-自定义指标进行扩缩)。 + +1. å•å‡»**创建**以创建HPA。 + +:::note 结果： + +HPA 已被部署到选定的命å空间。你å¯ä»¥åœ¨é¡¹ç›®çš„**èµ„æº > HPA** 视图中查看 HPA 的状æ€ã€‚ + +::: + +## èŽ·å– HPA æŒ‡æ ‡å’ŒçŠ¶æ€ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到具有 HPA 的集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æœåŠ¡å‘现 > Horizo​​ntalPodAutoscalers**。**Horizo​​ntalPodAutoscalers** 页é¢æ˜¾ç¤ºå½“å‰å‰¯æœ¬çš„æ•°é‡ã€‚ + +有关特定 HPA 的更详细指标和状æ€ï¼Œè¯·å•å‡» HPA çš„å称。你会转到 HPA 的详情页é¢ã€‚ + + +## 删除 HPA + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦åˆ é™¤çš„ HPA 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**æœåŠ¡å‘现 > Horizo​​ntalPodAutoscalers**。 +1. å•å‡»**èµ„æº > HPA**。 +1. 找到è¦åˆ é™¤çš„ HPA，然åŽå•å‡» **â‹® > 删除**。 +1. å•å‡»**删除**以进行确认。 + +:::note 结果： + +HPA 已从当å‰é›†ç¾¤ä¸­åˆ é™¤ã€‚ + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md new file mode 100644 index 00000000000..d48c98b3e8c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md @@ -0,0 +1,531 @@ +--- +title: 使用 kubectl 测试 HPA +--- + +本文介ç»å¦‚何在使用负载测试工具扩缩 HPA åŽæ£€æŸ¥ HPA 的状æ€ã€‚有关使用 Rancher UI（最低版本 2.3.x）检查状æ€çš„ä¿¡æ¯ï¼Œè¯·å‚阅[使用 Rancher UI ç®¡ç† HPA](./manage-hpas-with-kubectl.md)。 + +è¦è®© HPA 正常工作，æœåŠ¡éƒ¨ç½²åº”该具有容器的资æºè¯·æ±‚定义。按照以下 hello-world 示例测试 HPA 是å¦æ­£å¸¸å·¥ä½œã€‚ + +1. å°† `kubectl` 连接到你的 Kubernetes 集群。 + +1. å¤åˆ¶ä¸‹æ–¹çš„ `hello-world` 部署清å•ã€‚ + +
+ Hello World æ¸…å• + + ``` + apiVersion: apps/v1beta2 + kind: Deployment + metadata: + labels: + app: hello-world + name: hello-world + namespace: default + spec: + replicas: 1 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + resources: + requests: + cpu: 500m + memory: 64Mi + ports: + - containerPort: 80 + protocol: TCP + restartPolicy: Always + --- + apiVersion: v1 + kind: Service + metadata: + name: hello-world + namespace: default + spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-world + ``` + +
+ +1. 将其部署到你的集群。 + + ``` + # kubectl create -f + ``` + +1. æ ¹æ®ä½ ä½¿ç”¨çš„指标类型å¤åˆ¶ä»¥ä¸‹å…¶ä¸­ä¸€ä¸ª HPA： + +
+ Hello World HPA：资æºæŒ‡æ ‡ + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 1000Mi + ``` + +
+
+ Hello World HPA：自定义指标 + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi + - type: Pods + pods: + metricName: cpu_system + targetAverageValue: 20m + ``` + +
+ +1. 查看 HPA ä¿¡æ¯å’Œæ述。确认显示的指标数æ®ã€‚ + +
+ 资æºæŒ‡æ ‡ + + 1. 输入以下命令： + ``` + # kubectl get hpa + NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE + hello-world Deployment/hello-world 1253376 / 100Mi, 0% / 50% 1 10 1 6m + # kubectl describe hpa + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 20:21:16 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 1253376 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+
+ 自定义指标 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:36:28 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 3514368 / 100Mi + "cpu_system" on pods: 0 / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+ +1. 为æœåŠ¡ç”Ÿæˆè´Ÿè½½ï¼Œä»Žè€Œæµ‹è¯•ä½ çš„ pod 是å¦æŒ‰é¢„期进行了自动扩缩。你å¯ä»¥ä½¿ç”¨ä»»ä½•è´Ÿè½½æµ‹è¯•å·¥å…·ï¼ˆHeyã€Gatling 等），我们使用的是 [Hey](https://github.com/rakyll/hey)。 + +1. 测试 pod 自动扩缩是å¦æŒ‰é¢„期工作。

+ **使用资æºæŒ‡æ ‡æµ‹è¯•è‡ªåŠ¨æ‰©ç¼©ï¼š** + +
+ 扩展到两个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨ä½ çš„è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 使用情况扩展到两个 Pod。 + + 1. 查看你的 HPA。 + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10928128 / 100Mi + resource cpu on pods (as a percentage of request): 56% (280m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. 输入以下命令，确认你已扩展到两个 pod： + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-k8ph2 1/1 Running 0 1m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ 扩展到三个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨ä½ çš„è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 用é‡æ‰©å±•åˆ°ä¸‰ä¸ª Pod，并将 `horizo​​ntal-pod-autoscaler-upscale-delay` 设置为 3 分钟。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 9424896 / 100Mi + resource cpu on pods (as a percentage of request): 66% (333m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 4m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + ``` + 2. 输入以下命令，确认三个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-f46kh 0/1 Running 0 1m + hello-world-54764dfbf8-k8ph2 1/1 Running 0 5m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ 缩å‡åˆ° 1 个 Pod：所有指标å‡ä½ŽäºŽç›®æ ‡ + + 当 `horizontal-pod-autoscaler-downscale-delay` 的所有指标å‡ä½ŽäºŽç›®æ ‡ï¼ˆé»˜è®¤ä¸º 5 分钟）时，使用你的负载测试工具缩å‡åˆ° 1 个 pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10070016 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 6m horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 1s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + +
+ + **使用自定义指标测试自动缩放：** + +
+ 扩展到两个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 用é‡æ‰©å±•åˆ°ä¸¤ä¸ª Pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8159232 / 100Mi + "cpu_system" on pods: 7m / 20m + resource cpu on pods (as a percentage of request): 64% (321m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. 输入以下命令，确认两个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ 扩展到三个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + 当 cpu_system 用é‡è¾¾åˆ°ç›®æ ‡æ—¶ï¼Œä½¿ç”¨ä½ çš„负载测试工具扩展到三个 Pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3s horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + ``` + 1. 输入以下命令，确认三个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + # kubectl get pods + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ 扩展到四个 Pod：CPU 用é‡è¾¾åˆ°ç›®æ ‡ + + ä½¿ç”¨è´Ÿè½½æµ‹è¯•å·¥å…·æ ¹æ® CPU 用é‡æ‰©å±•åˆ°å››ä¸ª Pod。`horizontal-pod-autoscaler-upscale-delay` 默认设置为 3 分钟。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 4s horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + ``` + + 1. 输入以下命令，确认四个 pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-2p9xb 1/1 Running 0 5m + hello-world-54764dfbf8-5pfdr 1/1 Running 0 2m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ 缩å‡åˆ° 1 个 Pod：所有指标å‡ä½ŽäºŽç›®æ ‡ + + 当 `horizontal-pod-autoscaler-downscale-delay` 的所有指标å‡ä½ŽäºŽç›®æ ‡æ—¶ï¼Œä½¿ç”¨ä½ çš„负载测试工具缩å‡åˆ° 1 个 pod。 + + 1. 输入以下命令： + ``` + # kubectl describe hpa + ``` + 你应该会看到类似以下的输出： + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8101888 / 100Mi + "cpu_system" on pods: 8m / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 8m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + 1. 输入以下命令，确认å•ä¸ª pod 正在è¿è¡Œï¼š + ``` + # kubectl get pods + ``` + 你应该会看到类似以下的输出： + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md new file mode 100644 index 00000000000..6231ec1697a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md @@ -0,0 +1,178 @@ +--- +title: Kubernetes é•œåƒä»“库和容器镜åƒä»“库 +description: 了解容器镜åƒä»“库和 Kubernetes é•œåƒä»“库ã€å®ƒä»¬çš„用例以åŠå¦‚何在 Rancher UI 中使用ç§æœ‰é•œåƒä»“库 +--- + +é•œåƒä»“库是 Kubernetes 密文（Secret），包å«ç”¨äºŽå‘[ç§æœ‰å®¹å™¨é•œåƒä»“库](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/)进行身份验è¯çš„凭è¯ã€‚ + +“Registry†这个è¯å¯èƒ½æœ‰ä¸¤ç§æ„æ€ï¼Œå¯æŒ‡ä»£å®¹å™¨æˆ– Kubernetes é•œåƒä»“库： + +- **容器镜åƒä»“库**（以å‰ç§°ä¸ºâ€œDocker é•œåƒä»“库â€ï¼‰åŒ…å«ä½ å¯ä»¥æ‹‰å–和部署的容器镜åƒã€‚é•œåƒä»“库是一个无状æ€ã€å¯æ‰©å±•çš„æœåŠ¡å™¨ç«¯åº”用程åºï¼Œç”¨äºŽå­˜å‚¨å’Œåˆ†å‘容器镜åƒã€‚ +- **Kubernetes é•œåƒä»“库**是一个镜åƒæ‹‰å–密文，你的 deployment 使用该密文æ¥å‘容器镜åƒä»“库进行身份验è¯ã€‚ + +Deployment 使用 Kubernetes é•œåƒä»“库密文å‘ç§æœ‰å®¹å™¨é•œåƒä»“库进行身份验è¯ï¼Œç„¶åŽæ‹‰å–托管在仓库上的容器镜åƒã€‚ + +ç›®å‰ï¼Œåªæœ‰åœ¨ Rancher UI 中创建工作负载时（通过 kubectl 创建的工作负载ä¸å¯ä»¥ï¼‰ï¼ŒDeployment æ‰ä¼šè‡ªåŠ¨æ‹‰å–ç§æœ‰é•œåƒä»“库凭è¯ã€‚ + +## 在命å空间中创建镜åƒä»“库 + +:::note 先决æ¡ä»¶ï¼š + +你必须已ç»éƒ¨ç½²äº†ä¸€ä¸ªå¯ç”¨çš„ç§æœ‰é•œåƒä»“库。 + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ é•œåƒä»“库的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航中，å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. å•å‡»**é•œåƒä»“库**。 +1. 输入镜åƒä»“库的**å称**。 + + :::note + + Kubernetes 将密文ã€è¯ä¹¦å’Œé•œåƒä»“库都归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，命å空间或项目中的密文å称ä¸èƒ½é‡å¤ã€‚因此，为了é¿å…冲çªï¼Œé•œåƒä»“库的å称必须与工作空间中的其他密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 为镜åƒä»“库选择一个命å空间。 +1. 选择托管你的ç§äººé•œåƒä»“库的网站。然åŽï¼Œè¾“å…¥å‘é•œåƒä»“库进行身份验è¯çš„凭è¯ã€‚例如，如果你使用 DockerHub，请æ供你的 DockerHub 用户å和密ç ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**： + +- 已将你的密文添加到命å空间中。 +- ä½ å¯ä»¥é€šè¿‡å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**在 Rancher UI 中查看密文。 +- 如果工作负载在镜åƒä»“库的范围内，你在 Rancher UI 中创建的任何工作负载都将具有访问镜åƒä»“库的凭è¯ã€‚ + +## 在项目中创建镜åƒä»“库 + +:::note 先决æ¡ä»¶ï¼š + +你必须已ç»éƒ¨ç½²äº†ä¸€ä¸ªå¯ç”¨çš„ç§æœ‰é•œåƒä»“库。 + +如果你需è¦åˆ›å»ºç§æœ‰é•œåƒä»“库，请å‚阅相应è¿è¡Œæ—¶çš„文档： + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl 命令和镜åƒä»“库托管æœåŠ¡](https://github.com/containerd/nerdctl/blob/main/docs/registry.md) +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +在 Rancher 2.6 之å‰ï¼Œå¯†æ–‡å¿…须创建在项目级别。现在ä¸å†éœ€è¦é¡¹ç›®çº§åˆ«ï¼Œä½ å¯ä»¥é‡‡ç”¨å‘½å空间级别。因此，Rancher UI 进行了更新以å映这一新功能。但是，你ä»ç„¶å¯ä»¥æŒ‰ç…§éœ€è¦åˆ›å»ºé¡¹ç›®çº§åˆ«çš„é•œåƒä»“库。执行以下步骤进行æ“作： + +1. 在左上角，å•å‡»ä¸‹æ‹‰èœå•ä¸­çš„ **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 转到 `Legacy` 功能开关并å•å‡»**激活**。 +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ é•œåƒä»“库的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航中，å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. å•å‡»**é•œåƒä»“库**。 +1. 在顶部导航æ ä¸­è¿›è¡Œè¿‡æ»¤ï¼Œä»Žè€Œä»…查看一个项目。 +1. 输入镜åƒä»“库的**å称**。 + + :::note + + Kubernetes 将密文ã€è¯ä¹¦å’Œé•œåƒä»“库都归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，命å空间或项目中的密文å称ä¸èƒ½é‡å¤ã€‚因此，为了é¿å…冲çªï¼Œé•œåƒä»“库的å称必须与工作空间中的其他密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 为镜åƒä»“库选择一个命å空间。 +1. 选择托管你的ç§äººé•œåƒä»“库的网站。然åŽï¼Œè¾“å…¥å‘é•œåƒä»“库进行身份验è¯çš„凭è¯ã€‚例如，如果你使用 DockerHub，请æ供你的 DockerHub 用户å和密ç ã€‚ +1. å•å‡»**ä¿å­˜**。 + +**结果**： + +- 密文已添加到你选择的项目中。 +- ä½ å¯ä»¥é€šè¿‡å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**在 Rancher UI 中查看密文。 +- 如果工作负载在镜åƒä»“库的范围内，你在 Rancher UI 中创建的任何工作负载都将具有访问镜åƒä»“库的凭è¯ã€‚ + +:::note + +local 集群上的项目级别镜åƒä»“库仅在选择å•ä¸ªé¡¹ç›®æ—¶å¯è§ã€‚ + +::: + +## 使用ç§æœ‰é•œåƒä»“库 + +ä½ å¯ä»¥é€šè¿‡ Rancher UI 或 `kubectl` 使用ç§æœ‰é•œåƒä»“库的镜åƒæ¥éƒ¨ç½²å·¥ä½œè´Ÿè½½ã€‚ + +### 在 Rancher UI 中使用ç§æœ‰é•œåƒä»“库 + +è¦ä½¿ç”¨ç§æœ‰é•œåƒä»“库中的镜åƒæ¥éƒ¨ç½²å·¥ä½œè´Ÿè½½ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 点击**工作负载**。 +1. å•å‡»**创建**。 +1. 选择è¦åˆ›å»ºçš„工作负载类型。 +1. 输入工作负载的独特å称，并选择命å空间。 +1. 在**容器镜åƒ**字段中，输入ç§æœ‰é•œåƒä»“库中镜åƒçš„路径 URL。例如，如果你的ç§æœ‰é•œåƒä»“库ä½äºŽ Quay.io，你å¯ä»¥ä½¿ç”¨ `quay.io//`。 +1. å•å‡»**创建**。 + +**结果**：你的 deployment 能å¯åŠ¨ï¼Œèƒ½ä½¿ç”¨ä½ åœ¨ Rancher UI 中添加的ç§æœ‰é•œåƒä»“库凭è¯è¿›è¡Œèº«ä»½éªŒè¯ï¼Œå¹¶æ‹‰å–指定的容器镜åƒã€‚ + +### 通过 kubectl 使用ç§æœ‰é•œåƒä»“库 + +使用 `kubectl`创建工作负载时，你需è¦é…ç½® pod，从而使其 YAML 具有ç§æœ‰é•œåƒä»“库中镜åƒçš„路径。如果 Pod 是在 Rancher UI 中创建的，它åªä¼šè‡ªåŠ¨èŽ·å–对ç§æœ‰é•œåƒä»“库凭è¯çš„访问æƒé™ï¼Œå› æ­¤ä½ è¿˜å¿…须创建和引用镜åƒä»“库密文。 + +密文必须创建在部署工作负载的åŒä¸€å‘½å空间中。 + +下é¢æ˜¯ä¸€ä¸ªç¤ºä¾‹ `pod.yml`，它用于使用ç§æœ‰é•œåƒä»“库的镜åƒçš„工作负载。在此示例中，pod 使用æ¥è‡ª Quay.io çš„é•œåƒï¼Œè€Œä¸” .yml 指定了镜åƒçš„路径。pod 使用存储在å为 `testquay` çš„ Kubernetes 密文中的凭è¯æ¥å‘é•œåƒä»“库进行身份验è¯ï¼Œè¯¥å¯†æ–‡åœ¨ `name` 字段的 `spec.imagePullSecrets` 中指定： + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: private-reg +spec: + containers: + - name: private-reg-container + image: quay.io// + imagePullSecrets: + - name: testquay +``` + +在此示例中，å为 `testquay` 的密文ä½äºŽ Default 命å空间中。 + +ä½ å¯ä»¥é€šè¿‡ `kubectl` 使用ç§æœ‰é•œåƒä»“库凭è¯æ¥åˆ›å»ºå¯†æ–‡ã€‚此命令创建å为 `testquay` 的密文： + +``` +kubectl create secret docker-registry testquay \ + --docker-server=quay.io \ + --docker-username= \ + --docker-password= +``` + +è¦æŸ¥çœ‹å¯†æ–‡æ˜¯å¦‚何存储在 Kubernetes 中的，å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤ï¼š + +``` +kubectl get secret testquay --output="jsonpath={.data.\.dockerconfigjson}" | base64 --decode +``` + +结果如下所示： + +``` +{"auths":{"quay.io":{"username":"","password":"","auth":"c291bXlhbGo6dGVzdGFiYzEyMw=="}}} +``` + +部署工作负载åŽï¼Œä½ å¯ä»¥æ£€æŸ¥é•œåƒæ˜¯å¦å·²æ‹‰å–æˆåŠŸï¼š + +``` +kubectl get events +``` +结果应如下所示： +``` +14s Normal Scheduled Pod Successfully assigned default/private-reg2 to minikube +11s Normal Pulling Pod pulling image "quay.io//" +10s Normal Pulled Pod Successfully pulled image "quay.io//" +``` + +有关详细信æ¯ï¼Œè¯·å‚阅 Kubernetes 文档中的[创建使用你密文的 pod](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md new file mode 100644 index 00000000000..5717ca726a5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md @@ -0,0 +1,18 @@ +--- +title: 添加 Ingress +description: ä½ å¯ä»¥ä¸ºå·¥ä½œè´Ÿè½½æ·»åŠ  Ingress，从而æ供负载å‡è¡¡ã€SSL 终止和基于主机/路径的路由。了解如何添加 Rancher Ingress +--- + +ä½ å¯ä»¥ä¸ºå·¥ä½œè´Ÿè½½æ·»åŠ  Ingress，从而æ供负载å‡è¡¡ã€SSL 终止和基于主机/路径的路由。在项目中使用 Ingress 时，你å¯ä»¥é€šè¿‡è®¾ç½®å…¨å±€ DNS æ¡ç›®æ¥å°† Ingress 主机å编程到外部 DNS。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ  Ingress 的集群，然åŽå•å‡» **Explore**。 +1. 点击**æœåŠ¡å‘现 > Ingresses**。 +1. å•å‡»**创建**。 +1. 从下拉列表中选择一个现有的**命å空间**。 +1. 输入 Ingress çš„**å称**。 +1. [创建](./ingress-configuration.md) Ingress 转å‘**规则**。如果你的任何 Ingress 规则处ç†åŠ å¯†ç«¯å£çš„请求，请添加è¯ä¹¦ä»¥åŠ å¯†/解密通信。 +1. **å¯é€‰**：点击**添加规则**æ¥åˆ›å»ºå…¶ä»– Ingress 规则。例如，在创建 Ingress 规则以引导主机å请求åŽï¼Œä½ å¯èƒ½æƒ³åˆ›å»ºä¸€ä¸ªé»˜è®¤åŽç«¯æ¥å¤„ç† 404。 +1. 点击å³ä¸‹è§’çš„**创建**。 + +**结果**：已将 Ingress 添加到项目中。Ingress 会开始执行你的 Ingress 规则。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md new file mode 100644 index 00000000000..380629411b4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md @@ -0,0 +1,42 @@ +--- +title: Ingress é…ç½® +description: Ingress é…ç½® +--- + +:::note + +在 Kubernetes v1.21 åŠæ›´é«˜ç‰ˆæœ¬ä¸­ï¼ŒNGINX Ingress Controller ä¸å†é»˜è®¤è¿è¡Œåœ¨ hostNetwork 中。它改为将 hostPorts ç”¨äºŽç«¯å£ 80 å’Œç«¯å£ 443，因此你å¯ä»¥å°†å‡†å…¥ Webhook é…置为åªèƒ½é€šè¿‡ ClusterIP 访问。这确ä¿äº†åªèƒ½ä»Žé›†ç¾¤å†…部访问 webhook。 + +由于 controller 的这一更改，默认 RKE1 é…ç½®ä¸å†å°† `hostNetwork` 设置为 `true`。但是，你必须将 `hostNetwork` 设置为 `true` æ‰èƒ½ä½¿åŸºäºŽ TCP å’Œ UDP çš„ Service 正常工作。为此，请[编辑](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#使用-yaml-编辑集群)集群的 YAML 并按照[官方 RKE1 文档](https://rke.docs.rancher.com/config-options/add-ons/ingress-controllers#configuring-network-options)中的步骤æ“作。 + + +::: + +## 指定è¦ä½¿ç”¨çš„主机å + +如果使用此选项，Ingress 会将主机å请求路由到你指定的æœåŠ¡æˆ–工作负载。 + +1. 指定类型为 `Prefix` 的路径并指定路径，例如 `/`。 +1. 添加一个**目标æœåŠ¡**。 +1. **å¯é€‰**：如果你想在将请求å‘é€åˆ°ç‰¹å®šä¸»æœºå路径时指定工作负载或æœåŠ¡ï¼Œè¯·ä¸ºç›®æ ‡æ·»åŠ **路径**。例如，如果你希望将 `www.mysite.com/contact-us` 的请求å‘é€åˆ°ä¸Ž `www.mysite.com` ä¸åŒçš„æœåŠ¡ï¼Œåœ¨**路径**字段中输入 `/contact-us`。通常情况下，你创建的第一æ¡è§„则ä¸åŒ…å«è·¯å¾„。 +1. 输入æ¯ä¸ªç›®æ ‡æ“作的**端å£**å·ã€‚ + +## è¯ä¹¦ + +:::note + +你必须具有 SSL è¯ä¹¦ï¼ŒIngress å¯ä½¿ç”¨è¯¥è¯ä¹¦æ¥åŠ å¯†å’Œè§£å¯†é€šä¿¡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[添加 SSL è¯ä¹¦](../encrypt-http-communication.md)。 + +::: + +1. è¦åˆ›å»º Ingress Controller，å•å‡»**è¯ä¹¦**选项å¡ã€‚ +1. å•å‡»**添加è¯ä¹¦**。 +1. 从下拉列表中选择一个**è¯ä¹¦ - 密文å称**。 +1. 使用加密通信进入主机。 +1. è¦æ·»åŠ ä½¿ç”¨è¯ä¹¦çš„其他主机，请å•å‡»**添加主机**。 + +## 标签和注释 + +添加[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)å’Œ/或[注释](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/)æ¥ä¸ºä½ çš„ Ingress Controller æ供元数æ®ã€‚ + +有关å¯ç”¨çš„注释列表，请å‚阅 [Nginx Ingress Controller 文档](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md new file mode 100644 index 00000000000..c13f8dbf75e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md @@ -0,0 +1,68 @@ +--- +title: "四层和七层负载å‡è¡¡" +description: "Kubernetes 支æŒå››å±‚è´Ÿè½½å‡è¡¡å’Œä¸ƒå±‚è´Ÿè½½å‡è¡¡ã€‚了解对ä¸åŒ deployment 的支æŒ" +--- + +Kubernetes 支æŒå››å±‚è´Ÿè½½å‡è¡¡å’Œä¸ƒå±‚è´Ÿè½½å‡è¡¡ã€‚ + +## 四层负载å‡è¡¡å™¨ + +四层负载å‡è¡¡å™¨ï¼ˆæˆ–外部负载å‡è¡¡å™¨ï¼‰å°†æµé‡è½¬å‘到 Nodeport。四层负载å‡è¡¡å™¨æ”¯æŒè½¬å‘ HTTP å’Œ TCP æµé‡ã€‚ + +通常情况下，四层负载å‡è¡¡å™¨ç”±åº•å±‚云æ供商支æŒï¼Œå› æ­¤ï¼Œå¦‚果你在裸金属æœåŠ¡å™¨å’Œ vSphere 集群上部署 RKE 集群，则ä¸æ”¯æŒå››å±‚è´Ÿè½½å‡è¡¡å™¨ã€‚但是，å•ä¸ª[全局管ç†çš„ config-map](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) å¯ç”¨äºŽåœ¨ NGINX 或第三方 Ingress 上公开æœåŠ¡ã€‚ + +:::note + +ä½ å¯ä»¥ä½¿ç”¨éžäº‘è´Ÿè½½å‡è¡¡å™¨ï¼ˆä¾‹å¦‚ [MetalLB](https://metallb.universe.tf/)）æ¥éƒ¨ç½²é›†ç¾¤ã€‚但是，该用例比云æ供商支æŒçš„四层负载å‡è¡¡å™¨æ›´é«˜çº§ï¼Œè€Œä¸”ä¸å¯ä»¥åœ¨ Rancher 或 RKE 中é…置。 + +::: + +### 四层负载å‡è¡¡æ”¯æŒ + +ä¸åŒåº•å±‚云æ供商对四层负载å‡è¡¡å™¨çš„支æŒæœ‰æ‰€ä¸åŒï¼š + +| 集群部署 | 四层负载å‡è¡¡å™¨æ”¯æŒ | +----------------------------------------------|-------------------------------- +| Amazon EKS | ç”± AWS 云æä¾›å•†æ”¯æŒ | +| Google GKE | ç”± GCE 云æä¾›å•†æ”¯æŒ | +| Azure AKS | ç”± Azure 云æä¾›å•†æ”¯æŒ | +| EC2 上的 RKE | ç”± AWS 云æä¾›å•†æ”¯æŒ | +| DigitalOcean 上的 RKE | å—é™çš„ NGINX 或第三方 Ingress\* | +| vSphere 上的 RKE | å—é™çš„ NGINX 或第三方 Ingress\* | +| 自定义主机上的 RKE
（例如裸金属æœåŠ¡å™¨ï¼‰ | å—é™çš„ NGINX 或第三方 Ingress\* | +| 第三方 MetalLB | å—é™çš„ NGINX 或第三方 Ingress\* | + +\* å¯ä»¥é€šè¿‡å•ä¸ª[全局管ç†çš„ config-map](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) æ¥å…¬å¼€æœåŠ¡ã€‚ + +## 七层负载å‡è¡¡å™¨ + +七层负载å‡è¡¡å™¨ï¼ˆæˆ– Ingress Controller）支æŒåŸºäºŽä¸»æœºå’Œè·¯å¾„çš„è´Ÿè½½å‡è¡¡å’Œ SSL 终止。七层负载å‡è¡¡å™¨ä»…è½¬å‘ HTTP å’Œ HTTPS æµé‡ï¼Œå› æ­¤å®ƒä»¬ä»…侦å¬ç«¯å£ 80 å’Œ 443。亚马逊和谷歌等云æ供商支æŒä¸ƒå±‚è´Ÿè½½å‡è¡¡å™¨ã€‚此外，RKE 集群部署了 Nginx Ingress Controller。 + +### 七层负载å‡è¡¡æ”¯æŒ + +ä¸åŒåº•å±‚云æ供商对七层负载å‡è¡¡å™¨çš„支æŒæœ‰æ‰€ä¸åŒï¼š + +| 集群部署 | 七层负载å‡è¡¡å™¨æ”¯æŒ | +----------------------------------------------|-------------------------------- +| Amazon EKS | ç”± AWS 云æä¾›å•†æ”¯æŒ | +| Google GKE | ç”± GKE 云æä¾›å•†æ”¯æŒ | +| Azure AKS | ä¸æ”¯æŒ | +| EC2 上的 RKE | Nginx Ingress Controller | +| DigitalOcean 上的 RKE | Nginx Ingress Controller | +| vSphere 上的 RKE | Nginx Ingress Controller | +| 自定义主机上的 RKE
（例如裸金属æœåŠ¡å™¨ï¼‰ | Nginx Ingress Controller | + +### 七层负载å‡è¡¡å™¨ä¸­çš„主机å + +一些托管在云上的七层负载å‡è¡¡å™¨ï¼ˆä¾‹å¦‚ AWS 上的 ALB Ingress Controller）会为 Ingress 规则公开 DNS 地å€ã€‚你需è¦ï¼ˆä½¿ç”¨ CNAME）将你的域å映射到七层负载å‡è¡¡å™¨ç”Ÿæˆçš„ DNS 地å€ã€‚ + +其他七层负载å‡è¡¡å™¨ï¼ˆä¾‹å¦‚ Google Load Balancer 或 Nginx Ingress Controller）会直接公开一个或多个 IP 地å€ã€‚Google Load Balancer æ供了一个å¯è·¯ç”±çš„ IP 地å€ã€‚Nginx Ingress Controller 公开了è¿è¡Œ Nginx Ingress Controller 的所有节点的外部 IP。你å¯ä»¥æ‰§è¡Œä»¥ä¸‹ä»»ä¸€æ“作： + +1. é…置你自己的 DNS，从而（使用 A 记录）将你的域å映射到七层负载å‡è¡¡å™¨å…¬å¼€çš„ IP 地å€ã€‚ +2. 让 Rancher 为你的 Ingress 规则生æˆä¸€ä¸ª xip.io 主机å。Rancher 将使用你公开的其中一个 IP（å‡è®¾æ˜¯ `a.b.c.d`）生æˆä¸€ä¸ªä¸»æœºåï¼ˆå³ `..a.b.c.d.xip.io`）。 + +使用 xip.io 的好处是你å¯ä»¥åœ¨åˆ›å»º Ingress 规则åŽç«‹å³èŽ·å¾—一个有效的入å£ç‚¹ URL。此外，如果你设置自己的域å，则需è¦é…ç½® DNS æœåŠ¡å™¨å¹¶ç­‰å¾… DNS 传播。 + +## 相关链接 + +- [创建外部负载å‡è¡¡å™¨](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md new file mode 100644 index 00000000000..d4618d910e1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md @@ -0,0 +1,78 @@ +--- +title: 密文 +--- + +[密文（secret）](https://kubernetes.io/docs/concepts/configuration/secret/#overview-of-secrets)存储æ•æ„Ÿæ•°æ®ï¼Œä¾‹å¦‚密ç ã€ä»¤ç‰Œæˆ–密钥。它们å¯èƒ½åŒ…å«ä¸€ä¸ªæˆ–多个键值对。 + +:::note + +本文介ç»å¯†æ–‡çš„概述。有关设置ç§æœ‰é•œåƒä»“库的详细信æ¯ï¼Œè¯·å‚阅[é•œåƒä»“库](kubernetes-and-docker-registries.md)。 + +::: + +é…置工作负载时，你能够选择è¦åŒ…å«çš„密文。与 ConfigMap 一样，工作负载å¯ä»¥å°†å¯†æ–‡å¼•ç”¨ä¸ºçŽ¯å¢ƒå˜é‡æˆ–å·æŒ‚载。 + +除éžä½œä¸ºå­è·¯å¾„å·æŒ‚载，å¦åˆ™æŒ‚载的密文会自动更新。有关如何传播更新的密文，请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically)。 + +## 在命å空间中创建密文 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ å¯†æ–‡çš„集群，然åŽå•å‡» **Explore**。 +1. è¦å¯¼èˆªåˆ°å¯†æ–‡ï¼Œå•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**。 +1. å•å‡»**创建**。 +1. 选择è¦åˆ›å»ºçš„密文类型。 +1. 为密文选择一个**命å空间**。 +1. 输入密文的**å称**。 + + :::note + + Kubernetes 将密文ã€è¯ä¹¦å’Œé•œåƒä»“库都归类为[密文](https://kubernetes.io/docs/concepts/configuration/secret/)，命å空间中的密文å称ä¸èƒ½é‡å¤ã€‚因此，为了é¿å…冲çªï¼Œå¯†æ–‡çš„å称必须与工作空间中的其他密文ä¸ä¸€æ ·ã€‚ + + ::: + +1. 在**æ•°æ®**中，å•å‡»**添加**以添加键值对。你å¯ä»¥æ ¹æ®éœ€è¦æ·»åŠ ä»»æ„æ•°é‡çš„值。 + + :::tip + + ä½ å¯ä»¥é€šè¿‡å¤åˆ¶å’Œç²˜è´´çš„æ–¹å¼å°†å¤šä¸ªé”®å€¼å¯¹æ·»åŠ åˆ°å¯†æ–‡ä¸­ã€‚ + + ::: + + ![](/img/bulk-key-values.gif) + +1. å•å‡»**ä¿å­˜**。 + +**结果**：密文已添加到你选择的命å空间中。你å¯ä»¥åœ¨ Rancher UI 中å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**æ¥æŸ¥çœ‹å¯†æ–‡ã€‚ + +除éžä½œä¸ºå­è·¯å¾„å·æŒ‚载，å¦åˆ™æŒ‚载的密文会自动更新。有关如何传播更新的密文，请å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically)。 + + +## 在项目中创建密文 + +在 Rancher 2.6 之å‰ï¼Œå¯†æ–‡å¿…须创建在项目级别。现在ä¸å†éœ€è¦é¡¹ç›®çº§åˆ«ï¼Œä½ å¯ä»¥é‡‡ç”¨å‘½å空间级别。因此，Rancher UI 进行了更新以å映这一新功能。但是，你ä»ç„¶å¯ä»¥æŒ‰ç…§éœ€è¦åˆ›å»ºé¡¹ç›®çº§åˆ«çš„密文。请注æ„，你必须先å¯ç”¨`旧版`功能开关并查看å•ä¸ªé¡¹ç›®ã€‚执行以下步骤设置你的项目级别密文： + +1. 在左上角，å•å‡»ä¸‹æ‹‰èœå•ä¸­çš„ **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 转到 `Legacy` 功能开关并å•å‡»**激活**。 +1. 点击左上角下拉èœå•ä¸­çš„ **☰ > 集群管ç†**。 +1. 转到你创建的集群，然åŽå•å‡» **Explore**。 +1. å•å‡» **Legacy > 项目**。 +1. 在顶部导航æ ä¸­è¿›è¡Œè¿‡æ»¤ï¼Œä»Žè€Œä»…查看一个项目。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**密文**。 +1. å•å‡»**添加密文**。 + +**结果**：密文已添加到你选择的项目中。你å¯ä»¥åœ¨ Rancher UI 中å•å‡»**存储 > 密文**或**æ›´å¤šèµ„æº > 核心 > 密文**æ¥æŸ¥çœ‹å¯†æ–‡ã€‚ + +由于项目范围的 Secret 是在项目级别设置的，因此在命å空间级别所åšçš„任何更改都将被覆盖。 + +:::note + +local 集群上的项目级别密文仅在选择å•ä¸ªé¡¹ç›®æ—¶å¯è§ã€‚ + +::: + +## åŽç»­æ“作 + +现在你已将密文添加到命å空间，你å¯ä»¥å°†å…¶æ·»åŠ åˆ°ä½ éƒ¨ç½²çš„工作负载中。 + +有关将密文添加到工作负载的更多信æ¯ï¼Œè¯·å‚阅[部署工作负载](workloads-and-pods/deploy-workloads.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md new file mode 100644 index 00000000000..fe840f7005f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md @@ -0,0 +1,35 @@ +--- +title: 添加 Sidecar +--- + +_sidecar_ 是一个容器，用于扩展或增强 pod 中的主容器。主容器和 Sidecar 共享一个 pod，因此共享相åŒçš„网络空间和存储。你å¯ä»¥ä½¿ç”¨**添加 Sidecar** 选项将 Sidecar 添加到现有工作负载。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ  Sidecar 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 + +1. 找到è¦æ‰©å±•çš„工作负载。选择 **â‹® > + 添加 Sidecar**。 + +1. 输入 Sidecar çš„**å称**。 + +1. 在**通用**中，选择 Sidecar 类型。此选项确定 Sidecar 容器是在主容器之å‰è¿˜æ˜¯ä¹‹åŽéƒ¨ç½²ã€‚ + + - **标准容器**： + + Sidecar 容器部署在主容器之åŽã€‚ + + - **åˆå§‹åŒ–容器**： + + Sidecar 容器部署在主容器之å‰ã€‚ + +1. 在**容器镜åƒ**字段中，输入è¦éƒ¨ç½²æ¥æ”¯æŒä¸»å®¹å™¨çš„容器镜åƒçš„å称。部署时，Rancher 会从 [Docker Hub](https://hub.docker.com/explore/) 拉å–这个镜åƒã€‚输入与 Docker Hub 上完全相åŒçš„å称。 + +1. 设置其余选项。你å¯ä»¥åœ¨[部署工作负载](deploy-workloads.md)中了解它们。 + +1. 点击**å¯åŠ¨**。 + +**结果**：已根æ®ä½ çš„å‚数部署 Sidecar。部署åŽï¼Œä½ å¯ä»¥é€‰æ‹© **â‹® > 编辑** æ¥æŸ¥çœ‹ Sidecar。 + +## 相关链接 + +- [分布å¼ç³»ç»Ÿå·¥å…·åŒ…：å¤åˆå®¹å™¨çš„模å¼](https://kubernetes.io/blog/2015/06/the-distributed-system-toolkit-patterns/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md new file mode 100644 index 00000000000..6fecebc7341 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md @@ -0,0 +1,57 @@ +--- +title: 部署工作负载 +description: 阅读此步骤指å—以部署工作负载。部署工作负载以在一个或多个容器中è¿è¡Œåº”用程åºã€‚ +--- + +部署工作负载以在一个或多个容器中è¿è¡Œåº”用程åºã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡»**创建**。 +1. 选择工作负载的类型。 +1. 选择è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„命å空间。 +1. 为工作负载设置**å称**。 + +1. 在**容器镜åƒ**字段中，输入è¦éƒ¨ç½²åˆ°é¡¹ç›®çš„ Docker é•œåƒçš„å称。你å¯ä»¥é€‰æ‹©åœ¨å‰é¢åŠ ä¸Šé•œåƒä»“库主机（例如 `quay.io`ã€`registry.gitlab.com` 等）。部署时，Rancher 从指定的公共或ç§æœ‰é•œåƒä»“库中拉å–é•œåƒã€‚如果没有æ供镜åƒä»“库主机，Rancher 将从 [Docker Hub](https://hub.docker.com/explore/) 拉å–é•œåƒã€‚输入与镜åƒä»“库æœåŠ¡å™¨ä¸­æ˜¾ç¤ºçš„å称完全相åŒçš„å称，任何所需的路径，也å¯ä»¥é€‰æ‹©æ‰€éœ€çš„标签（例如 `registry.gitlab.com/user/path/image:tag`）。如果没有æ供标签，将自动使用 `latest` 标签。 + +1. 选择现有命å空间，或å•å‡»**添加到新命å空间**并输入新命å空间。 + +1. 点击**添加端å£**进入端å£æ˜ å°„，这让你å¯ä»¥è®¿é—®é›†ç¾¤å†…外的应用程åºã€‚如需更多信æ¯ï¼Œè¯·å‚阅 [Service](../../../../pages-for-subheaders/workloads-and-pods.md#services)。 + +1. é…置其余选项： + + - **环境å˜é‡** + + ä½ å¯ä»¥åœ¨è¿™é‡Œä¸ºå·¥ä½œè´Ÿè½½æŒ‡å®šçŽ¯å¢ƒå˜é‡ä»¥å³æ—¶ä½¿ç”¨å®ƒä»¬ï¼Œæˆ–者从其他æ¥æºï¼ˆä¾‹å¦‚密文或 [ConfigMap](../configmaps.md)）拉å–它们。 + + - **节点调度** + - **å¥åº·æ£€æŸ¥** + - **å·** + + 在此处为你的工作负载添加存储。你å¯ä»¥æ‰‹åŠ¨æŒ‡å®šè¦æ·»åŠ çš„å·ï¼Œä½¿ç”¨æŒä¹…å·å£°æ˜Žä¸ºå·¥ä½œè´Ÿè½½åŠ¨æ€åˆ›å»ºå·ï¼Œæˆ–从 [ConfigMap](../configmaps.md) 等文件中读å–è¦ä½¿ç”¨çš„å·æ•°æ®ã€‚ + + 部署 StatefulSet 时，应在使用æŒä¹…å·æ—¶ä½¿ç”¨å·å£°æ˜Žæ¨¡æ¿ã€‚这将确ä¿åœ¨æ‰©å±• StatefulSet 时能动æ€åˆ›å»ºæŒä¹…å·ã€‚ + + - **扩展/å‡çº§ç­–ç•¥** + + :::note AWS å·æ³¨æ„事项： + + è¦æŒ‚è½½ Amazon EBS å·ï¼š + + - 在 [AWS](https://aws.amazon.com/) 中，节点必须ä½äºŽåŒä¸€å¯ç”¨åŒºä¸­å¹¶å…·æœ‰é™„加/分离å·çš„ IAM æƒé™ã€‚ + + - 集群必须使用 [AWS 云æ供商](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#aws)选项。有关å¯ç”¨æ­¤é€‰é¡¹çš„更多信æ¯ï¼Œè¯·å‚阅[创建 AWS EC2 集群](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md)或[创建自定义集群](../../../../pages-for-subheaders/use-existing-nodes.md)。 + + ::: + +1. 点击**显示高级选项**并é…置： + + - **命令** + - **网络** + - **标签 & 注释** + - **安全和主机é…ç½®** + +1. 点击**å¯åŠ¨**。 + +**结果**：工作负载已部署到选定的命å空间。你å¯ä»¥åœ¨é¡¹ç›®çš„**工作负载**视图查看工作负载的状æ€ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md new file mode 100644 index 00000000000..c12dfdd469a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md @@ -0,0 +1,14 @@ +--- +title: 回滚工作负载 +--- + +在调试或å‡çº§ä¸é¡ºåˆ©æ—¶ï¼Œä½ å¯èƒ½éœ€è¦å°†åº”用程åºå›žæ»šåˆ°å…ˆå‰çš„版本。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. 找到è¦å›žæ»šçš„工作负载，然åŽé€‰æ‹© **â‹® > 回滚**。 + +1. 选择è¦å›žæ»šåˆ°çš„修订版。å•å‡»**回滚**。 + +**结果**：你的工作负载已æ¢å¤åˆ°ä½ é€‰æ‹©çš„å…ˆå‰ç‰ˆæœ¬ã€‚等待几分钟待æ“作完æˆã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md new file mode 100644 index 00000000000..6d2313f5514 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md @@ -0,0 +1,21 @@ +--- +title: å‡çº§å·¥ä½œè´Ÿè½½ +--- + +如果 Docker Hub å‘布新版本的应用程åºé•œåƒï¼Œä½ å¯ä»¥å°†è¿è¡Œæ—§ç‰ˆåº”用程åºçš„工作负载å‡çº§åˆ°æ–°ç‰ˆæœ¬ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦å‡çº§å·¥ä½œè´Ÿè½½çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 + +1. 找到è¦å‡çº§çš„工作负载并选择 **â‹® > 编辑é…ç½®**。 + +1. æ›´æ–°**容器镜åƒ**å’Œè¦æ›´æ”¹çš„选项。 + +1. 查看和编辑工作负载的**扩展/å‡çº§ç­–ç•¥**。 + + 这些选项能控制å‡çº§å¦‚何在è¿è¡Œçš„容器中进行。例如，对于å¯æ‰©å±•çš„ deployment，你å¯ä»¥é€‰æ‹©æ˜¯å¦è¦åœ¨éƒ¨ç½²æ–° Pod 之å‰åœæ­¢æ—§ Pod，以åŠé€‰æ‹©å‡çº§çš„批数é‡ã€‚ + +1. å•å‡»**ä¿å­˜**。 + +**结果**：工作负载会根æ®ä½ çš„设置æ¥å‡çº§å®¹å™¨ã€‚请注æ„，扩展 deployment 或更新å‡çº§/扩展策略ä¸ä¼šå¯¼è‡´ Pod é‡æ–°åˆ›å»ºã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md new file mode 100644 index 00000000000..15263fff292 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md @@ -0,0 +1,43 @@ +--- +title: Rancher Agent +--- + +Rancher 管ç†çš„集群上部署了两ç§ä¸åŒçš„ Agent 资æºï¼š + +- [cattle-cluster-agent](#cattle-cluster-agent) +- [cattle-node-agent](#cattle-node-agent) + +有关 Rancher Server 如何é…置集群并与集群通信的概述，请å‚阅[产å“架构](../../../pages-for-subheaders/rancher-manager-architecture.md)。 + +### cattle-cluster-agent + +`cattle-cluster-agent` 用于连接 [Rancher å¯åŠ¨çš„ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群的 Kubernetes API。`cattle-cluster-agent` 使用 Deployment 资æºè¿›è¡Œéƒ¨ç½²ã€‚ + +### cattle-node-agent + +`cattle-node-agent` 用于在执行集群æ“作时与 [Rancher å¯åŠ¨çš„ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群中的节点进行交互。集群æ“作包括å‡çº§ Kubernetes 版本和创建/æ¢å¤ etcd 快照。`cattle-node-agent` 使用 DaemonSet 资æºè¿›è¡Œéƒ¨ç½²ï¼Œä»¥ç¡®ä¿èƒ½åœ¨æ¯ä¸ªèŠ‚点上è¿è¡Œã€‚当 `cattle-cluster-agent` ä¸å¯ç”¨æ—¶ï¼Œ`cattle-node-agent` å¯ä»¥ä½œä¸ºå¤‡é€‰æ–¹æ¡ˆï¼Œç”¨æ¥è¿žæŽ¥ [Rancher å¯åŠ¨çš„ Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群的 Kubernetes API。 + +### 调度规则 + +`cattle-cluster-agent` 使用一组固定的容å¿åº¦ï¼Œæˆ–基于应用于 control plane 节点的污点动æ€æ·»åŠ çš„容å¿åº¦ã€‚è¿™ç§ç»“æž„å…许[基于污点进行驱é€](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions)为 `cattle-cluster-agent` 正常工作。 + +如果集群中存在 control plane 节点，则默认容å¿åº¦å°†æ›¿æ¢ä¸ºä¸Ž control plane 节点的污点匹é…的容å¿åº¦ã€‚默认的容å¿åº¦å¦‚下。 + +| 组件 | nodeAffinity nodeSelectorTerms | nodeSelector | 容å¿åº¦ | +| ---------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| `cattle-cluster-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | **注æ„**：这些是默认容å¿åº¦ï¼Œå¹¶å°†æ›¿æ¢ä¸ºä¸Ž controlplane 节点的污点匹é…的容å¿åº¦ã€‚

`effect:NoSchedule`
`key:node-role.kubernetes.io/controlplane`
`value:true`

`effect:NoSchedule`
`key:node-role.kubernetes.io/control-plane`
`operator:Exists`

`effect:NoSchedule`
`key:node-role.kubernetes.io/master`
`operator:Exists` | +| `cattle-node-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | `operator:Exists` | + +`cattle-cluster-agent` Deployment 使用 `preferredDuringSchedulingIgnoredDuringExecution` 的首选调度规则，倾å‘于在具有 `controlplane` 节点的节点上进行调度。当集群中没有å¯è§çš„ controlplane 节点时（通常是使用[æ供商托管的 Kubernetes 的集群](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)），你å¯ä»¥åœ¨èŠ‚点上添加 `cattle.io/cluster-agent=true` 标签，从而优先将 `cattle-cluster-agent` pod 调度到该节点。 + +有关调度规则的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes：将 Pod 分é…给节点](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/)。 + +`preferredDuringSchedulingIgnoredDuringExecution` é…置如下表所示： + +| æƒé‡ | è¡¨è¾¾å¼ | +| ------ | ------------------------------------------------ | +| 100 | `node-role.kubernetes.io/controlplane:In:"true"` | +| 100 | `node-role.kubernetes.io/control-plane:In:"true"` | +| 100 | `node-role.kubernetes.io/master:In:"true"` | +| 1 | `cattle.io/cluster-agent:In:"true"` | + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md new file mode 100644 index 00000000000..3941f3b37c3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md @@ -0,0 +1,31 @@ +--- +title: RKE1 å’Œ RKE2 差​​异 +--- + +RKE2，也称为 RKE Government，是一个完全符åˆæ ‡å‡†çš„ Kubernetes å‘行版，它专注于安全性和åˆè§„性。它被认为是 Rancher Kubernetes Engineï¼ˆå³ RKE1）的下一个迭代。 + +RKE1 å’Œ RKE2 有一些细微的差异，本文将é‡ç‚¹ä»‹ç»è¿™äº›å·®å¼‚。 + +### controlplane 组件 + +RKE1 使用 Docker æ¥éƒ¨ç½²å’Œç®¡ç† controlplane 组件，还使用 Docker 作为 Kubernetes 的容器è¿è¡Œæ—¶ã€‚相比之下，RKE2 å°† controlplane 组件作为由 kubelet 管ç†çš„é™æ€ pod å¯åŠ¨ã€‚RKE2 的容器è¿è¡Œæ—¶æ˜¯ Containerd，它å…许 Mirror 容器镜åƒä»“库等内容。使用 Docker çš„ RKE1 ä¸å…许 Mirror。 + +### Cluster API + +RKE2/K3s é…置是基于 Cluster API (CAPI) 上游框架之上构建的，这导致 RKE2 é…置的集群的行为通常与 RKE1 é…置的集群ä¸åŒã€‚ + +如果你在 RKE2 中更改集群é…置，节点**å¯èƒ½**会é‡æ–°é…置。这是由 CAPI 控制器而ä¸æ˜¯ Rancher 控制的。请注æ„，etcd 节点ä¸é€‚用相åŒçš„行为。 + +以下是一些å¯èƒ½å¯¼è‡´ä¸Šè¿°è¡Œä¸ºçš„é…置更改示例： + +- 编辑集群并å¯ç”¨`删除å‰æ¸…空`时，会删除现有的 controlplane 节点和 worker 并创建新节点。 + +如果你是习惯于 RKE1 é…置的用户，请注æ„æ–°çš„ RKE2 行为。 + +### åè¯è§£é‡Š + +从 RKE1 到 RKE2，æŸäº›æœ¯è¯­å·²æ›´æ”¹æˆ–å·²ä¸å†ä½¿ç”¨ã€‚例如，在 RKE1中，你使用**节点模æ¿**，而在 RKE2 中，你å¯ä»¥åœ¨åˆ›å»ºæˆ–编辑集群时é…置集群节点池。å¦ä¸€ä¸ªä¾‹å­æ˜¯ RKE1 中的**节点池（node pool）** 现在在 RKE2 中称为**主机池（machine pool）**。 + + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md new file mode 100644 index 00000000000..39f8b418151 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md @@ -0,0 +1,99 @@ +--- +title: 创建 DigitalOcean 集群 +--- + +在本节中，你将学习如何使用 Rancher 在 DigitalOcean 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 DigitalOcean 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ DigitalOcean 中é…置新节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 DigitalOcean 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + + + + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **DigitalOcean**。 +1. 输入你的 Digital Ocean 凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 DigitalOcean 创建[节点模æ¿](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿) 会å…许 Rancher 在 DigitalOcean 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **DigitalOcean**。 +1. 填写 DigitalOcean 的节点模æ¿ã€‚有关填写表å•çš„帮助，请å‚阅 [DigitalOcean 节点模æ¿é…ç½®](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md)。 + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **DigitalOcean**。 +1. 输入**集群å称**。 +1. 将一个或多个节点池添加到你的集群。将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +1. 在**集群é…ç½®**中，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +### 1. åˆ›å»ºäº‘å‡­è¯ + +如果你已ç»æœ‰ä¸€ç»„å¯ç”¨çš„云凭è¯ï¼Œè¯·è·³è¿‡æ­¤éƒ¨åˆ†ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **DigitalOcean**。 +1. 输入你的 Digital Ocean 凭è¯ã€‚ +1. å•å‡»**创建**。 + +### 2. 创建你的集群 + +使用 Rancher 在 DigitalOcean 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 将开关切æ¢åˆ° **RKE2/K3s**。 +1. å•å‡» **DigitalOcean**。 +1. 选择一个**云凭è¯**。如果存在多个则需è¦é€‰æ‹©ã€‚å¦åˆ™ï¼Œå®ƒæ˜¯é¢„选的。 +1. 输入**集群å称**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个主机池。请å‚阅[最佳实践](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点角色)了解角色分é…和计数的建议。 + 1. 为æ¯ä¸ªä¸»æœºæ± å®šä¹‰ä¸»æœºé…置。有关é…置选项的信æ¯ï¼Œè¯·å‚阅 [DigitalOcean 主机é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md)。 +1. 使用**集群é…ç½®**，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。有关é…置集群的帮助，请å‚阅 [RKE2 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md new file mode 100644 index 00000000000..b2918cf475a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md @@ -0,0 +1,283 @@ +--- +title: 创建 Amazon EC2 集群 +description: 了解使用 Rancher 创建 Amazon EC2 集群所需的先决æ¡ä»¶å’Œæ­¥éª¤ +--- + +在本节中，你将学习如何使用 Rancher 在 Amazon EC2 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 EC2 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ EC2 中é…置新节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 EC2 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +### 先决æ¡ä»¶ + +- **AWS EC2 访问密钥和密文密钥**，用于创建实例。请å‚阅 [Amazon 文档：创建访问密钥](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)æ¥åˆ›å»ºè®¿é—®å¯†é’¥å’Œå¯†æ–‡å¯†é’¥ã€‚ +- **已创建 IAM ç­–ç•¥**，用于为用户添加的访问密钥和密文密钥。请å‚阅 [Amazon 文档：创建 IAM 策略（控制å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start)æ¥åˆ›å»º IAM 策略。å‚阅下é¢çš„三个示例 JSON 策略： + - [IAM 策略示例](#iam-策略示例) + - [带有 PassRole çš„ IAM 策略示例](#带有-passrole-çš„-iam-策略示例)（如果è¦ä½¿ç”¨ [Kubernetes 云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)，或将 IAM é…置文件传递给实例，则需è¦ï¼‰ + - [å…许加密 EBS å·çš„ IAM 策略示例](#å…许加密-ebs-å·çš„-iam-策略示例) +- 为用户添加 **IAM ç­–ç•¥æƒé™**。请å‚阅 [Amazon 文档：为用户添加æƒé™ï¼ˆæŽ§åˆ¶å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console)，æ¥å°†æƒé™æ·»åŠ ç»™ç”¨æˆ·ã€‚ + +## 创建 EC2 集群 + +创建集群的步骤因 Rancher 版本而异。 + + + + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯å’Œ EC2 çš„ä¿¡æ¯æ¥åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯å’Œ-ec2-çš„ä¿¡æ¯æ¥åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Amazon**。 +1. 输入云凭è¯çš„å称。 +1. 在**默认区域**字段中，选择集群节点所在的 AWS 区域。 +1. 输入 AWS EC2 **Access Key** å’Œ **Secret Key**。 +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯å’Œ EC2 çš„ä¿¡æ¯æ¥åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 EC2 创建[节点模æ¿](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 EC2 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. 填写 EC2 的节点模æ¿ã€‚有关填写表å•çš„帮助，请å‚阅 [EC2 节点模æ¿é…ç½®](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md)。 +1. å•å‡»**创建**。 + + :::note + + 如果è¦ä½¿ç”¨[åŒæ ˆ](https://kubernetes.io/docs/concepts/services-networking/dual-stack/)功能，请关注其他é¢å¤–çš„[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + + ::: + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +将一个或多个节点池添加到你的集群。有关节点池的更多信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Amazon EC2**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个节点池。为æ¯ä¸ªèŠ‚点池选择你已创建的节点模æ¿ã€‚有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +1. 点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 使用**集群选项**选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。å‚è§[选择云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)æ¥é…ç½® Kubernetes 云æ供商。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + + :::note + + 如果è¦ä½¿ç”¨[åŒæ ˆ](https://kubernetes.io/docs/concepts/services-networking/dual-stack/)功能，请关注其他é¢å¤–çš„[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + + ::: + +1. å•å‡»**创建**。 + + + + +### 1. åˆ›å»ºäº‘å‡­è¯ + +如果你已ç»æœ‰ä¸€ç»„å¯ç”¨çš„云凭è¯ï¼Œè¯·è·³è¿‡æ­¤éƒ¨åˆ†ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Amazon**。 +1. 输入云凭è¯çš„å称。 +1. 在**默认区域**字段中，选择集群节点所在的 AWS 区域。 +1. 输入 AWS EC2 **Access Key** å’Œ **Secret Key**。 +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 创建你的集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 将开关切æ¢åˆ° **RKE2/K3s**。 +1. å•å‡» **Amazon EC2**。 +1. 选择一个**云凭è¯**。如果存在多个则需è¦é€‰æ‹©ã€‚å¦åˆ™ï¼Œå®ƒæ˜¯é¢„选的。 +1. 输入**集群å称**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个主机池。请å‚阅[最佳实践](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点角色)了解角色分é…和计数的建议。 + 1. 为æ¯ä¸ªä¸»æœºæ± å®šä¹‰ä¸»æœºé…置。有关é…置选项的信æ¯ï¼Œè¯·å‚阅 [EC2 主机é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md)。 +1. 使用**集群é…ç½®**，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。有关é…置集群的帮助，请å‚阅 [RKE2 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +### å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 + +## IAM ç­–ç•¥ + +### IAM 策略示例 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` + +### 带有 PassRole çš„ IAM 策略示例 + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*", + "arn:aws:iam::AWS_ACCOUNT_ID:role/YOUR_ROLE_NAME" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` +### å…许加密 EBS å·çš„ IAM 策略示例 +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:Encrypt", + "kms:DescribeKey", + "kms:CreateGrant", + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:CreateSnapshot" + ], + "Resource": [ + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:snapshot/*", + "arn:aws:kms:REGION:AWS_ACCOUNT_ID:key/KMS_KEY_ID" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots" + ], + "Resource": "*" + } + ] +} +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md new file mode 100644 index 00000000000..a3af5bca578 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md @@ -0,0 +1,141 @@ +--- +title: 创建 Azure 集群 +--- + +在本节中，你将学习如何通过 Rancher 在 Azure 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 Azure 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ Azure 中é…置新节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 Azure 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +:::caution + +当 Rancher RKE 集群在 Azure 中è¿è¡Œå¹¶ä¸”å‰é¢æœ‰ Azure è´Ÿè½½å‡è¡¡å™¨æ—¶ï¼Œå‡ºç«™æµé‡ä¼šå¤±è´¥ã€‚问题的解决方法如下： + +- 终止内部负载å‡è¡¡å™¨ä¸Šçš„ SSL/TLS +- 使用 L7 è´Ÿè½½å‡è¡¡å™¨ + +详情请å‚è§ [Azure è´Ÿè½½å‡è¡¡å™¨é™åˆ¶](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations)。 + +::: + +有关 Rancher 在 Azure 节点上安装的 Kubernetes 集群的更多信æ¯ï¼Œè¯·å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + +有关é…ç½® Azure 节点模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅 [Azure 节点模æ¿é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md)。 + +- [Azure 中的准备工作](#azure-中的准备工作) +- [创建 Azure 集群](#创建-azure-集群) + +## Azure 中的准备工作 + +在使用 Azure 等云厂商在 Rancher 中创建节点模æ¿ä¹‹å‰ï¼Œå¿…须先é…ç½® Rancher 以å…许æ“作 Azure 订阅中的资æºã€‚ + +为此，首先在 Azure çš„ **Active Directory (AD)** 中创建一个新的 Azure **service principal (æœåŠ¡ä¸»ä½“，缩写是 SP)**ï¼Œå®ƒæ˜¯å…·æœ‰ç®¡ç† Azure æƒé™çš„应用用户。 + +以下是创建æœåŠ¡ä¸»ä½“å¿…é¡»è¿è¡Œçš„ `az cli` 脚本模æ¿ï¼Œä½ å¿…须在其中输入你的æœåŠ¡ä¸»ä½“å称ã€è§’色和范围： + +``` +az ad sp create-for-rbac \ + --name="" \ + --role="Contributor" \ + --scopes="/subscriptions/" +``` + +创建æœåŠ¡ä¸»ä½“åŽä¼šè¿”回三个标识信æ¯ï¼Œåˆ†åˆ«æ˜¯*应用 ID（也称为客户端 ID）*，以åŠ*客户端密ç *。为 Azure 创建节点模æ¿æ—¶å°†ä½¿ç”¨è¿™äº›ä¿¡æ¯ã€‚ + +## 创建 Azure 集群 + + + + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 输入你的 Azure 凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 Azure 创建[节点模æ¿](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 Azure 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **Azure**。 +1. 填写 Azure 的节点模æ¿ã€‚有关填写表å•çš„帮助，请å‚阅 [Azure 节点模æ¿é…ç½®](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md)。 + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +使用 Rancher 在 Azure 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 输入**集群å称**。 +1. 将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池åŠå…¶æœ€ä½³å®žè·µçš„更多信æ¯ï¼Œè¯·å‚阅[本节](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +1. 在**集群é…ç½®**中，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +### 1. åˆ›å»ºäº‘å‡­è¯ + +如果你已ç»æœ‰ä¸€ç»„å¯ç”¨çš„云凭è¯ï¼Œè¯·è·³è¿‡æ­¤éƒ¨åˆ†ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **Azure**。 +1. 输入你的 Azure 凭è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 创建你的集群 + +使用 Rancher 在 Azure 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. 将开关切æ¢åˆ° **RKE2/K3s**。 +1. å•å‡» **Azure**。 +1. 选择一个**云凭è¯**。如果存在多个则需è¦é€‰æ‹©ã€‚å¦åˆ™ï¼Œå®ƒæ˜¯é¢„选的。 +1. 输入**集群å称**。 +1. 为æ¯ä¸ª Kubernetes 角色创建一个主机池。请å‚阅[最佳实践](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点角色)了解角色分é…和计数的建议。 + 1. 为æ¯ä¸ªä¸»æœºæ± å®šä¹‰ä¸»æœºé…置。有关é…置选项的信æ¯ï¼Œè¯·å‚阅 [Azure 主机é…ç½®å‚考](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md)。 +1. 使用**集群é…ç½®**，选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。有关é…置集群的帮助，请å‚阅 [RKE2 集群é…ç½®å‚考](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md)。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. å•å‡»**创建**。 + + + + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +### å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md new file mode 100644 index 00000000000..c360364aa91 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md @@ -0,0 +1,92 @@ +--- +title: 在 Nutanix AOS 中é…ç½® Kubernetes 集群 +--- + +è¦ä½¿ç”¨ Rancher 在 Nutanix AOS (AHV) 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群： + +1. 找到 Rancher 的内置 Nutanix [主机驱动并激活它](../../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#激活åœç”¨ä¸»æœºé©±åŠ¨)。 + +1. 创建一个节点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ Nutanix AOS 中é…置节点。 + +1. 在 Rancher 中创建 Nutanix AOS 集群。é…置新集群时，你需è¦ä¸ºå®ƒå®šä¹‰èŠ‚点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +有关é…ç½® Nutanix AOS 节点模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅 [Nutanix AOS 节点模æ¿é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md)。 + +有关在 Rancher 中é…ç½® RKE Kubernetes 集群的详细信æ¯ï¼Œè¯·å‚阅[集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 + +- [Nutanix AOS 中的准备工作](#nutanix-aos-中的准备工作) +- [创建 Nutanix AOS 集群](#创建-nutanix-aos-集群) + +## Nutanix AOS 中的准备工作 + +下文介ç»äº†è®¾ç½® Nutanix AOS çš„è¦æ±‚，以便 Rancher å¯ä»¥é…置虚拟机和集群。 + +:::note + +节点模æ¿å·²ä½¿ç”¨ Nutanix AOS 5.20.2 å’Œ 6.0.1 版本进行记录和测试。 + +::: + +### 在 Nutanix AOS ä¸­åˆ›å»ºå‡­è¯ + +在继续创建集群之å‰ï¼Œè¯·ç¡®ä¿ä½ æ‹¥æœ‰å…·æœ‰ç®¡ç†å‘˜æƒé™çš„ [Nutanix Prism Central 用户账å·](https://portal.nutanix.com/page/documents/details?targetId=Nutanix-Security-Guide-v6_0:wc-user-create-wc-t.html)。设置节点模æ¿æ—¶ï¼Œæ¨¡æ¿å°†éœ€è¦ä½¿ç”¨è¿™äº›å‡­è¯ã€‚ + +### 网络æƒé™ + +必须确ä¿è¿è¡Œ Rancher Server 的主机能够建立以下网络连接： + +- 连接到 Nutanix Prism Central APIï¼ˆé€šå¸¸æ˜¯ç«¯å£ 9440/TCP）。 +- 连接到创建的 VM 上的 22/TCP å’Œ 2376/TCP 端å£ã€‚ + +有关在基础设施æ供商上创建节点的端å£è¦æ±‚，请å‚阅[节点网络è¦æ±‚](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +### VM-VM å亲和性策略 + +建议设置 [VM-VM å亲和性策略](https://portal.nutanix.com/page/documents/details?targetId=AHV-Admin-Guide-v6_1:ahv-vm-anti-affinity-t.html)。这些规则å…许分é…了 etcd å’Œ controlplane 角色的虚拟机在分é…ç»™ä¸åŒèŠ‚点池时，在å•ç‹¬çš„ AHV 主机上è¿è¡Œã€‚è¿™ç§åšæ³•å¯ç¡®ä¿å•ä¸ªç‰©ç†æœºçš„æ•…éšœä¸ä¼šå½±å“这些平é¢çš„å¯ç”¨æ€§ã€‚ + +## 创建 Nutanix AOS 集群 + +1. [创建节点模æ¿](#1-创建节点模æ¿) +2. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#2-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 Nutanix AOS 创建[节点模æ¿](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 Nutanix AOS 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**创建**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **Nutanix**。 +1. 填写 Nutanix AOS 的节点模æ¿ã€‚有关填写表格的帮助，请å‚阅 Nutanix AOS 节点模æ¿[é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md)。 +1. å•å‡»**创建**。 + +### 2. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +使用 Rancher 在 Nutanix AOS 中创建 Kubernetes 集群。 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **Nutanix**。 +1. 输入**集群å称**，然åŽç‚¹å‡»**继续**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 使用**集群选项**选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)。 +1. 检查并确认你的选项。然åŽå•å‡»**创建**。 + +**结果**：集群已创建，并处于 **Provisioning** 状æ€ã€‚Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ + +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md new file mode 100644 index 00000000000..6872dae6a02 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md @@ -0,0 +1,162 @@ +--- +title: 创建 vSphere è™šæ‹Ÿæœºæ¨¡æ¿ +--- + +è¦é‡å¤ä¸”å¯é åœ°åˆ›å»ºè™šæ‹Ÿæœºé€šå¸¸éžå¸¸å›°éš¾ã€‚VMware vSphere 支æŒæž„建å¯ä»¥è½¬æ¢ä¸ºæ¨¡æ¿çš„虚拟机。然åŽï¼Œä½ å¯ä»¥ä½¿ç”¨è¯¥æ¨¡æ¿æ¥åˆ›å»ºé…置相åŒçš„虚拟机。Rancher 会在节点池中利用此功能æ¥åˆ›å»ºç›¸åŒçš„ RKE1 å’Œ RKE2 节点。 + +为了使用模æ¿åˆ›å»ºæ–°çš„虚拟机，Rancher 有虚拟机必须预先安装的[特定è¦æ±‚](#è¦æ±‚)。根æ®è¿™äº›è¦æ±‚é…置虚拟机åŽï¼Œä½ éœ€è¦[准备虚拟机](#准备虚拟机)，然åŽå†[创建模æ¿](#创建模æ¿)。准备工作完æˆåŽï¼Œè™šæ‹Ÿæœºå¯ä»¥[转æ¢ä¸ºæ¨¡æ¿](#转æ¢ä¸ºæ¨¡æ¿)并[移动到内容库中](#移动到内容库)ï¼Œç„¶åŽ Rancher 节点池就å¯ä»¥ä½¿ç”¨å®ƒäº†ã€‚ + + +## è¦æ±‚ + +Linux å’Œ Windows 虚拟机都需è¦ç‰¹å®šå·¥å…·æ‰èƒ½ä¾› vSphere 主机驱动使用。最关键的ä¾èµ–项是 Linux çš„ [cloud-init](https://cloud-init.io/) å’Œ Windows çš„ [cloudbase-init](https://cloudbase.it/cloudbase-init/)。二者都用于通过é…置主机å和设置 SSH 访问以åŠé»˜è®¤ Rancher 用户æ¥é…置虚拟机。如果需è¦å…¶ä»–é…置，用户å¯ä»¥æ ¹æ®éœ€è¦æ·»åŠ å…¶ä»–内容。此外，下é¢åˆ—出了其他è¦æ±‚以供å‚考。 + +:::note + +如果你有任何特定的防ç«å¢™è§„则或é…置，则需è¦åœ¨åˆ›å»ºæ¨¡æ¿ä¹‹å‰å°†å…¶æ·»åŠ åˆ°è™šæ‹Ÿæœºã€‚ + +::: + +## Linux ä¾èµ–项 + +下é¢åˆ—出了需è¦åœ¨æ¨¡æ¿ä¸Šå®‰è£…的包。ä¸åŒçš„å‘行版对应的å称会略有ä¸åŒï¼Œä¾‹å¦‚，æŸäº›å‘行版会默认æ供。 + +* curl +* wget +* git +* net-tools +* unzip +* apparmor-parser +* ca-certificates +* cloud-init +* cloud-guest-utils +* cloud-image-utils +* growpart +* cloud-initramfs-growroot +* open-iscsi +* openssh-server +* [open-vm-tools](https://docs.vmware.com/en/VMware-Tools/11.3.0/com.vmware.vsphere.vmwaretools.doc/GUID-8B6EA5B7-453B-48AA-92E5-DB7F061341D1.html) + +## Windows ä¾èµ–项 + +模æ¿ä¸Šéœ€è¦å®‰è£…的包如下： + +* Windows 容器功能 +* [cloudbase-init](https://cloudbase.it/cloudbase-init/#download) +* [Docker EE](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server#install-docker) - ä»…é™ RKE1 + +:::note + +RKE1 å’Œ RKE2 对应的 Windows 模æ¿çš„é…置有所ä¸åŒï¼š + +- RKE1 使用 Docker，因此任何 RKE1 模æ¿éƒ½éœ€è¦é¢„先安装 Docker EE +- RKE2 ä¸éœ€è¦ Docker EE，因此ä¸éœ€è¦å®‰è£… + +::: + +## åˆ›å»ºæ¨¡æ¿ + +ä½ å¯ä»¥æ‰‹åŠ¨åˆ›å»ºè™šæ‹Ÿæœºï¼Œä¹Ÿå¯ä»¥ä½¿ç”¨[其他替代方法](#手动创建的替代方案)æ¥åˆ›å»ºè™šæ‹Ÿæœºã€‚ + +### 手动创建 +1. 在 VMware 中按照[这些说明](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AE8AFBF1-75D1-4172-988C-378C35C9FAF2.html)手动创建虚拟机。虚拟机è¿è¡ŒåŽï¼Œä½ å¯ä»¥æ‰‹åŠ¨å®‰è£…上é¢åˆ—出的ä¾èµ–项，以便为 vSphere 主机驱动正确é…置虚拟机。 +2. æ ¹æ®ä½ çš„特定环境和è¦æ±‚按需定制。 +3. 在创建模æ¿ä¹‹å‰è¿›è¡Œæœ€åŽçš„准备工作。 + +### 手动创建的替代方案 + +下é¢åˆ—出了创建虚拟机的其他替代选项： + +* [VMware PowerCLI](https://developer.vmware.com/powercli) +* [Packer](https://www.packer.io/) +* [SaltStack](https://saltproject.io/) +* [Ansible](https://www.ansible.com/) + +Packer 是一ç§å¸¸ç”¨çš„替代方案。有关将其与 vSphere 一起使用的示例，请å‚阅[å‚考](https://github.com/vmware-samples/packer-examples-for-vsphere)。 + +## 准备虚拟机 + +创建具有所有必需ä¾èµ–项（以åŠä»»ä½•å…¶ä»–必需项）的虚拟机åŽï¼Œä½ å¿…须执行最关键的下一个步骤，å³å‡†å¤‡å°†è™šæ‹Ÿæœºè½¬æ¢ä¸ºæ¨¡æ¿ã€‚此准备步骤会é‡ç½®å…³é”®æ•°æ®ï¼ˆä¾‹å¦‚虚拟机主机åã€IP 等）以防止这些信æ¯è¢«å¸¦å…¥æ–°è™šæ‹Ÿæœºã€‚如果你无法执行此步骤，你也å¯ä»¥åˆ›å»ºä¸€ä¸ªå…·æœ‰ç›¸åŒä¸»æœºåã€IP 地å€ç­‰çš„虚拟机。 + +请注æ„，Linux å’Œ Windows 对应的准备步骤有所ä¸åŒã€‚ + +### Linux 准备 + +以下命令将在 Linux 中é‡ç½®ä½ çš„虚拟机： + +```bash +# 清ç†æ—¥å¿—。 +if [ -f /var/log/audit/audit.log ]; then + cat /dev/null > /var/log/audit/audit.log +fi +if [ -f /var/log/wtmp ]; then + cat /dev/null > /var/log/wtmp +fi +if [ -f /var/log/lastlog ]; then + cat /dev/null > /var/log/lastlog +fi + +# æ¸…ç† udev 规则。 +if [ -f /etc/udev/rules.d/70-persistent-net.rules ]; then + rm /etc/udev/rules.d/70-persistent-net.rules +fi + +# æ¸…ç† /tmp 路径。 +rm -rf /tmp/* +rm -rf /var/tmp/* + +# æ¸…ç† SSH 主机密钥。 +rm -f /etc/ssh/ssh_host_* + +# æ¸…ç† machine-id。 +truncate -s 0 /etc/machine-id +rm /var/lib/dbus/machine-id +ln -s /etc/machine-id /var/lib/dbus/machine-id + +# æ¸…ç† shell 历å²ã€‚ +unset HISTFILE +history -cw +echo > ~/.bash_history +rm -fr /root/.bash_history + +# 截断主机åã€ä¸»æœºå’Œ resolv.conf，并将主机å设置为 localhost。 +truncate -s 0 /etc/{hostname,hosts,resolv.conf} +hostnamectl set-hostname localhost + +# æ¸…ç† cloud-init。 +cloud-init clean -s -l +``` + +### Windows 准备 + +Windows 有一个å为 [sysprep](https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/sysprep--generalize--a-windows-installation) 的实用程åºï¼Œç”¨äºŽä¸€èˆ¬åŒ–é•œåƒå¹¶é‡ç½®ä¸Šè¿° Linux 项目。命令如下： + +```PowerShell +sysprep.exe /generalize /shutdown /oobe +``` + +## 转æ¢ä¸ºæ¨¡æ¿ + +1. 关闭并åœæ­¢è™šæ‹Ÿæœºã€‚ +2. å³é”®å•å‡»æ¸…å•åˆ—表中的虚拟机，然åŽé€‰æ‹©**模æ¿**。 +3. 点击**转æ¢ä¸ºæ¨¡æ¿**。 + +**结果**：æµç¨‹å®ŒæˆåŽï¼Œå³å¯ä½¿ç”¨æ¨¡æ¿ã€‚ + +有关将虚拟机转æ¢ä¸ºæ¨¡æ¿çš„更多信æ¯ï¼Œè¯·å‚阅 [VMware 指å—](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-5B3737CC-28DB-4334-BD18-6E12011CDC9F.html)。 + +## 移动到内容库 + +Rancher 支æŒä½¿ç”¨å†…容库æ供的模æ¿ã€‚内容库在 vSphere 中存储和管ç†å†…容，还支æŒå‘布和共享该内容。 + +以下是有关内容库的一些有用链接： + +* [创建内容库](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-2A0F1C13-7336-45CE-B211-610D39A6E1F4.html) +* [将模æ¿å…‹éš†åˆ°å†…容库](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AC1545F0-F8BA-4CD2-96EB-21B3DFAA1DC1.html) + +## å…¶ä»–èµ„æº + +以下是å¯èƒ½æœ‰ç”¨çš„其他资æºåˆ—表： + +* [Linux 模æ¿åˆ›å»ºæ•™ç¨‹](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-ubuntu-template) +* [Windows 模æ¿åˆ›å»ºæ•™ç¨‹](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-windows-template) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md new file mode 100644 index 00000000000..8e1905f7ecc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md @@ -0,0 +1,45 @@ +--- +title: 在 vSphere 控制å°ä¸­åˆ›å»ºå‡­è¯ +--- + +本节介ç»å¦‚何创建 vSphere 用户å和密ç ã€‚你需è¦å°†è¿™äº› vSphere 凭è¯æ供给 Rancher，Rancher æ‰èƒ½åœ¨ vSphere 中é…置资æºã€‚ + +下表列出了 vSphere 用户账å·æ‰€éœ€çš„æƒé™ï¼š + +| 特æƒç»„ | æ“作 | +|:-------------------------|:-----------------------------------------------------------------------| +| Cns Privileges | Searchable | +| Content library | Read Storage
(Required only if deploying Template from Content Library) | +| Cryptographic operations | Direct Access | +| Datastore | AllocateSpace
Browse
FileManagement (Low level file operations)
UpdateVirtualMachineFiles
UpdateVirtualMachineMetadata | +| 全局 | Set custom attribute | +| 网络 | Assign | +| èµ„æº | AssignVMToPool | +| 虚拟机 | Config (All)
GuestOperations (All)
Interact (All)
Inventory (All)
Provisioning (All) | +| vSphere 标签 | Assign or Unassign vSphere Tag
Assign or Unassign vSphere Tag on Object | + +以下步骤创建具有所需æƒé™çš„角色，然åŽåœ¨ vSphere 控制å°ä¸­å°†è§’色分é…给新用户： + +1. 在 **vSphere** 控制å°ä¸­è½¬åˆ° **Administration** 页é¢ã€‚ + +2. 转到 **Roles** 选项å¡ã€‚ + +3. 创建一个新角色。为新角色命å，并选择上é¢æƒé™è¡¨ä¸­åˆ—出的æƒé™ã€‚ + + ![](/img/rancherroles1.png) + +4. 转到 **Users and Groups** 选项å¡ã€‚ + +5. 创建一个新用户。填写表å•ï¼Œç„¶åŽå•å‡» **OK**。请务必记下用户å和密ç ï¼Œå› ä¸ºåœ¨ Rancher 中é…置节点模æ¿æ—¶å°†éœ€è¦ç”¨åˆ°è¿™äº›ä¿¡æ¯ã€‚ + + ![](/img/rancheruser.png) + +6. 转到 **Global Permissions** 选项å¡ã€‚ + +7. 创建一个新的全局æƒé™ã€‚添加之å‰åˆ›å»ºçš„用户，并为其分é…之å‰åˆ›å»ºçš„角色。å•å‡» **OK**。 + + ![](/img/globalpermissionuser.png) + + ![](/img/globalpermissionrole.png) + +**结果**：你已拥有 Rancher 所需的æ“作 vSphere 资æºçš„凭è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md new file mode 100644 index 00000000000..25e4cf7e087 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md @@ -0,0 +1,112 @@ +--- +title: 在 vSphere 中é…ç½® Kubernetes 集群 +--- + +在本节中，你将学习如何使用 Rancher 在 vSphere 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +首先，在 Rancher 中设置你的 vSphere 云凭è¯ã€‚然åŽï¼Œä½¿ç”¨äº‘凭è¯åˆ›å»ºä¸€ä¸ªèŠ‚点模æ¿ï¼ŒRancher 将使用该模æ¿åœ¨ vSphere 中é…置节点。 + +然åŽï¼Œåœ¨ Rancher 中创建一个 vSphere 集群，并在é…置新集群时为集群定义节点池。æ¯ä¸ªèŠ‚点池都有一个 etcdã€controlplane 或 worker çš„ Kubernetes 角色。Rancher 会在新节点上安装 RKE Kubernetes，并为æ¯ä¸ªèŠ‚点设置节点池定义的 Kubernetes 角色。 + +有关é…ç½® vSphere 节点模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅 [vSphere 节点模æ¿é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)。 + +有关在 Rancher 中é…ç½® RKE Kubernetes 集群的详细信æ¯ï¼Œè¯·å‚阅[集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-集群é…置文件å‚考)。 + + +- [vSphere 中的准备工作](#vsphere-中的准备工作) +- [创建 vSphere 集群](#创建-vsphere-集群) + +## vSphere 中的准备工作 + +本节介ç»è®¾ç½® vSphere çš„è¦æ±‚，以便 Rancher å¯ä»¥é…置虚拟机和集群。 + +节点模æ¿å·²ä½¿ç”¨ vSphere Web Services API 6.5 版本进行记录和测试。 + +### 在 vSphere ä¸­åˆ›å»ºå‡­è¯ + +在继续创建集群之å‰ï¼Œç¡®ä¿ä½ çš„ vSphere 用户拥有足够的æƒé™ã€‚设置节点模æ¿æ—¶ï¼Œæ¨¡æ¿å°†éœ€è¦ä½¿ç”¨è¿™äº› vSphere 凭è¯ã€‚ + +有关如何在 vSphere 中创建具有所需æƒé™çš„用户的说明，请å‚阅此[æ“作指å—](create-credentials.md)。这些步骤会产生需è¦å‘ Rancher æ供的用户å和密ç ï¼Œä»Žè€Œå…许 Rancher 在 vSphere 中é…置资æºã€‚ + +### 网络æƒé™ + +必须确ä¿è¿è¡Œ Rancher Server 的主机能够建立以下网络连接： + +- 连接到 vCenter Server 上的 vSphere APIï¼ˆé€šå¸¸æ˜¯ç«¯å£ 443/TCP）。 +- 连接到用于实例化集群虚拟机的所有 ESXi 主机上的 Host APIï¼ˆç«¯å£ 443/TCP）（*仅在使用 ISO 创建模å¼æ—¶éœ€è¦*）。 +- 连接到创建的 VM 上的 22/TCP å’Œ 2376/TCP 端å£ã€‚ + +有关在基础设施æ供商上创建节点的端å£è¦æ±‚，请å‚阅[节点网络è¦æ±‚](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 + +### 用于 vSphere API 访问的有效 ESXi 许å¯è¯ + +å…费的 ESXi 许å¯è¯ä¸æ”¯æŒ API 访问。vSphere Server 必须具有有效或评估的 ESXi 许å¯è¯ã€‚ + +### 具有 DRS 的集群的 VM-VM å…³è”规则 + +如果你的集群å¯ç”¨äº† DRS，建议设置 [VM-VM å…³è”规则](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.resmgmt.doc/GUID-7297C302-378F-4AF2-9BD6-6EDB1E0A850A.html)。这些规则å…许分é…了 etcd å’Œ controlplane 角色的虚拟机在分é…ç»™ä¸åŒèŠ‚点池时，在å•ç‹¬çš„ ESXi 主机上è¿è¡Œã€‚è¿™ç§åšæ³•å¯ç¡®ä¿å•ä¸ªç‰©ç†æœºçš„æ•…éšœä¸ä¼šå½±å“这些平é¢çš„å¯ç”¨æ€§ã€‚ + +## 创建 vSphere 集群 + +在 Rancher 中创建 vSphere 集群的æ“作å–决于 Rancher 的版本。 + +1. [创建云凭è¯](#1-创建云凭è¯) +2. [使用云凭è¯åˆ›å»ºèŠ‚点模æ¿](#2-使用云凭è¯åˆ›å»ºèŠ‚点模æ¿) +3. [使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群](#3-使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群) + +### 1. åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡» **VMware vSphere**。 +1. 输入你的 vSphere 凭è¯ã€‚如需帮助，请å‚è§[节点模æ¿é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)中的**è´¦å·è®¿é—®**。 +1. å•å‡»**创建**。 + +**结果**：已创建用于在集群中é…置节点的云凭è¯ã€‚ä½ å¯ä»¥åœ¨å…¶ä»–节点模æ¿æˆ–集群中å¤ç”¨è¿™äº›å‡­è¯ã€‚ + +### 2. 使用云凭è¯åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +为 vSphere 创建[节点模æ¿](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)会å…许 Rancher 在 vSphere 中é…置新节点。其他集群å¯ä»¥å¤ç”¨èŠ‚点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**创建**。 +1. å•å‡»**添加模æ¿**。 +1. å•å‡» **vSphere**。 +1. 填写 vSphere 的节点模æ¿ã€‚有关填写表格的帮助，请å‚阅 vSphere 节点模æ¿[é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)。 +1. å•å‡»**创建**。 + +### 3. 使用节点模æ¿åˆ›å»ºå…·æœ‰èŠ‚点池的集群 + +使用 Rancher 在 vSphere 中创建 Kubernetes 集群。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡» **VMware vSphere**。 +1. 输入**集群å称**并使用你的 vSphere 云凭è¯ã€‚点击**继续**。 +1. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ +1. 使用**集群选项**选择è¦å®‰è£…çš„ Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。如需获å–é…置集群的帮助，请å‚阅 [RKE 集群é…ç½®å‚考](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)。 +1. 如果你想ç¨åŽåŠ¨æ€é…ç½®æŒä¹…存储或其他基础设施，你需è¦ä¿®æ”¹é›†ç¾¤ YAML 文件æ¥å¯ç”¨ vSphere 云æ供商。有关更多信æ¯ï¼Œè¯·å‚阅[树内 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 +1. 将一个或多个节点池添加到你的集群。æ¯ä¸ªèŠ‚点池都使用节点模æ¿æ¥é…置新节点。有关节点池的更多信æ¯ï¼ŒåŒ…æ‹¬ä¸ºèŠ‚ç‚¹åˆ†é… Kubernetes 角色的最佳实践，请å‚阅[本节](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)。 +1. 检查并确认你的选项。然åŽå•å‡»**创建**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 +- **é…置存储**：有关如何使用 Rancher 在 vSphere 中é…置存储的示例，请å‚阅[本节](../../../../../pages-for-subheaders/provisioning-storage-examples.md)。è¦åœ¨ vSphere 中动æ€é…置存储，你必须å¯ç”¨ vSphere 云æ供商。有关更多信æ¯ï¼Œè¯·å‚阅[树内 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere cloud provider 文档](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md new file mode 100644 index 00000000000..aef1f3c08b4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md @@ -0,0 +1,66 @@ +--- +title: 将用户添加到集群 +--- + +如果你想为用户æ供对集群内 _所有_ 项目ã€èŠ‚点和资æºçš„访问æƒé™ï¼Œè¯·ä¸ºç”¨æˆ·åˆ†é…集群æˆå‘˜èµ„格。 + +:::tip + +如果你想为用户æ供对集群内 _特定_ 项目的访问æƒé™ï¼Œè¯·å‚è§[添加项目æˆå‘˜](../../../new-user-guides/add-users-to-projects.md)。 + +::: + +ä½ å¯ä»¥åœ¨ä¸¤ç§æƒ…况下添加集群æˆå‘˜ï¼š + +- å°†æˆå‘˜æ·»åŠ åˆ°æ–°é›†ç¾¤ + + ä½ å¯ä»¥åœ¨åˆ›å»ºé›†ç¾¤æ—¶å°†æˆå‘˜æ·»åŠ åˆ°é›†ç¾¤ï¼ˆæŽ¨è）。 + +- [å‘现有集群添加æˆå‘˜](#编辑集群æˆå‘˜) + + é…置集群åŽï¼Œä½ å§‹ç»ˆå¯ä»¥å‘集群添加æˆå‘˜ã€‚ + +## 编辑集群æˆå‘˜ + +集群管ç†å‘˜å¯ä»¥ç¼–辑集群的æˆå‘˜ï¼ŒæŽ§åˆ¶å“ªäº› Rancher 用户å¯ä»¥è®¿é—®é›†ç¾¤ä»¥åŠä»–们å¯ä»¥ä½¿ç”¨å“ªäº›åŠŸèƒ½ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ æˆå‘˜çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**æˆå‘˜è§’色**选项å¡ä¸­ï¼Œå•å‡»**添加æˆå‘˜**。 +1. æœç´¢è¦æ·»åŠ åˆ°é›†ç¾¤çš„用户或组。 + + 如果é…置了外部身份验è¯ï¼š + + - 在你键入时，Rancher 会从你的[外部身份验è¯](../../../../pages-for-subheaders/authentication-config.md)æºè¿”回用户。 + + :::note 使用 AD 但找ä¸åˆ°ä½ çš„用户？ + + ä½ çš„æœç´¢å±žæ€§é…ç½®å¯èƒ½å­˜åœ¨é—®é¢˜ã€‚请å‚阅[é…ç½® Active Directory 身份验è¯ï¼šæ­¥éª¤ 5](../../../new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md)。 + + ::: + + - ä½ å¯ä»¥åœ¨ä¸‹æ‹‰èœå•ä¸­æ·»åŠ ç»„，而ä¸æ˜¯å•ä¸ªç”¨æˆ·ã€‚下拉列表仅列出你（登录用户）所属的组。 + + :::note + + 如果你以本地用户身份登录，外部用户ä¸ä¼šæ˜¾ç¤ºåœ¨ä½ çš„æœç´¢ç»“果中。有关详细信æ¯ï¼Œè¯·å‚阅[外部身份验è¯é…置和主体用户](../../../../pages-for-subheaders/authentication-config.md#外部身份验è¯é…置和用户主体)。 + + ::: + +1. 分é…用户或组的**集群**角色。 + + [什么是集群角色？](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::tip + + 对于自定义角色，你å¯ä»¥ä¿®æ”¹å¯åˆ†é…çš„å•ä¸ªè§’色列表。 + + - è¦å°†è§’色添加到列表中，请[添加自定义角色](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md)。 + - è¦ä»Žåˆ—表中删除角色，请[é”定/解é”角色](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md)。 + + ::: + +**结果**：已将选中的用户添加到集群中。 + +- è¦æ’¤é”€é›†ç¾¤æˆå‘˜èµ„格，请选择用户并å•å‡»**删除**。此æ“作会删除æˆå‘˜èµ„格，而ä¸ä¼šåˆ é™¤ç”¨æˆ·ã€‚ +- è¦ä¿®æ”¹é›†ç¾¤ä¸­çš„用户角色，请将其从集群中删除，然åŽä½¿ç”¨ä¿®æ”¹åŽçš„角色é‡æ–°æ·»åŠ ç”¨æˆ·ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md new file mode 100644 index 00000000000..dee43a45458 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md @@ -0,0 +1,42 @@ +--- +title: 授æƒé›†ç¾¤ç«¯ç‚¹çš„å·¥ä½œåŽŸç† +--- + +æœ¬æ–‡ä»‹ç» kubectl CLIã€kubeconfig 文件和授æƒé›†ç¾¤ç«¯ç‚¹å¦‚何ååŒå·¥ä½œï¼Œä½¿ä½ å¯ä»¥ç›´æŽ¥è®¿é—®ä¸‹æ¸¸ Kubernetes 集群，而无需通过 Rancher Server 进行身份验è¯ã€‚本文旨在为[设置 kubectl 以直接访问集群的说明](use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)æ供背景信æ¯å’Œä¸Šä¸‹æ–‡ã€‚ + +### kubeconfig 文件说明 + +_kubeconfig 文件_ 是与 kubectl 命令行工具（或其他客户端）结åˆä½¿ç”¨æ—¶ç”¨äºŽé…ç½® Kubernetes 访问的文件。 + +æ­¤ kubeconfig 文件åŠå…¶å†…容特定于你正在查看的集群。你å¯ä»¥ä»Ž Rancher çš„**集群**视图中下载该文件。在 Rancher 中å¯ä»¥è®¿é—®çš„æ¯ä¸ªé›†ç¾¤éƒ½éœ€è¦ä¸€ä¸ªå•ç‹¬çš„ kubeconfig 文件。 + +下载 kubeconfig 文件åŽï¼Œä½ å°†èƒ½å¤Ÿä½¿ç”¨ kubeconfig 文件åŠå…¶ Kubernetes [上下文](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#kubectl-context-and-configuration)访问下游集群。 + +如果管ç†å‘˜[关闭了 kubeconfig 令牌生æˆ](../../../../reference-guides/about-the-api/api-tokens.md#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)，则 kubeconfig 文件è¦æ±‚ [Rancher CLI](./authorized-cluster-endpoint.md) 存在于你的 PATH 中。 + +### RKE 集群的两ç§èº«ä»½éªŒè¯æ–¹æ³• + +如果集群ä¸æ˜¯ [RKE 集群](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)，kubeconfig 文件åªå…许你以一ç§æ–¹å¼è®¿é—®â€‹â€‹é›†ç¾¤ï¼Œå³é€šè¿‡ Rancher Server 进行身份验è¯ï¼Œç„¶åŽ Rancher å…许你在集群上è¿è¡Œ kubectl 命令。 + +对于 RKE 集群，kubeconfig 文件å…许你通过两ç§æ–¹å¼è¿›è¡Œèº«ä»½éªŒè¯ï¼š + +- **通过 Rancher Server 身份验è¯ä»£ç†**：Rancher 的身份验è¯ä»£ç†ä¼šéªŒè¯ä½ çš„身份，然åŽå°†ä½ è¿žæŽ¥åˆ°è¦è®¿é—®çš„下游集群。 +- **直接使用下游集群的 API Server**：RKE 集群默认å¯ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹ã€‚此端点å…许你使用 kubectl CLI å’Œ kubeconfig 文件访问下游 Kubernetes 集群，且 RKE 集群默认å¯ç”¨è¯¥ç«¯ç‚¹ã€‚在这ç§æƒ…况下，下游集群的 Kubernetes API server 通过调用 Rancher 设置的 webhook（`kube-api-auth` å¾®æœåŠ¡ï¼‰å¯¹ä½ è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +第二ç§æ–¹æ³•ï¼ˆå³ç›´æŽ¥è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API server）éžå¸¸é‡è¦ï¼Œå› ä¸ºå¦‚果你无法连接到 Rancher，这ç§æ–¹æ³•å¯ä»¥è®©ä½ è®¿é—®ä¸‹æ¸¸é›†ç¾¤ã€‚ + +è¦ä½¿ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹ï¼Œä½ éœ€è¦é…ç½® kubectl，从而使用 Rancher 在创建 RKE 集群时生æˆçš„ kubeconfig 文件中的é¢å¤– kubectl 上下文。该文件å¯ä»¥ä»Ž Rancher UI çš„**集群**视图中下载，é…ç½® kubectl 的说明在[此页é¢](use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)。 + +[架构介ç»](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md)也详细解释了这些与下游 Kubernetes 集群通信的方法，并介ç»äº† Rancher 的工作原ç†ä»¥åŠ Rancher 如何与下游集群通信的详细信æ¯ã€‚ + +### 关于 kube-api-auth èº«ä»½éªŒè¯ Webhook + +`kube-api-auth` å¾®æœåŠ¡æ˜¯ä¸º[授æƒé›†ç¾¤ç«¯ç‚¹](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)æ供用户认è¯åŠŸèƒ½è€Œéƒ¨ç½²çš„。当你使用 `kubectl` 访问下游集群时，集群的 Kubernetes API server 会使用 `kube-api-auth` æœåŠ¡ä½œä¸º webhook 对你进行身份验è¯ã€‚ + +在集群é…置期间会部署 `/etc/kubernetes/kube-api-authn-webhook.yaml` 文件，并使用 `--authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml` é…ç½® `kube-apiserver`。这会将 `kube-apiserver` é…置为通过查询 `http://127.0.0.1:6440/v1/authenticate` æ¥ç¡®å®šæŒæœ‰è€… token 的身份验è¯ã€‚ + +`kube-api-auth` 的调度规则如下： + +| 组件 | nodeAffinity nodeSelectorTerms | nodeSelector | 容å¿åº¦ | +| -------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| kube-api-auth | `beta.kubernetes.io/os:NotIn:windows`
`node-role.kubernetes.io/controlplane:In:"true"` | none | `operator:Exists` | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md new file mode 100644 index 00000000000..c28853ad3e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -0,0 +1,105 @@ +--- +title: "使用 Kubectl å’Œ kubeconfig 访问集群" +description: "了解如何通过 kubectl Shell 使用 kubectl，或通过 kubectl CLI å’Œ kubeconfig 文件，æ¥è®¿é—®å’Œç®¡ç† Kubernetes 集群。kubeconfig 文件用于é…置对 Kubernetes 的访问。当你使用 Rancher 创建集群时，Rancher 会自动为你的集群创建 kubeconfig。" +--- + +本文介ç»å¦‚何通过 Rancher UI 或工作站æ¥ä½¿ç”¨ kubectl æ“作下游 Kubernetes 集群。 + +有关使用 kubectl 的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档：kubectl 概述](https://kubernetes.io/docs/reference/kubectl/overview/)。 + +### 在 Rancher UI 中使用 kubectl shell 访问集群 + +ä½ å¯ä»¥é€šè¿‡ç™»å½• Rancher 并在 UI 中打开 kubectl shell æ¥è®¿é—®å’Œç®¡ç†ä½ çš„集群。你无需进一步é…置。 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦ä½¿ç”¨ kubectl 访问的集群，然åŽå•å‡» **Explore**。 +1. 在顶部导航èœå•ä¸­ï¼Œå•å‡» **Kubectl Shell** 按钮。使用打开的窗å£ä¸Žä½ çš„ Kubernetes 集群进行交互。 + +### 在工作站使用 kubectl 访问集群 + +本节介ç»å¦‚何下载集群的 kubeconfig 文件ã€ä»Žå·¥ä½œç«™å¯åŠ¨ kubectl 以åŠè®¿é—®ä¸‹æ¸¸é›†ç¾¤ã€‚ + +è¿™ç§è®¿é—®é›†ç¾¤çš„替代方法å…许你在ä¸ä½¿ç”¨ Rancher UI 的情况下通过 Rancher 进行身份验è¯å¹¶ç®¡ç†é›†ç¾¤ã€‚ + +:::note 先决æ¡ä»¶ï¼š + +以下说明å‡è®¾ä½ å·²ç»åˆ›å»ºäº†ä¸€ä¸ª Kubernetes 集群，并且已将 kubectl 安装在工作站上。有关安装 kubectl 的帮助，请å‚阅官方 [Kubernetes 文档](https://kubernetes.io/docs/tasks/tools/install-kubectl/)。 + +::: + +1. 登录到 Rancher。点击 **☰ > 集群管ç†**。 +1. 转到è¦ä½¿ç”¨ kubectl 访问的集群，然åŽå•å‡» **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œå•å‡»**下载 KubeConfig** 按钮。 +1. å°† YAML 文件ä¿å­˜åœ¨æœ¬åœ°è®¡ç®—机上。将文件移动到 `~/.kube/config`。注æ„：kubectl 用于 kubeconfig 文件的默认ä½ç½®æ˜¯ `~/.kube/config`。但是你也å¯ä»¥è¿è¡Œç±»ä¼¼ä»¥ä¸‹çš„命令，使用 `--kubeconfig` 标志指定任何其他目录： +``` +kubectl --kubeconfig /custom/path/kube.config get pods +``` +1. 从工作站å¯åŠ¨ kubectl。使用它与 Kubernetes 集群进行交互。 + + +### 使用 kubectl 创建的资æºçš„注æ„事项 + +Rancher 会å‘现并显示由 `kubectl` 创建的资æºã€‚但是在å‘现资æºçš„时候，这些资æºå¯èƒ½æ²¡æœ‰åŒ…括所有必须的注释。如果资æºå·²ç»ä½¿ç”¨ Rancher UI/API 进行æ“作（例如，扩展工作负载），但是由于缺少注释，资æºçš„é‡æ–°åˆ›å»ºå¯èƒ½ä¼šè§¦å‘。åªæœ‰åœ¨é¦–次对å‘现的资æºè¿›è¡Œæ“作时，这ç§æƒ…况æ‰ä¼šå‘生。 + +## ç›´æŽ¥ä½¿ç”¨ä¸‹æ¸¸é›†ç¾¤è¿›è¡Œèº«ä»½éªŒè¯ + +本节旨在帮助你设置访问 [RKE 集群的替代方法](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +此方法仅适用于å¯ç”¨äº†[授æƒé›†ç¾¤ç«¯ç‚¹](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)çš„ RKEã€RKE2 å’Œ K3s集群。在 Rancher 创建集群时，Rancher 会生æˆä¸€ä¸ª kubeconfig 文件，其中包å«ç”¨äºŽè®¿é—®é›†ç¾¤çš„é¢å¤– kubectl 上下文。该上下文å…许你使用 kubectl 通过下游集群进行身份验è¯ï¼Œè€Œæ— éœ€é€šè¿‡ Rancher 进行身份验è¯ã€‚有关授æƒé›†ç¾¤ç«¯ç‚¹å¦‚何工作的详细说明，请å‚阅[此页é¢](authorized-cluster-endpoint.md)。 + +在 RKE2 å’Œ K3s 集群上，你需è¦[手动å¯ç”¨](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#对-rke2-å’Œ-k3s-集群的授æƒé›†ç¾¤ç«¯ç‚¹æ”¯æŒ)授æƒé›†ç¾¤ç«¯ç‚¹ã€‚ + +我们的最佳实践是使用此方法æ¥è®¿é—® RKEã€RKE2 å’Œ K3s集群。这样，万一你无法连接到 Rancher，你ä»ç„¶å¯ä»¥è®¿é—®è¯¥é›†ç¾¤ã€‚ + + +:::note 先决æ¡ä»¶ï¼š + +以下步骤å‡è®¾ä½ å·²ç»åˆ›å»ºäº†ä¸€ä¸ª Kubernetes 集群，并按照步骤[从工作站使用 kubectl 连接到集群](#在工作站使用-kubectl-访问集群)。 + +::: + +è¦åœ¨ä¸‹è½½çš„ kubeconfig 文件中查找上下文的å称，请è¿è¡Œï¼š + +``` +kubectl config get-contexts --kubeconfig /custom/path/kube.config +CURRENT NAME CLUSTER AUTHINFO NAMESPACE +* my-cluster my-cluster user-46tmn + my-cluster-controlplane-1 my-cluster-controlplane-1 user-46tmn +``` + +在此示例中，当你将 `kubectl` 与第一个上下文 `my-cluster` 一起使用时，你将通过 Rancher Server 进行身份验è¯ã€‚ + +使用第二个上下文 `my-cluster-controlplane-1`，你将使用授æƒé›†ç¾¤ç«¯ç‚¹è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç›´æŽ¥ä¸Žä¸‹æ¸¸ RKE 集群通信。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +现在你已ç»æœ‰äº†ç›´æŽ¥ä½¿ç”¨é›†ç¾¤è¿›è¡Œèº«ä»½éªŒè¯æ‰€éœ€çš„上下文å称，你å¯ä»¥åœ¨è¿è¡Œ kubectl 命令时将上下文å称作为选项传入。根æ®ä½ çš„集群是å¦å®šä¹‰äº† FQDN，这些命令会有所ä¸åŒã€‚以下几节æ供了示例。 + +当 `kubectl` 正常工作时，它确认你å¯ä»¥ç»•è¿‡ Rancher 的身份验è¯ä»£ç†è®¿é—®é›†ç¾¤ã€‚ + +### 直接连接到定义了 FQDN 的集群 + +如果集群定义了 FQDN，将会创建一个引用 FQDN 的上下文。上下文将命å为 `-fqdn`。当你想在没有 Rancher 的情况下使用 `kubectl` 访问这个集群时，你需è¦ä½¿ç”¨è¿™ä¸ªä¸Šä¸‹æ–‡ã€‚ + +å‡è®¾ kubeconfig 文件ä½äºŽ `~/.kube/config`： + +``` +kubectl --context -fqdn get nodes +``` +直接引用 kubeconfig 文件的ä½ç½®ï¼š +``` +kubectl --kubeconfig /custom/path/kube.config --context -fqdn get pods +``` + +### 直接连接到未定义 FQDN 的集群 + +如果集群没有定义 FQDN，则会创建é¢å¤–的上下文æ¥å¼•ç”¨ controlplane 中æ¯ä¸ªèŠ‚点的 IP 地å€ã€‚æ¯ä¸ªä¸Šä¸‹æ–‡å°†è¢«å‘½å为 `-`。当你想在没有 Rancher 的情况下使用 `kubectl` 访问这个集群时，你需è¦ä½¿ç”¨è¿™ä¸ªä¸Šä¸‹æ–‡ã€‚ + +å‡è®¾ kubeconfig 文件ä½äºŽ `~/.kube/config`： +``` +kubectl --context - get nodes +``` +直接引用 kubeconfig 文件的ä½ç½®ï¼š +``` +kubectl --kubeconfig /custom/path/kube.config --context - get pods +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md new file mode 100644 index 00000000000..d881d6c968e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md @@ -0,0 +1,39 @@ +--- +title: 添加 Pod 安全策略 +--- + +:::note 先决æ¡ä»¶ï¼š + +以下选项仅适用于[使用 RKE å¯åŠ¨çš„集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +::: + +当你的集群上è¿è¡Œäº†å…·æœ‰å®‰å…¨æ•æ„Ÿé…置的 pod æ—¶ï¼Œè¯·ä¸ºå…¶åˆ†é… [pod 安全策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md)，这是一组用于监控 pod 中的状æ€å’Œè®¾ç½®çš„规则。如果 pod ä¸ç¬¦åˆä½ çš„策略中指定的规则，则该策略会阻止它è¿è¡Œã€‚ + +ä½ å¯ä»¥åœ¨é…ç½®é›†ç¾¤æ—¶åˆ†é… pod 安全策略。如果你以åŽéœ€è¦æ”¾æ¾æˆ–é™åˆ¶ pod 的安全性，你å¯ä»¥åœ¨ç¼–辑集群时更新策略： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦åº”用 pod 安全策略的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在 **Pod 安全策略支æŒ**中，选择**å¯ç”¨**。 + + :::note + + 此选项仅适用于[ç”± RKE é…置的集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + + ::: + +4. 从**默认 Pod 安全策略**下拉列表中，选择è¦åº”用于集群的策略。 + + Rancher 支æŒäº†`å—é™`å’Œ`ä¸å—é™`çš„[ç­–ç•¥](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#默认-psp)，你也å¯ä»¥[创建自定义策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#创建-psp)。 + +5. å•å‡»**ä¿å­˜**。 + +**结果**：pod 安全策略应用于集群和集群内的任何项目。 + +:::note + +åœ¨åˆ†é… pod 安全策略之å‰å·²ç»è¿è¡Œçš„工作负载是ä¸å—é™åˆ¶çš„。å³ä½¿å®ƒä»¬ä¸ç¬¦åˆä½ çš„ pod 安全策略，在分é…策略之å‰è¿è¡Œçš„工作负载也会继续è¿è¡Œã€‚ + +è¦æ£€æŸ¥æ­£åœ¨è¿è¡Œçš„工作负载是å¦é€šè¿‡äº†ä½ çš„ pod 安全策略，请克隆或å‡çº§å®ƒã€‚ + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md new file mode 100644 index 00000000000..180a248025b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md @@ -0,0 +1,25 @@ +--- +title: åˆ†é… Pod 安全策略 +--- + +_Pod 安全策略_（PSP）是控制 Pod 安全的规范（如是å¦å¯ä»¥ä½¿ç”¨ root æƒé™ç­‰ï¼‰çš„对象。 + +## 添加默认 Pod 安全策略 + +使用 RKE 创建新集群时，你å¯ä»¥å°†å…¶é…置为立å³åº”用 PSP。创建集群时，使用**集群选项**æ¥å¯ç”¨ PSP。分é…给集群的 PSP 将是集群内项目的默认 PSP。 + +:::note 先决æ¡ä»¶ï¼š + +在 Rancher 中创建 Pod 安全策略。在将默认 PSP 分é…给新集群之å‰ï¼Œä½ å¿…须有一个å¯åˆ†é…çš„ PSP。有关说明，请å‚阅[创建 Pod 安全策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +::: + +:::note + +å‡ºäºŽå®‰å…¨è€ƒè™‘ï¼Œæˆ‘ä»¬å»ºè®®ä½ åœ¨åˆ›å»ºé›†ç¾¤æ—¶åˆ†é… PSP。 + +::: + +è¦å¯ç”¨é»˜è®¤ Pod 安全策略，请将 **Pod 安全策略支æŒ**选项设置为**å¯ç”¨**，然åŽä»Ž**默认 Pod 安全策略**下拉框中进行选择。 + +集群完æˆé…ç½®åŽï¼Œä½ é€‰æ‹©çš„ PSP 将应用于集群内的所有项目。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md new file mode 100644 index 00000000000..4ed2eb4d44d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -0,0 +1,376 @@ +--- +title: 从节点中移除 Kubernetes 组件 +description: 了解从 Rancher å¯åŠ¨çš„ Kubernetes 集群中删除节点时的集群清ç†è¿‡ç¨‹ã€‚了解删除了的内容，以åŠå¦‚何手动进行æ“作 +--- + +本文介ç»å¦‚何断开 Rancher 创建的 Kubernetes 集群中的一个节点，并删除该节点中的所有 Kubernetes 组件。此过程å…许你将节点用于其他用途。 + +在通过 Rancher 在基础设施æ供商的新节点上安装 Kubernetes 时，会创建资æºï¼ˆå®¹å™¨/虚拟网络接å£ï¼‰å’Œé…置项（è¯ä¹¦/é…置文件）。 + +从 Rancher å¯åŠ¨çš„ Kubernetes 集群中移除节点时（å‰æ是它们处于 `Active` 状æ€ï¼‰ï¼Œè¿™äº›èµ„æºä¼šè¢«è‡ªåŠ¨æ¸…ç†ï¼Œå”¯ä¸€éœ€è¦çš„æ“作是é‡å¯èŠ‚点。如果节点å˜å¾—ä¸å¯è®¿é—®å¹¶ä¸”无法自动清ç†è¿›ç¨‹æ—¶ï¼Œè¯·å…ˆæ‰§è¡Œæˆ‘们æ供的步骤，然åŽè¯¥èŠ‚点æ‰èƒ½é‡æ–°æ·»åŠ åˆ°é›†ç¾¤ã€‚ + +## 移除了什么？ + +清ç†ä½¿ç”¨ Rancher é…置的节点时，会根æ®è¦åˆ é™¤çš„集群节点类型移除以下组件： + +| 移除的组件 | [由基础设施æ供商托管的节点][1] | [自定义节点][2] | [托管集群][3] | [注册节点][4] | +|--------------------------------------------------------------------------------|----------------------------------------------|-------------------|---------------------|-----------------------| +| Rancher deployment 命å空间（默认：`cattle-system`） | ✓ | ✓ | ✓ | ✓ | +| ç”± Rancher 标记的 `serviceAccount`ã€`clusterRoles` å’Œ `clusterRoleBindings` | ✓ | ✓ | ✓ | ✓ | +| 标签ã€æ³¨é‡Šå’Œç»ˆç»“器 | ✓ | ✓ | ✓ | ✓ | +| Rancher Deployment | ✓ | ✓ | ✓ | | +| 主机ã€é›†ç¾¤ã€é¡¹ç›®å’Œç”¨æˆ·çš„自定义资æºå®šä¹‰ (CRD) | ✓ | ✓ | ✓ | | +| 在 `management.cattle.io` API Group ä¸‹åˆ›å»ºçš„æ‰€æœ‰èµ„æº | ✓ | ✓ | ✓ | | +| Rancher v2.x 创建的所有 CRD | ✓ | ✓ | ✓ | | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +## 通过 Rancher UI 删除集群中的节点 + +如果节点处于 `Active` 状æ€ï¼Œåˆ é™¤é›†ç¾¤ä¸­çš„节点会触å‘清ç†èŠ‚点的进程。完æˆè‡ªåŠ¨æ¸…ç†åŽè¯·é‡å¯èŠ‚点，以确ä¿æ‰€æœ‰éžæŒä¹…性数æ®å·²æ­£ç¡®åˆ é™¤ã€‚ + +**é‡å¯èŠ‚点**： + +``` +# 使用é‡å¯ +$ sudo reboot + +# 使用关机 +$ sudo shutdown -r now +``` + +## 手动移除集群中的 Rancher 组件 + +当节点ä¸å¯è¾¾å¹¶å·²ä»Žé›†ç¾¤ä¸­ç§»é™¤æ—¶ï¼Œç”±äºŽè¯¥èŠ‚点ä¸å¯è¾¾ï¼Œåˆ™æ— æ³•è§¦å‘自动清ç†è¿‡ç¨‹ã€‚请按照以下步骤手动删除 Rancher 组件。 + +:::danger + +下é¢åˆ—出的命令将删除节点中的数æ®ã€‚在执行命令之å‰ï¼Œç”±äºŽæ•°æ®å°†ä¼šä¸¢å¤±ï¼Œè¯·ç¡®ä¿ä½ å·²ç»å¤‡ä»½äº†è¦ä¿ç•™çš„文件。 + +::: + +### 移除注册集群中的 Rancher 组件 + +移除注册集群中的 Rancher 的过程略有ä¸åŒã€‚ä½ å¯ä»¥é€‰æ‹©åœ¨ Rancher UI 中简å•åœ°åˆ é™¤é›†ç¾¤ï¼Œä¹Ÿå¯ä»¥è¿è¡Œè„šæœ¬ä»ŽèŠ‚点中删除 Rancher 组件。两个选项的删除效果是一样的。 + +将注册集群分离 Rancher åŽï¼Œé›†ç¾¤çš„工作负载将ä¸å—å½±å“，你å¯ä»¥ä½¿ç”¨ä¸Žé›†ç¾¤æ³¨å†Œåˆ° Rancher 之å‰ç›¸åŒçš„方法æ¥è®¿é—®é›†ç¾¤ã€‚ + + + + +:::danger + +此过程将删除你的集群数æ®ã€‚在执行命令之å‰ï¼Œç”±äºŽæ•°æ®å°†ä¼šä¸¢å¤±ï¼Œè¯·ç¡®ä¿ä½ å·²ç»å¤‡ä»½äº†è¦ä¿ç•™çš„文件。 + +::: + +使用 Rancher UI（或 API）删除注册集群åŽï¼Œä¼šå‘生以下事件： + +1. Rancher 创建一个 `serviceAccount`，用于删除集群中的 Rancher 组件。此账å·åˆ†é…了 [clusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole) å’Œ [clusterRoleBinding](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding) æƒé™ï¼Œè¿™äº›æƒé™æ˜¯åˆ é™¤ Rancher 组件所必需的。 + +1. Rancher 会使用 `serviceAccount` 调度并è¿è¡Œä¸€ä¸ª [job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/)，以将 Rancher 组件从集群中清除。该 job 还将 `serviceAccount` åŠå…¶è§’色作为ä¾èµ–引用，因此该 job 在完æˆä¹‹å‰ä¼šåˆ é™¤å®ƒä»¬ã€‚ + +1. 已从集群中移除 Rancher。然而，集群ä»ç„¶å­˜åœ¨ï¼Œè¿è¡Œç€åŽŸç”Ÿç‰ˆæœ¬çš„ Kubernetes。 + +**结果**：已删除[移除了什么](#移除了什么)章节中为注册集群列出的所有组件。 + + + + +ä½ å¯ä»¥è¿è¡Œè„šæœ¬ï¼Œè€Œä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥æ¸…ç†å·²æ³¨å†Œçš„集群节点。 + +:::note 先决æ¡ä»¶ï¼š + +安装 [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)。 + +::: + +1. 打开æµè§ˆå™¨å¹¶å¯¼èˆªåˆ° [GitHub](https://github.com/rancher/rancher/blob/master/cleanup/user-cluster.sh)，然åŽä¸‹è½½ `user-cluster.sh`。 + +1. 在 `user-cluster.sh` 所在的目录下è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œä½¿è„šæœ¬å¯æ‰§è¡Œï¼š + + ``` + chmod +x user-cluster.sh + ``` + +1. **ä»…é™ç¦»çº¿çŽ¯å¢ƒ**：打开 `user-cluster.sh` 并将 `yaml_url` 替æ¢ä¸º `user-cluster.yml` 中的 URL。 + + 如果你没有使用离线环境，请跳过此步骤。 + +1. 在åŒä¸€ç›®å½•ä¸­ï¼Œè¿è¡Œè„šæœ¬å¹¶æä¾› `rancher/rancher-agent` é•œåƒç‰ˆæœ¬ï¼Œè¯¥ç‰ˆæœ¬åº”与用于管ç†é›†ç¾¤çš„ Rancher 版本相åŒï¼ˆ``）： + + :::tip + + ä½ å¯ä»¥æ·»åŠ  `-dry-run` 标志æ¥é¢„览脚本的结果，但ä¸è¿›è¡Œæ›´æ”¹ã€‚ + ``` + ./user-cluster.sh rancher/rancher-agent: + ``` + + ::: + +**结果**：脚本已è¿è¡Œã€‚已删除[移除了什么](#移除了什么)章节中为注册集群列出的所有组件。 + + + + +## 清ç†èŠ‚点 + + + + +在è¿è¡Œä»¥ä¸‹å‘½ä»¤ä¹‹å‰ï¼Œé¦–先通过 Rancher UI 删除节点。 + +è¦åˆ é™¤èŠ‚点： + +1. 点击 **☰** 并选择**集群管ç†**。 +1. 在集群表中，å•å‡»èŠ‚点所属集群的å称。 +1. 在第一个选项å¡ä¸­ï¼Œå•å‡»èŠ‚点状æ€æ—边的å¤é€‰æ¡†ã€‚ +1. å•å‡»**删除**。 + +è¦åˆ é™¤æ•´ä¸ªé›†ç¾¤è€Œä¸æ˜¯å•ä¸ªèŠ‚点，或者ä¸ä½¿ç”¨ Rancher UI é‡æ–°ç§»åŠ¨èŠ‚点，请按照以下步骤æ“作： + +1. 从节点中[删除](#docker-容器镜åƒå’Œå·) Docker 容器并[å¸è½½](#挂载)任何å·ã€‚ +1. é‡å¯èŠ‚点。 +1. [删除](#目录和文件)所有剩余的文件。 +1. 确认在é‡æ–°å¯åŠ¨åŽå·²æ­£ç¡®æ¸…ç†ç½‘络接å£å’Œ IP 表。å¦åˆ™è¯·å†é‡å¯ä¸€æ¬¡ã€‚ + +### Windows 节点 + +è¦æ¸…ç† Windows 节点，请è¿è¡Œ `c:\\etc\\rancher` 中的脚本。此脚本删除 Kubernetes 生æˆçš„资æºå’Œæ‰§è¡ŒäºŒè¿›åˆ¶æ–‡ä»¶ï¼Œè¿˜ä¼šåˆ é™¤é˜²ç«å¢™è§„则和网络设置： + +``` +pushd c:\etc\rancher +.\cleanup.ps1 +popd +``` + +è¿è¡Œæ­¤è„šæœ¬åŽï¼ŒèŠ‚点将é‡ç½®å¹¶å¯ä»¥é‡æ–°æ·»åŠ åˆ° Kubernetes 集群。 + +### Docker 容器ã€é•œåƒå’Œå· + +:::caution + +æ¸…ç† Docker 容器时è¦å°å¿ƒã€‚以下命令将删除节点上的*所有* Docker 容器ã€é•œåƒå’Œå·ï¼ŒåŒ…括与 Rancher 无关的容器： + +::: + +``` +docker rm -f $(docker ps -qa) +docker rmi -f $(docker images -q) +docker volume rm $(docker volume ls -q) +``` + +### 挂载 + +Kubernetes 组件和 secret 会留下以下挂载： + +* `/var/lib/kubelet` +* `/var/lib/rancher` +* `/var/lib/kubelet/pods/` 中的其他挂载 + +è¦å¸è½½æ‰€æœ‰æŒ‚载，请è¿è¡Œï¼š + +``` +for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done +``` + + + + +你需è¦ä»Ž Rancher æ供的 RKE2 节点中删除以下组件： + +* rancher-system-agent，用于连接 Rancher å¹¶å®‰è£…å’Œç®¡ç† RKE2。 +* RKE2 本身。 + +### 删除 rancher-system-agent + +è¦åˆ é™¤ rancher-system-agent，请è¿è¡Œ [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) 脚本： + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### 删除 RKE2 + +è¦åˆ é™¤ RKE2，请è¿è¡ŒèŠ‚点上已存在的 `rke2-uninstall` 脚本： + +``` +sudo rke2-uninstall.sh +``` + + + + +你需è¦ä»Ž Rancher æ供的 K3s 节点中删除以下组件： + +* rancher-system-agent：连接 Rancher å¹¶å®‰è£…å’Œç®¡ç† K3s。 +* K3s 本身。 + +### 删除 rancher-system-agent + +è¦åˆ é™¤ rancher-system-agent，请è¿è¡Œ [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) 脚本： + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### 删除 K3s + +è¦åˆ é™¤ K3s，请è¿è¡ŒèŠ‚点上已存在的 `k3s-uninstall` 脚本： + +``` +sudo k3s-uninstall.sh +``` + + + + +### 目录和文件 + +以下目录在将节点添加到集群时使用，应将该目录删除。你å¯ä»¥ä½¿ç”¨ `rm -rf /directory_name` æ¥åˆ é™¤ç›®å½•ã€‚ + +:::note + +节点分é…的角色决定了出现在节点上的目录。 + +::: + +| 目录 | +|------------------------------| +| `/etc/ceph` | +| `/etc/cni` | +| `/etc/kubernetes` | +| `/etc/rancher` | +| `/opt/cni` | +| `/opt/rke` | +| `/run/secrets/kubernetes.io` | +| `/run/calico` | +| `/run/flannel` | +| `/var/lib/calico` | +| `/var/lib/etcd` | +| `/var/lib/cni` | +| `/var/lib/kubelet` | +| `/var/lib/rancher` | +| `/var/log/containers` | +| `/var/log/kube-audit` | +| `/var/log/pods` | +| `/var/run/calico` | + +**清ç†ç›®å½•**： + +``` +rm -rf /etc/ceph \ + /etc/cni \ + /etc/kubernetes \ + /etc/rancher \ + /opt/cni \ + /opt/rke \ + /run/secrets/kubernetes.io \ + /run/calico \ + /run/flannel \ + /var/lib/calico \ + /var/lib/etcd \ + /var/lib/cni \ + /var/lib/kubelet \ + /var/lib/rancher\ + /var/log/containers \ + /var/log/kube-audit \ + /var/log/pods \ + /var/run/calico +``` + +### 网络接å£å’Œ iptables + +其余两个更改/é…置的组件是（虚拟）网络接å£å’Œ iptables 规则。两者都对节点ä¸æŒä¹…，这æ„味ç€å®ƒä»¬å°†åœ¨èŠ‚点é‡æ–°å¯åŠ¨åŽè¢«æ¸…除。è¦åˆ é™¤è¿™äº›ç»„件，建议é‡æ–°å¯åŠ¨å®ƒä»¬ã€‚ + +**é‡å¯èŠ‚点**： + +``` +# 使用é‡å¯ +$ sudo reboot + +# 使用关机 +$ sudo shutdown -r now +``` + +如果你想了解更多关于（虚拟）网络接å£æˆ– iptables 规则的信æ¯ï¼Œè¯·å‚阅下é¢çš„具体内容。 + +### ç½‘ç»œæŽ¥å£ + +:::note + +节点所在的集群所é…置的网络æ供商决定了节点上将出现的接å£ã€‚ + +::: + +| æŽ¥å£ | +|--------------------------------------------| +| `flannel.1` | +| `cni0` | +| `tunl0` | +| `caliXXXXXXXXXXX` (random interface names) | +| `vethXXXXXXXX` (random interface names) | + +**列出所有接å£**： + +``` +# 使用 ip +ip address show + +# 使用 ifconfig +ifconfig -a +``` + +**删除接å£**： + +``` +ip link delete interface_name +``` + +### Iptables + +:::note + +节点所在的集群所é…置的网络æ供商决定了节点上将出现的链。 + +::: + +iptables 规则用于将æµé‡ä»Žå®¹å™¨è·¯ç”±åˆ°å®¹å™¨ã€‚创建的规则ä¸æ˜¯æŒä¹…性的，因此é‡æ–°å¯åŠ¨èŠ‚点会将 iptables æ¢å¤åˆ°åŽŸå§‹çŠ¶æ€ã€‚ + +| 链 | +|--------------------------------------------------| +| `cali-failsafe-in` | +| `cali-failsafe-out` | +| `cali-fip-dnat` | +| `cali-fip-snat` | +| `cali-from-hep-forward` | +| `cali-from-host-endpoint` | +| `cali-from-wl-dispatch` | +| `cali-fw-caliXXXXXXXXXXX` (random chain names) | +| `cali-nat-outgoing` | +| `cali-pri-kns.NAMESPACE` (chain per namespace) | +| `cali-pro-kns.NAMESPACE` (chain per namespace) | +| `cali-to-hep-forward` | +| `cali-to-host-endpoint` | +| `cali-to-wl-dispatch` | +| `cali-tw-caliXXXXXXXXXXX` (random chain names) | +| `cali-wl-to-host` | +| `KUBE-EXTERNAL-SERVICES` | +| `KUBE-FIREWALL` | +| `KUBE-MARK-DROP` | +| `KUBE-MARK-MASQ` | +| `KUBE-NODEPORTS` | +| `KUBE-SEP-XXXXXXXXXXXXXXXX` (random chain names) | +| `KUBE-SERVICES` | +| `KUBE-SVC-XXXXXXXXXXXXXXXX` (random chain names) | + +**列出所有 iptables 规则**： + +``` +iptables -L -t nat +iptables -L -t mangle +iptables -L +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md new file mode 100644 index 00000000000..9a4c878d631 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md @@ -0,0 +1,37 @@ +--- +title: GlusterFS å· +--- + +:::note + +本文仅适用于 [RKE 集群](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +::: + +在将数æ®å­˜å‚¨åœ¨ GlusterFS å·ä¸Šçš„集群中，你å¯èƒ½ä¼šé‡åˆ°é‡å¯ `kubelet` åŽ pod 无法挂载å·çš„问题。`kubelet` 的日志将显示 `transport endpoint is not connected`。为了é¿å…è¿™ç§æƒ…况，你å¯ä»¥åœ¨é›†ç¾¤ä¸­å°† `systemd-run` 二进制文件挂载到 `kubelet` 容器中。在更改集群é…置之å‰æœ‰ä¸¤ä¸ªè¦æ±‚： + +- 该节点需è¦å®‰è£…了 `systemd-run` 二进制文件（å¯ä»¥é€šè¿‡åœ¨æ¯ä¸ªé›†ç¾¤èŠ‚点上è¿è¡Œ `which systemd-run` 命令æ¥æ£€æŸ¥ï¼‰ã€‚ +- `systemd-run` 二进制文件需è¦ä¸Ž hyperkube é•œåƒæ‰€åŸºäºŽçš„ Debian OS 兼容（å¯ä»¥é€šè¿‡åœ¨æ¯ä¸ªé›†ç¾¤èŠ‚点上è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ£€æŸ¥ï¼Œè¯·å°†é•œåƒæ ‡ç­¾æ›¿æ¢ä¸ºä½ æƒ³è¦çš„ Kubernetes 版本）： + +``` +docker run -v /usr/bin/systemd-run:/usr/bin/systemd-run --entrypoint /usr/bin/systemd-run rancher/hyperkube:v1.16.2-rancher1 --version +``` + +:::caution + +在更新 Kubernetes YAML 以挂载 `systemd-run` 二进制文件之å‰ï¼Œè¯·ç¡®ä¿åœ¨é›†ç¾¤èŠ‚点上安装了 `systemd` 包。如果在绑定挂载创建到你的 Kubernetes YAML _之å‰_ 未安装此包，Docker 将自动在æ¯ä¸ªèŠ‚点上创建目录和文件，并且ä¸å…许包安装æˆåŠŸã€‚ + +::: + +``` +services: + kubelet: + extra_binds: + - "/usr/bin/systemd-run:/usr/bin/systemd-run" +``` + +集群完æˆé…ç½®åŽï¼Œä½ å¯ä»¥é€šè¿‡æŸ¥æ‰¾ä»¥ä¸‹æ—¥å¿—æ¥æ£€æŸ¥ `kubelet` 容器日志记录，从而查看该功能是å¦å·²æ¿€æ´»ï¼š + +``` +Detected OS with systemd +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md new file mode 100644 index 00000000000..6e744c1295f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md @@ -0,0 +1,72 @@ +--- +title: æŒä¹…å­˜å‚¨çš„å·¥ä½œåŽŸç† +--- + +æŒä¹…å· (PV) 是 Kubernetes 集群中的一å—存储，而æŒä¹…å·å£°æ˜Ž (PVC) 是对存储的请求。 + +在 Kubernetes 中使用æŒä¹…存储有两ç§æ–¹æ³•ï¼š + +- 使用现有的æŒä¹…å· +- 动æ€é…置新的æŒä¹…å· + +è¦ä½¿ç”¨çŽ°æœ‰ PV，你的应用需è¦ä½¿ç”¨å·²ç»‘定到 PV çš„ PVC，并且 PV åº”åŒ…å« PVC 所需的最少资æºã€‚ + +对于动æ€å­˜å‚¨é…置，你的应用需è¦ä½¿ç”¨ç»‘定到存储类的 PVC。存储类包å«é…置新æŒä¹…å·çš„授æƒã€‚ + +![设置新的和现有的æŒä¹…存储](/img/rancher-storage.svg) + +有关更多信æ¯ï¼Œè¯·å‚阅[关于存储的官方 Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/volumes/)。 + + +## 关于æŒä¹…å·å£°æ˜Ž + +æŒä¹…å·å£°æ˜Ž (PVC) 是从集群请求存储资æºçš„对象，类似于你的 deployment 用于兑æ¢å­˜å‚¨è®¿é—®çš„凭è¯ã€‚PVC 以å·çš„å½¢å¼æŒ‚载到工作负载中，以便工作负载å¯ä»¥å£°æ˜Žå…¶æŒ‡å®šçš„æŒä¹…存储份é¢ã€‚ + +è¦è®¿é—®æŒä¹…存储，Pod 必须有挂载为å·çš„ PVC。此 PVC 让你的 deployment 应用将其数æ®å­˜å‚¨åœ¨å¤–部ä½ç½®ï¼Œè¿™æ ·ï¼Œå¦‚æžœ pod å‘生故障，就å¯ä»¥ç”¨æ–°çš„ pod 替æ¢å®ƒå¹¶ç»§ç»­è®¿é—®å…¶å¤–部存储的数æ®ã€‚ + +æ¯ä¸ª Rancher 项目都包å«ä½ åˆ›å»ºçš„ PVC 列表，å¯ä»Ž**èµ„æº > 工作负载 > å·**中查看。你å¯ä»¥åœ¨å°†æ¥åˆ›å»º deployment æ—¶å¤ç”¨è¿™äº› PVC。 + +### 新的和现有的æŒä¹…å­˜å‚¨éƒ½éœ€è¦ PVC + +无论工作负载打算使用现有存储，还是需è¦æ ¹æ®éœ€æ±‚动æ€é…置新存储，pod éƒ½éœ€è¦ PVC æ‰èƒ½ä½¿ç”¨æŒä¹…存储。 + +如果你为工作负载设置现有存储，则工作负载会挂载引用一个 PV çš„ PVC，该 PV 对应现有的存储基础设施。 + +如果工作负载需è¦è¯·æ±‚新的存储，则工作负载会挂载引用一个存储类的 PVC，该存储类能创建新 PV åŠå…¶åº•å±‚存储基础设施。 + +Rancher å…许你在项目中创建任æ„æ•°é‡çš„ PVC。 + +ä½ å¯ä»¥åœ¨åˆ›å»º deployment 时将 PVC 挂载到 deployment 中，也å¯ä»¥åœ¨åˆ›å»º deployment åŽåœ¨ deployment è¿è¡Œçš„时候将 PVC 挂载到 deployment 中。 + +## 使用 PVC å’Œ PV 设置现有存储 + +ä½ çš„ pod å¯ä»¥å°†æ•°æ®å­˜å‚¨åœ¨[å·](https://kubernetes.io/docs/concepts/storage/volumes/)中，但如果 pod å‘生故障，该数æ®å°±ä¼šä¸¢å¤±ã€‚为了解决这个问题，Kubernetes æ供了æŒä¹…å· (PV)。PV 是 Kubernetes 资æºï¼Œè¿™äº›èµ„æºå¯¹åº”于你的 pod å¯ä»¥è®¿é—®çš„外部存储ç£ç›˜æˆ–文件系统。如果一个 pod 崩溃，它的替代 pod å¯ä»¥è®¿é—®æŒä¹…存储中的数æ®ï¼Œè€Œä¸ä¼šä¸¢å¤±ä»»ä½•æ•°æ®ã€‚ + +PV å¯ä»¥æ˜¯ä½ æœ¬åœ°æˆ–托管的云上（Amazon EBS 或 Azure Disk）物ç†ç£ç›˜æˆ–文件系统。 + +在 Rancher 中创建æŒä¹…å·ä¸ä¼šåˆ›å»ºå­˜å‚¨å·ã€‚它åªåˆ›å»ºæ˜ å°„到现有å·çš„ Kubernetes 资æºã€‚因此，在你å¯ä»¥å°†æŒä¹…å·åˆ›å»ºä¸º Kubernetes 资æºä¹‹å‰ï¼Œä½ å¿…须先é…置存储。 + +:::note é‡è¦æ示： + +PV 是在集群级别创建的，这æ„味ç€åœ¨å¤šç§Ÿæˆ·é›†ç¾¤ä¸­ï¼Œæœ‰æƒè®¿é—®ä¸åŒå‘½å空间的团队å¯ä»¥è®¿é—®åŒä¸€ä¸ª PV。 + +::: + +### å°† PV 绑定到 PVC + +当 pod 设置为使用æŒä¹…存储时，pod 会挂载æŒä¹…å·å£°æ˜Ž (PVC)，PVC 的挂载方å¼ä¸Žå…¶ä»– Kubernetes å·ç›¸åŒã€‚在创建æ¯ä¸ª PVC 时，Kubernetes master å°† PVC 视为对存储的请求，并将其绑定到与 PVC 的最低资æºè¦æ±‚相匹é…çš„ PV。并éžæ¯ä¸ª PVC 都ä¿è¯ç»‘定到 PV。详情请å‚阅 Kubernetes [文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)。 + +> 如果没有匹é…çš„å·ï¼Œåˆ™å£°æ˜Žå°†ä¸€ç›´ä¿æŒä¸ºæœªç»‘定状æ€ã€‚当匹é…çš„å·å¯ç”¨æ—¶ï¼Œå£°æ˜Žå°†ä¼šè¢«ç»‘定。例如，é…置有许多 50Gi PV 的集群ä¸ä¼šåŒ¹é…请求 100Gi çš„ PVC。PVC å¯ä»¥åœ¨ 100Gi PV 添加到集群时进行绑定。 + +æ¢å¥è¯è¯´ï¼Œä½ å¯ä»¥åˆ›å»ºæ— é™é‡çš„ PVC，但åªæœ‰åœ¨ Kubernetes master å¯ä»¥æ‰¾åˆ°æ»¡è¶³ PVC 所需的ç£ç›˜ç©ºé—´é‡çš„ PV 时，PVC æ‰ä¼šç»‘定到 PV。 + +è¦åŠ¨æ€é…置新存储，挂载在 pod 中的 PVC 对应的必须是存储类而ä¸æ˜¯æŒä¹…å·ã€‚ + +## 使用 PVC 和存储类é…置新存储 + +存储类å…许你动æ€åˆ›å»º PV，而无需先在基础设施æ供商中创建æŒä¹…存储。 + +例如，如果工作负载绑定到 PVC 并且 PVC 引用 Amazon EBS 存储类，则存储类å¯ä»¥åŠ¨æ€åˆ›å»º EBS å·å’Œç›¸åº”çš„ PV。 + +ç„¶åŽ Kubernetes master 会将新创建的 PV 绑定到你的工作负载的 PVC，从而å…许你的工作负载使用æŒä¹…存储。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md new file mode 100644 index 00000000000..6582e5e0f50 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md @@ -0,0 +1,87 @@ +--- +title: 在 Rancher 中动æ€é…置新存储 +--- + +本文介ç»å¦‚何为 Rancher 中的工作负载é…置新的æŒä¹…存储。 + +本节å‡è®¾ä½ äº†è§£ Kubernetes 的存储类和æŒä¹…å·å£°æ˜Žçš„概念。如需更多信æ¯ï¼Œè¯·å‚阅[存储的工作原ç†](about-persistent-storage.md)。 + +新存储通常由 Amazon EBS 等云æ供商æ供。但是，新存储ä¸ä¸€å®šè¦åœ¨äº‘中。 + +如果你有一个å—存储池并且ä¸æƒ³ä½¿ç”¨äº‘æ供商，你å¯ä»¥ä½¿ç”¨ Longhorn 为 Kubernetes 集群æä¾›æŒä¹…存储。详情请å‚è§[本页é¢](../../../../../integrations-in-rancher/longhorn.md)。 + +è¦ä¸ºä½ çš„工作负载é…置新存储，请执行以下步骤： + +1. [添加一个存储类并将其é…置为使用你的存储](#1-添加一个存储类并将其é…置为使用你的存储) +2. [为使用 StatefulSet 部署的 Pod 使用存储类](#2-为使用-statefulset-部署的-pod-使用存储类) + +### 先决æ¡ä»¶ + +- 设置æŒä¹…存储需è¦`管ç†å·`çš„[角色](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色å‚考)。 +- 如果你è¦ä¸ºäº‘集群é…置存储，则存储和集群主机必须使用相åŒçš„云æ供商。 +- å¿…é¡»å¯ç”¨äº‘æ供商。有关å¯ç”¨äº‘æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../../../pages-for-subheaders/set-up-cloud-providers.md)。 +- ç¡®ä¿ä½ çš„存储å·æ’件å¯ä»¥å¯ç”¨ã€‚ + +默认情况下å¯ç”¨ä»¥ä¸‹å­˜å‚¨å·æ’件： + +| å称 | æ’件 | +--------|---------- +| Amazon EBS Disk | `aws-ebs` | +| AzureFile | `azure-file` | +| AzureDisk | `azure-disk` | +| Google Persistent Disk | `gce-pd` | +| Longhorn | `flex-volume-longhorn` | +| VMware vSphere Volume | `vsphere-volume` | +| 本地 | `local` | +| 网络文件系统 | `nfs` | +| hostPath | `host-path` | + +如果你的存储å·æ’件没有在上述列表中，你需è¦[使用功能开关æ¥å¯ç”¨ä¸å—支æŒçš„存储驱动](../../../../advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md)。 + +### 1. 添加一个存储类并将其é…置为使用你的存储 + +这些步骤æ述了如何在集群级别设置存储类： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦åŠ¨æ€é…ç½®æŒä¹…存储å·çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**存储 > 存储类**。 +1. å•å‡»**创建**。 +1. 输入存储类的å称。 +1. 从 **Provisioner** 下拉列表中，选择è¦ç”¨äºŽåŠ¨æ€é…置存储å·çš„æœåŠ¡ã€‚例如，如果你有一个 Amazon EC2 集群并且想è¦ä½¿ç”¨äº‘存储，请使用 `Amazon EBS Disk`。 +1. 在**å‚æ•°**选项å¡ä¸­ï¼Œå¡«å†™æœåŠ¡ç”¨äºŽåŠ¨æ€é…置存储å·æ‰€éœ€çš„ä¿¡æ¯ã€‚æ¯ä¸ªå·æ’件都需è¦ä¸åŒçš„ä¿¡æ¯æ¥åŠ¨æ€é…置存储å·ã€‚有关如何获å–此信æ¯çš„帮助，请å‚阅æœåŠ¡æ–‡æ¡£ã€‚ +1. å•å‡»**创建**。 + +**结果**：存储类å¯ä¾› PVC 使用。 + +有关存储类å‚数的完整信æ¯ï¼Œè¯·å‚阅官方 [Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/storage-classes/#parameters)。 + +### 2. 为使用 StatefulSet 部署的 Pod 使用存储类 + +StatefulSet ç®¡ç† Pod 的部署和扩展，åŒæ—¶ä¸ºæ¯ä¸ª Pod 维护一个粘性标识。在这个 StatefulSet 中，我们将é…置一个 VolumeClaimTemplate。StatefulSet 管ç†çš„æ¯ä¸ª Pod 都将部署一个基于此 VolumeClaimTemplate çš„ PersistentVolumeClaim。PersistentVolumeClaim 将引用我们创建的 StorageClass。因此，在部署 StatefulSet 管ç†çš„æ¯ä¸ª Pod 时，都会使用 PersistentVolumeClaim 中定义的 StorageClass æ¥ç»‘定到动æ€é…置的存储。 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°† StorageClass 用于工作负载的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡»**创建**。 +1. å•å‡» **StatefulSet**。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. 输入æŒä¹…å·çš„å称。 +1. 在*存储类*\*字段中，选择将为此 StatefulSet 管ç†çš„ pod 动æ€é…置存储的 StorageClass。 +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. 点击**å¯åŠ¨**。 + +**结果**：StatefulSet 管ç†çš„æ¯ä¸ª Pod 部署完æˆåŽï¼Œéƒ½ä¼šå‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到具有兼容 PVC çš„ Pod。 + +è¦å°† PVC 附加到现有工作负载， + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦å°† StorageClass 用于工作负载的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡» **â‹® > 编辑é…ç½®**，转到使用由 StorageClass é…置的存储的工作负载。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**中，å•å‡»**添加声明模æ¿**。 +1. 输入æŒä¹…å·å称。 +1. 在*存储类*\*字段中，选择将为此 StatefulSet 管ç†çš„ pod 动æ€é…置存储的 StorageClass。 +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. å•å‡»**ä¿å­˜**。 + +**结果**ï¼šå·¥ä½œè´Ÿè½½å°†å‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到 PVC。å¦åˆ™ï¼ŒRancher å°†é…置新的æŒä¹…存储。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md new file mode 100644 index 00000000000..ea7fde37298 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md @@ -0,0 +1,62 @@ +--- +title: iSCSI å· +--- + +在将数æ®å­˜å‚¨åœ¨ iSCSI å·ä¸Šçš„ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中，你å¯èƒ½ä¼šé‡åˆ° kubelet 无法自动连接 iSCSI å·çš„问题。æˆæ­¤é—®é¢˜çš„原因很å¯èƒ½æ˜¯ iSCSI å¯åŠ¨å™¨å·¥å…·ä¸å…¼å®¹ã€‚ä½ å¯ä»¥åœ¨æ¯ä¸ªé›†ç¾¤èŠ‚点上安装 iSCSI å¯åŠ¨å™¨å·¥å…·æ¥è§£å†³æ­¤é—®é¢˜ã€‚ + +将数æ®å­˜å‚¨åˆ° iSCSI å·çš„ç”± Rancher å¯åŠ¨çš„ Kubernetes 集群使用 [iSCSI å¯åŠ¨å™¨å·¥å…·](http://www.open-iscsi.com/)，该工具嵌入在 kubelet çš„ `rancher/hyperkube` Docker é•œåƒä¸­ã€‚该工具从æ¯ä¸ª kubeletï¼ˆå³ _initiator_）å‘现并å‘起与 iSCSI å·ï¼ˆå³ _target_）的会è¯ã€‚但是，在æŸäº›æƒ…况下，initiator å’Œ target 上安装的 iSCSI å¯åŠ¨å™¨å·¥å…·çš„版本å¯èƒ½ä¸åŒ¹é…，从而导致连接失败。 + +如果你é‡åˆ°æ­¤é—®é¢˜ï¼Œä½ å¯ä»¥åœ¨é›†ç¾¤ä¸­çš„æ¯ä¸ªèŠ‚点上安装å¯åŠ¨å™¨å·¥å…·æ¥è§£å†³è¯¥é—®é¢˜ã€‚ä½ å¯ä»¥é€šè¿‡ç™»å½•åˆ°é›†ç¾¤èŠ‚点并输入以下其中一个命令æ¥å®‰è£… iSCSI å¯åŠ¨å™¨å·¥å…·ï¼š + +| å¹³å° | 包å | 安装命令 | +| ------------- | ----------------------- | -------------------------------------- | +| Ubuntu/Debian | `open-iscsi` | `sudo apt install open-iscsi` | +| RHEL | `iscsi-initiator-utils` | `yum install iscsi-initiator-utils -y` | + + +在节点上安装å¯åŠ¨å™¨å·¥å…·åŽï¼Œç¼–辑集群的 YAML，编辑 kubelet é…置以挂载 iSCSI 二进制文件和é…置，如下é¢çš„示例所示。 + +:::note 注æ„事项 + +- 在更新 Kubernetes YAML 以挂载 iSCSI 二进制文件和é…置之å‰ï¼Œè¯·ç¡®ä¿å·²å°† `open-iscsi`（deb）或 `iscsi-initiator-utils`（yum）安装到你的集群节点。如果在绑定挂载创建到你的 Kubernetes YAML _之å‰_ 未安装此包，Docker 将自动在æ¯ä¸ªèŠ‚点上创建目录和文件，并且ä¸å…许包安装æˆåŠŸã€‚ + +- 下é¢çš„示例 YAML ä¸é€‚用于 K3s，仅适用于 RKE 集群。由于 K3s kubelet ä¸åœ¨å®¹å™¨ä¸­è¿è¡Œï¼Œå› æ­¤ä¸éœ€è¦æ·»åŠ é¢å¤–的绑定。但是，所有 iSCSI 工具ä»å¿…须安装在你的 K3s 节点上。 + +::: + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" +``` + +如果你使用的是 RHEL 7.9，你需è¦æŒ‚载文件 `/usr/lib64/libcrypto.so.10`，如下例所示。 + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.10:/usr/lib/libcrypto.so.10" +``` + +如果你使用的是 RHEL 8.6，你需è¦æŒ‚载文件 `/usr/lib64/libcrypto.so.1.1`，如下例所示。 + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.1.1:/usr/lib/libcrypto.so.1.1" +``` + + +:::tip + +è¦éªŒè¯ iscsiadm 是å¦æ­£å¸¸å·¥ä½œï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èŠ‚点上è¿è¡Œå‘½ä»¤ `docker exec kubelet iscsiadm --version`。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md new file mode 100644 index 00000000000..2232fc9c253 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md @@ -0,0 +1,85 @@ +--- +title: 设置现有存储 +--- + +本文介ç»å¦‚何在 Rancher 中为工作负载设置现有的æŒä¹…存储。 + +:::note + +本节å‡è®¾ä½ äº†è§£ Kubernetes çš„æŒä¹…å·å’ŒæŒä¹…å·å£°æ˜Žçš„概念。如需更多信æ¯ï¼Œè¯·å‚阅[存储的工作原ç†](about-persistent-storage.md)。 + +::: + +è¦è®¾ç½®å­˜å‚¨ï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ï¼š + +1. [设置æŒä¹…存储](#1-设置æŒä¹…存储)。 +2. [添加一个引用æŒä¹…存储的 PersistentVolume](#2-添加一个引用æŒä¹…存储的-persistentvolume)。 +3. [为使用 StatefulSet 部署的 Pod 使用存储类](#3-为使用-statefulset-部署的-pod-使用存储类) + +### 先决æ¡ä»¶ + +- è¦å°†æŒä¹…å·åˆ›å»ºä¸º Kubernetes 资æºï¼Œä½ å¿…须具有`管ç†å·`çš„[角色。](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色å‚考) +- 如果你è¦ä¸ºäº‘集群é…置存储，则存储和集群主机必须使用相åŒçš„云æ供商。 + +### 1. 设置æŒä¹…存储 + +在 Rancher 中创建æŒä¹…å·ä¸ä¼šåˆ›å»ºå­˜å‚¨å·ã€‚它åªåˆ›å»ºæ˜ å°„到现有å·çš„ Kubernetes 资æºã€‚因此，在你å¯ä»¥å°†æŒä¹…å·åˆ›å»ºä¸º Kubernetes 资æºä¹‹å‰ï¼Œä½ å¿…须先é…置存储。 + +设置æŒä¹…存储设备的步骤会因你的基础设施而异。我们æ供了使用 [vSphere](../../provisioning-storage-examples/vsphere-storage.md)ã€[NFS](../../provisioning-storage-examples/nfs-storage.md) 或 [Amazon EBS ](../../provisioning-storage-examples/persistent-storage-in-amazon-ebs.md)设置存储的示例。 + +如果你有一个å—存储池并且ä¸æƒ³ä½¿ç”¨äº‘æ供商，你å¯ä»¥ä½¿ç”¨ Longhorn 为 Kubernetes 集群æä¾›æŒä¹…存储。详情请å‚è§[本页é¢](../../../../../integrations-in-rancher/longhorn.md)。 + +### 2. 添加一个引用æŒä¹…存储的 PersistentVolume + +这些步骤æ述了如何在 Kubernetes 的集群级别设置 PersistentVolume。 + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ æŒä¹…å·çš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**存储 > æŒä¹…å·**。 +1. å•å‡»**创建**。 +1. 输入æŒä¹…å·çš„**å称**。 +1. 为正在使用的ç£ç›˜ç±»åž‹æˆ–æœåŠ¡é€‰æ‹©**å·æ’件**。将存储添加到由云æ供商托管的集群时，请使用云æ供商的云存储æ’件。例如，如果你有一个 Amazon EC2 集群并且想è¦ä½¿ç”¨äº‘存储，则必须使用 `Amazon EBS ç£ç›˜`å·æ’件。 +1. 输入å·çš„**容é‡**（å•ä½ï¼šGB）。 +1. 填写**æ’件é…ç½®**表å•ã€‚æ¯ä¸ªæ’件类型都需è¦ç£ç›˜ç±»åž‹ä¾›åº”商的信æ¯ã€‚有关æ¯ä¸ªæ’件的表å•å’Œæ‰€éœ€ä¿¡æ¯çš„帮助，请å‚阅æ’件的供应商文档。 +1. å¯é€‰ï¼šåœ¨**自定义**表å•ä¸­ï¼Œé…ç½®[访问模å¼](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)。此选项设置å¯ä»¥è®¿é—®å·çš„节点数é‡ï¼Œä»¥åŠèŠ‚点的读/写æƒé™ã€‚[Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes)中的表格列出了æ’件支æŒçš„访问模å¼ã€‚ +1. å¯é€‰ï¼šåœ¨**自定义**表å•ä¸­ï¼Œé…ç½®[挂载选项](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options)。æ¯ä¸ªå·æ’件都å…许你在安装过程中指定其他命令行选项。请查阅æ¯ä¸ªæ’件的供应商文档以获å–å¯ç”¨çš„挂载选项。 +1. å•å‡»**创建**。 + +**结果**：已创建你的新æŒä¹…å·ã€‚ + + +### 3. 为使用 StatefulSet 部署的 Pod 使用存储类 + +StatefulSet ç®¡ç† Pod 的部署和扩展，åŒæ—¶ä¸ºæ¯ä¸ª Pod 维护一个粘性标识。在这个 StatefulSet 中，我们将é…置一个 VolumeClaimTemplate。StatefulSet 管ç†çš„æ¯ä¸ª Pod 都将部署一个基于此 VolumeClaimTemplate çš„ PersistentVolumeClaim。PersistentVolumeClaim 将引用我们创建的 PersistentVolume。因此，在部署 StatefulSet 管ç†çš„æ¯ä¸ª Pod 时，都会绑定一个 PersistentVolumeClaim 中定义的 PersistentVolume。 + +ä½ å¯ä»¥åœ¨å·¥ä½œè´Ÿè½½åˆ›å»ºæœŸé—´æˆ–之åŽä¸º StatefulSet é…置存储。 + +以下步骤æ述了如何将现有存储分é…给新的 StatefulSet： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦ä¸º StatefulSet é…置存储的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载 > StatefulSets**。 +1. å•å‡»**创建**。 +1. 选择è¦éƒ¨ç½²å·¥ä½œè´Ÿè½½çš„命å空间。 +1. 输入 StatefulSet çš„å称。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. å•å‡»**使用现有的æŒä¹…å·**。 +1. 在**æŒä¹…å·**字段中，选择你创建的æŒä¹…å·ã€‚ +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. 点击**å¯åŠ¨**。 + +**结果**ï¼šå½“å·¥ä½œè´Ÿè½½è¢«éƒ¨ç½²æ—¶ï¼Œå®ƒä¼šå‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到 PVC。 + +以下步骤æ述了如何将æŒä¹…存储分é…给现有工作负载： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦ä¸º StatefulSet é…置存储的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载 > StatefulSets**。 +1. 转到è¦æ·»åŠ æŒä¹…存储的工作负载。å•å‡» **â‹® > 编辑**。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. å•å‡»**使用现有的æŒä¹…å·**。 +1. 在**æŒä¹…å·**字段中，选择你创建的æŒä¹…å·ã€‚ +1. 在**挂载点**字段中，输入工作负载将用于访问å·çš„路径。 +1. 点击**å¯åŠ¨**。 + +**结果**ï¼šå·¥ä½œè´Ÿè½½å°†å‘ Kubernetes master 请求指定的ç£ç›˜ç©ºé—´ã€‚如果在部署工作负载时具有指定资æºçš„ PV 是å¯ç”¨çš„，则 Kubernetes master 会将 PV 绑定到 PVC。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md new file mode 100644 index 00000000000..31f13278ae2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md @@ -0,0 +1,420 @@ +--- +title: 使用外部 Ceph 驱动 +--- + +本文介ç»å¦‚何在 RKE2 集群中使用外部 Ceph 驱动。如果你使用的是 RKE，则需è¦æ‰§è¡Œå…¶ä»–步骤。详情请å‚阅[本节](#在-rke-中使用-ceph-驱动)。 + + +## è¦æ±‚ + +ç¡®ä¿ ceph-common å’Œ xfsprogs 软件包安装在 SLE worker 节点上。 + +## 在 RKE 中使用 Ceph 驱动 + +以下资æºä¸Ž RKE 集群完全兼容，但需è¦ä¸º RKE 执行é¢å¤–çš„ kubelet é…置。 + +在 RKE 集群上，kubelet 组件在 Docker 容器中è¿è¡Œï¼Œé»˜è®¤æƒ…å†µä¸‹æ— æ³•åƒ rbd å’Œ libceph 一样访问主机的内核模å—。 + +为了解决这个é™åˆ¶ï¼Œä½ å¯ä»¥åœ¨ worker 节点上è¿è¡Œ `modprobe rbd`，或者é…ç½® kubelet 容器以将主机中的 `/lib/modules` 目录自动挂载到容器中。 + +对于 kubelet é…置，在 RKE 集群é…置之å‰ï¼Œå°†ä»¥ä¸‹å†…容放入 `cluster.yml` 文件中。你ç¨åŽè¿˜å¯ä»¥åœ¨ Rancher UI 中通过å•å‡»**编辑集群 > 以 YAML 文件编辑**并é‡æ–°å¯åŠ¨ worker 节点æ¥ä¿®æ”¹ `cluster.yml`。 + +```yaml +services: + kubelet: + extra_binds: + - '/lib/modules:/lib/modules:ro' +``` + +有关 `extra_binds` 指令的更多信æ¯ï¼Œè¯·å‚阅[本节](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds)。 + +## 在 RKE2 集群上安装 ceph-csi 驱动 + +:::note + +åªæœ‰åŠ¨æ€ RBD é…置需è¦æ‰§è¡Œè¿™äº›æ­¥éª¤ã€‚ + +::: + +有关 `ceph-csi-rbd` chart 的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://github.com/ceph/ceph-csi/blob/devel/charts/ceph-csi-rbd/README.md)。 + +è¦èŽ·å–有关 SES 集群的详细信æ¯ï¼Œè¯·è¿è¡Œï¼š + +``` +ceph mon dump +``` + +查看输出： + +``` +dumped monmap epoch 3 +epoch 3 +fsid 79179d9d-98d8-4976-ab2e-58635caa7235 +last_changed 2021-02-11T10:56:42.110184+0000 +created 2021-02-11T10:56:22.913321+0000 +min_mon_release 15 (octopus) +0: [v2:10.85.8.118:3300/0,v1:10.85.8.118:6789/0] mon.a +1: [v2:10.85.8.123:3300/0,v1:10.85.8.123:6789/0] mon.b +2: [v2:10.85.8.124:3300/0,v1:10.85.8.124:6789/0] mon.c +``` + +ç¨åŽä½ å°†éœ€è¦ fsid å’Œ mon 地å€å€¼ã€‚ + +## 使用 Helm 安装 ceph-csi 驱动 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +helm repo add ceph-csi https://ceph.github.io/csi-charts +helm repo update +helm search repo ceph-csi -l +helm inspect values ceph-csi/ceph-csi-rbd > ceph-csi-rbd-values.yaml +``` + +修改 `ceph-csi-rbd-values.yaml` 文件并åªä¿ç•™æ‰€éœ€çš„更改： + +```yaml +# ceph-csi-rbd-values.yaml +csiConfig: + - clusterID: "79179d9d-98d8-4976-ab2e-58635caa7235" + monitors: + - "10.85.8.118:6789" + - "10.85.8.123:6789" + - "10.85.8.124:6789" +provisioner: + name: provisioner + replicaCount: 2 +``` + +ç¡®ä¿å¯ä»¥ä»Ž RKE2 集群访问 ceph 监视器（例如通过 ping）。 + +``` +kubectl create namespace ceph-csi-rbd +helm install --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +kubectl rollout status deployment ceph-csi-rbd-provisioner -n ceph-csi-rbd +helm status ceph-csi-rbd -n ceph-csi-rbd +``` + +如果你想直接通过 Helm 修改é…置，则å¯ä»¥ä½¿ç”¨ `ceph-csi-rbd-values.yaml` 文件并调用： + +``` +helm upgrade \ + --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +``` + +## 创建 RBD Ceph èµ„æº + +``` +# 创建一个 ceph 池： +ceph osd pool create myPool 64 64 + +# 创建一个å—设备池： +rbd pool init myPool + +# 创建一个å—设备镜åƒï¼š +rbd create -s 2G myPool/image + +# 创建一个å—设备用户并记录密钥： +ceph auth get-or-create-key client.myPoolUser mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=myPool" | tr -d '\n' | base64 +QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== + +# Base64 加密 ceph 用户 myPoolUser： +echo "myPoolUser" | tr -d '\n' | base64 +bXlQb29sVXNlcg== + +# 创建一个å—设备管ç†å‘˜å¹¶è®°å½•å¯†é’¥ï¼š +ceph auth get-or-create-key client.myPoolAdmin mds 'allow *' mgr 'allow *' mon 'allow *' osd 'allow * pool=myPool' | tr -d '\n' | base64 +QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== + +# Base64 加密 ceph 用户 myPoolAdmin： +echo "myPoolAdmin" | tr -d '\n' | base64 +bXlQb29sQWRtaW4= +``` +## é…ç½® RBD Ceph 访问密钥 + +### ç”¨æˆ·è´¦å· + +对于é™æ€ RBD é…置（ceph 池中的镜åƒå¿…须存在），è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +cat > ceph-user-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-user + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sVXNlcg== + userKey: QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== +EOF + +kubectl apply -f ceph-user-secret.yaml +``` + +### 管ç†å‘˜è´¦å· + +å¯¹äºŽåŠ¨æ€ RBD é…置（用于在给定 ceph 池中自动创建镜åƒï¼‰ï¼Œè¯·è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +cat > ceph-admin-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-admin + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sQWRtaW4= + userKey: QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== +EOF + +kubectl apply -f ceph-admin-secret.yaml +``` + +## 创建 RBD æµ‹è¯•èµ„æº + +### 在 Pod 中使用 RBD + +```yaml +# pod +cat > ceph-rbd-pod-inline.yaml << EOF +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-inline +spec: + containers: + - name: ceph-rbd-pod-inline + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +EOF + +kubectl apply -f ceph-rbd-pod-inline.yaml +kubectl get pod +kubectl exec pod/ceph-rbd-pod-inline -- df -k | grep rbd +``` + +### 在æŒä¹…å·ä¸­ä½¿ç”¨ RBD + +```yaml +# pod-pvc-pv +cat > ceph-rbd-pod-pvc-pv-allinone.yaml << EOF +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ceph-rbd-pv +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ceph-rbd-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-pvc-pv +spec: + containers: + - name: ceph-rbd-pod-pvc-pv + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + persistentVolumeClaim: + claimName: ceph-rbd-pvc +EOF + +kubectl apply -f ceph-rbd-pod-pvc-pv-allinone.yaml +kubectl get pv,pvc,pod +kubectl exec pod/ceph-rbd-pod-pvc-pv -- df -k | grep rbd +``` + +### 在存储类中使用 RBD + +此示例用于动æ€é…ç½®ã€‚éœ€è¦ ceph-csi 驱动。 + +```yaml +# pod-pvc-sc +cat > ceph-rbd-pod-pvc-sc-allinone.yaml < /root/.bashrc << EOF +export PATH=$PATH:/var/lib/rancher/rke2/bin/ +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +EOF + +cat /var/lib/rancher/rke2/server/node-token +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +``` + +### RKE2 Agent/Worker é…ç½® + +``` +mkdir -p /etc/rancher/rke2/ + +cat > /etc/rancher/rke2/config.yaml << EOF +server: https://10.100.103.23:9345 +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +EOF + +curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh - +systemctl enable --now rke2-agent.service +``` + +è¦å°†é›†ç¾¤å¯¼å…¥ Rancher，请å•å‡» **☰ > 集群管ç†**。然åŽåœ¨**集群**页é¢ä¸Šï¼Œå•å‡»**导入已有集群**。然åŽåœ¨ server/master 上è¿è¡Œæ供的 kubectl 命令。 + +## 测试版本 + +è¿è¡Œ RKE2 节点的æ“作系统：安装了 kernel-default-5.3.18-24.49 çš„ JeOS SLE15-SP2 + +``` +kubectl version +Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.4", GitCommit:"c96aede7b5205121079932896c4ad89bb93260af", GitTreeState:"clean", BuildDate:"2020-06-22T12:00:00Z", GoVersion:"go1.13.11", Compiler:"gc", Platform:"linux/amd64"} +Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7+rke2r1", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-20T01:50:52Z", GoVersion:"go1.15.5b5", Compiler:"gc", Platform:"linux/amd64"} + +helm version +version.BuildInfo{Version:"3.4.1", GitCommit:"c4e74854886b2efe3321e185578e6db9be0a6e29", GitTreeState:"clean", GoVersion:"go1.14.12"} +``` + +RKE2 集群上的 Kubernetes 版本：v1.19.7+rke2r1 + +## 故障排除 + +如果你使用的是基于 SES7 çš„ SUSE ceph-rook，你å¯ä»¥é€šè¿‡ç¼–辑 `rook-1.4.5/ceph/cluster.yaml` 并设置 `spec.network.hostNetwork=true` æ¥å…¬å¼€ hostNetwork 上的监视器。 + +如果è¦æ“作 ceph-rook 集群，则å¯ä»¥åœ¨ Kubernetes 集群上部署一个工具箱，其中 ceph-rook 通过 `kubectl apply -f rook-1.4.5/ceph/toolbox.yaml` é…置。然åŽæ‰€æœ‰ä¸Ž ceph 相关的命令都å¯ä»¥åœ¨ toolbox pod 中执行，例如，è¿è¡Œ `kubectl exec -it -n rook-ceph rook-ceph-tools-686d8b8bfb-2nvqp -- bash`。 + +ceph æ“作 - 基本命令： + +``` +ceph osd pool stats +ceph osd pool delete myPool myPool --yes-i-really-really-mean-it +rbd list -p myPool +> csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 +> image +``` + +删除镜åƒï¼š`rbd rm csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 -p myPool` + +rook 工具箱中的 CephFS 命令： + +``` +ceph -s +ceph fs ls +ceph fs fail cephfs +ceph fs rm cephfs --yes-i-really-mean-it +ceph osd pool delete cephfs_data cephfs_data --yes-i-really-really-mean-it +ceph osd pool delete cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it +``` + +è¦å‡†å¤‡ cephfs 文件系统，你å¯ä»¥åœ¨ rook 集群上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +kubectl apply -f rook-1.4.5/ceph/filesystem.yaml +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md new file mode 100644 index 00000000000..a262560e90f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md @@ -0,0 +1,575 @@ +--- +title: 通过 AWS EC2 Auto Scaling 组使用 Cluster Autoscaler +--- + +本指å—介ç»å¦‚何使用 AWS EC2 Auto Scaling 组在 Rancher 自定义集群上安装和使用 [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/)。 + +我们将安装一个 Rancher RKE 自定义集群，该集群具有固定数é‡çš„具有 etcd å’Œ controlplane 角色的节点，以åŠæ•°é‡å¯å˜çš„具有 worker 角色的节点，它们由 `cluster-autoscaler` 管ç†ã€‚ + + +## 先决æ¡ä»¶ + +本指å—è¦æ±‚： + +* Rancher Server 正常è¿è¡Œã€‚ +* ä½ çš„ AWS EC2 用户具有创建虚拟机ã€Auto Scaling ç»„ä»¥åŠ IAM é…置文件和角色的适当æƒé™ã€‚ + +### 1. 创建自定义集群 + +在 Rancher Server 上，我们需è¦åˆ›å»ºä¸€ä¸ªè‡ªå®šä¹‰çš„ K8s 集群。请å‚阅[此处](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/)以检查版本兼容性。 + +è¯·ç¡®ä¿ cloud_provider å称设置为 `amazonec2`。创建集群åŽï¼Œæˆ‘们需è¦èŽ·å¾—： + +* clusterID：`c-xxxxx` 将用于 EC2 `kubernetes.io/cluster/` 实例标签。 +* clusterName：将用于 EC2 `k8s.io/cluster-autoscaler/` 实例标签。 +* nodeCommand：将添加到 EC2 实例 `user_data` 上，以包å«é›†ç¾¤ä¸Šçš„新节点。 + + ```sh + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum + ``` + +### 2. é…置云æ供商 + +在 AWS EC2 上，我们需è¦åˆ›å»ºä¸€äº›å¯¹è±¡æ¥é…置系统。为了在 AWS 上进行é…置，我们定义了三个ä¸åŒçš„组和 IAM é…置文件。 + +1. Autoscaling 组：将加入 EC2 Auto Scaling 组 (ASG) 的节点。`cluster-autoscaler` 将使用 ASG æ¥è¿›è¡Œæ‰©ç¼©å®¹ã€‚ +* IAM é…置文件：è¿è¡Œ cluster-autoscaler çš„ K8s 节点需è¦è¯¥æ–‡ä»¶ã€‚推è用于 Kubernetes master 节点。此é…置文件称为 `K8sAutoscalerProfile`。 + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:DescribeTags", + "autoscaling:DescribeLaunchConfigurations", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + +2. Master 组：将æˆä¸º Kubernetes etcd å’Œ/或 controlplane 的节点。该组ä¸ä¼šåœ¨ ASG 中。 +* IAM é…ç½®æ–‡ä»¶ï¼šé›†æˆ Kubernetes cloud_provider 需è¦è¯¥æ–‡ä»¶ã€‚或者，你也å¯ä»¥ä½¿ç”¨ `AWS_ACCESS_KEY` å’Œ `AWS_SECRET_KEY` æ¥ä»£æ›¿ [using-aws-credentials](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials)。此é…置文件称为 `K8sMasterProfile`。 + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + + * IAM 角色：`K8sMasterRole: [K8sMasterProfile,K8sAutoscalerProfile]` + * 安全组：`K8sMasterSg`。详情请å‚è§ [RKE 端å£ï¼ˆè‡ªå®šä¹‰èŠ‚点选项å¡ï¼‰](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#下游-kubernetes-集群节点) + * 标签： + `kubernetes.io/cluster/: owned` + * 用户数æ®ï¼š`K8sMasterUserData` Ubuntu 18.04(ami-0e11cbb34015ff725)，安装 Docker 并将 etcd å’Œ controlplane 节点添加到 K8s 集群。 + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--etcd --controlplane" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +3. Worker 组：将加入 K8s worker plane 的节点。Worker 节点将由 cluster-autoscaler 使用 ASG 进行扩缩容。 +* IAM é…置文件：æä¾› cloud_provider worker 集æˆã€‚ + æ­¤é…置文件称为 `K8sWorkerProfile`。 + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } + ] + } + ``` + +* IAM 角色：`K8sWorkerRole：[K8sWorkerProfile]` +* 安全组：`K8sWorkerSg`。详情请å‚è§ [RKE 端å£ï¼ˆè‡ªå®šä¹‰èŠ‚点选项å¡ï¼‰](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#下游-kubernetes-集群节点) +* 标签： + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` +* 用户数æ®ï¼š`K8sWorkerUserData` Ubuntu 18.04(ami-0e11cbb34015ff725)，安装 Docker 并将 worker 节点添加到 K8s 集群。 + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--worker" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +详情请å‚è§ [AWS 上的 RKE 集群](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md) å’Œ [AWS 上的 Cluster Autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md)。 + +### 3. 部署节点 + +我们é…ç½® AWS åŽï¼Œæˆ‘们需è¦åˆ›å»ºè™šæ‹Ÿæœºæ¥å¼•å¯¼é›†ç¾¤ï¼š + +* master (etcd+controlplane)：根æ®éœ€è¦éƒ¨ç½²ä¸‰ä¸ªé€‚当大å°çš„ master 实例。详情请å‚è§[生产就绪集群的建议](../../../../pages-for-subheaders/checklist-for-production-ready-clusters.md)。 + * IAM 角色：`K8sMasterRole` + * 安全组：`K8sMasterSg` + * 标签： + * `kubernetes.io/cluster/: owned` + * 用户数æ®ï¼š`K8sMasterUserData` + +* worker：使用以下设置在 EC2 上定义 ASG： + * å称：`K8sWorkerAsg` + * IAM 角色：`K8sWorkerRole` + * 安全组：`K8sWorkerSg` + * 标签： + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` + * 用户数æ®ï¼š`K8sWorkerUserData` + * 实例： + * 最少：2 + * ç†æƒ³æƒ…况：2 + * 最大：10 + +部署 VM åŽï¼Œä½ çš„ Rancher 自定义集群应该å¯ä»¥æ­£å¸¸è¿è¡Œäº†ï¼Œå…¶ä¸­åŒ…å«ä¸‰ä¸ª master 节点和两个 worker 节点。 + +### 4. 安装 Cluster-autoscaler + +此时，我们的 Rancher 集群应该已正常è¿è¡Œã€‚æˆ‘ä»¬å°†æ ¹æ® cluster-autoscaler 的建议，在 master 节点和 `kube-system` 命å空间上安装 cluster-autoscaler。 + +#### å‚æ•° + +下表显示了用于微调的 cluster-autoscaler å‚数： + +| å‚æ•° | 默认 | æè¿° | +|---|---|---| +| cluster-name | - | 自动扩缩的集群的å称（如果å¯ç”¨ï¼‰ | +| address | :8085 | 公开 Prometheus æŒ‡æ ‡çš„åœ°å€ | +| kubernetes | - | Kubernetes master ä½ç½®ã€‚如需使用默认值，则留空 | +| kubeconfig | - | 带有授æƒå’Œ master ä½ç½®ä¿¡æ¯çš„ kubeconfig 文件的路径 | +| cloud-config | - | 云æ供商é…置文件的路径。如果没有é…置文件，则为空字符串 | +| namespace | "kube-system" | è¿è¡Œ cluster-autoscaler 的命å空间 | +| scale-down-enabled | true | CA 是å¦åº”该缩å‡é›†ç¾¤ | +| scale-down-delay-after-add | "10m" | 扩容多久åŽæ¢å¤ç¼©å®¹è¯„ä¼° | +| scale-down-delay-after-delete | 0 | 节点删除多久åŽæ¢å¤ç¼©å®¹è¯„估，默认为 scanInterval 的值 | +| scale-down-delay-after-failure | "3m" | 缩容失败多久åŽæ¢å¤ç¼©å®¹è¯„ä¼° | +| scale-down-unneeded-time | "10m" | 在能进行缩容之å‰ï¼ŒèŠ‚点需è¦ä¸è¢«ä½¿ç”¨çš„时间 | +| scale-down-unready-time | "20m" | 在能进行缩容之å‰ï¼Œéžå°±ç»ªèŠ‚点应该需è¦ä¸è¢«ä½¿ç”¨çš„时间 | +| scale-down-utilization-threshold | 0.5 | 节点上è¿è¡Œçš„所有 pod çš„ CPU 或内存之和除以节点对应的å¯åˆ†é…资æºï¼Œä½ŽäºŽè¯¥å€¼æ—¶å¯ä»¥è€ƒè™‘缩å‡ä¸€ä¸ªèŠ‚点 | +| scale-down-gpu-utilization-threshold | 0.5 | 节点上è¿è¡Œçš„所有 pod çš„ GPU 请求总和除以节点的å¯åˆ†é…资æºï¼Œä½ŽäºŽè¯¥å€¼æ—¶å¯ä»¥è€ƒè™‘缩å‡ä¸€ä¸ªèŠ‚点 | +| scale-down-non-empty-candidates-count | 30 | 在一次迭代中被认为是空节点的最大数é‡ï¼Œè¿™äº›èŠ‚点会æˆä¸ºä½¿ç”¨æ¸…空æ¥ç¼©å®¹çš„候选节点 | +| scale-down-candidates-pool-ratio | 0.1 | 当先å‰è¿­ä»£çš„æŸäº›å€™é€‰èŠ‚点失效时，被视为è¦ç¼©å®¹çš„é¢å¤–éžç©ºå€™é€‰èŠ‚点的比率 | +| scale-down-candidates-pool-min-count | 50 | 当先å‰è¿­ä»£çš„æŸäº›å€™é€‰èŠ‚点失效时，被视为è¦ç¼©å®¹çš„é¢å¤–éžç©ºå€™é€‰èŠ‚ç‚¹çš„æœ€å¤§æ•°é‡ | +| node-deletion-delay-timeout | "2m" | CA 在删除节点之å‰ï¼Œç­‰å¾…删除 delay-deletion.cluster-autoscaler.kubernetes.io/ 注释的最长时间 | +| scan-interval | "10s" | é‡æ–°è¯„估集群扩缩容的频率 | +| max-nodes-total | 0 | 所有节点组中的最大节点数。Cluster Autoscaler ä¸ä¼šè®©é›†ç¾¤å¢žé•¿åˆ°è¶…è¿‡æ­¤æ•°é‡ | +| cores-total | "0:320000" | 集群中的最å°å’Œæœ€å¤§çš„核心数，格å¼ä¸º `:`。Cluster Autoscaler 会在该范围内扩缩集群。 | +| memory-total | "0:6400000" | 集群中最å°å’Œæœ€å¤§å†…å­˜åƒå…†å­—节数，格å¼ä¸º `:`。Cluster Autoscaler 会在该范围内扩缩集群。 | +| cloud-provider | - | 云æ供商类型 | +| max-bulk-soft-taint-count | 10 | å¯ä»¥åŒæ—¶æ·»åŠ /移除 PreferNoSchedule 污点的最大节点数。设置为 0 则关闭此类污点 | +| max-bulk-soft-taint-time | "3s" | åŒæ—¶æ·»åŠ /移除 PreferNoSchedule 污点的最大æŒç»­æ—¶é—´ã€‚ | +| max-empty-bulk-delete | 10 | å¯ä»¥åŒæ—¶åˆ é™¤çš„最大空节点数 | +| max-graceful-termination-sec | 600 | å°è¯•ç¼©å‡èŠ‚点时，CA 等待 pod 终止的最大秒数 | +| max-total-unready-percentage | 45 | 集群中未就绪节点的最大百分比。超过此值åŽï¼ŒCA å°†åœæ­¢æ“作 | +| ok-total-unready-count | 3 | å…许的未就绪节点数，与 max-total-unready-percentage æ— å…³ | +| scale-up-from-zero | true | 就绪节点数等于 0 时，CA 是å¦åº”该扩容 | +| max-node-provision-time | "15m" | CA 等待节点é…置的最长时间 | +| nodes | - | 以云æ供商接å—çš„æ ¼å¼è®¾ç½®èŠ‚点组的最å°ã€æœ€å¤§å¤§å°å’Œå…¶ä»–é…置数æ®ã€‚å¯ä»¥å¤šæ¬¡ä½¿ç”¨ã€‚æ ¼å¼æ˜¯ `::`。 | +| node-group-auto-discovery | - | 节点组自动å‘现的一个或多个定义。定义表示为 `:[[=]]` | +| estimator | - | "binpacking" | 用于扩容的资æºè¯„估器类型。å¯ç”¨å€¼ï¼š["binpacking"] | +| expander | "random" | è¦åœ¨æ‰©å®¹ä¸­ä½¿ç”¨çš„节点组扩展器的类型。å¯ç”¨å€¼ï¼š`["random","most-pods","least-waste","price","priority"]` | +| ignore-daemonsets-utilization | false | CA 为了缩容而计算资æºåˆ©ç”¨çŽ‡æ—¶ï¼Œæ˜¯å¦åº”忽略 DaemonSet pod | +| ignore-mirror-pods-utilization | false | CA 为了缩容而计算资æºåˆ©ç”¨çŽ‡æ—¶ï¼Œæ˜¯å¦åº”忽略 Mirror pod | +| write-status-configmap | true | CA 是å¦åº”该将状æ€ä¿¡æ¯å†™å…¥ configmap | +| max-inactivity | "10m" | 从上次记录的 autoscaler 活动åŽï¼Œè‡ªåŠ¨é‡å¯ä¹‹å‰çš„最长时间 | +| max-failing-time | "15m" | 从上次记录的 autoscaler æˆåŠŸè¿è¡ŒåŽï¼Œè‡ªåŠ¨é‡å¯ä¹‹å‰çš„最长时间 | +| balance-similar-node-groups | false | 检测相似的节点组，并å‡è¡¡å®ƒä»¬çš„èŠ‚ç‚¹æ•°é‡ | +| node-autoprovisioning-enabled | false | CA 是å¦åº”在需è¦æ—¶è‡ªåŠ¨é…置节点组 | +| max-autoprovisioned-node-group-count | 15 | 集群中自动é…ç½®ç»„çš„æœ€å¤§æ•°é‡ | +| unremovable-node-recheck-timeout | "5m" | 在å†æ¬¡æ£€æŸ¥æ— æ³•åˆ é™¤çš„节点之å‰ï¼ŒèŠ‚点的超时时间 | +| expendable-pods-priority-cutoff | -10 | 优先级低于 cutoff çš„ Pod 将是消耗性 pod。这些 pod å¯ä»¥åœ¨ç¼©å®¹æœŸé—´ä¸åŠ è€ƒè™‘地被终止，并且ä¸ä¼šå¯¼è‡´æ‰©å®¹ã€‚优先级是 null（ç¦ç”¨ PodPriority）的 pod ä¸æ˜¯æ¶ˆè€—性的 | +| regional | false | 集群是区域性的 | +| new-pod-scale-up-delay | "0s" | 生命短于这个值的 Pod å°†ä¸è€ƒè™‘扩容 | +| ignore-taint | - | 在扩缩容节点组时，指定在节点模æ¿ä¸­è¦å¿½ç•¥çš„污点 | +| balancing-ignore-label | - | 在比较两个节点组是å¦ç›¸ä¼¼æ—¶ï¼ŒæŒ‡å®šè¦å¿½ç•¥çš„标签（基本标签集和云æ供商标签集除外） | +| aws-use-static-instance-list | false | CA 在è¿è¡Œæ—¶è¿˜æ˜¯ä½¿ç”¨é™æ€åˆ—表获å–实例类型。仅适用于 AWS | +| profiling | false | 是å¦å¯ç”¨äº† debug/pprof 端点 | + +#### 部署 + +基于 [cluster-autoscaler-run-on-control-plane.yaml](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml) 示例，我们已ç»åˆ›å»ºäº†è‡ªå·±çš„ `cluster-autoscaler-deployment.yaml` 以使用首选的 [auto-discovery 设置](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#auto-discovery-setup)，更新容å¿åº¦ã€nodeSelectorã€é•œåƒç‰ˆæœ¬å’Œå‘½ä»¤é…置： + + +```yml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/controlplane + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + containers: + - image: eu.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler: + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + +``` + +准备好清å•æ–‡ä»¶åŽï¼Œå°†è¯¥æ–‡ä»¶éƒ¨ç½²åˆ° Kubernetes 集群中（也å¯ä»¥ä½¿ç”¨ Rancher UI 进行æ“作）： + +```sh +kubectl -n kube-system apply -f cluster-autoscaler-deployment.yaml +``` + +:::note + +你也å¯ä»¥é€šè¿‡[手动é…ç½®](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#manual-configuration)æ¥è®¾ç½® Cluster-autoscaler deployment。 + +::: + +## 测试 + +此时，cluster-autoscaler 应该已ç»åœ¨ Rancher 自定义集群中å¯åŠ¨å¹¶è¿è¡Œã€‚当满足以下æ¡ä»¶ä¹‹ä¸€æ—¶ï¼Œcluster-autoscaler 需è¦ç®¡ç† `K8sWorkerAsg` ASG，以在 2 到 10 个节点之间进行扩缩容： + +* 集群中有 Pod 因资æºä¸è¶³è€Œæ— æ³•è¿è¡Œã€‚在这ç§æƒ…况下，集群被扩容。 +* 集群中有一些节点长时间未得到充分利用，而且它们的 Pod å¯ä»¥æ”¾åˆ°å…¶ä»–现有节点上。在这ç§æƒ…况下，集群被缩容。 + +### 生æˆè´Ÿè½½ + +为了在 Kubernetes 集群上产生负载并查看 cluster-autoscaler 是å¦æ­£å¸¸å·¥ä½œï¼Œæˆ‘们准备了一个 `test-deployment.yaml`。`test-deployment` 通过三个副本请求 1000m CPU å’Œ 1024Mi 内存。通过调整请求的资æºå’Œ/或副本以确ä¿è€—å°½ Kubernetes 集群资æºï¼š + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hello-world + name: hello-world +spec: + replicas: 3 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + ports: + - containerPort: 80 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1024Mi + requests: + cpu: 1000m + memory: 1024Mi +``` + +准备好 test deployment åŽï¼Œå°†å…¶éƒ¨ç½²åœ¨ Kubernetes 集群的默认命å空间中（å¯ä»¥ä½¿ç”¨ Rancher UI）： + +``` +kubectl -n default apply -f test-deployment.yaml +``` + +### 检查扩缩容 + +Kubernetes 资æºè€—å°½åŽï¼Œcluster-autoscaler 应该扩容无法调度 pod çš„ worker 节点。它应该进行扩容，直到所有 pod 都能被调度。你应该会在 ASG å’Œ Kubernetes 集群上看到新节点。检查 `kube-system` cluster-autoscaler pod 上的日志。 + +检查完扩容åŽï¼Œæˆ‘们开始检查缩容。为此，请å‡å°‘ test deployment 的副本数，直到你能释放足够的 Kubernetes 集群资æºä»¥è¿›è¡Œç¼©å®¹ã€‚你应该能看到 ASG å’Œ Kubernetes 集群上的节点消失了。检查 `kube-system` cluster-autoscaler pod 上的日志。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md new file mode 100644 index 00000000000..9cb5a041693 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md @@ -0,0 +1,130 @@ +--- +title: é›†ç¾¤æ¨¡æ¿ +--- + +集群模æ¿åŒ…å« Kubernetes é…置和节点池é…​​置，å…许å•ä¸ªæ¨¡æ¿åŒ…å« Rancher 在云æ供商中é…置新节点并在这些节点上安装 Kubernetes 所需的所有信æ¯ã€‚ + +## 概述 + +集群模æ¿ä»¥ Helm Chart çš„å½¢å¼æ供。è¦ä½¿ç”¨é›†ç¾¤æ¨¡æ¿ï¼Œä½ éœ€è¦å…‹éš†å’Œå¤åˆ»æ¨¡æ¿ï¼Œæ ¹æ®ä½ çš„用例更改模æ¿ï¼Œç„¶åŽåœ¨ Rancher 管ç†é›†ç¾¤ä¸Šå®‰è£… Helm Chart。如果 Helm Chart 安装在 Rancher 管ç†é›†ç¾¤ä¸Šï¼Œä¼šåˆ›å»ºä¸€ä¸ªæ–°çš„集群资æºï¼ŒRancher 使用它æ¥é…置新集群。 + +使用模æ¿é…置集群åŽï¼Œå¯¹æ¨¡æ¿çš„任何更改都ä¸ä¼šå½±å“集群。使用集群模æ¿åˆ›å»ºé›†ç¾¤åŽï¼Œé›†ç¾¤çš„é…置和基础设施å¯èƒ½ä¼šå‘生å˜åŒ–，因为集群模æ¿æ²¡æœ‰ä»»ä½•æ‰§è¡Œé™åˆ¶ã€‚ + +### Kubernetes å‘行版 + +集群模æ¿å¯ä»¥ä½¿ç”¨ä»»ä½• Kubernetes å‘行版。现在，我们æ供一个带有 RKE2 Kubernetes 集群的示例。将æ¥æˆ‘们å¯èƒ½ä¼šæ供更多使用其他 Kubernetes å‘行版的示例。 + +### 版本控制 + +Rancher ä¸ç®¡ç†é›†ç¾¤æ¨¡æ¿çš„版本控制。版本控制在包å«æ¨¡æ¿çš„ Helm Chart 的仓库中处ç†ã€‚ + +## RKE2 é›†ç¾¤æ¨¡æ¿ + +请å‚阅 [RKE2 集群模æ¿](https://github.com/rancher/cluster-template-examples)的示例仓库。 + +## å‘ Rancher æ·»åŠ é›†ç¾¤æ¨¡æ¿ + +在本节中，你将学习如何将集群模æ¿æ·»åŠ åˆ° `local` 集群的 Chart 仓库列表中。这样，当用户安装新的 Kubernetes 集群时，Rancher å°†æ供集群模æ¿çš„选项。 + +:::note 先决æ¡ä»¶ï¼š + +- 你需è¦æœ‰åœ¨`本地` Kubernetes 集群上安装 Helm Chart çš„æƒé™ã€‚ +- 为了使 Chart 以创建新集群的形å¼å‡ºçŽ°ï¼ŒChart 必须具有注释 `catalog.cattle.io/type: cluster-template`。 + +::: + +1. 转到集群模æ¿ç¤ºä¾‹ä»“库。Rancher 的示例ä½äºŽ[æ­¤ GitHub 仓库中](https://github.com/rancher/cluster-template-examples)。 +1. å¤åˆ»ä»“库。 +1. å¯é€‰ï¼šé€šè¿‡ç¼–辑 `values.yaml` 文件æ¥ç¼–辑集群选项。有关编辑文件的帮助，请å‚阅集群模æ¿çš„ Helm Chart 自述文件。 +1. å°† Chart 仓库添加到 Rancher。点击 **☰ > 集群管ç†**。 +1. 转到 `local` 集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Chart Repositories**。 +1. 点击**创建**。 +1. 输入集群模æ¿ä»“库的å称。 +1. 点击**åŒ…å« Helm Chart 定义的 Git 仓库**。 +1. 在 **Git 仓库 URL**字段，输入仓库的 URL，例如，`https://github.com/rancher/cluster-template-examples.git`。 +1. 在 **Git 分支**字段，输入è¦ç”¨ä½œæ¨¡æ¿æºçš„分支。Rancher 的示例仓库使用了 `main` 分支。 +1. 点击**创建**。 + +**结果**：集群模æ¿å¯ä»Ž Rancher `local` 集群中的 **Apps** 获得。现在，你å¯ä»¥ä½¿ç”¨æ¨¡æ¿æ¥éƒ¨ç½²é›†ç¾¤ã€‚ + +:::note å—é™ç®¡ç†å‘˜è®¿é—®ï¼š + +如果你是å—é™ç®¡ç†å‘˜å¹¶ä¸”æ— æƒè®¿é—® `local` 集群，你ä»ç„¶å¯ä»¥æ·»åŠ æ–°çš„ RKE2 模æ¿å¹¶ç®¡ç†é›†ç¾¤ä»“库。è¦å¯¼èˆªåˆ° Chart 仓库，请在左侧导航æ å•å‡» **☰ > é›†ç¾¤ç®¡ç† > 高级选项 > 仓库**。你将绕过上述步骤 1 - 6，然åŽç»§ç»­æŒ‰ç…§æ­¥éª¤ 7 - 12 创建集群模æ¿ã€‚ + +::: + +## 使用集群模æ¿åˆ›å»ºé›†ç¾¤ + +:::note 先决æ¡ä»¶ï¼š + +- 你需è¦èŽ·å¾—é…置新 Kubernetes 集群的æƒé™ã€‚ +- 你将需è¦äº‘凭è¯æ¥ä½¿ç”¨æ¨¡æ¿é…置基础设施。 +- 为了以创建新集群的形å¼æ˜¾ç¤ºï¼Œé›†ç¾¤æ¨¡æ¿çš„ Helm Chart 必须具有 `catalog.cattle.io/type: cluster-template` 注释。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. å•å‡»é›†ç¾¤æ¨¡æ¿çš„å称。 +1. å®Œæˆ Helm Chart 的安装。 + +**结果**：在 Rancher é…置新集群åŽï¼Œä½ å¯ä»¥åƒç®¡ç†å…¶ä»– Rancher å¯åŠ¨çš„ Kubernetes 集群一样管ç†è¯¥æ–°é›†ç¾¤ã€‚如果集群模æ¿æœ‰å¯ä¾›ç”¨æˆ·é€‰æ‹©çš„选项，你å¯ä»¥é€šè¿‡ UI é…置这些选项。 + +## 更新使用集群模æ¿åˆ›å»ºçš„集群 + +如果集群所用的模æ¿æœ‰äº†æ–°ç‰ˆæœ¬ï¼Œä½ å¯ä»¥ä½¿ç”¨ **Apps > Installed Apps** 页é¢ä¸­çš„模æ¿æ›´æ–°é›†ç¾¤ã€‚ + +## 使用 Fleet 从模æ¿éƒ¨ç½²é›†ç¾¤ + +:::note 先决æ¡ä»¶ï¼š + +- 你需è¦èŽ·å¾—é…置新 Kubernetes 集群的æƒé™ã€‚ +- 你将需è¦äº‘凭è¯æ¥ä½¿ç”¨æ¨¡æ¿é…置基础设施。 +- 为了以创建新集群的形å¼æ˜¾ç¤ºï¼Œé›†ç¾¤æ¨¡æ¿çš„ Helm Chart 必须具有 `catalog.cattle.io/type:cluster-template` 注释。 +- 为了将模æ¿ç”¨ä½œæŒç»­äº¤ä»˜/GitOps 的一部分，集群模æ¿éœ€è¦éƒ¨ç½²åœ¨`本地`集群的 `fleet-local` 命å空间中。 +- 所有值都必须在模æ¿çš„ `values.yaml` 中设置。 +- Fleet 仓库必须éµå¾ªæ­¤å¤„çš„[指å—](https://fleet.rancher.io/gitrepo-content)。对于 RKE2 集群模æ¿ï¼Œåˆ™å¿…须把 `fleet.yaml` 文件添加到仓库。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œç‚¹å‡»**创建**。 +1. 点击**使用模æ¿åˆ›å»ºé›†ç¾¤**。 + +**结果**：Rancher é…置好新集群åŽï¼Œé›†ç¾¤ç”± Fleet 管ç†ã€‚ + +## å¸è½½é›†ç¾¤æ¨¡æ¿ + +1. 点击 **☰ > 集群管ç†**。 +1. å‰å¾€ `local` 集群并å•å‡» **Apps > Chart Repositories**。 +1. 转到集群模æ¿çš„ Chart 仓库，然åŽå•å‡» **â‹® > 删除。** +1. 确认删除。 + +**结果**：集群模æ¿å·²å¸è½½ã€‚æ­¤æ“作ä¸ä¼šå½±å“使用集群模æ¿åˆ›å»ºçš„现有集群。 + +能够访问 `local` 集群的管ç†å‘˜è¿˜å¯ä»¥å‰å¾€ **Apps > Installed Apps** 页é¢ï¼Œé€šè¿‡é›†ç¾¤æ¨¡æ¿æ¥ç§»é™¤å·²éƒ¨ç½²çš„集群。 + +## é…置选项 + +集群模æ¿éžå¸¸çµæ´»ï¼Œå¯ç”¨äºŽé…置以下所有选项： + +- 节点é…ç½® +- 节点池 +- é¢„å…ˆæŒ‡å®šçš„äº‘å‡­è¯ +- å¯ç”¨/é…置授æƒçš„集群端点，以在ä¸ä½¿ç”¨ Rancher 作为代ç†çš„情况下获得对集群的 kubectl 访问æƒé™ +- 安装 Rancher V2 monitoring +- Kubernetes 版本 +- 分é…集群æˆå‘˜ +- é…置基础设施，例如 AWS VPC/å­ç½‘或 vSphere æ•°æ®ä¸­å¿ƒ +- 云æ供商选项 +- Pod 安全选项 +- 网络æ供商 +- Ingress Controller +- 网络安全é…ç½® +- 网络æ’件 +- ç§æœ‰é•œåƒä»“库 URL å’Œå‡­è¯ +- 附加组件 +- Kubernetes 选项，包括 kube-apiã€kube-controllerã€kubelet å’ŒæœåŠ¡ç­‰ Kubernetes 组件的é…ç½® + +有关如何é…置模æ¿çš„详细信æ¯ï¼Œè¯·å‚阅集群模æ¿çš„ Helm Chart README 文件。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md new file mode 100644 index 00000000000..2174397f7ce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md @@ -0,0 +1,185 @@ +--- +title: 节点和节点池 +--- + +在 Rancher 中å¯åŠ¨ Kubernetes 集群åŽï¼Œä½ å¯ä»¥ä»Žé›†ç¾¤çš„**节点**选项å¡ç®¡ç†å„个节点。ä¸åŒçš„é…置集群[选项](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)对应ä¸åŒçš„å¯ç”¨èŠ‚点选项。 + +:::note + +å¦‚æžœä½ æƒ³ç®¡ç† _集群_ 而ä¸æ˜¯å•ä¸ªèŠ‚点，请å‚阅[编辑集群](../../../pages-for-subheaders/cluster-configuration.md)。 + +::: + + +## æ¯ä¸ªé›†ç¾¤åˆ›å»ºé€‰é¡¹çš„å¯ç”¨èŠ‚点选项 + +下表列出了 Rancher 中æ¯ç§é›†ç¾¤ç±»åž‹çš„å¯ç”¨èŠ‚点选项。å•å‡»**选项**列中的链接以获å–æ¯ä¸ªåŠŸèƒ½çš„更多信æ¯ã€‚ + +| 选项 | [由基础设施æ供商托管的节点][1] | [自定义节点][2] | [托管集群][3] | [注册的 EKS 节点][4] | [所有其他注册节点][5] | æè¿° | +| ------------------------------------------------ | ------------------------------------------------ | ---------------- | ------------------- | ------------------- | -------------------| ------------------------------------------------------------------ | +| [å°é”](#å°é”节点) | ✓ | ✓ | ✓ | ✓ | ✓ | 将节点标记为ä¸å¯è°ƒåº¦ã€‚ | +| [清空](#清空节点) | ✓ | ✓ | ✓ | ✓ | ✓ | 将节点标记为ä¸å¯è°ƒåº¦ _并且_ 驱é€æ‰€æœ‰ pod。 | +| [编辑](#管ç†å’Œç¼–辑å•ä¸ªèŠ‚点) | ✓ | ✓ | ✓ | ✓ | ✓ | 输入节点的自定义å称ã€æè¿°ã€æ ‡ç­¾æˆ–污点。 | +| [查看 API](#在-rancher-api-中查看节点) | ✓ | ✓ | ✓ | ✓ | ✓ | 查看 API æ•°æ®ã€‚ | +| [删除](#删除节点) | ✓ | ✓ | | * | * | 从集群中删除有缺陷的节点。 | +| [下载密钥](#通过-ssh-连接到由基础设施æ供商托管的节点) | ✓ | | | | | 下载 SSH 密钥以通过 SSH 连接到节点。 | +| [节点缩放](#扩缩节点) | ✓ | | | ✓ | | å‘上或å‘下扩展节点池中的节点数。 | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +[5]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +\* å¯é€šè¿‡ View API 访问的删除选项 + + +### 由基础设施æ供商托管的节点 + +在[托管在基础设施æ供商](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)中的节点上é…置由 Rancher å¯åŠ¨çš„ Kubernetes 集群时，你å¯ä»¥ä½¿ç”¨èŠ‚点池。 + +如果节点池被编辑，通过[节点池选项](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)é…置的集群å¯ä»¥çºµå‘扩容或缩容。 + +如果å¯ç”¨[节点自动替æ¢åŠŸèƒ½](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点自动替æ¢)，节点池还å¯ä»¥è‡ªåŠ¨ç»´æŠ¤åœ¨åˆå§‹é›†ç¾¤é…置期间设置的节点规模。该规模决定了 Rancher 为集群维护的 active 节点的数é‡ã€‚ + +Rancher 使用[节点模æ¿](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)æ¥æ›¿æ¢èŠ‚点池中的节点。æ¯ä¸ªèŠ‚点模æ¿éƒ½ä½¿ç”¨äº‘æ供商凭è¯æ¥å…许 Rancher 在基础设施æ供商中设置节点。 + +### 由托管 Kubernetes æ供商é…置的节点 + +用于管ç†[ç”± Kubernetes æ供商托管](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)的节点的选项在 Rancher 中有些é™åˆ¶ã€‚例如，你ä¸èƒ½ä½¿ç”¨ Rancher UI å‘上或å‘下缩放节点数é‡ï¼Œè€Œæ˜¯éœ€è¦ç›´æŽ¥ç¼–辑集群。 + +### 注册节点 + +虽然你å¯ä»¥ä½¿ç”¨ Rancher 将工作负载部署到[已注册集群](../../new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)，但你无法管ç†å•ä¸ªé›†ç¾¤èŠ‚点。导入的集群节点的所有管ç†éƒ½å¿…须在 Rancher 之外进行。 + +## 管ç†å’Œç¼–辑å•ä¸ªèŠ‚点 + +编辑节点å¯è®©ä½ ï¼š + +* 更改其å称 +* 更改其æè¿° +* 添加[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +* 添加/删除[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + +è¦ç®¡ç†å•ä¸ªèŠ‚点，请æµè§ˆåˆ°è¦ç®¡ç†çš„集群，然åŽä»Žä¸»èœå•ä¸­é€‰æ‹©**节点**。你å¯ä»¥é€šè¿‡å•å‡»èŠ‚点的 **â‹®** (**...**) 图标æ¥æ‰“开选项èœå•ã€‚ + +## 在 Rancher API 中查看节点 + +选择此选项以查看节点的 [API 端点](../../../pages-for-subheaders/about-the-api.md)。 + +## 删除节点 + +使用 **Delete** 从云æ供商中删除有缺陷的节点。 + +当你删除有缺陷的节点时，如果该节点在节点池中并å¯ç”¨äº†[节点自动替æ¢](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点自动替æ¢)，Rancher å¯ä»¥è‡ªåŠ¨å°†å…¶æ›¿æ¢ä¸ºå…·æœ‰ç›¸åŒé…置的节点。 + +:::tip + +如果你的集群由基础设施æ供商托管，并且你希望缩容集群，而ä¸æ˜¯åˆ é™¤æœ‰ç¼ºé™·çš„节点，请进行[缩容](#扩缩节点)而ä¸æ˜¯åˆ é™¤ã€‚ + +::: + +## 扩缩节点 + +对于由基础设施æ供商托管的节点，你å¯ä»¥ä½¿ç”¨ç¼©æ”¾æŽ§ä»¶æ¥ç¼©æ”¾æ¯ä¸ª[节点池](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点池)中的节点数é‡ã€‚此选项ä¸é€‚用于其他集群类型。 + +## 通过 SSH 连接到由基础设施æ供商托管的节点 + +对于[由基础设施æ供商托管的节点](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)，你å¯ä»¥é€‰æ‹©ä¸‹è½½å…¶ SSH 密钥，以便从桌é¢è¿œç¨‹è¿žæŽ¥åˆ°å®ƒã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦é€šè¿‡ SSH 连接到节点的集群，然åŽå•å‡»é›†ç¾¤å称。 +1. 在**主机池**选项å¡ä¸Šï¼Œæ‰¾åˆ°è¦è¿œç¨‹è®¿é—®çš„节点，然åŽå•å‡» **â‹® > 下载 SSH 密钥**。这将下载用于 SSH 的文件的 ZIP 包。 +1. å°† ZIP 文件解压缩到任何ä½ç½®ã€‚ +1. 打开终端。转到解压了的 ZIP 文件的ä½ç½®ã€‚ +1. 输入以下命令： + + ``` + ssh -i id_rsa root@ + ``` + +## å°é”节点 + +_å°é”_ 节点表示将节点标记为ä¸å¯è°ƒåº¦ã€‚此功能适用于在短期维护（如é‡å¯ï¼Œå‡çº§æˆ–åœç”¨ï¼‰æ—¶åœ¨èŠ‚点上执行短期任务。完æˆåŽï¼Œé‡æ–°æ‰“开电æºå¹¶é€šè¿‡å–消å°é”使节点å†æ¬¡å¯è°ƒåº¦ã€‚ + +## 清空节点 + +_清空_ 指的是先å°é”节点，然åŽé©±é€å…¶æ‰€æœ‰ pod 的过程。此功能对于执行节点维护（如内核å‡çº§æˆ–硬件维护）很有用。它å¯ä»¥é˜²æ­¢æ–°çš„ pod 部署到节点，åŒæ—¶èƒ½é‡æ–°åˆ†é…现有的 pod，从而é¿å…用户é‡åˆ°æœåŠ¡ä¸­æ–­çš„情况。 + +- 具有副本集的 pod 将被一个新的 pod 替æ¢ï¼Œè¯¥æ–° pod 将被调度到一个新节点。此外，如果 pod 是æœåŠ¡çš„一部分，则客户端将自动é‡å®šå‘到新的 pod。 + +- 对于没有副本集的 pod，你需è¦è°ƒå‡º pod 的新副本。如果该 pod ä¸æ˜¯æœåŠ¡çš„一部分，将客户端é‡å®šå‘到它。 + +ä½ å¯ä»¥æ¸…空处于 `cordoned` 或 `active` 状æ€çš„节点。清空一个节点时，该节点会被å°é”，然åŽä¼šè¯„估节点是å¦æ»¡è¶³æ¸…空的必备è¦æ±‚，如果满足è¦æ±‚，则会驱é€èŠ‚点的 pod。 + +但是，你å¯ä»¥åœ¨å¯åŠ¨æ¸…空时覆盖清空æ¡ä»¶ã€‚你还有机会设置宽é™æœŸå’Œè¶…时值。 + +### 激进和安全的清空选项 + +为集群é…ç½®å‡çº§ç­–略时，你将能够å¯ç”¨èŠ‚点清空。如果å¯ç”¨äº†èŠ‚点清空，你将能够é…置如何删除和é‡æ–°è°ƒåº¦ pod。 + +- **激进模å¼** + + 在这ç§æ¨¡å¼ä¸‹ï¼Œå³ä½¿ pod 没有控制器也ä¸ä¼šè¢«é‡æ–°è°ƒåº¦åˆ°æ–°èŠ‚点。Kubernetes 会认为你拥有自己的 pod 删除逻辑。 + + Kubernetes 还会认为你有处ç†ä½¿ç”¨ emptyDir çš„ pod 的实现。如果 Pod 使用 emptyDir 存储本地数æ®ï¼Œä½ å¯èƒ½æ— æ³•å®‰å…¨åœ°åˆ é™¤è¯¥ pod，因为一旦 Pod 从节点中删除，emptyDir 中的数æ®å°†è¢«åˆ é™¤ã€‚选择激进模å¼å°†åˆ é™¤è¿™äº› pod。 + +- **安全模å¼** + + 如果一个节点有独立的 pod 或临时数æ®ï¼Œå®ƒå°†è¢«å°é”但ä¸ä¼šè¢«æ¸…空。 +### 宽é™æœŸ + +ç»™æ¯ä¸ª pod 清ç†çš„超时时间，从而让它们优雅地退出。例如，pod å¯èƒ½éœ€è¦å®Œæˆä»»ä½•æœªå®Œæˆçš„请求ã€å›žæ»šäº‹åŠ¡æˆ–将状æ€ä¿å­˜åˆ°æŸä¸ªå¤–部存储。如果该值为负数，将使用 pod 中指定的默认值。 + +### 超时 + +在清空放弃之å‰åº”该继续等待的时间。 + +:::note Kubernetes 已知问题： + +Kubernetes 1.12 之å‰çš„版本中，在清空节点时ä¸ä¼šå¼ºåˆ¶æ‰§è¡Œ[超时设置](https://github.com/kubernetes/kubernetes/pull/64378)。 + +::: + +### 清空和å°é”çŠ¶æ€ + +如果有任何与用户输入相关的错误，节点会由于清空失败进入 `cordoned` 状æ€ã€‚ä½ å¯ä»¥æ›´æ­£è¾“入并å†æ¬¡å°è¯•æ¸…空节点，也å¯ä»¥é€šè¿‡è§£å°èŠ‚点æ¥ä¸­æ­¢ã€‚ + +如果清空没有错误，则节点会进入 `draining` 状æ€ã€‚当节点处于此状æ€æ—¶ï¼Œä½ å¯ä»¥é€‰æ‹©åœæ­¢æ¸…空，这将åœæ­¢æ¸…空过程并将节点的状æ€æ›´æ”¹ä¸º `cordoned`。 + +清空æˆåŠŸå®ŒæˆåŽï¼ŒèŠ‚点将处于 `drained` 状æ€ã€‚然åŽä½ å¯ä»¥å…³é—­æˆ–删除节点。 + +有关**å°é”和清空**的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/)。 + +## 标记 Rancher 忽略的节点 + +æŸäº›è§£å†³æ–¹æ¡ˆï¼ˆä¾‹å¦‚ F5 çš„ BIG-IP 集æˆï¼‰å¯èƒ½éœ€è¦åˆ›å»ºä¸€ä¸ªä¸ä¼šæ³¨å†Œåˆ°é›†ç¾¤çš„节点。 + +由于节点永远ä¸ä¼šå®Œæˆæ³¨å†Œï¼Œå› æ­¤å®ƒåœ¨ Rancher UI 中总是显示为ä¸å¥åº·ã€‚ + +在这ç§æƒ…况下，你å¯èƒ½å¸Œæœ›å°†èŠ‚点标记为 Rancher 忽略的节点，从而让 Rancher 仅在节点实际å‘生故障时将节点状æ€æ˜¾ç¤ºä¸ºä¸å¥åº·ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 或 `kubectl` 标记è¦å¿½ç•¥çš„节点。 + +:::note + +已知一个[未解决的问题](https://github.com/rancher/rancher/issues/24172)，å³æ ‡è®°ä¸ºè¢«å¿½ç•¥çš„节点å¯èƒ½ä¼šå¡åœ¨æ›´æ–°çŠ¶æ€ã€‚ + +::: + + +### 使用 kubectl 标记è¦å¿½ç•¥çš„节点 + +è¦æ·»åŠ  Rancher 忽略的节点，请使用 `kubectl` 创建具有以下标签的节点： + +``` +cattle.rancher.io/node-status: ignore +``` + +**结果**：如果将节点添加到集群中，Rancher å°†ä¸ä¼šå°è¯•ä¸Žè¯¥èŠ‚点åŒæ­¥ã€‚该节点ä»ç„¶å¯ä»¥æ˜¯é›†ç¾¤çš„一部分，并且å¯ä»¥ä½¿ç”¨ `kubectl` 列出。 + +如果在将节点添加到集群之å‰æ·»åŠ äº†æ ‡ç­¾ï¼Œåˆ™è¯¥èŠ‚点将ä¸ä¼šæ˜¾ç¤ºåœ¨ Rancher UI 中。 + +如果在将节点添加到 Rancher 集群åŽæ·»åŠ æ ‡ç­¾ï¼Œåˆ™ä¸ä¼šä»Ž UI 中删除该节点。 + +如果你使用 Rancher UI 或 API 从 Rancher server 中删除节点，å‡å¦‚ `nodeName` 在 Rancher API 下的 `v3/settings/ignore-node-name` 中列出，则ä¸ä¼šä»Žé›†ç¾¤ä¸­åˆ é™¤è¯¥èŠ‚点。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md new file mode 100644 index 00000000000..f8e5d94e510 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md @@ -0,0 +1,197 @@ +--- +title: 使用 Rancher 的项目和 Kubernetes 命å空间 +description: Rancher Projects å‡è½»äº†é›†ç¾¤çš„管ç†è´Ÿæ‹…并支æŒå¤šç§Ÿæˆ·ã€‚学习创建项目并将项目划分为 Kubernetes 命å空间 +--- + +命å空间是 Kubernetes 的概念，它å…许你在集群中使用虚拟集群。这对于将集群划分为å•ç‹¬çš„“虚拟集群â€å¾ˆæœ‰ç”¨ï¼Œæ¯ä¸ªâ€œè™šæ‹Ÿé›†ç¾¤â€éƒ½æœ‰è‡ªå·±çš„访问控制和资æºé…é¢ã€‚ + +项目是一组命å空间，是 Rancher 引入的一个概念。项目å…许你将多个命å空间作为一个组进行管ç†ï¼Œå¹¶åœ¨å…¶ä¸­æ‰§è¡Œ Kubernetes æ“作。你å¯ä»¥ä½¿ç”¨é¡¹ç›®æ¥æ”¯æŒå¤šç§Ÿæˆ·ï¼Œä»¥ä¾¿å›¢é˜Ÿå¯ä»¥åœ¨é›†ç¾¤å†…访问项目，而无需访问åŒä¸€é›†ç¾¤ä¸­çš„其他项目。 + +:::note + +UI 上ä¸å†å¼ºè°ƒé¡¹ç›®ï¼Œå› ä¸ºä¸éœ€è¦åœ¨é¡¹ç›®èŒƒå›´å†…创建 Kubernetes 资æºã€‚但是，如果å¯ç”¨äº†æ—§ç‰ˆåŠŸèƒ½å¼€å…³ï¼Œåˆ™ä»å¯ä»¥åœ¨é¡¹ç›®èŒƒå›´å†…创建诸如 [Secrets](../../new-user-guides/kubernetes-resources-setup/secrets.md#在项目中创建密文) 之类的资æºã€‚ + +::: + +本文介ç»é¡¹ç›®å’Œå‘½å空间如何与 Rancher 一起使用。 + +## 关于命å空间 + +命å空间是 Kubernetes 引入的一个概念。详情请å‚è§[命å空间的官方 Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)。 + +:::note + +Kubernetes 支æŒç”±åŒä¸€ä¸ªç‰©ç†é›†ç¾¤æ”¯æŒçš„多个虚拟集群。这些虚拟集群称为命å空间。命å空间适用于多用户分布在多个团队或项目中的环境。如果你的集群åªæœ‰å‡ ä¸ªåˆ°å‡ å个用户，你ä¸éœ€è¦åˆ›å»ºæˆ–考虑使用命å空间。 + +::: + +命å空间æ供以下功能： + +- **为å称æ供范围**：资æºå称在命å空间内必须是唯一的，但跨命å空间时则ä¸éœ€è¦ã€‚命å空间ä¸èƒ½ç›¸äº’嵌套，æ¯ä¸ª Kubernetes 资æºåªèƒ½ä½äºŽä¸€ä¸ªå‘½å空间中。 +- **资æºé…é¢**：命å空间æ供了在多用户之间划分集群资æºçš„方法。 + +ä½ å¯ä»¥åœ¨é¡¹ç›®çº§åˆ«åˆ†é…资æºï¼Œä»¥ä¾¿é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间都å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºã€‚你也å¯ä»¥é€šè¿‡å°†èµ„æºæ˜¾å¼åˆ†é…给命å空间æ¥ç»•è¿‡æ­¤ç»§æ‰¿æ¨¡å¼ã€‚ + +ä½ å¯ä»¥å°†ä»¥ä¸‹èµ„æºç›´æŽ¥åˆ†é…给命å空间： + +- [工作负载](../../../pages-for-subheaders/workloads-and-pods.md) +- [è´Ÿè½½å‡è¡¡å™¨/Ingress](../../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [æœåŠ¡å‘现记录](../../new-user-guides/kubernetes-resources-setup/create-services.md) +- [æŒä¹…å·å£°æ˜Ž](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [è¯ä¹¦](../../new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMap](../../new-user-guides/kubernetes-resources-setup/configmaps.md) +- [é•œåƒä»“库](../../new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [密文](../../new-user-guides/kubernetes-resources-setup/secrets.md) + +为了在 vanilla Kubernetes 集群中管ç†æƒé™ï¼Œé›†ç¾¤ç®¡ç†å‘˜è¦ä¸ºæ¯ä¸ªå‘½å空间é…置基于角色的访问策略。Rancher 在项目级别上分é…用户æƒé™ï¼Œé¡¹ç›®ä¸­çš„命å空间会自动继承项目的æƒé™ã€‚ + +有关创建和移动命å空间的更多信æ¯ï¼Œè¯·å‚阅[命å空间](../../new-user-guides/manage-namespaces.md)。 + +### 命å空间和 kubectl çš„ RBAC 问题 + +由于项目是 Rancher 引入的概念，因此 kubectl ä¸èƒ½å°†å‘½å空间的创建é™åˆ¶åœ¨åˆ›å»ºè€…å¯ä»¥è®¿é—®çš„项目中。 + +è¿™æ„味ç€å½“具有项目范围æƒé™çš„普通用户使用 `kubectl` 创建命å空间时，由于 `kubectl` ä¸éœ€è¦å°†æ–°å‘½å空间é™åˆ¶åœ¨ç‰¹å®šé¡¹ç›®ä¸­ï¼Œåˆ›å»ºçš„命å空间å¯èƒ½æ— æ³•ä½¿ç”¨ã€‚ + +如果你的æƒé™ä»…é™äºŽé¡¹ç›®çº§åˆ«ï¼Œåˆ™æœ€å¥½[通过 Rancher 创建命å空间](../../new-user-guides/manage-namespaces.md)，以确ä¿ä½ æœ‰æƒè®¿é—®è¯¥å‘½å空间。 + +如果普通用户是项目所有者，则该用户将能够在该项目中创建命å空间。Rancher UI 将阻止该用户在他们有æƒè®¿é—®çš„项目之外创建命å空间。 + +## 关于项目 + +在层次结构方é¢ï¼š + +- 集群包å«é¡¹ç›® +- 项目包å«å‘½å空间 + +ä½ å¯ä»¥ä½¿ç”¨é¡¹ç›®æ¥æ”¯æŒå¤šç§Ÿæˆ·ï¼Œä»¥ä¾¿å›¢é˜Ÿå¯ä»¥åœ¨é›†ç¾¤å†…访问项目，而无需访问åŒä¸€é›†ç¾¤ä¸­çš„其他项目。 + +在 Kubernetes 的基础版本中，RBAC 或集群资æºç­‰åŠŸèƒ½è¢«åˆ†é…给了å„个命å空间。项目å…许你让个人或团队åŒæ—¶è®¿é—®å¤šä¸ªå‘½å空间æ¥èŠ‚çœæ—¶é—´ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨é¡¹ç›®æ¥æ‰§è¡Œä»¥ä¸‹æ“作： + +- 将用户分é…到一组命å空间（å³[项目æˆå‘˜èº«ä»½](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)）。 +- 在项目中为用户分é…特定角色。角色å¯ä»¥æ˜¯æ‰€æœ‰è€…ã€æˆå‘˜ã€åªè¯»æˆ–[自定义](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md)。 +- 为项目分é…资æºã€‚ +- åˆ†é… Pod 安全策略。 + +创建集群时，会在其中自动创建两个项目： + +- [Default 项目](#集群的-default-项目) +- [System 项目](#system-项目) + +### 集群的 Default 项目 + +使用 Rancher é…置集群时，Rancher 会自动为集群创建一个 `default` 项目。你å¯ä»¥ä½¿ç”¨è¿™ä¸ªé¡¹ç›®æ¥å¼€å§‹ä½¿ç”¨é›†ç¾¤ã€‚但你始终å¯ä»¥åˆ é™¤è¯¥é¡¹ç›®ï¼Œå¹¶ç”¨å…·æœ‰æ›´å¤šæ述性å称的项目替æ¢å®ƒã€‚ + +如果你åªéœ€è¦ä½¿ç”¨ `default` 命å空间，那么你就ä¸éœ€è¦ Rancher 中 **Default** 之外的其他项目。 + +å¦‚æžœä½ éœ€è¦ **Default** 之外的其他项目，你å¯ä»¥åœ¨ Rancher 中创建更多项目æ¥éš”离命å空间ã€åº”用和资æºã€‚ + +### System 项目 + +故障排除时，你å¯ä»¥æŸ¥çœ‹ `system` 项目æ¥æ£€æŸ¥ Kubernetes 系统中é‡è¦çš„命å空间是å¦æ­£å¸¸ã€‚这是一个易于访问的项目，能让你é¿å…对系统命å空间容器进行é€ä¸ªæŽ’查。 + +è¦æ‰“开这个项目，请打开集群视图并å•å‡»**集群 > 项目/命å空间**。该视图显示了 `system` 项目中的所有命å空间。 + +`system` 项目： + +- 在你é…置集群时自动创建。 +- 列出存在于 `v3/settings/system-namespaces` 中的所有命å空间（如果存在）。 +- å…许你添加更多命å空间，或将项目内的命å空间移动到其他项目。 +- 无法删除，因为它是集群æ“作所必需的。 + +:::note + +在å¯ç”¨äº†é¡¹ç›®ç½‘络隔离的 RKE 集群中，`system` 项目会覆盖项目网络隔离选项，以便项目能与其他项目通信ã€æ”¶é›†æ—¥å¿—和检查å¥åº·çŠ¶æ€ã€‚ + +::: + +## é¡¹ç›®æŽˆæƒ + +普通用户仅在两ç§æƒ…况下会被授æƒè®¿é—®é¡¹ç›®ï¼š + +- 管ç†å‘˜ã€é›†ç¾¤æ‰€æœ‰è€…或集群æˆå‘˜å°†æ™®é€šç”¨æˆ·æ˜¾å¼æ·»åŠ åˆ°é¡¹ç›®çš„**æˆå‘˜**中。 +- 普通用户å¯ä»¥è®¿é—®ä»–们自己创建的项目。 + +## Pod 安全策略 + +Rancher 在 Kubernetes 之上进行了扩展，除了集群级别之外，还å…许在项目级别应用 [Pod 安全策略](https://kubernetes.io/docs/concepts/policy/pod-security-policy/)。但是，最佳实践是在集群级别应用 Pod 安全策略。 + +## 创建项目 + +本节介ç»å¦‚何创建具有å称以åŠå¯é€‰ pod 安全策略ã€æˆå‘˜å’Œèµ„æºé…é¢çš„新项目。 + +1. [命å新项目](#1-命å新项目) +2. [å¯é€‰ï¼šé€‰æ‹© pod 安全策略](#2-å¯é€‰é€‰æ‹©-pod-安全策略) +3. [推è：添加项目æˆå‘˜](#3-推è添加项目æˆå‘˜) +4. [å¯é€‰ï¼šæ·»åŠ èµ„æºé…é¢](#4-å¯é€‰æ·»åŠ èµ„æºé…é¢) + +### 1. 命å新项目 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºé¡¹ç›®çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. å•å‡»**创建项目**。 +1. 输入**项目å称**。 + +### 2. å¯é€‰ï¼šé€‰æ‹© Pod 安全策略 + +此选项仅在 Pod 安全策略已创建时å¯ç”¨ã€‚有关说明，请å‚阅[创建 Pod 安全策略](../authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +å°† PSP 分é…给项目将： + +- 覆盖集群的默认 PSP。 +- å°† PSP 应用于项目。 +- å°† PSP 应用到åŽç»­æ·»åŠ åˆ°é¡¹ç›®ä¸­çš„命å空间。 + +### 3. 推è：添加项目æˆå‘˜ + +使用**æˆå‘˜**为其他用户æ供项目访问æƒé™å’Œè§’色。 + +默认情况下，你的用户会被添加为项目的 `Owner`（所有者）。 + +:::note æƒé™è¯´æ˜Žï¼š + +- 如果用户分é…到了项目的`所有者`或`æˆå‘˜`角色，用户会自动继承`命å空间创建`角色。然而，这个角色是 [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole)，这表示角色的范围会延展到集群中的所有项目。因此，对于显å¼åˆ†é…到了项目`所有者`或`æˆå‘˜`角色的用户æ¥è¯´ï¼Œå³ä½¿åªæœ‰`åªè¯»`角色，这些用户也å¯ä»¥åœ¨åˆ†é…给他们的其他项目中创建命å空间。 + +- 默认情况下，Rancher çš„`项目æˆå‘˜`角色继承自 `Kubernetes-edit` 角色，而`项目所有者`角色继承自 `Kubernetes-admin` 角色。因此，`项目æˆå‘˜`å’Œ`项目所有者`角色都能管ç†å‘½å空间，包括创建和删除命å空间。 + +- 选择`自定义`æ¥ç«‹å³åˆ›å»ºè‡ªå®šä¹‰è§’色：[自定义项目角色](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#自定义项目角色)。 + +::: + +è¦æ·»åŠ æˆå‘˜ï¼š + +1. 在**æˆå‘˜**选项å¡ä¸­ï¼Œå•å‡»**添加**。 +1. 在**选择æˆå‘˜**字段中，æœç´¢è¦åˆ†é…项目访问æƒé™çš„用户或组。请注æ„，如果你å¯ç”¨äº†å¤–部身份验è¯ï¼Œåˆ™åªèƒ½æœç´¢ç»„。 +1. 在**项目æƒé™**中选择一个角色。如需更多信æ¯ï¼Œè¯·å‚阅[项目角色文档](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)。 + +### 4. å¯é€‰ï¼šæ·»åŠ èµ„æºé…é¢ + +资æºé…é¢ç”¨äºŽé™åˆ¶é¡¹ç›®ï¼ˆåŠå…¶å‘½å空间）å¯ä»¥ä½¿ç”¨çš„资æºã€‚有关详细信æ¯ï¼Œè¯·å‚阅[资æºé…é¢](projects-and-namespaces.md)。 + +è¦æ·»åŠ èµ„æºé…é¢ï¼š + +1. 在**资æºé…é¢**选项å¡ä¸­ï¼Œå•å‡»**添加资æº**。 +1. 选择一个**资æºç±»åž‹**。有关详细信æ¯ï¼Œè¯·å‚阅[资æºé…é¢](projects-and-namespaces.md)。 +1. 输入**项目é™åˆ¶**å’Œ**命å空间默认é™åˆ¶**的值。 +1. **å¯é€‰**：指定**容器默认资æºé™åˆ¶**，这将应用于项目中å¯åŠ¨çš„所有容器。如果资æºé…é¢è®¾ç½®äº† CPU 或内存é™åˆ¶ï¼Œåˆ™å»ºè®®ä½¿ç”¨è¯¥å‚数。å¯ä»¥åœ¨å•ä¸ªå‘½å空间或容器级别上覆盖它。有关详细信æ¯ï¼Œè¯·å‚阅[容器默认资æºé™åˆ¶](../../../pages-for-subheaders/manage-project-resource-quotas.md)。 +1. å•å‡»**创建**。 + +**结果**：项目已创建。你å¯ä»¥ä»Žé›†ç¾¤çš„**项目/命å空间**视图中查看它。 + +| 字段 | æè¿° | +| ----------------------- | -------------------------------------------------------------------------------------------------------- | +| 项目é™åˆ¶ | 项目的总资æºé™åˆ¶ã€‚ | +| 命å空间默认é™åˆ¶ | æ¯ä¸ªå‘½å空间的默认资æºé™åˆ¶ã€‚æ­¤é™åˆ¶åœ¨åˆ›å»ºæ—¶ä¼šæ²¿ç”¨åˆ°é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间。项目中所有命å空间的é™åˆ¶ä¹‹å’Œä¸åº”超过项目é™åˆ¶ã€‚ | + +## 删除项目 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°é¡¹ç›®è¿žæŽ¥çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到è¦åˆ é™¤çš„项目，点击 **â‹®**。 +1. 选择**删除**。 + +删除项目时，以å‰ä¸Žè¯¥é¡¹ç›®å…³è”的任何命å空间都将ä¿ç•™åœ¨é›†ç¾¤ä¸Šã€‚ä½ å¯ä»¥åœ¨ Rancher UI **项目/命å空间**页é¢çš„**ä¸åœ¨é¡¹ç›®å†…**选项å¡ä¸­æ‰¾åˆ°è¿™äº›å‘½å空间。你å¯ä»¥é€šè¿‡[移动](../manage-namespaces.md#将命å空间移动到å¦ä¸€ä¸ªé¡¹ç›®)æ¥å°†å®ƒä»¬é‡æ–°åˆ†é…给项目。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md new file mode 100644 index 00000000000..569a052d5fe --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md @@ -0,0 +1,75 @@ +--- +title: NFS 存储 +--- + +在将 NFS 存储å·æ’件用于 Rancher deployment 之å‰ï¼Œä½ éœ€è¦é…ç½® NFS æœåŠ¡å™¨ã€‚ + +:::note + +- 如果你已ç»æ‹¥æœ‰ NFS 共享，则无需é…置新的 NFS æœåŠ¡å™¨å³å¯åœ¨ Rancher 中使用 NFS å·æ’件。这样的è¯ï¼Œä½ å¯ä»¥è·³è¿‡æ­¤è¿‡ç¨‹çš„其余部分并直接[添加存储](../../../../pages-for-subheaders/create-kubernetes-persistent-storage.md)。 + +- 此教程演示了如何使用 Ubuntu 设置 NFS æœåŠ¡å™¨ã€‚你也应该能够将这些说明用于其他 Linux å‘行版（例如 Debianã€RHELã€Arch Linux 等）。有关如何使用å¦ä¸€ä¸ª Linux å‘行版创建 NFS æœåŠ¡å™¨çš„官方说明，请å‚阅å‘行版的文档。 + +::: + +:::note 推è： + +è¦ç®€åŒ–管ç†é˜²ç«å¢™è§„则的过程，请使用 NFSv4。 + +::: + +1. 使用远程终端连接，登录到你打算用于 NFS 存储的 Ubuntu æœåŠ¡å™¨ã€‚ + +1. 输入以下命令： + + ``` + sudo apt-get install nfs-kernel-server + ``` + +1. 输入以下命令，设置存储目录以åŠç”¨æˆ·è®¿é—®æƒé™ã€‚如果你想使用ä¸åŒçš„存储目录，请修改该命令： + + ``` + mkdir -p /nfs && chown nobody:nogroup /nfs + ``` + - `-p /nfs` å‚数在根目录下创建一个å为 `nfs` 的目录。 + - `chown nobody:nogroup /nfs` å‚æ•°å…许对存储目录的所有访问。 + +1. 创建 NFS 导出表。此表设置 NFS æœåŠ¡å™¨ä¸Šçš„目录路径，这些路径会暴露给将使用æœåŠ¡å™¨è¿›è¡Œå­˜å‚¨çš„节点。 + + 1. 使用文本编辑器打开 `/etc/exports`。 + 1. 添加你在步骤 3 中创建的 `/nfs` 文件夹的路径以åŠé›†ç¾¤èŠ‚点的 IP 地å€ã€‚为集群中的æ¯ä¸ª IP 地å€æ·»åŠ ä¸€ä¸ªæ¡ç›®ã€‚在æ¯ä¸ªåœ°å€åŠå…¶é™„带的å‚æ•°åŽé¢åŠ ä¸Šä¸€ä¸ªä½œä¸ºåˆ†éš”符的空格： + + ``` + /nfs (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) + ``` + + :::tip + + ä½ å¯ä»¥å°† IP 地å€æ›¿æ¢ä¸ºå­ç½‘，例如，`10.212.50.12/24`。 + + ::: + + 1. 通过输入以下命令更新 NFS 表： + + ``` + exportfs -ra + ``` + +1. 打开 NFS 使用的端å£ã€‚ + + 1. è¦æ‰¾å‡º NFS 正在使用的端å£ï¼Œè¯·è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + + ``` + rpcinfo -p | grep nfs + ``` + 2. [打开上一个命令输出的端å£](https://help.ubuntu.com/lts/serverguide/firewall.html.en)ã€‚ä¾‹å¦‚ï¼Œä»¥ä¸‹å‘½ä»¤æ‰“å¼€ç«¯å£ 2049： + + ``` + sudo ufw allow 2049 + ``` + +**结果**：已将 NFS æœåŠ¡å™¨é…置你的 Rancher 节点的存储。 + +## åŽç»­æ“作 + +在 Rancher 中，将 NFS æœåŠ¡å™¨æ·»åŠ ä¸ºå­˜å‚¨å·å’Œ/或存储类。添加æœåŠ¡å™¨åŽï¼Œä½ å¯ä»¥å°†å…¶ç”¨äºŽå­˜å‚¨ä»¥è¿›è¡Œéƒ¨ç½²ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md new file mode 100644 index 00000000000..3955afea1f2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md @@ -0,0 +1,15 @@ +--- +title: 在 Amazon EBS 中创建æŒä¹…存储 +--- + +本文介ç»å¦‚何在 EC2 中设置 Amazon çš„ Elastic Block Store。 + +1. 在 EC2 控制å°ä¸­ï¼Œè½¬åˆ°å·¦ä¾§é¢æ¿ä¸­çš„ **ELASTIC BLOCK STORE** 中，然åŽå•å‡» **Volumes**。 +1. å•å‡» **Create Volume**。 +1. å¯é€‰ï¼šé…ç½®å·çš„大å°æˆ–其他选项。你需è¦åœ¨å·è¦æŒ‚载到的实例所在的å¯ç”¨åŒºä¸­åˆ›å»ºå·ã€‚ +1. å•å‡» **Create Volume**。 +1. 点击 **Close**。 + +**结果**：已创建æŒä¹…存储。 + +有关如何在 Rancher 中设置新创建的存储的详细信æ¯ï¼Œè¯·å‚阅[设置现有存储](../../../new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md new file mode 100644 index 00000000000..1fb1c5d5e06 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md @@ -0,0 +1,72 @@ +--- +title: vSphere 存储 +--- + +è¦ä¸ºæœ‰çŠ¶æ€çš„工作负载æä¾› vSphere 存储，我们建议创建一个 vSphereVolume StorageClass。当工作负载通过 PersistentVolumeClaim 请求å·æ—¶ï¼Œè¿™ç§åšæ³•ä¼šåŠ¨æ€è°ƒé… vSphere 存储。 + +为了在 vSphere 中动æ€è°ƒé…存储，必须å¯ç”¨ vSphere æ供商。有关更多信æ¯ï¼Œè¯·å‚阅[树外 vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md) å’Œ[树内 vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)。 + + +### 先决æ¡ä»¶ + +为了在 [Rancher Kubernetes Engine (RKE)](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群中é…ç½® vSphere å·ï¼Œ[vSphere cloud provider](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere) 必须在[集群选项](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md)中显å¼å¯ç”¨ã€‚ + +### 创建一个 StorageClass + +:::tip + +你也å¯ä»¥ä½¿ç”¨ `kubectl` 命令行工具æ¥æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[关于æŒä¹…å·çš„ Kubernetes 文档](https://kubernetes.io/docs/concepts/storage/persistent-volumes/)。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 选择è¦ä½¿ç”¨ vSphere 存储的集群，然åŽå•å‡»**æµè§ˆ**。 +1. 在左侧导航æ ä¸­ï¼Œé€‰æ‹©**存储 > StorageClass**。 +1. å•å‡»**创建**。 +3. 输入存储类的**å称**。 +4. 在 **Provisioner**下，选择 **VMWare vSphere å·**。 + + ![](/img/vsphere-storage-class.png) + +5. å¯é€‰åœ°ï¼Œä½ å¯ä»¥åœ¨**å‚æ•°**下指定存储类的其他属性。有关详细信æ¯ï¼Œè¯·å‚阅 [vSphere 存储文档](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/blob/master/documentation/storageclass.md)。 +5. å•å‡»**创建**。 + +### 创建使用 vSphere å·çš„工作负载 + +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载**。 +1. å•å‡»**创建**。 +1. å•å‡» **StatefulSet**。 +1. 在**å·å£°æ˜Žæ¨¡æ¿**选项å¡ä¸Šï¼Œå•å‡»**添加声明模æ¿**。 +1. 输入æŒä¹…å·å称。 +1. 在**存储类**字段中，选择你创建的 vSphere 存储类。 +6. 输入å·æ‰€éœ€çš„**容é‡**。然åŽå•å‡»**定义**。 +7. 在**挂载点**字段中指定路径。这是å·å°†å®‰è£…在容器文件系统中的完整路径，例如 `/persistent`。 +8. å•å‡»**创建**。 + +### 验è¯å·çš„æŒä¹…性 + +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**工作负载 > Pod**。 +1. 转到你刚刚创建的工作负载，然åŽå•å‡» **â‹® > 执行命令行**。 +2. 请注æ„å·å·²æŒ‚载到的根目录下的目录（在本例中为 `/persistent`）。 +3. 通过执行命令 `touch //data.txt` 在å·ä¸­åˆ›å»ºä¸€ä¸ªæ–‡ä»¶ã€‚ +4. 关闭 shell 窗å£ã€‚ +5. å•å‡»å·¥ä½œè´Ÿè½½çš„å称以显示详细信æ¯ã€‚ +7. å•å‡» **â‹® > 删除**。 +8. 观察 pod 是å¦è¢«åˆ é™¤ã€‚为了让工作负载维æŒåœ¨æ‰€é…置的å•ä¸ªæœ‰çŠ¶æ€ pod 的规模，之åŽä¼šè°ƒåº¦ä¸€ä¸ªæ–°çš„ Pod æ¥æ›¿æ¢è¯¥ Pod。 +9. 替æ¢çš„ pod è¿è¡ŒåŽï¼Œå•å‡»**执行命令行**。 +10. 输入 `ls -l /` 以检查å·æ‰€æŒ‚载的目录的内容。请注æ„，你之å‰åˆ›å»ºçš„文件ä»ç„¶å­˜åœ¨ã€‚ + + ![workload-persistent-data](/img/workload-persistent-data.png) + +### 为什么使用 StatefulSet 替代 Deployment + +对于消耗 vSphere 存储的工作负载，你应该始终使用 [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/)，因为这ç§èµ„æºç±»åž‹æ—¨åœ¨è§£å†³ VMDK å—存储警告。 + +由于 vSphere å·ç”± VMDK å—存储支æŒï¼Œå› æ­¤å®ƒä»¬ä»…æ”¯æŒ `ReadWriteOnce` çš„[访问模å¼](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)。此设置é™åˆ¶å·ï¼Œä½¿å…¶ä¸€æ¬¡åªèƒ½æŒ‚载到一个 pod（除éžä½¿ç”¨è¯¥å·çš„所有 pod ä½äºŽåŒä¸€èŠ‚点上）。如果 deployment 资æºæ¶ˆè€— vSphere å·ï¼Œåˆ™ deployment 资æºæ— æ³•ç”¨äºŽæ‰©å±•åˆ°è¶…出å•ä¸ªå‰¯æœ¬ã€‚ + +å³ä½¿ä½¿ç”¨ä»…具有å•ä¸ªå‰¯æœ¬çš„ deployment 资æºä¹Ÿå¯èƒ½åœ¨æ›´æ–° deployment 时出现死é”情况。如果更新的 pod 被调度到ä¸åŒçš„节点，由于 VMDK ä»ç„¶è¿žæŽ¥åˆ°å¦ä¸€ä¸ªèŠ‚点，因此 pod 将无法å¯åŠ¨ã€‚ + +### 相关链接 + +- [用于 Kubernetes çš„ vSphere 存储](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/tree/master/documentation) +- [Kubernetes æŒä¹…å·](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md new file mode 100644 index 00000000000..62550604077 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -0,0 +1,78 @@ +--- +title: è¯ä¹¦è½®æ¢ +--- + +:::caution + +è½®æ¢ Kubernetes è¯ä¹¦å¯èƒ½ä¼šå¯¼è‡´é›†ç¾¤åœ¨ç»„件é‡å¯æ—¶æš‚æ—¶ä¸å¯ç”¨ã€‚对于生产环境，建议在维护时段内执行此æ“作。 + +::: + +默认情况下，Kubernetes 集群需è¦è¯ä¹¦ï¼ŒRancher å¯åŠ¨çš„ Kubernetes 集群会自动为 Kubernetes 组件生æˆè¯ä¹¦ã€‚在è¯ä¹¦è¿‡æœŸä¹‹å‰æˆ–被泄露åŽè½®æ¢è¯ä¹¦éžå¸¸é‡è¦ã€‚è¯ä¹¦è½®æ¢åŽï¼ŒKubernetes 组件会自动é‡å¯ã€‚ + +å¯ä»¥ä¸ºä»¥ä¸‹æœåŠ¡è½®æ¢è¯ä¹¦ï¼š + + + + +- etcd +- kubelet（节点è¯ä¹¦ï¼‰ +- kubelet（æœåŠ¡è¯ä¹¦ï¼Œå¦‚æžœ[å¯ç”¨](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)） +- kube-apiserver +- kube-proxy +- kube-scheduler +- kube-controller-manager + + + + +- admin +- api-server +- controller-manager +- scheduler +- rke2-controller +- rke2-server +- cloud-controller +- etcd +- auth-proxy +- kubelet +- kube-proxy + + + + +:::note + +å¦‚æžœä½ æœªè½®æ¢ webhook è¯ä¹¦ï¼Œä¸”è¯ä¹¦ç”¨äº†ä¸€å¹´åŽå·²ç»è¿‡æœŸï¼Œè¯·å‚阅此[页é¢](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md)。 + +::: + +### è¯ä¹¦è½®æ¢ + +Rancher å¯åŠ¨çš„ Kubernetes 集群能够通过 UI è½®æ¢è‡ªåŠ¨ç”Ÿæˆçš„è¯ä¹¦ã€‚ + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦è½®æ¢è¯ä¹¦çš„集群，然åŽå•å‡» **â‹® > è½®æ¢è¯ä¹¦**。 +1. 选择è¦è½®æ¢çš„è¯ä¹¦ã€‚ + + * è½®æ¢æ‰€æœ‰æœåŠ¡è¯ä¹¦ï¼ˆä½¿ç”¨ç›¸åŒçš„ CA） + * è½®æ¢å•ä¸ªæœåŠ¡ï¼Œå¹¶ä»Žä¸‹æ‹‰èœå•ä¸­é€‰æ‹©å…¶ä¸­ä¸€é¡¹æœåŠ¡ã€‚ + +1. å•å‡»**ä¿å­˜**。 + +**结果**：将轮æ¢æ‰€é€‰è¯ä¹¦ï¼Œç›¸å…³æœåŠ¡å°†é‡æ–°å¯åŠ¨ä»¥ä½¿ç”¨æ–°è¯ä¹¦ã€‚ + +### 补充说明 + + + + +虽然 RKE CLI å¯ä»¥ä¸º Kubernetes 集群组件使用自定义è¯ä¹¦ï¼Œä½† Rancher ç›®å‰ä¸å…许在 Rancher å¯åŠ¨çš„ Kubernetes 集群中上传这些è¯ä¹¦ã€‚ + + + + +在 RKE2 中，etcd å’Œ controlplane 节点都被视为相åŒçš„ `server`。因此，如果你轮æ¢å…¶ä¸­ä¸€ä¸ªç»„件的æœåŠ¡è¯ä¹¦ï¼Œåˆ™ä¸¤è€…çš„è¯ä¹¦éƒ½ä¼šè¢«è½®æ¢ã€‚è¯ä¹¦åªä¼šé’ˆå¯¹æŒ‡å®šçš„æœåŠ¡æ›´æ”¹ï¼Œä½†ä½ ä¼šçœ‹åˆ°ä¸¤ä¸ªç»„件的节点都进入更新状æ€ã€‚ä½ å¯èƒ½è¿˜ä¼šçœ‹åˆ°ä»… Worker 节点进入更新状æ€ã€‚这是在è¯ä¹¦æ›´æ”¹åŽé‡å¯ Worker，以确ä¿ä»–们获得最新的客户端è¯ä¹¦ã€‚ + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md new file mode 100644 index 00000000000..cfebc6443aa --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -0,0 +1,48 @@ +--- +title: åŠ å¯†å¯†é’¥è½®æ¢ +--- + +### RKE1 åŠ å¯†å¯†é’¥è½®æ¢ + +1. 使用以下两个选项之一æ¥å¯ç”¨åŠ å¯†å¯†é’¥è½®æ¢ï¼š + + - 在 Rancher UI 中的 **Cluster Options > Advanced Options > Secrets Encryption** 下选择 `Enabled` å•é€‰æŒ‰é’®ï¼š + + ![å¯ç”¨åŠ å¯†å¯†é’¥è½®æ¢](/img/rke1-enable-secrets-encryption.png) + + - 或者，应用以下 YAML： + + ```yaml + rancher_kubernetes_engine_config: + services: + kube_api: + secrets_encryption_config: + enabled: true + ``` + +2. 在 Rancher UI 中轮æ¢å¯†é’¥ï¼š + + 2.1. 点击 **☰ > 集群管ç†**。 + + 2.2. 在所选集群æ—边的å±å¹•æœ€å³ä¾§é€‰æ‹© **â‹® > Rotate Encryption Keys**： + + ![加密密钥轮æ¢](/img/rke1-encryption-key.png) + + + +### RKE2 åŠ å¯†å¯†é’¥è½®æ¢ + +_**v2.6.7 新功能**_ + +> **é‡è¦æ示**：加密密钥轮æ¢é»˜è®¤å¯ç”¨ï¼Œä¸èƒ½ç¦ç”¨ã€‚ + +è¦åœ¨ Rancher UI 中轮æ¢å¯†é’¥ï¼š + +1. 点击 **☰ > 集群管ç†**。 + +1. 在所选集群æ—边的å±å¹•æœ€å³ä¾§é€‰æ‹© **â‹® > Rotate Encryption Keys**： + + ![加密密钥轮æ¢](/img/rke2-encryption-key.png) + + +> **注æ„**：有关 RKE2 密文加密é…置的更多信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/security/secrets_encryption)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-namespaces.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-namespaces.md new file mode 100644 index 00000000000..1c8aeba0767 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/how-to-guides/new-user-guides/manage-namespaces.md @@ -0,0 +1,74 @@ +--- +title: 命å空间 +--- + +在 Rancher 中，你å¯ä»¥å°†é¡¹ç›®è¿›ä¸€æ­¥åˆ’分为多个[命å空间](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/)，项目是由物ç†é›†ç¾¤æ”¯æŒçš„，而命å空间是项目中的虚拟集群。当项目和 `default` 命å空间ä¸èƒ½æ»¡è¶³ä½ çš„需求时，你å¯ä»¥åœ¨é¡¹ç›®ä¸­åˆ›å»ºå¤šä¸ªå‘½å空间æ¥éš”离应用和资æºã€‚ + +ä½ å¯ä»¥åœ¨é¡¹ç›®å±‚级分é…资æºï¼Œä»Žè€Œä½¿é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间å¯ä»¥ä½¿ç”¨è¿™äº›èµ„æºï¼Œä¹Ÿå¯ä»¥å•ç‹¬ç»™æŸä¸ªå‘½å空间显å¼åˆ†é…资æºã€‚ + +å¯ä»¥ç›´æŽ¥åˆ†é…给命å空间的资æºåŒ…括： + +- [工作负载](../../pages-for-subheaders/workloads-and-pods.md) +- [è´Ÿè½½å‡è¡¡å™¨/Ingress](../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [æœåŠ¡å‘现记录](kubernetes-resources-setup/create-services.md) +- [æŒä¹…å·å£°æ˜Ž](../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [è¯ä¹¦](kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMap](kubernetes-resources-setup/configmaps.md) +- [é•œåƒä»“库](kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [密文](kubernetes-resources-setup/secrets.md) + +为了在 vanilla Kubernetes 集群中管ç†æƒé™ï¼Œé›†ç¾¤ç®¡ç†å‘˜è¦ä¸ºæ¯ä¸ªå‘½å空间é…置基于角色的访问策略。Rancher 在项目级别上分é…用户æƒé™ï¼Œé¡¹ç›®ä¸­çš„命å空间会自动继承项目的æƒé™ã€‚ + +:::note + +如果你使用 `kubectl`创建命å空间，由于 `kubectl` ä¸è¦æ±‚将新命å空间é™å®šåœ¨ä½ å¯ä»¥è®¿é—®çš„项目内，因此你å¯èƒ½æ— æ³•ä½¿ç”¨è¯¥å‘½å空间。如果你的æƒé™ä»…é™äºŽé¡¹ç›®çº§åˆ«ï¼Œåˆ™æœ€å¥½[通过 Rancher 创建命å空间](manage-namespaces.md)，以确ä¿ä½ æœ‰æƒè®¿é—®è¯¥å‘½å空间。 + +::: + +### 创建命å空间 + +创建一个新的命å空间æ¥éš”离项目中的应用和资æºã€‚ + +:::tip + +使用å¯ä»¥åˆ†é…给命å空间的项目资æºï¼ˆä¾‹å¦‚，[工作负载](kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md)ã€[è¯ä¹¦](kubernetes-resources-setup/encrypt-http-communication.md)ã€[ConfigMap](kubernetes-resources-setup/configmaps.md) 等）时，你å¯ä»¥åŠ¨æ€åˆ›å»ºå‘½å空间。 + +::: + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦åœ¨å…¶ä¸­åˆ›å»ºå‘½å空间的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦æ·»åŠ å‘½å空间的项目，并å•å‡»**创建命å空间**。或者，你也å¯ä»¥è½¬åˆ°**ä¸åœ¨é¡¹ç›®å†…**以创建ä¸ä¸Žé¡¹ç›®å…³è”的命å空间。 + +1. **å¯é€‰**：如果你的项目具有有效的[资æºé…é¢](../../pages-for-subheaders/manage-project-resource-quotas.md)，你å¯ä»¥è¦†ç›–默认资æº**é™åˆ¶**（é™åˆ¶å‘½å空间å¯ä»¥ä½¿ç”¨çš„资æºï¼‰ã€‚ + +1. 输入**å称**，然åŽå•å‡»**创建**。 + +**结果**：已将命å空间添加到项目中。你å¯ä»¥å¼€å§‹å°†é›†ç¾¤èµ„æºåˆ†é…给命å空间。 + +### 将命å空间移动到å¦ä¸€ä¸ªé¡¹ç›® + +在æŸäº›æƒ…况下（例如希望其他团队使用该应用时），集群管ç†å‘˜å’Œæˆå‘˜å¯èƒ½éœ€è¦å°†å‘½å空间移动到å¦ä¸€ä¸ªé¡¹ç›®ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°éœ€è¦ç§»åŠ¨å‘½å空间的集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦ç§»åŠ¨çš„命å空间，然åŽå•å‡» **â‹® > 移动**。 + +1. 选择è¦ç§»åŠ¨åˆ°å¦ä¸€ä¸ªé¡¹ç›®çš„命å空间。然åŽå•å‡»**移动**。你å¯ä»¥åŒæ—¶ç§»åŠ¨å¤šä¸ªå‘½å空间。 + + :::note 注æ„事项： + + - ä¸è¦ç§»åŠ¨ `System` 项目中的命å空间。移动命å空间å¯èƒ½ä¼šå¯¹é›†ç¾¤ç½‘络产生ä¸åˆ©å½±å“。 + - ä½ ä¸èƒ½å°†å‘½å空间移动到已é…ç½®[资æºé…é¢](../../pages-for-subheaders/manage-project-resource-quotas.md)的项目中。 + - 如果你将命å空间从已设置é…é¢çš„项目移动到未设置é…é¢çš„项目，则会删除该命å空间的é…é¢ã€‚ + +1. 为新命å空间选择一个新项目，然åŽå•å‡»**移动**。你也å¯ä»¥é€‰æ‹©**æ— **，从而将命å空间从所有项目中移除。 + +**结果**：你的命å空间已移至其他项目（或从所有项目中移除）。如果命å空间绑定了项目资æºï¼Œå‘½å空间会释放这些资æºï¼Œç„¶åŽç»‘定新项目的资æºã€‚ + +### 编辑命å空间资æºé…é¢ + +ä½ å¯ä»¥è¦†ç›–命å空间默认é™åˆ¶ï¼Œä»Žè€Œä¸ºç‰¹å®šå‘½å空间æ供对更多（或更少）项目资æºçš„访问æƒé™ï¼š + +有关详细信æ¯ï¼Œè¯·å‚阅[编辑命å空间资æºé…é¢](../advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/configuration-reference.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/configuration-reference.md new file mode 100644 index 00000000000..7a530783b36 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/configuration-reference.md @@ -0,0 +1,105 @@ +--- +title: é…ç½® +--- + +æ­¤é…ç½®å‚考用于帮助你管ç†ç”± `rancher-cis-benchmark` 应用创建的自定义资æºã€‚这些资æºç”¨äºŽåœ¨é›†ç¾¤ä¸Šæ‰§è¡Œ CIS 扫æã€è·³è¿‡æµ‹è¯•ã€è®¾ç½®æ‰«æ使用的测试é…置文件和其他自定义é…置。 + +è¦é…置自定义资æºï¼Œè½¬åˆ°**集群**仪表æ¿ã€‚è¦é…ç½® CIS 扫æ： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦é…ç½® CIS 扫æ的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark**。 + +### 扫æ + +扫æ是用æ¥æ ¹æ®å®šä¹‰çš„é…ç½®æ–‡ä»¶ï¼Œåœ¨é›†ç¾¤ä¸Šè§¦å‘ CIS 扫æ的。扫æ完æˆåŽä¼šåˆ›å»ºä¸€ä»½æŠ¥å‘Šã€‚ + +é…置扫æ时，你需è¦å®šä¹‰ä¸Ž `scanProfileName` å‚数一起使用的扫æé…置文件的å称。 + +下é¢æ˜¯ä¸€ä¸ª ClusterScan 自定义资æºç¤ºä¾‹ï¼š + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScan +metadata: + name: rke-cis +spec: + scanProfileName: rke-profile-hardened +``` + +### é…置文件 + +é…ç½®æ–‡ä»¶åŒ…å« CIS 扫æçš„é…置，包括è¦ä½¿ç”¨çš„ Benchmark 测试版本以åŠè¦åœ¨è¯¥ Benchmark 测试中跳过的测试。 + +:::caution + +默认情况下，一些 ClusterScanProfile 会作为 `rancher-cis-benchmark` Chart 的一部分进行安装。如果用户编辑了这些默认 Benchmark 或é…置文件，它们会在下一次 Chart 更新时被é‡ç½®ã€‚因此，建议用户ä¸è¦ç¼–辑默认的 ClusterScanProfile。 + +::: + +用户å¯ä»¥é€šè¿‡å…‹éš† ClusterScanProfile æ¥åˆ›å»ºè‡ªå®šä¹‰é…置文件。 + +跳过的测试会列在 `skipTests` å‚数下。 + +创建新é…置文件时，你还需è¦å‘½åé…置文件。 + +`ClusterScanProfile` 示例如下： + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" +spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" +``` + +### Benchmark 版本 + +Benchmark 版本是指使用 `kube-bench` è¿è¡Œçš„ Benchmark å称，以åŠè¯¥ Benchmark 的有效é…ç½®å‚数。 + +`ClusterScanBenchmark` 定义了 CIS `BenchmarkVersion` çš„å称和测试é…置。`BenchmarkVersion` å称是æ供给 `kube-bench` 工具的å‚数。 + +默认情况下，一些 `BenchmarkVersion` å称和测试é…置会作为 CIS 扫æ应用的一部分进行打包。å¯ç”¨æ­¤åŠŸèƒ½åŽï¼Œè¿™äº›é»˜è®¤ BenchmarkVersion 将自动安装，用户å¯ä»¥ä½¿ç”¨å®ƒä»¬æ¥åˆ›å»º ClusterScanProfile。 + +:::caution + +如果用户编辑了默认的 BenchmarkVersion，它们会在下一次 Chart 更新时被é‡ç½®ã€‚因此，ä¸å»ºè®®ç¼–辑默认的 ClusterScanBenchmark。 + +::: + +ClusterScanBenchmark 由以下字段组æˆï¼š + +- `ClusterProvider`：此 Benchmark 适用的集群æ供商å称，例如，RKEã€EKSã€GKE。如果此基准测试å¯ä»¥åœ¨ä»»ä½•é›†ç¾¤ç±»åž‹ä¸Šè¿è¡Œï¼Œåˆ™ç•™ç©ºã€‚ +- `MinKubernetesVersion`：集群è¿è¡Œæ­¤ Benchmark 测试所需的最低 kubernetes 版本。如果ä¸ä¾èµ–特定的 Kubernetes 版本，则留空。 +- `MaxKubernetesVersion`：集群è¿è¡Œæ­¤ Benchmark 测试所需的最高 Kubernetes 版本。如果ä¸ä¾èµ–特定的 Kubernetes 版本，则留空。 + +`ClusterScanBenchmark` 示例如下： + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + creationTimestamp: "2020-08-28T18:18:07Z" + generation: 1 + labels: + app.kubernetes.io/managed-by: Helm + name: cis-1.5 + resourceVersion: "203878" + selfLink: /apis/cis.cattle.io/v1/clusterscanbenchmarks/cis-1.5 + uid: 309e543e-9102-4091-be91-08d7af7fb7a7 +spec: + clusterProvider: "" + minKubernetesVersion: 1.15.0 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/custom-benchmark.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/custom-benchmark.md new file mode 100644 index 00000000000..6818ac56118 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/custom-benchmark.md @@ -0,0 +1,78 @@ +--- +title: 为集群扫æ创建自定义 Benchmark 版本 +--- + +æ¯ä¸ª Benchmark 版本都定义了一组测试é…置文件，这些文件定义了由 kube-bench 工具è¿è¡Œçš„ CIS 测试。 +`rancher-cis-benchmark` 应用安装了一些默认的 Benchmark 测试版本，这些版本列在了 CIS Benchmark 测试应用èœå•ä¸‹ã€‚ + +但是，æŸäº› Kubernetes 集群å¯èƒ½éœ€è¦è‡ªå®šä¹‰é…ç½® Benchmark 测试。例如，Kubernetes é…置文件或è¯ä¹¦çš„路径å¯èƒ½ä¸Žä¸Šæ¸¸ CIS Benchmark 的标准ä½ç½®ä¸åŒã€‚ + +现在，你å¯ä»¥ä½¿ç”¨ `rancher-cis-benchmark` 应用æ¥åˆ›å»ºè‡ªå®šä¹‰ Benchmark 版本，从而è¿è¡Œé›†ç¾¤æ‰«æ。 + +è¿è¡Œé›†ç¾¤æ‰«æ时，你需è¦é€‰æ‹©æŒ‡å‘特定 Benchmark 版本的é…置文件。 + +按照以下所有步骤添加自定义 Benchmark 版本并使用它è¿è¡Œæ‰«æ。 + +### 1. 准备自定义 Benchmark 版本 ConfigMap + +è¦åˆ›å»ºè‡ªå®šä¹‰ Benchmark 版本，你需è¦å…ˆåˆ›å»ºä¸€ä¸ªåŒ…å« Benchmark 版本é…置文件的 ConfigMap，并将其上传到è¦è¿è¡Œæ‰«æçš„ Kubernetes 集群。 + +å‡è®¾è¦æ·»åŠ ä¸€ä¸ªå为 `foo` 的自定义 Benchmark 版本，你å¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤å‡†å¤‡è‡ªå®šä¹‰ Benchmark 版本 ConfigMap： + +1. 创建一个å为 `foo` 的目录，并在该目录中放置所有é…ç½® YAML 文件， kube-bench 工具会æœç´¢è¿™äº›æ–‡ä»¶ã€‚例如，通用 CIS 1.5 Benchmark 版本的é…ç½® YAML 文件在[此处](https://github.com/aquasecurity/kube-bench/tree/master/cfg/cis-1.5)。 +1. 放置完整的 `config.yaml` 文件，其中包括所有è¦æµ‹è¯•çš„组件。 +1. å°† Benchmark 版本å称添加到 `config.yaml` çš„ `target_mapping` 中： + + ```yaml + target_mapping: + "foo": + - "master" + - "node" + - "controlplane" + - "etcd" + - "policies" + ``` +1. 通过创建 ConfigMap 将此目录上传到 Kubernetes 集群： + + ```yaml + kubectl create configmap -n foo --from-file= + ``` + +### 2. 将自定义 Benchmark 版本添加到集群 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰ Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark > Benchmark 版本**。 +1. å•å‡»**创建**。 +1. 输入自定义 Benchmark 版本的**å称**å’Œæ述。 +1. 选择è¦åº”用 Benchmark 版本的集群æ供商。 +1. 在下拉列表中选择你已上传的 ConfigMap。 +1. 添加最低和最高 Kubernetes 版本é™åˆ¶ï¼ˆå¦‚果有）。 +1. å•å‡»**创建**。 + +### 3. 为自定义 Benchmark 版本创建新é…置文件 + +è¦ä½¿ç”¨ä½ çš„自定义 Benchmark 版本è¿è¡Œæ‰«æ，你需è¦æ·»åŠ ä¸€ä¸ªæŒ‡å‘æ­¤ Benchmark 版本的新é…置文件： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰ Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark > é…置文件**。 +1. å•å‡»**创建**。 +1. 设置**å称**å’Œæ述。在本例中，我们将其命å为 `foo-profile`。 +1. 在下拉列表中选择 Benchmark 版本。 +1. å•å‡»**创建**。 + +### 4. 使用自定义 Benchmark 版本è¿è¡Œæ‰«æ + +指å‘你的自定义 Benchmark 版本的 `foo` é…置文件创建完æˆåŽï¼Œä½ å¯ä»¥åˆ›å»ºä¸€ä¸ªæ–°çš„扫æ，从而在 Benchmark 版本中è¿è¡Œè‡ªå®šä¹‰æµ‹è¯•ã€‚ + +è¦è¿è¡Œæ‰«æ： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æ·»åŠ è‡ªå®šä¹‰ Benchmark 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **CIS Benchmark > 扫æ**。 +1. å•å‡»**创建**。 +1. 选择新的集群扫æé…置文件。 +1. å•å‡»**创建**。 + +**结果**：已生æˆå¸¦æœ‰æ‰«æ结果的报告。如需查看结果，请å•å‡»æ˜¾ç¤ºçš„扫æå称。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md new file mode 100644 index 00000000000..6398f217c7e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md @@ -0,0 +1,46 @@ +--- +title: RBAC +--- + +本文介ç»ä½¿ç”¨ rancher-cis-benchmark 应用所需的æƒé™ã€‚ + +默认情况下，rancher-cis-benchmark 是集群管ç†å‘˜ç‹¬æœ‰çš„功能。 + +但是，`rancher-cis-benchmark` Chart 安装了这两个默认的 `ClusterRole`： + +- cis-admin +- cis-view + +在 Rancher 中，默认情况下åªæœ‰é›†ç¾¤æ‰€æœ‰è€…和全局管ç†å‘˜å…·æœ‰ `cis-admin` 访问æƒé™ã€‚ + +注æ„：如果你使用在 Rancher 2.5 中添加的 `cis-edit` 角色，该角色从 2.5.2 开始已被删除，删除的原因是该角色本质上与 `cis-admin` 相åŒã€‚如果你为 `cis-edit` 创建了任何 clusterrolebinding，请更新这些绑定以使用 `cis-admin` ClusterRole。 + +## Cluster-Admin 访问 + +默认情况下，Rancher CIS 扫æ是集群管ç†å‘˜ç‹¬æœ‰çš„功能。 +æ¢è¨€ä¹‹ï¼Œåªæœ‰ Rancher 全局管ç†å‘˜å’Œé›†ç¾¤çš„集群所有者å¯ä»¥ï¼š + +- 安装/å¸è½½ rancher-cis-benchmark 应用。 +- 查看 CIS Benchmark CRD 的导航链接 - ClusterScanBenchmarksã€ClusterScanProfilesã€ClusterScans。 +- 列出默认的 ClusterScanBenchmarks å’Œ ClusterScanProfiles。 +- 创建/编辑/删除新的 ClusterScanProfiles。 +- 创建/编辑/删除新的 ClusterScan，从而在集群上è¿è¡Œ CIS 扫æ。 +- 在 ClusterScan 完æˆåŽæŸ¥çœ‹å¹¶ä¸‹è½½ ClusterScanReport。 + + +## Kubernetes 默认角色的默认æƒé™æ‘˜è¦ + +rancher-cis-benchmark 创建了三个 `ClusterRole`，并将 CIS Benchmark CRD 访问æƒé™æ·»åŠ åˆ°ä»¥ä¸‹é»˜è®¤ K8s `ClusterRole`： + +| Chart 创建的 ClusterRole | 默认 K8s ClusterRole | 角色赋予的æƒé™ | +| ------------------------------| ---------------------------| ---------------------------| +| `cis-admin` | `admin` | 增刪查改（CRUD）clusterscanbenchmarksã€clusterscanprofilesã€clusterscansã€clusterscanreports CR | +| `cis-view` | `view ` | 列出 (R) clusterscanbenchmarksã€clusterscanprofilesã€clusterscansã€clusterscanreports CR | + + +默认情况下，åªæœ‰ cluster-owner 角色能管ç†å’Œä½¿ç”¨ `rancher-cis-benchmark` 功能。 + +默认情况下，其他 Rancher 角色（cluster-memberã€project-owner å’Œ project-member）没有管ç†å’Œä½¿ç”¨ rancher-cis-benchmark 资æºçš„æƒé™ã€‚ + +但是，如果 cluster-owner 想将访问æƒé™åˆ†é…给其他用户，他们å¯ä»¥æ‰‹åŠ¨åˆ›å»ºç›®æ ‡ç”¨æˆ·ä¸Žä¸Šè¿° CIS ClusterRole 之间的 ClusterRoleBinding，从而实现æƒé™åˆ†é…。 +`rancher-cis-benchmark` ClusterRole ä¸æ”¯æŒè‡ªåŠ¨è§’色èšåˆã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md new file mode 100644 index 00000000000..a7afe788a87 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md @@ -0,0 +1,53 @@ +--- +title: 跳过和ä¸é€‚用的测试 +--- + +本文列出了在 RKE çš„ permissive 测试é…置文件中跳过的测试。 + +> 在 v2.5 生æˆçš„报告中，此页é¢ä¸Šè¢«è·³è¿‡ä¸”ä¸é€‚用的测试将会计为ä¸é€‚用。跳过的测试计数åªä¼šæ¶‰åŠç”¨æˆ·å®šä¹‰çš„跳过测试。这样，你å¯ä»¥åŒºåˆ†ç”¨æˆ·è¦è·³è¿‡çš„测试与 RKE permissive 测试é…置文件中默认跳过的测试。 + +## CIS Benchmark v1.5 + +### CIS Benchmark v1.5 跳过的测试 + +| æ•°å­— | æè¿° | 跳过的原因 | +| ---------- | ------------- | --------- | +| 1.1.12 | ç¡®ä¿ etcd æ•°æ®ç›®å½•æ‰€æœ‰æƒè®¾ç½®ä¸º etcd:etcd（自动） | etcd æ•°æ®ç›®å½•æ‰€æœ‰æƒéœ€è¦ç³»ç»Ÿ ServiceAccount。有关如何é…置所有æƒçš„更多信æ¯ï¼Œè¯·å‚阅 Rancher 的强化指å—。 | +| 1.2.6 | ç¡®ä¿æ ¹æ®éœ€è¦è®¾ç½® --kubelet-certificate-authority å‚数（自动） | 在生æˆæœåŠ¡è¯ä¹¦æ—¶ï¼ŒåŠŸèƒ½å¯èƒ½ä¼šä¸ŽæŸäº›äº‘æ供商所需的主机å覆盖一起中断。 | +| 1.2.16 | ç¡®ä¿è®¾ç½®äº†å‡†å…¥æŽ§åˆ¶æ’件 PodSecurityPolicy（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 1.2.33 | ç¡®ä¿èƒ½æ ¹æ®éœ€è¦è®¾ç½® --encryption-provider-config å‚数（手动） | å¯ç”¨åŠ å¯†ä¼šæ”¹å˜æ¢å¤åŠ å¯†æ•°æ®çš„æ–¹å¼ã€‚ | +| 1.2.34 | ç¡®ä¿æ­£ç¡®é…置了加密æ供程åºï¼ˆæ‰‹åŠ¨ï¼‰ | å¯ç”¨åŠ å¯†ä¼šæ”¹å˜æ¢å¤åŠ å¯†æ•°æ®çš„æ–¹å¼ã€‚ | +| 4.2.6 | ç¡®ä¿ --protect-kernel-defaults å‚数设置为 true（自动） | 在é…置集群之å‰éœ€è¦ç³»ç»Ÿçº§åˆ«çš„é…置，æ‰èƒ½å°†æ­¤å‚数设置为 true。 | +| 4.2.10 | ç¡®ä¿æ ¹æ®éœ€è¦è®¾ç½® --tls-cert-file å’Œ --tls-private-key-file å‚数（自动） | 在生æˆæœåŠ¡è¯ä¹¦æ—¶ï¼ŒåŠŸèƒ½å¯èƒ½ä¼šä¸ŽæŸäº›äº‘æ供商所需的主机å覆盖一起中断。 | +| 5.1.5 | ç¡®ä¿æœªä¸»åŠ¨ä½¿ç”¨é»˜è®¤ ServiceAccount。（自动） | Kubernetes æ供了è¦ä½¿ç”¨çš„默认 ServiceAccount。 | +| 5.2.2 | 最å°åŒ–需è¦å…±äº«ä¸»æœºè¿›ç¨‹ ID 命å空间的容器准入（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.2.3 | 最å°åŒ–需è¦å…±äº«ä¸»æœº IPC 命å空间的容器准入（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.2.4 | 最å°åŒ–需è¦å…±äº«ä¸»æœºç½‘络命å空间的容器准入（自动） | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.2.5 | 使用 allowPrivilegeEscalation（自动）最å°åŒ–容器的准入 | å¯ç”¨ Pod 安全策略å¯èƒ½ä¼šå¯¼è‡´åº”用æ„外失败。 | +| 5.3.2 | ç¡®ä¿æ‰€æœ‰å‘½å空间都定义了网络策略（自动） | å¯ç”¨ç½‘络策略å¯ä»¥é˜²æ­¢æŸäº›åº”用进行相互通信。 | +| 5.6.4 | ç¡®ä¿ä¸ä½¿ç”¨ Default 命å空间（自动） | Kubernetes æ供了一个 Default 命å空间。 | + +### CIS Benchmark v1.5 ä¸é€‚用的测试 + +| æ•°å­— | æè¿° | ä¸é€‚用的原因 | +| ---------- | ------------- | --------- | +| 1.1.1 | ç¡®ä¿ API Server pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kube-apiserver çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.2 | ç¡®ä¿ API Server pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kube-apiserver çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.3 | ç¡®ä¿ Controller Manager pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.4 | ç¡®ä¿ Controller Manager pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.5 | ç¡®ä¿ Scheduler pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.6 | ç¡®ä¿ Scheduler pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.7 | ç¡®ä¿ etcd pod 规范文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 etcd çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.8 | ç¡®ä¿ etcd pod 规范文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 etcd çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.13 | ç¡®ä¿ admin.conf 文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸ä¼šåœ¨èŠ‚点上存储 kubernetes 的默认 kubeconfig 凭è¯æ–‡ä»¶ã€‚ | +| 1.1.14 | ç¡®ä¿ admin.conf 文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸ä¼šåœ¨èŠ‚点上存储 kubernetes 的默认 kubeconfig 凭è¯æ–‡ä»¶ã€‚ | +| 1.1.15 | ç¡®ä¿ scheduler.conf 文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.16 | ç¡®ä¿ scheduler.conf 文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 Scheduler çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.17 | ç¡®ä¿ controller-manager.conf 文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.1.18 | ç¡®ä¿å°† controller-manager.conf 文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 controller-manager çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 1.3.6 | ç¡®ä¿ RotateKubeletServerCertificate å‚数设置为 true（自动） | RKE é…置的集群直接使用 RKE 处ç†è¯ä¹¦è½®æ¢ã€‚ | +| 4.1.1 | ç¡®ä¿ kubelet æœåŠ¡æ–‡ä»¶æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet æœåŠ¡çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.1.2 | ç¡®ä¿ kubelet æœåŠ¡æ–‡ä»¶æ‰€æœ‰æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet æœåŠ¡çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.1.9 | ç¡®ä¿ kubelet é…置文件æƒé™å…·æœ‰ 644 或更严格的设置（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.1.10 | ç¡®ä¿ kubelet é…置文件所有æƒè®¾ç½®ä¸º root:root（自动） | RKE é…置的集群ä¸éœ€è¦æˆ–维护 kubelet çš„é…置文件。所有é…置在容器è¿è¡Œæ—¶ä½œä¸ºå‚数传入。 | +| 4.2.12 | ç¡®ä¿ RotateKubeletServerCertificate å‚数设置为 true（自动） | RKE é…置的集群直接使用 RKE 处ç†è¯ä¹¦è½®æ¢ã€‚ | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md new file mode 100644 index 00000000000..8431d4fbd66 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md @@ -0,0 +1,96 @@ +--- +title: 先决æ¡ä»¶ +--- + +### 1. 设置许å¯è¯ç®¡ç†å™¨å’Œè´­ä¹°æ”¯æŒ + +首先，完æˆè®¸å¯è¯ç®¡ç†å™¨è®¾ç½®çš„[第一步](https://docs.aws.amazon.com/license-manager/latest/userguide/getting-started.html)。 +然åŽï¼Œè½¬åˆ° AWS Marketplace。找到 “Rancher Premium Support Billing Container Starter Packâ€ã€‚最åŽï¼Œè´­ä¹°è‡³å°‘一项 Entitlement。 + +如果你已使用 “Rancher Setup†AWS Marketplace 产å“安装了 Rancher，请跳至[步骤 4](#4-创建-oidc-æ供程åº)。 + +> **注æ„**：æ¯é¡¹ Entitlement 都对一定数é‡çš„节点授予访问支æŒçš„æƒé™ã€‚ä½ å¯ä»¥åŽç»­æ ¹æ®éœ€è¦è´­ä¹°æ›´å¤šè®¸å¯è¯ã€‚ + +### 2. 创建 EKS 集群 +按照 [Rancher 文档](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md)创建 EKS 集群。进行到[安装 Rancher Helm Chart](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-安装-rancher-helm-chart)（最åŽä¸€æ­¥ï¼‰æ—¶ï¼Œ**åœæ­¢å¹¶è¿”回此页é¢**。该集群需è¦æ»¡è¶³ä»¥ä¸‹è¦æ±‚： + +- EKS 1.22 版本。 +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—®åŒ…å« Rancher åŠå…¶ç›¸å…³é•œåƒçš„é•œåƒä»“库。 +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—®å­˜å‚¨ CSP Adapter çš„ ECR 仓库。 +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—®è®¸å¯è¯ç®¡ç†å™¨æœåŠ¡ã€‚ +- 集群中的æ¯ä¸ªèŠ‚点都å¯ä»¥è®¿é—® STS æœåŠ¡çš„全局端点。 + +### 3. 安装 Rancher + +除了在 [Rancher 文档](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-安装-rancher-helm-chart)中指定的 Rancher 安装选项外，你还需è¦å¯ç”¨å…¶å®ƒæŒ‡æ ‡ã€‚ +ä½ å¯ä»¥é€šè¿‡ Helm CLI 使用以下选项æ¥å®Œæˆï¼š + +```bash +--set extraEnv\[0\].name="CATTLE_PROMETHEUS_METRICS" --set-string extraEnv\[0\].value=true +``` + +你还å¯ä»¥ä½¿ç”¨ values.yaml，如下所示： + +```yaml +extraEnv: + - name: "CATTLE_PROMETHEUS_METRICS" + value: "true" +``` + +你还需è¦å®‰è£… Rancher 2.6.7 或更高版本。 + +### 4. 创建 OIDC æä¾›ç¨‹åº + +按照 [AWS 文档](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html)为上一节中指定的集群创建 OIDC æ供程åºã€‚ + +### 5. 创建 IAM 角色 + +CSP Adapter éœ€è¦ IAM 角色æ‰èƒ½ç­¾å…¥/签出 Entitlement。 + +首先，é…置如下的信任策略。你需è¦å°† `MY_AWS_ACC` 替æ¢ä¸ºä½ çš„ AWS å¸å·ï¼Œå°† `MY_AWS_REGION` 替æ¢ä¸ºä½ çš„ AWS 区域，并将 `MY_OIDC_PROVIDER` 替æ¢ä¸ºä½ çš„ OIDC æ供商 ID： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${MY_AWS_ACC}:oidc-provider/oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:sub": "system:serviceaccount:cattle-csp-adapter-system:rancher-csp-adapter", + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:aud": "sts.amazonaws.com" + } + } + } + ] +} +``` + +接下æ¥ï¼Œä¸ºå…·æœ‰ä»¥ä¸‹æƒé™çš„角色使用策略： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "RancherCSPAdapterPermissions", + "Effect": "Allow", + "Action": [ + "license-manager:ListReceivedLicenses", + "license-manager:CheckoutLicense", + "license-manager:ExtendLicenseConsumption", + "license-manager:CheckInLicense", + "license-manager:GetLicense", + "license-manager:GetLicenseUsage" + ], + "Resource": "*" + } + ] +} +``` + +ä¿å­˜è§’色的å称。ç¨åŽå®‰è£… CSP Adapter 时将需è¦å®ƒã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md new file mode 100644 index 00000000000..0e9f63f4086 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md @@ -0,0 +1,28 @@ +--- +title: 常è§é—®é¢˜ +--- + +**安装 Adapter åŽï¼ŒRancher 中出现一æ¡æ¨ªå¹…消æ¯ï¼Œä¸Šé¢å†™ç€ "AWS Marketplace Adapter: Unable to run the adapter, please check the adapter logs"** + +此错误表示 Adapter 安装到集群中时å‘生了一个错误，导致它无法正确签入/签出许å¯è¯ã€‚ + +这通常是因为 IAM 角色设置ä¸æ­£ç¡®ã€‚请查看[先决æ¡ä»¶](./adapter-requirements.md)并确认： + +- 已创建一个 OIDC æ供程åºï¼Œå¹¶ä¸”它已关è”到è¿è¡Œ Rancher 的集群。 +- IAM 角色已é…置为信任此 OIDC æ供程åºã€‚ +- IAM 角色至少具有策略中概述的æƒé™ã€‚ + +如果上述所有é…ç½®å‡å·²æ­£ç¡®é…置，请è”系支æŒå¯»æ±‚帮助。 + +**我看到一æ¡æ¨ªå¹…消æ¯ï¼Œä¸Šé¢å†™ç€ "AWS Marketplace Adapter: You have exceeded your licensed node count. At least x more license(s) are required in AWS to become compliant"** + +此消æ¯è¡¨æ˜Žä½ æ²¡æœ‰è¶³å¤Ÿçš„ Entitlement æ¥æ»¡è¶³ Rancher 当å‰ç®¡ç†çš„节点数é‡ã€‚ + +请记ä½ä»¥ä¸‹é™åˆ¶ï¼š + +- æ¯ä¸ª Entitlement 仅对一定数é‡çš„节点有效。 +- 当å‰ç”± Rancher 管ç†çš„æ¯ä¸ªèŠ‚点都计入你的总使用é‡ï¼ˆå®‰è£…了集群 Rancher 的节点除外）。 +- æ¯ä¸ª Entitlement 最多å¯ä»¥è¢«ä¸€ä¸ª Rancher 实例使用。例如，如果你的账户中有两个正在è¿è¡Œçš„ Rancher 实例（æ¯ä¸ªéƒ½å®‰è£…在å•ç‹¬çš„ EKS 集群上），那么你至少需è¦ä¸¤ä¸ª Entitlement。 + +你最近å¯èƒ½è¿˜å¸è½½/é‡æ–°å®‰è£…了 Adapter。如果 Adapter 丢失了它当å‰ç®¡ç†çš„许å¯è¯çš„跟踪，则å¯èƒ½éœ€è¦ä¸€ä¸ªå°æ—¶æ¥è§£æžè®¸å¯è¯çš„实际状æ€ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md new file mode 100644 index 00000000000..08bae7cbe2b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -0,0 +1,155 @@ +--- +title: 安装 Adapter +--- + +> **é‡è¦æ示**：如果你å°è¯•é‡æ–°å®‰è£… Adapter，你å¯èƒ½ä¼šåœ¨é•¿è¾¾ä¸€å°æ—¶çš„时间内收到ä¸åˆè§„的错误消æ¯ã€‚ + +### Rancher 与 Adapter 的兼容性矩阵 + +:::note é‡è¦æ示： + +ä¸åŒç‰ˆæœ¬çš„ CSP Adapter ä¾èµ–于特定 Rancher 版本的功能。 +为了æˆåŠŸéƒ¨ç½²å’Œè¿è¡Œ Adapter，你需è¦ç¡®ä¿ Adapter 版本与必è¦çš„ Rancher 版本对应。 + +::: + +| Rancher 版本 | Adapter 版本 | +|-----------------|:---------------:| +| v2.7.0 | v2.0.0 | +| v2.7.1 | v2.0.0 | +| v2.7.2 | v2.0.1 | +| v2.7.3 | v2.0.1 | +| v2.7.4 | v2.0.1 | +| v2.7.5 | v2.0.2 | + + +### 1. 获å–对 Local 集群的访问æƒé™ + +> **注æ„**：åªæœ‰ç®¡ç†å‘˜ç”¨æˆ·æ‰èƒ½è®¿é—® Local 集群。因为 CSP Adapter 必须安装在 Local 集群中，所以此安装必须由管ç†å‘˜ç”¨æˆ·æ‰§è¡Œã€‚ + +首先，å•å‡» Local 集群并下载 kubeconfig 令牌。然åŽï¼Œä½¿ç”¨ä¸‹æ–¹çš„命令é…ç½® CLI 以使用此新令牌，你需è¦å°† `$TOKEN_PATH` 替æ¢ä¸ºæ–‡ä»¶ç³»ç»Ÿä¸Šä»¤ç‰Œçš„下载路径： + +```bash +export KUBECONFIG=$TOKEN_PATH +``` + +### 2. 创建 Adapter 命å空间 + +创建è¦å®‰è£… Adapter 的命å空间： + +```bash +kubectl create ns cattle-csp-adapter-system +``` + +### 3. 创建è¯ä¹¦å¯†æ–‡ + +Adapter 需è¦è®¿é—® Rancher 用æ¥ä¸Ž Rancher Server 通信的根 CA。有关 Rancher 支æŒçš„è¯ä¹¦é€‰é¡¹çš„更多信æ¯ï¼Œè¯·å‚阅 [Chart 选项页é¢](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)。 + +如果你的 Rancher 使用由公认的è¯ä¹¦é¢å‘机构（例如 Let's Encrypt）签å‘çš„è¯ä¹¦ï¼Œä½ å¯ä»¥è·³åˆ°[步骤 4](#4-安装-chart)。 + +但是，如果你的 Rancher 使用了自定义è¯ä¹¦ï¼ˆä¾‹å¦‚ Rancher 生æˆçš„è¯ä¹¦æˆ–ç”±ç§æœ‰è¯ä¹¦é¢å‘机构签å‘çš„è¯ä¹¦ï¼‰ï¼Œä½ éœ€è¦ä¸ºè¯¥ CA æä¾› PEM ç¼–ç æ ¼å¼çš„è¯ä¹¦ä»¥ä¾¿ Adapter å¯ä»¥ä¸Ž Rancher 通信。 + +首先，检索 Rancher 正在使用的è¯ä¹¦å¹¶å°†å…¶æ”¾å…¥å为 `ca-additional.pem` 的文件中。如果你使用 Rancher 生æˆçš„è¯ä¹¦ï¼Œä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å®Œæˆæ­¤æ“作： + +```bash +kubectl get secret tls-rancher -n cattle-system -o jsonpath="{.data.tls\.crt}" | base64 -d >> ca-additional.pem +``` + +然åŽï¼Œåˆ›å»ºä¸€ä¸ªä½¿ç”¨æ­¤è¯ä¹¦çš„密文： + +```bash +kubectl -n cattle-csp-adapter-system create secret generic tls-ca-additional --from-file=ca-additional.pem +``` + +> **é‡è¦æ示**：ä¸è¦æ›´æ”¹æ–‡ä»¶å或创建的密文的å称，å¦åˆ™å¯èƒ½ä¼šå¯¼è‡´ Adapter è¿è¡Œå‡ºé”™ã€‚ + +### 4. 安装 Chart + +首先，使用以下命令添加 `rancher/charts` 仓库： + +```bash +helm repo add rancher-charts https://charts.rancher.io +``` + +接下æ¥ï¼Œå®‰è£… CSP Adapter。你必须指定多个值，其中包括账å·å·ç ä»¥åŠåœ¨å…ˆå†³æ¡ä»¶ä¸­åˆ›å»ºçš„角色的å称。 + +ç¡®ä¿ä½ çš„ CSP Adapter 版本与你正在è¿è¡Œçš„ Rancher 版本匹é…，如[上文](#rancher-与-adapter-的兼容性矩阵)所述。 + +在下方的æ“作中，将 `$MY_ACC_NUM` 替æ¢ä¸ºä½ çš„ AWS è´¦å·ï¼Œå°† `$MY_ROLE_NAME` 替æ¢ä¸ºå…ˆå†³æ¡ä»¶ä¸­åˆ›å»ºçš„角色的å称。此外，将 `$CSP_ADAPTER_VERSION` 替æ¢ä¸ºä¸Ž[版本矩阵](#rancher-与-adapter-的兼容性矩阵)中的 Rancher 版本匹é…的版本。 + +> **注æ„**：如果你使用 shell å˜é‡ï¼Œè¯·ä¸è¦ä½¿ç”¨å¼•å·ã€‚例如，MY_ACC_NUM=123456789012 å¯ç”¨ï¼Œä½† MY_ACC_NUM="123456789012" 将失败。 + +> **注æ„**：使用欧盟和英国的 AWS Marketplace 列表的账å·éœ€è¦é¢å¤–指定 `--set image.repository=rancher/rancher-csp-adapter-eu` 选项。è¦æŸ¥çœ‹ä½ çš„è´¦å·åœ¨å®‰è£… Adapter 时是å¦éœ€è¦æ­¤é€‰é¡¹ï¼Œè¯·å‚阅 Marketplace 列表的使用说明。 + + + + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --version $CSP_ADAPTER_VERSION +``` + + +你也å¯ä»¥ä½¿ç”¨ `values.yaml` 并指定以下选项： + +```yaml +aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME +``` + +> **注æ„**：账å·éœ€è¦åƒä¸Šé¢é‚£æ ·ä»¥å­—符串格å¼æŒ‡å®šï¼Œå¦åˆ™å®‰è£…会失败。 + +然åŽï¼Œä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å®‰è£… Adapter： + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION +``` + + + + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --set additionalTrustedCAs=true --version $CSP_ADAPTER_VERSION +``` + +你也å¯ä»¥ä½¿ç”¨ `values.yaml` 并指定以下选项： + +```yaml +aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME +additionalTrustedCAs: true +``` + +> **注æ„**：账å·éœ€è¦åƒä¸Šé¢é‚£æ ·ä»¥å­—符串格å¼æŒ‡å®šï¼Œå¦åˆ™å®‰è£…会失败。 + +然åŽï¼Œä½¿ç”¨ä»¥ä¸‹å‘½ä»¤å®‰è£… Adapter： + +```bash +helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION +``` + + + + +### 5. 管ç†è¯ä¹¦æ›´æ–° + +如果你在[步骤 3](#3-创建è¯ä¹¦å¯†æ–‡) 中创建了一个用于存储自定义è¯ä¹¦çš„密文，则éšç€è¯ä¹¦çš„è½®æ¢ï¼Œä½ å°†éœ€è¦æ›´æ–°æ­¤å¯†æ–‡ã€‚ + +首先，使用以下命令删除 cattle-csp-adapter-system 命å空间中的原始密文： + +```bash +kubectl delete secret tls-ca-additional -n cattle-csp-adapter-system +``` + +然åŽï¼ŒæŒ‰ç…§[步骤 3](#3-创建è¯ä¹¦å¯†æ–‡) 中的安装步骤，将密文的内容替æ¢ä¸ºæ›´æ–°åŽçš„值。 + +最åŽï¼Œé‡æ–°å¯åŠ¨ rancher-csp-adapter deployment æ¥ç¡®ä¿æ›´æ–°åŽçš„值å¯ä¾› Adapter 使用： + +```bash +kubectl rollout restart deploy rancher-csp-adapter -n cattle-csp-adapter-system +``` + +> **注æ„**：有一些方法（例如 cert-manager çš„ [trust operator](https://cert-manager.io/docs/projects/trust/)）å¯ä»¥å¸®åŠ©ä½ å‡å°‘手动轮æ¢ä»»åŠ¡çš„æ•°é‡ã€‚这些选项ä¸å—官方支æŒï¼Œä½†å¯èƒ½å¯¹æƒ³è¦è‡ªåŠ¨åŒ–æŸäº›ä»»åŠ¡çš„用户有用。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md new file mode 100644 index 00000000000..f290d881ae9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md @@ -0,0 +1,21 @@ +--- +title: å¸è½½ Adapter +--- + +### 1. 使用 Helm å¸è½½ Adapter Chart： + +```bash +helm uninstall rancher-csp-adapter -n cattle-csp-adapter-system +``` + +### 2. 删除为 Adapter 创建的命å空间： + +```bash +kubectl delete ns cattle-csp-adapter-system +``` + +### 3. （å¯é€‰ï¼‰åˆ é™¤æœªå®Œæˆçš„用户通知： + +```bash +kubectl delete RancherUserNotification csp-compliance +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/supportconfig.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/supportconfig.md new file mode 100644 index 00000000000..f5f0d722487 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/cloud-marketplace/supportconfig.md @@ -0,0 +1,53 @@ +--- +title: Supportconfig Bundle +--- + +安装 CSP Adapter åŽï¼Œä½ å°†èƒ½å¤Ÿç”Ÿæˆä¸€ä¸ª Supportconfig Bundle。此 Bundle 是一个 tar 包，å¯ç”¨äºŽå¿«é€Ÿæ供支æŒä¿¡æ¯ã€‚ + +ä½ å¯ä»¥é€šè¿‡ Rancher 或通过直接访问安装 Rancher 的集群æ¥åˆ›å»ºè¿™äº› Bundle。请注æ„，建议通过 Rancher 访问。 + +> **注æ„**：无论采用何ç§æ–¹æ³•ï¼Œåªæœ‰ç®¡ç†å‘˜å¯ä»¥ç”Ÿæˆ/下载 Supportconfig Bundle。 + +### 通过 Rancher 访问 + +首先，点击汉堡èœå•ã€‚然åŽå•å‡» `Get Support` 按钮。 + +![Get Support](/img/support-help.png) + +在下一页中，å•å‡» `Generate Support Config` 按钮。 + +> **注æ„**：如果未安装 Adapter ，则ä¸ä¼šå‡ºçŽ°ç”Ÿæˆ Supportconfig Bundle 的选项。你必须安装 CSP Adapter æ‰èƒ½ç”Ÿæˆ Supportconfig Bundle。 + +![Get Support](/img/generate-support-config.png) + +### ä¸é€šè¿‡ Rancher 进行访问 + +首先，为安装 Rancher çš„é›†ç¾¤ç”Ÿæˆ kubeconfig。 + +> **注æ„**：如果 Rancher 宕机，你将无法使用 Rancher 生æˆçš„ kubeconfig 令牌访问集群。 + +é…置你的 shell 环境以使用此 kubeconfig 令牌： + +```bash +export KUBECONFIG=$MY_KUBECONFIG_PATH +``` + +建议在è¿è¡Œæ­¤å‘½ä»¤æ—¶åˆ›å»ºä¸€ä¸ªä¸´æ—¶å·¥ä½œç›®å½•ï¼Œå¦‚下所示： + +```bash +mkdir temp && cd temp +``` + +然åŽï¼Œæ£€ç´¢ Supportconfig Bundle： + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && tar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` + +这将在你的当å‰ç›®å½•ä¸­åˆ›å»ºä¸€ä¸ª `supportconfig_rancher.tar` 文件。 + +由于 gnu-tar å’Œ bsd-tar ä¸å…¼å®¹ï¼Œåœ¨ Mac 上è¿è¡Œè¿™äº›å‘½ä»¤çš„用户å¯èƒ½ä¼šé‡åˆ°é—®é¢˜ã€‚如果支æŒéƒ¨é—¨åœ¨è¯»å–你制作的 Supportconfig 时出现问题，你å¯ä»¥å…ˆå°è¯•åœ¨ä½ çš„路径上将 gnu-tar 作为 `gtar` 进行访问，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && gtar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/architecture.md new file mode 100644 index 00000000000..e039b353a31 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/architecture.md @@ -0,0 +1,8 @@ +--- +title: 架构 +--- + +Fleet å¯ä»¥ç®¡ç†æ¥è‡ª Git 的原始 Kubernetes YAMLã€Helm Chartã€Kustomize 或三者的任何组åˆçš„部署。无论æ¥æºå¦‚何，所有资æºéƒ½ä¼šåŠ¨æ€è½¬åŒ–为 Helm Chart，Helm 会用作引擎æ¥å°†æ‰€æœ‰èµ„æºéƒ¨ç½²åˆ°é›†ç¾¤ä¸­ã€‚这给了你高度的控制ã€ä¸€è‡´æ€§å’Œå¯å®¡è®¡æ€§ã€‚Fleet ä¸ä»…关注扩展能力，而且还æ供高度的控制和å¯è§æ€§ï¼Œä»Žè€Œè®©ç”¨æˆ·å‡†ç¡®äº†è§£é›†ç¾¤ä¸Šå®‰è£…的内容。 + +![架构](/img/fleet-architecture.svg) + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md new file mode 100644 index 00000000000..37ba154ae39 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md @@ -0,0 +1,54 @@ +--- +title: 在代ç†åŽä½¿ç”¨ Fleet +--- + +在本节中，你将学习如何在一个设置中å¯ç”¨ Fleet，该设置有一个具有公共 IP çš„ Rancher Server，以åŠä¸€ä¸ªæ²¡æœ‰å…¬å…± IP 但é…置为使用代ç†çš„ Kubernetes 集群。 + +Rancher ä¸ä¼šä¸Žå·²æ³¨å†Œçš„下游集群建立连接。部署在下游集群上的 Rancher agent 必须能够与 Rancher 建立连接。 + +è¦è®© Fleet 在代ç†åŽå·¥ä½œï¼Œä½ éœ€è¦ä¸ºä¸‹æ¸¸é›†ç¾¤è®¾ç½® **Agent 环境å˜é‡**。以下是集群级别的é…置选项。 + +ä½ å¯ä»¥é€šè¿‡ Rancher UI 为任何集群类型（包括注册集群和自定义集群）é…置这些环境å˜é‡ã€‚å¯ä»¥åœ¨ç¼–辑现有集群或é…置新集群时添加å˜é‡ã€‚ + +对于公共下游集群，[在 Rancher UI 中设置必è¦çš„环境å˜é‡](#在-rancher-ui-中设置环境å˜é‡)就足够了。 + +对于ç§æœ‰èŠ‚点或ç§æœ‰é›†ç¾¤ï¼Œåˆ™éœ€è¦åœ¨èŠ‚点上设置环境å˜é‡ã€‚然åŽï¼Œåœ¨é…置自定义集群或注册ç§æœ‰é›†ç¾¤æ—¶ï¼Œåœ¨ Rancher UI 中é…置环境å˜é‡ã€‚有关如何在 K3s Kubernetes 集群中的 Ubuntu 节点上设置环境å˜é‡çš„示例，请å‚阅[本节](#在ç§æœ‰èŠ‚点上设置环境å˜é‡)。 + +## å¿…è¦çš„环境å˜é‡ + +为代ç†æ·»åŠ  Fleet agent 环境å˜é‡æ—¶ï¼Œå°† 替æ¢ä¸ºä½ çš„ç§æœ‰ä»£ç† IP。 + +| å˜é‡å称 | 值 | +|------------------|--------| +| `HTTP_PROXY` | http://:8888 | +| `HTTPS_PROXY` | http://:8888 | +| `NO_PROXY` | 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local | + +## 在 Rancher UI 中设置环境å˜é‡ + +è¦å°†çŽ¯å¢ƒå˜é‡æ·»åŠ åˆ°çŽ°æœ‰é›†ç¾¤ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦æ·»åŠ çŽ¯å¢ƒå˜é‡çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. å•å‡»**高级选项**。 +1. å•å‡»**添加环境å˜é‡**。 +1. 输入[å¿…è¦çš„环境å˜é‡](#å¿…è¦çš„环境å˜é‡) +1. å•å‡»**ä¿å­˜**。 + +**结果**：Fleet agent 会在代ç†åŽå·¥ä½œã€‚ + +## 在ç§æœ‰èŠ‚点上设置环境å˜é‡ + +对于ç§æœ‰èŠ‚点和ç§æœ‰é›†ç¾¤ï¼Œä»£ç†çŽ¯å¢ƒå˜é‡éœ€è¦åœ¨èŠ‚点本身上设置，以åŠä»Ž Rancher UI é…置。 + +此示例显示了如何在 K3s Kubernetes 集群中的 Ubuntu 节点上设置环境å˜é‡ï¼š + +``` +ssh -o ForwardAgent=yes ubuntu@ +ssh +export proxy_private_ip= +export HTTP_PROXY=http://${proxy_private_ip}:8888 +export HTTPS_PROXY=http://${proxy_private_ip}:8888 +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md new file mode 100644 index 00000000000..f322aac9f3a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md @@ -0,0 +1,21 @@ +--- +title: Windows æ”¯æŒ +--- + +在 Rancher 2.5.6 之å‰ï¼Œ`agent` 在具有 Windows 节点的下游集群上没有原生的 Windows 清å•ã€‚这将导致集群的 `agent` pod 失败。 + +如果你从旧版本的 Rancher å‡çº§åˆ° 2.5.6+，你å¯ä»¥åœ¨*下游集群*中部署具有以下工作æµçš„工作 `agent`： + +1. å°é”所有 Windows 节点。 +1. 对 `agent` 工作负载应用以下容å¿åº¦ã€‚ +1. å–消所有 Windows 节点的å°é”。 +1. 删除所有 `agent` pod。使用新的容å¿åº¦æ¥åˆ›å»ºæ–° pod。 +1. `agent` pod è¿è¡Œå¹¶ä¸º Fleet å¯ç”¨äº†è‡ªåŠ¨æ›´æ–°åŽï¼Œå®ƒä»¬ä¼šæ›´æ–°åˆ°ä¸Ž Windows 兼容的 `agent` 版本。 + +```yaml +tolerations: +- effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/harvester.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/harvester.md new file mode 100644 index 00000000000..81823b2999a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/harvester.md @@ -0,0 +1,48 @@ +--- +title: Harvester é›†æˆ +--- + +Harvester 是 Rancher 2.6.1 新增的功能，[Harvester](https://docs.harvesterhci.io/) 是基于 Kubernetes 构建的开æºè¶…èžåˆåŸºç¡€æž¶æž„ (HCI) 软件。Harvester 安装在裸金属æœåŠ¡å™¨ä¸Šï¼Œæ供集æˆçš„虚拟化和分布å¼å­˜å‚¨åŠŸèƒ½ã€‚虽然 Harvester 使用 Kubernetes è¿è¡Œï¼Œä½†å®ƒä¸éœ€è¦ç”¨æˆ·äº†è§£ Kubernetes 概念，因此是一个更加用户å‹å¥½çš„应用。 + +### 功能开关 + +ä½ å¯ä»¥ä½¿ç”¨ Harvester 的功能开关æ¥ç®¡ç† Harvester 在 Rancher 虚拟化管ç†é¡µé¢çš„访问，用户å¯ä»¥åœ¨è¯¥é¡µé¢ç›´æŽ¥å¯¼èˆªåˆ° Harvester 集群并访问 Harvester UI。Harvester 的功能开关是默认å¯ç”¨çš„。如需了解 Rancher 中功能开关的更多详细信æ¯ï¼Œè¯·å•å‡»[此处](../pages-for-subheaders/enable-experimental-features.md)。 + +è¦å¯¼èˆªåˆ° Harvester 集群，请å•å‡» **☰ > 虚拟化管ç†**。在 **Harvester 集群**页é¢ä¸­ï¼Œå•å‡»é›†ç¾¤ä»¥è½¬åˆ°è¯¥ Harvester 集群的视图。 + +* 如果å¯ç”¨äº† Harvester 功能开关，则会从列出 Kubernetes 集群的任何页é¢æˆ–应用（例如 Fleet 和多集群应用）中过滤掉 Harvester 集群。 + +* 如果ç¦ç”¨äº† Harvester 功能开关，并且导入了 Harvester 集群，Harvester 集群将显示在**集群管ç†**页é¢çš„ Rancher 集群列表中。仅当功能开关为关闭时，Harvester 集群æ‰ä¼šæ˜¾ç¤ºåœ¨é›†ç¾¤åˆ—表中。 + +* é›†æˆ Harvester åŽï¼Œä½ å¯ä»¥å°† Harvester 集群导入 Rancher，对应的集群类型是 `Harvester`。 + +* 用户åªèƒ½åœ¨**虚拟化管ç†**页é¢ä¸Šå¯¼å…¥ Harvester 集群。在**集群管ç†**页é¢ä¸Šå¯¼å…¥é›†ç¾¤æ˜¯ä¸æ”¯æŒçš„，而且会出现警告。建议你返回**虚拟化管ç†**页é¢æ‰§è¡Œæ­¤æ“作。 + +### Harvester 主机驱动 + +[Harvester 主机驱动](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/) 通常å¯ç”¨äºŽ Rancher 中的 RKE å’Œ RKE2 选项。无论 Harvester 功能开关是å¦å¯ç”¨ï¼Œä¸»æœºé©±åŠ¨éƒ½æ˜¯å¯ç”¨çš„。请注æ„，默认情况下主机驱动是关闭的。用户åªèƒ½é€šè¿‡**集群管ç†**页é¢åœ¨ Harvester 上创建 RKE 或 RKE2 集群。 + +Harvester å…许通过 Harvester UI 上传和显示 `.ISO` é•œåƒï¼Œä½† Rancher UI ä¸æ”¯æŒã€‚这是因为 `.ISO` é•œåƒé€šå¸¸éœ€è¦é¢å¤–的设置，这会干扰干净的部署（å³æ— éœ€ç”¨æˆ·å¹²é¢„），并且它们通常ä¸ç”¨äºŽäº‘环境。 + +如需了解 Rancher 中主机驱动的更多详细信æ¯ï¼Œè¯·å•å‡»[此处](../pages-for-subheaders/about-provisioning-drivers.md#主机驱动)。 + +### 端å£è¦æ±‚ + +å¯ä»¥åœ¨[此处](https://docs.harvesterhci.io/v1.1/install/requirements#networking)找到 Harvester 集群的端å£è¦æ±‚。 + +此外，其他网络注æ„事项如下： + +- 请务必为 VM VLAN 网络å¯ç”¨ç‰©ç†äº¤æ¢æœºçš„ VLAN 中继端å£ã€‚ +- 按照[此处](https://docs.harvesterhci.io/v1.1/networking/clusternetwork)的网络设置指å—进行æ“作。 + +对于其他集群（例如 K3s å’Œ RKE1）的其他端å£è¦æ±‚，请å‚阅[这些文档](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters)。 + +### é™åˆ¶ + +--- +**仅适用于 Rancher v2.6.1 å’Œ v2.6.2**： + +- Harvester 0.3.0 ä¸æ”¯æŒç¦»çº¿çŽ¯å¢ƒå®‰è£…。 +- ä¸æ”¯æŒå°† Harvester 0.2.0 å‡çº§åˆ° 0.3.0，也ä¸æ”¯æŒå‡çº§åˆ°æ–°çš„ 1.0.0 版本。 + +--- diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md new file mode 100644 index 00000000000..811c89b3ea8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md @@ -0,0 +1,60 @@ +--- +title: 在 RKE2 å’Œ K3s 集群上安装 Istio 的其他步骤 +--- + +通过 **Apps** 页é¢å®‰è£…或å‡çº§ Istio Helm Chart 时： + +1. 如果è¦å®‰è£… Chart，请å•å‡»**在安装å‰è‡ªå®šä¹‰ Helm 选项**，然åŽå•å‡»**下一步**。 +1. 你将看到é…ç½® Istio Helm Chart 的选项。在**组件**选项å¡ä¸Šï¼Œé€‰ä¸­**å¯ç”¨ CNI** æ—边的框。 +1. 添加一个自定义覆盖文件，该文件指定 `cniBinDir` å’Œ `cniConfDir`。有关这些选项的更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](https://istio.io/latest/docs/setup/additional-setup/cni/#helm-chart-parameters)。下方是一个示例： + + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /opt/cni/bin + cniConfDir: /etc/cni/net.d +``` + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /var/lib/rancher/k3s/data/current/bin + cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d +``` + + + +**结果**：现在你应该å¯ä»¥æ ¹æ®éœ€è¦ä½¿ç”¨ Istio，包括 Sidecar 注入和通过 Kiali 进行监控。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/pod-security-policies.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/pod-security-policies.md new file mode 100644 index 00000000000..2ceed9acdce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/pod-security-policies.md @@ -0,0 +1,53 @@ +--- +title: 在具有 Pod 安全策略的情况下å¯ç”¨ Istio +--- + +如果你å¯ç”¨äº†é™åˆ¶æ€§ Pod 安全策略（Pod Security Policy），由于 Istio 需è¦æŸäº›æƒé™æ‰èƒ½è‡ªè¡Œå®‰è£…å’Œç®¡ç† pod 基础设施，因此 Istio å¯èƒ½æ— æ³•æ­£å¸¸è¿è¡Œã€‚在本文中，我们将é…置一个为 Istio å¯ç”¨äº† PSP 的集群，并设置 Istio CNI æ’件。 + +Istio CNI æ’件ä¸å†è¦æ±‚æ¯ä¸ªåº”用 pod å…·æœ‰ç‰¹æƒ `NET_ADMIN` 容器。如需更多信æ¯ï¼Œè¯·å‚阅 [Istio CNI æ’件文档](https://istio.io/docs/setup/additional-setup/cni)。请注æ„，[Istio CNI æ’件处于 alpha 阶段](https://istio.io/about/feature-stages/)。 + +:::note 先决æ¡ä»¶ï¼š + +- 集群必须是 RKE Kubernetes 集群。 +- 必须使用默认 PodSecurityPolicy 创建集群。 + +è¦åœ¨ä½¿ç”¨ Rancher UI 创建 Kubernetes 集群时å¯ç”¨ Pod 安全策略支æŒï¼Œè¯·è½¬åˆ°é«˜çº§é€‰é¡¹ã€‚在 Pod 安全策略支æŒä¸­ï¼Œå•å‡»å¯ç”¨ï¼Œç„¶åŽé€‰æ‹©ä¸€ä¸ªé»˜è®¤çš„ pod 安全策略。 + +::: + +1. [å°† PodSecurityPolicy 设置为ä¸å—é™åˆ¶](#1-å°†-podsecuritypolicy-设置为ä¸å—é™åˆ¶) +2. [å¯ç”¨ CNI](#2-å¯ç”¨-cni) +3. [éªŒè¯ CNI 是å¦æ­£å¸¸å·¥ä½œ](#3-验è¯-cni-是å¦æ­£å¸¸å·¥ä½œ) + +### 1. å°† PodSecurityPolicy 设置为ä¸å—é™åˆ¶ + +ä¸å—é™åˆ¶çš„ PSP 支æŒå®‰è£… Istio。 + +在安装 Istio 的项目或计划安装 Istio 的项目中，将 PSP 设置为 `unrestricted`。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 找到**项目: System**，然åŽé€‰æ‹© **â‹® > 编辑é…ç½®**。 +1. å°† Pod 安全策略选项更改为ä¸å—é™åˆ¶ï¼Œç„¶åŽå•å‡»**ä¿å­˜**。 + +### 2. å¯ç”¨ CNI + +通过 **Apps** 安装或å‡çº§ Istio 时： + +1. å•å‡»**组件**。 +2. 选中**å¯ç”¨ CNI**æ—边的框。 +3. å®Œæˆ Istio 的安装或å‡çº§ã€‚ + +你也å¯ä»¥é€šè¿‡ç¼–辑 `values.yaml` æ¥å¯ç”¨ CNI： + +``` +istio_cni.enabled: true +``` + +在集群中å¯ç”¨ CNI åŽï¼ŒIstio 应该能æˆåŠŸå®‰è£…。 + +### 3. éªŒè¯ CNI 是å¦æ­£å¸¸å·¥ä½œ + +通过部署[示例应用](https://istio.io/latest/docs/examples/bookinfo/)或部署你自己的应用，æ¥éªŒè¯ CNI 是å¦æ­£å¸¸å·¥ä½œã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/project-network-isolation.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/project-network-isolation.md new file mode 100644 index 00000000000..00b16856358 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/project-network-isolation.md @@ -0,0 +1,21 @@ +--- +title: 项目网络隔离的其他步骤 +--- + +在集群中： + +- ä½ åŒæ—¶ä½¿ç”¨äº† Canal 网络æ’件与 Rancher 2.5.8 之å‰ç‰ˆæœ¬ï¼Œæˆ–者åŒæ—¶ä½¿ç”¨äº† Rancher 2.5.8+ 以åŠä»»æ„支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE 网络æ’件（例如 Canal 或 Cisco ACI æ’件）。 +- å¯ç”¨äº†é¡¹ç›®ç½‘络隔离选项。 +- 安装了 Istio Ingress 模å—。 + +默认情况下，Istio Ingress Gateway pod 无法将入å£æµé‡é‡å®šå‘到工作负载。这是因为安装了 Istio 的命å空间无法访问所有命å空间。为此你有两个选项。 + +第一个选项是在需è¦è®© Istio 控制入å£çš„æ¯ä¸ªå‘½å空间中添加一个新的网络策略。你的策略需è¦åŒ…括以下几行： + +``` +- podSelector: + matchLabels: + app: istio-ingressgateway +``` + +第二个选项是将 `istio-system` 命å空间移动到 `system` 项目中，默认情况下该项目被排除在网络隔离之外。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md new file mode 100644 index 00000000000..5a40d032482 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md @@ -0,0 +1,120 @@ +--- +title: 选择器和抓å–é…ç½® +--- + +Monitoring 应用设置了 `prometheus.prometheusSpec.ignoreNamespaceSelectors=false`，å³åœ¨é»˜è®¤æƒ…况下跨所有命å空间进行监控。 + +这样，你å¯ä»¥æŸ¥çœ‹éƒ¨ç½²åœ¨å…·æœ‰ `istio-injection=enabled` 标签的命å空间中的资æºçš„æµé‡ã€æŒ‡æ ‡å’Œå›¾ã€‚ + +如果你想将 Prometheus é™åˆ¶ä¸ºç‰¹å®šçš„命å空间，请设置 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。完æˆæ­¤æ“作åŽï¼Œä½ éœ€è¦æ·»åŠ å…¶ä»–é…ç½®æ¥ç»§ç»­ç›‘控你的资æºã€‚ + + +### 通过将 ignoreNamespaceSelectors 设置为 True æ¥é™åˆ¶å¯¹ç‰¹å®šå‘½å空间的监控 + +è¦é™åˆ¶å¯¹ç‰¹å®šå‘½å空间的监控，你需è¦ç¼–辑 `ignoreNamespaceSelectors` Helm Chart 选项。你å¯ä»¥åœ¨å®‰è£…或å‡çº§ Monitoring Helm Chart æ—¶é…置此选项： + +1. 安装或å‡çº§ Monitoring Helm Chart 时，编辑 values.yml 并设置 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。 +1. 完æˆå®‰è£…或å‡çº§ã€‚ + +**结果**：Prometheus 将仅用于特定命å空间。æ¢è¨€ä¹‹ï¼Œä½ éœ€è¦è®¾ç½®ä»¥ä¸‹é…置之一æ‰èƒ½ç»§ç»­åœ¨å„ç§ä»ªè¡¨æ¿ä¸­æŸ¥çœ‹æ•°æ®ã€‚ + +### 让 Prometheus 检测其他命åç©ºé—´ä¸­çš„èµ„æº + +如果设置了 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`，则有两ç§æ–¹æ³•è®© Prometheus 检测其他命å空间中的资æºï¼š + +- **监控特定的命å空间**：在命å空间中添加一个 ServiceMonitor 或 PodMonitor 以åŠè¦æŠ“å–的目标。 +- **跨命å空间监控**：将 `additionalScrapeConfig` 添加到你的 rancher-monitoring 实例，从而抓å–所有命å空间中的所有目标。 + +### 监控特定命å空间：创建 ServiceMonitor 或 PodMonitor + +此选项用于定义在特定命å空间中è¦ç›‘控的æœåŠ¡æˆ– pod。 + +å¯ç”¨æ€§æƒè¡¡æŒ‡çš„是，由于你无法跨命å空间进行监控，因此你必须为æ¯ä¸ªå‘½å空间创建 ServiceMonitor 或 PodMonitor。 + +:::note 先决æ¡ä»¶ï¼š + +为 `` 定义 ServiceMonitor 或 PodMonitor。下é¢æ供了一个 ServiceMonitor 示例。 + +::: + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œæ‰“å¼€ kubectl shell。 +1. 如果 ServiceMonitor 或 PodMonitor 文件存储在本地集群中，请è¿è¡Œ `kubectl create -f .yaml`。 +1. 如果 ServiceMonitor 或 PodMonitor 没有存储在本地，请è¿è¡Œ `cat<< EOF | kubectl apply -f -`，将文件内容粘贴到终端，然åŽè¿è¡Œ ​​`EOF` æ¥å®Œæˆå‘½ä»¤ã€‚ +1. è¿è¡Œ `kubectl label namespace istio-injection=enabled` æ¥å¯ç”¨ Envoy sidecar 注入。 + +**结果**：Prometheus å¯ä»¥æŠ“å– ``。 + +
Istio 代ç†çš„ ServiceMonitor 示例
+ +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: envoy-stats-monitor + namespace: istio-system + labels: + monitoring: istio-proxies +spec: + selector: + matchExpressions: + - {key: istio-prometheus-ignore, operator: DoesNotExist} + namespaceSelector: + any: true + jobLabel: envoy-stats + endpoints: + - path: /stats/prometheus + targetPort: 15090 + interval: 15s + relabelings: + - sourceLabels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - action: labeldrop + regex: "__meta_kubernetes_pod_label_(.+)" + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: namespace + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: pod_name +``` + +### 跨命å空间监控：将 ignoreNamespaceSelectors 设置为 False + +此设置为 Prometheus æä¾›é¢å¤–的抓å–é…ç½®æ¥å®žçŽ°è·¨å‘½å空间监控。 + +å¯ç”¨æ€§æƒè¡¡æŒ‡çš„是 Prometheus 的所有 `additionalScrapeConfigs` 都维护在一个 Secret 中。如果在安装 Istio 之å‰å·²ç»ä½¿ç”¨ additionalScrapeConfigs 部署了监控，å‡çº§å¯èƒ½ä¼šå˜å¾—困难。 + +1. 安装或å‡çº§ Monitoring Helm Chart 时，编辑 values.yml 并将 `prometheus.prometheusSpec.additionalScrapeConfigs` 数组设置为下方的**其它抓å–é…ç½®**。 +1. 完æˆå®‰è£…或å‡çº§ã€‚ + +**结果**：Promethe 会抓å–所有带有 `istio-injection=enabled` 标签的命å空间。 + +
其它抓å–é…ç½®
+ +```yaml +- job_name: 'istio/envoy-stats' + scrape_interval: 15s + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:15090 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/cpu-and-memory-allocations.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/cpu-and-memory-allocations.md new file mode 100644 index 00000000000..3a19c21dcc5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/cpu-and-memory-allocations.md @@ -0,0 +1,63 @@ +--- +title: CPU å’Œå†…å­˜åˆ†é… +--- + +本文介ç»æˆ‘们建议在集群中分é…ç»™ Istio 组件的最少计算资æºã€‚ + +æ¯ä¸ªç»„件的 CPU 和内存分é…是[å¯é…ç½®](#é…置资æºåˆ†é…)的。 + +在å¯ç”¨ Istio 之å‰ï¼Œå»ºè®®ä½ å…ˆç¡®è®¤ä½ çš„ Rancher worker 节点是å¦æœ‰è¶³å¤Ÿçš„ CPU 和内存æ¥è¿è¡Œ Istio 的所有组件。 + +:::tip + +在规模较大的部署中，我们强烈建议通过为æ¯ä¸ª Istio 组件添加节点选择器，æ¥å°†åŸºç¡€è®¾æ–½æ”¾ç½®åœ¨é›†ç¾¤ä¸­çš„专用节点上。 + +::: + +下表总结了æ¯ä¸ªæ ¸å¿ƒ Istio 组件推èé…置的 CPU 和内存的最低资æºè¯·æ±‚å’Œé™åˆ¶ã€‚ + +Kubernetes 中的资æºè¯·æ±‚指的是，除éžè¯¥èŠ‚点至少具有指定数é‡çš„å¯ç”¨å†…存和 CPU，å¦åˆ™å·¥ä½œè´Ÿè½½ä¸ä¼šéƒ¨ç½²åœ¨èŠ‚点上。如果工作负载超过 CPU 或内存的é™åˆ¶ï¼Œåˆ™å¯ä»¥å°†å…¶ä»ŽèŠ‚点中终止或驱é€ã€‚有关管ç†å®¹å™¨èµ„æºé™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/)。 + +| 工作负载 | CPU - 请求 | 内存 - 请求 | CPU - é™åˆ¶ | 内存 - é™åˆ¶ | +|----------------------|---------------|------------|-----------------|-------------------| +| å…¥å£ç½‘å…³ | 100m | 128mi | 2000m | 1024mi | +| 出å£ç½‘å…³ | 100m | 128mi | 2000m | 1024mi | +| istiod | 500m | 2048mi | 没有é™åˆ¶ | 没有é™åˆ¶ | +| proxy | 10m | 10mi | 2000m | 1024mi | +| **总计：** | **710m** | **2314Mi** | **6000m** | **3072Mi** | + +## é…置资æºåˆ†é… + +ä½ å¯ä»¥ä¸ºæ¯ç§ç±»åž‹çš„ Istio 组件å•ç‹¬é…置资æºåˆ†é…。本节介ç»äº†æ¯ä¸ªç»„件默认分é…的资æºã€‚ + +为了更轻æ¾åœ°å°†å·¥ä½œè´Ÿè½½è°ƒåº¦åˆ°èŠ‚点，集群管ç†å‘˜å¯ä»¥é™ä½Žç»„件的 CPU 和内存资æºè¯·æ±‚。默认 CPU 和内存分é…是我们推è的最å°å€¼ã€‚ + +关于 Istio é…置的更多信æ¯ï¼Œè¯·å‚阅 [Istio 官方文档](https://istio.io/)。 + +è¦é…置分é…ç»™ Istio 组件的资æºï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `istio-system` 命å空间。在æŸä¸ª Istio 工作负载中（例如 `rancher-istio`），点击**â‹® > 编辑/å‡çº§**。 +1. 点击**å‡çº§**，然åŽé€šè¿‡æ›´æ”¹ values.yaml 或添加[覆盖文件](../../pages-for-subheaders/configuration-options.md#覆盖文件)æ¥ç¼–辑基本组件。有关编辑覆盖文件的更多信æ¯ï¼Œè¯·å‚阅[本节](cpu-and-memory-allocations.md#编辑覆盖文件)。 +1. 更改 CPU 或内存分é…ã€è°ƒåº¦å„个组件的节点，或节点容å¿åº¦ã€‚ +1. 点击**å‡çº§**。然åŽï¼Œæ›´æ”¹å°±èƒ½å¯ç”¨ã€‚ + +**结果**：已更新 Istio 组件的资æºåˆ†é…。 + +### 编辑覆盖文件 + +覆盖文件å¯ä»¥åŒ…å« [Istio Operator 规范](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/#IstioOperatorSpec)中的任æ„å€¼ã€‚åŒ…å« Istio 应用的覆盖文件åªæ˜¯è¦†ç›–文件潜在é…置的一个示例。 + +åªè¦æ–‡ä»¶åŒ…å« `kind: IstioOperator` 且 YAML 选项有效，文件就å¯ä»¥ç”¨ä½œè¦†ç›–。 + +在 Istio 应用æ供的示例覆盖文件中，以下部分能让你更改 Kubernetes 资æºï¼š + +``` +# k8s: +# resources: +# requests: +# cpu: 200m +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/disable-istio.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/disable-istio.md new file mode 100644 index 00000000000..66e0b1eb9b1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/disable-istio.md @@ -0,0 +1,42 @@ +--- +title: ç¦ç”¨ Istio +--- + +本文介ç»å¦‚何在集群中å¸è½½ Istio，以åŠå¦‚何在命å空间或工作负载中ç¦ç”¨ Istio。 + +## 在集群中å¸è½½ Istio + +è¦å¸è½½ Istio： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Installed Apps**。 +1. 在 `istio-system` 命å空间中，转到 `rancher-istio` 并å•å‡» **â‹® > 删除**。 +1. 删除 `rancher-istio` åŽï¼Œé€‰æ‹© `istio-system` 命å空间中所有剩余的应用，然åŽå•å‡»**删除**。 + +**结果**：已删除集群中的 `rancher-istio` 应用。Istio sidecar ä¸èƒ½éƒ¨ç½²åœ¨é›†ç¾¤ä¸­çš„任何工作负载上。 + +:::note + +ä½ ä¸èƒ½å†ç¦ç”¨å’Œé‡æ–°å¯ç”¨ä½ çš„ Istio 安装。如果你想ä¿å­˜è®¾ç½®ä»¥ä¾›å°†æ¥çš„安装使用，请查看并ä¿å­˜å„个 YAML，以便在之åŽçš„安装中å‚考/é‡å¤ä½¿ç”¨ã€‚ + +::: + +**å¸è½½ç–‘难解答**：如果你没有按照å¸è½½æ­¥éª¤æ“作，则å¯èƒ½ä¼šåœ¨å¸è½½è¿‡ç¨‹ä¸­é‡åˆ°ä»¥ä¸‹è­¦å‘Šï¼š + +`Error: uninstallation completed with 1 error(s): unable to build kubernetes objects for delete: unable to recognize "": no matches for kind "MonitoringDashboard" in version "monitoring.kiali.io/v1alpha1"` + +è¿™å¯èƒ½æ„味ç€å‡ ç§æƒ…况。第一ç§æƒ…况是你选择了 `istio-system` 命å空间中的所有应用并åŒæ—¶åˆ é™¤äº†å®ƒä»¬ï¼Œå¦ä¸€ç§æƒ…况是你在删除 `rancher-istio` Chart 之å‰åˆ é™¤äº†`rancher-istio` Chart ä¾èµ–项。由于å¸è½½æœªæ­£ç¡®å®Œæˆï¼Œä½ å°†éœ€è¦æ‰‹åŠ¨æ¸…ç† `istio-system` 命å空间中剩余的资æºã€‚如果ä¸æƒ³è¿›è¡Œæ‰‹åŠ¨æ¸…ç†ï¼Œä½ å¯ä»¥é‡æ–°å®‰è£… `rancher-istio`，然åŽæŒ‰ç…§æ­£ç¡®çš„顺åºå¸è½½å®ƒã€‚ + +## 在命å空间中ç¦ç”¨ Istio + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. 转到è¦å¯ç”¨ Istio 的命å空间，然åŽå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。或者，你也å¯ä»¥å•å‡»å‘½å空间，然åŽåœ¨å‘½å空间详情页é¢ä¸Šï¼Œå•å‡»**â‹® > å¯ç”¨ Istio 自动注入**。 + +**结果**：如果工作负载部署到此命å空间，它们将没有 Istio sidecar。 + +## 从工作负载中移除 Istio Sidecar + +在命å空间中ç¦ç”¨ Istio，然åŽé‡æ–°éƒ¨ç½²å…¶ä¸­çš„工作负载。这些工作负载将在没有 Istio sidecar 的情况下部署。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/rbac-for-istio.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/rbac-for-istio.md new file mode 100644 index 00000000000..ba56d0ff9b9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/istio/rbac-for-istio.md @@ -0,0 +1,43 @@ +--- +title: RBAC +--- + +本文介ç»è®¿é—® Istio 功能所需的æƒé™ã€‚ + +Rancher Istio Chart 安装了三个 `ClusterRole`。 + +## Cluster-Admin 访问 + +默认情况下，åªæœ‰å…·æœ‰ `cluster-admin` `ClusterRole` 的用户å¯ä»¥ï¼š + +- 在集群中安装 Istio 应用。 +- 为 Istio é…置资æºåˆ†é…。 + + +## Admin å’Œ Edit æƒé™ + +默认情况下，åªæœ‰ Admin å’Œ Edit 角色å¯ä»¥ï¼š + +- 为命å空间å¯ç”¨å’Œç¦ç”¨ Istio sidecar 自动注入。 +- å°† Istio sidecar 添加到工作负载。 +- 查看集群的æµé‡æŒ‡æ ‡å’Œæµé‡å›¾ã€‚ +- é…ç½® Istio 的资æºï¼ˆä¾‹å¦‚网关ã€ç›®æ ‡è§„则或虚拟æœåŠ¡ï¼‰ã€‚ + +## Kubernetes 默认角色的默认æƒé™æ‘˜è¦ + +Istio 创建了三个 `ClusterRole`，并将 Istio CRD 访问æƒé™æ·»åŠ åˆ°ä»¥ä¸‹é»˜è®¤ K8s `ClusterRole`： + +| Chart 创建的 ClusterRole | 默认 K8s ClusterRole | Rancher 角色 | + ------------------------------:| ---------------------------:|---------:| +| `istio-admin` | admin | 项目所有者 | +| `istio-edit` | edit | 项目æˆå‘˜ | +| `istio-view` | view | åªè¯» | + +Rancher 将继续使用 cluster-ownerã€cluster-memberã€project-ownerã€project-member 等作为角色å称，但会使用默认角色æ¥ç¡®å®šè®¿é—®æƒé™ã€‚æ¯ä¸ªé»˜è®¤çš„ K8s `ClusterRole` 都有ä¸åŒçš„ Istio CRD æƒé™ä»¥åŠå¯ä»¥æ‰§è¡Œçš„ K8s æ“作，包括 Create (C)，Get (G)，List (L)，Watch (W)，Update (U)，Patch (P)，Delete (D) å’Œ All (*)。 + + +| CRD | Admin | Edit | View | +|----------------------------| ------| -----| ----- +|
  • `config.istio.io`
    • `adapters`
    • `attributemanifests`
    • `handlers`
    • `httpapispecbindings`
    • `httpapispecs`
    • `instances`
    • `quotaspecbindings`
    • `quotaspecs`
    • `rules`
    • `templates`
| GLW | GLW | GLW | +|
  • `networking.istio.io`
    • `destinationrules`
    • `envoyfilters`
    • `gateways`
    • `serviceentries`
    • `sidecars`
    • `virtualservices`
    • `workloadentries`
| * | * | GLW | +|
  • `security.istio.io`
    • `authorizationpolicies`
    • `peerauthentications`
    • `requestauthentications`
| * | * | GLW | \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md new file mode 100644 index 00000000000..ab7f461150a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md @@ -0,0 +1,75 @@ +--- +title: Flows å’Œ ClusterFlows +--- + +有关如何é…ç½® `Flow` å’Œ `ClusterFlow` 的完整详细信æ¯ï¼Œè¯·å‚阅 [Logging Operator 文档](https://kube-logging.github.io/docs/configuration/flow/)。 + +有关如何解决 Logging 缓冲区的内存问题，请å‚阅 [Rancher 与 Logging æœåŠ¡çš„集æˆï¼šæ•…障排除](../../../pages-for-subheaders/logging.md#日志缓冲区导致-pod-过载)。 + +## Flows + +`Flow` 定义è¦æ”¶é›†å’Œè¿‡æ»¤å“ªäº›æ—¥å¿—，以åŠå°†æ—¥å¿—å‘é€åˆ°å“ªä¸ª Output。 + +`Flow` 是一个命å空间资æºã€‚æ¢è¨€ä¹‹ï¼Œåªæœ‰éƒ¨ç½²äº†è¯¥ Flow 的命å空间日志æ‰èƒ½è¢« `Flow` 收集。 + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `Flow`。 + +有关 `Flow` 自定义资æºçš„更多详细信æ¯ï¼Œè¯·å‚阅 [FlowSpec](https://kube-logging.github.io/docs/configuration/crds/v1beta1/flow_types/)。 + +### Matches + +匹é…语å¥ç”¨äºŽé€‰æ‹©ä»Žå“ªäº›å®¹å™¨ä¸­æ‹‰å–日志。 + +ä½ å¯ä»¥æŒ‡å®š match 语å¥ï¼Œç„¶åŽæ ¹æ® Kubernetes 标签ã€å®¹å™¨å’Œä¸»æœºåæ¥é€‰æ‹©æˆ–排除日志。匹é…语å¥ä¼šæŒ‰ç…§å®šä¹‰å’Œå¤„ç†çš„顺åºè¿›è¡Œè¯„估，直到应用了第一个匹é…的选择/排除规则。 + +ä½ å¯ä»¥é€šè¿‡å¡«å†™ Rancher UI 中的 `Flow` 或 `ClusterFlow` 表å•æ¥é…置匹é…。 + +使用 match 语å¥çš„详细示例，请å‚阅[日志路由的官方文档](https://kube-logging.github.io/docs/configuration/log-routing/)。 + +### Filters + +ä½ å¯ä»¥åœ¨ `Flow` 中定义一个或多个过滤器。过滤器å¯ä»¥å¯¹æ—¥å¿—执行å„ç§æ“作，例如，添加其他数æ®ã€è½¬æ¢æ—¥å¿—或解æžè®°å½•ä¸­çš„值。`Flow` 中的过滤器会按定义的顺åºåº”用。 + +有关 Logging Operator 支æŒçš„过滤器列表，请å‚阅 [Fluentd 过滤器的官方文档](https://kube-logging.github.io/docs/configuration/plugins/filters/)。 + +过滤器需è¦åœ¨ YAML 中é…置。 + +### Outputs + +æ­¤ `Output` 会接收æ¥è‡ª `Flow` 的日志。由于 `Flow` 是一个命å空间资æºï¼Œå› æ­¤ `Output` 必须与 `Flow` ä½äºŽç›¸åŒçš„命å空间中。 + +在 Rancher UI 中填写 `Flow` 或 `ClusterFlow` 表å•æ—¶ï¼Œä½ å¯ä»¥å¼•ç”¨`Output`。 + +## ClusterFlows + +为 `ClusterFlow` é…置匹é…ã€è¿‡æ»¤å™¨å’Œ `Output` çš„æ–¹å¼ä¸Ž `Flow` çš„é…置方å¼ç›¸åŒã€‚主è¦åŒºåˆ«åœ¨äºŽ `ClusterFlow` 是集群级别的，并且å¯ä»¥è·¨æ‰€æœ‰å‘½å空间é…置日志收集。 + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `ClusterFlow`。 + +`ClusterFlow` 选择集群中所有命å空间的日志åŽï¼Œé›†ç¾¤çš„日志会被收集并记录到所选的 `ClusterOutput`。 + +## YAML 示例 + +以下示例 `Flow` 转æ¢äº†é»˜è®¤å‘½å空间的日志消æ¯ï¼Œå¹¶å°†æ—¥å¿—å‘é€åˆ° S3 `Output`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: flow-sample + namespace: default +spec: + filters: + - parser: + remove_key_name_field: true + parse: + type: nginx + - tag_normaliser: + format: ${namespace_name}.${pod_name}.${container_name} + localOutputRefs: + - s3-output + match: + - select: + labels: + app: nginx +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md new file mode 100644 index 00000000000..1ad5b65f48b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md @@ -0,0 +1,295 @@ +--- +title: Outputs å’Œ ClusterOutputs +--- + +有关如何é…ç½® `Flow` å’Œ `ClusterFlow` 的完整详细信æ¯ï¼Œè¯·å‚阅 [Logging Operator 文档](https://kube-logging.github.io/docs/configuration/flow/)。 + +有关如何解决 Logging 缓冲区的内存问题，请å‚阅 [Rancher 与 Logging æœåŠ¡çš„集æˆï¼šæ•…障排除](../../../pages-for-subheaders/logging.md#日志缓冲区导致-pod-过载)。 + +## Outputs + +`Output` 资æºå®šä¹‰äº†ä½ çš„ `Flow` å¯ä»¥å‘é€æ—¥å¿—消æ¯çš„ä½ç½®ã€‚`Output` 是 Logging `Flow` 的最åŽé˜¶æ®µã€‚ + +`Output` 是命å空间资æºï¼Œæ¢è¨€ä¹‹ï¼Œåªæœ‰åŒä¸€å‘½å空间内的 `Flow` å¯ä»¥è®¿é—®å®ƒã€‚ + +ä½ å¯ä»¥åœ¨è¿™äº›å®šä¹‰ä¸­ä½¿ç”¨å¯†æ–‡ï¼Œä½†è¿™äº›å¯†æ–‡ä¹Ÿå¿…é¡»ä½äºŽåŒä¸€å‘½å空间中。 + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `Output`。 + +有关 `Output` 自定义资æºçš„更多详细信æ¯ï¼Œè¯·å‚阅 [OutputSpec](https://kube-logging.github.io/docs/configuration/crds/v1beta1/output_types/)。 + +Rancher UI æ供了用于é…置以下类型 `Output` 的表å•ï¼š + +- Amazon ElasticSearch +- Azure Storage +- Cloudwatch +- Datadog +- Elasticsearch +- File +- Fluentd +- GCS +- Kafka +- Kinesis Stream +- LogDNA +- LogZ +- Loki +- New Relic +- Splunk +- SumoLogic +- Syslog + +Rancher UI æ供了用于é…ç½® `Output` 类型ã€ç›®æ ‡å’Œè®¿é—®å‡­è¯ï¼ˆå¦‚果适用）的表å•ã€‚ + +有关 Logging Operator 支æŒçš„日志æ’件é…置示例，请å‚阅 [Logging Operator 文档](https://kube-logging.github.io/docs/configuration/plugins/outputs/)。 + +## ClusterOutputs + +`ClusterOutput` 定义了一个没有命å空间é™åˆ¶çš„ `Output`。åªæœ‰åœ¨ä¸Ž Logging Operator 部署在åŒä¸€å‘½å空间中时，它æ‰èƒ½ç”Ÿæ•ˆã€‚ + +ä½ å¯ä»¥é€šè¿‡åœ¨ Rancher UI 中填写表å•æ¥é…ç½® `ClusterOutput`。 + +有关 `ClusterOutput` 自定义资æºçš„更多详细信æ¯ï¼Œè¯·å‚阅 [ClusterOutput](https://kube-logging.github.io/docs/configuration/crds/v1beta1/clusteroutput_types/)。 + +## YAML 示例 + +安装 Logging åŽï¼Œä½ å¯ä»¥å‚考以下示例æ¥æž„建你自己的 Logging æµæ°´çº¿ï¼š + +- [ClusterOutput 设为 ElasticSearch](#clusteroutput-设为-elasticsearch) +- [Output 设为 Splunk](#output-设为-splunk) +- [Output 设为 Syslog](#output-设为-syslog) +- [ä¸æ”¯æŒçš„ Output](#ä¸æ”¯æŒçš„-output) + +### ClusterOutput 设为 ElasticSearch + +å‡è®¾ä½ æƒ³å°†é›†ç¾¤ä¸­çš„所有日志å‘é€åˆ° `elasticsearch` 集群。首先，创建一个集群 `Output`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-es" + namespace: "cattle-logging-system" +spec: + elasticsearch: + host: elasticsearch.example.com + port: 9200 + scheme: http +``` + +在与 Operator 相åŒçš„命å空间（`cattle-logging-system`）中，我们创建了这个 `ClusterOutput`（没有 elasticsearch é…置）。æ¯æ¬¡åˆ›å»º `ClusterFlow` 或 `ClusterOutput` 时，我们都必须将其放在 `cattle-logging-system` 命å空间中。 + +é…置日志的目的ä½ç½®åŽï¼Œæˆ‘们å¯ä»¥å°è¯•å°†æ‰€æœ‰æ—¥å¿—都é…置到该 `ClusterOutput`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: "cattle-logging-system" +spec: + globalOutputRefs: + - "example-es" +``` + +现在，我们应该能看到é…置的包å«æ—¥å¿—的索引。 + + +### Output 设为 Splunk + +有时候，你的应用程åºå›¢é˜Ÿå¯èƒ½åªæƒ³å°†æŸä¸ªå‘½å空间的日志å‘é€åˆ° `splunk` æœåŠ¡å™¨ã€‚对于这ç§æƒ…况，你å¯ä»¥ä½¿ç”¨å‘½å空间范围的 `Output` å’Œ `Flow`。 + +在开始之å‰ï¼Œå…ˆè®¾ç½®è¯¥å›¢é˜Ÿçš„åº”ç”¨ç¨‹åº `coolapp`。 + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: devteam +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coolapp + namespace: devteam + labels: + app: coolapp +spec: + replicas: 2 + selector: + matchLabels: + app: coolapp + template: + metadata: + labels: + app: coolapp + spec: + containers: + - name: generator + image: paynejacob/loggenerator:latest +``` + +`coolapp` è¿è¡Œæ—¶ï¼Œæˆ‘们将使用与创建 `ClusterOutput` 时类似的路径。但是，我们ä¸ä½¿ç”¨ `ClusterOutput`，而是在应用程åºçš„命å空间中创建 `Output`。 + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Output +metadata: + name: "devteam-splunk" + namespace: "devteam" +spec: + splunkHec: + hec_host: splunk.example.com + hec_port: 8088 + protocol: http +``` + +然åŽï¼Œå†æ¬¡ä¸º `Output` æ供一些日志： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: "devteam-logs" + namespace: "devteam" +spec: + localOutputRefs: + - "devteam-splunk" +``` + + +### Output 设为 Syslog + +å‡è®¾ä½ æƒ³å°†é›†ç¾¤ä¸­çš„所有日志å‘é€åˆ° `syslog` æœåŠ¡å™¨ã€‚首先，我们创建一个 `ClusterOutput`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-syslog" + namespace: "cattle-logging-system" +spec: + syslog: + buffer: + timekey: 30s + timekey_use_utc: true + timekey_wait: 10s + flush_interval: 5s + format: + type: json + app_name_field: test + host: syslog.example.com + insecure: true + port: 514 + transport: tcp +``` + +é…置日志的目的ä½ç½®åŽï¼Œæˆ‘们å¯ä»¥å°è¯•å°†æ‰€æœ‰æ—¥å¿—都é…置到该 `Output`： + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: cattle-logging-system +spec: + globalOutputRefs: + - "example-syslog" +``` + +### ä¸æ”¯æŒçš„ Output + +对于最åŽä¸€ä¸ªç¤ºä¾‹ï¼Œæˆ‘们创建一个 `Output` æ¥å°†æ—¥å¿—写入到ä¸æ˜¯å¼€ç®±å³ç”¨çš„目标ä½ç½®ï¼š + +:::note Syslog 注æ„事项： + +`Syslog` 是å—支æŒçš„ `Output`。但是，此示例ä»æ供了ä¸å—支æŒçš„æ’件概述。 + +::: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: syslog-config + namespace: cattle-logging-system +type: Opaque +stringData: + fluent-bit.conf: | + [INPUT] + Name forward + Port 24224 + + [OUTPUT] + Name syslog + InstanceName syslog-output + Match * + Addr syslog.example.com + Port 514 + Cluster ranchers + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentbit-syslog-forwarder + namespace: cattle-logging-system + labels: + output: syslog +spec: + selector: + matchLabels: + output: syslog + template: + metadata: + labels: + output: syslog + spec: + containers: + - name: fluentbit + image: paynejacob/fluent-bit-out-syslog:latest + ports: + - containerPort: 24224 + volumeMounts: + - mountPath: "/fluent-bit/etc/" + name: configuration + volumes: + - name: configuration + secret: + secretName: syslog-config +--- +apiVersion: v1 +kind: Service +metadata: + name: syslog-forwarder + namespace: cattle-logging-system +spec: + selector: + output: syslog + ports: + - protocol: TCP + port: 24224 + targetPort: 24224 +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: all-logs + namespace: cattle-logging-system +spec: + globalOutputRefs: + - syslog +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: syslog + namespace: cattle-logging-system +spec: + forward: + servers: + - host: "syslog-forwarder.cattle-logging-system" + require_ack_response: false + ignore_network_errors_at_startup: false +``` + +现在，我们分解这里的内容。首先，我们创建一个容器 Deployment，该容器具有é¢å¤–çš„ `syslog` æ’件并支æŒè½¬å‘自å¦ä¸€ä¸ª `fluentd` 的日志。接下æ¥ï¼Œæˆ‘们创建一个é…置为 Deployment 转å‘器的 `Output`。然åŽï¼ŒDeployment `fluentd` 会将所有日志转å‘到é…置的 `syslog` 目标。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/logging-architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/logging-architecture.md new file mode 100644 index 00000000000..c7e0510ebe9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/logging-architecture.md @@ -0,0 +1,28 @@ +--- +title: 架构 +--- + +本节介ç»äº† Rancher Logging 应用程åºçš„架构。 + +有关 Logging Operator 工作原ç†çš„更多详细信æ¯ï¼Œè¯·å‚阅[官方文档](https://kube-logging.github.io/docs/#architecture)。 + +### Logging Operator å·¥ä½œåŽŸç† + +Logging Operator 自动部署和é…ç½® Kubernetes 日志æµæ°´çº¿ã€‚它会在æ¯ä¸ªèŠ‚点上部署和é…置一个 Fluent Bit DaemonSet，从而收集节点文件系统中的容器和应用程åºæ—¥å¿—。 + +Fluent Bit 查询 Kubernetes API 并使用 pod 的元数æ®æ¥ä¸°å¯Œæ—¥å¿—，然åŽå°†æ—¥å¿—和元数æ®éƒ½ä¼ è¾“到 Fluentd。Fluentd 会接收和过滤日志并将日志传输到多个`Output`。 + +以下自定义资æºç”¨äºŽå®šä¹‰äº†å¦‚何过滤日志并将日志å‘é€åˆ° `Output`： + +- `Flow` 是一个命å空间自定义资æºï¼Œå®ƒä½¿ç”¨è¿‡æ»¤å™¨å’Œé€‰æ‹©å™¨å°†æ—¥å¿—消æ¯è·¯ç”±åˆ°å¯¹åº”çš„ `Output`。 +- `ClusterFlow` 用于路由集群级别的日志消æ¯ã€‚ +- `Output` 是一个命å空间资æºï¼Œç”¨äºŽå®šä¹‰å‘é€æ—¥å¿—消æ¯çš„ä½ç½®ã€‚ +- `ClusterOutput` 定义了一个所有 `Flow` å’Œ `ClusterFlow` 都å¯ç”¨çš„ `Output`。 + +æ¯ä¸ª `Flow` 都必须引用一个 `Output`，而æ¯ä¸ª `ClusterFlow` 都必须引用一个 `ClusterOutput`。 + +[Logging Operator 文档](https://kube-logging.github.io/docs/#architecture)中的下图显示了新的 Logging 架构： + +
Logging Operator 如何与 Fluentd 和 Fluent Bit 一起使用
+ +![Logging Operator 如何与 Fluentd 一起使用](/img/banzai-cloud-logging-operator.png) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/logging-helm-chart-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/logging-helm-chart-options.md new file mode 100644 index 00000000000..5c08fb1166a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/logging-helm-chart-options.md @@ -0,0 +1,94 @@ +--- +title: rancher-logging Helm Chart 选项 +--- + +### å¯ç”¨/ç¦ç”¨ Windows 节点 Logging + +è¦å¯ç”¨æˆ–ç¦ç”¨ Windows 节点 Logging，你å¯ä»¥åœ¨ `values.yaml` 中将 `global.cattle.windows.enabled` 设置为 `true` 或 `false`。 + +默认情况下，如果使用 Cluster Dashboard UI 在 Windows 集群上安装了 Logging 应用程åºï¼ŒWindows 节点的 Logging 就会å¯ç”¨ã€‚ + +在这ç§æƒ…况下，将 `global.cattle.windows.enabled` 设置为 `false` 会ç¦ç”¨é›†ç¾¤ä¸Šçš„ Windows 节点 Logging。 +ç¦ç”¨åŽï¼Œä»ä¼šä»Ž Windows 集群中的 Linux 节点收集日志。 + +:::note + +ç›®å‰å­˜åœ¨ä¸€ä¸ª[问题](https://github.com/rancher/rancher/issues/32325)，在 Windows 集群中ç¦ç”¨ Windows Logging åŽæ‰§è¡Œ `helm upgrade` æ—¶ä¸ä¼šåˆ é™¤ Windows nodeAgent。在这ç§æƒ…况下，如果已安装 Windows nodeAgents，用户å¯èƒ½éœ€è¦æ‰‹åŠ¨å¸è½½å®ƒä»¬ã€‚ + +::: + +### 使用自定义 Docker 根目录 + +如果使用了自定义 Docker 根目录，你å¯ä»¥åœ¨ `values.yaml` 中设置 `global.dockerRootDirectory`。 + +这将确ä¿åˆ›å»ºçš„ Logging CR 使用你指定的路径，而ä¸æ˜¯ä½¿ç”¨é»˜è®¤çš„ Docker `data-root` ä½ç½®ã€‚ + +请注æ„，这åªå½±å“ Linux 节点。 + +如果集群中有任何 Windows 节点，则更改将ä¸é€‚用于这些节点。 + +### 为自定义污点添加 NodeSelector 设置和容å¿åº¦ + +ä½ å¯ä»¥æ·»åŠ  `nodeSelector` 设置，并通过编辑 Logging Helm Chart 值æ¥æ·»åŠ å…¶ä»–`容å¿åº¦`。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](taints-and-tolerations.md)。 + +### å¯ç”¨ Logging 应用程åºä»¥ä½¿ç”¨ SELinux + +:::note è¦æ±‚： + +Logging v2 已在 RHEL/CentOS 7 å’Œ 8 上使用 SELinux 进行了测试。 + +::: + +[安全增强型 Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) 是对 Linux 的安全增强。被政府机构使用之åŽï¼ŒSELinux å·²æˆä¸ºè¡Œä¸šæ ‡å‡†ï¼Œå¹¶åœ¨ CentOS 7 å’Œ 8 上默认å¯ç”¨ã€‚ + +è¦é…åˆä½¿ç”¨ Logging V2 与 SELinux，我们建议你根æ®[此说明](../../pages-for-subheaders/selinux-rpm.md)安装 `rancher-selinux` RPM。 + +然åŽï¼Œåœ¨å®‰è£… Logging 应用程åºæ—¶ï¼Œåœ¨ `values.yaml` 中将 `global.seLinux.enabled` 更改为 `true`，使 Chart æ”¯æŒ SELinux。 + +### 其他日志æ¥æº + +默认情况下，Rancher 会收集所有类型集群的 [controlplane 组件](https://kubernetes.io/docs/concepts/overview/components/#control-plane-components)å’Œ[节点组件](https://kubernetes.io/docs/concepts/overview/components/#node-components)的日志。 + +在æŸäº›æƒ…况下，Rancher 也能收集其他的日志。 + +下表总结了æ¯ç§èŠ‚点类型å¯ä»¥æ”¶é›†çš„其他日志æ¥æºï¼š + +| 日志æ¥æº | Linux 节点（包括在 Windows 集群中） | Windows 节点 | +| --- | --- | ---| +| RKE | ✓ | ✓ | +| RKE2 | ✓ | | +| K3s | ✓ | | +| AKS | ✓ | | +| EKS | ✓ | | +| GKE | ✓ | | + +è¦å°†æ‰˜ç®¡ Kubernetes çš„æ供商作为é¢å¤–的日志æ¥æºï¼Œåœ¨å®‰è£…或å‡çº§ Logging Helm Chart 时，请å¯ç”¨ **Enable enhanced cloud provider logging** 选项。 + +å¯ç”¨åŽï¼ŒRancher 会收集æ供商开放å¯ç”¨çš„所有其他节点和 controlplane 日志，ä¸åŒæ供商å¯èƒ½æœ‰æ‰€ä¸åŒã€‚ + +如果你已ç»ä½¿ç”¨äº†äº‘æ供商的日志解决方案，例如 AWS CloudWatch 或 Google Cloud Operations Suite（以å‰ç§°ä¸º Stackdriver），由于原生解决方案å¯ä»¥ä¸å—é™åˆ¶åœ°è®¿é—®æ‰€æœ‰æ—¥å¿—，因此你无需å¯ç”¨æ­¤é€‰é¡¹ã€‚ + +### Systemd é…ç½® + +在 Rancher Logging 中，你必须为 K3s å’Œ RKE2 Kubernetes å‘行版é…ç½® `SystemdLogPath`。 + +K3s å’Œ RKE2 Kubernetes å‘行版将日志写入到 journald，它是 systemd çš„å­ç³»ç»Ÿï¼Œç”¨äºŽæ—¥å¿—记录。è¦æ”¶é›†è¿™äº›æ—¥å¿—，你需è¦å®šä¹‰ `systemdLogPath`。默认路径是 `run/log/journal`，但æŸäº› Linux å‘行版ä¸é»˜è®¤ä½¿ç”¨è¯¥è·¯å¾„。例如，Ubuntu 默认使用 `var/log/journal`。è¦ç¡®å®šä½ çš„ `systemdLogPath` é…置，请å‚阅以下步骤。 + +**Systemd é…置步骤：** + +* 在其中一个节点上è¿è¡Œ `cat /etc/systemd/journald.conf | grep -E ^\#?Storage | cut -d"=" -f2`。 +* 如果返回 `persistent`，则你的 `systemdLogPath` 是 `/var/log/journal`。 +* 如果返回 `volatile`，则你的 `systemdLogPath` 是 `/run/log/journal`。 +* 如果返回 `auto`，则检查 `/var/log/journal` 是å¦å­˜åœ¨ã€‚ + * 如果 `/var/log/journal` 存在，则使用 `/var/log/journal`。 + * 如果 `/var/log/journal` ä¸å­˜åœ¨ï¼Œåˆ™ä½¿ç”¨ `/run/log/journal`。 + +:::note 注æ„事项： + +如果返回的值ä¸åŒ…括在上述æ述中，Rancher Logging 将无法收集 controlplane 日志。è¦è§£å†³æ­¤é—®é¢˜ï¼Œä½ éœ€è¦åœ¨æ¯ä¸ª controlplane 节点上执行以下æ“作。 + +* 在 journald.conf 中设置 `Storage=volatile`。 +* é‡å¯ä¸»æœºã€‚ +* å°† `systemdLogPath` 设置为 `/run/log/journal`。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/rbac-for-logging.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/rbac-for-logging.md new file mode 100644 index 00000000000..2959a9f7946 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/rbac-for-logging.md @@ -0,0 +1,23 @@ +--- +title: Logging çš„ RBAC +--- + +Rancher Logging 有两个角色，分别是 `logging-admin` å’Œ `logging-view`。 + +- `logging-admin` å…许用户完全访问命å空间的 `Flow` å’Œ `Output`。 +- `logging-view` å…许用户*查看*命å空间的 `Flow` å’Œ `Output`ï¼Œä»¥åŠ `ClusterFlow` å’Œ `ClusterOutput`。 + +:::note 为什么选择一个角色而ä¸æ˜¯å¦ä¸€ä¸ªè§’色？ + +`ClusterFlow` å’Œ `ClusterOutput` 资æºçš„编辑æƒé™éžå¸¸å¼ºå¤§ã€‚任何拥有该æƒé™çš„用户都能编辑集群中的所有日志。 + +::: + +在 Rancher 中，集群管ç†å‘˜è§’色是唯一å¯ä»¥å®Œå…¨è®¿é—®æ‰€æœ‰ `rancher-logging` 资æºçš„角色。集群æˆå‘˜æ— æ³•ç¼–辑或读å–任何 Logging 资æºã€‚项目所有者和æˆå‘˜å…·æœ‰ä»¥ä¸‹æƒé™ï¼š + +| 项目所有者 | 项目æˆå‘˜ | +--- | --- +| 能够在其项目命å空间中创建命å空间级别的 `Flow` å’Œ `Output` | åªèƒ½æŸ¥çœ‹é¡¹ç›®å‘½å空间中的 `Flow` å’Œ `Output` | +| å¯ä»¥ä»Žé¡¹ç›®å‘½å空间中收集任何日志 | 无法在其项目命å空间中收集任何日志 | + +如果项目所有者和项目æˆå‘˜éœ€è¦ä½¿ç”¨ Logging，他们需è¦åœ¨é¡¹ç›®ä¸­è‡³å°‘有*一个*命å空间。如果没有，他们å¯èƒ½çœ‹ä¸åˆ°é¡¶éƒ¨å¯¼èˆªä¸‹æ‹‰åˆ—表中的 Logging 按钮。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/taints-and-tolerations.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/taints-and-tolerations.md new file mode 100644 index 00000000000..16a6a08fd26 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/logging/taints-and-tolerations.md @@ -0,0 +1,65 @@ +--- +title: 处ç†æ±¡ç‚¹å’Œå®¹å¿åº¦ +--- + +在 Kubernetes 节点上添加污点会导致 pod 排斥在该节点上è¿è¡Œã€‚ + +é™¤éž pod 对该节点的污点具有`容å¿åº¦`（toleration），å¦åˆ™ Pod 将在集群中的其他节点上è¿è¡Œã€‚ + +[污点和容å¿åº¦](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/)å¯ä»¥ä¸Ž `PodSpec` 中的 `nodeSelector` [字段](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)一起使用，从而实现*相åçš„*污点效果。 + +`nodeSelector` å¯ä»¥ä½¿ pod 被å¸å¼•åˆ°æŸç±»èŠ‚点。 + +两者都能让 pod 选择在哪个节点上è¿è¡Œã€‚ + +- [Rancher 日志堆栈中的默认实现](#rancher-日志堆栈中的默认实现) +- [为自定义污点添加 NodeSelector 设置和容å¿åº¦](#为自定义污点添加-nodeselector-设置和容å¿åº¦) + + +### Rancher 日志堆栈中的默认实现 + +默认情况下，Rancher 使用 `cattle.io/os=linux` æ¥å°†æ±¡ç‚¹åº”用到所有 Linux 节点，而ä¸å½±å“ Windows 节点。 +日志堆栈 pod 具有针对此污点的`容å¿åº¦`，因此它们能够è¿è¡Œåœ¨ Linux 节点上。 +此外，大多数日志堆栈 pod 仅在 Linux 上è¿è¡Œï¼Œå¹¶æ·»åŠ äº† `nodeSelector` 以确ä¿å®ƒä»¬åœ¨ Linux 节点上è¿è¡Œã€‚ + +此示例 Pod YAML 文件显示了与容å¿åº¦ä¸€èµ·ä½¿ç”¨çš„ nodeSelector： + +```yaml +apiVersion: v1 +kind: Pod +# metadata... +spec: + # containers... + tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + nodeSelector: + kubernetes.io/os: linux +``` + +在上é¢çš„示例中，我们确ä¿äº†çš„ pod 仅在 Linux 节点上è¿è¡Œï¼Œå¹¶ä¸”为所有 Linux 节点上的污点添加了`容å¿åº¦`。 + +ä½ å¯ä»¥å¯¹ Rancher 现有的污点或你自己的自定义污点执行相åŒçš„æ“作。 + +### 为自定义污点添加 NodeSelector 设置和容å¿åº¦ + +如果è¦æ·»åŠ ä½ è‡ªå·±çš„ `nodeSelector` 设置，或者è¦ä¸ºå…¶ä»–污点添加 `容å¿åº¦`，你å¯ä»¥å°†ä»¥ä¸‹å†…容传递给 Chart 的值： + +```yaml +tolerations: + # insert tolerations... +nodeSelector: + # insert nodeSelector... +``` + +这些值会将这两个设置添加到 `fluentd`ã€`fluentbit`å’Œ `logging-operator` 容器中。 +本质上，这些是日志堆栈中所有 pod 的全局设置。 + +但是，如果你想*ä»…*为 `fluentbit` 容器添加容å¿åº¦ï¼Œä½ å¯ä»¥å°†ä»¥ä¸‹å†…容添加到 Chart 的值中： + +```yaml +fluentbit_tolerations: + # insert tolerations list for fluentbit containers only... +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/longhorn.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/longhorn.md new file mode 100644 index 00000000000..ea34d0938ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/longhorn.md @@ -0,0 +1,70 @@ +--- +title: Longhorn - Kubernetes 的云原生分布å¼å—存储 +--- + +[Longhorn](https://longhorn.io/) 是一个轻é‡çº§ã€å¯é ã€æ˜“用的 Kubernetes 分布å¼å—存储系统。 + +Longhorn 是å…费的开æºè½¯ä»¶ã€‚Longhorn 最åˆç”± Rancher Labs å¼€å‘，现在正在作为云原生计算基金会的沙盒项目进行开å‘。它å¯ä»¥é€šè¿‡ Helmã€kubectl 或 Rancher UI 安装在任何 Kubernetes 集群上。有关其架构的更多信æ¯ï¼Œè¯·å‚阅[此处](https://longhorn.io/docs/latest/concepts/)。 + +使用 Longhorn，你å¯ä»¥ï¼š + +- å°† Longhorn å·ç”¨ä½œ Kubernetes 集群中分布å¼æœ‰çŠ¶æ€åº”用程åºçš„æŒä¹…存储 +- 将你的å—存储分区为 Longhorn å·ï¼Œä»¥ä¾¿åœ¨æœ‰æˆ–没有云æ供商的情况下使用 Kubernetes å· +- 跨多个节点和数æ®ä¸­å¿ƒå¤åˆ¶å—存储以æ高å¯ç”¨æ€§ +- 将备份数æ®å­˜å‚¨åœ¨ NFS 或 AWS S3 等外部存储中 +- 创建跨集群ç¾éš¾æ¢å¤å·ï¼Œä»¥ä¾¿ä½¿ç”¨å¦ä¸€ä¸ª Kubernetes 集群中的备份快速æ¢å¤ä¸» Kubernetes é›†ç¾¤ä¸­çš„æ•°æ® +- 计划å·çš„定期快照，并将定期备份调度到 NFS 或兼容 S3 的辅助存储 +- 使用备份æ¥æ¢å¤å· +- 在ä¸ä¸­æ–­æŒä¹…å·çš„情况下å‡çº§ Longhorn + +
Longhorn 仪表æ¿
+ +![Longhorn 仪表æ¿](/img/longhorn-screenshot.png) + +### 使用 Rancher 安装 Longhorn + +1. 满足所有[安装è¦æ±‚](https://longhorn.io/docs/latest/deploy/install/#installation-requirements)。 +1. 转到è¦å®‰è£… Longhorn 的集群。 +1. å•å‡» **Apps**。 +1. å•å‡» **Chart**。 +1. 点击 **Longhorn**。 +1. å¯é€‰ï¼šè¦è‡ªå®šä¹‰åˆå§‹è®¾ç½®ï¼Œè¯·å•å‡» **Longhorn 默认设置**并编辑é…置。如需自定义设置的帮助，请å‚阅 [Longhorn 文档](https://longhorn.io/docs/latest/references/settings/)。 +1. å•å‡»**安装**。 + +**结果**：Longhorn 已部署到 Kubernetes 集群中。 + +### 从 Rancher UI 访问 Longhorn + +1. 转到安装了 Longhorn 的集群。在左侧导航èœå•ä¸­ï¼Œå•å‡» **Longhorn**。 +1. 在此页é¢ä¸Šï¼Œä½ å¯ä»¥ç¼–辑 Longhorn 管ç†çš„ Kubernetes 资æºã€‚è¦æŸ¥çœ‹ Longhorn UI，请å•å‡»**概述**中的 **Longhorn** 按钮。 + +**结果**：你将转到 Longhorn UI，你å¯ä»¥åœ¨é‚£é‡Œç®¡ç† Longhorn å·åŠå…¶åœ¨ Kubernetes 集群中的副本，还å¯ä»¥æŸ¥çœ‹ä½äºŽå¦ä¸€ä¸ª Kubernetes 集群或 S3 中的 Longhorn 存储辅助备份。 + +### 从 Rancher UI å¸è½½ Longhorn + +1. 转到安装了 Longhorn 的集群，然åŽå•å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `longhorn-system` 命å空间并选中 `longhorn` å’Œ `longhorn-crd` 应用程åºæ—边的框。 +1. å•å‡»**删除**并确认**删除**。 + +**结果**：Longhorn 已被å¸è½½ã€‚ + +### GitHub 仓库 + +Longhorn 项目在[此处](https://github.com/longhorn/longhorn)。 + +### 文档 + +Longhorn 文档在[此处](https://longhorn.io/docs/)。 + +### 架构 + +Longhorn 为æ¯ä¸ªå·åˆ›å»ºä¸“用的存储控制器，并在存储在多个节点上的多个副本之间åŒæ­¥å¤åˆ¶è¯¥å·ã€‚ + +存储控制器和副本本身是使用 Kubernetes 编排的。 + +有关其架构的更多信æ¯ï¼Œè¯·å‚阅[此处](https://longhorn.io/docs/latest/concepts/)。 + +
Longhorn 架构
+ +![Longhorn 架构](/img/longhorn-architecture.svg) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md new file mode 100644 index 00000000000..0161de45166 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md @@ -0,0 +1,113 @@ +--- +title: å†…ç½®ä»ªè¡¨æ¿ +--- + +## Grafana UI + +ä½ å¯ä»¥ä½¿ç”¨ [Grafana](https://grafana.com/grafana/) 对存储在å„个地方的指标进行查询ã€å¯è§†åŒ–ã€å‘Šè­¦å’Œäº†è§£ã€‚你能与你的团队创建ã€æŽ¢ç´¢å’Œå…±äº«ä»ªè¡¨æ¿ï¼Œå¹¶åŸ¹å…»æ•°æ®é©±åŠ¨çš„文化。 + +è¦æŸ¥çœ‹æ—¶é—´åºåˆ—æ•°æ®å¯è§†åŒ–的默认仪表æ¿ï¼Œè¯·è½¬åˆ° Grafana UI。 + +### 自定义 Grafana + +è¦æŸ¥çœ‹å’Œè‡ªå®šä¹‰ç”¨äºŽæ”¯æŒ Grafana 仪表æ¿çš„ PromQL 查询，请å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md)。 + +### æŒä¹…化 Grafana ä»ªè¡¨æ¿ + +è¦åˆ›å»ºæŒä¹…化 Grafana 仪表æ¿ï¼Œè¯·å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md)。 + +### 访问 Grafana + +有关 Grafana çš„ RBAC，请å‚阅[本节](rbac-for-monitoring.md#grafana-çš„-rbac)。 + + +## Alertmanager UI + +安装 `rancher-monitoring` åŽï¼Œä¼šéƒ¨ç½² Prometheus Alertmanager UI，从而让你查看告警以åŠå½“å‰çš„ Alertmanager é…置。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅 [Alertmanager 工作原ç†](how-monitoring-works.md#3-alertmanager-的工作原ç†)。 + +::: + +### 访问 Alertmanager UI + +Alertmanager UI å¯è®©ä½ æŸ¥çœ‹æœ€è¿‘触å‘的告警。 + +:::note 先决æ¡ä»¶ï¼š + +必须安装 `rancher-monitoring` 应用。 + +::: + +è¦æŸ¥çœ‹ Alertmanager UI： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Alertmanager UI 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Alertmanager**。 + +**结果**：在新选项å¡ä¸­æ‰“å¼€ Alertmanager UI。如需é…置帮助，请å‚阅[官方 Alertmanager 文档](https://prometheus.io/docs/alerting/latest/alertmanager/)。 + +有关在 Rancher 中é…ç½® Alertmanager 的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 + +
Alertmanager UI
+ +![Alertmanager UI](/img/alertmanager-ui.png) + + +### 查看默认告警 + +è¦æŸ¥çœ‹é»˜è®¤è§¦å‘的告警，请转到 Alertmanager UI 并å•å‡»**展开所有组**。 + + +## Prometheus UI + +默认情况下，[kube-state-metrics service](https://github.com/kubernetes/kube-state-metrics) å‘ monitoring 应用æä¾› CPU 和内存利用率的大é‡ä¿¡æ¯ã€‚这些指标涵盖了跨命å空间的 Kubernetes 资æºã€‚æ¢è¨€ä¹‹ï¼Œå¦‚果你è¦æŸ¥çœ‹æœåŠ¡çš„资æºæŒ‡æ ‡ï¼Œä½ æ— éœ€ä¸ºå…¶åˆ›å»ºæ–°çš„ ServiceMonitor。由于数æ®å·²ç»åœ¨æ—¶é—´åºåˆ—æ•°æ®åº“中，你å¯ä»¥è½¬åˆ° Prometheus UI 并è¿è¡Œ PromQL 查询æ¥èŽ·å–ä¿¡æ¯ã€‚ä½ å¯ä»¥ä½¿ç”¨ç›¸åŒçš„查询æ¥é…ç½® Grafana 仪表æ¿ï¼Œä»Žè€Œæ˜¾ç¤ºè¿™äº›æŒ‡æ ‡éšæ—¶é—´å˜åŒ–的图表。 + +è¦æŸ¥çœ‹ Prometheus UI，请安装 `rancher-monitoring`。然åŽï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Prometheus UI 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Prometheus Graph**。 + +
Prometheus Graph UI
+ +![Prometheus Graph UI](/img/prometheus-graph-ui.png) + +### 查看 Prometheus 目标 + +è¦æŸ¥çœ‹ä½ æ­£åœ¨ç›‘控的æœåŠ¡ï¼Œä½ éœ€è¦æŸ¥çœ‹ä½ çš„目标。目标是由 ServiceMonitor å’Œ PodMonitor 设置的，指的是指标抓å–çš„çš„æºã€‚你无需直接编辑目标，但 Prometheus UI å¯ä¸ºä½ æ供抓å–的所有指标æ¥æºçš„概览。 + +è¦æŸ¥çœ‹ Prometheus 目标，请安装 `rancher-monitoring`。然åŽï¼š + + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Prometheus 目标的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. å•å‡» **Prometheus 目标**。 + +
Prometheus UI 中的目标
+ +![Prometheus 目标 UI](/img/prometheus-targets-ui.png) + +### 查看 PrometheusRules + +当你定义规则时（在 PrometheusRule 资æºçš„ RuleGroup 中声明），[规则本身的规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule)会包å«æ ‡ç­¾ï¼Œç„¶åŽ Alertmanager 会使用这些标签æ¥ç¡®å®šæŽ¥æ”¶å¯¹åº”告警的路由。 + +è¦æŸ¥çœ‹ PrometheusRule，请安装 `rancher-monitoring`。然åŽï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Prometheus 规则**。 + +你还å¯ä»¥åœ¨ Prometheus UI 中查看规则： + +
Prometheus UI 中的规则
+ +![PrometheusRules UI](/img/prometheus-rules-ui.png) + +有关在 Rancher 中é…ç½® PrometheusRule 的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md new file mode 100644 index 00000000000..3a2ec873a29 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md @@ -0,0 +1,248 @@ +--- +title: Monitoring çš„å·¥ä½œåŽŸç† +--- + +## 1. 架构概述 + +_**下文æ述了数æ®å¦‚何æµç» Monitoring V2 应用程åºï¼š**_ + +### Prometheus Operator + +Prometheus Operator 观察正在创建的 ServiceMonitorsã€PodMonitors å’Œ PrometheusRules。在创建 Prometheus é…置资æºæ—¶ï¼ŒPrometheus Operator 会调用 Prometheus API æ¥åŒæ­¥æ–°é…置。如本节末尾的图所示，Prometheus Operator 充当 Prometheus å’Œ Kubernetes 之间的中介，调用 Prometheus API æ¥åŒæ­¥ Prometheus 与 Kubernetes 中的监控相关资æºã€‚ + +### ServiceMonitor å’Œ PodMonitor + +ServiceMonitor å’Œ PodMonitor 以声明方å¼æŒ‡å®šéœ€è¦ç›‘控的目标，例如 Service å’Œ Pod。 + +- 目标是根æ®é…置的 Prometheus 抓å–间隔定期抓å–的，而抓å–的指标存储在 Prometheus 时间åºåˆ—æ•°æ®åº“（Time Series Database，TSDB）中。 + +- 为了执行抓å–，你需è¦ä½¿ç”¨æ ‡ç­¾é€‰æ‹©å™¨æ¥å®šä¹‰ ServiceMonitor å’Œ PodMonitor，这些标签选择器用于确定è¦æŠ“å–çš„ Service 或 Pod，并能确定端点（端点能确定抓å–如何在给定目标上进行，例如在 TCP 10252 中抓å–指标，通过 IP åœ°å€ x.x.x.x 进行代ç†ï¼‰ã€‚ + +- 开箱å³ç”¨åœ°ï¼ŒMonitoring V2 附带了一些预é…置的 Exporter，这些 Exporter æ ¹æ®å…¶æ‰€åœ¨çš„ Kubernetes 集群的类型进行部署。有关详细信æ¯ï¼Œè¯·å‚阅[抓å–和公开指标](#5-抓å–和公开指标)。 + +### PushProx çš„å·¥ä½œåŽŸç† + +- æŸäº›å†…部 Kubernetes 组件是通过部署在 Monitoring V2 中å为 **PushProx** 的代ç†æ¥æŠ“å–的。通过 PushProx å‘ Prometheus 公开指标的 Kubernetes 组件分别是 `kube-controller-manager`ã€`kube-scheduler`, `etcd` å’Œ `kube-proxy`。 + +- 对于æ¯ä¸ª PushProx Exporter，我们在所有目标节点上都部署一个 PushProx 客户端。例如，将 PushProx 客户端部署到 kube-controller-manager 的所有 controlplane 节点ã€kube-etcd 的所有 etcd 节点上，和 kubelet 的所有节点上。 + +- 我们为æ¯ä¸ª Exporter 部署了一个 PushProx 代ç†ã€‚导出指标的æµç¨‹å¦‚下： + +1. PushProx 客户端与 PushProx 代ç†å»ºç«‹å‡ºç«™è¿žæŽ¥ã€‚ +1. 然åŽï¼Œå®¢æˆ·ç«¯ä¼šè½®è¯¢ä»£ç†ä»¥èŽ·å–进入代ç†çš„抓å–请求。 +1. 当代ç†æ”¶åˆ°æ¥è‡ª Prometheus 的抓å–请求时，客户端会将其视为轮询的结果。 +1. 客户端抓å–内部组件。 +1. 内部组件通过将指标推回代ç†æ¥å“应。 + + +

使用 PushProx 导出指标的过程：
+ +![使用 PushProx 导出指标的过程](/img/pushprox-process.svg) + +### PrometheusRules + +PrometheusRule 用于定义指标或时间åºåˆ—æ•°æ®åº“查询触å‘告警的规则。规则是按时间间隔评估的。 + +- **记录规则**æ ¹æ®å·²æ”¶é›†çš„现有åºåˆ—创建一个新的时间åºåˆ—。它们常用于预先计算的å¤æ‚查询。 +- **告警规则**è¿è¡Œç‰¹å®šæŸ¥è¯¢ï¼Œå¦‚果查询评估为éžé›¶å€¼ï¼Œåˆ™ä»Ž Prometheus 触å‘告警。 + +### 告警路由 + +一旦 Prometheus 确定需è¦è§¦å‘告警，就会将告警转å‘到 **Alertmanager**。 + +- 告警包å«æ¥è‡ª PromQL 查询本身的标签，以åŠæŒ‡å®šåˆå§‹ PrometheusRule 时设置的其他标签和注释。 + +- 在接收任何告警之å‰ï¼ŒAlertmanager 会用é…置中指定的**路由 (Route)** å’Œ**接收器 (Receiver)** æ¥å½¢æˆä¸€ä¸ªè·¯ç”±æ ‘，所有传入的告警都会在该路由树上进行评估。路由树的æ¯ä¸ªèŠ‚点都å¯ä»¥æ ¹æ®é™„加到 Prometheus 告警的标签æ¥æŒ‡å®šè¦è¿›è¡Œçš„其他分组ã€æ ‡ç­¾å’Œè¿‡æ»¤ã€‚路由树上的节点（通常是å¶èŠ‚点）还å¯ä»¥æŒ‡å®šåˆ°è¾¾çš„告警需è¦å‘é€åˆ°ä»€ä¹ˆæŽ¥æ”¶å™¨ï¼Œä¾‹å¦‚ Slackã€PagerDutyã€SMS 等。请注æ„，Alertmanager é¦–å…ˆå‘ **alertingDriver** å‘é€å‘Šè­¦ï¼Œç„¶åŽ alertingDriver å†å°†å‘Šè­¦å‘é€æˆ–转å‘到正确的目标ä½ç½®ã€‚ + +- 路由和接收器也通过 Alertmanager Secret 存储在 Kubernetes API 中。当 Secret 更新时，Alertmanager 也会自动更新。请注æ„，路由仅通过标签å‘生（而ä¸æ˜¯é€šè¿‡æ³¨é‡Šç­‰ï¼‰ã€‚ + +
æ•°æ®å¦‚何æµç» Monitoring 应用程åº
+ + +## 2. Prometheus çš„å·¥ä½œåŽŸç† + +### 存储时间åºåˆ—æ•°æ® + +收集 Exporter 的指标åŽï¼ŒPrometheus 将时间åºåˆ—存储在本地ç£ç›˜æ—¶é—´åºåˆ—æ•°æ®åº“中。Prometheus å¯ä»¥é€‰æ‹©ä¸Žè¿œç¨‹ç³»ç»Ÿé›†æˆï¼Œä½† `rancher-monitoring` 使用本地存储æ¥å­˜å‚¨æ—¶é—´åºåˆ—æ•°æ®åº“。 + +存储åŽï¼Œç”¨æˆ·å¯ä»¥ä½¿ç”¨ PromQL（Prometheus 的查询语言）查询此 TSDB。 + +PromQL 查询å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼è¿›è¡Œå¯è§†åŒ–： + +1. 通过在 Prometheus çš„ Graph UI 中æ供查询，UI 会显示数æ®çš„简å•å›¾å½¢è§†å›¾ã€‚ +1. é€šè¿‡åˆ›å»ºåŒ…å« PromQL 查询和其他格å¼åŒ–指令的 Grafana 仪表æ¿ï¼Œè¿™äº›æŒ‡ä»¤ç”¨äºŽæ ‡è®°è½´ã€æ·»åŠ å•ä½ã€æ›´æ”¹é¢œè‰²ã€ä½¿ç”¨æ›¿ä»£å¯è§†åŒ–等。 + +### 为 Prometheus 定义规则 + +规则定义了 Prometheus 需è¦åœ¨å¸¸è§„ `evaluationInterval` 上执行的查询，从而执行æŸäº›æ“作，例如触å‘å‘Šè­¦ï¼ˆå‘Šè­¦è§„åˆ™ï¼‰æˆ–æ ¹æ® TSDB 中存在的其他查询（记录规则）预先计算查询。这些规则编ç åœ¨ PrometheusRules 自定义资æºä¸­ã€‚创建或更新 PrometheusRule 自定义资æºåŽï¼ŒPrometheus Operator 会观察å˜åŒ–，并调用 Prometheus API æ¥å®šæœŸåŒæ­¥ Prometheus 当å‰æ­£åœ¨è¯„估的规则集。 + +PrometheusRule 支æŒå®šä¹‰ä¸€ä¸ªæˆ–多个 RuleGroup。æ¯ä¸ª RuleGroup 由一组 Rule 对象组æˆï¼Œæ¯ä¸ª Rule 对象å‡èƒ½è¡¨ç¤ºå‘Šè­¦æˆ–记录规则，并具有以下字段： + +- 新告警或记录的å称 +- 新告警或记录的 PromQL è¡¨è¾¾å¼ +- 用于标记告警或记录的标签（例如集群å称或严é‡æ€§ï¼‰ +- 对需è¦åœ¨å‘Šè­¦é€šçŸ¥ä¸Šæ˜¾ç¤ºçš„其他é‡è¦ä¿¡æ¯è¿›è¡Œç¼–ç çš„注释（例如摘è¦ã€æè¿°ã€æ¶ˆæ¯ã€Runbook URL 等）。记录规则ä¸éœ€è¦æ­¤å­—段。 + +在评估[规则](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#rule)时，Prometheus 将执行é…置的 PromQL 查询，添加其他标签（或注释 - 仅用于告警规则），并为规则执行所需的æ“作。例如，如果æŸä¸ªå‘Šè­¦è§„则将 `team:front-end` 作为标签添加到é…置的 PromQL 查询，该标签会尾附到触å‘的告警，这将å…许 Alertmanager 将告警转å‘到正确的接收器。 + +### 告警和记录规则 + +Prometheus ä¸ä¼šç»´æŠ¤å‘Šè­¦æ˜¯å¦å¤„于 active 状æ€ã€‚它在æ¯ä¸ªè¯„估间隔内é‡å¤è§¦å‘å‘Šè­¦ï¼Œå¹¶æ ¹æ® Alertmanager 对告警进行分组和过滤æˆæœ‰æ„义的通知。 + +`evaluation_interval` 常é‡å®šä¹‰äº† Prometheus æ ¹æ®æ—¶é—´åºåˆ—æ•°æ®åº“评估告警规则的频率。与 `scrape_interval` 类似，`evaluation_interval` 的默认值也是一分钟。 + +规则包å«åœ¨ä¸€ç»„规则文件中。规则文件包括告警规则和记录规则，但åªæœ‰å‘Šè­¦è§„则æ‰ä¼šåœ¨è¯„ä¼°åŽè§¦å‘告警。 + +对于记录规则，Prometheus 会è¿è¡ŒæŸ¥è¯¢ï¼Œç„¶åŽå°†å…¶å­˜å‚¨ä¸ºæ—¶é—´åºåˆ—。如果需è¦å­˜å‚¨éžå¸¸æ˜‚贵或耗时的查询的结果，这ç§åˆæˆçš„时间åºåˆ—则éžå¸¸æœ‰ç”¨ï¼Œå› æ­¤ä½ å¯ä»¥åœ¨åŽç»­æ›´å¿«åœ°è¿›è¡ŒæŸ¥è¯¢å®ƒä»¬ã€‚ + +告警规则是更常用的。æ¯å½“告警规则评估为正数时，Prometheus 都会触å‘告警。 + +在触å‘告警之å‰ï¼ŒRule 文件会根æ®å®žé™…用例将标签和注释添加到告警中： + +- 标签用于标识告警的信æ¯ï¼Œå¹¶å¯èƒ½å½±å“告警的路由。例如，如果在å‘é€æœ‰å…³æŸä¸ªå®¹å™¨çš„告警时，你å¯ä»¥ä½¿ç”¨å®¹å™¨ ID 作为标签。 + +- 注释用于表示ä¸å½±å“告警路由ä½ç½®çš„ä¿¡æ¯ï¼Œä¾‹å¦‚ Runbook 或错误消æ¯ã€‚ + +## 3. Alertmanager çš„å·¥ä½œåŽŸç† + +Alertmanager 处ç†ç”± Prometheus server 等客户端应用å‘é€çš„告警。它负责以下任务： + +- 删除é‡å¤æ•°æ®ï¼Œåˆ†ç»„，并将告警路由到正确的接收器集æˆï¼ˆä¾‹å¦‚电å­é‚®ä»¶ã€PagerDuty 或 OpsGenie） + +- é™éŸ³å’ŒæŠ‘制告警 + +- 跟踪éšæ—¶é—´è§¦å‘çš„å‘Šè­¦ + +- å‘é€å‘Šè­¦çš„状æ€ï¼Œå³å‘Šè­¦æ˜¯å¦æ­£åœ¨è§¦å‘，或者是å¦å·²ç»è§£å†³ + +### ç”± alertingDrivers 转å‘çš„å‘Šè­¦ + +安装 alertingDriver åŽä¼šæ ¹æ® alertingDriver çš„é…置创建一个 `Service`，å¯ç”¨ä½œ Teams 或 SMS 的接收器 URL。接收器中的 URL ä¼šæŒ‡å‘ alertingDrivers。因此 Alertmanager é¦–å…ˆå‘ alertingDriver å‘é€å‘Šè­¦ï¼Œç„¶åŽ alertingDriver 将告警转å‘或å‘é€åˆ°æ­£ç¡®çš„目的ä½ç½®ã€‚ + +### 将告警路由到接收器 + +Alertmanager è´Ÿè´£å调告警的å‘é€ä½ç½®ã€‚它å…许你根æ®æ ‡ç­¾å¯¹å‘Šè­¦è¿›è¡Œåˆ†ç»„，并根æ®æ ‡ç­¾åŒ¹é…情况æ¥è§¦å‘告警。一个最顶层路由会接å—所有告警。然åŽï¼ŒAlertmanager 会根æ®å‘Šè­¦æ˜¯å¦åŒ¹é…下一个路由的æ¡ä»¶ï¼Œç»§ç»­å°†å‘Šè­¦è·¯ç”±åˆ°æŽ¥æ”¶å™¨ã€‚ + +虽然 Rancher UI 表å•åªå…许编辑两层深的路由树，但你å¯ä»¥é€šè¿‡ç¼–辑 Alertmanager Secret æ¥é…置更深的嵌套路由结构。 + +### é…置多个接收器 + +ä½ å¯ä»¥ç¼–辑 Rancher UI 中的表å•æ¥è®¾ç½®ä¸€ä¸ªæŽ¥æ”¶å™¨èµ„æºï¼Œå…¶ä¸­åŒ…å« Alertmanager 将告警å‘é€åˆ°ä½ çš„通知系统所需的所有信æ¯ã€‚ + +通过在 Alertmanager 或接收器é…置中编辑自定义 YAML，你还å¯ä»¥å°†å‘Šè­¦å‘é€åˆ°å¤šä¸ªé€šçŸ¥ç³»ç»Ÿã€‚有关详细信æ¯ï¼Œè¯·å‚阅[接收器é…ç½®](../../reference-guides/monitoring-v2-configuration/receivers.md#é…置多个接收器)。 + +## 4. Monitoring V2 特定组件 + +Prometheus Operator 引入了一组[自定义资æºå®šä¹‰](https://github.com/prometheus-operator/prometheus-operator#customresourcedefinitions)，å…许用户通过在集群上创建和修改这些自定义资æºæ¥éƒ¨ç½²å’Œç®¡ç† Prometheus å’Œ Alertmanager 实例。 + +Prometheus Operator ä¼šæ ¹æ® Rancher UI 中编辑的资æºå’Œé…置选项的实时状æ€æ¥è‡ªåŠ¨æ›´æ–° Prometheus é…置。 + +### é»˜è®¤éƒ¨ç½²çš„èµ„æº + +默认情况下，由 [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) 项目策划的一组资æºä¼šä½œä¸º Rancher Monitoring 安装的一部分部署到你的集群上，用æ¥è®¾ç½®åŸºæœ¬çš„ Monitoring/Alerting 堆栈。 + +ä½ å¯ä»¥åœ¨ [`rancher-monitoring`](https://github.com/rancher/charts/tree/main/charts/rancher-monitoring) Helm Chart 中找到部署到你的集群以支æŒæ­¤è§£å†³æ–¹æ¡ˆçš„资æºï¼Œè¯¥ chart 密切跟踪由 Prometheus 社区维护的上游 [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm Chart，并在 [CHANGELOG.md](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/CHANGELOG.md) 中跟踪å˜æ›´ã€‚ + +### 默认 Exporter + +Monitoring V2 部署了三个默认 Exporter，它们为 Prometheus æä¾›é¢å¤–的指标æ¥å­˜å‚¨ï¼š + +1. `node-exporter`：公开 Linux 主机的硬件和æ“作系统指标。有关 `node-exporter` 的更多信æ¯ï¼Œè¯·å‚阅[上游文档](https://prometheus.io/docs/guides/node-exporter/)。 + +1. `windows-exporter`：公开 Windows 主机的硬件和æ“作系统指标（仅部署在 Windows 集群上）。有关 `windows-exporter` 的更多信æ¯ï¼Œè¯·å‚阅[上游文档](https://github.com/prometheus-community/windows_exporter)。 + +1. `kube-state-metrics`：公开跟踪 Kubernetes API 中包å«çš„资æºçŠ¶æ€çš„其他指标（例如，podã€å·¥ä½œè´Ÿè½½ç­‰ï¼‰ã€‚有关 `kube-state-metrics` 的更多信æ¯ï¼Œè¯·å‚阅[上游文档](https://github.com/kubernetes/kube-state-metrics/tree/master/docs)。 + +ServiceMonitor å’Œ PodMonitor 将按照[此定义](#定义è¦æŠ“å–的指标)æ¥æŠ“å–这些 Exporter。Prometheus 会存储这些指标，你å¯ä»¥é€šè¿‡ Prometheus çš„ UI 或 Grafana 查询结果。 + +有关记录规则ã€å‘Šè­¦è§„则和 Alertmanager 的更多信æ¯ï¼Œè¯·å‚阅[架构](#1-架构概述)。 + +### Rancher UI 中公开的组件 + +安装 monitoring 应用åŽï¼Œä½ å°†èƒ½å¤Ÿåœ¨ Rancher UI 中编辑以下组件： + +| 组件 | 组件类型 | 编辑的目的和常è§ç”¨ä¾‹ | +|--------------|------------------------|---------------------------| +| ServiceMonitor | è‡ªå®šä¹‰èµ„æº | 设置 Kubernetes Service æ¥èŽ·å–其自定义指标。自动更新 Prometheus 自定义资æºä¸­çš„抓å–é…置。 | +| PodMonitor | è‡ªå®šä¹‰èµ„æº | 设置 Kubernetes Pod æ¥èŽ·å–其自定义指标。自动更新 Prometheus 自定义资æºä¸­çš„抓å–é…置。 | +| 接收器 | é…ç½®å—（Alertmanager 的一部分） | 修改将告警å‘é€åˆ°ä»€ä¹ˆä½ç½®çš„ä¿¡æ¯ï¼ˆä¾‹å¦‚，Slackã€PagerDuty 等）以åŠå‘é€å‘Šè­¦çš„其他必è¦ä¿¡æ¯ï¼ˆä¾‹å¦‚，TL​​S è¯ä¹¦ã€ä»£ç† URL 等）。自动更新 Alertmanager 自定义资æºã€‚ | +| Route | é…ç½®å—（Alertmanager 的一部分） | 修改用于根æ®æ ‡ç­¾è¿‡æ»¤ã€æ ‡è®°å’Œåˆ†ç»„告警的路由树，并将告警å‘é€åˆ°æ‰€éœ€çš„接收器。自动更新 Alertmanager 自定义资æºã€‚ | +| PrometheusRule | è‡ªå®šä¹‰èµ„æº | 定义其他查询，这些查询能触å‘告警或定义 Prometheus TSDB 中现有的物化视图。自动更新 Prometheus 自定义资æºã€‚ | + +### PushProx + +PushProx å…许 Prometheus 跨网络边界抓å–指标，这样，用户就ä¸ç”¨å¿…须为 Kubernetes 集群中æ¯ä¸ªèŠ‚点上的内部 Kubernetes 组件公开指标端å£ã€‚ + +由于 Kubernetes 组件的指标通常暴露在集群中节点的主机网络上，PushProx 部署了一个客户端 DaemonSet，这些客户端ä½äºŽæ¯ä¸ªèŠ‚点的主机网络上，并与ä½äºŽ Kubernetes API 上的å•ä¸ªä»£ç†å»ºç«‹å‡ºç«™è¿žæŽ¥ã€‚然åŽï¼Œä½ å¯ä»¥è®© Prometheus 通过代ç†å°†æŠ“å–请求å‘é€åˆ°æ¯ä¸ªå®¢æˆ·ç«¯ï¼Œè¿™æ ·ï¼ŒPrometheus 能从内部 Kubernetes 组件抓å–指标，而ä¸éœ€è¦æ‰“开任何入站节点端å£ã€‚ + +有关更多信æ¯ï¼Œè¯·å‚阅[使用 PushProx 抓å–指标](#使用-pushprox-抓å–指标)。 + +## 5. 抓å–和公开指标 + +### 定义è¦æŠ“å–的指标 + +ServiceMonitor å’Œ PodMonitor 定义了 Prometheus è¦æŠ“å–的目标。[Prometheus 自定义资æº](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md#prometheus)告诉 Prometheus 应该使用哪个 ServiceMonitor 或 PodMonitor æ¥ç¡®å®šä»Žå“ªé‡ŒæŠ“å–指标。 + +Prometheus Operator 观察 ServiceMonitor å’Œ PodMonitor。当它观察到二者被创建或更新时，它会调用 Prometheus API æ¥æ›´æ–° Prometheus 自定义资æºä¸­çš„抓å–é…置，并使该é…置与 ServiceMonitor 或 PodMonitor 中的抓å–é…ç½®ä¿æŒåŒæ­¥ã€‚此抓å–é…置告诉 Prometheus 从哪些端点抓å–指标，以åŠå¦‚何标记这些端点的指标。 + +Prometheus ä¼šæ ¹æ® `scrape_interval`（默认为一分钟）æ¥æŠ“å–定义在抓å–é…置中的所有指标。 + +抓å–é…ç½®å¯ä»¥ä½œä¸º Prometheus 自定义资æºçš„一部分被查看，该资æºåœ¨ Rancher UI 中公开。 + +### Prometheus Operator å¦‚ä½•è®¾ç½®æŒ‡æ ‡æŠ“å– + +Prometheus Deployment 或 StatefulSet 能抓å–指标，而 Prometheus çš„é…置由 Prometheus 自定义资æºæŽ§åˆ¶ã€‚Prometheus Operator 会观察 Prometheus å’Œ Alertmanager 资æºï¼Œå½“它们被创建时，Prometheus Operator 使用用户定义的é…置，为 Prometheus 或 Alertmanager 创建一个 Deployment 或 StatefulSet。 + +如果 Prometheus Operator 观察到正在创建的 ServiceMonitorã€PodMonitor å’Œ PrometheusRule，它就知é“需è¦åœ¨ Prometheus 中更新抓å–é…置。首先，会通过更新 Prometheus çš„ Deployment 或 StatefulSet å·ä¸­çš„é…置和规则文件æ¥æ›´æ–° Prometheus。然åŽï¼Œå†è°ƒç”¨ Prometheus API æ¥åŒæ­¥æ–°é…置，从而将 Prometheus Deployment 或 StatefulSet 修改到ä½ã€‚ + +### 如何公开 Kubernetes 组件指标 + +Prometheus 从称为 [exporter](https://prometheus.io/docs/instrumenting/exporters/) çš„ deployment 中抓å–指标，exporter 以 Prometheus å¯ä»¥æŠ“å–çš„æ ¼å¼å¯¼å‡ºæ—¶é—´åºåˆ—æ•°æ®ã€‚在 Prometheus 中，时间åºåˆ—由属于相åŒæŒ‡æ ‡å’Œç›¸åŒæ ‡è®°ç»´åº¦é›†çš„时间戳值æµç»„æˆã€‚ + +### 使用 PushProx 抓å–指标 + +æŸäº›å†…部 Kubernetes 组件是通过部署在 Monitoring V2 中å为 PushProx 的代ç†æ¥æŠ“å–的。有关 PushProx 的详细信æ¯ï¼Œè¯·å‚阅[此处](#pushprox-的工作原ç†)和上é¢çš„[架构](#1-架构概述)部分。 + +### 抓å–指标 + +Prometheus 直接抓å–以下 Kubernetes 组件： + +- kubelet\* +- ingress-nginx\*\* +- coreDns/kubeDns +- kube-api-server + +\* ä½ å¯ä»¥é€‰æ‹©é€šè¿‡ `hardenedKubelet.enabled` æ¥ä½¿ç”¨ PushProx，但这ä¸æ˜¯é»˜è®¤è®¾ç½®ã€‚ + +\*\* RKE å’Œ RKE2 集群默认部署 ingress-nginx，并将其视为内部 Kubernetes 组件。 + + +### 基于 Kubernetes å‘行版抓å–指标 + +指标的抓å–æ–¹å¼æ ¹æ® Kubernetes å‘行版而有所ä¸åŒã€‚有关术语的帮助，请å‚阅[此处](#åè¯è§£é‡Š)。详情è§ä¸‹è¡¨ï¼š + +
指标如何暴露给 Prometheus
+ +| Kubernetes 组件 | RKE | RKE2 | KubeADM | K3s | +|-----|-----|-----|-----|-----| +| kube-controller-manager | rkeControllerManager.enabled | rke2ControllerManager.enabled | kubeAdmControllerManager.enabled | k3sServer.enabled | +| kube-scheduler | rkeScheduler.enabled | rke2Scheduler.enabled | kubeAdmScheduler.enabled | k3sServer.enabled | +| etcd | rkeEtcd.enabled | rke2Etcd.enabled | kubeAdmEtcd.enabled | ä¸å¯ç”¨ | +| kube-proxy | rkeProxy.enabled | rke2Proxy.enabled | kubeAdmProxy.enabled | k3sServer.enabled | +| kubelet | 收集 kubelet 直接公开的指标 | 收集 kubelet 直接公开的指标 | 收集 kubelet 直接公开的指标 | 收集 kubelet 直接公开的指标 | +| ingress-nginx* | 收集 kubelet 直接公开的指标，由 rkeIngressNginx.enabled 公开 | 收集 kubelet 直接公开的指标，由 rke2IngressNginx.enabled 公开 | ä¸å¯ç”¨ | ä¸å¯ç”¨ | +| coreDns/kubeDns | 收集 coreDns/kubeDns 直接公开的指标 | 收集 coreDns/kubeDns 直接公开的指标 | 收集 coreDns/kubeDns 直接公开的指标 | 收集 coreDns/kubeDns 直接公开的指标 | +| kube-api-server | 收集 kube-api-server 直接公开的指标 | 收集 kube-api-server 直接公开的指标 | 收集 kube-appi-server 直接公开的指标 | 收集 kube-api-server 直接公开的指标 | + +\* RKE å’Œ RKE2 集群默认部署 ingress-nginx，并将其视为内部 Kubernetes 组件。 + +### åè¯è§£é‡Š + +- **kube-scheduler**：内部 Kubernetes 组件，该组件使用 pod 规范中的信æ¯æ¥å†³å®šåœ¨å“ªä¸ªèŠ‚点上è¿è¡Œ pod。 +- **kube-controller-manager**：负责节点管ç†ï¼ˆæ£€æµ‹èŠ‚点是å¦å¤±è´¥ï¼‰ã€pod å¤åˆ¶ï¼Œä»¥åŠç«¯ç‚¹åˆ›å»ºçš„内部 Kubernetes 组件。 +- **etcd**：Kubernetes 内部组件，它是 Kubernetes 用于æŒä¹…存储所有集群信æ¯çš„分布å¼é”®/值存储。 +- **kube-proxy**：内部 Kubernetes 组件，用于监控 API server çš„ pod/service 更改以ä¿æŒç½‘络最新状æ€ã€‚ +- **kubelet**：内部 Kubernetes 组件，用于为 pod 监视节点上的 API server 并确ä¿è¿™äº› pod 能è¿è¡Œã€‚ +- **ingress-nginx**：用于 Kubernetes çš„ Ingress controller，使用 NGINX 作为åå‘代ç†å’Œè´Ÿè½½å‡è¡¡å™¨ã€‚ +- **coreDns/kubeDns**：负责 DNS 的内部 Kubernetes 组件。 +- **kube-api-server**：负责为其他 master 组件公开 API 的主è¦å†…部 Kubernetes 组件。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md new file mode 100644 index 00000000000..006df34697c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md @@ -0,0 +1,365 @@ +--- +title: PromQL 表达å¼å‚考 +--- + +本文档中的 PromQL 表达å¼å¯ç”¨äºŽé…置告警。 + +关于查询 Prometheus 时间åºåˆ—æ•°æ®åº“的更多信æ¯ï¼Œè¯·å‚阅 [Prometheus 官方文档](https://prometheus.io/docs/prometheus/latest/querying/basics/)。 + + +## 集群指标 + +### 集群 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance))` | +| æ‘˜è¦ | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])))` | + +### 集群平å‡è´Ÿè½½ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
| +| æ‘˜è¦ |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"})`
| + +### 集群内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `1 - sum(node_memory_MemAvailable_bytes) by (instance) / sum(node_memory_MemTotal_bytes) by (instance)` | +| æ‘˜è¦ | `1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes)` | + +### 集群ç£ç›˜åˆ©ç”¨çŽ‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `(sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance) - sum(node_filesystem_free_bytes{device!="rootfs"}) by (instance)) / sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance)` | +| æ‘˜è¦ | `(sum(node_filesystem_size_bytes{device!="rootfs"}) - sum(node_filesystem_free_bytes{device!="rootfs"})) / sum(node_filesystem_size_bytes{device!="rootfs"})` | + +### 集群ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(node_disk_read_bytes_total[5m])) by (instance)`
written`sum(rate(node_disk_written_bytes_total[5m])) by (instance)`
| +| æ‘˜è¦ |
read`sum(rate(node_disk_read_bytes_total[5m]))`
written`sum(rate(node_disk_written_bytes_total[5m]))`
| + +### 集群网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| æ‘˜è¦ |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +### 集群网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| æ‘˜è¦ |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +## 节点指标 + +### 节点 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `avg(irate(node_cpu_seconds_total{mode!="idle", instance=~"$instance"}[5m])) by (mode)` | +| æ‘˜è¦ | `1 - (avg(irate(node_cpu_seconds_total{mode="idle", instance=~"$instance"}[5m])))` | + +### 节点平å‡è´Ÿè½½ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| +| æ‘˜è¦ |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| + +### 节点内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"})` | +| æ‘˜è¦ | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"}) ` | + +### 节点ç£ç›˜åˆ©ç”¨çŽ‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"}) by (device)) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device)` | +| æ‘˜è¦ | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"})) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"})` | + +### 节点ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| +| æ‘˜è¦ |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| + +### 节点网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| æ‘˜è¦ |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +### 节点网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| æ‘˜è¦ |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +## ETCD 指标 + +### ETCD 有一个 Leader + +`max(etcd_server_has_leader)` + +### Leader æ›´æ¢æ¬¡æ•° + +`max(etcd_server_leader_changes_seen_total)` + +### 失败的 Proposal æ•°é‡ + +`sum(etcd_server_proposals_failed_total)` + +### GRPC 客户端æµé‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m])) by (instance)`
| +| æ‘˜è¦ |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m]))`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m]))`
| + +### 对等æµé‡ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
in`sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)`
| +| æ‘˜è¦ |
in`sum(rate(etcd_network_peer_received_bytes_total[5m]))`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m]))`
| + +### æ•°æ®åº“å¤§å° + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(etcd_debugging_mvcc_db_total_size_in_bytes) by (instance)` | +| æ‘˜è¦ | `sum(etcd_debugging_mvcc_db_total_size_in_bytes)` | + +### æ´»åŠ¨æµ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance)`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance)`
| +| æ‘˜è¦ |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})`
| + +### Raft 方案 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
applied`sum(increase(etcd_server_proposals_applied_total[5m])) by (instance)`
committed`sum(increase(etcd_server_proposals_committed_total[5m])) by (instance)`
pending`sum(increase(etcd_server_proposals_pending[5m])) by (instance)`
failed`sum(increase(etcd_server_proposals_failed_total[5m])) by (instance)`
| +| æ‘˜è¦ |
applied`sum(increase(etcd_server_proposals_applied_total[5m]))`
committed`sum(increase(etcd_server_proposals_committed_total[5m]))`
pending`sum(increase(etcd_server_proposals_pending[5m]))`
failed`sum(increase(etcd_server_proposals_failed_total[5m]))`
| + +### RPC 速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m])) by (instance)`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m])) by (instance)`
| +| æ‘˜è¦ |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m]))`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m]))`
| + +### ç£ç›˜æ“作 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m])) by (instance)`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m])) by (instance)`
| +| æ‘˜è¦ |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m]))`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m]))`
| + +### ç£ç›˜åŒæ­¥æŒç»­æ—¶é—´ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
wal`histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))`
db`histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))`
| +| æ‘˜è¦ |
wal`sum(histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)))`
db`sum(histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le)))`
| + +## Kubernetes 组件指标 + +### API Server 请求延迟 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance, verb) /1e+06` | +| æ‘˜è¦ | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance) /1e+06` | + +### API Server 请求速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(apiserver_request_count[5m])) by (instance, code)` | +| æ‘˜è¦ | `sum(rate(apiserver_request_count[5m])) by (instance)` | + +### 调度失败的 Pod + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(kube_pod_status_scheduled{condition="false"})` | +| æ‘˜è¦ | `sum(kube_pod_status_scheduled{condition="false"})` | + +### Controller Manager 队列深度 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
volumes`sum(volumes_depth) by instance`
deployment`sum(deployment_depth) by instance`
replicaset`sum(replicaset_depth) by instance`
service`sum(service_depth) by instance`
serviceaccount`sum(serviceaccount_depth) by instance`
endpoint`sum(endpoint_depth) by instance`
daemonset`sum(daemonset_depth) by instance`
statefulset`sum(statefulset_depth) by instance`
replicationmanager`sum(replicationmanager_depth) by instance`
| +| æ‘˜è¦ |
volumes`sum(volumes_depth)`
deployment`sum(deployment_depth)`
replicaset`sum(replicaset_depth)`
service`sum(service_depth)`
serviceaccount`sum(serviceaccount_depth)`
endpoint`sum(endpoint_depth)`
daemonset`sum(daemonset_depth)`
statefulset`sum(statefulset_depth)`
replicationmanager`sum(replicationmanager_depth)`
| + +### 调度器 E2E 调度延迟 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06` | +| æ‘˜è¦ | `sum(histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06)` | + +### 调度器抢å å°è¯• + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(scheduler_total_preemption_attempts[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(scheduler_total_preemption_attempts[5m]))` | + +### Ingress Controller 连接数 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"}) by (instance)`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"}) by (instance)`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"}) by (instance)`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m]))) by (instance)`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m]))) by (instance)`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m]))) by (instance)`
| +| æ‘˜è¦ |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"})`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"})`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"})`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m])))`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m])))`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m])))`
| + +### Ingress Controller 请求处ç†æ—¶é—´ + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `topk(10, histogram_quantile(0.95,sum by (le, host, path)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | +| æ‘˜è¦ | `topk(10, histogram_quantile(0.95,sum by (le, host)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | + +## Rancher Logging 指标 + + +### Fluentd 缓冲区队列速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_output_status_buffer_queue_length[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(fluentd_output_status_buffer_queue_length[5m]))` | + +### Fluentd 输入速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(fluentd_input_status_num_records_total[5m]))` | + +### Fluentd 输出错误率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_output_status_num_errors[5m])) by (type)` | +| æ‘˜è¦ | `sum(rate(fluentd_output_status_num_errors[5m]))` | + +### Fluentd 输出速率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(rate(fluentd_output_status_num_records_total[5m])) by (instance)` | +| æ‘˜è¦ | `sum(rate(fluentd_output_status_num_records_total[5m]))` | + +## 工作负载指标 + +### 工作负载 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### 工作负载内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""}) by (pod_name)` | +| æ‘˜è¦ | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""})` | + +### 工作负载网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### 工作负载网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### 工作负载ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| æ‘˜è¦ |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +## Pod 指标 + +### Pod CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
| +| æ‘˜è¦ |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
| + +### Pod 内存利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""}) by (container_name)` | +| æ‘˜è¦ | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""})` | + +### Pod 网络数æ®åŒ… + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| æ‘˜è¦ |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod 网络 I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| æ‘˜è¦ |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| 详情 |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
| +| æ‘˜è¦ |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +## 容器指标 + +### 容器 CPU 利用率 + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| cfs throttled seconds | `sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| usage seconds | `sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| system seconds | `sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| user seconds | `sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | + +### 容器内存利用率 + +`sum(container_memory_working_set_bytes{namespace="$namespace",pod_name="$podName",container_name="$containerName"})` + +### 容器ç£ç›˜ I/O + +| Catalog | è¡¨è¾¾å¼ | +| --- | --- | +| read | `sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| write | `sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md new file mode 100644 index 00000000000..dfeefda6f97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md @@ -0,0 +1,251 @@ +--- +title: RBAC +--- +本节æ述了 RBAC 在 Rancher Monitoring 中的作用。 + + +## 集群管ç†å‘˜ + +默认情况下，åªæœ‰å…·æœ‰ cluster-admin `ClusterRole` 的用户æ‰èƒ½ï¼š + +- å°† `rancher-monitoring` 应用安装到集群上，并在 Chart 部署上执行所有其他相关é…置。 + - 例如，是å¦åˆ›å»ºäº†é»˜è®¤ä»ªè¡¨æ¿ï¼Œè¦å°†å“ªäº› Exporter 部署到集群上以收集指标等。 +- 通过 Prometheus CR 在集群中创建/修改/删除 Prometheus deployment。 +- 通过 Alertmanager CR 在集群中创建/修改/删除 Alertmanager deployment。 +- 通过在命å空间中创建 ConfigMap æ¥ä¿ç•™æ–°çš„ Grafana 仪表æ¿æˆ–æ•°æ®æºã€‚ +- 通过 `cattle-monitoring-system` 命å空间中的 Secret å°†æŸäº› Prometheus 指标暴露给 HPA çš„ K8s Custom Metrics API。 + +## 具有基于 Kubernetes ClusterRole æƒé™çš„用户 + +`rancher-monitoring` Chart 安装了以下三个 `ClusterRole`。默认情况下，它们会èšåˆåˆ°ç›¸åº”çš„ K8s `ClusterRole` 中： + +| ClusterRole | èšåˆåˆ°é»˜è®¤çš„ K8s ClusterRole | +| ------------------------------| ---------------------------| +| `monitoring-admin` | `admin` | +| `monitoring-edit` | `edit` | +| `monitoring-view` | `view ` | + +这些 `ClusterRole` æ ¹æ®å¯æ‰§è¡Œçš„æ“作æ供对 Monitoring CRD çš„ä¸åŒè®¿é—®çº§åˆ«ï¼š + +| CRD (monitoring.coreos.com) | Admin | Edit | View | +| ------------------------------| ---------------------------| ---------------------------| ---------------------------| +|
  • `prometheuses`
  • `alertmanagers`
| Get, List, Watch | Get, List, Watch | Get, List, Watch | +|
  • `servicemonitors`
  • `podmonitors`
  • `prometheusrules`
| * | * | Get, List, Watch | + +在较高级别上，默认情况下会分é…以下æƒé™ã€‚ + +### 具有 Kubernetes 管ç†å‘˜/编辑æƒé™çš„用户 + +åªæœ‰å…·æœ‰ cluster-adminã€admin 或 edit çš„ `ClusterRole` å¯ä»¥ï¼š + +- 通过 ServiceMonitor å’Œ PodMonitor CR 修改 Prometheus deployment 的抓å–é…置。 +- 通过 PrometheusRules CR 修改 Prometheus deployment çš„å‘Šè­¦/记录规则。 + +### 具有 Kubernetes 查看æƒé™çš„用户 + +åªæœ‰å…·æœ‰ Kubernetes `ClusterRole` 的用户å¯ä»¥ï¼š + +- 查看集群内部署的 Prometheuses çš„é…置。 +- 查看集群内部署的 Alertmanagers çš„é…置。 +- 通过 ServiceMonitor å’Œ PodMonitor CR 查看 Prometheus deployment 的抓å–é…置。 +- 通过 PrometheusRules CR 查看 Prometheus deployment çš„å‘Šè­¦/记录规则。 + +### 其他监控角色 + +Monitoring 还会创建其他 `Role`，这些角色默认情况下ä¸ä¼šåˆ†é…给用户，而是在集群中创建。你å¯ä»¥éƒ¨ç½²å¼•ç”¨è§’色的 `RoleBinding` æ¥å°†è§’色绑定到命å空间。è¦ä½¿ç”¨ `kubectl` 而ä¸æ˜¯é€šè¿‡ Rancher æ¥å®šä¹‰ `RoleBinding`，请å•å‡»[此处](#使用-kubectl-分é…-role-å’Œ-clusterrole)。 + +管ç†å‘˜åº”使用这些角色为用户æ供更细粒度的访问æƒé™ï¼š + +| 角色 | 用途 | +| ------------------------------| ---------------------------| +| monitoring-config-admin | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-monitoring-system 命å空间中的 Secret å’Œ ConfigMap。修改此命å空间中的 Secret/ConfigMap å¯ä»¥å…许用户更改集群的 Alertmanager é…ç½®ã€Prometheus Adapter é…ç½®ã€å…¶ä»– Grafana æ•°æ®æºã€TLS 密钥等。 | +| monitoring-config-edit | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-monitoring-system 命å空间中的 Secret å’Œ ConfigMap。修改此命å空间中的 Secret/ConfigMap å¯ä»¥å…许用户更改集群的 Alertmanager é…ç½®ã€Prometheus Adapter é…ç½®ã€å…¶ä»– Grafana æ•°æ®æºã€TLS 密钥等。 | +| monitoring-config-view | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看 cattle-monitoring-system 命å空间中的 Secret å’Œ ConfigMap。查看此命å空间中的 Secret/ConfigMap å¯ä»¥å…许用户观察集群的 Alertmanager é…ç½®ã€Prometheus Adapter é…ç½®ã€å…¶ä»– Grafana æ•°æ®æºã€TLS 密钥等。 | +| monitoring-dashboard-admin | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-dashboards 命å空间中的 ConfigMap。此命å空间中的 ConfigMap 将对应于æŒä¹…化到集群上的 Grafana 仪表æ¿ã€‚ | +| monitoring-dashboard-edit | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看/编辑 cattle-dashboards 命å空间中的 ConfigMap。此命å空间中的 ConfigMap 将对应于æŒä¹…化到集群上的 Grafana 仪表æ¿ã€‚ | +| monitoring-dashboard-view | å…许管ç†å‘˜ä¸ºç”¨æˆ·åˆ†é…角色，以便查看 cattle-dashboards 命å空间中的 ConfigMap。此命å空间中的 ConfigMap 将对应于æŒä¹…化到集群上的 Grafana 仪表æ¿ã€‚ | + + +### 通过自定义角色分é…监控角色 + +管ç†å‘˜å¯ä»¥åœ¨ Rancher UI 中分é…自定义角色以管ç†ã€ç¼–辑和查看监控。这些“角色â€æ˜¯åœ¨å®‰è£… Monitoring 应用程åºæ—¶é»˜è®¤åˆ›å»ºçš„。此外，这些角色也会被部署到相应的 Kubernetes 角色：adminã€edit å’Œ view `ClusterRoles`。 + +:::note é‡è¦æ示： + +将用户添加到集群时，UI ä¸ä¼šæä¾› `monitoring-admin`ã€`monitoring-edit` å’Œ `monitoring-view` 选项。这些监控角色åªèƒ½é€šè¿‡æ‰‹åŠ¨åˆ›å»ºè‡ªå®šä¹‰è§’色æ¥åˆ†é…，该自定义角色继承 Project Owner å’Œ Project Monitoring View 角色。 + +::: + +1. 创建自定义角色： + + 1.1 å•å‡» **☰ > Users & Authentication > Roles**。 + + 1.2 选择适当的选项å¡ï¼Œä¾‹å¦‚ **Cluster** 角色。然åŽå•å‡» **Create Cluster Role**。 + + 1.3 在 **Name** 字段中，创建自定义角色，例如 `View Monitoring`ã€`Edit Monitoring` 或 `Admin Monitoring`。 + + 1.4 å•å‡» **Inherit From > Add Resource**，然åŽä»Žä¸‹æ‹‰åˆ—表中选择所需的 Kubernetes 角色。 + + 1.5 å•å‡» **Create**。 + + +2. 将自定义角色分é…给新用户： + + 2.1 å•å‡» **☰ > Cluster Management > Cluster Explore > Cluster > Cluster Members > Add**。 + + 2.2 从显示的 **Select Member** 中æœç´¢ä½ çš„新用户å。 + + 2.3 å°† **Cluster Permissions** 中的新自定义角色分é…给新用户。 + + 2.4 å•å‡» **Create**。 + +**结果**：新用户现在应该能够看到 monitoring 工具。 + +### 其他监控集群角色 + +Monitoring 还会创建其他 `ClusterRole`，这些角色默认情况下ä¸ä¼šåˆ†é…给用户，而是在集群中创建。默认情况下，这些角色ä¸ä¼šèšåˆï¼Œä½†ä½ å¯ä»¥éƒ¨ç½²å¼•ç”¨è§’色的 `RoleBinding` 或 `ClusterRoleBinding` æ¥å°†è§’色绑定到命å空间。è¦ä½¿ç”¨ `kubectl` 而ä¸æ˜¯é€šè¿‡ Rancher æ¥å®šä¹‰ `RoleBinding`，请å•å‡»[此处](#使用-kubectl-分é…-role-å’Œ-clusterrole)。 + +| 角色 | 用途 | +| ------------------------------| ---------------------------| +| monitoring-ui-view | _自 Monitoring v2 14.5.100+ èµ·å¯ç”¨_ æ­¤ ClusterRole å…许用户在 Rancher UI 中查看指定集群的指标图。这是通过授予对外部监控 UI çš„åªè¯»è®¿é—®æƒé™æ¥å®žçŽ°çš„。具有此角色的用户有æƒé™åˆ—出 Prometheusã€Alertmanager å’Œ Grafana 端点，并通过 Rancher 代ç†å‘ Prometheusã€Grafana å’Œ Alertmanager UI å‘出 GET 请求。 | + +### 使用 kubectl åˆ†é… Role å’Œ ClusterRole + +#### 使用 `kubectl create` + +一ç§æ–¹æ³•æ˜¯ä½¿ç”¨ `kubectl create clusterrolebinding` 或 `kubectl create rolebinding` æ¥åˆ†é…一个 `Role` 或 `ClusterRole`。如以下示例所示： + +- 分é…给特定用户： + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx --namespace=my-namespace + ``` + + + +- 分é…给所有ç»è¿‡èº«ä»½è®¤è¯çš„用户： + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated --namespace=my-namespace + ``` + + + + +#### 使用 YAML 文件 + +å¦ä¸€ç§æ–¹æ³•æ˜¯åœ¨ä½ åˆ›å»ºçš„ YAML 文件中定义绑定。你必须先使用 YAML 文件é…ç½® `RoleBinding` 或 `ClusterRoleBinding`。然åŽï¼Œé€šè¿‡è¿è¡Œ `kubectl apply` 命令æ¥åº”用é…置更改。 + +- **Role**：以下 YAML 文件示例å¯å¸®åŠ©ä½ åœ¨ Kubernetes 中é…ç½® `RoleBinding`。你需è¦åœ¨ä¸‹é¢å¡«å†™å称。 + +:::note + +å称区分大å°å†™ã€‚ + +::: + +```yaml +# monitoring-config-view-role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: monitoring-config-view + namespace: cattle-monitoring-system +roleRef: + kind: Role + name: monitoring-config-view + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: User + name: u-b4qkhsnliz # this can be found via `kubectl get users -A` + apiGroup: rbac.authorization.k8s.io +``` + +- **kubectl**：以下 `kubectl` 示例命令用于应用 YAML 文件中创建的绑定。请记ä½ç›¸åº”填写你的 YAML 文件å。 + ```plain + kubectl apply -f monitoring-config-view-role-binding.yaml + ``` + +## 具有 Rancher æƒé™çš„用户 + +Rancher éƒ¨ç½²çš„é»˜è®¤è§’è‰²ï¼ˆå³ cluster-ownerã€cluster-memberã€project-ownerã€project-member）ã€Kubernetes é»˜è®¤è§’è‰²ï¼Œä»¥åŠ rancher-monitoring Chart 部署的角色之间的关系如下： + +
默认 Rancher æƒé™å’Œå¯¹åº”çš„ Kubernetes ClusterRole
+ +| Rancher 角色 | Kubernetes 角色 | Monitoring ClusterRole/Role | ClusterRoleBinding/RoleBinding | +| --------- | --------- | --------- | --------- | +| cluster-owner | cluster-admin | N/A | ClusterRoleBinding | +| cluster-member | admin | monitoring-admin | ClusterRoleBinding | +| project-owner | admin | monitoring-admin | 项目命å空间中的 RoleBinding | +| project-member | edit | monitoring-edit | 项目命å空间中的 RoleBinding | + +除这些默认角色之外，你还å¯ä»¥å°†ä»¥ä¸‹çš„其他 Rancher 项目角色应用于集群æˆå‘˜ï¼Œä»¥æ供对 Monitoring 的其他访问。这些 Rancher 角色将与 Monitoring Chart 部署的 ClusterRole 相关è”： + +
éžé»˜è®¤çš„ Rancher æƒé™å’Œå¯¹åº”çš„ Kubernetes ClusterRole
+ +| Rancher 角色 | Kubernetes ClusterRole | å¯ç”¨ Rancher 版本 | å¯ç”¨ Monitoring V2 版本 | +|--------------------------|-------------------------------|-------|------| +| 查看 Monitoring\* | [monitoring-ui-view](#monitoring-ui-view) | 2.4.8+ | 9.4.204+ | + +\* 如果æŸä¸ªç”¨æˆ·ç»‘定了 Rancher çš„ **View Monitoring** 角色，该用户åªæœ‰åœ¨æœ‰ UI 链接时æ‰æœ‰æƒè®¿é—®å¤–部 Monitoring UI。è¦è®¿é—® Monitoring Pane 以获å–这些链接，用户必须是至少一个项目的项目æˆå‘˜ã€‚ + +### 2.5.x 中的差异 + +在 Rancher 2.5.x 中，分é…了 project-member 或 project-owner 角色的用户将无法访问 Prometheus 或 Grafana，因为我们仅在集群级别创建 Grafana 或 Prometheus。 + +此外，虽然 project-owner ä»ç„¶åªèƒ½æ·»åŠ é»˜è®¤åœ¨é¡¹ç›®å‘½å空间内抓å–资æºçš„ ServiceMonitor/PodMonitor，但 PrometheusRule 并ä¸å±€é™äºŽå•ä¸ªå‘½å空间/项目。因此，å³ä½¿ project-owner 无法查看/编辑/删除在项目命å空间之外创建的任何规则，project-owner 在其项目命å空间内创建的任何告警规则或记录规则都将应用于整个集群。 + +### 分é…其他访问æƒé™ + +如果 cluster-admin 想è¦ä¸ºä¸å…·æœ‰ rancher-monitoring chart 角色的用户æä¾› admin/edit 访问æƒé™ï¼Œåˆ™å­˜åœ¨ä¸‹è¡¨çš„潜在影å“： + +| CRD (monitoring.coreos.com) | 是å¦ä¼šåœ¨å‘½å空间/项目之外造æˆå½±å“ | å½±å“ | +|----------------------------| ------| ----------------------------| +| `prometheuses` | 是。该资æºå¯ä»¥ä»Žæ•´ä¸ªé›†ç¾¤ä¸­çš„任何目标中抓å–æŒ‡æ ‡ï¼ˆé™¤éž Operator 本身进行了é¢å¤–çš„é…置）。 | 用户将能够定义应在集群中创建的新集群级 Prometheus deployment çš„é…置。 | +| `alertmanagers` | å¦ | 用户将能够定义应在集群中创建的新集群级 Alertmanager deployment çš„é…置。注æ„：如果你åªæƒ³å…许用户é…置路由和接收器等设置，你应该åªæ供对 Alertmanager Config Secret 的访问æƒé™ã€‚ | +|
  • `servicemonitors`
  • `podmonitors`
| å¦ï¼ˆé»˜è®¤ï¼‰ã€‚å¯ä»¥é€šè¿‡ Prometheus CR 上的 `ignoreNamespaceSelectors` 进行é…置。 | 用户将能够通过 Prometheus，在他们被授予此æƒé™çš„命å空间内的 Service/Pod 暴露的端点上设置抓å–。 | +| `prometheusrules` | 是。PrometheusRules 是集群范围的。 | 用户将能够根æ®åœ¨æ•´ä¸ªé›†ç¾¤ä¸­æ”¶é›†çš„任何系列，在 Prometheus 上定义告警或记录规则。 | + +| K8s èµ„æº | 命å空间 | 是å¦ä¼šåœ¨å‘½å空间/项目之外造æˆå½±å“ | å½±å“ | +|----------------------------| ------| ------| ----------------------------| +|
  • `secrets`
  • `configmaps`
| `cattle-monitoring-system` | 是。此命å空间中的 Config å’Œ Secret 会影å“整个监控/å‘Šè­¦æµæ°´çº¿ã€‚ | 用户将能够创建或编辑 Secret/ ConfigMap，例如 Alertmanager Configã€Prometheus Adapter é…ç½®ã€TLS 密文ã€å…¶ä»– Grafana æ•°æ®æºç­‰ã€‚这会对所有集群监控/告警产生广泛影å“。 | +|
  • `secrets`
  • `configmaps`
| `cattle-dashboards` | 是。此命å空间中的 Config å’Œ Secret å¯ä»¥åˆ›å»ºä»ªè¡¨æ¿ï¼Œå¯¹åœ¨é›†ç¾¤çº§åˆ«æ”¶é›†çš„所有指标进行查询。 | 用户将能够创建仅ä¿ç•™æ–° Grafana 仪表æ¿çš„ Secret/ConfigMap。 | + +## Grafana çš„ RBAC + +对于ç»è¿‡ Kubernetes 认è¯å¹¶å¯ä»¥è®¿é—® Rancher Monitoring Chart 部署的 Grafana æœåŠ¡çš„任何用户，Rancher å…许他们通过 Rancher Dashboard UI 访问 Grafana。默认情况下，所有能够访问 Grafana 的用户都被赋予 [Viewer](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#viewer-role) 角色，能查看 Rancher 部署的任何默认仪表æ¿ã€‚ + +但是，如果有需è¦çš„è¯ï¼Œç”¨æˆ·å¯ä»¥é€‰æ‹©ä»¥ [Admin](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#admin-role) 身份登录 Grafana。Grafana 实例的默认 Admin 用户å和密ç æ˜¯ `admin`/`prom-operator`，但你也å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ Chart 时替æ¢å‡­è¯ã€‚ + +è¦æŸ¥çœ‹ Grafana UI，请安装 `rancher-monitoring`。然åŽï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Grafana**。 + +
Grafana 中的集群计算资æºä»ªè¡¨æ¿
+ +![Grafana 中的集群计算资æºä»ªè¡¨æ¿](/img/cluster-compute-resources-dashboard.png) + +
Grafana 中的默认仪表æ¿
+ +![Grafana 中的默认仪表æ¿](/img/grafana-default-dashboard.png) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/windows-support.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/windows-support.md new file mode 100644 index 00000000000..c8de9476221 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/monitoring-and-alerting/windows-support.md @@ -0,0 +1,48 @@ +--- +title: Monitoring V2 çš„ Windows é›†ç¾¤æ”¯æŒ +--- + +_从 v2.5.8 èµ·å¯ç”¨_ + +从 Monitoring V2 14.5.100（Rancher 2.5.8 的默认版本）开始，Monitoring V2 å¯ä»¥éƒ¨ç½²åœ¨ Windows 集群上，并将使用 [prometheus-community/windows_exporter](https://github.com/prometheus-community/windows_exporter)（旧å是 `wmi_exporter`）æ¥æŠ“å– Windows 节点的指标。 + +## 集群è¦æ±‚ + +Monitoring V2 for Windows åªèƒ½ä»Žæœ€ä½Žæ˜¯ `wins` v0.1.0 çš„ Windows 主机中抓å–指标。è¦å®Œå…¨éƒ¨ç½² Monitoring V2 for Windows，你的所有主机都必须满足此è¦æ±‚。 + +如果你在 Rancher 2.5.8 中é…置新的 RKE1 集群，你的集群应该已ç»æ»¡è¶³æ­¤è¦æ±‚。 + +### 将现有集群å‡çº§åˆ° wins v0.1.0 + +如果集群是在 Rancher 2.5.8 之å‰é…置的（å³ä½¿å½“å‰ Rancher 版本是 2.5.8），你将无法æˆåŠŸéƒ¨ç½² Monitoring V2 for Windows，除éžä½ å°†æ¯å°ä¸»æœºçš„ wins 版本å‡çº§åˆ° v0.1.0 或以上版本。 + +为了方便此次å‡çº§ï¼ŒRancher 2.5.8 å‘布了一个全新的 Helm Chart，å为 `rancher-wins-upgrader`。 + +1. 使用以下覆盖部署 `rancher-wins-upgrader`： + ```yaml + # 通过先å‰å·²åˆ—入白åå•çš„进程路径 + # æ¥å¼•å¯¼ win-upgrader 安装，这是因为正常安装路径 + # c:\etc\rancher\wins\wins-upgrade.exe 通常ä¸ä¼šè¢«åˆ—入白åå•ã€‚ + # 因此，我们使用 Monitoring V1 之å‰æ‰€ç”¨çš„ + # 已列入白åå•çš„进程路径。 + masquerade: + enabled: true + as: c:\\etc\wmi-exporter\wmi-exporter.exe + ``` + :::note éžé»˜è®¤ Windows å‰ç¼€è·¯å¾„的注æ„事项： + + - 如果你使用具有éžé»˜è®¤ `win_prefix_path` çš„ `cluster.yml` æ¥è®¾ç½® RKE 集群，你需è¦å°† `c:\\` 替æ¢ä¸ºä½ çš„å‰ç¼€è·¯å¾„字段的值æ¥ä¿®æ”¹ `masquerade.as`。 + + - 例如，如果你使用 `win_prefix_path: 'c:\host\opt\'`，则需è¦è®¾ç½®ä¸º `as: c:\host\opt\etc\wmi-exporter\wmi-exporter.exe`。 + + ::: + +2. æˆåŠŸå‡çº§æ‰€æœ‰ä¸»æœºåŽï¼Œè¯·å†æ¬¡ä½¿ç”¨é»˜è®¤å€¼éƒ¨ç½² Helm Chart，以é¿å…与以下设置å‘生冲çªï¼š + ```yaml + masquerade: + enabled: false + ``` + +**结果**：主机已准备好安装 Monitoring V2。你å¯ä»¥é€‰æ‹©å¸è½½ `rancher-wins-upgrader` Chart，或将其ä¿ç•™åœ¨é›†ç¾¤ä¸­ä»¥æ–¹ä¾¿å°†æ¥å‡çº§ã€‚ + +有关如何使用它的更多信æ¯ï¼Œè¯·å‚阅 Chart çš„ [README.md](https://github.com/rancher/wins/blob/master/charts/rancher-wins-upgrader/README.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/neuvector.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/neuvector.md new file mode 100644 index 00000000000..42a2e900c59 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/neuvector.md @@ -0,0 +1,199 @@ +--- +title: NeuVector é›†æˆ +--- + +### Rancher 中的 NeuVector é›†æˆ + +[NeuVector 5.x](https://open-docs.neuvector.com/) 是一个开æºçš„，以容器为中心的安全应用程åºï¼ŒRancher å·²é›†æˆ NeuVector。NeuVector 在è¿è¡Œæ—¶ä¸ºå…³é”®åº”用程åºå’Œæ•°æ®æ供实时的åˆè§„ã€å¯è§å’Œä¿æŠ¤åŠŸèƒ½ã€‚NeuVector æ供具有 CIS Benchmark å’Œæ¼æ´žæ‰«æ的防ç«å¢™ã€å®¹å™¨è¿›ç¨‹/文件系统监控和安全审计。有关 Rancher 安全性的更多信æ¯ï¼Œè¯·å‚阅[安全文档](../pages-for-subheaders/rancher-security.md)。 + +NeuVector å¯ä»¥é€šè¿‡ Helm Chart å¯ç”¨ã€‚ä½ å¯ä»¥åœ¨ **Apps** 或 Rancher UI 中的 **Cluster Tools** 中安装该 Chart。安装 Helm Chart åŽï¼Œç”¨æˆ·å¯ä»¥è½»æ¾åœ°[在 Rancher ä¸­éƒ¨ç½²å’Œç®¡ç† NeuVector 集群](https://open-docs.neuvector.com/deploying/rancher#deploy-and-manage-neuvector-through-rancher-apps-marketplace)。 + +### 使用 Rancher 安装 NeuVector + +Harvester Helm Chart ç”¨äºŽç®¡ç† Rancher 中 NeuVector UI 的访问，用户å¯ä»¥åœ¨ Rancher 中直接跳转，然åŽéƒ¨ç½²å’Œç®¡ç† NeuVector 集群。 + +**通过 "Apps" 导航并安装 NeuVector Chart**： + +1. 点击 **☰ > 集群管ç†**。 +1. 在 Clusters 页é¢ä¸Šï¼Œè½¬åˆ°è¦éƒ¨ç½² NeuVector 的集群，然åŽå•å‡» **Explore**。 +1. 转到 **Apps > Charts**，然åŽä»Ž Chart 仓库中安装 **NeuVector**。 +1. ä¸åŒçš„集群类型需è¦ä¸åŒçš„容器è¿è¡Œæ—¶ã€‚é…ç½® Helm Chart 值时，转到**容器è¿è¡Œæ—¶**，然åŽæ ¹æ®é›†ç¾¤ç±»åž‹é€‰æ‹©è¿è¡Œæ—¶ã€‚最åŽï¼Œå†æ¬¡å•å‡»**安装**。 + +以下是一些例å­ï¼š + +- RKE1：`docker` +- K3s å’Œ RKE2：`k3scontainerd` +- AKS：`containerd` 适用于 v1.19 åŠæ›´é«˜ç‰ˆæœ¬ +- EKS：`docker` 适用于 v1.22 åŠä»¥ä¸‹ç‰ˆæœ¬ï¼›`containerd` 适用于 v1.23 åŠæ›´é«˜ç‰ˆæœ¬ +- GKE：`containerd`（请å‚阅 [Google 文档](https://cloud.google.com/kubernetes-engine/docs/concepts/using-containerd)了解更多信æ¯ï¼‰ + + :::note + + 在安装过程中一次åªèƒ½é€‰æ‹©ä¸€ä¸ªå®¹å™¨è¿è¡Œæ—¶å¼•æ“Žã€‚ + + ::: + +**通过集群工具导航并安装 NeuVector Chart**： + +1. 点击 **☰ > 集群管ç†**。 +1. 在 Clusters 页é¢ä¸Šï¼Œè½¬åˆ°è¦éƒ¨ç½² NeuVector 的集群，然åŽå•å‡» **Explore**。 +1. 点击左侧导航æ åº•éƒ¨çš„**集群工具**。 +1. 按照上é¢çš„步骤 4 相应地选择你的容器è¿è¡Œæ—¶ï¼Œç„¶åŽå†æ¬¡å•å‡»**安装**。 + +### 从 Rancher UI 访问 NeuVector + +1. 导航到安装了 NeuVector 的集群的 Cluster Explorer。在左侧导航æ ä¸­ï¼Œå•å‡» **NeuVector**。 +1. å•å‡»å¤–部链接以转到 NeuVector UI。选择链接åŽï¼Œç”¨æˆ·å¿…须接å—`最终用户许å¯åè®®`æ‰èƒ½è®¿é—® NeuVector UI。 + +### 从 Rancher UI å¸è½½ NeuVector + +**通过 "Apps" å¸è½½**： + +1. 点击 **☰ > 集群管ç†**。 +1. 在 **Apps** 下，点击 **Installed Apps**。 +1. 在 `cattle-neuvector-system` 下，选择 NeuVector 应用程åºï¼ˆå¦‚果需è¦ï¼Œè¿˜å¯ä»¥é€‰æ‹©ç›¸å…³çš„ CRD），然åŽå•å‡»**删除**。 + +**通过集群工具å¸è½½**： + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»å±å¹•å·¦ä¸‹è§’çš„**集群工具**，然åŽå•å‡» NeuVector Chart 下方的垃圾桶图标。如果需è¦ï¼Œé€‰æ‹©`删除与此应用关è”çš„ CRD`，然åŽå•å‡»**删除**。 + +### GitHub 仓库 + +NeuVector 项目在[这里](https://github.com/neuvector/neuvector)。 + +### 文档 + +NeuVector 文档在[这里](https://open-docs.neuvector.com/)。 + +### 架构 + +NeuVector 安全解决方案包å«å››ç§ç±»åž‹çš„安全容器，分别是 Controllerã€Enforcerã€Manager å’Œ Scanner。它还æ供了一个称为 All-in-One 的特殊容器（主è¦ç”¨äºŽ Docker 原生部署），能将 Controllerã€Enforcer å’Œ Manager 功能组åˆåœ¨ä¸€ä¸ªå®¹å™¨ä¸­ã€‚此外，还有一个 Updater，è¿è¡Œè¯¥ç¨‹åºæ—¶ä¼šæ›´æ–° CVE æ•°æ®åº“。 + +- **Controller**ï¼šç®¡ç† NeuVector Enforcer 容器；为管ç†æŽ§åˆ¶å°æä¾› REST API。 +- **Enforcer**：执行安全策略。 +- **Manager**：æ供一个 web-UI å’Œ CLI 控制å°æ¥ç®¡ç† NeuVector å¹³å°ã€‚ +- **All-in-One**：包括 Controllerã€Enforcer å’Œ Manager。 +- **Scanner**：对镜åƒã€å®¹å™¨å’ŒèŠ‚点执行æ¼æ´žå’Œåˆè§„性扫æ。 +- **Updater**：更新 Neuvector çš„ CVE æ•°æ®åº“（è¿è¡Œçš„时候）；é‡æ–°éƒ¨ç½² scanner pod。 + +
**NeuVector 安全容器：**
+ +![NeuVector 安全容器](/img/neuvector-security-containers.png) + +
**NeuVector 架构：**
+ +![NeuVector 架构](/img/neuvector-architecture.png) + +è¦äº†è§£æœ‰å…³ NeuVector 架构的更多信æ¯ï¼Œè¯·å‚阅[此处](https://open-docs.neuvector.com/basics/overview#architecture)。 + +### CPU å’Œå†…å­˜åˆ†é… + +以下是默认 NeuVector Chart 安装部署的最低计算资æºæŽ¨è。请注æ„，未设置资æºé™åˆ¶ã€‚ + +| 容器 | CPU - 请求 | 内存 - 请求 | +|------------|--------|---------| +| Controller | 3（æ¯ä¸ªæŽ§åˆ¶å™¨éœ€è¦ 1GB 1vCPU） | * | +| Enforcer | 所有节点上 (500MB .5vCPU) | 1GB | +| Manager | 1 (500MB .5vCPU) | * | +| Scanner | 3 (100MB .5vCPU) | * | + +\* Controllerã€Manager å’Œ Scanner 容器åˆè®¡è‡³å°‘éœ€è¦ 1GB 内存。 + + +### å¼ºåŒ–é›†ç¾¤æ”¯æŒ - Calico å’Œ Canal + + + + +- 如果 PSP 设置为 true，则所有 NeuVector 组件都是å¯éƒ¨ç½²çš„。 + +你需è¦ä¸ºå¼ºåŒ–集群环境进行é¢å¤–çš„é…置，如下所示： + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 安装（或å‡çº§åˆ°ï¼‰NeuVector 版本 `100.0.1+up2.2.2`。 + +- 在 **编辑选项** > **其它é…ç½®**下，选中å¤é€‰æ¡†æ¥å¯ç”¨ **Pod 安全策略**（无需其他é…置）： + + ![为 RKE1 强化集群å¯ç”¨ PSP](/img/psp-nv-rke.png) + +1. 点击å³ä¸‹è§’çš„**安装**。 + + + + + +- 如果 PSP 设置为 true，则å¯ä»¥éƒ¨ç½² NeuVector 组件 Controller å’Œ Enforcer。 + + +**仅适用于 NeuVector Chart 版本 100.0.0+up2.2.0**： + +- 对于 Managerã€Scanner å’Œ Updater 组件，需è¦è¿›è¡Œé¢å¤–çš„é…置，如下所示： + +``` +kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' +``` +
+ +你需è¦ä¸ºå¼ºåŒ–集群环境进行é¢å¤–çš„é…置。 + +> **注æ„**：你必须更新 RKE2 å’Œ K3s 强化集群中的é…置，如下所示。 + +1. 点击 **☰ > 集群管ç†**。 +1. 选择你创建的集群，并点击 **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 安装（或å‡çº§åˆ°ï¼‰NeuVector 版本 `100.0.1+up2.2.2`。 + +- 在 **编辑选项** > **其它é…ç½®**下，选中å¤é€‰æ¡†æ¥å¯ç”¨ **Pod 安全策略**。请注æ„，对于 `Manager runAsUser ID`ã€`Scanner runAsUser ID` å’Œ `Updater runAsUser ID`，你还必须输入大于 `0` 的值: + + ![为 RKE2 å’Œ K3s 强化集群å¯ç”¨ PSP](/img/psp-nv-rke2.png) + +1. 点击å³ä¸‹è§’çš„**安装**。 + + + + + +### å¯ç”¨ SELinux çš„é›†ç¾¤æ”¯æŒ - Calico å’Œ Canal + +è¦åœ¨ RKE2 集群上å¯ç”¨ SELinux，请执行以下步骤： + +- 如果 PSP 设置为 true，则å¯ä»¥éƒ¨ç½² NeuVector 组件 Controller å’Œ Enforcer。 + + +**仅适用于 NeuVector Chart 版本 100.0.0+up2.2.0**： + +- 对于 Managerã€Scanner å’Œ Updater 组件，需è¦è¿›è¡Œé¢å¤–çš„é…置，如下所示： + +``` +kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' +``` + +### ç¦»çº¿çŽ¯å¢ƒä¸­çš„é›†ç¾¤æ”¯æŒ + +- 所有 NeuVector 组件都å¯éƒ¨ç½²åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­çš„集群上，无需任何é¢å¤–é…置。 + + +### 支æŒé™åˆ¶ + +* ç›®å‰ä»…支æŒç®¡ç†å‘˜å’Œé›†ç¾¤æ‰€æœ‰è€…。 + +* ä¸æ”¯æŒ Fleet 多集群部署。 + +* Windows 集群ä¸æ”¯æŒ NeuVector。 + + +### 其他é™åˆ¶ + +* ç›®å‰ï¼Œå¦‚æžœ NeuVector partner Chart 已存在，则 NeuVector 功能 Chart 的安装会失败。è¦è§£å†³æ­¤é—®é¢˜ï¼Œè¯·å¸è½½ NeuVector partner Chart 并é‡æ–°å®‰è£… NeuVector 功能 Chart。 + +* Controller 未准备好时，有å¯èƒ½æ— æ³•ä»Ž Rancher UI 访问 NeuVector UI。在此期间，Controller å°†å°è¯•é‡æ–°å¯åŠ¨ï¼Œå¹¶ä¸”需è¦å‡ åˆ†é’Ÿæ‰èƒ½è¿›å…¥ active 状æ€ã€‚ + +* 安装 NeuVector Chart 时，ä¸ä¼šé’ˆå¯¹ä¸åŒçš„集群类型自动检测容器è¿è¡Œæ—¶ã€‚è¦è§£å†³æ­¤é—®é¢˜ï¼Œä½ å¯ä»¥æ‰‹åŠ¨æŒ‡å®šè¿è¡Œæ—¶ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/opa-gatekeeper.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/opa-gatekeeper.md new file mode 100644 index 00000000000..41c91274c66 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/opa-gatekeeper.md @@ -0,0 +1,111 @@ +--- +title: OPA Gatekeeper +--- + +为了确ä¿ä¸€è‡´æ€§å’Œåˆè§„性，æ¯ä¸ªç»„织都需è¦èƒ½å¤Ÿä»¥è‡ªåŠ¨åŒ–çš„æ–¹å¼åœ¨çŽ¯å¢ƒä¸­å®šä¹‰å’Œæ‰§è¡Œç­–略。[OPA（Open Policy Agent）](https://www.openpolicyagent.org/) 是一个策略引擎，用于基于策略控制云原生环境。Rancher 支æŒåœ¨ Kubernetes 集群中å¯ç”¨ OPA Gatekeeper，并且还安装了一些内置的策略定义（也称为约æŸæ¨¡æ¿ï¼‰ã€‚ + +OPA æ供了一ç§é«˜çº§å£°æ˜Žæ€§è¯­è¨€ï¼Œå¯ä»¥è®©ä½ å°†ç­–略指定为代ç ï¼Œè¿˜èƒ½æ‰©å±•ç®€å•çš„ API，从而å‡è½»ç­–略决策的负担。 + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) 是一个æä¾› OPA å’Œ Kubernetes 集æˆçš„项目。OPA Gatekeeper æ供： + +- 一个å¯æ‰©å±•çš„å‚数化策略库。 +- 用于实例化策略库的原生 Kubernetes CRD，也称为“约æŸâ€ã€‚ +- 用于扩展策略库的原生 Kubernetes CRD，也称为“约æŸæ¨¡æ¿â€ã€‚ +- 审计功能。 + +è¦äº†è§£æ›´å¤šå…³äºŽ OPA çš„ä¿¡æ¯ï¼Œè¯·å‚阅[官方文档](https://www.openpolicyagent.org/docs/latest/)。 + +## OPA Gatekeeper 集æˆçš„å·¥ä½œåŽŸç† + +Kubernetes 支æŒé€šè¿‡å‡†å…¥æŽ§åˆ¶å™¨ï¼ˆå‡†å…¥æŽ§åˆ¶å™¨ï¼‰webhook æ¥æ‰©å±• API Server 的功能，创建ã€æ›´æ–°æˆ–删除资æºæ—¶éƒ½ä¼šè°ƒç”¨è¿™äº› webhook。Gatekeeper ä½œä¸ºéªŒè¯ webhook 安装，并执行由 Kubernetes CRD（Custom Resource Definition）定义的策略。除了使用准入控制之外，Gatekeeper 还能审计 Kubernetes 集群中的现有资æºï¼Œå¹¶å¯¹è¿å当å‰ç­–略的情况进行标记。 + +OPA Gatekeeper ç”± Rancher çš„ Helm system Chart æ供，它安装在å为 `gatekeeper-system` 的命å空间中。 + +## 在集群中å¯ç”¨ OPA Gatekeeper + +:::note + +Rancher 2.5 改进了 OPA Gatekeeper 应用。无法从 Rancher 2.4 å‡çº§åˆ° Rancher 2.5 中的新版本。如果你在 Rancher 2.4 中安装了 OPA Gatekeeper，则需è¦åœ¨æ—§ UI 中å¸è½½ OPA Gatekeeper åŠå…¶ CRD，然åŽåœ¨ Rancher 2.5 中é‡æ–°å®‰è£…它。如需å¸è½½ CRD，请在 kubectl 控制å°ä¸­è¿è¡Œ `kubectl delete crd configs.config.gatekeeper.sh constrainttemplates.templates.gatekeeper.sh` 命令。 + +::: + +:::note 先决æ¡ä»¶ï¼š + +åªæœ‰ç®¡ç†å‘˜å’Œé›†ç¾¤æ‰€æœ‰è€…æ‰èƒ½å¯ç”¨ OPA Gatekeeper。 + +::: + +ä½ å¯ä»¥åœ¨ **Apps** 页é¢å®‰è£… OPA Gatekeeper Helm Chart。 + +### å¯ç”¨ OPA Gatekeeper + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸­ï¼Œè½¬åˆ°è¦å¯ç”¨ OPA Gatekeeper 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œç‚¹å‡» **Apps**。 +1. 点击 **Charts** 并点击 **OPA Gatekeeper**。 +1. å•å‡»**安装**。 + +**结果**：已将 OPA Gatekeeper 部署到你的 Kubernetes 集群。 + +## 约æŸæ¨¡æ¿ + +[约æŸæ¨¡æ¿](https://github.com/open-policy-agent/gatekeeper#constraint-templates)是 Kubernetes 自定义资æºï¼Œç”¨äºŽå®šä¹‰è¦ç”± Gatekeeper 应用的 OPA 策略的架构和 Rego 逻辑。有关 Rego 策略语言的更多信æ¯ï¼Œè¯·å‚阅[官方文档](https://www.openpolicyagent.org/docs/latest/policy-language/)。 + +å¯ç”¨ OPA Gatekeeper åŽï¼ŒRancher 默认会安装一些模æ¿ã€‚ + +è¦åˆ—出集群中安装的约æŸæ¨¡æ¿ï¼Œè¯·è½¬åˆ° OPA Gatekeeper 下的左侧èœå•ï¼Œç„¶åŽå•å‡»**模æ¿**。 + +Rancher 还支æŒé€šè¿‡å¯¼å…¥ YAML 定义æ¥åˆ›å»ºä½ è‡ªå·±çš„约æŸæ¨¡æ¿ã€‚ + +## 创建和é…ç½®çº¦æŸ + +[约æŸ](https://github.com/open-policy-agent/gatekeeper#constraints)是 Kubernetes 自定义资æºï¼Œç”¨äºŽå®šä¹‰è¦åº”用约æŸæ¨¡æ¿çš„对象范围。约æŸæ¨¡æ¿å’Œçº¦æŸå…±åŒå®šä¹‰ä¸€ä¸ªå®Œæ•´çš„策略。 + +:::note 先决æ¡ä»¶ï¼š + +集群中已å¯ç”¨ OPA Gatekeeper。 + +::: + +è¦åˆ—出已安装的约æŸï¼Œè¯·è½¬åˆ° OPA Gatekeeper 下的左侧èœå•ï¼Œç„¶åŽå•å‡»**约æŸ**。 + +å¯ä»¥ä»Žçº¦æŸæ¨¡æ¿åˆ›å»ºæ–°çš„约æŸã€‚ + +Rancher 支æŒé€šè¿‡ä½¿ç”¨æ–¹ä¾¿çš„表å•æ¥åˆ›å»ºçº¦æŸï¼Œä½ å¯ä»¥åœ¨è¯¥è¡¨å•ä¸­è¾“å…¥å„ç§çº¦æŸå­—段。 + +**以 YAML 文件编辑**选项也å¯ä»¥ç”¨äºŽé…置约æŸçš„ YAML 定义。 + +### 使 Rancher çš„ System 命å空间ä¸å—çº¦æŸ + +创建约æŸæ—¶ï¼Œè¯·ç¡®ä¿è¯¥çº¦æŸä¸åº”用于任何 Rancher 或 Kubernetes System 命å空间。如果ä¸æŽ’除 System 命å空间，则å¯èƒ½ä¼šå‡ºçŽ° system 命å空间下的许多资æºè¢«æ ‡è®°ä¸ºè¿å约æŸã€‚ + +è¦è®©çº¦æŸä»…é™åˆ¶ç”¨æˆ·å‘½å空间，请在约æŸçš„**匹é…**字段下指定这些命å空间。 + +此外，该约æŸå¯èƒ½ä¼šå¹²æ‰°å…¶ä»– Rancher 功能并拒ç»éƒ¨ç½²ç³»ç»Ÿå·¥ä½œè´Ÿè½½ã€‚为é¿å…è¿™ç§æƒ…况，请从你的约æŸä¸­æŽ’除所有 Rancher 特定的命å空间。 + +## åœ¨é›†ç¾¤ä¸­å®žæ–½çº¦æŸ + +如果**执行动作**为 **Deny**，约æŸä¼šç«‹å³å¯ç”¨ï¼Œå¹¶æ‹’ç»ä»»ä½•è¿å策略的请求。默认情况下，执行的值为 **Deny**。 + +如果**执行动作** 为 **Dryrun**，è¿å策略的资æºä»…会记录在约æŸçš„状æ€å­—段中。 + +è¦å¼ºåˆ¶æ‰§è¡Œçº¦æŸï¼Œè¯·ä½¿ç”¨è¡¨å•åˆ›å»ºçº¦æŸã€‚在**执行动作**字段中，选择 **Deny**。 + +## 集群中的审计和è¿è§„ + +OPA Gatekeeper è¿è¡Œå®šæœŸå®¡è®¡ï¼Œä»¥æ£€æŸ¥çŽ°æœ‰èµ„æºæ˜¯å¦è¿å强制执行的约æŸã€‚ä½ å¯ä»¥åœ¨å®‰è£… Gatekeeper æ—¶é…置审计间隔（默认 300 秒）。 + +Gatekeeper 页é¢ä¸Šåˆ—出了è¿å已定义的约æŸçš„情况。 + +此外，你也å¯ä»¥åœ¨**约æŸ**页é¢ä¸­æ‰¾åˆ°è¿å约æŸçš„æ•°é‡ã€‚ + +æ¯ä¸ªçº¦æŸçš„详细信æ¯è§†å›¾åˆ—出了è¿å约æŸçš„资æºçš„ä¿¡æ¯ã€‚ + +## ç¦ç”¨ Gatekeeper + +1. 导航到集群的仪表æ¿è§†å›¾ã€‚ +1. 在左侧èœå•ä¸­ï¼Œå±•å¼€é›†ç¾¤èœå•å¹¶å•å‡» **OPA Gatekeeper**。 +1. å•å‡» **â‹® > ç¦ç”¨**。 + +**结果**：ç¦ç”¨ OPA Gatekeeper åŽï¼Œæ‰€æœ‰çº¦æŸæ¨¡æ¿å’Œçº¦æŸä¹Ÿå°†è¢«åˆ é™¤ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/rancher-extensions.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/rancher-extensions.md new file mode 100644 index 00000000000..0f6de8e5900 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/integrations-in-rancher/rancher-extensions.md @@ -0,0 +1,84 @@ +--- +title: Rancher 扩展 +--- + +Rancher v2.7.0 引入了**扩展（Extension）**的新功能。扩展å…许用户ã€å¼€å‘人员ã€åˆä½œä¼™ä¼´å’Œå®¢æˆ·æ‰©å±•å’Œå¢žå¼º Rancher UI。此外，用户å¯ä»¥ç‹¬ç«‹äºŽ Rancher 版本对其 UI 功能进行更改和增强。有了扩展，用户能够在 Rancher 之上进行构建，从而更好地根æ®çŽ¯å¢ƒè¿›è¡Œå®šåˆ¶ã€‚用户还å¯ä»¥æ›´æ–°åˆ°æ–°ç‰ˆæœ¬ä»¥åŠå›žæ»šåˆ°ä»¥å‰çš„版本。 + +扩展是åªèƒ½åœ¨é›†ç¾¤ä¸­å®‰è£…一次的 Helm Chart。因此，这些 Chart å·²ç»è¿‡ç®€åŒ–，并与 **Apps** 下列出的常规 Helm Chart 分开。 + +内置的 Rancher 扩展示例包括 Fleetã€Explorer å’Œ Harvester。使用å¯æ‰‹åŠ¨æ·»åŠ çš„ Extensions API 的其他扩展示例包括 Kubewarden å’Œ Elemental。 + +## 先决æ¡ä»¶ + +> 你必须以管ç†å‘˜èº«ä»½ç™»å½•æ‰èƒ½æŸ¥çœ‹æ‰©å±•ç®¡ç†é¡µé¢å¹¶ä¸Žä¹‹äº¤äº’。 + +## 安装扩展 + +1. 点击 **Configuration** 下的 **☰ > Extensions**。 + +2. 如果它尚未安装到 **Apps** 中，则你必须通过å•å‡» **Enable** 按钮å¯ç”¨æ‰©å±• operator。 + + - 如果ä¸æ˜¯ç¦»çº¿å®‰è£…环境，请å•å‡» **OK** 添加 Rancher 扩展仓库。å¦åˆ™ï¼Œå–消选中å¤é€‰æ¡†å¹¶å•å‡» **OK**。 + + ![Rancher extension repository](/img/add-rancher-extension-repo.png) + +3. 在 **Extensions** 页é¢ä¸Šï¼Œå•å‡» **Available** 选项å¡é€‰æ‹©è¦å®‰è£…的扩展。 + +:::info + +在 v2.7.0 中，**Available** 选项å¡ä¸‹ä¸ä¼šæ˜¾ç¤ºå†…置扩展。因此，你需è¦æ‰‹åŠ¨æ·»åŠ æ‰€éœ€çš„仓库以安装扩展。一旦这些扩展å¯ç”¨äº†ï¼Œæˆ‘们将åŒæ­¥ç¤¾åŒºã€‚ + +::: +
+ +4. 如果没有显示å¯ç”¨çš„扩展，你å¯ä»¥æ‰‹åŠ¨æ·»åŠ ä»“库，如下所示： + + 4.1. 在å±å¹•å³ä¸Šè§’，点击 **â‹® > Manage Repositories > Create**。 + + 4.2. 添加所需的仓库å称，确ä¿æŒ‡å®šäº† Git 仓库 URL å’Œ Git 分支。 + + 4.3. å†æ¬¡ç‚¹å‡»å³ä¸‹æ–¹çš„ **Create**。 + + ![Manage repositories](/img/manage-repos.png) + +5. 在 **Available** 选项å¡ä¸‹ï¼Œå•å‡»æ‰€éœ€æ‰©å±•å’Œç‰ˆæœ¬ä¸Šçš„ **Install** 按钮，如下例所示。请注æ„，如果扩展程åºå¯ç”¨ï¼Œ**Update** 按钮将出现在该扩展上，因此你å¯ä»¥è½»æ¾æ›´æ–°æ‰©å±•ã€‚ + + ![Install Kubewarden](/img/install-kubewarden.png) + +6. 点击 **Reload** 按钮，该按钮将在你的扩展æˆåŠŸå®‰è£…åŽå‡ºçŽ°ã€‚请注æ„，对于刚刚安装扩展的登录用户而言，**除éž**他们é‡æ–°åŠ è½½é¡µé¢ï¼Œå¦åˆ™ä»–们ä¸ä¼šçœ‹åˆ° UI çš„å˜åŒ–。 + + ![Reload button](/img/reload-button.png) + +## å¸è½½æ‰©å±• + +ä½ å¯ä»¥é€šè¿‡ä¸¤ç§æ–¹å¼å¸è½½æˆ–ç¦ç”¨æ‰©å±•ï¼š + +1. 在 **Installed** 选项å¡ä¸‹ï¼Œå•å‡»è¦åˆ é™¤çš„扩展上的 **Uninstall** 按钮。 + + ![Uninstall extensions](/img/uninstall-extension.png) + +1. 在扩展管ç†é¡µé¢ï¼Œç‚¹å‡» **â‹® > Disable Extension Support**。这将ç¦ç”¨æ‰€æœ‰å·²å®‰è£…的扩展。 + + ![Disable extensions](/img/disable-extension-support.png) + +:::caution + +你必须在ç¦ç”¨æ‰©å±•åŽé‡æ–°åŠ è½½é¡µé¢ï¼Œå¦åˆ™å¯èƒ½ä¼šå‡ºçŽ°æ˜¾ç¤ºé—®é¢˜ã€‚ + +::: + +## 回滚扩展 + +在 **Installed** 选项å¡ä¸‹ï¼Œå•å‡»è¦å›žæ»šçš„扩展上的 **Rollback** 按钮。 + +![Roll back extensions](/img/roll-back-extension.png) + +:::caution + +回滚扩展åŽå¿…é¡»é‡æ–°åŠ è½½é¡µé¢ï¼Œå¦åˆ™å¯èƒ½ä¼šå‡ºçŽ°æ˜¾ç¤ºé—®é¢˜ã€‚ + +::: + +## å¼€å‘扩展 + +è¦äº†è§£å¦‚何开å‘你自己的扩展，请å‚阅官方[入门指å—](https://rancher.github.io/dashboard/extensions/extensions-getting-started)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-provisioning-drivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-provisioning-drivers.md new file mode 100644 index 00000000000..65868b38bd3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-provisioning-drivers.md @@ -0,0 +1,47 @@ +--- +title: é…置驱动 +--- + +使用 Rancher 中的驱动，你å¯ä»¥ç®¡ç†å¯ä»¥ä½¿ç”¨å“ªäº›ä¾›åº”商æ¥éƒ¨ç½²[托管的 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)或[云æœåŠ¡å™¨èŠ‚点](use-new-nodes-in-an-infra-provider.md)，以å…许 Rancher éƒ¨ç½²å’Œç®¡ç† Kubernetes。 + +### Rancher 驱动 + +ä½ å¯ä»¥å¯ç”¨æˆ–ç¦ç”¨ Rancher 中内置的驱动。如果相关驱动 Rancher 尚未实现，你å¯ä»¥æ·»åŠ è‡ªå·±çš„驱动。 + +Rancher 中有两ç§ç±»åž‹çš„驱动： + +* [集群驱动](#集群驱动) +* [主机驱动](#主机驱动) + +### 集群驱动 + +集群驱动用于é…ç½®[托管的 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)，例如 GKEã€EKSã€AKS 等。创建集群时å¯ä»¥æ˜¾ç¤ºçš„集群驱动，是由集群驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 集群驱动将显示为为托管 Kubernetes 集群创建集群的选项。默认情况下，Rancher 与几个现有的集群驱动打包在一起，但你也å¯ä»¥åˆ›å»ºè‡ªå®šä¹‰é›†ç¾¤é©±åŠ¨å¹¶æ·»åŠ åˆ° Rancher。 + +默认情况下，Rancher 已激活多个托管 Kubernetes 云æ供商，包括： + +* [Amazon EKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +* [Google GKE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +* [Azure AKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) + +还有几个托管的 Kubernetes 云æ供商是默认ç¦ç”¨çš„，但也打包在 Rancher 中： + +* [Alibaba ACK](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +* [Huawei CCE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) +* [Tencent](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) + +### 主机驱动 + +主机驱动用于é…置主机，Rancher 使用这些主机å¯åŠ¨å’Œç®¡ç† Kubernetes 集群。主机驱动与 [Docker Machine 驱动](https://docs.docker.com/machine/drivers/)相åŒã€‚创建主机模æ¿æ—¶å¯ä»¥æ˜¾ç¤ºçš„主机驱动，是由主机驱动的状æ€å®šä¹‰çš„。åªæœ‰ `active` 主机驱动将显示为创建节点模æ¿çš„选项。默认情况下，Rancher 与许多现有的 Docker Machine 驱动打包在一起，但你也å¯ä»¥åˆ›å»ºè‡ªå®šä¹‰ä¸»æœºé©±åŠ¨å¹¶æ·»åŠ åˆ° Rancher。 + +如果你ä¸æƒ³å‘用户显示特定的主机驱动，则需è¦åœç”¨è¿™äº›ä¸»æœºé©±åŠ¨ã€‚ + +Rancher 支æŒå‡ å®¶ä¸»è¦çš„云æ供商，但默认情况下，这些主机驱动处于 active 状æ€å¹¶å¯ä¾›éƒ¨ç½²ï¼š + +* [Amazon EC2](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) +* [Azure](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md) +* [Digital Ocean](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md) +* [vSphere](vsphere.md) + +还有其他几个默认ç¦ç”¨çš„主机驱动，但打包在 Rancher 中： + +* [Harvester](../integrations-in-rancher/harvester.md#harvester-主机驱动) - 在 Rancher 2.6.1 中å¯ç”¨ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-rke1-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-rke1-templates.md new file mode 100644 index 00000000000..3f4012ac5dd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-rke1-templates.md @@ -0,0 +1,126 @@ +--- +title: RKE æ¨¡æ¿ +--- + +RKE 模æ¿æ—¨åœ¨è®© DevOps 和安全团队标准化和简化 Kubernetes 集群创建的æµç¨‹ã€‚ + +RKE 的全称是 [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/)，它是 Rancher 用æ¥é…ç½® Kubernetes 集群的工具。 + +éšç€ Kubernetes 越æ¥è¶Šå—欢迎，管ç†æ›´å¤šå°åž‹é›†ç¾¤é€æ¸æˆä¸ºè¶‹åŠ¿ã€‚如果你想è¦åˆ›å»ºå¤§é‡é›†ç¾¤ï¼Œå¯¹é›†ç¾¤è¿›è¡Œä¸€è‡´ç®¡ç†å°¤ä¸ºé‡è¦ã€‚多集群管ç†é¢ä¸´ç€å®‰å…¨å’Œé™„件é…置执行的挑战，在将集群移交给最终用户之å‰ï¼Œè¿™äº›é…置需è¦æ ‡å‡†åŒ–。 + +RKE 模æ¿æœ‰åŠ©äºŽæ ‡å‡†åŒ–这些é…置。无论是使用 Rancher UIã€Rancher API 还是自动化æµç¨‹åˆ›å»ºçš„集群，Rancher 都将ä¿è¯ä»Ž RKE 集群模æ¿åˆ›å»ºçš„æ¯ä¸ªé›†ç¾¤åœ¨ç”Ÿæˆæ–¹å¼ä¸Šæ˜¯ä¸€è‡´çš„。 + +管ç†å‘˜å¯ä»¥æŽ§åˆ¶æœ€ç»ˆç”¨æˆ·èƒ½æ›´æ”¹çš„集群选项。RKE 模æ¿è¿˜å¯ä»¥ä¸Žç‰¹å®šçš„用户和组共享，以便管ç†å‘˜å¯ä»¥ä¸ºä¸åŒçš„用户集创建ä¸åŒçš„ RKE 模æ¿ã€‚ + +如果集群是使用 RKE 模æ¿åˆ›å»ºçš„，则ä¸èƒ½è®©é›†ç¾¤ä½¿ç”¨å¦ä¸€ä¸ª RKE 模æ¿ã€‚ä½ åªèƒ½å°†é›†ç¾¤æ›´æ–°ä¸ºåŒä¸€æ¨¡æ¿çš„新版本。 + +ä½ å¯ä»¥[将现有集群的é…ç½®ä¿å­˜ä¸º RKE 模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这样，åªæœ‰æ¨¡æ¿æ›´æ–°åŽæ‰èƒ½æ›´æ”¹é›†ç¾¤çš„设置。新模æ¿è¿˜å¯ç”¨äºŽå¯åŠ¨æ–°é›†ç¾¤ã€‚ + +RKE 模æ¿çš„核心功能å…许 DevOps 和安全团队： + +- 标准化集群é…置并确ä¿æŒ‰ç…§æœ€ä½³å®žè·µåˆ›å»º Rancher é…置的集群 +- é…置集群时，防止用户åšå‡ºä¸æ˜Žæ™ºçš„选择 +- 与ä¸åŒçš„用户和组共享ä¸åŒçš„æ¨¡æ¿ +- 将模æ¿çš„所有æƒå§”托给å—信任的用户进行更改 +- 控制哪些用户å¯ä»¥åˆ›å»ºæ¨¡æ¿ +- è¦æ±‚用户使用模æ¿æ¥åˆ›å»ºé›†ç¾¤ + +## å¯é…置的设置 + +RKE 模æ¿å¯ä»¥åœ¨ Rancher UI 中创建或以 YAML æ ¼å¼å®šä¹‰ã€‚当你使用 Rancher 从基础设施æ供商é…置自定义节点或一般节点时，它们å¯ä»¥æŒ‡å®šä¸ºç›¸åŒçš„å‚数： + +- 云æ供商选项 +- Pod 安全选项 +- 网络æ供商 +- Ingress Controller +- 网络安全é…ç½® +- 网络æ’件 +- ç§æœ‰é•œåƒä»“库 URL å’Œå‡­è¯ +- 附加组件 +- Kubernetes 选项，包括 kube-apiã€kube-controllerã€kubelet å’ŒæœåŠ¡ç­‰ Kubernetes 组件的é…ç½® + +RKE 模æ¿çš„[附加组件](#附加组件)的功能特别强大，因为它å…许多ç§è‡ªå®šä¹‰é€‰é¡¹ã€‚ + +## RKE 模æ¿çš„范围 + +Rancher é…ç½®çš„é›†ç¾¤æ”¯æŒ RKE 模æ¿ã€‚模æ¿å¯ç”¨äºŽé…置自定义集群或由基础设施æ供商å¯åŠ¨çš„集群。 + +RKE 模æ¿ç”¨äºŽå®šä¹‰ Kubernetes å’Œ Rancher 设置。节点模æ¿è´Ÿè´£é…置节点。有关如何将 RKE 模æ¿ä¸Žç¡¬ä»¶ç»“åˆä½¿ç”¨çš„å‚考，请å‚阅 [RKE 模æ¿å’Œç¡¬ä»¶](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md)。 + +å¯ä»¥ä»Žå¤´å¼€å§‹åˆ›å»º RKE 模æ¿æ¥é¢„先定义集群é…置。它们å¯ä»¥ç”¨äºŽå¯åŠ¨æ–°é›†ç¾¤ï¼Œä¹Ÿå¯ä»¥ä»ŽçŽ°æœ‰çš„ RKE 集群导出模æ¿ã€‚ + +现有集群的设置å¯ä»¥[ä¿å­˜ä¸º RKE 模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这会创建一个新模æ¿å¹¶å°†é›†ç¾¤è®¾ç½®ç»‘定到该模æ¿ã€‚这样，集群åªæœ‰åœ¨[模æ¿æ›´æ–°](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#更新模æ¿)的情况下æ‰èƒ½[使用新版本的模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)进行å‡çº§ã€‚新模æ¿ä¹Ÿå¯ä»¥ç”¨æ¥åˆ›å»ºæ–°é›†ç¾¤ã€‚ + + +## 示例场景 +如果一个组织åŒæ—¶æ‹¥æœ‰æ™®é€šå’Œé«˜çº§ Rancher 用户，管ç†å‘˜å¯èƒ½å¸Œæœ›ä¸ºé«˜çº§ç”¨æˆ·æ供更多用于集群创建的选项，并é™åˆ¶æ™®é€šç”¨æˆ·çš„选项。 + +这些[示例场景](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md)æ述组织如何使用模æ¿æ¥æ ‡å‡†åŒ–集群创建。 + +示例场景包括： + +- **强制执行模æ¿**：如果希望所有 Rancher é…置的新集群都具有æŸäº›è®¾ç½®ï¼Œç®¡ç†å‘˜å¯èƒ½æƒ³è¦[为æ¯ä¸ªç”¨æˆ·å¼ºåˆ¶æ‰§è¡Œä¸€é¡¹æˆ–多项模æ¿è®¾ç½®](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#强制执行模æ¿è®¾ç½®)。 +- **与ä¸åŒçš„用户共享ä¸åŒçš„模æ¿**：管ç†å‘˜å¯ä»¥ä¸º[普通用户和高级用户æä¾›ä¸åŒçš„模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#普通用户和高级用户模æ¿)。这样，普通用户会有更多é™åˆ¶é€‰é¡¹ï¼Œè€Œé«˜çº§ç”¨æˆ·åœ¨åˆ›å»ºé›†ç¾¤æ—¶å¯ä»¥ä½¿ç”¨æ›´å¤šé€‰é¡¹ã€‚ +- **更新模æ¿è®¾ç½®**：如果组织的安全和 DevOps 团队决定将最佳实践嵌入到新集群所需的设置中，这些最佳实践å¯èƒ½ä¼šéšç€æ—¶é—´è€Œæ”¹å˜ã€‚如果最佳实践å‘生å˜åŒ–，[å¯ä»¥å°†æ¨¡æ¿æ›´æ–°ä¸ºæ–°ç‰ˆæœ¬](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#更新模æ¿å’Œé›†ç¾¤)，这样，使用模æ¿åˆ›å»ºçš„集群å¯ä»¥[å‡çº§åˆ°æ¨¡æ¿çš„新版本](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#å‡çº§é›†ç¾¤ä»¥ä½¿ç”¨æ–°çš„模æ¿ä¿®è®¢ç‰ˆ)。 +- **共享模æ¿çš„所有æƒ**：当模æ¿æ‰€æœ‰è€…ä¸å†æƒ³è¦ç»´æŠ¤æ¨¡æ¿æˆ–想è¦å…±äº«æ¨¡æ¿çš„所有æƒæ—¶ï¼Œæ­¤æ–¹æ¡ˆæ述了如何[共享模æ¿æ‰€æœ‰æƒ](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#å…许其他用户控制和共享模æ¿)。 + +## 模æ¿ç®¡ç† + +创建 RKE 模æ¿æ—¶ï¼Œå¯ä»¥åœ¨ Rancher UI 中的**集群管ç†**下的 **RKE 模æ¿**中使用模æ¿ã€‚创建模æ¿åŽï¼Œä½ å°†æˆä¸ºæ¨¡æ¿æ‰€æœ‰è€…，这将授予你修改和共享模æ¿çš„æƒé™ã€‚ä½ å¯ä»¥ä¸Žç‰¹å®šç”¨æˆ·æˆ–组共享 RKE 模æ¿ï¼Œä¹Ÿå¯ä»¥å…¬å¼€æ¨¡æ¿ã€‚ + +管ç†å‘˜å¯ä»¥å¼€å¯æ¨¡æ¿å¼ºåˆ¶æ‰§è¡Œï¼Œè¦æ±‚用户在创建集群时始终使用 RKE 模æ¿ã€‚这使管ç†å‘˜å¯ä»¥ä¿è¯ Rancher 总是创建指定é…置的集群。 + +RKE 模æ¿æ›´æ–°é€šè¿‡ä¿®è®¢ç³»ç»Ÿå¤„ç†ã€‚如果è¦æ›´æ”¹æˆ–更新模æ¿ï¼Œè¯·åˆ›å»ºæ¨¡æ¿çš„新版本。然åŽï¼Œå¯ä»¥å°†ä½¿ç”¨æ—§ç‰ˆæœ¬æ¨¡æ¿åˆ›å»ºçš„集群å‡çº§åˆ°æ–°æ¨¡æ¿ä¿®è®¢ç‰ˆã€‚ + +在 RKE 模æ¿ä¸­ï¼Œæ¨¡æ¿æ‰€æœ‰è€…å¯ä»¥é™åˆ¶è®¾ç½®çš„内容，也å¯ä»¥æ‰“开设置以供最终用户选择值。它们的差别体现在，创建模æ¿æ—¶ï¼ŒRancher UI 中的æ¯ä¸ªè®¾ç½®ä¸Šçš„**å…许用户覆盖**标示。 + +对于无法覆盖的设置，最终用户将无法直接编辑它们。为了让用户使用这些设置的ä¸åŒé€‰é¡¹ï¼ŒRKE 模æ¿æ‰€æœ‰è€…需è¦åˆ›å»º RKE 模æ¿çš„新版本，这将å…许用户å‡çº§å’Œæ›´æ”¹è¯¥é€‰é¡¹ã€‚ + +本节中的文件解释了 RKE 模æ¿ç®¡ç†çš„细节： + +- [获å–创建模æ¿çš„æƒé™](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md) +- [创建和修改模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md) +- [强制执行模æ¿è®¾ç½®](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md#强制新集群使用-rke-模æ¿) +- [覆盖模æ¿è®¾ç½®](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md) +- [与集群创建者共享模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#与特定用户或组共享模æ¿) +- [共享模æ¿çš„所有æƒ](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#共享模æ¿æ‰€æœ‰æƒ) + +ä½ å¯ä»¥å‚è§æ­¤[模æ¿çš„示例 YAML 文件](../reference-guides/rke1-template-example-yaml.md)作为å‚考。 + +## åº”ç”¨æ¨¡æ¿ + +ä½ å¯ä»¥ä½¿ç”¨ä½ è‡ªå·±åˆ›å»ºçš„模æ¿æ¥[创建集群](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#使用-rke-模æ¿åˆ›å»ºé›†ç¾¤)，也å¯ä»¥ä½¿ç”¨[与你共享的模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md)æ¥åˆ›å»ºé›†ç¾¤ã€‚ + +如果 RKE 模æ¿æ‰€æœ‰è€…创建了模æ¿çš„新版本，你å¯ä»¥[将你的集群å‡çº§åˆ°è¯¥ç‰ˆæœ¬](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#更新使用-rke-模æ¿åˆ›å»ºçš„集群)。 + +å¯ä»¥ä»Žå¤´å¼€å§‹åˆ›å»º RKE 模æ¿æ¥é¢„先定义集群é…置。它们å¯ä»¥ç”¨äºŽå¯åŠ¨æ–°é›†ç¾¤ï¼Œä¹Ÿå¯ä»¥ä»ŽçŽ°æœ‰çš„ RKE 集群导出模æ¿ã€‚ + +ä½ å¯ä»¥[将现有集群的é…ç½®ä¿å­˜ä¸º RKE 模æ¿](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#将现有集群转æ¢ä¸ºä½¿ç”¨-rke-模æ¿)。这样，åªæœ‰æ¨¡æ¿æ›´æ–°åŽæ‰èƒ½æ›´æ”¹é›†ç¾¤çš„设置。 + +## 标准化硬件 + +RKE 模æ¿çš„目的是标准化 Kubernetes å’Œ Rancher 设置。如果你还想标准化你的基础设施，一个选择是将 RKE 模æ¿ä¸Ž[其他工具](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md)一起使用。 + +å¦ä¸€ç§é€‰æ‹©æ˜¯ä½¿ç”¨åŒ…å«èŠ‚点池é…置选项，但ä¸å¼ºåˆ¶æ‰§è¡Œé…置的[集群模æ¿](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md)。 + +## YAML 定制 + +如果将 RKE 模æ¿å®šä¹‰ä¸º YAML 文件，则å¯ä»¥ä¿®æ”¹æ­¤[示例 RKE æ¨¡æ¿ YAML](../reference-guides/rke1-template-example-yaml.md)。RKE 模æ¿ä¸­çš„ YAML 使用了 Rancher 在创建 RKE 集群时使用的相åŒè‡ªå®šä¹‰è®¾ç½®ã€‚但由于 YAML è¦åœ¨ Rancher é…置的集群中使用，因此需è¦å°† RKE 模æ¿è‡ªå®šä¹‰é¡¹åµŒå¥—在 YAML 中的 `rancher_kubernetes_engine_config` å‚数下。 + +RKE 文档也æä¾›[注释的](https://rancher.com/docs/rke/latest/en/example-yamls/) `cluster.yml` 文件供你å‚考。 + +有关å¯ç”¨é€‰é¡¹çš„更多信æ¯ï¼Œè¯·å‚阅[集群é…ç½®](https://rancher.com/docs/rke/latest/en/config-options/)上的 RKE 文档。 + +### 附加组件 + +RKE 模æ¿é…置文件的附加组件部分的工作方å¼ä¸Ž[集群é…置文件的附加组件部分](https://rancher.com/docs/rke/latest/en/config-options/add-ons/)相åŒã€‚ + +用户定义的附加组件指令å…许你调用和下拉 Kubernetes 清å•æˆ–将它们直接内è”。如果这些 YAML 清å•åŒ…括在 RKE 模æ¿ä¸­ï¼ŒRancher 将在集群中部署这些 YAML 文件。 + +ä½ å¯ä»¥ä½¿ç”¨é™„加组件执行以下æ“作： + +- å¯åŠ¨ Kubernetes 集群åŽï¼Œåœ¨é›†ç¾¤ä¸Šå®‰è£…应用 +- 在使用 Kubernetes Daemonset 部署的节点上安装æ’件 +- 自动设置命å空间ã€ServiceAccount 或角色绑定 + +RKE 模æ¿é…置必须嵌套在 `rancher_kubernetes_engine_config` å‚数中。è¦è®¾ç½®é™„加组件，在创建模æ¿æ—¶å•å‡»**以 YAML 文件编辑**。然åŽä½¿ç”¨ `addons` 指令添加清å•ï¼Œæˆ–使用 `addons_include` 指令设置哪些 YAML 文件å¯ç”¨äºŽé™„加组件。有关自定义附加组件的更多信æ¯ï¼Œè¯·å‚è§[用户自定义附加组件文档](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-the-api.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-the-api.md new file mode 100644 index 00000000000..db0ad233862 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/about-the-api.md @@ -0,0 +1,80 @@ +--- +title: API +--- + +## 如何使用 API + +API 有自己的用户界é¢ï¼Œä½ å¯ä»¥ä»Ž Web æµè§ˆå™¨è®¿é—®å®ƒã€‚这是查看资æºã€æ‰§è¡Œæ“作以åŠæŸ¥çœ‹ç­‰æ•ˆ cURL 或 HTTP 请求和å“应的一ç§ç®€å•çš„方法。è¦è®¿é—®å®ƒï¼š + + + + +1. å•å‡»å³ä¸Šè§’的用户头åƒã€‚ +1. å•å‡»**è´¦å· & API 密钥**。 +1. 在 **API 密钥**下，找到 **API 端点**字段并å•å‡»é“¾æŽ¥ã€‚该链接类似于 `https:///v3`，其中 `` 是 Rancher deployment 的完全é™å®šåŸŸå。 + + + + +转到ä½äºŽ `https:///v3` çš„ URL 端点，其中 `` 是你的 Rancher deployment 的完全é™å®šåŸŸå。 + + + + +## èº«ä»½éªŒè¯ + +API 请求必须包å«èº«ä»½éªŒè¯ä¿¡æ¯ã€‚身份验è¯æ˜¯é€šè¿‡ [API 密钥](../reference-guides/user-settings/api-keys.md)使用 HTTP 基本身份验è¯å®Œæˆçš„。API 密钥å¯ä»¥åˆ›å»ºæ–°é›†ç¾¤å¹¶é€šè¿‡ `/v3/clusters/` 访问多个集群。[集群和项目角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)会应用于这些键，并é™åˆ¶è´¦å·å¯ä»¥æŸ¥çœ‹çš„集群和项目以åŠå¯ä»¥æ‰§è¡Œçš„æ“作。 + +默认情况下，æŸäº›é›†ç¾¤çº§åˆ«çš„ API 令牌是使用无é™æœŸ TTL（`ttl=0`）生æˆçš„。æ¢è¨€ä¹‹ï¼Œé™¤éžä½ è®©ä»¤ç‰Œå¤±æ•ˆï¼Œå¦åˆ™ `ttl=0` çš„ API 令牌永远ä¸ä¼šè¿‡æœŸã€‚有关如何使 API 令牌失效的详细信æ¯ï¼Œè¯·å‚阅 [API 令牌](../reference-guides/about-the-api/api-tokens.md)。 + +## å‘出请求 + +该 API 通常是 RESTful 的，但是还具有多ç§åŠŸèƒ½ã€‚这些功能å¯ä»¥ä½¿å®¢æˆ·ç«¯å‘现所有内容，因此å¯ä»¥ç¼–写通用客户端，而ä¸å¿…为æ¯ç§èµ„æºç¼–写特定代ç ã€‚有关通用 API 规范的详细信æ¯ï¼Œè¯·å‚阅[此处](https://github.com/rancher/api-spec/blob/master/specification.md)。 + +- æ¯ç§ç±»åž‹éƒ½æœ‰ä¸€ä¸ª Schema，这个 Schema æ述了以下内容： + - 用于获å–此类资æºé›†åˆçš„ URL + - 资æºå¯ä»¥å…·æœ‰çš„æ¯ä¸ªå­—段åŠå…¶ç±»åž‹ã€åŸºæœ¬éªŒè¯è§„则ã€æ˜¯å¿…填还是å¯é€‰å­—段等 + - 在此类资æºä¸Šå¯ä»¥æ‰§è¡Œçš„æ¯ä¸ªæ“作，以åŠå®ƒä»¬çš„输入和输出（也作为 schema） + - å…许过滤的æ¯ä¸ªå­—段 + - 集åˆæœ¬èº«æˆ–集åˆä¸­çš„å•ä¸ªèµ„æºå¯ä»¥ä½¿ç”¨çš„ HTTP æ“作方法 + + +- 因此，你å¯ä»¥åªåŠ è½½ schema 列表并了解 API 的所有信æ¯ã€‚实际上，这是 API çš„ UI 工作方å¼ï¼Œå®ƒä¸åŒ…å«ç‰¹å®šäºŽ Rancher 本身的代ç ã€‚æ¯ä¸ª HTTP å“应中的 `X-Api-Schemas` 标头都会å‘é€èŽ·å– Schemas çš„ URL。你å¯ä»¥æŒ‰ç…§æ¯ä¸ª schema 上的 `collection` 链接了解è¦åœ¨å“ªé‡Œåˆ—出资æºï¼Œå¹¶åœ¨è¿”回资æºä¸­çš„其他 `links` 中获å–其他信æ¯ã€‚ + +- 在实践中，你å¯èƒ½åªæƒ³æž„造 URL 字符串。我们强烈建议将此é™åˆ¶ä¸ºåœ¨é¡¶å±‚åˆ—å‡ºçš„é›†åˆ (`/v3/`)，或获å–ç‰¹å®šèµ„æº (`/v3//`)。除此之外的任何内容都å¯èƒ½åœ¨å°†æ¥çš„版本中å‘生更改。 + +- 资æºä¹‹é—´ç›¸äº’之间有è”系，称为链接（links）。æ¯ä¸ªèµ„æºéƒ½åŒ…å«ä¸€ä¸ª `links` 映射，其中包å«é“¾æŽ¥å称和用于检索该信æ¯çš„ URL。åŒæ ·ï¼Œä½ åº”该 `GET` 资æºå¹¶éµå¾ª `links` 映射中的 URL，而ä¸æ˜¯è‡ªå·±æž„造这些字符串。 + +- 大多数资æºéƒ½æœ‰æ“作（action），表示å¯ä»¥æ‰§è¡ŒæŸä¸ªæ“作或改å˜èµ„æºçš„状æ€ã€‚è¦ä½¿ç”¨æ“作，请将 HTTP `POST` 请求å‘é€åˆ° `actions` 映射中你想è¦çš„æ“作的 URL。æŸäº›æ“作需è¦è¾“入或生æˆè¾“出，请å‚阅æ¯ç§ç±»åž‹çš„独立文档或 schema 以获å–具体信æ¯ã€‚ + +- è¦ç¼–辑资æºï¼Œè¯·å°† HTTP `PUT` 请求å‘é€åˆ°èµ„æºä¸Šçš„ `links.update` 链接，其中包å«è¦æ›´æ”¹çš„字段。如果链接丢失，则你无æƒæ›´æ–°èµ„æºã€‚未知字段和ä¸å¯ç¼–辑的字段将被忽略。 + +- è¦åˆ é™¤èµ„æºï¼Œè¯·å°† HTTP `DELETE` 请求å‘é€åˆ°èµ„æºä¸Šçš„ `links.remove` 链接。如果链接丢失，则你无æƒæ›´æ–°èµ„æºã€‚ + +- è¦åˆ›å»ºæ–°èµ„æºï¼ŒHTTP `POST` 到 schemaï¼ˆå³ `/v3/`ï¼‰ä¸­çš„é›†åˆ URL。 + +## 过滤 + +ä½ å¯ä»¥ä½¿ç”¨ HTTP 查询å‚数的公共字段在æœåŠ¡å™¨ç«¯è¿‡æ»¤å¤§å¤šæ•°é›†åˆã€‚`filters` 映射显示了å¯ä»¥è¿‡æ»¤çš„字段，以åŠè¿‡æ»¤åŽçš„值在你å‘起的请求中是什么。API UI 具有设置过滤和显示适当请求的控件。对于简å•çš„ "equals" 匹é…，它åªæ˜¯ `field=value`。你å¯ä»¥å°†ä¿®é¥°ç¬¦æ·»åŠ åˆ°å­—段å称，例如 `field_gt=42` 表示“字段大于 42â€ã€‚详情请å‚阅 [API 规范](https://github.com/rancher/api-spec/blob/master/specification.md#filtering)。 + +## æŽ’åº + +ä½ å¯ä»¥ä½¿ç”¨ HTTP 查询å‚数的公共字段在æœåŠ¡å™¨ç«¯æŽ’åºå¤§å¤šæ•°é›†åˆã€‚`sortLinks` 映射显示了å¯ç”¨çš„排åºï¼Œä»¥åŠç”¨äºŽèŽ·å–éµå¾ªè¯¥æŽ’åºçš„集åˆçš„ URL。它还包括当å‰å“排åºä¾æ®çš„ä¿¡æ¯ï¼ˆå¦‚果指定）。 + +## 分页 + +默认情况下，API å“应以æ¯é¡µ 100 个资æºçš„é™åˆ¶è¿›è¡Œåˆ†é¡µã€‚ä½ å¯ä»¥é€šè¿‡ `limit` 查询å‚数进行更改，最大为 1000，例如 `/v3/pods?limit=1000`。集åˆå“应中的 `pagination` 映射能让你知é“你是å¦æ‹¥æœ‰å®Œæ•´çš„结果集，如果没有，则会指å‘下一页的链接。 + +## æ•èŽ· Rancher API 调用 + +ä½ å¯ä»¥ä½¿ç”¨æµè§ˆå™¨å¼€å‘人员工具æ¥æ•èŽ· Rancher API 的调用方å¼ã€‚例如，你å¯ä»¥æŒ‰ç…§ä»¥ä¸‹æ­¥éª¤ä½¿ç”¨ Chrome å¼€å‘人员工具æ¥èŽ·å–用于é…ç½® RKE 集群的 API 调用： + +1. 在 Rancher UI 中，转到**集群管ç†**并å•å‡»**创建**。 +1. å•å‡»æŸä¸ªé›†ç¾¤ç±»åž‹ã€‚此示例使用 Digital Ocean。 +1. 使用集群å称和节点模æ¿å¡«å†™è¡¨å•ï¼Œä½†ä¸è¦å•å‡»**创建**。 +1. 在创建集群之å‰ï¼Œä½ éœ€è¦æ‰“开开å‘人员工具æ‰èƒ½çœ‹åˆ°æ­£åœ¨è®°å½•çš„ API 调用。è¦æ‰“开工具，å³é”®å•å‡» Rancher UI，然åŽå•å‡»**检查**。 +1. 在开å‘者工具中，å•å‡» **Network** 选项å¡ã€‚ +1. 在 **Network** 选项å¡ä¸Šï¼Œç¡®ä¿é€‰æ‹©äº† **Fetch/XHR**。 +1. 在 Rancher UI 中，å•å‡»**创建**。在开å‘者工具中，你应该会看到一个å为 `cluster?_replace=true` 的新网络请求。 +1. å³é”®å•å‡» `cluster?_replace=true` 并å•å‡»**å¤åˆ¶ > å¤åˆ¶ä¸º cURL**。 +1. 将结果粘贴到文本编辑器中。你将能够看到 POST 请求，包括被å‘é€åˆ°çš„ URLã€æ‰€æœ‰æ ‡å¤´ä»¥åŠè¯·æ±‚的完整正文。此命令å¯ç”¨äºŽä»Žå‘½ä»¤è¡Œåˆ›å»ºé›†ç¾¤ã€‚请注æ„，请求包å«å‡­è¯ï¼Œå› æ­¤è¯·å°†è¯·æ±‚存储在安全的地方。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/access-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/access-clusters.md new file mode 100644 index 00000000000..48e93b6f447 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/access-clusters.md @@ -0,0 +1,31 @@ +--- +title: 集群访问 +--- + +本节介ç»å¯ä»¥ç”¨æ¥è®¿é—® Rancher 管ç†çš„集群的工具。 + +有关如何授予用户访问集群的æƒé™çš„ä¿¡æ¯ï¼Œè¯·å‚阅[将用户添加到集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md)。 + +有关 RBAC 的更多信æ¯ï¼Œè¯·å‚阅[本节](manage-role-based-access-control-rbac.md)。 + +有关如何设置身份验è¯ç³»ç»Ÿçš„ä¿¡æ¯ï¼Œè¯·å‚阅[本节](authentication-config.md)。 + + +### Rancher UI + +Rancher æ供了一个直观的用户界é¢æ¥è®©ä½ ä¸Žé›†ç¾¤è¿›è¡Œäº¤äº’。UI 中所有å¯ç”¨çš„选项都使用 Rancher API。因此，UI 中的任何æ“作都å¯ä»¥åœ¨ Rancher CLI 或 Rancher API 中进行。 + +### kubectl + +ä½ å¯ä»¥ä½¿ç”¨ Kubernetes 命令行工具 [kubectl](https://kubernetes.io/docs/reference/kubectl/overview/) æ¥ç®¡ç†ä½ çš„集群。使用 kubectl 有两ç§é€‰æ‹©ï¼š + +- **Rancher kubectl shell**：通过å¯åŠ¨ Rancher UI 中å¯ç”¨çš„ kubectl shell 与集群交互。此选项ä¸éœ€è¦ä½ è¿›è¡Œä»»ä½•é…ç½®æ“作。有关详细信æ¯ï¼Œè¯·å‚阅[使用 kubectl Shell 访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 +- **终端远程连接**：你也å¯ä»¥é€šè¿‡åœ¨æœ¬åœ°æ¡Œé¢ä¸Šå®‰è£… [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)，然åŽå°†é›†ç¾¤çš„ kubeconfig 文件å¤åˆ¶åˆ°æœ¬åœ° `~/.kube/config` 目录æ¥ä¸Žé›†ç¾¤äº¤äº’。有关更多信æ¯ï¼Œè¯·å‚阅[使用 kubectl å’Œ kubeconfig 文件访问集群](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 + +### Rancher CLI + +ä½ å¯ä»¥ä¸‹è½½ Rancher 自己的命令行工具 [Rancher CLI](cli-with-rancher.md) æ¥æŽ§åˆ¶ä½ çš„集群。这个 CLI 工具å¯ä»¥ç›´æŽ¥ä¸Žä¸åŒçš„集群和项目进行交互，或者å‘它们传递 `kubectl` 命令。 + +### Rancher API + +最åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher API 与集群进行交互。在使用 API 之å‰ï¼Œä½ å¿…é¡»å…ˆèŽ·å– [API 密钥](../reference-guides/user-settings/api-keys.md)。è¦æŸ¥çœ‹ API 对象的ä¸åŒèµ„æºå­—段和æ“作，请打开 API UI（API UI å¯ä»¥é€šè¿‡å•å‡» Rancher UI 对象的**在 API 中查看**访问）。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/advanced-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/advanced-configuration.md new file mode 100644 index 00000000000..cf942d3dac5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/advanced-configuration.md @@ -0,0 +1,15 @@ +--- +title: 高级é…ç½® +--- + +### Alertmanager + +有关é…ç½® Alertmanager 自定义资æºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 + +### Prometheus + +有关é…ç½® Prometheus 自定义资æºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md)。 + +### PrometheusRules + +有关é…ç½® Prometheus 自定义资æºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/advanced-user-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/advanced-user-guides.md new file mode 100644 index 00000000000..0f8f5a7df69 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/advanced-user-guides.md @@ -0,0 +1,7 @@ +--- +title: é«˜çº§ç”¨æˆ·æŒ‡å— +--- + +高级用户指å—是“问题导å‘â€çš„文档，用户å¯ä»¥ä»Žä¸­å­¦ä¹ å¦‚何解决问题。高级用于指å—与新用户指å—的主è¦åŒºåˆ«åœ¨äºŽï¼Œé«˜çº§ç”¨æˆ·æŒ‡å—é¢å‘更有ç»éªŒæˆ–更高级的用户，这些用户对文档有更多的技术需求，而且已ç»äº†è§£ Rancher åŠå…¶åŠŸèƒ½ã€‚他们知é“自己需è¦åšä»€ä¹ˆï¼Œåªæ˜¯éœ€è¦é¢å¤–的指导æ¥å®Œæˆæ›´å¤æ‚的任务。 + +应该注æ„的是，新用户指å—和高级用户指å—都没有æ供详细的解释或讨论（这些文档ä¸åŒ…括在本部分）。æ“作指å—侧é‡äºŽå¼•å¯¼ç”¨æˆ·é€šè¿‡å¯é‡å¤ã€æœ‰æ•ˆçš„步骤æ¥å­¦ä¹ æ–°æŠ€èƒ½ã€æŽŒæ¡æŸäº›æ“作或解决æŸäº›é—®é¢˜ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/air-gapped-helm-cli-install.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/air-gapped-helm-cli-install.md new file mode 100644 index 00000000000..b5b58f77e56 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/air-gapped-helm-cli-install.md @@ -0,0 +1,31 @@ +--- +title: 离线 Helm CLI 安装 +--- + +本文介ç»å¦‚何使用 Helm CLI 在离线环境中安装 Rancher Server。离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +Rancher 安装在 RKE Kubernetes 集群ã€K3s Kubernetes 集群，或å•ä¸ª Docker 容器上对应的安装步骤会有所ä¸åŒã€‚ + +如需了解å„个安装方å¼çš„更多信æ¯ï¼Œè¯·å‚è§[本页](installation-and-upgrade.md)。 + +在安装指导中，我们为ä¸åŒçš„安装选项æ供对应的 _选项å¡_ 。 + +:::note é‡è¦æ示： + +如果你按照 Docker 安装指å—安装 Rancher，你将没有把 Docker 安装转æ¢ä¸º Kubernetes 安装的å‡çº§é€”径。 + +::: + +## å®‰è£…æ¦‚è¦ + +1. [设置基础设施和ç§æœ‰é•œåƒä»“库](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) +2. [收集镜åƒåˆ°ç§æœ‰é•œåƒä»“库](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md) +3. [设置 Kubernetes 集群（如果你使用 Docker 安装，请跳过此步骤）](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md) +4. [安装 Rancher](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md) + +## å‡çº§ + +如需在离线环境中使用 Helm CLI å‡çº§ Rancher，请按照[å‡çº§æ­¥éª¤](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md)进行æ“作。 + +### åŽç»­æ“作 +[准备节点](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/authentication-config.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/authentication-config.md new file mode 100644 index 00000000000..a77cbe0b204 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/authentication-config.md @@ -0,0 +1,132 @@ +--- +title: 身份验è¯é…ç½® +weight: 10 +--- + +Rancher å‘ Kubernetes 添加的关键功能之一，就是集中å¼ç”¨æˆ·èº«ä»½éªŒè¯ã€‚此功能å…许你的用户使用一组凭è¯å¯¹ä½ çš„所有 Kubernetes 集群进行身份验è¯ã€‚ + +è¿™ç§é›†ä¸­å¼çš„用户身份验è¯æ˜¯ä½¿ç”¨ Rancher 身份验è¯ä»£ç†å®Œæˆçš„，该代ç†ä¸Ž Rancher 的其他组件一起安装。这个代ç†éªŒè¯ä½ çš„用户，并使用一个 ServiceAccount 将用户请求转å‘到你的 Kubernetes 集群。 + +## 外部验è¯ä¸Žæœ¬åœ°éªŒè¯ + +Rancher 身份验è¯ä»£ç†æ”¯æŒä¸Žä»¥ä¸‹å¤–部身份验è¯æœåŠ¡é›†æˆï¼š + +| 验è¯æœåŠ¡ | +| ------------------------------------------------------------------------------------------------ | +| [Microsoft Active Directory](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md) | +| [GitHub](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md) | +| [Microsoft Azure AD](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md) | +| [FreeIPA](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md) | +| [OpenLDAP](configure-openldap.md) | +| [Microsoft AD FS](configure-microsoft-ad-federation-service-saml.md) | +| [PingIdentity](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md) | +| [Keycloak (OIDC)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md) | +| [Keycloak (SAML)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md) | +| [Okta](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md) | +| [Google OAuth](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md) | +| [Shibboleth](configure-shibboleth-saml.md) | + +åŒæ—¶ï¼ŒRancher 也æ供了[本地身份验è¯](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md)。 + +大多数情况下，应该使用外部身份验è¯æœåŠ¡ï¼Œè€Œä¸æ˜¯æœ¬åœ°èº«ä»½éªŒè¯ï¼Œå› ä¸ºå¤–部身份验è¯å…许对用户进行集中管ç†ã€‚但是你å¯èƒ½éœ€è¦ä¸€äº›æœ¬åœ°èº«ä»½éªŒè¯ç”¨æˆ·ï¼Œä»¥ä¾¿åœ¨ç‰¹å®šçš„情况下（例如在外部身份验è¯ç³»ç»Ÿä¸å¯ç”¨æˆ–æ­£åœ¨è¿›è¡Œç»´æŠ¤æ—¶ï¼‰ç®¡ç† Rancher。 + +## 用户和组 + +Rancher ä¾èµ–用户和组æ¥å†³å®šå…许登录到 Rancher 的用户，以åŠä»–们å¯ä»¥è®¿é—®å“ªäº›èµ„æºã€‚使用外部系统进行身份验è¯æ—¶ï¼Œå°†ç”±å¤–部系统æ供用户和组。这些用户和组被赋予集群ã€é¡¹ç›®ã€å¤šé›†ç¾¤åº”用ã€å…¨å±€ DNS æ供商等资æºçš„特定角色。当你将访问æƒé™æŽˆäºˆæŸä¸ªç»„时，身份验è¯æ供程åºä¸­å±žäºŽè¯¥ç»„的所有用户都将能够使用你指定的æƒé™è®¿é—®è¯¥èµ„æºã€‚有关角色和æƒé™çš„更多信æ¯ï¼Œè¯·å‚è§ [RBAC](manage-role-based-access-control-rbac.md)。 + +:::note + +本地认è¯ä¸æ”¯æŒåˆ›å»ºæˆ–管ç†ç”¨æˆ·ç»„。 + +::: + +详情请å‚è§[用户和组](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md)。 + +## Rancher 授æƒèŒƒå›´ + +å°† Rancher é…ç½®æˆå…许使用外部验è¯æ供程åºç™»å½•åŽï¼Œä½ éœ€è¦é…ç½®å…许登录和使用 Rancher 的用户。å¯ç”¨é€‰é¡¹å¦‚下： + +| 访问级别 | æè¿° | +|----------------------------------------------|-------------| +| å…许任何有效用户 | 授æƒæœåŠ¡ä¸­çš„ _任何_ 用户都å¯ä»¥è®¿é—®Rancher。通常ä¸å»ºè®®ä½¿ç”¨æ­¤è®¾ç½®ã€‚ | +| å…许集群和项目æˆå‘˜ï¼Œä»¥åŠæŽˆæƒçš„用户和组织 | 认è¯æœåŠ¡ä¸­çš„任何用户，以åŠæ·»åŠ ä¸º**集群æˆå‘˜**或**项目æˆå‘˜**的任何组都å¯ä»¥ç™»å½•åˆ° Rancher。此外，添加到**授æƒç”¨æˆ·å’Œç»„织**列表中的身份验è¯æœåŠ¡ä¸­çš„任何用户和组都能登录到 Rancher。 | +| ä»…å…许授æƒç”¨æˆ·å’Œç»„织 | åªæœ‰æ·»åŠ åˆ°**授æƒç”¨æˆ·å’Œç»„织**的身份验è¯æœåŠ¡ä¸­çš„用户和组能登录 Rancher。 | + +è¦åœ¨æŽˆæƒæœåŠ¡ä¸­ä¸ºç”¨æˆ·è®¾ç½® Rancher 访问级别，请执行以下步骤： + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. å•å‡»å·¦ä¾§å¯¼èˆªæ çš„**认è¯**。 +1. 设置好认è¯æ供程åºçš„é…ç½®åŽï¼Œä½¿ç”¨ **Site Access** 选项æ¥é…置用户的授æƒèŒƒå›´ã€‚上表说明了æ¯ä¸ªé€‰é¡¹çš„访问级别。 +1. å¯é€‰ï¼šå¦‚果你选择**å…许任何有效用户**以外的选项，你å¯ä»¥é€šè¿‡åœ¨æ˜¾ç¤ºçš„文本字段中æœç´¢ç”¨æˆ·ï¼Œå°†ç”¨æˆ·æ·»åŠ åˆ°**授æƒç”¨æˆ·å’Œç»„织**的列表中。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：Rancher 访问é…置已应用。 + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + +## 外部身份验è¯é…置和用户主体 + +é…置外部认è¯éœ€è¦ï¼š + +- 分é…了管ç†å‘˜è§’色的本地用户，以下称为 _本地主体_。 +- å¯ä»¥ä½¿ç”¨å¤–部认è¯æœåŠ¡è¿›è¡Œè®¤è¯çš„外部用户，以下称为 _外部主体_。 + +外部身份验è¯çš„é…ç½®å°†å½±å“ Rancher 中主体用户的管ç†æ–¹å¼ã€‚按照下é¢çš„列表æ¥æ›´å¥½åœ°ç†è§£è¿™äº›å½±å“。 + +1. 作为本地主体登录到 Rancher 并完æˆå¤–部身份验è¯çš„é…置。 + + ![Sign In](/img/sign-in.png) + +2. Rancher 将外部主体与本地主体相关è”。这两个用户共享本地主体的用户 ID。 + + ![Principal ID Sharing](/img/principal-ID.png) + +3. 完æˆé…ç½®åŽï¼ŒRancher 将自动退出本地主体。 + + ![Sign Out Local Principal](/img/sign-out-local.png) + +4. 然åŽï¼ŒRancher 会自动将你作为外部主体é‡æ–°ç™»å½•ã€‚ + + ![Sign In External Principal](/img/sign-in-external.png) + +5. 由于外部主体与本地主体共享一个 ID，因此**用户**页é¢ä¸ä¼šå†å•ç‹¬æ˜¾ç¤ºå¤–部主体的对象。 + + ![Sign In External Principal](/img/users-page.png) + +6. 外部主体和本地主体共享相åŒçš„访问æƒé™ã€‚ + +:::note é‡æ–°é…置以å‰è®¾ç½®çš„身份验è¯æä¾›ç¨‹åº + +如果你需è¦é‡æ–°é…置或ç¦ç”¨ä»¥å‰è®¾ç½®çš„æ供程åºç„¶åŽå†é‡æ–°å¯ç”¨å®ƒï¼Œè¯·ç¡®ä¿è¿›è¡Œæ­¤æ“作的用户使用外部用户身份登录 Rancher，而ä¸æ˜¯æœ¬åœ°ç®¡ç†å‘˜ã€‚ + +::: + +## ç¦ç”¨è®¤è¯æä¾›ç¨‹åº + +ç¦ç”¨èº«ä»½è®¤è¯æ供程åºæ—¶ï¼ŒRancher 会删除与其关è”的所有资æºï¼Œä¾‹å¦‚： +- Secrets +- 全局角色绑定。 +- 集群角色模æ¿ç»‘定。 +- 项目角色模æ¿ç»‘定。 +- 与æ供商关è”的外部用户，但是这些用户从未以本地用户身份登录到 Rancher。 + +由于此æ“作å¯èƒ½ä¼šå¯¼è‡´è®¸å¤šèµ„æºä¸¢å¤±ï¼Œå› æ­¤ä½ å¯èƒ½å¸Œæœ›åœ¨æ供程åºä¸Šæ·»åŠ ä¿æŠ¤æŽªæ–½ã€‚为确ä¿åœ¨ç¦ç”¨èº«ä»½è®¤è¯æ供程åºæ—¶ä¸ä¼šè¿è¡Œæ­¤æ¸…ç†ï¼Œè¯·å‘相应的身份认è¯é…置添加特殊注释。 + +例如，è¦ä¸º Azure AD æ供程åºæ·»åŠ å®‰å…¨æŽªæ–½ï¼Œè¯·æ³¨é‡Š `azuread` authconfig 对象： + +`kubectl annotate --overwrite authconfig azuread management.cattle.io/auth-provider-cleanup='user-locked'` + +在你将注释设置为 `unlocked` 之å‰ï¼ŒRancher ä¸ä¼šæ‰§è¡Œæ¸…ç†ã€‚ + +### 手动è¿è¡Œèµ„æºæ¸…ç† + +å³ä½¿åœ¨ä½ é…置了å¦ä¸€ä¸ªèº«ä»½è®¤è¯æ供程åºï¼ŒRancher 也å¯èƒ½ä¼šä¿ç•™ local 集群中已ç¦ç”¨çš„身份认è¯æ供程åºé…置的资æºã€‚例如，如果你使用 Provider A，然åŽç¦ç”¨äº†å®ƒå¹¶å¼€å§‹ä½¿ç”¨ Provider B，当你å‡çº§åˆ°æ–°ç‰ˆæœ¬çš„ Rancher 时，你å¯ä»¥æ‰‹åŠ¨è§¦å‘对 Provider A é…置的资æºçš„清ç†ã€‚ + +è¦ä¸ºå·²ç¦ç”¨çš„身份认è¯æ供程åºæ‰‹åŠ¨è§¦å‘清ç†ï¼Œè¯·å°†å¸¦æœ‰ `unlocked` 值的 `management.cattle.io/auth-provider-cleanup` 注释添加到 auth é…置中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/authentication-permissions-and-global-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/authentication-permissions-and-global-configuration.md new file mode 100644 index 00000000000..8d6758e8dcb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/authentication-permissions-and-global-configuration.md @@ -0,0 +1,53 @@ +--- +title: 身份验è¯ã€æƒé™å’Œå…¨å±€é…ç½® +--- + +安装完æˆåŽï¼Œ[系统管ç†å‘˜](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)需è¦é…ç½® Rancher æ¥é…置身份验è¯ï¼Œå®‰å…¨ï¼Œé»˜è®¤è®¾å®šï¼Œå®‰å…¨ç­–略，驱动和全局 DNS æ¡ç›®ã€‚ + +## 首次登录 + +首次登录 Rancher åŽï¼ŒRancher 会æ示你输入 **Rancher Server URL**。你需è¦å°† URL 设置为 Rancher Server 的主è¦å…¥å£ç‚¹ã€‚当负载å‡è¡¡å™¨ä½äºŽ Rancher Server 集群å‰é¢æ—¶ï¼ŒURL 需è¦è®¾ç½®ä¸ºè´Ÿè½½å‡è¡¡å™¨åœ°å€ã€‚系统会自动å°è¯•ä»Žè¿è¡Œ Rancher Server 的主机的 IP 地å€æˆ–主机å推断 Rancher Server çš„URL，上述推断仅在你è¿è¡Œå•èŠ‚点 Rancher Server æ—¶æ‰æ­£ç¡®ã€‚因此，在大多数情况下，你需è¦è‡ªå·±å°† Rancher Server çš„ URL 设置为正确的值。 + +:::danger + +Rancher Server çš„ URL 在设置åŽä¸å¯å†æ›´æ–°ã€‚因此，你需è¦è°¨æ…Žè®¾ç½®è¯¥ URL。 + +::: + +## èº«ä»½éªŒè¯ + +Rancher å‘ Kubernetes 添加的关键功能之一，就是集中å¼ç”¨æˆ·èº«ä»½éªŒè¯ã€‚此功能å…许将本地用户连接到外部身份验è¯ç³»ç»Ÿï¼Œä½¿ç”¨è¯¥ç³»ç»Ÿçš„用户和组进行身份验è¯ã€‚ + +有关身份验è¯å¦‚何工作åŠå¦‚何设置外部身份认è¯ç³»ç»Ÿï¼Œè¯·å‚è§[身份验è¯](authentication-config.md)。 + +## æŽˆæƒ + +Rancher 通过 _用户_ 进行授æƒç®¡ç†ã€‚用户的 _授æƒ_ 或系统访问æƒé™ç”±ç”¨æˆ·è§’色决定。Rancher æ供了预设角色，让你轻æ¾é…置用户对资æºçš„æƒé™ï¼Œè¿˜æ供了为æ¯ä¸ª Kubernetes 资æºå®šåˆ¶è§’色的能力。 + +有关授æƒå¦‚何工作åŠå¦‚何自定义角色，请å‚è§ [RBAC](manage-role-based-access-control-rbac.md)。 + +## Pod 安全策略 + +_Pod 安全策略（PSP）_ 是用æ¥æŽ§åˆ¶å®‰å…¨æ•æ„Ÿç›¸å…³ Pod 规范（例如 root 特æƒï¼‰çš„对象。如果æŸä¸ª Pod ä¸æ»¡è¶³ PSP 指定的æ¡ä»¶ï¼ŒKubernetes å°†ä¸å…许它å¯åŠ¨ï¼Œå¹¶åœ¨ Rancher 中显示错误消æ¯ã€‚ + +有关如何创建和使用 PSP，请å‚è§ [Pod 安全策略](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。 + +## é…置驱动 + +使用 Rancher 中的驱动，你å¯ä»¥ç®¡ç†å¯ä»¥ä½¿ç”¨å“ªäº›ä¾›åº”商æ¥é…ç½®[托管的 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)或[云æœåŠ¡å™¨èŠ‚点](use-new-nodes-in-an-infra-provider.md)，以å…许 Rancher éƒ¨ç½²å’Œç®¡ç† Kubernetes。 + +详情请å‚考[é…置驱动](about-provisioning-drivers.md)。 + +## 添加 Kubernetes 版本到 Rancher + +ä½ å¯ä»¥é€šè¿‡è¿™ä¸ªåŠŸèƒ½ï¼Œåœ¨ä¸å‡çº§ Rancher 的情况下，å‡çº§åˆ°æœ€æ–°å‘布的 Kubernetes 版本。Kubernetes 倾å‘于在次è¦ç‰ˆæœ¬åˆ é™¤æˆ–新增 API 接å£ã€‚本功能让你轻æ¾å‡çº§ Kubernetes è¡¥ä¸ç‰ˆæœ¬ï¼ˆå³ `v1.15.X`），但ä¸å‡çº§ Kubernetes 次è¦ç‰ˆæœ¬ï¼ˆå³ `v1.X.0`）。 + +Rancher 用于é…ç½® [RKE 集群](launch-kubernetes-with-rancher.md) çš„ä¿¡æ¯çŽ°åœ¨ä½äºŽ Rancher Kubernetes 元数æ®ä¸­ã€‚有关元数æ®é…置以åŠå¦‚何更改用于é…ç½® RKE 集群的 Kubernetes 版本，请å‚è§ [Rancher Kubernetes 元数æ®ã€‚](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md) + +Rancher 用于é…ç½® [RKE 集群](launch-kubernetes-with-rancher.md)çš„ Kubernetes 版本信æ¯åŒ…å«åœ¨ Rancher Kubernetes 元数æ®ä¸­ã€‚ + +有关元数æ®å¦‚何工作以åŠå¦‚何é…置元数æ®ï¼Œè¯·å‚è§ [Rancher Kubernetes 元数æ®](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md)。 + +## å¯ç”¨å®žéªŒåŠŸèƒ½ + +Rancher 包å«ä¸€äº›é»˜è®¤å…³é—­çš„实验功能。我们引入了功能开关，让你试用这些新功能。详情请å‚è§[功能开关](enable-experimental-features.md)的章节。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/aws-cloud-marketplace.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/aws-cloud-marketplace.md new file mode 100644 index 00000000000..980a427cf4d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/aws-cloud-marketplace.md @@ -0,0 +1,32 @@ +--- +title: AWS Marketplace é›†æˆ +--- + +## 概述 + +Rancher æ供了与 AWS Marketplace 的集æˆï¼Œå…è®¸ç”¨æˆ·å‘ SUSE 购买支æŒã€‚此集æˆå¸®åŠ©ä½ åœ¨éœ€è¦ä½¿ç”¨æ›´å¤šé›†ç¾¤æ—¶è½»æ¾è°ƒæ•´æ”¯æŒéœ€æ±‚。 + +## é™åˆ¶ + +- 必须使用 Rancher v2.6.7 或更高版本。 +- Rancher 必须在å¯ç”¨å…¶ä»–指标的情况下进行部署。 +- Rancher 必须安装在 EKS 集群上。 +- 必须通过 AWS Marketplace 购买至少一项 Rancher 支æŒçš„ Entitlement。 +- ä½ å¯èƒ½éœ€è¦é¢å¤–的设置æ¥æ”¯æŒä»£ç†/离线用例。有关详细信æ¯ï¼Œè¯·å‚阅[先决æ¡ä»¶](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md)。 + +## 如何使用 + +1. 完æˆ[先决æ¡ä»¶æ­¥éª¤](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md)。 +2. [安装 CSP Adapter](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md)。 + +## 常è§é—®é¢˜ + +**我以åŽåŽç»­å†è´­ä¹°æ›´å¤šèŠ‚点的支æŒå—？** + +是的。你需è¦è½¬åˆ°æœ€åˆè´­ä¹°æ”¯æŒçš„ AWS Marketplace æ¡ç›®å¹¶å¢žåŠ  Entitlement çš„æ•°é‡ã€‚ + +**我å¯ä»¥åœ¨åŒä¸€ä¸ª AWS 账户中使用多个 Rancher 实例å—？** + +是的。但是，安装 Rancher çš„æ¯ä¸ªé›†ç¾¤éƒ½éœ€è¦éµå®ˆå…ˆå†³æ¡ä»¶ã€‚ + +此外，一个 Entitlement æ¯æ¬¡åªèƒ½ç”±ä¸€å° Rancher management server 使用。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/backup-restore-and-disaster-recovery.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/backup-restore-and-disaster-recovery.md new file mode 100644 index 00000000000..b7460369da2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/backup-restore-and-disaster-recovery.md @@ -0,0 +1,96 @@ +--- +title: 备份和ç¾éš¾æ¢å¤ +keywords: [rancher 备份还原, rancher 备份与æ¢å¤, 备份æ¢å¤ rancher, rancher 备份与æ¢å¤ rancher] +--- + +在本节中，你将学习如何创建 Rancher 的备份，如何从备份中æ¢å¤ Rancher，以åŠå¦‚何将 Rancher è¿ç§»åˆ°æ–°çš„ Kubernetes 集群。 + +`rancher-backup` operator å¯ä»¥ç”¨æ¥å¤‡ä»½å’Œæ¢å¤ä»»ä½• Kubernetes 集群上的 Rancher。这个应用是一个 Helm Chart，å¯ä»¥é€šè¿‡ Rancher çš„ **Apps** 页é¢æˆ–使用 Helm CLI 部署。你å¯ä»¥è®¿é—®[本页é¢](https://github.com/rancher/charts/tree/release-v2.6/charts/rancher-backup)èŽ·å– `rancher-backup` Helm Chart。 + +`backup-restore` operator 需è¦å®‰è£…在 local 集群上，并且åªå¯¹ Rancher 应用进行备份。备份和æ¢å¤æ“作仅在本地 Kubernetes 集群中执行。 + + +## 备份和æ¢å¤ Docker 安装的 Rancher + +对于使用 Docker 安装的 Rancher，请å‚è§[备份](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md)å’Œ[æ¢å¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md)对 Rancher 进行备份和æ¢å¤ã€‚ + +## 备份和æ¢å¤åŽŸç† + +`rancher-backup` operator 引入了三个自定义资æºï¼Œåˆ†åˆ«æ˜¯ Backupsã€Restores å’Œ ResourceSets。将以下集群范围的自定义资æºå®šä¹‰æ·»åŠ åˆ°é›†ç¾¤ä¸­ï¼š + +- `backups.resources.cattle.io` +- `resourcesets.resources.cattle.io` +- `restores.resources.cattle.io` + +ResourceSet 定义了需è¦å¤‡ä»½å“ªäº› Kubernetes 资æºã€‚由于备份 Rancher 所需的值是预设的，因此 ResourceSet 无法通过 Rancher UI 进行é…置。请ä¸è¦ä¿®æ”¹æ­¤ ResourceSet。 + +在创建 Backup 自定义资æºæ—¶ï¼Œ`rancher-backup` operator 调用 `kube-apiserver` æ¥èŽ·å– Backup 自定义资æºå¼•ç”¨çš„ ResourceSet（å³é¢„设的 `rancher-resource-set`）资æºã€‚ + +然åŽï¼Œoperator 以 `.tar.gz` æ ¼å¼åˆ›å»ºå¤‡ä»½æ–‡ä»¶ï¼Œå¹¶å°†å…¶å­˜å‚¨åœ¨ Backup 资æºä¸­é…置的ä½ç½®ã€‚ + +在创建 Restore 自定义资æºæ—¶ï¼Œoperator 访问 Restore 指定的 `tar.gz` 备份文件，并从该文件æ¢å¤åº”用。 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 或 `kubectl apply` æ¥åˆ›å»º Backup å’Œ Restore 自定义资æºã€‚ + +:::note + +请å‚è§[此处](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md#2-使用-restore-自定义资æºæ¥è¿˜åŽŸå¤‡ä»½)获å–在 Rancher 2.6.3 中将现有备份文件æ¢å¤åˆ° v1.22 集群的帮助。 + +::: + +## 安装 rancher-backup operator + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 或 Helm CLI æ¥å®‰è£… `rancher-backup` operator。两ç§å®‰è£…方法都将 `rancher-backup` Helm Chart 安装在è¿è¡Œ Rancher Server çš„ Kubernetes 集群上。它是集群管ç†å‘˜ç‹¬æœ‰çš„功能，仅适用于 **local** 集群。（*如果你在 Rancher UI 中没有看到 `rancher-backup`，你å¯èƒ½é€‰æ‹©äº†é”™è¯¯çš„集群。*） + +:::note + +使用 `backup-restore` operator 执行æ¢å¤åŽï¼ŒFleet 中会出现一个已知问题：用于 `clientSecretName` å’Œ `helmSecretName` 的密文ä¸åŒ…å«åœ¨ Fleet çš„ Git 仓库中。请å‚è§[此处](./fleet-gitops-at-scale.md#故障排除)获得解决方法。 + +::: + +### 使用 Rancher UI 安装 rancher-backup + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ° `local` 集群并å•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Apps > Charts**。 +1. 点击 **Rancher 备份**。 +1. å•å‡»**安装**。 +1. å¯é€‰ï¼šé…置默认存储ä½ç½®ã€‚如需获å–帮助，请å‚è§[é…ç½®](../reference-guides/backup-restore-configuration/storage-configuration.md)。 +1. å•å‡»**安装**。 + +**结果**：`rancher-backup` operator 已安装。 + +在**集群仪表æ¿**中，你å¯ä»¥çœ‹åˆ°åˆ—在 **Deployments** 下的 `rancher-backup` operator。 + +如果需è¦åœ¨ Rancher 中é…置备份应用，在左侧导航æ ä¸­å•å‡» **Rancher 备份**。 + +### RBAC + +åªæœ‰ Rancher 管ç†å‘˜å’Œæœ¬åœ°é›†ç¾¤çš„所有者å¯ä»¥ï¼š + +* 安装 Chart +* 看到 Backup å’Œ Restore CRD 的导航链接 +* 通过分别创建 Backup CR å’Œ Restore CR 执行备份和æ¢å¤ +* 列出目å‰å·²æ‰§è¡Œçš„备份和æ¢å¤æ“作 + +## 备份 Rancher + +备份是通过创建 Backup 自定义资æºå®žçŽ°çš„。如需查看教程，请å‚è§[本页é¢](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md)。 + +## 还原 Rancher + +还原是通过创建 Restore 自定义资æºå®žçŽ°çš„。如需查看教程，请å‚è§[本页é¢](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md)。 + +## å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ + +ä½ å¯ä»¥æŒ‰ç…§[这些步骤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)执行è¿ç§»ã€‚ + +## 默认存储ä½ç½®é…ç½® + +é…置一个用于ä¿å­˜æ‰€æœ‰å¤‡ä»½çš„默认存储ä½ç½®ã€‚ä½ å¯ä»¥é€‰æ‹©å¯¹æ¯ä¸ªå¤‡ä»½è¿›è¡Œè¦†ç›–，但仅é™äºŽä½¿ç”¨ S3 或 Minio 对象存储。 + +如需了解å„个选项的é…置，请å‚è§[本页é¢](../reference-guides/backup-restore-configuration/storage-configuration.md)。 + +### rancher-backup Helm Chart 的示例 values.yaml + +当使用 Helm CLI 安装时，å¯ä»¥ä½¿ç”¨ç¤ºä¾‹ [values.yaml 文件](../reference-guides/backup-restore-configuration/storage-configuration.md#rancher-backup-helm-chart-的示例-valuesyaml) æ¥é…ç½® `rancher-backup` operator。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/backup-restore-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/backup-restore-configuration.md new file mode 100644 index 00000000000..b016cbe2f97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/backup-restore-configuration.md @@ -0,0 +1,8 @@ +--- +title: Rancher 备份é…ç½®å‚考 +--- + +- [备份é…ç½®](../reference-guides/backup-restore-configuration/backup-configuration.md) +- [还原é…ç½®](../reference-guides/backup-restore-configuration/restore-configuration.md) +- [存储ä½ç½®é…ç½®](../reference-guides/backup-restore-configuration/storage-configuration.md) +- [Backup å’Œ Restore 自定义资æºç¤ºä¾‹](../reference-guides/backup-restore-configuration/examples.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/best-practices.md new file mode 100644 index 00000000000..8c7c1fd9d07 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/best-practices.md @@ -0,0 +1,17 @@ +--- +title: 最佳实践 +--- + +æœ¬èŠ‚ä»‹ç» Rancher 实现的最佳实践，其中包括对 Kubernetesã€Dockerã€å®¹å™¨ç­‰æŠ€æœ¯çš„使用建议。最佳实践旨在利用 Rancher åŠå…¶å®¢æˆ·çš„è¿è¥ç»éªŒï¼ŒåŠ©ä½ æ›´å¥½åœ°å®žçŽ° Rancher。 + +如果你对用例的实际应用有任何疑问，请è”系客户æˆåŠŸç»ç†æˆ–支æŒä¸­å¿ƒã€‚ + +ä½ å¯ä»¥åœ¨å·¦ä¾§å¯¼èˆªæ å¿«é€Ÿæ‰¾åˆ°ç®¡ç†å’Œéƒ¨ç½² Rancher Server 的最佳实践。 + +如需查看更多最佳实践指å—，请å‚è§ï¼š + +- [安全类文档](rancher-security.md) +- [Rancher åšå®¢](https://www.suse.com/c/rancherblog/) +- [Rancher 论å›](https://forums.rancher.com/) +- [Rancher 用户的 Slack 群组](https://slack.rancher.io/) +- [B ç«™](https://space.bilibili.com/430496045/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/checklist-for-production-ready-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/checklist-for-production-ready-clusters.md new file mode 100644 index 00000000000..5c925f124ed --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/checklist-for-production-ready-clusters.md @@ -0,0 +1,48 @@ +--- +title: ç”Ÿäº§å°±ç»ªé›†ç¾¤æ£€æŸ¥æ¸…å• +--- + +本节将介ç»åˆ›å»ºç”Ÿäº§å°±ç»ªåž‹ Kubernetes 集群的最佳实践。这个集群å¯ç”¨äºŽè¿è¡Œä½ çš„应用和æœåŠ¡ã€‚ + +有关集群的è¦æ±‚（包括对 OS/Dockerã€ç¡¬ä»¶å’Œç½‘络的è¦æ±‚），请å‚阅[节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +本文介ç»äº†æˆ‘们推è用于所有生产集群的最佳实践的简短列表。 + +如需获å–推è的所有最佳实践的完整列表，请å‚阅[最佳实践](best-practices.md)。 + +### 节点è¦æ±‚ + +* ç¡®ä¿ä½ çš„节点满足所有[节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)，包括端å£è¦æ±‚。 + +### 备份 etcd + +* å¯ç”¨ etcd 快照。验è¯æ˜¯å¦æ­£åœ¨åˆ›å»ºå¿«ç…§ï¼Œå¹¶æ‰§è¡Œç¾éš¾æ¢å¤æ–¹æ¡ˆï¼Œä»Žè€ŒéªŒè¯å¿«ç…§æ˜¯å¦æœ‰æ•ˆã€‚etcd 是存储集群状æ€çš„ä½ç½®ï¼Œä¸¢å¤± etcd æ•°æ®æ„味ç€ä¸¢å¤±é›†ç¾¤ã€‚因此，请确ä¿ä¸ºé›†ç¾¤é…ç½® etcd 的定期快照，并确ä¿å¿«ç…§ä¹Ÿæ˜¯å­˜å‚¨åœ¨å¤–部（节点外）的。 + +### 集群架构 + +* 节点应具有以下角色é…置之一： + * `etcd` + * `controlplane` + * `etcd` å’Œ `controlplane` + * `worker`（ä¸åº”在具有 `etcd` 或 `controlplane` 角色的节点上使用或添加 `worker` 角色） +* 至少拥有三个角色为 `etcd` 的节点，æ¥ç¡®ä¿å¤±åŽ»ä¸€ä¸ªèŠ‚点时ä»èƒ½å­˜æ´»ã€‚增加 etcd 节点数é‡èƒ½æ高容错率，而将 etcd 分散到ä¸åŒå¯ç”¨åŒºç”šè‡³èƒ½èŽ·å–更好的容错能力。 +* ä¸ºä¸¤ä¸ªæˆ–æ›´å¤šèŠ‚ç‚¹åˆ†é… `controlplane` 角色，能实现主组件的高å¯ç”¨æ€§ã€‚ +* ä¸ºä¸¤ä¸ªæˆ–å¤šä¸ªèŠ‚ç‚¹åˆ†é… `worker` 角色，以便在节点故障时é‡æ–°å®‰æŽ’工作负载。 + +有关æ¯ä¸ªè§’色的用途的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 中的节点角色](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md)。 + +有关æ¯ä¸ª Kubernetes 角色的节点数的详细信æ¯ï¼Œè¯·å‚阅[推è架构](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)。 + +### Logging å’Œ Monitoring + +* 为 Kubernetes 组件（系统æœåŠ¡ï¼‰é…置告警/通知程åºã€‚ +* 为集群分æžå’Œäº‹åŽå‰–æžé…ç½® Logging。 + +### å¯é æ€§ + +* 在集群上执行负载测试，以验è¯ç¡¬ä»¶æ˜¯å¦å¯ä»¥æ”¯æŒä½ çš„工作负载。 + +### 网络 + +* 最å°åŒ–网络延迟。Rancher 建议尽é‡å‡å°‘ etcd 节点之间的延迟。`heartbeat-interval` 的默认设置是 `500`，`election-timeout` 的默认设置是 `5000`。这些 [etcd 调优设置](https://coreos.com/etcd/docs/latest/tuning.html) å…许 etcd 在大多数网络（网络延迟特别高的情况下除外）中è¿è¡Œã€‚ +* 集群节点应ä½äºŽå•ä¸ªåŒºåŸŸå†…。大多数云厂商在一个区域内æ供多个å¯ç”¨åŒºï¼Œè¿™å¯ä»¥æ高你集群的å¯ç”¨æ€§ã€‚任何角色的节点都å¯ä»¥ä½¿ç”¨å¤šä¸ªå¯ç”¨åŒºã€‚如果你使用 [Kubernetes Cloud Provider](set-up-cloud-providers.md) 资æºï¼Œè¯·æŸ¥é˜…文档以了解é™åˆ¶ï¼ˆå³åŒºåŸŸå­˜å‚¨é™åˆ¶ï¼‰ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cis-scan-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cis-scan-guides.md new file mode 100644 index 00000000000..269edaaec4a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cis-scan-guides.md @@ -0,0 +1,13 @@ +--- +title: CIS 扫ææŒ‡å— +--- + +- [安装 rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md) +- [å¸è½½ rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md) +- [è¿è¡Œæ‰«æ](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md) +- [定期è¿è¡Œæ‰«æ](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md) +- [跳过测试](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md) +- [查看报告](../how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md) +- [为 rancher-cis-benchmark å¯ç”¨å‘Šè­¦](../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) +- [为定时扫æé…置告警](../how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md) +- [创建è¦è¿è¡Œçš„自定义 Benchmark 版本](../how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cis-scans.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cis-scans.md new file mode 100644 index 00000000000..156d0ed946b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cis-scans.md @@ -0,0 +1,118 @@ +--- +title: CIS 扫æ +--- + +Rancher å¯ä»¥é€šè¿‡è¿è¡Œå®‰å…¨æ‰«ææ¥æ£€æŸ¥ Kubernetes 是å¦æŒ‰ç…§ CIS Kubernetes Benchmark 中定义的安全最佳实践进行部署。CIS 扫æå¯ä»¥è¿è¡Œåœ¨ä»»ä½• Kubernetes 集群，包括托管的 Kubernetes，例如 EKSã€AKS å’Œ GKE。 + +`rancher-cis-benchmark` 应用使用了 kube-bench ，这是 Aqua Security çš„å¼€æºå·¥å…·ï¼Œç”¨äºŽæ£€æŸ¥é›†ç¾¤æ˜¯å¦ç¬¦åˆ CIS Kubernetes Benchmark。此外，为了生æˆé›†ç¾¤çº§åˆ«çš„报告，此应用使用了 Sonobuoy æ¥èšåˆæŠ¥å‘Šã€‚ + + +## 关于 CIS Benchmark + +CIS（Center for Internet Security）是一个 501(c\)(3) éžè¥åˆ©ç»„织，æˆç«‹äºŽ 2000 å¹´ 10 月，其使命是识别ã€å¼€å‘ã€éªŒè¯ã€ä¿ƒè¿›å’Œç»´æŒç½‘络防御的最佳实践方案，并建立和指导社区，以在网络空间中è¥é€ ä¿¡ä»»çš„环境。该组织总部ä½äºŽçº½çº¦ä¸œæ ¼æž—布什，其æˆå‘˜åŒ…括大公å¸ã€æ”¿åºœæœºæž„和学术机构。 + +CIS Benchmark 是目标系统安全é…置的最佳实践。CIS Benchmark 是由安全专家ã€æŠ€æœ¯ä¾›åº”商ã€å…¬å¼€å’Œç§äººç¤¾åŒºæˆå‘˜ï¼Œä»¥åŠ CIS Benchmark å¼€å‘团队共åŒå¿—æ„¿å¼€å‘的。 + +在 CIS 网站上[注册](https://learn.cisecurity.org/benchmarks)以查看官方 Benchmark 文档。 + +## 关于生æˆçš„报告 + +æ¯æ¬¡æ‰«æ都会生æˆä¸€ä»½æŠ¥å‘Šï¼Œä½ å¯ä»¥åœ¨ Rancher UI 中查看该报告，并以 CSV æ ¼å¼ä¸‹è½½å®ƒã€‚ + +默认情况下使用 CIS Benchmark v1.6。 + +Benchmark 版本包å«åœ¨ç”Ÿæˆçš„报告中。 + +Benchmark æ供两ç§ç±»åž‹çš„建议，分别是自动（Automated）和手动（Manual）。Benchmark 中标记为 Manual 的建议ä¸åŒ…å«åœ¨ç”Ÿæˆçš„报告中。 + +一些测试会被标记为“ä¸é€‚用â€ã€‚由于 Rancher é…ç½® RKE 集群的方å¼ï¼Œè¿™äº›æµ‹è¯•ä¸ä¼šåœ¨ä»»ä½• CIS 扫æ中è¿è¡Œã€‚有关如何审核测试结果，以åŠä¸ºä»€ä¹ˆæŸäº›æµ‹è¯•ä¼šè¢«æ ‡è®°ä¸ºä¸é€‚用，请å‚阅 Rancher çš„ Kubernetes 对应版本的[自测指å—](./rancher-security.md#cis-benchmark-和自我评估)。 + +该报告包å«ä»¥ä¸‹ä¿¡æ¯ï¼š + +| 报告中的列 | æè¿° | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | CIS Benchmark çš„ ID å·ã€‚ | +| `description` | CIS Benchmark 测试的æ述。 | +| `remediation` | 为了通过测试需è¦ä¿®å¤çš„内容。 | +| `state` | 测试的状æ€ï¼Œå¯ä»¥æ˜¯é€šè¿‡ã€å¤±è´¥ã€è·³è¿‡æˆ–ä¸é€‚用。 | +| `node_type` | 节点角色，角色决定了在节点上è¿è¡Œçš„测试。主测试在 controlplane 节点上è¿è¡Œï¼Œetcd 测试在 etcd 节点上è¿è¡Œï¼ŒèŠ‚点测试在 Worker 节点上è¿è¡Œã€‚ | +| `audit` | 这是 `kube-bench` 为此测试è¿è¡Œçš„审计检查。 | +| `audit_config` | 适用于审计脚本的任何é…置。 | +| `test_info` | `kube-bench` 报告的测试相关信æ¯ï¼ˆå¦‚果存在）。 | +| `commands` | `kube-bench` 报告的测试相关的命令（如果存在）。 | +| `config_commands` | `kube-bench` 报告的测试相关的é…置数æ®ï¼ˆå¦‚果存在）。 | +| `actual_value` | 测试的实际值。如果由 `kube-bench` 报告，则会显示。 | +| `expected_result` | 测试的预期值。如果由 `kube-bench` 报告，则会显示。 | + +请å‚阅[集群加固指å—中的表格](./rancher-security.md)，以了解 Kubernetesã€Benchmarkã€Rancher 以åŠæˆ‘们的集群强化指å—的版本对应关系。å¦å¤–，请å‚阅强化指å—，以获å–ç¬¦åˆ CIS 的集群的é…置文件以åŠä¿®å¤å¤±è´¥æµ‹è¯•çš„ä¿¡æ¯ã€‚ + +## 测试é…置文件 + +以下是å¯ç”¨çš„é…置文件： + +- Generic CIS 1.6 +- Generic CIS 1.20 +- Generic CIS 1.23 +- RKE permissive 1.6 +- RKE hardened 1.6 +- RKE permissive 1.20 +- RKE hardened 1.20 +- RKE permissive 1.23 +- RKE hardened 1.23 +- RKE2 permissive 1.6 +- RKE2 hardened 1.6 +- RKE2 permissive 1.20 +- RKE2 hardened 1.20 +- RKE2 permissive 1.23 +- RKE2 hardened 1.23 +- K3s permissive 1.6 +- K3s hardened 1.6 +- K3s permissive 1.20 +- K3s hardened 1.20 +- K3s permissive 1.23 +- K3s hardened 1.23 +- AKS +- EKS +- GKE + +你还å¯ä»¥é€šè¿‡ä¿å­˜ä¸€ç»„è¦è·³è¿‡çš„测试æ¥è‡ªå®šä¹‰é…置文件。 + +所有é…置文件都会有一组ä¸é€‚用的测试，CIS 扫æ会跳过这些测试。RKE é›†ç¾¤ç®¡ç† Kubernetes çš„æ–¹å¼å¯¼è‡´è¿™äº›æµ‹è¯•è¢«è®¤ä¸ºä¸é€‚用。 + +RKE 集群扫æé…置文件有两ç§ç±»åž‹ï¼š + +- **Permissive**：此é…置文件有一组è¦è·³è¿‡çš„测试，跳过的原因是这些测试会在默认的 RKE Kubernetes 集群上失败。除了跳过的测试列表之外，é…置文件也ä¸ä¼šè¿è¡Œä¸é€‚用的测试。 +- **Hardened**：此é…置文件ä¸ä¼šè·³è¿‡ä»»ä½•æµ‹è¯•ï¼ˆä¸é€‚用的测试除外）。 + +EKS å’Œ GKE 集群扫æçš„é…置文件基于这些集群类型特定的 CIS Benchmark 版本。 + +è¦é€šè¿‡ “Hardened†é…置文件，你需è¦éµä»Ž[强化指å—](./rancher-security.md#rancher-强化指å—)并使用强化指å—中定义的 `cluster.yml` æ¥é…置一个强化集群。 + +默认é…置文件和支æŒçš„ CIS Benchmark 版本å–决于扫æ的集群类型： + +`rancher-cis-benchmark` æ”¯æŒ CIS 1.6 Benchmark 版本。 + +- RKE Kubernetes 集群默认使用 RKE Permissive 1.6 é…置文件。 +- EKS å’Œ GKE 有自己的 CIS Benchmark，由 `kube-bench` å‘布。这些集群默认使用相应的测试é…置文件。 +- RKE2 Kubernetes 集群默认使用 RKE2 Permissive 1.6 é…置文件。 +- RKEã€RKE2ã€EKS å’Œ GKE 以外的集群类型默认使用 Generic CIS 1.5 é…置文件。 + +## 跳过和ä¸é€‚用的测试 + +有关è¦è·³è¿‡å’Œä¸é€‚用的测试列表，请å‚阅[此页é¢](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md)。 + +ç›®å‰ï¼Œåªæœ‰ç”¨æˆ·å®šä¹‰çš„跳过测试会在生æˆæŠ¥å‘Šä¸­æ ‡è®°ä¸ºè·³è¿‡ã€‚ + +如果æŸä¸ªé»˜è®¤é…置文件将æŸä¸ªæµ‹è¯•å®šä¹‰ä¸ºè·³è¿‡ï¼Œåˆ™è¯¥æµ‹è¯•ä¹Ÿä¼šæ ‡è®°ä¸ºä¸é€‚用。 + +## RBAC + +有关æƒé™çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/cis-scans/rbac-for-cis-scans.md)。 + +## é…ç½® + +有关为扫æã€é…置文件和 Benchmark 版本é…置自定义资æºçš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/cis-scans/configuration-reference.md)。 + +## æ“ä½œæŒ‡å— + +è¦äº†è§£å¦‚何è¿è¡Œ CIS 扫æ，请å‚阅 [CIS 扫æ指å—](../pages-for-subheaders/cis-scan-guides.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cli-with-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cli-with-rancher.md new file mode 100644 index 00000000000..f1e68727a8d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cli-with-rancher.md @@ -0,0 +1,5 @@ +--- +title: Rancher CLI +--- + +Rancher CLI 是一个命令行工具，用于在工作站中与 Rancher 进行交互。以下文档将æè¿° [Rancher CLI](../reference-guides/cli-with-rancher/rancher-cli.md) å’Œ [kubectl Utility](../reference-guides/cli-with-rancher/kubectl-utility.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cloud-marketplace.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cloud-marketplace.md new file mode 100644 index 00000000000..1ef64aa10a8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cloud-marketplace.md @@ -0,0 +1,7 @@ +--- +title: äº‘å¸‚åœºé›†æˆ +--- + +Rancher æ供与云市场的集æˆï¼Œè®©ä½ èƒ½è½»æ¾è´­ä¹°äº‘æ供商上的安装支æŒã€‚此外，该集æˆè¿˜æ”¯æŒç”Ÿæˆ supportconfig bundle，你å¯ä»¥å°†è¯¥ bundle æ供给 Rancher。 + +此集æˆä»…æ”¯æŒ AWS。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cluster-configuration.md new file mode 100644 index 00000000000..68908371deb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/cluster-configuration.md @@ -0,0 +1,29 @@ +--- +title: 集群é…ç½® +--- + +使用 Rancher é…ç½® Kubernetes 集群åŽï¼Œä½ ä»ç„¶å¯ä»¥ç¼–辑集群的选项和设置。 + +有关编辑集群æˆå‘˜èµ„格的信æ¯ï¼Œè¯·è½¬è‡³[此页é¢](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md)。 + +### 集群é…ç½®å‚考 + +集群é…置选项å–决于 Kubernetes 集群的类型： + +- [RKE 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [GKE 集群é…ç½®](gke-cluster-configuration.md) +- [AKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) + +### ä¸åŒç±»åž‹é›†ç¾¤çš„管ç†åŠŸèƒ½ + +对于已有集群而言，å¯æ供的选项和设置å–决于你é…置集群的方法。 + +下表总结了æ¯ä¸€ç§ç±»åž‹çš„集群和对应的å¯ç¼–辑的选项和设置： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configuration-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configuration-options.md new file mode 100644 index 00000000000..f98a4990ceb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configuration-options.md @@ -0,0 +1,39 @@ +--- +title: é…置选项 +--- + +### Egress æ”¯æŒ + +默认情况下，Egress 网关是ç¦ç”¨çš„，但你å¯ä»¥åœ¨å®‰è£…或å‡çº§æ—¶ä½¿ç”¨ values.yaml 或[覆盖文件](#覆盖文件)å¯ç”¨å®ƒã€‚ + +### å¯ç”¨è‡ªåŠ¨ Sidecar 注入 + +默认情况下，自动 sidecar 注入是ç¦ç”¨çš„。è¦å¯ç”¨æ­¤åŠŸèƒ½ï¼Œè¯·åœ¨å®‰è£…或å‡çº§æ—¶åœ¨ values.yaml 中设置 `sidecarInjectorWebhook.enableNamespacesByDefault=true`。这会自动将 Istio sidecar 注入到所有已部署的新命å空间。 + +### 覆盖文件 + +覆盖文件用于为 Istio 进行更广泛的é…置。它å…许你更改 [IstioOperator API](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/) 中å¯ç”¨çš„任何值。你å¯ä»¥è‡ªå®šä¹‰é»˜è®¤å®‰è£…以满足你的需求。 + +覆盖文件将在 Istio Chart 默认安装的基础上添加é…置。æ¢è¨€ä¹‹ï¼Œä½ ä¸éœ€è¦ä¸ºå®‰è£…中已定义的组件进行é‡æ–°å®šä¹‰ã€‚ + +有关覆盖文件的更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](https://istio.io/latest/docs/setup/install/istioctl/#configure-component-settings)。 + +### 选择器和抓å–é…ç½® + +Monitoring 应用设置了 `prometheus.prometheusSpec.ignoreNamespaceSelectors=false`，å³åœ¨é»˜è®¤æƒ…况下跨所有命å空间进行监控。这样，你å¯ä»¥æŸ¥çœ‹éƒ¨ç½²åœ¨å…·æœ‰ `istio-injection=enabled` 标签的命å空间中的资æºçš„æµé‡ã€æŒ‡æ ‡å’Œå›¾ã€‚ + +如果你想将 Prometheus é™åˆ¶ä¸ºç‰¹å®šçš„命å空间，请设置 `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`。完æˆæ­¤æ“作åŽï¼Œä½ éœ€è¦æ·»åŠ å…¶ä»–é…ç½®æ¥ç»§ç»­ç›‘控你的资æºã€‚ + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)。 + +### 在具有 Pod 安全策略的情况下å¯ç”¨ Istio + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/pod-security-policies.md)。 + +### 在 RKE2 集群上安装 Istio 的其他步骤 + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md)。 + +### 项目网络隔离的其他步骤 + +详情请å‚阅[本节](../integrations-in-rancher/istio/configuration-options/project-network-isolation.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md new file mode 100644 index 00000000000..9cb1a64ca66 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md @@ -0,0 +1,38 @@ +--- +title: é…ç½® Microsoft AD FS (SAML) +--- + +如果你的组织使用 Microsoft è”åˆèº«ä»½éªŒè¯æœåŠ¡ (AD FS) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 AD FS 凭è¯ç™»å½•ã€‚ + +## 先决æ¡ä»¶ + +已安装 Rancher。 + +- 获å–ä½ çš„ Rancher Server URL。é…ç½® AD FS 时，请使用该 URL æ›¿æ¢ `` å ä½ç¬¦ã€‚ +- ä½ çš„ Rancher 必须具有全局管ç†å‘˜è´¦å·ã€‚ + +ä½ å¿…é¡»é…ç½® [Microsoft AD FS æœåŠ¡å™¨](https://docs.microsoft.com/en-us/windows-server/identity/active-directory-federation-services)。 + +- 获å–ä½ çš„ AD FS æœåŠ¡å™¨ IP/DNS å称。é…ç½® AD FS 时，请使用该 IP/DNS åç§°æ›¿æ¢ `` å ä½ç¬¦ã€‚ +- 你必须有在 AD FS æœåŠ¡å™¨ä¸Šæ·»åŠ  [Relying Party Trusts](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) çš„æƒé™ã€‚ + +## é…ç½®æ¦‚è¦ + +è¦è®© Rancher Server 使用 Microsoft AD FS，你需è¦åœ¨ Active Directory æœåŠ¡å™¨ä¸Šé…ç½® AD FS，并将 Rancher é…置为使用 AD FS æœåŠ¡å™¨ã€‚如果需è¦èŽ·å–在 Rancher 中设置 Microsoft AD FS 身份验è¯çš„指å—，请å‚è§ï¼š + +- [1. 在 Microsoft AD FS 中é…ç½® Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) +- [2. 在 Rancher 中é…ç½® Microsoft AD FS](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md) + +:::note SAML 身份æ供商注æ„事项 + +- SAML åè®®ä¸æ”¯æŒæœç´¢æˆ–查找用户或组。因此，将用户或组添加到 Rancher æ—¶ä¸ä¼šå¯¹å…¶è¿›è¡ŒéªŒè¯ã€‚ +- 添加用户时，必须正确输入确切的用户 IDï¼ˆå³ `UID` 字段）。键入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +::: + + +### åŽç»­æ“作 + +[在 Microsoft AD FS 中é…ç½® Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-openldap.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-openldap.md new file mode 100644 index 00000000000..5b8f3d014d9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-openldap.md @@ -0,0 +1,52 @@ +--- +title: é…ç½® OpenLDAP +--- + +如果你的组织使用 LDAP 进行用户身份验è¯ï¼Œåˆ™å¯ä»¥é…ç½® Rancher 与 OpenLDAP æœåŠ¡å™¨é€šä¿¡ï¼Œä»Žè€Œå¯¹ç”¨æˆ·è¿›è¡Œèº«ä»½éªŒè¯ã€‚这使 Rancher 管ç†å‘˜å¯ä»¥å¯¹å¤–部用户系统中的用户和组进行集群和项目的访问控制，åŒæ—¶å…许最终用户在登录 Rancher UI 时使用 LDAP 凭è¯è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +## 先决æ¡ä»¶ + +必须为 Rancher é…ç½® LDAP 绑定账å·ï¼ˆå³ ServiceAccount），æ¥æœç´¢å’Œæ£€ç´¢åº”该具有访问æƒé™çš„用户和组的 LDAP æ¡ç›®ã€‚建议ä¸è¦ä½¿ç”¨ç®¡ç†å‘˜è´¦å·æˆ–个人账å·ï¼Œè€Œåº”在 OpenLDAP 中创建一个专用账å·ï¼Œè¯¥è´¦å·å¯¹é…置的æœç´¢åº“下的用户和组需è¦å…·æœ‰åªè¯»æƒé™ï¼ˆå‚è§ä¸‹æ–‡ï¼‰ã€‚ + +> **使用 TLS？** +> +> 如果 OpenLDAP æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +## 在 Rancher 中é…ç½® OpenLDAP + +é…ç½® OpenLDAP æœåŠ¡å™¨ï¼Œç»„和用户的设置。有关填写æ¯ä¸ªå­—段的帮助，请å‚è§[é…ç½®å‚考](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md)。 + +> 在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **OpenLDAP**。填写**é…ç½® OpenLDAP æœåŠ¡å™¨**表å•ã€‚ +1. 点击**å¯ç”¨**。 + +### æµ‹è¯•èº«ä»½éªŒè¯ + +完æˆé…ç½®åŽï¼Œè¯·æµ‹è¯•ä¸Ž OpenLDAP æœåŠ¡å™¨çš„连接。如果测试æˆåŠŸï¼Œåˆ™è¡¨æ˜Ž OpenLDAP 身份验è¯å·²å¯ç”¨ã€‚ + +:::note + +与此步骤中输入的凭è¯ç›¸å…³çš„ OpenLDAP 用户将映射到本地主体账å·ï¼Œå¹¶åœ¨ Rancher 中分é…系统管ç†å‘˜æƒé™ã€‚因此，你应该决定使用哪个 OpenLDAP è´¦å·æ¥æ‰§è¡Œæ­¤æ­¥éª¤ã€‚ + +::: + +1. 输入应映射到本地主体账å·çš„ OpenLDAP è´¦å·çš„**用户å**å’Œ**密ç ** 。 +2. 点击**å¯ç”¨ OpenLDAP 认è¯**æ¥æµ‹è¯• OpenLDAP 的连接并完æˆè®¾ç½®ã€‚ + +**结果**： + +- OpenLDAP 验è¯é…ç½®æˆåŠŸã€‚ +- 与输入凭è¯å¯¹åº”çš„ LDAP 用户被映射到本地主体（管ç†å‘˜ï¼‰è´¦å·ã€‚ + +:::note + +如果 LDAP æœåŠ¡ä¸­æ–­ï¼Œä½ ä»ç„¶å¯ä»¥ä½¿ç”¨æœ¬åœ°é…置的 `admin` è´¦å·å’Œå¯†ç ç™»å½•ã€‚ + +::: + +## 附录：故障排除 + +如果在测试与 OpenLDAP æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·é¦–先仔细检查为 ServiceAccount 输入的凭è¯ä»¥åŠæœç´¢åº“é…置。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-shibboleth-saml.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-shibboleth-saml.md new file mode 100644 index 00000000000..a5908ccbfee --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/configure-shibboleth-saml.md @@ -0,0 +1,102 @@ +--- +title: é…ç½® Shibboleth (SAML) +--- + +如果你的组织使用 Shibboleth Identity Provider (IdP)) 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许用户使用 Shibboleth 凭è¯ç™»å½•ã€‚ + +在此é…置中，当 Rancher 用户登录时，他们将被é‡å®šå‘到 Shibboleth IdP æ¥è¾“入凭è¯ã€‚身份验è¯ç»“æŸåŽï¼Œä»–们将被é‡å®šå‘回 Rancher UI。 + +如果你将 OpenLDAP é…置为 Shibboleth çš„åŽç«¯ï¼ŒSAML 断言会返回到 Rancher，其中包括用于引用组的用户属性。然åŽï¼Œé€šè¿‡èº«ä»½éªŒè¯çš„用户将能够访问其所在的组有æƒè®¿é—®çš„ Rancher 资æºã€‚ + +> 本节å‡å®šä½ å·²äº†è§£ Rancher，Shibboleth å’Œ OpenLDAP 是如何ååŒå·¥ä½œçš„。有关工作原ç†çš„详细说明，请å‚è§[本页](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md)。 + + +## 在 Rancher 中设置 Shibboleth + +### Shibboleth 先决æ¡ä»¶ +> +> - ä½ å¿…é¡»é…置了 Shibboleth IdP æœåŠ¡å™¨ã€‚ +> - 以下是 Rancher Service Provider é…置所需的 URL： +> å…ƒæ•°æ® URL：`https:///v1-saml/shibboleth/saml/metadata` +> 断言使用者æœåŠ¡ (ACS) URL：`https:///v1-saml/shibboleth/saml/acs` +> - 从 IdP æœåŠ¡å™¨å¯¼å‡º `metadata.xml` 文件。详情请å‚è§ [Shibboleth 文档](https://wiki.shibboleth.net/confluence/display/SP3/Home)。 + +### 在 Rancher 中é…ç½® Shibboleth + +如果你的组织使用 Shibboleth 进行用户身份验è¯ï¼Œä½ å¯ä»¥é€šè¿‡é…ç½® Rancher æ¥å…许你的用户使用 IdP 凭è¯ç™»å½•ã€‚ + +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Shibboleth**。 +1. 填写**é…ç½® Shibboleth è´¦å·**表å•ã€‚Shibboleth IdP å…许你指定è¦ä½¿ç”¨çš„æ•°æ®å­˜å‚¨ã€‚ä½ å¯ä»¥æ·»åŠ æ•°æ®åº“或使用现有的 ldap æœåŠ¡å™¨ã€‚例如，如果你选择 Active Directory (AD) æœåŠ¡å™¨ï¼Œä¸‹é¢çš„示例将æ述如何将 AD 属性映射到 Rancher 中的字段： + + 1. **显示å称字段**：包å«ç”¨æˆ·æ˜¾ç¤ºå称的 AD 属性（例如：`displayName`）。 + + 1. **用户å字段**：包å«ç”¨æˆ·å/给定å称的 AD 属性（例如：`givenName`）。 + + 1. **UID 字段**：æ¯ä¸ªç”¨æˆ·å”¯ä¸€çš„ AD 属性（例如：`sAMAccountName`ã€`distinguishedName`）。 + + 1. **用户组字段**: 创建用于管ç†ç»„æˆå‘˜å…³ç³»çš„æ¡ç›®ï¼ˆä¾‹å¦‚：`memberOf`）。 + + 1. **Rancher API 主机**：你的 Rancher Server çš„ URL。 + + 1. **ç§é’¥**å’Œ**è¯ä¹¦**：密钥/è¯ä¹¦å¯¹ï¼Œç”¨äºŽåœ¨ Rancher 和你的 IdP 之间创建一个安全外壳（SSH）。 + + ä½ å¯ä»¥ä½¿ç”¨ openssl 命令进行创建。例如： + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP 元数æ®**：从 IdP æœåŠ¡å™¨å¯¼å‡ºçš„ `metadata.xml` 文件。 + + +1. 完æˆ**é…ç½® Shibboleth è´¦å·**表å•åŽï¼Œå•å‡»**å¯ç”¨**。 + + Rancher 会将你é‡å®šå‘到 IdP 登录页é¢ã€‚输入使用 Shibboleth IdP 进行身份验è¯çš„凭è¯ï¼Œæ¥éªŒè¯ä½ çš„ Rancher Shibboleth é…置。 + + :::note + + ä½ å¯èƒ½éœ€è¦ç¦ç”¨å¼¹å‡ºçª—å£é˜»æ­¢ç¨‹åºæ‰èƒ½çœ‹åˆ° IdP 登录页é¢ã€‚ + + ::: + +**结果**：已将 Rancher é…置为使用 Shibboleth。你的用户现在å¯ä»¥ä½¿ç”¨ Shibboleth 登录å登录 Rancher。 + +### SAML æ供商注æ„事项 + +SAML åè®®ä¸æ”¯æŒç”¨æˆ·æˆ–用户组的æœç´¢æˆ–查找。因此，如果你没有为 Shibboleth é…ç½® OpenLDAP，则请留æ„以下警告。 + +- 在 Rancher 中为用户或组分é…æƒé™æ—¶ï¼Œä¸ä¼šå¯¹ç”¨æˆ·æˆ–组进行验è¯ã€‚ +- 添加用户时，必须正确输入准确的用户 IDï¼ˆå³ UID 字段）。在你输入用户 ID 时，将ä¸ä¼šæœç´¢å¯èƒ½åŒ¹é…的其他用户 ID。 +- 添加组时，必须从文本框æ—边的下拉列表中选择组。Rancher å‡å®šæ¥è‡ªæ–‡æœ¬æ¡†çš„任何输入都是用户。 +- 用户组下拉列表仅显示你所属的用户组。如果你ä¸æ˜¯æŸä¸ªç»„çš„æˆå‘˜ï¼Œä½ å°†æ— æ³•æ·»åŠ è¯¥ç»„。 + +è¦åœ¨ Rancher 中分é…æƒé™æ—¶å¯ç”¨æœç´¢ç»„，你需è¦ä¸ºæ”¯æŒç»„çš„ SAML 身份验è¯æ供商é…ç½®åŽç«¯ï¼ˆä¾‹å¦‚ OpenLDAP）。 + +## 在 Rancher 中设置 OpenLDAP + +如果你将 OpenLDAP é…置为 Shibboleth çš„åŽç«¯ï¼ŒSAML 断言会返回到 Rancher，其中包括用于引用组的用户属性。然åŽï¼Œé€šè¿‡èº«ä»½éªŒè¯çš„用户将能够访问其所在的组有æƒè®¿é—®çš„ Rancher 资æºã€‚ + +### OpenLDAP 先决æ¡ä»¶ + +必须为 Rancher é…ç½® LDAP 绑定账å·ï¼ˆå³ ServiceAccount），æ¥æœç´¢å’Œæ£€ç´¢åº”该具有访问æƒé™çš„用户和组的 LDAP æ¡ç›®ã€‚建议ä¸è¦ä½¿ç”¨ç®¡ç†å‘˜è´¦å·æˆ–个人账å·ï¼Œè€Œåº”在 OpenLDAP 中创建一个专用账å·ï¼Œè¯¥è´¦å·å¯¹é…置的æœç´¢åº“下的用户和组需è¦å…·æœ‰åªè¯»æƒé™ï¼ˆå‚è§ä¸‹æ–‡ï¼‰ã€‚ + +> **使用 TLS？** +> +> 如果 OpenLDAP æœåŠ¡å™¨ä½¿ç”¨çš„è¯ä¹¦æ˜¯è‡ªç­¾å的或ä¸æ˜¯æ¥è‡ªè®¤å¯çš„è¯ä¹¦é¢å‘机构，请确ä¿æ‰‹å¤´æœ‰ PEM æ ¼å¼çš„ CA è¯ä¹¦ï¼ˆåŒ…å«æ‰€æœ‰ä¸­é—´è¯ä¹¦ï¼‰ã€‚你必须在é…置期间粘贴此è¯ä¹¦ï¼Œä»¥ä¾¿ Rancher 能够验è¯è¯ä¹¦é“¾ã€‚ + +### 在 Rancher 中é…ç½® OpenLDAP + +é…ç½® OpenLDAP æœåŠ¡å™¨ï¼Œç»„和用户的设置。有关填写æ¯ä¸ªå­—段的帮助，请å‚è§[é…ç½®å‚考](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md)。请注æ„，嵌套组æˆå‘˜èµ„æ ¼ä¸é€‚用于 Shibboleth。 + +> 在开始之å‰ï¼Œè¯·ç†Ÿæ‚‰[外部身份验è¯é…置和主体用户](authentication-config.md#外部身份验è¯é…置和用户主体)的概念。 + +1. 使用åˆå§‹çš„本地 `admin` è´¦å·ç™»å½•åˆ° Rancher UI。 +1. 在左上角，å•å‡» **☰ > 用户 & 认è¯**。 +1. 在左侧导航æ ï¼Œå•å‡»**认è¯**。 +1. å•å‡» **Shibboleth**，如果已é…ç½® SAML，则å•å‡»**编辑é…ç½®**。 +1. 在**用户和组æœç´¢**下，选中**é…ç½® OpenLDAP Server**。 + +## 故障排除 + +如果在测试与 OpenLDAP æœåŠ¡å™¨çš„连接时é‡åˆ°é—®é¢˜ï¼Œè¯·é¦–先仔细检查为 ServiceAccount 输入的凭è¯ä»¥åŠæœç´¢åº“é…置。你还å¯ä»¥æ£€æŸ¥ Rancher 日志æ¥æŸ¥æ˜Žé—®é¢˜çš„原因。调试日志å¯èƒ½åŒ…å«æœ‰å…³é”™è¯¯çš„更详细信æ¯ã€‚详情请å‚è§[如何å¯ç”¨è°ƒè¯•æ—¥å¿—](../faq/technical-items.md#如何å¯ç”¨è°ƒè¯•æ—¥å¿—记录)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/create-kubernetes-persistent-storage.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/create-kubernetes-persistent-storage.md new file mode 100644 index 00000000000..e59d36c843d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/create-kubernetes-persistent-storage.md @@ -0,0 +1,74 @@ +--- +title: "Kubernetes æŒä¹…存储：å·å’Œå­˜å‚¨ç±»" +description: "了解在 Kubernetes 中创建æŒä¹…存储的两ç§æ–¹å¼ï¼šæŒä¹…å·å’Œå­˜å‚¨ç±»" +--- + +在部署需è¦ä¿â€‹â€‹ç•™æ•°æ®çš„应用时，你需è¦åˆ›å»ºæŒä¹…存储。æŒä¹…存储å…许你在è¿è¡Œåº”用的 pod 之外存储应用数æ®ã€‚å³ä½¿è¿è¡Œåº”用的 pod å‘生故障，这ç§å­˜å‚¨æ–¹å¼ä¹Ÿèƒ½è®©ä½ ä¿ç•™åº”用数æ®ã€‚ + +本文å‡è®¾ä½ å·²äº†è§£ Kubernetes çš„æŒä¹…å·ã€æŒä¹…å·å£°æ˜Žå’Œå­˜å‚¨ç±»çš„概念。如需更多信æ¯ï¼Œè¯·å‚阅[存储的工作原ç†](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md)。 + +### 先决æ¡ä»¶ + +设置æŒä¹…存储需è¦`管ç†å·`çš„[角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色å‚考)。 + +如果你è¦ä¸ºäº‘集群é…置存储，则存储和集群主机必须使用相åŒçš„云æ供商。 + +è¦ä½¿ç”¨ Rancher é…置新存储，则必须å¯ç”¨äº‘æ供商。有关å¯ç”¨äº‘æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../pages-for-subheaders/set-up-cloud-providers.md)。 + +如果è¦å°†çŽ°æœ‰çš„æŒä¹…存储连接到集群，则ä¸éœ€è¦å¯ç”¨äº‘æ供商。 + +### 设置现有存储 + +设置现有存储的总体æµç¨‹å¦‚下： + +1. 设置你的æŒä¹…存储。å¯ä»¥æ˜¯äº‘存储或你自己的存储。 +2. 添加引用æŒä¹…存储的æŒä¹…å· (PV)。 +3. 添加引用 PV çš„æŒä¹…å·å£°æ˜Ž (PVC)。 +4. å°† PVC 挂载为工作负载中的å·ã€‚ + +有关详细信æ¯å’Œå…ˆå†³æ¡ä»¶ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md)。 + +### 在 Rancher 中动æ€é…置新存储 + +é…置新存储的总体æµç¨‹å¦‚下： + +1. 添加一个 StorageClass 并将它é…置为使用你的存储æ供商。StorageClass å¯ä»¥å¼•ç”¨äº‘存储或你自己的存储。 +2. 添加引用存储类的æŒä¹…å·å£°æ˜Ž (PVC)。 +3. å°† PVC 挂载为工作负载的å·ã€‚ + +有关详细信æ¯å’Œå…ˆå†³æ¡ä»¶ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md)。 + +### Longhorn 存储 + +[Longhorn](https://longhorn.io/) 是一个轻é‡çº§ã€å¯é ã€æ˜“用的 Kubernetes 分布å¼å—存储系统。 + +Longhorn 是å…费的开æºè½¯ä»¶ã€‚Longhorn 最åˆç”± Rancher Labs å¼€å‘，现在正在作为云原生计算基金会的沙盒项目进行开å‘。它å¯ä»¥é€šè¿‡ Helmã€kubectl 或 Rancher UI 安装在任何 Kubernetes 集群上。 + +如果你有å—存储池，Longhorn å¯ä»¥å¸®åŠ©ä½ ä¸º Kubernetes 集群æä¾›æŒä¹…存储，而无需ä¾èµ–云æ供商。有关 Longhorn 功能的更多信æ¯ï¼Œè¯·å‚阅[文档](https://longhorn.io/docs/latest/what-is-longhorn/)。 + +Rancher v2.5 简化了在 Rancher 管ç†çš„集群上安装 Longhorn 的过程。详情请å‚è§[本页é¢](../integrations-in-rancher/longhorn.md)。 + +### é…置存储示例 + +我们æ供了如何使用 [NFSã€](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) [vSphere](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) å’Œ [Amazon EBS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) æ¥é…置存储的示例。 + +### GlusterFS å· + +在将数æ®å­˜å‚¨åœ¨ GlusterFS å·ä¸Šçš„集群中，你å¯èƒ½ä¼šé‡åˆ°é‡å¯ `kubelet` åŽ pod 无法挂载å·çš„问题。有关é¿å…此情况å‘生的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md)。 + +### iSCSI å· + +在将数æ®å­˜å‚¨åœ¨ iSCSI å·ä¸Šçš„ [Rancher å¯åŠ¨çš„ Kubernetes 集群](../pages-for-subheaders/launch-kubernetes-with-rancher.md)中，你å¯èƒ½ä¼šé‡åˆ° kubelet 无法自动连接 iSCSI å·çš„问题。有关解决此问题的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md)。 + +### hostPath å· +在创建 hostPath å·ä¹‹å‰ï¼Œä½ éœ€è¦åœ¨é›†ç¾¤é…置中设置 [extra_bind](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds/)。这会将路径作为å·å®‰è£…在你的 kubelet 中，å¯ç”¨äºŽå·¥ä½œè´Ÿè½½ä¸­çš„ hostPath å·ã€‚ + +### å°† vSphere Cloud Provider 从树内è¿ç§»åˆ°æ ‘外 + +Kubernetes 正在é€æ¸ä¸åœ¨æ ‘内维护云æ供商。vSphere 有一个树外云æ供商，å¯é€šè¿‡å®‰è£… vSphere 云æ供商和云存储æ’件æ¥ä½¿ç”¨ã€‚ + +有关如何从树内 vSphere 云æ供商è¿ç§»åˆ°æ ‘外，以åŠå¦‚何在è¿ç§»åŽç®¡ç†çŽ°æœ‰è™šæ‹Ÿæœºï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 + +### 相关链接 + +- [Kubernetes 文档：存储](https://kubernetes.io/docs/concepts/storage/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/custom-resource-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/custom-resource-configuration.md new file mode 100644 index 00000000000..602ae0bfb63 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/custom-resource-configuration.md @@ -0,0 +1,8 @@ +--- +title: 自定义资æºé…ç½® +--- + +以下自定义资æºå®šä¹‰ï¼ˆCustom Resource Definition，CRD）用于é…ç½® Logging： + +- [Flow å’Œ ClusterFlow](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md) +- [Output å’Œ ClusterOutput](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-apps-across-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-apps-across-clusters.md new file mode 100644 index 00000000000..5a6382f71b1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-apps-across-clusters.md @@ -0,0 +1,15 @@ +--- +title: 跨集群部署应用 +--- + +### Fleet + +Rancher 2.5 引入了 Fleet，这是一ç§è·¨é›†ç¾¤éƒ¨ç½²åº”用的新方å¼ã€‚ + +使用 Fleet çš„æŒç»­äº¤ä»˜æ˜¯å¤§è§„模的 GitOps。如需更多信æ¯ï¼Œè¯·å‚阅 [Fleet](../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md)。 + +### 多集群应用 + +在 2.5 之å‰çš„ Rancher 版本中，多集群应用功能用于跨集群部署应用。我们已弃用多集群应用功能，但你ä»ç„¶å¯ä»¥åœ¨ Rancher 2.5 中使用该功能。 + +详情请å‚阅[此文档](../how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-rancher-manager.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-rancher-manager.md new file mode 100644 index 00000000000..c20dbce3552 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-rancher-manager.md @@ -0,0 +1,19 @@ +--- +title: 部署 Rancher Server +--- + +ä½ å¯ä½¿ç”¨ä»¥ä¸‹æŒ‡å—之一，在你选择的æ供商中部署和é…ç½® Rancher å’Œ Kubernetes 集群。 + +- [AWS](../getting-started/quick-start-guides/deploy-rancher-manager/aws.md)（使用 Terraform） +- [AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md)（使用 Amazon EKS） +- [Azure](../getting-started/quick-start-guides/deploy-rancher-manager/azure.md)（使用 Terraform） +- [DigitalOcean](../getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md)（使用 Terraform） +- [GCP](../getting-started/quick-start-guides/deploy-rancher-manager/gcp.md)（使用 Terraform） +- [Hetzner Cloud](../getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md)（使用 Terraform） +- [Vagrant](../getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md) +- [Equinix Metal](../getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md) +- [Outscale](../getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md)（使用 Terraform） + +如有需è¦ï¼Œä½ å¯ä»¥æŸ¥çœ‹ä»¥ä¸‹æŒ‡å—以了解分步步骤。如果你需è¦åœ¨å…¶ä»–æ供商中或本地è¿è¡Œ Rancher，或者你åªæ˜¯æƒ³çœ‹çœ‹å®ƒæ˜¯å¤šä¹ˆå®¹æ˜“上手，你å¯é˜…读以下指å—： + +- [手动安装](../getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-rancher-workloads.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-rancher-workloads.md new file mode 100644 index 00000000000..01d72560dde --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/deploy-rancher-workloads.md @@ -0,0 +1,8 @@ +--- +title: 部署工作负载 +--- + +这些指å—指导你完æˆä¸€ä¸ªåº”用的部署，包括如何将应用暴露在集群之外使用。 + +- [部署带有 Ingress 的工作负载](../getting-started/quick-start-guides/deploy-workloads/workload-ingress.md) +- [部署带有 NodePort 的工作负载](../getting-started/quick-start-guides/deploy-workloads/nodeports.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/downstream-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/downstream-cluster-configuration.md new file mode 100644 index 00000000000..3d09efe4a8e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/downstream-cluster-configuration.md @@ -0,0 +1,5 @@ +--- +title: 下游集群é…ç½® +--- + +以下文档将讨论[节点模æ¿é…ç½®](./node-template-configuration.md)å’Œ[主机é…ç½®](./machine-configuration.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/enable-experimental-features.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/enable-experimental-features.md new file mode 100644 index 00000000000..4f759f616fe --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/enable-experimental-features.md @@ -0,0 +1,125 @@ +--- +title: å¯ç”¨å®žéªŒåŠŸèƒ½ +--- + +Rancher 包å«ä¸€äº›é»˜è®¤å…³é—­çš„实验功能。在æŸäº›æƒ…况下，例如当你认为使用[ä¸æ”¯æŒçš„存储类型](../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md)的好处大于使用未ç»æµ‹è¯•çš„功能的风险时，你å¯èƒ½æƒ³è¦å¯ç”¨å®žéªŒåŠŸèƒ½ã€‚为了让你能够试用这些默认关闭的功能，我们引入了功能开关（feature flag）。 + +实验功能å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸‰ç§æ–¹å¼å¯ç”¨ï¼š + +- [使用 CLI](#å¯åŠ¨-rancher-æ—¶å¯ç”¨åŠŸèƒ½)：在使用 CLI 安装 Rancher 时，使用功能开关默认å¯ç”¨æŸä¸ªåŠŸèƒ½ã€‚ +- [使用 Rancher UI](#使用-rancher-ui-å¯ç”¨åŠŸèƒ½)：在**设置**页é¢å¯ç”¨åŠŸèƒ½ã€‚ +- [使用 Rancher API](#使用-rancher-api-å¯ç”¨åŠŸèƒ½)：安装 Rancher åŽå¯ç”¨åŠŸèƒ½ã€‚ + +æ¯ä¸ªåŠŸèƒ½å‡æœ‰ä»¥ä¸‹ä¸¤ä¸ªå€¼ï¼š + +- 默认值：å¯ä»¥é€šè¿‡åœ¨å‘½ä»¤è¡Œä½¿ç”¨æ ‡å¿—或环境å˜é‡è¿›è¡Œé…置。 +- 设置值：å¯ä»¥é€šè¿‡ Rancher API 或 UI 进行é…置。 + +如果没有设置值，Rancher 会使用默认值。 + +设置值是通过 API 设置的，而默认值是通过命令行设置。因此，如果你使用 API 或 UI å¯ç”¨æˆ–ç¦ç”¨æŸä¸ªåŠŸèƒ½ï¼Œå‘½ä»¤è¡Œä¸­è®¾ç½®çš„值将被覆盖。 + +如果你安装 Rancher åŽä½¿ç”¨ Rancher API 将功能开关设置为 true，然åŽåœ¨ä½¿ç”¨å‘½ä»¤å‡çº§ Rancher 时将功能开关设置为 false，在这ç§æƒ…况下，虽然默认值会是 false，但是该功能ä¾ç„¶ä¼šè¢«å¯ç”¨ï¼Œå› ä¸ºå®ƒæ˜¯é€šè¿‡ API 设置的。如果你éšåŽä½¿ç”¨ Rancher API 删除设置值（true）并将它设置为 NULL，则默认值（false）将生效。有关详细信æ¯ï¼Œè¯·å‚阅[功能开关页é¢](../getting-started/installation-and-upgrade/installation-references/feature-flags.md)。 + +## å¯åŠ¨ Rancher æ—¶å¯ç”¨åŠŸèƒ½ + +安装 Rancher 时，使用功能开关å¯ç”¨ä½ æ‰€éœ€çš„功能。通过å•èŠ‚点容器安装 Rancher，和在 Kubernetes 集群上安装 Rancher 对应的命令有所ä¸åŒã€‚ + +### Kubernetes 安装的情况下å¯ç”¨åŠŸèƒ½ + +:::note + +通过 Rancher API 设置的值会覆盖命令行传入的值。 + +::: + +使用 Helm Chart 安装 Rancher 时，使用 `--set` 选项。下é¢çš„示例通过传递功能开关å称（用逗å·åˆ†éš”）æ¥å¯ç”¨ä¸¤ä¸ªåŠŸèƒ½ï¼š + +对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +:::note + +如果你安装的是 alpha 版本，Helm è¦æ±‚你在命令中添加 `--devel` 选项。 + +::: + +### 离线安装的情况下渲染 Helm Chart + +如果你是在离线环境安装 Rancher 的，在使用 Helm 安装 Rancher 之å‰ï¼Œä½ éœ€è¦æ·»åŠ ä¸€ä¸ª Helm Chart 仓库并渲染一个 Helm 模æ¿ã€‚详情请å‚è§[离线安装文档](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md)。 + +以下是在渲染 Helm 模æ¿æ—¶ä¼ å…¥åŠŸèƒ½å¼€å…³å称的命令示例。下é¢çš„示例通过传递功能开关å称（用逗å·åˆ†éš”）æ¥å¯ç”¨ä¸¤ä¸ªåŠŸèƒ½ã€‚ + +Helm 命令如下： + +``` +helm template rancher ./rancher-.tgz --output-dir . \ + --no-hooks \ # é¿å…ç”Ÿæˆ Helm é’©å­æ–‡ä»¶ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # 设置在 Rancher 中使用的默认ç§æœ‰é•œåƒä»“库 + --set useBundledSystemChart=true # 使用打包的 Rancher System Chart + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +### Docker 安装的情况下å¯ç”¨åŠŸèƒ½ + +如果 Rancher 是使用 Docker 安装的，请使用 `--features` 选项。下é¢çš„示例通过传递功能开关å称（用逗å·åˆ†éš”）æ¥å¯ç”¨ä¸¤ä¸ªåŠŸèƒ½ï¼š + +``` +docker run -d -p 80:80 -p 443:443 \ + --restart=unless-stopped \ + rancher/rancher:rancher-latest \ + --features==true,=true +``` + + +## 使用 Rancher UI å¯ç”¨åŠŸèƒ½ + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å•å‡»**功能开关**。 +1. 如需å¯ç”¨æŸä¸ªåŠŸèƒ½ï¼Œæ‰¾åˆ°è¯¥å·²ç¦ç”¨çš„功能，并点击**â‹® > 激活**。 + +**结果**：该功能已å¯ç”¨ã€‚ + +### 使用 Rancher UI ç¦ç”¨åŠŸèƒ½ + +1. 在左上角，å•å‡» **☰ > 全局设置**。 +1. å•å‡»**功能开关**。你将看到实验功能列表。 +1. 如需ç¦ç”¨æŸä¸ªåŠŸèƒ½ï¼Œæ‰¾åˆ°è¯¥å·²å¯ç”¨çš„功能，并点击**â‹® > åœç”¨**。 + +**结果**：该功能已ç¦ç”¨ã€‚ + +## 使用 Rancher API å¯ç”¨åŠŸèƒ½ + +1. å‰å¾€ `/v3/features`。 +1. 在 `data` 中，你会看到一个数组，该数组包å«æ‰€æœ‰èƒ½é€šè¿‡åŠŸèƒ½å¼€å…³å¯ç”¨çš„功能。功能的å称在 `id` 字段中。å•å‡»è¦å¯ç”¨çš„功能的å称。 +1. 在左上角的 **Operations** 下，点击 **Edit**。 +1. 在 **Value** 下拉èœå•ä¸­ï¼Œå•å‡» **True**。 +1. å•å‡» **Show Request**。 +1. å•å‡» **Send Request**。 +1. 点击 **Close**。 + +**结果**：该功能已å¯ç”¨ã€‚ + +### 使用 Rancher API ç¦ç”¨åŠŸèƒ½ + +1. å‰å¾€ `/v3/features`。 +1. 在 `data` 中，你会看到一个数组，该数组包å«æ‰€æœ‰èƒ½é€šè¿‡åŠŸèƒ½å¼€å…³å¯ç”¨çš„功能。功能的å称在 `id` 字段中。å•å‡»è¦å¯ç”¨çš„功能的å称。 +1. 在左上角的 **Operations** 下，点击 **Edit**。 +1. 在 **Value** 下拉èœå•ä¸­ï¼Œå•å‡» **False**。 +1. å•å‡» **Show Request**。 +1. å•å‡» **Send Request**。 +1. 点击 **Close**。 + +**结果**：该功能已ç¦ç”¨ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/fleet-gitops-at-scale.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/fleet-gitops-at-scale.md new file mode 100644 index 00000000000..c27fe57a2cd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/fleet-gitops-at-scale.md @@ -0,0 +1,64 @@ +--- +title: 使用 Feet 进行æŒç»­äº¤ä»˜ +--- + +使用 Fleet çš„æŒç»­äº¤ä»˜æ˜¯å¤§è§„模的 GitOps。你å¯ä»¥ä½¿ç”¨ Fleet 管ç†å¤šè¾¾ä¸€ç™¾ä¸‡ä¸ªé›†ç¾¤ã€‚此外，它éžå¸¸è½»é‡ï¼Œå› æ­¤ä¹Ÿéžå¸¸é€‚用于[å•ä¸ªé›†ç¾¤](https://fleet.rancher.io/installation#default-install)。但是，它在[大规模](https://fleet.rancher.io/installation#configuration-for-multi-cluster)场景下的功能更加强大。大规模指的是大é‡é›†ç¾¤ã€å¤§é‡éƒ¨ç½²æˆ–大é‡å›¢é˜Ÿã€‚ + +Fleet 是一个独立于 Rancher 的项目，你å¯ä»¥ä½¿ç”¨ Helm 将它安装在任何 Kubernetes 集群上。 + + +## 架构 + +有关 Fleet 工作原ç†çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此处](../integrations-in-rancher/fleet-gitops-at-scale/architecture.md)。 + +## 在 Rancher UI 中访问 Fleet + +Fleet 预装在 Rancher 中，å¯ä»¥é€šè¿‡ Rancher UI 中的**æŒç»­äº¤ä»˜**选项进行管ç†ã€‚有关æŒç»­äº¤ä»˜å’Œ Fleet 故障排除技巧的更多信æ¯ï¼Œè¯·å‚阅[此处](https://fleet.rancher.io/troubleshooting)。 + +用户å¯ä»¥é€šè¿‡éµå¾ª **gitops** 的实践，利用æŒç»­äº¤ä»˜å°†åº”用部署到 git 仓库中的 Kubernetes 集群，而无需任何手动æ“作。 + +按照以下步骤在 Rancher UI 中访问æŒç»­äº¤ä»˜ï¼š + +1. å•å‡» **☰ > æŒç»­äº¤ä»˜**。 + +1. 在èœå•é¡¶éƒ¨é€‰æ‹©ä½ çš„命å空间，注æ„以下几点： + + - 默认情况下会选中 **fleet-default**，其中包括注册到 Rancher 的所有下游集群。 + + - ä½ å¯ä»¥åˆ‡æ¢åˆ°ä»…åŒ…å« **local** 集群的 **fleet-local**，或者创建自己的工作空间，并将集群分é…和移动到该工作空间。 + + - 然åŽï¼Œä½ å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„**集群**æ¥ç®¡ç†é›†ç¾¤ã€‚ + +1. å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸Šçš„ **Git 仓库**å°† git 仓库部署到当å‰å·¥ä½œç©ºé—´ä¸­çš„集群中。 + +1. 选择你的 [git 仓库](https://fleet.rancher.io/gitrepo-add)å’Œ[目标集群/集群组](https://fleet.rancher.io/gitrepo-targets)。你还å¯ä»¥å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸­çš„**集群组**在 UI 中创建集群组。 + +1. 部署 git 仓库åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 监控应用。 + +## Windows æ”¯æŒ + +有关对具有 Windows 节点的集群的支æŒï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md)。 + +## GitHub 仓库 + +ä½ å¯ä»¥å•å‡»æ­¤å¤„èŽ·å– [Fleet Helm Chart](https://github.com/rancher/fleet/releases)。 + +## 在代ç†åŽä½¿ç”¨ Fleet + +有关在代ç†åŽä½¿ç”¨ Fleet 的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md)。 + +## Helm Chart ä¾èµ– + +由于用户需è¦å®Œæˆä¾èµ–列表，因此为了æˆåŠŸéƒ¨ç½²å…·æœ‰ä¾èµ–项的 Helm Chart，你必须手动è¿è¡Œå‘½ä»¤ï¼ˆå¦‚下所列）。如果你ä¸è¿™æ ·åšï¼Œå¹¶ç»§ç»­å…‹éš†ä»“库并è¿è¡Œ `helm install`，由于ä¾èµ–项将丢失，因此你的安装将失败。 + +git 仓库中的 Helm Chart 必须在 Chart å­ç›®å½•ä¸­åŒ…å«å…¶ä¾èµ–项。你必须手动è¿è¡Œ `helm dependencies update $chart`，或在本地è¿è¡Œ `helm dependencies build $chart`，然åŽå°†å®Œæ•´çš„ Chart 目录æ交到你的 git 仓库。请注æ„，你需è¦ä½¿ç”¨é€‚当的å‚æ•°æ¥ä¿®æ”¹å‘½ä»¤ã€‚ + +## 故障排除 + +- **已知问题**：Fleet git 仓库的 clientSecretName å’Œ helmSecretName 密文ä¸åŒ…å«åœ¨ç”± [backup-restore-operator](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md#1-安装-rancher-backup-operator) 创建的备份或æ¢å¤ä¸­ã€‚如果我们有了永久的解决方案，我们将通知社区。 + +- **临时解决方法**：默认情况下，用户定义的密文ä¸ä¼šåœ¨ Fleet 中备份。如果执行ç¾éš¾æ¢å¤æˆ–å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤ï¼Œåˆ™éœ€è¦é‡æ–°åˆ›å»ºå¯†æ–‡ã€‚è¦ä¿®æ”¹ resourceSet 以包å«éœ€è¦å¤‡ä»½çš„其他资æºï¼Œè¯·å‚阅[此文档](https://github.com/rancher/backup-restore-operator#user-flow)。 + +## 文档 + +Fleet 文档链接：https://fleet.rancher.io/ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/gke-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/gke-cluster-configuration.md new file mode 100644 index 00000000000..7ae7e6292c2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/gke-cluster-configuration.md @@ -0,0 +1,321 @@ +--- +title: GKE 集群é…ç½®å‚考 +--- + +## Rancher 2.6 å˜æ›´ + +- 支æŒé¢å¤–çš„é…置选项： + - 项目网络隔离 + - 网络标签 + +## 集群ä½ç½® + +| 值 | æè¿° | +|--------|--------------| +| ä½ç½®ç±»åž‹ | 地区 (zone) 或区域 (region)。借助 GKE，你å¯ä»¥æ ¹æ®å·¥ä½œè´Ÿè½½çš„å¯ç”¨æ€§è¦æ±‚和预算创建一个é‡èº«å®šåˆ¶çš„集群。默认情况下，集群的节点在å•ä¸ªè®¡ç®—区域中è¿è¡Œã€‚选择多个区域时，集群的节点将跨越多个计算区域，而 controlplane 则åªä½äºŽå•ä¸ªåŒºåŸŸä¸­ã€‚区域集群也增加了 controlplane çš„å¯ç”¨æ€§ã€‚有关选择集群å¯ç”¨æ€§ç±»åž‹çš„帮助，请å‚阅[这些文档](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#choosing_a_regional_or_zonal_control_plane)。 | +| 地区 | 计算引擎中的æ¯ä¸ªåŒºåŸŸéƒ½åŒ…å«å¤šåœ°åŒºã€‚有关å¯ç”¨åŒºåŸŸå’Œå¯ç”¨åŒºçš„更多信æ¯ï¼Œè¯·å‚阅[这些文档](https://cloud.google.com/compute/docs/regions-zones#available)。 | +| 其他地区 | 对于地区性集群，你å¯ä»¥é€‰æ‹©å…¶ä»–地区æ¥åˆ›å»º[多地区集群](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#multi-zonal_clusters)。 | +| 区域 | 对于[区域性集群](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#regional_clusters)，你å¯ä»¥é€‰æ‹©ä¸€ä¸ªåŒºåŸŸã€‚有关å¯ç”¨åŒºåŸŸå’Œå¯ç”¨åŒºçš„更多信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/compute/docs/regions-zones#available)。地区å称的å‰é¢éƒ¨åˆ†æ˜¯åŒºåŸŸçš„å称。 | + +## 集群选项 + +### Kubernetes 版本 + +_å¯å˜ï¼šæ˜¯_ + +有关 GKE Kubernetes 版本的更多信æ¯ï¼Œè¯·å‚阅[这些文档](https://cloud.google.com/kubernetes-engine/versioning)。 + +### 容器地å€èŒƒå›´ + +_å¯å˜ï¼šå¦_ + +集群中 Pod çš„ IP 地å€èŒƒå›´ã€‚必须是有效的 CIDR 范围，例如 10.42.0.0/16。如果未指定，则会自动从 10.0.0.0/8 中选择一个éšæœºèŒƒå›´ï¼Œå¹¶æŽ’除已分é…ç»™ VMã€å…¶ä»–集群或路由的范围。自动选择的范围å¯èƒ½ä¸Žé¢„留的 IP 地å€ã€åŠ¨æ€è·¯ç”±æˆ–与集群对等的 VPC 中的路由å‘生冲çªã€‚ + +### 网络 + +_å¯å˜ï¼šå¦_ + +集群连接的 Compute Engine 网络。将使用此网络创建路由和防ç«å¢™ã€‚如果使用[共享 VPC](https://cloud.google.com/vpc/docs/shared-vpc)，与你的项目共享的 VPC 网络将显示在此处。你将å¯ä»¥åœ¨æ­¤å­—段中进行选择。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets)。 + +### 节点å­ç½‘/å­ç½‘ + +_å¯å˜ï¼šå¦_ + +集群连接到的 Compute Engine å­ç½‘。该å­ç½‘必须属于**网络**字段中指定的网络。选择一个现有的å­ç½‘，或选择“自动创建å­ç½‘â€æ¥è‡ªåŠ¨åˆ›å»ºä¸€ä¸ªå­ç½‘。如果ä¸ä½¿ç”¨çŽ°æœ‰ç½‘络，则需è¦ä½¿ç”¨**å­ç½‘å称**æ¥ç”Ÿæˆä¸€ä¸ªã€‚如果使用[共享 VPC](https://cloud.google.com/vpc/docs/shared-vpc)，与你的项目共享的 VPC å­ç½‘将显示在此处。如果使用共享 VPC 网络，则无法选择“自动创建å­ç½‘â€ã€‚如需更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets)。 + +### å­ç½‘å称 + +_å¯å˜ï¼šå¦_ + +使用æ供的å称自动创建å­ç½‘。如果为**节点å­ç½‘**或**å­ç½‘**选择了“自动创建å­ç½‘â€ï¼Œåˆ™ä¸ºå¿…填。有关å­ç½‘的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets)。 + +### IP 别å + +_å¯å˜ï¼šå¦_ + +å¯ç”¨[别å IP](https://cloud.google.com/vpc/docs/alias-ip)。这将å¯ç”¨ VPC 原生æµé‡è·¯ç”±ã€‚如果使用[共享 VPC](https://cloud.google.com/vpc/docs/shared-vpc)，则为必填。 + +### 网络策略 + +_å¯å˜ï¼šæ˜¯_ + +在集群上å¯ç”¨çš„网络策略。网络策略定义了集群中 pod å’Œ service 之间å¯ä»¥å‘生的通信级别。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/how-to/network-policy)。 + +### 项目网络隔离 + +_å¯å˜ï¼šæ˜¯_ + +选择å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´é€šä¿¡ã€‚请注æ„，如果å¯ç”¨**项目网络隔离**，则将自动å¯ç”¨**网络策略**å’Œ**网络策略é…ç½®**，å之则ä¸ç„¶ã€‚ + +### 节点 IPv4 CIDR å— + +_å¯å˜ï¼šå¦_ + +此集群中实例 IP çš„ IP 地å€èŒƒå›´ã€‚如果为**节点å­ç½‘**或**å­ç½‘**选择了“自动创建å­ç½‘â€ï¼Œåˆ™å¯ä»¥è¿›è¡Œè®¾ç½®ã€‚必须是有效的 CIDR 范围，例如 10.96.0.0/14。有关如何确定 IP 地å€èŒƒå›´çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing)。 + +### 集群次è¦èŒƒå›´å称 + +_å¯å˜ï¼šå¦_ + +Pod IP 地å€çš„现有次è¦èŒƒå›´çš„å称。如果选中，将自动填充**集群 Pod 地å€èŒƒå›´**。如果使用共享 VPC 网络，则为必填。 + +### 集群 Pod 地å€èŒƒå›´ + +_å¯å˜ï¼šå¦_ + +分é…给集群中 pod çš„ IP 地å€èŒƒå›´ã€‚必须是有效的 CIDR 范围，例如 10.96.0.0/11。如果未æ供，将自动创建。如果使用共享 VPC 网络，则必须æ供。有关如何确定 pod çš„ IP 地å€èŒƒå›´çš„更多信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_pods)。 + +### Service 次è¦èŒƒå›´å称 + +_å¯å˜ï¼šå¦_ + +Service IP 地å€çš„现有次è¦èŒƒå›´çš„å称。如果选中，将自动填充 **Service 地å€èŒƒå›´**。如果使用共享 VPC 网络，则为必填。 + +### Service 地å€èŒƒå›´ + +_å¯å˜ï¼šå¦_ + +分é…给集群中 Service 的地å€èŒƒå›´ã€‚必须是有效的 CIDR 范围，例如 10.94.0.0/18。如果未æ供，将自动创建。如果使用共享 VPC 网络，则必须æ供。有关如何确定 Service çš„ IP 地å€èŒƒå›´çš„详细信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_svcs)。 + +### ç§æœ‰é›†ç¾¤ + +_å¯å˜ï¼šå¦_ + +:::caution + +ç§æœ‰é›†ç¾¤éœ€è¦åœ¨ Rancher 之外进行é¢å¤–的规划和é…置。请å‚阅[ç§æœ‰é›†ç¾¤æŒ‡å—](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md)。 + +::: + +仅分é…节点内部 IP 地å€ã€‚除éžåœ¨ GCP 中执行了é¢å¤–çš„è”网步骤，å¦åˆ™ç§æœ‰é›†ç¾¤èŠ‚点无法访问公共互è”网。 + +### å¯ç”¨ç§æœ‰ç«¯ç‚¹ + +:::caution + +ç§æœ‰é›†ç¾¤éœ€è¦åœ¨ Rancher 之外进行é¢å¤–的规划和é…置。请å‚阅[ç§æœ‰é›†ç¾¤æŒ‡å—](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md)。 + +::: + +_å¯å˜ï¼šå¦_ + +é”定对 controlplane 端点的外部访问。仅当**ç§æœ‰é›†ç¾¤**也被选中时å¯ç”¨ã€‚如果选中，并且 Rancher 无法直接访问集群所在的虚拟ç§æœ‰äº‘网络，Rancher å°†æ供在集群上è¿è¡Œçš„注册命令，以使 Rancher 能够连接到集群。 + +### 主 IPV4 CIDR å— + +_å¯å˜ï¼šå¦_ + +controlplane VPC çš„ IP 范围。 + +### 主授æƒç½‘络 + +_å¯å˜ï¼šæ˜¯_ + +å¯ç”¨ controlplane 授æƒç½‘络，以阻止ä¸å—ä¿¡ä»»çš„éž GCP æº IP 通过 HTTPS 访问 Kubernetes master。如果选择，则å¯ä»¥æ·»åŠ é¢å¤–的授æƒç½‘络。如果集群是使用公共端点创建的，则此选项å¯ç”¨äºŽå°†å…¬å…±ç«¯ç‚¹çš„访问é”定到特定网络（例如è¿è¡Œ Rancher æœåŠ¡çš„网络）。如果集群åªæœ‰ä¸€ä¸ªç§æœ‰ç«¯ç‚¹ï¼Œåˆ™éœ€è¦æ­¤è®¾ç½®ã€‚ + +## 其他选项 + +### 集群æ’件 + +其他 Kubernetes 集群组件。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.locations.clusters#Cluster.AddonsConfig)。 + +#### æ°´å¹³ Pod 自动缩放 + +_å¯å˜ï¼šæ˜¯_ + +Horizo​​ntal Pod Autoscaler 通过自动增加或å‡å°‘ Pod çš„æ•°é‡æ¥è°ƒæ•´ Kubernetes 工作负载，从而å“应工作负载的 CPU æˆ–å†…å­˜æ¶ˆè€—ï¼Œä»¥åŠ Kubernetes 内部报告的自定义指标或集群外部设置的指标。详情请å‚è§[本页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/horizontalpodautoscaler)。 + +#### HTTP (L7) è´Ÿè½½å‡è¡¡ + +_å¯å˜ï¼šæ˜¯_ + +HTTP (L7) è´Ÿè½½å‡è¡¡å°† HTTP å’Œ HTTPS æµé‡åˆ†é…到托管在 GKE 上的åŽç«¯ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/tutorials/http-balancer)。 + +#### 网络策略é…ç½®ï¼ˆä»…é™ master） + +_å¯å˜ï¼šæ˜¯_ + +NetworkPolicy çš„é…置。仅跟踪 master 节点上是å¦å¯ç”¨äº†æ’件，ä¸è·Ÿè¸ªæ˜¯å¦ä¸ºèŠ‚点å¯ç”¨äº†ç½‘络策略。 + +### 集群特å¾ï¼ˆAlpha 功能） + +_å¯å˜ï¼šå¦_ + +打开集群的所有 Kubernetes alpha API 组和功能。å¯ç”¨åŽï¼Œé›†ç¾¤æ— æ³•å‡çº§ï¼Œå¹¶ä¸”会在 30 天åŽè‡ªåŠ¨åˆ é™¤ã€‚由于 GKE SLA æœªæ”¯æŒ alpha 集群，因此ä¸å»ºè®®å°† Alpha 集群用于生产环境。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/alpha-clusters)。 + +### Logging æœåŠ¡ + +_å¯å˜ï¼šæ˜¯_ + +集群用于写入日志的日志管ç†æœåŠ¡ã€‚è¦ä¹ˆä½¿ç”¨ [Cloud Logging](https://cloud.google.com/logging)，è¦ä¹ˆä¸ä½¿ç”¨æ—¥å¿—管ç†æœåŠ¡ï¼ˆä¸ä¼šä»Žé›†ç¾¤ä¸­å¯¼å‡ºæ—¥å¿—）。 + +### 监控æœåŠ¡ + +_å¯å˜ï¼šæ˜¯_ + +集群用于写入指标的监控æœåŠ¡ã€‚è¦ä¹ˆä½¿ç”¨ [Cloud Monitoring](https://cloud.google.com/monitoring)，è¦ä¹ˆä¸ä½¿ç”¨é›†ç¾¤ç›‘控æœåŠ¡ï¼ˆä¸ä¼šä»Žé›†ç¾¤ä¸­å¯¼å‡ºæŒ‡æ ‡ï¼‰ã€‚ + + +### ç»´æŠ¤çª—å£ + +_å¯å˜ï¼šæ˜¯_ + +设置时长 4 å°æ—¶çš„维护窗å£çš„开始时间。使用 HH:MM æ ¼å¼åœ¨ UTC 时区中指定时间。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/maintenance-windows-and-exclusions)。 + +## 节点池 + +在此部分中，输入æ述节点池中æ¯ä¸ªèŠ‚点的é…置的详细信æ¯ã€‚ + +### Kubernetes 版本 + +_å¯å˜ï¼šæ˜¯_ + +节点池中æ¯ä¸ªèŠ‚点的 Kubernetes 版本。有关 GKE Kubernetes 版本的更多信æ¯ï¼Œè¯·å‚阅[这些文档](https://cloud.google.com/kubernetes-engine/versioning)。 + +### é•œåƒç±»åž‹ + +_å¯å˜ï¼šæ˜¯_ + +节点æ“作系统镜åƒã€‚有关 GKE 为æ¯ä¸ªæ“作系统æ供的节点镜åƒé€‰é¡¹ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/node-images#available_node_images)。 + +:::note + +默认选项是 “Container-Optimized OS with Dockerâ€ã€‚GCP Container-Optimized OS 上的åªè¯»æ–‡ä»¶ç³»ç»Ÿä¸Ž Rancher 中的 [legacy logging](/versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-logging.md) 实现ä¸å…¼å®¹ã€‚如果你需è¦ä½¿ç”¨æ—§ç‰ˆæ—¥å¿—管ç†åŠŸèƒ½ï¼Œè¯·é€‰æ‹© “Ubuntu with Docker†或 “Ubuntu with Containerdâ€ã€‚[current logging feature](logging.md) 与 Container-Optimized OS é•œåƒå…¼å®¹ã€‚ + +::: + +:::note + +如果节点池镜åƒç±»åž‹é€‰æ‹© “Windows Long Term Service Channel†或 “Windows Semi-Annual Channelâ€ï¼Œè¿˜å¿…须至少添加一个 Container-Optimized OS 或 Ubuntu 节点池。 + +::: + +### 主机类型 + +_å¯å˜ï¼šå¦_ + +节点实例å¯ç”¨çš„虚拟化硬件资æºã€‚有关 Google Cloud 主机类型的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://cloud.google.com/compute/docs/machine-types#machine_types)。 + +### æ ¹ç£ç›˜ç±»åž‹ + +_å¯å˜ï¼šå¦_ + +标准永久性ç£ç›˜ç”±æ ‡å‡†ç£ç›˜é©±åŠ¨å™¨ (HDD) 支æŒï¼Œè€Œ SSD 永久性ç£ç›˜ç”±å›ºæ€ç¡¬ç›˜ (SSD) 支æŒã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/compute/docs/disks)。 + +### 本地 SSD ç£ç›˜ + +_å¯å˜ï¼šå¦_ + +é…ç½®æ¯ä¸ªèŠ‚点的本地 SSD ç£ç›˜å­˜å‚¨ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚本地 SSD 物ç†è¿žæŽ¥åˆ°æ‰˜ç®¡ä½ çš„ VM 实例的æœåŠ¡å™¨ã€‚与标准永久性ç£ç›˜æˆ– SSD 永久性ç£ç›˜ç›¸æ¯”，本地 SSD 具有更高的åžåé‡å’Œæ›´ä½Žçš„延迟。存储在本地 SSD 上的数æ®åªä¼šä¿ç•™åˆ°å®žä¾‹åœæ­¢æˆ–删除。有关详细信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/compute/docs/disks#localssds)。 + +### 抢å å¼èŠ‚点（beta） + +_å¯å˜ï¼šå¦_ + +抢å å¼èŠ‚点也称为抢å å¼è™šæ‹Ÿæœºã€‚通常是最长æŒç»­ 24 å°æ—¶çš„ Compute Engine 虚拟机实例，ä¸æä¾›å¯ç”¨æ€§ä¿è¯ã€‚详情请å‚è§[本页é¢](https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms)。 + +### 污点 + +_å¯å˜ï¼šå¦_ + +将污点应用于节点时，仅å…许容å¿è¯¥æ±¡ç‚¹çš„ Pod 在该节点上è¿è¡Œã€‚在 GKE 集群中，你å¯ä»¥å°†æ±¡ç‚¹åº”用到节点池，这会将污点应用到池中的所有节点。 + +### 节点标签 + +_å¯å˜ï¼šå¦_ + +ä½ å¯ä»¥å°†æ ‡ç­¾åº”用到节点池，这会将标签应用到池中的所有节点。 + +无效标签会阻止å‡çº§ï¼Œæˆ–阻止 Rancher å¯åŠ¨ã€‚有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 + +### 网络标签 + +_å¯å˜ï¼šå¦_ + +ä½ å¯ä»¥å°†ç½‘络标签添加到节点池以制定防ç«å¢™è§„则和å­ç½‘之间的路由。标签将应用于池中的所有节点。 + +有关标签语法和è¦æ±‚的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://cloud.google.com/vpc/docs/add-remove-network-tags)。 + +## ç»„è¯¦ç»†ä¿¡æ¯ + +在此部分中，输入æ述节点池的详细信æ¯ã€‚ + +### å称 + +_å¯å˜ï¼šå¦_ + +输入节点池的å称。 + +### åˆå§‹èŠ‚点数 + +_å¯å˜ï¼šæ˜¯_ + +节点池中åˆå§‹èŠ‚点数的整数。 + +### æ¯ä¸ªèŠ‚点的最大 Pod æ•°é‡ + +_å¯å˜ï¼šå¦_ + +GKE 的硬性é™åˆ¶æ˜¯æ¯ä¸ªèŠ‚点 110 个 Pod。有关 Kubernetes é™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅[本节](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#dimension_limits)。 + +### 自动缩放 + +_å¯å˜ï¼šæ˜¯_ + +节点池自动缩放会根æ®å·¥ä½œè´Ÿè½½çš„需求动æ€åˆ›å»ºæˆ–删除节点。详情请å‚è§[本页é¢](https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-autoscaler)。 + +### è‡ªåŠ¨ä¿®å¤ + +_å¯å˜ï¼šæ˜¯_ + +GKE 的节点自动修å¤åŠŸèƒ½å¯å¸®åŠ©ä½ å°†é›†ç¾¤ä¸­çš„节点ä¿æŒåœ¨å¥åº·çš„è¿è¡ŒçŠ¶æ€ã€‚å¯ç”¨åŽï¼ŒGKE 会定期检查集群中æ¯ä¸ªèŠ‚点的è¿è¡ŒçŠ¶å†µã€‚如果æŸä¸ªèŠ‚点在较长时间段内连续未通过å¥åº·æ£€æŸ¥ï¼ŒGKE 会为该节点å¯åŠ¨ä¿®å¤è¿‡ç¨‹ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[自动修å¤èŠ‚点](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-repair)。 + +### 自动å‡çº§ + +_å¯å˜ï¼šæ˜¯_ + +å¯ç”¨åŽï¼Œå½“ä½ çš„ controlplane [按照你的需求更新](https://cloud.google.com/kubernetes-engine/upgrades#automatic_cp_upgrades)时，自动å‡çº§åŠŸèƒ½ä¼šä½¿é›†ç¾¤ä¸­çš„节点与集群 controlplane（master）版本ä¿æŒåŒæ­¥ã€‚有关自动å‡çº§èŠ‚点的更多信æ¯ï¼Œå‚è§[此页é¢ã€‚](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades) + +### 访问范围 + +_å¯å˜ï¼šå¦_ + +设置访问范围是为你的节点指定æƒé™çš„旧版方法。 + +- **å…许默认访问**：新集群的默认访问是 [Compute Engine 默认 ServiceAccount](https://cloud.google.com/compute/docs/access/service-accounts?hl=en_US#default_service_account)。 +- **å…许完全访问所有 Cloud API**：通常，你åªéœ€è®¾ç½®äº‘å¹³å°è®¿é—®èŒƒå›´æ¥å…许完全访问所有 Cloud API，然åŽä»…授予 ServiceAccount 相关的 IAM 角色。授予虚拟机实例的访问范围和授予 ServiceAccount çš„ IAM 角色的组åˆå†³å®šäº† ServiceAccount 对该实例的访问é‡ã€‚ +- **为æ¯ä¸ª API 设置访问æƒé™**：或者，你å¯ä»¥è®¾ç½®æœåŠ¡å°†è°ƒç”¨çš„特定 API 方法的访问范围。 + +有关详细信æ¯ï¼Œè¯·å‚阅[为 VM å¯ç”¨ ServiceAccount](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances)。 + + +### é…置刷新间隔 + +刷新间隔å¯ä»¥é€šè¿‡ “gke-refresh†æ¥é…置，它是一个代表秒的整数。 + +默认值为 300 秒。 + +ä½ å¯ä»¥é€šè¿‡è¿è¡Œ `kubectl edit setting gke-refresh` æ¥æ›´æ”¹åŒæ­¥é—´éš”。 + +刷新窗å£è¶ŠçŸ­ï¼Œäº‰ç”¨æ¡ä»¶å‘生的å¯èƒ½æ€§å°±è¶Šå°ã€‚但这确实增加了é‡åˆ° GCP API å¯èƒ½å­˜åœ¨çš„请求é™åˆ¶çš„å¯èƒ½æ€§ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/helm-charts-in-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/helm-charts-in-rancher.md new file mode 100644 index 00000000000..ec9ffabac78 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/helm-charts-in-rancher.md @@ -0,0 +1,155 @@ +--- +title: Rancher 中的 Helm Chart +--- + +在本节中，你将学习如何在 Rancher ä¸­ç®¡ç† Helm Chart 仓库和应用。你å¯ä»¥åœ¨ **Apps** ä¸­ç®¡ç† Helm Chart 仓库。它使用类似目录的系统从仓库中导入 Chart 包，然åŽä½¿ç”¨è¿™äº› Chart æ¥éƒ¨ç½²è‡ªå®šä¹‰ Helm 应用或 Rancher 工具（例如监控和 Istio）。Rancher 工具以预加载仓库的方å¼æ供，并能部署为独立的 Helm Chart 。其他仓库åªä¼šæ·»åŠ åˆ°å½“å‰é›†ç¾¤ã€‚ + +### 版本控制方案 + +Rancher 功能 Chart 版本控制方案以 Chart 的主è¦ç‰ˆæœ¬å’Œä¸Šæ¸¸ Chart çš„ `+up` 注释（如果适用）为中心。 + +**主è¦ç‰ˆæœ¬**：Chart 的主è¦ç‰ˆæœ¬ä¸Ž Rancher 次è¦ç‰ˆæœ¬ç›¸å…³è”。当你å‡çº§åˆ°æ–°çš„ Rancher 次è¦ç‰ˆæœ¬æ—¶ï¼Œä½ åº”该确ä¿ä½ çš„所有 **Apps** Chart 也å‡çº§åˆ° Chart 的正确å‘行版本。 + +**功能 Chart**： + +| **Name** | **支æŒçš„最低版本** | **支æŒçš„最高版本** | +| ---------------- | ------------ | ------------ | +| external-ip-webhook | 100.0.0+up1.0.0 | 100.0.1+up1.0.1 | +| harvester-cloud-provider | 100.0.2+up0.1.12 | 100.0.2+up0.1.12 | +| harvester-csi-driver | 100.0.2+up0.1.11 | 100.0.2+up0.1.11 | +| neuvector | 100.0.0+up2.2.0 | 100.0.0+up2.2.0 | +| rancher-alerting-drivers | 100.0.0 | 100.0.2 | +| rancher-backup | 2.0.1 | 2.1.2 | +| rancher-cis-benchmark | 2.0.1 | 2.0.4 | +| rancher-gatekeeper | 100.0.0+up3.6.0 | 100.1.0+up3.7.1 | +| rancher-istio | 100.0.0+up1.10.4 | 100.3.0+up1.13.3 | +| rancher-logging | 100.0.0+up3.12.0 | 100.1.2+up3.17.4 | +| rancher-longhorn | 100.0.0+up1.1.2 | 100.1.2+up1.2.4 | +| rancher-monitoring | 100.0.0+up16.6.0 | 100.1.2+up19.0.3 | +| rancher-sriov (experimental) | 100.0.0+up0.1.0 | 100.0.3+up0.1.0 | +| rancher-vsphere-cpi | 100.3.0+up1.2.1 | 100.3.0+up1.2.1 | +| rancher-vsphere-csi | 100.3.0+up2.5.1-rancher1 | 100.3.0+up2.5.1-rancher1 | +| rancher-wins-upgrader | 0.0.100 | 100.0.1+up0.0.1 | + +
+ +**基于上游的 Chart**：对于基于上游的 Chart ，+up 注释用于表示 Rancher Chart 正在跟踪的上游版本。在å‡çº§æ—¶ï¼Œè¯·æ£€æŸ¥ä¸Šæ¸¸ç‰ˆæœ¬ä¸Ž Rancher 的兼容性。 + +- 例如，用于 Monitoring çš„ `100.x.x+up16.6.0` 跟踪上游 kube-prometheus-stack `16.6.0` 并添加了一些 Rancher è¡¥ä¸ã€‚ + +- 在å‡çº§æ—¶ï¼Œè¯·ç¡®ä¿ä½ æ²¡æœ‰é™çº§ä½ æ­£åœ¨ä½¿ç”¨çš„ Chart 版本。例如，如果你在 Rancher 2.5 中使用 Monitoring > `16.6.0` 版本，则ä¸åº”å‡çº§åˆ° `100.x.x+up16.6.0`。相å，你应该在下一个å‘行版中å‡çº§åˆ°é€‚当的版本。 + +### 预å‘布版本 + +预å‘布版本éµå¾ª [Semantic Versioning 2.0.0](https://semver.org/) 定义的[规范](https://semver.org/#spec-item-9)。例如，版本为 `0.1.3-dev.12ab4f` çš„ Helm chart 为预å‘布版本。默认情况下ä¸æ˜¾ç¤ºé¢„å‘布版本，必须进行é…ç½®æ‰èƒ½æ˜¾ç¤ºã€‚ + +è¦æ˜¾ç¤ºé¢„å‘布版本： + +1. å•å‡»å³ä¸Šè§’的用户头åƒã€‚ +1. å•å‡»**å好设置**。 +1. 在 **Helm Chart** 下，选择**包括预å‘布版本**。 + +### Charts + +从左上角的èœå•ä¸­é€‰æ‹© _Apps_，然åŽä½ ä¼šè½¬åˆ° Chart 页é¢ã€‚ + +Chart 页é¢åŒ…å«æ‰€æœ‰ Rancherã€Partner 和自定义 Chart 。 + +* Rancher 工具（例如 Logging 或 Monitoring）包å«åœ¨ Rancher 标签下 +* Partner Chart ä½äºŽ Partner 标签下 +* 自定义 Chart 将显示在仓库的å称下 + +所有这三ç§ç±»åž‹éƒ½ä»¥ç›¸åŒçš„æ–¹å¼éƒ¨ç½²å’Œç®¡ç†ã€‚ + +:::note + +ç”± Cluster Manager （旧版 Rancher UI 中的全局视图）管ç†çš„应用应继续仅由 Cluster Manager 管ç†ï¼Œè€Œåœ¨æ–° UI 中使用 Apps 管ç†çš„应用则仅能由 Apps 管ç†ã€‚ + +::: + +### 仓库 + +从左侧边æ ä¸­é€‰æ‹© _仓库_。 + +这些项目代表 helm 仓库，å¯ä»¥æ˜¯å…·æœ‰ index.yaml 的传统 helm 端点，也å¯ä»¥æ˜¯è¢«å…‹éš†å¹¶æŒ‡å‘特定分支的 git 仓库。è¦ä½¿ç”¨è‡ªå®šä¹‰ Chart ，åªéœ€åœ¨æ­¤å¤„添加你的仓库å³å¯ï¼Œå®ƒä»¬å°†åœ¨ä»“库å称下的 Chart 选项å¡ä¸­å¯ç”¨ã€‚ + +为 Helm Chart 仓库添加ç§æœ‰ CA： + +- **基于 HTTP çš„ Chart 仓库**：你必须将 DER æ ¼å¼çš„ CA è¯ä¹¦çš„ base64 ç¼–ç å‰¯æœ¬æ·»åŠ åˆ° Chart 仓库的 spec.caBundle 字段，例如 `openssl x509 -outform der -in ca.pem | base64 -w0`。点击 Chart 仓库的**编辑 YAML** 并进行设置，如下所示：
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + + +- **基于 Git çš„ Chart 仓库**：你必须将 DER æ ¼å¼çš„ CA è¯ä¹¦çš„ base64 ç¼–ç å‰¯æœ¬æ·»åŠ åˆ° Chart 仓库的 spec.caBundle 字段，例如 `openssl x509 -outform der -in ca.pem | base64 -w0`。点击 Chart 仓库的**编辑 YAML** 并进行设置，如下所示：
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + +:::note + +带有身份验è¯çš„ Helm Chart 仓库 + +Repo.Spec 包å«ä¸€ä¸ª `disableSameOriginCheck`，能å…许用户绕过相åŒæºçš„检查，将仓库身份认è¯ä¿¡æ¯ä½œä¸ºåŸºæœ¬ Auth 标头与所有 API 调用一起å‘é€ã€‚ä¸å»ºè®®é‡‡ç”¨è¿™ç§åšæ³•ï¼Œä½†è¿™å¯ä»¥ç”¨ä½œéžæ ‡å‡† Helm Chart 仓库（例如é‡å®šå‘到ä¸åŒæº URL 的仓库）的临时解决方案。 + +è¦å°†æ­¤åŠŸèƒ½ç”¨äºŽçŽ°æœ‰ Helm Chart 仓库，请å•å‡» â‹® > 编辑 YAML。在 YAML 文件的 `spec` 部分，添加 `disableSameOriginCheck` 并将其设置为 `true`： + +```yaml +[...] +spec: + disableSameOriginCheck: true +[...] +``` + +::: + +### Helm 兼容性 + +ä»…æ”¯æŒ Helm 3 兼容 Chart 。 + + +### 部署和å‡çº§ + +从 _Chart_ 选项å¡ä¸­é€‰æ‹©è¦å®‰è£…çš„ Chart 。Rancher å’Œ Partner Chart å¯èƒ½é€šè¿‡è‡ªå®šä¹‰é¡µé¢æˆ– questions.yaml 文件进行é¢å¤–çš„é…置，但所有 Chart 安装都å¯ä»¥ä¿®æ”¹ values.yaml 和其他基本设置。å•å‡»å®‰è£…åŽï¼Œå°†éƒ¨ç½²ä¸€ä¸ª Helm æ“作作业，并显示该作业的控制å°ã€‚ + +è¦æŸ¥çœ‹æ‰€æœ‰æœ€è¿‘的更改，请转到 _最近的æ“作_ 选项å¡ã€‚ä½ å¯ä»¥æŸ¥çœ‹å·²è¿›è¡Œçš„调用ã€æ¡ä»¶ã€äº‹ä»¶å’Œæ—¥å¿—。 + +安装 Chart åŽï¼Œä½ å¯ä»¥åœ¨ _已安装的应用_ 选项å¡ä¸­æ‰¾åˆ°è¯¥ Chart。在本节中，你å¯ä»¥å‡çº§æˆ–删除安装，并查看更多详细信æ¯ã€‚选择å‡çº§æ—¶ï¼Œå‘ˆçŽ°çš„å½¢å¼å’Œæ•°å€¼ä¸Žå®‰è£…相åŒã€‚ + +大多数 Rancher 工具在 _Apps_ 下方的工具æ ä¸­éƒ½æœ‰é¢å¤–的页é¢ï¼Œä»¥å¸®åŠ©ä½ ç®¡ç†å’Œä½¿ç”¨è¿™äº›åŠŸèƒ½ã€‚这些页é¢åŒ…括指å‘仪表æ¿çš„链接ã€å¯è½»æ¾æ·»åŠ è‡ªå®šä¹‰èµ„æºçš„表å•ä»¥åŠå…¶ä»–ä¿¡æ¯ã€‚ + +:::caution + +如果你使用 _在å‡çº§å‰è‡ªå®šä¹‰ Helm 选项_ æ¥å‡çº§ Chart，如果你的 Chart 有ä¸å¯æ›´æ”¹çš„字段，使用 _--force_ 选项å¯èƒ½ä¼šå¯¼è‡´é”™è¯¯ã€‚这是因为 Kubernetes 中的æŸäº›å¯¹è±¡ä¸€æ—¦åˆ›å»ºå°±æ— æ³•æ›´æ”¹ã€‚è¦é¿å…该错误，你å¯ä»¥ï¼š + +* 使用默认å‡çº§é€‰é¡¹ï¼ˆå³ä¸è¦ä½¿ç”¨ _--force_ 选项） +* å¸è½½çŽ°æœ‰ Chart 并安装å‡çº§åŽçš„ Chart +* 在执行 _--force_ å‡çº§ä¹‹å‰åˆ é™¤é›†ç¾¤ä¸­å…·æœ‰ä¸å¯æ›´æ”¹å­—æ®µçš„èµ„æº + +::: + +#### 旧版应用 + +**Apps > Installed Apps** 页é¢ä¸­ï¼Œæ—§ç‰ˆåº”用的å‡çº§æŒ‰é’®å·²è¢«ç§»é™¤ã€‚ + +如果你安装了旧版应用并想è¦å‡çº§å®ƒï¼š + +- 必须开å¯æ—§ç‰ˆ[功能开关](enable-experimental-features.md)（如果在å‡çº§å‰æœ‰æ—§ç‰ˆåº”用导致该开关未自动开å¯ï¼‰ +- ä½ å¯ä»¥ä»Ž cluster explorer å‡çº§åº”用，从左侧导航部分选择**旧版 > 项目 > 应用** +- 对于多集群应用，你å¯ä»¥è½¬åˆ° **≡ > 多集群应用**并在那里å‡çº§åº”用 + +### é™åˆ¶ + +Rancher CLI **ä¸èƒ½**用于安装[仪表æ¿åº”用程åºæˆ– Rancher 功能 Chart](helm-charts-in-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/horizontal-pod-autoscaler.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/horizontal-pod-autoscaler.md new file mode 100644 index 00000000000..eeee4305c97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/horizontal-pod-autoscaler.md @@ -0,0 +1,26 @@ +--- +title: Horizontal Pod Autoscaler +description: 了解 Pod 水平自动扩缩 (HPA)ã€‚å¦‚ä½•ç®¡ç† HPA 以åŠå¦‚何使用æœåŠ¡éƒ¨ç½²æ¥è¿›è¡Œæµ‹è¯• +--- + +[Horizontal Pod Autoscaler（HPA）](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)是一项 Kubernetes 功能，用于将集群é…置为自动扩缩其è¿è¡Œçš„æœåŠ¡ã€‚ + +Rancher æ供了一些附加功能æ¥å¸®åŠ©ç®¡ç† HPA，具体å–决于 Rancher 的版本。 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 创建ã€ç®¡ç†å’Œåˆ é™¤ HPA。仅在 `autoscaling/v2beta2` API ä¸­æ”¯æŒ HPA。 + +## ç®¡ç† HPA + +ç®¡ç† HPA çš„æ–¹å¼å› ä½ çš„ Kubernetes API 版本而异： + +- **Kubernetes API 版本 autoscaling/V2beta1**：å…许根æ®åº”用程åºçš„ CPU 和内存利用率自动扩缩 pod。 +- **Kubernetes API 版本 autoscaling/V2beta2**：å…è®¸æ ¹æ® CPU 和内存利用率以åŠè‡ªå®šä¹‰æŒ‡æ ‡è‡ªåŠ¨æ‰©ç¼© pod。 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher UI 创建ã€ç®¡ç†å’Œåˆ é™¤ HPA。在 Rancher UI 中，你å¯ä»¥å°† HPA é…ç½®ä¸ºæ ¹æ® CPU 和内存利用率进行扩缩。有关详细信æ¯ï¼Œè¯·å‚阅[使用 Rancher UI ç®¡ç† HPA](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md)。如需根æ®è‡ªå®šä¹‰æŒ‡æ ‡è¿›è¡Œ HPA，你ä»ç„¶éœ€è¦ä½¿ç”¨ `kubectl`。有关详细信æ¯ï¼Œè¯·å‚阅[é…ç½® HPA 以使用 Prometheus 自定义指标进行扩缩](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md#é…ç½®-hpa-以使用-prometheus-自定义指标进行扩缩)。 + +在 Rancher 2.0.7 åŠæ›´é«˜ç‰ˆæœ¬ä¸­åˆ›å»ºçš„集群自动满足使用 HPA 的所有è¦æ±‚（metrics-server å’Œ Kubernetes 集群é…置）。 +## 使用æœåŠ¡éƒ¨ç½²æµ‹è¯• HPA + +ä½ å¯ä»¥è½¬åˆ°ä½ çš„项目并å•å‡»**èµ„æº > HPA**æ¥æŸ¥çœ‹â€‹â€‹ HPA 当å‰çš„副本数。有关详细信æ¯ï¼Œè¯·å‚阅[èŽ·å– HPA 指标和状æ€](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md)。 + +你还å¯ä»¥ä½¿ç”¨ `kubectl` æ¥èŽ·å–你使用负载测试工具测试的 HPA 的状æ€ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[使用 kubectl 测试 HPA](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/infrastructure-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/infrastructure-setup.md new file mode 100644 index 00000000000..16b3b3d4d97 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/infrastructure-setup.md @@ -0,0 +1,8 @@ +--- +title: Kubernetes 集群基础设施 +--- + +è¦ä¸ºå…·æœ‰å¤–部数æ®åº“的高å¯ç”¨ K3s Kubernetes 集群设置基础设施，请å‚è§[本页é¢](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md)。 + + +è¦ä¸ºé«˜å¯ç”¨ RKE Kubernetes 集群设置基础设施，请å‚è§[本页é¢](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/install-cluster-autoscaler.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/install-cluster-autoscaler.md new file mode 100644 index 00000000000..87620b84498 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/install-cluster-autoscaler.md @@ -0,0 +1,24 @@ +--- +title: Cluster Autoscaler +--- + +在本文中，你将学习如何使用 AWS EC2 Auto Scaling 组在 Rancher 自定义集群上安装和使用 [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/)。 + +Cluster Autoscaler 是一个自动调整 Kubernetes 集群大å°çš„工具。该工具在满足以下æ¡ä»¶ä¹‹ä¸€æ—¶èƒ½è‡ªåŠ¨è°ƒæ•´é›†ç¾¤å¤§å°ï¼š + +* 集群中有 Pod 因资æºä¸è¶³è€Œæ— æ³•è¿è¡Œã€‚ +* 集群中有一些节点长时间未得到充分利用，而且它们的 Pod å¯ä»¥æ”¾åˆ°å…¶ä»–现有节点上。 + +为防止你的 pod 被驱é€ï¼Œè¯·åœ¨ä½ çš„ pod 规范中设置 `priorityClassName: system-cluster-critical` 属性。 + +Cluster Autoscaler è¿è¡Œåœ¨ Kubernetes master 节点上。它å¯ä»¥åœ¨ `kube-system` 命å空间中è¿è¡Œã€‚Cluster Autoscaler ä¸ä¼šç¼©å‡è¿è¡Œéžé•œåƒ `kube-system` pod 的节点。 + +ä½ å¯ä»¥åœ¨ worker 节点上è¿è¡Œ Cluster Autoscaler 的自定义 deployment，但需è¦å°å¿ƒä»¥ä¿è¯ Cluster Autoscaler 能正常è¿è¡Œã€‚ + +## 云æ供商 + +Cluster Autoscaler 为ä¸åŒçš„云æ供商æ供支æŒã€‚有关详细信æ¯ï¼Œè¯·å‚è§ [Cluster Autoscaler 支æŒçš„云æ供商](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#deployment)。 + +### 在 Amazon 上设置 Cluster Autoscaler + +有关在 Amazon 上è¿è¡Œ Cluster Autoscaler 的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md new file mode 100644 index 00000000000..b189e6d7a35 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md @@ -0,0 +1,352 @@ +--- +title: 在 Kubernetes 集群上安装/å‡çº§ Rancher +description: 了解如何在开å‘和生产环境中安装 Rancher。了解å•èŠ‚点和高å¯ç”¨å®‰è£… +--- + +在本节中，你将学习如何使用 Helm CLI 在 Kubernetes 集群上部署 Rancher。 + + +## 先决æ¡ä»¶ + +- [Kubernetes 集群](#kubernetes-集群) +- [Ingress Controller](#ingress-controller) +- [CLI 工具](#cli-工具) + +### Kubernetes 集群 + +设置 Rancher Server 的本地 Kubernetes 集群。 + +Rancher å¯ä»¥å®‰è£…在任何 Kubernetes 集群上。这个集群å¯ä»¥ä½¿ç”¨ä¸Šæ¸¸ Kubernetes，也å¯ä»¥ä½¿ç”¨ Rancher çš„ Kubernetes å‘行版之一，也å¯ä»¥æ˜¯æ¥è‡ª Amazon EKS ç­‰æ供商的托管 Kubernetes 集群。 + +ä½ å¯å‚考以下教程，以获得设置 Kubernetes 集群的帮助： + +- **RKE**：[安装 RKE Kubernetes 集群的教程](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md)ï¼›[为高å¯ç”¨ RKE 集群设置基础设施的教程](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md)。 +- **K3s**：[安装 K3s Kubernetes 集群的教程](../how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md)ï¼›[设置高å¯ç”¨ K3s 集群的基础设施的教程](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md)。 +- **RKE2:** ：[安装 RKE2 Kubernetes 集群的教程](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md)ï¼›[设置高å¯ç”¨ RKE2 集群的基础设施的教程](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md)。 +- **Amazon EKS**：[在 Amazon EKS 上安装 Rancher 以åŠå¦‚何安装 Ingress Controller 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md)。 +- **AKS**：[使用 Azure Kubernetes æœåŠ¡å®‰è£… Rancher 以åŠå¦‚何安装 Ingress Controller 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md)。 +- **GKE**：有关如何使用 GKE 安装 Rancher，包括如何安装 Ingress Controller 以便å¯ä»¥è®¿é—® Rancher Server，请å‚阅[此页é¢](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md)。GKE 创建 Kubernetes 集群时有两ç§è¿è¡Œæ¨¡å¼ï¼Œåˆ†åˆ«æ˜¯ Autopilot å’Œ Standard 模å¼ã€‚Autopilot 模å¼çš„集群é…置对编辑 kube-system 命å空间有é™åˆ¶ã€‚但是，Rancher 在安装时需è¦åœ¨ kube-system 命å空间中创建资æºã€‚因此，你将无法在以 Autopilot 模å¼åˆ›å»ºçš„ GKE 集群上安装 Rancher。 + + +### Ingress Controller + +Rancher UI å’Œ API 通过 Ingress 公开。æ¢è¨€ä¹‹ï¼Œå®‰è£… Rancher çš„ Kubernetes 集群必须包å«ä¸€ä¸ª Ingress Controller。 + +对于 RKEã€RKE2 å’Œ K3s，你ä¸éœ€è¦æ‰‹åŠ¨å®‰è£… Ingress Controller，因为它是默认安装的。 + +对于默认ä¸åŒ…å« Ingress Controller çš„å‘行版（例如 EKSã€GKE 或 AKS 等托管 Kubernetes 集群），你必须先部署 Ingress Controller。请注æ„，Rancher Helm Chart 默认情况下ä¸ä¼šåœ¨ Ingress 上设置 `ingressClassName`。因此，你必须将 Ingress Controller é…置为在没有 `ingressClassName` 的情况下也å¯ä»¥ç›‘视 Ingress。 + +上é¢çš„ **Amazon EKS**ã€**AKS** å’Œ **GKE** 教程中包å«äº†ç¤ºä¾‹ã€‚ + +### CLI 工具 + +设置 Kubernetes 集群需è¦ä»¥ä¸‹ CLI 工具。请确ä¿è¿™äº›å·¥å…·å·²å®‰è£…并在你的 `$PATH` 中å¯ç”¨ã€‚ + +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) - Kubernetes 命令行工具。 +- [Helm](https://docs.helm.sh/using_helm/#installing-helm) - Kubernetes 的包管ç†å™¨ã€‚请å‚è§ [Helm 版本è¦æ±‚](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md)选择 Helm 版本æ¥å®‰è£… Rancher。请为你的具体平å°å‚è§ [Helm 项目æ供的说明](https://helm.sh/docs/intro/install/)。 + +## 安装 Rancher Helm Chart + +Rancher 是使用 Kubernetes çš„ [Helm](https://helm.sh/) 包管ç†å™¨å®‰è£…的。Helm Chart 为 Kubernetes YAML 清å•æ–‡ä»¶æ供了模æ¿è¯­æ³•ã€‚通过 Helm，用户å¯ä»¥åˆ›å»ºå¯é…置的 deployment，而ä¸ä»…ä»…åªèƒ½ä½¿ç”¨é™æ€æ–‡ä»¶ã€‚ + +如果系统无法直接访问互è”网，请å‚è§[离线环境：Kubernetes 安装](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md)。 + +如果è¦æŒ‡å®šå®‰è£…çš„ Rancher 版本，请å‚è§[选择 Rancher 版本](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +如果è¦æŒ‡å®šç”¨äºŽå®‰è£… Rancher çš„ Helm 版本，请å‚è§[Helm 版本è¦æ±‚](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md)。 + +:::note + +本安装指å—å‡å®šä½ ä½¿ç”¨çš„是 Helm 3。 + +::: + +è¦è®¾ç½® Rancher： + +1. [添加 Helm Chart 仓库](#1-添加-helm-chart-仓库) +2. [为 Rancher 创建命å空间](#2-为-rancher-创建命å空间) +3. [选择 SSL é…ç½®](#3-选择-ssl-é…ç½®) +4. [安装 cert-manager](#4-安装-cert-manager)（除éžä½ è‡ªå¸¦è¯ä¹¦ï¼Œå¦åˆ™ TLS 将在负载å‡è¡¡å™¨ä¸Šç»ˆæ­¢ï¼‰ +5. [使用 Helm 和你选择的è¯ä¹¦é€‰é¡¹å®‰è£… Rancher](#5-æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹é€šè¿‡-helm-安装-rancher) +6. [éªŒè¯ Rancher Server 是å¦éƒ¨ç½²æˆåŠŸ](#6-验è¯-rancher-server-是å¦éƒ¨ç½²æˆåŠŸ) +7. [ä¿å­˜é€‰é¡¹](#7-ä¿å­˜é€‰é¡¹) + +### 1. 添加 Helm Chart 仓库 + +执行 `helm repo add` 命令，以添加包å«å®‰è£… Rancher çš„ Chart çš„ Helm Chart 仓库。有关如何选择仓库，以åŠå“ªä¸ªä»“库最适åˆä½ çš„用例，请å‚è§[选择 Rancher 版本](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md)。 + +- Latest：建议用于试用最新功能 + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable：建议用于生产环境 + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha：å³å°†å‘布的实验性预览。 + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + 注æ„：ä¸æ”¯æŒå‡çº§åˆ° Alpha 版ã€ä»Ž Alpha 版å‡çº§æˆ–在 Alpha 版之间å‡çº§ã€‚ + +### 2. 为 Rancher 创建命å空间 + +你需è¦å®šä¹‰ä¸€ä¸ª Kubernetes 命å空间，用于安装由 Chart 创建的资æºã€‚这个命å空间的å称为 `cattle-system`： + +``` +kubectl create namespace cattle-system +``` + +### 3. 选择 SSL é…ç½® + +Rancher Management Server é»˜è®¤éœ€è¦ SSL/TLS é…ç½®æ¥ä¿è¯è®¿é—®çš„安全性。 + +:::note + +如果你想在外部终止 SSL/TLS，请å‚è§[外部负载å‡è¡¡å™¨çš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)。 + +::: + +ä½ å¯ä»¥ä»Žä»¥ä¸‹ä¸‰ç§è¯ä¹¦æ¥æºä¸­é€‰æ‹©ä¸€ç§ï¼Œç”¨äºŽåœ¨ Rancher Server 中终止 TLS： + +- **Rancher 生æˆçš„ TLS è¯ä¹¦**：è¦æ±‚你在集群中安装 `cert-manager`。Rancher 使用 `cert-manager` ç­¾å‘并维护è¯ä¹¦ã€‚Rancher 会生æˆè‡ªå·±çš„ CA è¯ä¹¦ï¼Œå¹¶ä½¿ç”¨è¯¥ CA 签署è¯ä¹¦ã€‚ç„¶åŽ `cert-manager`负责管ç†è¯¥è¯ä¹¦ã€‚ +- **Let's Encrypt**：Let's Encrypt 选项也需è¦ä½¿ç”¨ `cert-manager`。但是，在这ç§æƒ…况下，cert-manager 与 Let's Encrypt 的特殊é¢å‘者相结åˆï¼Œè¯¥é¢å‘è€…æ‰§è¡ŒèŽ·å– Let's Encrypt é¢å‘çš„è¯ä¹¦æ‰€éœ€çš„所有æ“作（包括请求和验è¯ï¼‰ã€‚æ­¤é…置使用 HTTP 验è¯ï¼ˆ`HTTP-01`），因此负载å‡è¡¡å™¨å¿…须具有å¯ä»¥ä»Žäº’è”网访问的公共 DNS 记录。 +- **你已有的è¯ä¹¦**：使用已有的 CA é¢å‘的公有或ç§æœ‰è¯ä¹¦ã€‚Rancher 将使用该è¯ä¹¦æ¥ä¿æŠ¤ WebSocket å’Œ HTTPS æµé‡ã€‚在这ç§æƒ…况下，你必须上传å称分别为 `tls.crt` å’Œ `tls.key`çš„ PEM æ ¼å¼çš„è¯ä¹¦ä»¥åŠç›¸å…³çš„密钥。如果你使用ç§æœ‰ CA，则还必须上传该 CA è¯ä¹¦ã€‚这是由于你的节点å¯èƒ½ä¸ä¿¡ä»»æ­¤ç§æœ‰ CA。Rancher 将获å–该 CA è¯ä¹¦ï¼Œå¹¶ä»Žä¸­ç”Ÿæˆä¸€ä¸ªæ ¡éªŒå’Œï¼Œå„ç§ Rancher 组件将使用该校验和æ¥éªŒè¯å…¶ä¸Ž Rancher 的连接。 + + +| é…ç½® | Helm Chart 选项 | 是å¦éœ€è¦ cert-manager | +| ------------------------------ | ----------------------- | ------------------------------------- | +| Rancher 生æˆçš„è¯ä¹¦ï¼ˆé»˜è®¤ï¼‰ | `ingress.tls.source=rancher` | [是](#4-安装-cert-manager) | +| Let’s Encrypt | `ingress.tls.source=letsEncrypt` | [是](#4-安装-cert-manager) | +| 你已有的è¯ä¹¦ | `ingress.tls.source=secret` | å¦ | + +### 4. 安装 cert-manager + +> 如果你使用自己的è¯ä¹¦æ–‡ä»¶ï¼ˆ`ingress.tls.source=secret`）或使用[外部负载å‡è¡¡å™¨çš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止)，你å¯ä»¥è·³è¿‡æ­¤æ­¥éª¤ã€‚ + +仅在使用 Rancher 生æˆçš„è¯ä¹¦ï¼ˆ`ingress.tls.source=rancher`）或 Let's Encrypt é¢å‘çš„è¯ä¹¦ï¼ˆ`ingress.tls.source=letsEncrypt`）时，æ‰éœ€è¦å®‰è£… cert-manager。 + +
+ å•å‡»å±•å¼€ + +:::note é‡è¦æ示： + +由于 cert-manager 的最新改动，你需è¦å‡çº§ cert-manager 版本。如果你需è¦å‡çº§ Rancher 并使用低于 0.11.0 çš„ cert-manager 版本，请å‚è§[å‡çº§æ–‡æ¡£](../getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md)。 + +::: + +这些说明æ¥è‡ª [cert-manager 官方文档](https://cert-manager.io/docs/installation/kubernetes/#installing-with-helm)。 + +:::note + +è¦æŸ¥çœ‹è‡ªå®šä¹‰ cert-manager 安装的选项（包括集群使用 PodSecurityPolicies 的情况），请å‚阅 [cert-manager 文档](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration)。 + +::: + +``` +# 如果你手动安装了CRD，而ä¸æ˜¯åœ¨ Helm 安装命令中添加了 `--set installCRDs=true` 选项，你应该在å‡çº§ Helm Chart 之å‰å‡çº§ CRD 资æºã€‚ +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +# 添加 Jetstack Helm 仓库 +helm repo add jetstack https://charts.jetstack.io + +# 更新本地 Helm Chart 仓库缓存 +helm repo update + +# 安装 cert-manager Helm Chart +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 +``` + +安装完 cert-manager åŽï¼Œä½ å¯ä»¥é€šè¿‡æ£€æŸ¥ cert-manager 命å空间中正在è¿è¡Œçš„ Pod æ¥éªŒè¯å®ƒæ˜¯å¦å·²æ­£ç¡®éƒ¨ç½²ï¼š + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +
+ +### 5. æ ¹æ®ä½ é€‰æ‹©çš„è¯ä¹¦é€‰é¡¹ï¼Œé€šè¿‡ Helm 安装 Rancher + +ä¸åŒçš„è¯ä¹¦é…置需è¦ä½¿ç”¨ä¸åŒçš„ Rancher 安装命令。 + +但是，无论è¯ä¹¦å¦‚何é…置，Rancher 在 `cattle-system` 命å空间中的安装å称应该总是 `rancher`。 + +:::tip 测试和开å‘： + +这个安装 Rancher 的最终命令需è¦ä¸€ä¸ªå°†æµé‡è½¬å‘到 Rancher 的域å。如果你使用 Helm CLI 设置概念è¯æ˜Žï¼Œåˆ™å¯ä»¥åœ¨ä¼ å…¥ `hostname` 选项时使用伪域å。伪域å的一个例å­æ˜¯ `.sslip.io`，这会把 Rancher 暴露在它è¿è¡Œçš„ IP 上。生产安装中è¦æ±‚填写真实的域å。 + +::: + + + + +默认情况是使用 Rancher ç”Ÿæˆ CA，并使用 `cert-manager` é¢å‘用于访问 Rancher Server 接å£çš„è¯ä¹¦ã€‚ + +由于 `rancher` 是 `ingress.tls.source` 的默认选项，因此在执行 `helm install` 命令时，我们ä¸éœ€è¦æŒ‡å®š `ingress.tls.source`。 + +- å°† `hostname` 设置为解æžåˆ°ä½ çš„è´Ÿè½½å‡è¡¡å™¨çš„ DNS å称。 +- å°† `bootstrapPassword` 设置为 `admin` 用户独有的值。 +- 如果你需è¦å®‰è£…指定的 Rancher 版本，使用 `--version` 标志，例如 `--version 2.7.0`。 +- 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin +``` + +如果你安装的是 alpha 版本，Helm 会è¦æ±‚你在安装命令中添加 `--devel` 选项： + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +等待 Rancher è¿è¡Œï¼š + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + + +此选项使用 `cert-manager` æ¥è‡ªåŠ¨è¯·æ±‚和续订 [Let's Encrypt](https://letsencrypt.org/) è¯ä¹¦ã€‚Let's Encrypt 是å…费的，而且是å—ä¿¡çš„ CA，因此å¯ä»¥ä¸ºä½ æ供有效的è¯ä¹¦ã€‚ + +:::note + +由于 HTTP-01 质询åªèƒ½åœ¨ç«¯å£ 80 上完æˆï¼Œå› æ­¤ä½ éœ€è¦æ‰“å¼€ç«¯å£ 80。 + +::: + +在以下命令中， + +- å°† `hostname` 设置为公有 DNS 记录。 +- å°† `bootstrapPassword` 设置为 `admin` 用户独有的值。 +- å°† `ingress.tls.source` 设置为 `letsEncrypt`。 +- å°† `letsEncrypt.email` 设置为å¯é€šè®¯çš„电å­é‚®ä»¶åœ°å€ï¼Œç”¨äºŽå‘é€é€šçŸ¥ï¼ˆä¾‹å¦‚è¯ä¹¦åˆ°æœŸçš„通知）。 +- å°† `letsEncrypt.ingress.class` 设为你的 Ingress Controller（例如 `traefik`，`nginx`，`haproxy`） +- 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=letsEncrypt \ + --set letsEncrypt.email=me@example.org \ + --set letsEncrypt.ingress.class=nginx +``` + +如果你安装的是 alpha 版本，Helm 会è¦æ±‚你在安装命令中添加 `--devel` 选项： + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +等待 Rancher è¿è¡Œï¼š + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + +在此选项中，你使用你自己的è¯ä¹¦æ¥åˆ›å»º Kubernetes 密文，以供 Rancher 使用。 + +è¿è¡Œè¿™ä¸ªå‘½ä»¤æ—¶ï¼Œ`hostname` 选项必须与æœåŠ¡å™¨è¯ä¹¦ä¸­çš„ `Common Name` 或 `Subject Alternative Names` æ¡ç›®åŒ¹é…，å¦åˆ™ Ingress controller 将无法正确é…置。 + +è™½ç„¶æŠ€æœ¯ä¸Šä»…éœ€è¦ `Subject Alternative Names` 中有一个æ¡ç›®ï¼Œä½†æ˜¯æ‹¥æœ‰ä¸€ä¸ªåŒ¹é…çš„ `Common Name` å¯ä»¥æœ€å¤§ç¨‹åº¦åœ°æ高与旧版æµè§ˆå™¨/应用的兼容性。 + +:::note + +如果你想检查è¯ä¹¦æ˜¯å¦æ­£ç¡®ï¼Œè¯·æŸ¥çœ‹[如何在æœåŠ¡å™¨è¯ä¹¦ä¸­æ£€æŸ¥ Common Name å’Œ Subject Alternative Names](../faq/technical-items.md#如何在æœåŠ¡å™¨è¯ä¹¦ä¸­æ£€æŸ¥-common-name-å’Œ-subject-alternative-names)。 + +::: + +- 设置 `hostname`。 +- å°† `bootstrapPassword` 设置为 `admin` 用户独有的值。 +- å°† `ingress.tls.source` 设置为 `secret`。 +- 对于 Kubernetes v1.25 或更高版本，使用 Rancher v2.7.2-v2.7.4 时，将 `global.cattle.psp.enabled` 设置为 `false`。对于 Rancher v2.7.5 åŠæ›´é«˜ç‰ˆæœ¬æ¥è¯´ï¼Œè¿™ä¸æ˜¯å¿…需的，但你ä»ç„¶å¯ä»¥æ‰‹åŠ¨è®¾ç½®è¯¥é€‰é¡¹ã€‚ + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret +``` +如果你安装的是 alpha 版本，Helm 会è¦æ±‚你在安装命令中添加 `--devel` 选项： + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +如果你使用的是ç§æœ‰ CA è¯ä¹¦ï¼Œè¯·åœ¨å‘½ä»¤ä¸­å¢žåŠ  `--set privateCA=true`。 + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret \ + --set privateCA=true +``` + +**添加 TLS 密文（åƒä¸‡ä¸è¦é—æ¼æ­¤æ­¥éª¤ï¼‰**：现在 Rancher å·²ç»å®Œæˆéƒ¨ç½²ï¼Œä½ è¿˜éœ€è¦å‚考[添加 TLS 密文](../getting-started/installation-and-upgrade/resources/add-tls-secrets.md)å‘布è¯ä¹¦æ–‡ä»¶ï¼Œä»¥ä¾¿ Rancher å’Œ Ingress Controller å¯ä»¥ä½¿ç”¨å®ƒä»¬ã€‚ + + + + +Rancher Chart 有许多选项，用于为你的具体环境自定义安装。以下是一些常è§çš„高级方案： + +- [HTTP 代ç†](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#http-代ç†) +- [ç§æœ‰å®¹å™¨é•œåƒä»“库](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#ç§æœ‰ä»“库和离线安装) +- [外部负载å‡è¡¡å™¨ä¸Šçš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止) + +如需获å–完整的选项列表，请å‚è§ [Chart 选项](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)。 + + +### 6. éªŒè¯ Rancher Server 是å¦éƒ¨ç½²æˆåŠŸ + +添加密文åŽï¼Œæ£€æŸ¥ Rancher 是å¦å·²æˆåŠŸè¿è¡Œï¼š + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + +如果你看到 `error: deployment "rancher" exceeded its progress deadline` 这个错误，å¯è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ£€æŸ¥ deployment 的状æ€ï¼š + +``` +kubectl -n cattle-system get deploy rancher +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +rancher 3 3 3 3 3m +``` + +`DESIRED` å’Œ `AVAILABLE`的个数应该相åŒã€‚ + +### 7. ä¿å­˜é€‰é¡¹ + +请ä¿å­˜ä½ ä½¿ç”¨çš„ `--set` 选项。使用 Helm å‡çº§ Rancher 到新版本时，你将需è¦ä½¿ç”¨ç›¸åŒçš„选项。 + +### å®‰è£…å®Œæˆ + +安装已完æˆã€‚现在 Rancher Server 应该已ç»å¯ä»¥æ­£å¸¸è¿è¡Œäº†ã€‚ + +使用æµè§ˆå™¨æ‰“开把æµé‡è½¬å‘到你的负载å‡è¡¡å™¨çš„ DNS 域å。然åŽï¼Œä½ å°±ä¼šçœ‹åˆ°ä¸€ä¸ªæ¼‚亮的登录页é¢äº†ã€‚ + +如果é‡åˆ°ä»»ä½•é—®é¢˜ï¼Œè¯·å‚è§[故障排除](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-and-upgrade.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-and-upgrade.md new file mode 100644 index 00000000000..d57549034f6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-and-upgrade.md @@ -0,0 +1,94 @@ +--- +title: 安装/å‡çº§ Rancher +description: 了解如何在开å‘和生产环境中安装 Rancher。了解å•èŠ‚点和高å¯ç”¨å®‰è£… +--- + +本节介ç»äº† Rancher å„ç§å®‰è£…æ–¹å¼ä»¥åŠæ¯ä¸ªå®‰è£…æ–¹å¼çš„优点。 + +## åè¯è§£é‡Š + +本章节涉åŠä»¥ä¸‹åè¯ï¼š + +- **Rancher Server**：用于管ç†å’Œé…ç½® Kubernetes 集群。你å¯ä»¥é€šè¿‡ Rancher Server çš„ UI 与下游 Kubernetes 集群进行交互。Rancher Management Server å¯ä»¥å®‰è£…åˆ°ä»»æ„ Kubernetes 集群上，包括托管的集群，如 Amazon EKS 集群。 +- **RKE（Rancher Kubernetes Engine）**：是ç»è¿‡è®¤è¯çš„ Kubernetes å‘è¡Œç‰ˆï¼Œä¹Ÿæ˜¯ç”¨äºŽåˆ›å»ºå’Œç®¡ç† Kubernetes 集群的 CLI 工具和库。 +- **K3s（轻é‡çº§ Kubernetes）**：也是ç»è¿‡è®¤è¯çš„ Kubernetes å‘行版。它比 RKE 更新，更易用且更轻é‡ï¼Œå…¶æ‰€æœ‰ç»„件都在一个å°äºŽ 100 MB 的二进制文件中。 +- **RKE2**：一个完全åˆè§„çš„ Kubernetes å‘行版，专注于安全和åˆè§„性。 + +`restrictedAdmin` Helm Chart 选项在 **Rancher Server** å¯ç”¨ã€‚如果该选项设置为 true，åˆå§‹çš„ Rancher 用户访问本地 Kubernetes 集群会å—到é™åˆ¶ï¼Œä»¥é¿å…æƒé™å‡çº§ã€‚详情请å‚è§ [restricted-admin 角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#å—é™ç®¡ç†å‘˜)。 + +## 安装方å¼æ¦‚è¿° + +Rancher å¯ä»¥å®‰è£…在以下主è¦æž¶æž„上： + +### 使用 Helm CLI 安装的高å¯ç”¨ Kubernetes + +我们建议使用 Kubernetes 包管ç†å™¨ Helm 在专用的 Kubernetes 集群上安装 Rancher。在 RKE 集群中，需è¦ä½¿ç”¨ä¸‰ä¸ªèŠ‚点æ‰èƒ½å®žçŽ°é«˜å¯ç”¨é›†ç¾¤ã€‚在 K3s 集群中，åªéœ€è¦ä¸¤ä¸ªèŠ‚点å³å¯ã€‚ + +### 通过 AWS Marketplace 在 EKS 上安装 Rancher + +ä½ å¯ä»¥[通过 AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md) å°† Rancher 安装到 Amazon Elastic Kubernetes Service (EKS) 上。部署的 EKS 集群已生产就绪，并éµå¾ª AWS 最佳实践。 + +### å•èŠ‚点 Kubernetes 安装 + +Rancher å¯ä»¥å®‰è£…在å•èŠ‚点 Kubernetes 集群上。但是，在å•èŠ‚点安装的情况下，Rancher Server 没有高å¯ç”¨æ€§ã€‚而高å¯ç”¨æ€§å¯¹åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­è¿è¡Œ Rancher éžå¸¸é‡è¦ã€‚ + +但是，如果你想è¦çŸ­æœŸå†…使用å•èŠ‚点节çœèµ„æºï¼ŒåŒæ—¶åˆä¿ç•™é«˜å¯ç”¨æ€§è¿ç§»è·¯å¾„，那么å•èŠ‚点 Kubernetes 安装也是åˆé€‚的。你也å¯ä»¥ä¹‹åŽå‘集群中添加节点，获得高å¯ç”¨çš„ Rancher Server。 + +### Docker 安装 + +如果你的目的是测试或演示，你å¯ä»¥ä½¿ç”¨ Docker 把 Rancher 安装到å•ä¸ªèŠ‚点中。本地 Kubernetes 集群是安装到å•ä¸ª Docker 容器中的，而 Rancher 是安装到本地集群中的。 + +Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +### å…¶ä»–æ–¹å¼ + +如果你需è¦åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­æˆ–使用 HTTP 代ç†å®‰è£… Rancher，请å‚è§ä»¥ä¸‹ç‹¬ç«‹çš„说明文档： + +| ç½‘ç»œè®¿é—®æ–¹å¼ | 基于 Kubernetes 安装（推è） | 基于 Docker 安装 | +| ---------------------------------- | ------------------------------ | ---------- | +| å¯ç›´æŽ¥è®¿é—®äº’è”网 | [文档](install-upgrade-on-a-kubernetes-cluster.md) | [文档](rancher-on-a-single-node-with-docker.md) | +| 使用 HTTP ä»£ç† | [文档](rancher-behind-an-http-proxy.md) | [文档](rancher-on-a-single-node-with-docker.md)åŠ[é…ç½®](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) | +| 离线环境 | [文档](air-gapped-helm-cli-install.md) | [文档](air-gapped-helm-cli-install.md) | + +我们建议在 Kubernetes 集群上安装 Rancher，因为在多节点集群中，Rancher Server å¯ä»¥å®žçŽ°é«˜å¯ç”¨ã€‚高å¯ç”¨é…ç½®å¯ä»¥æå‡ Rancher 访问其管ç†çš„下游 Kubernetes 集群的稳定性。 + +因此，我们建议在生产级别的架构中，设置一个高å¯ç”¨çš„ Kubernetes 集群，然åŽåœ¨è¿™ä¸ªé›†ç¾¤ä¸Šå®‰è£… Rancher。安装 Rancher åŽï¼Œä½ å¯ä»¥ä½¿ç”¨ Rancher éƒ¨ç½²å’Œç®¡ç† Kubernetes 集群。 + +如果你的目的是测试或演示，你å¯ä»¥å°† Rancher 安装到å•ä¸ª Docker 容器中。Docker 安装å¯ä»¥è®©ä½ å®žçŽ°å¼€ç®±å³ç”¨ï¼Œä»¥ä½¿ç”¨ Rancher 设置 Kubernetes 集群。Docker 安装主è¦æ˜¯ç”¨äºŽæŽ¢ç´¢ Rancher Server 的功能，åªé€‚用于开å‘和测试。 + +[在 Kubernetes 上安装 Rancher 的说明](install-upgrade-on-a-kubernetes-cluster.md)介ç»äº†å¦‚何首先使用 K3s 或 RKE åˆ›å»ºå’Œç®¡ç† Kubernetes 集群，然åŽå†å°† Rancher 安装到该集群上。 + +如果 Kubernetes 集群中的节点正在è¿è¡Œä¸”满足[节点è¦æ±‚](installation-requirements.md)，你å¯ä»¥ä½¿ç”¨ Helm å°† Rancher 部署到 Kubernetes 上。Helm 使用 Rancher çš„ Helm Chart 在 Kubernetes 集群的æ¯ä¸ªèŠ‚点上安装 Rancher 的副本。我们建议使用负载å‡è¡¡å™¨å°†æµé‡å®šå‘到集群中的æ¯ä¸ª Rancher 副本上。 + +如需进一步了解 Rancher 架构，请å‚è§[架构概述](rancher-manager-architecture.md)，[生产级别架构推è](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)或[最佳实践指å—](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md)。 + +## 先决æ¡ä»¶ + +安装 Rancher 之å‰ï¼Œè¯·ç¡®ä¿ä½ çš„节点满足所有[安装è¦æ±‚](installation-requirements.md)。 + +## 架构建议 + +为了达到最佳性能和安全性，我们建议你为 Rancher Management Server 使用å•ç‹¬çš„专用 Kubernetes 集群。ä¸å»ºè®®åœ¨æ­¤é›†ç¾¤ä¸Šè¿è¡Œç”¨æˆ·å·¥ä½œè´Ÿè½½ã€‚部署 Rancher åŽï¼Œä½ å¯ä»¥[创建或导入集群](kubernetes-clusters-in-rancher-setup.md)æ¥è¿è¡Œä½ çš„工作负载。 + +详情请å‚è§[架构推è](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)。 + +### 在 Kubernetes 上安装 Rancher 的更多选项 + +å‚è§ [Helm Chart 选项](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)以了解在 Kubernetes 集群上安装 Rancher 的其他é…置，包括： + +- [å¼€å¯ API 审计日志æ¥è®°å½•æ‰€æœ‰äº‹åŠ¡](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-审计日志) +- [è´Ÿè½½å‡è¡¡å™¨ä¸Šçš„ TLS 终止](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#外部-tls-终止) +- [自定义 Ingress](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#自定义-ingress) + +在 Rancher 的安装指å—中，我们推è使用 K3s 或 RKE æ¥é…ç½® Kubernetes 集群，然åŽå†åœ¨è¿™ä¸ªé›†ç¾¤ä¸­å®‰è£… Rancher。K3s å’Œ RKE å‡æ供许多é…置选项，用于为你的具体环境自定义 Kubernetes 集群。有关选项和功能的完整列表，请å‚è§ï¼š + +- [RKE é…置选项](https://rancher.com/docs/rke/latest/en/config-options/) +- [K3s é…置选项](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +### 在 Docker 上安装 Rancher 的更多选项 + +å‚è§ [Docker 安装选项](rancher-on-a-single-node-with-docker.md)了解其他é…置，包括： + +- [å¼€å¯ API 审计日志æ¥è®°å½•æ‰€æœ‰äº‹åŠ¡](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志) +- [外部负载å‡è¡¡å™¨](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md) +- [æŒä¹…化数æ®å­˜å‚¨](../reference-guides/single-node-rancher-in-docker/advanced-options.md#æŒä¹…化数æ®) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-references.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-references.md new file mode 100644 index 00000000000..e7a5b5ac802 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-references.md @@ -0,0 +1,5 @@ +--- +title: 安装å‚考 +--- + +有关其他安装资æºï¼Œè¯·å‚阅以下å‚考指å—：[Rancher Helm Chart 选项](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md)ã€[TLS 设置](../getting-started/installation-and-upgrade/installation-references/tls-settings.md)å’Œ[功能开关](../getting-started/installation-and-upgrade/installation-references/feature-flags.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-requirements.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-requirements.md new file mode 100644 index 00000000000..0328552e0e8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/installation-requirements.md @@ -0,0 +1,155 @@ +--- +title: 安装è¦æ±‚ +description: 如果 Rancher é…置在 Docker 或 Kubernetes 中è¿è¡Œæ—¶ï¼Œäº†è§£è¿è¡Œ Rancher Server çš„æ¯ä¸ªèŠ‚点的节点è¦æ±‚ +--- + +本文æ述了对需è¦å®‰è£… Rancher Server 的节点的软件ã€ç¡¬ä»¶å’Œç½‘络è¦æ±‚。Rancher Server å¯ä»¥å®‰è£…在å•ä¸ªèŠ‚点或高å¯ç”¨çš„ Kubernetes 集群上。 + +:::note é‡è¦æ示： + +如果你需è¦åœ¨ Kubernetes 集群上安装 Rancher，该节点的è¦æ±‚与用于è¿è¡Œåº”用和æœåŠ¡çš„[下游集群的节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)ä¸åŒã€‚ + +::: + +Rancher UI 在基于 Firefox 或 Chromium çš„æµè§ˆå™¨ï¼ˆChromeã€Edgeã€Operaã€Brave）中效果最佳。 + +查看我们的[最佳实践](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md)页é¢ï¼ŒèŽ·å–在生产环境中è¿è¡Œ Rancher Server 的建议。 + +## Kubernetes 与 Rancher 的兼容性 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。请查阅 [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)，确ä¿ä½ çš„ Kubernetes 版本å—支æŒã€‚ + +## æ“作系统和容器è¿è¡Œæ—¶è¦æ±‚ + +所有支æŒçš„æ“作系统都使用 64-bit x86 架构。Rancher 兼容当å‰æ‰€æœ‰çš„ä¸»æµ Linux å‘行版。 + +[Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)列出了æ¯ä¸ª Rancher 版本测试过的æ“作系统和 Docker 版本。 + +è¿è¡Œ RKE 集群的节点需è¦å®‰è£… Docker。RKE2 或 K3s 集群ä¸éœ€è¦å®ƒã€‚ + +请安装 `ntp`（Network Time Protocol），以防止在客户端和æœåŠ¡å™¨ä¹‹é—´ç”±äºŽæ—¶é—´ä¸åŒæ­¥é€ æˆçš„è¯ä¹¦éªŒè¯é”™è¯¯ã€‚ + +æŸäº› Linux å‘行版的默认防ç«å¢™è§„则å¯èƒ½ä¼šé˜»æ­¢ Kubernetes 集群内的通信。从 Kubernetes v1.19 开始，你必须关闭 firewalld，因为它与 Kubernetes 网络æ’件冲çªã€‚ + +如果你ä¸å¤ªæƒ³è¿™æ ·åšçš„è¯ï¼Œä½ å¯ä»¥æŸ¥çœ‹[相关问题](https://github.com/rancher/rancher/issues/28840)中的建议。æŸäº›ç”¨æˆ·å·²èƒ½æˆåŠŸ[使用 ACCEPT ç­–ç•¥ 为 Pod CIDR 创建一个独立的 firewalld 区域](https://github.com/rancher/rancher/issues/28840#issuecomment-787404822)。 + +如果你需è¦åœ¨ ARM64 上使用 Rancher，请å‚è§[在 ARM64（实验功能）上è¿è¡Œ Rancher](../how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md)。 + +### RKE è¦æ±‚ + +容器è¿è¡Œæ—¶æ–¹é¢ï¼ŒRKE å¯ä»¥å…¼å®¹å½“å‰çš„所有 Docker 版本。 + +有关详细信æ¯ï¼Œè¯·å‚阅[安装 Docker](../getting-started/installation-and-upgrade/installation-requirements/install-docker.md)。 + +### K3s è¦æ±‚ + +对于容器è¿è¡Œæ—¶ï¼ŒK3s 默认附带了自己的 containerd。你也å¯ä»¥å°† K3s é…置为使用已安装的 Docker è¿è¡Œæ—¶ã€‚有关在 Docker 中使用 K3s 的更多信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://docs.k3s.io/advanced#using-docker-as-the-container-runtime)。 + +Rancher 需è¦å®‰è£…在支æŒçš„ Kubernetes 版本上。如需了解你使用的 Rancher 版本支æŒå“ªäº› Kubernetes 版本，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)。如需指定 K3s 版本，在è¿è¡Œ K3s 安装脚本时，使用 `INSTALL_K3S_VERSION` 环境å˜é‡ã€‚ + +如果你使用 **Raspbian Buster** 在 K3s 集群上安装 Rancher，请按照[这些步骤](https://rancher.com/docs/k3s/latest/en/advanced/#enabling-legacy-iptables-on-raspbian-buster)切æ¢åˆ°æ—§ç‰ˆ iptables。 + +如果你使用 Alpine Linux çš„ K3s 集群上安装 Rancher，请按照[这些步骤](https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-alpine-linux-setup) 进行其他设置。 + +### RKE2 è¦æ±‚ + +对于容器è¿è¡Œæ—¶ï¼ŒRKE2 附带了自己的 containerd。RKE2 安装ä¸éœ€è¦ Docker。 + +如需了解 RKE2 通过了哪些æ“作系统版本的测试，请å‚è§ [Rancher 支æŒçŸ©é˜µ](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions)。 + +## 硬件è¦æ±‚ + +本节æ述安装 Rancher Server 的节点的 CPUã€å†…存和ç£ç›˜è¦æ±‚。 + +## CPU 和内存 + +硬件è¦æ±‚æ ¹æ®ä½ çš„ Rancher 部署规模而定。请根æ®è¦æ±‚é…ç½®æ¯ä¸ªèŠ‚点。通过å•èŠ‚点容器安装 Rancher，和在 Kubernetes 集群上安装 Rancher çš„è¦æ±‚有所ä¸åŒã€‚ + +### RKE 和托管 Kubernetes + +这些 CPU 和内存è¦æ±‚适用于æ¯ä¸ªå®‰è£… Rancher Server çš„ Kubernetes 集群中的主机。 + +这些è¦æ±‚适用于 RKE Kubernetes 集群以åŠæ‰˜ç®¡çš„ Kubernetes 集群，例如 EKS。 + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | +| --------------- | ---------- | ------------ | -------| ------- | +| å° | 最多 150 个 | 最多 1500 个 | 2 | 8 GB | +| 中 | 最多 300 个 | 最多 3,000 个 | 4 | 16 GB | +| 大 | 最多 500 个 | 最多 5,000 个 | 8 | 32 GB | +| 特大 | 最多 1,000 个 | 最多 10,000 个 | 16 | 64 GB | +| 超大 | 最多 2,000 个 | 最多 20,000 个 | 32 | 128 GB | + +æ¯ä¸ªç”¨ä¾‹å’ŒçŽ¯å¢ƒéƒ½æ˜¯ä¸åŒçš„。请[è”ç³» Rancher](https://rancher.com/contact/) æ¥å®¡æ ¸ä½ çš„情况。 + +### K3s Kubernetes + +这些 CPU 和内存è¦æ±‚适用于æ¯ä¸ª[安装 Rancher Server çš„ Kubernetes 集群](install-upgrade-on-a-kubernetes-cluster.md)中的主机。 + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | æ•°æ®åº“å¤§å° | +| --------------- | ---------- | ------------ | -------| ---------| ------------------------- | +| å° | 最多 150 个 | 最多 1500 个 | 2 | 8 GB | 2 核，4 GB + 1,000 IOPS | +| 中 | 最多 300 个 | 最多 3,000 个 | 4 | 16 GB | 2 核，4 GB + 1,000 IOPS | +| 大 | 最多 500 个 | 最多 5,000 个 | 8 | 32 GB | 2 核，4 GB + 1,000 IOPS | +| 特大 | 最多 1,000 个 | 最多 10,000 个 | 16 | 64 GB | 2 核，4 GB + 1,000 IOPS | +| 超大 | 最多 2,000 个 | 最多 20,000 个 | 32 | 128 GB | 2 核，4 GB + 1,000 IOPS | + +æ¯ä¸ªç”¨ä¾‹å’ŒçŽ¯å¢ƒéƒ½æ˜¯ä¸åŒçš„。请[è”ç³» Rancher](https://rancher.com/contact/) æ¥å®¡æ ¸ä½ çš„情况。 + + +### RKE2 Kubernetes + +这些 CPU 和内存è¦æ±‚适用于安装了 RKE2 çš„æ¯ä¸ªå®žä¾‹ã€‚最低é…ç½®è¦æ±‚如下： + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | +| --------------- | -------- | --------- | ----- | ---- | +| å° | 最多 5 个 | 最多 50 个 | 2 | 5 GB | +| 中 | 最多 15 个 | 最多 200 个 | 3 | 9 GB | + +### Docker + +这些 CPU 和内存è¦æ±‚适用于[å•èŠ‚点](rancher-on-a-single-node-with-docker.md)安装 Rancher 的主机。 + +| 部署规模 | 集群 | 节点 | vCPUs | 内存 | +| --------------- | -------- | --------- | ----- | ---- | +| å° | 最多 5 个 | 最多 50 个 | 1 | 4 GB | +| 中 | 最多 15 个 | 最多 200 个 | 2 | 8 GB | + +## Ingress + +安装 Rancher çš„ Kubernetes 集群中的æ¯ä¸ªèŠ‚点都应该è¿è¡Œä¸€ä¸ª Ingress。 + +Ingress 需è¦éƒ¨ç½²ä¸º DaemonSet 以确ä¿è´Ÿè½½å‡è¡¡å™¨èƒ½æˆåŠŸæŠŠæµé‡è½¬å‘到å„个节点。 + +如果是 RKE，RKE2 å’Œ K3s 安装，你ä¸éœ€è¦æ‰‹åŠ¨å®‰è£… Ingress，因为它是默认安装的。 + +对于托管的 Kubernetes 集群（EKSã€GKEã€AKS），你需è¦è®¾ç½® Ingress。 + +- **Amazon EKS**：[在 Amazon EKS 上安装 Rancher 以åŠå¦‚何安装 Ingress 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md)。 +- **AKS**：[使用 Azure Kubernetes æœåŠ¡å®‰è£… Rancher 以åŠå¦‚何安装 Ingress 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md)。 +- **GKE**：[使用 GKE 安装 Rancher 以åŠå¦‚何安装 Ingress 以访问 Rancher Server](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md)。 + +## ç£ç›˜ + +etcd 在集群中的性能决定了 Rancher 的性能。因此，为了获得最佳速度，我们建议使用 SSD ç£ç›˜æ¥æ”¯æŒ Rancher 管ç†çš„ Kubernetes 集群。在云æ供商上，你还需使用能获得最大 IOPS 的最å°å¤§å°ã€‚在较大的集群中，请考虑使用专用存储设备存储 etcd æ•°æ®å’Œ wal 目录。 + +## 网络è¦æ±‚ + +本节æ述了安装 Rancher Server 的节点的网络è¦æ±‚。 + +:::caution + +å¦‚æžœåŒ…å« Rancher çš„æœåŠ¡å™¨å¸¦æœ‰ `X-Frame-Options=DENY` 标头，在å‡çº§æ—§ç‰ˆ UI 之åŽï¼ŒRancher UI 中的æŸäº›é¡µé¢å¯èƒ½æ— æ³•æ¸²æŸ“。这是因为æŸäº›æ—§ç‰ˆé¡µé¢åœ¨æ–° UI 中是以 iFrames 模å¼åµŒå…¥çš„。 + +::: + +### 节点 IP åœ°å€ + +无论你是在å•ä¸ªèŠ‚点还是高å¯ç”¨é›†ç¾¤ä¸Šå®‰è£… Rancher，æ¯ä¸ªèŠ‚点都应é…置一个é™æ€ IP。如果使用 DHCP，则æ¯ä¸ªèŠ‚点都应该有一个 DHCP 预留，以确ä¿èŠ‚点分é…到相åŒçš„ IP 地å€ã€‚ + +### 端å£è¦æ±‚ + +为了确ä¿èƒ½æ­£å¸¸è¿è¡Œï¼ŒRancher 需è¦åœ¨ Rancher 节点和下游 Kubernetes 集群节点上开放一些端å£ã€‚ä¸åŒé›†ç¾¤ç±»åž‹çš„ Rancher 和下游集群的所有必è¦ç«¯å£ï¼Œè¯·å‚è§[端å£è¦æ±‚](../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md)。 + +## Dockershim æ”¯æŒ + +有关 Dockershim 支æŒçš„详情，请å‚è§[此页é¢](../getting-started/installation-and-upgrade/installation-requirements/dockershim.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/istio-setup-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/istio-setup-guide.md new file mode 100644 index 00000000000..4ed26979729 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/istio-setup-guide.md @@ -0,0 +1,30 @@ +--- +title: è®¾ç½®æŒ‡å— +--- + +本文介ç»å¦‚何å¯ç”¨ Istio 并在你的项目中使用它。 + +如果你使用 Istio 进行æµé‡ç®¡ç†ï¼Œåˆ™éœ€è¦å…许外部æµé‡è¿›å…¥é›†ç¾¤ã€‚在这ç§æƒ…况下，你将需è¦æ‰§è¡Œä»¥ä¸‹æ‰€æœ‰æ­¥éª¤ã€‚ + +## 先决æ¡ä»¶ + +本指å—å‡è®¾ä½ å·²ç»[安装 Rancher](installation-and-upgrade.md)，且已ç»[é…置了一个å•ç‹¬çš„ Kubernetes 集群](kubernetes-clusters-in-rancher-setup.md)并è¦åœ¨è¯¥é›†ç¾¤ä¸Šå®‰è£… Istio。 + +集群中的节点必须满足 [CPU 和内存è¦æ±‚](../integrations-in-rancher/istio/cpu-and-memory-allocations.md)。 + +Istio 控制的工作负载和æœåŠ¡å¿…须满足 [Istio è¦æ±‚](https://istio.io/docs/setup/additional-setup/requirements/)。 + +## 安装 + +:::tip 快速设置æ示： + +如果你ä¸éœ€è¦å¤–部æµé‡åˆ°è¾¾ Istio，而åªæƒ³è®¾ç½® Istio 以监控和跟踪集群内的æµé‡ï¼Œè¯·è·³è¿‡[设置 Istio Gateway](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) å’Œ[设置 Istio çš„æµé‡ç®¡ç†ç»„件](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md)步骤。 + +::: + +1. [在集群中å¯ç”¨ Istio](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md) +1. [在è¦ä½¿ç”¨ Istio 的所有命å空间中å¯ç”¨ Istio](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md) +1. [添加注入了 Istio sidecar 的部署和æœåŠ¡](../how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md) +1. [设置 Istio Gateway](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) +1. [设置 Istio çš„æµé‡ç®¡ç†ç»„件](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md) +1. [生æˆæµé‡å¹¶æŸ¥çœ‹ Istio çš„è¿è¡Œæƒ…况](../how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/istio.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/istio.md new file mode 100644 index 00000000000..a485651a4cb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/istio.md @@ -0,0 +1,137 @@ +--- +title: Istio +--- + +[Istio](https://istio.io/) 是一ç§å¼€æºå·¥å…·ï¼Œå¯ä»¥è®© DevOps 团队更轻æ¾åœ°è§‚察ã€æŽ§åˆ¶ã€æŽ’查并ä¿æŠ¤å¤æ‚çš„å¾®æœåŠ¡ç½‘络中的æµé‡ã€‚ + +éšç€å¾®æœåŠ¡ç½‘络的å˜åŒ–和增长，微æœåŠ¡ç½‘络之间的交互å˜å¾—越æ¥è¶Šéš¾ä»¥ç®¡ç†å’Œç†è§£ã€‚在这ç§æƒ…况下，将æœåŠ¡ç½‘格作为å•ç‹¬çš„基础设施层是éžå¸¸æœ‰ç”¨çš„。Istio çš„æœåŠ¡ç½‘æ ¼å¯ä»¥è®©ä½ åœ¨ä¸ç›´æŽ¥æ›´æ”¹å¾®æœåŠ¡çš„情况下控制微æœåŠ¡ä¹‹é—´çš„æµé‡ã€‚ + +Rancher 与 Istio 集æˆï¼Œä½¿å¾—管ç†å‘˜æˆ–集群所有者å¯ä»¥å°† Istio 交给开å‘者团队，然åŽå¼€å‘者使用 Istio 执行安全策略，排查问题，或为è“绿部署，金ä¸é›€éƒ¨ç½²ï¼Œå’Œ A/B 测试进行æµé‡ç®¡ç†ã€‚ + +此核心æœåŠ¡ç½‘格支æŒä½†ä¸é™äºŽä»¥ä¸‹åŠŸèƒ½ï¼š + +- **管ç†æµé‡**：例如入å£å’Œå‡ºå£è·¯ç”±ã€æ–­è·¯ã€é•œåƒã€‚ +- **安全**：具有用于验è¯å’ŒæŽˆæƒæµé‡å’Œç”¨æˆ·çš„资æºï¼ŒåŒ…括 mTLS。 +- **å¯è§‚察性**：观察日志ã€æŒ‡æ ‡å’Œåˆ†å¸ƒå¼æµé‡ã€‚ + +[设置 Istio](istio-setup-guide.md) åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UIã€`kubectl` 或 ` Istioctl` æ¥ä½¿ç”¨ Istio çš„ controlplane 功能。 + +Istio 需è¦ç”± `cluster-admin` 设置åŽæ‰èƒ½åœ¨é¡¹ç›®ä¸­ä½¿ç”¨ã€‚ + + +## Rancher 2.5 的新功能 + +Istio å·²ç®€åŒ–äº†æ•´ä½“æž¶æž„ã€‚ç»“åˆ Pilotã€Citadelã€Galley å’Œ sidecar injector 创建了一个å•ç‹¬çš„组件 Istiod。Node Agent 功能也已åˆå¹¶åˆ° istio-agent 中。 + +以å‰ç”± Istio 安装的æ’件（cert-managerã€Grafanaã€Jaegerã€Kialiã€Prometheusã€Zipkin）现在需è¦å•ç‹¬å®‰è£…。Istio 支æŒå®‰è£…æ¥è‡ª Istio 项目的集æˆï¼Œå¹¶ä¿æŒä¸Žéž Istio 项目的兼容性。 + +ä½ ä»ç„¶å¯ä»¥é€šè¿‡å®‰è£… [Rancher Monitoring](monitoring-and-alerting.md) 或安装你自己的 Prometheus operator æ¥ä½¿ç”¨ Prometheus 集æˆã€‚Rancher çš„ Istio chart 还默认安装 Kiali，确ä¿ä½ å¯ä»¥å¼€ç®±å³ç”¨åœ°å…¨é¢äº†è§£å¾®æœåŠ¡ã€‚ + +Istio å·²ç»è„±ç¦»äº†ä½¿ç”¨ Helm 安装的方å¼ï¼ŒçŽ°åœ¨é€šè¿‡ Istioctl 二进制文件或 Istio Operator 进行安装。为了使用最简å•çš„æ–¹å¼ä¸Ž Istio 交互，Rancher çš„ Istio 会维护一个 Helm Chart，该 Chart 使用 Istioctl 二进制文件æ¥ç®¡ç†ä½ çš„ Istio 安装。 + +æ­¤ Helm Chart 将在 UI çš„**应用 & 市场市场**中æ供。有æƒè®¿é—® Rancher Chart 应用商店的用户需è¦å…ˆè®¾ç½® Istio，然åŽæ‰èƒ½åœ¨é¡¹ç›®ä¸­ä½¿ç”¨å®ƒã€‚ + +## Istio 附带的工具 + +我们的 [Istio](https://istio.io/) 安装程åºå°† istioctl 二进制命令包装在一个 Helm chart 中，其中包括一个覆盖文件的选项，用æ¥æ”¯æŒå¤æ‚的自定义é…置。 + +它还包括以下内容。 + +### Kiali + +Kiali 是一个全é¢çš„å¯è§†åŒ–辅助工具，用于绘制整个æœåŠ¡ç½‘格中的æµé‡å›¾ã€‚它å…许你查看它们的连接方å¼ï¼ŒåŒ…括它们之间的æµé‡é€ŸçŽ‡å’Œå»¶è¿Ÿã€‚ + +ä½ å¯ä»¥æ£€æŸ¥æœåŠ¡ç½‘格的è¿è¡ŒçŠ¶å†µï¼Œæˆ–深入查看å•ä¸ªç»„件的传入和传出请求。 + +### Jaeger + +Jaeger 是用于跟踪分布å¼ç³»ç»Ÿçš„工具。我们的 Istio 安装程åºåŒ…括能快速å¯åŠ¨çš„一体化 [Jaeger](https://www.jaegertracing.io/) 安装。 + +请注æ„，这ä¸æ˜¯ç¬¦åˆ Jaeger 生产è¦æ±‚的部署。此部署使用在内存中的存储组件，而 Jaeger 推è在生产环境中使用æŒä¹…存储组件。有关你所需的部署策略的更多信æ¯ï¼Œè¯·å‚阅 [Jaeger 文档](https://www.jaegertracing.io/docs/latest/operator/#production-strategy)。 + +## 先决æ¡ä»¶ + +在å¯ç”¨ Istio 之å‰ï¼Œå»ºè®®ä½ å…ˆç¡®è®¤ä½ çš„ Rancher worker 节点是å¦æœ‰è¶³å¤Ÿçš„ [CPU 和内存](../integrations-in-rancher/istio/cpu-and-memory-allocations.md)æ¥è¿è¡Œ Istio 的所有组件。 + +如果è¦åœ¨ RKE2 集群上安装 Istio，则需è¦æ‰§è¡Œä¸€äº›é¢å¤–的步骤。有关详细信æ¯ï¼Œè¯·å‚阅[本节](#在-rke2-集群上安装-istio-的其他步骤)。 + +## è®¾ç½®æŒ‡å— + +如需了解如何设置 Istio 并在项目中使用它，请å‚阅[设置指å—](istio-setup-guide.md)。 + +## å¸è½½ Istio + +è¦ä»Žé›†ç¾¤ã€å‘½å空间或工作负载中删除 Istio 组件，请å‚阅[å¸è½½ Istio](../integrations-in-rancher/istio/disable-istio.md)。 + +## 访问å¯è§†åŒ– + +> 默认情况下，åªæœ‰ cluster-admin å¯ä»¥è®¿é—® Kiali。有关如何å…许具有管ç†å‘˜ã€ç¼–辑或查看æƒé™çš„角色访问它们的说明，请å‚阅[本节](../integrations-in-rancher/istio/rbac-for-istio.md)。 + +在集群中设置 Istio åŽï¼Œä½ å¯ä»¥åœ¨ Rancher UI 中使用 Grafanaã€Prometheus å’Œ Kiali。 + +è¦è®¿é—® Grafana å’Œ Prometheus å¯è§†åŒ–： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å¯è§†åŒ–的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**监控**。 +1. 点击 **Grafana** 或任何其他仪表æ¿ã€‚ + +è¦è®¿é—® Kiali å¯è§†åŒ–： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦æŸ¥çœ‹ Kiali 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Istio**。 +1. å•å‡» **Kiali**。从这里，你å¯ä»¥è®¿é—®**æµé‡å›¾**或**æµé‡æŒ‡æ ‡**选项å¡ï¼Œä»Žè€Œå¯è§†åŒ–网络指标。 + +默认情况下，prometheus 会拾å–所有命å空间，并将数æ®ç”¨äºŽ Kiali 图。如果你想使用ä¸åŒçš„é…置进行 prometheus æ•°æ®æŠ“å–，请å‚阅[选择器/抓å–é…ç½®](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)。 + +你的角色决定了你对å¯è§†åŒ–的访问。åªæœ‰ `cluster-admin` 角色å¯ä»¥ä½¿ç”¨ Grafana å’Œ Prometheus。默认情况下，åªæœ‰ `cluster-admin` å¯ä»¥ä½¿ç”¨ Kiali UI，但是 `cluster-admin` å¯ä»¥é€šè¿‡ç¼–辑 Istio values.yaml æ¥å…许其他角色进行访问。 + +## 架构 + +Istio 安装了一个æœåŠ¡ç½‘格，它使用 [Envoy](https://www.envoyproxy.io) Sidecar 代ç†æ¥æ‹¦æˆªåˆ°æ¯ä¸ªå·¥ä½œè´Ÿè½½çš„æµé‡ã€‚这些 sidecar 拦截并管ç†æœåŠ¡ä¹‹é—´çš„通信，从而实现精细化观察并控制集群内的æµé‡ã€‚ + +åªæœ‰æ³¨å…¥äº† Istio sidecar 的工作负载å¯ä»¥é€šè¿‡ Istio 进行跟踪和控制。 + +如果命å空间å¯ç”¨äº† Istio，部署到命å空间的新工作负载会自动具有 Istio sidecar。你需è¦ä¸ºä¹‹å‰çš„工作负载手动å¯ç”¨ Istio。 + +有关 Istio sidecar 的更多信æ¯ï¼Œè¯·å‚阅 [Istio sidecare-injection 文档](https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/)。有关 Istio 架构的更多信æ¯ï¼Œè¯·å‚阅 [Istio 架构文档](https://istio.io/latest/docs/ops/deployment/architecture/)。 + +### 多个 Ingress + +默认情况下，æ¯ä¸ª Rancher é…置的集群都有一个 NGINX Ingress Controller æ¥å…许æµé‡è¿›å…¥é›†ç¾¤ã€‚Istio 还在 `istio-system` 命å空间中默认安装一个 Ingress Gateway。因此，你的集群将有两个 ingress。 + +![å¯ç”¨ Istio 的集群å¯ä»¥æœ‰ä¸¤ä¸ª ingress，分别是默认的 Nginx ingress 和默认的 Istio controller](/img/istio-ingress.svg) + +å¯ä»¥é€šè¿‡[覆盖文件](configuration-options.md#覆盖文件)æ¥å¯ç”¨å…¶ä»– Istio Ingress Gateway。 + +### Egress æ”¯æŒ + +默认情况下，Egress 网关是ç¦ç”¨çš„，但你å¯ä»¥åœ¨å®‰è£…或å‡çº§æ—¶ä½¿ç”¨ values.yaml 或[覆盖文件](configuration-options.md#覆盖文件)å¯ç”¨å®ƒã€‚ + +## 在 RKE2 集群上安装 Istio 的其他步骤 + +è¦åœ¨ RKE2 集群上安装 Istio，请按照[步骤](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md)进行æ“作。 + +## 在离线环境中å‡çº§ Istio + +现在，Istio Pod 安全策略默认å¯ç”¨ã€‚新值 `installer.releaseMirror.enabled` 已添加到 rancher-istio Chart 中，以å¯ç”¨å’Œç¦ç”¨æ”¯æŒç¦»çº¿å‡çº§çš„ Server。请注æ„，`installer.releaseMirror.enabled` 默认设置为 `false`。你å¯ä»¥åœ¨å®‰è£…或å‡çº§æ—¶æ ¹æ®éœ€è¦è®¾ç½®è¯¥å€¼ã€‚按照以下步骤执行： + +1. 在 Rancher UI 中é…置离线 Rancher 实例和离线自定义集群。 +2. 在集群中安装 Monitoring：**Cluster Explorer > Apps & Marketplace > Charts > Monitoring**。 +3. å°† Istio 所需的所有镜åƒæ‹‰å…¥åœ¨ç¦»çº¿çŽ¯å¢ƒä¸­ä½¿ç”¨çš„ç§æœ‰é•œåƒä»“库。 +4. 在集群中安装 Istio：**Cluster Explorer > Apps & Marketplace > Charts > Istio**。 + +:::note + +ä½ å¯ä»¥åœ¨æ–°å®‰è£…çš„ Istio 上å¯ç”¨ [Jaeger](https://www.jaegertracing.io/) å’Œ [Kiali](https://kiali.io/)ã€‚ä¸ºç¡®ä¿ Jaeger å’Œ Kiali 正常工作，请在安装期间将 `values.yaml` 中的 `installer.releaseMirror.enabled` 设置为 `true`。 + +::: + +5. å‡çº§ Istio。 + +:::caution + +如果你还没有执行æ“作，请设置 `installer.releaseMirror.enabled=true` 以å‡çº§ Istio。 + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/k3s-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/k3s-hardening-guide.md new file mode 100644 index 00000000000..fc356dfda17 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/k3s-hardening-guide.md @@ -0,0 +1,728 @@ +--- +title: K3s Hardening Guide +--- + +This document provides prescriptive guidance for how to harden a K3s cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for K3s clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 up to v1.25 | + +:::note +At the time of writing, the upstream CIS Kubernetes v1.25 benchmark is not yet available in Rancher. At this time Rancher is using the CIS v1.23 benchmark when scanning Kubernetes v1.25 clusters. +::: + +For more details on how to evaluate a hardened K3s cluster against the official CIS benchmark, refer to the K3s self-assessment guides for specific Kubernetes and CIS benchmark versions. + +K3s passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. K3s does not modify the host operating system. Any host-level modifications need to be done manually. +2. Certain CIS policy controls for `NetworkPolicies` and `PodSecurityStandards` (`PodSecurityPolicies` on v1.24 and older) restrict cluster functionality. + You must opt into having K3s configure these policies. Add the appropriate options to your command-line flags or configuration file (enable admission plugins), and manually apply the appropriate policies. + See further for more details. + +The first section (1.1) of the CIS Benchmark primarily focuses on pod manifest permissions and ownership. Since everything in the distribution is packaged in a single binary, this section does not apply to the core components of K3s. + +## Host-level Requirements + +### Ensure `protect-kernel-defaults` is set + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + protect-kernel-defaults: true +``` + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +This configuration needs to be done before setting the kubelet flag, otherwise K3s will fail to start. + +## Kubernetes Runtime Requirements + +The CIS Benchmark runtime requirements center around pod security (via PSP or PSA), network policies and API Server auditing logs. + +By default, K3s does not include any pod security or network policies. However, K3s ships with a controller that enforces any network policies you create. By default, K3s enables both the `PodSecurity` and `NodeRestriction` admission controllers, among others. + +### Pod Security + + + + +K3s v1.25 and newer support [Pod Security admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) for controlling pod security. + +You can specify the PSA configuration by setting the `defaultPodSecurityAdmissionConfigurationTemplateName` field in the cluster configuration in Rancher: + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted +``` + +The `rancher-restricted` template is provided by Rancher to enforce the highly-restrictive Kubernetes upstream [`Restricted`](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile with best practices for pod hardening. + + + + +K3s v1.24 and older support [Pod Security Policy (PSP)](https://v1-24.docs.kubernetes.io/docs/concepts/security/pod-security-policy/) for controlling pod security. + +You can enable PSPs by passing the following flags in the cluster configuration in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount +``` + +This maintains the `NodeRestriction` plugin and enables the `PodSecurityPolicy`. + +Once you enable PSPs, you can apply a policy to satisfy the necessary controls described in section 5.2 of the CIS Benchmark. + +:::note +These are manual checks in the CIS Benchmark. The CIS scan flags the results as `warning`, because manual inspection is necessary by the cluster operator. +::: + +Here is an example of a compliant PSP: + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false # CIS - 5.2.1 + allowPrivilegeEscalation: false # CIS - 5.2.5 + requiredDropCapabilities: # CIS - 5.2.7/8/9 + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false # CIS - 5.2.4 + hostIPC: false # CIS - 5.2.3 + hostPID: false # CIS - 5.2.2 + runAsUser: + rule: 'MustRunAsNonRoot' # CIS - 5.2.6 + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +``` + +For the example PSP to be effective, we need to create a `ClusterRole` and a `ClusterRoleBinding`. We also need to include a "system unrestricted policy" for system-level pods that require additional privileges, and an additional policy that allows the necessary sysctls for full functionality of ServiceLB. + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: system-unrestricted-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + hostIPC: true + hostNetwork: true + hostPID: true + hostPorts: + - max: 65535 + min: 0 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: svclb-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + allowedUnsafeSysctls: + - net.ipv4.ip_forward + - net.ipv6.conf.all.forwarding + fsGroup: + rule: RunAsAny + hostPorts: + - max: 65535 + min: 0 + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - restricted-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:system-unrestricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svclb-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - svclb-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-local-path-provisioner-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-coredns-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-cis-operator-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-psp +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-unrestricted-node-psp-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-unrestricted-svc-acct-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svclb-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svclb-psp +subjects: +- kind: ServiceAccount + name: svclb +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-local-path-provisioner-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-local-path-provisioner-psp +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-coredns-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-coredns-psp +subjects: +- kind: ServiceAccount + name: coredns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-cis-operator-psp-rolebinding + namespace: cis-operator-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-cis-operator-psp +subjects: +- kind: ServiceAccount + name: cis-operator-serviceaccount +``` + +The policies presented above can be placed in a file named `policy.yaml` in the `/var/lib/rancher/k3s/server/manifests` directory. Both the policy file and the its directory hierarchy must be created before starting K3s. A restrictive access permission is recommended to avoid leaking potential sensitive information. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +:::note +The critical Kubernetes additions such as CNI, DNS, and Ingress are run as pods in the `kube-system` namespace. Therefore, this namespace has a less restrictive policy, so that these components can run properly. +::: + + + + +### Network Policies + +CIS requires that all namespaces apply a network policy that reasonably limits traffic into namespaces and pods. + +:::note +This is a manual check in the CIS Benchmark. The CIS scan flags the result as a `warning`, because manual inspection is necessary by the cluster operator. +::: + +The network policies can be placed in the `policy.yaml` file in `/var/lib/rancher/k3s/server/manifests` directory. If the directory was not created as part of the PSP (as described above), it must be created first. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +Here is an example of a compliant network policy: + +```yaml +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-system +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-system +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: default +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: default +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-public +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-public +``` + +The active restrictions block DNS unless purposely allowed. Below is a network policy that allows DNS-related traffic: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-network-dns-policy + namespace: +spec: + ingress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Ingress +``` + +The metrics-server and Traefik ingress controller are blocked by default if network policies are not created to allow access. + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-metrics-server + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: metrics-server + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-svclbtraefik-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + svccontroller.k3s.cattle.io/svcname: traefik + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-traefik-v121-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ingress: + - {} + policyTypes: + - Ingress +``` + +:::note +You must manage network policies as normal for any additional namespaces you create. +::: + +### API Server audit configuration + +CIS requirements 1.2.22 to 1.2.25 are related to configuring audit logs for the API Server. K3s does not create by default the log directory and audit policy, as auditing requirements are specific to each user's policies and environment. + +If you need a log directory, it must be created before you start K3s. We recommend a restrictive access permission to avoid leaking sensitive information. + +```bash +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/logs +``` + +The following is a starter audit policy to log request metadata. This policy should be written to a file named `audit.yaml` in the `/var/lib/rancher/k3s/server` directory. Detailed information about policy configuration for the API server can be found in the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + +```yaml +--- +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata +``` + +Further configurations are also needed to pass CIS checks. These are not configured by default in K3s, because they vary based on your environment and needs: + +- Ensure that the `--audit-log-path` argument is set. +- Ensure that the `--audit-log-maxage` argument is set to 30 or as appropriate. +- Ensure that the `--audit-log-maxbackup` argument is set to 10 or as appropriate. +- Ensure that the `--audit-log-maxsize` argument is set to 100 or as appropriate. + +Combined, to enable and configure audit logs, add the following lines to the K3s cluster configuration file in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 +``` + +### Controller Manager Requirements + +CIS requirement 1.3.1 checks for garbage collection settings in the Controller Manager. Garbage collection is important to ensure sufficient resource availability and avoid degraded performance and availability. Based on your system resources and tests, choose an appropriate threshold value to activate garbage collection. + +This can be remediated by setting the following configuration in the K3s cluster file in Rancher. The value below is only an example. The appropriate threshold value is specific to each user's environment. + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 +``` + +### Configure `default` Service Account + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. + +For CIS requirement 5.1.5 the `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +This can be remediated by updating the `automountServiceAccountToken` field to `false` for the `default` service account in each namespace. + +For `default` service accounts in the built-in namespaces (`kube-system`, `kube-public`, `kube-node-lease`, and `default)`, K3s does not automatically do this. + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```shell +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Run the script every time a new service account is added to your cluster. + +## Reference Hardened K3s Template Configuration + +The following reference template configuration is used in Rancher to create a hardened K3s custom cluster based on each CIS control in this guide. This reference does not include other required **cluster configuration** directives, which vary based on your environment. + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,ServiceAccount # CIS 1.2.15, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount # CIS 1.2.15, 5.2, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 +``` + + + + +## Conclusion + +If you have followed this guide, your K3s custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our K3s self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-cluster-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-cluster-setup.md new file mode 100644 index 00000000000..67a402ff972 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-cluster-setup.md @@ -0,0 +1,7 @@ +--- +title: "Kubernetes 使用教程 " +--- + +本章节介ç»å¦‚何安装 Kubernetes 集群，使得 Rancher Server å¯ä»¥å®‰è£…在该集群上。 + +Rancher å¯ä»¥åœ¨ä»»ä½• Kubernetes 集群上è¿è¡Œã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md new file mode 100644 index 00000000000..702c3542c7e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md @@ -0,0 +1,76 @@ +--- +title: 在 Rancher 中设置 Kubernetes 集群 +description: é…ç½® Kubernetes 集群 +--- + +Rancher å…许你通过 Rancher UI æ¥åˆ›å»ºé›†ç¾¤ï¼Œä»Žè€Œç®€åŒ–了集群的创建æµç¨‹ã€‚Rancher æ供了多ç§å¯åŠ¨é›†ç¾¤çš„选项。你å¯ä»¥é€‰æ‹©æœ€é€‚åˆä½ çš„用例的选项。 + +本节默认你已对 Docker å’Œ Kubernetes 有一定的了解。如果你需è¦äº†è§£ Kubernetes 组件如何å作，请å‚è§ [Kubernetes 概念](../reference-guides/kubernetes-concepts.md)。 + +有关 Rancher Server é…置集群的方å¼ï¼Œä»¥åŠä½¿ç”¨ä»€ä¹ˆå·¥å…·æ¥åˆ›å»ºé›†ç¾¤çš„详细信æ¯ï¼Œè¯·å‚阅[产å“架构](rancher-manager-architecture.md)。 + + + +### ä¸åŒç±»åž‹é›†ç¾¤çš„管ç†åŠŸèƒ½ + +下表总结了æ¯ä¸€ç§ç±»åž‹çš„集群和对应的å¯ç¼–辑的选项和设置： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## 在托管的 Kubernetes æ供商中设置集群 + +在这ç§æƒ…况下，Rancher ä¸ä¼šé…ç½® Kubernetes，因为它是由 Google Kubernetes Engine (GKE)ã€Amazon Elastic Container Service for Kubernetes 或 Azure Kubernetes Service ç­‰æ供商安装的。 + +如果你使用 Kubernetes æ供商，例如 Google GKE，Rancher 将与对应的云 API 集æˆï¼Œå…许你从 Rancher UI ä¸ºæ‰˜ç®¡é›†ç¾¤åˆ›å»ºå’Œç®¡ç† RBAC。 + +详情请å‚阅[托管 Kubernetes 集群](set-up-clusters-from-hosted-kubernetes-providers.md)。 + +## 使用 Rancher å¯åŠ¨ Kubernetes + +在你自己的节点上é…ç½® Kubernetes 时，Rancher 使用 [Rancher Kubernetes Engine (RKE)](https://rancher.com/docs/rke/latest/en/) 作为库。RKE 是 Rancher 自己的轻é‡çº§ Kubernetes 安装程åºã€‚ + +在 RKE 集群中，Rancher ç®¡ç† Kubernetes 的部署。这些集群å¯ä»¥éƒ¨ç½²åœ¨ä»»ä½•è£¸æœºæœåŠ¡å™¨ã€äº‘æ供商或虚拟化平å°ä¸Šã€‚ + +这些节点å¯ä»¥é€šè¿‡ Rancher çš„ UI 动æ€é…置，该 UI 调用 [Docker Machine](https://docs.docker.com/machine/) 在å„ç§äº‘æ供商上å¯åŠ¨èŠ‚点。 + +如果你已ç»æœ‰ä¸€ä¸ªæƒ³è¦æ·»åŠ åˆ° RKE 集群的节点，你å¯ä»¥é€šè¿‡åœ¨èŠ‚点上è¿è¡Œ Rancher Agent 容器将节点添加到集群中。 + +有关详细信æ¯ï¼Œè¯·å‚阅 [RKE 集群](../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 + +### 在基础设施æ供商中å¯åŠ¨ Kubernetes 并é…置节点 + +Rancher å¯ä»¥åœ¨ Amazon EC2ã€DigitalOceanã€Azure 或 vSphere 等基础设施æ供商中动æ€é…置节点，然åŽåœ¨èŠ‚点上安装 Kubernetes。 + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)创建节点池。此模æ¿å®šä¹‰äº†è¦åœ¨äº‘æ供商中å¯åŠ¨çš„节点的å‚数。 + +使用由基础设施æ供商托管的节点的一个好处是，如果一个节点与集群失去连接，Rancher å¯ä»¥è‡ªåŠ¨æ›¿æ¢å®ƒï¼Œä»Žè€Œç»´æŠ¤é›†ç¾¤é…置。 + +Rancher UI 中状æ€ä¸º Active çš„[主机驱动](use-new-nodes-in-an-infra-provider.md#主机驱动)决定了å¯ç”¨äºŽåˆ›å»ºèŠ‚点模æ¿çš„云æ供商。 + +如需更多信æ¯ï¼Œè¯·å‚阅[基础设施æ供商托管的节点](use-new-nodes-in-an-infra-provider.md)。 + +### 在现有自定义节点上å¯åŠ¨ Kubernetes + +在设置这ç§ç±»åž‹çš„集群时，Rancher 会在现有的[自定义节点](use-existing-nodes.md)上安装 Kubernetes，从而创建一个自定义集群。 + +ä½ å¯ä»¥ä½¿ç”¨ä»»ä½•èŠ‚点，在 Rancher 中创建一个集群。 + +这些节点包括本地裸机æœåŠ¡å™¨ã€äº‘托管虚拟机或本地虚拟机。 + +## 注册现有集群 + +集群注册功能å–代了导入集群的功能。 + +注册 EKS 集群的优点更多。在大多数情况下，注册的 EKS 集群和在 Rancher 中创建的 EKS 集群在 Rancher UI 中的处ç†æ–¹å¼ç›¸åŒï¼ˆé™¤äº†åˆ é™¤ï¼‰ã€‚ + +删除在 Rancher 中创建的 EKS 集群åŽï¼Œè¯¥é›†ç¾¤å°†è¢«é”€æ¯ã€‚删除在 Rancher 中注册的 EKS 集群时，它与 Rancher Server 会断开连接，但它ä»ç„¶å­˜åœ¨ã€‚ä½ ä»ç„¶å¯ä»¥åƒåœ¨ Rancher 中注册之å‰ä¸€æ ·è®¿é—®å®ƒã€‚ + +详情请å‚è§[本页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)。 + +## 以编程方å¼åˆ›å»ºé›†ç¾¤ + +通过 Rancher 以编程方å¼éƒ¨ç½² Kubernetes 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 + +ä½ å¯ä»¥ä½¿ç”¨ Terraform 创建或导入 EKSã€GKEã€AKS 集群和 RKE 集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-components.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-components.md new file mode 100644 index 00000000000..7deb6057b8a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-components.md @@ -0,0 +1,17 @@ +--- +title: Kubernetes 组件 +--- + +本文列出的命令和步骤适用于 [Rancher å¯åŠ¨çš„ Kubernetes](../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群上的核心 Kubernetes 组件。 + +本文包括以下类别的故障排除æ示： + +- [etcd 节点故障排除](../troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md) +- [Controlplane 节点故障排除](../troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md) +- [nginx-proxy 节点故障排除](../troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md) +- [Worker 节点和通用组件故障排除](../troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md) + +## Kubernetes 组件图 + +![集群图](/img/clusterdiagram.svg)
+线æ¡è¡¨ç¤ºç»„件之间的通信。而颜色纯粹用于视觉辅助。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-resources-setup.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-resources-setup.md new file mode 100644 index 00000000000..146521b2bc5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/kubernetes-resources-setup.md @@ -0,0 +1,63 @@ +--- +title: Kubernetes èµ„æº +--- + +ä½ å¯ä»¥åœ¨ Rancher UI 中查看和æ“作 Kubernetes 集群中的所有自定义资æºå’Œ CRD。 + +## 工作负载 + +使用[工作负载](workloads-and-pods.md)将应用部署到集群节点，工作负载是包å«ç”¨äºŽè¿è¡Œåº”用的 pod 的对象，以åŠä¸ºéƒ¨ç½²è¡Œä¸ºè®¾ç½®è§„则的元数æ®ã€‚工作负载å¯ä»¥éƒ¨ç½²åœ¨é›†ç¾¤èŒƒå›´å†…，也å¯ä»¥éƒ¨ç½²åœ¨ä¸€ä¸ªå‘½å空间内。 + +部署工作负载时，你å¯ä»¥ä½¿ç”¨ä»»ä½•é•œåƒè¿›è¡Œéƒ¨ç½²ã€‚å¯ä¾›é€‰æ‹©çš„[工作负载类型](workloads-and-pods.md#工作负载类型)有多ç§ï¼Œå·¥ä½œè´Ÿè½½ç±»åž‹å†³å®šäº†ä½ çš„应用程åºçš„è¿è¡Œæ–¹å¼ã€‚ + +在工作负载部署之åŽï¼Œä½ å¯ä»¥ç»§ç»­ä½¿ç”¨å®ƒã€‚ä½ å¯ä»¥ï¼š + +- 将工作负载[å‡çº§](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md)到它è¿è¡Œçš„应用的更新版本。 +- 如果å‡çº§å‡ºçŽ°é—®é¢˜ï¼Œå°†å·¥ä½œè´Ÿè½½[回滚](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md)到以å‰çš„版本。 +- [添加一个 sidecar](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md)，这是一个支æŒä¸»è¦å·¥ä½œè´Ÿè½½çš„工作负载。 + +## è´Ÿè½½å‡è¡¡å’Œ Ingress + +### è´Ÿè½½å‡è¡¡å™¨ + +å¯åŠ¨åº”用程åºåŽï¼Œå®ƒä»…在集群中å¯ç”¨ã€‚无法从外部访问它。 + +如果你希望你的应用程åºå¯ä»¥ä»Žå¤–部访问，则必须å‘集群添加负载å‡è¡¡å™¨ã€‚如果用户知é“è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€å’Œåº”用的端å£å·ï¼Œè´Ÿè½½å‡è¡¡å™¨å¯ä»¥ä¸ºå¤–部连接创建一个访问集群的网关。 + +Rancher 支æŒä¸¤ç§ç±»åž‹çš„è´Ÿè½½å‡è¡¡å™¨ï¼š + +- [Layer-4 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#四层负载å‡è¡¡å™¨) +- [Layer-7 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#七层负载å‡è¡¡å™¨) + +有关详细信æ¯ï¼Œè¯·å‚阅[è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md)。 + +#### Ingress + +è´Ÿè½½å‡è¡¡å™¨åªèƒ½å¤„ç†æ¯ä¸ª service 的一个 IP 地å€ã€‚æ¢è¨€ä¹‹ï¼Œå¦‚果你在集群中è¿è¡Œäº†å¤šä¸ª service，则必须为æ¯ä¸ª service é…备一个负载å‡è¡¡å™¨ã€‚è¿è¡Œå¤šä¸ªè´Ÿè½½å‡è¡¡å™¨çš„花费å¯èƒ½éžå¸¸é«˜æ˜‚。因此，你å¯ä»¥ä½¿ç”¨ Ingress æ¥è§£å†³æ­¤é—®é¢˜ã€‚ + +Ingress 是一组充当负载å‡è¡¡å™¨çš„规则。Ingress 与一个或多个 Ingress Controller 一起动æ€è·¯ç”± service 的请求。Ingress 收到请求时，集群中的 Ingress Controller 会对负载å‡è¡¡å™¨è¿›è¡Œé…置，从而根æ®ä½ é…置的 service å­åŸŸæˆ–路径规则将请求定å‘到正确的 service。 + +有关详细信æ¯ï¼Œè¯·å‚阅 [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md)。 + +在项目中使用 Ingress 时，你å¯ä»¥é€šè¿‡è®¾ç½®å…¨å±€ DNS æ¡ç›®æ¥å°† Ingress 主机å编程到外部 DNS。 + +## æœåŠ¡å‘现 + +使用负载å‡è¡¡å™¨å’Œ/或 Ingress 将集群公开给外部请求åŽï¼Œä½ åªèƒ½é€šè¿‡ IP 地å€è®¿é—®é›†ç¾¤ã€‚è¦åˆ›å»ºå¯è§£æžçš„主机å，你必须创建æœåŠ¡è®°å½•ï¼Œè¯¥è®°å½•å°† IP 地å€ã€å¤–部主机åã€DNS 记录别åã€å·¥ä½œè´Ÿè½½æˆ–标记的 pod 映射到特定主机å。 + +有关详细信æ¯ï¼Œè¯·å‚阅[æœåŠ¡å‘现](../how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md)。 + +## åº”ç”¨ç¨‹åº + +除了å¯åŠ¨åº”用程åºçš„å„个组件外，你还å¯ä»¥ä½¿ç”¨ Rancher 应用商店æ¥å¯åŠ¨åº”ç”¨ï¼Œå³ Helm Chart。 + +## Kubernetes èµ„æº + +在 Rancher 项目或命å空间的上下文中，_资æº_ æ˜¯æ”¯æŒ Pod æ“作的文件和数æ®ã€‚在 Rancher 中，è¯ä¹¦ã€é•œåƒä»“库和密文都被视为资æºã€‚但是，Kubernetes 将资æºåˆ’分为ä¸åŒç±»åž‹çš„[密文（secret）](https://kubernetes.io/docs/concepts/configuration/secret/)。因此，在å•ä¸ªé¡¹ç›®æˆ–命å空间中，å„个资æºå¿…须具有唯一的å称以é¿å…冲çªã€‚资æºä¸»è¦ç”¨äºŽæ‰¿è½½æ•æ„Ÿä¿¡æ¯ï¼Œä½†ä¹Ÿæœ‰å…¶ä»–用途。 + +资æºåŒ…括： + +- [è¯ä¹¦](../how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md)：用于加密/解密进入或离开集群的数æ®çš„文件。 +- [ConfigMap](../how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md)：存储一般é…置信æ¯çš„文件，例如一组é…置文件。 +- [密文](../how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md)：存储密ç ã€token 或密钥等æ•æ„Ÿæ•°æ®çš„文件。 +- [é•œåƒä»“库](../how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md)：æºå¸¦ç”¨äºŽéªŒè¯ç§æœ‰é•œåƒä»“库的凭è¯çš„文件。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/launch-kubernetes-with-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/launch-kubernetes-with-rancher.md new file mode 100644 index 00000000000..966e88886a4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/launch-kubernetes-with-rancher.md @@ -0,0 +1,80 @@ +--- +title: 使用 Rancher å¯åŠ¨ Kubernetes +--- + +Rancher å¯ä»¥ä½¿ç”¨ä»»æ„节点å¯åŠ¨ Kubernetes 集群。在 Rancher 中将 Kubernetes 部署到这些节点上时，你å¯ä»¥é€‰æ‹© [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) 或 [RKE2](https://docs.rke2.io) å‘行版。Rancher å¯ä»¥åœ¨ä»»ä½•è®¡ç®—机上å¯åŠ¨ Kubernetes，包括： + +- 裸金属æœåŠ¡å™¨ +- 本地虚拟机 +- 由云厂商托管的虚拟机 + +Rancher å¯ä»¥åœ¨çŽ°æœ‰èŠ‚点上安装 Kubernetes，也å¯ä»¥åœ¨äº‘厂商中动æ€é…置节点并安装 Kubernetes。 + +Rancher 还å¯ä»¥åˆ›å»ºèŠ‚点池。在托管在云厂商的节点池上安装 Kubernetes 的一个好处是，如果一个节点与集群断开连接，Rancher å¯ä»¥è‡ªåŠ¨åˆ›å»ºå¦ä¸€ä¸ªèŠ‚点并将其加入集群，从而确ä¿èŠ‚点池的数é‡ç¬¦åˆè¦æ±‚。 + +## RKE + +### è¦æ±‚ + +如果你使用 RKE 建立集群，节点必须满足下游集群的[节点è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)。 + +### 在云厂商的新节点上å¯åŠ¨ Kubernetes + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)创建节点池。此节点模æ¿å®šä¹‰äº†è¦ç”¨äºŽåœ¨äº‘厂商中å¯åŠ¨èŠ‚点的å‚数。 + +在托管在云厂商的节点池上安装 Kubernetes 的一个好处是，如果一个节点与集群断开连接，Rancher å¯ä»¥è‡ªåŠ¨åˆ›å»ºå¦ä¸€ä¸ªèŠ‚点并将其加入集群，从而确ä¿èŠ‚点池的数é‡ç¬¦åˆè¦æ±‚。 + +有关详细信æ¯ï¼Œè¯·å‚阅[在新节点上å¯åŠ¨ Kubernetes](use-new-nodes-in-an-infra-provider.md)。 + +### 在现有自定义节点上å¯åŠ¨ Kubernetes + +在这ç§æƒ…况下，你希望将 Kubernetes 安装到裸机æœåŠ¡å™¨ã€æœ¬åœ°è™šæ‹Ÿæœºæˆ–云厂商中已存在的虚拟机上。使用此选项，你将在主机上è¿è¡Œ Rancher Agent Docker 容器。 + +如果è¦é‡å¤ä½¿ç”¨ä¹‹å‰çš„自定义集群中的节点，请在å¤ç”¨ä¹‹å‰[清ç†èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md)。如果你é‡å¤ä½¿ç”¨å°šæœªæ¸…ç†çš„节点，则集群é…ç½®å¯èƒ½ä¼šå¤±è´¥ã€‚ + +有关详细信æ¯ï¼Œè¯·å‚阅[自定义节点](use-existing-nodes.md)。 + +### 以编程方å¼åˆ›å»º RKE 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² RKE 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster)。 + +## RKE2 + +Rancher 2.6 支æŒç›´æŽ¥ä½¿ç”¨ Rancher UI é…ç½® [RKE2](https://docs.rke2.io/) 集群。RKE2，也称为 RKE Government，是一个完全符åˆæ ‡å‡†çš„ Kubernetes å‘行版，它专注于安全性和åˆè§„性。在 Rancher 2.6.4 åŠæ›´æ—©ç‰ˆæœ¬ä¸­ï¼ŒRKE2 é…置还处于技术预览阶段。 + +在 Rancher 2.6.5 中，RKE2 å·²ç» GA。 + +### è¦æ±‚ + +如果你使用 RKE2 建立集群，节点必须满足下游集群的[节点è¦æ±‚](https://docs.rke2.io/install/requirements)。 + +### 在云厂商的新节点上å¯åŠ¨ Kubernetes + +RKE2 基于使用上游[集群 API](https://github.com/kubernetes-sigs/cluster-api) 项目的新é…置框架。这个新é…置框架支æŒï¼š + +- å°† RKE2 集群é…置到 Rancher 具有主机驱动的任何æ供商上 +- 完全在 Rancher 中é…ç½® RKE2 集群 +- 除了 Canal 之外，还å¯ä»¥é€‰æ‹© CNI 选项， Calicoã€Cilium å’Œ Multus + +RKE2 é…置还包括在具有 Windows 节点的集群上安装 RKE2。 + +RKE2 çš„ Windows 功能包括： + +- Windows æ”¯æŒ vSphere 主机驱动 +- 用于 Windows RKE2 自定义集群的 Calico CNI +- Calico 的项目网络隔离 (PNI) +- ç”± containerd æ供支æŒçš„使用 RKE2 çš„ Windows 容器 +- 通过 Terraform é…ç½® Windows RKE2 集群 +- 直接从 Rancher UI é…ç½® Windows RKE2 自定义集群 + +è¦ä½¿ Windows æ”¯æŒ RKE2 自定义集群，请选择 Calico 作为 CNI。 + +### 在现有自定义节点上å¯åŠ¨ Kubernetes + +RKE2 还支æŒåœ¨é¢„é…置的虚拟机或裸机节点上安装自定义集群。 + +如果è¦é‡å¤ä½¿ç”¨ä¹‹å‰çš„自定义集群中的节点，请在å¤ç”¨ä¹‹å‰æ¸…ç†èŠ‚点。如果你é‡å¤ä½¿ç”¨å°šæœªæ¸…ç†çš„节点，则集群é…ç½®å¯èƒ½ä¼šå¤±è´¥ã€‚ + +### 以编程方å¼åˆ›å»º RKE2 集群 + +通过 Rancher 以编程方å¼éƒ¨ç½² RKE2 集群的最常è§æ–¹æ³•æ˜¯ä½¿ç”¨ Rancher 2 Terraform Provider。详情请å‚è§[使用 Terraform 创建集群](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster_v2)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/load-balancer-and-ingress-controller.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/load-balancer-and-ingress-controller.md new file mode 100644 index 00000000000..e88352bc8b3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/load-balancer-and-ingress-controller.md @@ -0,0 +1,61 @@ +--- +title: 在 Rancher 中设置负载å‡è¡¡å™¨å’Œ Ingress Controller +description: 了解如何设置负载å‡è¡¡å™¨å’Œ Ingress Controller 以在 Rancher 中é‡å®šå‘æœåŠ¡è¯·æ±‚，并了解负载å‡è¡¡å™¨çš„é™åˆ¶ +--- + +在 Rancher 中，你å¯ä»¥é€šè¿‡è®¾ç½®è´Ÿè½½å‡è¡¡å™¨å’Œ Ingress Controller æ¥é‡å®šå‘æœåŠ¡è¯·æ±‚。 + +## è´Ÿè½½å‡è¡¡å™¨ + +å¯åŠ¨åº”用程åºåŽï¼Œè¯¥åº”用程åºä»…在集群内å¯ç”¨ã€‚你无法从集群外部访问它。 + +如果你希望从外部访问应用程åºï¼Œåˆ™å¿…é¡»å‘集群添加负载å‡è¡¡å™¨æˆ– Ingress。如果用户知é“è´Ÿè½½å‡è¡¡å™¨çš„ IP 地å€å’Œåº”用的端å£å·ï¼Œè´Ÿè½½å‡è¡¡å™¨å¯ä»¥ä¸ºå¤–部连接创建一个访问集群的网关。 + +Rancher 支æŒä¸¤ç§ç±»åž‹çš„è´Ÿè½½å‡è¡¡å™¨ï¼š + +- [Layer-4 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#四层负载å‡è¡¡å™¨) +- [Layer-7 è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#七层负载å‡è¡¡å™¨) + +有关详细信æ¯ï¼Œè¯·å‚阅[è´Ÿè½½å‡è¡¡å™¨](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md)。 + +### è´Ÿè½½å‡è¡¡å™¨é™åˆ¶ + +è´Ÿè½½å‡è¡¡å™¨æœ‰å‡ ä¸ªéœ€è¦æ³¨æ„çš„é™åˆ¶ï¼š + +- è´Ÿè½½å‡è¡¡å™¨åªèƒ½å¤„ç†æ¯ä¸ª service 的一个 IP 地å€ã€‚æ¢è¨€ä¹‹ï¼Œå¦‚果你在集群中è¿è¡Œäº†å¤šä¸ª service，则必须为æ¯ä¸ª service é…备一个负载å‡è¡¡å™¨ã€‚è¿è¡Œå¤šä¸ªè´Ÿè½½å‡è¡¡å™¨çš„花费å¯èƒ½éžå¸¸é«˜æ˜‚。 + +- 如果你想将负载å‡è¡¡å™¨ä¸Žæ‰˜ç®¡çš„ Kubernetes 集群（å³æ‰˜ç®¡åœ¨ GKEã€EKS 或 AKS 中的集群）一起使用，则负载å‡è¡¡å™¨å¿…é¡»è¿è¡Œåœ¨è¯¥äº‘æ供商的基础设施上。请根æ®ä½ é…置集群的方å¼æŸ¥çœ‹è´Ÿè½½å‡è¡¡å™¨çš„兼容列表： + +- [æ”¯æŒ Layer-4 è´Ÿè½½å‡è¡¡](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#四层负载å‡è¡¡æ”¯æŒ) + +- [æ”¯æŒ Layer-7 è´Ÿè½½å‡è¡¡](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#七层负载å‡è¡¡æ”¯æŒ) + +## Ingress + +如上所述，使用负载å‡è¡¡å™¨çš„缺点是： + +- æ¯ä¸ªæœåŠ¡è´Ÿè½½å‡è¡¡å™¨åªèƒ½å¤„ç†ä¸€ä¸ª IP 地å€ã€‚ +- 如果你在集群中è¿è¡Œå¤šä¸ªæœåŠ¡ï¼Œåˆ™æ¯ä¸ªæœåŠ¡éƒ½å¿…é¡»é…备一个负载å‡è¡¡å™¨ã€‚ +- 为æ¯ä¸ªæœåŠ¡é…备负载å‡è¡¡å™¨çš„花费å¯èƒ½éžå¸¸é«˜æ˜‚。 + +相å，如果将æŸä¸ª Ingress 用作集群的入å£ç‚¹ï¼ŒIngress å¯ä»¥æ›´çµæ´»åœ°å°†æµé‡è·¯ç”±åˆ°å¤šä¸ª service。它å¯ä»¥å°†å¤šä¸ª HTTP 请求映射到 service，而无需为æ¯ä¸ª service æä¾›å•ç‹¬çš„ IP 地å€ã€‚ + +因此，如果你需è¦ä½¿ç”¨ç›¸åŒçš„ IP 地å€ã€Layer 7 å议或特æƒèŠ‚点端å£ï¼ˆ80 å’Œ 443）æ¥å…¬å¼€å¤šä¸ª service，你å¯ä»¥ä½¿ç”¨ä¸€ä¸ª Ingress。 + +Ingress 与一个或多个 Ingress Controller 一起动æ€è·¯ç”± service 的请求。Ingress 收到请求时，集群中的 Ingress Controller 会根æ®ä½ é…置的 service å­åŸŸæˆ–路径规则将请求定å‘到正确的 service。 + +æ¯ä¸ª Kubernetes Ingress 资æºéƒ½å¯¹åº”一个 `/etc/nginx/sites-available/` 中的文件，其中包å«ä¸€ä¸ªé…置对特定文件和文件夹的请求的 `server{}` é…ç½®å—。 + +Ingress 能为你的集群创建一个入å£ç«¯å£ï¼ˆä¸Žè´Ÿè½½å‡è¡¡å™¨ç±»ä¼¼ï¼‰ï¼Œå¯ä»¥ä½äºŽé›†ç¾¤çš„内部或外部。RKE å¯åŠ¨çš„集群中的 Ingress å’Œ Ingress Controller ç”± [Nginx](https://www.nginx.com/) æ供支æŒã€‚ + +Ingress 还支æŒå…¶ä»–功能，例如 SSL 终止ã€åŸºäºŽå称的虚拟主机等。 + +:::note 在高å¯ç”¨æ€§é…置中使用 Rancher： + +请é¿å…å°† Ingress 添加到 `local` 集群。Rancher å°† Nginx Ingress Controller 作为 Rancher 管ç†çš„ _所有_ 集群的全局入å£ç‚¹ï¼Œå…¶ä¸­åŒ…括 `local` 集群。因此，当用户å°è¯•è®¿é—®åº”用程åºæ—¶ï¼ŒRancher å¯èƒ½ä¼šç”±äºŽé‡æ–°åŠ è½½ Nginx é…置而断开连接。è¦è§£å†³è¿™ä¸ªé—®é¢˜ï¼Œæˆ‘们建议你仅在通过 Rancher å¯åŠ¨çš„集群中部署应用程åºã€‚ + +::: + +- 有关如何在 Rancher 中设置 Ingress 的更多信æ¯ï¼Œè¯·å‚阅 [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md)。 +- 有关 Ingress å’Œ Ingress Controller 的完整信æ¯ï¼Œè¯·å‚阅 [Kubernetes Ingress 文档](https://kubernetes.io/docs/concepts/services-networking/ingress/)。 +- 在项目中使用 Ingress 时，你å¯ä»¥é€šè¿‡è®¾ç½®å…¨å±€ DNS æ¡ç›®æ¥å°† Ingress 主机å编程到外部 DNS。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/logging.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/logging.md new file mode 100644 index 00000000000..a6f81844170 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/logging.md @@ -0,0 +1,117 @@ +--- +title: Rancher Logging é›†æˆ +description: Rancher 集æˆäº†ä¸»æµçš„日志æœåŠ¡ã€‚了解集æˆæ—¥å¿—æœåŠ¡çš„è¦æ±‚和优势，并在你的集群上å¯ç”¨ Logging。 +--- + +现在，Rancher 的日志管ç†ç”± [Logging operator](https://kube-logging.github.io/docs/) æ供支æŒï¼Œå®ƒå–代了以å‰çš„内部解决方案。 + +## å¯ç”¨ Logging + +ä½ å¯ä»¥è½¬åˆ°**应用**页é¢å¹¶å®‰è£… Logging 应用程åºï¼Œä»Žè€Œä¸º Rancher 管ç†çš„集群å¯ç”¨ Logging： + +1. 转到è¦å®‰è£… Logging 的集群，然åŽå•å‡» **Apps**。 +1. 点击 **Logging** 应用。 +1. 滚动到 Helm Chart README 的底部，然åŽå•å‡»**安装**。 + +**结果**：Logging 应用已部署到 `cattle-logging-system` 命å空间中。 + +## å¸è½½ Logging + +1. 转到è¦å®‰è£… Logging 的集群，然åŽå•å‡» **Apps**。 +1. 点击**已安装的应用**。 +1. 转到 `cattle-logging-system` 命å空间并选中 `rancher-logging` å’Œ `rancher-logging-crd` 框。 +1. å•å‡»**删除**。 +1. 确认**删除**。 + +**结果**：已å¸è½½ `rancher-logging`。 + +## 架构 + +有关 Logging 应用程åºå·¥ä½œåŽŸç†çš„更多信æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-architecture.md)。 + + + +## RBAC + +Rancher Logging 有两个角色，分别是 `logging-admin` å’Œ `logging-view`。有关如何以åŠä½•æ—¶ä½¿ç”¨è¿™äº›è§’色的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/logging/rbac-for-logging.md)。 + +## é…ç½® Logging è‡ªå®šä¹‰èµ„æº + +è¦ç®¡ç† `Flows`ã€`ClusterFlows`ã€`Outputs` å’Œ `ClusterOutputs`： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦é…ç½® Logging 自定义资æºçš„集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡» **Logging**。 + +### Flows å’Œ ClusterFlows + +有关é…ç½® `Flows` å’Œ `ClusterFlows` 的帮助，请å‚阅[此页é¢](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md)。 + +### Outputs å’Œ ClusterOutputs + +有关é…ç½® `Outputs` å’Œ `ClusterOutputs` 的帮助，请å‚阅[此页é¢](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md)。 + +## é…ç½® Logging Helm Chart + +有关在安装或å‡çº§ Logging 应用程åºæ—¶å¯é…置的选项，请å‚阅[此页é¢](../integrations-in-rancher/logging/logging-helm-chart-options.md)。 + +### Windows æ”¯æŒ + +ä½ å¯ä»¥ä»Ž Windows 节点[å¯ç”¨ Logging](../integrations-in-rancher/logging/logging-helm-chart-options.md#å¯ç”¨ç¦ç”¨-windows-节点-logging)。 + + +### 使用自定义 Docker 根目录 + +有关使用自定义 Docker 根目录的详细信æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-helm-chart-options.md#使用自定义-docker-根目录)。 + + +### 处ç†æ±¡ç‚¹å’Œå®¹å¿åº¦ + +有关如何在 Logging 应用程åºä¸­ä½¿ç”¨æ±¡ç‚¹å’Œå®¹å¿åº¦çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/logging/taints-and-tolerations.md)。 + + +### 在 SELinux 上使用 Logging V2 + +有关在å¯ç”¨äº† SELinux 的节点上使用 Logging 应用程åºçš„ä¿¡æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-helm-chart-options.md#å¯ç”¨-logging-应用程åºä»¥ä½¿ç”¨-selinux)。 + +### 其他日志æ¥æº + +默认情况下，Rancher 会收集所有类型集群的 controlplane 组件和节点组件的日志。在æŸäº›æƒ…况下，也会收集其他日志。有关详细信æ¯ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/logging/logging-helm-chart-options.md#其他日志æ¥æº)。 + + +## 故障排除 + +### 日志缓冲区导致 Pod 过载 + +æ ¹æ®ä½ çš„é…置，默认缓冲区大å°å¯èƒ½å¤ªå¤§å¹¶å¯¼è‡´ Pod 故障。å‡å°‘负载的一ç§æ–¹æ³•æ˜¯é™ä½Žè®°å½•å™¨çš„刷新间隔。这å¯ä»¥é˜²æ­¢æ—¥å¿—溢出缓冲区。你还å¯ä»¥æ·»åŠ æ›´å¤šåˆ·æ–°çº¿ç¨‹æ¥å¤„ç†å¤§é‡æ—¥å¿—试图åŒæ—¶å¡«å……缓冲区的情况。 + +有关如何é…置日志缓冲区æ¥æ»¡è¶³ä¼ä¸šéœ€æ±‚的更完整说明，请å‚阅[缓冲区](https://kube-logging.github.io/docs/configuration/plugins/outputs/buffer/)å’Œ [Fluentd é…ç½®](https://kube-logging.github.io/docs/logging-infrastructure/fluentd/)的官方 Logging Operator 文档。 + +### `cattle-logging` 命å空间正在é‡æ–°åˆ›å»º + +如果你的集群之å‰åœ¨æ—§ç‰ˆ Rancher UI 的全局视图中部署了 Logging，`cattle-logging` 命å空间å¯èƒ½ä¼šä¸æ–­è¢«é‡æ–°åˆ›å»ºã€‚ + +è¦è§£å†³è¿™ä¸ªé—®é¢˜ï¼Œä½ å¯ä»¥å°†æ‰€æœ‰ `clusterloggings.management.cattle.io` å’Œ `projectloggings.management.cattle.io` 自定义资æºä»Žç®¡ç†é›†ç¾¤ä¸­é’ˆå¯¹è¯¥é›†ç¾¤çš„命å空间中删除。 +这些自定义资æºä¼šå¯¼è‡´ Rancher 在下游集群中创建 `cattle-logging` 命å空间（如果ä¸å­˜åœ¨ï¼‰ã€‚ + +集群命å空间与集群 ID 匹é…，因此我们需è¦æ‰¾åˆ°æ¯ä¸ªé›†ç¾¤çš„集群 ID。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦èŽ·å– ID 的集群，然åŽå•å‡» **Explore**。 +2. 从以下其中一个 URL 中å¤åˆ¶ `` 的内容。`` 是集群命å空间å称。 + +```bash +# Cluster Management UI +https:///c// + +# Cluster Dashboard +https:///dashboard/c// +``` + +现在我们有了 `` 命å空间，我们å¯ä»¥åˆ é™¤å¯¼è‡´ `cattle-logging` ä¸æ–­é‡æ–°åˆ›å»ºçš„自定义资æºã€‚ +*警告*：请当å‰æœªä½¿ç”¨ç¡®ä¿ Logging（从旧版 Rancher UI 全局视图中安装的版本）。 + +```bash +kubectl delete crd clusterloggings.management.cattle.io -n +kubectl delete crd projectloggings.management.cattle.io -n +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/machine-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/machine-configuration.md new file mode 100644 index 00000000000..f7878f968b7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/machine-configuration.md @@ -0,0 +1,5 @@ +--- +title: 主机é…ç½® +--- + +主机é…置指的是如何将资æºåˆ†é…给虚拟机。请å‚阅 [Amazon EC2](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md)ã€[DigitalOcean](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md) å’Œ [Azure](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) 的文档以了解更多信æ¯ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-clusters.md new file mode 100644 index 00000000000..9e327a6138b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-clusters.md @@ -0,0 +1,32 @@ +--- +title: é›†ç¾¤ç®¡ç† +--- + +在 Rancher 中é…置集群åŽï¼Œä½ å¯ä»¥å¼€å§‹ä½¿ç”¨å¼ºå¤§çš„ Kubernetes 功能在开å‘ã€æµ‹è¯•æˆ–生产环境中部署和扩展容器化应用。 + +:::note + +本节默认你已对 Docker å’Œ Kubernetes 有一定的了解。如果你需è¦äº†è§£ Kubernetes 组件如何å作，请å‚è§ [Kubernetes 概念](../reference-guides/kubernetes-concepts.md)。 + +::: + +## 在 Rancher 中管ç†é›†ç¾¤ + +将集群[é…置到 Rancher](kubernetes-clusters-in-rancher-setup.md) 之åŽï¼Œ[集群所有者](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)需è¦ç®¡ç†è¿™äº›é›†ç¾¤ã€‚管ç†é›†ç¾¤çš„选项如下： + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## é…置工具 + +Rancher åŒ…å« Kubernetes 中未包å«çš„å„ç§å·¥å…·æ¥å助你进行 DevOps æ“作。Rancher å¯ä»¥ä¸Žå¤–部æœåŠ¡é›†æˆï¼Œè®©ä½ çš„集群更高效地è¿è¡Œã€‚工具分为以下几类： + +- å‘Šè­¦ +- Notifiers +- Logging +- Monitoring +- Istio æœåŠ¡ç½‘æ ¼ +- OPA Gatekeeper + +ä½ å¯ä»¥é€šè¿‡ **Apps** æ¥å®‰è£…工具。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-project-resource-quotas.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-project-resource-quotas.md new file mode 100644 index 00000000000..89c8a760f5e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-project-resource-quotas.md @@ -0,0 +1,46 @@ +--- +title: 项目资æºé…é¢ +--- + +如果多个团队共享一个集群，æŸä¸ªå›¢é˜Ÿå¯èƒ½ä¼šä½¿ç”¨è¿‡å¤šçš„å¯ç”¨èµ„æºï¼Œä¾‹å¦‚ CPUã€å†…å­˜ã€å­˜å‚¨ã€æœåŠ¡ã€Kubernetes 对象（如 Pod 或 Secret）等。你å¯ä»¥åº”用 _资æºé…é¢_ æ¥é˜²æ­¢è¿‡åº¦æ¶ˆè€—资æºã€‚资æºé…é¢æ˜¯ Rancher 用æ¥é™åˆ¶é¡¹ç›®æˆ–命å空间å¯ç”¨èµ„æºçš„功能。 + +本文介ç»å¦‚何在现有项目中创建资æºé…é¢ã€‚ + +你也å¯ä»¥åœ¨åˆ›å»ºæ–°é¡¹ç›®æ—¶è®¾ç½®èµ„æºé…é¢ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[创建新项目](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md#创建项目)。 + +Rancher 中的资æºé…é¢åŒ…å«ä¸Ž [Kubernetes 原生版本](https://kubernetes.io/docs/concepts/policy/resource-quotas/)相åŒçš„功能。Rancher 还扩展了资æºé…é¢çš„功能，从而让你将资æºé…é¢åº”用于项目。有关资æºé…é¢å¦‚何与 Rancher 中的项目一起使用的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md)。 + +### 将资æºé…é¢åº”用于现有项目 + +修改资æºé…é¢çš„使用场景如下： + +- é™åˆ¶æŸä¸ªé¡¹ç›®å’Œé¡¹ç›®ä¸‹çš„命åç©ºé—´èƒ½ä½¿ç”¨çš„èµ„æº +- 在资æºé…é¢å·²ç”Ÿæ•ˆçš„情况下，对项目å¯ç”¨çš„资æºè¿›è¡Œæ‰©å®¹æˆ–缩容 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œè¿›å…¥è¦åº”用资æºé…é¢çš„集群，然åŽå•å‡» **Explore**。 +1. å•å‡»**集群 > 项目/命å空间**。 +1. ç¡®ä¿ **Projects/Namespaces** 页é¢å¤„于 **Group by Project** 视图模å¼ã€‚ + ![Screenshot highlighting the "Group by Project" icon, above the list of projects. It resembles a folder.](/img/edit-project-config-for-resource-quotas-group-by-project.png) + +1. 找到è¦æ·»åŠ èµ„æºé…é¢çš„项目，选择与项目å称åŒè¡Œçš„ **â‹®**。 + ![Screenshot highlighting triple dots icon at the end of the same row as the project name.](/img/edit-project-config-for-resource-quotas-dots.png) + +1. 选择**编辑é…ç½®**。 + +1. 展开**资æºé™é¢**并å•å‡»**添加资æº**。你也å¯ä»¥ç¼–辑现有é…é¢ã€‚ + +1. 选择资æºç±»åž‹ã€‚有关类型的更多信æ¯ï¼Œè¯·å‚阅[é…é¢ç±»åž‹å‚考](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md)。 + +1. 输入**项目é™åˆ¶**å’Œ**命å空间默认é™åˆ¶**的值。 + + | 字段 | æè¿° | + | ----------------------- | -------------------------------------------------------------------------------------------------------- | + | 项目é™åˆ¶ | 项目的总资æºé™åˆ¶ã€‚ | + | 命å空间默认é™åˆ¶ | æ¯ä¸ªå‘½å空间的默认资æºé™åˆ¶ã€‚æ­¤é™åˆ¶ä¼šæ²¿ç”¨åˆ°é¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间。项目中所有命å空间的é™åˆ¶ä¹‹å’Œä¸åº”超过项目é™åˆ¶ã€‚ | + +1. **å¯é€‰**：添加更多é…é¢ã€‚ + +1. å•å‡»**创建**。 + +**结果**：资æºé…é¢å·²åº”用到你的项目和命å空间。如果你åŽç»­éœ€è¦æ·»åŠ æ›´å¤šå‘½å空间，Rancher 会验è¯é¡¹ç›®æ˜¯å¦å¯ä»¥å®¹çº³è¯¥å‘½å空间。如果项目无法分é…资æºï¼Œä½ ä»ç„¶å¯ä»¥åˆ›å»ºå‘½å空间，但命å空间将获得的资æºé…é¢ä¸º 0ã€‚ç„¶åŽ Rancher å°†ä¸å…许你创建任何å—æ­¤é…é¢é™åˆ¶çš„资æºã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-projects.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-projects.md new file mode 100644 index 00000000000..341878779ef --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-projects.md @@ -0,0 +1,37 @@ +--- +title: é¡¹ç›®ç®¡ç† +--- + +_项目_ 是 Rancher 中引入的对象，å¯å¸®åŠ©ä½ æ›´æœ‰ç»„ç»‡åœ°ç®¡ç† Kubernetes 集群中的命å空间。你å¯ä»¥ä½¿ç”¨é¡¹ç›®åˆ›å»ºå¤šç§Ÿæˆ·é›†ç¾¤ï¼Œè¿™ç§é›†ç¾¤å…许一组用户共享相åŒçš„底层资æºæ¥åˆ›å»ºåº”用，而应用之间ä¸ä¼šç›¸äº’å½±å“。 + +在层次结构方é¢ï¼š + +- 集群包å«é¡¹ç›® +- 项目包å«å‘½å空间 + +在 Rancher 中，你å¯ä»¥ä½¿ç”¨é¡¹ç›®å°†å¤šä¸ªå‘½å空间作为一个实体进行管ç†ã€‚在原生 Kubernetes（没有项目这个概念）中，RBAC 或集群资æºç­‰åŠŸèƒ½è¢«åˆ†é…给了å„个命å空间。如果集群中的多个命å空间需è¦åˆ†é…åŒæ ·çš„访问æƒé™ï¼Œåˆ†é…æƒé™ä¼šå˜å¾—éžå¸¸ç¹ç。å³ä½¿æ‰€æœ‰å‘½å空间都需è¦ç›¸åŒçš„æƒé™ï¼Œä½†ä¹Ÿæ— æ³•ä½¿ç”¨ä¸€ä¸ªæ“作中将这些æƒé™åº”用于所有命å空间。你必须é‡å¤åœ°å°†è¿™äº›æƒé™åˆ†é…ç»™æ¯ä¸ªå‘½å空间。 + +而 Rancher 通过引入项目的概念，通过å…许你在项目级别应用资æºå’Œè®¿é—®æƒé™ã€‚然åŽï¼Œé¡¹ç›®ä¸­çš„æ¯ä¸ªå‘½å空间都会继承这些资æºå’Œç­–略。因此你åªéœ€å°†èµ„æºå’Œç­–略分é…给项目å³å¯ï¼Œä¸éœ€è¦å°†å®ƒä»¬åˆ†é…ç»™æ¯ä¸ªå•ç‹¬çš„命å空间。 + +ä½ å¯ä»¥ä½¿ç”¨é¡¹ç›®æ‰§è¡Œä»¥ä¸‹æ“作： + +- [为用户分é…一组命å空间的访问æƒé™](../how-to-guides/new-user-guides/add-users-to-projects.md) +- 为用户分é…[项目中的特定角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)。角色å¯ä»¥æ˜¯æ‰€æœ‰è€…ã€æˆå‘˜ã€åªè¯»æˆ–[自定义](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md) +- [设置资æºé…é¢](manage-project-resource-quotas.md) +- [管ç†å‘½å空间](../how-to-guides/new-user-guides/manage-namespaces.md) +- [é…置工具](../reference-guides/rancher-project-tools.md) +- [é…ç½® Pod 安全策略](../how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md) + +### æŽˆæƒ + +éžç®¡ç†è€…用户åªæœ‰åœ¨[管ç†å‘˜](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)ã€[集群所有者或æˆå‘˜](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#集群角色)或[项目所有者](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)å°†éžç®¡ç†å‘˜ç”¨æˆ·æ·»åŠ åˆ°é¡¹ç›®çš„**æˆå‘˜**选项å¡åŽï¼Œæ‰èƒ½èŽ·å–项目的访问æƒé™ã€‚ + +创建项目的人自动æˆä¸º[项目所有者](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#项目角色)。 + +## åœ¨é¡¹ç›®ä¹‹é—´åˆ‡æ¢ + +è¦åœ¨é¡¹ç›®ä¹‹é—´åˆ‡æ¢ï¼Œè¯·ä½¿ç”¨å¯¼èˆªæ ä¸­çš„下拉èœå•ã€‚你也å¯ä»¥ç›´æŽ¥åœ¨å¯¼èˆªæ ä¸­åˆ‡æ¢é¡¹ç›®ï¼š + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ï¼Œè¿›å…¥è¦åˆ‡æ¢é¡¹ç›®çš„集群然åŽç‚¹å‡» **Explore**。 +1. 在顶部导航æ ä¸­ï¼Œé€‰æ‹©è¦æ‰“开的项目。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-role-based-access-control-rbac.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-role-based-access-control-rbac.md new file mode 100644 index 00000000000..a4cf2425ecb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/manage-role-based-access-control-rbac.md @@ -0,0 +1,25 @@ +--- +title: RBAC +--- + +Rancher 通过 _用户_ 进行授æƒç®¡ç†ã€‚如[身份验è¯](authentication-config.md)中所述，用户å¯ä»¥æ˜¯æœ¬åœ°ç”¨æˆ·ï¼Œä¹Ÿå¯ä»¥æ˜¯å¤–部用户。 + +é…置外部身份验è¯åŽï¼Œ**用户**页é¢ä¸Šæ˜¾ç¤ºçš„用户会å‘生å˜åŒ–。 + +- 如果你以本地用户身份登录，则仅显示本地用户。 + +- 如果你以外部用户身份登录，则会åŒæ—¶æ˜¾ç¤ºå¤–部用户和本地用户。 + +## 用户和角色 + +一旦用户登录到 Rancher，他们的 _授æƒ_，也就是他们在系统中的访问æƒé™ï¼Œå°†ç”± _全局æƒé™_ å’Œ _集群和项目角色_ 决定。 + +- [全局æƒé™](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md)： + + 定义用户在任何特定集群之外的授æƒã€‚ + +- [集群和项目角色](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)： + + 定义用户在分é…了角色的特定集群或项目中的授æƒã€‚ + +全局æƒé™ä»¥åŠé›†ç¾¤å’Œé¡¹ç›®è§’色都是基于 [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) 实现的。因此，æƒé™å’Œè§’色的底层实现是由 Kubernetes 完æˆçš„。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-alerting-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-alerting-guides.md new file mode 100644 index 00000000000..7769d685ae4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-alerting-guides.md @@ -0,0 +1,10 @@ +--- +title: Monitoring æŒ‡å— +--- + +- [å¯ç”¨ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [å¸è½½ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring 工作负载](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [自定义 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [æŒä¹…化 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [调试高内存使用率](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-and-alerting.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-and-alerting.md new file mode 100644 index 00000000000..71ac5e0ec93 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-and-alerting.md @@ -0,0 +1,92 @@ +--- +title: 监控和告警 +description: Prometheus å…许你查看æ¥è‡ªä¸åŒ Rancher å’Œ Kubernetes 对象的指标。了解监控范围以åŠå¦‚何å¯ç”¨é›†ç¾¤ç›‘控 +--- + +ä½ å¯ä»¥ä½¿ç”¨ `rancher-monitoring` 应用，将业界领先的开æºç›‘控和告警解决方案快速部署到你的集群中。 + + +### 功能 + +Prometheus 支æŒæŸ¥çœ‹ Rancher å’Œ Kubernetes 对象的指标。通过使用时间戳，Prometheus 能让你通过 Rancher UI 或 Grafana（与 Prometheus 一起部署的分æžæŸ¥çœ‹å¹³å°ï¼‰ä»¥æ›´å®¹æ˜“阅读的图表和视觉形å¼æ¥æŸ¥è¯¢å’ŒæŸ¥çœ‹è¿™äº›æŒ‡æ ‡ã€‚ + +通过查看 Prometheus 从集群的 controlplaneã€èŠ‚点和 deployment 中抓å–çš„æ•°æ®ï¼Œä½ å¯ä»¥éšæ—¶äº†è§£é›†ç¾¤ä¸­å‘生的所有事件。然åŽï¼Œä½ å¯ä»¥ä½¿ç”¨è¿™äº›åˆ†æžæ¥æ›´å¥½åœ°è¿è¡Œä½ çš„环境，例如在系统紧急情况å‘生之å‰é˜»æ­¢å®ƒä»¬ã€åˆ¶å®šç»´æŠ¤ç­–略，或æ¢å¤å´©æºƒçš„æœåŠ¡å™¨ã€‚ + +在 Rancher v2.5 中引入的 `rancher-monitoring` operator ç”± [Prometheus](https://prometheus.io/)ã€[Grafana](https://grafana.com/grafana/)ã€[Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)， [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) å’Œ [Prometheus adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) æ供支æŒã€‚ + +Monitoring 应用： + +- 监控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚ +- æ ¹æ® Prometheus 收集的指标定义告警。 +- 创建自定义 Grafana 仪表æ¿ã€‚ +- 使用 Prometheus Alertmanager 通过电å­é‚®ä»¶ã€Slackã€PagerDuty ç­‰é…置告警通知。 +- æ ¹æ® Prometheus 收集的指标，将预先计算的ã€ç»å¸¸éœ€è¦çš„，或计算æˆæœ¬é«˜çš„表达å¼å®šä¹‰ä¸ºæ–°çš„时间åºåˆ—。 +- 通过 Prometheus Adapter，将从 Prometheus 收集的指标公开给 Kubernetes Custom Metrics API，以便在 HPA 中使用。 + +有关监控组件如何ååŒå·¥ä½œçš„说明，请å‚阅 [Monitoring 工作原ç†](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +## 默认组件和部署 + +### å†…ç½®ä»ªè¡¨æ¿ + +默认情况下，监控应用将 Grafana 仪表æ¿ï¼ˆç”± [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) 项目策划）部署到集群上。 + +它害部署一个 Alertmanager UI 和一个 Prometheus UI。有关这些工具的更多信æ¯ï¼Œè¯·å‚è§[内置仪表æ¿](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md)。 +### 默认指标 Exporter + +默认情况下，Rancher Monitoring 会部署 Exporter（例如 [node-exporter](https://github.com/prometheus/node_exporter) å’Œ [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics)）。 + +这些默认 Exporter 会自动从 Kubernetes é›†ç¾¤çš„æ‰€æœ‰ç»„ä»¶ï¼ˆåŒ…æ‹¬å·¥ä½œè´Ÿè½½ï¼‰ä¸­æŠ“å– CPU 和内存的指标。 + +### 默认告警 + +Monitoring 应用会默认部署一些告警。è¦æŸ¥çœ‹é»˜è®¤å‘Šè­¦ï¼Œè¯·è½¬åˆ° [Alertmanager UI](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md#alertmanager-ui) 并å•å‡»**展开所有组**。 + +### Rancher UI 中公开的组件 + +有关 Rancher UI 中公开的监控组件列表，以åŠç¼–辑它们的常è§ç”¨ä¾‹ï¼Œè¯·å‚阅[本节](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#rancher-ui-中公开的组件)。 + +## RBAC + +有关é…ç½® monitoring 访问æƒé™çš„ä¿¡æ¯ï¼Œè¯·å‚阅[此页é¢](../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md)。 + +## æŒ‡å— + +- [å¯ç”¨ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [å¸è½½ monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring 工作负载](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [自定义 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [æŒä¹…化 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [调试高内存使用率](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) + +## é…ç½® + +### 在 Rancher 中é…ç½® Monitoring èµ„æº + +此处的é…ç½®å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚如需更多信æ¯ï¼Œè¯·å‚阅 [monitoring 的工作原ç†](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +- [ServiceMonitor å’Œ PodMonitor](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [接收器](../reference-guides/monitoring-v2-configuration/receivers.md) +- [路由](../reference-guides/monitoring-v2-configuration/routes.md) +- [PrometheusRule](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) +- [Prometheus](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) +- [Alertmanager](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### é…ç½® Helm Chart 选项 + +有关 `rancher-monitoring` Chart 选项的更多信æ¯ï¼ŒåŒ…括设置资æºé™åˆ¶å’Œè¯·æ±‚的选项，请å‚阅 [Helm Chart 选项](../reference-guides/monitoring-v2-configuration/helm-chart-options.md)。 + +## Windows é›†ç¾¤æ”¯æŒ + +如果 Monitoring 部署到 RKE1 Windows 集群，Monitoring V2 将自动部署 [windows-exporter](https://github.com/prometheus-community/windows_exporter) DaemonSet 并设置 ServiceMonitor，以从æ¯ä¸ªéƒ¨ç½²çš„ Pod 中收集指标。这将使用 `windows_` 指标填充 Prometheus，这些指标与 [node_exporter](https://github.com/prometheus/node_exporter) 为 Linux 主机导出的 `node_` 指标类似。 + +为了能够为 Windows 完全部署 Monitoring V2，你的所有 Windows 主机都必须至少具有 v0.1.0 çš„ [wins](https://github.com/rancher/wins) 版本。 + +有关如何在现有 Windows 主机上å‡çº§ wins 版本的更多信æ¯ï¼Œè¯·å‚阅 [Windows 集群对 Monitoring V2 的支æŒ](../integrations-in-rancher/monitoring-and-alerting/windows-support.md)。 + + +## 已知问题 + +有一个[已知问题](https://github.com/rancher/rancher/issues/28787#issuecomment-693611821)ï¼Œå³ K3s 集群需è¦çš„内存超过分é…的默认内存。如果你在 K3s 集群上å¯ç”¨ Monitoring，将 `prometheus.prometheusSpec.resources.memory.limit` 设置为 2500 Mi，并将 `prometheus.prometheusSpec.resources.memory.request` 设置为 1750 Mi。 + +如需获å–æ„è§å’Œå»ºè®®ï¼Œè¯·å‚阅[调试高内存使用情况](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-v2-configuration-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-v2-configuration-guides.md new file mode 100644 index 00000000000..cdc63d55f53 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-v2-configuration-guides.md @@ -0,0 +1,51 @@ +--- +title: é…ç½® +--- + +本文介ç»åœ¨ Rancher UI 中é…ç½® Monitoring V2 的一些最é‡è¦é€‰é¡¹ã€‚ + +有关为 Prometheus é…置自定义抓å–目标和规则的信æ¯ï¼Œè¯·å‚阅 [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 的上游文档。Prometheus Operator [设计文档](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md)中解释了一些最é‡è¦çš„自定义资æºã€‚Prometheus Operator 文档还å¯ä»¥å¸®åŠ©ä½ è®¾ç½® RBACã€Thanos 或进行自定义é…置。 + +## 设置资æºé™åˆ¶å’Œè¯·æ±‚ + +安装 `rancher-monitoring` æ—¶å¯ä»¥é…ç½® Monitoring 应用的资æºè¯·æ±‚å’Œé™åˆ¶ã€‚有关默认é™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/helm-chart-options.md#é…置资æºé™åˆ¶å’Œè¯·æ±‚)。 + +:::tip + +在空闲集群上，Monitoring å¯èƒ½ä¼šå ç”¨å¾ˆå¤š CPU 资æºã€‚è¦æ高性能，请关闭 Prometheus Adapter。 + +::: + +## Prometheus é…ç½® + +通常ä¸éœ€è¦ç›´æŽ¥ç¼–辑 Prometheus 自定义资æºã€‚ + +相å，è¦è®© Prometheus 抓å–自定义指标，你åªéœ€åˆ›å»ºä¸€ä¸ªæ–°çš„ ServiceMonitor 或 PodMonitor æ¥å°† Prometheus é…置为抓å–其他指标。 + + +### ServiceMonitor å’Œ PodMonitor é…ç½® + +有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md)。 + +### 高级 Prometheus é…ç½® + +有关直接编辑 Prometheus 自定义资æºï¼ˆå¯¹é«˜çº§ç”¨ä¾‹å¯èƒ½æœ‰å¸®åŠ©ï¼‰çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md)。 + +## Alertmanager é…ç½® + +Alertmanager 自定义资æºé€šå¸¸ä¸éœ€è¦ç›´æŽ¥ç¼–辑。在常è§ç”¨ä¾‹ä¸­ï¼Œä½ å¯ä»¥é€šè¿‡æ›´æ–°è·¯ç”±å’ŒæŽ¥æ”¶å™¨æ¥ç®¡ç†å‘Šè­¦ã€‚ + +路由和接收器是 Alertmanager 自定义资æºé…置的一部分。在 Rancher UI 中，路由（Route）和接收器（Receiver）并ä¸æ˜¯çœŸæ­£çš„自定义资æºï¼Œè€Œæ˜¯ Prometheus Operator 用æ¥å°†ä½ çš„é…置与 Alertmanager 自定义资æºåŒæ­¥çš„伪自定义资æºã€‚当路由和接收器更新时，Monitoring 应用将自动更新 Alertmanager æ¥å映这些更改。 + +对于一些高级用例，你å¯èƒ½éœ€è¦ç›´æŽ¥é…ç½® Alertmanager。有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 + +### 接收器 + +接收器（Receiver）用于设置通知。有关如何é…置接收器的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/receivers.md)。 +### 路由 + +路由（Route）在通知到达接收器之å‰è¿‡æ»¤å®ƒä»¬ã€‚æ¯æ¡è·¯ç”±éƒ½éœ€è¦å¼•ç”¨ä¸€ä¸ªå·²ç»é…置好的接收器。有关如何é…置路由的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../reference-guides/monitoring-v2-configuration/routes.md)。 + +### 高级é…ç½® + +有关直接编辑 Alertmanager 自定义资æºï¼ˆå¯¹é«˜çº§ç”¨ä¾‹å¯èƒ½æœ‰å¸®åŠ©ï¼‰çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-v2-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-v2-configuration.md new file mode 100644 index 00000000000..6cd47ac5d10 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/monitoring-v2-configuration.md @@ -0,0 +1,11 @@ +--- +title: Monitoring V2 é…ç½® +--- + +本文介ç»äº†åœ¨ Rancher 中é…ç½® Monitoring V2 çš„å¿…è¦é€‰é¡¹ï¼š + +- [接收器é…ç½®](../reference-guides/monitoring-v2-configuration/receivers.md) +- [路由é…ç½®](../reference-guides/monitoring-v2-configuration/routes.md) +- [ServiceMonitor å’Œ PodMonitor é…ç½®](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [Helm Chart 选项](../reference-guides/monitoring-v2-configuration/helm-chart-options.md) +- [示例](../reference-guides/monitoring-v2-configuration/examples.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/new-user-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/new-user-guides.md new file mode 100644 index 00000000000..7e46d5b69a8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/new-user-guides.md @@ -0,0 +1,9 @@ +--- +title: æ–°ç”¨æˆ·æŒ‡å— +--- + +新用户指å—（也称为**教程**）æ述了æŸäº›æ“作的实际步骤。这些文档是“学习导å‘â€çš„，也就是说用户通过“æ“作â€æ¥å­¦ä¹ ã€‚ + +新用户指å—旨在引导åˆå­¦è€…或 Rancher 的日常用户通过一系列步骤æ¥å­¦ä¹ å¦‚何进行æŸäº›æ“作。这些文档旨在帮助用户通过使用易于éµå¾ªã€æœ‰æ„义且å¯é‡å¤çš„æ“作æ¥äº†è§£å¦‚何完æˆä»»åŠ¡ã€‚这些指å—将帮助用户完æˆå·¥ä½œï¼Œå¹¶èƒ½ç«‹åˆ»çœ‹åˆ°æ•ˆæžœã€‚ + +正常æ¥è¯´ï¼Œæ™®é€š Rancher 用户的技术水平高于“åˆå­¦è€…â€ï¼Œä½†æ˜¯ï¼Œæˆ‘们的新用户指å—为åˆå­¦è€…以åŠç»éªŒä¸°å¯Œçš„ Rancher 用户æ供相åŒçš„指导。我们结åˆä½¿ç”¨äº†é«˜çº§è¯­è¨€å’ŒæŠ€æœ¯è¯­è¨€æ¥ä»‹ç»å„ä¸ªä¸»é¢˜ï¼Œå¹¶æŒ‡å¯¼ç”¨æˆ·å®Œæˆ Rancher 用户需è¦äº†è§£çš„通用任务。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/node-template-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/node-template-configuration.md new file mode 100644 index 00000000000..489dbbafa13 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/node-template-configuration.md @@ -0,0 +1,5 @@ +--- +title: 节点模æ¿é…ç½® +--- + +è¦äº†è§£èŠ‚点模æ¿é…置，请å‚阅[EC2 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md)ã€[DigitalOcean 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md)ã€[Azure 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md)ã€[vSphere 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md)å’Œ [Nutanix 节点模æ¿é…ç½®](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/nutanix.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/nutanix.md new file mode 100644 index 00000000000..a3e28a1182e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/nutanix.md @@ -0,0 +1,17 @@ +--- +title: 创建 Nutanix AOS 集群 +description: 使用 Rancher 创建 Nutanix AOS (AHV) 集群。集群å¯èƒ½åŒ…括具有ä¸åŒå±žæ€§çš„ VM 组，这些属性å¯ç”¨äºŽç»†ç²’度控制节点的大å°ã€‚ +--- + +[Nutanix Acropolis æ“作系统](https://www.nutanix.com/products/acropolis) (Nutanix AOS) 是适用于 Nutanix 超èžåˆåŸºç¡€æž¶æž„å¹³å°çš„æ“作系统。AOS 带有一个å为 [Acropolis Hypervisor（AHV）](https://www.nutanix.com/products/ahv)的内置虚拟机监控程åºã€‚ä½ å¯ä»¥ç»“åˆä½¿ç”¨ Rancher 与 Nutanix AOS (AHV)，从而在本地体验云环境的æ“作。 + +Rancher å¯ä»¥åœ¨ AOS (AHV) 中é…置节点并在其上安装 Kubernetes。在 AOS 中创建 Kubernetes 集群时，Rancher 首先与 Prism Central API 通信æ¥é…置指定数é‡çš„虚拟机。然åŽåœ¨è™šæ‹Ÿæœºä¸Šå®‰è£… Kubernetes。 + +Nutanix 集群å¯èƒ½ç”±å¤šç»„具有ä¸åŒå±žæ€§ï¼ˆä¾‹å¦‚内存或 vCPU æ•°é‡ï¼‰çš„ VM 组æˆã€‚è¿™ç§åˆ†ç»„å…许对æ¯ä¸ª Kubernetes 角色的节点大å°è¿›è¡Œç»†ç²’度控制。 + +- [创建 Nutanix 集群](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md#创建-nutanix-aos-集群) +- [é…置存储](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md) + +## 创建 Nutanix 集群 + +在[本节](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md)中，你将学习如何使用 Rancher 在 Nutanix AOS 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/other-installation-methods.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/other-installation-methods.md new file mode 100644 index 00000000000..f56c0cbb179 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/other-installation-methods.md @@ -0,0 +1,19 @@ +--- +title: å…¶ä»–å®‰è£…æ–¹å¼ +--- + +### 离线安装 + +按照[以下步骤](air-gapped-helm-cli-install.md)在离线环境中安装 Rancher Server。 + +离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +### Docker 安装 + +[å•èŠ‚点 Docker 安装](rancher-on-a-single-node-with-docker.md)适用于想è¦æµ‹è¯• Rancher 的用户。你无需使用 Helm 在 Kubernetes 集群上è¿è¡Œ Rancher，你å¯ä»¥ä½¿ç”¨ `docker run` 命令，把 Rancher Server 组件安装到å•ä¸ªèŠ‚点上。 + +Docker 安装仅用于开å‘和测试环境。 + +由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此，如果该节点å‘生故障，由于其他节点上没有å¯ç”¨çš„ etcd æ•°æ®å‰¯æœ¬ï¼Œä½ å°†ä¸¢å¤± Rancher Server 的所有数æ®ã€‚ + +Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/prometheus-federator-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/prometheus-federator-guides.md new file mode 100644 index 00000000000..742fd54b6fe --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/prometheus-federator-guides.md @@ -0,0 +1,8 @@ +--- +title: Prometheus Federator æŒ‡å— +--- + +- [å¯ç”¨ Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md) +- [å¸è½½ Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md) +- [自定义 Grafana 仪表æ¿](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md) +- [设置工作负载](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/prometheus-federator.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/prometheus-federator.md new file mode 100644 index 00000000000..f78b8d9543f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/prometheus-federator.md @@ -0,0 +1,104 @@ +--- +title: Prometheus Federator +--- + +Prometheus Federator（也称为 Project Monitoring V2）基于 [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator) 部署一个 Helm Project Operator。该 Operator ç®¡ç† Helm Chart 的部署，æ¯ä¸ª Operator 都包å«ä¸€ä¸ª Project Monitoring Stack，而æ¯ä¸ªå †æ ˆéƒ½åŒ…å«ï¼š + +- [Prometheus](https://prometheus.io/)（由 [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 在外部管ç†ï¼‰ +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/)（由 [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) 在外部管ç†ï¼‰ +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana)ï¼ˆé€šè¿‡åµŒå…¥å¼ Helm Chart 部署） +- 基于 [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) 社区策划资æºé›†åˆçš„默认 PrometheusRules å’Œ Grafana ä»ªè¡¨æ¿ +- 监视已部署资æºçš„默认 ServiceMonitor + +:::note é‡è¦æ示： + +Prometheus Federator 适åˆåœ¨å·²å®‰è£… Prometheus Operator CRD 的集群中与现有的 Prometheus Operator Deployment 一起部署。 + +::: + +## Operator å·¥ä½œåŽŸç† + +1. 在部署此 Chart 时，用户å¯ä»¥åˆ›å»º ProjectHelmCharts CR，并在**项目 Registration 命å空间 (`cattle-project-`)** 中将 `spec.helmApiVersion` 设置为 `monitoring.cattle.io/v1alpha1`（在 Rancher UI 中也称为“项目监控â€ï¼‰ã€‚ +2. 在看到æ¯ä¸ª ProjectHelmChartCR 时，Operator 会代表项目所有者在**项目 Release 命å空间 (`cattle-project--monitoring`)** 中自动部署一个 Project Prometheus 堆栈（基于 ProjectHelmChart 控制器在 **Operator / System Namespace** 中创建的 HelmChart CR å’Œ HelmRelease CR）。 +3. RBAC 将自动分é…到项目 Release 命å空间中，从而å…许用户查看 Prometheusã€Alertmanager 以åŠå·²éƒ¨ç½²çš„ Project Monitoring Stack çš„ Grafana UI（基于在项目 Registration 命å空间上针对[é¢å‘用户的默认 Kubernetes 角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)定义的 RBAC）。有关详细信æ¯ï¼Œè¯·å‚阅[é…ç½® RBAC](../reference-guides/prometheus-federator/rbac.md)。 + +### 什么是项目？ + +在 Prometheus Federator 中，项目是一组å¯ä»¥ç”± `metav1.LabelSelector` 标识的命å空间。默认情况下，用于标识项目的标签是 `field.cattle.io/projectId`，该标签用于标识给定 Rancher 项目中包å«çš„命å空间。 + +### é…置由 ProjectHelmChart 创建的 Helm 版本 + +æ­¤ ProjectHelmChart 资æºçš„ `spec.values` 对应于为底层 Helm Chart é…置的 `values.yaml` 覆盖，该 Helm Chart 是 Operator 代表用户部署的。è¦æŸ¥çœ‹åº•å±‚ Chart çš„ `values.yaml` 规范，你å¯ä»¥é€‰æ‹©ä»¥ä¸‹å…¶ä¸­ä¸€ç§æ–¹å¼ï¼š + +- 查看ä½äºŽ [`charts/rancher-project-monitoring` 中的 `rancher/prometheus-federator`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) çš„ Chart 定义（Chart 版本会绑定到 Operator 版本）。 +- 查找在æ¯ä¸ªé¡¹ç›® Registration 命å空间中自动创建的å为 `monitoring.cattle.io.v1alpha1` çš„ ConfigMap，其中包å«ç”¨äºŽé…ç½® Chart（直接嵌入到 `prometheus-federator` 二进制文件中）的`values.yaml` å’Œ `questions.yaml`。 + +### 命å空间 + +Prometheus Federator 是基于 [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator) çš„ Project Operator，Prometheus Federator æ供了三类命å空间供 Operator 查找： + +1. **Operator / System 命å空间**：部署 Operator 的命å空间（例如 `cattle-monitoring-system`）。此命å空间将包å«è¯¥ Operator 监视的所有 ProjectHelmChart 的所有 HelmChart å’Œ HelmRelease。**åªæœ‰é›†ç¾¤ç®¡ç†å‘˜æ‰èƒ½è®¿é—®æ­¤å‘½å空间**。 + +2. **项目 Registration 命å空间 (`cattle-project-`)**：Operator 在这些命å空间中监视 ProjectHelmChart。对于在项目å‘布命å空​​间中创建的自动分é…çš„ RBAC，应用于此命å空间的 RoleBinding å’Œ ClusterRoleBinding 也会作为 RBAC 的真实æ¥æºã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [RBAC 页é¢](../reference-guides/prometheus-federator/rbac.md)。**项目所有者（admin）ã€é¡¹ç›®æˆå‘˜ï¼ˆedit）和åªè¯»æˆå‘˜ï¼ˆview）应该有æƒè®¿é—®æ­¤å‘½å空间。** + + :::note 注æ„事项： + + - 如果æ供了 `.Values.global.cattle.projectLabel`（默认设置为 `field.cattle.io/projectId`），则 Operator 会自动生æˆé¡¹ç›®æ³¨å†Œå‘½å空间，并将它导入到命å空间绑定的项目中。æ¢è¨€ä¹‹ï¼Œå¦‚果观察到至少一个带有该标签的命å空间，则 Operator 会创建一个项目 Registration 命å空间。除éžå‡ºçŽ°ä»¥ä¸‹ä¸¤ç§æƒ…况，å¦åˆ™ Operator ä¸ä¼šè®©è¿™äº›å‘½å空间被删除。第一ç§æƒ…况是带有该标签的所有命å空间都消失了（例如，这是该项目中的最åŽä¸€ä¸ªå‘½å空间，在这ç§æƒ…况下，命å空间将标有标签 `"helm.cattle.io/helm-project-operator-orphaned": "true"`，表示å¯ä»¥åˆ é™¤ï¼‰ã€‚第二ç§æƒ…况是由于项目 ID 是在 `.Values.helmProjectOperator.otherSystemProjectLabelValues` 下æ供的（用作项目的拒ç»åå•ï¼‰ï¼Œå¯¼è‡´ Operator ä¸å†ç›‘视该项目。这些命å空间ä¸ä¼šè¢«è‡ªåŠ¨åˆ é™¤ï¼Œè¿™æ ·èƒ½é¿å…ç ´å用户数æ®ã€‚如果需è¦ï¼Œå»ºè®®ç”¨æˆ·åœ¨åˆ›å»ºæˆ–删除项目时手动清ç†è¿™äº›å‘½å空间。 + + - 如果未æä¾› `.Values.global.cattle.projectLabel`，则 Operator / System 命å空间也是项目注册命å空间。 + + ::: + +3. **项目å‘布命å空​​间（`cattle-project--monitoring`）**：Operator 代表 ProjectHelmChart 在其中部署项目监控堆栈的命å空间集。Operator 还将根æ®åœ¨é¡¹ç›® Registration 命å空间中找到的绑定，自动为项目监控堆栈在此命åç©ºé—´ä¸­åˆ›å»ºçš„è§’è‰²åˆ†é… RBAC。**åªæœ‰é›†ç¾¤ç®¡ç†å‘˜æ‰èƒ½è®¿é—®è¿™ä¸ªå‘½å空间。部署的 Helm Chart å’Œ Prometheus Federator 将为项目所有者（admin）ã€é¡¹ç›®æˆå‘˜ï¼ˆedit）和åªè¯»æˆå‘˜ï¼ˆview）分é…该命å空间的有é™è®¿é—®æƒé™ã€‚** + + :::note 注æ„事项： + + - 项目å‘布命å空间会自动部署并导入到 ID 在 `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` 下指定的项目中，如果未指定，且项目注册命å空间中指定了 ProjectHelmChart，则默认为 `.Values.global.cattle.systemProjectId` 的值。 + + - 项目å‘布命å空​​间的孤立约定与项目注册命å空间的相åŒï¼ˆå‚è§ä¸Šé¢çš„注释）。 + + - 如果 `.Values.projectReleaseNamespaces.enabled` 为 false，则项目å‘布命å空​​间与项目注册命å空间是相åŒçš„。 + + ::: + +### Helm 资æºï¼ˆHelmChartã€HelmRelease） + +在部署 ProjectHelmChart 时，Prometheus Federator 将自动创建和管ç†ä¸¤ä¸ªå­è‡ªå®šä¹‰èµ„æºï¼Œå®ƒä»¬ä¾æ¬¡ç®¡ç†ä»¥ä¸‹åº•å±‚ Helm 资æºï¼š + +- HelmChart CR（通过 Operator ä¸­çš„åµŒå…¥å¼ [k3s-io​​/helm-contoller](https://github.com/k3s-io/helm-controller) 管ç†ï¼‰ï¼šæ­¤è‡ªå®šä¹‰èµ„æºä¼šæ ¹æ®åº”用到 HelmChart CR çš„å˜æ›´ï¼Œåœ¨è§¦å‘ `helm install`ã€`helm upgrade` 或 `helm uninstall` çš„åŒä¸€å‘½å空间中自动创建一个 Job。此 CR ä¼šæ ¹æ® ProjectHelmChart 的更改（例如，修改 `values.yaml`）或底层项目定义的更改（例如，从项目中添加或删除命å空间）自动更新。 + +:::note é‡è¦æ示： + +如果 ProjectHelmChart 没有部署或更新底层项目监控堆栈，你å¯ä»¥å…ˆä½¿ç”¨æ­¤èµ„æºåœ¨ Operator / System 命å空间中创建的 Job æ¥æ£€æŸ¥ Helm æ“作是å¦æœ‰é—®é¢˜ã€‚通常åªèƒ½ç”±**集群管ç†å‘˜è®¿é—®**。 + +::: + +- HelmRelease CR（通过 Operator ä¸­çš„åµŒå…¥å¼ [rancher/helm-locker](https://github.com/rancher/helm-locker) 管ç†ï¼‰ï¼šæ­¤è‡ªå®šä¹‰èµ„æºä¼šè‡ªåŠ¨é”定已部署的 Helm 版本并自动覆盖对底层资æºçš„更新，除éžæ›´æ”¹æ˜¯ Helm æ“作导致的（`helm install`ã€`helm upgrade` 或 `helm uninstall` ç”± HelmChart CR 执行）。 + +:::note + +HelmRelease CR 会å‘出 Kubernetes 事件，用于检测底层 Helm 版本修改并将其é”定回原ä½ã€‚è¦æŸ¥çœ‹è¿™äº›äº‹ä»¶ï¼Œä½ å¯ä»¥ä½¿ç”¨ `kubectl describe helmrelease -n `。你还å¯ä»¥æŸ¥çœ‹æ­¤ Operator 的日志，了解检测到更改的时间以åŠå“ªäº›èµ„æºè¢«å°è¯•æ›´æ”¹ã€‚ + +::: + +这两ç§èµ„æºéƒ½æ˜¯ä¸º Operator / System 命å空间中的所有 Helm Chart 创建的，用于é¿å…低æƒé™ç”¨æˆ·çš„æƒé™å‡çº§ã€‚ + +### 高级 Helm Project Operator é…ç½® + +有关高级é…置的更多信æ¯ï¼Œè¯·å‚阅[此页é¢](https://github.com/rancher/prometheus-federator/blob/main/charts/prometheus-federator/0.0.1/README.md#advanced-helm-project-operator-configuration)。 + + + +### Local 集群上的 Prometheus Federator + +Prometheus Federator 是一个资æºå¯†é›†åž‹åº”用程åºã€‚ä½ å¯ä»¥å°†å…¶å®‰è£…到 Local 集群（**ä¸æŽ¨è**）。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/provisioning-storage-examples.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/provisioning-storage-examples.md new file mode 100644 index 00000000000..e13500f2aaf --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/provisioning-storage-examples.md @@ -0,0 +1,11 @@ +--- +title: é…置存储示例 +--- + +Rancher 通过å„ç§å·æ’件æ¥æ”¯æŒæŒä¹…存储。但是，在使用这些æ’件将æŒä¹…存储绑定到工作负载之å‰ï¼Œæ— è®ºæ˜¯ä½¿ç”¨äº‘解决方案还是你自己管ç†çš„本地解决方案，你都必须先é…置存储本身。 + +为了你的方便，Rancher æ供了é…置主æµå­˜å‚¨çš„å‚考文档： + +- [NFS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) +- [vSphere](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) +- [EBS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/quick-start-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/quick-start-guides.md new file mode 100644 index 00000000000..fd0013e0777 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/quick-start-guides.md @@ -0,0 +1,17 @@ +--- +title: Rancher éƒ¨ç½²å¿«é€Ÿå…¥é—¨æŒ‡å— +--- + +:::caution + +本章节中æ供的指å—，旨在帮助你快速å¯åŠ¨ä¸€ä¸ªç”¨äºŽ Rancher 的沙盒，以评估 Rancher 是å¦èƒ½æ»¡è¶³ä½ çš„使用需求。快速入门指å—ä¸é€‚用于生产环境。如果你需è¦èŽ·å–生产环境的æ“作指导，请å‚è§[安装](installation-and-upgrade.md)。 + +::: + +ä½ å¯ä»¥é˜…读本章节，以快速开始部署和测试 Rancher 2.xã€‚æœ¬ç« èŠ‚åŒ…å« Rancher 的简å•è®¾ç½®å’Œä¸€äº›å¸¸è§ç”¨ä¾‹çš„说明。未æ¥ï¼Œæˆ‘们会在本章节中添加更多内容。 + +我们æ供以下快速入门指å—： + +- [部署 Rancher Server](deploy-rancher-manager.md)：使用最方便的方å¼è¿è¡Œ Rancher。 + +- [部署工作负载](deploy-rancher-workloads.md)：部署一个简å•çš„[工作负载](https://kubernetes.io/docs/concepts/workloads/)并公暴露工作负载，以从集群外部访问工作负载。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-behind-an-http-proxy.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-behind-an-http-proxy.md new file mode 100644 index 00000000000..e468db53dd5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-behind-an-http-proxy.md @@ -0,0 +1,13 @@ +--- +title: 使用 HTTP 代ç†å®‰è£… Rancher +--- + +很多ä¼ä¸šæœ¬åœ°è¿è¡Œçš„æœåŠ¡å™¨æˆ–虚拟机ä¸èƒ½ç›´æŽ¥è®¿é—®äº’è”网，但是出于安全考虑，他们必须通过 HTTP(S) 代ç†è¿žæŽ¥åˆ°å¤–部æœåŠ¡ã€‚本教程将分步介ç»å¦‚何在这样的环境中进行高å¯ç”¨çš„ Rancher 安装。 + +å¦å¤–，用户也å¯ä»¥åœ¨æ²¡æœ‰ä»»ä½•äº’è”网访问的情况下离线设置 Rancher。详情请å‚è§ [Rancher 官方文档](air-gapped-helm-cli-install.md)。 + +## å®‰è£…æ¦‚è¦ + +1. [é…置基础设施](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md) +2. [é…ç½® Kubernetes 集群](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md) +3. [安装 Rancher](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-hardening-guides.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-hardening-guides.md new file mode 100644 index 00000000000..48386cdd871 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-hardening-guides.md @@ -0,0 +1,52 @@ +--- +title: Self-Assessment and Hardening Guides for Rancher +--- + +Rancher provides specific security hardening guides for each supported Rancher version's Kubernetes distributions. + +## Rancher Kubernetes Distributions + +Rancher uses the following Kubernetes distributions: + +- [**RKE**](https://rancher.com/docs/rke/latest/en/), Rancher Kubernetes Engine, is a CNCF-certified Kubernetes distribution that runs entirely within Docker containers. +- [**RKE2**](https://docs.rke2.io/) is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. +- [**K3s**](https://docs.k3s.io/) is a fully conformant, lightweight Kubernetes distribution. It is easy to install, with half the memory requirement of upstream Kubernetes, all in a binary of less than 100 MB. + +To harden a Kubernetes cluster that's running a distribution other than those listed, refer to your Kubernetes provider docs. + +## Hardening Guides and Benchmark Versions + +Each self-assessment guide is accompanied by a hardening guide. These guides were tested alongside the listed Rancher releases. Each self-assessment guides was tested on a specific Kubernetes version and CIS benchmark version. If a CIS benchmark has not been validated for your Kubernetes version, you can use the existing guides until a guide for your version is added. + +### RKE Guides + +| Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|--------------------|-----------------------|-----------------------|------------------| +| Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md) | [Link](rke1-hardening-guide.md) | + +### RKE2 Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned RKE2 | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md) | [Link](rke2-hardening-guide.md) | +| Standalone RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | + +### K3s Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned K3s cluster | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md) | [Link](k3s-hardening-guide.md) | +| Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment) | [Link](https://docs.k3s.io/security/hardening-guide) | + +## Rancher with SELinux + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a kernel module that adds extra access controls and security tools to Linux. Historically used by government agencies, SELinux is now industry-standard. SELinux is enabled by default on RHEL and CentOS. + +To use Rancher with SELinux, we recommend [installing](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) the `rancher-selinux` RPM. + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-managed-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-managed-clusters.md new file mode 100644 index 00000000000..15d2194487a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-managed-clusters.md @@ -0,0 +1,19 @@ +--- +title: Rancher 管ç†é›†ç¾¤çš„最佳实践 +--- + +### Logging + +有关集群级别日志和应用日志的建议，请å‚è§ [Logging 最佳实践](../reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md)。 + +### Monitoring + +é…ç½®åˆç†çš„监控和告警规则对于安全ã€å¯é åœ°è¿è¡Œç”Ÿäº§çŽ¯å¢ƒä¸­çš„工作负载至关é‡è¦ã€‚有关更多建议，请å‚阅[最佳实践](../reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md)。 + +### 设置容器的技巧 + +é…置良好的容器å¯ä»¥æžå¤§åœ°æ高环境的整体性能和安全性。有关容器设置的建议，请å‚è§[设置容器的技巧](../reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md)。 + +### Rancher ç®¡ç† vSphere 集群的最佳实践 + +[Rancher ç®¡ç† vSphere 集群的最佳实践](../reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md)概述了在 vSphere 环境中é…置下游 Rancher 集群的å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-manager-architecture.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-manager-architecture.md new file mode 100644 index 00000000000..ad6a3ea462f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-manager-architecture.md @@ -0,0 +1,17 @@ +--- +title: 架构 +--- + +本章节é‡ç‚¹ä»‹ç» [Rancher Server åŠå…¶ç»„件](../reference-guides/rancher-manager-architecture/rancher-server-and-components.md) ä»¥åŠ [Rancher 如何与下游 Kubernetes 集群通信](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md)。 + +有关安装 Rancher çš„ä¸åŒæ–¹å¼çš„ä¿¡æ¯ï¼Œè¯·å‚è§[安装选项概述](installation-and-upgrade.md#安装方å¼æ¦‚è¿°)。 + +有关 Rancher API Server 的主è¦åŠŸèƒ½ï¼Œè¯·å‚è§[概述](../getting-started/overview.md#rancher-api-server-的功能)。 + +有关如何为 Rancher Server 设置底层基础架构，请å‚è§[架构推è](../reference-guides/rancher-manager-architecture/architecture-recommendations.md)。 + +:::note + +本节默认你已对 Docker å’Œ Kubernetes 有一定的了解。如果你需è¦äº†è§£ Kubernetes 组件如何å作，请å‚è§ [Kubernetes 概念](../reference-guides/kubernetes-concepts.md)。 + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-on-a-single-node-with-docker.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-on-a-single-node-with-docker.md new file mode 100644 index 00000000000..b8875d47d51 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-on-a-single-node-with-docker.md @@ -0,0 +1,207 @@ +--- +title: 使用 Docker å°† Rancher 安装到å•ä¸ªèŠ‚点中 +description: 在开å‘和测试环境中，你å¯ä»¥ä½¿ç”¨ Docker 安装。在å•ä¸ª Linux 主机上安装 Docker，然åŽä½¿ç”¨ä¸€ä¸ª Docker 容器部署 Rancher。 +--- + +Rancher å¯ä»¥é€šè¿‡è¿è¡Œå•ä¸ª Docker 容器进行安装。 + +在这ç§å®‰è£…方案中，你需è¦å°† Docker 安装到å•ä¸ª Linux 主机，然åŽä½¿ç”¨å•ä¸ª Docker 容器将 Rancher 部署到主机中。 + +:::note 想è¦ä½¿ç”¨å¤–部负载å‡è¡¡å™¨ï¼Ÿ + +请å‚阅[使用外部负载å‡è¡¡å™¨çš„ Docker 安装](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md)。 + +::: + +Rancher çš„ Docker 安装仅推è用于开å‘和测试环境中。Rancher 版本决定了能å¦å°† Rancher è¿ç§»åˆ°é«˜å¯ç”¨é›†ç¾¤ã€‚ + +Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。详情请å‚è§[把 Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +## Rancher 特æƒè®¿é—® + +当 Rancher Server 部署在 Docker 容器中时，容器内会安装一个本地 Kubernetes 集群供 Rancher 使用。为 Rancher 的很多功能都是以 deployment çš„æ–¹å¼è¿è¡Œçš„，而在容器内è¿è¡Œå®¹å™¨æ˜¯éœ€è¦ç‰¹æƒæ¨¡å¼çš„，因此你需è¦åœ¨å®‰è£… Rancher 时添加 `--privileged` 选项。 + +## æ“作系统，Docker，硬件和网络è¦æ±‚ + +请确ä¿ä½ çš„节点满足常规的[安装è¦æ±‚](installation-requirements.md)。 + +## 1. é…ç½® Linux 主机 + +按照[è¦æ±‚](installation-requirements.md)é…置一个 Linux 主机，用于è¿è¡Œ Rancher Server。 + +## 2. 选择一个 SSL 选项并安装 Rancher + +出于安全考虑，使用 Rancher 时请使用 SSL（Secure Sockets Layer）。SSL ä¿æŠ¤æ‰€æœ‰ Rancher 网络通信（如登录和与集群交互）的安全。 + +:::tip 你是å¦éœ€è¦ï¼š + +- 使用代ç†ã€‚å‚è§ [HTTP 代ç†é…ç½®](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md)。 +- é…置自定义 CA æ ¹è¯ä¹¦ä»¥è®¿é—®æœåŠ¡ã€‚å‚è§[自定义 CA æ ¹è¯ä¹¦](../reference-guides/single-node-rancher-in-docker/advanced-options.md#自定义-ca-è¯ä¹¦)。 +- 完æˆç¦»çº¿å®‰è£…。å‚è§ [离线：Docker 安装](air-gapped-helm-cli-install.md)。 +- 记录所有 Rancher API 的事务。å‚加 [API 审计](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-审计日志)。 + +::: + +选择以下的选项之一： + +- [选项 A：使用 Rancher 生æˆçš„默认自签åè¯ä¹¦](#选项-a使用-rancher-生æˆçš„默认自签åè¯ä¹¦) +- [选项 B：使用你自己的è¯ä¹¦ - 自签å](#选项-b使用你自己的è¯ä¹¦---自签å) +- [选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦](#选项-c使用你自己的è¯ä¹¦---å¯ä¿¡-ca-ç­¾åçš„è¯ä¹¦) +- [选项 D：Let's Encrypt è¯ä¹¦](#选项-dlets-encrypt-è¯ä¹¦) +- [选项 E：Localhost 隧é“，ä¸ä½¿ç”¨è¯ä¹¦](#选项-elocalhost-隧é“ä¸ä½¿ç”¨è¯ä¹¦) + +### 选项 A：使用 Rancher 生æˆçš„默认自签åè¯ä¹¦ + +如果你在ä¸è€ƒè™‘身份验è¯çš„å¼€å‘或测试环境中安装 Rancher，å¯ä»¥ä½¿ç”¨ Rancher 生æˆçš„自签åè¯ä¹¦å®‰è£… Rancher。这ç§å®‰è£…æ–¹å¼é¿å…了自己生æˆè¯ä¹¦çš„麻烦。 + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest +``` + +### 选项 B：使用你自己的è¯ä¹¦ - 自签å +在你团队访问 Rancher Server çš„å¼€å‘或测试环境中，创建一个用于你的安装的自签åè¯ä¹¦ï¼Œä»¥ä¾¿å›¢é˜ŸéªŒè¯ä»–们对实例的连接。 + +:::note 先决æ¡ä»¶ï¼š + +使用 [OpenSSL](https://www.openssl.org/) 或其他方法创建自签åè¯ä¹¦ã€‚ + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 +- 在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括链中的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +::: + +创建è¯ä¹¦åŽï¼Œè¿è¡Œä»¥ä¸‹ Docker 命令以安装 Rancher。使用 `-v` 标志并æä¾›è¯ä¹¦çš„路径，以将è¯ä¹¦æŒ‚载到容器中。 + +| å ä½ç¬¦ | æè¿° | +| ------------------- | --------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | +| `` | CA è¯ä¹¦çš„路径。 | + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher:latest +``` + +### 选项 C：使用你自己的è¯ä¹¦ - å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ + +在公开暴露应用的生产环境中，请使用由å¯ä¿¡ CA ç­¾åçš„è¯ä¹¦ï¼Œä»¥é¿å…用户收到è¯ä¹¦å®‰å…¨è­¦å‘Šã€‚ + +ä¸å»ºè®®å°† Docker 安装用于生产环境。这些说明仅适用于测试和开å‘。 + +:::note 先决æ¡ä»¶ï¼š + +- è¯ä¹¦æ–‡ä»¶çš„æ ¼å¼å¿…须是 PEM。 +- 在你的è¯ä¹¦æ–‡ä»¶ä¸­ï¼ŒåŒ…括å¯ä¿¡ CA æ供的所有中间è¯ä¹¦ã€‚你需è¦å¯¹ä½ çš„è¯ä¹¦è¿›è¡ŒæŽ’åºï¼ŒæŠŠä½ çš„è¯ä¹¦æ”¾åœ¨æœ€å‰é¢ï¼ŒåŽé¢è·Ÿç€ä¸­é—´è¯ä¹¦ã€‚如需查看示例，请å‚è§[è¯ä¹¦æ•…障排除](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +::: + +获å–è¯ä¹¦åŽï¼Œè¿è¡Œä»¥ä¸‹ Docker 命令。 + +- 使用 `-v` 标志并æä¾›è¯ä¹¦çš„路径，以将è¯ä¹¦æŒ‚载到容器中。因为你的è¯ä¹¦æ˜¯ç”±å¯ä¿¡çš„ CA ç­¾å的，因此你ä¸éœ€è¦å®‰è£…é¢å¤–çš„ CA è¯ä¹¦æ–‡ä»¶ã€‚ +- 使用 `--no-cacerts` 作为容器的å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +| å ä½ç¬¦ | æè¿° | +| ------------------- | ----------------------------- | +| `` | 包å«è¯ä¹¦æ–‡ä»¶çš„目录的路径。 | +| `` | 完整è¯ä¹¦é“¾çš„路径。 | +| `` | è¯ä¹¦ç§é’¥çš„路径。 | + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +### 选项 D：Let's Encrypt è¯ä¹¦ + +:::caution + +Let's Encrypt 对新è¯ä¹¦è¯·æ±‚有频率é™åˆ¶ã€‚因此，请é™åˆ¶åˆ›å»ºæˆ–销æ¯å®¹å™¨çš„频率。详情请å‚è§ [Let's Encrypt 官方文档 - 频率é™åˆ¶](https://letsencrypt.org/docs/rate-limits/)。 + +::: + +你也å¯ä»¥åœ¨ç”Ÿäº§çŽ¯å¢ƒä¸­ä½¿ç”¨ [Let's Encrypt](https://letsencrypt.org/) è¯ä¹¦ã€‚Let's Encrypt 使用 HTTP-01 质询æ¥éªŒè¯ä½ å¯¹åŸŸå的控制æƒã€‚如果è¦ç¡®è®¤ä½ å¯¹è¯¥åŸŸå有控制æƒï¼Œä½ å¯å°†ç”¨äºŽè®¿é—® Rancher 的主机å（例如 `rancher.mydomain.com`）指å‘è¿è¡Œçš„主机的 IP。你å¯é€šè¿‡åœ¨ DNS 中创建 A 记录，以将主机å绑定到 IP 地å€ã€‚ + +ä¸å»ºè®®å°† Docker 安装用于生产环境。这些说明仅适用于测试和开å‘。 + +:::note 先决æ¡ä»¶ï¼š + +- Let's Encrypt 是è”网æœåŠ¡ã€‚因此，在内网和离线环境中ä¸èƒ½ä½¿ç”¨ã€‚ +- 在 DNS 中创建一æ¡è®°å½•ï¼Œå°† Linux 主机 IP 地å€ç»‘定到è¦ç”¨äºŽè®¿é—® Rancher 的主机å（例如，`rancher.mydomain.com`）。 +- 在 Linux 主机上打开 `TCP/80` 端å£ã€‚Let's Encrypt çš„ HTTP-01 质询å¯ä»¥æ¥è‡ªä»»ä½•æº IP 地å€ï¼Œå› æ­¤ç«¯å£ `TCP/80` 必须开放开所有 IP 地å€ã€‚ + +::: + +满足先决æ¡ä»¶åŽï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤ä½¿ç”¨ Let's Encrypt è¯ä¹¦å®‰è£… Rancher。 + +| å ä½ç¬¦ | æè¿° | +| ----------------- | ------------------- | +| `` | 你的域ååœ°å€ | + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --acme-domain +``` + +### 选项 E：Localhost 隧é“，ä¸ä½¿ç”¨è¯ä¹¦ + +如果你在开å‘或测试环境中安装 Rancher，且环境中有è¿è¡Œçš„ localhost 隧é“解决方案（如 [ngrok](https://ngrok.com/)），ä¸è¦ç”Ÿæˆè¯ä¹¦ã€‚此安装选项ä¸éœ€è¦è¯ä¹¦ã€‚ + +- 使用 `--no-cacerts` 作为å‚数，以ç¦ç”¨ Rancher 生æˆçš„默认 CA è¯ä¹¦ã€‚ + +登录到你的主机，然åŽè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +## 高级选项 + +使用 Docker å°† Rancher 安装到å•ä¸ªèŠ‚点时，有如下几个å¯å¼€å¯çš„高级选项： + +- 自定义 CA è¯ä¹¦ +- API 审计日志 +- TLS 设置 +- 离线环境 +- æŒä¹…åŒ–æ•°æ® +- 在åŒä¸€ä¸ªèŠ‚点中è¿è¡Œ `rancher/rancher` å’Œ `rancher/rancher-agent` + +详情请å‚è§[本页](../reference-guides/single-node-rancher-in-docker/advanced-options.md)。 + +## 故障排除 + +如需了解常è§é—®é¢˜åŠæ•…障排除æ示，请å‚è§[本页](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md)。 + +## åŽç»­æ“作 + +- **推è**：检查å•èŠ‚点[备份](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md)å’Œ[æ¢å¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md)。你å¯èƒ½æš‚时没有需è¦å¤‡ä»½çš„æ•°æ®ï¼Œä½†æ˜¯æˆ‘们建议你在常规使用 Rancher åŽåˆ›å»ºå¤‡ä»½ã€‚ +- 创建 Kubernetes 集群：[é…ç½® Kubernetes 集群](kubernetes-clusters-in-rancher-setup.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-security.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-security.md new file mode 100644 index 00000000000..15677f4a9a7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-security.md @@ -0,0 +1,85 @@ +--- +title: 安全 +--- + + + + + + + +
+

安全策略

+

Rancher Labs 会负责任地披露问题，并致力于在åˆç†çš„时间内解决所有问题。

+		 +

报告过程

+

请将安全问题å‘é€è‡³ security-rancher@suse.com。

+		 +

公告

+

订阅 Rancher 公告论å›ä»¥èŽ·å–版本更新。

+
+ +安全是 Rancher 全部功能的基础。Rancher 集æˆäº†å…¨éƒ¨ä¸»æµèº«ä»½éªŒè¯å·¥å…·å’ŒæœåŠ¡ï¼Œå¹¶æ供了ä¼ä¸šçº§çš„ [RBAC 功能](manage-role-based-access-control-rbac.md)，让你的 Kubernetes 集群更加安全。 + +本文介ç»äº†å®‰å…¨ç›¸å…³çš„文档以åŠèµ„æºï¼Œè®©ä½ çš„ Rancher 安装和下游 Kubernetes 集群更加安全。 + +### NeuVector 与 Rancher çš„é›†æˆ + +NeuVector 是一个开æºçš„ã€ä»¥å®¹å™¨ä¸ºä¸­å¿ƒçš„安全应用程åºï¼ŒçŽ°å·²é›†æˆåˆ° Rancher 中。NeuVector æ供生产安全ã€DevOps æ¼æ´žä¿æŠ¤å’Œå®¹å™¨é˜²ç«å¢™ç­‰åŠŸèƒ½ã€‚请å‚阅 [Rancher 文档](../integrations-in-rancher/neuvector.md)å’Œ [NeuVector 文档](https://open-docs.neuvector.com/)了解更多信æ¯ã€‚ + +### 在 Kubernetes 集群上è¿è¡Œ CIS 安全扫æ + +Rancher 使用 [kube-bench](https://github.com/aquasecurity/kube-bench) æ¥è¿è¡Œå®‰å…¨æ‰«æ，从而检查 Kubernetes 是å¦æŒ‰ç…§ [CIS](https://www.cisecurity.org/cis-benchmarks/)（Center for Internet Security，互è”网安全中心）Kubernetes Benchmark 中定义的安全最佳实践进行部署。 + +CIS Kubernetes Benchmark 是一个å‚考文档，用于为 Kubernetes 建立安全é…置基线。 + +CIS 是一个 501(c\)(3) éžè¥åˆ©ç»„织，æˆç«‹äºŽ 2000 å¹´ 10 月，其使命是识别ã€å¼€å‘ã€éªŒè¯ã€ä¿ƒè¿›å’Œç»´æŒç½‘络防御的最佳实践方案，并建立和指导社区，以在网络空间中è¥é€ ä¿¡ä»»çš„环境。 + +CIS Benchmark 是目标系统安全é…置的最佳实践。CIS Benchmark 是由安全专家ã€æŠ€æœ¯ä¾›åº”商ã€å…¬å¼€å’Œç§äººç¤¾åŒºæˆå‘˜ï¼Œä»¥åŠ CIS Benchmark å¼€å‘团队共åŒå¿—æ„¿å¼€å‘的。 + +Benchmark æ供两ç§ç±»åž‹çš„建议，分别是自动（Automated）和手动（Manual）。我们åªè¿è¡Œ Automated 相关的测试。 + +Rancher 在集群上è¿è¡Œ CIS 安全扫æ时会生æˆä¸€ä»½æŠ¥å‘Šï¼Œè¯¥æŠ¥å‘Šä¼šæ˜¾ç¤ºæ¯ä¸ªæµ‹è¯•çš„结果，包括测试概è¦ä»¥åŠ `passed`ã€`skipped` å’Œ `failed` 的测试数é‡ã€‚报告还包括失败测试的修正步骤。 + +有关详细信æ¯ï¼Œè¯·å‚阅[安全扫æ](cis-scan-guides.md)。 + +### SELinux RPM + +[安全增强型 Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) 是对 Linux 的安全增强。被政府机构使用之åŽï¼ŒSELinux å·²æˆä¸ºè¡Œä¸šæ ‡å‡†ï¼Œå¹¶åœ¨ CentOS 7 å’Œ 8 上默认å¯ç”¨ã€‚ + +我们æ供了 `rancher-selinux` å’Œ `rke2-selinux` 两个 RPM（Red Hat 软件包），让 Rancher 产å“能够在 SELinux 主机上正常è¿è¡Œã€‚有关详细信æ¯ï¼Œè¯·å‚阅[此页é¢](selinux-rpm.md)。 + +### Rancher å¼ºåŒ–æŒ‡å— + +Rancher 强化指å—基于 CIS Kubernetes Benchmark。 + +强化指å—为强化 Rancher 的生产安装æ供了说明性指导。有关安全管控的完整列表，请å‚阅 Rancher çš„ [CIS Kubernetes Benchmark自我评估指å—](#cis-benchmark-和自我评估)。 + +> 强化指å—æ述了如何ä¿æŠ¤é›†ç¾¤ä¸­çš„节点，建议在安装 Kubernetes 之å‰å‚考强化指å—中的步骤。 + +æ¯ä¸ªå¼ºåŒ–指å—版本都针对特定的 CIS Kubernetes Benchmarkã€Kubernetes å’Œ Rancher 版本。 + +### CIS Benchmark 和自我评估 + +Benchmark 自我评估是 Rancher 安全强化指å—的辅助。强化指å—展示了如何强化集群，而 Benchmark 指å—旨在帮助你评估强化集群的安全级别。 + +由于 Rancher å’Œ RKE å°† Kubernetes æœåŠ¡å®‰è£…为 Docker 容器，因此 CIS Kubernetes Benchmark 中的许多管控验è¯æ£€æŸ¥éƒ½ä¸é€‚用。本指å—将介ç»å„ç§ç®¡æŽ§ï¼Œå¹¶æ供更新的示例命令æ¥å®¡æ ¸ Rancher 创建的集群的åˆè§„性。你å¯ä»¥å‰å¾€ [CIS 网站](https://www.cisecurity.org/benchmark/kubernetes/)下载原始的 Benchmark 文档。 + +Rancher æ¯ä¸ªç‰ˆæœ¬çš„自我评估指å—都对应特定的强化指å—ã€Rancherã€Kubernetes å’Œ CIS Benchmark 版本。 + +### 第三方渗é€æµ‹è¯•æŠ¥å‘Š + +Rancher 会定期è˜è¯·ç¬¬ä¸‰æ–¹å¯¹ Rancher 2.x 软件栈进行安全审核和渗é€æµ‹è¯•ã€‚测试环境会éµå¾ª Rancher æ供的强化指å—。旧的渗é€æµ‹è¯•æŠ¥å‘Šå¦‚下。 + +结果： + +- [Cure53 渗é€æµ‹è¯• - 2019 å¹´ 7 月](https://releases.rancher.com/documents/security/pen-tests/2019/RAN-01-cure53-report.final.pdf) +- [Untamed Theory 渗é€æµ‹è¯• - 2019 å¹´ 3 月](https://releases.rancher.com/documents/security/pen-tests/2019/UntamedTheory-Rancher_SecurityAssessment-20190712_v5.pdf) + +### Rancher 安全公告和 CVE + +Rancher 致力于å‘社区披露我们产å“的安全问题。有关我们已解决问题的 CVE（Common Vulnerabilities and Exposures，通用æ¼æ´žæŠ«éœ²ï¼‰åˆ—表，请å‚阅[此页é¢](../reference-guides/rancher-security/security-advisories-and-cves.md)。 + +### Kubernetes 安全最佳实践 + +有关ä¿æŠ¤ Kubernetes 集群的建议，请å‚阅 [Kubernetes 安全最佳实践指å—](../reference-guides/rancher-security/kubernetes-security-best-practices.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-server-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-server-configuration.md new file mode 100644 index 00000000000..a0b6e3850b1 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-server-configuration.md @@ -0,0 +1,12 @@ +--- +title: Rancher Server é…ç½® +--- + +- [RKE1 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [AKS 集群é…ç½®](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) +- [GKE 集群é…ç½®](../pages-for-subheaders/gke-cluster-configuration.md) +- [使用现有节点](../pages-for-subheaders/use-existing-nodes.md) +- [åŒæ­¥é›†ç¾¤](../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-server.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-server.md new file mode 100644 index 00000000000..0d632b6def6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rancher-server.md @@ -0,0 +1,17 @@ +--- +title: Rancher Server 的最佳实践 +--- + +本指å—介ç»äº†è®© Rancher 管ç†ä¸‹æ¸¸ Kubernetes 集群的 Rancher Server è¿è¡Œå»ºè®®ã€‚ + +### 推è的架构和基础设施 + +有关在高å¯ç”¨ Kubernetes 集群上设置 Rancher Server 的通用建议，请å‚è§[本指å—](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md)。 + +### 部署策略 + +[本指å—](../reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md)旨在帮助你选择部署策略（区域部署/中心è¾å°„型部署），æ¥è®© Rancher Server 更好地管ç†ä¸‹æ¸¸ Kubernetes 集群。 + +### 在 vSphere 环境中安装 Rancher + +[本指å—](../reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md)介ç»äº†åœ¨ vSphere 环境中安装 Rancher çš„å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/resources.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/resources.md new file mode 100644 index 00000000000..45aa544fa7e --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/resources.md @@ -0,0 +1,25 @@ +--- +title: èµ„æº +--- + +### Docker 安装 + +[å•èŠ‚点 Docker 安装](rancher-on-a-single-node-with-docker.md)适用于想è¦æµ‹è¯• Rancher 的用户。你无需使用 Helm 在 Kubernetes 集群上è¿è¡Œ Rancher，你å¯ä»¥ä½¿ç”¨ `docker run` 命令，把 Rancher Server 组件安装到å•ä¸ªèŠ‚点上。 + +由于åªæœ‰ä¸€ä¸ªèŠ‚点和一个 Docker 容器，因此，如果该节点å‘生故障，由于其他节点上没有å¯ç”¨çš„ etcd æ•°æ®å‰¯æœ¬ï¼Œä½ å°†ä¸¢å¤± Rancher Server 的所有数æ®ã€‚ + +### 离线安装 + +按照[以下步骤](air-gapped-helm-cli-install.md)在离线环境中安装 Rancher Server。 + +离线环境å¯ä»¥æ˜¯ Rancher Server 离线安装ã€é˜²ç«å¢™åŽé¢æˆ–代ç†åŽé¢ã€‚ + +### 高级选项 + +安装 Rancher 时，有如下几个å¯å¼€å¯çš„高级选项：æ¯ä¸ªå®‰è£…指å—中都æ供了对应的选项。了解选项详情： + +- [自定义 CA è¯ä¹¦](../getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md) +- [API 审计日志](../how-to-guides/advanced-user-guides/enable-api-audit-log.md) +- [TLS 设置](../getting-started/installation-and-upgrade/installation-references/tls-settings.md) +- [etcd é…ç½®](../how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md) +- [离线安装 Local System Chart](../getting-started/installation-and-upgrade/resources/local-system-charts.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rke1-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rke1-hardening-guide.md new file mode 100644 index 00000000000..6a187df3273 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rke1-hardening-guide.md @@ -0,0 +1,504 @@ +--- +title: RKE Hardening Guide +--- + +This document provides prescriptive guidance for how to harden an RKE cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 up to v1.25 | + +:::note +At the time of writing, the upstream CIS Kubernetes v1.25 benchmark is not yet available in Rancher. At this time Rancher is using the CIS v1.23 benchmark when scanning Kubernetes v1.25 clusters. Due to that, the CIS checks 5.2.3, 5.2.4, 5.2.5 and 5.2.6 might fail. +::: + +For more details on how to evaluate a hardened RKE cluster against the official CIS benchmark, refer to the RKE self-assessment guides for specific Kubernetes and CIS benchmark versions. + +## Host-level requirements + +### Configure Kernel Runtime Parameters + +The following `sysctl` configuration is recommended for all nodes types in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.overcommit_memory=1 +vm.panic_on_oom=0 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Configure `etcd` user and group + +A user account and group for the **etcd** service is required to be set up before installing RKE. + +#### Create `etcd` user and group + +To create the **etcd** user and group run the following console commands. +The commands below use `52034` for **uid** and **gid** for example purposes. +Any valid unused **uid** or **gid** could also be used in lieu of `52034`. + +```bash +groupadd --gid 52034 etcd +useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin +``` + +When deploying RKE through its cluster configuration `config.yml` file, update the `uid` and `gid` of the `etcd` user: + +```yaml +services: + etcd: + gid: 52034 + uid: 52034 +``` + +## Kubernetes runtime requirements + +### Configure `default` Service Account + +#### Set `automountServiceAccountToken` to `false` for `default` service accounts + +Kubernetes provides a default service account which is used by cluster workloads where no specific service account is assigned to the pod. +Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. +The default service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. +Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### Configure Network Policy + +#### Ensure that all Namespaces have Network Policies defined + +Running different applications on the same Kubernetes cluster creates a risk of one compromised application attacking a neighboring application. Network segmentation is important to ensure that containers can communicate only with those they are supposed to. A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. + +Network Policies are namespace scoped. When a network policy is introduced to a given namespace, all traffic not allowed by the policy is denied. However, if there are no network policies in a namespace all traffic will be allowed into and out of the pods in that namespace. To enforce network policies, a container network interface (CNI) plugin must be enabled. This guide uses [Canal](https://github.com/projectcalico/canal) to provide the policy enforcement. Additional information about CNI providers can be found [here](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/). + +Once a CNI provider is enabled on a cluster a default network policy can be applied. For reference purposes a **permissive** example is provided below. If you want to allow all traffic to all pods in a namespace (even if policies are added that cause some pods to be treated as “isolatedâ€), you can create a policy that explicitly allows all traffic in that namespace. Save the following configuration as `default-allow-all.yaml`. Additional [documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) about network policies can be found on the Kubernetes site. + +:::caution +This network policy is just an example and is not recommended for production use. +::: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +``` + +Create a bash script file called `apply_networkPolicy_to_all_ns.sh`. Be sure to `chmod +x apply_networkPolicy_to_all_ns.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl apply -f default-allow-all.yaml -n ${namespace} +done +``` + +Execute this script to apply the `default-allow-all.yaml` configuration with the **permissive** `NetworkPolicy` to all namespaces. + +## Known Limitations + +- Rancher **exec shell** and **view logs** for pods are **not** functional in a hardened setup when only a public IP is provided when registering custom nodes. This functionality requires a private IP to be provided when registering the custom nodes. +- When setting `default_pod_security_policy_template_id:` to `restricted` or `restricted-noroot`, based on the pod security policies (PSP) [provided](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) by Rancher, Rancher creates `RoleBindings` and `ClusterRoleBindings` on the `default` service accounts. The CIS check 5.1.5 requires that the `default` service accounts have no roles or cluster roles bound to it apart from the defaults. In addition, the `default` service accounts should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +## Reference Hardened RKE `cluster.yml` Configuration + +The reference `cluster.yml` is used by the RKE CLI that provides the configuration needed to achieve a hardened installation of RKE. RKE [documentation](https://rancher.com/docs/rke/latest/en/installation/) provides additional details about the configuration items. This reference `cluster.yml` does not include the required `nodes` directive which will vary depending on your environment. Documentation for node configuration in RKE can be found [here](https://rancher.com/docs/rke/latest/en/config-options/nodes/). + +The example `cluster.yml` configuration file contains an Admission Configuration policy in the `services.kube-api.admission_configuration` field. This [sample](../reference-guides/rancher-security/psa-restricted-exemptions.md) policy contains the namespace exemptions necessary for an imported RKE cluster to run properly in Rancher, similar to Rancher's pre-defined [`rancher-restricted`](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) policy. + +If you prefer to use RKE's default `restricted` policy, then leave the `services.kube-api.admission_configuration` field empty and set `services.pod_security_configuration` to `restricted`. See [the RKE docs](https://rke.docs.rancher.com/config-options/services/pod-security-admission) for more information. + + + + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + # Leave `pod_security_configuration` out if you are setting a + # custom policy in `admission_configuration`. Otherwise set + # it to `restricted` to use RKE's pre-defined restricted policy, + # and remove everything inside `admission_configuration` field. + # + # pod_security_configuration: restricted + # + admission_configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [ calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator ] + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: "true" + generate_serving_certificate: true +addons: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + generate_serving_certificate: true +addons: | + # Upstream Kubernetes restricted PSP policy + # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: restricted-noroot + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: psp:restricted-noroot + rules: + - apiGroups: + - extensions + resourceNames: + - restricted-noroot + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: psp:restricted-noroot + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-noroot + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +## Reference Hardened RKE Cluster Template Configuration + +The reference RKE cluster template provides the minimum required configuration to achieve a hardened installation of Kubernetes. RKE templates are used to provision Kubernetes and define Rancher settings. Follow the Rancher [documentation](installation-and-upgrade.md) for additional information about installing RKE and its template details. + + + + +```yaml +# +# Cluster Config +# +default_pod_security_admission_configuration_template_name: rancher-restricted +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: false + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +```yaml +# +# Cluster Config +# +default_pod_security_policy_template_id: restricted-noroot +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +## Conclusion + +If you have followed this guide, your RKE custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rke2-hardening-guide.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rke2-hardening-guide.md new file mode 100644 index 00000000000..962462c7f3f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/rke2-hardening-guide.md @@ -0,0 +1,259 @@ +--- +title: RKE2 Hardening Guide +--- + +This document provides prescriptive guidance for how to harden an RKE2 cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE2 clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 up to v1.25 | + +:::note +At the time of writing, the upstream CIS Kubernetes v1.25 benchmark is not yet available in Rancher. At this time Rancher is using the CIS v1.23 benchmark when scanning Kubernetes v1.25 clusters. Due to that, the CIS checks 5.2.2, 5.2.3, 5.2.5, 5.2.6, 5.2.7 and 5.2.8 might fail. +::: + +For more details on how to evaluate a hardened RKE2 cluster against the official CIS benchmark, refer to the RKE2 self-assessment guides for specific Kubernetes and CIS benchmark versions. + +RKE2 passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. RKE2 will not modify the host operating system. Therefore, you, the operator, must make a few host-level modifications. +2. Certain CIS controls for Network Policies and Pod Security Standards (or Pod Security Policies (PSP) on RKE2 versions prior to v1.25) will restrict the functionality of the cluster. You must opt into having RKE2 configure these for you. To help ensure these requirements are met, RKE2 can be started with the profile flag set to `cis-1.23` for v1.25 and newer or `cis-1.6` for v1.24 and older. + +## Host-level requirements + +There are two areas of host-level requirements: kernel parameters and etcd process/directory configuration. These are outlined in this section. + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Ensure etcd is configured properly + +The CIS Benchmark requires that the etcd data directory be owned by the `etcd` user and group. This implicitly requires the etcd process run as the host-level `etcd` user. To achieve this, RKE2 takes several steps when started with a valid `cis-1.xx` profile: + +1. Check that the `etcd` user and group exists on the host. If they don't, exit with an error. +2. Create etcd's data directory with `etcd` as the user and group owner. +3. Ensure the etcd process is ran as the `etcd` user and group by setting the etcd static pod's `SecurityContext` appropriately. + +To meet the above requirements, you must: + +#### Create the etcd user + +On some Linux distributions, the `useradd` command will not create a group. The `-U` flag is included below to account for that. This flag tells `useradd` to create a group with the same name as the user. + +```bash +sudo useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U +``` + +## Kubernetes runtime requirements + +The runtime requirements to pass the CIS Benchmark are centered around pod security, network policies and kernel parameters. Most of this is automatically handled by RKE2 when using a valid `cis-1.xx` profile, but some additional operator intervention is required. These are outlined in this section. + +### Ensure `protect-kernel-defaults` is set + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +Both `protect-kernel-defaults` and `profile` flags can be set in the RKE2 template configuration file. +When the `profile` flag is set, RKE2 will set the flag to `true` if it is unset. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + profile: # use cis-1.23 or cis-1.6 + protect-kernel-defaults: true +``` + +### PodSecurity + +RKE2 always runs with some amount of pod security. + + + + +On v1.25 and newer, [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) are used for pod security. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 +``` + +When both the `defaultPodSecurityAdmissionConfigurationTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures the Pod Security Admission Controller with the PSA configuration template `rancher-restricted`, to enforce restricted mode in all namespaces, except the ones in the template's exemption list. + These namespaces are exempted to allow system pods to run without restrictions, which is required for proper operation of the cluster. + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + + + + + +On Kubernetes v1.24 and older, the `PodSecurityPolicy` admission controller is always enabled. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. + +:::note +In the following example the profile is set to `cis-1.6` which is the value defined in the upstream RKE2, but the cluster is actually configured to pass the CIS v1.23 hardened profile +::: + +```yaml +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 +``` + + +When both the `defaultPodSecurityPolicyTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures runtime pod security policies that allow the cluster to pass associated controls. + + + + +:::note +The Kubernetes control plane components and critical additions such as CNI, DNS, and Ingress are ran as pods in the `kube-system` namespace. Therefore, this namespace will have a policy that is less restrictive so that these components can run properly. +::: + +### NetworkPolicies + +When ran with a valid `cis-1.xx` profile, RKE2 will put `NetworkPolicies` in place that passes the CIS Benchmark for Kubernetes' built-in namespaces. These namespaces are: `kube-system`, `kube-public`, `kube-node-lease`, and `default`. + +The `NetworkPolicy` used will only allow pods within the same namespace to talk to each other. The notable exception to this is that it allows DNS requests to be resolved. + +:::note +Operators must manage network policies as normal for additional namespaces that are created. +::: + +### Configure `default` service account + +**Set `automountServiceAccountToken` to `false` for `default` service accounts** + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. The `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE2 install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +For namespaces created by the cluster operator, the following script and configuration file can be used to configure the `default` service account. + +The configuration bellow must be saved to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `sudo chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + echo -n "Patching namespace $namespace - " + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### API Server audit configuration + +CIS requirements 1.2.19 to 1.2.22 are related to configuring audit logs for the API Server. When RKE2 is started with the `profile` flag set, it will automatically configure hardened `--audit-log-` parameters in the API Server to pass those CIS checks. + +RKE2's default audit policy is configured to not log requests in the API Server. This is done to allow cluster operators flexibility to customize an audit policy that suits their auditing requirements and needs, as these are specific to each users' environment and policies. + +A default audit policy is created by RKE2 when started with the `profile` flag set. The policy is defined in `/etc/rancher/rke2/audit-policy.yaml`. + +```yaml +apiVersion: audit.k8s.io/v1 +kind: Policy +metadata: + creationTimestamp: null +rules: +- level: None +``` + +## Reference Hardened RKE2 Template Configuration + +The reference template configuration is used in Rancher to create a hardened RKE2 custom cluster. This reference does not include other required **cluster configuration** directives which will vary depending on your environment. + + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 + protect-kernel-defaults: true +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 + protect-kernel-defaults: true +``` + + + + +## Conclusion + +If you have followed this guide, your RKE2 custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE2 self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/selinux-rpm.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/selinux-rpm.md new file mode 100644 index 00000000000..6dd562f95e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/selinux-rpm.md @@ -0,0 +1,16 @@ +--- +title: SELinux RPM +--- + +[安全增强型 Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) 是对 Linux 的安全增强。 + +它由 Red Hat å¼€å‘，是 Linux 上 MAC（mandatory access controls，强制访问控制）的实现。系统管ç†å‘˜å¯ä»¥ä½¿ç”¨ MAC 设置应用程åºå’Œç”¨æˆ·æ˜¯å¦‚何访问ä¸åŒèµ„æºçš„，例如文件ã€è®¾å¤‡ã€ç½‘络和进程间的通信。SELinux 还通过默认é™åˆ¶æ“作系统æ¥å¢žå¼ºå®‰å…¨æ€§ã€‚ + +被政府机构使用之åŽï¼ŒSELinux å·²æˆä¸ºè¡Œä¸šæ ‡å‡†ï¼Œå¹¶åœ¨ CentOS 7 å’Œ 8 上默认å¯ç”¨ã€‚è¦æ£€æŸ¥ SELinux 是å¦åœ¨ä½ çš„系统上å¯ç”¨å’Œæ‰§è¡Œï¼Œè¯·ä½¿ç”¨ `getenforce`： + +``` +# getenforce +Enforcing +``` + +我们æ供了 [`rancher-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) å’Œ [`rke2-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md) 两个 RPM（Red Hat 软件包），让 Rancher 产å“能够在 SELinux 主机上正常è¿è¡Œã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/set-up-cloud-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/set-up-cloud-providers.md new file mode 100644 index 00000000000..a35d5a22313 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/set-up-cloud-providers.md @@ -0,0 +1,43 @@ +--- +title: 设置 Cloud Provider +--- + +_cloud provider_ 是 Kubernetes 中的一个模å—，它æ供了一个用于管ç†èŠ‚点ã€è´Ÿè½½å‡è¡¡å™¨å’Œç½‘络路由的接å£ã€‚ + +在 Rancher 中设置 cloud provider 时，如果你使用的云æ供商支æŒè‡ªåŠ¨åŒ–，Rancher Server å¯ä»¥åœ¨å¯åŠ¨ Kubernetes 定义时自动é…置新节点ã€è´Ÿè½½å‡è¡¡å™¨æˆ–æŒä¹…存储设备。 + +如果你é…置的节点云æ供商集群ä¸æ»¡è¶³å…ˆå†³æ¡ä»¶ï¼Œé›†ç¾¤å°†æ— æ³•æ­£ç¡®é…置。 + +**Cloud Provider** 选项默认设置为 `None`。 + +å¯ä»¥å¯ç”¨çš„云æ供商包括： + +* Amazon +* Azure +* GCE (Google Compute Engine) +* vSphere + +### 设置 Amazon 云æ供商 + +有关å¯ç”¨ Amazon 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md)。 + +### 设置 Azure 云æ供商 + +有关å¯ç”¨ Azure 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md)。 + +### 设置 GCE 云æ供商 + +有关å¯ç”¨ Google Compute Engine 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md)。 + +### 设置 vSphere 云æ供商 + +有关å¯ç”¨ vSphere 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[树内 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 + +### 设置自定义云æ供商 + +任何 Kubernetes Cloud Provider 都å¯ä»¥é€šè¿‡`自定义`云æ供商进行é…置。 + +对于自定义云æ供商选项，你å¯ä»¥å‚考 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/)，了解如何为你的云æ供商编辑 yaml 文件。特定云æ供商的详细é…置说明如下： + +* [vSphere](https://rke.docs.rancher.com/config-options/cloud-providers/vsphere) +* [OpenStack](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/openstack/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md new file mode 100644 index 00000000000..d93fda6f3a6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md @@ -0,0 +1,29 @@ +--- +title: 通过托管 Kubernetes æ供商设置集群 +--- + +在这ç§æƒ…况下，Rancher ä¸ä¼šé…ç½® Kubernetes，因为它是由 Google Kubernetes Engine (GKE)ã€Amazon Elastic Container Service for Kubernetes 或 Azure Kubernetes Service ç­‰æ供商安装的。 + +如果你使用 Kubernetes æ供商，例如 Google GKE，Rancher 将与对应的云 API 集æˆï¼Œå…许你从 Rancher UI ä¸ºæ‰˜ç®¡é›†ç¾¤åˆ›å»ºå’Œç®¡ç† RBAC。 + +在这个用例中，Rancher 使用æ供商的 API å‘托管æ供商å‘é€è¯·æ±‚。然åŽï¼Œæ供商会为你é…置和托管集群。集群创建æˆåŠŸåŽï¼Œä½ å¯ä»¥åƒç®¡ç†æœ¬åœ°é›†ç¾¤æˆ–云上集群一样，通过 Rancher UI 对集群进行管ç†ã€‚ + +Rancher 支æŒä»¥ä¸‹ Kubernetes æ供商： + +- [Google GKE (Google Kubernetes Engine)](https://cloud.google.com/kubernetes-engine/) +- [Amazon EKS (Amazon Elastic Container Service for Kubernetes)](https://aws.amazon.com/eks/) +- [Microsoft AKS (Azure Kubernetes Service)](https://azure.microsoft.com/en-us/services/kubernetes-service/) +- [Alibaba ACK (Alibaba Cloud Container Service for Kubernetes)](https://www.alibabacloud.com/product/kubernetes) +- [Tencent TKE (Tencent Kubernetes Engine)](https://intl.cloud.tencent.com/product/tke) +- [Huawei CCE (Huawei Cloud Container Engine)](https://www.huaweicloud.com/en-us/product/cce.html) + +## 托管 Kubernetes æä¾›å•†çš„èº«ä»½éªŒè¯ + +使用 Rancher 创建由æ供商托管的集群时，你需è¦è¾“入身份验è¯ä¿¡æ¯ã€‚Rancher 会使用验è¯ä¿¡æ¯æ¥è®¿é—®äº‘厂商的 API。有关如何获å–此信æ¯çš„详情，请å‚阅： + +- [创建 GKE 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +- [创建 EKS 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +- [创建 AKS 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) +- [创建 ACK 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +- [创建 TKE 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) +- [创建 CCE 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/single-node-rancher-in-docker.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/single-node-rancher-in-docker.md new file mode 100644 index 00000000000..ec2524a165f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/single-node-rancher-in-docker.md @@ -0,0 +1,5 @@ +--- +title: Docker 中的å•èŠ‚点 Rancher +--- + +以下文档将讨论 Docker 安装的 [HTTP 代ç†é…ç½®](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md)å’Œ[高级选项](../reference-guides/single-node-rancher-in-docker/advanced-options.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-existing-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-existing-nodes.md new file mode 100644 index 00000000000..d54ae37d50b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-existing-nodes.md @@ -0,0 +1,137 @@ +--- +title: 在现有自定义节点上å¯åŠ¨ Kubernetes +description: è¦åˆ›å»ºå…·æœ‰è‡ªå®šä¹‰èŠ‚点的集群，你需è¦è®¿é—®é›†ç¾¤ä¸­çš„æœåŠ¡å™¨ï¼Œå¹¶æ ¹æ® Rancher çš„è¦æ±‚é…ç½®æœåŠ¡å™¨ã€‚ +--- + +创建自定义集群时，Rancher 使用 RKE（Rancher Kubernetes Engine）在本地裸机æœåŠ¡å™¨ã€æœ¬åœ°è™šæ‹Ÿæœºæˆ–云æœåŠ¡å™¨èŠ‚点中创建 Kubernetes 集群。 + +è¦ä½¿ç”¨æ­¤é€‰é¡¹ï¼Œä½ éœ€è¦è®¿é—®è¦åœ¨ Kubernetes 集群中使用的æœåŠ¡å™¨ã€‚请根æ®[è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)é…ç½®æ¯å°æœåŠ¡å™¨ï¼Œå…¶ä¸­åŒ…括硬件è¦æ±‚å’Œ Docker è¦æ±‚。在æ¯å°æœåŠ¡å™¨ä¸Šå®‰è£… Docker åŽï¼Œä½ è¿˜éœ€è¦åœ¨æ¯å°æœåŠ¡å™¨ä¸Šè¿è¡Œ Rancher UI 中æ供的命令，从而将æ¯å°æœåŠ¡å™¨è½¬æ¢ä¸º Kubernetes 节点。 + +本节介ç»å¦‚何设置自定义集群。 + +## 使用自定义节点创建集群 + +:::note 使用 Windows 主机作为 Kubernetes Worker 节点？ + +在开始之å‰ï¼Œè¯·å‚阅[é…ç½® Windows 自定义集群](use-windows-clusters.md)。 + +::: + +### 1. é…ç½® Linux 主机 + +ä½ å¯ä»¥é€šè¿‡é…ç½® Linux 主机，æ¥åˆ›å»ºè‡ªå®šä¹‰é›†ç¾¤ã€‚你的主机å¯ä»¥æ˜¯ï¼š + +- 云虚拟机 +- 本地虚拟机 +- 裸机æœåŠ¡å™¨ + +如果è¦é‡å¤ä½¿ç”¨ä¹‹å‰çš„自定义集群中的节点，请在å¤ç”¨ä¹‹å‰[清ç†èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md)。如果你é‡å¤ä½¿ç”¨å°šæœªæ¸…ç†çš„节点，则集群é…ç½®å¯èƒ½ä¼šå¤±è´¥ã€‚ + +æ ¹æ®[安装è¦æ±‚](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md)å’Œ[生产就绪集群的检查清å•](checklist-for-production-ready-clusters.md)é…置主机。 + +如果你使用 Amazon EC2 作为主机，并希望使用[åŒæ ˆ (dual-stack)](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) 功能，则需è¦æ»¡è¶³é…置主机的其他[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + +### 2. 创建自定义集群 + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡»**自定义**。 +1. 输入**集群å称**。 +1. 在**集群é…ç½®**中，选择 Kubernetes 版本ã€è¦ä½¿ç”¨çš„网络æ供商，以åŠæ˜¯å¦å¯ç”¨é¡¹ç›®ç½‘络隔离。è¦æŸ¥çœ‹æ›´å¤šé›†ç¾¤é€‰é¡¹ï¼Œè¯·å•å‡»**显示高级选项**。 + + :::note 你使用 Windows 主机作为 Kubernetes Worker 节点？ + + - 请å‚阅[å¯ç”¨ Windows 支æŒé€‰é¡¹](use-windows-clusters.md)。 + - æ”¯æŒ Windows 集群的唯一网络æ’件是 Flannel。 + + ::: + + :::note Amazon EC2 上的åŒæ ˆï¼š + + 如果你使用 Amazon EC2 作为主机，并希望使用[åŒæ ˆ (dual-stack)](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) 功能，则需è¦æ»¡è¶³é…ç½® RKE 的其他[è¦æ±‚](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements)。 + + ::: + +6. 点击**下一步**。 + +4. 使用**æˆå‘˜è§’色**为集群é…置用户授æƒã€‚点击**添加æˆå‘˜**添加å¯ä»¥è®¿é—®é›†ç¾¤çš„用户。使用**角色**下拉èœå•ä¸ºæ¯ä¸ªç”¨æˆ·è®¾ç½®æƒé™ã€‚ + +7. 从**节点角色**中，选择è¦ç”±é›†ç¾¤èŠ‚点充当的角色。你必须为 `etcd`ã€`worker` å’Œ `controlplane` 角色é…置至少一个节点。自定义集群需è¦æ‰€æœ‰ä¸‰ä¸ªè§’色æ‰èƒ½å®Œæˆé…置。有关角色的详细信æ¯ï¼Œè¯·å‚阅[本节](../reference-guides/kubernetes-concepts.md#kubernetes-集群中节点的角色)。 + +:::note + +- 使用 Windows 主机作为 Kubernetes Worker 节点？请å‚阅[本节](use-windows-clusters.md)。 +- 裸机æœåŠ¡å™¨æ醒：如果你想将裸机æœåŠ¡å™¨ä¸“用于æ¯ä¸ªè§’色，则必须为æ¯ä¸ªè§’色é…置一个裸机æœåŠ¡å™¨ï¼ˆå³é…置多个裸机æœåŠ¡å™¨ï¼‰ã€‚ + +::: + +8. **å¯é€‰**：点击[显示高级选项](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md)æ¥æŒ‡å®šæ³¨å†ŒèŠ‚点时使用的 IP 地å€ï¼Œè¦†ç›–节点的主机å，或将[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)或[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)添加到节点。 + +9. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ + +10. 使用你惯用的 shell（例如 PuTTy 或远程终端）登录到你的 Linux 主机。粘贴剪贴æ¿çš„命令并è¿è¡Œã€‚ + +:::note + +如果è¦å°†ç‰¹å®šä¸»æœºä¸“用于特定节点角色，请é‡å¤æ­¥éª¤ 7-10。根æ®éœ€è¦å¤šæ¬¡é‡å¤è¿™äº›æ­¥éª¤ã€‚ + +::: + +11. 在 Linux 主机上è¿è¡Œå®Œå‘½ä»¤åŽï¼Œå•å‡»**完æˆ**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +### 3. ä»…é™äºšé©¬é€Šï¼šæ ‡ç­¾èµ„æº + +如果你已将集群é…置为使用 Amazon 作为**云æ供商**，请使用集群 ID 标记你的 AWS 资æºã€‚ + +[Amazon 文档：标记你的 Amazon EC2 资æº](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html) + +:::note + +ä½ å¯ä»¥ä½¿ç”¨ Amazon EC2 实例，而无需在 Kubernetes 中é…置云æ供商。如果你想使用特定的 Kubernetes 云æ供商功能，é…置云æ供商å³å¯ã€‚如需更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 云æ供商](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/)。 + +::: + +以下资æºéœ€è¦ä½¿ç”¨ `ClusterID` 进行标记： + +- **Nodes**：Rancher 中添加的所有主机。 +- **Subnet**：集群使用的å­ç½‘。 +- **Security Group**：用于你的集群的安全组。 + +:::note + +ä¸è¦æ ‡è®°å¤šä¸ªå®‰å…¨ç»„。创建 Elastic Load Balancer 时，标记多个组会导致错误。 + +::: + +应该使用的标签是： + +``` +Key=kubernetes.io/cluster/, Value=owned +``` + +`` å¯ä»¥æ˜¯ä½ é€‰æ‹©çš„任何字符串。但是，必须在你标记的æ¯ä¸ªèµ„æºä¸Šä½¿ç”¨ç›¸åŒçš„字符串。将值设置为 `owned` 会通知集群所有带有 `` 标记的资æºéƒ½ç”±è¯¥é›†ç¾¤æ‹¥æœ‰å’Œç®¡ç†ã€‚ + +如果你在集群之间共享资æºï¼Œä½ å¯ä»¥å°†æ ‡ç­¾æ›´æ”¹ä¸ºï¼š + +``` +Key=kubernetes.io/cluster/CLUSTERID, Value=shared +``` + +## å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md new file mode 100644 index 00000000000..4811fca8fd9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md @@ -0,0 +1,152 @@ +--- +title: 在云厂商的新节点上å¯åŠ¨ Kubernetes +--- + +在 Rancher 中使用节点模æ¿æ¥åˆ›å»º RKE 或 RKE2 集群时，æ¯ä¸ªç”Ÿæˆçš„节点池都会显示在新的**主机池**选项å¡ä¸­ã€‚ä½ å¯ä»¥é€šè¿‡æ‰§è¡Œä»¥ä¸‹æ“作æ¥æŸ¥çœ‹ä¸»æœºæ± ï¼š + +1. 点击**☰ > 集群管ç†**。 +1. å•å‡» RKE 或 RKE2 集群的å称。 + +## RKE 集群 + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)创建节点池。此节点模æ¿å®šä¹‰äº†è¦ç”¨äºŽåœ¨åŸºç¡€è®¾æ–½æ供商或云厂商中å¯åŠ¨èŠ‚点的å‚数。 + +在托管在云厂商的节点池上安装 Kubernetes 的一个好处是，如果一个节点与集群断开连接，Rancher å¯ä»¥è‡ªåŠ¨åˆ›å»ºå¦ä¸€ä¸ªèŠ‚点并将其加入集群，从而确ä¿èŠ‚点池的数é‡ç¬¦åˆè¦æ±‚。 + +å¯ç”¨äºŽåˆ›å»ºèŠ‚点模æ¿çš„云æ供商是由[主机驱动](use-new-nodes-in-an-infra-provider.md#主机驱动)决定的。 + +### èŠ‚ç‚¹æ¨¡æ¿ + +节点模æ¿ä¿å­˜äº†ç”¨äºŽåœ¨ç‰¹å®šäº‘æ供商中é…置节点时è¦ä½¿ç”¨çš„å‚数。这些节点å¯ä»¥ä»Ž UI å¯åŠ¨ã€‚Rancher 使用 [Docker Machine](https://docs.docker.com/machine/) æ¥é…置这些节点。å¯ç”¨äºŽåˆ›å»ºèŠ‚点模æ¿çš„云æ供商å–决于 Rancher 中状æ€æ˜¯ Active 的主机驱动。 + +在 Rancher 中创建节点模æ¿åŽï¼Œæ¨¡æ¿ä¼šè¢«ä¿å­˜ï¼Œä»¥ä¾¿ä½ å¯ä»¥å†æ¬¡ä½¿ç”¨è¯¥æ¨¡æ¿æ¥åˆ›å»ºèŠ‚点池。节点模æ¿ç»‘定到你的登录å。添加模æ¿åŽï¼Œä½ å¯ä»¥å°†å…¶ä»Žç”¨æˆ·é…置文件中删除。 + +#### 节点标签 + +ä½ å¯ä»¥ä¸ºæ¯ä¸ªèŠ‚点模æ¿æ·»åŠ [标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)，这样，使用节点模æ¿åˆ›å»ºçš„节点都会自动带有这些标签。 + +无效标签会阻止å‡çº§ï¼Œæˆ–阻止 Rancher å¯åŠ¨ã€‚有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 + +#### 节点污点 + +ä½ å¯ä»¥ä¸ºæ¯ä¸ªèŠ‚点模æ¿æ·»åŠ [污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)，这样，使用节点模æ¿åˆ›å»ºçš„节点都会自动带有这些污点。 + +由于污点å¯ä»¥åŒæ—¶æ·»åŠ åˆ°èŠ‚点模æ¿å’ŒèŠ‚点池中，因此如果添加了相åŒé”®çš„污点效果没有冲çªï¼Œåˆ™æ‰€æœ‰æ±¡ç‚¹éƒ½å°†æ·»åŠ åˆ°èŠ‚点中。如果存在具有相åŒé”®ä½†ä¸åŒæ•ˆæžœçš„污点，则节点池中的污点将覆盖节点模æ¿ä¸­çš„污点。 + +#### 节点模æ¿çš„管ç†å‘˜æŽ§åˆ¶ + +管ç†å‘˜å¯ä»¥æŽ§åˆ¶æ‰€æœ‰èŠ‚点模æ¿ã€‚现在，管ç†å‘˜å¯ä»¥ç»´æŠ¤ Rancher 中的所有节点模æ¿ã€‚当节点模æ¿æ‰€æœ‰è€…ä¸å†ä½¿ç”¨ Rancher 时，他们创建的节点模æ¿å¯ä»¥ç”±ç®¡ç†å‘˜ç®¡ç†ï¼Œä»¥ä¾¿ç»§ç»­æ›´æ–°å’Œç»´æŠ¤é›†ç¾¤ã€‚ + +è¦è®¿é—®æ‰€æœ‰èŠ‚点模æ¿ï¼Œç®¡ç†å‘˜éœ€è¦æ‰§è¡Œä»¥ä¸‹æ“作： + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 + +**结果**：列出所有节点模æ¿ã€‚ä½ å¯ä»¥é€šè¿‡å•å‡» **â‹®** æ¥ç¼–辑或克隆模æ¿ã€‚ + +### 节点池 + +使用 Rancher，你å¯ä»¥åŸºäºŽ[节点模æ¿](#节点模æ¿)创建节点池。 + +节点模æ¿å®šä¹‰äº†èŠ‚点的é…置，例如è¦ä½¿ç”¨çš„æ“作系统ã€CPU æ•°é‡å’Œå†…å­˜é‡ã€‚ + +使用节点池的好处是，如果一个节点被销æ¯æˆ–删除，你å¯ä»¥å¢žåŠ  Active 节点的数é‡æ¥è¡¥å¿ä¸¢å¤±çš„节点。节点池å¯ä»¥å¸®åŠ©ä½ ç¡®ä¿èŠ‚点池的计数符åˆè¦æ±‚。 + +æ¯ä¸ªèŠ‚点池必须分é…一个或多个节点角色。 + +æ¯ä¸ªèŠ‚ç‚¹è§’è‰²ï¼ˆå³ etcdã€controlplane å’Œ worker）都应分é…ç»™ä¸åŒçš„节点池。虽然你å¯ä»¥å°†å¤šä¸ªèŠ‚点角色分é…ç»™åŒä¸€ä¸ªèŠ‚点池，但ä¸è¦åœ¨ç”Ÿäº§é›†ç¾¤ä¸­æ‰§è¡Œæ­¤æ“作。 + +推è的设置： + +- 具有 etcd 角色且计数为 3 的节点池 +- 具有 controlplane 角色且计数至少为 2 的节点池 +- 具有 worker 角色且计数至少为 2 的节点池 + +**离线环境中的 RKE1 下游集群节点**： + +默认情况下，在é…ç½® RKE1 下游集群节点时（例如在 vSphere 中），Rancher 会å°è¯•è¿è¡Œ Docker 安装脚本。但是，Rancher Docker 安装脚本在离线环境中会è¿è¡Œå¤±è´¥ã€‚è¦è§£å†³æ­¤é—®é¢˜ï¼Œå¦‚æžœ Docker 已预安装到 VM é•œåƒä¸Šï¼Œä½ å¯ä»¥é€‰æ‹©åœ¨åˆ›å»ºèŠ‚点模æ¿æ—¶è·³è¿‡å®‰è£… Docker。为此，你å¯ä»¥åœ¨ Rancher UI **引擎选项**下的 `Docker 安装 URL` 下拉列表中选择 **æ— **。 + +
**引擎选项下拉列表**
+ +![引擎选项下拉列表](/img/node-template-engine-options-rke1.png) + +#### 节点池污点 + +如果你没有在节点模æ¿ä¸Šå®šä¹‰[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)，则å¯ä»¥ä¸ºæ¯ä¸ªèŠ‚点池添加污点。将污点添加到节点池的好处是你å¯ä»¥æ›´æ”¹èŠ‚点模æ¿ï¼Œè€Œä¸éœ€è¦å…ˆç¡®ä¿æ±¡ç‚¹å­˜åœ¨äºŽæ–°æ¨¡æ¿ä¸­ã€‚ + +æ¯ä¸ªæ±¡ç‚¹éƒ½å°†è‡ªåŠ¨æ·»åŠ åˆ°èŠ‚点池中已创建的节点。因此，如果你在已有节点的节点池中添加污点，污点ä¸ä¼šåº”用到已有的节点，但是添加到该节点池中的新节点都将获得该污点。 + +如果污点åŒæ—¶æ·»åŠ åˆ°èŠ‚点模æ¿å’ŒèŠ‚点池中，且添加了相åŒé”®çš„污点效果没有冲çªï¼Œåˆ™æ‰€æœ‰æ±¡ç‚¹éƒ½å°†æ·»åŠ åˆ°èŠ‚点中。如果存在具有相åŒé”®ä½†ä¸åŒæ•ˆæžœçš„污点，则节点池中的污点将覆盖节点模æ¿ä¸­çš„污点。 + +#### èŠ‚ç‚¹è‡ªåŠ¨æ›¿æ¢ + +Rancher å¯ä»¥è‡ªåŠ¨æ›¿æ¢èŠ‚点池中无法访问的节点。如果节点在指定的时间中处于 Inactive 状æ€ï¼ŒRancher 将使用该节点池的节点模æ¿æ¥é‡æ–°åˆ›å»ºèŠ‚点。 + +:::caution + +自我修å¤èŠ‚点池的功能帮助你替æ¢æ— çŠ¶æ€åº”用的 worker 节点。ä¸å»ºè®®åœ¨ master 节点或连接了æŒä¹…å·çš„节点的节点池上å¯ç”¨èŠ‚点自动替æ¢ï¼Œå› ä¸ºè™šæ‹Ÿæœºä¼šè¢«ä¸´æ—¶å¤„ç†ã€‚节点池中的节点与集群断开连接时，其æŒä¹…å·å°†è¢«ç ´å，从而导致有状æ€åº”用的数æ®ä¸¢å¤±ã€‚ + +::: + +节点自动替æ¢åŸºäºŽ Kubernetes 节点控制器工作。节点控制器定期检查所有节点的状æ€ï¼ˆå¯é€šè¿‡ `kube-controller` çš„ `--node-monitor-period` 标志é…置）。一个节点ä¸å¯è®¿é—®æ—¶ï¼ŒèŠ‚点控制器将污染该节点。å‘生这ç§æƒ…况时，Rancher 将开始其删除倒计时。你å¯ä»¥é…ç½® Rancher 等待删除节点的时间。如果在删除倒计时结æŸå‰æ±¡ç‚¹æ²¡æœ‰è¢«åˆ é™¤ï¼ŒRancher 将继续删除该节点。Rancher 会根æ®èŠ‚点池设置的数é‡æ¥åˆ›å»ºæ–°çš„节点。 + +#### å¯ç”¨èŠ‚ç‚¹è‡ªåŠ¨æ›¿æ¢ + +创建节点池时，你å¯ä»¥æŒ‡å®š Rancher 替æ¢æ— å“应节点的等待时间（以分钟为å•ä½ï¼‰ã€‚ + +1. 在创建或编辑集群的表å•ä¸­ï¼Œè½¬åˆ°**节点池**。 +1. 转到è¦å¯ç”¨èŠ‚点自动替æ¢çš„节点池。在 **Recreate Unreachable After** 字段中，输入 Rancher 在替æ¢èŠ‚点之å‰åº”该等待节点å“应的分钟数。 +1. 填写表å•çš„其余部分以创建或编辑集群。 + +**结果** ：已为节点池å¯ç”¨èŠ‚点自动替æ¢ã€‚ + +#### ç¦ç”¨èŠ‚ç‚¹è‡ªåŠ¨æ›¿æ¢ + +ä½ å¯ä»¥æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ä»Ž Rancher UI ç¦ç”¨èŠ‚点自动替æ¢ï¼š + +1. 点击 **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦ç¦ç”¨èŠ‚点自动替æ¢çš„集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 在**节点池**部分中，转到è¦å¯ç”¨èŠ‚点自动替æ¢çš„节点池。在 **Recreate Unreachable After** 字段中，输入 0。 +1. å•å‡»**ä¿å­˜**。 + +**结果**：已ç¦ç”¨èŠ‚点池的节点自动替æ¢ã€‚ + +### äº‘å‡­è¯ + +节点模æ¿å¯ä»¥ä½¿ç”¨äº‘凭è¯ï¼Œæ¥å­˜å‚¨ç”¨äºŽåœ¨äº‘æ供商中å¯åŠ¨èŠ‚点的凭è¯ï¼Œå…¶ä¼˜ç‚¹æ˜¯ï¼š + +- 凭è¯ä¼šå­˜å‚¨ä¸ºæ›´å®‰å…¨çš„ Kubernetes 密文，而且你无需æ¯æ¬¡éƒ½è¾“入凭è¯ä¾¿å¯ç¼–辑节点模æ¿ã€‚ + +- 创建云凭è¯åŽï¼Œä½ å¯ä»¥é‡æ–°ä½¿ç”¨è¯¥å‡­è¯æ¥åˆ›å»ºå…¶ä»–节点模æ¿ã€‚ + +- 多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯æ¥åˆ›å»ºèŠ‚点池。如果你的密钥被泄露或过期，则å¯ä»¥åœ¨ä¸€ä¸ªä½ç½®æ›´æ–°äº‘凭è¯ï¼Œä»Žè€Œä¸€æ¬¡æ›´æ–°æ‰€æœ‰ä½¿ç”¨è¯¥å‡­è¯çš„节点模æ¿ã€‚ + +创建云凭è¯åŽï¼Œç”¨æˆ·å¯ä»¥[管ç†åˆ›å»ºçš„云凭è¯](../reference-guides/user-settings/manage-cloud-credentials.md)。 + +### 主机驱动 + +如果你找ä¸åˆ°æƒ³è¦çš„主机驱动，你å¯ä»¥åœ¨ Rancher çš„[内置主机驱动](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#激活åœç”¨ä¸»æœºé©±åŠ¨)中查看并激活它，也å¯ä»¥[添加自定义主机驱动](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#添加自定义主机驱动)。 + +## RKE2 集群 + +Rancher 2.6 支æŒç›´æŽ¥ä½¿ç”¨ Rancher UI é…ç½® [RKE2](https://docs.rke2.io/) 集群。RKE2，也称为 RKE Government，是一个完全符åˆæ ‡å‡†çš„ Kubernetes å‘行版，它专注于安全性和åˆè§„性。 + +:::note + +对于 RKE2 集群模æ¿ï¼Œè¯·å‚阅[此页é¢](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md#rke2-集群模æ¿)了解更多信æ¯ã€‚ + +::: + +### 节点角色 + +RKE2 CLI 公开了 `server` å’Œ `agent` 两个角色，它们分别代表 Kubernetes 节点角色 `etcd` + `controlplane` å’Œ `worker`。通过 Rancher 2.6 中的 RKE2 集æˆï¼ŒRKE2 节点池å¯ä»¥åˆ†é…更细粒度的角色，例如 `etcd` å’Œ `controlplane`。 + +ä½ å¯ä»¥åœ¨ RKE2 CLI 中使用标志和节点污染，æ¥æŽ§åˆ¶è°ƒåº¦å·¥ä½œè´Ÿè½½å’Œ Kubernetes master 节点的ä½ç½®ï¼Œä»Žè€Œä½¿ç”¨ `etcd`，`controlplane` å’Œ `worker` 节点功能。这些角色没有在 RKE2 CLI 中实现为第一级角色的原因是，RKE2 被概念化为一组原始构建å—，使用 Rancher 等编排系统得到最佳利用。 + +在 Rancher 中实现这三个节点角色，表示 Rancher 管ç†çš„ RKE2 集群能够轻æ¾ä½¿ç”¨ä¸º RKE 集群推è的相åŒæž¶æž„的所有最佳实践。 + +在[推è的集群架构](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md)中，我们概述了æ¯ä¸ªè§’色集群应该有多少节点： + +- 至少拥有三个角色为 etcd 的节点，æ¥ç¡®ä¿å¤±åŽ»ä¸€ä¸ªèŠ‚点时ä»èƒ½å­˜æ´»ã€‚ +- 至少两个节点具有 controlplane 角色，以实现主组件高å¯ç”¨æ€§ã€‚ +- 至少两个具有 worker 角色的节点，用于在节点故障时é‡æ–°å®‰æŽ’工作负载。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-windows-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-windows-clusters.md new file mode 100644 index 00000000000..59d893a6cc8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/use-windows-clusters.md @@ -0,0 +1,277 @@ +--- +title: 在 Windows 集群上å¯åŠ¨ Kubernetes +--- + +使用 Rancher é…ç½®[自定义集群](use-existing-nodes.md)时，Rancher 通过 RKE（Rancher Kubernetes Engine）在现有节点上安装 Kubernetes。 + +在使用 Rancher é…置的 Windows 集群中，集群必须åŒæ—¶åŒ…å« Linux å’Œ Windows 节点。Kubernetes controlplane åªèƒ½è¿è¡Œåœ¨ Linux 节点上，Windows 节点åªèƒ½æœ‰ Worker 角色。Windows 节点åªèƒ½ç”¨äºŽéƒ¨ç½²å·¥ä½œè´Ÿè½½ã€‚ + +Windows 集群的其他è¦æ±‚如下： + +- åªæœ‰åœ¨åˆ›å»ºé›†ç¾¤æ—¶å¯ç”¨äº† Windows 支æŒçš„集群æ‰èƒ½æ·»åŠ  Windows 节点。无法为现有集群å¯ç”¨ Windows 支æŒã€‚ +- éœ€è¦ Kubernetes 1.15+。 +- 必须使用 Flannel 网络æ供商。 +- Windows 节点必须有 50 GB çš„ç£ç›˜ç©ºé—´ã€‚ + +有关完整的è¦æ±‚列表，请å‚阅[本节](#windows-集群的è¦æ±‚)。 + +æœ‰å…³æ”¯æŒ Windows çš„ Kubernetes 功能摘è¦ï¼Œè¯·å‚阅[在 Windows 中使用 Kubernetes 支æŒçš„功能和é™åˆ¶](https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#supported-functionality-and-limitations)çš„ Kubernetes 文档，或[在 Kubernetes 中调度 Windows 容器的指å—](https://kubernetes.io/docs/setup/production-environment/windows/user-guide-windows-containers/)。 + +### RKE2 Windows + +RKE2 é…置功能还包括在 Windows 集群上安装 RKE2。RKE2 çš„ Windows 功能包括： + +- ç”± containerd æ供支æŒçš„使用 RKE2 çš„ Windows 容器 +- 直接从 Rancher UI é…ç½® Windows RKE2 自定义集群 +- 用于 Windows RKE2 自定义集群的 Calico CNI +- 技术预览包å«äº† Windows Server çš„ SAC 版本（2004 å’Œ 20H2） + +è¦ä½¿ Windows æ”¯æŒ RKE2 自定义集群，请选择 Calico 作为 CNI。 + +:::note + +默认情况下，Rancher å…许 Windows 工作负载 pod 部署在 Windows å’Œ Linux Worker 节点上。在 RKE2 中创建混åˆé›†ç¾¤æ—¶ï¼Œä½ å¿…须编辑 Chart 中的 `nodeSelector`，从而将 Pod 放置到兼容的 Windows 节点上。有关如何使用 `nodeSelector` å°† pod 分é…给节点的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector)。 + +::: + +- Kubernetes v1.24.1 åŠæ›´é«˜ç‰ˆæœ¬æ”¯æŒ Windows RKE2 中的 HostProcess 容器。有关详细信æ¯ï¼Œè¯·å‚阅[上游文档](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/)。 + +## Windows 集群的è¦æ±‚ + +网络ã€æ“作系统和 Docker 的一般节点è¦æ±‚与 [Rancher 安装](installation-requirements.md)的节点è¦æ±‚相åŒã€‚ + +### æ“作系统和 Docker è¦æ±‚ + +我们对 Windows Server å’Œ Windows 容器的支æŒä¸Ž LTSC（长期æœåŠ¡æ¸ é“）和 SAC（åŠå¹´æ¸ é“）的 Microsoft 官方生命周期相匹é…。 + +有关 Windows Server 的支æŒç”Ÿå‘½å‘¨æœŸçš„日期，请å‚阅 [Microsoft 文档](https://docs.microsoft.com/en-us/windows-server/get-started/windows-server-release-info)。 + +### Kubernetes 版本 + +éœ€è¦ Kubernetes v1.15+。 + +如果你在 Windows Server 20H2 Standard Core 上使用 Kubernetes v1.21，则必须在节点上安装补ä¸â€œ2019-08 Servicing Stack Update for Windows Serverâ€ã€‚ + +### 节点è¦æ±‚ + +集群中的主机至少需è¦ï¼š + +- 2 æ ¸ CPU +- 5 GB 内存 +- 50 GB ç£ç›˜ç©ºé—´ + +Rancher ä¸ä¼šé…ç½®ä¸æ»¡è¶³è¦æ±‚的节点。 + +### 网络è¦æ±‚ + +在é…置新集群之å‰ï¼Œè¯·ç¡®ä¿ä½ å·²ç»åœ¨æŽ¥æ”¶å…¥ç«™ç½‘络æµé‡çš„设备上安装了 Rancher。这是集群节点与 Rancher 通信所必需的。如果你尚未安装 Rancher，请在继续阅读本指å—之å‰å…ˆå‚阅[安装文档](installation-and-upgrade.md)进行安装。 + +Rancher 仅支æŒä½¿ç”¨ Flannel 作为网络æ供商的 Windows。 + +有两个网络选项：[**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) å’Œ [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan)。默认选项是 **VXLAN (Overlay)** 模å¼ã€‚ + +对于 **Host Gateway (L2bridge)** 网络，最好为所有节点使用相åŒçš„第 2 层网络。å¦åˆ™ï¼Œä½ éœ€è¦ä¸ºå®ƒä»¬é…置路由规则。有关详细信æ¯ï¼Œè¯·å‚阅[é…置云托管 VM 路由的文档](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#云托管虚拟机的路由é…ç½®)。如果你使用的是 Amazon EC2ã€Google GCE 或 Azure 虚拟机，你需è¦[ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#ç¦ç”¨ç§æœ‰-ip-地å€æ£€æŸ¥)。 + +对于 **VXLAN (Overlay)** 网络，你必须安装 [KB4489899](https://support.microsoft.com/en-us/help/4489899) 修补程åºã€‚大多数云托管的 VM å·²ç»å…·æœ‰æ­¤ä¿®è¡¥ç¨‹åºã€‚ + +如果你在为 AWS 虚拟ç§æœ‰äº‘é…ç½® DHCP 选项集，请注æ„，你åªèƒ½åœ¨ `domain-name` 选项字段中指定一个域å。详情请å‚è§ [DHCP 选项文档](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_DHCP_Options.html)。 + +:::note + +一些 Linux æ“作系统支æŒä»¥ç©ºæ ¼åˆ†éš”的多个域å。但是，其他 Linux æ“作系统和 Windows 将该值视为å•ä¸ªåŸŸå，从而导致æ„外错误。如果你的 DHCP 选项集与具有多个æ“作系统实例的 VPC 相关è”，请仅指定一个域å。 + +::: + +### 带有 ESXi 6.7u2 åŠæ›´é«˜ç‰ˆæœ¬çš„ vSphere 上的 Rancher + +如果你在带有 ESXi 6.7u2 或更高版本的 VMware vSphere 上使用 Rancher，并使用 Red Hat Enterprise Linux 8.3ã€CentOS 8.3 或 SUSE Enterprise Linux 15 SP2 或更高版本，你需è¦ç¦ç”¨ `vmxnet3` 虚拟网络适é…器硬件å¸è½½åŠŸèƒ½ã€‚å¦åˆ™ï¼Œä¸åŒé›†ç¾¤èŠ‚点上的 pod 之间的所有网络连接会因为超时错误而失败。从 Windows pod 到在 Linux 节点上è¿è¡Œçš„关键æœåŠ¡ï¼ˆä¾‹å¦‚ CoreDNS）的所有连接也将失败。外部连接也å¯èƒ½å¤±è´¥ã€‚出现这个问题的原因是 Linux å‘行版在 `vmxnet3` 中å¯ç”¨äº†ç¡¬ä»¶å¸è½½åŠŸèƒ½ï¼Œè€Œä¸” `vmxnet3` 硬件å¸è½½åŠŸèƒ½ä¸­å­˜åœ¨ä¸€ä¸ªä¼šä¸¢å¼ƒå®¢æˆ·è¦†ç›–æµé‡çš„æ•°æ®åŒ…çš„ bug。è¦è§£å†³æ­¤é—®é¢˜ï¼Œå¿…é¡»ç¦ç”¨ `vmxnet3` 硬件å¸è½½åŠŸèƒ½ã€‚此设置ä¸ä¼šåœ¨é‡å¯åŽç»§ç»­ç”Ÿæ•ˆï¼Œå› æ­¤éœ€è¦åœ¨æ¯æ¬¡å¯åŠ¨æ—¶ç¦ç”¨ã€‚推èçš„åšæ³•æ˜¯åœ¨ `/etc/systemd/system/disable_hw_offloading.service` 中创建一个 systemd å•å…ƒæ–‡ä»¶ï¼Œè¿™ä¼šåœ¨å¯åŠ¨æ—¶ç¦ç”¨ `vmxnet3` 硬件å¸è½½åŠŸèƒ½ã€‚ç¦ç”¨ `vmxnet3` 硬件å¸è½½åŠŸèƒ½çš„示例 systemd å•å…ƒæ–‡ä»¶å¦‚下所示。注æ„，`` 必须自定义为主机的 `vmxnet3` 网络接å£ï¼Œå¦‚ `ens192`： + +``` +[Unit] +Description=Disable vmxnet3 hardware offloading feature + +[Service] +Type=oneshot +ExecStart=ethtool -K tx-udp_tnl-segmentation off +ExecStart=ethtool -K tx-udp_tnl-csum-segmentation off +StandardOutput=journal + +[Install] +WantedBy=multi-user.target +``` +然åŽåœ¨ systemd å•å…ƒæ–‡ä»¶ä¸Šè®¾ç½®é€‚当的æƒé™ï¼š +``` +chmod 0644 /etc/systemd/system/disable_hw_offloading.service +``` +最åŽï¼Œå¯ç”¨ systemd æœåŠ¡ï¼š +``` +systemctl enable disable_hw_offloading.service +``` + +### 架构è¦æ±‚ + +Kubernetes 集群管ç†èŠ‚点（`etcd` å’Œ `controlplane`）必须è¿è¡Œåœ¨ Linux 节点上。 + +部署工作负载的 `worker` 节点通常是 Windows 节点，但必须至少有一个 `worker` 节点è¿è¡Œåœ¨ Linux 上，æ‰èƒ½æŒ‰é¡ºåºè¿è¡Œ Rancher Cluster Agentã€DNSã€Metrics Server å’Œ Ingress 相关容器。 + +#### 推è架构 + +我们推è下表中列出的三节点架构，但你始终å¯ä»¥æ·»åŠ é¢å¤–çš„ Linux å’Œ Windows worker 节点æ¥æ‰©å±•é›†ç¾¤ï¼Œä»Žè€Œå®žçŽ°å†—余： + +| 节点 | æ“作系统 | Kubernetes 集群角色 | 用途 | +| ------ | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| 节点 1 | Linux（推è Ubuntu Server 18.04） | controlplane, etcd, worker | ç®¡ç† Kubernetes 集群 | +| 节点 2 | Linux（推è Ubuntu Server 18.04） | Worker | 支æŒé›†ç¾¤çš„ Rancher Cluster Agentã€Metrics Serverã€DNS å’Œ Ingress | +| 节点 3 | Windows（Windows Server 核心版本 1809 或更高版本） | Worker | è¿è¡Œ Windows 容器 | + +### 容器è¦æ±‚ + +Windows è¦æ±‚容器的版本必须与部署容器的 Windows Server 的版本一致。因此，你必须在 Windows Server 核心版本 1809 或更高版本上构建容器。如果你已ç»ä½¿ç”¨æ—©æœŸçš„ Windows Server 核心版本构建了容器，则必须使用 Windows Server 核心版本 1809 或更高版本é‡æ–°æž„建容器。 + +### 云æ供商è¦æ±‚ + +如果你在集群中设置了 Kubernetes 云æ供商，则需è¦è¿›è¡Œä¸€äº›é¢å¤–çš„æ“作。如果你想使用云æ供商的功能，例如为集群自动é…置存储ã€è´Ÿè½½å‡è¡¡å™¨æˆ–其他基础设施，你å¯èƒ½éœ€è¦è®¾ç½®äº‘æ供商。有关如何é…置满足æ¡ä»¶çš„云æ供商集群节点，请å‚阅[此页é¢](../pages-for-subheaders/set-up-cloud-providers.md)。 + +如果你的云æ供商是 GCE（Google Compute Engine），则必须执行以下æ“作： + +- 按照[步骤](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md) 在`cluster.yml` 中å¯ç”¨ GCE 云æ供商。 +- 在 Rancher 中é…置集群时，在 Rancher UI 中选择**自定义云æ供商**作为云æ供商。 + +## æ•™ç¨‹ï¼šå¦‚ä½•åˆ›å»ºæ”¯æŒ Windows 的集群 + +本教程æ述了如何使用[推è架构](#推è架构)中的三个节点创建由 Rancher é…置的集群。 + +在现有节点上使用 Rancher é…置集群时，你需è¦åœ¨æ¯ä¸ªèŠ‚点上安装 [Rancher Agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) æ¥å°†èŠ‚点添加到集群中。在 Rancher UI 中创建或编辑集群时，你会看到一个**自定义节点è¿è¡Œå‘½ä»¤**，你å¯ä»¥åœ¨æ¯å°æœåŠ¡å™¨ä¸Šè¿è¡Œè¯¥å‘½ä»¤ï¼Œä»Žè€Œå°†æœåŠ¡å™¨æ·»åŠ åˆ°é›†ç¾¤ä¸­ã€‚ + +è¦è®¾ç½®æ”¯æŒ Windows 节点和容器的集群，你需è¦å®Œæˆä»¥ä¸‹ä»»åŠ¡ï¼š + + +### 1. é…置主机 + +è¦åœ¨å…·æœ‰ Windows 支æŒçš„现有节点上é…置集群，请准备好你的主机。 + +主机å¯ä»¥æ˜¯ï¼š + +- 云托管的虚拟机 +- 虚拟化集群中的虚拟机 +- 裸金属æœåŠ¡å™¨ + +ä½ å°†é…置三个节点： + +- 一个 Linux èŠ‚ç‚¹ï¼Œç”¨äºŽç®¡ç† Kubernetes controlplane 并存储你的 `etcd`。 +- 第二个 Linux 节点，它将作为 worker 节点。 +- Windows 节点，它将作为 worker 节点è¿è¡Œ Windows 容器。 + +| 节点 | æ“作系统 | +| ------ | ------------------------------------------------------------ | +| 节点 1 | Linux（推è Ubuntu Server 18.04） | +| 节点 2 | Linux（推è Ubuntu Server 18.04） | +| 节点 3 | Windows（Windows Server 核心版本 1809 或更高版本） | + +如果你的节点托管在**云æ供商**上，并且你需è¦è‡ªåŠ¨åŒ–支æŒï¼ˆä¾‹å¦‚è´Ÿè½½å‡è¡¡å™¨æˆ–æŒä¹…存储设备），你的节点还需è¦æ»¡è¶³é¢å¤–çš„é…ç½®è¦æ±‚。详情请å‚è§[选择云æ供商](../pages-for-subheaders/set-up-cloud-providers.md)。 + +### 2. 在现有节点上创建集群 + +在现有节点上创建 Windows 集群的说明与一般[创建自定义集群的说明](use-existing-nodes.md)éžå¸¸ç›¸ä¼¼ï¼Œä½†æœ‰ä¸€äº›ç‰¹å®šäºŽ Windows çš„è¦æ±‚。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œå•å‡»**创建**。 +1. å•å‡»**自定义**。 +1. 在**集群å称**字段中输入集群的å称。 +1. 在 **Kubernetes 版本**下拉èœå•ä¸­ï¼Œé€‰æ‹© v1.19 或更高版本。 +1. 在**网络æ供商**字段中，选择 **Flannel**。 +1. 在 **Windows 支æŒ**中，å•å‡»**å¯ç”¨**。 +1. å¯é€‰ï¼šå¯ç”¨ Windows 支æŒåŽï¼Œä½ å°†èƒ½å¤Ÿé€‰æ‹© Flannel åŽç«¯æ¨¡å¼ã€‚有两个网络选项：[**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) å’Œ [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan)。默认选项是 **VXLAN (Overlay)** 模å¼ã€‚ +1. 点击**下一步**。 + +:::note é‡è¦æ示： + +对于 Host Gateway (L2bridge) 网络，最好为所有节点使用相åŒçš„第 2 层网络。å¦åˆ™ï¼Œä½ éœ€è¦ä¸ºå®ƒä»¬é…置路由规则。有关详细信æ¯ï¼Œè¯·å‚阅[é…置云托管 VM 路由的文档](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#云托管虚拟机的路由é…ç½®)。如果你使用的是 Amazon EC2ã€Google GCE 或 Azure 虚拟机，你需è¦[ç¦ç”¨ç§æœ‰ IP 地å€æ£€æŸ¥](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#ç¦ç”¨ç§æœ‰-ip-地å€æ£€æŸ¥)。 + +::: + +### 3. 将节点添加到集群 + +本节介ç»å¦‚何将 Linux å’Œ Worker 节点注册到集群。你将在æ¯ä¸ªèŠ‚点上è¿è¡Œä¸€ä¸ªå‘½ä»¤ï¼Œè¯¥å‘½ä»¤å°†å®‰è£… Rancher Agent 并å…许 Rancher 管ç†æ¯ä¸ªèŠ‚点。 + +#### 添加 Linux master 节点 + +在本节中，你需è¦åœ¨ Rancher UI 上填写表å•ä»¥èŽ·å–自定义命令，从而在 Linux master 节点上安装 Rancher Agent。然åŽï¼Œå¤åˆ¶è¯¥å‘½ä»¤å¹¶åœ¨ Linux master 节点上è¿è¡Œå‘½ä»¤ï¼Œä»Žè€Œåœ¨é›†ç¾¤ä¸­æ³¨å†Œè¯¥èŠ‚点。 + +集群中的第一个节点应该是具有 **controlplane** å’Œ **etcd** 角色的 Linux 主机。至少必须为此节点å¯ç”¨è¿™ä¸¤ä¸ªè§’色，并且必须先将此节点添加到集群中，然åŽæ‰èƒ½æ·»åŠ  Windows 主机。 + +1. 在**节点æ“作系统**中，å•å‡» **Linux**。 +1. 在**节点角色**中，至少选择 **etcd** å’Œ **controlplane**。推è选择所有的三个角色。 +1. å¯é€‰ï¼šå¦‚果点击**显示高级选项**，你å¯ä»¥è‡ªå®šä¹‰ [Rancher Agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) å’Œ[节点标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)的设置。 +1. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. SSH 到你的 Linux 主机，然åŽè¿è¡Œå¤åˆ¶åˆ°å‰ªè´´æ¿çš„命令。 +1. 完æˆé…ç½® Linux 节点åŽï¼Œé€‰æ‹©**完æˆ**。 + +**结果**： + +你已创建集群，集群的状æ€æ˜¯**é…置中**。Rancher 已在你的集群中。 + +当集群状æ€å˜ä¸º **Active** åŽï¼Œä½ å¯è®¿é—®é›†ç¾¤ã€‚ + +**Active** 状æ€çš„集群会分é…到两个项目： + +- `Default`ï¼šåŒ…å« `default` 命å空间 +- `System`ï¼šåŒ…å« `cattle-system`，`ingress-nginx`，`kube-public` å’Œ `kube-system` 命å空间。 + + +节点å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ‰èƒ½æ³¨å†Œåˆ°é›†ç¾¤ä¸­ã€‚ + +#### 添加 Linux Worker 节点 + +在本节中，我们通过è¿è¡Œå‘½ä»¤å°† Linux Worker 节点注册到集群中。 + +在åˆå§‹é…置集群之åŽï¼Œä½ çš„集群åªæœ‰ä¸€ä¸ª Linux 主机。接下æ¥ï¼Œæˆ‘们添加å¦ä¸€ä¸ª Linux `worker` 主机，用于支æŒé›†ç¾¤çš„ _Rancher Cluster Agent_ã€_Metrics Server_ã€_DNS_ å’Œ _Ingress_。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到你创建的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. å‘下滚动到**节点æ“作系统**。选择 **Linux**。 +1. 在**自定义节点è¿è¡Œå‘½ä»¤**中，转到**节点选项**并选择 **Worker** 角色。 +1. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. 使用远程终端连接登录到你的 Linux 主机。粘贴剪贴æ¿çš„命令并è¿è¡Œã€‚ +1. 在 **Rancher**中，å•å‡»**ä¿å­˜**。 + +**结果**：**Worker** 角色已安装在你的 Linux ä¸»æœºä¸Šï¼Œå¹¶ä¸”èŠ‚ç‚¹ä¼šå‘ Rancher 注册。节点å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ‰èƒ½æ³¨å†Œåˆ°é›†ç¾¤ä¸­ã€‚ + +:::note + +Linux Worker 节点上的污点 + +以下污点将添加集群中的 Linux Worker 节点中。将此污点添加到 Linux Worker 节点åŽï¼Œæ·»åŠ åˆ° Windows 集群的任何工作负载都将自动调度到 Windows Worker 节点。如果想将工作负载专门调度到 Linux Worker 节点上，则需è¦ä¸ºè¿™äº›å·¥ä½œè´Ÿè½½æ·»åŠ å®¹å¿åº¦ã€‚ + +| 污点键 | 污点值 | 污点效果 | +| -------------- | ----------- | ------------ | +| `cattle.io/os` | `linux` | `NoSchedule` | + +::: + +#### 添加 Windows Worker 节点 + +在本节中，我们通过è¿è¡Œå‘½ä»¤å°† Windows Worker 节点注册到集群中。 + +ä½ å¯ä»¥é€šè¿‡ç¼–辑集群并选择 **Windows** 选项，从而将 Windows 主机添加到集群中。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到你创建的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. å‘下滚动到**节点æ“作系统**。选择 **Windows**。注æ„：你将看到 **worker** 角色是唯一å¯ç”¨çš„角色。 +1. å°†å±å¹•ä¸Šæ˜¾ç¤ºçš„命令å¤åˆ¶åˆ°å‰ªè´´æ¿ã€‚ +1. 使用你喜欢的工具（例如 [Microsoft 远程桌é¢](https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-clients)）登录到 Windows 主机。在 **Command Prompt (CMD)** 中è¿è¡Œå¤åˆ¶åˆ°å‰ªè´´æ¿çš„命令。 +1. 在 Rancher 中，å•å‡»**ä¿å­˜**。 +1. å¯é€‰ï¼šå¦‚æžœè¦å‘集群添加更多 Windows 节点，请é‡å¤è¿™äº›æ“作。 + +**结果**：**Worker** 角色已安装在你的 Windows ä¸»æœºä¸Šï¼Œå¹¶ä¸”èŠ‚ç‚¹ä¼šå‘ Rancher 注册。节点å¯èƒ½éœ€è¦å‡ åˆ†é’Ÿæ‰èƒ½æ³¨å†Œåˆ°é›†ç¾¤ä¸­ã€‚你现在已拥有一个 Windows Kubernetes 集群。 + +### å¯é€‰çš„åŽç»­æ­¥éª¤ + +创建集群åŽï¼Œä½ å¯ä»¥é€šè¿‡ Rancher UI 访问集群。最佳实践建议你设置以下访问集群的备用方å¼ï¼š + +- **通过 kubectl CLI 访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#在工作站使用-kubectl-访问集群)在你的工作站上使用 kubectl 访问集群。在这ç§æƒ…况下，你将通过 Rancher Server 的身份验è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ï¼Œç„¶åŽ Rancher 会让你连接到下游集群。此方法å…许你在没有 Rancher UI 的情况下管ç†é›†ç¾¤ã€‚ +- **通过 kubectl CLI 使用授æƒçš„集群端点访问你的集群**：按照[这些步骤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#直接使用下游集群进行身份验è¯)直接使用 kubectl 访问集群，而无需通过 Rancher Server 进行身份验è¯ã€‚我们建议设置此替代方法æ¥è®¿é—®é›†ç¾¤ï¼Œä»¥ä¾¿åœ¨æ— æ³•è¿žæŽ¥åˆ° Rancher 时访问集群。 + +## Azure 中存储类的é…ç½® + +如果你的节点使用 Azure VM，则å¯ä»¥ä½¿ç”¨ [Azure 文件](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv)作为集群的存储类（StorageClass）。详情请å‚è§[此部分](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/user-settings.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/user-settings.md new file mode 100644 index 00000000000..5ed627b7b07 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/user-settings.md @@ -0,0 +1,15 @@ +--- +title: 用户设置 +--- + +在 Rancher 中，æ¯ä¸ªç”¨æˆ·éƒ½æœ‰å¾ˆå¤šä¸Žç™»å½•ç›¸å…³çš„设置，例如个人å好ã€API 密钥等。你å¯ä»¥ä»Ž**用户设置**èœå•ä¸­é…置这些设置。你å¯ä»¥å•å‡»ä¸»èœå•ä¸­çš„头åƒæ¥æ‰“开此èœå•ã€‚ + +![用户设置èœå•](/img/user-settings.png) + +å¯ç”¨çš„用户设置包括： + +- [API & 密钥](../reference-guides/user-settings/api-keys.md)：如果你想以编程方å¼ä¸Ž Rancher 交互，你需è¦ä¸€ä¸ª API 密钥。你å¯ä»¥æŒ‰ç…§æœ¬èŠ‚中的说明获å–密钥。 +- [云凭è¯](../reference-guides/user-settings/manage-cloud-credentials.md)：管ç†[节点模æ¿](use-new-nodes-in-an-infra-provider.md#节点模æ¿)使用的云凭è¯ï¼Œä»Žè€Œ[为集群é…置节点](../pages-for-subheaders/launch-kubernetes-with-rancher.md)。 +- [节点模æ¿](../reference-guides/user-settings/manage-node-templates.md)ï¼šç®¡ç† [Rancher 用æ¥ä¸ºé›†ç¾¤é…置节点](../pages-for-subheaders/launch-kubernetes-with-rancher.md)的模æ¿ã€‚ +- [å好设置](../reference-guides/user-settings/user-preferences.md)：设置 Rancher UI 的表é¢é¦–选项。 +- 登出：结æŸä½ çš„用户会è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/vsphere.md new file mode 100644 index 00000000000..db206b47f96 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/vsphere.md @@ -0,0 +1,56 @@ +--- +title: 创建 vSphere 集群 +description: 使用 Rancher 创建 vSphere 集群。集群å¯èƒ½åŒ…括具有ä¸åŒå±žæ€§çš„ VM 组，这些属性å¯ç”¨äºŽç»†ç²’度控制节点的大å°ã€‚ +--- + +import YouTube from '@site/src/components/YouTube' + +ä½ å¯ä»¥ç»“åˆä½¿ç”¨ Rancher 与 vSphere，从而在本地体验云环境的æ“作。 + +Rancher å¯ä»¥åœ¨ vSphere 中é…置节点并在其上安装 Kubernetes。在 vSphere 中创建 Kubernetes 集群时，Rancher 首先与 vCenter API 通信æ¥é…置指定数é‡çš„虚拟机。然åŽåœ¨å®ƒä»¬ä¹‹ä¸Šå®‰è£… Kubernetes。 + +vSphere 集群å¯èƒ½ç”±å¤šç»„具有ä¸åŒå±žæ€§ï¼ˆä¾‹å¦‚内存或 vCPU æ•°é‡ï¼‰çš„ VM 组æˆã€‚è¿™ç§åˆ†ç»„å…许对æ¯ä¸ª Kubernetes 角色的节点大å°è¿›è¡Œç»†ç²’度控制。 + +## Rancher 2.3 中的 vSphere 增强功能 + +我们更新了 vSphere 节点模æ¿ï¼Œä½¿ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹å¢žå¼ºåŠŸèƒ½åœ¨æœ¬åœ°ä½“验云æ“作： + +### 自我修å¤çš„节点池 + +使用 Rancher é…ç½® vSphere 节点的最大优势之一，是å…许你在本地集群中使用 Rancher 的自我修å¤èŠ‚点池（也称为[节点自动替æ¢åŠŸèƒ½](use-new-nodes-in-an-infra-provider.md#节点自动替æ¢)）。自我修å¤èŠ‚点池的功能帮助你替æ¢æ— çŠ¶æ€åº”用的 worker 节点。当 Rancher 使用节点模æ¿é…置节点时，Rancher å¯ä»¥è‡ªåŠ¨æ›¿æ¢æ— æ³•è®¿é—®çš„节点。 + +:::caution + +ä¸å»ºè®®åœ¨ master 节点或连接了æŒä¹…å·çš„节点的节点池上å¯ç”¨èŠ‚点自动替æ¢ï¼Œå› ä¸ºè™šæ‹Ÿæœºä¼šè¢«ä¸´æ—¶å¤„ç†ã€‚节点池中的节点与集群断开连接时，其æŒä¹…å·å°†è¢«ç ´å，从而导致有状æ€åº”用的数æ®ä¸¢å¤±ã€‚ + +::: + +### 实例和调度的动æ€å¡«å……选项 + +vSphere 的节点模æ¿å·²æ›´æ–°ã€‚当你使用 vSphere 凭è¯åˆ›å»ºèŠ‚点模æ¿æ—¶ï¼Œè¯¥æ¨¡æ¿ä¼šè‡ªåŠ¨å¡«å……你在 vSphere 控制å°ä¸­å¯ä»¥è®¿é—®çš„相åŒçš„虚拟机é…置选项。 + +è¦å¡«å……的字段设置需è¦æ»¡è¶³[先决æ¡ä»¶](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md#vsphere-中的准备工作)。 + +### 更多支æŒçš„æ“作系统 + +ä½ å¯ä»¥ä½¿ç”¨ä»»ä½•æ”¯æŒ `cloud-init` çš„æ“作系统æ¥é…ç½® VM。[cloud config](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) ä»…æ”¯æŒ YAML æ ¼å¼ã€‚ + +### 2.3.3 节点模æ¿åŠŸèƒ½çš„è§†é¢‘ä»‹ç» + +在这段 YouTube 视频中，我们演示了如何使用新的节点模æ¿åœ¨æœ¬åœ°çŽ¯å¢ƒä½“验云环境一样的æ“作。 + + + +## 创建 vSphere 集群 + +在[本节](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md)中，你将学习如何使用 Rancher 在 vSphere 中安装 [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes 集群。 + +## é…置存储 + +有关如何使用 Rancher 在 vSphere 中é…置存储的示例，请å‚阅[本节](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md)。è¦åœ¨ vSphere 中动æ€é…置存储，你必须å¯ç”¨ vSphere 云æ供商。请å‚阅[树内 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 + +## å¯ç”¨ vSphere 云æ供商 + +在 Rancher 中设置云æ供商时，Rancher Server å¯ä»¥è‡ªåŠ¨ä¸ºé›†ç¾¤é…置新的基础设施，包括新节点或æŒä¹…存储设备。 + +有关å¯ç”¨ vSphere 云æ供商的详细信æ¯ï¼Œè¯·å‚阅[树内 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md)å’Œ[树外 vSphere é…ç½®](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/workloads-and-pods.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/workloads-and-pods.md new file mode 100644 index 00000000000..d2f6bba4258 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/pages-for-subheaders/workloads-and-pods.md @@ -0,0 +1,78 @@ +--- +title: "Kubernetes 工作负载和 Pod" +description: "了解在 Kubernetes 中构建å¤æ‚容器化应用程åºçš„两ç§ç»“构：Kubernetes 工作负载和 Pod" +--- + +ä½ å¯ä»¥ä½¿ç”¨ä¸¤ç§åŸºæœ¬ç»“构（pod 和工作负载）在 Kubernetes 中构建å¤æ‚的容器化应用程åºã€‚构建应用程åºåŽï¼Œä½ å¯ä»¥ä½¿ç”¨ç¬¬ä¸‰ç§ç»“构（service）在集群中或互è”网上公开应用程åºã€‚ + +### Pod + +[_Pod_](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) 是一个或多个共享网络命å空间和存储å·çš„容器。大多数 pod åªæœ‰ä¸€ä¸ªå®¹å™¨ã€‚因此，我们讨论的 _pod_ 通常等åŒäºŽ _容器_。扩展 pod çš„æ–¹å¼ä¸Žæ‰©å±•å®¹å™¨çš„æ–¹å¼ç›¸åŒï¼Œå³é…置实现æœåŠ¡çš„åŒä¸€ pod 的多个实例。通常，Pod 会根æ®å·¥ä½œè´Ÿè½½è¿›è¡Œæ‰©å±•å’Œç®¡ç†ã€‚ + +### 工作负载 + +_工作负载_ 是为 pod 设置部署规则的对象。Kubernetes 基于这些规则执行部署，并根æ®åº”用程åºçš„当å‰çŠ¶æ€æ¥æ›´æ–°å·¥ä½œè´Ÿè½½ã€‚ +工作负载让你å¯ä»¥å®šä¹‰åº”用程åºè°ƒåº¦ã€æ‰©å±•å’Œå‡çº§çš„规则。 + +#### 工作负载类型 + +Kubernetes 将工作负载分为ä¸åŒçš„类型。Kubernetes 支æŒçš„最æµè¡Œçš„类型是： + +- [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) + + _Deployment_ 最适åˆç”¨äºŽæ— çŠ¶æ€åº”用程åºï¼ˆå³ä¸éœ€è¦ç»´æŠ¤å·¥ä½œè´Ÿè½½çš„状æ€ï¼‰ã€‚ç”± Deployment 类型工作负载管ç†çš„ Pod 是独立且一次性的。如果 pod 中断了，Kubernetes 会删除该 pod 然åŽé‡æ–°åˆ›å»ºå®ƒã€‚一个示例应用程åºæ˜¯ Nginx Web æœåŠ¡å™¨ã€‚ + +- [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + + 与 Deployment 相比，_StatefulSet_ 最适åˆåœ¨éœ€è¦ç»´æŠ¤èº«ä»½å’Œå­˜å‚¨æ•°æ®çš„应用程åºä¸­ä½¿ç”¨ã€‚适用的应用程åºç±»ä¼¼äºŽ Zookeeper（一个需è¦æ•°æ®åº“进行存储的应用程åºï¼‰ã€‚ + +- [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) + + _Daemonset_ ç¡®ä¿é›†ç¾¤ä¸­çš„æ¯ä¸ªèŠ‚点都è¿è¡Œ pod 的副本。如果你需è¦æ”¶é›†æ—¥å¿—或监控节点性能，这ç§ç±»ä¼¼ daemon 的工作负载效果是最好的。 + +- [Job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) + + _Job_ å¯åŠ¨ä¸€ä¸ªæˆ–多个 Pod 并确ä¿æŒ‡å®šæ•°é‡çš„ Pod 能æˆåŠŸç»ˆæ­¢ã€‚Job 最好用于è¿è¡Œæœ‰é™ä»»åŠ¡è‡³å®ŒæˆçŠ¶æ€ï¼Œè€Œä¸æ˜¯ç®¡ç†æ­£åœ¨è¿›è¡Œçš„应用程åºçš„所需状æ€ã€‚ + +- [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) + + _CronJobs_ 与 Job 类似。但是，CronJob 会基于 cron 的计划è¿è¡Œåˆ°å®ŒæˆçŠ¶æ€ã€‚ + +### Services + +在许多用例中，工作负载必须： + +- 由集群中的其他工作负载访问。 +- 暴露给外部。 + +ä½ å¯ä»¥é€šè¿‡åˆ›å»ºä¸€ä¸ª _Service_ 实现这些目的。Service 使用[选择器/标签（查看代ç ç¤ºä¾‹ï¼‰](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#service-and-replicationcontroller)æ¥æ˜ å°„到底层工作负载的 pod。Rancher UI 使用你选择的æœåŠ¡ç«¯å£å’Œç±»åž‹æ¥è‡ªåŠ¨åˆ›å»º service 以åŠå·¥ä½œè´Ÿè½½ï¼Œä»Žè€Œç®€åŒ–此映射过程。 + +#### Service 类型 + +Rancher 中有几ç§å¯ç”¨çš„ Service 类型。以下æè¿°æ¥è‡ª [Kubernetes 文档](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types)。 + +- **ClusterIP** + + > 在集群内部 IP 上公开 Service。如果你选择此值，Service åªèƒ½ä»Žé›†ç¾¤å†…访问。这是默认的 `ServiceType`。 + +- **NodePort** + + > 在æ¯ä¸ªèŠ‚点 IP 上的é™æ€ç«¯å£ï¼ˆ`NodePort`）上暴露 Service。`ClusterIP` service 是自动创建的，而 `NodePort` service 会路由到 ClusterIP service。你å¯ä»¥é€šè¿‡è¯·æ±‚ `:` 在集群外部è”ç³» `NodePort` service。 + +- **LoadBalancer** + + > 使用云æ供商的负载å‡è¡¡å™¨å‘外部公开æœåŠ¡ã€‚`NodePort` å’Œ `ClusterIP` service 是自动创建的，外部负载å‡è¡¡å™¨ä¼šè·¯ç”±åˆ°è¿™äº› service。 + +## 工作负载选项 + +以下文档介ç»äº†å¦‚何部署工作负载和使用工作负载选项。 + +- [部署工作负载](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md) +- [å‡çº§å·¥ä½œè´Ÿè½½](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md) +- [回滚工作负载](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md) + +## 相关链接 + +### 外部链接 + +- [Service](https://kubernetes.io/docs/concepts/services-networking/service/) diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/rancher-manager.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/rancher-manager.md new file mode 100644 index 00000000000..7df74298395 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/rancher-manager.md @@ -0,0 +1,15 @@ +--- +slug: / +title: "什么是 Rancher？" +sidebar_label: 什么是 Rancher？ +description: "Rancher 在 Kubernetes 之上添加了新的功能，让你一站å¼ç®¡ç†ä¸Šç™¾ä¸ªé›†ç¾¤ï¼Œç»Ÿä¸€ RBAC，开å¯ç›‘控和告警。了解更多。" +--- +Rancher 是一个 Kubernetes 管ç†å·¥å…·ï¼Œè®©ä½ èƒ½åœ¨ä»»ä½•åœ°æ–¹å’Œä»»ä½•æ供商上部署和è¿è¡Œé›†ç¾¤ã€‚ + +Rancher å¯ä»¥åˆ›å»ºæ¥è‡ª Kubernetes 托管æœåŠ¡æ供商的集群，创建节点并安装 Kubernetes，或者导入在任何地方è¿è¡Œçš„现有 Kubernetes 集群。 + +Rancher 基于 Kubernetes 添加了新的功能，包括统一所有集群的身份验è¯å’Œ RBAC，让系统管ç†å‘˜ä»Žä¸€ä¸ªä½ç½®æŽ§åˆ¶å…¨éƒ¨é›†ç¾¤çš„访问。 + +此外，Rancher å¯ä»¥ä¸ºé›†ç¾¤å’Œèµ„æºæ供更精细的监控和告警，将日志å‘é€åˆ°å¤–部æ供商，并通过应用商店（Application Catalogï¼‰ç›´æŽ¥é›†æˆ Helm。如果你拥有外部 CI/CD 系统，你å¯ä»¥å°†å…¶ä¸Ž Rancher 对接。没有的è¯ï¼Œä½ ä¹Ÿå¯ä»¥ä½¿ç”¨ Rancher æ供的 Fleet 自动部署和å‡çº§å·¥ä½œè´Ÿè½½ã€‚ + +Rancher 是一个 _全栈å¼_ çš„ Kubernetes 容器管ç†å¹³å°ï¼Œä¸ºä½ æ供在任何地方都能æˆåŠŸè¿è¡Œ Kubernetes 的工具。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/about-the-api/api-tokens.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/about-the-api/api-tokens.md new file mode 100644 index 00000000000..59b103fa476 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/about-the-api/api-tokens.md @@ -0,0 +1,86 @@ +--- +title: API 令牌 +--- + +默认情况下，æŸäº›é›†ç¾¤çº§åˆ«çš„ API 令牌是使用无é™æœŸ TTL（`ttl=0`）生æˆçš„。æ¢è¨€ä¹‹ï¼Œé™¤éžä½ è®©ä»¤ç‰Œå¤±æ•ˆï¼Œå¦åˆ™ `ttl=0` çš„ API 令牌永远ä¸ä¼šè¿‡æœŸã€‚令牌ä¸ä¼šå› ä¸ºæ›´æ”¹å¯†ç è€Œå¤±æ•ˆã€‚ + +è¦åœç”¨ API 令牌，你å¯ä»¥åˆ é™¤ä»¤ç‰Œæˆ–åœç”¨ç”¨æˆ·è´¦å·ã€‚ + +### 删除令牌 +è¦åˆ é™¤ä»¤ç‰Œï¼š + +1. 转到 `https:///v3/tokens`，在 Rancher API 视图中查看包å«æ‰€æœ‰ä»¤ç‰Œçš„列表。 + +1. 通过 ID 访问è¦åˆ é™¤çš„令牌。例如，`https:///v3/tokens/kubectl-shell-user-vqkqt`。 + +1. å•å‡»**删除**。 + +以下是使用 `ttl=0` 生æˆçš„完整令牌列表： + +| 令牌 | æè¿° | +| ----------------- | -------------------------------------------------------------------------------------- | +| `kubeconfig-*` | Kubeconfig 令牌 | +| `kubectl-shell-*` | 在æµè§ˆå™¨ä¸­è®¿é—® `kubectl` shell | +| `agent-*` | Agent deployment 令牌 | +| `compose-token-*` | compose 令牌 | +| `helm-token-*` | Helm Chart deployment 令牌 | +| `telemetry-*` | é¥æµ‹ä»¤ç‰Œ | +| `drain-node-*` | 用于清空的令牌（由于没有原生 Kubernetes API，我们使用 `kubectl` æ¥æ¸…空） | + + +### 在 Kubeconfig 令牌上设置 TTL + +管ç†å‘˜å¯ä»¥åœ¨ Kubeconfig 令牌上设置全局存活时间 (time-to-live，TTL)。如需更改默认 kubeconfig TTL，你å¯ä»¥å¯¼èˆªåˆ°å…¨å±€è®¾ç½®å¹¶å°† [`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) 设置为所需的æŒç»­æ—¶é—´ï¼ˆå•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚[`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) 的默认值为 0，表示令牌永ä¸è¿‡æœŸã€‚ + +:::note + +除了由 CLI 创建的用于[ç”Ÿæˆ kubeconfig 令牌](#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)的令牌之外，所有 kubeconfig 令牌都使用此设置。 + +::: + +### 在生æˆçš„ Kubeconfig 中ç¦ç”¨ä»¤ç‰Œ + +1. å°† `kubeconfig-generate-token` 设置为 `false`。此设置让 Rancher ä¸å†åœ¨ç”¨æˆ·å•å‡»ä¸‹è½½ kubeconfig 文件时自动生æˆä»¤ç‰Œã€‚如果åœç”¨æ­¤è®¾ç½®ï¼Œç”Ÿæˆçš„ kubeconfig 将引用 [Rancher CLI](../cli-with-rancher/kubectl-utility.md#使用-kubectl-å’Œ-kubeconfig-令牌进行-ttl-认è¯) æ¥æ£€ç´¢é›†ç¾¤çš„短期令牌。当这个 kubeconfig 在客户端（例如 `kubectl`）中使用时，你需è¦å®‰è£… Rancher CLI æ¥å®Œæˆç™»å½•è¯·æ±‚。 + +2. å°† `kubeconfig-token-ttl-minutes` 设置为所需的时长（å•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚`kubeconfig-token-ttl-minutes` 默认设置为 960ï¼ˆå³ 16 å°æ—¶ï¼‰ã€‚ + +### 令牌哈希 + +ä½ å¯ä»¥å¯ç”¨ä»¤ç‰Œå“ˆå¸Œï¼Œä»¤ç‰Œå°†ä½¿ç”¨ SHA256 算法进行å•å‘哈希。这是一个ä¸å¯é€†çš„æ“作，一旦å¯ç”¨ï¼Œæ­¤åŠŸèƒ½å°†æ— æ³•ç¦ç”¨ã€‚在å¯ç”¨åŠŸèƒ½æˆ–在测试环境中评估之å‰ï¼Œå»ºè®®ä½ å…ˆè¿›è¡Œå¤‡ä»½ã€‚ + +è¦å¯ç”¨ä»¤ç‰Œå“ˆå¸Œï¼Œè¯·å‚阅[本节](../../pages-for-subheaders/enable-experimental-features.md)。 + +此功能将影å“所有令牌，包括但ä¸é™äºŽä»¥ä¸‹å†…容： + +- Kubeconfig 令牌 +- æŒæœ‰è€…令牌 API 密钥/调用 +- 内部æ“作使用的令牌 + +### 令牌设置 +ä»¥ä¸‹å…¨å±€è®¾ç½®ä¼šå½±å“ Rancher 令牌的行为： + +| 设置 | æè¿° | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes) | 用户认è¯ä¼šè¯ä»¤ç‰Œçš„ TTL（å•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚ | +| [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes) | 默认 TTL，应用于所有 kubeconfig 令牌（除了[ç”± Rancher CLI 生æˆçš„令牌](#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)）。**此设置从 2.6.6 版本开始引入。** | +| [`kubeconfig-token-ttl-minutes`](#kubeconfig-token-ttl-minutes) | 在 CLI 中生æˆçš„令牌 TTL。**自 2.6.6 起已弃用，并将在 2.8.0 中删除**。请知悉，`kubeconfig-default-token-TTL-minutes` 将用于所有 kubeconfig 令牌。 | +| [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) | 除了由 [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes) 控制的令牌外，所有令牌的最大 TTL。 | +| [`kubeconfig-generate-token`](#kubeconfig-generate-token) | 如果为 true，则在用户下载 kubeconfig 时自动生æˆä»¤ç‰Œã€‚ | + +#### auth-user-session-ttl-minutes +存活时间（TTL）（å•ä½ï¼šåˆ†é’Ÿï¼‰ï¼Œç”¨äºŽç¡®å®šç”¨æˆ·èº«ä»½éªŒè¯ä¼šè¯ä»¤ç‰Œçš„到期时间。过期åŽï¼Œç”¨æˆ·å°†éœ€è¦ç™»å½•å¹¶èŽ·å–新令牌。此设置ä¸å— [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) çš„å½±å“。会è¯ä»¤ç‰Œæ˜¯åœ¨ç”¨æˆ·ç™»å½• Rancher 时创建的。 + +#### kubeconfig-default-token-TTL-minutes +存活时间（TTL）（å•ä½ï¼šåˆ†é’Ÿï¼‰ï¼Œç”¨äºŽç¡®å®š kubeconfig 令牌的到期时间。令牌过期åŽï¼ŒAPI 将拒ç»ä»¤ç‰Œã€‚此设置的值ä¸èƒ½å¤§äºŽ [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) 的值。此设置适用于在请求的 kubeconfig 文件中生æˆçš„令牌，ä¸åŒ…括[ç”± Rancher CLI 生æˆçš„](#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)令牌。 +**此设置从 2.6.6 版本开始引入**。 + +#### kubeconfig-token-ttl-minutes +存活时间（TTL）（å•ä½ï¼šåˆ†é’Ÿï¼‰ï¼Œç”¨äºŽç¡®å®šç”± CLI 生æˆçš„ kubeconfig 令牌的到期时间。当 [`kubeconfig-generate-token`](#kubeconfig-generate-token) 设为 false 时，则由 CLI 生æˆä»¤ç‰Œã€‚令牌过期åŽï¼ŒAPI 将拒ç»ä»¤ç‰Œã€‚此设置的值ä¸èƒ½å¤§äºŽ [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) 的值。 +**自版本 2.6.6 起已弃用，并将在 2.8.0 中删除。请知悉，此设置将被 [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes) 的值替æ¢**。 + +#### auth-token-max-ttl-minutes +身份验è¯ä»¤ç‰Œçš„最大生存时间 (TTL)（å•ä½ï¼šåˆ†é’Ÿï¼‰ã€‚如果用户å°è¯•åˆ›å»ºä¸€ä¸ª TTL 大于 `auth-token-max-ttl-minutes` 的令牌，Rancher 会将令牌 TTL 设置为 `auth-token-max-ttl-minutes` 的值。身份验è¯ä»¤ç‰Œæ˜¯ä¸ºéªŒè¯ API 请求而创建的。 +**2.6.6 版本更改：适用于所有 kubeconfig 令牌和 API 令牌。** + +#### kubeconfig-generate-token +如果设置为 true，则通过 UI 请求的 kubeconfig 将包å«ä¸€ä¸ªæœ‰æ•ˆçš„令牌。如果设置为 false，kubeconfig 将包å«ä¸€ä¸ªä½¿ç”¨ Rancher CLI æ示用户登录的命令。然åŽï¼Œ[CLI 将为用户检索和缓存令牌](../cli-with-rancher/kubectl-utility.md#使用-kubectl-å’Œ-kubeconfig-令牌进行-ttl-认è¯)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/backup-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/backup-configuration.md new file mode 100644 index 00000000000..24505753d86 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/backup-configuration.md @@ -0,0 +1,182 @@ +--- +title: 备份é…ç½® +--- + +ä½ å¯ä»¥åœ¨å¤‡ä»½åˆ›å»ºé¡µé¢é…置调度，å¯ç”¨åŠ å¯†ï¼Œå¹¶æŒ‡å®šå¤‡ä»½çš„存储ä½ç½®ã€‚ + + +## 调度 + +选择第一个选项æ¥æ‰§è¡Œä¸€æ¬¡æ€§å¤‡ä»½ï¼Œæˆ–选择第二个选项æ¥å®‰æŽ’定期备份。选择**定期备份**åŽï¼Œä½ å¯ä»¥é…置以下两个字段： + +- **调度**ï¼šè¯¥å­—æ®µæ”¯æŒ + - 标准 [cron 表达å¼](https://en.wikipedia.org/wiki/Cron)，例如 `"0 * * * *"` + - æ述符，例如 `"@midnight"` 或 `"@every 1h30m"` +- **ä¿ç•™æ•°é‡**：指定必须ä¿ç•™çš„备份文件数é‡ã€‚如果文件数é‡è¶…过 `retentionCount` 设置的值，最旧的文件将被删除。默认值为 10。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `schedule` | æ供用于调度定期备份的 cron 字符串。 | +| `retentionCount` | æä¾›è¦ä¿ç•™çš„备份文件数é‡ã€‚ | + +## 加密 + +rancher-backup 通过调用 kube-apiserver æ¥æ”¶é›†èµ„æºã€‚apiserver 返回的对象会被解密，所以å³ä½¿å¯ç”¨äº†[é™æ€åŠ å¯†](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/)，备份收集的加密对象也会是明文。 + +为é¿å…以明文形å¼å­˜å‚¨èµ„æºï¼Œä½ å¯ä»¥ä½¿ç”¨ä¸Žé™æ€åŠ å¯†ç›¸åŒçš„ `EncryptionConfiguration` 文件æ¥åŠ å¯†å¤‡ä»½ä¸­çš„特定资æºã€‚ + +:::note é‡è¦æ示： + +rancher-backup Operator ä¸ä¼šä¿å­˜ `EncryptionConfiguration`，因此在备份中加密对象时，你必须自行ä¿å­˜è¯¥æ–‡ä»¶ã€‚ + +例如，[å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)时，该文件å¯ç”¨äºŽåœ¨æ–°é›†ç¾¤â€‹â€‹ä¸­é‡æ–°åˆ›å»º secret。 + +::: + +Operator å°† `EncryptionConfiguration` 用作 `cattle-resources-system` 命å空间中的 Kubernetes Secret，在 Secret æ•°æ®ä¸­ï¼Œå®ƒä½äºŽå为 `encryption-provider-config.yaml` çš„ key 下é¢ã€‚ + +对于 `EncryptionConfiguration`，你å¯ä»¥ä½¿ç”¨ [Kubernetes 文档中æ供的示例](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#understanding-the-encryption-at-rest-configuration)。 + +为确ä¿åœ¨ secret 中使用正确的密钥，加密é…置文件必须命å为 `encryption-provider-config.yaml`。下é¢çš„命令使用 `--from-file` 标志æ¥åˆ›å»ºå…·æœ‰æ­£ç¡®å¯†é’¥å称的 Secret。 + +å°† `EncryptionConfiguration` ä¿å­˜åˆ°å为 `encryption-provider-config.yaml` 的文件中，并è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +```bash +kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system +``` + +这将确ä¿å¯†æ–‡åŒ…å«ä¸€ä¸ªå为 `encryption-provider-config.yaml` çš„ key，而且 operator 会使用该 key æ¥èŽ·å–加密é…置。 + +`Encryption Config Secret` 下拉èœå•å°†è¿‡æ»¤å¹¶ä»…列出拥有这个 key 的密文。 + +![](/img/backup_restore/backup/encryption.png) + +在上é¢çš„示例命令中，`encryptionconfig` 这个å称å¯ä»¥ä»»æ„修改。 + + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | æä¾› `cattle-resources-system` 命å空间中包å«åŠ å¯†é…置文件的密文的å称。 | + +## 存储ä½ç½® + +如果在备份中指定了 `StorageLocation`，operator 将从该特定的 S3 存储桶中检索备份ä½ç½®ã€‚如果没有指定，operator å°†å°è¯•åœ¨é»˜è®¤ operator 级别的 S3 存储和 operator 级别的 PVC 存储中查找这个文件。默认存储ä½ç½®æ˜¯åœ¨éƒ¨ç½² `rancher-backup` operator æ—¶é…置的。 + +如果你选择第一个选项，备份将存储在安装 rancher-backup Chart æ—¶é…置的存储ä½ç½®ä¸­ã€‚如果你选择第二个选项，你å¯ä»¥é…ç½®ä¸åŒçš„兼容 S3 的对象存储作为备份存储ä½ç½®ã€‚ + +### S3 + +S3 存储ä½ç½®åŒ…å«ä»¥ä¸‹é…置字段： + +1. **凭è¯å¯†æ–‡**（å¯é€‰ï¼‰ï¼šå¦‚果你需è¦ä½¿ç”¨ AWS 访问密钥（access key）和密文密钥（secret key）æ¥è®¿é—® S3 存储桶，请使用带有密钥和指令 `accessKey` å’Œ `secretKey` 的凭è¯æ¥åˆ›å»ºå¯†æ–‡ã€‚它å¯ä»¥æ˜¯ä»»æ„一个命å空间。你å¯ä»¥ç‚¹å‡»[此处](#credentialsecret-示例)查看示例密文。如果è¿è¡Œ operator 的节点在 EC2 中，并且设置了å…许它们访问 S3 çš„ IAM æƒé™ï¼Œåˆ™æ­¤æŒ‡ä»¤æ˜¯ä¸å¿…è¦çš„（如[本节](#ec2-节点访问-s3-çš„-iam-æƒé™)所述）。凭è¯å¯†æ–‡ä¸‹æ‹‰èœå•åˆ—出了所有命å空间的密文。 +1. **存储桶å称**：存储备份文件的 S3 存储桶的å称。 +1. **区域**（å¯é€‰ï¼‰ï¼šS3 存储桶所在的 AWS [区域](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/)。é…ç½® MinIO æ—¶ä¸éœ€è¦è¯¥å­—段。 +1. **文件夹**（å¯é€‰ï¼‰ï¼šS3 存储桶中存储备份文件的文件夹å称。ä¸æ”¯æŒåµŒå¥—文件夹（例如， `rancher/cluster1`）。如果此字段留空，则默认将备份文件存储在 S3 存储桶的根文件夹中。 +1. **端点**：用于访问存储桶区域中的 S3 çš„[端点](https://docs.aws.amazon.com/general/latest/gr/s3.html)。 +1. **端点 CA**（å¯é€‰ï¼‰ï¼šBase64 ç¼–ç çš„ CA è¯ä¹¦ã€‚如需获å–示例，请å‚è§ [S3 兼容é…置示例](#s3-存储é…置示例)。 +1. **跳过 TLS 验è¯**（å¯é€‰ï¼‰ï¼šå¦‚果你ä¸ä½¿ç”¨ TLS，则设置为 `true`。 + + +| YAML 指令å称 | æè¿° | å¿…å¡« | +| ---------------- | ---------------- | ------------ | +| `credentialSecretName` | 如果你需è¦ä½¿ç”¨ AWS 访问密钥（access key）和密文密钥（secret key）æ¥è®¿é—® S3 存储桶，请使用带有密钥和指令 `accessKey` å’Œ `secretKey` 的凭è¯æ¥åˆ›å»ºå¯†æ–‡ã€‚它å¯ä»¥åœ¨ä»»ä½•å‘½å空间中，åªè¦ä½ åœ¨ `credentialSecretNamespace` 中æ供该命å空间。点击[这里](#credentialsecret-示例)查看示例密文。如果è¿è¡Œ operator 的节点在 EC2 中，并且设置了å…许它们访问 S3 çš„ IAM æƒé™ï¼Œåˆ™æ­¤æŒ‡ä»¤æ˜¯ä¸å¿…è¦çš„（如[本节](#ec2-节点访问-s3-çš„-iam-æƒé™)所述）。 | | +| `credentialSecretNamespace` | 包å«è®¿é—® S3 的凭è¯çš„密文的命å空间。如果è¿è¡Œ operator 的节点在 EC2 中，并且设置了å…许它们访问 S3 çš„ IAM æƒé™ï¼Œåˆ™æ­¤æŒ‡ä»¤æ˜¯ä¸å¿…è¦çš„（如[本节](#ec2-节点访问-s3-çš„-iam-æƒé™)所述）。 | | +| `bucketName` | 存储备份文件的 S3 存储桶的å称。 | ✓ | +| `folder` | S3 存储桶中存储备份文件的文件夹å称。ä¸æ”¯æŒåµŒå¥—文件夹（例如， `rancher/cluster1`）。如果此字段留空，则默认将备份文件存储在 S3 存储桶的根文件夹中。 | | +| `region` | S3 存储桶所在的 AWS [区域](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/)。 | ✓ | +| `endpoint` | 用于访问存储桶区域中的 S3 çš„[端点](https://docs.aws.amazon.com/general/latest/gr/s3.html)。 | ✓ | +| `endpointCA` | Base64 ç¼–ç çš„ CA è¯ä¹¦ã€‚如需获å–示例，请å‚è§ [S3 兼容é…置示例](#s3-存储é…置示例)。 | | +| `insecureTLSSkipVerify` | 如果你ä¸ä½¿ç”¨ TLS，则设置为 `true`。 | | + +### S3 存储é…置示例 + +```yaml +s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### MinIO é…置示例 + +```yaml +s3: + credentialSecretName: minio-creds + bucketName: rancherbackups + endpoint: minio.35.202.130.254.xip.io + endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURHakNDQWdLZ0F3SUJBZ0lKQUtpWFZpNEpBb0J5TUEwR0NTcUdTSWIzRFFFQkN3VUFNQkl4RURBT0JnTlYKQkFNTUIzUmxjM1F0WTJFd0hoY05NakF3T0RNd01UZ3lOVFE1V2hjTk1qQXhNREk1TVRneU5UUTVXakFTTVJBdwpEZ1lEVlFRRERBZDBaWE4wTFdOaE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBCjA4dnV3Q2Y0SEhtR2Q2azVNTmozRW5NOG00T2RpS3czSGszd1NlOUlXQkwyVzY5WDZxenBhN2I2M3U2L05mMnkKSnZWNDVqeXplRFB6bFJycjlpbEpWaVZ1NFNqWlFjdG9jWmFCaVNsL0xDbEFDdkFaUlYvKzN0TFVTZSs1ZDY0QQpWcUhDQlZObU5xM3E3aVY0TE1aSVpRc3N6K0FxaU1Sd0pOMVVKQTZ6V0tUc2Yzc3ByQ0J2dWxJWmZsVXVETVAyCnRCTCt6cXZEc0pDdWlhNEEvU2JNT29tVmM2WnNtTGkwMjdub3dGRld3MnRpSkM5d0xMRE14NnJoVHQ4a3VvVHYKQXJpUjB4WktiRU45L1Uzb011eUVKbHZyck9YS2ZuUDUwbk8ycGNaQnZCb3pUTStYZnRvQ1d5UnhKUmI5cFNTRApKQjlmUEFtLzNZcFpMMGRKY2sxR1h3SURBUUFCbzNNd2NUQWRCZ05WSFE0RUZnUVU5NHU4WXlMdmE2MTJnT1pyCm44QnlFQ2NucVFjd1FnWURWUjBqQkRzd09ZQVU5NHU4WXlMdmE2MTJnT1pybjhCeUVDY25xUWVoRnFRVU1CSXgKRURBT0JnTlZCQU1NQjNSbGMzUXRZMkdDQ1FDb2wxWXVDUUtBY2pBTUJnTlZIUk1FQlRBREFRSC9NQTBHQ1NxRwpTSWIzRFFFQkN3VUFBNElCQVFER1JRZ1RtdzdVNXRQRHA5Q2psOXlLRW9Vd2pYWWM2UlAwdm1GSHpubXJ3dUVLCjFrTkVJNzhBTUw1MEpuS29CY0ljVDNEeGQ3TGdIbTNCRE5mVVh2anArNnZqaXhJYXR2UWhsSFNVaWIyZjJsSTkKVEMxNzVyNCtROFkzelc1RlFXSDdLK08vY3pJTGh5ei93aHRDUlFkQ29lS1dXZkFiby8wd0VSejZzNkhkVFJzNwpHcWlGNWZtWGp6S0lOcTBjMHRyZ0xtalNKd1hwSnU0ZnNGOEcyZUh4b2pOKzdJQ1FuSkg5cGRIRVpUQUtOL2ppCnIvem04RlZtd1kvdTBndEZneWVQY1ZWbXBqRm03Y0ZOSkc4Y2ZYd0QzcEFwVjhVOGNocTZGeFBHTkVvWFZnclMKY1VRMklaU0RJd1FFY3FvSzFKSGdCUWw2RXBaUVpWMW1DRklrdFBwSQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t +``` +### credentialSecret 示例 + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +:::note + +为é¿å…ç¼–ç é—®é¢˜ï¼Œä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤åˆ›å»º `credentialSecret`，更新 `accessKey` å’Œ `secretKey` 的值。 + +```bash +kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= +``` + +::: + +### EC2 节点访问 S3 çš„ IAM æƒé™ + +有两ç§æ–¹æ³•å¯ä»¥è®¾ç½® `rancher-backup` operator æ¥ä½¿ç”¨ S3 作为备份存储ä½ç½®ã€‚ + +一ç§æ–¹æ³•æ˜¯åœ¨ Backup 自定义资æºä¸­é…ç½® `credentialSecretName`（å³å¯ä»¥è®¿é—® S3 çš„ AWS 凭è¯ï¼‰ã€‚ + +如果集群节点在 Amazon EC2 中，你也å¯ä»¥ç»™ EC2 èŠ‚ç‚¹åˆ†é… IAM æƒé™ï¼Œä½¿å…¶å¯ä»¥è®¿é—® S3。 + +è¦å…许节点访问 S3，请按照 [AWS 文档](https://aws.amazon.com/premiumsupport/knowledge-center/ec2-instance-access-s3-bucket/)中的说明，为 EC2 创建一个 IAM 角色。在å‘角色添加自定义策略时，添加以下æƒé™ï¼Œå¹¶å°† `Resource` 替æ¢ä¸ºä½ çš„存储桶å称： + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups/*" + ] + } + ] +} +``` + +在创建角色并将相应的实例é…置文件附加 EC2 实例åŽï¼ŒBackup 自定义资æºä¸­çš„ `credentialSecretName` 指令å¯ä»¥ç•™ç©ºã€‚ + +## 示例 + +å¦‚éœ€èŽ·å– Backup 自定义资æºçš„示例，请å‚è§[本页](examples.md#备份)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/examples.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/examples.md new file mode 100644 index 00000000000..ca1ce934d84 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/examples.md @@ -0,0 +1,293 @@ +--- +title: 示例 +--- + +æœ¬èŠ‚åŒ…å« Backup å’Œ Restore 自定义资æºçš„示例。 + +默认的备份存储ä½ç½®æ˜¯åœ¨å®‰è£…或å‡çº§ `rancher-backup` operator æ—¶é…置的。 + +åªæœ‰ Restore 自定义资æºä½¿ç”¨åˆ›å»ºå¤‡ä»½æ—¶ä½¿ç”¨çš„加密é…置密文时，æ‰èƒ½è¿˜åŽŸåŠ å¯†çš„备份。 + +## 备份 + +æœ¬èŠ‚åŒ…å« Backup 自定义资æºçš„示例。 + +> **注æ„**：有关é…置以下选项的更多信æ¯ï¼Œè¯·å‚阅[备份é…ç½®å‚考页é¢](./backup-configuration.md)。 + +### 在默认ä½ç½®è¿›è¡ŒåŠ å¯†å¤‡ä»½ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-encrypted-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### 在默认ä½ç½®è¿›è¡Œå®šæœŸå¤‡ä»½ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-recurring-backup +spec: + resourceSetName: rancher-resource-set + schedule: "@every 1h" + retentionCount: 10 +``` + +### 在默认ä½ç½®è¿›è¡ŒåŠ å¯†çš„定期备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-enc-recurring-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 3 +``` + +### Minio 中的加密备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: minio-backup +spec: + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### 使用 AWS 凭è¯å¯†æ–‡åœ¨ S3 中备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### 使用 AWS 凭è¯å¯†æ–‡åœ¨ S3 中进行定期备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-recurring-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 +``` + +### 从具有访问 S3 çš„ IAM æƒé™çš„ EC2 节点进行备份 + +这个例å­è¡¨æ˜Žï¼Œå¦‚æžœè¿è¡Œ `rancher-backup` 的节点拥有这些[访问 S3 çš„æƒé™](backup-configuration.md#ec2-节点访问-s3-çš„-iam-æƒé™)，就ä¸å¿…æä¾› AWS 的凭è¯å¯†æ–‡æ¥åˆ›å»ºå¤‡ä»½ã€‚ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-iam-backup +spec: + storageLocation: + s3: + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +## 还原 + +æœ¬èŠ‚åŒ…å« Restore 自定义资æºçš„示例。 + +> **注æ„**：有关é…置以下选项的更多信æ¯ï¼Œè¯·å‚阅[æ¢å¤é…ç½®å‚考页é¢](./restore-configuration.md)。 + +### 使用默认备份文件ä½ç½®è¿˜åŽŸ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-default +spec: + backupFilename: default-location-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-29-54-07-00.tar.gz +# encryptionConfigSecretName: test-encryptionconfig +``` + +### 为 Rancher è¿ç§»è¿›è¡Œè¿˜åŽŸ +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-migration +spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + prune: false + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### 使用加密的备份还原 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-encrypted +spec: + backupFilename: default-test-s3-def-backup-c583d8f2-6daf-4648-8ead-ed826c591471-2020-08-24T20-47-05Z.tar.gz + encryptionConfigSecretName: encryptionconfig +``` + +### 从 Minio 还原加密的备份 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-minio +spec: + backupFilename: default-minio-backup-demo-aa5c04b7-4dba-4c48-9ac4-ab7916812eaa-2020-08-30T13-18-17-07-00.tar.gz + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + encryptionConfigSecretName: test-encryptionconfig +``` + +### 使用 AWS 凭è¯å¯†æ–‡è®¿é—® S3 从备份中还原 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: test-s3-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-49-34-07-00.tar.gz.enc + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +### 从具有访问 S3 çš„ IAM æƒé™çš„ EC2 节点进行还原 + +这个例å­è¡¨æ˜Žï¼Œå¦‚æžœè¿è¡Œ `rancher-backup` 的节点拥有这些[访问 S3 çš„æƒé™](backup-configuration.md#ec2-节点访问-s3-çš„-iam-æƒé™)，就ä¸å¿…æä¾› AWS 的凭è¯å¯†æ–‡æ¥ä»Žå¤‡ä»½ä¸­è¿˜åŽŸã€‚ + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: default-test-s3-recurring-backup-84bf8dd8-0ef3-4240-8ad1-fc7ec308e216-2020-08-24T10#52#44-07#00.tar.gz + storageLocation: + s3: + bucketName: rajashree-backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +## 在 S3 中存储备份的凭è¯å¯†æ–‡ç¤ºä¾‹ + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +## EncryptionConfiguration 示例 + +以下代ç ç‰‡æ®µæ¼”示了两ç§ä¸åŒç±»åž‹çš„密文åŠå…¶ä¸Žè‡ªå®šä¹‰èµ„æºçš„备份和还原的相关性。 + +第一个示例是用于加密备份文件的密钥。在这ç§æƒ…况下，Backup operator 将无法读å–密文加密文件。它åªä½¿ç”¨å¯†æ–‡çš„内容。 + +第二个示例是 Kubernetes 密文加密é…置文件，用于加密存储在 etcd 中的密文。**备份 etcd æ•°æ®å­˜å‚¨æ—¶ï¼Œè¯·åŠ¡å¿…åŒæ—¶å¤‡ä»½ EncryptionConfiguration**。如果你没有这样åšï¼Œè€Œä¸”备份数æ®æ—¶æ­£åœ¨ä½¿ç”¨å¯†æ–‡åŠ å¯†ï¼Œä½ å°†æ— æ³•ä½¿ç”¨æ¢å¤çš„æ•°æ®ã€‚ + + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: + - secrets + providers: + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - secretbox: + keys: + - name: key1 + secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY= +``` + + + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/restore-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/restore-configuration.md new file mode 100644 index 00000000000..5931f1f82d8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/restore-configuration.md @@ -0,0 +1,81 @@ +--- +title: 还原é…ç½® +--- + +ä½ å¯ä»¥åœ¨è¿˜åŽŸåˆ›å»ºé¡µé¢æ供还原备份的详细信æ¯ã€‚ + +![](/img/backup_restore/restore/restore.png) + + +## å¤‡ä»½æº +你需è¦æ供备份文件和备份文件存储ä½ç½®çš„详细信æ¯ï¼Œoperator 会使用这些信æ¯æ‰§è¡Œè¿˜åŽŸã€‚选择以下其中一个选项æ¥æ供这些详细信æ¯ã€‚ + + + + +### 使用已有的备份é…ç½®æ¢å¤ + +如果你选择这个选项，**目标备份**下拉èœå•ä¸­ä¼šå‡ºçŽ°é›†ç¾¤ä¸­å¯ç”¨çš„备份。从下拉èœå•ä¸­é€‰æ‹© Backup，选择åŽ**备份文件å**字段会自动填写，而且所选 Backup çš„ä¿¡æ¯ä¼šä¼ é€’ç»™ operator + +![](/img/backup_restore/restore/existing.png) + +如果 Backup 自定义资æºåœ¨é›†ç¾¤ä¸­ä¸å­˜åœ¨ï¼Œä½ éœ€è¦èŽ·å–准确的文件å，并使用默认存储目标或与 S3 兼容的对象存储æ¥æ供备份æºçš„详细信æ¯ã€‚ + + +### 使用默认存储目标æ¢å¤ + +如果你è¦ä»Ž operator 级别é…置的默认存储ä½ç½®ä¸­çš„备份文件进行æ¢å¤ï¼Œè¯·é€‰æ‹©æ­¤é€‰é¡¹ã€‚operator 级别的é…置是指安装或å‡çº§ `rancher-backup` operator æ—¶é…置的存储ä½ç½®ã€‚在**备份文件å**字段中æ供准确的文件å。 + +![](/img/backup_restore/restore/default.png) + +### 使用与 S3 兼容的对象存储æ¢å¤ + +如果在 operator 级别没有é…置默认存储ä½ç½®ï¼Œæˆ–者备份文件所在的存储桶与é…置的默认存储ä½ç½®ä¸ä¸€æ ·æ—¶ï¼Œè¯·é€‰æ‹©æ­¤é€‰é¡¹ã€‚在**备份文件å**字段中æ供准确的文件å。请å‚è§[本节](#从-s3-获å–备份文件å)了解从 S3 获å–备份文件å的具体步骤。填写 S3 兼容对象存储的所有详细信æ¯ã€‚它的字段与 [Backup 自定义资æº](backup-configuration.md#存储ä½ç½®)中 `backup.StorageLocation` é…置的字段一样。 + +![](/img/backup_restore/restore/s3store.png) + +## 加密 + +如果备份是在å¯ç”¨åŠ å¯†çš„情况下创建的，备份文件的åŽç¼€ä¸º `.enc`。如果你选择这类的 Backup，或者æä¾›åŽç¼€ä¸º `.enc` 的备份文件å，则会显示å¦ä¸€ä¸ªå为 **Encryption Config Secret** 的下拉èœå•ã€‚ + +![](/img/backup_restore/restore/encryption.png) + +从该下拉èœå•ä¸­é€‰æ‹©çš„密文必须与执行备份时用于 Backup 自定义资æºçš„密文内容相åŒã€‚如果加密é…ç½®ä¸åŒ¹é…，还原将会失败。 + +`Encryption Config Secret` 下拉èœå•å°†è¿‡æ»¤å¹¶ä»…列出拥有这个 key 的密文。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | æä¾› `cattle-resources-system` 命å空间中包å«åŠ å¯†é…置文件的密文的å称。 | + +:::note é‡è¦æ示： + +此字段仅在备份创建的过程中å¯ç”¨äº†åŠ å¯†åŠŸèƒ½æ—¶æ‰éœ€è¦è®¾ç½®ã€‚æ供错误的加密é…置将导致还原失败。 + +::: + +## 还原过程中修剪 + +* **Prune**：为了从备份中完全æ¢å¤ Rancher，并回到备份时的确切状æ€ï¼Œæˆ‘们需è¦åˆ é™¤ Rancher 在备份åŽåˆ›å»ºçš„所有é¢å¤–资æºã€‚如果 **Prune** 标志被å¯ç”¨ï¼Œoperator 就会删除这些资æºã€‚Prune 是默认å¯ç”¨çš„，建议ä¿æŒå¯ç”¨çŠ¶æ€ã€‚ +* **删除超时**：在删除资æºçš„时候，operator 编辑资æºä»¥åˆ é™¤ Finalizers，并试图å†æ¬¡åˆ é™¤å‰å°†ç­‰å¾…的时间。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `prune` | 删除备份中ä¸å­˜åœ¨çš„ç”± Rancher 管ç†çš„资æºï¼ˆæŽ¨è）。 | +| `deleteTimeoutSeconds` | 在删除资æºçš„时候，operator 编辑资æºä»¥åˆ é™¤ Finalizers，并试图å†æ¬¡åˆ é™¤å‰å°†ç­‰å¾…的时间。 | + +## 从 S3 获å–备份文件å + +这是 `rancher-backup` operator 用æ¥æ‰§è¡Œè¿˜åŽŸçš„备份文件的å称。 + +è¦ä»Ž S3 获å–这个文件å，请进入你的 S3 存储桶（或在执行备份时指定的文件夹）。 + +å¤åˆ¶æ–‡ä»¶å并将其存储在你的 Restore 自定义资æºä¸­ã€‚å‡è®¾ä½ çš„备份文件的å字是 `backupfile`： + +- 如果你的存储桶å称是 `s3bucket`，而且你没有指定文件夹，那么è¦ä½¿ç”¨çš„ `backupFilename` 会是 `backupfile`。 +- 如果你的存储桶å称是 `s3bucket`，而且基础文件夹是 `s3folder`，那么è¦ä½¿ç”¨çš„ `backupFilename` 会是 `backupfile`。 +- 如果 `s3Folder` 中有一个å为 `s3sub` çš„å­æ–‡ä»¶å¤¹ï¼Œè€Œä¸”你的备份文件存放在该文件夹中，那么è¦ä½¿ç”¨çš„`backupFilename` 会是 `s3sub/backupfile`。 + +| YAML 指令å称 | æè¿° | +| ---------------- | ---------------- | +| `backupFilename` | 这是 `rancher-backup` operator 用æ¥æ‰§è¡Œè¿˜åŽŸçš„备份文件的å称。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/storage-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/storage-configuration.md new file mode 100644 index 00000000000..2db4ed74ef5 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/backup-restore-configuration/storage-configuration.md @@ -0,0 +1,56 @@ +--- +title: 备份存储ä½ç½®é…ç½® +--- + +é…置一个用于ä¿å­˜æ‰€æœ‰å¤‡ä»½çš„默认存储ä½ç½®ã€‚ä½ å¯ä»¥é€‰æ‹©å¯¹æ¯ä¸ªå¤‡ä»½è¿›è¡Œè¦†ç›–，但仅é™äºŽä½¿ç”¨ S3 对象存储。 + +在 operator 级别åªèƒ½é…置一个存储ä½ç½®ã€‚ + + +## 存储ä½ç½®é…ç½® + +### 无默认存储ä½ç½® + +ä½ å¯ä»¥é€‰æ‹©ä¸é…ç½® operator 级别的存储ä½ç½®ã€‚如果选择此选项，你必须é…置一个与 S3 兼容的对象存储作为æ¯ä¸ªå¤‡ä»½çš„存储ä½ç½®ã€‚ + +### S3 兼容的对象存储 + +| å‚æ•° | æè¿° | +| -------------- | -------------- | +| 凭è¯å¯†æ–‡ | 从 Rancher 的密文中选择 S3 的凭è¯ã€‚[示例](examples.md#在-s3-中存储备份的凭è¯å¯†æ–‡ç¤ºä¾‹)。 | +| 存储桶å称 | 存储备份的 [S3 存储桶](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html)çš„å称。默认值：`rancherbackups`。 | +| 区域 | S3 存储桶所在的 [AWS 区域](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/)。 | +| 文件夹 | 存储备份的 [S3 存储桶中的文件夹](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html)。如果此字段留空，则默认将备份文件存储在 S3 存储桶的根文件夹中。 | +| 端点 | [S3 端点](https://docs.aws.amazon.com/general/latest/gr/s3.html)，例如 `s3.us-west-2.amazonaws.com`。 | +| Endpoint CA | 用于 S3 端点的 CA è¯ä¹¦ã€‚默认值：base64 ç¼–ç çš„ CA è¯ä¹¦ã€‚ | +| insecureTLSSkipVerify | 如果你ä¸ä½¿ç”¨ TLS，则设置为 `true`。 | + +### 使用现有的 StorageClass + +如果你通过选择 StorageClass 选项æ¥å®‰è£… `rancher-backup` Chart，将会创建一个æŒä¹…å·è¯´æ˜Žï¼ˆPersistent Volume Claim，PVC），而且 Kubernetes 会动æ€é…置一个æŒä¹…å·ï¼ˆPersistent Volume，PV），所有备份都会默认ä¿å­˜åˆ°è¯¥æŒä¹…å·ä¸­ã€‚ + +关于创建存储类的信æ¯ï¼Œè¯·å‚è§[本章节](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md)。 + +:::note é‡è¦æ示： + +强烈建议使用回收策略为 "Retain" çš„ StorageClass。å¦åˆ™ï¼Œå¦‚æžœ `rancher-backup` Chart 创建的 PVC 在应用å‡çº§æœŸé—´æˆ–æ„外被删除åŽï¼ŒPV 也会被删除，也就是说所有ä¿å­˜åœ¨å…¶ä¸­çš„备份都会被删除。 +如果没有这样的 StorageClass å¯ç”¨ï¼Œåœ¨é…ç½® PV åŽï¼Œè¯·ç¡®ä¿å…ˆå°†å…¶å›žæ”¶ç­–略设置为 "Retain"，然åŽå†å­˜å‚¨å¤‡ä»½ã€‚ + +::: + +### 使用现有的æŒä¹…å· + +选择一个用于存储备份的现有æŒä¹…å·ã€‚有关在 Rancher 中创建 PersistentVolumes 的更多信æ¯ï¼Œè¯·å‚è§[本节](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md#2-添加一个引用æŒä¹…存储的-persistentvolume)。 + +:::note é‡è¦æ示： + +强烈建议使用回收策略为 "Retain" çš„ Persistent Volume。å¦åˆ™ï¼Œå¦‚æžœ `rancher-backup` Chart 创建的 PVC 在应用å‡çº§æœŸé—´æˆ–æ„外被删除åŽï¼ŒPV 也会被删除，也就是说所有ä¿å­˜åœ¨å…¶ä¸­çš„备份都会被删除。 + +::: + +## rancher-backup Helm Chart 的示例 values.yaml + +使用 Helm CLI 时，å¯ç”¨äºŽé…ç½® `rancher-backup` operator çš„ `values.yaml` å¯ä»¥åœ¨ [backup-restore-operator](https://github.com/rancher/backup-restore-operator/blob/master/charts/rancher-backup/values.yaml) 仓库中找到。 + +有关 `values.yaml` 文件和在安装时é…ç½® Helm Charts 的更多信æ¯ï¼Œè¯·å‚è§ [Helm 文档](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing)。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md new file mode 100644 index 00000000000..afc1247854f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -0,0 +1,88 @@ +--- +title: Logging 最佳实践 +--- + +本指å—介ç»äº†é›†ç¾¤çº§åˆ«æ—¥å¿—和应用日志的最佳实践。 + +- [集群级别日志](#集群级别日志) +- [应用日志](#应用日志) +- [通用最佳实践](#通用最佳实践) + +在 Rancher 2.5 之å‰ï¼ŒRancher çš„ Logging 是一个é™æ€é›†æˆã€‚å¯ä¾›é€‰æ‹©çš„èšåˆå™¨æ˜¯å›ºå®šçš„（包括 ElasticSearchã€Splunkã€Kafkaã€Fluentd å’Œ Syslog），而且åªæœ‰ä¸¤ä¸ªé…置级别å¯ä¾›é€‰æ‹©ï¼ˆé›†ç¾¤çº§åˆ«å’Œé¡¹ç›®çº§åˆ«ï¼‰ã€‚ + +现在，Rancher 的日志èšåˆæ›´åŠ çµæ´»ã€‚通过新的 Logging 的功能，管ç†å‘˜å’Œç”¨æˆ·éƒ½å¯ä»¥éƒ¨ç½²ç¬¦åˆç»†ç²’度收集标准的日志记录，åŒæ—¶æ供更多的目标和é…置选项。 + +Rancher Logging 使用的是 [Logging Operator](https://github.com/kube-logging/logging-operator)。我们让你å¯ä»¥ç®¡ç†è¿™ä¸ª operator åŠå…¶èµ„æºï¼Œå¹¶å°†å®ƒçš„管ç†åŠŸèƒ½å’Œ Rancher 集群管ç†è”系起æ¥ã€‚ + +## 集群级别日志 + +### 抓å–集群内日志 + +æŸäº›ç”¨æˆ·å¯èƒ½æƒ³ä»Žé›†ç¾¤ä¸­è¿è¡Œçš„æ¯ä¸ªå®¹å™¨ä¸­æŠ“å–日志。但是你的安全团队å¯èƒ½è¦æ±‚你从所有执行点收集所有日志。 + +在这ç§æƒ…况下，我们建议你至少创建两个 _ClusterOutput_ 对象 - 一个用于安全团队（如果需è¦ï¼‰ï¼Œå¦ä¸€ä¸ªç”¨äºŽä½ è‡ªå·±ï¼Œå³é›†ç¾¤ç®¡ç†å‘˜ã€‚创建这些对象时，请选择一个å¯ä»¥å¤„ç†æ•´ä¸ªé›†ç¾¤çš„大é‡æ—¥å¿—的输出端点。此外，你还需è¦é€‰æ‹©åˆé€‚的索引æ¥æŽ¥æ”¶è¿™äº›æ—¥å¿—。 + +创建这些 _ClusterOutput_ 对象åŽï¼Œåˆ›å»ºä¸€ä¸ª _ClusterFlow_ æ¥æ”¶é›†æ‰€æœ‰æ—¥å¿—。ä¸è¦åœ¨æ­¤ flow 上定义任何 _Include_ 或 _Exclude_ 规则。这å¯ä»¥ç¡®ä¿ä½ èƒ½æ”¶é›†æ•´ä¸ªé›†ç¾¤çš„所有日志。如果你有两个 _ClusterOutputs_，请确ä¿å®ƒä»¬éƒ½èƒ½æ”¶åˆ°æ—¥å¿—。 + +### Kubernetes 组件 + +_ClusterFlows_ 能够收集 Kubernetes 集群中所有主机上所有容器的日志。如果这些容器包å«åœ¨ Kubernetes Pod 中，这个方法是适用的。但是，RKE 容器ä¸å­˜åœ¨äºŽ Kubernetes 内。 + +ç›®å‰ï¼ŒRancher 能æœé›† RKE 容器的日志，但ä¸èƒ½è½»æ˜“过滤。这是因为这些日志ä¸åŒ…å«æºå®¹å™¨çš„ä¿¡æ¯ï¼ˆä¾‹å¦‚ `etcd` 或 `kube-apiserver`）。 + +Rancher 的未æ¥ç‰ˆæœ¬å°†åŒ…å«æºå®¹å™¨å称，æ¥æ”¯æŒè¿‡æ»¤è¿™äº›ç»„件的日志。该功能实现之åŽï¼Œä½ å°†èƒ½å¤Ÿè‡ªå®šä¹‰ _ClusterFlow_ æ¥**ä»…**检索 Kubernetes 组件日志，并将日志å‘é€åˆ°é€‚当的输出ä½ç½®ã€‚ + +## 应用日志 + +对于 Kubernetes 和所有基于容器的应用而言，最佳实践是将应用日志引导到 `stdout`/`stderr`。容器è¿è¡Œæ—¶å°†æ•èŽ·è¿™äº›æ—¥å¿—并用它们进行**æŸäº›æ“作** - 通常是将它们写入文件。根æ®å®¹å™¨è¿è¡Œæ—¶ï¼ˆåŠå…¶é…置），这些日志å¯ä»¥æ”¾ç½®åœ¨ä»»æ„æ•°é‡çš„ä½ç½®ã€‚ + +在将日志写入文件的情况下，Kubernetes 通过在æ¯ä¸ªä¸»æœºä¸Šåˆ›å»ºä¸€ä¸ª `/var/log/containers` 目录æ¥æ供帮助。这个目录将日志文件符å·é“¾æŽ¥åˆ°å®ƒä»¬çš„实际目的地（å¯èƒ½å› ä¸ºé…置或容器è¿è¡Œæ—¶è€Œæœ‰æ‰€ä¸åŒï¼‰ã€‚ + +Rancher çš„ Logging å°†è¯»å– `/var/log/containers` 中的所有日志æ¡ç›®ï¼Œç¡®ä¿æ‰€æœ‰å®¹å™¨çš„所有日志æ¡ç›®ï¼ˆå‡è®¾ä½¿ç”¨é»˜è®¤é…置）å‡èƒ½è¢«æ”¶é›†å’Œå¤„ç†ã€‚ + +### 特定日志文件 + +日志收集仅从 Kubernetes 中的 Pod 中检索 `stdout`/`stderr` 日志。但是，我们也å¯èƒ½æƒ³ä»Žåº”用生æˆçš„其他文件中收集日志。在这ç§æƒ…况下，你å¯ä»¥ä½¿ç”¨ä¸€ä¸ªï¼ˆæˆ–ä¸¤ä¸ªï¼‰æ—¥å¿—æµ Sidecar。 + +è®¾ç½®æ—¥å¿—æµ Sidecar 的目的是获å–写入ç£ç›˜çš„日志文件，并将其内容传输到 `stdout`。这样一æ¥ï¼ŒLogging Operator å°±å¯ä»¥æŽ¥æ”¶è¿™äº›æ—¥å¿—，并把日志å‘é€åˆ°ç›®æ ‡è¾“出ä½ç½®ã€‚ + +è¦è¿›è¡Œè®¾ç½®ï¼Œç¼–辑你的工作负载资æºï¼ˆä¾‹å¦‚ Deployment）并添加以下 Sidecar 定义： + +```yaml +... +containers: +- args: + - -F + - /path/to/your/log/file.log + command: + - tail + image: busybox + name: stream-log-file-[name] + volumeMounts: + - mountPath: /path/to/your/log + name: mounted-log +... +``` + +这将添加一个容器到你的工作负载定义中，用于将 `/path/to/your/log/file.log` 的内容（在本示例中）传输到 `stdout`。 + +然åŽå°†æ ¹æ®ä½ è®¾ç½®çš„ _Flows_ 或 _ClusterFlows_ 自动收集该日志æµã€‚你还å¯ä»¥é€šè¿‡ä½¿ç”¨å®¹å™¨çš„å称，专门为该日志文件创建一个 _Flow_。示例如下： + +```yaml +... +spec: + match: + - select: + container_names: + - stream-log-file-name +... +``` + + +## 通用最佳实践 + +- å°½é‡è¾“出结构化日志æ¡ç›®ï¼ˆä¾‹å¦‚ `syslog`ã€JSON）。这些格å¼å·²ç»æœ‰äº†è§£æžå™¨ï¼Œå› æ­¤ä½ å¯ä»¥æ›´è½»æ¾åœ°å¤„ç†æ—¥å¿—æ¡ç›®ã€‚ +- å°½é‡åœ¨æ—¥å¿—æ¡ç›®å†…æ供创建该日志æ¡ç›®çš„应用的å称。这å¯ä»¥ä½¿æ•…障排除更容易。这是因为 Kubernetes 并ä¸æ€»æ˜¯å°†åº”用的å称作为对象å称。例如，æŸä¸ª Pod ID å¯èƒ½æ˜¯ `myapp-098kjhsdf098sdf98`，从这个 ID 中我们ä¸èƒ½èŽ·å–è¿è¡Œåœ¨å®¹å™¨å†…的应用的太多信æ¯ã€‚ +- 除了在集群范围内收集所有日志的情况外，尽é‡ä¸¥æ ¼é™å®š _Flow_ å’Œ _ClusterFlow_ 对象的范围。这使得在出现问题时更容易进行故障排除，并且还有助于确ä¿ä¸ç›¸å…³çš„日志æ¡ç›®ä¸ä¼šå‡ºçŽ°åœ¨ä½ çš„èšåˆå™¨ä¸­ã€‚严格é™å®šèŒƒå›´çš„一个例å­æ˜¯å°† _Flow_ é™åˆ¶åœ¨å‘½å空间中的å•ä¸ª _Deployment_，甚至是 _Pod_ 中的å•ä¸ªå®¹å™¨ã€‚ +- 除éžè¦è¿›è¡Œæ•…障排除，å¦åˆ™ä¸è¦è®©æ—¥å¿—太详细。太详细的日志会带æ¥è®¸å¤šé—®é¢˜ï¼Œå…¶ä¸­æœ€ä¸»è¦çš„是**带æ¥å¹²æ‰°**，å³é‡è¦äº‹ä»¶å¯èƒ½ä¼šæ·¹æ²¡åœ¨æµ·é‡ `DEBUG` ä¿¡æ¯ä¸­ã€‚ä½ å¯ä»¥é€šè¿‡ä½¿ç”¨è‡ªåŠ¨å‘Šè­¦å’Œè„šæœ¬æ¥ç¼“解这ç§é—®é¢˜ï¼Œä½†å¤ªè¯¦ç»†çš„日志ä»ç„¶ç»™æ—¥å¿—管ç†åŸºç¡€è®¾æ–½å¸¦æ¥è¿‡å¤§çš„压力。 +- 如果å¯èƒ½ï¼Œå°½é‡åœ¨æ—¥å¿—æ¡ç›®ä¸­æ供事务或请求 ID。这å¯ä»¥ä½¿è·¨æ—¥å¿—æºè¿½è¸ªåº”用活动å˜å¾—更容易，尤其是在处ç†åˆ†å¸ƒå¼åº”用时。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md new file mode 100644 index 00000000000..d880b03bd0b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -0,0 +1,111 @@ +--- +title: 监控最佳实践 +--- + +é…ç½®åˆç†çš„监控和告警规则对于安全ã€å¯é åœ°è¿è¡Œç”Ÿäº§çŽ¯å¢ƒä¸­çš„工作负载至关é‡è¦ã€‚在使用 Kubernetes å’Œ Rancher 时也是如此。幸è¿çš„是，你å¯ä»¥ä½¿ç”¨é›†æˆçš„监控和告警功能æ¥ç®€åŒ–整个过程。 + +[Rancher 监控文档](../../../pages-for-subheaders/monitoring-and-alerting.md)æ述了如何设置完整的 Prometheus å’Œ Grafana。这是开箱å³ç”¨çš„功能，它将从集群中的所有系统和 Kubernetes 组件中抓å–监控数æ®ï¼Œå¹¶æä¾›åˆç†çš„仪表æ¿å’Œå‘Šè­¦ã€‚但为了实现å¯é çš„设置，你还需è¦ç›‘控你的工作负载并使 Prometheus å’Œ Grafana 适应你的特定用例和集群规模。本文档将为你æ供这方é¢çš„最佳实践。 + +## 监控内容 + +Kubernetes 本身以åŠè¿è¡Œåœ¨å…¶å†…部的应用构æˆäº†ä¸€ä¸ªåˆ†å¸ƒå¼ç³»ç»Ÿï¼Œä¸åŒçš„组件之间能够进行交互。对于整个系统和æ¯ä¸ªå•ç‹¬çš„组件，你必须确ä¿å…¶æ€§èƒ½ã€å¯ç”¨æ€§ã€å¯é æ€§å’Œå¯æ‰©å±•æ€§ã€‚详情请å‚è§ Google å…费的 [Site Reliability Engineering Book](https://sre.google/sre-book/table-of-contents/)，尤其是关于 [Monitoring distributed systems](https://sre.google/sre-book/monitoring-distributed-systems/) 的章节。 + +## é…ç½® Prometheus 资æºä½¿ç”¨ + +在安装集æˆç›‘控时，Rancher å…许进行一些é…置，这些é…ç½®å–决于你的集群规模和其中è¿è¡Œçš„工作负载。本章将更详细地介ç»è¿™äº›å†…容。 + +### 存储和数æ®ä¿ç•™ + +Prometheus 所需的存储空间å–决于你存储的时间åºåˆ—和标签的数é‡ï¼Œä»¥åŠä½ é…置的数æ®ä¿ç•™é‡ã€‚需è¦æ³¨æ„的是，Prometheus 并ä¸æ˜¯ç”¨æ¥é•¿æœŸå­˜å‚¨æŒ‡æ ‡çš„。数æ®é€šå¸¸åªä¿ç•™å‡ å¤©ï¼Œè€Œä¸æ˜¯å‡ å‘¨æˆ–几个月。这是因为 Prometheus ä¸ä¼šå¯¹å­˜å‚¨çš„指标进行任何èšåˆã€‚ä¸èšåˆçš„好处是é¿å…稀释数æ®ï¼Œä½†è¿™ä¹Ÿæ„味ç€ä¸è®¾ç½®ä¿ç•™æ—¶é•¿çš„è¯ï¼Œæ‰€éœ€çš„存储空间éšç€æ—¶é—´çš„推移线性增长。 + +以下是计算所需存储空间的一ç§æ–¹æ³•ã€‚首先，通过以下查询æ¥æŸ¥çœ‹ Prometheus 中存储å—çš„å¹³å‡å¤§å°ï¼š + +``` +rate(prometheus_tsdb_compaction_chunk_size_bytes_sum[1h]) / rate(prometheus_tsdb_compaction_chunk_samples_sum[1h]) +``` + +接下æ¥ï¼Œæ‰¾å‡ºæ¯ç§’çš„æ•°æ®å¼•å…¥é€ŸçŽ‡ï¼š + +``` +rate(prometheus_tsdb_head_samples_appended_total[1h]) +``` + +然åŽä¸Žä¿ç•™æ—¶é—´ç›¸ä¹˜ï¼Œå¹¶æ·»åŠ å‡ ä¸ªç™¾åˆ†ç‚¹ä½œä¸ºç¼“冲区： + +``` +å¹³å‡å—大å°ï¼ˆä»¥å­—节为å•ä½ï¼‰x æ¯ç§’çš„æ•°æ®å¼•å…¥é€ŸçŽ‡ x ä¿ç•™æ—¶é—´ï¼ˆä»¥ç§’为å•ä½ï¼‰x 1.1 = 必需的存储空间（以字节为å•ä½ï¼‰ +``` + +ä½ å¯ä»¥è®¿é—®[这篇åšå®¢](https://www.robustperception.io/how-much-disk-space-do-prometheus-blocks-use)了解关于如何计算必须的存储空间的更多信æ¯ã€‚ + +ä½ å¯ä»¥å‚è§ [Prometheus 官方文档](https://prometheus.io/docs/prometheus/latest/storage)æ¥è¿›ä¸€æ­¥äº†è§£ Prometheus 的存储概念。 + +### CPU 和内存的请求和é™åˆ¶ + +在较大的 Kubernetes 集群中，Prometheus 会消耗大é‡å†…存。Prometheus 需è¦çš„内存与它存储的时间åºåˆ—和标签的数é‡ä»¥åŠæŠ“å–间隔有关。 + +ä½ å¯ä»¥è®¿é—®[这篇åšå®¢](https://www.robustperception.io/how-much-ram-does-prometheus-2-x-need-for-cardinality-and-ingestion)了解关于如何计算必须的内存的更多信æ¯ã€‚ + +所需的 CPU æ•°é‡ä¸Žä½ æ­£åœ¨æ‰§è¡Œçš„查询数é‡ç›¸å…³ã€‚ + +### 长期储存 + +Prometheus ä¸æ˜¯ç”¨äºŽé•¿æœŸå­˜å‚¨æŒ‡æ ‡çš„，它åªç”¨äºŽçŸ­æœŸå­˜å‚¨ã€‚ + +如果想è¦é•¿æ—¶é—´å­˜å‚¨éƒ¨åˆ†æˆ–全部指标，你å¯ä»¥åˆ©ç”¨ Prometheus çš„[远程读/写](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)功能将其连接到 [Thanos](https://thanos.io/)，[InfluxDB](https://www.influxdata.com/) 或 [M3DB](https://www.m3db.io/) 等存储系统。你å¯ä»¥è®¿é—®[这篇åšå®¢](https://rancher.com/blog/2020/prometheus-metric-federation)找到设置示例。 + +## 抓å–自定义工作负载 + +虽然集æˆçš„ Rancher Monitoring å·²ç»å¯ä»¥ä»Žé›†ç¾¤çš„节点和系统组件中抓å–系统指标，但你也需è¦ä¸ºéƒ¨ç½²åœ¨ Kubernetes 上的自定义工作负载抓å–æ•°æ®ã€‚为此，你å¯ä»¥é…ç½® Prometheus，让它在一定的时间间隔内对你应用的端点进行 HTTP 请求。然åŽï¼Œè¿™äº›ç«¯ç‚¹ä¼šä»¥ Prometheus æ ¼å¼è¿”回指标。 + +一般æ¥è¯´ï¼Œä½ ä¼šä»Žé›†ç¾¤ä¸­è¿è¡Œçš„所有工作负载中抓å–æ•°æ®ï¼Œç„¶åŽå°†æ•°æ®ç”¨äºŽå‘Šè­¦æˆ–调试问题。一般情况下，你åªæœ‰åœ¨å…·ä½“事件å‘生åŽæ‰éœ€è¦æŸäº›æŒ‡æ ‡æ•°æ®ã€‚如果数æ®å·²ç»è¢«æŠ“å–并存储了，那就好办了。由于 Prometheus åªæ˜¯çŸ­æœŸå­˜å‚¨æŒ‡æ ‡ï¼Œå› æ­¤æŠ“å–和存储大é‡æ•°æ®çš„æˆæœ¬å¹¶ä¸æ˜¯é‚£ä¹ˆé«˜ã€‚如果你使用的是 Prometheus 的长期存储方案，那么你也å¯ä»¥å†³å®šæŒä¹…存储哪些数æ®ã€‚ + +### 关于 Prometheus Exporters + +许多第三方工作负载（如数æ®åº“ã€é˜Ÿåˆ—或 Web æœåŠ¡å™¨ï¼‰è¦ä¹ˆå·²ç»æ”¯æŒä»¥ Prometheus æ ¼å¼å…¬å¼€æŒ‡æ ‡ï¼Œè¦ä¹ˆæœ‰æ‰€è°“çš„ exporter，æ¥å¯¹å·¥å…·çš„指标格å¼å’Œ Prometheus ç†è§£çš„指标格å¼è¿›è¡Œè½¬æ¢ã€‚通常，你å¯ä»¥å°†è¿™äº› exporter 作为é¢å¤–çš„ Sidecar 容器添加到工作负载的 Pod 中。很多 Helm Chart å·²ç»åŒ…å«äº†éƒ¨ç½² exporter 的选项。此外，你还å¯ä»¥åœ¨ [promcat.io](https://promcat.io/) å’Œ [ExporterHub](https://exporterhub.io/) 上找到一个由 SysDig 策划的 exporter 列表。 + +### Prometheus çš„ç¼–ç¨‹è¯­è¨€å’Œæ¡†æž¶æ”¯æŒ + +è¦æƒ³æŠŠä½ çš„自定义应用指标放到 Prometheus 中，你必须直接从你的应用代ç ä¸­æ”¶é›†å’Œæš´éœ²è¿™äº›æŒ‡æ ‡ã€‚幸è¿çš„是，对于大多数æµè¡Œçš„编程语言和框架æ¥è¯´ï¼Œå·²ç»æœ‰ä¸€äº›åº“和集æˆæ¥å¸®åŠ©è§£å†³è¿™ä¸ªé—®é¢˜ã€‚其中一个例å­æ˜¯ [Spring 框架](https://docs.spring.io/spring-metrics/docs/current/public/prometheus)中的 Prometheus 支æŒã€‚ + +### ServiceMonitor å’Œ PodMonitor + +一旦所有工作负载都以 Prometheus æ ¼å¼å…¬å¼€äº†æŒ‡æ ‡åŽï¼Œä½ å¿…é¡»é…ç½® Prometheus æ¥æŠ“å–æ•°æ®ã€‚Rancher 使用 [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator)，这使得使用 ServiceMonitors å’Œ PodMonitors æ¥æ·»åŠ å…¶ä»–抓å–目标å˜å¾—容易。很多 Helm Chart å·²ç»åŒ…å«ç›´æŽ¥åˆ›å»ºè¿™äº›ç›‘控器的选项。你也å¯ä»¥åœ¨ Rancher 文档中找到更多信æ¯ã€‚ + +### Prometheus 推é€ç½‘关（Pushgateway） + +有些工作负载的指标很难被 Prometheus 抓å–ï¼Œå°±åƒ Jobs å’Œ CronJobs 这样的短期工作负载，或者是ä¸å…许在å•ä¸ªå¤„ç†çš„传入请求之间共享数æ®çš„应用，如 PHP 应用。 + +è¦æƒ³èŽ·å¾—这些用例的指标，你å¯ä»¥è®¾ç½® [prometheus-pushgateways](https://github.com/prometheus/pushgateway)。CronJob 或 PHP 应用将把指标更新推é€åˆ° pushgateway。pushgateway 汇总并通过 HTTP 端点暴露它们，然åŽå¯ä»¥ç”± Prometheus 抓å–。 + +### Prometheus Blackbox 监控 + +有时，你å¯èƒ½éœ€è¦ä»Žå¤–部监控工作负载。为此，你å¯ä»¥ä½¿ç”¨ [Prometheus blackbox-exporter](https://github.com/prometheus/blackbox_exporter)，它å…许通过 HTTPã€HTTPSã€DNSã€TCP å’Œ ICMP 探测任何类型的端点。 + +## 在（微）æœåŠ¡æž¶æž„中进行监控 + +如果你有一个（微）æœåŠ¡æž¶æž„，在该架构中集群的多个å•ç‹¬çš„工作负载相互通信，那么拥有这些æµé‡çš„详细指标和跟踪是éžå¸¸é‡è¦çš„，因为这å¯ä»¥å¸®åŠ©ä½ äº†è§£æ‰€æœ‰è¿™äº›å·¥ä½œè´Ÿè½½ä¹‹é—´çš„通信方å¼ï¼Œä»¥åŠé—®é¢˜æˆ–瓶颈å¯èƒ½å‡ºçŽ°çš„地方。 + +当然，你å¯ä»¥ç›‘控所有工作负载中的所有内部æµé‡ï¼Œå¹¶å°†è¿™äº›æŒ‡æ ‡æš´éœ²ç»™ Prometheusï¼Œä½†è¿™ç›¸å½“è€—è´¹ç²¾åŠ›ã€‚åƒ Istio 这样的æœåŠ¡ç½‘格（å¯ä»¥é€šè¿‡[å•å‡»](https://rancher.com/docs/rancher/v2.6/en/istio/)在 Rancher 中安装）å¯ä»¥è‡ªåŠ¨å®Œæˆè¿™é¡¹å·¥ä½œï¼Œå¹¶æ供所有 Service 之间æµé‡çš„丰富的é¥æµ‹æ•°æ®ã€‚ + +## 真实用户监控 + +监控所有内部工作负载的å¯ç”¨æ€§å’Œæ€§èƒ½å¯¹äºŽç¨³å®šã€å¯é å’Œå¿«é€Ÿåœ°è¿è¡Œåº”用至关é‡è¦ã€‚但这些指标åªèƒ½å‘你展示部分情况。è¦æƒ³èŽ·å¾—一个完整的视图，还必须知é“你的最终用户是如何实际感知的。为此，你å¯ä»¥ç ”究å„ç§[真实用户监控解决方案](https://en.wikipedia.org/wiki/Real_user_monitoring)。 + +## 安全监控 + +除了通过监控工作负载æ¥æ£€æµ‹æ€§èƒ½ã€å¯ç”¨æ€§æˆ–å¯æ‰©å±•æ€§ä¹‹å¤–，你还应该监控集群和è¿è¡Œåœ¨é›†ç¾¤ä¸­çš„工作负载，æ¥å‘现潜在的安全问题。一个好的åšæ³•æ˜¯ç»å¸¸è¿è¡Œ [CIS 扫æ](../../../pages-for-subheaders/cis-scan-guides.md)并å‘出告警，æ¥æ£€æŸ¥é›†ç¾¤æ˜¯å¦æŒ‰ç…§å®‰å…¨æœ€ä½³å®žè·µè¿›è¡Œé…置。 + +对于工作负载，你å¯ä»¥æŸ¥çœ‹ Kubernetes å’Œ Container 安全解决方案，例如 [NeuVector](https://www.suse.com/products/neuvector/)ã€[Falco](https://falco.org/)ã€[Aqua Kubernetes Security](https://www.aquasec.com/solutions/kubernetes-container-security/) å’Œ [SysDig](https://sysdig.com/)。 + +## 设置告警 + +将所有的指标纳入监控系统并在仪表æ¿ä¸­å¯è§†åŒ–是很好的åšæ³•ï¼Œä½†ä½ ä¹Ÿå¸Œæœ›åœ¨å‡ºçŽ°é—®é¢˜æ—¶èƒ½ä¸»åŠ¨æ”¶åˆ°æ醒。 + +集æˆçš„ Rancher 监控已ç»é…置了一套åˆç†çš„告警，这些告警在任何 Kubernetes 集群中都是å¯ç”¨çš„。你å¯ä»¥æ‰©å±•å‘Šè­¦ï¼Œæ¥è¦†ç›–特定的工作负载和用例。 + +在设置告警时，你需è¦ä¸ºå¯¹ä½ åº”用éžå¸¸å…³é”®çš„工作负载é…置告警，但åŒæ—¶ä¹Ÿè¦ç¡®ä¿å‘Šè­¦ä¸ä¼šå¤ªé¢‘ç¹ã€‚ç†æƒ³æƒ…况下，你收到的æ¯ä¸€ä¸ªå‘Šè­¦éƒ½åº”该是一个你需è¦å…³æ³¨å¹¶è§£å†³çš„问题。如果你一直收到ä¸å¤ªå…³é”®çš„告警，你就有å¯èƒ½å¼€å§‹å®Œå…¨å¿½ç•¥å‘Šè­¦ä¿¡æ¯ï¼Œç„¶åŽé”™è¿‡çœŸæ­£é‡è¦çš„告警。因此，少é‡çš„å‘Šè­¦å¯èƒ½ä¼šæ›´å¥½ã€‚首先，你å¯ä»¥å…³æ³¨çœŸæ­£é‡è¦çš„指标，例如应用离线等。之åŽï¼Œè§£å†³å‡ºçŽ°çš„所有问题，然åŽå†åˆ›å»ºæ›´è¯¦ç»†çš„告警。 + +如果告警开始å‘é€ï¼Œä½†ä½ æš‚时无法处ç†ï¼Œä½ ä¹Ÿå¯ä»¥å°†å‘Šè­¦é™é»˜ä¸€å®šæ—¶é—´ï¼Œä»¥ä¾¿ä»¥åŽæŸ¥çœ‹ã€‚ + +如果需è¦äº†è§£æ›´å¤šå…³äºŽå¦‚何设置告警和通知通é“çš„ä¿¡æ¯ï¼Œè¯·è®¿é—® [Rancher 文档中心](../../../pages-for-subheaders/monitoring-and-alerting.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md new file mode 100644 index 00000000000..de11f24cedd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -0,0 +1,58 @@ +--- +title: Rancher ç®¡ç† vSphere 集群的最佳实践 +--- + +本指å—概述了在 vSphere 环境中é…置下游 Rancher 集群的å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 + +- [1. 虚拟机注æ„事项](#1-虚拟机注æ„事项) +- [2. 网络注æ„事项](#2-网络注æ„事项) +- [3. 存储注æ„事项](#3-存储注æ„事项) +- [4. 备份和ç¾éš¾æ¢å¤](#4-备份和ç¾éš¾æ¢å¤) + +
解决方案概述
+ +![解决方案概述](/img/solution_overview.drawio.svg) + +## 1. 虚拟机注æ„事项 + +### 利用虚拟机模æ¿æ¥æž„建环境 + +为了ä¿è¯è·¨çŽ¯å¢ƒéƒ¨ç½²çš„虚拟机的一致性，你å¯ä»¥è€ƒè™‘使用虚拟机模æ¿å½¢å¼çš„黄金镜åƒï¼ˆgolden image）。你å¯ä»¥ä½¿ç”¨ Packer æ¥å®žçŽ°ï¼Œä»Žè€Œå¢žåŠ æ›´å¤šè‡ªå®šä¹‰é€‰é¡¹ã€‚ + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ ESXi 主机上分离下游集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚ç‚¹è™šæ‹Ÿæœºåˆ†å¸ƒåœ¨å¤šå° ESXi 主机上，从而防止主机级别的å•ç‚¹æ•…障。 + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ Datastore 上分离下游集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚点虚拟机分布在多个 Datastore 上，从而防止 Datastore 级别的å•ç‚¹æ•…障。 + +### 为 Kubernetes é…ç½®åˆé€‚的虚拟机 + +在部署节点时，请éµå¾ª K8s å’Œ etcd 的最佳实践，其中包括ç¦ç”¨ swap，检查集群中的所有主机之间是å¦æœ‰è‰¯å¥½çš„网络连接，为æ¯ä¸ªèŠ‚点使用唯一的主机åã€MAC 地å€å’Œ `product_uuids`。 + +## 2. 网络注æ„事项 + +### 利用 ETCD 节点之间的低延迟和高带宽连接 + +å°½å¯èƒ½åœ¨å•ä¸ªæ•°æ®ä¸­å¿ƒå†…部署 etcd æˆå‘˜ï¼Œæ¥é¿å…延迟开销并å‡å°‘网络分区的å¯èƒ½æ€§ã€‚大多数情况下，1Gb 的连接就足够了。对于大型集群，10Gb 的连接å¯ä»¥ç¼©çŸ­æ¢å¤å¤‡ä»½æ‰€éœ€çš„时间。 + +### 为虚拟机æ供固定的 IP åœ°å€ + +ä½ å¯ä»¥ä¸ºä½¿ç”¨çš„所有节点都é…置一个é™æ€ IP。如果使用 DHCP，则æ¯ä¸ªèŠ‚点都应该有一个 DHCP 预留，以确ä¿èŠ‚点分é…到相åŒçš„ IP 地å€ã€‚ + +## 3. 存储注æ„事项 + +### 在 ETCD 节点上使用 SSD ç£ç›˜ + +ETCD 对写入延迟éžå¸¸æ•æ„Ÿã€‚因此，你å¯ä»¥å°½é‡ä½¿ç”¨ SSD ç£ç›˜æ¥æ高写入性能。 + +## 4. 备份和ç¾éš¾æ¢å¤ + +### 定期备份下游集群 + +Kubernetes 使用 etcd æ¥å­˜å‚¨å…¶æ‰€æœ‰æ•°æ®ï¼ŒåŒ…括é…ç½®ã€çŠ¶æ€å’Œå…ƒæ•°æ®ã€‚在ç¾éš¾æ¢å¤çš„情况下，备份这些数æ®æ˜¯è‡³å…³é‡è¦çš„。 + +### 备份下游节点虚拟机 + +å°† Rancher 下游节点的虚拟机纳入标准的虚拟机备份策略中。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md new file mode 100644 index 00000000000..d48c0cf2af2 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -0,0 +1,52 @@ +--- +title: 设置容器的技巧 +--- + +é…置良好的容器å¯ä»¥æžå¤§åœ°æ高环境的整体性能和安全性。 + +本文介ç»ä¸€äº›è®¾ç½®å®¹å™¨çš„技巧。 + +如果你需è¦äº†è§£å®¹å™¨å®‰å…¨çš„详细信æ¯ï¼Œä¹Ÿå¯ä»¥å‚è§ Rancher çš„[容器安全指å—](https://rancher.com/complete-guide-container-security)。 + +### 使用通用容器æ“作系统 + +在å¯èƒ½çš„情况下，你应该尽é‡åœ¨é€šç”¨çš„容器基础æ“作系统上进行标准化。 + +Alpine å’Œ BusyBox 等较å°çš„å‘行版å‡å°‘了容器镜åƒçš„大å°ï¼Œå¹¶ä¸”通常具有较å°çš„æ¼æ´žã€‚ + +æµè¡Œçš„å‘行版如 Ubuntuã€Fedora å’Œ CentOS 等都ç»è¿‡äº†å¤§é‡çš„测试，并æ供了更多的功能。 + +### 使用 From scratch 容器 +如果你的微æœåŠ¡æ˜¯ä¸€ä¸ªç‹¬ç«‹çš„é™æ€äºŒè¿›åˆ¶ï¼Œä½ åº”该使用 `From scratch` 容器。 + +`FROM scratch` 容器是一个[官方 Docker é•œåƒ](https://hub.docker.com/_/scratch)，它是空的，这样你就å¯ä»¥ç”¨å®ƒæ¥è®¾è®¡æœ€å°çš„é•œåƒã€‚ + +这个镜åƒè¿™å°†å…·æœ‰æœ€å°çš„攻击层和最å°çš„é•œåƒå¤§å°ã€‚ + +### 以éžç‰¹æƒæ–¹å¼è¿è¡Œå®¹å™¨è¿›ç¨‹ +在å¯èƒ½çš„情况下，在容器内è¿è¡Œè¿›ç¨‹æ—¶ä½¿ç”¨éžç‰¹æƒç”¨æˆ·ã€‚虽然容器è¿è¡Œæ—¶æ供了隔离，但ä»ç„¶å¯èƒ½å­˜åœ¨æ¼æ´žå’Œæ”»å‡»ã€‚如果容器以 root 身份è¿è¡Œï¼Œæ— æ„中或æ„外的主机挂载也会å—到影å“。有关为 Pod 或容器é…置安全上下文的详细信æ¯ï¼Œè¯·å‚è§ [Kubernetes 文档](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/)。 + +### 定义资æºé™åˆ¶ +你应该将 CPU 和内存é™åˆ¶åº”用到你的 Pod 上。这å¯ä»¥å¸®åŠ©ç®¡ç† worker 节点上的资æºï¼Œå¹¶é¿å…å‘生故障的微æœåŠ¡å½±å“其他微æœåŠ¡ã€‚ + +在标准 Kubernetes 中，你å¯ä»¥è®¾ç½®å‘½å空间级别的资æºé™åˆ¶ã€‚在 Rancher 中，你å¯ä»¥è®¾ç½®é¡¹ç›®çº§åˆ«çš„资æºé™åˆ¶ï¼Œé¡¹ç›®å†…的所有命å空间都会继承这些é™åˆ¶ã€‚详情请å‚è§ Rancher 官方文档。 + +在设置资æºé…é¢æ—¶ï¼Œå¦‚果你在项目或命å空间上设置了任何与 CPU 或内存相关的内容（å³é™åˆ¶æˆ–预留），所有容器都需è¦åœ¨åˆ›å»ºæœŸé—´è®¾ç½®å„自的 CPU 或内存字段。为了é¿å…在创建工作负载期间对æ¯ä¸ªå®¹å™¨è®¾ç½®è¿™äº›é™åˆ¶ï¼Œå¯ä»¥åœ¨å‘½å空间上指定一个默认的容器资æºé™åˆ¶ã€‚ + +有关如何在[容器级别](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container)和命å空间级别设置资æºé™åˆ¶çš„更多信æ¯ï¼Œè¯·å‚è§ Kubernetes 文档。 + +### 定义资æºéœ€æ±‚ +你应该将 CPU 和内存è¦æ±‚应用到你的 Pod 上。这对于通知调度器需è¦å°†ä½ çš„ pod 放置在哪ç§ç±»åž‹çš„计算节点上，并确ä¿å®ƒä¸ä¼šè¿‡åº¦é…置该节点资æºè‡³å…³é‡è¦ã€‚在 Kubernetes 中，你å¯ä»¥é€šè¿‡åœ¨ pod 的容器规范的资æºè¯·æ±‚字段中定义 `resources.requests` æ¥è®¾ç½®èµ„æºéœ€æ±‚。详情请å‚è§ [Kubernetes 文档](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container)。 + +:::note + +如果你为 pod 所部署的命å空间设置了资æºé™åˆ¶ï¼Œè€Œå®¹å™¨æ²¡æœ‰ç‰¹å®šçš„资æºè¯·æ±‚，则ä¸å…许å¯åŠ¨ pod。为了é¿å…在创建工作负载期间对æ¯ä¸ªå®¹å™¨è®¾ç½®è¿™äº›å­—段，å¯ä»¥åœ¨å‘½å空间上指定一个默认的容器资æºé™åˆ¶ã€‚ + +::: + +建议在容器级别上定义资æºéœ€æ±‚，å¦åˆ™ï¼Œè°ƒåº¦å™¨ä¼šè®¤ä¸ºé›†ç¾¤åŠ è½½å¯¹ä½ çš„应用没有帮助。 + +### é…置存活和就绪探测器 +ä½ å¯ä»¥ä¸ºä½ çš„容器é…置存活探测器和就绪探测器。如果你的容器ä¸æ˜¯å®Œå…¨å´©æºƒï¼ŒKubernetes 是ä¸ä¼šçŸ¥é“它是ä¸å¥åº·çš„，除éžä½ åˆ›å»ºä¸€ä¸ªå¯ä»¥æŠ¥å‘Šå®¹å™¨çŠ¶æ€çš„端点或机制。或者，确ä¿ä½ çš„容器在ä¸å¥åº·çš„情况下åœæ­¢å¹¶å´©æºƒã€‚ + +Kubernetes 文档展示了如何[为容器é…置存活和就绪探测器](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md new file mode 100644 index 00000000000..8ffe650d009 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -0,0 +1,84 @@ +--- +title: 在 vSphere 环境中安装 Rancher +--- + +本指å—概述了在 vSphere 环境中在 RKE Kubernetes 集群上安装 Rancher çš„å‚è€ƒæž¶æž„ï¼Œä»¥åŠ VMware 记录的标准 vSphere 最佳实践。 + + +
解决方案概述
+ +![解决方案概述](/img/rancher-on-prem-vsphere.svg) + +## 1. è´Ÿè½½å‡è¡¡å™¨æ³¨æ„事项 + +你需è¦ä½¿ç”¨ä¸€ä¸ªè´Ÿè½½å‡è¡¡å™¨å°†æµé‡è½¬å‘到 RKE 节点上的 Rancher 工作负载。 + +### 利用容错和高å¯ç”¨æ€§ + +请充分利用具有继承高å¯ç”¨åŠŸèƒ½çš„外部（硬件或软件）负载å‡è¡¡å™¨ï¼ˆå¦‚：F5ã€NSX-Tã€Keepalived 等）。 + +### 备份负载å‡è¡¡å™¨é…ç½® + +在ç¾éš¾æ¢å¤æ—¶ï¼Œå¯ç”¨çš„è´Ÿè½½å‡è¡¡å™¨é…ç½®å¯ä»¥åŠ å¿«æ¢å¤è¿‡ç¨‹ã€‚ + +### é…ç½®å¥åº·æ£€æŸ¥ + +让负载å‡è¡¡å™¨åœ¨å¥åº·æ£€æŸ¥å¤±è´¥æ—¶è‡ªåŠ¨å°†èŠ‚点标记为ä¸å¯ç”¨ã€‚例如，NGINX å¯ä»¥é€šè¿‡ä»¥ä¸‹é…ç½®æ¥å®žçŽ°è¿™ä¸€åŠŸèƒ½ï¼š + +`max_fails=3 fail_timeout=5s` + +### 利用外部负载å‡è¡¡å™¨ + +é¿å…在管ç†é›†ç¾¤å†…使用软件负载å‡è¡¡å™¨ã€‚ + +### 安全访问 Rancher + +将防ç«å¢™/ACL 规则é…置为åªå…许 Rancher 访问。 + +## 2. 虚拟机注æ„事项 + +### æ ¹æ® Rancher æ–‡æ¡£ç¡®å®šè™šæ‹Ÿæœºçš„å¤§å° + +请å‚阅[安装è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)。 + +### 利用虚拟机模æ¿æ¥æž„建环境 + +为了ä¿è¯è·¨çŽ¯å¢ƒéƒ¨ç½²çš„虚拟机的一致性，你å¯ä»¥è€ƒè™‘使用虚拟机模æ¿å½¢å¼çš„黄金镜åƒï¼ˆgolden image）。你å¯ä»¥ä½¿ç”¨ Packer æ¥å®žçŽ°ï¼Œä»Žè€Œå¢žåŠ æ›´å¤šè‡ªå®šä¹‰é€‰é¡¹ã€‚ + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ ESXi 主机上分离 Rancher 集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚ç‚¹è™šæ‹Ÿæœºåˆ†å¸ƒåœ¨å¤šå° ESXi 主机上，从而防止主机级别的å•ç‚¹æ•…障。 + +### 利用 DRS å亲和规则（å¯èƒ½çš„è¯ï¼‰åœ¨ Datastore 上分离 Rancher 集群节点 + +这样å¯ä»¥ç¡®ä¿èŠ‚点虚拟机分布在多个 Datastore 上，从而防止 Datastore 级别的å•ç‚¹æ•…障。 + +### 为 Kubernetes é…ç½®åˆé€‚的虚拟机 + +在部署节点时，请éµå¾ª K8s å’Œ etcd 的最佳实践，其中包括ç¦ç”¨ swap，检查集群中的所有主机之间是å¦æœ‰è‰¯å¥½çš„网络连接，为æ¯ä¸ªèŠ‚点使用唯一的主机åã€MAC 地å€å’Œ `product_uuids`。 + +## 3. 网络注æ„事项 + +### 利用 ETCD 节点之间的低延迟和高带宽连接 + +å°½å¯èƒ½åœ¨å•ä¸ªæ•°æ®ä¸­å¿ƒå†…部署 etcd æˆå‘˜ï¼Œæ¥é¿å…延迟开销并å‡å°‘网络分区的å¯èƒ½æ€§ã€‚大多数情况下，1Gb 的连接就足够了。对于大型集群，10Gb 的连接å¯ä»¥ç¼©çŸ­æ¢å¤å¤‡ä»½æ‰€éœ€çš„时间。 + +### 为虚拟机æ供固定的 IP åœ°å€ + +ä½ å¯ä»¥ä¸ºä½¿ç”¨çš„所有节点都é…置一个é™æ€ IP。如果使用 DHCP，则æ¯ä¸ªèŠ‚点都应该有一个 DHCP 预留，以确ä¿èŠ‚点分é…到相åŒçš„ IP 地å€ã€‚ + +## 4. 存储注æ„事项 + +### 在 ETCD 节点上使用 SSD ç£ç›˜ + +ETCD 对写入延迟éžå¸¸æ•æ„Ÿã€‚因此，你å¯ä»¥å°½é‡ä½¿ç”¨ SSD ç£ç›˜æ¥æ高写入性能。 + +## 5. 备份和ç¾éš¾æ¢å¤ + +### 定期备份管ç†é›†ç¾¤ + +Rancher 将数æ®å­˜å‚¨åœ¨å®ƒæ‰€åœ¨çš„ Kubernetes 集群的 ETCD datastore 中。与其它 Kubernetes 集群一样，你需è¦å¯¹è¯¥é›†ç¾¤è¿›è¡Œé¢‘ç¹ä¸”ç»è¿‡æµ‹è¯•çš„备份。 + +### 备份 Rancher 集群节点虚拟机 + +å°† Rancher 管ç†èŠ‚点的虚拟机纳入标准的虚拟机备份策略中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md new file mode 100644 index 00000000000..8c4d1541296 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -0,0 +1,36 @@ +--- +title: Rancher 部署策略 +--- + +本文æä¾› Rancher 实例的两ç§éƒ¨ç½²ç­–略，用于管ç†ä¸‹æ¸¸ Kubernetes 集群。æ¯ç§ç­–略都有优缺点。请按照你的实际情况选择最适åˆçš„部署策略。 + +## 中心è¾å°„åž‹ç­–ç•¥ +--- + +在中心è¾å°„型部署中，一个 Rancher 实例就å¯ä»¥ç®¡ç†é布全çƒçš„ Kubernetes 集群。Rancher 实例è¿è¡Œåœ¨é«˜å¯ç”¨ Kubernetes 集群上，并且会å—延迟影å“。 + +### 优点 + +* å¯ä»¥é€šè¿‡ä¸€ä¸ª controlplane ç•Œé¢æŸ¥çœ‹æ‰€æœ‰åŒºåŸŸå’ŒçŽ¯å¢ƒã€‚ +* Kubernetes ä¸éœ€è¦ Rancher 进行æ“作，并且å¯ä»¥å®¹å¿ä¸Ž Rancher 实例断开连接。 + +### 缺点 + +* å—é™äºŽç½‘络延迟。 +* 如果 Rancher 出现故障，在æ¢å¤ä¹‹å‰ä¸å¯ä»¥åœ¨å…¨çƒèŒƒå›´å†…创建新æœåŠ¡ã€‚但是æ¯ä¸ª Kubernetes 集群都å¯ä»¥ç»§ç»­å•ç‹¬ç®¡ç†ã€‚ + +## 区域型策略 +--- +在区域型部署模型中，Rancher 实例部署在é è¿‘下游 Kubernetes 集群的ä½ç½®ã€‚ + +### 优点 + +* 如果æŸä¸ªåŒºåŸŸçš„ Rancher 实例出现故障，其他区域内的 Rancher 功能ä»ç„¶å¯ä»¥ä¿æŒæ­£å¸¸ã€‚ +* Rancher 和下游集群之间的网络延迟大大é™ä½Žï¼Œæ高了 Rancher 的性能。 +* ä½ å¯ä»¥åœ¨æ¯ä¸ªåŒºåŸŸå†…独立å‡çº§ Rancher。 + +### 缺点 + +* 管ç†å¤šä¸ª Rancher 安装的开销较大。 +* 需è¦åœ¨å¤šä¸ªç•Œé¢ä¸­æŸ¥çœ‹ä¸åŒåŒºåŸŸçš„ Kubernetes 集群。 +* 在 Rancher 中部署多集群应用时，需è¦åœ¨æ¯ä¸ª Rancher Server 中é‡å¤éƒ¨ç½²æ­¥éª¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md new file mode 100644 index 00000000000..0c47b75fa06 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -0,0 +1,36 @@ +--- +title: Rancher è¿è¡ŒæŠ€å·§ +--- + +本指å—适用于使用 Rancher 管ç†ä¸‹æ¸¸ Kubernetes 集群的用例。高å¯ç”¨è®¾ç½®å¯ä»¥é˜²æ­¢ Rancher Server ä¸å¯ç”¨æ—¶æ— æ³•è®¿é—®ä¸‹æ¸¸é›†ç¾¤ã€‚ + +高å¯ç”¨ Rancher 安装指将 Rancher 安装到至少有三个节点的 Kubernetes 集群上，适用于所有生产环境以åŠé‡è¦çš„安装场景。在多个节点上è¿è¡Œå¤šä¸ª Rancher 实例能够实现å•èŠ‚点安装无法æ供的高å¯ç”¨æ€§ã€‚ + +如果你在 vSphere 环境中安装 Rancher，请å‚è§[最佳实践文档](on-premises-rancher-in-vsphere.md)。 + +在设置高å¯ç”¨ Rancher 安装时，请考虑以下事项。 + +### 在å•ç‹¬çš„集群上è¿è¡Œ Rancher +ä¸è¦åœ¨å®‰è£…了 Rancher çš„ Kubernetes 集群上è¿è¡Œå…¶ä»–工作负载或微æœåŠ¡ã€‚ + +### ç¡®ä¿ Kubernetes 节点é…置正确 +在部署节点时，请éµå¾ª K8s å’Œ etcd 的最佳实践，其中包括ç¦ç”¨ swap，检查集群中的所有主机之间是å¦æœ‰è‰¯å¥½çš„网络连接，为æ¯ä¸ªèŠ‚点使用唯一的主机åã€MAC 地å€å’Œ `product_uuids`，检查所需端å£æ˜¯å¦å·²ç»æ‰“开，并使用é…ç½® SSD çš„ etcd 进行部署。详情请å‚è§ [kubernetes 官方文档](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin)å’Œ [etcd 性能æ“作指å—](https://etcd.io/docs/v3.4/op-guide/performance/)。 + +### 使用 RKE 时：备份状æ€æ–‡ä»¶ï¼ˆStatefile） +RKE 将集群状æ€è®°å½•åœ¨ä¸€ä¸ªå为 `cluster.rkestate` 的文件中，该文件对集群的æ¢å¤å’Œ/或通过 RKE 维护集群éžå¸¸é‡è¦ã€‚由于这个文件包å«è¯ä¹¦æ料，我们强烈建议在备份å‰å¯¹è¯¥æ–‡ä»¶è¿›è¡ŒåŠ å¯†ã€‚请在æ¯æ¬¡è¿è¡Œ `rke up` åŽå¤‡ä»½çŠ¶æ€æ–‡ä»¶ã€‚ + +### 在åŒä¸€ä¸ªæ•°æ®ä¸­å¿ƒè¿è¡Œé›†ç¾¤ä¸­çš„所有节点 +为达到最佳性能，请在åŒä¸€åœ°ç†æ•°æ®ä¸­å¿ƒè¿è¡Œæ‰€æœ‰ä¸‰ä¸ªèŠ‚点。如果你在云（如 AWS）上è¿è¡ŒèŠ‚点，请在ä¸åŒçš„å¯ç”¨åŒºï¼ˆAZ）中è¿è¡Œè¿™ä¸‰ä¸ªèŠ‚点。例如，在 us-west-2a 中è¿è¡ŒèŠ‚点 1，在 us-west-2b 中è¿è¡ŒèŠ‚点 2，在 us-west-2c 中è¿è¡ŒèŠ‚点 3。 + +### ä¿è¯å¼€å‘和生产环境的相似性 +强烈建议为è¿è¡Œ Rancher çš„ Kubernetes 集群é…备 “staging†或 “pre-production†环境。这个环境的软件和硬件é…置应该尽å¯èƒ½æŽ¥è¿‘你的生产环境。 + +### ç›‘æŽ§é›†ç¾¤ä»¥è§„åˆ’å®¹é‡ +Rancher Server çš„ Kubernetes 集群应该尽å¯èƒ½æ»¡è¶³[系统和硬件è¦æ±‚](../../../pages-for-subheaders/installation-requirements.md)。越å离系统和硬件è¦æ±‚，你å¯èƒ½é¢ä¸´çš„风险就越大。 + +但是，已å‘布的è¦æ±‚å·²ç»è€ƒè™‘了å„ç§å·¥ä½œè´Ÿè½½ç±»åž‹ï¼Œå› æ­¤ï¼ŒåŸºäºŽæŒ‡æ ‡æ¥è§„划容é‡åº”该是扩展 Rancher 的最佳实践。 + +ä½ å¯ä»¥å°† Rancher 集æˆä¸šç•Œé¢†å…ˆçš„å¼€æºç›‘控解决方案 Prometheus 以åŠèƒ½å¯è§†åŒ– Prometheus 指标的 Grafana，æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿‡ç¨‹ã€‚ + +在集群中[å¯ç”¨ç›‘控](../../../pages-for-subheaders/monitoring-and-alerting.md)åŽï¼Œä½ å¯ä»¥é€šè¿‡è®¾ç½®å‘Šè­¦é€šçŸ¥ï¼Œæ¥äº†è§£é›†ç¾¤å®¹é‡çš„使用情况。你还å¯ä»¥ä½¿ç”¨ Prometheus å’Œ Grafana 监控框架，在你扩容时建立关键指标的基线。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md new file mode 100644 index 00000000000..8d5fc725d36 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md @@ -0,0 +1,61 @@ +--- +title: Rancher 扩展技巧 +--- + +本指å—介ç»äº†æ‰©å±• Rancher 时应该考虑的方法，以åŠè¿™æ ·åšçš„难度。éšç€ç³»ç»Ÿçš„增长，性能自然会é™ä½Žï¼Œä½†æˆ‘们å¯ä»¥é‡‡å–一些措施æ¥æœ€å¤§é™åº¦åœ°å‡å°‘ Rancher 的负载，并优化 Rancher 处ç†è¿™äº›å¤§åž‹è®¾ç½®çš„能力。 + +## 优化 Rancher 性能的技巧 +* 建议åŠæ—¶æ›´æ–° Rancher çš„è¡¥ä¸ç‰ˆæœ¬ã€‚在次è¦ç‰ˆæœ¬çš„整个生命周期中，我们都会进行性能改进和错误修å¤ã€‚ä½ å¯ä»¥æŸ¥çœ‹å‘行说明决定是å¦éœ€è¦å‡çº§ï¼Œä½†åœ¨å¤§å¤šæ•°æƒ…况下，我们建议你使用最新版本。 + +* æ€§èƒ½ä¼šå— Rancher 的基础设施和下游集群的基础设施之间的延迟影å“（例如，地ç†è·ç¦»ï¼‰ã€‚如果用户需è¦é布全çƒæˆ–分布在许多地区的集群/节点，最好使用多个 Rancher。 + +* 请始终å°è¯•é€æ¸æ‰©å¤§è§„模，åŒæ—¶ç›‘控和观察å˜åŒ–。通常情况下，性能问题越早解决越好，而且在其他问题出现之å‰æ›´å®¹æ˜“解决。 + +## 最å°åŒ–本地集群的负载 +扩展 Rancher 最大的瓶颈是本地 Kubernetes 集群中的资æºå¢žé•¿ã€‚local 集群包å«æ‰€æœ‰ä¸‹æ¸¸é›†ç¾¤çš„ä¿¡æ¯ã€‚下游集群的许多æ“作将在 local 集群中创建新对象，并需è¦ä½¿ç”¨ local 集群中è¿è¡Œçš„处ç†ç¨‹åºè¿›è¡Œè®¡ç®—。 + +### 管ç†å¯¹è±¡æ•°é‡ +ETCD 最终会é‡åˆ°å­˜å‚¨çš„ Kubernetes 资æºæ•°é‡è¶…过é™åˆ¶çš„问题。暂时没有这些数字的确切记录。根æ®å†…部观察，一旦å•ä¸ªèµ„æºç±»åž‹çš„对象数é‡è¶…过 60k（通常是 Rolebindings），就很有å¯èƒ½é‡åˆ°æ€§èƒ½é—®é¢˜ã€‚ + +Rolebindings 是在 local 集群中创建的，是许多æ“作的结果。 + +å°è¯•å‡å°‘ local 集群中的 rolebindings 时需è¦æ³¨æ„： +* 仅在必è¦æ—¶å°†ç”¨æˆ·æ·»åŠ åˆ°é›†ç¾¤å’Œé¡¹ç›®ä¸­ +* 删除ä¸éœ€è¦çš„集群和项目 +* 仅在必è¦æ—¶ä½¿ç”¨è‡ªå®šä¹‰è§’色 +* 在自定义角色中使用尽å¯èƒ½å°‘的规则 +* 考虑给用户添加角色是å¦å¤šä½™ +* 使用更少但更强大的集群å¯èƒ½æ›´é«˜æ•ˆ +* 检查是å¦èƒ½é€šè¿‡åˆ›å»ºæ–°é¡¹ç›®æˆ–新集群为你的用例å‡å°‘ rolebindings。 + +### 使用新版应用程åºè€Œä¸æ˜¯æ—§ç‰ˆåº”ç”¨ç¨‹åº +Rancher ä½¿ç”¨äº†ä¸¤ä¸ªåº”ç”¨ç¨‹åº kubernetes 资æºï¼Œåˆ†åˆ«æ˜¯ apps.projects.cattle.io å’Œ apps.cattle.cattle.ioã€‚æ—§ç‰ˆåº”ç”¨ç¨‹åº apps.projects.cattle.io 最åˆæ˜¯åœ¨ Cluster Manager 中引入的，现在已ç»è¿‡æ—¶ã€‚æ–°ç‰ˆåº”ç”¨ç¨‹åº apps.catalog.cattle.io å¯åœ¨å…¶å„自集群的 Cluster Explorer 中找到。新版应用程åºæ›´å—欢迎，因为它们ä½äºŽä¸‹æ¸¸é›†ç¾¤ä¸­ï¼Œè€Œæ—§ç‰ˆåº”用程åºä½äºŽ local 集群中。 + +我们建议删除 Cluster Manager 中出现的应用程åºï¼Œå¦‚有必è¦ï¼Œå¯ä»¥ä¸ºç›®æ ‡é›†ç¾¤æ›¿æ¢ä¸º Cluster Explorer 中的应用程åºï¼Œå¹¶ä»…在集群的 Cluster Explorer 中创建åŽç»­çš„应用程åºã€‚ + +### 使用授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE） +Rancher é…置的 RKE1ã€RKE2 å’Œ K3s 集群有一个 _授æƒé›†ç¾¤ç«¯ç‚¹_ 选项。å¯ç”¨åŽï¼Œä¼šåœ¨ä¸ºé›†ç¾¤ç”Ÿæˆçš„ kubeconfigs 中添加一个上下文，该上下文使用一个直接到集群的端点，并绕过 Rancher。但是，仅å¯ç”¨æ­¤é€‰é¡¹æ˜¯ä¸å¤Ÿçš„。Kubeconfig 的用户需è¦ä½¿ç”¨ `kubectl use-context ` æ‰èƒ½å¼€å§‹ä½¿ç”¨å®ƒã€‚ + +如果ä¸ä½¿ç”¨ ACE，所有 kubeconfig 请求都会先通过 Rancher 进行路由。 + +### 实验功能：å‡å°‘事件处ç†ç¨‹åºæ‰§è¡Œçš„选项 +Rancher 的大部分逻辑都å‘生在事件处ç†ç¨‹åºä¸Šã€‚æ¯å½“更新对象以åŠå¯åŠ¨ Rancher 时，这些事件处ç†ç¨‹åºéƒ½ä¼šåœ¨å¯¹è±¡ä¸Šè¿è¡Œã€‚此外，当缓存åŒæ­¥æ—¶ï¼Œå®ƒä»¬æ¯ 15 å°æ—¶ä¼šè¿è¡Œä¸€æ¬¡ã€‚在缩放设置中，由于æ¯ä¸ªå¤„ç†ç¨‹åºéƒ½ä¼šåœ¨æ¯ä¸ªé€‚用对象上è¿è¡Œï¼Œå› æ­¤è¿è¡Œè¿™äº›è®¡åˆ’会消耗巨大的性能。但是，你å¯ä»¥ä½¿ç”¨ `CATTLE_SYNC_ONLY_CHANGED_OBJECTS` 环境å˜é‡æ¥ç¦ç”¨å¤„ç†ç¨‹åºçš„这些计划执行。如果在大约 15 å°æ—¶çš„间隔内看到资æºåˆ†é…峰值，则此设置å¯èƒ½æœ‰å¸®åŠ©ã€‚ + +环境å˜é‡çš„值å¯ä»¥æ˜¯ä»¥ä¸‹é€‰é¡¹çš„逗å·åˆ†éš”列表。这些值指的是控制器的类型（包å«å’Œè¿è¡Œå¤„ç†ç¨‹åºçš„ struct）åŠå…¶å¤„ç†ç¨‹åºã€‚如果将控制器类型添加到å˜é‡ä¸­ï¼Œé‚£ä¹ˆè¯¥ç»„控制器将无法在é‡æ–°åŒæ­¥ç¼“存时è¿è¡Œå…¶å¤„ç†ç¨‹åºã€‚ + +* `mgmt` 指的是åªè¿è¡Œåœ¨ä¸€ä¸ª Rancher 节点上的管ç†æŽ§åˆ¶å™¨ã€‚ +* `user` 指的是为æ¯ä¸ªé›†ç¾¤è¿è¡Œçš„用户控制器。其中一些è¿è¡Œåœ¨ä¸Žç®¡ç†æŽ§åˆ¶å™¨ç›¸åŒçš„节点上，而其他则è¿è¡Œåœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­ã€‚该选项针对å‰è€…。 +* `scaled` 指的是在æ¯ä¸ª Rancher 节点上è¿è¡Œçš„缩放控制器。由于缩放处ç†ç¨‹åºè´Ÿè´£çš„关键功能，因此ä¸å»ºè®®è®¾ç½®ã€‚ + +简而言之，如果你å‘现 CPU ä½¿ç”¨çŽ‡æ¯ 15 å°æ—¶å‡ºçŽ°ä¸€æ¬¡å³°å€¼ï¼Œè¯·å°† `CATTLE_SYNC_ONLY_CHANGED_OBJECTS` 环境å˜é‡æ·»åŠ åˆ°ä½ çš„ Rancher 部署中，其值为 `mgmt,user`。 + +## Rancher 之外的优化 +å½±å“性能的一个é‡è¦å› ç´ æ˜¯ local 集群åŠå…¶é…置方å¼ã€‚该集群å¯èƒ½ä¼šåœ¨ Rancher è¿è¡Œä¹‹å‰å¼•å…¥ç“¶é¢ˆã€‚当 Rancher 节点出现资æºå ç”¨è¿‡é«˜çš„情况时，你å¯ä»¥ä½¿ç”¨ top 命令æ¥åˆ¤æ–­æ˜¯ Rancher 还是 Kubernetes 组件在超é¢æ¶ˆè€—资æºã€‚ + +### ä¿æŒä½¿ç”¨æœ€æ–°ç‰ˆæœ¬çš„ Kubernetes +与 Rancher 版本类似，我们建议让你的 kubernetes 集群ä¿æŒä½¿ç”¨æœ€æ–°ç‰ˆæœ¬ã€‚这将确ä¿ä½ çš„集群能包å«å¯ç”¨çš„性能增强或错误修å¤ã€‚ + +### 优化 ETCD +[ETCD 性能](https://etcd.io/docs/v3.4/op-guide/performance/)的两个主è¦ç“¶é¢ˆæ˜¯ç£ç›˜é€Ÿåº¦å’Œç½‘络速度。对任何一个进行优化都应该能æ高性能。有关 ETCD 性能的信æ¯ï¼Œè¯·å‚阅 [etcd 性能慢（性能测试和优化）](https://www.suse.com/support/kb/doc/?id=000020100)å’Œ[为大型安装调优 etcd](https://docs.ranchermanager.rancher.io/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs)。有关ç£ç›˜çš„ä¿¡æ¯ï¼Œä½ ä¹Ÿå¯ä»¥å‚阅[我们的文档](https://docs.Ranchermanager.Rancher.io/v2.5/pages-for-subheaders/installation-requirements#disks)。 + +ç†è®ºä¸Šï¼ŒETCD 集群中的节点越多，由于å¤åˆ¶è¦æ±‚ [source](https://etcd.io/docs/v3.3/faq)，它就会越慢。这å¯èƒ½ä¸Žå¸¸è§çš„缩放方法相悖。我们还å¯ä»¥æŽ¨æ–­ï¼ŒETCD 的性能将å—到节点间è·ç¦»çš„åé¢å½±å“，因为这将å‡æ…¢ç½‘络通信。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cli-with-rancher/kubectl-utility.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cli-with-rancher/kubectl-utility.md new file mode 100644 index 00000000000..62377d846c6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cli-with-rancher/kubectl-utility.md @@ -0,0 +1,32 @@ +--- +title: kubectl å®žç”¨ç¨‹åº +--- + +## kubectl + +kubectl 用于与 Rancher 进行交互。 + +### kubectl å®žç”¨ç¨‹åº + +安装 `kubectl`。详情请å‚è§[安装 kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)。 + +è¦é…ç½® kubectl，通过 Rancher Web UI 访问你的集群，å•å‡» `Kubeconfig`，然åŽå¤åˆ¶å†…容并将其粘贴到你的 `~/.kube/config` 文件中。 + +检查是å¦å¯ä»¥æˆåŠŸè¿è¡Œ `kubectl cluster-info` 或 `kubectl get pods` 命令。 + +### 使用 kubectl å’Œ kubeconfig 令牌进行 TTL è®¤è¯ + +_è¦æ±‚_ + +如果管ç†å‘˜[关闭了 kubeconfig 令牌生æˆ](../about-the-api/api-tokens.md#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)，当你è¿è¡Œ `kubectl` 时，kubeconfig æ–‡ä»¶éœ€è¦ [Rancher CLI](./rancher-cli.md) 存在于你的 PATH 中。å¦åˆ™ï¼Œä½ ä¼šçœ‹åˆ°è¿™æ ·çš„错误信æ¯ï¼š +`Unable to connect to the server: getting credentials: exec: exec: "rancher": executable file not found in $PATH`。 + +该功能å¯ä»¥è®© kubectl 与 Rancher Server 进行身份验è¯ï¼Œå¹¶åœ¨éœ€è¦æ—¶èŽ·å¾—æ–°çš„ kubeconfig token。目å‰æ”¯æŒä»¥ä¸‹éªŒè¯æ供程åºï¼š + +1. 本地 +2. Active Directory (ä»…é™ LDAP) +3. FreeIPA +4. OpenLDAP +5. SAML 身份æ供商：Ping，Okta，ADFS，Keycloak å’Œ Shibboleth + +如果你是第一次è¿è¡Œ kubectl（例如，`kubectl get pods`），它会è¦æ±‚你选择一个验è¯æ供程åºå¹¶ä½¿ç”¨ Rancher Server 登录。kubeconfig token 会被缓存到 `./.cache/token` 下你è¿è¡Œ kubectl 的路径中。该 Token 在[过期](../about-the-api/api-tokens.md#在生æˆçš„-kubeconfig-中ç¦ç”¨ä»¤ç‰Œ)或[从 Rancher Server 删除](../about-the-api/api-tokens.md#删除令牌)之å‰éƒ½æ˜¯æœ‰æ•ˆçš„。过期åŽï¼Œä¸‹ä¸€ä¸ª `kubectl get pods` 命令会è¦æ±‚ä½ å†æ¬¡ä½¿ç”¨ Rancher Server 登录。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cli-with-rancher/rancher-cli.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cli-with-rancher/rancher-cli.md new file mode 100644 index 00000000000..4cf44238004 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cli-with-rancher/rancher-cli.md @@ -0,0 +1,87 @@ +--- +title: Rancher CLI +description: Rancher CLI 是一个命令行工具，用于在工作站中与 Rancher 进行交互。 +--- + +Rancher CLI（命令行界é¢ï¼‰æ˜¯ä¸€ä¸ªå‘½ä»¤è¡Œå·¥å…·ï¼Œå¯ç”¨äºŽä¸Ž Rancher 进行交互。使用此工具，你å¯ä»¥ä½¿ç”¨å‘½ä»¤è¡Œè€Œä¸ç”¨é€šè¿‡ GUI æ¥æ“作 Rancher。 + +### 下载 Rancher CLI + +ä½ å¯ä»¥ç›´æŽ¥ UI 下载二进制文件。 + +1. 点击左上角的 **☰**。 +1. 在导航侧边æ èœå•åº•éƒ¨ï¼Œå•å‡»**简介**。 +1. 在 **CLI 下载**中，有 Windowsã€Mac å’Œ Linux 的二进制文件下载链接。你还å¯ä»¥è®¿é—®æˆ‘们的 CLI [å‘布页é¢](https://github.com/rancher/cli/releases)直接下载二进制文件。 + +### è¦æ±‚ + +下载 Rancher CLI åŽï¼Œä½ éœ€è¦è¿›è¡Œä¸€äº›é…置。Rancher CLI 需è¦ï¼š + +- ä½ çš„ Rancher Server URL，用于连接到 Rancher Server。 +- API æŒæœ‰è€…令牌（Bearer Tokenï¼‰ï¼Œç”¨äºŽå‘ Rancher 进行身份验è¯ã€‚有关获å–æŒæœ‰è€…令牌的更多信æ¯ï¼Œè¯·å‚阅[创建 API 密钥](../user-settings/api-keys.md)。 + +### CLI èº«ä»½éªŒè¯ + +在使用 Rancher CLI 控制你的 Rancher Server 之å‰ï¼Œä½ å¿…须使用 API æŒæœ‰è€…令牌进行身份验è¯ã€‚è¿è¡Œä»¥ä¸‹å‘½ä»¤è¿›è¡Œç™»å½•ï¼ˆå°† `` å’Œ `` 替æ¢ä¸ºä½ çš„实际信æ¯ï¼‰ï¼š + +```bash +$ ./rancher login https:// --token +``` + +如果 Rancher Server 使用自签åè¯ä¹¦ï¼ŒRancher CLI 会æ示你继续连接。 + +### 项目选择 + +在执行命令之å‰ï¼Œä½ å¿…须先选择一个 Rancher 项目æ¥æ‰§è¡Œè¿™äº›å‘½ä»¤ã€‚è¦é€‰æ‹©[项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)，请è¿è¡Œ `./rancher context switch` 命令。输入此命令åŽï¼Œä¼šæ˜¾ç¤ºå¯ç”¨é¡¹ç›®çš„列表。输入一个数字以选择项目。 + +**示例：`./rancher context switch` 输出** +``` +User:rancher-cli-directory user$ ./rancher context switch +NUMBER CLUSTER NAME PROJECT ID PROJECT NAME +1 cluster-2 c-7q96s:p-h4tmb project-2 +2 cluster-2 c-7q96s:project-j6z6d Default +3 cluster-1 c-lchzv:p-xbpdt project-1 +4 cluster-1 c-lchzv:project-s2mch Default +Select a Project: +``` + +输入数字åŽï¼ŒæŽ§åˆ¶å°ä¼šæ˜¾ç¤ºä½ æ‰€é€‰é¡¹ç›®çš„消æ¯ã€‚ + +``` +INFO[0005] Setting new context to project project-1 +INFO[0005] Saving config to /Users/markbishop/.ranchcli2.json +``` + +请确ä¿ä½ å¯ä»¥æˆåŠŸè¿è¡Œ `rancher kubectl get pods`。 + +### 命令 + +以下命令å¯ç”¨äºŽ Rancher CLI： + +| 命令 | 结果 | +|---|---| +| `apps, [app]` | 对商店应用（å³å•ä¸ª [Helm Chart](https://docs.helm.sh/developing_charts/)）或 Rancher Chart 执行æ“作。 | +| `catalog` | 对[应用商店](../../pages-for-subheaders/helm-charts-in-rancher.md)执行æ“作。 | +| `clusters, [cluster]` | 对[集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)执行æ“作。 | +| `context` | 在 Rancher [项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)之间切æ¢ã€‚有关示例，请å‚阅[项目选择](#项目选择)。 | +| `inspect [OPTIONS] [RESOURCEID RESOURCENAME]` | 显示 [Kubernetes 资æº](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#resource-types)或 Rancher 资æºï¼ˆå³[项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)å’Œ[工作负载](../../pages-for-subheaders/workloads-and-pods.md)）的详细信æ¯ã€‚按å称或 ID 指定资æºã€‚ | +| `kubectl` | è¿è¡Œ [kubectl 命令](https://kubernetes.io/docs/reference/kubectl/overview/#operations)。 | +| `login, [l]` | 登录 Rancher Server。有关示例，请å‚阅 [CLI 身份验è¯](#cli-身份验è¯)。 | +| `namespaces, [namespace]` | 执行命å空间æ“作。 | +| `nodes, [node]` | 执行节点空间æ“作。 | +| `projects, [project]` | 执行[项目](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md)æ“作。 | +| `ps` | 显示项目中的[工作负载](../../pages-for-subheaders/workloads-and-pods.md)。 | +| `settings, [setting]` | 显示 Rancher Server 的当å‰è®¾ç½®ã€‚ | +| `ssh` | 使用 SSH å议连接到你的æŸä¸ªé›†ç¾¤èŠ‚点。 | +| `help, [h]` | 显示命令列表或æŸä¸ªå‘½ä»¤çš„帮助。 | + + +### Rancher CLI 帮助 + +使用 CLI 登录 Rancher Server åŽï¼Œè¾“å…¥ `./rancher --help` 以获å–命令列表。 + +æ‰€æœ‰å‘½ä»¤éƒ½æ”¯æŒ `--help` 标志，该标志解释了æ¯ä¸ªå‘½ä»¤çš„用法。 + +### é™åˆ¶ + +Rancher CLI **ä¸èƒ½**用于安装[仪表æ¿åº”用程åºæˆ– Rancher 功能 Chart](../../pages-for-subheaders/helm-charts-in-rancher.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md new file mode 100644 index 00000000000..98e6b500737 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -0,0 +1,78 @@ +--- +title: EC2 主机é…ç½®å‚考 +--- + +有关 EC2 和节点的更多详细信æ¯ï¼Œè¯·å‚阅 [EC2 管ç†æŽ§åˆ¶å°](https://aws.amazon.com/ec2)的官方文档。 + +### 区域 + +构建集群的地ç†[区域](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html)。 + +### 地区 + +[地区](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones)，一个区域内用于构建集群的隔离ä½ç½®ã€‚ + +### 实例类型 + +[实例类型](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html)用于é…置集群，能确定硬件特性。 + +### æ ¹ç£ç›˜å¤§å° + +é…ç½®[根设备](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/RootDeviceStorage.html)的大å°ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚ + +### VPC/å­ç½‘ + +[VPC](https://docs.aws.amazon.com/vpc/latest/userguide/configure-your-vpc.html) 或特定的[å­ç½‘](https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html)（VPC 中的一个 IP 范围），用于添加你的资æºã€‚ + +### IAM 实例é…置文件å称 + +示例é…置文件的å称，用于将 IAM 角色传递给 EC2 实例。 + +## 高级选项 + +### AMI ID + +用于集群中节点的 [Amazon Machine Image（AMI）](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html)。 + +### 用于 AMI çš„ SSH 用户å + +用于连接到你å¯åŠ¨çš„实例的用户å。有关选定 AMI 的默认用户å，请å‚阅[此处](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connection-prereqs.html)。对于未列出的 AMI，请咨询 AMI æ供商。 + +### 安全组 + +选择默认安全组或é…置安全组。 + +请å‚考[使用主机驱动时的 Amazon EC2 安全组](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-安全组)，了解 `rancher-nodes` 安全组中创建的规则。 + +### EBS æ ¹å·ç±»åž‹ + +用于根设备的 [EBS å·ç±»åž‹](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html)。 + +### 加密 EBS å· + +å¯ç”¨ [Amazon EBS 加密](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html)。 + +### 请求 Spot 实例 + +å¼€å¯[请求 Spot 实例](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html)选项，并指定你愿æ„支付的最高实例价格（æ¯å°æ—¶ï¼‰ã€‚ + +### 仅使用ç§æœ‰åœ°å€ + +å¯ç”¨ä»…使用[ç§äººåœ°å€](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html)的选项。 + +### EBS 优化实例 + +使用 [EBS 优化实例](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html)。 + +### å…许访问 EC2 å…ƒæ•°æ® + +å¯ç”¨å¯¹ [EC2 元数æ®](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)的访问。 + +### 为元数æ®ä½¿ç”¨ Token + +使用 [Instance Metadata Service Version 2 (IMDSv2)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html)，å³åŸºäºŽä»¤ç‰Œè®¿é—®å…ƒæ•°æ®çš„方法。 + +### 添加标签 + +使用[标签](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html)添加元数æ®ï¼Œä»Žè€Œå¯¹èµ„æºè¿›è¡Œåˆ†ç±»ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md new file mode 100644 index 00000000000..6ea2549ccf8 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -0,0 +1,120 @@ +--- +title: Azure 主机é…ç½® +--- + +有关 Azure 的更多信æ¯ï¼Œè¯·å‚阅官方 [Azure 文档](https://docs.microsoft.com/en-us/azure/?product=featured)。 + +### 环境 + +Microsoft æ供了多个[云](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest)æ¥æ»¡è¶³åœ°åŒºæ³•å¾‹çš„è¦æ±‚： + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +### ä½ç½® + +é…置集群和节点的[ä½ç½®](https://docs.microsoft.com/en-us/azure/virtual-machines/regions)。 + +### 资æºç»„ + +资æºç»„æ˜¯ä¸€ä¸ªå®¹å™¨ï¼Œå…¶ä¸­åŒ…å« Azure 解决方案的相关资æºã€‚资æºç»„å¯ä»¥åŒ…括解决方案的所有资æºï¼Œæˆ–者仅包括你希望作为一个组æ¥ç®¡ç†çš„资æºã€‚ä½ å¯ä»¥æ ¹æ®ç»„织的需求æ¥å†³å®šå¦‚何将资æºåˆ†é…给资æºç»„。通常情况下，你å¯ä»¥å°†ç”Ÿå‘½å‘¨æœŸç›¸åŒçš„资æºæ·»åŠ åˆ°åŒä¸€ä¸ªèµ„æºç»„，以便轻æ¾åœ°æŒ‰ç»„进行部署ã€æ›´æ–°å’Œåˆ é™¤ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰èµ„æºç»„，也å¯ä»¥è¾“入资æºç»„çš„å称，然åŽç³»ç»Ÿå°†ä¸ºä½ åˆ›å»ºä¸€ä¸ªèµ„æºç»„。 + +有关管ç†èµ„æºç»„çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal)。 + +### å¯ç”¨æ€§é›†ï¼ˆéžæ‰˜ç®¡ï¼‰ + +è¦æ·»åŠ  VM 的现有[å¯ç”¨æ€§é›†](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview)çš„å称或 ID。 + +### é•œåƒ + +作为 ARM 资æºæ ‡è¯†ç¬¦æ供的æ“作系统镜åƒçš„å称。需è¦ä½¿ç”¨æ‰˜ç®¡ç£ç›˜ã€‚ + +### è™šæ‹Ÿæœºå¤§å° + +为节点池中的æ¯ä¸ª VM 选择一个大å°ã€‚有关æ¯ä¸ª VM 大å°çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/)。 + +## 高级选项 + +### å®¹é”™åŸŸæ•°é‡ + +容错域定义了共享公共电æºå’Œç½‘络交æ¢æœºçš„虚拟机组。如果å¯ç”¨æ€§é›†å·²åˆ›å»ºï¼Œå°†å¿½ç•¥å®¹é”™åŸŸæ•°é‡ã€‚ + +有关容错域的更多信æ¯ï¼Œè¯·å‚阅[此处](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work)。 + +### æ›´æ–°åŸŸæ•°é‡ + +更新域表示å¯ä»¥åŒæ—¶é‡æ–°å¯åŠ¨çš„虚拟机组和底层物ç†ç¡¬ä»¶ã€‚如果å¯ç”¨æ€§é›†å·²åˆ›å»ºï¼Œå°†å¿½ç•¥æ›´æ–°åŸŸæ•°é‡ã€‚ + +有关更新域的更多信æ¯ï¼Œè¯·å‚阅[此处](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work)。 + +### 购买计划 + +Azure 市场中的æŸäº› VM é•œåƒéœ€è¦è´­ä¹°è®¡åˆ’。如果适用，请为你选择的镜åƒé€‰æ‹©æ ¼å¼ä¸º `publisher:product:plan` 的购买计划。 + +### å­ç½‘ + +创建新 VNet 或引用现有 VNet 时的å­ç½‘å称。 + +默认值：`docker-machine` + +### å­ç½‘å‰ç¼€ + +创建 CIDR æ ¼å¼çš„æ–° VNet æ—¶è¦ä½¿ç”¨çš„å­ç½‘ IP 地å€å‰ç¼€ã€‚ + +默认值：`192.168.0.0/16` + +### 虚拟网络 + +è¦ä½¿ç”¨æˆ–创建的[虚拟网络](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview)（如果ä¸å­˜åœ¨ï¼‰ã€‚æ ¼å¼ä¸º `[resourcegroup:]name`。 + +### 公共 IP 选项 + +#### 没有公共 IP + +ä¸è¦åˆ†é…公共 IP 地å€ã€‚ + +#### é™æ€å…¬å…± IP + +分é…é™æ€å…¬å…± IP 地å€ã€‚ + +### 使用ç§æœ‰ IP + +使用é™æ€ç§æœ‰ IP 地å€ã€‚ + +### ç§æœ‰ IP åœ°å€ + +é…ç½®è¦ä½¿ç”¨çš„é™æ€ç§æœ‰ IP 地å€ã€‚ + +### 网络安全组 + +è¦ä½¿ç”¨çš„[网络安全组（NSG）](https://docs.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview)。使用此模æ¿çš„所有节点都将使用é…置的网络安全组。如果没有é…置网络安全组，则为æ¯ä¸ªèŠ‚点创建一个新的网络安全组。 + +### DNS 标签 + +公共 IP 地å€çš„唯一 DNS å称标签。 + +### 储存类型 + +用于 VM çš„[存储账å·](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview)类型。å¯é€‰é¡¹åŒ…括 Standard LRSã€Standard ZRSã€Standard GRSã€Standard RAGRS å’Œ Premium LRS。 + +### 使用托管ç£ç›˜ + +[Azure 托管ç£ç›˜](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview)是由 Azure 管ç†å¹¶ä¸Ž Azure Virtual Machines 一起使用的å—级存储å·ã€‚托管ç£ç›˜æ—¨åœ¨å®žçŽ° 99.999% 的高å¯ç”¨æ€§ã€‚托管ç£ç›˜é€šè¿‡æ供三个数æ®å‰¯æœ¬æ¥å®žçŽ°é«˜å¯ç”¨æ€§å’Œé«˜æŒç»­æ€§ã€‚ + +### 托管ç£ç›˜å¤§å° + +æ¯ä¸ªèŠ‚点的ç£ç›˜å¤§å°ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚ + +### SSH 用户å + +用于 SSH 连接到节点的用户å。 + +### å¼€æ”¾ç«¯å£ + +打开指定端å£ä¸Šçš„入站æµé‡ã€‚如果你使用现有的网络安全组，将忽略开放端å£ã€‚ + +默认值：`2379/tcp, 2380/tcp, 6443/tcp, 9796/tcp, 10250/tcp, 10251/tcp, 10252/tcp, 10256/tcp` å’Œ `8472/udp, 4789/udp` \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md new file mode 100644 index 00000000000..6fdddebab84 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -0,0 +1,33 @@ +--- +title: DigitalOcean 主机é…ç½® +--- + +有关 DigitalOceanã€Droplet 的更多详细信æ¯ï¼Œè¯·å‚阅[官方文档](https://docs.digitalocean.com/products/compute/)。 + +### 区域 + +é…置创建 Droplet çš„[区域](https://docs.digitalocean.com/glossary/region/)。 + +### å¤§å° + +é…ç½® Droplet çš„[大å°](https://docs.digitalocean.com/products/droplets/resources/choose-plan/)。 + +### æ“ä½œç³»ç»Ÿé•œåƒ + +é…置用于创建 Droplet çš„æ“作系统[é•œåƒ](https://docs.digitalocean.com/products/images/)。 + +### Monitoring + +å¯ç”¨ DigitalOcean 代ç†ä»¥è¿›è¡Œé¢å¤–çš„[监控](https://docs.digitalocean.com/products/monitoring/)。 + +### IPv6 + +为 Droplet å¯ç”¨ IPv6。 + +### ç§æœ‰ç½‘络 + +为 Droplet å¯ç”¨ç§æœ‰ç½‘络。 + +### Droplet 标签 + +将标签（tag, label）应用于 Droplet。标签åªèƒ½åŒ…å«å­—æ¯ã€æ•°å­—ã€å†’å·ã€ç ´æŠ˜å·å’Œä¸‹åˆ’线。例如，`my_server`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md new file mode 100644 index 00000000000..56082ed497a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -0,0 +1,47 @@ +--- +title: EC2 节点模æ¿é…ç½® +--- + +有关 EC2 和节点的更多详细信æ¯ï¼Œè¯·å‚阅 [EC2 管ç†æŽ§åˆ¶å°](https://aws.amazon.com/ec2)的官方文档。 + +### 区域 + +在**区域**字段中，选择创建云凭è¯æ—¶ä½¿ç”¨çš„åŒä¸€åŒºåŸŸã€‚ + +### äº‘å‡­è¯ + +ä½ çš„ AWS 账户访问信æ¯ï¼Œå­˜å‚¨åœ¨[云凭è¯](../../../user-settings/manage-cloud-credentials.md)中。 + +请å‚阅 [Amazon 文档：创建访问密钥](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey)æ¥åˆ›å»ºè®¿é—®å¯†é’¥å’Œå¯†æ–‡å¯†é’¥ã€‚ + +请å‚阅 [Amazon 文档：创建 IAM 策略（控制å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start)æ¥åˆ›å»º IAM 策略。 + +请å‚阅 [Amazon 文档：为用户添加æƒé™ï¼ˆæŽ§åˆ¶å°ï¼‰](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console)了解如何绑定 IAM。 + +å‚阅下é¢çš„三个示例 JSON 策略： + +- [IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#iam-策略示例) +- [带有 PassRole çš„ IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#带有-passrole-çš„-iam-策略示例)（如果è¦ä½¿ç”¨ [Kubernetes 云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)，或将 IAM é…置文件传递给实例，则需è¦ï¼‰ +- [å…许用户加密 EBS å·çš„ IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#å…许加密-ebs-å·çš„-iam-策略示例) + +### 验è¯å’Œé…置节点 + +为集群选择å¯ç”¨åŒºå’Œç½‘络设置。 + +### 安全组 + +选择默认安全组或é…置安全组。 + +请å‚考[使用主机驱动时的 Amazon EC2 安全组](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-安全组)，了解 `rancher-nodes` 安全组中创建的规则。 + +如果你自行为 EC2 实例æ供安全组，Rancher ä¸ä¼šå¯¹å…¶è¿›è¡Œä¿®æ”¹ã€‚因此，你需è¦è®©ä½ çš„安全组å…许 [Rancher é…置实例所需的端å£](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rke-上-rancher-server-节点的端å£)。有关使用安全组控制 EC2 实例的入站和出站æµé‡çš„更多信æ¯ï¼Œè¯·å‚阅[这里](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#WorkingWithSecurityGroups)。 + +### 实例选项 + +é…ç½®è¦åˆ›å»ºçš„实例。确ä¿ä¸º AMI é…置正确的 **SSH 用户**。所选的区域å¯èƒ½ä¸æ”¯æŒé»˜è®¤å®žä¾‹ç±»åž‹ã€‚在这ç§æƒ…况下，你必须选择一个确实存在的实例类型。å¦åˆ™å°†å‡ºçŽ°é”™è¯¯ï¼Œè¡¨ç¤ºè¯·æ±‚çš„é…ç½®ä¸å—支æŒã€‚ + +如果需è¦ä¼ é€’ **IAM 示例é…ç½®å称**（ä¸æ˜¯ ARN），例如è¦ä½¿ç”¨ [Kubernetes 云æ供商](../../../../pages-for-subheaders/set-up-cloud-providers.md)时，策略则需è¦å…¶ä»–æƒé™ã€‚有关示例策略，请å‚阅[带有 PassRole çš„ IAM 策略示例](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#带有-passrole-çš„-iam-策略示例)。 + +### 引擎选项 + +在节点模æ¿çš„**引擎选项**中，你å¯ä»¥é…置容器 daemon。你å¯èƒ½éœ€è¦æŒ‡å®šå®¹å™¨ç‰ˆæœ¬æˆ–容器镜åƒä»“库 Mirror。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md new file mode 100644 index 00000000000..25531ce9a1c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -0,0 +1,33 @@ +--- +title: Azure 节点模æ¿é…ç½® +--- + +有关 Azure 的更多信æ¯ï¼Œè¯·å‚阅官方 [Azure 文档](https://docs.microsoft.com/en-us/azure/?product=featured)。 + +账户访问信æ¯å­˜å‚¨åœ¨äº‘凭è¯ä¸­ã€‚云凭è¯å­˜å‚¨åœ¨ Kubernetes 密文中。多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯ã€‚ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰çš„云凭è¯æˆ–创建新的凭è¯ã€‚ + +- **Placement** 设置托管集群的地ç†åŒºåŸŸä»¥åŠå…¶ä»–ä½ç½®å…ƒæ•°æ®ã€‚ +- **Network** é…置集群中使用的网络。 +- **Instance** 自定义你的 VM é…置。 + + :::note + + 如果在与 VM ä¸åŒçš„资æºç»„中使用 VNet，则 VNet å称需è¦ä»¥èµ„æºç»„å称作为å‰ç¼€ã€‚例如，`:`。 + + ::: + +如果你使用 Docker，[Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) é…置选项包括： + +- **标签**：有关标签的信æ¯ï¼Œè¯·å‚阅 [Docker 对象标签文档](https://docs.docker.com/config/labels-custom-metadata/)。 +- **Docker 引擎安装 URL**：确定è¦åœ¨å®žä¾‹ä¸Šå®‰è£…çš„ Docker 版本。 + + :::note + + 如果è¦é…ç½® Red Hat Enterprise Linux (RHEL) 或 CentOS 节点，请将 **Docker Install URL** 字段ä¿ç•™ä¸ºé»˜è®¤å€¼ï¼Œæˆ–选择 **none**。由于 Docker å·²ç»å®‰è£…在这些节点上，因此将绕过 Docker 安装检查。 + + 如果没有将 **Docker Install URL** 设置为默认值或 **none**，你å¯èƒ½ä¼šçœ‹åˆ°é”™è¯¯æ¶ˆæ¯ï¼š`Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories`。 + + ::: + +- **é•œåƒä»“库 mirror**：Docker daemon 使用的 Docker é•œåƒä»“库镜åƒã€‚ +- **其他高级选项**：å‚è§ [Docker daemon 选项å‚考](https://docs.docker.com/engine/reference/commandline/dockerd/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md new file mode 100644 index 00000000000..4c00d381982 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -0,0 +1,18 @@ +--- +title: DigitalOcean 节点模æ¿é…ç½® +--- + +账户访问信æ¯å­˜å‚¨åœ¨äº‘凭è¯ä¸­ã€‚云凭è¯å­˜å‚¨åœ¨ Kubernetes 密文中。多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯ã€‚ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰çš„云凭è¯æˆ–创建新的凭è¯ã€‚ + +### Droplet 选项 + +**Droplet 选项**用于é…置集群的地ç†åŒºåŸŸå’Œè§„范。 + +### Docker Daemon + +如果你使用 Docker，[Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) é…置选项包括： + +- **标签**：有关标签的信æ¯ï¼Œè¯·å‚阅 [Docker 对象标签文档](https://docs.docker.com/config/labels-custom-metadata/)。 +- **Docker 引擎安装 URL**：确定è¦åœ¨å®žä¾‹ä¸Šå®‰è£…çš„ Docker 版本。 +- **é•œåƒä»“库 mirror**：Docker daemon 使用的 Docker é•œåƒä»“库镜åƒã€‚ +- **其他高级选项**：å‚è§ [Docker daemon 选项å‚考](https://docs.docker.com/engine/reference/commandline/dockerd/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md new file mode 100644 index 00000000000..21f03714a21 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -0,0 +1,66 @@ +--- +title: Nutanix 节点模æ¿é…ç½® +--- + +## è´¦å·è®¿é—® + +| å‚æ•° | å¿…å¡« | æè¿° | 默认 | +|:-----------------------------|:--------:|:-----------------------------------------------------------------|:----- +| 管ç†ç«¯ç‚¹ | ✓ | Prism Central 的主机å/IP åœ°å€ | +| 用户å | ✓ | Prism Central 用户的用户å | +| å¯†ç  | ✓ | Prism Central ç”¨æˆ·çš„å¯†ç  | +| å…许ä¸å®‰å…¨çš„通信 | | 设置为 true 以å…许与 Prism Central 进行ä¸å®‰å…¨çš„ SSL 通信 | False | + +## 调度 + +选择虚拟机è¦è°ƒåº¦åˆ°å“ªä¸ª Nutanix 集群。 + +| å‚æ•° | å¿…å¡« | æè¿° | +|:----------|:--------:|:---------------------------------------------------------------------------- +| 集群 | ✓ | 部署虚拟机的 Nutanix 集群的å称（区分大å°å†™ï¼‰ | + +## 实例选项 + +在**实例å‚æ•°**中，é…置此模æ¿åˆ›å»ºçš„ VM çš„ vCPU æ•°é‡ã€å†…存和ç£ç›˜å¤§å°ã€‚ + +| å‚æ•° | å¿…å¡« | æè¿° | 默认 | +|:---------------------|:--------:|:--------------------------------------------------------------------------------------------|:------- +| CPU | | 分é…ç»™ VM çš„ vCPU æ•°é‡ï¼ˆæ ¸å¿ƒï¼‰ | 2 | +| 内存 | | 分é…ç»™ VM çš„ RAM é‡ (MB) | 2 GB | +| Template Image | ✓ | è¦ä½œä¸ºè™šæ‹Ÿæœºä¸»ç£ç›˜è¿›è¡Œå…‹éš†çš„ç£ç›˜é•œåƒæ¨¡æ¿çš„åç§°ï¼ˆå¿…é¡»æ”¯æŒ cloud-init） | +| 虚拟机ç£ç›˜å¤§å° | | 虚拟机主ç£ç›˜çš„新大å°ï¼ˆä»¥ GiB 为å•ä½ï¼‰ | +| 其他ç£ç›˜å¤§å° | | è¦æ·»åŠ åˆ°è™šæ‹Ÿæœºçš„其他ç£ç›˜çš„大å°ï¼ˆä»¥ GiB 为å•ä½ï¼‰ | +| 储存容器 | | è¦é…置其他ç£ç›˜çš„存储容器 _UUID_ | +| Cloud Config YAML | | è¦æ供给虚拟机的 cloud-init（将使用 Rancher root 用户修补） | +| 网络 | ✓ | è¦é™„加到虚拟机的网络的å称 | +| 虚拟机类别 | | è¦åº”用于虚拟机的类别å称 | + +虚拟机支æŒé€šè¿‡ [Config Drive v2 datasource](https://cloudinit.readthedocs.io/en/latest/reference/datasources/configdrive.html) æ¥æ”¯æŒ [cloud-init](https://cloudinit.readthedocs.io/en/latest/) 的任何现代 Linux æ“作系统。 + +## 网络 + +节点模æ¿å…许你为虚拟机é…置多个网络。在**网络**字段中，你å¯ä»¥å•å‡»**添加**，然åŽåœ¨ AOS 中添加任何å¯ç”¨çš„网络。 + +## 虚拟机类别 + +类别用于将实体分组æˆé”®å€¼å¯¹ã€‚通常会根æ®æŸäº›æ ‡å‡†ä¸ºè™šæ‹Ÿæœºåˆ†é…一个类别。然åŽï¼Œä½ å¯ä»¥å°†ç­–略绑定到分é…（分组）了特定类别值的实体。 + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) å…许你在首次å¯åŠ¨æ—¶åº”用é…置，从而åˆå§‹åŒ–节点。这å¯èƒ½æ¶‰åŠåˆ›å»ºç”¨æˆ·æˆ–æŽˆæƒ SSH 密钥之类的æ“作。 + +è¦ä½¿ç”¨ cloud-init åˆå§‹åŒ–，请将使用有效 YAML 语法的 cloud config 粘贴到 **Cloud Config YAML** 字段中。è¦èŽ·å–支æŒçš„ cloud config 指令的注释示例集，请å‚阅 [cloud-init 文档](https://cloudinit.readthedocs.io/en/latest/topics/examples.html)。 + +ä¸å»ºè®®ä½¿ç”¨åŸºäºŽ cloud-init 的网络é…置，仅支æŒä½¿ç”¨ç”¨æˆ·æ•°æ® `runcmd`，ä¸æ”¯æŒ NoCloud 或其他网络é…置数æ®æºã€‚ + +建议使用 Nutanix IP Address Management（IPAM) 或其他 DHCP æœåŠ¡ã€‚ + +## 引擎选项 + +在节点模æ¿çš„**引擎选项**中，你å¯ä»¥é…置容器 daemon。你å¯èƒ½éœ€è¦æŒ‡å®šå®¹å™¨ç‰ˆæœ¬æˆ–容器镜åƒä»“库 Mirror。 + +:::note +如果è¦é…ç½® Red Hat Enterprise Linux (RHEL) 或 CentOS 节点，请将 **Docker Install URL** 字段ä¿ç•™ä¸ºé»˜è®¤å€¼ï¼Œæˆ–选择 **none**。由于 Docker å·²ç»å®‰è£…在这些节点上，因此将绕过 Docker 安装检查。 + +如果没有将 **Docker Install URL** 设置为默认值或 **none**，你å¯èƒ½ä¼šçœ‹åˆ°é”™è¯¯æ¶ˆæ¯ï¼š`Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories`。 +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md new file mode 100644 index 00000000000..7ac1c106d1d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -0,0 +1,95 @@ +--- +title: vSphere 节点模æ¿é…ç½® +--- + +## è´¦å·è®¿é—® + +| å‚æ•° | å¿…å¡« | æè¿° | +|:----------------------|:--------:|:-----| +| äº‘å‡­è¯ | * | ä½ çš„ vSphere è´¦å·è®¿é—®ä¿¡æ¯ï¼Œå­˜å‚¨åœ¨[云凭è¯](../../../user-settings/manage-cloud-credentials.md)中。 | + +你的云凭è¯å…·æœ‰ä»¥ä¸‹å­—段： + +| 凭è¯å­—段 | æè¿° | +|-----------------|--------------| +| vCenter 或 ESXi Server | 输入 vCenter 或 ESXi 主机å/IP。ESXi 是你创建和è¿è¡Œè™šæ‹Ÿæœºå’Œè™šæ‹Ÿè®¾å¤‡çš„虚拟化平å°ã€‚ä½ å¯ä»¥é€šè¿‡ vCenter Server æœåŠ¡æ¥ç®¡ç†ç½‘络中连接的多个主机并池化主机资æºã€‚ | +| ç«¯å£ | é…ç½® vCenter 或 ESXi Server 的端å£ã€‚ | +| 用户åå’Œå¯†ç  | ä½ çš„ vSphere 登录用户å和密ç ã€‚ | + +## 调度 + +选择虚拟机è¦è°ƒåº¦åˆ°å“ªä¸ªè™šæ‹Ÿæœºç›‘控程åºã€‚ + +**调度**中的字段应使用 vSphere 中å¯ç”¨çš„æ•°æ®ä¸­å¿ƒå’Œå…¶ä»–计划选项自动填充。 + +| 字段 | å¿…å¡« | 解释 | +|---------|---------------|-----------| +| æ•°æ®ä¸­å¿ƒ | * | 选择è¦è°ƒåº¦ VM çš„æ•°æ®ä¸­å¿ƒçš„å称/路径。 | +| 资æºæ±  | | è¦åœ¨å…¶ä¸­è°ƒåº¦ VM 的资æºæ± å称。资æºæ± å¯å¯¹ç‹¬ç«‹ä¸»æœºæˆ–集群的å¯ç”¨ CPU 和内存资æºè¿›è¡Œåˆ†åŒºï¼Œä¹Ÿå¯ä»¥åµŒå¥—使用。如果是独立 ESXi，请留空。如果未指定，则使用默认资æºæ± ã€‚ | +| æ•°æ®å­˜å‚¨ | * | 如果你有数æ®å­˜å‚¨é›†ç¾¤ï¼Œåˆ™å¯ä»¥æ‰“å¼€**æ•°æ®å­˜å‚¨**字段。此字段å…许你选择è¦å°† VM 调度到哪个数æ®å­˜å‚¨é›†ç¾¤ã€‚如果该字段未打开，你å¯ä»¥é€‰æ‹©å•ä¸ªç£ç›˜ã€‚ | +| 文件夹 | | æ•°æ®ä¸­å¿ƒä¸­ç”¨äºŽåˆ›å»º VM 的文件夹的å称。必须已ç»å­˜åœ¨ã€‚此下拉èœå•ä¸­çš„ VM 文件夹直接对应于 vSphere 中的 VM 文件夹。在 vSphere é…置文件中，文件夹å称应以 `vm/` 开头。 | +| 主机 | | 用于调度 VM 的主机系统的 IP。如果是独立 ESXi 或具有 DRS（分布å¼èµ„æºè°ƒåº¦å™¨ï¼‰çš„集群，将此字段留空。如果指定，将使用主机系统的池，而忽略**资æºæ± **å‚数。 | + +## 实例选项 + +在**实例å‚æ•°**中，é…置此模æ¿åˆ›å»ºçš„ VM çš„ vCPU æ•°é‡ã€å†…存和ç£ç›˜å¤§å°ã€‚ + +| å‚æ•° | å¿…å¡« | æè¿° | +|:----------------|:--------:|:-----------| +| CPU | * | è¦åˆ†é…ç»™ VM çš„ vCPU æ•°é‡ã€‚ | +| 内存 | * | è¦åˆ†é…ç»™ VM 的内存é‡ã€‚ | +| ç£ç›˜ | * | è¦æŒ‚载到 VM çš„ç£ç›˜å¤§å°ï¼ˆä»¥ MB 为å•ä½ï¼‰ã€‚ | +| 创建方法 | * | 在节点上设置æ“作系统的方法。å¯ä»¥ä½¿ç”¨ ISO 或 VM 模æ¿å®‰è£…æ“作系统。根æ®åˆ›å»ºæ–¹æ³•ï¼Œä½ è¿˜å¿…须指定 VM 模æ¿ã€å†…容库ã€çŽ°æœ‰ VM 或 ISO。有关创建方法的详细信æ¯ï¼Œè¯·å‚阅 [VM 创建方法](#vm-创建方法)。 | +| Cloud Init | | `cloud-config.yml` 文件的 URL 或用于é…ç½® VM çš„ URL。此文件å…许进一步定制æ“作系统，例如网络é…ç½®ã€DNS æœåŠ¡å™¨æˆ–系统守护程åºã€‚æ“ä½œç³»ç»Ÿå¿…é¡»æ”¯æŒ `cloud-init`。 | +| 网络 | | è¦æŒ‚è½½ VM 的网络的å称。 | +| guestinfo é…ç½®å‚æ•° | | VM 的其他é…ç½®å‚数。这些å‚数对应 vSphere 控制å°ä¸­çš„[高级设置](https://kb.vmware.com/s/article/1016098)。示例用例包括æä¾› RancherOS [guestinfo](https://rancher.com/docs/os/v1.x/en/installation/cloud/vmware-esxi/#vmware-guestinfo) å‚数，或为 VM å¯ç”¨ç£ç›˜ UUID (`disk.EnableUUID=TRUE`)。 | + + +### VM 创建方法 + +在**创建方法**字段中，é…置用于在 vSphere 中é…ç½® VM 的方法。å¯ç”¨çš„选项包括创建从 RancherOS ISO å¯åŠ¨çš„ VM，或通过从现有虚拟机或 [VM 模æ¿](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-F7BF0E6B-7C4F-4E46-8BBF-76229AEA7220.html)克隆æ¥åˆ›å»º VM。 + +现有 VM 或模æ¿å¯ä»¥ä½¿ç”¨ä»»ä½•çŽ°ä»£ Linux æ“作系统，该æ“作系统é…置为使用 [NoCloud æ•°æ®æº](https://canonical-cloud-init.readthedocs-hosted.com/en/latest/reference/datasources/nocloud.html)æ¥æ”¯æŒ [cloud-init](https://cloudinit.readthedocs.io/en/latest/)。 + +选择创建 VM çš„æ–¹å¼ï¼š + +- **使用模æ¿éƒ¨ç½²ï¼šæ•°æ®ä¸­å¿ƒ**：选择存在于所选数æ®ä¸­å¿ƒçš„ VM 模æ¿ã€‚ +- **使用模æ¿éƒ¨ç½²ï¼šå†…容库**：首先选择包å«ä½ çš„模æ¿çš„[内容库](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-254B2CE8-20A8-43F0-90E8-3F6776C2C896.html)，然åŽä»Žå¡«å……列表**库模æ¿**中选择模æ¿ã€‚ +- **克隆现有的虚拟机**：在**虚拟机**字段中选择一个现有虚拟机，新虚拟机将克隆自该虚拟机。 +- **使用 boot2docker ISO 安装**ï¼šç¡®ä¿ **OS ISO URL** å­—æ®µåŒ…å« RancherOS çš„ VMware ISO 版本的 URL (`rancheros-vmware.iso`)。请注æ„，è¿è¡Œ Rancher Server 节点必须能访问该 URL。 + +## 网络 + +节点模æ¿å…许你为虚拟机é…置多个网络。在**网络**字段中，你å¯ä»¥å•å‡»**添加网络**æ¥æ·»åŠ  vSphere 中å¯ç”¨çš„任何网络。 + +## 节点标签和自定义属性 + +æ ‡ç­¾ç”¨äºŽå‘ vSphere 对象清å•ä¸­çš„对象添加元数æ®ï¼Œä»¥ä¾¿å¯¹å¯¹è±¡è¿›è¡ŒæŽ’åºå’Œæœç´¢ã€‚ + +你的所有 vSphere 标签都将显示为节点模æ¿ä¸­å¯ä¾›é€‰æ‹©çš„选项。 + +在自定义属性中，Rancher 会让你选择你已ç»åœ¨ vSphere 中设置的所有自定义属性。自定义属性是键，你å¯ä»¥ä¸ºæ¯ä¸ªå±žæ€§è¾“入值。 + +:::note + +自定义属性是一项旧版功能，最终将从 vSphere 中删除。 + +::: + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) å…许你在首次å¯åŠ¨æ—¶åº”用é…置，从而åˆå§‹åŒ–节点。这å¯èƒ½æ¶‰åŠåˆ›å»ºç”¨æˆ·ã€æŽˆæƒ SSH 密钥或设置网络之类的æ“作。 + +è¦ä½¿ç”¨ cloud-init åˆå§‹åŒ–，请使用有效的 YAML 语法创建一个 cloud config 文件，并将文件内容粘贴到 **Cloud Init** 字段中。è¦èŽ·å–支æŒçš„ cloud config 指令的注释示例集，请å‚阅 [cloud-init 文档](https://cloudinit.readthedocs.io/en/latest/topics/examples.html)。 + +请注æ„，使用 ISO 创建方法时ä¸æ”¯æŒ cloud-init。 + +## 引擎选项 + +在节点模æ¿çš„**引擎选项**中，你å¯ä»¥é…置容器 daemon。你å¯èƒ½éœ€è¦æŒ‡å®šå®¹å™¨ç‰ˆæœ¬æˆ–容器镜åƒä»“库 Mirror。 + +:::note +如果è¦é…ç½® Red Hat Enterprise Linux (RHEL) 或 CentOS 节点，请将 **Docker Install URL** 字段ä¿ç•™ä¸ºé»˜è®¤å€¼ï¼Œæˆ–选择 **none**。由于 Docker å·²ç»å®‰è£…在这些节点上，因此将绕过 Docker 安装检查。 + +如果没有将 **Docker Install URL** 设置为默认值或 **none**，你å¯èƒ½ä¼šçœ‹åˆ°é”™è¯¯æ¶ˆæ¯ï¼š`Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories`。 +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md new file mode 100644 index 00000000000..a296187be6b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -0,0 +1,224 @@ +--- +title: AKS 集群é…ç½®å‚考 +--- + +## Rancher 2.6 å˜æ›´ + +- 支æŒæ·»åŠ å¤šä¸ªèŠ‚点池 +- 支æŒç§æœ‰é›†ç¾¤ +- å¯ç”¨è‡ªåŠ¨ç¼©æ”¾èŠ‚点池 +- 在云凭è¯ä¸­é…ç½® AKS æƒé™ + +## RBAC + +在 Rancher UI 中é…ç½® AKS 集群时，无法ç¦ç”¨ RBAC。如果在 AKS 中为集群ç¦ç”¨äº† RBAC，则无法在 Rancher 中注册或导入集群。 + +Rancher å¯ä»¥ä½¿ç”¨ä¸Žå…¶ä»–集群一样的方å¼ä¸º AKS 集群é…ç½®æˆå‘˜è§’色。有关详细信æ¯ï¼Œè¯·å‚阅 [RBAC](../../../pages-for-subheaders/manage-role-based-access-control-rbac.md)。 + +## äº‘å‡­è¯ + +:::note + +本节中的é…置信æ¯å‡è®¾ä½ å·²ç»ä¸º Rancher 设置了æœåŠ¡ä¸»ä½“。有关如何设置æœåŠ¡ä¸»ä½“的分步说明，请å‚阅[本节](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md#microsoft-azure-中的先决æ¡ä»¶)。 + +::: + +### 订阅 ID + +è¦èŽ·å–订阅 ID，请å•å‡»å·¦ä¾§å¯¼èˆªæ ä¸­çš„ **All Services**。然åŽå•å‡» **Subscriptions**。转到è¦ä¸Ž Kubernetes 集群关è”的订阅的å称，然åŽå¤åˆ¶ **Subscription ID**。 + +### 客户端 ID + +è¦èŽ·å–客户端 ID，请转到 Azure 门户，然åŽå•å‡» **Azure Active Directory**，å•å‡» **App registrations**，然åŽå•å‡»æœåŠ¡ä¸»ä½“çš„å称。客户端 ID 在 app registration 详细信æ¯é¡µé¢ä¸Šåˆ—为 **Application (client) ID**。 + +### å®¢æˆ·ç«¯å¯†ç  + +在创建客户端密ç å€¼åŽï¼Œä½ æ— æ³•å†èŽ·å–它的值。因此如果你还没有客户端密ç å€¼ï¼Œåˆ™éœ€è¦åˆ›å»ºä¸€ä¸ªæ–°çš„客户端密ç ã€‚ + +è¦èŽ·å–新的客户端密ç ï¼Œè¯·è½¬åˆ° Azure 门户，然åŽå•å‡» **Azure Active Directory**，å•å‡» **App registrations**，然åŽå•å‡»æœåŠ¡ä¸»ä½“çš„å称。 + +然åŽç‚¹å‡» **Certificates & secrets** 并点击 **New client secret**。å•å‡» **Add**。然åŽå¤åˆ¶æ–°å®¢æˆ·ç«¯å¯†ç çš„ **Value**。 + +### 环境 + +Microsoft æ供了多个[云](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest)æ¥æ»¡è¶³åœ°åŒºæ³•å¾‹çš„è¦æ±‚： + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +## è´¦å·è®¿é—® + +在本部分中，你需è¦é€‰æ‹©çŽ°æœ‰çš„ Azure 云凭è¯æˆ–创建一个新凭è¯ã€‚ + +有关é…ç½® Azure 云凭è¯çš„帮助，请å‚阅[本部分](#云凭è¯)。 + +## 集群ä½ç½® + +é…置集群和节点ä½ç½®ã€‚有关 AKS å¯ç”¨åŒºçš„详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/availability-zones)。 + +高å¯ç”¨æ€§ä½ç½®åŒ…括多个å¯ç”¨åŒºã€‚ + +## 集群选项 + +### Kubernetes 版本 + +å¯ç”¨çš„ Kubernetes 版本是从 Azure API 动æ€èŽ·å–的。 + +### 集群资æºç»„ + +资æºç»„æ˜¯ä¸€ä¸ªå®¹å™¨ï¼Œå…¶ä¸­åŒ…å« Azure 解决方案的相关资æºã€‚资æºç»„å¯ä»¥åŒ…括解决方案的所有资æºï¼Œæˆ–者仅包括你希望作为一个组æ¥ç®¡ç†çš„资æºã€‚ä½ å¯ä»¥æ ¹æ®ç»„织的需求æ¥å†³å®šå¦‚何将资æºåˆ†é…给资æºç»„。通常情况下，你å¯ä»¥å°†ç”Ÿå‘½å‘¨æœŸç›¸åŒçš„资æºæ·»åŠ åˆ°åŒä¸€ä¸ªèµ„æºç»„，以便轻æ¾åœ°æŒ‰ç»„进行部署ã€æ›´æ–°å’Œåˆ é™¤ã€‚ + +ä½ å¯ä»¥ä½¿ç”¨çŽ°æœ‰èµ„æºç»„，也å¯ä»¥è¾“入资æºç»„çš„å称，然åŽç³»ç»Ÿå°†ä¸ºä½ åˆ›å»ºä¸€ä¸ªèµ„æºç»„。 + +使用包å«çŽ°æœ‰ AKS 集群的资æºç»„将会创建一个新资æºç»„。Azure AKS ä»…å…许æ¯ä¸ªèµ„æºç»„对应一个 AKS 集群。 + +有关管ç†èµ„æºç»„çš„ä¿¡æ¯ï¼Œè¯·å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal)。 + +### Linux 管ç†å‘˜ç”¨æˆ·å + +用于创建到 Linux 节点的 SSH 连接的用户å。 + +AKS 节点的默认用户å是 `azureuser`。 + +### SSH 公钥 + +用于创建到 Linux 节点的 SSH 连接的密钥。 + +### Tags + +如果你的组织使用标签（Tag）æ¥ç®¡ç†å¤šä¸ª Azure æœåŠ¡çš„资æºï¼Œé‚£ä¹ˆé›†ç¾¤æ ‡ç­¾åˆ™éžå¸¸æœ‰ç”¨ã€‚这些标签ä¸é€‚用于集群内的资æºã€‚ + +## 网络选项 + +### è´Ÿè½½å‡è¡¡å™¨ SKU + +Azure è´Ÿè½½å‡è¡¡å™¨æ”¯æŒ standard å’Œ basic SKU（stock keeping units）。 + +有关 standard è´Ÿè½½å‡è¡¡å™¨å’Œ basic è´Ÿè½½å‡è¡¡å™¨çš„对比，请å‚阅官方 [Azure 文档](https://docs.microsoft.com/en-us/azure/load-balancer/skus#skus)。Microsoft 建议使用 standard è´Ÿè½½å‡è¡¡å™¨ã€‚ + +如果你选择了一个或多个å¯ç”¨åŒºï¼Œæˆ–è€…ä½ æœ‰å¤šä¸ªèŠ‚ç‚¹æ± ï¼Œåˆ™éœ€è¦ Standard è´Ÿè½½å‡è¡¡å™¨ã€‚ + +### 网络策略 + +默认情况下，AKS 集群中的所有 Pod 都å¯ä»¥æ— é™åˆ¶åœ°å‘é€å’ŒæŽ¥æ”¶æµé‡ã€‚为了æ高安全性，你å¯ä»¥å®šä¹‰æŽ§åˆ¶æµé‡çš„规则。Kubernetes 中的网络策略功能å…许你定义集群中 pod 之间的入å£å’Œå‡ºå£æµé‡è§„则。 + +Azure æ供了两ç§å®žçŽ°ç½‘络策略的方法。创建 AKS 集群时会选择网络策略选项。创建集群åŽæ— æ³•æ›´æ”¹ç­–略选项： + +- Azure 自己的实现，称为 Azure 网络策略。Azure ç½‘ç»œç­–ç•¥éœ€è¦ Azure CNI。 +- Calico Network Policies，一个由 [Tigera](https://www.tigera.io/) 创立的开æºç½‘络和网络安全解决方案。 + +你也å¯ä»¥é€‰æ‹©ä¸ä½¿ç”¨ç½‘络策略。 + +有关 Azure å’Œ Calico 网络策略åŠå…¶åŠŸèƒ½ä¹‹é—´çš„差异，请å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/use-network-policies#differences-between-azure-and-calico-policies-and-their-capabilities)。 + +### DNS å‰ç¼€ +为集群的 Kubernetes API server FQDN 输入唯一的 DNS å‰ç¼€ã€‚ + +### 网络æ’件 +有两个网络æ’件，分别是 kubenet å’Œ Azure CNI。 + +[kubenet](https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet) Kubernetes æ’件是 AKS 创建的集群的默认é…置。使用 kubenet 时，集群中的æ¯ä¸ªèŠ‚点都会收到一个å¯è·¯ç”±çš„ IP 地å€ã€‚Pod 使用 NAT 与 AKS 集群外部的其他资æºè¿›è¡Œé€šä¿¡ã€‚è¿™ç§æ–¹æ³•å‡å°‘了需è¦åœ¨ç½‘络空间中ä¿ç•™ä»¥ä¾› Pod 使用的 IP 地å€æ•°é‡ã€‚ + +如果使用 Azure CNI（高级）网络æ’件，Pod å¯ä»¥ä½¿ç”¨å®Œæ•´çš„虚拟网络连接，并且å¯ä»¥ä»Žè¿žæŽ¥çš„网络中通过 pod çš„ç§æœ‰ IP 地å€ç›´æŽ¥è®¿é—®ã€‚这个æ’件需è¦æ›´å¤šçš„ IP 地å€ç©ºé—´ã€‚ + +有关 kubenet å’Œ Azure CNI 之间差异的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/concepts-network#compare-network-models)。 + +### HTTP 应用路由 + +å¯ç”¨åŽï¼ŒHTTP 应用路由附加组件å¯ä»¥æ›´è½»æ¾åœ°è®¿é—®éƒ¨ç½²åˆ° AKS 集群的应用。它部署了两个组件，分别是 [Kubernetes Ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress/) å’Œ [External-DNS](https://github.com/kubernetes-incubator/external-dns) controller。 + +有关详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/http-application-routing)。 + +### è®¾ç½®æŽˆæƒ IP 范围 + +ä½ å¯ä»¥ä½¿ç”¨[授æƒçš„ IP 地å€èŒƒå›´](https://docs.microsoft.com/en-us/azure/aks/api-server-authorized-ip-ranges#overview-of-api-server-authorized-ip-ranges)æ¥ä¿æŠ¤å¯¹ Kubernetes API server 的访问。 + +Kubernetes API server 公开 Kubernetes API。该组件æ供管ç†å·¥å…·ï¼ˆä¾‹å¦‚ kubectl）的交互。AKS æ供带有专用 API server çš„å•ç§Ÿæˆ·é›†ç¾¤ controlplane。默认情况下，API server 分é…了一个公共 IP 地å€ï¼Œä½ åº”该使用基于 Kubernetes 或 Azure çš„ RBAC æ¥æŽ§åˆ¶å¯¹ API server 的访问。 + +è¦ä¿æŠ¤å¯¹å…¶ä»–å¯å…¬å¼€çš„ AKS controlplane å’Œ API server 的访问，你å¯ä»¥å¯ç”¨å¹¶ä½¿ç”¨æŽˆæƒçš„ IP 范围。这些授æƒçš„ IP 范围åªå…许定义的 IP 地å€èŒƒå›´ä¸Ž API server 通信。 + +但是，å³ä½¿ä½ ä½¿ç”¨äº†æŽˆæƒçš„ IP 地å€èŒƒå›´ï¼Œä½ ä»åº”使用 Kubernetes RBAC 或 Azure RBAC æ¥æŽˆæƒç”¨æˆ·åŠå…¶è¯·æ±‚çš„æ“作。 + +### 容器监控 + +容器监控使用 Metrics API 从 Kubernetes 中å¯ç”¨çš„控制器ã€èŠ‚点和容器中收集内存和处ç†å™¨æŒ‡æ ‡ï¼Œä»Žè€Œä¸ºä½ å¯è§†åŒ–性能数æ®ã€‚容器日志也能被收集。å¯ç”¨ç›‘控åŽï¼Œç³»ç»Ÿä¼šé€šè¿‡ Linux çš„ Log Analytics 代ç†çš„容器化版本自动为你收集指标和日志。指标会被写入指标存储，而日志数æ®ä¼šè¢«å†™å…¥ä¸Žä½ çš„ [Log Analytics](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview) 工作区关è”的日志存储。 + +### Log Analytics 工作区资æºç»„ + +[资æºç»„](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups)åŒ…å« Log Analytics 工作区。你必须至少创建一个工作区æ‰èƒ½ä½¿ç”¨ Azure Monitor Logs。 + +### Log Analytics 工作区å称 + +Azure Monitor Logs 收集的数æ®å­˜å‚¨åœ¨ä¸€ä¸ªæˆ–多个 [Log Analytics 工作区中](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/design-logs-deployment)。工作区定义了数æ®çš„地ç†ä½ç½®ã€è®¿é—®æƒé™ï¼ˆå®šä¹‰äº†å“ªäº›ç”¨æˆ·å¯ä»¥è®¿é—®æ•°æ®ï¼‰ä»¥åŠé…置设置（定价层和数æ®ä¿ç•™ç­‰ï¼‰ã€‚ + +你必须至少创建一个工作区æ‰èƒ½ä½¿ç”¨ Azure Monitor Logs。一个工作区å¯èƒ½å°±è¶³ä»¥æ»¡è¶³ä½ çš„所有监控数æ®ã€‚你也å¯ä»¥æ ¹æ®éœ€æ±‚创建多个工作区。例如，你å¯èƒ½æœ‰ä¸€ä¸ªå·¥ä½œåŒºç”¨äºŽç”Ÿäº§æ•°æ®ï¼Œå¦ä¸€ä¸ªå·¥ä½œåŒºç”¨äºŽæµ‹è¯•ã€‚ + +有关 Azure Monitor Logs 的详细信æ¯ï¼Œè¯·å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-platform-logs)。 + +### 支æŒç§æœ‰ Kubernetes æœåŠ¡ + +通常情况下，无论集群是å¦ä¸ºç§æœ‰ï¼ŒAKS worker 节点都ä¸ä¼šèŽ·å¾—公共 IP。在ç§æœ‰é›†ç¾¤ä¸­ï¼Œcontrolplane 没有公共端点。 + +Rancher å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€è¿žæŽ¥åˆ°ç§æœ‰ AKS 集群。 + +第一ç§æ–¹æ³•æ˜¯ç¡®ä¿ Rancher è¿è¡Œåœ¨ä¸Ž AKS 节点相åŒçš„ [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) 上。 + +第二ç§æ–¹æ³•æ˜¯è¿è¡Œå‘½ä»¤å‘ Rancher 注册集群。é…置集群åŽï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èƒ½è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API 的地方è¿è¡Œæ˜¾ç¤ºçš„命令。é…ç½®å¯ç”¨äº†ç§æœ‰ API 端点的 AKS 集群时，此命令将显示在弹出窗å£ä¸­ã€‚ + +:::note + +注册现有 AKS 集群时，集群å¯èƒ½éœ€è¦ä¸€äº›æ—¶é—´ï¼ˆå¯èƒ½æ˜¯æ•°å°æ—¶ï¼‰æ‰ä¼šå‡ºçŽ°åœ¨ `Cluster To register` 下拉列表中。ä¸åŒåŒºåŸŸçš„结果å¯èƒ½ä¸åŒã€‚ + +::: + +有关连接到 AKS 专用集群的详细信æ¯ï¼Œè¯·å‚阅 [AKS 文档](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster)。 + +## 节点池 + +### æ¨¡å¼ + +在 Azure ç•Œé¢ä¸­ï¼Œç”¨æˆ·èƒ½å¤ŸæŒ‡å®šä¸»è¦èŠ‚点池（Primary Node Pool）ä¾èµ–于 `system`（通常用于 controlplane）还是 `user`（Rancher 最常用的）。 + +ä½ å¯ä»¥æŒ‡å®šä¸»è¦èŠ‚点池（Primary Node Pool）的模å¼ã€æ“作系统ã€æ•°é‡å’Œå¤§å°ã€‚ + +`system` 节点池总是需è¦è¿è¡ŒèŠ‚点，因此它们ä¸èƒ½ç¼©å®¹åˆ°ä¸€ä¸ªèŠ‚点以下。至少需è¦ä¸€ä¸ª `system` 节点池。 + +对于åŽç»­çš„节点池，Rancher UI 强制使用默认的 `user`。`user` 节点池å…许缩容到零节点。`user` 节点池ä¸è¿è¡Œ Kubernetes controlplane 的任何部分。 + +AKS ä¸ä¼šå…¬å¼€è¿è¡Œ Kubernetes controlplane 组件的节点。 + +### å¯ç”¨åŒº + +[å¯ç”¨åŒº](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview)是区域内的唯一物ç†ä½ç½®ã€‚æ¯ä¸ªå¯ç”¨åŒºç”±ä¸€ä¸ªæˆ–多个é…备独立电æºã€å†·å´ç³»ç»Ÿå’Œç½‘络的数æ®ä¸­å¿ƒç»„æˆã€‚ + +并éžæ‰€æœ‰åŒºåŸŸéƒ½æ”¯æŒå¯ç”¨åŒºã€‚有关具有å¯ç”¨åŒºçš„ Azure 区域列表，请å‚阅 [Azure 文档](https://docs.microsoft.com/en-us/azure/availability-zones/az-region#azure-regions-with-availability-zones)。 + +### è™šæ‹Ÿæœºå¤§å° + +为节点池中的æ¯ä¸ª VM 选择一个大å°ã€‚有关æ¯ä¸ª VM 大å°çš„详细信æ¯ï¼Œè¯·å‚阅[此页é¢](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/)。 + +### æ“作系统ç£ç›˜ç±»åž‹ + +节点池中的节点å¯ä»¥ä½¿ç”¨æ‰˜ç®¡ç£ç›˜æˆ–临时ç£ç›˜ã€‚ + +[临时 OS ç£ç›˜](https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks)在本地虚拟机存储上创建，并ä¸ä¼šä¿å­˜åˆ°è¿œç¨‹ Azure 存储。临时 OS ç£ç›˜é€‚用于无状æ€å·¥ä½œè´Ÿè½½ï¼Œå…¶ä¸­çš„应用å¯ä»¥å®¹å¿å•ä¸ª VM æ•…éšœï¼Œä½†æ›´å®¹æ˜“å— VM 部署时间或é‡ç½®å•ä¸ªè™šæ‹Ÿæœºå®žä¾‹é•œåƒçš„å½±å“。使用临时 OS ç£ç›˜ï¼Œä½ å¯ä»¥ä½“验更低的 OS ç£ç›˜è¯»/写延迟和更快的 VM é‡ç½®é•œåƒè¿‡ç¨‹ã€‚ + +[Azure 托管ç£ç›˜](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview)是由 Azure 管ç†å¹¶ä¸Ž Azure Virtual Machines 一起使用的å—级存储å·ã€‚托管ç£ç›˜æ—¨åœ¨å®žçŽ° 99.999% 的高å¯ç”¨æ€§ã€‚托管ç£ç›˜é€šè¿‡æ供三个数æ®å‰¯æœ¬æ¥å®žçŽ°é«˜å¯ç”¨æ€§å’Œé«˜æŒç»­æ€§ã€‚ + +### æ“作系统ç£ç›˜å¤§å° + +æ¯ä¸ªèŠ‚点的ç£ç›˜å¤§å°ï¼ˆä»¥ GB 为å•ä½ï¼‰ã€‚ + +### 节点数 +节点池中的节点数。[Azure 订阅](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits)å¯èƒ½ä¼šé™åˆ¶æœ€å¤§èŠ‚点数。 + +### æ¯ä¸ªèŠ‚点的最大 Pod æ•°é‡ +æ¯ä¸ªèŠ‚点的最大 Pod æ•°é‡é»˜è®¤ä¸º 110，最大为 250。 + +### å¯ç”¨è‡ªåŠ¨æ‰©ç¼©å®¹ + +å¯ç”¨è‡ªåŠ¨æ‰©ç¼©å®¹åŽï¼Œä½ éœ€è¦è¾“入最å°å’Œæœ€å¤§èŠ‚点数。 + +å¯ç”¨ Auto Scaling åŽï¼Œä½ å°†æ— æ³•æ‰‹åŠ¨å¯¹èŠ‚点池进行扩缩容。扩缩容由 AKS autoscaler 控制。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md new file mode 100644 index 00000000000..5de68eb41ae --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -0,0 +1,149 @@ +--- +title: EKS 集群é…ç½®å‚考 +--- + +### è´¦å·è®¿é—® + +使用获å–çš„ä¿¡æ¯ä¸º IAM 策略填写æ¯ä¸ªä¸‹æ‹‰åˆ—表和字段： + +| 设置 | æè¿° | +| ---------- | -------------------------------------------------------------------------------------------------------------------- | +| 区域 | 从下拉列表中选择构建集群的地ç†åŒºåŸŸã€‚ | +| äº‘å‡­è¯ | 选择为 IAM 策略创建的云凭è¯ã€‚有关在 Rancher 中创建云凭è¯çš„更多信æ¯ï¼Œè¯·å‚阅[此页é¢](../../user-settings/manage-cloud-credentials.md)。 | + +### æœåŠ¡è§’色 + +选择一个[æœåŠ¡è§’色](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html)。 + +| æœåŠ¡è§’色 | æè¿° | +-------------|--------------------------- +| Standard：Rancher 生æˆçš„æœåŠ¡è§’色 | 如果选择此角色，Rancher 会自动添加一个æœåŠ¡è§’色以供集群使用。 | +| 自定义：从现有的æœåŠ¡è§’色中选择 | 如果选择此角色，Rancher å°†å…许你从已在 AWS 中创建的æœåŠ¡è§’色中进行选择。有关在 AWS 中创建自定义æœåŠ¡è§’色的更多信æ¯ï¼Œè¯·å‚阅 [Amazon 文档](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html#create-service-linked-role)。 | + +### 密文加密 + +å¯é€‰ï¼šè¦åŠ å¯†å¯†æ–‡ï¼Œè¯·é€‰æ‹©æˆ–输入在 [AWS 密钥管ç†æœåŠ¡ (KMS)](https://docs.aws.amazon.com/kms/latest/developerguide/overview.html) 中创建的密钥。 + +### API Server 端点访问 + +é…置公共/ç§æœ‰ API 访问是一个高级用例。有关详细信æ¯ï¼Œè¯·å‚阅 [EKS 集群端点访问控制文档](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html)。 + +### 专用 API 端点 + +如果你在创建集群时å¯ç”¨äº†ç§æœ‰ API 端点访问，并ç¦ç”¨äº†å…¬å…± API 端点访问，那么你必须进行é¢å¤–的步骤æ‰èƒ½ä½¿ Rancher æˆåŠŸè¿žæŽ¥åˆ°é›†ç¾¤ã€‚在这ç§æƒ…况下，一个弹窗将会显示，其中包å«éœ€è¦åœ¨è¦æ³¨å†Œåˆ° Rancher 的集群上è¿è¡Œçš„命令。é…置集群åŽï¼Œä½ å¯ä»¥åœ¨ä»»ä½•èƒ½è¿žæŽ¥åˆ°é›†ç¾¤çš„ Kubernetes API 的地方è¿è¡Œæ˜¾ç¤ºçš„命令。 + +以下两ç§æ–¹æ³•èƒ½é¿å…这个é¢å¤–的手动步骤： +- 在创建集群时，创建具有ç§æœ‰å’Œå…¬å…± API 端点访问æƒé™çš„集群。在集群创建并处于 active 状æ€åŽï¼Œä½ å¯ä»¥ç¦ç”¨å…¬å…±è®¿é—®ï¼ŒRancher 将能继续与 EKS 集群通信。 +- ç¡®ä¿ Rancher 与 EKS 集群共享åŒä¸€ä¸ªå­ç½‘。然åŽï¼Œä½ å¯ä»¥ä½¿ç”¨å®‰å…¨ç»„使 Rancher 能够与集群的 API 端点进行通信。在这ç§æƒ…况下，你ä¸éœ€è¦è¿è¡Œæ³¨å†Œé›†ç¾¤çš„命令，Rancher 就能够与你的集群通信。有关é…置安全组的更多信æ¯ï¼Œè¯·å‚阅[安全组文档](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html)。 + +### 公共访问端点 + +你也å¯ä»¥é€‰æ‹©é€šè¿‡æ˜¾å¼ CIDR å—æ¥é™åˆ¶å¯¹å…¬å…±ç«¯ç‚¹çš„访问。 + +如果你é™åˆ¶å¯¹ç‰¹å®š CIDR å—的访问，那么建议你也å¯ç”¨ç§æœ‰è®¿é—®ï¼Œä»¥é¿å…丢失与集群的网络通信。 + +å¯ç”¨ç§æœ‰è®¿é—®éœ€è¦æ»¡è¶³ä»¥ä¸‹æ¡ä»¶ä¹‹ä¸€ï¼š +- Rancher çš„ IP 必须是å…许的 CIDR å—的一部分。 +- 应该å¯ç”¨äº†ç§æœ‰è®¿é—®ã€‚此外，Rancher 必须和集群共享åŒä¸€ä¸ªå­ç½‘，并对集群有网络访问æƒé™ã€‚ä½ å¯ä»¥é€šè¿‡å®‰å…¨ç»„æ¥è¿›è¡Œé…置。 + +有关对集群端点的公共和ç§æœ‰è®¿é—®çš„更多信æ¯ï¼Œè¯·å‚阅 [Amazon EKS 文档](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html)。 + +### å­ç½‘ + +| 选项 | æè¿° | +| ------- | ------------ | +| Standard：Rancher 生æˆçš„ VPC å’Œå­ç½‘ | 在é…置集群时，Rancher 会生æˆä¸€ä¸ªå…·æœ‰ 3 个公有å­ç½‘çš„æ–° VPC。 | +| 自定义：从现有的 VPC å’Œå­ç½‘中选择 | 在é…置集群时，Rancher 将你的 controlplane 和节点é…置为使用你已ç»[在 AWS 中创建](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html)çš„ VPC å’Œå­ç½‘。 | + +有关更多信æ¯ï¼Œè¯·å‚阅 AWS 文档以了解[集群 VPC 注æ„事项](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html)。根æ®ä½ åœ¨ä¸Šä¸€æ­¥ä¸­çš„选择，按照以下说明进行æ“作。 + +- [什么是 Amazon VPC？](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) +- [VPC å’Œå­ç½‘](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) + +### 安全组 + +Amazon 文档： + +- [集群安全组注æ„事项](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) +- [VPC 的安全组](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) +- [创建安全组](https://docs.aws.amazon.com/vpc/latest/userguide/getting-started-ipv4.html#getting-started-create-security-group) + +### Logging + +å°† controlplane 日志é…置为å‘é€åˆ° Amazon CloudWatch。如果你将集群日志å‘é€åˆ° CloudWatch Logs，你需è¦æŒ‰ç…§ standard CloudWatch Logs 支付数æ®å¼•å…¥å’Œå­˜å‚¨è´¹ç”¨ã€‚ + +æ¯ä¸ªæ—¥å¿—类型å‡å¯¹åº”一个 Kubernetes controlplane 组件。有关这些组件的更多信æ¯ï¼Œè¯·å‚阅 Kubernetes 文档中的 [Kubernetes 组件](https://kubernetes.io/docs/concepts/overview/components/)。 + +有关 EKS controlplane 日志管ç†çš„更多信æ¯ï¼Œè¯·å‚阅[官方文档](https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)。 + +### 托管节点组 + +Amazon EKS 托管的节点组自动为 Amazon EKS Kubernetes 集群的节点（Amazon EC2 实例）进行预置和生命周期管ç†ã€‚ + +有关节点组如何工作以åŠå¦‚何é…置的更多信æ¯ï¼Œè¯·å‚阅 [EKS 文档](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html)。 + +#### 使用你自己的å¯åŠ¨æ¨¡æ¿ + +ä½ å¯ä»¥æä¾›å¯åŠ¨æ¨¡æ¿ ID 和版本，以便轻æ¾é…置节点组中的 EC2 实例。如果你æ供了å¯åŠ¨æ¨¡æ¿ï¼Œåˆ™ä»¥ä¸‹è®¾ç½®éƒ½æ— æ³•åœ¨ Rancher 中进行é…置。因此，如果你使用å¯åŠ¨æ¨¡æ¿ï¼Œåˆ™éœ€è¦åœ¨å¯åŠ¨æ¨¡æ¿ä¸­æŒ‡å®šä»¥ä¸‹åˆ—表中的所有必须和所需的设置。å¦è¯·æ³¨æ„，如果æ供了å¯åŠ¨æ¨¡æ¿ ID 和版本，则åªèƒ½æ›´æ–°æ¨¡æ¿ç‰ˆæœ¬ã€‚如果è¦ä½¿ç”¨æ–°æ¨¡æ¿ ID，则需è¦åˆ›å»ºæ–°çš„托管节点组。 + +| 选项 | æè¿° | å¿…å¡«/选填 | +| ------ | ----------- | ----------------- | +| 实例类型 | 为è¦é…置的实例选择[硬件规格](https://aws.amazon.com/ec2/instance-types/)。 | å¿…å¡« | +| é•œåƒ ID | 为节点指定自定义 AMI。与 EKS 一起使用的自定义 AMI å¿…é¡»[正确é…ç½®](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/)。 | 选填 | +| 节点å·å¤§å° | å¯åŠ¨æ¨¡æ¿å¿…须指定具有所需大å°çš„ EBS å·ã€‚ | å¿…å¡« | +| SSH 密钥 | è¦æ·»åŠ åˆ°å®žä¾‹ä»¥å¯¹èŠ‚点进行 SSH 访问的密钥。 | 选填 | +| ç”¨æˆ·æ•°æ® | [MIME 多部分格å¼](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data)çš„ Cloud init 脚本。 | 选填 | +| 实例资æºæ ‡ç­¾ | 标记节点组中的æ¯ä¸ª EC2 实例。 | 选填 | + +#### Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿ + +如果你ä¸æŒ‡å®šå¯åŠ¨æ¨¡æ¿ï¼Œä½ å°†èƒ½å¤Ÿåœ¨ Rancher UI 中é…置上述选项，并且å¯ä»¥åœ¨åˆ›å»ºåŽæ›´æ–°æ‰€æœ‰è¿™äº›é€‰é¡¹ã€‚为了利用所有这些选项，Rancher 将为你创建和管ç†å¯åŠ¨æ¨¡æ¿ã€‚Rancher 中的所有集群都将有一个 Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿ã€‚此外，æ¯ä¸ªæ²¡æœ‰æŒ‡å®šå¯åŠ¨æ¨¡æ¿çš„托管节点组都将具有一个管ç†çš„å¯åŠ¨æ¨¡æ¿ç‰ˆæœ¬ã€‚æ­¤å¯åŠ¨æ¨¡æ¿çš„å称将具有 “rancher-managed-lt-†å‰ç¼€ï¼ŒåŽé¢æ˜¯é›†ç¾¤çš„显示å称。此外，Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿å°†ä½¿ç”¨ “rancher-managed-template†键和 “do-not-modify-or-delete†值æ¥è¿›è¡Œæ ‡è®°ï¼Œä»¥å°†å…¶è¯†åˆ«ä¸º Rancher 管ç†çš„å¯åŠ¨æ¨¡æ¿ã€‚请注æ„，ä¸è¦ä¿®æ”¹æˆ–删除此å¯åŠ¨æ¨¡æ¿ï¼Œæˆ–将此å¯åŠ¨æ¨¡æ¿ä¸Žå…¶ä»–集群或托管节点组一起使用。因为这å¯èƒ½ä¼šä½¿ä½ çš„节点组“é™çº§â€å¹¶éœ€è¦é”€æ¯å’Œé‡æ–°åˆ›å»ºã€‚ + +#### 自定义 AMI + +如果你在å¯åŠ¨æ¨¡æ¿æˆ– Rancher 中指定了自定义 AMI，则必须[正确é…ç½®](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/)é•œåƒï¼Œå¹¶ä¸”å¿…é¡»æ供用户数æ®ä»¥[引导节点](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami)。这是一个高级用例，因此你必须è¦äº†è§£å…¶è¦æ±‚。 + +如果你指定了ä¸åŒ…å«è‡ªå®šä¹‰ AMI çš„å¯åŠ¨æ¨¡æ¿ï¼Œåˆ™ Amazon 将为 Kubernetes 版本和所选区域使用 [EKS 优化的 AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html)。你还å¯ä»¥ä¸ºèƒ½ä»Žä¸­å—益的工作负载选择[å¯ç”¨ GPU 的实例](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html#gpu-ami)。 + +:::note + +如果你在下拉èœå•æˆ–å¯åŠ¨æ¨¡æ¿ä¸­æ供了自定义 AMI，则会忽略 Rancher 中设置的å¯ç”¨ GPU 的实例。 + +::: + +#### Spot 实例 + +Spot 实例现在[å— EKS 支æŒ](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types-spot)。如果你指定了å¯åŠ¨æ¨¡æ¿ï¼ŒAmazon 建议ä¸è¦åœ¨æ¨¡æ¿ä¸­æ供实例类型。相å，Amazon 建议æ供多ç§å®žä¾‹ç±»åž‹ã€‚如果你为节点组å¯ç”¨äº†â€œè¯·æ±‚ Spot 实例â€å¤é€‰æ¡†ï¼Œé‚£ä¹ˆä½ å°†æœ‰æœºä¼šæ供多ç§å®žä¾‹ç±»åž‹ã€‚ + +:::note + +在这ç§æƒ…况下，你在实例类型下拉列表中所选的选项都将被忽略，你必须在“Spot 实例类型â€ä¸­è‡³å°‘指定一ç§å®žä¾‹ç±»åž‹ã€‚此外，与 EKS 一起使用的å¯åŠ¨æ¨¡æ¿æ— æ³•è¯·æ±‚ Spot 实例。请求 Spot 实例必须是 EKS é…置的一部分。 + +::: + +#### 节点组设置 + +以下设置也是å¯é…置的。在创建节点组åŽï¼Œé™¤â€œèŠ‚点组å称â€å¤–的所有选项都是å¯ç¼–辑的。 + +| 选项 | æè¿° | +| ------- | ------------ | +| 节点组å称 | 节点组的å称。 | +| 期望 ASG å¤§å° | 期望的实例数é‡ã€‚ | +| 最大 ASG å¤§å° | 最大的实例数é‡ã€‚在安装 [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) 之å‰ï¼Œæ­¤è®¾ç½®ä¸ä¼šç”Ÿæ•ˆã€‚ | +| æœ€å° ASG å¤§å° | 最å°çš„实例数é‡ã€‚在安装 [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) 之å‰ï¼Œæ­¤è®¾ç½®ä¸ä¼šç”Ÿæ•ˆã€‚ | +| Labels | 应用于管ç†çš„节点组中节点的 Kubernetes 标签。 | +| Tags | 管ç†çš„节点组的标签，这些标签ä¸ä¼šä¼ æ’­åˆ°ä»»ä½•ç›¸å…³èµ„æºã€‚ | + + +### é…置刷新间隔 + +`eks-refresh-cron` 设置已弃用。它已è¿ç§»åˆ° `eks-refresh` 设置，这是一个表示秒的整数。 + +默认值为 300 秒。 + +ä½ å¯ä»¥é€šè¿‡è¿è¡Œ `kubectl edit setting eks-refresh` æ¥æ›´æ”¹åŒæ­¥é—´éš”。 + +如果之å‰è®¾ç½®äº† `eks-refresh-cron` 设置，è¿ç§»å°†è‡ªåŠ¨è¿›è¡Œã€‚ + +刷新窗å£è¶ŠçŸ­ï¼Œäº‰ç”¨æ¡ä»¶å‘生的å¯èƒ½æ€§å°±è¶Šå°ã€‚但这确实增加了é‡åˆ° AWS API å¯èƒ½å­˜åœ¨çš„请求é™åˆ¶çš„å¯èƒ½æ€§ã€‚ + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md new file mode 100644 index 00000000000..ff4638fc589 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -0,0 +1,49 @@ +--- +title: ç§æœ‰é›†ç¾¤ +--- + +在 GKE 中，[ç§æœ‰é›†ç¾¤](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept)是一ç§é›†ç¾¤ï¼Œå…¶èŠ‚点仅通过分é…内部 IP 地å€ä¸Žå…¥ç«™å’Œå‡ºç«™æµé‡ç›¸éš”离。GKE 中的ç§æœ‰é›†ç¾¤å¯ä»¥é€‰æ‹©å°† controlplane 端点作为公开访问的地å€æˆ–作为ç§æœ‰åœ°å€ã€‚这与其他 Kubernetes æ供商ä¸åŒï¼ŒåŽè€…å¯èƒ½å°†å…·æœ‰ç§æœ‰ controlplane 端点的集群称为“ç§æœ‰é›†ç¾¤â€ï¼Œä½†ä»å…许进出节点的æµé‡ã€‚基于你的组织的网络和安全è¦æ±‚，你å¯èƒ½æƒ³åˆ›å»ºä¸€ä¸ªæœ‰ç§æœ‰èŠ‚点的集群，其中有或没有公共 controlplane 端点。从 Rancher é…置的 GKE 集群å¯ä»¥é€šè¿‡åœ¨**集群选项**中选择**ç§æœ‰é›†ç¾¤**（在**显示高级选项**下）æ¥ä½¿ç”¨éš”离的节点。通过选择**å¯ç”¨ç§æœ‰ç«¯ç‚¹**，å¯ä»¥é€‰æ‹©å°† controlplane 端点设为ç§æœ‰ã€‚ + +### ç§æœ‰èŠ‚点 + +由于ç§æœ‰é›†ç¾¤ä¸­çš„节点åªæœ‰å†…部 IP 地å€ï¼Œå®ƒä»¬å°†æ— æ³•å®‰è£… cluster agent，Rancher 将无法完全管ç†é›†ç¾¤ã€‚è¿™å¯ä»¥é€šè¿‡å‡ ç§æ–¹å¼æ¥å¤„ç†ã€‚ + +#### Cloud NAT + +:::caution + +Cloud NAT å°†[产生费用](https://cloud.google.com/nat/pricing)。 + +::: + +如果é™åˆ¶å¤–出的互è”网访问对你的组织æ¥è¯´ä¸æ˜¯ä¸€ä¸ªé—®é¢˜ï¼Œå¯ä»¥ä½¿ç”¨ Google çš„ [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) æœåŠ¡æ¥å…许ç§æœ‰ç½‘络中的节点访问互è”网，使它们能够从 Dockerhub 下载所需的镜åƒå¹¶ä¸Ž Rancher management server 通信。这是最简å•çš„解决方案。 + +#### ç§æœ‰é•œåƒä»“库 + +:::caution + +此方案ä¸å—官方支æŒã€‚如果 Cloud NAT æœåŠ¡ä¸è¶³ä»¥æ»¡è¶³ä½ çš„需求，则å¯ä»¥å‚考此方案。 + +::: + +如果è¦æ±‚é™åˆ¶èŠ‚点的传入和传出æµé‡ï¼Œè¯·æŒ‰ç…§ç¦»çº¿å®‰è£…说明，在集群所在的 VPC 上设置一个ç§æœ‰å®¹å™¨[é•œåƒä»“库](https://rancher.com/docs/rancher/v2.6/en/installation/other-installation-methods/air-gap/)，从而å…许集群节点访问和下载è¿è¡Œ cluster agent 所需的镜åƒã€‚如果 controlplane 端点也是ç§æœ‰çš„，Rancher 将需è¦[直接访问](#直接访问)它。 + +### ç§æœ‰ controlplane 端点 + +如果集群暴露了公共端点，Rancher 将能够访问集群，且无需执行é¢å¤–çš„æ­¥éª¤ã€‚ä½†æ˜¯ï¼Œå¦‚æžœé›†ç¾¤æ²¡æœ‰å…¬å…±ç«¯ç‚¹ï¼Œåˆ™å¿…é¡»ç¡®ä¿ Rancher å¯ä»¥è®¿é—®é›†ç¾¤ã€‚ + +#### Cloud NAT + +:::caution + +Cloud NAT å°†[产生费用](https://cloud.google.com/nat/pricing)。 + +::: + +如上所述，如果ä¸è€ƒè™‘é™åˆ¶å¯¹èŠ‚点的传出互è”网访问，则å¯ä»¥ä½¿ç”¨ Google çš„ [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) æœåŠ¡æ¥å…许节点访问互è”网。当集群进行é…置时，Rancher å°†æ供一个在集群上è¿è¡Œçš„注册命令。下载新集群的 [kubeconfig](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) 并在集群上è¿è¡Œæ供的 kubectl 命令。如果è¦é€šè¿‡èŽ·å–集群访问æƒæ¥è¿è¡Œæ­¤å‘½ä»¤ï¼Œä½ å¯ä»¥åˆ›å»ºä¸´æ—¶èŠ‚点或使用 VPC 中的现有节点，或者登录到æŸä¸ªé›†ç¾¤èŠ‚点或使用æŸä¸ªé›†ç¾¤èŠ‚点创建 SSH 隧é“。 + +#### 直接访问 + +如果 Rancher server 与集群的 controlplane è¿è¡Œåœ¨åŒä¸€ VPC 上，它将直接访问 controlplane çš„ç§æœ‰ç«¯ç‚¹ã€‚集群节点将需è¦è®¿é—®[ç§æœ‰é•œåƒä»“库](#ç§æœ‰é•œåƒä»“库)以下载上述的镜åƒã€‚ + +你还å¯ä»¥ä½¿ç”¨ Google çš„æœåŠ¡ï¼ˆä¾‹å¦‚ [Cloud VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/overview) 或 [Cloud Interconnect VLAN](https://cloud.google.com/network-connectivity/docs/interconnect)）æ¥ä¿ƒè¿›ç»„织网络与 Google VPC 之间的连接。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md new file mode 100644 index 00000000000..507f93870bc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md @@ -0,0 +1,151 @@ +--- +title: K3s 集群é…ç½®å‚考 +--- + +æœ¬æ–‡ä»‹ç» Rancher 中å¯ç”¨äºŽæ–°çš„或现有的 K3s Kubernetes 集群的é…置选项。 + +## 概述 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€æ¥é…ç½® Kubernetes 选项： + +- [Rancher UI](#rancher-ui-中的é…置选项)：使用 Rancher UI æ¥é€‰æ‹©è®¾ç½® Kubernetes 集群时常用的自定义选项。 +- [集群é…置文件](#集群é…置文件)：高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª K3s é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å…许你设置 K3s 安装中å¯ç”¨çš„任何[选项](https://rancher.com/docs/k3s/latest/en/installation/install-options/)。 + +## Rancher UI 中的é…置选项 + +:::tip + +一些高级é…置选项没有在 Rancher UI 表å•ä¸­å¼€æ”¾ï¼Œä½†ä½ å¯ä»¥é€šè¿‡åœ¨ YAML 中编辑 K3s 集群é…置文件æ¥å¯ç”¨è¿™äº›é€‰é¡¹ã€‚有关 YAML 中 K3s 集群的å¯é…置选项的完整å‚考，请å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/installation/install-options/)。 + +::: + +### åŸºæœ¬ä¿¡æ¯ +#### Kubernetes 版本 + +这指的是集群节点上安装的 Kubernetes 版本。Rancher 基于 [hyperkube](https://github.com/rancher/hyperkube) 打包了自己的 Kubernetes 版本。 + +有关更多详细信æ¯ï¼Œè¯·å‚阅[å‡çº§ Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 + +#### 加密密文 + +å¯ç”¨æˆ–ç¦ç”¨å¯†æ–‡åŠ å¯†çš„选项。å¯ç”¨åŽï¼Œå¯†æ–‡å°†ä½¿ç”¨ AES-CBC 密钥进行加密。如果ç¦ç”¨ï¼Œåˆ™åœ¨å†æ¬¡å¯ç”¨åŠ å¯†ä¹‹å‰æ— æ³•è¯»å–任何以å‰çš„密文。有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/advanced/#secrets-encryption-config-experimental)。 + +#### 项目网络隔离 + +如果你的网络æ供商å…许项目网络隔离，你å¯ä»¥é€‰æ‹©å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´çš„通信。 + +#### SELinux + +å¯ç”¨æˆ–ç¦ç”¨ [SELinux](https://rancher.com/docs/k3s/latest/en/advanced/#selinux-support) 支æŒçš„选项。 + +#### CoreDNS + +默认情况下，[CoreDNS](https://coredns.io/) 会安装为默认 DNS æ供程åºã€‚如果未安装 CoreDNS，则必须自己安装备用 DNS æ供程åºã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/networking/#coredns)。 + +#### Klipper Service LB + +å¯ç”¨æˆ–ç¦ç”¨ [Klipper](https://github.com/rancher/klipper-lb) æœåŠ¡è´Ÿè½½å‡è¡¡å™¨çš„选项。有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer)。 + +#### Traefik Ingress + +å¯ç”¨æˆ–ç¦ç”¨ [Traefik](https://traefik.io/) HTTP åå‘代ç†å’Œè´Ÿè½½å‡è¡¡å™¨çš„选项。有关更多详细信æ¯å’Œé…置选项，请å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/networking/#traefik-ingress-controller)。 + +#### Local Storage + +在节点上å¯ç”¨æˆ–ç¦ç”¨ [Local Storage](https://rancher.com/docs/k3s/latest/en/storage/) 的选项。 + +#### Metrics Server + +å¯ç”¨æˆ–ç¦ç”¨ [metrics server](https://github.com/kubernetes-incubator/metrics-server) 的选项。如果å¯ç”¨ï¼Œè¯·ç¡®ä¿ä¸ºå…¥ç«™ TCP æµé‡æ‰“å¼€ 10250 端å£ã€‚ + +### 附加é…ç½® + +集群å¯åŠ¨æ—¶å°†åº”用的其他 Kubernetes 清å•ï¼Œä¼šä½œä¸º[附加组件](https://kubernetes.io/docs/concepts/cluster-administration/addons/)æ¥ç®¡ç†ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/helm/#automatically-deploying-manifests-and-helm-charts)。 + +### Agent 环境å˜é‡ + +为 [K3s agent](https://rancher.com/docs/k3s/latest/en/architecture/) 设置环境å˜é‡çš„选项。你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/)。 + +### etcd + +#### 自动快照 + +å¯ç”¨æˆ–ç¦ç”¨å®šæœŸ etcd 快照的选项。如果å¯ç”¨ï¼Œç”¨æˆ·å¯ä»¥é…置快照的频率。有关详细信æ¯ï¼Œè¯·å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/backup-restore/#creating-snapshots)。 + +#### 指标 + +选择å‘公众公开或仅在集群内公开 etcd 指标的选项。 + +### 网络 + +#### 集群 CIDR + +用于 pod IP çš„ IPv4/IPv6 网络 CIDR（默认：10.42.0.0/16）。 + +#### Service CIDR + +用于 Service IP çš„ IPv4/IPv6 网络 CIDR（默认：10.43.0.0/16）。 + +#### 集群 DNS + +用于 coredns æœåŠ¡çš„ IPv4 集群 IP。应该在你的 service-cidr 范围内（默认：10.43.0.10）。 + +#### 集群域å + +选择集群的域。默认值为 `cluster.local`。 + +#### NodePort æœåŠ¡ç«¯å£èŒƒå›´ + +更改å¯ç”¨äºŽ [NodePort æœåŠ¡](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)的端å£èŒƒå›´çš„选项。默认值为 `30000-32767`。 + +#### 截断主机å + +将主机å截断为 15 个或更少字符。该字段åªèƒ½åœ¨é›†ç¾¤åˆå§‹åˆ›å»ºæ—¶è®¾ç½®ã€‚创建集群åŽï¼Œä½ æ— æ³•å†å¯ç”¨æˆ–ç¦ç”¨ 15 个字符的é™åˆ¶ã€‚ + +此设置仅影å“é…置了机器的集群。由于自定义集群在创建节点时设置主机å（å‘生在 Rancher 之外），因此该字段ä¸é™åˆ¶è‡ªå®šä¹‰é›†ç¾¤çš„主机å长度。 + +截断集群中的主机åå¯æ高与基于 Windows 的系统的兼容性。虽然 Kubernetes å…许最长的主机å长度为 63 个字符，但使用 NetBIOS 的系统将主机åé™åˆ¶ä¸º 15 个字符之内。 + +#### TLS å¯é€‰å称 + +在æœåŠ¡å™¨ TLS è¯ä¹¦ä¸Šæ·»åŠ å…¶ä»–主机å或 IPv4/IPv6 地å€ä½œä¸º Subject Alternative Name。 + +#### 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯ç”¨äºŽç›´æŽ¥è®¿é—® Kubernetes API server，而无需通过 Rancher 进行通信。 + +有关授æƒé›†ç¾¤ç«¯ç‚¹çš„工作原ç†ä»¥åŠä½¿ç”¨çš„原因，请å‚阅[架构介ç»](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +### é•œåƒä»“库 + +选择è¦ä»Žä¸­æ‹‰å– Rancher é•œåƒçš„é•œåƒä»“库。有关更多详细信æ¯å’Œé…置选项，请å‚阅 [K3s 文档](https://rancher.com/docs/k3s/latest/en/installation/private-registry/)。 + +### å‡çº§ç­–ç•¥ + +#### controlplane å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### Worker å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### 清空节点（controlplane） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +#### 清空节点（worker 节点） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +### 高级é…ç½® + +为ä¸åŒèŠ‚点设置 kubelet 选项。有关å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)。 + +## 集群é…置文件 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª K3s é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI 表å•æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å…许你设置 K3s 安装中å¯ç”¨çš„任何[选项](https://rancher.com/docs/k3s/latest/en/installation/install-options/)。 + +è¦ç›´æŽ¥ä»Ž Rancher UI 编辑 K3s é…置文件，å•å‡»**以 YAML 文件编辑**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md new file mode 100644 index 00000000000..7cdc1ec7c64 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -0,0 +1,361 @@ +--- +title: RKE 集群é…ç½®å‚考 +--- + +Rancher 安装 Kubernetes 时，它使用 [RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 或 [RKE2](https://docs.rke2.io/) 作为 Kubernetes å‘行版。 + +æœ¬æ–‡ä»‹ç» Rancher 中å¯ç”¨äºŽæ–°çš„或现有的 RKE Kubernetes 集群的é…置选项。 + + +## 概述 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€æ¥é…ç½® Kubernetes 选项： + +- [Rancher UI](#rancher-ui-中的é…置选项)：使用 Rancher UI æ¥é€‰æ‹©è®¾ç½® Kubernetes 集群时常用的自定义选项。 +- [集群é…置文件](#rke-集群é…置文件å‚考)：高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ ä½¿ç”¨ YAML æ¥æŒ‡å®š RKE 安装中å¯ç”¨çš„任何选项（除了 system_images é…置）。 + +RKE 集群é…置选项嵌套在 `rancher_kubernetes_engine_config` å‚数下。有关详细信æ¯ï¼Œè¯·å‚阅[集群é…置文件](#rke-集群é…置文件å‚考)。 + +在 [RKE å¯åŠ¨çš„集群](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md)中，你å¯ä»¥ç¼–辑任何åŽç»­å‰©ä½™çš„选项。 + +有关 RKE é…置文件语法的示例，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/example-yamls/)。 + +Rancher UI 中的表å•ä¸åŒ…括é…ç½® RKE 的所有高级选项。有关 YAML 中 RKE Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +## 在 Rancher UI 中使用表å•ç¼–辑集群 + +è¦ç¼–辑你的集群： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦é…置的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 + + +## 使用 YAML 编辑集群 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ ä½¿ç”¨ YAML æ¥æŒ‡å®š RKE 安装中å¯ç”¨çš„任何选项（除了 system_images é…置）。 + +RKE 集群（也称为 RKE1 集群）的编辑方å¼ä¸Ž RKE2 å’Œ K3s 集群ä¸åŒã€‚ + +è¦ç›´æŽ¥ä»Ž Rancher UI 编辑 RKE é…置文件： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到è¦é…置的 RKE 集群。å•å‡»å¹¶å•å‡» **â‹® > 编辑é…ç½®**。你将会转到 RKE é…置表å•ã€‚请注æ„，由于集群é…置在 Rancher 2.6 中å‘生了å˜æ›´ï¼Œ**â‹® > 以 YAML 文件编辑**å¯ç”¨äºŽé…ç½® RKE2 集群，但ä¸èƒ½ç”¨äºŽç¼–辑 RKE1 é…置。 +1. 在é…置表å•ä¸­ï¼Œå‘下滚动并å•å‡»**以 YAML 文件编辑**。 +1. 编辑 `rancher_kubernetes_engine_config` å‚数下的 RKE 选项。 + +## Rancher UI 中的é…置选项 + +:::tip + +一些高级é…置选项没有在 Rancher UI 表å•ä¸­å¼€æ”¾ï¼Œä½†ä½ å¯ä»¥é€šè¿‡åœ¨ YAML 中编辑 RKE 集群é…置文件æ¥å¯ç”¨è¿™äº›é€‰é¡¹ã€‚有关 YAML 中 RKE Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +::: + +### Kubernetes 版本 + +这指的是集群节点上安装的 Kubernetes 版本。Rancher 基于 [hyperkube](https://github.com/rancher/hyperkube) 打包了自己的 Kubernetes 版本。 + +有关更多详细信æ¯ï¼Œè¯·å‚阅[å‡çº§ Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 + +### 网络æ供商 + +这指的是集群使用的[网络æ供商](https://kubernetes.io/docs/concepts/cluster-administration/networking/)。有关ä¸åŒç½‘络æ供商的更多详细信æ¯ï¼Œè¯·æŸ¥çœ‹æˆ‘们的[网络常è§é—®é¢˜è§£ç­”](../../../faq/container-network-interface-providers.md)。 + +:::caution + +å¯åŠ¨é›†ç¾¤åŽï¼Œä½ æ— æ³•æ›´æ”¹ç½‘络æ供商。由于 Kubernetes ä¸å…许在网络æ供商之间切æ¢ï¼Œå› æ­¤ï¼Œè¯·è°¨æ…Žé€‰æ‹©è¦ä½¿ç”¨çš„网络æ供商。使用网络æ供商创建集群åŽï¼Œå¦‚果你需è¦æ›´æ”¹ç½‘络æ供商，你将需è¦æ‹†é™¤æ•´ä¸ªé›†ç¾¤ä»¥åŠå…¶ä¸­çš„所有应用。 + +::: + +Rancher 与以下开箱å³ç”¨çš„网络æ供商兼容： + +- [Canal](https://github.com/projectcalico/canal) +- [Flannel](https://github.com/coreos/flannel#flannel) +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Weave](https://github.com/weaveworks/weave) + +:::note Weave 注æ„事项： + +选择 Weave 作为网络æ供商时，Rancher 将通过生æˆéšæœºå¯†ç æ¥è‡ªåŠ¨å¯ç”¨åŠ å¯†ã€‚如果你想手动指定密ç ï¼Œè¯·å‚阅使用[é…置文件](#rke-集群é…置文件å‚考)å’Œ [Weave 网络æ’件选项](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/#weave-network-plug-in-options)æ¥é…置集群。 + +::: + +### 项目网络隔离 + +如果你的网络æ供商å…许项目网络隔离，你å¯ä»¥é€‰æ‹©å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´çš„通信。 + +如果你使用支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE 网络æ’件（例如 Canal 或 Cisco ACI æ’件），则å¯ä»¥ä½¿ç”¨é¡¹ç›®ç½‘络隔离。 + +### Kubernetes 云æ供商 + +ä½ å¯ä»¥é…ç½® [Kubernetes 云æ供商](../../../pages-for-subheaders/set-up-cloud-providers.md)。如果你想在 Kubernetes 中使用动æ€é…置的[å·å’Œå­˜å‚¨](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md)，你通常需è¦é€‰æ‹©ç‰¹å®šçš„云æ供商。例如，如果你想使用 Amazon EBS，则需è¦é€‰æ‹© `aws` 云æ供商。 + +:::note + +如果你è¦ä½¿ç”¨çš„云æ供商未作为选项列出，你需è¦ä½¿ç”¨[é…置文件选项](#rke-集群é…置文件å‚考)æ¥é…置云æ供商。请å‚考 [RKE 云æ供商文档](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/)æ¥äº†è§£å¦‚何é…置云æ供商。 + +::: + +### ç§æœ‰é•œåƒä»“库 + +集群级别的ç§æœ‰é•œåƒä»“库é…置仅能用于é…置集群。 + +在 Rancher 中设置ç§æœ‰é•œåƒä»“库的主è¦æ–¹æ³•æœ‰ä¸¤ç§ï¼šé€šè¿‡[全局默认镜åƒä»“库](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md)中的**设置**选项å¡è®¾ç½®å…¨å±€é»˜è®¤é•œåƒä»“库，以åŠåœ¨é›†ç¾¤çº§åˆ«è®¾ç½®çš„高级选项中设置ç§æœ‰é•œåƒä»“库。全局默认镜åƒä»“库å¯ä»¥ç”¨äºŽç¦»çº¿è®¾ç½®ï¼Œä¸éœ€è¦å‡­è¯çš„é•œåƒä»“库。而集群级ç§æœ‰é•œåƒä»“库用于所有需è¦å‡­è¯çš„ç§æœ‰é•œåƒä»“库。 + +如果你的ç§æœ‰é•œåƒä»“库需è¦å‡­è¯ï¼Œä¸ºäº†å°†å‡­è¯ä¼ é€’ç»™ Rancher，你需è¦ç¼–辑æ¯ä¸ªéœ€è¦ä»Žä»“库中拉å–é•œåƒçš„集群的集群选项。 + +ç§æœ‰é•œåƒä»“库的é…置选项能让 Rancher 知é“è¦ä»Žå“ªé‡Œæ‹‰å–用于集群的[系统镜åƒ](https://rancher.com/docs/rke/latest/en/config-options/system-images/)或[附加组件镜åƒ](https://rancher.com/docs/rke/latest/en/config-options/add-ons/)。 + +- **系统镜åƒ**是维护 Kubernetes 集群所需的组件。 +- **附加组件**用于部署多个集群组件，包括网络æ’件ã€ingress controllerã€DNS æ供商或 metrics server。 + +有关为集群é…置期间应用的组件设置ç§æœ‰é•œåƒä»“库的更多信æ¯ï¼Œè¯·å‚阅[ç§æœ‰é•œåƒä»“库的 RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/private-registries/)。 + +Rancher v2.6 引入了[为 RKE 集群é…ç½® ECR é•œåƒä»“库](https://rancher.com/docs/rke/latest/en/config-options/private-registries/#amazon-elastic-container-registry-ecr-private-registry-setup)的功能。 + +### 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯ç”¨äºŽç›´æŽ¥è®¿é—® Kubernetes API server，而无需通过 Rancher 进行通信。 + +:::note + +授æƒé›†ç¾¤ç«¯ç‚¹ä»…适用于 Rancher å¯åŠ¨çš„ Kubernetes 集群，å³åªé€‚用于 Rancher [使用 RKE](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#é…ç½®-kubernetes-集群的工具) æ¥é…置的集群。它ä¸é€‚用于托管在 Kubernetes æ供商中的集群，例如 Amazon çš„ EKS。 + +::: + +在 Rancher å¯åŠ¨çš„ Kubernetes 集群中，它默认å¯ç”¨ï¼Œä½¿ç”¨å…·æœ‰ `controlplane` 角色的节点的 IP 和默认的 Kubernetes 自签åè¯ä¹¦ã€‚ + +有关授æƒé›†ç¾¤ç«¯ç‚¹çš„工作原ç†ä»¥åŠä½¿ç”¨çš„原因，请å‚阅[架构介ç»](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +### 节点池 + +有关使用 Rancher UI 在 RKE 集群中设置节点池的信æ¯ï¼Œè¯·å‚阅[此页é¢](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 + +### NGINX Ingress + +如果你想使用高å¯ç”¨æ€§é…ç½®æ¥å‘布应用，并且你使用没有原生负载å‡è¡¡åŠŸèƒ½çš„云æ供商æ¥æ‰˜ç®¡ä¸»æœºï¼Œè¯·å¯ç”¨æ­¤é€‰é¡¹ä»¥åœ¨é›†ç¾¤ä¸­ä½¿ç”¨ NGINX Ingress。 + +### Metrics Server 监控 + +这是å¯ç”¨æˆ–ç¦ç”¨ [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/) 的选项。 + +æ¯ä¸ªèƒ½å¤Ÿä½¿ç”¨ RKE å¯åŠ¨é›†ç¾¤çš„云æ供商都å¯ä»¥æ”¶é›†æŒ‡æ ‡å¹¶ç›‘控你的集群节点。如果å¯ç”¨æ­¤é€‰é¡¹ï¼Œä½ å¯ä»¥ä»Žä½ çš„云æ供商门户查看你的节点指标。 + +### Pod å®‰å…¨ç­–ç•¥æ”¯æŒ + +为集群å¯ç”¨ [pod 安全策略](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。å¯ç”¨æ­¤é€‰é¡¹åŽï¼Œä½¿ç”¨**默认 Pod 安全策略**下拉èœå•é€‰æ‹©ä¸€ä¸ªç­–略。 + +你必须有已é…置的 Pod 安全策略æ‰èƒ½ä½¿ç”¨æ­¤é€‰é¡¹ã€‚ + +### 节点上的 Docker 版本 + +表示是å¦å…许节点è¿è¡Œ Rancher ä¸æ­£å¼æ”¯æŒçš„ Docker 版本。 + +如果你选择使用支æŒçš„ Docker 版本，Rancher 会ç¦æ­¢ pod è¿è¡Œåœ¨å®‰è£…了ä¸æ”¯æŒçš„ Docker 版本的节点上。 + +如需了解å„个 Rancher 版本通过了哪些 Docker 版本测试，请å‚è§[支æŒå’Œç»´æŠ¤æ¡æ¬¾](https://rancher.com/support-maintenance-terms/)。 + +### Docker 根目录 + +如果è¦æ·»åŠ åˆ°é›†ç¾¤çš„节点为 Docker é…置了éžé»˜è®¤ Docker 根目录（默认为 `/var/lib/docker`），请在此选项中指定正确的 Docker 根目录。 + +### 默认 Pod 安全策略 + +如果你å¯ç”¨äº† **Pod 安全策略支æŒ**，请使用此下拉èœå•é€‰æ‹©åº”用于集群的 pod 安全策略。 + +### 节点端å£èŒƒå›´ + +更改å¯ç”¨äºŽ [NodePort æœåŠ¡](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)的端å£èŒƒå›´çš„选项。默认为 `30000-32767`。 + +### 定期 etcd å¿«ç…§ + +å¯ç”¨æˆ–ç¦ç”¨[定期 etcd å¿«ç…§](https://rancher.com/docs/rke/latest/en/etcd-snapshots/#etcd-recurring-snapshots)的选项。 + +### Agent 环境å˜é‡ + +为 [rancher agent](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md) 设置环境å˜é‡çš„选项。你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚如果 Rancher Agent 需è¦ä½¿ç”¨ä»£ç†ä¸Ž Rancher Server 通信，则å¯ä»¥ä½¿ç”¨ Agent 环境å˜é‡è®¾ç½® `HTTP_PROXY`，`HTTPS_PROXY` å’Œ `NO_PROXY` 环境å˜é‡ã€‚ + +### æ›´æ–° ingress-nginx + +使用 Kubernetes 1.16 之å‰ç‰ˆæœ¬åˆ›å»ºçš„集群将具有 `OnDelete`çš„ `ingress-nginx` `updateStrategy`。使用 Kubernetes 1.16 或更高版本创建的集群将具有 `RollingUpdate`。 + +如果 `ingress-nginx` çš„ `updateStrategy` 是 `OnDelete`，则需è¦åˆ é™¤è¿™äº› pod 以获得 deployment 正确的版本。 + +### Cluster Agent é…置和 Fleet Agent é…ç½® + +ä½ å¯ä»¥ä¸º Cluster Agent 和集群的 Fleet Agent é…置调度字段和资æºé™åˆ¶ã€‚ä½ å¯ä»¥ä½¿ç”¨è¿™äº›å­—段æ¥è‡ªå®šä¹‰å®¹å¿åº¦ã€äº²å’Œæ€§è§„则和资æºè¦æ±‚。其他容å¿åº¦ä¼šè¢«å°¾é™„到默认容å¿åº¦å’Œ Control Plane 节点污点的列表中。如果你定义了自定义亲和性规则，它们将覆盖全局默认亲和性设置。定义资æºè¦æ±‚会在以å‰æ²¡æœ‰çš„地方设置请求或é™åˆ¶ã€‚ + +:::note + +有了这个选项，你å¯ä»¥è¦†ç›–或删除è¿è¡Œé›†ç¾¤æ‰€éœ€çš„规则。我们强烈建议你ä¸è¦åˆ é™¤æˆ–覆盖这些规则和其他亲和性规则，因为这å¯èƒ½ä¼šå¯¼è‡´ä¸å¿…è¦çš„å½±å“： + +- `affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution` 用于 `cattle-cluster-agent` +- `cluster-agent-default-affinity` 用于 `cattle-cluster-agent` +- `fleet-agent-default-affinity` 用于 `fleet-agent` + +::: + +如果将 Rancher é™çº§åˆ° v2.7.4 或更低版本，你的更改将丢失，而且 Agent 将在没有你的自定义设置的情况下é‡æ–°éƒ¨ç½²ã€‚é‡æ–°éƒ¨ç½²æ—¶ï¼ŒFleet Agent 将回退到使用内置默认值。如果é™çº§æœŸé—´ Fleet 版本没有更改，则ä¸ä¼šç«‹å³é‡æ–°éƒ¨ç½²ã€‚ + + +## RKE 集群é…置文件å‚考 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ åœ¨ RKE 安装中设置任何[å¯ç”¨é€‰é¡¹](https://rancher.com/docs/rke/latest/en/config-options/)（`system_images` é…置除外）。使用 Rancher UI 或 API 创建集群时，ä¸æ”¯æŒ `system_images` 选项。 + +有关 YAML 中 RKE Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/)。 + +### Rancher 中的é…置文件结构 + +RKE（Rancher Kubernetes Engine）是 Rancher 用æ¥é…ç½® Kubernetes 集群的工具。过去，Rancher 的集群é…置文件与 [RKE é…置文件](https://rancher.com/docs/rke/latest/en/example-yamls/)的结构是一致的。但由于 Rancher 文件结构å‘生了å˜åŒ–，因此在 Rancher 中，RKE 集群é…ç½®é¡¹ä¸Žéž RKE é…置项是分开的。所以，你的集群é…置需è¦åµŒå¥—在集群é…置文件中的 `rancher_kubernetes_engine_config` å‚数下。使用早期版本的 Rancher 创建的集群é…置文件需è¦é’ˆå¯¹è¿™ç§æ ¼å¼è¿›è¡Œæ›´æ–°ã€‚以下是一个集群é…置文件示例： + +
+ 集群é…置文件示例 + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker +enable_cluster_alerting: false +enable_cluster_monitoring: false +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # ç›®å‰ä»…æ”¯æŒ Nginx ingress provider +# # è¦ç¦ç”¨ Ingress controller，设置 `provider: none` +# # è¦åœ¨æŒ‡å®šèŠ‚点上ç¦ç”¨ Ingress，使用 node_selector，例如： +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` +
+ +### 默认 DNS æ供商 + +下表显示了默认部署的 DNS æ供商。有关如何é…ç½®ä¸åŒ DNS æ供商的更多信æ¯ï¼Œè¯·å‚阅 [DNS æ供商相关的 RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/add-ons/dns/)。CoreDNS åªèƒ½åœ¨ Kubernetes v1.12.0 åŠæ›´é«˜ç‰ˆæœ¬ä¸Šä½¿ç”¨ã€‚ + +| Rancher 版本 | Kubernetes 版本 | 默认 DNS æ供商 | +|-------------|--------------------|----------------------| +| v2.2.5 åŠæ›´é«˜ç‰ˆæœ¬ | v1.14.0 åŠæ›´é«˜ç‰ˆæœ¬ | CoreDNS | +| v2.2.5 åŠæ›´é«˜ç‰ˆæœ¬ | v1.13.x åŠæ›´ä½Žç‰ˆæœ¬ | kube-dns | +| v2.2.4 åŠæ›´ä½Žç‰ˆæœ¬ | ä»»æ„ | kube-dns | + +## YAML 中的 Rancher 特定å‚æ•° + +除了 RKE é…置文件选项外，还有å¯ä»¥åœ¨é…置文件 (YAML) 中é…置的 Rancher 特定设置如下。 + +### docker_root_dir + +请å‚阅 [Docker 根目录](#docker-根目录)。 + +### enable_cluster_monitoring + +å¯ç”¨æˆ–ç¦ç”¨[集群监控](../../../pages-for-subheaders/monitoring-and-alerting.md)的选项。 + +### enable_network_policy + +å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®ç½‘络隔离的选项。 + +如果你使用支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE 网络æ’件（例如 Canal 或 Cisco ACI æ’件），则å¯ä»¥ä½¿ç”¨é¡¹ç›®ç½‘络隔离。 + +### local_cluster_auth_endpoint + +请å‚阅[授æƒé›†ç¾¤ç«¯ç‚¹](#授æƒé›†ç¾¤ç«¯ç‚¹)。 + +示例： + +```yaml +local_cluster_auth_endpoint: + enabled: true + fqdn: "FQDN" + ca_certs: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +### 自定义网络æ’件 + +ä½ å¯ä»¥ä½¿ç”¨ RKE çš„[用户定义的附加组件功能](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/)æ¥æ·»åŠ è‡ªå®šä¹‰ç½‘络æ’件。部署 Kubernetes 集群之åŽï¼Œä½ å¯ä»¥å®šä¹‰è¦éƒ¨ç½²çš„任何附加组件。 + +有两ç§æ–¹æ³•å¯ä»¥æŒ‡å®šé™„加组件： + +- [内嵌附加组件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#in-line-add-ons) +- [为附加组件引用 YAML 文件](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#referencing-yaml-files-for-add-ons) + +有关如何通过编辑 `cluster.yml` æ¥é…置自定义网络æ’件的示例，请å‚阅 [RKE 文档](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/custom-network-plugin-example)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md new file mode 100644 index 00000000000..c1771ed32d3 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md @@ -0,0 +1,360 @@ +--- +title: RKE2 集群é…ç½®å‚考 +--- + +æœ¬æ–‡ä»‹ç» Rancher 中å¯ç”¨äºŽæ–°çš„或现有的 RKE2 Kubernetes 集群的é…置选项。 + +## 概述 + +ä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼ä¹‹ä¸€æ¥é…ç½® Kubernetes 选项： + +- [Rancher UI](#rancher-ui-中的é…置选项)：使用 Rancher UI æ¥é€‰æ‹©è®¾ç½® Kubernetes 集群时常用的自定义选项。 +- [集群é…置文件](#集群é…置文件å‚考)：高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE2 é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件让你能设置更多å¯ç”¨äºŽ RKE2 的其他[安装选项](https://docs.rke2.io/install/configuration)。 + +## 在 Rancher UI 中使用表å•ç¼–辑集群 + +è¦ç¼–辑你的集群： + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 转到è¦é…置的集群，然åŽå•å‡» **â‹® > 编辑é…ç½®**。 + +## 使用 YAML 编辑集群 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ª RKE2 é…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å¯ä»¥è®©ä½ ä½¿ç”¨ YAML æ¥æŒ‡å®š RKE2 安装中å¯ç”¨çš„任何选项。 + +è¦ç›´æŽ¥ä»Ž Rancher UI 中编辑 RKE2 é…置文件： + +1. 点击 **☰ > 集群管ç†**。 +1. 转到你è¦é…置的集群，然åŽå•å‡» **â‹® > 以 YAML 文件编辑**。 +1. 编辑 `rkeConfig` å‚数下的 RKE 选项。 + +## Rancher UI 中的é…置选项 + +:::tip + +一些高级é…置选项没有在 Rancher UI 表å•ä¸­å¼€æ”¾ï¼Œä½†ä½ å¯ä»¥é€šè¿‡åœ¨ YAML 中编辑 RKE2 集群é…置文件æ¥å¯ç”¨è¿™äº›é€‰é¡¹ã€‚有关 YAML 中 RKE2 Kubernetes 集群的å¯é…置选项的完整å‚考，请å‚阅 [RKE2 文档](https://docs.rke2.io/install/configuration)。 + +::: + +## 主机池 + +本å°èŠ‚介ç»äº†é€šç”¨çš„主机池é…置。对于针对特定基础设施æ供商的é…置，请å‚阅以下页é¢ï¼š + +- [Azure](../downstream-cluster-configuration/machine-configuration/azure.md) +- [DigitalOcean](../downstream-cluster-configuration/machine-configuration/digitalocean.md) +- [EC2](../downstream-cluster-configuration/machine-configuration/amazon-ec2.md) + +### æ± å称 + +主机池的å称。 + +### ä¸»æœºæ•°é‡ + +池中的主机数。 + +### 角色 + +å°† etcdã€controlplane å’Œ worker 角色分é…给节点的选项。 + +### 高级é…ç½® + +#### è‡ªåŠ¨æ›¿æ¢ + +如果节点无法访问的æŒç»­æ—¶é—´è¾¾åˆ°è¯¥å€¼ï¼Œåˆ™ä¼šè¢«è‡ªåŠ¨åˆ é™¤å’Œæ›¿æ¢ã€‚ + +#### 删除å‰æ¸…空 + +通过在删除节点之å‰é©±é€æ‰€æœ‰ pod æ¥æ¸…空节点。 + +#### Kubernetes 节点标签 + +å°†[标签](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)（label）添加到节点，让对象选择更加简便。 + +有关标签语法的详细信æ¯ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)。 + +#### 污点 + +å°†[污点](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/)（taint）添加到节点，防止 pod è¢«è°ƒåº¦æˆ–åœ¨èŠ‚ç‚¹ä¸Šæ‰§è¡Œï¼ˆé™¤éž pod 具有匹é…的容å¿åº¦ï¼‰ã€‚ + +## 集群é…ç½® + +### åŸºæœ¬ä¿¡æ¯ +#### Kubernetes 版本 + +这指的是集群节点上安装的 Kubernetes 版本。Rancher 基于 [hyperkube](https://github.com/rancher/hyperkube) 打包了自己的 Kubernetes 版本。 + +有关更多详细信æ¯ï¼Œè¯·å‚阅[å‡çº§ Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md)。 + +#### 容器网络æ供商 + +这指的是集群使用的[网络æ供商](https://kubernetes.io/docs/concepts/cluster-administration/networking/)。 + +:::caution + +å¯åŠ¨é›†ç¾¤åŽï¼Œä½ æ— æ³•æ›´æ”¹ç½‘络æ供商。由于 Kubernetes ä¸å…许在网络æ供商之间切æ¢ï¼Œå› æ­¤ï¼Œè¯·è°¨æ…Žé€‰æ‹©è¦ä½¿ç”¨çš„网络æ供商。使用网络æ供商创建集群åŽï¼Œå¦‚果你需è¦æ›´æ”¹ç½‘络æ供商，你将需è¦æ‹†é™¤æ•´ä¸ªé›†ç¾¤ä»¥åŠå…¶ä¸­çš„所有应用。 + +::: + +Rancher 与以下开箱å³ç”¨çš„网络æ供商兼容： + +- [Canal](https://github.com/projectcalico/canal) +- [Cilium](https://cilium.io/)\* +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Multus](https://github.com/k8snetworkplumbingwg/multus-cni) + +\* 在 [Cilium CNI](../../../faq/container-network-interface-providers.md#cilium) 中使用[项目网络隔离](#项目网络隔离)时，你å¯ä»¥å¼€å¯è·¨èŠ‚点入å£è·¯ç”±ã€‚详情请å‚è§ [CNI æ供商文档](../../../faq/container-network-interface-providers.md#cilium-中跨节点的-ingress-路由)。 + +有关ä¸åŒç½‘络æ供商以åŠå¦‚何é…置它们的更多详细信æ¯ï¼Œè¯·æŸ¥é˜… [RKE2 文档](https://docs.rke2.io/install/network_options)。 + +##### åŒæ ˆç½‘络 + +所有 CNI 网络æ’件都支æŒ[åŒæ ˆ](https://docs.rke2.io/install/network_options#dual-stack-configuration)网络。è¦åœ¨åŒæ ˆæ¨¡å¼ä¸‹é…ç½® RKE2，请为你的[集群 CIDR](#集群-cidr) å’Œ/或 [Service CIDR](#service-cidr) 设置有效的 IPv4/IPv6 CIDR。 + +###### é¢å¤–é…ç½® {#dual-stack-additional-config} + +使用 `cilium` 或 `multus,cilium` 作为容器网络接å£æ供商时，请确ä¿**å¯ç”¨ IPv6 支æŒ**选项。 + +#### 云æ供商 + +ä½ å¯ä»¥é…ç½® [Kubernetes 云æ供商](../../../pages-for-subheaders/set-up-cloud-providers.md)。如果你想在 Kubernetes 中使用动æ€é…置的[å·å’Œå­˜å‚¨](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md)，你通常需è¦é€‰æ‹©ç‰¹å®šçš„云æ供商。例如，如果你想使用 Amazon EBS，则需è¦é€‰æ‹© `aws` 云æ供商。 + +:::note + +如果你è¦ä½¿ç”¨çš„云æ供商未作为选项列出，你需è¦ä½¿ç”¨[é…置文件选项](#集群é…置文件å‚考)æ¥é…置云æ供商。请å‚考[本文档](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/)æ¥äº†è§£å¦‚何é…置云æ供商。 + +::: + +#### 默认 Pod 安全策略 + +为集群选择默认的 [pod 安全策略](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md)。请å‚阅 [RKE2 文档](https://docs.rke2.io/security/pod_security_policies)æ¥äº†è§£æ¯ä¸ªå¯ç”¨ç­–略的规范。 + +#### Worker CIS é…置文件 + +选择一个 [CIS benchmark](../../../pages-for-subheaders/cis-scan-guides.md) æ¥éªŒè¯ç³»ç»Ÿé…置。 + +#### 项目网络隔离 + +如果你的网络æ供商å…许项目网络隔离，你å¯ä»¥é€‰æ‹©å¯ç”¨æˆ–ç¦ç”¨é¡¹ç›®é—´çš„通信。 + +如果你使用支æŒæ‰§è¡Œ Kubernetes 网络策略的 RKE2 网络æ’件（例如 Canal），则å¯ä»¥ä½¿ç”¨é¡¹ç›®ç½‘络隔离。 + +#### CoreDNS + +默认情况下，[CoreDNS](https://coredns.io/) 会安装为默认 DNS æ供程åºã€‚如果未安装 CoreDNS，则必须自己安装备用 DNS æ供程åºã€‚有关其他 CoreDNS é…置，请å‚阅 [RKE2 文档](https://docs.rke2.io/networking#coredns)。 + +#### NGINX Ingress + +如果你想使用高å¯ç”¨æ€§é…ç½®æ¥å‘布应用，并且你使用没有原生负载å‡è¡¡åŠŸèƒ½çš„云æ供商æ¥æ‰˜ç®¡ä¸»æœºï¼Œè¯·å¯ç”¨æ­¤é€‰é¡¹ä»¥åœ¨é›†ç¾¤ä¸­ä½¿ç”¨ NGINX Ingress。有关其他é…置选项，请å‚阅 [RKE2 文档](https://docs.rke2.io/networking#nginx-ingress-controller)。 + +有关其他é…置选项，请å‚阅 [RKE2 文档](https://docs.rke2.io/networking#nginx-ingress-controller)。 + +#### Metrics Server + +这是å¯ç”¨æˆ–ç¦ç”¨ [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/) 的选项。 + +æ¯ä¸ªèƒ½å¤Ÿä½¿ç”¨ RKE2 å¯åŠ¨é›†ç¾¤çš„云æ供商都å¯ä»¥æ”¶é›†æŒ‡æ ‡å¹¶ç›‘控你的集群节点。如果å¯ç”¨æ­¤é€‰é¡¹ï¼Œä½ å¯ä»¥ä»Žä½ çš„云æ供商门户查看你的节点指标。 + +### 附加é…ç½® + +集群å¯åŠ¨æ—¶å°†åº”用的其他 Kubernetes 清å•ï¼Œä¼šä½œä¸º[附加组件](https://kubernetes.io/docs/concepts/cluster-administration/addons/)æ¥ç®¡ç†ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/helm#automatically-deploying-manifests-and-helm-charts)。 + +### Agent 环境å˜é‡ + +为 [Rancher agent](https://rancher.com/docs/rancher/v2.6/en/cluster-provisioning/rke-clusters/rancher-agents/) 设置环境å˜é‡çš„选项。你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹è®¾ç½®çŽ¯å¢ƒå˜é‡ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/reference/linux_agent_config)。 + +### etcd + +#### 自动快照 + +å¯ç”¨æˆ–ç¦ç”¨å®šæœŸ etcd 快照的选项。如果å¯ç”¨ï¼Œç”¨æˆ·å¯ä»¥é…置快照的频率。有关详细信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/backup_restore#creating-snapshots)。请注æ„，如果使用 RKE2，快照会存储在æ¯ä¸ª etcd 节点上，这与 RKE1 ä¸åŒï¼ˆRKE1 æ¯ä¸ªé›†ç¾¤åªå­˜å‚¨ä¸€ä¸ªå¿«ç…§ï¼‰ã€‚ + +#### 指标 + +选择å‘公众公开或仅在集群内公开 etcd 指标的选项。 + +### 网络 + +#### 集群 CIDR + +用于 pod IP çš„ IPv4 å’Œ/或 IPv6 网络 CIDR（默认：10.42.0.0/16）。 + +##### åŒæ ˆç½‘络 + +è¦é…ç½®[åŒæ ˆ](https://docs.rke2.io/install/network_options#dual-stack-configuration)模å¼ï¼Œè¯·è¾“入有效的 IPv4/IPv6 CIDR。例如 `10.42.0.0/16,2001:cafe:42:0::/56`。 + +使用 `cilium` 或 `multus,cilium` 作为[容器网络](#容器网络æ供商)接å£æ供商时，你需è¦è¿›è¡Œ[附加é…ç½®](#dual-stack-additional-config)。 + +#### Service CIDR + +用于 Service IP çš„ IPv4/IPv6 网络 CIDR（默认：10.43.0.0/16）。 + +##### åŒæ ˆç½‘络 + +è¦é…ç½®[åŒæ ˆ](https://docs.rke2.io/install/network_options#dual-stack-configuration)模å¼ï¼Œè¯·è¾“入有效的 IPv4/IPv6 CIDR。例如 `10.42.0.0/16,2001:cafe:42:0::/56`。 + +使用 `cilium` 或 `multus,cilium` 作为[容器网络](#容器网络æ供商)接å£æ供商时，你需è¦è¿›è¡Œ[附加é…ç½®](#dual-stack-additional-config)。 + +#### 集群 DNS + +用于 coredns æœåŠ¡çš„ IPv4 集群 IP。应该在你的 service-cidr 范围内（默认：10.43.0.10）。 + +#### 集群域å + +选择集群的域。默认值为 `cluster.local`。 + +#### NodePort æœåŠ¡ç«¯å£èŒƒå›´ + +更改å¯ç”¨äºŽ [NodePort æœåŠ¡](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)的端å£èŒƒå›´çš„选项。默认值为 `30000-32767`。 + +#### 截断主机å + +将主机å截断为 15 个或更少字符。该字段åªèƒ½åœ¨é›†ç¾¤åˆå§‹åˆ›å»ºæ—¶è®¾ç½®ã€‚创建集群åŽï¼Œä½ æ— æ³•å†å¯ç”¨æˆ–ç¦ç”¨ 15 个字符的é™åˆ¶ã€‚ + +此设置仅影å“é…置了机器的集群。由于自定义集群在创建节点时设置主机å（å‘生在 Rancher 之外），因此该字段ä¸é™åˆ¶è‡ªå®šä¹‰é›†ç¾¤çš„主机å长度。 + +截断集群中的主机åå¯æ高与基于 Windows 的系统的兼容性。虽然 Kubernetes å…许最长的主机å长度为 63 个字符，但使用 NetBIOS 的系统将主机åé™åˆ¶ä¸º 15 个字符之内。 + +#### TLS å¯é€‰å称 + +在æœåŠ¡å™¨ TLS è¯ä¹¦ä¸Šæ·»åŠ å…¶ä»–主机å或 IPv4/IPv6 地å€ä½œä¸º Subject Alternative Name。 + +#### 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯ç”¨äºŽç›´æŽ¥è®¿é—® Kubernetes API server，而无需通过 Rancher 进行通信。 + +在 Rancher å¯åŠ¨çš„ Kubernetes 集群中，它默认å¯ç”¨ï¼Œä½¿ç”¨å…·æœ‰ `controlplane` 角色的节点的 IP 和默认的 Kubernetes 自签åè¯ä¹¦ã€‚ + +有关授æƒé›†ç¾¤ç«¯ç‚¹çš„工作原ç†ä»¥åŠä½¿ç”¨çš„原因，请å‚阅[架构介ç»](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +我们建议使用具有授æƒé›†ç¾¤ç«¯ç‚¹çš„è´Ÿè½½å‡è¡¡å™¨ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[推è的架构](../../rancher-manager-architecture/architecture-recommendations.md#授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„)。 + +### é•œåƒä»“库 + +选择è¦ä»Žä¸­æ‹‰å– Rancher é•œåƒçš„é•œåƒä»“库。有关更多详细信æ¯å’Œé…置选项，请å‚阅 [RKE2 文档](https://docs.rke2.io/install/containerd_registry_configuration)。 + +### å‡çº§ç­–ç•¥ + +#### controlplane å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### Worker å¹¶å‘ + +选择å¯ä»¥åŒæ—¶å‡çº§å¤šå°‘个节点。å¯ä»¥æ˜¯å›ºå®šæ•°å­—或百分比。 + +#### 清空节点（controlplane） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +#### 清空节点（worker 节点） + +在å‡çº§ä¹‹å‰ä»ŽèŠ‚点中删除所有 pod 的选项。 + +### 高级é…ç½® + +为ä¸åŒèŠ‚点设置 kubelet 选项。有关å¯ç”¨é€‰é¡¹ï¼Œè¯·å‚阅 [Kubernetes 文档](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/)。 + +## 集群é…置文件å‚考 + +高级用户å¯ä»¥åˆ›å»ºä¸€ä¸ªé…置文件，而ä¸æ˜¯ä½¿ç”¨ Rancher UI æ¥ä¸ºé›†ç¾¤é€‰æ‹© Kubernetes 选项。é…置文件å…许你为 RKE2 设置[å¯ç”¨çš„选项](https://docs.rke2.io/install/configuration)，其中包括已ç»åœ¨ [Rancher UI é…置选项](#rancher-ui-中的é…置选项)ä¸­åˆ—å‡ºçš„é€‰é¡¹ä»¥åŠ Rancher 特定的å‚数。 + +
+ + 集群é…置文件片段示例 + + +```yaml +spec: + cloudCredentialSecretName: cattle-global-data:cc-s879v + kubernetesVersion: v1.23.6+rke2r2 + localClusterAuthEndpoint: {} + rkeConfig: + chartValues: + rke2-calico: {} + etcd: + snapshotRetention: 5 + snapshotScheduleCron: 0 */5 * * * + machineGlobalConfig: + cni: calico + disable-kube-proxy: false + etcd-expose-metrics: false + profile: null + machinePools: + - controlPlaneRole: true + etcdRole: true + machineConfigRef: + kind: Amazonec2Config + name: nc-test-pool1-pwl5h + name: pool1 + quantity: 1 + unhealthyNodeTimeout: 0s + workerRole: true + machineSelectorConfig: + - config: + protect-kernel-defaults: false + registries: {} + upgradeStrategy: + controlPlaneConcurrency: "1" + controlPlaneDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 + workerConcurrency: "1" + workerDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 +``` +
+ +### chartValues + +此选项用于为 RKE2/K3s 安装的 System Chart 指定值。 + +示例： + +```yaml +chartValues: + chart-name: + key: value +``` +### machineGlobalConfig + +RKE2/K3s é…置嵌套在 `machineGlobalConfig` å‚数下。在这里所åšçš„任何é…置更改都将应用到æ¯ä¸ªèŠ‚点。你å¯ä»¥åœ¨æ­¤å¤„应用[RKE2 的独立版本](https://docs.rke2.io/reference/server_config)中å¯ç”¨çš„é…置选项。 + +示例： + +```yaml +machineGlobalConfig: + etcd-arg: + - key1=value1 + - key2=value2 +``` + +### machineSelectorConfig + +æ­¤å‚数与 [`machineGlobalConfig`](#machineglobalconfig) 相åŒï¼Œåªæ˜¯å¯ä»¥åœ¨é…置中指定 [label](#kubernetes-节点标签) 选择器。该é…置仅应用于与标签选择器匹é…的节点。 + +å…许多个 `config` æ¡ç›®ï¼Œå¯ä»¥ä¸ºæ¯ä¸ªæ¡ç›®æŒ‡å®šå„自的 `machineLabelSelector`。用户å¯ä»¥æŒ‡å®š `matchExpressions`ã€`matchLabels`ã€æŒ‡å®šäºŒè€…或都ä¸æŒ‡å®šã€‚如果你çœç•¥äº† `machineLabelSelector`，则与将 config 放入 `machineGlobalConfig` 的效果相åŒã€‚ + +示例： + +```yaml +machineSelectorConfig + - config: + config-key: config-value + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # 有效的è¿ç®—符：Inã€NotInã€Exists å’Œ DoesNotExist + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md new file mode 100644 index 00000000000..823aaa2de81 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -0,0 +1,41 @@ +--- +title: åŒæ­¥ +--- + +åŒæ­¥å…许 Rancher 更新集群值，以便与托管在 AKSã€EKS 或 GKE 中的集群对象ä¿æŒåŒæ­¥ã€‚这使得 Rancher 以外的æ¥æºèƒ½å¤ŸèŽ·å–托管集群的状æ€ã€‚这是 UI 中显示的内容。 + +:::caution +如果你åŒæ—¶å¤„ç†æ¥è‡ªå¦ä¸€ä¸ªæ¥æºçš„更新，你å¯èƒ½ä¼šä¸å°å¿ƒè¦†ç›–一个æ¥æºçš„状æ€ã€‚如果你在完æˆä¸€ä¸ªæ¥æºçš„æ›´æ–°åŽ 5 分钟内处ç†å¦ä¸€ä¸ªæ¥æºçš„更新，也å¯èƒ½ä¼šå‘生这ç§æƒ…况。 +::: + +### å·¥ä½œåŽŸç† + +è¦ç†è§£åŒæ­¥æ˜¯å¦‚何工作的，则必须ç†è§£ Rancher Cluster 对象上的两个字段： + +1. 集群的 config 对象，ä½äºŽé›†ç¾¤çš„规范上： + + * 对于 AKS，该字段称为 AKSConfig + * 对于 EKS，该字段称为 EKSConfig + * 对于 GKE，该字段称为 GKEConfig + +2. UpstreamSpec 对象 + + * 对于 AKS，它ä½äºŽç¾¤é›†çŠ¶æ€çš„ AKSStatus 字段中。 + * 对于 EKS，它ä½äºŽé›†ç¾¤çŠ¶æ€çš„ EKSStatus 字段中。 + * 对于 GKE，它ä½äºŽé›†ç¾¤çŠ¶æ€çš„ GKEStatus 字段中。 + +定义这些对象的结构类型å¯ä»¥åœ¨å®ƒä»¬å¯¹åº”çš„ operator 项目中找到： + +* [aks-operator](https://github.com/rancher/aks-operator/blob/master/pkg/apis/aks.cattle.io/v1/types.go) +* [eks-operator](https://github.com/rancher/eks-operator/blob/master/pkg/apis/eks.cattle.io/v1/types.go) +* [gke-operator](https://github.com/rancher/gke-operator/blob/master/pkg/apis/gke.cattle.io/v1/types.go) + +除集群å称ã€ä½ç½®ï¼ˆåŒºåŸŸæˆ–地区）ã€å¯¼å…¥å’Œäº‘凭è¯å¼•ç”¨å¤–，所有字段å‡å¯ä¸ºç©ºã€‚ + +AKSConfigã€EKSConfig 或 GKEConfig 表示所需的状æ€ã€‚零值会被忽略。é…置对象中éžé›¶çš„字段å¯ä»¥è¢«è®¤ä¸ºæ˜¯â€œç®¡ç†çš„â€ã€‚在 Rancher 中创建集群时，所有字段都是éžé›¶çš„，因此都是“管ç†â€çš„。在把一个已存在的集群注册到 Rancher 时，所有å¯ä¸ºç©ºå­—段都是 nil 并且ä¸æ˜¯â€œç®¡ç†â€çš„。一旦 Rancher 更改了这些字段的值，这些字段就会被管ç†ã€‚ + +UpstreamSpec 代表集群在托管的 Kubernetes æä¾›å•†ä¸­çš„æƒ…å†µã€‚å®ƒæ¯ 5 分钟刷新一次。刷新 UpstreamSpec åŽï¼ŒRancher 会检查集群是å¦æ­£åœ¨è¿›è¡Œæ›´æ–°ã€‚如果它正在更新，则ä¸åšä»»ä½•è¿›ä¸€æ­¥å¤„ç†ã€‚如果它目å‰æ²¡æœ‰æ›´æ–°ï¼ŒAKSConfigã€EKSConfig 或 GKEConfig 上的任何 "管ç†" 字段都会被最近更新的 UpstreamSpec 上的相应值覆盖。 + +有效的期望状æ€å¯ä»¥è¢«è®¤ä¸ºæ˜¯ UpstreamSpec，加上 AKSConfigã€EKSConfig 或 GKEConfig 中的所有éžé›¶å­—段。这是 UI 中显示的内容。 + +如果 Rancher å’Œå¦ä¸€ä¸ªæ¥æºè¯•å›¾åœ¨åŒä¸€æ—¶é—´æˆ–在更新完æˆåŽçš„ 5 分钟å°è¯•æ›´æ–°é›†ç¾¤ï¼Œä»»ä½•ç®¡ç†å­—段都å¯èƒ½é™·å…¥ç«žäº‰çŠ¶æ€ã€‚以 EKS 为例，集群å¯èƒ½å°† PrivateAccess 作为管ç†å­—段。如果 PrivateAccess 为 false，在 11:01 在 EKS 控制å°ä¸­å¯ç”¨ï¼Œç„¶åŽ Rancher 在 11:05 之å‰æ›´æ–°æ ‡ç­¾ï¼Œé‚£ä¹ˆè¯¥å€¼å¾ˆå¯èƒ½è¢«è¦†ç›–。如果在集群处ç†æ›´æ–°æ—¶æ›´æ–°äº†æ ‡ç­¾ï¼Œä¹Ÿä¼šå‘生这ç§æƒ…况。如果集群已注册并且 PrivateAccess 字段为 nil，则ä¸åº”å‘生此示例中æ述的问题。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md new file mode 100644 index 00000000000..c301e408d1f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -0,0 +1,53 @@ +--- +title: Rancher Agent 选项 +--- + +Rancher 在æ¯ä¸ªèŠ‚点上部署一个 Agent æ¥ä¸ŽèŠ‚点通信。本文æ述了å¯ä»¥ä¼ é€’ç»™ Agent 的选项。è¦ä½¿ç”¨è¿™äº›é€‰é¡¹ï¼Œä½ éœ€è¦[使用自定义节点创建集群](../../../../pages-for-subheaders/use-existing-nodes.md)，并在添加节点时将选项添加到生æˆçš„ `docker run` 命令。 + +有关 Rancher 如何使用 Node Agent 与下游集群通信的概述，请å‚阅[产å“架构](../../../rancher-manager-architecture/communicating-with-downstream-user-clusters.md#3-node-agents)。 + +## 通用选项 + +| å‚æ•° | 环境å˜é‡ | æè¿° | +| ---------- | -------------------- | ----------- | +| `--server` | `CATTLE_SERVER` | å·²é…置的 Rancher `server-url`，Agent 将通过这个地å€è¿žæŽ¥ Server。 | +| `--token` | `CATTLE_TOKEN` | 在 Rancher 中注册节点所需的 Token。 | +| `--ca-checksum` | `CATTLE_CA_CHECKSUM` | 使用已é…置的 Rancher `cacerts` 进行 SHA256 校验和验è¯ã€‚ | +| `--node-name` | `CATTLE_NODE_NAME` | 覆盖用于注册节点的主机å（默认为 `hostname -s`）。 | +| `--label` | `CATTLE_NODE_LABEL` | å‘节点添加节点标签。对于多个标签，请传递é¢å¤–çš„ `--label` 选项。(`--label key=value`) | +| `--taints` | `CATTLE_NODE_TAINTS` | 将节点污点添加到节点。对于多个污点，请传递é¢å¤–çš„ `--taints` 选项。(`--taints key=value:effect`) | + +## 角色选项 + +| å‚æ•° | 环境å˜é‡ | æè¿° | +| ---------- | -------------------- | ----------- | +| `--all-roles` | `ALL=true` | 将所有角色（`etcd`ã€`controlplane`ã€`worker`）应用到节点。 | +| `--etcd` | `ETCD=true` | 将角色 `etcd` 应用到节点。 | +| `--controlplane` | `CONTROL=true` | 将角色 `controlplane` 应用到节点。 | +| `--worker` | `WORKER=true` | 将角色 `worker` 应用到节点。 | + +## IP 地å€é€‰é¡¹ + +| å‚æ•° | 环境å˜é‡ | æè¿° | +| ---------- | -------------------- | ----------- | +| `--address` | `CATTLE_ADDRESS` | 节点将注册的 IP 地å€ï¼ˆé»˜è®¤ä¸ºç”¨äºŽè¿žæŽ¥ `8.8.8.8`çš„ IP）。 | +| `--internal-address` | `CATTLE_INTERNAL_ADDRESS` | ç§æœ‰ç½‘络上用于主机间通信的 IP 地å€ã€‚ | + +### åŠ¨æ€ IP 地å€é€‰é¡¹ + +出于自动化目的，你ä¸èƒ½åœ¨å‘½ä»¤ä¸­ä½¿ç”¨ç‰¹å®šçš„ IP 地å€ï¼Œå› ä¸ºå®ƒå¿…须是通用的æ‰èƒ½ç”¨äºŽæ¯ä¸ªèŠ‚点。为此，我们æä¾›äº†åŠ¨æ€ IP 地å€çš„选项。它们用作现有 IP 地å€é€‰é¡¹çš„å€¼ã€‚æ”¯æŒ `--address` å’Œ `--internal-address`。 + +| 值 | 示例 | æè¿° | +| ---------- | -------------------- | ----------- | +| 接å£å称 | `--address eth0` | 将从给定的接å£ä¸­æ£€ç´¢ç¬¬ä¸€ä¸ªé…置的 IP 地å€ã€‚ | +| `ipify` | `--address ipify` | 将使用从 `https://api.ipify.org` 检索到的值。 | +| `awslocal` | `--address awslocal` | 将使用从 `http://169.254.169.254/latest/meta-data/local-ipv4` 检索到的值。 | +| `awspublic` | `--address awspublic` | 将使用从 `http://169.254.169.254/latest/meta-data/public-ipv4` 检索到的值。 | +| `doprivate` | `--address doprivate` | 将使用从 `http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address` 检索到的值。 | +| `dopublic` | `--address dopublic` | 将使用从 `http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address` 检索到的值。 | +| `azprivate` | `--address azprivate` | 将使用从 `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text` 检索到的值。 | +| `azpublic` | `--address azpublic` | 将使用从 `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-08-01&format=text` 检索到的值。 | +| `gceinternal` | `--address gceinternal` | 将使用从 `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip` 检索到的值。 | +| `gceexternal` | `--address gceexternal` | 将使用从 `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip` 检索到的值。 | +| `packetlocal` | `--address packetlocal` | 将使用从 `https://metadata.packet.net/2009-04-04/meta-data/local-ipv4` 检索到的值。 | +| `packetpublic` | `--address packetlocal` | 将使用从 `https://metadata.packet.net/2009-04-04/meta-data/public-ipv4` 检索到的值。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/kubernetes-concepts.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/kubernetes-concepts.md new file mode 100644 index 00000000000..17a1bc2b969 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/kubernetes-concepts.md @@ -0,0 +1,64 @@ +--- +title: Kubernetes 概念 +--- + +本文解释了 Kubernetes 的相关概念，以便让你更好地了解 Rancher çš„è¿è¡Œæœºåˆ¶ã€‚本文仅对 Kubernetes 组件进行了简å•çš„æ述。如需了解更多详情，请å‚è§ [Kubernetes 组件的官方文档](https://kubernetes.io/docs/concepts/overview/components/)。 + +## 关于 Docker + +Docker 是容器打包和è¿è¡Œæ—¶ç³»ç»Ÿçš„标准。开å‘者在 Dockerfiles 中构建容器镜åƒï¼Œå¹¶åœ¨ Docker é•œåƒä»“库中分å‘容器镜åƒã€‚[Docker Hub](https://hub.docker.com) 是市é¢ä¸Šä¸»æµçš„公有镜åƒä»“库。许多ä¼ä¸šè¿˜åˆ›å»ºç§æœ‰ Docker é•œåƒä»“库。Docker 主è¦ç”¨äºŽç®¡ç†å•ä¸ªèŠ‚点上的容器。 + +:::note + +由于 Kubernetes å·²ç»æˆä¸ºäº†å®¹å™¨ç®¡ç†çš„主æµå·¥å…·ï¼Œæ‰€ä»¥ä»Ž Rancher 2.x 版本开始，我们ä¸å†æ”¯æŒ Docker Swarm。如果你有使用 Docker 管ç†å®¹å™¨çš„需求，å¯ä»¥å®‰è£… Rancher 1.6 进行æ“作。 + +::: + +## 关于 Kubernetes + +Kubernetes 是容器和集群管ç†çš„标准。YAML 文件规定了组æˆä¸€ä¸ªåº”用所需的容器和其他资æºã€‚Kubernetes æ供了调度ã€ä¼¸ç¼©ã€æœåŠ¡å‘现ã€å¥åº·æ£€æŸ¥ã€å¯†æ–‡ç®¡ç†å’Œé…置管ç†ç­‰åŠŸèƒ½ã€‚ + +## Kubernetes 集群是什么 + +集群是å¯ä½œä¸ºä¸€ä¸ªç³»ç»ŸååŒå·¥ä½œçš„一组计算机。 + +_Kubernetes 集群_ 是使用 [Kubernetes 容器编排系统](https://kubernetes.io/)æ¥éƒ¨ç½²ã€è¿ç»´å’Œä¼¸ç¼© Docker 容器的集群，让你的组织实现应用自动化è¿ç»´ã€‚ + +## Kubernetes 集群中节点的角色 + +Kubernetes 集群中的æ¯ä¸ªè®¡ç®—资æºç§°ä¸ºä¸€ä¸ª _节点_ 。节点å¯ä»¥æ˜¯è£¸é‡‘属æœåŠ¡å™¨æˆ–虚拟机。Kubernetes 将节点分为 _etcd_ 节点ã€_controlplane_ 节点和 _worker_ 节点。 + +一个 Kubernetes 集群至少包å«ä¸€ä¸ª etcd 节点，一个 controlplane 节点和一个 worker 节点。 + +### etcd 节点 + +Rancher 在å•èŠ‚点和高å¯ç”¨å®‰è£…中å‡ä½¿ç”¨ etcd 作为数æ®å­˜å‚¨ã€‚在 Kubernetes 中，etcd 也是存储集群状æ€çš„节点的角色。 + +Kubernetes 集群的状æ€ä¿å­˜åœ¨ [etcd](https://kubernetes.io/docs/concepts/overview/components/#etcd) 中。etcd 节点è¿è¡Œ etcd æ•°æ®åº“。 + +etcd æ•°æ®åº“组件是一个分布å¼çš„键值对存储系统，用于存储所有 Kubernetes 的集群数æ®ï¼Œä¾‹å¦‚集群å作和状æ€ç®¡ç†ç›¸å…³çš„æ•°æ®ã€‚建议在多个节点上è¿è¡Œ etcd，以ä¿è¯åœ¨å‘生故障时有å¯ç”¨çš„备份数æ®ã€‚ + +å³ä½¿ä½ å¯ä»¥ä»…在一个节点上è¿è¡Œ etcd，但 etcd 需è¦å¤§å¤šæ•°èŠ‚ç‚¹ï¼ˆå³ quorum）的åŒæ„æ‰èƒ½æ›´æ–°é›†ç¾¤çŠ¶æ€ã€‚集群需è¦åŒ…å«è¶³å¤Ÿæ•°é‡çš„å¥åº· etcd èŠ‚ç‚¹ï¼Œä»¥å½¢æˆ quorum。å‡è®¾é›†ç¾¤ä¸­æœ‰ n 个节点，quorum çš„æ•°é‡åˆ™éœ€è¦æ˜¯ (n/2)+1。如果集群节点数é‡æ˜¯å¥‡æ•°ï¼Œæ¯æ–°å¢žä¸€ä¸ªèŠ‚点，都会增加 quorum 所需节点数。 + +一般情况下，集群中åªè¦é…置三个 etcd 节点就能满足å°åž‹é›†ç¾¤çš„需求，五个 etcd 节点能满足大型集群的需求。 + +### controlplane 节点 + +controlplane 节点上è¿è¡Œ Kubernetes API serverã€scheduler å’Œ Controller Manager。这些节点执行日常任务，以确ä¿é›†ç¾¤çŠ¶æ€å’Œä½ çš„é…置一致。因为 etcd 节点ä¿å­˜äº†é›†ç¾¤çš„全部数æ®ï¼Œæ‰€ä»¥ controlplane 节点是无状æ€çš„。虽然你å¯ä»¥åœ¨å•ä¸ªèŠ‚点上è¿è¡Œ controlplane，但是我们建议在两个或以上的节点上è¿è¡Œ controlplane，以ä¿è¯å†—余性。å¦å¤–，一个节点å¯ä»¥æ—¢æ˜¯ controlplane 节点，åˆæ˜¯ etcd 节点。 + +### Worker 节点 + +æ¯ä¸ª [worker 节点](https://kubernetes.io/docs/concepts/architecture/nodes/)都能è¿è¡Œï¼š + +- **Kubelets**：监控节点状æ€çš„ Agent，确ä¿ä½ çš„容器处于å¥åº·çŠ¶æ€ã€‚ +- **工作负载**：承载应用和其他 deployment 的容器和 Pod。 + +Worker 节点也è¿è¡Œå­˜å‚¨å’Œç½‘络驱动，有必è¦æ—¶ä¹Ÿä¼šè¿è¡Œ Ingress Controller。你å¯ä»¥æ ¹æ®éœ€è¦ï¼Œåˆ›å»ºå°½å¯èƒ½å¤šçš„ worker 节点æ¥è¿è¡Œä½ çš„[工作负载](../pages-for-subheaders/workloads-and-pods.md)。 + +## 关于 Helm + +在 Rancher 高å¯ç”¨å®‰è£…的场景下，你å¯ä»¥ä½¿ç”¨ Helm 工具，把 Rancher 安装到 Kubernetes 集群上。 + +Helm 是 Kubernetes 的包管ç†å·¥å…·ã€‚Helm Chart 为 Kubernetes YAML 清å•æ–‡ä»¶æ供了模æ¿è¯­æ³•ã€‚通过 Helm，用户å¯ä»¥åˆ›å»ºå¯é…置的 deployment，而ä¸ä»…ä»…åªèƒ½ä½¿ç”¨é™æ€æ–‡ä»¶ã€‚如果你需è¦åˆ›å»ºè‡ªå·±çš„ deployment 商店应用，请å‚è§ [https://helm.sh/](https://helm.sh) 上的文档。 + +有关 ServiceAccount å’Œ Cluster Role Binding 的更多信æ¯ï¼Œè¯·å‚è§ [Kubernetes 官方文档](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/examples.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/examples.md new file mode 100644 index 00000000000..98169f63376 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/examples.md @@ -0,0 +1,23 @@ +--- +title: 示例 +--- + +### ServiceMonitor + +ä½ å¯ä»¥åœ¨[此处](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml)找到 ServiceMonitor 自定义资æºçš„示例。 + +### PodMonitor + +ä½ å¯ä»¥åœ¨[此处](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/example-app-pod-monitor.yaml)找到 PodMonitor 示例，还å¯ä»¥åœ¨[此处](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/prometheus-pod-monitor.yaml)找到引用它的 Prometheus 资æºç¤ºä¾‹ã€‚ + +### PrometheusRule + +PrometheusRule 包å«ä½ é€šå¸¸æ”¾ç½®åœ¨ [Prometheus 规则文件](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/)中的告警和记录规则。 + +è¦åœ¨é›†ç¾¤ä¸­æ›´ç»†ç²’度地应用 PrometheusRules，你å¯ä»¥ä½¿ç”¨ Prometheus 资æºçš„ ruleSelector 字段，从而根æ®æ·»åŠ åˆ° PrometheusRules 资æºçš„标签æ¥é€‰æ‹©è¦åŠ è½½åˆ° Prometheus 上的 PrometheusRule。 + +ä½ å¯ä»¥åœ¨[此页é¢](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/alerting.md)找到 PrometheusRule 示例。 + +### Alertmanager é…ç½® + +有关示例é…置，请å‚阅[本节](./receivers.md#alertmanager-é…置示例)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/helm-chart-options.md new file mode 100644 index 00000000000..476af17585c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -0,0 +1,68 @@ +--- +title: Helm Chart 选项 +--- + +## é…置资æºé™åˆ¶å’Œè¯·æ±‚ + +安装 `rancher-monitoring` æ—¶å¯ä»¥é…置资æºè¯·æ±‚å’Œé™åˆ¶ã€‚ + +默认值在 `rancher-monitoring` Helm Chart çš„ [values.yaml](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/values.yaml) 中。 + +下表中的默认值是所需的最低资æºé™åˆ¶å’Œè¯·æ±‚： + +| 资æºå称 | 内存é™åˆ¶ | CPU é™åˆ¶ | 内存请求 | CPU 请求 | +| ------------- | ------------ | ----------- | ---------------- | ------------------ | +| alertmanager | 500Mi | 1000m | 100Mi | 100m | +| grafana | 200Mi | 200m | 100Mi | 100m | +| kube-state-metrics subchart | 200Mi | 100m | 130Mi | 100m | +| prometheus-node-exporter subchart | 50Mi | 200m | 30Mi | 100m | +| prometheusOperator | 500Mi | 200m | 100Mi | 100m | +| prometheus | 2500Mi | 1000m | 1750Mi | 750m | +| **总计** | **3950Mi** | **2700m** | **2210Mi** | **1250m** | + +å»ºè®®è‡³å°‘é… 50Gi 存储。 + + +## Notifiers çš„å¯ä¿¡ CA + +如果你需è¦å°†å—信任的 CA 添加到 Notifiers，请执行以下步骤： + +1. 创建 `cattle-monitoring-system` 命å空间。 +1. 将你信任的 CA 密文添加到 `cattle-monitoring-system` 命å空间。 +1. 部署或å‡çº§ `rancher-monitoring` Helm Chart。在 Chart 选项中，引用**å‘Šè­¦ > 补充密文**中的密钥。 + +**结果**：默认的 Alertmanager 自定义资æºå°†æœ‰æƒè®¿é—®ä½ ä¿¡ä»»çš„ CA。 + + +## 其它抓å–é…ç½® + +如果无法通过 ServiceMonitor 或 PodMonitor 指定你想è¦çš„抓å–é…置，你å¯ä»¥åœ¨éƒ¨ç½²æˆ–å‡çº§ `rancher-monitoring` æ—¶æä¾› `additionalScrapeConfigSecret`。 + +[scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) 指定一组目标以åŠæŠ“å–这些目标的å‚数。在一般情况下，一个抓å–é…置指定一个 job。 + +Istio 就是一个å¯èƒ½ç”¨åˆ°è¿™ä¸ªé…置的例å­ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md)。 + + +## é…置打包在 Monitoring V2 中的应用 + +我们使用 Monitoring V2 æ¥éƒ¨ç½² kube-state-metrics å’Œ node-exporter。node-exporter 部署为 DaemonSet。对于 Monitoring V2 helm Chart，values.yaml ä¸­çš„æ‰€æœ‰ä¸œè¥¿éƒ½éƒ¨ç½²ä¸ºå­ Chart。 + +我们还部署了ä¸ç”± prometheus 管ç†çš„ Grafana。 + +如果你查看 Helm Chart 在 kube-state-metrics 中的功能，则还有很多å¯ä»¥è®¾ç½®çš„值没有在顶层 Chart 中显示。 + +但是，你å¯ä»¥åœ¨é¡¶å±‚ Chart ä¸­æ·»åŠ è¦†ç›–å­ Chart 的值。 + +### 增加 Alertmanager 的副本 + +作为 Chart 部署选项的一部分，你å¯ä»¥é€‰æ‹©å¢žåŠ éƒ¨ç½²åˆ°é›†ç¾¤ä¸Šçš„ Alertmanager 副本的数é‡ã€‚这些副本使用相åŒçš„底层 Alertmanager Config Secret 进行管ç†ã€‚有关 Alertmanager Config Secret 的更多信æ¯ï¼Œè¯·å‚阅[本节](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md#多个-alertmanager-副本)。 + +### 为æŒä¹…化 Grafana 仪表æ¿é…置命å空间 + +è¦è®© Grafana 监控所有命å空间中的 ConfigMap，请在 `rancher-monitoring` Helm chart 中指定以下值： + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +请注æ„，Monitoring Chart 用于添加 Grafana 仪表æ¿çš„ RBAC 角色仅能让用户将仪表æ¿æ·»åŠ åˆ°å®šä¹‰åœ¨ `grafana.dashboards.namespace` 中的命å空间，默认为 `cattle-dashboards`。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/receivers.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/receivers.md new file mode 100644 index 00000000000..e961ec75d77 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/receivers.md @@ -0,0 +1,299 @@ +--- +title: 接收器é…ç½® +--- + +[Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret åŒ…å« Alertmanager 实例的é…ç½®ï¼Œè¯¥å®žä¾‹æ ¹æ® Prometheus å‘出的告警å‘é€é€šçŸ¥ã€‚ + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-alertmanager-的工作原ç†)。 + +::: + + +## 在 Rancher UI 中创建接收器 + +:::note 先决æ¡ä»¶ï¼š + +- 已安装 Monitoring 应用。 +- 如果你使用现有的 Alertmanager Secret é…ç½® Monitoring，则它必须具有 Rancher çš„ UI 支æŒçš„æ ¼å¼ã€‚å¦åˆ™ï¼Œä½ å°†åªèƒ½ç›´æŽ¥ä¿®æ”¹ Alertmanager Secret æ¥è¿›è¡Œæ›´æ”¹ã€‚请注æ„，对于通过使用路由和接收器 UI 支æŒçš„ Alertmanager é…置类型，我们会继续进行强化。因此如果你有增强功能的请求，请[æ交 issue](https://github.com/rancher/rancher/issues/new)。 + +::: + +è¦åœ¨ Rancher UI 中创建通知接收器： + + + + +1. 转到è¦åˆ›å»ºæŽ¥æ”¶å™¨çš„集群。å•å‡» **监控 -> å‘Šè­¦ -> AlertManagerConfigs**。 +1. å•å‡»**创建**。 +1. 点击**添加接收器**。 +1. 输入接收器的**å称**。 +1. 为接收器é…置一个或多个æ供程åºã€‚如需获å–填写表å•çš„帮助，请å‚阅下方的é…置选项。 +1. å•å‡»**创建**。 + + + + +1. 转到è¦åˆ›å»ºæŽ¥æ”¶å™¨çš„集群。å•å‡»**监控**，然åŽå•å‡»**接收器**。 +2. 输入接收器的å称。 +3. 为接收器é…置一个或多个æ供程åºã€‚如需获å–填写表å•çš„帮助，请å‚阅下方的é…置选项。 +4. å•å‡»**创建**。 + + + + +**结果**：告警å¯ä»¥å‘接收器å‘é€é€šçŸ¥ã€‚ + +## 接收器é…ç½® + +通知集æˆæ˜¯é€šè¿‡ `receiver` é…置的，[Prometheus](https://prometheus.io/docs/alerting/latest/configuration/#receiver) 文档对此进行了说明。 + +### 原生和éžåŽŸç”ŸæŽ¥æ”¶å™¨ + +默认情况下，AlertManager æ供与一些接收器的原生集æˆï¼Œè¿™äº›æŽ¥æ”¶å™¨åœ¨[本节](https://prometheus.io/docs/alerting/latest/configuration/#receiver)中列出。所有原生支æŒçš„接收器都å¯ä»¥é€šè¿‡ Rancher UI 进行é…置。 + +对于 AlertManager ä¸æ供原生支æŒçš„通知机制，å¯ä½¿ç”¨ [webhook 接收器](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config)实现集æˆã€‚ä½ å¯ä»¥åœ¨[此处](https://prometheus.io/docs/operating/integrations/#alertmanager-webhook-receiver)找到æ供此类集æˆçš„第三方驱动程åºåˆ—表。Alerting Drivers 应用能让你访问这些驱动程åºï¼Œä»¥åŠå®ƒä»¬ç›¸å…³çš„集æˆã€‚å¯ç”¨åŽï¼Œä½ å°†å¯ä»¥åœ¨ Rancher UI 中é…ç½®éžåŽŸç”Ÿçš„接收器。 + +ç›®å‰ Rancher Alerting Drivers 应用支æŒè®¿é—®ä»¥ä¸‹é›†æˆï¼š +- Microsoft Teams，基于 [prom2teams](https://github.com/idealista/prom2teams) é©±åŠ¨ç¨‹åº +- SMS，基于 [Sachet](https://github.com/messagebird/sachet) é©±åŠ¨ç¨‹åº + +ä½ å¯ä»¥åœ¨ Rancher UI 中å¯ä»¥é…置以下类型的接收器： + +- Slack +- 电å­é‚®ä»¶ +- PagerDuty +- Opsgenie +- Webhook +- 自定义 +- Teams +- SMS + +ä½ å¯ä»¥åœ¨ YAML 中使用自定义接收器选项，从而é…置无法通过 Rancher UI 表å•é…置的接收器。 + +## Slack + +| 字段 | 类型 | æè¿° | +|------|--------------|------| +| URL | String | 输入你的 Slack webhook URL。有关创建 Slack webhook 的说明，请å‚阅 [Slack 文档](https://get.slack.help/hc/en-us/articles/115005265063-Incoming-WebHooks-for-Slack)。 | +| é»˜è®¤é¢‘é“ | String | 输入è¦å‘é€å‘Šè­¦é€šçŸ¥çš„频é“å称。格å¼ï¼š`#`。 | +| ä»£ç† URL | String | webhook 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | Bool | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +## 电å­é‚®ä»¶ + +| 字段 | 类型 | æè¿° | +|------|--------------|------| +| é»˜è®¤æ”¶ä»¶äººåœ°å€ | String | 接收通知的电å­é‚®ä»¶åœ°å€ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | Bool | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +SMTP 选项： + +| 字段 | 类型 | æè¿° | +|------|--------------|------| +| å‘件人 | String | ä½ çš„ SMTP 邮件æœåŠ¡å™¨ä¸Šå¯ç”¨çš„电å­é‚®ä»¶åœ°å€ï¼Œç”¨äºŽå‘é€é€šçŸ¥ã€‚ | +| 主机 | String | SMTP æœåŠ¡å™¨çš„ IP 地å€æˆ–主机å。示例：`smtp.email.com`。 | +| 使用 TLS | Bool | 使用 TLS 进行加密。 | +| 用户å | String | 用户å，用于通过 SMTP æœåŠ¡å™¨è¿›è¡Œèº«ä»½éªŒè¯ã€‚ | +| å¯†ç  | String | 密ç ï¼Œç”¨äºŽé€šè¿‡ SMTP æœåŠ¡å™¨è¿›è¡Œèº«ä»½éªŒè¯ã€‚ | + +## PagerDuty + +| 字段 | 类型 | æè¿° | +|------|------|-------| +| 集æˆç±»åž‹ | String | `Events API v2` 或 `Prometheus`。 | +| 默认集æˆå¯†é’¥ | String | 有关获å–集æˆå¯†é’¥çš„说明，请å‚阅 [PagerDuty 文档](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/)。 | +| ä»£ç† URL | String | PagerDuty 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | Bool | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +## Opsgenie + +| 字段 | æè¿° | +|------|-------------| +| API 密钥 | æœ‰å…³èŽ·å– API 密钥的说明，请å‚阅 [Opsgenie 文档](https://docs.opsgenie.com/docs/api-key-management)。 | +| ä»£ç† URL | Opsgenie 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + +Opsgenie å“应者： + +| 字段 | 类型 | æè¿° | +|-------|------|--------| +| 类型 | String | 计划程åºã€å›¢é˜Ÿã€ç”¨æˆ·æˆ–å‡çº§ã€‚有关告警å“应者的更多信æ¯ï¼Œè¯·å‚阅 [Opsgenie 文档](https://docs.opsgenie.com/docs/alert-recipients-and-teams)。 | +| å‘é€è‡³ | String | Opsgenie 收件人的 IDã€å称或用户å。 | + +## Webhook + +| 字段 | æè¿° | +|-------|--------------| +| URL | 你所选的应用的 Webhook URL。 | +| ä»£ç† URL | webhook 通知的代ç†ã€‚ | +| å‘é€å·²è§£å†³å‘Šè­¦ | 如果告警已解决（例如 [已解决] CPU 使用率过高问题），是å¦å‘é€åŽç»­é€šçŸ¥ã€‚ | + + + +## 自定义 + +此处æ供的 YAML 将直接附加到 Alertmanager Config Secret 的接收器中。 + +## Teams + +### 为 Rancher 管ç†çš„集群å¯ç”¨ Teams 接收器 + +Teams 接收器ä¸æ˜¯åŽŸç”ŸæŽ¥æ”¶å™¨ï¼Œå› æ­¤éœ€è¦å¯ç”¨åŽæ‰èƒ½ä½¿ç”¨ã€‚ä½ å¯ä»¥é€šè¿‡è½¬åˆ°åº”用页é¢ï¼Œå®‰è£… rancher-alerting-drivers 应用，然åŽé€‰æ‹© Teams 选项，从而为 Rancher 管ç†çš„集群å¯ç”¨ Teams 接收器。 + +1. 在 Rancher UI 中，转到è¦å®‰è£… rancher-alerting-drivers 的集群，然åŽå•å‡» **Apps**。 +1. 点击 **Alerting Drivers** 应用。 +1. å•å‡» **Helm 部署选项**选项å¡ã€‚ +1. 选择 **Teams** 并å•å‡»**安装**。 +1. 记下使用的命å空间，åŽç»­æ­¥éª¤ä¸­å°†éœ€è¦è¯¥å‘½å空间。 + +### é…ç½® Teams 接收器 + +å¯ä»¥é€šè¿‡æ›´æ–° ConfigMap æ¥é…ç½® Teams 接收器。例如，以下是最å°çš„ Teams 接收器é…置： + +```yaml +[Microsoft Teams] +teams-instance-1: https://your-teams-webhook-url +``` + +é…置完æˆåŽï¼ŒæŒ‰ç…§[本节](#在-rancher-ui-中创建接收器)中的步骤添加接收器。 + +使用以下示例作为 URL，其中： + +- å°† `ns-1` 替æ¢ä¸ºå®‰è£… `rancher-alerting-drivers` 应用的命å空间。 + +```yaml +url: http://rancher-alerting-drivers-prom2teams.ns-1.svc:8089/v2/teams-instance-1 +``` + + + +## SMS + +### 为 Rancher 管ç†çš„集群å¯ç”¨ SMS 接收器 + +SMS 接收器ä¸æ˜¯åŽŸç”ŸæŽ¥æ”¶å™¨ï¼Œå› æ­¤éœ€è¦å¯ç”¨åŽæ‰èƒ½ä½¿ç”¨ã€‚ä½ å¯ä»¥é€šè¿‡è½¬åˆ°åº”用页é¢ï¼Œå®‰è£… rancher-alerting-drivers 应用，然åŽé€‰æ‹© SMS 选项，从而为 Rancher 管ç†çš„集群å¯ç”¨ SMS 接收器。 + +1. 在左上角，å•å‡» **☰ > 集群管ç†**。 +1. 在**集群**页é¢ä¸Šï¼Œè½¬åˆ°è¦å®‰è£… `rancher-alerting-drivers` 的集群，然åŽå•å‡» **Explore**。 +1. 在左侧导航æ ä¸­ï¼Œå•å‡»**应用 & 应用市场**。 +1. 点击 **Alerting Drivers** 应用。 +1. å•å‡» **Helm 部署选项**选项å¡ã€‚ +1. 选择 **SMS** 并å•å‡»**安装**。 +1. 记下使用的命å空间，åŽç»­æ­¥éª¤ä¸­å°†éœ€è¦è¯¥å‘½å空间。 + +### é…ç½® SMS 接收器 + +å¯ä»¥é€šè¿‡æ›´æ–° ConfigMap æ¥é…ç½® SMS 接收器。例如，以下是最å°çš„ SMS 接收器é…置： + +```yaml +providers: + telegram: + token: 'your-token-from-telegram' + +receivers: +- name: 'telegram-receiver-1' + provider: 'telegram' + to: + - '123456789' +``` + +é…置完æˆåŽï¼ŒæŒ‰ç…§[本节](#在-rancher-ui-中创建接收器)中的步骤添加接收器。 + +使用以下示例作为å称和 URL，其中： + +- 分é…给接收器的å称（例如 `telegram-receiver-1`）必须与 ConfigMap 中 `receivers.name` 字段中的å称（例如 `telegram-receiver-1`）匹é…。 +- å°† URL 中的 `ns-1` 替æ¢ä¸ºå®‰è£… `rancher-alerting-drivers` 应用的命å空间。 + +```yaml +name: telegram-receiver-1 +url http://rancher-alerting-drivers-sachet.ns-1.svc:9876/alert +``` + + + + +## é…置多个接收器 + +ä½ å¯ä»¥ç¼–辑 Rancher UI 中的表å•æ¥è®¾ç½®ä¸€ä¸ªæŽ¥æ”¶å™¨èµ„æºï¼Œå…¶ä¸­åŒ…å« Alertmanager 将告警å‘é€åˆ°ä½ çš„通知系统所需的所有信æ¯ã€‚ + +也å¯ä»¥å‘多个通知系统å‘é€å‘Šè­¦ã€‚一ç§æ–¹æ³•æ˜¯ä½¿ç”¨è‡ªå®šä¹‰ YAML æ¥é…置接收器。如果你需è¦è®©ä¸¤ä¸ªç³»ç»ŸæŽ¥æ”¶ç›¸åŒçš„消æ¯ï¼Œåˆ™å¯ä»¥ä¸ºå¤šä¸ªé€šçŸ¥ç³»ç»Ÿæ·»åŠ é…置。 + +你还å¯ä»¥é€šè¿‡ä½¿ç”¨è·¯ç”±çš„ `continue` 选项æ¥è®¾ç½®å¤šä¸ªæŽ¥æ”¶å™¨ã€‚这样，å‘é€åˆ°æŽ¥æ”¶å™¨çš„告警会在路由树（å¯èƒ½åŒ…å«å¦ä¸€ä¸ªæŽ¥æ”¶å™¨ï¼‰çš„下一级进行评估。 + + +## Alertmanager é…置示例 + +### Slack +è¦é€šè¿‡ Slack 设置通知，你å¯ä»¥å°†ä»¥ä¸‹ Alertmanager Config YAML 放入 Alertmanager Config Secret çš„ `alertmanager.yaml` 键中，你需è¦æ›´æ–° `api_url` æ¥ä½¿ç”¨æ¥è‡ª Slack çš„ Webhook URL： + +```yaml +route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + receiver: 'slack-notifications' +receivers: +- name: 'slack-notifications' + slack_configs: + - send_resolved: true + text: '{{ template "slack.rancher.text" . }}' + api_url: +templates: +- /etc/alertmanager/config/*.tmpl +``` + +### PagerDuty +è¦é€šè¿‡ PagerDuty 设置通知，请使用 [PagerDuty 文档](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/) 中的以下示例作为指导。此示例设置了一个路由，该路由æ•èŽ·æ•°æ®åº“æœåŠ¡çš„告警，并将告警å‘é€åˆ°é“¾æŽ¥åˆ°æœåŠ¡çš„接收器，该æœåŠ¡å°†ç›´æŽ¥é€šçŸ¥ PagerDuty 中的 DBA，而其他告警将被定å‘到具有ä¸åŒ PagerDuty 集æˆå¯†é’¥çš„默认接收器。 + +ä½ å¯ä»¥å°†ä»¥ä¸‹ Alertmanager Config YAML 放入 Alertmanager Config Secret çš„ `alertmanager.yaml` 键中。你需è¦å°† `service_key` 更新为使用你的 PagerDuty 集æˆå¯†é’¥ï¼Œå¯ä»¥æ ¹æ® PagerDuty 文档的 "Integrating with Global Event Routing" 找到该密钥。有关é…置选项的完整列表，请å‚阅 [Prometheus 文档](https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config)。 + +```yaml +route: + group_by: [cluster] + receiver: 'pagerduty-notifications' + group_interval: 5m + routes: + - match: + service: database + receiver: 'database-notifcations' + +receivers: +- name: 'pagerduty-notifications' + pagerduty_configs: + - service_key: 'primary-integration-key' + +- name: 'database-notifcations' + pagerduty_configs: + - service_key: 'database-integration-key' +``` + +## CIS 扫æ告警的示例路由é…ç½® + +在为 `rancher-cis-benchmark` å‘Šè­¦é…置路由时，你å¯ä»¥ä½¿ç”¨é”®å€¼å¯¹ `job:rancher-cis-scan` æ¥æŒ‡å®šåŒ¹é…。 + +例如，以下路由é…置示例å¯ä»¥ä¸Žå为 `test-cis` çš„ Slack 接收器一起使用： + +```yaml +spec: + receiver: test-cis + group_by: +# - string + group_wait: 30s + group_interval: 30s + repeat_interval: 30s + match: + job: rancher-cis-scan +# key: string + match_re: + {} +# key: string +``` + +有关为 `rancher-cis-benchmark` å¯ç”¨å‘Šè­¦çš„更多信æ¯ï¼Œè¯·å‚阅[本节](../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md)。 + +## Notifiers çš„å¯ä¿¡ CA + +如果你需è¦å°†å—信任的 CA 添加到 Notifiers，请按照[本节](helm-chart-options.md#notifiers-çš„å¯ä¿¡-ca)中的步骤æ“作。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/routes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/routes.md new file mode 100644 index 00000000000..2a3da893016 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/routes.md @@ -0,0 +1,64 @@ +--- +title: 路由é…ç½® +--- + +路由（Route）é…置是 Alertmanager 自定义资æºçš„一部分，用于控制 Prometheus 触å‘的告警在到达接收器之å‰çš„分组和过滤方å¼ã€‚ + +当路由更改时，Prometheus Operator 会é‡æ–°ç”Ÿæˆ Alertmanager 自定义资æºä»¥å映更改。 + +有关é…置路由的更多信æ¯ï¼Œè¯·å‚阅[官方 Alertmanager 文档](https://www.prometheus.io/docs/alerting/latest/configuration/#route)。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + + + +## 路由é™åˆ¶ + +Alertmanager æ ¹æ®æŽ¥æ”¶å™¨å’Œè·¯ç”±æ ‘æ¥ä»£ç† Prometheus 的告警，该路由树根æ®æ ‡ç­¾å°†å‘Šè­¦è¿‡æ»¤åˆ°æŒ‡å®šæŽ¥æ”¶å™¨ã€‚ + +Alerting Drivers 为 Alertmanager 将告警代ç†åˆ°éžåŽŸç”ŸæŽ¥æ”¶å™¨ï¼Œä¾‹å¦‚ Microsoft Teams å’Œ SMS。 + +在é…置路由和接收器的 Rancher UI 中，你å¯ä»¥é…置有一个根的路由树，然åŽå†é…置一个深度，这样的树就有两个深度。但是如果在直接é…ç½® Alertmanager 时使用 `continue` 路由，你å¯ä»¥è®©æ ‘更深。 + +æ¯ä¸ªæŽ¥æ”¶å™¨ç”¨äºŽä¸€ä¸ªæˆ–多个通知æ供商。因此，如果你需è¦å°†å‘é€åˆ° Slack çš„æ¯ä¸ªå‘Šè­¦ä¹Ÿå‘é€åˆ° PagerDuty，你å¯ä»¥åœ¨åŒä¸€ä¸ªæŽ¥æ”¶å™¨ä¸­é…置两者。 + +## 路由é…ç½® + +### 标签和注释的注æ„事项 + +标签用于识别å¯èƒ½å½±å“通知路由的信æ¯ã€‚告警的标识信æ¯å¯èƒ½åŒ…括容器å称，或应接收通知的团队的å称。 + +注释用于标识ä¸å½±å“告警接收者的信æ¯ï¼Œä¾‹å¦‚ Runbook URL 或错误消æ¯ã€‚ + + +### 接收器 +路由需è¦å¼•ç”¨ä¸€ä¸ªå·²ç»é…置好的[接收器](./receivers.md)。 + +### 分组 + +| 字段 | 默认 | æè¿° | +|-------|--------------|---------| +| 分组ä¾æ® | N/A | 用于分组的标签列表。标签必须是唯一的。如果æ供了特殊标签“...â€ï¼ˆç”±æ‰€æœ‰å¯èƒ½çš„标签èšåˆï¼‰ï¼Œæ ‡ç­¾å¿…须在列表中是唯一的元素。接å—字符串列表。有关详细信æ¯ï¼Œè¯·å‚阅[上游文档](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#route)。 | +| 组等待时长 | 30s | 在å‘é€ä¹‹å‰ï¼Œç¼“冲åŒä¸€ç»„告警的等待时间。 | +| 组间隔 | 5m | 等待多长时间æ‰å‘é€å·²æ·»åŠ åˆ°å‘Šè­¦ç»„的告警，其中该告警组的åˆæ¬¡é€šçŸ¥å·²è¢«å‘é€ã€‚ | +| é‡å¤é—´éš” | 4h | 等待多长时间åŽï¼Œæ‰é‡æ–°å‘é€å·²å‘é€çš„告警。 | + +### åŒ¹é… + +**Match** 字段指一组相等匹é…器，用于根æ®å‘Šè­¦ä¸Šå®šä¹‰çš„标签æ¥è¯†åˆ«è¦å‘é€åˆ°æŒ‡å®šè·¯ç”±çš„告警。在 Rancher UI 中添加键值对时，它们对应于以下格å¼çš„ YAML： + +```yaml +match: + [ : , ... ] +``` + +**Match Regex** 字段指一组正则表达å¼åŒ¹é…器，用于根æ®åœ¨è¯¥å‘Šè­¦ä¸Šå®šä¹‰çš„标签æ¥è¯†åˆ«è¦å‘é€åˆ°æŒ‡å®šè·¯ç”±çš„告警。在 Rancher UI 中添加键值对时，它们对应于以下格å¼çš„ YAML： + +```yaml +match_re: + [ : , ... ] +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md new file mode 100644 index 00000000000..e514d60a928 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -0,0 +1,33 @@ +--- +title: ServiceMonitor å’Œ PodMonitor é…ç½® +--- + +ServiceMonitor å’Œ PodMonitor 都是伪 CRD，它们映射 Prometheus 自定义资æºçš„抓å–é…置。 + +这些é…置对象以声明方å¼æŒ‡å®š Prometheus 抓å–指标的端点。 + +ServiceMonitor 比 PodMonitor 更常用，推è用于大多数用例。 + +:::note + +本节å‚考å‡è®¾ä½ å·²ç»ç†Ÿæ‚‰ Monitoring 组件的ååŒå·¥ä½œæ–¹å¼ã€‚有关 Alertmanager 的详细信æ¯ï¼Œè¯·å‚阅[本节](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md)。 + +::: + +### ServiceMonitor + +这个伪 CRD 映射到 Prometheus 自定义资æºé…置的一部分。它以声明方å¼æŒ‡å®šåº”如何监控 Kubernetes æœåŠ¡ç»„。 + +创建 ServiceMonitor 时，Prometheus Operator 会更新 Prometheus 抓å–é…ç½®ï¼Œä»Žè€ŒåŒ…å« ServiceMonitor é…ç½®ã€‚ç„¶åŽ Prometheus 开始从 ServiceMonitor 中定义的端点抓å–指标。 + +如果集群中的任何 Service 与 ServiceMonitor `selector` 字段中的标签匹é…ï¼Œåˆ™ä¼šæ ¹æ® ServiceMonitor 指定的 `endpoints` 进行监控。有关å¯ä»¥æŒ‡å®šçš„字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ Prometheus Operator çš„[规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor)。 + +有关 ServiceMonitor 工作原ç†çš„更多信æ¯ï¼Œè¯·å‚阅 [Prometheus Operator 文档](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/running-exporters.md)。 + +### PodMonitor + +这个伪 CRD 映射到 Prometheus 自定义资æºé…置的一部分。它以声明方å¼æŒ‡å®šåº”如何监控 Pod 组。 + +创建 PodMonitor 时，Prometheus Operator 会更新 Prometheus 抓å–é…ç½®ï¼Œä»Žè€ŒåŒ…å« PodMonitor é…ç½®ã€‚ç„¶åŽ Prometheus 开始从 PodMonitor 中定义的端点抓å–指标。 + +如果集群中的任何 Pod 与 PodMonitor `selector` 字段中的标签匹é…ï¼Œåˆ™ä¼šæ ¹æ® PodMonitor 指定的 `podMetricsEndpoints` 进行监控。有关å¯ä»¥æŒ‡å®šçš„字段的更多信æ¯ï¼Œè¯·æŸ¥çœ‹ Prometheus Operator çš„[规范](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#podmonitorspec)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/prometheus-federator/rbac.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/prometheus-federator/rbac.md new file mode 100644 index 00000000000..a519b5a8cbc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/prometheus-federator/rbac.md @@ -0,0 +1,31 @@ +--- +title: RBAC +--- + +æœ¬æ–‡ä»‹ç» Prometheus Federator RBAC。 + +如[命å空间](../../pages-for-subheaders/prometheus-federator.md#命å空间)部分所述，Prometheus Federator 期望集群中具有项目级别æƒé™ï¼ˆä¾‹å¦‚，具有由å•ä¸ªæ ‡ç­¾é€‰æ‹©å™¨ç¡®å®šçš„命å空间组的æƒé™ï¼‰çš„项目所有者ã€é¡¹ç›®æˆå‘˜å’Œå…¶ä»–用户，除了项目 Registration 命å空间（默认导入到项目中）和那些已ç»åŒ…å«å…¶é¡¹ç›®çš„命å空间之外，在任何其他命å空间中都åªæœ‰æœ€ä½Žæƒé™ã€‚因此，为了让项目所有者将特定 Chart æƒé™åˆ†é…给其项目命å空间中的其他用户，Helm Project Operator 将自动监视以下绑定： + +- ClusterRoleBindings +- 项目å‘布命å空​​间中的 RoleBindings + +如果 Helm Project Operator 观察到其中一ç§ç»‘定的更改，Helm Project Operator 会检查绑定指å‘çš„ `roleRef` 是å¦ä¸Žå…·æœ‰ä»¥ä¸‹å称的 ClusterRole 匹é…： + +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view` + +默认情况下，这些 roleRef 分别对应 `admin`ã€`edit` å’Œ `view`，它们都是 [Kubernetes é¢å‘用户的默认角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)。 + +:::note + +对于 Rancher RBAC 用户，这些 [Kubernetes é¢å‘用户的默认角色](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)与`项目所有者`ã€`项目æˆå‘˜`å’Œ`åªè¯»`默认项目角色模æ¿ç›´æŽ¥å¯¹åº”。 + +::: + +如果 `roleRef` 匹é…，Helm Project Operator 将为所有用户和组过滤绑定的 `subjects`，并使用它为项目 Release 命å空间中的æ¯ä¸ªè§’色自动构造一个 RoleBinding，该 RoleBinding çš„å称与角色相åŒå¹¶å¸¦æœ‰ä»¥ä¸‹æ ‡ç­¾ï¼š + +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +默认情况下，Prometheus Federator 部署的底层 Chart `rancher-project-monitoring` 会为æ¯ä¸ªé¡¹ç›®å‘布命åç©ºâ€‹â€‹é—´åˆ›å»ºä¸‰ä¸ªé»˜è®¤è§’è‰²ï¼Œè¿™äº›è§’è‰²èƒ½æŽˆæƒ `admin`ã€`edit` å’Œ `view` 用户查看项目监控堆栈的 Prometheusã€Alertmanager å’Œ Grafana UI，从而æ供最低æƒé™ã€‚如果集群管ç†å‘˜æƒ³è¦ä¸ºæŸäº›ç”¨æˆ·åˆ†é…é¢å¤–çš„æƒé™ï¼Œä¸€ç§åšæ³•æ˜¯ç›´æŽ¥å°†é¡¹ç›® Release 命å空间中的 RoleBinding 分é…ç»™æŸäº›ç”¨æˆ·ã€‚å¦ä¸€ç§åšæ³•æ˜¯åˆ›å»ºå¸¦æœ‰ä¸Šè¿°ä¸¤ä¸ªæ ‡ç­¾çš„角色，然åŽï¼Œé¡¹ç›®æ‰€æœ‰è€…å¯ä»¥æŽ§åˆ¶åœ¨é¡¹ç›® Registration 命å空间中分é…这些 RBAC 角色的用户。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-cluster-tools.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-cluster-tools.md new file mode 100644 index 00000000000..7f1ed5206c7 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-cluster-tools.md @@ -0,0 +1,48 @@ +--- +title: 集群工具：Logging，Monitoring å’Œå¯è§†åŒ– +--- + +Rancher åŒ…å« Kubernetes 中未包å«çš„å„ç§å·¥å…·æ¥å助你进行 DevOps æ“作。Rancher å¯ä»¥ä¸Žå¤–部æœåŠ¡é›†æˆï¼Œè®©ä½ çš„集群更高效地è¿è¡Œã€‚工具分为以下几类： + + +## Logging + +Logging 支æŒï¼š + +- 获å–和分æžé›†ç¾¤çš„çŠ¶æ€ +- 在你的环境中å‘现趋势 +- 将日志ä¿å­˜åˆ°é›†ç¾¤å¤–部的安全ä½ç½® +- éšæ—¶äº†è§£å®¹å™¨å´©æºƒã€pod 驱é€æˆ–节点死亡等事件 +- æ›´è½»æ¾åœ°è°ƒè¯•å’Œè§£å†³é—®é¢˜ + +Rancher å¯ä»¥ä¸Ž Elasticsearchã€splunkã€kafkaã€syslog å’Œ fluentd 集æˆã€‚ + +有关详细信æ¯ï¼Œè¯·å‚阅 [Logging 文档](../pages-for-subheaders/logging.md)。 +## 监控和告警 + +ä½ å¯ä»¥ä½¿ç”¨ Rancher，通过业界领先并开æºçš„ [Prometheus](https://prometheus.io/) æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚ + +å¯ç”¨ç›‘控åŽï¼Œä½ å¯ä»¥é€šè¿‡è®¾ç½®å‘Šè­¦å’Œé€šçŸ¥å™¨æ¥é…置接收告警的机制。 + +通知器是通知你告警事件的æœåŠ¡ã€‚ä½ å¯ä»¥é€šè¿‡é…置通知器æ¥å‘最适åˆé‡‡å–纠正措施的员工å‘é€å‘Šè­¦é€šçŸ¥ã€‚支æŒä½¿ç”¨ Slackã€ç”µå­é‚®ä»¶ã€PagerDutyã€å¾®ä¿¡å’Œ webhook å‘é€é€šçŸ¥ã€‚ + +告警是触å‘这些通知的规则。在接收告警之å‰ï¼Œä½ å¿…须在 Rancher 中é…置一个或多个通知器。你å¯ä»¥åœ¨é›†ç¾¤æˆ–项目级别设置告警范围。 + +如需更多信æ¯ï¼Œè¯·å‚阅[监控文档](../pages-for-subheaders/monitoring-and-alerting.md)。 + +## Istio + +[Istio](https://istio.io/) 是一ç§å¼€æºå·¥å…·ï¼Œå¯ä»¥è®© DevOps 团队更轻æ¾åœ°è§‚察ã€æŽ§åˆ¶ã€æŽ’查并ä¿æŠ¤å¤æ‚çš„å¾®æœåŠ¡ç½‘络中的æµé‡ã€‚ + +Rancher v2.5 改进了与 Istio 的集æˆã€‚ + +如需更多信æ¯ï¼Œè¯·å‚阅 [Istio 文档](../pages-for-subheaders/istio.md)。 +## OPA Gatekeeper + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) 是一个开æºé¡¹ç›®ï¼Œå®ƒå¯¹ OPA å’Œ Kubernetes 进行了集æˆï¼Œä»¥é€šè¿‡è®¸å¯æŽ§åˆ¶å™¨ Webhook æ供策略控制。有关如何在 Rancher 中å¯ç”¨ Gatekeeper 的详细信æ¯ï¼Œè¯·å‚阅 [OPA Gatekeeper](../integrations-in-rancher/opa-gatekeeper.md)。 + +## CIS 扫æ + +Rancher å¯ä»¥é€šè¿‡è¿è¡Œå®‰å…¨æ‰«ææ¥æ£€æŸ¥ Kubernetes 是å¦æŒ‰ç…§ CIS Kubernetes Benchmark 中定义的安全最佳实践进行部署。 + +如需更多信æ¯ï¼Œè¯·å‚阅 [CIS 扫æ文档](../pages-for-subheaders/cis-scan-guides.md)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/architecture-recommendations.md new file mode 100644 index 00000000000..10ae10563b0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -0,0 +1,106 @@ +--- +title: 架构推è +--- + +如果你准备在å•ä¸ªèŠ‚点上安装 Rancher，我们推èä½ [分开部署 Rancher 与下游集群](#分开部署-rancher-与下游集群)。 + +## 分开部署 Rancher 与下游集群 + +下游集群，是è¿è¡Œä½ è‡ªå·±çš„应用和æœåŠ¡çš„下游 Kubernetes 集群。 + +如果你通过 Docker 安装了 Rancher，è¿è¡Œ Rancher Server 的节点应该与你的下游集群分开。 + +如果你需è¦ä½¿ç”¨ Rancher 管ç†ä¸‹æ¸¸ Kubernetes 集群，那么è¿è¡Œ Rancher Server çš„ Kubernetes 集群也应该与下游集群分开。 + +![分开部署 Rancher Server 与下游集群](/img/rancher-architecture-separation-of-rancher-server.svg) + +## 为什么高å¯ç”¨ï¼ˆHA）更适åˆç”Ÿäº§çŽ¯å¢ƒä¸­çš„ Rancher + +我们建议在高å¯ç”¨ Kubernetes 集群上安装 Rancher Server，以ä¿æŠ¤ Rancher Server çš„æ•°æ®ã€‚在高å¯ç”¨å®‰è£…中，负载å‡è¡¡å™¨å……当客户端的å•ç‚¹å…¥å£ï¼Œå¹¶åœ¨é›†ç¾¤ä¸­çš„多å°æœåŠ¡å™¨ä¹‹é—´åˆ†é…网络æµé‡ï¼Œè¿™æœ‰åŠ©äºŽé˜²æ­¢ä»»ä½•ä¸€å°æœåŠ¡å™¨æˆä¸ºå•ç‚¹æ•…障。 + +我们ä¸å»ºè®®åœ¨å•ä¸ª Docker 容器中安装 Rancher，因为如果该节点å‘生故障，则其他节点上将没有å¯ç”¨çš„集群数æ®å‰¯æœ¬ï¼Œå¹¶ä¸”ä½ å¯èƒ½ä¼šä¸¢å¤± Rancher Server 上的数æ®ã€‚ + +### K3s Kubernetes 集群安装 + +底层 Kubernetes 集群的一ç§é€‰æ‹©æ˜¯ä½¿ç”¨ K3s Kubernetes。K3s 是 Rancher CNCF 认è¯çš„ Kubernetes å‘行版。K3s æ˜“äºŽå®‰è£…ï¼Œä»…éœ€è¦ Kubernetes 内存的一åŠï¼Œæ‰€æœ‰ç»„件都在一个å°äºŽ 100 MB 的二进制文件中。K3s çš„å¦ä¸€ä¸ªä¼˜ç‚¹æ˜¯å…许外部 Datastore ä¿å­˜é›†ç¾¤æ•°æ®ï¼Œå› æ­¤å¯ä»¥æŠŠ K3s æœåŠ¡å™¨èŠ‚点视为无状æ€ã€‚ + +
è¿è¡Œ Rancher Management Server çš„ K3s Kubernetes 集群的架构
+ +![è¿è¡Œ Rancher Management Server çš„ K3s Kubernetes 集群的架构](/img/k3s-server-storage.svg) + +### RKE Kubernetes 集群安装 + +在 RKE 安装中，集群数æ®åœ¨é›†ç¾¤ä¸­çš„三个 etcd 节点上å¤åˆ¶ï¼Œä»¥åœ¨æŸä¸ªèŠ‚点å‘生故障时æ供冗余和进行数æ®å¤åˆ¶ã€‚ + +
è¿è¡Œ Rancher Management Server çš„ RKE Kubernetes 集群的架构
+ +![è¿è¡Œ Rancher Management Server çš„ RKE Kubernetes 集群的架构](/img/rke-server-storage.svg) + +## Kubernetes 安装的负载å‡è¡¡å™¨æŽ¨èé…ç½® + +我们建议你为负载å‡è¡¡å™¨å’Œ Ingress Controller 使用以下é…置： + +* 把 Rancher çš„ DNS 解æžåˆ°å››å±‚è´Ÿè½½å‡è¡¡å™¨ä¸Šã€‚ +* è´Ÿè½½å‡è¡¡å™¨åº”该把 TCP/80 端å£å’Œ TCP/443 端å£çš„æµé‡è½¬å‘到 Kubernetes 集群的全部 3 个节点上。 +* Ingress Controller 会把 HTTP é‡å®šå‘到 HTTPS，在 TCP/443 端å£ç»ˆç»“ SSL/TLS。 +* Ingress Controller 会把æµé‡è½¬å‘到 Rancher deployment çš„ Pod 上的 TCP/80 端å£ã€‚ + +
在 Kubernetes 集群中安装 Rancher，并使用四层负载å‡è¡¡å™¨ï¼ŒSSL 终止在 Ingress Controller 中
+ +![Rancher HA](/img/ha/rancher2ha.svg) + +## Kubernetes 安装环境 + +我们强烈建议你把 Rancher 安装到托管在云æ供商（如 AWS EC2 å’Œ Google Compute Engine（GCE）等）上的 Kubernetes 集群上。 + +为了达到最佳性能和安全性，我们建议你为 Rancher Management Server 创建一个专用的 Kubernetes 集群。ä¸å»ºè®®åœ¨æ­¤é›†ç¾¤ä¸Šè¿è¡Œç”¨æˆ·å·¥ä½œè´Ÿè½½ã€‚部署 Rancher åŽï¼Œä½ å¯ä»¥[创建或导入集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)æ¥è¿è¡Œä½ çš„工作负载。 + +## Kubernetes 安装的推è节点角色 + +如果 Rancher 安装在 K3s Kubernetes 或 RKE Kubernetes 集群上，以下建议适用。 + +### K3s 集群角色 + +在 K3s 集群中有两ç§ç±»åž‹çš„节点，分别是 Server 节点和 Agent 节点。你å¯ä»¥æŠŠå·¥ä½œè´Ÿè½½è°ƒåº¦åˆ° Server 节点和 Agent 节点上。Server 节点è¿è¡Œ Kubernetes master。 + +对于è¿è¡Œ Rancher Management Server 的集群，我们建议使用两个 server 节点。ä¸éœ€è¦ Agent 节点。 + +### RKE 集群角色 + +如果 Rancher 安装在 RKE Kubernetes 集群上，该集群应具有三个节点，并且æ¯ä¸ªèŠ‚点都应具有所有三个 Kubernetes 角色，分别是 etcd，controlplane å’Œ worker。 + +### Rancher Server 和下游 Kubernetes 集群的 RKE 集群架构对比 + +我们对 Rancher Server 集群上 RKE 节点角色建议，与对è¿è¡Œä½ çš„应用和æœåŠ¡çš„下游集群的建议相å。 + +在é…置下游 Kubernetes 集群时，Rancher 使用 RKE 作为创建下游 Kubernetes 集群的工具。注æ„：Rancher 将在未æ¥çš„版本中添加é…置下游 K3s 集群的功能。 + +我们建议下游 Kubernetes 集群中的æ¯ä¸ªèŠ‚点都åªåˆ†é…一个角色，以确ä¿ç¨³å®šæ€§å’Œå¯æ‰©å±•æ€§ã€‚ + +![Rancher Server 集群中和下游集群中节点的 Kubernetes 角色对比](/img/rancher-architecture-node-roles.svg) + +RKE æ¯ä¸ªè§’色至少需è¦ä¸€ä¸ªèŠ‚点，但并ä¸å¼ºåˆ¶æ¯ä¸ªèŠ‚点åªèƒ½æœ‰ä¸€ä¸ªè§’色。但是，我们建议为è¿è¡Œåº”用的集群中的æ¯ä¸ªèŠ‚点，使用å•ç‹¬çš„角色，以ä¿è¯åœ¨æœåŠ¡æ‹“展时，worker 节点上的工作负载ä¸å½±å“ Kubernetes master 或集群的数æ®ã€‚ + +以下是我们对下游集群的最低é…置建议： + +- **三个仅使用 etcd 角色的节点** ，以在三个节点中其中一个å‘生故障时，ä»èƒ½ä¿éšœé›†ç¾¤çš„高å¯ç”¨æ€§ã€‚ +- **两个åªæœ‰ controlplane 角色的节点** ，以ä¿è¯ master 组件的高å¯ç”¨æ€§ã€‚ +- **一个或多个åªæœ‰ worker 角色的节点**，用于è¿è¡Œ Kubernetes 节点组件，以åŠä½ éƒ¨ç½²çš„æœåŠ¡æˆ–应用的工作负载。 + +在设置 Rancher Server 时，在三个节点上使用全部这三个角色也是安全的，因为： + +* 它å…许一个 `etcd` 节点故障。 +* 它通过多个 `controlplane` 节点æ¥ç»´æŠ¤ master 组件的多个实例。 +* 此集群上没有创建除 Rancher 之外的其他工作负载。 + +由于 Rancher Server 集群中没有部署其他工作负载，因此在大多数情况下，这个集群都ä¸éœ€è¦ä½¿ç”¨æˆ‘们出于å¯æ‰©å±•æ€§å’Œå¯ç”¨æ€§çš„考虑，而为下游集群推è的架构。 + +有关下游集群的最佳实践，请查看[生产环境清å•](../../pages-for-subheaders/checklist-for-production-ready-clusters.md)或[最佳实践](../../pages-for-subheaders/best-practices.md)。 + +## 授æƒé›†ç¾¤ç«¯ç‚¹æž¶æž„ + +如果你使用[授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)，我们建议你创建一个指å‘è´Ÿè½½å‡è¡¡å™¨çš„ FQDN，这个负载å‡è¡¡å™¨æŠŠæµé‡è½¬åˆ°æ‰€æœ‰è§’色为 `controlplane` 的节点。 + +如果你在负载å‡è¡¡å™¨ä¸Šä½¿ç”¨äº†ç§æœ‰ CA ç­¾å‘çš„è¯ä¹¦ï¼Œä½ éœ€è¦æä¾› CA è¯ä¹¦ï¼Œè¿™ä¸ªè¯ä¹¦ä¼šåŒ…å«åœ¨ç”Ÿæˆçš„ kubeconfig 文件中，以校验è¯ä¹¦é“¾ã€‚详情请å‚è§ [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)å’Œ [API 密钥](../user-settings/api-keys.md#创建-api-密钥)的相关文档。 + +注册的 RKE2 å’Œ K3s 集群å¯ä»¥ä½¿ç”¨ ACE 支æŒã€‚点击[这里](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#对-rke2-å’Œ-k3s-集群的授æƒé›†ç¾¤ç«¯ç‚¹æ”¯æŒ)äº†è§£åœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­å¼€å¯ ACE 的步骤。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md new file mode 100644 index 00000000000..56800763102 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -0,0 +1,132 @@ +--- +title: 与下游集群通信 +--- + +æœ¬èŠ‚ä»‹ç» Rancher 如何é…置和管ç†è¿è¡Œåº”用和æœåŠ¡çš„下游集群。 + +下图显示了 Cluster Controllerã€Cluster Agent å’Œ Node Agent 让 Rancher 控制下游集群的。 + +
与下游集群通信
+ +![Rancher 组件](/img/rancher-architecture-cluster-controller.svg) + +以下æ述对应于上图中的数字： + +1. [认è¯ä»£ç†](#1-认è¯ä»£ç†) +2. [Cluster Controller å’Œ Cluster Agent](#2-cluster-controller-å’Œ-cluster-agent) +3. [Node Agents](#3-node-agents) +4. [授æƒé›†ç¾¤ç«¯ç‚¹](#4-授æƒé›†ç¾¤ç«¯ç‚¹) + +### 1. 认è¯ä»£ç† + +在此图中，å为 Bob 的用户希望查看在å为 User Cluster 1 的下游集群上è¿è¡Œçš„所有 Pod。在 Rancher 中，他å¯ä»¥è¿è¡Œ `kubectl` 命令æ¥æŸ¥çœ‹ +Pod。Bob 通过 Rancher 的认è¯ä»£ç†è¿›è¡Œèº«ä»½éªŒè¯ã€‚ + +认è¯ä»£ç†å°†æ‰€æœ‰ Kubernetes API 调用转å‘到下游集群。它集æˆäº†æœ¬åœ°èº«ä»½éªŒè¯ã€Active Directory å’Œ GitHub 等身份验è¯æ–¹å¼ã€‚在æ¯ä¸ª Kubernetes API 调用请求时，认è¯ä»£ç†ä¼šéªŒè¯è¯·æ±‚方的身份，并在转å‘ç»™ Kubernetes master 节点之å‰ï¼Œè®¾ç½®æ­£ç¡®çš„ Kubernetes 消æ¯å¤´ã€‚ + +Rancher 使用 [ServiceAccount](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 与 Kubernetes 集群通信，该 ServiceAccount 为在 Pod 中è¿è¡Œçš„进程æ供身份。 + +默认情况下，Rancher 生æˆä¸€ä¸ª [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)，文件包å«å‡­è¯ä¿¡æ¯ï¼Œç”¨äºŽä¸º Rancher Server 连接下游集群的 Kubernetes API Server 的代ç†ã€‚kubeconfig 文件 (`kube_config_rancher-cluster.yml`) 包å«å¯¹é›†ç¾¤çš„完全访问æƒé™ã€‚ + +### 2. Cluster Controller å’Œ Cluster Agent + +æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤éƒ½æœ‰ä¸€ä¸ª Cluster Agent，用于打开与 Rancher Server 中对应的 Cluster Controller 之间的通é“。 + +æ¯ä¸ªä¸‹æ¸¸é›†ç¾¤æœ‰ä¸€ä¸ª Cluster Controller 和一个 Cluster Agent。æ¯ä¸ª Cluster Controller 都能： + +- 检测下游集群中的资æºå˜åŒ– +- 将下游集群的当å‰çŠ¶æ€å˜æ›´åˆ°ç›®æ ‡çŠ¶æ€ +- é…置集群和项目的访问控制策略 +- 通过调用所需的 Docker Machine 驱动和 Kubernetes 引擎（例如 RKE å’Œ GKE）æ¥é…置集群 + +默认情况下，Cluster Controller 连接到 Cluster Agent，Rancher æ‰èƒ½ä¸Žä¸‹æ¸¸é›†ç¾¤é€šä¿¡ã€‚如果 Cluster Agent ä¸å¯ç”¨ï¼ŒCluster Controller å¯ä»¥è¿žæŽ¥åˆ° [Node Agent](#3-node-agents)。 + +Cluster Agent，也å«åš `cattle-cluster-agent`，是è¿è¡Œåœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­çš„组件。它具有以下功能： + +- 连接 Rancher å¯åŠ¨çš„ Kubernetes 集群中的 Kubernetes API。 +- 管ç†é›†ç¾¤å†…的工作负载，pod 创建和部署。 +- æ ¹æ®æ¯ä¸ªé›†ç¾¤çš„全局策略，应用定义的角色和绑定。 +- 通过与 Cluster Controller 之间的通é“，实现集群和 Rancher Server 之间的通信，包括事件，统计数æ®ï¼ŒèŠ‚点信æ¯å’Œå¥åº·çŠ¶å†µã€‚ + +### 3. Node Agents + +如果 Cluster Agent（也称为 `cattle-cluster-agent`）ä¸å¯ç”¨ï¼Œå…¶ä¸­ä¸€ä¸ª Node Agent 会创建一个连接到 Cluster Controller 的通é“与 Rancher 通信。 + +`cattle-node-agent` 使用 [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) 资æºè¿›è¡Œéƒ¨ç½²ï¼Œä»¥ç¡®ä¿å®ƒèƒ½åœ¨ Rancher å¯åŠ¨çš„ Kubernetes 集群中的æ¯ä¸ªèŠ‚点上è¿è¡Œï¼Œç”¨äºŽåœ¨æ‰§è¡Œé›†ç¾¤æ“作时与节点交互。集群æ“作的包括å‡çº§ Kubernetes 版本，创建或æ¢å¤ etcd 快照等。 + +### 4. 授æƒé›†ç¾¤ç«¯ç‚¹ + +授æƒé›†ç¾¤ç«¯ç‚¹ï¼ˆACE）å¯è¿žæŽ¥åˆ°ä¸‹æ¸¸é›†ç¾¤çš„ Kubernetes API Server，而ä¸ç”¨é€šè¿‡ Rancher 认è¯ä»£ç†è°ƒåº¦è¯·æ±‚。 + +> 授æƒé›†ç¾¤ç«¯ç‚¹ä»…适用于 Rancher å¯åŠ¨çš„ Kubernetes 集群，å³åªé€‚用于 Rancher [使用 RKE](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) æ¥é…置的集群。它ä¸é€‚用于导入的集群，也ä¸é€‚用于托管在 Kubernetes æ供商中的集群（例如 Amazon çš„ EKS）。 + +授æƒé›†ç¾¤ç«¯ç‚¹çš„主è¦ç”¨é€”： + +- 在 Rancher ä¸å¯ç”¨æ—¶è®¿é—®ä¸‹æ¸¸é›†ç¾¤ +- 在 Rancher Server 和与下游集群之间相è·ç”šè¿œæ—¶é™ä½Žå»¶è¿Ÿ + +`kube-api-auth` å¾®æœåŠ¡ä¸ºæŽˆæƒé›†ç¾¤ç«¯ç‚¹æ供用户验è¯åŠŸèƒ½ã€‚当使用 `kubectl`访问下游集群时，集群的 Kubernetes API Server 使用 `kube-api-auth` æœåŠ¡ä½œä¸º webhook 对用户进行身份验è¯ã€‚ + +与授æƒé›†ç¾¤ç«¯ç‚¹ä¸€æ ·ï¼Œ`kube-api-auth` 的身份验è¯åŠŸèƒ½ä¹Ÿä»…适用于 Rancher å¯åŠ¨çš„ Kubernetes 集群。 + +> **示例场景：** å‡è®¾ Rancher Server ä½äºŽç¾Žå›½ï¼ŒUser Cluster 1 与用户 Alice å‡ä½äºŽæ¾³å¤§åˆ©äºšã€‚Alice å¯ä»¥ä½¿ç”¨ Rancher UI æ“作 User Cluster 1 中的资æºï¼Œä½†å¥¹çš„请求必须从澳大利亚å‘é€åˆ°ç¾Žå›½çš„ Rancher Server，然åŽé€šè¿‡ä»£ç†è¿”回澳大利亚，å³ä¸‹æ¸¸é›†ç¾¤æ‰€åœ¨çš„ä½ç½®ã€‚地ç†è·ç¦»å¯èƒ½å¯¼è‡´æ˜Žæ˜¾å»¶è¿Ÿï¼Œå› æ­¤ï¼ŒAlice å¯ä»¥ä½¿ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹æ¥é™ä½Žå»¶è¿Ÿã€‚ + +为下游集群å¯ç”¨æŽˆæƒé›†ç¾¤ç«¯ç‚¹åŽï¼ŒRancher 会在 kubeconfig 文件中é¢å¤–生æˆä¸€æ®µ Kubernetes 上下文，用于直连到集群。该文件具有 `kubectl` å’Œ `helm`的凭è¯ã€‚ + +如果 Rancher 出现问题，你需è¦ä½¿ç”¨æ­¤ kubeconfig 文件中定义的上下文æ¥è®¿é—®é›†ç¾¤ã€‚因此，我们建议你导出 kubeconfig 文件，以便在 Rancher 出现问题时，ä»èƒ½ä½¿ç”¨æ–‡ä»¶ä¸­çš„凭è¯è®¿é—®é›†ç¾¤ã€‚详情请å‚è§ä½¿ç”¨ [kubectl å’Œ kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)访问集群的章节。 + +## é‡è¦æ–‡ä»¶ + +维护ã€æŽ’除问题和å‡çº§é›†ç¾¤éœ€è¦ç”¨åˆ°ä»¥ä¸‹æ–‡ä»¶ï¼Œè¯·å¦¥å–„ä¿ç®¡è¿™äº›æ–‡ä»¶ï¼š + +- `rancher-cluster.yml`：RKE 集群é…置文件。 +- `kube_config_rancher-cluster.yml`：集群的 Kubeconfig 文件，包å«å®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚如果 Rancher 出现问题时，你å¯ä»¥ä½¿ç”¨æ­¤æ–‡ä»¶è®¤è¯ç”± Rancher å¯åŠ¨çš„ Kubernetes 集群。 +- `rancher-cluster.rkestate`：Kubernetes 集群状æ€æ–‡ä»¶ï¼Œæ–‡ä»¶åŒ…å«ç”¨äºŽå®Œå…¨è®¿é—®é›†ç¾¤çš„凭è¯ã€‚注æ„：仅在使用 RKE v0.2.0 或更高版本时，æ‰ä¼šåˆ›å»ºæ­¤è¯¥æ–‡ä»¶ã€‚ + +> **注æ„**：åŽä¸¤ä¸ªæ–‡ä»¶å中的 `rancher-cluster` 部分å–决于你命å RKE 集群é…置文件的方å¼ã€‚ + +有关在没有 Rancher 认è¯ä»£ç†å’Œå…¶ä»–é…置选项的情况下连接到集群的更多信æ¯ï¼Œè¯·å‚è§ [kubeconfig 文件](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md)。 + +## é…ç½® Kubernetes 集群的工具 + +Rancher 使用什么工具é…置下游集群，å–决于集群的类型。 + +### Rancher 为托管在云æ供商中的节点å¯åŠ¨ Kubernetes + +Rancher å¯ä»¥åŠ¨æ€å¯åŠ¨äº‘上（如 Amazon EC2ã€DigitalOceanã€Azure 或 vSphere 等）的节点，然åŽåœ¨èŠ‚点上安装 Kubernetes。 + +Rancher 使用 [RKE](https://github.com/rancher/rke) å’Œ [docker-machine](https://github.com/rancher/machine) æ¥é…置这类型的集群。 + +### Rancher 为自定义节点å¯åŠ¨ Kubernetes + +在é…置此类集群时，Rancher 会在现有节点上安装 Kubernetes，从而创建自定义集群。 + +Rancher 使用 [RKE](https://github.com/rancher/rke) æ¥å¯åŠ¨æ­¤ç±»é›†ç¾¤ã€‚ + +### 托管的 Kubernetes æ供商 + +é…置此类集群时，Kubernetes 由云æ供商安装，如 GKEã€ECS 或 AKS 等。 + +Rancher 使用 [kontainer-engine](https://github.com/rancher/kontainer-engine) é…置此类型的集群。 + +### 导入的 Kubernetes 集群 + +è¿™ç§æƒ…况下，Rancher 需è¦è¿žæŽ¥åˆ°ä¸€ä¸ªè®¾ç½®å¥½çš„ Kubernetes 集群。因此，Rancher ä¸æä¾› Kubernetes，åªè®¾ç½® Rancher Agent 实现与集群通信。 + +## Rancher Server 组件和æºç  + +下图展示了 Rancher Server 的组件： + +![Rancher 组件](/img/rancher-architecture-rancher-components.svg) + +Rancher çš„ GitHub 代ç ä»“库如下： + +- [Rancher Server 主代ç åº“](https://github.com/rancher/rancher) +- [Rancher UI](https://github.com/rancher/ui) +- [Rancher API UI](https://github.com/rancher/api-ui) +- [Norman](https://github.com/rancher/norman)（Rancher çš„ API 框架） +- [类型](https://github.com/rancher/types) +- [Rancher CLI](https://github.com/rancher/cli) +- [应用商店](https://github.com/rancher/helm) + +以上仅列出部分 Rancher 最é‡è¦çš„仓库。详情请å‚è§[å‚与 Rancher å¼€æºè´¡çŒ®](../../contribute-to-rancher.md#rancher-仓库)ã€‚å¦‚éœ€èŽ·å– Rancher 使用的所有库和项目，请å‚è§ `rancher/rancher` 仓库中的 [`go.mod` 文件](https://github.com/rancher/rancher/blob/master/go.mod)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/rancher-server-and-components.md new file mode 100644 index 00000000000..b715f4be1e4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -0,0 +1,35 @@ +--- +title: Rancher Server å’Œ Components +--- + +大多数 Rancher 2.x 软件å‡è¿è¡Œåœ¨ Rancher Server 上。Rancher Server 包括用于管ç†æ•´ä¸ª Rancher 部署的所有软件组件。 + +下图展示了 Rancher 2.x 的上层架构。下图中，Rancher Server 管ç†ä¸¤ä¸ªä¸‹æ¸¸ Kubernetes 集群，其中一个由 RKE 创建，å¦ä¸€ä¸ªç”± Amazon EKS 创建。 + +为了达到最佳性能和安全性，我们建议你为 Rancher Management Server 创建一个专用的 Kubernetes 集群。ä¸å»ºè®®åœ¨æ­¤é›†ç¾¤ä¸Šè¿è¡Œç”¨æˆ·å·¥ä½œè´Ÿè½½ã€‚部署 Rancher åŽï¼Œä½ å¯ä»¥[创建或导入集群](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md)æ¥è¿è¡Œä½ çš„工作负载。 + +下图介ç»äº†ç”¨æˆ·å¦‚何通过 Rancher 的认è¯ä»£ç†ç®¡ç† [Rancher å¯åŠ¨çš„ Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群和[托管的 Kubernetes](../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md) 集群： + +
通过 Rancher 的认è¯ä»£ç†ç®¡ç† Kubernetes 集群
+ +![架构](/img/rancher-architecture-rancher-api-server.svg) + +ä½ å¯ä»¥æŠŠ Rancher 安装到å•ä¸ªèŠ‚点或高å¯ç”¨ Kubernetes 集群上。 + +在生产环境中，建议安装到高å¯ç”¨ Kubernetes 集群。 + +Rancher çš„ Docker 安装仅推è用于开å‘和测试环境中。Rancher 版本决定了能å¦å°† Rancher è¿ç§»åˆ°é«˜å¯ç”¨é›†ç¾¤ã€‚ + +è¦æŸ¥çœ‹å·²éƒ¨ç½²çš„资æºï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +```bash +kubectl get all -n +``` +如果你具有管ç†å‘˜æƒé™ï¼Œä½ è¿˜å¯ä»¥åœ¨ Rancher UI 中看到列出的资æºï¼š + +1. å•å‡» **☰** 并选择一个集群。 +1. 从侧é¢å¯¼èˆªèœå•ä¸­é€‰æ‹©**更多资æº**，从而按类型查看已部署的资æºã€‚ +1. 从侧é¢å¯¼èˆªèœå•ä¸­é€‰æ‹©**集群** > **项目/命å空间**，然åŽé€‰æ‹©ä¸€ä¸ªå‘½å空间，从而按命å空间查看已部署的资æºã€‚ + Rancher backup operator å¯å°† Rancher 从å•ä¸ª Docker 容器è¿ç§»åˆ°é«˜å¯ç”¨ Kubernetes 集群上。有关详细信æ¯ï¼Œè¯·å‚阅[å°† Rancher è¿ç§»åˆ°æ–°é›†ç¾¤](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md)。 + +ä¸ç®¡ Rancher Server 是如何安装的，它都应该è¿è¡Œåœ¨ä¸Žå…¶ç®¡ç†çš„下游集群ä¸åŒèŠ‚点上。如果 Rancher 安装在高å¯ç”¨çš„ Kubernetes 集群上，它需è¦è¿è¡Œåœ¨ä¸Žå…¶ç®¡ç†çš„集群ä¸åŒçš„集群上。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-project-tools.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-project-tools.md new file mode 100644 index 00000000000..ad05d4166e0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-project-tools.md @@ -0,0 +1,32 @@ +--- +title: 项目工具：Logging，Monitoring å’Œå¯è§†åŒ– +--- + +Rancher åŒ…å« Kubernetes 中未包å«çš„å„ç§å·¥å…·æ¥å助你进行 DevOps æ“作。Rancher å¯ä»¥ä¸Žå¤–部æœåŠ¡é›†æˆï¼Œè®©ä½ çš„集群更高效地è¿è¡Œã€‚ + + +## Notifiers 和告警 + +通知器和告警是两个ååŒå·¥ä½œçš„功能，它们å¯ä»¥å°† Rancher 系统中的事件告知你。在å¯ç”¨å®ƒä»¬ä¹‹å‰ï¼Œä½ å¿…须先安装监控应用。 + +通知器是通知你告警事件的æœåŠ¡ã€‚ä½ å¯ä»¥é€šè¿‡é…置通知器æ¥å‘最适åˆé‡‡å–纠正措施的员工å‘é€å‘Šè­¦é€šçŸ¥ã€‚支æŒä½¿ç”¨ Slackã€ç”µå­é‚®ä»¶ã€PagerDutyã€å¾®ä¿¡å’Œ webhook å‘é€é€šçŸ¥ã€‚ + +告警是触å‘这些通知的规则。在接收告警之å‰ï¼Œä½ å¿…须在 Rancher 中é…置一个或多个通知器。你å¯ä»¥åœ¨é›†ç¾¤æˆ–项目级别设置告警范围。 + +## Logging + +Logging 支æŒï¼š + +- 获å–和分æžé›†ç¾¤çš„çŠ¶æ€ +- 在你的环境中å‘现趋势 +- 将日志ä¿å­˜åˆ°é›†ç¾¤å¤–部的安全ä½ç½® +- éšæ—¶äº†è§£å®¹å™¨å´©æºƒã€pod 驱é€æˆ–节点死亡等事件 +- æ›´è½»æ¾åœ°è°ƒè¯•å’Œè§£å†³é—®é¢˜ + +Rancher å¯ä»¥ä¸Ž Elasticsearchã€splunkã€kafkaã€syslog å’Œ fluentd 集æˆã€‚ + +有关详细信æ¯ï¼Œè¯·å‚阅 [Logging](../pages-for-subheaders/logging.md)。 + +## Monitoring + +ä½ å¯ä»¥ä½¿ç”¨ Rancher，通过业界领先并开æºçš„ [Prometheus](https://prometheus.io/) æ¥ç›‘控集群节点ã€Kubernetes 组件和软件部署的状æ€å’Œè¿›ç¨‹ã€‚有关详细信æ¯ï¼Œè¯·å‚阅 [Monitoring](../pages-for-subheaders/monitoring-and-alerting.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..ddf9809440a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3148 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 518 1 0 Feb26 ? 00:03:06 /usr/bin/containerd root 794 1 0 Feb26 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 94093 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7927fe0998264b8fa49789e8e2793a21ea8f22001cb580de46e1fcc01b1d1455 -address /run/k3s/containerd/containerd.sock root 94107 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ccec5bdaf56b4aad4018e8fefabfa103122905ecf4f908dbddaf9fbb985bc5c -address /run/k3s/containerd/containerd.sock root 94188 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d0f840cefd5b5ba804b2795420963e2c43a0c8b7b6b2c796e859c57ba0d38b3f -address /run/k3s/containerd/containerd.sock root 95349 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3849ee071d3f5a8e8a63ee1c4d590f88e0d21bc5fc25e20bbecb1e104b2835d0 -address /run/k3s/containerd/containerd.sock root 95421 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id a608e22cc1520272c91a2aca6a029e662a5d13defc7cf67de6a59594b859c550 -address /run/k3s/containerd/containerd.sock root 96569 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id f46f18b427a026156a41a54daad5c55a7d04bee1d112ae32e9c3d07c8cfc5a4e -address /run/k3s/containerd/containerd.sock root 97820 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id aa5731bfdbd6ff9c87043ccd4e30adc1c384eb8653b97169629598852931375d -address /run/k3s/containerd/containerd.sock root 97868 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 99f90b82253554453d0d1e57a23e770f000cf8972ca751387fd5682e9e1607f2 -address /run/k3s/containerd/containerd.sock root 100124 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 632f8fcf8174c12048354376e1f22e87ac95baa26a46e77f4cb8c1549f00fc4f -address /run/k3s/containerd/containerd.sock root 101049 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 05cc13a10b18601034393b12dcb0b140799cef2f3c7f712de671054cb65a4cff -address /run/k3s/containerd/containerd.sock root 105484 105441 2 21:41 ? 00:00:10 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 107618 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d7baeb7e8d463b1afd831ca4367d4c552c8e87afc29f7cae7cb5bf987c2e4b1f -address /run/k3s/containerd/containerd.sock root 107751 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 8fd7e8824474bb98e13d39dca0e130444b7b6fb917b7dc5e662c5a5f669fe584 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'644' is equal to '644' OR '640' is present OR '600' is present OR '444' is present OR '440' is present OR '400' is present OR '000' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md similarity index 99% rename from docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md rename to i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md index 3e7cd1e1e06..8944de04c27 100644 --- a/docs/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md @@ -2,10 +2,6 @@ title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.24 --- - - - - This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md new file mode 100644 index 00000000000..1f5b7dfd37b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md @@ -0,0 +1,3148 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.25 +--- + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.25 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 519 1 0 22:09 ? 00:00:00 /usr/bin/containerd root 801 1 0 22:09 ? 00:00:00 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 3864 1 0 22:31 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id d00174abbc275f6bb85c7f0be1d3154b9c91982a10b9dba6b5cb280f4d4c531d -address /run/k3s/containerd/containerd.sock root 4105 1 0 22:31 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7c2b546b4d2380bcb51278661f34cff94fad2ba06978e13f8f1b92dafcc89d43 -address /run/k3s/containerd/containerd.sock root 4206 1 0 22:31 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 68d8a55ff4663985be004608dbf78b0362f5522e18490c81d4c8dc9963de1556 -address /run/k3s/containerd/containerd.sock root 5374 1 0 22:31 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id ca0ae9e0b37dfd7b1ce05f72e1bc5a1be8f5cb08f2b4543081536de3bdbc925d -address /run/k3s/containerd/containerd.sock root 5443 1 0 22:31 ? 00:00:01 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3ea3c1cdbbd5adb8efd5c67a46aadd0fca9918dc0ad1f7cafe38b83171e3dc1b -address /run/k3s/containerd/containerd.sock root 7130 1 0 22:32 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4d838297d35a31003106ac5989c3547433985bb2964b47baad12cee6e375645e -address /run/k3s/containerd/containerd.sock root 7639 1 0 22:32 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 341cb9bcd8486aa2f1acb8e1ae51baebd630ac6ed266643266c34d677f61c7d0 -address /run/k3s/containerd/containerd.sock root 10308 1 0 23:17 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id c534fbee8e0d06fd9b29bf8fc70a138975c6b18db25f1faf2615677dfdb4199e -address /run/k3s/containerd/containerd.sock root 11370 1 0 23:18 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ff4b8776dac7a35b83616d341dbe4d5a689ac7fb9b8eee8db5978e3968380ea -address /run/k3s/containerd/containerd.sock root 13736 13723 2 23:21 ? 00:00:10 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 16022 1 0 23:29 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9027256349086e458119478e5e00384b1b76fbf5e6dbee23699f596a88d9f2bc -address /run/k3s/containerd/containerd.sock root 16159 1 0 23:29 ? 00:00:00 /var/lib/rancher/k3s/data/630c40ff866a3db218a952ebd4fd2a5cfe1543a1a467e738cb46a2ad4012d6f1/bin/containerd-shim-runc-v2 -namespace k8s.io -id 929bf369fc5881654f4c1925624151ddb7cea51073267b8d213d966ba45406f3 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'644' is equal to '644' OR '640' is present OR '600' is present OR '444' is present OR '440' is present OR '400' is present OR '000' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:42 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:42Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:44 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:44Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-31-124 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c42d922-ed1e-4e15-8414-d399d179d897 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 23:21:44 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:44Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-31-124 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c42d922-ed1e-4e15-8414-d399d179d897 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Feb 27 23:21:44 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:44Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-31-124 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c42d922-ed1e-4e15-8414-d399d179d897 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 23:21:44 ip-172-31-31-124 k3s[13723]: time="2023-02-27T23:21:44Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-31-124 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c42d922-ed1e-4e15-8414-d399d179d897 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..34bb63f89f0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3083 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/!(*key).pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 5 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 2 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): replicationcontrollers is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicationcontrollers" in API group "" in the namespace "default" Error from server (Forbidden): services is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "services" in API group "" in the namespace "default" Error from server (Forbidden): daemonsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "daemonsets" in API group "apps" in the namespace "default" Error from server (Forbidden): deployments.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "deployments" in API group "apps" in the namespace "default" Error from server (Forbidden): replicasets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicasets" in API group "apps" in the namespace "default" Error from server (Forbidden): statefulsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "statefulsets" in API group "apps" in the namespace "default" Error from server (Forbidden): horizontalpodautoscalers.autoscaling is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "horizontalpodautoscalers" in API group "autoscaling" in the namespace "default" Error from server (Forbidden): cronjobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "cronjobs" in API group "batch" in the namespace "default" Error from server (Forbidden): jobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "jobs" in API group "batch" in the namespace "default" true +``` diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md similarity index 99% rename from docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md rename to i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md index 3d12d66fb51..25108b848b5 100644 --- a/docs/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md @@ -2,10 +2,6 @@ title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.24 --- - - - - This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md new file mode 100644 index 00000000000..0cec041061f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md @@ -0,0 +1,3085 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.25 +--- + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.25 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=600 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/!(*key).pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 5506 5484 2 22:01 ? 00:00:05 kube-controller-manager --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --v=2 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --allow-untagged-cloud=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --cluster-cidr=10.42.0.0/16 --node-monitor-grace-period=40s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --service-cluster-ip-range=10.43.0.0/16 --profiling=false --configure-cloud-routes=false --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5506 5484 2 22:01 ? 00:00:05 kube-controller-manager --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --v=2 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --allow-untagged-cloud=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --cluster-cidr=10.42.0.0/16 --node-monitor-grace-period=40s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --service-cluster-ip-range=10.43.0.0/16 --profiling=false --configure-cloud-routes=false --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 5506 5484 2 22:01 ? 00:00:05 kube-controller-manager --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --v=2 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --allow-untagged-cloud=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --cluster-cidr=10.42.0.0/16 --node-monitor-grace-period=40s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --service-cluster-ip-range=10.43.0.0/16 --profiling=false --configure-cloud-routes=false --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5506 5484 2 22:01 ? 00:00:05 kube-controller-manager --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --v=2 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --allow-untagged-cloud=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --cluster-cidr=10.42.0.0/16 --node-monitor-grace-period=40s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --service-cluster-ip-range=10.43.0.0/16 --profiling=false --configure-cloud-routes=false --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 5506 5484 2 22:01 ? 00:00:05 kube-controller-manager --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --v=2 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --allow-untagged-cloud=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --cluster-cidr=10.42.0.0/16 --node-monitor-grace-period=40s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --service-cluster-ip-range=10.43.0.0/16 --profiling=false --configure-cloud-routes=false --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5506 5484 2 22:01 ? 00:00:05 kube-controller-manager --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --v=2 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --allow-untagged-cloud=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --cluster-cidr=10.42.0.0/16 --node-monitor-grace-period=40s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --service-cluster-ip-range=10.43.0.0/16 --profiling=false --configure-cloud-routes=false --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5671 5649 0 22:01 ? 00:00:01 kube-scheduler --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5671 5649 0 22:01 ? 00:00:01 kube-scheduler --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 5188 5167 3 22:01 ? 00:00:08 /usr/local/bin/etcd --client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --initial-advertise-peer-urls=https://172.31.31.51:2380 --listen-peer-urls=https://172.31.31.51:2380 --initial-cluster=etcd-ip-172-31-31-51=https://172.31.31.51:2380 --initial-cluster-state=new --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --peer-client-cert-auth=true --listen-client-urls=https://172.31.31.51:2379 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --advertise-client-urls=https://172.31.31.51:2379 --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-31-51 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --election-timeout=5000 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --heartbeat-interval=500 root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem root 19036 18926 5 22:05 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5188 5167 3 22:01 ? 00:00:08 /usr/local/bin/etcd --client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --initial-advertise-peer-urls=https://172.31.31.51:2380 --listen-peer-urls=https://172.31.31.51:2380 --initial-cluster=etcd-ip-172-31-31-51=https://172.31.31.51:2380 --initial-cluster-state=new --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --peer-client-cert-auth=true --listen-client-urls=https://172.31.31.51:2379 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --advertise-client-urls=https://172.31.31.51:2379 --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-31-51 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --election-timeout=5000 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --heartbeat-interval=500 root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem root 19036 18926 4 22:05 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-31-51 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem ETCDCTL_ENDPOINTS=https://172.31.31.51:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 5188 5167 3 22:01 ? 00:00:08 /usr/local/bin/etcd --client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --initial-advertise-peer-urls=https://172.31.31.51:2380 --listen-peer-urls=https://172.31.31.51:2380 --initial-cluster=etcd-ip-172-31-31-51=https://172.31.31.51:2380 --initial-cluster-state=new --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --peer-client-cert-auth=true --listen-client-urls=https://172.31.31.51:2379 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --advertise-client-urls=https://172.31.31.51:2379 --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-31-51 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --election-timeout=5000 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --heartbeat-interval=500 root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem root 19036 18926 2 22:05 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5188 5167 3 22:01 ? 00:00:08 /usr/local/bin/etcd --client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --initial-advertise-peer-urls=https://172.31.31.51:2380 --listen-peer-urls=https://172.31.31.51:2380 --initial-cluster=etcd-ip-172-31-31-51=https://172.31.31.51:2380 --initial-cluster-state=new --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --peer-client-cert-auth=true --listen-client-urls=https://172.31.31.51:2379 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --advertise-client-urls=https://172.31.31.51:2379 --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-31-51 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --election-timeout=5000 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --heartbeat-interval=500 root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem root 19036 18926 3 22:05 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-31-51 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem ETCDCTL_ENDPOINTS=https://172.31.31.51:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 5188 5167 3 22:01 ? 00:00:08 /usr/local/bin/etcd --client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --initial-advertise-peer-urls=https://172.31.31.51:2380 --listen-peer-urls=https://172.31.31.51:2380 --initial-cluster=etcd-ip-172-31-31-51=https://172.31.31.51:2380 --initial-cluster-state=new --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --peer-client-cert-auth=true --listen-client-urls=https://172.31.31.51:2379 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51.pem --advertise-client-urls=https://172.31.31.51:2379 --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-31-51 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-31-51-key.pem --election-timeout=5000 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --heartbeat-interval=500 root 5354 5332 14 22:01 ? 00:00:33 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem root 19036 18926 2 22:05 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 5354 5332 14 22:01 ? 00:00:34 kube-apiserver --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxsize=100 --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-group-headers=X-Remote-Group --storage-backend=etcd3 --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --authentication-token-webhook-cache-ttl=5s --etcd-prefix=/registry --service-node-port-range=30000-32767 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --profiling=false --audit-log-format=json --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --allow-privileged=true --requestheader-username-headers=X-Remote-User --anonymous-auth=false --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --api-audiences=unknown --etcd-servers=https://172.31.31.51:2379 --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --advertise-address=172.31.31.51 --audit-log-maxage=30 --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --requestheader-extra-headers-prefix=X-Remote-Extra- --bind-address=0.0.0.0 --service-account-lookup=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-allowed-names=kube-apiserver-proxy-client --service-account-issuer=rke --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --service-cluster-ip-range=10.43.0.0/16 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 6239 5834 2 22:02 ? 00:00:04 kubelet --authorization-mode=Webhook --v=2 --root-dir=/var/lib/kubelet --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51-key.pem --cgroups-per-qos=True --streaming-connection-idle-timeout=30m --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-31-51.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --address=0.0.0.0 --cluster-domain=cluster.local --fail-swap-on=false --make-iptables-util-chains=true --volume-plugin-dir=/var/lib/kubelet/volumeplugins --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --pod-infra-container-image=rancher/mirrored-pause:3.6 --node-ip=172.31.31.51 --resolv-conf=/etc/resolv.conf --event-qps=0 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --protect-kernel-defaults=true --cluster-dns=10.43.0.10 --container-runtime=remote --authentication-token-webhook=true --anonymous-auth=false --feature-gates=RotateKubeletServerCertificate=true --cloud-provider= --read-only-port=0 --hostname-override=ip-172-31-31-51 --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): replicationcontrollers is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicationcontrollers" in API group "" in the namespace "default" Error from server (Forbidden): services is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "services" in API group "" in the namespace "default" Error from server (Forbidden): daemonsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "daemonsets" in API group "apps" in the namespace "default" Error from server (Forbidden): deployments.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "deployments" in API group "apps" in the namespace "default" Error from server (Forbidden): replicasets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicasets" in API group "apps" in the namespace "default" Error from server (Forbidden): statefulsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "statefulsets" in API group "apps" in the namespace "default" Error from server (Forbidden): horizontalpodautoscalers.autoscaling is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "horizontalpodautoscalers" in API group "autoscaling" in the namespace "default" Error from server (Forbidden): cronjobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "cronjobs" in API group "batch" in the namespace "default" Error from server (Forbidden): jobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "jobs" in API group "batch" in the namespace "default" true +``` + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..83cac7d6e59 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3196 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"zF6yWmUNDvLZW5QnHG580wVBYOwq9S+VXJ3G4JjDaEk="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md similarity index 99% rename from docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md rename to i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md index 548d3e92d47..ffb7755d7ad 100644 --- a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md @@ -2,10 +2,6 @@ title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.24 --- - - - - This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md new file mode 100644 index 00000000000..4d524d3fab4 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.25.md @@ -0,0 +1,3196 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.25 +--- + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.25 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:05 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 15378 3910 99 23:32 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3980 3910 19 23:26 ? 00:01:06 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"TSpBkJhIU0sRx+84IZuBZ1qO+eaRdW31C7QCnF3+n8s="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4128 4029 2 23:27 ? 00:00:06 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4126 4014 0 23:27 ? 00:00:02 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4126 4014 0 23:27 ? 00:00:02 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-25-112 ETCD_UNSUPPORTED_ARCH= POD_HASH=ab0b8a2ee7711940d3d951edece075f3 FILE_HASH=068666c5f959fc1023cb1761daaaed212727c04120747d825c78fd7683122e6d NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3727 3667 3 23:26 ? 00:00:11 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-25-112 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=5c1cd514-db7b-4692-a1c4-cacb2656161f --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVXpPVE00TUI0WERUSXpNREl5TmpJek1qVXpPRm9YRFRNek1ESXlNekl6TWpVegpPRm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFNemt6T0RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJOUHllSlcwNE9lMUN6clo5cHplN09WWlliMlh4QmNvWUJ5Z3JRcUwKYUdpR29tN2xLNGs2ZW1uaUZQekpiOU9EQ0hDRmYyVUZaVXZDQTQxcUVmNVB3NlNqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUnhUdGJTbTVMTnFrditkYXQrCmczalBWWXJjcERBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlBN08wV1NNVHo4djgwM3BZK3hFeDR2SUZaVEMraVoKUHVDc082eFRuVVVtb2dJaEFPY1NqQVdkUVJ0UmJvaUhJbTd2RHZuM1czT1JmYlUwaXU2UUhNcXdsTGs4Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJRjc3RlZRT1l5RzR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU16a3pPREFlRncweU16QXlNall5TXpJMU16aGFGdzB5TkRBeQpNall5TXpJMU16aGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUXg5Z1lxUzVCbndwTDIKcUdOeStjNnR0Yk15VmE3ZTdzT0J1UWdaWHBHZ1hNcGxOZHFhVy9lZjNZTjQwTk5RUnl2SWdWeTMzU0ZHSFJ0VgpqaXBpOXRZbG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVyNlUvYVNMcm1ONkxiaDZHS1JJa3NoM1M1bEF3Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPS2paZmxRVU11RnZldlFkYzg3ckxPcnhoNUtyUGhlQUtkY0Y4YWdielFJQWlCcHBmUGNMMFRoZ1g5UAptcCtOZHJqa1hvQU5SRTlEWVdIRUlRbDdubytDdWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFV6T1RNNE1CNFhEVEl6TURJeU5qSXpNalV6T0ZvWERUTXpNREl5TXpJek1qVXoKT0Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTXprek9EQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQlhFZ0x2Z3JLV09KdkZtVnJhNEhyY05YdmNwN3JMN3VFNW1IcFpkCmJmT2xkRDRkVlJ4NjRxak9DeUNpc2Vsczk4WDJLSXlieGNSNkpnbFU2VXRoOU5xalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlN2cFQ5cEl1dVkzb3R1SG9ZcApFaVN5SGRMbVVEQUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpRUF3R2xqWXUxZkJpMHZROFczbWxueXVDNUJqMlBBCm14Sm1uS3BVSG8ydjJBZ0NJRndiblM3ajROUGtCT2hzRjJBeFhEZlZzdExoRWpqbmhPRHlQek1kT01STQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUpZblVvZ2U2bHVuNFN1WVVaN3VBQTVGYXV6blBaQzV2WHlpc1R2SVRjOXFvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFTWZZR0trdVFaOEtTOXFoamN2bk9yYld6TWxXdTN1N0RnYmtJR1Y2Um9GektaVFhhbWx2MwpuOTJEZU5EVFVFY3J5SUZjdDkwaFJoMGJWWTRxWXZiV0pRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** fail + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" --count=0 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** fail + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +error: the server doesn't have a resource type "psp" +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** true + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/kubernetes-security-best-practices.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/kubernetes-security-best-practices.md new file mode 100644 index 00000000000..af9746e899f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/kubernetes-security-best-practices.md @@ -0,0 +1,11 @@ +--- +title: Kubernetes 安全最佳实践 +--- + +### é™åˆ¶äº‘å…ƒæ•°æ® API 访问 + +AWSã€Azureã€DigitalOcean 或 GCP 等云æ供商通常会在本地å‘实例公开元数æ®æœåŠ¡ã€‚默认情况下，此端点å¯è¢«è¿è¡Œåœ¨äº‘实例上的 pod 访问，包括在托管的 Kubernetes（如 EKSã€AKSã€DigitalOcean Kubernetes 或 GKE）中的 pod，并且å¯ä»¥åŒ…å«è¯¥èŠ‚点的云凭è¯ã€é…置数æ®ï¼ˆå¦‚ kubelet 凭è¯ï¼‰ä»¥åŠå…¶ä»–æ•æ„Ÿæ•°æ®ã€‚为了é™ä½Žåœ¨äº‘å¹³å°ä¸Šè¿è¡Œçš„è¿™ç§é£Žé™©ï¼Œè¯·éµå¾ª [Kubernetes 安全建议](https://kubernetes.io/docs/tasks/administer-cluster/securing-a-cluster/#restricting-cloud-metadata-api-access)，å³é™åˆ¶æŽˆäºˆå®žä¾‹å‡­è¯çš„æƒé™ï¼Œä½¿ç”¨ç½‘络策略é™åˆ¶ pod å¯¹å…ƒæ•°æ® API 的访问，并é¿å…使用é…置数æ®æ¥ä¼ é€’密文。 + +建议你å‚阅你使用的云æ供商的安全最佳实践，获å–é™åˆ¶å¯¹äº‘å®žä¾‹å…ƒæ•°æ® API 访问的建议和详情。 + +è¦èŽ·å–更多å‚考资料，请å‚阅 MITRE ATT&CK 知识库 - [ä¸å®‰å…¨å‡­è¯ï¼šäº‘å®žä¾‹å…ƒæ•°æ® API](https://attack.mitre.org/techniques/T1552/005/)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/psa-restricted-exemptions.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/psa-restricted-exemptions.md new file mode 100644 index 00000000000..d623ef71dfb --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/psa-restricted-exemptions.md @@ -0,0 +1,62 @@ +--- +title: PodSecurityConfiguration 示例 +--- + +以下 PodSecurityConfiguration 包å«äº† `rancher-restricted` 集群正常è¿è¡Œæ‰€éœ€çš„ Rancher 命å空间è±å…。 + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-elemental-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-resources-system, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator] +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/security-advisories-and-cves.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/security-advisories-and-cves.md new file mode 100644 index 00000000000..b34a5f15626 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/security-advisories-and-cves.md @@ -0,0 +1,41 @@ +--- +title: 安全公告和 CVE +--- + +Rancher 致力于å‘社区披露我们产å“的安全问题。我们会针对已解决的问题å‘布安全公告和 CVE（Common Vulnerabilities and Exposures，通用æ¼æ´žæŠ«éœ²ï¼‰ã€‚Rancher GitHub 上的[安全页é¢](https://github.com/rancher/rancher/security/advisories)也会å‘布新的安全公告。 + +| ID | æè¿° | 日期 | 解决 | +|----|-------------|------|------------| +| [CVE-2023-22648](https://github.com/rancher/rancher/security/advisories/GHSA-vf6j-6739-78m8) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。在用户注销并é‡æ–°ç™»å½•åˆ° Rancher UI 之å‰ï¼ŒAzure AD 中的æƒé™æ›´æ”¹ä¸ä¼šå映给用户。 | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2022-43760](https://github.com/rancher/rancher/security/advisories/GHSA-46v3-ggjg-qq3x) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。攻击者å¯ä»¥é€šè¿‡ Rancher UI 利用多个跨站脚本 (XSS) æ¼æ´žã€‚ | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2020-10676](https://github.com/rancher/rancher/security/advisories/GHSA-8vhc-hwhc-cpj4) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。具有更新命å空间æƒé™çš„用户å¯ä»¥å°†è¯¥å‘½å空间移动到他们无æƒè®¿é—®çš„项目中。 | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22647](https://github.com/rancher/rancher/security/advisories/GHSA-p976-h52c-26p6) | 在 Rancher 2.6.12 å’Œ 2.7.3 åŠä¹‹å‰çš„版本中å‘现了一个问题。Standard åŠä»¥ä¸Šç”¨æˆ·èƒ½å¤Ÿå°†ä»–们的æƒé™æå‡ä¸º local 集群中的管ç†å‘˜ã€‚ | 2023 å¹´ 5 月 31 æ—¥ | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22651](https://github.com/rancher/rancher/security/advisories/GHSA-6m9f-pj6w-w87g) | 由于 webhook 的更新逻辑失败，Rancher 准入 webhook å¯èƒ½ä¼šé…置错误。准入 webhook 在资æºå…许进入 Kubernetes 集群之å‰ä¼šå¼ºåˆ¶æ‰§è¡ŒéªŒè¯è§„则和安全检查。webhook 在é™çº§çŠ¶æ€ä¸‹è¿è¡Œæ—¶å°†ä¸å†éªŒè¯ä»»ä½•èµ„æºï¼Œè¿™å¯èƒ½å¯¼è‡´ä¸¥é‡çš„æƒé™æå‡å’Œæ•°æ®æŸå。 | 2023 å¹´ 4 月 24 æ—¥ | Rancher [v2.7.3](https://github.com/rancher/rancher/releases/tag/v2.7.3) | +| [CVE-2022-43758](https://github.com/rancher/rancher/security/advisories/GHSA-34p5-jp77-fcrc) | 在 Rancher 2.5.0 至 2.5.16ã€2.6.0 至 2.6.9 å’Œ 2.7.0 版本中å‘现了一个问题，Rancher Git 包中存在命令注入æ¼æ´žã€‚这个包使用 Rancher 容器镜åƒä¸­å¯ç”¨çš„底层 Git 二进制文件æ¥æ‰§è¡Œ Git æ“作。特制的命令如果没有消除歧义，å¯èƒ½ä¼šåœ¨é€šè¿‡ Git 执行时造æˆæ··æ·†ï¼Œå¯¼è‡´åœ¨åº•å±‚ Rancher 主机中进行命令注入。 | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43757](https://github.com/rancher/rancher/security/advisories/GHSA-cq4p-vp5q-4522) | æ­¤é—®é¢˜å½±å“ Rancher 2.5.0 到 2.5.16，2.6.0 至 2.6.9 å’Œ 2.7.0。我们å‘现 Rancher 之å‰å‘布的安全公告 [CVE-2021-36782](https://github.com/advisories/GHSA-g7j7-h4q8-8w2f) 没有解决æŸäº›æ•æ„Ÿå­—段ã€Secret Tokenã€åŠ å¯†å¯†é’¥å’Œ SSH 密钥，这些字段ä»ç„¶ä»¥æ˜Žæ–‡å½¢å¼ç›´æŽ¥å­˜å‚¨åœ¨ Kubernetes 上 `Clusters` 之类的对象。在 Rancher 中，集群中已认è¯çš„ `Cluster Owners`ã€`Cluster Members`ã€`Project Owners` å’Œ `Project Members` å¯ä»¥çœ‹åˆ°å…¬å¼€çš„凭è¯ã€‚ | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43755](https://github.com/rancher/rancher/security/advisories/GHSA-8c69-r38j-rpfj) | 在 Rancher 2.6.9 å’Œ 2.7.0 åŠä¹‹å‰çš„版本中å‘现了一个问题。`cattle-cluster-agent` 使用的 `cattle-token` Secret 是å¯é¢„测的。é‡æ–°ç”Ÿæˆ Token 之åŽï¼ŒToken 的值ä¾ç„¶ç›¸åŒã€‚如果 Token 被泄露并且出于安全目的需è¦é‡æ–°åˆ›å»ºï¼Œè¿™å¯èƒ½ä¼šé€ æˆä¸¥é‡çš„问题。Rancher çš„ `cattle-cluster-agent` 使用 `cattle-token` æ¥è¿žæŽ¥åˆ° Rancher é…置的下游集群 Kubernetes API。 | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1) å’Œ [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) | +| [CVE-2022-21953](https://github.com/rancher/rancher/security/advisories/GHSA-g25r-gvq3-wrq7) | 在 Rancher 2.5.16ã€2.6.9 å’Œ 2.7.0 åŠä¹‹å‰çš„版本中å‘现了一个问题。由于授æƒé€»è¾‘缺陷，任何下游集群上ç»è¿‡èº«ä»½è®¤è¯çš„用户都能在 Rancher `local` 集群中打开一个 shell pod (1)，而且对 kubectl 具有有é™çš„访问æƒé™ (2)。预期的行为是：除éžæ˜Žç¡®æŽˆäºˆæƒé™ï¼Œå¦åˆ™ç”¨æˆ·åœ¨ Rancher `local` 集群中没有这样的访问æƒé™ã€‚ | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [GHSA-c45c-39f6-6gw9](https://github.com/rancher/rancher/security/advisories/GHSA-c45c-39f6-6gw9) | æ­¤é—®é¢˜å½±å“ Rancher 2.5.0 到 2.5.16，2.6.0 至 2.6.9 å’Œ 2.7.0。åªä¼šå½±å“é…置了或é…置过外部身份认è¯æ供程åºçš„ Rancher 设置。我们å‘现，当在 Rancher 中é…置外部身份认è¯æ供程åºç„¶åŽå°†å…¶ç¦ç”¨æ—¶ï¼ŒRancher 生æˆçš„ Token 如果关è”了通过现已ç¦ç”¨çš„身份认è¯æ供程åºæŽˆäºˆè®¿é—®æƒé™çš„用户，那么 Token ä¸ä¼šè¢«æ’¤é”€ã€‚ | 2023 å¹´ 1 月 24 æ—¥ | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1)ã€[v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) å’Œ [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-31247](https://github.com/rancher/rancher/security/advisories/GHSA-6x34-89p7-95wg) | 在 Rancher 2.5.15 å’Œ 2.6.6 åŠä¹‹å‰çš„版本中å‘现了一个问题。授æƒé€»è¾‘缺陷å…许在下游集群中通过集群角色模æ¿ç»‘定 (CRTB) 和项目角色模æ¿ç»‘定 (PRTB) æ¥æå‡æƒé™ã€‚任何有æƒé™åˆ›å»º/编辑 CRTB 或 PRTB 的用户（例如 `cluster-owner`ã€`manage cluster members`ã€`project-owner` å’Œ `manage project members`）都å¯ä»¥åˆ©ç”¨è¯¥æ¼æ´žï¼Œåœ¨åŒä¸€é›†ç¾¤çš„å¦ä¸€ä¸ªé¡¹ç›®æˆ–ä¸åŒä¸‹æ¸¸é›†ç¾¤çš„å¦ä¸€ä¸ªé¡¹ç›®ä¸­èŽ·å¾—所有者æƒé™ã€‚ | 2022 å¹´ 8 月 18 æ—¥ | [Rancher 2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) å’Œ [Rancher 2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36783](https://github.com/rancher/rancher/security/advisories/GHSA-8w87-58w6-hfv8) | 2.5.12 到 2.6.3 çš„ Rancher 版本无法正确清ç†é›†ç¾¤æ¨¡æ¿ answer 中的凭è¯ã€‚此错误å¯èƒ½ä¼šå¯¼è‡´æ˜Žæ–‡å­˜å‚¨ä»¥åŠå‡­è¯ã€å¯†ç å’Œ API 令牌被暴露。在 Rancher 中，已认è¯çš„ `Cluster Owner`ã€`Cluster Member`ã€`Project Owner` å’Œ `Project Member` å¯ä»¥åœ¨ `/v1/management.cattle.io.clusters`ã€`/v3/clusters` å’Œ `/k8s/clusters/local/apis/management.cattle.io/v3/clusters` 端点上看到暴露的凭è¯ã€‚ | 2022 å¹´ 8 月 18 æ—¥ | [Rancher 2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) å’Œ [Rancher 2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36782](https://github.com/rancher/rancher/security/advisories/GHSA-g7j7-h4q8-8w2f) | 在 2.5.15 到 2.6.6 çš„ Rancher 版本中å‘现了一个问题，其中密ç ã€API 密钥和 Rancher çš„ ServiceAccount 令牌（用于é…置集群）等æ•æ„Ÿå­—段直接以明文形å¼å­˜å‚¨åœ¨ `Cluster` ç­‰ Kubernetes 对象上（例如，`cluster.management.cattle.io`ï¼‰ã€‚ä»»ä½•èƒ½å¤Ÿè¯»å– Kubernetes API 中的对象的用户都å¯ä»¥æ£€ç´¢è¿™äº›æ•æ„Ÿæ•°æ®çš„明文版本。该问题由 Florian Struck（æ¥è‡ª [Continum AG](https://www.continum.net/)）和 [Marco Stuurman](https://github.com/fe-ax)（æ¥è‡ª [Shock Media B.V.](https://www.shockmedia.nl/)）å‘现并报告。 | 2022 å¹´ 8 月 18 æ—¥ | [Rancher 2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) å’Œ [Rancher 2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2022-21951](https://github.com/rancher/rancher/security/advisories/GHSA-vrph-m5jj-c46c) | æ­¤æ¼æ´žä»…å½±å“通过 [RKE 模æ¿](https://rancher.com/docs/rancher/v2.6/en/admin-settings/rke-templates/)é…ç½® [Weave](https://rancher.com/docs/rancher/v2.6/en/faq/networking/cni-providers/#weave) å®¹å™¨ç½‘ç»œæŽ¥å£ (CNI) 的客户。在 Rancher 2.5.0 到 2.5.13 å’Œ Rancher 2.6.0 到 2.6.4 版本中å‘现了一个æ¼æ´žã€‚如果将 CNI 选为 Weave，RKE 模æ¿çš„ç”¨æˆ·ç•Œé¢ (UI) ä¸åŒ…括 Weave 密ç çš„值。如果基于上述模æ¿åˆ›å»ºé›†ç¾¤ï¼Œå¹¶ä¸”å°† Weave é…置为 CNI，则 Weave 中ä¸ä¼šä¸º[网络加密](https://www.weave.works/docs/net/latest/tasks/manage/security-untrusted-networks/)创建密ç ã€‚因此，集群中的网络æµé‡å°†ä¸åŠ å¯†å‘é€ã€‚ | 2022 å¹´ 5 月 24 æ—¥ | [Rancher 2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) å’Œ [Rancher 2.5.14](https://github.com/rancher/rancher/releases/tag/v2.5.14) | +| [CVE-2021-36784](https://github.com/rancher/rancher/security/advisories/GHSA-jwvr-vv7p-gpwq) | 在 Rancher 2.5.0 到 2.5.12 å’Œ Rancher 2.6.0 到 2.6.3 中å‘现了一个æ¼æ´žï¼Œè¯¥æ¼æ´žå…许能创建或更新[全局角色](https://rancher.com/docs/rancher/v2.6/en/admin-settings/rbac/)的用户将他们或其他用户å‡çº§ä¸ºç®¡ç†å‘˜ã€‚全局角色能授予用户 Rancher 级别的æƒé™ï¼Œä¾‹å¦‚能创建集群。在已识别的 Rancher 版本中，如果用户被授予了编辑或创建全局角色的æƒé™ï¼Œä»–们ä¸ä»…仅能授予他们已ç»æ‹¥æœ‰çš„æƒé™ã€‚æ­¤æ¼æ´žå½±å“使用能够创建或编辑全局角色的éžç®¡ç†å‘˜ç”¨æˆ·çš„客户。此场景最常è§çš„用例是 `restricted-admin` 角色。 | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) å’Œ [Rancher 2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-4200](https://github.com/rancher/rancher/security/advisories/GHSA-hx8w-ghh8-r4xf) | æ­¤æ¼æ´žä»…å½±å“在 Rancher 中使用 `restricted-admin` 角色的客户。在 Rancher 2.5.0 到 2.5.12 å’Œ 2.6.0 到 2.6.3 中å‘现了一个æ¼æ´žï¼Œå…¶ä¸­ `cattle-global-data` 命å空间中的 `global-data` 角色授予了应用商店的写æƒé™ã€‚由于具有任何级别的应用商店访问æƒé™çš„用户都会绑定到 `global-data` è§’è‰²ï¼Œå› æ­¤è¿™äº›ç”¨æˆ·éƒ½èƒ½å†™å…¥æ¨¡æ¿ `CatalogTemplates`) 和模æ¿ç‰ˆæœ¬ (`CatalogTemplateVersions`)。在 Rancher 中创建的新用户默认分é…到 `user` 角色（普通用户），该角色本ä¸è¯¥å…·æœ‰å†™å…¥åº”用商店的æƒé™ã€‚æ­¤æ¼æ´žæå‡äº†èƒ½å†™å…¥åº”用商店模æ¿å’Œåº”用商店模æ¿ç‰ˆæœ¬èµ„æºçš„用户的æƒé™ã€‚ | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) å’Œ [Rancher 2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [GHSA-wm2r-rp98-8pmh](https://github.com/rancher/rancher/security/advisories/GHSA-wm2r-rp98-8pmh) | æ­¤æ¼æ´žä»…å½±å“使用ç»è¿‡è®¤è¯çš„ Git å’Œ/或 Helm 仓库通过 [Fleet](https://rancher.com/docs/rancher/v2.6/en/deploy-across-clusters/fleet/) 进行æŒç»­äº¤ä»˜çš„客户。在 [`v1.5.11`](https://github.com/hashicorp/go-getter/releases/tag/v1.5.11) 之å‰ç‰ˆæœ¬ä¸­çš„ `go-getter` 库中å‘现了一个问题，错误消æ¯ä¸­æ²¡æœ‰åˆ é™¤ Base64 ç¼–ç çš„ SSH ç§é’¥ï¼Œå¯¼è‡´è¯¥ä¿¡æ¯æš´éœ²ã€‚Rancher 中 [`v0.3.9`](https://github.com/rancher/fleet/releases/tag/v0.3.9) 之å‰çš„ Fleet 版本使用了该库的æ¼æ´žç‰ˆæœ¬ã€‚æ­¤é—®é¢˜å½±å“ Rancher 2.5.0 到 2.5.12（包括 2.5.12ï¼‰ä»¥åŠ 2.6.0 到 2.6.3（包括 2.6.3）。该问题由 Raft Engineering çš„ Dagan Henderson å‘现并报告。 | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) å’Œ [Rancher 2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-36778](https://github.com/rancher/rancher/security/advisories/GHSA-4fc7-hc63-7fjg) | 在 Rancher 2.5.0 到 2.5.11 å’Œ Rancher 2.6.0 到 2.6.2 中å‘现了一个æ¼æ´žï¼Œå½“从é…置的ç§æœ‰ä»“库下载 Helm Chart 时，对åŒæºç­–略的检查ä¸è¶³å¯èƒ½å¯¼è‡´ä»“库凭è¯æš´éœ²ç»™ç¬¬ä¸‰æ–¹æ供商。仅当用户在 Rancher çš„`应用 & 应用市场 > 仓库`中é…ç½®ç§æœ‰ä»“库的访问凭è¯æ—¶æ‰ä¼šå‡ºçŽ°æ­¤é—®é¢˜ã€‚该问题由 Martin Andreas Ullrich å‘现并报告。 | 2022 å¹´ 4 月 14 æ—¥ | [Rancher 2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) å’Œ [Rancher 2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) | +| [GHSA-hwm2-4ph6-w6m5](https://github.com/rancher/rancher/security/advisories/GHSA-hwm2-4ph6-w6m5) | 在 Rancher 2.0 到 2.6.3 中å‘现了一个æ¼æ´žã€‚Rancher æ供的 `restricted` Pod 安全策略（PSP）与 Kubernetes æ供的上游 `restricted` 策略有差别，因此 Rancher çš„ PSP å°† `runAsUser` 设置为 `runAsAny`，而上游将 `runAsUser` 设置为 `MustRunAsNonRoot`。因此，å³ä½¿ Rancher çš„ `restricted` 策略是在项目或集群级别上强制执行的，容器也å¯ä»¥ä»¥ä»»ä½•ç”¨æˆ·èº«ä»½è¿è¡Œï¼ŒåŒ…括特æƒç”¨æˆ· (`root`)。 | 2022 å¹´ 3 月 31 æ—¥ | [Rancher 2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | +| [CVE-2021-36775](https://github.com/rancher/rancher/security/advisories/GHSA-28g7-896h-695v) | 在 Rancher 2.4.17ã€2.5.11 å’Œ 2.6.2 以åŠæ›´é«˜çš„版本中å‘现了一个æ¼æ´žã€‚从项目中删除与æŸä¸ªç»„å…³è”çš„`项目角色`åŽï¼Œèƒ½è®©è¿™äº›ä½¿ç”¨è€…访问集群级别资æºçš„绑定（Binding）ä¸ä¼šè¢«åˆ é™¤ã€‚导致问题的原因是ä¸å®Œæ•´çš„授æƒé€»è¾‘检查。如果用户是å—å½±å“组中的æˆå‘˜ï¼Œä¸”能对 Rancher 进行认è¯è®¿é—®ï¼Œé‚£ä¹ˆç”¨æˆ·å¯ä»¥åˆ©ç”¨æ­¤æ¼æ´žè®¿é—®ä»–们ä¸åº”该能访问的资æºã€‚暴露级别å–决于å—å½±å“项目角色的原始æƒé™çº§åˆ«ã€‚æ­¤æ¼æ´žä»…å½±å“在 Rancher 中基于组进行身份验è¯çš„客户。 | 2022 å¹´ 3 月 31 æ—¥ | [Rancher 2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3)ã€[Rancher 2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) å’Œ [Rancher 2.4.18](https://github.com/rancher/rancher/releases/tag/v2.4.18) | +| [CVE-2021-36776](https://github.com/rancher/rancher/security/advisories/GHSA-gvh9-xgrq-r8hw) | 在 Rancher 2.5.0 到 2.5.9 中å‘现了一个æ¼æ´žï¼Œè¯¥æ¼æ´žå…许ç»è¿‡è®¤è¯ç”¨æˆ·é€šè¿‡ API 代ç†æ¨¡æ‹Ÿé›†ç¾¤ä¸Šçš„任何用户，而无需知é“被模拟用户的凭è¯ã€‚问题的原因是 API 代ç†åœ¨å°†è¯·æ±‚å‘é€åˆ° Kubernetes API 之å‰æœªåˆ é™¤æ¨¡æ‹Ÿæ ‡å¤´ã€‚能认è¯è®¿é—® Rancher çš„æ¶æ„用户å¯ä»¥å†’å……å¦ä¸€ä¸ªåœ¨ Rancher 认è¯ç”¨æˆ·ï¼Œä»Žè€Œå¯¹é›†ç¾¤è¿›è¡Œç®¡ç†å‘˜çº§åˆ«çš„访问。 | 2022 å¹´ 3 月 31 æ—¥ | [Rancher 2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) å’Œ [Rancher 2.5.10](https://github.com/rancher/rancher/releases/tag/v2.5.10) | +| [CVE-2021-25318](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25318) | Rancher 2.0 çš„ä¸å¯ç¼–辑版本å‘现了一个æ¼æ´žï¼Œåœ¨è¯¥ç‰ˆæœ¬ä¸­ï¼Œæ— è®ºèµ„æºçš„ API 组如何，用户都å¯ä»¥è®¿é—®èµ„æºã€‚例如，Rancher 应该å…许用户访问 `apps.catalog.cattle.io`，但å´é”™è¯¯åœ°æŽˆäºˆäº†å¯¹ `apps.*` 的访问æƒé™ã€‚ä½ å¯ä»¥åœ¨[这里](https://github.com/rancher/rancher/security/advisories/GHSA-f9xf-jq4j-vqw4)找到**下游集群**å’Œ **Rancher 管ç†é›†ç¾¤**中å—å½±å“的资æºã€‚除了å‡çº§åˆ°æ‰“了补ä¸çš„ Rancher 版本之外，暂时没有直接的缓解措施。 | 2021 å¹´ 7 月 14 æ—¥ | [Rancher 2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) å’Œ [Rancher 2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-31999](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-31999) | Rancher 2.0.0 çš„è¡¥ä¸ç‰ˆæœ¬å‘现了一个æ¼æ´žï¼Œæ¶æ„çš„ Rancher 用户å¯ä»¥é’ˆå¯¹æ‰˜ç®¡é›†ç¾¤çš„ Kubernetes API 的代ç†å‘起一个 API 请求，以获å–他们无æƒè®¿é—®çš„ä¿¡æ¯ã€‚这是通过在 Connection 标头中传递 “Impersonate-User†或 “Impersonate-Group†标头æ¥å®žçŽ°çš„，然åŽä»£ç†ä¼šåˆ é™¤è¯¥æ ‡å¤´ã€‚此时，请求ä¸ä¼šæ¨¡æ‹Ÿç”¨æˆ·åŠå…¶æƒé™ï¼Œè€Œæ˜¯ä¼šç±»ä¼¼ Rancher management server 的请求，并错误地返回信æ¯ã€‚该æ¼æ´žä»…å½±å“对集群具有一定级别æƒé™çš„ Rancher 用户。除了å‡çº§åˆ°æ‰“了补ä¸çš„ Rancher 版本之外，暂时没有直接的缓解措施。 | 2021 å¹´ 7 月 14 æ—¥ | [Rancher 2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) å’Œ [Rancher 2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25320](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25320) | Rancher 2.2.0 çš„è¡¥ä¸ç‰ˆæœ¬å‘现了一个æ¼æ´žï¼Œäº‘凭è¯æ²¡æœ‰æ­£ç¡®é€šè¿‡ Rancher API 验è¯ã€‚具体地说，是通过用于与云æ供商通信的代ç†ã€‚任何登录并具有有效云æä¾›å•†äº‘å‡­è¯ ID çš„ Rancher 用户都å¯ä»¥é€šè¿‡ä»£ç† API 调用该云æ供商的 API，并且云凭è¯ä¼šè¢«ç»‘定。该æ¼æ´žä»…å½±å“有效的 Rancher 用户。除了å‡çº§åˆ°æ‰“了补ä¸çš„ Rancher 版本之外，暂时没有直接的缓解措施。 | 2021 å¹´ 7 月 14 æ—¥ | [Rancher 2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) å’Œ [Rancher 2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25313](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25313) | 所有 Rancher 2 版本上都å‘现了一个安全æ¼æ´žã€‚使用æµè§ˆå™¨è®¿é—® Rancher API 时，URL 没有正确转义，导致它容易å—到 XSS 攻击。这些 API 端点的特制 URL å¯èƒ½åŒ…括嵌入页é¢å¹¶åœ¨æµè§ˆå™¨ä¸­æ‰§è¡Œçš„ JavaScript。暂时没有直接的缓解措施。请ä¸è¦å•å‡»æŒ‡å‘ Rancher Server çš„ä¸å—信任链接。 | 2021 å¹´ 3 月 2 æ—¥ | [Rancher v2.5.6](https://github.com/rancher/rancher/releases/tag/v2.5.6)ã€[Rancher v2.4.14](https://github.com/rancher/rancher/releases/tag/v2.4.14) å’Œ [Rancher v2.3.11](https://github.com/rancher/rancher/releases/tag/v2.3.11) | +| [CVE-2019-14435](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14435) | æ­¤æ¼æ´žè®©å·²éªŒè¯çš„用户å¯ä»¥é€šè¿‡ Rancher 使用的系统æœåŠ¡å®¹å™¨å¯è®¿é—®çš„ IP æå–ç§æœ‰æ•°æ®ã€‚这包括但ä¸é™äºŽäº‘æ供商元数æ®æœåŠ¡ç­‰æœåŠ¡ã€‚虽然 Rancher å…许用户为系统æœåŠ¡é…置白åå•åŸŸï¼Œä½†è¿™ä¸ªæ¼æ´žä»ç„¶å¯ä»¥è¢«ç²¾å¿ƒè®¾è®¡çš„ HTTP 请求利用。该问题由 Workiva çš„ Matt Belisle å’Œ Alex Stevenson å‘现并报告。 | 2019 å¹´ 8 月 5 æ—¥ | [Rancher 2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) å’Œ [Rancher 2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-14436](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14436) | 该æ¼æ´žå…许有æƒç¼–辑角色绑定的项目æˆå‘˜ä¸ºè‡ªå·±æˆ–其他用户分é…集群级别的角色，从而授予他们对该集群的管ç†å‘˜è®¿é—®æƒé™ã€‚该问题由 Nokia çš„ Michal Lipinski å‘现并报告。 | 2019 å¹´ 8 月 5 æ—¥ | [Rancher 2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) å’Œ [Rancher 2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-13209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-13209) | 该æ¼æ´žè¢«ç§°ä¸º[跨网页 Websocket 劫æŒæ”»å‡»](https://www.christian-schneider.net/CrossSiteWebSocketHijacking.html)。该攻击å…许攻击者以å—害用户的角色/æƒé™è®¿é—®ç”± Rancher 管ç†çš„集群。它让å—害用户登录到 Rancher Server，然åŽè®¿é—®ç”±æ”»å‡»è€…托管的第三方站点。完æˆåŽï¼Œæ”»å‡»è€…å°±å¯ä»¥ä½¿ç”¨å—害用户的æƒé™å’Œèº«ä»½å¯¹ Kubernetes API 执行命令。该问题由 Workiva çš„ Matt Belisle å’Œ Alex Stevenson 报告。 | 2019 å¹´ 7 月 15 æ—¥ | [Rancher 2.2.5](https://github.com/rancher/rancher/releases/tag/v2.2.5)ã€[Rancher 2.1.11](https://github.com/rancher/rancher/releases/tag/v2.1.11) å’Œ [Rancher 2.0.16](https://github.com/rancher/rancher/releases/tag/v2.0.16) | +| [CVE-2019-12303](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12303) | 项目所有者å¯ä»¥æ³¨å…¥é¢å¤–çš„ fluentd 日志é…置，从而在 fluentd 容器内读å–文件或执行任æ„命令。该问题由 Untamed Theory çš„ Tyler Welton 报告。 | 2019 å¹´ 6 月 5 æ—¥ | [Rancher 2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4)ã€[Rancher 2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) å’Œ [Rancher 2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-12274](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12274) | 如果节点使用的内置主机驱动使用了文件路径选项，则节点å¯ä»¥è¯»å– Rancher Server 容器内的任æ„文件，包括æ•æ„Ÿæ–‡ä»¶ã€‚ | 2019 å¹´ 6 月 5 æ—¥ | [Rancher 2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4)ã€[Rancher 2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) å’Œ [Rancher 2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-11202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11202) | å³ä½¿å·²è¢«æ˜¾å¼åˆ é™¤ï¼ŒRancher 的默认管ç†å‘˜ä¼šåœ¨ Rancher é‡å¯æ—¶é‡æ–°åˆ›å»ºã€‚ | 2019 å¹´ 4 月 16 æ—¥ | [Rancher 2.2.2](https://github.com/rancher/rancher/releases/tag/v2.2.2)ã€[Rancher 2.1.9](https://github.com/rancher/rancher/releases/tag/v2.1.9) å’Œ [Rancher 2.0.14](https://github.com/rancher/rancher/releases/tag/v2.0.14) | +| [CVE-2019-6287](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6287) | 如果将项目æˆå‘˜æ·»åŠ åˆ°å¤šä¸ªé¡¹ç›®ä¸­ï¼Œåˆ™æˆå‘˜è¿˜èƒ½ç»§ç»­è®¿é—®è¢«åˆ é™¤çš„项目中的命å空间。 | 2019 å¹´ 1 月 29 æ—¥ | [Rancher 2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) å’Œ [Rancher 2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) | +| [CVE-2018-20321](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-20321) | 任何有æƒè®¿é—® `default` 命å空间的项目æˆå‘˜éƒ½å¯ä»¥åœ¨ pod 中挂载 `netes-default` ServiceAccount，然åŽä½¿ç”¨è¯¥ pod 对 Kubernetes 集群执行管ç†ç‰¹æƒå‘½ä»¤ã€‚ | 2019 å¹´ 1 月 29 æ—¥ | [Rancher 2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) å’Œ [Rancher 2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) - 对于这些版本或更高版本，我们有对应的[回滚说明](../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md)。 | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md new file mode 100644 index 00000000000..5146e3aeb0c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md @@ -0,0 +1,66 @@ +--- +title: 关于 rancher-selinux +--- + +è¦è®© Rancher 使用 SELinux，你必须手动为 SELinux 节点å¯ç”¨ä¸€äº›åŠŸèƒ½ã€‚为了解决这个问题，Rancher æ供了一个 SELinux RPM。 + +`rancher-selinux` RPM ä»…åŒ…å« [rancher-logging 应用程åº](https://github.com/rancher/charts/tree/dev-v2.5/charts/rancher-logging)的策略。 + +`rancher-selinux` çš„ GitHub 仓库在[这里](https://github.com/rancher/rancher-selinux)。 + +## 安装 rancher-selinux RPM + +:::note è¦æ±‚ + +rancher-selinux RPM 已在 CentOS 7 å’Œ 8 上进行了测试。 + +::: + +### 1. 设置 yum 仓库 + +设置 yum 仓库，从而直接在集群中的所有主机上安装 `rancher-selinux`。 + +è¦ä½¿ç”¨ RPM 仓库，在 CentOS 7 或 RHEL 7 系统上è¿è¡Œä»¥ä¸‹ bash 代ç ç‰‡æ®µï¼š + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/7/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` + +è¦ä½¿ç”¨ RPM 仓库，在 CentOS 8 或 RHEL 8 系统上è¿è¡Œä»¥ä¸‹ bash 代ç ç‰‡æ®µï¼š + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/8/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` +### 2. 安装 RPM + +安装 RPM： + +``` +yum -y install rancher-selinux +``` + +## é…ç½® Logging 应用程åºä»¥ä½¿ç”¨ SELinux + +:::note è¦æ±‚ + +Logging v2 已在 RHEL/CentOS 7 å’Œ 8 上使用 SELinux 进行了测试。 + +::: + +在主机上安装 `rancher-selinux` RPM åŽï¼Œåº”用程åºä¸ä¼šè‡ªåŠ¨è¿è¡Œã€‚它们需è¦åœ¨ RPM æ供的å…许的 SELinux 容器域中è¿è¡Œã€‚ + +è¦å°† `rancher-logging` Chart é…ç½®ä¸ºæ”¯æŒ SELinux，请在安装 Chart 时将 `values.yaml` 中的 `global.seLinux.enabled` 更改为 true。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md new file mode 100644 index 00000000000..b5e4a22cad9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md @@ -0,0 +1,9 @@ +--- +title: 关于 rke2-selinux +--- + +`rke2-selinux` 为 RKE2 æ供策略。如果 RKE2 安装程åºè„šæœ¬æ£€æµ‹åˆ°å®ƒè¿è¡Œåœ¨åŸºäºŽ RPM çš„å‘行版上，它会自动安装。 + +`rke2-selinux` çš„ GitHub 仓库在[这里](https://github.com/rancher/rke2-selinux)。 + +有关在å¯ç”¨ SELinux 的主机上安装 RKE2 的更多信æ¯ï¼Œè¯·å‚阅 [RKE2 文档](https://docs.rke2.io/install/methods#rpm)。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-webhook.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-webhook.md new file mode 100644 index 00000000000..caa0335ca2a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rancher-webhook.md @@ -0,0 +1,36 @@ +--- +title: Rancher Webhook +--- + +Rancher-Webhook 是 Rancher çš„é‡è¦ç»„件，它与 Kubernetes 结åˆä½¿ç”¨ï¼Œç”¨äºŽå¢žå¼ºå®‰å…¨æ€§å¹¶ä¸º Rancher 管ç†çš„集群å¯ç”¨å…³é”®åŠŸèƒ½ã€‚ + +如 [Kubernetes 文档](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)中所述，它与 Kubernetes çš„å¯æ‰©å±•å‡†å…¥æŽ§åˆ¶å™¨é›†æˆï¼Œå…许 Rancher-Webhook 检查å‘é€åˆ° Kubernetes API Server 的特定请求，添加自定义和 Rancher 相关的验è¯ï¼Œä»¥åŠ Rancher 相关请求的å˜åŒ–。Rancher-Webhook 使用 `rancher.cattle.io` `ValidatingWebhookConfiguration` å’Œ `rancher.cattle.io` `MutatingWebhookConfiguration` 管ç†è¦éªŒè¯çš„资æºï¼Œå¹¶è¦†ç›–任何手动编辑。 +Rancher å°† Rancher-Webhook 作为å•ç‹¬çš„ deployment å’ŒæœåŠ¡éƒ¨ç½²åœ¨ local 和下游集群中。Rancher 使用 Helm ç®¡ç† Rancher-Webhook。需è¦æ³¨æ„的是，Rancher å¯èƒ½ä¼šè¦†ç›–用户对 Helm 版本所åšçš„修改。 + +## 为什么我们需è¦å®ƒï¼Ÿ + +Rancher-Webhook 对于让 Rancher ä¿æŠ¤é›†ç¾¤å…å—æ¶æ„攻击并å¯ç”¨å„ç§åŠŸèƒ½è‡³å…³é‡è¦ã€‚ +Rancher ä¾èµ– Rancher-Webhook 作为其功能的组æˆéƒ¨åˆ†ã€‚如果没有 Webhook，Rancher å°†ä¸æ˜¯ä¸€ä¸ªå®Œæ•´çš„产å“。 +它为 Rancher 管ç†çš„集群æ供了必è¦çš„ä¿æŠ¤ï¼Œé˜²æ­¢å®‰å…¨æ¼æ´žå¹¶ç¡®ä¿é›†ç¾¤çš„一致性和稳定性。 + +## 常è§é—®é¢˜ + +### 带有 Calico CNI çš„ EKS 集群 + +当 Kubernetes API Server å°è¯•è”ç³» Rancher-Webhook 时，使用 Calico CNI è¿è¡Œ EKS 集群的用户å¯èƒ½ä¼šé‡åˆ°é”™è¯¯ã€‚ +æ ¹æ® [Calico 文档](https://docs.tigera.io/calico/latest/getting-started/kubernetes/managed-public-cloud/eks#install-eks-with-calico-networking)，此问题的解决方法是为 Webhook 部署设置 `hostNetwork=true`。用户å¯ä»¥åœ¨å—å½±å“的集群上使用下é¢çš„ Helm 命令更改此设置。 + +```bash +helm repo add rancher-charts https://charts.rancher.io +helm upgrade --reuse-values rancher-webhook rancher-chart/rancher-webhook -n cattle-system --set global.hostNetwork=true +``` +**注æ„**：这个临时解决方法å¯èƒ½ä¼šè¿å环境的安全策略。此解决方法还è¦æ±‚ä¸»æœºç½‘ç»œä¸Šæœªä½¿ç”¨ç«¯å£ 9443。 + +### ç§æœ‰ GKE 集群 + +使用ç§æœ‰ GKE 集群时å¯èƒ½ä¼šå‘生错误，导致 Kubernetes API Server 无法与 Webhook 通信。以下错误消æ¯å¯èƒ½ä¼šå‡ºçŽ°ï¼š + +``` +Internal error occurred: failed calling webhook "rancher.cattle.io.namespaces.create-non-kubesystem": failed to call webhook: Post "https://rancher-webhook.cattle-system.svc:443/v1/webhook/validation/namespaces?timeout=10s": context deadline exceeded +``` +出现此问题的原因是防ç«å¢™è§„则é™åˆ¶äº† API Server 与ç§æœ‰é›†ç¾¤ä¹‹é—´çš„通信。è¦è§£å†³æ­¤é€šä¿¡é—®é¢˜ï¼Œä½ å¿…须通过添加防ç«å¢™è§„则æ¥å…许 GKE Control Plane é€šè¿‡ç«¯å£ 9443 与 Rancher-Webhook 进行通信。请å‚阅 [GKE 文档](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules)，了解更新防ç«å¢™è§„则的详细信æ¯å’Œæ­¥éª¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rke1-template-example-yaml.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rke1-template-example-yaml.md new file mode 100644 index 00000000000..5efdc493a95 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/rke1-template-example-yaml.md @@ -0,0 +1,111 @@ +--- +title: RKE1 示例 YAML +--- + +以下是一个供å‚考的 RKE 模æ¿é…置文件示例。 + +RKE 模æ¿ä¸­çš„ YAML 使用与创建 RKE 集群时相åŒçš„自定义项。但是，由于 YAML ä½äºŽ Rancher é…置的 RKE 集群的上下文中，因此 RKE 文档中的自定义项需è¦åµŒå¥—在 `rancher_kubernetes_engine` 指令下。 + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker + +enable_cluster_alerting: false +# This setting is not enforced. Clusters +# created with this sample template +# would have alerting turned off by default, +# but end users could still turn alerting +# on or off. + +enable_cluster_monitoring: true +# This setting is not enforced. Clusters +# created with this sample template +# would have monitoring turned on +# by default, but end users could still +# turn monitoring on or off. + +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # ç›®å‰ä»…æ”¯æŒ Nginx ingress provider +# # è¦ç¦ç”¨ Ingress controller，设置 `provider: none` +# # è¦åœ¨æŒ‡å®šèŠ‚点上ç¦ç”¨ Ingress，使用 node_selector，例如： +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/single-node-rancher-in-docker/advanced-options.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/single-node-rancher-in-docker/advanced-options.md new file mode 100644 index 00000000000..3eae5197503 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/single-node-rancher-in-docker/advanced-options.md @@ -0,0 +1,105 @@ +--- +title: Docker 安装高级选项 +--- + +### 自定义 CA è¯ä¹¦ + +如需 Rancher 在验è¯æœåŠ¡æ—¶ä½¿ç”¨ CA æ ¹è¯ä¹¦ï¼Œè¯·åœ¨å¯åŠ¨ Rancher å®¹å™¨æ—¶å…±äº«åŒ…å« CA æ ¹è¯ä¹¦çš„目录。 + +使用命令示例æ¥å¯åŠ¨æŒ‚载了ç§æœ‰ CA è¯ä¹¦çš„ Rancher 容器。 + +- å·æ ‡å¿— (`-v`) åº”æŒ‡å®šåŒ…å« CA æ ¹è¯ä¹¦çš„主机目录。 +- 环境å˜é‡æ ‡å¿— (`-e`) 与 `SSL_CERT_DIR` 和目录共åŒå£°æ˜Žä¸€ä¸ªçŽ¯å¢ƒå˜é‡ï¼Œè¯¥å˜é‡æŒ‡å®šå®¹å™¨å†…挂载了 CA æ ¹è¯ä¹¦çš„目录ä½ç½®ã€‚ +- ä½ å¯ä½¿ç”¨ `-e KEY=VALUE` 或 `--env KEY=VALUE`将环境å˜é‡ä¼ ç»™ Rancher 容器。 +- ä½ å¯ä½¿ç”¨ `-v host-source-directory:container-destination-directory` 或 `--volume host-source-directory:container-destination-directory`在容器内挂载主机目录。 + +在以下示例将ä½äºŽ `/host/certs` 主机目录中的 CA è¯ä¹¦ï¼ŒæŒ‚载到 Rancher 容器内的 `/container/certs` 上。 + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /host/certs:/container/certs \ + -e SSL_CERT_DIR="/container/certs" \ + --privileged \ + rancher/rancher:latest +``` + +### API 审计日志 + +API 审计日志记录了通过 Rancher Server 进行的所有用户和系统事务。 + +默认情况下，API 审计日志会写入到 Rancher 容器内的 `/var/log/auditlog`。你å¯å°†æ­¤ç›®å½•å…±äº«ä¸ºå·ï¼Œå¹¶è®¾ç½® `AUDIT_LEVEL` 以å¯ç”¨æ—¥å¿—。 + +有关更多信æ¯å’Œé€‰é¡¹ï¼Œè¯·å‚è§ [API 审计日志](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md)。 + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /var/log/rancher/auditlog:/var/log/auditlog \ + -e AUDIT_LEVEL=1 \ + --privileged \ + rancher/rancher:latest +``` + +### TLS 设置 + +如需使用ä¸åŒçš„ TLS é…置，你å¯ä½¿ç”¨ `CATTLE_TLS_MIN_VERSION` å’Œ `CATTLE_TLS_CIPHERS` 环境å˜é‡ã€‚例如，将 TLS 1.0 设为å¯æŽ¥å—的最低 TLS 版本，如下： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_TLS_MIN_VERSION="1.0" \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +å‚è§ [TLS 设置](../../getting-started/installation-and-upgrade/installation-references/tls-settings.md)了解更多信æ¯å’Œé€‰é¡¹ã€‚ + +### 离线环境 + +如果è¦è®¿é—®æ­¤é¡µé¢ä»¥å®Œæˆç¦»çº¿å®‰è£…，在è¿è¡Œå®‰è£…命令时，你必须把ç§æœ‰é•œåƒä»“库的 URL 添加到 Server 标志å‰é¢ã€‚例如，将带有ç§æœ‰é•œåƒä»“库 URL çš„ `` 添加到 `rancher/rancher:latest` å‰é¢ã€‚ + +**示例**： + + /rancher/rancher:latest + +### æŒä¹…åŒ–æ•°æ® + +Rancher 使用 etcd 作为数æ®å­˜å‚¨ã€‚如果 Rancher 是使用 Docker 安装的，Rancher ä¼šä½¿ç”¨åµŒå…¥å¼ etcd。æŒä¹…化数æ®ä½äºŽå®¹å™¨çš„ `/var/lib/rancher` 路径中。 + +ä½ å¯ä»¥å°†ä¸»æœºå·æŒ‚载到该ä½ç½®ï¼Œæ¥å°†æ•°æ®ä¿ç•™åœ¨è¿è¡Œå®ƒçš„主机上： + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +### 在åŒä¸€ä¸ªèŠ‚点中è¿è¡Œ `rancher/rancher` å’Œ `rancher/rancher-agent` + +如需使用å•ä¸ªèŠ‚点è¿è¡Œ Rancher 并将åŒä¸€ä¸ªèŠ‚点添加到集群，你必须调整映射给 `rancher/rancher` 容器的主机端å£ã€‚ + +å¦‚æžœå°†èŠ‚ç‚¹æ·»åŠ åˆ°é›†ç¾¤ä¸­ï¼ŒèŠ‚ç‚¹å°†éƒ¨ç½²ä½¿ç”¨ç«¯å£ 80 å’Œ 443 çš„ NGINX Ingress Controller。而这将与我们建议用于暴露 `rancher/rancher` 容器的默认端å£å†²çªã€‚ + +请知悉我们ä¸å»ºè®®å°†æ­¤è®¾ç½®ç”¨äºŽç”Ÿäº§çŽ¯å¢ƒï¼Œè¿™ç§æ–¹å¼ä»…用æ¥æ–¹ä¾¿è¿›è¡Œå¼€å‘/演示。 + +如需更改主机端å£æ˜ å°„，将 `-p 80:80 -p 443:443` 替æ¢ä¸º `-p 8080:80 -p 8443:443`： + +``` +docker run -d --restart=unless-stopped \ + -p 8080:80 -p 8443:443 \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md new file mode 100644 index 00000000000..7d7405a255b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md @@ -0,0 +1,61 @@ +--- +title: HTTP 代ç†é…ç½® +--- + +如果你通过代ç†æ¥æ“作 Rancher，并想è¦é€šè¿‡ä»£ç†è®¿é—®æœåŠ¡ï¼ˆä¾‹å¦‚拉å–应用商店），你需è¦æä¾› Rancher 代ç†çš„ä¿¡æ¯ã€‚由于 Rancher 是用 Go 编写的，Rancher 使用如下常è§çš„代ç†çŽ¯å¢ƒå˜é‡ã€‚ + +è¯·ç¡®ä¿ `NO_PROXY` 包å«ä¸ä½¿ç”¨ä»£ç†çš„网络地å€ï¼Œç½‘络地å€èŒƒå›´å’ŒåŸŸå。 + +| 环境å˜é‡ | 用途 | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| HTTP_PROXY | å‘èµ· HTTP 连接的代ç†åœ°å€ | +| HTTPS_PROXY | å‘èµ· HTTPS 连接的代ç†åœ°å€ | +| NO_PROXY | å‘起连接时ä¸ä½¿ç”¨ä»£ç†çš„网络地å€ï¼Œç½‘络地å€èŒƒå›´å’ŒåŸŸå | + +:::note é‡è¦æ示： + +`NO_PROXY` 必须大写æ‰èƒ½ä½¿ç”¨ç½‘络范围 CIDR 表示法。 + +::: + +## 基于 Docker 安装 + +ä½ å¯ä½¿ç”¨ `-e KEY=VALUE` 或 `--env KEY=VALUE`将环境å˜é‡ä¼ ç»™ Rancher 容器。在 [Docker 安装](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md)中，`NO_PROXY` 必须的值为： + +- `localhost` +- `127.0.0.1` +- `0.0.0.0` +- `10.0.0.0/8` +- `cattle-system.svc` +- `.svc` +- `.cluster.local` + +以下示例中，代ç†æœåŠ¡å™¨å¯ä»¥é€šè¿‡ `http://192.168.0.1:3128` 访问。此外，在访问 `192.168.10.0/24` ç½‘ç»œèŒƒå›´ä»¥åŠ `example.com` 域å下的æ¯ä¸ªä¸»æœºåæ—¶å‡ä¸ä½¿ç”¨ä»£ç†æœåŠ¡å™¨ã€‚ + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e HTTP_PROXY="http://192.168.10.1:3128" \ + -e HTTPS_PROXY="http://192.168.10.1:3128" \ + -e NO_PROXY="localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,cattle-system.svc,192.168.10.0/24,.svc,.cluster.local,example.com" \ + --privileged \ + rancher/rancher:latest +``` + +特æƒè®¿é—®æ˜¯[å¿…é¡»](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#rancher-特æƒè®¿é—®)的。 + +### 离线代ç†é…ç½® + +你现在å¯ä»¥åœ¨é…置的离线集群中é…置主机驱动集群，以使用代ç†è¿›è¡Œå‡ºç«™è¿žæŽ¥ã€‚ + +除了如上为代ç†æœåŠ¡å™¨è®¾ç½®é»˜è®¤è§„则外，你还需è¦é¢å¤–添加如下所示的规则，以从代ç†çš„ Rancher 环境中é…置主机驱动集群。 + +æ ¹æ®ä½ çš„设置é…置文件路径，例如 `/etc/apt/apt.conf.d/proxy.conf`： + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/system-tools.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/system-tools.md new file mode 100644 index 00000000000..ce21d6c41fd --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/system-tools.md @@ -0,0 +1,31 @@ +--- +title: 系统工具 +--- + +:::note + +系统工具（System Tools）自 2022 å¹´ 6 月起已弃用。 + +::: + +## 日志 + +请使用 [logs-collector](https://github.com/rancherlabs/support-tools/tree/master/collection/rancher/v2.x/logs-collector) æ¥æ”¶é›†ä½ çš„集群日志。 + +## Stats + +如果è¦å¤åˆ¶ stats 命令，你å¯ä»¥åœ¨é›†ç¾¤èŠ‚点上è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š + +:::note + +以下命令需è¦é›†ç¾¤èŠ‚点上的 `sysstat` 包。 + +::: + +``` +/usr/bin/sar -u -r -F 1 1 +``` + +## Remove + +请使用 [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) 工具。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/api-keys.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/api-keys.md new file mode 100644 index 00000000000..287b71feaf0 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/api-keys.md @@ -0,0 +1,59 @@ +--- +title: API 密钥 +--- + +## API å¯†é’¥å’Œç”¨æˆ·èº«ä»½éªŒè¯ + +如果你想通过外部应用程åºæ¥è®¿é—® Rancher 集群ã€é¡¹ç›®æˆ–其他对象，你å¯ä»¥ä½¿ç”¨ Rancher API。但是，在你的应用程åºå¯ä»¥è®¿é—® API 之å‰ï¼Œä½ å¿…须为应用程åºæä¾›ç”¨äºŽå‘ Rancher 进行身份验è¯çš„密钥。你å¯ä»¥é€šè¿‡ Rancher UI 获å–密钥。 + +使用 Rancher CLI ä¹Ÿéœ€è¦ API 密钥。 + +API 密钥由四个组件组æˆï¼š + +- **端点**：其他应用程åºç”¨æ¥å‘ Rancher API å‘é€è¯·æ±‚çš„ IP 地å€å’Œè·¯å¾„。 +- **访问密钥**：Token 的用户å。 +- **密文密钥**：Token 的密ç ã€‚如果应用程åºæ示你输入两个ä¸åŒçš„字符串进行 API 身份验è¯ï¼Œä½ é€šå¸¸éœ€è¦å°†ä¸¤ä¸ªå¯†é’¥ä¸€èµ·è¾“入。 +- **æŒæœ‰è€…令牌**：连接在一起的令牌用户å和密ç ã€‚如果应用程åºæ示你输入一个身份验è¯å­—符串，则使用此字符串。 + +:::note + +用户å¯ä»¥é€‰æ‹©å¯ç”¨[令牌哈希（token hashing）](../about-the-api/api-tokens.md)。 + +::: + +## 创建 API 密钥 + +1. 从å³ä¸Šè§’选择**ç”¨æˆ·å¤´åƒ > è´¦å· & API 密钥**。 + +2. å•å‡»**创建 API 密钥**。 + +3. **å¯é€‰**：输入 API 密钥的æ述并选择有效期或范围。我们建议设置到期日期。 + + API 密钥过期åŽå°†å¤±æ•ˆã€‚有效期短一点会更安全。 + + æœ‰æ•ˆæœŸå°†å— `v3/settings/auth-token-max-ttl-minutes` 约æŸã€‚如果超过 max-ttl，则会以 max-ttl 为有效期创建 API 密钥。 + + 范围将é™åˆ¶ API 密钥，使其仅适用于指定集群的 Kubernetes API。如果集群é…置了授æƒé›†ç¾¤ç«¯ç‚¹ï¼Œä½ å°†èƒ½å¤Ÿç›´æŽ¥é’ˆå¯¹é›†ç¾¤çš„ API 使用设定了范围的令牌，而无需通过 Rancher Server 进行代ç†ã€‚有关详细信æ¯ï¼Œè¯·å‚阅[授æƒé›†ç¾¤ç«¯ç‚¹](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-授æƒé›†ç¾¤ç«¯ç‚¹)。 + +4. å•å‡»**创建**。 + + **步骤结果**：已创建 API 密钥。将会显示你的 API **端点**ã€**访问密钥**ã€**密文密钥**å’Œ**æŒæœ‰è€…令牌**。 + + 使用**æŒæœ‰è€…令牌**通过 Rancher CLI 进行身份验è¯ã€‚ + +5. 将显示的信æ¯å¤åˆ¶åˆ°å®‰å…¨ä½ç½®ã€‚此信æ¯ä»…显示一次，因此如果你丢失了密钥，则必须制作一个新的。 + +## åŽç»­æ“作 + +- å°† API 密钥信æ¯è¾“å…¥åˆ°å°†å‘ Rancher API å‘é€è¯·æ±‚的应用程åºä¸­ã€‚ +- è¦äº†è§£ Rancher 端点和å‚数的更多信æ¯ï¼Œä½ å¯ä»¥ä¸º Rancher UI 中的对象选择**在 API 中查看**。 +- API 密钥å¯ç”¨äºŽ API 调用和 [Rancher CLI](../../pages-for-subheaders/cli-with-rancher.md)。 + +## 删除 API 密钥 + +如果è¦æ’¤é”€ API 密钥，请将其删除。如果出现以下情况，你需è¦åˆ é™¤ API 密钥： + +- API 密钥å¯èƒ½å·²ç»æ³„露。 +- API 密钥已过期。 + +è¦åˆ é™¤ API，选择密钥并å•å‡»**删除**。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/manage-cloud-credentials.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/manage-cloud-credentials.md new file mode 100644 index 00000000000..b4f7f71c543 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/manage-cloud-credentials.md @@ -0,0 +1,51 @@ +--- +title: 管ç†äº‘å‡­è¯ +--- + +如果è¦åˆ›å»º[由基础设施æ供商托管](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)的集群，则å¯ä»¥ä½¿ç”¨[节点模æ¿](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)æ¥é…置集群节点。这些模æ¿ä½¿ç”¨ Docker Machine é…置选项æ¥å®šä¹‰èŠ‚点的æ“作系统镜åƒä»¥åŠè®¾ç½®/å‚数。 + +节点模æ¿å¯ä»¥ä½¿ç”¨äº‘凭è¯æ¥è®¿é—®åœ¨åŸºç¡€è®¾æ–½æ供商中é…置节点所需的凭è¯ä¿¡æ¯ã€‚多个节点模æ¿å¯ä»¥ä½¿ç”¨ç›¸åŒçš„云凭è¯ã€‚云凭è¯çœåŽ»äº†ä¸ºåŒä¸€äº‘æ供商é‡æ–°è¾“入访问密钥的麻烦。云凭è¯å­˜å‚¨åœ¨ Kubernetes 密文中。 + +åªæœ‰å­˜åœ¨æ ‡è®°ä¸º `password` 的字段时，节点模æ¿æ‰ä¼šä½¿ç”¨äº‘凭è¯ã€‚默认的 `active` 主机驱动将其账å·è®¿é—®å­—段标记为 `password`，但å¯èƒ½æœ‰ä¸€äº› `inactive` 主机驱动没有使用它们。这些主机驱动ä¸ä¼šä½¿ç”¨äº‘凭è¯ã€‚ + +ä½ å¯ä»¥åœ¨ä¸¤ç§æƒ…况下创建云凭è¯ï¼š + +- [在为集群创建节点模æ¿æœŸé—´](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)。 +- 在**用户设置**中。 + +所有云凭è¯éƒ½ç»‘定到创建者的用户é…置文件。**ä¸èƒ½**与其他用户共享。 + +## ä½¿ç”¨ç”¨æˆ·è®¾ç½®åˆ›å»ºäº‘å‡­è¯ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. å•å‡»**创建**。 +1. å•å‡»äº‘凭è¯ç±»åž‹ã€‚下拉列表中的选项å–决于 Rancher 中的 `active` [主机驱动](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md)。 +1. 输入云凭è¯çš„å称。 +1. æ ¹æ®æ‰€é€‰çš„云凭è¯ç±»åž‹è¾“入所需的值，从而å‘基础设施æ供商进行身份验è¯ã€‚ +1. å•å‡»**创建**。 + +**结果**：已创建云凭è¯ï¼Œå®ƒå¯ä»¥ç«‹å³ç”¨äºŽ[创建节点模æ¿](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)。 + +## æ›´æ–°äº‘å‡­è¯ + +如果访问凭è¯æ›´æ”¹æˆ–泄露了，你å¯ä»¥é€šè¿‡æ›´æ–°äº‘凭è¯æ¥è¿›è¡Œè½®æ¢ï¼ŒåŒæ—¶ä¿ç•™ç›¸åŒçš„节点模æ¿ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. 选择è¦ç¼–辑的云凭è¯ï¼Œç„¶åŽå•å‡» **â‹® > 编辑é…ç½®**。 +1. 更新凭è¯ä¿¡æ¯å¹¶å•å‡»**ä¿å­˜**。 + +**结果**：已使用新的访问凭è¯æ›´æ–°äº‘凭è¯ã€‚[添加新节点](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)时，所有使用此云凭è¯çš„现有节点模æ¿éƒ½ä¼šè‡ªåŠ¨ä½¿ç”¨æ›´æ–°çš„ä¿¡æ¯ã€‚ + +## åˆ é™¤äº‘å‡­è¯ + +如果è¦åˆ é™¤äº‘凭è¯ï¼Œäº‘凭è¯ä¸èƒ½æœ‰å…³è”的节点模æ¿ã€‚如果你无法删除云凭è¯ï¼Œè¯·[删除与该云凭è¯å…³è”的所有节点模æ¿](manage-node-templates.md#删除节点模æ¿)。 + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡»**云凭è¯**。 +1. ä½ å¯ä»¥åˆ é™¤å•ä¸ªäº‘凭è¯æˆ–进行批é‡åˆ é™¤ã€‚ + + - è¦å•ç‹¬åˆ é™¤ä¸€ä¸ªäº‘凭è¯ï¼Œè¯·é€‰æ‹©ä½ è¦ç¼–辑的云凭è¯ï¼Œç„¶åŽå•å‡» **â‹® > 删除**。 + - è¦æ‰¹é‡åˆ é™¤äº‘凭è¯ï¼Œè¯·ä»Žåˆ—表中选择一个或多个云凭è¯ã€‚å•å‡»**删除**。 +1. 确认你需è¦åˆ é™¤è¿™äº›äº‘凭è¯ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/manage-node-templates.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/manage-node-templates.md new file mode 100644 index 00000000000..5fbb8845fce --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/manage-node-templates.md @@ -0,0 +1,54 @@ +--- +title: 管ç†èŠ‚ç‚¹æ¨¡æ¿ +--- + +如果è¦é…ç½®[由基础设施æ供商托管](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)的集群，则å¯ä»¥ä½¿ç”¨[节点模æ¿](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#节点模æ¿)æ¥é…置集群节点。这些模æ¿ä½¿ç”¨ Docker Machine é…置选项æ¥å®šä¹‰èŠ‚点的æ“作系统镜åƒä»¥åŠè®¾ç½®/å‚数。你å¯ä»¥åœ¨ä¸¤ç§æƒ…况下创建节点模æ¿ï¼š + +- [é…置节点池集群](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)。 +- 在任何时间使用[用户设置](../../pages-for-subheaders/user-settings.md)。 + +创建节点模æ¿æ—¶ï¼Œå®ƒä¼šç»‘定到你的用户é…置文件。节点模æ¿ä¸èƒ½åœ¨ç”¨æˆ·ä¹‹é—´å…±äº«ã€‚ä½ å¯ä»¥ä»Žç”¨æˆ·è®¾ç½®ä¸­åˆ é™¤ä¸å†ä½¿ç”¨çš„旧节点模æ¿ã€‚ + +## åˆ›å»ºèŠ‚ç‚¹æ¨¡æ¿ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. å•å‡»**添加模æ¿**。 +1. 选择一个å¯ç”¨çš„云æ供商。然åŽæŒ‰ç…§å±å¹•ä¸Šçš„说明é…置模æ¿ã€‚ + +**结果**：模æ¿å·²é…置。你å¯ä»¥ç¨åŽåœ¨[é…置节点池集群](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)时使用该模æ¿ã€‚ + +## æ›´æ–°èŠ‚ç‚¹æ¨¡æ¿ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. 选择è¦ç¼–辑的节点模æ¿å¹¶å•å‡» **â‹® > 编辑**。 + + :::note + + 默认的 `active` [主机驱动](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md)和任何标记了 `password` 字段的主机驱动都需è¦ä½¿ç”¨[云凭è¯](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#云凭è¯)。 + + ::: + +1. 编辑所需信æ¯å¹¶å•å‡»**ä¿å­˜**。 + +**结果**：节点模æ¿å·²æ›´æ–°ã€‚添加新节点时，使用此节点模æ¿çš„所有节点池都会自动使用更新的信æ¯ã€‚ + +## å…‹éš†èŠ‚ç‚¹æ¨¡æ¿ + +通过用户设置创建新的节点模æ¿æ—¶ï¼Œå¯ä»¥å…‹éš†çŽ°æœ‰æ¨¡æ¿å¹¶å¿«é€Ÿæ›´æ–°å…¶è®¾ç½®ï¼Œè€Œæ— éœ€ä»Žå¤´å¼€å§‹åˆ›å»ºæ–°çš„模æ¿ã€‚克隆模æ¿é¿å…了为云æ供商é‡æ–°è¾“入访问密钥的麻烦。 + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. 找到è¦å…‹éš†çš„模æ¿ã€‚然åŽé€‰æ‹© **â‹® > 克隆**。 +1. 填写表å•çš„其余部分。 + +**结果**：已克隆和é…置模æ¿ã€‚ä½ å¯ä»¥ç¨åŽåœ¨[é…置节点池集群](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md)时使用该模æ¿ã€‚ + +## åˆ é™¤èŠ‚ç‚¹æ¨¡æ¿ + +ä¸å†ä½¿ç”¨èŠ‚点模æ¿æ—¶ï¼Œä½ å¯ä»¥å°†å…¶ä»Žç”¨æˆ·è®¾ç½®ä¸­åˆ é™¤ã€‚ + +1. 点击 **☰ > 集群管ç†**。 +1. å•å‡» **RKE1 é…ç½® > 节点模æ¿**。 +1. 从列表中选择一个或多个模æ¿ã€‚然åŽç‚¹å‡»**删除**。出现æ示时确认删除。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/user-preferences.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/user-preferences.md new file mode 100644 index 00000000000..96650669366 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/reference-guides/user-settings/user-preferences.md @@ -0,0 +1,62 @@ +--- +title: 用户å好 +--- + +ä½ å¯ä»¥é€šè¿‡å好设置æ¥ä¸ªæ€§åŒ–ä½ çš„ Rancher 体验。è¦æ›´æ”¹å好设置： + +1. å•å‡»å³ä¸Šè§’的用户头åƒã€‚ +1. å•å‡»**å好设置**。 + +## 语言 + +选择 Rancher UI 显示的语言。选项包括： + +- English +- 简体中文 + +## 主题 + +选择 Rancher UI 的背景颜色。如果选择**自动**ï¼ŒèƒŒæ™¯é¢œè‰²ä¼šåœ¨ä¸‹åˆ 6 点从浅色å˜ä¸ºæ·±è‰²ï¼Œç„¶åŽåœ¨æ—©ä¸Š 6 点å˜å›žæµ…色。 + +## ç™»å½•é¡µé¢ + +选择登录åŽæ˜¾ç¤ºçš„页é¢ã€‚选项包括： + +- 主页。 +- 上次访问的页é¢ã€‚ +- 你选择的特定集群。 + +## 显示设置 + +选择信æ¯çš„显示方å¼ï¼š + +- æ—¥æœŸæ ¼å¼ +- æ—¶é—´æ ¼å¼ +- æ¯é¡µè¡Œæ•° +- 在侧边æ æ˜¾ç¤ºçš„é›†ç¾¤æ•°é‡ + +## 确认设置 + +_从 v2.7.2 èµ·å¯ç”¨_ + +选择缩å‡èŠ‚点池时是å¦è¦æ±‚确认。 + +## 高级功能 + +- å¯ç”¨â€œåœ¨ API 中查看â€ã€‚ +- 显示由 Rancher 管ç†çš„ system 命å空间（ä¸è¦ç¼–辑或删除）。 +- 使用快æ·é”®ï¼ˆshift+T）æ¥åˆ‡æ¢æ·±è‰²/浅色主题。 +- éšè—所有类型æ述。 +- å¯ç”¨æ‰©å±•å¼€å‘者功能。 + +![](/img/user-preferences-2-7.png) + +## YAML 编辑器 + +- 默认 +- Emacs +- Vim + +## Helm Chart + +选择仅显示已å‘布的 Helm Chart 还是åŒæ—¶åŒ…å«é¢„å‘å¸ƒç‰ˆæœ¬ã€‚å¦‚æžœç‰ˆæœ¬ç¬¦åˆ [Semantic Versioning 2.0.0](https://semver.org/) 定义的[规范](https://semver.org/#spec-item-9)，则那么该版本为预å‘布版本。例如，è¦æ˜¾ç¤ºç‰ˆæœ¬ä¸º `0.1.3-dev.12ab4f` çš„ Helm chart，你需è¦å…ˆé€‰æ‹©`包括预å‘布版本`。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/security/security-scan/security-scan.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/security/security-scan/security-scan.md new file mode 100644 index 00000000000..32cf9e17a4c --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/security/security-scan/security-scan.md @@ -0,0 +1,5 @@ +--- +title: 安全扫æ +--- + +CIS 安全扫æ的文档已移至[此处](../../pages-for-subheaders/cis-scan-guides.md)。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/shared-files/_cluster-capabilities-table.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/shared-files/_cluster-capabilities-table.md new file mode 100644 index 00000000000..c418e1eaf54 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/shared-files/_cluster-capabilities-table.md @@ -0,0 +1,23 @@ +| æ“作 | Rancher å¯åŠ¨çš„ Kubernetes 集群 | EKSã€GKE å’Œ AKS 集群1 | 其他托管的 Kubernetes 集群 | éž EKS 或 GKE 注册集群 | +| --- | --- | ---| ---|----| +| [使用 kubectl å’Œ kubeconfig 文件æ¥è®¿é—®é›†ç¾¤](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) | ✓ | ✓ | ✓ | ✓ | +| [管ç†é›†ç¾¤æˆå‘˜](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) | ✓ | ✓ | ✓ | ✓ | +| [编辑和å‡çº§é›†ç¾¤](../pages-for-subheaders/cluster-configuration.md) | ✓ | ✓ | ✓ | ✓2 | +| [管ç†èŠ‚点](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md) | ✓ | ✓ | ✓ | ✓3 | +| [管ç†æŒä¹…å·å’Œå­˜å‚¨ç±»](../pages-for-subheaders/create-kubernetes-persistent-storage.md) | ✓ | ✓ | ✓ | ✓ | +| [管ç†é¡¹ç›®ã€å‘½å空间和工作负载](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) | ✓ | ✓ | ✓ | ✓ | +| [使用应用目录](../pages-for-subheaders/helm-charts-in-rancher.md) | ✓ | ✓ | ✓ | ✓ | +| é…置工具（[Alertsã€Notifiersã€Monitoring](../pages-for-subheaders/monitoring-and-alerting.md)ã€[Logging](../pages-for-subheaders/logging.md) å’Œ [Istio](../pages-for-subheaders/istio.md)） | ✓ | ✓ | ✓ | ✓ | +| [è¿è¡Œå®‰å…¨æ‰«æ](../pages-for-subheaders/cis-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | +| [è½®æ¢è¯ä¹¦](../how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md) | ✓ | ✓ | | | +| [备份](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md)å’Œ[æ¢å¤](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Rancher å¯åŠ¨çš„集群 | ✓ | ✓ | | ✓4 | +| [在 Rancher æ— æ³•è®¿é—®é›†ç¾¤æ—¶æ¸…ç† Kubernetes 组件](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) | ✓ | | | | +| [é…ç½® Pod 安全策略](../how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md) | ✓ | ✓ | | + +1. 注册的 EKSã€GKE å’Œ AKS 集群与从 Rancher UI 创建的 EKSã€GKE å’Œ AKS 集群的å¯ç”¨é€‰é¡¹ä¸€è‡´ã€‚ä¸åŒä¹‹å¤„是，从 Rancher UI 中删除已注册的集群åŽï¼Œé›†ç¾¤ä¸ä¼šè¢«é”€æ¯ã€‚ + +2. 无法编辑已注册的集群的集群é…置选项（[K3s å’Œ RKE2 集群](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md)除外）。 + +3. Rancher UI 为已注册的集群节点æ供了å°é”ã€æ¸…空和编辑节点的功能。 + +4. 对于使用 etcd 作为 controlplane 的注册集群，必须在 Rancher UI 之外手动创建快照以用于备份和æ¢å¤ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/shared-files/_common-ports-table.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/shared-files/_common-ports-table.md new file mode 100644 index 00000000000..b37d40dc276 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/shared-files/_common-ports-table.md @@ -0,0 +1,19 @@ +| åè®® | ç«¯å£ | æè¿° | +|:--------: |:----------------: |---------------------------------------------------------------------------------- | +| TCP | 22 | Node Driver SSH é…ç½® | +| TCP | 179 | Calico BGP ç«¯å£ | +| TCP | 2376 | Node Driver Docker daemon TLS ç«¯å£ | +| TCP | 2379 | etcd 客户端请求 | +| TCP | 2380 | etcd 对等通信 | +| UDP | 8472 | Canal/Flannel VXLAN 覆盖网络 | +| UDP | 4789 | Windows 集群中的 Flannel VXLAN 覆盖网络 | +| TCP | 8443 | Rancher webhook | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 9100 | Monitoring 从 Linux node-exporter 中抓å–æŒ‡æ ‡æ‰€éœ€çš„é»˜è®¤ç«¯å£ | +| TCP | 9443 | Rancher webhook | +| TCP | 9796 | Monitoring 从 Windows node-exporter 中抓å–æŒ‡æ ‡æ‰€éœ€çš„é»˜è®¤ç«¯å£ | +| TCP | 6783 | Weave ç«¯å£ | +| UDP | 6783-6784 | Weave UDP ç«¯å£ | +| TCP | 10250 | Metrics Server 与所有节点 API 的通信 | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | +| TCP/UDP | 30000-32767 | NodePort 端å£èŒƒå›´ | diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/general-troubleshooting.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/general-troubleshooting.md new file mode 100644 index 00000000000..01d4356c9ca --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/general-troubleshooting.md @@ -0,0 +1,44 @@ +--- +title: 一般故障排除 +--- + +本文用于帮助你解决使用 Rancher æ—¶é‡åˆ°çš„问题。 + +- [Kubernetes 组件](../pages-for-subheaders/kubernetes-components.md) + + 对以下核心 Kubernetes 集群组件进行故障排除： + * `etcd` + * `kube-apiserver` + * `kube-controller-manager` + * `kube-scheduler` + * `kubelet` + * `kube-proxy` + * `nginx-proxy` + +- [Kubernetes resources](other-troubleshooting-tips/kubernetes-resources.md) + + 本节介ç»äº†å¯¹èŠ‚点ã€Ingress Controller å’Œ Rancher Agent ç­‰ Kubernetes 资æºè¿›è¡Œæ•…障排除的选项。 + +- [网络](other-troubleshooting-tips/networking.md) + + 介ç»äº†è§£å†³ç½‘络问题的步骤。 + +- [DNS](other-troubleshooting-tips/dns.md) + + 解决集群的å称解æžé—®é¢˜ã€‚ + +- [对安装在 Kubernetes 上的 Rancher 进行故障排除](other-troubleshooting-tips/rancher-ha.md) + + 解决[安装在 Kubernetes 上的 Rancher Server](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) 的问题。 + +- [Logging](other-troubleshooting-tips/logging.md) + + 了解å¯ä»¥é…置哪些日志级别，以åŠå¦‚何é…置日志级别。 + +- [审计日志中的用户 ID 跟踪](other-troubleshooting-tips/user-id-tracking-in-audit-logs.md) + + 了解 Rancher 管ç†å‘˜å¦‚何通过外部身份æ供程åºç”¨æˆ·å从 Rancher 审计日志和 Kubernetes 审计日志中跟踪事件。 + +- [过期的 Webhook è¯ä¹¦](other-troubleshooting-tips/expired-webhook-certificate-rotation.md) + + 了解如何在æ¯å¹´åˆ°æœŸåŽè½®æ¢ Rancher webhook è¯ä¹¦å¯†é’¥ã€‚ diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md new file mode 100644 index 00000000000..6c9a8d5291a --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md @@ -0,0 +1,51 @@ +--- +title: Controlplane 节点故障排除 +--- + +本文适用于具有 `controlplane` 角色的节点。 + +## 检查 controlplane 容器是å¦æ­£åœ¨è¿è¡Œ + +具有 `controlplane` 角色的节点上å¯åŠ¨äº†ä¸‰ä¸ªå®¹å™¨ï¼š + +* `kube-apiserver` +* `kube-controller-manager` +* `kube-scheduler` + +容器的状æ€åº”该是 **Up**。**Up** åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name='kube-apiserver|kube-controller-manager|kube-scheduler' +``` + +输出示例： +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +26c7159abbcc rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-apiserver +f3d287ca4549 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-scheduler +bdf3898b8063 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-controller-manager +``` + +## controlplane 容器日志记录 + +:::note + +如果你添加了多个具有 `controlplane` 角色的节点，`kube-controller-manager` å’Œ `kube-scheduler` 会通过 leader 选举æ¥ç¡®å®š leader。åªæœ‰å½“å‰ leader 会记录执行的æ“作。有关检索当å‰çš„ leader 的更多信æ¯ï¼Œè¯·å‚阅 [Kubernetes leader 选举](../other-troubleshooting-tips/kubernetes-resources.md#kubernetes-leader-选举)。 + +::: + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs kube-apiserver +docker logs kube-controller-manager +docker logs kube-scheduler +``` + +## RKE2 Server 日志 + +如果 Rancher é…置的 RKE2 集群无法与 Rancher 通信，你å¯ä»¥åœ¨ä¸‹æ¸¸é›†ç¾¤ä¸­çš„ Server 节点上è¿è¡Œæ­¤å‘½ä»¤ï¼Œä»Žè€ŒèŽ·å– RKE2 Server 日志： + +``` +journalctl -u rke2-server -f +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md new file mode 100644 index 00000000000..72d6b470bfc --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md @@ -0,0 +1,365 @@ +--- +title: etcd 节点故障排除 +--- + +本文介ç»äº†å¯¹å…·æœ‰ `etcd` 角色的节点进行故障排除的命令和æ示。 + + +## 检查 etcd 容器是å¦æ­£åœ¨è¿è¡Œ + +etcd 容器的状æ€åº”该是 **Up**。**Up** åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name=etcd$ +``` + +输出示例： +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +605a124503b9 rancher/coreos-etcd:v3.2.18 "/usr/local/bin/et..." 2 hours ago Up 2 hours etcd +``` + +## etcd 容器日志记录 + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs etcd +``` +| 日志 | 解释 | +|-----|------------------| +| `health check for peer xxx could not connect: dial tcp IP:2380: getsockopt: connection refused` | æ— æ³•è¿žæŽ¥åˆ°ç«¯å£ 2380 上显示的地å€ã€‚检查 etcd 容器是å¦åœ¨æ˜¾ç¤ºåœ°å€çš„主机上è¿è¡Œã€‚ | +| `xxx is starting a new election at term x` | etcd 集群失去了集群仲è£æ•°é‡ï¼Œå¹¶æ­£åœ¨å°è¯•å»ºç«‹ä¸€ä¸ªæ–°çš„ leader。è¿è¡Œ etcd 的大多数节点关闭/无法访问时，å¯èƒ½ä¼šå‘生这ç§æƒ…况。 | +| `connection error: desc = "transport: Error while dialing dial tcp 0.0.0.0:2379: i/o timeout"; Reconnecting to {0.0.0.0:2379 0 }` | 主机防ç«å¢™æ­£åœ¨é˜»æ­¢ç½‘络通信。 | +| `rafthttp: request cluster ID mismatch` | 具有 etcd 实例日志 `rafthttp: request cluster ID mismatch` 的节点正在å°è¯•åŠ å…¥å·²ç»æ·»åŠ å¦ä¸€ä¸ªå¯¹ç­‰èŠ‚点（peer）的集群。你需è¦ä»Žé›†ç¾¤ä¸­åˆ é™¤è¯¥èŠ‚点，然åŽå†é‡æ–°æ·»åŠ ã€‚ | +| `rafthttp: failed to find member` | 集群状æ€ï¼ˆ`/var/lib/etcd`）包å«åŠ å…¥é›†ç¾¤çš„错误信æ¯ã€‚你需è¦ä»Žé›†ç¾¤ä¸­åˆ é™¤è¯¥èŠ‚点，清ç†çŠ¶æ€ç›®å½•ï¼Œç„¶åŽå†é‡æ–°æ·»åŠ ã€‚ | + +## etcd 集群和连接检查 + +è¿è¡Œ etcd 的主机的地å€é…置决定了 etcd 监å¬çš„地å€ã€‚如果为è¿è¡Œ etcd 的主机é…置了内部地å€ï¼Œåˆ™éœ€è¦æ˜¾å¼æŒ‡å®š `etcdctl` 的端点。如果任何命令的å“应是 `Error: context deadline exceeded`，则 etcd 实例ä¸å¥åº·ï¼ˆä»²è£ä¸¢å¤±æˆ–实例未正确加入集群）。 + +### 检查所有节点上的 etcd æˆå‘˜ + +输出应包å«å…·æœ‰ `etcd` 角色的所有节点，而且所有节点上的输出应该是相åŒçš„。 + +命令： +``` +docker exec etcd etcdctl member list +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list" +``` + +输出示例： +``` +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +``` + +### æ£€æŸ¥ç«¯ç‚¹çŠ¶æ€ + +`RAFT TERM` 的值应该是相等的，而且 `RAFT INDEX` 相差ä¸èƒ½å¤ªå¤§ã€‚ + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table +``` + +输出示例： +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | 333ef673fc4add56 | 3.2.18 | 24 MB | false | 72 | 66887 | +| https://IP:2379 | 5feed52d940ce4cf | 3.2.18 | 24 MB | true | 72 | 66887 | +| https://IP:2379 | db6b3bdb559a848d | 3.2.18 | 25 MB | false | 72 | 66887 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### 检查端点å¥åº· + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint health +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl endpoint health --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") +``` + +输出示例： +``` +https://IP:2379 is healthy: successfully committed proposal: took = 2.113189ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.649963ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.451201ms +``` + +### æ£€æŸ¥ç«¯å£ TCP/2379 上的连接 + +命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health" + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +输出示例： +``` +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +``` + +### æ£€æŸ¥ç«¯å£ TCP/2380 上的连接 + +命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +输出示例： +``` +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +``` + +## etcd å‘Šè­¦ + +etcd 会触å‘告警（例如空间ä¸è¶³æ—¶ï¼‰ã€‚ + +命令： +``` +docker exec etcd etcdctl alarm list +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +è§¦å‘ NOSPACE 告警的输出示例： +``` +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +``` + +## etcd 空间错误 + +相关的错误消æ¯æ˜¯ `etcdserver: mvcc: database space exceeded` 或 `applying raft message exceeded backend quota`。告警 `NOSPACE` 会被触å‘。 + +解决： + +- [压缩键空间](#压缩键空间) +- [对所有 etcd æˆå‘˜è¿›è¡Œç¢Žç‰‡æ•´ç†](#对所有-etcd-æˆå‘˜è¿›è¡Œç¢Žç‰‡æ•´ç†) +- [检查端点状æ€](#检查端点状æ€) +- [解除告警](#解除告警) + +### 压缩键空间 + +命令： +``` +rev=$(docker exec etcd etcdctl endpoint status --write-out json | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*') +docker exec etcd etcdctl compact "$rev" +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +rev=$(docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT endpoint status --write-out json | egrep -o '\"revision\":[0-9]*' | egrep -o '[0-9]*'") +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT compact \"$rev\"" +``` + +输出示例： +``` +compacted revision xxx +``` + +### 对所有 etcd æˆå‘˜è¿›è¡Œç¢Žç‰‡æ•´ç† + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl defrag +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl defrag --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','")" +``` + +输出示例： +``` +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +``` + +### æ£€æŸ¥ç«¯ç‚¹çŠ¶æ€ + +命令： +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table" +``` + +输出示例： +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | e973e4419737125 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | 4a509c997b26c206 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | b217e736575e9dd3 | 3.2.18 | 553 kB | true | 32 | 2449410 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### 解除告警 + +如果压缩和整ç†ç¢Žç‰‡åŽç¡®å®šæ•°æ®åº“大å°ä¸‹é™äº†ï¼Œåˆ™éœ€è¦è§£é™¤å‘Šè­¦æ¥å…许 etcd å†æ¬¡å†™å…¥ã€‚ + +命令： +``` +docker exec etcd etcdctl alarm list +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm disarm" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +输出示例： +``` +docker exec etcd etcdctl alarm list +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +## é…置日志级别 + +:::note + +你无法å†åŠ¨æ€æ›´æ”¹ etcd v3.5 或更高版本中的日志级别。 + +::: + +### etcd v3.5 åŠæ›´é«˜ç‰ˆæœ¬ + +è¦é…ç½® etcd 的日志级别，请编辑集群 YAML： + +``` +services: + etcd: + extra_args: + log-level: "debug" +``` + +### etcd v3.4 åŠæ›´æ—©ç‰ˆæœ¬ + +在早期的 etcd 版本中，你å¯ä»¥ä½¿ç”¨ API 动æ€æ›´æ”¹æ—¥å¿—级别。使用以下命令æ¥é…置调试日志： + +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +è¦å°†æ—¥å¿—级别é‡ç½®å›žé»˜è®¤å€¼ (`INFO`)，你å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤ã€‚ + +命令： +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +## etcd 内容 + +如果è¦æŸ¥çœ‹ etcd 的内容，你å¯ä»¥æŸ¥çœ‹æµäº‹ä»¶ï¼Œä¹Ÿå¯ä»¥ç›´æŽ¥æŸ¥è¯¢ etcd。详情请å‚阅以下示例。 + +### 查看æµäº‹ä»¶ + +命令： +``` +docker exec etcd etcdctl watch --prefix /registry +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT watch --prefix /registry +``` + +如果你åªæƒ³æŸ¥çœ‹å—å½±å“的键（而ä¸æ˜¯äºŒè¿›åˆ¶æ•°æ®ï¼‰ï¼Œä½ å¯ä»¥å°† `| grep -a ^/registry` 尾附到该命令æ¥è¿‡æ»¤é”®ã€‚ + +### 直接查询 etcd + +命令： +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only +``` + +如果 etcd 版本低于 3.3.x（Kubernetes 1.13.x åŠæ›´ä½Žç‰ˆæœ¬ï¼‰ä¸”添加节点时指定了 `--internal-address`，则使用以下命令： +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT get /registry --prefix=true --keys-only +``` + +ä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤æ¥å¤„ç†æ•°æ®ï¼Œä»Žè€ŒèŽ·å–æ¯ä¸ªé”®çš„计数摘è¦ï¼š + +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only | grep -v ^$ | awk -F'/' '{ if ($3 ~ /cattle.io/) {h[$3"/"$4]++} else { h[$3]++ }} END { for(k in h) print h[k], k }' | sort -nr +``` + +## æ›´æ¢ä¸å¥åº·çš„ etcd 节点 + +如果你 etcd 集群中的æŸä¸ªèŠ‚点å˜å¾—ä¸å¥åº·ï¼Œåœ¨å°†æ–°çš„ etcd 节点添加到集群之å‰ï¼Œæˆ‘们建议你修å¤æˆ–删除故障/ä¸å¥åº·çš„节点。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md new file mode 100644 index 00000000000..c2ee756591f --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md @@ -0,0 +1,68 @@ +--- +title: nginx-proxy 故障排除 +--- + +æ¯ä¸ªæ²¡æœ‰ `controlplane` 角色的节点上都部署了 `nginx-proxy` 容器。`nginx-proxy` 基于具有 `controlplane` 角色的å¯ç”¨èŠ‚点æ¥åŠ¨æ€ç”Ÿæˆ NGINX é…置，从而æ供对这些 `controlplane` 角色节点的访问。 + +## 检查容器是å¦æ­£åœ¨è¿è¡Œ + +该容器称为 `nginx-proxy`，它的状æ€åº”该是 `Up`。`Up` åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name=nginx-proxy +``` + +输出示例： + +``` +docker ps -a -f=name=nginx-proxy +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +c3e933687c0e rancher/rke-tools:v0.1.15 "nginx-proxy CP_HO..." 3 hours ago Up 3 hours nginx-proxy +``` + +## 检查生æˆçš„ NGINX é…ç½® + +生æˆçš„é…置包括了具有 `controlplane` 角色的节点的 IP 地å€ã€‚ä½ å¯ä»¥ä½¿ç”¨ä»¥ä¸‹å‘½ä»¤æ¥æ£€æŸ¥é…置： + +``` +docker exec nginx-proxy cat /etc/nginx/nginx.conf +``` + +输出示例： +``` +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + + server ip_of_controlplane_node1:6443; + + server ip_of_controlplane_node2:6443; + + } + + server { + listen 6443; + proxy_pass kube_apiserver; + proxy_timeout 30; + proxy_connect_timeout 2s; + + } + +} +``` + +## nginx-proxy 容器日志记录 + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs nginx-proxy +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md new file mode 100644 index 00000000000..ae6b283c056 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md @@ -0,0 +1,34 @@ +--- +title: Worker 节点和通用组件故障排除 +--- + +本文包括了è¿è¡Œåœ¨æ‰€æœ‰è§’色节点上的组件，因此适用于æ¯ä¸ªèŠ‚点。 + +## 检查容器是å¦æ­£åœ¨è¿è¡Œ + +具有 `worker` 角色的节点上å¯åŠ¨äº†ä¸¤ä¸ªå®¹å™¨ï¼š + +* kubelet +* kube-proxy + +容器的状æ€åº”该是 `Up`。`Up` åŽé¢æ˜¾ç¤ºçš„时间指的是容器è¿è¡Œçš„时间。 + +``` +docker ps -a -f=name='kubelet|kube-proxy' +``` + +输出示例： +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +158d0dcc33a5 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-proxy +a30717ecfb55 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kubelet +``` + +## 容器日志记录 + +容器的日志记录å¯èƒ½åŒ…å«é—®é¢˜çš„ä¿¡æ¯ã€‚ + +``` +docker logs kubelet +docker logs kube-proxy +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/dns.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/dns.md new file mode 100644 index 00000000000..c5ae0ff8872 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/dns.md @@ -0,0 +1,220 @@ +--- +title: DNS +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥é›†ç¾¤ä¸­çš„å称解æžé—®é¢˜ã€‚ + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，为 Rancher HA é…置了 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + +在è¿è¡Œ DNS 检查之å‰ï¼Œè¯·æ£€æŸ¥é›†ç¾¤çš„[默认 DNS æ供商](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#默认-dns-æ供商)，并确ä¿[覆盖网络正常è¿è¡Œ](networking.md#检查覆盖网络是å¦æ­£å¸¸è¿è¡Œ)，因为这也å¯èƒ½å¯¼è‡´ DNS 解æžï¼ˆéƒ¨åˆ†ï¼‰å¤±è´¥ã€‚ + +### 检查 DNS pod 是å¦æ­£åœ¨è¿è¡Œ + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns +``` + +使用 CoreDNS 时的示例输出： +``` +NAME READY STATUS RESTARTS AGE +coredns-799dffd9c4-6jhlz 1/1 Running 0 76m +``` + +使用 kube-dns 时的示例输出： +``` +NAME READY STATUS RESTARTS AGE +kube-dns-5fd74c7488-h6f7n 3/3 Running 0 4m13s +``` + +### 检查 DNS æœåŠ¡æ˜¯å¦æ˜¾ç¤ºæ­£ç¡®çš„ cluster-ip + +``` +kubectl -n kube-system get svc -l k8s-app=kube-dns +``` + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/kube-dns ClusterIP 10.43.0.10 53/UDP,53/TCP 4m13s +``` + +### 检查是å¦æ­£åœ¨è§£æžåŸŸå + +检查是å¦æ­£åœ¨è§£æžå†…部集群å称（在本例中为 `kubernetes.default`），`Server:` åŽé¢æ˜¾ç¤ºçš„ IP 应与 `kube-dns` æœåŠ¡çš„ `CLUSTER-IP` 一致。 + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup kubernetes.default +``` + +输出示例： +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: kubernetes.default +Address 1: 10.43.0.1 kubernetes.default.svc.cluster.local +pod "busybox" deleted +``` + +检查是å¦æ­£åœ¨è§£æžå¤–部å称（在本例中为 `www.google.com`）： + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup www.google.com +``` + +输出示例： +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: www.google.com +Address 1: 2a00:1450:4009:80b::2004 lhr35s04-in-x04.1e100.net +Address 2: 216.58.211.100 ams15s32-in-f4.1e100.net +pod "busybox" deleted +``` + +如果è¦æ£€æŸ¥æ‰€æœ‰ä¸»æœºçš„域å解æžï¼Œè¯·æ‰§è¡Œä»¥ä¸‹æ­¥éª¤ï¼š + +1. 将以下文件å¦å­˜ä¸º `ds-dnstest.yml`： + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: dnstest + spec: + selector: + matchLabels: + name: dnstest + template: + metadata: + labels: + name: dnstest + spec: + tolerations: + - operator: Exists + containers: + - image: busybox:1.28 + imagePullPolicy: Always + name: alpine + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + ``` + +2. 通过è¿è¡Œ `kubectl create -f ds-dnstest.yml` å¯åŠ¨å®ƒã€‚ +3. 等待 `kubectl rollout status ds/dnstest -w` 返回 `daemon set "dnstest" successfully rolled out`。 +4. 将环境å˜é‡ `DOMAIN` é…置为主机能够解æžçš„完全é™å®šåŸŸå（Fully Qualified Domain Name，FQDN），例如 `www.google.com`，并è¿è¡Œä»¥ä¸‹å‘½ä»¤è®©æ¯ä¸ªä¸»æœºä¸Šçš„å„个容器解æžé…置的域å（它是å•è¡Œå‘½ä»¤ï¼‰ï¼š + + ``` + export DOMAIN=www.google.com; echo "=> Start DNS resolve test"; kubectl get pods -l name=dnstest --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do kubectl exec $pod -- /bin/sh -c "nslookup $DOMAIN > /dev/null 2>&1"; RC=$?; if [ $RC -ne 0 ]; then echo $host cannot resolve $DOMAIN; fi; done; echo "=> End DNS resolve test" + ``` + +5. 完æˆæ­¤å‘½ä»¤åŽï¼Œå¦‚果一切正确，则输出如下： + + ``` + => Start DNS resolve test + => End DNS resolve test + ``` + +如果你在输出中看到错误，则表示主机无法解æžç»™å®šçš„ FQDN。 + +在以下错误输出示例中，IP 为 209.97.182.150 的主机阻止了 UDP 端å£ï¼š + +``` +=> Start DNS resolve test +command terminated with exit code 1 +209.97.182.150 cannot resolve www.google.com +=> End DNS resolve test +``` + +è¿è¡Œ `kubectl delete ds/dnstest` æ¸…ç† alpine DaemonSet。 + +### CoreDNS 相关 + +#### 检查 CoreDNS 日志记录 + +``` +kubectl -n kube-system logs -l k8s-app=kube-dns +``` + +#### 检查é…ç½® + +CoreDNS é…置存储在 `kube-system` 命å空间中 configmap çš„ `coredns` 中。 + +``` +kubectl -n kube-system get configmap coredns -o go-template={{.data.Corefile}} +``` + +#### 检查 resolv.conf 中的上游å称æœåŠ¡å™¨ + +默认情况下，é…置在主机（在 `/etc/resolv.conf` 里）上的å称æœåŠ¡å™¨ä¼šç”¨ä½œ CoreDNS 的上游å称æœåŠ¡å™¨ã€‚ä½ å¯ä»¥åœ¨ä¸»æœºä¸Šæ£€æŸ¥æ­¤æ–‡ä»¶ï¼Œæˆ–å°† `dnsPolicy` 设置为 `Default`（将继承其主机的 `/etc/resolv.conf`）并è¿è¡Œä»¥ä¸‹ Pod： + +``` +kubectl run -i --restart=Never --rm test-${RANDOM} --image=ubuntu --overrides='{"kind":"Pod", "apiVersion":"v1", "spec": {"dnsPolicy":"Default"}}' -- sh -c 'cat /etc/resolv.conf' +``` + +#### å¯ç”¨æ—¥å¿—查询 + +ä½ å¯ä»¥é€šè¿‡åœ¨ configmap `coredns` çš„ Corefile é…置中å¯ç”¨ [log plugin](https://coredns.io/plugins/log/) æ¥å¯ç”¨æ—¥å¿—查询。为此，你å¯ä»¥ä½¿ç”¨ `kubectl -n kube-system edit configmap coredns`，或è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥æ›¿æ¢é…置： + +``` +kubectl get configmap -n kube-system coredns -o json | sed -e 's_loadbalance_log\\n loadbalance_g' | kubectl apply -f - +``` + +这样，所有查询都会记入日志，并且å¯ä»¥ä½¿ç”¨[检查 CoreDNS 日志记录](#检查-coredns-日志记录)中的命令进行检查。 + +### kube-dns 相关 + +#### 检查 kubedns 容器中的上游å称æœåŠ¡å™¨ + +默认情况下，é…置在主机（在 `/etc/resolv.conf` 里）上的å称æœåŠ¡å™¨ä¼šç”¨ä½œ kube-dns 的上游å称æœåŠ¡å™¨ã€‚有时，主机会è¿è¡Œæœ¬åœ°ç¼“å­˜ DNS å称æœåŠ¡å™¨ï¼Œè¿™æ„å‘³ç€ `/etc/resolv.conf` 中的地å€å°†æŒ‡å‘ Loopback 范围（`127.0.0.0/8`）内的地å€ï¼Œè€Œå®¹å™¨å°†æ— æ³•è®¿é—®è¯¥èŒƒå›´ã€‚对于 Ubuntu 18.04，这是由 `systemd-resolved` 进行的。我们会检测 `systemd-resolved` 是å¦æ­£åœ¨è¿è¡Œï¼Œå¹¶è‡ªåŠ¨ä½¿ç”¨å…·æœ‰æ­£ç¡®ä¸Šæ¸¸å称æœåŠ¡å™¨çš„ `/etc/resolv.conf` 文件（ä½äºŽ `/run /systemd/resolve/resolv.conf`）。 + +使用以下命令检查 kubedns 容器使用的上游å称æœåŠ¡å™¨ï¼š + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do echo "Pod ${pod} on host ${host}"; kubectl -n kube-system exec $pod -c kubedns cat /etc/resolv.conf; done +``` + +输出示例： +``` +Pod kube-dns-667c7cb9dd-z4dsf on host x.x.x.x +nameserver 1.1.1.1 +nameserver 8.8.4.4 +``` + +如果输出显示 Loopback 范围（`127.0.0.0/8`ï¼‰å†…çš„åœ°å€ ï¼Œä½ å¯ä»¥é€šè¿‡ä»¥ä¸‹ä¸¤ç§æ–¹å¼è§£å†³æ­¤é—®é¢˜ï¼š + +* ç¡®ä¿åœ¨é›†ç¾¤èŠ‚点上的 `/etc/resolv.conf` 列出了正确的å称æœåŠ¡å™¨ã€‚如果需è¦äº†è§£å¦‚何进行æ“作，请å‚阅你的æ“作系统文档。请确ä¿ä½ åœ¨é…置集群之å‰æ‰§è¡Œæ­¤æ“作，或在修改åŽé‡å¯èŠ‚点。 +* 通过é…ç½® `kubelet` æ¥ä½¿ç”¨ä¸åŒçš„文件进行å称解æžï¼Œä½ å¯ä»¥ä½¿ç”¨å¦‚下的 `extra_args`（其中 `/run/resolvconf/resolv.conf` 是具有正确å称æœåŠ¡å™¨çš„文件）： + +``` +services: + kubelet: + extra_args: + resolv-conf: "/run/resolvconf/resolv.conf" +``` + +:::note + +由于 `kubelet` 在容器内è¿è¡Œï¼Œå› æ­¤ `/etc` å’Œ `/usr` 中文件的路径ä½äºŽ `kubelet` 容器内的 `/host/etc` å’Œ `/host/usr` 中。 + +::: + +请å‚阅[使用 YAML 编辑集群](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#使用-yaml-编辑集群)了解如何应用此修改。集群é…置完æˆåŽï¼Œä½ å¿…须删除 kube-dns pod 以激活 pod 中的新设置： + +``` +kubectl delete pods -n kube-system -l k8s-app=kube-dns +pod "kube-dns-5fd74c7488-6pwsf" deleted +``` + +ä½ å¯ä»¥[检查是å¦æ­£åœ¨è§£æžåŸŸå](#检查是å¦æ­£åœ¨è§£æžåŸŸå)æ¥å°è¯•å†æ¬¡è§£æžå称。 + +如果è¦æ£€æŸ¥é›†ç¾¤ä¸­çš„ kube-dns é…置（例如，检查是å¦é…置了ä¸åŒçš„上游å称æœåŠ¡å™¨ï¼‰ï¼Œä½ å¯ä»¥è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ—出 kube-dns é…置： + +``` +kubectl -n kube-system get configmap kube-dns -o go-template='{{range $key, $value := .data}}{{ $key }}{{":"}}{{ $value }}{{"\n"}}{{end}}' +``` + +输出示例： +``` +upstreamNameservers:["1.1.1.1"] +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md new file mode 100644 index 00000000000..8cb5fd34104 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md @@ -0,0 +1,28 @@ +--- +title: 过期 Webhook è¯ä¹¦è½®æ¢ +--- + +如果你的 Rancher 版本安装了 `rancher-webhook`，æŸäº›ç‰ˆæœ¬åˆ›å»ºçš„è¯ä¹¦å°†åœ¨ä¸€å¹´åŽè¿‡æœŸã€‚如果è¯ä¹¦æœªç»­è®¢ï¼Œä½ éœ€è¦è½®æ¢ä½ çš„ webhook è¯ä¹¦ã€‚ + +在 Rancher v2.6.3 åŠæ›´é«˜ç‰ˆæœ¬ä¸­ï¼Œrancher-webhook deployments å°†åœ¨åˆ°æœŸæ—¥æœŸåŽ 30 天或更短的时间内自动更新其 TLS è¯ä¹¦ã€‚如果你使用的是 v2.6.2 或更低版本，你å¯ä»¥é€šè¿‡ä¸‹é¢ä¸¤ç§æ–¹æ³•æ¥è§£å†³è¿™ä¸ªé—®é¢˜ã€‚ + +##### 1. 如果用户具有集群访问æƒé™ï¼Œè¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼š +``` +kubectl delete secret -n cattle-system cattle-webhook-tls +kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io --ignore-not-found=true rancher.cattle.io +kubectl delete pod -n cattle-system -l app=rancher-webhook +``` + +##### 2. 如果用户没有集群访问æƒé™ï¼Œä½¿ç”¨ `kubectl`： + +1. 删除 local 集群 `cattle-system` 命å空间中的 `cattle-webhook-tls` 密文。 + +2. 删除 `rancher.cattle.io` mutating webhook。 + +3. 删除 local 集群 `cattle-system` 命å空间中的 `rancher-webhook` pod。 + +:::note + +webhook è¯ä¹¦è¿‡æœŸé—®é¢˜ä¸æ­¢ç¤ºä¾‹ä¸­åˆ—出的 `cattle-webhook-tls`。你需è¦ç›¸åº”地填写你过期的è¯ä¹¦å¯†æ–‡ã€‚ + +::: \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md new file mode 100644 index 00000000000..33e7a22738d --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md @@ -0,0 +1,255 @@ +--- +title: Kubernetes èµ„æº +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥æœ€é‡è¦çš„ Kubernetes 资æºï¼Œå¹¶åº”用于 [Rancher å¯åŠ¨çš„ Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) 集群。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，为 Rancher HA é…置了 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + + +## 节点 + +### 获å–节点 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤å¹¶æ£€æŸ¥ä»¥ä¸‹å†…容： + +- 集群中的所有节点都已列出，没有缺失的节点。 +- 所有节点的状æ€éƒ½æ˜¯ **Ready**（如果未处于 **Ready** 状æ€ï¼Œè¯·è¿è¡Œ `docker logs kubelet` 检查该节点上的 `kubelet` 容器日志) +- 检查所有节点是å¦æŠ¥å‘Šäº†æ­£ç¡®çš„版本。 +- 检查 OS/Kernel/Docker 值是å¦æŒ‰é¢„期显示（此问题å¯èƒ½ä¸Žå‡çº§ OS/Kernel/Docker 相关）。 + + +``` +kubectl get nodes -o wide +``` + +输出示例： + +``` +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +controlplane-0 Ready controlplane 31m v1.13.5 138.68.188.91 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +etcd-0 Ready etcd 31m v1.13.5 138.68.180.33 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +worker-0 Ready worker 30m v1.13.5 139.59.179.88 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +``` + +### 获å–节点状况 + +è¿è¡Œä»¥ä¸‹å‘½ä»¤åˆ—出具有 [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) 的节点： + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' +``` + +è¿è¡Œä»¥ä¸‹å‘½ä»¤ï¼Œåˆ—出具有 active [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) 的节点，这些节点å¯èƒ½é˜»æ­¢æ­£å¸¸æ“作： + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{if ne .type "Ready"}}{{if eq .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{else}}{{if ne .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{": "}}{{.status}}{{"\n"}}{{end}}{{end}}{{end}}{{end}}' +``` + +输出示例： + +``` +worker-0: DiskPressure:True +``` + +## Kubernetes leader 选举 + +### Kubernetes Controller 管ç†å™¨ leader + +Leader ç”± Leader 选举确定。确定 Leader åŽï¼ŒLeader（`holderIdentity`）会ä¿å­˜åœ¨ `kube-controller-manager` 端点中（在本例中为 `controlplane-0`）。 + +``` +kubectl -n kube-system get endpoints kube-controller-manager -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +### Kubernetes 调度器 Leader + +Leader ç”± Leader 选举确定。确定 Leader åŽï¼ŒLeader（`holderIdentity`）会ä¿å­˜åœ¨ `kube-scheduler` 端点中（在本例中为 `controlplane-0`）。 + +``` +kubectl -n kube-system get endpoints kube-scheduler -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +## Ingress Controller + +默认的 Ingress Controller 是 NGINX，作为 DaemonSet 部署在 `ingress-nginx` 命å空间中。Pod 仅会调度到具有 `worker` 角色的节点。 + +检查 pod 是å¦è¿è¡Œåœ¨æ‰€æœ‰èŠ‚点上： + +``` +kubectl -n ingress-nginx get pods -o wide +``` + +输出示例： + +``` +kubectl -n ingress-nginx get pods -o wide +NAME READY STATUS RESTARTS AGE IP NODE +default-http-backend-797c5bc547-kwwlq 1/1 Running 0 17m x.x.x.x worker-1 +nginx-ingress-controller-4qd64 1/1 Running 0 14m x.x.x.x worker-1 +nginx-ingress-controller-8wxhm 1/1 Running 0 13m x.x.x.x worker-0 +``` + +如果 pod 无法è¿è¡Œï¼ˆå³çŠ¶æ€ä¸æ˜¯ **Running**，Ready 状æ€æœªæ˜¾ç¤º `1/1`ï¼Œæˆ–è€…æœ‰å¤§é‡ Restarts），请检查 pod 详细信æ¯ï¼Œæ—¥å¿—和命å空间事件。 + +### Pod è¯¦ç»†ä¿¡æ¯ + +``` +kubectl -n ingress-nginx describe pods -l app=ingress-nginx +``` + +### Pod 容器日志 + +下é¢çš„命令å¯ä»¥æ˜¾ç¤ºæ‰€æœ‰æ ‡è®°ä¸º “app=ingress-nginx†的 pod 的日志，但是由于 `kubectl logs` 命令的é™åˆ¶ï¼Œå®ƒåªä¼šæ˜¾ç¤º 10 行日志。有关详细信æ¯ï¼Œè¯·å‚阅 `kubectl logs -h` çš„ `--tail`。 + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +如果需è¦æŸ¥çœ‹å®Œæ•´çš„æ—¥å¿—ï¼Œè¯·åœ¨å‘½ä»¤ä¸­å°¾éš pod å称： + +``` +kubectl -n ingress-nginx logs +``` + +### 命å空间事件 + +``` +kubectl -n ingress-nginx get events +``` + +### 调试日志 + +è¦å¯ç”¨è°ƒè¯•æ—¥å¿—： + +``` +kubectl -n ingress-nginx patch ds nginx-ingress-controller --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--v=5"}]' +``` + +### 检查é…ç½® + +在æ¯ä¸ª pod 中检索生æˆçš„é…置： + +``` +kubectl -n ingress-nginx get pods -l app=ingress-nginx --no-headers -o custom-columns=.NAME:.metadata.name | while read pod; do kubectl -n ingress-nginx exec $pod -- cat /etc/nginx/nginx.conf; done +``` + +## Rancher Agents + +Rancher Agent 用于实现与集群的通信（通过 `cattle-cluster-agent` çš„ Kubernetes API）和与节点的通信（通过 `cattle-node-agent` 的集群é…置）。 + +#### cattle-node-agent + +检查æ¯ä¸ªèŠ‚点上是å¦å­˜åœ¨ cattle-node-agent pod，状æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +检查特定或所有 cattle-node-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +检查 cattle-cluster-agent pod 是å¦å­˜åœ¨äºŽé›†ç¾¤ä¸­ï¼ŒçŠ¶æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +检查 cattle-cluster-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` + +## Jobs å’Œ Pods + +### 检查 Pod 或 Job 的状æ€æ˜¯å¦ä¸º **Running**/**Completed** + +è¿è¡Œä»¥ä¸‹å‘½ä»¤è¿›è¡Œæ£€æŸ¥ï¼š + +``` +kubectl get pods --all-namespaces +``` + +如果 Pod 的状æ€ä¸æ˜¯ **Running**，你å¯ä»¥é€šè¿‡è¿è¡Œå‘½ä»¤æ¥æ‰¾åˆ°æ ¹æœ¬åŽŸå› ã€‚ + +### æè¿° Pod + +``` +kubectl describe pod POD_NAME -n NAMESPACE +``` + +### Pod 容器日志 + +``` +kubectl logs POD_NAME -n NAMESPACE +``` + +如果 Job 的状æ€ä¸æ˜¯ **Completed**，你å¯ä»¥é€šè¿‡è¿è¡Œå‘½ä»¤æ¥æ‰¾åˆ°æ ¹æœ¬åŽŸå› ã€‚ + +### æè¿° Job + +``` +kubectl describe job JOB_NAME -n NAMESPACE +``` + +### Job Pod 容器的日志 + +``` +kubectl logs -l job-name=JOB_NAME -n NAMESPACE +``` + +### 驱é€çš„ Pod + +å¯ä»¥æ ¹æ® [eviction ä¿¡å·](https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/#eviction-policy)æ¥é©±é€ Pod。 + +检索被驱é€çš„ Pod 列表（podname 和命å空间）： + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' +``` + +è¦åˆ é™¤æ‰€æœ‰è¢«é©±é€çš„ pod： + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace delete pod $epod; done +``` + +检索被驱é€çš„ pod 列表ã€è°ƒåº¦èŠ‚点以åŠåŽŸå› ï¼š + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace get pod $epod -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName,MSG:.status.message; done +``` + +### Job æœªå®Œæˆ + +如果你å¯ç”¨äº† Istio 而且你部署的 Job 未完æˆï¼Œä½ éœ€è¦æŒ‰ç…§[这些步骤](../../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md)将注释添加到 Pod 中。 + +由于 Istio Sidecars 会一直è¿è¡Œï¼Œå› æ­¤å³ä½¿ä»»åŠ¡å®Œæˆäº†ï¼Œä¹Ÿä¸èƒ½è®¤ä¸º Job 已完æˆã€‚这是一个临时的解决方法，它ç¦æ­¢äº† Istio 和添加了注释的 Pod 之间的通信。如果你使用了这个方法，由于这个 Job 无法访问æœåŠ¡ç½‘格，因此你将ä¸èƒ½ç»§ç»­ä½¿ç”¨ Job 进行集æˆæµ‹è¯•ã€‚ \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/logging.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/logging.md new file mode 100644 index 00000000000..7b74521fbb9 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/logging.md @@ -0,0 +1,72 @@ +--- +title: Logging +--- + +## 日志级别 + +Rancher 使用了以下日志级别： + +| å称 | æè¿° | +|---------|-------------| +| `info` | 记录信æ¯æ€§æ¶ˆæ¯ã€‚这是默认的日志级别。 | +| `debug` | 记录å¯ç”¨äºŽè°ƒè¯•çš„更详细消æ¯ã€‚ | +| `trace` | 记录内部功能的éžå¸¸è¯¦ç»†çš„消æ¯ã€‚éžå¸¸å†—长，并且å¯èƒ½åŒ…å«æ•æ„Ÿä¿¡æ¯ã€‚ | + +### 如何é…置日志级别 + +#### Kubernetes 安装 + +* é…ç½® debug 日志级别 + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set debug; done +OK +OK +OK +$ kubectl -n cattle-system logs -l app=rancher -c rancher +``` + +* é…ç½® info 日志级别 + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set info; done +OK +OK +OK +``` + +#### Docker 安装 + +* é…ç½® debug 日志级别 + +``` +$ docker exec -ti loglevel --set debug +OK +$ docker logs -f +``` + +* é…ç½® info 日志级别 + +``` +$ docker exec -ti loglevel --set info +OK +``` + +## Rancher 主机调试日志 +如果需è¦è§£å†³åœ¨åŸºç¡€è®¾æ–½æ供商中创建对象的问题，你å¯ä»¥ä½¿ç”¨ `rancher-machine` 调试日志。 + +ä½ å¯ä»¥åœ¨å¯åŠ¨ Rancher 时设置环境å˜é‡æ¥ä¸º `rancher-machine` å¯ç”¨è°ƒè¯•æ—¥å¿—。 + +`CATTLE_WHITELIST_ENVVARS` 环境å˜é‡å…许用户将特定环境å˜é‡åˆ—入白åå•ï¼Œè®©å®ƒä»¬èƒ½åœ¨é…置期间传递给 `rancher-machine`。 + +`MACHINE_DEBUG` å˜é‡åœ¨ `rancher-machine` 中å¯ç”¨è°ƒè¯•æ—¥å¿—。 + +因此，通过设置 `MACHINE_DEBUG=true` 并将 `MACHINE_DEBUG` 添加到 `CATTLE_WHITELIST_ENVVARS` 中的å˜é‡é»˜è®¤åˆ—表（例如 `CATTLE_WHITELIST_ENVVARS=HTTP_PROXY,HTTPS_PROXY,NO_PROXY,MACHINE_DEBUG`），你å¯ä»¥åœ¨é…ç½® RKE1ã€RKE2 å’Œ K3s 集群时在 `rancher-machine` 中å¯ç”¨è°ƒè¯•æ—¥å¿—。 + +:::caution + +åƒä¸Šé¢çš„ `trace` 日志级别一样，`rancher-machine` 调试日志å¯ä»¥åŒ…å«æ•æ„Ÿä¿¡æ¯ã€‚ + +::: diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/networking.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/networking.md new file mode 100644 index 00000000000..47c806b0278 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/networking.md @@ -0,0 +1,121 @@ +--- +title: 网络 +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥é›†ç¾¤ä¸­çš„网络问题。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，为 Rancher HA é…置了 `export KUBECONFIG=$PWD/kube_config_cluster.yml`）或通过 UI ä½¿ç”¨äº†åµŒå…¥å¼ kubectl。 + +### 仔细检查你的（主机）防ç«å¢™ä¸­æ˜¯å¦æ‰“å¼€äº†æ‰€æœ‰å¿…éœ€çš„ç«¯å£ + +仔细检查你的（主机）防ç«å¢™ä¸­æ˜¯å¦æ‰“开了所有[必需的端å£](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。其他所需的端å£éƒ½ä½¿ç”¨ TCP，而覆盖网络使用 UDP。 +### 检查覆盖网络是å¦æ­£å¸¸è¿è¡Œ + +ä½ å¯ä»¥å°† Pod 调度到集群中的任何主机，但是 NGINX Ingress Controller 需è¦èƒ½å¤Ÿå°† `NODE_1` 请求路由到 `NODE_2`。这会在覆盖网络上进行。如果覆盖网络无法正常工作，由于 NGINX Ingress Controller 无法路由到 pod，因此 TCP/HTTP 连接会间歇性失败。 + +è¦æµ‹è¯•è¦†ç›–网络，你å¯ä»¥å¯åŠ¨ä»¥ä¸‹ `DaemonSet` 定义。这样，æ¯å°ä¸»æœºä¸Šéƒ½ä¼šè¿è¡Œä¸€ä¸ª `swiss-army-knife` 容器（镜åƒç”± Rancher 工程师开å‘，å¯åœ¨[此处](https://github.com/rancherlabs/swiss-army-knife)找到），我们使用该镜åƒåœ¨æ‰€æœ‰ä¸»æœºä¸Šçš„容器之间è¿è¡Œ `ping` 测试。 + +:::note + +此容器[ä¸æ”¯æŒ ARM 节点](https://github.com/leodotcloud/swiss-army-knife/issues/18)（例如 Raspberry Pi）。Pod 日志会显示为 `exec user process caused: exec format error`。 + +::: + +1. 将以下内容å¦å­˜ä¸º `overlaytest.yml`： + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: overlaytest + spec: + selector: + matchLabels: + name: overlaytest + template: + metadata: + labels: + name: overlaytest + spec: + tolerations: + - operator: Exists + containers: + - image: rancherlabs/swiss-army-knife + imagePullPolicy: Always + name: overlaytest + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + + ``` + +2. 使用 `kubectl create -f overlaytest.yml` å¯åŠ¨å®ƒã€‚ +3. 等待 `kubectl rollout status ds/overlaytest -w` 返回 `daemon set "overlaytest" successfully rolled out`。 +4. 从åŒä¸€ä½ç½®è¿è¡Œä»¥ä¸‹è„šæœ¬ã€‚它会让æ¯ä¸ªä¸»æœºä¸Šçš„所有 `overlaytest` 容器相互 ping 通： + ``` + #!/bin/bash + echo "=> Start network overlay test" + kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.metadata.name}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read spod shost + do kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.status.podIP}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read tip thost + do kubectl --request-timeout='10s' exec $spod -c overlaytest -- /bin/sh -c "ping -c2 $tip > /dev/null 2>&1" + RC=$? + if [ $RC -ne 0 ] + then echo FAIL: $spod on $shost cannot reach pod IP $tip on $thost + else echo $shost can reach $thost + fi + done + done + echo "=> End network overlay test" + ``` + +5. 命令è¿è¡Œå®ŒæˆåŽï¼Œå®ƒä¼šè¾“出æ¯æ¡è·¯ç”±çš„状æ€ï¼š + + ``` + => Start network overlay test + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.7.3 on wk2 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.0.5 on cp1 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.2.12 on wk1 + command terminated with exit code 1 + FAIL: overlaytest-v4qkl on cp1 cannot reach pod IP 10.42.7.3 on wk2 + cp1 can reach cp1 + cp1 can reach wk1 + command terminated with exit code 1 + FAIL: overlaytest-xpxwp on wk1 cannot reach pod IP 10.42.7.3 on wk2 + wk1 can reach cp1 + wk1 can reach wk1 + => End network overlay test + ``` + 如果你在输出中看到错误，则说明两å°ä¸»æœºä¸Šçš„ Pod 路由存在问题。在上é¢çš„输出中，节点 `wk2` 在覆盖网络上没有连接。原因å¯èƒ½æ˜¯æ²¡æœ‰ä¸º `wk2` 打开覆盖网络的[必需端å£](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#网络è¦æ±‚)。 +6. 你现在å¯ä»¥é€šè¿‡è¿è¡Œ `kubectl delete ds/overlaytest` æ¥æ¸…ç† DaemonSet。 + + +### 检查主机和对等/隧é“设备/设备上的 MTU 是å¦æ­£ç¡®é…ç½® + +如果 MTU 在è¿è¡Œ Rancher 的主机ã€åˆ›å»º/导入集群中的节点或两者之间的设备上é…ç½®ä¸æ­£ç¡®ï¼ŒRancher å’Œ Agent 会记录类似以下的错误信æ¯ï¼š + +* `websocket: bad handshake` +* `Failed to connect to proxy` +* `read tcp: i/o timeout` + +有关在 Rancher 和集群节点之间使用 Google Cloud VPN 时如何正确é…ç½® MTU 的示例，请å‚阅 [Google Cloud VPN：MTU 注æ„事项](https://cloud.google.com/vpn/docs/concepts/mtu-considerations#gateway_mtu_vs_system_mtu)。 + +### 已解决的问题 + +#### 由于缺少节点注释，使用 Canal/Flannel 时覆盖网络中断 + +| | | +|------------|------------| +| GitHub issue | [#13644](https://github.com/rancher/rancher/issues/13644) | +| 解决于 | v2.1.2 | + +è¦æ£€æŸ¥ä½ çš„集群是å¦å—到影å“，è¿è¡Œä»¥ä¸‹å‘½ä»¤æ¥åˆ—出æŸå的节点（此命令è¦æ±‚安装 `jq`）： + +``` +kubectl get nodes -o json | jq '.items[].metadata | select(.annotations["flannel.alpha.coreos.com/public-ip"] == null or .annotations["flannel.alpha.coreos.com/kube-subnet-manager"] == null or .annotations["flannel.alpha.coreos.com/backend-type"] == null or .annotations["flannel.alpha.coreos.com/backend-data"] == null) | .name' +``` + +如果没有输出，则集群没有影å“。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/rancher-ha.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/rancher-ha.md new file mode 100644 index 00000000000..630f4f13de6 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/rancher-ha.md @@ -0,0 +1,79 @@ +--- +title: Rancher HA +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥ä½ çš„ Rancher Kubernetes 安装。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，`export KUBECONFIG=$PWD/kube_config_cluster.yml`）。 + +### 检查 Rancher Pod + +Rancher pod 会部署为 `cattle-system` 命å空间中的一个 Deployment。 + +检查 pod 是å¦è¿è¡Œåœ¨æ‰€æœ‰èŠ‚点上： + +``` +kubectl -n cattle-system get pods -l app=rancher -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +rancher-7dbd7875f7-n6t5t 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qbj5k 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qw7wb 1/1 Running 0 8m x.x.x.x x.x.x.x +``` + +如果 pod 无法è¿è¡Œï¼ˆå³çŠ¶æ€ä¸æ˜¯ **Running**，Ready 状æ€æœªæ˜¾ç¤º `1/1`ï¼Œæˆ–è€…æœ‰å¤§é‡ Restarts），请检查 pod 详细信æ¯ï¼Œæ—¥å¿—和命å空间事件。 + +#### Pod è¯¦ç»†ä¿¡æ¯ + +``` +kubectl -n cattle-system describe pods -l app=rancher +``` + +#### Pod 容器日志 + +``` +kubectl -n cattle-system logs -l app=rancher +``` + +#### 命å空间事件 + +``` +kubectl -n cattle-system get events +``` + +### 检查 Ingress + +Ingress 应该具有正确的 `HOSTS`（显示é…置的 FQDN）和 `ADDRESS`（将被路由到该主机地å€ï¼‰ï¼š + +``` +kubectl -n cattle-system get ingress +``` + +输出示例： + +``` +NAME HOSTS ADDRESS PORTS AGE +rancher rancher.yourdomain.com x.x.x.x,x.x.x.x,x.x.x.x 80, 443 2m +``` + +### 检查 Ingress Controller 日志 + +如果访问你é…置的 Rancher FQDN 时没有显示 UI，请检查 Ingress Controller 日志以查看å°è¯•è®¿é—® Rancher æ—¶å‘生了什么： + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +### Leader 选举 + +Leader ç”± Leader 选举确定。确定 Leader åŽï¼ŒLeader（`holderIdentity`）会ä¿å­˜åœ¨ `cattle-controllers` ConfigMap 中（在本例中为 `rancher-7dbd7875f7-qbj5k`）： + +``` +kubectl -n kube-system get configmap cattle-controllers -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"rancher-7dbd7875f7-qbj5k","leaseDurationSeconds":45,"acquireTime":"2019-04-04T11:53:12Z","renewTime":"2019-04-04T12:24:08Z","leaderTransitions":0} +``` + diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/registered-clusters.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/registered-clusters.md new file mode 100644 index 00000000000..58eff6a97ed --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/registered-clusters.md @@ -0,0 +1,67 @@ +--- +title: 注册集群 +--- + +本文列出的命令/步骤å¯ç”¨äºŽæ£€æŸ¥ä½ æ­£åœ¨æ³¨å†Œæˆ–已在 Rancher 中注册的集群。 + +请确ä¿ä½ é…置了正确的 kubeconfig（例如，`export KUBECONFIG=$PWD/kubeconfig_from_imported_cluster.yml`）。 + +### Rancher Agents + +Rancher Agent 用于实现与集群的通信（通过 cattle-cluster-agent çš„ Kubernetes API）和与节点的通信。 + +如果 cattle-cluster-agent 无法连接到é…置的 `server-url`，集群将ä¿æŒåœ¨ **Pending** 状æ€å¹¶æ˜¾ç¤º `Waiting for full cluster configuration`。 + +#### cattle-node-agent + +:::note + +cattle-node-agents åªä¼šåœ¨ä½¿ç”¨ RKE 在 Rancher 创建的集群中显示。 + +::: + +检查æ¯ä¸ªèŠ‚点上是å¦å­˜åœ¨ cattle-node-agent pod，状æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +检查特定或所有 cattle-node-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +检查 cattle-cluster-agent pod 是å¦å­˜åœ¨äºŽé›†ç¾¤ä¸­ï¼ŒçŠ¶æ€æ˜¯å¦ä¸º **Running**，并且é‡å¯æ¬¡æ•°ä¸å¤šï¼š + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +输出示例： + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +检查 cattle-cluster-agent pod 的日志记录： + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md new file mode 100644 index 00000000000..bdbac691333 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md @@ -0,0 +1,22 @@ +--- +title: 审计日志中的用户 ID 跟踪 +--- + +Rancher 使用以下审计日志æ¥è·Ÿè¸ªæœ¬åœ°å’Œä¸‹æ¸¸é›†ç¾¤ä¸­å‘生的事件： + +* [Kubernetes 审计日志](https://rancher.com/docs/rke/latest/en/config-options/audit-log/) +* [Rancher API 审核日志](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) + +Rancher 2.6 增强了审计日志，在 Rancher 和下游 Kubernetes 审计日志中都包å«äº†å¤–部身份æ供程åºå称（å³å¤–部身份æ供程åºä¸­ç”¨æˆ·çš„通用å称）。 + +在 Rancher 2.6 之å‰ï¼Œå¦‚果管ç†å‘˜ä¸çŸ¥é“外部身份æ供程åºä¸­çš„用户åå’Œ Rancher 中的 userId (`u-xXXX`) 之间的映射，则无法通过 Rancher 审计日志和 Kubernetes 审计日志æ¥è¿½è¸ªäº‹ä»¶ã€‚ +è¦çŸ¥é“这个映射，集群管ç†å‘˜éœ€è¦èƒ½å¤Ÿè®¿é—® Rancher APIã€UI å’Œ local 管ç†é›†ç¾¤ã€‚ + +有了这个功能之åŽï¼Œä¸‹æ¸¸é›†ç¾¤ç®¡ç†å‘˜å°±èƒ½å¤ŸæŸ¥çœ‹ Kubernetes 审计日志，并且å¯ä»¥ä¸åœ¨ Rancher 中查看任何内容就能知é“哪个外部身份æä¾›ç¨‹åº (IDP) 用户执行了æŸé¡¹æ“作。 +如果集群传出了审计日志，日志系统的用户就能够识别外部身份æ供程åºä¸­çš„用户。 +Rancher 管ç†å‘˜çŽ°åœ¨èƒ½å¤ŸæŸ¥çœ‹ Rancher 审计日志，并使用外部身份æ供程åºç”¨æˆ·åæ¥è·Ÿè¸ª Kubernetes 审计日志。 + +### 功能æè¿° + +- 在下游集群å¯ç”¨äº† Kubernetes 审计日志åŽï¼Œä¼šåœ¨â€œå…ƒæ•°æ®â€çº§åˆ«ä¸ºæ¯ä¸ªè¯·æ±‚记录外部身份æ供程åºçš„用户å。 +- 在 Rancher å¯ç”¨ Rancher API 审计日志åŽï¼Œæ¯ä¸ªåˆ°è¾¾ Rancher API Server 的请求记录（包括登录请求）对应的外部身份æ供程åºç”¨æˆ·å也会记录在 `auditLog.level=1` 中。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-latest/v2.7.md b/i18n/zh/docusaurus-plugin-content-docs/version-latest/v2.7.md new file mode 100644 index 00000000000..4e36ad8730b --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-latest/v2.7.md @@ -0,0 +1,16 @@ +--- +title: v2.7 +description: Dummy file used to redirect to the base url +--- + + + +import {Redirect} from '@docusaurus/router'; + +const Home = () => { +return ; +}; \ No newline at end of file diff --git a/package.json b/package.json index 736dfa216f7..60696343f2c 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "scripts": { "docusaurus": "docusaurus", - "start": "docusaurus start", + "start": "NODE_OPTIONS='--max-old-space-size=3096' docusaurus start", "build": "docusaurus build", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", diff --git a/sidebars.js b/sidebars.js index d9c5f34deb4..b4e164c5130 100644 --- a/sidebars.js +++ b/sidebars.js @@ -1069,8 +1069,8 @@ const sidebars = { }, items: [ "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23", - "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24", - "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", ], }, { @@ -1082,8 +1082,8 @@ const sidebars = { }, items: [ "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23", - "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24", - "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", ], }, { @@ -1095,8 +1095,8 @@ const sidebars = { }, items: [ "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23", - "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24", - "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27", ], }, ], @@ -1116,6 +1116,7 @@ const sidebars = { "reference-guides/rancher-security/kubernetes-security-best-practices", "reference-guides/rancher-security/security-advisories-and-cves", "reference-guides/rancher-security/psa-restricted-exemptions", + "reference-guides/rancher-security/rancher-webhook-hardening", ], } ] diff --git a/static/img/backup-container.png b/static/img/backup-container.png deleted file mode 100644 index e1fce01bd2c..00000000000 Binary files a/static/img/backup-container.png and /dev/null differ diff --git a/static/img/click-register-new-app.png b/static/img/click-register-new-app.png deleted file mode 100644 index ac467965d19..00000000000 Binary files a/static/img/click-register-new-app.png and /dev/null differ diff --git a/static/img/create-backup-tarball.png b/static/img/create-backup-tarball.png deleted file mode 100644 index e659d50f351..00000000000 Binary files a/static/img/create-backup-tarball.png and /dev/null differ diff --git a/static/img/default-grafana-dashboards.png b/static/img/default-grafana-dashboards.png deleted file mode 100644 index 9918dae8236..00000000000 Binary files a/static/img/default-grafana-dashboards.png and /dev/null differ diff --git a/static/img/dir-backup-tarball-clear.png b/static/img/dir-backup-tarball-clear.png deleted file mode 100644 index 35395285fdf..00000000000 Binary files a/static/img/dir-backup-tarball-clear.png and /dev/null differ diff --git a/static/img/dir-backup-tarball.png b/static/img/dir-backup-tarball.png deleted file mode 100644 index eeb2a4687e6..00000000000 Binary files a/static/img/dir-backup-tarball.png and /dev/null differ diff --git a/static/img/docker-container-ps-output.png b/static/img/docker-container-ps-output.png deleted file mode 100644 index edd911fb566..00000000000 Binary files a/static/img/docker-container-ps-output.png and /dev/null differ diff --git a/static/img/no-ingress.png b/static/img/no-ingress.png deleted file mode 100644 index fa1a0a0fdd2..00000000000 Binary files a/static/img/no-ingress.png and /dev/null differ diff --git a/static/img/persistent-volume.png b/static/img/persistent-volume.png deleted file mode 100644 index 327da071bf2..00000000000 Binary files a/static/img/persistent-volume.png and /dev/null differ diff --git a/static/img/rancher_overview_2.png b/static/img/rancher_overview_2.png deleted file mode 100644 index 00ce8eb2c27..00000000000 Binary files a/static/img/rancher_overview_2.png and /dev/null differ diff --git a/static/img/rke-etcd-backup.png b/static/img/rke-etcd-backup.png deleted file mode 100644 index b40c550618d..00000000000 Binary files a/static/img/rke-etcd-backup.png and /dev/null differ diff --git a/static/img/stop-rancher-container.gif b/static/img/stop-rancher-container.gif deleted file mode 100644 index 92468116973..00000000000 Binary files a/static/img/stop-rancher-container.gif and /dev/null differ diff --git a/versioned_docs/version-2.0-2.4/backups/backup/backup.md b/versioned_docs/version-2.0-2.4/backups/backup/backup.md deleted file mode 100644 index 065e8159532..00000000000 --- a/versioned_docs/version-2.0-2.4/backups/backup/backup.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: Backup ---- - - - - - -This section contains information about how to create backups of your Rancher data and how to restore them in a disaster scenario. - - - Rancher server backups: - - [Rancher installed on a K3s Kubernetes cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-k3s-installed-rancher.md) - - [Rancher installed on an RKE Kubernetes cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md) - - [Rancher installed with Docker](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) - -For information on backing up Rancher launched Kubernetes clusters, refer to [this section.](../../how-to-guides/advanced-user-guides/manage-clusters/backing-up-etcd.md) - -If you are looking to back up your [Rancher launched Kubernetes cluster](../../pages-for-subheaders/launch-kubernetes-with-rancher.md), please refer [here](../../how-to-guides/advanced-user-guides/manage-clusters/backing-up-etcd.md). diff --git a/versioned_docs/version-2.0-2.4/backups/restore/restore.md b/versioned_docs/version-2.0-2.4/backups/restore/restore.md deleted file mode 100644 index 7467b146f48..00000000000 --- a/versioned_docs/version-2.0-2.4/backups/restore/restore.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Restore ---- - -If you lose the data on your Rancher Server, you can restore it if you have backups stored in a safe location. - -- [Restoring backups for Rancher installed with Docker](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) -- [Restoring backups for Rancher installed on an RKE Kubernetes cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) -- [Restoring backups for Rancher installed on a K3s Kubernetes cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-k3s-installed-rancher.md) - -If you are looking to restore your [Rancher launched Kubernetes cluster](../../pages-for-subheaders/launch-kubernetes-with-rancher.md), please refer to [this section](../../how-to-guides/advanced-user-guides/manage-clusters/restoring-etcd.md). diff --git a/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/creating-apps.md b/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/creating-apps.md index 270d7674c0c..beeea68d679 100644 --- a/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/creating-apps.md +++ b/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/creating-apps.md @@ -65,7 +65,6 @@ Before you create your own custom catalog, you should have a basic understanding A file that contains questions for a form. These form questions simplify deployment of a chart. Without it, you must configure the deployment using key value pairs, which is more difficult. The following image displays the difference between a Rancher chart (which includes `questions.yml`) and a native Helm chart (which does not). -
Rancher Chart with questions.yml (left) vs. Helm Chart without (right)
![questions.yml](/img/questions.png) diff --git a/versioned_docs/version-2.0-2.4/reference-guides/about-the-api/api-tokens.md b/versioned_docs/version-2.0-2.4/reference-guides/about-the-api/api-tokens.md index 8e6a1a9b8c1..64e450463e2 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/about-the-api/api-tokens.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/about-the-api/api-tokens.md @@ -3,7 +3,7 @@ title: API Tokens --- - + By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. diff --git a/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/kubectl-utility.md b/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/kubectl-utility.md index 003aea86dba..255b8ad4ac1 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/kubectl-utility.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/kubectl-utility.md @@ -3,7 +3,7 @@ title: kubectl Utility --- - + ## kubectl diff --git a/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/rancher-cli.md b/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/rancher-cli.md index c0850617aff..9b50fd35f2e 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/rancher-cli.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/cli-with-rancher/rancher-cli.md @@ -4,7 +4,7 @@ description: Interact with Rancher using command line interface (CLI) tools from --- - + The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. diff --git a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md index 4997518b135..828c9ca1ce8 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Node Template Configuration --- - + For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md index 3c969b1d06e..d8c0f3dcdab 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Node Template Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md index 22fd6532146..24bfabe753a 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Node Template Configuration --- - + diff --git a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md index a0e092fee87..3c363d053cf 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -3,7 +3,7 @@ title: Rancher Agent Options --- - + Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. diff --git a/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/architecture-recommendations.md index 36794f75b40..b7fc1fc46a0 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -2,6 +2,10 @@ title: Architecture Recommendations --- + + + + Kubernetes cluster. If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the cluster running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) ## Separation of Rancher and User Clusters diff --git a/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index fabdcd6f8b9..e45c9e17140 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -2,6 +2,10 @@ title: Communicating with Downstream User Clusters --- + + + + This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. diff --git a/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/rancher-server-and-components.md index 6bf86ee8d7c..7c22dba7b10 100644 --- a/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/rancher-server-and-components.md +++ b/versioned_docs/version-2.0-2.4/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -2,6 +2,10 @@ title: Rancher Server and Components --- + + + + The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). diff --git a/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.0/rancher-v2.3.0.md b/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.0/rancher-v2.3.0.md deleted file mode 100644 index a5d9ae8ec66..00000000000 --- a/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.0/rancher-v2.3.0.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: Rancher v2.3.0 ---- - -### Self Assessment Guide - -This [guide](../../../reference-guides/rancher-security/rancher-v2.3-hardening-guides/rancher-v2.3.0-self-assessment-guide-with-cis-v1.4.1-benchmark.md) corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark: - -Self Assessment Guide Version | Rancher Version | Hardening Guide Version | Kubernetes Version | CIS Benchmark Version ----------------------------|----------|---------|-------|----- -Self Assessment Guide v2.3 | Rancher v2.3.0-2.3.2 | Hardening Guide v2.3 | Kubernetes 1.15 | Benchmark v1.4.1 - -### Hardening Guide - -This hardening [guide](../../../reference-guides/rancher-security/rancher-v2.3-hardening-guides/rancher-v2.3.0-hardening-guide-with-cis-v1.4.1-benchmark.md) is intended to be used with specific versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -Hardening Guide Version | Rancher Version | CIS Benchmark Version | Kubernetes Version -------------------------|----------------|-----------------------|------------------ -Hardening Guide v2.3 | Rancher v2.3.0-v2.3.2 | Benchmark v1.4.1 | Kubernetes 1.15 \ No newline at end of file diff --git a/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.3/rancher-v2.3.3.md b/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.3/rancher-v2.3.3.md deleted file mode 100644 index c353c8a9a99..00000000000 --- a/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.3/rancher-v2.3.3.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: Rancher v2.3.3 ---- - -### Self Assessment Guide - -This [guide](../../../reference-guides/rancher-security/rancher-v2.3-hardening-guides/rancher-v2.3.3-self-assessment-guide-with-cis-v1.4.1-benchmark.md) corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark: - -Self Assessment Guide Version | Rancher Version | Hardening Guide Version | Kubernetes Version | CIS Benchmark Version ----------------------------|----------|---------|-------|----- -Self Assessment Guide v2.3.3 | Rancher v2.3.3 | Hardening Guide v2.3.3 | Kubernetes v1.16 | Benchmark v1.4.1 - -### Hardening Guide - -This hardening [guide](../../../reference-guides/rancher-security/rancher-v2.3-hardening-guides/rancher-v2.3.3-hardening-guide-with-cis-v1.4.1-benchmark.md) is intended to be used with specific versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -Hardening Guide Version | Rancher Version | CIS Benchmark Version | Kubernetes Version -------------------------|----------------|-----------------------|------------------ -Hardening Guide v2.3.3 | Rancher v2.3.3 | Benchmark v1.4.1 | Kubernetes 1.14, 1.15, and 1.16 \ No newline at end of file diff --git a/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.5/rancher-v2.3.5.md b/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.5/rancher-v2.3.5.md deleted file mode 100644 index cf519162c6c..00000000000 --- a/versioned_docs/version-2.0-2.4/security/rancher-2.3.x/rancher-v2.3.5/rancher-v2.3.5.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: Rancher v2.3.5 ---- - -### Self Assessment Guide - -This [guide](../../../reference-guides/rancher-security/rancher-v2.3-hardening-guides/rancher-v2.3.5-self-assessment-guide-with-cis-v1.5-benchmark.md) corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark: - -Self Assessment Guide Version | Rancher Version | Hardening Guide Version | Kubernetes Version | CIS Benchmark Version ----------------------------|----------|---------|-------|----- -Self Assessment Guide v2.3.5 | Rancher v2.3.5 | Hardening Guide v2.3.5 | Kubernetes v1.15 | Benchmark v1.5 - -### Hardening Guide - -This hardening [guide](../../../reference-guides/rancher-security/rancher-v2.3-hardening-guides/rancher-v2.3.5-hardening-guide-with-cis-v1.5-benchmark.md) is intended to be used with specific versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: - -Hardening Guide Version | Rancher Version | CIS Benchmark Version | Kubernetes Version -------------------------|----------------|-----------------------|------------------ -Hardening Guide v2.3.5 | Rancher v2.3.5 | Benchmark v1.5 | Kubernetes 1.15 \ No newline at end of file diff --git a/versioned_docs/version-2.5/backups/docker-installs/docker-installs.md b/versioned_docs/version-2.5/backups/docker-installs/docker-installs.md deleted file mode 100644 index cc211cee5e0..00000000000 --- a/versioned_docs/version-2.5/backups/docker-installs/docker-installs.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Backup and Restore for Rancher Installed with Docker ---- - -- [Backups](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) -- [Restores](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) \ No newline at end of file diff --git a/versioned_docs/version-2.5/cluster-provisioning/rke-clusters/options/options.md b/versioned_docs/version-2.5/cluster-provisioning/rke-clusters/options/options.md deleted file mode 100644 index 9fc981415b2..00000000000 --- a/versioned_docs/version-2.5/cluster-provisioning/rke-clusters/options/options.md +++ /dev/null @@ -1,302 +0,0 @@ ---- -title: RKE Cluster Configuration Reference ---- - - - - - -When Rancher installs Kubernetes, it uses [RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) as the Kubernetes distribution. - -This section covers the configuration options that are available in Rancher for a new or existing RKE Kubernetes cluster. - -You can configure the Kubernetes options one of two ways: - -- [Rancher UI](#rancher-ui-options): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. -- [Cluster Config File](#cluster-config-file): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the options available in an RKE installation, except for system_images configuration, by specifying them in YAML. - -The RKE cluster config options are nested under the `rancher_kubernetes_engine_config` directive. For more information, see the section about the [cluster config file.](#cluster-config-file) - - -## Rancher UI Options - -When creating a cluster using one of the options described in [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md), you can configure basic Kubernetes options using the **Cluster Options** section. - -### Kubernetes Version - -The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). - -### Network Provider - -The [Network Provider](https://kubernetes.io/docs/concepts/cluster-administration/networking/) that the cluster uses. For more details on the different networking providers, please view our [Networking FAQ](../../../faq/container-network-interface-providers.md). - ->**Note:** After you launch the cluster, you cannot change your network provider. Therefore, choose which network provider you want to use carefully, as Kubernetes doesn't allow switching between network providers. Once a cluster is created with a network provider, changing network providers would require you tear down the entire cluster and all its applications. - -Out of the box, Rancher is compatible with the following network providers: - -- [Canal](https://github.com/projectcalico/canal) -- [Flannel](https://github.com/coreos/flannel#flannel) -- [Calico](https://docs.projectcalico.org/v3.11/introduction/) -- [Weave](https://github.com/weaveworks/weave) - - -**Notes on Weave:** - -When Weave is selected as network provider, Rancher will automatically enable encryption by generating a random password. If you want to specify the password manually, please see how to configure your cluster using a [Config File](cluster-provisioning/rke-clusters/options/#cluster-config-file) and the [Weave Network Plug-in Options](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/#weave-network-plug-in-options). - -### Project Network Isolation - -Project network isolation is used to enable or disable communication between pods in different projects. - - - - -To enable project network isolation as a cluster option, you will need to use any RKE network plugin that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin. - - - - -To enable project network isolation as a cluster option, you will need to use Canal as the CNI. - - - - -### Kubernetes Cloud Providers - -You can configure a [Kubernetes cloud provider](../../../pages-for-subheaders/set-up-cloud-providers.md). If you want to use [volumes and storage](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) in Kubernetes, typically you must select the specific cloud provider in order to use it. For example, if you want to use Amazon EBS, you would need to select the `aws` cloud provider. - ->**Note:** If the cloud provider you want to use is not listed as an option, you will need to use the [config file option](#cluster-config-file) to configure the cloud provider. Please reference the [RKE cloud provider documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to configure the cloud provider. - -If you want to see all the configuration options for a cluster, please click **Show advanced options** on the bottom right. The advanced options are described below: - -### Private registries - -The cluster-level private registry configuration is only used for provisioning clusters. - -There are two main ways to set up private registries in Rancher: by setting up the [global default registry](../../../how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md) through the **Settings** tab in the global view, and by setting up a private registry in the advanced options in the cluster-level settings. The global default registry is intended to be used for air-gapped setups, for registries that do not require credentials. The cluster-level private registry is intended to be used in all setups in which the private registry requires credentials. - -If your private registry requires credentials, you need to pass the credentials to Rancher by editing the cluster options for each cluster that needs to pull images from the registry. - -The private registry configuration option tells Rancher where to pull the [system images](https://rancher.com/docs/rke/latest/en/config-options/system-images/) or [addon images](https://rancher.com/docs/rke/latest/en/config-options/add-ons/) that will be used in your cluster. - -- **System images** are components needed to maintain the Kubernetes cluster. -- **Add-ons** are used to deploy several cluster components, including network plug-ins, the ingress controller, the DNS provider, or the metrics server. - -See the [RKE documentation on private registries](https://rancher.com/docs/rke/latest/en/config-options/private-registries/) for more information on the private registry for components applied during the provisioning of the cluster. - -### Authorized Cluster Endpoint - -Authorized Cluster Endpoint can be used to directly access the Kubernetes API server, without requiring communication through Rancher. - -> The authorized cluster endpoint is available only in clusters that Rancher has provisioned [using RKE](../../../pages-for-subheaders/rancher-manager-architecture.md#tools-for-provisioning-kubernetes-clusters). It is not available for clusters in hosted Kubernetes providers, such as Amazon's EKS. Additionally, the authorized cluster endpoint cannot be enabled for RKE clusters that are registered with Rancher; it is available only on Rancher-launched Kubernetes clusters. - -This is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self signed certificates. - -For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) - -We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint) - -### Node Pools - -For information on using the Rancher UI to set up node pools in an RKE cluster, refer to [this page.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) - -## Advanced Options - -The following options are available when you create clusters in the Rancher UI. They are located under **Advanced Options.** - -### NGINX Ingress - -Option to enable or disable the [NGINX ingress controller](https://rancher.com/docs/rke/latest/en/config-options/add-ons/ingress-controllers/). - -### Node Port Range - -Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). Default is `30000-32767`. - -### Metrics Server Monitoring - -Option to enable or disable [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/). - -### Pod Security Policy Support - -Option to enable and select a default [Pod Security Policy](../../../how-to-guides/advanced-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md). You must have an existing Pod Security Policy configured before you can use this option. - -### Docker Version on Nodes - -Option to require [a supported Docker version](../../../pages-for-subheaders/installation-requirements.md) installed on the cluster nodes that are added to the cluster, or to allow unsupported Docker versions installed on the cluster nodes. - -### Docker Root Directory - -If the nodes you are adding to the cluster have Docker configured with a non-default Docker Root Directory (default is `/var/lib/docker`), please specify the correct Docker Root Directory in this option. - -### Recurring etcd Snapshots - -Option to enable or disable [recurring etcd snapshots](https://rancher.com/docs/rke/latest/en/etcd-snapshots/#etcd-recurring-snapshots). - -### Agent Environment Variables - -_Available as of v2.5.6_ - -Option to set environment variables for [rancher agents](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. If rancher agent requires use of proxy to communicate with Rancher server, `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables can be set using agent environment variables. - - -## Cluster Config File - -Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the [options available](https://rancher.com/docs/rke/latest/en/config-options/) in an RKE installation, except for `system_images` configuration. The `system_images` option is not supported when creating a cluster with the Rancher UI or API. - -- To edit an RKE config file directly from the Rancher UI, click **Edit as YAML**. -- To read from an existing RKE file, click **Read from a file**. - -![image](/img/cluster-options-yaml.png) - -### Config File Structure in Rancher v2.3.0+ - -RKE (Rancher Kubernetes Engine) is the tool that Rancher uses to provision Kubernetes clusters. Rancher's cluster config files used to have the same structure as [RKE config files,](https://rancher.com/docs/rke/latest/en/example-yamls/) but the structure changed so that in Rancher, RKE cluster config items are separated from non-RKE config items. Therefore, configuration for your cluster needs to be nested under the `rancher_kubernetes_engine_config` directive in the cluster config file. Cluster config files created with earlier versions of Rancher will need to be updated for this format. An example cluster config file is included below. - -
- Example Cluster Config File - -```yaml -# -# Cluster Config -# -docker_root_dir: /var/lib/docker -enable_cluster_alerting: false -enable_cluster_monitoring: false -enable_network_policy: false -local_cluster_auth_endpoint: - enabled: true -# -# Rancher Config -# -rancher_kubernetes_engine_config: # Your RKE template config goes here. - addon_job_timeout: 30 - authentication: - strategy: x509 - ignore_docker_version: true -# -# # Currently only nginx ingress provider is supported. -# # To disable ingress controller, set `provider: none` -# # To enable ingress on specific nodes, use the node_selector, eg: -# provider: nginx -# node_selector: -# app: ingress -# - ingress: - provider: nginx - kubernetes_version: v1.15.3-rancher3-1 - monitoring: - provider: metrics-server -# -# If you are using calico on AWS -# -# network: -# plugin: calico -# calico_network_provider: -# cloud_provider: aws -# -# # To specify flannel interface -# -# network: -# plugin: flannel -# flannel_network_provider: -# iface: eth1 -# -# # To specify flannel interface for canal plugin -# -# network: -# plugin: canal -# canal_network_provider: -# iface: eth1 -# - network: - options: - flannel_backend_type: vxlan - plugin: canal -# -# services: -# kube-api: -# service_cluster_ip_range: 10.43.0.0/16 -# kube-controller: -# cluster_cidr: 10.42.0.0/16 -# service_cluster_ip_range: 10.43.0.0/16 -# kubelet: -# cluster_domain: cluster.local -# cluster_dns_server: 10.43.0.10 -# - services: - etcd: - backup_config: - enabled: true - interval_hours: 12 - retention: 6 - safe_timestamp: false - creation: 12h - extra_args: - election-timeout: 5000 - heartbeat-interval: 500 - gid: 0 - retention: 72h - snapshot: false - uid: 0 - kube_api: - always_pull_images: false - pod_security_policy: false - service_node_port_range: 30000-32767 - ssh_agent_auth: false -windows_prefered_cluster: false -``` -
- -### Default DNS provider - -The table below indicates what DNS provider is deployed by default. See [RKE documentation on DNS provider](https://rancher.com/docs/rke/latest/en/config-options/add-ons/dns/) for more information how to configure a different DNS provider. CoreDNS can only be used on Kubernetes v1.12.0 and higher. - -| Rancher version | Kubernetes version | Default DNS provider | -|-------------|--------------------|----------------------| -| v2.2.5 and higher | v1.14.0 and higher | CoreDNS | -| v2.2.5 and higher | v1.13.x and lower | kube-dns | -| v2.2.4 and lower | any | kube-dns | - -## Rancher specific parameters - -Besides the RKE config file options, there are also Rancher specific settings that can be configured in the Config File (YAML): - -### docker_root_dir - -See [Docker Root Directory](#docker-root-directory). - -### enable_cluster_monitoring - -Option to enable or disable [Cluster Monitoring](../../../pages-for-subheaders/monitoring-and-alerting.md). - -### enable_network_policy - -Option to enable or disable Project Network Isolation. - -Before Rancher v2.5.8, project network isolation is only available if you are using the Canal network plugin for RKE. - -In v2.5.8+, project network isolation is available if you are using any RKE network plugin that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin. - -### local_cluster_auth_endpoint - -See [Authorized Cluster Endpoint](#authorized-cluster-endpoint). - -Example: - -```yaml -local_cluster_auth_endpoint: - enabled: true - fqdn: "FQDN" - ca_certs: "BASE64_CACERT" -``` - -### Custom Network Plug-in - -You can add a custom network plug-in by using the [user-defined add-on functionality](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/) of RKE. You define any add-on that you want deployed after the Kubernetes cluster is deployed. - -There are two ways that you can specify an add-on: - -- [In-line Add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#in-line-add-ons) -- [Referencing YAML Files for Add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#referencing-yaml-files-for-add-ons) - -For an example of how to configure a custom network plug-in by editing the `cluster.yml`, refer to the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/custom-network-plugin-example) diff --git a/versioned_docs/version-2.5/reference-guides/about-the-api/api-tokens.md b/versioned_docs/version-2.5/reference-guides/about-the-api/api-tokens.md index 2cc866cbe50..1136b4956d1 100644 --- a/versioned_docs/version-2.5/reference-guides/about-the-api/api-tokens.md +++ b/versioned_docs/version-2.5/reference-guides/about-the-api/api-tokens.md @@ -3,7 +3,7 @@ title: API Tokens --- - + By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. diff --git a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/backup-configuration.md b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/backup-configuration.md index 6a94ae60a51..12d8c9a053e 100644 --- a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/backup-configuration.md +++ b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/backup-configuration.md @@ -3,7 +3,7 @@ title: Backup Configuration --- - + The Backup Create page lets you configure a schedule, enable encryption and specify the storage location for your backups. diff --git a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/examples.md b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/examples.md index 1cc04a7938d..610cfab0650 100644 --- a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/examples.md +++ b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + This section contains examples of Backup and Restore custom resources. diff --git a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/restore-configuration.md b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/restore-configuration.md index 3b2f93cc099..a6db2a37239 100644 --- a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/restore-configuration.md +++ b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/restore-configuration.md @@ -3,7 +3,7 @@ title: Restore Configuration --- - + The Restore Create page lets you provide details of the backup to restore from diff --git a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/storage-configuration.md b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/storage-configuration.md index 3f0e04d0fbb..2a99b969db0 100644 --- a/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/storage-configuration.md +++ b/versioned_docs/version-2.5/reference-guides/backup-restore-configuration/storage-configuration.md @@ -3,7 +3,7 @@ title: Backup Storage Location Configuration --- - + Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible object store. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index bd56e50d30e..dc307e21494 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -3,7 +3,7 @@ title: Logging Best Practices --- - + In this guide, we recommend best practices for cluster-level logging and application logging. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md index 20e14e6b9e0..3dbe0bd9c20 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -3,7 +3,7 @@ title: Monitoring Best Practices --- - + Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. This is not different when using Kubernetes and Rancher. Fortunately the integrated monitoring and alerting functionality makes this whole process a lot easier. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md index 3012223f423..9d24fb3c5db 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -3,7 +3,7 @@ title: Best Practices for Rancher Managed vSphere Clusters --- - + This guide outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md index 628bd1c36f9..cdc2183847a 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -3,7 +3,7 @@ title: Tips for Setting Up Containers --- - + Running well-built containers can greatly impact the overall performance and security of your environment. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md index 7251e64930f..87d4f348d3a 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -3,7 +3,7 @@ title: Installing Rancher in a vSphere Environment --- - + This guide outlines a reference architecture for installing Rancher on an RKE Kubernetes cluster in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md index a158b49e238..ba971bfde8f 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -3,7 +3,7 @@ title: Rancher Deployment Strategy --- - + There are two recommended deployment strategies for a Rancher server that manages downstream Kubernetes clusters. Each one has its own pros and cons. Read more about which one would fit best for your use case. diff --git a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index a5290120701..36a246dc735 100644 --- a/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/versioned_docs/version-2.5/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -3,7 +3,7 @@ title: Tips for Running Rancher --- - + This guide is geared toward use cases where Rancher is used to manage downstream Kubernetes clusters. The high-availability setup is intended to prevent losing access to downstream clusters if the Rancher server is not available. diff --git a/versioned_docs/version-2.5/reference-guides/cli-with-rancher/kubectl-utility.md b/versioned_docs/version-2.5/reference-guides/cli-with-rancher/kubectl-utility.md index ba9782062b7..3ff216dc9fa 100644 --- a/versioned_docs/version-2.5/reference-guides/cli-with-rancher/kubectl-utility.md +++ b/versioned_docs/version-2.5/reference-guides/cli-with-rancher/kubectl-utility.md @@ -3,7 +3,7 @@ title: kubectl Utility --- - + ## kubectl diff --git a/versioned_docs/version-2.5/reference-guides/cli-with-rancher/rancher-cli.md b/versioned_docs/version-2.5/reference-guides/cli-with-rancher/rancher-cli.md index cd762cd7368..fbf757d7f43 100644 --- a/versioned_docs/version-2.5/reference-guides/cli-with-rancher/rancher-cli.md +++ b/versioned_docs/version-2.5/reference-guides/cli-with-rancher/rancher-cli.md @@ -4,7 +4,7 @@ description: Interact with Rancher using command line interface (CLI) tools from --- - + The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md index ad127081b1a..5d6ee5eb8c8 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Node Template Configuration --- - + For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md index 5b529bb37aa..bfe53fc1835 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Node Template Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md index 584a6b14437..d4e70a8fc12 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Node Template Configuration --- - + Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md index 015f6eda551..49afe925b82 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -3,7 +3,7 @@ title: VSphere Node Template Configuration --- - + The following node template configuration reference applies to Rancher v2.3.3+. diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md index 4964d9cf4c0..6861ccbee5c 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -3,7 +3,7 @@ title: EKS Cluster Configuration Reference --- - + diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md index db4cb75d76d..103cf012ccb 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -3,7 +3,7 @@ title: Private Clusters --- - + _Available as of v2.5.8_ diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md index 659bc58eefa..2cbaa955b68 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -3,7 +3,7 @@ title: Syncing --- - + Syncing is the feature for EKS and GKE clusters that causes Rancher to update the clusters' values so they are up to date with their corresponding cluster object in the hosted Kubernetes provider. This enables Rancher to not be the sole owner of a hosted cluster’s state. Its largest limitation is that processing an update from Rancher and another source at the same time or within 5 minutes of one finishing may cause the state from one source to completely overwrite the other. diff --git a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md index a0e092fee87..3c363d053cf 100644 --- a/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md +++ b/versioned_docs/version-2.5/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -3,7 +3,7 @@ title: Rancher Agent Options --- - + Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. diff --git a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/examples.md b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/examples.md index 7c586657eb2..c65722cfa97 100644 --- a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/examples.md +++ b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + ### ServiceMonitor diff --git a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/helm-chart-options.md index 0a0cb764e2b..fb2f68e6460 100644 --- a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/helm-chart-options.md +++ b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -3,7 +3,7 @@ title: Helm Chart Options --- - + ## Configuring Resource Limits and Requests diff --git a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/receivers.md b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/receivers.md index a3e33a76e18..bfe2f5ac7df 100644 --- a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/receivers.md +++ b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/receivers.md @@ -3,7 +3,7 @@ title: Receiver Configuration --- - + The [Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret contains the configuration of an Alertmanager instance that sends out notifications based on alerts it receives from Prometheus. diff --git a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/routes.md b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/routes.md index 9115fbced37..62e68e8bc53 100644 --- a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/routes.md +++ b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/routes.md @@ -3,7 +3,7 @@ title: Route Configuration --- - + The route configuration is the section of the Alertmanager custom resource that controls how the alerts fired by Prometheus are grouped and filtered before they reach the receiver. diff --git a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md index c46c7970881..6e7150b1105 100644 --- a/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md +++ b/versioned_docs/version-2.5/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -3,7 +3,7 @@ title: ServiceMonitor and PodMonitor Configuration --- - + ServiceMonitors and PodMonitors are both pseudo-CRDs that map the scrape configuration of the Prometheus custom resource. diff --git a/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/architecture-recommendations.md index b0a4a11590e..14230cf82dd 100644 --- a/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -2,6 +2,10 @@ title: Architecture Recommendations --- + + + + Kubernetes cluster. If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the node running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) ## Separation of Rancher and User Clusters diff --git a/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index b676e11d6ee..71a22d20698 100644 --- a/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -2,6 +2,10 @@ title: Communicating with Downstream User Clusters --- + + + + This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. diff --git a/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/rancher-server-and-components.md index edb00a9424c..c7092bcec23 100644 --- a/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/rancher-server-and-components.md +++ b/versioned_docs/version-2.5/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -2,6 +2,10 @@ title: Rancher Server and Components --- + + + + The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). diff --git a/versioned_docs/version-2.6/backups/docker-installs/docker-installs.md b/versioned_docs/version-2.6/backups/docker-installs/docker-installs.md deleted file mode 100644 index cc211cee5e0..00000000000 --- a/versioned_docs/version-2.6/backups/docker-installs/docker-installs.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Backup and Restore for Rancher Installed with Docker ---- - -- [Backups](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) -- [Restores](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) \ No newline at end of file diff --git a/versioned_docs/version-2.6/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/versioned_docs/version-2.6/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md index b74d1d7d4e9..a4e4f9e0771 100644 --- a/versioned_docs/version-2.6/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md +++ b/versioned_docs/version-2.6/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -113,20 +113,23 @@ This reference contains variables that you can use in `questions.yml` nested und | label | string | true | Define the UI label. | | description | string | false | Specify the description of the variable.| | type | string | false | Default to `string` if not specified (current supported types are string, multiline, boolean, int, enum, password, storageclass, hostname, pvc, and secret).| -| required | bool | false | Define if the variable is required or not (true \| false)| | default | string | false | Specify the default value. Only used if there is no corresponding value in the `values.yml` file. | | group | string | false | Group questions by input value. | -| min_length | int | false | Min character length.| -| max_length | int | false | Max character length.| -| min | int | false | Min integer length. | -| max | int | false | Max integer length. | | options | []string | false | Specify the options when the variable type is `enum`, for example: options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer"| -| valid_chars | string | false | Regular expression for input chars validation. | -| invalid_chars | string | false | Regular expression for invalid input chars validation.| | subquestions | []subquestion | false| Add an array of subquestions.| | show_if | string | false | Show current variable if conditional variable is true. For example `show_if: "serviceType=Nodeport"` | | show\_subquestion_if | string | false | Show subquestions if is true or equal to one of the options. for example `show_subquestion_if: "true"`| + + :::note `subquestions[]` cannot contain `subquestions` or `show_subquestions_if` keys, but all other keys in the above table are supported. diff --git a/versioned_docs/version-2.6/reference-guides/about-the-api/api-tokens.md b/versioned_docs/version-2.6/reference-guides/about-the-api/api-tokens.md index 143964542e8..0d813665761 100644 --- a/versioned_docs/version-2.6/reference-guides/about-the-api/api-tokens.md +++ b/versioned_docs/version-2.6/reference-guides/about-the-api/api-tokens.md @@ -3,7 +3,7 @@ title: API Tokens --- - + By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. diff --git a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/backup-configuration.md b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/backup-configuration.md index 2a792778e0a..e7ef2b056bf 100644 --- a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/backup-configuration.md +++ b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/backup-configuration.md @@ -3,7 +3,7 @@ title: Backup Configuration --- - + The Backup Create page lets you configure a schedule, enable encryption and specify the storage location for your backups. diff --git a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/examples.md b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/examples.md index 4336a204d98..bab7533302e 100644 --- a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/examples.md +++ b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + This section contains examples of Backup and Restore custom resources. diff --git a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/restore-configuration.md b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/restore-configuration.md index 5e958fcd674..b162a6ef873 100644 --- a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/restore-configuration.md +++ b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/restore-configuration.md @@ -3,7 +3,7 @@ title: Restore Configuration --- - + The Restore Create page lets you provide details of the backup to restore from diff --git a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/storage-configuration.md b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/storage-configuration.md index 3ad8b94487f..b1b8bad0896 100644 --- a/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/storage-configuration.md +++ b/versioned_docs/version-2.6/reference-guides/backup-restore-configuration/storage-configuration.md @@ -3,7 +3,7 @@ title: Backup Storage Location Configuration --- - + Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible object store. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index 9fc13c9ce70..dc200ae62b1 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -3,7 +3,7 @@ title: Logging Best Practices --- - + In this guide, we recommend best practices for cluster-level logging and application logging. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md index 820cfae1e49..35f8af067e4 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -3,7 +3,7 @@ title: Monitoring Best Practices --- - + Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. This is not different when using Kubernetes and Rancher. Fortunately the integrated monitoring and alerting functionality makes this whole process a lot easier. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md index 3012223f423..9d24fb3c5db 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -3,7 +3,7 @@ title: Best Practices for Rancher Managed vSphere Clusters --- - + This guide outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md index 1298ff18b87..0cec0a91864 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -3,7 +3,7 @@ title: Tips for Setting Up Containers --- - + Running well-built containers can greatly impact the overall performance and security of your environment. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md index c19ac243d98..74243559753 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -3,7 +3,7 @@ title: Installing Rancher in a vSphere Environment --- - + This guide outlines a reference architecture for installing Rancher on an RKE Kubernetes cluster in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md index cc8811dc77e..99583f75579 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -3,7 +3,7 @@ title: Rancher Deployment Strategy --- - + There are two recommended deployment strategies for a Rancher instance that manages downstream Kubernetes clusters. Each one has its own pros and cons. Read more about which one would fit best for your use case. diff --git a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index 1863ff82503..1ed05696fa2 100644 --- a/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/versioned_docs/version-2.6/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -3,7 +3,7 @@ title: Tips for Running Rancher --- - + This guide is geared toward use cases where Rancher is used to manage downstream Kubernetes clusters. The high-availability setup is intended to prevent losing access to downstream clusters if the Rancher server is not available. diff --git a/versioned_docs/version-2.6/reference-guides/cli-with-rancher/kubectl-utility.md b/versioned_docs/version-2.6/reference-guides/cli-with-rancher/kubectl-utility.md index 45502753a08..fefa1a66b1b 100644 --- a/versioned_docs/version-2.6/reference-guides/cli-with-rancher/kubectl-utility.md +++ b/versioned_docs/version-2.6/reference-guides/cli-with-rancher/kubectl-utility.md @@ -3,7 +3,7 @@ title: kubectl Utility --- - + ## kubectl diff --git a/versioned_docs/version-2.6/reference-guides/cli-with-rancher/rancher-cli.md b/versioned_docs/version-2.6/reference-guides/cli-with-rancher/rancher-cli.md index f647eca1163..445d2b6446b 100644 --- a/versioned_docs/version-2.6/reference-guides/cli-with-rancher/rancher-cli.md +++ b/versioned_docs/version-2.6/reference-guides/cli-with-rancher/rancher-cli.md @@ -4,7 +4,7 @@ description: Interact with Rancher using command line interface (CLI) tools from --- - + The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md index 6043df8f1ba..4640d142697 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Machine Configuration Reference --- - + For more details about EC2 nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md index 98d299762b0..ba7e806295e 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Machine Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md index 1d62fe9d9d7..634c7f48561 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Machine Configuration --- - + For more details about DigitalOcean, Droplets, refer to the [official documentation](https://docs.digitalocean.com/products/compute/). diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md index 2589d800c95..78bdc608163 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Node Template Configuration --- - + For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md index 8dcfd402deb..80f266d590b 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Node Template Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md index 584a6b14437..d4e70a8fc12 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Node Template Configuration --- - + Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md index ff68438637c..e3294c0c885 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -3,7 +3,7 @@ title: Nutanix Node Template Configuration --- - + ## Account Access diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md index 159248fc1c6..baae6beac1a 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -3,7 +3,7 @@ title: VSphere Node Template Configuration --- - + ## Account Access diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md index 136c71a45a2..67cd280e470 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -3,7 +3,7 @@ title: AKS Cluster Configuration Reference --- - + ## Changes in Rancher v2.6 diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md index 49d7e9d38f3..d9fe3d29059 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -3,7 +3,7 @@ title: EKS Cluster Configuration Reference --- - + ### Account Access diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md index c6d7893c7ae..0df4836a237 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -3,7 +3,7 @@ title: Private Clusters --- - + In GKE, [private clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept) are clusters whose nodes are isolated from inbound and outbound traffic by assigning them internal IP addresses only. Private clusters in GKE have the option of exposing the control plane endpoint as a publicly accessible address or as a private address. This is different from other Kubernetes providers, which may refer to clusters with private control plane endpoints as "private clusters" but still allow traffic to and from nodes. You may want to create a cluster with private nodes, with or without a public control plane endpoint, depending on your organization's networking and security requirements. A GKE cluster provisioned from Rancher can use isolated nodes by selecting "Private Cluster" in the Cluster Options (under "Show advanced options"). The control plane endpoint can optionally be made private by selecting "Enable Private Endpoint". diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md index 0b6e10c248c..a9a02deaaef 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -127,7 +127,7 @@ ACE is available on Kubernetes clusters provisioned by or registered with Ranche ::: -ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters.md) on RKE2 and K3s clusters in Rancher v2.6.3 and later. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. +ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) on RKE2 and K3s clusters in Rancher v2.6.3 and later. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md index 7d8916c6785..51f9d04929c 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -3,7 +3,7 @@ title: Syncing --- - + Syncing allows Rancher to update cluster values so that they're up to date with the corresponding cluster object hosted in AKS, EKS or GKE. This enables sources other than Rancher to own a hosted cluster’s state. diff --git a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md index a0e092fee87..3c363d053cf 100644 --- a/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md +++ b/versioned_docs/version-2.6/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -3,7 +3,7 @@ title: Rancher Agent Options --- - + Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. diff --git a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/examples.md b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/examples.md index 7c586657eb2..c65722cfa97 100644 --- a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/examples.md +++ b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + ### ServiceMonitor diff --git a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/helm-chart-options.md index fabd9fc222f..112c41c9ea0 100644 --- a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/helm-chart-options.md +++ b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -3,7 +3,7 @@ title: Helm Chart Options --- - + ## Configuring Resource Limits and Requests diff --git a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/receivers.md b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/receivers.md index b14f320a3ee..631e589009c 100644 --- a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/receivers.md +++ b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/receivers.md @@ -3,7 +3,7 @@ title: Receiver Configuration --- - + The [Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret contains the configuration of an Alertmanager instance that sends out notifications based on alerts it receives from Prometheus. diff --git a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/routes.md b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/routes.md index 38e29ec82a0..9fb4e3896e6 100644 --- a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/routes.md +++ b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/routes.md @@ -3,7 +3,7 @@ title: Route Configuration --- - + The route configuration is the section of the Alertmanager custom resource that controls how the alerts fired by Prometheus are grouped and filtered before they reach the receiver. diff --git a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md index 780d13c2673..30fccb634c8 100644 --- a/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md +++ b/versioned_docs/version-2.6/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -3,7 +3,7 @@ title: ServiceMonitor and PodMonitor Configuration --- - + ServiceMonitors and PodMonitors are both pseudo-CRDs that map the scrape configuration of the Prometheus custom resource. diff --git a/versioned_docs/version-2.6/reference-guides/prometheus-federator/rbac.md b/versioned_docs/version-2.6/reference-guides/prometheus-federator/rbac.md index 47887415d85..8b54ce9559f 100644 --- a/versioned_docs/version-2.6/reference-guides/prometheus-federator/rbac.md +++ b/versioned_docs/version-2.6/reference-guides/prometheus-federator/rbac.md @@ -2,6 +2,10 @@ title: Role-Based Access Control --- + + + + This section describes the expectations for Role-Based Access Control (RBAC) for Prometheus Federator. As described in the section on [namespaces](../../pages-for-subheaders/prometheus-federator.md#namespaces), Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: diff --git a/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/architecture-recommendations.md index 0440180b112..8940a45f8c1 100644 --- a/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -2,6 +2,10 @@ title: Architecture Recommendations --- + + + + If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the node running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) ## Separation of Rancher and User Clusters diff --git a/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index 0c5b83a3ffe..586b55b0db8 100644 --- a/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -2,6 +2,10 @@ title: Communicating with Downstream User Clusters --- + + + + This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. diff --git a/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/rancher-server-and-components.md index 7a588ac91b3..9d25e318ee3 100644 --- a/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/rancher-server-and-components.md +++ b/versioned_docs/version-2.6/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -2,6 +2,10 @@ title: Rancher Server and Components --- + + + + The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). diff --git a/versioned_docs/version-2.6/security/hardening-guides/rke-1.6-benchmark-2.6/Rancher_v2-6_CIS_v1-6_Benchmark_Assessment.pdf b/versioned_docs/version-2.6/security/hardening-guides/rke-1.6-benchmark-2.6/Rancher_v2-6_CIS_v1-6_Benchmark_Assessment.pdf deleted file mode 100644 index 25f33709b3f..00000000000 --- a/versioned_docs/version-2.6/security/hardening-guides/rke-1.6-benchmark-2.6/Rancher_v2-6_CIS_v1-6_Benchmark_Assessment.pdf +++ /dev/null @@ -1,14862 +0,0 @@ -%PDF-1.7 -%🖤 -1 0 obj -<< -/Type /Pages -/Kids [ 6 0 R 8 0 R 38 0 R 76 0 R 114 0 R 152 0 R 186 0 R 222 0 R 258 0 R 292 0 R 304 0 R 311 0 R 315 0 R 317 0 R 320 0 R 325 0 R 327 0 R 351 0 R 376 0 R 409 0 R 442 0 R 466 0 R 505 0 R 510 0 R 513 0 R 516 0 R 518 0 R 521 0 R 523 0 R 526 0 R 529 0 R 531 0 R 534 0 R 536 0 R 539 0 R 542 0 R 544 0 R 547 0 R 550 0 R 552 0 R 555 0 R 557 0 R 561 0 R 564 0 R 567 0 R 569 0 R 572 0 R 574 0 R 577 0 R 580 0 R 582 0 R 585 0 R 588 0 R 590 0 R 593 0 R 595 0 R 598 0 R 601 0 R 603 0 R 606 0 R 609 0 R 611 0 R 614 0 R 616 0 R 619 0 R 622 0 R 624 0 R 627 0 R 629 0 R 632 0 R 635 0 R 637 0 R 640 0 R 643 0 R 645 0 R 662 0 R 676 0 R 679 0 R 682 0 R 685 0 R 688 0 R 691 0 R 694 0 R 696 0 R 700 0 R 702 0 R 705 0 R 707 0 R 710 0 R 712 0 R 715 0 R 717 0 R 719 0 R 722 0 R 724 0 R 727 0 R 729 0 R 732 0 R 734 0 R 736 0 R 739 0 R 741 0 R 743 0 R 745 0 R 748 0 R 750 0 R 753 0 R 757 0 R 766 0 R 774 0 R 776 0 R 781 0 R 786 0 R 792 0 R 796 0 R 800 0 R 806 0 R 810 0 R 814 0 R 817 0 R 819 0 R 844 0 R 855 0 R 858 0 R 862 0 R 865 0 R 882 0 R 889 0 R 891 0 R 893 0 R 895 0 R ] -/Count 131 ->> -endobj -2 0 obj -<< -/Title (RKE CIS v1.6 Benchmark - Self-Assessment Guide - Rancher v2.6) -/Creator (pandoc) -/Producer (WeasyPrint 54.3) ->> -endobj -3 0 obj -<< -/Type /Catalog -/Pages 1 0 R -/Outlines 1057 0 R -/Names << -/Dests << -/Names [ (TOC) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (api-server) [ 510 0 R /XYZ 41.740157 784.430079 0 ] (apply-security-context-to-your-pods-and-containers-manual) [ 893 0 R /XYZ 84.259843 324.051496 0 ] (authentication-and-authorization) [ 743 0 R /XYZ 41.740157 784.430079 0 ] (cb1) [ 317 0 R /XYZ 84.259843 207.170079 0 ] (cb1-1) [ 317 0 R /XYZ 101.433071 205.473057 0 ] (cb10) [ 351 0 R /XYZ 84.259843 342.035606 0 ] (cb10-1) [ 351 0 R /XYZ 101.433071 340.338585 0 ] (cb101) [ 624 0 R /XYZ 84.259843 504.320079 0 ] (cb101-1) [ 624 0 R /XYZ 101.433071 502.623057 0 ] (cb104) [ 629 0 R /XYZ 84.259843 772.370079 0 ] (cb104-1) [ 629 0 R /XYZ 101.433071 770.673057 0 ] (cb107) [ 632 0 R /XYZ 84.259843 313.070079 0 ] (cb107-1) [ 632 0 R /XYZ 101.433071 311.373057 0 ] (cb110) [ 637 0 R /XYZ 84.259843 560.570079 0 ] (cb110-1) [ 637 0 R /XYZ 101.433071 558.873057 0 ] (cb113) [ 640 0 R /XYZ 84.259843 110.570079 0 ] (cb113-1) [ 640 0 R /XYZ 101.433071 108.873057 0 ] (cb116) [ 645 0 R /XYZ 84.259843 418.070079 0 ] (cb116-1) [ 645 0 R /XYZ 101.433071 416.373057 0 ] (cb116-10) [ 645 0 R /XYZ 99.933071 189.008799 0 ] (cb116-11) [ 645 0 R /XYZ 99.933071 170.061778 0 ] (cb116-12) [ 645 0 R /XYZ 99.933071 151.114756 0 ] (cb116-13) [ 645 0 R /XYZ 99.933071 132.167735 0 ] (cb116-14) [ 645 0 R /XYZ 99.933071 113.220714 0 ] (cb116-15) [ 645 0 R /XYZ 99.933071 94.273692 0 ] (cb116-16) [ 662 0 R /XYZ 99.933071 799.173057 0 ] (cb116-17) [ 662 0 R /XYZ 99.933071 780.226036 0 ] (cb116-18) [ 662 0 R /XYZ 99.933071 761.279014 0 ] (cb116-19) [ 662 0 R /XYZ 99.933071 742.331993 0 ] (cb116-2) [ 645 0 R /XYZ 99.933071 397.426036 0 ] (cb116-20) [ 662 0 R /XYZ 99.933071 723.384971 0 ] (cb116-21) [ 662 0 R /XYZ 99.933071 704.437950 0 ] (cb116-22) [ 662 0 R /XYZ 99.933071 685.490928 0 ] (cb116-23) [ 662 0 R /XYZ 99.933071 666.543907 0 ] (cb116-24) [ 662 0 R /XYZ 99.933071 647.596885 0 ] (cb116-25) [ 662 0 R /XYZ 99.933071 628.649864 0 ] (cb116-3) [ 645 0 R /XYZ 99.933071 378.479014 0 ] (cb116-4) [ 645 0 R /XYZ 99.933071 340.584971 0 ] (cb116-5) [ 645 0 R /XYZ 99.933071 321.637950 0 ] (cb116-6) [ 645 0 R /XYZ 99.933071 302.690928 0 ] (cb116-7) [ 645 0 R /XYZ 99.933071 283.743907 0 ] (cb116-8) [ 645 0 R /XYZ 99.933071 264.796885 0 ] (cb116-9) [ 645 0 R /XYZ 99.933071 226.902842 0 ] (cb117) [ 662 0 R /XYZ 84.259843 569.399864 0 ] (cb117-1) [ 662 0 R /XYZ 101.433071 567.702842 0 ] (cb120) [ 662 0 R /XYZ 84.259843 196.152842 0 ] (cb120-1) [ 662 0 R /XYZ 101.433071 194.455821 0 ] (cb121) [ 676 0 R /XYZ 84.259843 550.251496 0 ] (cb121-1) [ 676 0 R /XYZ 101.433071 548.554475 0 ] (cb124) [ 679 0 R /XYZ 84.259843 571.820079 0 ] (cb124-1) [ 679 0 R /XYZ 101.433071 570.123057 0 ] (cb127) [ 682 0 R /XYZ 84.259843 575.570079 0 ] (cb127-1) [ 682 0 R /XYZ 101.433071 573.873057 0 ] (cb13) [ 376 0 R /XYZ 84.259843 711.620079 0 ] (cb13-1) [ 376 0 R /XYZ 101.433071 709.923057 0 ] (cb13-10) [ 376 0 R /XYZ 99.933071 501.505821 0 ] (cb13-11) [ 376 0 R /XYZ 99.933071 482.558799 0 ] (cb13-12) [ 376 0 R /XYZ 99.933071 463.611778 0 ] (cb13-13) [ 376 0 R /XYZ 99.933071 444.664756 0 ] (cb13-14) [ 376 0 R /XYZ 99.933071 425.717735 0 ] (cb13-15) [ 376 0 R /XYZ 99.933071 406.770714 0 ] (cb13-16) [ 376 0 R /XYZ 99.933071 387.823692 0 ] (cb13-17) [ 376 0 R /XYZ 99.933071 368.876671 0 ] (cb13-18) [ 376 0 R /XYZ 99.933071 349.929649 0 ] (cb13-19) [ 376 0 R /XYZ 99.933071 330.982628 0 ] (cb13-2) [ 376 0 R /XYZ 99.933071 690.976036 0 ] (cb13-20) [ 376 0 R /XYZ 99.933071 312.035606 0 ] (cb13-21) [ 376 0 R /XYZ 99.933071 293.088585 0 ] (cb13-22) [ 376 0 R /XYZ 99.933071 274.141563 0 ] (cb13-23) [ 376 0 R /XYZ 99.933071 255.194542 0 ] (cb13-24) [ 376 0 R /XYZ 99.933071 236.247520 0 ] (cb13-25) [ 376 0 R /XYZ 99.933071 217.300499 0 ] (cb13-26) [ 376 0 R /XYZ 99.933071 198.353477 0 ] (cb13-27) [ 376 0 R /XYZ 99.933071 179.406456 0 ] (cb13-28) [ 376 0 R /XYZ 99.933071 160.459434 0 ] (cb13-29) [ 376 0 R /XYZ 99.933071 141.512413 0 ] (cb13-3) [ 376 0 R /XYZ 99.933071 672.029014 0 ] (cb13-30) [ 376 0 R /XYZ 99.933071 122.565391 0 ] (cb13-31) [ 376 0 R /XYZ 99.933071 103.618370 0 ] (cb13-32) [ 409 0 R /XYZ 99.933071 799.173057 0 ] (cb13-33) [ 409 0 R /XYZ 99.933071 780.226036 0 ] (cb13-34) [ 409 0 R /XYZ 99.933071 761.279014 0 ] (cb13-35) [ 409 0 R /XYZ 99.933071 742.331993 0 ] (cb13-36) [ 409 0 R /XYZ 99.933071 723.384971 0 ] (cb13-37) [ 409 0 R /XYZ 99.933071 704.437950 0 ] (cb13-38) [ 409 0 R /XYZ 99.933071 685.490928 0 ] (cb13-39) [ 409 0 R /XYZ 99.933071 666.543907 0 ] (cb13-4) [ 376 0 R /XYZ 99.933071 634.134971 0 ] (cb13-40) [ 409 0 R /XYZ 99.933071 647.596885 0 ] (cb13-41) [ 409 0 R /XYZ 99.933071 628.649864 0 ] (cb13-42) [ 409 0 R /XYZ 99.933071 609.702842 0 ] (cb13-43) [ 409 0 R /XYZ 99.933071 590.755821 0 ] (cb13-44) [ 409 0 R /XYZ 99.933071 571.808799 0 ] (cb13-45) [ 409 0 R /XYZ 99.933071 552.861778 0 ] (cb13-46) [ 409 0 R /XYZ 99.933071 533.914756 0 ] (cb13-47) [ 409 0 R /XYZ 99.933071 514.967735 0 ] (cb13-48) [ 409 0 R /XYZ 99.933071 496.020714 0 ] (cb13-49) [ 409 0 R /XYZ 99.933071 477.073692 0 ] (cb13-5) [ 376 0 R /XYZ 99.933071 596.240928 0 ] (cb13-50) [ 409 0 R /XYZ 99.933071 458.126671 0 ] (cb13-51) [ 409 0 R /XYZ 99.933071 439.179649 0 ] (cb13-52) [ 409 0 R /XYZ 99.933071 420.232628 0 ] (cb13-53) [ 409 0 R /XYZ 99.933071 382.338585 0 ] (cb13-54) [ 409 0 R /XYZ 99.933071 363.391563 0 ] (cb13-55) [ 409 0 R /XYZ 99.933071 344.444542 0 ] (cb13-56) [ 409 0 R /XYZ 99.933071 325.497520 0 ] (cb13-57) [ 409 0 R /XYZ 99.933071 306.550499 0 ] (cb13-58) [ 409 0 R /XYZ 99.933071 287.603477 0 ] (cb13-59) [ 409 0 R /XYZ 99.933071 268.656456 0 ] (cb13-6) [ 376 0 R /XYZ 99.933071 577.293907 0 ] (cb13-60) [ 409 0 R /XYZ 99.933071 249.709434 0 ] (cb13-61) [ 409 0 R /XYZ 99.933071 230.762413 0 ] (cb13-7) [ 376 0 R /XYZ 99.933071 558.346885 0 ] (cb13-8) [ 376 0 R /XYZ 99.933071 539.399864 0 ] (cb13-9) [ 376 0 R /XYZ 99.933071 520.452842 0 ] (cb130) [ 685 0 R /XYZ 84.259843 560.570079 0 ] (cb130-1) [ 685 0 R /XYZ 101.433071 558.873057 0 ] (cb133) [ 688 0 R /XYZ 84.259843 556.820079 0 ] (cb133-1) [ 688 0 R /XYZ 101.433071 555.123057 0 ] (cb136) [ 691 0 R /XYZ 84.259843 369.020079 0 ] (cb136-1) [ 691 0 R /XYZ 101.433071 367.323057 0 ] (cb139) [ 696 0 R /XYZ 84.259843 565.251496 0 ] (cb139-1) [ 696 0 R /XYZ 101.433071 563.554475 0 ] (cb14) [ 409 0 R /XYZ 84.259843 171.512413 0 ] (cb14-1) [ 409 0 R /XYZ 101.433071 169.815391 0 ] (cb142) [ 696 0 R /XYZ 84.259843 102.004475 0 ] (cb142-1) [ 696 0 R /XYZ 101.433071 100.307453 0 ] (cb145) [ 702 0 R /XYZ 84.259843 514.251496 0 ] (cb145-1) [ 702 0 R /XYZ 101.433071 512.554475 0 ] (cb148) [ 707 0 R /XYZ 84.259843 309.320079 0 ] (cb148-1) [ 707 0 R /XYZ 101.433071 307.623057 0 ] (cb151) [ 712 0 R /XYZ 84.259843 103.070079 0 ] (cb151-1) [ 712 0 R /XYZ 101.433071 101.373057 0 ] (cb154) [ 719 0 R /XYZ 84.259843 613.970079 0 ] (cb154-1) [ 719 0 R /XYZ 101.433071 612.273057 0 ] (cb157) [ 724 0 R /XYZ 84.259843 403.070079 0 ] (cb157-1) [ 724 0 R /XYZ 101.433071 401.373057 0 ] (cb160) [ 729 0 R /XYZ 84.259843 178.070079 0 ] (cb160-1) [ 729 0 R /XYZ 101.433071 176.373057 0 ] (cb163) [ 736 0 R /XYZ 84.259843 628.970079 0 ] (cb163-1) [ 736 0 R /XYZ 101.433071 627.273057 0 ] (cb166) [ 745 0 R /XYZ 84.259843 595.251496 0 ] (cb166-1) [ 745 0 R /XYZ 101.433071 593.554475 0 ] (cb169) [ 750 0 R /XYZ 84.259843 267.651496 0 ] (cb169-1) [ 750 0 R /XYZ 101.433071 265.954475 0 ] (cb17) [ 442 0 R /XYZ 84.259843 543.320079 0 ] (cb17-1) [ 442 0 R /XYZ 101.433071 541.623057 0 ] (cb17-10) [ 442 0 R /XYZ 99.933071 333.205821 0 ] (cb17-11) [ 442 0 R /XYZ 99.933071 314.258799 0 ] (cb17-12) [ 442 0 R /XYZ 99.933071 295.311778 0 ] (cb17-13) [ 442 0 R /XYZ 99.933071 276.364756 0 ] (cb17-14) [ 442 0 R /XYZ 99.933071 257.417735 0 ] (cb17-15) [ 442 0 R /XYZ 99.933071 238.470714 0 ] (cb17-16) [ 442 0 R /XYZ 99.933071 219.523692 0 ] (cb17-17) [ 442 0 R /XYZ 99.933071 200.576671 0 ] (cb17-18) [ 442 0 R /XYZ 99.933071 181.629649 0 ] (cb17-19) [ 442 0 R /XYZ 99.933071 162.682628 0 ] (cb17-2) [ 442 0 R /XYZ 99.933071 522.676036 0 ] (cb17-20) [ 442 0 R /XYZ 99.933071 143.735606 0 ] (cb17-21) [ 442 0 R /XYZ 99.933071 124.788585 0 ] (cb17-22) [ 442 0 R /XYZ 99.933071 105.841563 0 ] (cb17-23) [ 466 0 R /XYZ 99.933071 799.173057 0 ] (cb17-24) [ 466 0 R /XYZ 99.933071 780.226036 0 ] (cb17-25) [ 466 0 R /XYZ 99.933071 761.279014 0 ] (cb17-26) [ 466 0 R /XYZ 99.933071 742.331993 0 ] (cb17-27) [ 466 0 R /XYZ 99.933071 723.384971 0 ] (cb17-28) [ 466 0 R /XYZ 99.933071 704.437950 0 ] (cb17-29) [ 466 0 R /XYZ 99.933071 685.490928 0 ] (cb17-3) [ 442 0 R /XYZ 99.933071 503.729014 0 ] (cb17-30) [ 466 0 R /XYZ 99.933071 666.543907 0 ] (cb17-31) [ 466 0 R /XYZ 99.933071 647.596885 0 ] (cb17-32) [ 466 0 R /XYZ 99.933071 628.649864 0 ] (cb17-33) [ 466 0 R /XYZ 99.933071 609.702842 0 ] (cb17-34) [ 466 0 R /XYZ 99.933071 590.755821 0 ] (cb17-35) [ 466 0 R /XYZ 99.933071 571.808799 0 ] (cb17-36) [ 466 0 R /XYZ 99.933071 552.861778 0 ] (cb17-37) [ 466 0 R /XYZ 99.933071 533.914756 0 ] (cb17-38) [ 466 0 R /XYZ 99.933071 514.967735 0 ] (cb17-39) [ 466 0 R /XYZ 99.933071 496.020714 0 ] (cb17-4) [ 442 0 R /XYZ 99.933071 465.834971 0 ] (cb17-40) [ 466 0 R /XYZ 99.933071 477.073692 0 ] (cb17-41) [ 466 0 R /XYZ 99.933071 458.126671 0 ] (cb17-42) [ 466 0 R /XYZ 99.933071 439.179649 0 ] (cb17-43) [ 466 0 R /XYZ 99.933071 420.232628 0 ] (cb17-44) [ 466 0 R /XYZ 99.933071 401.285606 0 ] (cb17-45) [ 466 0 R /XYZ 99.933071 382.338585 0 ] (cb17-46) [ 466 0 R /XYZ 99.933071 363.391563 0 ] (cb17-47) [ 466 0 R /XYZ 99.933071 344.444542 0 ] (cb17-48) [ 466 0 R /XYZ 99.933071 325.497520 0 ] (cb17-49) [ 466 0 R /XYZ 99.933071 306.550499 0 ] (cb17-5) [ 442 0 R /XYZ 99.933071 427.940928 0 ] (cb17-50) [ 466 0 R /XYZ 99.933071 287.603477 0 ] (cb17-51) [ 466 0 R /XYZ 99.933071 268.656456 0 ] (cb17-52) [ 466 0 R /XYZ 99.933071 249.709434 0 ] (cb17-53) [ 466 0 R /XYZ 99.933071 211.815391 0 ] (cb17-54) [ 466 0 R /XYZ 99.933071 192.868370 0 ] (cb17-55) [ 466 0 R /XYZ 99.933071 173.921348 0 ] (cb17-56) [ 466 0 R /XYZ 99.933071 154.974327 0 ] (cb17-57) [ 466 0 R /XYZ 99.933071 136.027305 0 ] (cb17-58) [ 466 0 R /XYZ 99.933071 117.080284 0 ] (cb17-59) [ 466 0 R /XYZ 99.933071 98.133262 0 ] (cb17-6) [ 442 0 R /XYZ 99.933071 408.993907 0 ] (cb17-60) [ 505 0 R /XYZ 99.933071 799.173057 0 ] (cb17-61) [ 505 0 R /XYZ 99.933071 780.226036 0 ] (cb17-7) [ 442 0 R /XYZ 99.933071 390.046885 0 ] (cb17-8) [ 442 0 R /XYZ 99.933071 371.099864 0 ] (cb17-9) [ 442 0 R /XYZ 99.933071 352.152842 0 ] (cb172) [ 753 0 R /XYZ 84.259843 607.820079 0 ] (cb172-1) [ 753 0 R /XYZ 101.433071 606.123057 0 ] (cb174) [ 753 0 R /XYZ 84.259843 298.679014 0 ] (cb174-1) [ 753 0 R /XYZ 101.433071 296.981993 0 ] (cb176) [ 757 0 R /XYZ 84.259843 715.370079 0 ] (cb176-1) [ 757 0 R /XYZ 101.433071 713.673057 0 ] (cb179) [ 757 0 R /XYZ 84.259843 375.479014 0 ] (cb179-1) [ 757 0 R /XYZ 101.433071 373.781993 0 ] (cb179-2) [ 757 0 R /XYZ 99.933071 354.834971 0 ] (cb179-3) [ 757 0 R /XYZ 99.933071 335.887950 0 ] (cb179-4) [ 757 0 R /XYZ 99.933071 279.046885 0 ] (cb179-5) [ 757 0 R /XYZ 99.933071 260.099864 0 ] (cb18) [ 505 0 R /XYZ 84.259843 720.976036 0 ] (cb18-1) [ 505 0 R /XYZ 101.433071 719.279014 0 ] (cb180) [ 757 0 R /XYZ 84.259843 200.849864 0 ] (cb180-1) [ 757 0 R /XYZ 101.433071 199.152842 0 ] (cb182) [ 766 0 R /XYZ 84.259843 655.220079 0 ] (cb182-1) [ 766 0 R /XYZ 101.433071 653.523057 0 ] (cb182-2) [ 766 0 R /XYZ 99.933071 634.576036 0 ] (cb182-3) [ 766 0 R /XYZ 99.933071 615.629014 0 ] (cb182-4) [ 766 0 R /XYZ 99.933071 558.787950 0 ] (cb182-5) [ 766 0 R /XYZ 99.933071 539.840928 0 ] (cb183) [ 766 0 R /XYZ 84.259843 480.590928 0 ] (cb183-1) [ 766 0 R /XYZ 101.433071 478.893907 0 ] (cb185) [ 776 0 R /XYZ 84.259843 490.251496 0 ] (cb185-1) [ 776 0 R /XYZ 101.433071 488.554475 0 ] (cb186) [ 776 0 R /XYZ 84.259843 429.304475 0 ] (cb186-1) [ 776 0 R /XYZ 101.433071 427.607453 0 ] (cb188) [ 776 0 R /XYZ 84.259843 116.807453 0 ] (cb188-1) [ 776 0 R /XYZ 101.433071 115.110432 0 ] (cb189) [ 781 0 R /XYZ 84.259843 772.370079 0 ] (cb189-1) [ 781 0 R /XYZ 101.433071 770.673057 0 ] (cb191) [ 781 0 R /XYZ 84.259843 459.873057 0 ] (cb191-1) [ 781 0 R /XYZ 101.433071 458.176036 0 ] (cb192) [ 781 0 R /XYZ 84.259843 398.926036 0 ] (cb192-1) [ 781 0 R /XYZ 101.433071 397.229014 0 ] (cb194) [ 786 0 R /XYZ 84.259843 799.370079 0 ] (cb194-1) [ 786 0 R /XYZ 101.433071 797.673057 0 ] (cb195) [ 786 0 R /XYZ 84.259843 738.423057 0 ] (cb195-1) [ 786 0 R /XYZ 101.433071 736.726036 0 ] (cb197) [ 786 0 R /XYZ 84.259843 410.926036 0 ] (cb197-1) [ 786 0 R /XYZ 101.433071 409.229014 0 ] (cb198) [ 786 0 R /XYZ 84.259843 349.979014 0 ] (cb198-1) [ 786 0 R /XYZ 101.433071 348.281993 0 ] (cb2) [ 320 0 R /XYZ 84.259843 730.370079 0 ] (cb2-1) [ 320 0 R /XYZ 101.433071 728.673057 0 ] (cb201) [ 792 0 R /XYZ 84.259843 230.570079 0 ] (cb201-1) [ 792 0 R /XYZ 101.433071 228.873057 0 ] (cb202) [ 792 0 R /XYZ 84.259843 169.623057 0 ] (cb202-1) [ 792 0 R /XYZ 101.433071 167.926036 0 ] (cb204) [ 796 0 R /XYZ 84.259843 583.970079 0 ] (cb204-1) [ 796 0 R /XYZ 101.433071 582.273057 0 ] (cb205) [ 796 0 R /XYZ 84.259843 523.023057 0 ] (cb205-1) [ 796 0 R /XYZ 101.433071 521.326036 0 ] (cb207) [ 800 0 R /XYZ 84.259843 700.370079 0 ] (cb207-1) [ 800 0 R /XYZ 101.433071 698.673057 0 ] (cb208) [ 800 0 R /XYZ 84.259843 639.423057 0 ] (cb208-1) [ 800 0 R /XYZ 101.433071 637.726036 0 ] (cb21) [ 510 0 R /XYZ 84.259843 565.251496 0 ] (cb21-1) [ 510 0 R /XYZ 101.433071 563.554475 0 ] (cb210) [ 800 0 R /XYZ 84.259843 296.926036 0 ] (cb210-1) [ 800 0 R /XYZ 101.433071 295.229014 0 ] (cb211) [ 800 0 R /XYZ 84.259843 235.979014 0 ] (cb211-1) [ 800 0 R /XYZ 101.433071 234.281993 0 ] (cb213) [ 806 0 R /XYZ 84.259843 640.370079 0 ] (cb213-1) [ 806 0 R /XYZ 101.433071 638.673057 0 ] (cb214) [ 806 0 R /XYZ 84.259843 579.423057 0 ] (cb214-1) [ 806 0 R /XYZ 101.433071 577.726036 0 ] (cb217) [ 810 0 R /XYZ 84.259843 428.570079 0 ] (cb217-1) [ 810 0 R /XYZ 101.433071 426.873057 0 ] (cb218) [ 810 0 R /XYZ 84.259843 131.823057 0 ] (cb218-1) [ 810 0 R /XYZ 101.433071 130.126036 0 ] (cb219) [ 814 0 R /XYZ 84.259843 799.370079 0 ] (cb219-1) [ 814 0 R /XYZ 101.433071 797.673057 0 ] (cb221) [ 819 0 R /XYZ 84.259843 768.620079 0 ] (cb221-1) [ 819 0 R /XYZ 101.433071 766.923057 0 ] (cb221-10) [ 819 0 R /XYZ 99.933071 596.399864 0 ] (cb221-11) [ 819 0 R /XYZ 99.933071 577.452842 0 ] (cb221-12) [ 819 0 R /XYZ 99.933071 482.717735 0 ] (cb221-13) [ 819 0 R /XYZ 99.933071 463.770714 0 ] (cb221-14) [ 819 0 R /XYZ 99.933071 444.823692 0 ] (cb221-15) [ 819 0 R /XYZ 99.933071 425.876671 0 ] (cb221-16) [ 819 0 R /XYZ 99.933071 406.929649 0 ] (cb221-17) [ 819 0 R /XYZ 99.933071 387.982628 0 ] (cb221-18) [ 819 0 R /XYZ 99.933071 350.088585 0 ] (cb221-19) [ 819 0 R /XYZ 99.933071 331.141563 0 ] (cb221-2) [ 819 0 R /XYZ 99.933071 747.976036 0 ] (cb221-20) [ 819 0 R /XYZ 99.933071 198.512413 0 ] (cb221-21) [ 819 0 R /XYZ 99.933071 179.565391 0 ] (cb221-22) [ 819 0 R /XYZ 99.933071 160.618370 0 ] (cb221-23) [ 819 0 R /XYZ 99.933071 103.777305 0 ] (cb221-24) [ 844 0 R /XYZ 99.933071 799.173057 0 ] (cb221-25) [ 844 0 R /XYZ 99.933071 780.226036 0 ] (cb221-26) [ 844 0 R /XYZ 99.933071 761.279014 0 ] (cb221-27) [ 844 0 R /XYZ 99.933071 742.331993 0 ] (cb221-28) [ 844 0 R /XYZ 99.933071 723.384971 0 ] (cb221-29) [ 844 0 R /XYZ 99.933071 704.437950 0 ] (cb221-3) [ 819 0 R /XYZ 99.933071 729.029014 0 ] (cb221-30) [ 844 0 R /XYZ 99.933071 685.490928 0 ] (cb221-31) [ 844 0 R /XYZ 99.933071 666.543907 0 ] (cb221-4) [ 819 0 R /XYZ 99.933071 710.081993 0 ] (cb221-5) [ 819 0 R /XYZ 99.933071 691.134971 0 ] (cb221-6) [ 819 0 R /XYZ 99.933071 672.187950 0 ] (cb221-7) [ 819 0 R /XYZ 99.933071 653.240928 0 ] (cb221-8) [ 819 0 R /XYZ 99.933071 634.293907 0 ] (cb221-9) [ 819 0 R /XYZ 99.933071 615.346885 0 ] (cb222) [ 844 0 R /XYZ 84.259843 607.293907 0 ] (cb222-1) [ 844 0 R /XYZ 101.433071 605.596885 0 ] (cb225) [ 855 0 R /XYZ 84.259843 434.451496 0 ] (cb225-1) [ 855 0 R /XYZ 101.433071 432.754475 0 ] (cb228) [ 858 0 R /XYZ 84.259843 799.370079 0 ] (cb228-1) [ 858 0 R /XYZ 101.433071 797.673057 0 ] (cb231) [ 858 0 R /XYZ 84.259843 448.229014 0 ] (cb231-1) [ 858 0 R /XYZ 101.433071 446.531993 0 ] (cb234) [ 862 0 R /XYZ 84.259843 799.370079 0 ] (cb234-1) [ 862 0 R /XYZ 101.433071 797.673057 0 ] (cb237) [ 865 0 R /XYZ 84.259843 394.701496 0 ] (cb237-1) [ 865 0 R /XYZ 101.433071 393.004475 0 ] (cb237-10) [ 865 0 R /XYZ 99.933071 222.481281 0 ] (cb237-11) [ 865 0 R /XYZ 99.933071 203.534260 0 ] (cb237-12) [ 865 0 R /XYZ 99.933071 165.640217 0 ] (cb237-13) [ 865 0 R /XYZ 99.933071 127.746174 0 ] (cb237-14) [ 865 0 R /XYZ 99.933071 108.799152 0 ] (cb237-15) [ 865 0 R /XYZ 99.933071 89.852131 0 ] (cb237-16) [ 882 0 R /XYZ 99.933071 799.173057 0 ] (cb237-17) [ 882 0 R /XYZ 99.933071 780.226036 0 ] (cb237-18) [ 882 0 R /XYZ 99.933071 761.279014 0 ] (cb237-19) [ 882 0 R /XYZ 99.933071 742.331993 0 ] (cb237-2) [ 865 0 R /XYZ 99.933071 374.057453 0 ] (cb237-3) [ 865 0 R /XYZ 99.933071 355.110432 0 ] (cb237-4) [ 865 0 R /XYZ 99.933071 336.163410 0 ] (cb237-5) [ 865 0 R /XYZ 99.933071 317.216389 0 ] (cb237-6) [ 865 0 R /XYZ 99.933071 298.269367 0 ] (cb237-7) [ 865 0 R /XYZ 99.933071 279.322346 0 ] (cb237-8) [ 865 0 R /XYZ 99.933071 260.375324 0 ] (cb237-9) [ 865 0 R /XYZ 99.933071 241.428303 0 ] (cb238) [ 882 0 R /XYZ 84.259843 683.081993 0 ] (cb238-1) [ 882 0 R /XYZ 101.433071 681.384971 0 ] (cb24) [ 513 0 R /XYZ 84.259843 125.570079 0 ] (cb24-1) [ 513 0 R /XYZ 101.433071 123.873057 0 ] (cb241) [ 895 0 R /XYZ 84.259843 768.620079 0 ] (cb241-1) [ 895 0 R /XYZ 101.433071 766.923057 0 ] (cb241-10) [ 895 0 R /XYZ 99.933071 596.399864 0 ] (cb241-11) [ 895 0 R /XYZ 99.933071 577.452842 0 ] (cb241-12) [ 895 0 R /XYZ 99.933071 520.611778 0 ] (cb241-13) [ 895 0 R /XYZ 99.933071 501.664756 0 ] (cb241-14) [ 895 0 R /XYZ 99.933071 482.717735 0 ] (cb241-15) [ 895 0 R /XYZ 99.933071 463.770714 0 ] (cb241-16) [ 895 0 R /XYZ 99.933071 444.823692 0 ] (cb241-17) [ 895 0 R /XYZ 99.933071 425.876671 0 ] (cb241-18) [ 895 0 R /XYZ 99.933071 406.929649 0 ] (cb241-2) [ 895 0 R /XYZ 99.933071 747.976036 0 ] (cb241-3) [ 895 0 R /XYZ 99.933071 729.029014 0 ] (cb241-4) [ 895 0 R /XYZ 99.933071 710.081993 0 ] (cb241-5) [ 895 0 R /XYZ 99.933071 691.134971 0 ] (cb241-6) [ 895 0 R /XYZ 99.933071 672.187950 0 ] (cb241-7) [ 895 0 R /XYZ 99.933071 653.240928 0 ] (cb241-8) [ 895 0 R /XYZ 99.933071 634.293907 0 ] (cb241-9) [ 895 0 R /XYZ 99.933071 615.346885 0 ] (cb242) [ 895 0 R /XYZ 84.259843 347.679649 0 ] (cb242-1) [ 895 0 R /XYZ 101.433071 345.982628 0 ] (cb27) [ 518 0 R /XYZ 84.259843 425.570079 0 ] (cb27-1) [ 518 0 R /XYZ 101.433071 423.873057 0 ] (cb3) [ 320 0 R /XYZ 84.259843 523.623057 0 ] (cb3-1) [ 320 0 R /XYZ 101.433071 521.926036 0 ] (cb30) [ 523 0 R /XYZ 84.259843 715.370079 0 ] (cb30-1) [ 523 0 R /XYZ 101.433071 713.673057 0 ] (cb33) [ 526 0 R /XYZ 84.259843 211.820079 0 ] (cb33-1) [ 526 0 R /XYZ 101.433071 210.123057 0 ] (cb36) [ 531 0 R /XYZ 84.259843 451.820079 0 ] (cb36-1) [ 531 0 R /XYZ 101.433071 450.123057 0 ] (cb39) [ 536 0 R /XYZ 84.259843 730.370079 0 ] (cb39-1) [ 536 0 R /XYZ 101.433071 728.673057 0 ] (cb42) [ 539 0 R /XYZ 84.259843 294.320079 0 ] (cb42-1) [ 539 0 R /XYZ 101.433071 292.623057 0 ] (cb45) [ 544 0 R /XYZ 84.259843 575.570079 0 ] (cb45-1) [ 544 0 R /XYZ 101.433071 573.873057 0 ] (cb48) [ 547 0 R /XYZ 84.259843 129.320079 0 ] (cb48-1) [ 547 0 R /XYZ 101.433071 127.623057 0 ] (cb51) [ 552 0 R /XYZ 84.259843 350.570079 0 ] (cb51-1) [ 552 0 R /XYZ 101.433071 348.873057 0 ] (cb54) [ 557 0 R /XYZ 84.259843 538.070079 0 ] (cb54-1) [ 557 0 R /XYZ 101.433071 536.373057 0 ] (cb55) [ 557 0 R /XYZ 84.259843 301.323057 0 ] (cb55-1) [ 557 0 R /XYZ 101.433071 299.626036 0 ] (cb56) [ 561 0 R /XYZ 84.259843 799.370079 0 ] (cb56-1) [ 561 0 R /XYZ 101.433071 797.673057 0 ] (cb59) [ 564 0 R /XYZ 84.259843 350.570079 0 ] (cb59-1) [ 564 0 R /XYZ 101.433071 348.873057 0 ] (cb6) [ 320 0 R /XYZ 84.259843 138.376036 0 ] (cb6-1) [ 320 0 R /XYZ 101.433071 136.679014 0 ] (cb62) [ 569 0 R /XYZ 84.259843 598.970079 0 ] (cb62-1) [ 569 0 R /XYZ 101.433071 597.273057 0 ] (cb65) [ 574 0 R /XYZ 84.259843 772.370079 0 ] (cb65-1) [ 574 0 R /XYZ 101.433071 770.673057 0 ] (cb68) [ 577 0 R /XYZ 84.259843 271.820079 0 ] (cb68-1) [ 577 0 R /XYZ 101.433071 270.123057 0 ] (cb71) [ 582 0 R /XYZ 84.259843 571.820079 0 ] (cb71-1) [ 582 0 R /XYZ 101.433071 570.123057 0 ] (cb74) [ 585 0 R /XYZ 84.259843 144.320079 0 ] (cb74-1) [ 585 0 R /XYZ 101.433071 142.623057 0 ] (cb77) [ 590 0 R /XYZ 84.259843 440.570079 0 ] (cb77-1) [ 590 0 R /XYZ 101.433071 438.873057 0 ] (cb80) [ 595 0 R /XYZ 84.259843 715.370079 0 ] (cb80-1) [ 595 0 R /XYZ 101.433071 713.673057 0 ] (cb83) [ 598 0 R /XYZ 84.259843 275.570079 0 ] (cb83-1) [ 598 0 R /XYZ 101.433071 273.873057 0 ] (cb86) [ 603 0 R /XYZ 84.259843 556.820079 0 ] (cb86-1) [ 603 0 R /XYZ 101.433071 555.123057 0 ] (cb89) [ 606 0 R /XYZ 84.259843 125.570079 0 ] (cb89-1) [ 606 0 R /XYZ 101.433071 123.873057 0 ] (cb9) [ 327 0 R /XYZ 84.259843 509.420079 0 ] (cb9-1) [ 327 0 R /XYZ 101.433071 507.723057 0 ] (cb9-10) [ 327 0 R /XYZ 99.933071 318.252842 0 ] (cb9-11) [ 327 0 R /XYZ 99.933071 299.305821 0 ] (cb9-12) [ 327 0 R /XYZ 99.933071 280.358799 0 ] (cb9-13) [ 327 0 R /XYZ 99.933071 261.411778 0 ] (cb9-14) [ 327 0 R /XYZ 99.933071 242.464756 0 ] (cb9-15) [ 327 0 R /XYZ 99.933071 223.517735 0 ] (cb9-16) [ 327 0 R /XYZ 99.933071 204.570714 0 ] (cb9-17) [ 327 0 R /XYZ 99.933071 185.623692 0 ] (cb9-18) [ 327 0 R /XYZ 99.933071 166.676671 0 ] (cb9-19) [ 327 0 R /XYZ 99.933071 147.729649 0 ] (cb9-2) [ 327 0 R /XYZ 99.933071 488.776036 0 ] (cb9-20) [ 327 0 R /XYZ 99.933071 128.782628 0 ] (cb9-21) [ 327 0 R /XYZ 99.933071 109.835606 0 ] (cb9-22) [ 327 0 R /XYZ 99.933071 90.888585 0 ] (cb9-23) [ 351 0 R /XYZ 99.933071 799.173057 0 ] (cb9-24) [ 351 0 R /XYZ 99.933071 780.226036 0 ] (cb9-25) [ 351 0 R /XYZ 99.933071 761.279014 0 ] (cb9-26) [ 351 0 R /XYZ 99.933071 742.331993 0 ] (cb9-27) [ 351 0 R /XYZ 99.933071 723.384971 0 ] (cb9-28) [ 351 0 R /XYZ 99.933071 704.437950 0 ] (cb9-29) [ 351 0 R /XYZ 99.933071 685.490928 0 ] (cb9-3) [ 327 0 R /XYZ 99.933071 469.829014 0 ] (cb9-30) [ 351 0 R /XYZ 99.933071 666.543907 0 ] (cb9-31) [ 351 0 R /XYZ 99.933071 647.596885 0 ] (cb9-32) [ 351 0 R /XYZ 99.933071 628.649864 0 ] (cb9-33) [ 351 0 R /XYZ 99.933071 609.702842 0 ] (cb9-34) [ 351 0 R /XYZ 99.933071 590.755821 0 ] (cb9-35) [ 351 0 R /XYZ 99.933071 571.808799 0 ] (cb9-36) [ 351 0 R /XYZ 99.933071 552.861778 0 ] (cb9-37) [ 351 0 R /XYZ 99.933071 533.914756 0 ] (cb9-38) [ 351 0 R /XYZ 99.933071 514.967735 0 ] (cb9-39) [ 351 0 R /XYZ 99.933071 496.020714 0 ] (cb9-4) [ 327 0 R /XYZ 99.933071 431.934971 0 ] (cb9-40) [ 351 0 R /XYZ 99.933071 477.073692 0 ] (cb9-41) [ 351 0 R /XYZ 99.933071 458.126671 0 ] (cb9-42) [ 351 0 R /XYZ 99.933071 439.179649 0 ] (cb9-43) [ 351 0 R /XYZ 99.933071 420.232628 0 ] (cb9-44) [ 351 0 R /XYZ 99.933071 401.285606 0 ] (cb9-5) [ 327 0 R /XYZ 99.933071 412.987950 0 ] (cb9-6) [ 327 0 R /XYZ 99.933071 394.040928 0 ] (cb9-7) [ 327 0 R /XYZ 99.933071 375.093907 0 ] (cb9-8) [ 327 0 R /XYZ 99.933071 356.146885 0 ] (cb9-9) [ 327 0 R /XYZ 99.933071 337.199864 0 ] (cb92) [ 611 0 R /XYZ 84.259843 440.570079 0 ] (cb92-1) [ 611 0 R /XYZ 101.433071 438.873057 0 ] (cb95) [ 616 0 R /XYZ 84.259843 685.370079 0 ] (cb95-1) [ 616 0 R /XYZ 101.433071 683.673057 0 ] (cb98) [ 619 0 R /XYZ 84.259843 238.070079 0 ] (cb98-1) [ 619 0 R /XYZ 101.433071 236.373057 0 ] (client-certificate-authentication-should-not-be-used-for-users-manual) [ 743 0 R /XYZ 84.259843 684.651496 0 ] (configure-image-provenance-using-imagepolicywebhook-admission-controller-manual) [ 891 0 R /XYZ 84.259843 684.651496 0 ] (consider-external-secret-storage-manual) [ 889 0 R /XYZ 84.259843 568.851496 0 ] (contentsbox) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (controller-manager) [ 676 0 R /XYZ 41.740157 784.430079 0 ] (controls) [ 311 0 R /XYZ 84.259843 612.381496 0 ] (create-administrative-boundaries-between-resources-using-namespaces-manual) [ 893 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-a-minimal-audit-policy-is-created-automated) [ 745 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-a-unique-certificate-authority-is-used-for-etcd-automated) [ 736 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-all-namespaces-have-network-policies-defined-automated) [ 865 0 R /XYZ 84.259843 538.851496 0 ] (ensure-that-default-service-accounts-are-not-actively-used.-automated) [ 817 0 R /XYZ 84.259843 206.451496 0 ] (ensure-that-encryption-providers-are-appropriately-configured-automated) [ 645 0 R /XYZ 84.259843 577.220079 0 ] (ensure-that-service-account-tokens-are-only-mounted-where-necessary-manual) [ 844 0 R /XYZ 84.259843 449.446885 0 ] (ensure-that-the---anonymous-auth-argument-is-set-to-false-automated) [ 510 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the---audit-log-maxage-argument-is-set-to-30-or-as-appropriate-automated) [ 598 0 R /XYZ 84.259843 445.970079 0 ] (ensure-that-the---audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate-automated) [ 603 0 R /XYZ 84.259843 727.220079 0 ] (ensure-that-the---audit-log-maxsize-argument-is-set-to-100-or-as-appropriate-automated) [ 606 0 R /XYZ 84.259843 295.970079 0 ] (ensure-that-the---audit-log-path-argument-is-set-automated) [ 593 0 R /XYZ 84.259843 183.470079 0 ] (ensure-that-the---authorization-mode-argument-includes-node-automated) [ 539 0 R /XYZ 84.259843 464.720079 0 ] (ensure-that-the---authorization-mode-argument-includes-rbac-automated) [ 544 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated) [ 534 0 R /XYZ 84.259843 183.470079 0 ] (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated-1) [ 776 0 R /XYZ 84.259843 332.207453 0 ] (ensure-that-the---auto-tls-argument-is-not-set-to-true-automated) [ 712 0 R /XYZ 84.259843 258.470079 0 ] (ensure-that-the---basic-auth-file-argument-is-not-set-automated) [ 513 0 R /XYZ 84.259843 295.970079 0 ] (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated) [ 691 0 R /XYZ 84.259843 524.420079 0 ] (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated-1) [ 696 0 R /XYZ 84.259843 257.404475 0 ] (ensure-that-the---cert-file-and---key-file-arguments-are-set-as-appropriate-automated) [ 702 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated) [ 632 0 R /XYZ 84.259843 483.470079 0 ] (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated-1) [ 781 0 R /XYZ 84.259843 675.273057 0 ] (ensure-that-the---client-cert-auth-argument-is-set-to-true-automated) [ 707 0 R /XYZ 84.259843 464.720079 0 ] (ensure-that-the---encryption-provider-config-argument-is-set-as-appropriate-automated) [ 640 0 R /XYZ 84.259843 295.970079 0 ] (ensure-that-the---etcd-cafile-argument-is-set-as-appropriate-automated) [ 637 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the---etcd-certfile-and---etcd-keyfile-arguments-are-set-as-appropriate-automated) [ 624 0 R /XYZ 84.259843 689.720079 0 ] (ensure-that-the---event-qps-argument-is-set-to-0-or-a-level-which-ensures-appropriate-event-capture-automated) [ 796 0 R /XYZ 84.259843 208.126036 0 ] (ensure-that-the---hostname-override-argument-is-not-set-manual) [ 796 0 R /XYZ 84.259843 425.926036 0 ] (ensure-that-the---insecure-bind-address-argument-is-not-set-automated) [ 577 0 R /XYZ 84.259843 427.220079 0 ] (ensure-that-the---insecure-port-argument-is-set-to-0-automated) [ 582 0 R /XYZ 84.259843 727.220079 0 ] (ensure-that-the---kubeconfig-kubelet.conf-file-ownership-is-set-to-rootroot-automated) [ 753 0 R /XYZ 84.259843 163.687950 0 ] (ensure-that-the---kubeconfig-kubelet.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 753 0 R /XYZ 84.259843 454.079014 0 ] (ensure-that-the---kubelet-certificate-authority-argument-is-set-as-appropriate-automated) [ 531 0 R /XYZ 84.259843 652.220079 0 ] (ensure-that-the---kubelet-client-certificate-and---kubelet-client-key-arguments-are-set-as-appropriate-automated) [ 526 0 R /XYZ 84.259843 427.220079 0 ] (ensure-that-the---kubelet-https-argument-is-set-to-true-automated) [ 521 0 R /XYZ 84.259843 164.720079 0 ] (ensure-that-the---make-iptables-util-chains-argument-is-set-to-true-automated) [ 796 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the---peer-auto-tls-argument-is-not-set-to-true-automated) [ 729 0 R /XYZ 84.259843 333.470079 0 ] (ensure-that-the---peer-cert-file-and---peer-key-file-arguments-are-set-as-appropriate-automated) [ 719 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the---peer-client-cert-auth-argument-is-set-to-true-automated) [ 724 0 R /XYZ 84.259843 558.470079 0 ] (ensure-that-the---profiling-argument-is-set-to-false-automated) [ 590 0 R /XYZ 84.259843 595.970079 0 ] (ensure-that-the---profiling-argument-is-set-to-false-automated-1) [ 679 0 R /XYZ 84.259843 727.220079 0 ] (ensure-that-the---profiling-argument-is-set-to-false-automated-2) [ 696 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the---protect-kernel-defaults-argument-is-set-to-true-automated) [ 792 0 R /XYZ 84.259843 445.970079 0 ] (ensure-that-the---read-only-port-argument-is-set-to-0-automated) [ 781 0 R /XYZ 84.259843 301.829014 0 ] (ensure-that-the---request-timeout-argument-is-set-as-appropriate-automated) [ 611 0 R /XYZ 84.259843 595.970079 0 ] (ensure-that-the---root-ca-file-argument-is-set-as-appropriate-automated) [ 688 0 R /XYZ 84.259843 727.220079 0 ] (ensure-that-the---rotate-certificates-argument-is-not-set-to-false-automated) [ 800 0 R /XYZ 84.259843 138.881993 0 ] (ensure-that-the---secure-port-argument-is-not-set-to-0-automated) [ 585 0 R /XYZ 84.259843 314.720079 0 ] (ensure-that-the---service-account-key-file-argument-is-set-as-appropriate-automated) [ 619 0 R /XYZ 84.259843 408.470079 0 ] (ensure-that-the---service-account-lookup-argument-is-set-to-true-automated) [ 614 0 R /XYZ 84.259843 164.720079 0 ] (ensure-that-the---service-account-private-key-file-argument-is-set-as-appropriate-automated) [ 685 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the---streaming-connection-idle-timeout-argument-is-not-set-to-0-automated) [ 786 0 R /XYZ 84.259843 641.326036 0 ] (ensure-that-the---terminated-pod-gc-threshold-argument-is-set-as-appropriate-automated) [ 676 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated) [ 627 0 R /XYZ 84.259843 239.720079 0 ] (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated-1) [ 800 0 R /XYZ 84.259843 542.326036 0 ] (ensure-that-the---token-auth-file-parameter-is-not-set-automated) [ 518 0 R /XYZ 84.259843 595.970079 0 ] (ensure-that-the---use-service-account-credentials-argument-is-set-to-true-automated) [ 682 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the-admin.conf-file-ownership-is-set-to-rootroot-automated) [ 325 0 R /XYZ 84.259843 586.670079 0 ] (ensure-that-the-admin.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 325 0 R /XYZ 84.259843 702.470079 0 ] (ensure-that-the-admission-control-plugin-alwaysadmit-is-not-set-automated) [ 552 0 R /XYZ 84.259843 520.970079 0 ] (ensure-that-the-admission-control-plugin-alwayspullimages-is-set-manual) [ 557 0 R /XYZ 84.259843 708.470079 0 ] (ensure-that-the-admission-control-plugin-eventratelimit-is-set-automated) [ 547 0 R /XYZ 84.259843 314.720079 0 ] (ensure-that-the-admission-control-plugin-namespacelifecycle-is-set-automated) [ 564 0 R /XYZ 84.259843 520.970079 0 ] (ensure-that-the-admission-control-plugin-noderestriction-is-set-automated) [ 572 0 R /XYZ 84.259843 258.470079 0 ] (ensure-that-the-admission-control-plugin-podsecuritypolicy-is-set-automated) [ 569 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the-admission-control-plugin-securitycontextdeny-is-set-if-podsecuritypolicy-is-not-used-manual) [ 557 0 R /XYZ 84.259843 501.723057 0 ] (ensure-that-the-admission-control-plugin-serviceaccount-is-set-automated) [ 557 0 R /XYZ 84.259843 264.976036 0 ] (ensure-that-the-anonymous-auth-argument-is-set-to-false-automated) [ 776 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the-api-server-only-makes-use-of-strong-cryptographic-ciphers-automated) [ 662 0 R /XYZ 84.259843 411.552842 0 ] (ensure-that-the-api-server-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 315 0 R /XYZ 84.259843 553.851496 0 ] (ensure-that-the-api-server-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 315 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-audit-policy-covers-key-security-concerns-manual) [ 748 0 R /XYZ 84.259843 333.470079 0 ] (ensure-that-the-certificate-authorities-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 757 0 R /XYZ 84.259843 519.629014 0 ] (ensure-that-the-client-certificate-authorities-file-ownership-is-set-to-rootroot-automated) [ 766 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the-cluster-admin-role-is-only-used-where-required-manual) [ 817 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-cni-in-use-supports-network-policies-manual) [ 865 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-container-network-interface-file-ownership-is-set-to-rootroot-manual) [ 317 0 R /XYZ 84.259843 170.823057 0 ] (ensure-that-the-container-network-interface-file-permissions-are-set-to-644-or-more-restrictive-manual) [ 317 0 R /XYZ 84.259843 347.570079 0 ] (ensure-that-the-controller-manager-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 315 0 R /XYZ 84.259843 277.251496 0 ] (ensure-that-the-controller-manager-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 315 0 R /XYZ 84.259843 423.051496 0 ] (ensure-that-the-controller-manager.conf-file-ownership-is-set-to-rootroot-automated) [ 327 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the-controller-manager.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 325 0 R /XYZ 84.259843 209.270079 0 ] (ensure-that-the-etcd-data-directory-ownership-is-set-to-etcdetcd-automated) [ 320 0 R /XYZ 84.259843 365.776036 0 ] (ensure-that-the-etcd-data-directory-permissions-are-set-to-700-or-more-restrictive-automated) [ 320 0 R /XYZ 84.259843 694.023057 0 ] (ensure-that-the-etcd-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 317 0 R /XYZ 84.259843 478.370079 0 ] (ensure-that-the-etcd-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 317 0 R /XYZ 84.259843 609.170079 0 ] (ensure-that-the-kubelet---config-configuration-file-has-permissions-set-to-644-or-more-restrictive-automated) [ 766 0 R /XYZ 84.259843 383.493907 0 ] (ensure-that-the-kubelet---config-configuration-file-ownership-is-set-to-rootroot-automated) [ 766 0 R /XYZ 84.259843 195.693907 0 ] (ensure-that-the-kubelet-only-makes-use-of-strong-cryptographic-ciphers-automated) [ 810 0 R /XYZ 84.259843 392.223057 0 ] (ensure-that-the-kubelet-service-file-ownership-is-set-to-rootroot-automated) [ 750 0 R /XYZ 84.259843 553.851496 0 ] (ensure-that-the-kubelet-service-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 750 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-kubernetes-pki-certificate-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 351 0 R /XYZ 84.259843 184.188585 0 ] (ensure-that-the-kubernetes-pki-directory-and-file-ownership-is-set-to-rootroot-automated) [ 327 0 R /XYZ 84.259843 668.570079 0 ] (ensure-that-the-kubernetes-pki-key-file-permissions-are-set-to-600-automated) [ 442 0 R /XYZ 84.259843 702.470079 0 ] (ensure-that-the-proxy-kubeconfig-file-ownership-is-set-to-rootroot-automated) [ 753 0 R /XYZ 84.259843 763.220079 0 ] (ensure-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) [ 691 0 R /XYZ 84.259843 727.220079 0 ] (ensure-that-the-scheduler-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 317 0 R /XYZ 84.259843 739.970079 0 ] (ensure-that-the-scheduler-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 315 0 R /XYZ 84.259843 146.451496 0 ] (ensure-that-the-scheduler.conf-file-ownership-is-set-to-rootroot-automated) [ 325 0 R /XYZ 84.259843 340.070079 0 ] (ensure-that-the-scheduler.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 325 0 R /XYZ 84.259843 470.870079 0 ] (ensure-that-the-seccomp-profile-is-set-to-dockerdefault-in-your-pod-definitions-manual) [ 893 0 R /XYZ 84.259843 604.851496 0 ] (etcd-node-configuration-files) [ 702 0 R /XYZ 41.740157 784.430079 0 ] (extensible-admission-control) [ 891 0 R /XYZ 41.740157 784.430079 0 ] (general-policies) [ 893 0 R /XYZ 41.740157 784.430079 0 ] (header_bottom_text) [ 6 0 R /XYZ 85.009843 392.718898 0 ] (if-proxy-kubeconfig-file-exists-ensure-permissions-are-set-to-644-or-more-restrictive-automated) [ 750 0 R /XYZ 84.259843 423.051496 0 ] (kubelet) [ 776 0 R /XYZ 41.740157 784.430079 0 ] (logging) [ 745 0 R /XYZ 41.740157 784.430079 0 ] (master-node-configuration-files) [ 315 0 R /XYZ 41.740157 784.430079 0 ] (minimize-access-to-create-pods-manual) [ 817 0 R /XYZ 84.259843 307.251496 0 ] (minimize-access-to-secrets-manual) [ 817 0 R /XYZ 84.259843 523.851496 0 ] (minimize-the-admission-of-containers-wishing-to-share-the-host-ipc-namespace-automated) [ 855 0 R /XYZ 84.259843 238.710432 0 ] (minimize-the-admission-of-containers-wishing-to-share-the-host-network-namespace-automated) [ 858 0 R /XYZ 84.259843 603.629014 0 ] (minimize-the-admission-of-containers-wishing-to-share-the-host-process-id-namespace-automated) [ 855 0 R /XYZ 84.259843 589.851496 0 ] (minimize-the-admission-of-containers-with-added-capabilities-manual) [ 862 0 R /XYZ 84.259843 323.081993 0 ] (minimize-the-admission-of-containers-with-allowprivilegeescalation-automated) [ 858 0 R /XYZ 84.259843 252.487950 0 ] (minimize-the-admission-of-containers-with-capabilities-assigned-manual) [ 862 0 R /XYZ 84.259843 207.281993 0 ] (minimize-the-admission-of-containers-with-the-net_raw-capability-manual) [ 862 0 R /XYZ 84.259843 453.881993 0 ] (minimize-the-admission-of-privileged-containers-manual) [ 855 0 R /XYZ 84.259843 720.651496 0 ] (minimize-the-admission-of-root-containers-manual) [ 862 0 R /XYZ 84.259843 584.681993 0 ] (minimize-wildcard-use-in-roles-and-clusterroles-manual) [ 817 0 R /XYZ 84.259843 423.051496 0 ] (network-policies-and-cni) [ 865 0 R /XYZ 41.740157 784.430079 0 ] (overview) [ 304 0 R /XYZ 84.259843 704.822079 0 ] (pod-security-policies) [ 855 0 R /XYZ 41.740157 784.430079 0 ] (prefer-using-secrets-as-files-over-secrets-as-environment-variables-manual) [ 889 0 R /XYZ 84.259843 684.651496 0 ] (rbac-and-service-accounts) [ 817 0 R /XYZ 41.740157 784.430079 0 ] (rke-cis-v1.6-kubernetes-benchmark---rancher-v2.6-with-kubernetes-v1.18-to-v1.23) [ 304 0 R /XYZ 84.259843 784.970079 0 ] (scheduler) [ 696 0 R /XYZ 41.740157 784.430079 0 ] (secrets-management) [ 889 0 R /XYZ 41.740157 784.430079 0 ] (testing-controls-methodology) [ 304 0 R /XYZ 84.259843 166.076079 0 ] (the-default-namespace-should-not-be-used-automated) [ 893 0 R /XYZ 84.259843 193.251496 0 ] (title-block-header) [ 6 0 R /XYZ 84.259843 502.818898 0 ] (verify-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) [ 810 0 R /XYZ 84.259843 670.970079 0 ] (worker-node-configuration-files) [ 750 0 R /XYZ 41.740157 784.430079 0 ] ] ->> ->> ->> -endobj -4 0 obj -<< -/ExtGState << -/a1.0 << -/ca 1 ->> -/A1.0 << -/CA 1 ->> ->> -/XObject << ->> -/Pattern << ->> -/Shading << ->> -/Font 1078 0 R ->> -endobj -5 0 obj -<< -/Filter /FlateDecode -/Length 2625 ->> -stream -xœ½šM¯c· †÷þ^…©/ È¢@6i4Å  Zt‘3 Š;&]ôï÷Õ±õáëw>2Sôö="u$‰zDÊÖ³àõBñQ]C­­d?_ޜޞ$”$ø;jÜõx—tVñ’¶\ÓùMX ¥¥Xòùé„gŽúÓé/_œþužch-GMç__u•[(µIDÕ 5¢J)”S&ª¬ØSÁŒT<ú£ªÀ o51Ìð+QU˜a^ÔU fÄ¢ÎT0C[*D¥;4ŠP w‹D§)d zfº¨!×Õè´D¦3 ¹¸&eºrΩ2ÖÔD™.Á–¤îÂt°Å­¦Ë°Å² ÕÁ–X‹1]-£Óˆ®ÂxW¥:Ø"Y”éš…Ôª;Õ5褖ú¨‹’BªfÂtª!Á7ê°¢0I¹]„-YR£:Ø’L"Ólq¼˜œHVj%«5öÕmbÊt ¶ÄXœê` &Ÿ fØ"%3`…Î%Æ€XJðA V ^½2 -ÄÚ‚—b ±¥à¹Æ….* lIž La‹cåRl±æŒa *0˜Á–èÆX` ­Ã[ ¸uiÊX`à­‹fÆpa¦0ˆk5;c¹p¥ÆX``®Œèb¹Æu-åÈX`À®y͌08¸k†;©¶À @Ç`Z±ÀÁ]S)ŒîbEÆÌj€fÆøJˆµ -cƒ»±Š38¸ ¶ª+kÐ Ü Àvnp7&‰ŒîF™±ŽÀ¡Ñ¸ „$À–±û=tÑ ÐXˆK ¸« ÛYÓ UÐÚ2cAwµbú¨®-q²n1ÊAsnŒ ÜÅFeŒ Ü>+c°£FÆ‚îªa§f:pWcÆ‚î"nHŒp‡  °¿¸.Õµ ­VÆ‚îK‘±»p -’P] p$e,Œƒ e,Èà.¶÷ÆXÁ]IfŒÜGØCu°Åjd,@\–-YÓVƒD‚éÀ]QÌ0Ó»"¥1dp·o›Œ¹tA¬Œ00 -ŽŒ¹zh¥dÆ‚Ü:„|T—ʉ±  È¢:ØâÅ X‡f­2”[,FÆ‚a BÆD¡)¢ošÀ¸ Ͷ Rd,ÉBmîŒ%eäcU¦Ë1Ô\S]…N cA)°+cr©P±è øÐÃ%Ƙ:cAi°K…±[Q¨‘9Ó)lAÍX€H:UÆ‚a ¢AÆ‚+²Ì,ŒØ¡‘Kb¨™Î¥ç™•± z=Td,À2A¦ RRl& 0?=ÙNŒH~l"Í¡:ØbÉ j=ݬŒµÂ–(‘±¾ÙóÍÂXж âc,h’C“ €qèD šVä›Ñ °.{¾Y š òM¤xdÝ6Ë=ß,ŒÍaKBˆBu°AS#ëLB¾ :1]†-±9cA˹盕±aòMDôôH ö|³Ðãp75‰éÀ]D¦™±䛈lYn/o*ˆÏ¸²öŒ³ÒzöVzd `/v«BÏ ðí9=4€™È,k¦§ØV¡T¡Çþ"ó&\OŸö#ª w(Ø3kéÙ°,ÑýÌ9V5{Gµëùد?Ÿ¾ü ›ïùçŸ^qœ”—ÂùÍsO¬s?çU9¿Ùånö„ZÏOÛ‡ä[;·5~SB<~íHÓ–K‹M»Ñ¼o0”Wï+ åüxñðàãñ»—'¿J_ôÙBÞ¡×í¶óˆHõüòÍéË?üø§ïþúfìåëÓß¾ê>\òÞµÇ-øÇ .óµœ:QoÌrâkÜ㨟 sÔóà*Ì–¤®{u­§òõßÏ/¿}‡aˆv6Ã[òµ]wû:ùWGQñ?ö¶óõ}ô}»>ú½Ù‚¡½Ú÷®{;x¸ç¡ÿ{òƒ=éÚ÷ñ\Û³ý¥m|ÒŸkûß¼Ä]½þ-È"nóÜ6gï >õUº­¥­¥¡ØŒoG½|iÜzï+ZŸ; v‡ëÉtÀC?–ƾ†±`'yw‘o¾ÿãïÿüýÿËE®ÿï§þ*ÿßLù­ýôYÓøö”ˆD¬Ù ²´~l dëóÒÎ= -Bˆä:¦MÖó;î<ûßÎZ·óéG=¼~9°nõß°!‚èãߣ®ëYb$é¹XËýŒéQ „ÇwáíRf[Hã3Ùå¤ðàö¼&p¹]¶Ù¸¥q äu·õ=ab¥¾®!îÌú(îO/Aoâ~Þ]ƒÝJyš‚¤ MájÐ÷º«ÁuãÅj² Ó£P¶î–8ú°G{~4‡./á4}:ݦÔe›Î÷ /'tkÏ¥OTÚOÙFSct… ™mFµ)M£æœŸóDD—¾Ñ´c2§~›µž«\…u8’ÏzmÔ#¢Ë‰Ö»µ–æL•Y«®[ÛùeÊru«sà‰ì£›´ô/ð¶5"6.ïŹÏÀ˜kŽËoâƒðrB\íâ§S*¤vZ¬4\oÚÖîWYþ²¤ˆ¯oC…¥9d=ÒëŸWT¤)íä9.ôŽÛz¶¶J«“¾6§ø~vyžÔq6êeÕ݆KK™>»—V!õ;tgí9bwÒmÈ6yiü© {±N¯›âŽ°1>“•»P†èn‰Ótå§;ñ²3Í¥šdv–us÷4Û]â<Ùù´‹Ëœqí©÷´w¶»ùü^ŽÍŸ‹»ë×ÑÉG™»Ýêl.Ã…%^}:ÄcµN:õ¯ÇåZdlǽœ~9Û\´[Ø‘g"cÜ£.=ÿu¾ÅûŸ'?7R—ë>eÒ"N¶Þ‰÷¬ïX?RÜ¿‡¹l6aô ûK¬uÕMÃæ>ãÆk_·Ëû¨©“ :µ»0­1ÜÄqºnTÛF&½Gxy6Š3ùܹYöµhãáÐõ¤wX…Âþ8(Ö¥³µ%Ÿ·&QêU×–­ÇíFöFQÜ:l˲;3»CîÌ¡ïöû>ÖûÚ -[Ñ‹¶t¾¶Ù½é!;žb«‚ØöÂÞ°ëÞi»³‡ýÉËë³çýfr+ K'Ú§ôi—ÎëÒ <.µ£˜ï‹sϱs/ŒÅ»Fh¶eÛÚ³䬥±B ¿+ì>ºIëºíV!íúÝu:6Š‰ ÈdÈf‹eØ”o·a9ž¸ÒQ*3¦XÒ_ö‘Nû¬ÌALû¬ä‡º½Ý±ãÍhÉf”vw.ëédÕœ±d%²¼ß=¥¾jÚ=öí÷}Ýö¼=†/CòŠLeÝ–j~ÿ>eíÐ)Vk¼Ž“­ë°õqcÇï·¤æ †xÿ -®xÝŽJ~ó=ï?;yÒo€wžšÒãC=~Š¿ÛÑ,–ž*ö_^Yzkÿ¸ˆÒ£"õ»£“ãõ_¬PK -endstream -endobj -6 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 5 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -7 0 obj -<< -/Filter /FlateDecode -/Length 3667 ->> -stream -xœí\]¯Ü¶}ß_±Ï¢C?€¢mó’6AÒmÓ¢öÖNP\púпßCI$‡Zí¬ï—»kãÚÒÙ9<3‡Cj¥k?·ø'y;¥”cðÇÓÛûƒ™"ü™-†“ùãïð#ÿû‰)2ç£v21EoŽoÖ¸Éùà9JünÄMLJvþ—Ï? Y&N„C¶“MÖ:r˜B¦lÝñç×ïa…µ™Bùç?ÿpøâ%|?þðïÛÏ桇HS6Ì–ŽDqŠÑöópÁ"HtIâe¸¡áŽ:Žæ[;_‡Û/¢Ô;‘u0{†ÐÞº0ï ­ãÓAâ!L®·-ðÔ[é=âpµ–þuô4Œ§ã…@lqD¨Y÷4ȶi—à7½•ÞcgjŸïÑoì!qŒCà-.Sž:G‡8Ð…8A¢Ù•?Çóƒ+í£y{Aº—¥(q‰›Do½IôL¢­¢R¢Ò“qôagD7‰Þ$z“è3It_Š÷_è^’îM¢7‰Þ$úd Ý÷%ÚñQŠ™ö«¨ÄÏ$š’XM‹‹:>J´ãR\;(uŠ¼#Q‡ôm­´åj-üè$*p!.Ñ〲l›v pÓ[éíLíó=¦´ ö‡8Æ¡ -¦ãCÊä)œKtŒ]ˆÉ8Ä3‰:`'ãÐñA¢)-zÐ!öB„0Úè¥D;Sû|?B¢b4o/H÷²‡l‘¸Iô&Ñ›DÏ$úÀ½è Qéù8ú°3¢›Do½Iô™$º/Åû/t/I÷&Ñ›Do}²…®ûU´ã£ÙîWQ‰o%ê|«iqQlj -\ŠËç›H¢ãQ¢‰ëí•Q¢IÜé=‚x>OŽŽ©ÑñQ¢—bLâV‚@Y¶M»K¼^öØ™Úç{ô[{ˆCãPÓñ!eòDgÝÄ.ÄAà˜FÎ%šÍ^::Ž§ã£D³¸µÖSº÷8 Cì…8X:“¨`jŸïÇH´æíé^–âÍ"7‰Þ$z“è™Dº••žŒ£;#ºIô&Ñ›DŸI¢ûR¼ÿB÷’to½Iô&ÑG-t­µ$AÂö—ŒE£ýo½¾M×pÙPénköˆ0ev!Í" ±çšM˜˜%VlMì] -F³I;Ï9(6¾cXYá‘È.#‘;2Ì…CöåA–hÂ!CÄ.Ää™irœ Í>ûìŒÆa€ÏÞ¯qX¾A+Qã0†2Õ±Õ8LðÙyÃ2eÊI™8û‰²‹¤pŒ™(yÈL³ E¶Yá0Xš(p&…À K…Ã@ðÙG—ƒƒÏ.Y§pˆìB1H)(Ÿm†R5øl B¯Ø0|6ÖD%ŸCÀŠ&—7D4›0ÙDì5Q§mt”4cšlðÆj&THôćå&»g¯Õ”á³CéR8ŒX+Y‚¥Âa4ðÙÆ#ê 5Ék5É>™œ­VS"ê I9k5ÃÆ­¦DÔA­ÓjJD4¤pQ j6ðÙc&Ó8DD¦:«qˆ:hˆVS"ê ±“2'DÔAck5%b—s$­¦ÄŒÅ\ŠY«)1—EsŠZMIÆM9d“É°1¤Õ”dá3›ìÁgS8ÄJkHòFá09øLÎj5%9øl±QQò9yøl|ÐjJâ²NFåÕ8dbå§Õ”í)!‰´š‚IwJ!bß ØDžÊ›sZMI >{ÈTã[8pµš‚ l –ˆš |&KZMÁžgžµš’-|6µš’q³óZMA^L1y«Õ”Leãé³VSÈS X½("—'Ì/N«)×è#ņ᳃ê5>"¯Ì 9Àg[Ô£ØDøŒÕ¡VSr„ÏÆD­¦ää°c·¬Õ”\¾äLDAã0ó¢3ZM»v -õB!F{nﵪ‚]/ÜöLF¡FðÛa¨ðˆ'ÇÐâ6©×]èÞÞ•±à1ó;´XÓáË~»y‘æ¼o¾¼ø»³Ç~·óÑo^°YZ^,Xs*#fÏ#öte/êñÅÛÃ_~óíïÿü óâÍáï¿å¾üüúÇ_¾|æß³$<§h™Ždó•øA`á„ÍöQžfP;ï×A^ÚÏóâŠ6 K8Ìë(ç~nþwßÿöûoÿôtÍsIØgl+gÿ|Í£jbô|͇²Ö|¶æcÑç•æã#šÇ,{-´h>a³w-´ióÿµÐ>¢ù\ÞÃxÆæ] mšßmÚ/èçe±!~s5ÎùäæÆÿð×?~ý·ïzã 6$øA5b”PŸú1¯é*¦2kçpeÚ'V'cÇhü5~РqKGtÂaXÎ9¯#+µ2¯6°Ç¾Çx^áÅ™ù¸:·^OÜõÀü+ã#¬µæëüëÅΚåÿâÏü™°¯}4xô‡_Âæ´öýê>>ªÄ U¬p5pQþ_ûšƒ@;|qŸKöû Ù^ g¿qîß\ów¬Y-#W_gn+ïAp°òêìÚÏŠ×\¨¾`Õ?sì_.¼rS\¹_íæëªí› Ï´ú%™/wÖ›×Ø,cwYVö|0T„C©dNœÕÁ«ƒÃgÎ-É>D¢äMÒ¹NNµ-X±™“ôdÝË×ëñ.)ëço„NôgVû°ú!®­Bj‚I£LB•h–DÏÜ–dËYÇØÆlúy볊)ôäœĘs·7¯ž9u‘4ñ¯‰6szRýNåÁÈÁïmNð'’´Lfm¢âÕ‡WëµnÍ…SÏ‹ûç›-Ÿu’™J>üó{ºx2v·¬Ç3ÎGWžp`Šaþ&ùðÓû˜”±Æìæ¯ìË㬖hb›Câ ߯½—oî­òEkyúc½ÇîËÝëèøé¾›¸ï÷ k*Ì²Ï N^¤SêÇs;U -õvÃz?|eìá{Ø÷g_N½;”gO}yø‹Öpy¶ØÏÏx•·Åg¼üzkxÕ¸Ù™i{¼péü{½ß5«õW}›‰-O¯.¿\œ¬öË7eo6û© <´e'^Ž£3ÞÂw{žüìËšïp=4­-o«aÃN‡ò%ZÞZºò¬Ãr˜[ã×Ãò´E¿žëen‘8Õ³"‚¥ãW{—Ñ—×fA(‹¨õ,4W8TŸö½´í ö ¯ÂÖò9hDw†&sµ¥v˜[Ì6×%uV„¿ÚRáÔÀÓݺ-z·‹’mUvÉ4Ì §rC¹Z¶€ø§¨¼ƒr~njÓåeƒö¹ˆMiSM$ßìrµÛN‡]»µ5n‘ŠÍ*õKse¾»ÒSÝ¥Fü6s´¢˜&îÄiùršj$Jj¬MyCgàéPæ3øîÀqÇšû\a¸¦žh·ÁÉô|éhyäz9‚4+Vò)ÿ.S7´Ì<óõ-¥ÊKý¬wR´Ùà‘$‰‡6ëø=¬ÝVÐecl9+ù²ÉìØ—I·Y7ÆTP&ð˜«{e’j“ÇÛ–u .SXå§Í•4Õ t×an©|7ÀÝOnR-Ï^¯`°"ݹµÛáÐæÎ; Çq|D¯Aä|Osaó[¸¤~ªt˜L«v½3vÇ0)t¸O£w3\ÕÚf'›s;ì"Û«¸§Ã‡¹Ì‘cÊåV7?ø.—]öøØ|…Ÿüñëú0ßRÇÉ´Ù‚ÚÜ:À²’¹«0Qn`“É‹i¿Ã6u[®ÙÐê8‘˜¯}¿Üô‰½ZÚ6ƒÚö©¹s(`j©KÖ fXOÛBæ±±é{9uupè¦> -endobj -9 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 671.061024 504.679134 638.361024 ] -/BS << -/W 0 ->> -/Dest (rke-cis-v1.6-kubernetes-benchmark---rancher-v2.6-with-kubernetes-v1.18-to-v1.23) ->> -endobj -10 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.750423 648.111024 496.429134 633.111024 ] -/BS << -/W 0 ->> -/Dest (rke-cis-v1.6-kubernetes-benchmark---rancher-v2.6-with-kubernetes-v1.18-to-v1.23) ->> -endobj -11 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 626.361024 504.679134 593.661024 ] -/BS << -/W 0 ->> -/Dest (controls) ->> -endobj -12 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 485.690364 618.411024 496.429134 603.411024 ] -/BS << -/W 0 ->> -/Dest (controls) ->> -endobj -13 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 581.661024 504.679134 548.961024 ] -/BS << -/W 0 ->> -/Dest (master-node-configuration-files) ->> -endobj -14 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 485.521907 573.711024 496.429134 558.711024 ] -/BS << -/W 0 ->> -/Dest (master-node-configuration-files) ->> -endobj -15 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 530.961024 498.679134 507.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -16 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 510.561024 491.179134 495.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -17 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 491.361024 498.679134 467.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -18 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 470.961024 491.179134 455.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -19 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 451.761024 498.679134 428.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -20 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 431.361024 491.179134 416.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -21 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 412.161024 498.679134 388.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -22 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 391.761024 491.179134 376.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -23 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 372.561024 498.679134 349.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -24 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 352.161024 491.179134 337.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -25 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 332.961024 498.679134 309.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -26 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 312.561024 491.179134 297.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -27 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 293.361024 498.679134 269.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -28 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 272.961024 491.179134 257.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -29 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 253.761024 498.679134 230.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -30 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 233.361024 491.179134 218.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -31 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 214.161024 498.679134 190.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -32 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 193.761024 491.179134 178.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -33 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 174.561024 498.679134 151.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-ownership-is-set-to-rootroot-manual) ->> -endobj -34 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 154.161024 491.179134 139.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-ownership-is-set-to-rootroot-manual) ->> -endobj -35 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 134.961024 498.679134 111.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-permissions-are-set-to-700-or-more-restrictive-automated) ->> -endobj -36 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 114.561024 491.179134 99.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-permissions-are-set-to-700-or-more-restrictive-automated) ->> -endobj -37 0 obj -<< -/Filter /FlateDecode -/Length 3687 ->> -stream -xœí\KÜÈ ¾÷¯ès€­Éb=Ç<. d#Á"ÈÁîµ7Æ69äT©%öŒ§½öccf$êS‹UzÁÙËÿo@~å.ç’b8_>ž~%-Œèì}41ŒŽ ±g “çLL ŽS! Lò‚Aô†ï@ŠŽ#ƒ‰É¢3gï ï,:³/&¦ˆÎ²7\ ½èL1yÃЋÎX¢‰Ñ1ÃP¢”ÉÄ èì3ÃÇDgP ÿAŠ.”àML@r”c&»J61ƒ©þƒÑ»9Ã0ŠÎœÙÄ$Ñ™!Ã0‰Î!‰É¢3%,†ÿH¨ ÆC±ü§ˆÎBˆ…!/:—bøyÑÙK!±0 :{HÙðÉrŽJˆ6&:ʉ³ácD(²á?DÙQ’ß&G1b6üG²“#.`cDgFŸ £(:‡P²á?$%(e#5ȧløØÆ‘™!r6|Œ¤ Š÷„lùÔAñ!²0Aê ÆløR‡9ƒ‰‚Ÿ ÿ R1QI†ÿ©ƒÏÙÄHD.)þ¤"c´1¢s`N†ÿ©ƒbÁ`cDgJ†©ƒ’£1þ¤"$°1¢3xŸ ÿ RÑc±1ÙAᜠ“:à¤^¦døLSÑÆD'ƒâdøK”V‚£§høK”R‰Ñð–:(s °1¢3 ÿa©ƒ@PlŒè,Å0þÃRe7EÃXê ÌG£½d<Ãdž%Óü®0Ë”g™ ê‰ÃeqKe2øie²nvDó>z^Ìüû‡Ó/ß‚óçþsúð‹iYE¥âY<àÌRc[ÊxhZw¹,Z¤ÙÚ­@š<ÆTÛYÉç‘õ“$sôN†“¹4&±WrqóŽ.2QÚJ‡Ž/§QN,i¶µ=ÈSo¥õ©Tô8¨.½¬ŒÐåjY#\ÉAf Ýz\Iylw <ȇÑ=–Ú·÷eÍC<à!®yˆ¼•¯\&»¶Ò xÀà Ï·ÆÃÒcÔ"rÍC—®yèò™[ùàÒC+éŠ8àaŒ6ú¡ÇÁRûö¾œ¼+¤ÿÎ×vˆ£ùxºÇ¡¸ò扗} Ñ—½ -ѨÿÒùzã !:j¸}ÜÑKˆ¾„èKˆ~¦ÝŧOtB÷%D_Bô%DŸ5Ñ]Ý!šnÉ is_M/å4‡ó|lhçÖZ? Îä<ø’y»º¦BÎ&&:¢˜ LFGœ3‚…Éz7M”0E/(cârŒIÞ»€!a¶0Ñ…ãt³áèBÌ1š˜,Aê#E£7Hãt³ê£7H(ðtîsä`p‘:N™Áà"…¬7áB¶0Ôq|¥¨7)C@ƒ¯£–%Ê_’#„{q “]Ohq‘ƒK„X,.ŠÞàMLt)E$ƒ¯,y:•ŒÞà+{ Dô ¾2—B0¸6\ŽÀà"£ä }¶0¤¹8{6øÊ’2 y_9WJ6øÊ,¹.Qaƒ¯ÌÑ•Â .rÔ)‹‹Xœ§’£…Ñg "d²øÊ’‘ÅÞâ+'A¤dñUȦ,.¤¸C( .¤:H³…pP(ƒ¯"›,£70H)q6ø’R¡7?$-® .JGž‚Åi Éò|±øÃä(¤@_â^Ž¤hxƒ¯Ù‘TÁdð%L¹DÁâ"É´˜¼ÅE&'Y“‰)zƒÅWa½¡Ž`ðÆŽ‘滔ǠäXò<lÈlˆËœ :Tçâ‹ BvQo'lI»º Ðg'Ãmñtñ †íÄ`„ó>üð>À;‡~õæTæg¦ŠL/õÖ2z­Î^’/j‰7O¿üÍw¿þîO–¡¼ùpúÛ+1xŸüúïç7¿;ýö4þÔæ£Ì;o4_>½y)4Rú?_ó2ƒŸ¯y}R?có‰nRûŒæƒxþ-jß>£ù”oRûŒæ™à&µïžÑ¼ÌÝ,jY»x\ó—GãÔüdÝD]X“—I”vðÛ?þé÷ù㦃ôéú'oÖs›';°øyYGgÅ·¬ÿ ßÉÁ,~žï°KÖ:@Ïh>Ú¥Íó32¾G;²´ý¸jßíqq{œžÄôuédñ2Í<ŒZØüÕÇe€Ù{Kÿúd#³.çc,KìðN~¼½’-ý;Ét©¹`ôo=Gÿ6Ù¥·¯x=Î8Ëø²´¯ûog¢ê6õ©çÓÒÞe–OçÄ®GÕQÛÙÓ¾ßÊf›ïÛUÐí†U[zÑÃb«ª¿Æ›;\ÌvÅ1òªÝ]Žâ8 -³N“ž—¹-ícâ[Ž}/Û†öiàÁw»O6¿ôó+“£~7P4]Î?ûÅ/õOÝÙ´eÜÚ²öÏÕÇxÖµúܤwé:Þ‰S™«ÇÛœ¦¯€Ó;ÇØt>.?KŒ­e–Ýdͺ±Û}ø˜¾ÍG¾“}b—MÜänwæ×ô¯þŸãÖÀ_&ðd¶çð6Ñåk%ú«ŠF­1ïD’.îo“ôöNÙq°{uDÝŸìþ¡Ûü0X²êÛ¯*bM£¦+£>:²»³üôôVÚmgx÷sv†»EµiHÜòvôß•H.™ó\îC$¾ ñÕ’ny°+t»UgVñþ5êùæò…¦ÙÔ/KØ4omúd®ú¥PР]n•Ü¸:6;,\ŒGc@•VÄ<.䎯Bq÷˜{x‰–mÊvcì2DÎâ% —{;+Vß.Çü2`èý>úŠSY¦*Î2ZòŸgõ›èpõËâïe^p±g¢={ë]Æm>Ùþ•ç´ÏÇlX–Õ†wºBWmXu]e.x¼y´eìX3%)[êÓ$ë¿jË;]kþø~]9ÚT?-Çß-ãI³Ý®2ürA[nä{Wâ\—PÕžù'¶g\d—µ½Ú±GÍÎüfö}§þW3Ë®¶v½åÔÜض—xm3ø:«»\Ûä>>SÂáj­úÌ.’=‰³GøÅ^þk3âËb“‹¿KüÎ<™vŒ[;~>e@‡K´Jè.¨­&YG$|? `˜:×ë*SaÛú‹þíexÜ ëLM/ô„×X^Íž¤×cÓñeÍÿ)ì-÷së=ãÄ¢™÷é3äåLɥ˜âô|Ôé_èàR!—¦' tÚŽ¡Äù!«' –ÞõÙ70¥&s½}¢ ‰D¼»z ^–ZµÈÄyå@•«2øbM*Üù­WЛÙxNÕ×ÆÈDA™å¸$ÑÆý²½ç;¸øÕ€iíËSûu’4è3N’¦1´ÄxuëÿÇgÔÏž°~–Áë§f(0âô¥¥4ÉÏœœh3VÞ`²4œN¾ÄõcC-çÒgÆA?2¹kØYðósOú]!%@¿Žç-ëg‹¶â¡½Ô·çWâ Ïè:Hs?mðØi¨²U§µP´t¡ïfWYk1UârZ5Ë4â¼k¥Ôæ]úÑÒ<²ÒŒÈ#+ñ -«íÖŠ×fKÔ&@<ºsê£óÙæ’yGdz›4t$UÙußaÝw­¶ça¡+ûÌÔ÷Óø -ùIùïSbg÷ö=ÔêÒävIÃëGW!Ò¹èt'Ç(9D"«æ‡k%O>Ǿxr}‰¤^<{}û\¿Å âo*<ÈrûÀÐÔòöþÁiúRóΞf€Ã·ãö_7 GËG ¦+Dm7éR‘KÖ·Ô}– /üÁæxuìÛÿÏ µ -endstream -endobj -38 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 37 0 R -/Resources 4 0 R -/Annots [ 39 0 R 40 0 R 41 0 R 42 0 R 43 0 R 44 0 R 45 0 R 46 0 R 47 0 R 48 0 R 49 0 R 50 0 R 51 0 R 52 0 R 53 0 R 54 0 R 55 0 R 56 0 R 57 0 R 58 0 R 59 0 R 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R 65 0 R 66 0 R 67 0 R 68 0 R 69 0 R 70 0 R 71 0 R 72 0 R 73 0 R 74 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -39 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-ownership-is-set-to-etcdetcd-automated) ->> -endobj -40 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-ownership-is-set-to-etcdetcd-automated) ->> -endobj -41 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -42 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -43 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -44 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -45 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -46 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -47 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -48 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -49 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -50 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -51 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -52 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -53 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-directory-and-file-ownership-is-set-to-rootroot-automated) ->> -endobj -54 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-directory-and-file-ownership-is-set-to-rootroot-automated) ->> -endobj -55 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 473.570079 498.679134 450.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-certificate-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -56 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 453.170079 491.179134 438.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-certificate-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -57 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 433.970079 498.679134 410.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-key-file-permissions-are-set-to-600-automated) ->> -endobj -58 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 413.570079 491.179134 398.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-key-file-permissions-are-set-to-600-automated) ->> -endobj -59 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 394.370079 504.679134 361.670079 ] -/BS << -/W 0 ->> -/Dest (api-server) ->> -endobj -60 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.982112 386.420079 496.429134 371.420079 ] -/BS << -/W 0 ->> -/Dest (api-server) ->> -endobj -61 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 343.670079 498.679134 320.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---anonymous-auth-argument-is-set-to-false-automated) ->> -endobj -62 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 323.270079 491.179134 308.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---anonymous-auth-argument-is-set-to-false-automated) ->> -endobj -63 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 304.070079 498.679134 280.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---basic-auth-file-argument-is-not-set-automated) ->> -endobj -64 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 298.670079 491.179134 283.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---basic-auth-file-argument-is-not-set-automated) ->> -endobj -65 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 264.470079 498.679134 241.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---token-auth-file-parameter-is-not-set-automated) ->> -endobj -66 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 259.070079 491.179134 244.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---token-auth-file-parameter-is-not-set-automated) ->> -endobj -67 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 224.870079 498.679134 201.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-https-argument-is-set-to-true-automated) ->> -endobj -68 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 219.470079 491.179134 204.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-https-argument-is-set-to-true-automated) ->> -endobj -69 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 185.270079 498.679134 161.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-client-certificate-and---kubelet-client-key-arguments-are-set-as-appropriate-automated) ->> -endobj -70 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 164.870079 491.179134 149.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-client-certificate-and---kubelet-client-key-arguments-are-set-as-appropriate-automated) ->> -endobj -71 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 145.670079 498.679134 122.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-certificate-authority-argument-is-set-as-appropriate-automated) ->> -endobj -72 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 125.270079 491.179134 110.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-certificate-authority-argument-is-set-as-appropriate-automated) ->> -endobj -73 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 106.070079 498.679134 82.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated) ->> -endobj -74 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 85.670079 491.179134 70.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated) ->> -endobj -75 0 obj -<< -/Filter /FlateDecode -/Length 3500 ->> -stream -xœå\M·½Ï¯˜³Ó$«ŠdI|qÃA„Fƒ4ÖÚVäò÷óØÓü˜™ÞšÝQ%Vê®®®f¿GV=²»7ì=þ~ðOáàJÑœxø°û¸ó.‹ÇŸÅãdg9ü'üÌCÈ®žMe|,.Åè9ï?ÌÌ5ÐþærÅ<‡yÜýõ‹Ý/۱žœ÷JÜÿú~vÉìr1—§] -n,jâh¸$—¹xŸŸvÑèrJ”ƒáR\.œ)=í<»â)å§}‚w%Nbù$WHKTÃ'FW¤eËTä$bàmVÖhÐØ;ð‹ îÈ)…$A¢SôÂ`a(Åi*T *Bb§%e¶0Ì$x ÃœÄÙ°àã_ âÎ'½…¡Šó¹H20ŒmÖ¤d`}v@ªa äQJ†1¨C c²'5ús¤àB)Y #e‡v‡`p™®ÇÅÀ0²:ø6¸ˆ"."ç Ã\,^²ÅEB›#ÃÂ0“£É[\duD Ý°ˆ#aOåHja¨Ù,l?íCžã@40$¯ŽcFR0|‚8f)b`H18FjŽ†³ã¥‘CÚ@ê°|ÔI@20$'”S60$ N$y6ú3Iv’™½…aB›5ælq‘Ô¥YfA Å­[\”àç’, Kv)% WÃGɥ¢†ª.{¤ƒ ö‚¢Œ‚a`È!¸LÈô†ŒÍ,¨†ÉeHž``ÈmVd`È$®B3|ªÀB †ÌØdEB4|„\I%²…¡(Ä\BG4|’8õŒŠaø ¢¸ãå“]R‹ ÔAî“…!ê B$E‹ ÔAU ACtvç³\ˆ/Î#±FC ì J1¸@ÔeÒbù$çAF0¸Š.ø¤–V*.D!K¬ -³ LÙR«"Þ…T+ªå“P—=[zUÚ¬¥XzU Ó#Ä”¥W‘ ]$K¯b;Üz±ô*Š€‹*ÀÂR=*º¡ÑŸZ|QK¯b`9Ô²ôj‚VGÍÈ–^MÐê”P - . Ga¸[z5A«³Gƒ‹­ÎAƒ¥Wчî#dp‘ ÕYD-½ -!ï Œc2¸HÐê ÉÒ« Z½ªpK¯B9t²ôjÊÉ •¥W‘R eZzÇI†Êr¾Xz5{ºŒÉ®5}õÉ%R±ôjÑ%A:40̘Rãþ£¥WsD›jËÀJóiò–^Åu’ÄÀÝ3jÍ–^ÅŒSêâ-½ŠIæÔè9a¾ì¡- æËP6jq‘1_æ –^ÍóeQóe( K¯fE›5%K¯fÅ|9ˆ·ôjATTnK¯ÔA¤Â:¨ÙK¯ÔA”¶ôjVGç)–^-Ðê˜TFK¯huT&±ô*d(æÔUÐ>Ðê YzÌ©«2| Õ±ë-½Z ÕÕfø@«©É×ðV¹&Ës|­ÞðV¾fù`¾ëÅžöQhõˆb`éU…VG–WK¯¢*aNÉҫС®fK¯*´:F²·ôªB«cT¥W¡g1§–léUL 0§æ`éU…V'…<´0„VgŒ.K¯*´:æ!ÑÒ« -­ŽùƒXzU¡Õ¡ûÕÒ« -­Î…ÉÒ« -­,½ªÐê†êÓ>Á{L˜kß°½ÇŒY2’°å0eÎ)ØK¬íVFâ´œ"&Í9úÈu zcáZP‹ÐÊmUé\Œ|¡†·l{××ýi÷Õ[ä¦ýOÿÜ=|±®´_úí›—ïšlш¾j;¯K“‘ŽÞ|Ø}õû~÷Ã÷ƼyØýí7uõ?åë¿ïß|»ûæ ‚¿4|Â,öJø·¯COvøw7„Où¸~õDxè²@¯O(ƒñJx¹!|&“ÚÃ3ú½Eí­á¡,jkøüúðBá*µzCx(·kÔÞÐïå«ÔnýaQ‹I|àׇǬۤ¶ÜÖs2”‘Em¹­ç@Ž›Ô–Û2¦Öç2WÂßÐsªÄ¸Fí =Å7™ÜêEÊÜŒêñQs\ú¦!å!1ývlÆO•²þÿÎ{ ËòlÎ ªÔm¯hë1ÁHaø1·Ol«N]~Ÿ2ŽcÞqŒuÀÿ?ŽË~<^³žW…×}Î#æâ[Ï}»ÆôÇcK;×v,q°Ïh7?ýZŒz/Õ§>ÿ±¸ncWŸð*^÷§{ZÚì§ûX¯çf\®sÝ9î‡ÿpb”#ž1™x¥s¼îÄôs²y¨ût.ê±Ú¦¥‡5N:Å|ñKc¹7Y9xXãÒ„éÛuo£žòQqžq‰ë9­3g+7K»l-üb8Çï>¼D.užuþŒxáõXµó×QqO²^+ž_‹5gõ{x?éÇûðC¨·ñ:?ò¿ÁÑzÎÃñ'fKœ)§½| Õgðñ‚#jœ„õÚkû®ðRçÈ×yIŸ/´bÈs×1Rù‰g±¼ß·cÛGßWò“Ï1­1X×ö‘ïõñZ[»ïš¿­võ2ÚÊ«íõ}„!6¯h>Ά>·>²œ£Û¥ÖñºÝâoé”õŽçøÝ'§Ö÷ ž¡EÊgÄKXÛÐôaZýyâ§ÆNãïajïËyÊçxÞ‡' ñ9ÚD?#ž>E{GõŒO¡O$ÉsôÉÛψ£“9ëªéۘʃ'^÷/®ýrnÊ9†÷á&…òrïuˆÃˆsÑ¿Ëê#ïÞG›7/˜çáÛø}Ñ\ø&ŸO²Qßëx†¸÷ZÄ3¸5Ï,c$¿ëŽíhO¹´…†ŽºÇéZí–û­#4?9~OöÕOŽgN×j¬Üoþßñ¤–“½}?jÜ’ÞÞ¿.š§Í;ÓhÏ}0,Q¯Õ@¹ßý|‘[ jyôáx·†u¿Ÿw÷ÁöN¸•x­>ÉýæÐ/Æ­ù®qÛúÃÝóccºå‘~ s­Vý9~Ë94¯±½h–‰Ÿûñ¨õ»©ë<Þiž{ëúì2çy¿^Ÿ>§üjNåËÿ§õ‰ëœÞiŽ| §]ÿÿø‰ÆçÊåëø õuŸô%t}pÙŽfA«„Å¥úÂŽî)»¬sZ^Úýò—zcYÉååEŸ}¨_\JÐTä‰}֫דÂÅÃÓ´•9ú÷ZìXs}¶>;ýæ»ïÿð—ïƳÓåùÕûc?[ú[:® SãO§þÙÖdp¾Œ÷wlrÿ­ïÍk[‹Ñ£}=MýcÝÞêOqík …}=~{64µ§­¥õ{hcëòÝŽ;A¹d¸Ô×=ÒK}=7—û¾~‹•|Zß絛w-H‡ý§.ß>ì^ëçÐÞ -¼?~+=í¬þÇîvay7¼þA¨_¥Õ-˜ë«ççæGЮõ­þ 3×WQžg®ÆuÓ÷Xšc·v]XÏ=©¾¨vÜÔÆu³¾(7ΗvÇsl{u/¼aÄ™)\šëÝ{V³,_öÒº—zS$µ6×Ï{@ž}GÀqâU3R×d”K£Ÿ.7Ì“Ú|cßÔÎ9Œ½é3ta¢¿ùÆ<Ñi;\–έ›Öº-6t£ï6š¥Ý*ͳ§ Óa·aüÒ·Ðõ;–~|b-º²KëHÜý´ùm˜»M¿5št¦r÷*ãTmȦŒ®N¥¿a[0Z­HÓnðÔ6OÍ©2иöeê7ñÂxØÕ¼sa~ÜIÞð–‘+¼´®7iŠÛÍÅþ2¬¹4¨R趴O\ÿ=¦ -éÖšy–úâHhæ½q‘:6»ù¤ÙžzÖá-çá;ÁU¿È§ ;b¿á_“n÷îˆX'È&{ÖÖ¼š¤zò85‡Þ뺹¦°†OÏ•³Ñ·FàrÃ,½+?ž˜G;¥Utöv±¦î.=î0§ž;gsîŒ×_”ÐnbÔ4õùf<,…Ï͵ë—v‘aŽ¾W»q±É8†¤0Ì#>.æ6Z{v -ª}s ²­Š{Øý¼[Ê\¤úÙ!ºgýf‰NdWØÿ >ßâç;Þÿ±½‰}¬ãÑ÷l{n=1Ï•,Fj^5ǨÝ؇ÍdŒ<¥ýaeøJë ½ŽÇ8åk§û‘Ø›gè4ô£³Q†“9ö®[{Ä@F ãá Å.dnåfìã.µ›óõ÷Mì{‡ª;óí`·Œc4Jò~ -‰½ê:J:ŽrœNÄλÓu´ì¤™µCÎéÐ×öóLÕkM;“Œ¨»4Žñ(³sèf[îbrá0Ìá$0‡ù¢zÒžF¿zxÝÌûÚäÔF@µöÔÞ­³µßXµö„·m¥e7îöšC˯*iƒw ÔcÑ4ö¨‹œ14†´à“¹NÖ2N[d¾(7ÛÉE[¡èé¢>½i¶1·6¥õ´ËrÇe¥Ü5Å°þ<#-3+D™YI¾5n«x]-Q@2wç<îÎÏ®%ˆ-Ígw+Oj¶Ëkóéµ[µÝO·Á£!i(S?N“ ÏWå¿×ŒÍ7µ·¼N–FBéK…šC¦H{­r§Ô_„ÁRgÍ\¦µ’Ÿc/ž\.‘´Å“ùJ “vª¿æàÃSMxĽ~` -µþ"²'NS¿yK/àÉÀ¶ß«'GËŸu…¨ïæ:U-õó;_0È5d6׊Î>žXþþ‘ä§" -endstream -endobj -76 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 75 0 R -/Resources 4 0 R -/Annots [ 77 0 R 78 0 R 79 0 R 80 0 R 81 0 R 82 0 R 83 0 R 84 0 R 85 0 R 86 0 R 87 0 R 88 0 R 89 0 R 90 0 R 91 0 R 92 0 R 93 0 R 94 0 R 95 0 R 96 0 R 97 0 R 98 0 R 99 0 R 100 0 R 101 0 R 102 0 R 103 0 R 104 0 R 105 0 R 106 0 R 107 0 R 108 0 R 109 0 R 110 0 R 111 0 R 112 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -77 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-node-automated) ->> -endobj -78 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-node-automated) ->> -endobj -79 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-rbac-automated) ->> -endobj -80 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-rbac-automated) ->> -endobj -81 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-eventratelimit-is-set-automated) ->> -endobj -82 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-eventratelimit-is-set-automated) ->> -endobj -83 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwaysadmit-is-not-set-automated) ->> -endobj -84 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwaysadmit-is-not-set-automated) ->> -endobj -85 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwayspullimages-is-set-manual) ->> -endobj -86 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwayspullimages-is-set-manual) ->> -endobj -87 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-securitycontextdeny-is-set-if-podsecuritypolicy-is-not-used-manual) ->> -endobj -88 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-securitycontextdeny-is-set-if-podsecuritypolicy-is-not-used-manual) ->> -endobj -89 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-serviceaccount-is-set-automated) ->> -endobj -90 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-serviceaccount-is-set-automated) ->> -endobj -91 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-namespacelifecycle-is-set-automated) ->> -endobj -92 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-namespacelifecycle-is-set-automated) ->> -endobj -93 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 473.570079 498.679134 450.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-podsecuritypolicy-is-set-automated) ->> -endobj -94 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 453.170079 491.179134 438.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-podsecuritypolicy-is-set-automated) ->> -endobj -95 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 433.970079 498.679134 410.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-noderestriction-is-set-automated) ->> -endobj -96 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 413.570079 491.179134 398.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-noderestriction-is-set-automated) ->> -endobj -97 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 394.370079 498.679134 370.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-bind-address-argument-is-not-set-automated) ->> -endobj -98 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 373.970079 491.179134 358.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-bind-address-argument-is-not-set-automated) ->> -endobj -99 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 354.770079 498.679134 331.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-port-argument-is-set-to-0-automated) ->> -endobj -100 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 349.370079 491.179134 334.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-port-argument-is-set-to-0-automated) ->> -endobj -101 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 315.170079 498.679134 291.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---secure-port-argument-is-not-set-to-0-automated) ->> -endobj -102 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 309.770079 491.179134 294.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---secure-port-argument-is-not-set-to-0-automated) ->> -endobj -103 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 275.570079 498.679134 252.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated) ->> -endobj -104 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 270.170079 491.179134 255.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated) ->> -endobj -105 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 235.970079 498.679134 212.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-path-argument-is-set-automated) ->> -endobj -106 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 230.570079 491.179134 215.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-path-argument-is-set-automated) ->> -endobj -107 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 196.370079 498.679134 172.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxage-argument-is-set-to-30-or-as-appropriate-automated) ->> -endobj -108 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 175.970079 491.179134 160.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxage-argument-is-set-to-30-or-as-appropriate-automated) ->> -endobj -109 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 156.770079 498.679134 133.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate-automated) ->> -endobj -110 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 136.370079 491.179134 121.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate-automated) ->> -endobj -111 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 117.170079 498.679134 93.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxsize-argument-is-set-to-100-or-as-appropriate-automated) ->> -endobj -112 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 96.770079 491.179134 81.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxsize-argument-is-set-to-100-or-as-appropriate-automated) ->> -endobj -113 0 obj -<< -/Filter /FlateDecode -/Length 3631 ->> -stream -xœí\K䶾÷¯è³Ë,²ªH†I|qÃF Œ ‡ÙƬ`ÖÀ:‡üý%‘,uK¥i­ÀÌ g¤Ò§b±ž$Åœü~ò'! )åÈx¾¼?}8¹!’“Ÿ±8//ý ‡rwHgŠ4¸ì3„ó{Mç4ðD~rÚ!+.O§~vúyœý3#¹ó/qàDÉ‚$7pL¼a[ì2€q)"¸m8‘—\01 cð.[‘80˜/"ûì\²0"³‡ì¢ "3`21(2»[‘Ù961äÊY˜4Pb41ŒÅ˜èã½3|"Ä&&‰Ì”œ3¼’ÈL.›˜,2cHÎp1ïDæÀÑþãÈì3›™½Çlø—(% `b¼Èì’φù 2;€lø<`FgbИX®Y˜4`ÌÉÄ -ÆÇløg7 gÃ<‹Ì”ÈÄD‘™³á?>ŠÌˆÁÄ$‘9DŸ ÿ‘PŒƒlùO™Å &8‘(g™ "³ƒ˜ ÿ‘,7„Œlcx)R2|,/‡ÉðŸÒ¢üµ0ˆC`öÉðÉNC  6Fd&ï’ácEfÄœ ÿ RCˆÉÄH ÁÅdøèfd6Fd¦døX2(ÞƒÉò©ƒâCÁ ÔAŸÉ'ä>%°1,pÑð”:ècÈÑð”:(ñœLŒÔAO9FÃPê 'Ï6FdF¢høJ ¢™„høJ”í£á?(uÐC#2ƒsÑð”:èÏ6& )EÃǤ R/c4üG†©‚¶1§5,"eGâg’ªÈm*ã IÝé2i¶µY4:s¬|ô©gý&ɽu“¢ 3‰E¸¡‹›wt–Ò5U5|9iz I³·¢ÇÎ¥µÈ!W´îT§^Jèô¢™#ÜÐAF ÝZ\PIóö« -VtÕ{Õ¢ÒÔº¾/K;ð†xi¦kúÂeÒñšze¿aEOƒÃf‡¹E.EäÖº´C§Ov¼¦+—V-.¨ ;À†T`´Þ«•¦Öõ}9¹!‡òs¾=°CTõæýFèn‡â›•%^Cô5D_Cô&D¹üÄóíÁ3BTK¸ì=¯ôè5D_Cô5D?Qˆ®‡âóº[¡û¢¯!ú¢w tOˆÀ«Usݲ”ÓDT3æ ºf´òh­ßç08Ï.ÅëÙµX*È<}\0ÝÄðrô &ù1c “ÊChmaÊ -ÓBç&ºò Š22G±—g††ÌQ”ÁgÈEÙ‘=FCæèqH€L–ÌÁ ‰EKlaxÈÝøPg ƒo¦…¼ML– !`20e-Eþxo`ð‘³%³Ä4Äœ¢%s ƒÞ¡%s”D›Ðƒ%s¢!Iɖ̆±%³$SDƒaÓäBKÎÛ˜<ÐNì$ vb'yIL;±“|âN줆¸;I -ZÚ‰„4¤ØI$õw'vI¾ß‰$9ÊíÄŽ8×àvb'IŽ‚ØI’£`'v’ä(¿;Ir”߉‘d;±“%G…ØÉ’£p'v²ä(܉\âîÄN.qwb'KŽâØ‘bÚ÷tlb$¯îÄN&âNìd–¼º;™%¯îÄN.ã˜ØÉ’£ÜNìdÉQn'v²ä(؉,9 -vbœ$)¿<Êãc¯›è¹Y–ïõ./‘0¸Õuy §uøæÂü‡•KzsÊÓ&¦,ã½ò¬×»ª®h]âÏoÞŸ¾øËþỿKWÞ¼;ýëKçJÌ}õïó›oN_¿æÏe/E.í°ç—³óÈXÅfŸî`/åwØ?¼œ}ytïwØ¿½ƒ½ ,Ó>È'¼œ=z2MûpŸç`L¦iîó -`šöá>Ï¡„¦i ûüqì罇, ”=,(Z?³/3Ýàƾþö»¿þãÛ«îpÍèìÀº—}°ëá>ÏO`Va¹ƒ=Ûõö¾ÀÊ`Vaw°g;°ÞÞ™ñ·#ë–ÿºÛûÙíý¸5ÒÕAÝà0zpë¼Q>Pø«ÿeÔœóIB#–O9–kIã5’9/Š\2m´#·ŽŸzáÄg<æ ?^/÷]ä¿›¯ù¹½™ïØFáñ0Ó+æ2ñåàùº¾ŸÕq˜?4·5µ?8g²u–ݲ]©ENúí±Ë=òušŸ|Š<“¯8EZð]µÏÃÁö©ú*úȳfÌ}ÏYÙºýðÝL—>Î×Ã1¶«ºlÇT}ÈÔ#_ëñ ûd’ÉÆ®}ÞþÎìóø•ç/g—Bƒðð8Ù­¶uLŒYºóp­»Å©cl8îÖÝ·áåXV]vªvÑv6¹O{¼6~¾}ùQ÷ñùvækýþ6v–Þàm;su«q·¼fc#öVì|ÍKëlÌJGÅù­˜:ö×:>¦öêeé`ßðàxçYï—¥~Û5ü”yøȱNÙéý)Æ:XbíÛ…~¥<Œ[ ¿ÖÙA¶ ¼7®áés¸-`Òÿ¨“ˬ“:ðÓuûFZÅž[«§{f[>üZcŒ×úûmr]ÙY¾3¶)vŒ/³ãÇÚË]÷{Å^[ãçèm¬ó8mÍöºOM½…k½d¦ƒ¤câªÈÊÿY'ázÞ0÷ζÒ:¬ý[Œ#jL¤õ~̽67«|ëuÅWÆ{æþÍöû1ÎkL}¦k}þxQ<®üëÙvìK‡P‚i~´°³tˆÝ˜­Ó¾'˜ª˜±¸¼["Ta`VþC½n)‚“žLm¯êðüÿàÑ éHv³àV«-s{Um¦hm|Þv9F{«”uè -¥Ðè~³Ó6g§Ü3Ç¡¶lýS# jσ*¨^ÅÁyæRg6ë/oÎúø*HÒŸÊŒ‡¬Ö°SUk©çêwUwÇÙÆÒi‚›YÞb%îHû%Üœ­Uû½pôò©V×~YùýùmÜÖò;Èo)oÎúª} é§uuú9Ïðif}7g}Õ= -Ôõ¯eô_A‰1Ôœ÷®ëskë¯VÕ°®xò­ßþ6O 2_ëö 'ÎmNñªÑ~<„5¹ÍK”-)Æé£oÇ¥Ytí±×Ø¿põm…ù{çvó3×ú\7’´B(ãŲ§,ŸCb&yÜÃtúùc EИÃÇ]Jeªá‚,Ãú]JsëeÛ„eª9Êä%Z±”€ô“ÔK™6†j¬üª&ê6Ä:PwÊÖÝoôôs è<Ñ=v?Ñ>³æ~ö½t°à_§JžÅ7«iöíΑ'*[ÄRÞeàhÜ'2n¡ôe'`¡Ÿ)"'_í£¦á¼qëøúª 5¿Ñª¼=Ê7¦×]©“?íx*/ã)({Fa éHÈå]?×ä'1{®_9_±ì£ù8r!·®ñB¨ÀF»œ ”¯J\!Õ­uóòe‡éÐSíM¹ŸêmaŠO_ÏJL ¯åN†[ré}ùfÍD¦ñõLa>ã& -q•Yˆ!jlgØoÜ%KêQDº%:Õ\'KLæŠõí07›ßEתeþŠõQ™Ó"^NÒl¸¦>­R=4š¯Ú-/Ë©ò)¡r£RE6ƒ`³Ó -©|Må†ø¹«¬QŒÙ®+«ù!ÍÄT .WÜ -érZÅÍܨY*6Tê·æªù.JwõšâWh£Žfª¤‰'u -e›lµƒ&s±@µµKÊoü ñr*yç†üt¢¸‚¦ž+U×ÓÄ ø6rrÝ_:5¦ª*†F+[íÊß)UP£–Ì3”½-Pè³ÞH‰ÍF^*IÓ¹e\Ó`ìX¥.ˆ±ù¬Ö—L8Vð%é6tÓØ‚ªT¦èãžìÊš[òX’¡y]#—VõÓr¥&º*„4×ÉÔ\ùiAîrR UqöÚƒrwj|;™[î|ÒäØ,^ÞvW;Ñë+Ÿ¯ÄËXØðš\\?ÕF:Ù»VízcŠØ“¤ÐÉ=>ä­-;AÎí°ÙZŽœ~:eÎò¹|͵¼ðhù:8ÿO0ßÈç?'<ÿ­n#ŸêxyÞܾo¹uAÖ•ÌûPM¸Kö>7b Eô¨Ò~'CêXªÞÐê¸÷*_c¿ÝõÄ^‘Ð2(´«šH]‡Šì›ëzJ3d/WZl™{mÓ ötj礙PÞT¥’Ý9MýZè%ù¬XÊYö’å ‚êF9ÑLåT5˜»d 1‹CêFšê‹ü¨MJ[êD #Êiè×°—YͺÒÆ^(‚ºaÁA7šò¬ýâðºÛî³È\# P[joÔ'Mm+Ô–ðÖ©a<ååi«9á\Njðv 5^AÅ^hƒœ}h‹í£Ššúm3€t£Xi‹Fk¡hé"´oÏPç«L<ßVÕ2ö8­j)¶1E§þ¤5MÚ*M‰¤­Â7Ø·V¼6Z -mDÚcïëÈ6–L+4Öw7*vd¨´Û¶qÙv­¶gÕ ì‚p™º~Ý _”ÿ^;«›Ì×P‹¥HmIÕ7•B >œsî$fÉ!åË•1©µ’gßc/žÜ.‘ÔÅÝ’˜C&íÁEÛáI.äý ŠÕü•÷Û²[íÒó°ù ¶õ¯„!ÌoWˆÚi,SEÊ ËÛ’y†è6žn§›e°ïÿ¶;dd -endstream -endobj -114 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 113 0 R -/Resources 4 0 R -/Annots [ 115 0 R 116 0 R 117 0 R 118 0 R 119 0 R 120 0 R 121 0 R 122 0 R 123 0 R 124 0 R 125 0 R 126 0 R 127 0 R 128 0 R 129 0 R 130 0 R 131 0 R 132 0 R 133 0 R 134 0 R 135 0 R 136 0 R 137 0 R 138 0 R 139 0 R 140 0 R 141 0 R 142 0 R 143 0 R 144 0 R 145 0 R 146 0 R 147 0 R 148 0 R 149 0 R 150 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -115 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---request-timeout-argument-is-set-as-appropriate-automated) ->> -endobj -116 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---request-timeout-argument-is-set-as-appropriate-automated) ->> -endobj -117 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-lookup-argument-is-set-to-true-automated) ->> -endobj -118 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-lookup-argument-is-set-to-true-automated) ->> -endobj -119 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -120 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -121 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-certfile-and---etcd-keyfile-arguments-are-set-as-appropriate-automated) ->> -endobj -122 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-certfile-and---etcd-keyfile-arguments-are-set-as-appropriate-automated) ->> -endobj -123 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -124 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -125 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -126 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -127 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-cafile-argument-is-set-as-appropriate-automated) ->> -endobj -128 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-cafile-argument-is-set-as-appropriate-automated) ->> -endobj -129 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---encryption-provider-config-argument-is-set-as-appropriate-automated) ->> -endobj -130 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---encryption-provider-config-argument-is-set-as-appropriate-automated) ->> -endobj -131 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 473.570079 498.679134 450.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-encryption-providers-are-appropriately-configured-automated) ->> -endobj -132 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 453.170079 491.179134 438.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-encryption-providers-are-appropriately-configured-automated) ->> -endobj -133 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 433.970079 498.679134 410.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-only-makes-use-of-strong-cryptographic-ciphers-automated) ->> -endobj -134 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 413.570079 491.179134 398.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-only-makes-use-of-strong-cryptographic-ciphers-automated) ->> -endobj -135 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 394.370079 504.679134 361.670079 ] -/BS << -/W 0 ->> -/Dest (controller-manager) ->> -endobj -136 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 483.326107 386.420079 496.429134 371.420079 ] -/BS << -/W 0 ->> -/Dest (controller-manager) ->> -endobj -137 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 343.670079 498.679134 320.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---terminated-pod-gc-threshold-argument-is-set-as-appropriate-automated) ->> -endobj -138 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 323.270079 491.179134 308.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---terminated-pod-gc-threshold-argument-is-set-as-appropriate-automated) ->> -endobj -139 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 304.070079 498.679134 280.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-1) ->> -endobj -140 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 298.670079 491.179134 283.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-1) ->> -endobj -141 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 264.470079 498.679134 241.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---use-service-account-credentials-argument-is-set-to-true-automated) ->> -endobj -142 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 244.070079 491.179134 229.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---use-service-account-credentials-argument-is-set-to-true-automated) ->> -endobj -143 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 224.870079 498.679134 201.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-private-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -144 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 204.470079 491.179134 189.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-private-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -145 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 185.270079 498.679134 161.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---root-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -146 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 164.870079 491.179134 149.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---root-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -147 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 145.670079 498.679134 122.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) ->> -endobj -148 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 125.270079 491.179134 110.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) ->> -endobj -149 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 106.070079 498.679134 82.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated) ->> -endobj -150 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 85.670079 491.179134 70.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated) ->> -endobj -151 0 obj -<< -/Filter /FlateDecode -/Length 3797 ->> -stream -xœí]K䶾÷¯è³Ó|Tñ>$ñÅA Y$0‚Ö]Á¬;‡üý%‘U”(jz¦× bÍbv[ŸJUdU}Ҥ^sÕôçSCE0*Æ<\o.?_´ -¨ég’h¦ÓßÒ¯üãƒUI#{õQùd“q×£I+)uQÀOûÇ°÷¡(‘ð_?¹ü$/ñ‰ ˆK&MA+³†}0U½Q°«ÍÛEÀFeY±À+©æŒ1EZ´M 7Ùg¯5`m«4[lÐ uÛžg%lXGµÇNêú¹i³Œ¯tlÝïq‹¡Ó 6`ã~»ã~ÆVªûsÆØ®ûíŽûmë~k\d°°(ÑÖý¦ï~†k×…=vR×Ï7bcrùçºýðË—ÏÞ¥¯?üëòþ“‰˜—jO>ôIºK;é?ƒ“Ž'ótôù'\·î cÓ—¦ç¾Ó›“'!OB>„}âÝ;[Ý£éIÇ“Ž¿y:Þ3]5&%ÖjAGÆó¢ç&áBO!dW™¹:e¦Ÿë/ï° - ’èH&ªà‚Ã8AP F=ñF™h¢¶#™ @ƒ¶0É1´ÞBÉ$!‚O™è•Ú'3Iö؇)úÐRVùÐR"ùÐÒÇ®—à÷2)6Z‡àó§i‡cÕ)]éySd<í³ys¥3ʼO,xÆEahðõ8cÁ²‘æ"+”qé¸&,@g†›ÁÆF]ÊóS‹[ÖÂÉq°©vmªÀ›ÁFàbœTKݶëàÖÂÙS}·í–Ánâ`Ú8,c€À›”Éåv‹6q°;q¸Sv3ê؈Ý8àN°Ã2 -\¦4[lÐ&f'FÆÁlÆ᩾¿_±Q"zóa‡ºûTl¼("qRô¤èIÑ E_¸yÒP´iyÓ{ßéÑIÑ“¢'E?EûT¼¢»GÝ“¢'EOŠ¾j¢+–´ƒ (ãŽwëš­”=\èÙn±ˆ‹òr\[¯cX­­A…€ùľŒ5 -S°ÞŒd‚òÀá@Æ9´ñ:d’ -èBµPE GmF -FãFm&ç%£µÙ;•ònΨ͔ëÚ€ÇQ›ƒWÚûhý@†ŠŠ1)oi d¢2È$PÖ:q_´V6 Z;’ñŠâêÓ Í`¬r!Å0h3P¾‚³mKe6‚5£6;­’Fmv^aŒèGm¦RêA7ˆ)o¯îÊ ¨pÀðZ…êrÀ˜ªì˜;¢Jܘ7ÐÆÜDuë€;åê˜;¨2ÜAªQö€;H5Êp©F¹î Õ(wÀ¤ÜAªQpÀª© -¸Cã Âî Õ(À¤å¸ƒžêêw0  -Ü¡´ \sc¨ç®L¢ºzÀLI¥îxªQú€;žj”9àŽ§e¸C©¬ìw<Õ({ÀO5ÊpÇSrÜñT£à€;žjô¸³Þ’“±%O<èïÉ7âØΦ¼IéÎZEàÍšÄçmÿÎvBƒonþÒ|PwæÈooÿ2.V>Š™³énÖ*|%îm -<±–j1Pùò›9²@›9²À›µŠÀ››·Õbƒ©ÛvÜàb6Ì÷€ÙS}·we°ed°å}`Æ›§æ¨ÔoÖ*«8Ø80ni°Y«k»q°;q°m–•ƒÀå]\¶(Ñ6f'FÆÁmo³§úþ~Ŧ¼è͇êîSQzQFâ¤èIÑ“¢Š¾pS¾¡hÓŸ¦÷¾Ó£“¢'EOŠ~$Šö©xÿDwº'EOŠž}ÕDW¬iC¨ûͦ|/zÏKæ]œõl7åÅEyAKíh×›gû2^+äçñvd‚¼З   gk½A .7Œ²½ýF»ÒÏÙ)uoe -¼©ŽQ‹»x"±|]5£Öl¤¹¨¹ÍUk˜ÀE½‹šŸ"–¨¸Ý(qe%`4ka‹`k}íc´a«À›ª)pQï„E‰rûšªÙ8²ÁÅÓÏl‘=Õ÷w[ed°eU’Á–5,‰‡é-{ܨ°©š«8Ø8X‡¸©š\7n'®ÃRÃ.RZX”h³×ÞË:R'}Åö€è͇êîSQâ2'EOŠžÝPô…Û E›þ4½÷=)zRô#Q´OÅû'º{Ô=)zRô¤è«&ºbI›ÐŠ2ÉAÜ{^0ïáBÏv{@\´ûš] N¹ƒWúbH -^é‹U8x¥/&bìÁk‘1EZÖ_‹LºÙfèËÊ¿ƒ×"“¡¨¼™¬;ô!>ôarxèÃæЇ B߇ë-üg¼ÙHßùZäÏS¿{s1~ù.*õ¹cà³Ô~2é4µù͇Ëg_~ýÍÿò5õäÍûËß>×Ú|O¿á‹¿_ß|uùò ièO³ú4?Þ­Îb¡W^'CÙ“Õÿá»ß÷ÍŸ§ÞR á1êWÞñ¨½MWësá»'½¼ý’Šíúü ûúï_®ˆB#ïßè×½\=ê\yÆêý+Ô{: >¾B}rÃÐfõoïUï¼2V;¤õcbAöýóý/³ß*ž‹pÐHÓ¡ýì/FîŽ÷ƒÊÚ0 -wšhúA -¨Á]£EšË÷ãîJÄýˆ1 ¹ð)OqŸa¢MÚ#7åo¶€2” ‡‚¬ßä.ÐøCvœ×ò/U É0Žùß÷ó1âÜšZζh€„|Gm²eò£¯¬ÑûYŒMÿÂlÔfÅtŒ~6†8Ëç0,ùsƒ-2téô‹n¾­qoß-¾Íºáí,;éÁùx²¥—ŽÖ?]ã[~9/l£]äÒ¬/Û)r×Ô6 ‹þE~²³ô¡8VßFÎÌ_ßÕ™]ÿácý(ü æÏS",z';þñþœú‡¹„.ýs«aè/·ö×Ýqèèh¶ ƒö6™Ù(v¶õKCÌ’@7á<»±Üt]‘-Lô#g8—WlûLÄÃ@(ÑÅY†VÌÄ(2Æ°“êµï¾°þóåúÛàÚA¶m²2ö³n:/e¬üyrxÍ–¡ÃQ:üEÙ××(©MìÍ~ÏSú´×Egd}®¸TœiøyheÍoAߧþÁ>]µu¥/òÙRTªíÿŽcR¶ïÃðøQ"çßkëLW߯^{F¾Þ°j|ûüõÈcÞ”íÇ8þJ1þŸ­GC¿ùµß<—ïÇ#ý -ñø¿­e4¾î±o_æWXæU¸\7õÚ¶ÛUÃm©9~Ëg»Êí©Ÿe¶æšµñmd–˜'œåZíØ×rÒú˜YóÑ.ƒ_M@W”I3é”ÕÌx]è‡ÎúGOÏ(9BžþfÒ¢#M‡vª×Ôà;"¿®j÷8½p@\ÚúÌ™ÑÏÈ)3õÃsö8ÍA™ì.käqÆíçG—iïû¹A¹ÔL¿u¡?ŠfŒQLýö£ùÊõQå¿ »1&oˆèB‰^‰Ž[:÷ž£Üãø„/mÙðóaõ1ß1YÍõÖ~zÐZäEþ( ê´ÍÆõ¼NŽ1 Û¸®£r‘.³ÜÜ—éËv]Ù HÎÍï™ûü mºº BÊïœO·–.?=G$Ç'¤üµùîQ&PþÆÕä#îÜ>Z¬ç»of°e˜w¶— |û6‡]ÒBWÛws¨K¡³ä6W†$Ša™–”£âÎïEæsïxú1í›”½Ÿ OÆí²d¯EgùÜ›bØeú1¥Š(¦U?rØäng-nIN­6›»?_0RÙ"Š–Ñù}mÓw ب„_1(j¶ÄÍLž6×K§ÿæç*µü‡1ùÛ3LŠpÿ7q°ÈÏ;Í.†¦†9ùËŒÂùÁd ×ð…=ßaÛÂåaïC8ƒËG]uA}X¼b·‹qùi„•¤ÃrÏÓ¤ªœÀåc -øz,—¹¹,Ùr”I0î€t¥7[8÷>?¼2ÃŒÊ-G¾6}i3¬¤,+ä aã´q jaŽaâd*²¶~L5æÖ¦K×þ"kƒç¼]Ȭ[£O]4™GQU¼kuÅœhTª(ɨqê@ùI ø©.ª‚YÏ‹¨Y0–D‚*—Š\º]ºr‹6¬‘ -U*ò¥©xž›Â©îbu|›|´ T&žÄ¡Ñ®|laŸ#Pb=ÝcCÀÔeðvÉug?]0t¤‘kÍéÓtBo…£æ|a4Äâ*¢fÁò½ùü÷\*°¢¹òLò—Š›¢@±‘ÌÍ -·N’¸¯Uz0XV¸Ë„PsVúËDÝ‘ÏE·JW5¨p™Àó+QUµ¯Å£…Mͺ -çVüSk¥ui™ck*?50·+U)Ù‹1oDºcÕË°¯µóI¡FÜx¨àqÀ‹œ/àmØ` çÔÅÃV×ÑŽ FEa.£O\ØZ«“I©~d’õFÜÛåÇË4ÌY‡6å'Iåtû˜š¹þ›d¾¢ß^àú§ò|Ï<Ž[]«…­µµåHf­+!<„óS5¬´ Qö6‘e±dCÇ­õørÍ…½HšZAM=+Ad -ØÖﵮ Ï༭¼X'2¯ Øó¡+#3Î\kBåÙ:Œ|Îñ|*é(‹ònòHŠ é@*¥Ca0qËšfæ„”FªësûA†Âe[â@L#ò¡ãsÀìT]°©BŒ¸L£Œ4ššötýbz½:îK“}a@Fki¯è“DkÇ2Z ^uÓ¡oë˜ã®ù —=Tu9Á=W'9L žZ@s sT ‘/[P…‚5FË@QË…«_,…¬1”6ùå²â–©Ç±ë¥PçŒþ(=2*Õ‰(£â7²YoñêlÉÕ ÊtÜ;Í’u.;˜—WWXÒlkZÛe´½Šn7ÄóÌTóe¸‘|Qý{ wºÏ'õ¤š­ë–†¦LpÖ]SžîD渚PGhÕ Qì•Ü}Íxód»ER6O¤%“_pNCù¶Ó„§K˜Ç°ñ ¡jy®|粤»]ºÏ»÷Ÿ&sÊ-ÛO;Dõ0ä¥"¦ùý˜Ÿ1AïìÙ¦Í6Ø·ÿÏ®·£ -endstream -endobj -152 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 151 0 R -/Resources 4 0 R -/Annots [ 153 0 R 154 0 R 155 0 R 156 0 R 157 0 R 158 0 R 159 0 R 160 0 R 161 0 R 162 0 R 163 0 R 164 0 R 165 0 R 166 0 R 167 0 R 168 0 R 169 0 R 170 0 R 171 0 R 172 0 R 173 0 R 174 0 R 175 0 R 176 0 R 177 0 R 178 0 R 179 0 R 180 0 R 181 0 R 182 0 R 183 0 R 184 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -153 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 790.370079 504.679134 757.670079 ] -/BS << -/W 0 ->> -/Dest (scheduler) ->> -endobj -154 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.309749 782.420079 496.429134 767.420079 ] -/BS << -/W 0 ->> -/Dest (scheduler) ->> -endobj -155 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 739.670079 498.679134 716.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-2) ->> -endobj -156 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 734.270079 491.179134 719.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-2) ->> -endobj -157 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 700.070079 498.679134 676.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated-1) ->> -endobj -158 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 679.670079 491.179134 664.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated-1) ->> -endobj -159 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 660.470079 504.679134 627.770079 ] -/BS << -/W 0 ->> -/Dest (etcd-node-configuration-files) ->> -endobj -160 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.237972 652.520079 496.429134 637.520079 ] -/BS << -/W 0 ->> -/Dest (etcd-node-configuration-files) ->> -endobj -161 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 609.770079 498.679134 586.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---cert-file-and---key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -162 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 589.370079 491.179134 574.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---cert-file-and---key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -163 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 570.170079 498.679134 546.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -164 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 564.770079 491.179134 549.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -165 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 530.570079 498.679134 507.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -166 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 525.170079 491.179134 510.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -167 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 490.970079 498.679134 467.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-cert-file-and---peer-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -168 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 470.570079 491.179134 455.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-cert-file-and---peer-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -169 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 451.370079 498.679134 427.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -170 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 430.970079 491.179134 415.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -171 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 411.770079 498.679134 388.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -172 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 406.370079 491.179134 391.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -173 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 372.170079 498.679134 348.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-unique-certificate-authority-is-used-for-etcd-automated) ->> -endobj -174 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 351.770079 491.179134 336.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-unique-certificate-authority-is-used-for-etcd-automated) ->> -endobj -175 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 332.570079 504.679134 299.870079 ] -/BS << -/W 0 ->> -/Dest (authentication-and-authorization) ->> -endobj -176 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.986019 324.620079 496.429134 309.620079 ] -/BS << -/W 0 ->> -/Dest (authentication-and-authorization) ->> -endobj -177 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 281.870079 498.679134 258.470079 ] -/BS << -/W 0 ->> -/Dest (client-certificate-authentication-should-not-be-used-for-users-manual) ->> -endobj -178 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 261.470079 491.179134 246.470079 ] -/BS << -/W 0 ->> -/Dest (client-certificate-authentication-should-not-be-used-for-users-manual) ->> -endobj -179 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 242.270079 504.679134 209.570079 ] -/BS << -/W 0 ->> -/Dest (logging) ->> -endobj -180 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.506282 234.320079 496.429134 219.320079 ] -/BS << -/W 0 ->> -/Dest (logging) ->> -endobj -181 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 191.570079 498.679134 168.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-minimal-audit-policy-is-created-automated) ->> -endobj -182 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 186.170079 491.179134 171.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-minimal-audit-policy-is-created-automated) ->> -endobj -183 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 151.970079 498.679134 128.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-audit-policy-covers-key-security-concerns-manual) ->> -endobj -184 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 146.570079 491.179134 131.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-audit-policy-covers-key-security-concerns-manual) ->> -endobj -185 0 obj -<< -/Filter /FlateDecode -/Length 3770 ->> -stream -xœí\Ë®ÇÝó+¸6àq?ªú^$ñÆA AW´d#¸2 g‘ßÏé™éî²Y¼Þk%¦JœšÓ]§ëÑâvoð÷s‹Ù)¥ívwfŠlðgFl.æÛßã%ÿ†è¦l˜­Û‡4…ì²õû;kÐ+:õIˆï!U}‡k'Rü÷Ïv¿È&!w¢I£§h&{,ѶîíD§Â¦ó°bkâäzÇBž{'Mµ¶¢7!=È¡y± -À±Ü×Ð]ãFeßndY)¶½¦¯ihç gé_iþ´5à#±ÜÎt"ܘß1—[3jæ_ÕYë†æwgÌï¶æ·t,,4JéÖüvlþ.nCúº‘†v> ³/ö§o~ýi÷ÅÌþ§ïÞ6'æÆJm$ÆIz6í¤ý¤néxKÇß}:†ò'îOß<"7cÙŒ< FsKÈ[BÞò* 9N¼ÇîVÏ¥é-oéø»OÇÇlW­­râðØåÖ1r›.ÉE?5!‡Ù=”±ÙÿúnƒA.qÒ1ir1Åâ±³&`lP1ÁLrk5 8s$k ¢Í±ñ:œÉ;“Lg¬ŠÉàì²1IÀ³2žÇ8dºƒ!u 8›MP0œ *ÆaÚËž k˜0ÙHÅx7Ù˜½! “€F%~!£[ÃàÌÉ%~ƒ3›¬b8“OF‰1‡FnD£Ä‹àìrP1 œ£¬ÄKàlÙ«˜ Î&¹¬ÄncmV⎚L&£bÊd™îi˜0™˜“Šq³?Þ¥ÉY‰ïÁ™«g¶”•øñÎD^Å08ûskpöÆf%~|gçŠÁ‚b,ç¬ÄàlRR1 œ f3%~|Šš)¨˜ì§œ"'%ƤÀJJü`˜rô^Å`—”C@èkpælUŒgƬ¨ÄÖl,Ü”“?äÁÙǤbœ½‰I‰"pÆ‚¡bœ1Ù%%ƈÁ‰š”ø¡Î2Ëö…1 i¼Mɪ˜äÁ†I‰JyJÑç¨ÄežRIÃpÙrqŽQ‰ÆÖ-± *Æ‚31G%~Ø‚³O¤b8{ë£cìÁÙ– ƒ·6ZCàl‰Jü0³qYÅ0O˜xSTbŒƒñ1*ñÃ؉ÇdƒŠÁ~9Fâ¨ÄÇ<ÅIÅ$pÆ%Æ0L‘=–g ά†š ¡j‚?g¶è±sÓ0œ!¤ŒgcPâ'8pÆ BÅxp6žƒ?Á—“_ ̺ñûrD\B\2Î0dæ·Žö¯Îcøò‘˜^ô X‡5†.ßTN²£¢ßF~ReÀÄèFçÜ.ßÖº\ JC3æ~f”ˆù˜~ZjˆX [/McœÉŸu…tsÖòM©AÈe• kÜHìÛ ¼‘ÇÞK/6tKí½-7Hgoü@[?Ô‚C—oB&L|Rr8òƒ;ã'ýNj±,#?Ä3~ˆ[?¬5!ß” šÆtã{ÆBÞF/«ÝRc{?ãc21šgR÷|*n¬(kœ)b•åì4Î\Á0DgÎSÀÄ5ÎX°CÌl4ÎØÄHÆkœ#8‡ÄAãœÀéž4Î œѪqÎàÌÎ’’_Ö`á z]~ k²6+³V`Ðö2-ÞK†2/”« õÏÇ*è¸ö,VDQ{v9 –!<]œÇð³µçƒ[x³ƒ–¯{„}ðØxÄì‘÷™ Èk÷o>ì¾øúÛïþü·o1–7ïwÿø'¼<^ù«îß|³ûú 4(:ò¢"Ϭ+•û½aÂjh‘øEÅŸ~øãßýõº*r™^XEr“{YÞO_Vq÷ ©,¬/ª‚È<ÈÝÏQ%é!î~ûtØÀ=È݇g¨À¢ùw?CEÀ¾å!î~  -9ƒP9º o³Ç¢5Ñ…)d~=}1^N¾gªHîrò=WE¼œ|åÅOW‘íåä{®Št9ùÊ+lT\Š§ò +ÕÍ’º"E¼ì¢Ã vý1ìðbcè^iy?ß³Ë=Š‹¼È\Xñ¸G¯^¥µ¦/Ãq‘WÜÜ®b߯:Öña{½Œ[©ò@Öº1ಠÁ™ËŒ$!ãÒ¢ƒ ´ŸI½í7²S PÚÑÚ®‘­øÐ Äu@¡¶·þî]o3ã½À›Õ(aíG¨\Ïúéˆs8Ò—6f^)W;Ô{®;®üלšTCGièbÛªæ×ëuœÍ±¹s4°Ÿ£ÕÞ+¿y¬5Ò@æ ñ‰6<Í¥FZŒUq×ËÌdΞzÙ\µúÔ²N[IgXiWm/K²¶sõgÔì®ÚËÛËUßÜ>_¹üå+¡ß›ï¯3Â}ÇgOÄÕwáʾ;³›j÷Ê,n+å3óA®²óûã*J]ÙÒ†ªíâ±í®ã“òÊ™ÓiõI¼®Oj»yŽX}ÑN™ï—Xœebw°±ë•æ©ºz_ÉŽáì ±Ú1]ÙŽrG´Ú³íâìb£í®Á´Õ}³{sֱ笧⪭]·Õóæ4Õžéä4ø¾\?²«Ÿ F&žBùI•¼÷±|ÍÖÅ0?ç²ûå!2–X$›Ÿd)›7±Í!ñ™'YVíåQ«|lˆ.×'f•íLô<ÜÃx^ö-®†[X -ý¾š? -OÜ]F릯™­Ü{×C¯üïƒØåEîÖ)¯¹i}? -·†J ·tÔݸ ->uSÚÆPÓêô1”;NØòR¾ÔmöòÄñåÒgùžãv!¹ê7;[ÚîÏ4 öcC­?k&ö¶üôÀòÛ±âbÅ/Ÿ6ØÙLSq@ydÒN¼¼ƒÊøX|·çúÍÛ˜æŸ~y¸×·¦õE¶›ì°³¾<1~„œŸÏYÞæÖ9„ëÛò,Xoϵ™_RØÕ«’‹â-ƒ=—Ñ—/,b8›¿^…F…Cå aï$¶wØ^c&B>¡®‹‘“¹b]{››Ï½ëÔ¥é¬pÅ–ZÓƒ„‡ÔúcéýPêl“¹j]gšÌ R¹I¹"›C¨ùi *OëŸ?7µëòØq»/¼†#á*L5¨árÅ D‡Ý·öÆÍS±¡Roš«å;•ê>5Ãd³V)¦‰{qi¯o·âPÞÏâš­mv²9··=ÉF+îa÷ón^æœg—Ë·ýÊ/¿l¿Jd÷ÿæ¼þµ£ý_ê£ÆË:îL›-\›[7b¹’9ç« /ŠËƒ½UØÒF‰i¿‹mêX®ÑÐÖqçÄ|M½¹é{EÚ6ƒÚvW -¹ÛPˆ] ]g½° +ÂÑÛFæ¹¾é öréëà Æ—‡Ö++\Èáà2õ{¾/É{Ñ%® -´/é¸KN4Ä…ì—BaîÌ64K@J%Íô…?IWø¢K\ˆmD¹ôýõeVv]eó(„¬hLvÓ1Y©4oø H?9½ží÷•r¨P¤mjoÒ{)m+Ò6á¥~¾ Û˶æø}¹¨ÉÛ-Ôúò"÷|ÛäôÔè[ Ú\ÈÒÔ›­–J©Ê6JëBѦ ÈL•µcåÖfÕ,óˆÓÐJ±í)ºôgii–^iFdé•p‚-ýÖ¯í–|Û± çØGg:²í%Ó@dë&¥ŽôUvª›¶ºëj»àN$ô©éÍøù¤ùï)¹3|Fy„Ú”Flj% ƒ…šlôÎïËv™æ*?w)‰ZÉ£ÛèÅ“ÓI-žHMpíÞD‹x;Cá7ò墫õ›¿gše3Òã pö[Nã'Êýä×/DÏ¢vËQ‘s*ß03 Ižm4g>ïº;)ƒ}ÿ_Ëg¥ -endstream -endobj -186 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 185 0 R -/Resources 4 0 R -/Annots [ 187 0 R 188 0 R 189 0 R 190 0 R 191 0 R 192 0 R 193 0 R 194 0 R 195 0 R 196 0 R 197 0 R 198 0 R 199 0 R 200 0 R 201 0 R 202 0 R 203 0 R 204 0 R 205 0 R 206 0 R 207 0 R 208 0 R 209 0 R 210 0 R 211 0 R 212 0 R 213 0 R 214 0 R 215 0 R 216 0 R 217 0 R 218 0 R 219 0 R 220 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -187 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 790.370079 504.679134 757.670079 ] -/BS << -/W 0 ->> -/Dest (worker-node-configuration-files) ->> -endobj -188 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.434505 782.420079 496.429134 767.420079 ] -/BS << -/W 0 ->> -/Dest (worker-node-configuration-files) ->> -endobj -189 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 739.670079 498.679134 716.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -190 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 719.270079 491.179134 704.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -191 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 700.070079 498.679134 676.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-ownership-is-set-to-rootroot-automated) ->> -endobj -192 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 679.670079 491.179134 664.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-ownership-is-set-to-rootroot-automated) ->> -endobj -193 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 660.470079 498.679134 637.070079 ] -/BS << -/W 0 ->> -/Dest (if-proxy-kubeconfig-file-exists-ensure-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -194 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 640.070079 491.179134 625.070079 ] -/BS << -/W 0 ->> -/Dest (if-proxy-kubeconfig-file-exists-ensure-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -195 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 620.870079 498.679134 597.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-proxy-kubeconfig-file-ownership-is-set-to-rootroot-automated) ->> -endobj -196 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 600.470079 491.179134 585.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-proxy-kubeconfig-file-ownership-is-set-to-rootroot-automated) ->> -endobj -197 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 581.270079 498.679134 557.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -198 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 560.870079 491.179134 545.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -199 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 541.670079 498.679134 518.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -200 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 521.270079 491.179134 506.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -201 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 502.070079 498.679134 478.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-certificate-authorities-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -202 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 481.670079 491.179134 466.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-certificate-authorities-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -203 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 462.470079 498.679134 439.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-client-certificate-authorities-file-ownership-is-set-to-rootroot-automated) ->> -endobj -204 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 442.070079 491.179134 427.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-client-certificate-authorities-file-ownership-is-set-to-rootroot-automated) ->> -endobj -205 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 422.870079 498.679134 399.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-has-permissions-set-to-644-or-more-restrictive-automated) ->> -endobj -206 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 402.470079 491.179134 387.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-has-permissions-set-to-644-or-more-restrictive-automated) ->> -endobj -207 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 383.270079 498.679134 359.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-ownership-is-set-to-rootroot-automated) ->> -endobj -208 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 362.870079 491.179134 347.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-ownership-is-set-to-rootroot-automated) ->> -endobj -209 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 343.670079 504.679134 310.970079 ] -/BS << -/W 0 ->> -/Dest (kubelet) ->> -endobj -210 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 484.911067 335.720079 496.429134 320.720079 ] -/BS << -/W 0 ->> -/Dest (kubelet) ->> -endobj -211 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 292.970079 498.679134 269.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-anonymous-auth-argument-is-set-to-false-automated) ->> -endobj -212 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 272.570079 491.179134 257.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-anonymous-auth-argument-is-set-to-false-automated) ->> -endobj -213 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 253.370079 498.679134 229.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated-1) ->> -endobj -214 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 232.970079 491.179134 217.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated-1) ->> -endobj -215 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 213.770079 498.679134 190.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated-1) ->> -endobj -216 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 193.370079 491.179134 178.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated-1) ->> -endobj -217 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 174.170079 498.679134 150.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---read-only-port-argument-is-set-to-0-automated) ->> -endobj -218 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 168.770079 491.179134 153.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---read-only-port-argument-is-set-to-0-automated) ->> -endobj -219 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 134.570079 498.679134 111.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---streaming-connection-idle-timeout-argument-is-not-set-to-0-automated) ->> -endobj -220 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 114.170079 491.179134 99.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---streaming-connection-idle-timeout-argument-is-not-set-to-0-automated) ->> -endobj -221 0 obj -<< -/Filter /FlateDecode -/Length 4007 ->> -stream -xœí\K$· ¾÷¯èsËEê{H⋃6²H`9¬;»6‚Yvùû¡ªõ`U©Ø³Ó³ð¥g1;U,J¤ø‘¥z¸³å_8þ/¡3)åð|ùxúõdM$Ë? Çêd¹ü=ÿÊÎESZûtÆ fÈΟ?JzH&\ÉOLN7È¢—§Ó?þpúeÞ•;{c'çßÞK–ˆ&ºœ“;fI<,Œ³ÂLŒè\8fÉ`’eÅQaáQA‚cgÑ$"ïeœ³ ø5ž`²ÍH¤ð˜ì¦$“ƒ§l&g()<ìVÖÅš>EŒYÓ_ˆ.OGÙØœ2hú2(gQay6(òŒcܽ¦OòlñB'ðAÃ4“à¼Wôë ¤„VÁ”u1Þ»¼Âã¼auÈ+ú€ËƇ¬‚)ŸCˆ -¦àAð5}|4H6YS@o0ÆL³!uÔô!2äÙ’ -¦Œƒ!ŽŠ¨`Ê`bOEMŸè C -NÃ4fѤašÈ+kIŒ%§KD§aš£‰¬MR0õÖsÂä,¥èãmæŒILÙ9eBH -¦lΙ9jyÌRàŒ¨ðxÏ9Ó§¬`ê}6ÙÙ¬å1Ä9³h­ðãœIœ}5žÈ9\Ðl/dS-çÌÀÒxçLÏ™^áI`œµè5}RâœY*…'#çLd4ŽyÐZΙŽg'pÎLÑ*˜¢Ι£‚)r%’×ôäœYf0…Ç[Ι!GSôs&ZÔôA0>ò ®`Š˜8g&žZBΙ¨é,çLή -¦‚)3nÚbZK­IyF,ÚZN³¸ ™ìR®»•Ü4羑¿ýtúò—çŸþ{úð‡¥œ ‘ËKÄj#g,ÛKHžÚšÖƒÎÅ"»J§‡Açt8úôëÈd#BV@tÆqævôÀBçæ3¡vÁ—“ äQíJºw£—.‘8yUn¡Ÿ ^äx½ùdKww—(©C¿Ò7L ¼¢ûÑK—(,5·÷eƒ;ÀÁ­q´¥¯\f)\vÔp€Ã {0ˆ‡*‘CcŠà€k¸¾ÝÐ…K ‰’ºÆÁàà$;U¢°ÔÜÞ^æe_~Îû=DÅh>„îq(J+J$!úÑGˆîB4”ŸxÞ|Bˆ®Æ³}˜Œè¢}„èg -Ñy(~z¡{º}„è#Dï*tå-êȯngÍo\ÑE?û¢Ñ²G9oÖÖÐĘ©mONyËBßf˜ó“ ¢ -Oâ >û¶Å;çÉËVºÕt.[×ÌdCÙºÎÙfEçP¶®…¬è\ÙÈŠÎÁeöNYÑ9™€„YÓ™“Jô`“¦³çC—BÒtF_¢ ’¦3f“³KmxÊCe«/aÛΙò0.³i<ɹжҧ<÷ Ú–Ø”'Yƒ€)j:§`Ø™ûíˆ)§RÊÉMç̱“gÛcžhKì8ŠÎ±Ü^£˜‚¢s,·×1(:G(±Ã3›¢sä´Ç§ ›Âã9v,i:c‰›HÓ9vbDT|5rr€_TbÇrh(F/$ -KÍí}Yã@8ЇZõ -úÊe¢‰»:{ƒà‡´«³³ËSòyC­z]¸´¸¢®pp8ˆÀ装¥æö¾cCYŒæãAè‡âÊŠ‰Gˆ>Bô¢»}á†ò*DWš¯F&#z„è#D!ú™BtŠŸ^è…î#D!úÑ» -ÝÕ£ÞÀ‹Z€²Ù²ÚQÎèÅðÇšùˆ.;Úo)‹VîÌ0±âi»¼Î˜ÊÖ".[yG<¼LOÙÚe åˆ'XS¢å©êCž`BŠŽ¢ÂË–E—4ždq_žÀ=âIh|‚´ä ÆEþŽyœå ¶1(†f&öþmR,Í^é ûØu‡í˜©ÜùuréoOùú -J>϶v3—åx·Ùžß~<}ùçþôÃwã¼ýpúçWÖ2f×ßøæ_ç·ßœ¾~Ë>UDà ð é™ÓúçQž4ÇÏ+‡É3Dä;DDÿ,¸ï@Ï‚ûÇ;Dp^zÜÏáB}7+,7’¨ÜŒsgâX„rä!_ûÝ_þþíFñ¯ù8(ß¾;E”»g·Ð¸WDº|wŠˆþvðÝ)"¹ÛÁw¯ˆp;øŠzÛ&ã‚÷Üs¶å ”Ûn^>ŽÌãxkó1@Ïúhy»Î}å…‹ÎÎûü몌ò—+&[nô$k‰Ï)ðo9æ똮׈Ûàü‹×ã­òpÓå—üµ-Aåáë®Çå:¾á«+}ás•çC½ÞÚä*¯^+í‹nEÖB/ú½«ºÚÚŸúáeð—¿Ëu¡{ѯS•S§Ÿ¹]Ë[ðŖ–ÇUÇÖç¢ Š±™µ_äµÖªßFï^£ª£Äb±ý¥âëuºÚ~±]¨mëX oÇ, ÙKîwÇ*lmúJXåòÈ…ŠÕ¯‹U¡-ºÖZâ‡LÊõ,bŠ*>qô÷É8¸Ã3\Š=}³Ÿ«˜àÕ't¯åP·áå•m˜„}ÜÈE„Õß?ƒ¯:_ipí³õ¿üý°Öi±Yá}Wõ¿´˜Rí¶vÄ{­Æç79Ú ±°ðOì¶Ðƒè+î±yXóåq+ÝOpØùÕrc¬óM¹­ÅÕâ7ο{?0ìvuuLJ_ÒÖ”×öØÍE–â£;_Vódy{umã­O/ý¼Ð^ÉÊrÿ¶à+û¬a¶8m|/§°æ}q^¹‘{ežiuRó³xÇ­-_£ò6ì¦ÖµEgŸß5àâ;ùêóÏÅÉàï߸CË|D#¿6ûy‰˜3ᙘþ~5 ÂÖƯ„å[5MÁ.¼N|ݧϙ®útó‰IÍÚ훆[{¬ë„[¿­ž•.êÞ¥í»:®šïÆœ¬Ú1ní¸øÔEôÑü+¼0?Ž5«s`°>zvc«% [à]ƒLPbÛI+HˆXÏCUz5iFbNuu*”•Jßëu‹²†§ô M5£ãuð‹õë -ÆŠYn5ËÈžyªðÆfÐ^ÙmuoZ[é^ £fH0Éä/«Èƒß­@·8´Ì-ì³ümçÿ^gÇæ-J»½dæk6 -¢‚¹cä7>DAŠõñrE¿[…mm_s‰õvV¥–¹_ÅbÞ­¶ãkÉìs‹¶Cµší’'ªÉs[½Ï+ÂÖGÛ‰‘+É£<äœf¯ä¶özê¢<”mu^¸yÏ$µ´oœ‡¯ªÓɲb¶}%J‹Yi²ð¬ä߶>7AÓd’nØ0+åoË´„¸µí6³€ˆT‰Jä.Õk.›6R—6pm ™ŠÏ(D±úìÔ¾-¶e@Cä2ߘ»E}›*ŠAÞ­=¡»bó±a:+o¡Yo´{‘Øz„åC[¹¼Öࣉ庰ÜP?ýò–@ÌÞÄåžyq0ä2O©7Í«ôòp‚Sn²,¯¥Öý¿âvÑÓÔ×–òôý5´Y.W[Ì'™£UÎö½èž©Êµ÷¶¯íWþúá -c¡ -¸E0ÎÖvP×}Û=ÖÞ? ØúÞx…·¡ßÙÝJúõD‰C“Y-gˇ”<.A„dâB?S4¬]HÐps‹¥Ýù éò-Ì_;Wý<¦5åCŽ Ï×ogŠ“ʽ¯õñt}D ü°Gº1™…Ñ–üÄ°gƒ2OðLr!ÖCÛû*ïmh—“óåq­ §/Ï|\sïÜ÷—qÊC'£=µfþšw • ¸ -ž¹ep{r}yºïJf(_ÏBW…BÓ™‰£C”¼£ÃÑð&Ùy+ˆ´'Z!n9&sã…~˜;æLìªKÓ9ã…(àÔˆ—‹õ[êÓ” -®Ó Yl§y¡TîTjœì8MHåQ¹ñ Ûº.;õë5hß +÷5êQçËoBºœ¦|µ7êHÅΕFÓÜ,?T®^^–;¦-6ªTNOâÔYß×äPhX—÷+#tñr*ygG~:QœpÓÈ–šëI¢ývr²Ã_5¦f*ÍF ç€åÿkª N-™g9(7`]ë@ž !%6;ym$I=ëàÌåû±nGå¾cì>+íť̈́¿$ÝÎÝ-¶¢ -“ zÌM½’¤zòX“]÷ºN.)¬Ù§çJI´M 7ÈÔ]ùiEzRUvö&,8áîÔûäÐsç“$ÇŽ¸+ßèúö~…Ï7âe™ØpK.®ŸšAÛg»!L‡bœy¤Ñ§…Ü¢µg'—s?A6›q/§ŸOË4ž —Gí=ûôú9^wþó|ÿÿ9áù¯í©Æë<¶g è¹uE–3€oÞ$äNìa#ˆ€"í²Kƒ—š7ôy@äkÍíHìÓõ êúUI¤aCA†îºå{ÂÃ2¤/+öBæ^lÆ„}=õmp¶|øܪœÈáði×ü˜’Ï¢K>+¬cJç«¢!ŸÈNùTÌC³•šÅ!¥nú¢?J(|‘%NDQNý¸†cš•]7Ú2 -Á‚N4F·êšWúL”~qxÝ{U9´(ÔžÚ;õIRûÀ -µ'¼9Õ/§a}Úç¿|¼ºï°PïË‹Øó½È¡1J \HÔ4šU’B±ÑVBÛDÑÓÓl£õcÓ)ÔfÍ,ˈÓÔJ±×ƒú³´4ITºI¢v¼¥ß6ãõjÉ÷ˆ¤;Ç1:;8{-™&´ [w*Nßh{Ù¸–ÝfÛ³E¨LíhF;Îå¿—ÄÎôiÈ×jkÄ¥¾¥aË'…]ôàϹ”;å;Dg_>eM“Ø+ùä6úæÉ~‹¤mžHI /Ú½Žýí@…'¾o_]Õ×nše;Ò§àð…Šù³«Þøú6Ò²CÔOcY*RNåË,6•÷Ú\´û’?î¶Á¾ÿ?„Ø/ -endstream -endobj -222 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 221 0 R -/Resources 4 0 R -/Annots [ 223 0 R 224 0 R 225 0 R 226 0 R 227 0 R 228 0 R 229 0 R 230 0 R 231 0 R 232 0 R 233 0 R 234 0 R 235 0 R 236 0 R 237 0 R 238 0 R 239 0 R 240 0 R 241 0 R 242 0 R 243 0 R 244 0 R 245 0 R 246 0 R 247 0 R 248 0 R 249 0 R 250 0 R 251 0 R 252 0 R 253 0 R 254 0 R 255 0 R 256 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -223 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---protect-kernel-defaults-argument-is-set-to-true-automated) ->> -endobj -224 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---protect-kernel-defaults-argument-is-set-to-true-automated) ->> -endobj -225 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---make-iptables-util-chains-argument-is-set-to-true-automated) ->> -endobj -226 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---make-iptables-util-chains-argument-is-set-to-true-automated) ->> -endobj -227 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---hostname-override-argument-is-not-set-manual) ->> -endobj -228 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 705.770079 491.179134 690.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---hostname-override-argument-is-not-set-manual) ->> -endobj -229 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---event-qps-argument-is-set-to-0-or-a-level-which-ensures-appropriate-event-capture-automated) ->> -endobj -230 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---event-qps-argument-is-set-to-0-or-a-level-which-ensures-appropriate-event-capture-automated) ->> -endobj -231 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated-1) ->> -endobj -232 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated-1) ->> -endobj -233 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---rotate-certificates-argument-is-not-set-to-false-automated) ->> -endobj -234 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---rotate-certificates-argument-is-not-set-to-false-automated) ->> -endobj -235 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (verify-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) ->> -endobj -236 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (verify-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) ->> -endobj -237 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-only-makes-use-of-strong-cryptographic-ciphers-automated) ->> -endobj -238 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-only-makes-use-of-strong-cryptographic-ciphers-automated) ->> -endobj -239 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 473.570079 504.679134 440.870079 ] -/BS << -/W 0 ->> -/Dest (rbac-and-service-accounts) ->> -endobj -240 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.154476 465.620079 496.429134 450.620079 ] -/BS << -/W 0 ->> -/Dest (rbac-and-service-accounts) ->> -endobj -241 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 422.870079 498.679134 399.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cluster-admin-role-is-only-used-where-required-manual) ->> -endobj -242 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 402.470079 491.179134 387.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cluster-admin-role-is-only-used-where-required-manual) ->> -endobj -243 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 383.270079 498.679134 359.870079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-secrets-manual) ->> -endobj -244 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 377.870079 491.179134 362.870079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-secrets-manual) ->> -endobj -245 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 343.670079 498.679134 320.270079 ] -/BS << -/W 0 ->> -/Dest (minimize-wildcard-use-in-roles-and-clusterroles-manual) ->> -endobj -246 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 338.270079 491.179134 323.270079 ] -/BS << -/W 0 ->> -/Dest (minimize-wildcard-use-in-roles-and-clusterroles-manual) ->> -endobj -247 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 304.070079 498.679134 280.670079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-create-pods-manual) ->> -endobj -248 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 298.670079 491.179134 283.670079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-create-pods-manual) ->> -endobj -249 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 264.470079 498.679134 241.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-default-service-accounts-are-not-actively-used.-automated) ->> -endobj -250 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 244.070079 491.179134 229.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-default-service-accounts-are-not-actively-used.-automated) ->> -endobj -251 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 224.870079 498.679134 201.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-service-account-tokens-are-only-mounted-where-necessary-manual) ->> -endobj -252 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 204.470079 491.179134 189.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-service-account-tokens-are-only-mounted-where-necessary-manual) ->> -endobj -253 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 185.270079 504.679134 152.570079 ] -/BS << -/W 0 ->> -/Dest (pod-security-policies) ->> -endobj -254 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.622493 177.320079 496.429134 162.320079 ] -/BS << -/W 0 ->> -/Dest (pod-security-policies) ->> -endobj -255 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 134.570079 498.679134 111.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-privileged-containers-manual) ->> -endobj -256 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 129.170079 491.179134 114.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-privileged-containers-manual) ->> -endobj -257 0 obj -<< -/Filter /FlateDecode -/Length 3771 ->> -stream -xœí]K$· ¾÷¯è³ËEê{H⋃6²H`9Ìvvm³Ö9ä҃ªV©zfv,Üc÷vÕ×,‘â'’’º¦Æœ5ÿ÷¥áBôÏ—÷§'­ E¥cˆ0³Ç‘2@1N5Þ(֧݄Rî’2̻٬FáL&*°nÆi$ÎX;±´Qê §l‹²†xÈOdŒUlÙ‰=`¢².:=နÎù §`B°göX¯tÐN­Bp -q¨ãÌ"E–=9á”yPÄQá'œ2™Šx¤âÌoS -fÆ©Ê1£aÆi •¼t }èÀúÐqª9ò¡³qìÃí‹ _채t4Øbé¤i(}Ë‹ãуÊ#ð®Â8.’C‡o+sÔ”t‘h¬Õ‹š‘.¼®GBqWyü’´¯+×ÔZ©ýbì6ã ´Ëxï*ÀEÍ;dÛ0tp‡ûÖJÕ(<5ö÷¥çwxÀž‡\Þ §èªòlx€@òà®*×~ȃßáÁ÷<ä: p1¤…Æíx0;<¼ö^hžûû[,¢7ïwBw?;/ -&î!zÑ{ˆ^…è3·Xºí,ïzï=º‡è=Dï!ú‰BtŠOŸèî…î=Dï!zÑMtå]–醠¢âζu×m¥ìá¢ë-qÑî’ÝÛÀÌ·<¢²ÛžtºKlº=àÉ)°ÅÂÿð²~¾Åâ[,ޓƒ-¶E¹ƒ-ü±Ó@> Zú0ð(;òa00öáv‹EÃK'ýü-–î]Tw& 4\$‡ßVžÀCOà]帨õ  Å]å ^~—/ñÐZiƒ®õHØ×Ð.ã ¼«<5ChìP'Û†¡ƒ;\·VšÆ橱¿{»%Ù¾ç!×wC&*Uy6<À yW•'3äÁìð`zr¸ÒMc‡v<˜D`ÔÞKÍSc¿`‹EôæýNèÄýäF³{ˆÞCô¢ÏÝbéBTZÆ_‘÷¿‡è=Dï!ú)BtŠOŸèî…î=Dï!zÑMt寠Z'¾îxÛºë¶RvñÖÎõ‹¸(-Ç1çŽþwayÉ.îÙ‘ *¶¥ÿXê^ϾŒå‘?ҵ݆‹íkŒ¶:iJïn||ô‡×§¸þ"v<;ö¯mÉñˆÜ6û ϯߟ¾úÓüñû¿rG^¿;ýãk­ ò‹øå^ýóüúÛÓ7¯YÁSU8’ŸV¤»rnPá_ ‚ƒ?­ -«9qÞ "¼@…·7Ñý˜bæ«àüx Ý7ª0.?¡À±f|ôu‹múEݤä›ï¾ÿóß¾(‰ÏïùÛ‚ï*Ý|7ªØº*DÖ5»êÍóûá¹ß/PRÆýx*6®J¿^Næ\º}øØU—ç÷#"Ý”©zã>@îœù2®G¹4)Œ·øe²Žü®íò¿Æ‹f'ä÷rþ/~« y>~ÓΑmE/dÝúZŽamk¹Ž1æk\ûli…^n—hmc¹î¡µ»è-rÍžÔÖr\dpmc×æ7Y²#Wƒ±Ó£6š#“ïÈfû`íÇb¯köÛ>ë5Ù-äŠÏ\³G³=äBëÏr fÛÃêc}™ÚÊ“ÌÐÙzųû=óÜø±éÝM}鶾ü¹ŒéÜ)—þ÷ÌåÌw`¶¾«6úÕ¶¥¤ëíÿ#NyA=ç6|Üf½Uç»õµ´ó°ÆØâã$×÷E&¬ã ½CöõâãÒFñeµwêK·õåÇáȦÇèÌ9Š¿-G¥æ,/ÿtÞ’_lñƒYýS|~à‹´žûâáó¯[¬YÇ¢M8·k)¡Wh¹¯ö¡é\|Wl&1¦sû4÷£ßúñùœ¤Ç£Ôú7Ÿ'E'ʽÑïËyøc)ÕÝËgæ×gøo‘¶¥|]ôá´ö¢ßúï&.Ú:̘´Sº´u´º/Ä”IíA‘·Å±¹ãèv:nÚà—“ÃtÐP¸mµåò;æÆCöˆËÕj4ûÁ–í·£©—¿ˆT*ki¯è°ù•³8ë#:ðù‘çìîêÅén•úT¿ÉÙIíçÁl¯ˆÒVì«Å‹Ú’åŒ}x{kd9ÚúçÉ3a”m Êr#;hIK$!—–%ŠPש^uö’ÆfõdÅÔk?š|Mu¾ÖÜYrc ÛuÈAgJdʦÊüµæ[h#£êº¹Má[Žß´Q»øÁ[&~­Ù­Dõïæ¬4ŒŽ£½ÀÒé’’·‘D-²7N:ú6HðÝŒ N,Œö£(ljRU–GüÃuŽ…2ÛÙx¹D`¦y]®¤÷"rÔö(ŸRÚÅyxeéë<*H·h¹øxÆ5u"ºÍêr;kdˆ(¤ÛG]ÞÙ-ÈžØÂô(Ï1ò’AùHàÝòeåé—[DR?|´Ê/ßG¦Á ŠLtv¾ÌÚÓíÞf²C¾<ø"o`¤ðð–†1±,žÞ®±P†Ô²éæZ[6¸Å"H —¼!TK™(­)$º*ônMö 9i íx”H!k™Œºöi3lßl’|”Wß|8QàÂ"Ì–ÑéQ—Ç,CP~ÁÏä[çÞÌâisÞ¹tyÚö‡*•À­UzTtÀóútnq’å×/%ÞŸLL_¿¦ŠÖ#†YmáG¦=*Àˆå»íC8ùP׶°~7^±ËÉØtoÆFÒR¹ÉÞÄÚ8ƒù¨ô&]Oå2»†/”³«âÈW:s §Þ§[yV˜I  l>sÕrÅf[ƒ(e[ƒíÂCØX-@ºµP×`ŽÉXd¡Æʹ…fºtôYð‚Îx9±Z»E‡(˜ŠAñ.èŠYaT¬(ÉJVžPº/æ -üR—¦‘ɬŸ Ö <•4mÌæ£*‹Üºœ†r¹5ªLù*Ú¥±x¾™Ò†º ÕñlñQF9M<ŠS£m9ìátFåZ1nà -¼œRÞ¹‚OäÒÔr…¦2ô$hE»º—†úP\Å¡Y0wv˜þ]SU4ežå }óeJò¬)I±YáÞIw5ëàÈé õæ -å¶Ó£tþâéÕ@>%Ý*]=Ö¡Âe_îE(M»š‹&ù,‰¶’Ο"ˆ ùD6ʧBal–uf¦)•T×'ûQRa“.q"¦éԶϰ•YÙtÁ–^4âb4]Ãh¤ÒØÙ30úÙáõb޳ɮD@Bkj¯è£DkÇZ޵˩ëOkͱ˟Ç(ÁÛ> -endobj -259 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-process-id-namespace-automated) ->> -endobj -260 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-process-id-namespace-automated) ->> -endobj -261 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-ipc-namespace-automated) ->> -endobj -262 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-ipc-namespace-automated) ->> -endobj -263 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-network-namespace-automated) ->> -endobj -264 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-network-namespace-automated) ->> -endobj -265 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-allowprivilegeescalation-automated) ->> -endobj -266 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-allowprivilegeescalation-automated) ->> -endobj -267 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-root-containers-manual) ->> -endobj -268 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 626.570079 491.179134 611.570079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-root-containers-manual) ->> -endobj -269 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-the-net_raw-capability-manual) ->> -endobj -270 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-the-net_raw-capability-manual) ->> -endobj -271 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-added-capabilities-manual) ->> -endobj -272 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-added-capabilities-manual) ->> -endobj -273 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-capabilities-assigned-manual) ->> -endobj -274 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-capabilities-assigned-manual) ->> -endobj -275 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 473.570079 504.679134 440.870079 ] -/BS << -/W 0 ->> -/Dest (network-policies-and-cni) ->> -endobj -276 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.070980 465.620079 496.429134 450.620079 ] -/BS << -/W 0 ->> -/Dest (network-policies-and-cni) ->> -endobj -277 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 422.870079 498.679134 399.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cni-in-use-supports-network-policies-manual) ->> -endobj -278 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 417.470079 491.179134 402.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cni-in-use-supports-network-policies-manual) ->> -endobj -279 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 383.270079 498.679134 359.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-all-namespaces-have-network-policies-defined-automated) ->> -endobj -280 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 362.870079 491.179134 347.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-all-namespaces-have-network-policies-defined-automated) ->> -endobj -281 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 343.670079 504.679134 310.970079 ] -/BS << -/W 0 ->> -/Dest (secrets-management) ->> -endobj -282 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.118587 335.720079 496.429134 320.720079 ] -/BS << -/W 0 ->> -/Dest (secrets-management) ->> -endobj -283 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 292.970079 498.679134 269.570079 ] -/BS << -/W 0 ->> -/Dest (prefer-using-secrets-as-files-over-secrets-as-environment-variables-manual) ->> -endobj -284 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 272.570079 491.179134 257.570079 ] -/BS << -/W 0 ->> -/Dest (prefer-using-secrets-as-files-over-secrets-as-environment-variables-manual) ->> -endobj -285 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 253.370079 498.679134 229.970079 ] -/BS << -/W 0 ->> -/Dest (consider-external-secret-storage-manual) ->> -endobj -286 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 247.970079 491.179134 232.970079 ] -/BS << -/W 0 ->> -/Dest (consider-external-secret-storage-manual) ->> -endobj -287 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 213.770079 504.679134 181.070079 ] -/BS << -/W 0 ->> -/Dest (extensible-admission-control) ->> -endobj -288 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.131038 205.820079 496.429134 190.820079 ] -/BS << -/W 0 ->> -/Dest (extensible-admission-control) ->> -endobj -289 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 163.070079 498.679134 139.670079 ] -/BS << -/W 0 ->> -/Dest (configure-image-provenance-using-imagepolicywebhook-admission-controller-manual) ->> -endobj -290 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 142.670079 491.179134 127.670079 ] -/BS << -/W 0 ->> -/Dest (configure-image-provenance-using-imagepolicywebhook-admission-controller-manual) ->> -endobj -291 0 obj -<< -/Filter /FlateDecode -/Length 2237 ->> -stream -xœíZK7¾×¯¨s€("õ ’Í%‹ d rðÆ íœ=ìß_J%R¬®êjœ26Ú]úŠ")¾D© ³¥¿Ÿý“=˜œKŠ~^ÞMï'kR°ô§Qlíõ÷ôÑcBSl€sÌ&,àæwXâJL]Vð…àÈprŽ11 ÿôÙô=%–!@M0qJÖÀ5{0~ŠÌeR0Ødp0VxLD0µÒM¡‹^ŠÂ«U<)pƒE¡7hÒ¼ñȲ†ÁCä #Úy£³ö¯6Þš?†+X…½.~nÌ7Ì?p°Æz1€‡æÇæÇ­ùÁ_ã*‚•DnÍÇæ°,]ÉF:´óBÙX\ý3ïþømúâ ;ÿößéíg-17V’•¼;NÒ›i§í§}ð’Ž/éø·OÇXÿ¤yÿðŒtܬe³òx°š—„|IÈ—„üSò8ñžÛ­ÞJÓ—t|IÇ¿}:>§]HÌÄa GNǃ‹jåïáŠ'ä!3˜±¶ôóOÆ”g4Ù8ôî„&xãèìâ M´ÆÅB>£‰Æå áLŠ6o”3}R6#útB“½ñ>`9Ó§Xã£wÏh¢ñÙ¹boÓ ez°è݉>h³ HÖ'4àMð6¸}­ ¡„|¦FRŽ˜OhéLéLGé”ÿ'1†ž -š )øC­>Ù>!š˜\Ž'1F.—œ½M“Mºó˜¼Iwbóz_tó˜£IwbžrÔ¤;1…rþNÌ;ª\ùNÌÓ’L¾ó¨Ø݉y‡´sÅ|/BG•+P°6åD²3mb¶]µ]å’&ÇäëõÜA}ðêˇ b¿þ‹st‰Ê'‡Ì…ŒUœ˜ÞM_|ýíwÿüñ[ZËÃÛé—WÖ‚§‰÷ú×ùá›éë’p"£¬" -Iˆк‰âÕ ÃTÿøù«Ÿ¿ûáÏH­Ç_,‚2 -ÿZª§Ÿ+➯k·àרºãëLJuxCæ¬Eúö„èß”ãžtðo×÷Ž>×±_èûw¥«ŽÔݬzPyêOx—L&ú„`ªiß¾+»yU$¤õ¹¾kŠÑ¸Þ[W%šÒõ;®t¬|›Sñ¢æ†u!›:mÐ ´m\yv±‘ê{Ö¯Éež¡Gñ ýYt}sj°¤ VmÔæø¾^–Çv¸’eׯF«lqär&—Rk åî|ºoº šyØ¡­µÚóq؃1M×¹éζ³ëšÚûn[j Ü›§tì¿EÙ5ÿ°œÊ«Ñõç&ãé5ÆW#°»ß«¥ëšW9Ì‹}Õø.[ÿŠŽÏ©MݵMW¹ëº$n9NqØõ}Ás_vèûZ\_ÿÒ^ùH|ÜueŒJçnÏðxì·¼.ðjoG),Iå¹y(Ç«\ê¼>Ýfµ°ç6ëú¸«x¾K’¿ÏÃF-î†s|êœãú’}µv]Øí’í²Ù`x£þ×ÎÁkgYfê!µ;)¶~‡Ú ©6NÅ™Ô:šº7QoM-(õúÇM—^[B8Ù䈥tÙu·K.îqõl†Ok¹â0Âe íf¶Ò÷ÙêÚÒi¸¤CwÍ£2[}÷´ÒÕÊ°I“·«û+ÞÌUyêÏâ~n¬ûnsõ2Âhß·2¥×e¨¬ey]îy?…\Ž´ ÓÆR3J-n@Š{¤ó@ÃgÚŸI»˜‘ýÍÒ0ߘÚ~#~/Týgc:~9(Ù÷ߔՠӯ½ü©1é}-€ ëÁy0`:÷ÇÚìía:^áÂìVxy`BÁ– \½¦¹¢tµO[‹0'°?bàÕÔù§¹µ´ j¬‚@ša×Õ×˼&'„l\EQ…ˆEÀÁÐkÚÁpL¼ ƒ³ - {Ð*q¦œ,L‹ò(g™ -ŠêÚt ÜÏ´˜”;ÏÀe"±î½¢‚![­`N)U L)ñ⧨^“íÀÏ-³öäLy¯¼FçËf$/t…é e:¤ëÜ‚x* US [~¨2B½ÞuÜÆš:Je⢆`?náX=À¾¶YÅ îÀeªug_¦¨Ã¨6pèiÐ)¾g;âe )³©(5£Ã´¯ÿ®¥"Z+O{¨W‹À ôh©¹)ðÖHRuü‘À§A«ÌUn‡ïlèkÑj±ØU&Sx*¬^-RR<¶0HÔ \KÛGj¥-+Aâ$”/xè$U)ØYXîAø8Jí¼h8‰Ç!zYÄØ¢Šy—¶±ùk¸†~f!^¯F¯„)p(FEaÀ£Œ^ÌÙ*Õ J‘Ç‘dG;î2ý>µm]ÀRRsÓÛ;|˜ÿG4ßÐçß“ŸÿÅWNë>ŽVªJmÝÀz'Ctì»p½àaPÒFèUÙ0äA8dGTõÚévv¦©  o5† ŒºtjU– 'àreEid>Õ7cÃ^‡ŽGbÌPu —CÃ<Þ¹±%ÏŠ%*éØÒé­G5‘š) •À24Û¨YR ÓWý½v…«²Ô@µuèÆ;?¶YÍš±¶ -EâAMö°aìA -}”þèôúd¿w•#g@E¥´ zѨ,¬¢RðŽQ׆q;”=ÇÍuÀÉ;,$¼œÊ='MÎHÑZøÍ@ǨBó˜Ö ‚êÛåBÊa–1á˜X§Ø§±YÚŠó¡•’ôý][:h¯ˆƒöJÜÑV¾¼ãI·ä¤ -:œÓX”ÒKæ,êÙ‚úAéÛËö[Ù¼ÛÎj~(GgjÇ´°£ü¨ú÷1¹sx£~Dµ¹,W–6jÉ¡›KmwrŒTCh!tjöYÝ•<{ÎùåÉþŠ„/O´$rÚM@ñvC… ½(÷_(VýG÷ÓŠ=\Òó pó×®ãß?œqý"´"¦zT %ûúŸ2%ydO~ûr»«°ïÿ0ÒO% -endstream -endobj -292 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 291 0 R -/Resources 4 0 R -/Annots [ 293 0 R 294 0 R 295 0 R 296 0 R 297 0 R 298 0 R 299 0 R 300 0 R 301 0 R 302 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -293 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 790.370079 504.679134 757.670079 ] -/BS << -/W 0 ->> -/Dest (general-policies) ->> -endobj -294 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.986019 782.420079 496.429134 767.420079 ] -/BS << -/W 0 ->> -/Dest (general-policies) ->> -endobj -295 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 739.670079 498.679134 716.270079 ] -/BS << -/W 0 ->> -/Dest (create-administrative-boundaries-between-resources-using-namespaces-manual) ->> -endobj -296 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 719.270079 491.179134 704.270079 ] -/BS << -/W 0 ->> -/Dest (create-administrative-boundaries-between-resources-using-namespaces-manual) ->> -endobj -297 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 700.070079 498.679134 676.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-seccomp-profile-is-set-to-dockerdefault-in-your-pod-definitions-manual) ->> -endobj -298 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 679.670079 491.179134 664.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-seccomp-profile-is-set-to-dockerdefault-in-your-pod-definitions-manual) ->> -endobj -299 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 660.470079 498.679134 637.070079 ] -/BS << -/W 0 ->> -/Dest (apply-security-context-to-your-pods-and-containers-manual) ->> -endobj -300 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 655.070079 491.179134 640.070079 ] -/BS << -/W 0 ->> -/Dest (apply-security-context-to-your-pods-and-containers-manual) ->> -endobj -301 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 620.870079 498.679134 597.470079 ] -/BS << -/W 0 ->> -/Dest (the-default-namespace-should-not-be-used-automated) ->> -endobj -302 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 615.470079 491.179134 600.470079 ] -/BS << -/W 0 ->> -/Dest (the-default-namespace-should-not-be-used-automated) ->> -endobj -303 0 obj -<< -/Filter /FlateDecode -/Length 4785 ->> -stream -xœÍ\ÛŽäÈq}¯¯¨gÃÍûØ€,y –5°=Ì–¶%=V~ðïû$ÉÌŒ$YU]cÙÓèébD2·<™dQ_~>iü—œžRÊ1¸ëíë始š¢Wø7föoñýU+'¯sHþúµlœbö&†ëûÿmåð÷ËïvùëxŽ¾š)ç`´¿þíÇÂrvŠ)Få +ƒ¥´%,龍ôsf,¨á£3nÏŠPÃåä j8ca%¨a]ÔvÏÊPÃDí jèì#ai=´QŠò ˆrÎÂÓ~ -pz`<£§²É”ÁÓÑ0žµSˆN{Íxy -!øÄxºø¬4ãyèâµsŠñ ‹³)2^€.6XEyÐŤh/BïdÂKÐÙ•(º¨ 4ãe;ùœœ£¼ žJ1íyFùÉ'kãi=y䦥<Ì()DÂ3Ð%(Ÿ)ºx« ãYèâðÃxÀ ocJd¶š2»­²šñC ‹ŠAfÐTp‰a€‰qrÙ8&ÙÉ%— -˜”'£e0`²Ÿ\È‘á€U<£X]¼ ¬†.3—ò ‹ÍŽa5ÐX ]Œ³ ,ÐÖ![XÀ­SY3,°À[§t`X`¸PS1,°@\›‚cX`¹H¥Ì°Às-°“aèbºG†¨k}0 ,`׺Xà®uZ1,pÀ]kq&åA$0àãdu² p×j8à.¦‘aX€¨NÈÃÀ°¹2™”ÃÜ5I9†¸k"JåÅ sÐ2,pÀ]ãa«ÜÀ]ã•aXà€»Æ™À°‰2퀻€ÏÛè€eX€zž± l2hKxà®Î(dN­&r`Xà»:!|”—'áq2oáåI‡xà. -•eXເÏÄ°?¡¢†¸«-*5ãwµÉŠaî¢oð “FÀ°õmÒ -©KyyR9%†¸ X2 P…'•€$”'$’fX0žúP†¸‹òžà®òÖ2,À]åÐöPt±É0,@_¦-™ÓhV'e€ŒÜUf<à®R13,ÀÝR6„X&1,€‚º`ð $7åÂxhù(/L8ö ¢Á²(º¸è ±ž²Í‰aA4ÐÅð è‚–a:‚)ktßty].t}à  :E†@²)eçD°ƒW/˜)EÀ5å%ðtdX£›PŠ5쥦„IÏ°?•v‰aÔO;†1CL†(ES2èÌOC4Ù Ð邧5Âd  ºA†É$¬2ƒbX€ -µ$\ÍxN•ufbX\˜J«È°Ó+M %åAÀÃħ,¶=Ã,~°ØÄ2‡ò ‹õŽaAJ¦,7” ‹Q†ar³¬7#¬  :>†Y…©´É ãà)Å° ë„õ¦q 0/Ëz31,ÈVa½‰%™·Ù†²ÞŒ ²ƒ.- -åA4M™Ì[`Ö›@'Æ ÐÅdÇ° ‡PÖ›‰aÚ ¬7ÑÑÓ-TÖ›‘nw}21pi`X€¬7ÑÙ²µ½ðúˆþŒ3SYq&ºc ½µ•n(`/ªU¤{ -à[ÖHtÓjbe™Ý5@YS+ºm €¿èÇÜ–Ý'¹E53ÅbdÆ´pèîÛš£³Öý±¿ýùòÝßëŸÿëòö³y§¬þí0<±60X£¡«±×¯ëBYp úûH·>ÈôÙ&.¬ìƒ`Ý„«¨øãh1ê“žMBÁÀ¿xÝ vZ8\g=Ú‡…±û}¤ ;9[û¸0”¬§"ºÅ>.ZŒúˆ}¾4j -YêFûÊÂro÷ûHv r¶öqaeo,]jþl-F}Ä>€ïTjˆÝÄÏcfìí~éÂŽAÎÖ>. ½¬ÍKšíã¢Å¨Ø'“½ÐóÏÞ¥w9§óOaå°Ì;&Æ,ìçì‘I.í9žo FúÎ.¬ÚÃE~Ø¡‡åñ‰VÆÅóŽ=\Ø>v/ãcŸ´2^ÇøÁðt¤ïìãÂj¼¸ÈÆkÐÃòü‹CÝbùzÇ.ì ïE¼^ÅûB2^ÇxÈêÃHßÙÇ…Õxq‘Œ× ‡åù‡:ÌòõŽ=\ØYýñzµ~éVwd¼$]Æ…×»‘¾³ «ñâ"?¯AËó/}Ë×;öpagõXÄë›Õcïý#xoïÑ…œS¼W×T šïjoÊr›†Q®ÎƒuÇèȪß@ß[Ç…m¬ã’_µNhÅ{)A?ˆõ븰“Z-D|»Z=Dï+Y-è{û¸°Mô¸ä×¢7hÅ;Ç!z<—ïXÇ…Tn½oV¹‡è#'«Œ}o¶‰—üZô­x9Dçò븰“:.£÷Íêø½¡¾¬7è{û¸°Mô¸ä×¢7hÅ»Ê!z<—ïXÇ…Tu½Wªú~ì?_tžŒjO}ꇺsÙµÚ¤ë篗ï~ù›û—ÿøÍU_?¿]þðóòŒ‡RæGü¦_|Ò?/w¯ð9à÷†¡Ó|Ù[vøÕøÍë8œç×ó丄cüõz¥•ã¸þ²L[Çázî‡åsyîĹEŽûq¯Õò·è:óÈyõÚMW¿Ð¥¾þË:ö¶êUÎýãõó÷ù±|ìŽ,¾û¨ýƒ/Ëßã½9[m -¿X´«ºaÐÓ«)d[ž«z?½-vÏ>[ýê~è:—¿ýú+ýì¿›å¸Ä ðæ1nM‰+h&w½›?Âês³È™ùENîלu¹-c嵋¾>­y‘–ó}\ìÿåçò@åìƒÿÕå»XçÅà 3™lµÊ3”Çõ¼ÊJÒßË#™9'å¯ÿ}ùÝáDë®ÄHWïæ–Ö‹ž³ZdW?š–çÔ0áBÎ5jÖl¼˜×¦9“O2Àø0)£Eا=¼Ž‘ã×HÏãÍ2ÞÛ:#I„E&µÏY¶ÓÝérs.ù˜î^\#ŒYÛgû8Ó;ª­cÂrN³Õwð·Ólt‹óss6ÊÃR'éOg“ [”¦ï²i¶»"c챚í(¿¬ñù²Ø8çÊ<îÑÙRžíkö¤yylÒIÚ¾AÌ©}zÍ;äÊ+v¯±·ëü©óIðkn×1•6çIÍ‘š¿·>éõdŽ¦Óù™·9^pÖÏ?i³ÛÛ8ÏëšëUo1/« _ùn­oë5É9­Oçã[×kæÇ„÷®UÎXýÏ|hõ·ñ—׬WõóqUÕž¹ÚÙ•_ýQe¦5Vßù$|—æ­þ­ç­òw•ð§N*çi¾Y»óÕQ~¥3ºm-WîζYÿnrªúhæ­±jÇêû–³ocnRÿÝÍ—^qüÍ#uñ(ªî~Ä -o»MóÜ ûœªãjlgÙ‹êø“Ž©É¤±ÿ8ùvê·¼óÛÐWÛ®ÿ8~ ;–.[m:lµé®û?ím0}÷Ñæ æA™ƒzzp¿½Ž=ŒëNŽÉ}?q§ï±ÏOå¼¢S}‡l￧ã{*ÿ=ËøjÝ ÚëùPþœÊ|E7‘üwÉAhr'VãjþT–ÛÉz>Î"qÖ7Ék éFθ8˵—ý#rƒÄôå|Ûë·ß1XõL¢GYvÎäîb9î.ÜÛYð)äÂ7×{…Z“î­Ý¶X¿]·õªÏÔÙ[ ké0Ž™õ©x&ÖÇ¥6ä¤ðÏü\üØgmúÙ+Iü¨5³Í£2þ´ß{¬Ÿ’=Ŷç8êõeœªíÍ?'ý™Ø5<óQùžF²Ú¨çæË“=„ì½ôjŸØ±òëšiîook–/«¯D¾µK®ëLö„uý·1kñò ,Kã Ýd—MbøÇy­£¹ö]ÈòXåå¯9¥D æþ`¥6¦~QxÞÎÑx—¯íƒ.Kù7¬ë±²ŸLyÀQœÐ–’Õä–þn·Ü÷ãîi; Ê6u‡¸÷ÏßÿO¿þ؆nA’icsµ¶|´àÀôô)ÿ·’êzwæ ³ÉZ9‹ç™óÖ»8¯mËlvc”Ÿ\ìözwwcX!±ãë -ñ­ÏΊmW&-2ë*q‹,u¥ÝVæWš{(8 û<»OÑ)nül÷HVපòÚ*ºð+j(¿-‚«ýV’µÊºŠÜ‘È‘92ìJíQü‰îKú)ÓJW»懓݄yõvºrÖþ/ìh~•÷@DÓô÷½RÜÝ‘6ª­ö½ïþh»²â?²ã…_U=ÿÔ_~ë/–7ƒ>¤Ö<¯;ô>Κ³½Cz(ži·JÛì€ÎvšžSRÙI9±ãèžœîB±9´Ù’ùÚƉ™Œ«ôIúmWy{’Úú¤uªçb›Ó›ë¶kmw’ë.eÅå­¢Ãòsjnnw=C·Ev¶œÜÖgì¬Êý9Ƕ»Á ÿîVÌç×±¢‹võÎK÷w»ÃûÕýüõL•¢&ö™ †:0è½™/æ¶×c~~§PèPn ÷;…å+¼Î¦ é¯Ü)LAš¸¦ãš¢~Øäûù]Pyñ(l3åf64 AÒŸ¿K(ÄœÛöÌ=õ\¶±‚“÷Ô7h·ËÐ:ÉÖUÔ.¶Þs«ãÌ"¯ÞWæ!DBàc~â>÷ÿ·½’F_g5ß/á~=Ú9ïrÞú̹moã‹Èú£=ŽM×ت˜Ð‰u~ ë56¦¼Ÿ\÷’¶äCÕñÄOM§Pþéè¼æÕå±]øÖǶ¤*´?8¸¶›—m¸¯ŽmVMÀ×oׯ†F/Ìë„lý›¦m·àî;»;T{[¯^¾N«Oð"Ë;álº7¹ËרϟÎã -}*ïÁN¤=UWóómqm¡›MîÖÏlEµ¿7±‘oµDöíEº„ùuz~%ŒBÕ¦¼3îÏ%.ùZ^A”uº†MÏŽÖ׃3ç7íýÔF­/ßSå}<å] Ë›ùÄÁ:~©‹_/0j±™_”¥fr2õ “ß/åqyGÈÎõM ;äB\?ª&Ë鶧Ti·‹FçíHëë#µâ¹FÛö Œ¯Ö”ó}=­|Ý©:ÈÎ_á^.Lˆ83è=¹X¯&½’ËËæÒd×£ÐTñ¡ê bèäØ.°Ÿx—¬­D¿'*q¹N6®ê£ËËIšëB'6Õ¥ë´k¢çñvÁeí–úN©åWUTõ. J³B©Ü¨¾Žlq-N„t»â'UE;³ñEÔLmý5‘\—ë8Bº]è¸Ušo‘ŠmTê§æêù®JOu›šã í&žýåí™bŽ([?ŽäP"Pc­’ȳ#Þ.å)ñùýâ#í;V(_SO­ÛÈIõ|éÔ˜ª«05+­¼f¡ü¿@…oÔ‚<óíZJ•W¥ô£~‘27yt’¤‡†:Žy@»ØÇ -wé[ÎJé¤ÈøºmtóØ@.ô˜«z¤xŒdݲ®‘ „Uÿ4¬”DU•Àå:Ù·T~È]Oߦ*’½^,h‘î¾Éíäаó]’c‹¸.ï½iú6¹"ç+ñ66·%—ÔOõ"lT«výb‚Ø(tr‡Ñ÷™\gkC§ò.Ïú±O2Vqo—¿\æ2g¬7åM°º¼ÆMv]ëOuý¿ÿyq×­ßYê¸Q -LÃÖ,+Yyégz\^Z‰mÚ¢qö;Y§>Ö×lhuÜ×®Ÿ®:°×‘º!¨n\Iô݇‚lZêm…gü ñ¶ñbkd^M/ØË¡­Æá2¶¼q¦jU¾¥#´Áaê<ÛKòUˆÄQÚK:¸Îˆq …âP\0wÍ5KBÊ‹4×ý …-×¢(‡¶ó\/³Rt¥ÍVˆ!h­<‚–̓>DéO¯—㾪ê (Ôíú.©Í°Bm€Ç©ó×.{“¸¶šc¯å NÞî¡&ËŠ¹g[“Ó§Fo-Üp sTPS?màåE]¥ ­…¢ÁhªÒšÄXu -ëiÕ-³Å‰z)¶ž¢Sÿ"=íeTš½ŒJØ-rkÅkÝ’m —é»uªl½d"´ ÏnT×GÚJÛ_Û×®Õö*Ìp]‘Ð;SÕO󻑿̺·ÊFÕŸygD§þ¦-j§£5v~yºJ!C\yÿetIl•<}ÎùÞÉ~‡¤îÈ+!“³VE]^ƒÏUx#ßgQõùü´¬¨IÏ9àð•e|'|~2þ­Dí0–¥byí¹-Mo*okê`c¼l¹aûdþùà=T§ -endstream -endobj -304 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 303 0 R -/Resources 4 0 R -/Annots [ 305 0 R 306 0 R 307 0 R 308 0 R 309 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -305 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 740.570079 409.271317 725.570079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://releases.rancher.com/documents/security/2.6/Rancher_v2-6_CIS_v1-6_Benchmark_Assessment.pdf) ->> ->> -endobj -306 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 311.068925 661.574079 462.272294 646.574079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke-1.6-hardening-2.6/) ->> ->> -endobj -307 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 646.574079 183.998124 631.574079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke-1.6-hardening-2.6/) ->> ->> -endobj -308 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 351.653153 216.824079 461.230790 201.824079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://www.cisecurity.org/benchmark/kubernetes/) ->> ->> -endobj -309 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 201.824079 165.615800 186.824079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://www.cisecurity.org/benchmark/kubernetes/) ->> ->> -endobj -310 0 obj -<< -/Filter /FlateDecode -/Length 3495 ->> -stream -xœí[K$· ¾÷¯è³Ë’H½Ã@8 bÄñ">9¬ÛÁl€uþûù¨*I¬.ÎÌîúâCf03]¤J¢øøHªjÂÕãûÓ€_•ƒ«µ•Ì×ÛÛË»‹w%y|õ‡‹Îþ?%]ƒçâRh¹¦ë[!Pq¥¥Xòõñ‚_w—zøãåÛO.ÿ9ޮѵ–cHן“+µŸ VË2X)¹RbÊ+‡-¶f± F*ùÌ*ƒ[M bpŒÕ`UˆA\Y bÄØbAŒÐR1XÁCŽ½7yÄ3S4x!¹ ¥g‹ƒËµÅfò -x¡D‹Gärá‚Åk.眪ÅcÈ’š/A–˜½Åƒ,LµX¼ Y(“7y%ÖB¯@–í4ƒW! ¼«š<È⳯‘K­2›¼ž¯¥žyÑ'—*‘·x!¸ß$“‡ˆ‚‘r1x²dŸšÉƒ,‰|´xYß8‘¨ÔjDk”è&OÁâ%Èca“Y`| b†,¾d ANpÉ€XŠãÙXÉqåj¡@¬Íq)dÁ@lÉqnÅÂò¼, Yg (@Fäš<ÈB-, Y0ÀÂ"È™,,  -Ã[,, À-û,, à-û-, .Äô—jf  Wj0—€@á^,,  .¥-, À.qÍp—8x ¸K„;Md[Xtt*YXÀÀ] -¾XXÀÀ]„Q´°VuðÃla|ÅÅZ½… ÜÕ³… Ü)Àä‡$ ¸¶`enànL>ZXÀÀÝÈ1[XGqÀoVÀ]@H²ËÈ€µ°ù¼H`2Q–XX€»¡!ý1 ´r¡¶laAî† -ó™¼æBƸ…–]ȹYX€»HTdaAî>«…Àx‡Œ-,HÀÝ@ÈÔ¸bó$à.ê†daÜÁ ¿¹àáº&¯9ßjµ° wKÑÂdaç+ÄäG -ŒGjaAî"½7 2p×'" 2p×3Ê“Y¨F P×€‡°5bŪóHañ€»>À¸ë}idதM rB¬@@‡*8ZX+»VJ¶° 7J>“—®“…ŃY&²pa PX»F­ZXP"d¡-,(² d°°kÕ·Ù@¸‹Ù0dA¥haÌÕÆlaAIý´jñrtµ®M^/ Ja‡T,,@/å*‚Þ¼“rɈ^` Jƒ, Š\¨Ì-^€,(²-,@¥ ^ÔYP ZXPcE—™½…ÈÐè%¡j‹Ç^úÌjaAåì¤T´°a‚NHiò `ÂÂØGšídaš4›hsLd¡ÄÔ¥Ý¬Ô -Y¢À7¥ß,4YPñYXÐ|vR&[XÏ{ Z¨è7#[X€¸”~³ZXÐÈ£ßD‹gÄm£,ýf±° 1dI(QLdAÑÔŒ¸&¡ß:Y¼ Ybc ZÎÒoV Pf ßDEo Té7‹y\ÜM Èdñ€»¨L³…¸@¿‰ÊÖêí=€7Ôg6³JÇYÍèMÈ­æ‘ö"[óÌÀ|¥G2 &:ËšÍS¤U0ƒ7 <ðõŸ0a;}ÒGTîP3k‘M6„£ºŸ=ÇFO ÛÎÇ~þñòÙ$ßëÿ½<|ÒOÊÆw¨çÞX7‰ƒØ`ï·”DNb—‚¦?^2‹Âîé"çÍÒ垨yŒ½]Y]°SüöñH—:¢ÓÕŠ±æ1Zɧ¨7½EQtÞÓQ0®ÑsÅ5ê¹ãÒU^:9ÐçîÕŠJS¶¾o—oaúwÇ™&—4ݺ @)+uÁ=bÞÒ–ùÎ~œ)æwŒZ {þpð ¸Ä×GW '\Ž®àý=É|†JùL=¸BxÂ*ÍÓöÅ\–+¤'\!]5Þ= +™£çŠêÁè WPô¹{µ¢Ò”­ïÛÿCò7’&vþþõuÂöŒ"K¸[‘,#²>ÒÁ뷗ϾüêoùÇWÀ÷×—~ŽÆæ üö;üTïSÚþ¢¿òŒò/Eü·Q$vž\óÃÎg\c,’…çÛ>&ï¼xØ~ö¹;¯­¹úsÚî2ôë¸Ë"s¼ÙçÛšcýˆµ"è4ÖÜeïãò6ŸÌßåÞ¸x°×;ïå‹]_¿zBŸMëST8UåÕ’a›n¨Dþ&Ú–ìâ õܶë!–Ì#Ÿ)©yòÑDSde.™{ªuÌ[öõŠR}\s¤ße[›ÊáÆ›Ú=Âsj‘G½Œýv9ÕÚ]Gj-Óîdëzæ»ís±»‹ˆk ÷ès×»ýó®Ë¬æ6trgìÛùj?Ú…Ãý|ÏêN~õÝ>ÖÁ>·èsîIí[ü&ÂÕc]zèã•>§m?`Ÿ3´÷õRVúÚÇJˆÍýïki} ™ú=éU¸k(š-ü°‡jZ±Ôç{Øæùòµ<ísýý—Ï~·ããaNrµ¡n’¹’tC\Ãþ(OtƧë/—oP€œ÷I´|nýwË_‡†^»nš²éMù˜‚ã‰ov=¬1¬|oŽ}Øõ}Sø1}ødƒ~Âïc’"lð=îågõ}¸_雡(_Ñé'Mÿ`}¦y^ß*¶Gú:`ÉÀ·ÛÂY_Gz©â¶hÓF·ŽùrÂÌg0ùl·}¬ò™.Û¾pŠAµÎayŸçaáNÓÞÒV1Î]  ™o˜#ÄþLÍ­<£4KÈCqÃÁ©ðýŽGÿ6Œ~Ù},rƒªªoþÑi%ÚÕ%J²<ؾ¿w±÷qËK- õ¯ëù”±u* zÜ!¬åÀõ¨‡ |òæ4ß8/£<`Ú¥>êñϯ^ýáOß,=Ž²eú·W¶ï~ù’͈äX.¿d³'s2혦ý§)ßy`Út3•·V ){§~£ÔìËxI#1[öþà[~ö†æ¦ÔÏÛ;e•ÒûÚ8É  hÃ/ÚXç†dc— -òT\…(9Ÿê`Uò¿ˆ7U[¼YØ:ê…;3)ÈþÚQ›óR+Áš…B¤jî6vi¿ø4|Þ+Ûíïlt¶ë-[àóáëýˆª$,›äõiÊš:ñšÞøâ¼ïÜãí«O;?‘›äÕ"½£ô—EŽsM¾+ï;*1cà­\ {ÚˆïÓea\Ø;;ÚKÀæm£G^¥·.Ãû<ûÜ#%EÃ%ó¿Ð&õ=¤±‡/îÍøî’ûc‰ÐMû\]ˆòîÚ5³“ìTÚU0¥zâ0̺¢Ãõ‰;ûïæ¨ý%@/ïÈ3¡í Au±ßŠ¨·9ÈÕû ;¾“k‹üx‘¶Ø 3¯xx,Äý£Ÿsq˜á5h·K€·û‘‚(ûǃ¤’ÙØÜŸÆmrÎ~ ¢~”¼-lqgg²ì ¿“¥¬Žö«ÉY,0lí«ò›x"Þ.‰g$.òã%ctZXáÓp=M$5ï$W¿üeQKªBhš<î‘ßT¤IäéO—’G¶ëj-"±9ÉG%izž¨Ã–—5V©+”2}Vë+ToŒУ§ÆT¥2E/mˆ' 5ÁãHÓë&Y lègb¥&ú!„œ/Orš®üx /9Ó Õäçb9(wOsÞEÎ;5¹L‹yþ>åó*ŸÄ[Ol|Oׯc‘EŽ~f»µ˜".Á -‹¼`ô±“G´Nt’wŠÇÇdVƽ]~ºô4‡’96|Èäu2]u…ë/ó -?ÿ¾ðõ¯ãùå–ÇQõãÄÖYg2yù¸¾'Y^FÄ6ŠYÁþ"‡ºÆ¦á 3Ǩðš×í~û&‚†ÉÕÄ´t¨Èqºn ¤4“ž!Þî´8 ™_k›•°·K›“–[ž|©p¡·ƒËºx´RòUM‰+ºR:¸ոГâR-Ø–d1Å!õ"Sõ"?kS¬¥.T!—´x¼Ò¬žzÐú.Ô”À¤/ôÄô¢í !ôG‡×¯¶û.r Ô í“ú¨©scB€gS©_æãåÌ9t•‹¼KCs.R±G³ÈY¡±J >\hUÔºnÛ$½(ÚaÑ‘(&\€æmÎX†Ly¿m¨¥ï¸šZ*³¦XÔŸ´¦“¶ÊTbÒVɧ±2ïÈx³Z¢Y%íÎeíί‘³–¬-ë»'•×H´óÚ|\{dÛ«Ú/AòªLýº-F~þ}Lì˜ñÖ¨ñÝOFB]oü Qs(©ÿ—¯9CXÞÃ-\ÕQÉßóüÙÉù„dœè•`ÇD¾ùw<[„G0ÚË 5ÕøW=û¶æÍ-}˜ž|uÊ~lBóø²ÍË"­¢üûIÑ[å­ñâí§(ý(ŽOú÷ÿÚ V -endstream -endobj -311 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 310 0 R -/Resources 4 0 R -/Annots [ 312 0 R 313 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -312 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 443.985184 754.370079 487.998612 739.370079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://kubernetes.io/docs/tasks/tools/) ->> ->> -endobj -313 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 297.976884 739.370079 309.064286 724.370079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://stedolan.github.io/jq/) ->> ->> -endobj -314 0 obj -<< -/Filter /FlateDecode -/Length 3612 ->> -stream -xœí\M$· ½÷¯ès€•%‘ú  øâ F,âCú½k#è °Î!?Õ%‰UÅÏÌîÀ±w<î®G•DQä#USUáìñó*àŸÊÁÕÚJæóåýéÃÉ»’<þ[Zlñßð)é<—BË5ß @Å•–bÉçë ÿìuóëéû?þ½='œ£k-Çο¼“+µŸ QƒÈ2D)¹Rbʆ(‡)¶f‰ F*ù(*Pƒ[M–jpŒÕU¨A\E jÄØAÐR1DÁC½7ePÄ3S4d!¹ £gKƒËµÅfÊ -d¡DKFärá‚%k.眪%cè’š–,A—˜½%ƒ.LµX² ](“7eÐ%ÖB–¬@—ë4CV¡ ¼«š2èâ³–¬‘K­2›²™¯¥eÑ'—*‘·d!¸ß$S†ˆÂ"åbÈ"tÉ>5S]ùhɺ0~,x"Q©ÕˆÖ(ÑMž‚%KÐ%Ʀ º`ñ-ˆºø’- -@9á%‹b)Ž[d‹b%Ç•«Å±6Ç¥E±%ǹ‹ÈÈb°ˆ€YIb”‹õÔɱ) -hk›¾¡‡ªZd‘½]Яæî¤úîV‚Çáöï{‚\Æ{3#—ét£ÿNôJŸ EȘ©Ïá°ŒNyyî),Ͼú\]ˆòrŒsfG².í,ÏZUOú²…ÅÐá|Ç™Ë+E>ŒVë[F¼ßÁË’Øx‹ë×>È„£Ùn¦À©Ha“F¯ Ü£u°“¼´¨AfeÜËéçÓ’æ"¥Øä†e -ò¾ -]u…óÑæ[|þuâó_ú Ì·<ý`‹8¸uëL&o7ª„åmGa£ÀÈŠö'êl›º7Œ<£âkž§ûIì½e †TƒiÚPÁq¸n ¤,“î/;+ŽBæc×f&ìÛ!õÉa’Gk»V8ÐÓÁa2š)ù¬ºÄ‘4)RŽêDèNq¨lS³šâzazÑŸõRŒ¥T!‡4e<Ó¬îºcË,T”À¤tÇô m£¡ô“Ãë£×}U9÷tPû@¯tžÒr˜·‡#çÐYzðN ¾HÅ"g†Æ,-xs }T¡už¶6HzPîØfО(]ó=–®S^OëfYf\M+•QSLôgmé¤We1éUɇ¶ÒoÏx£Z¢Q%íÎeÎÎÏ–£–¬–õÙåÙ’:v›·c÷l{VÓà©Hž•©Ÿ§¥CË'ñßSbÇ|,ÀjÕ–+#¡ÎW - Qs(iyK¤¯9ƒCX^ôS¸ªK%>ç©´ë‘°Ž‰| ò¾O[…+íת«þ.Pû´æÍ)=Îw¾›Á|ˆ#,/|[žþX.Ã"[Ey¿#IÑ[åµTÅßóPGÞ\>Y~þ.¶Ø -endstream -endobj -315 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 314 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -316 0 obj -<< -/Filter /FlateDecode -/Length 3668 ->> -stream -xœí\M·½Ï¯˜³ÑEVñ 0|à Q#Fçä M´6‚U9‡üý¼êi~twí®´«…­ØZ¬fXÅ&‹ÅWEv÷ú3áç…ÇE¼+¥æ$çË»Óû¹ ÿ–›Â¢þ¿9ž=IvÑ×Tâù -8»\cÈé|{»â\ýöô·/NÿÞ^ãÏÁÕš‚çŸÞªJØå’3ECU¡"φ*F—sˆÉP%ï0ÄZ-`Ä,AŽª RK´T€!!CUƒ%{>ª*`„ìÅR†¯1*OÀá‘©á`è|t FO–.x—J ÕÔeè|–ŽÙ¥,>zKW]J)K'À+yK%z²tÀ"\²¥KÀ‰ÉÔK(™-]–ëTCW€ÞUL°P"oé*»X‹ˆ©«ÐQÉå¨ ],Ìdé¼w¾É¦+ -“”²¡ À’(VS,‘)X:Á¥ODÎ¥«5èêfboé"°„ÅÔ &ßb€€…r²(+È)/YrvRƒX$ -;)R,¥:É™-5:I5[<Àä¡ Þ"&`‰’,&`,‚•kꀅ«X\ÀXPÁâf` Â0ØVà-0èV¨z‹ |+ä“Å ÂL²¸€Á¸\’X\À \¸Rµ¸€Á¹ î´¸€AºXîÙâërLÁâí²”dqƒwY¥jqAï"P±Å¼ ú,€ã"j°¸ ‚w=#R[:ð®•,.ˆà]ä Ñ⸃óH,.@|sžàº¦®:ª¥X\Á» ¥`q¢°£&1uÙÁ‘¼Å cGÈC-.Hà]„÷jqAïRd¶¸ wIö˜:`á,.@^–­±¦‘¬: -` -KÞ%¶tà]¢\-.Hà] ›¤¬‚P,.@‡,8X\Š¸šs²¸ U‚)Ÿ©KåhqA&"(ËÔ‹d±¸‰µ«\‹Å9 ‡`qAÀ‚”Áâd®zdßæöXà.æþ@€™¢Å`2WªˆÅ9&ìÇ`UK—‚+tmê -t>[\³8„boqöR®`Ñ[\‚wš.Y\Ðy±¸ W`ÁR±¸¡È•€ÌÜÒy`A’mq2]è¼·¸ `A6hqA »ÌD Bc/ S[:!Ýg‹ Š$§©¢ÅX&Øi‚)M°€&,.Àüèf;Z\€Í6›Øæ˜:`á(”t»Y,.(X‹ à›ºßÌTd|TJNÓd‹ @ãÐY\P}Á~3ˆÅX—ºß,T&ì7±Å3Ömå¤ûÍlqA`‰HQL° iªÆº'a¿ v²t XB‹ jJºß, ÍÀ~½y$Pt¿™Íãðn¬`&KÞEfš,.@ûMd¶ÖÞž@¼1#?³•EwœÅ<1 PoDl5 Ü‹h•Í3ùêÉ<4Lì,K2O V¡ôdøù˜8ázú4Q%¸CFÌ,YY±,Ù}ßsŒj|GµëùØO?œ¾|à{þá?§›/–“²öãáü,I7ÖÊȸ+æ|å–ö;L$ÛA"ÀÇ\£ r =hq=tSÅ·õÜ¿ø-d VÌÉÌ22Ùô‹­‚Ç\q2±~ô5ŸëÑì¿{u¿\Ï6Óy² Y[À^½;}ùÍ·ùÓ÷ߢÍW7§¿¥çDRˆôLA?á4$ü -dë÷ˆeļû~ýó«—s?z“ûšºíG›FzG7¸\»Hk“ú=^õ‘¯ß—.êÚ]ÚvÛ`i}Ѻ—ëw…Þâ·¬º|½FZ_¸–^_û¤µŸE'Wp#ou×v®ž0tWoXË7t\€aÕåm;ˬ–µ½ËäiïÛñ½ö#¬˜÷Vì­¿ÞöÒFÑGÝF|¹">:PõºekÉÖG÷‰ÖgûÌÊ~wëô:zs][}§v4ú[,}¹oìÒft¦Ncm_qÛI¬ µÚÛY¨Ó¯ë#NˆmßlgY)w3‹6^Iwá}rd9¶ýë-!ííñ[lùØÊÁnŸIláÝ[^?oliŒ¹xB3óü±cò‚Æø´eöÎêF{‹îÃí,{;wÙÇØ{"ˆÆyŸ¯¿4Ùhf€YÖÛî ?E8ìÖmVÇ5Ÿ6êè$úQçØÎóD½£tÞ'GcÛ¿î¨#²·ÇsF=ßür"ÌôýÞÈq¯-ÓÁ–?s„‰z/ÿþóf0Üÿm”ù{·yØÛüóÙŸè}Tù‘âØÎóD -½ŸzÞ'GŠcÛ¿îH‘üÞ¿EŠÇFŠÄ[þÌ‘"åèÊý‘âò¼Q"¤#£oG¿eíÆ(ÍS:»Ï9øÛµíË®Íæum?m=ñ1g\©î-ø¤ÝÍW;?ûïT˜¦ö[ou½Þw2³+Ÿ îÛYÆõzØ1ú{q¤zŽ'Ç“cÛókøÍäkÿËøÃu,Ž Áí ?Ly4þÛqÚ̃‡þ,n^û>æ_S;áAÛ–ýøckcæ´´ÊçتKíò0ön+.¯¹‡äÆ+3XqçÍÚÑÈM´)æh¿×u4ìº]w÷Ù¢„¸³Eh1¼Ùvòõo^-w$¹ß‘ŒÞÆ(ë™áT>x•úâ4Hiº‹ù±×<ê.æzóÒ}<ãu\ƒ>NA.R-$‡;šÒz½š¦=ÇŽ7%‡Iñ–œ|ùò÷øë¸Á9(ªÑÌT¾þÎNô†³Ï‘$Õ3d¾½Àre¶ôºQø.¦¼ÓûÐ-ᣠÜÝ]îBÍúDeõœÈ@íoºZÀ‹EôžãÕ}|²³)\܇QnãðaØg¡œË$¯£,7;»¥çFªw)û,œÁ÷åÐo”«7}@l®{²Kóvù!ÕÑO~þñçžòTžÍú¹ÝÖ{zÚR“¾z‚6ÓÓÒ–c;w¥-ðýïÖqrDCQ_BJ5Ô:=×ÓùÿÁ*Ðýñ) öHÍÙï`Ù>¨R5 ËúhÀÚ®YZ»ë9•yÇ« B—Ûά÷Ù›‰ý6÷Lùí ñeÁ´ìçæêè*»Ý^û¾ßÅ-ý­;¼eä±hzû-Ëšðl2í3¶1¦ñý)-_úå|J±YßÐ;'q¬óRÏúÀg!ߦÍ/†öç;®\Þk|ßk­¯:’¾ý O¾^߃œ -ký÷Ë„¿ÃÊ—ÚZ_^K¢E\B+ ñíI_KC,2ã± -ׯÔÛßù§É.Hh²«ûšˆŒ´~$¥™Äõ+R­4®í2}\P¨Uâå¹kdžW&ëèÉùU¬¯öÇk)u(15ÌŽe®;>(öL“0…4u7ÄA¯‚wÓ¥!ìÐgÓùiú[ݧé¼Ox9¡[ÞKoM©¾PÔšjÖ Ôe<ª][Í>!ÒçÉ]N†ðµ¦“ÙõÓ¬écÙWaiŽ$½^mõ ÑådÖ[[‹}¦r¯UÆ¥µY~@®Î¥Þ]¦@žõ]åi·¯[±n[û\S™ü&„—S”¾‡øö³Q;® Ø\oòÔnþ2¤¹4Sai6™>Ôªÿ_©"v©2ÏòÅKw)}0}”F'º6»xk¤Yž:ëˆe/yÔÌåsî>;ÛË2ê+éöÚÝbéd²Ižkƒ§$ÕÉc+öÝëºX)¬Ù§så,¤Ý qì®|»œ±/ÕH½³ä'w½Ý!N;ogqî3îõ-ƒŽ··;ù|^–À&{±º~i q íFg“p) ñ ÑÛEÜVkg'}sº}‹ÌŠ¸—Ó§%ÌŽA·Ü‰½¾47g]þü_Ôy‰ßäüçö”ö5Žêl:·nÄs$ÓW¬ËŠõ•ë&ìËf™hˆ}ucó†ÇC˜øZÆå4ˆ½ÕôA}×ÎÂ8l8‰CwÝày²L¼GxÙY±'2O›°¯EnƒC7¬Ï÷7T(ÌÃA± |žšDI«Ž­„éBæFQœ:¬Ù¦:äÜI7½â—y*ôhe.Li„yèd„Ù¹é&[F1UA -ÌsanXüÜiÝà1@?zy=yÞWÈ©­•vjïÒÛYÚ¦ÒNx¶”—bÚ{Ìá³Úâêmñ´ö¸'9ciŒÔB6…ÙG'i—­âÜ©4Ù¦Ó(:]@FMÖ[Ì SZ/kfYF\L+åžS é³¥ã<+݈qž•t¨«í¶ˆ×³%î PœÝ9ÑѨÙsÉbÈÒ|u—ʨÉMvì[¶}·h{ž†!H™)Ëâ¡æ£øï1kÇ|)ƪÕ~–“_Æ{MÔâ3^þT •”À!¢og)ÓQÉG_sÿÙÉñ„¤Ì=a:œ0SöúG‡l·PÔ‡SSíÙ—U2‡ôq¸ó1û¦å¯Nàßz@Ô‹Y·ŠúGfX“Þ¢ïÆg²_iZŽòæødùùž‰Ê^ -endstream -endobj -317 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 316 0 R -/Resources 4 0 R -/Annots [ 318 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -318 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 205.473057 101.433071 186.723057 ] -/BS << -/W 0 ->> -/Dest (cb1-1) ->> -endobj -319 0 obj -<< -/Filter /FlateDecode -/Length 3955 ->> -stream -xœí\K$· ¾÷¯è³Ë”D½Ã‡1bd‘ŒÆm`ÀÎ!?UzU§§wwœlï`¶»H=(ŠüHªªÆž ?[ü—ÙšœKŠ|¾üxúéD&¿ÚbwQÙ_á7…³%N&Øs8ÿ(ŸL*Á¥x~:Ύ˵ùÓé¯þ±ïcÏΔ 矿{“rJV‹¬WX!˜”\ˆ -+Zƒ%–¢± FHìøÈJƒK b°sYaeˆá9Ydˆá’e1l IaY‚Ö©<BÌÞ)<L„Ò£ÆsÖÄ\\Qy <›œÆóÞÄÄ6XWLŒ1dÇ%²/@–`™IãAö9i¼Y|ô¤ò ‹ËÉk¼Y´S^†,°®¬ò E²¯xJfVy<Ê)yŽ‚ Ù{ÒxÖšÛô*…MŠIá9È)•Y‚'§ñ°†9;)7³†9CGNÃئԛIÂBŸ†…¢‘4YÃÀ8xD›Qo:Ö°~)õfÖ° xB½‰OñÛâ£Ô›IÂÂ% EQyISQü˜„zè¤ñ"dq…5,(1J½™5,@šz½z$¥ÞLêqp7 “Æî"3àõ&2[­¶'oHÈÏtf–Š3«'è ˆ­ê‘{­’zf@_©‘ÔCˆ‰Ê2GõÔaLK걑ñ¶Ó§õˆ*ÂbfN²È·`d÷£æ˜Íü3Ͷ󱟿?}ò€à{þþŸ§ÇêIYÿ±0~ϱÖb_Åì7ðÊÔñ&’m'H$„€ïÒG„He -i‹íÐM„D(F¸-çñÅîE†Ä"³E2SWÆ»yQ*X앪¬oÝç•d:BÝ‹ú/ {»_¯wöyu½®óÞ«×;û¼º¬R'Á9=ñÕ¼Tÿ©²ÞÙçEY½ü;¿¼$2Òmz]Ö¿®Óö -À‹Ú¼÷Úë}^ÝÖyïµ×;û¼†¬GøýÍ›òŒíG}Óú -=NÑëîÆ|¥-ïq¡µ¹eNmÙsÌÃòWC½^~èCÈòC&6zlÓ4ÍÓåeчÖ¾}\Û–Ðè¶áJ³µ«}¾iãˆjý¢ŠÇ©z™·oÇ'4šÛæ·ßiäo©È¢.ÙëÈñ¦ŸnuÛ¸ÛŸ¿©ÞåG”E&ê|X -Æ­PÉF³Gå·íóNÙ+BÉ‹õ_œ”Èû¨ ä;x'÷Y7ÕŒKëÃ-;Ugýý_üöwžÎâg[ùLÛ'ór½)þÙIÝ6h\úô±iÑ~z—[°+·K@ûØ×8‚~lðû¸‡ÞuœèåŒFîÇ»ÙÃR¼%‡Ü|FŽ÷N>Žc×ìãV& XvÏ$ªe–Û!v„ˇ9Ö½ó¼-ÞóxS¿îZû½ÙgI>¾Û+Ïzh©KÛ›ºþ4cEWë´¶Ž¡£²¤ -ôz™`MÍûZl·¥Òx·uÆ×:«ëîvéŸßÃ.³}5;»©¯xÐWO1ó~2÷=ïØvy¥Ô”¦/NŒÞ~ëuŸ³Ùrù [úõN_·Ö,·Ä÷k~›´•QÿåR‹BÉŽ,¥zÿàfÚzgŸ&mEΆj7eÁ´u?É­´µ?LÁÎðÉiŸîMSé¡{I{jØSÅ•—¯Ö—ñÕP·­ %ÚÌäÔ”Ï5§è)/µß -DÇ`~ –ò0L‘ÛÎW“ÖÔòã rwª¢x'ã Æ{gŸ÷9dú%l8¸pˆéá -f:œ_”ä´ÛÆ’Âuûê6µAø¾d!¯Í£—ª5…oYÓ°”DãT¢BöjM 6«庖£%±)%äd¯&¼iIÅv«ðpé,»þ’!ÝÕåC³£Hñ–ͽ}ç21y,1c‘÷—‰aZØÿT©Øi{™G’ÞQ·ËrY’¶ËÞ+÷%â\Gãî‚1¦Uù_z\·$¥š.^·¨KTä ñ¼w+êŽã¼]Q—8<'Ç{uDZÿÿŠº¯uðkQG/u)_ëìע2ôõß(ê®î7Ü‹«¯[ìÉ»A{]8ž8¢Þ»Ñ0¦î}äv÷zí¾Ç—iëÃN{‘¹îko¯éùµð¨ÛdÛ‹k›m¶koÞËñZ=.ð5·ùG-Ó}§Ï×Û>,ØÖm*ÐÀîí¿}v¿|˜:¯è¯(¬¹Ï·é¯ß6û´øíj¿K^qK?ùZ?Ã6,ÍÛ”¯9ٕͼ÷~ »»¿÷£p}û¦”rvÙ›åºÒû;û|0‡%$y:2´é¿Ì!Ç~’·»7÷ÁvyF6[*é?xرŸôªDýª=²–:I7å9®²<7,÷Å&/êñy¹þÐÉÙ^=â±W †”wU}>;XLIòxçsÕ½ûnScÅP‘ƒßúŽe© [åº:¼ù‹ð¾›XÕßs—Ç £„î®rÑþ]Ë1Ý3x<Æ¿…ÿ} ¡¯á°?b}LÙÖWU(fc¼ËzŽl¼ì‹@ø—gÛ·ÍVEÛó3=ëÀ?Ví¥`’÷„äñíá墵ÿ©nø()ÙÄ^DÖø¨’xášüt’8Y!3ÏJô²ÛWc!§qW´  8™rÝpGíë¬~·¯.ôÕHÿлÉCL½‘¯–n+DôŒöH–Õ“±,/ÁfãÛU¢Èóƒ8äµípv|‘l=-Äp$Ò2Ý$;îòXyib¨.Nâ}U]¶¿·uiÙÎ[ÄËIŒ¼¦>©Tyõ®ÕµëhÐü"TÔÐ[Ž á±O -érRˆSš±™ƒ¿ìš¼À°s7$íJo§.'µ]-ŒJ£Už]K×üešºÏCñ -í²œ%y«ñòýëž,c¯)/vãÄË)ððÄI~:…¤´+P¿•#Ñ/ãr¦i/“šrW\³ÓäñoùƒŠ0¨‚<õ‹åaRr2;¯æ$⛃¼WÒJuXÓ€å4Û.ê²) ›]õe3)ítG롱uQÙBOã)^©{²V7Èa]?+W"u!0Ý$‡aÊO;ò”3 W 4&‹v1÷0Æä8°ói%§±ãVÞÇòŽq›ïÄK l|MÓÏ}’Iv4¢Ýœl!NÁ -“©?ÿözn½ -endstream -endobj -320 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 319 0 R -/Resources 4 0 R -/Annots [ 321 0 R 322 0 R 323 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -321 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 728.673057 101.433071 709.923057 ] -/BS << -/W 0 ->> -/Dest (cb2-1) ->> -endobj -322 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 521.926036 101.433071 503.176036 ] -/BS << -/W 0 ->> -/Dest (cb3-1) ->> -endobj -323 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 136.679014 101.433071 117.929014 ] -/BS << -/W 0 ->> -/Dest (cb6-1) ->> -endobj -324 0 obj -<< -/Filter /FlateDecode -/Length 3542 ->> -stream -xœíœ]oÇ †ïϯ8×2á 9_@‘‹)Z Ôh/‚^(§v‚B.àô¢¿/÷ì|ì.%Y–ÔÄu-X:KÎÎp9œ‡œÕ®ü™ðõ…Ç·"Þ•Rs’óåÝéý‰\Ž„K‹ÍÁ¢þÿs<{’좯©Äó;pv¹ÆÓùö„o»Ã¹ùí鯿:ýs{Ž?Wk ->žz£*a—KÎ U…Š<ª]Î!&C•¼Ã%Öj©`FÌä¨Ê0Cj‰– -fHÅP˜Á’=Uf„ìÅRÁ _c6Tž`‡D¦†Cç£Kpz²tÁ»Tj¨¦.Cçs°tÌ.eñÑ[ºêRJ±X:-±’·t¶D/B–¶—léláÄdê`K(™-]†-Þ©†®ÀDW1u°…yKWÙÅZDL]…ŽJ.G] èba&K罋ˆM6uXQ˜¤” ]€-‰b5u°%2KÇ°EðeéÀ‰È¹cµ]ÝLì-]„-!d1u°“o $ØB9YÀ -rÊ%‹!g'5ˆPØI‘bQ ”ê$g¶0jt’j¶8Àä¡ Þl‰’,°‡-‚•kê` W±XÀ¶ ÅfØ„-0h+ˆ‹ Ü -Uo±€Á[!Ÿ,0€ 3Ébƒ¸\’X,` ¡T-0˜Ë`§Åt±Ü³Åu9¦`±€]–’,0¸ËâÉb€»Ì8ÓÔÁ°ÅÐѱ/l±@À]ö”-¸‹e,`Vâ0Y,@¬¸P -Y,p7‹`ê²Ãd‹K°27¸"‹î ÉbÅ?dVà.ír¶° ï¡ l±¹€²ÄbAw}Eú3Ö4hå|©ÉbAw}Áô™ºê|†Çu /;ŸRµXÁ]$*¶XÁ]à³X,ã2j°XÁ]ÏÈÔ–Üõ¡’Å‚î¢nˆ Σ°X€üæ[,ÈYR±·X€½”+Xô x§å’Ř‹¹Â,‹HE®Tæ–ÎÃÙ PéBç½Å‚` ªA‹%ì2Y,@†Æ^®¶tBºÏ, Š$§¥¢Å,ì4AJS[€ ‹˜ÝlG‹Øü`³‰mŽ©ƒ-ÅbA)A·›ÅbA)°%P°X€ØÔýf¶XP ¶ â³XP)9-“-ãÐY,¨¾`¿ÄbÖ¥î7‹Å‚Ê„ý&¶xƺ­œt¿™-T-%Š©ƒ-(šª±nÁ$ì7A'K—`K¨b± ¦¤ûÍb±eö›¨èÍ[E÷›Ù¼]îÆ -2Y:p•i²X€ì7QÙZ{{xcF}f+‹î8‹yÇ€€ÞˆÜjÞ2 °Ù*›÷ ðÕ=’yÓfbgY’y×iJOæmQÉ ×»Oó-ª„pÈÈ™%ëEV, Auß÷£ßÑìzì§N_Þ ùžø×éí¯–;eíË£@XZ ê+,ƒP1ÝÕ·ÇåsÂs± †=¦­šë0ʃëM65 -éNÿ6–ÂÐo·Ãz\óÛøPãg6’Û¸(s°ttX*1D|Îà´ÊúÉÔGžòŒvl(Úp ŠL¬»$›Æ¿y}ÂêY­¸Þ˜í‡žc3 Ç]B<¿~wúò÷¯^ýöwFl¿~{úî×hzC$å«/<>Ç|ý)iý¹û¿?ÜFûAMJrYÛ­z•E^eq=§lÛôc¿öÿ·óëW§¯_¢—šÖkLê[ì½Ór¿ Ûªžc¡ëÕ}ýÍŸþð—oÆÕacˆÞµWüןñ:ŠÊ°Äë0¤^“·×6°f< "*ŒÝ€ß-ÞùªÙ» Ô8-nO±A/DÚ‡ò”H{‰Ó[­÷DØ“£ËŒìÐi²«N±*EoÉúPÌØ@IˆYÜýLט="·‘›|«%º1òýj¯\òµíg‹¤+eô¯ã ÆŽa;®Ôõ¼zm¯1ÚÇå5¶ãÔ_Ú­¬Ë±Ïye.öÎ×’†½óu,}\WŇ8>ÌŽ_|­\¾þ_ú ÃË1meÝž¶~ÛuåUvYý•×k¬ãúó'Ó\å©osÔp9¬÷ùZ´>Û¬÷Δ´òâíµ¿Æ¹ŸÄZœë àc? {”xaøz±»ÍíÛqKÜÄ6ÿ÷Ú[Ó]öÒ¸æ¥_™ü®\4®AaHBªFß‹Íi°3¦áÛ>·ë<.cÔ§ÛqgëuÆËê|ot=®º|=§ùéçÃfÃ";¬Ói-¾Yç/{›-ýÒ}¾f¿÷‡´¸ZsÊÖ˜×5¿ŽÛÖxˤiØßú½fÈ1KßéF¬}[×ßüÖûNÊ¡¯>x•cßìh»Ê-¢¶ÈþˆÚÚÞìÎiÑÑèøýDWnuÏ®æy’rÞûøº’Vû œfÑ»tÏOEýM=ý¼ õ7wØûd*ûþ¼©(yïωŠú; y˜Šeâ¥ÈØ ”¶º%ZÖ¾›WtŒ_f úÏóÞó3ã´çO· -©:yÞûyÞ&ŠwÙûdÞûþ¹y»°äfZíZë.»x -Û˜[üz™86_WȽ¾{ÜÅÏvÄòù]+\e{S#ÊL“#IZ-ò·»?Ʈ˔×1ÒjË(öL“0…4 7ÄAš=^Ÿ©î®KCØMŸ]ç§éomCž¦ó>áå„ay/½5¥úfNëªy7P—ñdTíÒØZö ‘>O†èr2„_PëZ0™]?Íš>ß|–HÒÛÕÖÎ]Nf»µ·Øg*÷VeœZ›ç‡)#Ô¹tDzËT¨g}éwZ#ÄíãV¬5WŸk*SÜ„ƒðrBM&ñí)f£u¬ ØBoòÔoñ2¤¹4Wai6™>ªß¯¨ˆ]ªäY>èa¾u0AtmvñÖI³P¬ï.7a_6“0È„ý!öe´-zׇÜòN¦§Ó{ké;A}×ÎÂ8|8‰CÝàyòL¼GxÙy±2O›‘°¯‡Ü.ð>(߬ÂÁ|98,CÇ#%Ÿ§.q¤MGJ‡VÂt"æNq8 X‡e35 çAºëÕ~™§‚u¬é`*#ô‡NFš»n²å*¦&(y>˜;?Z7öFôòzò¼¯&§¶TÚÑÞ¥·³´_˜J;ðl)/‡i{ØsŸõ -Þá¡ÞOk{‘3–Æ(-ds0Çè$-ã´µAœ•&Û ÚEÇdÔd½ÇÜlJëiÍ-ËÓK¹×Cúãìé8ÏJwbœg%Új¿-ãõj‰{çpÎãêh´ìµd1di>»Ke´ä&;Ž-Û±[¶=O—!Ã4*S§ÅCËâßǬë9»»V«öµÜñe¼ „D->sàåo¾PI }m7K™n•<úœûïï´{'óH˜'Ì”½þõÛ„[(êÊ©«ö—}ìÓ*™—ô8Üù¦ÕQ¡w†–?ß°¼:±Ü ê‡Y·Šú×ZX‹Þ¢/™g²ß<¹þ~sûdùú\Å£ù -endstream -endobj -325 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 324 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -326 0 obj -<< -/Filter /FlateDecode -/Length 4635 ->> -stream -xœÕ\K$9n¾ç¯Èó£‘Dê,æ`` { /¼pc}øÐÛ³ £ÆÀ¬þûþ¨="’™U]k—»¤E~¤ŠpW‹¿ïþËìLÎ%E¾Þ~¹üz±&‹µÄ梲ÿ€O -Wg9™àJÌáú‹(™T‚OñúrÁ»Ë¹øËåß~sùÏmwõ¦”è]¸þõ«°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ ±1BbÏ÷¬1¸ä ± {ŸV†ÄÉÑ=«@ Ÿk,ˆáJH -ËYÈá¼µ*‚Xfò -Ï¡ô¨ñ¼31_T^Ï%¯ñˆLLì‚ÓxÅÄCÖx YB±NãȳÕx…)'! E²*²øœHã%Èâ¡¢ð2due•Yl´Nã2¡df•WÀ³9å{ž·Á„Ld5žs&À6IåÁ£0I1)<Y¢ EåA–@Ök<‚,Œ?œ”rV¼Õ‹w“%§ñdñ>±Êƒ,˜| |„,6E àAFpIß’áâYŸÉp次€ÏÅpJ¤Á€/Áp,Iò<ï4 YG ÈA†çª<ÈB…5, YP@Ã"Èâ™4,  -ÃZ4, À-Ûâ4, à-[5, .Ä´—rd  S*0—€@îž4,  .…è5, À.qŽp—ØY ¸K„š*²À€5,:r™4,`à.9›4,`à.ÜÈkX€Y5°Ã¨alÅøœ­† ÜõÙ²† Üõ !@å%$ ¸ë† Enà®ÖkXÀÀ]Ï>jXC1À«fÀ]@HÐÓÈ€Õ°ñ ´2.—¨aAéSyŸ+~ -cÑ° w¨HÂÜ|f €ñÕkX€»Ž©5p×ùb5,À]ä A؃qH4,@|3ÎÂtU^1¶ä¬aAî–¼†ˆÂÆf ‰ÊK†ä4,‹kX ²`¯aAÌlJJQÂX,xHùT^4¸$ Bd©<ȉ5,@bm -•¬aAò…¼×° yÈ‚”AÃd¦8dßêò²À\ÔõCdŠÉL.̤±ƒV5^ô&'ÀµÊË๤aAJlŠ†XK™ §×°o$]Ò°b€çXÂT \EÄ"“=2sç ’l é‚眆ÙCdƒdŸ±ÊŒVÃDh¬%¡jÇVÖ™YÂÌÑHª¨aÜ+M ¥Êƒ,€ 0?²Ø`ñƒÅ&–9*²P` rö²ÜÌä Y¼õÀ6e½™4,(² ãÓ° Øh$MÖ°0žµ—±Þô¬aüRÖ›YÂBëM,ñ¿-e½™4,( YR•Y4ÅoIXo4^„,¾°†%FYof f`½‰Œ^ÝȲÞLêvp7 “Æî"3àëMd¶ÚÚÞxCB~¦3³¬8³ºc`½±UÝ2°À^D«¤îX€¯¬‘ÔMˆ‰•eŽê®Â*˜ÎªÛø‹|Œï0aÙ}š·¨"Ì!!fæ$ƒ,p Fvß×£=(¶ìýõÏ—ï?#ø^ÿü_—ŸSwÊÚŸƒñGYX3Kï æxej{Œ,+!r¾©’ˆ‘ÊÓ×m7Á·\û·2‹ÔéLo:ÆbÁa¶(êÒ¾¾Ò{H{¯ú¿ûtAçmß·ün\N*Å2Ñù|ýôËåûßýþ_þé¿GŸ~¾üô[ ããvß_~øÎýVò'¸I^~·~Çõ»Ñƒµœ—²˜1Ëk](¨^ßÑײŒ@ü®í´´×èüóúÉ âYÙ ©m»ÑžÊ8½m.k¿øæÛh·Ê×Ê~ÞÕYÛlcîc@ý@?üûõÓ§f Ì3ðSk %÷òViM°z=)¦£ŒûÓcžÔ³_0‰¯º ºÎ9©Ðfyç¶p†ÍÖZÝ}û¯’}Ï6×µ\\çxž×ª×gúôöNŸ<Ú®¿Ã$ÓjÏ]s¹¤ëlžß†µM7ømôáÜyð“-ó¼A?5äÜþ¶áF¼¦µÁaŒvõÍS'd¦µ®¿ ›½ˆãQ'oÝ„©¤Yö+´÷Z|[КmôÿcüÁ -Èäwˆ?÷íÜûëS9äfòß(®Ü·Ý°ÿÎ'¿LøØ,ò絟Ï;Ÿßãà4?ƸgŒšqí®? £×¾[ŒÖðqùýT·q?þÐÚ˜1*®ô9ÆŠÛÜŽeïzÐâóšƒpjX·"È>þH/k;ÖŽEú˜bˆôÛæ£ÉS¿f¬õ¶ç=ß­ÖñOu–÷:눥.¾¿®þ±A¿=ò-eëX¾.s÷¼_Æ‚M¡Rë×·\£Íý*?ÅÕÃÀ¢~*€G!(íÛ5ñþÝ'¬Åê}cÊXå_í–µ˜ÇZÌFd=ÈI:˜öõÛëë¼iù¶®Ú°>´( -&/{IÖ[²å»¥\ØF~éuy7Úøþüñïÿá_ÇZ®™[ Lu -ý½«V7÷[³l&QË·”ÀÛMÀW¨u¿LS Ç$úrXvCQbÉ]¢Œxå²™§àõuþw¦€[¯ËôKG¡Ë‹òd -¬d]¼zQÞå0aù®ÞZTnÑ{¥çqÚ’µ•åm{§!±Üh" ÐiÿiÑoPú¶—"w³m)åäFBÎuSü¹dçê¼ËÎÊ{H<ˆ»fL {l ‡-$ݶ“ºIÇv†ÓËRK¥Fµ6ãâ©^kcNõ6Fx—€oÛšÛX°g¢ß¥¨SÛá@öÌá@æÖÞ}º:ÝŸŽËlBË<¥²3ïIîšùhl[;0ßsUþ¯Qmˆ ËLÞF=°”IY¢D]M\ä†[Ï¡xÄ‹äôt®ÎÇQTd#Gh¬êÝ<¢n<4k/[O[–—;Oßy[G„tËîKÏÉãf”ඳºý~*sòôz;dÙx±Û]§‡“~¡yÌb6±É-Èçær¦ÆÇ1–,1,ËéDÝXt¸ † Þ¥ ¢õÄz9WåÃ(&:¹½èäHî3/š-ªÇ¼]ΔÊá´c¦×AGŠXÐ9‘£©ƒ<„ŽSU>ŽbŠ &ò¹­w;ÚÃÅa$-‡Ÿëá!ï †déh s²Î‡Qm’‡(²·‰OÁHW]ت´¹vƒ”}αÉìue#eÆXåÞT?/ú^ ØV r:BŽ[äÉ^G’«óñ°)F#Fu3;!´Þk»#”×Åìêq>­eÂ]j´yàU)ÉêÈùx¢Ÿ}¯‰Îr,×LW¤ù˜ÂáRïd7ÑrÐPÎö… /úZοnCÑWù¬›µ›—CzÎæâvÍÏ[·kç`–ÔÚá¨v‘Í,9 ­Õ–ÛN__g¾Ñôè fÙ¥âé-òÊC3vý[tE@kL|dMWâxþHz‚)¸â(ªú:ì¿>NÉrüSû™ñ‡bê#‚Yí¿î£âsÔJ}Æ•VÜ×#ÁjÉȶ3¥mUÜn_íá,ûdRŠVN¨’ö',ŽÐã\“$ÈS´¥„ìÏëŒCwÙX—ù]³?)+„å>ZK»¤Lf±%¥ÍC‰äÞËF˜Z;œHh&Ç¿3 AäD’ÅÊIÐ,Ç7Ÿ[Ç©*Ç8¢éìY(o6ŽM³³q„/gö™2ñ„$³ê’<«WϬ?Wü©*Gñ{üÒo:ô-ë‡ý'o69äÞJíðN÷çŽ2Ê€@Qœ·Q×áÉŒ2 -›5MœÊ(#ÒˆúM¯É(å9¸1ÒÛóåÉ:&v¡K„Ú"ïixϤaÛìûd”÷m¾&£TjcFYä5Á{xÝÙŒò\•cò²‘T_Ðò®Æ±iöõ¥ÜV•‡ÙŠ§SÊ“u>Œêur4‹ÙéŽy—Tþa} •·àÄâK™,ëj9,r …*¶úD×Õír6í8Zˆ&åPXm× Ÿä©Ií±¨ý“òTG çíþB™Î¯¯©™o'+Ý8Š×Oš†å8^KRú‰ÈõÄ®(Sèõôäô´@û½?Ý_û[Oþ×£}ë ÎMûíTï$Ï椫ôÚ~ØO㯗XŸþuõ ΃´¿dy`ßH¦“ÊUž8Ζ(×isUÑîú f}±Ö¯½”ü’•×oÈ£×uZ狵ü¯u¹È_Í¥äh|}/Ž­äìÛÅ ¿\ä½H¬™Çùú²ן¶·Å®Ôl´ü™Ç¾$cןãL?ƒ¡Fê‡VMžWeÛ -Q}bséX!¢ft÷d½5n%Ë»¥$ô,W±‹b“ÄÑ ÏeGƒ£â!Ù‘ˆážh§îÙs“ÇÉ»ºêâ vÑgÕ¹iú[YŸ¦é|F¼]Ð-í©/*UÞhÓšjÚõ¶Óhªtjh%û„pŸ'…t»(Äïlkš1™?Íš¼`!æfHÜË•VN!Ý.j¹µµÐg*õRyT-MóC”aê”»âÚmzî%ÉËò&±Ô~nÉò˜DŸk›'»ñwÄÛy0ß‘_.!)¥ÃÀ -šéÍDšÚíäl‡½ ªY~Á5Mžª–ÿ¨*ÈS8î&%·ÆÕèD|³“·Jšé±£kpœFÙI].¥n³³¾\¶JyÝ^ºklCT6Ñ‘ÚÞtìà±%»nu,ÖôÓ±r&Ú&ºäÐMùeCr†îªÁö΢›Ì=ôv9vì|™É©Ï¸“×\ty{»“Í7â­6Þ“Åôsëde9Ò̧w6‡`…A0úRÉÍ[;:É«ûÚÏádZĽ]þr©aÎc!!™h$'omš³.wýo”ùŸÿ¸ðõŸÛk–8ŽzëßwlÝçH&ïøË'ÉòοFìn3=O°?È.²¡YCãÞOxÍ£ºÀÞJºŽ ®sgb:œÈ¾›®<ë14žo;-öDæ[çfìå’ÚàdûM^0ѤÂÅ<’sUG#$_§&q%EGH—ýTs£¸œ:,C²˜bs']õ"?ÏS!7æ‹)K<avnºÑê(¦"Hi¾˜–;´£­²‘GúÍîõÍ󾊛µC{§¾ÌÔ>0¡vÀÓ©T/ãö²Çª»»Íy‡†z[4ùõ$g¸ÆH-xs1ÛèDÍ£ÚZ Ìr£m:m¢Ãh¶Ñz‹©É×jM-uÄYÕRê9Å þeÖt˜g¥+1̳ïÊJ»-âõl‰zfsNctv”ì¹dVhq®Ý©©ÿñ̲ -endstream -endobj -327 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 326 0 R -/Resources 4 0 R -/Annots [ 328 0 R 329 0 R 330 0 R 331 0 R 332 0 R 333 0 R 334 0 R 335 0 R 336 0 R 337 0 R 338 0 R 339 0 R 340 0 R 341 0 R 342 0 R 343 0 R 344 0 R 345 0 R 346 0 R 347 0 R 348 0 R 349 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -328 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 507.723057 101.433071 488.973057 ] -/BS << -/W 0 ->> -/Dest (cb9-1) ->> -endobj -329 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 488.776036 99.933071 470.026036 ] -/BS << -/W 0 ->> -/Dest (cb9-2) ->> -endobj -330 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 469.829014 99.933071 451.079014 ] -/BS << -/W 0 ->> -/Dest (cb9-3) ->> -endobj -331 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 431.934971 99.933071 413.184971 ] -/BS << -/W 0 ->> -/Dest (cb9-4) ->> -endobj -332 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 412.987950 99.933071 394.237950 ] -/BS << -/W 0 ->> -/Dest (cb9-5) ->> -endobj -333 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 394.040928 99.933071 375.290928 ] -/BS << -/W 0 ->> -/Dest (cb9-6) ->> -endobj -334 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 375.093907 99.933071 356.343907 ] -/BS << -/W 0 ->> -/Dest (cb9-7) ->> -endobj -335 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 356.146885 99.933071 337.396885 ] -/BS << -/W 0 ->> -/Dest (cb9-8) ->> -endobj -336 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 337.199864 99.933071 318.449864 ] -/BS << -/W 0 ->> -/Dest (cb9-9) ->> -endobj -337 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 318.252842 99.933071 299.502842 ] -/BS << -/W 0 ->> -/Dest (cb9-10) ->> -endobj -338 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 299.305821 99.933071 280.555821 ] -/BS << -/W 0 ->> -/Dest (cb9-11) ->> -endobj -339 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 280.358799 99.933071 261.608799 ] -/BS << -/W 0 ->> -/Dest (cb9-12) ->> -endobj -340 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 261.411778 99.933071 242.661778 ] -/BS << -/W 0 ->> -/Dest (cb9-13) ->> -endobj -341 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 242.464756 99.933071 223.714756 ] -/BS << -/W 0 ->> -/Dest (cb9-14) ->> -endobj -342 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 223.517735 99.933071 204.767735 ] -/BS << -/W 0 ->> -/Dest (cb9-15) ->> -endobj -343 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 204.570714 99.933071 185.820714 ] -/BS << -/W 0 ->> -/Dest (cb9-16) ->> -endobj -344 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 185.623692 99.933071 166.873692 ] -/BS << -/W 0 ->> -/Dest (cb9-17) ->> -endobj -345 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 166.676671 99.933071 147.926671 ] -/BS << -/W 0 ->> -/Dest (cb9-18) ->> -endobj -346 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 147.729649 99.933071 128.979649 ] -/BS << -/W 0 ->> -/Dest (cb9-19) ->> -endobj -347 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 128.782628 99.933071 110.032628 ] -/BS << -/W 0 ->> -/Dest (cb9-20) ->> -endobj -348 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 109.835606 99.933071 91.085606 ] -/BS << -/W 0 ->> -/Dest (cb9-21) ->> -endobj -349 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 90.888585 99.933071 72.138585 ] -/BS << -/W 0 ->> -/Dest (cb9-22) ->> -endobj -350 0 obj -<< -/Filter /FlateDecode -/Length 4932 ->> -stream -xœÍ]K$9n¾ç¯Èó£ÕƒÔXÌa5ì1¼ðÂû`øÐ]ݵ £ÆÀ¬þûþ¨ ="ƒ•™½[]¨® Q¢ù‘T(”îlñóƒÃ™œÉ¹¤Hç—_N¿ž¬Ilñ¯ÖX]TòŸð›øì,%îÄÌç_¤ $“ -ûÏo'üwu9W;ýÇoNÿ³nãÎÞ”½ãó_¿ -‰‚I9%Ë -©€d]PHÌ&%ÏQ!Egp‹¥h$ˆÁ‰9ÒHÃN -ÉYÈá¼µ* ‚X¢àšc1èQ£ygb.¾¨´šK^£…`b"ÇN£cä¬Ñ²p±N£1daGd5d¡“F‹%Ä`Udñ9– ‹Çè…–! ´+«4Èb£u­Ã%©´šÍ)oiÞ²á‚ÕhΆn•‹Â$ŤЕVŒKqÅn1ÊÆÅX4,`à.Uа€»€Ï¬a0ÞÀ£z ¸ë<µFî:_¬† ÜEÜÀ@ŒC aü›qª«ÒŠ±%g ¸ XòÀ ›$*-(’Ó°`l,âP "pî½hX»–Cа w-!ìQi%d¯aâÐ`¶ŠM#X5Ö)4p×:Ì°FîZ›Š†¸+nS˜¤Àg  Aì5,ˆ™LI)jX‹ !ŸJ‹׬aA²(`@–Jƒ,”HÃÖ¦„’5,H²ï5,H² dа)Ñ·š@¨‹šdA¤¨aÌäB¤aAâˆ| £ªÑ¢79®UZÍ% R"Wì4,@.e2Œ^ü‘pIÈš# R,0 àŠLöˆÌ5šƒ,²5,@¤ šsdY jX}F–­†ðÐÈ%1Ô¬ä™YÂLÑH¨¨aÌ™&R¥AÀ„†˜I¶YÃ$?H6‘æ¨4Ș4,ÈÙKº™5,Ȳxë5,€nJ¾™4,(² âÓ° Øh$LÖ°0šµ—‘ozÒ°v)ùfÖ° ‹|)žb·%DÉ7“†… #DQiASQ옄|è¤Ñ"dñ…4,(1J¾™5,@˜|½º$%ßLêrp— I£w™F p|‘­–Û[/'Äg:1KÆ™Õ èeøVuÉÀ{á­’ºf`¾’#©‹™eŽêªÜ*ˆÎªËø‹xŒ6˜pY}š—¨"Ô!Ágæ$7Y`„è¾ç£Zx§Úe}ì¯>ýöœïùÏÿ{zýM])k?Ê(JbÍb"¾`ºW¦²#dèˆ -c !à#mDˆT†°¸,º‰pÅp·åÜ?¸µÈXdvfêÑÜ/2‡© -Q—õî6O’uŒ¢YäïÞ;;‚nøqìÚ¸lóôqûƒ¤Ézw›§Ëšc$d²6®ûU…Ü«¼+]çí‡!«“?*ä^å' ú ZIp‘^€? -%(™$/¼¼ˆ€°Ÿü¯óÏ?½/%¢-BÊt%å®`·ÙÊ$¢fäu[¶H=-D¶´°·aGÆPŸ‘Ķ[f² ›/ƒ ŒåsÀ¯ûrùì?I¦‰@‚!EÚ½ÆAž19ăJcL½Lap—Û á"!:Km:ÓRwíN‡²4Y‘o;”§5öË–õ)D÷ʜۗKã?ü¼6=Àò [J9{àDÛr;æz°ÍCöú=ÌɶA[L¦n§Ÿ3øŸ+ø´>Dy€ï’ª»]YÁ¢=N%›Ùéûž_HPr ÅÞVÇÚ|°JòLŸ‹õÏ«»jáΨ{¬ ·'Ö†ç]€µmý÷¬ûïg¬í|ÝXô=Övv¿ °i¬-ŸGë}i°vÓ® °¶œÞ¬,ë¶ÅEöÖ3nWýxZõ"»¥²“$êL)(Ÿ#ÙÁ6o%Ã{Ä©ÁÉö½g@éš]5–X• ·ðU~÷82¶È“w…ã>ôYoR±²–¤´>´ˆ2 W¹#¾ËëyÅF³‡ÓIÙ8–ÏX ;=/»yÁKàÞÑ©ÈÏÛ>æX›¥{Ù(Ûhò ×rÔÅÄl,bŠà¯Ø>ìbÄcSFJUŽ¢ëá ¶­÷ŒaŠÍ•Öi¹ëÞI1¨$ëzܽì—'àk­é¬ùìªõØk59Öÿä*•{ë›-géd}Ù¯³ï´&× Ü~Ëò{Lß?—ÓÃŽ‡µ‰D›0rª}íKmÊ%‡â™Í)ÿxP›( šeŠÙ2A›brYG>à¿ì×ÙwÐ&q)«3pÔÅE Z}¹Â*\¹¸(Oµ#sÒFã.ëƒH—d,k-í>>Øæã¸8ÄÌ1æpÀÅuuc)iÍþ9KI[ž÷,%)­ùâxû#ªW=+j°Q“œdÿ™ì-Dv_B¨«"·µäP“£$òfOvfù]”dÍ~VžžkŽU©ë €6ˆ¬eã.{㢅åíLÀ±&g<çì;?5]³mϸê€ë2¼|Ýý~x 5ù8Ͳ% JyÊbÒšÝ -þ4ÑÁEȲ󃉛·ýQ?Öæã {’×IÅ'ë÷Ší3­-Ç{-¥õ‰–ÒúîDë F¢õÈÝDKi}g¢õHÿ#ÑÒîýoŸhi÷p04žVÒ¶\…ÆS¢µåpOhr6‘m¬› †ÆÛ|P"+ïYy7þ{D=köÏ ·<ï •Öß“ ¦àšÂáÐøX“£$^ž•ùÌü}”dÅþþÐXÞ‡v)ÚGlÇš|œ `„:…™íSC‡5ÛýÐ8jälŽœ#y%®n¸=Їš|œ–W°8×>%4^³;0ÀâÈ­#Yã’áBÖb÷vlóa†˜-°Ï㓾ÛwµÏów‡wŸNZ½îàØ.ß±GxÛÚ½¶ßã´a -À•Öû!t÷Jëï°¨ñºg?Ñû÷umŒö%’¼†½·ÛävÕoPûu/+ð1{»/Ðͪ të}9ÁAθqòª©o‡íAƱ&ÏôÉÐ!§pÙ"‡©ÐñXh9CǪƒ»¡cÛš‡qõü* +¹Èqlõ À9ÛcM>Œ'ˆVÎ’¨çŽÝšÎwƒÀ ot,ýÅ¥£xžßø¤hlòrÚ–ôð‡?þë?ÿûGžd:ÄçHÂýââ}^zZ¼ÔáVÇ_pÐ}ѧMÖ€7–× )É™OŽì^4z°Íß{â¸Ë‹¤±9ÝÛ. y)j¶Å}.»ZÚÛX=6š’°kwC׫Ÿ×ÄW|g÷ÕÊÒmiÓw0u¹wW=R[õXKßµ( _÷œ§Qzç±d¥íìýé¸â>W£We‰Wwøz—I%9´JÞ— ºI-¦Ãa2Ÿ´|^Ì+ðåšÓ¢×*ñeœG§²‘*!Ï5×V{ÇüãÍ”¼½B=’å¦ùl³k~7^þV˜bºe…²±x×÷,›»ôWC'Ý¡©ÝÕ‰Ôã«€ý²™ùHß?׺ló-­ë•°æf“ÖÕ%žEÛr•c«qpH…srWÞÒ89UfÑžùécM>š¾eÊ·ôíæ|«3¼ÀN}öËyÞqÿ‘¸¥Î;ü${úÛP< ]ôc•w¥ãýE€Ew.º\Ñõ¬Ê—ºþëŒÉ‹þñè§éãé†ç(ì¢ÏCç¦éouˆûC…/'t®KßÔR º«6ºÞö²0 Uz)·š}B¨Ï“RôrR -°5a2;}š59iðR˜›"Q¯WZ=¥èå¤Ö[¸qŸ©ÔkåÑ´´‘¢ U¹¼Rö2IŽßŸlĆöq]eÚ\#ðzã7…/'¦n‰£øíÄI©Í+,7Õ› ÃÄ·g;ôe”Ê {—O0ÍV&ç´Éÿ¨à^*ÈS?8ê*%g-Ž«Ñ‰Øf/^Ò\;ê6ŽÒ¨; —K©ëì<^.[¥¾€n¯ÝGlU: ÙTžJO@ªƒÇºØu­ëÅam|:VÎ…¶ îF1wU~[9¹›*ÛÞYt“ºsç;ŠcÇη¹8õwrpf—·ót¾¾TÇF×Å¢ú¹u2ŠåáMSŸÞÙT8(Œâ£oµ¸YkG'ù2€öq™æq_N9U7ç{YÆ‹ÁÉ9ÐsÔåÎÿ‡:?á÷¿Otþ—vðàÅ{ÛÑÂwl]ÏžL¾5 ,–oh…Ýl¦BOìb—G]nÚÐý¸÷^Óhn°·š®#¨ëÔ¹ÇNž«®wa¾Qør5Š=ùÖ¹ûrÚÍ¡› GV6©p1ß.ó …á’ÏK\IÕáÒA%?5ÄÅÌ—S‡eH¶SrÈOóTékº˜Â¹ ƒFÃÍά[Y½‹© -Bà0_ÌŒÉÍ–•<ŠÐ›×7Ïû"rl ¥Ú{éÛ\ÚoLJ;à饡^Æõe÷9¡®£6ã#Ôy…ÉöBr†iŒÐ‚V³ŽN¥y4[*ðÜ)µ²U§ÍQt¸@™mecj2Å¥Y–zÇY¥ÔcŠQú—y¤yž•>ˆ<ÏJÜÔ¾Íãõh)ôˆguNãîì¨Ùcɬ”Źu/¥Q3´²mß´î»yÛót4‰#2µ£oj>„ØÎ¥ÍÕ*„V«ýÔ•—ÇQ½rpªKÁ‡úíK6ÇèÎu½•åi©äî6·×N¶+$mídî Óa(›œ|–.ÂeŸ0±jß±¥7“‡ÂÊ-Ý7ÊÒЀ9»Yª_¤R¥ëQ¿”9^¾7)HЛåë’uï¯í~^-ŸÔŸÿS†Ò‰ -endstream -endobj -351 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 350 0 R -/Resources 4 0 R -/Annots [ 352 0 R 353 0 R 354 0 R 355 0 R 356 0 R 357 0 R 358 0 R 359 0 R 360 0 R 361 0 R 362 0 R 363 0 R 364 0 R 365 0 R 366 0 R 367 0 R 368 0 R 369 0 R 370 0 R 371 0 R 372 0 R 373 0 R 374 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -352 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-23) ->> -endobj -353 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-24) ->> -endobj -354 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-25) ->> -endobj -355 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-26) ->> -endobj -356 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-27) ->> -endobj -357 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-28) ->> -endobj -358 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-29) ->> -endobj -359 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-30) ->> -endobj -360 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-31) ->> -endobj -361 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-32) ->> -endobj -362 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-33) ->> -endobj -363 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-34) ->> -endobj -364 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-35) ->> -endobj -365 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-36) ->> -endobj -366 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-37) ->> -endobj -367 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-38) ->> -endobj -368 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-39) ->> -endobj -369 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-40) ->> -endobj -370 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-41) ->> -endobj -371 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-42) ->> -endobj -372 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-43) ->> -endobj -373 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-44) ->> -endobj -374 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 340.338585 101.433071 321.588585 ] -/BS << -/W 0 ->> -/Dest (cb10-1) ->> -endobj -375 0 obj -<< -/Filter /FlateDecode -/Length 4737 ->> -stream -xœÍ]K$¹q¾×¯¨³€å’Œà t0 Á^Â/äÇÙÞ B¯•þûþ‚™|dVTevO¯UÓ¨îÊ>"ƒ_™IŽ»Zü|ãð+³39—ùúòóå—‹5)Xü«%6•ý'|R¸:ËÉWbן…@ɤ|Š×× ~í.ç⯗ÿüÍå¿·uÜÕ›R¢wáú÷ÏÂb2)§dƒÂ*`YG -+“’QaEgp‹¥h,ˆ{¾e%ˆÁ%1Øû¬°2Ä NŽnYbøäXcA WBRXÎBç­UyÄ2“Wx.˜¥G特ø¢òx.yGdbbœÆ+&ƲÆcÈŠu/@–à˜­Æƒ,L9i¼Y(’UyÅçD/Aí…—! ¬+«<Èb£u¯ %3«¼žÍ)ßò¼ &d"«ñœ3¶I*…AŠIáyÈm(*²²^ãdaüh<àD ”³â­^¼›,9 ‹÷‰UdÁàkà#d±)j2‚Kø” ÏøL†3g |.†S" | †cIuày§YÈ8jH@²0•WŒKиâ·Ð²q1 pŠ4,À]ÀgÖ°oQ½†¸ë‘Zãw/VÂÜEÞ4,€9‡@ÃÄ7ã,LWåcKÎà.`ÉkX€(ll’¨¼d`HNÀ±±ÈC5,ˆÀ]„÷¢aAîÚ@¤aAîZFÚ£ò e¯aòðඊO#Y5Ö)4p×:Œ°ÆîZ›Š†¸+aS˜„ೆÐ öÄ̦¤5,ˆÅ‚‡”OåEƒë aA² @–ʃ,œXÃ$Ö¦PÉ$YÈ{ ’‡,H4,@F`ŠCö­N ÌE0dA¦¨aÌä¬aA -ó1hUãEor\«¼ žK¤Ä¡ØiX€¹”Épz ðFÒ% xŽ5,H²ÀU4,@(2Ù#3×x² ÉÖ°™.xÎiX=dA6¨aAö³Ìh5,@„Æ\ªÖxlež™5,ȤŠÀM0ÓRª<Șаã#“í a&?˜lbš£ò Ö° g/ÓͬaAÎÅ[¯alSæ›IÂb! 2> ŠFÒd ãàY«aAqóMÏÀ/e¾™5,(d1ßÄOñÛBQæ›IÂÂ% EQyISQü˜„ù&ÐIãEÈâ kXPb”ùfÖ°iæ›ÈèÕ%,óͤ.wC2i<à.2Ó¨a.0ßDf«Íí-€7$äg:3ËŒ3«+Ð[Õ% ìE´JêšøÊI]4€˜˜Y樮 ¬‚鬺l`¿ÈÇø–Õ§y‰*ÂbfNr“nÁÈîûœc£;Å–õ±¿ÿåòí'ßë_þçòå7u¥¬ý8?q¬kÌobñã ¼2µ=2 Z‘«ˆð]•DŒT†˜À¸.»‰˜Æ¸åÚ¿¸­ÐY¤vHgê½ñ¦cLF‹¢.íÛ+}„´·ªÿ§ï/À˜e}3^'%ÂÑ<¦ôýÏ—oÿÇÿ×?ÿm~ÿåòÃoe-ÁZÎÖʺ‚ü…áX~Á‡A[¿Ì.ÆÞýô»ÿº~ÿÝÜ,ÇD‹¹MÙö#M#ū͇¼T—ëÚäKWrÍk÷üeíæÓJëµ]E㥠Oøöǵ®Ðã½öáôWÖÛû2ÑÖ¾¥Oí¶G›7·?©¹ìo?´&äöÃÚL\éqífÕ¼}9½«a®ÛÚuë-¬t·¶ ÍÒæZ®ÖùqmGTK“*¾ ÕK¿•ï‡Zk“k;N"ZZ>õÚ¯m§¥¾óT%_·ºâÏ‹žš9ˆ¼u¸ót¯qé§Êüy"¹¶?-÷[MFꥇ}#e¸§·­oÏkßmØ×{¢8Ú¬ôuL4/°²h€˜”ö--åÿ=à¢.m""àYV¸EÆ‚X *¦î¶>{èóö:ïB˜X02^Ü›¡â%ÝA ÃDÕò Ú„Ö뢆~é1Õlòn´ñí¿|÷Ýþù?ø4S«Þ×aõ·^Z=¼ }3»K\ù“ÇVSYÿò“úe¿£u‘…L{¯óC€7Ræ+22ñYýo¯óÿ£~n½.êï—b#M^ñ@ýV¦¼zUþÝ7NP+®Ãò·zöú_Vžk?®Ci€¾–åm{÷†Ác^Ęºâíöý -¥o{ñHq0Â¥\an.’Ü=–ì\ üa Cpäv¨Öä@5Š^¶ƒº„´u€w†ÓËÒú7:j;;lÆÅS½ÖFðS[³6òN–|ÛÆ‚;S;eÈÅ_”z´¥µ¶ƒÝÝoÜÝÇËVæYÎÚ/?u§‹¡Ã;Î$ dH:‹¬a€ˆï|dº§ªü£mˆ+ËìLÞFÕ`5Ùç*} Dáº3„ΑŠ½;M±ÙË”ò±ÏÕyE̵ýCÏoÖ½QÚÞJw´îiW§y׎ß=¤Üzä¾ßÙcº—~9çů›BÙÜfýþI—©—Ý˽»ÿýýíï!¼(†øÛ½º‡“<6ÇMfYI–õƒ,“þ‡–y®ÊÓ¦<Nè¶$ÝÃ?팩 rÚ*ÿVÓ=o'’ÕW—d!3ø“ËG‰àÉ:Ï£TÊ““·^zû>ö~ylÙZl›-›ËI¯Ô&DcsŠ×lb!’5éÇÃr¦Æó J’ö¨ÀrôA¹Ù¢ä’;äÏUyÅ Vˆ1ÓCk1¼g™»L©Yt›´ï•É6Êr²<*8ퟬó4êÄÌ`U(ݳ³Eý#¹/ß댾­ï”¡Þ}è]ÖevÈ;%Ú•¿ ‡µÍ_1”¾[¦)Ä> -ÍíÞë³—ÙO ÜÓ$‡Àâå %cšFò”á(.¬ó<¦Éldʉϙæ?øÓ‡NÌ秕.7…kY†äÏ»!ó”eJ·^ïçö^ì“ל¯¯œŒ}§j<ÏÀ'y5 º¼)öqö†²<˜BJ“äuc:ÌOÖyÕ”h"<"žô ëuY^ì°CánÝEËÕö3©ö¢ÁþG®yô§6ŠÕ):-û¢}°ÉÄ`#ç³qªÆÓ {ð€îdååì·xD¨ïéŒúš£©7y” ž«ò<Š‘·ýÆBxq'ìœlÈ7n1¿‡³MOtHCúnl 5=»Ðr²ÎÓ$9ÈÛAèúÎâqu´ÉfDšms?ÌsèýÔê<|Ü´y,Ñ·µÃ‰Å ÉáoÌB6ÒûL@‰³è¹*ÏcItÊåÎBÿ[bÓÜlažèÜ]˜ÈYkË}Ä$9àîå{¬ðSUžFᲇ¹‡œq¢†ïöàWUOÞå@ ÙýQæõ¸èGe^…$ŒFLâzXô#ò¬mwÑJÅEªïkraÇGˆ~²Îóå„À?9'&ýñÄ×æ„ÛæßšÞÖ–™$Ïe½õ&[by²c…3“WkÿjyåÈí4™ÏfF#CS´w&3š2´ÛÞ’9w‰ìÅö²eÛÛtøòÉ:OƒÉ²•Ä pc„?$ -îÚûa5"¿f!bHuž½]3´Ã´ß{kœœ÷µ.³vïë1rµú¯æc²¢tÛvÏM)ØŸÖý»0X{ rv™ìh-Óvànt‡El¬ÊªF’«Ûáövo+š”ƒû(_«Ã'Ùë~ok«ÿ¼h³¾ÕWM¶'ÆeyÑMvŠ¹ž}Û€¸n(l!EŸu#àgÛ1ôM‚ëþUøºP'eÛ®´<¾·-¶õ}Åu3‚ÿd—͇²ÛmÝè¸i¿íqäÙlü,ËÛË=Ü ã/—XÏlpõܳq^ö»F6b@99'"[J6W-ÏVÔšõ8Ä_z)ù ’•C“dA§ë|±–ÿ¥øÏ™3ˆþÅk—ÓÌl%gß.ùõ"§Ù±Bfn› ÉB\¿ÚÞÜÂïh/@Qàö¾¤¬˜®_û.˺ÃnùŠ™oõC«&§ °m…¨î³_:Vˆ¨Ý-Y’egi6´^Å.JˆMfGƒ<— ŽŠ‡dGv"†[¢ºdÏM'¯™tÕÅAì¢ÏªsÓð·²>MÃùˆørA·´§¾ªT9‡¬5Õ´ lo4š„*ZÉ> ÜÇI!½\â7¶5ÍÌΟFMNsYˆ¹÷r¥•SH/µÜÚZè#•z©<ª–¦ù!Ê0uÊ]ñ -­êh¥&9âtòKíë–,gô±¶y²C|¹îž8ȯ—”Òa`… Íôf"Mívr¶Ã^5å¦*¸f£ÉYò{ŠÐ©‚<õ‹ãnRržÍ¸ˆovòVI3=vÔaMŽÓ(;© J·ÙY_.[¥¼€n/Ý5¶¡N*›è©4ñ¤:xlÉ®[]' „5ýt¬œ‰¶ î9tS~݇œ¡»j°½³è&s½ÝAŽ;_grê#îäp¢.oow²ùF|©÷d1ýÜ:d™]5óéMÄ!@aŒ¾VróÖŽNràjû:œL‹¸/—¿^j˜ó¼äù‘œœµ7g]îú¿(ó>»ðõßÚá.K÷¶£…ïغ!Ï‘LNfÍ'ɲ™º»ÛLDÏì²Ë£lhÖÐã¸÷^ó¨n°·’®#¨ëÜ™†'²ï¦+Ç Í„Ä—{"óµc3örIíæРɱ@M*\Ì·ƒË> -endobj -377 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 709.923057 101.433071 691.173057 ] -/BS << -/W 0 ->> -/Dest (cb13-1) ->> -endobj -378 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 690.976036 99.933071 672.226036 ] -/BS << -/W 0 ->> -/Dest (cb13-2) ->> -endobj -379 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 672.029014 99.933071 653.279014 ] -/BS << -/W 0 ->> -/Dest (cb13-3) ->> -endobj -380 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.134971 99.933071 615.384971 ] -/BS << -/W 0 ->> -/Dest (cb13-4) ->> -endobj -381 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 596.240928 99.933071 577.490928 ] -/BS << -/W 0 ->> -/Dest (cb13-5) ->> -endobj -382 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 577.293907 99.933071 558.543907 ] -/BS << -/W 0 ->> -/Dest (cb13-6) ->> -endobj -383 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 558.346885 99.933071 539.596885 ] -/BS << -/W 0 ->> -/Dest (cb13-7) ->> -endobj -384 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 539.399864 99.933071 520.649864 ] -/BS << -/W 0 ->> -/Dest (cb13-8) ->> -endobj -385 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 520.452842 99.933071 501.702842 ] -/BS << -/W 0 ->> -/Dest (cb13-9) ->> -endobj -386 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 501.505821 99.933071 482.755821 ] -/BS << -/W 0 ->> -/Dest (cb13-10) ->> -endobj -387 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 482.558799 99.933071 463.808799 ] -/BS << -/W 0 ->> -/Dest (cb13-11) ->> -endobj -388 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 463.611778 99.933071 444.861778 ] -/BS << -/W 0 ->> -/Dest (cb13-12) ->> -endobj -389 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 444.664756 99.933071 425.914756 ] -/BS << -/W 0 ->> -/Dest (cb13-13) ->> -endobj -390 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 425.717735 99.933071 406.967735 ] -/BS << -/W 0 ->> -/Dest (cb13-14) ->> -endobj -391 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 406.770714 99.933071 388.020714 ] -/BS << -/W 0 ->> -/Dest (cb13-15) ->> -endobj -392 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 387.823692 99.933071 369.073692 ] -/BS << -/W 0 ->> -/Dest (cb13-16) ->> -endobj -393 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 368.876671 99.933071 350.126671 ] -/BS << -/W 0 ->> -/Dest (cb13-17) ->> -endobj -394 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 349.929649 99.933071 331.179649 ] -/BS << -/W 0 ->> -/Dest (cb13-18) ->> -endobj -395 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 330.982628 99.933071 312.232628 ] -/BS << -/W 0 ->> -/Dest (cb13-19) ->> -endobj -396 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 312.035606 99.933071 293.285606 ] -/BS << -/W 0 ->> -/Dest (cb13-20) ->> -endobj -397 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 293.088585 99.933071 274.338585 ] -/BS << -/W 0 ->> -/Dest (cb13-21) ->> -endobj -398 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 274.141563 99.933071 255.391563 ] -/BS << -/W 0 ->> -/Dest (cb13-22) ->> -endobj -399 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 255.194542 99.933071 236.444542 ] -/BS << -/W 0 ->> -/Dest (cb13-23) ->> -endobj -400 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 236.247520 99.933071 217.497520 ] -/BS << -/W 0 ->> -/Dest (cb13-24) ->> -endobj -401 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 217.300499 99.933071 198.550499 ] -/BS << -/W 0 ->> -/Dest (cb13-25) ->> -endobj -402 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 198.353477 99.933071 179.603477 ] -/BS << -/W 0 ->> -/Dest (cb13-26) ->> -endobj -403 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 179.406456 99.933071 160.656456 ] -/BS << -/W 0 ->> -/Dest (cb13-27) ->> -endobj -404 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 160.459434 99.933071 141.709434 ] -/BS << -/W 0 ->> -/Dest (cb13-28) ->> -endobj -405 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 141.512413 99.933071 122.762413 ] -/BS << -/W 0 ->> -/Dest (cb13-29) ->> -endobj -406 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 122.565391 99.933071 103.815391 ] -/BS << -/W 0 ->> -/Dest (cb13-30) ->> -endobj -407 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 103.618370 99.933071 84.868370 ] -/BS << -/W 0 ->> -/Dest (cb13-31) ->> -endobj -408 0 obj -<< -/Filter /FlateDecode -/Length 4928 ->> -stream -xœÍ]M¯#»qÝëWhmàÑ$«øýr²&‹?µÄ梲O -gg9™àJÌáü³(™T‚OñüqÂ?W—cñÓþêô?Û:îìM)Ñ»pþë7a1™”S²Aa°¬#…‚Iɇ¨°¢3¸ÅR4ĉ=ïY bpÉAcA ö>+¬ 1ˆ“£=«@ Ÿk,ˆáJH -ËYÈá¼µ*‚Xfò -Ï¡ô¨ñ¼31_T^Ï%¯ñˆLLì‚ÓxÅÄCÖx YB±NãȳÕx…)'! E²*²øœHã%Èâ¡¢ð2due•Yl´Nã2¡df•WÀ³9å=ÏÛ`B&²Ï9`›¤òàQ¤˜ž‡,ц¢ò K ë5AÆ_œ”rV¼Õ‹w“%§ñdñ>±Êƒ,| |„,6E àAFpIß’áâYŸÉp次€ÏÅpJ¤Á€/Áp,Iò<ï4 YG ÈA†çª<ÈB…5, YP@Ã"Èâ™4,  -ÃZ4, À-Ûâ4, à-[5, .Ä´—rd  S*0—€@îž4,  .…è5, À.qŽp—ØY ¸K„š*²À€5,:r™4,`à.9›4,`à.ÜÈkX€Q5°Ã¨alÅøœ­† ÜõÙ²† Üõ S€ÊK>H0p×Ü‚6sw}°^ÃîzöQÊþX5îB‚@¬†˜ïÁó¤a3a‰†¸ë -¦?ŧVÆå5,À]—1|*¯— qÅo¡eãb,à.&*Ò° wŸYÃ`¼ÁŒê5,À]G˜©5p×ùb5,À]Ä A؃q4,Àüfœ…骼blÉYÂÜ,y 0 ›$*/’Ó°`l,âP "pÓ{Ñ° wm Ò° w-#ìQy…²×°q xp[ŧ¬ë¸kFXãw­MEÂÜ•iS˜„ೆÐ -öÄ̦¤5,ˆÅ‚‡OåEƒë aA² @–ʃ,œXÃÖ¦PÉ$YÈ{ ’‡,4,@D`ŠCô­¦æ¢æ Y)jX$3¹0kXBD>­j¼èMN€k•—ÁsI”Ø`*v —2N¯aÞH¸¤aÄϱ†©@¸Š†˜ŠLöˆÌ5žƒ,²5,@¤ žsdY jX}F–­†˜¡‘KBÕ­ä™YÂÌÑH¨¨aÜ™&RåAÀ„†I¶ƒ†H~l"ÍQy…kX³—t3kX3dñÖkXÛ”|3iXP,dAħaA±ÑH˜¬a`)vYûëŸO¿þŠÉ÷üçÿ=½ÿª®”­ŒŸ8JbÄE|Áp®LmŽ´ºéÔEBøH"•.¤,.‹n"$¦bL·åܾ¸­ÈXdvfêñØ/2‡¡¢¨Ëzw'ÉÚu”ÑÂAŽyTR#ÈÿT½NÖyº^Ç~»’4Yï®ó,Yií°†´ AñY&ËDu¢ÉL™RIï«ñ 9ÑDX¥¨x™‘3È%,o -ÿ×Àiéþ²îÝ.±#14ëíùËϧ_ÿóO?ýã?ý;ºýò~úÃÖrþÍNþ¿üâr-ôÿ:ùéôÛ/[5Àä¡(„‘¥:"ª¡àMÍÍUyHu !»FÀ‰ÀLÕ|~Suóykˆ„#Ù ¸Ø´& ð|Q3&ÝúñYVýðqº·xL=NÒ“¼ïÀ†ÃÊ5¦Æ +•!MøID¢Ø%c¿HöÖ¥”;©åP'ILõ Kù©Ä׶4@ÐiKÁh'Yv«áçM[›«ò2¶†ü )(âN§»'"K~ûÌbAKäwöH-7‹~‡:¶½ I÷9RA²^“?’l®Îë«Õ‡Þ–)­\À›ݾ¸¤ŸÎfD Ûæ/SÀQm™ñ™A­}Ïiª-²Î§Õ†ºÝ!À;'„È—Gúæ–î}0U¥6Ï;Ǻ ãCý#v,ÀS¯Ö3÷Ô•(e¾6­uúçz#G­8ŒKÅ“ÒŠû6q'ä‚KDÖ¬(¤Åªÿ؃¸àtsÈNf\¸…eZ‰¬oƒÀdפÛÀSyîÿŒ¨mÛÜýx¾uÔÝ€Èâyb–şɸf®Êë ‡<‚“5ëü@\ãcu@—Üa\s»è³âŸ“,JÉ‚#!xÍ…:Ð\׋käÉv¨ºªöî߈=9¸ÇG*mñûÇ#(´òÎùì¯Äûó6Häáz:)Ù·+·àê—öë-峟·*8×µÝ}«ô>D,8ð7U®Üïñ¸%’j.&ÒZZ“½‰”T5Ø7Ï”&G/[ŽövÑg9,Q6Î×'{É}"> -{'ë¼ ÃYùÃÌPT‡ _‡Hám»ÜÓ—yf"ámG÷FÂûÚ! R ŽËÇŽ;ĨšTÇ]Û¿'fÜ·Öæªk½-«!Á]Ê 7ÈöÃl5mÌÅ„`í%]°K²sÔHˆdáæ^R®>¸ºísu^&ô $»âæ§D‚ÛæªÙÒ6Ê;œ½F'ص֌?v70¥ÑvmÞ“¤)µ—Yá.Ãÿqf p0ÿûe& ±+qƒ6&ïw8Î}ÃBÊ6iµ×‘ y>îHò7‹©kÒ,qÇq;l½I”=­¯‹µ=ŠÄ\&ÕÚ›4ebáAvÐ…â’jéŸF9C6£‡›EŸ=°#C#†>û06¡äd׋dðùÂSàrÛÜãëo} lßâ1¼‰[[05*µïX=Sj!ódªs0> ïŒèjF¥öÌz…bêËù¡þc2%r*jí™õºËi•hýO®× )×¾•©Øl€Ñ} ÷¬×ɾg„Ý -¥ºmÕ]–åoâÏ\—‰Ò8 ª,!ý¹ôvfá'fc1é¿jöaø!`×âDtÕb3¥öOg•Úi‰?§—Б §Cܽ¼½ÄÌ9hµgÅgW_íµ™ÌõŸë¦é¡{ïËöŸhÿÙà?Ïýò²Øšçt]Í‚ŸØ ÀVõ 9ðë‘ß¾…»À/¢Mä›ô -^ˆøúhq|²Î뀟l€ F:~³ Ø“ÌmóÏI2÷mÞ“d*µÃ’`ò6ÁÜl@º±5„ÅË­c tŽ¦…ã)rªÊË ¼ÑoúÖï5’m󣑄a¡°ç²×Ð*â6y xã"’ãi®Êë €¼Ú”)Ò­`³Šï!ʶÙã§aq[>foç=Uåu„‰nYú| -ºmî±]Žòº˜¼Pëä-6yK(×7;nk}®Îë¨]Þõ·E^™~ª}oš}F¾oñž\©}G®Ô¾;D‚‚?r÷=Wjß‚?ÒÁµ{GH)‹‰éò¹g}C“Ÿ²|îXkx`L†EÛ‡­bx4ü€Ãrï£2$ZÅ°ÂñHÿýQôUÐŒ.² d–@ÿ;­€m·«Žë‘AÖî8xÝÏç¬B^††V‹î¯‡Èà äiõ£"/M{—“îé3VÄ/l`ÿU„¤HºSÖfŽf4iQ^‹Œ-}bSépúfÀ—êéc:¼ Š-ä½³Ó¯•ÌUy™ !Zy/ºž¡£n6¸#Ó."f@Pç3ýÉ:¯£.’·†˜ÝLŒeïNâ¶Í?'Óß·yO¦¯ÔþÎL £ñͺéL®ÊëI¤’ôú71’Mó÷gú1ÊûÖãvgй*¯3Y0µõ-fBÛf3ýX@iÅÏ¿”8Uåe¬l¬óÁÞ²ôùLÛܱ‚1oC„|,p -v|”ÕOÖyË»µ]ë»7{Ä~\6ñLììV½í`n`ßC¸¯íÞ×ÏüSýdÖ”Úw¼£Ô^ =skôñžžI~~?;`oÈg¤ç£­6·‹~‡¹o{‰²K^Î ;èfчºõ‚x=& õ\äXÅXI8Uå’>2r=WŠäÐO2 )ÈØvp/d(µCÏ4ÖM¾ëp2c@œ«Pœ›dçª¼Ì IÞ‰‘¸ãæp~ü]ÚFÇÒ_\:Šçñ€‰D† “âåÉÈo÷oÿò¿^a™k¤iý2µ þÕ^…/eÂZÆ_æ"÷§O†­C©8YRd;· ?Yçï=p¡<àêY«1é…Ý2=ȉ¹£/îüóÛò¿ïËîŸíãmIØÐõ.»Ð߶õZý•>„+o•»•=|Ûçò´XÔ¹UIÛºËÚÛÔ¬RW »qíTÞ·+̟ܵSÚ¹Ò\•åZïWWœ0JöOÃȬmÀð×]±Ÿ/Fɉ±TdÓý^U·ªÃÔ¦ØÉA_ÈÎå¡t„ssUþÞîÒÅ¥lrt”tœÛ™²½ ãJ*¦DWÂ'·`Y ÏÒò}Á»zn†”[°Žß«]†»w*¯vÂGå]Ém§u€;þ~9{)ÉAΖ‰Êp”SÔÃ"ƒ¸jg ÝU@²U œæS’œæZ·´&9§LSŸ¨Æ»¨±N ±‡Ärdїž/ ÀµÌªNwQwCÄÁdºÞ7Ûv«WõÇe8Þ—Ýëv™šâåÓ†&.í,m·éêë2loËT–¯ÚK߃<µ0ÜCXïa7Œ¿œb=oÏÕ3WmÌuW–#2¼€—ª¦l‰Û1H®*Ú?©Y²ÿ¥•’A²rà­¬Ôa/–ò¿Ôÿù$›®ëk~rG=‰ÚVröëE'œä$rVÈÌÆO’…¸|µ­-Y»»¢½¥’)×%1sÚåkiËtzùêÃz7R?¬Õä„8Ù±^‰\:Vˆ¨Ýž,w„p!ËiîÙÐr›(!®2KØØä±lo°W<$ËùöD;t×ÉrÖÌZÖ·¯¥9ˆMôQunþµ¬OÃpÞ"¾Ð-]S?Tªœ!½6µj‰÷J£A¨Ò¨a-Ù„Û8)¤·“BüÁ®M3³ñ‡Q““8/ļ·re-§ÞNj¹¥µÐF*µR¹W-«æ»(ÝÔ)7Å+´ª£…šäç)±´~Ý’£ŒÀ:Ö6vãwÄ·“l³Ù‘?N!)¥CÇ -VÓ‰4´ÛÈÙv{éTD‡‹ªàš+MÎ1”/PU§~qÜLJ¢š~Õ;ßlä­’Fzl¨Ãš§^vP—K©Ùì¨/—­R^@·•nÛP• tD¶5xlÉ®Y]# „­úiX9í*ºëäÐLùcCîr†æªÁ¶Î¢Ì=´v;96ìüÉ©¸“ƒe›¼­ÝÁæWâ[Øøš,¦Ÿ×N:YVWói Ä.@¡“;Œ~Tòê­ äÇ2Ö¯ÝÉ´÷íô—Sæ</t$'ç¤Q—;ÿÊü„ÏŸøü¯ëÁœ—yÜÛ†¾aë†<Îdò«y’,¿²±›Û DÏìw²Ë½lX­¡ÍãÞxͽºíÀ¾–t A]ãŽÄÐu8}3]ïhÐL¸A|»Òb d¾wlú„}¹¤õæÐ É‘®«T¸o—¹ó¨OÉç¡I\IÑ>¥ƒË~¨ˆ‹±Q\–.ÙFL1ȱ“¦z‘ŸÇ¡µ¾ñb#ä’:û4;6½Òê] EÓx16Ìnì´läQ„~ؽ¾{Ü‘ãêBmÐÞ¨#µÝ˜PàéTª—q{Ùæªyæê¼]C­-|ZÓ]£‡¼¹mt æ^m)ÆNy¥m:]'Š Ù•ÖZL«Lq©¶ª¥ÞqVµ”ZLÑ©5ÆQiJ ã¨Ä]YiwñZ´D- -£9§~w¶—l±dVhq¬Ý¨ÜKÒJÛ÷ÍÛ¾×Ùö<ÜwAbLm¯v%¿G|çRçjB+µþ­+#.÷£¬1Q³Kä©þ:™Í1:yFŽ”À(ÇŒ¡©õ7èôj²ÑL¹¥û , u˜³»•¡úCCø³,µË$©¢ü®IЛåçP’uêRQ]B¡ÍòIýûÿ¿ht& -endstream -endobj -409 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 408 0 R -/Resources 4 0 R -/Annots [ 410 0 R 411 0 R 412 0 R 413 0 R 414 0 R 415 0 R 416 0 R 417 0 R 418 0 R 419 0 R 420 0 R 421 0 R 422 0 R 423 0 R 424 0 R 425 0 R 426 0 R 427 0 R 428 0 R 429 0 R 430 0 R 431 0 R 432 0 R 433 0 R 434 0 R 435 0 R 436 0 R 437 0 R 438 0 R 439 0 R 440 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -410 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb13-32) ->> -endobj -411 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb13-33) ->> -endobj -412 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb13-34) ->> -endobj -413 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb13-35) ->> -endobj -414 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb13-36) ->> -endobj -415 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb13-37) ->> -endobj -416 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb13-38) ->> -endobj -417 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb13-39) ->> -endobj -418 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb13-40) ->> -endobj -419 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb13-41) ->> -endobj -420 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb13-42) ->> -endobj -421 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb13-43) ->> -endobj -422 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb13-44) ->> -endobj -423 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb13-45) ->> -endobj -424 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb13-46) ->> -endobj -425 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb13-47) ->> -endobj -426 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb13-48) ->> -endobj -427 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb13-49) ->> -endobj -428 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb13-50) ->> -endobj -429 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb13-51) ->> -endobj -430 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb13-52) ->> -endobj -431 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb13-53) ->> -endobj -432 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb13-54) ->> -endobj -433 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb13-55) ->> -endobj -434 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb13-56) ->> -endobj -435 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb13-57) ->> -endobj -436 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb13-58) ->> -endobj -437 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb13-59) ->> -endobj -438 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb13-60) ->> -endobj -439 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb13-61) ->> -endobj -440 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 169.815391 101.433071 151.065391 ] -/BS << -/W 0 ->> -/Dest (cb14-1) ->> -endobj -441 0 obj -<< -/Filter /FlateDecode -/Length 4517 ->> -stream -xœÍ\K$· ¾÷¯ès+’H½#‡  bd‘ŒÖíÝÁ8€Cþ~>ªJªfw×쎓öbÅóË ÿÛ=ÎÅ_NûÅé_Û:îìM)Ñ»pþ郰˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ ±1BbÏ׬1¸ä ± {ŸV†ÄÉÑ5«@ Ÿk,ˆáJH -ËYÈá¼µ*‚Xfò -Ï¡ô¨ñ¼31_T^Ï%¯ñˆLLì‚ÓxÅÄCÖx YB±NãȳÕx…)'! E²*²øœHã%Èâ¡¢ð2due•Yl´Nã2¡df•WÀ³9åkž·Á„Ld5žs&À6IåÁ£0I1)<Y¢ EåA–@Ök<‚,Œ8(å¬x«ï&KNãÈâ}b•Y0ùøYlŠÀƒŒà’†>%Ãų>“áÌYCŸ‹á”Hƒ_‚áX’†dxÞi@@²Žƒ, ÏUy… -kX@² €†DÅ3iX@@[†µhX@€[¶ÅiX@À[¶.jX@\ˆi5, .åÈ ¦T4, `.;5, €.Ü=iX@@] -ÑkX@€]â5, à.±³0p—5UdkXt4ä2iXÀÀ]r6iXÀÀ]¸‘×°³j`‡QÃØŠñ9[ ¸ë³e ¸ëB€ÊK>H0p× A‹ÜÀ]¬×°€»ž}Ô°†b€?VÍ€»€ §«aâ=xž4,@cÆ#-Ñ° w]AøS|he\.QÂÜuÓ§òŠq WüZ6.Æ¢aAî"P‘†¸ øÌã "ª×° w!Rk<à®óÅjX€»È‚†0ãhX€øfœ…骼blÉYÂÜ,y …Í@•— ÉiX06y¨†¸‹ð^4,ˆÀ]ˆ4,ˆÀ]ËH{Td¡ì5,@^ÜVñi$«Æz …ÆîZ‡ÖxÀ]kSÑ° w%ljX“|Ö°dÁ^˜ٔ”¢†±Xðò©¼hð4,H„ÈRy…kX€ÄÚ*YÂä! y¯aAò)ƒ†ÈLqȾÕåd¹¨ë†,È5,’™\˜5,H!b=­j¼èMN€k•—ÁsIÂ”Ø ; °–2N¯aÞHº¤aÄϱ†©@¸Š†E&{dæÏA$Ù ÓÏ9 ²‡,È5,È>c•­†ˆÐXKBÕ­¬3³†™£‘TQø Vš@J•Y`~d±4,Àâ‹M,sTd¡Àäìe¹™5,Ȳxë5,€mÊz3iXP,dAƧaA±ÑHš¬a`ü ëûW?¿ûêô›wŠdàqqà°€õˆœ€Tëoþôç?üõOc¬X‚£]éSú‘>ñãÀ´ÊDEäÙ—2£ö?uäæär»¥÷]—wcDÈ&›Ú‘aÕ¬æ®Ý«ò9v÷s˜›ljß±·»ó­Ît1ÞN}öÇiÞ‘ëÉƶóYw$Ö˜¡é÷Ú·<ËœûfÃnkßM¶°ÚÍbÇøûÛfçËó}-ë?Lãk¶F?Íëïɧh­ë/«}XË´g±ÑËb£].šÚ· ¿cý]i^km—ÛXW›?¢z?«þ›¹·¼ÕÎìùN²eZg&Dø£žd®WGÅCcÕc/WÞ9Ë&yëÆ;;ÄÕ»?.í5¯Û‰$‹Ù¿n§kœ×¶â]9J¼%‡c©3ÚÆWgjA'E6yA-ûX”¶«…áÜlÃZš5ÖñK? áãúl'}»Ž2?µî:î9›¼jŠÅ¶¾¥OmÜ£Í{º%·hm¬ÞPÛ‰+=N'¦|y,{×Ã\·µëÖ1¬t·¶@]Ú -Jâïo×vìj;M‡î¥ßÊ÷C¯µÉ0ûÍòSŸý„"QžïêŠöºâ«lÐiLK?‹/r˳ýn‹L2öŽBö®²gƒ´ÏÛ&ÇâóÃnP\m2¬tºé¯Þʦ>ÖÄAißÒ•ë«GŠ՞)Ù¬¹#ÍÃZ/@Ôt=,¿¾Î'e®¯Èa‹ÐýÑ;êòn´q›ÙUWŽë”úk—­Sïoš•?¹o5›õ7£šÉK¨H(áG¤ ‰²”¬ó¿Qÿ|ˆÕwÔoeûŽWË»ü£å)2¡Åð¿›÷çPºoóð‘oMƒÏ&—L }¤ý»E?CéÛ^‡.å sÃtp‘Í—û’«ó Ê71!x̆2&ÐÝ6–”EXŸàáô²ÔÒ©)ÙÓÚÙ`3.žêi ¹^%Ï»ÝÔÆ‚;S;eÈÕ›z´¥õE„Ý÷*ñÝÊ<˹$¾wdÝébèð†3¥bØ“¼pID‹åâ#Ó=TåÿhC\™¼÷Wxû|?-?K¾5)\Uc°lbŽXŽŸ¦ØìeË÷®ÖyE'ïØ€Úþ®ç7ëÞ(mo¥;Z÷€Ýr»{׎ß=¤\{ä¾ßÙcº—~ÜÖ¹åů›BÙÜfýû½.S/»—{7þýøöcÅ¿ÜË¡{¸w@t±YÞôÊþ~–Mùû–y¨Êó&K,ÍrÈNõð÷;cj“œ¶Ê¿‚ÕtÓÛò%ç’¼(IdrL.?JÖy¥F9ùæäTê]oßÇÞ÷-[‹m³es9è•úÄdk81§xβv$yg|ZŽÔxžI),B–œoLŠÑÊ9èä’;äUyÅÈy¡bÌt×Zg ïYæ.Sj}kk\Þú m—Wù‡³ýƒužGrTÑJ·ìlQgÿIÓî´µc¯§ õîCo]AÒy§D»òwá°¶ù3†ÒO–é@ -±Bs»·úìeö wÃ4C4>z9AÄL†äÀ£¸t°Îó˜f`#SN|Ì4ÿÏ?}êÄ|¾[é2(!p½nÑ‹v(Üí»h¹ÚÕËòËößrÏ£¿ÂQ|hójárcêÉt9õˆC5žgÚЬ\žz•G„dÏLçMä£lðX•çQŒÜJB 7‘´µK³µ}ø‰™aÊõÌž•sðÈ?#÷H³Çê|*rÿϹ[ý¹øTÃSÛ•TÞh>ÜhÐÞ@6ߟð¨ãÅüöòÆÛÉnúû7§û·óÛÒ†KsFÝƺǿ Óøý÷ô짹èì¦âârÎä¾yªò4%RdKÔë[âê«×¸äê2ËÕ‘mãó) Ûµc6Ö%dºJm™YãEA·ì`Ôñ!6Ý/úfØ”åjÒ`9£LÅôhOõ`'Ä&¹“W„V͈òrÌçå ¯ø ÇkiÝ’#·µ\ZWÞu¶Oö×ë•— DÏ}Ƚ2Ý é -}Xäþ®ïÛ´Ù=».·çàѤ|ëë%•$×Ãn‚¯§‹Û)帪¢-âÊtRz…ißÎ -|Mç¹þ0¢~?··ž/¨¯¾ÓÏÊÚq7Ú~¶¾BXÏx¹ õåÚzqÓ~;ƒ:ɳ9—Y– ™e WÓøã)ÖkŽ®^u·ðz¤uE^β Hå,W+³%¤ãë´¹ªhw¾Q³~AèÇ^jý¨•ï ÈÓå‹CÓÃZþÇ:á?œœ¼‡²‹a/±•,‹Ù=ùå$€a…Ìf–’åãfØÝf"zž`]eC³†ÇåöVÚѤºÀÞJºŽ ®sgb:œÈ¾›®ÜHš wˆ—{"ó¹s3öòHmpè†ä&}“ -ópð˜FH>OMâIŠŽ.û©"æFñ8uX†d1Å çNºêE~ž§‚¤¯éaJ#ä‘G˜›n´:Š©R`šæ†ÙÍ–<ŠÐŸì^Ÿ=ï«È±y€P;´wêËLíj> -endobj -443 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 541.623057 101.433071 522.873057 ] -/BS << -/W 0 ->> -/Dest (cb17-1) ->> -endobj -444 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 522.676036 99.933071 503.926036 ] -/BS << -/W 0 ->> -/Dest (cb17-2) ->> -endobj -445 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 503.729014 99.933071 484.979014 ] -/BS << -/W 0 ->> -/Dest (cb17-3) ->> -endobj -446 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 465.834971 99.933071 447.084971 ] -/BS << -/W 0 ->> -/Dest (cb17-4) ->> -endobj -447 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 427.940928 99.933071 409.190928 ] -/BS << -/W 0 ->> -/Dest (cb17-5) ->> -endobj -448 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 408.993907 99.933071 390.243907 ] -/BS << -/W 0 ->> -/Dest (cb17-6) ->> -endobj -449 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 390.046885 99.933071 371.296885 ] -/BS << -/W 0 ->> -/Dest (cb17-7) ->> -endobj -450 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 371.099864 99.933071 352.349864 ] -/BS << -/W 0 ->> -/Dest (cb17-8) ->> -endobj -451 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 352.152842 99.933071 333.402842 ] -/BS << -/W 0 ->> -/Dest (cb17-9) ->> -endobj -452 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 333.205821 99.933071 314.455821 ] -/BS << -/W 0 ->> -/Dest (cb17-10) ->> -endobj -453 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 314.258799 99.933071 295.508799 ] -/BS << -/W 0 ->> -/Dest (cb17-11) ->> -endobj -454 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 295.311778 99.933071 276.561778 ] -/BS << -/W 0 ->> -/Dest (cb17-12) ->> -endobj -455 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 276.364756 99.933071 257.614756 ] -/BS << -/W 0 ->> -/Dest (cb17-13) ->> -endobj -456 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 257.417735 99.933071 238.667735 ] -/BS << -/W 0 ->> -/Dest (cb17-14) ->> -endobj -457 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 238.470714 99.933071 219.720714 ] -/BS << -/W 0 ->> -/Dest (cb17-15) ->> -endobj -458 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 219.523692 99.933071 200.773692 ] -/BS << -/W 0 ->> -/Dest (cb17-16) ->> -endobj -459 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 200.576671 99.933071 181.826671 ] -/BS << -/W 0 ->> -/Dest (cb17-17) ->> -endobj -460 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 181.629649 99.933071 162.879649 ] -/BS << -/W 0 ->> -/Dest (cb17-18) ->> -endobj -461 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 162.682628 99.933071 143.932628 ] -/BS << -/W 0 ->> -/Dest (cb17-19) ->> -endobj -462 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 143.735606 99.933071 124.985606 ] -/BS << -/W 0 ->> -/Dest (cb17-20) ->> -endobj -463 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 124.788585 99.933071 106.038585 ] -/BS << -/W 0 ->> -/Dest (cb17-21) ->> -endobj -464 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 105.841563 99.933071 87.091563 ] -/BS << -/W 0 ->> -/Dest (cb17-22) ->> -endobj -465 0 obj -<< -/Filter /FlateDecode -/Length 4916 ->> -stream -xœÍ]ϯä¸q¾÷_ñΖ&Y,þ{à Y#† ìÃ"‡Ù—ÁÛëüïç+µDR­z-J¯'èí™V‹,‘U«(Ší^,þ|çðWÎä\R /¯¿\~½X“Øâ¿©Äêbbÿ ŸÄ/ΆdØ•˜ùå!P2©°Oñåí‚¿n.ûâo—¿þæò?k÷âM)Ñ;~ùÇa2)§dYa°¬#…ÅlRòVt·XŠÆ‚œ‚[V‚¡dÖXP#xŸV†’£-«@ Ÿ\ÐXPÃN -ËYèá¼µ*ŠØÈ+<Ç&¢Ó£ÆóÎÄ\|Qy <—¼Æ#21ÇNãcä¬ñtábÆcèÂ.«ñ K œ4^„.ɪ<èâs"— ‹Gï…—¡ ¬+«<èb£u¯á’CPy<›SÞò¼eÙÈj<ç Ã6IåÁ£0H1)<]¢å¢ò  “õ KÀœ`J9+ÞêŻɒÓx ]¼OAåA ¾†>B›¢ð #¸¤a€OÉ„âƒ>“ 9d |.&¤D øÂ&Ä’4 ëÀóN²Ð…CÔ€t ð\•]¨ ÈCа€ºø@Ð6ÀZ4, Àm°ÅiX@ÀÛ`]Ô°€¸PÓjX@@\Ê1hX@€\˜RÑ°€€¹ìÔ°€ºp÷¤au‰£×°€»rÔ°€€»œÕ° w‰ ©ò  Xà£!—IÂÜ%g“†¸ 7ò`T ì0jX[1>g«aAîúlƒ†¸ë¦•— |4,À]ϸmæîz¶^ÂÜõÁG `(øcÕh¸ a=€.X 0߃çIÃTf< ¸ë -¦?ŧVÆå5,`à®Ë>•WŒKèqÅoÑËÆÅX4,`à.&*Ò°€»€Ï¬a0Þ`Fõ0p×fjÜu¾X ¸‹¸5,€9‡@ÃÌoÆY˜®Ê+Æ–œ5,`à.`ÉkX€YØØ $QyÉÀœ†cc‡jX»˜Þ‹†¸k™HÂܵaʃ.”½†ˆkÀƒÛ*>`ÕX¤ÐxÀ]ë0¸km*Dà®L›Ä$Ÿ5,€‚Q°×° æ`JJQÂX,xùT^4¸f ’Y*º„4,@`m -•¬aAòÐ…¼×° yè‚AÃD¦8Dßjz]`.j~  "E €d&—4,H‘¡W5^ô&'ÀµÊË๤aAJÁ`*v —2N¯aÞH¸¤aÔÏ R.p 0™ì™k<]dkX€H<ç4,Ⱥ Ô° ûŒ,3Z 0C#—DWk¼`%ÏÌä„ŠÀMi)UtLhX€ñ‘d›5,@òƒdiŽÊƒ.ÄAÂœ½¤›YÂœ¡‹·^Ãئä›IÂb¡ "> ŠFÂd ãàY«aAqù¦À/%ß̲È7‘â)~[(J¾™4,(º0B•]4ÅoIÈ7N/B_‚†%FÉ7³†3o"¢W—²ä›I].îr2i<à."Ó¨a.o"²Õr{ àå„øLgfÉ8³ºb`½Œ¹U]2°À^ÌVI]3°_É‘ÔE¨‰Ì2GuÕÓ*˜ÎªËø‹x,l0áºúÔ/QE˜CÂœ™“Üd[D÷5çhÅèb×õ±üíòÛϘ|_þö¿—¯¿™VÊ–?ÆO!JbÍâ"¾`¸Wfª.` ™ˆaÒ -ž‘%RiJ:Àâ¼è&Jb*Æt[^ê·V‹ÎÁÌtg¡o™‚ÃPQÔu=,ó(]iiî‡ôÁ*œ€(ßi8Vòáz!- ˆHl@V3D³‚Ôwy„¦è ‰Q"JQñ‚épTDãhº/ü/Ÿ.pï¹ùëÊq½ìôF¶…õ˧_.¿ý÷ü×û3šýôõòÓ÷ֆ׾sòo‘ÿ‘ÏÿùòéÇ;µË솸¸uíR¡¥=aéHk­cd¬|ö*‘!S¨±©Ä}¸àœ¬Íñ­/HœfóÿŸ|ýÎîZáï>­¨‰Ò–R¦eú{#·ƒ2§Pã[€*ä”觢… Ú¾ƒÃl"Ù,±Óªº oæN±u<‹ƒƒÆ¶Î}j(Ò|E¿f¾Îf°h8k¼XûÆ,ÒÊtZd5É ¼s÷¬bHäyŒêy«O!‡bU]oüs›&ÏÔ;<"¬É‘ŠLD²š=eu÷;|Häy:<ÉS.Öësv™»Iïž,‘{AŽ¼sÝ/úîXµ"Ï˲ØÝUè~чG^VS˜Z|,˜iƒ {ˆ>(ó|q —Ç -²—¿I ¸®þh$¸•v¤Ý‰Á#7NÅ–XÎh ö2°H•þfñd‹ê4Gã¢)½7uáÙ¶†#‘‘HŠ ‘nHï¼<ïûјÌÓ`²‡©3ǘé!“຺Ÿfòs "f$ŸÅä3Û÷½·ÆɺVZØ×½ŸV«1Þšô7s‡–£l[•ý5ö¿Þ¾|N²à&‹©bE‰Ân"?&ò4†'[²+”ÞÁí|'lU»L–/rôòlA¶«Ñô8änŸ ÊM)F·¦u(ÏòãRñ¤Ô2–gµGÛŽäY²£m%^% Ë´TöÖ+ež\afûÈm]Ýq<_;êf@"ÚÄtLa8®yžáÍœ}x“ïÆ5!Ë[.øýÇ¿÷‹>*®aÙŠ?ôWJ‘Ý^¼ûgPæùâšiokFÀ¨/Û£©ÉÞý×ÙbËŽøÈD›ýÄþ<þ(u­ÞO]êZ!1Ü> -ÉJ¶õÊ-LˉsýÓ-å³#·µÒ×$êž=*zåvû5‘¼«åb"­¦%ÛÈJeÓ}ÕñŒLÐGät»{¿èÃ6"ËäyhÜÜ {ežÐaåí5[ä% Íaùs)¼®W}ÚjÏH$¼nèh$¼•f^?*­RÖï[ŒªiuuÜ¥þ#1㶶:WÝöÛ¼ÂîZfàqt·‚¿mg,&ìk«é €M“£y Ô{¤ÈêRnÚÂyß/Ædž&ôˆVö°Oï;>"\W7™-­£¼ÝÙ«s‚mm'w6t°­óH’¦HϳÂ!Ãÿþج:†Ø‡¸A“¯ãŽs¦íöÐLÁ%þÈãqG’ýÌYL]Ófø1¦,%ÊžX«çólíŸ÷b±DªµWmÊÀƒ¼KÆÅ%ÕÒß‹¢ÃlëPÜî}Tô€¨Ýxøì‘‘¡âîÞœA™ç‹äõ£€ry@¹®îüú[[ÛÖ¸oíé¾"}`õL‘îCæÁTç0×·Æã}ÕãVzd½Ž¸˜é„|ªý¶›@»÷‘ûŸ^vŒ¨Dkp½®K¹¶µŒmí`tSÑõºe2íˆÞ‘8Ýu×Õ]ü“yž(-KT iýñôvFá'fÌ -‰ÉßT{~zØÔ8]µØl+}àù¬"æøsx ýŒ-:s÷rŽG!³&=(>ÖÈy{k&cíçémŽtêÞÛ²ý;½ÿhðï績Al͇Âz_‚ŸØ ÀVõ 1ðë"¿M ‡À¯`>ŽÉ‹iá…ˆ¯÷Çežü’•õ%þ¿QlIæºúÇ$™Û:$™Šô·Ï#³0!6Æ1GSˆxwŠy#‘×ð2šÖ÷|ØHVÕßNŸ‚7ä3MKòcï/Œ‰<Ï°„T)—{›ÁF;¾…(ëj÷Ÿ†¥(ûå™ñŽyžŽÎ²UÑÊ)]HA×ÕÛì8ˆÀ”䱉œ ‘ótÆÁý^“yšnÏÓÁ*$§Þ=Ò¾×Õ>"ßÖx$W¤„àŠôáüŒ-?s÷-W¤†àgÚo!¸vï)e11]?GÖ74]ð)óçÀZÉ1émO[E÷hø„Ýrï‰Qé“VÑ­pœi¿=ŠÞ± -é‹,™%Ðÿ U°cä+h\×==XÖî{ÝÏǬBŽC¯Ý_w5‡žåiõ‰Q‘ãüËI÷ô«`ñ ËÁŸ² -FHŠ¤;emæ¨VAƒV倠Ȝ޳Š¡t8}3àKõô>¾ àb~­dLäy‚’Í‘’Ýë› dú˜Ü ¥â"gúƒ2ÏÓ],‘xŽIß÷Ñ$n]ýc2ýmG2}Eúƒ™~Žr†““³»F3ý1‘ç1’,oI9J÷ö®~ÀHVÕÏô3ú¨Ø ã& c"O3ÅÊKCÁÆ{'Œv|Ë„ÖÕîgúÅôU’Ó%Gg1‘çéh¢iïr|Ì9!ëê:8 _Jàñ÷ež§‹Y°ê¾spS¿GìûyÏÀ.ÀΪW Œíì~ÚH»¯Ëgü©þ ºõEúÀ;1ŠôüXè‘[£÷÷ tç½{?sô‹A*Û±û»E?`îëVŠ¬s0fè}…î=¥ÐÍÆš?ÍÇü%iå'äð»ÒXõÚ-²£Ï4šê1#/îF7ñû8>ÆšŒü¾P~Ú3’äHLñóßýá¿ÿ˺×agþËÕæÄà|l6'×,Yzйk†5•‘H-ϘçØÑJä&{iñï[·ƒMëxñZ>|·‡‰]KÙxýLuÍߧzæº9Íí}–MsØ‚KWþª~žÛîô™êàîx¹‡jñË0þz‰ÓÑ®n:ÞÛÆi2–;#;ÜåŒ,ÇÂ}67u´{yGrúÕ”_k)ù$+g«ËÃÇiXû‹¹ü¯Ó€ÿr‘]MÓ>z_ôÀNdh|K~»È^…Œ É’…8µµ.Ù©vC{…£%SnK;-µrâå«çånDž19ŒTvù/Wâ׆"$£Û’åî¸3Y~8$š¯bU`©ÄVaè˶ -›à.Y^ykDÞm×\#Ë!aKY_¿–:æ VÕû®sÝð/e}ê†óñõ‚fé–ú¦Råç -–ª–ÞÅ̶ШSªT*/%뀄:N -éõ¢¿³KÕˆ£\åw£&‡>_‰y1¤PË•¥œBz½¨åæÚ¸ŽTª¥r-KÏ7Uš©S®¯Ð¦>š©I~ ©óKË×59Ê,cì Ùß_/ª'6òÛ…“RšVX^L¯'RWo%gÛì¥QS^º -®¹ÐäÈ\ùû -\©‚<ÓªIɱ×íª5"¾YÉëNêé±¢NÐzÀ…ÔÊvÝ…ø¬Úlß_.[¥¼€n-]{lEíº¬£§²¨' UÁcMvÕê*Y l韊•=Ñ.J ¹FæjÊo+rÓ“««²­E×™;×z9Vì|ëÉ©Ž¸“3Ì«¾µÞÎæâë4±…[²˜~^id ãó©uĦ@¡‘Œ¾MäÅ[+:Éï2-_›“i3îëåï—išóÄòâ(ÒI'?ÉÑG]îåŸ(ó#>ÿ} /ÿ±œ}ǽ­há+¶®ÈýL&?à”ÉòƒN ±ºMGô¡ƒýFv¹•åÅê<î}‡×¡‰ÛìKIWÔUnOäÖ‡ÙWÓõŽºžá;Ä×›^¬ÌGǦMØ×KZnÍœ¾h…‹þvp™Ú”üÒU‰+)Ú¦tpƒïqÑWŠË®ÁÒ4[©)Ù7R»^ôýPÈóõþ¢ #ä’/´i¶¯z¡MwÑALýE_qp}£e¥¢ôi÷úð¸Ï*ÇÅ„Z¡½Rßzj½1¡VÀÓ©4]Æõesh:ãqqÞÖCµ.ê|jÓ\£…auÑÛhGÍMl.À}£a¡­]&Š - Ù…VkL‹Nq[ºeºã¬öRª1E£þ½ïiîG¥v"÷£7e¥ÞeÆ«ÑÕˆ{sNíîl+YcɬÐb/]©¡•¤…¶m;¬Û^fÛ—î6BS$¶ÈÔ61Þ”<…g|ç*s³ -¡•Z­Œ¸Ü~5up‰> -endobj -467 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb17-23) ->> -endobj -468 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb17-24) ->> -endobj -469 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb17-25) ->> -endobj -470 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb17-26) ->> -endobj -471 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb17-27) ->> -endobj -472 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb17-28) ->> -endobj -473 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb17-29) ->> -endobj -474 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb17-30) ->> -endobj -475 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb17-31) ->> -endobj -476 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb17-32) ->> -endobj -477 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb17-33) ->> -endobj -478 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb17-34) ->> -endobj -479 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb17-35) ->> -endobj -480 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb17-36) ->> -endobj -481 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb17-37) ->> -endobj -482 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb17-38) ->> -endobj -483 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb17-39) ->> -endobj -484 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb17-40) ->> -endobj -485 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb17-41) ->> -endobj -486 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb17-42) ->> -endobj -487 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb17-43) ->> -endobj -488 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb17-44) ->> -endobj -489 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb17-45) ->> -endobj -490 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb17-46) ->> -endobj -491 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb17-47) ->> -endobj -492 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb17-48) ->> -endobj -493 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb17-49) ->> -endobj -494 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb17-50) ->> -endobj -495 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb17-51) ->> -endobj -496 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb17-52) ->> -endobj -497 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb17-53) ->> -endobj -498 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb17-54) ->> -endobj -499 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb17-55) ->> -endobj -500 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb17-56) ->> -endobj -501 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb17-57) ->> -endobj -502 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb17-58) ->> -endobj -503 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb17-59) ->> -endobj -504 0 obj -<< -/Filter /FlateDecode -/Length 3174 ->> -stream -xœÍ[KÇ ¾Ï¯˜³—É"ë:p(ˆ#Brr×’`@Î!?»»3Ã}I2­vwš¬›du/Ÿ __3~TåPk+YÏwNOJ"üÛF\\lìïñ]Ò™IKHÜrMçFJK±äóý ?®.×á÷§¿}uú×å>ÇÐZŽœÎ¿¼3–J(µJ«E,+¥PJLÙae¸ÅÖ<ÄHE£Þ² -ÄÐV“Ç‚cuXbˆ–[Vƒ±°z,ˆÁ-‡Å98¹<Bª§¡ôìñ"‡\[l.¯€Ç%z<‘‹rb×BÎ9U§%5b— KbUòxE¥—!‹d!—Yb-âñ -d‰ÐNsx²À»ªËƒ,”‰=^“ZUuy <ª¥Þò"¥ªy<æà›âòQ0R./B–L©¹<È’„¢ÇÈ¢øòxÀ‰$¥V'Z£E·°ÇK%Æ¢.²ÀøÄ Y¨dAÁpÉÀXJÐÕX%hÕê¡@¬-h)âÁ@l)hnÅÃ!/²B%iö@²("×åAiêaDÈ‚ˆ@–¨âamÞâan•{X À[%Îb’‡Ä•šÕÃä•š‡Ì`§‡ÐE¸ ¨+)G °+Z³‡Üeò°@»"˜éò Øàc®âaw…©xX À]„Qô°V ðÃìa|%ÄZÉÃîÆJêawcA -py% ÅÃîÆ„[ð27p7&Š(p7jÌÀQð‡Üj¸ I~9Y° ߃ÅÃ,"Ê p—ÒŸÓ@«Àµe p—+ÌçòZà;q -ιyX€»HTâaAî>«‡Àø€Œ=,HÀ]djÜåØÈÂÜEÝ<,€;Fàaò[`‚뺼¨ÕêaAî–¢‡ÈÂ*Äå•Gb ÆP‡zX»HïÍ ܥ$âaAî’¢ìqyEjô°u x['¦Q¬Š@ -Ü%†…=p—¨4 2p×Ò¦‡¹!V `@=,ÈUC+%{X‡’Ïåå€ëäaA! ËåA-êa -ëФU J„,£‡%B”  "Q}»íd»¸ýBTŠÉBmª””ÑA«/ÇP àÚåUð¸xXPŠ¤bö°½T¨z ðÁÊ% x¬”Y* …Q™{<†,(²=,@¥ ³‡5BTƒÔXÑefò°½$Tíñ”¬Ï¬TÍÁJE &è4”.²&<,€}¬ÙN ùA³‰6ÇåAIêaA­ÑÚÍêaA­%Rô°¾iýfñ° dAÅçaA£¬Lö°0‘‡+úͨ .­ß¬4!ô›hñœ¸m’­ß,4…, %ŠËƒ,(šš·À$ô›@'—!KlêaAËÙúÍêaÊ ô›¨èÝ#jýfq €»©™<p•iö°è7QÙz½=xSA}æ3«uœÕ=1 @oBnu Ø‹lUÜ3øZä@Lt–5»§H«`2¹ÇüE=¦7˜°Ÿ>­GTîP3k±›l Eu?zŽ9L¶ŸýòÓé›·H¾çŸþ}zÿÕvRÖ¿Î/š­±N"±ÁÜ€«°-§(!kB›² ù>aŠ‰PÚ‘ŠÇ‘›‰ˆDŒdÛÎã_ - yMbF)³Ý—®Û¢O`J²+éK§|!I§‚5ÂKHW ¡àFž±¤î)õ™s¾¸V×}§Ž&âÉ™ômÆ(ÄJÞÃpÕP [Ùñ¨Õž9çm·4äEC‡¹ %“g8æÝXö8i ÛÐ|wüŽÇï6 ­ï/C$å˱I®ø=´è ß]Îó;½¯Ã“×åcx"îÅŠg4[æi*ÙQc×À¶C¿Ãr)õ¦¡º¸ö¢÷î -0ÒwÍÎ:WšÛd¹ÖÆûÉ£‹ô¹w‹V®5N/ -Åg%tßhÝP¶ö|éÁhÇç"۱ʣaûÌ9Ÿ“ðè(õ‰^zûŒ„õv·Èu”m´îºÌÓK³qc8iÏÛûÕó½NHóºáQåùÔciñ:C%=¼­nrÜzœ¢ûKµðÕ†yœØ©Ñî=h2Ÿ•ÝŸ7åÿÍßdKiúÛ£öîšûþèuÑ-r$ÊUkÛÒ:}=9ä ýÜ6Ú½Ö>óUu{évXÒ^rI"ÔЊ =ätñÝy[µ8‹ZÉG`Ø–ðâ»cº£ò[UK»cšÓê»#áÐá´ùpâ÷{Z5úVíäý{Àh>Ö9ÖÐÛÃÙïŽê¨^­ŸŽ½yô q©ßëk3~<åí|“·g\„Ú›£½sÎÄìD­v6*:ZzÞÍçfn¯}£Ž·‰È0ÚáòþªÑrqŒÿ¸üɛӿ¹îþäŸ6rýb’ïOöæ‡:dÕŸI6âñ‘ÆZÊ}à ÝJh×#^t|lcq‹¹ýcLýnl~êÓìD%qìWûÆ33ß’íî)ðA¶·gjã*QRî2[#2Ôuì\pN|’ÌB 1ÝiÙn’£vyØž¶ ÕåI¢¯ªãÅü}l,‹9#Þ°­\Sï]ª}©®ÝHƒ&‹PmPS9 ¢ÃNéî俦¾´Â˜ƒ¿XÍž|ìÄÚIǸÖÇ9¤»“;îX- K•1ªÎ©­k~Š2]]êP¼CÛttP‹½¸ÄòsìvXÉÙ,ÐmMuñ›xC¼;%‘8É÷§TœÑib¥îz+Q–u¹Òô—IEÃq¨ -¡Ùivnl?w¨Hƒjȳ}`.eÏ~æÕÜÄbs/•´Òó@õ4ÀZæØE]\ÊðÙU_Ö5ÝŽ7У‡Æ.¨‹Êzi]<©—d^7Èa]?+W"u!°Ý$§áÊ÷ä)g¡šhl–yq÷4Öä<°ó~%—aq¶yCÞ±îâóx·%6½&›ë×¾É$GÙnn¶§`…Iž0z¿‘{´t²—ûÇd^ƽ;ý|ÚÒ\”í(# Û{)kÕÅçÿ`Ìk|ÿó¤ç?õ!{4Ð"l½ ¯™ÌÞb¬Ï$Û[8Âf!F]`’¹Î±©{ÃÈãÖ –+šM§ ì}$åÁ]‰iêp!ÇẑeÑLz„xw¥ÅQÈ|®mfÂÞ/¥ß¶{„Ö¥ÂÅz;¸¬“'3%Ÿ—%qeCgJg{^°LÄź(.— Û”ìBLsÈu“¡z“_WSˆíµ\,e„]ÊäéL³ëÒ¶ÝÅ2%°¬ëÂÊë¦íBGèO¯Ï¶û!rî`Ôíƒz¿RÇužO•í2_^Žœ#Û™PÞ©¡±–,±'£È™¡1K ½¸X}t¡Ö9íÖMµÓ.6í‰bÀhÔicÅÒeÊÇ´®–펫«¥2jŠIýyÕtZ­2”˜V«ä›±¶nÏx£Z’Q¥Õ˼;š#G-YZ^gªÎ‘Òi·{ëåÞ=Ûž—ÛÐ)Hž•)Íiéfä'áߧÄÎ>çêÂÕ¿¶“®óÕ$jå"Q¶¿¡š30Dí…¾¢u9*yñœÇÏNnOHúÙɺÌT„ -Ûßõø"܃ўf,Kõ¿ùñ§5roée -xðŒ[† m/volÛѸ,Ö*Úßqˆ½Õ^?-ÄîQÑv„’/ŽO¶¯ÿ÷ž„ -endstream -endobj -505 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 504 0 R -/Resources 4 0 R -/Annots [ 506 0 R 507 0 R 508 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -506 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb17-60) ->> -endobj -507 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb17-61) ->> -endobj -508 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 719.279014 101.433071 700.529014 ] -/BS << -/W 0 ->> -/Dest (cb18-1) ->> -endobj -509 0 obj -<< -/Filter /FlateDecode -/Length 4966 ->> -stream -xœí]K»qÞŸ_qÖn›ï`ÜEÉ bĈ/‚,¤#`@Î"?_±›d5»ºO¬ÇÒ`4ÝU|‹?Ù/}UøùIã¿äô”RŽÁ]oŸ/_.jŠ^á_I±:)êßã7ú«V.N^çüõ3 lœbö&†ëëÿ §<ùë忺ü×:¾š)ç`´¿þù©œbŠQyA•¡RÚ -*蓼*è MÌYRÁ q[U„.'/©`†3& ª3¬‹ÚnUf˜¨¤‚:û(¨´‚Ú(%ê`ˆrÎA§ýàô 錞BÊ&‹ºŽFÒY;…è´×’.O!Ÿ$ƒ->+-é‹:Øâ­2’·Ižð6¦$ŒVC£Û*«%‡-ÆD'ê` :_b`‹ŠA¢Œ ‰xIâãä²q ˜d'—\’XÀ¤<¹­D&ûÉ…%°JCg´DVÁï‚ÄVÇ‘+ê`‹ÍNâk` H\`-l1ÎJ\`Á¶h‘¸À‚nÊZâ ¾uJ‰ ,f*‰ ,צà$.° \@)K\`Á¹Ü)qéb¸G‰ ,X×ú`$.° ]ëR¸À‚w­ÓJâÞµ9El€%.;NV'+qïZ­¢Ä¼‹ad$.@¯NÀa¸X™LJJâÞ5I9‰ x×DL¢.NƒVâÞ5Mfnð®ñÊH\àÀ»Æ™ q€2” €wA!^` VâÌ÷Ð+q -› ‰ Ërwq¿qßÿ|?ßßØà ô/^· >f0o1sÄÙ?ßÁ÷cð=8øÄAöXȹ3  ¼ïolà=rbQ8YèB¹¥}W?›ÕTîjòd‚¡ åeUù5yŽW¢´µn¼Ë×v ¯KµÕSÙ©p«z½ÆºWD[ÎóÍmu^Ñõ$ÚG]Õ+y/ñ]ëNõýÖHoµP¯Ëqò*Y“DcÏfú†Fo7¨þîÝ=ZîÃßì&çR¹~E·•DÚƒ¿¾û|ùõo÷/ÿôo¿Co½{¹üûo”O*¬t¹ýBøCoó¹ J¹¤”‡Þçzüó\ßýrÑÀŒjw“ýÔOÙÈÐ»ðŒ±‰ª_×øóOú7˱[jO³ÌëåoXþV¹Ÿ- ´hÊ}˜åè‡r¾‘/iQEù-éjÙf¨ -&'4yZÛÐòW9«‡ê(:²ï=K£鞧¤½õr‹Í¡ÛËÛQl9ïë<úÚåÙ.÷²®ƒÊWæÞæíimWݧ”Þ‘M·¥×ÃbBàuñ˜2 еn»ô•TÇbÊÑ·åK—Jèvœm9ÅÕ¿¡´ãÈŽhöìP½-äÿÖ>ê C}$ÚF·eå@BÙ3f{™k v_£Ê‹£ª·•ìq±§-í§zÂb{^þ¹c5Í ¦cŒò–óOKÿxf“fé–>ÔæÐïyôMÃf6W<ÄZæ¶þ6ÎÚ8‘}E¼ák?ª¹Ê®}0öC³)°¶U¿ëî÷š¾Ù‡~ÛéçÚ†‚u²ãý‘ϬÞøŒóHmWZÛªµÄi½ÝÇf¿?7îê6¬ùê8´×¥Á^³”Éñ_ÇÐoß•YÒ¶¸#ºÉæ2KÒ-S&iãA_(Ò"c3ë£y¾*VY&ÒÉÐE}´ºª0kç¤Ü8»jWk]ÓNµõÍÞ”è^ScçÉöùåïÿá_ÙdKðZ¦=w«ÓÏ,÷–M7s'ìV˜Ã¤2Ý3TX)©t÷BëÔÅÊþ|\¢AˆîN¢P¢¹—¸ »ÇBí¹—««L¨’r—É´z¬NTö®‡L¹-Ûù`w<ä>­'y7Ô1Vï±0سäÉi-»ÛzgËVK–Þõ<Ý'×…(·ó^nê7p…5ßÎó.Œ²ä0]îcÓç¥ë´0—ÆŸÈI —ˆu½ŽbxK“³ÿ°ž@WAÝ T¶Ýo›éζ¡Ò2²wHÏ•j !,£û3Êý<‡¤w2Ï[V î£ûD¸O½¯ÓÍÒß/òh¢tsˆ¼ÖYÅ [Çòæiyâ—p÷tý?ŸGÈû} Yq™zYhÕÐǧà$NÁƺ)gOK¸u…GÈó¤ykÐÓ.Ó­‡È;™çÙ·!¯ñ³þÆŽŽ²(õË_ú}I¼Ñѱ]tï‡4—¿‹Ü˜E_?.ó6í÷›“xð8Ô‡rc­{eôP@EŒ:‰²syžeÝlzÆheOtÑÜVæ:µæ©2…ÁË{/` -š,CYí­´îÖ½VÒ|Ж‹kÞ[ÕgöËž—hÇÈlhÓB9ã6ŠéÇÍ/çë*>ÕÌÏqÔï 5…Ç‘z.ϳ!qÌ!R„èºAeÚrÆ!×ÜÖèYq×ÐÛãÛ1žå½­í+vTdÔ‘¥çQ'Ž¤¥}…×;·Ã±Yò䑧{;Z»olõ6ò§éùötãCŠ¡œƒá2Oæy:dšCm^glŽ¬#”„­×WÜ÷þ ùî[ ‹ma·Ñ Öuu4ï °Ç~Λ 1v Ä£ð\ž§Cá¹xQ\mìqÖ¶ãRWH¹ODÖØ“œK¨¯ÖQBÑsþye².Œ‹y}6íÛºâIawË݉!鹤Tž%9FÞ¹Ã#+¹={kž»«:±±ÇŽ§{2ÏÓ!ö8Þ|šð­kÞGVoß!ºydT¬Ò»c®öñ5üÉZ}œF½Ì±«¡ß¥WL8îåý{„¶üwtLãØñš}Ø^©ØÛ‹#~íé=¿Ñ}nsçß+¤Y?Η'ó<òî\§Ö·›±.ôìêêà8»Žw¤°«††¯‡÷vËGÄ>²»îôW»£¶ƒ`_Æ4iëèžý;>—çé|gÿô 1åt=²o¹º¾Å÷âÚ¦†Ž3 -Êﮞ¶ûO‰o‰§oC;ï l%÷îÍlïòìŠèdž§Cß¹xóÞµv'­RÆØIÚE$c±â_Ͻe$ï ´ÇŠ„¦Dˆºw7ÊÉ<φRrÙiþ_¦–#´ ÄhYU»sèÅ5 VO_ÛPµ‹?láǺ÷÷Ë›£‡'<}µ"d“3{dëÓ»Iîtáöµ‘õ›· ®Ÿ)A‘ôÉ›®=•#½åtï‰óiÞ?.Ï„ù¡b»¸±®8ËCõË=¬¦>…Rî0\Ø °…š4ýÔ»€?ò_ÎË~5¥ ª?»÷ÜU{(Ö”àÕüÐ/{˜¶•_neö¬ÄÍ˼TÚ°éÆ/—PÞÖ«ËÛU@Lbè“.×à&Ký‚A˜èM¿ÖéÚmº8Z_wr–á|i©–oã(z]>½*yþp;YÒ)þù¢³›Èÿåiüò UÄÉÔ“.~½ÐwLœ v®?ÒGLÂåPµ²œno¨²HŠ^˜2¤´~RËa=N3RìßóûšÞ/ëTMdËVçŠ!r½SëÕ¤1} &Mv9 Íz ´ {Ž§íöŒwÅÚ*&ô[¡bÕu±qÕMïo® ]ØLç®Ó¬ûkZYw oTkGé«(¥/PÔ¢ªwj2ËŒÊMêkÊÖ!®õ“ ¢7êl„?©Z´Cg6=ë5z÷,LH®¥Ë5 º]ÄtKi¾õTl©RÏš«ç»)ê65Ç ²{ÕF¤[±1¢l=\‹é-­¯Ub¸1áíâ]‰]\Þ¢´Mí;W(_¡Ç…–•ÛÄIu¼t)­Þæ# Í*£· Óÿ3Uø&%æ)Ú5HÑ›ÌûY¯„Æf¯Ä塱Ž“< ]ìi™»tŒ ³Ü_:)!=‘nKÝ<¶’2—1yÌÕ<"©Fk±n¨kb¢°êŸÆ•\¨ª¨®‹}ƒòëJÜíôm¨zÕ* šÁÝ·r»84î|åâØz\Ók雽­\†ù*¼•‰Íb‚~ª•t±Qm¶ë•1a7 ¤ÐÅF_‹¸ŽÖÆNô©­zØ™4ãÞ.º”iÎXoèmÁjúÊ -ºôõæüþçÅ]ÿ¹¾þkžÇjla·®Ä|&£or¥“búFW¶aÄÆ1ÚïbzZ_ÑÐæqc_»ž]ub¯)ucPÝ´\è»™Ø4èm™güð6x±2oí›>aϧ¶6ŽÞ^A/„¯Vá„7§©ëlŸ’¯¬HœQÒ>¥Cë ˈ^(NY…¹[¶2“É+i®'ûï -Ku±FЩí:קY^t••V°$-?á;Í+Í+{£¿zx½¹ß“C$mÔÞ¤¯\ÚFÒFx²Ô–Ó°>msŽ-/~¨ƒ·{¨•eÙس-ÈéC£‡nuÂ1ʤ©g[x^©«²U¥u¢ht™ª²Vb¬6…%[uKiq½[LÑ¥âžö¼Wš=ï•°IKåÖ¯EK¶@žÃ9öÖ©ž²Å’Iž»I]Oi«l[·[×]gÛ+k†ë†„™ªžÍoR~ÿ}ÍØß (¥ª?egD§þ! LÔNGklù¶©J!€C}ž*:öþÃÇóïì‡×„µ*júJ­lÂ+ù¾‚U¿`+gËJlÒc¶†:ÍmÞä¨Ëg -Ë`Ù j§‘–ŠôURKAo¢©EµófGÚB‰«í“òó¿Î$Ô -endstream -endobj -510 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 509 0 R -/Resources 4 0 R -/Annots [ 511 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -511 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 563.554475 101.433071 544.804475 ] -/BS << -/W 0 ->> -/Dest (cb21-1) ->> -endobj -512 0 obj -<< -/Filter /FlateDecode -/Length 4928 ->> -stream -xœí]ËŽ,¹qÝ×WÔZÀ¤ø~†dØcX°à yaxÑ]3-Áè10ò¿ïÌ$ÉŒ¬Êªî{§FðmôíÊ>‚ÁÃà 3‹©Ï -?ßiü—œžRÊ1¸óå§ÓÏ'5E¯ð¯¤X]õñýY+'¯sHþü lœbö&†óû ÿ —<ùûéßsúïu}6SÎÁhþë¤rvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS€Óƒ¤3z -)›,ê"t:Igí¢Ó^Kº<…|’t¶ø¬´¤ó°Åk甤ƒ-Φ(él±Á*Q[LŠVÒEØbà,èlº’¨ƒ-*(-é²|NΉº J1muFùÉ'k•¤ÓzòÀ¦uQè¤-Aù,ê`‹·ÊH: [~$xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØ_ÿ|úí &ßóŸÿçôö›²SV4„’Ú!¾Â00Ýí}k4ߢh†*ÿŠ…0ð‘×óóº Ç/Oû2× »—[–rJ”¥1%ª·OjÇž¿|úý—5dJ|~ÊŽeùʾ¸°Ø¿®‡;˜uÔ]fP "ÿ¼>ÒŽÁŽñº´Ç²rðٚŶËh²ÝþN„Ýo¸Oó< °æ»†(BÅ#=ký¯›z>fÿªl£‡ —1„[¨:–çéPåñÔ/Å;w÷î§ñÍ2¢è¯¢ß¤ðˆzÆeZs\Gʱp—û`ž§CêXõEˆ«-ãÄ;¼¾zàâuÎO÷‰FNkõ åPzsÀ®'sĘçÚjKËç…¶Q›¾î -QF¬{ ž=˜çÙënij7ö@Ÿ/Îûx$ó´«Å›QËšˆqæy:4_q?ºšâw:GÎ:r×v5 »ÞÃå>1õž;ÿ­rºgÿ2?{@Wü×hâ‘Î×É;è{`Ÿõ`ž§Cß8õo‘çîÙ©º‚ÎÍJéÂêܹ—öõövÌcÒƒ+®ƒyžÉÛsýU"ù3ö -§·9@ÏålâðÑ<ÿè³åÇFÃ8ŠúŠOF¯gñéÑ;¶ó<zýgCÇÞV-Ò“(ãJ§î}6TŒÅŸ -áVÑfvʼñ·l¿ó‰Ѧe¸ØHeè~ì9¯”>ð\ÁÁÛ×eîÝSlrÃíÞAë{°ó<Z¯Ç®­çÞÖ(Y=«©Õz_µÎ€;»ì­g†h`D-=W¿zj0­Ó•gWǡ׽­óoб ¦ìýîì-Ô‘¥êwîqæÑVlŽ¬<îŸ5ûw¾…Ø“=˜çٮǯ#;J«Ý¤ÀT=?ðÈC½Õö©ì$Û3>/ËùŠ£pƒ(͸ÙJAËf…´W×2Ì'p«e~m<ÎïË ó‰zÛA²½Ÿ‹æy:$_q­°žÝÜzÊžî±íqa-kàRËî¼nîË9V‡êbTцñî÷ø]‰+£±Ôi·yWL#cú]ÞƒyžÓ¿ùÅž,¨ßõkXçqFeÔ}Ýâ?—IÌ.1åæ‹ùWúyï^òÞwV¤»R4†Ç™ÀQñµÙhŒg$L·4?¬ã¡‚ÕÁ_âŒÂÛ)ã—=«éÀ> ?ßÂï±Áz}÷†¼ÞË,éýìùγ¾Ù¿ØDy[¼g™Þ¬Óõ}ƒÒK¥¹=p·“§UùNôºE­6Õ=F59ªý²Ô–j/?):MËÒiÊKùvé OœfÏPyú‡m9ÁÒQctœí¶œÒêêÍPŠW}7¾ N«.¬ovÚVý42Ǽ׻¤uWqG^­û²¦[ÆZ¹ûRǯfvÔ±{º±ÌÆr‘d?éåþ^I«fÛ•­íºê·¼ñÛÐWÍö°ÆíÌŠµo–ecóñîM·[ÂÅ–õ%Æïù$¦7jÓ^O‡ÄÑY·^hïŠÏ<óaZ·W_õcÆÚx]®Ylåc¡òÙù•³-Z›Ëñχ"¿ƒy:”ìC>ßì¥3F½Ì墣qɧ—ácYp1wÂn…9L*ÓѾC…µ«ù¦8A§ÞªÞ-ÑØ<ù€‹(”èßnåv©œ„‚hÏ­Ü>L٣ϤÜüAÞèÚ›2åλã¡oõ°ÃÍÖ;[N{¥“/ðïä,oú¹¥¢ÿ!Rô®:gºt+¿XÒÿ\:ü',¾0×åVy ˆ*bŒ=?ŠßOô'ˆ±(6Å$\>ªV–Óm XeLÞqÊcJLjùØ׉4kªe¨­¡ü¾f£Ãyª‰l9žv®X"gÐ[1µ^MzÓ‹tÒd—«ÐL¡¨ {Ž§íöŒ7ÅÚ*&ô[¡bÕu±qÕM¯7×….l¦s×iÖý5­‰¬;¯ /'TkGé»(¥×wÔ¢ªwj2ËŒÊMêkÊÖ!®õ“ ºœáwªíЙMÏzAŸ…©ɵt¹¦D—“˜n)Í·žŠ-UêYsõ|7¥C–ðû² ÛK‰ôf06F”­×bÚ"h}­ÃÙ/'ïÚHìâ÷“Bjß¹Bù -=.´¬Ü&Nªã¥Ké<Âù†f•ÑÒôÿL¾I‰yÊí¤èø~Õ+¡±ÙÄk'qyh¬ã$h{Zæ.cÃ,÷—NJHO¤ÛR7­¤ÌeLs5Hª‘ÇZ¬Ꚙ(¬ú§q%ªjªëbß ü¾w;}ª^µÊ‚fp÷­Ü.;ß¹8¶×t¦³·•Ë0_…—2±¹QLÐOµ’.6ªÍv½2&솺¸Óè{×ÑÚØ©á¶|ìƒLšq/§¿œÊ4g¬7´¼ ããúi}þ_¤ù¿ÿurç©g¢Ïó¸Q-LãÖ•˜ÏdôB³tPL/8«Â6l˜Ð8Fû]¬SOë+Ú%ŸY‘¸¢¤}J‡Ö–¼P\² -s·le&’WÒ\Oö;Þ–êb,Œ KÛu®O³¼è*+­`I[~Á všWšWöF?<¼>Üï‹É¡Ž’6joÒw.m #i#[CæÔfg¨¼ã±<Q6ˆÚe¤¥"½ÒÕRЛèMtQiù6z9¬{µ}R~þ,Q—_ -endstream -endobj -513 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 512 0 R -/Resources 4 0 R -/Annots [ 514 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -514 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 123.873057 101.433071 105.123057 ] -/BS << -/W 0 ->> -/Dest (cb24-1) ->> -endobj -515 0 obj -<< -/Filter /FlateDecode -/Length 4716 ->> -stream -xœÝ]K䶾÷¯è³Ë|?€À‡1bd‘‚f{wlãvùûùJ->D•$v÷ìXëÌN«Šùñc±ÄåYàçK‰ÿ‚‘CÑ;s¾ütúù$oþ)f£ú{üz{–ÂøÁÊè‚=ÿDí­òîürÂÍeüåô/Nÿ™ç‘g5Äè”´ç_>’ÊèÁï…eT*!5£²vð^YǨœp‹1r*˜a½Qf©ò0ÃÄ`9Ì0JF`†6^ê¥* å¥áT0CFë•°C*!X ÆhÅè¤Ýq:%¢Š¬ÎC'½âtZÎi%§‹ƒsÎNg`‹Br: [¬4Fp:ØbtðœÎÁí´`u°E¯9‡- -­]€-@W`u°E8!9]ÔƒÁV¡Á‡¥N ;Ø µàtRØÔ¬# -ä<£S°Å Yl±Z(N§a‹Á§OXíC`F«¢Ñ­…–œÎÂ¥¼au°Ï1€r°ExÇQFÐ@¼Äq€ò~0QŽTЃ &p, BŒ÷š£í`\ôh!¡S’#-`‹5Žc-a‹ÁÈeu°EGÃqV° 8.ж(£9.Ð`[´p\ A·FDÉqß!Ç„ 3ÇŒ«ƒ3hP. 9.Ðà\ îä¸@ƒt1Ü=Ǭ«­ShЮ6Áq\ Á»ÚHÁqïjœ¬¶À€-ƒæ¸À€wµžãÞÅ0R WàÐq\¬ *Áqïª Ç¼«<¦VçŒAÍqï*‹[àfnð®²Bq\`À»Ê(Çq€2€ë €wA!–w` –ãÌ÷Ð)Íq -ÜŽ ,xWFL̘[ 2DÇqïÊ€îcuq-ÎŒ[´ò ‹Xð.&*Íqï‚>Çàø3ªâ¸À‚w¥ÆLÍéÀ»REÁqïÂo°ƒ„Àqæ·A -@—ÕÅAÄ8.°à]В⸳𠘄Õù@’€Œ?”ãÞÅô9.pà]aµæ¸Àw…ÛÃê`‹Šãø5ÐaØ2cÎê ˜‚Ów…Ds:ð®>r\àÀ»4mr\à< Tà¸ð‚Ç.˜!zï8.pQ@—Õ¹×–ã/ ° ,V[Œ7À±¢Žã¯`‹VŠã¯` \Ž à QÂûf—°pa׶ÀSä¸L6„h ÇÞ:¬ÇЪœÎ©!xÐ5« ÐIÏq÷fÀT,9.ÀZjô€àr—8.€ÐIÃq°C…ãLECPðÌ9„-p²9.€§ ”[à r\TÀ*Ó Ž 0Cc-‰¦ætFÐ:3p\ŒÈUä¸Ã+M0%«ƒ-  Ž Ð?´Ø¶`ñƒÅ&–9¬¶hk8.AÑr3p\lQBq\lÒzÓs\lÇÇqAn 7™ãÐ8tBp\eÀzSŽ 0.i½8.ˆZ`½‰%3n£v´ÞôD[,\V[à4EfÜ‚“°Þ;q:[T4Dçh½8.€›õ&ÕC2‹ÌVÍËåz’ÙyYÙÆt-§|ÿ:¿ûöôÍ»2訹ÜÔNŽúëy7Æ°°TS˜#@ÔBß|÷×?ÿý»ÒBXl¢t*•î–,¸ÖB2ÐÆh¦»5t§×4°f¬¿ªv±ðZš -©ùáëdï zð›ö,(óF$æÜ„kgžGðú)`JÜ œ¦~ÏØ`0§Ëé/ýúë5«£ÏzÒ=5i>L'¹R“>ý~˜þÆeZóq>.Lƒ×ÙØÑóñµÀrºç¸”ö~,ß”ëSÝ+(Ãz/#Ft¢¬/ÏAPV̆«»²ÄT ]fbG+f|¿dGû¡ôfD“®P–z«eL7ïµ1ÍÇ —mFëSëeºdf´I¦œ†m©¼ô9·Ãs]c›VLo|«_AjT·#µ/ÏѪÄ&r‘²á•¤KÎØäšf¾ŸqWÓÛ¶AÆ1¶Ê{™Û7ÚÑúò:êØ‘4ÝßÈ뮺ÖÍg5å‰+#O–ûÈ÷}©|Œ–?UÉ·†LzÎ6¡ŒBÛÆÀ·ƒÌÎ<‡C¦ÞäÐÖSË­W#k %nÙê3î{: ß½²dIŸGƒ˜×Uм‚Ââû«3!f…}y‡Â>1÷Œ©°ÆYï‹.õÀR[îc‘ÕödÍ¥7Ô—ê]Ñiu]™ÌËoýâº>Ömñdj»Ku;>¤ªýAz ÆÐí6òúòy;>dÛ“~ÞÒ ôU=»˜ÉV>sÞ®‡Y-,x‡›Ù±ûyÛŸ3ª©âC%¯ä“ÚôV>¢ÄGE·ÑÛ—çhèÕ;~ejÁçùßÌ+OMÏV«–-¾«ÑòVkäѦ5oäýä33>mö4Ú˜RË“=+¹5{SžÝUX]|ÇnÄvæ9b·ýÍÃø€®yoY½½ws˨˜¥7Û\«ÍíkøÎ<‡Cî¶ú[‹è¼g¿%ºZwGß -šÝíëþÎ<‡Có¶ß»ðGw<nvΈÙut(õ0Ûø«-bÞœ?“ìýҶ‘·•›Gë·V>)mÜrÜ°³Æ¾êÞìo½Â³*ÓèZþ[Ckë±$.mî3G±îô>¸úóè\Ä‹WzdzûÎ<‡CèNܵó¹Qö š²EÒRW”ÑÓÄ¿¤Kˆ6Sú¶ü„Œ¡¹žÕí®Ùó£›oEãÖN®Ü–­mª|w<-5Åç4Ñð; mzÝFm_ž£¡Öîø©‚GK=j¹F<_}WE¿S¯(·ÝËë{„–ü·µ &slûÌÞ-ŸT¬ÅbYO ~ö4Ü1]³ºlGÕÞR+oçËÎ<‡CÞÎsúf}»ëLÏr»DóìÚîH©žªz=¼-o{KtÝl 鳨­ ¸ò/}¤6ÞíEœ:óÁ;ñÓ|ÊtÝ·œ=ߪc~nSFLJ+ -Æßžžæý§Ä·ÄÓ—æ>wPÖŽ’½½¢¶Š]ö®ˆ:ó}}þæÞ³víRZ߉‹¢ßâU¾âçÇsŒä”_ŽÐQ{»Q:ó ¥nÇ¿ì]uÔO)›ÕF‹ìa´*ò"ïÅœùy~Ž‚µ®zºá²: -¾·[ÃÉÛ÷uæ9\ïvúp¡iÕ•±:®&poŠV ®Ì,ɦƒ"Ýûë5§£Ù¿‚ªxûJ°3ÏÑPåwv-þʼssï¾ßL#jÇöwD½:ó);O_«¸â'óWvh·û¢zâ²Ü7K8¯®~–”W\UL!íº±rŶŽ¸ðMONR¼öŽ½­Þܾ»3Ïáкã½áþ“!áÖ½=‘¹v÷Œñ‰6Ý“XîhŸ™Í"i+H­"[½ß·êÌs8¤îDÃîŽ3Šûw“¼bäíó‹š­ òŽ€yŽ†ÈÐ÷M–‡v#ßòDá-Góíïß -‚ÃѽÎ<‡CðŽ¯úJûé~WÀ+‡jÕ¦.Ÿñ(_Aoõ¤µ÷ûy‡Þßõ3þ–ß«Œš7ݵ¸‚Æ;âŽy‡Æÿte7ôâ[¶pM^54Hø5÷¦.ÞÉö4qY³_êóáØäÞñìÎÿõÖ7JÌPtaÐPÅ=)º>Û?ÏqP³gtŒÈSŒöY,¾ÃusþzWNIªò¶ûùðÖ‹xÇsîÎ<‡CꎯúÄøŽ©«·•½õ{T‡][ßËjólÅPoyOÝÓ§]!® ö¶3ÏáûØîÃãùy{2‡]-Þùî€x‡Û™çhhFkmð£Ë©í—‰ŠÝǶ³ixëU”é¡{ý›äÓk€Ç­‘é•Âé¾z¶D~²¡œá÷ýôºyoÑS–ŽÚ£w±Çêíõs»Iv ¶|×}zû™~Z •e£H:gR‡³’¢§£ÖÞË­>^=¸ñ-Ínò̦ݨé{{RŠü&P•Þå=>M¿Îçvd1¾×Û|,Ý6¾×Û]ÓÓ{½© Æ9™ÒºëïXÖôy,'$f˜ê£ù›òÓ³OÕÏÊ·SÝ•=fzÇx¾›îaÑ?ŸÜxĈ™. RÑ9”ggÚrâ㙎5 B™ºMŽ -Ï+9ÇÓ;Ω¦=ñEç»\Oû¬.¦ô?þÓIF3PûÓ迾'F1&3ÛŠ_Ntø¢aÄÆ ªSLÂé£Èe™fÙåD'Ä6¥¶ƒ˜>Æ\8½”üúQÙt7”ߦlt(†)‘…¸VÌ‘ÓÉ¥˜î^ rÓ–aÐӕ˦X—l†°hê´¥À’qW,µ¨„v)UuE¬L²GÒG¹é\fÓ릓U÷§´ÊWݹ%¼œP­n¥/¬”ŽÍKE¥ÖU"ËteTÌR›Ræ1¹ŸÑåÄ¿©hƒÎÌúª×èð¡«0$ ™œ.¦tŒèrbÓM¥ÙÜS>§ -%kL-_L)P×!7<#Ûh’z:‘·#B§s±£H}-B…µ^NÖä‘XÄ/'ë™Ô¶p…° zµPWåfq/EJ;¯Ÿ04“ŒŽn¡ÿ¯Ta³”˜gü@'+ÈT@}U*¡±™ÅóFªå.³ŽáZ@_ÒVÍ%½Ï˜­ÛKÁ¤'ÒÍ©s‹Í¤U“Ur“yDR™<æb™Q—ÅDa©}2WÖB‘Œ@uEl3”_fâb§ÍC`O•9YÁÝær‹Øeî|©Å>÷¸¤³´²½¹Ü -óIx'6ÓŠ ú!URÄJäÙ®TV ‹a …".4ú2ŠÓhÍìDç§eq3îåôãiœæ”¶*âƒÓ’Ž†¬½.yþÒ|‹ßŸÌù/é,¢ë<®Df •¹u&®g2:H8tŠé`á$Ìæ*SÑ~ËPÒÚ„†<+Uñµ)ÙE!ö”Rf•Y[ miÃJ¬2t•ÔUËØ á¥iÅìÈ<Ú7e¾^êts¨FÓ)VÉ*\Ô·ƒËPtºLÉçªH\QÒ2¥KzCx•u¡¸¬*ŒÅ²™™Ⱥ’Üôd¿©»BS]ÕEåFÐ¥.:S¦Ùºè$ï¢JX×uÁFÖ•Æ™=ŒÑw¯‡û}2Ù¥@ÒLíYúRKó‘4/Õ㥛_æ9GŸé" ÞÒB¹,]=œ24Škaf5F+i(Ù¦¶®Ô$Ù¬Ò4Qdº€L$Y.Ñ'›Ü”-5ËxÇm%Ÿ}Š"ý±ni[÷JnD[÷Š[¤¥rÓŒ—½% [ÃÙ—»%eö%#suî,5%¥N²eÝf^wšmÏÕm˜bˆ+ž©(Ùì"å]üwÏعæi¢\ªô3FFd(§÷a¢6Òk¥Ï‘Üà8ÄЙºÞ„*TrsžíØÉ2B’b'uMèŽÁh-¼ÞVLx"î+ª¢^¦ƒùlQ°·t[0¡¡BsbÏVc”c€(_zZ*Ú8ž€+B  ½àpC(óðÉøóŒîZ -endstream -endobj -516 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 515 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -517 0 obj -<< -/Filter /FlateDecode -/Length 4933 ->> -stream -xœå]K¯äºqÞ÷¯èµ+óý /8pn#FÉÂðâLÏÁ¹®³ÈßÏW%ª¤VŸ9wÐpf0Ó­*>ŠÅ‹¥ÖW…¿?hü—œžRÊ1¸ëí§ËÏ5E¯ðgN±º˜ÕÄ¿è¯Z¹8yCòןH`ã³71\ß.øo¸äÉß.ÿõ«ËßÖyôÕL9£ýõï_IåìSŒÊ ª •ÒVPy?Åh|TAOhbÎ’ -føèŒÛª"Ìp9yI3œ1IP%˜a]Ôv«Ê0ÃDí$ÌÐÙGA¥ìÐF)QC”sÖ:í§§IgôR6YÔEèt4’ÎÚ)D§½–ty -!ø$élñYiIça‹×Î)I[œMQÒØbƒU¢¶˜­¤‹°ÅÀ;YÐ%Øt%Q[TPZÒe;ùœœu:•bÚêŒò“OÖ*I§õäM+ê0¢ÐI! -:[‚òYÔÁo•‘t¶8ü•tà ocJÂh54º­²ZÒyØbLt¢¶ ó%0¶¨$ -Àšˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lq¹¢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.†{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]k‘SÔÁXâ°ãdu²8ð®Õ*J\àÀ»FFâôê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„1h%.pà]ãÑiæ﯌ļkœ (øG‰Ñxâåp¶€`%.À|± °É ,‘¸ÀƒwuÆô'Œi°Õ¤Sxð®Nè>Q—'áqaÜÂË“!K\àÁ»˜¨¬Ä¼ úL€ã'̨FâÞÕ3µ¤ïj“•Ä¼‹¸ÁK\8L€Ä˜ß&­]Q—'•S’¸ÀƒwAKFâÌ“J`Q'IK\2žâP‰ xÓ{–¸ €w•·Vâ‚ÞUa¨ƒ-6‰ ×@‡a+Œi«“2` -IÞU=,éÀ»JÅ,qAïÒ´)qAˆ$0Iâ8! -6„ä¦c¸ dB>Q&\{‰ ¢‚Àƒ²DlqÑI\€ÀzÊ6'‰ ¢-Ö‰ ¢-$.@D0eè[\ÀÀE\8Ø‚HQâ0Ù”²sD°ƒW%]0SŠ kQ— ÓQâ‚Ý„©XK\€µÔ”0è%.ÁO.I\3 ÓNₘa †ŠÄ˜Š¦d™K: [dK\€H:­%.H¶ ”¸ ™„UfP`†ÆZ®–tNÑ:3I\\˜(T”¸Ã+M0¥¨ƒ-  ‰ Ð?´Øö`ñƒÅ&–9¢¶Xï$.HÉÐr3I\l1ÊH\lÒz3J\lAÄ'qAVa¢0YâÐ8tJI\uÂzÓ8‰ 0.i½™$.ÈVa½‰%ž0n³ ´ÞŒd[ÜÖR¢!æº^ÑÈ{‰?¶ó{½"„z-ÖD&` ÈÆžÍôÁFÛÖ£™:aîP -Ê0óŸ¤ Zšú™Éfù@ƒðˆÒØlh-bí£Ü*ñ?}º€L‹Ë>}»dæ#¦Ó˜¿~úéòëùñÇþýŸ^/úRîëoÐøôiùt¾|–k_®½f²8¤ Ë'í<×ïõÓ½vݽºPظ¤±EwÊíÊ[™ÖCU¯–O,¥æºè_kËëÐÕó×r›-™Ù…6|ê½yîeÔúj£ŠF7ßþöÏ×O?^~÷i ÎyUUÀI¸q™fÖC<ŸËò-xþ%`œóÿ;ïAéx7;pá û~ëé›|ôÏ?úìóYdžÂ8„ª]ÊÊÐ¥;Rˆt“IéyqyˆÝ“yž ¼Ú’ð¦·*²â€žÈzæuÝ{îeMZ‚úÜIfE`mf§ÌGÏvóí,#À}a#•¡[wëÚA©u£ô\ž§C©;æØ=Ä©qŸB*7TNTÑ:#i;LÏG|yÊÐØt#JRGË -m¿ÈÔýÑóÓRY ªÜd´KÎÝCê¹Áz9}÷†¼Îl°%/Ÿª”˧_ÒQ¾ëYÖ>Óëšå¡Ûš{à”Ëw9yYÑNíКV›ê£šÕ~+µ…RMàå;Eçi,=OQÊ·¥'<ñuZ¼Båé/Ûr‚¥ÃFt }[ÎÊ‹aöÀ‘˜PwìP½-s/Õö‘çÍ2^ÛèÁ† œ Y({Fd^òϽþZÊzáHëhp±¤->ihÐ%­`“ªm×¥lÉ_ô u/¥L6(_õ{EÛ²+Se‡¾ôc{›CiWµëVúF-ŸsÝÅ/ á®ØÇ|ÒÚ}+eJ}R|`‡|Ĺs¹¬O×#¼ûpN\nnu1äõHô¶ôQ<ôMÜø&œVÿÜXßì´­úideŸ·¤uG±Œ#¯Ö}ÉÙ¤¤mãW3;êXŒ=ÝXfc¸ÆH²Ÿt¹·×­ôÅÒ®C¿å߆¾j¶‡-K.÷K'ylx6Þ½évK¸X³>g|Îö=ŸÄòFmÚëtXç¡Ú{vVЇ~ôX¯Ë5ÅV>*ŸQŸ§Óv>C€œŠúNæy×!¥ ÷|³—N{˜Ë7C©»Çåž.ÃDzàbé„Ý -s˜T¦ÃýC…µ«ù†8A§Þ¦Þ-ÑØ<ù€‹(”è_ïåvi~&Ñž{¹}˜²GŸI¹ç f¼¡dïzÈÌ™:쎇¾×A‡»­wv>ïmµdé]ÏÓSOp]ˆr;ïå¦~XóqžwÉ`”%‡Ðe›>—^8¬c òEN -Ú=à †õóéóz’\˜v/hÜeôô`¦çt†J瑽CzÁéJ`´¤g¬éTü!éÌó$KÝÆ}ôDÎ÷©—íÉ´»7ŽèÜ»u­^ê"g=jû”¾]înNÅ[#ã2ûœÒ³>û8mŒ= ®VÃifÁuéyç:ûõPáF#€Sðæ­9wÐîdžgÃh´á£É‡­èÕ–møòIÿâr-ê<;öñ2¤ùR>‹Ü˜¢¯ÿ¾”ϼMûýf¯;l]ìˆ9yœódž'AY7;è#”=rØèhS–,íØq|£N8rñ\‡Ôö†Ò£Ç]Ó›ü»Hæq¤žËótHM‡|ø­‡rÞ}Øhèíñøî1üXñmmßlÇx“|8ª±Iuã5.ÀíÚßMÉ“wFžîíhí¾±ãà–Ñ.2sãCzÇ€s0 ÜCæ¹<φ̤9tŒÔš÷8²ŽP"ày_žï>Yº§ß°zGðÎs=ösÞLÊ»@ˆ9~0ã\ž§Cá¹x±õ »µ±wT`Ù -\óÑRGsTnFúõ­Žá•5ÖêñøX…ð„Mû¶®xRØswbÈÄãÁ“Næy:ä݉!Çž¢Œèc=»™Év¾KÑ«GX-lxçèñ{ì~=Žçœê¯ð‡Ç&~ùQ»ƒ^#j3e• =~ŒÞsyž½wâÊñÚ°"ò/CÏòå|ÇÑò=eÛF>«þ˜ÅxÔø6ÔÁò>º’Û³·æ¹»ªÛAlÏ#ö\žgCl>Ž7Ÿ&üÖ5ï#«·ïÝ<2*VéÝ1×fóøþdž§CîqŒú¶£óaœý=Ñ]Ñzwôí Ù=¾î?™çéÐ|÷nâÑ;„4;¯Þu°32øîPía-ãï1ß?«ìóֶΑ•Û’ßA#‹IÏ>6|2ÏÓ¡ñ8ŽÝ»_È9×þ¶õt»ƒ}л‡+µw=بIJ ú‡cÊgwÖîÝ»}ïCò;âÕ“yž ™ZÝ X?xÚþX=´ùøÁá³…? ¶,ïF‹=ìé½ðô’³Ì^µÖ°z7ÉhÖ£ ÛWG]õðº®õéI?Š`ÓÕ9Ò{÷Î.˜¯ SÍw²ÃrèÕ–C¹4϶ãõån‰©çæ½ìe~j‡0I÷µCa>ûP…Î saÆv8[õó;å@0?è;×W*χ(ÙAÎV~=ÉìYÍåü܆M7þ| óû0õüNdÒ¤ ýhÂ5¸‰ÎÇ|¥wp&e®Ý¦gGëëNÎù§&~n©Ê¯O(z!5½ŒtîV~QÒÿ -©}ç -å+ô¸Ð²r›8©Ž—.©º -C³Êè=£ôÿB¾I‰yæ/Ú5HÑ»‚ûU¯„Æf¯Ä塱Ž“< ]ìi™»tŒ ³Ü_:)!=‘nKÝ<¶’2—1yÌÕ<"©Fk±n¨kb¢°êŸÆ•\¨ª¨®‹}ƒòÛJÜíôm¨zÕ* šÁÝ·r»84î|ãâØz\Ó‹Ÿ›½­\†ù*¼Í›ÅýT+éb£Úl×+cÂnH¡‹;¾Íâ:Z;Ío=)_û “fÜÛ寗yš3Öz!XM¿cÀ£.}ý_¤ùÿþûâ®ÿV_œ»ÌãF5¶0[Wb>“ѯޤ“búœ*lÆ c´ßÅ:õ´¾¢¡ÍãÆ0¾v=»êÄ^SêÆ ºi¹Ðw2±iÐ5Ú2Ïøámðb d¾µoú„½\ÚÚ8z¢‚^¹\­Âo.S×Ù>%_Y‘¸¢¤}J‡Ö–¼P\² -s·le&’WÒ\Oö;Þ–êb,Œ KÛu®O³¼è*›[Á’ ¶ü‚ì4¯4¯ìŒ~÷ðúæ~/&‡:HÚ¨½I߸´5Œ¤ðd©/Ãú²Í9v~¡Þî¡V–ec϶ §Z¸ÕÇ(“¦ž­$ð¼RWe«JëDÑè2Ue­ÄXm -%[uËÜâ$z)¶˜¢KÿÊ=íy¯4'zÞ+a“–Ê­3^‹–l €<‡sì­S=e‹%“ > -endobj -519 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 423.873057 101.433071 405.123057 ] -/BS << -/W 0 ->> -/Dest (cb27-1) ->> -endobj -520 0 obj -<< -/Filter /FlateDecode -/Length 4780 ->> -stream -xœí]]¯ä¸q}ï_ÑÏVæ÷`ìƒÉ1²È y0ü0Ó;×Fp7À:þû9ÅÉ»ÔR÷½3Öž‹žnUñ£DK”¤Ï -ßiü—œžRÊ1¸óåçÓ/'5E¯ð¯¤XõøDÖÊÅÉë’?ÿL§˜½‰áüzÂÃ!Oþzúï_þw™GŸÍ”s0ÚŸÿú™TÎN1Ũ¼ ÊP)m•÷SŒÆAô„SÌYRÁ q·ª3\N^RÁ gLT fXµ½Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS@£IgôR6YÔEèt4’ÎÚ)D§½–ty -!ø$élñYiIça‹×Î)I[œMQÒØbƒU¢¶˜­¤‹°Å u² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Åt«3ÊO>Y«$Ö“6­¨ÃˆB'…(è l ÊgQ[¼UFÒYØâð'éÀÞÆ”„Ñjht[eµ¤ó°Å˜èDlAçK `lQ1H€4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ0rEl±ÙI\` lA‰ ¬…-ÆY‰ ,ØÖ-XЭSYK\`Á·Né qáÂL%qãڜĔ (e‰ ,8ׂ;%.° ] ÷(qëZŒÄ´k] -Xð®uZI\àÀ»Ö"§¨ƒ-°Ä`ÇÉêd%.pà]«U”¸Àw1ŒŒÄèÕ 8 +“III\àÀ»&)'q)@ÔÅ cÐJ\àÀ»Æ㤙¼k¼28ð®q&H\ Là%zà]Pˆ—ÝØ‚•¸ó=tÆJ\€Â&·DâÞÕÓŸ0¦ÁV“N9H\àÁ»:¡ûD]žtD‹ ã­<é²Ä¼‹‰ÊJ\àÁ» Ï$q8~ÂŒj$.ðà]m1SK:ð®6YI\àÁ»ð¼Ä€Ã¤áH\€ùmÒ -ÐuyR9%‰ r”«ùÜ,Ós»LëÕõ.bÉGŸVgM3—ïA­î2ë‚P§tõÌÒ^Ä>²|/Cºù{<Ïj»Q=³--íYœ_¯¿žOm‡~>ßÿéüá‡Óï>,AW<Ñë0±„—‰îât_–·àôKÀËÚ;ð¼éZ4©ãðø8w“ºôeÙì­œËÚ1¸_7nÝUáìæôcù<[=¤[÷ÇnÇ#6? -Åj¿ÏëkËѶŸX¾¶¬ý× KÑûJ”XìÂ'qä$ÜÅìΖ•ƒßÖ̶]fL—ï -c¥'k_žÃ!ê¾×D¨x¦g)¾ñ-³ÏÛì—QeÕã+ÁyŽ†*{ßËù{óÎýûn|3¨ Ø>õÚ™çpH¹ï1ñ¸âó‡¸g[ÑU_÷²¾Ú㲋hÚ¸âd^¿ÔV\,¦à?vÙ³qᇮœÔxíÝhÜ -Z™/¥Í”U2tIÿ>Z÷å9Z7ü¯¯¸†ÿbHÐýw«C-ó=™®$´ÌGW'j,ä&ÝG–nŒƒûnï&¯²È–7“ò.ªî#u_žÃ!u#ötœ‘õðˆ¢µ‹ó_ òöíEÍdDºîßyºÕ ®DØšéwæ9"݆O¸2ë>ÕèŠÂ×㵶o2à'¢{;óÁ¾ê{ x-m¾Z¹nO3lóõÞÁû|Æ£ |½ìJkÈès,Õ¶Öä;ó½¾ëžë>ï„ð¯†ÐÃnY\AãqÇy‡Æ ÿtôöÇ+'C/•5uE쀄¿çÆÔVÛÕX¸lØ/õíp¬Œ\¯ìÌs4äú}WpŸ½¶þÕ:ì'“ö…½¿¾Åß¾?ÛiL³Á¹þ‰XëÎ<‡Cî>ÿµE’¢Ü¢#Â(ºh`qOŠ®/6ÏK4ì-yŠÑ¾\£øþÓòó]9c$‰å7óK7´º0/ eóý7í¼s…â:÷Î<‡CꆯúQðk+ZƉ´úbÏŧk~ºN4rZ«OØ fvصàÄaŽóÜ‹¡¶´|^X97:§/»B\AìþìÎ<‡CìÛvÏÏ{»'sØÕâ¦×"£9<áãîÌs44‡û>î[WS÷vOï¹j»˜…]ïáµ»ZŸ*§kö¯{èûá‘_l$¯ ï‰8ëÎ<‡C߆ŸúÈsDªî óf¥tauÞÛÅþEb+Hæ>éÎ×Î<‡CòÛb®ß$’ß#VÀ8ߟZV—£>¶Ï?¶Ù¼Ç`߉+èeþéÞ+¶;ó½{CÇÞV-Ý«ZQ1rßÂ9¬¢Í¬”ù(â·lpÇhÓ<ÜOl¤2t?·ÏKFi|b_ÁÎQ/+ÒRsñίä,/™û¥¥šß;§èU4ô‚ëKéØÁœþ—Òá?cd¹)Ô±TÞ¥Š8™zÐů'zG˜ÄÎõ¹!&áüSµ²œnã»Ê.˜—ã”Ç”Ö×gx3°¾þ4¾ž å÷5=»Ý©šÈ–§—_+„Èô­˜Î^MzÓ{ÖÒdç£ÐLñ¡Ú a/Ðñ´½ÀžqS¬­bB+T¬º.6®Ú£é½­éB6ÓyÓiÖý5­‰¬;ï /'TkGé«(¥·;Õ¢jëÕd–•›Ô×”­C\ë'At9 ÂïT-Ú¡3›žõ½#ã*LH®¥Ë5 ºœÄtsi¾õTl©RÏškËwS:Ômj /È.l¢ŒôâH6F”­?—â@=PûZ%†s#¼œ¼k#±‹_O> -©}ç -å+ô¸Ð²r›8©Ž—.¥Î\ahV½a€þ¿R…oRbžòC»)zKH?ê•ÐØlâe#qyh¬ã¤Ð.ö´¬¹tŒ ³¼½tRBz"Ý–ºµØBÊšŒÉc®æI5òXŠuC]…Õöi\É…ªêºØ7(¿.ÄÝN߆*À^+ šÁÝ·r»84î|åâØz\Ó+_š½­\†ù*¼”‰Íb‚~ª•t±Qm¶ë•1a7 ¤ÐÅF_‹¸ŽÖÆNå9 óÏ>Ȥ÷rúË©LsÆz“ñ#XMo0ã^—>ÿ i~ÀçNîüïõ•×yܨƦqëBÌg2zßeÚ)¦÷_Va6Lh£ý.Ö©§õ m7†ñµëÙU'öšR7ÕMË…¾·!›]£-kGxZ±92oí›>a_m=9Tcée+Õ*ðÓÁaê:Û§ä3+G”´OéÐ:Ã2 CVaî–-Ì$@òJZÓ“ýŽw…¥ºØs#èÐvëÓ,/ºÊÊY°$p-?à;Í+Í {£Ÿ^oî÷ÙäPGIµ7é+—¶#i#> -endobj -522 0 obj -<< -/Filter /FlateDecode -/Length 4848 ->> -stream -xœÝ]Kä8r¾ç¯ÈóÃåûs0°†=†^¸a ª²»vaÔ˜õÁß_PâCTè‘Y5íw!;¥>BäÇ”’RW‰¿þ‹V‰SðözûùòËEŠà$þå‹“¬þ>Á]•´A8•|tןI`‚Ééà¯ïü7œöÉß/ÿö»Ë-ó¨«)y­Üõ¯ßHe1éU‚J*èœ!hç•W—˜§‚.Xmת3lŠŽSÁ «udTf”Y«ÌÐAYN3TrQ) ;”–’ÕÁi­ÑŒN9áÑèžÓi%|L:±º -šÓ#|°Ê)N—„÷ÞENga‹KRq:[œ²Vr:ØbM œÎÃãdu°EÇ`8]€-­“]„-@Wdu°Ez©8]2Â¥h-«KÐÉâZ§¥.#9R›†ÕaD¡“|`t¶x髃-ÎHÍé l±øãtà gBŒÌhÕ4º4ŠÓ9Ø¢u°¬¶ ó9жÈà9 -ÀÄKè„MÚr$ £6Úȱ€ŽIØ G:9a} -© ÓŠ##a‹³žc£`‹ÅÈeu°Å$ËqÑ° 8.0¶hk8.0`[ ´p\`@·V&ÅqßZ©<Ç„ 3%ÇŒk¢·P. ”8.0à\îä¸À€t1ÜǬkœ×Ю±Ñs\`À»Æ*Éq¬¶À€…QÑp\`Á»FÉÀqïbiŽ Ы8ô+BÇ(9.°à]¥å¸À‚wuÀÀê‚À4Xð®v¸næïj'5Ǽ«­ö(ü#Yo¼ -q¼;[@°`¾‡NŽ P˜ÐpK8.pà]•0ý1cl%TLžãÞUÝÇê’P-ÎŒ[´²PÞ'Ž x•á¸ÀwAŸ‘ãp¼ÀŒª9.pà]e0Ss:ð®ÒIr\àÀ»ðÇ€ƒPp8.Àü&”tY]2ÅÈqï‚–4ǘ……Œ`V€¤8. ?”ãÞÅôž8.ðà]éŒá¸Àƒw¥…ÛÃê`‹‰šãø5ÐaØ2cΪLÁéÀ»R¡‡9xWÊ8.ðà]š69.ð:r\¼`ÍqV¤<Ç>Ièàò±:/pî8.Êbu°ÅËqk‘LŠ [ŒÖ [à2p\@$ï› ` àÂƶÀSä¸L&b²–ã‚à<â1´*§óZĺfu:8.Á -LÅŠãÄR"bÐs\‚ä.q\3 S–ã‚` † -ǘŠDÔðÌ9‚-p²9.€§ RD [à r\uD”é%ǘ¡K¢©9•gFŽ ¢õ‚\EŽ 0Li‚)YlMp\€þ¡`Ûq\€àÁ&ÂV[Œ³Ĩ)ÜŒÄ[´Ô›oŽ ’„-ðø8.HÒ r“9.C'%ÇIEÄ›Úr\€qIñfä¸ ‰x!3n“ñoŽ ’…-. -«ƒ-pš3nÁIˆ7ÁNœÎÃ,ÇÉ{Š7#Çp3o£g—"Å›].ïºfâtà]x¦žãœ Þ„gËÅöÄëü3^)âŒìŠõ:Ì­ì’÷b¶ -ìšùRŒÄ.ÀLD–ѳ«˜V¡T’]6à_øcvÅ ÓêS¿D净93ºÈ„aaáÝט£%3ɦõ±¿þùòûL¾×?ÿ÷åíwy¥¬ü)€ßXŸkð´O:¡¿ÁW"—gaÂÙÖ4‘…0ð‘80’æu½äzÁGÇ{6Ó'½ôß~¹€¹§Ucí° 34xôËÏ—ßÿáÿüÿúGôÖ—·Ë¿ÿ ­ÐHi£”´ZCߎÒÞð±ÍÇ1»ÃˆR_üë—Ÿúzh‘ËKDŒiY ÇYê®H*Žd¹ØשºœUú¾Íz?››\jÇYg&slhiI–M÷³éiþöüeP¾šæ­ÙAë·¹®0åÍçßpNe»Î&Õ¥‹“ J¯š¦ë‚46Mnm;ULž‹¢bÝFõªä3ó%m4 (]éE9]¦4Ë&»¡Úä»K+Í®Z³—ôÕþÒ’r=©+{¨‹LPr¯ÉèpÙfc[”®£²s»¼Îmf˜ø™l«ëÚ#.¯Q©][àq,m!ȺÚetüáKfSù¾½6™hi ~µv˜ò¢ càôvŒrož‡8z&\‚\¯0I“³a¾´+V±¥Ö©iê©2®Ú _Õ‚û´Ì$ó?ýôwÿ/d2”Ý?(êÆÛô …`bæsï©6+„g$- ¶É0ˆs=‰d?î—¨M‚S„“À”èÞŽr.BkÏÚs”qdrè3.·}!`ÎW2_Qn©ƒ2óí+ë¼Ùh!ûm.«´ÒPÇD0]}}ùÁž9ï4З²Ã«G°D‘döªïoyZOEÓùÀ_çQnê7ðyØŸÕò‘¢‹‰p›.ͽ°[¤Nút—¼ãÙiæ]÷ºœ á®S¿ŸÉìÒ¬§xº/‘hp¨4ì ÒÓp‰fs†˜Ü& -ùR¦»ãÑgßL„nT¾¶óúm–#“êÊøFl›Â—ù§éq9rkÝãhWË|FO¢ãç¾Æ „FYÐF‹vÎ8`û=•åIðÙ¬NvŸw÷ÿyŽ2t·z›£*ÿ„‰zߺrTšœ¶ÌM1[³æ'+Rr1¨¡Â=~¢›Ÿ÷òÓÉÆÈŸºåÛD¦©|HO®X Žµ“yž™n—CGO­¶^¬=”øu«/¸ïå ùî3¥Zú:ä²®qQr…ÂæûY§…tÖböQx.ÏÓ¡ðœ¿X{ÆvHØâ¬×.þ³ÇH¹EÖØ“=—ÞQ_©#»¢¯Sþ)2Y–?úÅ}}&nÛºàIf Ôø¶÷é)Ÿ˜ŸÌØGÞ¹<φ³¿'º ZGßšãýqÿÉ<φf¿ï÷®üÑ‚›+bvFF¿:Tz˜F‹üÕ1ß?‹ìum[ãÈûÊ­£uƒ[}ç“Òïr¥ÊÏãï¢ñdž§Cã¾»u¿°ç\ê™ÄÓõé…ÞÝÔŠ<ðäêÎÂbýQ²+d+ô/ã­CÚ‰•µœîCóÒ2ðWOæy:dø«Ÿåtݳn¹¸¿uâùðüܨž?¸{ZŸ?%¾%ž¾ ×ùðsîèëÖ.ÏFD'ó<úâ9óè^»å¢”ÑwâVÑïñÇ:_ñ·ÇsÉ\Zuc¬æhצ)bðmf¡•lŠ4é^çßÃ3:ÊS"Œ¢7/ÓoâÍœ·æ³-_Wý•EŸöeªƒ†Z®Ÿm)œ@yℼñ|Æuðøêï6ŸÅ×¹îúê®>ö.ˆU>­Ï\Ç`Çx^~-TË™Y˜m»Í˜Îïc=ðtäÉ<φ¨tp—8>Ö³´¾ñ[fŸÙÏ£*©û#Á“yžUO-þóÎݽûi|3¨8=°êu2ÏÓ!åàîk·®ø«ùÆOhÏEY—å~YÂyuý½¤quk -å©›²­Î#ëÂwÝ9)ëµ<ÛšüýOcŸÌóth=ð¿¾c ÿ«!áÞg άÌÝñtO^ŸÓ½Èõóã=³ÅJÚR»•­³¿·:™çÙJÍð‘ÇZv7Ôxôq’O\zûí-›UHþiÞ#28ôž£÷cÐƉ©Ûr²âð0ÉìÊfëÍ ¯jØ[q¹ßŠ¤—ØxÕ@W -´ŸêÖnúÛä]å½üìA•mÒæ»ñ´u\ù}¯.;tä9rZKªÛÄ‘î[Éy·Žy;º¼5[šäy«4/ÛVuóq݆-Cp®öE¢üÄhÝVsµü²…[gÏb›º4{xùVÝøËÅç}UÞ^zxéš^sõV#Ò•ö"ŽÒXUºMå†Vלù•;¿ÔTô?D’6æ§M™s·ö'sú_r‡ÿ|QÉ -jb„é2‹£.'Mü~¡7¦XFl­Ð'Å$œe-˪’°Ên Ö Ò˜Ò8!çÃT §­F¦Cľåw%í³eeIdò^®SÅŒ9½Z‹éê¥P³˜Þ:…™Ï|5…¶T«ÂV íÓ¶[ÆC±2²ºµPvÕ5±¶ÅE»”צóMXMï›NuÝ_ÒêÐuçžðvAµf”¾³Rz×E)ª´®–Uf:£R•º’²vˆ­ýĈnFøƒ,E[tfÕw½F;†OÂX€dkºTÒ1¢Û…M7—æjO…š*¶¬©´|3¥AÝÄÚðŒ,·Ñ, ô­nŒHS—bÚAµöµŒnôJx»8[Gb¿_\`R»ÆÒèõBÓ•[ÅQ6¼4)­gLGšEFû-ÓÿU¸*%æÉÊVHÑžéí¬UBc³Š—ÔË}e˵€²¡¥íšK…P1Û·{&=‘nM][l!횬“‡TÌ#’ªä±«Šº*& -+íS¹²ÊbªkbW¡ü¾7;]ªNÖʼêàîj¹Mì+w¾÷âP{\ÑøÕÞZn‡ù"¼å‰ÍŽb‚~,•4±–u¶k•uÂfH¡‰¾gq­•è¥^å° 2nƽ]þrÉÓÂMû¥z£è}.½×¥®ÿƒ4?áóŸ{ý§²ø4kYÙBWn]ˆû™ŒÞþOŠiËÅ"¬Ã¦jÛÑ~«ØÒº‚†:kÝñµmÙe#ö’RUUUÛ ]kÃN¬+tµ2]˸ámhÅêÈ|´oÚ„=šrq´ÿ+m=_¬ÂI98MgÚ”|íŠÄ%mS:´VwqÒŠÓ®ÂÔ,[˜I€ì+©MOöÛ¾+ ÕÕtnš¦³mší‹.²|]¸À¦?é ¶ª¯4-ìaŒ~xx}¸ßg“}$­Ô^¥ï½´^I+áñR“Oýò´Î9&oZok¡Z–éÆž©NN͵°‹“£4¶ls×Wj‹lQi™(*]@&‹¬–ŠM~ÎVš%_qd[)TŸ¢IÿÒ·´ë{¥6¢ë{ůÒR¹eÆ«Þ’©ëáÚÕÉ–²ú’‘‘ù>w•Ú–ÒÙºn»¬»Ì¶×î2l3Ä7ÏT¶ln•ò!þ{dì°oàR•¿¼2¢b{å&j«‚Ñ&¿EUFïÁ!–^„lì–Jîγ¿v²^!)k'}MŠVÙŒ‘AÑûpyÞ¡HÇŠ®¨ò®\>[’ì%Ý×ÌÒP£¹õ; ò óºe^ ª§BEzÿ©!§7ÒkÛ‚äß ‘—Pn‹å“ü÷¿É8_« -endstream -endobj -523 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 522 0 R -/Resources 4 0 R -/Annots [ 524 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -524 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 713.673057 101.433071 694.923057 ] -/BS << -/W 0 ->> -/Dest (cb30-1) ->> -endobj -525 0 obj -<< -/Filter /FlateDecode -/Length 4792 ->> -stream -xœí]K$9¾×¯¨3Ò&~?$Ä ‹@ Fp@¦k§A¨iáÀßç gÚŽtFVe×Ô°µˆiõte„‘áÏ_„U.}VøùJã¿äô”RŽÁ/ßž¾;©)z…¥ÄꢨßèÏZ¹8yCòçoI`ã³71œßNøo¸äÅßNúÑéë:úl¦œƒÑþüÏO¤rvŠ)FåU†Ji+¨¼Ÿb4>ª 'ÜbÎ’ -føèŒÛª"Ìp9yI3œ1IP%˜a]Ôv«Ê0ÃDí$ÌÐÙGA¥ìÐF)QC”sÖ:í§§IgôR6YÔEèt4’ÎÚ)D§½–ty -!ø$élñYiIça‹×Î)I[œMQÒØbƒU¢¶˜­¤‹°ÅÀ;YÐ%Øt%Q[TPZÒe;ùœœu:•bÚêŒò“OÖ*I§õäM+ê0£0H! -:[‚òYÔÁo•‘t¶8üH:ð„·1%a¶šÝVY-é03»Éï®sÓäˆ !ŸÛ -ÛÚ]Åûí¾’l}w‡Ûši)è=NÞ/D2òVáÇ>-6ìeºŸâÚ1DÚIä&h¸¾³Ê VX#Õ -(cá'À|ƒE…r«Â?ûpK-VÌàí’™dI#êþðíéÇ¿úúë_üò̯§?ÿDV~ú•Æ_oç¿î²\‡å:-×~ù›·2­‡6ª^ͱ&P ˜òë>-º¥Œóë:îuh#.}PÝ0”¹ eõPþÓ ¿ fÛ~m»ènØØ|¡…vê5k¯¾öƒýGúÒê§9øúôókP–eÊ JÊÚ‘QÀ¹ŠãcU>Ç_¾9qøî@å=°®eZ?fùû²´õ øyfu‡éò\ð–¡Kx ÒîÖN…Ø=XçÙÀ‹eöUôÞœüCø ¡ï™5×î¡öö}Òta¸'ÑÞ4Ú÷3éÁ:O‡FwKdµ¿I%âšúzDÂ÷7ʬ¿ÚöÇ…Ë.½<É~8»ƒ\–˜b¥’U2´B¾ŽÜcuž¹×“ØÏK°¾„Æu¹ -õcøõs’í1^´:B|Ù”¹Å¹<%¤¸ln­ÂÖy:äË_«ÇêßÑ£#ÂW(ºh«Ìe@‘ƒÜTÇIê¯Ó’%Ü[ÿ…͈°F¯[,ÆÑNˆKCÛî…!rôó&C’‘jzÞy8;8XçÙjn䪅ܱzÑ2N|‡×9JíË\ßú-§µþ†v¨¼9`׊‡1ÖqW8°•åqaçÞèž¾ì -q±wä³ë<boä³{Q¯`ž*ÏûüLæiW‹7³–4ß‘ã¬óth¾žã~îjªŒFÍ7m#n†v9VĹ>ÂV3&à׿Un×´òÊóß?Æ5šx¦óågòúîØe=XçéÐw#Oý_ä¹÷ìT]Açf¥ta}:Ù_nïAF²å9éÁ×Á:φdûy{®?H$?b¯€qz‹zng“‡~àõGŸ½Ì¨ul6Œ³¨¯øvÐËòÓ£OmÖy:ô^Ïi7£5¬ZzxdV:uﳡb䨗Î3+«h3;m¾ñ·l7¸Ïe¸oØLeègÀ;vPzÇ{ Öy:”ÞxSÁâêOõô Cȸ—: ©¢µ ÓÍyå÷ù”á‡üäÀ²U¹Éh—œ»…Ôcuž©×óÚÿ?9ØÉX{ÍV¿ö‘x?qÝæÞ3Å&7Ün­îŽ=؃už ­îzîÚFîu’²gO7Wûª5îì²·‘²µæ•!šå  ¹µÝªyÝëºþ jÊÞïÎÞBYÔ– ï›qæŽÙVlŽì>y&4#aº•ùf¬þ# -¿O¿ž½7VÓ'ç5ü|ë3 Çê<~ý½]¿+?<Ír×öN—¼ƒž›˜:’ÛÞ·Ó¤v÷RÃn›ãþÊÒNÙwÀZ”oŸù¢‘ ‰vÉ>åé<mµIe$~þÛßýú¿eû”´CCkºö¬¿ì}=¦yx˜­ íäݹ¬'öy©w6_oäiðð'æÝ¥-÷:ÿ¥zóSÅEvYäÕ®O³-¥¯Ë’_^ÝbKí»\×ÕVäöÌ^>äÖÄÝ:?¯xÀ|Zäf©Ÿ±Ôùn[·e®¼N•Ð=_G¤”ëØå×3Ü-×ÅÞæÅã^ -jô’¢=kž™°¿Ü«26—¥·°tVí»8…lép§¥}»Ü¹§ÈµŒ@ñî7Ûv‚¥O>Óé:ÛvÔâÙSÅ[×ìHvÏÕïeåEzZcfæl£S–‚r&d¡í2/ó\¿ éuië#ŸoæÓâ‹j“_¢y꾦{,HˆK»‹ÿ:Ú–²‚ý}NõùäƒZ!®´eeû(k1¯}U.û8ªÑÅÆņz?E&Ùç»=þãR6uýhS¥Î/¹¿–îÛ ¾æ @íhÝï¹öW³¥RŸa¥úŽæ EGs|»²çªïÌè»2³Íz¼¼í>ÜåÐѯµÌ+ãvðêí1 –gY•4³£ÎéØËmêZïƸéåii);Gîk¾sß±ûqÓl쾬ÌOl—ñMç͈Ë+ø*úM„tklÏÑ骿‚„µ•ªÏÓÚ?{Íî±ò[?ëüAö‡qÝAµ¹g˜„|;Æä…{~ŒWý¼‰ <{ì埒G´^×còš{j\Ú~¤ÕOÁ–„Ó©b‡Ö1ëÜõY÷.`|³×ç)yš‘ïÓ470ôB_–¥Êó ìv˜Ã¤25tÈ!Ä·Lë/v[46O>à" --ú×[µ]*¬… Ús«¶Sö3©6[z[¶Ù›2åäRçƒÝñÐë­;7ïÞÙrˆ¨r‡çé(=¸.Dù>oÕ¦q¯Zó8Ï»d0Ë’C -ºMŸ—Q¸ÚǸl9‰øɼąjûp KRV »—üo“k:’2ÓáoC§efï^6¡XŽ“/ÏÕoÞÁ:O²yÓ¸/;uûÔGڮϾ‡žqãÍÅ«‡žãÆ !µ,™FF˜—ë~6†ÒÃHÛÔÍV³ ó1~ÛA'd<#6Þæ›Õ?XçIÐÙÍþ:ñÑø‡¢â8¢F¸F–¯a‡5T£À<ç­…úÊW ?7åìSÔcÿý~9K*zÔô”ÀÑiP™MÕ`u³È UßlO:ëá@£µûÐ$"orz¤s×ö|Gû ¦î‰„å³ËZƒYÛ ]‚š©þÖKŽþÂÖw~ηk ,¾,¿Ï³Ü´ý#š‚Úì«”þ>.cvQ«5bk¿®Ÿ˜=«õe^>¹Qîa3ŒßB9?P—3dUH“6tÈü9¸‰¶#b>Ó™…IY§ë°éâh}Þ©YŽæÿ®•ZNëWt€/Þ8åÏ.–òß•ÿö¤³›ÈÿeÓ°œ¬­Š™‡Åo':YÝ bçúÎã 1 ——ªµåtÛȬ² x2Ny,‰©–—}·“âæüÒøz7Tß×jtâ"ª©W4æŽ!j½ÓÝ«I/b:>Mv¹ -ÍÊ¿›°7èxÙÞ`¯xS¬­bB¿*Ö]WíÑtšis]èÂf:wfÃ_˚ȆóšðrB·v”¾‰R:Û¬6U½‹U}•YfTnR_K¶qmœÑå$¿Rµi‡Álz6jt²è,LH®•Ëµœ ºœÄrKk¾Tl¥R¯š«ç»)ê65Ç ² {"éë6ØQ¶¾\‹i£»µJ 7f#¼œ¼k3±‹ßN> -¥}ç -å+ô¸Ð²v›8©Ž—.©º -S³Êè\Fú¦ -ߤÄ<å…v Rt¶j¿êÐÜlⵓ¸<4Öq’´‹½,s—Ž±a–ûK'%”'Òm¥›ÇVRæ2&¹šG$ÕÈc-Ö uMLVýÓ¸’ U5Ýu±oP~[‰»¾MU¯ZgA3¸ûÖn‡Æo\Ûˆk:(·ÙÛÚe˜¯ÂK lnôSí¤‹jÑ®wÆ„Ý0Bw}+â:[;•-–—}’I÷rúÛ©„9c½¡Íµ`5ûγ.}þ7Ê|ß¿ŸÜù7õ Ñ9ŽÕØÂ4n]‰y$£o IÅô­!Uئ Çh¿‹uêe}EC‹ãÆ0¾v½ºêÄ^KêÆ ºi¹Ðw2±iÐ5Ú2Ïø+ÂËàÅ–È|îØô€=_Úzs´YHGÔV«pÁo—©ëlÉgÖ$®¨héÐ:Ã*â‚7ŠKÖaî–­Ì$@òNšëÉ~LJ‚6¼øK#èÒvëa–7]eå.X¤À–_ð†ææ•=‚ÑwO¯Ï÷ÅäPgIµ7é—¶#i#[¼ÝC­-ËæžmINŸ=µp« ŽQ&M½ÚRÀóN]•­:­¢ÑdªÊZ‹±Ú–jÕ-厓è¥ØrŠ.ý÷´ç£Òœèù¨„MYj·F¼–-Ù–yçØïNõ’-—L‚,ðÚMêzI[e۾ݺïmÏì6\7$ôÌTõj~Sò.þ»gîÌu†]©Tý);#:õ£¹¨ŽÖØòmk*…qô…Ñ%¶Uòî:×÷N¶;$uï„÷„ᘜµ*júÞ<Ù„7(òmkª~§ž\-+ñ–Þçak¨ÓœÚì •/N*Ûeƒ¨]FZ*Ò÷¤YJz}½KTZ~K\9cµ}R~þ<Γ -endstream -endobj -526 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 525 0 R -/Resources 4 0 R -/Annots [ 527 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -527 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 210.123057 101.433071 191.373057 ] -/BS << -/W 0 ->> -/Dest (cb33-1) ->> -endobj -528 0 obj -<< -/Filter /FlateDecode -/Length 4698 ->> -stream -xœÝ]ˮܸÝ÷Wôzsø~Á,L8È ƒÉ"Èâºí;ƒà:€'‹ü~ÕâCTI¢ºÛwäñE»[U¤X"‹%JgŽ¿7ÿy-˜÷ÁY}¾|:}>qæ Ç¿!Åä`Pÿˆ3gÁµcFëÍùS(Ç\0ÒÙóË ÿ5‡uò—Ó?¾9ýgšGœ% ÁJaο|Œ*­˜óÎqC¨T\(Be sNK¨¬`¸Ä(Ì0NK=W9˜¡ƒ7” -fh)=¡ò0Ci'Ô\`†tBS*˜!‚q„JpØ!$礆p­•$tÂ0‹J·”N -f}Ô9è„“”N)fFPºÀ¬µÆS: [Là‚ÒØb„ÖœÒÁ­¼£t¶(«8©ƒ-Ò;Eél‘¨@è0휢h@ô ŽâÅtRPD 8l1ÚRL lÑ蹤¶¨ ).P¶ ÅJÁ©Å -l«Š èVó (.Pà[Í…¥¸@pa&§¸@q•·šâʔŠ-œ«À(.º»£¸@u•±’âÚUÚ[Š xWiÁ).Ðà]¥“ÔÁ˜â°#SÂ+Š 4xW î(.Ðà]t#IqZ•‡–â`…Iï9ż+=×hð®tHc胊â Þ•—@Üà]i¸¤¸@ƒw¥––â…8é €wA!†v` –âŒ÷ÐIEqNÆ$ÜŠ xW DŸ[1ქ¸À€w…Gó‘ºÀ„CýµÌ„µâÞÅ@¥(.0à]Ч§¸Ï0¢JŠ xW(ŒÔ”¼+dàð.üCqàÀŠ 0¾1Á]RÞS\`À» %IqFaÆ=˜„Ô9 Š @ƌå¸À‚w1¼Š ,x—¥(.°à]®áö:Ø¢¼¤¸~ tè¶DŸ†³Ê¸SP:ð.haJÞåÜŠ ,x7›XÒS\¼`Iqõšç,Å6pèàò‘:Ëpl(.pÊ"u°E;MqkTð8 [””8 [à2P\€ÀÀ…œhØO‘â0óAkŠ œ±˜¡V)•Ì;Ð5©óÐ Gqsša(`.Å<:=Å xÝ%Š `tBS\àlAW¡¸Cóž9¥°N6Åðt¡‚â/a ¼AŠ ¼ô˜eZNqFhÌ%QÕ”Nó8ÏôxmYt).@7ÁLLIê` h‚â´OœlŠ 0ùÁdÓR[”Ñx/ãtÓS\à=l‘\R\lÆù¦£¸ pØâ‚À-‹n2Å qè8§¸ ù¦Ô _Æù¦§¸ (Žù&¦xD¿ ÊÆù¦£¸ hØbà¢:ا)ýœ„ù&؉ÒYØ"ƒ¦¸ X盞⸘o£'C>Î7.ïšf¢tà]x¦–â`¾ Ï–šÛs¯qðÏh¥3NOF 8¨×`l%CÜ‹ÑÊ‘1òs$2h31³ô–Œ`X…Rp2lÀÁ¿ðÇôŒ®Ñ§:De‡1Ó»x‘ÝBûÏsŽ’L-$»ÆÇ~ùéôíßóOÿ==3DÊÒŸ€ƒ0¤Öð¯Ð d@sxß—¯,Z°A¨ aà-y¢.#hq ºE#1üÅçù‰å0üÇ¡x•Š‡G”Å£9€ñ[Ùcú:Ä+‹wfy Á…`´µH£‚ŒìÈÁ¯äz’ø÷ïNêhÅ5š…2É|p*&ÃZšó»O§oÿôöíþø7ÀèÝó響ãܘïÞˆø-›owý†K‡î€¿ãÇ]I]ü­FÝS“æÃø=Ê¥õéóaüó´úãh—¿~ëÑn=Åè%c5ê.㱦Í×æ²ÁÞœ/•éšóºTöwÿ:¿{{úþÝ8ð82pxÖú²ÜƒµB,[ Æ\Ø~žbK?]ë×ðñ8Õóûª­R(má2`IUKmå§m¢í´Í†4\ð;”²’“òäò¹Ý.3Öqžt\/ýÎõðÜ_ÖP§¢ªg×êiœÂÝ ÔÎ<‡Cª\eC -!B4¬’t|ΫLs™¢gÂ\Mk›3Ę*ïejß`GBFêYâÚëÈž4^ßÀê¶:VÍo9æ =O”ëÈ×})v™–=eÉ·ˆL•ù0Æâ0ËÔÜn!³/ÏáiV94×b;^×ÈZC‰×ú„ûžÈw@–(ésoàÓ² -šPX<@1‡‡³ŽÂ¾<‡CaŸ·˜[FWHXâ¬÷E—Z` ©-÷‘Èj[²æÒå¥2Gôý5¿2sŽo½âº<å—mðdª»Ku¤¨ýÁ·ôC¬iy}yŽ†<¹áC¶-é¦5=C_Õ²³‘lá7åûçrˆ¹ÂŒw¨‘ý»Ÿ×ý9-›òG>”¢ñJ¾x¯¥Ñ++QH¸—1Š³ŠÞÎ<‡Cï†_™jðyúyå©iÙjÖ²Æw5Z^k†<Ø´ä¼}f§͞F5ªO¢ {frKö¦<›³ºÄß±±}y‡Øuó0>à½sÞ=³·WðnöôŠIz½Áµvÿ¾3Ïá»î£þÖ":ãì×DwBëfï[@³ß?ïïÌs44«u¿wænxÔ蜳Ò3êèPjáØ[L㯶ˆyuþL²÷sÛ -Gî;oî­ Üª*Ÿ4®4æbXa°ŠÆÎ<‡Cãº;CVªá -Y±eî™OK¿Ýº«3µä#7<9»³0‰?r2B6Cò2ž+¤uDÖ†twK ȼÁ_íÌs8dnø«µ‚ª]±ÇGø†Gó~P[ÅFãª,é…4[¨íËs8ÔnÄSïõ wû[¸W¥]ËKhm=–Ä¥Íuæ(ÖÞU~î³xñBo¸uß™çhÕq×ÎûFÙ3hFÈICtH^Qï&Îø%µXB´Ó·çOÈhšËiPݮ٘Ü?ê°y/Ûµ#µ¿›ë²µíC•ï†»¥ºøœ:¿SÇUz«¨íÌs8Ônø©œFK=*±ž†?_}W?c«H»ÞÊË+„æü·¶ -&sl{ÏÞÎïT,ÅbIO ¾÷4öÜ!]3»l{ÕÖR­öóegžÃ!oã>}3¿õu¢e'wÛѵ]‘RÝ5”õ|x)ZÞ"vOt]¯ 髨- ¸ò/gBig·"Ny‡àøé>eºöÄ-'÷·ê˜€›Ú”ÑñኂásÇÝÓ¼ú4òmäéKs(k{ÉÖJQ]Å.{gDyŽ†>ÓçonÝk×Ô,¥õ¨(ú¬ò¿>ž»§'Ó(5ÅW„#Ä|DÔÖj”Î<‡Cé†Ù;ë¨ïR6³ÙÂâ¬Èñ¼sâç¹) -JÔºjé†Ëê(øÖj £ö¯5êÌs¸Öíôá|S« }u˜MàÚdœ1Ø2²ÄHvœ1Ȩ{=¦t1Oša$½ŠÞùe<ïû*Ÿ.ùë²ò3uÚ§kqÂÏkG['Ä<þŠ¼aÆóˆë ñUßmîÅW_žÃákÃúµ«¶ÚÕÆöŠ‹È*kãžëhìhÓ³Bù<ø­ähÛeÄô0ã]`¬VGvæ9¢ìÆ]b[ËÆøÆ×Ì>÷ÙO£ÊŠý3ÁÎ<‡CÕƪÅ_™wv·îÃøfìQþ°½!êÕ™çpHÙ¸ûZÅ¿˜o¼°B»]Õ—¥ž,¡¼ºú^RžqU1…´êƈÛ:â»xí k[­Ý¿»3Ïáкá½âþ‹!aïÚ‚žÈÜŽÕ=C|¢M÷Äçë#Ú{f“HÚR«ÈVïóVyŽ†T· »9ÎÈo_MòÀÈÛ×5£énXØ™çpˆì{’å®ÕÈ{î(¼f žþþÍ ø†è^gžÃ!xÃW}Ðzú»ßðàg¨mêò‚ðôVwZ{ŸèÌs8ônø®_ñS~é5¯ºjq7Ä;ó ~Ã?]X ={êîɳ† ¿æÚÔvwœÉ \Ö¬—úz8váT7<‘Ý™çpÈí»ƒ{ó3§¯Ðf=µ.ìüz¿½?hë©M³õ.µb­y‡Ü>ÿuï%&(ºh¨âž1º>Y?OqP³ftˆÈÇí3Ÿ=õ;½*§$UyÛõüGxë…¿á>wgžÃ!uÃW}"|ÇT‹ÕÛÊ^û=²Ã®µç²Ú¦µ×Ïp®ñ÷pŸæcyq¶óÇ•RœßŒeWö ç0Õ5˜t ³fü|²ÃJbØE[Ï„ŒÛìž­fqª ç¸k“çJ‹Ôlb¨hq^È9lNü9§÷+æq ø}Õu3ãê`LÿyhðO'4‹õY⺷(Ä^¦ƒ"~9Že5!ÖšÉNqŽ?y>—)a–]@¶Ž…6¥2Œ?C>yÜ¿äúSšt51¿IÙâž?š§DjØõæZ0!DN+æâxõœ‰Q÷çõLG6›bl²ÂrB]§-',7ÅBñJhæB^WÄR'{DÜÏ-W-Âlz]u¢jþ”Vºª9ׄ—ŠU­ô…”Æ]AÓ©RíJžeª2*d©I)sƒèÜN„èr"„ox:µFcf}Õjqoµ«Ð' éœ.¤t„èr"Óg3¹¥\NåKÖj¾˜R ®|®xB6ÔÑ(uqÃñªp•~NÅ6¶@jkî+ÜÈ™ðr2:÷Ä"~9G¤6…+¸IЫ…ª:o{^ðR¤ñ9‰ë/tÍ$‹;SÅÿ¯Ta²42ÏðCè ©¸»\9*…ľ™ÅÓJªå6³Ž¦j@hWÒVÕ%œË˜­ëKxN¤¤›Sç›H«*«ä.$ó"Ieò˜ŠEF]G -Kõ“¹²òdŠ+b“¡ü2;Mîª{*ÌŠ -î&Ÿ·ˆmæΗZìr‹‹¸U`¶7Ÿ·Â|^†M·â}Ÿ -)bÉóhW -«„Å0B}Ä©·fvŠÛŸ§Ÿ¥“Q#îåôóiæ¤22à‡U"î|[{]âü?¤y‹Ï¿Oúü—´ÕÚu—<³…ÌÜ:×#YÜ'ÝwŠã¾éI˜»M%”º¢ý"¾¤5 y—²âk]²óBì)¥È *²¶šR‡•XfèJ¡ªš1+ÂKS‹Ù‘¹·mÊ€}=TéâPŒŠ›ô%«pP_}Ñ©2$Ÿ«Sâ(&-C:´ZVqPŸ‡U¡X613².$W}´_×M¡bYÕAåFÄCUtº ³õ©“l¸Š* \`UÔ'Ö¢.4Lì!Œ¾¹{ÝÝî£É6õ€(ÍÔž¥/µ4_X”f£¥j8´ÓÃ<æ¨s—ªúžÊNNéŵГƒ£•Ô—lcSª“lRh(2]@Æ“,ŸÑ%›ì˜-UËpÅž¬%—}Š"ý¹®iS·J®DS·Š¥çM#^ö–Tv€L gW®Ž—”Ù—ô„ÌÖ¹³T—”*ÉæeëiÙi´=W—¡‹!¶x¦¼d3³”7ñß-}皧‰BP©Òß¾lNŠZ §¤:‡èîxkÁ!:nî´¯B%»ó¬ÇNæ’;©KBs0­wx[0ኰ­¨Nõ2îëJg œ¼¤}@„† -ÍñYdH15Æ2‡Q>tqªh°Á7÷>npï¸ CEÃÍ<= Ÿ ÿ^3îZ -endstream -endobj -529 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 528 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -530 0 obj -<< -/Filter /FlateDecode -/Length 4925 ->> -stream -xœÝ]Mä8r½ç¯Èó£å÷`Ìa5ì1¼ðÂõaáCuv×.Œ³>øïû%’!*¤TV×´Û…êJEð#|| R”R_~~Ðø/9=¥”cp×ÛÏ—_.jŠ^á_I±º(ê?â7ú«V.N^çüõgØ8ÅìM ×· þ.yò·Ëüæòßë<új¦œƒÑþú·¯¤rvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS€Óƒ¤3z -)›,ê"t:Igí¢Ó^Kº<…|’t¶ø¬´¤ó°Åk甤ƒ-Φ(él±Á*Q[LŠVÒEØbà,èlº’¨ƒ-*(-é²|NΉº J1muFùÉ'k•¤ÓzòÀ¦uQè¤-Aù,ê`‹·ÊH: [~$xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØßþrùí &ßë_þçòú›²SV4„’Ú!¾Â00Ýí}k4ßÙ1Ó`ÙC‰©Ú˜»YD¸l³‘Y˜ðèßuûae+L%c‘+¡@«SKÅàžRºÃ2+C³d·õáÄF»äܼžËò-xý5`šóL«›t^ÏwÃÿäêç6¬Fêhêõ0jºeHò¡ÄËk¶úµÄöÄu™Õ?߆ö»e¨Ò¦ -Q¯;HöïàâsyžÉÇ®eHk(º­{ê9{váÙ‘ÿG.¬e \jKc†ö9V‡êTц8Ô=ðùÑh,uÚmÞÓȘNï`çsyžÓÇÑð8Óœ^üÖyœQõN_·xÄÏe³KL9ÖÔÏ›ù##f„9žÆð8ø!*>šÆxFÂtKóe¬þgÞN¿¦ÇºNÓx ?ßÁïÉ<φ_sKø]ùyäi»–H"-q¤„›˜:Û®VWc„²ƒÿ’vg#Å1ì¶10n®,åÐ:À}¸^·Y¾m!SOŠõD»d»­†Ž·i«M*=ñû?üÛ¿üé½'Êö ­éËÊç<{Á4ï#µ!<;§õÄ<Ÿk«æë< ÞýÊ<»”å^ç¿”¯¤ ˵_ʺ-±ãmÑ-uQš¹ÇxÜÖëî=0ç%¿Í^=åÆÄÝøçÚJ?÷sç¢ÅZÝ- —CÇWß`iÜ:M¹¶Ë¯g-°½¼õ%dýL±ÊÚ-ŸZ§wToõ@Xšx›-îÌ–Nb.m¶K/yš–Þ*=õe[N°t›’ŽÂmËQK»æ6—vÙA§7e;ToËÊäo3ó‚`‰ Êaheäç9AãëRÖ Gt÷¹ùºø¢Úä—¹2u_SKŸÆ¥ÜÅ/KZÁþ†½ô™Ë«å—r¬lÅ浯wwü›Fû–úk[ŠL²Í3[^–´©ëG›*®ç1Á|–ûg©Ínð3g -*GëÞæZ_CJ~†“5;±¾tóõ̇IðËÂA¢?-³õóýôc»÷pêr·ûL9ûþ;ô݆ËÏø®òƯÓ×çl0ǘÀbpÝ.³”Éù¬ÎsãªÃ?[n¸Ò‘âS«Ž“yÞuÛû—¾Ùkó”¼U‰ËBaãvƒ^(вàvî„Ý -s˜T¦ã¢C…5là7d\ ¢<.Ñ p…ýë½Ü.•ÓÕ!ˆöÜËíÔ=úLÊ]‚Úñn¦½ë!S[r>عU×Evè±£[xã͇{­w¶œ ´Z²ô®çé=\¢ÜÎ{¹©ßÀÏÖ|œç]2eÉ!¤ÝǦ_vÇuŒ‹L‘“À>@x'ˆËʲ@ÿ¼tV »·˜Øëôœqì­Ö\6w±.ϳa?*}„ý†ÑáËj«6Àüò—~ã|-ê<;Ëô2¤ù²ü]äf ™uýý²üÍÛ´ßoV|ÏÖ¦Œ²HÇ´+bÔ9”Ìó$(ëf›p„²GNÐÝlà§åÆs{§Ðghá(ÑspëŸß}ŒK«Í͘®ßAª#õ\ž§Cª?äÃo=löîCtCoûÄx–÷¶¶¯Ø1þŽ ­FÒÒ¾Âë]Ûá³Yò䑧{;Z»o,Æ8¸º‹ÌÐøž†u„{È<—çé9tŒÔš÷8²ŽP"ëáy_žï>Yº§ß|Gð -{ìç¼™”wsŒÂsyž …é\¼Øz†ÝJÛ;3oc®ùè©#÷‰ÈÚ9Zþ@}«ã¥Ÿçü«£j@”°>³ißÖO -ûkîN ™xŽ{7ñèBšbFߪ=L£e|aDÌwçÏ*û¼µ­säcå¶ÑºÃ­™Å¤g‰?™çéÐxÇîÝ/äœK=ó-ëévgü wWjïz`W‰;dôÇïÏî¬ÕcîvùŽxõdž§CæxõƒgíAÕC{>[ô³ƒZ¶7JïE+§çî¡ö\žgC-¡àP¿58|8àú€›Uï}ÉȲ ÏsJ/GyOø!Õ¿y¦i{KôË{¦¢GozzÇ6½0*³×V5\ÞMr†õðÌö5öA&͸·Ë_/eš3Özö&XMï„çQ—¾þ/Òü„ßÿº¸ë¿Ö—Îó¸Q-LãÖ•˜Ïdô "餘¾Q¤ -Û°aBãíw±N=­¯hhó¸1Œ¯]Ï®:±×”º1¨nZ.ô݇Llt¶Ì3þ@x¼Ø™oí›>aÏ—¶6Žž%¢××V«pÁ›ƒËÔu¶OÉWV$®(iŸÒ¡u†eÄ/—¬ÂÜ-[™I€ä•4דýŽw…¥ºØ #èÒvëÓ,/ºÊJ+X„À–_ð‚æ•æ•=‚Ñï^ßÜï‹É¡Ž’6joÒ7.m #i#> -endobj -532 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 450.123057 101.433071 431.373057 ] -/BS << -/W 0 ->> -/Dest (cb36-1) ->> -endobj -533 0 obj -<< -/Filter /FlateDecode -/Length 4910 ->> -stream -xœí]Mä8r½ç¯Èó£å÷°Øƒ5ì1¼ðÀ û`øЕӵ £ÆÀ¬þû~A‰dˆJ)³ª{³îBu¥"H*Èx| †”’>+üü ñ_rzJ)ÇàΗ_N¿žÔ½Â¿RbuPÔ?á7ú³V.N^çüùØ8ÅìM ç·þyñ·Ó¿ÿæôßë:úl¦œƒÑþü—/¤rvŠ)FåU†Ji+¨¼Ÿb4>ª 't1gI3|tÆ]«"Ìp9yI3œ1IP%˜a]ÔöZ•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚOƒ$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ƒÑÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓµÎ(?ùd­’tZOØ´¢3 -N -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) ³ÕÐì¶ÊjIça‹1щ:ØçK `lQ1H€4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ0sEl±ÙI\` lA‰ ¬…-ÆY‰ ,ØÖ-XЭSYK\`Á·Né qáÂL%qãڜĔ (e‰ ,8ׂ;%.° ]L÷(qëZŒÄ´k] -Xð®uZI\àÀ»Ö¢¦¨ƒ-°Ä`ÇÉêd%.pà]«U”¸Àw1ŒÄðê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„9h%.pà]ãÑiå﯌ļkœ (øG‰Ñxâåp¶€`%.Àz± ±É ,‘¸ÀƒwuÆò'Ìi°Õ¤Sxð®NpŸ¨Ë“ŽqaÞb”'B–¸Àƒw±PY‰ Q&{‰ ¢‚Àƒ²DlqÑI\€ÀzÊ6'‰ ¢-Ö‰ ¢-$.@D0eè[ÜÀÀEÜ8Ø‚HQâ0Ù”²sD°èJº`¦A×¢.A§£Ä1º K±–¸{©)aÒK\‚Ÿ(\’¸f@§Ä1ÃL‰ °MÉ 2—t¶ È–¸‘.tZK\ lA4(qA2 »Ì $.À -½$†ZÒ9EûÌ$qAra¢PQâLì4Á”¢¶€&$.€h³í%.Àæ›MlsDl±ÞI\’¡íf’¸ %Øb”‘¸Ø¤ýf”¸ +Ø‚ˆO€ÂDa²Ä qè”’¸ ë„ý¦q`^Ò~3I\­Â~[½žþãwJ¹ðû4þz³ü¨Í]æc÷ó-z».ç^—¿µ\mçZDOô[œ_ôiÑÇåØÍ©L)?¶¿”óa]¿§¶ç×vÔöÉŽ{l.}Œë6ë9ê¹Þ* Où­öû±_/l,GÛ~fõâ cã¯ÕïÿóüéÇÓ>­X¢ÓD‡Í8ß ßÄî±*ïÁî×€,¶º7 ë• •ê&r¥Õ·Ë¨Wʸ¡,ý..1ᶋÝè²å¸”}Î÷y–W[[[/ý3•á6cSÚ®mhöCû|Š°.Sx™¶¥œ[—§T?Ÿ ;ÊÊßË™ë<ð°3º‰¼°öüÕD<[Ð1꫾ ä¨;ÊŠ÷Fr©õ*òFÄî•WLæn ‰£Òl·[t_†Y2¾g}۩Ǭ½6ÆamÓ‘sm"ØvDŠÃ‚Õy:»›Ü9z_Wž´ƒG”°ä@×È-ÍûY†ô6«âÚ¦†ŽŸg”ß×Q[üîªü²nË西ð-ñôeèçÊÆYÒ× ôõÒQÂ$müúŽÕy:ô 6¯0¬¬+N;rÇÃÁ ¿?ž{ÏLÞ@iM‰¥÷Pz¬ÎÓ¡t'¾<ºå`Þ·#² ÂhK—ßaånHb´•§.#tèÛò®áñ%À“ÙõîÁ:Ïæ]s0†KènÌÕ²›@ß íB_YLšw †t/󱤣:u‡Qõ–¢óËÒî «çz}~®²I¦‹—ý<Ÿƒ6 ­Ý°ØR9ê¤yeÇóýñÕ£¥ãø:Vçéðµa=êcæ«»|f\«|˜ôc°c<.ý±¬|¶f±í²`ºìx7Ë?ÀXÇê<¢nGM„ŠG³rcÜw{wy•e¶¼™”wPu©Çê<Rw²a癇G1NË`æíûËšm ²Çwž¾H‚P"ì®ôÇê<"wbÂU÷ž¬þ]W¾å ¯µ}—y_ÁîìÞÁ:φ`·«~‚7Ðr²·®ÛÓ -Ûb½@ð±˜ñY¾^v¥5dø[µ½=ùÁ:O‡ÞØõÈuŸBø7Cè=³æVƨD“¦ojíÝ@ãyǃuž;ñéíWN/•=uE쀄¿ž—ÙùØ-…ˆû¥¾ŽÝ@nº?p°ÎÓ!÷ØÜG¯­s„÷“I÷…}¿¾'Þ¾™?Çi,³Ã¹þ\ëÁ:φ\,~m™¤(èˆðŠ.XÞ“²ë" ÷Œ–Œ<åh_ç,¾yG}~WΘIbu[ÆKZCÆñpº4´Íï—¸çƒ÷Qø®s¬ótH݉U? ±cEË8ñŽQ_Ýsñ2קëD#§µó ÷‚™v­8qX#Æ:·r¨­,_6úF}úº;Ä Ä>Ϭótˆ}ß݇Ïç½?’yÚÝânÔ²æb܃užÍ·cÜ÷î¦nÝ=}äªíjvÝÃ[ßjYýV9]³ÿ<ß{@ï¾#ò«Íd}á<ëÁ:φ¾°§þ-òÜ=™ªè¼Ú)]Ø9oÝÅþUrHæ1éÁ×Á:O‡ä÷å\¿K$D®€q:ÿrjÙ]Žqø8¼þ8fË=Ǿ™¸^Ÿ½b{°ÎÓ¡wçÞÐÑ[îå®ïªVTŒÅï -áVÑf6Ú¼ñ{¶ßyÇhÓ2ÜÏl¦2t?vŸ×J¸¯à`§CéÎ}÷|;z绲ã^¾ Ó©ëoNãî÷|å ²U¹©<%`ïÉë<RãÁoýÿ•ùCžÛõë1û×mn]S¼ç1ñìÁ:O‡ÖÛ±kóÜë%«{5µZçUë -¸‘eož¢µt_ýê®Á´.Wî]3B5®{]׿BÇ‚š’ûÝÈ-Ô™¥êwî˜qæÙVlŽ¬Ÿ<®Ÿ5û7¾…Èɬót¾¿Þó$ úÆGCRùGòRT×we'Ùž[íá(«¯m¥ åj‡´u®e˜àVËƵñ8¿.7¬'êuÉé.>Vçé|;ƵÂ~öêšÐSzv¸Æ¶Å…µ­K-»òzu]αsì<‘I´a¼ú=~WâÆl,ç´×uWL#?fë,ïÁ:φétð›_ì΂ú]¿†ugTFÝñu‹GüÜ&1»Ä”WßÍ¿áç­kÉ[ßY‘®JÑW?DÅ·V£1ž‘0½z‹‡ -V‡ñWÞO¿ìÞXMS×ç=ü«ótøÝÉí -ø]óï`µ;]òVxl:~3æ@lûÑOšs »mŒùöK9ÖVùöPò¡˜'Ú!ü?Yèô˜Ý ­6©xâü—ú·?²<%ehhO×®µô+~¦î0SÒid粞˜ç¥öj>¾’§qtyŒÕËõÑ¢‘^fäÒf96k»æŒŸ¥ý\¥žŸ½ÑcÖ·~-¶–¶CGØ^*^82ìYóa_?½f=:µõ2âŽp¤Ú7\‹õ̪ª/e†oÖõe.ÃGµ•SlÓxzš²¥·é,}°‹Ç=­ i±ƒžšóóu;ÁÒ£¦éu&×픾Wï„Ò¿[v¤´e‡ê}q—%3/¶Ñkm‚r&d¡íù½Í5’;²hœ‰³ÌòÝJ›|å¦ùsí+Ùãb/[úOç ‹íyùä~„Ö2¯l6™Ž)ª;gÄÿxf“fåÖ¼£<îšƼœ"ÍŒ®€ˆµÑkÚtoÓL,½\È*eÕÜQe׃0:¢ÙXçêÀë>ð+JÐkgŒ“’·?ÓU#àŸ–‡.GóôÂ)zlrfÏpnëøn‘eûú‰Ïõ¹Égú×åîD4Io[³éŒ8ÒåH(—Ø¿,¸¬ -y)S1¥¬¾0øÙõuØè¯]†·`6/÷¶yÏà83ww”ûÓ·¸vKiß3—†îº¼ÜÏRúpåÆ_O¡ÓÃýöSººM—Öçšåv¿¶RËkí½é†Þr0¿óŽ,å-ÿË –Žº ”WP©"N¦tñÛ‰^Aæ±s}UÚ“pù¨Z[N·E®Ê.Lã”Ç’Ö×G„³•ÐúúÑøÚªïk5z4¼Sµ-GŸO,Q3èk1õ^MzÓkÜÒd—£ÐLñ¡Ú aoÐñ²½Á^qW¬­bB-Tìt]l\µGÓk?ÚÐ….l¦ó¡ÓÌýµ¬‰Ì·„—NkGé›(¥—GÕ¦êèÕd–•›Ô×’Í!®ùI]N‚ðU›vpfÓ3¯Ñ+8faª@r­\®åÑå$–[ZóÍS±•J½j®#ßMéP·© ¼ »°h1Ò{)ÙQ¶~\‹)@i¾ÆÜqc®„—“wm&vñÛÉG¡´ï\¡|…ZÖn'ÕñÒ¥ôÄ¥ù¦f•Ñ èÿ™*|“ó”Ú5HÑKHúQ? ÍÍ&^—‡Æ:Níb/ˆKÇØ0ËÇK'%”'Òm¥Ûˆ­¤lȘ<æj‘T#µX7Ô51QXŸÆ•\¨ª8]ûå·•¸ÛéÛTØëÉ‚fp÷­Ý.;߸86kz£L³·µË0_…—²°¹QLÐOõ$]lT[íúɘ°RèâN£oE\gkc§òˆÑåcŸdÒŠ{9ýùT–9c½Éø¬¦¤ñ¨KŸÿe~ÄïÜùŸë9æuܨƦqëJÌW2zf:(¦×kVa›6Lh£ý.Ö©—õ m7†ñµëÕU'öZR7ÕMË…¾!›]£-CxF±2ïõM_°çC[;‡ÓXz—Kµ -¼;8L]gû’|fM∊ö%ZgXEðFqÈN˜»e+3 ü$mèÉ~Ç]aé\쀅th»Îõe–7]e¥¬B`ËxÃNó“æ•=‚ÑO¯wû}19Ô@ÒFíMúÆ¥­c$m„'Km9 ëöæØ3ÔÉÛG¨µeÙܳ-ÈéS£‡nuÀ1ʤ©W[ -x~RWe«“Ö…¢ÑdªÊZ‹±Ú–juXJ“8J±Å]úg>Òž{¥ ¢ç^ We©ÝºâµhɶÈs8ÇÞ;ÕK¶X2 ²Àk7©ë%m•]ŸÛ­Ï]WÛ3ë†ë†„™ª^Í_•|ˆÿ™;s! !•ª?%3¢S‡j§£5¶¼–\¥À!ŽÞ,]b©’»ëÜÎ\gHjî„Ÿ µ*jzÁ¼lÂy_Ášª/Ÿ—«e%vé¾RCæÔUf¨¼a¸\÷) ¢vi«H/·ô&zjTZ¾P@™½°JŸ”Ÿÿ×ËñE -endstream -endobj -534 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 533 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -535 0 obj -<< -/Filter /FlateDecode -/Length 4829 ->> -stream -xœÝ]K$¹q¾×¯¨³€Mñ|ÆÈ°×°`Áù`ø0]3-Áè5°òÁß_0“ÌŒ|TuϨVÛÛÓ•|D’?FYL}UøùAãŸHzˆ1O×ÛÏ—_.jNá¿œbv‘ÕÄopW­( N'ÝõgØ0„äLð×· þY\öÉß.ÿñ›ËÿÌóè«RòF»ë_¿²ŠìbÊ ª•ÒVP97„`œT^¸Å”$Ìp ­UfPŠNRÁ 2& -ª3,mת3LÐ$©`†N.*­`‡6J‰:¢ˆ¬tÚ î%уÉ$Q ÓÁH:kH;-éÒà½wQÒlqIiIç`‹ÓDJÒÁ²1H:[¬·JÔÁƒ•t¶´Nt¶]QÔÁå•–tÉ.E"Q— S1ĵÎ(7¸h­’tZØ´¢# -䃠3°Å+—DlqVIga áGÒ'œ 1 -£Õðè¶ÊjIç`‹1DlAçK `°ÀD‰ `à/ØH\à# )/qO -:¸|¢Î¸ve‰:ØB$.€c=$›¢ÄÁÀkŒÄÁÀ¸ À#’†÷-†°pã‚-ð%.“ 1I\œG<†V•tÞ 1€®E]„N‰ B S±–¸±Ô1è%.Áì.I\3 Ó$qAH°CEâLEC4ðÌ%†-p²%.€§ ÖD[à J\MD”é•Ä˜¡K¢©%)Ž3£Ä‘üÀ®¢Ä&ˆ4Á”¢¶€&$.@ÿp°í$.@ðƒ`aŽ¨ƒ-Ö‘Ä17£Ä1£ŒÄÀ&Ç›A₤` <>‰ ’ò»É€Æ¡SJ₤#âMC`\r¼%.HV!ÞDˆ'ŒÛd=Ç›Aâ‚D°ÅÁEu°NSÆ-8 ñ&ØIÒyØbI\¼çx3J\7ñ&’‡©©A‹Ó¢‰©ÓmºÖzn2,f›5œ™|g4«¡‚F_Y/Úzwž·•×50Ã{›×+y”øÐ:Ëÿ]׌X…z1kó²¢8Ñس™>Ðè5 ÷éæWýµÃ&èË€‡1@?ý|ùíïÿðoÿò§? ·>½^þó¡R.âFÑ þ">wŸoøûeÒ‡éšuYø_ñê_ë"û¬¸¶Sz7ýU£ÎMiœ›Ò…1oþœÆ´ô:Ú‘ÓùIgš]ËüYF£=,3¥ŒÏ“ü6–SäYgF½žòØé~(þø_×O?m´_êÛ›Œ‹Í¦úÑ .*›÷Òn;7ÅkkN›«“ä²'ü¥Wèëzv5“m|–¼ÿZ-¬xGšÙßc÷ë¾?GfQÿćf±tþíGíz;Q›!©hx x½çò<züÊ“+,µÅ»¨eïz´|¯9Û´å¼L>³àÓVO£›Õgë ÷Dr[ö–<‡QÝb›ïx±çò<bݾ¿ù4>à{cÞ{¢·ïàÝÜ3*féiŸk¾?†?™çé»ï£þ½­è|gOt´Ž¾ 4ÓýqÿÉÊ|Ë<}[ÜçÊ–£äèYÑЭ]žˆNæy:ôó7öÚIŠR–¾“´Š~?ÖùŠ¿>ž{ÏHÞ@ióá ‘uô4ÊÉ‹9óóÂmÕºëé—õ«àGOk„tÿ³F'ó<[ïÆ“>\\´êÆXÍÑîÍpÄàÛÌÂ+Ù1Ö½Œ×’Îúî;8“Þ²w~›Ê}éòQËß×U¿eѧý<ÖÁC-×O¶Nàâ>D|Å~·ù$¾Næy:|xXöq×Wwõ±qÁ¬òa}|æ>v,¯Ë·…j9Ó·Ô³m· Ó9â•+>ðtäÉuf]Vúf‰äÕõ{I5âêÖÊS7å\–GÖ…ïÚ9)ëµ<ÛšôýOcŸÌóth=ð¿¾c ÿÍpï³gVæîxº'¯O,Ó}Vëç#–{f³•´ ¤v+[g¿ou2ÏÓ!õ`5ìáuFõøÓ$¸òöë[5Û@äOžÌótˆ<÷M–w=|ÏŽÂ÷·¿ÿnüÀêÞÉ<φ`îÕoá ¸ÜõÖ‰jÓ¦SNã³@¼Â÷Ó‘¥Á¡§¿®…ÏñLÝ ¨³‡I Z3ZŸ•yÕ‹£>çç¡H~W‘W$¦ÀÇûnvd¾ŽÑm>ûÆOl9ûqzŠ=,ç+˜rBRŽQÔ¸S•c 5|øµÁ)Ÿ–äÇôùÀÅ4Êóa…~ü­ç}yÕΪÌð™êã“É8?{”Ý9—µürÎegÏìœÊ4EØùVÝøËÅçcªu~UòqІßetõ4p Ò•ÆŽÊ’.ݦsCëëFÎü¨_j*þ"Åï‰à3Âs·öSú_r‡ÿ|щnfñ.*‹£)Müváø &ÌI1 §ª–Å~ëBv ‡!-SZ7¨écª…óQOãGãÊÝp~W²ñIw¤J"›+„ÈéõZÌw¯=‰ù%Hq°Ó•¯¦ð¡†UØ -¤>m+°e<k«:¡[ UW]*öh>4¿6oÂjzßtºëþ’Ö„®;÷„· ªµKé›(åW¯”¢JëUe¶3*U©+)k‡Pí'At»ÂT)šÐ™Ußõ`? -cÕt©¤D·‹˜n*ÍÕž -5UlYSiùfJƒºµáYn£Iø­nÝQ¶|œ‹=÷@ék;ܘ•ðvqTGb¿]\R»ÆðEÒZh»r«8ª†—&åõäñ†f‘ññßüïH®J™yòMR|„»j•ðجây#õr_Y‡¤ÐZÚ®¹t³}{騄ôLº5um±™´k²NR1Iª’Ç\¬+ꪘ)¬´OåÊ^¨Š¨®‰]…òÛLÜìtu¨:U+󺃻«å6±¯ÜùÖ‹CíqÍïc¨öÖr;Ìá-Ol´3ôc©¤‰ª³]«¬6Ã@ -MÜhô-‹Ëh­ìÄï˜+Û “fÜÛå/—<Í!Ô0|b±·š_/Ô{]úúHó~ÿûB×-çÙó¸Q•-LåÖ™¸ŸÉøetñ¤˜Ïe/Â:l:¡¡Žö›XÇ–Ö4ÔyܘŽ¯©eWØKJ]TWm/t­ ;±©Ð5Úv-ãv„·E+VGæ½}Ó&ìñÒ–›ã˜ùMÅ*\ô·ƒËØt¶MÉ×®H\qÒ6¥CK¦Ëˆ‹¾P\v¦fÙÌLd_Imz¶Ÿú®°\Wwѹ|i›ŽÚ4Û]dù.º$pmÑLº¯4ÍìŒ~xx½»ß'“},­Ô^¥o½´ÞK+áÉR›/ýü²Î96^\ok¡Z–íÆž­NN͵ ÙEÑN[¶)ë+¥"›UZ&ŠJ©"«%†b“Ÿ²•fÉwÅV -Õ§hÒ¿ô-íú^©èú^ñ«´\n™ñª·d«äz8‡vwª¥¬¾dd¾Ï]¥ÔRÚ"[×MóºËl{ínƒš!¾y¦ªes«”ñß#cG|É…”ªüä•Û`0Q“ÖØüR_½‡¿—-Pì–Jîγ¿v²^!)k'}M莬UAóë™eÞ HÇŠ®¨òêf9[Râ-Ý×ÂÒP£¹õ+Iòû9óg^ ª—CE~¯e§7ò[ƒ’_Q’@³å“üóÿjé  -endstream -endobj -536 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 535 0 R -/Resources 4 0 R -/Annots [ 537 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -537 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 728.673057 101.433071 709.923057 ] -/BS << -/W 0 ->> -/Dest (cb39-1) ->> -endobj -538 0 obj -<< -/Filter /FlateDecode -/Length 4874 ->> -stream -xœí]M$¹q½×¯¨ó›â÷`è`@‚´†/<°‚Ó5ÓZ½vuðß÷ f’ŒdFfe÷ôì– O£§+#ø >>™™,}UøùVã¿äô”RŽÁ]o?^~º¨)z…%Åꢨ¿ÇoôW­\œ¼Î!ùë$°qŠÙ›®/ü7\òä/—ÿþæò÷u}5SÎÁhýù3©œbŠQyA•¡RÚ -*蓼*è MÌYRÁ q[U„.'/©`†3& ª3¬‹ÚnUf˜¨¤‚:û(¨´‚Ú(%ê`ˆrÎA§ýàô 錞BÊ&‹ºŽFÒY;…è´×’.O!Ÿ$ƒ->+-é‹:Øâ­2’·Ižð6¦$ŒVC£Û*«%‡-ÆD'ê` :_b`‹ŠA¢Œ ‰xIâãä²q ˜d'—\’XÀ¤<¹­D&ûÉ…%°JCg´DVÁï‚ÄVÇ‘+ê`‹ÍNâk` H\`-l1ÎJ\`Á¶h‘¸À‚nÊZâ ¾uJ‰ ,f*‰ ,צà$.° \@)K\`Á¹Ü)qéb¸G‰ ,X×ú`$.° ]ëR¸À‚w­ÓJâÞµ9El€%.;NV'+qïZ­¢Ä¼‹ad$.@¯NÀa¸X™LJJâÞ5I9‰ x×DL¢.NƒVâÞ5Mfnð®ñÊH\àÀ»Æ™ q€2” €wA!^` VâÌ÷Ð+q -› ‰ ¯ëjå!‘̶σžÊу½U§æ¿XJ•¼Å¦£6ԺþOš. míýí_®¾»üîÃ`ee4“ -ÑwˆÉsY¾“_Š9Bq€Uû›ä.2ªaðëu1«¯–ýqNÓÚ§gÙ.|IÎm{^§¡vÓMž–'®í+º;úù@ åÔëVž [º£TAˆé=«d(<ÄíÉ<\¬ê;"¶öØ™žøU×é6(ÔïC®÷Ú>"{eã0Y´<Âä²Is‡pµeôIHq™Ö#ÇÈ=—çáëŽ9×­=VÿŽ¾BÑM@Cè\enkŠäÖˆ <HRÏøœ–á­ùŸØˆk¤ñ¼Áâ2úà óÒP¶{bˆý¼ vÚƒÏÓ±ÁÉ<‡Ô;êG!p¬^´Œ_áuŽRû4ç·~Ëi­¾¡JoNصâÄaŽó¸liù¼°Ó6jÓ -Áï|ï ö ÑìÉ<‡Ø;ñìެΗ/ç}y$ó°KÅ»Q‹Œfó†÷džGC³9Žq¿t5Uz£Æ›¶7C¹œ+â\ïašV#&à׿Un×´òÊóß?Æ5šx¤óõGòúìë¹ôdž‡Cß8õÿ"Ͻf›ê›•ÒÕéd_|½½‡$ó˜ôäŠëdž‡Cò—m¸þS"ù=ö -§·9@ÏålâðÑ<ÿ賧µŽ†qõßzY|JCc" wÑ{.Ïá÷8¦ÝôÖ°jièá3ã°Ò©{Ÿ #G=užYqXE›Ù)󵈿g»y‡v.#À}b#•¡[wë’QjYÜy¥'ó<Jíq¬º‹¸ºÃS=ýÄ2î¥Hªh-Èts\ùkÞeøg¾s`YŒªÜd´KÎÝCê¹<‡Ôã¸öÿïìD¬¼f«_ûHlO\—¹wO±É ·{­o؃=™çáÐz»¶ž{^£¤¬ÁÙÝÍÕ¾jwvÙ[Ï ÑÀˆZóÌÍbІܛÚîÕ¸îyƒŽ5eïwgo¡Ž,*ˆ×8ó†ÑVlŽ¬<îŸ5ûÂ߰'{2ÏÃ!ø8~ùkÜQZí&†¤êùGÞÔK‘ñçxtÜác@ÆW…DiÆ…ÌV -Z6+¤½º–`Þ[-ókãq~_n˜OÔ³Œd§^ÏÅ'ó<’ÝqŒk…õìæžÐCöìpm kY—Zvçus_α:ôPÿ£Š6Œw¿>?¥N»Í»bÓoØå=™çá0}g—w˜iN/~ ë<ΨŒz§¯[<âç2‰Ù%¦ë?êç½{É›±3ÚÂíÎÛ™ÀQñÑl4Æ3¦[šOëx¨`uð—8£ðvÊøeÈjzÁTÃÏ÷ð{.ÏÃá÷ÎÞ®€ß•ŸGžf±k{Ò%ï`…Ǧ¦ÎĶoÛiR»{)Ža·qe)§ì¿Š;`m–o¯FPOŠõD»dotü‚6©ôÄïþôÿö_bû”´CCkºv¯¿Oݦyx­ íäÝ9­'öyª-›¯7ò4z˜ÇY=]÷y{•K™åÚ¬íšw­øHíu•|~î‘γ¾µ+Î>åRË]:ã¹—ä -ÿpÌ,5…îU«eKíèi-UÌiΧn‹Õa17¬ìŒq -ÙÒ !‹ÖW[¨ŽÅN”£?mË –^Ÿ£#¶å”^­ž ¥mvÐÂ;v¨Þòẇçq-ØFGuåLÈBÙ3j{™kv_“Ÿ‰sè ¸¢‹M¾rËü¹¶•ìq±§-íOKß–Ím¹ÚQXÓ<³‘`–ºâœwÞÑZúÇ3›4K·ôaÝ7Üñ»}Ó0¤™Í±–¹­¿T»´iÇWz¹UÒª¹Ê®}0öC³)°¶U¿ëî÷ÕhÖ뾨~‘ÊŸ™æÐO~ôÓÌJ½¬bÿ.©¶›73ÐÚ^Ϩú% msÞÚÆ"Ëóç™OF¦]³¬[žíüù‡óN—k}ÚGñkùhÞûü¶¬}(¥qjù%bݺ¶Í8ÖþÛšãƨ,Z?[^´¢£DNEe'ó¼éu·w Ç|µ)OÉ{ mùf%¹~\ŽéåÆ¢eÿÜ »æ0©LÇD VZæÖD=õ6òn‰3­¸ˆB‰þù^n—Ê©*!ˆöÜËíÔ=úLÊͲmA¨½ë!SŽ+s>ØýR"Üm½³åäz‹ú ž§ósà:Æ[<ï¨ßÀÖ¼Ÿç]2eÉ!dØǦÏK/Ö1á2'ÑI¼Ä°»=ªÏ‚ˆU`g÷‚µm0DçPe:ñe¨´ŒìÒ£ÃÓ4öîë'ó<ÈR´q_"$ïsŸúx[d=õmµ<9OÅëQXÂù3e”—v"ªRÔG¨jˆ‰ËÔÌÂƆªMH! ÊM9{T6Txˆ¨+:¼5ç\;™çÑ•µ>BÔæQìaóµ°[²uËoœ¯EgQ|Ò|êë–EÍ@¶z?-ó6í/7׼ߣ;ÀGGÌÉÇ#Oæy”u³m8BÙkÝ9ÚâäéŒêì=þÂ_½‘`x¬GkÔööÌk9|$©.¾©çò<RÃ!þj'Œ/íÜÄÒ½­í;ýREIu[’x=°k;|6Kž¼3òtoGk7{ôèÌ.2cãÃÓG{œÌópÈL‡ú%¯ r„­n‘ñ¼ïÞYº§ß<®ô<ŽàöØÏy3)ï!æ…çò< -µ:0~½w¦·ä'Bkçɳõ_øma›öm]¥°m%œjðýrzXôè%O'§Ó1`™FÖðv7ÉxÕEÜöp¥«´Z¯óP$}m€MWåH'íí­òÌçy¡ÄüayÙä%öj·õ–yÅÔ•aÝ ¦¨P©¾1ý¹»µ¬—Mõr£"ÏrÓnö°}‰ Ø­4Õ6£ËËÞ”ÿ¦V›á­üºÎìYm†çe­RÚ°éÆŸ.¡œ©Ë©Á*¤IúZkp=ó•N©LÊ:]»MGëëNÎòe ?µTË÷3(:²™Žë,ÝÊ/–ô?•ÿñ¢³›ÈÿåNa9K]q2õ¢‹_.t–¾ÄÎõÛwÄ$\>ªV–Óíîe•Ý@ qÊcJ¬fÕò±ßâ¤%îüÑøÚÊïk6:ãЩšÈ–SþæŠ!r½SëÕ¤1}Ašìrš)´ùÚ„½@ÇÓö{Æ»bmú­P±êºØ¸j¦ók›ëB6Ó¹ë4ëþšÖDÖGÂÛÕÚQú"JéôZTõ®QMf™Q¹I}MÙ:ĵ~D·‹ üVÕ¢:³éY¯ÑY²³0U ¹–.×t‚èvÓ-¥ùÖS±¥J=k®žï¦t¨ÛÔ/Ènì1€H_°Âƈ²õãZLw¥[_«Äpc6ÂÛÅ»6»øåâ£Úw®P¾B -+·‰“êxéÒ˜ª«04«ŒNâ¤ÿgªðMJÌS>h× E§éö«^ Í&^;‰ËCc'y@»ØÓ2wéf¹¿tRBz"Ý–ºyl%e.cò˜«yDR<ÖbÝP×ÄDaÕ?+¹PU#P]ûå—•¸ÛéÛPõªU4ƒ»oåvqhÜùÂűõ¸¦£‘›½­\†ù*¼•‰Íb‚~ª•t±Qm¶ë•1a7 ¤ÐÅF_Š¸ŽÖÆNåüåcdÒŒ{»üp)Óœ±ÞÐÕ`5ôÏ£.}ý¤ù¿»¸ë¿×£eçyܨƦqëJÌg2ú^˜tRLßS…mØ0¡qŒö»X§žÖW4´yÜÆ×®gWØkJÝT7-úîC&6 ºF[æ ¼ ^lÌ—öMŸ°çK[GwŠéPâj.xsp™ºÎö)ùÊŠÄ%íS:´Î°Œ¸à…â’U˜»e+3 ¼’æz²ßñ®°T»`a]Ú®s}šåEWYiK‚Øò ^°Ó¼Ò¼²G0úÍÃë‹û}19Ô@ÒFíMúÂ¥­a$m„'Km¹ ëË6çØr“µÞî¡V–ec϶ §Z¸ÕÇ(“¦žmIày¥®ÊV•Ö‰¢ÑdªÊZ‰±Ú–lÕ-¥ÅIôRl1E—þÀ=íy¯4'zÞ+a“–Ê­3^‹–l €<‡sì­S=e‹%“ > -endobj -540 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 292.623057 101.433071 273.873057 ] -/BS << -/W 0 ->> -/Dest (cb42-1) ->> -endobj -541 0 obj -<< -/Filter /FlateDecode -/Length 4751 ->> -stream -xœí]K丑¾ç¯È³¡ù~‹9°a±Æ¶á=,öP3eè10öÁ¿PŠQ¡Wfu -p²3AŠ!òãÇ`ˆÕUâï…ÿ¢U"Ƽ½Þ~ºü|‘"8‰CŠÉÁ þŸà®JÚ œJ>ºëO$0A„ätð×· þëÛäo—ÿùÕåïÓ<êªEJ^+wýǤ²F„‚tŒ*A%•aTΉ´óŒÊ+KL‰SÁ ¬¶sU€6EÇ©`†Õ:2ª3Œ ÊÌU fè ,§‚*¹À¨”„JKÉê`ˆ´ÖhF§œð¨tÏé´>&X]€NÍéŒ>Xå§KÂ{ï"§³°Å%©8ƒ-NY+9l±&Nça‹ñF²:Ø¢c0œ.ÀÚIŒ. +²:Ø"½Tœ.áR´–Õ%èd q®ÓÒ ‘œN)á€MÃêУÐH>0: [¼t‰ÕÁg¤æt¶Xüq:ð„3!F¦·jêÝFÅélÑ:XV[Ðøh[dð  â%ŽtÂ&m9ÐÑmäX@Ç$l†£œ°>ŽŒTÐiÅ‘°ÅYÏ1Q°Å¢ç²:Øb’å¸ÀhØ‚[´5°­Z8.0 [+“â¸À€o­TžãÂ…™’ãÆ5Ñ[Ž (PJp®wr\`@ºèîãÖ5ÎkŽ h×Øè9.0à]c•ä¸À‚wANV[`Ž ÀŽÂ¨h8.°à]£dà¸À‚wÑ4ÇhUzŽ €¡c”Xð®ŽÒr\`Á»:``uA Ž ,xW;\7rƒwµ“šã ÞÕV{Ž þ‘¬7Þ…8Þ€- XŽ 0ÞC§ Ç8™ÐpK8.pà]•0ü1}l%TLžãÞUÍÇê’P5Îô[Ô²PÞ'Ž x•á¸ÀwAŸ‘ãp¼Àˆª9.pà]e0Rs:ð®ÒIr\àÀ»ðÇ€ƒPp8.Àø&”tY]2ÅÈqï‚–4Ç……Œ`V€¤8. ?”ãÞÅðž8.ðà]éŒá¸Àƒw¥…ÛÃê`‹‰šãø5СÛ2}ΪLÁéÀ»R¡…9xWÊ8.ðà]69.ð:r\¼`ÍqV¤<Ç>Ièàò±:/pì8.Êbu°ÅËqk‘LŠ [ŒÖ [à2p\@$ï›ÀÀ…XØO‘ã0™ˆÉZŽ ‚ó˜¡V9×"Ð5«‹Ð©ÀqAV`(V`.%":=Ç xAîÇ0:e9. ¶ «p\€¡HD ÏœÓ)Ø'›ãxºÐ)ÅqAÔ°Þ ÇQGÌ2½ä¸#4æ’¨jNg%Í3#ÇÑzA®"Çè&˜i‚)YlMp\€ö¡É¶ã¸“L61Íau°Å8ËqAŒš¦›‘ã‚a‹–šã`“æ›ã‚$a <>Ž ’ô‚ÜdŽ @ãÐIÉqARóMm9.@¿¤ùfä¸ ‰ù&¦xL¿MÆÓ|3p\,lqpQXlÓ”˜~ NÂ|ìÄéµ!*8Œ™1ÐE&t ï¾Ì9j2³ìûÇ_.¿~Áà{ýË?/¯¿"eùOÁAR[øWè:¡¹¼o…Ë7 ­Ä ´ƒ…0ð‘˜å –˜a´õHc’&väàWJ;Iü›Ou´â-‡ù/1¶Ú]?ýtùõ¾ûîw¿ÿoÀèÓëåÿCYß~£è;L¿ãxì§ß˜›JWê~¼ö[Ò±¼úò\jÊËtÙ–Û´ÌœŽ;Ï!»_qŒ“8ÝcÌcuW¾óªéµÌÊBÝ»¸P~.ãsSN ]=(ùíÿ]?}wùí§) /õH DIb–f60¼/Ë3þÐÅ”wt_§ß¹YÝK׬¡Â‡š?çw’o2‚Šýq”åstMÞB1Ãd ¶Þ ä›L—._ ãncY¾»æ[W†ªP›t»¬ÏPóó:±Köæ<©)ïušvèVš‡+Eõâugž³3«UÄ.´ÝÙBZi Åœ§C.¯'¾Üb{ÊÊ=d‰ 3¹Äçòåy_ž$õŠIz»N´ðGêÐOH±‰|ÂuäîËs:äÚu®Ýìû ÝïÆÙ_Ý­›½oÍÕ‘¥À¶µÊJ¿…æ}yN‡æu§wæŒnxÜè\³Ò3ìËÜû Þâ:gµGÌWçÏ,û<·­rä±ó–޺ĭKJ·ï¥ÂvëhÜ—çth\÷cgÈÊ5Ü ‹Z†c>ß[¸y¿/¨íGk¤×q»uW§iÙGîx²ðå­³AM¹«•ÏП½Œ×iL¾Éô2¦{j\â‘©ðWwæ92õ†¿úΣö» *~79ŸÁ½ÎùóyßðlÞÏj«ßiéV‡ŽJ»-ÔîËs:Ô®ûªOû†‡ý­f´^óoSd°á,?ý.È_Bkï±d.í®³„°ô>¸òKï”ýõ, ´ ¤îœMíÌs:„n]wŒnºõ9»²GÒÒw”ÁAŸóKn±Œh;¦ïÏŸ‘Ñ!´”Ó¡º T;ŽØ|…ïûëúÜÔeoÛM¾¹oeµÕç´É ø–n}­£v_žÓ¡vÃO•7û2µYùùmŠ¥X,ë Ü*ŠrÏÒu³Ë¾WÕòxäyœ/wæ9ò̺ÚÏog}iÙ½>·À?*9™ŸoFË{ĉ®Û$}؈Ú‚ÿ2D¡Œ ~+â´3Ïé¼?}§܃®#qËÉý­6&¦6tüpGÁðyâÖép7ò-ñô­»Î ”õ½¤Ž èkb—{gD;óœ}ûüÍ­í–›¥ô¾E?â5¾âÇã¹gzòJ«¯GHDBÔÖŠ”yN‡Ò ÿrﬣ½KÙÍ6zd£YQ?ÝÈ]Òùh“–‚o-Õ°òøz£yÎÖºv§»Z]è«ÃlצiÆàëÈB‘lš1hÒ}¾s:Ê“gYoÈ;¿çýÜä³5[Ö0O¦«Mûr/ƒ& å¼~´%så‰wä 3ž÷¸_íÝæ½øÚ—çtøÚð°m㦭µ±¿ã‚XåÝÚxÏutvôÇÃõ˜æ<ømôhÛmÄô0ã]`¬VHîÌs:DmÜ%Žµ,Å7>2û°¶Õ™ãk±wæ9Z7ü¯¯8‡ÿbH8º¶`OdîÀêž!>ѧ{‘óõý=³I$m©MdËi!õ„ªu¤îËs:¤nDÃŽ3ÊÇW“¼cäíãEÍùÀ -ÀyN‡È}O²<µùÈ…¯Ùú{m2îË#Ø?ÝÛ™çlö¾ê;­§çÐrhýÕ;?CµhÓ.Ÿñ,_@os§uïó;󜽾ë~Êï]zÍW]µ¸€ÆâŽ;óœþéÂjèÙS®rM™5tHø%צö«¸i&3pY·^êãpìrx{gžÓ!wß܇Ÿ9ýÚíÖ“qëÂÞƒ_Ÿñ·Wã}=õi687<kÝ™çlÈ ûü×£¯“˜ èÆ ¡‰{Rt}²~žã nÍè‘§í«œ=Ãu8»*§$5yûõügxåExà>÷Î<§CꆯúÂøŽ¹MÉ<«ùÔ†;ìZ{.«Ï³C-iÛqaáÚèš¾ì q±ø³;óœ±Ï­><ŸŸ÷¼'sÚÙâƒïø¸;óœÍë>î³³©Õ÷ª´-¾p×v2 -ÛÚÂKOµL>YN÷ì_îkèûðŠÈ/Ö“^xõ@œugž³¡/~Ý'²OÁsG"U+è\}CÇÚ*ö/{X@òoÚ™çtH~.æú!‘ü±‚†Ó'o͸1~x_mþ¾ÎÆ5ûžL\@oãŸî½c»3ÏéÐ{ì BO=«šQÑsT»*¤å°Œ6½pΣˆß²ýàŠÖ¦±ØšžÚ û±u^ (}`]ÁÎ<§Céƺ‚#OGo<+˽IbXÛÙ?9ýµß€ñï¤ÆG•V ¯ Øz·ÀÎ"”ýº×iþ:FÔ ±ß…ØBîY2?sp ÇézÛ`sh®³õ„ºûgÅþ…§PÒ1ÙyN‡àuÿõÈ›4艂¤\ó<ÔJAÎWeGÞžµ7÷´(d߬涒Ó2›!-•5öýÜjÚ7‚æklïËuã‰|]@r|€‹÷å9’×}\ÃÌgg÷„NÙ²Ý=¶%.Ìçê¸Ô4w^g÷ålSÆÆ™Xú»ßý³+½q(ÓÌóN˜&cúûqg‘àÐüŽvU¥½AR³QIòf’ Üη5É›ƒ\鯦éáÂÚRØÄ+\X›íÂC@üíŸþëþSDz¥Õ÷ÛôÄ/Ãr·;=Ñq~9›R÷Àûf¨üˆ~ÌàÆQEXª;Hßwª# ß)v¸ñJiýý3œkü=œ'æÁt,nÒR~rÂ]?9¿ËnìÎáškpù¾í›ñç‹v“RÃŽ‚ÒG¡4m9|õVÛ‡t¥¬"À«r³©¡¢Õu!ç°QóÏ%Õ¸w³¤íi+¯ûÆÎÍÁ˜þç¡Áº¨dÕ?QÊ}ŸU9ˆ£ÎUüv¡}v-#¶˜gí“pü)˹¬Ê ‹ìf"õ)rü™ÊÉË?µËWCù]ÎFûY™™a {ÁŒ9½š‹éê¥P£˜ö*ŽÂŒG¾˜â|¶ÂzBÛ¦­'¬7ÅÊÈFèæBÙWÅÚf{ímWªÎWa1½­:Õ4N«CÓœkÂÛÅš^úÆJi‡Ô|ª\»Z™iŒJEêrÊÒ ¶´#º]á72ŸÚ¢1‹¾i5Úgî.ŒH¶¤K9#º]ØtãÙ\i©PRÅš5嚯¦T¨›X*ž‘ u4Jm¾ÞôiòÏ©ØS 䶖±Áž ogKO¬â·‹ LjW¹Bº ½Vhšóq”/UJ·ß¡kfíÒEÿß©Â)1ÏðCÙ)Úi¯ÕB¨oñ´’Z¹/¬c¹P6Ô´Mu© -fÛúRQ2é‰tKêRciSe<¤l‘T!©XÔ1QX®ŸÂ•­Pf#P\»å·‰¸ÚéJWØsa^5pwå¼Uì w¾µâPZ\Ѷ‰ÅÞrÞóYx6Û‹ ú1RÅZ–Ѯ֫a …*®4ú6ˆso-ì4¼×~üY;7âÞ.½ Ãœ6N'üðFÑ.À­×¥®ÿBšïðùÛÅ^ÿ3o;wǵ,l¡ ·NÄíHF{ÆÇbÚC> K·i„Ú6´_Å*Ö´.£¡ŒãZ7|mkvY‰=§T…AUѶBWë°ë]­LS3nExëj±82϶M°ï‡&_Š1´aa¶ -íåà0V©Còµ9%Ž(iÒ¡µºÉˆƒö¤8l -LÕ²‰™ȶRõd¿m›ÂPYÍAãFС©:[‡ÙöÔY6\E“.°iÚ[Õš&ö0F?ܽžn÷ÑdŸ{I µé[+-FÒBx¼Ô ‡~zXÆs¥ƒÜyk •s™¦ï™âäÔ®Q] ;9h1ÚHcÍ6&pm¡6Ë&…æ¢Ðd2ËÊC¶ÉÙrµ WÙZ -ŧ¨Ò¿¶5íÚV)•èÚVñ³´tÞ<âoÉȵpõêdMY|ÉÈÈ|›»HmMi²l^¶–GÛks¶â«g*k67Kùÿ=Òwîyº(—*ÿ ‘ëF­¨­ -F›k"w'z±´}z°± •γ;™GHrì¤- Í!¬12(àmÁ„7(Ò¶¢9ÕÛ¸Ç-Ÿ-Iö’ŽUª4'g‘!#ÌøDå0ÐTÑ¥a³sÑÉ“ -R±¡¢a æ4|2üý?Ö-/ -endstream -endobj -542 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 541 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -543 0 obj -<< -/Filter /FlateDecode -/Length 4930 ->> -stream -xœÝ]Kä8r¾ç¯Èó£å#øŒ9XÃà /Ò”Bðø¤ ƒË'êü„k'qAP8P–¨ƒ-Hâ8ÖS²)J\ l±ÆH\ lË q<‚)ixßbx[1> ØOQâ0Ù‘ÄÁyÄcèUIçÍèZÔEèt¸ š°k‰ KM“^âüÄî’Ä0:M„[0U$.ÀR4EÏ\ÒiØ'[âxºÐi-qA4°Þ ÄÑDD™^I\€±$ºZÒ‘â83J\ÉOì*J\€i‚HL)ê` hBâŒÛNâ?6æˆ:ØbI\£áp3J\#l1ÊH\lr¼$.H -¶À㓸 )?±›,qh:¥$.H:"Þ4$qæ%Ç›Qâ‚dâM„x¼MÖs¼$.H[\Q[à4%aÞ‚“o‚$‡-&‘ÄÉ{Ž7£Äp3o£·"Ç›AÜ.ïºf’tà]x¦^â\ Þ„g+Åö -Äëü3Y9âŒâŽõ:¬­â–÷bµ -âžùrŒ$nÀLD–Ñ‹»XV¡ÔJÜ6Pà_øc´á„y÷©ß¢ò€CÀš72aZ¼ûs´dv'Ù¼?ö—?]~û‹ïõOÿsyýMÞ)+?BNMð¯0 LÂp;xßÍ·Îa¶ †=’–+ ©¥AƒË&…åŽÿ]·V–ÂP6¹yŽý9¬SK½`ž\:!F<`xl¦>œç®ÉAºÖZ¶µuQ_oë#ÉÖ‡ó|º­JBÝp0×õŠFÞKü¹ƒßañͶ^ŽˆŒÇ =›é“¶µþă”]2¬{Ì&q‚–þÎä³|¢Á -á;üA46^?AŠˆ|­ÿýÏPébżK_/;óáÑi,Æ]þåòÛúé§øÇ~½üÇß)E·ÐøëÂò÷†ì¯ðüÑ¢YVÓè–ÎêEf–4‹œâ"÷ó_ø¹Lã[Z.§¦ê¯×]ÚlíÊvë:j?”»Sgóë¢/éÔ:Û\ÒlúbIkݶÍ5Íצ+mûËú–†ü`·ó?¯?ÿtùÝÏkHæH*C’4oèhtóŸËòÿ5À›Ò£à]uòëz@j/ 7ŸÓP\ôPæu „ÑD늱˿µœ¸ü1ON[@YÚðe‘wÀ­ Ô=Øi8ý×nBÔÉôcSuDx €Õ D½ìVm ' N›61Äï~ÿ¯ÿüÇß·‘@°…à¿‹eùómîS{w˜¦æܳsZÇ´óRZ5_oäqìÝ®¬.]ë-îée:.eæk³¶‹Ëâ{mŠ¶ºr>7F£‹Y_ÛæÞ=Õ¾ïÎË­$ÊÄÓãe©É7l1>™\ ÍDÅ2õ2_÷½Q[©ºÖǹLu[,ö‹©¾·!œJË÷á­+vp‹(GÝ–ã-©|#d[NÑÒ«>·ëȬº;v¨Öî¿õèÎóY°oˆyEðà„²gĶ2×lˆà~f®1 Ê_lr…SæÏ¥­l…–6·?.c}›çwþëåvd–4¯Ý,0K]aΛ¯¿-ãã:›t—nCmûÝ}S1¤;› B)s[¥viÓN_1k¸2Žjn§²ë>Ç¡Ú件•~×­ßW3Y¯Ç¢ô‹TþÌ2‡ýÆ~š©••íße¡§•}Ööºž}J¿D¡­aÎ[Ú˜eiþõåþÞÉz9j{œ7ã奛•]HËóÉaŸŒ$¾Ûº¤Š’°,ÇkY‘´q;Ñ”’‹A¡ÈahD8k0j”ø&Âñæݹ<φ"gýŠ*ךáo¿ùÀ!¹[þòo˜¯E¶‹îËæëòw‘›ÅÉÔå÷ëò7mÓ~¿õå=›-;(£Ð£N¢ì\ž'AY3Ûë#”ímEm¶^ÔáÞ'&£Év(+£×£R¶è¨ßû6 ¡†ËÃ6__ŸÙ/{Q‘YѦ…r†ýÞì·úµ¬Øt¦®Ü§z=ØEj0#õ\ž§Cj<äC !º¬²vЩ-grÍ°y¼â®a´Ý€Œ b\—÷¶¶/ÛQQf–žg8“ʶ%óºï®íðÙ,yÒÎÌÓ­µÝ·.;¸3³‹ÌTùŸò„+Gwý½“yž ™^rhíÅqÅî‘u„’ÍþÀ}_žï>Yº¥¯³A­ë·çFúæû‘3“rä1‡(<™çéPxÎ_¬#ÓÝ$G·ŽÊK·{Rî~ uä>YãHö\ú@}õî2ƒàeοº«DmVT¤û¶®xRØ©¢;>¤ïýÁ“œÌótÈ»ãCŽ#Ö=½A_7²›•lç³äý×z„haÃ;ÒÊþ»_ý92Cý =x%õY»ƒÞÎGÔfJ*~>ð½çò<zïø•¥_×+¯|F¶‹ZŽø®GË÷Š‘³M{ÞÈËâ3 >mõ4„‡$Žäöì-yîFu;ˆm¾ãyÄžËólˆ ÇþæÓø€y‰Þ¾ƒwóȬX¥§c® æñþdž§Cî±úÿmGçÓ8û{¢» õîìÛA3=÷ŸÌóth>ö{7þèBZ+bfF¿;TF˜g‹üÕ1ß?‹ìek[ãÈÇÊ­³u[;Ÿ”Ï°P:¿»vŒÆsyžÇ~ìYÝSe/‹§ë=æƒÑ=ŒÔÆiGþ÷•¸C¶Aÿð@ðÙµòàíû×¥d¾Ã_=™çÙïø«Ÿ¼j -ªÚ{ü ßðÙ¼µ±Ûe¾ÉÏ¡Ý{íà\ž§CíýÔú†û[Ÿp¯ŠÝÈ{h=–Â¥C;ë.Ö;½©þÍKwîˆÆwÜ»?™çézgßõä}£êœxe&GEi¾›¸á—2bѤÖÏsqþˆbñݧÝ+>Øü(ǧGzw|ýfõ¬ÃøªÐwKcó9)¹ ~'ñûßǨ=—çéP{ÇOU2Zú•ðÝOÏ}ò YÒS0•cÇ{ö~{§bo/Vôú{OËÌÍé†èrœU÷îÆô¾<—çÙ—îܧâÛÍ\Fvuwp\]Ç'Rº»†¦‡÷vËGÄ>²»NHú›ÝQ“œ:ÿ2ÄI[âçï|2ÏÓ!øÎþé|Ê3èzdßru«ßk›*:¾Î(È¿¸{ZŸ?e¾ež¾ í¼ƒ²q–Ü{V4u{—g#¢“yž}çüÍ{÷ÚIŠRv^³~·?ÖùŠ{<÷‘™¼ƒÒæ+š"#êÞÓ('ó<Jïø—g£Žþ.åmŒÈÎã¨(¨ú,æÊÏ k´]ën¤.ëwÁï=­‘ÒãÏÌól£«ÕI'.ݺw8‰š:áóòAËóV6‡ †u/Ëûà‚Žó”£èí—ùp»ä­ù¨åï몯Yôi¿ÌupÄPËõ‹-…8Oœ¡Wlùp; -Àþ°œIÆÂñyÜ|ºTꎸª¨º›äˆÊ IÛC{®z8(iýΊäÃèm¼`%>¿mï%ómnp~Å/´ÌÓ²£‘¬Xž‘6å-§üë²ÚäÕHÍ/ks<¿ñ´¼Dž_^NËýNëU{Á|ùÜŽPõåk“•PóKõÝËßµüòòwgÏêåï´"·áÇq½ø|¡ÎgÑ*Ÿ×ðaõWOã&€ä#ŸahI—aÓ¹£õu'g>âÿךj9õ_ñAÀ|dÖþbIÿkð_.:ÑÄýŸOÅÈ't«,Ž¦\4ñÛ…Oh'ALÔŽÖ¸#fáòQÕ²Hד:Šìš SSZ7©åc;΃_ך?WZÃù]ÉÆ'ç‘*‰l>;n®X"§×[1·^Mzó)÷q²Ë•¯¦ðËÃUØ -¤>m+°e¼+ÖVuB·ª®º&6TìÑ|*jí:ß„Õô¾ët7ü%­ Ýp oTkGé›(å³µKQ¥wª2Û•ªÔ•”u@¨Ž“ º]áªM̪ïFO(…±‰jºTÒ ¢ÛEL·”æêH…š*¶¬©ô|3¥AOIٗݺ#omG7G”-×b>…¥ŽµŠnÌFx»8ª3±‰ß..©]ã -å -ôz¡íÊ­â¨^š”wæO˜šEÆç;òÿ3U¸*eæÉ4UHñ­íªUÂs³Š×ÔË}e’z@Shi»îÒ!TÌöýAHϤ[S×[I».ëä!ó˜¤*y¬Åº¢®Š™ÂJÿT®ì…ªêšØU(¿­ÄÍNW§ªSµ2¯;¸»ZnûÊo½8ÔçÚJ#Ú:à;Ìá-/l4Šú±TÒÄ|0RO­¬6Ã@ -MÜhô-‹Ël­ìÄ_"R>¶I&­¸·ËŸ/y™3Ö>Ä[ÍçÇ÷^—¾þ/Òü„ßÿºÐõ_Ê¥ó:nTe S¹u%îW2þ¶‘xRÌß>R„uÚtBCí7±Ž-­+h¨ë¸1_SË®±—”º2¨®Ú^èZvbS¡k´ízÆoC/VGæ£cÓìùÒ–ÆñI'|Ôm± -}sp›Î¶%ùÚ‰+NÚ–thÉtqÑŠË®ÂÔ,[™É€ì+©]ÏöS?–ëê.:7‚/mÓQ[fû¢‹,·¢KØö}Á¤ûJÓÊÁèwO¯ûb²/3€¥•Ú«ô­—Ö†±´ž,µùÒ¯/ëšcó!!eò¶ªeÙnîÙêä´©Ñ\ Z]ôí¤±e[¸¾R*²U¥e¡¨t™*²Zb(6ù%[é–Üâ(öR¨>E“þ¹ïi×JíD׊ߤårËŠW½%[ ×Ã9´Ö©–²ú’Qù>w•RKi‹l[7­ë.«íµk5C|óLUËæ6)ßÅï™;sžaBJU~òΈŽíˆo,Ô¤ƒ56k›ŠÞƒCˆ¿x#Pì¶JÎs¼w²Ý!){'}MŽ‰¬UAó÷ïÉ&¼A‘î+º¢ÊwóÉÙ’›ôX[CæÔfg(SÞ…ÌDõ2p¨Èß·fÙéü51AiùÓ|Ûhµ}’þ#‚Ó -endstream -endobj -544 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 543 0 R -/Resources 4 0 R -/Annots [ 545 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -545 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 573.873057 101.433071 555.123057 ] -/BS << -/W 0 ->> -/Dest (cb45-1) ->> -endobj -546 0 obj -<< -/Filter /FlateDecode -/Length 4939 ->> -stream -xœí]M$7r½×¯¨óÊå÷`è`` [ /¼ð`}0|è.©%-Zöïû3IF2#«²jzF¥…§ÑÓ•üƒÌ,¦>+ü|£ñ_rzJ)ÇàΗŸO¿œÔ½Â¿’buQÔÆoôg­\œ¼Î!ùóÏ$°qŠÙ›Îï'ü7\òäï§ÿøÝéÖyôÙL9£ýù¯?ÊÙ)¦•T*¥­ ò~ŠÑø ¨‚žÐÄœ%ÌðÑ·UE˜árò’ -f8c’ J0ú¨íV•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚON’Îè)¤l²¨‹Ðéh$µSˆN{-éòBðIÒ9Øâ³Ò’ÎïS’¶8›¢¤ °Å«Dl1)ZIa‹w² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Å´Õå'Ÿ¬U’NëÉ›VÔaD¡“Bt¶峨ƒ-Þ*#é,lqø‘tà ocJÂh54º­²ZÒyØbLt¢¶ ó%0¶¨$ -Àšˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lq¹¢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.†{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]k‘SÔÁXâ°ãdu²8ð®Õ*J\àÀ»FFâôê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„1h%.pà]ãÑiæ﯌ļkœ (øG‰Ñxâåp¶€`%.À|± °É ,‘¸ÀƒwuÆô'Œi°Õ¤Sxð®Nè>Q—'áqaÜÂË“!K\àÁ»˜¨¬Ä¼ úL€ã'̨FâÞÕ3µ¤ïj“•Ä¼‹¸ÁK\8L€Ä˜ß&­]Q—'•S’¸ÀƒwAKFâÌ“J`Q'IK\2žâP‰ xÓ{–¸ €w•·Vâ‚ÞUa¨ƒ-6‰ ×@‡a+Œi«“2` -IÞU=,éÀ»JÅ,qAïÒ´)qAˆ$0Iâ8! -6„ä¦c¸ dB>Q&\{‰ ¢‚Àƒ²DlqÑI\€ÀzÊ6'‰ ¢-Ö‰ ¢-$.@D0eè[\ÀÀE\8Ø‚HQâ0Ù”²sD°ƒW%]0SŠ kQ— ÓQâ‚Ý„©XK\€µÔ”0è%.ÁO.I\3 ÓNₘa †ŠÄ˜Š¦d™K: [dK\€H:­%.H¶ ”¸ ™„UfP`†ÆZ®–tNÑ:3I\\˜(T”¸Ã+M0¥¨ƒ-  ‰ Ð?´Øö`ñƒÅ&–9¢¶Xï$.HÉÐr3I\l1ÊH\lÒz3J\lAÄ'qAVa¢0YâÐ8tJI\uÂzÓ8‰ 0.i½™$.ÈVa½‰%ž0n³ ´ÞŒd[’‡Œˆ¹©A‹Ë¦‰éþ·V–Ãp2¹±. -´1]©¥~0Q)ÝaQŒõ¡9³›|wž›&GDùÜ>hÙÖî*^o÷•dëÝy>ÊVÛº5SÒR-`ki‚v@ÂY> -‹ND1il6Äúʈו[%þÇO'Àbż·Ü.™ùˆC4Ýøó§ŸO¿ÿ—ï¾û§þwøéÓÛé?ÿA)g¾ýFã¯}]ðö›×.7nù›  ]®=~!Ó™¥¥2“f/7ÌåÕr@JãA“RoK}o벩.*Ç,¶Ðµ¤+e/åÞlÇ`Çx]ÚcY9ølÍbÊC 4Ûn¿ý¯ó§ïNø´I‰ÈïÃÕ±,Ÿƒ«/',ï¯À‰ ñH·ZgwîÁÒ-]ùz',œ¿¾ý2¤è.Ç Dc^ á¦æy6Pa¥w„¤~-Ò¹»w?Œl–Eýî Å2*¡žq™bÂëH9–çéâ®ò»Ìõp—Ko¥åïrí—kï×zò°·‹Î¬Óø¸ô¤š{¥÷ò/½ZóVVºjC\§©u“½n°·ÚOuzžîÖÆjçK—‰¶íÔ_Û%µÝ½ ¶Vÿ]–²3Ó½­m/~2;he‚׬’¡EÓu´Ëóth½|¥u/TÓmçÖ˵ç´6dj¡œA¥¼/…Ý?·:Ô:ß -‘;mvRýC›9âükE™î…¥ ëúšß.xµ‡hΛIaÉL¨ºŽÔcyž©×ãº[hÇ?GSëáEŒÓÄô#ÒGîÌ[D6#¡"åkŽ¸†¼¡ýGêÚC¤éñ§‡IJ„[3ýÁ<φHs#&Ü™uÇY¿õZÞÊwÙ!xëkŽ€ýåïÁöþ•òÁ/ì´ÚôeWˆ;ˆ} ž=˜çé{#ž½±ú|qÞçG2O»Z¼µÈhvĸó<šÝõ÷sWSüNçÈYGîÚ®fa×{¸Ü'â#¦Þsç¿UN÷ì_ægèïŠãM<Òùò#y}ì³Ìót軧þ=òÜ=;UWйY)]X;÷Ò¾ÜÞÃ’yLzpÅu0ÏÓ!ùóö\“Hþˆ½ÆémÐs9›8|ôÏ?úlyÆÀ±Ñ0Ž¢¾âÛA/‹OÞ±=˜çéÐ{ãÙб·†U‹ô$ʸҩ{Ÿ #Gñ§B8‡U´™2ïEü-Ûï|âF´iî{6Rº{ÎKF©โƒyž ¥þÆs{ˆ«;<ÕÓ¯ !ã^ꀤŠÖ‚L§6O~í=ÜßòÏbTå&£A–îRåy:¤^kÿÿÎÁN4ÀÊk¶úµÄöÄu™{÷›Üp»wÐúÈ÷~›{°þzìÚzîm’Õ³šZ­÷Uë ¸³ËÞzfˆFÔÒsõ«§Ó:]yvuܪqÝÛ:ÿ jÊÞïÎÞBYª~çàŽgmÅæÈÚÉ#¡áþY³ç[(þ=ÙƒyžÁ×ãב¿Æ¥ÕnR`Hªžxä¡^ŠjûTv’ퟗå|ÅQ¸A”f\Èl¥ e³BÚ«kæ¸Õ2¿6ç÷å†ùD½ÉHê~.>˜çٮǸVXÏnî =eÏ÷Øö¸°–5p©ew^7÷å«Cõ1ªhÃx÷{ü®Ä•ÑXê´Û¼+¦‘1ýÀ.ïÁ%íÐК®Ýk©»6ݦyy± ñäá9­'z­­›¯7ò%-éÅxÔê‹4‹®<Ðìqa+§òÉŽŽ ¥^;^¯À–|ŧ‡œh¹ÉoÍ'yn·cFàÒæTQÇÚ¹ø„·¹ØºjU¾IÍcQÖvµ®‡|©.3RtXšVíˆq -ÙÒ¹K;ìÒwž|š_¡ýý¶œ`é):¤t[Nóµ[Ê -×ìHÚîÙ¡z[Ü…µ¯`aé‚mtXmP΄,”]0œçüÔ¾ÒF*ë…c³ãÒü°ø¢Úä—Ù/u_SKÆ¥ÜÅÕþ™‘dû+öçµöýºß»MWýhǶ:f_©Ó3l,caÄbkËem_k‹­yé“—¥­þJûjÚ·¹~½È-óuJzÖç[,ýâæYÂ\Ÿy6}Çbé—xÕ~ô_ÁnµgñakK8ÞÞ2‹†™P÷*a«ÜÓ©ã[3;êX=ÝX¦®ùÇÈ~ÒË]Ã’VuÎìؼê»(ùNÆðRnÅâ8þjžÙÂýá—¶Ž÷ •yÄ3¬)inac¢òˆºÚ¾¼i_›Kø<2—[îèþ»ŽR:šÞ†²VÉ‘v­ÇmŸgûG=×ýÎH÷§¹ê§lòà§y¾ZçÊÙcLœ½Ÿ‚E r9¸øPL|0ÏCÇi}`0웽!OÉ{ UùV1u˸Ö XÍ°[a“Êt(íPaùíêâz·Dƒ¨Ç\D¡Dÿv+·Kå çD{nåöaÊ}&åæ8·%€½é!S^Žà|°;úZÜl½³åœR:³ñÏÓiÝp]ˆr;oå¦~¿XóqžwÉ`”%‡ðl›>/½pµŽq $r’F€ˆ'ˆ«žöM 6Ó¯¢h»o#O:ö>ÓÓc­elwÚûór`côÈééetäbfç?6®»™äµUÿlÏ<ëáÔÀµ Q$½©Å&Œ^´*ÒᦒÿÈ7%›™…å›$K$I]ÙV–K·šêO½DR¯,ÂñsS¡Püz´@tErÓV¬o‚ÚDÝ¥¾%J-3YdQ™býëÖö¬"ȼ<[Ú°éÆ_N¡Ò«ËAí*¤Iz“Ë9¸‰‚ܘÏt0pRÖéÚmº8ZŸwr–÷ßüÒR-¯ÄQtJ>\º•_,é)þ3²n"ÿ—¥ky}…*bŒ=?ŠßOôú'ˆëëßb.U+Ë鶜®² &ï8å1%&Hµ|ìknš5Õ2ÔÖP~_³Ñ±²NÕD¶¬:W,‘3è­˜Z¯&½ˆé0i²ËUh¦Ð Ô„½@ÇÓö{Æ›bmú­P±êºØ¸j¦#ÛëB6Ó¹ë4ëþšÖDÖׄ—ªµ£ô]”Ò‹'jQÕ»F5™eFå&õ5eë×úI]N‚ðU‹vè̦g½FÇwÏÂTäZº\Ó ¢ËIL·”æ[OÅ–*õ¬¹z¾›Ò¡nSs¼ »°}©Hï´bcDÙúq-¦mÖ×*1ܘðrò®Ä.~?ù(¤ö+”¯ÐãBËÊmâ¤:^º”Njœ?ahV~LÿÏTá›”˜§|ЮAŠ0ïW½›M¼v—‡Æ:Nò€v±§eîÒ16Ìr餄ôDº-uóØJÊ\Æä1Wóˆ¤y¬Åº¡®‰‰ÂªWr¡ªF º.ö Êï+q·Ó·¡J‡/ Ü}+·‹CãÎw.Ž­Ç5Fßìmå2ÌWá¥LlnôS­¤‹j³]¯Œ »a ….î4ú^Äu´6v*G†-û “fÜËé§S™æŒõ†–—a|\}¬ÏCšïðûß'wþ×zš÷<ÕØÂ4n]‰ùLF¯âJÅôj®*lÆ c´ßÅ:õ´¾¢¡ÍãÆ0¾v=»êÄ^SêÆ ºi¹Ðw2±iÐ5Ú2Ïø+ÂËàÅÈ|nßô {¾´µq´\¦sà«U¸àÍÁeê:Û§ä3+W”´OéÐ:Ã2â‚ŠKVaî–­Ì$@òJšëÉ~Ç»ÂR]ì‚…ti»Îõi–]e¥, B`Ë/xÁNóJóÊÁ臇×g÷ûbr¨#€¤Ú›ôK[ÃHÚO–ÚrÖ—mαe¡Yo÷P+˲±g[Ó‡F-Üê‚c”IS϶$ð¼RWe«JëDÑè2Ue­ÄXm -K¶ê–Òâ$z)¶˜¢Kâžö¼Wš=ï•°IKåÖ¯EK¶@žÃ9öÖ©ž²Å’Iž»I]Oi«l[·[×]gÛ3k†ë†„™ªžÍoR>ÄŒ9Ï° !¥ª?egD§þþ LÔNGkly¥©J!€C½•*ºÄ¶JîÎs}ïd»CR÷NxMèŽÉY«¢¦—ÓÊ&¼Ÿâ<‡]W°¢ê‹kålY‰MºÏÂÖP§9µÙ*o',Ï‹” ¢vi©H/#µô&z‡ZTZ~À G¾Ú>)?ÿä8Vp -endstream -endobj -547 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 546 0 R -/Resources 4 0 R -/Annots [ 548 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -548 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 127.623057 101.433071 108.873057 ] -/BS << -/W 0 ->> -/Dest (cb48-1) ->> -endobj -549 0 obj -<< -/Filter /FlateDecode -/Length 4840 ->> -stream -xœÝ]K䶾÷¯èsË|?€À‡1bd‚f{wmãvùûùŠ¢Jî™]k½ƒÙiUñQ"?~,–¨¦¼ -ü|!ñ_0r!zg®·Ÿ.?_Äà­À¿”bv‘Ôßá×Û«ÆVFìõ'h?øh•w×ç þë.ÛäÏ—ïwùÏ<¼ª!F§¤½þòžTF>x/,£ŠP ©•µƒ÷Ê:Fåä€[Œ‘SÁ ë2K•‡&Ë©`†Q*0ª3´ñR/Uf(/ §‚2ZϨ¤€R Áê`ˆ0F+F'íàÐèŽÓ)9¸Udu:é§ÓzpÞH+9]œs6p:[l’ÓYØb¥1‚ÓÁ£ƒçt¶h§«ƒ-*xÍé,uJØÁ­§“r°À¦fuQè$ç‚-NØÈê`‹ÕBq: [ ~8xÂj3Zn-´ät¶(å «ƒ-è|Ž”ƒ-Â;Ž0‚â%Ž”÷ƒ‰Êp$ ‚L0câ`¼× ¨hã¢çx@ ’h[¬qh [ F.«ƒ-:Ž ´‚-HÀqÖ°EÍqÛ …ã º5"JŽ 4øÖé8.Ð \˜)8.Ð`\œá¸@ƒr¥Èqçjp'Ǥ‹áî9.Ð`]mâ¸@ƒvµ Žã ÞÕF -Ž xWkädu°æ¸ì8h4Ǽ«¥ðð.†‘⸽:‡Žã`eP!Ž xWa8.0à]å1°:?` jŽ xWYÜ7sƒw•ŠãÞUF9Ž ”ü#Xo¼ -±¼;[@°`¾‡NiŽ PØ à–p\`Á»2búcÆ4Øj!:Ž ,xWt«‹ƒôhqfÜ¢•é\ä¸À‚w1QiŽ ,xô8.ǘQǼ+5fjNÞ•* -Ž ,x~ƒå¸p$Ž 0¿ Rº¬."†Àqǘ…À$¬Î’ä¸d<ø¡8ð.¦÷Èqï -«5Ǽ+ ÜV[tPÀ¯ÖÓpV¡Àœ¼+$z˜Ów…ð‘ãÞ¥i“ãçI Ç0p€¬8.pÁ Ñ{Çq‹:¸|¬Î ¸¶xe±:Øb¼á¸Žõu x[´Rx[à2p\`ˆÞ7»<€-€ »>0°ž"Ç`²!Dc8.ðÖa=†VåtN Áƒ®Y]€NzŽ ¼7¦bÉqÖRCÀ ç¸?»ÄqÌ€NŽ |„-*`*‚‚gÎé$l“Íq<]è¤ä¸ (Øoã‚ V™Np\€kI45§3‚Ö™ã‚`Ü@®"Ç&Xi‚)YlMp\€þ¡Å¶å¸‹,6±Ìau°E[ÃqAŠ–›ã‚`‹Šã`“Ö›žã‚(` <>Ž ¢p¹É€Æ¡‚ã‚(Ö›Êp\€qIëÍÀqAÔëM,ñ˜qµ£õ¦ç¸ Øbᢰ:ا)2㜄õ&؉Ó9Ø¢¢á¸ :GëÍÀqÜ ¬7áѳ!@ëMφ À»6‚™8xž©ã¸Xo³åÖöÄk=ü3^hÅ؈õZÌ­lÈ@€{1[y6f @¾´Fbƒ0+Ëàب¦U(¥`Ãü Ì,8aŒ>µ!*8xÌ™ÁÓMF ﾬ9j2½’lŒýòÃåË'L¾×þ{ùð»)Ë?BJmà_a¨ˆî¶ð¾%n_Ù*•ó=,»+1Uëc5K‚§0™… þ]—f¶ÂÔïæõ*Üõ²bOÓ9 kìÑL¯l´ÎõÃñÁ`JÕƒ»áR¦9&`•B˜ÉwfyEƒÖpp -Ò計D12àþ -3Kü‡7Œ§ÉŠ1T[.¥¶Ù|P?ÖìFÙ뛟._þù›oþø§¿ío>\þù{$}‚«'¿úBâ³1ã_+¦ë0]»é¯ž§3nžN}ÀçÛ$‹UžÓÙ[“çÃ<¯ø0Ï_lÈ×~ü‹Å_µ#Ì<5vÚZ–vK¹}½íý=e›Õô7Ô:³-ÉfßÜgll óû,ö»ú¹”m»vmîI¹¯þu}óÍåë7sPÂéÊ $ÿòŽey Ž_¾ÅjL[ø-8Êíîæê#_û¦ÇißW9OÁ©ª8i1¸Àa×/m…ÿV±æwêšð_°›ËëÇN×&¬­ æL¾Ç·MšÜï[›¶,ít›Û2³E2Üõ»¡|0Ïé°¬6¹ø4ýjîïy0—Õñ¨6MÕô}ƒ…T‡ìê÷]9œ ¾«»ãò­±˜êÔ˼ îæš³R^ß`ý¶Še}?-Ìs:,ÛM^nçɽ¾Îý«ìØÞÊñ<Ù÷ÍV?/æþ~Œô˜aæwâ·~(Øî|$²¹Ÿ‹z_†ÃtIónî %¬víÅÎ'ýxdð;+«åiÃÕßÕ†B–2-Ç·1|,Ïé0¼í·~kò#ÂäCrXiýÒSGüZòÃiò'nî}™ä<}ìú‹ŸYªÔŠnj>G½5hãè¹/=VVQÓczj·¯¿ýÛ_þñmm·çT6á“°9ã@XïgôcØšêo*ĺ+r¼ì*¤:ä;Þ/FgV6UÈd"E6Ñ{0ÏIÐ[vÊúMÎÌ×Íô¥ÇÑ …J;ý¥_?^³:ú¬'ÝS—æÝôw’+5éóï»éo\¦Íž‡í½”éšH½ [wèí¼ÙJ¯_걬Yy¹Î•FŠ”9_#¢ìXž“ ¬šÔ‘õW™_žÆ.<“[úíöœG€IhÒ Êroõë!7ïµ”æ}‡„Û´ÖË>b·ŽKõ©õ²“n™m’)§›ç©¼…¿þáx]©M庵ŠÔ¨îGê±sÞ©‡Y-,x‡›Ù_b÷‡mΨ®þ‰•ì¼’>jWÐÛøˆEP´‘`½Çòœ ½fǯ쟀t+"ûÔõl³jÙâ»-ŸjœlZóFÞN>3ãÓOƒ{JzïJnÍÞœgwUÇ#ÖTßñ0bæ9b·ýÍÓø€/]óÞ³zûÞÍ=£b–Þls­1÷¯áæ9r·}ÔßZDçÕ8ûS¢;£uwô­ ÙÝ¿î?˜çthÞö{þèŽÁÍÎ1#£å¦ÑÒ?OïóÉù3ËÞ.m«y_¹e´®qkã“ÒË®B¦MîÛh<–çlh´Û~ìY¹…dQϼd=ž`îôîæJí0k²ú»'äG#k)Ý‹æ%™öõ`žÓ!sÇ_}åYûUPuWìñ5|óy?+¨mb£ôb -RÙ=ÔËs:ÔîÄS_êÞío½Â³ªvA§|½ÇÒí*,ùrëA±³e牨}àÙýÁ<§CèNÜõàs£âp{ÈÚrnÓª(ŽOüÒíMOS~·/ú‘=o÷Ø|/ûÝ#­¿»ºŸô]“痢¶úœ&Ú~§¡ŶQ{,ÏÙPëvüTÁ£¥ ÷vœæ]δã4ﲦœíZòßÖ.ö-1=ŸïžT¬ÅbYO ß=ž=†nuÙª½'¤NÞÏ—óœy;Ïéû}èÞß™=ìg×~GJóÔPµëáµhyØ{¢ëfIŸmDmÁéà µñn/ât0Ïé¼?}Oy]÷Ä-gÏ·Ú˜€ŸÛTÐñnDAú}ÁÓÓ²ÿ”ø–xúÖÝçÊo„í<çwMìòèŠè`žÓ¡ï˜¿¹÷¬Ýp«”•½íûc¯øùñÜKFò -J«¯Gh„¨Ý£å9JýŽytÕÑ>¥ìV=²ÂhUäEÙ‹9óóü5jÝôtÇem|o·†—÷ï5:˜çt½{Ї ]«®½“&ÆwÿT~§|êaŠdÓŠÞ-VoÇkNGyÊ{œ“žÞÉT·©Ü·M>Só·u•·,Ú´Oc´`(åºÉ–Ì ”'ŒÈËï0¾ø>x|µO›âëXžÓákÇÃz´›¾º«Ýˆ b•Wëã#÷ÑÙÑ_ç·…J9ôΗšl»M˜N+ÞÆz`wäÁ<§CÔÎSâðXÏR|ãsfŸ—Ù¿‚ªxÿJð`ž³¡*ììZü•yçîÞ}5¾™FÔŽ?ˆzÌs:¤ì<}mâŠÍ7^Ù¡Ýï‹:—åÞ,ἺöYRYq51…¼ëÆÊÛÄ…ïzr’ãµìm æþÝØóœ­;þ×'\Ã4$Ü»·àHdîŽÝ=)>ѧ{Ëýý3³Y$mí{«jdëèûVóœ©;Ñ°‡ãŒâñÝ$¯yûü¢f+ˆ|`àÁ…+[¡ÈF º*¶ÊÏ3qµÓ–¡ -°çÊœlànK¹Uì -w>·b_z\Ò¹^ÅÞRnƒù,¼¥‰Íôb‚~È•T±e¶«•5ÂjH¡Š+>'q­…è¬âü±2nƽ]~¼¤iNi«">8-é˜ÊÖë’×ÿ!Í7øý÷Å\ÿšÏEçq% -[¨Â­3q;“Ñ¡Æá ˜9ÎÂ2l¡2 íW± 5­Íh(ó¸R _›š]TbÏ)eaPY´­ÐÖ6lĪ@WIÝ´ŒÝÞºV,ŽÌKû¦NØã¥Î7‡j4¨•­ÂE{;¸ U§ë”|mŠÄ%­Sº¤oên2â¢-—M…±Z63“ÙVRšžì7mWhª«¹hܺÔUgê4Ûeé.š$pu{Ñld[iœÙÃýððzq¿O&»<HZ¨½HŸ[i¹1’Âã¥:]ºùe™st:G,ÞÚB¥,ÝŒ=]œœ:4ªkaf-Fi¨Ù¦¶­ÔdÙ¬ÒE•þض´m{¥4¢m{Å-ÒR¹yÆ+Þ’.máìë݉š²ø’‘¹6w‘ššRgÙ²n3¯;϶×æ6L5ÄUÏTÔlv‘ò!þ{dìŒyº(—*ÿ¤Èˆ õ$ALÔFz­ô5’»œ‡:ßכЄJîγ;YFHr줭 Ý1­…—ÀÛŠ ÏPÄ}ESÔót#Ÿ- -ö–îk&4TiN,"Céœ÷ßL¢réi©hc:W„@§Q{Á‘¶ Þfá“ôó+þ -endstream -endobj -550 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 549 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -551 0 obj -<< -/Filter /FlateDecode -/Length 4902 ->> -stream -xœí]M$¹q½×¯¨³€Mñût Ã^ÂìƒàÃtͶ£×ÀÊÿ}¿`&ÉHfdVVOϺ í4zº2‚ÁàãcÌÌÒW…Ÿï4þKNO)åÜõöã始š¢WøWR¬.Šúøþª•‹“×9$ý‘6N1{Ãõí‚ÿ†Kžüíò¿ºü÷:¾š)ç`´¿þõR9;Å£ò‚*C¥´TÞO1UК˜³¤‚>:㶪3\N^RÁ gLT fXµÝª2Ì0Q;I3töQPi;´QJÔÁåœ5‚Nû)ÀéAÒ=…”Mu:¤³v -Ñi¯%]žB>I:[|VZÒyØâµsJÒÁgS”t¶Ø`•¨ƒ-&E+é"l1ðNt ¶]IÔÁ”–tÙN>'çD]†N¥˜¶:£ü䓵JÒi=y`ÓŠ:Œ(tRˆ‚ÎÀ– |u°Å[e$…-?’<ámLI­†F·UVK:[Œ‰NÔÁt¾Ä&ÀƒDAñ’Ä&ÆÉeã$0ÉN.¹$±€Iyr1Z‰Lö“ 9J<`•†Îh‰¬‚-Þ‰ ¬†-#WÔÁ›ÄÖÀ$¸ÀZØbœ•¸À‚mÐ"qÝ:•µÄ|ë”X.ÌTX0®MÁI\`A¹€R–¸À‚s-¸Sâ ÒÅpX°®õÁH\`A»Ö¥ qïZ§•Ä¼k-rŠ:ØK\vœ¬NVâÞµZE‰ xÃÈH\€^€Ã q°2™””ļk’r8𮉘D]œ0­Ä¼k<š ÍÜà]ã•‘¸Àw3Aâeÿ(1ï‚B¼À¬Ä˜ï¡3Vâ6„%xð®Î˜þ„1 ¶štÊAâÞÕ Ý'êò¤#<.Œ[xyÒ!d‰ ">“•‰VœIÜ1P ^¹UÜ2Pà^ÌVQÜ3P _Z#‰›0+ËÄ]L«Pj%n(ð/â1·á„y÷‰oQÀ!bÎL‘™1,¢û¶æèÉìN²y쯾üú3&ßëŸÿçòú«²SV4„’Ú!¾Â00Ýí}k4ßÂoŠi°ì¡ÄTmÌÝ, "\¶ÙÈ,Lxôïºý°²¦’±È•P Õ 0–ŠÁ=¥t‡e0V†fÉnëÃyîšäkû e[»x½ÝI’­çùp[#H ‹f„˜¼^ ά¼›úc»¿WLü¸­Á€ Y4ödž6Ù¶‘’©J‡bRgb¢4AKS?³øÁ,h0U‚ˆ0 ͆fPÐ"Ö>Ê­ÿîÓdºX1ïÓ·Kf>b:ÉÑøë§/¿þ§ï¿ÿ‡ü7 ðÓëåO¿Q`¿ß~§éo\ÿõ~¹NËuXÿ³!Yd7üZ9½A:CzÝÓ·4K¤kv¼.ùÇ"’Q¯øœæå+òS^3´#nózµ.K®ÒÎÒÖÑð³O;>ªv¾tÝÆÏC´úí^?}ùý§5øʪi-®R ˆñz.Ë×àõ[À4çC˜~fÝ]»²ºÐvH>ârQû2ç·ž¥SC}C9”Þœ°‹ÃÈ…5LÇ<ÕÖ9;Ø[Ó":v·ã¶Q›VðÝ+ëe¾’m­ŽZ6Â{p¥S¦Gñz2ϳ+ýCĹg<ëaÇ{òužzŽòZþÚÓ•€I÷Àž‘¤8AŽå ¤Eå$_m:SWiO%R»FßóE–ˆð Uv #\Ù4ÛN¥`³J†–JÇh>—çéÐìù·ñÈQOÖ[d”Þ02g­zÜ år>¬ˆs½‡i^X˜€_;üV9¸]#½Îóß?Æ5šx˜óíGòúü;¸ô\ž§Cß õo‘çîrÒ9tŽ\ÙlÉÌNaÄÕv¶zoßÉ<$=¹ê:™çé|'Žý[Dòš@8çô6蹜M>úç}ö2£Ö±Ñ0Ž¢¾Ü“ÑkX|J7Ca" ÷Ð{2ϳ¡×Ç´›ÞV- =|fV:4csÞÚpÔK癇U´™2Eü=ÛÍ´sî © Ýâ¸[×JYÜy¥çò<JcÕ]ÄÕíêé†Ê •$U´dº9®\ÍÐ7õŽ(×Þßjöþèùi©,FUn2Ú%çî!õ\ž§Cêq\ûµ³÷Ϻ¶iZGÖˆBýAÑ+¯Ùê×>Û×eŽÑLó×mh¯>àÕwìÀžÌóth=Ž][Ͻ®QRÖà7†¾¯ZgÀ-öÖ3C40¢Ö¼2D³´!÷¦¶;B5®{]çß cAMÙûÝÙ[¨#‹Ê²á±gÞ1ÚŠÍ‘µ“GBiÝÎflß±'{2ϳ!Øǯ#;J«Ý¤ÀT=?ðÈ»z)2þºÙác@ÆW…DiÆ…ÌV -Z6+¤½º–`>€[-ókãq~(7Ì'êuÉöq.>™çé|ãZa=»9zÊžÎØö¸°–5p©eÇ®›s9ÇêÐCýCŒ*Ú0}|~4Kv›wÅ42¦ß±Ë{2ÏÓaúÎ.ï0Óœ^üÖyœQõN_·xÄÏe³KL9ÖÔÏ{gÉ›±3ÚÂíÎÛ™ÀQñÑl4Æ3¦[š/ëx¨`uð—8£ðvÊøí±®Óôx‰†Ÿïá÷\ž§Ãï½]¿+?<Íb×v›KÞÁ -ML‰mß·Ó¤v÷RÃnãþÊRNÙwÀÚ,ßnŒ¤ž ë‰vÉî!tÎÑ×ڤÒ¿ÿÿþó¿ÿíSÒ ­éÚYË"s³'Lóð0ZÚÉ»sZOìóR[6_oäKZÒ;Šï¨Å7i]µ¾õŒ [9•Ovtô,õÚùózõµ.ϸ%Ígf7³ÇŒvƹ'N¹>p×ÿé7ë– >n Ý[U¦^fkzTÊò©îqÊC«ÛbiXL Ü6¯èÞqKÏ/¶Ù¥'=y8-^A9ú˶œ`éÆzzxs[Nó¼[Ê -‡v`RÞ±Cõ¶ßZû -2æ1/ØFñåLÈBÙ3¢{™k„vt’Ÿ‰èÖ¸¹?›|åùsm+ÙãbO[ÚŸf´Ûóò7Èí(È®i^»óä’>ÔÝ®¥<³I³tKÖ=Å¿ûÑ7 CšÙ\ñk™ÛúuÍg+¾e_é匪¤Ul”2ŒýÐl -¬mÕïºû³Ê§,ש÷s-·åg8óõ%ú,n|6ØÛwå;Üâ§×µ¯ÖÌ·øŒãZqöâÌ5§]1Ï|š?qûËÕGÃ8ð¦û²Èòbsµ% }å+Ê~Ë£ßÜROÅïÏ•]z3µnÖ×¹9kKŒ]vTì X÷®í›yvÍ•kƨ.ÐS*¶Ü¥M"ŸŠêNæy×ÍòÎùf/=íæ=†¿|ØI]9.çôr0iÙÄ>wÂn…9L*ÓC¦C…•ºù†7ÑS=†Þ-ÑØ<ù€‹(”è_ïåv©<“‚hϽÜ>L٣ϤÜü&ÝÄÚ»2åe'λ㡟ëF†»­w¶h¬‚?»Ðm&z‹E¦çŇJËÈÞ!½èt%0ïÔ¹­˜“yžd)Û¸/zÄ}êó/G7óBi³£›þùÜáÄ‚éÒg7Oæyw³Ëóè»þåèæÿqƒÿÑà ËY?Žåsyž ËI²ñÓôëßëÁ0ãìNÈOñéÇyùdž§Ã²=äå_n¶ÑÌ7?¸ÙÁïîáÄΓ¨[]L“¶ŽÖ÷E=•çé0|ÿ¬7ŸùFÒ ³jÙ¬Zä̆0ð¹¢â´ ô6ôc;óÊ`“5sûj]ï»}oç¹ç•š”¦´¼açmçò<ÚîD²rs£zøö½ÚËc=GhÜ<Œ4²Ú8BÎïÀÏG{m_€bôá¡íl6}…}7åìSÔC…GûÙÅ>S›s7Ìó$Xmû9èÃÙ}|`WÀH9ˆõý@VÇùZÔyv«ýç!Í—þHgÙ:_¶øÛcŸ_–¿y›öçÛQü¸<Ê  *bN>Dw2Ï“ ¬›Mï“»¿¶?ó€ÇQ<ÅæèØ{H‚¿ AºÍý¹Àf‡G18|p{©9>ŽÔsyž ©ZéCBüÚ'!Þý„Çøž‘%³ø,çmm_±ã̳÷u(-í+Äص>›%OÞzš…Ú¾û¯Ë=Í?.oX‹½èéíòôª´Ì^ØÖðx7ÉøÕ fûª«^úµŽsP$}µ‚MW”åHo#Ü‹rÌóŠ£ÌyayÓQ=nÎìæ§Å­¦FFõØ™XQ©~ÌþCw‰’–£ôr¼Ÿg¹i·½¨~úT?ž×ªq—Wâ,·(ð£íV~=Úgö¬naÈË\]Ú°éÆŸ.¡¼US—7+«€U¥¡¯^¸7ÑfL¤àŸÞÈi®Ý¦‹£õu'gùŠŸZªå;,½Öš^iZº•_,é*þãEg7‘ÿË}Rå}󪈓©]üv¡ïp‚ع~³Õ1 —ª•…•¶d7lœò˜ÑœZ>ö¼(Ä›?_[Cù}ÍFïtª&²åMˆsÅ‚9ƒÞŠ©õjÒ‹˜¾³!Mv¹ -Í:bnÂ^ ãi{=ã]±¶Š ýV¨Xu]l\µGÓ;~›ëB6Ó¹ë4ëþšÖDÖGÂÛÕÚQú&JéMñµ¨ê]£šÌ2£r“úš²uˆký$ˆnAøªE;tfÓ³^£÷íÎÂTäZº\Ó ¢ÛEL·”æ[OÅ–*õ¬¹z¾›Ò¡N÷ÏíËnì&ÈH_BÃƈ²õãZL÷çµ¾V‰áÆl„·‹wm$vñÛÅG!µï\¡|…ZVn'ÕñÒ¥´ÿ6ÂЬ2z[)ý?S…oRbžòA»)zãp¿ê•ÐØlⵓ¸<4Öq’´‹=-s—Ž±a–ûK'%¤'Òm©›ÇVRæ2&¹šG$ÕÈc-Ö uMLVýÓ¸’ U5Õu±oP~[‰»¾ U¯ZeA3¸ûVn‡Æo\[kz}t³·•Ë0_…·2±¹QLÐOµ’.6ªÍv½2&솺¸Óè[×ÑÚØ©¼OeùØ™4ãÞ.¹”iÎXoèþ±`5}ºôõ‘æ{üþ×Å]ÿ¥¾~wžÇjla·®Ä|&£ïÎI'Åô]:U؆ Çh¿‹uêi}EC›Ça|ízvÕ‰½¦ÔAuÓr¡ï>dbÓ k´ežñÂÛàÅÈ|mßô {¾´µqt?½¸¹Z… Þ\¦®³}J¾²"qEIû”­3,#.x¡¸dænÙÊL$¯¤¹žìw¼+,ÕÅ.XA—¶ë\ŸfyÑUVZÁ’ ¶ü‚ì4¯4¯ìŒ~÷ðúê~_Lu´Q{“¾qikIáÉR[.Ãú²Í9¶ÜJVo÷P+˲±g[Ó‡F-Üê‚c”IS϶$ð¼RWe«JëDÑè2Ue­ÄXm -K¶ê–Òâ$z)¶˜¢KÿÂ=íy¯4'zÞ+a“–Ê­3^‹–l €<‡sì­S=e‹%“ )?ÿQéu% -endstream -endobj -552 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 551 0 R -/Resources 4 0 R -/Annots [ 553 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -553 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 348.873057 101.433071 330.123057 ] -/BS << -/W 0 ->> -/Dest (cb51-1) ->> -endobj -554 0 obj -<< -/Filter /FlateDecode -/Length 4706 ->> -stream -xœí]K䶾÷¯èsË|_@àC‰ƒ1²HA»íÁl€uùûù(‰Q¥W÷ìXd½Ýª"©ùñc±D‰ò*ð÷•Äždç}p–®·O—ÏÑ9#ð¯O19èÕ?àãÌU -r‘ÁzsýÚu.åìõù‚ÿšÃ:ùóåü{šG^U‚UÒ\ùU¤;ç†Q¨„ԌʘÎ9e,£²²Ã%†À©`†q¤h®r0ƒ‚7œ -fRžQy˜¡ÉI=W˜¡œ$N3d0ŽQI;¤‚ÕÁA¤£“¦³¨tËé”ì¬*°:tŠÓiÝYGÒHN:k­ñœŽ`‹ Br:[Œ$œ¶öŽÓYØ¢­¬¶(ï4§s°E¡v£ó°èò¬¶+$§ º3Á±ºðÎÏuJ˜Îx­§“²3À¦fuèQh$ë‚-V˜Àê`‹ÑBq: [œÇÊÂá,GèA]ä%Ž”sE (¯;òä9P>täœæh@Ó‘ Žã-$tJrD l1d9&жz.«ƒ-:ÇZÁ$à¸@kØ¢Hs\ Á¶´p\ A·$‚ä¸@ƒoIHËqáÂLÁqãjo‰ã ʔǜ«Áh.º»ã¸@ƒuµ±Šã ÚÕä-Ǽ«I -Ž ¼«5r²:Øs\vì´ôšãïj)ÇÞE7R U;àÐr\¬tÊ{Áqw•Äqw•ÃÀê\‡>¨9. ð®2¸näï*#ÇÞU¤,ÇJþ¬7Þ…Þ€- XŽ 0ÞC§4Ç(¬SpK8.0à]0ü1}lÕI,Ǽ+=šÕ…N:Ô8ÓoQË´6p\`À»¨4Ǽ úô€ã;Œ¨ŠãÞ•#5§ïJǼ ¿Áp\8tÇß:)]V:¼ç¸À€wAKŠãŒÂð`Vç:Ir\2îüPŽ ,xÃ{à¸À‚w…Ñšã Þ·‡ÕÁíÇðk C·eú4œÕN(0§ï -‰ætà]!\à¸À‚wã°ÉquQ <Ç0°ƒ¬8.°žºàœå¸À\>Vg;Ž œ€À€²Xl!GÀ±î‚žã§`‹VŠã§` \Ž àtAÂûf§°paç[à)r\&ë| â¸À‹ùj•ÓYÕyºfu:é8.pŽ: Å’ãÌ¥:NÏq¾‹îÇ0:I¸[ÐU8.ÀPÔyÏœÓIØ'›ãxºÐIÉqW°Þ Ç^yÌ2­à¸#4æ’¨jNG"Î3=Çžl]EŽ ÐM0ÓS²:Øšà¸í'Û†ãL~0ÙÄ4‡ÕÁmˆãïUœnzŽ ¼‡-J(Ž €Í8ßt[àñq\„í¢›Ìqh:!8.Òc¾©ˆãôË8ßô-0ßÄé·AÛ8ßt‚-. -«ƒ-pšÓoÁI˜o‚8…-*ÇÁÚ8ßôÀÍÀ|=ðq¾éØpx×0§ïÂ3µàóMx¶ÜÜ^€xƒÆ+}œqz6b @½c+2à^ŒVŽoœ#±A˜‰™¥·lÔÃ*”R°aþ…?F3N¢OuˆÊc¦wñ"ºÁ»ÏsŽ’L/$âc¿ütùú=ßëOÿ¹<ý¦”¥? ¡OMð¯Ð T@sxß—¯-Zv½z aà=y¢.#%hq ºE#1üÅ×ù‰å0ü‡þô:Ÿ>`ïO{ [$Ð=~õ±ÁÊâƒY^Ð`ÉF[‹4:¨ÈN€üJA“Ä¿{wPG+†h>¬ÌÇx‰É0)s}÷éòõ¿ûî÷ø+`ôîéòß -xß|%ã·¿Õø z£Ûð›Üøí§iA¶É!Õyߣ"]Œøåü©¼1o¯û8êÆ4døsöå·å¤ãª¼ô;}ÓSu._•e›òoU¤òoók”²¤'‹é¹à÷u$Å7ÿ¼¾ûîòí»)°zosZâˆÀYÅâ¾,`ñK@S×5¶MM š¦ÍMò¡èRõ¯ÁÔ4å°°j›ÑW0?p¾tŽ{ýaȯM•N4próói¿lkUÕMnÕu¸T»eAd¢ÓÏ*ŽW«¸Û™çlÀÃLg y³–tÓšž¡¯jY© ¬ýž´Vs¾žd[ô´¤cçèxÈî§% }-ŠI5çÉPÉéµ|ù^»€^)ÐÀ «è®£w_žÓ¡—Vy3×àÓô;óÊû¦e]AÐßÕhÙ–k4&¤,!7#œA}oÓ’+òafkw#_ó­9‡,h›ô¼×£®Z²7å©Ýš§iÚÁÅY@lq ÷#v_žÓ!vÝÙ<¸Õk™?Ýœ_Ì‘÷ÞÍ‘^1IO\[ÜM£#R(Ä9ñ:r÷å9r×}Ôí¾ÿ¶ÐýbœýšèNhÝì}<š•8>÷ß™çlhVë~ïÌÝð ¸Ñ9#f¥gÄÉë}ÄÞbµEÌ«óg’}˜ÛV8òX¹¹·.p«ª|Ò¸|IÈþ¶Å:÷å9×ýزR WÈŠ-óÈ|ZùíÖ]©%¹áÉÌ—\ˆh!<6Cò2ž*¤í«õé—y‡¿º3Ïé¹á¯¾ð¨ý"¨:x| ßðlÞÏj«Ðh¼Õ«¼Tf µûòœµñÔG}ÃÃþV5Z¯ù·)2؈–~gä/¡µõX—6×™£XwzÜùsïœÅ‹y„ê:†ºs6µ3ÏÙª7â®;oeÏ ![$õÑ!5  úœ_R‹%DÓ˜¾-?!£Ah>OƒêŒWì8bóQ4f¾o¯ëCU—­m?VùÖnh-¡¶øœL¿“â­ÿuÔîËs:Ônø©‚GK=j¹žF< ¾«ŠŸ±U”]oej[m…ÿ’_˜9)•õ¡âØ÷S›¥ß©XŠÅ²ž@}ïiì¹}ºfvÙöª­Û£ÚÜÁ—ûòœy7é›ùí¬¯3-;¹;ØŽ®~¬îªz>¼-o{$ºN+Hz³µWþ¥óÔäìVÄigžÓ!x#~ú€O¹]Gâ–“û[uLÀMmÊèøq@Aÿyàîi_Žù6òô­¹Î ”µ½¤Œ<ú¨Š]îíÌs6ôÑ>së^;q³”Öwâ¢èGü±ÊW|{<÷HO^@iñáu>"jk5ÊÎ<§C醹wÖQߥlf-²{„ÅY‘?ÍÈ‘Òøh“–n¸¬Ž‚o­Ö s|­ÑÎ<§kÝ>œoju¡¯ö³ \›Š3[F–ÉŽ3u†cNó¤FÒëèßÆr?Tù¨ä¯ÏÕÏ“ãˆU§}?œ#Nr¹v´%qBÌãäõ3ž—¸_õÝæ½øÚ—çtøÚð°îm㪭µ±pYåÅÚxÏu4v´ÇýõèªüÖj´í6bºŸñòŒeîX¹3ÏÙe6îûûZ6Æ7Þ2ûrGá5{@{¯íMÆ}|GtogžÓ!xÃW}¡õôZ­¿zág¨mÚå3žá è­î´î}`gžÓ¡wÃw}ÃOù½H¯yÕU‹<ÝqÇyΆF·áŸ.¬†ž=õa -×äYCƒ„_smj»Š;Îdz.kÖK½Ž]@îOdïÌs:ä{÷3§¯Ðf=·.ì%øõ{5~ÐÖS›f‹sîÌs:äîó_¾Qb‚¢ƒ†*î£ë“õó5kFûˆ|ŒÑ>‰Ù3\‡ó׫rÚHR•·]φ·^¸;îsïÌs:¤nøªïß1Õ¢®8ñ•ßØ£vصö\V›g-†zä%]ñš¾ì qáåVwø³;óœ ±þ±Õ‡çóó÷dN;[¼óÝþwgžÓ¡yÝÇ}t6µú^•ºÅîÚNFa*-¼ôTËä“äñžýûaíAü>¼"ò‹õäôÝgÝ™çtè{Ý'²OÁsG"U+è\}CÇÚ*ö/{X@òoÚ™çtH~,æú&‘ü±‚ŠÓ'o͸1~x[uþ¶ÎÆ5ûžLäÑ*ÿtïÛyΆÞpì B=«šPÑrT½*¤æ°„6µPæQÄoÙ~pÅ kÓØèǪ§Vè¾o×JïXW°3ÏéPº±®àÈÓÑÏÊro’è×v¶ON¿ö0ÞðƒPù¨‚ºþ5[ïØ™çtHÝùtÐÿïð+ ëØ®™Ö{=½·k­wÄ`wæ9Z7Þ0¤§-‘Y¯Õ”bWM#àÆ[‡[o Em\W?Y5è§éúµ«mD(ùuOÓü3tŒ¨éc¿ ±…üöôÌÁ§îèm½Í®ºÎÚjîŸeûËS(?Œ{w8ƒÆ6qßÒ¸ûF¨¶É°ÝL²ÒùÆ!iûkükahGüY8]Ü´Wû+œN -.îsa÷í÷ùÓß¾/°ë_œòq¸I¢ã64Ca}'UÜÔ÷j©‹O®¸p{Dy¡I¦f“}EËëBÎ~+äÏ9Õ¸;²ˆ&ÆͲ†­“«ƒ1ýç¾Á?]d .Ö$a'SÑ‹½JEü|‰;Ù#&Œä;ÅQ8þ¹,’)a–ÝÀî mJm:1þ ¹pmÒOeÒÕÄü&e‹; ‘H‰t¿ÇÎpbFˆœVÎÅñêE'GqÜ Øwz<²Ùc“Í–©N[ -,7ÅR‹JhæBQ®ˆ%{dÜ=.W-Âlz]u²jþ”V¹ª9ׄ· N«[é3+{¦¢Rí*‘eº2*d©I)sƒPn'Ft»0¯D*šÐ˜Y_µZÜÉmú$ÊéBJLjn6ÝXšÉ-år*_²†TóÅ”uísÅ3²¾ŽF©‹Û›W}Dèôs*¶±R[ _áFÍ„·‹¡Ü‹øùb“Ú®&A¯êªÜ,ö¢à¥Hã”Ã/tÍ$‹û`Åÿª0Y™§ÿ!)C*îeWŽÊIbßÌâi%Õr›Y‡¸äJÚªº¤s³u}I/˜ô‘tsê\ciUe•Ü…d^$©LS±Ì¨ËâHa©~2WÖB‘ŒÀéŠØd(?OÄÅN“»*ÀžNfew“Ë-b›¹ó¹»Üâ2nL˜íÍåV˜OÂ[?°Q+ŽÐ÷é$E¬DíÊÉ*a1 ¤PÄ…FŸ{qê­™ú·Ò?K'ãFÜÛåçK?Ì)mTÀ«eÜg·öºäõ¿Hó>ÿºÐõÏic·aW"³…ÊÜ:×#YÜ•ÝïÇ]Ú“0w›J¨¨¢ý"–¾¤5 yWªâk*ÙE!ö”Rf•Y[ M©ÃJ¬2t•ÔU͘á­©ÅìÈ<Ú6eÀuº8œFÇ-“U8¨/‡¾èt’¯U‘8ŠIË-©*#êBqX0Ë&fF@Ö'ÉUí§º)t)%Ùä¤i Èt™H²\¢K6Ù1[ª–þŠ=[K.ûEús]Ó¦n•\‰¦n;KËM#^ö–tv€L gW®N””Ù—ôŒÌÖ¹³”JJdósÓôÜi´½V—AÅ[[ì%«&4ThNÌ"CºÓc˜³åC§Š&ôÛ‰ N¤’ õï:£Iø¤ÿûl# f -endstream -endobj -555 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 554 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -556 0 obj -<< -/Filter /FlateDecode -/Length 4245 ->> -stream -xœí\K·¾Ï¯˜s·ù(¾€@‡1bDHA«‘×F° - ç¿Ÿ¯ØM²º»æµ»rÄZ¬vºŠÅâÇ*NwÛ£ÁÏWÿe²SÎ%E:ž>>Ì”‚Á¿ZbuQÕßá7…£5”¦`KÌáø‘>M©—âñé€ÿ6—²øÓ᯿:üs]ÇÝTJt6úžU䧔S2AQ¨ŒõŠ*„)%¢¢ŠvÂKÑT€9Ú«`PÉAS9—U OÉú½ª†K–4`Ø’¢²8¬3FÕˆ!òNÑÙ0E=j:g§˜‹+ª.Ag“ÓtÞO1‘ VÓ•)Ʋ¦#` ÅXM€%X"£é€…|Nš.‹Þ¨:`q9yM—€ÅÁ:EÑe`weU,&«éŠŸBÉDª®@grÊ{3a -Ù{£é¬|Ó«:¬(LRLŠÎK4¡¨:` Þ8Mç…ð£éÀÁ§œ•Õêxu{ã­¦ Àâ\"U,˜|\“¢FXAó’Æ.¥‰Š#\öeÊ ¸\&JÉk4àJ˜(–¤ñ€7:g5"ðXE ¼ÂÊUuÀâ i\à° €ÆÞ‹#¯qÛ¼Eãº%S¬Æ|KÆF <0ÆŒës$ <(®T4.ðà\îÔ¸Àƒt±Ü“ƬëCtxЮ§5.ðà]OÖh\@à]ïQSÕ Xã°ãämöx×[“4. ð.–‘Ó¸³:Á£Æð•Éål4. ð®Ë†4. ð®KØT]š°½ÆÞuCÐvnð® Æi\@à]G.j\G™À?FÀ»  ‡À‚Õ¸û=tÎk\€Æ&‡°Dã‚ÞµÛŸ²¦ÁV“Í%j\À»6cúT]™l‚Å•u +O6Æ¢qAïb£òð.è3k\ŽŸ°£: x×zìÔš¼k]1ð.↠qÜa²4.Àþ6Y×Uue2%g x´ä4.À.<™ &Qui‚#Y @Æ“AªqAïb{/Dð® Þk\Á»†ö¨:`ñÙi\€¸:,[eM#XŒSh:𮱘aMÞ5& "x—·M bbËà„(Øi\3M%¥¨qA,:„|ª.N¸$Ae©:`¡D °žŠ/Yã‚ä€Å;§qArÀ‚AãDS±ˆ¾ÕôXà.j~@À‚HQã0Ù” ‘Æ)Däc°ª¦‹nÊ t­ê2t6i\MØŠ­ÆÈ¥¦ŒE¯q~âpIãÀ€Î’Æ© –ŠÆØŠ¦ì™k: ,²5.@¤ µd,ˆ5.È.#ËŒFãìÐÈ%ajMG†ó̬qA¦8q¨¨q– 2M0¥ªЄÆ˜N¶ƒÆH~l"ÍQuÀâi\³ãt3k\3°8ã4.€or¾™4.(Xñi\PLœ8LÖ¸41›‘o:Ò¸ë’óͬqAñù&RÓ•™3άžPoÀÞªp/v«¤ž/çHê¡`"³ÌQ=5À¶ -¥5ê±ÿ"£'̧Oòˆ*ÂöÌœxË‚Ý÷œcógŠÍçc?ýpøú›ïñ‡UOÊÚE€PKâ+,W0ÝÑ·Åð=Ÿ9Þ˜™¤!>§ƒHe€´ ÅåÐAbûãÇý‡rg証(òYöZ„asÿ`¢Ú:!)F~àxÏï®srB„PŽýƒÕ±SÉ~‡­4¬w×y%¬ÃF’¤pÞn×뼺]e¿·ÚõÆ:¯…Õ÷åR¸hNа 2·ä ÚÍ -»³Ê+.0ƒdÑaD_裂HäA†V…óîb]PÌgöýRÀG|g±Qºp|÷ñðõïß¾ýíïþ ;½{<üí×1Å›¯,ÿuËß0ÿ%Zþ>à7ÏŸ"%z\ëC\ÿ5(—F*¢l“-m´¾±köþ{ûYÇÓêÐiƒ´ë£è7îõ”öX8u\½ís}Ùٮ٦µg6å66Q± -»Rã{Q¦Ùãû†õÍßïÞ¾y·vΚaÝçÏ·Uy‰?7.å’{áfÝ…Nëiú2§•Oª…»oÝv» Zß‹Ü“(ã6ã#чÝôŸ6íhÒ¦ïÓÃ¥¥Xûôûº+šQš¿³º×£o¬ó¥¹´u©¹Û¬ÍÃi±k¾.|¸Ó镹nóëÂÜ&ÓºF“Ûþ/Íó–®wkdë3YÁ]öÛ@÷m³®§mE½/sÞ§{™C×}uc/u;‘ãÔý×wz%ËGïv¾æ¿·Õùâü—.r²æ¿+;oy:Žr5ŒÈø\ÎøJPø¼ùÆãÞ¶Xxßg÷åß]xrÆÿkÙæ—m ‹\øn_­ï NÇå?(\oûß“Fž 8„3Ñ/Eäo ñÓÖå:ß|û§?üåÛ1S‡ÛØYýL㚭ẕ7+¶{<[x.˜Þ·ÑÍ×;ùR–õÄú¤í¢+ t<÷rnŸq Zúõóç>ƒK»¶­Ò[Œè¬4"ÛÍÑÒCõæ<°ð¯;-c¥<‹€Ó¯mP±Ça¯fGóžÿˆ>íz.hÁeN³Çظ +®ÆøTÍó]Ëx|hx¸­elèÓ~Ø·=9ò×Úûv*î‡[ž{GÎçp˜ù/ûÛ¥ùÒìóŠW°ñí ÑòD¥íÙ—G›kß~ÉsËóÇó5ÏÂ)4Ö™?7ÿb<”FÙêsyöÓŠ½,£>Ž:÷­ÌãÀÁ·î´¹çºõúûe~‚ÀdE¹e­»dwo¶¶i~YÛj˜›?¤Öæ¾ÛêuÿÕmżÚ<±>… ¶óÐ1E1¶fw;ì.ùdŽR6óvfžg–¿h'·µSmÏ.ë,ŒùjvXóÙ‚Aú¬‘œ$ùh.Û9kY?Ý^Y±AšÛíÜakm Û ¾’\ÕxêGÍów£Íè綛 î[Ëo^8^nc´yq¼Hš×ãu$ìZóè6&$k¦èëaßâqSLxcg½b0:^$39Ї VZÙ&ƒv–/B‚yÎvXâd -}¿é°Qu7qödü›Ë-:_¦q‘”ÃãµÚX|·KŒ*žkµCœJÀœiµÙQ{úÜ‚0ÕB®ÞFJ!ú3jGa{L‘E˜K¢?%Û¥{e/»:zòõ]þvë–çûš`º˜ôq^«ÍóNðîõ,OÙa•eŠå‚o†²ÌÂÅ>ž‘€‡&—Bvt_ - ÿiÀ¦ŠÐ¼â‘¶X(€2°ÖmæýüÙµë¦?Ýž´P?¯*nǸN¼MUÏâØ"õ9Þ¨Ô-ðÍÍVço¬ü:ÉNÀ‚ö(ãË’};÷%;¦YË<3ÙÙ·ýK²ÓìmlóK²£ÚÉmíô¿šìø(Yï+/8βñ€mF;ßj6{>%C7v¾Œ±—=ËÈb}4}îórZÛJÌÒÿ:±{9¿h¯¸³×‰ÝëÎí­I*npß“äñ¢®doèö$ïÆ:_L’—µmÊ6þLIÞºÃ×Hòö-Þ“äixnOòöµ_žäéú“¼gX^$yÏ°¼Hò^Íò"É;-ÉK9L…ï.÷%y©…ŠÿïI^6Ò€Û$¯‡x-´ÍËcÜ`ÝÚQ$j·|·ãä]âæ§.[X û.¦=¡‰±™Œ{YÚ³og¶Õà"ŽlÎáxqÚ³o»úr¡ÓãÒÖƒÓVöpB„@µì¹f~×ú"kó$ÃÑë>4ÂÇ.Ù”ï¦_»Žûÿ £´W!· ­O¢ïEÖm°à×Ý"Ûب…^ضÝÒÚúàyá[6øÙ9O¶M›­†¶Ç35ë«e>õRËÛf ?€Îί¢KùOuÂ?ê­Q-¬o†0UŒ8>lÅO~3)b„GîF1 —¦·E¶Ç¦Mv:XxpÙ–D²m–#€å |þèB ×­?±JwíŠ×Àܱ"DÍh÷b½™ì"æ·«äÉ/W±Cál¶ Gƒ$ËŽGÅ«b놽Јî†ØQÃcùiÜnº8„º4Óßʺ$¦ó’ðtàcÔ­ôI•ò;ZSͺˆZšÌ P¥KC+Ù'„ú<)¢ÓA~eZÓ„Éìz1küdì,ÌÍ‘¨—+­œ":ÔrKk¡ÏTê¥ò¨Zšå”áê>wÃ+²“Hò¿.J¬ãÛǵ˜s‰>×& ¿q;á騯Ä!~:„¤”ƒ+8­Ü ½h·‹³þ2¤)7Sai6?WÈÿÏTº”™§~°Ô]ŠŸ W£^›]¼6’”ÇÎ:¤YÀRe…¹lJÝg¥½l6Jy&Ý^º[l%&òT<&©Nk±í^×ÅLaÍ>+¥Ð4ènˆCw姕xà }©Ó;‹V¸{èíqìÜù$ũϸå½;ÞÞ®ðù&<Õ¶bvýÜ:bgún7: ¤0ăFŸª¸­ÖÎNüòªöq,2mÇ=~<ÔmÎùàø¨:zËï-‘Q—=þeÞâ÷:þ±=(;ïãÎt¶p[Wb¹“ñ[®òb~ëUöe#„Ží±Í£lhÞÐ÷qç_Ó¨n±·’¶3¨íZ) ÆBìºë:ë…eÂáicÅȼtnƆ=_ú68>zçG¬*\Èáà2[òQ4‰+.:¶thɉŠ¸âRtX²LvHÙI7=ã'9žû"ŒàK?t4¶YÙt“ÕQˆ"½¼ “•–ô³—׋ç}Û -`i§ö.}’Ò>0–vÂÓ¥¾^Æõeßsü‘/Úâêmy±ö|rÆÒ¡­.¤ -iÕ–AvJM¶ê´m. 3MÖ[L S\ª5³ÔgÕJ©ÇCú£´t³Òä¬Ä]Yn·íx=Zò= -ÒÓ%{,™Y”µ»”FIßdû¾iÝwÛmb4€Ä™šQ-ìJ>‹ÿž³væ:›S­Tû©'#6WK`£&›¼óõm¡&Çá£û(‹£’»ë\>;ÙŸ´³Ù¦c"ïM²üÞWÂåºB4ÕÞ «W+FÒ}PŽ†Í™ÝÉP}ñ_}À®õËÄ©"¿çÓsЛùõdÉXý›ºT¿¥“Ç'õç?À‘“Í -endstream -endobj -557 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 556 0 R -/Resources 4 0 R -/Annots [ 558 0 R 559 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -558 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 536.373057 101.433071 517.623057 ] -/BS << -/W 0 ->> -/Dest (cb54-1) ->> -endobj -559 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 299.626036 101.433071 280.876036 ] -/BS << -/W 0 ->> -/Dest (cb55-1) ->> -endobj -560 0 obj -<< -/Filter /FlateDecode -/Length 4816 ->> -stream -xœÝ]ÉŽäHr½ÇWÄy€æøb¾BŒ µ *HA‡Ê¬ÊÙztÐïëI_è4.]Õ•ÈÊ ™/F÷çÏÍŒ R_~~Ðø/’bLÁÓõõçË/5§ðo,±8ÕÂopW­( N'ÝõgØ0„äLð×÷ þëÛâï—ÿøÝå–uôÕ )y£Ýõ¯_YEv1åU‚Ji+¨œB0Î *¯œbJ’ -f¸@†Öª3(E'©`U„–‚¶kU‚&h’T0C'•V°C¥D QDÖ:íA÷’ÎèÁÇd’¨ Ðé`$µƒ¤–tiðÞ»(鶸¤´¤s°Åi"%é` Ù$‡-Ö[%ê`‹‰ÁJº[ F' º[€®(ê`‹òJKºd—"‘¨KЩâZg”\´VI:­lZQ‡…IòAÐØâ•K¢¶8«Œ¤³°…ð#éÀΆ…Õjxu[eµ¤s°Å˜@¢¶`ò%0¶¨à% -À -˜—$0! ” I$`¢(R”XÀÄ4PV¢“Ü@>‰¬ÒÐ-U°Å‘—˜ÀjØBX¹¢¶ØDX[P@âka‹!+qÛÐ"qÝ’JZâ ¾%¥½Ä„ 3•ÄŒk£'‰ ,(PJXp®wJ\`AºXîAâ ֵΉ ,h×RôXð®%­$. 𮵨)ê` ,qØq°:Z‰ ¼kµ -xËÈH\€Y€C/q°2˜•ÄÞ5Q‘ÄÞ5[€¨ Ö •¸€À»Æ᤼kœ2x×ñ(øG‰Þxâdw¶€`%.À~± ±ÁÀ-‘¸ÀwuÂö'¬i°Õ cò8𮎘>Q—0âºÅ(Úû$qïb£²8ð.è3J\Ž°£‰ xW[ìÔ’¼«MR8ð.ü'qà0h8`´tE]TŠQâÞ-‰ ° *‚ID]$-qÈxPðC%.ðà]lïIâÞUÎZ‰ Qç;‰ ‚‚À²Dl¡@À±’MQâ‚``‹5Fâ‚`` \‰ à IÃûÃظˆñÁxŠ€É†˜ˆ$.Î#èJ:o†@×¢.B§ƒÄ!Ѐ­XK\€XjˆXô€àv—$.€Ði’¸ $Ø‚¥"q¶¢!xæ’NÃ8ÙÀÓ…Nk‰ ¢-ð%.ˆ&"ÊôJâìЈ%1Ô’ŽÇ™Qâ‚H~`WQâ,Dš`JQ[@`~8Øv øA°‰0GÔÁëHâ‚ ‡›Qâ‚a‹QFâ`“ãÍ qAR°ŸÄIùÝd‰ @ãÐ)%qAÒñ¦!‰ °.9ÞŒ$«o"ÄÖm²žãÍ qA"Øâ࢈:ا) 뜄xì$éÉ„eAðîKÌQ‹ÙbS~쯾üþ36ßëŸÿ÷òö»1S–4ÀoÉs`íx‰˜„é] cs„¾¶áý‡ „}wTaBª&jâœrc±c³M×òA/ †½l±†+3žµÝ"NИ(ëEKo­òpK5& »0<·¶_¬DÉƣ‡ÖYþw]80q¹ÐoÀ2†#ŽN4öl¥msÿp8”q÷·bÓFhw[CË;jcòU…€–G)› ïH ÄŠ…ÿþÓä4w?å½Ë¡¶.Û}”0ÆF]?ý|ùý?ýôÓ?üã¿¡ÛOo—ÿü;l[FÁÃÿñÏô:ýuz’;;{üý¯ë§ŸvúûªÄ¡É²?î‘€B‹ŠâÜMbÙû „þû• -ëÝÛQeìuÇ{/YsT‹Ñž6BeúŒÞóhͧ3ŽÒA“czœœ·òèÐ×¹©we¾Ìg¹1³>ÿ~™ÿ¦¦ìË·ö¯†¤®Ïž1Ã>#š@1ê$ÊÎÕy”U³ÁÞ{Œø¶DÍûPaÆ<Ò/+~©³Ä€Ñd”åÙŠËY!¿œµ±Ì× ¯S,“ûê÷º±?³Ýö¨;@fA›ÚéØv ÙúÝáí|_ã˜6LO¡×o 5¥Û‘z®Î³!u¼¹a©B´îx%ëÔš3v¹æu‰žwu³í:d¬㚺¯KûF;22òÊÒ³ß ­¤ùüF^÷ͱí>›¹NÚXyºžG9ï×ÆÇèùsáïÈÈ´¦ð!_¾'‚þ™'ë<2í.‡öžZ½Y{(ñëQ_pßç'ä»G K×òe5¨e_Í(¬¾93(Gž³Âsuž…çüÅ23Ô a‹³^ª®$¤öÜ'"«ŸÉ–Koè/÷1º¢/Sý)2Y¶ßûÅm6nÛºàI!IK>¤mýA¾Õ!Ž—§÷‘w®ÎÓ!ïÀ‡ìg2,Gz…¾ffW;ÙÆgÉû/ýÑŠw¤ý#v¿íûsdºþg>4ºóJ~õU»ÞÆGÔæ\ÆòdgC/ø•yß– ¯|îf¶‰Zöø®EË·Š‘ÛÜÔÊy™}fÁ§-žÆF¦ò¦HnËÞ\ç0ª“KÕw<Ø“už±ûþæÓø€yo‰Þ¾ws˪X”§}®%º=†?Yçé»ï£þÖ2:ãìo‰îŒÖÃÕ·æp{ܲÎÓ¡yßï]ù£„´;Ä쬌6;”g˜W‹ëüÕ1ßœ?³ìem[åÈÛÚ-«u‹[Ÿ”¿œ¨ôxSò>ÏÕy64º}?vëza˹<3‰§Ë};³»©Ýy½UʭП½Œ·i'2kc¹íK22ÝþêÉ:O‡ÌõÁ»öCPuSîñ¾á³y?¨mr£üEŽñVÌ#Ôž«ót¨=ȧ~Ô7¼ÙßzÀµ*êt=ÿm¡µ÷X2—vçY²XwzRÿeu®òŽãÚýÉ:O‡Ðƒ¼ëÉëFÅ3èvÈIcvÈL(㫉+~É3–Mjy+sç÷(.ýt¨îïÚX\?:aó­hìïiýÝ2–½m_šzw\-uÕç¤äøÄ_ìÙGí¹:φZà§*-íNhõ~õ6ù®†çY1~–·ïZóßÞ]0…cûkö~}¥b++zíµ§yåŽåºè²_UGWH½¹/OÖy:ä\§ïâÛÕZfvqu°ß]û;Rš«†¦‡·²å=boɮӒ¾ÛŒÚ‚ÿ2ÄA[â/Ÿ ø\§CðAþô>åtÝ’·\\ßjsaiSAÇ— ãﮞ–ûO™o™§_»ó<@Y¿JŽîõMîòlDt²Îӡyt­¤(¥÷¤,ú-þXã+~<÷‘•¼Òê+Â"#êèn”“už ¥áÀ¿<u´W)»h£Göˆ0ŽŠ‚*÷b.ü¼°DAÍZ73ÝqY›?º[#˜Ûï5:Yçéf÷¤»QÝX«c4s31øº³p&›#ú—éXÒqad½eïüun÷¥©Gµ~ÛWù–E[öóÔ ¥]?Û’9ëÄ ycÄóˆóñÕ^m>‹¯suž_Ö½sÜÌÕMsì'\0«t7ò-W¾å -¾ýý[Apº#»w²ÎÓ!øÀW}Ðýô~VÀƒ¿CµiÓ)ŸñY¾ÞæJëÙótè=ð]¿ãoù=dÕ|Ó»7ÐxGÞñd§Cãºq7ôê[®rM‰:$ü-ïMíïâŸ&©Õê~©ï‡c7{Ç7²OÖy6äjuîîÝ_:ýÖín(“n {Á~ÄáÞM ôãÔ—Y“îŸæG•‡Yvüš6~†wjž|^ðzXäžùq†ëGN_u÷ˆçåÑ$¿£ÐÆ+œeJë¿õ¼CóuÊoO¿ós{¾ÊŸï‡ÖZ•',™üŒÄ1K©¦{UF©ñy‰ôµNÝø¼D?•çç%ò³ÜpY?ý–'Óú¹˜a6÷ǬÇõ9§4?ÙvѾ›ûnì¡ùÙå\>‡Õ4þrñãë)ôøŠ"åã  ¿ÃðêiàT~àÇ€ò«-,é…+[¡ÊF »*vÊï qµÓ•¥êTéÌëî®´[žpç{+eÆ5¿‡©Ø[Úm0Ÿ…¯ãÆF½˜¡s'UlTÙíjg°R¨âJ£ï£8¯ÖÂNünÙü±.2iÇ}½üå2ns3 ¿ZÃ[ͯl½.}ý?”ù ¿ÿ}¡ë¿ä÷ØLû¸Q…-LáÖ…¸ÝÉøAÑñ¤˜_J›…eÙ4BC íW±Žµ¬Ëh(û¸1 _S­®*±ç’º0¨.ÚVèê6bS k´mFÆí_»Q,ŽÌGç¦nØÓ¡Í'Çïá7 e«pОcÕÙº%_›&qÄEë–-™¦"ÚFqØt˜ªe 3m'eèÙ~j§Âr_ÍAãFð¡­:ªÛlÛt–gÑ lÛƒ¶aÒm§ia`ôÝËëÃó>›ìó -`i¡ö"}o¥åÄXZO–ÚñÐ/ËžcÇ×läÅ[G¨´e›µg‹“S—Fu-hqÐb´‘ÆZm.àÚN)Ëæ¢Ðd*ËJ‹!ÛäçjyXÆ3Žâ(…âSTé_Ú‘ví¬”Atí¬øUYn7ïxÅ[²År-œC=;UK_2 -2ßÖ.Rª%m–­û¦eßy·½6§AÕ_=SU«¹UÉ»øïžµ3Õé²R©ü3fFt¬o~ÃFM:Xc¯ü¶"½‡¿5PlR%7×ÙϬ3$9wÒö„éÈZ4ð¶aÂ;éXÑ4õ>¿4O®–”xJ· €ª4§V™¡ñ½Üc~sL•ÃÀ¡¢KãÛSUŒüöà ´˜*š¾ü±HŸŒ?ÿHÛ -endstream -endobj -561 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 560 0 R -/Resources 4 0 R -/Annots [ 562 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -562 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb56-1) ->> -endobj -563 0 obj -<< -/Filter /FlateDecode -/Length 4959 ->> -stream -xœÝ]Mä8r½ç¯Èó£å÷°Øƒ5ì1¼ð û`øЕ=µ £ÆÀ¬þû~A‰dˆ -)•ÕÕ½ééBu¥"ø >>)J©¯ -??hü—œžRÊ1¸ëíçË/5E¯ð¯¤X]õŸðýU+'¯sHþú3 lœbö&†ëÛÿ —<ùÛåßsùïu}5SÎÁhýëO¤rvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS€Óƒ¤3z -)›,ê"t:Igí¢Ó^Kº<…|’t¶ø¬´¤ó°Åk甤ƒ-Φ(él±Á*Q[LŠVÒEØbà,èlº’¨ƒ-*(-é²|NΉº J1muFùÉ'k•¤ÓzòÀ¦uQè¤-Aù,ê`‹·ÊH: [~$xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØ_ÿ|ùígL¾×?ÿÏåõ7e§¬þh%µC|…a`2ºÛ#úÖh¾…-ÞÓ`ÙC‰©Ú˜»YD¸l³‘Y˜ðèßuûae+L%c‘+¡@«ÿ`,ƒ{JéË`¬ Í’ÝÖ‡óÜ59"&È×öA˶vñz»“$[Îóá¶FÍ1y½  ÉÈ{‰?¶ó{½ Ãa[¯F,`Æ@=›éƒ¶m¤dê„Ò¡˜ÀÙ˜(MÐÒÔÏL~0ËL• " Hc³¡´ˆµr«Ä÷é2]¬˜÷éÛ%31ÆähüõÓÏ—ßþã?þý?ü+øéõò¿S`¿ßÿ éo\ÿõ~¹NËuXÿ³!Yd7üZ9½A:CzÝÓ·4K¤kv¼.ùÇ"’Q¯øœæå+òS^3´#nózµ.K®ÒÎÒÖÑð³O;>ªv¾tÝÆÏC´úý^?ýxùç5øʪi-®R ˆñz.Ë×àõ[À4çC˜~fÝ]»²ºÐvH>ârQû2ç·ž¥SC}C9”Þœ°‹ÃÈ…5LÇ<ÕÖ9;Ø[Ó":v·ã¶Q›VðÝ+ëe¾’m­ŽZ6Â{p¥»Lâõdžg,Vú‡ˆ rÏxÖÃŽ÷äë:=õ ÝÊkùkOW&ÝOzF’â9–3•7’|µéL]¥=•HíA~ÏY"Âo€V=Ø5ŒpewÐl;•"€Í*Z*£ù\ž§C³;äßÆ#G=Y{ l‘QzÃÈœµêq3”Ëù°"Îõ¦ya5b~íð[åàvô:ÏWü×hâaηÉ;èóïàÒsyž}w‚Ô_#ÏÝå¤sè¹²Ù’™Âˆ«ílõÞ¾!’yHzrÕu2ÏÓ!ùNûkDòš@8çô6蹜M>úç}ö2£Ö±Ñ0Ž¢¾Ü“ÑkX|J‡¡0‘„{è=™çÙÐkŽcÚMo «–†>3+š±9om8ê¥óÌŠÃ*ÚÌN™"þžíæÚ¹Œ÷…T†nqÜ­k¥,î<Òsyž¥Ç±ê.âêöNõô CHå†Ê ’*Z 2ÝW®fèwÆÅ›zG”ˆkïo5{ôü´ƒT£*7í’s÷z.ÏÓ!õ8®ýÚÙû{£®mšÖ‘5¢PP4ÀÊk¶úµÄöÄu™c4ÓüuÚ«xõ;°'ó<Zc×Ös¯k””5ø!†ï«Öpg‹½õÌ Œ¨5¯ Ñ,mȽ©íŽPë^×ù7èXPSö~wöêÈ¢²lxlÄ™wŒ¶bsdíä‘PZ·³ÙdÛwìÉžÌól¶ÇñëÈ_ãŽÒj7)0$UÏ<ò®^ŠŒ?¤nv¸ÆñGáQšq!³•‚–Í -i¯®e˜àVËüÚxœß”æõºƒdû8ŸÌótH>Žq­°žÝÜzÊžî±íqa-kàRËn»nîË9V‡êbTцñÖ÷ÀçG£±Ôi·yWL#cú»¼'ó<¦ïìòó1Íéů‘aÇ•Qïôu‹Gü\&1»Ä”cýGý¼w/y3vF[¸Ýy;ø!*>šÆxFÂtKóe¬þgÞN¿=Öuš/Ñðó=üžËótø½³·+àwå瑧YìÚŽ¹ä¬ðØtÀÔ™Øö};Mjw/Å1ì¶10î¯,å”ýWq¬Íòí`$õ¡XO´Kv†Ð9G_j“JOüáÿòOÿöG¶OI;4´¦k÷Z;Øcš—‡ÛOžÓzb —Úºùz#_Ò’ÞQŒG­¾I³èÊͶr*ŸìèZêµóçõ -lÉW|zʉ;±DziŸšÛÓWgK»Â,Ÿ·Ä»i±ý¶è_™/Xû‹Ý¡ûªúPÑ~/ŸÕY|\ìXü]ê¡<·5:,M ¼M^ÑéqKO×6ùÚªcñÊÑ_¶åKGëéñÍm9Íïn)+ÚiyÇÕÛR|XÛWp1zÁ6zŒ7(gBÊžñÜË\ã³÷ù™‰ÇÍý±Øä+óÌŸk[É{ÚÒþ¥¿[ÿ×~ÚQ0PÓ0\Ì»KúP÷»–þñÌ&ÍÒ-}XwwüîGß4 é5fK{b-s[¿®ùŽe_éå.U7uŒ2ŒýÐl -¬mÕïºûsÊ© ý¶ÓÏ3Óú)Ž~jØYú²øáuí‹5¯-i9nç&ÎKsÚÕò׌~ˆs¹jÁ\-ç57s¼»%Oí_îÏ–f±§r•¯ã&ú+oÆó†+ßÇ“ZÕ°z]×;såz¬W¾c³@ÏšØrØš'>›Ìó®#ï”ùf/=³æ=†°|Ë’à0.Êôr{Ѳ©yî„Ý -s˜T¦GE‡ -+ýòmkêîz3y·Dcóä.¢P¢½—Û¥òdu¢=÷rû0e>“ró£¶-µw=dÊ+KœvÇCßë8ÂÝÖ;[ž¤'©Þáyz†® Qnç½ÜÔoà k>Îó.Œ²äìcÓç¥ëCq‘“¢ö@x'ˆÁw;­Ï‚…Ug÷‚²mÐCï¢ÈôÔ÷PiÙ;¤®æé9ß3‡6Oæy’iã¾èý÷©ÏlÑ7.òêˆún·×ÔÃ7Ôç:e¯ËïÓúv‘»97nˆH7[‘fFeY’|ˆßvÐIxÎH3)Miyrøçò< :»Ùå‰ñ]tþªQužM‚¬6mL{è[—EMóvÖpvÊ–“>Ù§¨‡ -˜4¹ØXÑšsÇ6Oæy¬6&MAauó ‚€‘²Áä—¿ôçkQçÙ¢ÏCš/ý¨zY´/‹ vœýËò7³´/ß;Æú¸ƒk‰l¯ˆ9y8ødž'AY7›Þ“qÀˆ\;Úàç‡Ôƃj{‡¿øƒgÒñç:X&Ì:| e©9>ŽÔsyž ©Yòá×ðz÷ÁµññÉ[¸âõÛÚ¾ÓÕ‘´´¯ðz`×vøl–k8½›ä,ë¦ÙöENW=¼H“]®B3…nò6a/Ðñ´½Àžñ®X[Å„~+T¬º.6®Ú£é]¹Íu¡ ›éÜušuMk"ëÎ#áí‚jí(}¥ôÆõZTõ®QMf™Q¹I}MÙ:ĵ~D·‹ üAÕ¢:³éY¯Ñ{kgaª@r-]®éÑí"¦[Jó­§bK•zÖ\=ßMéP§Shû²;‚éË\ØQ¶~\‹é”[ëk•nÌFx»x×Fb¿]|RûÎÊWèq¡eå6qR/]Su†f•Ñ[?éÿ™*|“ó”Ú5HÑ›{ûU¯„Æf¯Ä塱Ž“< ]ìi™»tŒ ³Ü_:)!=‘nKÝ<¶’2—1yÌÕ<"©Fk±n¨kb¢°êŸÆ•\¨ª¨®‹}ƒòÛJÜíôm¨zÕ* šÁÝ·r»84î|ãâØz\Ók˜›½­\†ù*¼•‰Íb‚~ª•t±Qm¶ë•1a7 ¤ÐÅFߊ¸ŽÖÆNå½$ËÇ>Ȥ÷vùË¥LsÆzC'¸‚Õô­<êÒ×ÿEšñû_wýçúÛy7ª±…iܺ󙌾ƒ&ÓwÒTa6Lh£ý.Ö©§õ m7†ñµëÙU'öšR7ÕMË…¾û‰Mƒ®Ñ–yÆoƒ[ óµ}Ó'ìùÒÖÆщ4zrµ -¼9¸L]gû”|eE⊒ö)ZgXF\ðBqÉ*Ìݲ•™H^Is=ÙïxWXª‹]°0‚.m×¹>Íò¢«¬´‚%Alù/Øi^i^Ù#ýîáõÕý¾˜ê i£ö&}ãÒÖ0’6“¥¶\†õe›sl9ÌUo÷P+˲±g[Ó‡F-Üê‚c”IS϶$ð¼RWe«JëDÑè2Ue­ÄXm -K¶ê–Òâ$z)¶˜¢KÿÂ=íy¯4'zÞ+a“–Ê­3^‹–l €<‡sì­S=e‹%“ > -endobj -565 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 348.873057 101.433071 330.123057 ] -/BS << -/W 0 ->> -/Dest (cb59-1) ->> -endobj -566 0 obj -<< -/Filter /FlateDecode -/Length 4805 ->> -stream -xœí]ËŽ,·‘Ý×WÔÚ€Ò|_ÀÀ xdŒaÁö˜Eß’ZÆ e@ž…Nd%ÉŠ|TUßv^Àºhug‘äáa0’™Ôg…ßhü/’bLÁÓùòóé—“‚SøoL1»Õßã'¸³V§“îü3 lBr&øóÛ ÿë.Ûäo§¿üêô÷y}6CJÞhwþǬ";„‚r‚*A¥´TÎ !ç•×n1%I3\ C·ª3(E'©`U„–‚¶·ª3LÐ$©`†N.*­`‡6J‰:¢ˆ¬tÚ î%уÉ$Q ÓÁH:kH;-éÒà½wQÒlqIiIç`‹ÓDJÒÁ²1H:[¬·JÔÁƒ•t¶´Nt¶]QÔÁå•–tÉ.E"Q— S1Ä[QnpÑZ%é´°iEF:ÉAg`‹W.‰:Øâ¬2’ÎÂÂ?Ižp6Ä(ŒVãÛ*«%ƒ-Æu°/1€ñ°E/QFÐÀ¼$q€ a dH"í@‘¢Ä&¦B° ˜äò)H<`•†Îh‰¬‚-Ž¼ÄVÃÂÈu°Å&’¸ÀØ‚X [ Y‰ ,Ø–€‰ ,è–TÒXð-)í%.° \˜©$.°`\=I\`A¹€R’¸À‚s-¸Sâ ÒÅpX°®uÞH\`A»–¢—¸À‚w-i%qw­ENQ[`‰ ÀŽƒÕÑJ\@à]«U¸€À»FFâôêz‰ €•ÁĨ$. 𮉊$. ð® ˜D]0­ÄÞ5· ÍÜà]㔑¸€À»†Œ—¸@À?JôÀ» '»°+qæ{茕¸… n‰Ä¼«¦?aLƒ­“—¸ÀwuD÷‰º4è€Æ-ZyÐÞ'‰ x••¸ÀwAŸQâpü€ÕH\àÀ»Úb¦–tà]m’’¸Àwá78‰ ‡AøóÛ  +êÒ RŒ8ð.hÉH\€YxPL"ê i‰ @ƃ‚*qïbzOxð®rÖJ\àÁ»Šàöˆ:Øb£‘¸~ t¶Â˜†³:(¦tà]¥ÑÃ’¼«THxð.O›øÀ%.€¼`#q4¤¼Ä>)èàò‰:?àÚI\”%ê` ’¸ŽõlŠ[¬1[à2H\`HÞ·¸<€-€‹¸> ØOQâ0Ù‘ÄÁy¬ÇЪ’Λ!е¨‹Ðé qA4`*Ö`-5D z‰ @ð»KÀ è4I\lÁP‘¸SÑ ‰ ’ò»É€Æ¡SJ₤#Ö›†$.À¸äõf”¸ Y…õ&–x¸MÖóz3H\¶8¸(¢¶ÀiJ¸'a½ v’t¶˜D$ïy½%.€›õ&ßÚV ò¾rËh] Vm+Mòã$¥Çeä:÷å9i•[o•[¸A÷ #Ç~¾ö„u·ã¾ ¶Ÿª‘ÞÄíÞmQ×÷n®£çÉ——Î=ç®V~ƒþìb¼6HòÍæ —)ÝSóÒ2ðYwæ927œÕwžµßUyö»4³¸š#`†ä§ãy? ¨­n'qèÍDmÜj÷å9j×}Õ§}ûý­f¶^ó/sdHþhÑõü·„ÖÞcÉ\ÚÝg¶ýQïCª¿ŒNÕߌPS}ÉÝk©yŽ†P³îî™ÝLësv3d$æÂqUÄ1Rø%÷XF4Méûò32:„–z:T„jÇ=6ß‹ÆÂ÷ý}}nÚ²·í‡&_Ü™·²€ÚêsRrüNâPì:j÷å9j7üT%£¥ ­^O£^¯¾«áŸ©WŒ_ïeê{m…ÿ²_X8)—õ¹áØ—¹ÍõkÛñ^ç“—ò%OàRQ”G[]ö£ªÖ·€<÷_îËs8ä­û¡ýúöf¬ =;¢£×çø±ãG­fëó%_.§Ñ=b·Ò ^ű|Êg½¦7þeˆ–ïÁoEœvæ9‚7â§Oø”{ÐuOÜ2§5SD§Œª0·© ã‡+ -ÆŸWµúX€²ü2/Ëè©7ñ-óô¥»Ï ”õ£¤Î2úl»Ü»"Ú™çhè³ûüÍ$t3+I«”Þw’¢è÷øc¯øõñÜ3#y¥ÕW„#4DF”ÞBé¾<‡C醹wÕÑ>¢ìV=²G„ñª(L?ÝÌ]Òùh³žî¸¬‚×(úBï¶þo„‹f»w÷å9\ïîôábת cu\MàÞ ¯|Y8’Í+úÏ×kIÇyò -#ë-{ç—©ÜÏM>ªùÛºÆu2ÏXmÚ—k¼`(åúÉ–Ì œ'^‘7®xÞã>d|µ›÷âk_žÃákÃÃz´›¾º«ýÌ*ïÖÇ{£¿ïÇ6åàok&Û.¦Ç¯ÌX¤îg¬yŽ†(ÚxJëYŽo|Íìóœý ¨²÷¯wæ9ªÖ½œ5ïÜÝ»ïÆ7ÓˆÚð‡é¨×Î<‡CÊÆÓ×&®øÅ|ã.î¹´/jO\vMëWœW×>K*+®&¦wÝ8½`ÛŽ¸ð]ONr¼v5·€ÖÆ—Ú»W`gžÃ¡uÃÿúÀ5üC½{ öDæîØÝ3Æ'út/êvDÿÌlI“‘êšÈïþväUë;±÷å9RÝF4ìá8£z|7É;FÞ¾¾¨Ù"ظ3Ïá¹á¾Çnä{ž(|ä蟵}•qß?ÝÛ™çpÞðUßi?½„–»ö_-<·÷¾^ÞÁÇÛ¸†ðô6OZ÷¾°3Ïáлá»~ä#…ÐÃîZ”Ñèˆ;îÌs44ú ÿta7ôÍ[®rMY5tHøWîMíwqóJfä²n¿Ô×ñ ȵ÷Çvæ9r÷=Á}ø…ÓFh·ŸLÚöüúŒ¿½?èÛ©O³Å¹ÄZwæ9r÷ù¯%’Ô¿'ÕqA»O@Úý,Å=9º>Û?/qP·gtŒÈsŒöUݼÃuwþvWNIjòöûù?ömĤ>ðœ{gžÃ!uÃW}|ÇÜŠ¶áÄÞÕ|ê Ãv­½—ÕçY‹¡–´í¼°po|O_v…(#6<àÏîÌs4ĆçvÏÏ{Þ“9ìjñÁo„|Üy‡æu÷ÙÕÔêGUÚ_xj;›…©öðÒ[-³Ÿ,çgö/×½üûî‘_l$/ ï8ëÎ<‡CßǾ‘}ž»'Rµ‚ÎÕ/t¬íbÿ"±‡$?ð=¡y‡äçb®_%’ß#VÐpúì«ÁïÛ¡Íß·Ù´Ç`ß›‰ ÃjüÓ½Olwæ9zã}_zê]ÕŒŠž£Ú]!-‡e´™…2ïEü–íwî¸mšFýÐŒÔÝíóZ@éû -væ9J7öÜóvôÆ»²Ò—$ƽý›ÓýŒ¯øÉAl|TEÃø™€­o ìÌs8¤î|;èßOä†ml×ÍÛH¼Ÿ§¾Ûµ€Öb°;ó­_²ó(Èl÷jj5«æp!Ê^z¦ózÔò¾úÙ®Á8O7î]í#BÙ¯{ç¿AÇ„š1ö»[È#KåwîqæÑ6Úšûl=¡îùY±á-”ô@Lvgž£!8­û¯÷|Iƒßø(HÊ-ßñÈC½Ôí®ì(Û³öåž…◕­­ì´Ü¬–êšF€ynµíAó=¶ÏåºùD½. ÙÞÏÅ;óÉë>®Ö³7Ï„Ù³Ý3¶%.Ìeu\j›'¯7Ï娩cã‹L¢ ýÓïþ]‰•Ñ8ÖioóΘFÆôQÞy‡éo~5; ò»~ë­Ÿ‘u£¯‹?â®e2³KLyónþJ?/=K^zgEz*Åc¸Ÿ \ç¯ÍF½?#azö=¸Æ±Úµ—8£´÷)ã·Ù«ùP5vÞÂï¾<‡ÃïFlWÀï¬WÞÁ*;]ÒVZß´3f‡oûÞ_š£»e ôñ•æK9Vfù麟?‚Cã;>÷”OïHÍQ"F›I6Ps{ðH>¾ãÌÿzXø ŽÄÀ‡þÚx†“I)ð99 ƒoÿðÇßÿùÍÔÌùûñÚÜìã«Ã—ëðâëü¹J­¯"Ç4ãÐèÈOSèØUܼ<}Ö.äßסÊÝsíŠq+ -§õן±¬éﱜ˜á4ÕÇÛV8?/›ÂU?+ßMu7öŒe¸æ\¾‡›nüåäÇóžôxæŸòqІ>{˜ÍC:óSÓ¹ÎݦdžÖç…œãQÊ¿”TÓéÊŠ\äö®G/7Sú_Æÿù¤ Üþ< ¯'¡ªQM¾¨â·Ÿ„K‚˜h0;Å,œþT¥,Ò9a‘]À‹aH}JëòI5ÌU8ýi\¾Îïr6>¡ˆTNdÇ3z® BäôúVÌw¯=‰ù4á8ØéÊSœÏ6CX ¤6m-°fÜk«¡»ª¦º*6”íÑ|ú\i:_…Åô¶étÓý9­ Mw® /'Tk{é›(å3LsQ¹u*2Û•ŠÔ唥C¨ô“ ºœá7*MèÌ¢ozO‚» -c•t)§D—“˜n*Í•ž -%U¬YSnùjJ…º¥áÙØF“4ðñèÍQ6ÿ9{îÜ×*6¸17ÂËÉQ‰UüvrAHí*W(—¡× -mSnGUñR¥üÁë_šYÆçhñÿ¯Tኔ™güCSŸ…W¯j%<6‹xÞH­ÜÖ!©4…š¶i.BÁlÛ^:*!=“nI]Zl&mš¬‘‡”Íc’*ä1낺"f -ËíS¸²ªlª«bW ü6W;]ª{®Ìëî®”[žpç[+¥Ç5lXì-å6˜ÏÂË8±Q/fèÇ\IUf»ZY#¬†ª¸ÒèÛ(Σµ°ÓøeúéÏ:Ȥ÷rúÛiœæŒu&áo5ŸÓÛz]úüO¤ù?ÿ{¢óçƒá®ó¸Q…-LáÖ™¸ÉøT÷¸Ş¼ga6ÐPCûU¬cMë2ÊÄŒkž. -!¥ÊÿÆȈŽõ(ULÔ¤ƒ5öœØ݉ރCˆ8›PÉÝyÖc'·’;ikBw d­ -x[0á Š´­hŠz›N¡•³%%ÞÒ} „†*Í©›ÈìvDå2ðRÑ¥ñ8r1È“J‹¡¢ñÛ¼q>ÿý?íh -endstream -endobj -567 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 566 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -568 0 obj -<< -/Filter /FlateDecode -/Length 5029 ->> -stream -xœÝ]K$9n¾ç¯Èó«÷XÌÁÀö^xáÁú`øÐݵ £ÆÀ¬þûþ¨I ãQÕµ´§QS¤E}"©(¥¾+üûAãÉé)¥ƒ»?~¹ýzSSô -ÿ•«‡Âþ#~¢¿kåâäuÉß!‚SÌÞÄp½áÃ#/þzû·ßÜþk]GßÍ”s0Úßÿú•XÎN1Ũ¼ÀÊ`)m–÷SŒÆô„!æ,± †Î¸-+B —“—XÓV‚ÖEm·¬ 1LÔNbA }XZAm”yD9gÀÓ~ -PzxFO!e“E^OG#ñ¬BtÚk‰—§‚OÏAŸ•–x²x휒xÅÙ%^€,6X%ò ‹IÑJ¼Y ´“^‚,°®$ò ‹ -JK¼l'Ÿ“s"/ƒ§RL[žQ~òÉZ%ñ´ž ² -¹ÉÆÁSJ‚¬âMã$,Àº¤x3IX­B¼‰OX·ÙŠ7£„ÙAEäA8MYX·À$Ä›@'‰ ‹ÉN‚Å›I¸ˆ7áÑ‹)DñfÓÀ]ŸL¸ Ï4HX€Ä›ðl¥Ø^x}„&3EœIÌ(@¯ÇÞ*¦ °»Us -àK1’˜4€˜ˆ,S³ØVÁÔJL(à/ü1·Á„9ûÄSTæ±g¦HƒÌXÞ}‹9z1»SlÎýõÏ·ß~Âæ{ÿóß^~S2eõŸ†ñ[(°¶ð)í‚ù^M¥=‡ζ¡ ˆ$„€ï©CBÄÜ…Ô€Å%éFBb+Æv›ïíƒ^‹ ‰If g¦ŒÌ­úE¨ 1W6ˆ²¾¹Î‡Ë -‡¾°‡÷Æû…'+JyZúT>KÿÝ·NÄ6 -ÃOŸ\€ÉÒ^­ôBoMúï~¾Á=6ªåƒèÌV¾ÃÑÐ&ÝþåöÛßÿá_þéOÀìýürû÷ßý~°z´ÿñý»å3Ñ2ù¥€Ÿ4Ó½^~‡åw¥{¥\šËb)(÷y¦COåyC_ÊP (kä¬Ýåw¡™uߥ^ØÒ©}’£ÒÝËÒ¯?WYݧ¥]ª÷÷Ÿº¤¿ÌõG*³–u…®mèC+âqÕ<ñC½—™WÊ?–¡15ñY»U•ê3¦Ä±>"S‡êj/²PÇó@Ã2ÂÀ‡FpRÀulËtú2ŽE}hGÙ¶,ùG%·i§©ß-m…C9BÜ“Cõ±}Õñó ©e£\lPø&´]Ì:ÏõË<¼,m}âæÊL5.eT™¨ïRV©š8}v‘™î2.>OÄ›m£—µÛÓÖ–VuV¶t¨ç4ê¢Ô§¶¿°þâò"o•# ãf[l•¯Øc-ÏlgĘ3|‘±…°ö£u}¨§ vÍ ©¯¸/ËáÈGŒ™ÚYµ+a–íØÞì{±«Øew×ä‘Ž¬²ƒŽfÿ`uüýÏÅ‘µ=8 ƒ­âÈÒ9ˆIÚxx?i*F¨¸óûÖ:ï -(_w2”©Ãx'› e6®ªœ”Û8À®ö:«¦=j뛼Üt£Šü?ýô÷ÿð¯Ì.[`õ:«g¹À­e^é< »"ŒW™rÜC‡Õ嘗ËÒOñ®fPaýñ ƒNó.¬²äàïÛ¦_"¾ã> -¥õ'cR¢Ô)p'ˆ1§!`û¼vªVA‹Ý D¶Ž>¢g:®:-+{ôœVÀ¼þ¤T’߇ w±Î·ü ìs”¬ÝÇ>…¹¶ºFì5Œæ> +n)ç†g^újtlÕÕrÅ­¬u^Öui‹åõ› õ9ο‹UT9õ⌫û¥n¥x8³aß=]z-TMÔc¾XçÙlØ{ÇÜs-~DâIn+Ü3lêŠg«–Ç9±7øvñÔ¿‡MŒãþôãõ¸^U<ˆûZLUKÀÕDw‹û6IO!æsSÎ>E=txóù¬;ÊX‘ËôfıU_«ó$VÝb¾ ü!2WD<€f‰v9öËoú‰ó³È£Ïvá}Ê|Y~/t³¤¸uýù²üάìçï béì12ØÙyjÚz´² C·uÍÊ.Öy+ëbÓ«^çqYØOK–©â_Õôçã½°$õ]·˜õáÀ·a=k¥Ì×ÁÚáÝ€}¼?³ßö|¸vl™ÍÚ´Ðΰÿ—,óèÇ¿\ï«èTïûV»–êôÛ-õZ§³Tˆ‡’…èê±Ú§¶˜qˆ5;ù¦b™ÃlûÁ26ãYÝÇZ¾"ǘ™òc«•´Œ¯àz`ÏvølÖomVžfÞ‚ïúkç&žú®e††‡ô§+ÎA€pf™×ê ³¿§uWk=]}²5'ýö¸ÿb§³æc¿wãžxÒîÜ,æ`eðìPaZ-ã9ûh1ß?+íóV¶Ž‘ok·­ÖlMÌ'¥‹¹”.l×ê<5û±˪f–E3ó-ñt{#ô`v#µw¼³—!ÛXÿpJz5³VÊ}Ó¾´c™ïðW/Öy:Ë<ñW?x×þ«zSîñ#|Ãgó~v¬–åFé“òW#gV{­Î³Ym>ɧ~«oøfëΪÞñ~t©7z,ÃÛ†­^Íb½Óûúß¼qsr"šßqv±ÎÓYèIÞõâ¹Qó .¼GT¢¢<Ÿ&nðex?´œ8¡ýá}é÷¼÷ô™ßjãÛ#ÜßÝ}Ïô «÷ŽÓÒÜ}N—ý¿Ó¥³·õ.Öy:«=ñS•l-|'<{µ¾ýLo¢Ö·¯M8žåýw„¶øwôŒø÷j9ŸN*ör±¢'0¾U^=†!ºWÕÙ iïÀËkužÎòNÎéÇ÷Ó/üeÏêtpÜ]Ç7RØ©¡áñð^¶|´Ø·d×Ý%ýŸÍ¨íX0ó/cš´uôײ'|­Î³Y0ùEW¶ü÷8•WÌë-‰ËÕO -ĵLÍ<¾ÌfP~¾áø´½€J€K@ýÆybf›?Ûôÿq¹„,zÌ”§ ØC69³;ÍšÍ91±ÍuQíî«».ïZ¿Ï麰H×XRŽtaßÞÛÜæ뜻)ïö†ù*[¯ÈX¼½rmB½Mª¾^ÞîY¦·ì©j¾>ákWùꜗ‘J®ˆÊÖ«$Rÿ<^/Sú£¿e¦ú³«EZûõú&ÏêZ’¼˜DÃf½…rñ¤.—«<0ôí÷à&úƒ¡˜ïtÙeRÖé:mº(Zßwj–ïtøµ•Z¾æAÑÍÏtëçüìa)ÿk™ð_n:»‰ô_®°*W²«BN¦>tòë®äwÙ¹~Ö ™ˆËGÕÚ‚iÚ §<–´~RËÇ~÷½Ê>4¾Ž†êûZ®Jtª²å²À¹cˆšAoÉ4z5é…L_k&»<…& -]ƒÐˆ½AÇËö{ÅS²¶Šý–¨Xwl\•GÓ5¸Mu¡›è\ušM-k"›Î#âã†níH}©t™zmªjרF³L¨Ü¨¾–lâÚ< ¤ÇM þ jÓ“ÙølÖèJÚ™˜ª!¹V.×réqË-­ù6S±•J½j®šï¢tS·©)^ =Øýt‘¾§…­eëÇ5™î>ks­³³!>nÞµ•Øɯ7…Ò¾c|‰¼%ZÖn#'Õí¥SÉsš?aiV]èIÿŸ¡Â7*!Où ]3)º”·?õNhm6òZIœê8IÚÅ^–©KÇØl–ëK'%”'Ðm¥›ÆVT¦2F¹ŠG ÕÀcMÖÍê™ ¬ê§a%'ª*ºëdßLùuEîrú¶T½jÍÌÝ·v;94ì|åäØf\Ó ËMÞÖ.³ùJ|”Íd2ýT;éd£Ún×;cÄ.@¡“;Œ¾r]­ è[cêǾȤ÷qûË­lsÆzCw«é ¸×¥ïÿƒ2?áç?oîþÏõ†Úy7ª¡…iغ"óŒ¾^&]$Ó×ÍTb[6ŒhƒýNÖ©—õÕÚ>n Ãk׫«쵤nª—}×!#›fºF[¦@| ZlŽÌ·ÎMß°çG[Gw6ÑÝÆU*<ðáà1uží[ò5‰'*Ú·tpañÀÅ#ë0wÉVb’AòNšêI~ǧÂR_ì¹ôh;Ïõm–7]ie¬\`ËxÃNóNóJAèw/¯ož÷EäPWQ´7ê+§¶µžLµå1¬ÛžcËuGuñv µ¶,[{¶99}it×­¸2jêÕ–žwê*mÕiÝ(\€¦*­µ«La©VÕRFœD-ÅæStê_¸¦=Ÿ•¦DÏg%lÊR»uÇkÞ’mçæûèT/Ù|É$Яݨ®—´•¶íÛ­û®»í ÃuAB÷LU¯æ7%ß…ïY;â¥ÕR©ú¯dFtêwºS¶XGklùš>•B†8ú¦•èK•¼¹Îqîd›!©¹Þ¦crÖª¨é e^ÁÈç ÖTý2F¹ZVâÞ¦!5Ôan{ÅxùÆ­’£, ¢ö)T¤/سäô&ú^ ¨´|ÉxܤOÊ¿ÿñø=¹ -endstream -endobj -569 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 568 0 R -/Resources 4 0 R -/Annots [ 570 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -570 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 597.273057 101.433071 578.523057 ] -/BS << -/W 0 ->> -/Dest (cb62-1) ->> -endobj -571 0 obj -<< -/Filter /FlateDecode -/Length 4815 ->> -stream -xœí]M丑½ç¯È³‘ùüŒ9ðbwŒ5v°ÝƒáCUN—EÍã=øïï#%’!*”©ÊªngîBu¥"H*|| †”’>+ü|§ñ_$=Ř‚§óåçÓ/'5§ð¯”Xõø Ééä£;ÿœ6L!9üùõ„ÿ†C^üõôß¿:ýﺎ>›)%o´;ÿõsV‘B A9A• RÚ -*禌ó‚Êë ]LIRÁ ÈÐV`¥è$Ì c¢ Š0ÃRÐv«J0ÃM’ -fèä‚ Ò -vh£”¨ƒ!ŠÈA§Ýäát/錞|L&‰ºFÒY;ù@ÚiI—&ｋ’Ž`‹KJK:[œ&R’¶AÒyØb½U¢¶˜¬¤ °ÅÀ;IÐEØtEQ[”WZÒ%;¹‰D]‚NÅ·:£Ü䢵JÒi=9`ÓŠ:Ì( ’‚ÎÀ¯\u°ÅYe$…-„Ižp6Ä(ÌV“g·UVK:[Œ $ê` _bãa‹ -^¢Ì )ó’Ä&„‰’!‰L´EŠ ˜˜& -ÁJ4`’›È§ ñ€U:£%"° -¶8òX [3WÔÁ›Hâk` -H\`-l1d%.°`[Z$.° [RIK\`Á·¤´—¸À‚pa¦’¸À‚qmô$qåJIâ εàN‰ ,HÓ=H\`ÁºÖy#qíZŠ^â Þµ¤•ÄÞµ5El€%.;NVG+qw­VAâïb‰ 0ªpè%.V&£’¸€À»&*’¸€À»&` ua´x×8tAZ¹Á»Æ)#qw /q€2” €wA!N` Vâ¬÷Ð+q› ‰ xW',Âœ[M:&/qïêˆáuiÒæ-¼ñ•ÖÌr'¦!ºo{Ž^Ìî›ócýóé×OX|Ïþ¿Ó˯J¦¬þh¥4!¾Â40 Ãí}ktßú²à©ò¯Xï©“©©A‹KÒ-‰å/ÿ;o?¬,‡á?–ÓÛvú„U¼œžò‚ê69‚íëhñ«| Á -›#¬¶el2™9Ä•ŠV…ûé .VÌ9ÐvÈÌÇz‰Í0ÁæO?Ÿ~ýo?üð/ÿúŸ€Ñ§—Ó£ùï¿Óùoœÿ:·ü ‹ü²üMìs­CCÙ¥ ªëâPæŸÍú\µÝZÆ]0wÑFÎÒX9wÑ}^ŸƒÜN_´ÐN=fíÕÏõ/½?Wéóà­—:v8§šÿ"\.õ´úþOçO?œ~÷i ¼àrÌÀÒ7°z¬Ê{°ú% Š­íˆº¢ÕÍnf>Õå#Ôêðxå<,¿žƒÉË>Ì—axŸ·ÐÛÚœé^ÊåÀ;FsslÖy´ÁÅnãÈ趉5Á8QóÈ!lQæ1j´Œ°Øye²îy>–t¹ŽY&hÕÛ'È.K»Ï¬õúü\ܱeÅ//û4ŸÃ*Ö®_l©„ëÄyêåƒú!ãËÒÛñu¬ÎÃá‹®²ÇÝcÌÆêMcìg\dVù°1>ÒÁŽñ¸ôDzvðٚŶ˂él»Ýa,wc«ópˆº2eTÜ3²Ö}Ûìó>ûwPÕ¼¿…ªcuU×£œ¿7ï¼yt?Œo–u#6<úÉ#C)ïϯ"å`GCŠ¹1µIÁØØ®ãᶫ±îe«GãÖ1v{7ÛMÕ9^î3ëcµó©ËDÛvÎ_û%õ½Æ÷ãÖ6oAKÛ|Ûû²¶}ޢÅRÚLIE“X×Ñz¬ÎáõFüõ7ð_ ºnçPëz+Dîô™¤ó}æˆsÏ=²)÷ÄÊùõùšß.x•¥·œ™”#ŸQu©Çê<Ro¤Ân qœÿMäwPÄ8M,ÿi·o/e¶ƒÈß¹|cB s¥?Vçáy#&ÜYuÇU¿Z2P‡“_y|ôßdÒWF°½#»w°Î£!ØÞˆU?Á;hy ²ÇնŔÏ,Öû‹á;èíqh¾Ž[µ[{òƒu½7b×#}>á_ ¡o™5×ú0F%‚OšÎ}íÝAãyǃu7âÓ1Ú¯œ £TöÔ±þ~£ÌÎWÛ~Z¸Œå5²ìÛáØäÆ·çÖy8仂{ï…õ¯ŽÐ°.·A¡þ~}O¼}50úi,sƒséŽ\ëÁ:†\:¿¶LR=:"|…¢‹€–÷ÌÙuŽ@‘ƒhˆ’‘Ï9Ú—9‹_¢„{ëó[rÆL«Û2^Ò2úƒ„uih›ß/±ñóÁû(èŽëÜë<RoĪOBìX½h'¾Áë«{.žçúù:ÑÈií|Â`æ€]+NÖˆ±Îµj+Ë×…¾å>}ÙâbïˆgÖy8ľïÖÃÇ‹óÞÉ<ìnñfÔ²ƒæ;b܃uÍ×cÜ÷î¦ø•Î‘³Ž\µ]­ÂÔG¸\'â3¦^sç¿Už¯Ù?Í÷ä¿o¾#ò‹Íd}îŽ<ëÁ:†>w#NýGä¹·dª® s³Sº°s^»…ý‹ävÌcÒƒ;®ƒuÉï˹~“Hþˆ\ãô¶è¹M>ú×}¶Üc@l6Œ³¨ïøvÐËâÓ£WlÖy8ôÞ¸7t­a×"݉2îtjbä(~Wç°Š6³Óæ[Ëö7Þq#Ú´Ìú‰ÍT†îûîóÚAé÷¬óp(½q_Áâj†§zú™!dÌ¥Hªh-È$µ¹ƒðkçp¿å+žÅ¨Š&£±e¢H=XçÑê~;èŸWä; yn×­}$ö'¬ÛÜ»¦Øä†Û½ƒÖ;r°ë<Z¯Ç®mä^Ö(YÝ«©Õ:¯ZWÀ,{™!Q›ï«_Ý5×åʽ«cF¨Æu/ëút,¨)¹ßÜBYª~çà 3ÎÜ1ÛŠÍõ“GBÃõ³fÿηPü9ÙƒuÁ×ãב¿ÆŒÒ*›ä’ªç¹k”‚ÚÞ•e{Æûe9_qn¥2[sвÙ!ík™æ¸Õ2¿6ç×å†õD½ì 9ÞÁÅÇê<’¯Ç¸VØÏn® =äÈ×Øö¸°¶5p©eW^7×åˆCçbTцñê÷ø]‰+³±œÓn뮘FÄt¸#Ë{°Î£a:ü滳 ~ׯaÇ•QoŒu‹GÜÜffv‰)7ßÍ¿2Î{×’÷¾³"]•Êsx\ Ü_[ÆxFÂt+óÓ:*Xü%®(¼Ÿ2~Ù½±:?ÜRÃÏ·ð{¬ÎÃá÷FnWÀïÊÏW¾ƒÕîtI;Xá±éø͘±í½ß(ÜË¥Ãn›c~…}ãR΀µU¾=–)¡ØH´CŒÿdÉçg‡DùÑÏÚÄ2¿ûÃüþ¿þÀò”9C“÷tíZË"[®ú™æáa¶6´gïÎe]fŸçÚ³ùx#_Êf=åø.÷ø"­ «Þ·‘!¿•çö³=Ëyíüy½ûZêq`ÔÜ%¢X]¡ûd?t6[|sYlWúÄú_ìöÝWÕ‡ª|£šÇ¤¬-Õýëd›ÔeFŒöK×üªO.?·Îæç–×>5ûó9¿åçáü´mÇÛüP¿üàèm;Íï´´å¯Ú㞪÷¥ø¸ç/Ø– ¶ žÓ\?÷¯ô1·õÄqÚ1j>/¾¨6¹eŒÝ×ny^PÁ²YÆšÙ?3“lóÊÊñ½ÈÒÒ÷§¥ÍyÕºâϤ6þÔ‹]œ^Ãh·¯Ï!Æh¦ç}üU|¼ø*ŸO/å,ósJ}6Þkî`cBóJa.u®°±Iýsµ}îûUŸ™ÑgÅ7Õ¿ø'u?íŽe-óÒí³ÈKy_ó•ÇpU®ëÔ¹­™|üvÚÔµ^ãÙOz¹rXÊ*Ʀ¯:;~£ß†±j¶{Ö¯:6.¥¹ÐlÞÁBÏ.õhéÿËÚëufi—óˆÚâ¼b¼ÎïÆiŽùŠ—»ê+/ùÊ™ÞïrŽ´Ø[íˆC¿ÙšZí+x¬åNzÏܯsõã||Õ?®¾6åêŒî¿ó.ö<÷ÂÛk‘ÖËÓ.ƒƒÁ.¿é#?¯2±‡g¶€ýf‘ñùöQ›õ•çü3à݇h2¿æÆÆ36Œ”B~2¬æewäåÌTõË­ß¾C¯…‚i)ãØ°û‚mJ»™r*D -Ü<£ý4Ë 18Äþy\Ëù–e® £…Ö~zÌž¥¤åƵ҇Í0þròå Ǻ<å^aFj“_ƒsö4å)¤s~ªrT–t6]­Ï;5Ë˃~i¥–÷ ©üŠüxéùeCì`)ÿKðŸ}"²¬ñfy÷‡*âhêA¿žò»_Hõ õ†8 —ªµEºÅÀUvÁ4Li,i]}6+ ”­««½Éõ]­–ŸÉKª²å©´ó‰!jz½çÞ«I/âüþœ8ÙåÈ7Sœ¯6CØ$^¶7Ø+Þk«˜Ðm…Š®‹ U{t~ÞzsïÂf:wfÃ_ËšÀ†óšðrÂií(}¥ù­µ©ê]£šÌ2£R“ºZ² µqD—“ üNÕ¦ ƒÙôlÔò³Ïga¬@¢V.Õr‚èrË-­¹6R¡•Š½jªžï¦t¨ÛØ/È.l3ò ÁØQ¶~\‹ó~¥5B÷Ž³^NŽÚLìâד Bi×¹B¹ -=.´¬Ý&Žªã¥Kó£ÕæO˜šU–ŸÿŸ©Â5ifžòASƒT~ú{?ê'És³‰×NârßX‡$h -½,s—¡a–ûKG%”ϤÛJ7­¤ÌeLR5/“T#µX7Ô5q¦°êŸÆ•\¨ª8]»åו¸ÛéÚTØëɼfpw­Ý.ö;_¹8´×ùQþÍÞÖ.Ã|^ÊÂF£8C?Ö“t±Qmµë'cÂnH¡‹;¾q­ÊSö–}’I+îåô—SYæŒu&ჷ:¿™†G]úü7”ù¿ÿs¢ó¿×G¡Ïë¸Q-LãÖ•˜¯dù=fñ 8¿×¬ -Û´aBCŒö»XÇ^ÖU4´uÜÆ×Ô««N쵤n ª›– ]÷!›]£-óŒ»"¼ ^lÌ{Ǧ/Øó¡­Ãil~ˆ~µ -¼;8Œ]gû’|fMâ(íK:´dXEðFqÈN˜ºe+33 ùIšë³ýćÂæs±FäCÛuÔ—YÞt••^°"-?à “æ'M+{£ïž^ï÷Åd_g@–6joÒW.mËÒFx²Ô–C¿>lkŽ=çƒ:y»‡Z[–Í=Û‚œ>5zhA«ŽQ&½ÚRÀñ“R•­NZŠF©*k-†j“_ªU·”GÑK¡Å]úîiÇG¥9ÑñQñ›²¹ÝºâµhɶÈq8‡Þ;ÕK¶X2 -2Ïk7)õ’¶Ê¶ç¦õ¹ëj{fÝ nˆï‘©êÕܦä]üwÏÜ™ë Y©Tý)™ûËC°P“ÖØò>X½‡P~¥W ÈR%o®s=w²ÍÔÜ ?†c"kUÐù;² ¯P¤Û -ÖT}ë¯\-)±Kos€ê4§6™¡òjÇr·$ˆÚaÈ[Åü&W›ƒÞ˜_@”–¯æ«>O«ôIùùþS˜K -endstream -endobj -572 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 571 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -573 0 obj -<< -/Filter /FlateDecode -/Length 4943 ->> -stream -xœÝ]ÉŽäƽ×WÔY€èÜÀÐÁ€ [† ØÇîši FË€äƒß/’Ì…ÉàRÕ­1åiôt1"—`æË—A)¯?_JüŒBˆÞ™ëí‡Ë1x+ð/•˜$õ·øõö*…ñƒ•Ñ{ýÚ>ZåÝõõ‚ÿºÃ¶øëåo_\þ5¯#¯jˆÑ)i¯?}"•ÑƒÞ ˨"TBjFeíིŽQ99àcäT0Ãz£ÌRåa†‰Ár*˜a” -Œ*À m¼ÔKU„ÊKé`†ŒÖ3*)`‡TB°:"ŒÑŠÑI;8 ºãtJ.DY‡NzÅé´œ7ÒJNçœ œÎÀ…ät¶XiŒàt°Åèà9ƒ-ÚiÁê`‹ -^s:[F'2º[€®Àê`‹pBrº¨ƒ1¬.B'‚Kv°AkÁé¤,°©YV&ÉyF§`‹6²:ØbµPœNÃNž°Ú‡À¬VE«[ -9…-JyÃê` &Ÿcå`‹ðŽ£¬ x‰ãåý`¢2 ¨ LàX@…8ï5G*ÚÁ¸è9ÐBB§$GZÀkÇZƒ•Ëê`‹Ž†ã­` -p\ 5lQFs\ Á¶há¸@ƒnˆ’ã ¾5B:Ž 4f -Ž 4Wg8.Р\@)r\ Á¹ÜÉqéb¹{Ž 4XW[§8.Р]m‚ã¸@ƒwµ‘‚ãÞÕ5Yl€9.;ZÍqïj)<Ǽ‹e¤8.À¬À¡ã¸XT‚ãÞUAŽ xWyl¬ÎXƒšãÞU§ÀíÜà]e…â¸À€w•QŽãeÿÖï‚B,ïÀ,ÇØï¡Sšã46(¸%Xð®ŒØþ˜5 ¶dˆŽã Þ•ÓÇêâ =FœY·åA:9.°à]lTšã Þ}Ž ÀñvTÅqïJšÓw¥Š‚ã Þ…ß`9. €ãìoƒ€.«‹ƒˆ!p\`Á» %ÅqváA0 «ó€$9.~(Ǽ‹í=r\àÀ»ÂjÍqï -·‡ÕÁÇðk Ã²eÖ4œÕA(0§ï -‰ætà]!|ä¸ÀwiÛä¸Ày¨Àq à+Ž \0CôÞq\ࢀ.«sŽ-Ç^@`AY¬¶o8.€c=DÇ^Á­Ç^Á¸ À#¢„÷͆°paã[à)r\&B4†ãoâ1Œ*§sjtÍêtÒs\཰KŽ K ‹žãü@îÇ0:i8.ð¶`©p\€­h -ž9§“°N6Çðt¡“’ã‚ ` ¼AŽ ‚ -ˆ2à¸;4bI 5§3‚âÌÀqA0n W‘ã,Dš`JV[@`~(ض øA°‰0‡ÕÁm Ç!( -7Ç!À%ÇÀ&Å›žã‚(` <>Ž ¢p¹É€Æ¡‚ã‚(âMe8.Àº¤x3p\µ@¼‰Y·Q;Š7=ÇÑÀ …ÕÁ8M‘Y·à$Ä›`'Nç`‹Š†ã‚èś㸈7áѳ)@ñ¦gÓà]ÁLœ¼ ÏÔq\€Ä›ðl¹Ø^€x­‡Æ+EœÍP¯ÅÞʦ ¸»•gsäK1›4€™ˆ,ƒc³ØV¡”‚Mð/ü1³à„1ûÔ¦¨àà±gO'±, ¼ûsÔbz¥Ø˜ûé»Ë¯ž°ù^¿û÷åå‹”)Ë?à×ÆQ`i¨ˆé] ©9ƒ¾¶¢ý‡ „}T!|¬&Jâ”r#±c³×òAÎ †½d±„+“ÎË´Ý"N˜(íXKï­òî–bÀ_°ðÜÚ~áŲFî–ÞµOÓ¿ëòÃŽ™`i¦cð%ÍñÖ­ôŽF/áü›ðö˜3vך /ÆòüðÃåW_ÿéÏøëŸ0_^.ÿ5ÈÄaƒ@”…ß>ã¯üøÕ?®¾¹|ý!§.Ð þÑiR„ ÷Aäû´´w7Csg•‡Ð6 N.Î|ÐQÑ–C°"ÌbtLîu¤r(µ-¸Ã°Ì¡i°~ÿÍ7¿ýÝ_ê`Q~×د¾”¿¦ÁÿZ9Ê­žŽþ¦á[ïü."?]‡Ô‚ &… S?‘d_m·¨tµãÀ3-Ú—½ÚØO)WàkÏ^mxÃ)¥âj›'ôŸGl:£4R;m¦$¼±N¯Œù4µ•G©ë# iúkgÌuöLum\ÊvÏ.ùÃÉ7¸ä)+„¡sž?ϽÚ4oÂ&?á½FÞÀ…Oƒ@k›6N³°Ùè£ÐÔ*;IÊ=[°Žãé ÔdŸ©Uê¿n¤(3QY¡íxlÝDe/Tf»Ú©¢ìj¤7eòx|jlmƲŒÓmnËÌvË*ÞåcuN‡e³ÉƧ™WË°Ï„¹­ŽIµiʨfî,¤>d׿ïÚálð]ß›o­ÅÔ§^Ö]°p·ãЮ•êúë·U,ÛxùXÓaÙmòr»SîÍuž_„‘i¼•ãy²Ÿ›­y^ìþýé1ÃìðÄoý>P°ÝyIds¿õÞ ‡éRæãÜJXíÆ‹ÝOúõÈàwÖVËÓ<†«§K÷ xm4åõ¶1|¬Îé0¼í·žkò#BO¶Xi=ÓSG<[òÄSôõ¼ôe’Sðô?à̧¯ŽÇ|Z›­˜¯Äs~Œç¬Övó%bŠõB:—e¼g†mð²ëp+Þú© K‰3)]¾‰èƒuN‚èïio6YÙv³ÜíÐ)€F¥þÒ¯Y}Ö“î©+óqú;É•šôù÷ãô76eŸ?w&hlkÖ^èVW·R¥XAY°1â ÊŽÕ9 ʪÙ1‰É -O=M¦Ì}y¤Ÿ·÷ALB“nP–g«‘Ü|ÖR™OnSü—9°‹íRj½í¤ÛAfA›dÚé¸9e˜{þåx_iLåú~°†T#ÅÝH=XçlH5j“9„ȼËêN'–œ±É5+¹¦„Ìn¶íʮݷ›êÞæö%;ú¬D—›­¤éü¯»æXwŸÕT'®¬<Ùx¶Ž_¹f²áY¬"S>¤ûn™Çꜙf“CË(ö;v‹¬-”pù¦¶îÓ ùî=%kùEFû¥_Á+(¬¾Ÿ±jÖ8BÌ6 -Õ9 -ù‹efšìÎZ¤‘"˜\¦Ë`qHí¹EÖʵ‰„ô;ú+™ÁóX5‰Q‹åú­3žd®)›Ò´þàÁüÑÁ:§CÞŽÙÏdÇöèkfv±“­|æ¼ÿÒ-,x‡ÛÙßb÷˶?gT×ÿćJv^ÉϾjyôÚÆG”jˆ"(ºn½ëœ ½vǯ쯊t‘}êf¶‰Z¶ø®EË犑“MkÞÈóä33>mñ4¸k§÷Frköæ:»QÝ -b«ïx±ÇꜱÛþæi|À·Æ¼÷DoŸÁ»¹gUÌÊ›®u÷Çð뜹Û>êÿ[FçÝ8ûs¢;£uwõ­ 9Ü÷¬s:4oû½ tǃàv炘•Ñf‡ò Ójé¯õˆùìü™eÏKÛ*GÞ×nY­+Ü꟔žÖ dú–Ö&Ö9ݶ»@VáY43o‰§ËÝ ³»©=pWÌZ†lþîªùÑÌZ*÷¦}i™ø«ëœ™;þê;ïÚ»rïážÍûYAm“¥o¶¦oŒì¡öXÓ¡v'ŸúVßðnë®U=potª×{,݆¥^Îb=è}pý/î¶Ù¹"ê¸v°Î麓w=xݨxÜ}em;·)*ŠãÕÄ¿t÷†¦+ŽŠi¿»Wú‘ûàî±ù^4öw´þîê=¦›z\-õÕç4Ñð; }Óyµëœ µ~ÇO}PÀPÚu“-™¨N‘—¿×øæóàñÕ^m>Š¯cuN‡¯ëÑ9næê®9v#.ˆUÞmŽœGgGœ¿-TÚ¡ï|©É¶Û„éñ®0ÖwG¬s:Dí\%Í,å7~Éìó6ûW€%ïÖ9ªâÎ]‹ÿcÞ¹{vßo¦µãDz^ëœ);W_›¼âÏæ¯Ü¡Ýßu$/Ë}³„óêÚkI%âjr -ù®+Wl;¾ëÊIÎ×>pokl|©£÷ -¬s:´îø_Ÿ1†ÿÙpï½G2swÜÝ“ò}¹'±¼?¢¿f6ˤ­=‚²f¶Ž~ßê`Ó!u'öpžQ<~7É;fÞ~yY³‚Èo§{‹É³ôÒ$z¢nlžC\`¸[duùYËgÖ^e÷ÐÝùã6Ð$½1L‡+9rÑÓC¶×¶¡>ÎUzô‚›¨)Å”/ÆK)Ê×{U~@GÚ"ÇT’M[›Hë0ŸêBNëpcyzXA$ÝÊGeÝø[Þè¦vB†àÔ=f’ê¡Mœµo§¾{Ìôàr6ŸÃb¼¸ô°x™^"œtEo»:3éã•P„62O›L-¯+5Ó{Ø~,¥èˆ½­…žÔŸ¦µ=˜Êÿ˜&ü‡‹Œf ñ'B_£$’8¨|Půz–aÄÆ ê ˜„ÓGQÚ22,²xÕ±/ ÇULciœ¼Ùñ£²ùl¨¾ÍÕè±¥FäB:=à{옢¦“K1½ä$¦W‘…AOG®˜BO¨-ÂÚ iËÖkÅ]±Ô¢Ú¥P4ÝU±2ÙI¯®(C窰˜Þl¦?—U¾™Î-áí‚nu/}e¥ô¤ÜT]%ŠL7FÅ"µ¹d™Sæ‰Ý.ŒðK‘›6˜Ì¢of^#1 -C’)åb.Ljn¶ÜÔš-3åK©P«Æ<òÕ” -uÊÀ3²4F“ÔÓ»›5"tþ8;š<×"4¸Q áíbMY‰Uüz±ž)m+W›¡× -uÓnQñR¥”Î?aif=„Ÿþ©Â)1Oú M½H£ÕNhmñ|Z¹+¬c¸ÆײÍpIï fÛñB\Ï”'Ò-¥ËˆÍ¤Í5r³yDR…<æbYPWÄDay| -W¶B‘@wUl ”_gâj§-KÕŠÒ™“ Ümi·Š]áÎ×VìËŒKz+J±·´Û`> oic3½˜ r'U¬DÙíjg°R¨âJ£¯IœWka'zÓcþX·ãÞ.ß_Ò6‡èAÑãç–ô’¯Öë’×ÿ Ì7øýçÅ\ÿ˜ß*1îãJ¶P…[gâv'£WB†ƒbzªi–eÓ•ih¿Še¨emFCÙÇ•jøÚÔê¢{.) ƒÊ¢m…¶Ža#VºJêfdì†ðÖbqdÞ:7uÃu>9zœ>½$[…ƒötpªN×-ùÚ4‰#*Z·thj*â m‡M‡±Z63“ÙvR†žì7íThê«9hÜ:ÔUgê6Û6eé,š"pu{Ð6ldÛiœÙÃýðòzó¼O&»¼HZ¨½H_[i91’Âã¥:ºùaÙstz}^¼u„J[ºY{º89uiT×ÂÌZŒ6ÒP«Ml۩ɲY§y£(t™È²Ò¢Ï6¹©Z–tÆ%_|Š*ý¾iÛÎJDÛΊ[”¥vóŽW¼%] ÛÂÙ׳µdñ%#smí"5µ¤Î²eßfÞwÞm¯Íi˜jˆ«ž©¨Õì¢äCü÷ÈÚa_6ÕÊ?)3"C}6j#½V:½Z[çÀ!†ÞŽèMhR%w×ÙÎ,3$9wÒö„éŒÖÂKzI:oÂ+q_Ñ4•_ ÎW‹‚=¥û€I Uš[¾(½%7¥-S‚¨z -é¥ØšœÞ@ïòô‚UPºóèy–>I?ÿ$O¨D -endstream -endobj -574 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 573 0 R -/Resources 4 0 R -/Annots [ 575 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -575 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 770.673057 101.433071 751.923057 ] -/BS << -/W 0 ->> -/Dest (cb65-1) ->> -endobj -576 0 obj -<< -/Filter /FlateDecode -/Length 4848 ->> -stream -xœí]K$9¾×¯¨3Ò&~?$Ä ‹@ FpXq˜®™¡^¤…Ÿ/œi;ÒY•]ݳ[‹˜VMUFøþü9lg¹ôYáï+ÿ’ÓSJ9w¾|{úèþ•«‹¢þ#^ÑŸµrqò:‡äÏß’ÀÆ)fob8¿œðßpÉ“¿œþò“Ó?×yôÙL9£ýù_ŸIåìSŒÊ ª •ÒVPy?Åh|TAO¸Åœ%ÌðÑ·UE˜árò’ -f8c’ J0ú¨íV•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚON’Îè)¤l²¨‹Ðéh$µSˆN{-éòBðIÒ9Øâ³Ò’ÎïS’¶8›¢¤ °Å«Dl1)ZIa‹w² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Å´Õå'Ÿ¬U’NëÉ›VÔ¡G¡‘Bt¶峨ƒ-Þ*#é,lqø“tà ocJBo5Ô»­²ZÒyØbLt¢¶ ñ%0¶¨$ -@šˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lqè¹¢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.º{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]k‘SÔÁXâ°ãdu²8ð®Õ*J\àÀ»èFFâ´ê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„>h%.pà]ãq ÒÈ Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\€ñ:c%.@a“AX"qïêŒáOèÓ`«I§$.ðà]Ð|¢.O:ÂãB¿…—'B–¸Àƒw1PY‰ `>¯Jº`¦A×¢.A§£Ä1º C±–¸s©)¡ÓK\‚Ÿ(\’¸f@§Ä1Ãt‰ 0MÉ 2—t¶ È–¸‘.tZK\ lA4(qA2 ³Ì $.À¹$\-霢yf’¸ ¹0Q¨(qº fš`JQ[@ }h²í%.Àä“MLsDl±ÞI\’¡éf’¸ %Øb”‘¸Ø¤ùf”¸ +Ø‚ˆO€ÂDa²Ä qè”’¸ ë„ù¦q _Ò|3I\­Â|S<¡ßfh¾%.ȶx„(¢¶ hÊB¿'a¾ v’t¶˜ì$.È!Ð|3I\€0óMDôâ’@¢ùf— À»>ƒ™$x‘i¸˜o"²•æö -Äë#â3Y™hÆ™Äêõ[Å%îÅhÅ5ò¥9’¸h31³LA\5À° -¥Vâ²ÿ"sN˜WŸøU"ÆÌé&3º…Ctßæ=™ÝI6¯ýëo§Ÿ~Äà{þÛ¿OÏ?)+eõO#@(©â+t“ÑÜÑ·ÆíÛ€À¸ ÌDRd! ¼'s7Rƒ—E72Ãý;o?¬,‡ád:rc^h-“p„asý`¢RºÃ¤óCcf7ùÕynš!äsû e[»«x½ÝW’­¯Îóî¶ÒÒ+¦Ð8×õŠFÞJü¾ßëÍ(g[/¢yx=B6öh¦w6Ú¶“©JƒÒ2'beb´4A;ô±WfyGƒ¦óˆÒØlh<Ib&¤Ü*ñ/>œ@­‹óª}»dæ#ÂÓ*?øöôÓß|ýõ¯~ý' ðÃó雟)páÏ¿Òx÷v~w—å:,×i¹öË{ÞÊ´ʨz5¿c"£À‹åå>/º%óë<îy(#.uPÞ0¤¹ iõþó ¿`#;̶üZvÑÝ°±ùB åÔkV^ýìûÔ¥ÕÏÿzþðõé—Ö ,s«”4Õ@§%J¸7ÿëa4ž·"XCF8a\ÊvO ‘£Ÿ7’ŒTÓãÎÃÑÁÁ<†Ts#Vý(ÄŽÕ‹–qâ+¼ÎQjŸæüÖo9­Õ7”CéÍ»Vœ8Œcw…[Z>.ìÜÝÓ—!î öŽxö`ž‡CìxvoTç3˜‡ŠóÞÉ<ìlñfÔ²ƒæ;b܃yÍ×cܷΦJkÔxSàÑÖâf(—óaEœë-LãªÇ¼ìðªrp»¦™WžßWü×hâ‘ΗïÉ;è»c•õ`ž‡Cß8õ‘ç^³Ru›™Ò…Õéd_|¹µɖǤg\ó<’íÛÖ\”H~µÆém Ðs9›8|ôÏ?úìiF­c½aìE}Æ·ƒ^Ÿݵ=˜çáÐ{=¦Ý´Ö0kièá#ã0Ó©kŸ #G=užYqXE›Ù)󵈿e»y‡û\z€ûÄz*C·ØnÖµƒÒ;ž-8˜çáPzã¡‚=ÄÕžêé'†q-u@REkA¦›ãÊr—áǼs`YŒªÜd´KÎÝBê±<‡Ôëqíÿwv¢V^³Õ¯}$ÞO\—¹·§Øä†Û-£Õݱ{0Ï£¡Õ]][Ë=¯QRæàlwsµ®ZGÀUöÖ2C40¢Ö<3D³´!÷¢¶+B5®{^çß cAMYûÝY[¨=‹Ê²áu=ÎÜÑÛŠÍ‘Ý'„†ý³fØAðk²ó<‚¯Ç¯#+J«Õ¤ÀT=?ðÈ]­Ž; ã -×2¾â(Ü J3.d¶Rв™!íÕµôóÜj™_ó}¹aEüzöl¬¦¯ûkøùÖwŽåy4üúk»~W~yšÅ®íI—¼ƒ›˜:ÛÞ·Ò¤v×RÃnëãúÊRNYWÀÚ(ß¾¨F-@(Öí’}‹ËÓ!:Új“JKüò÷øíŸÏÖ)i…†ætm¯e‘=uo˜æå¡Ç6Ä“‡ç´žè©ÞÝ|½‘§ÁË—mٔƅm%½_çq±¶Öü™^5ý&µ‹]~n¡Î"KújÇbw©'Ô}Èá‰;üî7³.›ßk•)ZÓå#7‹½ê>-þ¡<—:,&nK S ²¥›[ló ÕA¨ËÓŸ¶åK_g¦#s¶åû«/C¹—kvÐ)O²ªß ùºÝùÝÌ=[°ŽN -Ê™…²gÌö2×ìø#?ëÐ3ps{,6ùÊ.óçz¯d‹=m¹ÿ§Åö¼¼ù> -žjšgÖLÇå×´–öñÌ&ÍÒ-mXWwüžFß4 ifsÅC¬enë¯ýË·>!ûJ/;Q%­šïSÙµÆvh6voÕïºû½¦oö׶`Xò¦Û#Õ5sÎ5ŸE5úÌÕç2ç¥=NºÅGókÕ<óWZû@뫶Òim+[gîXc¿öŸ1‰ÁOÁ–¯xÒ‘F‡â‘ƒyîú¢í;"¾Ù󔼤åm:‚Ö8ÑË–šeCÇÜ»æ0©LÇÕ V:âKµÔåêên‰-ë.¢P¢¾•Û¥rºS¢=·rû0e6“róÇK[øeozÈ”cvÇCß×üÍ»w¶œ`B§9Üáy:Ç ® Q¾Ï[¹©ÝÀÖ¼Ÿç]2èeÉa¨ÜǦÏK+\­c ?ENJtJï1àl©³ÁsÐؽ eÐyx™Nž*-={‡ôèà—…À¬×tʧ¥³&n|!÷Pž™„5îKA]ã>õ‘‡àl"5ô&q‚;öLCùÚ÷~}B}¬aø6Œ_=Ã&o{_ˆÙ°D4Ç6ç¶ncjÐÖ°—ÀàyÁvpí¦œ}Šz¨ð®3…è3F½5Ç;˜çÑpé¼ã}\o…0R¦¸~y§Wœ¯Eg1|Ò|êË–)…YôõõiyÏ,íÓ÷=â½ß£3Ù›Ž˜ƒ'Ìó (ëf‡p e¯ytæÚ#Lf|Tfïñ“³ -<Ö£-[VõÃW‰ßAjŒ¯Gê±<‡Ô|•°/ÿ_àÚÙD’½¬í;ü¥†Ú“–û+¼ص>›%OÞéyšÅ?¾û¯ÅW6@v¿TKG´Õ9ôñs5Žez4lj¥¯Òè[¾¹ÇA¶Ú¥ây?> 彸tO¿ybèyìÄ ˆ\δÃôE=çO§ÒevD^ßÍ$7 Vƒéí!_g=¬¶Ž·Q$ý˜…MgD“­ß¶ÍçyFYb¯°|¹.¯e¶½±ôoS#t=Ï<˼bËpŸ»ÛJ´^—üžgX‘Ü´%o6K Šm!¨¶ä\¾ôšæ¥8¾„ÙʯKÌžÕòg^bÆr›füîÊ9¦ºœe­Bš´¡»87Ñ^pÌg:;5)ëtm6]­Ï;9ËO„|×R-¿¢è q:D¶4+¿XÒWüÛ“În"ÿ—=‘r¿*âdêE¿œèœ v®o¬Ü“pù¨ZY˜ ˜Av›Æ))1«PËǾ™CSù£ñõn(¿¯ÙèäM§j"[Ξœ+„ÈôVLw¯&½ˆéW2Òd—«ÐL¡¥¸&ì:ž¶Ø3Þk«˜Ðo…ŠU×ÅÆU{4ªÜ\º°™Î]§Yó×´˜}šCÂË ÕÚQú"JélþZTõ®QMf™Q¹I}MÙĵvD—“ üJÕ¢³éY«Ñ dz0U ¹–.×t‚èrÓ-¥ùÖR±¥J=k®žï¦t¨ÛÔ/È.lÃ3ÒÏþ°>¢lý¸ÓÞ\kk•nÌFx9y×zb¿œ|RûÎÊWèq¡eå6qR/]Suºf•Ñù°ôÿL¾I‰yÊí¤èŒç~Õ+¡¾ÙÄk'qyh¬ã$h{Zæ.cÃ,÷—NJHO¤ÛR7­¤ÌeLs5Hª‘ÇZ¬Ꚙ(¬ú§q%ªjªëbß ü²w;}ëª^µÊ‚fp÷­Ü.;_¸8¶×t`w³·•Ë0_…—2°¹QLÐOµ’.6ªv½2&솺¸ÓèK×ÞÚØ©œQ³|ìLq/§¿ŸÊ0g¬7´Ï¬¦ßŸàQ—>ÿi¾Æë'wþ]=ðxÇjla·®Ä|$£_+JÅôëEUغ Çh¿‹uêi}ECÇa|ízvÕ‰½¦ÔAuÓr¡ï>dbÓ k´ežñW„—Á‹-ykÛô{¾´õæhߎʮVá‚ß.S×Ù>$ŸY‘¸¢¤}H‡Ö–¼P\² -s·le&’WÒ\Oö;Þ–êb,Œ KÛu®³¼è*+wÁ’ ¶ü‚ì4¯4¯ìŒ¾»{½¹Ý“Cí$mÔÞ¤/\ÚnŒ¤ðd©-—a}ÙÆ[¶ÜjçíjeYÖ÷l rz×è¡…[]pŒ2iêÙ–žWêªlUi(]@¦ª¬•«MaÉVÝRî8‰^Š-¦èÒ¿sO{Þ*͉ž·Jؤ¥rëˆ×¢%Û ÏáûÝ©ž²Å’Iž»I]Oi«l[·[×]GÛ3» × =2U=›ß¤¼‹ÿîé;sžaBJUÿÊʈNý'0P;­±åWU -âè‡{¢Kl©äÕy®¯lWHêÚ ¯ Í19kUÔôû² /PäÛ -VTýmO9[Vâ-½ÎÂÒP§9µY*?àVÖ*ËQ»Œ4U¤ßk´ô&ú™©¨´ü”+-_VË'åï¿oº( -endstream -endobj -577 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 576 0 R -/Resources 4 0 R -/Annots [ 578 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -578 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 270.123057 101.433071 251.373057 ] -/BS << -/W 0 ->> -/Dest (cb68-1) ->> -endobj -579 0 obj -<< -/Filter /FlateDecode -/Length 4736 ->> -stream -xœí][ì¸q~ï_ÑÏ–æ¥x?pàl#‹ØFæôîØf ¬ó¿Ÿjñ"ªtëž3Ö>ƒ>ݪ"©ùñc±D‰ê*ñ÷”!zG×ÛÏ—_.Rx+ñoH19Ô?àãíUIòª肽þœÆ ­öîúvÁÝa›üíò§_]þ6Í£®ZÄ贲׿ÿ”Td„ÞK˨"TRFe­ð^[Ǩœ¸Ä9Ì°ž4ÍUfP –SÁ Ò:0ª3 yeæª3´WÄ©`†ŠÖ3*%a‡ÒR²:"‰ŒftÊ -‡JwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¶XE$9l!<§s°Å8#YlÑÁNça‹FíDF` ÐXl‘N*N°1±º >ÌuZZaƒ1’Ó)%,°iXzÉyF§a‹“6²:ØbÔœÎÀ§OXãC`z«N½ÛH£8…-Z{bu°Ï1€v°EzÇQzH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHØbÉqL`l!ô\V[L$Ž Œ†-HÀq1°E“á¸À€m há¸À€nIFÅqß’TŽãÂ…™’ãÆ5ÁÇ” (EŽ 8×€;9.0 ]twÏqëë4Ç´k(8Ž x×’x×ädu°æ¸ì(Œ -†ãï%=ÇÞE7Ò Upè8.V„Ar\@à]$q\@à]í1°:/Ð ÇÞÕ—ÀÜà]m¥æ¸€À»š´ã¸@àÉzà]PˆåÝØ‚å¸ã=tÚp\€Â„†[Âq愈áéÓ`+¡BtXð® -h>V…ò¨q¦ß¢–…r.r\`Á»¨ Ǽ ú €ãFTÍqï*ƒ‘šÓw•Ž’ã Þ…ß`9.„‚ÀqÆ7¡$ Ëê¢1Ž ,x´¤9.À(,d“°:/$ÅqÈXHø¡8ð.†÷ÈqïJk Ǽ+ n«ƒ-&hŽ à×@‡nËôi8«Bj0§ïJ…ætà])}ä¸ÀwÓ°ÉqóI Ç0PÀ Ö¸@"zï8.pQB—Õ9cËq—XP«ƒ-ä‰ã8Ö"š8.ð¶­9.ð¶Àeจ -Þ7;=€-€ ;? ØO‘ã0™‘ˆãoæc¨UNç´tÍêtÊs\à= ÅŠãÌ¥D@§ç¸/’»ÄqÌ€NÇ>ÂtŽ 0‰ á™s:[àds\O:¥8.¶Àä¸ è€Y¦“`„Æ\UÍéH¦yfภÉUä¸Ý3M0%«ƒ-  Ž Ð>i²m9.Àä“MLsXl1–8.A§éfภØ¢¥æ¸ØLóMÏqA”°ÇQ:‘ÜdŽ @ãÐIÉqATóMM _¦ùfภ‰ù&¦xL¿Æ¥ù¦ç¸ l±pQXlÓ™~ NÂ|ìÄélÑ‘8.ˆÎ¥ùfà¸næ›ðèÙ@HóMφ À»6‚™8xž©ã¸˜o³åæöÄk=ü3^ÒŒ3° êµ[Ù÷b´òlÌ@‚|Ó‰ ÀLÌ,ƒc£V¡T’ Hð/ü1šqÂ=úÔ†¨àà1fŸ.2¢[¼û2ç¨ÉÌB²{|ìï¹üúƒïõ/ÿ{yýÕ)Ë -šà_¡èˆæ¶ð¾.ß84´ƒ aà#y’>V#hq º%#1ü¥×ù‰å0ü‡áô¦œ>bNO 7‰ÆJ¥L_‡Ø`cñÁ,ïh°Ä䣭Cub'@~¥¤Iâß~¹¨£÷h9lÌÇx‰É0i{ýòóå×ÿöý÷ÿúûÿŒ¾¼^þü/RZû›ï¾)Ü¿-òÛÐÉÜøíGùת#¿}—g,è*åårú4VÕr¬œ¦Iùœ/ŸC'ßðë=?fb5ÝX¾ÕÝ55ç3aÙV¸¸CšÁ®\w·æ:|.ç7ÿ}ýòýåw_¦`<Ì;ü“Ã…Q-qÅ*þöeyßv˜®®ÀnÖŒ~ZÍ3è5ͪR¨±šW~Ošª;Ÿ t|לn§ì~½C$C¯‡0éîüfÌ«¦×òí»,ÝÏ@„eÐiP^ÅîΣÍÍë„–ìÍybs¾×iÚ¡géÄ:ŽØ}yN‡XZçÛ…VH7â¶6¦ûr:ä¦òzîË-¶ç\¹‡,qdæ÷‚ønXÿXž<Ö+&éiƒk«ÓiMB -Å4YGî¾<§CºÝ÷?ºß³?Ý­›½oÍÕ…M·TˆI·…æ}yN‡æu¿wænxÜè\³Ò3èeî}¤Þb;µĠóg–}ÛV9òX¹¥·.p«n|Ò´pDª!`¼ŠÆyΆF½îÇΕk¸AVj™g&Ó:l·îêL-ûÈO¾¼u6¨)wµòú³—ñÚ É7ƒ^ÆtOK È|À_Ý™çtÈÜðWßyÔ~TåÑï&ç3¸×9>ïžÍûY@m M7ÙtPÚn¡v_žÓ¡v#˜ú¬oxØßjFë5ÿñ6EÑrÓï‚ü%´öKæÒî:KëAïƒ;é³`ñBÛêÎÙÔÎ<§CèFÜuÇè¦[Ÿ³!{$ Ñ!}Gô9¿äˈ¦1}_~FF‡ÐržÕ¾ÚqÄæ£h,|ß_×צ.{Û~lòõÈx+áSîAב¸åäþVðS› -:~¼£`øÚ¤¥;.k£à[«5È_i´3ÏéZw§ºZ]è«ÃlצӌÁÕ‘%E²ÓŒA'Ý×û1§Kyò #ëMòÎoc¹_›|Tó·çæÉiÄjÓ¾ÜÏ‘& ¥\7Ú’9!å wä 3ž÷¸_íÝæ½øÚ—çtøÚð°m㦭µ±»ã"±Ê»µñžëèìè‡ë1M9ømôhÛmÄô0ã]`¬ÖFîÌs:DmÜ%µlŠo|föyÎ~UVŸ îÌs6TÙU‹ÿ`Þ9ܺïÆ7cÚð‡íQ¯yN‡”»¯M\ñ›ùÆ +´ûuQ{ⲓhZ?ãl¼ºö^R™q51…¼êƪÛvÄ…Ý9ÉñÚÖ¶Z{|5öÎ<§Cë†ÿõsøo†„£k öDæ¬îâ}º9_Ñß3›DÒÚD¶v?sµ/Ïéº {8Î(_MòŽ‘·Ï5ãéX¸3ÏÙéö=ÉòÔjä#w>²ô÷Ú>eÜwÁD÷væ9‚7|ÕwZOÏ¡åÐú«w~†jѦ]>ãY¾€ÞæNëÞçvæ9z7|×Oü”ß»ôš]µ¸€ÆâŽ;óœþéÂjèÙS¶rM™5tHøG®MíWq§™ÌÀeÝz©Ïñão¯ÆúzêÓlp® Öº3Ïé»Ï=úF‰ Šn š¸gŠ®OÖÏsÔ­"ò)Fû*gÏpÎß®Êé#IMÞ~=ÿÞzá¸Ï½3Ïéºá«¾0¾c®EÓpâ¿®Gï°kí¹¬>ÏZ µ¤mÇ……kK×ômgˆ ˆ}ÀŸÝ™çtˆ}nõáùü¼ç=™ÓÎ|w@xÀÇÝ™çlhë>î³³©Õ÷ª´-¾p×v2 -SmᥧZ&Ÿ,O÷ì_îkÒ÷á‘߬'/ ï8ëÎ<§CßÇ>‘} -ž;©ZAçê:ÖV±“ØÃ’x£ÐÎ<§Còs1×O‰ä÷ˆ4œ>ykÆñÃûzhó÷u6®1Ø÷dâzÿtïÛyN‡ÞcozêYÕŒŠž£ÚU!-‡e´é…2"~Ëöƒ+nX›Æ@?6=µA÷cë¼x”ÆÖìÌs6”ÆuGžŽÞxV–{“Ä°¶³rú£ß€ñ‰ïÄÆG•$†×l½[`gžÓ!uçÓAÿ¼sÀ¯0lc»vZGìõ<õÞ®´>ƒÝ™çthÝxÙ¶@Af»VSÉi\5€oî½µi]ýdÕ`˜¦Ö®ö¡ì×½NóÏÐ1¢fˆý.ÄÊ+Øó3zœ~ · 6ûæ:[O¨»Vì_x -%>“Ý™çt^÷_¼I#=ñQ”k¾ã‘‡ZÉËùªìÀÛ³öæž…웕üÜÖä´ÌfHKç{€~n5íAó5¶÷åºñD¾$ÿ0îÿá-ݦ½OÓ±ÙN¤Àw3ÉZç›ä-<®é¯‡£qèàœˆ´ñ¯ W8¦}Ú+'ÁïwøÏÿã*ü†Wýt¿¥™Úbx¨èvoÊtœ_d¥Ô=H9¤nHÍáÆj†¼—TeuèLßwX$hÝá8ܤJiÝý3”5þÊ ™xÆó¥Z)"T×OÊ·ã¹{†2ls 6_칸aÏ'5ìû']J§¯ŽDê>^Ó>SAR¹ÙÔPÑêºsØNù—’jÜaY¦MÓ†[÷í—›ƒ1ý/Cƒÿ|Q‘DªÿD$÷ÝPå :TñÛ%í†KŒ˜à“î'áøS–²Hå„Ev{û”Æ -9þŒ¥pcóOmóÕ¤ü6gK»‘̰̉OÏýÄŒ9š‹ÓÕK¡FqÚQ83¹bŠuÙfkÔ¦­ÖŒ›bed#´s¡lNWÅš²=*í@WªÎUa1½­:Õ4N«}ÓœkÂÛ§5½ô•¦}LsQ¹vµ,2Ó‹Ô攥A¨´#º]áw2MhÌ¢oZ-íw† $*ébNLjn6ÝXš--åKªP³Æ\óÕ” -uJÅ3²¡ŽF©O[¤7}Dšüs*v©r[ËÐàFÏ„·‹¥Ò«øíb=“ÚV®6C¯š¦Ü"²â¥JÓ£À÷_èšY–öÒJÿß©Âibžá‡¢©´^=ª'I}³ˆ§•ÔÊ]aâj@‘¯i›êRÞ̶õ¥‚dÒ'Ò-©KM¤M•5r³y‰¤ -yLŪ ®ˆ…åú)\Ù -e6§«b[ ü6W;méª{>™S Üm)·Š]áηVìK‹«´¹a±·”Û`> oÃÀF½8A?ä“T±–e´«'k„Õ0BW}Ĺ·vÞH?þ¬Œqo—¿^†aN«#~8£Ò^½­×¥®ÿ‡4ßãó?ºþGÞî>ŽkYØBnˆÛ‘,íìvŠÓNïYXºM#ÔÔÐ~«PÓÚŒ†2ŽkÝð5Õì²{N© -ƒª¢m…¶Öa#ÖºZ™¦fìŠðÖÕbqdžm›:`ßM¾8œÆ¤m³U8h/‡¡êL’¯M‘8JIë-é&#ÚBqØœ0VË&f&@¶')UŸì§¶)L:WsиéÐTÕa¶-:ˆ«h’À6íA[0©ö¤qbcôÃÝëévMv¹$i¡ö"}k¥åÂ’´/5á›–1Ç\ÓAî¼µ†JY¦é{¦89µkTׂ&-Fi¨Ùƶ=)eÙä¤y (t™Ì²R¢Ï6¹1[®–áŠ[K¾øUú׶¦mÛ*¥mÛ*n–6•›G¼â-™âÙξ^¬)‹/™ks)Õ”&Ëæç¦é¹óh{m.ƒª!®z¦²f³³”ñß#}瞧‹Bp©òßQ¡n§Šš”7Ú\crw‚sàJ›œ{ -M¨äpžõØÉ> -endobj -581 0 obj -<< -/Filter /FlateDecode -/Length 4971 ->> -stream -xœÝ]ËŽì8rÝçWäz€Öð|F/ Œa·ánŒ†·òÞšQm Ç ÿ¾OPâCT¤¤¬ª¹Èq_Tg*‚yxA‰L}Uø÷ƒÆÿ"é)Æ<]o¿\~½¨)8…ÿrŠÕEVÿÁ]µ¢09|t×_X`Ã’3Á_ß.øßpÙ'»üûo.ÿ½Î£¯fJÉí®ùÆ*²Sˆ!('¨TJ[AåÜ‚q^Py=áS’T0Ã2´U˜A):I3Ș(¨"Ì°´ÝªÌ0A“¤‚:¹ ¨´‚Ú(%ê`ˆ"²FÐi7y4º—tFO>&“D]€N#é¬| í´¤K“÷ÞEIG°Å%¥%ƒ-N)I[ÈÆ é£Äàø 3ª‘¸ÀwµÅL-éÀ»Ú$%qïÂop“† qæ·I+@WÔ¥I¥%.pà]Ð’‘¸³ð¤"˜DÔ… @Ò€Œ'?TâÞÅôž$.ðà]嬕¸ÀƒwÁíu°ÅF#qüè0l…1 guRL!éÀ»J£‡%xW©$.ðà]ž6%.ð&J\'xÁFâiJ!x‰ |RÐÁåu~µ“¸ ((KÔÁ -$që)Ù%.¶Xc$.¶Àe¸Á”4¼o1<€-€‹l§(q˜lŠ‰Hâ‚à<â1´ª¤ófŠt-ê"t:H\M˜ŠµÄˆ¥¦ˆA/q~bwIâ˜&‰ B‚-*`*š¢g.é4l“-q<]è´–¸ ØoPâ‚h"¢L¯$.À XM-éHqœ%.ˆä'v%.À0A¤ ¦u°4!qú‡ƒm'q‚›sDl±Ž$.ˆÑp¸%.ˆ¶e$.69Þ $[àñI\”ŸØM–¸4R$o’¸ã’ãÍ(qA² -ñ&B7·zc†9_-¿¤½µº²}]½½Í¥žl¯-£úT£Ç¾Ñ¹Õ »m]ùÙÎ¥½ò½³ ·¹Çµ_ªñ}ù†ŸO#øGT¸”o—6r¹Ýsyúë¶oy¡…æmËÉ÷^ÚÓç¶Ù³ƒŸ?Ëv¨v/ÜÞõþ¸íÍ•]·ÁØÕ&ßÝ[iwÝÚ½¤¯ö‡¡ßîôs¹‡Œu¶ãË^›Yµi³ž7Ê}ŵ­ZK|Õl£e¼m9ªñóøcÏ6~odeÛÌk¬—ñ2úÖ»ÉÛnòÃÕS~ÅÉ<ï -ú?Ñ¡pÕÞ¦ˆ`Ú(Ñ¡ÈУ =Ëí¦—¹îV˜ü¤?8*,ô“!PœÄ²÷K4èYçq„ÝëQnŠù9³÷¢=G¹Ÿ’CŸI¹óDYZ¬LJö°…L~‹œ·wZ¨Ä®nŒscç*QWŸ -lB†´•Þ=Ùü,…וßÑòüFšÎù>rs¿O¬ù¼–§h0Ê"aj¼M—–^Ø­ct#EN"~^èÀ;^t³ þ²ž,WŒ½ç”l'}~3'ñ3ð¡Ò<²ï‘/Æ.®ýéÌó$ÁTå>4í÷)4ìü9»·ëѤú ¼sADZyŠ]ÊöåËAˆˆ§ôz*2B›>‹TÑ“æi9£&æ{Ø"‡¦”\ z¨p9ŽÞÎô%~h³¿”¢Ëâ‘tjË»\3,e®¸kèm7 cƒ×å½­íËvd”‘µ<ÃGÒr™×}wm‡ïfÉ“îŒ<Ýî£Þ÷­‹¾vÜC¦W•ù­Z"ày2ϳ!Óë]­­8ÎØ=²öP²YP¸ïËòÝg K·ôu4¨u] ÍwPØ|? Ú3böQx.ÏÓ¡ðœ¿(F÷8ë¥[1ž¡KH¹ODÖØ“=—>P_}ÜÉ x™ó¯ ªQ›éã}[W<)¬NÑé{ðäãî“yžy>äØ“ãÛ#úºžÝÌdw¾KÞ­Gˆ6¼#Íì±ûuߟ#3Ô¿ð¡–¬ÿú£öz;Q›)©hø}Ì}ôžËótè=ð+Çw†ˆÈ}z¶‹Zöø®GË÷Š‘çCºr/‹Ï,ø´ÕÓÙ?Éݳ·ä9ŒêdĆæ;žFìÉ<φذïo>øј÷‘èí;x7ŒŠUzÚçÚ`áOæy:äîû¨ÿßVt>³¿'º ZGß4»Çãþ“yžÍû~ïÆ=ð ¤Ù¹"fgdô«C¥‡y´ŒoùŽˆùîüYd/[ÛG>Vn­÷¸µóIùÌ¥ó^Á}4žËóthÜ÷c7ÈêÞ„¨({ùX<]Ÿ+ïôîn¤öŽ÷òï­mÐ?¼žzve­¼úþyIFf|‡¿z2ϳ!3ø«Ÿìo}³*t÷ö$h=–a¯SÍWV±Þé}Hõo^ù?x"ßñìþdž§CèÁºëÉçFÕ38±#GEi~š¸á—awZ~âh„ò 2„Š›qztN±ùQ4Žoôþî¸dõ®Ã½]ˆ'ž–ÆæsRrüNâýöû¨=—çéP{à§*-ýLx´®ì½ä­peï'o-ûìíAÒ[0•cÇgö~û¤âÞZ¬è Œ{Z‹Ç0D—ã¨:zBšÔã|y2ϳ!/<§wÇŽc]èÙÕÓÁqvßHéžš>¾·Z>"ö‘ÕuÚAÒßìŠÚwþeˆ“¶ÄïÜ ø\ž§CðÁúé|Ê3èzdÝrõ|«_k›*:¾Î(ÈxzZß?e¾ež¾ ÷y€²q”½+šºµË³ÑÉýWyx½†àêibÜÐ|äS#-éÒm:7´¾ÞÉ™TáךjùÅG/ó±›¹[û‹%ý¯¹Ã¹èD·>!Ÿ‰®²8šrÑÄoÞõG‚˜¨âp fáòUÕ²àR›AvO†))­+§ÖuGð†­ù«qån8¿+Ùø¬BR%‘ͧõÍ Bäôz+æ»W“^Äü»q²Ë•¯¦ð–á*lRŸ¶Ø2ŠµUÐm…ª«®‰ {4ŸC[›Î7a5½o:ÝuI‹xÍœÞ.¨ÖŽÒ7Qʧ™—¢JëUe¶3*U©+)k‡Pí'At»ÂT)šÐ™UßõŸ ; cÕt©¤D·‹˜n)ÍÕž -5UlYSiùfJƒºµáÙ­;\%ð¥tcDÙòu-æ3Cj_«ØáÆl„·‹£:›øíâ‚Ú5®P®@¯Ú®Ü*Žªá¥Iy}`þ†¡Yd|¢&ÿ¦ -W¥Ì<ù‹¦ -)>·]µJxlVñº‘z¹¯¬CR h --m×\:„ŠÙ¾½à/é™tkêÚb+i×d<¤b“T%µXWÔU1SXiŸÊ•½P#P]» -å·•¸ÙéêPuªVæuwWËmb_¹ó­‡Úãš8®öÖr;Ìá-Ol4Šú±TÒÄFÕÙ®UÖ ›a …&n4ú–Åe´VvâŸm)_Û “fÜÛåÏ—<Íë Ÿâ­æû{¯K_ÿi~Âß]èú/åˆØy7ª²…©Üº÷3ÿ¾K<)æß{)Â:l:¡¡Žö›XÇ–Ö4ÔyܘŽ¯©eWØKJ]TWm/t­ ;±©Ð5Úv-ãv„·¡«#óѾiö|iËÍñù&|¸p± -ýíà26mSòµ+Wœ´MéÐ’é2â¢/—]…©Y¶2“ÙWR›ží§¾+,×Õ]tn_Ú¦£6ÍöEY¾‹. \`Û_ô“î+M+{£ß=¼>Üï‹É¾Œ–Vj¯Ò·^ZoŒ¥•ðd©Í—~}Yç›)ƒ·µP-ËvcÏV'§ æZÐê¢Çh'-Û’Àõ•R‘­*-E¥ ÈT‘ÕC±É/ÙJ³ä;Žb+…êS4éŸû–v}¯ÔFt}¯øMZ.·ÌxÕ[²Õr=œC»;ÕRV_2 -2ßç®Rj)m‘më¦uÝe¶½v·AÍß> -endobj -583 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 570.123057 101.433071 551.373057 ] -/BS << -/W 0 ->> -/Dest (cb71-1) ->> -endobj -584 0 obj -<< -/Filter /FlateDecode -/Length 4858 ->> -stream -xœí]]$9±}¯_QÏH›øûCB< `¯@Ý<¬x˜®Ý¡^¤…‡û÷9áLÛ‘ÎȪ¬šžÙZÄ´zº2ÂáðñqØ™åÔg…Ÿ¯4þKNO)åÜùòý釓š¢WøWR¬.Šúøþ¬•‹“×9$þž6N1{Ãùí„ÿ†KžüíôçŸþ±Î£ÏfÊ9íÏÿüŽTÎN1Ũ¼ ÊP)m•÷SŒÆAô„&æ,©`†Î¸­* —“—T0ÓU‚ÖEm·ª 3LÔNRÁ }TZÁm”u0D9g Ó~ -pztFO!e“E]„NG#é¬BtÚkI—§‚O’ÎÁŸ•–t¶x휒t°ÅÙ%]€-6X%ê`‹IÑJº[ ¼“]‚-@Wu°E¥%]¶“ÏÉ9Q—¡S)¦­Î(?ùd­’tZOØ´¢# -¢ 3°%(ŸElñVIga‹Ã¤OxSF«¡Ñm•Õ’ÎÃc¢u°/1€ °EÅ QFÐD¼$q€‰qrÙ8‰L²“K.I,`Rž\ŒV¢“ýäBŽX¥¡3Z"«`‹wAb«a‹ÃÈu°Åf'q5° $.°¶g%.°`[´H\`A·Ne-qß:¥ƒÄ„ 3•ÄŒkSpXP. ”%.°à\ À‚t1ܣĬk}0XЮu)H\`Á»Öi%qïZ‹œ¢¶À€'«“•¸Àw­VQâÞÅ02 W'à0H\¬L&%%qœÄ¼k"¦Q'ŒA+qï&H37x×xe$.pà]ãL¸@™À?JŒÀ» /‡°+qæ{茕¸…Ma‰Ä¼«3¦?aLƒ­&r¸ÀƒwuB÷‰º<é ã^žtYâÞÅDe%.ðà]Ðg’¸?aF5x𮶘©%xW›¬$.ðà]Ä ^âÀaÒ$.Àü6i芺<©œ’ļ Z2`žT“ˆº8HZâñ¤‡J\À»˜Þ³Ä¼«¼µð®r{Dl±ÉH\€¸: [aL#X”SH:ð®ÒèaIÞU*f‰ x—¦M‰ B$IÀÀ Q°‘¸ $7åƒÄ!+èò‰º0áÚK\”%ê`‹‹NâÖS¶9I\ l±ÆH\ lAÈ q"‚)kDßâò¶.âúÀÁDŠ€É¦”“¸ ú€õ¼*é‚™R]‹ºŽÄè&LÅZ⬥¦„A/q~¢pIâ˜vÄ [0T$.ÀT4%ƒÈ\ÒiØ‚ [âDºÐi-qA2°Ñ ÄÉ$¬2ƒ’¸34Ö’pµ¤sŠÖ™Iâ‚äÂD¡¢Ä&Xi‚)ElMH\€þ¡Å¶—¸‹,6±Ìu°Åz'qAJ†–›Iâ‚”`‹QFâ`“Ö›Q€` ">‰ ² -…ɀơSJ€Ö›ÆI\€qIëÍ$qA¶ -ëM,ñ„q›m õf”¸ ;Ø⢈:Ø‚ ) 㜄õ&ØIÒØb²“¸ ‡@ëÍ$q ¬7Ñ‹[‰Ö›QÜ.ïú f’tà]D¦Aâ\`½‰ÈVZÛ+¯ˆÏde¢gw ¨×cn· ¸³U÷ È—ÖHâ¦ÌÄÊ2q×Ó*”Z‰Û -ü‹xÌm8aÞ}â[Tpˆ˜3S¤Ff ‡è¾­9z2»“lÞûç_O?ÿˆÉ÷ü×^VvÊêF€PR;ÄW&£»=¢oæ[mÊÄL$EÂÀGò1w#5hqÙt##1ýÑ¿óöÃÊrN¦#7ÖEöh˜©¥~0Q)ÝaQŒõ¡9³›|wž›&GDùÜ>hÙÖî*^o÷•dëÝyÞËVÛº5SÒR-`ki‚v@ÂYÞ -‹ND1il6Äúʈו[%þÕ‡`±bÞ[n—Ì|Ä!„nüùÃ÷§Ÿÿîë¯óÛÿƒŸ>¼ž¾ù…RÎüò+¿ö#xè‚¿°ß¼t¹qËß]èríñ ™Î,-•4˜4{¹a.¯–‚Tך”z]ê{]—MuQ9f±…®%]){)÷f;;ÆëÒËÊÁgkÛPb ÙvûË¿œ?|}úõ‡5HJD~®Žeù\}8ayN‰GºÕú;»s–néÊ—;aÉàüåí—!Ew€8%ój70u0ϳ -+½#$õc‘Îݽûnd³Œ(ú«èw)–Q õŒË^Gʱ²ta]_óÛå¯öÍy3),™ Uבz,ÏÓ!õz\w ãøçhj=<¢ˆqš˜~DúÈy‹È†Àa$T¤|É×7´ÿH]{ˆ4=¾óô0 B‰pk¦?˜çÙinÄ„;³î8ë·^Ë[Yã.;äQo}ÉðÝ ¿ü!ØÞ¿R>˜çé|#V}ï åd³m‹)_X¬÷>3> ÂwÐÛãPzèÂ9,Õn­Éæy:ôÞˆ]oŽÿ÷CøCè=£æZƨDðIÓ…¡M¢½;h|`×ñ`ž§CãøtŒö‡µþØKeM]; áÇëeV_-ûãÂel_ƒd?Ž•‘kÕý{ó<ríõ8öÓb¬¡qnƒBý>üú)ñöÕýƒÑOcšœkØk=˜çé{,~m;IQöèˆðŠ.ؾ'í®sŠäÖˆ(;ò´Gû:ïâ—(áÑü/lDŒ;I,oÛñ’æÑN˜—†²Ý CäèçM„´ƒÔîuÌótH½«~bÇêEË8ñ¯s”Ú—9?Ý'9­Õ7”CéÍ»Vœ8Ìcžk{¨--ŸvÚFmú¼+ÄÄ>ÏÌótˆ½ÏÞØ}¾8ïÓ#™§]-ÞŒZd4»b܃yž ÍîzŒû©«)~§sä¬#wmW³°ë=\îñSï¹óß*§{öçgèïŠãM<Òùü#y}ì³Ìót軧þ'òÜ=;UWйY)]X;÷Ò>ßÞÃ’yLzpÅu0ÏÓ!ùÓö\’H~½ÆémÐs9›8|ôÏ?úlyÆÀ±Ñ0Ž¢¾âÛA/‹OÞ±=˜çéÐ{ãÙб·†U‹ô$ʸҩ{Ÿ #Gñ§B8‡U´™2ïEü-Ûï|âF´iî[6Rº{ÎKF©โƒyž ¥þÆs{ˆ«;<ÕÓ/ !ã^ꀤŠÖ‚L§6O~é=ÜŸòÏbTå&£A–îRåy:¤^kÿ{ç`'`å5[ýÚGb{âºÌ½{ŠMn¸Ý;h}äûF?Í=X=vm=÷ºFÉêYM­ÖûªuÜÙeo=3D#jé¹úÕSƒi®<»:îÕ¸îuƒŽ5eïwgo¡Ž,U¿spLj3Œ¶bsdíä‘Ðpÿ¬Ù¿ó-ÿÀžìÁ%íÐК®ÝkaŸÙS¦yzµ õäå9­'z©-œ¯7ò¤vv…¨··åÎû Ü“½ÜR‡Ÿ=ÞYaÎ×ì­i/,γLoÖé¸=Üþ’®ôÅ!ç[îüoŸªò-ig2;T÷]ñ Ùp™Q ÃReXÕã²¥óo—ºìâ;_üIËÓßnË –‡¢H·å”¶W?‡â›+v$m÷ìP½-îÂÚG}bæQ,ØFÑåLÈBÙ36{™k¬uœ‘Ÿ‰aÌ¥öïb“¯L2®m%{\ìiKûlÛóò7Èí(X«i^ÞMÇå÷¯–þñÌ&ÍÒ-}Xw wünGß4 ifsÅC¬enë×5_/²¯ôrש¤Us;•]û`ì‡fS`m«~×Ýï5}³¿ŽßÔû¹–Ûò3œùzO$]õ™ßøl°·óç&‰—Öœä9/Uÿ$¡Í†}jÅ7#ž½é>)ë8ʬo«tÏKØ}©G n¶ÞÙrÖ%û÷€çéÄg¸.D¹·rS¿ó¬y?Ï»d0Ê’C°MŸ—^¸ZÇFËœDgËzðNçö°= VÁšÝ À¶œžéŒâ¡Ò2²;éýq9ò/zdôôº:´/³ÓÝLrƒØªw¶§ËõpîÜÚ(’ÞõaÆ.éxLÉ}äóÝìÆŸ…å»a™ï2[›,jØÜVâ®Oø9Ž©@(î¯söëLVå)ôò® l~fófyb}‰7xLÒʯs;³g5·çåIÊÒ†M7þp -å˜W]ŽúV!MÚл@ÎÁM´‘󙎖MÊ:]»MGëóNÎò•Zªå¥*ŠÎY§3vK·ò‹%ý¥Ã¿Ç’ÆM¡.bÊ TcäùQüv¢`8Aì\_ Ý“pù¨ZYN·…U•]0uÇ))1=ªåc_}Ñœ©þ¯­¡ü¾f£ƒIª‰l9šs®X"gÐ[1µ^MzÓKDÒd—«ÐL¡ù§ {Ž§íöŒ7ÅÚ*&ô[¡bÕu±qÕM‡N7×….l¦s×iÖý5­‰¬;¯ /'TkGé›(¥WÔ¢ªwj2ËŒÊMêkÊÖ!®õ“ ºœáWªíЙMÏz€ž…©ɵt¹¦D—“˜n)Í·žŠ-UêYsõ|7¥CݦæxAva;‘ÞŠÄƈ²õãZL‹íÖ×*1ܘðrò®Ä.~;ù(¤ö+”¯ÐãBËÊmâ¤:^º”Îú›?ahVŸKÿÏTá›”˜§|ЮAŠŽÀîW½›M¼v—‡Æ:Nò€v±§eîÒ16Ìr餄ôDº-uóØJÊ\Æä1Wóˆ¤y¬Åº¡®‰‰ÂªWr¡ªF º.ö Êo+q·Ó·¡êU«,hwßÊíâиó‹cëqMç™7{[¹ óUx)›ÅýT+éb£Úl×+cÂnH¡‹;¾q­Ê¡SËÇ>Ȥ÷rúÛ©LsÆzC‹Ë€ >®ÏÕçÿGš¯ñû÷“;ÿ¾ž=ÏãF5¶0[Wb>“ÑËœÒA1½Ü© -Û°aBãíw±N=­¯hhó¸1Œ¯]Ï®:±×”º1¨nZ.ô݇Llt¶Ì3þŠð2x±2ŸÚ7}ž/mm-–é$ñj.xsp™ºÎö)ùÌŠÄ%íS:´Î°Œ¸à…â’U˜»e+3 ¼’æz²ßñ®°T»`a]Ú®s}šåEWYiK‚Øò ^°Ó¼Ò¼²G0úááõÉý¾˜ê i£ö&}ãÒÖ0’6“¥¶\†õe›slYgÖÁÛ=ÔʲlìÙäô¡ÑC ·ºàeÒÔ³- <¯ÔUÙªÒ:Q4º€LUY+1V›Â’­º¥´8‰^Š-¦èÒ¿qO{Þ+͉ž÷Jؤ¥rëŒ×¢%Û Ïá{ëTOÙbÉ$ÈÏݤ®§´U¶­Û­ë®³í™5ÃuCBLUÏæ7)â¿GÆÎœgØ…RÕŸ²3¢Sƒ&j§£5¶¼S¥À!ŽÞk]b[%wç¹¾w²Ý!©{'¼&tÇä¬UQÓëMeÞNqžÃ®+XQõÕ§r¶¬Ä&Ýçak¨ÓœÚì •÷Û•'ÊQ»Œ´T¤×YZ -z½…+*-ߢNå–4ß>)?ÿDé÷¸ -endstream -endobj -585 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 584 0 R -/Resources 4 0 R -/Annots [ 586 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -586 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 142.623057 101.433071 123.873057 ] -/BS << -/W 0 ->> -/Dest (cb74-1) ->> -endobj -587 0 obj -<< -/Filter /FlateDecode -/Length 4741 ->> -stream -xœÝ]M¯äºqÝ÷¯èµ'óû0¼0àÀ~AŒÇÊÁáGèAñÇÊûÁDe8PA&˜À±€ -q0ÞkŽT´ƒqÑs< …„NIŽ´€-Ö8Ž ´„-=—ÕÁ ÇZÁ$à¸@kØ¢Œæ¸@ƒm ÐÂqÝ%Ç|k„th.Ìh0®Îp\ A¹€Rä¸@ƒs5¸“ã ÒEw÷h°®¶Nq\ A»ÚÇqïj#Ǽ«5r²:Øs\v´ šãÞÕRxŽ xÝHq\€V€CÇq°2¨Ç¼«‚0ð®òXÐ5Ǽ«,.¹Á»Ê -Åqï*£ÇÊþ¬7Þ…XÞ€- XŽ 0ÞC§4Ç8Ù à–p\`Á»2bøcú4Øj!:Ž ,xW4«‹ƒô¨q¦ß¢–é\ä¸À‚w1PiŽ ,xô8.ÇQǼ+5FjNÞ•* -Ž ,x~ƒå¸p$Ž 0¾ Rº¬."†Àqï‚–Ç…À$¬Î’ä¸d<ø¡8ð.†÷Èqï -«5Ǽ+ ÜV[tPÀ¯Ý–éÓpV¡Àœ¼+$Z˜Ów…ð‘ãÞ¥a“ãçI Ç0p€¬8.pÁ Ñ{Çq‹:¸|¬Î 8¶xe±:Øb¼á¸Žõu x[´Rx[à2p\`ˆÞ7;=€-€ ;?0°ž"Ç`²!Dc8.ðÖa>†ZåtN Áƒ®Y]€NzŽ ¼7†bÉqæRC@§ç¸?»ÄqÌ€NŽ |„-è*`(‚‚gÎé$l“Íq<]è¤ä¸ (Øoã‚ f™Np\€sIT5§3‚æ™ã‚`Ü@®"Çè&˜i‚)YlMp\€ö¡É¶å¸“L61Íau°E[ÃqAŠ¦›ã‚`‹Šã`“曞ã‚(` <>Ž ¢p¹É€Æ¡‚ã‚(æ›Êp\€~IóÍÀqAÔóMLñ˜~µ£ù¦ç¸ Øbᢰ:ا)2ýœ„ù&؉Ó9Ø¢¢á¸ :GóÍÀqÜ Ì7áѳ!@óMφ À»6‚™8xž©ã¸˜o³åæöÄk=ü3^hÆ؈õZŒ­lÈ@€{1Zy6f @¾4Gbƒ03Ëàب†U(¥`Ãü ̬8aŠ>Õ!*8xŒ™ÁÓEFt ï>Ï9J2½‘lŠýíÏ—_Æà{ýóß/¯¿#eéOÂASøWè*¢¹-¼o‰Ë×ÌÑ2vOZ*ÔÇb” ÎA62 -Ãý»®,,…¡?-‹•¸æu¹sÖÖÞLïl´NåÃíAW¢â ÊëKE̱ÇfeòYÞÑ`\‡4:*¢Pô 8¿Â,ÿîÓ½i¶b -ÔæC©m2Ļ͟~¾üú?þø/ø`ýÓëå¿ž%®õó‡†? æöÛ$tÖMߣì3ŽítlÂümæt~)Oé(_Ö½Té夓zúÖ²´ÓorHMŸT~.o>¶M9I?æѳN-Ó$;èü[×Hvå|¾’k¾¼Ö¾1?åûŸë§/¿ÿT:<5•›ÛÈ6mÜ?Ã4Qa|qQëüþOÿþ¯ÿù§Ò:T+&ÐYéªÉ‚©’²¦Ú‹T³ø¼Ni`ÍX~U ø Ìfá15ŽøòÛdïöðÙî-ècV$ÖÞí*yžé+ߢ‹Pðx§d ©æ»ÂNLµæo3÷'»¡³ûòs“æËü=Ë•*}eü|™¿c•öeÆô×eÿ0LÿÈ}pÆtÛÚ>l#߯SÍçKeúæ¼>•½2Ì53bD'ÊúòœeÅl¸Ù;(3¯Kt™™ ­hØö¥j­TË_J+`F4U ›[+,[%1ªñB¾6H¸áw,e%;å©ísºdf´Iæ< ÛÒùÒï\¯ýeuZ1½ñ­~©QÝÔ¾!ß½²dIŸ{ƒX–UмÂâû«É:!f…}yN‡Â>1·Œ©°ÅY/ëyÕR[îc‘Õ¶dÍ¥w”—Ê]Ñ—)ÿ43Yž¿õ‹ëòtضuÁ“©înÕuøªöéDÃÆûÈëËs:äømKúeM¯ÐWµìj$ÛøÍyÿ¹f¶°ândÆî×}Ψ¦ü™•\^Ë·ïµè­|D‰Ÿ"( -È/ÏÙЫüÊTƒ¯ËïÌ+Ÿ›–­f-{|W£å£æÈ‹XTë¼Ì>3ãÓfO£)µ<Ù3“Û²7å9œÕñˆÕÅwìFlgžÓ!vßß<øìœ÷žÙÛx7÷ôŠEz³ÏµÚÜ?‡ïÌs:äîû¨ÿlwãìDwBëaïÛ@³»Þß™çthÞ÷{WþèÁÎ1;=£Ž¥¦ÞbµĖóg’½¬m+yßysoÝâÖÊ'¥‡„ í£±/ÏÙÐhöýØ­û…5çRË<3ŸVá¸uwgjÉGnxrugal„l…þäe¼VH눬éž—xdšüÕÎ<§Cæ¿úΣö» ê®Øã{ø†gó~6P[ÅFieÏR•Î<§CíA<õYßðnëîU™F×òßZ[%qis9Šõ ÷Á•Ÿ{ç*^¼ÐîÝwæ9Bâ®÷ÚÕI+Kç¹Í³¢8ÝM\ñKj±„h3§oÏŸÑ»’ªYµ±¸Ôaó½hlWÔþn»‚k±Ö¡]yvÇÝRS|Ní¿ÓЂÛ}Ôöå9jíŸ*x´Ô#¡–ûiÄëä»*úÌ­¢Ü~+o¯ZóßÞ*˜Ì±í={·¾S±‹e=úÞÓÜsÇtÍì²íUGwH­¼Ÿ/;óœy÷é›ùíª¯3-»¸;ØŽ®íŠ”ꮡªçÃ[Ñò±÷D×Í’¾ÛˆÚ‚+ÿÒ‡AjãÝQÄ©3Ïé|?}§ìA×=qËÅý­:&à—6et|™P0~ž¸{š×ŸßOßšë<@YÛKŽÖŠÚ*vÙ;#êÌs:ôõù›G÷Ú 7KÙXµÿ°?VùŠßÏ=Ó“7PZ|E8BC D­FéÌs6”ºÿò<¿1"Œyf%ûy~‰‚µ®Zºá²: -~´ZÃÉû×uæ9]ëvúp¡©Õ¾:Î&pmŠf ®Œ,ɦƒ"ÝËtÌé(Oša$½&ïü6Ÿ÷¥ÊgJþº¬ü”EöóTMòyÝlKâÊ&ä3ž÷¸_õÝæ^|õå9¾<¬GÛ¸j«»ÚØM¸ Vy·6Ǝö8=-”Ï3?17Úv›1=Îx7ëÕ‘yN‡¨ƒ»Äá±–¥øÆ÷Ì>ÏÙ¿ªxÿL°3ÏÙPåV-þƒyçîÖ}7¾™{Ô?ìˆzuæ9Rî¾VqÅoæo¬Ðn×EõÄe¹'K8¯®¾—”g\UL!­ºÉÏ}·¶uÄ…ïºs’⵬mõæþÕØyN‡Öÿëçðß ÷®-è‰ÌݱºgŒO´é>‹õúˆöžÙ"’¶Ô*²Õû¼UgžÓ!õ öpœQ<¾šä#oß_Ôl‘¬ìÌs6D†¾'YžZ|Ï…ìÌÓßÿ,D÷:óœÁ¾ê;­§ú]ïü Õ¦M]>ãY¾ÞêNkïóyN‡Þßõ;~Êï]z͇®ZÜ@ãqÇÎ<§C㺱zõÔ‡-\“g þ‘kSÛUÜã;á¤X­—ú~8v¹<‘Ý™çlÈ}wp~æô£Ú¬'ãÖ…½¿>ãoïÆÚzjÓpn| ÖÚ™çtÈíó_ï}£ÄE7 UÜ“¢ë‹õó5kFLj<Åh_Åê®»ó׫rÚHR•·]φ·^Äîswæ9R|Õêͪ+~«ÞVöÑoìQví=—ÕæÙ‹¡Þóž.º¦o;CÜ@ìþlgžÓ!ö¹Õ‡çóóž÷dN;[|ðÝñ·3ÏÙÐŒÚÚ%àg§Sû/‡·mÃðÞ«(ÓM÷ú“äó+ƒÇ¥‘é•Âéºz–D~³®œá÷Óüª{oÑR–¶ù£÷ÀÇêÍùs‡I ¶~Ï~zü•þZ •eã”´Ç¥W$EOÛBl½—[}<¸ñ-ÍnöÌæÕ¨é¹=)E~¨Jïòï¦Oã¹™EŒïõ6_K³ïõvSzz¯75Á8&SZ7}ÆsÍ¿Çó„Ä sy4~S~º÷é'ýâüv.»²ÇÌïÏ×`Ó5¬šñ—‹·7‘ãWÂ…A*ÚóêÌ@KN|¼Ò–*Ah#S³É±¢åu#ç¸sè/9Õ¼™¨ ýÅho™i§Ñê`NÿËØà?_d4Õ?õþiã?1Š1˜ÙVüv¡ #6fPbÎ?E>—‘)a–Ý.´ëAlSj;ˆùgÌ'§—’O?•MWCùmÊFr‘éqKŠ©`FˆœN®Åtõb³˜6Ï ƒž\6źd3„儦N[NX2Š¥•Ð®…¢*®ˆ•IöHÚl)W+Âlz]u²jþ”Vùª9÷„· ŠÕ­ô•Ò–}éT©v•È2]³Ô¦”¹ALn'Ft»0ÂD:µAcf}Õj´ñÑ$ H&§‹)#º]ØtóÙln)ŸS…’5¦š/¦¨ë+ž‘u4K=í\õ¡ÓÏ¥ØQ ¤¶¡ÂZ okrO,â·‹õLj[¸BؽZ¨«ófq/EJ+§_èšIFÛÆÐÿUØ,%æÐÎ -2 >*…PßÌâe%Õr—YÇp5 /i«ê’ÞgÌÖõ%ƒ`ÒéæÔ¹ÆÒªÊ*¹É<"©LK±Ì¨Ëb¢°T?™+k¡HF ¸"¶Êo q±Óæ® -°§Âœ¬ànóy‹Øeî|«Å>·¸¤}¼²½ù¼æ“ð6l¦ôC*¤ˆ•È£])¬Ã@ -E\hôm§ÞšÙ‰ö&N?K'ãFÜÛå/—q˜SÚªˆNKÚ–²öºäõÿæG|þ÷b®ÿ–öAšÆq%2[¨Ì­ q=’Ñ&Æ¡SL›'aî6•P™Šö‹X†’Ö&4äq\©Š¯MÉ. -±§”23¨ÌÚZhKVb•¡«¤®jÆîoM-fGæÙ¶)öt¨ÓÅ¡M;h%«pP_CÑé2$_«S−–!]Ò«Œ8¨OŠÃªÀX,[˜I€¬ ÉUOö›º)4•UTnê¢3e˜­OdãUTIàëú >±‘u¡qacôÃÝëévŸMv©4S{–¾ÕÒ|a$Í„ÇKõxè–‡yÌÑW:H·ÔP>—®úžÎNNéŵ0‹ƒ£•4”ls[j’lQh(2]@&’,ŸÑ'›Üœ-UËxÅ­%Ÿ}Š"ýK]Ó¶n•\‰¶n·JKçM#^ö–tv€l g_®N””Ù— ŒÌÕ¹³Ô””:ÉÖe›eÙi´½V—aŠ!®x¦¢d³«”ñß#}gÊÓD!¸TéoŒŒÈPvÄ@m¤×J_#¹;Á9pˆ¡ý|½ U¨äî<û±“u„$ÅNê’ЃÑZx ¼m˜ðE> -endobj -589 0 obj -<< -/Filter /FlateDecode -/Length 4902 ->> -stream -xœå]]äºq}ï_ÑÏ®ÌïÀðCÎ bÄÈ"y0ü°Ó»c#˜kà:ùû9E‰d‰*©Õ³s g³ÓªâG±xxX¤(µ¾*üü ñ_rzJ)Çà®·Ÿ.?_Խ¿’buQÔÄoôW­\œ¼Î!ùëO$°qŠÙ›®oü7\òäo—ÿúÕåoë<új¦œƒÑþú÷¯¤rvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS€Óƒ¤3z -)›,ê"t:Igí¢Ó^Kº<…|’t¶ø¬´¤ó°Åk甤ƒ-Φ(él±Á*Q[LŠVÒEØbà,èlº’¨ƒ-*(-é²|NΉº J1muFùÉ'k•¤ÓzòÀ¦uQè¤-Aù,ê`‹·ÊH: [~$xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØßÿrùõgL¾×¿üÏåõWe§¬þh%µC|…a`2ºÛ#úÖh¾ X›b,{(1Us7Kƒ—m62 ý»n?¬l…©d,rc%hõO©Z*÷”Ò–ÁXš%»­ç¹krDL¯íƒ–me>bõ2' ¶>œçÃm ˜Ä°hFˆ¹®W4ò^âí|V/­ˆ7õÒÞ‚ ;ÆžÌôÁFÛÖ£™:¡t(e˜ùˆOÒ-MýÌä³| Á - xD„il64ƒ‚±öQn•øŸ>]@¦‹ó>}»dæ#¦Ó˜¿~úéòëùñÇþýŸ^/úRîëoÐøëÓü×ùåïrí—k¯™,iÂü—vžëçú×½vݽºPØxIcÝm(o´+oeZeT½šÿb)Uê¢ß֖ס-ªç¯å6[2³3 -mø:Ô{[ä¹—Që«iŒZ|4ú¸ùö·¾~úñò»Okp–UÕNÂË4³âù\–oÁó/ãœÿßÁxJÀ»Ù oØç[Oßä£xþÑg/3d -ãªv)+C—îHU ÒM&¥Ëâò»'ó<xµ9$áMoUdÅ=‘õÌëº÷Üç5im꥓̊À*ÚÌN™"þžíæÚ¹Œ÷…T†nqÜ­k¥Ö=ŽÒsyž¥î˜c÷§Ľ0„Tn¨œ0 ©¢µ i;LÏG|yÊÐØt#JRGË -m¿ÈÔýÑóÓRY ªÜd´KÎÝCê¹™çé|ãZa=Ûøç©{váÙ‘ÿG.¬e \jKc†ö9V‡êbTц8Ô=ðùÑh,uÚmÞÓȘ~ÇNïÉû¡ÙXÛªßu÷{MßìC¿íôsmCÁ:ÙñùÐgqã3Î$µ]im«îîˆJ¤Çð+bN<™çIPƳê#”=rDåh+G¤ìóp|{G¸Qÿ\GHúçw’`G -Ç­á]¤zó8RÏåy:¤†C>üÖ£ï>¢2ôöxèsƒ~õ¶¶¯Ø1ÞZnð¯FÒÒ¾Âë]Ûá³Yò䑧{;Z»olõvp£a™±ñ!=™î ÷y.ÏÓ!3rèxê½y#ë%ÒMsž÷óòÝG K÷ô›c9,.žÑ¼sš¿Ç~´‡;bŽóŸËól(LçâEqµ±ÇY/lÇe¸ /!uä>Y;¬ -Ò¨ouxëeοº±8Æž+±ißÖO -»[îN ™x}ݯÇñœ3CýõØö¸åý‹Úô²Q›)«dîîðœÌótè½WŽ‡—†‘ÿ<ô,?†|Àw-ßó¨ÝhäEõÃùãÕÛP?úèJnÏÞšçîªn±=v<Øsyž±ÇñæÓĀߺæ}dõö¢›GFÅ*½;æÚ¬_ßÌólÈÍÇ1ê?ڎ·qö÷DwEëÝÑ·ƒfóøºÿdž§CóqÜ»‰GïDÒì¼zB~gdðÝ¡ÚÃ4ZÆcÂ#b¾;VÙËֶΑ•Û­ÞA#‹IÏ>lz2ÏÓ¡ñ8ŽÝ ‹¤h({ù¶õt»/}л‡+µw=§Ä² ú‡Ã­gwÖê!Ò~ÏD~G¼z2ÏÓ!óN¼úÁ³ö‡ ê¡½Çˆ Ÿ-úÙ{ÅÛ¥—–Ãkwß1q.Ó³áV«;[ªß>r}Àíª÷>Ç?-ÃóRÒûÞ€HõožØÞýãòÖ·èÑ›žÞxO¯oËì%r ˜w“Üa=>³})ÖU/"[Ÿ°A‘ôu6] Ð.·~÷|ù:Ϧå´E˜SÓc/åPVýÁ厞©grÊý–9†jG€I÷µ»»œÏYŽ÷–ã³y™½Û±sÕOŽÅ÷«vØ”Ãÿj>‚ËŽ·òë±^fÏê2Õék6Ýøó%”7}êò¶gÒ¤ }Ä5¸‰ž²ŠùJoMÊ:]»MGëëNÎò%?·TË÷j(zÕ6½fµt+¿XÒÿ\:ü§‹În"ÿ—g Ê;ðU'S/ºøíBßà±sý…;b.U+ 3Èn`Ú8å1%½goùØ’ ÃEóGãkk(¿¯ÙèÝ”NÕD¶¼q®X"gÐ[1µ^MzÓ÷H¤É.W¡™BÇc›°èxÚ^`ÏxW¬­bB¿*V]WíÑôÞáæºÐ…Ítî:ͺ¿¦ÅjÖœÞ.¨ÖŽÒ7QJo¯¯EUïÕd–•›Ô×”­C\ë'At»ÂT-Ú¡3›žõ½x¦ -$×ÒåšNÝ.bº¥4ßz*¶T©gÍÕóÝ”uz~e_vcEúb6F”­×bz>¦õµJ 7f#¼]¼k#±‹ß.> -©}ç -å+ô¸Ð²r›8©Ž—.©º -C³Êè ªôÿL¾I‰yÊí¤è-ÈýªWBc³‰×NâòÐXÇIÐ.ö´Ì]:ƆYî/”žH·¥n[I™Ë˜<æj‘T#µX7Ô51QXõOãJ.TÕTמAùm%îvú6T½j•Íàî[¹]w¾qql=®é•ÖÍÞV.Ã|ÞÊÄæF1A?ÕJºØ¨6Ûõʘ°RèâN£oE\Gkc§ò>—åcdÒŒ{»üõR¦9c½¡g_‚Õô <êÒ×ÿEšñûßwý·úJày7ª±…iܺ󙌾Ï'Ó÷ûTa6Lh£ý.Ö©§õ m7†ñµëÙU'öšR7ÕMË…¾û‰Mƒ®Ñ–yÆoƒ[ ó­}Ó'ìùÒÖÆѳ<ô2éj.xsp™ºÎö)ùÊŠÄ%íS:´Î°Œ¸à…â’U˜»e+3 ¼’æz²ßñ®°T»`a]Ú®s}šåEWYiK‚Øò ^°Ó¼Ò¼²G0úÝÃë›û}19Ô@ÒFíMúÆ¥­a$m„'Km¹ ëË6çØòL¼ÝC­,ËÆžmAN=´p« ŽQ&M=Û’ÀóJ]•­*­E£ ÈT•µcµ),Ùª[J‹“è¥ØbŠ.ý+÷´ç½Òœèy¯„MZ*·Îx-Z²-òα·Nõ”-–L‚,ðÜMêzJ[eۺݺî:Û^Y3\7$ôÈTõl~“ò]ü÷ž±3çv!¤Tõ§ìŒèÔ_¢‰Úéh-ß‹¨RàG_m]b[%ç9Þ;ÙîÔ½^ºcrÖª¨é.eÞ È÷¬¨úí—r¶¬Ä&=æak¨ÓœÚì •¯8+û˜eƒ¨]Ò“$†¾ÑÐRЛ苘¢Òòû#è)ò¸Ú>)?ÿ.4‹T -endstream -endobj -590 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 589 0 R -/Resources 4 0 R -/Annots [ 591 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -591 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 438.873057 101.433071 420.123057 ] -/BS << -/W 0 ->> -/Dest (cb77-1) ->> -endobj -592 0 obj -<< -/Filter /FlateDecode -/Length 4876 ->> -stream -xœí]Mä8r½ç¯Èó£å÷°Øƒ5ì1¼ðÀ û`øЕӵ £ÆÀ¬þû~A‰dˆJ)³ª{³îBu¥"øb<>CJIŸ~~Ðø/9=¥”cpçË/§_OjŠ^á_)±:(êŸðýY+'¯sHþü lœbö&†óÛ ÿ ‡¼øÛéßsúïu}6SÎÁhþËR9;Å£ò‚*C¥´TÞO1UÐN1gI3|tÆ]«"Ìp9yI3œ1IP%˜a]ÔöZ•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚOƒ$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ƒÑÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓµÎ(?ùd­’tZOØ´¢3 -N -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) ³ÕÐì¶ÊjIça‹1щ:ØçK `lQ1H€4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ0sEl±ÙI\` lA‰ ¬…-ÆY‰ ,ØÖ-XЭSYK\`Á·Né qáÂL%qãڜĔ (e‰ ,8ׂ;%.° ]L÷(qëZŒÄ´k] -Xð®uZI\àÀ»Ö¢¦¨ƒ-°Ä`ÇÉêd%.pà]«U”¸Àw1ŒÄðê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„9h%.pà]ãq -ÒÊ Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\€õ:c%.@c“AX"qïêŒåO˜Ó`«I§$.ðà]à>Q—'1â¼Å(O:„,qïb¡²xð.è3I\ŽŸ°¢‰ `?†Q•tÁL)‚®E]‚NG‰ bt–b-qöRS¤—¸?Q¸$qÌ€N;‰ b†-˜*`)š’Ad.é4lA-q"]è´–¸ Ø‚hPâ‚dv™AI\€{I µ¤sŠö™Iâ‚äÂD¡¢Ä˜&Øi‚)ElMH\ÿÐfÛK\€Í6›Øæˆ:Øb½“¸ %CÛÍ$qAJ°Å(#q°IûÍ(qAV°ŸÄY…‰Âd‰ @ãÐ)%qAÖ ûMã$.À¼¤ýf’¸ […ý&¶x¼Í6Ð~3J\lñQDlAД…y NÂ~ì$él1ÙI\C ýf’¸aö›ˆèÅ”@¢ýfÓà]ŸÁL’¼‹È4H\€ì7ÙJ{{âõñ™¬L´ãLbÆ@z=ÖV1e À½X­¢˜3P _Ú#‰I˜‰e -bÖË*”Z‰iþE<æ®8aÎ>ñU"ÖÌé$3¦…Ctßö½˜Ý(6çÇþò§Óo?cñ=ÿéN¯¿)™²ú£ ”Òñ¦Ép·Gô­qú6€ -ÌDRd! |¤s7Rƒ—¤‰åþ¯?¬,‡á?•îmë>cŸ»Ç„Æ3y9aû:Z|g•4˜:ÁjPÆfCìÈ!®TnUøï>ÔÅŠ9Ú™ùX/±vÆŸ?ýrúí?þøãßÿÿFŸ^Oÿñ;¥\øý½YþÔæ.ó±ûŒy„½]—s¯ËßZ®¶s-¢‚'z„-Î/ú´èãrìæ¿T¦”Û_Êù°®ßú©íùµµ}²ã›Ë9Æu›µÚ7Â[…á)¿Õ~?ž× ËѶŸY½8èØøkõûÿ<úñô‡Ok –è´Ña3ÎwDÂ7±{¬Ê{°û5 ‹­î Èz%C¥º‰\iõí2ê•2n(K¿‹KL¸íb7ºl9.e_‡þ>Ïòjkk륦2Üf¬qJÛµ Â~hŸOÑÖe -/Ó¶”sërã”êýÉ°£¬ü½œy°Î³;£›È kÏ_MtÁ³£¾zàË@Žº£¬xo$—Z¯"oDì^yÅdî’8*Ív»E÷e˜%#á{vnc;õ˜µ×Æ8¬m:Ò×&‚m§A„¡8Œ!ì!øX§C°»É£÷uåI;xD KÞtÜÒ¼ŸY`Ha³*®mjèøyFAù}µÅï®Ê/붌^Úñ ßO_†óÜAÙ8Kú:°¾@:JB˜¤ßCß±:O‡¾cÁ憕uÅIbGîx8câ÷Çsï™É(í¡"¡)¢ôJÕy:”îÄ—G·ÌãVcDvAm‰âò;¬Ü )CŒ¶òôÀe„î}[Þ5<£x2»Þ=XçÙ¼kÆpiÕ¹Zv87C;†ÐW“æƒ!ÝË|,é¨NÝaT½¥èü²´ûÂê¹^Ÿ÷U6É´bñ²Ÿç>hÃÐÚ ‹-•¨Nš‘Wv<q2¾z´t_Çê<¾v"¬G}Ì|u—ÃŒ b•óñ‘óìËùXÖ>[³ØvY0]v¼Œå`¬cužQ·£&BÅ#ž¥üÆ÷Ì>ï³UéþàÁ:O‡ªÛQÎ_›wîöî‡ñÍ2£vâaû@Öë`gCŠ½1ñ¼âW‹‡¼gÛÑÕX÷²½Ûó²«lÚ¸ãdQçå¾°s¬v~î²GóÂw]6©ùÚ›Ù¸ ´²XJ›)«dè‚ém´«óth݉¿¾áþ«!A÷Ï­µ®wwfn¸’Ðf0C]¨¹«rŸY¹1»¼Ê2[ÞLÊ»@¨ºÔcuž©;Ù°‡óŒÌÃ#Š§‰å?0óöýeÍ6Ùã;O_$A(vWúcuž‘;1áƪ{OVÿ®+ -ßrŒ×ھ˼¯Œ`÷@vï`gC°Û‰U?Áh¹Ù[×íi…m±Þ øXÌø,ß@/»Ò2|Ž­ÚÞžü`§CïNìzäºÏ!ü›!ôžYsëƨD“¦ojíÝ@ãyǃuž;ñéíWN/•=uE쀄¿ž—Yì–ÆÂeÃýRßÇn 7ÝŸ8Xçé{ì -î£×Ö¿9B‡ûɤûÂ>‚_ßoßÌŒã4–Ùá\ÿ@®õ`gC®?¿¶LR”GtDø -E ,ïIÙuŽ@‘ƒ†{FKFžr´¯sß¿¼£>¿+gÌ$±º-ã%­!ãx8a]Úæ÷K\óÁû(ü×¹Öy:¤îĪŸ…رŽ¢eœxǨ¯î¹x™ëÓu¢‘ÓZ½`æ€]+NÖˆ±Î­j+Ë×…s£súº;Ä Ä>Ϭótˆ}ß݇Ïç½?’yÚÝânÔ²æb܃užÍ·cÜ÷î¦nÝ=}äªíjvÝÃ[ßjYýV9]³ÿ<ß{@ï¾#ò«Íd}á<ëÁ:φ¾°§þ-òÜ=™ªè¼Ú)]XŸ·îbÿ*¹‡ $ó˜ôàŽë`§Còûr®ß%’?"WÀ89µì.Ç8|^³åƒcßLÜ@/‹O^±=Xçéлsoèè­a×r×wU+*FŽâw…p«h3mÞ‹ø=Ûï¼ãF´i™îg6Sº»Ïk¥ÜWp°ÎÓ¡t羂{¾½ó]Ùq/_éÔõ7§¿q÷{¾rYŒªÜTž°÷dƒuž ©ñà·ƒþÿÊ|‡!Ïíúõ‰ç×mn]S¼ç1ñìÁ:O‡ÖÛ±kóÜë%«{5µZçUë -¸‘eož¢µt_ýê®Á´.Wî]3B5®{]׿BÇ‚š’ûÝÈ-Ô™¥êwî˜qæÙVlŽì“=Xçé|;~½çIô†¤:ò<䥨®ïÊN²=·ÛÃQ(>V)^ÛJAËÕi«¯e˜àVËƵñ8¿.7¬'êuÉé.>Vçé|;ƵÂ~öêšÐSzv¸Æ¶Å…µ­K-»òzu]α>vžÈ$Ú0^ý¿+qc6–>íuÝÓÈÙz Ë{°Î³a:ü滳 ~ׯaÇ•Qw|Ýâ?·IÌ.1åÕwóoøyëZòÖwV¤«R4‡Ç•ÀQñ­ÕhŒg$L¯Çâ¡‚Õa¼Ä…Ÿ§Œ_vo¬¦‡©kŒó~Õy:üîävü®ÆùÆw°Ú.y+<6¿s ¶ýè'Í9†Ý6Æü -ûÆ¥œk«|{ (y€PÌíþŸ¬ ôŽ zÌnÐV›T<ñ‡?þË?ýÛYž’24´§k×ZØgßGÄ´‘fmC=ò\Ö ½Ô3œ¯äii>Ê4Âsùyd¯GÕÛÞvmSU™ŸËu†˜ûk¶/ö”>B¯ÛsÅ#eÄ qÖ|ˆiTUùþ3 ™/³<Íg«.Koaé&¬Ú÷ôTcKoµYÚ·Ëh{âç´ŒÖÏ×íK|¦×Š\·SκŽZ(#pËŽ”¶ìPý\Ü…°™ç§`½^&(gBÚžQ·FCGQG3q‡¹T/.6ùÊóçz®d‹½l9ê',¶çåoÏ£ §–yeH69sÝ93µøÇ3›4+·ø°æÿäq×ôPäõà4iftD¬^ЦZƒ¿ÓCöö5ººM—Öçšå]r¿¶RËëå½q†Þ60¿{Ž,å-ÿÔZ¯d_^¥Š8™zÐÅo'z˜ÄÎõcGLÂå£jm9Ý *» @ŒSKZ_ÕÍV)ëëGãëÙP}_«Ñ#Úª…lyHùܱ DÍ ¯ÅtöjÒ‹˜^§–&»…fŠÕf{ƒŽ—í öŠ»bmúk¡bÝu±qÕM¯ßhCº°™Î‡N3÷ײ&2wÞ^NèÖŽÒ7QJ/qªMÕÑ5ªÉ,3*7©¯%›C\ó“ ºœáª6íà̦g^£WaÌÂTäZ¹\Ë ¢ËI,·´æ›§b+•zÕ\G¾›Ò¡nSxAva‘\¤÷C²9¢lý¸S€Ò|%¸ãÆ\ /'ïÚLìâ·“Biß¹Bù -=.´¬Ý&Nªã¥KéÉGó'LÍ*£ Ðÿ3Uø&%æ)´k¢—ô£Þ ÍÍ&^—‡Æ:Níb/ˆKÇØ0ËÇK'%”'Òm¥Ûˆ­¤lȘ<æj‘T#µX7Ô51QXŸÆ•\¨ª讋}ƒòÛJÜíômªìµ³ Ü}k·‹CãÎ7.ŽÍãšÞìÒìmí2ÌWá¥,lnôSí¤‹j«]ïŒ »a ….î4úVÄu¶6v*ú\>öI&­¸—ÓŸOe™3Ö›ŒÁjzQºôùQæGüþ×Éÿ¹¾c^Çjla·®Ä|%£×Z¦ƒbzÍe¶iÄÆ1ÚïbzY_ÑÐÖqc_»^]ub¯%ucPÝ´\èû2±iÐ5Ú²‘ñ7„—a[ ó^ßô{>´õäÐ¥wªT«pÀO‡©ël_’ϬIQѾ¤Cë «ˆÞ(Y‡¹[¶2“É;iCOö;î -K}±FСí:×—YÞt••³`E[~ÀvšwšWöF?<½Þí÷ÅäPgIµ7é—¶#i#B­-ËæžmANŸ=´p«ŽQ&M½ÚRÀóN]•­:­ E£ ÈT•µcµ),Õê°”3Nâ(ÅStéŸùH{î•6ˆž{%\•¥vëŠ×¢%Û ÏáûÙ©^²Å’I^»I]/i«ìºo·î»®¶gv®zdªz5Uò!þ{dîÌu†,„Tªþ”̈Ný]RX¨ŽÖØòzp•B‡8zÃct‰¥Jî®s;wr!©¹ÞÜ19kUÔô¢wÙ„7(ò¾‚5U_/WËJ<¥û@H ušSW™¡ò¦ßrý¥$ˆÚa¤­"½ØÛRЛè}¤Qi9aOY¼´JŸ”ŸÿçØ -endstream -endobj -593 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 592 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -594 0 obj -<< -/Filter /FlateDecode -/Length 4808 ->> -stream -xœÝ]Kä8r¾ç¯Èó£å#øŒ9XÃà /<°†UÙ]»0j Ìúà¿ï/(ñ!)ôȬšv¶»P]©R ‘?F„””¾*üü ñ_$=Ę‚§ëí—˯5§ð/—˜dõŸðÜU+ -ƒÓÉGwý…6 !9üõý‚ÿ‡}ñ÷Ë¿ýîò_ó:új†”¼Ñîúׯ¬";„‚r‚*A¥´TÎ !ç•×.1%I3\ CkU€”¢“T0ƒŒ‰‚* KAÛµ*Á 4I*˜¡“ ‚J+Ø¡R¢†("kvƒG§{Igôàc2IÔèt0’ÎÚÁÒNKº4xï]”t[\RZÒ9Øâ4‘’t°…l ’ÎÃë­u°ÅÄ`%]€-½“]„-@Wu°Ey¥%]²ƒK‘HÔ%èT q­3Ê .Z«$Öƒ6­¨ÃŒÂ ù è lñÊ%Q[œUFÒYØBø‘tà gCŒÂl5<»­²ZÒ9ØbL Q[0ø[Tð` ÌK˜J†$0Ñ)J,`b(+Ñ€In Ÿ‚ÄVi茖ˆÀ*ØâÈKL`5l!Ì\Q[l"‰ ¬-( qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ]L÷ qëZçÄ´k)z‰ ,x×’Vx×ZÔu°–¸ì8X­ÄÞµZ‰ ¼‹id$.À¨À¡—¸XLŒJâHâ%@Ô…sÐJ\@à]ãp ÒÊ Þ5N‰ ¼kÈx‰ ”ü£Do¼ -q²;[@°`½‡ÎX‰ p²ÁÀ-‘¸ÀwuÂò'Ìi°Õ cò8ð®Ž>Q—Ðã¼E/Úû$qïb¡²8ð.è3J\Ž°¢‰ xW[¬Ô’¼«MR8ð.ü'qà0h8`}´tE]TŠQâÞ-‰ ° -*‚ID]$-qÈxPðC%.ðà],ïIâÞUÎZ‰ Qç;‰ ‚‚À²Dl¡@À±’MQâ‚``‹5Fâ‚`` \‰ à IÃûÃظˆñÁxŠ€É†˜ˆ$.Î#C¯J:o†@×¢.B§ƒÄ!Ѐ¥XK\€Xjˆ˜ô€àv—$.€Ði’¸ $Ø‚©"q–¢!xæ’NÃ8ÙÀÓ…Nk‰ ¢-ð%.ˆ&"ÊôJâ¬Ðˆ%ÑÕ’ŽÇ™Qâ‚H~`WQâLDš`JQ[@`|8Øv øA°‰0GÔÁëHâ‚ ‡›Qâ‚a‹QFâ`“ãÍ qAR°ŸÄIùÝd‰ @ãÐ)%qAÒñ¦!‰ 0/9ÞŒ$«o"Äæm²žãÍ qA"Øâ࢈:ا) óœ„xì$éE净53¾È„iAðîkÌÑŠÙbc~쯾üþ‹ïõÏÿ}yû]Δ• ð[ò9°OûdÆ|5äó"L8Û† ¶>R‡©©A‹SÒÄRŒå6]ë=7³ÍÎL¾2šµ‹PAc¬¬m½»Î§Û - >ƒƒ÷6oW4ò¨ð¡u–ÿ]׌´ ¬u» LN›!Š=[é^úo¾€¹Ç¬±¿vØ}ð0&èÏ¿\~ÿ‡?þó?þë1Z?¿]þýo&¡R.â/Œ¢>ã/&¢7|F¬N/ã1Üñ±,—{eµ“¹é/׋S™8Ês¹0žKMå³ÌOíNm²œÜØ.׿k3ëÑÞ|œÆsÌê½Œå‹ U~íp]¹rœë‡ÿãúóOý˜ú~Ì]÷65ýµ5!ug–wÝÉ—¾ê77?›=Õ¯ÝÉãðÖ.;›>uqÖ¿¶ap¶u‰þ2–áêyhr6•sÜ.µaçãY=³×…üqч4µsû¥ Yn#vmH6ë¾íÝvá2ÌÛ5Òù¾ŒçøÃÏyzÛJ˜pÎÍÓ›sCpŒÃš‡ŒeÕS½u"Ù‰pEˆ^p½ƒM†½=,áˆÓ­hJ«c×ÔCm]µÎ&¼ŒÊ,ñ?ýôwÿ/%ò¸Ðo.ËãmðŽ×e3º›È|[ÊÚg§E£®¥L†ë5Ìð…Ä)¼E£yfožq®ã\« »¤w²ÎGÜ ß‚ûàõîqŸzé—ÎÅì ‹ºYø6ÉÌT¦.«Ó1ÍëdçeÂ×bÆŽÞÐ|†VÜ-gµžÎy‹Ñö°XqFœåZ/°89ƒ1[³Æ!!Ês1èEƒ{8´ÖL9ËÞ¥Cž¬ól8D7ïá°²µYü #'ÕnúË¿a<uüÙNº—E™/ÓßIn&7U—ß/ÓßÔ•-ú7[¡™—(óº!FDÙ¹:O‚²fvp{(Ûd-µ`­×n´J/i£”ƒjˆ™Vœ -ùù¨å2_H¸•Xo’¿Íõ%8Ù:wÖ ³¢M çY°mö|ýµoçÛÊ}ª»~KýR£¿©çê<R!ØEª€­¼RtjÍ»\s›£gÆ]‹Ñv d¬㺺·¹}ÙŽ‚Œ2³ô8ëÄ™4]_æußÛÅg3ÕI3O·ë¨×}ë|Œ%šVo ™¤+ò#D0À ód§C¦ÙåÐ¥§V{¯GÖJüº×gÜ÷ò„|÷ÈÒ­| jÞVCó -›ïGÎ Ê‘gÄì£ð\§Cá9±Ž uHØâ¬×.ÿBÇH]rŸˆ¬åHö\zG{¥ìŠ¾ŽõÇÈd~þ¥_Ü·g㶭3žr]tàCRïòã1ßBßGÞ¹:O‡¼r9’aÞÓ+ôu#»ZÉ6>KÞmGˆV¼#­ì±ûmߟ#³hâC£^Éo>k7ÐÛùˆÚ IEÃ7§÷Ñ{®Î³¡×ø•¥ßæ+¯¼,F¶‹Zöø®GË·Š‘û|ÓÊy|fÁ§­žÆ2§´äÉ3‘Ü–½¥ÎaT'#Ö5ßñ4bOÖy:Äîû›Oã~4æ½'zûÞÍ=³bVžö¹ÖÙûcø“už¹û>êÿ·ŒÎ§qö·DwAëáìÛ@³¿?î?Yçéмï÷®üÑBZ+bvfFŸ*#̳Å-üÕ%b¾9ÙëڶƑ÷·ÎÖ-ní|Rþ¥ÒùÁé}4ž«ólhôû~ìÖýžsÝëÇâéz—zgtw#µâ#/xruga–Tb†l…þâe¼uH;‘YËå>´.ÉÈôø«'ë<2üÕO^µ?Uwå?Ã7|6ïgµ]n”¿l’Ÿd;Bí¹:O‡Úƒ|êG}ûý­O¸WE Ý’ÿ¶ÐºôX -—.®³f±ô>¤öëì\å‹7úÀ½û“už¡yד÷ªg°X!—HÊÙ¡é_M¿”+ˆ&5"tç/Q\ÛY zùÔÆìþÑ ›ïEãòé‘Þß­}¹´íKWﻥ¾ùœ”Ü¿“øËGû¨=WçÙPüT%£¥_ ­Þ/Ãųïjøwã÷Gù¡g÷„§`*Ç.ïÙûõŠ­\¬è ô÷ž¦™›Ë-¢Ëå¬:ºCôý|y²ÎÓ!ïà>ý"¾]Íuadgw—«ëò‰”éãá­lù±÷d×iIßmFmÁâ -Oðï"ø\§CðAþô>åtÝ“·œÝßêsanSEÇ—ù÷wOëó§Ì·ÌÓ·Åu l9KŽž ]îòlDt²Îӡyt¯¤(eé;IYô{ü±ÎWüþxî#3y¥ÍW„#4DFÔÑÓ('ë<Jãy6êèïR.¢%²3Â8* -ª>‹9óóÂ-kÝô‚Ëú,øÑÓQßÿ¬ÑÉ:O7º'}¸¸èÕ¹š£ \›áˆÁ·•…3Ù1Ö½ŽÇ’Žë”£è-{ç·é¼¯]=jõû¶ê·,ú²/c0ÔóúÉ–Â \'ŽÈËÏg\‡Œ¯þnóY|«ótø:ð°ãn¬îc?â‚YåÓÆøÌu,ìX—o Õóðw¾ÌdÛmÂtŽx7ë§#OÖy:DÜ%Ž,ç7¾göù˜ý¨Š÷G‚'ë<ªÒÁS‹ÿǼs÷è~ßL3êÀNd½NÖy:¤Ü}íòŠ¿™o¼ñ„öò¹¨3yYé›%’W×ßKªW—S(OÝ8½aÛ‰¼ð]wNJ¾ög[“½ÿiì“už­þ×7Œá3$ÜûlÁ™ÌÜO÷äüIJ܋Z?±¼g6ˤm µËlý¾ÕÉ:O‡ÔƒlØÃyFõøÓ$Ÿ˜yûþ²fˆ|à À“už ‘ …3_eùÐãÈ÷ÜRø–S@øú÷w á?MÛQ‡Ñvü*Þ£1u»[VÜ9€iÙf½âU/¶qœïƒSò{hl¼ 1Þºuk¿óuŒòî!~ŠÊÆwÓó$¼[ù†º){Ìd/O¹þì©q3¼¯ jy¿™i³¸¼É]åyS7¯ÚFrÓçº/a†ÐÔïà4í?HÓ¾J³ó—= ;{f{¦)FÉ×°Æ_/>oA¬ó6ôÊ#Î4üžš«§C¡®¼íqT–t6;Z_7jæ·ûüZKñÿ)~ïÿœ‡µ?˜Êÿšü—‹N4pÿ3ƒŒ/çPYM9hâ÷ ¿œ…1Ñ`NŠY8}Tõ\¤KÁ*»ˆÃ–%­Ôô1Õ“óf9ãGÄ8¾Õw¥ïFª²yÛرaAˆš^¯Å|õjГ˜_p;ùj -oþV…í„Ô—m'lÅÚªNèÖBÕ5×ĆŠ=š7D¯]ç›°šÞw”5¡Î=áí‚fíRú.JùµåT¥wª2Û•ªÔ•’u@¨Ž“ º]᪜š0˜UßoN> -cÕr©”D·‹Xn:›«#j©Øª¦ÒóÍ”ukÇ ²ÜG“4ð»º9¢lù8{2Ö*v¸1+áíâ¨ÎÄ&~¿¸ ”v+”+Ðë…¶;oGÕðÒ¤œ‘?ajoíÌÿT᪔™'ÐT!ÅÛ³·£ÖÏÍ*žwR/÷•uHêM¡•íºK‡P1Û÷—ŽJ(Ϥ[K×›I».ëä!ó˜¤*yÌź¢®Š™ÂJÿT®ì…ªæšØU(¿ÏÄÍNW§ªSµ1¯;¸»zÞ&ö•;ß{q¨#®y¯ýjo=o‡ù"¼å…–b†~,4±QuµkuÂfH¡‰¾gq™­•øýaåc›dÒŠ{»üå’—9„†wvõVó«cz¯K_ÿe~Âï^èúOe¯òq7ª²…©Ü:÷+¿h,žóæEX§M'4ÔÑ~ëØʺ‚†ºŽÓñ5µêª{)©+ƒêªí…®õa'6ºFÛ®gÜŽð¶èÅêÈ|tlÚ‚=Úrq¼S-ïr_¬ÂA98ŒMgÛ’|íN‰#.Ú–thÉtqП‡]ƒ©Y63“Ù7R»ží§~(,·ÕtnÚ¦£¶Ìö§.²|]¸À¶?èOLºo4ÍìŒ~xz}xÜ'“}™,­Ô^¥ï½´^K+áÉR›ýü°®96oòZ&oë¡z.ÛÍ=[œ65škA³ƒ£4¶jS×7JE6k´,•. SEVÏŠM~ªVº%_q{)TŸ¢IÿÒ÷´ëG¥v¢ëGůÊòyËŠW½%[ ×Ã9´«S­dõ%£ ó}í*¥VÒÙºmš·]VÛkwÔ ñÍ3U­š[•|ˆÿ™;â ¤Rå'gFtlo÷ÀBM:Xcó [UôBüέ@±K•Ü]g?w²Î”ÜIߒ欜µ*h~õ®lÂ;éXѪ¼–W®–”xI÷u€j4·~ÝD~÷bÎsæQ= *ò«V-;½‘ß”üú‰ñVõ,}’þ½ux¬ -endstream -endobj -595 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 594 0 R -/Resources 4 0 R -/Annots [ 596 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -596 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 713.673057 101.433071 694.923057 ] -/BS << -/W 0 ->> -/Dest (cb80-1) ->> -endobj -597 0 obj -<< -/Filter /FlateDecode -/Length 4869 ->> -stream -xœí]M$¹q½×¯¨³€Mñû0t0 CZÂäƒàÃtͶ£WÀJÿ}¿`&ÉHfdVvMïn­áiÔTe?‚ÁÇÇ 3‹¥¯ -ßhü—œžRÊ1¸ëíûË5E¯ð¯¤X]õñŠþª•‹“×9$ýž6N1{Ãõí‚ÿ†KžüíòŸ¿ºümG_Í”s0Ú_ÿþ©œbŠQyA•¡RÚ -*蓼*è MÌYRÁ q[U„.'/©`†3& ª3¬‹ÚnUf˜¨¤‚:û(¨´‚Ú(%ê`ˆrÎA§ýàô 錞BÊ&‹ºŽFÒY;…è´×’.O!Ÿ$ƒ->+-é‹:Øâ­2’·?Ižð6¦$ŒVC£Û*«%‡-ÆD'ê` :_b`‹ŠA¢Œ ‰xIâãä²q ˜d'—\’XÀ¤<¹­D&ûÉ…%°JCg´DVÁï‚ÄVÇ‘+ê`‹ÍNâk` H\`-l1ÎJ\`Á¶h‘¸À‚nÊZâ ¾uJ‰ ,f*‰ ,צà$.° \@)K\`Á¹Ü)qéb¸G‰ ,X×ú`$.° ]ëR¸À‚w­ÓJâÞµ9El€%.;NV'+qïZ­¢Ä¼‹ad$.@¯NÀa¸X™LJJâÞ5I9‰ x×DL¢.NƒVâÞ5Mfnð®ñÊH\àÀ»Æ™ q€2” €wA!^` VâÌ÷Ð+q -› ‰ I´4õ3“ß™å VXÀ#" Hc³¡´ˆµr«Äÿüé2]¬˜÷éÛ%31ÆähüõÓ÷—_ÿþÛoÿåwÿ~z½üùŸØï7ßh¼{;¿»Ûr–ë´\ûå=oe`9å¾Òß ×CÙU§æw,iJ^zÕ4­³¼¿,eQզ׵­­\$òñŽM$Ï,ïë:!{ ˳”Wë.º¥\¿¤q~ÇWZ(§^³òêg?´O›ßü×õÓ·—ß~Z²¬¤ - i˧ ŸËò5þ1 ›ó!tïtCéê‚÷OÏ÷ ™£6ԺþOš. mí•¡Hw•Þ˧'ó<±²?D〬öžä^"¢©ŸG$ü|½Ìê«e^ˆìÖÓ“ì—C°;ȵ=‚Ó¬’¡eÐ1rÏåy:äºCÝŸ~ÎqìOŽÐ¸N·A¡þ~½×öÙ+‡ù¢åæ—Mš{œËRBŠË´š>Fî¹°{2ÏÓ¡õ8vm=÷ºFIYƒ³»›«}Õ:îì²·ž¢µæ•!šÅ  ¹7µÝªqÝë:ÿ jÊÞïÎÞBYT– ïqæÑVlŽ¬<îŸ5ûƒŒ`÷ÀžìÉ<φ`w¿Žü5î(­v“CRõüÀ#õRdü9Þw¸ÆñGáQšq!³•‚–Í -i¯®e˜àVËüÚxœß—æõºƒdû~.>™çé|ãZa=»¹'ô”=;ÜcÛãÂZÖÀ¥–ÝyÝÜ—s¬=Ô?Ĩ¢ ãÝïÏFc©Ónó®˜FÆô»¼'ó<¦ïìòó1Íéů‘aÇ•Qïôu‹Gü\&1»Ä”cýGý¼w/y3vF[¸Ýy;ø!*>šÆxFÂtKóe¬þgÞN¿ìÑXM_Öðó=üžËótø½³·+àwå瑧YìÚžtÉ;Xá±é€©3±íc;Mjw/Å1ì¶10î¯,å”ýWq¬Íòí‹=Ô„b=Ñ.Ùw`¼stxˆ6©ôÄoÿðïÿú§?°}JÚ¡¡5]»×Â>‡Ù¦yy± ñäá9­'z©­›¯7ò4z™{˜¼;§Ÿ½ºõ¨¯y^æ÷9Ë,r?Ë:CÌúfÿbS©'ô^ï{¥GŠÇO¹8pÿy©¡Z]ZZîµ²šL÷\³šYR®íòò,ýrM–WïÖ–+Ú æs>³¡ylIO^V·¥…aiZàm -Š¾iélœ¥MvéeOsBZz†zé˶œ`é‹£t8ɶµ´knsi÷‘˜´wìP½-+P¯š™Ûèš œ Y({Fû…½Ý×ägâ+zznFÎb“¯¼Ôû¶ô™Y|¾¤-í§zÂb{^ރ܎‚‰šæ• SÑ:çwÖþñÌ&ÍÒ-}X÷wüîGß4 ifsÅC¬enëo#¼8ÙWz¹‡UÒ.#_ÙµÆ~h6Ö¶êwÝý¾b½î‹ê©ü™¡ýG?Íl¶ÇdçX¬Ïü‘„6ƹÌÎ1Ìg+nY·_æ˜c~)þ¬ãÎvP»ò¡ŸòÆOq©cÁ„þ"Í -û±?ãrdWÄšwm—‘êý"¯²#}ÃÚ–¯Ò!:§"º“yú¢ç†r¾ÙK'5x¡-ß褮—rz¹)iYÀ0wÂn…9L*Ó)C…•–ùf7QO½½[¢±yÂÌìTJô¯÷r»TÎ -A´ç^n¦ìÑgRnþ€n `í]™rPŸóÁîxè§zˆánë-gfÐùxžNŽ‚ëB”Ûy/7õ8Ášó¼K£,9„ ûØôyé…Ã:Æ^䤤=ÐÞ bÈÞógAÄ*°³{ÁÚ6¢Ø2u4TZFöéÑñœ Ñ){å ž÷nŸŸÌó$ËØÆ}èêCîcSî8’ö°°·4í£Km–‹Šo£ðÅÒ°­9‡ë¥#[~¼tÒ>ºâ°³!0/Óñë‚À$ ÏM9ûõPáú2 ± É[syž }t^Ìú6|›¼¥çÝò0¯[^q¾už=¬ñyHóEµ‡‚˨õõeyÏ,íËO=/}Ü#B$Õsò1Ì“yžeÝì PöžG„Ž¶Rùã@ã#A{Ùð¯øHJ<×#<ýóé>ú¿ƒÔhÞÔsyž©é¶CÆ/ªíÜ,¾­í;ýå:’–ö^ìÚŸÍ’'ïŒ<ÝÛÑÚ;®zt£g™¹ñáéÃCNæy6dj8ûpªþŠï'rˆ­îÅñ¼ŸŸð>Zº§ß<Å‚^¾Oþq9é,zô˜§SÞéȲÌNkØ»›äÔj(½=ꪇ÷ÖÑ6Š¤Ÿ8° "¬Á" -¸k›ïæU_‰¼Âò¥íeK Ônâ,£ÛÔø\Ï«Ãò˜R}Kñ»î¶«/Û ek9ÏrÓ¶çÙJ2(v3EµmÓòÕÞ4o—ñíÖV~ÝÆdö¬¶ó1–6lºñ‡K(§[êr± -iÒ†~áÜDw¼#•‰NÆ´N×nÓÅÑúº“³üpÄ-Õò[ŠŽ—¦£EK·ò‹%ý¥Ã¿¿èì&ò¹TÎ}WEœL½èâ· ûï±sýfÒ1 —ª•…5µd7iœò˜k -µ|ì7°h¡14¾¶†òûšÎctª&²åD¹bAˆœAoÅÔz5éEL¿&»\…f -m—5a/Ðñ´½Àžñ®X[Å„~+T¬º.6®Ú£é¬ÝæºÐ…Ítî:ͺ¿¦5‘uç‘ðvAµv”¾‰R:±½U½kT“YfTnR_S¶q­ŸÑí"¿Qµh‡ÎlzÖktîí,LH®¥Ë5 º]ÄtKi¾õTl©RÏš«ç»)êtŸo_vc7y#ý #ÊÖk1ÝGl}­ÃÙoïÚHìâ·‹Bjß¹Bù -=.´¬Ü&Nªã¥Kcª®ÂЬ2:5”þŸ©Â7)1Où ]ƒüÛ¯z%46›xí$.uœäíbOËÜ¥cl˜åþÒI é‰t[ê汕”¹ŒÉc®æI5òX‹uC]…Uÿ4®äBU@u]ì”ßVân§oCÕ«VYРÛÅ¡qçÇÖ㚎qnö¶ræ«ðV&67Š ú©VÒÅFµÙ®WÆ„Ý0Bw}+â:Z;•óM–}I3îíò×K™æŒõ†î…«éW xÔ¥¯ÿƒ4ßâõßwý·z î<ÕØÂ4n]‰ùLF¿a“NŠé7mª° &4ŽÑ~ëÔÓúŠ†6ÃøÚõìª{M©ƒê¦åBß}ÈĦA×hË<ã„·Á‹-ùÚ¾éö|ikãèÞ \­Âo.S×Ù>%_Y‘¸¢¤}J‡Ö–¼P\² -s·le&’WÒ\Oö;Þ–êb,Œ KÛu®O³¼è*+­`I[~Á všWšWöF?<¼¾ºß“C$mÔÞ¤o\ÚFÒFx²Ô–Ë°¾lsŽ-·ÅêàíjeY6öl rúÐè¡…[]pŒ2iêÙ–žWêªlUi(]@¦ª¬•«MaÉVÝRZœD/ÅSté_¹§=ï•æDÏ{%lÒR¹uÆkÑ’mçpŽ½uª§l±ddçnR×SÚ*ÛÖíÖu×ÙöÊšáº!¡G¦ªgó›”ñß#cgÎ3ìBH©ê_ÙÑ©‰Úéh-¿¨RàG?ç]b[%ïÎs¼w²Ý!©{'¼&tÇä¬UQÓ¯:Ê&¼A‘ï+XQõålY‰MzŸ„­¡Nsj³3T~Ö«ìU– ¢vi©H¿âg)èMôãCQiùY^zÚîójû¤üý/Ù¡TÖ -endstream -endobj -598 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 597 0 R -/Resources 4 0 R -/Annots [ 599 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -599 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 273.873057 101.433071 255.123057 ] -/BS << -/W 0 ->> -/Dest (cb83-1) ->> -endobj -600 0 obj -<< -/Filter /FlateDecode -/Length 4737 ->> -stream -xœí][ì¸q~ï_ÑÏ–æ¥x?pàl#‹ØFæôîØf ¬ó¿Ÿjñ"ªtëž3Ö>ƒ>ݪ"©ùñc±D‰ê*ñ÷”!zG×ÛÏ—_.Rx+ñoH19Ô?àãíUIòª肽þœÆ ­öîúvÁÝa›üíò§_]þ6Í£®ZÄ贲׿ÿ”Td„ÞK˨"TRFe­ð^[Ǩœ¸Ä9Ì°ž4ÍUfP –SÁ Ò:0ª3 yeæª3´WÄ©`†ŠÖ3*%a‡ÒR²:"‰ŒftÊ -‡JwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¶XE$9l!<§s°Å8#YlÑÁNça‹FíDF` ÐXl‘N*N°1±º >ÌuZZaƒ1’Ó)%,°iXzÉyF§a‹“6²:ØbÔœÎÀ§OXãC`z«N½ÛH£8…-Z{bu°Ï1€v°EzÇQzH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHØbÉqL`l!ô\V[L$Ž Œ†-HÀq1°E“á¸À€m há¸À€nIFÅqß’TŽãÂ…™’ãÆ5ÁÇ” (EŽ 8×€;9.0 ]twÏqëë4Ç´k(8Ž x×’x×ädu°æ¸ì(Œ -†ãï%=ÇÞE7Ò Upè8.V„Ar\@à]$q\@à]í1°:/Ð ÇÞÕ—ÀÜà]m¥æ¸€À»š´ã¸@àÉzà]PˆåÝØ‚å¸ã=tÚp\€Â„†[Âq愈áéÓ`+¡BtXð® -h>V…ò¨q¦ß¢–…r.r\`Á»¨ Ǽ ú €ãFTÍqï*ƒ‘šÓw•Ž’ã Þ…ß`9.„‚ÀqÆ7¡$ Ëê¢1Ž ,x´¤9.À(,d“°:/$ÅqÈXHø¡8ð.†÷ÈqïJk Ǽ+ n«ƒ-&hŽ à×@‡nËôi8«Bj0§ïJ…ætà])}ä¸ÀwÓ°ÉqóI Ç0PÀ Ö¸@"zï8.pQB—Õ9cËq—XP«ƒ-ä‰ã8Ö"š8.ð¶­9.ð¶Àeจ -Þ7;=€-€ ;? ØO‘ã0™‘ˆãoæc¨UNç´tÍêtÊs\à= ÅŠãÌ¥D@§ç¸/’»ÄqÌ€NÇ>ÂtŽ 0‰ á™s:[àds\O:¥8.¶Àä¸ è€Y¦“`„Æ\UÍéH¦yfภÉUä¸Ý3M0%«ƒ-  Ž Ð>i²m9.Àä“MLsXl1–8.A§éfภØ¢¥æ¸ØLóMÏqA”°ÇQ:‘ÜdŽ @ãÐIÉqATóMM _¦ùfภ‰ù&¦xL¿Æ¥ù¦ç¸ l±pQXlÓ™~ NÂ|ìÄélÑ‘8.ˆÎ¥ùfà¸næ›ðèÙ@HóMφ À»6‚™8xž©ã¸˜o³åæöÄk=ü3^ÒŒ3° êµ[Ù÷b´òlÌ@‚|Ó‰ ÀLÌ,ƒc£V¡T’ Hð/ü1šqÂ=úÔ†¨àà1fŸ.2¢[¼û2ç¨ÉÌB²{|ìï¹üúƒïõ/ÿ{yýÕ)Ë -šà_¡èˆæ¶ð¾.ß84´ƒ aà#y’>V#hq º%#1ü¥×ù‰å0ü‡áô¦œ>bNO 7‰ÆJ¥L_‡Ø`cñÁ,ïh°Ä䣭Cub'@~¥¤Iâß~¹¨£÷h9lÌÇx‰É0i{ýòóå×ÿöý÷ÿúûÿŒ¾¼^þü/RZû›ï¾)Ü¿-òÛÐÉÜøíGùת#¿}—g,è*åårú4VÕr¬œ¦Iùœ/ŸC'ßðë=?fb5ÝX¾ÕÝ55ç3aÙV¸¸CšÁ®\w·æ:|.ç7ÿ}ýòýåw_¦`<Ì;ü“Ã…Q-qÅ*þöeyßv˜®®ÀnÖŒ~ZÍ3è5ͪR¨±šW~Ošª;Ÿ t|לn§ì~½C$C¯‡0éîüfÌ«¦×òí»,ÝÏ@„eÐiP^ÅîΣÍÍë„–ìÍybs¾×iÚ¡géÄ:ŽØ}yN‡XZçÛ…VH7â¶6¦ûr:ä¦òzîË-¶ç\¹‡,qdæ÷‚ønXÿXž<Ö+&éiƒk«ÓiMB -Å4YGî¾<§CºÝ÷?ºß³?Ý­›½oÍÕ…M·TˆI·…æ}yN‡æu¿wænxÜè\³Ò3èeî}¤Þb;µĠóg–}ÛV9òX¹¥·.p«n|Ò´pDª!`¼ŠÆyΆF½îÇΕk¸AVj™g&Ó:l·îêL-ûÈO¾¼u6¨)wµòú³—ñÚ É7ƒ^ÆtOK È|À_Ý™çtÈÜðWßyÔ~TåÑï&ç3¸×9>ïžÍûY@m M7ÙtPÚn¡v_žÓ¡v#˜ú¬oxØßjFë5ÿñ6EÑrÓï‚ü%´öKæÒî:KëAïƒ;é³`ñBÛêÎÙÔÎ<§CèFÜuÇè¦[Ÿ³!{$ Ñ!}Gô9¿äˈ¦1}_~FF‡ÐržÕ¾ÚqÄæ£h,|ß_×צ.{Û~lòõÈx+áSîAב¸åäþVðS› -:~¼£`øÚ¤¥;.k£à[«5È_i´3ÏéZw§ºZ]è«ÃlצӌÁÕ‘%E²ÓŒA'Ý×û1§Kyò #ëMòÎoc¹_›|Tó·çæÉiÄjÓ¾ÜÏ‘& ¥\7Ú’9!å wä 3ž÷¸_íÝæ½øÚ—çtøÚð°m㦭µ±»ã"±Ê»µñžëèìè‡ë1M9ømôhÛmÄô0ã]`¬ÖFîÌs:DmÜ%µlŠo|föyÎ~UVŸ îÌs6TÙU‹ÿ`Þ9ܺïÆ7cÚð‡íQ¯yN‡”»¯M\ñ›ùÆ +´ûuQ{ⲓhZ?ãl¼ºö^R™q51…¼êƪÛvÄ…Ý9ÉñÚÖ¶Z{|5öÎ<§Cë†ÿõsøo†„£k öDæ¬îâ}º9_Ñß3›DÒÚD¶v?sµ/Ïéº {8Î(_MòŽ‘·Ï5ãéX¸3ÏÙéö=ÉòÔjä#w>²ô÷Ú>eÜwÁD÷væ9‚7|ÕwZOÏ¡åÐú«w~†jѦ]>ãY¾€ÞæNëÞçvæ9z7|×Oü”ß»ôš]µ¸€ÆâŽ;óœþéÂjèÙS¶rM™5tHøG®MíWq§™ÌÀeÝz©Ïñão¯ÆúzêÓlp® Öº3Ïé»Ï=úF‰ Šn š¸gŠ®OÖÏsÔ­"ò)Fû*gÏpÎß®Êé#IMÞ~=ÿÞzá¸Ï½3Ïéºá«¾0¾c®EÓpâ¿®Gï°kí¹¬>ÏZ µ¤mÇ……kK×ômgˆ ˆ}ÀŸÝ™çtˆ}nõáùü¼ç=™ÓÎ|w@xÀÇÝ™çlhë>î³³©Õ÷ª´-¾p×v2 -SmᥧZ&Ÿ,O÷ì_îkÒ÷á‘߬'/ ï8ëÎ<§CßÇ>‘} -ž;©ZAçê:ÖV±“ØÃ’x£ÐÎ<§Còs1×O‰ä÷ˆ4œ>ykÆñÃûzhó÷u6®1Ø÷dâzÿtïÛyN‡ÞcozêYÕŒŠž£ÚU!-‡e´é…2"~Ëöƒ+nX›Æ@?6=µA÷cë¼x”ÆÖìÌs6”ÆuGžŽÞxV–{“Ä°¶³rú£ß€ñ‰ïÄÆG•$†×l½[`gžÓ!uçÓAÿ¼sÀ¯0lc»vZGìõ<õÞ®´>ƒÝ™çthÝxÙ¶@Af»VSÉi\5€oî½µi]ýdÕ`˜¦Ö®ö¡ì×½NóÏÐ1¢fˆý.ÄÊ+Øó3zœ~ · 6ûæ:[O¨»Vì_x -%>“Ý™çt^÷_¼I#=ñQ”k¾ã‘‡ZÉËùªìÀÛ³öæž…웕üÜÖä´ÌfHKç{€~n5íAó5¶÷åºñD¾$ÿ0îÿá-ݦ½OÓ±ÙN¤Àw3ÉZç›ä-<®é¯‡£qèàœˆ´ñ¯ W8¦}Ú+'ÁïwøÏÿã*ü†Wýt¿¥™Úbx¨èvoÊtœ_d¥Ô=H9¤nHÍáÆj†¼—TeuèLßwX$hÝá8ܤJiÝý3”5þÊ ™xÆó¥Z)"T×OÊ·ã¹{†2ls 6_칸aÏ'5ìû']J§¯ŽDê>^Ó>SAR¹ÙÔPÑêºsØNù—’jÜaY¦MÓ†[÷í—›ƒ1ý/Cƒÿ|Q‘DªÿD$÷ÝPå :TñÛ%í†KŒ˜à“î'áøS–²Hå„Ev{û”Æ -9þŒ¥pcóOmóÕ¤ü6gK»‘̰̉OÏýÄŒ9š‹ÓÕK¡FqÚQ83¹bŠuÙfkÔ¦­ÖŒ›bed#´s¡lNWÅš²=*í@WªÎUa1½­:Õ4N«}ÓœkÂÛ§5½ô•¦}LsQ¹vµ,2Ó‹Ô攥A¨´#º]áw2MhÌ¢oZ-íw† $*ébNLjn6ÝXš--åKªP³Æ\óÕ” -uJÅ3²¡ŽF©O[¤7}Dšüs*v©r[ËÐàFÏ„·‹¥Ò«øíb=“ÚV®6C¯š¦Ü"²â¥JÓ£À÷_èšY–öÒJÿß©Âibžá‡¢©´^=ª'I}³ˆ§•ÔÊ]aâj@‘¯i›êRÞ̶õ¥‚dÒ'Ò-©KM¤M•5r³y‰¤ -yLŪ ®ˆ…åú)\Ù -e6§«b[ ü6W;méª{>™S Üm)·Š]áηVìK‹«´¹a±·”Û`> oÃÀF½8A?ä“T±–e´«'k„Õ0BW}Ĺ·vÞH?þ¬Œqo—¿^†aN«#~8£Ò^½­×¥®ÿ‡4ßãó?ºþGÞî>ŽkYØBnˆÛ‘,íìvŠÓNïYXºM#ÔÔÐ~«PÓÚŒ†2ŽkÝð5Õì²{N© -ƒª¢m…¶Öa#ÖºZ™¦fìŠðÖÕbqdžm›:`ßM¾8œÆ¤m³U8h/‡¡êL’¯M‘8JIë-é&#ÚBqØœ0VË&f&@¶')UŸì§¶)L:WsиéÐTÕa¶-:ˆ«h’À6íA[0©ö¤qbcôÃÝëévMv¹$i¡ö"}k¥åÂ’´/5á›–1Ç\ÓAî¼µ†JY¦é{¦89µkTׂ&-Fi¨Ùƶ=)eÙä¤y (t™Ì²R¢Ï6¹1[®–áŠ[K¾øUú׶¦mÛ*¥mÛ*n–6•›G¼â-™âÙξ^¬)‹/™ks)Õ”&Ëæç¦é¹óh{m.ƒª!®z¦²f³³”ñß#}瞧‹Bp©òßQ¡n§Šš”7Ú\crw‚sàJ›œ{ -M¨äpžõØÉ> -endobj -602 0 obj -<< -/Filter /FlateDecode -/Length 4927 ->> -stream -xœÝ]K$9¾×¯¨óJëõû!¡= `+VŒà€8L×LƒP/Ò¿ÏÎô#‘êî쨷*#ìt8üùsØÎt©«Ä¿¯þ­1¦àíõöã始ÁIü—S,.²úüwUÒáTòÑ]$ "$§ƒ¿¾\ð¿á²OþrùóW—.ó¨«)y­Üõ_ŸIe1éU‚J*èœ!hç•WUL‰SÁ ¬¶kU€6EÇ©`†Õ:2ª3Œ ʬU fè ,§‚*¹À¨”„JKÉê`ˆ´ÖhF§œðpºçtZ “N¬.@§‚ætƬrŠÓ%á½w‘ÓYØâ’TœÎÁ§¬•œ¶X§ó°Åx#YlÑ1N`‹†w£‹°芬¶H/§KF¸­eu :C\ë´tÂEc$§SJ8`Ó°:ô(4’ŒNÃ/]bu°Å©9-ÿ8x™#Ó[5õn#ât¶h,«ƒ-h|Ž´‡-2xŽЃñÇ:a“¶ èh„6r, c6ÃÑ€NNXŸÇF*è´âˆÀHØâ¬ç˜À(ØbÑsYl1Ér\`4lAŽ Œ-ÚŽ ØÖ-Э•Iq\`À·V*ÏqáÂLÉqãšè-Ç” (%Ž 8×€;9.0 ]t÷Àqëç5Ç´klôð®±Jr\`Á»Æ '«ƒ-0Ç`GaT4Xð®Q2p\`Á»èFšã´ª=ÇÀŠÐ1JŽ ,xWGi9.°à]0°º РǼ«ªÀÜà]í¤æ¸À‚wµÕžãE€$ €wA!Ž` –ãŒ÷ÐiÃqn&4ÂŽ xW% LŸ[ “ç¸ÀwUDó±º$T€Ç™~ / å}â¸Àw1PŽ xô9.Ç Œ¨šãÞU#5§ï*$Ǽ‹¸Áq\8…€ãŒoBI@—Õ%!SŒ8ð.hIs\€QXÈ&auAHŠã±ˆC9.ðà] ï‰ãÞ•ÎŽ ÷±qÖ»ùó©K3ÚÏÓ§|¦%¾–§–QÒy¹W){–Û¥ÑCýlW†ÊÃ}8ÂPömiÃÊr™f·¦!ù_¯¾»üêæy®u²Ïey ²@§´èê°Ò·Ù©¡zà’Kãj7ÝSûeÃÙ±QO4²@ºê #`"cwêl~À/—ùÈæÚÉG_Èm@×4Ÿš®uð—ñ]Göƒ=›à¥ªŠVÑ‚¥‚ŸÐ{2Ï£ÁWé]Bæð»ðóHÒ¾¥£6ÑßÓVCæÏk,Œ¶HØAð¥¿zŸ¸lÿ1ON[pYêðq–wØ­} ”=Ø©)ý'†èÕ·Q5Ô¦– d×õ²‹Dó6 bsKüêû?üîOß·–Ph:Ä'RÍ–-¾‡Éºzyè±ñäá)­#z*µ›®Wò8z¹÷0ywJ?yuíQWò<Ñ0:}·Ô#?w6™©ŒœÎMycL6ÔúÌ6ær}齧œ{çþ¥g“¸¬{©C®7ùܴ놎ÎfßôùÚÌ®K?_“ÝÅ×Å·ò‰B–®Î Õszò¹¼MS~®®ïë©i?<Ú»žëiæ6w4BŨÍ>­ïã -ìÐæáú>r®×Tç\ï=;h¿›·C¶º,üAmª'†`l£Md/-æ9̽'ì/1Ù°Ü|M~&öÒ·Yïg›\a©Ö¶¹Íôìó9m®œ1|›#z¾%ÍsןtÃåÍןçöqMªK7·¡Ò»~£o*†TgsÁC(÷\—_û{ío¼¯ˆ‡\iÇ™¤Yú`l‡j“ïêVü®šßKújÚm£'¾Úó“‘£Ÿ&nÛâµ³œÖø¬ö×ù&2õ ÓýßtþcÒ_ð|³Ï5¹Ýì\§8·³:í3ýsúŒª7óíž ˜þ.mÐœ Ÿ¾eã:Úy÷&O÷isûT\w2Ï«]Þ1 sÕ^LH¢sèÒl@—»ú8›S“Ü™.l˜a³Àä…L´q9Xè8à„ -‰dßîßQ›$œÇE`îèžrÛ˜÷ù½gí9Êí¼HmÆåÎIñX Z=¤ótÖy³á¡²vàÆu†Ø…ª¶+™Š­¦li-;¬½5y/‹Öõ_áyz¢®ó¯çQnj7ðƒÑïçy5zY´¶±éÒÜ -»eŒa<ËI(èïx6pÏS §eð°èÌV¶‚èɨDÏ …æž½Az–¶t&£§ß,H–6MvIïdž™ÌVî³y‡x›ûºáuìI[XØš ¶Þ%W“FÙ/¦ôS¦aesÚ—ÈnÚqÓ+ºÂ<Üv¡eEà*4`ÐgEJ.5¸‡>zðgF’3­m“>BßÉ<†>çÕú*GkÈ‹å jy7Úib¯Ü†Î54«CšOóç,×%À,ŸæÏÔ¥}úÒãÒk–o6PF[Ó1ò$ÊÎåy”5³éI—m”m-n­²žäî‚j Ç;”•ÖŠËV©»ý"Ûç uº<,öåéí{OSÔ}dV´)æ>Ã"rŽwÇ çóeeŸ*¹¹@¿‰ÔîGê¹<†T¯vùCˆ* }fÐÉ5gìrÍ°½à®¡µÝ€Œ­‘>ç½-íËvd”ž5ïC²=i®_æuß]›á»žó¤ž§Z=j½oÝ ng»g ™^W>¤'£múy2ÏÃ!Óìrhõâ8b÷ÈÚCÉjS`à¾Èwï,ÕÒ×Þ —eKM+¶ØÏ:-?{BÌ> -Ïåy8ž‹Ù™Äg=­g.{H¹EÖØ’=—ÞQ^ݲ&<ùÈÂÉ<‡¼ƒrlÉñ ’}]Ë®F²ï\ô_Ëaf +ÞáFö·Øý¼ÏY=”?ó¡V˺üü½v½]Œ¨´H2jz¦v½çò<zÃA\9>?6̈ÜÇ¡e»YËßõhùRsälÓV4ò4ÇÌLL[# 汋»gr[ö–<‡³:±¡ÅŽ§{2ÏÃ!v?Þ|˜ð­sÞ{fo_ º¹§W,ÒÛ}® öþ9üÉ<‡ÜýõÿmEçÝ8ûK¢» õ°÷m 9Ü?ï?™çáм÷®âу‚+bvzF¿:TZ˜zËø¤öˆ˜/ΟEö´¶­qä}÷­½u‹[»˜”Î}‘ÊîVžÌóhhŒûqì -YÝÓeOo›O×½éÖÝ©½âÝŠ­²ú‡GŒÏ®¬•Gy_?.ñÈŒ¯ˆWOæy8dÄ«ïX?}CLy]÷¬[.ö·ú5°´©¢ãÓ„‚ü÷†ÝÓúü)ñ-ñôm¨çÊÆ^rô¬hêÖ.ÏΈNæy8ô‹7öÚ-7KÙx.þÕñX+þïñÜ[zòJ[¬ˆ@HDBÔÑÓ('ó<J•<0ÏN;úmÊaº1B;CŒ¦EAÖ‡1^X -[wM=Y¿ Î<®ñÃ|ÎVpð„£3Æ騬ÔÛUÛô0ÉA–FÖG]ÕpÜÓòÜ’Ø7ñªÑR)Йt[o”èÏÓúq~¿ÀO/U›ÙeÆ™)˜ŸaÕå-”ü„áÌ™-äôÂêçÖù”ù%ßüâtš×«)­—íàù{;(@Ö—‚u>H@N/Âv/ÁÖû—s;{/æ¦y\ÊuX5ãOŸÏVTù|]é“h:€ÿê­ Ó]:a¤sU¥ÙTv´ºnäÌ?[ðSM5ÿ’¤Ãé`ËܬýÅœþ§Üà?^T²‚üŸO>ȧŽË,Žº\4ñË…Ž›°ŒØÚv|˜„óWYïeU=¡Èn © Ò˜Ò¸r.\wd½N3}Õ®Ô†ò»’N´²$2ù<¼©`Fˆœ^­ÅT{)Ô,¦“û£0󕯦ÐK¡UØnhû´í†-ã¡XÙ ÝZ(»âšXÛb¢“^«ë|VÓ{ש®ùKZºæÜÞ.(ÖŒÒVJç…—[ïjYe¦3*U©+)kƒØÚNŒèva„_Ërk‹Æ¬ú®ÕèÔÕI lM—J:Ft»°é滹ÚR¡¦Š-k*žo¦4¨›XÏÈnݱ&~Š¤ë#Ò”¯K1’QÛZÆ7z%¼]œ­=±‰_..0©]ã -é -ôz¡éî[ÅQ6¼4)ÍÞ¦oèšEFgVÒÿ'ªpUJÌ“¿([!Eçζ«VõÍ*^:©—ûÊ:–󀲡¥íÜ¥B¨˜íý¥¢dÒéÖÔÕc iç²NR1Hª’ÇR¬*ꪘ(¬ø§re/”Å×Ä®Bùe!nvºÚU¬…yÕÁÝÕû6±¯ÜùÒ‹CmqE‡W{ë};Ìá-lvôc)¤‰µ¬£]+¬6Ã@ -MÜhô%‹Ko­ìD?ŒR¾¶NƸ·Ëß/y˜ÓÆi:ñÁEgâ÷Q—ºþi¾Ãß?.öúûrë4ŽkYÙBWn]ˆû‘Œ~A%žÓ/ªaí6PÛŽö›XÅ–Ö4Ôq\뎯mË.±—”ª2¨ªÚ^èš;±®ÐÕÊtžq;ÂÛàÅȼµmÚ€=]šR9:Á‚Žï-V᢯.cÓ™6$_»[⊒¶!Z«»Œ¸èoŠË®ÀÔ,[˜I€ì ©®'ûm߆Êê.º0‚.MÓÙ6Ìö·.²\‹. B`Ó_ô7¶ª/4-ìaŒ~u÷zs»Ï&ûÒHZ©½J_zi­I+áñR“/ýò²Ž9&þP:oóP½—éúž©ANë-´°‹‹£4¶ls×j‹lQh(*]@&‹¬Þ1›üœ­¸%×8²^ -5¦hÒ¿÷žv}«T'º¾Uü*-Ý·Œx5Z25r=œC«l)k,™ïsW©m)M‘­Ë¶Ë²Ëh{íªa›!¾E¦²es«”¯â¿×ô)Ï° -Á¥*ÿòʈŠíØr ÔV£Mþ%:½‡Xú1‘`c·Trwžýµ“õ -IY;éKBskŒ Š~S7áŠt¬ènU~oÏ–$[¥ûÀ, 5š“«•¡ü£Ry 0/ÕË@SEú 9CAo¤Ÿ¾ RñçVÒc·ÅòIþ÷_H­þ7 -endstream -endobj -603 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 602 0 R -/Resources 4 0 R -/Annots [ 604 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -604 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 555.123057 101.433071 536.373057 ] -/BS << -/W 0 ->> -/Dest (cb86-1) ->> -endobj -605 0 obj -<< -/Filter /FlateDecode -/Length 4924 ->> -stream -xœí]M$¹q½×¯¨³€Mñû0t0 ÃZC‚ìƒàÃtͶ£×ÀJÿ}½`&ÉHfdUVuÏl­àtwe?‚ÁÇÇ 3‹©Ï -ÿ¿Óø•œžRÊ1¸óåÇÓO'5E¯ð¯¤X]õñýY+'¯sHþü# lœbö&†óÛ ¿†Kžüíô_¿:ýï:>›)ç`´?ÿõR9;Å£ò‚*C¥´TÞO1UК˜³¤‚>:㶪3\N^RÁ gLT fXµÝª2Ì0Q;I3töQPi;´QJÔÁåœ5‚Nû)ÀéAÒ=…”Mu:¤³v -Ñi¯%]žB>I:[|VZÒyØâµsJÒÁgS”t¶Ø`•¨ƒ-&E+é"l1ðNt ¶]IÔÁ”–tÙN>'çD]†N¥˜¶:£ü䓵JÒi=y`ÓŠ:Œ(tRˆ‚ÎÀ– |u°Å[e$…-ÿ%xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØ_ÿ|úõgL¾ç?ÿíôú«²SVÿk%µC|…a`2ºÛ#úÖh¾EÑ Uþ aà#yȈ˜»‘´¸lº‘‘˜þèßyûae9 'Ó‘ë¢@{™Â¥~0Q)ÝaQŒõ¡9³›|wž›&GDùÜ>hÙÖî*^o÷•dëÝy>ÊVÛº5SÒR-`ki‚v@ÂY> -‹ND1il6Äúʈו[%þçO'Àbż·Ü.™ùˆC4Ýøó§O¿þÝ÷ßÿË¿þüôéõô§Rʇß|§éošÿºËr—ëE^ÓŠeÑF󊿘áYä ?¸6¤{™¯%å1n){ÑÛÏ]–r_X>×óóº ÇOûy®v/7,¶,å”<(KcJT¯ÔŽÿ>úþôÛOkÈ”øü>”Ëò”} pa±\w0먻:8Ì @Dþq|¤ƒãuieåà³5‹m—Ðd»ý'ºßp/žæy6@aÍw Q„ŠGzÖú_6õ¼ÏþTÙF9A/.c·Pu,ÏÓ¡Êâ©Ÿ‹wîîÝã›eDÑ_E?;Há!õŒË´æ¸Ž”cyž)×Ã¥éÕÉ ¡”÷k=yØÛEgÖij¸EáõJïä_zµæ­6¬tÕ†¸NÓÂ7Ýí“B:ÏÓýÀÚXíüÜe¢m;õ×vImw¯ƒ­Õ—¥ìÌt¯kÛ‹ŸÌZY(…ÅQVÉТü:Zåy:´Þˆ¿Òºª‡é±†Ö˵ç´6dj¡œA¥¼¯…Ý?·:Ô:ß -‘;mvRýC›9âüKE™î3KÖõ5¿]nóªé1š£ï¡ê*Ræy6¤šëqÝ-4ŽãŸ£©õðˆ"Æibúé#wæ-"‡‘P‘ò-G\CÞÐþ#uí"²ÇwžVB(nÍôó<"oÄ„;³î8ë·^Ë[Yã.;äQo}Ëðà¿ü!ø¾ƒyžÁ7bÕ@ðZîAö8Û¶˜ò…Åz€àc1ã³ |½=¥‡zœÃRíÖšü`ž§CïØõæøÿ8„3„Þ3j®µaŒJŸ4]Ú$Ú+£Ñ>°ïx0ϳ¡ÑÞˆOÇh¼m2ôRYSWÄHøùz™ÕWËþ¼pÛ× Ù/‡cwkïß8˜çé{=Ž}_Œõ3 4®ÓmP¨?†_ßo_Ý?ý4¦¹Å¹ìµÌótÈ=¿¶¤({tDø -E lß“v×9ErkD”yÚ£}wñK”ðhþ6"Æ$–·íxIsÈè'ÌKCÙî…!rôó&BÚAêw¹æy:¤ÞˆU? ±cõ¢eœx‡×W\¼Ìùé>ÑÈi­¾¡JoصâÄaŽó\ÛCmiù¼°Ó6jÓ×]!ʈuijó<bÝxöÆèóÅyïdžvµx3jÙAó1îÁ•d{Æçe9_qn¥2[)hÙ¬öêZF€ùnµÌ¯Çù}¹a>Q¯;H¶÷sñÁ%íÐК®ÝkaŸÙó=¦yzµ õäå9­'z©-œ¯7ò4zš{™<<§Ÿ=»õª¯y^ø®£w.»èýœ§³Å¬oíXl+yCooß/(½S¼ÈÝ»ûOK ÕúÒâ²'¶x˜=÷Ò1¬g•k»üx–~¹v—îíêU¾‡Í#Y–O­Ó“×ÕeiiXšxÛ’¢¼,༴Í.½îižHKOQ¯}Ù–,oFGènËQK»æ6—v_³ùŽª·eåê]3ó„`¥”3! eÏè_£²£¹ûšüLf.A‹M¾rUïÛÒgfñù’¶´Ÿê ‹íyùävLÔ4¯lD™ŠÚ9ï¼C¶ôg6i–néú¹ãw?ú¦aH3›+b-s[ñmäɾÒË}­’vae×>û¡ÙXÛªßu÷ûŠ%ôº/ª_¤ògƺê§8úif·=fÛcµ-£µ1â™O’ÐÎ8—ËÛ'sÈuþlp—^QŒYú©Œ¼”é¯ú'oü³<¹ZÛTÚY1ýºmÓØG|\Ö¾ï\ÛïA4{¿H³Ïûú‡ßï¨gçÈíÏXg¯Ûo¤ú¿È+ûrΠEí¹%}(Š<˜ç¡Î>0|ôÍ^:¯Ô{P‡|s•ºd\>êåF¨eÊÜ »æ0©LÇVÚçìDmõ¶÷n‰ÆæÉ\D¡Dÿz+·KåTíD{nåöaÊ}&åæ· ÙÞô)¯«p>Ø}«'n¶ÞÙrr,¢ù€çéüt¸.D¹·rS¿¬ù8Ï»d0Ê’CH²MŸ—^¸ZǸh9I#ê|@ïä,o$ú©¥¢ß)zoY]º•_,é*þ#t˜ˆÕ<Ì/QEŒ±çGñÛ‰^(ã1Øæ ˜„ËGÕÊrº-+«ì‚É;NyL‰ R-ûÚ“fMµÌµ5”ß×ltЯS5‘-GÝÎ Bä z+¦Ö«I/bz)Ošìrš)45a/Ðñ´½Àžñ¦X[Å„~+T¬º.6®Ú£é÷æºÐ…Ítî:ͺ¿¦5‘uç5áå„jí(}¥ô*ZTõ®QMf™Q¹I}MÙ:ĵ~D—“ üNÕ¢:³éY¯Ñê³0U ¹–.×t‚èrÓ-¥ùÖS±¥J=k®žï¦t¨ÓÒ|_vaû3‘Þ2Æƈ²õãZLKÿÖ×*1ܘðrò®Ä.~;ù(¤ö+”¯ÐãBËÊmâ¤:^º”Î6œ?ahVGM¿gªðMJÌS>h× EGÊ÷«^ Í&^;‰ËCc'y@»ØÓ2wéf¹¿tRBz"Ý–ºyl%e.cò˜«yDR<ÖbÝP×ÄDaÕ?+¹PU#P]ûå·•¸ÛéÛPõªU4ƒ»oåvqhÜùÆűõ¸¦÷4{[¹ óUx)›ÅýT+éb£Úl×+cÂnH¡‹;¾q­ÊqpËÇ>Ȥ÷rúË©LsÆzCËË€0>®£ÖçÿCšïñó?'wþ}=_}žÇjla·®Ä|&£—£¥ƒbzYZ¶aÄÆ1ÚïbzZ_ÑÐæqc_»ž]ub¯)ucPÝ´\è»™Ø4èm™güáeðb dÞÛ7}ž/mm-—édþj.xsp™ºÎö)ùÌŠÄ%íS:´Î°Œ¸à…â’U˜»e+3 ¼’æz²ßñ®°T»`a]Ú®s}šåEWYiK‚Øò ^°Ó¼Ò¼²G0úááõî~_Lu´Q{“¾qikIáÉR[.Ãú²Í9¶,4ëàíjeY6öl rúÐè¡…[]pŒ2iêÙ–žWêªlUi(]@¦ª¬•«MaÉVÝRZœD/ÅSté_¸§=ï•æDÏ{%lÒR¹uÆkÑ’mçpŽ½uª§l±ddçnR×SÚ*ÛÖíÖu×ÙöÌšáº!¡G¦ªgó›”ñß#cgÎ3ìBH©êÿ²3¢S# &j§£5¶¼dV¥À!ŽÞ]b[%wç¹¾w²Ý!©{'¼&tÇä¬UQÓë‚eÞ È·¬¨ú*a9[Vb“îs€°5ÔiNmv†Êû"Ëeƒ¨]FZ*Òëa-½‰Þj•–oÉgÅßo=þwѦ’ -endstream -endobj -606 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 605 0 R -/Resources 4 0 R -/Annots [ 607 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -607 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 123.873057 101.433071 105.123057 ] -/BS << -/W 0 ->> -/Dest (cb89-1) ->> -endobj -608 0 obj -<< -/Filter /FlateDecode -/Length 4708 ->> -stream -xœÝ]KäH¾×¯¨3Òš|?$Ä ‹@¬Áq˜©™ÙêEZ8ð÷ùÂv>œÛYU=‡iõt9"áÌ/¿Œ g9åUàç‰ÿ‚‘CÑ;s½ýxùé"oþ)£ú;üz{•ÂøÁÊ肽þHí­òîúrÁÍeüåòן]þ¹Ì#¯jˆÑ)i¯ÿúD*£¼–QE¨„ÔŒÊÚÁ{e£rrÀ-ÆÈ©`†õF™µÊà ƒåT0Ã(U€Úx©×ª3”—†SÁ ­gTRÀ©„`u0D££“vphtÇé”\ˆ*²:ôŠÓi=8o¤•œ.Î98-6 -Éé,l±ÒÁé`‹ÑÁs:[´Ó‚ÕÁ¼æt¶(´Ndt¶]ÕÁá„ätQ6cX]„NÖ:%ì`ƒÖ‚ÓI9X`S³:Œ(t’óŒNÁ'ldu°Åj¡8†-?œÇÊÁáGAñÇÊûÁDe8PA&˜À±€ -q0ÞkŽT´ƒqÑs< …„NIŽ´€-Ö8Ž ´„-#—ÕÁ ÇZÁ$à¸@kØ¢Œæ¸@ƒm ÐÂqÝ%Ç|k„th.Ìh0®Îp\ A¹€Rä¸@ƒs5¸“ã ÒÅp÷h°®¶Nq\ A»ÚÇqïj#Ǽ«5r²:Øs\v´ šãÞÕRxŽ xÃHq\€^€CÇq°2¨Ç¼«‚0ð®ò˜X05Ǽ«,n›¹Á»Ê -Åqï*£ÇÊþ¬7Þ…XÞ€- XŽ 0ßC§4Ç(lPpK8.°à]1ý1cl5ÈǼ+ºÕÅAz´83nÑʃt.r\`Á»˜¨4Ǽ ú €ą̃Šã Þ•35§ïJǼ ¿Ár\8 ǘß)]VCà¸À‚wAKŠã̃`VçIr\2üPŽ xÓ{ä¸Àw…ÕšãÞn«ƒ-:(Ž à×@‡aËŒi8«ƒP` -NÞ=ÌéÀ»BøÈqïÒ´Éqó$Pã8À V¸`†è½ã¸ÀE\>Vç\[Ž ¼€À‚²Xl1Þp\Çzˆ:Ž ¼‚-Z)Ž ¼‚-p8.€G0D ï›]ÀÀ…]ØO‘ã0Ù¢1xë°C«r:§†àA׬.@'=ÇÞ›S±ä¸k©!`Ðs\‚È]â¸f@' Ç> Ž 0 AÁ3çt¶ÀÉ渞.tRr\l7ÈqAP«L'8.À µ$ššÓAëÌÀqA0n W‘ã ¬4Á”¬¶€&8.@ÿÐbÛr\€Å›Xæ°:Ø¢­á¸ EËÍÀqA°E Åq°IëMÏqA°ÇQ¸ÜdŽ @ãÐ ÁqA”ëMe8.À¸¤õfภjõ&–x̸ÚÑzÓs\ l±pQXlÓ™q NÂzìÄélQÑp\£õfà¸nÖ›ðèÙ@ õ¦gÃà]ÁLœ¼ ÏÔq\€ ¬7áÙrk{âµþ¯ ´â lÄ@€z-æV6d À½˜­<3 _Z#±A˜‰•eplÔÓ*”R°aþ…?fVœ0EŸê•<æÌàé&#†…wŸ×%™ÞH6ÅÇþõýåçï1ù^¿ÿ÷åóÏÆHYú‘pÆÔþ†Šèn ï[âöµÃs´ †Ý“–*õ±%AƒsŒÂtGÿ®ë KaèwËj%îy]o Éœ„†µµ7Ó+­Sýp{0”¨z¬/0 t`&`=0+“ïÌòŠ ¬àà8¤ÑQ…b\Àùf‘øWï.M³S 6_Jm“ù ~¬Ø²×w?^~þ»o¿ýÍoÿ ¬¿û|ùÛ/À³H/õô S˜Û/¿‘[7ýeïqm§kæ¿fNç—ò”ŽòYµ–åt¨×6e™ÏUÞ¹ÜTÉŠ­¿üûõÝ·—_¿+CŠÃÍ­à¨õ±Zwc„ - 1…Ô@÷ÿë?þé÷ùc¹,%aÕGö­ø•“ ¤0Ö«#ÙHöMipcýU…`p‡…OÒT8¶ñÇlïXðŠ², ³B$^ÜcgžgÐø%@HáÙ&„dĨ%ºrïÛù/ýúéšÕÙ‚hù¾Ióqþ;Ë•šõé÷ãü7Vi?Ìèý4Û•ÛŒ -§Œ =ëšÕŽùÑCe-ÊKuú¦\ŸêÞ@Vs1¢e}yN‚²b6Ù”%~Iè"^£¶¢á´Uo¥VþXz‰Óòfî­°ìã–½6¦ùÔ á†Ï±Ô•ì\Ô§¶ËuÈÌh“L9éº*/}Îí𹿮±MeÕξÕo 5ªû‘Ú—çlHUb—9„HÙðJÒ‰5gìrÍm‰žw5½mdlÍöcÞÛҾю„Œ4²ä4êØ‘4ßßÈ뮺ÖÍg5ç‰#O–ûÈ÷}+vÙ–?UÉ·…LzŠ6£Œ×ÆÀw€ÌÎ<§C¦ÞåÐÜŠíŒ]#k%nÝê î{B¾{ dÉ’>±¬« y…Å÷3V ð¡!f…}yN‡Â>‘]MlqÖ‡õêe©-÷±Èj{²æÒ;êKuŒ®è‡)ÿ´2Y–ßúÅu}:lÛºàÉÔv·ê>|HUûƒäêpÙÜ™çtÈ;ð!ÛžôË–^¡¯êÙÕL¶ñ™óþs=ÌjaÅ;ÜÌþŒÝŸ÷ý9£šúg>T²ñJ¾ø¨Ý@oå#J|AQÈs½}yΆ^}àW6‘–vEdß7=[­Zöø®FË[­‘G›¶¼‘³ÏÌø´ÙÓ¨fõEœáž•Ü–½)Ï᪎G¬.¾c7b;óœ±ûþæi|Àg×¼÷¬ÞÞÀ»¹gT,Ò›}®Õæþ5|gžÓ!wßGý‹è¼g¿%ºZGßšÝýëþÎ<§Có¾ß»òG<nvΈÙut(õ0Ûø«-bÞœ?“ìÃڶ‘÷•›Gë·V>)mËrÜŽ³Æ¾rÓ«' ‹ø£`#d+ô'/ãs…´ŽÈÚ˜î©y‰G¦yÀ_íÌs:dø«¯x|ÕO›{ñÕ—çtø:ð°í㪯îêc7á‚XåÕú¸ç>;Úëôm¡\}çKͶÝfL+Þ Æz`wdgžÓ!êà)qx¬g)¾ñ5³Ïsöo *Þ¿ìÌs6Tùƒ]‹ÿcÞ¹»w_oæuàû¢^yN‡”ƒ§¯U\ñ‹ùÆ;´Û}Q=qYî›%œWW?KÊ+®*¦vÝX¹a[G\ø®'')^ûÀÞVoîßÝ™çth=ð¿Þp ÿÅpïÞ‚žÈÜ»{ÆøD›î½XïhŸ™-"iH­"[½ß·êÌs:¤DÃŽ3ŠÇw“¼bäíë‹šm ò€yΆÈÐ÷M–§v#ßóDá-Góíïÿ‡¢{yN‡à_õ•öÓ?ý®€WþÕ¦M]>ãY¾ÞêIkï÷:󜽾ëWü-¿W5oºkqÄ;óœþéÆnèÕ·>láš¼jhð¿Ü›Úîâß¼&Åj¿Ô×ñÈ}àÙyΆÜØ÷÷á5B›ýdܾ°×à×güíÝøAÛNmšÎÄZ;óœ¹}þë½o”X èÆ ¡Š{Rt}±žã fÏ葧íg±ú×Ýùë]9m$©ÊÛîç?Ã[/âϹ;󜩾jõþÒ¿Uo+{ë7ö¨»ö¾—ÕæÙ‹¡Þóž.º§/»BÜ@ìþlgžÓ!ö¹Ý‡çóóž÷dN»Z|ðÝñ·3ÏÙÐŒÖÚ%àg—Sû/‡mÓðÞ«(ÓC÷ú7ÉçW[#Ó+…Ó}õl‰übC9Ãï»ùeòÞ¢§,¤GoZÕ»é3æ“@lý&ûôªõ+ý´*/ÊF‘tŠ¤W$EO/l½—[}š<¸ñ-ÍnöÌæݨé{{RŠü&P•Þå=>MŸæs;2‹ßëm>•nßëí¦ôô^oê‚qN¦´núËš?å„Ä s}4S~zöé'ý¢|;×]ÙcæwŒç{°éVÝøÓňÈñ)á 2yuf -'>^éÐ’ ´‘©ÛäØÐòº‘s<›ó§œj>®SÐ ^tzËt–gu1§ÿiìð/2šÚŸFÿt´žŘÌl+~¹ÐÑŠ†3¨N1 ç"—edJ˜e· +Û”ÚbþsáôRò飲én(¿MÙèÈ #R"=ú0UÌ‘Óɵ˜î^ rÓñ”aÐó•Ë¦X—l†°hê´¥À’ñP,µ¨„v-UuE¬L²GÒqF¹é\fÓ릓U÷§´ÊWݹ'¼]P­n¥/¬”ÅKE¥ÖU"ËteTÌR›Ræ1¹ŸÑí¿©hƒÎÌúª×èh¡ILNS:Ft»°éæÒlî)ŸS…’5¦–/¦¨ëž‘m4K=·[¡ÓÇ¥ØQ¤¾¡ÂZ okòH,â—‹õLj[¸BؽZ¨«r³8ˆ‚—"¥Ó' Í$£ƒYèÿ‰*l–óŒèd™ -¨¯J%46³xÙHµÜeÖ1\ HãKÚª¹¤÷³u{É ˜ôDº9un±…´j²Jîc2H*“ÇR,3겘(,µOæÊZ(’¨®ˆm†òËB\ì´y¨ì©2'+¸Û\n»Ì/µØç—tRV¶7—[a> oãÄfZ1A?¤JŠX‰<Û•Ê*a1 ¤PÄ…F_Fq­™èôßô± 2nƽ]~¸ŒÓœÒVE|pZÒÁµ×%¯ÿAšoñû‹¹þ!44ÍãJd¶P™[âz&£c‚C§˜Ž NÂÙäæl©YÆ;l+ùìSéuKÛºWr#ÚºWÜ*-•›f¼ì-éìÙξÜ()³/™«sg©))u’­ë6˺Ól{­nÃC\ñLEÉfW)â¿GÆΔ§‰Bp©ÒÏ‘¡œÍ‡‰ÚH¯•¾Frw‚sàC'æzªPÉÝyöc'ëIŠÔ5¡;£µðxÛ0áŠx¬¨Šz™5ä³EÁÞÒ} À„† -͉Udh<9}ŒQŽ¢|éi©hãx¾­Îwö‚?ÂmŒÈ™Eødüù/£æ  -endstream -endobj -609 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 608 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -610 0 obj -<< -/Filter /FlateDecode -/Length 4919 ->> -stream -xœå]M$¹q½×¯¨³€Mñû0ö @†½† ÈÁ‡éši F¯•þû~ÁL’‘ÌȬ¬žÞAÁÚAoUFð#|| 2™Yúªðïÿ%§§”r îzûùòËEMÑ+üWR¬.Šúø‹þª•‹“×9$ý™6N1{Ãõí‚ÿ —<ùÛå?~sùïu}5SÎÁhýÛWR9;Å£ò‚*C¥´TÞO1UК˜³¤‚>:㶪3\N^RÁ gLT fXµÝª2Ì0Q;I3töQPi;´QJÔÁåœ5‚Nû)ÀéAÒ=…”Mu:¤³v -Ñi¯%]žB>I:[|VZÒyØâµsJÒÁgS”t¶Ø`•¨ƒ-&E+é"l1ðNt ¶]IÔÁ”–tÙN>'çD]†N¥˜¶:£ü䓵JÒi=y`ÓŠ:Œ(tRˆ‚ÎÀ– |u°Å[e$…-ÿ$xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØßþrùígL¾×¿üÏåõ7e§¬þÓJj‡ø -ÃÀdt·Gô­Ñ|°6Å4XöPbª6æn–.Ûld&<úïºý²²¦’±È•P Õ?¤j©ÜSJwXcE`h–ì¶>œç®É1A¾¶/Z¶•ùˆÕËœ$Øúpž·5`â!&¯”!y/ñÇv~¯ë¡^„©ðÆ@=›éƒ¶­G3uBéP -Ê0óŸ¤ Zšú™Éfù@ƒðˆÒØlh-bí£Ü*ñï>]@¦‹ó>}»dæ#¦Ó˜¿~úùòÛþé§ü§?½^þüJ¹¯?þ ñéÓüéüò¹\ûåÚk&‹Cš0ÒÎsý^?Ýk×Ý« U€—4vÑ݆òF»òV¦õPFÕ«ùK©Rýµ¶¼mQ=-·Ù’™QhÃסÞÛ"ϽŒZ_McÔâ£ÑÇÍ·?þçõÓO—ßZƒ³¬ªpn\¦™õÏç²| ž çüwã=(=ïf.¼aßo=}“~àùGŸ½Ìul(ŒC¨Ú¥¬ ]º#UH7™”.‹ËCìžÌólàÕæ„7½U‘ôDÖ3¯ëÞsŸ×¤µ!¨—N2+«h3;e>Šø{¶›hç2Ü6RºÅp·®”Z÷8JÏåy:”ºcŽÝCœ÷ÂR¹¡r€¤ŠÖ‚L¤í0=ñåm([@cÓ(I-+´ý*S÷GÏO;Heªr“Ñ.9w©çò<RƒÚo½¿7êê÷6²FêŠXyÍV¿ö‘Øž¸.sŒfš¿nC{õ¯ò0ôäJìdž§CëqìÚzîuâ×æQݽL|cë 膞¸bŒFÔšW†hƒ6äÞXž!êuƒŽ5Ö¯ãÐÒ®adQY6<6âÌ;F[±9²vòH(­ÛÙì2‚z|/ádžgC°9Ž_GþòãìÏz܆¤êùGÞÕK‘ñç€ÔÑ?Æ€Œ¯8 -7ˆÒŒ ™­´lVH{u-#À|·Zæ×Æã/,Í0Ÿ¨×$Ûǹødž§CòqŒk…õl㟧îÙ…gGþ¹°–5p©u,ÚçXz¨ˆQEâP÷ÀçG£±Ôi·yWL#cú;½'ó<¦ïlñó1Íéů‘aÇ•Qïôu‹Gü\&1»Ä”cýGý¼™ÿÇ12bF˜ãi 3¢â£ÙhŒg$L·4_ÖñPÁêà/qFáí”ñÛc]§éØž†Ÿïá÷\ž§Ãï½]¿+?<Íb×I$µÞ_åXá±é€©3±íûvšÔî^ŠcØmc`Ü_YÊ)û¯âX›åÛ gê B±žh—ìÞ¬uŽµk“JOüþÿö/úÛ§¤ZÓµ-ì{ž½aš—‡ÛOžÓzb —Úºùz#OÃŽÐ’ß³«\7f¨kjòƒÜŽ‚«šæ• Ó±Gyçý­¥<³I³tKÖ]Ä¿ûÑ7 CšÙ\ñk™ÛúuÍ×Æì+½Ü•*iÕÜÎ6žôÚ|¼}Së-ø$ÿ}0çYÜ'~5~|GßlÇëñXÛTýYì\úªa{ùÔ L^ʯö’í/½M¥#¯Gܹî?™3çt$/»ÜaÞ¡×öž<&úµL­“­:>ÇØöMÁ–C"ôÄ©ØçdžwÕùÀ Ç7{½›’÷2ò-AJã¢G/·ï,›ZçNØ­0‡Ie:â>TXéŽo Ïüñ¸Dcóä.¢P¢½—Û¥òDH¢=÷rû0e>“r— |¼­bïzÈ”G-vÇCßëvÿÝÖ;[N=[-Yz×óôì\¢ÜÎ{¹©ßÀ Ö|œç]2eÉ…|€M¿Ðâqc¨+sRö@x'ˆÁ­Yˆ”OΫ€ÉîAÛ ƒž¡Ëô´ÊPiÙ;¤èNñL`ÞšsGyNæy’_ã>˜|Ä}ê3Ÿ²†>_…ýò¦Àê\׸€7¹†m[õ¹NÉë²ëÔi¾,Ÿ‹Ü˜E_ÿ¾,Ÿ™¥}ùÞ‘ÍÇÇŠÞtÄœ<ðz2Ï“ ¬›ÂÊ9Žu´mÍ^ǯöŽ49¾•)Jy®ãRl¶|ï Æôãm]¤Æø8RÏåy:¤æC>üÖcKï>Ž5ôöxÀyƒ~ðú¶¶o—ífY¤¥}…×»¶Ãw³äÉ;#O÷v´vßXŒqpSm™I5>¤·08½‡Îåy6d–§fê!RkÞãÈ:B‰t@„çýü„|÷ÈÒ=ýæÚë8‚wPØc?‡ «œ@ˆ9Fá¹Ö³›™lç»ý·z„Õ†wŽZ¿Çî×ãxΙ¡þúˆÂð`ɯ?jwÐËbDm¦¬’Q÷¶_Næy:ôÞ‰+ǃzÊÈz–ï•ðGË÷|Øo7yQýA”ñ0öm¨cØc|h%·goÍswU'#6÷Øñ4bOæy6ÄæãxóibÀo]ó>²zûÑÍ#£b•Þsm6¯áOæy:äǨÿßvt>Œ³¿'º+Z4ûÇ×ý'ó<šãÞM“Ñ/¥“bú¥ *lÆ c´ßÅ:õ´¾¢¡ÍãÆ0¾v=»êÄ^SêÆ ºi¹Ðw2±iÐ5Ú2Ïøámðb d¾µoú„=_ÚÚ8zÞ†^K]­Âo.S×Ù>%_Y‘¸¢¤}J‡Ö–¼P\² -s·le&’WÒ\Oö;Þ–êb,Œ KÛu®O³¼è*+­`I[~Á všWšWöF¿{x}s¿/&‡:HÚ¨½I߸´5Œ¤ðd©-—a}Ùæ[U©ƒ·{¨•eÙس-ÈéC£‡nuÁ1ʤ©g[x^©«²U¥u¢ht™ª²Vb¬6…%[uKiq½[LÑ¥åžö¼Wš=ï•°IKåÖ¯EK¶@žÃ9öÖ©ž²Å’Iž»I]Oi«l[·[×]gÛ+k†ë†„™ªžÍoR¾‹ÿÞ3væ<Ã.„”ªþ+;#:õ×ñc¢v:ZcË/,ª8ÄÑäD—ØVÉÃyŽ÷N¶;$uï„ׄµ*jú­LÙ„7(ò}+ªþŽ¦œ-+±I9@Øê4§6;CåÇÒÊÞfÙ j—ô4ˆ¡ßF´ô&úI§¨´ü&Š²²Ú>)ÿþO—{ -endstream -endobj -611 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 610 0 R -/Resources 4 0 R -/Annots [ 612 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -612 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 438.873057 101.433071 420.123057 ] -/BS << -/W 0 ->> -/Dest (cb92-1) ->> -endobj -613 0 obj -<< -/Filter /FlateDecode -/Length 4784 ->> -stream -xœí]K䶾÷¯è³Ë|?€À‡1bd‘‚vÚÁ8€Cþ~¾’D²Ä.µÔ=³kmÌN«Š"ùñc±Ä–ôYáçsÿ’ÓCJ9w¾ütúù¤†èþ)£ú[üFÖÊÅÁë’?ÿD‡˜½‰áürÂÝ%OþrúÛg§-óè³rFûó/ß“ÊÙ!¦•T*¥­ ò~ˆÑø ¨‚ÐÄœ%ÌðÑw­Š0Ãåä%ÌpÆ$A•`†uQÛkU†&j'©`†Î> -*­`‡6J‰:¢œ³FÐi?tztF!e“E]„NG#é¬BtÚkI—‡‚O’ÎÁŸ•–t¶x휒t°ÅÙ%]€-6X%ê`‹IÑJº[ z' º[€®$ê`‹ -JKºlŸ“s¢.C§RL×:£üà“µJÒi=x`ÓŠ:Ì( Rˆ‚ÎÀ– |u°Å[e$…-?’<ámLI˜­†f·UVK:[Œ‰NÔÁ ¾Ä&ÀƒD˜Añ’Ä&ÆÁeã$0É.¹$±€Iyp1Z‰Löƒ 9J<`•†Îh‰¬‚-Þ‰ ¬†-3WÔÁ›ÄÖÀ$¸ÀZØbœ•¸À‚mÐ"qÝ:•µÄ|ë”X.ÌTX0®MÁI\`A¹€R–¸À‚s-¸Sâ ÒÅtX°®õÁH\`A»Ö¥ qïZ§•Ä¼k-rŠ:ØK\v¬NVâÞµZE‰ xÓÈH\€Q€Ã q°2˜””ļk’r8ð®‰XD]0­Ä¼k<š ­Üà]ã•‘¸Àw3Aâeÿ(Ñï‚B¼ìÀ¬ÄXï¡3Vâ6¸%xð®ÎXþ„9 ¶tÊAâÞÕ Ã'êò #z\˜·èåA‡%.ðà],TVâÞ}&‰ ÀñVT#qïj‹•ZÒwµÉJâÞ…ßà%. @â¬oƒV€®¨ËƒÊ)I\àÁ» %#qVáA%0‰¨‹€¤%. -~¨Ä¼‹å=K\À»Ê[+qAï*·GÔÁ›ŒÄðk Ã´æ4œÕA0…¤ï*–tà]¥b–¸ €wiÙ”¸ D˜$q à‰ BrCŽ1H\²‚.Ÿ¨ ®½ÄQAàAY¢¶¸è$.€c=d›“ÄÑÀkŒÄÑÀ¸ À#²†÷-n` à"îl§(q˜lHÙ9‰ ¢Ø¡W%]0CŠ kQ— ÓQâ‚Ý€¥XK\€½Ô0é%.Áä.I\3 ÓNₘa ¦ŠÄXŠ†dà™K: [àdK\O:­%.H¶À”¸ ™„]fP`…Æ^]-霢}f’¸ ¹0«(q¦ vš`JQ[@`|h³í%.Àæ›MlsDl±ÞI\’¡íf’¸ %Øb”‘¸Ø¤ýf”¸ +ØO€Â@n²Ä qè”’¸ ë„ý¦q`^Ò~3I\­Â~[’‡Œˆ¹©A‹sÐŒÄòGÿÎ×–ÃðoÇêm­>c«w´ €º ÅB°}í-¾3ˬ°9ÂjÆfCìÈÁ¯Tn‘ø·ïNêlÅ­—Ì|¬—Ø ;Øüî§Óøúëßýþ/€Ñ»çÓߣ”_~®Ù_÷<ý¥û~Ö¥Yçç¿ó5£\Éçf™žóØeZ¯¦¿pÇ|ô[ë,iæò=¨Õ]f]êt®´Á,íõHì#Ë÷Ü¥›ÿöí,¶ÕòWÛÒÒžE{üzý¥=¥Z{¾üÇùÝק¯Þ-A7z¢Ó4±„—‰nât_–×àôCÀÛÚð¼Zt©ãðx?“í†ôyÙíµœËm‚ܯ+H×á*pvsú¾ü‹žµžÒuøc³ã›ï…b±ß÷ízb}ÙÛöË×Öõÿd)z_ˆ›]ø$Žœ„›˜Ý™çh Åê&©*-e¤h4­¾F=SLiéwn²ëGíùÑÌI«R)ë‰ìû¥Íð”¶éÙ%šjù|–vÈgñó - ÝÊê¤-Jï; [–»ü1æ+~z<÷š™,£Ô4_ŽÐQz¥;ó ¥fÿܻë`£Ñï6zd£]Qœ»•»"¥óÑ#Ýq¡»GßêèrŒnÑ$³=ºûòntwúp©ëÕ•¹:î&Ð6C;†ÐV“¦ƒ!ÝÓt-é(OÙa½%ïü2—ûÄò¹–Ÿ×5î“iÅâißOuІ¡–f[ -'Pž4!oÜñ¼E;d|5oi?¾öå9¾6<¬GǘÕ]c&\«¼ÙïiGgG=¶Ç²rðٚٶˌéqÇ»ÂXéÆÚ—çpˆºí5*YŠo|Êìó:ûeTYuÿNpgž£¡ÊÞör~mÞ¹{tߌoæµáÛ¢^;ó)·=&Wü`¾q÷¬;ºâë^Öw{}\vMëwœÌ«ã7’ꎋÅüû&{4.|ד¯½[A+ó¥´²J†néßFë¾<‡Cë†ÿõ÷ð º}®u¨e¾»#sÝ„:ƒâèîD‰…\¥{ÏÒõqpßìÝäUÙòfPÞBÕm¤îËs8¤nDÃŽ3²îQ´vsþDÞ>½¨™ŒH×ü;O_u‚+¶VúyŽ†H·á®¬º÷Dõﺣð1g@¯í“Œû® øèÞÎ<‡Cð†¯ú^AË]‡¯VîÛÓ -[}½7@ð>Ÿñ(_A/»Ó2Æ[µ­=ùÎ<‡Cï†ïºç¾Ï!ü£!ô°GWÐø@ÜqgžÃ¡qÃ?í½ýþÎI7J㞺 ¶C¯y0µÖÇN5Ž\Ö—út8VF®W÷Çvæ9rý¾;¸Þ[ÿèíΓIçÂÞ‚__ãoߌôýÔ§Ùà\ÿ@¬ugžÃ!wŸÿZ#IQîÑá ]4°¸'Eׇç%êÎŒŽyŠÑ>OQ|ÿôŠüüTNIbyûÃüÒ—jÝNX—º²ùy‰«~ÞyŽÂ?pŸ{gžÃ!uÃW}/øŽ¥-ãÄ;z}qæâiÊO÷‰zN«õ gÁÌ»œØ­}ž[1Ôš–¯ +m£6}Øâ -bðgwæ9b_wúðx~Þë=™Ãî7½Íáwgž£¡9Üöq_»›ºuzzÏ]ÛÅ*ìÚ¯}«eñ[ätÏþýtö€þÞ}"òƒÍäô=gÝ™çpèÛðSÿyîžHÕ t^í”.¬Î[§Ø?HìaÉÜ'ݹãÚ™çpH~]Ìõ“Dò[Ä -§ó硫»ËÞïûçïûl>c°ï›‰+èeþéÞ;¶;ó½gCûÑêv-w}Wµ ¢ç(~*„sXA›Y)ó^ÄoÙ~ç‰Ѧy¸ïØLeè~윗ŒÒøÀ¹‚yŽ†Ò¸q®àžoGo|WVzŒÄx¶³ÿæôÇ~üÅ'|ç 2U¹a|LÀÖ³væ9Rw~;èÿwä†<¶ë—}$¶'.Ë\»§xÏ“0â1Øy‡ÖÇ ÙåTdò³šZ-ãªe\‰²×‘é¼µt®~qj0-ÓgWûˆPñëž—ù¯Ð1£fŒý®ÄÊÌRå;wÌ8óÀlmŽ¬ÜêîŸUûW¾…ˆÉîÌs8ßö_ïy’}ã£"©ô|Ç#RTק²“lÏ­'÷pŠOVŠ×¶’ÓrµCZ«kžæ ¸Õ²~­<ÎïËuë‰z^y–ºŸ‹wæ9’Óm× -ûÙ«{B‡ÙîÛ–²:.µìÎëÕ}9ÇêØx"“hC÷»ÿ®ÄÙ8Öi¯ó.˜FÆôQÞy‡éßüb' Êwý*Ö¹ŸQuc¬«?â§2‰Ù%¦¼únþq^»—¼öé®Íá~%ðW|k5êý Ó‹çÁ1hÄj×_âŠÂÛ)ã—Õô¸~ÞÂï¾<‡ÃïFlWÀo|«žtÉ+Xá¾iÿ͘¾í[?iÎ1ìÖ9ÐÇWØ7.åX]åëƒji$ÅF¢^büë½&C´Õ&#ñÕ7þã_¿aqJŠÐО®ÞkaŸç;¦ör7c+⩇§´žè©´nº¾’'µšÊšv4ê“l‰ùš3RoW,#!­ª,ý‘íóÓH56™í¨eN=¿««-ïê çÌ«´]KYÏ+ëµg˜›{Kß©æ^)k¥j½Mé•}™-³ÉaakŒCÈ–Þå4ÛjçQôÔÃiîEêÑï®Ë –tN/Ó¹.glCéÝ0¶¹`÷Ûù‰ÊÑ£ Oo“¢g"gö€æJ›I6ïúqÎå¡Ègúé)­µ EÒ«Ôl:íthžõâÄRùýDEã$ óaºùBåQ«ã„ÊsšÒÅzó:?´L úkç @CA42£ªÝʆ+pà6hG®Ò<™ârâåû¹nfO™„µ ¾´áj>…ñ)úz|“Š -iІ^µvn O$æ3=¹?ÁÕeØôØÑú¼’s|AÝÏ5ÕüÎ:E¯±¡WL/´csúŸÇÿ ³Ô ¡ÌËñýRj'S.šøåDïs‚ع6¹7Ä$œ?ªZ–Ó•+Šì‚5=¹Oi}yþ7#ëËGãKk(¿/Ùè¹ïN•Dv|òùT± DΠ¯ÅÔz5èYLïhKƒ¯B5Ňb3„­@ÇÓ¶[ÆM±¶Š ýµP±êšØ¸b¦wzÔ® MXMç]§Ùð—´&²á¼%¼œP­í¥/¢”Þ UŠ*½kT•YfT®R_RÖquœÑå$?W¥h‡Á¬z6jô~I˜ -\M—K:At9‰éæÒ|©XS¥–5—žo¦4¨ÛT;^]Ø¢饓lŽ([>.ÅF ŒµJ 7æJx9yWgb¿œ|RûÆÊèq¡eåVqR /MJ«™>aj½€þŸ¨ÂW)1ÏøA» -)zÃH»j•Ðܬâe'qy¨¬ã¤Ð.¶´¬»tŒ³¼¿tRBz"ÝšºöØBʺŒÉc.æIUòXŠuE]…•þ©\É…ªêšØW(¿,ÄÍN_§*À^* šÁÝ×r›8Tî|áâXG\Óëbª½µ\†ù"¼Œ ›ëÅýT*ib£êj×*cÂfH¡‰¾Œâ2[+;Ï?¶I&­¸—Ó§q™3Ö›ŒÁjzû÷ºôù?Hó5~ÿyrç?•×mLë¸Q•-LåÖ…˜¯dô®Ì´SLïÎ,Â:m˜Ð8FûM¬SKë ê:n ãkײ«Fì%¥® ª«– }ëC&6ºF[Ö3þ†ðÒõbud^;6mÁž.miª±ô¢–b.xsp™šÎ¶%ùÌŠÄ%mK:´Î°Œ¸à…â’U˜›e 3 ¼’Úõd¿ãCa©.vÁܺ´MçÚ2Ë‹.²±, \`Ë/xÁNóJóÂÁ臧׫Ç}69”@ÒJíUúÂ¥µa$­„'Kíx–—uͱgº(“·õP-˲¹g«“Ó¦Fs-Üâ‚c”ISË6'ð¼RWd‹JËBQé2UdµÄXl -s¶Ò-c‹“ØK±úMú#ïiÏG¥v¢ç£®ÒR¹eÅ«Þ’­çpŽ­uª¥¬¾ddç®R×RÚ"»®Û-ë.«í™5Ã5CBóLUËæ¯R>ÄÌ)O…R•Ÿ12¢S{Aj§£5v|ç¸J!€C½62ºÄB%wç¹;¹Ž”Ø ¯ Ã18kUÔôöxÙ„(ò¶‚UÞ,/gËJlÒ} „†Í©«ÈÐøúà1d>ˆêe¤­"½-Ü’Ó›è%§Qi9Æ:¾3o>þ ~;êÏ -endstream -endobj -614 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 613 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -615 0 obj -<< -/Filter /FlateDecode -/Length 4953 ->> -stream -xœÝ]K$9¾×¯¨3ÒæúýÐÁ"V Fp@zzf¡^¤…Ÿ/œ~¥3òQÕ=C-Óª©Ì?"íÏŸ#œYNyøûJâ¿`äBôÎ\Ÿ¼üt“·ÿRŠÅIRÿo¯R?Y]°×I ýä£UÞ]_.øo8í“¿\þò‹Ë?—yäUM1:%íõ_Ieôäƒ÷Â2ª•šQY;y¯¬cTNN¸Ä9Ì°Þ(³Vy˜ab°œ -f¥£ -0C/õZa†òÒp*˜!£õŒJ -Ø!•¬†c´btÒNî8’“ QEVç¡“^q:­'ç´’ÓÅÉ9g§3°ÅF!9…-V#8l1:xNç`‹vZ°:Ø¢‚לÎÃ…Ö‰Œ.À +°:Ø"œœ.êÉÆ` «‹Ð‰àÃZ§„lÐZp:)' ljV‡…NržÑ)Øℬ¶X-§Ó°ÅàÓ'¬ö!0£UÑèÖBKNga‹RÞ°:Ø‚Îç@9Ø"¼ã(#h"^â8@y?™¨ G*èÉ8P!NÆ{ÍÑ€Šv2.zŽ´Ð)É°ÅÇ1–°Å`ä²:Ø¢£á¸@+Ø‚h [”Ñh°­Z8.Р[#¢ä¸@ƒoŽã Â…™‚ã ÆÕÁŽ 4(PŠhp®wr\ Aºîžã ÖÕÖ)Ž 4hW›à8.Ðà]m¤à¸À€wµFNV[`Ž ÀŽ“–As\`À»Z -Ïqïb)Ž Ыpè8.V&‚à¸À€wU†ãÞUS«óÆ æ¸À€w•Å%p37xWY¡8.0à]e”ã¸@™À?‚õÀ» Ë»°Ëqæ{è”清M -n Ǽ+#¦?fLƒ­&¢ã¸À‚we@÷±º8IgÆ-Zy’ÎEŽ ,x•æ¸À‚wAŸãpü„Uq\`Á»Rc¦ætà]©¢à¸À‚wá7XŽ ‡IÂà¸óÛ$ Ëêâ$bXð.hIq\€YxLÂêü IŽ @Æ“€Êqïbz8ð®°Zs\àÀ»ÂÀíau°EÅqüè0l™1 gu -LÁéÀ»B¢‡9xW9.pà]š69.pž*p\'xÁŠã̽w¸( ƒËÇêÜ„sËqXP«ƒ-ÆŽ àXOQÇÀqW°E+ÅqW°.Çð¦(á}³ál\ØøÀÀxŠ€É¦á¸À[‡x ­Ê霚‚]³ºôxo&LÅ’ãÄRSÀ ç¸?‘»ÄqÌ€NŽ |„-*`*š‚‚gÎé$l“Íq<]è¤ä¸ (Øoã‚ ¢L'8.À XMÍ錠83p\Œ›ÈUä¸Ã‘&˜’ÕÁÐÇè -¶-Ç~l"Ìau°E[ÃqAŠÂÍÀqA°E Åq°Iñ¦ç¸ -Øã‚(ÜDn2Ç qè„ภʀxSŽ 0.)Þ D-o"ÄcÆmÔŽâMÏqA4°ÅÂEau°NSdÆ-8 ñ&؉Ó9Ø¢¢á¸ :Gñfà¸nâMxôì’@ xÓ³Ëà]ÁLœ¼ ÏÔq\€Ä›ðl¹Ø^€x­‡Æ+Eœ]1 ^‹¹•]2à^ÌVž]3 _Š‘ØE˜‰È28vÕÓ*”R°Ëü ̬8a^}ê—¨àà1gO1, ¼ûs´dz#Ù¼>ö¯._?aò½þðï˧_¤•²ò'~mÖð1TDƒ¯¦TžA„ g[ÑDÂÀ{ò>6#%h1/º‘‘˜Š1ÝÆk=K“a1Ù,á̤+3‹z*Hô•v¬­7çy{[áGb&†÷Ö׋ÑÈyøÐ:Mÿ®ëƒ#á3õ:€Î8¢8Öس™ÞÐè5 õîæžWݵÃ&èK‡1@ßýxùúÛïÿð»?Þz÷éò×_Ò -&A«5ôá(Ì3>²|l³[Œ(ù᛿]ß}××C‹\N bŒËz¨h8ÎBuERq$KžŸ«KiP•¦ïç¬wÙ$Ødc;N:=›c|KK²dºË¦ÇüíøË |5ͧf­ß¦ºüœ7Ä9•m;›d—.Ì6Hµjš® âØ4©µM.ª˜œ‹¢bíFõ²äÓù’6š -¶ô¢˜/SèeŒÝPmrÝ¥•f—­ÙKúj¿ºm£›Ë5-ÉŽ§½&£Ã¡ÍJ¶»®°´Uæk£/ú¬Ú§À¢kËtî²M!_«Ïúbk×d‡¹ŸlîŸÊ)OgSj£†±k‚|é›çfCj¯_¹ÛNzl§ÒGڶ볱O>w;•!{ÔoeH¦úÕœfaoÆúÖеٮEžÔæ»mgW1[Ú2æòJ¿õ7ó5—ñ+t—§ôuSr¿/áñ/íQ¥ÎŽB û.ÍBºN™pð”N³-a"~S®U˜È@WÝÌukž»|wøž;a³Bxà"ÒòÔPa™ÕäC®'’ì›ý•Žp¾qâ™í§£Üˆxh5×9Öž£ÜÖÁ;AŸq¹Í º|%ùŠRK”™n“ëôF ™¹¬ÒJC3©tõõ=æ{rÞ™8—²Ã«GPN+½ÝÑò´n¦C{OËê7p"¹7ky*ù`àpmcÓÆÜ »u€@ª¹é=³CdÞq¬ûHþ}¿ôÁ -±Ç º˜®'е+I÷¿"­4•¦‘½Ezp½3Ù4šHî>éËóoýspŸFD»Ã}â©ŸÎwFPeÆmȪËßV-Gó<õwù> éò÷Èe¤“­ ŸÃˆo®f–ûN®—×R¯m¼V¹Ì§ÕŒ@:¦z‹®¶IXæß³­Öaº|¥Í>-iD*ì(¨ 82ˆÍPÏeyœ6«µÙÃimÕÚõ\¿|sžµ´·{¬UÉgWµ‹ê*ke—/±UrñÆ2SŒ6x9T¸ÇXFÈ›ëdžABe,ºË¾‡˳F?z% -•6ÓÇÏ笎ŽsÐ!Ÿ†4òw–«,Èòù¿c—öý—öeƒ×òJ~(·²ïÊ”kˆ'Qv.󠬙mÄÊƹÉäyÒŠ%'/Æ ­ü¡õ&¡Iw(+½–½RçÏ~Žü8 ¡®T-çE}j»ìyyh™m’)g˜?SŒä–²bÓ™ºR›vs·ñ£~©VÞŽÔsy©n—9„H9ðJщ5gÜíõ ½md¬c»¼ÏKû’edÉyÔ±#)__âu×ëáXå¢TSAÑÓ6ûè=—çáÐ{àW+T«Õº§¡g»¨eïz´|©9Ù´å¼Ï>3ãÓVOcc•ð¦HnËÞ’ç0ªÛ@lóÏ#ö\ž‡C쾿ù0>àkcÞ[¢·/àÝÜ2*éÍ>×:q{ 2Ï£!×íû¨ÿo+:oÆÙ_Ý­‡£oÍêö¸ÿdž‡Có¾ß»òG<nv®ˆÙýêPéáù«]ã=¡/ÍŸEö~m[ãÈÛÊ­£u‹[;Ÿ”~.dú%È>Ïåy84îû±[w€{Î¥žyM<]ŸgØéÝÝH­øÈO®î,,Ö»B¶Bñ2>uH;±²–Ò½j^Ú@æþêÉ<‡Ìõgí7AÕMkoá>š÷ãÖwk£ôë¹ôÌãjOæy4ÔúƒõÔ×ú†7û[op¯ê =¡t¯÷ÁÕ_Gçj½x¡wÜ»?™çáz°îzò¾Qõ †rDRZR3ÊènâŠ_JD±|vxˆóG×zTOm,î°ùV4ŽOôþnmËѶ]¾;î–úæsšh'ø&=àv2ÏáöÀO™çÑÖO_áSžA×-ë–‹û[ýš€_ÚTÑñaFAú¼âîi}þ”ø–xúy¸Î”£äèYÑЭ]žˆNæy8ôó7îµ.J}'ný¬ó~<÷š‘¼Òæ+š!êèi”“y¥þåÙ¨£¿K9D#²Â(*ò¢>‹¹ðóümÕºëéËúUð£§5‚¿ýY£“y®wOúpahÕ±š¢ \›¢ˆÁµ™…V²)bP¤{Ÿwb`t”§DE¯ŸæÝtÎ[ó™–¿¯«þÊ¢Oû4×AC-×e[ -'Pž0#/Eîúê¦>v3.ˆUÞ¬Ï\Ç`Çx^~-TËÉ¿L¶=gL§ˆ—g¬xÇÓ‘'ó<¢î‡ûz–Ö7~Îìó:û7PeoOæy8T<µø?æ›{÷Íø&¨8Þ±êu2ÏÃ!åàîk·®øÙ|ã'´Çç¢Î¬Ër¿,Ἲþ^R¸º5…òÔMÙÐéžuá›îœ”õÚ;žm¥e›8›éÑð*Å öÃøφ[/8³8wÃ>i‰bL÷$ÖHŒ·Í‹i¬Ì»Az‹^µô& Ú"1v›KV€&9€cÙ¼`½ áU»(.÷7@‘ô® -¨‹žvNÝÚÝ@}œg³ô[w—g¬Ó—»Ÿ´YZù=¥*;"$NÊ+S‰Kļq×ÇÖliw„¼9XÚ„+Îò´Y•m3±|\7üJ÷Äs}´ãå'øt”ÕòËfa=‹ ébžQÓ5¬ºñ§‹K;Ë´ ¼pðŠ½&æêÌD·WÚu8mdé6™Z^7r¦—ëüTSÑÿ Ú‚Ÿ¶_NÝÚŸäô?¥ÿñ"£™¨ý‰)æwcˆ$ªœ4ñË…Þb±1“:)&a>µ,#KÂ*{ãú)Ž)µD>ŒµpÚÚa>T¶\ å·%íY‹`A•3sÅŒ9\‹éêÅ$³˜Þ/&Ï\5…6µªÂV éÓ¶[ÆC±Ô¢ÚµPtÕ5±2ÅÚ‘¯ÆÚç´mG1½o:ÙuI«|×{Âç ªÕ£ô…•Ò[-JQ¥u•¨2Ý«Ô–”µCLí'Fô|a„_‰R´AgV}×k´7ø, H¦¦‹%#z¾°éri¶ö”¯©BËKË7SÔu¨ ÏÈRe©§fucDèr¸Ó^©µ¯Eèp£VÂç‹5u$6ñËÅz&µm\!l^/Ô]¹UDÃK“Rü8ahí¬LÿÏTa«”˜'HS!E»£·³V Í*^6R/w•u ×Òø–¶k.é}Ålß^¤˜ôDº5um±…´k²Nîc1Hª’ÇR,+ꪘ(¬´OåÊ^(Š¨®‰m…òËBÜì´u¨ZQ+s²ƒ»­å6±«ÜùÒ‹}íqI[ÝW{k¹æ‹ð9MlfôC©¤‰•¨³]«¬6Ã@ -MÜhô%‰Ëh­ìD¯ï*‡mq3îóåï—4Í!¬P´c¥Ó’ÞÜÒ{]òú¤ùŸ\Ìõ÷e«ðyW¢²…ªÜº÷3½ç+œÓ{¿Š°›N¨LGûM,CKk ê<®TÇצeØKJYTVm/´­ ;±ªÐURw-cw„ÏC+VGæµ}Ó&ìùT—‹£8i“ùbNúËÁih:ݦäkW$Î(i›Ò¡5ªËˆ“¾PœvÆfÙÂLd_Imz²ßô]¡©®î¤s#èT7iÓl_t‘¥«è’ÀÖýI_°‘}¥qacôÝÃëÕýžMve´R{•¾ôÒza$­„ÇKu:uËÓ:çè´ye¼­…jYº{º:9mh4×Â,NzŒvÒв嶯ÔÙ¢Ò2QTº€LY-Ñ›\ÎVš%]q`[ÉWŸ¢IÿÞ·´í{¥6¢í{Å­ÒR¹eÆ«Þ’®íáìÛÕ‰–²ú’‘¹>w•š–RÙºn³¬»Ì¶×î2L3Ä5ÏT´lv•ò.þ»gì°ïàR•¿´2"C{¹&j#½V:½/UçÀ!†^yåMè–Jnγ¿v²^!)k'}MèŽÉh-¼¤7ßò&¼@]Qå­¸|¶(ØKº­˜¥¡Fsë·=¤W¦õÌ´@TO=…Šô¦SMNo ´yÁ¿ýa~äc±|’þþ ‹åXÜ -endstream -endobj -616 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 615 0 R -/Resources 4 0 R -/Annots [ 617 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -617 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 683.673057 101.433071 664.923057 ] -/BS << -/W 0 ->> -/Dest (cb95-1) ->> -endobj -618 0 obj -<< -/Filter /FlateDecode -/Length 4799 ->> -stream -xœí]Mä8r½ç¯Èó£å÷`ìÁÀö^xáÆú`øЕ=µ £ÆÀ¬þû~A‰dˆ -)UÙÕ39 w¡ºR$ >>C™L}UøùNã¿äô”RŽÁ]o?^~º¨)z…¥Äꢨÿˆßè¯Z¹8yCò×I`ã³71\ß.øo¸äÅß.ÿþ›Ë¯ëè«™rFûë_ •³SL1*/¨2TJ[Aåý£ñAP=¡‹9K*˜á£3n«Š0Ãåä%ÌpÆ$A•`†uQÛ­*à µ“T0Cg•V°C¥D QÎY#è´Ÿœ$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ÂïdA—` ЕDlQAiI—íäsrNÔeèTŠi«3ÊO>Y«$Ö“6­¨ÃŒÂ …(è l ÊgQ[¼UFÒYØâð#éÀÞÆ”„Ùjhv[eµ¤ó°Å˜èDlÁàK `lQ1H€4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ0sEl±ÙI\` lA‰ ¬…-ÆY‰ ,ØÖ-XЭSYK\`Á·Né qáÂL%qãڜĔ (e‰ ,8ׂ;%.° ]L÷(qëZŒÄ´k] -Xð®uZI\àÀ»Ö¢¦¨ƒ-°Ä`ÇÉêd%.pà]«U”¸Àw1ŒÄÕ 8 +“III\àÀ»&)'q%@ÔÅ sÐJ\àÀ»Æ£ ÒÊ Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\€õ:c%.@c“AX"qïêŒåO˜Ó`«I§$.ðà]0|¢.O:Âã¼…—'B–¸Àƒw±PY‰ `?¯Jº`¦A×¢.A§£Ä1º K±–¸{©)aÒK\‚Ÿ(\’¸f@§Ä1ÃL‰ °MÉ 2—t¶ È–¸‘.tZK\ lA4(qA2 »Ì $.À -½$\-霢}f’¸ ¹0Q¨(q¦ vš`JQ[@`|h³í%.Àæ›MlsDl±ÞI\’¡íf’¸ %Øb”‘¸Ø¤ýf”¸ +Ø‚ˆO€ÂDa²Ä qè”’¸ ë„ý¦q`^Ò~3I\­Â~[R‡Œˆ¹©A‹KÒŒÄòGÿ®Û+Ëa8™ŽÚØÊ$EaØ|0QiÝaSŒý¡5³›üî:wMŽˆòµ½Ð²­ÝUü¾ÝW’­ï®óñ¶byÅçú¾²‘w -ìà÷ûfÚ«oî‹ ÓLÙÖ“u>ØdÛ¦K¦! átXŒÈ…ZO´Ã {g•4Xa3è0 ŒÍ†VSP$öAÊ­ -ÿý§ ˆu±bÎÙ·Kf>â;…Òøë§/¿ý§ï¿ÿ‡ü7àïÓëå?þN)÷Ãï¾ÓøëÓü×ùåo½~]_û8ÿË)†2·¡¬Þ)_õ~Ýf«ŸÙëZÇ e—6 Ú¥ì÷h_­WtwúØìÐB;õšµW_û¡ÿgî¥ÕïþóúéûËï?­AVvJ3ÍPò;8Zgqy®Ê×àò[À1çC8Öa¶kHø=èä­L롪Wó_ìª  $Nøÿ0èo;ð¥'MŒ´ñÅî‚b¶Cüž¬ólÆnÿ›#x-ïAv-Óîc–¿/K[„à"ç„=̘çBøzm£RJQ;§ -÷Ð{®ÎÓ¡×óïÝùÿqÿÙúžYsÔ‡1$|Òtaè“hïý\z®ÎÓ¡ñNp: «ýMò(×Ô×#~¹Qf÷«m^¸ìÖË“ì×ñ;Èe!)¶ûY%Ci¦cäž«ótÈ=Žc¿.Æú×å6(Ôï_oëE«#¬/›2w8×ðX”â²¹·ÿ:YçÙkÎůÕcõïèÑá+Ý4„ÎUæ¶F ÈAnªc$õŠ×i‰­ÿÂfDX#×­×ÑNX—†¶Ý CäèçM„´ƒÔwžŽNÖy:¤Þ‰U? ±cõ¢eœø¯s”Ú—¹¾õ[Nk÷Ú¡òæ„]+NÖˆ±Ž;àÀV–¯ ;}£>}ÛâbˆgOÖy:ÄÞ‰gï$@Ÿ/ÎûúHæiw‹w£–4?㞬óth>Žq¿v7UF£Æ›¶7C»œ+â\aZV3&à׿Un×´óÊóß?Æ5šx¤óíg²Œ>û@žõdgCŸ½§þ-òÜ{2UèÜì”nìž;Ò¾]îaÉ<&=¹ã:Yçéüu9×_%’?"WÀ8½­zng‡~àõGŸ½Ì¨ul6Œ³¨ïøvÐËâÓ³ÏkOÖy:ôÇ´›Ñv- =|ev:5÷ÙP1rÔK癇U´™6ß‹ø{¶›èç2Ü6SºÅp÷^;(}à]'ë<JＯ`q5ÃS=ýÂ2æR$U´dº9®ü%Ÿ2üšŸ8£*7í’swz²Î³!Õǵÿÿä`'`í5[ýÚGbâºÍ½gŠMn¸Ý;h} {²ÎÓ¡õ8vm#÷ºFIÙƒ³§›«¼j]w²ìmd†h`D­yeˆf1hCîMm3B5®{]×ß cAMÉýîäêÌ¢¶lxߌ3̶bsdýä‘Ððü¬Ùvü@Nöd§Cðqü:òטQZe“CRõüÀ#Rdü9>3\c ÈøŠ£pƒ(͸ÙJAËf‡´w¯e˜àVËüÚxœ?—ÖõºƒäôŸ«ótH>Žq­°ŸÝ<zÊ‘ž±íqamkàRËž¼nžË9v=܈QEƧߟÍÆrO»­»bÓþ,ïÉ:φi'Ë;¬Ç´¦¿F†ugTF½3Ö-ñs›ÄìSŽ÷?ç½gÉ›¹3ÚÂíÎÛ•ÀQñÑj4Æ3¦[™/ëx¨`u𗸢ð~ÊøeïÕtü„†Ÿïá÷\§ÃïÜ®€ß•ŸGžf±k{§KÞÁ -MÇÅœˆmË4©Ý\ŠcØms`̯,í”ü«˜k«|ûà$¡ØH´Kö©BŸΤM*#ñû?üë?ÿé,OIÚÓµg-ìõK÷ˆižfmC=yy.뉅^jçë<íe…æ¶æ]ü,›Gc¹æ¬4Úë¾zik•í¡Qê÷.¶ùy¤:›,õZ{³çϸ:hîêç,ª´C/#³‚•)×vùõ¬§ËuéÅâÕÚE™`¾Þ³ÈÙ«uyG÷½-½ -Kwªž>wnéܱ¥v]OžO³‹§¿lÛ –>”O?mÛQK¿æ>—~Ù‘Òžª÷eåò·™ù@° Ê™…¶g”÷6רí¾&?WÑ;çæ1P»¼®}%{\ìeKÿÓ‚äCfûQ0! wÎO.åCÍ„-ãã™Mš•[Æ°æe¿Óy!kß4 ifsÅC¬mnï_gméómßWzy~Uʪ¹ŸÊ®}0ŽC³)°¾U¿ëî÷Õl×ë±àóplf¦C?™ŸÆþ2Øì«yí:ÏZž3WõW|ç{ˆ¾³lœ^»ÅŎ=ìÍk»×n¿ýçFÿù§`ì˸j|[›ãñÇ~ymÿÌÁk©<´ùh«VS°åßtÀÙ©hðd‡>vÿa oöb“¼5ÈIiŠŽÛ@½<д,Иa÷†9L*ÓáUà +­óD9 q}|½Û¢±yòQhÑ¿Þ« \ÓYo!ˆöÜ«íÔ=ÆLªÍßÜÛ‚_{×C¦¢ê|°;ú¹Þq·÷ΖóŒèl—"À‚U`h÷‚½m0E§cf:‡n¸i™Ù{¤ .F'h*§ËéFǤw®Î“l÷eå¸O}þu=WËÛ@ÇY?‡k»7©•ÑŽóxÎ.á¹a5.‹; xžó¼.xÞ%å“ Ù§¨‡a¹C5ãÒ[sî­z'ë<–éT»},oÞ|>¤› ϹåmÅnùóµ¨óìm#Ÿ‡2_úÛ“Ëvl qÛ[˜¿,3+ûòs¯r÷f¥œcGÌÉ7„ž¬ó$(c'_h}³÷¼[é(«Ëß™4¾;iï?î}>×»‰úë‡ß/sø)„Õ?.g¶EQõt^=º–Ùp Ÿw‹Üc]*¶ÇX]õppØz5A“ôe 6!0GÄé|ýµÄü0ÇH…YÂòñÈecL3½¥K—ˆÍÔõG/Í–˜ðó³Â«¬EËæ´l0ó,7-Æâ®À6öº')ʇèÒ²IeÉÖ~M -0{VI¼0béÃfº„rN§.g5«&mèË®ÁMôl)æ+ š”uº›.ŽÖךå+0~j¥–oÅPtP6’Z†•_,å*þãESúZÍL1Ÿ`¯ŠaµÅoú'ˆëéÚ;b./UkËé–ý­²7Ny,‰5S-/{Š˜Òù¥ñµ7Tß×jt²¤Sµ-g+Î7„¨ôVL½W“^Äô-i²ËUh¦Ðæ² {ƒŽ—í öŠwÅÚ*&ô[¡b·ëbãª=šN n® ]ØLç®ÓløkYÙp oÜÖŽÒ7QJgÏצªwj2ËŒÊMêkÉ6 ®“ º]áwª6í0˜MÏFNð…©ɵr¹–D·‹XniÍ·‘Š­TêUsõ|7¥CݦæxAvcQ"}­ ›#ÊÖ—k1eíÛX«Äpc6ÂÛÅ»6»øíâ£PÚw®ÀV-o…–µÛÄIu¼tiLÕU˜šUFçŸÒÿ3Uø&%æ)/´k¢3ŒûU¿ ÍÍ&^;‰ËCc'y@»ØË2wéf¹¿tRBy"ÝVºyl%e.cò˜«yDR<ÖbÝP×ÄDaÕ?+¹PU#p».ö Êo+q·Ó·©êU»YÐ î¾µÛÅ¡qçÇ6⚤nö¶væ«ðV67Š ú©Þ¤‹j«]¿vÃ@ -]Üiô­ˆëlmìTNkY^öI&­¸·Ë_.e™3ÖÊ«éûxÔ¥¯ÿ‹2ßã÷¿.îú/õ@ßy7ª±…iܺ󕌾'Ó·óTa›6Lh£ý.Ö©—õ m7†ñµëÕU'öZR7ÕMË…¾û‰Mƒ®Ñ–yÆoƒ[ óµcÓìùÒÖÎQ&œŽ‚®Vá‚w—©ël_’¯¬I\QѾ¤Cë «ˆ Þ(.Ù s·le&’ߤ¹žìw|((›Ë/XA—¶ë\_fyÓUVzÁŠ ¶ü‚7ì4¿i^Ù#ýðôúêq_Lu´Q{“¾qiëIáÉR[.Ãú²­9¶$‘ëäíjmY6÷l rúÔè¡…[]pŒ2iêÕ–žßÔUÙê¦u¡ht™ª²Öb¬6…¥ZuKéq½[LÑ¥ážö|Tš=•°)KíÖ¯EK¶@žÃ9öÞ©^²Å’I^»I]/i«l{o·¾w]m¯¬®zdªz5¿)ùÿ=2wæ:CB*UJfD§~>j§£5¶|«¡J!€C}1Mt‰¥JÞ]ç8w²ÍÔÜ ¿†crÖª¨éû)eÞ È÷¬©úÝ•rµ¬Ä.½ÏBj¨ÓœÚd†Ê”•|fIµËH[Eú>BKAo¢¯QŠJË4JZ¥OÊÏÿÖ‡8 -endstream -endobj -619 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 618 0 R -/Resources 4 0 R -/Annots [ 620 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -620 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 236.373057 101.433071 217.623057 ] -/BS << -/W 0 ->> -/Dest (cb98-1) ->> -endobj -621 0 obj -<< -/Filter /FlateDecode -/Length 4672 ->> -stream -xœí]K丑¾ç¯È³¡ù¾Ã^Øc¬±ƒm؇źs¦l,j Œ}ðßßJñ!*ôʬ®QîBu¥"Rˆüø1bŠê*ñó”!zG×Û—Ÿ.Rx+ño(19Ôßá×Û«’ä…UÑ{ý1 Œ>ZíÝõõ‚ÿºÃ¶øëåÏ¿¸ümj£®ZÄ贲׿ÿTd„ÞK˨"TRFe­ð^[Ǩœ¸Å9Ü°ž4ÍUnP –SÁ Ò:0ª7 yeæª7´WÄ©à†ŠÖ3*%á‡ÒR²:8"‰ŒftÊ -‡JwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¾XE$9|!<§sðÅ8#Y|ÑÁNçá‹FíDFà ÐX|‘N*N°1±º >ÌuZZaƒ1’Ó)%,°iXzÉyF§á‹“6²:øbÔœÎÀ§OXãC`z«N½ÛH£8…/Z{buðÏ1€vðEzÇQzH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHøbÉqL`|!ô\V_L$Ž Œ†/(Àq1ðE“á¸À€m há¸À€nIFÅqß’TŽãÂ…›’ãÆ5ÁÇ” (EŽ 8×€;9.0 ]twÏqëë4Ç´k(8Ž x×’x×X²:øs\vFÃqw’žãï¢iŽ Ъ8t+B‡ 9. ð®’8. ð®öX胆ãïj‹[àFnð®¶Rs\@à]MÚq\ ðd£ð.(Äòá|Ár\€ñ:m8.ÀÉ„FXÂq愈áéÓ`+¡BtXð® -h>V…ò¨q¦ß¢–…r.r\`Á»¨ Ǽ ú €ãFTÍqï*ƒ‘šÓw•Ž’ã ÞEÜ`9.„BÀqÆ7¡$ Ëê¢1Ž ,x´¤9.À(,d“°:/$ÅqÈXHÄ¡8ð.†÷ÈqïJk Ǽ+ a«ƒ/&hŽ ×@‡nËôi«Bj0§ïJ…ætà])}ä¸ÀwÓ°ÉqóI ÇpP -Ö¸@"zï8.pQB‡Õ9cËq—XP«ƒ/ä‰ãÖ"š8.ð¾­9.ð¾ dจ -Ñ7;=€/€ ;? ø‚H‘ã0™‘ˆãoæc¨UNç´tÍêtÊs\à= ÅŠãÌ¥D@§ç¸/R¸ÄqÜ€NÇ>ÂtŽ 0‰ ™s:_ds\€H:¥8.¾ ä¸ è€Y¦“`„Æ\UÍéH¦yfภ)Tä¸Ý3M0%«ƒ/  Ž Ð>i²m9.Àä“MLsX|1–8.A§éfภø¢¥æ¸ØLóMÏqA”ðÇQ:‘ÂdŽ @ãÐIÉqATóMM _¦ùfภ‰ù&¦xL¿Æ¥ù¦ç¸ |±QX|AЙ~ NÂ|ìÄé|Ñ‘8.ˆÎ¥ùfà¸a曈èÙ”@HóMϦ À»6‚™8x‘©ã¸˜o"²åæöÄk=â3^ÒŒ3° êµ[Ù”÷b´òlÎ@‚|Ó‰MÀMÌ,ƒc³V¡T’MHð/â1šqÂ=ûÔ¦¨àà1fŸn2¢[¢û2ç¨ÅÌB±{~ìï¹üò3ßë_þqyùÅ)Ë? -ÂPš_¡èˆæ¶ˆ¾nß84´ƒáà#6É «“ -´8&Ý’“þÒ¿ëüÃÄs8þÝpyS.1Šß/ŒÇÔÊÓ×!7Øx|Ðä NÁhëPÆDØ C\)iRø7Ÿ.êèÅ=Z÷1^b2LÚ^?ýxùåï¿ýö?~÷߀ѧ—ËÿüJJúá×ߨô7ÜÿÚhMŸM§“÷¿ó$P3üZו±ã_Ð#ÝF]W&ÙÙËX>Ûrëùó¶·©ƒ£žhÔAHŸ«½Nþ¢n”©÷‡Þ›î³mâôž‹OªÞG¹ï[õ«ÜCöWW;%ý¿×Oß^~ûi -¶!À–&Ÿ«Hº |î3yŸ_–˜Î®À²T¡íª®…ÕDܼʩµýßis6äêõu»ï,t¿g¿'º3Z7{ßšÍñYÿN›Ó¡y=îÅ£7:ĬôŒ65”[8õÛÅ«=bÞ?³ìËÜ·Ê‘ÇÎ[zë·61iZX'Õð@mûlN‡Æõ8v†¬\à ²RË<3ŸÖa»uWgj9Fîx²ð%›|”l†l†þe¼4HÛ‘YÊ=5.- óxu§Í鹯¾ñ¨ý&¨:”{|‹ØðlÑZÓäFÓ"”¶¨Ýis6Ôš|ê³±ááxë TQ§ëùo ­}Ä’¹´»Ï’Åz0úà®_zç,_¼€Ð6‡ºs6µÓætÝÈ»î|nT"ƒn„ì‘4d‡ôeéQâŒ_r‹eDÓX¾?FF‡ÐrÕ¾úqÄç£h,|ßß×—¦.{ß¾oìxTjjÌIÑ -Ä”¥¬£vŸÍéP»§J-íHhÔzùr]uú[E»õV¦¾ÕVø/Ç……“ò¹¾4Û?°wó'K¹X6hŸ==w(×Í.û^µõ„Ô„ørŸÍé·ñœ¾›ßÎú:Ó²“§ƒýèÚ/Gižêv>¼”-ï{$»N+Hú°5ÁÔÄ—>eÈ»­ŒÓN›³!˜6ò§OÄ”{Ðu$o9y¾ÕæüÔ§‚Žïï(~Ÿxz:œÇŽ|›xúÖÝçÊú^RÇô5¹Ë½3¢6§Cß¾xsëY;q³”>vâ²èGâ±&Vüx<÷LO^@i‰µµe§ÍéPº_îu´O)»ÙFìaiVäeYˆ9‰óü5kÝ´tÇem|kµ…ãkvÚœ®uwÆp¡«Õ…¾:Ì&po:Í\YR&;ÍtÒ}¹sºd“gYoRt~Ïû¥±£jß^k˜'§«-ûù~4a(çu£/™’M¸#o˜ñ¼Å}°ø²íÓæøÚis6|ÙëÑ6nÚêP»;.«¼Yï¹Îþx¸ÓœŸ}»˜f¼2û<çÿªìñ™àN›Ó¡jcÕâÏÌ;‡[÷ÍøfìQñ°} ëµÓætHÙxúÚä¿Zl¼°B»_µ'/Ë}³„‹êÚgIeÆÕäòª«|Û‘>ôä$çkXÛêäñÕØ;mΆV·½ãþ«!áèÚ‚=™¹«{†üD_¯èŸ™M2i Hm2[{¿oµÓætHÝȆ=œg”¯&yÃÌÛÇËš- ò€;mN‡È}ßdyj5ò‘' -ïÙ˜¯~ÿË øìÞN›Ó!x#V}£õôO¿(à¿CµèÓ®˜ñ,çÑë›'­{¿°Óælèõ±ëþ–ß›ôšw]µ¸€ÆòŽ;mN‡Æøta5ôì[¶rM™5tHø9צö«¸ÓLfà²n½ÔÇáØä>ðì6§Cî¾'¸çô½Ú­'ãÖ…½¿>o¯æúzêËlqî¹Ö6§Cî¾øõè%&(º1hhòž)»>Y?ÏqP·ftÈȧí‹œ}‡ë°}»*§Ï$5¶ýzþ3¼õ"<ðœ{§ÍÙ6bÕÏLì˜k±yUÙ{¿±Gïðkí{Y½ÍZõÈ{ºÒ=}ÝâbˆgwÚœ±Ï­><_œ÷|$sÚÙâƒïĸ;mN‡æõ÷ÙÙÔê{UÚ_xj;…×^C™Ÿ¹·¿YžžÙ¾¯=H¯ˆüj=y}äYwÚœ}ïûìSðÜ‘LÕ -:Wßб¶Šý«äx$ÇÞ(´ÓælHŽÏå\?$’ß"WÐpúä­7&ï롵ïël\c°ï›‰ èmâÓ½OlwÚœ½ÇÞ ôÔwU3*zŽjW…´–ѦÎyñ[¾\qÃú4öú¾é© º[çµ€ÒÖì´9J7ÖùvôÆwe¹7IäWÿ¬oÀøÀOb£JÃk¶Þ-°ÓætHÝùí ?9XÞ€ºûY¹Ÿ§ÞÛUÐúݸ'‡·hX›ö#M»jÄf‹ÑÍ"ˆœo’·Õ¸¦ŸrnÄšC€!Òf¼&\pRôiÿš±ßþñ¿þð§?Vˆ ¯øážFOU<,d¿Ýçãüò¥dÙ‹AIбúÝ}H¶C5'ˆ¥æ­Í•þ‹ðÓbû—; ‰ÑTÖ݇sŸ‡ó„ ­ñz)‰šìÓT×OÎoÇk7þ ç°Í=Ø|³füéâ†}˜Ô°ŸtA(6ë½:iݿ״÷S†Tn65T´º.X[ÿTJ»Ë´bÚë¾%rs0–ÿihð/*’HõŸÈâ¾C©ÄAçƒ*~½¤j‰xp§8 Dzœ‹T.Xd7p®±/i¬ãÇXNnlþ¨m¾›do³YÚ9ˆd.d†½sîf„°tj.Nw/…Åi—ß ÌxäŠ+ÖeŸ!¬'¤¶l=a5Ü+#¡ es¹*Ö”ýQiW¸Ru® -‹ëmÕ©¦ùsYí›æ\Þ.¸¬é¥¯¬4í-šO•kWË"3S±Hm.Y„J;1¢Û…~#ó© YôM«¥ÚîÂD¥\ÌåÑí–ÏfKKùR*TÓ˜k¾ºR¡nB©xF6ÔÑ(õiÛò¦H“?NÅ.µ@nkÜè™ðv±Tzb¿^¬gJÛÊÒfèµBÓœ·ˆƒ¬x©Òôõ³û'tÍ,Kû[¥ÿïTa‹41ÏðAQTÚ£®Õ‹¤¾YÄÓJjå®°q5 ÈײMu)ï fÛúRA2åé–Ò¥Æ&ҦʹٽDR…<¦bUPWĉÂrý®l…2;ËU±-P~ˆ«Ÿ¶tU€=_Ì©·Š]áÎ×VìK‹«´á`ñ·œ·Á|Þ†zq‚~È©b-ËhW/Ö«c …*®4ú:ˆso-ì4¼~üX;7âÞ.½ Ãœ6VG|pF¥ýsÛ¨K]ÿ‰2ßâ÷ÿ.týϼaÛ}ײ°….Ü:·#YÚm=ì§Ý׳°t›F¨©¡ý*V¡–µ e׺ákªæ²{.© -ƒª¢m…¶Öa#ÖºZ™¦fìŠðÖÕb džm›:`ßM¾9\Ƥ­þ²W8ho‡¡êL’¯Í)q”ŠÖ!ZÒ!Ú“â°¹`¬žMÜL€l/Rª>ùOmS˜t­æ  #Ò¡©:ªÃl{ê,î¢)‚Ø´í‰Iµ§î^O·ûè²Ë= I µék+-7–¤…ðx©Ýô°Œ9æšrç­5TÎeš¾gJS»F -hrÐb´‘†j6°íE)Ë&ÍE¡ Èd–•3úì“Írµ wØZò%¦¨Ò¿¶5mÛV)•hÛVq³²é¼yÄ+Ñ’)máìëÝÉZ²Ä’‘¹ÖºH©–4Y6¿6M¯GÛksTq52•ÕÌÎJ>Äô»M—…àJåŸ!3¢BÝâ5)o´¹ÆîçÀ!”6÷šTÉa›õÜÉ> -endobj -623 0 obj -<< -/Filter /FlateDecode -/Length 4988 ->> -stream -xœÝ]K䶾÷¯èsË|?€À‡1bd‘ŒfzwlãvùûùŠâKTI­î/:ÙÁì´ªø(?~,R”Zž~¾ø/9…½3çËO§ŸObòVà_J±¸HêïðëíY -ã'+£ öü ´Ÿ|´Ê»óë ÿ —}ò×Óßsú×2<«)F§¤=ÿò‰TFO>x/,£ŠP ©•µ“÷Ê:Fåä„&ÆÈ©`†õF™µÊà ƒåT0Ã(U€Úx©×ª3”—†SÁ ­gTRÀ©„`u0D££“vrpºãtJN.DY‡NzÅé´žœ7ÒJN'çœ œÎÀ…ät¶XiŒàt°Åèà9ƒ-ÚiÁê`‹ -^s:[¼]€-@W`u°E8!9]Ô“ÁV¡Á‡µN ;Ù µàtRNØÔ¬# -ä<£S°Å Yl±Z(N§a‹Á§OXíC`F«¢Ñ­…–œÎÂ¥¼au°Ï1€r°ExÇQFÐD¼Äq€ò~2QŽTГ &p, BœŒ÷š£íd\ôh!¡S’#-`‹5Žc-a‹ÁÈeu°EGÃqV° 8.ж(£9.Ð`[´p\ A·FDÉqß!Ç„ 3ÇŒ«ƒ3hP. 9.Ðà\ îä¸@ƒt1Ü=Ǭ«­ShЮ6Áq\ Á»ÚHÁqïjœ¬¶À€'-ƒæ¸À€wµžãÞÅ0R W'àÐq\¬L*Áqïª Ç¼«<¦Vç'ŒAÍqï*‹&p37xWY¡8.0à]e”ã¸@™À?‚À» ˇ°Ëqæ{è”清M -a Ǽ+#¦?fLƒ­&¢ã¸À‚we@÷±º8I3ã^ž¤s‘ã ÞÅD¥9.°à]Ðgà¸?aFUXð®Ô˜©9xWª(8.°à]Ä –ãÀa’8.Àü6I貺8‰Ç¼ ZR`žD“°:?H’ãñ$‡r\àÀ»˜Þ#Ǽ+¬Ö8ð®0{XlÑAq\€¸: [fL#X„Sp:ð®èaNÞÂGŽ x—¦MŽ œ' -ÀÀ Q°â¸À3EïÇ. -èò±:7áÚr\à”Åê`‹ñ†ãÖSÔ1p\àlÑJq\àlAÈÀq"‚)JDßìò¶.ìúÀÀDŠ€É¦á¸À[‡õ¼Ê霚‚]³ºôxo&LŒ㬥¦€AÏq~¢p‰ã˜4ø[0T8.ÀT4…ÈœÓIØ‚ ›ãDºÐIÉqAP°Ñ ÇA¬2à¸34Ö’p5§3‚Ö™ã‚`ÜD¡"Ç&Xi‚)YlMp\€þ¡Å¶å¸‹,6±Ìau°E[ÃqAŠ–›ã‚`‹Šã`“Ö›žã‚(` ">Ž ¢p…ɀơ‚ã‚(Ö›Êp\€qIëÍÀqAÔëM,ñ˜qµ£õ¦ç¸ Øb¢°:Ø‚ )2㜄õ&؉Ó9Ø¢¢á¸ :GëÍÀq ¬7ѳ[Ö›žÝ.ïÚfâtà]D¦Žã\`½‰È–[Û ¯õˆÏxe g`w ¨×bne· ¸³•g÷ È—ÖHì¦ÌÄÊ28v×Ó*”R°Ûü‹x̬8aÞ}ê·¨àà1gOŒÑ}]s´dz#Ù¼?ö˧/Ÿ0ùžø÷éå7i§¬üH)µA|…a "ºÛ"ú–h¾viÃE¤ÉBxO2ÂÇf¤-æM72Óý;¯?,,‡ád:rc]äh/À`º¹~0Q*Ý`QŒõ¢9³™|sž«&{Dñ\?HÞÖ檾Þæ+ÎÖ›ó¼»­À.–¬ç²^W¿oç·zõéª^ƒÙX9ŒÞØ£™ÞÙh]©R‡Rˆ†yØ%LÐcìÆ,ïh°Àrñ¡Cͧ I¬„„Y$þ݇¨5[1ïÚ×ËÎ|DxS¥²ç?¾üã7ßüþ?¼œ¾ÿ­Àºå«/$þš—ù¯ ùú ¿—,©Y7V´J9Éz‘¯sÞ’¶–Ó·L§^Z&6yIg/]ž—e^ñ²Ì_m(×~þ«mg§Ìí*vÚV–vk¹}£íc›ŠÍjéÇä;ßÙì»vÆζ°lgµß}õó‡oN_X‚2­­nÃñ±,oÁñ¯ßwáë·Ûr]w+×Á¨¸Ý-ÿÞÕE¾Õ?Ât´§ä©0U &=Wp"èúµ­ˆüÔü•º2ü+tKyãÐ|ÂÚÚùÕ”6>wiŠ?>[yÓ=ª[q|0Ï£Yª]"ÖÒ*Š.ËžzÌžÍ$;’ÿH„¥¬HµéÒ¨¡}¦«Cõû¡Î?Ô=ùÞhLuêuÞÓð˜Ö·sóÁ<‡i³ËÎãdLzò«ï°Þ…Q¯ôu Fì\&1;Ç”cý{ý¼šüÇ12b†™ài 3AÅö$q³ÑÌp˜®i>.ƒ¡„ÕÁ_ìŒÒ·“Ço x¤ÛG~¾†ßcy¿ûÁ1‡ß…ŸGžî×I„DrXéÓSG[šú ¾ô»ŠP6ðŸÒ\–6õ¹Ü쿆›œ–±¿!¿¡Þ:±ÀN*KóöQŒ ^ÚxÃÇzôC²1ÛPÚ“dœ}¶ÙcŸrÚÐô£MÛ©œ~äÇö™k÷j”ËÖÆR~‰EJú‚%3uýgæ¹H]ß.ìÙõ}—ƨZö—Õ͇¦g$·Óï Ò!ÅÞéú )måÙÙQÆ´oéÆ2eÉw¥ß¨Ol‹ó<¹ç;¿ò]×þ±¯ªí®kWÆJÂiöùrÆØ7#.|%Ù83mõÝFý,´îÓ]ßDWu.)þ K_¤þ“oŸq•XÙF‡Øœ´dÐÚ¶ûgc¶®þF‡ŠC]ó·Û…çÇ(ÜX;96Êé`Ø¡(ü`ž»nW¼cøm«½.NÁZ [6üNÃy\~ËLº òæNج0ºID:ô3TØw¿á'ôWû%Rte.7Öé •Í;n …naaºú˜…3»»>îÄ_k½ÑéÝ¿Ãót®sžoçµÜÔoà.­ÞÏó&(Œ²€òv°icî…Ý:ÆEËI>@xDZˬ´`}^Nú‹À[oÓë`•NG:¿7TšFöéY:š1˜Uà2¬ñèÀÁþí¹cydë¡rŸ¥“^ÛÜ'žúiiØ&(#ª_Öo2ßx­nSŒ÷Òòa±órgYÞj;Š»Ï¥ÛVtÙÊS=7·ëÓPÏå3¶å†qÝWº_h nWqŽÒx -l¥h—8ÚàåPáÞ˜r´jÌãC+`ØDumLÌóhcŠNÛ:&†[‹-5Úh²ù/ýúùšÕÑç¼|•OCšùo–+•õå÷cþ»´ÏŸ{¶½g qetp® FDÙ±<‚²f6ÃÝFÙÖëj3õYìnê`št‡²_‘9S}j»ìyq½ÌŠ6É”3°mŠâÇ’/ÇëJ>•bó&Ñ&R¿©Çò<RÃ.r‘e+X:±æŒ]®Ù87³ˆUÆ[d[·Øl—÷²´/Ù1²Îù,FRn_âu×]ëá³ÊyâÆÈ“­µÝ—.Æع帉ÌXùÆÀw ™Çò<2·ìÎÔC¤V½×#k%Üñ™>ïÓòÝ{ K¶ô«Óy/ãæQè[ìg¬š„5Ž³‹Âƒy…ÇâÅÚ3Ý-ª­£&ó¶ì’ö:r‹¬s– é7Ô·8Ãù<ç_1¢V3*Ò‡m[<ÉìÛ™+1¤ïãÁƒgæy8ä]‰!Çž2èëzv5“m|æ¢ÿZ³ZXñ7³¿Åî—ýxΨ¡þ̇j¸iðëÚ ôv1¢TSAÑS6ûè=–çáÐ{%®1+"û4ôl·jÙã»-ŸkœlÚŠFžsÌÌÄ´5ÒØØû\{{s”³Ø;Gì±<†Ø°o>L øÖ5ï-«·ÏÝÜ2*éÍ>×yûþ`ž‡Cî~Œúÿ¶£ónœý9Ñ]Ðzuôm=Ësûºÿ`ž‡Có~Ü»ŠG¯DÜì\³32úÝ¡ÒÃ4ZÆFÄ|vþ,²çµm#o+·ŽÖ-níbRzÜ‘{°ó<÷ãØ­û…=çRϼe=]ï¸ïôîîJíŽG|¶vÈV莹ÝYKéÞ4/m óŽxõ`žGCf¼¯¾ó¬ý.¨ºiïñ=bÃG‹~xÔÆno”Þš“Nå]AíÁ<‡Ú+û©o oŽ·Þá^ÕÏy§|cÄ2<6Yó•]¬;£®þÕ£CWîˆÆ;îÝÌóp½²ïzð¾Q <–VEq¾›¸â—áA×tÇQ1å~×H„{¨¯G@÷àÚ-6ߊÆñôHï®ÍïÏ:x,~µ-æ4ÑNˆ; ½EiµÇò<j¯Ä©‚GK?^{¤¶<ÆMÔ–³{Êí÷ò=r§`Ø—5ˆ|~¸S±µËFããñ%bV—㨺v‡4Æ;øòXžGCÜ°£~|ÒþÀ{8·Çéu<’ÒÝ6Tý‚xk»|„ì-ÛëfJÿ³[jÂßå÷Žy‹Þ¶ôÎuzsXì^cVq{5ɘ–ã²ë1åðò«å‰ZI_8 ÃYÑÓ;ú¶ÎÓÒŒª< çæátîþ2ߦDŽó UÎà¦óynJ¬&æ“>5è¤ó¸ùA£ô°OÌ«õúàg.+ˆLõ¥mÅüÀP÷ U-¿< ÕÙ³x0Œê´¥ «nüùäÒ»&ezß°pa’Š¾àìÌDÏúx¦÷[¡,Ý&“£åy#gú‡ŸkªüÍ‚^öL/úLÝÚ_äô?§ÿé$£™ÈÿéIäôv‘ÄA•‹&~=Ñ[ø #6¦=Î|ELÂüQÔ²Œ¬OGÙDì§8¦Ô¶¼%¯{„šÏÓ³5¿-ÙèíˆF”D:½p®˜"§“k1µ^L2‹é› ¤󕫦Ѓ>UØ -4}ÚV`ËxU,µè„v-]uM¬L±GÒ›o«ë\VÓ{×É®ûKZå»îÜ^N¨VÒWVJïO/Eï*Qeº3*V©-)k‡˜ÚOŒèrb„_ˆR´AgV}×kôÚY -LMK:Ft9±éri¶ö”¯©BË‹ç›) ê:TÇ3²K÷šO_ÍÒ¡ËÇ¥˜žP¯}-B‡µ^NÖÔ‘Øį'ë™Ô¶q…°z½PwåVq /MêCq†f‘Ñ;<éÿ™*l•ó¤ÒTHÑ{xÛU«„Æf/ÔË]eÃy@ßÒvî’ÞWÌöþ’A0é‰tkêê±…´sY'÷±˜G$UÉc)–uULVüS¹²Šbªkb[¡üº7;mªô.Î,t²ƒ»­å6±«ÜùÚ‹}íqI/U®öÖr;Ìá%MlfôC©¤‰•¨³]«¬6Ã@ -MÜhô5‰Ëh­ìD_S>¶AÆ͸—Ó§4Í)m=Åë´¤ïè£.yþÒ|ƒßžÌùÏ奴ó<®De U¹u!îg2úF™pPLß0S„uØtBe:ÚobZZ[ÐPçq¥:¾6-»hÄ^RÊÊ ²j{¡m>ìĪBWIÝyÆî/ƒk óÖ¾iö|©Kãè©dzq± -}spšN·)ù܉+JÚ¦thê2â¢/—]…±Y¶0“ÙWR]Oö›¾+4ÕÕ]ta]ê¦3mší‹.²ÔŠ. B`Ý_ôÙWö0Fß=¼ÞÜïÙdWFI+µWék/­ #i%<^ªÓ¥[^Ö9G§zËàmªeénìéä´¡ÑB ³¸è1ÚICË–ؾRSd‹JËDQé2QdµD_lr9[qKjq`½äkLѤ?öž¶}¯T'Ú¾WÜ*-•[f¼-éÙξµN´”5– ŒÌõ¹«Ô´”ºÈÖu›eÝe¶=wÍ0Í×"SѲÙUÊ»øïž±3çv!¸Tå'íŒÈÐ^㎉ÚH¯•NßÌ'‚sàC_®âMè¶Jnγ¿w²Þ!){'}MèŽÉh-¼¤ïXäMx…"^WtE•ï_ä³EÁ6é60[CæÄjg(}ÉVÚçLDõÒÓR‘¾SOSÐè«€¼ü›ãhåi±}’~þ ý9Ü -endstream -endobj -624 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 623 0 R -/Resources 4 0 R -/Annots [ 625 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -625 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 502.623057 101.433071 483.873057 ] -/BS << -/W 0 ->> -/Dest (cb101-1) ->> -endobj -626 0 obj -<< -/Filter /FlateDecode -/Length 4876 ->> -stream -xœí]Mä8r½ç¯Èó£å÷°Øƒ5ì1¼ðÀ û`øP•Óµ £ÇÀ¬þû~A‰dˆ -e*³ª{³îBu¥"ø ->>CJIŸ~~Ðø/9=¥”cpçË/§_OjŠ^á_)±:(êŸðýY+'¯sHþü lœbö&†ó—þyñ/§ÿÍé¿×uôÙL9£ýù/ŸIåìSŒÊ ª •ÒVPy?Åh|TAO8Åœ%ÌðÑ·UE˜árò’ -f8c’ J0ú¨íV•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚON’Îè)¤l²¨‹Ðéh$µSˆN{-éòBðIÒ9Øâ³Ò’ÎïS’¶8›¢¤ °Å«Dl1)ZIa‹w² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Å´Õå'Ÿ¬U’NëÉ›VÔaFaBt¶峨ƒ-Þ*#é,lqø‘tà ocJÂl54»­²ZÒyØbLt¢¶`ð%0¶¨$ -À šˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lq˜¹¢¶Øì$.°¶ €ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.¦{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]kQSÔÁXâ°ãdu²8ð®Õ*J\àÀ»˜FFâŒê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„9h%.pà]ãq -ÒÊ Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\€õ:c%.@c“AX"qïêŒåO˜Ó`«I§$.ðà]0|¢.O:Âã¼…—'B–¸Àƒw±PY‰ `?¯Jº`¦A×¢.A§£Ä1º K±–¸{©)aÒK\‚Ÿ(\’¸f@§Ä1ÃL‰ °MÉ 2—t¶ È–¸‘.tZK\ lA4(qA2 »Ì $.À -½$\-霢}f’¸ ¹0Q¨(q¦ vš`JQ[@`|h³í%.Àæ›MlsDl±ÞI\’¡íf’¸ %Øb”‘¸Ø¤ýf”¸ +Ø‚ˆO€ÂDa²Ä qè”’¸ ë„ý¦q`^Ò~3I\­Â~[ÔÅŠ9Ú™ùX/±vÆŸ?ýrúí?þøãßÿÿFŸÞNÿñ;¥\øýšþºù¯Öó_oyZŽÕü!žbÊoÓÅ¥lmk9¦2> íøåodKYCm¿övÛ_».§ƯY~ß G‡Þ¬m¨}¸*¿¬Û2ziÇã÷¿—ápõÃ0—árëáje^;¨´U„Qü—ß°&œ†”·)|¤"#tèÛ]ÛèÄÓ¶/%s{tÕyºÑuW9È£9òÁ8Wiä4+ƒåƆ¾¬  W†t¯ó±¤£:Æ­—$‹%Ç\–v_Y=×ëó¾°mœ—+^öeîÃ*ÖnXl©œ@uÒŒ<õöAç!㫇LÇñu¬ÎÓáëFxõè³±ºkŒÃŒ b•ã#ç1Ø1—ó±¬|¶f±í²`šl·;Œ•`¬cužQ×£&BÅ##ký÷Í>ï³_F•Q÷ïÖy6T™ëQÎ_›wîÝã›eF݈‡ ~hd\¦ìÐu¤«ótH¹1µIÅØØ𺣫±îe·çÆÝcêönvœ,ªó¼ÜgvŽÕΗ.mÛéßÇýso‰ƒawK»ÐÒ6ßùI†y—ºƒVKi3e• ¥O¯£õX§Cëøëîá¿tÿÜúPëzw§åìúÛ fˆó¯=²)÷ÂÊ…uÍo«4ÚRYbË›IyUבz¬ÎÓ!õF6ìá$#áEŒÓÄò˜yûþ²f2"mï<ÝVŠP"ÜZéÖy6DÚ1áΪ{OJÿ®Ë ßr|ôßeÞwÁd÷Öy:߈U?Á;h¹ÙãjÛbÊWë}‚Ōςðô²Ë­!c̱U»µ'?XçéÐ{#v=rÝçƒþÍzϬ¹vcT"ø¤éÂpN¢½;h| ïx°ÎÓ¡ñF|:Fûã•“a”Êžº"v@Â_o”Yµí—…ËX^ƒdßÇÊÈuêþÀÁ:φ\wì -î£×Ö¿9BãºÜ…úcøõ=ñöÕüÁè§±Ì ÎuäZÖy:ä‹_[&)ʾBÑE@Ë{Rv#Pä ·FDÉÈSŽömÎâû×wÔçw匙$V·e¼¤5dô‡Ö¥¡m~¿ÄÆÏï£p\ç>Xçéz#V}bÇêEË8ñ¯¯î¹xëÓu¢‘ÓZ½`æ€]+NÖˆ±Îµj+Ë×…s£súº;ÄÄ>Ϭótˆ}ß݇Ïç½?’yÚÝâͨEF³ Æ=XçÙÐì¯Ç¸ïÝM]»uúÈUÛÕ*ìú—ëD|ÆÔkîü·ÊéšýË|ïý½ûŽÈ¯6“wÐ÷@žõ`§Cß8õo‘çîÉT]Açf§ta}^»‹ý«ävÌcÒƒ;®ƒužÉï˹~—Hþˆ\ãô¶è¹M>ú×}¶ÜcàØlgQßñí —Å§G¯Ø¬ótè½qoè8ZîEºeÜéÔÜgCÅÈQü®Îamf§Í{Ëö;ï¸mZf€û™ÍT†îÇîó’Q¸¯à`gCi¸q_Áâj†§zú•!dÌ¥Hj_ç£Ñrjsá·Îá~ÏW‹Q•›Œvɹ[H=VçézðÛAÿå@¾ÃçvýÚGâùÄu›{×›Üp»wÐú@ö`§CëõصÜÛ%«{5µZçUë -¸“eo#3D#jé¾úÕ]ƒi]®Ü»:f„j\÷¶®¿AÇ‚š’ûÝÉ-Ô™¥êwî˜qæÙVlŽìú½\ŠcØms`̯°o\Ê°¶Ê·‡‚ÑHŠD;ÄøOÖzâv‚4h«M*#ñ‡?þË?ýÛYž’24´§k×Zê•Áî Ó¼<Ì؆xòð\Ö½Ö³›7ò´örѽ©u–:Ìå¨mÒ÷ŒÍ\¶¶Qú¨;§x»]o{/ ‚©½y?¾ô{éûÜCn¶ÜÍã{Öú¿–«w#‹DyêáнX½[ÊŒul?Û†#vöüÌK¸[¾³Í£^VO­ËO_ï„Å-aå§-=—ñ‡]ÎÁÓš’–‘¡‘ýyÛN°ôÐJz0ú¶µœ×|Î弯ØâܳCõsYùƒafNl£äåLÈBÛe¶ä¹~÷·¥­> ºÏÍçÅÕ&¿¬³©ûÚ/O$rqiwñ_ÇÍRV°¿Ï>;|P+씶¬l_vÀUÛÑÅÆņz>E&Ø·ú<ô_q\êpFÈý³ÖÝÖZ·Æ¥ 6Æk6bãàæõÇ\ }Uì•q`þô¶ŸcÁRûA—«¾ô_ -<2禗vCÍ‚ÃR¹ZTç³föÖ¹{¹±M]ë_甚m¿5ÖͦÀìõW}G_Í×D—ñNñ¿‹ù¥þ°íŽï^WVœRž­8åXïÆwžù”«_Æl¸RÖW@¶šÉ¾ÌÒ¾µ¡Mz¨¾ -I%¡Í}Û®ûi»2ïÛ‚é§å)©ÑÃ(Ooˆ¡çœföÐÕjß,r#²Þ>¢µ>èôL?cèÜý„&éõH6 èé‰ÂR€FË-¦RUXnÚ^àI®jA\^ÊT -ÐË0¾²éågWÖ¡(_†º¸2ÏrÓ–WF a»ì”þ^š¸¬§^k¿Â–Ù³¢Ô¼ÜrVÎa3Œ¿žBy2¶.oG¶&mèõIçà&š­1ŸéiÜIY§ë°éâh}Þ©Y^:õk+µ¼‡JÑ«)è±äóKªØÁRþ×2à¿ "tS¨1`ygŒ*âdêA9Ñ;ƒœ v®’7Ä$\>ªÖ–Ó-.­² öŽqÊcIëë3}Yðj}ýh|=ªïk5z–³Sµ-O3ž;„¨ôVLg¯&½ˆé½Ki²ËQh¦øPm†°7èxÙÞ`¯xS¬­bB¿*Ö]WíÑôœþæºÐ…Ítî:͆¿–5‘ ç5áå„ní(ý"Jém/µ©ê]£šÌ2£r“úZ² ˆkã$ˆ.'AøƒªM; fÓ³Q£gæÏÂTäZ¹\Ë ¢ËI,·´æÛHÅV*õª¹z¾›Ò¡nSs¼ »° ^¤ɱ9¢lý¸Ó> µJ 7f#¼œ¼k3±‹¿œ|JûÎÊWèq¡eí6qR/]JE›?ajV=qœþŸ©Â7)1Où ]ƒ½5 õNhn6ñÚI\ë8ÉÚÅ^–¹KÇØ0Ëý¥“Êé¶ÒÍc+)s“Ç\Í#’jä±놺&& -«þi\É…ªîºØ7(Y‰»¾MU€½v4ƒ»oívqhÜù…‹cqM¯€hö¶væ«ðR67Š ú©vÒÅFµÕ®wÆ„Ý0BwýRÄu¶6v*O]\>öI&­¸—ÓŸOe™3Ö›ŒÁjz£ºôùQæGüþ×Éÿ¹>B^ǨÖþMãÖ•˜¯dôþ»tPLïëÂ6m˜Ð8Fû]¬S/ë+Ú:n ãk׫«N쵤n ª›– }÷!›]£-óŒ¿"¼ ^lÌ{Ǧ/Øó¡­'‡n,½|¡Z…~:8L]gû’|fM∊ö%ZgXEðFqÈ:Ìݲ•™HÞIs=ÙïøPXê‹°0‚m×¹¾Ìò¦«¬œ+‚ØòÞ°Ó¼Ó¼²G0úáéõîq_Lu´Q{“~áÒvb$m„'Km9 ëöæØ3ÔÉÛ=ÔÚ²lîÙäô©ÑC ·:àeÒÔ«-<ïÔUÙªÓºP4º€LUYk1V›ÂR­º¥œq½[LÑ¥æžö|Tš=•°)KíÖ¯EK¶@žÃ9ö³S½d‹%“ ¼v“º^ÒVÙ¶o·î»®¶gv®zdªz5¿)ùÿ=2wæ:CB*UJfD§þÒ,ÔNGkly°J!€C½ -.ºÄR%w×¹ž;ÙfHjî„÷„ᘜµ*jz#´lÂ(òmkª¾-Z®–•xJ÷9@H ušS›ÌPy%h¹4[DíÞÇcè À–‚ÞD/.ŒJË×ò(òºJŸ”Ÿÿ” û -endstream -endobj -627 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 626 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -628 0 obj -<< -/Filter /FlateDecode -/Length 4760 ->> -stream -xœÝ]KäÈq¾×¯¨³€¥òù„=a¯aÁ‚öÁðaºfZ‚Ñk`åƒÿ¾¿H2$ƒªîÕh=]ŒÈG0óË/#‚,R_~~Ðø/’bLÁÓõöóå—‹‚Sø—K̲úOø Áé䣻þ̆œ þúvÁ‹Ã¾øÛå?~sùŸy}5CJÞhwýëWV‘B A9A• RÚ -*熌ó‚Ê맘’¤‚.¡µ*À JÑI*˜AÆDAa†¥ íZ•`† š$ÌÐÉA¥ìÐF)QC‘5‚N»Ácн¤3zð1™$êt:Igíài§%]¼÷.J:‚-.)-élqšHI:ØB6Iça‹õV‰:Øbb°’.ÀƒÑI‚. +Š:Ø¢¼Ò’.ÙÁ¥H$êt*†¸Öå­U’NëÁ›VÔaEa’|t¶xå’¨ƒ-Î*#é,l!üH:ð„³!Faµ^ÝVY-él1&¨ƒ-˜|‰Œ‡-*x‰°‚æ%‰L%C ˜hŠ%01 ‚•hÀ$7OAâ«4tFKD`lqä%&°¶V®¨ƒ-6‘ÄÖÀ¸ÀZØbÈJ\`Á¶´H\`A·¤’–¸À‚oIi/qáÂL%qãÚèIâ Ê”’ÄœkÁX.–{¸À‚u­óFâ Úµ½Ä¼kI+‰ ¼k-jŠ:ØK\v¬ŽVâïZ­‚ÄÞÅ22`VàÐK\¬ &F%qwMT$qwMÀ ê€5h%. ð®q8iç琉ÄÞ5d¼ÄÊþQ¢7Þ…8Ù€- X‰ °ßCg¬Ähl0pK$.pà]°ý kl5蘼ļ«#¦OÔ¥AŒ¸°n1ʃö>I\àÀ»Ø¨¬Ä¼ úŒ€ãì¨FâÞÕ;µ¤ïj“”ļ ¿ÁI\8 €ÄØß­]Q—•b”¸ÀwAKFâ샊`QIK\2üP‰ ã¯þòã]?ýtùç|ž¶'rðǧÉñ1œ„Wì -ÐòÎÝ ÍUÂÚ4898p8óÁ&ÃX ¡Š¢ÕèPéu¤z¨­«°Ã°æШj(4©(Ný$–ý¸ß¢± ÄŽƒ ´è^j©œ)ð^´ç¨6|a”ÚHµé3ú/#6Q©ƒ6s -žœ·#D_§¶Ê(-úÈ#H]ýŒù…=S]—ֲó‡ÃÇÞpö îyÎ aè|Ïó¨6Ï›rÙKø¨‘'x£ðhfí`Ó¥ivû}TšÚd'Í™gÖñ2=šÜ ·Ê=à×Ee±ÖÇÎOTöÊe¦±kέ&Îb,:Í+{ƒò4¨}"0g™H)±ó´Ëy'ë¼g7øU¸¶ìpŸú<Î}žÿ0Íùë|EåU·µºBW&µÕXÚ(幟Œ­ÅjeYåØ%æ–}êÖ×3X=þ5rΞӪßb[êX&ÌmÈçùua×í=ç/#Ô TŸÐƉg/°]€ž«ò$ølV³…Ûø¼ç9*_Ûæ¨Ê?aäçÆ^f•‘2qSÌÖ¬ù‰¹ô¢Ã=~2ÑÝÍO'ë<ÉüW~2)íÎá³øÛñ€F£ÚMù7ŒÇ¢Ž?ÛI÷yQæËôw’3éËï—éoêʾ|kÏ¥q[m¯ôíVn–Q†ð½!FCÙÉ:O‚²f¶¡=”ÕÝaBM;ˆSÓqé—n¶Ê(i³Ä€Éh²ÊÊlÅù¬ÔµßA…eoWÉý™í¶³î™mZhgÁ¶9"ò Ô¾žï+©îÆyµ»n Õºû‘z®ÎÓ!•vùPô=ô‚WŠN­9c—knsô̸k1ÛnŒb\W÷6·/ÛQQV–W¸’¦ó˼î»c»øl¦:icåévõ¼o±äOÓêm"ÓW>ä«ïD0àÈQ;Yçév9té©ÕÑ둵‡¿õ÷}~B¾ûdéV¾®5ï«¡y…Í÷#gåÈ3böQx®ÎÓ¡ðœ¿Xg†:$lqÖK——£c¤.¹ODÖr&{.½£¿ÒGvE_Æúcd2oé÷ýÙ¸mëŒ'…(øÔûƒ|§BÌW—w‘w²Î³!|ÈåL†ùH¯Ð×Íìj'Ûø,yÿµ!ZXñŽ´³¿Çî×}ŽÌ¢ÿ‰^x%¿úªÝ@oç#j3$ _·ÝGï¹:O‡Þ¿²Œàëüoå•Ï‹™í¢–=¾ëÑò­bd)·WÏãEµ|ÝbG¯žÆF¾ñ®Hî _zÕm ¶ùŽç{®ÎÓ!vßß|ð½1ï=ÑÛ7ðnîY³òtÀµñþþd§Cî¾ú÷–Ñù0Îþ–è.h=\}2š¾?î?YçÙÐìöýÞ•?zàA^yÛX}v¨Ì0¯·ðW—ˆùæüYd/kÛGÞ×n»j¹ÆÎ'åï*ï)ÞGã¹:O‡Æ}?vëzaϹ<3ëÝ ;³»©yÁ“«+ ³ü£3d+ô/ãµCÚ‰ÌÚÑõëã}i™ø«'ë<2üÕÞµ?Uwå?Â7|6ïgµ]n”¿‡‘ïp¤þëê\å‹e„ú®ÝŸ¬ólõyדתg°Ø!—HÊÙ!3¢Œ¯&®ø¥ÌXA4©ù‹8‰âÚÏÕË»6f×NØ|/—wôþnË¥m_ºz\-õÍç¤äøÄßËÙGí¹:O‡Ú?UÉhéwÂr_ãVõ:ú®Æµ{!ߟåí{„Öü·wLåØå5{¿¾R±•‹=þÚÓ´r©»syçÙÈóðå¹:O‡¼ƒëô‹øvµÖ…™]\î®Ë;Rº«†¦‡·²åKÄÞ“]§$}·µ wþeˆƒ¶”¿Ù±àsužÁùÓwø”gÐuOÞrv}«Ï „¹M_Fäßw\=­÷Ÿ2ß2Oßçy€²å*9ºW4t¹Ë³ÑÉ:φ¾pÎß<ºÖNR”²÷­‚Gü±ÎWüþxî=+y¥ÍW„#4DFÔÑÝ('ë<Jü˳QG•rm,‘ÆQQPõ^Ì™ŸwæûC .ë³àGwk„Þ;y¯ÑÉ:O7»'}¸¸Õµš£ œ›áˆÁ·…3Ù1ð7§ÌËx,é¸N‰0ŠÞ²w~›Ú}éêQ«ß÷U¿eÑ—ý<öÁCm×O¶Nà:qD^Žx>âpwäÉ:φ¨xp•8>6³œßøžÙç}öo ÊÞ ž¬ót¨:¸kñoÌ;wÏî‡ñÍ´¢üáø@Öëd§CÊÁÕ×.¯ø«ùÆwh/ï‹:“—•¾Y"yuýµ¤qu9…r×Ó¶È ßuå¤äk¸·5ÆûïÆ>YçéÐzà}ÃþWC½÷œÉÌÝqwOÎO,Ë}Vëû#–×Ìf™´G&u™­³ß·:YçÙš²açÕãw“|`æíûËšm ò;OÖy:Džû&Ë»îF¾çŠÂ·\·¿ÿnü@vïd§Cð¯úA÷Ó¿ûYüªM›NùŒÏ‚ð­‡(¶øúì÷NÖy:ôø®ßñ·ü>dÕ|Ó»+ÿ4=ô68LœãþðÓ`S÷ Ý -ÁÃ"ˆ+OÖZ?qõªŒ?| MòÛ®l¼²S™? zëÑ[æë˜jÉbòS:eºàTnÍÓZÕ‡}˜ò¸®0«ñ²iV•ÝE_Û4åGwù±H½ÜWÖ!i4…V¶.BÅl?^:*¡<“n-]Gl&톬“‡TÌc’ªä1늺*f -+ãS¹²ªbºkbW¡ü67;]]ªNÕμîàîj»Mì+w¾õâPg\ó=ª½µÝóExË-Å ýX:ib£ên×:ë„Í0B7}Ëâ²Z+;ñ[ -ËǶȤ÷vùË%osˆ ?<Ý[Í/¨ê½.}ý?”ù ¿ÿ}¡ë¿”7"Œû¸Q•-LåÖ™¸ßÉøu†ñ¤˜_oX„uÙtBCí7±Ž­¬+h¨û¸1_S«®±—’º2¨®Ú^èÚvbS¡k´íFÆío‹Q¬ŽÌ{ç¦mØã¡-'ǃçwi«pПcÓÙ¶%_»&qÄEÛ–-™®"úFqØu˜še33}'uèÙ~ê§Âr_ÝAçFð¡m:jÛlßt‘å³èŠÀ¶ýAß0é¾Ó4³G0úáåõîyŸLöe°´R{•¾õÒzb,­„'Km>ôóúçØüõ²xÛÕ¶l·ölurÚÒh®ÍzŒvÒتM\ß)Ù¬Ó²QTº€LYm1›üT­ K>ã(ŽR¨>E“þ¥i×ÏJD×ÏŠ_•åvËŽW½%[ ×Ã9´³S­dõ%£ ó}í*¥VÒÙºoš÷]vÛkwÔ ñÍ3U­š[•|ˆÿY;â«R¤Rå'gFtlïÂFM:Xcók¡UôBüf¿@±K•Ü]g?w²Î”ÜIߦc kUÐü‚oÙ„7(Ò±¢kª¼ü[®–”xJ÷ €j4·~±M~ÃkNYæQ= *ò -;½‘ßC”ü¢›|?òm–>É?ÿFÁ`Å -endstream -endobj -629 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 628 0 R -/Resources 4 0 R -/Annots [ 630 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -630 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 770.673057 101.433071 751.923057 ] -/BS << -/W 0 ->> -/Dest (cb104-1) ->> -endobj -631 0 obj -<< -/Filter /FlateDecode -/Length 4950 ->> -stream -xœí]M$¹q½×¯¨³€Mñ#ø:!­aÁ‚òAðaºfZ‚Ñ+`¥ƒÿ¾™I2’™•]Ó»* žFOWFð#|| 23YúªðóÆ‘ôc -ž®·.?^Ԝ¿’buQÔ¿ÇopW­(LN'Ýõ‡,°a -É™à¯oü7\òäo—ÿüÅå/ë<új¦”¼ÑîúׯYEv -1åU‚Ji+¨œ›B0Î *¯'41%I3\ C[U€”¢“T0ƒŒ‰‚* KAÛ­*Á 4I*˜¡“ ‚J+Ø¡R¢†("kv“‡Ó½¤3zò1™$êt:Igíäi§%]š¼÷.J:‚-.)-élqšHI:ØB6Iça‹õV‰:Øbb°’.Àï$Aa ÐElQ^iI—ìäR$u :CÜêŒr“‹Ö*I§õä€M+ê0¢ÐI>:[¼rIÔÁg•‘t¶~$xÂÙ£0ZMÝVY-él1&¨ƒ-è|‰Œ‡-*x‰0‚¦ÌK˜&J†$0ÑN)J,`bš(+Ñ€In"Ÿ‚ÄVi茖ˆÀ*ØâÈKL`5l!Œ\Q[l"‰ ¬-H qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ] ÷ qëZçÄ´k)z‰ ,x×’Vx×Zäu°–¸ì8Y­ÄÞµZ‰ ¼‹ad$.@¯NÀ¡—¸X™LŒJâHâ)@Ô… cÐJ\@à]ãÐiæ琉ÄÞ5d¼ÄÊþQb4Þ…89€- X‰ 0ßCg¬Ä(l2K$.pà]0ý cl5阼ļ«#ºOÔ¥Ix\·ðò¤½O8ð.&*+qï‚>£Äàø 3ª‘¸ÀwµÅL-éÀ»Ú$%qï"np“F qæ·I+@WÔ¥I¥%.pà]Ð’‘¸³ð¤"˜DÔ… @Ò€Œ'…8TâÞÅôž$.ðà]嬕¸Àƒw!ìu°ÅF#qâè0l…1`uRL!éÀ»J£‡%xW©$.ðàÝÄï¹,߂ߟ¶)Áv„kí®3ÝðwgX§Û@P@S/Ÿí SÛaw¯í#¬W6Úµ/Zž$øiLÓÊ‘a›ï>5Í@¡”#ÀCÜžÌólÀÕæ˜pií±úwôèˆðŠn|'*s[#P$ Z#"ç±@’zÅçˆ4/ßÿ…¿FÏ[,N £?H˜”†²é…!rôóІ]¤Zzw„p2ÏÓ!•9¶L¡µÇG~³ŒßáuŽRû2ç·nËi­¾¡œœÞœ°kʼnÃ1æ¡liù¼°Ó¶Ü¦‚÷Êza!Ç`[«CœGvû@L{2ÏÓ!öN0»7«³~® ïÛ#™Æ¥v ÃgÃ…?ZïF-;h~ Â=™çéÐ|ã~ëRªôF7m=n†r9VÄQïá{™QKl4Œ£¨¯øvÐËâÓü*&½çò<zcÚMo «–†>3+ºñÙP1rÔK癇U´™2ß‹ø{¶›hç2è © Ýâ¸[×JYÜy¥çò<JcÕ]ÄÕžêé†q/u@REkA&ÍqåßóÃÿåÛ–ŨŠ&£)ÝAêÉ<φT{×þÿƒh€•×luk‰í ë2÷n(6¹ávï õ=Ø“yž­Ç±kë¹×5JÊœÝÚ\í«Öpg—½õÌ Œ¨5¯ Ñ,mȽ©íŽPë^×ù7èXPSö~wöêÈÊeYÿ¾gmÅæÀÚÉ#¡áþY³ßï ø=Ù“yžÁÇñëÈ_ãŽÒj7É3$UÏ<òP/ÆŸãÐq‡kŒ_qn¥2[sвY!íÕµŒóÜj™_óûrÃ|¢^wàâsyžÉÇ1®Ö³›{BOÙ³Ã=¶=.¬e \jÙ×Í}9buè¡þ!Fmï~|~4Kv›wÅ4"¦é]Þ“yž Ótg—w˜óœ^üÖyœQõN_·xÄÍeff—˜r¬ÿ¨Ÿ÷î%oÆÎh ·;mg7DÅG³ÑÏH˜ni¾¬ã¡‚ÕÁ_âŒÂÛ)ã·Çº¤ó+}~¾‡ßsyž¿wövü®ü<ò4‹]Û“.i+<60u&¶}l§Iíî¥ÃnãþÊRNÙwÀÚ,ßFÏ=@(Öí’=·M‰ò ïÚÄÒ¿þÝ¿ÿë~Çö)óM^Óµ{-Ë@š=aš‡‡ÑÚО½;§u™}^jËæë<.£5{tÖµzÓß±YÒ/å”zÜìíÎÛ²JÚ‹ñ,ù‘¥¡E^ü}ÆÁNsÿ±¶Ý.¿Žõâr]Ú°ø§¶Cå½\>c³Ø·´‡¥§lëmF„ö‹i~e“ËoãØ|Ãb“]úÉeF³‹¿lËñ6¿ª”_‡ß–£–vÍ~*¾:²#Æ=;ToËʹÌ<¢Ûò±^‘ñI(»à5Íù F^—²>svŸ›¯‹/ªMn™åb÷uncÁEXÊ]ü×1¶¤ìoxÒKŸÙ¹<Ž·R–•í˳¹yí«UÙÇ^~(6.6Ôö™`ßêóPÅñ0ˆtÍ^,‚¢s -ò-Ö µq©—!fY¸2wÂn…ÉO*åㆠ-ë”·Ùsכ߻%›&çq„Ýë½ÜWùô ïE{îåv~J}&åæ·ÐÙÞõ)ÇZ‘óvÇC?×ãw[O¶¼ažß¶}Àóùœ¸Î¹÷rç~Zóqž§h0Ê"!ÜÙǦKK/Ö1.DN -Þ=à/.Ú lÂ]¥v/ÐÜrù¼¢”O*-#{ôÄ…À2ε -t÷ᨓyždݸ/*wÄ}êóþƒòÇ£³ôÄd7ÃÕçº|ZØ9[ÐͦÊhÃy,æS°Øp–Iž\ ‹ižœ c±f‹CšRr1è¡Â#ÆüRÿŒ)g͹‡ôNæy:æ3Böq¸yì|Øh.EËÅ´ü†ùZÔ9öÀÈç!Í—þ`rY6˜E_¿,KûòsÏP÷˜RL¡#æ䣠'ó< Êú[£Z¡ì=)mçòG’ÆÇ’öõá¯Ik<×cDýóÃʾ~ #{7ROæy:¤ÚC>üÖÇy~Li|Ynç†ø@òmmßéHêHZÚWxݳk;|6Kž´3òtoGk7{eöèfÓ.2©ña>¹ø{È<—çéé9ô[^‘ä[Ýäy??!ß}²tO¿y4ëuÁ;(ì±93)G>#æ…çò< -ÏÅ‹­gØ-½‡ Þ÷zø–ûDdí»?ì€ÿŒ§:ü~9Y-8ô´Ë§Êç#Ò;¨­aön’;­[Ûƒ§®z8ìk½K"óW*Øx5@b -ù½=Š|“ÑÔ]~9pc¹©˜!ÑnÀ/Q‘©ûeΚÝØnÎdÝ×Þ5ec¹)Rn:¤YnÚÍYÕ÷â¼ÚÜ´,õ}^öFòüÆnž´òê7q¼ê7ùkƒ«mØtã_NÓÔåDeåã¤MþÊ…«§)ß3 éšOðŒÊ’®Ý¦‹£õu'gù¢Š[ªå»+T>Î:eZº•_,é,þÃEç"ÔÌó9󪈣©]üvÉß3@‚˜¨? -pGœ…ËGÕÊ"Ýž,¨²H8LiLiݤ–ýñƒ¼A34®¶&çw5[>ÿ‘TMdË ˆsÅ‚9½ÞŠsëÕ¤qþ®†8ÙåÊ7Sò ‡&ìOÛ ìULè¶BŪëbCÕÏöm®ó]ØLç®Ó¬ûkZXw oTkGé›(Í'Ä×¢ªwj2ËŒJMêjÊÖ!ÔúIÝ.‚ð;U‹&tfÓ³^ËçìÎÂXD-]ªéÑí"¦[Js­§BK{ÖT=ßMéP·±9^ÝØ#:!ù #ÊÖkq~Š¤õµŠ 7f#¼]µ‘ØÅo„Ô®s…rz\hY¹MUÇK—†X]…¡Yeù”ÒüÿL®I3󔚤òIÃýªW’Çf¯Äå¾±IÐzZæ.BÃ,÷—ŽJHŸI·¥n[I™Ë˜<¤j^&©Fk±n¨kâLaÕ?+¹PU#P]»å·•¸ÛéÚPuªUæ5ƒ»kåv±oÜùÆÅ¡õ¸ÎÇF7{[¹ óUx+â ýX+éb£Úl×+cÂnH¡‹;¾q­ÊÙ(ËÇ>Ȥ÷vùó¥LsÆ:“Ÿ&ðVçoAàQ—¾þÒ|ßÿ¾Ðõßê±»ó> -endobj -633 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 311.373057 101.433071 292.623057 ] -/BS << -/W 0 ->> -/Dest (cb107-1) ->> -endobj -634 0 obj -<< -/Filter /FlateDecode -/Length 4712 ->> -stream -xœí]ËŽ,¹qÝ×WÔZÀP|_€1 2¤1,xà yaxÑ·fZ‚Ñ#`¤…߇YÉG2#•Õ·• ø6úve‘äáa0’YTW‰Ÿïþ ¤DÑ;ºÞ~¹üz‘Â[‰CŠÉÅ þ¿Þ^•$/¬Š.Øë/I`¼ðÑjï®oü×]¶Éß.ÿù›Ë_§yÔU‹Vöú·Ÿ“ŠŒðÁ{iU„J*è¬Þkë•S·#§‚Ö“¦¹Êà ŠÁr*˜AZF`†!¯Ì\a†öŠ8ÌPÑzF¥$ìPZJVC$‘ÑŒNYáÐèŽÓi%\ˆ:²:òšÓ#œ'e§‹Â9g§#Øb£TœÎ«ˆ$§ƒ-d‚çt¶g$«ƒ-:xÃé tภ-xÁšãHDïÇ.Jèàò±:'pm9.ð Êbu°…?ïyO洫Ńßàø¸;óœÍë>©ÕïUi{|á©íd¦ÚÃKoµL~³<=³¹ï=HÞùÍFòúÄYwæ9ú>öìSðÜ#‘ªt®~CÇÚ.öo{Xø"¬ß(´3ÏÙž‹¹~J$¿G¬ áôÉ·fÜ?¼o‡6ßfãƒ}o&. ·ñO÷>±Ý™çtè}ì„žzW5£¢ç¨vWHËamz¡ÌG¿eûƒ;nX›Æ@?5#µA÷±}^ (=°¯`gžÓ¡tc_Á#oGo¼+Ë}“Ä°·³sú£¿ã?9*I _°õÝ;óœ©;ßúÿ'üÃ6¶k§mÄÞÏSßÛÅ£5ˆÁîÌs6´Æo2Ó(Èl÷j*9«æpã+‡{o GmÚW?Ù5¦é†½«}D(ûu¯Óü3tŒ¨b¿ ±…<²d~çৌ¶ÁfßÜgë uÏÏŠý o¡Ä1ÙyN‡àuÿõ‘oÒHo|$å–ïxäP/y9ß•x{Ö¾¹§E!ûÍJ~nkrZf+¤¥ºÆ ß[Mû ùÛçrÝ|"_lpñ¾<§Còºk˜õìì™Ð){¶{ƶą¹¬ŽKMóäuö\Žš:6¾‘‰µ¡úÝ¿+±2‡:Í<ï„i2¦ÏfðÝoÓ¹”ét…ØõP€¼™d·óƒ!òñ -×ôÓÓˆt˜D:”Õ„+œŠ>c’€ø»?þû¿þé ÓWZý|LŸøexQîv§§t¿œM©{à}H34þD7fpãRC%°Tw0ý½S]úb‡¯)­»ÿeŸ‡rBžLÇúÒCÚ”?9 þ®Ÿ”oǺ{†2ls6ßÃ÷}7þzqÃy^Ó@àU¹ÛÔÐÐêºs8êö×’j<ýV¦ñÒaH÷£q›‹1ý¯C‡ÿrQ‘DjÿD)÷“*å :_TñÛ%TJŒ˜°ÎÚ)NÂñ£,e‘Ê ‹ìfö"ö)rüKáÆæÚæ»IùmΖN!™™á •{ÅŒ9š‹ÓÝK¡Fq:í53^¹bŠuÙfkÔ¦­ÖŒ›bed#´s¡lª«bMÙ•N+M窰˜Þ6jº?§Õ¾éÎ5áí‚jM/}c¥éŒÉ\Tn]-‹Ì4FÅ"µ9eé*ýĈnFøÌE:³è›^K'uÝ…!‰Jº˜Ó1¢Û…M7–fKOù’*Ô¬1·|5¥BÝ„ÒðŒlh£QêÓñÕÍ‘&œŠ]êÜ×24¸Ñ3áíb©ŒÄ*~»XϤ¶•+¤ÍÐk…¦)·ˆƒ¬x©Òôzûý†f–¥sŽÒÿwª°Eš˜gø ¨@*UV¯j%ilñ´‘Z¹+¬C\ (ò5mÓ\Êû‚Ù¶½TLúDº%ui±‰´i²Fîc6/‘T!©XÔq¢°Ü>…+[¡ÌF º*¶ÊoqµÓ–¡ -°çÊœjànK¹Uì -w¾µb_z\¥ƒçŠ½¥ÜóYx&6êÅ ú!WRÅZ–Ù®VÖ«a …*®4ú6ˆóh-ì4|{øø±2nƽ]þr¦9m¬ŽøàŒJ稶^—ºþ/Òü€ßÿ¹ÐõßòÁ]÷y\˺pëDÜÎdéÔí°SœNáÎÂ2l¡¦†ö«X…šÖf4”y\놯©f—•ØsJUTm+´µ ±.ÐÕÊ4-cW„·®‹#ólßÔ û~iòÍ¡“Ž|ËVᢽ\†ª3uJ¾6Eâ*%­S:´¤›Œ¸h ÅeSa¬–MÌL€l+)MŸì§¶+Lª«¹h܈tiªŽê4ÛeÃ]4Ià›ö¢-˜T[iœØÃ}xx=Ýï£É.€$-Ô^¤o­´ÜX’Âã¥f¸tÓË2ç˜kºÈƒ·¶P)Ë4cÏ'§êZÐä¢Åh# 5Û˜À¶•R–M*ÍE¡ Èd–•}¶ÉÙr³ wØVòŧ¨Ò¿´-mÛ^)hÛ^q³´©Ü<ãoÉȶpöõîdMY|ÉÀÈ\›»H©¦4Y6¯›¦uçÙöÚÜUC\õLeÍfg)ñß‘±sÏÓE!¸TùgˆŒ¨PºÄDMÊm®1¹;Á9p¥¨=…&TòpžõØÉ[”ì-=ÖLh¨ÒœœE†Œ0càs•KŸ–Š6ÇEËÒqé^*6T4콤Iødøù?‚Þ- -endstream -endobj -635 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 634 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -636 0 obj -<< -/Filter /FlateDecode -/Length 4907 ->> -stream -xœÝ]KäÈq¾×¯¨³€¥òùŒ=a­!Á‚öAða¦fZ‚Ñk`¥ƒÿ¾¾H2LF‘¬îÖ äiôt1"‘‘_F$É,}UøùAã¿HzŠ1O×ÛÏ—_.j -Ná_.±ºÈì?à7¸«V&§“îú3l˜Br&øëëÿ —}ñ×Ëýêò¿ë:új¦”¼Ñîú—oÌ";…‚r+¥´XÎM!ç–׆˜’Ä‚.¡-+@ JÑI,ˆAÆD!†¥ í–• † š$ÄÐÉ¥äÐF)‘A‘5O»ÉCé^â=ù˜Ly<ŒÄ³vò´Ó/MÞ{%A—”–x²8M¤$d!ƒÄóÅz«Dd11X‰ ‹v’À‹ÞEdQ^i‰—ìäR$y <CÜòŒr“‹Ö*‰§õäà›VäaFÁH><Y¼rIäAg•‘x²~$pÂÙ£0[ Ïn«¬–x²HäA_Bã!‹ -^‚Ì ‰qIÂÂDÉ&Ú‰"E LL…`%0ÉMäSpÀ* žÑXYy ¬†,„™+ò ‹M$a5$,°²²X -Á[$,°€[RIKX`·¤´—°Àp!¦’°Àqmô$aä•’„˜kX€.¦{°Àu­óF ص½„¸kI+ ¸k-jŠ<È–°è8Y­„ܵZ ¸‹id$,€U'ø¡—°¾2™•„Ü5Q‘„Ü5K€È æ •°€€»ÆaÒÊ Ü5N ¸kÈx à(ðG‰Ñpâäp²`%,Àzž± ±É ,‘°ÀwuÂò'Ìi Õ¤cò8எ0ŸÈK“и0o¡åI{Ÿ$,pÀ],TVÂÜ|F €ñVT#aîj‹•ZâwµIJÂÜEÜà$,€;L€„Xß&­àº"/M*Å(aî–Œ„X…'$"/Lp$-aÀxRˆC%,ðÀ],ïIÂÜUÎZ P`…ôñ G› ¯ŸEd>ŠV…ÿùÓPºH1ïÒ×ËN|DtK£q×O?_~ýÛŸ~ú—ýxৗËÿI)ºýøƒÆ_–¿7TAäqÍ´ZF·rV/4³”Y躟ÿ"ÁÏmßÊr;µüнîÊfyl׶[÷Qëø¡­(È:™_~)§ÖõXæRf£‹¥¬uÛ1×2_¯ŒaÔ—õ­ ùAç_?ýtùͧµKæL*»$iÞÐÑPóŸ«ò/þ{8oJ:ïJÉ/kƒT/Nn">§;ŽâšÑG‡2/[GeA¶®Øwù·¶×Æëä²Å)Ë>/ôÎqë(}r.ÿµ›u2ýXÜ©"løƒê Q/»U[#HBЦMÌ–øÍïÿýßþó÷ÍH¶¼ãw‘,öíšµaª–‡éZÝ5<—u ?_Êèæë =ZŽ Ÿ§TXh~®ÏÚ£k·ÕËíºYà –vŠŒK¿\·È›åðM†Z†zÖñ)¥ú^©,cµË¯ë,·\ç1,ú(ãP_ [êƺzj]žõ¢n³ ¿šEó½LHôJZ¾û¶Èd»8†ð8ëÛÓ_·íxË©)ßþض£–qÍzʺړkí9TËJl#3ÏbA6¾ æ!nÚÎþ™æúÙG^–¶>÷~×tn¾-º(2¹eY‹M×<ÆìaiwÑ_󱥬 õ'½ØÌÎíõþ–Û²²|¼|3JÙ}»QYÆE†2žL“äsMÖS.”©øö<—:½¥öY÷f^ë6ÆÒ~ JùâkÜèìGóJanƒnWòìê.ŒºËsÔ¬íålÓaöÕ‚C~ÇîVñ-ý^ùú¤rÙŠº“£ÌéÐÊmêRïÀnlWæ⼊íé.mt×´U•ÝwãZ|%ûé¢ó•ŸíÍ›Ñ/ÿÊ´~œAÝ·]ßglmçñ¸æƒî&ÙvOGVKþU׬¢ç¸ÖI™oYÚ•‡w´Vò˜eÌý\+¸;¬6ªÉÛœöñmÍSëÉ:oJ¾?0VuUÞDSDRk”¬æéå¶Užž¶ ‰f#Üí0ùI%¾e5tX–±ÙìK?‰i?î·hlšœÇEZt/Gµ)æ;¼Þ‹òÕv~J6“jçÀ¬h¬7öPC&?:EÎÛ;¢oC8>ô± -Ù­œcnrÑ´¥Žžl¾‹Á;ºoÐ<ߡê|ÇyT›í ±æã4OÑ`–EBˆuß7]Z¬°Ûǘ¢ˆ˜D˜|Éw¼˜”äôîËz^Âö^p» ù™˜ÄwŸ‡Nó̾zäu0K+,Ñè¬ó$yzÅ>~^cûÔç~I’ê2£ú$¸Ð¨áXMâ_Ö>£>—m=Cçtc=#7;5câÞ÷œö{¾gûàxΪï¥e!~Y|oKSJ.=t¸çwŽoZÍ>ä¬)ñÍ–ýMÎsužÍïß¿ïwÍð·ß¤á­ ·üåß0_‹<þ¼¤\úóPæëòw¡³ðËï×åoêÊ~ùÞ+Ò[6¥îx™ÍcÔI/;WçI¼¬‰õž—ÝÛ²ÛlÏ}Q»{Äì0Ù›lçe æ·ÁjÒ"#gîÏÜo{N÷=³z›ÚÐ6Gº~M+2é+ëT«»÷îzj2{ê¹:Ïæ©^íâ¡ä!ºluÚ§¶˜±‹5Ã&û -»k»Á36㺺·µ|YŽâeféyÖ‰3i_Æuß]Ûá³Yê¤;3O·qÔqߺcçÖ=Ïôºâ!? Kügž¬ótžiw1tŒÔªözÏÚó’Í­Žû>?!Þ}„géV¾Îµî«yó/l±93)Gž=fß ÏÕy:/</VËt·UFëV«|éö[J¾²ã©#ö‰ž5Z²ÇÒú«wáÙ ¾ÌõWw¬ÕàQ›åã}YW8)ìmÑA éûxðäã'ë<çÄ£%ÃZÓïë,»YÉî|–¢ÿÚ-lpGZÙß#÷Ë~M øÞœ÷‘ìí;D7Ì -i§÷Öz<‡?Yçé”Îïøí{ã¹:Ïæq?Ž½w¿°Ç\¶Ì{òézWzǺ»™ÚøÀñˆ¿âþ£wÈ6Þ?<8}vg­< üöuIöÌø†xõd§ó̃xõƒWíñª‡ö?"6|¶èçŽ×v{£ü^t~ríðõŒSužÎköSß>o}À½*x#þÝóÖ1b)X:Œ³îb½1úúß¼ŒrpG4¾áÞýÉ:Oç¡û®'ïÕÈàÄ«E9+JóÝÄ ¾‹&µ~tÈóG/ßë= {ê™õÆñé‘>Þ_SZ=ë0¾RõÀÝÒØbNJnBÜIñèyº“užÍkÓAœªdoéW£74ó‰n~CÓ,V1~ßÊoyqMz -¦bìxÏÞoïTÜÛ‹#þÞÓ2ss¹!»gÕÑÒ¤ÇË“užÎóîÓùíf® –]ÝW×ñ‰”éóá{»å£Ç>²»~âéÒ¼µ;ÜÅ—!NÚ?±àÁçê<쟾#¦<ã]ì[®îoõ{a-SõŽ¯³äßwÜ=­ÏŸ2Þ2N߆qxÙ8KŽžMÝÞåÙŒèd§ó¾sñæѽv’²”;¯£¿9ëbÅ<œ{ÏL¾ã¥-VD 4Eö¨£§QNÖy6/…N%î‡iG›rH7F×Î.ÆiQPõaÌU Önж­;S`Öoƒ küa9[)8hÂñ¹Ò|JRêŽjª6=,r`ÂòÂÈöð™«üY¿S‚&ùPu¯–JÏ!»÷F ¿vnÊ+Ì~~}Ù.j,g>zay†Õ”·Pò† d´Pók¤ßš ò)Ëk¯ù•Ð´ìWsY¯Úk´Ëçñ5öÜ_>JAͯ•v¯ÇÖöË««<«×yÓ².å1lÌøËÅçóôt>SUyÄ$†]¿zšømð€Iù,>Kº˜MgEëëšù¨ú_j©åôzÅÚòa†óÑöÝÅRþ—lðŸ/:ÑÄúÏç:䓦U&GS.ùõÂ'“@&j‡C™¸|Tµ-Òõ¬‰B»¤Â”Æ’ÖMjùؤà×iæÆ•Ñp}Wªñ p¤J!›Ï@›;ˆ¨éõ–Ì£W“^È|Z{œìrå«(ü:h%¶©/ÛlÉÚªŽè¶DÕu×ȆŠ<šO÷¬ªóXEïU§;ó—²&tæÜ#Þ.èÖŽÔW‘ÊgD—¦Švª4Û •*Õ•’Õ Tí$nøƒ*MŒYùÕø¤Í™‹#Q-—J9t»ˆå–Ö\µT¨¥b«šŠæ›(ÍÕùÜû´[whK௟è戲åãšÌçŠT[«ØùÙoGu&6òëÅ¡´kX¡\q½žh»v+9ªæ/ÊÙÛü S³ÐøœBþ† -W©Œ<ùƒ¦êR|Öh»jðܬ䵒zº¯¨C’4…V¶S—¡úl¯/•PžA·–®[Q;•uôŠx R<Öd]½®’Š~*VöDU„@wìª+¿®ÈMNW§ªSµ3¯;wwµÝFö;_{r¨çã†Ê Ú:à;Ÿ/Ä[^Øh$³ëÇÒI#óQ?Å}jg± Ph䣯™\fkE'þ2Œò±M2iŽ]þ|ÉËœ±ÎðYÞj>½ºôõÿPæ'üþÏ…®¿+oÎë¸Q-LÅÖ¹_Éø[3âI2‹F!ÖiÓ u°ßÈ:¶²®xC]ÇéðšZuÕ€½”ÔAuåöD×tØ‘Mu]£m§·C¼ Z¬Ì{mÓìùÒ–ÁñÙ|dk‘ -ýpp϶%ùÚ5‰+.Ú–tpÉtqÑ7ŠË®ÃÔ$[‰ÉÙwRUÏòSo -Ë}u]Á—¶ñ¨-³}Ó…–GÑAlû‹¾aÒ}§i% ô›§×»í¾ˆìË `j…öJ}í©u`L­€'Sm¾ôë˺æØ|ìC™¼MCµ-ÛÍ=[ƒœ65ZhA«‹ÞG;jlÕ–®ï” -mÕiY(*\€¦ -­¶ŠL~©VÔ’GE-…S4êŸ{M»Þ*U‰®·Šß”åvËŠW£%[ ×»sh£S­d%£@ó}íJ¥VÒÚ¶oZ÷]VÛk7 j‚ø™ªVÍmJ¾ ÿÞ2wæ:Ã.„TªüäÛQÕX¨Iklþö1½†D Øm•<\gïd»CRöNúž`Ž‰¬UAó÷ÈÉ"¼‚‘Ž]Så;æäjI‰CzLÂÖPƒ9µÙÊ_$”÷óQ½ œ*ò÷†Yz#ÝIPZ>“·ôÝjû$ÿü ª‘Î -endstream -endobj -637 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 636 0 R -/Resources 4 0 R -/Annots [ 638 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -638 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 558.873057 101.433071 540.123057 ] -/BS << -/W 0 ->> -/Dest (cb110-1) ->> -endobj -639 0 obj -<< -/Filter /FlateDecode -/Length 4922 ->> -stream -xœí]M$¹q½×¯¨³€Mñû0ö @†´† ØÁ‡žÚm F¯€•þû~ÁL’‘ÌȪ¬êžÙZAÓèéÊ~ƒAfSŸ~¾Ñø/9=¥”cpç˧ŸNjŠ^á_I±º(ê?â7ú³V.N^çüùGØ8ÅìM ç·þ.yò·Óÿêô×u}6SÎÁhþÛ¤rvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS€Óƒ¤3z -)›,ê"t:Igí¢Ó^Kº<…|’t¶ø¬´¤ó°Åk甤ƒ-Φ(él±Á*Q[LŠVÒEØbà,èlº’¨ƒ-*(-é²|NΉº J1muFùÉ'k•¤ÓzòÀ¦uQè¤-Aù,ê`‹·ÊH: [~$xÂÛ˜’0Z n«¬–t¶¨ƒ-è|‰L€-*‰0‚&â%‰LŒ“ËÆI$`’\rIb“òäb´ ˜ì'r”xÀ* ÑX[¼ X [F®¨ƒ-6;‰ ¬-H qµ°Å8+qÛ: Eâ ºu*k‰ ,øÖ)$.° \˜©$.°`\›‚“¸À‚r¥,qçZp§Ä¤‹á%.°`]냑¸À‚v­KAâ ÞµN+‰ x×Zäu°–¸ì8Y¬Ä¼kµŠ8ð.†‘‘¸½:‡Aâ`e2))‰ x×$å$.pà]1ˆº8a Z‰ x×x4Aš¹Á»Æ+#qïg‚ÄÊþQb4Þ…x9€- X‰ 0ßCg¬Ä(l2K$.ðà]1ý cl5锃ļ«ºOÔåIGx\·ðò¤CÈxð.&*+qï‚>“Äàø 3ª‘¸ÀƒwµÅL-éÀ»Úd%qï"nð“F qæ·I+@WÔåIå”$.ðà]Ð’‘¸³ð¤˜DÔÅ @Ò€Œ'…8Tâ‚ÞÅôž%.à]å­•¸ €w•CØ#ê`‹MFâÄ5ÐaØ -cÁê¤ ˜BÒw•FK:ð®R1K\À»4mJ\" L’¸Nˆ‚Ä!¹)Ç$.YA‡OÔ… ×^₨ ð ,Q[\t °ž²ÍIâ‚h`‹5Fâ‚h` B‰ LY#ú—°p׶ R”¸L6¥ìœÄѬÇàUIÌ”"èZÔ%èt”¸ F7a*Ö`-5% z‰ @ð…KÀ è´“¸ fØ‚¡"q¦¢)Dæ’NÃÙ Ò…Nk‰ ’-ˆ%.H&a•”Ę¡±–„«%S´ÎL$& -%.À0ÁJL)ê` hBâô-¶½ÄXü`±‰eŽ¨ƒ-Ö;‰ R2´ÜL¤[Œ2›´ÞŒd[ñI\U˜(L–¸4Rd°Þ4NâŒKZo&‰ ²UXob‰'ŒÛl­7£ÄÙÁEÔÁMY·à$¬7ÁN’.À“Ä9Zo&‰ f`½‰ˆ^ÜH´ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^x}D|&+­8“¸c @½s«¸e À½˜­¢¸g @¾´F7 `&V–)ˆ»˜V¡ÔJÜ6Pà_Äcnà óîߢ -€CÄœ™"52cX8D÷mÍÑ“ÙdóþØßþ|úõ &ßóŸÿ~zýUÙ)«?BIí_a˜Œîöˆ¾5šoQ4HC•ÅBøH2"æn¤-.›nd$¦?úwÞ~XYÃÉtäƺ(Ð^F4°¹~0Q)ÝaQŒõ¡9³›|wž›&GDùÜ>hÙÖî*^o÷•dëÝy>ÊVÛº5SÒR-`ki‚v@ÂY> -‹ND1il6Äúʈו[%þͧ`±bÞ[n—Ì|Ä!„nüùÓ§_ÿþ»ïþõwÿ ?}z=ýé_”òáÛo4ýMó_wY®ãr½Èk:C±,Úh^ñ3¼3‹<á׆tŸçkIGyŒ[Ê^ôö²ËRîg–Ïõü¼.Ä#ˆÄñËÓ¾Ìu`Âîå†Å–¥œ’eiL‰êõƒÚñ?çOß~ûi ™Ÿß‡²cYÞƒ²/.,ö¯€ëáfuW‡ˆÈ?®ƒ´c°c¼.í±¬|¶f±í²šl·ßŠp¢û ÷âé`žgÖ|×E¨x¤g­ÿeSÏûìßA•m”ãôâ2†p UÇò<ªÜ!žú¹xçîÞý0¾YFýUô»ƒQϸLkŽëH9–çér=\já‘î!Q ‘ÜJy¿Ö“‡½]tf¦†[^Q¯ôAþ¥WkÞjÃJWmˆë4-|ÓÝ>)¤ó<ݬÕΗ.mÛ©¿¶Kj»{l­þ»,eg¦{]Û^üdvÐÊB),Ž²J†å×Ñz,ÏÓ¡õFü•Ö½P=L5´^®=7 µ!S å *å})$èþ¹Õ¡ÖùVˆÜi³“êÚÌç?W” é^Xº°®¯ùír›WMÑí,xUW‘z0ϳ!Õ\ën¡qÿM­‡G1NÓH¹3oÙ8Œ„Š”¯9âò†ö©k‘=¾óô°B‰pk¦?˜çéy#&Ü™uÇY¿õZÞÊwÙ!xëkŽ€ýåÁìðÌót¾«~‚wÐr²ÇٶŔŸY¬÷>3> ÂwÐÛãPz¨Ç9,Õn­Éæy:ôÞˆ]oŽÿCøWCè=£æZƨDðIÓ…¡M¢½2íûŽó<íøtŒöÇÛ&C/•5uE쀄Ÿ¯—Y}µì—…ËؾÉ~9»ƒ\{ÿÀÁ3DŽ~ÞDH;H}à.÷Á½žÅ§GïØÌólèõ7ž {kXµHO¢Œ+º÷ÙP1r*„sXE›Ù)ó^Äß²ýÎ'nD›–à¾g#•¡û±ç¼vPúÀsó<Jo˜çé|=ƵÂzvsOè){v¸Ç¶Ç…µ¬K-»óº¹/çXz¨ˆQEÆ»ßãw%®ŒÆR§Ýæ]1Œévyæy:Lüæ{² ~ׯaÇ•Qoôu‹Gü\&1»Ä”›/æ_éç½{É{ßY‘îJÑg?DÅ×f£1ž‘0ÝÒ|¿Ž‡ -V‰3 -o§Œ_öl¬¦û4ü| ¿Çò<~oìí -ø]ùùÊw°Ú“.y+<6¿s ¶}ô…{{)Ža·q…}ãRÞk³|;j†z€P¬'Ú%;‡…Ž! Új“JOüöÿñoÿõ¶OI;4´¦k÷Z–;€aö„iFkC;ywNë‰}>×–Í×y<œzÙ.ÌåÍÏI-ù.+/4o½gº<§uq)Ó×;’Ûü®¦}™½zÈ»ñOµ…~îçÎEëU”V„Ž¯¾ÁÒ¸ušrm…–.×¥‹—«WUùÞ5\Y>µNOžR—ÅaizàmNŠNì²tbóÒf»ô¶'¯R¯syúûm9ÁÒqftd´knsi÷5;0qïØ¡z[Vþ(¨™yA°ŽNÊ™…² êóœŸÚWÚHe½p4wŸ›_øŽ¼yϯûÚ/£ÉÅ¥ÜÅ/KZÁþ†½ô™­¨^d ÍK™lD‚_®ùÕm7‹-û#qÖ›úKõúu¶C/m°Ìo5>(éYÿ­ƒùØÍóƒ¹ÔñÃ|û犫âãØÓúZOXl«6†+>ÚQXË\õgýÙ0êNå1¬”;:u¼jfs{±§ËÔ5_ãÙWz¹gXÒª¹Ê®}0öC³)0{«ßu÷;æÛk~Ê? øn6ïàa={ÜÆëzƸ5[ȸn|å™Ï’à‡eŒ‹þ«e,òêo—¯ù«œŽ¸òW+—Ͳ›q÷ýÏã+£®¶…NP\sŽ[ðË8 röA#‚:r9FûP}0ÏC‡»}`èìû)VnJÞchË7– .ãÒY/”aY6wÂn…9L*ÓÉC…u -ä7¨{ë-ÿÝÍ“¸ˆB‰þõVn—ʉâ!ˆöÜÊíÔ=úLÊ͈n {ÓC¦¼ªÃù`w<ôµ¹ÙzgË©¹t‚螧³ãáºåvÞÊMý.´æã<ï’Á(KáÙ>6}^záj ?.qFZ<½„„ŽÒÌì\ÏÆ7“Ü ‰º4ÛžùxÖÃiD Q$½Ç&ŒƒÉåH‡ÖJ«5Š{)Þ45. -Ë78–ø’œÒVt‹ƒŒ_sÙs¬ó¸Ÿã‡êÔ§-so™Gò,7-Ng1HP›øµÔ·Ä¡|ÎX•_çOfÏjþÌËó§¥ ßŽÝøÓ)”×u9€_…4iCoè9cF!fÌg:ð9)ëtí6]­Ï;9Ë{~j©–W)zû|]º•_,é*þ#–˜ÒÔ̪ókITÅ~¿èµ4Nc™nŠI¸|T­,§Û"¶Ê.˜ã”Ç”˜jÔò±¯tiþQ —ÖÖP~_³ÑqÁNÕD¶˜;W,‘3è­˜Z¯&½ˆéÕ>i²ËUh¦—7a/Ðñ´½Àžñ¦X[Å„~+T¬º.6®Ú£é(øæºÐ…Ítî:ͺ¿¦5‘uç5áå„jí(}¥ôB‘ZTõ®QMf™Q¹I}MÙ:ĵ~D—“ üFÕ¢:³éY¯Ñ±ì³0U ¹–.×t‚èrÓ-¥ùÖS±¥J=k®žï¦t¨Ó‚_va»A‘ÞUÆƈ²õãZL -­¯Ub¸1áå„ØmÄo'…Ô¾s…òz\hY¹MœTÇK—Ò ‰ó' Í*£C­éÿ™*|“ó”Ú5HÑÁôýªWBc³‰×NâòÐXÇIÐ.ö´Ì]:ƆYî/”žH·¥n[I™Ë˜<æj‘T#µX7Ô51QXõOãJ.TÕTמAùm%îvú6T½j•Íàî[¹]w¾qql=®é-ÍÞV.Ã|^ÊÄæF1A?ÕJºØ¨6Ûõʘ°RèâN£oE\Gkc§r¨Üò±2iƽœþr*Óœ±ÞÐB- Žë#­õùÿæ;üþïÉÿ½žÒ>ÏãF5¶0[Wb>“Ñ+ÖÒA1½r­ -Û°aBãíw±N=­¯hhó¸1Œ¯]Ï®:±×”º1¨nZ.ô݇Llt¶Ì3þŠð2x±2ïí›>aÏ—¶6Žžt¾µ -¼9¸L]gû”|fE⊒ö)ZgXF\ðBqÉ*Ìݲ•™H^Is=ÙïxWXª‹]°0‚.m×¹>Íò¢«¬´‚%Alù/Øi^i^Ù#ýððzw¿/&‡:HÚ¨½I߸´5Œ¤ðd©-—a}Ùæ{¦‹:x»‡ZY–=Û‚œ>4zháV£Lšz¶%畺*[UZ'ŠF©*k%ÆjSX²U·”'ÑK±Å]úîiÏ{¥9Ñó^ ›´TnñZ´d[ä9œcoê)[,™Y๛Ôõ”¶Ê¶u»uÝu¶=³f¸nHè‘©êÙü&åCü÷ÈØ™ó »RªúSvFtêï5ÁDít´Æ–WÕª8ÄÑÛÆ¢Kl«äî<×÷N¶;$uï„ׄµ*jzé°lÂù¶‚U_H,gËJlÒ}¶†:Í©ÍÎPyëdyN£lµËHKEzɬ¥ 7Ñ»ñ¢Òò}zü ¬¶OÊÏÿ´î -endstream -endobj -640 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 639 0 R -/Resources 4 0 R -/Annots [ 641 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -641 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 108.873057 101.433071 90.123057 ] -/BS << -/W 0 ->> -/Dest (cb113-1) ->> -endobj -642 0 obj -<< -/Filter /FlateDecode -/Length 4695 ->> -stream -xœÝ]Kä8r¾ç¯Èó£å#ø{X` { />tgwí¨10ëƒÿ¾¿Ä‡¨PJ™US«Þ.TW*‚ùñc0ÄõUáçÿ"é!Æ<]o?_~¹¨!8…cŠÅŨþ ¿Á]µ¢08|tןY`Ã’3Á__/ø¯»l“¿^þã7—ÿYæÑW3¤äv׿~cÙ!Ä”T *¥­ rnÁ8/¨¼p‹)I*˜áZ«Ì ¤‚dLTfX -Ú®U f˜ IRÁ \TZÁm”u0DY#è´<ÝK:£“I¢.@§ƒ‘tÖ>vZÒ¥Á{#Øâ’Ò’ÎÁ§‰”¤ƒ-dct¶Xo•¨ƒ-&+él1h$è"lº¢¨ƒ-Ê+-é’\ŠD¢.A§bˆkQnpÑZ%é´°iEF:ÉAg`‹W.‰:Øâ¬2’ΤO8bF«áÑm•Õ’ÎÁc‰:؂ΗÀxØ¢‚—(#h`^’8À„0P2$‘€‰v HQbÓ@!X‰Lrù$°JCg´DVÁG^b«a aäŠ:ØbI\` lA‰ ¬…-†¬ÄlK@‹ÄtK*i‰ ,ø–”öX.ÌTX0®ž$.° \@)I\`Á¹Ü)qéb¸‰ ,X×:o$.° ]KÑK\`Á»–´’¸€À»Ö"§¨ƒ-°Ä`ÇÁêh%. ð®Õ*H\@à] ##qzu½ÄÀÊ`bTx×DEx×L¢. ƒVâï‡[fnð®qÊH\@à]CÆK\  à%zà]Pˆ“ÝØ‚•¸ó=tÆJ\€Â·DâÞÕ ÓŸ0¦ÁVƒŽÉK\àÀ»:¢ûD]t@‹ ã­ö×?_~û“ïõÏÿ{yùÍ)Ë?˜š€g “ÐÝÞ·Æí[æh {$-WR5Jƒç …éŽÿ]×–ÂП–ÕSB½à…$Úz4Ó;½nö?|º€_¦Ø¦ç²±´óÎÀ 3` ÀèÓÏ—ßþñOÿúÏÿþ'àâÓËå?ÐG¥Üü‚b Ÿ Ó&“és`‰c ÓµcË_8ÍïÿëúéǶNÃËìÄí²N®F’ÿñÓØt¶ts€—4¶œ‰`Ð\ DhyÚjZûÁ,oi빉q pbpXè&}ŒdÜ¢U»S®|jŠr©­+A‡hºqc7üÓ?þÃ?þ[íõm¤¦_Š¿ÿACæ4wÇüÙÍoÜUóç0ý¥Û|mj>.§¤s>Õ<”ˈ˴£~].µy?Ov3DŠ s–õõ—:|w­ç{É)àÞÄ´vö¨ `ÃXçfë[PóÐ æ8Z³4Fprpʺ -ï!Zó‚a§³< (ñÄpÒóœÓ°å¦W¸3KÜr/À§ÅjnþË¿aºuüÙκÏ]š¯óßYn̬Ͽ_ç¿©IûeÆî·Ù®Ôáu 4§±Ý˜ë±ìš1ÖÉF{TW^ìÆámÙFZÉ(3ÊVĨc(;˜ç$(«fÃÂ;(£—%ºhfR§æëÜÒ_šÞÊ­üµöfD“mPÖ3rî%¿ìµ1Í· ·Ž-_–ú±>³]ö¨ÛAfA›ÊéØ–ËËŸK;¼¯klSÝ´sèõH5ñq¤Ës:¤Ò]>”¢uÇ+Y§Öœq—knKô,¸«ëm×!c…˜Ö¸-ííÈÈÈ#KO£NIóý¼î›kÛ}6sž´1òÞä m Ó>äÈ= ð{È<–çtÈôw9ô#<ÅÓñÝ{ K×ôe4¨e]Í(¬¾93(Gžs…Çòœ…ÇüÅÒ3Ô a‹³¾T]î{Hí¹ODVß“-—>P_®ctE¿Lù§•É²üÞ/në³qÛÖO涻5÷±ãCÚÖä§qŒLßEÞÁ4ºóJ~õQ»ÞÆGÔfH*ŽùÞGï±<§CïŽ_™[ðeù·ðÊç®g›UË=¾kÑòQkä»q«/³Ï,ø´ÅÓècJ=OYÉmÙ›óì®ê6[}Çãˆ=–çtˆ½ïožÆ|ëš÷‘ÕÛx7ŒŠEzÚáÚðøþ`žÓ!÷¾ú÷Ñy7ÎþHtg´îŽ>ͤ_÷Ìs64Ó}¿wåîxÒì\sgd´Ñ¡ÜÃVn­ÜJOÊßKPzÜtÇòœ÷ýØ­ç…-çrϼe=]wßéÝ»+µì#w<¹z²Ð=‹•"d+ôg/ã¥AÚÈÚ˜îMóÒ2ŸðWæ92wüÕwžµßUÅßÃ7<›÷³Ú&6Ê{8MÔÆí¡öXžÓ¡v'žúVßðaëžUQ§ëùo ­½Ç’¹´»ÏÅzÒûê/£s/–êžxv0ÏÙêv⮟Ϡ›!{$Ñ!3¡ŒŸ&®ø%÷XF4Íéûò32:„–z:T÷»6ÏØü(ûÝ#­¿[Ú²·ík“¥®úœ”Ü¿“âÞV½ƒyN‡Ú?UÉhigB«ï§Q/“ïjøwîãï÷òö¡5ÿÝÛS8¶fï×O*¶b±¢'Ð>{šG[]ö£jï ©sOðå±<§CÞÎsún}»ëBÏ.žö³k¿#¥yjhÚõðV´¼Gì#Ñuºƒ¤ï6¢¶àÆ¿ÄGm)ø½ˆÓÁ<§CðNüô >åt=·\<ßjcaiSAÇ× ãïžž–ý§Ì·ÌÓ·î>wPÖ’½½¢¾‰]]Ìs6ôùcþæÞ³v’V)½ï$EÑñÇ_ñûã¹·Œä ”V_ŽÐQ{»Qæ9JwüË£«Žö)e·Úè‘="ŒWEA•½˜ ?/,QP£ÖMOw\ÖFÁ÷vkx÷ø^£ƒyN×»}¸ØµêÆXW¸7Ã+_gŽdóŠÁ°îËt-é8O^ad½eïü6—û¥ÉG5[Wù–E›öóT/J¹~¶%sç‰òÆÏ{܇Œ¯öióQ|Ës:|íxXÏöqÓWõ±ŸpÁ¬òn}|ä>:;úëüm¡RçË̶ÝfL+^™±Â»#æ9¢ÂÎSâø\Ïr|ã{fŸ·Ù¿*óøJð`žÓ¡jg×âߘwîÝwã›yDíøÃቨ×Á<§CÊÎÓ×&®ø«ùÆ;´û}QGâ²Ò7K$¯®}–TV\ML!ïºqzöqᇞœäxí{[Cx|7öÁ<§CëŽÿõkø_ î-8™{`wÏŸèÓ}Vëýý3³E$MFjl"[G¿ou0ÏÙw¢aOÇÕó»IÞ1òöýEÍ6ùÄÀƒyN‡ÈcßdyÓnäGž(|ä¾ýýwƒà'¢{óœÁ;¾ê;í§ó»Þù;T›6òÏ‚ð ô6OZ~à`žÓ¡wÇwýŽ¿å÷.£æCw-n¼퉸ãÁ’Ôäí÷óŸá­é‰çÜóœ©;¾êgÁw̭ؼ­ì£ßØcØuï{Y}ž{1ÔGÞÓÅ÷ôë®7Þ‡ªžphf:fµzÛÄó¹zowfN»`<þú€Ÿæ7š‡žw|š¿î;5/H/ ÞM²ÙõëÔóKʯü³ýÆoÉGÚx5@f -üöÿ­w#›oÓ,:¾)×ϳã¼#0wJkUÞÆhòû”Ç'š§º±kÕøneúV¹e|·²ŸÒó»•Î#/rZ?ý–—‰û¹œ˜»f®9”óóó§ùeä‹òÝ\wcÍïy.÷àò=¬ºñ—‹O±ÐãIFŠ="ÃG^= üØ?¤+Ÿœ•%»M ­¯9Ç")©æ3##ÅGˆLJ6sú_Æÿù¢ ÜþÌ&ÓùnjG“/ªøõÂçû‘ &ÌA1 窔E:',²X9 ©OiÝ æ©Î/†ž>—ï†ó»œÏ] •Ùñä©bAˆœ^¯Å|÷jг˜ÏHŒƒ¯|1Åùl3„µ@jÓÖkÆ]±¶ªºµP5ÕU±¡læ3uJÓù*,¦·M§›îÏiMhºóžðvAµ¶—¾ŠR>™-•[ר"³Q©H]NY:„J? ¢ÛEþ rÑ„Î,ú¦×ø|›I3¨¤K9 º]Ätsi®ôT(©bÍšrËWS*Ôm, /ÈÆ6š¥}mÆÜ“û¡{îÜ×*6¸1+áí⨌Ä*~½¸ ¤v•+”ËÐk…¶)·ˆ£ªx©RÞ}6}ÂÐÌ2>„ÿŸ¨Â)3ÏøASŸðS¯j%<6‹xÙH­ÜÖ!©4…š¶i.BÁlÛ^:*!=“nI]Zl!mš¬‘‡”Íc’*ä±낺"f -ËíS¸²ªlª«bW üºW;]ª{®Ìëî®”[žpçk+¥Ç5×Tì-å6˜ÏÂÛ8±Q/fèÇ\IUf»ZY#¬†ª¸Òèë(Σµ°A›?ÖA&͸·Ë_.ã4g¬3 ¼Õ|ú`ëuéëÿ!Íøýï ]ÿ%w3ÍãF¶0…[âv&ã³jãA1Ÿ]›…eØ4BC íW±Ž5­Ëh(ó¸1 _SÍ®*±ç”º0¨.ÚVèj6bS k´mZÆÝÞºV,ŽÌ[û¦NØÓ¥Í7‡j,””­ÂE{;¸ŒUgë”|mŠÄ'­S:´dšŒ¸h ÅeSaª–-Ìd@¶•”¦gû©í -Ëu5Á—¶ê¨N³mÑY6ÞE“.°m/Ú‚I·•¦…=‚ÑO¯7÷ûl²Ï#€¥…Ú‹ôµ•–ci!> -endobj -644 0 obj -<< -/Filter /FlateDecode -/Length 5008 ->> -stream -xœå]I%¹q¾¿_ñÎ&Å%¸†dØcX°à||è~Ý%Á¨10òÁß_0“K2#—ª®i?ÀÝxU/#¸ƒÁ/‚K²ô]áÿ?"é)Æ<Ý?ß~¹©)8…9Åê!³ÿˆOpw­(LN'Ýýg&Ø0…äLð÷×~ }ò×Û¿ÿæö_ë<ún¦”¼Ñîþ·¯Ì";…‚r+¥´XÎM!ç–ך˜’Ä‚.¡-+@ JÑI,ˆAÆD!†¥ í–• † š$ÄÐÉ¥äÐF)‘A‘5O»ÉCé^â=ù˜Ly<ŒÄ³vò´Ó/MÞ{%A—”–x²8M¤$d!ƒÄóÅz«Dd11X‰ ‹v’À‹ÖEdQ^i‰—ìäR$y <CÜòŒr“‹Ö*‰§õä`›VäaD¡“|x²xå’ȃ,Î*#ñ,d!ü—xÀ gCŒÂh5<º­²Zâ9ÈbL ‘YÐùYTð`MŒK˜&J†$0ÑN)J(`bš(+Á€In"Ÿ‚„ViðŒ–€À*ÈâÈKH`5d!Œ\‘Yl" ¬,H aµÅ•°Àm Ö"aÜ’JZ ¼%¥½„€ 1•„ˆk£' , ¦”$,°À\ ì”°Àt1܃„¨k7XÀ®¥è%,°À]KZIX@À]k‘SäA°„@ÇÉêh%, à®Õ*HX@À] ##azu‚z `+“‰QIX@À]IX@À]àD^˜0­„Ü5M<7p×8e$, à®!ã%,€¡LÀ%FÀ]@ˆ“ÃÈ€•°þ‘—& qaÜBË“ö>IXà€»pTVÂÜ|F €ñ<ª‘°Àwµ…§–xÀ]m’’°Àw78 `“F aüÛ¤LWä¥I¥%,pÀ]À’‘°^xRH"òÂCÒŒ'…8TÂÜ…{Oxà®rÖJX໊öˆ<Èb£‘°q x¶Â˜F°:)¤xÀ]¥Ñø«THxà.»M |`‚‰@À Q°‘°ÀGšR^ŸxùDžŸðì$, -Èy…IX€ÀzJ6E ‚,Ö ‚,$,@D0%è[œ@˜‹8? È‚HQ ّ„ÁyÌÇ U‰çÍàZäEðt° šàŠµ„˜KMƒ^ÂüÄá’„µr ˈBÛáSè¤×iþ­l×®"ÓËR¶oõŽí‘d£¾ý/ëôY.Óåk=ió»ÿ¸ÿôãí÷?­ *Çì³Aq‹8Ñ÷Яeùü5L/¥#Ó£¯‹Úõ‘Ûé-¨8ÝiÚ÷jÊ/wVWîêb~‹É–n?Më¡ŒÂWó﫦Y‡dIœÂp+í¬õ>~=3æ½j”l8”8þ8´ã‹yžÍ1ÿügÉ{Öô ï½:=—SÒWú¨‡>ÿ¨³Ï³ÕR7ÆQTäRvÇzmƒTÞŒƒñ§Ö{-ÏÓY/â𦷊e…Ázz·ø²î=v×=nm0êsÙ†k3;e¾ÕâÏd7ÐÎeЗn¤vÖ-Ž€Óºv¬´ >/[éµwR°¡`Â`IÅZ³eÒT®<ô;ƒâM½£•Äf-+kûU¼÷Gû§KíBTE“щÎ,õZž§³Ôã¸ö[½÷÷¶ºò½Ž¬Ñ -õE]yUV·Ö‘Øž°.sŒfª¾C{õ>®š>½8»˜çÙ¬ÕÇ®µç^ÖV’'àÎb|Ã[< =3`Å ŒVk^:‹îbÐj¹.Ï`!êec‹ÕX§vÊÈⲬÛˆ3ïmYæе³„âºU~¿cÁöí«ó<ǯ#~ËI«¥$ßYRÑü€#ïê¥Ðáç`©›å­1ìðª·ÂEé ;Y9hÙÌöêZF€ùlµ^+ŽîÒ þD½ìX²{_Ëót–|ãZa>[ñç©{vÁÙÿG,,e Xj©Kc†öQW‡êbTQ†0Ô=àùÑhÌuÚmÞÒÈ6ýŽ5Þ‹yžÎ¦OVyÌ>=ë5t¶ÞÇQOúºÆ#n.“‘]Bʱþ£~ÞøÿqŒŒ6#øxã'pCT|äÆxF²éšæË:ʶ:èKô(};Eûµ-Ö%ÍÇ5ô|b¿ó<›ýÚ“µ]Á~Wzqº‹]s$Õz}µ·•>6lêJlû¾•&µ»–BíÖ10®¯,åäõWq¬zùº1Ï=ƒP]OÔÇn ÛòKÚjsOüþÿúÏúC·NÉ+4<§«{-Ëö_˜5aª†‡ÑZ­µ;§uŒ>ŸKËægeg~ÉO~ΓW«KžÇª¥µ2ßu¼4§¥°”麘Í.uvòäg©œå9—µÈÉyè…åš5}Iµ±WíŸK놖P‘úÓVk´hHñJnﯻÈשAJÎS¤ô‹x¾—‹Ï\ûdù½”E.»ÔçX¾ÒR”£¿lËñ–mñ‹ÛrJ¯dy|Öþ‘ü.‹,‡jmYõB¶„y< ²ñ "^‘ñI(;[kšósûr¹¬O½u6ë0_]¸Îšôò=4}eK -K¹‹þŠü3îÈòW Ô­ŸgûXh+»-÷P¯ql»YdÙ]3ßìÔŸë,Ï/³ziƒy´öT>:Ý ùŸK[çøvùøõÈ,úÍéÜü»æý:ë¸Ô‘ó»åûRNikí/·¤ûÜéþÐ>õ8Ú]Ôë­d¤ÒúP~gÑ9mTíSZì³ÔµØõ]Û;y¤ñ¢ø…‡`Ê_íáåײlŒ¤îŽOϧˆØ‹o}0÷€5Jy{žw²k#Û0Ê.NÑ­ÚŠZÛ;ËnÛ†Õ2AŸ÷Óàjú.$?‘±ù%ØñDÿlAçbžÿkkòò!z²Fùã™°d–Ãúó®ѾäïWåüƼ·IÝ#‡B–_ 9Vÿ•Ï£üè¦Ä¦lv0bÇ.ùE.qäwX¦Üȳ]÷kYžF1_Eµ)‚çf­9 à2ÒËy†/²bÚ‹°1ð,üNPAÙ³1Ïó¨Öz~›o¡8RmùŒKéu©©lÚu~$OV6…‰Qð§áÉqÒ -OB0S°A}=<¹˜çùÂ~Ñ09ï£=0ƒeƶ[œã˜]EL«ÖÅåµ´n%ïªZsV€0çpZ(IQuõ¤wÛ›¦.ôØC†ÃpåàÄUæ+¨ÅÄ/ìñ–ò7aŒº ×w:ý»…0{NùF˯ŽÛpº/p1ÏÓ bÔŒâ2î̸ånÛ,”÷›PÃ&AЮ‹ªù~ZºK~Äà«7|ªÃú ~äZžçé¾0%Hì^E,;„¼RÈ«µvÙ™a´á•@FœzRÉti;:ã§5Ÿº4…¶ìÞu‚Xb뤭WƒØ=Ý2ó?zQ껞ßìV6'÷^XŠ“3ó WGýÅ<ÏcpޛÊ$ú£ùëãÚ¶–|t#^­»@¹.ùN¶äÌi„tœô£"¤˜Ð©NyŠwâW“#ß(t&Ùµ<Ï!apO^øʲ£™1-«sö+Îâ¾:@+ÖÀºø~ÅûJ”$䦳܉0"ùŽ–÷Ôm†JR Ce›{ÞG>+Á¨Ióµ%ñ]õ›|ÙƤ”Ûå¶|u‚”›wϾ~_ÿâÏü 9Ê+²²¥œµ7¿Tt-Åæ¥i|ÎJá ä £ûüë…^s|-‹Nì5)ÞíN¬P„擄xæ{®ÎÌ/æyo”òå -&:ùPÊ<#¿>%[×û³~]7ûªë°³)ó\¢v¶¹Ý…Õƒ‰é6ßž¡l¼;ÃûÚù3«¸”åyŒ‚Ï;$çÜÎ>Ó[bUÜ*Èé7!ÂÞ4$ñ½FÑ¿»Å· æ[wŽ~)Ëó(œoGsÑÐÎÑ°²H°óz´‚“Sš™g!ÛIÚoPÈ—›P‚»ó^²›<_i’º‹Uª<§INÄ(§½¶×oÜõ£±þÛÞ?Šä+1 -Ò”ß$c'lçœüâˆËpê‰-¡˜)Ç.–½ðŒsìðʬ¯ª£«G–cGÜ¿L7õ‘jG“¼Ú­Éõ-Gdú=ôUùNµ£6ÝÞ}=2’–û,r*Ò•nüåæóíW:߀¨|œ´á+’ïþû%ÝùÆ­¨,éÒm:+Zßwræ‹¥©©ø'HŠ¯Ÿä«Çr·öKú_r‡ÿ|cLgýçófù^X•ÉÑ”‡F~½ñ½À$‰Ú¡µ2—¯ª–Eºž¤(´X˜Ò˜ÒºI-_ÛA9ÛN^¸ÒÎïJ6¾¯‰TIdóEsÅ9½Þ’¹õ<Êg2ß­'»<ù*ŠóEf[Ô§m¶Œ§dÞ‚lD·%ª®ºF6TäÑ|_UoÄ*z¯:ÝuIkB×GÄÇ㵑ú*RùF×RTÑ.¦…f;¡R¥º’²vÕ~H›@üA•¢átåw½Æ÷âÍÄX ‰jºTÒ ¤ÇML·”æjO…š*¶¬©h¾‰ÒLÝƪxö蓾,¾#Ê–¯k2Ÿu¬}Y}³³!>n˜ÊІüzsAHíV(WL¯'Ú®ÜJŽªÙK£†XT…¡Yh|«ÿœ¡ÂU*#Oþ‡\²ñÍ€í©UÂc³’×Jêé¾¢IÐZÚN]˜$W›íõ¥£Ò3èÖÔUc+j§²ŽRAª‚Çš¬«ÕU2CXÑOÅÊž¨Š¨®‘]5å×¹ÉéêPuªVæugî®–ÛȾbçkOµÇ5_óXå­åv6_ˆìØh$³éÇRI#óŠH1ŸZYGl‚¹Áèk&—ÑZÑ)¿i¾|mƒLò¸Û_oÙÍë8ˆD¯ùÖâ>êÒ÷ÿAšñùÏÝÿ¥\“7ûq£*Z˜Š­+rïÉøŽûx‘ÌwÞb6ÑPû¬cKëŠ5T?nL‡×Ô²«ì%¥®ª+·'º¦ÃŽlªéòa̦w@| Z¬Ì·öMsØó£-ãå,¾`±H…‡¾9xŒg›K¾wE≓6—.™.#úBñØU˜šd+1Ù ûJªêY~ê»Âr]ÝCFð£m> -endobj -646 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 416.373057 101.433071 397.623057 ] -/BS << -/W 0 ->> -/Dest (cb116-1) ->> -endobj -647 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 397.426036 99.933071 378.676036 ] -/BS << -/W 0 ->> -/Dest (cb116-2) ->> -endobj -648 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 378.479014 99.933071 359.729014 ] -/BS << -/W 0 ->> -/Dest (cb116-3) ->> -endobj -649 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 340.584971 99.933071 321.834971 ] -/BS << -/W 0 ->> -/Dest (cb116-4) ->> -endobj -650 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 321.637950 99.933071 302.887950 ] -/BS << -/W 0 ->> -/Dest (cb116-5) ->> -endobj -651 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 302.690928 99.933071 283.940928 ] -/BS << -/W 0 ->> -/Dest (cb116-6) ->> -endobj -652 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 283.743907 99.933071 264.993907 ] -/BS << -/W 0 ->> -/Dest (cb116-7) ->> -endobj -653 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 264.796885 99.933071 246.046885 ] -/BS << -/W 0 ->> -/Dest (cb116-8) ->> -endobj -654 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 226.902842 99.933071 208.152842 ] -/BS << -/W 0 ->> -/Dest (cb116-9) ->> -endobj -655 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 189.008799 99.933071 170.258799 ] -/BS << -/W 0 ->> -/Dest (cb116-10) ->> -endobj -656 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 170.061778 99.933071 151.311778 ] -/BS << -/W 0 ->> -/Dest (cb116-11) ->> -endobj -657 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 151.114756 99.933071 132.364756 ] -/BS << -/W 0 ->> -/Dest (cb116-12) ->> -endobj -658 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 132.167735 99.933071 113.417735 ] -/BS << -/W 0 ->> -/Dest (cb116-13) ->> -endobj -659 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 113.220714 99.933071 94.470714 ] -/BS << -/W 0 ->> -/Dest (cb116-14) ->> -endobj -660 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 94.273692 99.933071 75.523692 ] -/BS << -/W 0 ->> -/Dest (cb116-15) ->> -endobj -661 0 obj -<< -/Filter /FlateDecode -/Length 4427 ->> -stream -xœÍ\K$I ¾×¯¨3ÒaÇKB{@Á"ˆVzŠiêEZ8ð÷ù™ñÈ*WeNOô´º§ÒŽ‡Ãa¶#3Ë-~¾qø“Ù™œKŠ|¾üpúñdM -ÿj‹ÍEeÿ¿)œåd‚+1‡óB dR >ÅóË ®.çæ/§?ÿäôÏmwö¦”è]8ÿë“°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ,±1BbÏ·¬1¸ä ± {ŸV†ÄÉÑ-«@ Ÿk,ˆáJH -ËYÈá¼µ*‚Xfò -Ï¡ô¨ñ¼31_T^Ï%¯ñˆLLì‚ÓxÅÄCÖx YB±NãȳÕx…)'! E²*²øœHã%Èâ¡¢ð2due•Yl´Nã2¡df•WÀ³9å[ž·Á„Ld5žs&À6IåÁ£°I1)<Y¢ EåA–@Ök<‚,Œœ”rV¼Õ‹w“%§ñdñ>±Êƒ,Ø| |„,6E àAFpIß’áâYŸÉp次€ÏÅpJ¤Á€/Áp,Iò<ï4 YG ÈA†çª<ÈB…5, YÐ@Ã"Èâ™4,  -ÃZ4, À-Ûâ4, à-[5, .Ä´—rd  S*0—€@îž4,  .…è5, À.qŽp—ØY ¸K„ž*²À€5,:r™4,`à.9›4,`à.ÜÈkX€]5°Ã¨alÅøœ­† ÜõÙ²† Üõ !@å%$ ¸ë– Enà®ÖkXÀÀ]Ï>jXC1À«fÀ]@HÐÓÈ€Õ°ñ ´2.—¨aAíSyŸ+~ -cÑ° w¨HÂÜ|f €ñÕkX€»Ž©5p×ùb5,À]ä A؃qH4,@|3ÎÂtU^1¶ä¬aAî–¼†ˆÂÆf ‰ÊK†ä4,‹kX ²`¯aAÌlJJQÂX,xHùT^4¸$ Bd©<ȉ5,@bm -•¬aAò…¼×° yÈ‚”AÃd¦8dßjyY`.j}À™¢†@2“ ³†)DÔcЪƋÞä¸Vy<—4,H‰ B±Ó°µ”Épz ðFÒ% xŽ5,H²ÀU4,@(2Ù#3×x² ÉÖ°™.xÎiX=dA6¨aAöUf´ B£–„ª5[©3³†™£‘TQø *M ¥Êƒ,€ °?Rl Pü ØD™£ò Ö° g/åfÖ° gÈâ­×°¶)õfÒ° XÈ‚ŒOÂb£‘4YÃÀ8xÖjXP\F½éYÃø¥Ô›YÂBõ&J<Åo E©7“†…!K@Š¢ò ’¦¢ø-0 õ&ÐIãEÈâ kXPb”z3kX€4õ&2zõH K½™Ôãàn(@&ÜEf5,ÀêMd¶Zmo¼!!?Ó™Y*άžX@o@lU ,°Ñ*©gà+5’zh1QY樞 ¬‚é¬zl`¿ÈÇø–Ó§ùˆ*ÂbfN²È·`d÷½æÍèN³å|ì_;ýô Á÷ü·ŸžROÊÚƒñG)¬ƒ¸ˆ/ØnÀ•©Ã1 -a–Gü¡*!|M"•!¤,®‡n"$B1ú”sÿà¶"Cb‘Ù!™©+ãy^T -[EQ—õ³û¼‘¬CG¨ QWÁºYI(êĪ^öys½ÎóNJRdýì>o.+‹e/ùÁf^UȽƻґü;ß~Ø3TŽ ¹×øm…Ö–!Ê:ïQ =ØçÍw}ž÷¨…ìóV²R›Óz -(ÛΆÈsí¸’õM¢~f—·IÑŠŠ—¬ ¡õ®åMãŸ8!€®Ó/÷fúå$7òx†n½=øáôÓ_÷Ý/õGLûáùôýϬe¤WÁûÃçðÿ_ξ{0n,– b3® eéÛÎ(7¬KiÄMç@“ ?›d‚||Y>sZÿχåõ$'Þ¸Hš¼cèàÈä֥̬ ö:gl¡‡ÖÖâwgGů©~ý¶6›ƒa”¶˜Ó%9Y­ÆC+?ÔãUFþ5l» £r ežnÛi²%U?À^¶”rfäÍ©_Œ;:ØçÝèÈ¡B߀tXS’xÁ¾ç*d³›áªš›{–ãþ;ʾ R´:›‹ÓdxÐÑáöáÁ9¹¹Cä¯È£DqrŒ•µÞ»€àëá ÄÕz§¯‰rƒÉÁ³º {R“ †YÛ…]!t¼ ¢ÕÜ0j‡*!Ú{ûÅèämø(÷ûð{Á/ìœgå³mØOm':\°¶õOS›F{^æ‰vGâ¿AêH|톇Ö+GîÞådµÞîÓ^ï˜LAŒˆšÏîî*zm`¯õÍê>N–çô„í’3ézNJÖ°%»=ˆ=Öçý@,¶ÙE¬‡ˆ= -µHn†ý~rsŽcFt»5Ö›Tl‰Esæ&¼í†9t Ê[Pº—¸ˆ: G*ö‹œçd9„yl‡º¼ÃHò@Û¿­al† #|œrÚt7íÉr|Vl–[õr*]O‚+þP—w£x/'Θ¶¤7Iz¶Ãµº©*ZU°¸Lv’™Vð±.ïGÁ>Ör¦è–½I½×úªÚ¨®.¹ß™‰4æÛQÓæ_ ží,±Vz.99FÅ+KÝ“ìXŸ÷w@àêCˆ1“¾•¯‹9\m'øÜd_éVÿãU¢çÕÈⱸãs’Û r,Jñ wêòn¼S0Ë®PÊ·ôn´è7'd¾¸NÏó18Ë-îR"×~ñ»ßÿæO¿3x–4@p@fÑЕ¬¹Î*¼¸´ ­_pÂýUß6’»Õ±ž’D#èx×)öùo\èòf26{ œºqn-Nä¹ÞÙo|ôÓ¶¸IúhÓ³µËTj¶Äþ¢ÀwûüÚ²t–'*ãÏýŸÆZ»WÜÁ –§ J–Ra«Æ~ØÇWRÅ+­…éú¸+0ê#—} Ew…ÕäMfŸÖÏ«[H‘*×au ~®{±¬wL*Áj*9_MZmãŽÛ0—æ^ž#$ªÏ—²8Xz³ç¬#!Ú±¸ƒ}Þ›ÅñÚ¾yªf¯,I³ÕVŠñv’¹_Ζ#OzXŸRR-DZ`/~×9ëgà’ËÚý8ôÙ¬'t¼_,mYCÅîÕ[F*°¡¯m%mƒ:9d«¿ZpŸë~+Gµäµ¯4©Öý©aëä¡×û·ãI,šç¯^”ùV=¢öèfµ¿9ÖQ/Kt¨³4 žV-ò9š¦.K©ÚŠkâD[MÖ1§ÕÚ-!›VÓVbÇ®tl¸ÜàÀ¼Îr+|Â=z |˜Ç‰$O¬É[·ãÔxZå ãHñŽ™ïÉaÇZª¾Úú”$sÆ:'oê°E»%¯× íbÅ÷-Xtr×zi‘§áqÕß”%TÙËØsm]Z›ç!GÍ[r×ëOëþ„I&7µ[÷ÐùGzOöZ7݆Ü$s³‡ÔƼߵ~´®éŽ®BÛG»¬ÓÒV×ûÐeš²­®w7ôÞÚwùÓÕ¾ÝÙ綆jë"ÇÓCùµ9ô®¼•Õ­k“ÿä·òžW}Úa#‹ÿOýìCYøF–æ·—­†Õ[&Ï‚„ ûÅw¼Ð?®>qE—¶âB§§õ–ËÚ¾¶åõ6Í:f8"wã?­·cì:F\çjíÄv¡ŸËWo½™\õ¯õûVkø,Oóùë¹iíGË-¯:÷e]o\ø"ßW×龜ÏÐï 6ßè7.{æÒWXÃÿX¯_äW¯Òo¾Á÷ûú}8Û«q´C¡{5DŽQîœúZ_Ë=T{gÃû¼›Ãžœå}N±èŃ„¨VLÏ) ½Âô£Šû–hl‘7›®&liM %yN|÷nRiUÝŽž÷z#ï“cTåÙë¢){¦õ~íS õ {‘îh¨§õòëjŽ%‘Ù] \…- Ζ¶»z¦ú²‹<øÿ -ÍË+ŸP]Lú:÷z˾!O!ÿvšçìáe™1Ð}Û k1öxÈÖççSÀ,R™Gyª¼LãwÔØm²­ä¾}Æôì®nÙlOp0¤|qås}&É»&÷Îoüú8K«õ*Þ¯µŸ(¥Wê«‚|ËëÝš~œòà°äßM©Ku;rXq|¡û–3·üwýÜóS7rb‰'-oœsé>~Ëm'y6yxYcT]÷×Ûøã)Öw¦\}oÖÆlœ—/Ö@åh$ì¥r–÷´²%vmÛ\U´;ßéY¿ŽäÇÞJþ‚då¥eya­në|±¶ÿ±nø(óÙÄVØ×o°• +×ä—“|›+däßþ YˆëGÛÇb×í‚0˜L¹n‰Pc×ãDBâ]±´­Fú‡ÖM0@ìöíJ|`™X!¢gt·dY½5n%Ë7rdCëU좖wâç¶cÀÑq—,O b¸%ÚiºA–Z[ß?–¾ç vÑgÕ¹iû[[Ÿ¦í|D¼œ0-]S_TªœÅ·¡šv½í4š„*Z˾!Ü÷I!]N -ñÛ†flfçO»&oS.ÄÜ ‰{»ÒÚ)¤ËIm·ŽúN¥Þ*®¥i~ˆ2LrW¼B»L§vI¾bhòKíã–,‡D}¯mžìÆß/§ÀÝùå’Ò: ¬°¡™ÞL¤iÜNÎvØË ÊSÉË'¸f£É»hòwŠÐ©‚<õƒãnRò>鸓ˆovòVI3=vÔaMŽÓh;©Ë¥ÔmvÖJ¥½€noÝ5¶¡N*›è©4ñ¤:xlÉ®[]' „5ýt¬œ‰¶ é9tS~Ù‡œ¡»j°}²è&s}ÜAŽ;_frê;îäåà.ow²ùF¼ÔÀÆ×d1ýÜ&dy˜°™OŸl"Á -ƒ<`ô¥’›·vt’/^Ñœwh¨E“ïQOr†kŒÔ‚7³NÔ<º­ Â<)7ÚfÒ(:\€f­˜šLqíÖÔRWœU-¥žS êßgM‡yWºü+ñ¦­ŒÛ"^Ï–¨'@a6ç4VgGËžKf…çÞÊ£%5Úíܼ»EÛó´ ‚Ä‘™ÚÑ-Ü´|þ½Æw–>W§Z«öSOF\_G€@-/rzªß0isŒõÑc”ä‰ótTòÙ}ŸÜž´³“y&l‡a"›œ|W¨. eŸ1 Õ¾GTïV¬º¤ÏS€r44`ÎÞœ Õ/‹«OªÔ¢~™¤T”ï†$Iz³|¥U²N`ãéæø¤þükä.° -endstream -endobj -662 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 661 0 R -/Resources 4 0 R -/Annots [ 663 0 R 664 0 R 665 0 R 666 0 R 667 0 R 668 0 R 669 0 R 670 0 R 671 0 R 672 0 R 673 0 R 674 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -663 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb116-16) ->> -endobj -664 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb116-17) ->> -endobj -665 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb116-18) ->> -endobj -666 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb116-19) ->> -endobj -667 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb116-20) ->> -endobj -668 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb116-21) ->> -endobj -669 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb116-22) ->> -endobj -670 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb116-23) ->> -endobj -671 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb116-24) ->> -endobj -672 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb116-25) ->> -endobj -673 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 567.702842 101.433071 548.952842 ] -/BS << -/W 0 ->> -/Dest (cb117-1) ->> -endobj -674 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 194.455821 101.433071 175.705821 ] -/BS << -/W 0 ->> -/Dest (cb120-1) ->> -endobj -675 0 obj -<< -/Filter /FlateDecode -/Length 4938 ->> -stream -xœí]ÉŽ$I½çWäißiÄ ƒ@ Zp@º²»@¨©áÀïóÌËËÈÈê†NDMMvf˜ùbnþü¹¹Ç¦Ï -ßiü“œžRÊ1¸óåÓéóIMÑ+üWR,ŠúwøDÖÊÅÉë’?"SÌÞÄp~9áŸÕ!OþrúãN_æÑg3åŒöç|$•³SL1*/¨2TJ[Aåý£ñAP=¡‰9K*˜á£3n«Š0Ãåä%ÌpÆ$A•`†uQÛ­*à µ“T0Cg•V°C¥D QÎY#è´Ÿœ$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ÂïdA—` ЕDlQAiI—íäsrNÔeèTŠi«3ÊO>Y«$Ö“6­¨ÃˆB'…(è l ÊgQ[¼UFÒYØâð'éÀÞÆ”„Ñjht[eµ¤ó°Å˜èDlAçK `lQ1H€4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ0rEl±ÙI\` lA‰ ¬…-ÆY‰ ,ØÖ-XЭSYK\`Á·Né qáÂL%qãڜĔ (e‰ ,8ׂ;%.° ] ÷(qëZŒÄ´k] -Xð®uZI\àÀ»Ö"§¨ƒ-°Ä`ÇÉêd%.pà]«U”¸Àw1ŒŒÄèÕ 8 +“III\àÀ»&)'q)@ÔÅ cÐJ\àÀ»Æ£ ÒÌ Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\€ù:c%.@a“AX"qïêŒéOÓ`«I§$.ðà]Ð}¢.O:Â㸅—'B–¸Àƒw1QY‰ p°‘¢Ä`²)eç$.ˆ>`=¯Jº`¦A×¢.A§£Ä1º S±–¸k©)aÐK\‚Ÿ(\’¸f@§Ä1à ‰ 0MÉ 2—t¶ È–¸‘.tZK\ lA4(qA2 «Ì $.À µ$\-霢uf’¸ ¹0Q¨(q† Vš`JQ[@ h±í%.Àâ‹M,sDl±ÞI\’¡åf’¸ %Øb”‘¸Ø¤õf”¸ +Ø‚ˆO€ÂDa²Ä qè”’¸ ë„õ¦q`\Òz3I\­ÂzKƒ™$x‘i¸Xo"²•Öö -Äë#â3Y™hÅ™Äêõ˜[Å-îÅlÅ=ò¥5’¸i3±²LAÜ5À´ -¥Vâ¶ÿ"sN˜wŸøU"æÌ©‘ÃÂ!ºïkŽ‘Ì^I6ïýã/§¿Çä{þË?OÏ?*;eí°ái‚¦ÑÛ&£·?à,×&~8!2]‰-hº¥¹†µ°ìZáåÄĈòÀšû²”cU—ŠœU§kÖÔÌ6&½°–0ñË ‹¬HWbï{ÚQ†Qî–¼0éhô¨jxGòïåôGtôg^Æð&H%bZ£¬w=­ -6ƒº2æÑÕ“KݺD -¡^v5zšúš÷ëd©úæFÞ\æt..ˆeMŒ2„"/(Á-Å´zniñk#\@hˆ—âr? ,½:MÓ¦!%CH1%¤^‹}héÕq!‡P!Ô¥£Ñ£ªáÉ¿—·ûM.EêôßyûC YîцOò€¼:ȸ߸ïßßÛàû?|þ‹çí;3˜·˜9b‰ì·Á÷6øÞ߃Od÷…œWäÛÀ{xÿgïž‹Âɺ@'ÊmBm~67ª©,ÝÙäÉCÊuUùš<û+QÚZ7Þåsÿ!¯KµÕSÙ©p‹z½ÆºWD[ïÎóÕmupŒÖ–öQõŠFÞJ|ÓºC}¿5r®l]¯Ãpö*Y“DcfúŠFo7¨~úî„-Wá;»É¹TÎ_Ñe%‘öàÏï>~ü³ßüöWø zëÝóéOß+žT0XiÚP·J|{ƒd€²ò~>vÏõ“f¥Åÿ -(iéÛ½oúŸüùü¦T¿Úì»qÈFŽÆ؆çŒMdÞÚ¢òíªeé'ßé勇úêw“û¹vJK–»§Y^,Œ‚¼¦EåStµuÔ@wuµ2HhÎé©3î±â#r6¬ÇͯÌ’q›eºÆ?»¾²_±ëy`ºÔ[ÇJIÿ4ú·Œ“ŠýáÛù‘ÜWfR»ë‹V¾˜ùzÉ7³~ö®Ä=¶G’ˆ2l.q]˜åvtBEº2a±XéÞ<¯Š>kh4ºLíEE†Nk+Äa9)·Ž—´kµÎ®é‡ÚúˆªDW;‡O¿üᇟÿâ÷,|"—û:}^Z€RéDz@eî„«æ0©L8­*lS@@F ‚ªuëµ ]>à -%úç[¹]*׆ Ús+7ÖË™P%å.JóX ìM™r¡½óÁ^ñû¸ ݪŽ™Ô—a×Lî-D[‡d«ï´L7 Gyí÷< ,óÜôp:¡Þ¥×ô]É1JÄ>¿]·A\„U¿Øç¯ì5°HÂÚC­ǵϵwëùôE]‘ÏÝK¢mÒQ\ì˜JÖ<˜Y öZP¹ ÚèêëL×9®*-¬p…0érÙJ~6dºò’¤û„y0Ï—¬-ÿ#¼™òoª÷|Z”GX]—íèZçq‘J»ÉL7O£,m+çIÀ^`º5+Ô|ÔŽyIµÌ3‡ÑËölÊM£]ÅŽã8€÷pÞ1g ó ¹ã<×`õ¹â|4릜=m,+ÜøùW¼z”¥T® ÜÅøÁ<†qÚêÛÁøzFéßq`¤„h¾~Sx—程KÝ"͇ú]ñhjø¬ÛçCývõ·[>ÿK3çmIwäXê]—邵}´Ëó hfg¿‰ªeO5ïûë\cÛEÞË’åZš±¨ …mÍ+òÚ±]ý65OCn³óê$ÄÍ -%ÅÃÚ—yçÅý~=½Z(gå»Õ‡¥¬ù¹md¬Tç6„S»x¡ó<BƒÙåã=^zùý1”<75´ÏÛ<«Øå™q}C¾“mé6†UýqÛæy{g9J‡þ --ã½tlf>˜çáèv¹òϵ±m”€¤´ôö¢gp–_¡GBò^Šr×ÑŠ[Vè[G£‹ö¬FIÏ»7‡¤eù>÷øö -’Y¼xtÖ?˜çá¼cö5ÔÞš§õGâ:¸¬Ð¤W룦W«ž¿‚¾23§‘^³ÓLnÐû$¬¾ßÚ¿áÈ+íøïDWК^Á»Çò<Z÷cÔÿ^,ömô–·üÝí½:ÉHŽ,.5~ÊÑi¾äƒy ÉñF,ûþz|Æg\ËO•G¹Z>óq­kÕ¼6£'oeFJ·æTa~¸j뽑‘pŽdÌQWÐ6bOG÷­‚¿…¶cym7âÕuܵòüy}‡èýνýv«^c3p·¡Õ¹BdIÏvJ×û -BÙ êiÁ¬ãZ†J¿ŠÛ³õñwÇ»WP8âFòä²ÓîÖì}0Ïáp?ÖüV(ìHYõ ´cÕQö´EÄÕ~E|v0ÏÃõð~|¶¾¤®{uo†£‹ žëLTÇ#g£Çî©^À$è(i¨ªzK‘ö¥–ûÄò¹‘Ÿ×Õ‘ÅÓ¾Ÿë(:µrCµ¥–Sò¤º/ù|;hf—í—Q–T¸eó<ÊÒ~ìô¿Ð»¯FéÊŽõq±Ë²rèLœ©¶]ع;Û¯ž¯ é{yó<šöc£¯7nïD&CØ]È 3šuüÚ¼s"¯ Ê¿‚£Žåy8TíÇ:¯¯0#þ‘y÷¬7â$‡ØãÐ%‚ó<¢öc«ÇïÁo03ßX{e¶wttà`žGCO¾±ßtÏ™¡+ékgpÊ­&|íµ¾VÉ e­Ò,Îq†å÷æ|Û=•lï+¶’—·¶Ûnç…í~^–¾ã{X¯ÙÏÏ,Ž¢g.ÒáÖÙ§ƒy•7ö¥h?ÿ› 5.ÓmP¨wÐÙöñíJW÷ò¿|^Aï+b¼ƒy½7ö³¾äJ©Õ~?#¿8“ÃÎà\½z*ìì‰]»zjç¼iGíšC?ú€çKó+öãæy8„¾/]ðçœ/½uÕîb׹ŴW¸iq*7••Ì­åÏÑL†åòPÈ=0Q(öXƒAÁ=úîå{#Ï‚^W›ñkšo·7¬§¡Ààwõ™–Ñ£W=Åz!›œÙ#2;@o&¹Çí5ÛÃ6Ï_\Þõ€"ée66 P—#=ÿõÚ=tÝ°¢Ëz¶^åN±R´@MM»O‚ÜÛn¢,³«šo®ü8º­Ü3Qo˜-7YæY^n˜ jÜL[›ïU¿ÁÖ”ëÿÔ|³*»1·—ïÕ¸I·Ú³¸1;×õsiæ?ŸByŽ±.ϲW!MÚÐËnÎÁM´¬ùLÏNNÊ:ݺMGëó•œåAŸ{ªúÖ E/ ‡Hϯb5ýçÒáŸN:»‰ü_ž^PÞð¡Š8™v0Ä/'zËÄÎG Ü“°þT½,§ûšìÆ¥GɬRZ?©úsT½ê•Íàî{¹C:w¾pqì=®éýÝÞ^.Ã|^ÊÄæÖb‚~j• ±Q}¶•1á0 ¤0ăF_Š¸ÖÎNô²ös 2iƽœþz*Óœ±ÞÐS‚Õôþuéó¿æ|þvrç_·£Íó¸Q-LçÖ…˜Ïdô¶²tPLo/kÂ>l˜Ð8FûC¬ÓHëú> -endobj -677 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 548.554475 101.433071 529.804475 ] -/BS << -/W 0 ->> -/Dest (cb121-1) ->> -endobj -678 0 obj -<< -/Filter /FlateDecode -/Length 4772 ->> -stream -xœí]K$¹q¾×¯¨³€¥ø~ÂÈÖ°`Áû`øÐ];-Áè°ÒÁß_0ùJfTfVÏh·ì4zº2‚HòãA2“¥®?ß(ü­1¦àíõöÃåÇ‹ÁIüË)VYý'üwUÒáTòÑ] "$§ƒ¿¾_ðßt9&¿üׯ.]çQW-RòZ¹ëß>“ÊbÒ1ª•T†Q9'BÐÎ3*¯n1%N3\°ÚnUfا‚VëȨ"Ì06(³U%˜¡ƒ²œ -f¨ä£Rv(-%«ƒ!ÒZ£r£Ñ=§ÓJø˜tbu:4§3Fø`•Sœ. ｋœÎ—¤ât¶8e­ät°Åš8‡-ÆÉê`‹ŽÁpº[4Z'1º[€®Èê`‹ôRqºd„KÑZV— “1Ä­NK'\4Fr:¥„6 «ÃˆB'ùÀè4lñÒ%V[œ‘šÓØbñÃéÀ΄™Ñªiti§s°Eë`YlAçs  =l‘Ás€$ˆ—8Ð!›´åH@G#l´‘c“°!ŽtrÂú80RA§GFÂg=ÇFÁ‹‘Ëê`‹I–ã£a p\` lÑÖp\`À¶há¸À€n­LŠã¾µRyŽ fJŽ ×Do9.0 \@)q\`À¹ÜÉqéb¸Ž X×8¯9.0 ]c£ç¸À€wU’ã Þ59Yl€9.; -£¢á¸À‚w’ã ÞÅ0Ò Wpè9.V„ŽQr\`Á»:JËqïêÀê‚À4Xð®v¸Îsƒwµ“šã ÞÕV{Ž þ‘l4Þ…8>€- XŽ àï¡Ó†ã&4ÂŽ xW%¸?fLƒ­„ŠÉs\àÀ»*¢ûX]* Å™q‹VÊûÄqïÂQŽ xô9.Ç xTÍqï*OÍéÀ»J'Éqï"npB!à¸þM( 貺$dŠ‘ãÞ-iŽ à……Œ`V€¤8. ‰8”ãÞ…{Oxð®tÆp\àÁ»Ò"ìau°ÅDÍqâè0l™1`UH ¦àtà]©ÐÃœ¼+eHxð.¹MŽ | ŽÀ@(Xs\à£)ÏqO:„|¬Î \;Ž ‚„À²Xl±Ár\€ÀZ$“"ÇAã5ÇAÄ  "I!úf§°paç¶ Rä¸L&b²–ã‚à<æchUN絈tÍê"t*p\‚pÅŠãÌ¥DÄ ç¸/(\â¸f@§,Ç!Á Ž àŠDÔˆÌ9‚-²9.@¤ RD [ r\uÄ,ÓKŽ à¡1—DSs:+iž9.ˆÖ -9.À0ÁLLÉê` h‚ãôM¶Ç˜ü`²‰i«ƒ-ÆYŽ bÔ4ÝŒÄ[´Ô›4ß $ [ñq\¤&s\‡NJŽ ’Š˜ojËqÆ%Í7#ÇÉHÌ71ÅcÆm2žæ›ã‚da‹CˆÂê` ‚¦ÄŒ[pæ›`'Nça‹N–ã‚ä=Í7#Ç30ßDDÏ. Dšov¹¼ë˜‰Ów™zŽ pù&"[nn/A¼. >ã•‘fœ‘]1 ^ßÊ.Hp/¼U`× $È—æHì¢ÌÄÌ2zvÕnJ%Ùe þEKTpð™1ÐM& ‹è¾Í9z2s'Ù²>ö·?_~ýç{ýóß/o¿Ê+eõG!@È©-â+ ÐÝÑ·Âí‚&¿LE¾d!Bê&*bYr#áüèßuûae7Ì&ѳ"O+À–¥zðP.ÝbJŒÙ&Ù-~8Ï¡ÉñAº¶Š·µ·ÔXoo*ÎÖ‡ó|u[áþ1au7×õ²F%þº?@ålëuàyí1xcÏfúÊF›V¢N µÚS°d \ ´Ó{0ËW4Xb2èÐ#Iš¼)(ó iW‰ûéb-V,köír0ñ‚£Ôîúé‡Ë¯ÿðÝwÿòûÿ?½]þû7³–o¿Qøkãò×¹r}+׺\‡å¯Fé$¥BÄáü"Ã4E*¹üÖòj9-êeÚ·©d´/KÙ9]§«õTWõÞ~'™:J¹sžzo+ûõVG6Z_¯¿ýŸë§ï.¿û´Kžñ,`ƒVð1Þàë\–/Á×?V)íª6ñë©8uçëÐÌo Ìì³ú»êöTÒ¸µ|†NƒP©Ç½ ];4š½5ÿKùe R!°);nï¡–Û`×m±º§¹}â>Ôhw§Gc¢†¹‘£.ÚÅÚÉ<Ï6Ì°wÑ6µXkí‘t6­:ô¶_ÿ{tDȈ€Ùv&£YÕŽ;ävˆ–ŠPRÛ!‘CÛµË!ÞÊ›]ðÀ:æ¶ûÝÿý_ÿó½íò(S4÷*]G¦n÷±¶µµ‹#Û–´ˆå0¢ë=.×yœÅRÆÚIPÛ,k½4)_.×-×͹”Ûl,õæüƒƒêYÛVëÎyÒRÕ?`*·ü©¦ŽcSSëJpœ¶ƒµCÙòÒRd•%«n¥6_ªñcùšö^1±ÅŒ§”oJ  BÅrrÏßoËñ†h£j[NnÚÂ>ßýž´·ÊÛ!û½äž¬÷G½¡—ÁØF–^ZÄÔLÙ {™kduTißëÈíì–댦·b_ñ¦¶½*=þ²ÖgdêÒ'¦´‰)÷æ˽U´zþ>¢n½þŒ~ÝQHy•Þmë8·‡ý\úÚu›ó}ÄŽ%ç—zÜxŸÕ抗ÐÓÍe.,w®=óHÞiObW± —¼Ò¬ÛiîËVžîÉíµ“‘s;e»KÖr›C¿ÎŒ0ÚQÛ#+²÷e+n¸¯±ýË}ólÇ1œBàÊBñ÷MÏU¬î{á›õx©cnŽ!Œw›<£ 1§•¥…¨ÝâdžMŠ¿bðàš½!‰ˆÉ¦–lðá?ÏzT>fp|K'Ü­0y!m,MV -ËC*vG'Í·û%jô¬ó¸L‰îí(·yÖ{Öž£Ü΋äÐg\îì~ç Þ¶Î8YçͲŸ§°gªczéÞS›°ižT¼M¶ºaP牶—·šLy[8¸gãGzvF ÛçÇukøÿ+öšóJhÌäË*F†y.oqËë‘ã·Óõ{}Nª=ô5d…YcèÕÐ7ºïȺom¬HÉÅ ¦ -÷ç(h_P„æ€mÞ'Ý_º<—çÙGϵì oæùö7 ‹*vYÊ)ÈMË/« kÝ*Í÷åoA±ÖE_¿/mù\Êügóg´9Ýc¨wm¢í»}´Ëó$hëf{¿Ê;Ë~Órݽ•èUÞº4õðj¹³,ddTºáÚLŸuÉS‘[í<@‡¸Õê|™†,ÓÔéÞêê·>®§¡W1åLm—cm¿–Õv®Óã BCã6òTlÃBÏåy:„¦]><Ûã¹—_Ρ丩¢}Y<(ŸåºžGÿ[;ÜÙÊ6*¹Ùû9Zˆ÷rà½xÎ3ŸÌólHôj—+xîôb÷ܳ'8ËMèá|´e“0GÜõ‘]Ëy”´¼{>$®ËßðùCÛNw<Ä‹g½þÉÛóu5di_–:òã3µ\_l)åä<±¬K¾=~äÙyûï ,„ÇQv.ÏÓ¡l?vúgèÝ£t²c¾Îv™¡Ú‰ÓŶ۰×cûåÛç·?°–w2ϳ¡)îÇF_oÜ>ˆÌa!Ó/h¦}“¯Ë; òªôãu2ÏÓ¡j?Öùøx}øDæ£^/ö8É&wîÁ½“yžQû±Õó÷àÏà™æ^qX;:»p2ÏÓ¡ç`½é‘¡3é{;8ùeqîõ¿²2ÛÞf ÃLžÛ­+·ÍÎÛ°ú9¿Ï7¬a}d=? q@Hû£Ý§“yž •é`]ê‰Öó´NÏûmP¨vÐY×ñç7[ËZþ—Â;èý@Œw2ÏÓ¡÷`=ëKž”šÖ›ÆùÕNΰƒs÷é)¿³&vïé©}Ó†Ú™C¿g8ô ÷KÓÖãNæy:„þ²_ºâÏ'Ø/=zjwµê\cÚ{\;¬ßI+òDæhös2ÏÓ!ù ~ý¢8ì¹F‹€G¼ùî³{=ÏŠ[§•ø™ã³Mã[-sDs ¿~xkÿl¦gê’ûaí/‡È/=läOåì¨àÐ ŽÎͦãŸÒpUÔa’øÔ—e¶Çé\Õt„Ñú}I‡Æ›xÕ@ x׸»oÓгzx;¯””÷'( -oÇ”8M×7p¨‰}‰ËrÜ&——q?÷.Ëoã”|ó‹¶i‘çd½ì/ÿ–Ïý(Ù^ÖùÉR¹¼¬;¼HÜʯ/ö¬^$Oee&ßæ¼ø|^ ÊgÆJ…Òt¨üÕ[A F!]éŒÂ(UµÛTnhu½“3ÅÿcKUNç—t`/Ö˜»u¼(éÌþÃE%+¨ýóÉù$m™ÅQ׋.~¿Ðá–[ÛG8“°|”­,L0ô$»!ƒHsJãêYgÑ ô*Ñòìé{~W³Ñ wVÖD&Ÿñ¶TÌ‘Ó«­˜î^ -UÄt}¦\ùf -½HÛ„½@;¦íöŒ‡beä t[¡ªëbm«=ŠN/mMç»°™>6º¿¦ÅìUŸÞ.¨ÖÌÒwVJg`×¢jëjÙdf0*5©«)[‡ØÖOŒèva„ßÈZ´Eg6ýÐkt’è"ŒH¶¥K5#º]Øt¥4×z*´T±gMµå»)ê&¶†gd·áØ’@_¯1Œiêǵ˜Nãh}-〽Þ.ζ‘ØÅï˜Ô®s…tz£Ð å6q”/]J'e-Ÿ04«ŒÎa¤ÿªpMJÌ“?(Û Eg©ö«^ Í&^7Ò(÷u,×ʆžvh.BÃìØ^*J&=‘nKÝZl%šl‡TÍ#’j䱫†º&& -«íÓ¸rÊjªëb× ü¾w;]ªN¶Ê¼àîZ¹]ìw¾âÐz\ÑÁ¸ÍÞVî€ù*¼eÇfg1A?ÖJºXËæízeƒ°RèâN£ïY\Gkc'ú²ú±2ÎãÞ.¹d7§ÓtJ†7ŠÎy£.uý?¤ù¿ÿ{±×«‹.~\ËƺqëJô¢ÞÞB­,3Œ=Ó‚œ>4zhaW#FiìÙJ7Vj«lUiu. “UÖJ Õ&_²ÕfÉwÙV --¦èÒ¿Œ-íÆ^ièÆ^ñ›´Tnõx-Z2-r#œC¿;ÙS¶X222?ænRÛSš*ÛÖm×uWo{nÃvC|LeÏæ6)?Ä;Kži‚KUòʈŠý(n8j«‚Ñ&»šŒÞƒC,}AF°qX*y8ÏþÚÉv…¤®Œ5¡;„5FEß“Ç›ðE:V EÕïÐã³%ÉÞÒc À, uš“›•¡üEIyý1/µKzš_Ó÷¢ -z#}KŠ?¥’6F^WË'ùçÿLÀR -endstream -endobj -679 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 678 0 R -/Resources 4 0 R -/Annots [ 680 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -680 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 570.123057 101.433071 551.373057 ] -/BS << -/W 0 ->> -/Dest (cb124-1) ->> -endobj -681 0 obj -<< -/Filter /FlateDecode -/Length 4788 ->> -stream -xœí]M丑½ç¯È³‘ùý,æ°Àök¬± ïa°‡êœ.F±þûû‚ÉR*«Û3i`ºP©~ƒAJbé«ÂÏWÿ%§§”r îzûáòãEMÑ+ü+)VEýGüFÕÊÅÉë’¿þ@§˜½‰áúvÁÃ%Oþvùß_]þ¶Î£¯fÊ9í¯ÿD*g§˜bT^Pe¨”¶‚Êû)Fム -zBs–T0ÃGgÜVa†ËÉK*˜áŒI‚*Á 뢶[U†&j'©`†Î> -*­`‡6J‰:¢œ³FÐi?8=H:£§²É¢.B§£‘tÖN!:íµ¤ËSÁ'Iç`‹ÏJK:[¼vNI:ØâlŠ’.À¬u°Å¤h%]„-ÞÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓVg”Ÿ|²VI:­'lZQ‡…N -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) £ÕÐè¶ÊjIça‹1щ:؂ΗÀØ¢b(#h"^’8ÀÄ8¹lœD&ÙÉ%—$0)O.F+Ñ€É~r!G‰¬ÒÐ-U°Å» 1Õ°ÅaäŠ:Øb³“¸ÀØ‚X [Œ³X°­Z$.° [§²–¸À‚oÒAâ Â…™Jâ Ƶ)8‰ ,(PÊXp®wJ\`AºîQâ Öµ>‰ ,h׺$.°à]ë´’¸Àw­ENQ[`‰ ÀŽ“ÕÉJ\àÀ»V«(qïb‰ Ыp$.V&“’’¸ÀwMRNâÞ5S€¨‹Æ •¸ÀwG¤™¼k¼28ð®q&H\ Là%Fà]Pˆ—ÃØ‚•¸ó=tÆJ\€Â&ƒ°DâÞÕÓŸ0¦ÁV“N9H\àÁ»:¡ûD]žt„Ç…q /O:„,qïb¢²xð.è3I\ŽŸ0£‰ ñ-ª8DÌ™)R#3†…CtßÖ=™ÝI6ïýýÏ—_¿`ò½þù—×_•²ú£ ”Ôñ†Éènè[£ù6 8,–Á°GÒR¥1w£4hpÙd#£0ÝÑ¿ëöËÊRJ¦"7ÖAÖþ´¬SK½`žRºÃ"ëCsd7õá$Ÿ÷4TG¹R_-Ÿ±öU·ÕÅ¥\–¦”Må½.u?ŸrlàŽý®¶ÝÏ´ÑìQÛ:ÉTomkño7ó_m_»ö³\Fcu²|^õ>©>P·™N°š›x»° CÈgé.ÙÒ.»øÏ“/Óâ+òï÷Ûr‚¥%$ݦؖSÚ`{Bió‘­wìP¬?o¬}äS3;Á6º]”C|%”=c¾—¹Æp÷µ ½ŽÖföIù]pKimõ½^0ö²Ö“/JÞZWXü“|SÛ8þ…v,Õ4¯lœ™Ž%Ê«Í¡¯ýè÷iékßm.íHK>ÌõxÞÎjsÅKìéÆ2Ûø<áÏ þ$îò jΫìÚOc_¶òk“?ôSýTì6[ú¸í}:ŽinCõESdëË0N†H–öDlYЭo´£-Ãédžw-ξ`t䛽´·ŠEQbxT†ž_‡!…"©Ë-›†çNØ­0,Yé–ÆPa¥Ï©:5e’}}\¢±pnÀEJô¯÷r»Tî† Ús/·SÆêÐH¹Ë>®Yì]™òhóÁîxÈ}»¡ŽUpizOmÃqÍô:ØêÙG­Ï™<ÙgÈs×sÀé„x—ÞÓk´L%bŸß¯Û|Åÿ_°×|ГÁBH‰ãÚç¥ëÃs‘ÏèÎ|öଠä„øë@`¼Ù½€lðÐó™îl•VØ!Lçt%?ƒ7¹é;„y2Ï“¬*o:ïxS½#o\½¿Ôp~½œ)S¸f£8vyà ÙÇeá*¬G{‰'_–ðjgÙíúúDÒ}Ð}D6´Å%D`¡`CdfS¼<¥Ìã9ûõPá= ² îšes'ó<½ Gh¹_BVYÌûå“‚î<ÿŠº¸Ö­Ò|¿|.è6K0Ûöâ¾_>ÝòÝõƒ§9nƒ6r,õ®Ët3émçò< ÚºÙ˜õŽbFµ³Ó9lÛøqËËo{¹m} =¼Úá]¶G -*=»¶Ãw³ä©È­vÞA„8Ç·_{ kK\ç—°Çõ4ôj¡œÁw%þkYõs]®o·!0˜"8ØÅ{=—çéšùðl—^~9‡’gঊöy3cù®Öõ—çé|'–}ÙÏøŒkývÝ¿—Ï|:@눠%¯ ÂèÉ[™‘Òœ*=|±g룑‘p7£ÏQ;h뱧£Þmçò<ÚâxuŒ»ÏÈãl;D/kôúÝ ½ÆfàfC­s@dIÏž÷j„²AÔÇ!‚ãZ†J?Äí«ÙzDüÃñ®ŒÂØãFòä²ÓîÞì}2ÏÓ¡ð8Öü¹PØ2ô ´cÕPöq‹ˆÝ~G|v2ÏÓõðq|6>2Ö¼z4ámæu™‰–ñHwÄhÆ1¤û¸<¦#è(©¨Zô–"íÛRîG–Ïõü¼®†,žöe®£<ÎSË ‹-K9%OZö%_oÍì²ý;(‹æq”Ëót(;Žþz÷Ý(쯋]–•CwâÌbÛÝë±³ýêuMïØË;™çÙДŽc£/7nD&CØCÈ 3šé¾É—åG¹ótºyœ£Næy:TÇ:ï¯O0#þ ‘ù謗zœä²?÷0ßÉ@gÝÇ·ƒnÙËÿüQ(£7¿#Æ;™çéÐ{g?ësž”ö›øùÕvg÷é©p°'¶÷ôÔÁ}Ó†Ú‘C¿8ô ï—æwìÇÌótýå~éŠ?Ÿà~齧vW»Î5¦ÝãZ¶§ÜT2÷V?'ó<’ïįŸ‡=×HðÈl~øì^ϳâÖa'~äøbÓeŒhîEì¾éÙ;û'ó<Rµ:j7]8N£C×øFµûˆÓÞrLüöø2…>úÐÞ^ØÒH×±³¤—“Ò£e/l˜vhýq9Ç(zô‚§3œéD¢ÌŽEjxº›ä|ê«2Ûƒ^®z8\gý6 Š¤Ìmº $G:ókï]z2ð×ÉË>ÉòöÅàíH†%J3õýrqX¢²µ©ù•ÛO½ËÊ»8Ë+¾åõé<ËË+³AõW‚—ïýxÕ^Q6å¹R5¿¶Ë^knå×W’™=«×Úó²/SÚ°éÆ/¡œ]§Ëù¥*¤I:àüÜDÛE1_é¼¼¤¬ÓµÛtq´¾îä,ÇÂÿØR-'Å+:<–,ÝÊ/–ô?–ÿᢳ›Èÿå|‡rª³*âdêE¿]èTo'ˆë‡DÜ“pùªZYN·3'ªì‚ŒSSZ?©åk?˜‚^$š¿‚=CÏïk6:mÍ©šÈ–óÆæŠ!r½SëÕ¤1Œž&»\…f -½ZÛ„½@ÇÓö{Æ»bmú­P±êºØ¸j¦“4›ëB6Ó¹ë4ëþšÖDÖGÂÛÕÚQú&Jé<æZTõ®QMf™Q¹I}MÙ:ĵ~D·‹ üJÕ¢:³éY¯Ñ©–³0U ¹–.×t‚èvÓ-¥ùÖS±¥J=k®žï¦t¨ÓÙû²;¼%ÒŸz`cDÙúu-¦³AZ_«Äpc6ÂÛÅ»6»øíâ£Úw®P¾B -+·‰“êxéR:#lþ†¡Yet& ý?S…oRbžòE»):׳_õJhl6ñÚI\ë8ÉÚÅž–¹KÇØ0Ëý¥“Òé¶ÔÍc+)s“Ç\Í#’jä±놺&& -«þi\É…ªêºØ7(¿­ÄÝN߆ªW­² Ü}+·‹CãÎ7.Ž­ÇéØ¡Úˆ>†ù*¼•‰Íb‚~ª•t1×SáÓ*cÂnH¡‹;¾q­èOÔ¯}I3îíò—K™æŒõ†ÎÍVÓ™ã<êÒ×"Í·øýëÅ]ÿ«r9ÏãF5¶0[Wb>“Ñ_¨H'Åô+ª° &4ŽÑ~ëÔÓúŠ†6ÃøÚõìª{M©ƒê¦åBß}ÈĦA×hË<ã„·Á‹-ùܾéö|ikãè:µZ… Þ\¦®³}J¾²"qEIû”­3,#.x¡¸dænÙÊL$¯¤¹žìw¼+,ÕÅ.XA—¶ë\ŸfyÑUVZÁ’ ¶ü‚ì4¯4¯ìŒ~÷ðúì~_Lu´Q{“¾qikIáÉR[.Ãú²Í9¶ƒQo÷P+˲±g[Ó‡F-Üê‚c”IS϶$ð¼RWe«JëDÑè2Ue­ÄXm -K¶ê–Òâ$z)¶˜¢KÿÂ=íy¯4'zÞ+a“–Ê­3^‹–l €<‡sì­S=e‹%“ > -endobj -683 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 573.873057 101.433071 555.123057 ] -/BS << -/W 0 ->> -/Dest (cb127-1) ->> -endobj -684 0 obj -<< -/Filter /FlateDecode -/Length 4747 ->> -stream -xœí][,7~Ÿ_1ÏH1¾_$”$qˆ‡=›³‰Ð)ðÀßç+·oí®îéÙs’L»Úî*_ÊUåÏe»Û£®¿Ÿ)ü‹V‰SðöúüÝåû‹ÁIüä«›Ìþ -Á]•´A8•|t×ïˆ`‚Éé௯ü›nÇ䯗¿þâòÏuuÕ"%¯•»þ뱬!† ÃJ`Ie–s"í<ÃòJ ‰)q,ˆá‚ÕvË -æè8Ä°ZG†!†±A™-+A ”åXC%–’Ci)Y‘ÖÍð”J÷O+ácÒ‰åðTÐÏáƒUNq¼$¼÷.r< Y\’Šã9È┵’ãAkbàx²o$˃,:ÃñdÑÐNbx²À»"˃,ÒKÅñ’.EkY^OÆ·<-pÑÉñ”¾iXzŒäÃÓÅK—XdqFjŽg ‹Å/ÇN8bdz«¦Þm¤QÏA­ƒeyÆç@{È"ƒç =H.q C6iË€ŽFØh#‡:&aC0 èä„õ)p8`¤O+Œ„,Îz Œ‚,=—åA“,‡FC$à°ÀÈ¢­á°Àm-¼…øµ2) ðÖJå9,0\ˆ)9,0@\½å°ÀráJ‰ÃÌ5ÀN @Ý=pX`€ºÆyÍaì=‡¸k¬’Xà®1ÈÉò ˜Ã £0* ,p×(8,°À]t#Ía¬*à‡žÃøŠÐ1J ,pWGi9,°À]0°¼ Ð ‡¸«šÀÜÀ]í¤æ°ÀwµÕžÃ8ŠþH6îB@,‡ïÁÓ†Ã&4 pW% LŸZ “ç°ÀwU„ùX^*@ãL¿…–…ò>qXà€»¨ ‡¸ øŒãFTÍaî*ƒ‘šãw•N’ÃÜEÜà8,€;…€ÃŒoBI¸.ËKB¦9,pÀ]À’æ°£°HÂò‚€#) ÆB"å°Àw1¼' AG  @¬9,ðÑŠ‚ç°À' B>–çî‡A‚àY,²Ø`9,@`-’I‘ !‹ÑšÃ‚ ! B ˆ¤}³ÓÈwaç² Rä°H&b²–Âà<æcÐ*ÇóZĸfy<8,Á - ÅŠÃÌ¥DD§ç°/(\â°b€§,‡!At 0‰¨™s<YdsX€H<¥8,ˆ² ä° êˆY¦—`„Æ\ªæxVÒ<3rX­*rX€n‚™&’åAÀ‡°M¶‡˜ü`²‰i˃,ÆY bÔ4ÝŒÄY´ÔÀ7i¾8,H² âã° I/(Læ°0ž”$1ßÔ–ÃôKšoF ’‘˜obŠÇôÛd<Í7‡ÉB‡…åAM‰é·À$Ì7NÏC,‡É{šoF f`¾‰ˆž]ˆ4ß ìrp×% Çî"2õàóMD¶ÜÜ^x]@|Æ3#Í8#»b ½c+»d ½­»f ¾4Gb &f–ѳ«VÁT’]6À_Äcvƒ ËêÓ¸Dåácf ÔÈ„naÝ·9GOfv’-ëcÿúæòË' ¾×oþ}yùE^)«¿ -BNm_¡ès;Dß -Í7Áa– ‚Ý“–* © ¥ƒe‘„ÂpG?×íÅJRJ¢"7æAžæþ0®¥^ O.ÝbŒù€¦1²‹zwž›"DéÚ./kWÑXo×'ëÝy>¹¬öÄ”æº^VÈ[‰?­ñŸƒÃo뵘AkÀ {6Ó'Ú´ú j™ -½Þc¸ˆ\ø‘ïÌò –˜¾#ôHc’¦ñ ˆ™´«Ä¿~w”)–Uúv;ˆˆNahÔîúî»Ë/ÿÅ¿ýÝŸáï^.û•Dñùg -Ÿö=®]¹ŽË§óO—ë—òYòiÔ†HK*Óÿ$}ÊåÏ¥’Æ­éÖNå•ÏZ{Z>sU5È[ó?•¿¸–+×áwÊŽÛ6Ôr«Zšò¹jÓ¬ŸXëüüï×w_\~óní@yÞ“H#„FÔêÈB‡>w.ËÇøÜáj)ºÚ¤®¦j¸ˆ}.¼JSûõçlÎÑ=FóÏn]Ín9·ªrLîYe¿é*ÕÚ@OªÃx!ÕµÛaTTB)³î~óåŸþð—/»îrS—Ï°¸¤nmXËÙtâH®%-F[tåÚ¾å~C“«ûN§²[‘nÚbŸrï9›”ÏPÊ3E¦çRV‘£•õ¡Ôõ\îk}/Y¿Y»§ÔéGuþ­¶Ø-`Ѥ“¥¶AJj}–Î÷Ž¿ÒBMc×iò½é-lšZ<¶–òH«¶ƒHßz鉌l´å¥EÄÅ”½ô‚^æÚ«»®µïud=»å>{ÖK÷pJkªîUñî§5?ÛT›˜¢SÚæKÛƞ´3ûÌÊ»KÏÓÝÏ(¯Ò‡ºv³>¨e[».snGì¾äüRÛYe®þzº¹Ì]Ïé3÷ú}š¹ê rÉ+ÍZO³-[y~h“;ÔS˜õ”å.6¬å6»Ž}~–££ä=¹è²êíãQqêŸnÐåhs×Û@2¬Ûs¨»´ÑݼYƒ¼{~N´¬ 7éšÑaµÍž]:›éϨMô¡óÓZÒ*ÿ2f¬1¯âæ|ÄÞä mr9­,-.FŸ'ó¼iÚû ãN×äMVDL'µdÏ anà塆Ìn†Ài1Ân…É ™h³hª°CÙ-j0˜ˆöùq‰Ú@¹7)ѽÜÊmcÞ[Ål”“çVnçEr°—;IólÐÜÔÎ-YçÍŽ†ì‡)džêX…íº[jrϳїIV7Ìå:¤_¡aö8幩9ø©@ œo±í -ôÖæ·ëÖ—øÿ ­æ¼SL9w¤Ñ¯]*<¬cžø°xfÑq“fyvªCÁœ{¿¨VA°Ù l·#=É’hÏxª4£Â`Z¨£‚„v‘?¹˜'ó<È|½á&=eq€›ò‰›þ8“÷†Ë3n§¡'¦¾“œ«‰G jç2í[O¢—°Š_DØè£Ê¾8åúAóñP‚“!oý &Ùÿ#ÈicEJðH5UxÔm{-þì,¼0Ƽ {¼Lz.Ï£õG»èû}`qÚgÖí²¾™?)MËË kÞ*Í×å³ø£.az[[ýº|Úr]Êü¹¬oó¡yŽ!ëÚD›ƒÇÞv.σx[;ªÃHUî¬\O¨3#ßh•–·¢ÝdáÕŠ}YøÊ^é†{3]ë’§zn•ó†×q·Bâ§"ÚÖy—ˆãzš÷*¦œIw9ê÷kZÕs]lÙxhj؆pD`° 7·Nåy4õòÏZ<[ù霗<6Uo_–¢Êµ\×sÏþUÓÃA¬²,A­{é­½$¯Ü‹çFæ“yÎÍ!VÞ¹Ó[6³eO`–›¼‡óä[»ŽÙæèâvéÉîlO±í™zIË{4†Äuù<¿kçtÇ“‡xñì¨2ÏÃyòqŒÙæXfí «½ìj%f¾5o^Žž«$דåw¼/̱§WÃV‡…ï}ÏÌ~µ•ƒ‘;íøq¢oõoÀÝsyÎ[cÔ/û´ÑzXÚâw“ww,Úñä!.ÕN¤`éò†'ŸËóhžnIJOûñÙ8âšq;?ðV¨ùô‡o=¨ä5žé=iKÓ\ºS¹õ¸=Y=”>FñÞzìié  v7¼ídž‡ó¶ñêwMšŸ=oôÀ¶Bôt0G¯×v²Ú07j“GæôÃÓ~ózCP6õž_µmå ^馸}5ZÏw¼»ã…=nÔ> ›¬²·Fï“yÎ cÍŸÊ ›§LäV¬š—½ßzÄ®…ߟÌóp>ŽÏæG›VF8´M¿”‘¨ôGÚ‡£Gï}yÈŠáQ]½ªð EÚÏ¥Ü÷C>Ûóu5ÏÓ>-u䇱j¹¾ÈRÊÉybY—|¹¿4²óòïxYÒ÷{Ù¹<æeñ8vú9X÷Í^:É1ßg¹ÌPíÄé"Ûó°×cùåËÎsáoXË;™çá¼é86útýöNÏ<ì.Ïô‹7çÇ°?)îÜã‘{oÜQ'ó<œWÇ:oï¯0"þ€žyï¨{œd“;÷áÉ<çQDZÕã[ð'™o̽â°vtvàdžGóžtc½éž¡3齜ü: ûÏTæÿÀ³[o]ÏOCEçÛÁ¥ý­Ý§“yÎ+o¬K=ÐzþOâ­anã…êÀ;ë:¾™xe-ÿã{áŽ÷¾!Æ;™çá¼÷ÆzÖÇ<)5­7;ò«œag÷é)°&¶÷ôÔÁ¾ióÚC¿f0ô÷KÓÖãNæy8ýÿ~é -?`¿ôÖS»«UçÓîaí°~'­È™[³Ÿ“yÍ“¡–c°ý¨@챺ë÷ ç‡ïõ<+p–âgÏ2/ØÌ!Í6,øªœ¬êèäd:( ‡5ÿ¼™ä†;ֶǫ\Õt¤ÍúIdž›xÕðº褭½7hÿ\¯lçÙlyÆ"¥vBKu}K‚ÔëËØ™ÇV¹¼¾ù¡›-¿1Q^ͯ^¦…ž_§ô²¿2Z®ûñ²½¬óÓry}sx嵕__¿äY½~›Êì9·acÆï/>Ÿ§ò©¡ÒG¡4+~õVФ>¤+R¥±ªšMeE«ëNÎ|û÷-U9Ÿ]Ò‘­t\ßrxûpSÒŸ þÝE%+Hÿù|…|–²Ìä¨ëM'¿^è,mË­í‡4Ü ±\ÊV–Uí̇J{à‘æ”Æ Y.ûÁôºÇr©]m åw5qfeMdò)_KÅ 9½Ú’©õR¨B¦óÈ£0åÎ7QèµËFìÚ1m/°g¼IVFD·%Ê¡ºNֶʣèüʦ:߉MôQuj0M«Ã`Î#âóÕš™úÊRéäZTÕ®–f¡R£ºš²Ä6;1¤ç CüLÖ¢-ŒÙøƒÕè,É…«#Ù–.Õt éù¦+¥¹f©ÐRÅž5UÍwQº«Óùû´çáð”@_°0ôiêåšLço4[Ë8øÞŸ/ζžØɯ˜Ô®c…tÕõF¢Êmä(»¿t*̵\¡kVÄGÿ¨pJÈ“/”m.E§iö»^ õÍF^+i¤û†:–Ó€²¡§Ô¥Bh>;êKEɤ'Ðm©›ÆVÔAe=¤*T5Y5¯kd‚°ªŸ†•#QV!P]'»æʯ+r—Óµ®êd«Ì«ÁÝ]+·“}ÃÎבšÅéØŸÚˆ>øÁç+ñ9lv&“ëÇZI'Ó‘8Õ}Ze± Pè䣯™\{kC'úº‡zÙ;7â>_¾½äaN§éLoô=F]êú¤ùÿ¸ØëëÑ’Ë8®eC Ý°uEG2ú^ˆx’LßQ‰­Û DmØïd{ZW½¡ãZxm{vÙ½¦T AUãŽD×u8us]­Ì w@|ž´Ø™µM°—[SGgDС¤U*ÜŒÍÁmì<Ó‡äëP$î(iÒÁµzȈ›±Pܦ.ÙJLrȱ’¦z’ߎ¦0T×p3„tk:Ïöav,ºÒr+†$Íx3lÕXiZÉÃýæîõÑv/"ûÚˆÚ ½Q_GjkQàñT“oýú¶9&‘P;o×P+Ë }Ï´ §wZØÕÍè£5öl%+µ•¶ª´ .@“•ÖJ U&_²UµäGVK¡Åúí¨i7Z¥)ÑVñ›´TnñZ´dZäFw½u²§l±ddh~Ìݨ¶§4•¶­Û®ë®£íuh†í‚ø™ÊžÍmR¾ ÿÞÒw–<Ó*—ªþæ•ûą­ -F›üýZ2z ±ô ÁÆa©äî<Çk'Û’ºv2ÖskŒ Š¾)áŒt›1U¿EÏ–$Û¤ûÀ, u˜“›•¡üU9y=3/µÛ@SEúf,CAo¤/ôRñçaÒfµ|’ÿ RÚVh -endstream -endobj -685 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 684 0 R -/Resources 4 0 R -/Annots [ 686 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -686 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 558.873057 101.433071 540.123057 ] -/BS << -/W 0 ->> -/Dest (cb130-1) ->> -endobj -687 0 obj -<< -/Filter /FlateDecode -/Length 4783 ->> -stream -xœí]K$¹q¾×¯¨³€Mñ| H°×°`Á û øÐÓ;-Aè5°òÁß_0ùJfTVVÏh·h=]ÁGdðãÇ ™ÉÒW…Ÿo4þ‹¤—Sðt}ýñòÓE-Á)üË)6YýüwÕŠÂâtòÑ]d KHÎ}¿à¿érLþ~ù¯_]þg›G_Í’’7Ú]ÿú™Ud—CPNP%¨”¶‚ʹ%ã¼ òzÁ-¦$©`† dh¯ -0ƒRt’ -f1QPE˜a)h»W%˜a‚&I3trAPi;´QJÔÁEd Ónñpº—tF/>&“D]€N#é¬]| í´¤K‹÷ÞEIG°Å%¥%ƒ-N)I[ÈÆ éQç\;‰ ‚‚À²Dl¡@ °^’MQâ‚``‹5Fâ‚`` B‰ ,I#ú§°pç[)J\&[b"’¸ 8ù¼*é¼Yb]‹º„@ †b-qæRKD§—¸¿p¸$qÌ€N“Ä!Át‰ 0-Ñ 2—t¶ È–¸‘.tZK\ lA4(qA4³L¯$.À¹$\-éHñ<3J\É/*J\€n‚™&˜RÔÁЄÄhžl;‰ 0ùÁdÓQ[¬#‰ b4<ÝŒÄ[Œ2›<ß $[ñI\”_8L–¸4R$1ß4$qú%Ï7£ÄÉ*Ì71Åúm²žç›Aâ‚D°Å!Du°ASú-8 óM°“¤ó°Å$’¸ yÏóÍ(q Ì7Ñ‹K‘ç›A\.ïºf’tà]D¦^â\`¾‰ÈVšÛ+¯ ˆÏdeägW ¨×al— ¸£U× È—çHâ¢ÌÄÌ2zqÕÃ*”Z‰Ë -ü‹xŒvœ°®>KTp3cà›Lè„è¾Í9z2{#Ùº>ö×?]~ý‚Á÷ú§ÿ½¼ý*¯”Õ!§&ÄWè&¡¹¢oÛ·Íã2sû>…M©›¨AŠeÉMÄàÇÿ®û»a6ŽÜ˜y^ ° T©<”K'L‰1;0#ØÆ[š^¢n¡ì«½Ì-öº¯ïud?»õ:ãª`Ë•ÅVßëõï:t}nSSÚÄŸØro¾Ü[Å­—ï3ãNÀ6oçWlr^m}gÐçÒÖ®Ûœï#v,9¿ÖãÆû¬6W¼„žn.³öÑ3þÌ}ýÀŸÌ;®bA­y•ÝúinËVžîÉùɪÙOÙîÒ†^ÑÛvyb¶cËeå÷$‡¹‘Çê½mäz\®T?ùž¶âKÄ]MïÖv¬÷¸Úsè7³Ã×t/¼ü¨õ—ùèC0 ÛÚ°ræ¶ÏWÞ˜#%Þÿõ6O:yÛÏM¼Üv)Ìó¡©ÿW ‘\³FFL©C¤Ü…ç¹.`‡á}m„›&¿¨ÄÛgS…•†sÓÖ°%±ìÛã…s=.‚P¢{»—›bÞmÆŒI²ç^nç—äÐfRîXÌS{×C&?ÆEÎÛ¢ÏSp7Õ± 0Mo©]p8OÞ&[Ý0Õ©³aêåm¦:Sž»žNÜ£ø‘Vã}ò€^"¶ùýºÇGüÿ[Íy½̆ÔÜ‘F\»TZð°Ž9Dù Õyà,/å̸OÛ€bÚ[Ý>pâg{ï¢O•fV¸A˜Ä›+ù›÷‚òÞá!ažÌó$SËÆ›”÷8oò¦z‡´©W¶)‹°8@ƒLZûªÓ¬—:EÙN±Öð@žf•ÎãÃÙþ¶B †²á/•@î­àoDKKJ.=Ux„=~p¥àÈ1`cÌûÁÇK´çò<ö˜u°wo™—h]Ë9TO믨 [Ý&ÍåoÁ£)áf[€û¡ü¥ò¹”ù÷6¢íÐÆ[¬9–[—oS£í\ž'A[7›ŸØ8ˆÕå͉un­¸oòV¶›Zx³¬[i2*Ýpm§Ï¦ä©È­vÞA„¸ —‡u²=Ý[]å7÷ëièÕB9“ïr´í·²êç:Éß!45nC°p0…»[V§ò<B½>äó-ž[ùåJž›*Ú×%òYmëyd“cŽAf4fõ~§—‘èÍÀ{ñÜÈ|2ÏÓ!Ñrå=ž;½¨?·ì Îrz$$ßÛš£Ù;Üõ‘ÝÙ¹—´¼GcHÜ–¿ãó‡¶×n yˆÏŽú'ó<’cÌݾ½´ï][IØN›··Fänöç«^M-}ydŽ=ý­ç ÇÍèý$Ì~³·Ç‘7îãç‰6n 5|€wÏåy:´Ǩ?_,öh£õ°´çïfïͱè’‡¸ôìãW'ó<’ÃXöåv|6Ž¸vÜFr+Ô|æóZg•¼Ö ½'íeFJ7sªô¸Å-[Œ„½‹>FÉh =öä ™»ƒ¶“yžmwâÕ9îšÁˆø7D棣^ìq%wîѽ“yžQDZÕó·à/02ß™{Åaíèì -ÀÉ<φžtg½é‘¡3é[;8ù5‰qîõ3¿š³{fêu¨ï`?´®Ü6;_‡ÕÏÏn}t=? qŸ´Hû{»O'ó<*ï¬K=Ñzþ/‚Ö°M·C¡>@g]ÇŸßà-kù_Þ o ÷1ÞÉ°w2ÏÓ!ôû¥þ|‚ýÒ=%~ÉÃú¢%OdîÍ~Næy6$ku'€ý¢@칺‚G†óÇ÷zž ¹NKñ3Ég›Æ[æfü¡œcZÕñÎ|QŽEjø¼›äë û£]®z:NgûΊäÌm¼ .>óëÖ¼n†W…ól¶<ãΑR{U¾Œ¥¦¾%ÁîõeìÌc«Z_éüÜ›-¿1Q^Õ̯<¦Už_côª¿fZ>÷WÙU{õÔä§ÿÔúÚäðJg+¿¾F:سy4•Ùs¾‡]3þtñùì:Ï/U>.ÚðçWO OêCºòyyQYÒµÙtv´¾ÞÈ™…ÿ©¥*'Å+><–ÌÍ:^”ô?åÿñ¢-ìÿüÞ}>ÕYeq4õ¢‹ß/|Ð b¢þòþ1 ËGÕÊ"ÝΨ²WnXҜҺzîÖp`¿î±~4®Þ çw5Ÿ¶Fª&²ù¼±µbAˆœ^ïÅ|÷jÑEÌ'£ÇÅ–+ßLá×›°HcÚ^`ÏxW¬­„n/TCu]l¨Ú£ù$Íæ:ß…ÍôÑuzhþšÖ„¡9„¯Tkgé»(åó˜kQÕ»F5™ŒJMêjÊÖ ÔÚI½^á7ªMh̦ZOµ\…±‰ZºTÓ ¢×‹˜®”æZK…–*ö¬©z¾›Ò¡ncs¼ {ÕüUCQ¶~ÜŠù܇ÖÖ*¸1;áëÅQë‰]ü~qAHí:W(W¡7 -íPnGÕñÒ¥|jÓú ]³ÊøL@þ¥ -פÌ<ùƒ¦)>׳_õJ¸o6ñÖI£Ü7Ö!ÉšBO;¸K‡Ð0;úKG%¤gÒm©›Ç6ÒÁeƒ<¤j“T#­X7Ô51SXõOãÊQ¨ª¨®‹]ƒòûFÜít­«:Õ*óz€»kåv±oÜù>ŠCkq͇´6{[¹æ«ð5l4‹ú±VÒÅFµÑ®W6»a ….î4úžÅµ·6vâ/ž¨{'“FÜ×ËŸ/y˜3Ö>ËÀ[ÍgŽQ—¾þÒ|‡ß¿\èúoõËu7ª±…iܺ#CE<)æo¬¨ÂÖm¡¡ö»XÇžÖU4´qܘ¯©gWØkJÝT7í(t݇ƒØ4èmϸáëäÅÈ|iÛô{½´õæøl>µZ…‹ñvp»Îö!ù:‰+NÚ‡thÉ q1ŠË¡ÂÔ-ۘɀ+i®gûil -Ëu CÁ—¶ë¨³cÑU–ïbH‚ØŽcÁ¤ÇJÓÆÁèw¯/n÷b²¯=€¥Ú›ô}”¶ci#79[Râ-=æai¨ÓœÚ­ å/íÉë™y¨]ž*òwtYz#µHPZ>1‘ ÍòIþù›…r% -endstream -endobj -688 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 687 0 R -/Resources 4 0 R -/Annots [ 689 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -689 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 555.123057 101.433071 536.373057 ] -/BS << -/W 0 ->> -/Dest (cb133-1) ->> -endobj -690 0 obj -<< -/Filter /FlateDecode -/Length 4601 ->> -stream -xœí\K$· ¾÷¯è³—õ ^€áC‰ƒ1²HF³ãÁ8€Cþ~>ªJKÅ~ÌîÚ˜ žÁLw‘zP$õ‘R©Êž ~?µø—É.9—éüøã駓YR0ø©%v•ý þR8[Ci ¶ÄÎ?2Á§%•àRå¬ÌVdzÛo5^€,Î%RyÆ×ÀEÈbRÔ 3ha\Ò0À¥´Pq¤€Ë~¡LYC—ËB)y \ Å’4ðÆ‚ç¬Þ@–@QCo! aæª<Èâ iXàdA ¼‡,Ž¼†hKð <à–L±xà-5,ð\ˆi4,ð@\Ÿ#iXà¹p¥¢aæz`§† ‹éž4,ð@]¢Ó°Àv=å¨aîz²FÃîzš*²À5,:.Þf¯aw½5IÃîb9 `Õ~5,€¯,.g£aw]6¤aw]BPyiÁôp× A‹ÜÀ]ŒÓ°€€»Ž\Ô°Ž²Œš w!AO VÃÄ{ðœ×°-i‰†¸k Ÿ2§V‹Í%jX€»6Ã|*¯,6Aãʼ…–cÑ° w¨¼†¸ øÌãDT§aAîZH­ñ€»Ö£aAî"oÀ‹@ÃķŸ®Ê+‹)9kX€»€%§a¢ðb2D奎d5,/y¨†¸‹ð^4,ˆÀ]¼×° w !íQyÅg§aòð0m•9du1H¡ñ€»Æ¸kL*Dà.‡M bb‚Ë@ÀY°Ó° fZJJQÂX xHùT^\p4,H„ÈRy…iX€Äz)¾d ’ƒ,Þ9 ’ƒ,H4,@F°‹ì[]@¸‹º> È‚LQà ْ ‘†)D¬Ç UÝ’àZåeðlÒ° %ZŠ­†XK-“^ÃüÂé’††9;^nf r†,Î8 à›¼ÞLYñiXPL\8MÖ°0ž1›±Þt¤aæ%¯7³†Å¬7±ÄSæmñ‘×›IÂB% EQyISQæ-0 ëM “Æ‹ÅÒ° ÄÈëͬaÒ ¬7‘Ñ«[™×›IÝ.î†dÒxÀ]d¦QÃ\`½‰ÌV[ÛoHÈÏtfægVw  7 ¶ª[Ø‹h•Ô=ðå5’ºi1±²ÌQÝ5@XÓuÛÀ‘ÑÖÝ'¹Eá 13'dÁ´ d÷}Í1Šù ÅÖý±Ÿ¿?}ö€à{þþߧ§OêNYûµHjiB~…ià -Ì}[ ßG4ÇeÆ(ò½G!•!¢(n[n,"‚ÿœ_vrClµ±*Š¼€U/¨k÷À¡Ú:aIŒÕãˆ9$~q›"'äåÜ¿X]Ö¡)ÙïP•&ë‹ë|tYåøé¦ìð¡ y«ðÇ5þè7Û¤ôë8NĠ¢ -{o¥,´ïó¥°Ø „¤½$bIbÀ¦Ø «|D óÈ#Êøâ8š"±2´+ü»7'ë&źgß/…øÈï,¥ ç7?ž>ûãW_ýþ…¾y:}û¹Áªå‹O->)¯Ÿ!l×ÛµÛ®ÓúéÐ:Ic‘q„¸Ò°L1Ö¬­½ÖN¯_F›ô4õŠô°¶]ËѾ\ë§É¸ë/éò£ô±µ;×icÛÉïŽ<–‘b»þâç7_¾|³w–ºâYÚ"ÆÄxÿî«ò!þõK¸U)Wݪ©ø­p©<™ó­Pó“âf´ºYûÛ™½lež>»Nw¡­Ÿð L‡kty[ý‡íOq‘懶óq ­Ýî–y¯‹Ý˜fýäë®Æww6ÇqX¨amØDW}íÎ:¯Íٰ¾êm“ƺ¶%è´*¬÷Ÿ³E¥‡H˜=›f0’žÕä¸n7½¥ù@O(YwÈDŒÐ]¿ùV½Ù…ìrÕÝ—_ÿåOûzè®Î2Ëk¯í³¬^éúörv–k-‹<³¹o½>з²>ý¦QžùîóÖ>=­Ÿ\ÆÇÑ÷_í»}wQй½ÇtØ^²Ÿu6Ý¥º,U÷m]X±¡kÂl=Û1*)÷,CdÒ 5môë -¿Ò °Ò‘èSÔ«ý -·ý¸2nËr\ŽïábŒ•Ó6.¿i‰úXµÎíÙïŽíDÏ›|ÃëØNµ0{½c¬rûõ¯ZSX¤ZùIXE——ïåêòš1æÚ. ëW¬:¾A !‡WÚ^½´¹÷æaö¸ÖG·›ÛÆÓ¼v‹O¾ÙÈnã~Øó«~Üf;¿ÙÁoc‹W¼YŒ³ú\+ó4ääÃÍ縮uWug}лÍ'¹Ž£µ¹ù ÷ä8›ÌͯÒ(7·¹¢ê}ú¬±òŠ>ÅBó³Ö5~¯§Ù–½½(Æ®éÉ›YOÁ¶v»ŒÂ®óü—rôBÆulg‡ú±êðŠÁÓ%9>8“:¶ýÿI…0ëã·LJÕSšõtO&µâɘ?GÜu›>Â1 -Ôòe{Ë~òðVïj¶Ö1ó^šdj×t­™t¶bÕ~®µù:ïEL|ÞÇæcPÁYâÛWwŽî¬ó^·B>â–Qèò´¦ˆºeT§Î¼×m·©çEX[p±Ã­>N4uØ௺B[Ì×­•/®·è<”q‘”ÃÓ­Ú”ëé»UynÕq)6Ój×€>oåú›rõX;…è/hˆÞM›]S» 77,uØ,›·’Ÿ&YÃh¯ß ÑÞnëwªsSsðÓ#”ßÇj¡Þ³Eµùí¾]ÄWüÿˆV Ñ.Îædæ‰$ý:l[…×û˜·,u<ãS¸˜ÕMJN"ÂÛ} ß%_þRBuLXø¬sáS…S§.f²©_àsd&;>?q0ï¬óJ¶Ú;n&¾W{7ÍCK§[*½Í*á;­uÖt_Y.4©ö¼E¾†Þý,¯yàÈâÝê1½Ê1awó§É–öŸ—nrõ±^ÔÁ#×¼§ ÃæÓ7î®*¯Ä߆Ԉ9WümMÛLøƒ¿®÷Ã-˜û±(ñ©ÿËXÔqFl#Í[ þP*8L(!';ux ‡²óS–ɹž•¼~{ù¾:¯Ä/:eàô5¿˜¢¾æ#hÛF)ãöÖ—ö¼]™ïÆÔåê¶qÚoN·}Òö}kó-»9žeÈÝs<[— -á»q˜á®:¯ÄÛ†ØÑ]]-£ßúŸÑþÂi”]Ýv#x²ðîÈC‹Xió¼K‘Ímušç69oxæq»:ÛiœuÃc[;ãn÷Ó½×*íLº«+¯)j6=·–ƒ‡¦Žm¼ë—€Á”nyè}u^‡æ«xx¯Å«•îó’×€MÍÛ×m¨í»Ù÷ó’@]΂UEÎ1é¸xî«ÜËwFæûê¼6OäÓú×<ñÎÝ}àe¶ì˜&ïÑ<ùÖ±­êsvq»Þçäâ«T~‹Skªi×™NóB¨Ò©¡•ì¡n'…ôxRˆŸšÖ4Á˜/¬ÆïÂX‰¹9õr¥•SH'µÜÖZè–J½TUKÓüe¸ºÏ]ñ -íQùLü‚H1GŒo_÷d>•Ømm²ðw >žõ™8Èϧ”Òa`… Íõ$Ñ‹v;9›á/ƒÊÏz®ß05ß$ÀÿW¨ÊÈS¿Xê.ÅoW£ž›¼W’¤ÇŽ:¤iÀRe…ºlJÝg¥¾l6JyÝ^ºklG*ôTšx R<ödÛ½®“š~:VJ¢iB »AÝ•Ÿwä!gèS5˜ÞY´ÂÝCowcÇÎgINÝâ–_íÒåíí -ŸoÄÇØh&³ëçÖÉ ;Ó£ÝèL‡`…A0ú\Ém¶vtâ×U¶¯c’i÷ñôé†9çƒãÑ[~S™Ìºìù?(óþþy¢óŸÛ«1Ö8îLG ×±uG–‘Œßk™ï$ó{.±OAt$`meCó†ÇxM£ºÀÞJÚŽ ¶s%1  -²ë®ë¬š Wˆ“{"ó¡¶{½ômp|‚‘_ªÒ¤Â….óàù’Ï¢I\qÑÒÁ%'*âB6ŠKÑa’íÄd‡”tÕ³ü$Má¹/q!Ò¾ôƒG#Ìʦ­ŽBA -ìå…l˜¬ì´ìäQ„~ïéõÁvßDŽm0µC{§>KjS;àéT_/ãþ²Ç_ðµÉ;4ÔÛòbîùžäŒ©1R Ú]HÔ<ªm‚ì”m×i .@3Ö[LM¦¸Ukj©#Ϊ–RÏ)õ©é ­Ò•¤Uâ¡,·Û"^Ï–|O€‚tç4FgFÉžKf…eíN¥QÒ7Ú±oÚ÷Ý¢íY ƒ† qd¦fT ‡’ï…ï3wÖ:Ó.„VªýÖ›Çˤ¨É&ï|}?¸É1Cˆ_ñ˜(‹­’×¹¾wrÜ!i{'²'˜c!ïM²ü¦w]„g0Êm†hª½^¯VŒ:¤—)@Ù0g;CõU¿u¯²nõËÄKE~³·ç¤—ͲÉXý= ¼vÛ'õ÷¿ùM-R -endstream -endobj -691 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 690 0 R -/Resources 4 0 R -/Annots [ 692 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -692 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 367.323057 101.433071 348.573057 ] -/BS << -/W 0 ->> -/Dest (cb136-1) ->> -endobj -693 0 obj -<< -/Filter /FlateDecode -/Length 3527 ->> -stream -xœÕ\K$· ¾÷¯èsË’H½€À‡’ bÄÈ"99Ìv<6‚Ùëò÷óQUzT5û1³vÜÙFo·H=X"ù‘TW;Z¼¾pø/³39—ùxúxøt°&‹µÇ¦QÙßâÂÑYN&¸s8~%“Jð)_øoל»¿þú«Ã¿¶cÜÑ›R¢wáøãwÂb2)§dƒÂ*`YG -+“’QaEgp‰¥h,ˆ{>g%ˆÁ%1Øû¬°2Ä NŽÎYbøäXcA WBRXÎBç­UyÄ2“Wx.˜ˆMÏ;sñEå%ð\òÈÄÄ.8WLŒ1dÇ%ë4^€,Á1[Y˜rÒx²P$«ò ‹Ï‰4^‚,»S^†,°®¬ò ‹Öi¼B&”̬ò -x6§|Îó6˜‰¬ÆsÎØ&©Ã)‘¾ñ$ È:ð¼Ó€€,d 5$ Yž«ò Ö°€H0p×\‚¹»>X¯aw=û¨a Ŭš w!AO VÃÄ{ð ŠFÒd ãàY«aAqõ¦g à—Rof -YÔ›(ñ¿-¥ÞL†,)ŠÊƒ,HšŠâ·À$Ô›@'!‹/¬aA‰QêͬaÒ Ô›ÈèÕ#,õfR €»¡™4p™iÔ° Ô›ÈlµÚÞxCB~¦3³TœY=1°€Þ€ØªX`/¢URÏ ,ÀWj$õÐb¢²ÌQ=5@XÓYõØÀ‘ñ&,§OóU„9$ÄÌœä" Ü‚‘Ý÷šct£ Ý–ó±¿?|ù„à{üþ߇ç_Õ“²örHjoF~7ðêȾ.Ÿ KðU4HöªÎ²l*C, \ÙD,<ùw<ÿ²‘¢~[×¥¾nAÜ®ëz¤ó\€óP@FÁZO'Q_9ä'Ø¢B|èCÅ ÁÈIZÞtþÍûLs•b9õìÍI|DH”¿ìÃñýÇ׿÷î·¿û3 çýóáo¿¶°¥¯¾pò™¶Ÿ!¬í¼¶ãöÕ(ï€÷Þn¡;ºð]ä’¶]Þ¡­÷¼Wx¼~çÓÊ£µýa’'´þ_ýýøþÝáë÷[¥ÕÜm±/Ie/ò-=ß7äsôüs¨…àõvµåÝ–&]½¢V„ëŸñ‰Ô‚ýJÏx£í…÷aik<ã›I­|‚yøÓ:ï‡iñóZݬæ¾OËd§yã*Ë:Oƒ¹b±}~ýu D¿ ¿nbr>ýZ»sÌ£j„kVöÿ Ý7[éNŽ}»ÊEÓ<øN~•í´^o\Á0‹ì¬‰^XwŽy8k⫘õÓùí+-s²°WYf\¬êOŒ;¯±È VÞ€Q÷y8«ºžè¼Ý_ "þŒ–ùê¨7Ò$.…ˆc)þ®[Ô}c΢®çV¯Á_ 2÷_·?ò–#8 ‹7¬çÎ1f=þzÎÔ‹œVù]¦žFAÔ - Vôô¾edô¼ÏèÃÈè7…WÔûmæÚõ©EïxQ—;¸Q]h²÷ -¤Œ9÷ÕH]ﻜ§•>Ëù¼íS‹R?I[Y¿`•S%7À¤åœèºUÞ7æá¬òzîÕv½[áþ -Ÿíò®–Mì šÆª–n¬õ‹XkÚö;³BwÅ:×£Ž~\Ñxvùü|/¼`½oÈñîópÖ{ã0k©ûë Y5º;lªZjÉCûü´ÃÞt¥5j_:ó똲µˆ¾ÆÞ«>LV»ÇÐ(Úx~È<[ì%o;ó·ÃÚ§ÏÀ×7œÆÝ9æá,ôzÎøHøÚ½¥,Ú½ÇJæãÙÙRÂîظãçóîSñ¼†/køý¤Œi8{šÆØí:¬ÝgMí3_S›ïÖÒt~gÙÔBæVõsç˜G³dº‘¿~VöXž ZÀk¢¹›,^ÉeÛ˜ ¶îNâ÷_ez:A÷f4òS¨_9ÚrËRïóp–z=§=Óà>Šî4W­7-ѺYÓFÛª1Õ#N“Æ×ÊSå³ÿù«É¨þT¶÷ «¬q¡¢R«IΫxoXÖÈ9eãˆSŒ·,ë¾1gY×óÍMŽváÓ9ëV1[ZË×÷Fóeíô\q];†"Uâ’äŽ,QÜ×ßüéùfúA Öç¿[\ÅêO§å UÚ¢À -n”…>Ȇ¬ W©Ê”“”f¸•|Öƒ\ù ây 0B÷Ò7.ï:×ú½Î“›âÖõ$Õñ’T§…¿™?¬kOòÔ9Ât ¡]ÃW{5~:Äzg¡«w—Ûˆàåñ³cd#çÓ©ånÆl‰]S›«íŽFÖ‡ö>õ^ës|Vní—Û:—‡ü¦ÆÚÿSUøǃ+ldÿÅ—gnl%g߃ürg®X!3òù;ÉB\¿Ú>»Ö±ÓN@²dʾ'cׯ¥ON¡}EºÇøІɽpl['ªwƒ- +DŒŒîœ,W]ÉòÜZ6´¶b%Ä&3ˆcBžûŽ ÇÀ›dʼnΉvZn=7yœÜçÜ·.b}Þ:7©¿õõiRç5âé€eiO}Q©ò´L›ªí®·F“P¥SCëÙÂ]O -étPˆ_Ø65C™?iMî9^ˆ¹÷~¥õSH§ƒÚo-tM¥Þ+¡¥íüe˜:å¾ñ -­îÑJMò îä#–Ú×-9Šš®mžìÆŸO‡ÀÝùå’Ò; ¬°¡™ÞL¤iÞNÎvØË J²¼|ƒk6šÜ±)ÿ/P:U§~qÜMJ±ˆøf'o7i¦ÇŽ:¬í€ã4úNÛåRê6;ï—ËVé/ Û{÷ÛP§-›è©4ñ¤:xlÉ®[]' „µýéX9mË rè¦ü²!9CwU{[,ºÉÜCŸwcÇΗ™œºÆÜBßåíóN6߈§ØxOÓÏm‘Aö¶G»±ØD‚yÀèK%7oíèT9_¿'Ó"îéðá†9OÁ|‰ä䉰9ërÇÿ Ï;¼ÿyàãÛ-ÈK÷¶£…ïغ!Ï‘LžÎw’åyâFìn3=O°?È.¾¡YCãÞOxÍc¸ÀÞzºŽ ®sgb{8‘}7]ïhÚ™p…xÚíbOd>W7#`/Mj‡eHn^oR¡1_šyðh„äã4%ZÒu„tpÙOј'EsZ° É6bŠA΋ô­ùyVÉZScJ#¤IƒÇ#ÌÎS7Z½Š© R`šóÄìæEËFEè7»×gë}96j‡öN}™©ý„ÚO§RmÆm³ÇÔýh4ç;Ôç¢É÷¨'9Ã5FjÁ›Æl£5ak‡0/ʶY´Š ÙFë3¦&S\‡µm©WœÕ]J=§Ôæ³Vú&†Y+ñ¬¯ÌÛ"^Ï–¨'@a6ç4®ÎŽž=—Ì --Σ;•GOj´óµy»v‹¶Çé2xGfjÇ°pÖóMø÷ßYÆìN!´^íUOF\í P³Kä©þ›c†°©¯ÿnpœä -endstream -endobj -694 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 693 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -695 0 obj -<< -/Filter /FlateDecode -/Length 4517 ->> -stream -xœí][d9 ~¯_QÏHrqnÚ$,Áâa¦v„zøû|Î9I|N¹n½=»èíí©:v.Žã|¶S©´;Zü|åðO!gJ©9Ññôéðù`MŽÿµ›‡Æþ~s<:KÙDWS‰ÇOLÙä}NǧþÙ=ÊâO‡?ÿäðÏmwô¦Öä]<þë;fQ0¹äl£Âª`YVŒ&g“ÂJÎ`ˆµj,ˆ3y:geˆAµD1Èû¢° -Ä”]8gUˆá³#1\Ya9 9œ·VåAK¼ÂsÑ$(=i<ïL*ÕW•—ÁsÙk¼LÊä¢ÓxÕ¤”bÑxYbµNãEÈ‘Õx…BÉ/A–‚UyÅ—4^†,Ú© -¯@XWQyÅ&ë4^ &ÖB¤ò*x¶ärÎó6šXB°Ï9a›AåaEa’RVx²$«Êƒ,1X¯ñd!üh<àD ¹eµz^ÝÁ§ñ"dñ>“ʃ,˜| |‚,6' °‚ ã’†>gCÕ“¾C…Š†¾TC9 |†RÍëÀóN‚`!K¤¤!Ap…°rUd •4,² €†!@OA´%X‹†pK¶: ð–¬K.Ä´ n(‰4,€\˜RÕ° s°SÂÐÅrÏ nˆÉkX»JÒ° w9«awC@M•Y`ÀMp%hX@ÀÝàlÖ°€€»XF^Ã̪& `+Æ—b5, à®/–4, à®Ïp*/¬Á aw}Ä4Ï ÜõÑz ¸ëÉ' `(øcÕh¸ ‰z8Y°À߃烆hÌx„%D஫pÊšZWjÒ° w]Áô©¼j\†Æ•u -—RÕ° wᨂ†¸ ø,ã <ª×° w]€§ÖxÀ]ç«Õ° w7D `Æ!аþÍ8 ÓUyÕØZŠ†¸ XòÀ [€$*/’Ó°`l,âP pî½jX€»6† aAîZBØ£ò K(^ÃÄ5àaÙ*kÁª±H¡ñ€»Öa†5p×Ú\5,HÀ]v›¤Ì_4,€€Q°×° 25ç¤aAª<„|*/ž¿y`ñ 刅"¶–ý¶øÞßÛâ{pñ©‹ì±ó‚|[xo ïÿlá=r")4Pxß5.âfkZÒHG_Ož7”׬ò9u®g¢¼µî#Õãx£ç¥.8Óv*hÓotÈ{­I•õá:/.+EËŸ'ñ>ê¦_UÈ[…oJw×ÜŸ ƒÓú½ åÍÒ/+æ´·ZÚY:¾×Fï¬óâó.û½×Fï¬ó²žoõýüÝk£§Ãk%CTÚ'|@'ó§ÇwŸ?ýÅï~ÿ›?ým¾ûxøËϬ…DZԺ̟Èáÿd-ñïüü‚ùõãòã×=¾ûæà0*;Îä}5ÅÜ9 ´áCá®÷½µWZzõåë¯øÑ­¯i}íô¸ôÎean¡CïíùŒ¾–Eí7†¥èW~ÅïiÕiöKï—z­Ý¸<ìª.ãÚo«¿ÊÛäHS)[ï»Õ©KÜÿ,ÿ€fë^³óåIH+zl’¯šêsjOkoií&Éöa·p¥ ¸Þ~Xg ¦iÜžûö¼ø#$>¦tÞNÓF×pj£¿&Gö—ä°s,m&ûøx6<Ï’*WK–°•¶Kœmn-kZÕí52ÇɲPo'­c/ký“°Ä¤aXËi¶ß,ÛO ãºíù»UŽ8åà6F¹uþœ¿ªóº×Ë°'dî¶{›çý·•w‡® bŸC»ð.[êe?/CÎ$ÆÛçÂ͹èåǘÄüìWíÒþ5}·×W[#,ïûÝG1Ö²•Õ¹KÈ¥¡Ö‚PŒ[D¹)3•Ì vlm¿¯Ÿ_¼k~.ŒX,“ µù9>Fæ‹óð„&©•ðÖy–o\]¢ñ|ÐãEGž?¶&ÚZ,íý¤£Þ뢚ñèBò–Âço}XÜ毿ùæ—¿ú£p›<qîSw^ =áÄ–I¸ØaMÆV>"´ë°Ã‘k&2– __oÑ#m‰ Yi1~¼UvÁ'êRRå¹Uge«Òj7W‡«[4njÈ·£êS¸ !ún*Ю„VZý ´šwõ?îdŒÝmß=…vj$8MÒ›šç³“P]Êú8oÕæyžÿrš§â±Ê -ÁU^¶ÍX×Y¸Úd¬¼þTL"ÊمⲨ²cŽ¶tЄKAÊyÀg+ŸöÛuÚVöУvȸÂ.>³ÒÎ8]½;ë|Ÿ|ëK`Ÿ½‚}ö}w7Û#ÙeÅô`ú´å7Z·²îÛ[\ðZNÏ÷÷ÿõýÖð¾f}òÖðE†VÃú¤ .ª öL­‘“±m‡×,/"ˆî®³Xã1>_·¼;ë¼6˃Z¯Y^G˜‘Øô×<­£¥˜q}å ˜áÞ_àUÁ »2ß®¯+ßûY®ý~»{¿Öûa}R/Xùa±À&fŠ*Œºn9÷Õy%–3ÅæïR]¶œ[3ÒgaX”ëáöÜ`XÒ®­ÕõºK -t½ŸaÁNi§?‹öÆFHÚZÂ=} ˪‹ü-–ü,ÓõM’ÓÄéhÏñýªõ–4-‘O~—vZ÷ºõÞWçµY/â•«ÖK[íYØ[×ÎZšÅÒı>+ ³´÷´M>÷³{&Ç~Öe¶²¯ÛeËS¶¸Ç°ª²·‘Uí|Àf•ö>EF¶MoD ²¿ºê/Îø{o¡É ¬ ÀJv²ñ–g¾³Î«³P_‡Å(9q·–Æÿ‘,y`ìnæ›Li's8ǽiá,!<ŽUwÖyu–p#FÛmÝ­^ðr-¬ÂØüÇucu]לïy<{æ}X7f×ñݪV~à5ZÛý êѬ/û–%˾_úà°p´›VYÖvZ5Ž´gºü¬LÆf÷ZÙ}u^•]çþf÷ÙVº“cÿÜä -¢Î=ý*ÛioZñ·º?^°¦gÄWwÖymÖ”¯ÇW/·n´LaaYfZóÚüÒ¸óˆEêV•ÝãugWgU×c¢ç¯×Wà¿ e>êõ²ØËÊó…ÝŠ²ï¬óê,êzlõúgð‡öÌúï'(2ò`"tóXL{;n_ú°JûøXJÉ=wZÞ·ÐÇ'[¾èá•»§ð—¤§r‡>{õ~Õéšg½_s·—=ÒR"ñ©7±ßþ¼#-çírô¥Vo ‰û¾ÜYçÕ}Á$ãs¬>ý@g_v=¾Äá¥ÉGN¿¨ÝüE©þýÏ¿\ÐÒk<óí‹0ÏѾ8órÚg`®Øè݇`þ°ÞÆ#º‰|eª¾Vq¹ÃÀ›EnÀE‡ÏϸŸÝ°=&ùÖP° ÕÌ7—\:á¿[¢‹e¤5öï(]E½*Èw4wÓƒ¯oÛ•:¢’îYëB÷4½ô 3ÂÞÉ¿_‘ý´õœ£ýîA„<¯[×\Ãmr>Ÿ©ÝÀãÚ-l6ã<_ÓzLd8ÊõÈ·þÈõisMÑîx¡f»Üöó(µÞwkù -<¾þh¹ W<¬å?· ÿ„àœLêáx»›Ò62Ì8îÉO¾›”2ÑŒéo™¸¾µ£-r#Eè´" jWNÇ®ogÁžÈ®ˆÚGÃõc¯ÆKœl/Ú­)KÇ -5“;'óèì¬d¾ßµ˜°>¥! -#ú ÎI– Ί7É.XAŒçD+º›dO]Ç÷ Õ¥I¢KÕ91ý½¬Ïb:¯OtöÔ'•Ê·Jö¦ºv½´ „ªƒ{É1!4æI!ñ·äΈ_ÙÞ4a2_Ìß͵K7$åj/§NµÜÚZ3•G©2«Ö®ù)Ê4õP†âÚIäÚ™/¬kĆþvKætp̵-Ânüñt@*KgäöÍÈóÒqb…Ýô$1ˆv¹Øi/“šKW–f§ñÍFüïqPyÚGäøv²ù4;áµ9È[%Iz¨CšåYV¨Ë廿‹Ló¸§ÃßÍÍù=§l AqÞ~?Òÿƒ2ßà÷:þ¶¥wñãÞ´ð[7déÉøžír'™ïÝîılÑ“€ýIve–݆÷^à5Íêv{/邺Á•Ä8u(È~˜®wAh&^!žvZÌ÷›é°—ÇÐÇ (_ò֥ƒËä…é’¢I ÿž³vÔo©k¥úOÛqe^n GM.Úß+±%%`ñ•Ó™ŠØ*y¸Îõ½“Ëw+Êž0†B°Ùñ_žÑEx£Þfˆ¦ú_¥Ñ«U«é1([CæÎîpíO´Ï¾ÛÑxÌœ*ò_áíºRø‚ôl/Ü1Ð>NÞlŸ´ŸÿÐÐ -endstream -endobj -696 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 695 0 R -/Resources 4 0 R -/Annots [ 697 0 R 698 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -697 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 563.554475 101.433071 544.804475 ] -/BS << -/W 0 ->> -/Dest (cb139-1) ->> -endobj -698 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 100.307453 101.433071 81.557453 ] -/BS << -/W 0 ->> -/Dest (cb142-1) ->> -endobj -699 0 obj -<< -/Filter /FlateDecode -/Length 3427 ->> -stream -xœÕœKÇ Çïó)æl@å*’õ8HĈÁ99H#­`@Î!_?vw=f†óØ•ll4ØÝi²ì*Ö¬š…½ÇëUÀ¯"Á•Rs’ýáÓîóλ=þ-%Ž.õüä¸^²‹¡¦÷ŸTÀÙå)§ýã¿N.ç⻿³û÷q°'Wk¢÷¿|T•°Ë%g U…Ê6T1ºœ)&C•‚Ã-Öj©`FÌBr®Ê0Cj‰– -fQ1Tf°äÀçª -3(±T0#Ô˜ Uð°#÷¦†x&C¢Kôdé(¸T*US—¡ ™,³KYB –®º”R,–N`K¬>Xº[bñ–¶—léláÄÞÔÁ*™-]†-„Ñ©†®ÀxW1u°Å',]ek1u:_r9ב.foéBp¾É¦+ -“”²¡#Ø’|¬¦¶Dödé¶^–œˆœK1V+éêfÏÁÒEØB”ÅÔÁL¾EJ°Åçd!+È)—,PÎN*‰*ì¤H±(@¥:É™- PNRÍØè(X `[¢$‹`‹`åš:ØÂU,0Á°XÀ [HØbƒ¶o±XÀÀ­ø,0x+>$‹ àÂLo±€A\.I,0 Wª Ìe°ÓbºXîÙbƒºY,``—¥$‹ î²o±@À]fÔ4u°l±tt -[,p—ƒÏ ÜÅ2"‹˜U?L à+ŽJñ Ü¥âÅb€»”L]vXƒl±@À]Š¸+rƒ»=Y,p—„’Å8Š¼™ €»@H´ÓØÀZ,@¼‡ŽØbs„´ÄbAwCEø3Ö4håB©ÉbAwCÁô™ºêBƈë£ìBJÕbAw¨ØbAwÏb±Œwˆ¨d± ‚»©-¸¨z‹ÜEÞ-À\@`±ñÍ×5uÕùZŠÅ‚îKd±QØù’˜ºìàHÁb`ì<òP‹ ÜEx¯ ¸ë#³Å‚îzAÚcê` ²X€¼:,[cM#Yuž@ -Kîú€¶tà®÷¹Z,HஆM‹)«€ŠÅè“Å‚TÄÕœ“Å‚T=tHùL]r¸Ž ²‡ Y¦¶H‹H¬]åZ,d‚-Ld± lAÊ`±«Ù·¹=€-ps °™¢ÅÌ•*b± Ç„ýFÕÒ%r%צ®@²Å‚œÅ!‹ØK¹‚Eo±€wš.Y,€Ð±X+lÁR±X€Pä -!3·t¶ ɶX€Lº,‚-È-*Øe&o±{I µ¥¯ûÌb± Hrš*Z,À2ÁN¤4u°˜°X€ùÑÍv´X€Í6›Øæ˜:ØÂQ,”BºÝ, J-äÉb|S÷›ÙbAõ°ŸÅ‚ê“Ó4Ùb0÷ j(Øo’X,ÀºÔýf±XPÙc¿‰-ž±n+'Ýof‹U`KDŠbê` ’¦j¬[0 ûMÐÉÒ%ØBU,Ô”t¿Y, ÍÀ~½y$Pt¿™Íãp7VÉÒ»ÈL“Å\`¿‰ÌÖÚÛ{€7fäg¶²èŽ³˜'舭æ‘{­²yfà_Ý#™‡0;Ë’ÌS„U(ƒ7 <ø‹|LΘ°ž>ÍGT î3KÖ›¬X‚ì¾ï9F1¾Pl=ûå§Ý·ï|÷?ýg÷ðÍrRÖ^ ÂRZàÏXT1ÝÙwÀísB"§qY¥¾gTQr&@q;rSüôßþüÍ‘Ý0ûÍqïaF÷DÕ´õfé¯hæù°ÿþí|YÏ6“¶­]Òã ,2-àFo?í¾ýþ‡¿þùÇàovÿøœ¾xß㈼„ “õ}ÖŒIÏÖë¨:•?h™ïþ¹ûzî“t›]5¡=îS» Öâß¿]ÆŒûüfdIëi㱃ÀÂ-Ú—xb•/ëmˆq HbpØè“B+wçålÜ¥u¾E¿ »s`B€Nq™†?½~ý‡?þmLƒ‡1B¥€¡ú>|÷*è¸ñ…÷²ÖÑé‘Ã*‹iý»ÈÊ&²õ½Èö÷a”Ã4®åhȾĦÕ]z;~ý‘¸õÝl ê^[}?Ù–ÿÆV¯ßc¿×‹c`û›ºÁæ=ð£DÈ{oùÛ}U^ˆ¿ «±¯ø›Ž›óæ‹?ðõqï×a«×F¹é"‡"úuÈä21n^2ƒ¨ª§l*‹5çjD"}Òá5 - mŠÑ )Ýp‹;뼿èB*vÕ/â1,élп˜…ð~å„©«“ŽOÊ|ŒQ9Ñ(·ü|8y¿Õ[ÖûÇͮƶU¦tŸmß}çK»~8ñï8Ýï° žƒ-V‹F¬3%UÓÀëžs_â9Ãlä€W<çÖŒ´Yè¥k»r ¼@@?ëë³ë.ºýtF;'4ÓöN#Jó„{úêžUWû—Höyf‹¾ñ0¢gôçþª÷Fž¨ç\e9›¸î½÷ÕyqÞ›®rï¹ùÇâ±OÍ•ú¬œÏ³î§²§u›myØOî£{U9õóØßVêÑ*m}Îyܶr—r'9æiÞpÔ_ÝÆ/^Î×(wV2X©A6ÞŒÌ÷ÕyqZ¯òõ×Τ¿Ô“;cOf~±)ØÌçÜn{û§³êÎ:/ÍøFŽÖf®œŒê…(·¤U¸7BÖÌØÂ˶®uëO¸&Õ½_¯-Ö¡æU›žuͶvßOõdÔŸûêž5—}·ö¡iao7m¶lí,u¶<Ò?<ý>tWaÛÁËæÜì^/»¯Î‹ó²ëùÜÿÃì>ÛKOì8½^ìâ©ídc±í°ÝoÚxXÔö ÞôŒüêÎ:/Λ®çW_oÝ>Ñ3'{’g¦m_›¿6wžâ‘¼*?ƒQ÷Õyq^u='zþz}ñWô̧F=™Î²rÀ<`ÀneÙwÖyi%×s«—?ƒ¿udîófûX/G nÔGšõî:}JØÝäf‘^qþ™bûln¯¯Ë{¡I}žŸËž0ù5ëGà—›éã:\ËÑcÚ†5mÇàuº°%'Z¦Pë†+­›—¸l¸ÖÃjù86DËauZËëaµnUN²}Š¶éÚû¥ÒŽÕ·þÞm‡ÜzÀ³}"wÔ~Üúžì‘íà¼ßCl÷p6Ÿwiy”#,óy ߤÏûï“8õ®\÷úøHñ,¡M[X:ì/Ô\¾%ñ¹—Ú¾8áõYJ}ŽfýVÅt±•ÿ¼Lø§]¨âtüuÁ®9ûE\¨] ñãNrC,âèN± -··¾·%¡ì²ÃN?M©§%9¶¡CíëIûú–b»­[5}ø@|+ÄËÇïkdž5S8ëÝ{6±~Q 8Þ®R7%¦f3„£A™ËŽGÅ›âÀ~Æs¡Ÿºb’fOÐËúÐ¥!ì¦ÏC¦éoe)OÓyMxØ¡[>•>šR}<¹5ÕF—|—ñdTíÒØJö ‘>O†è°3„¯|kZ0™]?Íš>äµ -Ks$éåj+gˆ;³ÜÖZì3•{©2ªÖ6òÔáê\úÀ²eŒ6iÖo>Mk™µy˜ÅIg Íµ/“ßЙð°‹ÒWâ?îb6JÇÁ -›ëÍBžÚíâ⇿ )2ém¨°4›L‘Ñß+*b—*y–7AºKécnãjt¢k³‹i–§N±F He§á -9wŸÇ+o”WèöÒ}ÄŽ¤ÓMò\›y -©cqè^×ÅŠ°6>•³Ð7#ÐÝÇîÊGâagìKÎÞ:Kar÷ØÛâÔÙù8‹sŸñ Ï,v{{»“Ï7áa lr*V×/­“!&ߣÝèlÃ…!}\Ämµv:é÷°Ú۱Ȭˆ{Øý¼[Âq¤Š7‰ƒ>‚?g]aÿ_”yŸídÿ—öÌ×ÇÉwZPgë‘xŽdú…­r§X¿ÀÕ„}ÙLB’ ûCÊ(›7ô8N4ñZFu?ÀÞJ†NÐе³0Ž1œÄÔ]—O#¯'£Ø™/›°×Kn7‡nXŸlVáb¾\–¡ã’÷S“¸Ò¢#¤C+4UÄÅÜ(.§ë°ìÈLuȹ“>ôj¿ÌSÁÚ×t1¥zÉC'#ÌÎM7ÙrS¤À<_Ì K˜;­GöF?{y}ñ¼o&§¶TÚÑÞ¥³´ß˜J;ðl)/—éø²Ç^l‹wŒPo‹§µÇ=ÉKc¤rt1ûè$-£ÚV ÎJ“uÚEÇd¾Éz‹¹Ù”¶jmX–;.æ(åžS éÏóHÇyVú ÆyVÒYYm·E¼ž-qO€âìÎyÜ%{.Y Yškw©Œ’Üdç}Ëqß-Úî§ÛaH™©ÕâYÉgñï9kÇ|ÄÖ*Õ^ËÉH(ã)ij ™‰—/¾û’"úÝ¥,e:*yrëg'ç'$íìdî Óá„Ùç ÿ…mÂ#õ¶bjªý÷vµêÍ[zÚ\|ÜÜ~ zùër¬¸õˬ[EýÊ:kÒ[ô›vÙÛÏ%.'uåèødyýTåC÷ -endstream -endobj -700 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 699 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -701 0 obj -<< -/Filter /FlateDecode -/Length 4782 ->> -stream -xœí]K$I ¾×¯¨3Ò&ñp¼$´$,±Äaºv„zü}>GfD83]¯žÙÝÛÓêéJ;‡ý…™i?_Yü—ÉN9—éxúxøt0S -ÿj‰ÕEeÿ ¿)­¡4[bÇLðiJ%¸/ü·¹”Å_ßþìð¯u{tS)ÑÙpü÷f‘ŸRNÉ…UÀ2Ö+¬¦”\ˆ -+Ú C,EcAŒÈÑž• •4Ä ç²ÂÊÃS²~Ï*Ã%K bØ’²rXgŒÊƒ †È;…gáô¨ñœb.®¨¼žMNãy?ÅD6XW¦cÈ K(Æj¼Y‚%2²ÏIãEÈâ£7*²¸œ¼ÆKÅA;EáeÈëÊ*²˜h¬Æ+~ -%©¼žÉ)ïy΄)dïƳv -°M¯òàQ˜¤˜žƒ,Ñ„¢ò KðÆi<Y?8|ÊYñVÇÞí·/@ç©<È‚É×ÀEÈbRÔ 41.iàRš¨8Ò@Àe?Q¦¬¡€Ëe¢”¼®„‰bIxcÁsVo K ¨!·…à¹*²øBxYP@Ãï!‹#¯aÚ¬Eø%S¬†xKÆF <b <×çHx@.L©hXà¹Ø©aèÂÝ“†¨ëCtxÀ®§5,ðÀ]OÖhX@À]ïQSåA°†@ÇÉÛì5, à®·&iX@À]¸‘Ó°³:Á£†°•Éål4, à®Ë†4, à®KXT^šàƒ^Ã!h+7p×ã4, à®#5,€¡LÀ£FÀ]@HÐÃÈ€Õ°ë=xÎkX€Æ&‡°DÂܵËŸâÓ@«Éæ5,À]›1}*¯L6AãŠßBË“±hX€»X¨¼†¸ øÌã'¬¨NÂܵ+µÆîZWŒ†¸‹¸!hXs˜, °¾MÖÀtU^™LÉYÂÜ,9 ° -O&IT^š`HVÀñd‡jX»XÞ‹†¸k‚÷Dà®!„=*²øì4,@\ÜVñi«“q@ -Ü53¬ñ€»Æ¤¢aAîò²©aALLpYÃ8! -vÄLSI)jX‹!ŸÊ‹®ƒ†É€Y*²P" XOÅ—¬aArÅ;§aAr!ƒ†ˆ¦b}«éd¹¨ùADŠɦ\ˆ4,H!"ƒV5^tSN€k•—Á³I”hÂRl5,@.5e8½†ø‰Ã% x–4,H²ÀU4,ÀR4e‡È\ãYÈ‚ [ÃDºàY«aAvÑ †Ùed™ÑhX€¹$T­ñÈpž™5,È'5,€› ÓRª<ȘаóÃÉvаÉ’M¤9*²ø@äì8ÝÌä YœqÀ69ßLYñiXPLœ8LÖ°0ž1›‘o:Ò°~ÉùfÖ° xƒ|)žâ·ÅGÎ7“†… K@ˆ¢ò ‚¦¢ø-0 ù&ÐIãEÈâ -iXPbä|3kX€0ù&"zuK s¾™Ôíàn(@&ÜEd5,ÀòMD¶Zno¼!!>Ó™™3άî@oÀÚªn`/V«¤î€/çHê¦ÄDf™£ºk€eLkÔmüE)°±TÄʲߜïÍùÞœïNçSì¾óŒC¾9Þ›ãýÄïžIáä)ò7åäã”Ã,n2SMéèÊä¢ãå%«|MË™(ï­»@åØ?èy©õvª[´ê7Xä½Æù¨Êzw/.k°™¿PâÔU¿ª× -_•î¦¹W„,Eé—øŒ…ÉÞe]Ø+}A¡÷;T¿|wÀŒÖc`ø[h"Êõ ,>W’xþøîãáç¿úÃ÷—?`¶Þ=þú cl=waŒËÆÀt Eü¦™÷4ÁÍ×”¿þÛñÝ7gúà³½nÖŹ*ìÌPÁï ¿ïqÍÝ K(nîî4ÊU1ZÙgîô8wkaž¦Ÿ\ûj\ -'t.òIçs“†¯¿²ü­[jiÁ.ãò·ÑÃÜ3—­>Íô&ñŽ¾”åðoUà2J.Wi›QqùÚÞ"CSxoãÃÒÆéJýдºÈn–zmliŒoU'ß¡UÚjµµHyh¤¶¼íÉ/¿¬ ·H´\×Ñ4XFdž0;$F“D=³._5vZFñãJn,0@è6éC³©ÙY›u†¾Û·=yÄöí˜e\ó˜…ªrxgÎÉaÆXVú>¡ÈÆÅ¢!`µÒvó¡ê¶Ï˹­÷Òr…)n¤7Ë‹¢NZÚ_ô8¬niëŒoÏ?ú¡k±ôçQÞaþ²ži«‹&Sí'ÌÞ¼¢c;úiº©2?µ9Z#À=úŸM¯¡µ—ñJýžÓ›âý|xTöY¯?,vÝì+,ãoåÛ·î¢ãVÝ÷¬¹ùQjmîuÀz Í®Í,¿ñë±mõØûŠBæe‰ -«åH·©n»çæiÑÍð‹‹ºÈ;]HmãÊkY«>ìý«BÅÞ;Vg.ÉN6ndŸ±umË _~õ®†4¾‰„†J iø€›ËÖ, Èy¨." º·Î«Ë%ê™ÁÀxё㯬 B¬’ mC!K­×Y5ýÒúÐåå´ÎzççÈè·ß|óëßüYDFljKA§4ÌôàEð0OÂÙKœLáÃK›\ש]–<žb㿾ܢó,#.’Òbx¾V›r=ë£*ϵڨRت´Ú—4V½ñÚï¯jÏóI:$tñœöz[ii»'©#WFä‘z>´¬èª6=Ï.ÕÓ ¯Ð&N‚³óá‡Ñ&e¯Ê„Ðá¼-†²höbŒžƒ°¿éTïðÙ&5÷¨9ÇÓzQ\xþ\жŠø4tás‡›N«&Ï€ïl€u˜@‘é"ÈÝXçsR½ïë‚O—°Î¼—KÕfÎÃÚÃÆR´”{^—7K¨ÙmV¤7·vµíÓŽþjHKóg¿$6n±Ž•Ü6mœ~`Yo÷‹ò%¿è6ŸF0ܹîe@ª?d3R:é4•8+_wxÉ'Y4ûfSÈ¡ùðÐEŸ¸±Î£ùDäSjç}âúÊ2o*ðŠåÏ|Žíñt†—g~ã­Ê|·ü]øÎ-üöûÝò—–Ï$Úâxdc›ÍfëZ±Ø|K¥û8hí×›[tQýðŒU¹Ò-Äó,RqW­ê¶:bUClò—¬ªi°£êvæÒ]H¢îf殡Ñ¥Ù›“Ƶ´E»0,a×ÎFöj‰qM[Yו¾Ô1¶­,¿éÓl-ïóW®Y_g,8tŒ#€©õÄô ¾­ÎÃYp|.>¤5]Á­j9ÏmkaÍëØè KtöNë|m’n¡Û1ôr]Þ3Ö™ÊýÖy[‡³Îr_åʵ³¬¦í²_ýªïç±â®f.¯Ëöë-f·¾Ý¡fT[¾ÙÈÕÊo¬u+ÓN.1ÖþyÓ×n•—8ù¼é÷ÌÔ£‰3Ö—Fü‡d–ïòÍöÚê~cG³¾t9fì«{ÃŒmNbçY½—^?K"¾ü‘,å¶1ž±¦WÄŠ7Öy8kº+Úg¡Å§ xf÷³wiÕrËjÔ­5­Ëôø»y¥­¿Ÿ³J¶ì†7ÆŸö8¶]=5‹þáãæ3Ö)bºÈwqaê¯íÝXçá¬óÿ(üŒ¬âëü2™Ç%ž±ÆÃ~þË-»57Öy8k¼÷=šÅ=Pæq“¿b÷ôµV›Å~!ßqs½{ø¢ÕÞXçѬ6_Ùcü©Yí9Kû¾ðøy-‹öýÂmø}Æ’GÜÉÏI!Bà}-ï¾±ÎÃYòåXµïTl-èiƉsñ[ÿ&Ff+­îf÷c¿msj‘yTO9¾¶Þ-Bðú7=4¬¤[Æ“bi7ìŒ[g¾@&vÆúÝ)ÝXçá¬ïJ,Jk­×Ù?ãËÛïú$þÝ”‰lh=7ŽÃ)»z”]¢,âNë¦b²ãç\¶ÖÛê<œµ^ŽUwÜ✈¯V+ÿ{+¿åß»b>´ÈoaÍy‹,#¦$¬nž§,_±Èë<šE–+qè—¾pD ä迱¼Ž—ÙîíG¶¿Ý¥ÿœ=ÍâîÏÓo¬óp–öãìi®¬ecÅgW,ûÆ>±'ÉO«§u¯YämuÎ"¯ÄŽwZËÿeŒøîþñ½sp·º¼±ÎÃYÞÛžå—Þýé2mÚWóó3ØÇê¼{òÖJfƒÖ\ùÊzsÿã.EÜZÝÏèwd çÃÜ|×'ÍuOóuãñV.ÛŽñv:+wZÚ{mÑhSöÑKʲïç~Ñz»q‘¡mq>HÁGÛŸ¿ù»¡ýiyN\ -˜‘ÀÏÈäŠxì\7®«E®˜Òþ!uí~æÝ#ÍÖ'‘Ñ$¿ Â磃ŔÄÏTÔÖçCDÖñ $Ž`ÀóRŽü<Òl<Ù6m¶*ÚÏÔ¬¯ÝøÔK-oâ0üpn~0ëüšq±”ÿT'üãÁšXÿõ.ÝúÔ|SÉõ͆ürà·&B&·ú^!3qùhz[dûÃvZòÓ6%}˜ÌòqÜ^̇°ç.´ÑpýЪñÓ,É´B¾>ÏqîX!¢f´{2ÞLv!ó›'òä—«ØEáš:q4H²ìhpT¼J¶ÞbØèn5y,?©¸«.b]ªÎŠéoe]Óy‰x: [¿¥¾¨T~Þ}kªi×™NóB¨Ò©¡•ìB}ž?¾cGüÊ´¦ “ÙùbÖø©Á317C¢^®´r -étPË-­…>S©—Ê£jiš¢ S÷¹+^¡Ä-ø‰_¥#|ÄøöqMæ;Àû\›,ìÆ툧R6Ú‘ë#[ö¥ÃÀ -šéI¢ívr6Ã^•O”ΟàšÆÏ\åÿg¨ÊÈS?Xê&ÅÏMW£öÍN^+IÒcGÒ4`)²B]6¥n³R_ˆ”ò º½t×ØŠ*T&èHgLo:vðX“m·ºNfkúéX)‰¦ î9tS~Y‘‡œ¡»j0½³h…¹‡Þî ÇŽ/’œúŒ[~v—··+l¾Oua£-™M?·NÙ™¾ÚÎqPä£/•Ü¼µ£¿Ø§}N¦­¸§Ã?u™s>8¾[9zËïtQ—=þe¾Áï?tü}{ÖмŽ;ÓÑÂul]‘åJÆoÊ7’ù@ØÝF Ød›GÙЬ¡¯ãÎ ¼¦QÝ `o%mGPÛ¹’†ÙuÓuÖ Í„ ÄÓF‹=ùܹ ö|éÛàøîk~üt“ -r8¸ÌƒçÇ’|M⊋Ž%\r¢".d£¸–!ÙJL6HÙIW=ËOr*<÷%.DÁ—~ðh,³²éF«£E{y!&+;-+y¡_í^Ÿ=ï‹È±yS;´wꋤö1µžNõõ2®/ûšãëÌÍy‡†z[^øžïAÎpZÐêBÚ¨ æQm)d§Ôh«NÛBÑá4Óh½ÅÔdŠKµ¦–:â¬j)õ˜bPÿ!5ä¬t%9+qW–Ûm+^–|€‚4ç4FgFÉKf…eíN¥QÒ7Ú¾oZ÷ÝVÛ£ AâˆLͨv%_…¯ñõAdZ©öSwFlÝÇBM6yçë›MŽBü2œDâak÷×¹¼wrþ¡ï²'LÇDÞ›dù˜º/`”ë ÑT{_¦^­uH÷)@Ù0·{lœ­/E«{‘uƒ¨_&NùˆžƒÞ̯nJæÌcäžvÛ'õç‚šo& -endstream -endobj -702 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 701 0 R -/Resources 4 0 R -/Annots [ 703 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -703 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 512.554475 101.433071 493.804475 ] -/BS << -/W 0 ->> -/Dest (cb145-1) ->> -endobj -704 0 obj -<< -/Filter /FlateDecode -/Length 4699 ->> -stream -xœí]K䶾÷¯è³3|_@C€‰ƒ1²HA;³Á8€Cþ~>ªÅ‡¨R‹Ý=»Ñ:ÙÁì´ªHªD~üXU¢Zê,ñó¥Â”!zGççN?ž¤ðVâßTbq0©¿Á¯·g%É «¢ öüC/|´Ú»óë ÿu‡mñ×Ó_¿8ýsYGµˆÑieÏ?}›Td„ÞK˨"TRFe­ð^[Ǩœ¸Ä9Ì°ž4­UfP –SÁ Ò:0ª3 yeÖª3´WÄ©`†ŠÖ3*%a‡ÒR²:"‰ŒftÊ -‡NwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¶XE$9l!<§s°Å8#YlÑÁNça‹FïDF` ÐXl‘N*N°1±º >¬uZZaƒ1’Ó)%,°iXfÉyF§a‹“6²:ØbÔœÎÀ§OXãC`f«N³ÛH£8…-Z{bu°ƒÏ1€v°EzÇQfH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHØbÉqL`l!Ì\V[L$Ž Œ†-(Àq1°E“á¸À€m há¸À€nIFÅqß’TŽãÂ…™’ãÆ5ÁÇ” (EŽ 8×€;9.0 ]LwÏqëë4Ç´k(8Ž x×’x×Ôdu°æ¸ì(Œ -†ãï%=ÇÞÅ4Ò`Tpè8.V„Ar\@à]$q\@à]í±°:/0 ÇÞÕ—À­Üà]m¥æ¸€À»š´ã¸@àÉzà]PˆåÝØ‚å¸ë=tÚp\€Æ„†[Âq愈å™Ó`+¡BtXð® ->V…òèqfÞ¢—…r.r\`Á»X¨ Ǽ ú €ãVTÍqï*ƒ•šÓw•Ž’ã Þ…ß`9.„‚ÀqÖ7¡$ Ëê¢1Ž ,x´¤9.À*,d“°:/$ÅqÈXHø¡8ð.–÷ÈqïJk Ǽ+ n«ƒ-&hŽ à×@‡iËÌi8«Bj0§ïJ…ætà])}ä¸ÀwÓ²ÉqóI Ç0PÀ Ö¸@"zï8.pQB—Õ9cËq—XP«ƒ-ä‰ã8Ö"š8.ð¶­9.ð¶Àeจ -Þ7ÀÀ…¶ÀSä¸L&B$â¸À[‡x ½ÊéœÁƒ®Y]€NyŽ ¼'¥Xq\€XJLzŽ @ð"¹KÀ èq\à#lÁTá¸K‘ž9§S°N6Çðt¡SŠã‚ a ¼AŽ ‚ˆ2ä¸+4bIt5§#™âÌÀqA '’«Èq¦ "M0%«ƒ-  Ž 0>)ض øA°‰0‡ÕÁc‰ã‚t -7Ç!À-5ÇÀfŠ7=ÇQÂx|DéDr“9.C'%ÇQÄ›š8.À¼Lñfภ‰x!3o£q)ÞôD‚-. -«ƒ-pš"3oÁIˆ7ÁNœÎÁ‰ã‚è\Š7Çp3o£gS!Å›žM€wm3q:ð.Â×)7ØX|c•74X"8ÂjëPÆDØ ƒ_)iQø×ïNêlÅ%Zó±^"&mÏï~8ýâ÷_}õÛßý0z÷rúÛ/1Gž¤$ý«/UúLóß¿ÊU¹²ø… z-û¿("/²T'µ—ÛÁ¤• -ÇXȧ‚Ö^äVwýåo*ƒ5ñr¾ôùýüùiC7ÿt¦+óaþ;ëµ®å¦ßüç©ŒþÕßÏï¾:ýæÝ“wx‚IcDQïag¬Ê#ØùA¨y2eØÌrøRR71};ÿ ³NÖrýðÓË|ìÆÚÉŸs{4×#ß”Éõ©+3·gÃòÒ°¬vóuå:6ÖsMp“sÛn£íܦ]Úwê¶Ý[0NYö J82Q"Ê7;8¬s4 #Ò¹†ä~ÔÊèö=®˜^ÎÇ-jÝR–‘=Òι8(µDS9§¼‚ž»Ñr´þØ@¯¡›WðÁ:‡C/]çaÕph7 -7!§ãXûü9¢e§?6®ó–~š®ƒ–úÊãh­Î#!˜–²®£u¬ÎáÐzÝÑY]{1# ü}nõ~®ã-G³ éiílfÝb&„¦-ßµ¥×6ôm%S½\G"õíÌíkÕ8­nvpŸ[û6Ö¸˜’„Vˆö6VçpÛñK{þ鹦±Òûºê¦Ñ›Cz’ìêÚŸ£ n¯]Z¶{S(D󯿳:Û„/ïå@(Ä£IW?|X8½s;h¬s44éë¾á*8Œëžòé:N*Ùò×âZ¶@"uçì‘×F+/KtΔzŸ{r;ªÁmÜsæ&¯o ­úd”Ró:(m÷6VçpH»îÇq~ Å:eõkGãÿ¾ÙÒ7»6›ž—ý³aè;r„ƒu‡ÌŸ­mÛ!c…ÛŒhçß´þ]áÙcyfýiÕsͱé>ë¹N¼âËu«wBWŸ½âVîÉfÓÕÕ ²®õƒ[·»˜ÝïóŒÞ@c¸=Þ¬s84^÷ïÃwo,U˯bʆßúë¡æZ·fåÓÚO½†Ô~v³}·ÿnq©i|Ãh…BôÖ9zÍN®ñ†ÑhãIótc×+`5´+ª Û#¹àÉŒ¬6¶n}ÈÞV¿lk…¾6î¹zã3çß–ó0þðjf ®ì¦ñ]Ú†€¡ÞËÆ Ö9w|ÎGFõ ǧæ´R¦Ë$•™ó¾³½ÍF]Ỷ?îÍd{û}˜Á:‡C㎟ù1òU_íiŽ(¬Ï)rY¢ÛâÜu„±â«Ï&ªÛ@ìwÀë±;¾è£1ï-±í§àÉf{§R Ý<‚©ñ-•SIî x°ÎÑL×ýчïU¿gJtßs_}ЃØZ婹÷œ¶X‡0m¾Ž¸±:‡CÜÎýênu.ˆ¹Âmv(pâÛù«=b>9fÙÓÚ¶GÞržMi£é»tîû#Æ ôÚÛï Ö9zw6J†ýѽÇæ¢ãÉÕ}—.ÿÈåWèïî—æ?›U»úÃY¤Á:‡CáŽßù¨º1÷øóó õ~x”Úfäht4Xçh(µ;¹Îü­ûîä53Ã-ÿäo¡µ÷ç2—nìè½×ûàÎ_fç*›>Æߺõ ú=S]_Mù ~µwø¨ƒu‡Ü5^î&®ø%XF4Í<Ò÷~·#¬øi®CAïû:J·Œè­hìw´þnAZoÛ‡¦ÞÞÝRÉ÷GËõF]/#_.¾«¶Í#)é¡Ç=K¶ñ7Jç|tq½cu‡Þë>êÇÞ#T8¶¿gïÖ÷q¶r±¬'ÐíëÔÙcèbï~VqëÆê)“þ‰ÆöÍ•<ÜÁ‡cu‡¨sçynǬnãá­l¹“÷g×é -’>[Ÿò¯‰G±k|ËÑÁƒuŽ†b·ã ë–¼åâî_›è÷Œgt|¸ `úýhûÃ÷QÖÏ6jÛ¹×NœÞ{;üêÌí÷‘ë™c÷áoòÇ_ñ‹çnð«Ûûp?Ý#{êÃä÷{Yvnùš®ú*}ùŽ2Îï"w¬Îá;æk¤ôOñ¶#ÝqY›¯Yô¿Ã¬s¸ßy¾¥ÃzÏÜÞƒkÓ/ËÇùS&;ÅSÚÏë^7=Ο}­üˆòΟçvG¾: ?ƒÒ–ùê€æ+¦xð-®ƒÅ—o3pƒø¬s4|ù/éÞ1~‹¯‡x«1¹ŽÎŽþ8?KUÚÁg£gÛžgLO/ÏXþŽlÚ`Ã!êºw“PqÏȦìÏçÌ>Ù¿ª;îÄÖ9ªvv þ—yçæÑ}3¾™gÔNÜïïØ¥7XçpHÙÉ^µÏj¬Èacÿz¿/j$/Ë=YÂyuí½¤q59…¼'ɪ ÛòÂ7ÝWÊùÚ«Ù¸/UºãÛhë ­á¾,ÕLjá?nÝ[0’™»aïÓ⻣Ú=dýþˆþžÙÀ7-„æIÑïò¬s8¤Ž=±üPžñÖÝ$o˜—üü²fˆ¼ãIåÁ:‡Cä'x‚äŽïLú$3€y6þgƒà;²{ƒu‡à±}î§ø›ÞøªM›†|Æ£ œGolž qcŽPm/&¬s4ôÆǾIñMŸùT=ì®Å 4Þ‘w¬s84îø§»¡WO}ØÊ5[ß8ûßÜ›ÚïâN‘ÌÄeÌ÷à}»Ü;ž„¬s8ä^÷cþž¥OÐn··/ì-øõûjþ ï§¾ÌçÞ‘k¬s8äÞödÊè7J,PôÌ ¡É{¦ìúbÿ<ÇAÝžÑ)#Ÿr´/rõ ×ÍõÛ]9}&©©»þvØf ùèO#¤~3¿·Á[ ªMï¬Lo^ˆÍk -½ã$Áë7_ÿéùºÂkÚþíe§/±tóÐÌIóÆ}¥dù*0==9»t£È.uW"•:4éït(Ýèy¹C’ëTÖ]~§¶æÏS;!Ãq>_ZžSý”üôý¢};Ÿ»±gjÃ6×`ó5¬†ñÇ“›ÞÕ£¦÷µI„Òé…®gG"ÝsòñœÞ¤!•‡MM­Î5§×àþXJÍoÆ•éeyéEI—×æ6sù§ÿá¤"‰Ôÿ‰(.o±”“8è|Pů§ôSbÄDBŠ“pþ(K[¤rÁ"{ßzû’Æ -9Œ¥qcóGmóÕ¤ú6WKo—!™ ™éý*—3BÔtj-NW/…šÅéM°A˜ùÈS¬Ë6CX¤¶lm°VÜ+#¡] esº*Ö”íQéÍa¥ë\ÓÛ®SÍðç²Ú7ÃyMø|ÂiM/}e¥éý“¹©Ü»Z™iŒŠEjsÉ2 TƉ=Ÿá—27MÌ¢oF-½Åë" HTÊÅ\Ž=ŸØrsk¶Œ”/¥B­sÏWS*ÔM(ÏȦ>š¥>½Úº™#ÒäK±K#ÇZ†7z%|>Y*3±Š_OÖ3¥må -i3ôZ¡iÚ-â +^ª4m}¸|ÂÔ̲ô¤ôÿ…*l‘&æ™>(*Jï1«Gõ$inñ²“Z¹+¬C\(òµlÓ]Êû‚Ù¶¿TLùDº¥té±…´é²Fîc6/‘T!¥XÔq¢°Ü?…+[¡ÌFàtUl ”_âj§-S`Ï'sª»-íV±+ÜùÚŠ}q•^JWì-í6˜ÏÂçia£^œ òIªX˲ÚÕ“5ÂjH¡Š+¾Nâ<[ ;Mßv>¬“Œ[qŸOߟ¦eN«#>8£Ò;V[¯Kÿ2_á÷':ÿ1¿Ô벎kYØBn]ˆÛ•,½‘; ŠÓº³°L›F¨©¡ý*V¡–µ e׺ákªÕe%ö\RUEÛ -míÃF¬ tµ2MÏØ+Â箋#óèØÔûrhòÅá4&½.[…ƒörpªÎÔ%ùÜ4‰£T´.éÐ’n*â m‡Í cµlafd{’ÒõÉ~j‡Â¤s5‘MÕQ]fÛ¦³lºŠ¦\`Ó´ “jOö0Fß=½÷Ùd—g@’j/Ò×VZ.,I áñR3ºåaYsÌ9äÉ[{¨´eš¹gŠ“S§Fu-hqÐb´‘†Zm.`Û“R–-NšŠBÉ,+-úl“›«ån™®8°½ä‹OQ¥ß·=mÛQ)hÛQq«²©Ý¼âoÉȶpöõêd-Y|ÉÀÈ\[»H©–4Y¶>7-ÏWÛssT qÕ3•µš]•¼‹ÿî™;—:]‚+•¦Ìˆ -õ5˜X¨Iy£Í9&w'8¡ôrjO¡I•Ü\çzîd!ɹ“öLAÆH¯€· ^¡ˆûŠ¦©×ù ¢|µ(ÙKº­˜ÔP¥9¹Ê aætæ” *‡>…Š6N¯’–!¤W©{©ØTÑ´·÷ý"}2ýüpš -endstream -endobj -705 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 704 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -706 0 obj -<< -/Filter /FlateDecode -/Length 5044 ->> -stream -xœÝ]M丑½ç¯È³‘ùüs0à…=‹5ÖØÆîÁØCWö”E±û÷÷‘ÉRª²«Ç w¡ºR ~„ÈÇÁ ÄÔW…Ÿï4þ‹¤§Sðt½ýtùù¢¦àþ•«‹"þ#~ƒ»jEar:ùè®?妜 þúvÁÃ%Ïþvùï_]þº.£¯fJÉí®û1‹ÈN!† œ J)m‘sSÆyAäõ„[LIA ÈÐV ¥è$Ô c¢ ŠPÃRÐv+JPÃM’jèä‚ Ò -zh£”(ƒ"ŠÈA¦ÝäÑé^’=ù˜Le2Œ$³vò´Ó’,MÞ{%A—”–dº8M¤$t!ƒ$óÐÅz«Dt11XI ‹Aï$A¡ ÐEtQ^iI–ìäR$e 2CÜÊŒr“‹Ö*I¦õä€M+Ê0£0H>2]¼rI”Ag•‘dº~$xÂÙ£0[MžÝVY-Ét1&(ƒ.|‰Œ‡.*x‰0ƒ¦ÌK˜&J†$0ÑN)J,`bš(+Ñ€In"Ÿ‚ÄViÈŒ–ˆÀ*èâÈKL`5t!Ì\Q]l"‰ ¬.È qµÐÅ•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…šJâ ƵѓÄ” (%‰ ,8ׂ;%.° ]L÷ qëZçÄ´k)z‰ ,x×’Vx×Z”eЖ¸ì8Y­ÄÞµZ‰ ¼‹id$.À¨NÀ¡—¸X™LŒJâHâïš Ê„9h%. ð®q¸ÉrƒwSFâï2^âeÿ(Ñï‚Bœì@¬Ä°÷+q*› ܉ xW'˜?aNƒ­&“—¸ÀwuÄð‰²4é€æ-zyÒÞ'‰ x†ÊJ\àÀ» Ï(q8~‚E58𮶰Ԓ ¼«MR8ð.ü'qà0i8À¾MZº¢,M*Å(qï‚–ŒÄ°Â“Š`Q&IK\2žüP‰ ‰ ’òSv“%.C¦”ÄIG¬7 I\€y™×›Qâ‚dÖ›Xâ ó6YŸ×›Aâ‚DÐÅÁEeÐNSæ-8 ëM°“$óÐÅ$’¸ yŸ×›Qâ¸Xo£C1¯7ƒ.ïºf’dà]x¦^â\`½ ÏVZÛ+¯ ðÏdaÌ+Î(F ¨×Á¶Š!î…µ -bÌ@|óI @M¬,££0«j%† øþm8aŽ>ñ•lf ù&¦Á»okŽžÍîd›ãcûóåןa|¯þûåõW%RV4„’›à_a˜„ávð¾5nßúìøÕ Ù»2çfCêjiáfËjÁàå×퇕®P5+‹ÒX ù¼ú bµ4 î)µ–ÁX˜l%»®ï.sWåŸ ]Û-ëÚûˆ·Û;IÒõÝe>^×¼u.æº]YÉ;™?vð{»Œ´mW‹ÆcÈÊž-ôÁJÛ6SR„2 Ù)ƒåË|'H³ég*¿³È*¬°€‡Gè‘Ç&“-(hkE«Ì¿ùt™.ZÌqúvÉÔ‡O§a»~úéòëßÿðÿþî?ÀO¯—?ý‹RôY)ç¾ÿNã³3Ë߸üµó_°˜¢ü.éDËß°ü­ùk=~þkòâe.{ϧ†ö†zr~sB/,‰JÞ¢ßÒ&½Êeª®9o-¿©nÝŽï-ßow·.Ü·»Éºµ6jݺ÷VßÿÏõÓ—ß~Zƒ°¬žÞ‡ÛsE¾·ß®)ÂÕËÃâØðÆ×uþ<|yƒ¤•¯Ã¼Ô[d?Ðq;mj¡žzÍ꫟݃3m•û¡5ìë½¹½¾HÛ´oU=è5Loee(ç¨ -L¸IE“ÐC,Ÿ,ól`Öæ|‰d¿EF #ÖjÄÍP/'Ê8ê#œÂjÆ`Y¬íð[ÓAìùušÿ®È1¬ÑÔf…ÚÞÏÇÏäôÙ÷3éÉ2O‡>:æÒFž»ËIçÐ9reÓ%1=…Wï³µ{û†HæŽiF¥¼ª>Fò¹2O‡ä;Nì?#’÷Ðô„sNo6@Ïõlœð±xù±Ï^fÔ› ã,ªzízÌ=͘Àø»è=WæéÐ{ìÓnFkX²4ôpË8,s²Å漵ᨗÎ3+«h3;u¾ñ÷t7pŸË  /l¦2t‹3àn[2J ó;Ï¢ôd™gC©9öUw§Ľ0„Tn¨œ0 ©¢µ “f¿re¡ô‹7íŽ(ÞßÊz´}ÚA*óQMFS$º‡Ôsež©Ç~í×Zï_uõs›Y# -õy¬¾¦«[÷‘x?a]çèÍ´þº ÷«xõ8ìÉ2O‡ÖcßµÜë%e ~cˆáAÕji™+Fo`D­yeˆf>hCîMm#BÕ¯{]—ß cAM üîÄêÌÊuYÿ¾g˜mEçÀî“{Bq}ŸM¿ƒà"²'Ë<‚ýב¿ÆˆÒ*šä’jÏ<òÐ(ÆŸR7®Ñd|ÅQ¸A”f\ÈtÍNËf…´×Ö2Ìp«eýÚxü…åì‰z•‘lÕû¹ød™gC²=öq­°žÝl=åÈl{\Xë¸ÔË3nÊkCí>ª¨CÚøüh6–6í¶ìŠidL?å=Yæé0}'Ê;ØãlÓK¿†uîgTF½3ÖÍqs™Ù%¦Û?ç½äÍÜuáz§­%pƒW|dFFÂtËóeí¬ý%Z~Ÿ2~ˆíž,ótø½Ûð»ê瑧™ïZ<‰¨ÖñUŽî›˜:ãÛ>iR»±bØms`Œ¯,õ”øëAl³ËkÝÝwômÙÏ»íÇlm|Œád™§Cå˜-ï¹—e÷ó¥÷jùKKb”tœóätcyýý"ÎyNÅ^Çh’fþe@C\£d³6Ä…W«µÑ÷µ[Ykw3CdDÑñÕ“ež Qtì{º½uì0ºÛžUëýÑƨõßq„ OîÕ5"Kɺ4î=±Ââèim¦%ŽN•V`<&Pysl=¬õÈÎ*uO’›\!­{¨ÔZ÷Òû¥ßOu´åý§¥½¢\÷ÊÏ ÷ÒFZõ>Ìù)ës›G_û¥y¿j7„É'›ÏñXÚµËظ)ˆs?—>ü²­ÇÛü’[>Ha[Oé‡Ú·¾ôÓù=Þ=T¿—2Þõþò8˜™¯ÝòTä“P÷ŒÑ^çs½¯ë˜<…~O¹Ýví—ûŒK¾[eŠåZзÔ]ó¼2|›u›ó^è2µ?\µk}œï^ÒNÿÚ±V4Ó¹Ž{¨unû@/O&•²jÖ_Ùõ½ýØÚòLçeý5{²ËXD6õºÇùµ§¥o -V³Ÿü9¦o\ë õa¿¹±ßö9è ÿtÝ 8>Ï—ûq‡úÄ4èc–z9–ë|ý–O²åµž|p…‰Ú8uÇo9Yæ¡—«>ÐaqM_“¦è +‡2ÔÆ@µ^ ´eæj„Ý“ŸTʇ Vzá[ùTB -ß×h`œÇEjt¯÷JS,gxx/ês¯´óSr3©ôQñ÷’š‹bïöžÍ'dÀö{½w?`qçŽl “͇Iwt·7m]*o)?Л¤h -6ë~™Þ¤h0«"ÁÔícÑ-a­ã6FQä 0‹xÆ‹Na 徬âÊ!±{NÆÖˆçSŽR>Odh´ôäÉaðÂrÆ V,ó$‹³Æuå„…}®SŸ·Ašso·©Íæ£øâÇÁƒÇêsu—×K€Ù,ËöqãFz($£ÆÌÈ)nðç.æmQv Ckƒ¹Òã¦òìj¬SçÛ–]⌊Nͨ6[ÂâB0°Í¨4»e&WB˜M4¥äbÐCƒG³)fǾΠ{n6,ót³)é£Ùt*ˆN˦ƒUíU+}Û‘EÕ_Å¢!ÏÕ^Ù:ÜŒ å3±º ›‘~R1<âeôZª¯ÜPHÊ4„œÝ8=YæIPÕÕÎ'Ýì£jGö6ÑIâhbiOõ4»G_Òïñ=Ò~ÈË‹;6ã(ÄI[ʾ÷Ÿ+ót¦‡xñ)Ñt/ŒšGüU­¶×ÆM…Þ®œX_ÐïD§pVÈÐñÆ­»]~uæýèJÌ¿ü!åÜ=îDñ€³x¶Ð³áI«cwQ¿²Ž|9 4½À#ÃeƒÔ;î…V#üe6¶å÷k e]àä]†—-•Tõ/ï:7€þq9.8Œ¤ËgáçƒÝ;^®òn–;¬ÁƒíqYW=Q¶Ž/ ÊüE6^ ~µn7º`~œãve­‰!1·ù‘ØIËúR«þp\H”‡@蔈ÐÈ›Vu8KtbÙ)&˦m ª ôŠm¼ª¶áSΙÊåóf Û(jõ× ¦Ïj“)-p+÷°ÆŸ/¾œªË9ÐÊÃû1ù‹"®ž¦üvH×|îhT–t6]:Z_wJ–¯×ø¹åZ¾qCåC¸ó¬eXùÅ’ÿç2à?]t¢)÷Ù%.§ã«’\LãüvÉߎ@B2Qßj¾“œ—ªÕÇß i7°l˜Ò˜ÓºI-ûöv­ÌKLµ•wµX>µ’TÍd˹sÃB"Jz½MÎw¯&½$ço˜ˆ“]®|S%op´Ä^!ñ¼½Â^ðn²¶Š%ºm¢bÍõdCUO$n]ç{bSwfÃ_óbUiN%Þ.hÖŽ©obj>×¾VU{ר–f™R©¥ºš³ µq’n!ñ;U«& f“³Q˧ω±‰Z¾Tó I·‹˜o©Íµ‘ --WìESíù®J‡º­ã…´{$ä¯ÌasDÙúqœŸ`hc­"ÃÙ$Þ.ŽÚLìÉo„Ü®s…rz<Ѳz[rT/=5¯çO˜š5-Ÿ­šÿŸ©ÂµÔÌ<僦©|>r¿êä¹Ù’×ÄÓ}c’z@SèyYwéfyaq%äϤÛr·[¥².cé!Uõ2I5òX'놺–œ)¬öOãJž¨ªh®'»å·Ur×Óµ©êTkÌkw×êíɾqçOmÄu>ìºéÛêe˜¯‰·bØhLÎе‘žlT³v½1–Ø)ôäN£o%¹ÎÖÆN夗åcŸd’Ž]þr)fÎXgòÓ ÞêüÝ ÜëÒ×ÿCžðû¿ºþ{=,x¶ãF5¶0[WÉÜ’åoú‰'“ó7ÿÔÄ6mX¢!Fû=YÇž×U44;n ãkêÅU'öšS7ÕMÊ]ïC–lt¶¬gÜAâmèÅæÈ|íØtƒ=_ÚzsùiŒ|ÌtÕ -üvp»Ìv“|eUâ*gí&R2¬ .x¥¸d ¦®ÙJÍ HÞHëú¬?ñ¡°¹-vÁ܈|i»Œº™åU×´r, \`Ë/xŤy£i¥ ôÃÓë«Ç}QÙ×Sµ·Ô7žÚn,§6“Sm¹ôëËfsly°¡NÞÞC­.ËæžmNNŸݵ ÕÇ(K½Ø’ÁñF©¦­­†¢ÑÒTMk5†ª“_ŠÕn)wÅ^ -ͧè©á=íø¨´Nt|Tü&o®·Z¼æ-Ùæ9çÐïNõœÍ—ŒBšç¥[*õœ¶¦mÛ¦uÛÕÚ^ÙmPWÄwÏTõbn“ó!þ{dîÌe†(„”«þ”ȈŽýx}jÒÁ[¾1QEïÁ!”¿ô&Pd¡’w—9Žl#$5vÂ[ÂpLd­ -:÷¥¬Âé¾€UU¿S.–”xKïë!4ÔiNm"CåËÏJ ³ˆÚeÈKÅü]‡6;½1ESPZ~Þ?Gê^Vá“òóÿ´"Žù -endstream -endobj -707 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 706 0 R -/Resources 4 0 R -/Annots [ 708 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -708 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 307.623057 101.433071 288.873057 ] -/BS << -/W 0 ->> -/Dest (cb148-1) ->> -endobj -709 0 obj -<< -/Filter /FlateDecode -/Length 4677 ->> -stream -xœí]K䶾÷¯è³Ó|_@àC‰ƒ1²HA3m`6€Cþ~>JâCT©¥îž]hmï`vZU$U"?~¬*IMu–øù\á¿@J„½£óåý駓ÞJüJÌõ·øõö¬$yaUtÁžß'ñÂG«½;¿žð_wØ=ýã³ÓæuÔY‹Vöüó÷IEFøུŒ*B%•aTÖ -ïµuŒÊ)KŒ‘SÁ ëIÓRåaÅ`9Ì ­£ -0ÃWf©Š0C{Eœ -f¨h=£Rv(-%«ƒ!’ÈhF§¬pètÇé´.DY‡NyÍéŒΓ²ŠÓEᜳÓl±Q*Nga‹UD’ÓÁ2Ás:[Œ3’ÕÁ¼át¶hôNdt¶]ÕÁé¤âtÑ«‹ÐÉàÃR§¥6#9R›†ÕaFaœgt¶8i#«ƒ-ÖHÍé l!üp:ð„5>f¶ê4»4ŠÓYØ¢µ'V[0øh[¤w`‰ÄKhïEM è` - èyo8ÐÑ -rÑs<`¤‚N+ŽŒ„-–ÇFÁÂÌeu°ÅDâ¸ÀhØ‚[4Ž Ø–€Ž è–dTð-Iå8.0 \˜)9.0`\q\`@¹€Rä¸À€s ¸“ãÒÅt÷°®±Ns\`@»†‚ã¸À€w )ÉqwAMV[`Ž ÀŽÂ¨`8. ð®QÒs\@à]L#ÍqFU‡Žã`Eè$ÇÞÕAÇÞÕK«ósÐp\@à]mq ÜÊ ÞÕVjŽ ¼«I;Ž þ‘¬7Þ…XÞ€- XŽ °ÞC§ ÇhLh¸%Xð®ŠXþ˜9 ¶*DÇq婢ácuQ(gæ-zY(ç"Ǽ‹…Êp\`Á» ÏÀq8^`EÕXð®2X©9xWé(9.°à]ø –ãÀA(8`}Jº¬. -Cà¸À‚wAKšã¬ÂB0 «ó@R€Œ…„Êqïby8ð®´Æp\àÀ»’àö°:Øb‚æ¸~ t˜¶Ìœ†³*¤Sp:ð®TaNÞ•ÒGŽ x7-›8Ÿ:p\¼`Íq $¢÷Žã%tpùX8¶x e±:ØBž8.€c-¢‰ã¯a‹Ñšã¯a \Ž àˆ¨à}³ál\Øø€` ÂLŽ °‰ á™s:[àds\O:¥8.¶Àä¸ è€(ÓIŽ °B#–DWs:’)Î r"¹Š`š ÒS²:Øšà¸ã“‚mËq‚›sXl1–8.A§p3p\lÑRs\l¦xÓs\%lÇÇqA”N$7™ãÐ8tRr\U@¼©‰ãÌËoŽ ¢‘ˆ7â1ó6—âMÏqA$Øbᢰ:ا)2óœ„xìÄélÑ‘8.ˆÎ¥x3p\7ñ&²¹ÝÌI66õ324ƒˆF—Ë•zªAÆ3ƒ´+ÜWfŒî®íÒ´}éê^æç/³w }¶áÄ4ÚõÖÚ¿³ÎáзáˆÒ¼×‡Ñ_™Ë¥×»9¿—[l'Ë£”ÛKžÐRŸÿšŽ¯¿Ñ1ü>©©›ò øÅy•y Ä-9ÔvëÍb*}´‚ÖÆíTˆCeÐ),¿ŽÖ}u‡Öë¾ê¢{žkü«ÙÊÿÄpå!ÿÖóÐ³à ¿—¶ŸxDêêSaRÖ&l rg£!Roø¡oî/½±GÐØQß!oÆãqDíçiÛϾg;·ý㤙ۣôu‡´ë~bšã¥~`#žnUº6ÿghéP¼ºb©_1÷5‰Éts Ñ´¶[ˆÜWçpˆÜðoDË/ÒGüˆÙ]}:²ZHK.leÑwÖ9ò~ËY¾uö§ØÔµÏÆç+ÜgîÈ?î¬s4šë~_ALèzueC2צãêhë€_ë¤{³Îر¬¦9B V*}™Ú{nÚ¢Úf{E‡µeŸÆó&:+íºÉ†œ-JuЖ—É—aÿ -ÊîðùvÖ9Ê®û|»F·¥›F7¯h~ä‰Â™ºûë.ÉH2“¿4¡ŠÕMéÊäUuÒk]Ë ¿ßñŸ‡2+«£¹#§·³ÎáPsÝ/+#gæ#8ËÔv«‰jã½½¶ÕΞպŒq)Ó­¶ùÒȨv˪Ÿã×&Ä® }Û+±üˆÖÛý~V˶ô£qøùBj|LIB+ -D[ÛWçpÛðK{þaò„‹;nºê†Ñ›¢z–ìêÚŸcq‡p­]š·{S4DÓ¯Ym"˜'¹#ZAS“ÏóA(CÞm=7³³ÎáÐtÝ7\ćqÙ“»|ºŽ“ÖîõD0÷‰¹ûÊåœ=òÚhåeŽ®Â™²CïscOnç»jGAp'÷œyã7+o¿Ï±³ÎÑf÷=kØú%å÷…¾ÙµÙt™÷ÏV„aïÈî¬s8dnølw?i´ôoZÿnößÓ -ÏL£?¬z®96Ýg=Õ‰W|¹nõNèê³WÜÊÝÞU\¬Ø-²®õƒ[¶;›ÝOyF¯ ÑÞïî¬s84~"/µ¼²T-¿ˆ)_~믇äây™Å¬lî¼õO£]½7¾Æ¥WâßU.m|Ãh…B-ôî«s8ônäoÙ]½çqtÒÝ®~죆vE5a}$g<™‘ÕÆÖÌÓbÅÖî©­úÚX¸çê•ÏoÿÜ$F'oOpg£¡ÑmøœŒêŽÍi‹'Õºˆ(?¹[Úl³QWønö¼Ð™lgn¿³³Îáиó•b䫾ڳ¬ONÝðLámqî2ÂXðÕ'Õ­ öŽ{à;ë±¾è£1ï-±íÇàÉ{Ÿ3oßü:ºWÜø–Ê‹!$·¼¯Îá¼ó]—{ïU¿gLtßs_}§±¶Êû;žiÜYçhˆó÷«»Õ¹ æ -o´Ù¡<Â7¿;ò1–eÏKÛfyGÈy6¥¦ïìó#ã -zÍí÷ƒvÖ9z7ž• Û£{5ŽÍDÿ6CߥË?rùÃú»ûå{óŽŸÊªí›÷[öf‘vÖ9 -o|ŸúTݘ{üåù†z?+(mž‘Üí¬s8”nä:ð·î»“×Ì 7ÿ[¿†ÖÞŸË\ºòDï½Þwþ2;Ùô}ü­[Ï fªë«!ÿ±öUUwø¨;ë ¹aÃGãÝÄ¿tß0ÜqÔLïwO„?­{Ÿgáû:J·ŒèÍïk_y÷» ­·í»¦ÞÖÝRÉ÷GËõF]/3¼ ”Þ·Í[)éëQWžY -¿‰Îùèâz÷Õ9z7Þá^å·zF¨plÏÞ-ïã¬åbYO {®Sg¡‹½ûYÅ­‹·Lú76Û×Vò`ïàÃ}u‡¨ sã}î‰Ù]ßÇàäýÙuº‚¤OÖ§|ÄkZAqã[î}2xgáxÃÝ®[ò–‡ø¶€ÅóáÛ(ëgÉ-ßSf燯|OÒ2£¼ý>ÒÎ:GCfÜwþ&¬ñ]‘n)]zl&mº¬‘û˜ÍK$UÈc.VuEœ(,÷OáÊV(³8]Ûå×™¸ÚiËTØóÉœjànK»Uì -w¾¶b_F\¥âŠ½¥ÝóYx6êÅ ú!Ÿ¤Šµ,«]=Y#¬†ª¸Òèë γµ°Óð­àÓÇ:ɸ÷rúñ4,sÚXñÁ•ö=m½.uþÊ|ߟèü—¼ÑÖ¸ŽkYØBn‰Û•,í’vŠÓ®ÙYX¦M#ÔÔÐ~«PËÚŒ†²ŽkÝð5Õê²{.© -ƒª¢m…¶öa#ÖºZ™¦gìá¥ëÅâÈ<:6uÁM¾8œÆ¤-Ú²U8h/‡¡êL]’ÏM“8JEë’-é¦"ÚFqØœ0VËff&@¶')]Ÿì§v(L:WsиéÐTÕe¶m:ˆ«hŠÀ6íAÛ0©ö¤qfcôÝÓëáqŸLvy$i¡ö"}m¥åÂ’´/5á›–5ÇœÓAž¼µ‡J[¦™{¦89ujTׂf-Fi¨Õ¦¶=)eÙì¤y¡(t™Ì²Ò¢Ï6¹©Zî–áŠÛK¾øUúcÛÓ¶•Ò‰¶·(›ÚÍ+^ñ–Lq€l g_¯NÖ’Å— ŒÌµµ‹”jI“eËsÓüÜyµ=7—AÕW=SY«ÙEÉ»øïž¹3Öé²\©ü3dFT¨[Sb¡&å6ç˜Üà8„҆ўB“*¹¹ÎõÜÉ2C’s'í™0‚Œ‘^o+&¼B·MS¯Ó®ž|µ(ÙKº­˜ÔP¥9¹È a¦´ã *‡>…Š6Û;ËÒöæ^*6U4dï.³ôÉðóŽøÆ -endstream -endobj -710 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 709 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -711 0 obj -<< -/Filter /FlateDecode -/Length 5059 ->> -stream -xœí]ËŽìÈqÝ×WÔZÀPù~Æ,ÈÇ°`ÁÒBðâvÍ´£GÀÈ ÿ¾O$™™Ád°È®û˜A·Ñ·‹ùfž<d‘úªðóÆÉé)¥ƒ»Þ~¼ütQSô -ÿJ‰ÕAQÿ¿Ñ_µrqò:‡ä¯?’ÀÆ)fob¸¾]ðßpÈ‹¿]þô«ËßÖuôÕL9£ýõï?ÊÙ)¦•T*¥­ ò~ŠÑø ¨‚žpŠ9K*˜á£3n«Š0Ãåä%ÌpÆ$A•`†uQÛ­*à µ“T0Cg•V°C¥D QÎY#è´Ÿ=H:£§²É¢.B§£‘tÖN!:íµ¤ËSÁ'Iç`‹ÏJK:[¼vNI:ØâlŠ’.À¬u°Å¤h%]„-£“]‚-@Wu°E¥%]¶“ÏÉ9Q—¡S)¦­Î(?ùd­’tZOØ´¢+ -“¢ 3°%(ŸElñVIga‹Ã¤OxSV«¡Õm•Õ’ÎÃc¢u°“/1€ °EÅ QVÐD¼$q€‰qrÙ8‰L²“K.I,`Rž\ŒV¢“ýäBŽX¥¡3Z"«`‹wAb«a‹ÃÊu°Åf'q5°$.°¶g%.°`[´H\`A·Ne-qß:¥ƒÄ„ 3•ÄŒkSpXP. ”%.°à\ À‚t±Ü£Ä¬k}0XЮu)H\`Á»Öi%qïZ‹š¢¶À€'«“•¸Àw­VQâÞÅ22`V'à0H\¬L&%%qœÄ¼k"\€¨‹Ö •¸ÀwÇ)Hž¼k¼28ð®q&H\ Là%Fà]Pˆ—ÃØ‚•¸þ:c%.@c“AX"qïê ÷'¬i°Õ¤Sxð®N˜>Q—'1âºÅ(O:„,qïÂQY‰ #¨¿Ìj­[mXéª q]¦öMöºÁÞj?õéy¹Ø9V;?v™hÛNÿõ¼¤sw¯ƒ­uünKÛ™é^׶—q2;hµ¢ZÏÕy:´ºû¼–Ö³PG˜.¶Y®37 µ!S í *í})$èþ¹õ¡ÖõVˆÜ9g'õ?œ3Gœ©(Ê}d庿6n·¼Ú5ç§cƒC¨ºÔsuž©÷ƒº#4ŽëŸ£©Íðˆ"Æibùé#wæ-"‡•P‘ò5W\CÞpþgúÚEdï<]úG(=ý¹:O‡Èƒ˜pÇëŽ^¿ÍZÞÊwÙ¡Žxëk®€ýíÁF½¿|²Î³!ØĪŸÁ;hy²GoÛbÊë}Ÿ‹Ÿá;èíq(]"w[µ£ùÉ:O‡ÞƒØõpý>„5„¾gÕÜ;‡1*ƤéÂpN¢½;h| ÷x²ÎÓ¡ñ >£ýa¯?ÎRÙSWÄHøùf™õWÛþ¸pËkì—ñ;ÈMïÏœ¬ótȽÇ~ZŒõ3 4®ËmP¨?¿~J¼}70ŽÓXæ€sí¹Ö“už ¹ö\üÚ2IQÑá+Ý4°¼'e×9ErkD”Œ<åh_ç,~‰­ÿÂVĘIbu[ÆKò!ãx8Á/ m»†Èqœ7ÒRYÎôltp²ÎÓ!õ Vý(ÄŽu-ãÄwŒ:G©}™ëÓu¢‘ÓZC;TÞœ°kʼnƒëÜË¡¶²Ü/ìœÓ—Ý!î öxöd§CìA<{}¾8ïÓ#™§Ý-F-;h~ Æ=YçéÐ|?ÆýÔÝ¿Ò9rÖ™«¶+/ìú —ëD|ÅÔkîü·ÊéšýÇùÞú»âǸFt¾üJ–Ñçȳž¬ólèsqê?"Ͻ'Su›Òõ¹s-íËåvÌcÒ“;®“užÉŸ–sýE"ùsä -§7 çv6qø8¼þ8fË=Ž­†qõßzY|zöŠíÉ:O‡ÞƒCÇÙv-Ò(ãN§æ>*FŽâw…p«h3;m¾ñG¶¿óŽѦe¸ïÙJeè~ì>¯”>p_ÁÉ:O‡Òƒû -öW3ôÐÿ£Š6ŒW¿ÇïJÜY¥O»­»bÓá,ïÉ:φépò›_ì΂ú]¿†ugTF=˜ëø¹Mbv‰)ÇþïÍóÞµä½ï¬HW¥h žÀQñ=o4Æ3¦[™ï×ñPÁê0^¢Gáç)ã÷ÜîÉ:O‡ßƒÜ®€ßÕ8ßùV»Ó%ï`…Ǧã7cNĶ~£p/—âvÛó+ì—÷2`›«¼F°Ý/y`Aß–ëy·ýœmðïÏ1œ¬ót¨<ÈÙò‘{Y®~¾ôQ-ë·•1Kí ô³èëï÷òg*s*÷:f“4óð/Ò%›kaC^xµ[c_»ÕßZ<ú&ax ¿z²ÎÓ!êÜý®›]ÇÎ}˜«Ùã×wF£ÖÇ^ßshkD–’miÜ{b‡ÅÑÓúÌK5­åâŒç*oŽý±ûµ¹²{$貟|!­Tž¬ól¨ŒŸvÀþ)3P󨿇ܶû±[Ùʾ(²ôïzµp¸šûÜô%Dœ÷Väñµ©äñÎœ‹ûNÖy:äÄ}BŒäƽȰÞËèæ©*÷Õ{Y…ïÄo¾3d–VÏ -lùlÜwǵ›lÑëK_üœ<øý«­ Åí!a4ûy*wÊÖÙo‡ì Z1;z¹6©ÌþoÿŸÿþÇßóøªö>ÇP%njkoÇÓ•Ú´»}©¼1o䣷áY–Oœã#Å!öqKŸ¹·Ûm6.ýºÛº®v×;Ðmã6×ríسÙMó,œö¤ù°ÓH«òüù̽ê£G9²è¶ô–nª}OÏA´ôü¥}»ÌŠ§ü@ZF‘¢æï·íK‰¤‘oÛ)g^G=”‘ºgGJ{v¨~.îÆÎfÈÌ,-ØF¤  …¶gtö6×hëH«³êÂ2–Ë9Q¿í8,ç™–r·Ê˱`oi»–yeÈ6ë>ç+ Ë<ÔñðÕ›õyö¡ÇFòøf5ŽAÊf6×yµÍíèå~¤RWÍö+»>·q[_ÙìëÊêãÙƽÚû<Ôyªí¶ú '¾Þ-“d;¸'ß'3ŽÓÌ2uì2àܳqI¹ö9¬YcÄ¡7 '®÷Guôæ®™û6ö{¿x;wÇ!¸aÌÒ_3uÝQ=ñ3Xô’ËæMÒÆ«ƒ¨èd‡{øÃ!ߟ;FökDÎ8¦Ç<¸^ü¿eqž…Ýs˜T¦§‡=V"ã· -¸’²øö~“ÆæÉD©IÿzTÝ¥ò¸}zŸdÑQu¦ì1obõ{£Æ¿ýÔ¢ {8‚–g`;ìŽàqRäàœlŠ“¥W‡ˆçt8¢–æØÍ~`DªD›Œõ_iD]2X^ÉÁ·ÞÁ¤_Òg÷;uüay8môèÆÓ‹yèñ²™=ë¶ñÅa‘z¨Aïö9¨W= -¥}ç -å+ô¸Ð²v›8©Ž—.¥ÇÏŸ°4«ŒôNÿÏTá›”˜§|ЮAŠ^ÖÐz'´6›x=H\ë8i´‹½,.cÃ,/”PžH·•n#¶’²!cò˜«yDR<ÖbÝP×ÄDau|Wr¡ªF ».ö Êo+q·Ó·¥êUë,hwßÚíâиó‹c›qMoÞhö¶væ«ðV›ÅýT;éb£š·ë1a7 ¤ÐÅFߊ¸®ÖÆNåñ†ËǾÈ${»üõRÜ"UC[´`5½HŠG]úú(ó~ÿçâ®ÿQß\0ûq£[˜Æ­+1÷dôÚÁtRL¯!¬Â¶l˜Ð8Fû]¬S/ë+š7†ñµëÕU'öZR7ÕMË…¾!›]£-GxF±2Ÿ:7Ýaχ¶žm8éÕ*ðÓÁaê:Û]ò•5‰#*Ú]:´Î°Š8àâu˜»e+3 ¼“6ôd¿ãSa©/vÀÂ:´]纛åMWY9 V!°å¼a§y§ye`ôÃËë“ç}19Ô@ÒFíMúÆ¥íÄHÚO–ÚrÖ‡ÍçØ+ÔÅÛG¨µeÙÚ³-ÈéK£‡nuÀ1ʤ©W[ -xÞ©«²U§ÕQ4º€LUYk1V›ÂR­K9ã$ŽRl1E—þ•´ç³ÒÑóY ›²Ônõx-Z²-òαŸê%[,™Y൛Ôõ’¶Ê¶}»ußÕÛ^Ùi¸nHè‘©êÕü¦äCü÷ÈÚ™ë Y©Tý)™ú»~਎ÖØòúf•B‡8z_t‰¥JÞ]ç~îd›!©¹Þ¦crÖª¨éEܲ oPäckª¾¤[®–•xJï!5ÔiNm2CåM¬åª{IµÃH[Ezñ²¥ 7Ñû"£Òò%SºÏ®Ò'åçÿQ…RÏ -endstream -endobj -712 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 711 0 R -/Resources 4 0 R -/Annots [ 713 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -713 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 101.373057 101.433071 82.623057 ] -/BS << -/W 0 ->> -/Dest (cb151-1) ->> -endobj -714 0 obj -<< -/Filter /FlateDecode -/Length 4658 ->> -stream -xœí]Y$9~¯_QÏHk|„/iÅX1‚ÄCwÍ6õ"-<ð÷ùœ™>ÒyTuϳ0­šªŒðiþá<¬®_)üH‰¢wt½}ùá"…·ÿ†³ƒAý->Þ^•$/¬Š.Øë÷I`¼ðÑjﮯü׶É_/úÉåó<êªEŒN+{ýçwIEFøུŒ*B%•aTÖ -ïµuŒÊ)SŒ‘SÁ ëIÓRåaÅ`9Ì ­£ -0ÃWf©Š0C{Eœ -f¨h=£Rv(-%«ƒ!’ÈhF§¬phtÇé´.DY‡NyÍéŒΓ²ŠÓEᜳÓl±Q*Nga‹UD’ÓÁ2Ás:[Œ3’ÕÁ¼át¶h´Ndt¶]ÕÁé¤âtÑ«‹ÐÉàÃR§¥6#9R›†ÕaD¡“œgt¶8i#«ƒ-ÖHÍé l!üq:ð„5>f´ê4º4ŠÓYØ¢µ'V[Ðùh[¤w`‰ÄKhïEM è` - èyo8ÐÑ -rÑs<`¤‚N+ŽŒ„-–ÇFÁÂÈeu°ÅDâ¸ÀhØ‚[4Ž Ø–€Ž è–dTð-Iå8.0 \˜)9.0`\q\`@¹€Rä¸À€s ¸“ãÒÅp÷°®±Ns\`@»†‚ã¸À€w )ÉqwANV[`Ž ÀŽÂ¨`8. ð®QÒs\@à] #ÍqzU‡Žã`Eè$ÇÞÕAÇÞÕS«ócÐp\@à]mq -ÜÌ ÞÕVjŽ ¼«I;Ž þ‘¬7Þ…XÞ€- XŽ 0ßC§ Ç(Lh¸%X𮊘þ˜1 ¶*DÇq婢îcuQ(gÆ-ZY(ç"Ǽ‹‰Êp\`Á» ÏÀq8^`FÕXð®2˜©9xWé(9.°à]ø –ãÀA(8`~Jº¬. -Cà¸À‚wAKšãÌÂB0 «ó@R€Œ…„Êqïbz8ð®´Æp\àÀ»’àö°:Øb‚æ¸~ t¶Ì˜†³*¤Sp:ð®TèaNÞ•ÒGŽ x7M›8Ÿ:p\¼`Íq $¢÷Žã%tpùX8¶x e±:ØBž8.€c-¢‰ã¯a‹Ñšã¯a \Ž àˆ¨à}³ál\Øø€`  Ž 0‰ á™s:[àds\O:¥8.¶Àä¸ è€(ÓIŽ 0C#–DSs:’)Î r"¹Š`˜ ÒS²:Øšà¸ý“‚mËq‚›sXl1–8.A§p3p\lÑRs\l¦xÓs\%lÇÇqA”N$7™ãÐ8tRr\U@¼©‰ãŒËoŽ ¢‘ˆ7â1ã6—âMÏqA$Øbᢰ:ا)2㜄xìÄélÑ‘8.ˆÎ¥x3p\7ñ&dÄ° x÷%æ¨ÉÌJ²q}쟽üô “ïõ¯ÿº¼üdX)Ë -š€g ÑÝÞ·Âé‡s° †Ý“6Uêc5J§E¶d¦»ôïºü1³†~;¯˜bê ÿ $ÖÖ£™ÞÑèe³ÿüÃü2®mºT6B;—–30È4Ø0úðý姿øÝïóÇß^.þ RÚg|@±„ß„é“ÉøÛ')­5ŒÇ6é’ü%¥ùÙ_®¾iëÔ)̎ɡיªQÇä¿ø04)Ýìá%-‡!¦B{mš¶šÖ¾3Ë[Úzjbœœœ}H#g'iÑî”+›¢*c FÐ! -„®íÐ ¿þæ›_þêµäÚHŽ¢Ÿ}¥¾NM=}ûé[ß)M–Ñˤsãw*'uÝ:9dVÕ2rþAn&™Ê ó¼åXÍó=B#ýþ¢ìÏH,ƒhuì(k¶O~ÖŽµÌOL-2 š0X³8`Šháüun•“i öFL8ø±›Cç`žÓز1v¨Ãå~o0ß'‹Oú\±„×ÛŠ.Œú¬›¥ù8}Oz­'}þ|œ¾iúMMYºnŽÙ¤+cAwçAó±2¤ÍX¿Ulò#m1ŒSUZš‚“z‘¢ÞCÕÁ<'AU5n ª°–íØÌu-æ-<0`×sƒ,ÖÞÊ=™ó&mõÞ ûnnõlg+åt¶HtsÙ ];u±ç¨¦<¦«SöÈkòö¶û{ÚkÁºpîà~yçö|,ÏéLñâ)Ñ´Ã[r^æ>õ#0Ë3'NHÔêNt>7eRמ?‡’®Ø»‚NkîGç±<§C§Ûä×væZ +·v\Î~ÃØuÆõ\˜§-Ç=gçºuµaæ•Þæè(våôZ{›v5çZ~wu-fù–'_ºzWæ âM¬¡¯ú&èt9¨ÝÙýXžÓ¡oÛgìc’ÌQm¬ òÒã½Ôø—ÿ%¤;GMæ_ñ`ž³¡ÉlûŠif*­ø¼ÁgjÙ{[³–žf£‚V?OSfàãL;|Þ2KæèÆáó¼ä±~öäýùýæt6>K×Ðõ~Çòœ?"?ð QÅt¾Oä±Õ¦+h¬>œMwY­9˜çthÜñûΆ¸E‡Ü{*·OˆÚf½0]ïa¸v½ÚcyN‡Ú5Æÿ5Ô®!íSññËÜîšÃ1þæ‘LÕïLwéÁñÞ‹»æ9’iÛW]»:“üL½á¿åŸE+9o·ú1óßú˜º‰<†‘ÐÄøÜjxAD£+Wz¨¢¤ ã™AÚ•q¥å;Db+èÓáîHé`žÓ¡oÇ¥y«½¿2–ûk€-ÿŠD:Y­ñlÑÕYV‰¨ñ;•Qî¦ÙFë±<§C붯ºhÁžçÿj6ó?1\ùNÈ¿wÆ<õ(h¯ÂÊ DVŸ2ñ‹I7[…=DËs:Dîø¡ïî/½³GÐØQß!oÆãqDýõ´å÷«ôoYÓ´òþ8ý`ž³!ÍþwÖ4ghéP¼:c©ÿ]î³Íšdº'Ñ´¶{ˆ<–çtˆÜñïDËÒGüŒ«?¶¹îlµÞÒÞ -úÁ<§CÞÿ×,ß{õ§ØÔ•ÏÆçkÜ÷ÀúãÁ<§CàÎõjס¢;” ‹87 W{;Ýî­q¬“îy<κt7kJ«iŽƒ™Jߦòž›²¨–ÙÖQî•lÓ>õ&:+åºÉ†¼Z”ò¤»(ÀeòåSØÏ£Ì=àóÌs6”¹mŸïPï6½tWïæmº/ºp¦î¾×îÚ}ª¨buÓ·l÷®Ýî÷x×î -jXÓ;˜çt¨ÙöËJÏ™yÎVj»Ùdv§@‡€ÞÛ+çÐ=:ÝŠqIÓͶù†g>Q¹eÖÏñk³ÄΠ}Ù+±|¾Çü^?Ï=°Þw0Ïéü…ùyŸðë}´œ­=VÐÛ>Órtö>–çtèÝñ7¼ý»Ãxþ_ZvÚã"œ­5Hix´zyL}0ÏÙÐêw®C˜]{±¬lô+A©'¦' jÚÞéVk†ÎßlŸ4dW³û•Ÿ>~V˲ô[ãðÖ }ãcJ:öHíÁ<§CØŽ_Úó³N¸õDÀÐ{SÔAÏ’]û:W×Ê¥y¹wEC4}üxÌêlÁ<ÉÑÐ -šx^å`žÓ¡iÛ7\ćqÙ’‡|ºŽ“Ö®õΞédÜuåRg¼6ZéŸ0Íœ);ô>7öär>V; -‚7žÁ¹÷Ê›÷÷_ç8˜çtH;v¯aë—üØžýä¾ÙÖhºÍÛg/¬Ìs6d†Ÿí O:õþMëßÍîã{Zᙩ÷‡YÏ5Ǧû­§½ƒíÍÁú»BÃ5X7Ý£ìÆãṸ´Š­êJ¶ÎoÖGî°¬äðæaú®Âmxó°Ó§7§.MrÒºñS^éí¦rB†ÒTß“ßXœÖ$§W‚ÏÊ·SÝ=4½¹œƒÍç°èÆ.nØKB û Éäè´áàÕ‘H·~ûxMûWiHånSCC«ëJÎa›ÆJªiçF™6sJyŒÛ:6Sú†ÿþ¢"‰Ôþ‰Æ]Öä ^{׉_/i—=bÄDB'áôS–²Hå„Ev›zû”Æ -9ýŒ¥ðôÚäñ§¶ùlR~›³¥ÝHæDfxÿÿX1#DN§–âtöR¨Iœv* ÂLG®˜b]¶ÂZ µik5ã®XÙíR(›êªXS¶G¥mJÓ¹*,¦·M§šîÏiµoºsKx» ZÓK_YiÚ-•[WË"3Q±HmNY:„J?1¢Û…~%sÑ„Î,ú¦×Ò.3£0d QIs:Ft»°é¦Òlé)_R…š5æ–¯¦T¨›Pž‘ m4I}Úzµ#pGtî‡VìRä¾N› L]P‡nÞ.–ÊH¬â׋õLj[¹BÚ ½Vhšr‹8ÈŠ—*MÔø C3ËÒéÿ‘*l‘&æ~(*JûìÔ£ZI›E‘nI]Zl&mš¬‘û˜ÍK$UÈc.VuEœ(,·OáÊV(³¨®ŠmòëL\í´e¨ì¹2§¸ÛRn»Â¯­Ø—WiÓ¤bo)·Á|Þ†‰zq‚~È•T±–e¶«•5ÂjH¡Š+¾â9TcÒvEÙ*´§ƒÃPu¦NÉצH¥¤uJ‡–t“m¡8l*ŒÕ²™™ m%¥é“ýÔv…Iu5‘MÕQfÛ¢³l8‹& \`Ó´“j+3{£^oî÷Éd—G@’j/Ò×VZN,I áñR3ºùa™sÌ°?Q¼µ…JY¦{¦89uhTׂf-Fi¨Ù¦¶­”²lViž( -]@&³¬”è³MnÊ–›e8ãÀ¶’/>E•þ­miÛöJiDÛöŠ[¤Måæ¯xK¦8@¶…³¯g'kÊâKFæÚÜEJ5¥É²eÝ4¯;϶×æ4¨âªg*k6»Hùÿ=2vØ=¾¸TùoXQ¡nÓ†‰š”7Ú\crw‚sàJ›§z -ÍRÉÝy¶×N–+$yí¤­ Ý!Èéð¶bÂ+q_Ñõ:ípÇg‹’=¥û€Yª4·Ü‘mØD{X«ˆÊ¡O¡¢ÃV§2„´Õ¯—ü&Sé¡E³å“áï?ܼ{a -endstream -endobj -715 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 714 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -716 0 obj -<< -/Filter /FlateDecode -/Length 4929 ->> -stream -xœí]K丑¾ç¯È³‘ù~†¼Ø… ÜØ=,öЕ3ecQ³Àxû÷ý…R$CTè‘YÕcµí.dg*‚ùñc0DIúªð÷ÆÉé!¥ƒ»Þ~¼ütQCô -ÿƳƒQý>Ñ_µrqð:‡ä¯?’ÀÆ!fob¸¾]ð_wÈ“¿]þó—ÿçÑW3äŒö׿ü@*g‡˜bT^Pe¨”¶‚Êû!Fム -zÀ)æ,©`†Î¸¥* —“—T0ÓU‚ÖEm—ª 3LÔNRÁ }TZÁm”u0D9g Ó~hô éŒBÊ&‹ºŽFÒY;„è´×’.!Ÿ$ƒ->+-é'çD]†N¥˜–:£üà“µJÒi=x`ÓŠ:Œ(tRˆ‚ÎÀ– |u°Å[e$…-’<ámLI­†F·UVK:[Œ‰NÔÁt¾Ä&ÀƒDAñ’Ä&ÆÁeã$0É.¹$±€Iyp1Z‰Löƒ 9J<`•†Îh‰¬‚-Þ‰ ¬†-#WÔÁ›ÄÖÀ$¸ÀZØbœ•¸À‚mÐ"qÝ:•µÄ|ë”X.ÌTX0®MÁI\`A¹€R–¸À‚s-¸Sâ ÒÅpX°®õÁH\`A»Ö¥ qïZ§•Ä¼k-rŠ:ØK\v¬NVâÞµZE‰ xÃÈH\€^€Ã q°2˜””ļk’r8𮉘D]0­Ä¼kQ{‰ ¢‚Àƒ²DlqÑI\ÇzÈ6'‰ ¢-Ö‰ ¢-p$.€G0d ï[\ÀÀE\8ØOQâ0Ù²sD°C«Jº`†A×¢.A§£Ä1ºS±–¸k©!aÐK\‚È]’¸f@§Ä1à ‰ 0 ÉÀ3—t¶ÀÉ–¸ž.tZK\ l7(qA2 «Ì $.À µ$šZÒ9EëÌ$qAra WQâ ¬4Á”¢¶€&$.@ÿÐbÛK\€Å›Xæˆ:Øb½“¸ %CËÍ$qAJ°Å(#q°IëÍ(qAV°ŸÄY…Üd‰ @ãÐ)%qAÖ ëMã$.À¸¤õf’¸ […õ&–x¸Í6Ðz3J\lñpQDlÓ”…q NÂzì$él1ÙI\C õf’¸nÖ›ðèÅ@¢õfÃà]ŸÁL’¼ Ï4H\€¬7áÙJk{âõþ™¬L´âLbÄ@z=æV1d À½˜­¢3P _Z#‰A˜‰•e -bÔÓ*”Z‰aþ…?æœp>ñU"æÌé$3†…ƒw_×-™]IvýåO—_~Æä{ýÓÿ]^1FÊÊŸ†ƒ0¦vð¯0 LFw{xߧo€ehb&’" aà3yȈ˜›‘´8ÝÈHLôïºü1³†7Vokõ³øX½ƒC†qãÈ Ë×Þâ³| Á -‹#̶il6ÄN€üJåf‰óé NVÜc õ™ù‹agüõÓ—_þÛ·ßþË¿þ0úôzù¯_) ë×ßhúŽ÷o¯¦ãÛtæßUn¦oÀGëé÷­•WËy½Ã5TH:~\˜ëJ™.µ2ýdÏ,½c²Ìò½ÎóªÇ°ò¦²J9£î‡I7¥q¾³Á7[å¤y›Py‹vz=^—V¿þïë§o/¿ý4ÞèÞ‡Š%L¸LŒ´‰ÕcYÞƒÕ/Q,m· Úw›íºC-¡V»£ïVŸÐwaþ]a¿Õ×.ÝôÝC ØNP-ù«minÏì|üzýuh*á|zûu7„>O¶ØÎî~ˆ–rnm¶ů ¤ÅVäMÜÌs6àb%µ…\Bœü%¹”+ˆv‰ô­_Ñ!´öB‡êŠ€Øzé‘}•ìûózaHëmûžåë‘Ë'ÕÜ·'z«·Ó¨WŠ­#-}¦z -¡ì -zm¥P‡ŸpÉsÈ{è=–çtèu›¼»ÖË®ïµ þ#^á=YËzaû¹›ºÑ;Úv¼gçhªåKnÀ­¡¨ðÚ˜ÎÍÓõ£Jš7êhèÑ,ؾ6kÿËs:Dí8›}ë•ø¡ãGÝP6öÞŠ#WÒèÐñÒ^zÁ©üûr(ßã5­ ˜¹–Ê ã*hoåt0ÏéP¼ã@צßÙËCú ãœ8ﱊŽïï(?¯“¸2û¹"¿ÍË2z*ÇO|K<}ëFÑÊúQ".Ùúsíæ'ùá½w°Ã¯†ûŽ×Ióœ ™fÛß|Êc¾â?Ï=àW³úzºGö؆ä÷Çé³ákšæ7jº”§mˆ»È=–çtÈ=ækV¤t>Ú¬§;.#t÷è[åŠ'|±ƒyN×âÛ¾˜ï°ÞóA?VǵÎÍÐz*´™Å¤ûzÊîå~,é(OY½%ïü6•ûÂò¹–Ÿ×5®“iÆâi?ßë C-7L¶N <é>.ÇõàGœ‡Œ/€;Š¯cyN‡¯/éÙ>f}õP‡;.ˆs?¬œGgG<žeåà·5“m· ÓãŠWf,ûD4í`ž³!Ên{7„Šgz–¢?_3û¼ÏþT±ˆULƒ¶.†°‡ªcyN‡ªmÏãoÍ;÷î‡ñÍ4¢vÖýÖ?¾º:˜çtHÙ‰^±¸â[9tqϺ¢+¾îùÃ;qÙY4­_3¯Ž_Kª+.SðŸ›l+ú¿~èºR‰×nFãVÐÊ|)m†¬’¡#Ûh=–çth}.Jõ%Öð_ n,8™ë®$ÔÌGW'J,d‘î³ZnŽè¯™Í"i2R]óÑí~ñ.ª6‘z0ÏÙêv¢Vg|t+ÉÆ%¿¾¨Ù -"›çé†:¸ao¦?˜çtˆÜñ WfÝG¢úÄ!ÖÐ_kû*ã¾+~"ºw0Ïé|l[ß»¼‚–‡"ì+×íi†­¾Þ øëÚ¸‚Þæ‡ÒgÎa©¶·&?˜çtèÝñ]\÷ù „ÿl=í®Eþ‰¸ãÁé«níž>rÕv6 »ÖÃkwµÌ>EN×ì?ß÷Ð÷Œìˆüb#y}OÄYæ9úvüÔ¿Gž{$RµÎÅJéÆê\Ý㿲ºûRH~âëƒyN‡ä÷Å\¿J$D¬€q:¿?u\]ö~xß<ßfÓƒcw&ÊèÌ?=zÅö`ž³¡7îì í{«[µ¯”>±¯à`žÓ¡tg_Á#wGïÜ++=IbÜÛÙß9ýs?ã+¾r™zô׃yN‡ÔƒwýóʼÃÇvý¼ÄóyÏã•Öxõ‰ìÁ<§Cë¶ïZ{îuŽ’Ù^M­æqÕ2®DÙkÏtÞ@ZÚW?Û5˜æéƽ«}D¨øu¯óü tL¨c¿+±…2²T¹çàgžm£Í‘'÷„ºëgÕþ•»PÒ1ÙƒyΆà´ó¤ ž3Bw|T$•–ïxä©^Šj¹+;Éöl=¹‡£P|îT\ÚJNËb…´V×4Ìp«eíZyœ_—ëæõº‚dû8Ìs:$oû¸VXÏ.® ²g»klk\XÊê¸Ô²+¯‹ërŽÕ±ó¼*цþêw¯ÄÆhë´Ë¼3¦‘1ýD”÷`žÓaúà_lgA¹×¯bû…Qwúºú#þ^&1»Ä”‹{ó7úyíZòÚ=+ÒU)ÃýLà;¯xk6êý Ó³çÁ1hÄj×^âŒÂÏSÆï3ÏÏü:c»i'¶+àwÖÎ÷`Õ.y+Ü7íïŒ9àÛ>{GáZ,Å1ìÖ1ÐÇWØ—[°ÅU^#Ø =ëå6]Ï»­Çl³z<Æp0ÏÙP™wb¶¼å^¦«Ÿ/S«ºö=¶(=«ªÜC¹1“¾|¾—SšC±×>š¤Ù ÿÒ¡!ÍQ²¸ÖÅ…g«µÞ÷µK]×âÞ„ù‰øêÁ<§CÔ±ý®‹UÇÊ>ÌYïñë;ý£æß}Ïï÷ìÊê‘¥d[*÷XaqôÔ:óGMsù‘'ÿŽéúúØ~­g®¬æ'ž´~0ÏéPù¾=Gøgìrœï½6ëåükÈ­«»”Íì[ñ"Çú]‹¡Ž®îu.ê<έ¹mjy܇3Çü¾ƒyN‡¼¿Oð‘\¿éÆûغ†ÍT…ûÊ^VážøÅ1]diö¬Á&‘ÏúuwœÛ±ˆm\_úâç´3ƒo_m­(þnzÏFôèpOï¥7edöÚŽ -ÝÝ$;H]¾ä£¼*ãJ=ÄÁÇa ìÚt…sçr¤wÒô~ûû?üûüž…èqÛ?Ü—£Ã¦ÍïÓ£zˣѵfNݸL,äUˆ‹ˆ„º‡ˆ©A„¾ïËBêš{÷Ûž)m¸Ʋ¦ßc9©,]¦úz”ŸBôñ®Ÿ•ï§º™=cžƒ/ç°èÆŸ.a|·’߯§B´¡ð^ƒ(ró•Þç””uºt›Z_WrŽ¯-þ©¦šÞd¬èå†ôb«ûkŽÙÁ”þ§±Ã¼èìj"‘û[GÕ(N¦4ñÛ…Þ:ë±Ã -ä ˜„ÓOUËrº$¬²¸8¹Oi}y+ŒÎµpëËOãËÙP~_²ÑÛ€œ*‰ìø>œ{Å‚9ƒ^ŠéìÕ '1½¹7 v: -ՊͶOÛ -lwÅÚ*&ôK¡bÕ5±qÅMoz«Mš°šÎ›N³î/iMdݹ%¼]P­í¥o¢”ÞZŠ*­kT•YfT®R_RÖqµŸÑí"¿Q¥h‡Î¬zÖkôÖµ»0 ¹š.—t‚èvÓM¥ùÚS±¦J-k.-ßLiP·©6¼ Ûh’Fz9#Ê–Ÿsq (}­ÃYoïêHlâ·‹Bj߸Bù=.´¬Ü*Nªá¥Iéár÷_šEFﬢÿïTá«”˜gü¡]…½w®µJhlVñ¼‘¸5~úÙ™4ãÞ.¾ŒÓœ±ÞdüVÓ;q¹×¥¯ÿ4ßâó?wý]y Û}7ª²…©Ü:ó™ŒÞ žŠéêEX‡ Çh¿‰uji}ACÇa|íZvÕˆ½¤Ô•AuÕr¡omÈĦB×hËZÆoo]+VGæ½}Ó&ìû¡-'‡j,½¾¯X…~:8LMgÛ”|eE−¶)ZgXFðBqÈ*ÌͲ™™H^Imz²ßñ®°T;`nÚ¦smšåEÙx, \`ËxÁNóJóÌÁ觇׻û}29”@ÒJíUúÆ¥õÄHZ O–Úñ0Ìëœc¯tPok¡Z–ecÏV'§ æZ¸ÙÇ(“¦–mJày¥®Èf•–‰¢ÒdªÈj‰±Ø¦l¥YÆ3Nb+ÅêS4éŸyK{Þ+µ=ï•°HKå–¯zK¶:@žÃ9¶³S-eõ%“ ¯þV ƒ<먴*¯šúYødüû+õü -endstream -endobj -717 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 716 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -718 0 obj -<< -/Filter /FlateDecode -/Length 4732 ->> -stream -xœí]KäÈq¾×¯¨³€¥òùŒ=a­aA‚öAðaºvZ‚Ñk`¥ƒÿ¾¿Hæ‹dÉêy¨$y=]ŒÈGddäI2K_~¾Óø/’žbLÁÓõöÓåç‹š‚Sø—K,.2ûwø Éé䣻þĦœ þúvÁ«Ë±øÛå?qùŸe}5SJÞhwýó'f‘B A9•ÀRÚ -,禌óËë CLIbA ÈЖ ¥è$Ä c¢ÀŠÃRÐvËJÃM bèä‚ÀÒ -rh£”ȃ ŠÈ§Ýä¡t/ñŒž|L&‰¼žFâY;ù@Úi‰—&ｋ ‹KJK<Yœ&R²AâyÈb½U"²˜¬Ä Å@;IàEÈëŠ"²(¯´ÄKvr)‰¼žŠ!nyF¹ÉEk•ÄÓzr°M+ò°¢0I><Y¼rIäAg•‘x²~$pÂÙ£°Z ¯n«¬–x²HäAL¾„ÆC¼XAã’„&„‰’! L´EŠ -˜˜& -ÁJ0`’›È§ á€U<£% ° -²8òX Y+WäA›HÂk -HX`-d1d%,°@[‚µHX`·¤’–°ÀoIi/aàBL%aâÚèI ȅ)% ,0×;%,°],÷ aêZç„°k)z ,p×’Vp×ZÔy,aÐq²:Z ¸kµ -pËÈHX€Y`‡^ÂØÊdbTp×DEp׸‘&¬A+awÃ$Ï Ü5N ¸kÈx `(ðG‰Ñpâäp²`%,€¿ÏX ÐØd–HXà€»:Áý kh5阼„¸«#¦Oä¥Ih\X·Ðò¤½O8à.••°ÀwŸQÂ`üj$,pÀ]má©%pW›¤$,pÀ]Ä N˜äHXÿ6iÓyiR)F p°d$,€žT’ˆ¼0Á´„ãI!•°ÀwáÞ“„¸«œµxà®"„="²Øh$,@\–­°¦¬NÊ)$pWiÌ°Äî*’„¸ËnS˜`¢„pBl$,𑦂—°À'B>‘ç'\; ‚Á²Dd¡@ °ž’MQ‚` ‹5F‚` B LI#úÓÈsó‚,ˆ%,’M1IXœG>­J[Tæà3càA&, BtßrŽ^ÌÞ)6ïýù—_~„ó½þñ/—×_ä²ú£aü–<'Öˆï'ÞvÁ|¯¦Ü!ÃD°mر„ð=uXˆº°X6ÝXH¸b¸ÛtmôRdHÌ2k3yd´è©‚Æ\Y/Êúp//+ò?„DˆÞ–ýÊB>”Îò¿ëöà Øö ß;ƒ,Nöl¥/(ôÖ ÿùÃÁ±Qm7ø»~9X*’w„ÚÄ뇟.¿üÕo~ûoÿñÌ݇×Ëþ Øï¾ÿNó_¬D*ˆD8q¦9]þúò·ÒRç²0~E/3ºÉ×z)‹.ò/ýLã_n‹iäû5×Í4ø&ºá÷µ·‘û(rQ8Ñî§r}“Úüþ¿®~8¥Å4jñu°¬ˆƒbTé¬*.tå5áGåù® ª¸\f]Ç–_VŽ)m—ë<˜¢øÚ®zÁ$Ò Wê©eù¬Ä[Q„/ðãÐÙî‘Òó6q»-cÀ@³2ó¤ý¸mÇ[Ž¡ò>ݦUÆ59{OîÉ¡úXú¸ñ¸y>DÙx¿Ö+ -mç…æúÙX^K[G ³Ìa6äºF£®†ç‡:¡´_ôØí§´%Œ£/‚ÞO®_øÍÀ?–6Ý`è²^ãzìë…Ä} ÆR:°~¯.}»y9»#¯¿gÏû¶Leª®¶=È&é>~-å|o3ƒîcªs˜e}ííU[ZÚ‘.ZëR´ [æ}°“¦s_ô0ÚË=;’ïI}æëOeº®ƒbvW\Êõ¡O:¸×¢ÓaKŽ.»ŸÖÞ®îèhí;Aîo·¾E³ècö-Ë5RñõWr¨d{ðÉ7Nr¨Äûì&jãàYãÄ -…«Gë¼+`-ÑÔdx'ãl2œ9#R)*Ú„XT{UÓ.µuM^c‘¼:,‹cýú‡þå_?ÄXlž%΢[­Êr³CŒ5OÂÝ‘&ªÄ{¨««»ÊS[\>O±²ßï·hlB†ˆ‹ ´è^j#-ç[Þ‹òÕv~Js&ÕÞÓXvl•W{¨=ËúHü=íµ¶Bi»g¡c#²ð£–ïJ#:Ô¦åÙ¥¼©òm’¢)Øh¬û6Ú¤ˆü=DBètß]*šÝí€Ñr^o"Ù¼œñbcÜ^–NtàÚ{Aë6(䛲‰o¬:Íš¼rIUÀrÍĘ7S÷Aî\ÏI!¿Ö±¥í`ú8ºÁ圷¹¯9§VÙŠWWawW¥Üë²¼*®ºÙõ:5l]ÛÞºOÝûËa?ÍŸmIãL± ‡ÇöiÅ»}ãñœ__|Kzg}µµzà]ȶ¾Ò$Õ [”¥)%ƒ^u¸·¶(è¶Nj”Î7Fv×ÖÉ:O·¶¢Û[[Ç -Z$ÞÓÁ/æ½}¶ÙÛ^œù•·(ócù[øÆ~ýý±ü¥ò™†¶rz¶Ú;ªëÂt›¯[m´\»~ëŒ.†ðycUp·ÕB,Ï"%shUçê<‰U5±¡²=«ªlÈ»ž¹°D‘y5sGèÆF´7{s²º”ƒÖhçº%lÚYÉž-Ñ/i ë:èKcÝÜ[y§Ö–÷ùÞmÖ—lÁÎ4Œ£'m‰#ñ} >Yçé,ؾ ŸÒšp+[ÎkÝÖXòÚ -¬ôŠ‰Å~Ð:_†6I¶ÐõZ¹&ïë$ÿ¸už«ótÖéwñuô\˪ÚN[ï—×~ìw1sqY¶]¯1»ömº 9z]óÕJ®Z~e­k™6r cmŸW}m¼üˆ“¯«~ïø eß±¾ÿ!)懖£>òî'ë<õínjͻWÌXç$zžÕ3¸ôþYâË¿’¥œãkzG¬x²Î³Y“ßõë Å—<ÓÛÙÛóZ¦x£f­aY¦yàgO›?ÇKÖì†7ï_¶8v&»ÿöq³l~ˆéí Ÿ¬ót–÷ÿ{–_z÷§É´j_ÌÏïaß;öOÖy: <¸_½z×t“®¬,of ¯¦ÕÙæ'ŸŸÝ5/óuåñÓ¬\¶>çÛèðTæVÚ{Ú¢ÞæØG{Vr,ûqî—á¬µë‹ u·ˆëðSü|üë×ÿŽ•½#æ;YçÙ¬,íÇ|§fw˜¥‡f·z´ú²YÅL³ú{ï©ÝݪD^ù«Æço}jwõy~jW¶šôŽ=½“užÎjöã²þêõr;µ+o²xR`eëì¨SÏè¬vŒ[™•·­c8óæÓ{Ûm^¿æ¯ÃNèA×mßÉåûk®Åyéû}'ë<%ÿÅy_ñëcky6}ܱÞñ–“Þûd§³Þƒq'ÚÈr„ÈÿoÏZôñ2œ½=ÈúfêÆZÓã9õÉ:Ïf­Z܈>á^×AbÛÚXoñT”כٌӳڮÉ[:«€s|¥PÜÎ^oý¬h½mË|n"~nÃðwå¯à0Ž/÷ɤ4œÖìê°ÈÕW&·gG]õêè«å[•‰Û -|(°µ¤ÀÇÝÝ{§Ò|šóŽ«û’½•ƒRòŽëuɹL}3?P¦—§d`Þ§®¶üNf9ø"ôfz>¼Á«~(Fù¼>|&÷W‘ɇE l´öëÁƒ<‹Ã9Ê]y ›iüùâó±:Ý«|œ´á³ý¯ž&NŠCºòQ‘QYÒuÚtV´¾Þ©™¿áçVª|I‚âs“ùÌÌ<­ãE)ÿsžðŸ.:ÑÄúχ?åÍU&çVä· hO™¨Ÿ u@fbù¨Z[¤ÛT•vP†)­K"aQåc?µŠ_(?WGÃõ]­Æ ’ª…l>joîX ¢¦×[2^MºùKâdË•o¢ð!ؤ±lo°W<$k«¢ÛÕÐ]'ªòh>D¶©Îwb}T¦¿–5a˜Î=âí‚níšú&Rù(òÚTÕ®Qf¡R£ºZ²MµyH·‹@üNÕ¦ “Ùøìñ®31VC¢V.ÕrévË•Ö\›©ÐJÅ^5UÍwQº©ÛØ/ÐnÃÉn¿ådX#ÊÖK2Ÿ -ÖæZÅÁn̆x»8j+±“ß..¥]Ç -åªéD;´ÛÈQu{éT~;nþ„¥Yi|&ÿ?C…kTFžüAS3)>Ò¶_õNxm6òRI#Ý7Ô!IšB/;¨K‡ÐlvÔ—ŽJ(Ï ÛJ7-¨ƒÊzHU<©K²nV×È aU? +G¢ªB »NvÍ”ßä.§kKթ֙׃¹»Ön'û†o#9´×|>I“·µ;Ø|%Þ²c£5™M?ÖN:Ù¨æízg± Pèä£o™\WkC'þΕú±/2ÉãÞ.ºd7g¬3|‚“·šÛ£.}ý_”ù¿ÿ}¡ë¿×ó]g?nTC Ó°uA=9KK¥êOÞѱŸˆGM:Xcó—Ü©è=0„ø{JÅa«äá:û{'Û’ºw2ö„é˜ÈZ4]¡,Âé˜14U¿ÊP®–”8¤Ç l u˜ÛÑ¿¯*ïCæ ¢v8U䯧³ôFþV ´|H7oØúÅöIþù?¡S § -endstream -endobj -719 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 718 0 R -/Resources 4 0 R -/Annots [ 720 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -720 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 612.273057 101.433071 593.523057 ] -/BS << -/W 0 ->> -/Dest (cb154-1) ->> -endobj -721 0 obj -<< -/Filter /FlateDecode -/Length 4685 ->> -stream -xœå]K䶾÷¯èsÓ|?€À‡’ bÄÈ"99Ìôzl³Ö9äïç£$>D•$v÷ÌXãì`vZU¤T"?~¬*QMqæøùJà?¯ó>8«Ï—ϧ/'Îœáø7”˜ êïñëÌYpí˜ÁzsþÊ1Œtöü|ÂÍa]üùô÷ßœþ=¯#Î’…`¥0矈*­˜óÎqC¨T\(Be sNK¨¬`¸Å(Ì0NK½T9˜¡ƒ7” -fh)=¡ò0Ci'ÔR`†tBS*˜!‚q„JpØ!$礆p­•$tÂ0‹F·”N -f}Ô9è„“”N)fFPºÀ¬µÆS: [Là‚ÒØb„ÖœÒÁ­¼£t¶(«8©ƒ-Ò;Eél‘h@è0휢h@ô ŽâÅtRPD 8l1ÚRL lѹ¤¶¨ ).P¶ ÅJÁ©Å -l«Š èVó (.Pà[Í…¥¸@pa&§¸@q•·šâʔŠ-œ«À(.†»£¸@u•±’âÚUÚ[Š xWiÁ).Ðà]¥P“ÔÁ˜â°#SÂ+Š 4xW î(.Ðà] #Iqz•‡–â`…Iï9ż+=×hð®t˜HcƒŠâ Þ•·@ÍÜà]i¸¤¸@ƒw¥––â…8é €wA!†v` –âÌ÷ÐIEqNÆ$ÜŠ xWLĘ[1ქ¸À€w…G÷‘ºÀ„C‹ã­Ì„µâÞÅD¥(.0à]Ч§¸Ï0£JŠ xW(ÌÔ”¼+dàð.üCqàÀŠ 0¿1Á]RÞS\`À» %IqfaÆ=˜„Ô9 Š @ƌå¸À‚w1½Š ,x—¥(.°à]®áö:Ø¢¼¤¸~ t¶Ä˜†³Ê¸SP:ð.èaJÞåÜŠ ,x7N›XÒS\¼`Iqõšç,Å6pèàò‘:Ëpl(.pÊ"u°E;MqkTð8 [””8 [à2P\€ï› ` àBƶÀS¤¸LÆ|Кâg,â1´*¥³’yº&u:á(.pN3LÅ‚âÄRÌcÐS\‚gÑ]¢¸f@'4Å.À Š 01/á™S:[àdS\O:!(.ð¶À¤¸ÀK(ÓrŠ 0C#–DSS:Ícœé).ðÚ²è*R\€a‚HLIê` h‚âôO ¶ Å~l"Ì!u°EMq÷2†›žâïa‹ä’â`3Æ›Žâ‚Àa <>Š ·,ºÉ€Æ¡ãœâ‚ <âM©).À¸Œñ¦§¸ (Žx!1nƒ²1Þt [ \R[à4bÜ‚“o‚(…-2hŠ ‚µ1ÞôÀÍ@¼ žL øo:2]Þ5ÌDéÀ»ðL-Å8@¼ Ï–Ší9ˆ×8øg´ÒLjӓê5˜[É”÷b¶rd΀ƒ|cŒD& `&"Koɬ¦U('Óü L/8aÌ>Õ)* 88Ì™ÞÅ› Þ}Ž9J1µRlÌýüãéëL¾çÿszúÍ)K?ÂPZÿÂ0Ýmà} ܾ²èhÁ¡,„·Ô‰F¸PŒ Å)éÄôÿ—f–Ãðï‡Ë«|ù€Y|¼<È[ -„вGø:ä+‹¯¬ò‚sG˜m-ʨ #;rð+¹žþÝÇ€:Y1æ@óae>æKÃZšóÇϧ¯ÿøáÃïÿðWÀèãÓé¿åܘo¾ø«ýø׸æXT²Ë$“E‡•‹üè£C†Ï|üÕv*×\CëæZkçÕóó¦óärr~žáÚSáoüuã1©‹ŸÕ¤{hÊ|šþNr RßüóüñÃéÛs€ ^ã ’ˆµ;˜ê«r¦^JA7 ›røý4ý ËfÔ?L]– Ð@c€M‚‡jàah&8„¥lCÝ\³…«®ý4‡–~å†7Ð}¬ìIçùTìÈðUÄ’=íP³s»§a³ã 4Ñ™…Û)ÍÎ:ë hˆP¶–P”[ò‚Ï¡ôDêÅYoÈõ–t;ÈÌhdY/}Î(yê¿Ö€8Q¡Ð-õTÑŒœ¤ãËQ±9š.óö)×]A¦Êtf⣠ùèum#³¯Îá©79P7½md,cª½Ì{o襄ŒÄ;bä$’g¦Þ¦<[«æ³œê„^Ë©;¢+Ùµ5m6«¦®$µÕvyÞÙè~H#z•ÃÖƾ:‡Cã¶sw¾{ d‰R>>¿ÖŒßÚûÑÕ½®ÊÇ¥“º…Ôvt“m×ںǥ•k ¾oì¡·¯ÎáлíO^Ó©/ýqìe–3`2Ô3ªòë=9ãÉ„¬KÕ˵ÙÚêæçZ ¯²}ÁÕ+Ÿ)ÿ6_‡ð‡#«sf—•ÿhcâ]ívÐØYçhh”;>ç=½ú´íÏiÙôÎćR¼=§å2Oó¿yä<4¶WÈß⻺=öýß4V>£,p/cZp}u‡Æ?óbäM_íqŠ(*,#âÒ\£J]ç.#Œ_½›¨n±•_©"Bt{¾hgÃ!vǽ7æ½&¶} ž¼bTÌÊëJv(t¯ ¸ò-…cC‰ï!¸¯Îá¼íÞ›Ñy1Î~Kt'´^1úz=ˆµY^ñŠÿâb ?<`ßD\g£!Nmûœí윳Áuv(õpäÓø«-bÞœ?“ìqiÛŒ#oˆ)Ï&Ÿ£j»xíÛ#ÆôªëŸuÖ9z·}Té÷{w3ŽMDÓ‹ç.Mþ‘Ê.П¼Œ§ -iyÇ÷2k«âCvg‘:ë…;~çK êÊÜã¯Ï7¼×ûYAiñ»££Î:‡CéN®óë¶'yÕÈ°ó¿ùkhmý¹Ä¥ -rëF~‹lzËÚ3h怶­†üÇ -¿ê|ÔÎ:GC®ÞñQÃø4qÁ/©Ç¢õÄ#më'd4ͽРº]µ1{~ÔÑ£×¢±]=Rû»i­mŸªz{OK9Ý5×+±]†?¾«Œ¿Óue\&¼²`IWþ&>Zë‚ {èí«s8ônû¨¯½F(slûÌÞ.Ÿã¬åbIO ~ö4ñÚP®‰½ÛQEÍy4´h&l_›Éµ¹ûêQ;þfÛz©Úõ:ÕSCYÇÃkÙrËoÏ®ë $½[Ÿò¯iÅ•oÙ»:¸³ÎáP¼ãv ëš¼åìé_póËèø4¢`ø½ãéi^ù6òô¥E;(kG µí_3#¥ñÑf=ÝpY/Yô•¿Áë¬s¸ßy¹¥ÁzËÔÚ…{“1ž²ef‰™ìOɨ{)]¬“⯤WÑ;¿Lç}¬êéR¿¾V~¥.û0^# ù¼v²%qB¬ãÇq9ă/q4¾ê \/¾úê_;^Ò­}\õÕU}lG\DÎ}±>î¹ÆŽö8½H•ÏƒÏJN¶]&L/ÍXö†lZg£!Ên{7·ôlÌþ¼gö¹ÏþTÝð$¶³ÎáPµ³Zðæ«{÷ÅøfQ;q¿½a•^gÃ!e'{Uå_-rXY¿Þ®‹êÉËRo–P^]ý,)G\UN!­I2bŶŽ¼ðUÏ•R¾v3·‚Vý*èÎ:‡CëmYª×ˆá_ ×®-èÉÌ]±öiÈO´åør}DûÌl–I£‘êª7AŒdÜhë÷Þé¬s4¤º¾7–ïÊ3^»šäó’ï/k¶‚ÈÞTî¬s8D¾Á$׬ -~Ë@¼ÿ«Að Ù½Î:‡Cpßʾ{×ÓßýM -/üÕªM]>ãQ¾‚ÞêÍÐçÕöbòÎ:‡CïŽïú–oŒ¼B»jqå‹›nÈ;vÖ9ýŽº²zñÖ‡)\“£† ¿äÚÔvwŒd.kÖK½Ž]Aî oBwÖ9r·ýØ»¿gé­Ú¬¶£Ö…½¿ÞãooæÚvjËìqî ¹ÖÎ:‡Cîuo¦ô~£Ä E UÞ3f×gëç)jÖŒù˜£}â‹w¸®®_¯Êi3IUÝv=ÿÛ¾¸‚ÔžswÖ9Rw|ÕÂwL­X}[Ù[cì°k뽬¶ÎVõšo1‹÷ôº"Øpƒ?ÛYçhˆ ;þìNôx~ÞýžÌa£Å]¯eÍ7ø¸u‡æΕƒ7úª{ß%º÷Ôv6 o}ezæ^ÿ&ùô-ÀúÑôeËé¾zVD¾ÚH^Aß yÖÎ:‡Cß}ßÜó.yîšLÕ:7¿¡cuÿJt÷ZH¾á-ëÎ:‡Cò}9×w‰ä—ÈTœ>ûÖŒ ᇷíP×oÛlZcÐ÷fbFï÷Ó~Π£MÜk3îªí+2dw‹ì t¹ÙEÚ2âZÚ {Ž‹Í*†#ªƒ‹{³DÈ}ûÝ_þô·ï -ä†×réa;ÈÓë|éõi!xþBf9„A íØŒfhæØí±»K÷Å¿Ã2¼¸Üîilê!4Šeíø;œkú<œÇ'mº^ £bý¸ÅúÙùÍtíʞᦺ“îaÑ_NvØcH ûÌqë™q#Ú³Õ,®üsá÷5ò\i‘ºM -Î+5‡í{¿äRÓŽ¾~ÂÐL²¸wSü¤ -“¥‘y†BgHÅý×ÊQ¹H›Y™+k!OFàrEl2”Ÿgâb§ÉC`O³¢‚»Éç-b›¹ó¹»Üã"n¦—íÍç­0Ÿ„—abÓ­8Bߧ‹±äy¶+«„Å0B}Äi´fv¾Õ}úX5ã^N?†iN*#>X%âÞ°µ×%ÎÿE™øý×IŸÿœ6#çqÉ3[ÈÌ­3q=“ÅÄ}§8î,ž„yØTB©+Ú/báKY“Ðçq)+¾Ö¥:/ÄžJŠÌ "kk¡)mX‰e†®ªj³!¼4­˜™{û¦LØã¡J7‡Ë¨¸]² -õíàÐ*Sò¹:%ŽbÑ2¥C«eUõIqX]0ËffF@ÖÉMí×uW¨x­ê r#â¡*:]¦ÙúÔI6ÜEU.°ªêkQ_4Ìì!Œ¾yxÝÝï“É6€(ÍÔž¥Ïµ4ßX”f£¥j8´óÃ<ç¨s—ªÆžÊNNŵгƒ£•Ô—jSS_T'Ùì¢i¢ÈtO²|F—l²SµÔ,Ã{²•\ö)Šô§º¥MÝ+¹MÝ+vQ6ž7ÍxÙ[RÙ25œ]¹;^Jf_Ò2[×ÎR]Jª$[^[ϯfÛsuºb‹gÊK5³(yÿÝ2vÆ:M‚*•~†ÌˆðeûNLÔZ8%Õ9DwÇ[ ÑqSm§}•*¹ºÎvîd™!I¹“úJ覕âNo+&> -endobj -723 0 obj -<< -/Filter /FlateDecode -/Length 4981 ->> -stream -xœå]K$¹q¾×¯¨³€MñýŒ=a¯aÁ‚öAða¦fZ‚Ñk`åƒÿ¾¿`&ÉHfä£{zWyÝ]ÁG$ùñ‹à#³ô]áÿw¿’ÓSJ9wütûù¦¦èþ•«‹¢þ~¢¿kåâäuÉß"SÌÞÄp½á×pÉ“¿Þþã7·ÿ^çÑw3åŒö÷¿|%•³SL1*/¨2TJ[Aåý£ñAP=ás–T0ÃGgÜVa†ËÉK*˜áŒI‚*Á 뢶[U†&j'©`†Î> -*­`‡6J‰:¢œ³FÐi?4ztFO!e“E]„NG#é¬BtÚkI—§‚O’ÎÁŸ•–t¶x휒t°ÅÙ%]€-6X%ê`‹IÑJº[ Z' º[€®$ê`‹ -JKºl'Ÿ“s¢.C§RL[Q~òÉZ%é´ž<°iEF:)DAg`KP>‹:Øâ­2’·ÿ’<ámLI­†F·UVK:[Œ‰NÔÁt¾Ä&ÀƒDAñ’Ä&ÆÉeã$0ÉN.¹$±€Iyr1Z‰Lö“ 9J<`•†Îh‰¬‚-Þ‰ ¬†-#WÔÁ›ÄÖÀ$¸ÀZØbœ•¸À‚mÐ"qÝ:•µÄ|ë”X.ÌTX0®MÁI\`A¹€R–¸À‚s-¸Sâ ÒÅpX°®õÁH\`A»Ö¥ qïZ§•Ä¼k-rŠ:ØK\vœ¬NVâÞµZE‰ xÃÈH\€^€Ã q°2™””ļk’r8ð®‰p¢.NƒVâÞ5· ynð®ñÊH\àÀ»Æ™ q€2” €wA!^` Vâø{茕¸…Ma‰Ä¼«3ÜŸ0¦ÁV“N9H\àÁ»:¡ûD]žtD‹ ã­<é²Ä¼ Ge%.ðà]Ðg’¸?Á£‰ ØhÛz4S'”¥…MDÇÄai‚–\?3ùY>Ð`… <"€46ò  EÌ}”[%þûo ÓÅŠy¾]2óÓi8Gãï?þtûí?ýðÃ?üã¿?¾Üþøwp³jþqîûï4®}šÿº¸ü},ò庤}YdfIói–{»\»u÷þ‚êÁ/Ÿ™Þ/eš2Iþu°kÉã³çªíæîS/Ÿ¿@ÖzO™ì{Úôûÿ¼ÿøÃíw?®WfLoÃêµ,ß‚Õ_¢9Atnj€ÛgEæ–.aT¡Z` {´]w]íÚ -³ -ÕßÇP¶Å¦!’:TVPû:Ô÷Xä¼¾—uš:¤Zž¸®óÊði÷¥…rêu+O†)í,UÐaÒf´KÎàôbžg*f÷GHm„Ö=v•È~mÔÕÏmd(ÔèÔë‘Ôt˨\!+¯Ùê×m$ÞO\—ÙˆslçÇp¿zŸTµíIË2˜ûÒŒâ­×ò<ZÝ!¯¶ž{Y£„øµµ¨î­L|c«ûsCÏ \1†#jÍ Ctîò†ÜË3 D½¬óoб Æzf§^îkYT– oq棭ØÙ}ò0(­ï³ÙvÌÒ˾–çé|¼ŽüåGïÏz܆¤Úò¼«—"ãÏ©£=~ _qn¥2[)hñ£×Þ«kæ¸Õ²vm<þ™¥ü‰zÙArz_ËótH>Žq-CZCÑcÝSÏÙ³ ÏŽü?ra-kàRëX3ÜŸcuè¡þ!FmˆCÝŸÆR§Ýæ]1ˆi£ÞÎÎó<¦Íq4<úcòé¥]#Ã:3*£žôu‹Gü\&1»Ä”cýGý¼ñÿã1#øxã'ðCT|äÆxFÂtKóe¬í%z~Ÿ2~y¬KxqÙœ­:\Ìótø=Ž%ü®Úyäi»–H"-q¤„›˜ºÛ¾o™Ií®¥8†Ý6Æõ•¥š-µüƒÏXÙî—u`AÓl2Ì••#ãß¾Æp1ÏÓ¡òdÁ–·<³ÎóßÒªiùë–ý2Ë( ÉYôõç‹ü™Ò\ZxW“4óðŸ4¤5JÚJÛÝøŽÌqEªÅ¾v«kõnFÈ¢Þ±ºz1ÏÓ!ê8öô{óØ¡w·-»pšY÷Îèó6çÝqmÇfµè`鿧~¼ÕÚPÜÕPï牘·õ~»dçOlvôàŽ6©ôþï~ÿ¯ÿüï¿çñU­¼Bo ÓÆߎ·+³µ…h†û¹rÇ|½‘ÇÛYF?TÖÌÔ‚ÔC‹¼Ö™~Ißd|E…ÕWb&·Øù‰{émÙ%íc°¾7Uz¬´þ•ævš7÷—j?ÖV)wÖ-£èØÀp7­ÿUoÙÒjdõc±*,æ„•žÎZz®l±Ã.=æiý Í-]Zñ˶œ`éÐ%=س-§ÜCmÝPZêÈŽ”öìPý^J×û£ž03‹ ¶Ñ^²PöŒÜ^扽­ç^]û=Q½í:0”†ÅƼü ²½¥ìšæ…¡Þ¬ëœwH—~¨íá«·ëýìCäöõjlƒ†Íl®ýk™Û6 ðµÔl¿²ë{Û±Õ˜Í~ÑéÞž­Ý«]¬«b?-mS°Jv|ð白imƒÖ‡ífÆv{3]a¥~?Fñq_|Á‘mÁ ¶™¥<Žë:6ÆÈ1þl9rFU™¤W§g¾.åy×Á¿ i|;H§å½ŒåE‚ݸ”­n™C›;a·Â&•é™¡ÂJ5|³ß•E‡ïK46Oè]§¢P¢9ËíRy¾,Ñž³Ü>L٣ϤÜG-VÜçøØÓÖ³ôôbå°×zçK'wdSœ,=(+ÝÑikZê]WNп£5²D›Œõ¿Nkºd0ª’ƒÛÛÇ¢_¾ŽëƒH‘ƒð hÏ1l,‹½Ÿ×rœØ½€cëÐé ÜLϺ •––Ü!¹u%,OO7©dèTÿ!É]Ìó$Ó·ÎuÉqúÄ]׺ÏÇmžÕ$}\Îß;ª·wH‹§–äùIXõ©Øë‰ÄìƇÉÿ¸ý#-!d™]%lþÔuãâk›ÚØÐê`¡÷¸5=‡&ë)ÙÞÒ==ÀWQæ&]MÏ€y)˓಴¦§ÒŽqyÞ/ò´Vd¤èõ#5¶‰KèÅÂéÆHyÍ -ÉaXÙÛË>E=TxÄF1ÆÖÆ^[\¿˜çIz½±QÌú¨×/mS¸e[Ç.KŸêf˜ K³¾êVi¾,ýîv[>;V–alÖ( ‚Ì ‡žþ·l’2 !W·¦/æyTu³é)Ö}TíñÈÞ1'ù/ÇdOuÌœÝÇ;Roîñ-‡•¯ø÷ÓöÚA°içbš´u4w9Aðµõî×ò<úŽcÆ+嘋¼ôþ^bñå_ )×îQFS~G¬x1ϳ¡)ÇŠú…µâç>ÓÛÞ;òZfñF ­ãVsõÀ_fO[~¾ÅKÖÙ m×|ÞòØ•µž_?nÞA'‹é½³]ÏÐy-ÏÓ¡óo(ü†YÅ›V"¿iæqÔ¦;hì1Üå£ó<Oâ¾gCÜÍ<.!øyŒ}µl½ðêƒ6ó<jOÖÿ¿¡vi¿ØÎÐÞ zàI¯Eu‘÷ÙÄûj¦gòVÇáêÑÖ™9áV§ë„eïÉIÂœà>{aMÓmžûÎoÀvå4¤Q0küÃòÞ±èÑSžÞ¹N/Ëì5f t§IN V7²¶¯eºëáUXë½.I_8`ÓÝIànëwwºÌ×y¾ì{`†`ópe§2/{ÄõQ˜º;Vºf™ÉP³ÖCT_{s—²åPS9ô´×6íÀŸêûúA±#•ªä2åÈ¥šL±`­üzp‹Ù³:<–—ÙO¹‡M7þ| å]“º¼oX…4iC_Hpn¢ç-c¾Óû-“²N×nÓ¥¡õ}'gù‡Ÿ[ªå›½ì™^ôYº•_,é.þÓMÓ1[5þù-쪈Ë2Í ~½Ñ[ø v®)=“pù¨ZY˜|˜Aö‹Æ))­ŸÔò±c¥m¾ù£ñõn(¿¯ÙèíˆNÕD¶¼p®X"gÐ[1ݽšô"¦o2H“]®B3…+5a/Ðñ´½ÀžñT¬­bB¿*V]WíÑôæÛÖt¡ ›é¼é4ëþš3[sIø¸¡Z;J_E)½?½U[ר&³Ì¨Ü¤¾¦lâZ? ¢ÇM~§jÑÙô¬×è-´³0U ¹–.×t‚èqÓ-¥ùÖS±¥J=k®-ßMéP·©5¼ {°£Þ‘¾š…eëǵ˜N&·¾V‰áÆl„›wm$vñëÍG!µï\¡|…ZVn'ÕñÒ¥´g1ÂЬ2z‡'ýž©Â7)1Où ]ƒ½‡·_õJhl6ñº‘¸<4ÖqR h{ZÖ\:ƆYÞ^:)!=‘nKÝZl%eMÆä1Wóˆ¤y¬Åº¡®‰‰Âjû4®äBU@u]ì”_Wân§oCÕ«VYРÛÅ¡qç+ÇÖãš^ªÜìmå2ÌWá£867Š ú©VÒÅF5o×+cÂnH¡‹;¾q­ÊÛD–}I÷qûó­¸9c½¡“ÈÁjúŽuéûÿ"Íøù¯›»ÿK})íìÇjla·®ÄÜ“Ñ7ʤ‹bú†™*lÆ c´ßÅ:õ´¾¢¡ùqc_»ž]ub¯)ucPÝ´\è{2±iÐ5Ú²–ñÂÇЊ-ùÖ¾é{¾´õæèd5½Î¸Z… ~;¸L]g»K¾³"qEI»K‡Ö–¼P\² -s·le&’WÒšžìw¼+,ÕÅ.XA—¶ë\w³¼è*+wÁ’ ¶ü‚ì4¯4¯ìŒ~÷ðúæ~_Lu´Q{“¾ri»1’6“¥¶\†õeó9¶R®ƒ··P+˲±g[Ó‡F-Üê‚c”IS϶$ð¼RWe«J«£ht™ª²Vb¬6…%[m–rÇIl¥ØbŠ.ý3oiÏ{¥5¢ç½6i©ÜêñZ´d[ä9œc¿;ÕS¶X2 ²Às7©ë)m•mëv뺫·½³ÛpÝÐ#SÕ³ùMÊwñß{ÆÎœgX…RÕÿeeD§þw8j§£5¶|3ŸJ!€C}¹Jt‰-•¼9ÏñÚÉv…¤®ðšÐ“³VEMß±(›ð -E>W°¢ê÷/ÊÙ²oém , ušS›•¡ò%[e²,µËHSEúN=KAo¢¯ŠJËO÷Ò2qZ-Ÿ”ÿÿ@_9º -endstream -endobj -724 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 723 0 R -/Resources 4 0 R -/Annots [ 725 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -725 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 401.373057 101.433071 382.623057 ] -/BS << -/W 0 ->> -/Dest (cb157-1) ->> -endobj -726 0 obj -<< -/Filter /FlateDecode -/Length 4701 ->> -stream -xœå]K·¾Ï¯˜sÓ|_@àC‰‚1"$‡ íÊk#XsÈßÏÇžæ£ÙÕÓ])-[‹ÑLW‘ìjòãWUì—ºJü}¥ð_ %BˆÞÑõñÃåãE -o%þ %&ƒú{|¼½*I^X]°×I`¼ðÑjï®Ïü×m¶ÅŸ/ÿÍåßÓ:êªEŒN+{ýù‡¤"#|ð^ZF¡’Ê0*k…÷Ú:Få”À!ÆÈ©`†õ¤i®ò0ƒb°œ -fÖQ˜aÈ+3WE˜¡½"N3T´žQ) ;”–’ÕÁId4£SV8tºãtZ ¢Ž¬ÎC§¼ætÆçIYÅé¢pÎÙÀé¶Ø(§³°Å*"Éé` ™à9ƒ-ÆÉê`‹Þp:[4z'2º[€®Àê`‹tRqºh„ˆÕEèdða®ÓÒ -Œ‘œN)aMÃê0£0HÎ3: [œ´‘ÕÁk¤æt¶þ8xÂ3[ušÝFÅé,lÑÚ«ƒ-|Ž´ƒ-Ò;Ž0ƒDâ%Ž´÷‚¢&Žt0‚Žtˆ‚¼7 èh¹è90RA§GFÂKŽc£` aæ²:Øb"q\`4lAŽ Œ-š ÇlK@ ÇtK2*Ž ø–¤r.Ì”0® Ž8.0 \@)r\`À¹ÜÉqébº{Ž X×X§9.0 ]CÁq\`À»†”一À»Æ &«ƒ-0Ç`GaT0x×(é9. ð.¦‘渣*€CÇq°"t’ãïê ‰ãïjÀê¼À4xW[ç¹Á»ÚJÍqw5iÇq€"À?’À» ˇ°Ëqü=tÚp\€Æ„FXÂqïª÷ÇÌi°•P!:Ž ,xW «‹Byô83oÑËB99.°à]8*Ãqï‚>Çàxª9.°à]eà©9xWé(9.°à]Ä –ãÀA(À¿ %]V…Œ!p\`Á» %Íq¼°LÂê¼Ç c!‡r\àÀ»pï‘ãÞ•ÖŽ xWÂV[LÐ ®Ó–™ÓV…Ô` -NÞ• -#ÌéÀ»RúÈqï&·ÉqóI Ç0P -Ö¸@"zï8.pQB‡Õ9mËq—XP«ƒ-ä‰ãÖ"š8.ð¶­9.ð¶ dจ -Ñ7›ÀÀ…Ͷ Rä¸L&B$â¸À[‡| ½ÊéœÁƒ®Y]€NyŽ ¼'W¬8.@.%&=Ç x‘Â%Ž `tŠ8.ð¶`ªp\W$‚FdÎélAÍq"]è”⸠hØ‚hã‚ ²L'9.€‡F.‰®æt$Sž8.äD -9.À4A¦ ¦du°4ÁqÆ'%Û–ã$?H6‘æ°:Øb,q\‚NéfภØ¢¥æ¸ØLù¦ç¸ JØ‚ˆã‚(Ha2Ç qè¤ä¸ ª€|Sǘ—)ß D#‘o"Åcæm4.回ã‚H°Å"Dau°ASdæ-8 ù&؉Ó9Ø¢#q\Kùfà¸aòMDôì’@Hù¦g— À»6‚™8x‘©ã¸È7Ùr¹½ñZøŒW†”qvÅ@‚z-|+»d Á½ðVž]3 ß”#±‹0™epìªÜ*”J²Ëü‹xŒfœp[}j—¨ààá3ƒO1-Ñ}É9j1³Pì¶>öó—¯ßÁù^üÏåé7ÃJYþS†Ò„ø -Ó@G ·Eô­pøÆa •„4XÔIFøXT ÅqÑ- ÷—þ]ç?&–Ãðï‡Ý›²û/~Û=&0Ó(¤¯ÃÚ`cñÎ*¯hpÚ ¼­Cub'@q¥¤Iáß½½¨£·5в٘‰d˜´½¾ýpùúoÞüþŒÞ>]þñ[ d}ó•JßþömAkÆßvüuô8n»q{,!šœÊú6†2Œž¦í!ß•ôP÷W¾Í´œz?O£q«§ÇQìÐíc[ZíØDãø`¿Ê¤€òfkú”:ªùÝõE鿧zœäºã Sû­_ê£oþy}ûæòíÛ)‡¨ô@8ž(‘•™Ìn«òÌ~ -¨"ŽÕY÷ß FÃåî}wûž í+Á¾ê‹P¡“Û9ýx‡ÏcÛO<Ó*ý.‚1ÉS†ØÖI÷pÛÎ:coe5Mbà©ôãØÞCÓÕ6Û}(9¬-ûî¶ßDg¥]7Ú0¶3ÔA[ -\&Ÿ>…ý (;óm¬s:”Ýù6n3J»F7{4ã‰Â™ºûö —d$™1^QÅêÆïAgº2Ù«Žz­k¹áóžÿ=”YðŽúÀºÞÆ:§CÍý¸¬Œœ™Žààé²÷꼉jó½} ¶ÖÎoX¼(ue:o›!ü§j·xýœ¿6+A¬íÛ^ÈåohÝçé«}ëœÉ_Xœ×!àPüu-g뽦‰÷¶zïuΆ^³#Þ‰öw!‡‰ü¿<´¬ôÇ+d8÷Ö ¥Y@«ÙŸSo¬s:´Þ5·x×>F,+ýJP‰wc×Èh:šIóx3ëW³û•Ÿ>Vó¶ôKóðë…¦‰1% ­(­!l[Ó!l%.íù‡Y'œqÓU7ŒÞ˜uЃd½k¿ÙÂ¥viÚî®lˆÆ¿m³:Ûd0ïä†lhMÍzžBòέ¡i[Ó¡é~l8Ëã¼'7Åt'-ë-ˆ`Îsç•Ë>{äµÙÊÓ]…3e‡Þ‡ÆžÜÎûjGAp›'÷œ¹óÌÉýç96Ö9Òhå/—¼l}ÿW›Ý›MÓþYË0èÀúàÆ:§CæJÌvø2£y|ÓÆw…wÆ8Žå™qô¯çšmÓýÖcx'–ë¼wBW¿zÅyîö¬âÌc·Èº×nÞîdv¿Ë3zv¾»±Îéиr>ø,|÷ÈRµü,§|bø­?’³ëef³²9óÖ_v÷Üø—Þɹ´‰ £vH5Öл­Îéл²Ö¸c4&gõn£“Îvõ°ÏZjÂòHNx2#«Í­™«ÅŠ­ÝU[3ôµ¹pÏÕ ¿_ÿºI¶‰]ºIC½¶³±ÎÙÐhWbΗŒê V8>7§Í®Të2¢|åni³]ºÃw“ë…®d[³ÿ<ÌÆ:§CãJœù -9òÝXíAÖ+§v\S¸/Ïg3¾úb²ºÄ¹·åË<nWbÑ—æ¼{rÛÏÁ“G¯3§Fv*t/ ¸‰-•Ã’\Cð¶:§CðÆ{]Žž«~-Îþœè>r^}c±äåÝk7Ö9âÜÊùêÎ;ÄÜávu(ðî{G>²ìanÛ„#d\dSÚhúÎ>¼$c\@¯Ù>hcÓ¡wåZÉ°>ºwóØœAôw3ôç]ºõGnýp†þî|ùÖuÇ/Åk»æþ–­«H뜅;o¦>‚ªk¿¼Øð¥ÑÏJ›k$·fG뜥+k/ˆ·ŽÉkf†›~ä/¡µç2—.\Ñ{4úàö_fçl5}ë62诙êújXÿXàW FÝXçlÈõ+1j¼MœñK÷D€áŒ£fz¿»"¬ÄiÝý<³Ø×QÚ3¢»ï×¾sïwAZoÛû¦ÞÚÙRÉ÷GËõFÝ/3Ü ”î·Í])é‘T ×,ù&ÞÄOç|tq ½Ûꜽ+÷p/Œòk]#T8¶?gïæçq–ÖbÙH »®S爡˽ûYÅùÙ]&ýíKžÜÛ|¸­Îéµo®ÜïÂ]1»éy N_]§;HúbcÊ—DM (nbË­Wo¬s:¯Ä£еgÝòO ˜]¾Ž²~–ìyrL™i\¾ðœ¤ÅÇMÉýç‘6Ö92öóð»â±&VüuñÜŽ¸º=×ÅÓ=²‡>Lq¿—åjÃ¥X3Ô¸Q¥G#+ãü*r·Õ9r·Åš)ý]¼íH¿à™á@,¶±Îéz|åþ–ëŸâ™­îK~6{<¾Ú¸­øÚVçtøZ‰’ŽŽñkŸúTJ–›)õp Y&¤qYj ’Ôu –µÛÒ÷àB“«|º O’ëTÖÝ>C[ãm;>$.¹o›ê'øø›~Ò¾÷ÝØ3´a›c°ùfÃøñâ†wѨá}dÒ!$Òé…¥WG"ym¯éý7ARyØÔÐÑêºPsxÍëÇRj|ó«L/ƒK/º½¶ÙËüÃEE©ÿQÜÞÒ(qÐy£ŠŸ/é-Ĉ‰„Þ(NÂñ§,m‘Ê‹ì|ëEìK+äø3–ÆÍ?µÍG“êÛ\-½=…d.d†÷‡ÜvÌQÓ©¹8½j§7aÆ-WL±.Û amÚ²µÁZqU¬Œl„v.”ÍîªXS¶G¥7c•®sUXLo»N5ßËjß ç=áã»5½ô™•¦÷+æ¦rïjYd¦1*©Í%Ë€P'Fôxa„_ÉÜ4a0‹¾µô–ª›0d Q)s9FôxaË­Ù2R¾” -µjÌ=_M©P7¡t<#úh”úôêæfŽH“NÅ.@kÜè™ðñb©ÌÄ*~¾XÏ”¶•+¤ÍÐk…¦i·ˆƒ¬x©Ò”<Þ~ajfYzÇOúÿF¶Hó ?H¥÷tÕ­º“47‹xÚI­ÜÖ!®ùZ¶é.å}ÁlÛ_È¢˜ò‰tKéÒciÓeÜÇl^"©BS±*¨+âDa¹ -W¶B™ÀîªØ(?OÄÕN[¦*ÀžwæTw[Ú­bW¸ó¹û2â*½t­Ø[Úm0Ÿ…ƒc£^œ òNªXËâíêÎa5 ¤PÅ•FŸqž­…†§|?ë$ã<îãå§Ëàæ´±:â‡3*½C´ºÔõ¿(óŸ]èúçüÒª›ײ°….Ü:·ž,½q:l§7Pga™6PSCûU¬B-k3Š׺ákªÕe%ö\RUEÛ -míÃF¬ tµ2MÏØ;ÂÇ®K óÒ±©û¶iòÁa7&½î,[…öp°ªÎT—|mšÄV*Z]:´¤›ŠØhÅf³ÃX-›˜™Ùî¤t}²ŸÚ¡0i_ÍFF¤MSuTÝlÛt– GÑAlÚ¶aRíNãÄÆèÃÓëÅã>šìò HÒBíEúÜJË%i!<^j†M7Ý,>Ç\ÓFž¼µ‡J[¦™{¦9ujÔЂ&-Fi¨ÕƶÝ)eÙd§ÙQº€LfYiÑg›ÜX-wËpÄí%_bŠ*ý©íiÛŽJéDÛŽŠ›•MífW¢%S ÛÂÙ×£“µd‰%#smí"¥ZÒdÙ|ß4Ýwö¶×æ0¨âjd*k5;+yˆÿŽÌ[n‚+•ÿ†•êká¨Iy£Í5¦p'8¡ôòeO¡Y*Ù]çþÚÉ|…$¯´{Âp2Fz¼-˜ð E\W4M=oÈä«EÉÒ¾`–†*ÍÉÙÊf\ΈʦO©¢Ã«’eéUá^*v©(]WÖ-Ÿ ÿóV]ó -endstream -endobj -727 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 726 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -728 0 obj -<< -/Filter /FlateDecode -/Length 4935 ->> -stream -xœí]M$¹q½×¯¨³€Mñû0ö @†½† ØÁ‡éÚm F¯•þû~ÁL’‘ÌÈÊìšžÝZAÓè©Ê~Dƒ‘Õ,}UøùFã¿äô”RŽÁ]o?^~º¨)z…¥Äꢨÿˆßè¯Z¹8yCò×I`ã³71\ß.øo¸äÅß.ÿù›Ëÿ¬ëè«™rFûë_ •³SL1*/¨2TJ[Aåý£ñAP=ás–T0ÃGgÜVa†ËÉK*˜áŒI‚*Á 뢶[U†&j'©`†Î> -*­`‡6J‰:¢œ³FÐi?8=H:£§²É¢.B§£‘tÖN!:íµ¤ËSÁ'Iç`‹ÏJK:[¼vNI:ØâlŠ’.À¬u°Å¤h%]„-ÞÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓVg”Ÿ|²VI:­'lZQ‡…A -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) ³ÕÐì¶ÊjIça‹1щ:Ø‚Á—ÀØ¢b(3h"^’8ÀÄ8¹lœD&ÙÉ%—$0)O.F+Ñ€É~r!G‰¬ÒÐ-U°Å» 1Õ°ÅaæŠ:Øb³“¸ÀØ‚X [Œ³X°­Z$.° [§²–¸À‚oÒAâ Â…™Jâ Ƶ)8‰ ,(PÊXp®wJ\`Aº˜îQâ Öµ>‰ ,h׺$.°à]ë´’¸Àw­EMQ[`‰ ÀŽ“ÕÉJ\àÀ»V«(qïb‰ 0ªp$.V&“’’¸ÀwMRNâÞ5K€¨‹æ •¸ÀwÇ-H+7x×xe$.pà]ãL¸@™À?JŒÀ» /‡°+qÖ{茕¸Ma‰Ä¼«3–?aNƒ­&r¸ÀƒwuÂð‰º<é ó^žtYâÞÅBe%.ðà]Ðg’¸?aE5xð®¶X©%xW›¬$.ðà]Ä ^âÀaÒ$.Àú6i芺<©œ’ļ Z2`žT“ˆº8HZâñ¤‡J\À»X޳ļ«¼µð®r{Dl±ÉH\€¸:L[aN#X”SH:ð®ÒaIÞU*f‰ x—–M‰ B$IÀÀ Q°‘¸ $7åƒÄ!+èò‰º0áÚK\”%ê`‹‹NâÖS¶9I\ l±ÆH\ lAÈ q"‚)kDßâö¶.âþÀÁDŠ€É¦”“¸ ú€ý¼*é‚™R]‹ºŽÄè&,ÅZâ쥦„I/q~¢pIâ˜vÄ [0U$.ÀR4%ƒÈ\ÒiØ‚ [âDºÐi-qA2°Ñ ÄÉ$ì2ƒ’¸+4ö’pµ¤sŠö™Iâ‚äÂD¡¢Ä˜&Øi‚)ElMH\€ñ¡Í¶—¸›l6±Íu°Åz'qAJ†¶›Iâ‚”`‹QFâ`“ö›Q€` ">‰ ² -…ɀơSJ€ö›ÆI\€yIûÍ$qA¶ -ûMlñ„y›m ýf”¸ ;Ø⢈:Ø‚ ) óœ„ý&ØIÒØb²“¸ ‡@ûÍ$q ì7Ñ‹)DûÍ(¦ À»>ƒ™$x‘i¸Øo"²•öö -Äë#â3Y™hǙČõz¬­bÊ@{±ZE1g @¾´G“0;ËĬ–U(µÓ -ü‹xÌm8aÎ>ñU"ÖÌé&3¦…Ctßö½˜Ý)6çÇþúçËo?cñ½þù/¯¿)™²ú£ ”Òñ¦Énè[ãö-š†ßUùW,„Ô!#bîFjÐâ’t##±üÑ¿ëöÍÊrN¦£6öEr)Q6÷&*­;lŠ±?0´fv“ß]çÐäˆ!_Û-ÛÚ]Åûí¾’l}w·•h[hœ¼_ˆdäQá|Úl.ØËt?䇕Ma¦Rëi‚v€ë;«| Á -;c„Zel6´4o°©PnUøwŸ.`©ÅŠ9Þ.™ù–4V㯟~¼üöŸ¿ûîÿéß1˜Ÿ^/ú¥Üß~£ñêÓüêüòZ¯_××>ί  åÂPæ6”Õ;å«Þ¯Ûlõ3{_븡ìÒ†A»”Jí«õŠîà›Zh§^³öê{?Üÿ™¾´úö¿®Ÿ¾»üþÓdeÛ1ÏYÊdc;D‹Ö]\ž«ò%¸üpÌù.ë0Û5$ütòV¦õÐFÕ«ù[ÔÇÉŸþ? úÛß|é±M#í"ªSt¿'ë<€±uþêÞAË{]Ë´~Ìòú²´õA.rNØÃŒy.„ï ×6*¥|¯sØ–‡#ôž«ótèu÷ù÷pþÂ6„¾gÖÜ»‡1$|Òta¸'ÑÞ4ú¸ô\§CãAp: «½&y”ˆkêû ¿Ü(³þjÛŸ.»õò$ûõpìrYHŠ½sVÉPÎæ>rÏÕy:äÞc¿,Æú×å6(Ôï_oëE«#¬/›2œkx,JHqÙí¿NÖy6äšsñkõX}=:"|…¢›€†Ð¹ÊÜÖ9È­Au,¤^ñ>-Q£õ_ØŒk¤ñºÁâ2úà ëÒж{aˆý¼‰vÚãÎÓÑÁÉ:O‡ÔƒXõ³;V/ZƉïð:G©}™ë[¿å´ÖßЕ7'ìZqâ°FŒuÜleùº°sotO_w‡¸ƒØâÙ“už±ñìAôùâ¼/džv·xµì ù÷d§Cóý÷KwSe4j¼)ðhq3´Ëù°"Îõ¦ua5c~íð[åàvM;¯<¿®ø1®ÑÄ#¯?“eôÙò¬'ë<úìAœú·ÈsïÉTÝAçf§tc}îú×}ö2£Ö±Ù0΢¾ãÛA/‹OÏ>¯=YçéÐ{?¦ÝŒÖ°kièá+ã°Ó©¹Ï†Š‘£^:Ϭ8¬¢Íì´ù^ÄÙn>à>—à¾g3•¡[œ‡}í ôOœ¬ót(=ø\Áâj†§zú…!dÌ¥Hªh-Èts\ùK>eø5?9p,FUn2Ú%çz²Î³!ÕÝkÿþä`'`í5[ýÚGâýÄu›{Ï›Üp»wÐú@öd§CëýصÜë%eΞn®òªuÜɲ·‘¢µæ•!šÅ  ¹7µÍÕ¸îu]ƒŽ5%÷»“[¨3‹Ú²á}3Î<0ÛŠÍ‘Ý'„†çgÍþ°ƒàr²'ë<‚ïǯ#¥U6)0$UÏ<òÐ(EÆŸãÐ1Ã5Æ€Œ¯8 -7ˆÒŒ ™­´lvH{}-3À|·Zæ×Æãü¹Ü°ž¨×$§¸ø\§Còý× -ûÙÍ3¡§ÙáÛÖ¶.µìÉë湜c}è¡ÿ!FmŸ~|~o6–>í¶îŠiDLû²¼'ë<¦ýA–wXiM/~ ë<ΨŒz0Ö-ñs›ÄìSŽýßç½gÉ›¹3ÚÂíÎÛ•ÀQñ½ÕhŒg$L·2߯㡂ÕÁ_âŠÂïSÆï¹Ý“už¿¹]¿+?<Íb×öI—¼ƒ›Žs"¶},Ó¤vs)Ža·Í1¿²´Sò¯w2`›§¼F°Ý/y`Aß–çy·ýœ­÷ïÏ1œ¬ót¨<ÈÙrϽ,O?_¯ÆåÕ-Å(é4—!¹1‹¾þ~/¿§2§r¯c6I³þe@CZ£dó,lÈ ¯vkcìk·ºÖïf†ì êüêÉ:O‡¨sŸwÝì:v>‡¹=þ|g\cÔúuá“{mÈR²-{Oì°8zZŸyÉ£¦S¥Ï TÞûcŸ×zäÉjè‘ Ë~ò…´Py²Î³¡2|ÙgÎðOzçQ[òAý=ä¶ÝÝÊVöíD‘¥×s¨…ÃÕÜç¦/!â¼7#ŸMí Çpæ\Üw²ÎÓ!ï îb$7îE†ù^¼kØJU¹¯~–Õœ6þEÌY⟠lùlÜwǵ›lÑçK_ýžVðûO[ŠÛ‰4úy*Ÿ”­£ß.Ùq!;:US›TFÿ÷ø·ù?ðøªöÎâ¦6÷vVº²S[ø€v·/•7æë|Xm¼eôKmÍQ=­~çsÌDžâ!·Øñ™¯ÀK›º·SÊÞÖuý°ãn6‡n/¿R®øÿŒÃé(¬îð?Õûñ %‹?}V`ç¾½êþ,¾"{n3 -tXº«~=ócé˜×¥_»Œ“§ŒAZ|Hqô÷Ûv‚¥3èœÍm;ÅÕç¡øéž)íÙ¡ú½Ð˜´û£ñ13o ¶Ñy«”²ÐöŒ×ÞæÝ×uL]X|¹ÜõÛ®ÃfXlÌËkí-m×2¯ ëfÝçüLt‡ê_×·>Î>ôhIöoR£V4³¹Ž{¬mn} —O(•ºj¶_Ùõ½~l}f³¯óªû³ù½Úû8Ôqªí¶ú '¾~~&Évðµ}ÇOfã§]Þ©soä›ûžù* ÷oØû°æ‘›Þ0ì¸ÞÕÑãžñ»û§ÆxÛwýÜà/³ôÁçVŸ›CA²š‚-‡äÐ9«&iãÕÑ© çêóåi=æâ¬w>à¼ó‡MT Ù?×%t·cÝæ¼di…ñÁ’ô¡B™TÂó§¸Ç$‚FTÑÉ´ÞñT•'Á!;y‰N¦Ý⻺ºóË1‘Ñ£oO_‘A=fvêdsâa‘§UÚÛöwÕÃY…kfD“ôý06a[E:RUâEB¬ùaæDz! -Ë‘×6×ÛäWžÔˆà…÷~Þ¬T†/3c NË&gIà™¶!dœX€¬{Ð[þÔx þù¡µ_ƒkfÏ*¸ÎËàrߎÃøÓ%”£u9^…4iCßsETKOàc¾ÒqÄIY§ë°éâh}Ý©Y¾uç§VŠþ‡HÑÙüt.sV~±”ÿ© øM‰5ãyþÒ UÄXRý(~»Ð—¦8Aì\O1ˆI¸¼U­-§[Æ¢Ên`…8å±$¸_-o{Zƒ„ù­ñõn¨¾¯Õè0[§j![Žs;„¨ôVLw¯&½ˆé‹gÒd—«ÐL¡@² {ƒŽ—í öŠ‡bmú­P±îºØ¸j¦ƒÊ›ëB6Ó¹ë4þZÖD6œ÷„· ºµ£ôM”Ò×]Ô¦ªwj2ËŒÊMêkÉ6 ®“ º]á7ª6í0˜MÏF Ÿ…©ɵr¹–D·‹XniÍ·‘Š­TêUsõ|7¥CݦæxAvc©¿HߤÅ戲õíZL™«6Ö*1ܘðvñ®ÍÄ.~»ø(”ö+”¯ÐãBËÚmâ¤:^º4¦ê*LÍ*£#—éÿ™*|“ó”7Ú5HѱéýªwBs³‰×NâòÐXÇIÐ.ö²Ì]:ƆYî/”PžH·•n[I™Ë˜<æj‘T#µX7Ô51QXõOãJ.TÕtמAùm%îvú6U½jÍàî[»]w¾qql#®é üfok—a¾ -oeas£˜ Ÿj']lT[ízgLØ )tq§Ñ·"®³µ±S9ÓjyÛ'™´âÞ.¹”e›fCY¢`5}¥ ºôõÿPæ;üþ÷Å]ÿµž!>¯ãF5¶0[Wb¾’Ñ€¥“búB°*lÓ† c´ßÅ:õ²¾¢¡­ãÆ0¾v½ºêÄ^KêÆ ºi¹Ðw2±iÐ5Ú2Ïø;ÂÛàÅÈ|éØô{¾´õæ(ëE§ÏW«pÁo—©ël_’¯¬I\QѾ¤Cë «ˆ Þ(.Y‡¹[¶2“É;i®'û -K}± FÐ¥í:×—YÞt••»`E[~ÁvšwšWöF?<½¾xÜ“C$mÔÞ¤o\ÚnŒ¤ðd©-—a}ÙÖ[Huòvµ¶,›{¶9}jôЭ.8F™4õjKÏ;uU¶ê´.. SUÖZŒÕ¦°T«n)wœD/ÅSté_¸§=•æDÏG%lÊR»uÅkÑ’mçpŽýîT/ÙbÉ$ȯݤ®—´U¶íÛ­û®«í•Ý†ë†„™ª^ÍoJ>Ä̹΅JÕŸ’Ñ©ëj§£5¶|‘ªJ!€C}Vt‰¥JÞ]ç~îd›!©¹Þ†crÖª¨é+qeÞ ÈÇ -ÖTýº\¹ZVâ-½ÏBj¨ÓœÚd†Êw"–¬[IµËH[Eú -TKAo¢on‹JËŸ÷ O^¥OÊÏÿuÊfú -endstream -endobj -729 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 728 0 R -/Resources 4 0 R -/Annots [ 730 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -730 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 176.373057 101.433071 157.623057 ] -/BS << -/W 0 ->> -/Dest (cb160-1) ->> -endobj -731 0 obj -<< -/Filter /FlateDecode -/Length 4694 ->> -stream -xœí]Kä8r¾ç¯ÈóÃå#ø>XÃà ÜX ªª»vaÔ˜õÁß%ñ!*ôʪîUϺ Ù)Eð"?~Œ ¤¤ºKüý ð_ %BˆÞÑýåçÛ/7)¼•ø7¤˜ êŸðñö®$yaUtÁÞNã…V{w»á¿î´Mþvû÷ßÜþ{žGݵˆÑieïù’Td„ÞK˨"TRFe­ð^[Ǩœ¸Ä9Ì°ž4-UfP –SÁ Ò:0ª3 ye–ª3´WÄ©`†ŠÖ3*%a‡ÒR²:"‰ŒftÊ -‡FwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¶XE$9l!<§s°Å8#YlÑÁNça‹FëDF` ÐXl‘N*N°1±º >,uZZaƒ1’Ó)%,°iXF:ÉyF§a‹“6²:ØbÔœÎÀ§OXãC`F«N£ÛH£8…-Z{bu°Ï1€v°EzÇQFH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHØbÉqL`l!Œ\V[L$Ž Œ†-HÀq1°E“á¸À€m há¸À€nIFÅqß’TŽãÂ…™’ãÆ5ÁÇ” (EŽ 8×€;9.0 ] wÏqëë4Ç´k(8Ž x×’x×ädu°æ¸ì(Œ -†ãï%=ÇÞÅ0Ò Wpè8.V„Ar\@à]$q\@à]í1°:/0 ÇÞÕ—ÀÍÜà]m¥æ¸€À»š´ã¸@àÉzà]PˆåÝØ‚å¸ó=tÚp\€Â„†[Âq愈éÓ`+¡BtXð® -è>V…òhqfÜ¢•…r.r\`Á»˜¨ Ǽ ú €ãfTÍqï*ƒ™šÓw•Ž’ã Þ…ß`9.„‚Àqæ7¡$ Ëê¢1Ž ,x´¤9.À,,d“°:/$ÅqÈXHø¡8ð.¦÷ÈqïJk Ǽ+ n«ƒ-&hŽ à×@‡aËŒi8«Bj0§ïJ…ætà])}ä¸ÀwÓ´ÉqóI Ç0PÀ Ö¸@"zï8.pQB—Õ9sËq—XP«ƒ-ä‰ã8Ö"š8.ð¶­9.ð¶Àeจ -Þ7ÀÀ…¶ÀSä¸L&B$â¸À[‡x ­ÊéœÁƒ®Y]€NyŽ ¼'©Xq\€XJ zŽ @ð"¹KÀ èq\à#lÁPá¸S‘ž9§S°N6Çðt¡SŠã‚ a ¼AŽ ‚ˆ2ä¸34bI45§#™âÌÀqA '’«Èq† "M0%«ƒ-  Ž Ð?)ض øA°‰0‡ÕÁc‰ã‚t -7Ç!À-5ÇÀfŠ7=ÇQÂx|DéDr“9.C'%ÇQÄ›š8.À¸Lñfภ‰x!3n£q)ÞôD‚-. -«ƒ-pš"3nÁIˆ7ÁNœÎÁ‰ã‚è\Š7Çp3o£g—BŠ7=»\ÞµÌÄéÀ»ðLÇ8A¼ Ï–‹í%ˆ×zøg¼2¤ˆ3°+Ôk1·²KÜ‹Ùʳkä›b$vÑf"² Ž]5À´ -¥’ì²ÿ£'Œ«Oí•<æÌàÓEF ‚w_bŽšÌ¬$×Çþò§ÛoŸ0ùÞÿô?·×ß +eùOÁARðŒa #ºÛÂûV¸|ãV+! ÂÀGò$#|¬F*Ðâ´è–ŒÄô—þÝ—3ËaøOÌýý§†î¸lèRÙˆš\Z)~5"zèÓÏ·ßþþÿúÏüšüÓëí?~—Bt))Hð>éçV2ô $Ç„Ï똆ÂßýçýÓm}èHt¡ÅÔ0¯/U¡>Éÿih5SZÍÃù[ ~9üÅaŽ "U…Ù iè“YÞÓÎSóâààú?ëÄ¥ ð‚%-ÚœråcS”Sel¶:ubwÒvè‚úñÇøÇ«]€æüAý.uÁøMnúžÎS`ZÅÅâ“ŽK?áó²¢ £>ëfi>Oß“^ëIŸ?Ÿ§ošŽ©)K§îŸll´vüN:zdº»š¾_›´S>z™ò¨I~¤-dúð˜J]bR/RÔ{ :˜ç"¨ªfƒù6P•[0÷Тçr ‡y §Öí{nÅÚ[¹'sÞ¢­Þt_æv„ÎNÅ”ÓÙ> ÑÍe3tíÔÅ^£šò˜®NÙ#¯ÉÛÛîÏ´× -‚©pÁ½ÀôáÛCð±<—C°{ˆ/‰¦Þó:ÚSø±Yž9qB¢V'ÑùÜ”IKË‹èz¿‡Îcy.‡Î_‘øŽ¨â:?&òØjÓ4VΦ“Ž,ÖÌs94îø}WCÜ…"Cî=•—¯ˆÚf½0Ý - a¸­µÚcy®†Z³³Æø·†Ú5¤}->~ÛÒúá–óVù›G²©~gz€‡Ž÷^Ü}0Ïå¼í«–•ŠAÏ#O¬ùo¹ÅgÑJÎÛ­~Ìü·>¦n"a$41>·^Ñèrº’O5Èxfv`e\iù‘Ø -úÈŸŽ”æ¹úv|Qš·úÐû+c¹´z7æG"¬ÄÆ®"ðJÑÕUV‰LãwbVŒ2èt£}­Çò\­Û¾ê¢{žkü«ÙÌÿÄpå!ÿìŒyéQÐÞ…•ˆ¬>%‘&=‡öy,ÏÕI;~è‡ûKì4vÄwÈ›ñxÑp¶ž¶ü~•þ=kš¤ÏÇéó\i5ÍZ:¯ÎXêo—û¨Y“L«"šÖv‘Çò\‘;¾ãI´ü*}Äo¸úCÍ}g«…´äÂÞ -úÁ<—CÞÿ¯Y~ôêO±©+ŸÏ׸ïõǃy®†@»s¿Úu¨èãÃeÃb®MÆÕÞÖœë¤{ϳ.=ÊšÒjš#Ä`¦Ò/SyÏMYTËlë(ÏJ¶iŸÆz•rÝdC^-JyÒSà2ùú5ìçQfðùæ¹ʶ}¾C½ÛôÒ©ÞÍ3ÚôPtáLÝ}¯=µûTQÅê¦oÙ>sö©Ýîx|jw5¬éÌs9Ôlûe¥ç̼g+µÝl2{R C@ïƒí•sènŸ¤éfÛ| ©ç¿V¹eÖÏñk³ÄΠ}Ù+±|~Æü¬ŸgXï;˜çrHþÎü¼¯øŒõ>Z®Ö+èmßi9:{Ës5ôºqÃÛ?…ÆóÿþвÓál­AJãÕéó1õÁ<—CëÎ}è³kï#–•~%(õÄÓ”Ç52š÷fAÒóÒß̺ÕÕì~姟ղ,ýÞ8üõB×ø˜’„Vˆöv,Ïå¶ã—öüìn½0ôÞuгdg×¾ŽÅµri^î©hˆ¦ÏYm"˜'y ZAÓï«Ìs94mû†‹ø0.[òO×qÒÚ½ÞÙ;0Œ»¯\êì‘×F+ý¦™3e‡ÞçÆž\ÎçjGAðÆ;8gï¼¹xþ>ÇÁç{zõ+ßšÓOªuQ~r·”Ù®FmðÝìy¡W²ƒ>æ`žË¡ñà;*7}µgYŸœ:ñLá¹8wa,ø껉êVûÀ=ðƒy.‡Ø_ô½1ï™Øö[ðä£Ï™_ö¯VÜø–Ê‹aIî!øXžË!øà».Þ«þ(Îþ–è~ä¾úAbu–à™Æƒy®†¸¸s¿º› b6x£]Ê=|úÝ‘o°,{^Ú6ãÈ¢@γ)e4mgŸß1®üæ¨>?è`žË¡wçYÉ°ß»›qlŽ ú·úû.Ýú#·~¸@w¿üèºã÷2kÇæý–£«Hó\…'ߧ~U'×}¾á{½Ÿµßg>ÿûhó\¥;kïð·»“׌ 7ÿ^ü†cÖÞŸë~!°¢÷Qíw'ÏÜ7Z<ŸÔÒ\ÎË¿>à£Ìs5ä*¹ã¤Æñvâ‚`ºŸn9j¦ù»GŠ£Ö½Ð³p‚}í¦3]zú…í—¿ ÔzÛ>7ùön—J¾=Z²7j;Íð2PzIÜ6¯¥¤OêCK?MÛx‹ž¶i+Æ´¡@lv7(˜ÝM²Ñå^Síi µ±§ŠLûšp×@bôi뎵-ô—ÑÞÏrÓûKn<ÏTªÞåÖy‚áäÔ}C÷ÊaKúR»}Ø’ÀéÓ–‰E’\§´nü eMÇC9!3ÕTß“·2H>ƒõ³òíTwcMÛ#”k°ùÝøËÍ [Шa2é5è´OéÝ‘H¯…ùxOÛÞiHånSCC«ûJÎaw×_JªiÃW™ö€Kûÿ ÝÚžLé:ü益$Rû'ö7g”ƒxøIÜNüvK›s#&ú 8 §CYÊ"•ÙË-ý}ìS+ätKáiK…ñPÛ|5)¿ÍÙÒ¦)$s"3l2VÌ‘Ó©¥8]½j§ Nƒ0Ó™+¦X—m†°HmÚZ`͸+VF6B»ʦº*Ö”íQiC¬Òt® -‹émÓ©¦ûsZí›îܾÜP­é¥o¬4m«˜‹Ê­«e‘™Æ¨X¤6§,B¥ŸÑËþ sÑ„Î,ú¦×ÒæT£0d QIs:FôrcÓM¥ÙÒS¾¤ -5kÌ-_M©P7¡4<#Úh’ú´cs3F¤É‡s±K=ûZ†7z!|¹Y*#±ŠßnÖ3©må -i3ôZ¡iÊ-â +^ª4­¯ŒGšY–¶öIÿTa‹41Ïp ¨@*mÏUÏj%ilñ¼‘Z¹+¬C\ ¤ÔBŠ²½/˜mÛKɤO¤[R—›I›&kä>fóIò˜‹UA]' -ËíS¸²Êlª«b[ ü6W;mª{®Ì©[Å®pç[+ö¥ÇUÚk­Ø[Êm0Ÿ…/ÃÄF½8A?äJªXË2ÛÕÊa5 ¤PÅ•Fßq­…ÒþÑù°2nÆ}¹ýù6LsÚXqàŒJ[‡¶^—ºÿ/ÒüˆÏÝèþ/y¯ªqײ°….Ü:·3YÚh:§§³° ›F¨©¡ý*V¡¦µ eO¿—î;YÊ.+±ç”ª0¨*ÚVhk6b] «•iZÆn_ºV,ŽÌ{û¦NØã©É‡jLÚå,[…“örpªÎÔ)ùÞ‰³”´NéÐ’n2â¤-§M…±Z633²­¤4}²ŸÚ®0©®æ¤q#Ò©©:ªÓl[t– WÑ$ lÚ“¶`Rm¥qfcôÃÃëÝý>™ìòHÒBíEúÖJË…%i!<^j†S7?-sŽ~à?ÞÚB¥,ÓŒ=Sœœ:4ªkA³“£4ÔlSÛVJY6«4O…. “YVJôÙ&7eËÍ2\q`[ÉŸ¢Jÿܶ´m{¥4¢m{Å-Ò{&d™®²ìõ´pöõêdMY|ÉÀÈ\›»H©¦4Y¶¬›æuçÙöÞ\UC\õLeÍf)â¿GÆ»!—*ÿ +#*ÔÝ1Q“òF›{LîNpBiÏeO¡Y*9g{íd¹B’×NښЂŒ‘^o+&¼A÷MQoÓƘ|¶(ÙK:×ÌÒP¥¹ån“F˜isX *§>…Š6;$ËÒá^ò»O¦BÕólùdøû?ûdZ -endstream -endobj -732 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 731 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -733 0 obj -<< -/Filter /FlateDecode -/Length 4770 ->> -stream -xœí]]äºq}ï_ÑÏ®Ìâø!€ä1r‘…ý`äa§ï® cn€ë<äïçPÉUj©{f®µIv0;­*~”ÈÃÃb‘jé«ÂÏwÿEÒCŒ)xºÞ~ºü|QCp -ÿÆ‹‹Qý~ƒ»jEap:ùè®?e CHÎ}½à¿î’'½üñW—ÿ\æÑW3¤äv׿}É*²Cˆ!('¨TJ[AåÜ‚q^Py=àS’T0Ã2´V˜A):I3Ș(¨"Ì°´]«Ì0A“¤‚:¹ ¨´‚Ú(%ê`ˆ"²FÐi7x4º—tF>&“D]€N#é¬| í´¤Kƒ÷ÞEIG°Å%¥%ƒ-N)I[ÈÆ éæK,†É¸ë§Ÿ.¿þçï¿ÿÝ?ý;`ôéëåOÿ -SÓ/Ño¾Ó¸vqúKaþ{›åóõ˜öë,3sšÏ“Ü©ùº”õÒ>ç44—mr˜Å´Í\³ÁÙ9ŸëÊOÍŽRwµKOeç2Çt´Lçü|=×½¨Ï/ï­ÔWê—l×ê7ÿqýôýå·Ÿ– =Ç ÖÙ‘Âl•9à.®Žey ®>NX†ÞõMWšÿ˲Æn Uº|v~Ùu»é“ÑqHšírGÝ—nˆ¸îž»·¾œrÍÊ«ðóK›ŽÔ%ÝŸÖsÛÕ©zèËÎÑ÷H8¶FS$ÚÁðÁ2å92é>¿vÓtmÁaúê¦nwû¿ÆsëQZÊrݽðúêhìÚÓ±ùJçÕ_˜ǹi¹ÍyÔ9X§­»È=–çtÈ=æhV¤tÚ¢§;.ËèîÑ·ÉOxbóœ®Åïûb®ÃzÏýX͸¶¸7ƒÇú6³˜ˆ_\›¬{™®%]ÎcŠ¯5ëmvÍos¹/,µü¼.MóŒÅÓ~žêÈ«…Z®Ÿm)œóÄi\ª¯ït"¾ óxŽâë`ž³áËìxIÏö1뫇úØO¸Èœûn}|ä>:;úëñ~,+Ÿ­™m»Í˜—»2cû8cÌs:DÝ÷n2*žéYë¾möy›ý¨ba«¸ÃÁû=TËs:TÝ÷<þÞ¼ópï¾ßÌ#jgÝoâã««ƒyN‡”è *~ØÊ¡ zÖ]ñuoÌÞ Ê.¢iýzœyuŽ§ûÂî±Øù¹ÉDÛ…û{ïµµýnsÙ»Ñ8­–ùRÚ IE“·†î¢õ`ž³¡Õ>¥úˆ5ü‡!A·ÏµµÌ÷hÜ·ßF¨#˜!.oM”XÈ*Ýg–®ß"pÍÞ=^µÍG#gåÈgTÝGê±<§CêNÔê=âŒ=Š§}t\òÛ‹šm ²ùw.™‡+á÷fúƒyN‡ÈŸpcÖ}$ªÿHòý^Û7÷Ý@ðѽƒyN‡à_õ=¼–‡"ì›öy†­¾Þ; ø˜Ïx„Ëè¥æ‡æ£åDXªí­Éæ9ziÇw=²ïóNÿÅú†}©Å=ô^‰Ð&Uç»{íÝ@ãqǃyN‡Æÿ´÷öû“®—Æ5uAl‡„¿_/³úJÙŸg.ëK};»\÷x à`žÓ!÷¾ûÖ½õ_¡ÝQ;é\Ø{ðë[üí»ñƒ¾ú4{œûD¬õ`žÓ!÷˜ÿZ#IAnÑá Ý4°¸gŽ®sŠÔ#ò9FûuŠâ»—7äç§rúHË[#^ÒÒ· óRW6?/±jçƒç(ÜûÜóœ ©nÇWý,øŽ¥-ãÄZ}qæâeÊŸ÷‰zN«õ­NÊM{W{v-8±›#ú<÷b¨5-Ÿ6î-ßÓÇ®7û„?{0Ïé»ãÏîÄ@Ïçç½Ý“9íjq×kÙ@ó>îÁ<§CóÁ“ƒOúª÷NOÙµ]ÌÂÔzxë‘–Åo‘ç=ûÏÓÙƒüwÁGND~ØHÞ@ßqÖƒyN‡¾?õ#Ï=©ºƒÎÕJéÆêÜ<㿱ºû ${î“\qÌs6$û·Å\¿I$¿G¬€qzôTÎÊïÛçïÛl>cpì±Ä ô2ÿôèŽíÁ<§CïÎÙо·ºUËCªTôÅO…p+h3e>Šø=ÛW¿85—éƳ«}D¨øu_—ùWè˜Q3Æ~7b ed©òÌÁ#Î<1ÚF›»Oî uûgÕþ§PÂ1ÙƒyN‡àûþë#_2’Ÿø¨H*-ßñÈS½ÔúTv”íéÏËr¾â(\!J3.d¶f§eµBÚªkæ¸Õ²v­<Î÷åºùD}Ý@²{‚‹å9’ïû¸VXÏ®ö„NÙ³ÝÛ–²:.µlçuµ/G¬ÝÕßù¨¢ ýîwÿ¬ÄÑ8Öi×yL#cú‰(ïÁ<§ÃôÁ'¿ØÉ‚ò¬_Å:÷3 -£îôuõGÜTffv‰)WÏæßéç­½ä­gV¤]©<†û™Àu^ñ½Ù¨÷g$L×4?.ý¡«]{‰3 -¿OùK·žˆíÌs6üÆØ®€ßE;ßy«žtIXá¾iÿdÌßöÙ' -·b)Ä°[Ç@_aO\Þ‹€­vy`»›ã.ù»^nó~Þm;fíã1†ƒyN‡Ê˜-o¹—y÷óenÕ0ÿ-O+çïª*ÏÐCn̬/¿?ÊŸsšC±×>š¤Ù ÿÒ¡!.Q²Ú ëâ‹ÕZïûÚµ®jqïIÂøD|õ`žÓ!êØy×Õªcãæ¢÷øþN?Ǩåß¾‡—Ï{veõÈR²-•{¬°8zjiŽ£Æ¥\\ñ˜@á;>v^ë™ÕØüžvfðû»­Å?Ìߤ:Üå·ˆåïÂN싹+tw“ì uý5ÞåË°¯ù§‡¢Ÿ1è¯(2¿BÏÆ+ G)äoÏÐûíïÿí_þð{‚sg¾LK‡Ñaóóá÷ù«zót5sêÆeb!¯B\™Hr÷dbjɧeaîš©ûÇcÏ9­Ÿ~DzæÏc9±,]æú2ôrþ¢“~Q¾›ëföŒe8v®Üê¾øñí z|ƒŽòqÐ&¿bïêiÈ‘ƒ®ù QYÒ¥ÛôØÐúº‘s|1áÏ5Õü®B•__”_]1½È]Ìé;ü§‹N4äöÏ$2½WLâhÊE¿^ò{åHV ÅY8Tµ,Ò%a•ÝÀÅaH}JëÊ÷¾ëT ·®|4®ÜMÎïJ¶ü}ÿ¤J";~ãýT± DN¯×â|÷jг8¿›/v¾òÕç‹Í¶‰§m¶Œ»bmºµP±êšØP±Gçw¹Ô¦óMXMçM§Y÷—´&°î¼'¼]P­í¥¯¢4¿¬UZר*³Ì¨T¥®¤¬BµŸÑí"¿S¥hBgV=ëµü^•I ¨¦K% º]Ätsi®öT¨©bËšJË7SÔm¬ /ÈÆ6š¥!¿l”eËÇ¥Øç(}­"ÃY oGu$6ñëÅ!µk\¡\ZVnGÕðÒ¤ùËå¦OšE–ßJ‘ÿŸ¨ÂUifžñƒ¦ -©üf™vÕ*Éc³Š—Äå¾²I- )´´¬¹t³¼½°"ÒgÒ­©k‹-¤¬É˜<¤b^&©JK±®¨«âLa¥}*Wr¡*F º&vʯ q³ÓÕ¡ -°—ʼfpwµÜ&ö•;_¹8Ô×ù5AÕÞZ.Ã|ÞƉzq†~,•4±Qu¶k•1a3 ¤ÐÄF_Gq­•ÆïÛ?¶A&͸·Ë_.ã4g¬3 ¼Õù­wÜëÒ×ÿFšïñû× ]ÿµ¼fešÇªla*·.Ä|&ËïHÅù©EX‡ b´ßÄ:¶´® ¡ÎãÆ0¾¦–]5b/)ueP]µ\èZ2±©Ð5Ú²–qw„·®«#óÖ¾iötiËÍ¡›_ÐS¬Â¿\Ʀ³mJ¾²"q•“¶)Z2,#.x¡¸d¦fÙÂÌ H^Imúl?ñ®°¹.vÁ܈|i›ŽÚ4Ë‹.²ñ.X¸À–_ð‚IóJÓÂÁ觇כû}6Ù—¥•Ú«ô•Këei%> -endobj -735 0 obj -<< -/Filter /FlateDecode -/Length 4743 ->> -stream -xœí]K$¹q¾×¯¨³€¥ø¾cdØkX°à}|è®–`ôXùà¿ï/˜ÉGfFefõ̬J’§ÑÓ•|D?~Œ`U²ÌUãç;ƒÿ•RŽ®·Ÿ.?_´Š^ã_)±¸(êßá7ú«Ñ•79$ý‰.ª˜½áú~Á«Ë±øûå?~uùïesµ*ç`¿þé3«È©˜bÔ^Pe¨´q‚Ê{£õAP£p‹9K*˜á#YÚª"Ì œ¼¤‚dmT f8ŠÆmUfØhHRÁ “}TFÃcµu0D9+èŒWN’ÎR¶YÔEèL´’Î9"o$]V!Ÿ$ÁŸµ‘t¶xC¤%l!—¢¤ °Å§El±):Ia‹…w² K°èJ¢¶è ¤ËNùœˆD]†N§˜¶:«½òÉ9-éŒQØt¢3 -ƒ¢ ³°%hŸElñN[Iç` áGÒ'¼‹) ³ÕòìvÚIça‹µ‘DlÁàK `lÑ1H€¤˜—$°1*Ê–$°É)J”$°)+ŠÑI4`³Wr”xÀi58 [<‰ œ-„™+ê`‹Ë$q³°$.p¶Xr8°--8Ð-él$.pà[Ò&H\à@¸0SK\àÀ¸.’¸Àr¥,qç:p§Ä¤‹é%.p`]烕¸Àv¥ qï:2Zâï:‡š¢¶À€•3ÉI\@à]gt”¸€À»˜FV〉ƒÄÀŠ²)i‰ ¼k“&‰ ¼k#–Qæ “¸€À»Ö㤕¼k½¶x×’ ( -ü£Åh¼ -ñr8[@°`½‡Î:‰ И²K$.ðà]“±ü sl¥LÊAâÞ5 Ã'ê²2æ-¼¬LYâÞÅBå$.ðà]Ðg’¸¯°¢Z‰ [TpˆX3Sä›Ì˜„è¾å½˜»SlÚûÓ.¿~Áâ{ýÃÿ\Þ~UvÊêøN¬9åmŒ7øJ•ö&‚mË [?R‡ˆ¹i@‹ó¦‰¥Ëm¾¶fi2,f› ‚™rg´è©‚ÁX¹ Úúp¯n«ã¸Õ{DoË~E# -ZçøßuûâÀH0œÐ¯MS¼dcÏVúŠFoý÷Ÿ.Ž­n»ÁßõË©HÞf›®Ÿ~ºüú7¿ý×þ÷ßbì>½]~ÿwà~ÿýw†ÿbþ˜ŽDA8i’y3ÿ óß*÷ZSšÊüš^'9|S®Yί‹~ÕÝ𚺼¶cÃôšÛ.m¢åùoèm·òC£ Þ6Þ&¿Í6Ýú½,l ݪöç±­ïÿóúé‡SÞΣ·?7Z ½qýº½Ö™^Þl©s› sÏaì’q‰”›·qç>Ý<@(úÀë·©=óã¶à8Æ)ûh›vŠ“æÁ,m…];B¼g‡î÷R”ú {ËàmãýÔ  %´Í¶¹Ï“;K{f¾Wšî·³ú_ëîùÒêµ/OÙææ±lõµß°ãÏ5j™·Þ¿Ÿ4öY®?ϸöƒOÌPnÆ7—õÃ\hv ø¯åÖ÷Å>õu^ro»~³¿ ÷µöa³) öV,˜îËÞ†92âJÄO?ëy\ÛòÃ}¥¥ Kü‡‹üŒßÁÞâ#Ýdz¼^ŸÕ»>Í.}fçyÕ±Ú9ð7ŸJ¸ázÇo>”pƒ÷ª‘¨[U'©2ùõ¢}ž=»Û£å<ßDrÄÛYà™ f–Iïu¹˜­ƒ9°ÜnüÆfæ·VOÞ!9,è•°l •˜Ìɬó%iØ7áºD{\§_Æ%©å Ë,¬fZZδi¹ªyÙ Ë¿XæuõKÍ“–}LËéÒÚäN}N›Îã’¬ßÃeÃ\ ,‡¨á2ëø\nƒhëHåìS4«÷0ÉïÆÎøâ7ÒeÞ‡ÛÅäÉ:φI -~“ÇÌ/rhÄa¿ÎÓ~‚¹ÝÑ¥¾ß`hUæÇùoÝ°³¾þþ8ÿ¥ù5 mYaŽÔ=Û1_Óíõ©ú]¾?ã‹2ï *††Ç£HÙ¢ê\'AU7;ë=Tý’ìÆ Ú½)ÙZÚAk¶ó ›vV¶$†¥l®ƒ¾Ä{¬;rnÕ§^#o¨»¶=>â/Á^7Ž£˜”qÄì>‚OÖy6{ó!^|J4ðVAÎ[MË—º6«¼râŒDkDçëÐ&É]ßC+×콃NGç¹:O‡NÚå×qåÚ «z;oW¿2÷S_q#—–eÛõš³k߶Û0F´M¯WvÕò+´®mÚØ5Ük{½êk³Ê<ù¶ê÷ÎÔ¢‰{èëñ’IþÀl2G«ûÉ:O‡¾ý˜±­î•3>¯ ódgCgø+Š¿ «8ƒÎ¯“yìùô{ çùQª3»5'ë<â¾gCÜe§¼ŽTnßµÃ~!x9%{¸ï}²ÎÓ¡ö`ño µ÷ö­øømiËfÏþ4ßAr;ù‘#"ÞGy÷É:O‡äýXµíT¬ô:ñĽø­½3f+µîj÷c¿­sê!ó(3aÈñ¥Ýð†ˆA×Þé¡Ž’†ŒWi'vÆÕ_!“ѵy8S:YçÙÐbQZz½Œþ¹¼~¯oä¿S™ÈJÖrãÐøLÙÕ³ìÅ!î4Ve,?°Ösuž­û±êƃkžâ«ÅÊÿ"påWBþ£+æSÏ‚ñ]X½ƒÈS9åøÉ‘t„Èsuž‘qèW—¾rD0ØÑ¿BÞ‚Çó„†GûÛ_ïÒÉžfŒçé'ë<Òþ<{š ´¬P|wÅ2»Ü—†=I~À¶|Zö‘'ë<"ÓAìø Zþ*cÄ_p÷' ï;{«´§ŽvÐOÖy:äýÿžå×Þýi6­Úóó{Ü÷ýÇ“užïW¯žsÜä‡+”•Í Ü› -\è£ÍŸ¤æç-ë^§ëªãO³rYKK„8¬Tö6·÷:´E½Í±öYɱìËÔoy$®¶fên×áOQðçÊß¾…ýwPö˜ïd§CÙ~Ìwjt‡QzhtëŠV–ªœiWï}j÷¥£JÔÍõøù›G?µ»z=}jWFMþÀžÞÉ:φš¼—µ‘sË\ìÔ®V“Å'VXÇ`GíœúŒÎjǸ•Y­¶õÎ<1ôÑvÛª_ó×a'H\A×mßÉåûcšÅyùû}'ë<’ÿÂâ¼oøëc´<›?î w|¦åäê}²ÎÓ¡÷ F܉öBŽùÿå¡åÀ_!ÃÙÛƒ¬OtnÐÏ©OÖy:´¼}bu]Ljmgc½Ä#1?™ØP3ŽÎj·f:,bÙÿøD¡¸›½ÞùYçÏfÛ–ýÒ<üû…\­‘›&e %¢ˆ­ôl3ú 4]S°U¸÷P@À9ñ W-.°ë>6oÞk—–í>”Ñü§kQç‡$æEŸHˆ ~7G=<ïù(ömÎÃéf D‡E SÁÝžƒu5«c¼–Oéf>8,òÇÈÈ‘î»÷Œ®ý<å±õ4§²f·çÙ-fÞyà2õ¹Þò¹’™/xÈIO‡v|îP(ÏøΔ7ò$/‡hÝ ™_¯ä)ýñô\Ÿíií׃B{‡•Ìï¢M÷°ÆŸ/¡AiÊ1Ä:$e,OÁ5âM–˜¯|ìeÒŽL6Sm®wj–owø¹•š¿ðAóÐ|þgÖñb.ÿsðŸ.&“bÿ—ªÊá캈Ë&+ñû…ç'ALÔO¹:³p~©[[dÚ¡YUv+F•×%Wz~ÙOÖ┧—Ö×»áú¾VãC‘pØzÅs`êX¢f0[1ß½Vfó$åæ«ÐLáÃ6š°7HcÙÞ`¯x(6NB¿ê¡».¶Tí1| ns]èÂfúè:3 -kã0œ{ÂÛݺµô]”ò±êµ©ê]«›Ì Få&õµdjã$ˆnAø®M³é‡QãÃi'aª@¢V.×r‚èvËÍ­ù6R±•J½j®žï¦t¨»Ô/ÈnÃés‘¿±e˜#ÚÕ—K1Ÿ ÖÆZ§7v#¼]<µ™ØÅï…Ò¾s…öz£Ð í6qÒ/]ÊO[N¯05«Œöäÿ'ªðMÊÌS^jâãyûUï„çf/4ÊCc’<`(ö²ƒ»LŒ ³£¿LÒBy&ÝVºyl!\6Èc®æ1I5òXŠMC]3…Uÿ4®…ºîºØ7(¿/ÄÝNߦª×­³`¸ûÖn‡Æï£8¶7|Ör³·µ;`¾ -oea£µ˜¡Ÿj']lu[ízgƒ°RèâN£ïE\gkc'þþ˜ú²O2iŽ]þx)ËœuÞòIZÁþê€1ê2×ÿE™ðû_ºþK=«vZÇ­nla·.ÄãJÆ_4“NŠù‹gª°M›Ahi ý.6©—õ m·vàkêÕu'öZÒ45M; -}÷á ¶ ºÖ¸Á3~Gx[y±2_:6}Áž.]½9>ŒO9®Váb¼\¦®s}I¾M⊋ö%Z²CE\Œârè0wËf2 ÇNšëÙ~‡Âq_ÃÅFð¥ë:êËìØt••»Š vãÅØ0™±Ó¼°G0úÃÓë‹Ç}69ÔÀÒFíMú>JÛ±´ž,uå2,/ÛšãÊ![uòvµ¶Ü0÷\ rúÔè¡-.FŒÒÔ«ÍüØ)UÙ¢ÓºP4º€LWYk1V›Â\­º¥Üq½[LÑ¥=íÇQiNô㨄MYn·®x-Zr-ò#œc¿;ÝK¶X2 ²0ÖnRê%]•mû¦eßuµ½·AÝÐ#SÝ«ùMÉñßGæŽx€µTªþ”“úéîX¨ÉDg]ùÂ>B‡çJ¤4l•<\gïd»CR÷NÆž0ŠœÓÑðW/Ê&¼C‘CSõkåjY‹·ô˜„­¡NsÛãÆËwo•=DzAÔ.#§ŠüU{ŽƒÞÄßµ‘§úÖÿb ¥üüf6Ü -endstream -endobj -736 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 735 0 R -/Resources 4 0 R -/Annots [ 737 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -737 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 627.273057 101.433071 608.523057 ] -/BS << -/W 0 ->> -/Dest (cb163-1) ->> -endobj -738 0 obj -<< -/Filter /FlateDecode -/Length 4678 ->> -stream -xœå]K䶾÷¯èsÓ|?€À‡’ bÄÈ"99Ìôzl³Ö9äïç£Z|ˆ*IT÷ÌXãì`vZU|”È‹%ª)Î?_ üçµ`ÞgõùòùôåÄ™3ÿ†“‹Aý=~9 ®3"XoΟ£@9悑ΞŸOø¯¹¬“?Ÿþþ›Ó¿§yÄY²¬æüóQ¥sÞ9nU€Š E¨ŒaÎIc • ·¥‚Æi©ç*3tð†RÁ -¥'Tf(í„š«ÌNhJ3D0ŽP ;„äœÔÁ®µ’„NfÑè–ÒIÁ¬2:p’Ò)ŬÓÂJ˜µÖxJ§a‹ \P:[ŒÐšS:Ø¢•w”ÎÂe'u°Ez§(ƒ-­‡-@—'u°…[.(]P̯5© Ðqïü\'¹aÆ+Å)Ì›ŠÔaD¡“¬#t¶Xn©ƒ-FqIélÑø¡tà £œ÷Äh•qt+®¥3°EJ§IlAçS -láÎR€Ä"/Q c:HM‘€ôŠi¯=ÅÒ¦S È`˜¶ÁQ< ¸€N -Š‡-F[Š ”€-#—ÔÁ4ÅJÂ$ ¸@)Ø"µ¢¸@m5ÐBqÝjÅ -|«¹°(.Ìä(0®òVS\ @¹€R ¸@s¸“âÒÅpw(°®2VR\ @»J{Kqï*-8ż«r’:ØS\vdJxEqï*Áż‹a$).@¯2àÐR\¬0é=§¸@ƒw¥çšâ Þ•S©s cPQ\ Á»Òਙ¼+ —hð®ÔÒR\ 0ð'½ð.(ÄÐîlÁR\€ù:©(.@aLÂ-¡¸À€wEÀôGŒi°>XŠ xWxt© L8´81nÑÊLX(.0à]LTŠâÞ}zŠ Àñ 3ª¤¸À€w…ÂLMéÀ»BNqïÂo0LÀ ¸óÐ%uñà=ż Z’`f܃IHc’ ¸dÌ8üPŠ ,xÓ{ ¸À‚w¹QŠâ Þån©ƒ-ÊKŠ à×@‡aKŒi8«ŒK0¥ïr¦tà]Î] ¸À‚wã´IquQ =Å0Á –X¯YpÎR\`‡.©³ ׆âÇ!0 ,R[´ÓÀ±fAOq“°EIIq“°.ÅðXð¾Éål\Èõ†-ð).“1´¦¸À‹õZ•ÒYɼ]“:p8§¦bAqÖRÌcÐS\‚gÑ]¢¸f@'4Å.À Š 01/á™S:[àdS\O:!(.ð¶À¤¸ÀKU¦å`†ÆZMMé4ëLOq×–EW‘â ¬4Á”¤¶€&(.@ÿÄŶ¡¸‹,6±Ì!u°EMq÷2.7=ÅÞÃÉ%ÅÀf\o:Š ‡-ðø(.ܲè&S\‡ŽsŠ ‚ðXoJMqÆe\ozŠ ‚âXob‰GŒÛ l\o:Š ‚†-. -©ƒ-pš1nÁIXo‚(…-2hŠ ‚µq½é).€›õ&žÔÑŠk 4_Væc¾ÄbXKsþøùôõ?|øýþ -}|:ýã· 05þ~ÿ†ñ³ÁïCt°8×?|ó•@Z㯵ÿŽ×@:¦Q£î2^ÛiZ3æ5a.‹uŲ&å¥:]S®«ê~erLóp•>^§²+{R9ŸŠ2Ö­K›LìñÓzµÚ%ø7ÿ<üpúöã<ƒGyOt@à*H³·¾,÷àí5`†åé -Ì„r3^ð9”nH]8é -¹Üìƒn–j‚(']Wå¥Ï"Oýu pÝ\OµÍ°I:>«Cé2mŸR/ ËiAfbðã=Γ«¸ìÌs4`bµ³ŠÌ¦·MƒŒbLÕ£—iï ½”‘HG\ ‰$™±÷±Ò/Y]«æ³ó„Rå>2*.Å.Ó’§,ù›U“WÈZk;/w2ºÒˆ^@£ÒûÑØ—çphÔë>­kÂoíýèê^—FåcÑ¥:ÖÚŽn²íZ[·¸´rƒ£.t\­£·/ÏáлîLîéÔ‘Ôãµw”™Ï€QÄŒªürONx2!ëRõrí@¶¶ºiY3ôU¶Ï¸zá3åÜæzgx6²zgöÊ}´1Ô‚®v[hìËs84nøœ÷ôêÓº?§eÓ;#Jñöœ–Óßð^ïg¥ÅOì^uæ9J7bwø[·=É«F†þÍÈ_BkëÏ%.mP£X7zTýyt΢é}ü-kÏ ™Ú¶âKüzƒÚ™çpÈÝðQÃõiâŒ_R%Dë‘GÚÖOÈhš{¡Au»ecòü¨£G÷¢±Ý:Rû»i­mŸª|[OK9Ý5×+±ž†?]}WÇzeÜØ©hôêÊßĤo­ 6l ·3ÏÑЫ×}Ô×Þ ”9¶}foçÏq–b±¤'P?{ymH׬½ÛQEÍy4´h&l_šÉµÚχy‡¨ ³m½Ôí~ê©¡¬×ÃKÑrËo®ë$½[Ÿò¯iÅ•oÉ5@[«¦Î<‡Cñ†?Ú®=qËÉÓ¿:&à¦=–Ññ銂á÷Ž§§yóiäÛÈÓ—fm ¬%äªmãY»¦üðÖ;ØâW¿ÿ9RgžÃ!³ï9ü.¬òÿ¿xn‡_]?‡küéÙCF¿ßñ¼ÛpÉ×4ÅoñC¡¬ÛBngž£!×ôùš)6éé†Ëê(x‰¢/´ø ¾XgžÃµøº/f¬·|@íMQ¸7×S¶Ì,1’×S2ê¯×”.æI믤WÑ;¿Œå>VùtÉ_×5¬“ãŒU§}¸Ö ¹\;Ú’8!æñ×q9¬_â>h|Õ¸^|õå9¾6¼¤[û¸ê«]}l¯¸ˆœûb}Üsíõp?ª*Ÿ•m»Œ˜V¼ ŒuC4­3ÏáµîÝDTÜÒ³1úóžÙç>ûiTÙžÄvæ9ªìÆnÁ_˜wv÷î‹ñÍ8¢6Öýö†]zy‡”èUW|µ•ÃÂþõv_TO\–z³„òêêgIyÅUÅÒž$#lëˆ ïz®”⵫Ѹ´šý» ;ó­·E©^c ÿjHØ»· '2·cïÓŸhÓ=ðùþˆö™Ù$’¶€ÔêM#7Úú­·G:ó©}o,ßgÜ»›äã’ï/jF#ÒÝð¦rgž£!Ò½Á${v¿å ÞÿÕ ø†è^gžÃ!¸ogß½ûéïþ&…~‡jѦ.Ÿñ(_@oõfˆ ès,Õ¶Öäy‡Þ ßõ-ßy+„v×âoˆ;væ97üÓ…Ýг·>Láš¼jhðKîMmwqÇ•ÌÀeÍ~©÷ñ _uÛÐyŽ†\¿îÇÞý=KoÐf·µ/ì%øõ{5~жS›fë+Ìnˆµvæ9r÷½™Òû]4TqÏ]ŸìŸ§8¨Ù3:DäcŒö‰ÏÞáÚ¿Þ•ÓF’ª¼í~þ·}q©7<çîÌs8¤nøª„ï˜Z±ú¶²·þÆÙa×Ú{Ymžµêžo1‹÷ôº+ÄÄÞàÏvæ9b7üÙèñü¼û=™Ã®7½Íá·3ÏÑÐ:wÞè«n}—èÖSÛÉ,¼öM”é™{ý›äãWûF~ìÙùj#y}7ÄY;ó}÷}sÏ»ä¹=‘ªt®~CÇâÿ…ÕÝk!ù†·¬;óÉ÷Å\ß%’_"VPqúä[3.„Þ¶C¿m³qAß›‰ è­üÓÞ'¶y‡Þ½¡mo5«–]ïª&T´Uï -©9,¡M.”¹ñ[¶ïÜqCÚ4Žý©©ºoÛç•QúýxŽ…3èPÏðŒ'Q„êXŒ ÍÍ$Hœ¢‘Ž¢8ÇŸjvĘ…CÂâ¶ÊŸá`êàâ™/Zß~÷—?ýí»­áåñ®ážØ,ÆËËuÓf¼N/ù Áó׆Ëa±>’¥½BÇ MááPš8þ6‹ÆM¡OWBð1­½þeŸ‡r|ZFŒõÅÅ~Ì7J¹«~R¾ë®ìÊ0Õ=˜t³nür²ÃÙEb8¿Ž[Ï„ŒÜž­fqª çx^’çJ‹Ômbhhq^È9 ü%§O -æñðÀxpÔõáêbLÿeèðÏ'4‹íIâzª'Ä^¦‹"~>ÅS]5!ÖšÉNqŽy.K‹”0Ë.àZÇB›RÆÇ!®Lú(Mº›˜ß¤lñ´ÍS"5œ7s­˜"§sq¼{ÎÄ(Ž'ãz¦Æ+›M16Ù a)P×iK%ã¦X(^ Í\È«êŠXêdˆ'©å¦³E˜M¯›NTÝŸÒJWuçšðrBµª•>“Òxg**µ®äY¦*£B–š”2wˆÎýDˆ.'BøOEktfÖW½O5» -}’ÎéBJGˆ.'2ÝXšÉ=år*_²†ÔòÅ”uåsò¡F©‹G}Wc„«ôq*¶±R_s_áF΄—“Ñy$ñóÉ8"µ)\ÁM‚^-TU¹YìyÁK‘Æ×$®Ÿ04“,ž ÿ¿R…ÉÒÈ<á3¤â¹nåªTÇfO©–ÛÌ:šj¡]I[5—p.c¶n/á9‘>’nN[l"­š¬’»Ì‹$•Éc*uY),µOæÊZÈ“¨®ˆM†òóD\ì4y¨ì©2+*¸›\nÛÌϵØåñ¾lo.·Â|^†‰M·â}Ÿ*)bÉólW*«„Å0B}Äi´fv¾í}üX5ã^N?†iN*#>X%♳µ×%ÎÿEšøý×IŸÿœ9»Îã’g¶™['âz&‹'”ûNq<±< ó°©„RW´_Ä—´&¡!ÏãRV|­Kv^ˆ=¥™AEÖÖBSÚ°Ë ])TÕ2fExiZ1;2÷öM™°¯—*ݪQñx¼d.êÛÁ¥/:U¦äsU$®bÒ2¥C«e•u¡¸¬* Ų‰™u%¹é£ýºî -ëª.*7"^ª¢Óeš­‹N²á.ª$pU}Q¬E]i˜ØC}óðº»ßG“mQš©=KŸki¾±(Í„GKÕpi§—yÎQçx‘oi¡\–ªÆžÊNNŵГ‹£•Ô—lcSWª“lRiš(2]@Æ“,—è’MvÌ–še¸cO¶’Ë>E‘þT·´©{%7¢©{ÅÎÒÆrÓŒ—½%• SÃÙ•»ã%eö%=!³uî,Õ%¥J²yÝzZwšmÏÕmèbˆ-ž)/ÙÌ,åMüwËعæi¢Tªô3DF„/Ç‚b¢ÖÂ)©Î!º;ÞZpˆŽ‡u;í«PÉî<뱓y„$ÅNêšÐL+ÅÞLx†"l+ª¢žÇUél“·´¯ˆÐP¡9>‹ )¦ÆPæ Ê—..MŽÖæÞÇ£åd¨(Ÿ6§'!”áçKWÖ> -endstream -endobj -739 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 738 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -740 0 obj -<< -/Filter /FlateDecode -/Length 3553 ->> -stream -xœÍœK·Çïó)æl@4É*¾€À‡1bDpAÖXk#XsÈ×Ï¿º›f×ÌήddµXÍtÕdñWENϺ³Åχÿ2;“sI‘Ï—§O'kR°ø·”Ø],êïñ›ÂÙYN&¸s8%“Jð)žOøoº‹?žþþÕéßû:îìM)Ñ»pþõƒ¨˜LÊ)Ù ¨ -TÖ‘¢ -Á¤äCTTÑÜb)š -f„Äžª3¸ä ©`{ŸU†ÄÉÑQU`†OŽ5Ìp%$Eå,ìpÞZUC,3yE炉ô¨é¼31_T]‚Î%¯éˆLLì‚ÓtÅÄCÖt [B±NÓسÕt°…)'Ma E²ª¶øœHÓ%Øâ1:EÑeØïʪ¶Øh¦+dBÉ̪®@gsÊG·Á„Ld5s&À7IÕaEa’bRt¶DŠªƒ-¬×t[?šœ”rVV«—ÕM–œ¦ °ÅûĪ¶`ò5ø[lŠ°‚ŒpIc€OÉpñ¬AÀg2œ9kð¹N‰4 ø Ç’4uÐy§€,l 5ƒ-Œ•«ê` ÖX@¶ €Æ"Øâ™4hËðܲ-Nc·l]ÔX@.Ì´ Ä¥Yc¹p¥¢±€À\;5 ‹åž4¨K!zì稱€À]bg50¸K„šª¶À5€Ž†\& î’³Icƒ»XF^cfÕÀ£ÆøŠñ9[ îúlYcƒ»>!¨ºd°Icƒ»>à´È îú`½Æw=û¨±ŽbÀ«fà.ôt¶° ï¡ó¤±´DcAw]AøSÖ4he\.QcAw]Æô©ºb\ˆ+ë£l\ŒEcAw¨HcAwϬ±Œ7ˆ¨^cAw!Rk:p×ùb5pyCÐXw0 €ÆÄ7ã,\WÕcKÎ ¸ ,yˆÂÆfDÕ%Gr cc‘‡j,ˆà.Â{ÑXÁ]ˆ4Dp×2ÒU[({Èk Ã²UÖ4’Uc=H¡éÀ]ë0Úܵ6Ü•°©± &ø¬±dÁ^cAÌlJJQcA,:¤|ª.\ÉB€,U[8±Æ$Ö¦PÉ ’‡-ä½Æ‚äa RÈLqȾÕíl»¨û†-È5€d&f)DìÇ0ªš.z“p­ê2t.i,H‰ B±ÓX€½”ÉXô x#é’Ƙc©À,E&{dæšÎÁ$Ù éBçœÆ‚ìa ²AÙgì2£ÕX€½$†ZÓ±•}fÖX9I5`™`§ Rª:ØLh,ÀüÈf;h,Àæ›MlsTl¡À rö²ÝÌ r†-ÞzðMÙo&ÅÂd| ŠFÒdÀ8tÖj,(.c¿éYcÖ¥ì7³Æ‚BûMlñ”u[(Ê~3i,( [RU[4eÝ‚IØo‚Nš.Â_XcA‰Qö›YcÒ ì7‘Ñ«GYö›I=.wC™4¸‹Ì4j,Àö›Èlµ½½xCB~¦+³ì8³zb`ހتX°Ñ*©gð•=’zh3±³ÌQ=5@X…ÒYõØÀ‚¿ÈÇøÀ„õôi<¢Šp‡„˜™“ÜdÁ²`d÷mÏÑ‹Ñ•bëùد?Ÿ¾þÁ÷üóN_-'eõÇ!AXJ3ò+,_0ÝÙ·ÃíS”Mëb,{Vaé6•n–·c61 Oþov¶ÂÔï—~©õ[·×~嘹†xDƆu9 L}f•/h°Åvñ5¢ /<‚“!“´¼+üûw'¸æfÅzêÙ.ó!±ýeÎï>ž¾þÓÛ·øãßà8ïNÿøµüð͇×à×Wþ C Án×¼éßã}ØË8n¯¶2y{¥õ;V,ü’¤fk»KÛµ^-¿µQ¾Lmoe¤Þ¡ß¸·½õÂ! õ>Lý]6ùØßþÌb»ê¤}Ÿ‹nºï6>ó}9¥zÝÚûæŸçwoOß¾Û;ß’ƒ®Î‡ôÂ;ÎÌOøë}U>Ç_ 7ņö†›Öaj®ópÿ4ü?\®¾oËjvAwÃ5Ý~5ݶ$Ç¥4¶×l û1Rï'íÛ¬cpçËt¿‹Ýº«Êy}u<Ù> G“ÈwÓWï¬óÚœ{¦›ÞJûhžùã0¢®²À†Ü\;Í~œ¼böÌ -®‡Á£K—7Ͻ u&±ûúïؼ†Â`§ÛîkZYÒÅç­8ÿ‚ն؜†û,ƒmyŸÍþxщŸïÁ÷ÕyuÌ·y;ñ+Ì¡˜qOª#?qäE³”~Nž:ÛÓÂªÂ«Ñ å¶JÆæ}­¯mø/ÀVƵqüýPfŠ'öáŠ'‡°ø¾:¯Î“o'¸4xZó¢Ë~¦^çÌnœù?³°¶5±”x(ã§ûã¡7õ?%¨ª iê{âù­Õ¸ôIǺ;Òè>_@çûê¼:Ÿ¾ ÏñXbú2®iðõ1ϨD}b®[>Ö6…ì)çþoÍó!þÏkdö%ÆËž#A˜²â[ÑhÎg4Ÿne~ÚçC‹¯Nã¥F”ñ>Uÿõc®+þÂÅ?uöpg×æ¿þv~¬ùïnœgN¹ë’Iä-Ô|eÌM'Ÿº'·Ýí®æ åŠÿ/e¯¤ðà»m ̇+[;²àŸÖ»Õ®VŠ;ÛÃvè_p²›Œë«%=Sðôü†;ë¼:¯¼óîF‘Ù•õuÕ¸½ò6¢˜%—×2"÷~Ó×ߟô÷RæžóŠÃQ’"üûÉòÞKÚ1݄ۜî™óqTË}é¨kýVÈzÁëu^GÝÎ=õ}ì4»Ç‘ݘæ÷³3ǼGªG\n´5{–Õmiì½c‡5zOë³l‡¨ybª¶Ï*7çþBgäíƒÙ+^Ù3A.Á„ZOyå}u^W>‘=~þ,3P¯Ë:k»Y~¢þ5Ïm»:Êvö]É"—þ¹Ÿ¡. ·kŸ‡¾”ŒóÖŠ¼™oñÆÎß—÷ÝYçµy=‘÷)9Ï{‘i½/£ë‡HUÙGu½+L›x9Ÿ,õ]t›TžÍûî´·ãpZtãÃ¥ßüžžˆàuuèݼøûí“Þ0áAžr—lËðÁqsÝ'‹<á©Ç™ëG·gù™]1n>ÏhR¾âAÙá’ä©q½o¿ûëŸøn8Brç?¬[‡%aCç/ëVnM©à’ºe›XáUÁ% ‘é0u‘×u[(S³N¿È½”ëïÒÖö~i'×­ËÖŸ¸žÔ—#ú´êw퇭ïÁž¥0ÜC¨÷p˜ÆO§¸<Ýã–'_…—%°ñ,×ϵ“.ö¶E»ÞÙ ì† -]Ü1ú¸ˆëjmtZ>CØÞöE¦EÜËé—Óæ<_ð&’“oeŒY—;ÿeÞâ÷_'>ÿ¥>¸Æqo-|cëNwnzÀ^/©Þº!y€´Z…‹ñvp™»ŽzH>MâJŠö-û¡".ÆFq9tXºe;3Å!ÇNÚЋý> -endobj -742 0 obj -<< -/Filter /FlateDecode -/Length 3203 ->> -stream -xœí[M·½Ï¯˜³Ñü(~†|q#„øä %Á(€œCþ~^õ4Éê™·²V>äàÕbVÃWl²X¬z,²¹áìñó*àW“àZëµÈùòáôñä]Íÿ¶‡Â&þ>5Ÿƒ—êrè¥åóRuµçXËùz¯»¢­~=ýøÕéßÇgÂ9ºÞK ùüë;IrµÕê3uˆ|HD”³«5æBD%8 ±w&‚¹J”GQ…Ò[f"¨!16"jP#I éQÔ¡F¬A˜j„ž+=BôžÊ ˆI‘ÈBvF/Lƒ+­ÇNe²P#“¥äJ•“uWJÉɺäî“eè’ƒˆg2è"©U&+Ð%•ä© ºÄV“UèaNd ºÀ»•A_|`²ž\îM„Ê:d¾Õö(‹>»ÜRòL‚ËðÍDeˆ(LR©D¡Kñ¹StÉÉG&KÐEðÃdà‰œjk$Z£Fwò)0Y†.1V¡2è‚Ég  tñµ0 -@9å%ƱV'= -#Ø’“&±@lÝI­‰Ñ@ìÙIé•ñ@ò²$]²Æ)@AäRtI]¤]PqAJÐ%Jb\À¶oa\@·â{`\À·âCa\@¸PÓ3.H`ÜÔŠ0.H \¸Rg\À¹ Üɸ tî•qAë¦\"ã‚ÚMÒ -ã‚ÞM<ã濾'© ºÀ€] --1.ðn -¾2.ð.Â(2.À¬:øaa\_q±5ϸ@À»±ya\ àÝX±PYuˆÁĸ@À»1clåïÆì#ãïF‰…qÅ<ÍÀ» ÌÓè‚e\€õ²˜ 1‘–0.ÈàÝбü‘˜[¹Ðza\Á»¡aú¨¬»Paq·°² ¥tƼ‹…*1.Èà]Ðgc\ŽwXQ#ã‚ Þ +5“wCìžqAï"oÈŒ à. `\€õÍ×¥²î|oqAï‚–"ã¬ÂÎ70 •UG -Œ @ÆÎ#e\PÀ»XÞ;ã‚Þõ9%Ƽëi•A—Ô"ãä5!lIL#Yu>‚)˜ ¼ëf˜ÉÀ»Þ×θ €wuÙd\Pª±1.€‚Ypd\Pš¸^ka\Pº‡ )•‡rf\P=€ Ê¢2è"U ±v=õƸ Fè’bd\P#tAÊÀ¸ëÙ7Ý@¸ ÝtA¦È¸LæZa\PsÁ~ Ve²]« k*k…ʸ VqXŠãì¥\CÐ3.Á;M—@ È‚0.¨º T`)r-"3g²]d3.@¦ YŒ Z„.È´Ø°Ë,žqVhì%aj&¯ûÌƸ Iqš*2.@˜`§ ¦¤2èš`\€ùÑÍvf\€Í6›ØæPtIY´u»Ù´]¢Œ à›ºß¬Œ º‡.Èøt_œ¦ÉŒ @ãyϸ ‡†ýfƈKÝo6Æ=yì7±Å#qÛSÑýfe\кd¤(T]4u·à$ì7ÁNLV K츠—¢ûÍƸiö›Èèé‘@Óýf¥ÇàÝÜÁLLÞEfZ €ý&2[¶·÷ Þ\‘ŸqaÓg£'Ô›±¶Ò#îÅjUé™ùê‰@Mì,[¡§XV! žxð/ò1yà„Ûé“=¢*p‡Š5³UdGX²û¹çXÕÒÕnçc¿þ|úú ßóÏÿ9½ÿj;)?êYƒ4ÙŽ³ýáƒ`{ˆ¼¶ø -¸!3½ƒhzÔ¹–{p;µØÀËÉÀØ–‚‚zìõˆc7Ø6Üt‡¼vÔ6ºôbFbàë ›ìHïàœWÝÙeµ»à5’«A× WWË:̾—Ó˜è¶eMJŲ¦A‡|0Àù‚ßœè\Ö¾æ:`uÙÏ.QC1çS­“m§ÈÌ2ëÿjìhÇk¬náÍeÍ+÷¡jªÊpGX·Ï£.¾=€Zðч,Ž¬ixËì.èºB|¨qjƇjßù¬º³; Z*Ô‡&º½ºZÖaö½¼Dîÿ%r5U×çÇ/„eÖ™cøÀòÉ ;ØÍØþ%ø^‚ï|EÿÕóã—gŸUØŒØâàÙ/Á÷|/Á÷Ìà£Aö¼”ó‰€| ¼—ÀûƒÞ秜ûý?½>A³íR þïâDÚö:@ßÒW=Ò<¿þpúú»þúç¿ÿ€ýãë÷§|£‡aø|Då=Fç}nø`ï)oñÑïa/_ðA}‘UÎñ&×g7ߥ~ûÏóëïŸÐGߪO}T…û.·&óÞüO¬»[ó¡ÃæÝ W«ˆý²KRôAŒEßåÆÔFovÀ‡ëÀÞïƒÜýí«ðÍ­çmpedz1F'FiÆmµ!ooßïÛæFÝå{=m/—ýù½YgïcÓ}×u¶ïÆ—~S·²ôÏë¸æ³ùÖ–{k³^6 –52ÿVÿÛåb´o ÓÑùË®IÙU(‡¾ž’L¿Jûhs¹ù—¶¡–?=¶S’}ëõŠÇv6k½ÙuËÆÿ¨)ú§ôðËÊÛ¬“0"ºé5›â%–NÚ!;Æ6íîÚïfæ‡eñ¶—ÝVþöÿV7ííŽˮ¿õ•_v]Þí¶Îwº4£ßh{ÌˮnjšòIûʽ ,UÝ<ûÖö,Ç›GëwXÐGèë-⇓wÚ1’lÄlöñ7›luÌœNŠ´v¹,jœÏÈb’{Ý侌}òÆp7~÷z?ÜîLúsÖ;p¥ÇÞͱò~êøU>}0I¡÷Þ,±\™ùE“z¼}½ê;¶Zi(©‹Å¶H[§3©ì†Øw³Æ¼Ì9\m›¶=œåÝ­žÎÔæ"»©·©è7|[¢Êr«ñý~ú·þÞìndBåÐ~6!7BpŸÚ9†<Æð0Oe{÷¶û¾4¢^=÷ê{÷³¾ol>IÓ6C‡óOn×j?ÎZûM[¯—oôÅëí®)ìõ?nþ+ƒ¸2Ö‚íVœßàGaÁדފ‹¬å7`÷¯~¶%a®O»œBÒìë®fÊÎï_×"–òøó>ŸÇcú¶ -ÔGIcàÖ1ñd °ŽÞ»°Ãz³´¹´—ÊT¹{ŸàjPlÝÕàzð7á¼ó#èMw Ž2ô zaš®,pªnMÌôº±šéüxAö8Æ»Ð+Eõ>ÛhjX+åÀ’QªO4šsBdÎ4=_ùÑ´`2§ÜÌšÞ -¸m8’Ìz}Ô#ÐåDëí­å9SuÖjëÑ>,¿TY®žÚ4<Á.&Ñ«zUÞĈOãëÖeεoÆo∭“ÌH\ð¶%{¬Wø<\Ï‚É´;áæ—¿,[²ÝTÍé;Uý}£ŠÉâe²Ž0 ©«®1W¨uú¬µWhžÔWÒµ§Å¨1™Ákê)IMò8ÂazÝ„•Â†}&WZÐ%ÐÝ‚ótåë^zæªpöÑY ÆÝólwÁerçÕÂuÎxÐK.SßÙ®ñù^¶…Mîauý6:Ypôsµ[p)RXð¢Ñëhì¤÷Ç×dlŽœ~9mË\L9v=´KAïlÚ¬+œÿ‹:ßã󯓜ÿ2ñnëxô“-âäÖlW2½áß>ÖÿœacÀ(†öÚª›‡7Ìu›&QÒªkI‡T¢yÛ(Š¦Ã¾4;¨©i;™¦WýÅNEÒ¾LÁ¤ZLK&k™µMl…©‚8Ù‚mX‚í´ô!JqxýîyßU.#Ô>Ñ«EçÀ„ÇÑ´˱8לtÖÂÞe¡ÙV2±—f’³Bc¥r(X5h[í²íTvèt,“.€ùÍëÐ©ì ³l#nÔJuæ ýÅZ:ÛY™FÌvVÊC]mw¬x3[J3ÊÖë_5g.ÙVìÓ•U3 ì±o9ö=VÛ³†,EÊÊLýz,?Ôü"þû’Ø¡Gã¬ÖøÙNFB[×ê°PK¨)¦í/%}+"zÙ½J3G%Ï~æK/uÙž0NRò5èß¼r®ôߘ¦ÆßÃòǺ§Czžž¼ŸH_d„ힶ7 ÛÑ,VÝ*êß8&Mz›þiFõO¼ØÐc =v)‡#”íç€lò} -endstream -endobj -743 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 742 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -744 0 obj -<< -/Filter /FlateDecode -/Length 4971 ->> -stream -xœí]ˎ云ÝçWäÚÀ•ù¾ã.ð`| 6Üð]³èʾecP=@Û ÿ¾OPâCRè‘Uåžô¸»©>Bä‰Ã %Qúªð÷Æ‘ôc -ž®·Ï—/5§ð/§˜dõïñ Áé䣻~f CHÎ}¹à¿ÅaŸüåòãÏ.ÿ;Ï£¯fHÉí®ý‰Ud‡CPNP%¨”¶‚ʹ!ã¼ òzÀ)¦$©`† dh­ -0ƒRt’ -f1QPE˜a)h»V%˜a‚&I3trAPi;´QJÔÁEd Ónðht/éŒ|L&‰ºFÒY;ø@ÚiI—ｋ’Ž`‹KJK:[œ&R’¶AÒyØb½U¢¶˜¬¤ °Å u’ ‹°芢¶(¯´¤Kvp)‰ºŠ!®uF¹ÁEk•¤ÓzpÀ¦uð(t’‚ÎÀ¯\u°ÅYe$…-„?Ižp6Ä(x«aï¶ÊjIç`‹1DlAçK `‰¬ÒÐ-U°Å‘—˜ÀjØBð\Q[l"‰ ¬-H qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ]¸{¸À‚u­óFâ Úµ½Ä¼kI+‰ ¼k-rŠ:ØK\v¬ŽVâïZ­‚ÄÞ…‰ Ыpè%.V£’¸€À»&*’¸€À»&`ua€Z‰ ¼kNA¹Á»Æ)#qw /q€2€” €wA!N` VâŒ÷Ð+q - ‰ xW' ‚Oƒ­“—¸ÀwuD÷‰º4è€ü­Qç;‰ ‚‚À²Dl¡@ °’MQâ‚``‹5Fâ‚`` B‰  I#ú§°pç[)J\&b"’¸ 8ùZUÒy3ĺu:$. ÅZâÌ¥†§—¸?p¸$qÌ€N“Ä!Á¸ŠÄŠ†h™K: [dK\€H:­%.ˆ¶ ”¸ šˆY¦W`„Æ\M-éHñ<3J\É*J\7ÁLL)ê` hBâôO¶Ä˜ü`²‰iŽ¨ƒ-Ö‘Ä1žnF‰ b„-F‰ €Mžo‰ ’‚-ˆø$.HÊ&K\‡N)‰ ’Ž˜o’¸~ÉóÍ(qA² -óMLñ¿MÖó|3H\¶8„(¢¶ hJ‚ß‚“0ß;I:[L"‰ ’÷<ߌ ÌÀ|½¸$y¾Äåð®K`&IÞEdê%.À曈l¥¹½ñº€øLVFžqFqÅ@zÆVqÉ@{1ZqÍ@|yŽ$.ÀLÌ,£W 0¬B©•¸l À¿ˆÇhÅ ãêS¿D净13>É· D÷uÎÑ’ÙdãúØ_ÿ|ùùG ¾×?ÿíòü³¼RVþŽ4Þ6 ½ýù‚Áôq­ïÄ/GD¦ ±M—´ W¿æU‹,¼]:1¢<°ƒfľÌå˜ÕÅ,ïªÓˆ5§ÔmôÖI'~¹`²€éBì\M۪녾•ÛÄíL^:i;éVUk©}o—ÑÑ_ú2Zk‚T†5ΊpØÓ*c9¸+Cj]­1¸LK—HÁ8Ôó®FOs_÷=Ôu²TýK׌ýévÞ‹3b»S 2„B_P`‚›‹yö\Òâ×J8ƒPÏ!ÔËü°Ôê4›„” !ÕA("õRì|CK­®öò"„ª´t«ªµŽÔ¾·oŽû⸩ó¿ëú‡@²}ë´sø,;䦓õíÖ·ý7çûæ|ÿfÎçù_¸®Üá|Áýw 1Gö7çûæ|ßœïNçì¾sÃ!¿9Þ7Çû7s¼{BNL -Kž/”[ƒÚÜhnPCž4ÒÕ¤Áxà ÊÓ¬ò5yög¢¼´n¥ký!ÏKµÕC^© Y½NcÞ«Œõ¢­wçyw[I+¾žÄ먳zE#Zwªï#“êu6 NE´œlìÉLïhôzê?>\Уù.0|'ˆb¾~Å·•^ƒ¿~ø|ùù/û»_ÿñ·è­Ï—?ý‚WoñÑøäÛ/”2ÏJ9£}œ>7kþýý_?üpÑÀ‰ªw}×;oÐðW´†±‘«œ×òýwúÓošjŒ£ÌééÛOßEË(Žiá\ŠžF9Ú>³œ¦²ø$F‹§4·I==·´¥l -Sš©Ç3éžGÁvw›òÞš}ùØ7ûŠ=ùw¸£µÒ²µÔZ¥³’K-V•³i5Ap›jóS5¾/X™C Kùv²Ø±õqõqìû™gÕ háa¡yaåÅÍÓR ÖnSÙ©Ó=ϱÅ6ŒAÔÜ“Ç¡vî¹Koד}ç1J|¯Å6F+þB¨sÀÛc4µ2c4 -ø¤!%Çaö¼Â]|b(™°æ¬(ñåö}|žËóhøD“îá³²¸Y|‡†ž0ð4%ó'´iËJÇ¿í¤û¸HóiúžäÆLúòù4}§.íÓ×¹^ã¯2Êß \£Î¡ìdžAY3€ÛAYa¤Êf+:U¦”“þ©ë­ÒÊŸZ/1`2šl‡²Ò[qÞ+…!©cÕ‚¢šö¶Ïœ¹>³]vÖ ³¢M å,Ø6GÄ~.+6©+·©îÚ9,õH%?RÏåy8¤ú]>”¢Ë‰]èÔš3v¹f1vϸkÑÛnŒb\—÷6·/ÛQQ> ß½²tK_½AÍëjhÞ@a‹ýÈ™Aû1û(<—çÑPèÏÅ‹µgúEÝ-ÎzêÖeè©KìÉžK隷ԑCѧ1ÿ83™—¿Œ‹ûúlܶuÆ“ÂľùÙ‘˜ï÷ßEÞÉ<‡¼ƒrÙ“aÞÒ+ôu=»É6~KÑ­G˜-¬xGÙßb÷ó~(ÈŸ7\=­÷Ÿ2ß2Oßçy€²¥—Ý+šºµË³3¢“y}çâÍ£kí$ÍR–±“´Š~O<ÖÅŠÿz<÷OÞ@i‹ ‘ut7ÊÉ<‡Òƒøò쬣¿J¹˜m,‘Ƴ¢ ê½˜³8/ÌQÐV­»ž^pY¿ -~t·F -÷ßkt2ÏÃõîÉ..ZuÃWól"Œq[ßF^ÉæƒaÝÓx,é8O™a½åèü6•ûÔ壖¿¯«>eѧý8ÖÁ†Z®Ÿl)œÀy∼<ãyóñÕ_m>‹¯sy _Z„X¯í䮳îêd?ƒiåÝ:ùÌy,ìX—Ç…j9üЗ™l»M ÎSÞJY¿Ÿö ­ïø](>™”ºíE+Ž“Àf½iÙ¸bµwåü)8É/²ñj€ŽxïÜ­gàÌOcó–çs3NÌ_ÖÈò¶ Ó]÷¦<7—â—ߨñ¡ûŸÚ¨‘Ÿ¡ócúüÀ|š®°qZ?~ê3 ^Õ Æ•Ó©>~Z•óó5ŒîAþZ¾Sõ¡þbOy°¾žƒ+ç°êÆ/Ÿ÷€Öù=ÊceøEAWO£)ðã ¼´%]ºMç†ÖלùõJ_jªéKŠ_ÂÀp¯cê¦ô_r‡¾èD·Þ-"¿Eeq4å ‰_.üvÄDmˉ1 §Ÿª–Eºî`Qd7#oóH‰pZM?Û6üàøÓ¸r6œß•l¼k1©’Èæ}{ÇŠ!rz½óÙ«AOb~ÃPìtä«)ü {¶©OÛ -lÅÚªNèÖBÕU×ĆŠ=šw¤¯Mç›°šÞ7¤5¡ëÎ=áí‚jíRú"JyZWŠ*­kT•ÙΨT¥®¤¬BµŸïÓ´~§Jфάú®×xwøQ ¨¦K% º]ÄtSi®öT¨©bËšJË7SÔm¬ /ÈnÝV0_™Öùˆ²åç\Ì;œÔ¾V±ÃY oGÕ›8ï͵NíW(W × mWnGÕðÒ¤¼Þ4þ‚kï­ÍÿT᪔™'ÿÐT!Åûã·£V ûfÏ©—ûÊ:$µ€¦ÐÒvÍ¥C¨˜íÛ QžI·¦®-6“vMÖÉC*æ1IUò˜‹uE]3…•ö©\Ù U1Õ5±«P~™‰›®ºªSµ2¯;¸»ZnûÊ/½8Ô×ü²ƒjo-·Ã|ÞòÀFK1C?–JšØ¨:ÚµÊ:a3 ¤ÐÄF_²¸xke'~[ùÙœLqo—¿\ò0g¬3¼‹·šßÝÓG]úúw¤ùŸÿ¹Ðõ7eS¹q7ª²…©Ü:÷#¿é-žó›ßŠ°ºM'4ÔÑ~ëØÒº‚†:ŽÓñ5µìª{I©+ƒêªí…®µa'6ºFÛ®eÜŽð¶hÅȼµoÚ€=Úrr¼+¿f X…ƒþtp›Î¶!ùÚ‰#NÚ†thÉtqЊîÂÔ,›™É€ì+©MÏöSß–ëêº0‚mÓQfû¢‹,ŸE—!°íú‚I÷•¦™=‚ѯv¯7÷ûd²/ÀÒJíUúÒK뉱´ž,µùÐÏë˜có†6Åy[ Õ²lç{¶9Í5ZhA³ƒ£4¶lS×WJE6«´ •. SEVK Å&?e+Í’Ï8Š­jLѤé[Úõ½RÑõ½âWi¹Ü2âÕhÉÖÈõpíìTKYcÉ(È|Ÿ»J©¥´E¶®›æu—ÑöÚ5C|‹LUËæV)_ůñqÇI)UùË+#:¶×«` &¬±ù¹*z!~éY nWÍûóì¯l¿Ý£¯ Ý1µ*h~÷±l éXÑUÞ‹,gKJ<¥û@Xj4·ÚTç—_æuǼ@TOù]·–ƒÞȯè jc¿Ð²YC˜-¡ä¿*X¤ -endstream -endobj -745 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 744 0 R -/Resources 4 0 R -/Annots [ 746 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -746 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 593.554475 101.433071 574.804475 ] -/BS << -/W 0 ->> -/Dest (cb166-1) ->> -endobj -747 0 obj -<< -/Filter /FlateDecode -/Length 4469 ->> -stream -xœí]K$¹q¾×¯¨³€Mñ|†dØkXðÂù`ø0];-Áè5°òÁß_d&ÉHVdUvÍckM£§;#øˆ ~ü"ÈÌbÛ³Á×wÿe²SÎ%E:_~:ý|2S -ÿ曋Yý¾S8[Ci -¶ÄÎ?±À§)•àR<¿ðßp)‹¿þã7§ÿÙÖ±g7• ç¿~bù)å”LPT*c½¢ -aJÉ…¨¨¢p‹¥h*˜9ºV%˜A%M3ȹ¬¨2Ì𔬿V˜á’%M3l IQY;¬3FÕÁC䢳aŠpzÔtÎN1WT]‚Î&§é¼Ÿb"¬¦+SŒ1dMG°%c5]€-ÁM[Èç¤é"lñÑU[\N^Ó%Øâà¢è2lº²ªƒ-&«éŠŸBÉDª®@grÊ×:g²÷FÓY;`Ó«:Ì( RLŠÎÁ–hBQu°%xã4‡-„/Mž>å¬ÌVdzÛo5]€-Î%Ru°ƒ¯1€‹°Å¤¨QfÐļ¤q€Ki¢âH#—ýD™²Æ.—‰Rò ¸&Š%i<à…ÎY¼-¢ÆÞÂÂÌUu°ÅÒ¸À;Ø‚x[y <Ø–€ <è–L±xð-5.ð \˜i4.ð`\Ÿ#i\àA¹€RѸÀƒs=¸SãÒÅtOx°®Ñi\àA»žrÔ¸Àƒw=Y£qw½GMU[` ÀŽ“·Ùk\@à]oMÒ¸€À»˜FNãŒêF €•Éål4. ð®Ë†4. ð®Kª.M˜ƒ^ãﺀ[Ð"7x×ã4. ð®#5.P&ðQ³ð.($èélÁj\€xó ±É!-Ѹ €wmAøSæ4Øj²¹D x×f Ÿª+“Mð¸2oáåÉÆX4.à]*¯qAï‚>³Æàø Õi\À»Ö#Rk:ð®uÅh\À»È‚Æ€Ãd‘h\€ø6Y誺2™’³Æ¼ Zr -O&ƒIT]š$«qÈx2ÈC5.ˆà]„÷¢qAïšà½Æ¼kiªƒ->; ×@‡i«Ìi$«“q` -MÞ5#¬éÀ»Æ¤¢qAïrØÔ¸ &¸¬q œ; b¦©¤5.ˆÅ@‡”OÕÅ ×Aã‚d  ,U[(‘ÆH¬§âKÖ¸ 9ØâÓ¸ 9Ø‚”AãdS±È¾Õål\ÔõÁdŠ€É¦\ˆ4.H!b=¯jºè¦œ@ת.Cg“Æ)Ñ„Pl5.ÀZjʘô€à'N—4.€ÐYÒ¸ Ø‚©¢qBÑ”2sMga ’l éBg­ÆÙÁdƒd—±ÊŒFãDh¬%ájMG†×™Yã‚LqâTQãL¬4Á”ª¶€&4.Àøðb;h\€Å›Xæ¨:Øâi\³ãåfÖ¸ gØâŒÓ¸ØäõfÒ¸ Ø‚ŒOã‚bâÄi²Æ qèŒÑ¸ ØŒõ¦# 0/y½™5.(Þ`½‰%ž2o‹¼ÞL‚-)Šªƒ-HšŠ2oÁIXo‚4]„-®Æ%F^of f`½‰Œ^ÝȼÞLêvx70“¦ï"3àëMd¶ÚÚÞ€xCB~¦+3¯8³ºc`@½±UÝ20à^D«¤î/¯‘ÔM˜‰•eŽê®Â*”Ö¨Ûü‹|Œ®8aÙ}’[TpHˆ™9ñML BvßÖ½˜ß)¶ìýõϧß~Dð=ÿùO¯¿™wÊê—E‚0—&äW˜®`¸²o‹Û÷h~7ó¿ÙBøH6"•n¤-®›nl$Âÿ;_ÿ²±†ÿ0wï[÷Q|éžX…Ç*cù:ZüÎ*_Ð`ƒÅ¢mD_³ ‡¼ÒЦðï?œÔÕŠe´] -ó/±&Î~:ýöŸ¿ÿþÿéߣ¯§ÿüPXFñWäQ°ÛñODr¿ûÎB‡¥oÖ½,×U‡ÕÍ\ÖÑrÝä!»¬í½ˆ¶¨·)û@ Dö‡oYöãÒ/‚Do7®6¬íÌuЖ-¿„ý¿û¯ó‡ïOø°Êœ=Î@!–ý)Æ;Ø:Vås°õ5 …¥è Hµ¡Cô®¡ ÿ}›¡ú¯çûð¢üîÝjÓe…p\ÊÌÓ‰þÖa»ÜPx`¨p¤¹‰“ƒuž (XiÜDÊeñh€»B\G+¯?×ë°^‡°Õ³‡ƒ_un[&¤u$Í2*}DPÕZ·Ú°ÑUÒ¶Lí›í¥ÁÞj?÷d¹Oâ«»Lµm§ÿz_Ú½Óë`kõßem»ÝëÖöÙOn­¾QT@Ú[LvœŠÝFë±:O‡VºÍky; -ÕÃü0«r¹­ ™Vig@ÐÜÞ×B‚í¿·>̶Þ‘;÷LZÿÃ=KÄ…—Š2¥ÜGQ.nûk~»àÕž¨®‰8£ê6RÕy:¤ÞNêî¡qœÿMm„G NSËH¹³\#²!p˜ )ßrÆ5ä ÷¤¯]Döô.ð#j¤ñn¤?Vçéy''܉ºcÔo£V®e»üPÇ(¼õ-gÀ§AùÛA°3ï_/¬ólvwrÕ/à´¼Ùc´m9å‹Èõ¾‚åŒÏ‚ðôö<”åa©voE~°ÎÓ¡÷Nîzwþ9„3„¾gÖܺ‡1+Q|Òtq¸'ÕÞ4>°÷x°ÎÓ¡ñN~:fûÃZ¥yM]; á—eÑ_mûãÊeb_ƒe¿ŽÝAn~ÿÀÁ:O‡ÜÛyìçåX¿B -í—á×ÏÉ·oîŒ~ËÜá\ÿÀ^ëÁ:φ\,m;II÷èˆð Š. -ľ'ï®KªD[DÌ;ò¼GûºìâÏY£õ_ÄŒw’Dݶã¥ÅѤĥ¡mzˆý|•!í Uì™ÍÖy:¤ÞÉU?*¹cõ¢œø¯K”ú—¥>?'9­õ7´ÃåÝ»6œ8Ĉ±Î­=ÔVVÆ…{ã{úº+ÄÄ>Ϭótˆ½“ÏÞÙ}¾<ïó3™§]-ÞÍZvÐü@Ž{°ÎÓ¡ùvŽû¹«)ù¤sä¬#Om7Q˜úÏω䌩ÏÜåw•ó3ûË»üsÃi‹&™é|ý™¬£Øg=XçÙÐGwòÔ¿Ež{ÏNÕ t^­”.¢Ïgi_oïaÉ2'=¸â:Xçéüy{®¿J$‰½Áé-Ø¥«<|ôƒ¬?úl}Ç€ÄlgQ_ñí Wä§GŸØ¬ótè½óbè8ZêE{e\éԽφŠ‘£ä[!’Ã*ÚÜN›ïEü=ÛßùÆjÓ:èG1Sº{Ïk¥¼Wp°ÎÓ¡ôÎ{{ˆ«;<ÕÓ/!ã^ꀤŠÖ™d®Þ üÖ{¸¿æ'A䨆&gCº÷¾ý±:φÔp;¯ýû“ƒl@´×l [©÷“¶mî=Slr'íÞAë{°ë<Zoç®mä^·(Ù¼«iÍv_µFÀ]ö62C60¢–ß«ß¼5˜·åæwWÇ¡š×½në_¡cEͼ÷»³·Pg–©Ÿ9xÇŒs̶Ùæ$îSfBÃó³fÿÎGPÂ#ŸoúuîɆÛùëÈ_ãŽÒf7) -$UÏ<òÐ(%sýVvÖíß—•|%Qx…(+¸PØÊIËÕ -i¯¯u¸/À­^øµñ¸|.7Äóºƒäü«ótH¾ãze={õLè)GvxƶDžµ­K½xòzõ\ŽDvèÈQUƧßãg%nÌƹO]wÃ4*¦ã»¼ë<¦ãÁO~‰7 êgýÖežQõÎX·|$,m2³kL9ökœ÷ž%ï}fE{*ÅsxŒaÈŠoE£1ŸÑ0ÝÊü¸Í‡f¬þR#Š¼O¿âÝXËÇ4Yøù~Õy:üÞÙÛUð»ñóÏ`µ7]ÊVdn:~2æ@nûè' -÷öRH`·ÍqE|âRßkQ¾0À#@1íã?yŠ|z_,ćZ—ç‘øÃÿí_þôG±OÉOûxMמµôßÙ®yx˜­ íìÝ¥l`öy©w¶\_Éײ$ß býE”ß aÑQ4íSl$Y©,m²-ã2ŸVùZ§¯ÿ×úÒþµïpY¼|Ä­|¾PwëòÃf¼VÜ -ëÎ)æ ÿ³[ﲌ=`ªeq5)nl |žŠçó4W[ü:*a¾Ó¥ ¾Cûãu;Ñóa3| áu;³ç>®¶…ŽÃ;r޳ìž_Gº¢cö¼[æ¯bl ¹X”¶gtÆ^Ÿïun;uŸÏö›¥Lë»tDivÌuÜR®£wDîjw{m¯ãÒfì·õuËÏÙŒ¾˜ïåµ·F¤¥i‹¬S}6Þ¯´³ÞkOÖ17Ÿ¤þ{õ{ëë²ÖYëÚÎg?¬§£¤€ |2,ŸoRÄa+-,Þ-r' -^ÍR89ó×æºïÑ$‹ìói•Ä' idÊpwŸ–ðTá9¿`W—”uKÜ.¯VÌeê©Cò²ºnrLd5$ñO¿É ³²¾ZGë°Ç>ló·ém×á™_ÃËÛ©²i?T*êöl TÖ×Cæ{¸ÆŸOq>ËΧ"š˜'ëøØddg§©œù®Œå‰­ÃfgGÛóNÍù°éŸ[©õüiÃGRòqdËáÔâb-ÿó<à?ÍÁÔ•¿ç³bÍ,ή^tñÛ‰Ï -&ELÔƒÀ1 ×_Mk‹l‹)UvAž—¦2–ô¡žå#õWêÝpýP«ñNdj!?Ÿb´t¬Q3Úk1ß½™ì*æó–óä׫ØL ±Ú aodÙÞ`¯xWl½Âp-4¢».vTí±|>_s]ìÂfºtÃ_˺$†ó–ðrB·~”¾©R>åµ6U½ëL“yaTiÒPK¶¡6NŠèrR„ß™Ú4a0›^ŒŸ•·sµr¥–SD—“Znm-´‘J­TîUKõ|7¥CÝçæxEvÉYâäÅ1¾þºsÑÆ¡³ãÆ] /§@m&vñÛ)$¥tè\aB…žzÑngÓñÒ¥|~Ñò¦f•ñIcüÿB¡I™yæ_,5HñiýªwÂs³‰·N’òØX‡4XJ½¬p—M©aVúËf£”gÒm¥›Ç6Rá2!O¥šÇ$ÕÈc+¶ uMÌVýÓ¸R -M5ÝuqhP~Ûˆ»¡MU€½v­€{hívqlÜù&Å©¸å£›½­]ù*¼ÌF1C?×NºØ™ízgBØ )tq§Ñ·Y\gkc§ùÜ‚õ×>É´ˆ{9ýå4‡9çƒ+ø%zË'ˬ˞ÿe¾Ç÷Ÿèü¯õè¼%Ž;ÓØÂ5n݈e$ãsïóA1Ÿƒ_…mÚ¡#Aû]ls/*ZwNð5õê¦{-iƒÚ¦•ÂÐ}(Ä®A×Y/<n/ƒ["ó¹cÓöréëÍ¡χ.V«p!o—¹ë|ÉgÑ$®¸héÐ’q!Å¥è°tË6f2 e'Íõl?É¡ðÜ—¸i_ú®£feÓU6ß…(‚ØË Ù0YÙiÙØ£ýðôúìq_MŽu°´Q{“¾Ii»1–6ÂÓ¥~¾ŒÛËsü™/êäíjmy1÷|KrúÔè©m.$F…4÷jk ;¥*ÛtZE£ ÈL•µSµ)®Õª[æ;Ϊ—RË)ºô/ÒÓAŽJsb£¯Êr»5âµlÉ·(H8§~w¦—l¹dVdQÖnRê%}•]÷MÛ¾k´=‹Û nH왩éÕÂUɇø³Ôv!´RõkÞ±¹6‹@M6yçç¿drŒàâ#àe±Uòî:·÷N®wHêÞ‰ì Ã1‘÷&YþKPº oP”û -ÑTý+QzµbÔ[zŸvOí½VðÎÐü§@æÇ(óQ»L¼Tä¿üã9éÍü ’±ú¾ûúf¤Í›-”ùëÿ­ -Æê -endstream -endobj -748 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 747 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -749 0 obj -<< -/Filter /FlateDecode -/Length 4217 ->> -stream -xœí\K·¾Ï¯˜sÓ|_€áC‰ƒ1"ć i,ÙVäò÷ó»IVw×ÌÎJò+^­Vš®â£X¯Šœîvg‹ŸOþ)äL)5':_ÞžÞ¬ÉÑâOk±¹hì¯ñ›ãÙYÊ&ºšJ<¿eBÈ&×ès:?œðÏîR68}ó»Ó¿·}ÜÙ›Z“wñüÃkfQ0¹äl£Âª`YVŒ&g“ÂJÎ`‰µj,ˆ3y:²2Ä Z¢Æ‚ä}QXbÊ.YbøìHcA WcVXÎBç­Uyįð\4 JOÏ;“JõUåeð\ö/“2¹è4^5)¥X4A–X­Óx²DGd5d¡P²ÆK%¤`Udñ%—!‹‡vªÂ+ÞUTd±É:Wƒ‰µ©¼ -ž-¹yÞFKVã9g"|3¨³hXP(.5,@˜`§ ¤Ty0¡aìÛí¨a6?Ølb›£ò Kˆ¤aA)ž·›EÂR ‹·^Ãø&ï7³†ÕBT|T› —ÉÆÁ³VÂê -ö›ž4,@\ò~³hXPƒÅ~[<%nkH¼ßÌT‚,%ŠÊƒ,(šª·À$ì7N/A_Išï7‹†(3°ßDE¯ Þofõ¸¸+IãwQ™& pý&*[moo¼1£>Ó™…wœE=1°€ÞˆÜªX`/²UVÏ ,À—÷Hê¡ÄÄβ$õÔiLgÕc üE=FLXNŸäU‚;däÌ’y‘aA¨îÇžc6 Wš-çc?|wúô%’ïù»ÿœÞü®”õöÈA -˜†µ}…µßž lQ×&A~¹ 2Ý‘`º·¸¦=±Z4âå$ÈØ–{ìÖŽÝ`it1êÚÞZÈ&¨±A~8a³€éŽãl;¦“DšãNò\Ƀ ÎEÏ©¦v4ý^NßÀÐïäS›•Œ´ÆA‡zÐÁùœmÎ.lË\§­²Ëzv‰ìˆnkk˜š-M$¬¬Íÿ ô(×+´.ÉÍeųîCY”á¶dÞ>÷¶øt n|h’·>$騚º·ŒéçŇŠîCEøPAë=9¦ÙvL'‰Ò‡’êCƒ:=§šÚÑô{yŽÜŸ%r¹Tç?çãe7Úkx«äÕ ÛèMèþ9øžƒï7|‰ÿäóñÂO -,V,±ñìçà{¾çà{bð©Aö´’óJ@>ÞsàýÆï)%'6…&PâoÊsâyq³5mÓHíöRlI}Wù>}nïDùlÝGªçñAß—ºàL;ª Í¼Ñaßk}Hª¬OîóÑe-5ó—à¼1ßÌ« -ùXãG¥»ËöMÈãaÏï_œ œvGþ¯dˆJû.ˆoÑÈ|ž}~ñöôé_ýõÏÿ - ñæôϬu‚ZGü þ¾´6züFké5~Ëò¹ñÜ£ÌôÏÿy~ñå•9ù¶‰1'OãÓ:,† Šß ~yªux(ÐƼÐ{;žÒ÷¶oVQÒ2­«@€qSØ'óRø·÷‰¿Ä÷¡t)äbå¢}ùü÷ϺþŸÖÿ;=.³sÛ&å«…Þ¥>Ð׶¬À>­cŒ•ô~iöáyb_qãì´ÐúÑÇ®šK[ùmêkkãÒnmR–üMÓ^Ó¼º>3ìêªý¾j?5Û®í–6$ŠÃî /¯´UCÃgêìc_Á¢$¬'dišê¾VW¶—u¥i]bÚ¬ ˜O„áËaÈ´ø/[…ÇsßÇI¿Oá{vŽãÈêÞÐäîÖ}#×ôzYÆir‹ë%Bnêšöú ÷Ä9-™¶‘·!Ì]n!ËuTqnú[ë÷fó}å_m:‰Vû¤m›noê4tJsìX]¦Õ§»ý6íòqM{oü4®c:¡ï˜ùû‘ø ÊÛ¹'NtüU柎¨/·²qú¸¯D. -Ò[æú·yg®§Íq¿Îý^ç ê¬òù™ Ü·×y?ƒv]å#dã8?N!rWäýà rû·A¢Ûëã9ƒ|h‰á Ó_a‰9w;ƒ¬’úË¢ém¤¬hõjF‚š ’hÛ\Xíªg•eìÍ®#oQ}Ñêí£îdž áº×ðÍ<ô«ßÉ$¾yû#ä¡ã8ǘ½)G»9ûÇÉ/DZ{8Ä¥ÄAmžîù=î­Ð·›XÄö¡8×-qJbÛa>-¢Ö¹{®Ö0rù|S·e¿þØÇ8•VºÌµ—;d/+þ¥…ÞÚÜ84Š›C#‘7v¹i -­úB?"·ðœKŒM]Ϙ¼¥›l¾‘í¶¶¸(9QÍg7çá‡6óø^Kt›ŠûâE;D 㤓oo¯oÝŒÁ_©òílÅ°b¬•Oíó^§£ëy#…š±^ªçû.­‰¶KûCHG}ÖE5ãÒ…8ä-ÑðÀa9“üÓ—_þág’lÚØ¿ƒþBÛŠÅW§«ÉØÊ÷ßï¦ë À~H#…<2š–ol†Gh£½œrŽDœ{*Ù%À,R‰üvÝKŒâi7FÓKÙŽÓWÐx¯·Ivèp$Ý©ËÃ8;9¥®‡ÎßÜ?W—«…ÌËu­ôqöJHÀ”9‚{Û;Câ¾>?wH yù®ó”cõI ‰v&ÜÕw®×D-kö -ag¶×;×|%ÌÞÑ~ç¦c¬½ÛŠÚË–Ùîÿßu-ÞñƒÓüÕ?×[Rv¥<â¢wöùå¸(¼†’¼ŽÚG÷Çuã%¯n÷r˜e|3Æ«RósÉü0Tõ6«ßˆùµéÛER/8ôÂùX˜ò㮕,ÛMڲܕ ZßÝ"Úd¼£Bôˆ+ÝÙçC¾€ü1ê€êÒ-⼺Ùnõ²ü² èeû±õ¬=H “`übÍdr¾ðΟoŒ'¶ƒ=BzdHpŸ[Ö+^æóôºÓËîëó ñ²)6¹»¼lg…ƒ¥òOì‰ù'öÄ|'^ñ¦8ðLJÄtµGo{Ó}}~qÞ”nbÖG³F¸ž<ßýy²=ku#QŽ$˜çéÂþ$>Ôõ´`=ÝW¾yðL­‘oÙ͸˒_¯O3刞‘ßä’ª¯U<5ÜìÑ&xÕñQª~kÐáÁ›­ú0$¿Æ(”³‡óÔÌOþ]ÓüV‚¿ásÜÐOÀ«8ç]·”¾ëÛmOBÆÙçëYŸ7Ý÷©7K±Ät¿;UïŸ÷§àm¾õ„¼xäy4Æï'_BžÍÉÏûf|wJí V×Þb`S1ÎókŽÎ‰L`»Ô3?5[l ×Í暢ÝùJÏör¨w£Õú¾(˯àLJ——I‰‹µý»fð·'WÉ°þÛ1q{·‹mäâûÅ$?œøÝ>¤‰æYó#d&®í ÙÊïhà&ßC¸k‰BÊ®çù6WWËGûj¸ìÝø™K²½QhO.+DôLîHæÕ[ãV2¿©˜°^¥!JL]fÆ÷1 ɶsÀÙñQ² Vã‘hÅt“ì©Ëãøyú¡º4‰Ct©:'ÌßÛ¢òw/'LöÔ•ÊoeéCuíz;hAU5ö–à 4ì¤ø&ÓñÛ‡&sð…ÕøÙö…Xº#ÑhW{;…t9©íÖÑâ°T­ÊìZ»æ§(ÓÕCŠWhñP澉±¡Ü’ù«„ak[„ßøñrŠ4"q’ÛÅÇÖqb…Ýõ$1ˆq¹Øé/“šKWB³ÓøÉ`þwŠ8¨Œ<탣áRütÿ¼š“plòVI’žê¦Gy¶êr9Ÿ•úrÅ*ítG롱 U¨LÐsíâ1H ðØ’ÝðºAfëúX)‰¶ é&9W~اœq„j´c²ä„»Ç1î$§’œ‡Å¿ªaÈ;Æ>߉—–ØhOf×/}’Iövd»9™ NÁ -“ïûj9Ìa(›¿¹YáŒú8C Õßê¬w«V]ÒÓpõ-;êY®½º³K¶¢q™y«Èoê \ô~Á`¶WžÐ¢õ$p{„Ò~þŸ¨; -endstream -endobj -750 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 749 0 R -/Resources 4 0 R -/Annots [ 751 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -751 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 265.954475 101.433071 247.204475 ] -/BS << -/W 0 ->> -/Dest (cb169-1) ->> -endobj -752 0 obj -<< -/Filter /FlateDecode -/Length 3999 ->> -stream -xœí\Ko$·¾Ï¯˜sÓ|É"øÀAâ FŒ,’ƒ‘ƒv²²h¬sÈßÏWìæ£{J3#­Ö+8^A«é*>ŠÅâWénw´øùÂá?&g˜KNt<½?|8X“£Å¿ÚbsQÙßá7Ç£³”Mt%q<¾BÈ&—ès:>ðßîrnþpøûoÿÞöqGoJIÞÅãOï„EÁdÎÙF…UÀ².(¬MÎ>&…•œÁKÑX#fòtÎʃ -G1È{VX 1eÎYbøìHcA WbVXÎBç­Uyįð\4 JOÏ;“¸ø¢ò2x.{‚I™\t¯˜”RdG%ë4^„,ÑYY(pÖx ²„¬Êƒ,žsÐx²xh§(<†,°.VyÅ&ë4^ &&Ry<Ë™ÏyÞF9«ñœ3¶TN6)e…ç!K²±¨<ȃõ/@ÂÆNÄ™•ÓêåtœÆ‹ÅûL*²`ó5ð ²Øœ4À 2‚Køœ Ox†˜XCÏÅPÎAƒ_¢¡T²†Á:ð¼Ó€ XÈ)iHd!œ\•YB! ‚‡,h aAÅSа m Ö¢aAÜ’-N¼%ë’†€ 1­†ˆ8‘† S*`nvjXº8îYÃ‚Ô 1y `7' p7³p7ôTy¬aÐÑÇAÃîg³†ÜÅ1ò`W ì0iX[1žÙjX@À]Ï–4, à®Ïp*/œÁ aw}Ä4Ï ÜõÑz ¸ëÉ' `(øcÕh¸ ‰z8Y°À߃烆Ìx„%Dà®+pÊ™ZÇ%iX»Ž±}*¯—¡qåÜBËÆ¥T4,ˆÀ]8ª aAî>YÃ`¼GõDà® ðÔ¸ë|±Dà.↨aÌÁ8À¿gaº*¯[˜5,ˆÀ]À’×°^ØX’¨¼l`HNÀ±±ˆC5,HÀ]¸÷¢aAîÚ‚† ¸k aʃ,½†ˆkÀñUÎ4‚Uc=Bãw­Ãk<வ¹hX€»â65,HYž5,€€Q°×° 1™’sÒ°  B>•— ®£†Ù‚Y*²P& X› -kX=d ÞkX=dAÈ a"S¢o5=€,05? È‚HQà ™áB¤aAŽ ù´ªñ’7œ×*ÁsYÂœÉÀ; Kơװo$\Ò°b€çHÂ\ ŽŠ†pE†="sç ‚l é‚眆ì! ¢A Ø3²Ìd5,€‡F. Uk<²’g²†LÉH¨¨aŽ 2M ¥Êƒ,€ °?’lG ü ÙDš£ò Kˆ¤a³—t“5,`†,Þz `›’of Š…,ˆø4,(6 “5,Œƒg­†Å1òMOà\J¾É”`‘o"ÅSÎm IòͬaA!È¢¨<È‚ ©(瘄|è¤ñdñ…4,()I¾É Ì@¾‰ˆ^- °ä›Y-wc2i<à."Ó¤a.o"²Õr{ àñ™ÎdÉ8Y­X@o„oUKØ o•ÕšøJŽ¤ &2KNjÕnLgÕ²þ"£3LXªOs‰*Á2|&gYdÁ± D÷=çÍÂ#Í–úØO?¾¼ƒó=þðŸÃýoj¥¬ý8µ5!¾Â1ðÛ};,?$$˜U2ö”¶2i.C(\‹l"Üü;žØH -AETôF”$÷øh×y¬ZòÅ:ï­›cŸWè<ï­›cŸ—1r‹‚€â†ƒ­ñËn¾T–y‘;`=r0àY-ËèGøf0%~š$}Z×/QuB›P¼D!p-È-mÿîÍi•bù®£_º»1$äùž|<¾yøòß|óû?ü[ùæþðýoªH¸ä÷«ß|søúÍ@z™ o§ úåt2j•ë¹Îðõ·ùÓß¾f€F°¹pžëß,é¼0õ…?ºõoZÿ6z´–xi‹géíB—ƬÐ[Û°ô~ç-~O ÞñiŸÒÔv•… Bº[ûµëûižÖçn×—×ù×õtùN‹lu¼Óà×±ÒÔ7/;q“êiV½h[4Ò$³{M´¿²#ÿ|œ'ý,´æiÚi\)K·]¨”>§Uê´Š›6r{"Y„8«œaÕ~íó¢ÙE®óqR¬¡V¦ÏÆi;^åIU“äðõË"U;ÖRwº­OvÊËÉ7É’OE[d“ñe}ÒXì°¢n…÷ë<ÍâÒzm'}»U×ëþÔ¾ëºû>„!¯:ŸbÉmn™S[÷ó¢nÃ~ý±±žˆÅÒWzöS¿~=Ý ;/òU9ß®mîÖ>q9Ý4ÏåÖu­ó¸u\ kïÓÆ”¨k·«=5ýÜý9ÛuãJóûs5hÒÎù‹º‹{Ý-H°Ì+ߊ.È5ì¥Îá¦vëY’¶Ë9XÖ õ­k) ºÉ2ö|gg*ú‹§¡'º¿¸¦âvkZðdœ±ù¼Ãïl|nƒ°Šr$dñìÙùxÅOßØçYaϧðÌòU#öñVwͲu±9ædÚwç²lÁ£Ó!¶EêÄ»é@Uè®èÊhgä¬v7äóZŽpsl;'Ÿ‡‹ªðtí§ö­ÿÚo1çm ÐuÔ‡¡«³qvr̺ì:½¿}®&×ö8]îãì#& Ôj{)n\6ùÛú|n“òFù%3 B3ùv[¸‹á\‹+ -µhd·mïv¦÷vÚö†Ú;3ìcíÍrŠÑ$ð¼ ¼|öw¿s}Ä—j)÷‘pÊŽ¯eI7öy=¦ÈpU µ;Õ¯Ÿâ—5×%0[ÍíNOÆT?K^nøpy5ýòkp[5^-`Ôƒñó`Wn¯(òEænÒê­qäD¥™E”¯®nAµû|LÊý)ü¹Ü2pÁ¢Ä?žYŠßø%|¼Üƶá´‚òžƒÙÞR÷À2äÜ/øÅ:*Þý2ÁŽ25k“›®¨Òêkz[ŸWb¡Cl„ůó~ ~n9+B] st[9kŠ\?UIKô>µ¶7ž_+_iýîGò(ëxlœó¢Áíå)¹?g¨r ð¦ôÕÚM!¬ïäé\’ÙBª¶fMnŠgC‹s¬–9ó(>öñ[©Àni}ìî›E®´ÓäIEÞ2ú¼lé,¹w}ö´Ï+ó´ÒYòù19ìXËóJgçcÿ•Îí×ÿ‹,Ù1¦v&—ØÂNe¯‹:K{ýì%2EG·—Dz ;ùŸR˾öAî溹¶<¦Œökyì£ÊcYn劽½^ÎôŸ]fcùz3Çág,³m'½Tfcyi5#’§ ÈÊ ŽMïÆ>¯¤ˆÑãÎñ’Ö2Û'¤W]´(rç?yJô¦¢ÅüZ°Ø©‘f5~?[‹ßå3´÷úî×)YšqçyE‡óq+:Àæ¿[o'ÎEªÈçK™îNî vµÉÌ:¿—¹Ý'xt»Û·CÊ{1ÒYn½×œ¨Ë¿[ÜB=hXª‡9„´ª (‡±©x5î–3÷z×»q¸ª;IÓÁ( ÝwµNÛ•l¯Ñ,@³Îw·nåiä°›ñ[d’gS#9c[ÃÙ6~8¤ú‰«ZùvÐË{Ž‰L})Gyl…m ׶ÍUE»ã#=ëÛ>ôVë ¬<Ã)Ïï,os˜.ÖöꆿIíÌׇ«m%³oƒüp‡ëI!#±ó7’…¸~´},r‡í¯œMÙ·„˳ëÇVâ—>¶ÕHÿغÉC€1ß®ä ,+DôLîœ,«·Æ­dyA›°^¥.JLMf \û€4·ŽŽWÉ.؉ωvšn=5yœ<ÐÖU—±‹>«ÎMÛßÚúÌäÜwÜɳ’]Þ>îdóxªŽöd1}n“ ²·ÝÛÉ&â  0ÈF*¹ÖŽNòþ—öq2Íãž?ª›ó!z)Y¤àäÑÿ9êrÇÿ¢Í7øý׎nÏš-~ÜÛŽ¾cë†<{2yQ ßH–Ç4b?6ÑÓûƒìx´Íº÷~ÂkÝíöÖÒuu;ãÐáDöÝt½ “fââi§ÅÈ|ìÞ ‡½\†¶8)ËSŠM*\ÌËÁ%^.ù8 ‰+i:\:¸ä§Ž¸˜Åå4a’măœ'éªùiÞŠ sMS!—aðh¸ÙyèF««˜š óÅ<0¹yÒ²‘GúÙÇë£÷}9µ Ôíú0Sû„ÚO§†z™¶—Ýç„Z7k‡wh¨¦³z3ŽÆ-hs1ÛèDåÑmmçI©Ñ6“6GÑá4Ûh}ÄÜdJk·¦–ºbVµ”{L1¨?ΚŽó®t%ÆyWÒY[·y¼-…ÅÙœóX-{,É --ͽ;•FËÐhçsÓvîæmÓ2h’FdjG·xÖòYø÷œ³³ôÙU!´Ví§VF§³á¨ÉåàC}ážå”€!$ïLÉÄS©äÉ}.×NÎ+$­v2Ï„í0‚ÍN^¨‹ðF¹Î˜†j¯UÔ»«.éi -xô1÷s†T†ê»³jÑ»ˆúe–TQ^•$èeyÃO¶N¯ÍÒúæݦ„RþüØ¢ø -endstream -endobj -753 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 752 0 R -/Resources 4 0 R -/Annots [ 754 0 R 755 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -754 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 606.123057 101.433071 587.373057 ] -/BS << -/W 0 ->> -/Dest (cb172-1) ->> -endobj -755 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 296.981993 101.433071 278.231993 ] -/BS << -/W 0 ->> -/Dest (cb174-1) ->> -endobj -756 0 obj -<< -/Filter /FlateDecode -/Length 4883 ->> -stream -xœÕ]K$·‘¾×¯¨³Ñ|_ÀB^Ø2ÖXcöAØCOiÚ¢e@òÁ¿`ò•™Q•U5-i<®®Œà# ~Œˆd昳ÆÏW‰ŒJ)Ç@ç˧OZE¯ñ¯”X]ö_ðýÙhŠÊ›’?ÿÀUÌÞÆp~;ács9;ýí7§¬ë˜³U9küù§OÌ"§bŠQ{•ÁÒÆ ,ïUŒÖŒÂs–XÃG²´gEˆA9y‰1ÈÚ$°ÄpÛ³2Ä°ÑÄ‚&û(°Œ†Æj-ò ˆ&rV௔$ž5*¤l³È‹à™h%žs*D2ÞH¼¬B>I<‚,>k#ñ\çÝe:"¶×9ç¹_›E! -JçøßyÿåHH9÷ yPø}…Ö†…déwX›a?Á%ÞK$½·Ò»ÏûÜñÐ’(íã•ÞIÚ¡¦äygÍÜŤ&„†ðˆØý”T{gw×ìÜïl{Y®óî²"üG„bÙ‹=^XG…ßqaíw†ß}8ÁZn¿„ó4Íð,ìt~8ýö÷þï?ýõÏø‡×Ó·ÿÁ©N­šÓžüûš¦ ~ ´úÝ[übk2ß}ý¿çßÌýp¶8h¬ûᦖæ}ZªóuiòãÒ_Síž^k7/•êµ®¢ÑÒ‹Ç|ý±Öez¢—>ìþrÞëD«}sŸÒ°G›»áOjÎÛáûÖß×fB¥‡ÚMÕ¼¾Ü!zZÄ+b~¬e^jVŧÚ^ëËÔaÕ~Lm×æQ§µë?Ö¡³ºÝ¤ž×1Üg›¢.£¯4»Èk¾ÛÓ¸œ±·TÇ_׺+2Å¥_¾?Åcc=6s)}˜©\ZtÊe‹nÃ2©n ›ÆÁÃØò™&éÎ_†Žèõæxàê¯Çc©öÑL+ŽÕôûE\wtT[‡`q¬Êìü&Å#ÚœçÑ:OÁc4ªQL—-Gip½uFÔ½C!j½.ªé—Æù./‚DFZ]@éß|óŸøŸJeÚü×_6…Ëò·P™ÞJÿX§àjwHtælݦ»†MeòkkÚ´f¦ï?I­½ 9Ù´–å[¯S•¿öÔè]‚4]Ûfzµî¦¢—´n§`1õÊ«eº[?t¹kg#ç¬ë®ó×ûûjr­—Úí:F_Y9+ÊÉÁØ¡«AôÂÑ’¸¯Î¯½$º¼Öð½ø˜%Ò’¸i¦ªµ Tý®7Söic–§)oh¾1ÑÞÖÖdøÖ@5—äçïöåßÍ”e“³g(ž<œ8Îh• èM“»³Î—crœfNP¡MîçYáïgÊ‹/WMñ¥Ocw¯zÄñ˜N›ƒèÛêùætMÎZó1»ÛêÆþZÌoçóÍñÌ·¡6–]îŠà´ofä4<‰Kòö¦éÝYçs"ŸÃà 7,÷Õ¾ÐíæoµœÅå¼]F·¨â²Åý­4š¬–Ö7gk»€ß»É÷€õòˆÖÛ-3ŽÐ£¹ÊÝzs %^«õ&ÁrIåìáòl:¼i¹)4+LðB ñ/î]U¾4»á–Ý~¶MˆÖ•Õ“\ýr² ¢À‡j¬!Ñ6L\BCõo]¶;_µo¶# ]ÂÒ¶NFˆ·¢·5LáZK,´|Ž¶[ùi ÎíuÝø>•6Òf- Ñ}Cñ"ƒ®õ·ÎÀeêcÖÁ²*îQ|˜ÿmÓͬ·0t3ëlÆ“ëõLÆw¹Ö]´¶ýZ¥]?ôÜ3yÔá ŠÝè»·­×sSpãr+äõËd…ò)&‘ø¸eôšÁíÛésX3=ã-9H_“C±<—àÚ·}W†«¦[ŠMNY®Ò¯Y<§{2^ËüÌv0%/ÓqÆë¦îÂv|òÚY~g½ñuK_øw-«g ª,mýšjGõ»”©ki¨iüûussŒ™ïÂ:«Ã­ÔL™×Pç¤aŽ»j¿ðóÙãrÞ í7yÊ^YÞº”H—³Kq¹U`½áãAž°³upé›åãu~™Ã¯Q±_r Ûä]ic`„a"å;g0§DìÖ˜÷ˆIšú·¥Š¶îJ°ðΩññœ•‰.Ù#GûÎ:¿vŒ×=•@æÖh^¹TCäm¦ÊOáø•lÜœBðuå÷²´Ùe¯Mƒ·*¤§ô¬´³èg(}ÝKŒ*ÃøŸ Ô÷e`ï¬ó7¨Þ7 ²Uý×pbk 60D.Èw,Ö˜}]‚ä²à.R± a”`…yÓݲÉ쮫íØÔ’°ûv¶ Õe#:’,bkp1ÑVE²×£Ú ºvXRíã¾S9ëáƒjÓZöEæ5ä,qÖš¸¢¡Ã¤Îë¦Ïžû; "7Å`èü3š$¶güÎ ïÒ$6e,e/Õ~R“äI¥œ\¾¦É’ÊjGS»3L†M¿-èË{Úá(} Šw|+ÙË¡†}žSLö {`Vؤ'5øeð>J¨0»Œ+j‚®¨ùˆ>ç̹œÊLåôçMœ¾³Î¯½y`aðãtâ2½NÉ÷å{ÙL‡KyUž&Ú”OÝ€ÕwÉhk±ïS8‹db¶Ríþù‘(ƒi‚yïk÷åùRGõéH1&D¢4‹Æì=[ ?””Åœ¤–^~á9|9Zè%ý™“Rqp¶|6²?tÛ'ô°ÈÁü5Ëß?Ev6›ÛwëÅ&ùõx.Ë=›ÈO”_;Ðg?-K¤ pXv-×]äé@W½aÛR1zuh‚­¡=+Ôöð~Ø¡.)†—’ܤéÎzßÛAšvp£ô÷R—äE÷gfVí·Gµ&yVÏ*å%Õ·Œáëí4þx -åͦ¼G‡Tîö$~™ öd~+É™wȤáJµi3EÑæ|¥fyéགÄûç×R”i/jùË„ÿpbO¹$~øXyg˜.ädÛÅ ¿øq$‰ÆQ²2ëWÝÛbÚÐ.@(„mÛ’Xþº~Ç×Ü8³áÛh¸¾oÕøY~Ò­+O³/ DÔ fOæÑke*™ß»—”«W¡‹‚í=wâhæ²£ÁQñlœžˆ~OÔSwƒl©Écø=-]ua»è³êÌ4ý­¬ÓtÞ"^NèÖm©o"•ßöÕšjÚ…Ùhn*wªo%û„PŸ't9 įtkš0™?Í¿3e!¦fHÔËåVN ]Nb¹Úšï3{©4ªæ¦ù!Ê0u—ºâÚe:âùE¢ÓáÛ…mf2Ÿ ìsÍ'î댥;ˆ—“§¾ùíä£Pڬо™ÞLtS»œô°—A埖oXšÆoœàÏ*|§2ò”/†ºIñCãjtÂk³“×Jšé¡£Ià›fGEÛ1v›õÅÒïË3èöÒ]c+ꤲ‰sAªƒÇšlºÕu2CXÓOÇÊ™¨›èn}7å·yÈéûRõºwÌdî¾·;È¡cçÛLŽ}Æ ¿¨ËÛÛl¾/ec£-™M?µNÙê¾ÛÎ&â  0ÈFß -¹­ÖŽNüè^û:™´ã^NߟÊ6g·âgøv³×eÎÿB™oðû':ÿW{…ʲ[ÝÑÂvl]‘çŒßšî$óûP±/›‰hi‚ýA6i”õÍú>ní„×4ªëì­¤éj:w&ú¡Ã‰l»é–ç»fü âe£ÅîÈ|îÜŒ {¹tmpü8.¿|§I…‹y8¸LƒçÆ–|žšÄ[:¸d§Š¸˜ÅåÔa’­Ädƒœ;éªgùiž -Ç}M“Á—nðhl³sÓVF1 ì拹a2s§y% ôÓËë³ç½ŠÚ -`j‡öN}›©}`Lí€'S]¹ ë˾縒Ch‹wh¨·å¦µçº“3–Æp-hu1ÛèDM£Z-àçN©ÑV¶¢ÃhºÑz‹±Éjµ¦–2â$j)vŸbP¿Ÿ5íçYéJôó¬„]Yn·íxÝ[rÝò³9Ç1:=Jv_2 ´0×îT%]£íû¦ußm·=Oà!Hž©Õü®äSø÷ÌÚ_#•j?%3bÒxé?¼d¢³®¼G^§€!į”¦TÉÃunçNö’–;™{2üN-çt4ü?È"¼‘SSí «e-é1©¡sû—÷”WB—`IõËÈ¡"¿Þ±Ó›øŵQË/ó)i N»|\¥PÊÏÿîÊ£ý -endstream -endobj -757 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 756 0 R -/Resources 4 0 R -/Annots [ 758 0 R 759 0 R 760 0 R 761 0 R 762 0 R 763 0 R 764 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -758 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 713.673057 101.433071 694.923057 ] -/BS << -/W 0 ->> -/Dest (cb176-1) ->> -endobj -759 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 373.781993 101.433071 355.031993 ] -/BS << -/W 0 ->> -/Dest (cb179-1) ->> -endobj -760 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 354.834971 99.933071 336.084971 ] -/BS << -/W 0 ->> -/Dest (cb179-2) ->> -endobj -761 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 335.887950 99.933071 317.137950 ] -/BS << -/W 0 ->> -/Dest (cb179-3) ->> -endobj -762 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 279.046885 99.933071 260.296885 ] -/BS << -/W 0 ->> -/Dest (cb179-4) ->> -endobj -763 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 260.099864 99.933071 241.349864 ] -/BS << -/W 0 ->> -/Dest (cb179-5) ->> -endobj -764 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 199.152842 101.433071 180.402842 ] -/BS << -/W 0 ->> -/Dest (cb180-1) ->> -endobj -765 0 obj -<< -/Filter /FlateDecode -/Length 4700 ->> -stream -xœí\[¯$7n~ï_ÑÏ X¦$ê,ü°À‰ƒ,²È y0ò0nŸÙEpÀ›‡üý|T•.UžÎØël2gN© EQ)‰]öLøùÊâ¿ÌÖä\RäóåÇÓO'2)þÕ›‡Êþ#~S8[âd‚-1‡óBðɤ\Šç÷þÛ=ÎÅßOÿö›Ónëس3¥DgÃù/oÂboRN‰‚Â*`‘õ -+“’ QaEk0ÄR4ĉY bpÉAcA v.+¬ 1<'ë¬1\²¬± †-!),KÃ:"•AˆÙ;…gƒ‰PzÔxΚ˜‹+*/g“ÓxÞ›˜Ø«ñŠ‰1†¬ñ²„BVãÈ,3i<ÈÂ>'!‹žTdq9y— ‹ƒvŠÂËÖ•Ud¡HVãoBÉÌ*¯€G9å#ÏQ0!{OÏZ`›^åaEa’bRx²D -EåA–àÉi<Y?8|ÊYY­NV·'o5^€,Î%Vy“¯!€‹…RÔ +È.iàR2\k à²7œ9k(àr1œ’×`À•`8–¤á€' ž³x‚,£†ÞBÆÊUyÅÖ°À;È‚xY{ <Жa-xÀ-S±xà-“x.Ä$ <×çÈx@.L©hXà¹Ø©aèb¹' ; ×€‡e«¬i«†BãwÉb†5p—( "pWܦ†1 Áe  Aì4,ˆ™MI)jX ‡OåEƒç aA" KåAN¬akS|É$Y¼s$Y2hX€ˆÀ‹è[Ý@˜‹º?`È‚HQà ™É…YÂ"öcЪƋÎä¸Vy<›4,H‰ \±Õ°{)“±è5,À —4,€àYÖ° È‚¥¢a\‘É‘¹Æ³A¶†ˆtÁ³VÃ‚ì ¢A ²ËØeFÒ°{I¨Zã1É>3kX9 5,À2ÁNH©ò `BÃÌl¶ƒ†Øü`³‰mŽÊƒ,>°†9;Ùnf r†,Žœ†°MÙo& -AD|ŠFÂd ãàiXPlÆ~Ó±†X—²ßÌOØob‹§¬Ûâ£ì7“†…!K@ˆ¢ò ‚¦¢¬[`ö›@'!‹+¬aA‰Qö›YÄØo"¢W²ì7“z\Ü Ȥñ€»ˆL£†xÀ~‘­¶·'oHˆÏtf–gVO Ðà[Õ#öÂ[%õÌ€¾²GR &v–9ª§p«`ZR ø‹xŒ˜°œ>ÍGTæà3s’A, Ftß÷£˜¿Rl9ûËŸN_„ó=ÿé¿NŸ~SOÊÚ…ñ{Ž²±v°/9vÁ|¯Lmf!(쳬T¾TIÄHeˆiŒë±›ˆ g ‡[ÎýƒÝ - ™Ej‹p¦Ž7c³`1[>êÒ>_é I;Ô${ˆäm‚)MjBà #Î]Síƒu¾¸fç~‡’4YŸ®óÅeEp€ÄŒE°éWò^á»Òyùw>~8 -y\w¿ûpB ï¨Z5§IOÞ ².Ÿ?üxúú÷øçü×?@ >¾û-\DƒØ€´õï÷ß|e+3ð1/Ÿƒ]ÿÆõo£"ÎKYØñZº©ÏúZ–¡ð'ü^Zo?Mü¼´_ëK¹²þ£ýÞO2Mý7Éy™ÚÊc,µ~k[dÊßüûù÷i¶ÌšýnîðãVq}0k§]!"”_…¸l…êeóV™­ú¦>ÝP€ýá:OêÑ÷˜ÜÒzÚ*˜¥ÎeUJ\µg5Èz`„'ð[«|hJ–>Å.rÛ‰^BÁzÜxh§Ž±Mp¬:¹%GL×ä 1–:Ñm|¢s'ó¤Ê&ÇΑp£´-²Iû!/õ‡±O†Y–ö«a}Zûú¸ö‹:üq|åѤ»ê¾Ïûú™Ö6§….zÿ­¿ÚV·ÁµÌd{µ½rS·y?þ}_¢ù=.îuî×Ï›E·€O_üßoe¶¾þnl|ÐÈ߃sr+¬£6†e ]¶õáã:'aèéŠý:’£CDÞAi¿Éóûpõ‚1"8‡½_Z‚‡ /!tµcLw5Ï×yÉ®Æ8ÙÄclÆ'›D²T°‰=¸°Çþè¬ïòn´ñõ?|ûíßýý¿ ¿Ó¦¹.Kùü†_7Ldoì3g1£…6©^îñZ¼†m {”XÖ4¶Ø4üt:«þù:¿Œê¹õº¨¾?Zº¼1ßR=ɬB7y€æÚÃòWxš—jÞl¥çñʺj[YÞ¶wuRÁ9{ îiÿfÑÏPú¦4o°­-¥œaF˜.²½»)Ùƒu¾DˆúE dØÀ%,h~0— D·ä°¶Ü'‰ñ+m L®{µ;.Îì o»«ÎÃOfr§WÏú`uJ;Íñ4“ä29«-&¸Ÿ2ïQ%ût¯6ö%„ ´Ú÷ûÎu/¢WjK8Ð`_dþ®†¼ÃÇ".ôŠ†øm·ó®þ´ë3µóÎh˜àˆ†>¼¢I ñ6 ò!MÂ!븭ö‹šd}Ô”kš[ëÇS»3LÆ]¿«RoO»;Ê"BÐÀN³—»>¡ä”Ý+À¬°aM/j8Ê 5|SÒPa#7¡Ó ]´Ød¹I”›š\O×oãôcuþÚþ{ÈËBõŽ¢ -Ï#ÒÞ-ßËn:ìXÊ›ò<ÑÊToŽ0’Ð6b?¦“`Öjµïö-)'Ó$æ}¨Ý—纯 x§=LE€I•fј{ÄÅHÒG‰œJÖZúø ÏáÇ{ ]Æístñ•YKûŒÅœÃ•±†·Éá+1åp7TC –õ®Ü‘£‘ z]Ë÷äŽö0\,Ö,7FÙé¹â¹Þ÷¼B0’šS°Ü9IÖ§•=Åmðz¬Î¯¼¢ähÂÙ9}ß÷ü²D„`ãÈí϶ñ‡¦Ùb3D6ï´Ú¯˜'?hžsœyìùrÅD²\Ê’éÄrãWâ]y¬Î¯ÇDŠì›²¤lª&Ò°¬Ü›Ú@&zÊÖïš|Ì0&³:Ôn“Îûýqºo®hŒŠdä¨R­AäøÖiësl)ÝÚ¨=áµ-Û·'¼öózŸ½öu½?ûÀ{-Ö´~–4„Wä…”)iµÙ -¿º½Úûò«~üqï­Ìå#\4à‚çýÒün–J…+¦t­·t,IÀÕƒu~5p%;M‡¹âÑ^€«m“ÏÂÕ±ö—+MªíùÊ3¥´ö… ëØò3õ‚î'Ⱥ¡û!ëyd)òÄÌðö°ä•Û€ ö†-L—”ôy¹Œ‘ÏîãÒ3â"—±ïW-îsld‚ÂldÚȼ0'ˆk@t©0 Ro«X¾Sà­|F»·¿vu%Ž)L÷*!_¿ê<€¹¤ž§šÍà|͘«–·Áü±:m0ïw#ê%sÉY?[±ÓýG<.ìN[}>»'}ùTç˜- Î2´MɺY±v_æ›ÇÍ+0& àë浚Q½}o¦4gøk·üÇRù2N‘´÷]§UßWL1`×ÖÍŠ ¹$ ë¶)>Xç®)ÞÈÉùY,2»[IÓAë1Ÿcµ„î—¡–k°;¦®wëV)ï•mñ.¼8œR©ö÷@&MtÉ˱ØÇ’”. 캻݆=Ÿ› ô6Gó«Õ¯Ñ­ÞœïÐu×ò}Úåõ&¿ãz™¼<ö¶Î1Yi¥ÍãWj(žgÅ/Ws4.׉6yI}ÞÙžÍ]²e×ÆŸ’µFž‹2uš½RÏÓè(ui×-ô3ä;)$éoB±×òŽíTTµ4峬r·<¨Ocœ"sµÍ9iM•79{M^c~-/êØö/™%óV½ïëúŒcL‡õ©%êºäýx›ü›ÄÄI-·WûjùP›¤ÆÉ«¶qhc¸ô…É;ל®ËsÍ¥"ê9cÚÚ«™ e•?¬ÑO³µöü¶ö­à¡u7u:»25y§Ù!ÕÈéV»%íÚukîY³‰®“5Q¨åö¼±Òðm;g›ùÔóëÄ®ÝÛUÞš—××>ê^æyië¬ÍÛe‡en‹wuM_&™ž[¢ë-[Yó!4¶£ª³ÝëQõ1e'ï~}ݲ;ì·­ù×Æ1­‘i>¿·Î㕵ÔÊôÏaꋶksSîÑκóÝMcÚÏa³ÅžÏ¸ÇÈËŶ9{À‹c“hüÆ‹"&ùë‡÷ýÿÈé í–½vÿoæyË·Ìüˆ{Žíüa‹ýÇõKw)œåè2˜‹+eú_?#¹[äΑHÛÌ¿6u¶»lÕíyš”·mù|®)ŠI¾ ªíï÷qU=Æ°}óñeòdkú îæ8·ûö·ámzN›§²æòìâ†öyÔþÖXa¶éMûÍMòlü}Y2[–1¦ñ§S¬_´¶õesMnÌònãe^ÊYRç2y¶mÚlU´=_©YßaöS/%ÿƒDò¦ù–{Öùa-ÿSðOr1›3¬¯ ¢Jή= òûI^AÅ -™yxÔ;d!®©·…ý–ÛÑ.'+Q×¾¤†ÖËûñÕ„ÐF#õC«&_ fj…|ýrìÒ±BDÍhd=»’å5^Ö§ØE ±É âhç²£ÁQñ.ÙzšˆáH¤©»AvÜä±òÚ‡®º8ˆ]ôYuvšþVÖ¥i:o/'të÷Ôw•*/jM5í:ê4? U:5´’}B¸Ï“BºœâWÔšfLfçO³&¯`Xˆ¹÷r¥•SH—“Znm-ô™J½TUKÓüe˜ºÏ]ñ -í2EºIÞK8­òíã–,OŸkʓݸñr’ïÈÈ倫Òa`…fz3ÑOívr¦a/ƒŠ­Ïª*,ÍF“/°Ëÿ T„Nä©,w“’—PŒ§Ñ‰¬ÍNÞ*i¦ÇŽ:¬i@rö슶Sê6;ëK¾9~,/ ÛKwm¨“Ê&z*M<©[²íV×ÉaM?+g"5!ÐÝ ‡nÊïò3ô¥¨wídî¡·;ȱcçûLN}Æ­¼Q¤ËÛÛl¾/Õ±ñž,¦Ÿ['ƒì¨{»ÑÙD‚yÀè{%·ÕÚÑIÞ’Ø>ŽE¦yÜËéϧêæœNnH£·ò‚¬9ê²çÿF™oñû'>ÿS{#ÃâÇu´p[7äÙ“Éëóƒdy½b#öe3O°?È6²¡YC÷ãÎMxÍ£: `o%mGPÛ¹31 Nd×MW¾­84n/;-ö@æsçf8ìåÑ·Á¡/ïòhRáaóàùá’ÏS“x’¢Ã¥ƒËnªˆ‡¹Q{ÞW‘c[BíÐÞ©ï3µL¨ðtª¯qûØ}Ž¯×òmñ õ¶ü´ö|rÆÒ¡of¨yT[ „¹Sn´M§ÍQt¸­·˜šLq­ÖÔRGœU-¥S êŸgM‡yVºÃ<+ñPVÚm¯GK¾@a6ç4FG£d%³B‹síNåQÒ7Ú±oÞöݼíyAâˆLiT ‡’/áß+kG}ÿ‰VªýÔ“›Ç;Œà¨%wÒùúZjÊ1CXÞ,˜8OG%O×¹}vr> -endobj -767 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 653.523057 101.433071 634.773057 ] -/BS << -/W 0 ->> -/Dest (cb182-1) ->> -endobj -768 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.576036 99.933071 615.826036 ] -/BS << -/W 0 ->> -/Dest (cb182-2) ->> -endobj -769 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 615.629014 99.933071 596.879014 ] -/BS << -/W 0 ->> -/Dest (cb182-3) ->> -endobj -770 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 558.787950 99.933071 540.037950 ] -/BS << -/W 0 ->> -/Dest (cb182-4) ->> -endobj -771 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 539.840928 99.933071 521.090928 ] -/BS << -/W 0 ->> -/Dest (cb182-5) ->> -endobj -772 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 478.893907 101.433071 460.143907 ] -/BS << -/W 0 ->> -/Dest (cb183-1) ->> -endobj -773 0 obj -<< -/Filter /FlateDecode -/Length 2689 ->> -stream -xœ¥šMo$·†ïý+ælÀt‘Uü‚ øâ FÉ!Èa=XÙ0f¬sÈßÏËîæÇ̼+¯×$5«Èfñ£Vqä/‚ï¯=~󮔚“]®¶›¸_{»Â®þ;~r¼x±ì¢¯©Äˇ&Ðìr!§Ëmï‡âZý¶ýó«í?÷mü%¸ZSðñòëû¦2u¹ä,‘¨*T╨bt9‡˜ˆ*y‡!ÖÊT0#f ö¬Ê0Ãj‰L3,„BTf¨e¯Ïª -3BöÆT0Ãט‰Ê ìðA„ê`ˆ˜i :]¤'¦ Þ¥RC¥º ÏéT]Êæ£gºêRJ±0Á–XÅ3]„-Ñ› ÓÁÓ’™.ÁM*T[BÉÊt¶ÌN%º[°» -ÕÁIâ™®ª‹µ˜Q]…NJ.Ϻ ÑÅ¢*L罋؛Juð(,RÊD`K’X©¶D•Àt -[ ßLNDÍ¥o Í»UÔ3]„-!d£:Ø‚Åg ¶HN ð ׸ÄrvVƒ1„¢ÎŠFPª³œ•a Ôè,ÕÌ8 â¡ ž@¶DKŒêa‹Ás©¶h5Æ ° TaK0e,PÐÖ°[ ¸5©ž±@Á[Ÿ À…™ÂX  ®–dŒ -äb+UÆsìd,P@îž ÔÕ˜c»j%1(¸«æ…±ÀÀ]U´¤:Ø‚ ÌX::õE ÜU/™±ÀÀ]¸Q`,Àª:ìÃÄX€½âB)ÂX`àn(bŒ#€ê²ƒ*c»!bìäwC”ÀX`àn° °Qø#4wÈÃØÀ2༇.(c^æÂÆ‚îúŠãø4hå|©‰± ‚»¾`ù¨®:Ÿ1ãÄo1ËΧT "¸‹ƒJ "¸ |Æ0ÞáD ŒÜõŠ“šéÀ]ª0DpqCd,ÀvpcÎ7ç[—ꪓZ -cAw¥ÀX€SØII¨.;l$ÏX;AÊXÀ] »U ¸+†°‡ê`‹–ÀX€¸:¸-ñi«NHÁtà®x¬0Ó»"¹2$p·›Œ)7A(Œ0Ð! -Œ©˜«9'Æ‚T:„|T—Ê‘± È¢:ØbÙ X»ªµ0ä[4Æ‚` BÆD®zDß4=€-Ø.4?0Ø‚H‘±$s¥š1ä˜aV™.W2pMu:Ÿ r6‡£Ø3 —rNÏXÀ».1À è¼1ä -[à*Œ8Š\ ˆÌ™ÎÃÙŒˆt¡óž± Ø‚h± „‚,3 cNhä’˜j¦3iyfa,(–\  à&È4AJªƒ-ÀcÖ§%Û‘±É’M¤9T[4cA)¡¥›…± Ø$0`o¶|33T-ˆø ª$×ÂdÆ`:Æ‚ê òÍ`ŒðË–oÆ‚ª‚|)ñÛª©å›™± l‰Q¨¶ hªÄoÁ$ä› Ó%ت1Ô”Z¾Y f ßDDO¯JË73½.wc™˜ÜEdš P@¾‰È–åöðÆŒøŒ+KË8 ½1 7âl¥Wöâ´ÊôÎ@ß–#ÑK˜‰Ì²$zk€cJ/ôÚ@À_ÄcöÄ„ãöi½¢JØgfÉmnaˆîGÎ1«é'ª÷c¿þ´}ó‡ïå§ÿn¯_í7eÇ÷³úÛ— ~pÜÁa_Ã5ÔRK»±v5†ûòaûæ»þö—üû_^·ý ‹ŒÙ«H,øÁ3€'ÖžãQF,å,#Šð7»ºýo“ù£ &f¯o­îõxFø-á=~Ê©ËGëý¡íõzô+g?»Î›ö>NÙ®GyÆÌ×QÇŸ¶/å½?;ÿ¦ÅÖs öîw<êííǘþüïËË÷Ÿ˜ÓºÎi›ÆñÊ×ij“Ý™Û^ÿã¢ñ^oÛêùÓü`§üõK‡q¶»Þ/ç^/˸.ÝÃÔõ6ûs\ú’ÓÖsÚïêå·¦®=>ÌÝ2¦Ç%ì[±oX¦}cœ§-þ\²ï^à(‡«ÀÙ)MÒk¨uñ¼æµpÚϨÒÆëôSÈà¼wæ~zöÞðâßpQ¼²Ý‘ãÅq„À´+ÌC=¨ylÀ°µ/g;íØ2¾žuâ\ž6½mÛuoÙ—ñýQ¯Íäî¥iÙ‚õ‡‡-ÒŸ—~ïïÝôVËs‹÷÷­¹Øs·´­ÏØÇ𴌷´‡G~O‘‘j!elwèà§Ó¶.õÒB2Äkæû²ù}¢ýå-÷O>ŽZç‡Òî'Zlz|R±Îúy?láL›ÿ‡²‹ÅGñmkÇFÄ8OÂgŠ›ð|”ñ.äñáAvÝptL„h™ü³¸^œ?Åíò½8=Ki˜‚ ¥á|¡­uç gÃß{•EŸ…²t7ÅÁº=¾%kcêÒÓשóËò÷º!/Ëù–ðº¡[}”Þ¨´]ùõWõÙEHÔeºU‡4öšcAl¬]7"üZú« ‹9ô˪µÄé–¾‘lÔ«½]7Zï|[+•G­2›Ö>óÓ”¹ÕµŒ‰'²}ŽNinŸ&.>"ÚïÅ©­@_kD±sß„'áuCoOâÛ3©'+$ö­· -uyï™ûeJìŸS×ì²v¶ß*â6òìÈÀ{³–:ÎÒì¤ùæßOÒ*Oƒ:ÆfÀ[žu—éò9=»Î—/Bê7èŽÚcÆî¤Ë”-ò\»y R÷b?vÝ7„õù¬\…Ò@wSÇV¾Ý‰§q¸*6{ï,ùe»ÇñÞ)Nƒ·UœÇŠûv0ìï]ö|^÷ƒÍÅmë—ÞɧÝìlNÃ…)ž½íâî­ƒNí³Íþ8Œ¸×íçm?æ‚ÆPñô"}]£.ùê|Ÿ_6»üµçQÇ9dÐ" ¶Þ‰×“¬}Z>SÜ>íÂá6‹0Ø‚ý)öeÖ}7Œs<„…×6›Ë{¯éAýЮÂ8çp‡±uƒ×efâÂëÃ,Ž@æ®Í<°¢öÁ¡mx· -…u8(–©Óy$_–W¢ÔªÎ#Z KCÖ—¢¸tX§ewf¶ ¹v2¦¾ÙoëRhëk),aD+êÔÙŠ¥ -B`] ë‹Í¯Ö;{ˆÑ_ì^xÝO“S÷€&hÒÛ*kÒ<.Õ½˜î‹ãÌÑK+tç34Þ¥‹ïér¦kÌÐÂî -ë]¤e6;+ĵSë²»NûA1p™tÙxcî6¥³YŸ–}Ä…ÎR1Å”þ¼Ît\WeLb\W%=Õmïí'Þˆ–t@qÝÎyŽNfÍK"Kkë!µYS»ì¹o»ï»Ÿ¶—e6 I32•Ù,>Õü"þ}‰ïmn!X­þ½ßŒø2oqP›Ïtÿg2))!Ö>ÌV–«’ßÝæí»“ç’~w²ö„åp¦*Ù· ä&Ü ¨¿­X^Õÿe7«B‡ôû&à“W¸üòvÿ¿|D£˜[ªØþ L[Ð[Ú§×Yøeî~ Ô~ôî -eÿþ?>56> -endstream -endobj -774 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 773 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -775 0 obj -<< -/Filter /FlateDecode -/Length 4239 ->> -stream -xœí\K$G¾÷¯è3’‹ÈÌÈ—dù€#ˆ‡qÛk Í"-øû|‘U™ÕÓÓ3ƒ‘wG³Ý‘Èx|‘SUîLøùÌá¿Ân)¥æÄçˇÓÇ-9þµ»‹Æþ#~s<;â¼DWS‰çByÉ5úœÎ'üwu©›?žþò³Ó?ö}ÜÙ/µ&ïâùŸß‹Ã’KÎ V‹\0X1.9û˜ Vr –X«Å‚1³ç#+C ®%Z,ˆÁÞƒU Fàì‘U!†ÏŽ-Äp5fƒår8Odò 1oð\\”ž,žwK*ÕW“—ÁsÙ[¼–”ÙEgñê’RŠÅâ1d‰•œÅ‹%:f²x…CÉ/A–™<ÈâK/CíTƒW ¼«˜<ÈB‰œÅ«a‰µ0›¼ -•\Ž¢Jp‡Œ=³dYdEX0²ûQsÌfá‰fëùØ?¿=ýü›ïùÛÞÿ¬”õñ(A -˜†µ}…µ?œ ”‡Èk“"?‚\™^‘`º·¸¦kb;µhÄËI‘‘åœxì㞎ª®4ºšÎ!×ÜZ+Ùõ¢V¢È' ¨H¯È1Ž¶s:MLsÜIž+yTÔ¹è9ÕÔŽ¥ßËé/0ôG=ÆÔ&@%c[“®Hwá{Žšo ‡˜2×ij‡Íe;ºD ñC·75,-¶ÖRF¶¦TjÔËUJ×äæ±j‰Ùv¡¬Êp{²TϽ-¾ˆ;šä½ i:’Ÿî,c:'Û¦áBd»)*h}MŽiz˘Nµ %Ó…u.zN5µcé÷ò)pÿ'+™ºü;¿ «µ3×ðÁÈ'ƒLëMëþSð} -¾ŸXð%ù—ÏÇ//>%°^±RÄÞ³?ߧàû|/ >3È^–r>ŸïSàýÄï%)'ŠÂ%p’?”³œ»ÆUÜLK+ùìëâ“—å­ª|MŸÛ•¨­ûÈõ<¾Øu© ni'¼›7:Ô½äC2e}qŸ7’uê(F’¿'É9êÝz½³Ï›ëUÏ{¯^ïìóæ²ÊߪBNì®ìi -ù\ãg¥»+¦ŽÆ—Ó¼Í{¯ñïìóæ -ÕóÞkü;û¼…¬Ç³É_¼;!˜Û €ø¬¼0—ö§K¹£(ËŸ_Îï>œ~þËßÿá·þ=Æ|÷þô×ω¤€ äÚ7Dþ¢Xˆ×,Ÿï×Ϙ¿øÛùÝ—'‡Õиyð³y©læÕŠL¹Ÿå‹ÏÜçÛwÞf,+-ºí3mŸ¾I"máÄ_­tè¼]è[ÛÆïcú«9.ø%E/û¹!ÞÚ¿ÓÕøcl‘ëAµ¡MSnöim/sÜ&kšrjù›,÷ë¸^ë˜ë*—XLÏ!ãÓWÐóÕzÆÚiê²Y\dºlÖN›IÏ ¯Ff‚ës‡ÍF1mÞó~Ï}}'ù‹˜Üuu§é¢ë7µuÜ’#û§ä ¹ÑÿXŸØ‹LÙäî»DŒ`5Æn¾zYõÜäÜÖÛÆt«/t߸I[»4ei¶»¨1úõÖ‡Þotµ¦½ï+¿¿î;ü«ûßUDÛº­×ë_ãa?ŸÈÞõ*ë±õ¬ãqÒg N]Ž¶ÛØÍŸã\ çãzö~¿¶|¹Û^BßÖÃI­q7÷-]gêjàƒÆ†MöôB›>ãKæàÇnÿtÛ'äÎã>†´•ës—í3«XÊ·ùÎßÔO8èçÆzÄv·Ö#ü.ì!.lx}=&¯íš|îoMoÛõ°oó|µñ¶6q¿ËÔx›/sVzÞ|œ¯ír°ëM}Ń¾”žš<WX·OšóiY»Ïw\BH>ÊÞ‹ß÷ëg>r7:d—í3o<ÚxøðéóÊšì+‚»þa¥5¹ª’ërŒãŸòáè¦.8|ŽŒý{ê\àŠ¦“®¿t…3Ê?âEͧ}­xè±õ¾˜§N^ç7utÀêç⧭¹NßÞáàæGÍ·º½¾¾½^î×Ó:~e·Ú%^ùrÓ—Œÿ´^oé‰8ýjº9—«yüæ‹ç{þñËw­Š£4ϼ„ÚJHyÚÁåP|9; É- T•ÇKû¼ªòØ -ŽÅË}oX/&òrŸ-‘j!¾®B÷YWÕŒKâ·yǵ(ùÍ—_þê×RE‰¸ÅV*ð¥§îÛTª¾áÉ kZ¨Ê£WötÎmàS¿}90dè×ï÷×p·Mq”d¢Ó/äfáìBqÙ,Å,'iúD¢ø”ÅvfÖÊPy¬ En·ƒÝt ;ûüh(ÂËKŽÕ§;÷Úˆ¯øäÄžäƪ¨ öGò}Ïë5ëVGzå4—½S5Þkîóý:fmÛÝcë#.¢êÝ!©±ïw^D¬š¢ó¶ón»Gì™”BðŽîv•x¬Âä™–*w_MÚ,û„Ó§vc`s`yî Þé¨>ãôwöù>Ça?„ï'¹Ïøiߧ‡í7(ß×çNùLVô°÷]¾:×îðÝqt!÷ˆûš’¯ÏœH©³¢8åü¡Î‹Ö\´çÇsŸ¡³-§_ÓU½»—«å:4é‡:á¥çGwŸ e§üW­+¿ûæYT×wÏ—”Õ™Vç¯ùÜ6þÃìÿ¶gO9²œ]+DxÝÙÓqœ—=a;xJškyÝÙÓqìŸÖÙ“<0°_ÿu-7jr3iwîÔëZ¦ð°­-ª±ýZ¼ý9’ÿô‚ó¡›z ×zûÿ;Gº-ãëΆnê,|íÿêikÿ&çC7õ”¯õôÜ9R;Ú®=d _Ó½Æ~4؃¯¬&O‘¯ÄÒ‰G»ÚÛ¤ËÉl·‡¥òhUf×Ú5?E™®ÊP¼A»¨Sž,¯VS1B¡Ý“åaØšŠò ^N‘G$Nr»‡÷Ø:N¬ Ø]Oƒw M™Ô\ºªš&ÏàÊÿ+TÄAäi_—’çèçÕœDbs÷JÒô4P‡- 8γ­R—Ëyø¬Ö—+d´Ð­‡ÆvT¥2Eϵ‹' 5ÀcOvÃëY ¬ëg`¥&RÓMr®ü¸#O9ãÕHc²ä”»Ç1î$§šœ‡Å¼aÈ;ÆU>߉—¶±ñ5Y\¿ôI&ÙÓØíædŠ8(Lò„ÑÇFîÑ:ÐI^ôÖ¿Î ³vÜËé»SÛæ|ˆ^’圼ãGg]îüo´ù¿?ñùwýæóu÷4ÐÂlÝ‘õN&o„+w’å q8ÂF=+ØŸdWfÛؽaìãÞ+¼æÙ&°÷–n ¨\MŒS‡Šì‡ëz”fâ âåJ‹#‘ù¾¶™özúâ0M×t©p¡—ƒË2yanÉg5$®¤éÜÒÁe¯:âBŠK5a’íćԓ Õ‹ü¬Md.u¡Ò¹ “Çs›ÕCwZ[…j‚8è =0;=iÝÉcýêðúÞvßDN=„: }P5u,L¨ðljh—i9öœÐþ$ÚƒwjhŒTì…‘äÌИ©ï.´*j™Ý¶QOʶ›´o.@£N#æ.SÚºuµ´SKyä“úÖtÔVJŒÚ*éÐVÆí;ÞÈ–ÂH€¢vç> -endobj -777 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 488.554475 101.433071 469.804475 ] -/BS << -/W 0 ->> -/Dest (cb185-1) ->> -endobj -778 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 427.607453 101.433071 408.857453 ] -/BS << -/W 0 ->> -/Dest (cb186-1) ->> -endobj -779 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 115.110432 101.433071 96.360432 ] -/BS << -/W 0 ->> -/Dest (cb188-1) ->> -endobj -780 0 obj -<< -/Filter /FlateDecode -/Length 3952 ->> -stream -xœå\K$· ¾÷¯è³Ë’H½Ã‡’ bÄÈÂ99¬Û;6‚Ùëò÷óQUzTû1³³Æ:žAÏt‘zPù‰dWµ;Zü~îð'³39—ùxzwx°&‹ŸÚbsQÙßâ•ÂÑYN&¸s8¾%“Jð)øsv97<üã³Ã¿·}ÜÑ›R¢wáøË[a1™”S²Aa°¬#…‚Iɇ¨°¢3Xb) b„Äž÷¬1¸ä ± {ŸV†ÄÉÑžU †OŽ5Äp%$…å,äpÞZ•A,3y…炉PzÔxÞ™˜‹/*/ç’×xD&&vÁi¼bbŒ!k<†,¡X§ñd ŽÙj<È”“Æ‹…"Y•Y|N¤ñdñÐNQx²Àº²Êƒ,6Z§ñ -™P2³Ê+àÙœòžçm0!Yçœ °MRyð(lRL -ÏC–hCQy%õ ãWã'¥œoõâÝdÉi¼Y¼O¬ò 6_C!‹MQƒx\Ò0À§d¸xÖ@Àg2œ9k(às1œi0àK0KÒp€¬Ï; ÈB–ÀQCr…á¹*²Pa ÈC4а€²x& hË° p˶8 xËÖE € 1­†Ä¥YÃ䔊†Ì%`§†Ð…»' ¨K!z °Kœ£†Ü%vVÃʃ,0` €Ž†\& ¸KÎ& ¸ 7ò`W ì0jX[1>g«aw}¶¬aw} ò’’† ÜõKÐNnà®ÖkXÀÀ]Ï>jXC1À«FÀ]@HÐÃÈ€Õ°ç=xž4,À`Æ#,Ñ° w]Áñ§ø4Ðʸ\¢†¸ë2¶Oåã4®ø-´l\ŒEÂÜÅAEà.à3kXŒ78Q½†¸ë'µÆî:_¬†¸‹¸!hXs0€†8ߌ³0]•WŒ-9kX€»€%¯aNac3Då%CrŒEªaAîâx/Dà® DDà®e„=*²Pö ®n«ø4‚Uc=Bãw­Ãk<வ©hX»rljX“|Ö°DÁ^˜ٔ”¢†±Xðò©¼hp4,H„ÈRy…kX€ÀÚ*YÂä! y¯aAò!ƒ†ˆLqˆ¾Õô²À\Ôü€! "E €d&f RˆÈÇ U½É p­ò2x.iXÅNÃäR&Ãé5,À —4,€à9Ö° ÈWÑ°G‘É‘¹ÆsA¶†ˆtÁsNÂì! ¢A ²ÏÈ2£Õ°'4rI¨Z㱕<3kX9 5,€› ÓRª<Șаû#ÉvаÉ’M¤9*²P` rö’nf r†,Þz `›’o& Š…,ˆø4,(6 “5,Œƒg­†Åe䛞5,€_J¾™5,(d‘o"ÅSü¶P”|3iXP²„(*² h*Šß“o4^„,¾°†%FÉ7³†3o"¢WKYòͤ– €»¡™4p‘iÔ°È7Ùj¹½ð†„øLgfÉ8³Z1°€Þ€³U-X`/N«¤Ö ,ÀWr$µh1‘Yæ¨V p¬‚é¬Z6°À_Äc¼Ã„¥ú4—¨"Ì!áÌÌIYàŒè¾ç£]h¶ÔÇ~ùéðžǟþsxø¬VÊÚ¯ƒñGI¬‹¸/ØnÀ•©Ã1LÄÚ^Îò=£‹ˆÊÑ×’›ˆˆƒ‡m9ö7n+0ä‰B™º.ž§Ežà°QUIŸÚåÅ%… ^ˆÜ¶óª2Þj|S:’Ÿãþ*äØÅ ã:ï½;gŸWè<ï½{gŸ’uÒÀèA–Ÿ ×ûú¼¼^§yïÖë}}^\Ö˜äƒ[Ê=Nu«ñ :Õlÿðú€¨bùD#§mÆÑê#àðxýîðÅ×ßüí/ß}ƒ…¿~8|ÿ%Ž:¶6dkqDX>á}’Üt‹Áã”ç²ðø "—¿úçñõ«ÃׯëÚ©ƒ^–…ؽÔæbàSe®D“ºžØåY[º* Gò­*^‚5œÀH³aDçšã6ë¢À~é(tÈ„JûêmUäŸ_½úãŸþ>)ŸLpøês÷¥(kù/Ê«ô¸Òy¥'ü¯j¼8±G†VÅr6±Ì%$ã†2¹Î]ç{ØÊÁ3ïíÚ>oyÜ®¶×UÖ/·ë~¬o1Žµˆ[y§I6;Ý̧›ðEËuR5°È¨›.Ì6ü€‰´"Ýb¾¼š³h‰ÂrâjæuE‹îǤ^êÂE*0g“Ö½`ò.•fÀòÙ ¬ÓÉÛ«6gŸAŠbûåŠíÛ7ë‹&Û_mEhb'ÝfÒD§­ív›‹g×nö™sÛFí$¿œ,É“Ô²+Õ’\»Å«ù‰¼+v¶²5ÿ «…ÕõˆõýÐ|}€é†¾¶Åõ%V+VY6Oó¤‰ÏëûÄcœ:OX|¯Ëd÷c-{s¶/qÈÔÛðJ_|äå†Y¹ß·µÓú -ÓÞ¯×u <¼Uæµ?´ÃhÈÜûÙmûêݧÏ2IamãÅ: ïžÇ‰$Uù¤k?Ž]×µè©êêŠ$…3];Ö²ÑÇi¼Šlò‰g´Œ G»Úëi±“*çºÞ:¦[l¤íqmûvÕÅ*S³«n ‚€Ê;·Å‡•®‹ýO6«Øñ²ßÍ.›WuÎ×Ïk03Ï·`Ç:_¼¤çu^ôqèãÄ?Oƒ¾÷Ñe ’Äfý­õœ¯ÿaá]’§öñÓ>ž÷¿ O Ü<ëzvn¶‘«ºM纽hKÍFìä§nœÀMÎJS1kÆ«Uoñ‰ötÃŽ»=…5fj¶pÅk¼Ól)ÊõU}•¾â…)*©ëLkìv¿Äh—×#{xm=Î9¯¶l•1yiWçt{_o6Øì±÷‰‹î+ÿÍÚ&,ã/2]Ó»žÜjëÓYpnC.ìÿ¤§*Ï›3œë;Æiúëçbó ¬ƒÀóa‰0ýÃòŸ¤_|‹Ökèlj.}07­±ÃŸoϼ*k÷Ç«ºÚáûå˜ÁÞ/x»?O8ò$wóìç«úhº‹ÃÏm#4ìštzɾ›_‡ù[®ê,ç3i……K™T@Ò‘K­îÈk.QöùF&ugŸO¦|Po}p)»øäòAO“FX~qBÄ«¶È§þgö²Á - >6sxF9àþ^ -‹r[˜õµú°l$lRy»»ütgŸOÆ€°§OX y¿^ýi;ñï¨þƒ3œüéW¬?m'½VŠr³ÑbÀwןîìó©ÕŸ¢Ü»xÙö?ùúS¢h2LØ=¡þÔ ð#ןÂY>Pi~;gµÚÓÊ£‰?Ç/\‡šë?U^zJMªÞ@Ô^mäEëK© úØxéóêKûqžV_BüsI;Öò¼úÒ~ìßW})‡óõŸû -5ÙWÿ –å1mÞš‹»'èñ“®­\Õ]ÚéîEj0ö7^W¹ª³²ÓÙ³ê/ó™g?ºÊµ5w¾æ[õ©­ÐiýŸVž]y´ø¤O ÿùu™{ÎÈ­ï³qø¿¸½ØÉí:ÊUípø·Wo6üáu”«ºÚaöÿg½¥ÈMÉ[¿¹^oùv½/ ä1D¹G­Lwöõtáf“ÙÁþ>Àv_ÍÑÝq³M½0¤<ƒKùè‘”$·­járuç·‹ë÷M¾G=È®1B Þ\Ì6:Qóè¶6ó¤Üh›IÛAÑá4Ûh}ÄÔdŠk·¦–ºâ¬j)õ˜bPž5æ]éJ ó®Ä][·x=Z¢…ÙœÓX-{,™Zœ{w*–Ôhû¹y;w;mÓ2xGdjG·°kù,ü{Žï¨Èh­Úo­Œ¸<žlÄAÍ.‘§úeU6Ç aù¾Äy*•<¹ÏõÚɾBÒj'óLØÃD69ùÚ1]„G0ÊmÆ4TûJ2½[±ê’ž¦€‹ˆê3Õï©/×Q¿L’*Ê×L‘½Y¾#Yýñ¦ZâúÉê\B©¿ÿ÷ú¥º -endstream -endobj -781 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 780 0 R -/Resources 4 0 R -/Annots [ 782 0 R 783 0 R 784 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -782 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 770.673057 101.433071 751.923057 ] -/BS << -/W 0 ->> -/Dest (cb189-1) ->> -endobj -783 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 458.176036 101.433071 439.426036 ] -/BS << -/W 0 ->> -/Dest (cb191-1) ->> -endobj -784 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 397.229014 101.433071 378.479014 ] -/BS << -/W 0 ->> -/Dest (cb192-1) ->> -endobj -785 0 obj -<< -/Filter /FlateDecode -/Length 4580 ->> -stream -xœå\KäȾç¯È³ ǃñ °±;† nxƒ=tewÙ0jïÁßCñ’ÄTªjjÆlª+EƃÁ ¿ )¥ÌUãçƒÿ•RŽ®·ï/?\´Š^ã_i±º(ì?á7ú«Ñ•79$ýž .ª˜½áúrÁ›Ë¹ùËå¿qùßusµ*ç`¿þã+³È©˜bÔ^`e°´qË{£õA`£°Äœ%Äð‘,íYbPN^bA ²6 ¬1Eãö¬ 1l4$± †É> -,£!‡±Z‹<¢‰œxÆ«¥‰g -)Û,ò"x&Z‰çœ -‘Œ7/«‚O ‹ÏÚH<Y¼!Ò²KQâÈâ‚Ó"²ØÄ‹ÅB;Yà%ÈëJ"²è ÄËNùœˆD^O§˜ö<«½òÉ9-ñŒQ¶éD< -›¢À³%hŸEdñN[‰ç áGâ'¼‹) ÞjÙ»vFâyÈbm$‘Y°ùØYt Àƒã’„6FEÙ’69E‰’„6eE1: löŠBŽ8mÀ³F§!‹§ !3…à¹"²¸L8 YÐ@Âç ‹%'aÚ¬E¸%„xKÚ bj ×¥@8@.L)KXà€¹Ø)aèÂÝ£„¨ë|°8À®£$,pÀ]GFKX@À]çÐSäA°„@GåLrp×%, à.ÜÈJX€]U°Ã alEÙ”´„ܵI“„ܵG€È‹ ->è$, à®õX‚trw­×VÂîZ²AŠþh1îB¼@¬„8ïÁ³N ¦, ­J¼`UŠ€k‘—À3Q‚Iá(6 —R N/a^q¸$aÄÏ„1C¸Š„8ŠT²ˆÌ%ž,²%,@¤ ž1$ Y JXlB–´„8¡‘KBÕ4ç™I‚DAq¨(aÜ™&RäAÀ„„ØN¶½„H~l"ÍyÅy’° %Ëéf’° %Èbµ•°¶Éùf”° kÈ‚ˆO‚¬ƒâ0YÂÀ8xZKXMB¾iIÂø%ç›I‚ì4òM¤x‚ßf8ߌd‚,!ŠÈƒ,š²à·À$ä›@'‰ ‹Í$aAóÍ$a 䛈èÅ’@â|3Šåà®Ï@&‰ÜEd$,ÀòMD¶Rn¯¼>">“™‰3Î$V 4 ×ãlKØ‹Ó*Š5 ðåI,@Ld–)ˆU«`-– 4ðñí0a©>Í%ªsˆ83SäEf¸!ºï9Çhæî4[êcÿøëå—Ÿqø^ÿú—ç_”JYû10~GkÏ.b3¶p¥Êp„±¶åó‡„|oèÂ"Ä]IÕé—›týÒ8ßí"(‚Ž­¾~úþòËÿúöÛßþçŸ1í§çËw¿BŒmµ&ÿëo >ÓmùëÍB÷®^üýŸë§oæC¨¨3×QÖóñÙFԄކ嚾֡Sº‰Ð®Ÿ××>6~óiÄP¬PE ×ùÄæ;¸Ê¢ó‡?þî/‹¶Ä3côÈ‹æÑ©X:o°´Ì£Ïþ˯ÛÜ+“0\ Ïo¹Ö<•éÐŒNöy“ýæÃ7Õ¢‰È©_m?*¶›xwb‹”?fCÞLÜ ‰·'OcÖ¹Ë|Ïk9hæ½Åà~µ^‡·c}‹yÔ>l"¦òn“lzûÆË%ƒ¼ËÊÆ ÃõOì,-K·0Uƒf-9¿\ûP ý¹¬hÑý˜ÔòžÌ%Õͤegï½E–Ý Ø„ŽÁ‡F²ÏAûŸÂö-²ÆÛן믛l¿Ú -ÓØNºÍĉîÖ¶Ûm.l®ÍºŸ³ËÎòçŸmnÑnùèIwýr²b›H®ëeÑŠMdŸÁoóQþœ†œ¶Ë±–¯ù¯¯^ÖÄÖÿÔ°fÀùŠžÆAÄ¿}¼8Æ*}u=ÌäÛºz™ÝÈdÆØÝ oûv¦ŽYŽ›ç¥}‘¡µ×Õ[ífÍcO¨®5©û»¦ ¿¬ Øf2cõ‹m­gƒxM£a¬xÖrkW4ìÆgýÔÛIÛv’…6ØuÛaÔ¼F²œ6N%ãÜÀ®yœà¸hÊ7æ÷ã,žQå eGrÄtO=ÖRvœ¦]®… ? 4!—Æ.Þp[Nœ"g]ïb¥Ëþê*{iûµê¢Yõóz¿èNSxíºöÑÏ•.K‹w ÏÚõí¶Ñl§Éq¨Û¼]¿G^ºÈ>n…¶óÈ…n7^èvXõ÷ò7ý.ÑGµµ4­Õ®uâýFO´È°øÈ‚x÷öHWd¶ºê{«'ÿ2#.(ë¾ µu œ˜1¢®?¼ÒØ_·_uy[ä>²£…Õ1¸m‰y­·aÝÿØÿí¡ÞœhcÂXKäx=¼‡Gë1¦Éƒ_ªû®…1iiWæ4{íöe¦} :ÏSå®mü2þÊÍ£¸Ù÷"ß¡®¼¤«-þ¯öÒS‘çó_ýäÃiÒ·ŸÎ¢æX‡Ïú%îµÏËßâ· Aw«cåéÊ«“ Ÿi|1VÛÏ möç"WÅ첆vR7Ÿ-‘á®vØ}»Ftñ¿ÎGǾLµç¢Ó¦ÿ0üyk_þ6á‡ßJ>Ò°¡­±ó«NñžöxÿŽxÒ16ßhþãŸÆ~x7éóËñzû¾4L¬kmüy¯üÆšŸŸÿHo~‡ý÷õÿXÇ8v(‡§Rqå^.‰vÊ„RX=]‡;ÙçÃP|F›`æg*À­'ü7Uà@É@¤Ÿ´Ç0‘-uäÓ¸“}>Œ~p”ýܸõÄÿ*pÑ~ˆ&柳·žô¨‘¿Vö:~)=0ú“}>ZŽq;°}®€™°ÄÕm¯ßRë67Õ*š 5»\bݵ¶JœqcÞm•n_q’+p;›|yĉ{?XâÅÚg3Gcj¿Ä•×klã´8wgy!6+²ÚðÕ6<ªýžìóA,oˆÍ_î¹oy; ëµ…ZÞ›­ó}j¶"ÒñãH×Q,Ž\ ç' évõåHåìS4› P.QjvCòñMƒG¶v²Ï±µŽr)Ø#[+ðmÉd‹ÆÝþzKka^ÏŒo ÙšU1­üÞÆõ¶ Q®[Æâ7¶-Øj©røú÷©þ í¶\^™ãŽ%D×Ï.Çš§ÌÏ[¹>ĆØ)ÆzMƒ_êߊ&¶Ö(MûýRÿ†ú9Œ¶oÃæ»'4Åý,yw;úY÷ys¶¹©Ít&Ùí˜Ï2­îàÜAÃ6n#ß?;KB¤G½pgy¹ã‰hf,?({lyçú|4Ë+]ß·¼­54m–là5;aÖý·»±²²f5Û¹Ÿ¦~ÛÌçË&Û9ˆ‚Ú™ÝåŒëu§úœuºC\<²ÎG»ÐO5#Œ³±èå¾Ã›žÏÏÕóñ<ÅôúK5÷­ôaÞ»Å͆y~OßylÚŒ÷ ?È#þòü…§”Ê—wŽ-ò\Ÿg‘Ç1[;鶚ä]_Uiô´£îþü;ª-¯±„³ëœóœ-¾÷õßÉŒ·Wôâ†å}ÞÞ¹ÙYìˆ)ðW§` -ˆ'û|8‹=Ž-ûζ]ùºÇ¡¾£nm5½¿c]aŒ±;Í…ø±ãäÓ>Îœ­‰-a[Oé(ÄŒ´Áì>·ðÄÕª=M´¼—yŽ–{Õ÷Σ;V8ÅŒg3œ“}>šý Ðü@ǵhí¸u›9[ùq6·9PÜ–ÿŒp\ÏG~ë—…µo‚^)p5ÏS¿'AG~/G7g»¿­ÕšU’þ§úŵè±Ûž_ÚÃßèÊÓ÷àºÝ>lòÀL÷ßškÏô_Íæúu­ Cò«\º–GP#Éó^¥‹Ÿ6˜kö¶~J dß|f×<ŶêX}b£lw9×t -£™C«Åô;¯5+µý‰3=î½{ -§ÌWŸÐéO¤ÍøíÎþ$Ïê)\«e »müáÊ—•Mya…IËo´ºR\ŠùÊ_NÚ‘iÛfŠ¢ÍõNÏò°z«új0Ío áoŠ/ï ›.jûʆ1™ë¿<êX^㣠9Ùv1È/¾L™h®Éühlßk&»±;âíâ©{â ¿\|ZûÚ7Ó›‰n·““ö2¨15Uñ“•Æ_çÿ¨ðÊÈS>ê&Å/rWcöÍN^+i¦‡Ž:$iÀPm'u™»ÍÎú2I ít{ë®±uRÙD¹‰Ç ÕÁcM6Ýê:™!¬é§cåLÔML7Ⱦ›òËŠ<äôÝU½î“3™»ïãrèØù2“cßqÃoåèòöq'›oÄ[9ØhKfÓOm’A¶ºŸvc²‰8( ò€Ñ—BnÞÚщß4Ø>'“NÜÛåo—rÌYç-?ñœá—LÍQ—¹þm¾Åïß/tý}{«ÁrŽ#ÖoóÛŽ­+ò|’ñ+ ÓI2¿¢°»ÛLDKì²I£­oÖÐÏqk'¼¦Ñ]`o-MGPÓ¹3ÑNdÛM×7iÆo-ö@æÇîÍ8°—Kׇi¿£I…‹y9¸LƒçÆ‘|†Ä7G:¸d§Ž¸˜Åå4a’­Ädƒœ'éªgùiÞ -ÇsMSÁ—nðh³óÐVV15Aìæ‹y`2ó¤y% ô›ÝëGï{94`j‡öN}™©}aLí€'S]¹ ëË~æ¸òðSsÞ¡¡>–›|Ïõ g¸Æ-hu1ÛèDM£[màçI©ÑV“¶ƒ¢ÃhºÑúˆ±Éj·¦–²â$j)ö˜bPÿ6kÚϻҕèç] »¶> -endobj -787 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb194-1) ->> -endobj -788 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 736.726036 101.433071 717.976036 ] -/BS << -/W 0 ->> -/Dest (cb195-1) ->> -endobj -789 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 409.229014 101.433071 390.479014 ] -/BS << -/W 0 ->> -/Dest (cb197-1) ->> -endobj -790 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 348.281993 101.433071 329.531993 ] -/BS << -/W 0 ->> -/Dest (cb198-1) ->> -endobj -791 0 obj -<< -/Filter /FlateDecode -/Length 4809 ->> -stream -xœí]Ë®$¹qÝ×WÔZÀ¤ø¾adØcX°à†½0¼¸]ê+Áè10ò¿ïC&‘̨¬¬Û-¹ ¸/ºoe‘ÁÃÃf[_~~Ðø'’^bLÁÓõöóå—‹Z‚SøSJl.Šú÷øÜU+ -‹ÓÉGwý9 lXBr&øë× þ™.yñ¯—ýÕå?·uôÕ,)y£ÝõÏ_²ŠìbÊ ª•ÒVP9·„`œT^/¸Å”$Ìp íUfPŠNRÁ 2& -ª3,m÷ª3LÐ$©`†N.*­`‡6J‰:¢ˆ¬tÚ-N÷’ÎèÅÇd’¨ Ðé`$µ‹¤–tiñÞ»(鶸¤´¤s°Åi"%é` Ù$‡-Ö[%ê`‹‰ÁJº[ ¼“]„-@Wu°Ey¥%]²‹K‘HÔ%èT q¯3Ê-.Z«$Ö‹6­¨ÃŒÂ ù è lñÊ%Q[œUFÒYØBø‘tà gCŒÂl5yv[eµ¤s°Å˜@¢¶`ð%0¶¨à% -À Z2/I`BX(’HÀD»P¤(±€‰i¡¬D&¹…| -X¥¡3Z"«`‹#/1Õ°…0sEl±‰$.°¶ €ÄÖÂCVâ ¶% Eâ º%•´Ä|KJ{‰ ,f*‰ ,×FOXP. ”$.°à\ À‚t1݃Ĭk7XЮ¥è%.°à]KZI\@à]kQSÔÁXâ°ãbu´x×j$. ð.¦‘‘¸£º‡^â`e11*‰ ¼k¢"‰ ¼k–QÌA+qwÃ-H+7x×8e$. ð®!ã%.Pð£ð.(ÄÉálÁJ\€õ:c%.@c‹AX"qïê„åO˜Ó`«EÇä%.pà]1|¢.-:Àã¼…—í}’¸Àw±PY‰ xô%.Ç/XQļ«-VjIÞÕ&)‰ xqƒ“¸pX4‰ °¾-Zº¢.-*Å(qï‚–ŒÄX…Á$¢.,’–¸d¼(Ä¡xð.–÷$qï*g­Ä¼«a¨ƒ-6‰ ×@‡i+Ìi«‹2` -IÞU#,éÀ»J…$qïæeSâ²ÀD‰ `à‚(ØH\à#-)/qO -:„|¢Î/¸ve‰:ØB$.@`½$›¢ÄÁÀkŒÄÁÀ„  "X’Fô-¦°pó‚-ˆ%.“-1I\œG>¯J:o–@×¢.B§ƒÄ!Ђ¥XK\€\j‰˜ô€à—.I\3 Ó$qAH°SEâ,EK4ˆÌ%†-²%.@¤ ÖD[ J\MD–é•ÄX¡‘KÂÕ’ŽTÎ3£Ä‘ü’CE‰ 0Mi‚)ElMH\€ñÉɶ“¸É’M¤9¢¶XGÄhrº%.ˆ¶e$.6s¾$.H -¶ ⓸ )¿ä0YâÐ8tJI\tD¾iHâÌËœoF‰ ’UÈ7‘â ó6YŸóÍ qA"Ø⢈:Ø‚ ) óœ„|ì$é$Í1Ÿðë¹:ß߯¬ßÓ~=Wç»ÛŠ$­¢m¿¢‘ -ßI•Óö:™S¾ŸŒ9B.‘e7aåƒ6G'ÌÔ'«|GƒÕbrÐêQÆ&“y07Ò3øŠþ›Oð}µb}”Ð/™ù;5Öo㮟~¾üúïúéoÿîŸ1˜ŸÞ/ÿöDjýëÂ?h\ÓûúÛùõwÖQýL±ê\ýXDtÛ¶Ñʛ܇aõ[_­¬ûRuµ ¹;}j¡8ÙlÆçö»Ùt¦¯r?ÊÌ÷8|ñã¿_?ýtùí§-ˆJ‚¶rbo£)=Àݹ*ß‚»¿ÜR:„Ûû:„ÎT÷MCHM^¡äl…êh½•u׫õ7’øg&î­þ6[Hµ¡.õìvxûð¿1›ô -éßÒ×M°ÁOýÏ¢­ý¼ýÜŸzÝf½½7šúèm°ò÷ ™Z5€Ù J9p<ÄäÉ:¯JmI°×D‚œvVÛ2Ýãê -ù!óÕöRÝTßlëÌ}»-±ºÄÐäѽoë×}à¸í]—Øg½µÝјUmF»-oëBí Àü¤Té²CrŒÐsu^¡tÈ›”Þâ¤eìm_~^.Myˆ·GÞ½zæËó¨±^XÊÓ^f¤rú7¡†`+›¹½½¸-³™v*;ûäïÞE0 “C.œWñG>Wçå|h6¾9 -èzÆ9/²Q§iß·£Àý̨Ó$Î\NœÏî!„£ržiŒ¯7h;ºç~:÷§ûhçkïûv€ÎD¥'ë¼:ãÒÿO_FÝoŠÑï!ðtì.#Õ°¸S›%©hò¾ê!ROÖy5¤š±jCÄ›ÀAqëe‰OfDöÍl딨 <ÇŸ<*ò™»´:Ïšé>:;ŒÜ×)… +³Á]$Žø’B\nï!ñ\—C⃘ôÁ(òåQ”ø±q HGÈÄUedoÛí+)þÿnÚfÏ.#lõÞ¦8tê¿´×fóg¡ms#>4M1¿üçê¼cÊÝΉ›~Ï;=ñÜ[ºiÓSÿ¡æñ:žåú(O+¦8ê÷x‡Å]ù‡ËÓûãQ?WçåFý8Vë±Qœ¼z”ßâÞÌ{ÍC+*L\óM“uŸ×kI—ë˜6§«Þbn›[m÷3«G£>ïKã&5¦œæeßÖ>òFHo×W[G¹iü¥ÞŸ¿œ×ËöË(³|ï$ÊNÖy5”Ùã8ëÿÂè~¥“óu±Ë²vðÙšjÛ­ÞoÝÓ,<ø~MöyÎ:YçåÐt+}¿yû$2žB¦_ѬÃ÷ægyUîu®ÎË¡ê8þùø|}ñ/ˆÌ§W½'Þ¥=Yçåu[½þþ/¬Ìö¦è{S'ë¼zèÁÞËl7õmäj»½*ž'Í9X‹è?¯Q<0çYâ¾ÔA{ÒóʇO•Ä§ëìYCÛ…˜w@[¹·)uÞùùokã¹gwÊöŸ|~‹Ã!ô\—Cèqv´¯½Ù˱Ãûß²~ø”iš»ü•ï1…¤ÍStª«zÛ•Ó.×Õ²ßǦ;HûÀûu'ë¼Ò<ïœ8a·;xçm¢§GBOõ'kzéÝ“ùÍ‚Íþßo}ÜäÝ×ù‰ýþ=õÔó‹»(cûZ¤Îe'ë¼Ê<·œß-ãO‰…§ÏÏx¿ì™ûǨqÓzGI:~ÂÒŸöL«ZY-ù3ÐÛÁ,h³Š=!zƾÎýOÖë³N¼·ŽÚþ„<Úi)ÏÿÛh÷Köš¹#ÊçhËhÿöwÿôÿò;öödf÷ì>÷G„jº÷&ë; -ò­e]È1G›óëõN>ïoÛµñFÌZd]>¯O~WY)§G;ôÞV±zj¿ýiôÚG[Ñ»ŠÔ>Úµ}=l(eBŠßO9ÚsGó]åÒ>óG»ß~íxüµÊUÉxœÉyø¸Ü{nû¶"Dûj¢ç¶y•¿¤eó!Õ6[ÇÎe_Çá/ý‡};Þæo°åSöí”{h¾õåžì°ñžjÜKö}¿¿<õ)ž`[>-Ã#\òIh»`ø¶b£ØYï·´Y߈m¸(e¿T_¸1>v—í‰bªv¶6Úu­“³Ð"g÷´c¬wu;Nêgßì8ô­›ï̯ýÜ2›y5î×ìæŸKë3«-n×òZ?áïæGÅ°¬+ŽmõÃmصŸ»|ÞÛÎùüЇaö!=ÀDWÙÿ¶ÞßÑØ®O²«M¾­LmUcå*¿é×·wîÛ¨õ±Z>zä¯}”úˆÖ6‹}z?oŠßêu/ª6}®ú·ZÆm×€¢Ó«®qÇÃИ3wæóUéçmâ2W«Ñ·aÅç‘Ÿ‚žý”çœÍ{ž®î}¾×ݥܶYwŒòÞjùªNU]Û m7j]2æݧ,+ö'fåÑr¯ïÛyº_ƒÏ¬¿ÃgòÚ»åŠuŸóÄ¡¿vümÚø1§g츶>p|Ü™+ZÜÒõÕ_w±ò Ðy1 <7Ì»ÏcÎŽv¥5˜ùiÏó÷¸£]7^¬÷Úô}]UÃ'm>´9|äϹýG~ý8Oú#ÑŒ›†E¶4Ÿî¾DeÔâmùfc>YL…à|‹ê\}ò;&|®Ûki‰Îâå/PÝyÕ­,;üµ¿uîv˜ü¢R>eê°…b…2ÒºQ¾Âa›Nœz¡TLƒD\Äà–ä0ö^L|$”‰z'È» ŒÓSùÖi>ÂË;*G/èd—PÒ~Ñ:Dퟸï} $£h I%ÄôÛŽ;þÊßOÚ¼]8í8ô~ÞFïnÌ6ÅÚ>Þ”OéÒ‘”‘Á[W×¢'Æ̵å oŸAåóãR>©iê´Œì=ÐGÓœÏx:õÃ=Ù“u^d·l`?…#ì«·ú×2ì³=‡Ís+ö¶éî»wÞìÏ’Ú8ü¾@LvùðÍ|’DbçYtç?,òÀ×ûÓ/ڗ߯zúîþÈh2Ÿè(*Kº ›.ŽÖ×;5Ëy¾¿ôRõˆ_•OýË'>­çÿ²‹Zþ—2à?_tŽ§Ô: Öã8UcúºYüõ’ ÄDc‹ì8 ëGÕÛ"ÝwÜšì6 KšKbªªúqlËåù»~Ìëë¨ïZµ|L©VÈ–ƒbÖŽ!jz½ç»W‹®â|¤m\l½òÝç›ÍŽ‰— ŽŠÅÚ*&t{¡bÝ ±¡fÎG u×ù!ì¦s×i6ü­¬ l8„· ºµ³ô«(Íi¶¦šwq7™eF¥.u­dêã$ˆnAøƒjM³ëÙ¨åãÈVal@¢^.µr‚èvËÕÖ\©ÐKÅQ55ÏSÔsÖ|_vc[×!ŸÑÍ戲íãVœwUûX«ÈpcvÂÛÅQŸ‰Cüõâ‚PÚ ®P®A -k·‹£xÒü•ƒõ¦f“åÃœò¿+U¸.ÍÌS>hêʲ«ÑIž›]¼u—ûÎ:$y@Se™»t³Ü_:*¡|&Ý^º{l#e.còšy™¤:ylź£®‹3…5ÿt®äBÕŒ@wCì:”¿nÄÃNקªS½3¯Ü]owˆ}çί\úˆë|º^···Ë0ß„·²°Ñ,ÎЭ“!6ª¯v£3&††xÐè×"n³µ³SùÎhý8&™´âÞ.º”eÎXgr’ç­Î‡Åò¨K_ÿe~Âßÿ¸ÐõÛédë:nTg Ó¹u#æ+Y>Z<žç£Æ›°O&4Ähˆue]CC_Ça|M£ºÄÞJêΠºk¹Ð 2±éÐ5Ú2ϸámòbd¾ulÆ‚½^ÚvsèÆæsíšU¸à·ƒË8tv,ÉWÖ$®rѱ¤CK†UÄo—¬Ã4,Û˜™É;é®Ïö -›ûb,ŒÈ—vèh,³¼é&+wÁŠ ¶ü‚7Lšwš6öFxz}ó¸W“}›YÚ©½K¿ri¿±,í„'Km¹ôÛ˾æØ’ï·É;<ÔÛ²lîÙ䌩1B Ú\pŒ2iÕjÇ;¥&ÛtÚŠN©&ë-†f“¯Õš[ÊGÑK¡ÇCú'îiÇG¥;ÑñQñ»²¹Ý¶âõhÉöÈq8‡qwj”ì±ddž×îR%m“íû¦mßmµ½²Û aˆ‘©ÕÜ®ä‡øï#sg­3íBH¥ÚOÙÑqœç‰…št°Æ–ÿ¢EEïÁ!”OÙÙVÉÓuŽ÷Nö;$mï„÷„áXÈZtþÏvd¾B‘+XSí?â‘«%%ÞÒs¶†Í©ÝÎPùßÊf]Ù ê—!§Šù?W±9èùLø ´üžÕ¿a³…R~þ6),\ -endstream -endobj -792 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 791 0 R -/Resources 4 0 R -/Annots [ 793 0 R 794 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -793 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 228.873057 101.433071 210.123057 ] -/BS << -/W 0 ->> -/Dest (cb201-1) ->> -endobj -794 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 167.926036 101.433071 149.176036 ] -/BS << -/W 0 ->> -/Dest (cb202-1) ->> -endobj -795 0 obj -<< -/Filter /FlateDecode -/Length 4072 ->> -stream -xœÝ\KÇ ¾Ï¯˜s•«Š¬`øÀA¢ F‚ÉÁÈA[6‚U9‡üý|¬îzt7gvvµräh±Úi²,ù‘¬îw¶øyåð_fgr.)òùòþôádM -ÿj‹ÍEeÿ¿)œåd‚+1‡ó{!P2©Ÿâùá„ÿv—só‡Óßsú׶;{SJô.œþAXL&å”lPX,ëHa…`Rò!*¬è –XŠÆ‚!±ç#+A .9h,ˆÁÞg…•!qrtdˆá“c1\ Ia9 9œ·VåAËL^á¹`"”5žw&æâ‹ÊKà¹ä5‘‰‰]p¯˜cÈ!K(Öi¼Y‚c¶²0å¤ñ"d¡HVåAŸi¼Y<´S^†,°®¬ò ‹Öi¼B&”̬ò -x6§|äyLÈDVã9gl“T< -›“Âó%ÚPTd d½Æ#ÈÂøÑxÀ‰@)gÅ[½x7Yr/@ï«<È‚Í×ÀGÈbSÔ d—4 ð).ž5ð™ gÎ -ø\ §D ø Ç’4 ëÀóN²%pÔ€dax®Êƒ,TXÃò 4, ‚,žIÃÚ2¬EÃܲ-NÃÞ²uQÃàBL«aq)GÖ°€¹0¥¢as Ø©atáîIÃêRˆ^Ãìç¨aw‰Õ°€»Dè©ò Xà£!—IÃî’³IÃØU;ŒÀVŒÏÙjXÀÀ]Ÿ-kXÀÀ]ŸT^2ðAÒ°€»>` Zäîú`½† Üõ죆0ü±j6Ü„=€,X ïÁó¤a3i‰†¸ë -ŸâÓ@+ãr‰à®ËØ>•WŒKиâ·Ð²q1 pŠ4,À]ÀgÖ°oQ½†¸ë‘Zãw/VÂÜEÞ4,€9‡@ÃÄ7ã,LWåcKÎà.`ÉkX€(ll’¨¼d`HNÀ±±ÈC5,ˆÀ]„÷¢aAîÚ@¤aAîZFÚ£ò e¯aòðඊO#Y5Ö)4p×:ì°ÆîZ›Š†¸+aS˜„ೆÐ öÄ̦¤5,ˆÅ‚‡”OåEƒë aA² @–ʃ,œXÃ$Ö¦PÉ$YÈ{ ’‡,H4,@F`ŠCö­–æ¢Ö Y)jX$3¹0kXBD=­j¼èMN€k•—ÁsIÂ”Ø ; PK™ §×°o$]Ò°b€çXÂT \EÄ"“=2sç ’l é‚眆ÙCdƒdŸQeF«a"4jI¨Z㱕:3kX9I5,€› ÒRª<Șаû#ÅvаÅŠM”9*²P` röRnf r†,Þz `›Ro& Š…,Èø4,(6I“5,Œƒg­†ÅeÔ›ž5,€_J½™5,(dQo¢ÄSü¶P”z3iXP²¤(*² i*Šß“Po4^„,¾°†%F©7³†H3Po"£W²Ô›I=.î†dÒxÀ]d¦QÃ\ ÞDf«ÕöÀò3™¥âÌê‰ôÄVõÈÀ{­’zf`¾R#©‡•eŽê©Â*˜ÎªÇø‹|Œ˜°œ>ÍGTæ3s’E¸#»ï5ÇhFWš-çc?ÿxúâ-‚ïùÇŸÞý¦ž”µã'ŽRXK­%Ç.Øoà•©ã1*L$Û^HŸÓG„Heé‹ë¡›‰PŒp[ÎýƒÛŠ ‰Ef‡d¦®Œ7ó¢TpØ+Šª¬OîóB²1ò3¤DÈÞî×ë}^\¯ó¼÷êõÎ>/.k¦3jYËÛyU!kü¨t$ÿÎÇG!>÷Û7'äïÞöëWãrÚôD‘œÏç7ïO_|ýÍŸÿø·o †7ïNß~‰ðÑ 6²ë¯^¹/×Ïo%i6æ…Üú7®=XËyi {²üÝB‡nêõ¾¶ÅõWŽ´¤-ÿ°ð„ÆÐ[?áÉï~›¿öm2¤¥´©ýãVŽ:Þ,ÿ?Îo^ߥ·2ëíÛ¶,çí4½]EsCYbv·ª!̪ ~j“–ñÛØM]ö;lo—Üú5V5 =ËØ—¯–EÆuuq^–X/#‘@„Y×E¡©WæXT)ã¹ïãD’¤­ Æ©k UžX×|KŽ˜®ÉaÇZêÖòØæàEתlr@-”±«9ËXe•s]ïbËžÚUöÚö‡U«LÝÛþÆ!‹ŒYålc´ëµ}·Ò§5mÝcìõ¡o·“fGMŽ›ºÍûõï]MtAT7=o]ËÇUÎîRK;÷ýÞvWWvOÐyÓå •ßtýnê“Ç> _œýp•%^Óû-½y{ÐÛ~±‹¾?¢‹¶—ñöþÊ®6†´•ëê?—õošü"Ýæ×±nÈèÜms ô´®ÂõMù½Îóó2f•Ï}§êm½îûÅ ÊWþÛµM6ÂÊs ÓÎ>& -¶ÔqÊÖ¿æ=¾žnêz¸ÊZ»ÂŽ¾vÝ7«{»ú”¶FÁ7ñq$Ìã÷Ýò·âÜx ¸¬Óʳ+Op}ZøB“ø$øíß -í¦žØ\÷¢­ërÔUÇ’`{lé±ng;¡ÅˆÙ>®øJÇnqìUÕß5Œþ èk)Ížoêê€ãuÜÕ6ª-Ó¤§ïo¯—ÛuÃÅf—³_¸Õ.§µUßâIßwŽÿ˜^ŸS·tFžv:[r'»‰½-øúMMèiTsr'²VrCÝ%ÊH]651°sðÔ>ϪTÖÜßx9ZÅz /GQÖ[2ÊŽ}AÀmÖE5ýÒQèòÊS(W¼­Á^¿þÝïÿ:U²Ýk5À—V¬a‡™ôÕ K4¶ÈM‰Ý„-«PW–Ôb1‡muÑeh×ï¶×å« ØÄ^‰T»`+çØû¨ÖAš‘TG½’è]3 –#ºR+B¹s×Ç›tgŸÏÆ€XŽú]Ê.>Ù€¸ÕA¼ßÄ«{Ù¸b òúíÄÝZb"­&¼L²Ùiì'o†3¹ÌÖëÆ»F…в§ ™jëUÞ±Š’Ç&ŠÜ ÜMZwöŠÑËýãfÀÄ:Å1ú;û|Ì Ê§°ý½aûöíúK“íÏçn²™4Ñik»ÝæâîÚmûIFkžó‹Í­Úí'*r;½Þ%+÷E}7äüÔÇQÞuÔÆOcΖwµãªÞ¶L¹JËc÷õÈsÏŽÜN®æµ÷¨¼Ì*ÿv³«cF9a";­Ðm5[³?Éênž,E_ä0t‹ç,Ç©–¾Vy=0ó‚w=nÙá¦z¸)o ×äµ;Ÿwu{±àc‚ñë;a¸©Ó¼_÷ËŸF(c~¶' ·t•ì^W×N"„üØ‹:ŠBö§7×ã÷ëù´§ «lŸíéÁM]ñc{ÿ²§ Ãî?ÿÓƒ›z;àêÿæ”!ËSÐ[[ëzónmÁöÓ»@“o­sõñ†W“mŠÌ¢«–¿‹?ù¼Ûÿ+q䥱c¾ Qû¯¶ÕÚ͘næ Ùïu÷˜Œ·î¸4_Õbt·ù0é½îÇýT–7i6”š§^~¡5O;5åu[F4原úž¥£©½k\æ~‚–ó^ËÝOËðÛ%N¯R5 5*…Íê®ìÀAKÿ\Û6|ŸsÀ<Í¡ìD³Õ>fÚÏ}¿VP·î´ò²÷‚ yþð£3öã8 íî»\R¸&ÇGgâDZï¾/ùq/¸”ýú·–»ŒÙôÝ×ùˆ×Ll¯wŽîš=âAŠ‡ºûÆ7ï—Ë÷Jzö æ+›ù+)ë¾~³>U_{µç ¯1ÆâK™ž ìçy6yäøîøa{ ëìvjmÏF‹~Ën&y6Õ©ÌÚÛøáëãÛ®¾Âcc6ÎË;¾çÈFŠ®TÎòÈx†¯»¶m®*Ú¯ô¬oFè­Ö—¥­¼?%ÏÎ/oROkûuÃß#ô±‰-ØÕm%gß.ùá$÷ÔX!3ˆùYˆëGÛÇb×p£]N\ö-)»~QZŽ„—rËfô­›Jvd'b8í4Ý {nò8y™¤«.b}V›¶¿µõiÚÎ[ÄË ÓÒžú Rå•Ä6TÓ®·F“P¥SCkÙ7„û>)¤ËI!¾²mhÆfvþ´kòbÇBÌ͸·+­BºœÔvëh¡ïTê­òèZšæ‡(ÃÔ)wÅ+´Ë”É&ù¶ƒÉG,µ[²$h}¯mžìƈ—Sàüp -IiVØÐLo&Ò4n'g;ìePSnª‚k6š</ÿ/P:U§~pÜMJ^mWcñÍNÞ*i¦ÇŽ:¬iÀqm'u¹”ºÍÎúrÙ*ít{ë®± uRÙDO¥‰' ÕÁcKvÝê:Y ¬é§cåL´ML7È¡›òÆ<ä ÝUƒí“E7™{èãrìØù0“Sßq'ï)uyû¸“Í7â¥6Þ“Åôs›d½íÑnL6‡`…A0úPÉÍ[;:Éw/´ÃÉ´ˆ{9ýtªaÎSðòÜ@$'¯ÝÎY—;ÿm^ã÷Ÿ'>ÿ©½ç±Äqo;ZøŽ­òÉäKòdùÒ†Fìn3=O°?È.¶¡YCãÞOxÍ£»ÀÞZºŽ ®sgb:œÈ¾›®w4i&Ü ^vZì‰ÌÇîÍØË%µÅa’7„šT¸˜—ƒË·ÏNŽ'$íìdž Ûa˜È&'_[¦‹ðFyœ1 Õ¾ÒLïV¬º¤§)àê+¦ú{põ{kêó_õ€¨_&)åkªH’Þ,ß®‘¬Ó?âõ7oŽPêÏ“¬õ -endstream -endobj -796 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 795 0 R -/Resources 4 0 R -/Annots [ 797 0 R 798 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -797 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 582.273057 101.433071 563.523057 ] -/BS << -/W 0 ->> -/Dest (cb204-1) ->> -endobj -798 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 521.326036 101.433071 502.576036 ] -/BS << -/W 0 ->> -/Dest (cb205-1) ->> -endobj -799 0 obj -<< -/Filter /FlateDecode -/Length 4101 ->> -stream -xœå\K#·¾ëWèÀ4É*¾Ã‡1bd‘ŒÖ²ÇF0`Cþ~¾¢øjuI£™uÉ´RWñQ,¿z¨[îhñ÷™Ã™É¹¤ÈÇÓ»Ãûƒ5)Xü«-6•ý-^)åd‚+1‡ã;!P2©Ÿâññ€ÿ..×懿ýæðÏmwô¦”è]8þò£°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ,±1BbÏ{V‚\rÐXƒ½Ï -+C âähÏ*Ã'Ç b¸’Ârr8o­Êƒ –™¼ÂsÁD(=j<ïLÌÅ•—ÀsÉk<"»à4^11Æ5C–P¬Óx²Çl5daÊIãEÈB‘¬Êƒ,>'Òx ²xh§(¼ Y`]YåA­Óx…L(™YåðlNyÏó6˜‰¬ÆsÎØ&©<œ(lRL -ÏC–hCQy%õ ãOã'¥œ•Óêåt“%§ñdñ>±Êƒ,Ø| |„,6E p‚Œà’†>%Ãų>“áÌYCŸ‹á”Hƒ_‚áX’†dxÞi@@²Žƒ,Œ“«ò Ö°€³†Àxê5,À]GðÔ¸ë|±à.↠aÌÁ8À¿gaº*¯[rÖ° wK^Ãxac3Då%CrŒEªaAî½ "p×" "p×2•Y({ ×€‡c«œi«Æz …ÆîZ‡ÖxÀ]kSÑ° wÅmjX“|Ö°DÁ^˜ٔ”¢†±Xðò©¼hp4,H„ÈRy…kX€ÀÚ*YÂä! y¯aAò!ƒ†ˆLqˆ¾Õô²À\Ôü€! "E €d&f RˆÈÇ U½É p­ò2x.iX¸b§ar)“qè5,À —4,€à9Ö° È‚£¢a\‘É‘¹ÆsA¶†ˆtÁsNÂì! ¢A ²ÏÈ2£Õ°¹$T­ñØJž™5,È„Šà˜ ÓRª<Șаû#ÉvаÉ’M¤9*²P` rö’nf r†,Þz `›’o& Š…,ˆø4,(6 “5,Œƒg­†Åe䛞5,À¹”|3kXPÈ"ßDŠ§œÛBQòͤaAaÈ¢¨<È‚ ©(瘄|è¤ñ"dñ…5,(1J¾™5,@˜|½ZÈ’o&µ\Ü Ȥñ€»ˆL£†¸@¾‰ÈVËí-€7$Äg:3KƙՊôøVµd`½ðVI­X€¯äHjÑb"³ÌQ­À­‚é¬Z6°À_Äc¼Ã„sõi-QE˜C‚ÏÌIYp,ÑýÈ9f3ºÒì\ûå§Ãçoá|?ýëðð›Z)ëÆOkbs‹/Øoà•©ã12LÛ^H_ÒG„He -é‹­è&BÂÃÝ–ãøà¶"Cb‘Ù!˜©+ãͼHöŠ¢*ë³û¼’¬SG1C@ôv¿^ïìóêz]ç½W¯wöyuYIrDä²–·óªB>ÕøIéHþ÷žØ| àb›÷ÞÍ¿³Ï«+t÷ÞÍ¿³Ï«ªŒD*g[Ê3Õ}^]¯ë¼÷êõÎ>¯/+’^ø|™âéCõTãW·“Jpíƒþ}‚|.’çÎkù<úÈ<ß7þÛÖ&œÇï2Už;ód¡×1¸ŸZ›8×U¯ÓylÎ_þýøæë+º+«îªº5UqÞ¶áú”¡½Û9Ý*ª¨¯/ÓcžÏËûÃù¤|‡‘éÔÞSãÙÆûOg¾Ð/ò q±"‹¦`Rñ’U#U²%ÃÓ_Â0÷YϪ—ŽÂ‘¢1œ„·•ÿðõ׿ûý_&* ¢røò3'&r:¿W³ó vñ~Þ„«"…´Eê«Ê¶£5¶ÕÇnmìÜæî2ôë‡íuH]lâðIW]‘gùJ{U_¤I=¨þ¢áÛëäS0\j&_ÂÅÀµxrÓ€îìóébäRvñÙıÑùr¯Nì-#¹¶%–‹‰‡!ùîkÚ˜mî:ßÃV^y/1¸/¶ë~®ïl­˜ˆk¼Ó"›]ƾßxár¥æÉÖëÆÛ¼Gè̵̂iEãŽìi‡¤^¾.ò]ËŤug¯=À¯°Ü¥ët†7þÎ>·~ Û'©²^·}û¶½h±ýf+3\ÜÚÓ9†ÛÚî°¹xqínR»È?.Kb“¢{Q-É5ßëú9q=¼ž²ú!ËVÆ~ŽºßŸ±S?ó—1U_Ût5öh1D×Ù?õø¡; -Ç—¡w7==nèqÊi‰WÛÉééD÷$×í´Ü£fZÕ\£­kÒ†3Z íõè½k7m-hôéŽS‹#²b¥ÍÕ \V¿®¼†úßw§¸Èá—öUÓ§[Q'–šÏ‚%:MŒYlj$ßzÈ5ûq6P¬ë¾!ÎÏ59ì\ËF=£pªlr‡UD -ï´»ž–“‰f[oÏÄøíÜŸž•ÝÊ,.“׌TÞ¥}=, íEP³ËhÛ<ï÷öáL÷'Óoƺ¹t©¯Íœ—2öL$ê{Ó3Æ¡ëËþròiY÷:æ½:ÉS/—¼Ú¯gFn;Ÿ ÉF¾–I=¹ÿÓVÎÅ…›º šnûžÑ Ôó âù§tðö¨gÅ›ìq)Ò [^ÎÏ-ï™f_“¶·#+¯{tS¯i§×kgº¯Å.x¹…êܽêV_ÐýÀ3ÏõT*®ýöêR”ë›ú*;}]«´bÞ5þ9b¿¾žO¿`xKOqç;>¬°ØÚ@Å°cÒsª†â êuhUÆ!õ=¶*a«&7õ³÷wV÷Ñßý¥·;™¯˜)¶"Ó­ùžiÞ% Ë]q)äúº,·+«íó³*¦7÷hïsþg*«m¬gWLoêkïK>éÊj²ñBÞçTVå9ë·hwWVïìóÉÆG| íû®_YÝNø_ª¬B熳ƒHµ²šåY_¿á¼»²zgŸOÆ€²¯Ï1yûkWV·ÿUVsÌr×d*¿feu;é­ÊªÜdÙ XîgWï¼mô÷õùÔ*«Åæ[¶ÿ«UV{ìÔ슚­ù¶ÛŸ|…Wî?µ‘`iÏ«ðöÔÇ®î®õÇd÷qð3ê4j]ô‰ZêÈi6{v9éþJoI«º¿ÛìèV#}ô‘Ys϶šzÝÊ+¿—;ƒ°yYéUèZíVûm»µ5Œä±^¹ß³,wÊèz²ÉHµ¿¯¶ßKutwYm݆”gÚ)=©$¹ \;25Eýñì zäâÛÍ1U¥!9JeI½Z馗Ææµ²@?Õ‰ôíálB÷C­Ë~E»+ ÕùZÉh-=nÆï©à"O?fc ¡¯a·ï±>Îàê#mþËyyæýÙHFŸäËYy‚ØõmsUÑîx¥gý¥€÷£Uûñ+Ïʳ$ç_X.Zû÷uÃß䮋ØOc}Ð×Vröýb’’>°BfžGú ²ÛG;Æb7¢ÓNðÅɔ˖pt¶}œ0"ÞïüQ¢ÓÙ?ônr>ÛÞˆê-èç‰"zF·'Ëê­q,ËgCí*QBì2ƒ8äµípv|’ìÈ.Ä°'ÚeºIöÜå‘;—úÇ2ö\ngꢯªsËö÷¶ný]ÄÓÓÒ%õQ¥Ê#º}¨®]o¡Ê †Þrl}RH§ƒBüÌö¡›9øˮɃNgbî†Ä£]éíÒé ¶k£…±Si´Ê³k隟¢LS§<¯ÐN‹«Mòë˱Ô?nÉòÝÛØkÄŽÓnüŽx: (âùñ’Ò:L¬°¡›ÞJ¤eÜAÎvÚˤ¦ÜU…£Ùiò˜ˆü†Š0¨‚<õƒãaRò¨×¼š“ÈÙä­’Vz¨Ãš§ÙvQ—KiØìª/—­Ò^@w´ÛP•-ôTºxR<¶d7¬nº~V®DÛ…Àt“†)?nÈSÎ0Žj°c²èscÜIŽ;Wr;îä¹½!ïw±ùNVr?­ä·HúÇyÈ4{:ü|¨nÎSðR"‰ää1ô5êrÇ£Í×xýãÀÇ?õçžÎ~ù^ŸßlÝWO&?Z’ï$˘tâ86 Ñóû“ìòlº5 ?îý‚×<»Û ì½¥êw%†©Ã…ì‡ézG‹f âéB‹#ùн™û|I}q˜†ä‰¹..Öåà2OM—|\†Ä•4.\öKG\¬ƒâr™°LÉ6bŠA®“ Õ‹ü¼nÉ\ËÅFÈ%MO7»ÝiuK„À´^¬³['-y¡_|¼>xߛȱŸ¡hÔÇ•:&Ôx:•êeÜ^ŸCµZÖïÔЋ–³G#È™Gc†¼¹Xmt¡æÙ­5ë¤Üi›I»£pší´1bê2ÅÖ­«¥®8«ZJ#¦˜ÔŸWM‡uW†ú+q×VÆíoDK4 °šsš«³³åˆ%³B‹kïAåÙ’:m?7oçîÞö¸,ƒ§ qF¦vv »–/¿—œõ¡(­Uÿ«•—ç“ÂpÔìyª?þfsŒÀ–ßïHœ—RɳûÜ®ì+$½v²Î„í0Ld““ŸñÓEx£<ÍX†ê?ñ§w+V]Òópõ‘ký¶ú;NµÔ] Dã2Iª(?ÛFôfùµ™dõGÚz%õ¢„Rÿþl›R& -endstream -endobj -800 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 799 0 R -/Resources 4 0 R -/Annots [ 801 0 R 802 0 R 803 0 R 804 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -801 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 698.673057 101.433071 679.923057 ] -/BS << -/W 0 ->> -/Dest (cb207-1) ->> -endobj -802 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 637.726036 101.433071 618.976036 ] -/BS << -/W 0 ->> -/Dest (cb208-1) ->> -endobj -803 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 295.229014 101.433071 276.479014 ] -/BS << -/W 0 ->> -/Dest (cb210-1) ->> -endobj -804 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 234.281993 101.433071 215.531993 ] -/BS << -/W 0 ->> -/Dest (cb211-1) ->> -endobj -805 0 obj -<< -/Filter /FlateDecode -/Length 4920 ->> -stream -xœí]K$¹q¾×¯¨³€Mñ|  ÈÖ°`Áû`øÐS3-Áè°ÒÁß_°øJfdVvÏ.ÞAoUFð$?~Œ`f²ôUáßÿ‹¤—Sðt½ý|ù墖àþË)VYýGüwÕŠÂâtòÑ]f KHÎ}»àÓå˜üíòüeG_Í’’7Ú]ÿú•Ud—CPNP%¨”¶‚ʹ%ã¼ òzAS’T0Ã2´U˜A):I3Ș(¨"Ì°´ÝªÌ0A“¤‚:¹ ¨´‚Ú(%ê`ˆ"²FÐi·xtº—tF/>&“D]€N#é¬]| í´¤K‹÷ÞEIG°Å%¥%ƒ-N)I[ÈÆ é)èàò‰:¿àÚI\”%ê` ’¸Žõ’lŠ[¬1[à2H\`IÞ·ÀÀEŒ¶ÀS”¸L¶ÄD$qApñzUÒy³Äºu:$.,ÅZâÄRKĤ—¸¿°»$qÌ€N“Ä!ÁL‰ °-ÑÀ3—t¶ÀÉ–¸ž.tZK\ l7(qA4Q¦W`…F,‰®–t¤8ÎŒDò »Š`š ÒSŠ:Øš¸ãÃÁ¶“¸Á‚M„9¢¶XGÄh8ÜŒÄ[Œ2›o‰ ’‚-ðø$.HÊ/ì&K\‡N)‰ ’Žˆ7 I\€yÉñf”¸ Y…x!ž0o“õo‰ ÁEÔÁ8MI˜·à$Ä›`'Iça‹I$qAòžãÍ(qÜ Ä›ðèÅ-Èñf· À».™$xž©—¸ˆ7áÙJ±½ñºÿLVFŽ8£¸c @½k«¸e À½X­‚¸g @¾#‰›0‘eôâ®–U(µ· øþm8á¾û4nQyÀ!`ÍŒ™0-Þ}‹9z2»“ì¾?ö×?]~ý‚Å÷ú§¿]^•wÊê? ð[òXð´O&a¼ÁWK.aÂÙ6¼±…0ð#y؈º‘´X6ÝØH,ÅXnÓµ}Ñk“a1Û¬áÌä–Ѫ^„ -ce½hë»ó|'[{qœŠ•ÞÛù~=™ç»÷ëXïÙ~=™ç»ÛJü±¬¢±^°œdä£Ä­³üßuûå‘à¡^BØ¢cú”Œ=›é;½%ŠøtÁŠxß÷×›X Ö7ߧŸ/¿þíþåŸþí­O¯—ÿø‘w¾”¢¨ï‚ñ'hNÑ Yùî þÀTúËoþóú駱Þ<ô -‘xZ×ÃE# Q†‹J÷ñ¥HG/w9W•Ó~½§¡b½ÓBIç»)\f6³–Q¯KõZäC“j9¹YŸK:)oIÃuUûr{ù_‡6ÅM C‘æ.r®4) =Žb/Mw}š™~–»TÌv¥kSÌ]–¿£åihÆÐE¹É¯½¸›ëPÔ¦·2‡®jCwºªØœÛBÅÎ×ahÒPwiƒÖG]Ê_×}z -rJª ŸŠ>HCÚû$÷ßK)·ÌN“ûÀ¿vàîF Œc[¾µÓƉqaûÎnðÈýr+Ÿ_êÞ.{OŸë=hÖÇíѺ×ÀLin‹Ê,S1ש·”Ðð®‡q RÏç¢és‚Ë¿ÛtØOnÓOºÌ‹0ácÀÐÑ\cóøooþ綿“x -Ò8¼:ÝŠ§fš1{Ø?êg*·Ìc(ÈðçëýÓryæÎoùÚÝe/gœ¸ºÊ¼¶ƒõ:ÇÛøúaüœ4ßûn³&| /´6§5ïçôŸ‹Œ;;ôÓ—ãöR½®ÜVں¶îx®6çùAC?,ÿ¨ŸÌ†çõÿÇùèÐDõÖë:6ð}õ”~û){Š¶Å4øÞûÞ|[1w4ñªã’} 5z—ïÍó!½8“h‘BR(m2¼!ƒ([¥vö0©Özïšv©­köF˜?بìpþþ§ŸþñwÿÚμT¸ßü ê·ûg†ÃÈ–kÏû ìV˜ü¢oÍOVÏS:4¾Â¡”KÝÕ†zýº¾v¡Ú€Alîö®—móÝpŒ½Ýl $y¢îø´{⥔r¨Á÷o½£¼ïv “yž@(:Díß òENó îVl-!©„f]qR]»k™¥î\ßëÚuÜëv8ÓÛw‡GÉó’ýâ»î6ئ†²Ïƒ—îÛ夌 Þ²z¸êQ ^ÙÝŽlܼúm¼È$¾M7UšGvôd}09Z”#þdžo Éÿ7°Åêûêeô'Œû¾î>å„97¥­ø±&Í©±Œ¢W5¶ºMr[ýåuþ,·k;æ2Ûµ^ç³æŽ8þþm}²ƒ8¯+z =“ðNö1âÎåyÄu³ƒ;BÜß=bγ Ë7wY°1\èqBÝh,¸Ù¿–”\ zªðˆ±d76‹jÑ–‚÷ðx2Ï“à±1 sþ9ªä(±ö¸Ý^ϲêräX£M&S¢c–å¿[¿žÓpùºE÷ÓXÍ;®|~.Ÿ^Ð1ƒÝî×Y—ëØA‚uT±£ç)™GÌt2Ï“ ¡›õ¡X{ðKù,lbÊ^¤®_ÊgÙ'ÍŸ•y>è£{84ø†_ûš[™É½L¬h‡4ƒŸgæ2_e†[ï#ÉlXËme¤µ0û—\wÝ3Ü /5>A¬µ$ ß>FÞ¹<φ<¯9hFCíÍ)¼g$ô:ÿ<+”UÔÌuòÍQÑa ¾mëÊR“Óš'ÕË03œPÍûU˜%fÚ‰ˆç^7Ž£äg4ñC‡è<™çéÐiyñF¡­jZ(gBôý~ÀÄM¯çëj^b*ñì ”d¤ê}”>Œ‰gÞ¬œç¶òÍŒSybßý/Ç÷ŘU;Fä¹ãÙçdžgCaxàg>Ñj-" ®¶vªS hýD¿¶5Û?¯ÖãŠ_ó%¡í“Ï+ù­úuÈ÷Yè#·µ£¡Ù¬Ë¯m5z?Fƒ_iÜ’å›w‡>™çé|ì‹6*¬{uäÀG;ŒßÎ3O8“>ê‘ì¬Üað-9yýœÌótˆ;ö5y®EGóù=|&!¥îðlâñ¯—ô͸K£»neÃÌ£3ŠfNÊo—wm£©yÿ`•Ø«<™çéPyìOnV§i~¯8ò6Zœz|B]æ?÷ðE¤>Ø©~¶¨kå5TݸòO~©#µ~päÇ=ŠxÂà7ò1J»‡÷ùNæy6„ƾfx#Lˆ“V²—múyÅÌ{‡n‹¼½|÷Á߇ë…Õ\ð”îîe¾7êšï_núd‡w÷?°ïy2ÏÓ!øÁ¾ç£K>Ý*®ž{|ä¸)}áïkÏ\N#Ÿí!dDåNÃv­Ðvàwoî¤ß†zŽž!™ûùv€Îø¥'ó<:ýÒÿ`zÞoòÑ÷xÚwßAêàwž½»y2ÏÓ!õ¯Zñ"pдÓ-ñÉŒÈæ£M»?Ù+ïãÏ1Êré9Ð3þ¬™Ú!={9ÖµóÌé)wîlî"±û—â¹g}Næy6$¦>éƒQ<äËQ”ø±Ý‡"†‰«Vwʈ¶•”ŽGs§`,G˜=›ˆ°æîǵ­?—7Þ=˜ËÞÙOÝ?4Œ¦Èg7>@àÉ­¦óñšÊÃ#5|þöÕÓÂÈéÊǹEeI×aÓ¹£õu'g>µü—–ªd®ølS>×î~ÊùpQÒÿ’üç‹N´pÿó¬¾:¬²8šzÑÅo~Ó—1ÑbNŠYX¾ªVéš°Én Ç°¤9¥u‹*_S+œ_Ó¸åI{~W³ña`¤j"›ÃºW,‘Óë­˜[¯]Ä|pw\l¹ò͸p© {4¦íöŒÅÚªAè¶B5T×ņª=|¬FýšÚ˜óYÕô±ëô0ü5­ Ãp oTkgé›(åã‚kQµwj2;•šÔÕ”m@¨“ º]áªM̦F]¼ cµt©¦D·‹˜®”æÚH…–*ö¬©ö|7¥CÝÆÖñ‚,÷Q‘þ%‚aŽ([¿®Å|XRkܘðvqÔfb¿]\R»ÎÊUèB;”ÛÄQu¼t)Çî÷oü:c‘ñ‘uüÿ;U¸&eæÉ_45Hñ±“ýªWÂs³‰×4Ê}c’z@Sèi‡îÒ!4ÌŽý¥£Ò3é¶Ô­ÇVÒ¡ËyHÕ<&©Fk±n¨kb¦°Ú?+G¡ªF º.v Êo+q·Óµ©êT«ÌëÛžqçÛ(mÄ5Ÿ!Úìm嘯Â[^Øh3ôc­¤‹j«]¯lvÃ@ -]Üiô-‹ëlmìÄ¿‹P¿öI&­¸·ËŸ/y™3Ö>ÍÀ[ÍGb^—¾þ7Òü„¿ÿºÐõŸëŒ÷uܨƦqëJ<®dü -ñ¤˜P¡ -Û´„†Úïb{ZWÑÐÖqc¾¦ž]ub¯)ucPÝ´£Ðõ>ĦA×h;ôŒ;Þ¦^lŽÌ·ŽM_°ï—¶6ÕX>½³Z…‹±9¸Œ]gû’|ŠÄ'íK:´d†Œ¸ ÅåPaê–­Ìd@Ž•´®gûi -Ëu ƒÁ—¶ë¨/³cÑU–[1$ lÇ‹±`Òc¥ie`ô‡§×7{1Ù×ÀÒFíMú6J[ÃXÚO–Ú|é×—mͱù`‹:y{µ²ì0÷lsrúÔè®­.FŒÒس•n¬”ªlUi](]@¦ª¬•ªM¾d«Ý’[Å^ -ͧèÒ?=íÆQièÆQñ›´\n]ñš·d›äF8‡Þ:ÕS6_2 -2?ænRê)m•më¦uÝuµ½Í nˆïž©êÙÜ&å‡øï#sG<@TJUÿåû©ÅX¨Iklþ!*½‡ÿ–@ 8l•¼;ÏñÞÉv‡¤îŒ5a8²VÍ?)&›ðEz¬Šª?7&gKJlÒû:@Øê4·=î5ÿ¦LÞ{ÌDí2p¨È?!eÙéüËAÉÇ¿æm |Áj %ÿû‡°0 -endstream -endobj -806 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 805 0 R -/Resources 4 0 R -/Annots [ 807 0 R 808 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -807 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 638.673057 101.433071 619.923057 ] -/BS << -/W 0 ->> -/Dest (cb213-1) ->> -endobj -808 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 577.726036 101.433071 558.976036 ] -/BS << -/W 0 ->> -/Dest (cb214-1) ->> -endobj -809 0 obj -<< -/Filter /FlateDecode -/Length 4510 ->> -stream -xœí\K$Ém¾×¯¨³€MŃñ  È°×°`Á ù øÐSš–`ôXùà¿ï‘ñ`f²²»¦[R Øiôt%ÉøHFV¦½ü|gñ_&»ä\R¤ëíÇËO³¤`ð¯¶Ø\Töoñ›ÂÕJK°%æpý‘ >-©—âõå‚ÿv—²ùËå?qùŸm{uK)ÑÙpýóWf‘_RNÉ…UÀ2Ö+¬–”\ˆ -+ÚK,EcAŒÈÑ‘• •4Ä ç²ÂÊÃS²þÈ*Ã%K bØ’²rXgŒÊƒ †È;…gáô¨ñœ]b.®¨¼žMNãy¿ÄD6XW–cÈ K(Æj¼Y‚%2²ÏIãEÈâ£7*²¸œ¼ÆKÅA;EáeÈïÊ*²˜h¬Æ+~ %©¼žÉ)y΄%dïƳv ðM¯ò°£`¤˜žƒ,Ñ„¢ò KðÆi<Y?8|ÊYÙ­Žw·7Þj¼YœK¤ò Œ¯!€‹Å¤¨Avи¤a€Ki¡âH—ýB™²†.—…Rò ¸Š%i8àÏY ¼,¢†ÞBÂÎUyÅÒ°À;È‚xYy <Жà-xÀ-™b5,ðÀ[26jXà¸ÓhXà¸>GÒ°ÀráJEÃÌõÀN <@Û=iXàº>D§aìzÊQÃÜõd†Üõ=UdkXt\¼Í^Ãîzk’†ÜÅ6rÀª ü0jX_Y\ÎFÃîºlHÃ òÒ‚=è5, à® X‚¹».§aw¹¨aeþ5îB‚ž@¬†ˆ÷à9¯a[Ò pׄ?eO­›KÔ° wm†ùT^Yl‚Æ•} -/6Æ¢aAî"Py pð™5,Æ/ˆ¨NÂܵ‘Zãw­+FÂÜEÞ4,€;, €†ˆo‹5p]•WSrÖ° wKNÃDáÅd ‰ÊK ÉjX0^ òP "pá½hX»&x¯aAîBÚ£ò ‹ÏNÃä5àaÛ*{ÉêbBãw……5pטT4,ˆÀ]›ÄÄ—5,€€ ²`§aAÌ´””¢†±ðò©¼¸à:hX ¥ò %Ò°‰õR|É$Y¼s$Y2hX€Œ`)Ù·Z@¸‹ZdA¦¨alÉ…HÂ"ê1hUãE·ä¸Vy<›4,H‰„b«aj©%cÓkX€_8]Ò°b€gIÂT ¶Š†EKvÈÌ5ž…,H²5,@¦ žµdY jX]F•†ˆÐ¨%¡jG†ë̬aA¦¸pª¨a¶ *M ¥Êƒ,€ `.¶ƒ†(~Pl¢ÌQyÅÒ° gÇåfÖ° gÈâŒÓ°¾ÉõfÒ° È‚ŒOÂbâÂi²†€qðŒÑ° ØŒzÓ‘†Ø—\of Š7¨7Qâ)û¶øÈõfÒ° d HQTdAÒT”} LB½ tÒx²¸B”¹ÞÌ Í@½‰Œ^=È\o&õ¸¸ -Iãw‘™F pz™­VÛoHÈÏtfæŠ3«'Ð[Õ#ìE´Jê™ør¤@LT–9ª§«`Z£à/ò1:`Âzú$¨"Ü!!fæÄ‹,Ø„ì~Ô³™¿Ól=ûó/¿|Bð½þñ/Ï¿¨'eýÇ"A¨­ ù¶+0w@öm±|Ï®ŠÉjÌÓ¦2ŲÂvÌÆb!àñ¿ëñÃFVˆÊ¢7*¡ÈÕ?ÏfÚÄÀž::¡ FEà8JNYîóªÈ 9A¹ŽV—UèHÌ;•¤ÉúpŸ’uê¨pà)æÛõúÆ>®W9ï[õúÆ>%«ö,Ü´š“S2Ä=F“¼€Ë_ˆú`—Ü`å;òÁˆ6¾8ŽŸET>†6ÿñ‡  ´I±žÒK!>2:‹ÐèÂõ‡/¿ü—ï¿ÿ§þèé‡çËïÿ(N¿úÎ⯠øEÞ@n½öøìøÏø{8 ¯mPFÃôÔøqösà9\;æ}Y¯5÷és9Ÿ@»µq¿ˆ~4û˹ªÌ Ù"Ú>­s@Ì9nl²ôµò:pøî:|¸'ã¯þëúÃ÷—_ÿ°u’ZÛT'!”¢ÈˆCø©_½­Ë{üê/áN¥œºÓ§7߃2R“ó˃2ŠµqC¤øÕ]‡ïËt€˜“§c§¾óÆ>ŸÍyPŸy5í¨+8ñ ¦]wzncÓ¸ý;/5ë³e˜_fzÞö§ÖgZë|¼Þ¾Ê×é·ö·I»~·Ýì:Oçw:}ÝÊ4Ú=mé¦&øW¼l¦^ÌÛª€7öùt^vž¯Ñ.2 ¯*GOqjŸ£m÷¾3¯궞Æ2H‹o<©4ïpǨV£¥ãÞNvAßUÝcÝcò ì°ßØuêچ׎ã"¶6jc#¬=.Å ŒÎ_I±.Wkÿú7ÿþ¯¿ûÍ´vEwÖÎØû=ÃVòuU¼ö†fÅÛÊ ‰s¾××ë=·q‚XUší™ï¾Š$¬4znóÖâpŽÅr!gÞƒž¦Œõo”ó¬Z|“Ú¢TÛïûêBóóá+mf;WÅ+%á_URz¡¡®qd&¶ÒM­!dÆ)#ÀV‹ÄcßÚ*c[^”ëò†OÒ=“ª­Ë7m‡Ès¬ZçñìŽãDÏ·ø«,Çqª…kN3×Xåöëoµ¦°Hµò³°Š.¯Ï÷ä5sÍu\šÖ¯2Xu üÕ§ˆ+eì*³sëÍÓ&ôµéJñÔÕÞ;Oíëîý;íyÒ/úÜvÅ«ŽUÓÆuÝéœoÝ©NÃ~Ýgë±ö|=ÌïòŒÜÑ(cÒÚ®Êg×þÝNÔjâ~]uÔûÄU÷•ÿÔÚ„uü.Så5ߣ$ôLmü½]v=ÕW:èKè©ÊóÔÆës†ö×Ìù¤¬Õ·Ú:ù<#`hÇaÏíÌû¸v<–ÛùEZÏ*\iãºÇ*=·ö´¶å3 >ƒ¨r!×m¿ÿV{l2ª2õ_ñ¦£žÔ]•ƒ®ž¶Å6`¼]£Á½Hp/ -¼7ôê·cíô×m–î|2tl‘~wgšç3=‘Ýë©ëwŒCs=AÝÞìG}?õ=¾L߸Í}þp¾^ê×Ïm¬¶†Î˜læZûþY±áT_¬~«¯éõÛ±ïT^”r[y«>7ß‘2‡õºãGèû¢ÍUç1Mú&ËÜ3±ºç¿®8PyÝŽ ÿƺÚ\ôÀž1R.¹_…\Ãæ7a_a‹pš—PÙë«ûû˜77,ùr\/r瓱ßéßÚ‚Úx»8ïv1©êâiæûª1ð=V_oYñWkc ú•ˆó{GoëóM·¹>°\ CÞÄßñ ïj¹XÓ–}©ØÊ-õÐs5Ô½‰¡%SbÙMÜÓ3Û·F3Ìr«—SãÐUò¾nKBÚT¾¯À;Ž³Ipc]ÉÉš{r¼»p;Ž]õ}›žÑƒlO´éiÓ{ï{î¾8é@-ÞzÂó–ÂQ)üV{ï ”SÝúƒnÅ Ï­ßMŸóÝþ7‘|D‘Ø­~w6o8›·ê½~í>ÿJ?Þ|]yó¦ëþ†ëþf«¼Ñº»Éz¸|_¾ysøsÉõÞÄ|n?·oýþÇëlsƒÿÞÔ§\e]½Ëÿ?Pgk›ýõöÙ)Ž¥=ŽÉ"Xâq#vs?ǘââÞo”Ãÿ{ü%¸Åä%Tß c“Ï.¿rÀÿÆ>Ÿæ€Ÿ0 6e>àßÖ‹¯ìCÜÅ~®~7á8Ø7kZåz÷-öê!»ê˜.-Ñf2N=WÇ-¢ß¶Ç÷RÀ´_]ÄÏáñ4àp›W›¼â%ÇgûóMW»{òi« Éïíòùê`¢’øQWM û»Š5 ÞÌæ›ì­O6¸¾Ñ[1Sï♹áy£w‹¯Çøb÷oø¹qÈ,@#šC![ç{šÉ®,rÆø=Py6RiAÈÎ $ÌøÓ%ÖG¶m}m‡‰y±Žßëu´p LåʉgãÉv³Ùªh{½Ó³¾ í§Ñª½ Íð;Søyùõíi⢵ÿ©üNj囥fÝVëËŒL%g×/&ùå D -™hÞy…ÌÄöÑŒ±ÈŽ»*v¦¥ì[kLû8o½0­ù.èìz7~ȘLoäëc¶ëÄ -=£=’yõf±Ì/Ë‹oWqˆb—Ä9 ɶsÀÙñU2ÒNA G¢ÓM²£.åH ÕÅI¢KÕYaþÞÖ%aÎ3âí‚iýžú¢Rù5D}¨®]gÍ ¡Ê †Þr„†Òí¢¿3}h‚1_X_æ°sw$íJo§nµ]- K¥Ñ*Ï®¥k~Š2]¿ºrŸv·'¿áPìãûÇ-™ï† [›,üƈ·K ±'ùå’Ò:L¬0¡»ž$z1î g3ýeRSîªÂÖì4~žÿ_¡" *#Oý`i¸¿Îb^ÍIxoòVI’ê¦Ki¶ê²) Ÿ•úB¢´gЭ‡Æ6T¡2AO¥‹Ç 5ÀcK¶Ãë™!¬ëg`¥$š.¦›ä0\ùeCžr†±Uƒ“E+Ü=Œq'9ì|‘ä4,nùÝ$CÞ1®ðùN¼ÕÀF{2»~î“L²3#ÚÍÉq -P˜ä £/•Üwë@§úxDû87™qo—?]j˜s>8ÎÔ£·üª-™uÙëÿ¡Í÷øýï ]ÿ­¿ÛaãÎ ´p[7dÉøÅŒùd~Qc'Žm#ˆŽìO²Í³mèÞ0â¸s¯iv7Ø{K;Ô®$†©CAvÃuõB3á„xÛiq$2ïµÍ Øë¥ï‹Ã4žß -ҥ…\.óäù’¯bH\qÓÒÁ%':âBŠK1a™’mÄd‡”“ Õ³ü$Máy.q!Ò¾ô“G3ÌÊ¡;­®B4A -ìå…˜¬œ´läQ„þæíõn»7‘cßLÐ>¨/’:ÆÔx:Õ×˸½1Ç_ù¢oÞ©¡1–{Ï$gn™ZÐæBú¨ æÙ­5rRê´Í¤=P ¸ÍtÚ1u™bëÖÕRWœU-¥‘SL꟤¦ƒ´ÊPbV‰‡¶­Dã2q©È¯¦öœôf~£f2Vÿ&*µß/›#”úóÿÔõ« -endstream -endobj -810 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 809 0 R -/Resources 4 0 R -/Annots [ 811 0 R 812 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -811 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 426.873057 101.433071 408.123057 ] -/BS << -/W 0 ->> -/Dest (cb217-1) ->> -endobj -812 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 130.126036 101.433071 111.376036 ] -/BS << -/W 0 ->> -/Dest (cb218-1) ->> -endobj -813 0 obj -<< -/Filter /FlateDecode -/Length 2870 ->> -stream -xœµšK¹ Çïý)ú¼€µ¤H½€…6Hd‘EÉ!ÈÁéxvŒxsÈ×ÏŸÕ¥GuÓ3~lÜèq©’Xù©n>^¯ªr¨µ•¬çËûÓ‡…’ÿ¶‡‹Mý#Þ%™´„Ä-×t~o)¡´K>?ðçærmþtúë7§ïás ­åÈéüË;S©„RK¡ä¨TÄâ¨R -¥Ä”Uæ€GlÍSÁŒT4꽪À m5y*˜¡1VGUa†ha¹W5˜ «§‚ÜRqTL°ƒ#‘«ƒ!¤*ÑÑq -“ž=]äk‹ÍÕè¸DO'rQNìéZÈ9§ê鶤FìélI¬Jž¶¨Ôâé2l‘,äê`K¬E<]-³Ó]…-ð®êê` ebO×$¤VU]]ƒŽj©÷ºH)¤*BžŽ9$ø¦¸:D)GaK¦Ô\lIBÑÓ lQ¼<8‘¤ÔêDk´èöt ¶ÄXÔÕÁ,¾G€˜a •ì!ŒKb)A[T±JЪգ@¬-h)âa ¶4·âq@ˆ¡‹ì@¶$Í „a‹"r]l‘¦ $Â4ðX [¢ŠÇmÞâ±@€[¥Æ ¼Uâì±@\˜I Ä•šÕc¹p¥æ±@À\;= ‹p/ Ô•”£ÇvEköX à®(“ÇwEp§«ƒ-p` c®â±@Á]a* ÜEEXÕ?Ì à+!ÖJ Ü•Ôc‚»±` pu% Åc‚»1á¼܉¢Çw£Æ챎Àr³pI~:[XØï¡‹â±…ˆ´ÄcAw¹aûsb´ -\[öXÀ]®X>W×̸·˜åÀ97 ÜÅF% ¸ |V`|ÀŽ=$p—;µ§w96òXÀ]ä ÉcÜ!0Øß\×Õµ@­V Ü–¢ÇìÂ*HâêJ€#±ÇÀ8òPÜÅöÞ<dp—’ˆÇ‚ î’"íqu°EjôX€¼:„­ÓHVEÂÓ»ÄXaOî•æ± ƒ»¶mz,Èűz,€YpôX«†VJöXA‡”ÏÕå€ëä±  Èru°E‹z,@bš´ê± DØ"1z,(¶ eðX€Œ 4Föí–°îâÖ -[)z,ÉBmª JʨÇ0«ž.ÇP píê*t\<”¢[1{,@-*‚Þc,]òX3 cõXPlA¨x,ÀVjDfîé¶ ÉöX€L:f5Âdƒ j¬¨23y,ÀZSí锬ά ªæ`©¢Ç„ *MÒÕÁ`ÂcÖÇŠíä±ÅŠM”9®¶HRµF+7«Ç‚ZaK¤è±¾iõfñXж ãóXÐ(K“=ãÐy,h\QoFõX€¸´z³z,hB¨7Qâ9qÛ$[½Y<4…- )Š«ƒ-Hšš·`êMÐÉÓeØ›z,h9[½Y= Í@½‰ŒÞ=¨Vo÷¸ÜM dòtà.2Ó챨7‘Ùzµ=¼© ?ó•Õ*ÎêžЛ°·ºGöb·*V#¹‡0•eÍU(™Üc‘鮧OëU†;왵ØC6„…"»5Çl&iv=ûå§Ó·o±ùžúÏéñ›í¤¬¿Î/š­°N"±a¹«°u§(0‘kGÛÌ@Ø÷·˜ ¥MPÜÜÌDlÄØlÛy|à£Á°×,f¤2Ûsé:,êÆBIv-ýÜ[~uK‹„]™Ûq\×Æ—¿hØ¿óýßHéãbïF}cÃZ úaCo ÐÚæ´˜ú™·üÓI¨-‘¬d´’ îˆX¤å¤‡Æ¿y8!Î÷á¯GÈã’%u»±%)æ6ÒùáýéÛß¿yóÛßýÃ><žþövÔèš^¿b|ÖËõÿÄ»<ïrÝåÿÿýüðæã㢈C„RËí8® …äzë6µ¥Ë}èm¸Ç£ºêÞííëQ§ýúñx½™úÝñ1Rœ§mç-Úñ®»,¶ÑÒ÷öØß?LºØìçýñóyÛY`ò6áßÿð§?üå‡9á(ÿ)ýo1kÍ:³vÿ\®³„Ê{»N¦3ùãöDשŸƒF;EjV¯Ý º-ì?_w›nŒÝ­û±ôÂIÙ>>ëúŸxÏ×ëÿÃóí´úק·û[×ß}Ådæ'ÃgÊ"—£ïŸË7×¼† ÖáÇ}ÂFFçdGðÆç¶ìjcò_lòÂ\ßïg¾aÎÑ‘Ñ¥}ÿ„©Š>KÙ<76ï®îl“áªÁ#yŸ$Ë.Õ ﶷénÍW·W ²€³É´ ß!A{ä=,¯kò¨{Xä%DòÞO퓾÷vŸË5´´ÞôŸö±{¶>Òò ©?ÃëÛeüpÊ[éÁÛñ¡äC]ŠÊ¹I[—v¶rµr_6Þ&šÏ¹sûVïÃhµÑGvögußõ[Àåboÿa[ð÷'F©`óoap=”§MŒÊ#ÝŠŸNö¥Œ:bäjñÅ&Ü?ÒèK¹7²ËiÛèo["TiÿØFç¿×¶ÍÎûS¿Í’eÔƒ±_Y \v„¸3ó½Øžžïbûb«Ù¯ò0TÂÙ¡®mg‡óÆÅ,´Ó½–á¦8j·‡í dL]žÂaú:u¼,o˲œÏ /' +·Ò'WjÇ齫>»(7ºL£Ú¦Þr,ˆŽurD—“#|E½kÅbý²jv(qÖîH:ÚµÞÎ]Nn»½·4VªŒVuÞÚúÌOS¦«Kïȶ9ڥž©_b„¤<Š³­@_kTˆÓoâðrB­wâ§S*Në4YA©»Þ*”¥ß!®4ýeJQHïS…Ðì2+éìïiH<ÛÖáRv,3¯æ ›C|œ¤UžuÔ›Ö2Û.ÓÅ¥ Ÿ]ç‹+9í º£õ˜±ƒt™²E^Z7Ï 5àqóðº!6„õù¬\…ÔÀpSœ†+?ÄÓÎ4B5Ñ,óâîiô;Åy°ói—±âlglÃÞÑïâó]xÙ66½›ë×>ÈG»ÝlNÃ…)ž}ÚÄ=Zìwýã 2oǽœ~>mÛræhµ^¶¯ŒÖ¬‹ÏÿE›7xÿë¤ç?ö3Šë>iÐ"¶ÄëNf?0¨Ÿ(¶tá›EuÁþsmS÷†±Ç¸ðZçí4ÁÞ[ò (í*Lsq®Y–™IÏ/7³8™¯]›¹a_/¥?†;ÝêVáb}\Ö©“¹%Ÿ—.qeMç–­ÆåF\¬âr°MËfšC®ƒŒ©7ûu] -±±–‹%°K™:ÛìÚu—mO±4A -,ëÅÚ±ò:h;ØãýÅáõÕë¾›œ{˜t }HŸVéx0“àùRÙ.óñrì9²Õû=xç ¾d‰=IÎ ™ZèábõÑEZçm{ƒ´ª]v´oQ—K·)ï·õiÙž¸º³TFN1¥?¯3ÖU“˜ÖUÉwm­ß¾ãlIF”Vw.óéh¶¹dudy½{Hu¶”.»[c÷Ýö¼<†NCòÌLiÞ–îZ~ÿ¾$v®÷ÜœBx­úk;á:Oõ±Q+‰²ýP“jÎ`ˆÚwíEërTòÙ÷<vrBÒÏNÖ‘°AE¨°ýäÖ7á Šö²béªÿ׿­‘ûHŸ7ýzä^a'CÛo®¶Ãºí€h\+í'–bIoµ_†b÷¨h;²÷åp„²½þyšç -endstream -endobj -814 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 813 0 R -/Resources 4 0 R -/Annots [ 815 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -815 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb219-1) ->> -endobj -816 0 obj -<< -/Filter /FlateDecode -/Length 4003 ->> -stream -xœí\K$· ¾÷¯è³•õ ^€áƒ_ÄH€E|r˜íݵÌXç¿Ÿê’Īb÷tïÌÂÏ z¦‹dIE~¤Tw´ø}ãð§3¥Ôœèxútø|°&G‹Ÿ&±:hì¿ã“ãÑYÊ&ºšJ<~bBÈ&×ès:>ðgs(Å¿|søÏúwô¦Öä]<þþYL.9Û¨°*XÖ…£ÉÙǤ°’3b­ jÄLžö¬ 5¨–¨± y_V² {V…>;ÒXPÃÕ˜–³ÐÃykU±DÁ+<M‚Ñ“ÆóΤR}Uy<—½Æ Á¤L.:WMJ)GÐ%Vë4^„.ÑY](”¬ñt )X•]|ÉAãeèâaªð -tw•]l²NãÕ`b-D*¯‚gK.{ž·ÑÄ‚ÕxΙß *…IJYáyè’l¬*ºÄ`½Æ Ð…ð«ñ€1äR”hõÝÁ§ñ"tñ>“ʃ.˜| |‚.6' A†qIß³¡êI_‚¡BEC_ª¡œƒ¾FC©f ‚uày§A°Ð%RÒ 8èBˆ\•]B% ‚‡.а èâ)hX€¶oÑ° nÉV§aAÞ’uIÂÀ…šVÃ‚Ä %‘† Wª`nvjXº÷¬aAꆘ¼†°¨$ p7³p7œ©ò  X࣠® ¸œÍpaä5,À¬øaÒ°¾b|)VÃîúbIÃîúŒ ò²A  ¸ë#† enà®ÖkX@À]O>iXG1À«VÀ]@HÔËè€Õ°ù<4,@cÆ£,Ñ° w]EúSbhe\©IÂÜuÓ§òªqWâV6.¥ªaAî"Q "pðY4,ÆdT¯aAL­ñ€»ÎW«aAî¢nˆÀŒC aò›q®«òª±µ "p°ä5,@6¶IT^6p$§aÀØXÔ¡$à.Ò{Õ° wm AÂܵ„²GåA—P¼†¨kÀCØ*1bÕX¤ÐxÀ]ë0økm®$à.§M Rf‚/@Aƒ*ØkX -™šsÒ° U J>•— Ž£†Ù‚Y*ºP& PX›jÑ° {è¼×° {è‚’AÃT¦:TßêòºÀ]ÔõATŠÉL©D䘰ƒU5^ò¦dÀµÊ+๬aAÎdŠ†XK™‚ ×°o¸\Ò°j€çHÂ\¡ BEä"S<*sç  Šl Pé‚眆ÅCTƒ_°ÊLVÃdh¬%ajG–×™EÂBÉp©¨aÂ+M ¥Êƒ.€ 0?¼ØŽ`ñƒÅ&–9*º„H”ây¹Y4,(ºxë5,€oòz3kXP-tAŧaAµÉp™¬a`$騚º·ŒîçŇŠîCEøPô–Ó”ÝI¢ô¡¤úРÎAÏ®¦u4ûž^#÷‰\.Õùç¸ÿ¢ ìÊ:c Ÿô€¼d+» Û¿ßkðýÉ‚/ñO>î¿Ü|Ra1biˆ•g¿ßkð½ßÁ§Ù}%ç…€| ¼×Àû“Þí%ç~½ÿÃÛ4k7Õà%CTÚå¾JŸyKóøöÓáÛþÛ_þñ3Öo?þùµ®àãð!Þ–ç ÚÖzþàا3ð=B†ò":á¼¹Xñý„O£}ÿ¯ãÛŸ.èÁWÓ‡Ü5wÑNÃ'z|ʹ›ˆnb:7å*&{ÜÿófbMl%¾QÀûÄ×k}(½eW¾ã¾›“ô /ºåZþwz<ŽeYzw¦7#d…¾È¶‘|íôvó4Ôhí°!ùî6\×£µËõK»EÛ§Ù^“óë1´~O oÓ?·Ñ'.>,4¡ûy¼wX›¶Öî‹4{oÇuïöÿÖtkË7Ë`$ö´h“5Òª„!E†…Þ:ûP³útï÷í¤À[Ü|ž³…ÝbÓïŠÁÛKzØ9î‹ç4›ø³+ºñí4É’OUi»yîiiSÄIk¿ÎÙï6lv\yåbágo›žF±{äÔ½}8ŸÛ=ŸÏ™Q EÀ¾/Ýó¥×O=Üu»ÓÖ6 S]¦Ã‡5l›­®ÒV«h_ô–>»…¾§Úl²eu]î$ì¸ó9ªÏô›t˜ÙÚÈû1œõ¿jË´µ¥ª×âû¢Gk·.ú<Ì9n²ËØš|O Ý¿ºl:#rK%blr.º=Ò2žÓâŸ]¯BkºÏ5úÆ—·¶éò[Ÿ¦%K4?ör|WíXvv¼ÁC/Åæ[¿[ù€ˆ›Ž7M×*ìwGlÞƒÏÀk6Ä€76¼/˜æÞ/ýXb¢çž®C÷—EŸ~n×ï¥m>,Ç$ò½=Ë_û}º¦‰³&™UÔí{·ÞûµwõÊë‘ëšaβ8–õDÞ—×ç½L–'. _ ËïÛ¹/ËGG—ôxv–ß·ýˆx g íÈLñBÖöʋϼ»äG[T½jÛ°ÿȾÿêÙ±÷ý:²{+„¸•ÜíQ±ÊtOGeº?"GeÞóD·t¯QFÛW÷sô ”•„ŒR™½mà -æb»·Û°nmø2ˆ‘øîÔ@Œ};÷!FjwŸ~ÄØ·ý,ÄhÑ‘ Gô¦™«K»ª/[TVa£+þ»ŽÔ«6,ÛqÞ“›G?[» §Õ·­0‘¦×ã}«ãDZÕó4ùr¾Ïµðí”ùÙ‰§¨oÔ|åš@âôðæ•÷M„yù!'Þ}~ÄïÛ¹/â‹ —ôxvÄïÛþê5•9}:߯õ »r÷Uû­¾VŽ/pŒút„•éÍ/±£8Ö%u¡—™cG$rþ~ëu?Cc—p«ß&wû ]ÎOyï9ï>Þaá²µð¥=˾KÀxá…U$µ~Ú|î*nTï}ÜØ·Ó}¿éÓb㪱\ÒãÙ¸±o»Wxc›û}'*îY'é÷xŸÝìè \Ý`W÷$™-›¾›Õü‹íjåþ„óÖÜÏ̼[Ðwzúnà -ié_ìfqÜP!ÜŒ—,»mÜ‹,úá¼;¬ÎÑÀÀ^Llž»ÇWmXw6T°·ûå°ÁÃ2ÿQ÷ßmõØ}rµ¾Ü¡â¿Üw;}Gäñ¡îâ.z ùrÍŽYccÈ>¨®ÈHÈBÑËWÛ®_B K›ÍÉKe°RqqåîǷˤí{{ŒüÈtª¾Vqòr“ê "×ïcUîY^zßßáúÃ[iT4Éï @ñd¨f¾Å^»¸Ù.f~à =ñã$ˆù}ñC§¹º/Ž³`FÛq´K<õ=‹g³2Ýü_Ç}ÇuéƒþabŒô·Ñ~¡ÄÊ6†ØÇ°›ÆχÔqíqA›Šqžß'pLdØCr=òã)¨3ÉõisÍÐîxáÌö†ÏCjy1ƒåg5ù9ó[ÄÁ"ÿ¹Mø'”dR/ -ÚCÔ¶‘‹ï“üxà‡èI!ÍÊâ 2—¯v´En*v:¸Àë7’!»|ÕLˆý«}4|~ì§ñà d»Ph·÷Ÿ;Vˆ83¹=™Go[Èü"‚bÂr”†*1uAœ ’” Ο$»`1î‰Vt7Éžº>Ž\¦K“8T—¦sbú»¬Ïb:¯Ot¶ÔG•Ê?÷¦ºu½´ ”ªƒ»ä˜ó¤ønŽñíM&sðŬñCdgbéŽDC®v9…t:¨rKkqÌTRežZ»å§*ÓÕC†Wh'Qñg~³Šˆ,]}ŸIæBu̵-ÂoüŽx:D‘8Éíž½tœXacw=I ¢ÝA.vúˤ¢p\L…Ðì4~‡ÿž¡"*#Oûâh¸?F7f'›ƒ¼6’¤§:¤YÀQž²Â\.çá³Ò^®XEžAwH‹­¨Âd‚žkWAj€Çšì†× 2CX·ÏÀJI´] t7Éq¸òãŠ<õŒ#Tá콳䄻ÇÑî$§’œÇŒ;~&rè;Ú>߉§–ØhKf×/½“Iövd»Ù™ NÅ -“ “G3Íʦ;­Bˆ ò@6LNvZWú(Jqx={Þ•S¦hÔGIcê<ÚaZŽœŽ|ЃwZh´Dì…QäÌИ¥­¤ -j™§-QvJ¶ê´'Š ÙN-æ®SZNëfi#.ª•ò¨)&õ7ié(ge1ÊYI;Yn·g¼Q-…QEéÎyŽÎNÉQK…–äÙƒJS2tÚ¾oZ÷ݳíQ ƒ¦"iV¦vžw’_„_;êÔšTÿm;#®Ì§°‘¨ÉåàC{±ž-)Cˆß’©ˆ­’»ÏùÒg€eOŽï© -ÁfǯHÔUx£>ÍMõ×'ê§U«é>\|œ]½ïݵwdµæÛÑ8̼TäWâ.z ¿É'Û ÷ÁÓ²V[(í÷ÿ"Dx} -endstream -endobj -817 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 816 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -818 0 obj -<< -/Filter /FlateDecode -/Length 5439 ->> -stream -xœå]K$¹q¾×¯¨³€¥øˆà0t0 Ã^ÂäƒàCO«[‚Ñk`åƒÿ¾¿`e2™™Q•¬šÞQÞÙžé ¾"Éà>Ò-þüàðW&gr.)Òùõ§ÓÏ'k[üWs¬jòïñ“øì,%îÄÌ矄’I…}ŠçþÚ<öÙ?Nÿþ«Ó­Ë¸³7¥Dïøü×7I¢`RNɲ’Td]P’˜MJž£’Á+–¢% NäiŸ”À•ÌZØ ï³’”ÁF äÂ>©€ ŸiI`ÃNJ’³àÃykÕ40b‰‚WÒ›ˆNZšw&æâ‹š–æ’×ÒB01‘c§¥c䬥xáb–Æà…‘ÕÒÀ …œ´´^B VM/>§ ¥%ðâÑ;EIËàÒ•Õ4ðb£uZZ †K&RÓ -ÒlNyŸæ-Î!X-Í9ÃÍ ¦aFabRÒg«aw}¶¤aw}‚ -PÓ’Á p×3^AÓÜÀ]ÏÖkX@À]O>jXA1À«ZÀ]@ëæxÀjX}44,@eÆÃ,Ñ°€»®@ý)she\.QÃáSÓŠq =®Ì[ô²q1 ¸ E4,`à.à3kXŒ7Ш^ÃîºM­¥w/VÃîÂn` ÆÁаúÍ8 ÑUÓŠ±%g ¸ Xò@ ›$jZ2$§aÀØXØ¡Dà.Ô{Ñ° w-‡ aAîZ‚Ù£¦—½†°k†i«Ìi«Æz …–ܵ#¬¥w­MEÂܵ©aALBðYÃ0h`{ b&SRŠÄb‘“OM‹ϬaA² 0 KM/”HÃÖ¦„’5,H¼ï5,H¼Àdа)ַꀈ‹êx¥¨aÌäB¤aAâ ½ª¥Eor\«ii.iX¨b§a|)“1é5,À1—4,Hs¤aA*àSEè"“=,s-ÍÙÀÒEšsd^` jX}†—­†ÐÐð%ÑÕZYñ3³†™¢SQÃLxš@J5 ¼&4,Àøˆ³ÍÀù³ 7GM/IÂœ½¸›YÂœÁ‹·^ÃȦø›IÂbÁ ,> ŠFÌd ãH³VÂâ2üMO`^Š¿™5,(ÁÂß„‹§ÌÛ¢ø›IÂBà…a¢¨iàFSQæ-0 þ&ÐIK‹àÅÒ° Ä(þfÖ°füMXôjH ‹¿™Ôpp— IKîÂ2àþ&,[Í··^N°ÏôÄ,gV#ÐËЭjÈÀ{¡­’3°_ñ‘Ô Ø„g™£5€ZE¢³jØÀaÑ.ѧ>D! :3'yÉ‚iA°î›Ï±d W²]âcýóé×/P¾ç?ÿ÷éýW5R6ÿqþ@Qk¸±ø‚á\™ZUA®Dª‚ÁGÊ©,L:Àât&¡Š¡n˹ýâÖ,ƒcáÙÁ˜©oF}»ð†*D×»Ë|¯ûnÿû/'àË%¶Ï]Bøñöüå§Ó¯û»ýç?üu~y?ýñï ôd-g o?¯ø=‰•„ÿ¡6?Ì=\ÒÜŸ~óç/?ömy+îІ×mIõ6\²ÿö ºªÆt Àž%úqé*ï¤L®Æ\[c®­wï.òPçN} -'V ^ ž½”ÇÔ…}ni×Ñ<·zéƒöèaaOì®»ýŸ~üñþñß–n—®¥¯øÉÓïoøA%5Á~êv? K¾Ðiª÷ËPH:Ç Âvùýk×߯œDS¨p1aàW)õ°÷ÙHÙ¾¿ï.ò}ú›æV/ýÝ]àÆ®K·:ÜŠ$%Ê¿ùÁÉ ¼^þe·¡ÓDÓóµN…}E0¹Á°=êÌ›Y¿¡×­ÀñM°Rà´ ,Pp›±¡"ŸYŸ1Ü ßðçP–¡µánC—§¡Lø÷\W+g‰jQµ*V•÷8v½4|ë‹m±+íì‰ÏW„)U§W±Caº™õ³„ÉÃ[ñ9†bëâ†-0bŽ8,ó|â ¥¹X¯ÃõלÀò$^ïk1#¿;Þüë×ÏõÚ×ÑB˜iòqÃæ`zY("¢ÌJi~Ó…±®”@çd¾cìÇÊü­Ưشh^ ª0Bÿ3>û×Õþ±Ž¦N&A888$©Ø‹Rçñ°Ãp"¸÷^-ͳE±ã-jZ¾"È … ¿^Z A"·…c¤ÄóˆF„ü' ÄšhD{¥[2<”D”â!„ßÎúY,›’K®®ÃJ>ƒežÂeå9Æt‹ ­¡·Áy8¶ –Ù½ndÌ2(•³¹8­ôË/¨\^Ž0¢`TaLø ¾—Ø*²Ð,?ºÀ‡ Á–"A¶#¿õÓžÙäè%¢ OÀ@y{äç –yB—€˜MtÅÅ܈E^Dlž ³H5àŸÄ°º³ øz$B¾†Þ!“v*oë–›µQˆÜ üA{¹Ÿ8¥=YzÙý}çÌ“¦,~ mò4õ7üÞèѹiSÌþjúoÔìü¯¤Õ ?O~»áqn/\á'ëmÎü­‚/½àdYˬkZD&©Hìòö,+ó4*<¨1ѯEŸêº×/Ãxš¯š¨:æý¨tfC˜u¬•>n»ÌjYÜUJ×7¢q«Pk^‹ÏõÚ(Ö½°”Ú^$*µ ´¤àÚµ‡·£¹LØw}âî´âDÄ¥šÛÝM¨´™H›çmz«çÆ$¨¬pÉ]}3oÍþ¿²›ëµg}Ò’&d‰ÇŸ)¬ýˆgp{Ò–yšIKÞš@ºF{DzqØEÃl¿Ú[ø^ãâwèÿ€fké³¥._5Íf± SÙmŸLÿ6ÑÚ‡×Ä,È~T¡ÐY–øs®ËÕ·Ål¬ÌóˆÁâKVöÀªžÿÿ‘ÙxCM|r'6¯>Á ‹Éeh”u·i4‚"»œ•Ò‡Ú”e¯g&­ôaÛìØ$ÈŸMJé!mÊ)à;Dµý難*8êSš£)x–¥tÙæBLGæÞ`™ç™Ò²=/ø̬kŽ ú³oÝÔ½fß¾ô=fŸÒöK÷FÃÆŸÆÅä« ø`A¶ßã!iõ¼^Ñ+Ùt¢ ùìS6.Ëéš#!+ó¶eefœ×J· °ò†ŽWJÙ‡E§µ¯}¸Ä•Ú^&£àýÊT,GyS -psIç×»UnOõ±2Ï3ÕYNP¹…ô­\49âÒ+Ðd®Óá]†àsÚºÕ|¦ááf¼Å%\wÏ­èfŽrWE¤˜Ïì‹Öñѱ±"Ï#*IŽ´‰‡õ©»uµcqÒœ£ ©8ÙÐ2º‹{°ÌÓt·\“ÄrÒkd×ÅÝßE+×ÍhÑÊÕÊÂðºÀ¾Þ{BìK_E‘áSp´ÄG•~º†5Lè!‚P@Á=ÀQ·j_Ú½Ï?ã¨Ò‹=ð]}æÃ=’]4ÁñNpç=AùÃu%ù$&­¿‡6vua8E†Û_ÔmÎ~`ĺÍÙz_ßµ:pC A$'÷Í ŸÚ,ó< è唩 iD}(®š¹¹„³µýy#bÛSÇ>Ãð±Ñ ÑØ”mÒ8þ_úI¾kïhñehÊîkMC?ñnÇOEè\=ÿ]]ïÃá•‹Î׺ÎáÀÊEĨÖkÙí-ŽÃÊe‰2Æ)Ij×îƒ,ó<à#×Ñy²ñÊŽ»O8 »nâÞSûÒ÷œŠØ—¾÷0¬Öþã‡a•Ú^ñÊêÃ/vöÆ—ür9RverEq“Ü=¼-d°ÌóL®ëŸxe1/áÙØÄØv÷ŠÍ`në|_çmÃ÷ªc‹ëzÃ}õªcfýÆÉö§}éCˆèí„Ú}Û㧺3çHì´}`­žkÆqN†|)$ßk(rw6‡£õÛÁ2O3…äþEXó`SWPŸeoÚyøPV‡õJ•Ç-[¡´âÓ±¬ïi¸ãêD÷‘7îìo¥ø¹:?[åcà$Wgµ_©‚ªb;<ø,ó"“.Vc.ÿbç+UæúýtËr’K Y¾0%w—îÒæ† ‡Y€ ò­Þ—|v›Cë[ˆQ¥|^MÎoJè(Éä_”Kˆ%(îß.÷MÕ“êñÄ‘N5{$PN—[nÝ4Q} ¤OÞÌt›®¨4éÛz{îÛ‚ÍíBãéöÜx·ÓåÇñòSëš~¯õäyŠL퉀JùWÛnå]ÕÏSÛ?µîÞçwØ ãϧXoÖwõë*6f± e¯S$#‡û¡å6ÿl¡Pçasµ£ÝùJÉúѺŸ[.ù$+Ÿ¶‘=uXû‡)ÿÏuÀ:‰-ý/P}ùæ”­äì燅üq’oŽ‘B&šïF>$ qúÕ¶º`>ø íª.™²ÍØØé×v)t½ØN ;¿”繘Ü×JvÎTO°M +D”ŒnO–··ÆMdùnæõô+r¢¾— -©Ï»T¸<$»`;"kn!{šùqòÖuq!6Öû®sÝðÏy}ê†óñõ„fÖú¡RåkQsUsïzÛh¡cª4*Ï9Û€P'…ôzRˆ?عjP®¥w£&ßܸó,HÔò•9ŸBz=©ù¦Ú¸Tj¹òR´Ì=¿°²ˆzÈ­ãZí£‰šäC”Ý‘{¾çqèÉrÝlk›;¹ñ;âë Z•vä'%7/Xay½žºz9ÛE^ª¸ˆ—ß05gš|±@þ¾@7ª OýÅQ)Yå^ž–Fdn6òº“zzl¨CZ8JKÞ®»\JMfûþrÙ*ùt[îÖc+j×e]nŠnUÇk²kR×Èasÿ4¬ì‰vfÍ-dn¢ü±"/|r›ª²@=£ëÄ[½ 96ìüèÉ©¸“OÈ4~[½ÌÏÄתØhKÑÏs# Yœ‡Y|Zcqa  °ý¨äy¶6t’«Uç_—I¦iÜ×Ó_NUÍÁööâŒÅàä‹h½ÕåÎÿƒwUâI².*©ä»‚xè+Åc×`Y8[±)Ù7Òº^ø§~(dDÿЙò–4ZÔl_õL«oÑe ú‡¾br}£eÅÂôÃÓë›Ç}b9Î3@¨ Úõ£§¶j<êc\?6ꎇyò.=Ôê -ÝÜ ÍÈY¦ÆbZÐê¡—ÑŽš—bS¶jtV .@³3­Õ˜fžâTlî–úÆYí¥ÔlŠ…ú—¾§¹•Ö‰ÜJÜå•zg׬¥Ð îÅ9-og—œÍ–Ì --ö¥•–œa¦íÛ¦uÛ³¶=w¯A #q±LíRŒw9¿GæŽúÅ#-×ü§FF\^>ZE- øP¿CnsŒÀ’OI&Ê]¨äî2·c'ûÉ;é[Âp -Á&'_”×Yø@B9N誚¿6¯+V}¥û:@ -0·ÿ>Uý¤0þ›Dí1‰«(_—¨^ÎòáÓd*ªa £Ð*„Rÿü/C<Á€ -endstream -endobj -819 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 818 0 R -/Resources 4 0 R -/Annots [ 820 0 R 821 0 R 822 0 R 823 0 R 824 0 R 825 0 R 826 0 R 827 0 R 828 0 R 829 0 R 830 0 R 831 0 R 832 0 R 833 0 R 834 0 R 835 0 R 836 0 R 837 0 R 838 0 R 839 0 R 840 0 R 841 0 R 842 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -820 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 766.923057 101.433071 748.173057 ] -/BS << -/W 0 ->> -/Dest (cb221-1) ->> -endobj -821 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 747.976036 99.933071 729.226036 ] -/BS << -/W 0 ->> -/Dest (cb221-2) ->> -endobj -822 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 729.029014 99.933071 710.279014 ] -/BS << -/W 0 ->> -/Dest (cb221-3) ->> -endobj -823 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 710.081993 99.933071 691.331993 ] -/BS << -/W 0 ->> -/Dest (cb221-4) ->> -endobj -824 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 691.134971 99.933071 672.384971 ] -/BS << -/W 0 ->> -/Dest (cb221-5) ->> -endobj -825 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 672.187950 99.933071 653.437950 ] -/BS << -/W 0 ->> -/Dest (cb221-6) ->> -endobj -826 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 653.240928 99.933071 634.490928 ] -/BS << -/W 0 ->> -/Dest (cb221-7) ->> -endobj -827 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.293907 99.933071 615.543907 ] -/BS << -/W 0 ->> -/Dest (cb221-8) ->> -endobj -828 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 615.346885 99.933071 596.596885 ] -/BS << -/W 0 ->> -/Dest (cb221-9) ->> -endobj -829 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 596.399864 99.933071 577.649864 ] -/BS << -/W 0 ->> -/Dest (cb221-10) ->> -endobj -830 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 577.452842 99.933071 558.702842 ] -/BS << -/W 0 ->> -/Dest (cb221-11) ->> -endobj -831 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 482.717735 99.933071 463.967735 ] -/BS << -/W 0 ->> -/Dest (cb221-12) ->> -endobj -832 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 463.770714 99.933071 445.020714 ] -/BS << -/W 0 ->> -/Dest (cb221-13) ->> -endobj -833 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 444.823692 99.933071 426.073692 ] -/BS << -/W 0 ->> -/Dest (cb221-14) ->> -endobj -834 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 425.876671 99.933071 407.126671 ] -/BS << -/W 0 ->> -/Dest (cb221-15) ->> -endobj -835 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 406.929649 99.933071 388.179649 ] -/BS << -/W 0 ->> -/Dest (cb221-16) ->> -endobj -836 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 387.982628 99.933071 369.232628 ] -/BS << -/W 0 ->> -/Dest (cb221-17) ->> -endobj -837 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 350.088585 99.933071 331.338585 ] -/BS << -/W 0 ->> -/Dest (cb221-18) ->> -endobj -838 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 331.141563 99.933071 312.391563 ] -/BS << -/W 0 ->> -/Dest (cb221-19) ->> -endobj -839 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 198.512413 99.933071 179.762413 ] -/BS << -/W 0 ->> -/Dest (cb221-20) ->> -endobj -840 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 179.565391 99.933071 160.815391 ] -/BS << -/W 0 ->> -/Dest (cb221-21) ->> -endobj -841 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 160.618370 99.933071 141.868370 ] -/BS << -/W 0 ->> -/Dest (cb221-22) ->> -endobj -842 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 103.777305 99.933071 85.027305 ] -/BS << -/W 0 ->> -/Dest (cb221-23) ->> -endobj -843 0 obj -<< -/Filter /FlateDecode -/Length 3630 ->> -stream -xœÍ\KÇ ¾Ï¯˜³—YEÖ |à Q#Fçä°šhm«rùûùØÝõèîÌìJN¤Åît‘õ`³Èduü‘ðóµÇŸ"Þ•Rs’ãéÃáã\Ž„K]caÿ€ßž$»èk*ñøA œ]®1ät|:àÏYsîþtøëW‡íÇøcpµ¦àãñ—÷Êv¹äLÑ`U°È³ÁŠÑåb2XÉ;Üb­ bÄ,A.YbH-ÑbA ¡¬1X²çKV…!{±XÃט –'Èá‘Ƀ $ÂÁàù蔞,^ð.•ªÉËàù,³KY|ô¯º”R,O K¬ä-^„,ыŃ,Â%[¼Y81™<ÈJf‹—!K€vªÁ+ÖULd¡DÞâUv±“WÁ£’Ë%/Pt±0“ÅóÞEØ&›‹…Àx‡ˆ,,ˆÀ]ψÔ¸ëC% "pyC´°æà< ßœ'˜®É«Žj)Dà.`)XX€(ì¨IL^v0$oaÀØòP pá½ZX€»™-,HÀ]¤=&²p  ¯nkø4’UGHañ€»ä±Ã¸K”«… ¸«aÓ‚”•Š…Ð! ¤"®æœ,,H•ÀCÊgò’C;ZX „È2yE²XX€ÄÚU®Å‚ ‡`aA)ƒ…È\õȾÍò²À\Ìú@ 2E €d®T rL¨Ç U‹—‚+pmò -x>[X³8„boaj)Wàôà¦K@ ð¼XX+d«XX€PäJ@fnñ6ª©ž-³lDÝ6ÀVb>á@÷8Ä EûTù5óßM»(:oV¶O¿y­ø„hZW:j y)?¯+þ®!_Žâ³>~Š•‚©øª«h‰S wq ¯¯wýõìVÑ‹Å+äÝèz×Ï•nŽåô¸4/inÍnImù–„w ùò’¤ O_ô¸²Ø&öºäeòòýwe/^O“<•ê­Ñq‹‚jðqû,Ϧ.½*V}¤m¥tœê¯Pƒ>?Œ(ºTß}ÿç?þøýÐ@_Mƒ€®½"]([dV^ZûÄÖ'¬Îèÿa§P Ã@n§ÉÐ'qÌË íU»sÌÿ»b—WMbJ…MÃò~Ý*}¡d6® ƒ{¿}n§±¸ã]<ãå½ÌYn7‘Ç)‘šÆöìwëßäëôw/0)}|œbØ&µ™NäÉ|òv½™ǵ7Ó’ÇEæÕqÆ¢ -;¸Mõý¢‹ŽŸ3?¤š/6¿ûÆ|JÙúkX¡x¾f…ôp˜<¬;ÒSðÍ"ZKÑewVWi¿x©—T鮵_`u¢ÃÏ[]·¨¼UlÃdu‹ÇlÖV9.-éq†°_ðšÅaf=\îKÕîòÅÙ[•kövu¿Í®.дfo㨩+T’¹ï¾4DïlŸµ±–9îm»É7›YmxAÁÍÆG ä~n°ÙT]ƒ ¢Z«Ati§ ½Ï×nóåÑGÞoózÃ÷äL¾°Ÿo±áÓÆ£ëë®>°Ùú=*³Ê—}èqb§.ô¦™®í4i5©é~L<f5›§ÒéÂKgYkÑ“ÆÉKíØ2¼wž'±>Ô—Î.çYîëa’Û_“#±<' Z,GžO¥f$òú"¤è„—s/–Dëø>o,"·|c³¦f›õ.Èxº”cµ´µßrÍc Õg·L?P5¶ý~Æ;dç:û ©Ó¤çwÓØwWuÏõ±Èš-nëäÑ^î}ŠqîO—r}ê}Ík /ŸîwZ¿§Þi›3®6Û×ÙöÉû}~Øž|e­`£¾õ«Ïƒêô ­Çœ›]nĘËÇn­ <ú³nº1¥¾òŽ°JNâ³;¼_ÓÆL¡†pRàÝTU `C¥«¹lnÖ@óýÚO-zñmûWÓVúRù¤i›·ëæ®óV†‡ JNÕvóÇÉ-Òp½§{ˆí.¶ñã!-o;øå7Bu郾Lâ4^äzÔ7, -±ø¶m~Q´?>3rù"ÁÇÞkûnéë†úªÉúŃ©±õÿ¸løD ä>-.,ïÓB.¡5ùé ï‹AÁåY‰Û%õ¹Ä÷XÕh'$QÙÕóžHQh»Mó–õ2Äv7:>¶azJ'Ô:ñò„z]Ø bdò—d½{r~#ë»ô(d·Vê¢ÄÔdqL(sß1áx“ì™&b¼$Ò´Ü ë!pëúeí{b}VŸ¶¿õ yÚÎkÄÓËò9õɤjÖ¦jÚ Ôi< U;5¶ž}C¤ï“A: â×Ô¦lfçO»¦ïA­ÄÒ Iz¿Úú¤ÓÁì·ÍûNåÞ«Œ¡µi~ˆ2LKW¼A;MI_Ö/M>BÜ.÷dÍ?ú^S™ì&\OÄc¹ ?b6zÇ›éÍDžæíäBÃ^5—¦*¸f£é[$úw…ŠØ©Š<Ë…—nRú&ØhEÔ7;y¯¤™ž:ꈥ/yôÔåsî6;ëË2ú+èöÞ]c;ꤲ‰žkOAªƒÇžì»Õu²BXÓOÇÊ™HM,7ȱ›òÓŽ<äŒÝU#õÅ’ŸÌ=öy9uì|šÉ¹ï¸××úº¼}ÞÉæñ´69'«é—¶È ëS³f>}±‰8( ò€Ñ§…ܼµ£“~U©]'³"îéðóa scÐcÍÄ^ßRŸ³.üú¼Áï?rüS{-jã:Z„Ž­;òÉô;MåN²~Ç©»ÛLÄ ì²/£olÖÐãx^ËNØ[OßÔwîLŒC‡9tÓ ž'ÍÄ+ÄÓ™{"ó©{3öÚävsúà\_¨kR¡1ßšeðx„äã4%ZÚu„tp%Lј'EsZ°ÉvbªA΋tÕ«ü2oëZScJ#´Éƒ'#ÌÎS7ÚrS¤À<7æ‰ÅÏ‹Ö<†Ð¯v¯OÞ÷MäÔ<@©Ú;õi¦öSj<›ÊK3í›=æðr®Þœwh¨ÏÅ“ïqOr†kŒÔBvÙF'jöq^Tm·h .@£Fë3æ&SÚ†5µ,w\L-åžS êϳ¦ã¼+]‰qÞ•tÑWçm¯gKÜ 8›swG£gÏ%‹AKóèN•Ñ“írmٯݢíqº ‚¤‘™Ò/z¾ -ÿ^ã;똳S«WûYNF|/#P‹Ïxùn8•”¼¾ ’> -endobj -845 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb221-24) ->> -endobj -846 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb221-25) ->> -endobj -847 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb221-26) ->> -endobj -848 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb221-27) ->> -endobj -849 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb221-28) ->> -endobj -850 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb221-29) ->> -endobj -851 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb221-30) ->> -endobj -852 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb221-31) ->> -endobj -853 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 605.596885 101.433071 586.846885 ] -/BS << -/W 0 ->> -/Dest (cb222-1) ->> -endobj -854 0 obj -<< -/Filter /FlateDecode -/Length 4350 ->> -stream -xœí\K$9¾×¯¨3ÒÛ~IhH‹`ĈÃLí õ øû|áL?23ºªºvW»ìöôtWe„ápÄç§3ÝÙâç ‡?™É¹¤Èç˧Óç“5)Xü«%6•ýGü¦pv–“ ®ÄΟ„@ɤ|Šç§þì.çâO§¿üìô¯mwö¦”è]8ÿûƒ°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ ±1BbÏGV‚\rÐXƒ½Ï -+C âäèÈ*Ã'Ç b¸’Ârr8o­Êƒ –™¼ÂsÁD(=j<ïLÌÅ•—ÀsÉk<"»à4^11Æ5C–P¬Óx²Çl5daÊIãEÈB‘¬Êƒ,>'Òx ²xh§(¼ Y`]YåA­Óx…L(™YåðlNùÈó6˜‰¬ÆsÎØ&©[Ö°€»>a PyÉÀIÃîú€!h+7p×ë5,`à®g5,€¡àU£à. $èádÀjX€õkX ¢`¯aAÌlJJQÂX,xùT^4¸$ Bd©<ȉ5,@`m -•¬aAò…¼×° yÈ‚AÃD¦8DßjzY`.j~À‘¢†@2“ ³†)DäcЪƋÞä¸Vy<—4,H‰ –b§ar)“áôà„K@ ðkX -d«hX€¥ÈdÈ\ã9È‚ [ÃDºà9§aAöÑ †Ùgd™ÑjX€¹$T­ñØJž™5,È„ŠÀMi)UdLhX€ù‘d;hX€äÉ&Ò•Y(°†9{I7³†9Co½†°MÉ7“†ÅBD|„ÉÆÁ³VÂâ2òMÏÀ/%ß̲È7‘â)~[(J¾™4,( YB•Y4ÅoIÈ7N/B_X£ä›YÄÈ7Ñ«[YòͤnwC2i<à."Ó¨a.o"²Õr{ à ñ™ÎÌ’qfuÇÀzÖVuËÀ{±Z%uÏÀ|%GR7 &2ËÕ],«`:«nXà/â1>`²û4oQE˜Cš™“ ²À-Ñ}Ï9F1z¦Ø²?öï¿Ÿ~þ‹ïùïÿ9}üYÝ)k?bAœ0Ùö³ýé… =D\'òÈ‘éŽL€éVà÷ĺkQ‰—ÓDF”tpb±O[:²º\éSw±æZz’m¢^¦‘L䧒d¤;r½ìèn&ÆÑî ‘>ÊñŸà‡Ë®ÎÇ…'e맔_þíüöÍÉàJ?iöŸœ¼ÆÁ¯)9OYD]¤ûò ÷‹&áôמíB«¹é»]?¿^¤²"1¿_ˬ4Ì[i/~û¬4¿ÒÖök[+ËzM“fÊ:ú«æÞ­Ÿ«F«&wmVÙx7ŽUÛU–ûuXö:´ï'=ñÔg4‘Õ^Ö™Škóqn–Œ%×Ú¥Ðfx± iCÆá¾>¶In}Èñšc;UïVÙB•ïšÉ?'‡z®úçU¯—Å:ƒSe“cVѲEi[dóqÛ<µ9ÙOóñùltÑKýž†ž8;á°³‹Ë"ã°ÕÑÿ°ê8LváÖï½ý«z+û±U»÷“g¯ú“v›=îõg?®²¶r{Tp‹½oÆ°¶£Ëµ6h L¥½ÄŸµÁ>có×ÉÇÓÐY+[ù+†ØôzMŸäöú «>»L«_µöœ» Iî1Þ‡„a Ò B¸ŽS_­ß÷ʸ&+k}-³·«ÃÍ{_ ý´×¾ª­õº4Ž‘ÎkB×N³«V«cd¾Õs“ävW§ÍRr¾{x>k¤ÖŸ0«ÚûUœg¹Cÿ-àü±¦“†×ñŠ9>'Ç7ÆùcÛ?œç¸Ûç¥Lí3.uhç ߶oÐãš>óAŸ×°]m#Èé•M‹¿žýé«·5« ‘ŸÉù’ó&‘ä\ÖdSÒΙÈKë<”Ó­ ˆñrø ã5T¼±&Ø’-ï³Ç­×E5ýÒQèò8¬#OK’ò›7o~õë?$…?ìBÖР¸Akƒ¾õúc™Ÿí¼ W+r„p×yƒƒ%Ìض»@ØîiðÄsæň¿žÊõQx_^—NÎkçÌuþÒ…·j9ä‘mÒjßî;Ê™a'玵ëˆøV rª%d1 -]·®'»ykú¶[`±6}Kv–g -Š“s(è/ŽªÛÄ­ÚÙÃO2GMëwé e(—5Íwûêß­ÖäÌ1åè£ÖÚ»)TÊ“×\ñ* 4ä·ÍÞÞöõú,¿Ø n4Â_e#ö.`»¯Î÷ lCÞœáL¡`4`;ÆŽ«ÁïÖA‰QkœJëz¼Ÿ²õÓ^¶y˜²ƒÇ¦î1—5r+ÛÞܪ¬òûî–!{ù|Œ;uÞå–“ÑÜàPû&$ÈáüL®äðP߶ãñÞ˜LqÞFRûŸfÔnµÛ!µÍÂîzÏïí¸gœµ=6ÊÝ€ì0#tËYï«óƒqVyr@Î"úg¢ÖU}Ó \Ì€ßÈﺸËœÿu6§Ô¾Ýw†‘ö­vO¶Çãsíx‹œ¸Ø‚ÕM“bY›: Ý’Ê“<Ûˆ‹ôˆF<ò7y: €ð€F`C¦H4®Õï§ynQf[+o´[`Ø.OW4ä@#¢‰÷D4J‹ÆŽ­îa»Ãõ6 Q®'®Ð‹%Pö´Úï–qVkØG¾ šÜ€Y0º.Þuèé·7ÄsÕœ*Ö“ÚÅ´Û>/s9'œûÄPßØ8nÈs•Ež`ÚuZ·åžIÚb¶ úä9e' -Ë7àòÎ:ßä.Òw‘»EXÓÔl·jz¬?m{UÏÚçrû*o½qIÛ‡½÷òë–Š£XQÂ0®XQ·4¶?ÚvL·¢y; Û±Ñ8[§¹·íðš%y¼v±$^G|Àî«òC³Ÿ„Ðàšý¼MëóǾo–'ÈB‘'Äõ[¥ÏlÁ¯Q¯ÛïJ¶ßåÁ¬1 n¿ËJKë¶î÷¿­žS‘£“ß?¶­~lçeÛêÅ…çä°c,m«Ûþñl«Úíÿi[}ï ßÏ-Óö:¼û–©Þ^q»önl«ÿq}d+…³döX‰cñ¥LO€õUõf‘«h[yŽg‹mƒ4)ïjB¸/»%ÉãÏ…&bÖ>7«Óܦ¯,¡W] Ö…Á‡azêÞOP&¼#QÙœýù¸LOM`:ÌLðæ_û{·NÇe¸ñ¦ý°ö=ÉÓܳ!´1¦ñó)ÖÇt]}UƒE’í¼¼ËéÙÌK9Ë£ÁÙ»6m®*¹°^³¾ës/µ¾ËÊ{2äé:­óÅZþsðOXÝ´õ¬¾ÀÆV2¢±°'?$Èg…Ì<Åd!®_mo‹]_cí‚0QNJîJ -¨­_ÇB,‘ÙòÕ‡6©Z5y°K´oWâKÇ -5£;’eôÖ¸•,/ʆ֫ØE ±É,§C{ƒ<— ŽŠ7ÉŽìD G¢ºdÏM'/ 誋ƒØEŸUç¦éoe}š¦óñrB·´§>©TyõLkªiAm£Ñ$TéÔÐJö á>O -IŽÒˆ_ØÖ4c2;š5y€!æfHÜË•VN!]Nj¹µµÐg*õRyT-MóC”aê”»âÚe -V“¼ÕnòKíë–,ñPŸk›'»ñâå¸{â ×ãÓÇÒa`… Íôf"Mívr¶Ã^5å¦*¸f£ÉãÏòwŠÐ©‚<õ‹ãnRò -ƒq5:ßìä­’fzì¨Ãš§QvR—K©Ûì¬/—­R^@·—îÛP'•Mt‰°{Ó±ƒÇ–ìºÕu²@XÓOÇÊ™h›ènC7å§ yȺ«Û;‹n2÷ÐÛäرói&§>ãNÞGÑåííN6߈—º°ñž,¦Ÿ['ƒ,;ÅÍ|zgqPä£O•Ü¼µ£“¼c¯}N¦­¸—Ó?Nu™ó¼Ü8ˆääõJsÔåÎÿE™7øýç‰Ï¿kçþ—uÜÛŽ¾cë†<¯dò2¾|'Y^Î׈Ým&¢ç öÙåQ64kèë¸÷^ó¨n°·’®#¨ëÜ™†'²ï¦ëMš Wˆ—{ óMçf,ØË%µÁÉmyD“ -ópp™Æ’|žšÄ•K:¸ì§Š¸˜ÅåÔa’măœ;éªùyž -’¾¦‹)ŒK<ËìÜt£ÕQLEÓ|17Ìnî´läQ„~ؽ¾ñ¼¯"ÇæBíÐÞ©O3µL¨ðt*Õ˸½ìkÕ;Íy‡†z[4ùõ g¸Æ-xs1ÛèDÍ£ÚZ Ìr£m:m E‡ Ðl£õS“)®ÕšZꈳª¥ÔcŠAýǬé0ÏJWb˜g%ÊJ»mÅëÑõ(ÌæœÆèì(ÙcɬÐâ\»Sy”¤F;öÍÛ¾Ûj{ž†ÁC8"S;ª…CɇðïßQÒJµŸº3âòxjv‰<Õ—Û#0„å½t‰ó´Uòâ:×÷NžËܦÃ0‘MN^O­‹ðF¹Í˜šj¯®Ö««ée -P¶†Ìérõý¤õ6LÝ ê—IREy1IЛå-ŠÉ>óˆ¯Û(q³…RþÁ=aè -endstream -endobj -855 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 854 0 R -/Resources 4 0 R -/Annots [ 856 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -856 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 432.754475 101.433071 414.004475 ] -/BS << -/W 0 ->> -/Dest (cb225-1) ->> -endobj -857 0 obj -<< -/Filter /FlateDecode -/Length 4386 ->> -stream -xœÝ\KäÆ ¾÷¯è³WXU¬ä Aâ AŒ,’ƒ‘Ãl{'F0`ç¿Ÿ’ê!‰Ó­níw0;-²,ùÉ–dÏ„ŸÏ-þËlMÎ%E>_¾=}w"“áßÔbu1±¿Äo -gKœL°%æpþV>™T‚Kñür›˱ùËéŸþ³îcÏΔ çï? ‹½I9% - -«€EÖ+¬LJ.D…­ÁKÑX#$v¼g%ˆÁ%1ع¬°2Äðœ¬ß³ -ÄpɲƂ¶„¤°,AëˆT!fïž &BéQã9kb.®¨¼žMNãyobb¬Æ+&ƲÆcÈ -Y K°Ì¤ñ ûœ4^„,>zRyÅåä5^‚,Ú) -/CXWVy…"YW¼ %3«¼å”÷³†ÀxƒÕiX€»Öã¤ÖxÀ]ë -iX€»ˆ‚†0chX€óÍX‚骼b¨ä¬aAî–œ†8… e ‰ÊK†d5,BªaAîâx/Dà.ï5,ˆÀ]b„=*²øì4,@\ÜVñi«†BãwÉb‡5p—( "pWŽM b‚Ë@@ƒ(ØiX3›’RÔ° !ŸÊ‹×AÂD @–ʃ,œXÃÖ¦ø’5,H²xç4,H² dа)Ñ·š@˜‹š0dA¤¨aÌä¬aA -ù´ªñ¢39®U^Ï& Rbƒ£ØjX€\Êd8½†x#á’†<ˤYà*à(2Ù!2×x² ÈÖ°‘.xÖjXdA4¨aAvYf$ pB#—„ª5“ä™YÂÌÑH¨¨aÜ™&RåAÀ„†ØI¶ƒ†H~l"ÍQyÅÖ° g'éfÖ° gÈâÈiXÛ”|3iXP² âÓ° P4&kXHÂb3òMÇÀ/%ßÌOÈ7‘â)~[|”|3iXP²„(*² h*Šß“o4^„,®°†%FÉ7³†3o"¢WKYòͤ– €»¡™4p‘iÔ°È7Ùj¹=xCB|¦3³dœY­ 7àlUKìÅi•Ôš|%GR‹™eŽjÕÇ*˜–Ô²ñæêÓX¢Š0‡„33'Yd[0¢û–sôfþ•fs}ìû~õ„Ã÷ü¯ÿžž?›*eõÇÂø=GI¬ƒ¸ˆ+ØnÀ•™†ã©˜„ø~#B¾ºˆ©t-@q)¹‰ˆ8ˆqØ–sû`×C^‘Ø"”™ÖÅã´È,6ÊGUÒ{»¼¹¤ð,äõTJçuE•ñVã›ÒyùwÞ¸!¤CbtXÈ[ßVÈÁÔ€¶y™÷°yëóöö9Ì{Ø@õysY£xŽ¿lþ­Æ?–…&`Ýa!o5~c!}SŽ“2’4d¿èƒG¸4O_T ¢ÞÙå-öœŒ“˜?¢•/Nb$|Èn‰Wûá„ãr™~þ&¦]ZšŠ£aèÖÑù÷§_ýñ‹/~ÿ‡¿aÚϧ¯~MÄŸ~ó¹Åßç¿–¿õ:.ü´\?ãï?Ͼ¸27b*’8¯ç–é§?õáë°B~¡Õ)}çᨤàú5=´[èAùß\NŠ”ˆ $°Þ žouØdäÅÚÊnuŽR&³bï:OËá[H‡¸ö êÕÚe‹.›-«º¦µ~ý'ðè€ä,EtĈ¼î×’lãÃéÖp«3R„¯È©ÔY@žácÀiýŠÎÄ®Bµ«[ƒI a¤‹Ê`O]±Íwž¯û}œëÆm Û¯mðÇYÐß}X#T–ï,²L))µc]µ7IJ&.¤” Y¼ -f 6Švl.vÛÅy×Í¿êØüž×é²´©ã†Ž@mÜc222^æµl[c›d•ß§[Vìäcp1nÔyÈ#6cñ}ï›` eèìmÉ᡹lÇÏ[C81™¯^ØQÚ`gÅÒº ›ë-¿cuW•À7Å)‚DBk²ÅŽø¾z°ÏûqV©‡f¨Àê‘몾i6fC8¤ö°šâ Z©tYÊÅ*½oσab´Þáipûëã8Â^*8Ø4)惩ÁÐ-©œ—oñq‘шã8}'€p¿F`C¦ ƒÓz‡Ã>×Ȳž”7Æ-0l›ŠóW4ä*@#”‰GBJ Âö£na»Áõ1 ¬rª-Þ¯?KðÙI…qßûi^çd Ûp·B“tñÔ §U–Äsã"A<Ù÷äSÙ"lŸ|õwùëŸþþ—î«.Ï{ä0Ê"‹Ì¹|Nóø0_á ýy’k^}ŸTlçBͤ2ýú¸<›¡Oîȱ¢°| .õù!æ‘®9J×PÓò\Öa@M¬Â6ÛPyí“÷¤nï­½]¼ÌßaEN¾m|ÝŠš…¤ÙBpžOó¬¬h -uëÉ“¼{ ┓ÝLxÍ‚ÒÅS8xçt¬Ë»³Ÿœ¯ÚÏhbYßÀ( ’´ËÁ<ö‰É:˪5Ø\C%ü†ásµ„\¦0ª¦åï׳åÌÞÈV-zðÈ ó79ǾO}c™cׇç9šWÙ†0nÃW¿~E[7öûJç4iXeê+ë©‘Ô’6myÑ~S¤Es±Ï-©”ãÁ-LýysÚ\v˜0®·8Ã+LÐO¦Žã8ÑË÷§rÞ~œ¹HµÈ§5]‘ƒ]~Mêk+hë›,röAE6¹W3"öD̸›–Ô1,sŒ:«Þ0y•üÆù/ k™>§®'®c‰|V=[îèy}÷iÑq]ë€ñ¡Uo¼]ÛäÅn‘iÐߌUºþèy‘µ¶«}C—ôºùìì}³.[Ç" ¡Í1âÂC?¬5<-¼Á7¦Tþ²Œñ¼Ñi¬4 VÚGZó˜Wu·: ±ËÕ‘f©ž¹K;k¯ˆ7ãÏþÛmzô¯íY v^n”9\}?ØççN~Û1˜å†sGz¥êÇ-»¯'ou÷½t÷Þµµ¯¼ï{ß[z×uûSÔÞÐÛP|@oCõýA½ å÷WôFôPý]íàC&æˆæxþ`ŸŸغ¼È¥ …B·‡Xèû8ri÷©ƒÎ{,׿ÕzÞ¶´)Øb¶›m:äî‘,ÁŠÖû&ÔDkÍô`=4·#CÉ99v½AÜ!Í!yeþWJûºg¸O7%Ès^nH»Um:Øçý¸q‘‡rXJ5;7~øûz¼°?”±×âÝ[ÆÞ÷¾§Œ­Ì}Ga?!H†ðVÅöåÑ/ ¡²{D#Þóˆ÷kÄË÷„ERmîÇ ûòØV‰œJ¾¢¡» û’g›åØý¨öå¥{ݯ?†%ÔØkßû§*ìG¯ˆšù ûëI¯•eå9“6ööyo…Ù˜â5Ä}×…ýdÓ5+zûÂþzÂk”ØUk8ZØ?Öå½ÙO -éªý܉&? °Ÿåþ~DZD÷ös·Ì_\q¿Êx¸,/Ou%Î7D,%ÁçA¦e)׶Òe-!>/ö4ÿuy£ƒ8Œ¹*‘îõò¶%ø,O±¬¼÷±ü~œûJð¹Ä×ä ¾–ÇJðû±9%øb·k{×%ø•ßÐTv?æ/]ϺŸt_/Û_Õ¥ßêòXyùü¦eûé)±•,7Êö_.Ï¥p– -bÞnºÿ¬>ÎÔÎ×›Mnœ§õ Úßµ¶›»ä×a -†”!$Ï꽤ˆk¸Ü]uºé²~ëRæ l:ª–ãÁU²‹™|àpøVÇÒâÚq0‘²¤2 ªŒ{šæ{Z¶ãÒ¡`5~Xæä©.ÞÖêvÛøÝ)NÏœÚé½³±N^LtŽl¼ìK9Ës®™È¿Ó¶ŽKûï¦ ÿ'%›XÏÆém,4‘—…-ùå$á>+dæ~ÀÞ qùHm,¶í¼®´ Fä©Û–ÇhùØu‰Ñæ.ÔÕHÿP»ÉŠLµ‘Ÿžœ'Vˆèíž,«'c²¼Ñ(¿\Å&JˆUfû€<¶íöŽ7ÉÖÓ@ {" Óu²ã*•'à›êb'6ÑGÕÙaûk[—†í¼F¼œ0­ßR_Tª¼G¥Uµë¨Ñü TiÔP[¶ á¶O -érRˆŸSš±™?ìš<>s5$níJm§.'µÝ2Zh;•Z«Ü»–ªù.J7uŸ›âÚe|“¼¢mðòõãš,1UÛkʃݸñršneÙ’_N!)­CÇ - -ÕôF¢ÆmäLÝ^:5åª*¸f¥É³¼òÿ ¡Qy¦–›IÉóøýªO"¾ÙÈk%ôØP‡5 XN½í .›R³ÙQ_6“Ò^@·µn[Q• ôTªxR <ÖdÛ¬®‘ª~VŽDªB`ºNÍ”_Vä.gh®¨Mí`î¡ÛɱaçËHNmÇ­¼\¡ÉÛÆl¾/ÓÁÆ[²˜~®“t²Ü ^ͧM6»`…Nî0ú2‘«·6t’ÆÕÝÉ´÷rúæ4sÎ'_?Doå]AcÔeÏÿC›/ðûïŸÿ\NŸÏqG -\ÃÖy<ÉäÍrù YÞ4W‰Ím¢ãö;ÙæÞ6Tkhç¸s^sïNØkKÛÔ6îH ]‡Ù5ÓuÖš Wˆ—[ óC÷¦Ø󥯋“¯\äµU*\ŒËÁeî<ßäó0$®¤i?Òå®\7tÄÅ8(.‡ K—l%¦ä8IS½ÈÏãVx™k¸Â¹ôÇý˜‡®´iC„À~¼f;NZVò(B?ì^?xß‘cõ¡6hoÔ—‘Ú&Ôx:ÕO—q}ÙÎ?}'P·k¨åßó-Èé®ÑC ^]Œ6:Psï¶4ã¤\i«IëAÑà4ª´6bª2Å¥[UË´â¬j)µ˜¢S¿5Æ]iJ ã®Ä][·žx-Zò- -£9§¾:ê-[,™Z{7*÷–¾ÒösózîzÚž‡ep$öÈ”z·°kùþ=â;sŸMBkU¦ÊˆÍýu.8¨Ù&ïüô†^Ê1CX^²–8¥’»û\¯ì+$µv2΄í0ì=%+ïZÖEx£Üf CÕ÷0ëÝ -©KºOJi¨Ãí*CÓË6§/d¦Q»L’*Ê»uåÙϜ啀‰¬þ /e”´*¡L?ÿLçE= -endstream -endobj -858 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 857 0 R -/Resources 4 0 R -/Annots [ 859 0 R 860 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -859 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb228-1) ->> -endobj -860 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 446.531993 101.433071 427.781993 ] -/BS << -/W 0 ->> -/Dest (cb231-1) ->> -endobj -861 0 obj -<< -/Filter /FlateDecode -/Length 4391 ->> -stream -xœå\K$· ¾÷¯èsË”D½€À‡’ b$È"99ŒÛ;6‚q€uùûù¨*=ªŠÓÝ3;6Îf§‹Ôƒ¢È¤ªªí™ðó…Å™­É¹¤Èç˧'2)þÕ›‹Êþ3~S8[âd‚-1‡óBðɤ\Šç§þÛ]ÎÍŸNûÕé_Û>öìL)ÑÙpþ郰؛”S¢ ° -Xd½Â -Á¤äBTXÑ,±1BbÇGV‚\rÐXƒË -+C ÏÉú#«@ —,k,ˆaKH -Ëä°ŽHåAböNáÙ`"”5ž³&æâŠÊKàÙä4ž÷&&¶Áj¼bbŒ!k<†,¡Õx²ËL²°ÏIãEÈâ£'•Y\N^ã%Èâ ¢ð2due•Y(’ÕxÅ›P2³Ê+àQNùÈsLÈޓƳÖئWyð(lRL -ÏA–H¡¨<È<9ç! ãGã'‚O9+ÞêÄ»=y«ñdq.±Êƒ,Ø| \„,”¢ð #¸¤a€KÉpq¬€ËÞp次€ËÅpJ^ƒW‚áX’†ž,xÎj@à ²Žx Yž«ò ‹/¬aw 4,ð²8öx -ÃZ4,ð€[¦b5,ðÀ[&5,ð\ˆIx ®Ï‘5,ð€\˜RÑ°Às=°SÃÐ…»' • ®ƒ†‰@€,•Y8±†H¬Mñ%kXdñÎiXdAÊ a2S,²oµ<€,0µ>`È‚LQà ™É…YÂ"ê1hUãEgr\«¼ žM¤Ä¡ØjX€ZÊd8½†x#é’†<ˤYà* ™ì™k< YdkX€LÍGTæ3s’E¸#»ï5ÇhæŸi¶œýôýéËßó÷ÿ>=þªž”µ ã÷¥°â"®`»W¦« I.ðÙV !àkúˆ© !-`q=t!Šn˹°[‘!±Èl‘ÌÔ•ñ~¸%¤ëÝBÞhüÆBú6/’ bHƒQ_ ‚ˆ¸f6àJD}a—·ØsBŒ¬*¢•/N¢ õñ¦ñoÞŸHëôËYw¿´>4¹;ºut~ÿãéËß¿{÷ÛßýÓ¾<ýý×Düá«/,þ†¼üå°þm×qå§õúÿq~ÿîÊÜ@_*Ršlç–éP ?ŒáÛ°B~¥µ)ýàŒ(¸qÍßMíVzFþ«ëÂÉ1b”¤.áÂã­ÎABGFå¡­ìVç(V’˜C纾5€DJdV°U¯Ö®[tÙmYÓ5mõë?€GwHÎrL‰(ŒÐör¡Œ1Y²n ·:#¹C‚€ªå•: Èä| HKžÑ™ØUhvuk09Â@ vQìa(¶ûÎãu_¢o—߶q{Ãß÷ëÌÍ¿~¿Å¨,çÂÙ(<"+Òµ")óUX»¯Ë«`í Ѭ‹‹$ i;‚lÔáìq«úŽ «Íû±ßÝOÊø|ز «\ž¶8N[1÷ãï]vrØÅî-¿÷ì¿;ù鲶iýw¶º7I¨ö° JIá~O`»³Ïçcr’‘¡6«šÀ«=êÔ|¸ÇÙ#Q5#ù}¸‰— Ilœwj¾ ®Q1£¾·…’Òûf¤N>ã«æ†¬à|ì}_¬°J<ïMl`[ Ý'1}Ò3»É[«“Ýìmè9LÇ¢äTRdà”.%Ï·\ú¾>ŸK#±rì=ƒêÍŒoÅFOY6CÞe€6Èí.®Ç ‡Þ·çŽÐ9’$9t8Îý09þqr0Lìä A‘bÉYz|¸%•sd¬]ç×hĹzàJ¬õ¾=·—Û÷¸HÚÜßNÆ\†ÛBx2„l…ö†3kȵȉ,7ÒÒúúP­ Ž£î»‚ò­”=¶”ýú[n÷Ú Ç‰ÇÞË:«5ì+¡<¦„‹®‹‡ž=ôcñÔ¸JÏÓÁŒ<ôª)W}óëoþô‡¿~3|³†5ц„¢,²Èœëç´ìËužÐ«\ËêǤb/ˆfElu3©Ìc¿Ó3gKƒ:y΋ôå<ÞÙçS~ŽJÞÕ<‹’–ׄà²MZÍö¥ý><å­7VïIÃÞ{û5 ±þVäP^±¢n!i±Äâ%<ÎVTÕzr•÷hAlJ €¬Ý„×,C7kÈâeø|+¾Þ×ås³ê5ûy!šXÖ÷¿˜šš6Iúåd >ÈCtØТZƒÍ-QÂo˜>—ÕZv©¦Ï´þýn±œÅ‚Åz›…¯ÞÀ‹Uõöqë1•ÖSîÉV¯rT+u;m­ÖZqp7F•…wr7kŸ¼UJ˜¾FžúçA/ ËÁf%ËÓ–ÐqxxÆ¡áÑÆ*·8ÔôYõáÖµ5lùT,ÙåkÁy:ÎX:äsû8ÒÇ.Íÿ¯êÜíõ²ÇÚ—Z>;cPËk[î^ÛÌëKÊZ¯åó<ÖÕö>´6—©M ‘_^†iYž'¾Ó.ÿ'xöRº_×y¯ë%Ë9®¿žGNò· çmP®Èsðo€rÇq^†r%Äçäød”;Ž½9i/Óî³­f»Úì«Ì»ldnvØXQ­µïãíÖØQbªšÛï![t7õö: m® Å®¡œm(û@› lŸÍ/·nÇ>ö •×ñóìÃ͇ïNº»¢/UWM¶ÖîÛQE4d檾ÊA_W²ê±:»Çi=ë kö£EÆ9"´l½ë}_MÅ­>êX»(µÉ×v6ô\5vÔé5½Y²{Å<9KÚ.rjAb,¨—RŠS>gL#¼ÆH&ÞT2íÓ —ª¿~¿>„[¿ZÎA^•—‡¨Ëôôy¿Õr³É[+ívÄñà³Ý=r»½c…!å{"|>ËÃ(%É«Ïݯ’ŠÛµê;.§r¾©¿,÷ãæ»®ù–[R·%üa­xiµÑGªjËzW›'S™2°}e^ç{>–Ÿ|Ï÷$g¸ÆH-xs1ÛèDÍ£ÛÚ Ì“r£m&m¢Ã…<‡ßh}ÄÔdŠk·¦–ºâ¬j)õœbP˜5æ]éJ ó®ÄC[·E¼ž-ùž…ÙœÓX–=—Ì --ν;•GKßhǹy;w‹¶çi<‰#3¥Ñ-Z¾ -ÿ^ã;KŸÝ)„ÖªýÔ“›ÇÛ÷Ôò*‚óõ )Ç aùNœÄy:*yqŸëg'Ç’vv2Ï„í0ì=%+_©‹ðF¹Í˜†j_›©w+¤.ée -PŽ†ÌÑád¨~7Z}6·õË$¥¢|¢—¤7Ë78%²úì¼£äÍJýù/PýB -endstream -endobj -862 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 861 0 R -/Resources 4 0 R -/Annots [ 863 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -863 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb234-1) ->> -endobj -864 0 obj -<< -/Filter /FlateDecode -/Length 4891 ->> -stream -xœí][¹­~ï_ÑϬ•Dêû A² Ä8yXœ‡qï8A0àäáüý|T—.UžÌxœ ^ïxºH](ŠüH©Ô²;ZüyçðWfgr.)òñôéðù`M -ÿÕ«‡Êþ~R8:ËÉWbÇOB dR >ÅãÓmçâO‡¿üâðuwô¦”è]8þóQXL&å”lPX,ëHa…`Rò!*¬è †XŠÆ‚!±ç=+A .9h,ˆÁÞg…•!qr´gˆá“c1\ Ia9 9œ·VåAËL^á¹`"”5žw&æâ‹ÊKà¹ä5‘‰‰]p¯˜cÈ!K(Öi¼Y‚c¶²0å¤ñ"d¡HVåAŸi¼Y<´S^†,°®¬ò ‹Öi¼B&”̬ò -x6§¼çyLÈDVã9gl“T< -““Âó%ÚPTd d½Æ#ÈÂø£ñ€RΊ·zñn²ä4^€,Þ'Vy“¯!€Å¦¨A<È.iàS2\[Ö°€»>!¨¼dàƒ¤aw}À´È ÜõÁz ¸ëÙG `(øcÕl¸  z:Y° ÞƒçIÃ4f<Ò pׄ?ŧVÆå5,À]—1}*¯— qÅo¡eãb,à.iX€»€Ï¬a0Þ ¢z p×"µÆî:_¬†¸‹¼!hXs0 €†ˆoÆY˜®Ê+Æ–œ5,À]À’×°QØØ $QyÉÀœ†cc‘‡jX»ïEÂܵHÂܵŒ´GåAÊ^Ãä5àÁmŸF²j¬Rh<à®u˜aܵ6 "pW¦†1 Ág  Aì5,ˆ™MI)jX‹)ŸÊ‹ÏAÂdA€,•Y8±†H¬M¡’5,H²÷$Y2hX€ŒÀ‡ì[]@˜‹º>`È‚LQà ™É…YÂ"ÖcЪƋÞä¸Vy<—4,H‰ B±Ó°k)“áôà¤K@ ðkX -d«hX€Pd²Gf®ñdA’­a2]ðœÓ° {È‚lPÂì3V™ÑjX€µ$T­ñØÊ:3kX9I5,€›`¥ ¤Ty0¡aæGÛAÃ,~°ØÄ2GåA -¬aAÎ^–›YÂœ!‹·^Ãئ¬7“†ÅBd|¤ÉÆÁ³VÂâ2Ö›ž5,€_Êz3kXPÈb½‰%žâ·…¢¬7“†…!K@Š¢ò ’¦¢ø-0 ëM “Æ‹ÅÖ° Ä(ëͬaÒ ¬7‘Ñ«[YÖ›IÝ.î†dÒxÀ]d¦QÃ<`½‰ÌV[Û[oHÈÏtf–gVw , 7 ¶ª[Ø‹h•Ô= ð•5’ºi1±²ÌQÝ5@XÓYuÛÀ‘ñλOóU„9$ÄÌœdnÁÈîûšc£ ÅÎûcÿüëá»ßã_ÿuøø‹ºSÖþˆmqRÀ4fÛÌö§‚å!òÚ8‘Ÿ@ÎÈL7dL·²×¸%Ö]‹J<&2–¥@'û´¦c5˜+}êym+=É6QOÓH&òÓ‹¬H7äFÙÞÝLäÑî ‘¿ÒkH»ßKùÕûº¨–ð»°aÎõU‹œ€H²]||ÿéðݯÿðÇÿùß? Í÷?þÒZ—ñãdÃQ^yà|fРoðãñ@{\øt¦ñGüœð—ßR'~ÿÇ÷?\CN*t9¤khÍé.›öñܽ?›qNÔÏU½“…xE»žrkÕåïß¹_ŽAÕß¼ô°ð‚[~Çåw£‡ó@¤¬(€?œéUÒ¤ÐóVòó³(¤õQëäu½mÿ´üø!Oˆ8ÚZOÐRn*¿ž¨¥?}²Þ¹ûõÌ[=ÛÒͤ·‹Ð¤_Ûf3.ÍÇU», t« Ð䓶ÎmÈXÝOûv"ÉË9p²o§ê÷a‘-Tù®ÈAÞ^’Þ‹•VÝ-¶PçØŸ-X‘MEË>¥íj3ÒV™mê²=‰MT=ˆ;f[kvvÖ[kGd®ÎºÐk·ÈÝìyÑó°½aw­ü5@We>^Õ;ou#õ›^Ck+-í]ÿm¢ÃJ[ì¦! O~ÝõW–6ݘ©[?ÛÅ~‡~¥ž¦ã:öe™‡Þy™_.“ìK]·—æõãh;„«ú‹;ý•Iæ´—¿¡nûÜü«ÉXËò u}äen?LÈ}ZìÇ.í•aµÞìËiy^l$(²õ2<én)¿³õ­Ï<.z ›±äi|ñª.³f‹m~ÚZÛÎÝ¥nG©0Pþ5"U›]þ8E³ÙÛleãš^Û˜Û-£ÏŸ' -1oõØaXýˆ°Íû%Ryžt—&YíÚÚk«‹%m~…ˆµo§#/m]µÖàø’_±ömW›,Ò\Úzн±Gž]šÍ-èw Å·9a­¿Œk‹J×RU-M Wç5Ðv̵¾´÷Ó@¿s66ÉÚ¢jGóÉE¶Ùg‰>ÜÕŒ$$2Ö‘§.ÏÙ†'T_Ú¥8éét¶¡ öç­WL‚Ò¾¥sù_¿Çº¦ª¦œÙb1“åüœ¬``†äýu>ík¡ç×yÑRhY/‡06CÅËA k‚-ÙòvYäÂYú£wÔå]iã»ßýðÃo~ûç±Jê¯AI ü:à±·W“¥Ê¿‘,¹–ô|˜¦B¾K°,1å8ô%Çf -¼ª ²è75ñ°óªô™Uþ3Á­×óDôGG¡‹ëⵉ°âo¨äªo.Xj±rCçͪë’R}2(ÄN–­7”yµè(qÝK -¸"/Û£;£ Ðp]°»ª¼ÊÎÃkL÷;ÆzrÅëÓ=Ì-t÷tábãANœ³œøY7>CÜåÚ1ÃüR ¯ÔvvH‚M7¦”LÈ.ÛpÛ˜®}-c’S¢™%ŽÕ/Y@A¸%ÙuÞž9%ä062`@‡ñ*Ô,£e¦k3c¿1»°ùí×Ï’aö¬ñ’˜P)’`Ęw¦—/q1sJíð¨còˆÂV¾/÷Œ¹¿¯ÎÏ0†¼ äCNc“0hþ¹ßû×Íþ8G'‹!ܘ8øR*¶ ­Þ·y{Ú‘Rqfòjí°,|xmÆ[Ô´á‚yBfVrqÇlb!’S¼×ãžoÇ4¢laKªiD{A-rž”¼œÏ½é,W‹¾„gK&—Pƒ—"çú}¼•ÜYçíA8¢qŽ!”ž¤5ôv8§Û™Áðîu'÷er‚ÜÙjúûÚ_1¸<܈‚Yu©xRÇ%¹Êys€.à@–/]…TRºið׋¾šÁ#Tb¹“Dz\¾T“«_—ì¾:oÐࣥ·òÕX5g)“¸;ý]7üÜôw_û¼3ÜöÍ6F'7œÜ’çPtÃŒ9Ö¯K¦4É×ë¼Û3cËÁ­‚kk²sòánµk7o»ÂÆÉ›>f¸Þb©©—äKãØ,„Úžá—ÎËTNÚõÖ¥Î&­h¿…÷5úlV¾Ú‡š%3Q”×òõK'Ô¨p_7“²dQØ©è`ðÓ¤Þ -÷¸Ã½V]TÇþx«v†¸‚Vûvß…±Œ/š*µëˆøþ,Xë1Ÿ°6Ÿ»€fßÚCݘ]o,¥áÚµÓcÝ“êõv‘6Ž°yÞò{;WœìfVÀˆ‡¹dRgüæbU¾»ŠKꌻÇ[µ“7^6Óµ¿9?œÅÏ2Gm¶›îÆÊk ÅbÍÑFº?y¸³Î›‰â†Å’ž3Ü·Ø k›mšûqÂîñîní=¾¦„v Ly˜q3ñºó~Wôñ{/Ý׈¾õ‚'®Ûä»þnFß }ÿFt‘£Éý ívx»æLÄÉžeÔF!/%_'v·µóEIä ýŒÅ7²]Ÿ÷D|ù檷9… Ð&dclå;—×]ù¾:oÇ•YÖ-Ù'ûÕ"þº‹çFü}íçDü}íçFüôï½5Nn[ÈZíDøêèË1Šíz¾`ém|¸#:ËÝ>%b¡®Jxºà“‚B #ÁʉS ·öî¬óvÜA.$“íÒð:‘mÕÜ3Æ.g¶$y£[ Ílлo›ãØVj7TþJ÷&R“ÄŒâ(¾hlsÜÜÕž_˜ÝtùÙ%9ÐÚMYêåVN^}ê-H1¾¥è!vÆxN}ïÐ ¼Y.è Õþf˜rÁá‘ 'hXîÞ÷7wÖy;_äµŨǿ—¾¿Y7û:ïoöm>çýRû ß߈—Ë• ™ù; ³ÜépÕ<î¬ófÌ£ÞŠá]$D­×´M»³„›Ä )ÿOËmI^ع©3_ÊtùE×øÍ"7\%W¿¤²»XA9Nxyí“äþ;a—$7»hßû÷rdÖOGÎå¸9µmåüµ‰z0u •~:ÂÜNDõ”Ó¡Oý~0­Å.gz=Äm?BÝ>oÖþ$«’úÓðUûí8ø$O;žÛÇÚ¾ßNãçC¬7¹zK…m8/×Ø#’y)G¹)[ i›6WíŽjÖË?÷Rò7HV®”ë¡ê´ÎKùÏuÂ?$Íý‹©žïî´•œ}{䧃ÜÝÊ -ù¤¿“,Äå£ím!¢û í‘ïˆmJR0vù8ŽëÒ8_Úh¤~hÕÄûضBTo•9w¬Q3º=YF!Ëý·ÙÐò»(!6™A ò\v48*Þ$;²1ì‰vên=7yœÜ—ÖU±‹>«ÎMÓßÊú4Mç5âé€niK}R©rëfkªi8Ùh4 U:5´’}B¸Ï“B’/îˆïlkš1™?ÍšÜ]v&æfHÜË•VN!j¹¥µÐg*õRyT-MóC”aê”»âÚi:ÒžäBïÉG,µk²œÂîsmód7~G<äçŽ\¿8º/VH°'ÒÔn'g;ìePSnª‚k6šÜü$Ÿ¡"tª Oýป”ÜÞ6žF'⛼VÒLuXÓ€ã4ÊNêr)u›õå²UÊ èöÒ]c+ꤲ‰žJO@ªƒÇšìºÕu²@XÓOÇÊ™h›ènC7å§yȺ«Û;‹n2÷ÐÛäرói&§>ãN®âëòöv'›oÄS l¼%‹éçÖÉ KÂÝ̧w6‡`…A0úTÉÍ[;:É1˜öq8™qO‡¿j˜ó¼$¹K¹´þ†³;þ?Êü€Ÿ¿øøûöçs÷¶£…ïغ"Ï‘Lî!Ïw’å^òFìn3=O°?È.²¡YCãÞOxÍ£ºÀÞJºŽ ®sgb:œÈ¾›®¹š Wˆ§{"ó¥s3öù‘ÚàÐ É%xM*<ÌÃ!9óÖy4BòqjORt„tpÙOñ07ŠÇ©Ã2$[‰)9wÒU/òó<rpc~˜Òy¤Áãf禭Žb*‚˜æ‡¹avs§e%"ô‹Ýë‹ç}96j‡öN}š©}`B퀧S©>Æõc9t”‡æ¼CC½-š|z’3\c¤¼z˜mt¢æQm)æN¹ÑV¶@Ñá4Ûh½ÅÔdŠKµ¦–:â¬j)õœbPÿ6k:̳ҕæY‰»²Òn‹x=[¢ž…ÙœÓ%{.™Zœkw*’Ôhû¾yÝw‹¶Çi<‰#3µ£ZØ•|þ½ÄwÔK&´RíOÝqy\þ‰@-çÐ<ÕÏÅæ!,Wr'ÎÓVɳë\ß;¹|õäܦÃ0‘MNþe]„'0ÊmÆÔTûW{ôjŪCzž”­¡s»+A\ý§ê=-uƒ¨?&Y*Ê¿ÄB’ôf¹@>Ù W„ð²²ÞB©þ (Ó -endstream -endobj -865 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 864 0 R -/Resources 4 0 R -/Annots [ 866 0 R 867 0 R 868 0 R 869 0 R 870 0 R 871 0 R 872 0 R 873 0 R 874 0 R 875 0 R 876 0 R 877 0 R 878 0 R 879 0 R 880 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -866 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 393.004475 101.433071 374.254475 ] -/BS << -/W 0 ->> -/Dest (cb237-1) ->> -endobj -867 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 374.057453 99.933071 355.307453 ] -/BS << -/W 0 ->> -/Dest (cb237-2) ->> -endobj -868 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 355.110432 99.933071 336.360432 ] -/BS << -/W 0 ->> -/Dest (cb237-3) ->> -endobj -869 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 336.163410 99.933071 317.413410 ] -/BS << -/W 0 ->> -/Dest (cb237-4) ->> -endobj -870 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 317.216389 99.933071 298.466389 ] -/BS << -/W 0 ->> -/Dest (cb237-5) ->> -endobj -871 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 298.269367 99.933071 279.519367 ] -/BS << -/W 0 ->> -/Dest (cb237-6) ->> -endobj -872 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 279.322346 99.933071 260.572346 ] -/BS << -/W 0 ->> -/Dest (cb237-7) ->> -endobj -873 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 260.375324 99.933071 241.625324 ] -/BS << -/W 0 ->> -/Dest (cb237-8) ->> -endobj -874 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 241.428303 99.933071 222.678303 ] -/BS << -/W 0 ->> -/Dest (cb237-9) ->> -endobj -875 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 222.481281 99.933071 203.731281 ] -/BS << -/W 0 ->> -/Dest (cb237-10) ->> -endobj -876 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 203.534260 99.933071 184.784260 ] -/BS << -/W 0 ->> -/Dest (cb237-11) ->> -endobj -877 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 165.640217 99.933071 146.890217 ] -/BS << -/W 0 ->> -/Dest (cb237-12) ->> -endobj -878 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 127.746174 99.933071 108.996174 ] -/BS << -/W 0 ->> -/Dest (cb237-13) ->> -endobj -879 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 108.799152 99.933071 90.049152 ] -/BS << -/W 0 ->> -/Dest (cb237-14) ->> -endobj -880 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 89.852131 99.933071 71.102131 ] -/BS << -/W 0 ->> -/Dest (cb237-15) ->> -endobj -881 0 obj -<< -/Filter /FlateDecode -/Length 3174 ->> -stream -xœÍ›KÇ Çïó)æÀe²Èz†$ -bĈ‚äd#XsÈ×ÏŸ=]žá¾$±×«™&ëÁf±~Ūîå3áç+Æ?U9ÔÚJÖó݇ÓÇ…’ÿm%›ú{ü–tfÒ·\Óùƒ ¤„ÒR,ù|Â?W—kñûÓß~sú÷±Ÿch-GNçŸß™J%”Z -%GÕ "G•R(%¦ì¨2Übkž -f¤¢QoUfh«ÉSÁ ±:ª -3D Ë­ªÁŒXX=Ìà–Š£b‚‰\ !U‰ŽŽSÈpzöt‘C®-6WW ã=HÈE9±§k!眪§SØ’±§K°%±*y:Ø¢R‹§Ë°E²«ƒ-±ñt¶Dx§9º -[]ÕÕÁÊÄž®IH­ªººÕRou‘RHU„<sHˆMqu˜Q¤\]„-™Rsu°% EO'°EñãéÀ‰$¥Vg¶F›ÝBž.Á–‹º:Ø‚Á÷3l¡’=`ã’Ç€XJÐÕƒ@¬´jõ(k ZŠxˆ-Í­xbè"{ ‚-I³GaØ¢˜¹®¶HSa -x,-QÅc€¶ŠhñX À­RcÞ*qöX .Ì$âJÍê±@€\„RóX `®€ ÐÅt/ Ô•”£ÇvEköX à®(“ÇwEPÓÕÁ°ÇÐ1WñX à®0 -îbEÕ€8Ì +!ÖJ Ü•Ôc‚»±` pu%`ŠÇwcÂ-x+7¸E -îFÙc%€?äfà.’üt¶° °ÞCÅc i‰Ç‚îrÃòçÌiÐ*pmÙcAw¹bø\] \àqgÞÂËsn ¸‹…J<$pø¬ Àø€5z,Hà. VjOîrlä± »È’Ç„C`$ °¾&„®«kZ­ ¸ ,EX…UÄÕ•€@b€q ä¡ 2¸‹å½y,Èà.%Ü%EÚãê`‹Ôè±y t˜¶ÎœF²(‚žÜ%Æ{:p—¨4ܵeÓcA.&ˆÕc È‚£Ç‚\5´R²Ç‚Ü:¤|®.\'… H@–«ƒ-ZÔcëФU%‰ÑcA‰°)ƒÇd¡1²ow{[.îþ@a 2E Y¨MÕcAIû1xÕÓåj®]]…Ž‹Ç‚R4`)fØK…ŠIï±€–.y,€бz,( ¶`ªx,ÀRjDfîé¶ ÉöX€L:f5Âdƒ j¬ØefòX€{I¸ÚÓ)Ù>³z,¨šƒ¥Š 0M°Ó)]l&<`|l³<`óƒÍ&¶9®¶HRµFÛnVµÂ–HÑcbÓö›ÅcA#Ø‚ŒÏcA£,MöXŒCGä± qÅ~3ªÇÌKÛoVMûMlñœyÛ$Û~³x,h -[RW[45gÞ‚IØo‚Nž.ÖØÔcAËÙö›ÕcÒ ì7‘Ñ»GÕö›Å=.wS™<¸‹Ì4{,Àö›Èl½½=¼© ?ó•ÕvœÕ=1  7amu ìÅjUÜ3|mäÀLì,kvO °¬BÉäø‹|Lo˜p9}Z¨2¡`ͬÅn²aZ(²û±ç˜Åäb—ó±Ÿ<}ý‹ïùÇÿœÞÿf;)ë?ŒàͶ±N6EbÃpWakN·Bî ˜…0ðS꘥M#XÜÝÌH,ÅXnÛy|á£É°Ølf$3ÛéÚ/v -Œ¡’ìÛúâ:_ÈÖé#;+ÁD±5|q’n¬5&ôüúÌ:_ܯk¿ÓIž­/®óåmEc„ܽѱ_ßÈ' -?iØçÛ/O)HpžmäS…¿°‘ÒûÅ .¶gK;ŠlKv PZr²Xú²_bÄÑ媥¤E[ÛlìÊH…ûæÌïÝ_ž ŒËÅløWáÙHç7N_ÿáõëßýþ/èöÍûÓß¿±í‡ý¾úÇùÍëGZC†œ…*0rhÍPdz÷ê+ÞšøöÍÑg5Å–?×ËϪñI^þ%œÛ°ð0@š]çj_ð™âþ‰•JëC¾Bè#W¥Ö`øNz¼ègxçØ ’6›m{²Mž2ìYU~}“Ä"<ªDz` ëe5ïŸ?ìƒúļÁ.¸$Ëî\fÞS³ÎöLµ±W;áÿ^iÿìV^õ@9‘ù#ï=åó²úòv*+ØHnøö»?ÿñ¯ßMDµö-¤mº[ßvDÈÒÖ§yçR&õ2ñòüÏWnÀÛC“X7ôci ,?`ϬóÿDöŠ=I¢XÌ—¡²Ç‰kpÝÜ»ýs'‰QwP%uƒ2ß,!òöŠDÉo7ɱœ¾ß?ï®ì¹;†ZÚåý~Ò¸—„`´Íe­ÕÁ=Ô’,áVöï{8Jº\§=õýæƒËD›F{üÒì óªÓmL -×VGèÙc‘í8éñp}^ÏIr~‰¨ Ê#QKoŸŸ·—é‘Ò#b“õ¨Ñ¥ž#ÎÊ¥«õsKVžÕ÷ ¢N’<u#¢Ê·ÔgØu–ému³ã6â»ÞT _uøXÄÙ¶=z°¹~Vúô¼*¿ºx³ÇõÇÛ£ãÝ=÷ý¾Ç/ Ý${¿Áv¾m92þz²Èþ¹=`èÉÓ™¯’•cØ¡I{¹Cá†Vì<ì¡ ‹ï.ÈÛVYà,bE•¼O ;ºSÃ-~Û^¦*_иå(t L Z}w)Ç´mÞƒøýe91ù¶ÊçËïÀhÞÛÙÛ+ÿÛ=Øïö¬ ^µŸö¾{tŸ@ãR¿‡W×Ãøñ”·s]Þží2)ŽöòÏ9k—v¶³äJ¢ãƒ7GóùšÛ+SG©ý-*²«v¨~yÅj¹ØËÜüÉ›ó¿…îåÚÄ5ö‹)¾?Ù/êˆUC|¦Ø„ûWm)÷‚Cv”ЮKbzÑþµÆmÎ]¾ÆÔïÆê§^ÍrU¥^H¶³¸KÇŽ53ߊíî)ð.¶·†jý*SRî6C8ÔµìlpV|RÌB‹0Ý -iénŠ£v{Øž2 ×å)¦¯®ãeø{ÙX–á|LxwB·r-½w¥–@ô¦ºw# ™,Fµ!M½äãäˆîNŽð+êM+sè—Q³'>aí¤£\ëåÑÝÉ-··–ÆH•QªÎª­{~š2C]êp¼#Û|´K‹½¹Ì’þõ(Î6}¬©.qo„w§¤c&Nñý)§tš¬ ÔCoÊÒîWšñ2¥¥vWajv™—Û¿T¤!5òl_XGHÙ3¯y5;±¹9ÄG'­ò<¨£žXË,»¸‹K1»ú‹+9å º£ôðØAº¸l‘#=¢Ñtð8ŠyDݺ+W!u#ÐݧÊ÷ñ´3©šht–y ÷4Úâ<Øy¿ŠËq¶˜ÃÞÑîó]x·-lz-¶Ð¯½“)Ž4V»ÙÙ"œ† -S<1z¿‰ûlt²—2û×9ɼ÷îôÓi[梤h[ø,lïã¬YŸÿ‹2¯ñû¯“žÿÔ]ÖñHƒq°õ ^W2{{³>Slosvá˜6‹0ê‚ý)æ:˦ cqáµÎê4ÁÞKò (í*LÓ‡‹8ŽÐ,‹gÒ#»+/ŽDæsÇf.Ø—Ké7Gv”reëíà²NÌ%ù¼4‰++:—th5.q±6ŠË¥Ã6-;˜i¹v2\oöë:b}-Ka—2u:—Ùµé.Ûîb)‚XÖ‹µaåµÓv°Ç1ú“§×gûnrî3À¤íCz¿JÇ™tÏ—Êv™—cÍ‘íL¨OÞé¡Ñ–,sOF’3§ÆL-ôp±Æè"­³Ú^ ­j—:í ÅÀdÔe£ÅÒmÊ{µî–펫ë¥2rŠ)ýiõtZGe81­£’oÊZ»}ÅÙ’Œ(­á\æÝÑ,9rÉêÈòZ{Hu–”.»í[}÷Õö¼Ü†NCòÌLiVK7%?‰Ÿ2w.u®N!¼Rýg;á:_™ÀB­\$ÊöW0Tsf{â†-yѺ•¼¸Îãg'·'$ýìdí ÃT„ -Ûß3ù&ÜCÑžV,Mõ¿uò«5roéexðÝ“[… m/´olÛѸ,¶U´¿_Kz«½v[ˆÝ£¢íÈŽQÞŽP¶ŸÿaUÐÍ -endstream -endobj -882 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 881 0 R -/Resources 4 0 R -/Annots [ 883 0 R 884 0 R 885 0 R 886 0 R 887 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -883 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb237-16) ->> -endobj -884 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb237-17) ->> -endobj -885 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb237-18) ->> -endobj -886 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb237-19) ->> -endobj -887 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 681.384971 101.433071 662.634971 ] -/BS << -/W 0 ->> -/Dest (cb238-1) ->> -endobj -888 0 obj -<< -/Filter /FlateDecode -/Length 3342 ->> -stream -xœí[K$· ¾÷¯è³•%‘z†|q#ñ!Èa·ã±ôXç¿Ÿª’ÄêæÌÎì:ÈÁ³³3]"õ (ò#¥R‡³ÇÏ›€?•ƒ«µ•ÌçˇÓÇ“w%yüë5…Îþ ~K:ÏÅ¥ÐrMçB âJK±äóõ„?7E]ýzúñ«Ó¿ŽmÂ9ºÖr éüëOÂbr¥–â“Áj`ù@+%WJLÙ`åà0ÅÖ,ÄH…#ß³ -ÄàV“Å‚c5Xb—@÷¬1b l± Fh©¬à!GˆÞ›<â™)¼\†Ò³Å‹ÁåÚb3y¼P¢Å#r¹pHÁâ5—sNÕâ1dIÍ‹— K -ÌÞâA¦Z,^†,”É›<Èk!‹W K„všÁ«ÖUMdñÙ‹×È¥V™M^Ï×RïyÑ'—*‘·x!¸Û$“Â"åbð"dÉ>5“Yùhñ²0~,p"Q©ÕðÖ(ÞMž‚ÅK%ÆÂ&²`ñ-ˆ²ø’-€9Á% b)Ž[d b%Ç•«…±6Ç¥±%ǹ Èðb°€€YX»ž‘ö˜<ÈB5ZX€¼<¸­áÓHV@ -‹Üõ+lñ€»Þ—faAîJØ´° !ÄjatÈ‚£…¹²k¥d róà!å3yÙ¡œ,,(„È2y… [X€ÄÚ5jÕ‚! ÅhaA‰)ƒ…È\ ȾÍíd¹˜û†,È-,’¹Ú˜-,()c?­Z¼]-€k“WÁ Å‚RØ! °—rNoaÞIºdaÄ/°…¥A¸Š…E®Fdæ/@$Ù Ó/ j„,È-,¨±b—™½…ˆÐØKBÕ½ì3«…•³“TѸ vš@J“Y`}d³,,Àæ›MlsLd¡ÄÔe»Y-,¨²D-,€mÊ~³XXЖ6*aMœù`€ñßMd-K[k]ö³KÔC ǵÆRËbë%R«lUzÔóUZ×än²jŽÅ¶¡¢;*‚pG²lŸG]<Ý6´ÈGÒtdMÃZæpAâŠaCÕ¶¡ªl¨¢ö-9åUw§‰Ú†²iC“º&½†ZÚ±ô{yõÜÿ‹çJª.ÿÎ÷Ê´3çðÁvÈGì 7¥ûWç{u¾ß™óeùWÎ÷/p>-°š±VÄÁ²_ïÕù^ï…Îg:ÙËRÎGòÕñ^ïwæxÏO9ï÷ûx{‚dýR >;æÚ_È[ú"Gšç·N_÷ßÿø×°|ûpúÛ7Þ‡Šß€ß"ÇòøŸ½gП)mÏ ¼”¿ýûùí÷Œ!oÊçÒ-zòÌh¶O~·wå÷Ï Ýn]††w|Þ¬"ö½Ž8Ëe€³¼“TÇ¡~û&|³„O‚–ðÜöÓÆK{;L¨òE¤ÙŸßíuÏVDçI½>C]n[ü°×W¼÷qoåyéXÏ×ßêlÓ¼’ã²7e<Sw7ŽÞ†w/ûÊ&{¾þ½|ìscÕW]4iç/»ä}*ù08'–e ýˆ®êÖ‡ÈþqßO&9ЖK÷ýôù¼[ó|OÈAÑ?&Ç®+.K'iNq³tC6¹<“=ÇÜŒ¾ûz_6{êrÒÞWÞ~;OéÝ?ìõ†LïÖuYª²¡½¿Þö²Ùœ%s¯Ÿ·2«>º]Æãx}-‡< - ËG6›Ý=©s¾Õ‹èº·›²tÿÔضoÞèmèàý+½æ°Ó†L¿ÚiÔ{®/ ¦=!%Z–O#dZ³áù -ǪMôç÷Kë»×½ùˆ¦ù›%íÏé'~;á€Pü È}?/CNü˜_Œ ÷}÷(øXÔ¬,þ9Þ›ž×GÄÛÑæf.³¯¸Ë¦äÓÞ,Þ.{ÝѾ×{RÏåVáVû‰¤¼t ÃÍwÞ°¾ÑÙîåÒF€! „M»Î.ÿD~Øuqk'» ~÷v%ÐïJûs’»¯¹ÅÖÔë¤ýmÃ3ª<ýBÂxù´~ÿªiªZ7t)¿Yñ¹yWje©bÊñ'üÖ]ñ²e«L%ùë·͘–úº ¼WªÞO[½ào2߇Í$…Ù¯ [׳µ4ñÝn^ÊŒý'µÌÙ42fs¸[ƧÜßù‡~ïËçêB”‹áçÌÈòä¾ÍYîTOƲ…®èp~¤e¿NÿqÖÚoØ{¹t'.¶ë÷ª°×ÿØü¢»<âP¿ ë;¹ÆQXäëInC³Af^Áìd!î~öÅaÆÆA»œÉ®ë¦&%ç÷Ç@)ǘÆl¤}Íä-5‘8JâÛÀ-s¸'Ëì½ ;Yn”WG{)OQ°go“¸:d]wu¸~’È+bº'z5Ü"Gò¹4U—qŠ®UÔòº±¨å|ŠxÁsÌwQ¯&U®†v‘K)¡Ú¤¦Qs.Ïu2H²-¿#¾ñ£kÆbN¾Z5¹ ´ë0$žõÚ¨g.'³ÞÞ[š+Uf­ºš¶¡ù%Ê2uªSñí¢’Ì"_‘Q>âi<É’ƒÌµöUÙM¼#^NÈQøŽÜbîk§…> ÓÓDRýNrõË^µÔ¡*¸æ É] -ù»AEšTAžþxš”܇Z¥5ˆøæ$•¤éy¢[\V]¥®PÊ´Y­¯P½Q_@wÖž;P•Ê½´!ž€Ô#9L«›d°¡Ÿ‰•šè‡n‘Ó4åë¼äLÓUaìc°”¹§Ùï"ç‰WM.sŃ\n›òÎ~•Íâ¥6¾%‹é×1È"G?£ÝL—`…E^0zíäá­ä ;ãq9™q/§_N=ÌEJ±Éa=¹«­³®pþê|ßžøü§qx¿Åñè'Zĉ­²ŽdòÍžúL²|Óg§Û(bdû‹ꪛ†5Ì8£Âk^ÍýöQ3L “«‰iéP‘ã4ÝHi&=A¼Ühq&2_º6+`oE“Ã0$×ʆT(èé XVH>«.Q’ª+¤ƒËQ5DAwŠ¢°-ÉbŠAêA¦êE~ÖKA2–*¨4BŠ´x¼Â¬îzÐú,T¤À¤ ºczÐvÇú³Ýë‹×}9ê„öI½jꜘP'àÙTêÅ|,ΘCg) ç]š}‘ò=šIÎr•Zð¡ mTQëj¶WHzP´Ã #PL¸ÍÚì± ™òÞl¨¥Ï¸šZ*3§XÔ_´¦“^•©Ä¤W%ßÕ•~GÄ›ÙÍ(is.kv~Õœ¹d5hY·žT^5iÐîÇæãØ#ÚžÕ4x ’WfêW³tWó³ðïs|Ç|%fÕ?ýd$ÔušC¡HýÒ¾æ aù’K᪎J^Üæs/sê‘°Ž‰| ò]w[„+íÓ ÕÕø¼Ý¬ysJ/SÀ£÷’͘¡Ù±¿ùìD³Xd«(ßm&Iz«|%«øG^hò~Œòþp„Òþ =ð¾ -endstream -endobj -889 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 888 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -890 0 obj -<< -/Filter /FlateDecode -/Length 3187 ->> -stream -xœí[M·½Ï¯˜³Ñ$«ø>ðÅAŒâCƒ4±l£rùûyÕÓüè™·’V>øàÝÅît¿b“ÅbÕc‘Í gßWÿªWk+YÏ—÷§'ïJòøÙJn6ñßðWÒ9x-.…–k:¿7@Š+-Å’Ï×þÝݮů§¿:ýçøL8G×ZŽ!ýÉD*®ÔR|"¢‘BD)¹RbÊD”ƒC[c"¨‘ŠF}¨¡­&&‚c%¢ -5DKGQƒ±e"¨Z*D<ôÑ{*ƒ"^U"‘…ä2Œž™,—k‹Ê -d¡D&q¹hHÉšË9§Êd -]RóÉtIAÕ3tQ©…É2t‘,žÊ K¬E˜¬@—ë4"«ÐÞU© ºøì“5q©UU*kùZê£,úäRñL‚KðM¡2D)"‹Ð%ûÔ¨ º$ñ‘ɺ(~™ <‘¤ÔJ¢5Zt‹—Àd ºÄX”Ê  Ÿ1@ÌÐÅ—Ì(䌗ÄRœ¶¨Œb§U+cX›ÓR„Ñ@lÉin…ñ€øY ŒÄC—¤™1袈\*ƒ.Ò”qDè‚Œ D KTa\ `[…·0.ЭúøV}ÈŒ „ 5=ããJÍʸ@@¹p¥Æ¸@À¹îd\ ]„{a\ `]I92.ЮhÍŒ ¼+<ãïŠàI*ƒ.p`Æ`G'¡ -ãïJð…q‚wF‘qFÕÁ3ãøŠ‹µzÆ -ÞÕ+ãïÆ‚)€ÊŠC -ãïÆ„.°™¼“Œ ¼5fÆpþñ4ï‚BO  –qæ{È¢0.@e."-a\À»¡aú#1 ¶r¡¶Ì¸ wCÅðQYs¡Àâ$naernŒ x•0.Hà]Ðge\Žw˜Q#ã‚Þ ‚™šÉÀ»!6ϸ w‘7$Æp0.Àü悇ëRYs¾Õʸ wAK‘qfaç+˜„ÊŠƒ#Æ c瑇2.Èà]LïqAïú$¸ ƒw½"í¡2è"52.@^–Ä4’Uç#˜‚ÉÀ»>`„™ ¼ë}iŒ 2xצMƹ+ã(èGƹªk¥dƹyÈòQYv¸OŒ Š@YT]´(ã$Ö®I«Œ J„.#ã‚¡ RÆÈ\ ȾéòºÀ]èú@¡ 2EÆ`2W›*ã‚’2Öc°*“åèj]SY…,Æ¥¨ÃT`-å*‚žqÞYºÄ¸j@”qAiС¸S‘«™9“è‚$›q2]ÈB`\P#tA6ȸ ÆŠUföŒ 0Cc- S3™z[gVÆU³³T‘qÂ+M0%•AÐãŒ-¶ã,~°ØÄ2‡Ê ‹$e\Pk´åfe\P+t‰>2.€oÚz³0.hº ãc\Ð|v–&3.Cæ=ã‚*Ö›Q .m½Y4ñXob‰Gâ¶I¶õfa\к$¤(T]45·à$¬7ÁNL–¡Klʸ ålëÍʸiÖ›Èèé–@µõf¡ÛàÝÔÀLLÞEfšàëMd¶lmïA¼© ?ãÂj+ÎJw <¨7an¥[Ü‹ÙªÐ=òµ5Ý4€šXYÖLw 0­B<Ý6ðà_äcúÀ ·Ý§u‹*à -æÌZ¬“ a¡ÈîÇšc“'ŠÝöÇ~ýùôõL¾çŸÿ{z÷Õ¶SÖÍ7’)h£Fûý Áòym^à+àŠÌôÐt/ rÍ÷à¶k±—ÓcY -væ±×#ŽÕ`Ýð¥9äµ½ô¢Û‚^–ž,ðõ„ÅV¤wpJ³ìhnuÖ;áÙ“ë‚ÎNϦ¦u˜}/§1ÐÖ:¦5A*ÓšòÁç ~sîCe­¨Ãa[>÷²¸z>4ᣭ8²¦î-£¹`ó -ñ¡Ê}¨.>TQúNy–Í­àêC™úÐ@g§gSÓ:̾——Èý]"×Ruû9?^–=Xgôá=È'ƒì`·Åö/Á÷|°àËöSÎϾUá¥Ç«!žý|/Á÷|Ï >dÏK9ŸÈ—À{ ¼?Xà}~Êù¸ÞÿÓë4ÛÕ೩S­Ûë{K_lKóüúýéëï~øëŸÿþÖ¯ßþñ÷¡â/Ü>½xñ™ÞâkOµkÈRÆõ ï ÿöŸç×ß?Ñž½5íYQñª³÷åVU׫LñÖÄÖtž÷Ö|J·{}wk.4 ü8 ôjÞb}ìD³ˆ1Û»Û(µ·ê·¯Â7½“˵®­ÞðnŸÚö¿¶?cÚh7ÒeïÞÊt\¤k½?Óº÷zu¹Îó¹ÑFÞe—¥Ü›gô^ï{Ï4Ý´Ü5܆órÓ&áSÞìx݇ûí®Y<~êO7ÍõalIO=G¬>Ç|ÑíÝnÁt{οµGkv núàÙ­–wsåƒÊš •}t7߬·:LŸð¯Çz²Ø6¹Åx¬gÍ7»niÓï#zHôOéá÷1(ûøè’KÌÝìHNös#uožÛzLí}Œ7}7½­þ·‹—ÿ´Û"Í1°v·ë2íµgÙëÝí×õï±Ìôïþ³ÕYvò¬«Gör~üQ»ê}ßÓˆÌ5*—ö÷vÃ>^ß½Þ·Ô¶³†þœììXn±µe;vß­ûŒ"ßÐ#›·{ë[} ù¥Ï¨ÒN"Þ#<±{×ÀXÞÜʆ3ÖIzF ²Ÿ™Äq#ƶ—I‹™òÍ5úld¸S€äF~» 7×j7ܨÐ}×÷î±µ÷fw·Ëîöõ®þt7\oÔ°õ!õ>< ã‡SÞÞ™…íÜ„ÏÕ…h+ÏYÁ™ö¾úlïéª }ØÂfèp~âÉí8ê‡Qj?¡êíЊ½°¼_]nöò¶FW—;‡o§Éü×Øo&|=ÙiB%°êœ>¸_úQ—†1¯tìr -bYË]IIÎï—sò‘Ô/cê½±çSÌÞò c¿³¸5L@<™Ã#l½÷.ì°È¬Nö»øI8ˆ_Àôú¥¹ Gíú{ƒ?L—'8T_M–áïecY†ócàYYïïD¯µs`½ªn]Ì“E©6ÐÔKŽÑ1N²´ö|å{ÕŠÁòeÔìmú ¬Ý‘t”k½.'Zn¯-‘*£T¶nù©Êtu©Ãð», Z±#æKŒxé—GØæë1Ö¾.~@,9tDâ„·¥Ìcé4¹Â§îz+(K½®~úËD±”ÙM…Ð옽‹´ÿ7ªH5æÙ.‚—²óón6b±9࣑V<ÖQf e–]ÌJ>»Ú RÞHw”; ‹ÉËk?ªÎƒ<Žp^7`£°nŸÁ•+è»hnÂi¸òõO=ÓU8{o,‡ÅÝÓ¨wÂypçu…Ëñ`‡C†¾£ÞÅç;xÙ&6½‡ÍõkodÂÑÙn6¶€S1„'^7¸Gë`';ðÞ/g±÷rúå´MsQRl¶Ù%ÁÎ:®YW8ÿe¾Çß¿OzþKßüºÍãѶˆƒ[ð:“ÙÉøú™°”ïà›ŒºÐþ„CeS÷†1ǸðµÎÇý$ö^2  Cº‚iÚpãpÝd±Lúx¹³âHd~ëØÌ ûv+½shFìXF× -7kwp[§Læ”|^ªÄS:¤—q³VŠÛ¥Á65;¨i¹62Loúë:bm-7Ka·2e:§ÙµêŽm½XŠ –õf­XÃÚh;èC”þâðúÍ㾫œ{:¨} ×3tGe»ÍÇÛ1çÈÙnzðN ºd‰=IÎ ™ZèáfõÑ­ó±½@ZÕŽíÅ  `¾c£ÆÒuÊûcÝ,[+µR9ÅDY-ÖQFLë¨ä‡²VoŸñF¶$#J«;—Ù;?KŽ\²,¯OTgIéØcÛzl»Ï¶ç¥:É33õó±ôPò‹øïKb‡n)³RýwÛ uGÃD­¡H”톾æ Q;$^´.[%Ï~æKC­-a8œŠø컢\…+íÓ‚¥ªþ=RþXó´KÏ3À“çúè €°}Yh{s°mÛbKEûn XÒ[í+ Å?ñB@÷m”Ëa eûý?‹½ÊÒ -endstream -endobj -891 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 890 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -892 0 obj -<< -/Filter /FlateDecode -/Length 4332 ->> -stream -xœí\K$· ¾÷¯ès€•õ~†|q#ñ!Èa¶wÇF0`Cþ~>ª‹«ŠÝÓ=;FžÌN‰TIE~$Õ]åŽ?ïþ«Ñ™Z[Éñxú|ør°¦$‹½ÇªÑÙÃoIGgc1ɵ\Óñ3B1¥%_òñé€ÿ6MÙýéðóŸÿ^ßãŽÞ´–½KÇß>+Sj)6)¬–uAa¥dJñ)+¬ì –ؚƂ©D÷¬1b«IcAŒè}UXb„X\سÄðÅE1\KEa9 9œ·VåAcð -Ï%“¡ô¬ñ¼3¹6ßT^ϯñB0¹D—œÆk&眪Ƌ%5ë4^‚,ÉÅh5d‰¡—!KÈÁª<Èâk ¯@í4…W! ¬«ª<Èb³u¯“ZQå5ðl-uÏó6™TC°Ï9“`›AåÁ£°I¹(<Y²MMåA–¬×x²Düh<àD -¥VÅ[=yw°Ái¼Y¼/QåAl¾†>C[²ð C¸¤a€/ÅÄ棾k¬ -øÚL,%h0à[21·¢á@°<ï4 ²¤˜5$²Dx®Êƒ,¡E ‚‡,è aAÅÇ aAÚFX‹†pmsàm´.kX¸ÓjX€¸¡æ¨aA䔚†˜€€.ܽhX€º!e¯aAì†X³†¸¢³Dàn¸SåA°†@G\ Dànp¶hX»p#¯avÕÀ³†°ãkµDய6jX»¾ ¨¼bàƒAÂÜõ KÐ"7p×'ë5,ˆÀ]}Ö°†b€?VÍ€»€¤§«aâ=x>hX€ÁŒGZ¢aAð§ø4Ðʸڲ† ¸ë*¶Oå5ã -4®ø-´l\ÎMÂÜE  -$à.à³jXŒ7ˆ¨^ÂÜu‘ZãwoVÂÜEÞ4,€9‡@ÃÄ7ã,LWå5c[­$à.`ÉkX€(ll’¨¼b`HNÀ±±ÈC5,ÈÀ]„÷¦aAîÚ‚†¸k#Ò•YBõ ¯n«ø4’Uc=Bãw­Ãk<வ¥iX»65,È…¾jX ²`¯aA®Ñ´R²†¹Yðò©¼lÐN Bd©<ÈKÔ°‰µi¡U Š‡,Á{ Š‡,H4,@F`šCö­–æ¢Ö² SÔ°Hfj‹QÂ’2ê1hUãeoj\«¼ -ž+” B±Ó°µ”©pz ð†Ò% x.jXPd«hX€PdªGf®ñdA’­a2]ðœÓ° zÈ‚lPÂê+ªÌl5,@„F- Uk¼h©Î¬Ô˜ ¥ŠÀMPi)UdLhX€ý¡b;iX€âÅ&Ê•YBŠÔê©Ü¬Ô -Y¼õÀ6©Þ,4 YñiXÐl6”&kXÏZ š«¨7}Ô°~IõfÕ° ‹z%žâ·-dª7‹†-B–„EåA$MMñ[`êM “ÆËÅ·¨aAË™êͪaÒ Ô›ÈèÕ#JõfQ €»©™4p™iÖ° Ô›ÈlµÚÞxSA~¦3+UœU=1°€Þ„تX`/¢UQÏ ,À—j$õÐb¢²¬Y=5@XÓYõØÀ‘Å&œOŸäU†9ÄÌZh‘ n‘Ýšcv ºÏÇ~ûåðÍ‚ïñ—ÿÿÔOÊø‡l#‘“¦±Û¾a·? ”‡Èk³ ?\‘™nÈ0Í}®yKì§x:2²< ƒ#‹}ZÓQÕÕNÓ9äšKo!› žÄJùé€b醜Òè;§“Ä<Çä¹’'A‹žSMíhú=~ÆF‘cLmT -ÂÝŠt¶çl· ÜA[YÚÜj‡à²]¢Ù¡[o5všöZîØdmú'¡F¹\¡tIî+–Xt*r B·&SõÌ}qµ#®Lh’×&$éH~ØXÆtŽÂ¦bBV7!+L¨¢÷–œò´–1$Jʪ ê\ôœjjGÓïéÍqÿ/ŽK™:ý;î/•Ú™kø¬;äE'“z“ºs¾7çûƒ9_¦帿¸Ãù„ÀrÅBkË~s¾7ç{s¾;Ou²ûRÎ ùæxoŽ÷s¼ÛSÎ}¹ÿýû$ëß©ÁßMŒµ@Ò:Ñ<¾ÿ|ø懟þúç¿ÿ„úñýãáßZë*~~èTÞZ¿´ä–¿ #~Ïü€ßäÏíxÂo^þRßüÝ?ï<¸†ßÙy7›(dMˆ¹¸ …â|¨$ÖY’ïÞ¹o§4ýo\¤Ê‹$õ,M*çkâuéЦOãI’äÎãôë|¾¦þ¼’~/ñš#Ѫ–¾‹cœ²Ü—V+í<¾·ÓhŽ‡åžÊZœýXþ!Óv¾´h“ûܮ˶Õå3 ]ðz–í¸èNÊÆzÍûõÚôGÜëÖëÖrZ¬!/¢g)³‡7Å‚c™ÃÐuƒtê>îÇɪéËûqúÚæžA¾krIŽE'q±—•íø³‡(²Ñ—b²>7eìnËMxÐã2ï ÿAØeYú.:ûä–¾ŠLì݃ËÒ¾â?½ßÊ´ý_ÆYdOéªNÛvÝ} 4ÿÇE†¼ø {š²'^Ë‚>¬‡.G:;íõ1Öž÷cµÖ¹¯[ÓxÎÝŽb!xSžG±4½ÇoÐexœD*%žV:÷Íyò¢aF{Ö\ï· óhG»u? -«ȹ[ÊÀcLK]úÑüŸfüpþ¦­FyﻤQàÌ£Øsosß÷š`IËÚ¾X«2üKß¹×ÃÄÐèsЯÇÄý8÷a"Àë’_‰û±{V‘·–»¶Ú³E®±mÄþÇ>œÛŒSëK†…çIq™1õñl?n™Ã§s{g'q½'öq-S|¼ªã¸ÕÃKÖ²Ò‰\_™:ëã¦éáÄãµ…E×,û/f»ßÏÞGûXaês;·¾gWõ”wzºE2žlcj\æfý°´y=rÂÓÄ Ž+)Ïñ9—"c¥éè÷š|Äïc?ØU¦1Iu~Øμ–ö@}8—M‹þíÕX«fg]ž«æÃî…Ÿ¯#ÏÔ%}!œ£ùToZdKB·Nô[ÖÛûÞºJ”’ùÎV®¾&Þ¯²«¯Ó³/_Ó]²ªîD^5d¾ ] ŒíÒõéÏûJU”?‰v:W^!oìk»Ñ_ÚûÜí6xuÍ;|~][}¹^z®ð:kÜcïï±FÞ;ŽmE±[%×í6{ÚØ®-sQ›^²Á­?:ö«~wU;LÞÁ'UàɨDŒø0í›sû~ÏÇëºàú!Þ¨“—éà,ûµµõZ¹ÏUé8ì&]ªçF-sc|½VC®³r{=Ÿøv)/›Õ„ŽÑÏÕu‰O“Ê5Ýå=ïòu»ÉÕ§î†ýɼñŠí©ù&_s -êv¶UiOAØÐÈF~¼¬½Íó­¾/¼Ž2ûŒÓ6Þ–s¯a3Q¯ „wؾ­ý‡lÓاF.˜¦õ<ÙÊ}\çè·Øö32ï°úšMJ•9>ÛçõZ®Ýé·æ.n®ƒüH®ØÃWú] -sÏèhŸ#g¡c¶•“XKbÝ/yœyý+ØÃ-ó?[°8>(ØïïœûãâÇ›XžÊí'Åãž?ÓÉU8c[*Çseµ>Yïk§ÛÛÓ–þƒ°ŠÍ™K8]>Ižè1O:V§Ö│½ö¢c÷Ä9P²wžR¿Õë뜚”œŒ…S“ý8÷šTë.ɱxÓËOMöcß{’ìoD¶W=u¾à×úªŽýV;¤>£eN׳»Î¶kÈTÄz°eæ>[Î(DöÊÙ?¡þóz°ó¤êÚúv'7Wõ·ú•2ï g8› -|¥Ã­måsÕÆÕk¸a]^œºõùAȒΓ+¹þBTøeŸzwœð×JÏ?? –Ý ÊÉü-ç×÷|w¶œ¥ÿ=¬Ïê6}Ø:å' ÛÏ »ß®¥f·Z"l÷bµ2v}–þrÕï=Õÿ -¿gºwUzjÿwÂøýØ| -Ç™òö{ø¡ülèžOó¶§Æ>%TOxÅɈŒ;·}BØÚv­2VpEÉ×+}Xd{2?èzßâî=±pu² ↪K·ÖÕvo¤Îä)zz¸¦+gÝVYw uÇËr}ìD)Ò7CƒÜjt º?¼_Vè/q±ÇD/åÈÍ·&žsYžƒ¸¡ËõG%”§b–Ù÷Q|ÿ^n†¤7ÒÐ|4±zˆKûþL?¡ütöS-r•ß£Ûß±«²OϪ'º—ŸFÔy½ÍÕú|ËIß2áCc|>©ò¬b?Í™x »mürÈýaD×H·¹çé5¨Ý ÄÒŽôdµ!:Þ6×íŽîìïùù2z-¯þ±ô6zôü^ ÑXúéþ!+šÌAª¿¦Ãvrõܘ䧽¦%*äg¤{†LÄåÒŽ±(×ÚÐNèû`›ž!»\Îè_úÄ«¡ûßFÏEËB€ì<±BÄÙíÉ´zkÜB¦WÝT–V¢¤Ì2ÓwàÆ€QöΟ%»`1í‰VL7É>²<ŽªË“8D—ªsbû¹/ªRñtÀ´aK}R©ô‚ Šµëí !TÔÄ=džıO -‰¾0¸#¾³Zåú[û3Yý€h4éSMO/] ”ôVzW\±µŠç]X¡ôŸÿf£» -endstream -endobj -893 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 892 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -894 0 obj -<< -/Filter /FlateDecode -/Length 4324 ->> -stream -xœÍ\K$¹q¾×¯¨³€¥HF€±2ì5,Xð@>>Ì”z$½V>øïû V’ɬŠªÌžiI½½5ÝÁGd0øÅ#™Î??üS8¸RjN|¾ü|úåä]ÿZ‹ÍEcÿŸ,çà9; 59ÿ¬Ê.W‰9_OøçærnþzúÏ_þgÛ'œ£«5Š翼(‹Éå’³ƒUÁò –ˆË9J2X)8Üb­ bHæÈ÷¬ 1¸±Xƒc,«@ âèžU!FÌ-ÄU²Á -r„è½Éƒ ž™¢Á ┞,^ .•«ÉËà…-‘K™ƒ‹W]JIŠÅcÈ"Õ‹'E³·x…©d‹— %ò&²Ä’ÉâeÈ¡jð -du“Y|òÁâUrR ³É«àù’Ë=/zqRˆ¼Å Á l“Lv)eƒ!KòRMdòÑâdaüX<à„P.ÅØ­Qw7y -O KŒ™MdÁâ[dñ9Y€ä—, ˆ9;®‘-ˆ…. -ÄRçL Ä*ŽSÍàÅ`yÈ"œ,$ Y;×äAªlaEÈ‚A–ÈdamÖbanÙ×`aoÙ‡dap!¦·°€€¸T[X@€\˜Rµ°€€¹ì´°€ºØîÙÂê’¤hav‰K²°€€»ÄÁ[XÀÀ]"ô4ylaÐÑQ(daw)ølaw±¢…XU;LÀV\,Å[XÀÀÝX<[XÀÀݘáL^v؃daw£à,Ï Ü⣅ Ü“…0üñf4Ü„ˆ@¬…ð÷àE²°ƒ¹ˆ°ÄÂî† -÷gìi • ¥& ¸ -–ÏäU24nì[hÙ…”ª…Ü…£" ¸ ø,ã“—®Å‚ìA@–Ƀ,œÙÂÖ®R-äY(F r„,,,@Dàj@ôm¦æbæ Y)ZX$s¥2[X%!ƒV-^Š®dÀµÉ+à…laAÎìàŠƒ…È¥\Á¦·°ï4\²°b€Ø‚\! ¶Š…pE®DDæ/@Ù Ò/ J„,ˆ-,(± ËLÞÂxhä’PµÅc¯yf±° pr*ZX€m‚LHiò `¬&Ûba’$›HsLd!a J‰šn J,ÑG `›šof ª‡,ˆø,,¨>9 “-,Œƒç½…5䛑-,À¾Ô|³XXPÉ#ßDŠgìÛJIóÍlaAeÈ"QLdAÐT} LB¾ t²x ²ÄÊÔ”4ß, Ì@¾‰ˆÞ, Í7³Y.îJ2Y<à."Óda.o"²µr{à•ŒøÌfÍ8‹Y1ð€^o5KØ o•ÍšøjŽd &2˒̪Ü*˜Á›eüE<Æw˜p­>Í%ªsÈð™%ëMVl Ft?rŽµ=hv­ýåO§_†ó=ÿéO_Õ*eý'Àø‰“&ÖH7RË ¸rm8F0ªA3aÿ4 !à·ôQ!r]… €Å¥è¦BÂÃÝÖóø#lE†Ä*s@0ÓîŒçy‘),%[Ö7÷y'YW%Ѹ(âg%!膯QÇnéõ`Ÿw×ë<ïª$KÖ7÷ywYi!OBX/ÛyM!÷ïJGúßùþ! ’êÃBî5~G!ïá?à¯Õ÷tžl!C,履O¿þÍoÿý_ÿ[¬Î§¯§?ü`™½—⑯âsÁßYãxüÀ†ñYètå…?þø_çO?ÍsE¯ =ü¡lçÒá=]›ÿæTÕªŽgKÓºeÛÌ1hŸÒÂYØœoO†vßÜå›ÌtÑ©‹WãÆÕ¨qœ 2HÏwŠ–>ëUã2"\Äݪý_~úéŸþù?Vµ«jù >eùû Âd$.j˲”+ûR}½.…òCÐ?,¿ùˤo­¨/¦©…þHhQŽêƆ /NÇœõýæ.}sŸõªïqH†¸!?S¸×@ôaË?]„Ëõ·*qCç…ž–ëGJEÀH -!°ßSæӦߡÄí,‚1°`µž“&_$ -Ï;Ôå=Àÿ]–{•;e‡¾‚øÍZî±teYÊŒßWàz8¸hÝ•[Ü»|Ʊǽ‘%ûE£ß»ÞÁ¯’ÄòÀ˜r+˨ÇÝ5¦§Mߢ"òéXb«öøÍW„Ù{’ìóñÌ 9zKõцë/*…åż¾nÍŒãÙÉÍ︽ö×_vL *p*BL7b2̨2™¹ˆÑ[^lclÏòàsŠ¼aíõù{;ŒU^ͺ0mͶàùs|÷o‡ýÃlÃ,†°³pH™sõ5UcÌýeGàÄ…)š½¥G[3¾EM/ÌYàp‘Y$eDZËznGz|ÓH°_ -z„Á2䨥 ÕËÌ9íBøó¦ïáäÅåC;)P9Ç´ìóñ \Ÿm‹¤TÈŽòzœÓ~d°îîí$Ç"­£_j°zþ+:—Ï{Q±ª&"™÷¥±Š…ÐmðDZ¬ZÞ3øçMßÍàE\IQŸ9ˆ>B)¥=x.Ù±>Ð൲T}óƒóÆ,Êjb}3´´U3þ/{¦²:£í´m¦—í #ºq|2öá¨å~>5Mþ|ÜwúKë¿•§a¸Þ}k—odœÜḯ÷ã·1â4ßW·94À‚è£d)ŸCùº×;ˆË0?ŸMýìõŽú§(°«©wÄ{#èáN€ÆÃõ á&ǾѷømÈA/-=ß—=éc·$’-KÜÕ›>4×ÒYZߟ›“ÓC -Õê}Ho)i¨êêÇóßÁlÑÓí)ùqT;ÖçÄ\TvhÄM õ sôÁ¯[½AœÀž£ §9ÿßÀL0`Ÿ}rx.ɵLŠw°àX—½K€Åô ®éd0ùèfÈCæ7È}ï·Œ¹?O0³3NLjü%˜R,¥ÅñÔZ0ƹØ~‡IOÏû„ÝOâ‚ÏíPÎs[;Öçã#úÏ^Oì›ÆvYÃÃöXìE?Ç3¾íðÇLgZòûÞY‹vÇ‚û^¦8RXü9§ý¾w3Ëχêîz>>G–D¦kO= p%>›;ÝE=F ”¼u?/4Bê1rñMïG¨ÿeòáÁÆÀ4®õxÎÑòëÁ>gãé‘_ŠEä]˯Ûaߧüz?æ[ʯFïï,¿r°æä‘Ä$­¹§¢ç’ž[Ç¡.Ç8*"–*ÁÞ×86ÃÎÆ!_nRUSñëc¤Ïàô%”vó©âuù0Š}yO -àØv‡uQ“­Òãnaô{hõ¼é{•E²«‰õˆu•c§ŽuùxÅ@ÉØà…¥§AóApÚ;Û ÞZù6zË}M~ä·;¨¯}ÄØŽ¬íţϛ~Ͼkw{èVÒ㤜jÈo:‚¥Õw™Îiém–³?ý8Ö­†RÂ*”v-ê{aDí´ÔSUìó÷ƪqâG_É¾ê §fÉhI«ÛÉžt| ÚËÍã•zÿ(eð¾¡jÌSU¸{ø‘òߟ0võø@_¨® 0 d›Ób6zbo˜N?FÖO÷ÉõZÒ|¤ìþdŸVK«¾Du3iÓïÓË}ßjzûìšÞ“” ÌRŸY ÿ|â>/)ÊekÖƒžúÝÄŒž¦9¦çÇæ~ƒÕÁJžYÝ°¨¼•ôräduË!Æfm¥ÉqoqHI«”n&|fq%¦n=@ÚŒXKßYyjpǺ|4{k¯=¶·§ëÝ5÷»åý¬‡ÇD¿;AOÕ×éu„¡¯Ý&;ú¹ó#D;‡›hkvR¿8K¢5àšõ]›GF_®×<&à,^®‡—ÛÆÐׂøz:6,œØ 5øq -WKâªívêöÅÂÚ8½@¤Â½Ò›ÇN×πѴŒSzf™ïóbì?NónÆ—eîI^6иé÷ðãí2þrJí±ÐÞöˆ×ºêc98sÒu©g}`V†N<¶ä0¬nº~VÎDß…Àt+Y†)¿nÈ«œ2¶ªø1Y -“¹Ëw%§¯39úrôwŒ;Ù|'^šcã[²š~铬d­wó“MÄU0€ÂJ^aôµ‘ûnè¤G2ûŸë&³<îåôçSss‘D«l.QÐïú˜£®pþ?´ù Ÿÿ>ñùßúË¥W?ý@‹8°uCž=™~3T9HÖ“â8¶ÍDŒ<ÁþJem+݆qÂk^»ûØ{Ë04 îL”U‡9ÓÕ·¼VÍÈâåF‹#ùÞµYöõ’úÍaÒ×’»T¸˜o‡ôüõàÑê’ÏÓ¸Ò¦«K—ãÔ󠸜&¬«d1Õ çI†êU~ž—‚t®éb -#ô’V¯nvºÓÚ]LMÓ|1Ìaž´nä1„þæíõÝ뾈œúPê€öA}©ãÆ”:ϦR»LÛËás¨Õ„úæ]54Æ¢iïÑrÖ­±†¼¹˜mt¢–µÛÒ@æI¹Ó6“vG1à4ßicÄÜeJK·®–vÇÅÔR1ÅJýó¬i™We(QæUIwmuÜîñF´D#’Ùœózw~m9bÉbÐÒÜ{PymIv?7oçîÞö<ݯ‚¤52õk7¹kùMø÷-{ǬÓ[­úO«Œ„²~µžŠÔ¾aÓ—”€!¬_’”¹L¥’7÷y^;¹¯ôÚÉ<–Ã1‘ÏA¿+Õáõ”¯>ì9cªªÝ­zó–Þ¦£4´ÂÜýS•öey­ÀÖ -Dã2kª¨ßIôýJ¯ìƒY*je }Qš7%”öóÿ’Qd} -endstream -endobj -895 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 894 0 R -/Resources 4 0 R -/Annots [ 896 0 R 897 0 R 898 0 R 899 0 R 900 0 R 901 0 R 902 0 R 903 0 R 904 0 R 905 0 R 906 0 R 907 0 R 908 0 R 909 0 R 910 0 R 911 0 R 912 0 R 913 0 R 914 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -896 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 766.923057 101.433071 748.173057 ] -/BS << -/W 0 ->> -/Dest (cb241-1) ->> -endobj -897 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 747.976036 99.933071 729.226036 ] -/BS << -/W 0 ->> -/Dest (cb241-2) ->> -endobj -898 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 729.029014 99.933071 710.279014 ] -/BS << -/W 0 ->> -/Dest (cb241-3) ->> -endobj -899 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 710.081993 99.933071 691.331993 ] -/BS << -/W 0 ->> -/Dest (cb241-4) ->> -endobj -900 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 691.134971 99.933071 672.384971 ] -/BS << -/W 0 ->> -/Dest (cb241-5) ->> -endobj -901 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 672.187950 99.933071 653.437950 ] -/BS << -/W 0 ->> -/Dest (cb241-6) ->> -endobj -902 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 653.240928 99.933071 634.490928 ] -/BS << -/W 0 ->> -/Dest (cb241-7) ->> -endobj -903 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.293907 99.933071 615.543907 ] -/BS << -/W 0 ->> -/Dest (cb241-8) ->> -endobj -904 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 615.346885 99.933071 596.596885 ] -/BS << -/W 0 ->> -/Dest (cb241-9) ->> -endobj -905 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 596.399864 99.933071 577.649864 ] -/BS << -/W 0 ->> -/Dest (cb241-10) ->> -endobj -906 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 577.452842 99.933071 558.702842 ] -/BS << -/W 0 ->> -/Dest (cb241-11) ->> -endobj -907 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 520.611778 99.933071 501.861778 ] -/BS << -/W 0 ->> -/Dest (cb241-12) ->> -endobj -908 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 501.664756 99.933071 482.914756 ] -/BS << -/W 0 ->> -/Dest (cb241-13) ->> -endobj -909 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 482.717735 99.933071 463.967735 ] -/BS << -/W 0 ->> -/Dest (cb241-14) ->> -endobj -910 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 463.770714 99.933071 445.020714 ] -/BS << -/W 0 ->> -/Dest (cb241-15) ->> -endobj -911 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 444.823692 99.933071 426.073692 ] -/BS << -/W 0 ->> -/Dest (cb241-16) ->> -endobj -912 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 425.876671 99.933071 407.126671 ] -/BS << -/W 0 ->> -/Dest (cb241-17) ->> -endobj -913 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 406.929649 99.933071 388.179649 ] -/BS << -/W 0 ->> -/Dest (cb241-18) ->> -endobj -914 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 345.982628 101.433071 327.232628 ] -/BS << -/W 0 ->> -/Dest (cb242-1) ->> -endobj -915 0 obj -<< -/Title (RKE CIS v1.6 Benchmark - Self-Assessment Guide - Rancher v2.6) -/Dest [ 6 0 R /XYZ 85.009843 502.818898 0 ] -/Count 141 -/First 916 0 R -/Last 1052 0 R -/Parent 1057 0 R ->> -endobj -916 0 obj -<< -/Title (RKE CIS v1.6 Kubernetes Benchmark - Rancher v2.6 with Kubernetes v1.18 to v1.23) -/Dest [ 304 0 R /XYZ 84.259843 784.970079 0 ] -/Count 2 -/First 917 0 R -/Last 918 0 R -/Parent 915 0 R -/Next 919 0 R ->> -endobj -917 0 obj -<< -/Title (Overview) -/Dest [ 304 0 R /XYZ 84.259843 704.822079 0 ] -/Count 0 -/Parent 916 0 R -/Next 918 0 R ->> -endobj -918 0 obj -<< -/Title (Testing controls methodology) -/Dest [ 304 0 R /XYZ 84.259843 166.076079 0 ] -/Count 0 -/Prev 917 0 R -/Parent 916 0 R ->> -endobj -919 0 obj -<< -/Title (Controls) -/Dest [ 311 0 R /XYZ 84.259843 612.381496 0 ] -/Count 0 -/Prev 916 0 R -/Parent 915 0 R -/Next 920 0 R ->> -endobj -920 0 obj -<< -/Title (1.1 Master Node Configuration Files) -/Dest [ 315 0 R /XYZ 41.740157 784.430079 0 ] -/Count 21 -/Prev 919 0 R -/First 921 0 R -/Last 941 0 R -/Parent 915 0 R -/Next 942 0 R ->> -endobj -921 0 obj -<< -/Title (1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 315 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 920 0 R -/Next 922 0 R ->> -endobj -922 0 obj -<< -/Title (1.1.2 Ensure that the API server pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 315 0 R /XYZ 84.259843 553.851496 0 ] -/Count 0 -/Prev 921 0 R -/Parent 920 0 R -/Next 923 0 R ->> -endobj -923 0 obj -<< -/Title (1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 315 0 R /XYZ 84.259843 423.051496 0 ] -/Count 0 -/Prev 922 0 R -/Parent 920 0 R -/Next 924 0 R ->> -endobj -924 0 obj -<< -/Title (1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 315 0 R /XYZ 84.259843 277.251496 0 ] -/Count 0 -/Prev 923 0 R -/Parent 920 0 R -/Next 925 0 R ->> -endobj -925 0 obj -<< -/Title (1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 315 0 R /XYZ 84.259843 146.451496 0 ] -/Count 0 -/Prev 924 0 R -/Parent 920 0 R -/Next 926 0 R ->> -endobj -926 0 obj -<< -/Title (1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 317 0 R /XYZ 84.259843 739.970079 0 ] -/Count 0 -/Prev 925 0 R -/Parent 920 0 R -/Next 927 0 R ->> -endobj -927 0 obj -<< -/Title (1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 317 0 R /XYZ 84.259843 609.170079 0 ] -/Count 0 -/Prev 926 0 R -/Parent 920 0 R -/Next 928 0 R ->> -endobj -928 0 obj -<< -/Title (1.1.8 Ensure that the etcd pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 317 0 R /XYZ 84.259843 478.370079 0 ] -/Count 0 -/Prev 927 0 R -/Parent 920 0 R -/Next 929 0 R ->> -endobj -929 0 obj -<< -/Title (1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive \(Manual\)) -/Dest [ 317 0 R /XYZ 84.259843 347.570079 0 ] -/Count 0 -/Prev 928 0 R -/Parent 920 0 R -/Next 930 0 R ->> -endobj -930 0 obj -<< -/Title (1.1.10 Ensure that the Container Network Interface file ownership is set to root:root \(Manual\)) -/Dest [ 317 0 R /XYZ 84.259843 170.823057 0 ] -/Count 0 -/Prev 929 0 R -/Parent 920 0 R -/Next 931 0 R ->> -endobj -931 0 obj -<< -/Title (1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive \(Automated\)) -/Dest [ 320 0 R /XYZ 84.259843 694.023057 0 ] -/Count 0 -/Prev 930 0 R -/Parent 920 0 R -/Next 932 0 R ->> -endobj -932 0 obj -<< -/Title (1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd \(Automated\)) -/Dest [ 320 0 R /XYZ 84.259843 365.776036 0 ] -/Count 0 -/Prev 931 0 R -/Parent 920 0 R -/Next 933 0 R ->> -endobj -933 0 obj -<< -/Title (1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 702.470079 0 ] -/Count 0 -/Prev 932 0 R -/Parent 920 0 R -/Next 934 0 R ->> -endobj -934 0 obj -<< -/Title (1.1.14 Ensure that the admin.conf file ownership is set to root:root \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 586.670079 0 ] -/Count 0 -/Prev 933 0 R -/Parent 920 0 R -/Next 935 0 R ->> -endobj -935 0 obj -<< -/Title (1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 470.870079 0 ] -/Count 0 -/Prev 934 0 R -/Parent 920 0 R -/Next 936 0 R ->> -endobj -936 0 obj -<< -/Title (1.1.16 Ensure that the scheduler.conf file ownership is set to root:root \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 340.070079 0 ] -/Count 0 -/Prev 935 0 R -/Parent 920 0 R -/Next 937 0 R ->> -endobj -937 0 obj -<< -/Title (1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 209.270079 0 ] -/Count 0 -/Prev 936 0 R -/Parent 920 0 R -/Next 938 0 R ->> -endobj -938 0 obj -<< -/Title (1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root \(Automated\)) -/Dest [ 327 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 937 0 R -/Parent 920 0 R -/Next 939 0 R ->> -endobj -939 0 obj -<< -/Title (1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root \(Automated\)) -/Dest [ 327 0 R /XYZ 84.259843 668.570079 0 ] -/Count 0 -/Prev 938 0 R -/Parent 920 0 R -/Next 940 0 R ->> -endobj -940 0 obj -<< -/Title (1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 351 0 R /XYZ 84.259843 184.188585 0 ] -/Count 0 -/Prev 939 0 R -/Parent 920 0 R -/Next 941 0 R ->> -endobj -941 0 obj -<< -/Title (1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 \(Automated\)) -/Dest [ 442 0 R /XYZ 84.259843 702.470079 0 ] -/Count 0 -/Prev 940 0 R -/Parent 920 0 R ->> -endobj -942 0 obj -<< -/Title (1.2 API Server) -/Dest [ 510 0 R /XYZ 41.740157 784.430079 0 ] -/Count 35 -/Prev 920 0 R -/First 943 0 R -/Last 977 0 R -/Parent 915 0 R -/Next 978 0 R ->> -endobj -943 0 obj -<< -/Title (1.2.1 Ensure that the --anonymous-auth argument is set to false \(Automated\)) -/Dest [ 510 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 942 0 R -/Next 944 0 R ->> -endobj -944 0 obj -<< -/Title (1.2.2 Ensure that the --basic-auth-file argument is not set \(Automated\)) -/Dest [ 513 0 R /XYZ 84.259843 295.970079 0 ] -/Count 0 -/Prev 943 0 R -/Parent 942 0 R -/Next 945 0 R ->> -endobj -945 0 obj -<< -/Title (1.2.3 Ensure that the --token-auth-file parameter is not set \(Automated\)) -/Dest [ 518 0 R /XYZ 84.259843 595.970079 0 ] -/Count 0 -/Prev 944 0 R -/Parent 942 0 R -/Next 946 0 R ->> -endobj -946 0 obj -<< -/Title (1.2.4 Ensure that the --kubelet-https argument is set to true \(Automated\)) -/Dest [ 521 0 R /XYZ 84.259843 164.720079 0 ] -/Count 0 -/Prev 945 0 R -/Parent 942 0 R -/Next 947 0 R ->> -endobj -947 0 obj -<< -/Title (1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate \(Automated\)) -/Dest [ 526 0 R /XYZ 84.259843 427.220079 0 ] -/Count 0 -/Prev 946 0 R -/Parent 942 0 R -/Next 948 0 R ->> -endobj -948 0 obj -<< -/Title (1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate \(Automated\)) -/Dest [ 531 0 R /XYZ 84.259843 652.220079 0 ] -/Count 0 -/Prev 947 0 R -/Parent 942 0 R -/Next 949 0 R ->> -endobj -949 0 obj -<< -/Title (1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow \(Automated\)) -/Dest [ 534 0 R /XYZ 84.259843 183.470079 0 ] -/Count 0 -/Prev 948 0 R -/Parent 942 0 R -/Next 950 0 R ->> -endobj -950 0 obj -<< -/Title (1.2.8 Ensure that the --authorization-mode argument includes Node \(Automated\)) -/Dest [ 539 0 R /XYZ 84.259843 464.720079 0 ] -/Count 0 -/Prev 949 0 R -/Parent 942 0 R -/Next 951 0 R ->> -endobj -951 0 obj -<< -/Title (1.2.9 Ensure that the --authorization-mode argument includes RBAC \(Automated\)) -/Dest [ 544 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 950 0 R -/Parent 942 0 R -/Next 952 0 R ->> -endobj -952 0 obj -<< -/Title (1.2.10 Ensure that the admission control plugin EventRateLimit is set \(Automated\)) -/Dest [ 547 0 R /XYZ 84.259843 314.720079 0 ] -/Count 0 -/Prev 951 0 R -/Parent 942 0 R -/Next 953 0 R ->> -endobj -953 0 obj -<< -/Title (1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set \(Automated\)) -/Dest [ 552 0 R /XYZ 84.259843 520.970079 0 ] -/Count 0 -/Prev 952 0 R -/Parent 942 0 R -/Next 954 0 R ->> -endobj -954 0 obj -<< -/Title (1.2.12 Ensure that the admission control plugin AlwaysPullImages is set \(Manual\)) -/Dest [ 557 0 R /XYZ 84.259843 708.470079 0 ] -/Count 0 -/Prev 953 0 R -/Parent 942 0 R -/Next 955 0 R ->> -endobj -955 0 obj -<< -/Title (1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used \(Manual\)) -/Dest [ 557 0 R /XYZ 84.259843 501.723057 0 ] -/Count 0 -/Prev 954 0 R -/Parent 942 0 R -/Next 956 0 R ->> -endobj -956 0 obj -<< -/Title (1.2.14 Ensure that the admission control plugin ServiceAccount is set \(Automated\)) -/Dest [ 557 0 R /XYZ 84.259843 264.976036 0 ] -/Count 0 -/Prev 955 0 R -/Parent 942 0 R -/Next 957 0 R ->> -endobj -957 0 obj -<< -/Title (1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set \(Automated\)) -/Dest [ 564 0 R /XYZ 84.259843 520.970079 0 ] -/Count 0 -/Prev 956 0 R -/Parent 942 0 R -/Next 958 0 R ->> -endobj -958 0 obj -<< -/Title (1.2.16 Ensure that the admission control plugin PodSecurityPolicy is set \(Automated\)) -/Dest [ 569 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 957 0 R -/Parent 942 0 R -/Next 959 0 R ->> -endobj -959 0 obj -<< -/Title (1.2.17 Ensure that the admission control plugin NodeRestriction is set \(Automated\)) -/Dest [ 572 0 R /XYZ 84.259843 258.470079 0 ] -/Count 0 -/Prev 958 0 R -/Parent 942 0 R -/Next 960 0 R ->> -endobj -960 0 obj -<< -/Title (1.2.18 Ensure that the --insecure-bind-address argument is not set \(Automated\)) -/Dest [ 577 0 R /XYZ 84.259843 427.220079 0 ] -/Count 0 -/Prev 959 0 R -/Parent 942 0 R -/Next 961 0 R ->> -endobj -961 0 obj -<< -/Title (1.2.19 Ensure that the --insecure-port argument is set to 0 \(Automated\)) -/Dest [ 582 0 R /XYZ 84.259843 727.220079 0 ] -/Count 0 -/Prev 960 0 R -/Parent 942 0 R -/Next 962 0 R ->> -endobj -962 0 obj -<< -/Title (1.2.20 Ensure that the --secure-port argument is not set to 0 \(Automated\)) -/Dest [ 585 0 R /XYZ 84.259843 314.720079 0 ] -/Count 0 -/Prev 961 0 R -/Parent 942 0 R -/Next 963 0 R ->> -endobj -963 0 obj -<< -/Title (1.2.21 Ensure that the --profiling argument is set to false \(Automated\)) -/Dest [ 590 0 R /XYZ 84.259843 595.970079 0 ] -/Count 0 -/Prev 962 0 R -/Parent 942 0 R -/Next 964 0 R ->> -endobj -964 0 obj -<< -/Title (1.2.22 Ensure that the --audit-log-path argument is set \(Automated\)) -/Dest [ 593 0 R /XYZ 84.259843 183.470079 0 ] -/Count 0 -/Prev 963 0 R -/Parent 942 0 R -/Next 965 0 R ->> -endobj -965 0 obj -<< -/Title (1.2.23 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate \(Automated\)) -/Dest [ 598 0 R /XYZ 84.259843 445.970079 0 ] -/Count 0 -/Prev 964 0 R -/Parent 942 0 R -/Next 966 0 R ->> -endobj -966 0 obj -<< -/Title (1.2.24 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate \(Automated\)) -/Dest [ 603 0 R /XYZ 84.259843 727.220079 0 ] -/Count 0 -/Prev 965 0 R -/Parent 942 0 R -/Next 967 0 R ->> -endobj -967 0 obj -<< -/Title (1.2.25 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate \(Automated\)) -/Dest [ 606 0 R /XYZ 84.259843 295.970079 0 ] -/Count 0 -/Prev 966 0 R -/Parent 942 0 R -/Next 968 0 R ->> -endobj -968 0 obj -<< -/Title (1.2.26 Ensure that the --request-timeout argument is set as appropriate \(Automated\)) -/Dest [ 611 0 R /XYZ 84.259843 595.970079 0 ] -/Count 0 -/Prev 967 0 R -/Parent 942 0 R -/Next 969 0 R ->> -endobj -969 0 obj -<< -/Title (1.2.27 Ensure that the --service-account-lookup argument is set to true \(Automated\)) -/Dest [ 614 0 R /XYZ 84.259843 164.720079 0 ] -/Count 0 -/Prev 968 0 R -/Parent 942 0 R -/Next 970 0 R ->> -endobj -970 0 obj -<< -/Title (1.2.28 Ensure that the --service-account-key-file argument is set as appropriate \(Automated\)) -/Dest [ 619 0 R /XYZ 84.259843 408.470079 0 ] -/Count 0 -/Prev 969 0 R -/Parent 942 0 R -/Next 971 0 R ->> -endobj -971 0 obj -<< -/Title (1.2.29 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate \(Automated\)) -/Dest [ 624 0 R /XYZ 84.259843 689.720079 0 ] -/Count 0 -/Prev 970 0 R -/Parent 942 0 R -/Next 972 0 R ->> -endobj -972 0 obj -<< -/Title (1.2.30 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate \(Automated\)) -/Dest [ 627 0 R /XYZ 84.259843 239.720079 0 ] -/Count 0 -/Prev 971 0 R -/Parent 942 0 R -/Next 973 0 R ->> -endobj -973 0 obj -<< -/Title (1.2.31 Ensure that the --client-ca-file argument is set as appropriate \(Automated\)) -/Dest [ 632 0 R /XYZ 84.259843 483.470079 0 ] -/Count 0 -/Prev 972 0 R -/Parent 942 0 R -/Next 974 0 R ->> -endobj -974 0 obj -<< -/Title (1.2.32 Ensure that the --etcd-cafile argument is set as appropriate \(Automated\)) -/Dest [ 637 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 973 0 R -/Parent 942 0 R -/Next 975 0 R ->> -endobj -975 0 obj -<< -/Title (1.2.33 Ensure that the --encryption-provider-config argument is set as appropriate \(Automated\)) -/Dest [ 640 0 R /XYZ 84.259843 295.970079 0 ] -/Count 0 -/Prev 974 0 R -/Parent 942 0 R -/Next 976 0 R ->> -endobj -976 0 obj -<< -/Title (1.2.34 Ensure that encryption providers are appropriately configured \(Automated\)) -/Dest [ 645 0 R /XYZ 84.259843 577.220079 0 ] -/Count 0 -/Prev 975 0 R -/Parent 942 0 R -/Next 977 0 R ->> -endobj -977 0 obj -<< -/Title (1.2.35 Ensure that the API Server only makes use of Strong Cryptographic Ciphers \(Automated\)) -/Dest [ 662 0 R /XYZ 84.259843 411.552842 0 ] -/Count 0 -/Prev 976 0 R -/Parent 942 0 R ->> -endobj -978 0 obj -<< -/Title (1.3 Controller Manager) -/Dest [ 676 0 R /XYZ 41.740157 784.430079 0 ] -/Count 7 -/Prev 942 0 R -/First 979 0 R -/Last 985 0 R -/Parent 915 0 R -/Next 986 0 R ->> -endobj -979 0 obj -<< -/Title (1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate \(Automated\)) -/Dest [ 676 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 978 0 R -/Next 980 0 R ->> -endobj -980 0 obj -<< -/Title (1.3.2 Ensure that the --profiling argument is set to false \(Automated\)) -/Dest [ 679 0 R /XYZ 84.259843 727.220079 0 ] -/Count 0 -/Prev 979 0 R -/Parent 978 0 R -/Next 981 0 R ->> -endobj -981 0 obj -<< -/Title (1.3.3 Ensure that the --use-service-account-credentials argument is set to true \(Automated\)) -/Dest [ 682 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 980 0 R -/Parent 978 0 R -/Next 982 0 R ->> -endobj -982 0 obj -<< -/Title (1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate \(Automated\)) -/Dest [ 685 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 981 0 R -/Parent 978 0 R -/Next 983 0 R ->> -endobj -983 0 obj -<< -/Title (1.3.5 Ensure that the --root-ca-file argument is set as appropriate \(Automated\)) -/Dest [ 688 0 R /XYZ 84.259843 727.220079 0 ] -/Count 0 -/Prev 982 0 R -/Parent 978 0 R -/Next 984 0 R ->> -endobj -984 0 obj -<< -/Title (1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true \(Automated\)) -/Dest [ 691 0 R /XYZ 84.259843 727.220079 0 ] -/Count 0 -/Prev 983 0 R -/Parent 978 0 R -/Next 985 0 R ->> -endobj -985 0 obj -<< -/Title (1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 \(Automated\)) -/Dest [ 691 0 R /XYZ 84.259843 524.420079 0 ] -/Count 0 -/Prev 984 0 R -/Parent 978 0 R ->> -endobj -986 0 obj -<< -/Title (1.4 Scheduler) -/Dest [ 696 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 978 0 R -/First 987 0 R -/Last 988 0 R -/Parent 915 0 R -/Next 989 0 R ->> -endobj -987 0 obj -<< -/Title (1.4.1 Ensure that the --profiling argument is set to false \(Automated\)) -/Dest [ 696 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 986 0 R -/Next 988 0 R ->> -endobj -988 0 obj -<< -/Title (1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 \(Automated\)) -/Dest [ 696 0 R /XYZ 84.259843 257.404475 0 ] -/Count 0 -/Prev 987 0 R -/Parent 986 0 R ->> -endobj -989 0 obj -<< -/Title (2 Etcd Node Configuration Files) -/Dest [ 702 0 R /XYZ 41.740157 784.430079 0 ] -/Count 7 -/Prev 986 0 R -/First 990 0 R -/Last 996 0 R -/Parent 915 0 R -/Next 997 0 R ->> -endobj -990 0 obj -<< -/Title (2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate \(Automated\)) -/Dest [ 702 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 989 0 R -/Next 991 0 R ->> -endobj -991 0 obj -<< -/Title (2.2 Ensure that the --client-cert-auth argument is set to true \(Automated\)) -/Dest [ 707 0 R /XYZ 84.259843 464.720079 0 ] -/Count 0 -/Prev 990 0 R -/Parent 989 0 R -/Next 992 0 R ->> -endobj -992 0 obj -<< -/Title (2.3 Ensure that the --auto-tls argument is not set to true \(Automated\)) -/Dest [ 712 0 R /XYZ 84.259843 258.470079 0 ] -/Count 0 -/Prev 991 0 R -/Parent 989 0 R -/Next 993 0 R ->> -endobj -993 0 obj -<< -/Title (2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate \(Automated\)) -/Dest [ 719 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 992 0 R -/Parent 989 0 R -/Next 994 0 R ->> -endobj -994 0 obj -<< -/Title (2.5 Ensure that the --peer-client-cert-auth argument is set to true \(Automated\)) -/Dest [ 724 0 R /XYZ 84.259843 558.470079 0 ] -/Count 0 -/Prev 993 0 R -/Parent 989 0 R -/Next 995 0 R ->> -endobj -995 0 obj -<< -/Title (2.6 Ensure that the --peer-auto-tls argument is not set to true \(Automated\)) -/Dest [ 729 0 R /XYZ 84.259843 333.470079 0 ] -/Count 0 -/Prev 994 0 R -/Parent 989 0 R -/Next 996 0 R ->> -endobj -996 0 obj -<< -/Title (2.7 Ensure that a unique Certificate Authority is used for etcd \(Automated\)) -/Dest [ 736 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 995 0 R -/Parent 989 0 R ->> -endobj -997 0 obj -<< -/Title (3.1 Authentication and Authorization) -/Dest [ 743 0 R /XYZ 41.740157 784.430079 0 ] -/Count 1 -/Prev 989 0 R -/First 998 0 R -/Last 998 0 R -/Parent 915 0 R -/Next 999 0 R ->> -endobj -998 0 obj -<< -/Title (3.1.1 Client certificate authentication should not be used for users \(Manual\)) -/Dest [ 743 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 997 0 R ->> -endobj -999 0 obj -<< -/Title (3.2 Logging) -/Dest [ 745 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 997 0 R -/First 1000 0 R -/Last 1001 0 R -/Parent 915 0 R -/Next 1002 0 R ->> -endobj -1000 0 obj -<< -/Title (3.2.1 Ensure that a minimal audit policy is created \(Automated\)) -/Dest [ 745 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 999 0 R -/Next 1001 0 R ->> -endobj -1001 0 obj -<< -/Title (3.2.2 Ensure that the audit policy covers key security concerns \(Manual\)) -/Dest [ 748 0 R /XYZ 84.259843 333.470079 0 ] -/Count 0 -/Prev 1000 0 R -/Parent 999 0 R ->> -endobj -1002 0 obj -<< -/Title (4.1 Worker Node Configuration Files) -/Dest [ 750 0 R /XYZ 41.740157 784.430079 0 ] -/Count 10 -/Prev 999 0 R -/First 1003 0 R -/Last 1012 0 R -/Parent 915 0 R -/Next 1013 0 R ->> -endobj -1003 0 obj -<< -/Title (4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 750 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 1002 0 R -/Next 1004 0 R ->> -endobj -1004 0 obj -<< -/Title (4.1.2 Ensure that the kubelet service file ownership is set to root:root \(Automated\)) -/Dest [ 750 0 R /XYZ 84.259843 553.851496 0 ] -/Count 0 -/Prev 1003 0 R -/Parent 1002 0 R -/Next 1005 0 R ->> -endobj -1005 0 obj -<< -/Title (4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 750 0 R /XYZ 84.259843 423.051496 0 ] -/Count 0 -/Prev 1004 0 R -/Parent 1002 0 R -/Next 1006 0 R ->> -endobj -1006 0 obj -<< -/Title (4.1.4 Ensure that the proxy kubeconfig file ownership is set to root:root \(Automated\)) -/Dest [ 753 0 R /XYZ 84.259843 763.220079 0 ] -/Count 0 -/Prev 1005 0 R -/Parent 1002 0 R -/Next 1007 0 R ->> -endobj -1007 0 obj -<< -/Title (4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 753 0 R /XYZ 84.259843 454.079014 0 ] -/Count 0 -/Prev 1006 0 R -/Parent 1002 0 R -/Next 1008 0 R ->> -endobj -1008 0 obj -<< -/Title (4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root \(Automated\)) -/Dest [ 753 0 R /XYZ 84.259843 163.687950 0 ] -/Count 0 -/Prev 1007 0 R -/Parent 1002 0 R -/Next 1009 0 R ->> -endobj -1009 0 obj -<< -/Title (4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 757 0 R /XYZ 84.259843 519.629014 0 ] -/Count 0 -/Prev 1008 0 R -/Parent 1002 0 R -/Next 1010 0 R ->> -endobj -1010 0 obj -<< -/Title (4.1.8 Ensure that the client certificate authorities file ownership is set to root:root \(Automated\)) -/Dest [ 766 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 1009 0 R -/Parent 1002 0 R -/Next 1011 0 R ->> -endobj -1011 0 obj -<< -/Title (4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive \(Automated\)) -/Dest [ 766 0 R /XYZ 84.259843 383.493907 0 ] -/Count 0 -/Prev 1010 0 R -/Parent 1002 0 R -/Next 1012 0 R ->> -endobj -1012 0 obj -<< -/Title (4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root \(Automated\)) -/Dest [ 766 0 R /XYZ 84.259843 195.693907 0 ] -/Count 0 -/Prev 1011 0 R -/Parent 1002 0 R ->> -endobj -1013 0 obj -<< -/Title (4.2 Kubelet) -/Dest [ 776 0 R /XYZ 41.740157 784.430079 0 ] -/Count 13 -/Prev 1002 0 R -/First 1014 0 R -/Last 1026 0 R -/Parent 915 0 R -/Next 1027 0 R ->> -endobj -1014 0 obj -<< -/Title (4.2.1 Ensure that the anonymous-auth argument is set to false \(Automated\)) -/Dest [ 776 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 1013 0 R -/Next 1015 0 R ->> -endobj -1015 0 obj -<< -/Title (4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow \(Automated\)) -/Dest [ 776 0 R /XYZ 84.259843 332.207453 0 ] -/Count 0 -/Prev 1014 0 R -/Parent 1013 0 R -/Next 1016 0 R ->> -endobj -1016 0 obj -<< -/Title (4.2.3 Ensure that the --client-ca-file argument is set as appropriate \(Automated\)) -/Dest [ 781 0 R /XYZ 84.259843 675.273057 0 ] -/Count 0 -/Prev 1015 0 R -/Parent 1013 0 R -/Next 1017 0 R ->> -endobj -1017 0 obj -<< -/Title (4.2.4 Ensure that the --read-only-port argument is set to 0 \(Automated\)) -/Dest [ 781 0 R /XYZ 84.259843 301.829014 0 ] -/Count 0 -/Prev 1016 0 R -/Parent 1013 0 R -/Next 1018 0 R ->> -endobj -1018 0 obj -<< -/Title (4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 \(Automated\)) -/Dest [ 786 0 R /XYZ 84.259843 641.326036 0 ] -/Count 0 -/Prev 1017 0 R -/Parent 1013 0 R -/Next 1019 0 R ->> -endobj -1019 0 obj -<< -/Title (4.2.6 Ensure that the --protect-kernel-defaults argument is set to true \(Automated\)) -/Dest [ 792 0 R /XYZ 84.259843 445.970079 0 ] -/Count 0 -/Prev 1018 0 R -/Parent 1013 0 R -/Next 1020 0 R ->> -endobj -1020 0 obj -<< -/Title (4.2.7 Ensure that the --make-iptables-util-chains argument is set to true \(Automated\)) -/Dest [ 796 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 1019 0 R -/Parent 1013 0 R -/Next 1021 0 R ->> -endobj -1021 0 obj -<< -/Title (4.2.8 Ensure that the --hostname-override argument is not set \(Manual\)) -/Dest [ 796 0 R /XYZ 84.259843 425.926036 0 ] -/Count 0 -/Prev 1020 0 R -/Parent 1013 0 R -/Next 1022 0 R ->> -endobj -1022 0 obj -<< -/Title (4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture \(Automated\)) -/Dest [ 796 0 R /XYZ 84.259843 208.126036 0 ] -/Count 0 -/Prev 1021 0 R -/Parent 1013 0 R -/Next 1023 0 R ->> -endobj -1023 0 obj -<< -/Title (4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate \(Automated\)) -/Dest [ 800 0 R /XYZ 84.259843 542.326036 0 ] -/Count 0 -/Prev 1022 0 R -/Parent 1013 0 R -/Next 1024 0 R ->> -endobj -1024 0 obj -<< -/Title (4.2.11 Ensure that the --rotate-certificates argument is not set to false \(Automated\)) -/Dest [ 800 0 R /XYZ 84.259843 138.881993 0 ] -/Count 0 -/Prev 1023 0 R -/Parent 1013 0 R -/Next 1025 0 R ->> -endobj -1025 0 obj -<< -/Title (4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true \(Automated\)) -/Dest [ 810 0 R /XYZ 84.259843 670.970079 0 ] -/Count 0 -/Prev 1024 0 R -/Parent 1013 0 R -/Next 1026 0 R ->> -endobj -1026 0 obj -<< -/Title (4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers \(Automated\)) -/Dest [ 810 0 R /XYZ 84.259843 392.223057 0 ] -/Count 0 -/Prev 1025 0 R -/Parent 1013 0 R ->> -endobj -1027 0 obj -<< -/Title (5.1 RBAC and Service Accounts) -/Dest [ 817 0 R /XYZ 41.740157 784.430079 0 ] -/Count 6 -/Prev 1013 0 R -/First 1028 0 R -/Last 1033 0 R -/Parent 915 0 R -/Next 1034 0 R ->> -endobj -1028 0 obj -<< -/Title (5.1.1 Ensure that the cluster-admin role is only used where required \(Manual\)) -/Dest [ 817 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 1027 0 R -/Next 1029 0 R ->> -endobj -1029 0 obj -<< -/Title (5.1.2 Minimize access to secrets \(Manual\)) -/Dest [ 817 0 R /XYZ 84.259843 523.851496 0 ] -/Count 0 -/Prev 1028 0 R -/Parent 1027 0 R -/Next 1030 0 R ->> -endobj -1030 0 obj -<< -/Title (5.1.3 Minimize wildcard use in Roles and ClusterRoles \(Manual\)) -/Dest [ 817 0 R /XYZ 84.259843 423.051496 0 ] -/Count 0 -/Prev 1029 0 R -/Parent 1027 0 R -/Next 1031 0 R ->> -endobj -1031 0 obj -<< -/Title (5.1.4 Minimize access to create pods \(Manual\)) -/Dest [ 817 0 R /XYZ 84.259843 307.251496 0 ] -/Count 0 -/Prev 1030 0 R -/Parent 1027 0 R -/Next 1032 0 R ->> -endobj -1032 0 obj -<< -/Title (5.1.5 Ensure that default service accounts are not actively used. \(Automated\)) -/Dest [ 817 0 R /XYZ 84.259843 206.451496 0 ] -/Count 0 -/Prev 1031 0 R -/Parent 1027 0 R -/Next 1033 0 R ->> -endobj -1033 0 obj -<< -/Title (5.1.6 Ensure that Service Account Tokens are only mounted where necessary \(Manual\)) -/Dest [ 844 0 R /XYZ 84.259843 449.446885 0 ] -/Count 0 -/Prev 1032 0 R -/Parent 1027 0 R ->> -endobj -1034 0 obj -<< -/Title (5.2 Pod Security Policies) -/Dest [ 855 0 R /XYZ 41.740157 784.430079 0 ] -/Count 9 -/Prev 1027 0 R -/First 1035 0 R -/Last 1043 0 R -/Parent 915 0 R -/Next 1044 0 R ->> -endobj -1035 0 obj -<< -/Title (5.2.1 Minimize the admission of privileged containers \(Manual\)) -/Dest [ 855 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 1034 0 R -/Next 1036 0 R ->> -endobj -1036 0 obj -<< -/Title (5.2.2 Minimize the admission of containers wishing to share the host process ID namespace \(Automated\)) -/Dest [ 855 0 R /XYZ 84.259843 589.851496 0 ] -/Count 0 -/Prev 1035 0 R -/Parent 1034 0 R -/Next 1037 0 R ->> -endobj -1037 0 obj -<< -/Title (5.2.3 Minimize the admission of containers wishing to share the host IPC namespace \(Automated\)) -/Dest [ 855 0 R /XYZ 84.259843 238.710432 0 ] -/Count 0 -/Prev 1036 0 R -/Parent 1034 0 R -/Next 1038 0 R ->> -endobj -1038 0 obj -<< -/Title (5.2.4 Minimize the admission of containers wishing to share the host network namespace \(Automated\)) -/Dest [ 858 0 R /XYZ 84.259843 603.629014 0 ] -/Count 0 -/Prev 1037 0 R -/Parent 1034 0 R -/Next 1039 0 R ->> -endobj -1039 0 obj -<< -/Title (5.2.5 Minimize the admission of containers with allowPrivilegeEscalation \(Automated\)) -/Dest [ 858 0 R /XYZ 84.259843 252.487950 0 ] -/Count 0 -/Prev 1038 0 R -/Parent 1034 0 R -/Next 1040 0 R ->> -endobj -1040 0 obj -<< -/Title (5.2.6 Minimize the admission of root containers \(Manual\)) -/Dest [ 862 0 R /XYZ 84.259843 584.681993 0 ] -/Count 0 -/Prev 1039 0 R -/Parent 1034 0 R -/Next 1041 0 R ->> -endobj -1041 0 obj -<< -/Title (5.2.7 Minimize the admission of containers with the NET_RAW capability \(Manual\)) -/Dest [ 862 0 R /XYZ 84.259843 453.881993 0 ] -/Count 0 -/Prev 1040 0 R -/Parent 1034 0 R -/Next 1042 0 R ->> -endobj -1042 0 obj -<< -/Title (5.2.8 Minimize the admission of containers with added capabilities \(Manual\)) -/Dest [ 862 0 R /XYZ 84.259843 323.081993 0 ] -/Count 0 -/Prev 1041 0 R -/Parent 1034 0 R -/Next 1043 0 R ->> -endobj -1043 0 obj -<< -/Title (5.2.9 Minimize the admission of containers with capabilities assigned \(Manual\)) -/Dest [ 862 0 R /XYZ 84.259843 207.281993 0 ] -/Count 0 -/Prev 1042 0 R -/Parent 1034 0 R ->> -endobj -1044 0 obj -<< -/Title (5.3 Network Policies and CNI) -/Dest [ 865 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 1034 0 R -/First 1045 0 R -/Last 1046 0 R -/Parent 915 0 R -/Next 1047 0 R ->> -endobj -1045 0 obj -<< -/Title (5.3.1 Ensure that the CNI in use supports Network Policies \(Manual\)) -/Dest [ 865 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 1044 0 R -/Next 1046 0 R ->> -endobj -1046 0 obj -<< -/Title (5.3.2 Ensure that all Namespaces have Network Policies defined \(Automated\)) -/Dest [ 865 0 R /XYZ 84.259843 538.851496 0 ] -/Count 0 -/Prev 1045 0 R -/Parent 1044 0 R ->> -endobj -1047 0 obj -<< -/Title (5.4 Secrets Management) -/Dest [ 889 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 1044 0 R -/First 1048 0 R -/Last 1049 0 R -/Parent 915 0 R -/Next 1050 0 R ->> -endobj -1048 0 obj -<< -/Title (5.4.1 Prefer using secrets as files over secrets as environment variables \(Manual\)) -/Dest [ 889 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 1047 0 R -/Next 1049 0 R ->> -endobj -1049 0 obj -<< -/Title (5.4.2 Consider external secret storage \(Manual\)) -/Dest [ 889 0 R /XYZ 84.259843 568.851496 0 ] -/Count 0 -/Prev 1048 0 R -/Parent 1047 0 R ->> -endobj -1050 0 obj -<< -/Title (5.5 Extensible Admission Control) -/Dest [ 891 0 R /XYZ 41.740157 784.430079 0 ] -/Count 1 -/Prev 1047 0 R -/First 1051 0 R -/Last 1051 0 R -/Parent 915 0 R -/Next 1052 0 R ->> -endobj -1051 0 obj -<< -/Title (5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller \(Manual\)) -/Dest [ 891 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 1050 0 R ->> -endobj -1052 0 obj -<< -/Title (5.7 General Policies) -/Dest [ 893 0 R /XYZ 41.740157 784.430079 0 ] -/Count 4 -/Prev 1050 0 R -/First 1053 0 R -/Last 1056 0 R -/Parent 915 0 R ->> -endobj -1053 0 obj -<< -/Title (5.7.1 Create administrative boundaries between resources using namespaces \(Manual\)) -/Dest [ 893 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 1052 0 R -/Next 1054 0 R ->> -endobj -1054 0 obj -<< -/Title (5.7.2 Ensure that the seccomp profile is set to docker/default in your pod definitions \(Manual\)) -/Dest [ 893 0 R /XYZ 84.259843 604.851496 0 ] -/Count 0 -/Prev 1053 0 R -/Parent 1052 0 R -/Next 1055 0 R ->> -endobj -1055 0 obj -<< -/Title (5.7.3 Apply Security Context to Your Pods and Containers \(Manual\)) -/Dest [ 893 0 R /XYZ 84.259843 324.051496 0 ] -/Count 0 -/Prev 1054 0 R -/Parent 1052 0 R -/Next 1056 0 R ->> -endobj -1056 0 obj -<< -/Title (5.7.4 The default namespace should not be used \(Automated\)) -/Dest [ 893 0 R /XYZ 84.259843 193.251496 0 ] -/Count 0 -/Prev 1055 0 R -/Parent 1052 0 R ->> -endobj -1057 0 obj -<< -/Count 142 -/First 915 0 R -/Last 915 0 R ->> -endobj -1058 0 obj -<< -/Length1 41480 -/Filter /FlateDecode -/Length 7851 ->> -stream -xœí| xU¶ð¹÷Ô­®^Òéîtg_:{–˜˜ ¥AˆlbpM „ Ba#„ÁIP""b@ ˆ@DQDdgÜž 8èL@œ‡ŽÆ¤òNU'!8ã›™÷ýßÿþÿûêœô­{oÝåœsÏVõ})`D…é¹¹o®™¾{ Àc©7ò¦#sƒ®¦v)µKoÊ»eBÒ'×´RûšsÓ„IÃϽûR*@}$Ý¿|󰉣ÌC3 ð¤ƒÚßÞ2!-cÎwÏ/`MÔ¾}ÆÜ‚R|Y¢õò¾ öö‹zavÔ €ŸÕÓµ¨tÖÜù×.¾àÖut׬‚²R0Âc£¨m›uÏÒ¢ßì¸Ñð­:¢xfAaÌ_ÃGÓܺŸ]Lö'Âʨ½Ú ÅsÞ»[r]à4~ü»÷”Ì(XÔQJûO¡ý’çÍ-¸·4ä<ÿ˜Æo¤ñÞysg&_¸þÀ&âŸ.-)[ØùL¥ýS´û¥ f–1}MÕۮŠɎßÙÔ÷kuÑ]9ß@Œ¼aK¥výmó¬)JvSS~ y¦¹j•'èþS”¾R/zQë :õØD+€äáÁþ5$;«AtdŠMÔŒö_ñPÄ]4Á*#*ç’¿+W4²|à…Ų[u³Í¦¹ìl>°Î;µ»ÒI(êÉ׿ ü,Ò¼ºÛ">¥_]ï1Ô®þW×KÁ!–ÁqQezÛǵ«œéoóŸûÛb¨ÿúר¦ùn¨ü÷8ñƒtÊõķ5æK­0ŸÿÒôu]pˆ‚WõûÛüu:ÑCZ¿ô9Ìï^ÇP»/”`< ¤{{¤¸þB‹`€`€`€`€`€`€`€`€`€`€`€`€`€`€`€`€`€`€`Àÿk VBš°ø¦Hó­Mÿúžö5=7•L¿JPCW/¡¤_@dÃ0n‚Ñ0&@Ál(Åúˆþp Ö3â61Ša,èìì<ûßá¿ñ÷÷pÐ$ 9ƒ~^G#O]Øè»u²÷Í)±ýûý¨éu˜¼×°ÔÛÔÙ™7YŠSEd#&*Rbü§?uóÓþýÆæMö6vŒѵêÈüÔ7a2UµuSÿÈú=mÓF‘H£ó½3Š½:Œü cæàþÐÄ—û:P±Íß'âwø·:ü֎ߨxYÅÿLÄ¿Úñë:¼”ˆ_=8L|¥âÅ:¼P‡­mø—6ü³Š_Æ/†ãyÿ”ŸŸ› >¯Ãs4ðÜ<ûYš8Û†Ÿ¥á§*~¢âÇønü¨?Tñ¾¿ ß;ŒTñ]þî2gÇÝ»ÅîBÜÕà»±Á…ÏÑ϶áNŸQq‡ŠO»p»ŠOm³‹§2p›Ÿ,ÄzR_‡[UÜò„MlQñ >þX˜x¼Ûì…áfn²àF7Ôˆ *Ö`-Mª­ÃG±‹GSð;>܆ëk‹õ*ÖTO5‡±f¹Tý«DQ= «}Ò¯ñ!×­ Ö©¸v>Hl>8 «ÖXE•×X±’:* q5Iju"®râ/U\¹Â)Vª¸Â‰¨¸\Å -}¿X¶LüBÅeËðþB,Ÿèå‰xŸŠKU¼×ŽKl¸Ø‚‹T\؆em¸  ç·a©Š%*ÎSñžXœ£âÝÎáâî 8[Åâe8‹E*ÎT±PÅ*NW±`0æ·á6œ¦â*NUqÊd‹˜Ò†“-ø³à0ñ³ ¼]ÅI´ó¤á8у˜CLÅÛÜxë˜ q«ŠyV¼EÅñ7;Äxovà8ÇÒ±*Žíc‚ptT€íÀQx“Š¹u8²G¨x#ï/nlÃá‡qØXô©8TÅ®w‰Üx}N ¸Þ…9CDŽ¯3‡à`©xÝ@·¸® f;Ä@7fgYE¶³¬xm4f`Æ5V‘¡â5VLO³ŠôL³â€þf1ÀýÍØ/ûöI} ±OªKôIÄT¦$'Š”a˜œˆI‰V‘ˆ‰VLP1^Ÿ@Œ%>c]è-Ę6Œ&¢ 1*#I‚‘*F´aøp £F˜Š¡…B’ -Q1˜&‡¡GE·ŠA*ºh€KE'ñꎎeXˆvlÁ"@E¶£UE‹Í**4LQÑäF¹%º)‘xzQ%¯å¼?2‚Š¬‰®zˆõýÿà›€ÿ¢X4Á ÂW¡g;¨¥}Ÿu>õÔó}° -QÏkì«âý©o\‚S4²N`ƒl dR/À{‚Ãe6öÓƒ˜› 2ÉHã¥ýÒmR“t^: ¥2餔/•±LÜ&n;è7_ç.81ÐÄ>†28„_b&¶H#$;|Œ'±>§]´Øyªa;”-nV¼œßF=ÇÄIØLXB÷O²-ìQwˆ­„3°%> -¶°3Ä× øVâD^Aá9“ýÇh­“43”I Î0 ¨¼/õõ´×t½ŒÂþ⌎— ‚vžÛå&ÙmŠ§]4‰í`¯±Vù¨‡Søsœ°UR¼´SÕ~ `>TÓÚ›µ9r[J¼kX®­Î—Hù¬¾”òMÓií×5ŽhÏýü6¡}·…~Kdñ4„­Â*¢T»'Mc¤4šO+˜–×%˜wS­öÀ>èuPM+éüÊÅ·4óqéS⹚=Ä¿…“8R¡Hº¨}ÛÖ W5÷'!úèXŸø‘ý‡ø&÷ò9„› /ð è$ÜÞ¥RÃO£ð®‡É—¿–¿6Í6Uš¾P”E:î6ßaÞh~ß2Ç@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ÿ/àkVu¢õ5[0(dó -¾œàq.1¾} 8N·¦Åzb yDÇç|åvmìTìU{Vr¡’£½õHkM˜é‰ÿàÔ)U¥qŸHSÅ°€ÏçŽõ¯˜ Þ¬ÄÈ‘1Ìê8=¶1pâäfì»nJëQÿ"—[i×±&O‰cû1PâÓÆ:EVb¦“(QÙu›ùÓ¾½A*Õ4ªíL-PG»— ¶ø’ÃÂ#04Ò)$p -! w<é|4 Þ½^‚z g–ÈÊQŽö±ž‰cƒ'Þ1¶Ñ=ñ¢;_¹nÊÑÓ­¯¼ât ê¢æ²NÉ!.˜ÄÖép† "Ú|“¤ÛÅí¦û¤ûÄâˆÊ0“R˜.EˆÈ…°X^^±0r¬[¾"bEäNØáœÓ‰‰¬l˜umR|œlÊÊÎÎÌX® ~ÚQo«Š[¹.ÑgŽ ‹ŠÄؘˆD:2âìñ¹ËçÚÏ9.jHL§ûÜ'à;ÉOâIé„8!›¶Gï‹æÓØ4'{ÜÁ™áÌ“DÄsÔhâ½àt@lF0ß¾fëÖ5ôcæq{óTà}s>eB½ô™Ú¡^dy,bÜc8äж'~rÛ!¾´)!Iý«úÕϦ©_ýåOêŸÙCl)›ÎžŽÖþwÅ ŠIƒ8éЯ}ÃÀIj)$®)sZÐNˆ´¬ušhQ´ÃMµ 0 ³b’õÿ/1 éÛÑíPsÎn%F¯jÏEñWþú”¸}^cÓ|ÃY 4*0C)¬³‰)\F³ÌÂøíl2ϳÍbÅü^¶˜ß ¤%¦{•J¶†/·mä›°N -¡“g,–e²xŒÅxÞ¢^ä‰jùç|Ð;k:îZsFØ;ÂpO[_V¡>@–uœ´¹•8WÀ ^¸Ñµs­ëVky.ÆiUxPXŒ{d°‹`†H—KšÑN¦CÚ9HW]:^ÇÅAéûãˆæŒÕ.¾§’H§¥i]0•R|,{„xê‰'žR[XßGׯTµré|ÛòûkŸV/ýÐñ?ÞñQåÚu«x‘zCÉ‚ù¥;^Ù[µÍí=±ñÍ÷ÉúÊ:ÏŠdÒ¿0Èö…§ÞÇV³ «/ˆégîºS=¦þQ}O=vç]§Fb[ÛzQ¥Iq-IÑ -|¹Vâµð€R+=gÌlÂHlA§í‘Vú¾˜–æòœ]¿ãØØÎu âßµß Î4¨¹ gºx^Kª™±ÓoÈ9Z@t Ò°ÇgÑOBû|Ѻƒº/ˆ LK2Ið07Æ)Y"OÂd9É”¤$™½ÑÙ,›ç²\^,I‹Ä’ 5òÓFy£)FsPÁÁ!AñHïŒwÆz5÷Àœ~‡ +¿áä{/Y{ï‡o±7´¯ì¨R®­}˜·×üB-fuÓ;ªÄ™wÿøÐ!~KÇÅÊ•+WišSI±iÙI2ü—`ãv+Ž‰VÌÜdá11ÑÃ-ÖèÉÃÀó¤ûÑÐZ§T &’ÓH‰¶Xc"Lfïo -sÇ¥8><ÚÚÞzŽb”.-néNÄñ†³[µìšTº.$œÀTΘԴÔ[RÑïcô¢;½Ø¸¤ä¬à®ˆ–œœ””umBfF°4ªìí»ž~aÉŽû>ûƒú‘zþ—·.x®¥rsùgo±of¿/¶¿>0{ùâ3cÂú¾wà½OÒÓ~72wÍ/æÝÚÿ•]oœKÒt¡œøî/•“Å$B å$1Ö³ž ‘›íNïê˜C‘ÍñMÎu!6ÁгbAÅ=2‰NÿíÓ­Nýèž»ÜNÞâ -4“:hÑy^zTztzLº7=6=nh²/Êí‹ñy}±¾¸¼¨¼è¼˜êwQ—¢ÄPʆò¡ž¡¢Ÿ)MI3÷³”@ +á%ž’ó´ùš€cƒ{d«G*ÊMºÐMREû>ÛɃw›>ãwsÔËê1–Úþ35ñ§×ln¶ó;§9víµ{úôc×1 ůÕŽnØ¿g‹¦iÄØw$ë ˜â‹fSž•Y%l°Ë-d“Y(Öqn-xY´àeÕ‚×ØF»^×Òöœ£í9GºtÃ9G©ãb5ÒÙŸ'ÏSïÑ€¦-°“ƒËÊÔ”˜×8ãf–¦þ¾¹±qÏK²{S^ñŒêö4ü}õøûˆ®C”Cêt]ë C3 É•vg“mƒ…qÆk–›ëÖ|öåV=Ì‘8]äöå{ô´+Þ©Ÿ´‡*™NÝR¤Â¦ûï¯ÝÝÜ<ü…E¯¾Á·wüœoÙºåÈöŽJÙݱefáWÚ¿JBY*kÿ·m‚¾>»|DÚ -\0E‚\ÅÑNáTã²½•ôßaö™óÌùæR3鿶ôöj”ÿC½ìþºøxœÖ³/Jýéøݺ¸¹Wðs?Àz–|—Ä-½“yªè«Ü΋ø,¥Œ¢ö -^%~¥<ÂëÄå)îÒ¢6·RÄNÁdI‹Ù})>c¾­ -WIUâ!ŠÌ›qƒ©ŸL¯›Þ5ý /áߤKR¸µ`œÉe±ñ‡šyâ_:öð9—:Ž5ËîöÙìlÇåŽÝ<¾ã#â÷Šüâ®qÓó”ê pt¥:¥â’ýB#Éîï[»deŠ"ûŠƒ©¾$Ùe 9Êä±UFy±)¢%Ìag ¢ÈyN%0/2T Ï׃v{{«ÿ©1'çÜ圣­º"ø‚ÒòJjê _Nø8¡3ÁLš¡ë‚§·~\Q_QRG¾²âù#Í Uïh^°ä¡ÍÍC—Þ· «î_üÍgšÚ<ù¸¦6|˶Ç^~ª£RÊß3kúýݧ]HQ>~•Ö¶üc­=×­µûó=¿õðë­çŸè-m¬©­ß3-¢}­c|Ar³ šmM®u¡fWà­èòŒ ÕQ—'ç?4¬Êå -S…Ra®°TXËmöŠÀ -G…³ÜUv)Ìyõ“gr†æsü^—=º{Wí#»w?r‰¹Ô‹—¾V¿bNüøüñãç¿xóØ—«oª­êrCƒÈÛ¸Ùu~Éà¢Ð ô +[é$­Xio2·˜,²J®ËA¹—n[ä/O¿­9ÈýyA[ƒ4™$ýH !8&ft¿ÇŸin|hUЀHÜïrž8Ò±ÄQ4CÚ­„âÚ1Ú-Îwå>ºRŸ WRŠwU’{µ§*T‹w‰MWrŸÛ"»IqÇLѨ:}UîCð-º®Î}ºSHÖLrn¤%Òi@n¼ŸµŸmˆyˆeˆuˆÍê/Kà)–kŸ 4wš§OpJtJLª756!yµeµuµmu€K{3Ĺl‘­hôc :0 Ã1#¥(srZêÐÔ»R+R—§Ö¤Ö§^J ¥rþ“,-Yøq’¥É׎ß9µªjú£C>ý·?N}힢7 -V¬›¹Ë·kã'¿)Ú/ Ý“’2q¢ot¬½Ï¦ªÇÄÇÉÊšrëؼÄÀ„Ú[vGkZ>å_ÅÒrŠÎv¡â³àd-J¥ÅJ2¦øìpÙ5-×C†ÿ¡Øo—ö=ï÷ÏZ4 Ý"—IYœl +WW-{é¥3Û*+Åõ×ÕõUã7o}‡çW³4＇ô|²n_n⋼baë,¬ÅÝd#ûr[Ç“¥åz4…äרs=fVâyE3³ Š ~ÅîÉ’ØÍÌžkjºqï¢Wßd¿e‡øŽŽ‚­[lçå?Ôï.šq wjÜ_O6^!åƒ ?ø’»ü8ãB» —2wy8Gx™œ=r&$0ioÜ,=O~ÚK/í ˜öâ ô_!þw^½ž3”ž·"Ú+¯_âwór^ÁWóå|=ßÎm#3šõgŽp —’ ‰¥bªäU² ‹ ÆÁRº’ ¹l4Ž–rÅ(٧ܷ³)8EÊSŠ ˆÍÆÙÒ,Q,ç+‹`!+Çrz*¹O^«XVQb±)ŠCrÙÇkôùÉÓ¨£líY³D)œÓÜÀ5ÂT*¦‹îljáY=x%Ó‚±(¿*Ó%$·Óî©ÐŸbshZnH_¥#£„÷1CŒ¬$D›ã’Æ ¸"¨£ZÙ®‹+$"&þÙ'e•ýôÙë€ Á¦„–°¨X-\gdq¶:Z3èÏÊ]§90{`¨ºÏ¼—ÝÒRHíx™<~%õÍ|Ÿ&ή³G’&»ÿ”'E%kÂìn7k¼[¶:wI0Þœ’ 8‚CÝŠÃ ‘²aŽ‰—Ü‹31] B#½ÏÆ:y¥-iƒÇÛígérÎßóCmW‹ÿG¹°_O{ŸE=<ìê}.=gC)LÕ¾2$™‰•tý»CZA0µüu -Ëíªc¯~©W]@(ßU—ÁÍŠàF(RX - `6Ì‚bX^H¤^È€tÂLªM§^NcBýÀL(€¹ÐzGÃ<?€jÃàB/ÜÖ³V™ÞšI×™4g1•…4Òò/ìšÝ³ëDÚi1í¥}õ`Öè( 9ÿÞŽ#¨v7Í»Ñí Kúj3õ:G^Ze•¥4f:­;›Æyi~ í^ ßûñ:ôUʈ"zŽƒ9Ô«íZ¦áižÎË’ßÀ«fuÏAÿÇ:ÿH'ûNxg§>RÿÖÓû•%Õúõ–ʞ«vÇÖ3ÃA£µ¯Q9(c yFþ.ŒÊÒ|FRN¡Rûº£ÓëGåBY$qFtŽ 2—ΔÁXGåx’&ƒI0XÐÞ im? ÍÉ>"wÜD°ßS°p„ê_¼ý«VÝ5m$›3sÁ<âÃÏ£{\¿güm$rœ -4ß/SõÊØ(‘¤îJçq}ì‡0ˆ®úZ©ãÁžQv—k®,âï¿÷¿ ©]´ÔtµýÔnî5bsïkçs;ŸëÕ{J/·÷¿ýê+ô’Øÿyè¬éœó/û Iwñ{°WOW½óÕÿAIÿÞ™v.ëñ£Ò(Mª=Útª÷ŠÇ; þåiµƒ¬.‘ΠÙ‹fa£È²ÆídSšEq°Óý™HÈi\Ù®f›’n•‚æd‘ŸÎ&4ÑÜQd5£ Í´Æx²Á[ ­´Ömdÿ“hÍ)´¦fýc¨äÿH»‰®º©ë*_5BÖí÷§€"Ñb%¨f'®5pPÍ .½¤û¡`zjõûŽ-°ܺ︯`AÁt¨)X0wÔL_P0jfÌ+£²xæ*—.¸jfÍ,¡ú¬3ç@MqÁ<S> -stream -xœí9yxSU¾çœ›&-PhÚ¦‘²¦ Pºå&鞦Mš=Ý’Ò%é’î;m)Ô²”UÞ*ꌈúͪ<A?Ë(.øé7#3Qtt|3ãèÆró~ç&--¢ó½?ßûæžž{î9çwÎo9¿í¤#„f ˆAzsi‰Ë¸Ú± -¡y÷ÂèB³«¢pEÙýg ŸVâJUÕöÂè74õøú˜ùBa3 ÿ‹¶îáÖ¿]‘æ#$r#;ÒÞâkŽ¼Rs§¡jÛa \Îl‡õ°R´÷ Þþ™âhè"Ä4w¯hòåÿ¶¨ öÛcÇ{|·÷a'ê†ïÅ”ž^_O‹\‘Å"4ç]{¸oÅÊÁÀ HôõÑyDy`¡5°`sý¬Ü †ù/A<öøIÚ~rà¸ÀEî2s’y -º"DPðu‚xîjQ8s’ßiòsŽ9‡Ô §Û`†: a¬*í ¾…Ag;©‚1{°ÅuHÅK‹N¡øn´Ê@…7æ[øF„æ¢(™{& Z -¶”öQ‰<¨­AëÎ -Í*ÁjòP²£j˜­Î>¿uùôØ|Í?1P||éE;Ðãè-(çÐgè -aâ|ŸÂÿ )¤ÜMÞ ïðåR°0ˆÉdZ™;™½Ì1æ DBQ -´— Cð°à„àrX|˜'¬'ìž°ƒa¯Bù ì¯a°€0 -Ê<áRaƒp—ðUáQ’È"ºGtTô;Ñ7á3Âg‡—„÷…ï?!hŠØñXÄóïF|5M0m6”Eÿo‹êGJ>h‘œT`ÄÌÁ±‰äJêðôüCeæ¾ Ìå'³À*Pb -¡P$‘k•l\œT“B4jm††•@Gƒ’ùD‚`\(”Dk5j ³Òj ´žìd¨ åÙýuÚŠöp_˜:©Ð•ÅmÏÎTë>ÉËÍØY™-(+ o)^ª ¿*V8ôeB—KTïgE|#–; -°D-þ",‹åŠ -SÒ5Q_Ò µ<<ýkgäGRºa¦;`¿d¡YʼnѬDH$±ÑR‰ülWG}_W[½'ŒÁI¾ˆîÜÞG¸K°R+k‚+¥m´FM”6.ZKDim]ý}õý] '":qü#{±²#Âǽwd”û¤ŽÒ“‡Ûñ(ù‰¡#IEJ©[¨ˆWÐJa5åx?ä°/ÆrŒ÷s_Õ`1óÔµQAõ):xi€öéhBÑã[%$j‰‰ò8Ïظ8V¥Õjt}wíèëÛqWÞ#5Œf©Ôl4˜¥dÞ«¯yúµ×ž®¶Ù†+kÖØlkjo"ø„?Bd†Ô"FÊåâĽd£ËŸ Ñ #…“Xò)ä+HçD1‰äÚv¡H&‘iž½·)ßd´3U‚-mÃM#¸Š[ðô–Eö -sßšfמ¡Gå1=pD‘Ï@SƒÐMD¾¡¬P•(¹†ÕjÇñ6¶›XsVCÝníüYx•Èm2Y›[¤·yëŠô:‡¨š ·§»5ºšÈ°¨:G–G½¢5×aͶ:pŽN£×kA~ý‘*rMC‘prÌŠY‰,%Ëãúžææ=ÆýEø§(Ù¿Ÿæ¶Pš­÷d‘wi$À2‰vœì›è¥dŠed:·_Z£vµ'kW”wlžYéÖ—”–kJ¢—ãcdë……KŠzò,…Ý5Yúúââ -vÞ ´Q:òGò"8 ØL:!b@“¨‰’©À|g‚ŒãÿZY]…¥]…mk5]Ž¶ÁŒû¦¹9R«ÊM®R'ÙuçuÎÆ~c·Þ{`¯è,{@nfõ•¶âjÊ• ^×ÈÛ4û¤Ú&ÓÈð5îÏ^ÏàÉ]þA²¥ñÞåK’ Z7"äÌÑT‹'N.î…àdT -! -…¤nÏÕv9]:­Îª_åp¬Ô[txV©©¨ÔUhp}Zçq:k¼v|;W[ɲ•ÚìJIL½¡À§Ñø - Ñê2E™§ŠŠôÅÅzƒ™—W€¨Aâ©öj)"å7䦜P!JÌKV«„Hs ¹ëîêOîªmwŸYîv˜\åÅwê6?œÌftÚ»¶<|ä±å½÷»Ìvo•ÅZE-›·dÈk¨ïL=È ¤'=}4kƒJzüß‘p¨Èé<ëtRëy“Mä9€š†Ð"™HÃÆÈ• ƒÿÄÉl¸Þq­¡|dä‘GÈs~ûEÜÀí£;Ò…j2¸âù=$n™x\þ‹Æ%:ØHX|&‘¶ÍÄ]¿jnV28î!½ÉTøàEG©áÑýا²-ÕT¤“±ÔR•®,zQ®F“ëÿXŸ•mø–êDRà*ù y Í£ÖIåH#Æ$×’Á†„œäÝëxBÙ3¯Ça,,´:àSùÄ€o¯Ÿ:èímÉ7×»]ë†üæÏVàŸò²x™ÙÇ'r1¢\Ìâ݇««œ;÷o÷ìÜ62BÆ*íêÚã"cþûwnÚ|¿ÿ<¥vÁ¯ÃmüiŒ„š/ø±L2Êqx.w'"+þdÿo;9aÁžŒü¹Phðmø,÷ö·ßzȘí/6î ˜ rýò*oSÙ£Ì$!µâåAjçõZ­« †UÖÖ­Ó¹?ã¡pï.¯o_mí>ߊ'•øë<ƒa}yù:Úv›iùNðÞzÐÓÛ¢´F %2¨,¼I«‡Ûãñà&àÔŒs¹×É÷;¬ -­@£°‚FO€õPžfº—Ž—&\Ðø uÃøMî3ØiáJîuœuÜk°ÕKØÀä Éãßa øy¶ D£ßpG=6 g™(¸sÍRÇȺ-ÆL”ç’þ O#¦·Õã~ø¯“0ÿ5ø~äŽóÕgr–Éyb°˜%ýî~ ¥×C’üï‘þ÷ÉRЂ¤À÷ä õ,´nˉ“‰PÎÇ—€óbmÜçõîk¤2´Ù©äEžƒTªžmm<%Æ—{=¯n#Æž-à% Ò£*&§æ"‚žQ5# †Öló|¶zí/-¿Ø¶ ïZÉ}LÆj6æm - žÂuä2PC-.L)œtú¼ÉIQ(ÊŽ“JÖí`5¾qR«œÏøwônÝ1s‚Üæ})Oã{FŒ$‡è]ôΤ7V"—GƒÇÍ¿x¨×óYIÖåzšÛ°Íf³Ú·áyÜçdLëͱôFsçq«5Og„ÓÈ Gƒ /£Ǥ•¨ä3ÁIVM…,…lŠþbûÂÌÆû²ÂÌ"WNv‡±q}xs˜-/Ë’”ŸU®¹Ó"J¯ÌJ7°‹Uñ³¢΂¬ª´Æ²Åy9šÄ´ÙQ1‹œ…lM&PJq8âý!åG®ÁWOz^›óßGVبÎØÊ*€‰¡š  Ãã±D|ÈSVêñ˜†l¶¡"ÐasmMM-ã,•CÙÙC•øØ!äv7ìŠ_lL ¹›û¾ƒûÞ÷ÑGA3ñ&|¤“Ì@R͵fD Ã3ö›Ö£-§|»G[÷=įI÷ÿ']=JJüLjÖÆð9× -§k©YP<šÏ=ZŽŠ¹ïáÒô,·û(¦x}ô841¢QUò0vsz\Î=‹O””XHB‰ ï€6 rØŒ¨oÂYõõÇ™CöëAÝ­`‚0arš˜iXœp€N8˜_Û)DG`Yøú>ê¬÷™èvÓ1P‰™É{‰Rsé=rŸq $¨ßWH%IëH˜ðòüæ!ÙßœL&‡¬–Îx¯wN—ٺڈϨ*XÖ­R¹Y¶B…Û7˜t©Íþó­iù–Í®!­ S¤%þiiÙô½ó@ZfëÐ-¤•“†üsÛÒ ® ®àà6þ7‘ljÈwA)„¦š2RÀp¶°KNI0ÐX¼Öd\[Z|{A]JIZ‘O„kpä¼2OFJI“iSEÅÆ"ãÝlWVS{q-#×ù &hž,Íš$ +È–­ô˜Íõ¹é+ñ9N‰ÏÙ•Þ -GªVÉëw™×(•Æpª7Só`Í$bãn ^ÝïÕ•tßY3«®Ð^œöéÉB"ÉT¡ÎØW¶V©òê*V¦éATî ¦ó“¥Œ:™p2±Álø&z|0ôø‘欚r³¥Âã™SŸCR4ˆ…ÚòÊšåÜÛ @——¦ïTèò=ÄäY¡Ü6t­’Â>¡[Ú2Y?~Q«öŸ'p[#‹Üuô+~¥têʨ RÞ£5ËåññP“mf¸/˜´†Õ蜯 g5×€̓–‘A£ù†³Ïï7@jÃ\¾ ;ÿÎ}ÍýõF^paR^pÁ?úÁ(~’¤Œg‹“¥rC<Á; ½a0 ›³kÊŠÌnþ’òduÆK; &É˳œ{g¼%Š’¤]n¶Mœ@J0Ÿû!üâºqù×ØðÀ$ñ+ŠyéS¯}⳸ŸÎ¾^o¶ŽXí‹Ó—kÔît•K•^žúíÞTT´±¢b“©4Å[XèMNö¼É¼⮃J¾•âé¼Ùê -餩kî<ÐWùA”ŸÊ=È,né™â¨u\¿¥gÿ„¯0wÍözã»-–[yÖü”æëàŽý;"¨1Õ ûéÆ9‰‚SãÓ‹K÷„Ø= ™nv +Í`™ܵy}ýÑ£m›×Õ?sôôi…cßz‹û÷ž&Ž#‹`ˆÏ_bøÄb÷|]÷Ç-Gž Šëâp2Ÿ3âv À< -°SòŠûp$·Oã¾Ãýv»ï´Ù²9° ½ÌÌ¡žA - -ñ²ÇÓÉÜã ñûè•ÐM‚É=- z2ö9¥Ã-ø#üÿ«Ýú#®?àäÿë´ÕÐ_°€àÓ¦ßrÕ¡oßÏ„¾”‹N†¾h>ºúCÓðm¡o!š­@}h  Ô†ÚÑ D<JãëB䆑hKy¨>€éE+ùþŒtÂ\¿¢­‚¶ÆøùÅüNƒ°b%ʆ• -{wð«P#äfMÙ£t¬¾WÁ¾Í°vƦâZrS¿p¶|7ò¼(4¡”s §BR5`Nƒ¢2ÿ ü Ø©pÉ7Áÿ»Oà9ú‹ï-ú߀ih.œ6šnÈ_<|`Êü2ÀÇðó±Í-«Uh>¼}°f*,<ÿðH! -endstream -endobj -1060 0 obj -<< -/Length1 47128 -/Filter /FlateDecode -/Length 11769 ->> -stream -xœí} xEÚpU½]=Ý=w23ÉäšÂ$„3 Ç2b¸¹ p“È!"`pñÀƒA@”s1bVAñ\p]ED?A]7"ë.†¤ò¿Õ“@`uw¿çù¿çÿŸÿ™÷¥«»«««Þzïª$ ¡„8,ìÕkÈËÆV>EÈ Ã°6¥wAÏ^¾ _¼Ÿ…÷ã{ºéVxÞÿ…> ÷-·ö8óÖ«aB^jŽÏ/ÜtK»œiÍm3¡[ñ~è¸iÅåÿ‘RGÈ wð~í¸;fór{á» aCJË'Lɦœ'äæñø¼rBñÌrbA$/ À{Û„©sK ]sŸ&dŽßêʼn%ÅãC=aÿ­ñyÞD¬p~Ÿ´ïåûÍ'N›uçxGjö·ß™Z6®øƒmï"d”ÌÒiÅw–û_a˱ýblœ^<­¤µíz¤÷yìŠò²™³êO’8¾ì?X>£¤|~ñƯ ì"Äò4‘¼b£wÿîûô¿qvý‘p*Ÿ|÷ÔòüÙÆ?ûê«Eª¶\õ`[Í$C¾g™&R ÑÎÕW׿®-7{jñ;d –ÙÌ -F\$‚=þ EûPV°½„cM¾oÓ¢gøˆ”²8|Áª¨ -cŠliÒ÷ ÒžãI/QÕ#«FÊ%cñøx<ˆÇ(<žÂ£¢á~“¡9ûkc¨í‰¡úÉ^~š”ª›ñ<‡ìmúœÕDïÕø ÛY~"{ÕÑ:~ªáY>{ƒ,糉Ñاöéf¶9\_ýKã* g^J†ñ=¤RÙgžGñidÌ#™æõRÉö‘J³Ýàèµ6”l“õ¼"úžl?àûoàM!~,“Q -@“Ì¿Y&še‚YúÌÒK=(%N½æ¼o^Ç™¥“:È|î4ïä5P;µ‘G°ÎnÖÙÉ~¢Pµ¢ä¸ù°\ˆuVj ¬“OËÖÉ ºù¦f–b3Kù†ºãñ¶üúxªšóâf©˜­Àœ3k¨Y’Hý¨ÿ µ—ZóZ—ZC€Ÿ/öæ?/€‹½á5ð“€\ð¿^ü]ÀyߧÁ9ßUü;ÕTG”¿}kð¿åÀ·üµ¾yÔÇ¿ðu |Ugñ欀3¾ð_¾ð¹€Ó>«SŸ&òSãáÓD8¹>ŸŸ|æŸÔÀÇaøËûaþ—øèCÿÈžpñ=pÂÇ?°òãAøÀ -Æ®÷±ÿ÷Ãpì1?–GÿäáG3àOïÅñ?yà½88‚¤Âaz÷~HÀ»ïòw_w*ïDêßów -áˆòvÞðæx8¸ÌÅ -x#^p@Àþ×:óý5ðÚsÉüµÎ°ïÕ$¾/^Ýëæ¯&ÁÞWœ|¯^Ùcã¯8a ^ÆÁ^°[ÀK^x1þ(`—€v$À ~¨òÁóØÏó5°OÛkà9lÿ\2Tâ©r<+`[<#`«€§lð6 Ø´ÑÁ7 Øè€e2jC ¬ÇWÖ§Á:<­«§pòO¥À“Ö>ñ -_+à‰5…ü‰Wà‰…Êš¥a¾¦ÖD”ÕV¡v¬ðx[X‰/®L‹Ôà -|uE³Ár¬ZÞ~§ß xù𨖹`i~'`‰€G<,à! -Xü@˜/ð@îpŸ€{s`ÑJø­€{,ôC…w X `¾€y5pW Ì0çŽ-|Ž€;¶ÀìYÉ|v ÌJ†™50cÜ. ¼¬5/k Ók`Z L­)& ˜$`â8Ÿ˜”æ@Éxƒ—oÀøˆ2n¬ÁÇÙ`¬ÅE^^¼Š¨›yaŒ£ -…÷£Œ‘ÌG -w#’a¸€a5p›€¡x©*àVCÒà ¾ÙÏ×ÀÍøàf? ºÉÏÕÀMÝü&? tÃi0 ¿‡ðBÿ~nÞßýú:x?7ôu@ŸèÝËÃ{{¡—zÖ@Á ^à„Ðãú0ïQ×cŸ×‡!ÒÝÉ#ºÿÆÁ»;á7èÖÕλù «ºŒ‡Îò=p€Nñ—›ÄóÂÛÑÃs“ w¿ÒÑ°óŽè¸Pécã<Ð!¢äØ }öÞ^@6öŸ½ÚÙ m<´iÝ™·©ÖÞ0oÝZ‡–ã!K@ /d&¸yfd!œÍÓ‘­š§Aºš;oV!'„"JÐÒÒ 5ÅÏSÃâŒç)~HÙ>ãQ%ÙIþþË'ÔiJ7e‡r²[ÌVJ•{H–ùì}e­2O9ªÌ#Ã$et€<$d í‡yö†P?-`ï‘DÒß®¡]øa~˜œ 'è lYIæ0ƒ¾M íè0ºßú‰üDx—Ëré9ú5R¼Š¼øAÖ¥˜¬Â¾‡tŸ%?™ -öJ–ò¬?AÞ _Ìï O¦23L…6üây²•LFÎ|A?¡z,!¥”]$ÕtÛÂ.ÒtÊãh¹9ÞSŠ”·•ñ)r‡2èèå(Ù‚Ÿ kŠ/ÔR:ÛIœ‡ãT³7Ønœã>r -ç…£³Ql[CNÑítRLÈ}t»Rd«$“5êe9'yCÞgï!?™üx˜<¬¶'?)*9h‘²UrŒ„ùJhÈÒO#+i?Ë"œ Ndfº„¼K ?El¥©©d¥’ O!íŒ-häKÞcù0–¬5q9ÝM–“Ýd¦™4güjÛ† kÀ2ry’œÆkýœY9;¦ÃÛðmSTP¾âíùlþtâ¬ÔjzÈBøVª-GüVCÃÈh‚ß°º¬Ã­[¬—l=mÏÚÎÙçÚW:¸£c‰ã¤³¥³ÔçšâÚéΉ‹‹;ÿŒ‡Ç0†1Œa cÃÆ0†1Œa cÃÆ0†1Œa cÃÆ0†1Œa cÃÆ0†1Œa cø?€ã<û¼Šw¶÷$!„’*ñ(j±“@ÄI6[69,Tшխ¸®ÚÚƒ9qùù퉫¶:›†2XnǸN!•y=q ˆVÿ~ù*šµpa…øáôÕ“'é¿]NeÏ›Åì¢ÙsJÄN6«›v‹áV 8\ÇæT7vŸ—Û‘e†|q^³l^ˆ@³V-ÿýjñÃ9úÖ©SôÍï~ÝOž7üCöZF÷²/Ùi$î%ö,Ù¦P¢`Ø×ì+䕱Yu°Óâ”lý>!,Ÿ.ÂÖî—ÈVF‰WpÐö¤]´ñût¢XE‰ò/gê{°j~ÛÆ¿ÄÈ… -íšd^8’M;Ðtæø¬îï§ø‰Ÿ§ÉßF[ÿ%oÇÏ+I é‘x½Ò •ÞµÎU‰ÄgKP}qžDIŽå:s¡Úu.›º2Ò›©n—¯CNu‘P¸Í¶Oš?òäùó&Ó{Ä~ñ‰øX짚I3h„UÓijgÅ7âì7ßÐD±DL£ËéL:‹.Óäü¾&„+H³AZD¼°]aÛù Ù®k‰jDN­È›êƒÑç\¨>nN›»Ã8wwè<"6Ñ‘t:r©šðfªö¹”+~ž7¢œÆÙ%’‡#Y$UáJõ%¦r®„y"q,sZ+ãÖ*äy–èåÄ¡ù¼4Ù~קý«ìC†½L ~ÿuÃÖ¯vçç»ó/ŽwÙýofq9¾s'ä[\ü»áÍh$ƒ%´ôµL Žž˜ØŒ4Kh–˜Krr{~¼WB¯Dg!)¤’nš——ÛQòÒâÎËëT¼Õ²G/½ÁB}{-™=âû~+î¢6šu÷!š,ÎÒdúE» &-¼qíÓªMõw}ðÎðÁú/•s8Ãd@¤ñVÆë+ì•jpeà…”Êøªôµêª,_¼ßC\~_†ËO ¨§e¹jÏ CWÇ廣¶qæ™Z”°øñœë\~6õù¼%Ô,#Óí“¢–¤¦ç6\4¥ö<þ¤¨{f~}ûÔVozzÓšþî¡» ÷žñùTšNCA8óÀŠÏ¾‡iV^§ÉãJ']Y8ttË,š ¾¶ÑÓ¨…£P: 0´³"A²ƒ„ìaVËvNa‰NmIR5ÅæA“>Þõ`uŽ)’3Ç»V£yG¢|× ‚»ƒ;]*ˆ›ý FÐ-Zyâ„x¬n²²ªn)l¯,þ*ÎSí‡\| -­€áø©¤[¤™bI]‘Tiq­t¯ðTZvØÙód‰}U4#4Ágˆ+Íõ)òî ë\ƒU|%™&Î Ò½’5Äë!Ws™ö1¼ZwGëáÙÔM³ÅÅ'?Ï»ëdñ#6‡¶ëÕëÁÙw,nÕFZiÒ—ÎÏ¡•6'"xRéS+Sl+í+\•ÍÖ¦¬ -Ûô$?„ã“ü æai±g\çšäI¡žûPŠ´™êõ4PCÝ&EHjŽIjÔŒs|ìl BiI ½cÌÖA•û]=ÖŒ=’ …ˆ^Ï”oß PZ±Z ‰g@.ô^òÒ =¨”Ôí²ÌºöüÄÇ—eúºÈŸ8Ó‹¥“v¤w¤E¢-³’¨•i¤ª ZxÚªìD¯nƒæIΖIÞ =)9’œ¡æÙhèÝ$‹LCˆ -y‡6p5ǹ›‡¢v5Ž`óÜŽyñž+fïN5kê”3ÄüÅÑ$³“&=¼xÕè ?Cö}ôÄãF;vĨqlíӧϞ=½lvEÖ¶ŠWßzs_Ŷ¬–¯.ûìË/?[ö*½uxQÑðácŠ¤ì'ãÜ(ûĨì-þJƒ¬¤+”JÔÅJ×ZߪpJØ’Ÿ@š%ÙMÙ¬­=ÈÁFsîÕÃNªšÞŒävl”w³ ÚH;Nô½Q[Wîw¬qV|Mó©…6§ñØ3i?­()E¥(- QO+”wNµ~òwÚLÜ!V‰ß‰ávnÑ¢ßÞ{ïo-BŠÏ"Ù•"”‹£¦ƒm%;•­*§L!ñš«íUºÒ†H'ñ,$—‡R$ˆ-È°ƒ5õÇ‚W"!©*·¤ª*7R5ƒ«\—×,UŠ‡– - -©º¡(6²]S–¨ÌÐ5 W%º -Bv€4õ­ëéù¢:§™:g‰þ35ðrtë vÆP£Ô¨ ´B«ÐgëŒ×"ž6\qZŠ°…µ–zÐÖWéÅ{k}ôa0\ÊoS'Â$¥”OP‹l³È?„PsrŽËè– -!¤”.TçgGtÒÌÕ¬]³AÍZØ 3éîøÆ‹†h’ŒÆ„¸‘£G<ñ_³fÏšý_¬ÏüÅâSñaÝ=¬íDJaù 7Þ,ÖÍ;®¸XÌeþæ¯/ùËq~bïÑi«Ño”¢¾B[ö“.‘$›K'>°cìÚ®íúZX•ßÆFÔVI.ÌêÓqÐõf”æì]í’Ç$3ZFUotj ¦·“ùž´>jòW ÄÃbÝEg/øjò”c3ÿT]ý§™Ç¦ îtÝ@Kh)Ýp]'q¸o¸øÍ×âbA_ÔFä©šoò4´xÉ~} ÝïÓ˜Ï ¼«-ñéÍ”09hä\öŽA~tcÔÝà!(²'!š!ªÈ-Öí¡êŸ/~W÷#]I‡ÐçL*-t§¨Bœ¬ì¨½ýÛÓŸ}CÓ‹g•ˆ‹O?#þQ2«XæJH‡òÒa%Ù²_ÝÇö“%Ú~ƒr¥IÀfŠñàÁFÙe¿è²²—ÛQõ­Mâ^ÅY7ƒM¨[Í6_:‰¡ç”ømÑ|Ç`Ý̲õKä£Ø-æ›ý«œCúW¹†ŒÄĈÈuã (}ÑÅ#|³ÿ½˜ýUH]lèG½ÝÌëî‰Äkœajt@G? ¨šìÓ%ÕŒâèP͆ú¸OuÙºBËèo Ãx¡¥ &H R'ZŠlp§¥ÂRn»O¹_ÝfIÈd9¬‹Öƒõ׆°á|˜6F+b¥|šVÎîDC«ÐâK´gµøBsòh?øϤ‘µ¢¯Õµ¿H«Û.é­;ÍBuÝjϱu»šr™“„ˆUrX¡>  ¢ÒÉYŸit]²+“—"ëÌ÷,RKläO‘gÑ,,2Mž€é†Nó CDCfX¹®£ÜPÛ+F{‚z^5À(GâÕ•¸¨]öLf|,Õ™Ssêæµ´€î5Z±î¬»‰ ° 0ncÃ-ɬÌRf¬`Ë´eú3¬ÊRe$)TÁÕGýàr´¤aèLsá&ÞÚPm¸>È1‘N€m²^ä¸O{H\KD/y'³Öt“ߢÂ¥‰­ÄV±œŸ¨ý5­”‚Ú“¾´·‡ü'ä…NæGR-yò/ÁòT‹–“? q•ZX{Å‚³6¢³Îo˜u~“\N1çj¡¤0ÒÑP‰Õ¯^G{Ó~êP:\d@'ªEÖ=ôEÕágI–ß°–¾8ùˆåV6ÂRÊ&Y¬¦à©””Iöv ÓÄàº/LŠ‰BbRkþU^£Ï4H#—Bêvےß®¿J;{‚K·ø¬5#$ wTu$eùqùùM“*|„¼LL‘zHNLö'%%''¥äyó|=½=}C½C}ƒÒJ¼%¾¢4çeïg>sž„ø #e§çNš4w£¨`py¿tÙM "ï‹Ò;Ý>º˜P:LÜ#~ª;ÌO¼õácûÚÄUÜ#†Ñ™åƒQ#–£goƒž=“<iIŠ'ÑmXÓŠ—¾¡xßHÜîV¶‡×ºWµH3¬d Iö;<³®O«¯>(#RÃJ=þY×ÙË*;2¤<‹æ§æ§åòƒýÒúú‡…©£ÒFFG„&§”¥–¥•&Ë‚ÓC³¬³l³ìóóƒóC+­ÛžH[X\ÚbÝbÛbß–º-m[`[p[¨E¡tÝ ÉÊ"º"jŽ9½qff4)ÌQ”Ãó¾øнÃgoþùÏâ¤øàwâ¿–.¥Öùwß?rñŠÏÒ uÌ£ -ß"vºnÀ ®7$†rŽìýÇßóriÏ7Øk@Z(ûÏ;NŸ#—0ïá“ͼ§CÄ¡FÏuèÚ¯ãšëxÿ*°ÃtÂÜtÂæªøBuN6Ýѯ8yƒ ·¨.éÜ/ TvD{63*•,Šd‚\ ³Tʸ<¦•¦bƒ°4O̯¤_R0C?¾k) Ö8¡yþ²G¢‘Ö„¹XU1Mh"$* < µÞ¤7í}”ž¼ŸzK5w£K¡!ƒ.¢Ð‡é¢ºD.:ÒÊÀhôèFˆ¶\úR:-ÒOME_Ê‘dô¥FžÕ@Úõ<¤Üš‡ë -#O7peÁRÙ°9ÎÂ60ܨÒÅj6+æÑ/†X-Ä.y¨‘|Ù¿Ê* Û•v¤Ñõæü²çý¥yË3÷pwa#ˆØÝpgÞw0²#&Žf£y±±›½`T!&qf ó×unUm>– ŠŸ{5ždM²e²©´à˜RÚÚ9ò0vä(9<[Ͷdk9zGœwG_è¥ôá=õ~V™Ž`#àVåV>Xl¬ÐoµÚÊH-cÓa¢2ÑR¢MÔ§Ó­“le¶90G»S¿Ãz§m±å>íAÛKl¼¨ìæ/hûlý/ËÅL7i¨½7¡3ñ`tºX/ -ÄßÅQ€’zCé&ô¡e—E‰ÕW£æž6=ÙŠH‹«õ ˜F4)ÍÔ/•Y”ÎÜG:[ôFyØ¥(œ¦(, Š÷Ÿi\•Í"l+Bí30!C’aÚrigÈ6n2Ê É&hHÜú `Cè–º·Íè_Í<µëëf³¥Þ‘ªqR~ãf¡ -aíäBúø‘lê åÊp!NîÛ‡~{Xý—Ê›Ê<´ª0y2ÒRW‰?`%N8ª®ss§¬K?ä^e#é h×íÖn°{ºdÈ%lŽ;F¢Kô¯Îýt.¿Á› h—Ù-Ø-Ô=óÆà¡Â`ah:º­»ƒw‡Ê3 >z2ødè¹às¡Wƒ¯†¼9iÙÒ"[ÒÆ¥îK[Xž¶,°1m}`GZUÀ%½ØåulnØÊ•‹@Óou¸j3‡m,¿}äÍ%ÉuXŸ]÷lÿW¯Í>¸ÿw3ߺuæ7³h;j§ô+¸ñÑiYÔݳ¥´ððÆ7w§ÜzSÛ¶Ô’ú½üÛtôï£Uæ¦)äºH²ºÁvÌI6xW9%oJ<æ‡q‘D›nOr™!+çB4•2wPÎeïº)­(M&îMV¤`îO`gzÑ«™9wîÌ™óæÍë·köQjˆŸŽÎÞÕO¬¡¥_o]¿~ë3ëÖ=ÃNŒ-/‰:Ä— -ÇnDrP¦RV¹(+?® “Èz\q±76¸• (¢$K;ñt¹zMqáœëG$kLr…\O\‰r án²ºPrû.¿iåÓO¯òxdÈs·‰÷Å6ÌÚ {Vé&>ÍÉ~þÉ'ŸÏi/N¸úñ"v -È|c”\Õ#q®(¯6c÷~L[å8D7G!vIêaí"y%µEšÂ™ gª]g®ðÊÌlLyJZ:äø )ó¶Èmá~;f?Qãè¬%ïfÌ {Ù°Ÿ«7ŽEûR@ì[Xû®äž<¢zÍ Eþ|2¹!"Þ¿Pýˆvœ¯³Ñ×Ų­JIö2Ík'Ìîì’bÒw°Éæ‰äÜ9¹Kíž*‰ô†|×0-HxTó¸QûŠ}í#·· cÕ1ñ<íO›QvËæMü­‹u(½ûî -Duv{šKhí,^_^º`öt)ÕL\ì.AÆ“›#).NmÚ•®#›êNƒÅ[ˆEçv{O§ÕÝÐmŒ›Vé^ºV×v=x0ºÈ=“S[í:'FM#|¡e¡ÆjÉEo4š¦»ÑTò:À’·t¥9âObÍŽG>T=ëT0°žÔ®‡"J¾ô¼)WÑ‹;”¹(Õ¤$#Å Ä{ÄwÜ¿..ÍA?dß™¾!îr 2lÄîˆxƒ=Ô.YAÒr&*eñã9™ð ¨[µlXÏš[7(å„ÜPt÷¦q{šlTÁù{âµ½7{G?TgÅÞIÇÞ5²jsuÙü;g–ÏŸ¿oì(zCÍ%zý¨q[jÝâñe0Dòr×luóÊ5ë6¯X¹ù[‰þ<ù›@r#‰v…PãdÜ1ï!×&evÒÛm·;]rѵä®ÕH·ôiÙ»Šü ýQõÌucšµ““Ò }ay’|ÌÇ;k¬7ï§ e…ÚŒP%|+gwðùÚl1Xû={œ?¡y¯^¢"Õt-]_wŠ E±FõÔ¾@Ÿ@§¶‘~ ÚDçÐÀ•´—È&&ɾü3!d/âå|!o`„ÉÕósµ´šm¸FÝ…ö&“"YÉÞ8]± jItž K‡Ci;SÐtÓRHM¹{Mh :!wÇËzá5Ÿ+{ -^)?pH¬¢´×ÀÒ2&VEO(ÇÛ‰=ž0kl™8íÜ—uCY{JÒœ)[×Õ}Âúì™òÌ“u+E›Ç•“FÅÙ]«³;YgÏü{õþ}âÑFE*¤Ê6øÐL¤BþT­M$^ÝG6ØÅ­JÔ{8@o—k~ªöb§øîþÞñÐ4V£)7zK,Ù9 Ì™=þlTžâqZ|&^¦½aÞ³6<+JÄ;¢ñzõ ^')©Cùh¤DÆ™.‘Ô+qæcý v¦bŒ‰˜Ñ¦ITv9sm¨ _fEC<ŽoB(‹“Á<ë|vv‡ÜVm­zúúgwîJO§®Ž;çg·³kiÿ°£2Mò5å¹™?…Ù´H’ƒkNØà¦;µ ÄЬ:ÓÑ\qŽ!ž&ý«Ü²ˆ3³‡™5¼’5¬¾ÐU:=éð»“îñòÓNàt9ÝƒØ ä-bE¨ D¨ŸÜ8Å\™L¸Yz»Xú›Q»Å{Ç_رƒ?%^¯'"<°S=yá8=‰‹­ß ÅëÑ‹¨J‘¹žŒ‰µôKXå¢;ãTFâvwoôi®¤¨mDñ™œ+-y¡Ì\ãQûä+ÙZ/Ò‹bÄVŠ´;Õèbx€/VW’•t5¬VVð5êVx‘¾ ͯìG0ô&P z‹9JQm ¨—ÖËÙ÷Á¸™Š³·Ñß^³Ñ1ºÑQîGt”ûu?âô/îGÈ=ÿ[_y ‘Ëry¶ÑÛÜtcL5* “}Ñ@ò÷c2o$ø˜WqÜ¥º°›8Ýcø¬´9kųԖZ+½¹5lkçÈ%¹´#ëÂ;«-´ëlݽYtû¡·ÖÇ6†*Cµ‘úëpÛG)+RÆò"µÈR¤7Š¬³Y¹2“—«å–rm–QnE¨÷jê÷[¶-u<¦=a[æØž-Ê3üÚ3úë6 -D}Q{Ùö=‡”wÕìcøHù„ŸÕþªeýÖ6ÒV¼ù†B}訽¯ÒVxŒ‹¹¯îsQpµ Ô)¬öÒz`µ¢A{ ”Ÿ•Îô2R­Äª§j:‘'»ª¥Z,ª’*U7e%OŠüò-ê8%a…§(ô´a1¯(·P«­QvÎ7Ñ¿Êse…qðøÁË;ùÿ¹?A¢»  ¡Þºê7À}0ŒÒÇÂT}&Ü¥ß a‰¾Tœ,ƒeð¸ºV_§¯‡çõ*øš.q?쇣úQýƒÓúiýsø¾×¿×ÿAÎÃ?Ôz=S±i`ij%E‹7vºÒJkn  •.ZG#Û^Àú+ýµ£Ì~?YÈ–*ªKµ…Æãd[£¬V×h+§Õ*öšòš&õê¨rTÛo|DŽ±ÓÊ_ÔÓÚ1ãkò9û^ùFý^ûÜø™ü¤v–¾šB3¥FßSé¼³ßÐyxÞ*î«ýYÜǺ±t±‹¨;]w€ŽOI»óãŠ*åæ¤y‘œ©Ä‰Vf³ƒCþϵNȉ­ÐÜœa‡Á\œë´O€ìt9V]“ ½¼«QhMäeœé \è?Ñ}Æa*ûkÃ`©™¢Ú|ÅaÉͧ6‹Ý.ë1Óiíjâ´Ž78­D…êÀ9j˜…k>êå^ÕkiΛ«Í-ÒÆ®ƒ<5Ï’ïÈwö$=i?è§ôä=Õ ¬D½ŸÝÏïWÛ;g+qq±Ú¾Ú±•mƒmÊ3Žgœ¤{a·R¥W¯Y_¶¿ìx‡¶v¼ëüeù—z|óx:uò£‡kƒ!¦Tdõ~õ‰£å#oI@»¨ÓØÅKy¯¬ø[çþäú(B”^Èu<IÒÐ H—Æx¹6pMÚQ tïÕ©¢{?’ã¾üsÏ«~—ƒF -†³¡–I¬Ô2—Ýa¹×²Ü¢¡’k.Š¼\µp—5²´vÖ|­»µŸ6Æú;í)íxUC%ÖPµ/¡Zó›ñÊBåÒÐlx¿nXëcë+E5ukêI +î ¨rO “ôŠÄ“dÝîsd%z*RT{¨g }H ¹`Ää¿ë‘#Gj»iO"îÄ”à†‹­³gnòZB;“ÒpU€i0Z­ o¶Ü°È€ÜŽò:„ï.ÿ(ÕÜèÝHK£kJK?¤q楹Q JåfVªž¿%§ øÒmÃÌ‹è¾AãÞÁmÃP™Z¡‡êa®éüd4®üºb'v+­pêöøž^÷¤Fê‘t“Û‘xŸÎâØ)ƒ¯SÈ&±Óe“ôc;×wÑ-ˆû|òþä*L&—'%G’5jþ -4Ý⸲Éñ±Üì÷VÿÎLˆÇÅñxU•ÜëŸÒær¿ãRªŸoØ…1©ö’HÄ͉f€VàêÑÕÓ)_ŵ ôºâ5ÅiÝ +ëÈ&·¾ÓÎLb/ﾘ”\»ïÂæ±·‘’sºR¿øÚÜ{‘kYžõÏ»/H‹AZZ’¶dhÄïð·²g»|­+H.$<ÕÞ¼g³Œ!íšðñ‚Ü ’ŠàM -¤oh{ª9ªB+{KÔ…æ;SCru+µÁT‡§éW«„ü…ˆ«Õ¢jj”«ÐD3”Q’ã×êG#Ÿ¯hˆ¹w!g(¹EnŒ$$ºí­\Ž -’Š³óU¤©öôž¡ð–MgW]+ç—”ÖlCz*yÖ¦KúNWJ°qb¿¬æ¦þ­¢7ÈïLÁü’®K9ý¢º›`!#ä×å Ã4ÛüÞ¼¼¦Ä‡wÑkF4Ú«áHl¸Vš\s’H§7\«$ÞOn e¤œÌ%3È$2L$³Ì¯ÑC®IÉFì€Wc±EôÀ6³ÈLøÕ/ÇJÈ _–­/®Ÿ]¿¹¾ë?7©)z\Sû\ý²úeÿ¢çvMÆø÷ðOmê§üzãú çÍ×ÔúOå‚«[ˆçÄWõÅâÀ¯ô;[¾'cYpUý»ÿ<Êeš/S¥æ×xr5­×RþK >ï"‹Í›nŠ‹â¼8o^¯Èû -ÔÛ%¿ÄE”ßì«FY&Io”ª+RÄ–Ãë\CÓe9׿tyŽ” -iˆúÒ§H%ñ˜¾ô®âÅcɲâÓ¦“ecgO"ËÆOŸ‰åÄ’XÎ1•,›PR†×f”L!Ë&OÇ6KÆbÍ”âéÅdÙÔâ² ,Ñ'ß;­xÖD²lúYS6¡xY6cötl9«tú,'ÊþÅo›²œ:iBñU¾[!QßM‘;òÌqþ’Bš‘e{S|•yÿ<òõ¦øíXˆ˾ø%}ÿ7ì}þY -endstream -endobj -1061 0 obj -<< -/Length1 17264 -/Filter /FlateDecode -/Length 12304 ->> -stream -xœÝ|yxTŶo {êÝs§;Ý¡“Nè$B&’¦4Â<ažÂ<*¢‡C ˆ&P"ǃ˜ƒÂAŒ<!ˆ=ˆ9\q@#z=€ -â­Ú݉À»¾÷¾ïýñ¾ïõNí]µ‡ªµV­Zë·ÖÞ€0BÈ;ÅŒôÌ”=/ ”ÿœêÓ«w~û ý ´;õ:dx¯Ç.FèA´cû 8hæÔ_¡ÝÚùC†§fÌùäƒ4„ðnhœ:oòB|ŠíqS,öj¥ Ïê"DÒ¦/œ1o,™s ®óû÷̘¼h!’aCùû m˜1wÉô™i𼩡Œº™Ó&EW¸6@ÿ|¼ì™pÂৠ ×Q»™ó{2ëÌ,èšxàÜS'wm×φ±N4Ì›üäBñ#ü5\[ËéŸ?yÞ´¨ëÙÀoÏ(8÷Ø‹»ó ã?ï/|tÚÂ.'=„äHÄeE&Ô->òü‰æn7PŒ‚øïÂ/”ðã;? GˆE g¥h*üÏÉóX4ôs–7aѽ¿Þڙɨ µ6AT= q¶ûÎâgøU1S¬‚¦'x¤£é8$zIG!8Þï¿¡Ó{!?ŠEË%;³ã­ò<üå$„/ݹ¢efëœ:á,`¤aP'Pê¡x¡äÂy œ¯2Ú»‚÷ãMPŠ \…sÓ¡¬†rÊ(ëBçÊCçK¡¬‚²"x?ÞU®C‰„âãý‡îç}œ€ÒÚ?oŸ -ÑÊëCºwqèÞÅ¡1ù¹T(›‚ôkÏØxÿ!ÚÏûÄcZyW£·Ñq´ ¶ãZÙ¡fTæA}ªGëðlÔ„VA‹kÝ@_#ºspB±ÃkGõ…£IÓ =€ú¡Ñh<šˆ–ß¹Ãu u@@ã&¢ÉwîÜùò÷í~ yS–DŒ:ÄZö_¿¢}þ‡ -cßíMép_3Ö"ÇîCC÷—ÄÖݹ3´PˆGï£öQŸ²OðÅ_þ£‹—S: Z[‡½{…ºí=©œ^UÞ‚Óp¾w/íuŸèƒ¿~“öÅNû´åéø.O[¦uI9HŠ±cÿæñbÏ(ì@ˆÂ¾ ØŽÔô½ Ù`oÕêmoF[aoÒêÆýß÷{ú°-‡–¦Œb=Ê€½ªõ§ÓîR@¬ËZ]Ò §Ú¢ÁþÑŒ2F[–Ó£·½•A;L]N¹Y&þÂè/G…›7F‹7ËèÍbáÆõñÆhzÃ/\O ÿþ9Uü÷-ús*ý/FbôÇ zÍN¨ Í@b3£ÍuwÎúïß÷¡ß]-¿« W‹è·Œ~óu¤ø £_GÒ¯½2‡~É覗ÿå/ߢÿrÓKô F/2úùgñsF?sÐ ôÓO⧌~²^/~â /§u¡MÐhêBÏ3zîCU<Çè‡*=Ë茞yÚ*ž‰¢ÿ §ï3ú^m,õ‰ŒžfôÔrú.£'}‡Ñ[b£Ç=ÆèÛŒ…þŽÚé­ÿÇa±žÑ¼5^üÇaúbá­Ã>ñ­ñô-¿pØG1z°‚Ö•÷ßd´µ·èÐ×F_/¢û‹èßMtŸîeô5æo¡¯2º‡Ñ¿Ùh £¯ì6‰¯dÐÝ&úò.«ørÝe¥})EüërúR -ý £;}‘Ñên±ºˆîxÁ"îpÓ,ôy•ngt ²Ñ­FZµ¥£XÅè–Ž´Ư¬ Ï+}të¹Ãô¹baó3>qóxºÙ/lbôYF7B{ãaúŒ–ƒ0Ê{Ò Àí;]¯§ep¢¬ˆ–‚ÐJ}ôi+]ÇèZFK]³Ú*®atµ•þ£«}Êš/>5œ®d´øIºâÏËÅŒþy9]î¡bt™‰.eô Fgtñcq±™.®ÃÈAxÌ@;*,²ÑE~áQFat!£ æTÐùó’ÄùÃé¼$:—Ñ9t6£³2èÌ[tÆa:ÑiŒ1:uŠGœÊèd§xèdF'1:‘Ñ côâ_DǽKÇBc¬ŽÑSÐèB;ÅèHFŽt‹gÐŒgt£-§Cb§ƒ„SÄAŒE´Z‡ioí'zåÐó­âƒ6ú`ñûuB~O³˜o¥ùuA«§ß$ö4Óžuø(´üyÑo¢þ:\ ­<ƒNÌ3м:ì÷ =í$t¿E»1Ú5‰va4œ[D;§GˆÐF³Sìb6£Yh§´±Óš ‡LF3àÆ FÓárzM‹ ©PKuÑŽºp±ãašÒ!LL±Ó”:‡í`±ŠÂhNn…ü€OLfô¸ómOºˆíMb4‘Ñ3õ…狾޴™Æ3g6‹qŒzcSDïr›BcPŒìa4šÑ(m£‘0+‘nÁ¨›Q£NèÁY@Ã)bx>uØ-¢#…Ú-4 î ³S -ƒŽÃ~­§…8ÏC/á× Çe0æ´‚À‘ ƒžß×Ñû¨¶Mè%´êœ²U@ÿEt•¢ëh ¹ŠÆ@ýà//8\Sp Ü„nBO5¤;™÷€Þ¶¢­x §EWÎÐe±‰$C¯€„¦ íb“¸…ËŽMâOp¡h©N²ËñÀ—Û.üN'ƒÑ9x~AÇÑGègxµ/Û–Õ‰àqZŸÕлdkfÝ.]fÝš¿Åï|y¿ÃóB»H!ôÆŠû;XŒzSú¼h5<¯bS…M1¨T'Y!‚&Èmô:·Q²i&ús(NÍ(pqð3 ü´Ó°(Ê>Ž˜èóåˆÖL«OHf[ñô\¶oÛ—‹§³­¹xè"¯UÌòeZ¹®˜pözgÞ­¬ë«©†¯œ-Ðk<ÌmzÈßEùDIt¹#¨3Ò'Ib¾Åú²±Ò^! J‚,*ÁªÇg¡í¢-éèQ°‰¡á®¿´Œb«ItfŒŽóÁÈYÙ('»ÕŸdeƒ¼n% /&¯Æ®¬¢‚MÅcß]8ãääÏ°~tQ禚šš¸c¥•C–oÈð½ôŒ«ÿ˜tô±ž<ÁÖ.æ½I¨ÐVéPKu»Œ•RLi쮨Êø -©ÊñJûð0DínO‚ÅCãb캘ö@/|¾Ùª-ñfPM®œÀçþM€ä®?äûâ³~w‚mDSeÓvöý͘þΔ]û÷oÙºµtûÆ5£ëg.ùG¿ X\GcO>÷ÏïÚÊêT±þ©ª]Kç-Z–”t(6ö³7–½ÄýÄE˜C0߃Ԉ(5æ#ª—+EL×ê°AEĔv&K À‘抮ðš-ä„S âSÜÝ"ãˆ,”7‹\bíñÇÞïNž<ÕR"F¾§ï2_bÛqÑ.¹r\ -Œºûã9:¢R¶–Z6Ø+¤­5VÉ5êANVã%Æç¸!´’¯„¤Å€‚0˜U+— rØѽbi!×[’Guø[Ø—¿ýtUeÙº-#¼Vì?{vÿŠ^ï»;N_ùòTõ)<åÉ?ÿùÉ¥Å+¹î¯Ž -4ŽÚ¡®þ¸0 U†ÿÍb,5l°TÆI•QqU¾0‰â˜8ÇàæS|àls«+ý»¡Íp+À ÕtÏfå8‹ÌXŸmÀª#VnÚSYr™Ïô‰T>Óÿþ ?³‹ìû~›IEë4¶¬ƒ©Æ^ì.šƒÕëÿ…uìi¶€•°E1@sЯ -‰ZLé7’h°Sq´€ :pów=èd­!G‰ì,Ëw =hþzà¢Ï¦€ÿç `JeÀð€{`ãXà -ï!8@yo$0‚ï¢z’å+¹­–O€Þyý6Q€àM¤¯I¯Ê˜ $!‘+\††dšÀèˆgaaIà5:ìö%ÁKßfGÙñ7ñÎ]x'Ÿ•B¬V8/–Âðñ°ßç´´…Â!»ƒò~O}÷cz¤®š7xÿ” Gfºøà„ÂÞ 'ñÆ­SÖ ݵ0}ôüÞ½ÞÊí|ôÅ«ÆÌÑ=•ý Q>Šdo [ÅE° -:¡…þô¤ävNj2è";ѽNÝ^ƒ5±ÝÞd§gozrV¢QJ2 )ÙŽ"Œ vo²!¥c,¯°n ­lhÎóS M ÚBoã6Ío,‹¥b¹8¬Ø^ì8^ ­XäañY|Õ´Æ%³H3•Ù9N¸lØ‚©xÎœ•+çÌ).{Æ·jØ‹çν8tM»}o²‹¸?¶eÏÍMË[Ö…mcãñ“¸ÛÙ[Ä»rÛ¶•OmÛƾ]×ëç·Þú¹ {ÞözÀ3Ù›l©ÝQâváq^ƒÿÉnþ d⃙ߣyEmñ÷D>€º Ö>Q¡H>Qt>ª*P°O¥*òðš*xM¹ï§)‚ªHZC'¦ªzQ0 ªÕe*šÍ•ƒwÕ°ÃûeŠñx„ŠÌ¨6b=¨AÐÉ’²¯#Êx4S¯Œ½4žÌl‚éï7ð©³ó[nÌ=+Æ·ôµ[Éx5[Æm÷.àçàGx$–û37ªTu•¶µ¸Rý{ŒUïŽ SˆˆLžp1#*]‡<¶4/_´!„¢ùŠf ¡ðEÈ ÓÖŠÏ{—Ùözñ#?î«xa÷ß?»jåfÖúê×U«6½Ìn²ßX9ÕrqÙúgבé¬ÇÂåízçu/ØÃÏT7òX ,r:¬87êèp¿Žö‡WRãë†ý–J]­ŠË0 t)-‚“ö»£;œæû]?BmûfçéƒvŽe—Ùqœ‡£ÇîÔ¿f䉆†…¯ôËjßWà…x>®jßþLw?@Þ÷Ù?Ù~¾ö×Ć…$–äwH•ºµ¨Ò,!³N¦†tÙcI³wÅF×—‘mµ$xã­aAZ¼V|àÎéSwФ"<÷bo±vv×mܸ}û 1•mfÅ€©Ÿƒq0:#†šµ÷‡KM£µJ¥ðw‹ºt€P¦A›’††¶Éh³¼œ ·Œ Z%GOpdSPÓò¥†¨Ág…гµó‡…zþ&U붓ì6ÈÃÝÆ_¥ ZÆ‘i-Ûy}kZr´xíKa(ÌQ4êæ÷º#P8¨Œºgž<‘ne‡eHY.£Þyh²Øù`Âë1cÖø,¯0ô¡ÆÇÙÖ€ý8zž¡ÕL3¨;ÓvŠ5ÅxñÆ]‘¿}5IB\¢T’¿ h­*¯bº‚Ë´ 5WÙí -TÒ $áÎöñëIûÆÆ–y.t;)º•LN¶är)ž‚^%­Wðšª6;B%xÞê‘QŽÎÀ³ÁNyø‘Ñ&Æ,ØŸ¢ÝÙIœ8ŽsÙIåí55ÂJ¢õ”Ök¥ß«øt -„;²E$”¢ªÄ³¡þpËÑ )á Í’´ EàFc ,†‹N%ALP²Iy˜ŒR -uEd©¸DY¦3²ˆ$"ó›ÅR²œ t¡ýh©<–Ž ¥By”2‡.¥OÊN03 03 eM8OüÐr˜Œ¼ÎÜ[¹„f’--/Ö“æ—Zꀛ›EZöÙå7hj†=4u’,lJ½”µÃûq#9(›B1ßHxn L<ŒIZã’ýf£ñþv¢¢êœØEŠK—€ÛÓ¥½.çÒl¥³Îd–aS)„Ña^d@€kküHˆU @góav³ŠÝ8l+ä×[ÉbbàvëÓî.ÐVØ< L  Ìçd€ Å2± -é²¥©­&#Hoî]Ñ“À©tb>ƒ Nâ’Û“¹³<Š‘Ùòb²TV],à~Ò(<Ï”¸ :Aoa_ŠþõøÄ”ãbÓíLáý[ÉÂû·3Á’-†õ:®5WâÓUši¥£Â\åB†l)ÖåÒˆjµi¸5O06ì®:]W²µª¤¤jkɹ_ZZnþhù…\Åýpû¼hû»q?¶”ã?㵸³b[µÅš CÉ~®´ 0¥6‹ª‰ÆÎVäÑeÚƒ+28‘Zªßå\ @D‰Ë¿+{~ûz6¸… »sû»Óbj˙ͫWoÚõågÿ³e7Ç‹ÙÏ¿V-®ŒE‹ÕYIši²z¬.‹ÁŒ\6`ÞÕj øŒXµ«¥AB®–˜ã¤ø»r(²S¾[¤\ŽPnýÂå°®ôÝJK«t" .$¡ €-aXUßú"ú˽’âïù7P ™0…fû;` ªä£}‚@ó%9¨TC8¼V/ˆÖpDUjEQ&ªº3kº]õÒ¢ƒÆ›‹Îšk e‰»qÚ‚L´Åw¦˜ wjïâµ]Ë#´ ›ü:xþ°ÆO÷Z6õíF\}K/¶|pqã†ÊgÈÛö»ØL¼äÅñ-ëĦ/>Ü|˜Œn¹öÔš•ëB~»?È<ý‡¿»Ñ@LzŸ'ƣ般úbb<ùªÞ#`Çëö—]•Vü:zY¨ôUX«’<ª>&RF ‘¹¦t»œ—–ÄcÓ@óþ²@ Û\ü›êæFð%£ÊH†eFe‚92jËi;ÿs.%1‘ÇšícÒëãN(ß5ï¹Ùe¿fŸ³èý™U5‹7Îýà-l¼4²^Üù~×n«™:3Þ•þáMÿJIù¸_¯uË>ëN­¯~÷¿÷T˜Ç02Šò›¤ Ñë~µ‹Èß‘ié¯6wO±”F– ^àV¬1ÚÒë®åAÚûfjCÖUºª,x­Ç챦°üž¶5 -¾…‹" Ö½¿ æë³x“'±8§ › æØ%#G=é5Nçä×ÌžUó`ï.¤;Ý0ÌïîÚ­[W÷¸¹ôf`æWoçuíÒ¥kœptÂ-0ÐßóüòI€lIkÈ7‰øt QIÓxxç ¦ÝîR7å‡Vˆ;:ÎÇq*Î$3ñijlÀ 6àl˜ ´)ô¥¶ 貿x?ªTUõ*˜Q½Ö„ a—OE&>J È€}ȯ‚ƒT¨^A1ðÐ,øÙž^F©¡W†nmþà^‚ਹ™{ …Û]—Ž+Ñ„#Ĩ<›¬ò$pgu #êúòõ ˆIJ²Î§O0d‘.4Kè"vUºè:é³ #Ð<†ÒQÂHq¬4V¦ŒÒêÇf(3 KèãÂÝãúxPLkfXÿ£ñ”ÆŸ(¯9·§üÄ¥#§€ÍîE㟠õ·’龦l —z ¼ã ó<„žZQĶ>-.ÁDäJôHÏ?#Öçë?à~t’¤Ø^è*¥¢®*ÇWW¬÷aËýº -÷çî@;EAïqm/&è õO’%âz=ÓkVÌŠD(^ˆ“’Ÿ> °Çh¥P?,"îTEh@þF~ - xð§¬'¾ô){–-¾€¯ƒ[}Ÿd¶ä®’U-Ë©“ëFk¦ ÑþŽ÷óª …óªäs7 €†Š–‘ÐMô n²N[{¡Èó©œA²`iú>ÆsñœO™—›ydSàTËòÂø û™>!Ùa…{üf¡Vz“Ô¢7>Óî<%ÐR-éâuÄ[3^|ïþé'V(ÙË ”ßÿŽk°ˆ ûÞqpÁø¨­>ðÉ!mâo\ÆÀȤudZ‹ßùȨ»Dø ø- -Žlõf€Í¢„þô“dÿõ‹rIà#k_I3þÎÄüªÃ"©ï…ÞØ4ÎÞNeó†E¡Œg¼°ŒÇh¾ÿ°¨±—YFµ.¹Ìæ-=]Ö®ÞVå2b—à6©’¡W¬ 9z<`¹ÎMvFðMY*8ÐÀõP¾Ïš -û£ÓbÒbÓ¼iqÕ¨W“jµZ¿3¼ÚYíªvWG˜Æã¶@N‡ÿmi€Ì¬„»2`íH}÷ÿ²tî–×ðÁƒ]_/þÛ{·ÿý ^³iÂѱÓ–žèžK2Y8má¹Cí¶¬ÜU4ñí‡E¯Y’Ý©.1qØ°ŒMœÓ·S˜à4 -¥ù#\-3ëÊUæúÈmnd³õq$)¢ 8ËȾÀÐ`•'-ïÊX`-§«½XIHäSO+Ö®^óôÓkV¯mù²Ã¶é§¿½ÚXTÕ±®Ž¤~t᳦óŸ_ O+„PåGö=;1jøzþ -FÇHÞRý†RãQ *u /£–2]={m$D€–´F½ž0ºñß…½­r|ýŸÆì3ìÃÂC[÷Û4þÕC‡÷LÚÒ379wÆvØ:?Ðáíž]þõÁ™Ë]{p¹,:l@’%(Ù#ÊLö2¥ÊT·Q§QC«Mß—Ë…Oø½rá”6sA|~÷k}:¦®®cUQãÕïNOÛÊÌkW¯.-]½z-=Kü­yýðQ¸+v`+ÎÅô]øü|ÓgZé)¹8€žÞþˆ4±Zª[/…×b©Ìˆßv—…Õ«¢)‰²èÂ%Ô/Êf ÑÖüØ€‹ê -WEËõkZÐ :’£Ur0rPÉ„ÂÀ—G^+<:oÖñ±ì6»€cúè—:aãÚÕ{,dÂéÍw;ç¾ ÂËÅaØ€ýì‹;_Ù·ÏX©D[êR³`ƒR+á2´Í$SI˜Œt¢d4ëí<{ >)ôýMO­fظä|Ú!|ååŠ/$ž:¯d+êXN~åuÉ^õÐŒ©åTz¶|pÝkštX_ÁZ¯}ý‰nƒ'2QGm¥aáº2Ox™™Âjõ˜'{ Ñ$CÉfóö…ÅÚš;Ë•+ Á…jiâ1yîýÚf'¿ã¦î䮩ufeÒ1%+K6”¯]QRwí›»†O©~𹵶Ìkø¹©u$÷ÌÇŸ9óév1ÀQ‘µ;ìܧ,Ÿ0wÁ2Vp—‘c6¢ì"5Ùuð»t&Ê¿0/³Ö¶Aô'¡þFð_v¾µeØ­YC?¹Ú÷ARµ4·f„ȃ³YYZW—¾{Ñß^!µ-ýImå†7ÿÖR"Ù[^š4õ2—Ú~¯Œ§CFÔÑïVj‘¡A݆ŽI¤V õX¢ªl‚ɺÒè|yJƒ·…ó8ÓñÖÇx^`>žÇ.㘺:aR µ¼œæ“yŠb#©Æ1BÒÙ-"¬–)¸–ZLÛè1å¸M¯1 ’ _ù74„TV,¸¦D®€ë‚oÿ´µ -Ùq|ãõ¿²ïÆÀ’²Ó¯Êo—?·}¹ô׫\šÍÀÝ~ͲðwQœAœàœ s‚¿tÊ=‹N €œ€ê‚I†ch[(Ë Ã’Ô õ6€¼›3Z!•û ü mp™c§aQ”pâ]Jé$f+d:YLÁ¸*Š¬‡KtK²KI¢‰b’Ü•v˜+ç(ýi?y´4ZžCg ³¤Yòú¤¸DZ"GÝ—{h®Ç_þ‹õÁì2›²®^²–á³lLKÉ_Éò€÷jà]•b@â å"ªÖNÐZ€*’ôuü+‘À{ÁTðißHÁÓp1“cG|õ¯o°Oñ%\ñ§?o?Œþö ô;¿!…u‡þâOtÙÌ:AFÑ‘’ì0”ÅÒúÈcn‹Œ¬fe4Ø:È<8Ê5(¢w¼åú€}†öYGŒ-<ˆ"îí<:Ð-$\¨Ý®\×^£ò¸Äëï“&¤‰iRšœ¦¤éÒÔ4}^xž3Ï•ç΋ȋ̋ʋÎóÓb!”Ë.µº°}Èþ4êNÁ1؃;òµ[ÏF -#`\î3ýQm~°ÞT…ÓcÑàûhÞð.„ë¶ÍúÚ¸ aƒ{0ÃrWøÍ·§§WáŸJ‚x¡d}Ë)I->Š½Ã¾xpj¾‚ œ¢ul˜#<¡! ¨ ±Ô›«"»Ekx¥ —¿ûæ»)ºÛã{ ¹5,Y½zݺիKHDJÕ´SW¿=  åàA’Ì)_ÜòôðÑ8üs8î2jxùo7™ÎMX2²¢!03²ž"+­4ÕëŽÉª$!lZƒ– οÇ=Íï³ ¶XcQ,Žµ¦¡4œiI³ú‘?hñ[‡¢¡x¨e¨Õ6'ܧ)NaU×¥ýöÔ‚ÜŽ¬êõx6]ÒñOÞk9'Lúì‰qí¸^6¦H@M"Zì÷kùg(ýàôÄxò£=.-á@¥ø¨`/uu•Y…2_½µ*):˜‚)™úÉ’=®wRÐEÞ“‚°°o…>Ò¼+ ÑšT¹/á ÝpH%­ï¶éûCª‡/_6îÍþëÖ78¼vöŒ·F,]sCé½ãÙ §ÇìrtìøÐðýãMÛ—ï:_Ÿ•5utq:1ÅlZñâ^/pº -䮈/À -L÷»L¢Q²âcJ­ªèu”Òb3Ý‹4š¯wkÐ>cãA`Œ{¬ž•LÇ;Ù„ ‹Ï\>Sá†ø;VÞRý§)›v5’Iå¸GŸF -î[õU*>fà+¾?,}mճߡ EkŠ‚‹  &¯›³ð©§Lå‘=»ñn¾ìù¢'KnïÜ=¹è2ŸË"àpž0 Ihš¿=ø öœÁ˜|˜„$ ‘¡Äc¾c"·‹E¡ê-kosµÑÿ ÌÛŸ¨°&ž(R¨""ñÒ–ÆÁ½qÁöèàÉGÐ=·«[s@‹[ý½õà6UÊ3ÓzÈÒM’“''œÁä„3˜œpÞ•œ8öGɉ뜜f%Z“m¹ë2½NÐÇ M"h”à£t‘j„>Æ€p{’ ¶—ÚËñJC.És¥\9Séfè­ôÓèûF*…ºQúцéd.Ìgë¦é'KÅ'•?éÕ§˜U7áÉ·£Æ¨É$YìAzˆÝÔ¾êhu6™%ÎT—‘t‰°BX*þI]¡º@n­¹ ÌsµÇ«Ÿßy¼–ÝÚ÷úþ} À'H~K=]w»šLoÙÊgu Hr5HR^ñ÷ "K€@™ l'˜: ȇK«:u*æ½ -²Õ9A²ùª,`A'¡é§H§4B8åÊÿy*CÖäé´#OdzäÇñY’1Т:Ô.B'uŒ0RU€CÑrØ\×ÄêØÁ&\Ǽ‡ÛãDaRË·-uøm–Gú›ƒŸãœÎï±8µ¡jo£Ódäj£':G\V§ ðžÓlAü`µMF³Ód2æ[ŒȤ+“[MǬ³‘g4mŠ­Ÿ©w°ƒß‰ik+÷ž´î½œj©Q –u¶ioËã¢X›Ûà6&˜Ì­ý¬ýl£*Ï“J2á9C ^æp>×wì35®ãÓã6Ï{h\6ä8ˆûÇ3a¿v”|( иÎÇ_O-ÐúÌ*2QÎ=Ï·>Ýú, þc§;Ÿ 1÷ÿO¡—8¹sG»S»~¡dA¹v ý? Ú‘_1´=!ÁÝF8gÙa ¢ì{ ?ìóaÃ@Ñ$ØógT˜½ -Ž€q–;¡~‚×ùs’v=aò£“§ ´©K‹Òf<:mJ›;yA,ß?6æðÞ§µßÿÞXœ -endstream -endobj -1062 0 obj -<< -/Type /FontDescriptor -/FontName /LXRMZQ+DejaVuSans -/FontFamily (DejaVu Sans) -/Flags 4 -/FontBBox [ 0 -14 557 560 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 560 -/StemV 80 -/StemH 80 -/FontFile2 1058 0 R ->> -endobj -1063 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /LXRMZQ+DejaVuSans -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 317 ] 16 [ 360 317 ] 20 [ 636 636 ] 25 [ 636 ] 36 [ 684 686 698 ] 40 [ 631 ] 42 [ 774 ] 44 [ 294 ] 46 [ 655 ] 53 [ 694 634 ] 68 [ 612 ] 70 [ 549 634 615 352 ] 75 [ 633 277 ] 78 [ 579 277 974 633 611 ] 85 [ 411 520 392 633 591 ] ] -/FontDescriptor 1062 0 R ->> -endobj -1064 0 obj -<< -/Length 772 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -32 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0003> <0020> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0048> <0065> -<0051> <006e> -<0046> <0063> -<004b> <0068> -<0050> <006d> -<0044> <0061> -<0055> <0072> -<004e> <006b> -<0010> <002d> -<004f> <006c> -<0049> <0066> -<0024> <0041> -<0056> <0073> -<0057> <0074> -<002a> <0047> -<0058> <0075> -<004c> <0069> -<0047> <0064> -<0015> <0032> -<0052> <006f> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1065 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /LXRMZQ+DejaVuSans -/Encoding /Identity-H -/DescendantFonts [ 1063 0 R ] -/ToUnicode 1064 0 R ->> -endobj -1066 0 obj -<< -/Type /FontDescriptor -/FontName /ENPKVN+Poppins -/FontFamily (Poppins) -/Flags 4 -/FontBBox [ -27 -132 753 704 ] -/ItalicAngle 0 -/Ascent 1049 -/Descent -349 -/CapHeight 704 -/StemV 80 -/StemH 80 -/FontFile2 1059 0 R ->> -endobj -1067 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /ENPKVN+Poppins -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 266 ] 5 [ 291 ] 7 [ 621 ] 10 [ 158 453 453 485 ] 15 [ 197 550 209 475 627 319 574 588 628 627 634 545 630 629 212 ] 32 [ 722 ] 36 [ 673 612 771 706 512 503 777 691 245 ] 46 [ 598 431 860 702 785 578 787 607 586 540 674 675 975 ] 60 [ 583 540 422 ] 64 [ 422 ] 66 [ 732 256 675 675 606 675 619 328 675 639 245 247 514 245 1029 639 639 675 675 372 521 363 639 560 819 478 562 454 ] 95 [ 290 ] 204 [ 218 ] ] -/FontDescriptor 1066 0 R ->> -endobj -1068 0 obj -<< -/Length 1430 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -79 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0003> <0020> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0048> <0065> -<0051> <006e> -<0046> <0063> -<004b> <0068> -<0050> <006d> -<0044> <0061> -<0055> <0072> -<004e> <006b> -<0010> <002d> -<004f> <006c> -<0049> <0066> -<0024> <0041> -<0056> <0073> -<0057> <0074> -<002a> <0047> -<0058> <0075> -<004c> <0069> -<0047> <0064> -<0015> <0032> -<0016> <0033> -<0045> <0062> -<005a> <0077> -<001b> <0038> -<0052> <006f> -<0030> <004d> -<0031> <004e> -<004a> <0067> -<0029> <0046> -<0017> <0034> -<0033> <0050> -<001a> <0037> -<0018> <0035> -<0013> <0030> -<005d> <007a> -<002f> <004c> -<003a> <0057> -<005c> <0079> -<001c> <0039> -<005b> <0078> -<0027> <0044> -<0032> <004f> -<0037> <0054> -<0053> <0070> -<000f> <002c> -<001d> <003a> -<002b> <0048> -<0039> <0056> -<003c> <0059> -<000b> <0028> -<000c> <0029> -<004d> <006a> -<0054> <0071> -<000a> <0027> -<005f> <007c> -<0012> <002f> -<000d> <002a> -<0020> <003d> -<0042> <005f> -<0043> <0060> -<0005> <0022> -<003e> <005b> -<0040> <005d> -<00cc> <2019> -<0007> <0024> -<0038> <0055> -<003d> <005a> -<0034> <0051> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1069 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /ENPKVN+Poppins -/Encoding /Identity-H -/DescendantFonts [ 1067 0 R ] -/ToUnicode 1068 0 R ->> -endobj -1070 0 obj -<< -/Type /FontDescriptor -/FontName /DYCYPS+DejaVuSans -/FontFamily (DejaVu Sans) -/Flags 4 -/FontBBox [ -9 -92 733 729 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 729 -/StemV 80 -/StemH 80 -/FontFile2 1060 0 R ->> -endobj -1071 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /DYCYPS+DejaVuSans -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 348 ] 11 [ 457 457 ] 16 [ 415 379 365 695 695 695 695 695 695 695 695 695 695 399 ] 36 [ 773 762 733 830 683 ] 44 [ 372 ] 46 [ 774 637 995 836 ] 51 [ 732 ] 53 [ 770 720 682 ] 57 [ 773 1103 ] 60 [ 724 ] 66 [ 500 ] 68 [ 674 715 592 715 678 435 715 711 342 ] 78 [ 665 342 1041 711 687 715 715 493 595 478 711 651 923 645 651 582 ] 5038 [ 741 ] ] -/FontDescriptor 1070 0 R ->> -endobj -1072 0 obj -<< -/Length 1182 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -61 beginbfchar -<0014> <0031> -<0016> <0033> -<0017> <0034> -<0018> <0035> -<0011> <002e> -<0003> <0020> -<0028> <0045> -<0051> <006e> -<0056> <0073> -<0058> <0075> -<0055> <0072> -<0048> <0065> -<0057> <0074> -<004b> <0068> -<0044> <0061> -<0024> <0041> -<0033> <0050> -<002c> <0049> -<0059> <0076> -<0053> <0070> -<0052> <006f> -<0047> <0064> -<0046> <0063> -<004c> <0069> -<13ae> <00660069> -<004f> <006c> -<0050> <006d> -<0019> <0036> -<000b> <0028> -<000c> <0029> -<0015> <0032> -<005a> <0077> -<001d> <003a> -<004a> <0067> -<001a> <0037> -<001b> <0038> -<001c> <0039> -<0026> <0043> -<0031> <004e> -<004e> <006b> -<0049> <0066> -<0030> <004d> -<0013> <0030> -<005c> <0079> -<0010> <002d> -<002e> <004b> -<0045> <0062> -<005d> <007a> -<0035> <0052> -<0025> <0042> -<002f> <004c> -<0036> <0053> -<005b> <0078> -<0027> <0044> -<0054> <0071> -<0039> <0056> -<0037> <0054> -<0042> <005f> -<003a> <0057> -<0012> <002f> -<003c> <0059> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1073 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /DYCYPS+DejaVuSans -/Encoding /Identity-H -/DescendantFonts [ 1071 0 R ] -/ToUnicode 1072 0 R ->> -endobj -1074 0 obj -<< -/Type /FontDescriptor -/FontName /IJJFHS+DejaVuSansMono -/FontFamily (DejaVu Sans Mono) -/Flags 4 -/FontBBox [ 0 -155 598 728 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 728 -/StemV 80 -/StemH 80 -/FontFile2 1061 0 R ->> -endobj -1075 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /IJJFHS+DejaVuSansMono -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 602 602 602 602 602 602 602 602 602 602 602 ] 15 [ 602 602 601 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 601 602 602 602 602 602 602 602 602 ] 46 [ 602 602 602 601 602 602 602 602 602 602 602 602 602 602 602 ] 62 [ 602 602 602 ] 66 [ 601 ] 68 [ 601 601 601 601 601 602 602 602 601 602 601 601 601 602 601 601 602 601 601 601 601 602 601 602 602 602 602 602 602 ] ] -/FontDescriptor 1074 0 R ->> -endobj -1076 0 obj -<< -/Length 1570 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -89 beginbfchar -<0031> <004e> -<0052> <006f> -<0057> <0074> -<0024> <0041> -<0053> <0070> -<004f> <006c> -<004c> <0069> -<0046> <0063> -<0044> <0061> -<0045> <0062> -<0048> <0065> -<0058> <0075> -<0050> <006d> -<0047> <0064> -<0056> <0073> -<0055> <0072> -<0003> <0020> -<0010> <002d> -<0051> <006e> -<0020> <003d> -<0008> <0025> -<001f> <003c> -<004b> <0068> -<0012> <002f> -<0049> <0066> -<0021> <003e> -<0038> <0055> -<001d> <003a> -<002a> <0047> -<0059> <0076> -<000a> <0027> -<001a> <0037> -<0013> <0030> -<0054> <0071> -<004e> <006b> -<0042> <005f> -<005a> <0077> -<0011> <002e> -<0006> <0023> -<0004> <0021> -<0037> <0054> -<004a> <0067> -<005c> <0079> -<0007> <0024> -<0014> <0031> -<002c> <0049> -<0033> <0050> -<0027> <0044> -<0035> <0052> -<003e> <005b> -<0005> <0022> -<005e> <007b> -<0060> <007d> -<0040> <005d> -<001e> <003b> -<005b> <0078> -<000b> <0028> -<000c> <0029> -<002f> <004c> -<000d> <002a> -<005f> <007c> -<0015> <0032> -<0009> <0026> -<0019> <0036> -<0017> <0034> -<002b> <0048> -<0036> <0053> -<0028> <0045> -<0030> <004d> -<0032> <004f> -<0029> <0046> -<003f> <005c> -<0022> <003f> -<005d> <007a> -<0016> <0033> -<003b> <0058> -<000f> <002c> -<0025> <0042> -<0026> <0043> -<001c> <0039> -<003a> <0057> -<001b> <0038> -<0018> <0035> -<003c> <0059> -<004d> <006a> -<0039> <0056> -<0034> <0051> -<0023> <0040> -<002e> <004b> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1077 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /IJJFHS+DejaVuSansMono -/Encoding /Identity-H -/DescendantFonts [ 1075 0 R ] -/ToUnicode 1076 0 R ->> -endobj -1078 0 obj -<< -/LXRMZQ 1065 0 R -/ENPKVN 1069 0 R -/DYCYPS 1073 0 R -/IJJFHS 1077 0 R ->> -endobj -xref -0 1079 -0000000000 65535 f -0000000015 00000 n -0000001112 00000 n -0000001250 00000 n -0000040655 00000 n -0000040789 00000 n -0000043487 00000 n -0000043686 00000 n -0000047426 00000 n -0000047832 00000 n -0000048037 00000 n -0000048244 00000 n -0000048379 00000 n -0000048515 00000 n -0000048673 00000 n -0000048832 00000 n -0000049065 00000 n -0000049299 00000 n -0000049514 00000 n -0000049730 00000 n -0000049971 00000 n -0000050213 00000 n -0000050436 00000 n -0000050660 00000 n -0000050892 00000 n -0000051125 00000 n -0000051339 00000 n -0000051554 00000 n -0000051781 00000 n -0000052009 00000 n -0000052218 00000 n -0000052428 00000 n -0000052657 00000 n -0000052887 00000 n -0000053098 00000 n -0000053310 00000 n -0000053529 00000 n -0000053748 00000 n -0000057509 00000 n -0000057974 00000 n -0000058175 00000 n -0000058377 00000 n -0000058592 00000 n -0000058808 00000 n -0000059005 00000 n -0000059203 00000 n -0000059422 00000 n -0000059642 00000 n -0000059843 00000 n -0000060045 00000 n -0000060273 00000 n -0000060502 00000 n -0000060712 00000 n -0000060923 00000 n -0000061138 00000 n -0000061354 00000 n -0000061585 00000 n -0000061817 00000 n -0000062020 00000 n -0000062224 00000 n -0000062361 00000 n -0000062499 00000 n -0000062693 00000 n -0000062888 00000 n -0000063078 00000 n -0000063269 00000 n -0000063460 00000 n -0000063652 00000 n -0000063844 00000 n -0000064037 00000 n -0000064276 00000 n -0000064516 00000 n -0000064731 00000 n -0000064947 00000 n -0000065154 00000 n -0000065361 00000 n -0000068935 00000 n -0000069413 00000 n -0000069609 00000 n -0000069806 00000 n -0000070002 00000 n -0000070199 00000 n -0000070398 00000 n -0000070598 00000 n -0000070798 00000 n -0000070999 00000 n -0000071197 00000 n -0000071396 00000 n -0000071630 00000 n -0000071865 00000 n -0000072064 00000 n -0000072264 00000 n -0000072467 00000 n -0000072671 00000 n -0000072873 00000 n -0000073076 00000 n -0000073276 00000 n -0000073477 00000 n -0000073673 00000 n -0000073870 00000 n -0000074059 00000 n -0000074250 00000 n -0000074442 00000 n -0000074635 00000 n -0000074825 00000 n -0000075016 00000 n -0000075202 00000 n -0000075389 00000 n -0000075601 00000 n -0000075814 00000 n -0000076029 00000 n -0000076245 00000 n -0000076458 00000 n -0000076671 00000 n -0000080377 00000 n -0000080880 00000 n -0000081082 00000 n -0000081285 00000 n -0000081487 00000 n -0000081690 00000 n -0000081901 00000 n -0000082113 00000 n -0000082334 00000 n -0000082556 00000 n -0000082785 00000 n -0000083015 00000 n -0000083216 00000 n -0000083418 00000 n -0000083616 00000 n -0000083815 00000 n -0000084028 00000 n -0000084242 00000 n -0000084441 00000 n -0000084641 00000 n -0000084852 00000 n -0000085064 00000 n -0000085210 00000 n -0000085357 00000 n -0000085571 00000 n -0000085786 00000 n -0000085978 00000 n -0000086171 00000 n -0000086382 00000 n -0000086594 00000 n -0000086813 00000 n -0000087033 00000 n -0000087232 00000 n -0000087432 00000 n -0000087640 00000 n -0000087849 00000 n -0000088045 00000 n -0000088241 00000 n -0000092113 00000 n -0000092584 00000 n -0000092721 00000 n -0000092859 00000 n -0000093051 00000 n -0000093244 00000 n -0000093443 00000 n -0000093643 00000 n -0000093800 00000 n -0000093958 00000 n -0000094171 00000 n -0000094385 00000 n -0000094581 00000 n -0000094778 00000 n -0000094970 00000 n -0000095163 00000 n -0000095386 00000 n -0000095610 00000 n -0000095811 00000 n -0000096013 00000 n -0000096210 00000 n -0000096408 00000 n -0000096605 00000 n -0000096803 00000 n -0000096963 00000 n -0000097124 00000 n -0000097321 00000 n -0000097519 00000 n -0000097654 00000 n -0000097790 00000 n -0000097973 00000 n -0000098157 00000 n -0000098349 00000 n -0000098542 00000 n -0000102387 00000 n -0000102874 00000 n -0000103033 00000 n -0000103193 00000 n -0000103414 00000 n -0000103636 00000 n -0000103839 00000 n -0000104043 00000 n -0000104266 00000 n -0000104490 00000 n -0000104694 00000 n -0000104899 00000 n -0000105130 00000 n -0000105362 00000 n -0000105575 00000 n -0000105789 00000 n -0000106018 00000 n -0000106248 00000 n -0000106466 00000 n -0000106685 00000 n -0000106921 00000 n -0000107158 00000 n -0000107376 00000 n -0000107595 00000 n -0000107730 00000 n -0000107866 00000 n -0000108059 00000 n -0000108253 00000 n -0000108464 00000 n -0000108676 00000 n -0000108879 00000 n -0000109083 00000 n -0000109274 00000 n -0000109466 00000 n -0000109680 00000 n -0000109894 00000 n -0000113976 00000 n -0000114463 00000 n -0000114666 00000 n -0000114870 00000 n -0000115075 00000 n -0000115281 00000 n -0000115471 00000 n -0000115662 00000 n -0000115899 00000 n -0000116137 00000 n -0000116368 00000 n -0000116600 00000 n -0000116804 00000 n -0000117009 00000 n -0000117217 00000 n -0000117426 00000 n -0000117634 00000 n -0000117843 00000 n -0000117996 00000 n -0000118150 00000 n -0000118347 00000 n -0000118545 00000 n -0000118706 00000 n -0000118868 00000 n -0000119050 00000 n -0000119233 00000 n -0000119398 00000 n -0000119564 00000 n -0000119761 00000 n -0000119959 00000 n -0000120161 00000 n -0000120364 00000 n -0000120513 00000 n -0000120663 00000 n -0000120845 00000 n -0000121028 00000 n -0000124874 00000 n -0000125345 00000 n -0000125566 00000 n -0000125788 00000 n -0000126002 00000 n -0000126217 00000 n -0000126435 00000 n -0000126654 00000 n -0000126858 00000 n -0000127063 00000 n -0000127239 00000 n -0000127416 00000 n -0000127615 00000 n -0000127815 00000 n -0000128010 00000 n -0000128206 00000 n -0000128404 00000 n -0000128603 00000 n -0000128755 00000 n -0000128908 00000 n -0000129095 00000 n -0000129283 00000 n -0000129477 00000 n -0000129672 00000 n -0000129818 00000 n -0000129965 00000 n -0000130167 00000 n -0000130370 00000 n -0000130537 00000 n -0000130705 00000 n -0000130861 00000 n -0000131018 00000 n -0000131225 00000 n -0000131433 00000 n -0000133745 00000 n -0000134040 00000 n -0000134184 00000 n -0000134329 00000 n -0000134531 00000 n -0000134734 00000 n -0000134948 00000 n -0000135163 00000 n -0000135348 00000 n -0000135534 00000 n -0000135712 00000 n -0000135891 00000 n -0000140751 00000 n -0000141006 00000 n -0000141262 00000 n -0000141529 00000 n -0000141795 00000 n -0000142002 00000 n -0000142208 00000 n -0000145778 00000 n -0000146009 00000 n -0000146207 00000 n -0000146396 00000 n -0000150083 00000 n -0000150286 00000 n -0000154029 00000 n -0000154252 00000 n -0000154386 00000 n -0000158416 00000 n -0000158655 00000 n -0000158789 00000 n -0000158923 00000 n -0000159057 00000 n -0000162674 00000 n -0000162877 00000 n -0000167587 00000 n -0000167978 00000 n -0000168112 00000 n -0000168244 00000 n -0000168376 00000 n -0000168508 00000 n -0000168640 00000 n -0000168772 00000 n -0000168904 00000 n -0000169036 00000 n -0000169168 00000 n -0000169301 00000 n -0000169434 00000 n -0000169567 00000 n -0000169700 00000 n -0000169833 00000 n -0000169966 00000 n -0000170099 00000 n -0000170232 00000 n -0000170365 00000 n -0000170498 00000 n -0000170631 00000 n -0000170763 00000 n -0000170894 00000 n -0000175901 00000 n -0000176300 00000 n -0000176433 00000 n -0000176566 00000 n -0000176699 00000 n -0000176832 00000 n -0000176965 00000 n -0000177098 00000 n -0000177231 00000 n -0000177364 00000 n -0000177497 00000 n -0000177630 00000 n -0000177763 00000 n -0000177896 00000 n -0000178029 00000 n -0000178162 00000 n -0000178295 00000 n -0000178428 00000 n -0000178561 00000 n -0000178694 00000 n -0000178827 00000 n -0000178960 00000 n -0000179093 00000 n -0000179226 00000 n -0000179361 00000 n -0000184173 00000 n -0000184636 00000 n -0000184771 00000 n -0000184904 00000 n -0000185037 00000 n -0000185170 00000 n -0000185303 00000 n -0000185436 00000 n -0000185569 00000 n -0000185702 00000 n -0000185835 00000 n -0000185969 00000 n -0000186103 00000 n -0000186237 00000 n -0000186371 00000 n -0000186505 00000 n -0000186639 00000 n -0000186773 00000 n -0000186907 00000 n -0000187041 00000 n -0000187175 00000 n -0000187309 00000 n -0000187443 00000 n -0000187577 00000 n -0000187711 00000 n -0000187845 00000 n -0000187979 00000 n -0000188113 00000 n -0000188247 00000 n -0000188381 00000 n -0000188515 00000 n -0000188649 00000 n -0000188782 00000 n -0000193785 00000 n -0000194248 00000 n -0000194382 00000 n -0000194516 00000 n -0000194650 00000 n -0000194784 00000 n -0000194918 00000 n -0000195052 00000 n -0000195186 00000 n -0000195320 00000 n -0000195454 00000 n -0000195588 00000 n -0000195722 00000 n -0000195856 00000 n -0000195990 00000 n -0000196124 00000 n -0000196258 00000 n -0000196392 00000 n -0000196526 00000 n -0000196660 00000 n -0000196794 00000 n -0000196928 00000 n -0000197062 00000 n -0000197196 00000 n -0000197330 00000 n -0000197464 00000 n -0000197598 00000 n -0000197732 00000 n -0000197866 00000 n -0000198000 00000 n -0000198134 00000 n -0000198268 00000 n -0000198403 00000 n -0000202995 00000 n -0000203386 00000 n -0000203521 00000 n -0000203654 00000 n -0000203787 00000 n -0000203920 00000 n -0000204053 00000 n -0000204186 00000 n -0000204319 00000 n -0000204452 00000 n -0000204585 00000 n -0000204719 00000 n -0000204853 00000 n -0000204987 00000 n -0000205121 00000 n -0000205255 00000 n -0000205389 00000 n -0000205523 00000 n -0000205657 00000 n -0000205791 00000 n -0000205925 00000 n -0000206059 00000 n -0000206193 00000 n -0000206326 00000 n -0000211317 00000 n -0000211828 00000 n -0000211962 00000 n -0000212096 00000 n -0000212230 00000 n -0000212364 00000 n -0000212498 00000 n -0000212632 00000 n -0000212766 00000 n -0000212900 00000 n -0000213034 00000 n -0000213168 00000 n -0000213302 00000 n -0000213436 00000 n -0000213570 00000 n -0000213704 00000 n -0000213838 00000 n -0000213972 00000 n -0000214106 00000 n -0000214240 00000 n -0000214374 00000 n -0000214508 00000 n -0000214642 00000 n -0000214776 00000 n -0000214910 00000 n -0000215044 00000 n -0000215178 00000 n -0000215312 00000 n -0000215446 00000 n -0000215580 00000 n -0000215714 00000 n -0000215848 00000 n -0000215982 00000 n -0000216116 00000 n -0000216250 00000 n -0000216384 00000 n -0000216518 00000 n -0000216651 00000 n -0000216783 00000 n -0000220032 00000 n -0000220271 00000 n -0000220405 00000 n -0000220539 00000 n -0000220674 00000 n -0000225715 00000 n -0000225938 00000 n -0000226073 00000 n -0000231076 00000 n -0000231299 00000 n -0000231434 00000 n -0000236225 00000 n -0000236428 00000 n -0000241436 00000 n -0000241659 00000 n -0000241794 00000 n -0000246649 00000 n -0000246852 00000 n -0000251775 00000 n -0000251998 00000 n -0000252133 00000 n -0000257000 00000 n -0000257223 00000 n -0000257358 00000 n -0000262131 00000 n -0000262334 00000 n -0000267334 00000 n -0000267557 00000 n -0000267692 00000 n -0000272677 00000 n -0000272880 00000 n -0000277784 00000 n -0000278007 00000 n -0000278142 00000 n -0000283091 00000 n -0000283314 00000 n -0000283449 00000 n -0000288275 00000 n -0000288478 00000 n -0000293483 00000 n -0000293706 00000 n -0000293841 00000 n -0000298855 00000 n -0000299078 00000 n -0000299213 00000 n -0000304128 00000 n -0000304331 00000 n -0000309308 00000 n -0000309531 00000 n -0000309666 00000 n -0000314447 00000 n -0000314650 00000 n -0000318970 00000 n -0000319201 00000 n -0000319336 00000 n -0000319471 00000 n -0000324362 00000 n -0000324585 00000 n -0000324720 00000 n -0000329754 00000 n -0000329977 00000 n -0000330112 00000 n -0000334992 00000 n -0000335195 00000 n -0000340299 00000 n -0000340522 00000 n -0000340657 00000 n -0000345547 00000 n -0000345750 00000 n -0000350768 00000 n -0000350991 00000 n -0000351126 00000 n -0000356049 00000 n -0000356272 00000 n -0000356407 00000 n -0000361218 00000 n -0000361421 00000 n -0000366467 00000 n -0000366690 00000 n -0000366825 00000 n -0000371758 00000 n -0000371981 00000 n -0000372116 00000 n -0000376932 00000 n -0000377135 00000 n -0000382112 00000 n -0000382335 00000 n -0000382470 00000 n -0000387421 00000 n -0000387624 00000 n -0000392507 00000 n -0000392730 00000 n -0000392865 00000 n -0000397809 00000 n -0000398032 00000 n -0000398167 00000 n -0000402979 00000 n -0000403182 00000 n -0000408184 00000 n -0000408407 00000 n -0000408542 00000 n -0000413541 00000 n -0000413764 00000 n -0000413899 00000 n -0000418682 00000 n -0000418885 00000 n -0000423879 00000 n -0000424102 00000 n -0000424237 00000 n -0000429096 00000 n -0000429299 00000 n -0000434327 00000 n -0000434550 00000 n -0000434685 00000 n -0000439559 00000 n -0000439782 00000 n -0000439917 00000 n -0000444664 00000 n -0000444867 00000 n -0000449930 00000 n -0000450153 00000 n -0000450289 00000 n -0000455240 00000 n -0000455443 00000 n -0000460278 00000 n -0000460501 00000 n -0000460637 00000 n -0000465662 00000 n -0000465885 00000 n -0000466021 00000 n -0000470808 00000 n -0000471011 00000 n -0000475993 00000 n -0000476216 00000 n -0000476352 00000 n -0000481349 00000 n -0000481572 00000 n -0000481707 00000 n -0000486477 00000 n -0000486680 00000 n -0000491763 00000 n -0000492098 00000 n -0000492234 00000 n -0000492368 00000 n -0000492502 00000 n -0000492636 00000 n -0000492770 00000 n -0000492904 00000 n -0000493038 00000 n -0000493172 00000 n -0000493306 00000 n -0000493441 00000 n -0000493576 00000 n -0000493711 00000 n -0000493846 00000 n -0000493980 00000 n -0000494113 00000 n -0000498615 00000 n -0000498926 00000 n -0000499061 00000 n -0000499196 00000 n -0000499331 00000 n -0000499466 00000 n -0000499601 00000 n -0000499736 00000 n -0000499871 00000 n -0000500006 00000 n -0000500141 00000 n -0000500276 00000 n -0000500412 00000 n -0000500548 00000 n -0000505561 00000 n -0000505784 00000 n -0000505920 00000 n -0000510767 00000 n -0000510990 00000 n -0000511126 00000 n -0000515989 00000 n -0000516212 00000 n -0000516348 00000 n -0000521170 00000 n -0000521393 00000 n -0000521529 00000 n -0000526387 00000 n -0000526610 00000 n -0000526746 00000 n -0000531422 00000 n -0000531645 00000 n -0000531781 00000 n -0000535383 00000 n -0000535586 00000 n -0000540178 00000 n -0000540409 00000 n -0000540545 00000 n -0000540680 00000 n -0000544182 00000 n -0000544385 00000 n -0000549242 00000 n -0000549465 00000 n -0000549601 00000 n -0000554375 00000 n -0000554578 00000 n -0000559697 00000 n -0000559920 00000 n -0000560056 00000 n -0000564808 00000 n -0000565011 00000 n -0000570145 00000 n -0000570368 00000 n -0000570503 00000 n -0000575236 00000 n -0000575439 00000 n -0000580443 00000 n -0000580646 00000 n -0000585453 00000 n -0000585676 00000 n -0000585812 00000 n -0000590572 00000 n -0000590775 00000 n -0000595831 00000 n -0000596054 00000 n -0000596190 00000 n -0000600966 00000 n -0000601169 00000 n -0000606179 00000 n -0000606402 00000 n -0000606538 00000 n -0000611307 00000 n -0000611510 00000 n -0000616355 00000 n -0000616558 00000 n -0000621376 00000 n -0000621599 00000 n -0000621735 00000 n -0000626488 00000 n -0000626691 00000 n -0000630319 00000 n -0000630522 00000 n -0000633800 00000 n -0000634003 00000 n -0000639049 00000 n -0000639272 00000 n -0000639408 00000 n -0000643952 00000 n -0000644155 00000 n -0000648447 00000 n -0000648670 00000 n -0000648806 00000 n -0000652880 00000 n -0000653111 00000 n -0000653247 00000 n -0000653383 00000 n -0000658341 00000 n -0000658612 00000 n -0000658748 00000 n -0000658884 00000 n -0000659018 00000 n -0000659152 00000 n -0000659286 00000 n -0000659420 00000 n -0000659556 00000 n -0000664331 00000 n -0000664594 00000 n -0000664730 00000 n -0000664864 00000 n -0000664998 00000 n -0000665132 00000 n -0000665266 00000 n -0000665402 00000 n -0000668166 00000 n -0000668369 00000 n -0000672683 00000 n -0000672922 00000 n -0000673058 00000 n -0000673194 00000 n -0000673329 00000 n -0000677356 00000 n -0000677595 00000 n -0000677731 00000 n -0000677867 00000 n -0000678003 00000 n -0000682658 00000 n -0000682905 00000 n -0000683041 00000 n -0000683177 00000 n -0000683313 00000 n -0000683449 00000 n -0000688333 00000 n -0000688564 00000 n -0000688700 00000 n -0000688836 00000 n -0000692983 00000 n -0000693214 00000 n -0000693350 00000 n -0000693486 00000 n -0000697662 00000 n -0000697909 00000 n -0000698045 00000 n -0000698181 00000 n -0000698317 00000 n -0000698453 00000 n -0000703448 00000 n -0000703679 00000 n -0000703815 00000 n -0000703951 00000 n -0000708536 00000 n -0000708767 00000 n -0000708903 00000 n -0000709039 00000 n -0000711984 00000 n -0000712207 00000 n -0000712343 00000 n -0000716421 00000 n -0000716624 00000 n -0000722138 00000 n -0000722537 00000 n -0000722673 00000 n -0000722807 00000 n -0000722941 00000 n -0000723075 00000 n -0000723209 00000 n -0000723343 00000 n -0000723477 00000 n -0000723611 00000 n -0000723745 00000 n -0000723880 00000 n -0000724015 00000 n -0000724150 00000 n -0000724285 00000 n -0000724420 00000 n -0000724555 00000 n -0000724690 00000 n -0000724825 00000 n -0000724960 00000 n -0000725095 00000 n -0000725230 00000 n -0000725365 00000 n -0000725500 00000 n -0000725634 00000 n -0000729339 00000 n -0000729626 00000 n -0000729761 00000 n -0000729896 00000 n -0000730031 00000 n -0000730166 00000 n -0000730301 00000 n -0000730436 00000 n -0000730571 00000 n -0000730706 00000 n -0000730842 00000 n -0000735267 00000 n -0000735490 00000 n -0000735626 00000 n -0000740087 00000 n -0000740318 00000 n -0000740454 00000 n -0000740590 00000 n -0000745056 00000 n -0000745279 00000 n -0000745415 00000 n -0000750381 00000 n -0000750716 00000 n -0000750852 00000 n -0000750986 00000 n -0000751120 00000 n -0000751254 00000 n -0000751388 00000 n -0000751522 00000 n -0000751656 00000 n -0000751790 00000 n -0000751924 00000 n -0000752059 00000 n -0000752194 00000 n -0000752329 00000 n -0000752464 00000 n -0000752598 00000 n -0000752731 00000 n -0000755980 00000 n -0000756235 00000 n -0000756370 00000 n -0000756505 00000 n -0000756640 00000 n -0000756775 00000 n -0000756911 00000 n -0000760328 00000 n -0000760531 00000 n -0000763793 00000 n -0000763996 00000 n -0000768403 00000 n -0000768606 00000 n -0000773005 00000 n -0000773372 00000 n -0000773508 00000 n -0000773642 00000 n -0000773776 00000 n -0000773910 00000 n -0000774044 00000 n -0000774178 00000 n -0000774312 00000 n -0000774446 00000 n -0000774580 00000 n -0000774715 00000 n -0000774850 00000 n -0000774985 00000 n -0000775120 00000 n -0000775255 00000 n -0000775390 00000 n -0000775525 00000 n -0000775660 00000 n -0000775795 00000 n -0000775931 00000 n -0000776127 00000 n -0000776353 00000 n -0000776479 00000 n -0000776625 00000 n -0000776765 00000 n -0000776962 00000 n -0000777196 00000 n -0000777427 00000 n -0000777683 00000 n -0000777922 00000 n -0000778169 00000 n -0000778399 00000 n -0000778641 00000 n -0000778866 00000 n -0000779110 00000 n -0000779338 00000 n -0000779573 00000 n -0000779791 00000 n -0000780022 00000 n -0000780236 00000 n -0000780471 00000 n -0000780689 00000 n -0000780933 00000 n -0000781160 00000 n -0000781392 00000 n -0000781639 00000 n -0000781844 00000 n -0000782020 00000 n -0000782215 00000 n -0000782420 00000 n -0000782626 00000 n -0000782833 00000 n -0000783087 00000 n -0000783317 00000 n -0000783540 00000 n -0000783751 00000 n -0000783962 00000 n -0000784177 00000 n -0000784393 00000 n -0000784607 00000 n -0000784857 00000 n -0000785072 00000 n -0000785291 00000 n -0000785509 00000 n -0000785725 00000 n -0000785937 00000 n -0000786142 00000 n -0000786349 00000 n -0000786554 00000 n -0000786755 00000 n -0000786982 00000 n -0000787212 00000 n -0000787441 00000 n -0000787658 00000 n -0000787875 00000 n -0000788101 00000 n -0000788337 00000 n -0000788581 00000 n -0000788797 00000 n -0000789010 00000 n -0000789238 00000 n -0000789452 00000 n -0000789664 00000 n -0000789847 00000 n -0000790061 00000 n -0000790265 00000 n -0000790490 00000 n -0000790723 00000 n -0000790936 00000 n -0000791158 00000 n -0000791355 00000 n -0000791529 00000 n -0000791719 00000 n -0000791916 00000 n -0000792108 00000 n -0000792319 00000 n -0000792527 00000 n -0000792731 00000 n -0000792966 00000 n -0000793179 00000 n -0000793388 00000 n -0000793583 00000 n -0000793780 00000 n -0000793963 00000 n -0000794138 00000 n -0000794323 00000 n -0000794517 00000 n -0000794718 00000 n -0000794942 00000 n -0000795164 00000 n -0000795405 00000 n -0000795628 00000 n -0000795877 00000 n -0000796109 00000 n -0000796356 00000 n -0000796593 00000 n -0000796847 00000 n -0000797070 00000 n -0000797248 00000 n -0000797444 00000 n -0000797671 00000 n -0000797890 00000 n -0000798099 00000 n -0000798331 00000 n -0000798552 00000 n -0000798775 00000 n -0000798983 00000 n -0000799238 00000 n -0000799486 00000 n -0000799709 00000 n -0000799936 00000 n -0000800148 00000 n -0000800343 00000 n -0000800543 00000 n -0000800722 00000 n -0000800922 00000 n -0000801105 00000 n -0000801320 00000 n -0000801525 00000 n -0000801716 00000 n -0000801901 00000 n -0000802140 00000 n -0000802372 00000 n -0000802608 00000 n -0000802830 00000 n -0000803024 00000 n -0000803241 00000 n -0000803454 00000 n -0000803655 00000 n -0000803849 00000 n -0000804039 00000 n -0000804236 00000 n -0000804424 00000 n -0000804629 00000 n -0000804799 00000 n -0000804997 00000 n -0000805192 00000 n -0000805363 00000 n -0000805568 00000 n -0000805801 00000 n -0000806004 00000 n -0000806185 00000 n -0000806249 00000 n -0000814191 00000 n -0000818994 00000 n -0000830855 00000 n -0000843251 00000 n -0000843485 00000 n -0000843911 00000 n -0000844737 00000 n -0000844890 00000 n -0000845121 00000 n -0000845718 00000 n -0000847203 00000 n -0000847353 00000 n -0000847588 00000 n -0000848127 00000 n -0000849364 00000 n -0000849517 00000 n -0000849761 00000 n -0000850356 00000 n -0000851981 00000 n -0000852138 00000 n -trailer -<< -/Size 1079 -/Root 3 0 R -/Info 2 0 R ->> -startxref -852230 -%%EOF diff --git a/versioned_docs/version-2.6/security/hardening-guides/rke-1.6-hardening-2.6/Rancher_v2-6_CIS_v1-6_Hardening_Guide.pdf b/versioned_docs/version-2.6/security/hardening-guides/rke-1.6-hardening-2.6/Rancher_v2-6_CIS_v1-6_Hardening_Guide.pdf deleted file mode 100644 index 8984cc57c5e..00000000000 --- a/versioned_docs/version-2.6/security/hardening-guides/rke-1.6-hardening-2.6/Rancher_v2-6_CIS_v1-6_Hardening_Guide.pdf +++ /dev/null @@ -1,8259 +0,0 @@ -%PDF-1.7 -%🖤 -1 0 obj -<< -/Type /Pages -/Kids [ 6 0 R 8 0 R 30 0 R 45 0 R 53 0 R 67 0 R 77 0 R 99 0 R 129 0 R 164 0 R 194 0 R 232 0 R 270 0 R 310 0 R 350 0 R 390 0 R 425 0 R 464 0 R 499 0 R 534 0 R 557 0 R 591 0 R ] -/Count 22 ->> -endobj -2 0 obj -<< -/Title (RKE Hardening Guide with CIS v1.6 Benchmark) -/Creator (pandoc) -/Producer (WeasyPrint 54.3) ->> -endobj -3 0 obj -<< -/Type /Catalog -/Pages 1 0 R -/Outlines 611 0 R -/Names << -/Dests << -/Names [ (TOC) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (cb1) [ 45 0 R /XYZ 84.259843 441.572079 0 ] (cb1-1) [ 45 0 R /XYZ 101.433071 439.875057 0 ] (cb1-2) [ 45 0 R /XYZ 99.933071 420.928036 0 ] (cb1-3) [ 45 0 R /XYZ 99.933071 401.981014 0 ] (cb1-4) [ 45 0 R /XYZ 99.933071 383.033993 0 ] (cb1-5) [ 45 0 R /XYZ 99.933071 364.086971 0 ] (cb10) [ 390 0 R /XYZ 84.259843 416.811778 0 ] (cb10-1) [ 390 0 R /XYZ 101.433071 415.114756 0 ] (cb10-10) [ 390 0 R /XYZ 99.933071 244.591563 0 ] (cb10-100) [ 499 0 R /XYZ 99.933071 590.755821 0 ] (cb10-101) [ 499 0 R /XYZ 99.933071 571.808799 0 ] (cb10-102) [ 499 0 R /XYZ 99.933071 552.861778 0 ] (cb10-103) [ 499 0 R /XYZ 99.933071 533.914756 0 ] (cb10-104) [ 499 0 R /XYZ 99.933071 514.967735 0 ] (cb10-105) [ 499 0 R /XYZ 99.933071 496.020714 0 ] (cb10-106) [ 499 0 R /XYZ 99.933071 477.073692 0 ] (cb10-107) [ 499 0 R /XYZ 99.933071 458.126671 0 ] (cb10-108) [ 499 0 R /XYZ 99.933071 439.179649 0 ] (cb10-109) [ 499 0 R /XYZ 99.933071 420.232628 0 ] (cb10-11) [ 390 0 R /XYZ 99.933071 225.644542 0 ] (cb10-110) [ 499 0 R /XYZ 99.933071 401.285606 0 ] (cb10-111) [ 499 0 R /XYZ 99.933071 382.338585 0 ] (cb10-112) [ 499 0 R /XYZ 99.933071 363.391563 0 ] (cb10-113) [ 499 0 R /XYZ 99.933071 344.444542 0 ] (cb10-114) [ 499 0 R /XYZ 99.933071 325.497520 0 ] (cb10-115) [ 499 0 R /XYZ 99.933071 306.550499 0 ] (cb10-116) [ 499 0 R /XYZ 99.933071 287.603477 0 ] (cb10-117) [ 499 0 R /XYZ 99.933071 173.921348 0 ] (cb10-118) [ 499 0 R /XYZ 99.933071 154.974327 0 ] (cb10-119) [ 499 0 R /XYZ 99.933071 136.027305 0 ] (cb10-12) [ 390 0 R /XYZ 99.933071 206.697520 0 ] (cb10-120) [ 499 0 R /XYZ 99.933071 117.080284 0 ] (cb10-121) [ 499 0 R /XYZ 99.933071 98.133262 0 ] (cb10-122) [ 534 0 R /XYZ 99.933071 799.173057 0 ] (cb10-123) [ 534 0 R /XYZ 99.933071 685.490928 0 ] (cb10-124) [ 534 0 R /XYZ 99.933071 666.543907 0 ] (cb10-125) [ 534 0 R /XYZ 99.933071 647.596885 0 ] (cb10-126) [ 534 0 R /XYZ 99.933071 628.649864 0 ] (cb10-127) [ 534 0 R /XYZ 99.933071 609.702842 0 ] (cb10-128) [ 534 0 R /XYZ 99.933071 590.755821 0 ] (cb10-129) [ 534 0 R /XYZ 99.933071 571.808799 0 ] (cb10-13) [ 390 0 R /XYZ 99.933071 187.750499 0 ] (cb10-14) [ 390 0 R /XYZ 99.933071 168.803477 0 ] (cb10-15) [ 390 0 R /XYZ 99.933071 149.856456 0 ] (cb10-16) [ 390 0 R /XYZ 99.933071 130.909434 0 ] (cb10-17) [ 390 0 R /XYZ 99.933071 111.962413 0 ] (cb10-18) [ 390 0 R /XYZ 99.933071 93.015391 0 ] (cb10-19) [ 425 0 R /XYZ 99.933071 799.173057 0 ] (cb10-2) [ 390 0 R /XYZ 99.933071 396.167735 0 ] (cb10-20) [ 425 0 R /XYZ 99.933071 780.226036 0 ] (cb10-21) [ 425 0 R /XYZ 99.933071 761.279014 0 ] (cb10-22) [ 425 0 R /XYZ 99.933071 742.331993 0 ] (cb10-23) [ 425 0 R /XYZ 99.933071 723.384971 0 ] (cb10-24) [ 425 0 R /XYZ 99.933071 704.437950 0 ] (cb10-25) [ 425 0 R /XYZ 99.933071 685.490928 0 ] (cb10-26) [ 425 0 R /XYZ 99.933071 666.543907 0 ] (cb10-27) [ 425 0 R /XYZ 99.933071 647.596885 0 ] (cb10-28) [ 425 0 R /XYZ 99.933071 628.649864 0 ] (cb10-29) [ 425 0 R /XYZ 99.933071 609.702842 0 ] (cb10-3) [ 390 0 R /XYZ 99.933071 377.220714 0 ] (cb10-30) [ 425 0 R /XYZ 99.933071 571.808799 0 ] (cb10-31) [ 425 0 R /XYZ 99.933071 552.861778 0 ] (cb10-32) [ 425 0 R /XYZ 99.933071 533.914756 0 ] (cb10-33) [ 425 0 R /XYZ 99.933071 514.967735 0 ] (cb10-34) [ 425 0 R /XYZ 99.933071 496.020714 0 ] (cb10-35) [ 425 0 R /XYZ 99.933071 477.073692 0 ] (cb10-36) [ 425 0 R /XYZ 99.933071 458.126671 0 ] (cb10-37) [ 425 0 R /XYZ 99.933071 439.179649 0 ] (cb10-38) [ 425 0 R /XYZ 99.933071 420.232628 0 ] (cb10-39) [ 425 0 R /XYZ 99.933071 401.285606 0 ] (cb10-4) [ 390 0 R /XYZ 99.933071 358.273692 0 ] (cb10-40) [ 425 0 R /XYZ 99.933071 382.338585 0 ] (cb10-41) [ 425 0 R /XYZ 99.933071 363.391563 0 ] (cb10-42) [ 425 0 R /XYZ 99.933071 344.444542 0 ] (cb10-43) [ 425 0 R /XYZ 99.933071 325.497520 0 ] (cb10-44) [ 425 0 R /XYZ 99.933071 306.550499 0 ] (cb10-45) [ 425 0 R /XYZ 99.933071 287.603477 0 ] (cb10-46) [ 425 0 R /XYZ 99.933071 268.656456 0 ] (cb10-47) [ 425 0 R /XYZ 99.933071 249.709434 0 ] (cb10-48) [ 425 0 R /XYZ 99.933071 230.762413 0 ] (cb10-49) [ 425 0 R /XYZ 99.933071 211.815391 0 ] (cb10-5) [ 390 0 R /XYZ 99.933071 339.326671 0 ] (cb10-50) [ 425 0 R /XYZ 99.933071 192.868370 0 ] (cb10-51) [ 425 0 R /XYZ 99.933071 173.921348 0 ] (cb10-52) [ 425 0 R /XYZ 99.933071 154.974327 0 ] (cb10-53) [ 425 0 R /XYZ 99.933071 136.027305 0 ] (cb10-54) [ 425 0 R /XYZ 99.933071 117.080284 0 ] (cb10-55) [ 425 0 R /XYZ 99.933071 98.133262 0 ] (cb10-56) [ 464 0 R /XYZ 99.933071 799.173057 0 ] (cb10-57) [ 464 0 R /XYZ 99.933071 780.226036 0 ] (cb10-58) [ 464 0 R /XYZ 99.933071 761.279014 0 ] (cb10-59) [ 464 0 R /XYZ 99.933071 742.331993 0 ] (cb10-6) [ 390 0 R /XYZ 99.933071 320.379649 0 ] (cb10-60) [ 464 0 R /XYZ 99.933071 723.384971 0 ] (cb10-61) [ 464 0 R /XYZ 99.933071 704.437950 0 ] (cb10-62) [ 464 0 R /XYZ 99.933071 685.490928 0 ] (cb10-63) [ 464 0 R /XYZ 99.933071 666.543907 0 ] (cb10-64) [ 464 0 R /XYZ 99.933071 647.596885 0 ] (cb10-65) [ 464 0 R /XYZ 99.933071 628.649864 0 ] (cb10-66) [ 464 0 R /XYZ 99.933071 609.702842 0 ] (cb10-67) [ 464 0 R /XYZ 99.933071 590.755821 0 ] (cb10-68) [ 464 0 R /XYZ 99.933071 571.808799 0 ] (cb10-69) [ 464 0 R /XYZ 99.933071 552.861778 0 ] (cb10-7) [ 390 0 R /XYZ 99.933071 301.432628 0 ] (cb10-70) [ 464 0 R /XYZ 99.933071 533.914756 0 ] (cb10-71) [ 464 0 R /XYZ 99.933071 514.967735 0 ] (cb10-72) [ 464 0 R /XYZ 99.933071 496.020714 0 ] (cb10-73) [ 464 0 R /XYZ 99.933071 477.073692 0 ] (cb10-74) [ 464 0 R /XYZ 99.933071 458.126671 0 ] (cb10-75) [ 464 0 R /XYZ 99.933071 439.179649 0 ] (cb10-76) [ 464 0 R /XYZ 99.933071 420.232628 0 ] (cb10-77) [ 464 0 R /XYZ 99.933071 401.285606 0 ] (cb10-78) [ 464 0 R /XYZ 99.933071 382.338585 0 ] (cb10-79) [ 464 0 R /XYZ 99.933071 363.391563 0 ] (cb10-8) [ 390 0 R /XYZ 99.933071 282.485606 0 ] (cb10-80) [ 464 0 R /XYZ 99.933071 344.444542 0 ] (cb10-81) [ 464 0 R /XYZ 99.933071 325.497520 0 ] (cb10-82) [ 464 0 R /XYZ 99.933071 306.550499 0 ] (cb10-83) [ 464 0 R /XYZ 99.933071 287.603477 0 ] (cb10-84) [ 464 0 R /XYZ 99.933071 268.656456 0 ] (cb10-85) [ 464 0 R /XYZ 99.933071 249.709434 0 ] (cb10-86) [ 464 0 R /XYZ 99.933071 230.762413 0 ] (cb10-87) [ 464 0 R /XYZ 99.933071 117.080284 0 ] (cb10-88) [ 464 0 R /XYZ 99.933071 98.133262 0 ] (cb10-89) [ 499 0 R /XYZ 99.933071 799.173057 0 ] (cb10-9) [ 390 0 R /XYZ 99.933071 263.538585 0 ] (cb10-90) [ 499 0 R /XYZ 99.933071 780.226036 0 ] (cb10-91) [ 499 0 R /XYZ 99.933071 761.279014 0 ] (cb10-92) [ 499 0 R /XYZ 99.933071 742.331993 0 ] (cb10-93) [ 499 0 R /XYZ 99.933071 723.384971 0 ] (cb10-94) [ 499 0 R /XYZ 99.933071 704.437950 0 ] (cb10-95) [ 499 0 R /XYZ 99.933071 685.490928 0 ] (cb10-96) [ 499 0 R /XYZ 99.933071 666.543907 0 ] (cb10-97) [ 499 0 R /XYZ 99.933071 647.596885 0 ] (cb10-98) [ 499 0 R /XYZ 99.933071 628.649864 0 ] (cb10-99) [ 499 0 R /XYZ 99.933071 609.702842 0 ] (cb11) [ 534 0 R /XYZ 84.259843 331.262799 0 ] (cb11-1) [ 534 0 R /XYZ 101.433071 329.565778 0 ] (cb11-10) [ 534 0 R /XYZ 99.933071 159.042585 0 ] (cb11-11) [ 534 0 R /XYZ 99.933071 140.095563 0 ] (cb11-12) [ 534 0 R /XYZ 99.933071 121.148542 0 ] (cb11-13) [ 534 0 R /XYZ 99.933071 102.201520 0 ] (cb11-14) [ 557 0 R /XYZ 99.933071 799.173057 0 ] (cb11-15) [ 557 0 R /XYZ 99.933071 780.226036 0 ] (cb11-16) [ 557 0 R /XYZ 99.933071 761.279014 0 ] (cb11-17) [ 557 0 R /XYZ 99.933071 742.331993 0 ] (cb11-18) [ 557 0 R /XYZ 99.933071 723.384971 0 ] (cb11-19) [ 557 0 R /XYZ 99.933071 704.437950 0 ] (cb11-2) [ 534 0 R /XYZ 99.933071 310.618756 0 ] (cb11-20) [ 557 0 R /XYZ 99.933071 685.490928 0 ] (cb11-21) [ 557 0 R /XYZ 99.933071 666.543907 0 ] (cb11-22) [ 557 0 R /XYZ 99.933071 647.596885 0 ] (cb11-23) [ 557 0 R /XYZ 99.933071 628.649864 0 ] (cb11-24) [ 557 0 R /XYZ 99.933071 609.702842 0 ] (cb11-25) [ 557 0 R /XYZ 99.933071 590.755821 0 ] (cb11-26) [ 557 0 R /XYZ 99.933071 571.808799 0 ] (cb11-3) [ 534 0 R /XYZ 99.933071 291.671735 0 ] (cb11-4) [ 534 0 R /XYZ 99.933071 272.724714 0 ] (cb11-5) [ 534 0 R /XYZ 99.933071 253.777692 0 ] (cb11-6) [ 534 0 R /XYZ 99.933071 234.830671 0 ] (cb11-7) [ 534 0 R /XYZ 99.933071 215.883649 0 ] (cb11-8) [ 534 0 R /XYZ 99.933071 196.936628 0 ] (cb11-9) [ 534 0 R /XYZ 99.933071 177.989606 0 ] (cb12) [ 557 0 R /XYZ 84.259843 446.115778 0 ] (cb12-1) [ 557 0 R /XYZ 101.433071 444.418756 0 ] (cb12-10) [ 557 0 R /XYZ 99.933071 273.895563 0 ] (cb12-11) [ 557 0 R /XYZ 99.933071 254.948542 0 ] (cb12-12) [ 557 0 R /XYZ 99.933071 236.001520 0 ] (cb12-13) [ 557 0 R /XYZ 99.933071 217.054499 0 ] (cb12-14) [ 557 0 R /XYZ 99.933071 198.107477 0 ] (cb12-15) [ 557 0 R /XYZ 99.933071 179.160456 0 ] (cb12-16) [ 557 0 R /XYZ 99.933071 160.213434 0 ] (cb12-17) [ 557 0 R /XYZ 99.933071 141.266413 0 ] (cb12-18) [ 557 0 R /XYZ 99.933071 122.319391 0 ] (cb12-19) [ 557 0 R /XYZ 99.933071 103.372370 0 ] (cb12-2) [ 557 0 R /XYZ 99.933071 425.471735 0 ] (cb12-20) [ 591 0 R /XYZ 99.933071 780.226036 0 ] (cb12-21) [ 591 0 R /XYZ 99.933071 742.331993 0 ] (cb12-22) [ 591 0 R /XYZ 99.933071 723.384971 0 ] (cb12-23) [ 591 0 R /XYZ 99.933071 704.437950 0 ] (cb12-3) [ 557 0 R /XYZ 99.933071 406.524714 0 ] (cb12-4) [ 557 0 R /XYZ 99.933071 387.577692 0 ] (cb12-5) [ 557 0 R /XYZ 99.933071 368.630671 0 ] (cb12-6) [ 557 0 R /XYZ 99.933071 349.683649 0 ] (cb12-7) [ 557 0 R /XYZ 99.933071 330.736628 0 ] (cb12-8) [ 557 0 R /XYZ 99.933071 311.789606 0 ] (cb12-9) [ 557 0 R /XYZ 99.933071 292.842585 0 ] (cb2) [ 53 0 R /XYZ 84.259843 753.620079 0 ] (cb2-1) [ 53 0 R /XYZ 101.433071 751.923057 0 ] (cb2-2) [ 53 0 R /XYZ 99.933071 732.976036 0 ] (cb3) [ 53 0 R /XYZ 84.259843 654.779014 0 ] (cb3-1) [ 53 0 R /XYZ 101.433071 653.081993 0 ] (cb3-2) [ 53 0 R /XYZ 99.933071 634.134971 0 ] (cb3-3) [ 53 0 R /XYZ 99.933071 615.187950 0 ] (cb3-4) [ 53 0 R /XYZ 99.933071 596.240928 0 ] (cb4) [ 53 0 R /XYZ 84.259843 282.344928 0 ] (cb4-1) [ 53 0 R /XYZ 101.433071 280.647907 0 ] (cb5) [ 53 0 R /XYZ 84.259843 217.647907 0 ] (cb5-1) [ 53 0 R /XYZ 101.433071 215.950885 0 ] (cb5-2) [ 53 0 R /XYZ 99.933071 197.003864 0 ] (cb5-3) [ 53 0 R /XYZ 99.933071 178.056842 0 ] (cb5-4) [ 53 0 R /XYZ 99.933071 159.109821 0 ] (cb5-5) [ 53 0 R /XYZ 99.933071 140.162799 0 ] (cb6) [ 67 0 R /XYZ 84.259843 749.870079 0 ] (cb6-1) [ 67 0 R /XYZ 101.433071 748.173057 0 ] (cb6-2) [ 67 0 R /XYZ 99.933071 729.226036 0 ] (cb6-3) [ 67 0 R /XYZ 99.933071 710.279014 0 ] (cb6-4) [ 67 0 R /XYZ 99.933071 672.384971 0 ] (cb6-5) [ 67 0 R /XYZ 99.933071 634.490928 0 ] (cb7) [ 77 0 R /XYZ 84.259843 716.781496 0 ] (cb7-1) [ 77 0 R /XYZ 101.433071 715.084475 0 ] (cb7-10) [ 77 0 R /XYZ 99.933071 544.561281 0 ] (cb7-11) [ 77 0 R /XYZ 99.933071 525.614260 0 ] (cb7-12) [ 77 0 R /XYZ 99.933071 506.667238 0 ] (cb7-13) [ 77 0 R /XYZ 99.933071 487.720217 0 ] (cb7-14) [ 77 0 R /XYZ 99.933071 468.773195 0 ] (cb7-2) [ 77 0 R /XYZ 99.933071 696.137453 0 ] (cb7-3) [ 77 0 R /XYZ 99.933071 677.190432 0 ] (cb7-4) [ 77 0 R /XYZ 99.933071 658.243410 0 ] (cb7-5) [ 77 0 R /XYZ 99.933071 639.296389 0 ] (cb7-6) [ 77 0 R /XYZ 99.933071 620.349367 0 ] (cb7-7) [ 77 0 R /XYZ 99.933071 601.402346 0 ] (cb7-8) [ 77 0 R /XYZ 99.933071 582.455324 0 ] (cb7-9) [ 77 0 R /XYZ 99.933071 563.508303 0 ] (cb8) [ 77 0 R /XYZ 84.259843 372.023195 0 ] (cb8-1) [ 77 0 R /XYZ 101.433071 370.326174 0 ] (cb8-2) [ 77 0 R /XYZ 99.933071 351.379152 0 ] (cb8-3) [ 77 0 R /XYZ 99.933071 332.432131 0 ] (cb8-4) [ 77 0 R /XYZ 99.933071 294.538088 0 ] (cb8-5) [ 77 0 R /XYZ 99.933071 275.591066 0 ] (cb9) [ 99 0 R /XYZ 84.259843 652.281496 0 ] (cb9-1) [ 99 0 R /XYZ 101.433071 650.584475 0 ] (cb9-10) [ 99 0 R /XYZ 99.933071 423.220217 0 ] (cb9-100) [ 194 0 R /XYZ 99.933071 590.755821 0 ] (cb9-101) [ 194 0 R /XYZ 99.933071 571.808799 0 ] (cb9-102) [ 194 0 R /XYZ 99.933071 552.861778 0 ] (cb9-103) [ 194 0 R /XYZ 99.933071 533.914756 0 ] (cb9-104) [ 194 0 R /XYZ 99.933071 514.967735 0 ] (cb9-105) [ 194 0 R /XYZ 99.933071 496.020714 0 ] (cb9-106) [ 194 0 R /XYZ 99.933071 477.073692 0 ] (cb9-107) [ 194 0 R /XYZ 99.933071 458.126671 0 ] (cb9-108) [ 194 0 R /XYZ 99.933071 439.179649 0 ] (cb9-109) [ 194 0 R /XYZ 99.933071 420.232628 0 ] (cb9-11) [ 99 0 R /XYZ 99.933071 404.273195 0 ] (cb9-110) [ 194 0 R /XYZ 99.933071 401.285606 0 ] (cb9-111) [ 194 0 R /XYZ 99.933071 344.444542 0 ] (cb9-112) [ 194 0 R /XYZ 99.933071 325.497520 0 ] (cb9-113) [ 194 0 R /XYZ 99.933071 306.550499 0 ] (cb9-114) [ 194 0 R /XYZ 99.933071 287.603477 0 ] (cb9-115) [ 194 0 R /XYZ 99.933071 268.656456 0 ] (cb9-116) [ 194 0 R /XYZ 99.933071 249.709434 0 ] (cb9-117) [ 194 0 R /XYZ 99.933071 230.762413 0 ] (cb9-118) [ 194 0 R /XYZ 99.933071 211.815391 0 ] (cb9-119) [ 194 0 R /XYZ 99.933071 192.868370 0 ] (cb9-12) [ 99 0 R /XYZ 99.933071 385.326174 0 ] (cb9-120) [ 194 0 R /XYZ 99.933071 173.921348 0 ] (cb9-121) [ 194 0 R /XYZ 99.933071 154.974327 0 ] (cb9-122) [ 194 0 R /XYZ 99.933071 136.027305 0 ] (cb9-123) [ 194 0 R /XYZ 99.933071 117.080284 0 ] (cb9-124) [ 194 0 R /XYZ 99.933071 98.133262 0 ] (cb9-125) [ 232 0 R /XYZ 99.933071 799.173057 0 ] (cb9-126) [ 232 0 R /XYZ 99.933071 780.226036 0 ] (cb9-127) [ 232 0 R /XYZ 99.933071 761.279014 0 ] (cb9-128) [ 232 0 R /XYZ 99.933071 742.331993 0 ] (cb9-129) [ 232 0 R /XYZ 99.933071 704.437950 0 ] (cb9-13) [ 99 0 R /XYZ 99.933071 366.379152 0 ] (cb9-130) [ 232 0 R /XYZ 99.933071 685.490928 0 ] (cb9-131) [ 232 0 R /XYZ 99.933071 666.543907 0 ] (cb9-132) [ 232 0 R /XYZ 99.933071 647.596885 0 ] (cb9-133) [ 232 0 R /XYZ 99.933071 628.649864 0 ] (cb9-134) [ 232 0 R /XYZ 99.933071 609.702842 0 ] (cb9-135) [ 232 0 R /XYZ 99.933071 590.755821 0 ] (cb9-136) [ 232 0 R /XYZ 99.933071 571.808799 0 ] (cb9-137) [ 232 0 R /XYZ 99.933071 552.861778 0 ] (cb9-138) [ 232 0 R /XYZ 99.933071 533.914756 0 ] (cb9-139) [ 232 0 R /XYZ 99.933071 514.967735 0 ] (cb9-14) [ 99 0 R /XYZ 99.933071 347.432131 0 ] (cb9-140) [ 232 0 R /XYZ 99.933071 477.073692 0 ] (cb9-141) [ 232 0 R /XYZ 99.933071 458.126671 0 ] (cb9-142) [ 232 0 R /XYZ 99.933071 439.179649 0 ] (cb9-143) [ 232 0 R /XYZ 99.933071 420.232628 0 ] (cb9-144) [ 232 0 R /XYZ 99.933071 401.285606 0 ] (cb9-145) [ 232 0 R /XYZ 99.933071 382.338585 0 ] (cb9-146) [ 232 0 R /XYZ 99.933071 363.391563 0 ] (cb9-147) [ 232 0 R /XYZ 99.933071 344.444542 0 ] (cb9-148) [ 232 0 R /XYZ 99.933071 325.497520 0 ] (cb9-149) [ 232 0 R /XYZ 99.933071 306.550499 0 ] (cb9-15) [ 99 0 R /XYZ 99.933071 328.485109 0 ] (cb9-150) [ 232 0 R /XYZ 99.933071 287.603477 0 ] (cb9-151) [ 232 0 R /XYZ 99.933071 268.656456 0 ] (cb9-152) [ 232 0 R /XYZ 99.933071 249.709434 0 ] (cb9-153) [ 232 0 R /XYZ 99.933071 230.762413 0 ] (cb9-154) [ 232 0 R /XYZ 99.933071 211.815391 0 ] (cb9-155) [ 232 0 R /XYZ 99.933071 192.868370 0 ] (cb9-156) [ 232 0 R /XYZ 99.933071 173.921348 0 ] (cb9-157) [ 232 0 R /XYZ 99.933071 154.974327 0 ] (cb9-158) [ 232 0 R /XYZ 99.933071 136.027305 0 ] (cb9-159) [ 232 0 R /XYZ 99.933071 117.080284 0 ] (cb9-16) [ 99 0 R /XYZ 99.933071 309.538088 0 ] (cb9-160) [ 232 0 R /XYZ 99.933071 98.133262 0 ] (cb9-161) [ 270 0 R /XYZ 99.933071 799.173057 0 ] (cb9-162) [ 270 0 R /XYZ 99.933071 780.226036 0 ] (cb9-163) [ 270 0 R /XYZ 99.933071 761.279014 0 ] (cb9-164) [ 270 0 R /XYZ 99.933071 742.331993 0 ] (cb9-165) [ 270 0 R /XYZ 99.933071 723.384971 0 ] (cb9-166) [ 270 0 R /XYZ 99.933071 704.437950 0 ] (cb9-167) [ 270 0 R /XYZ 99.933071 685.490928 0 ] (cb9-168) [ 270 0 R /XYZ 99.933071 666.543907 0 ] (cb9-169) [ 270 0 R /XYZ 99.933071 647.596885 0 ] (cb9-17) [ 99 0 R /XYZ 99.933071 290.591066 0 ] (cb9-170) [ 270 0 R /XYZ 99.933071 628.649864 0 ] (cb9-171) [ 270 0 R /XYZ 99.933071 609.702842 0 ] (cb9-172) [ 270 0 R /XYZ 99.933071 590.755821 0 ] (cb9-173) [ 270 0 R /XYZ 99.933071 571.808799 0 ] (cb9-174) [ 270 0 R /XYZ 99.933071 552.861778 0 ] (cb9-175) [ 270 0 R /XYZ 99.933071 533.914756 0 ] (cb9-176) [ 270 0 R /XYZ 99.933071 514.967735 0 ] (cb9-177) [ 270 0 R /XYZ 99.933071 496.020714 0 ] (cb9-178) [ 270 0 R /XYZ 99.933071 477.073692 0 ] (cb9-179) [ 270 0 R /XYZ 99.933071 458.126671 0 ] (cb9-18) [ 99 0 R /XYZ 99.933071 271.644045 0 ] (cb9-180) [ 270 0 R /XYZ 99.933071 439.179649 0 ] (cb9-181) [ 270 0 R /XYZ 99.933071 420.232628 0 ] (cb9-182) [ 270 0 R /XYZ 99.933071 401.285606 0 ] (cb9-183) [ 270 0 R /XYZ 99.933071 382.338585 0 ] (cb9-184) [ 270 0 R /XYZ 99.933071 363.391563 0 ] (cb9-185) [ 270 0 R /XYZ 99.933071 344.444542 0 ] (cb9-186) [ 270 0 R /XYZ 99.933071 325.497520 0 ] (cb9-187) [ 270 0 R /XYZ 99.933071 306.550499 0 ] (cb9-188) [ 270 0 R /XYZ 99.933071 287.603477 0 ] (cb9-189) [ 270 0 R /XYZ 99.933071 268.656456 0 ] (cb9-19) [ 99 0 R /XYZ 99.933071 252.697023 0 ] (cb9-190) [ 270 0 R /XYZ 99.933071 249.709434 0 ] (cb9-191) [ 270 0 R /XYZ 99.933071 230.762413 0 ] (cb9-192) [ 270 0 R /XYZ 99.933071 211.815391 0 ] (cb9-193) [ 270 0 R /XYZ 99.933071 192.868370 0 ] (cb9-194) [ 270 0 R /XYZ 99.933071 173.921348 0 ] (cb9-195) [ 270 0 R /XYZ 99.933071 154.974327 0 ] (cb9-196) [ 270 0 R /XYZ 99.933071 136.027305 0 ] (cb9-197) [ 270 0 R /XYZ 99.933071 117.080284 0 ] (cb9-198) [ 270 0 R /XYZ 99.933071 98.133262 0 ] (cb9-199) [ 310 0 R /XYZ 99.933071 799.173057 0 ] (cb9-2) [ 99 0 R /XYZ 99.933071 612.690432 0 ] (cb9-20) [ 99 0 R /XYZ 99.933071 233.750002 0 ] (cb9-200) [ 310 0 R /XYZ 99.933071 780.226036 0 ] (cb9-201) [ 310 0 R /XYZ 99.933071 761.279014 0 ] (cb9-202) [ 310 0 R /XYZ 99.933071 742.331993 0 ] (cb9-203) [ 310 0 R /XYZ 99.933071 723.384971 0 ] (cb9-204) [ 310 0 R /XYZ 99.933071 704.437950 0 ] (cb9-205) [ 310 0 R /XYZ 99.933071 685.490928 0 ] (cb9-206) [ 310 0 R /XYZ 99.933071 666.543907 0 ] (cb9-207) [ 310 0 R /XYZ 99.933071 647.596885 0 ] (cb9-208) [ 310 0 R /XYZ 99.933071 628.649864 0 ] (cb9-209) [ 310 0 R /XYZ 99.933071 609.702842 0 ] (cb9-21) [ 99 0 R /XYZ 99.933071 214.802980 0 ] (cb9-210) [ 310 0 R /XYZ 99.933071 590.755821 0 ] (cb9-211) [ 310 0 R /XYZ 99.933071 571.808799 0 ] (cb9-212) [ 310 0 R /XYZ 99.933071 552.861778 0 ] (cb9-213) [ 310 0 R /XYZ 99.933071 533.914756 0 ] (cb9-214) [ 310 0 R /XYZ 99.933071 514.967735 0 ] (cb9-215) [ 310 0 R /XYZ 99.933071 496.020714 0 ] (cb9-216) [ 310 0 R /XYZ 99.933071 477.073692 0 ] (cb9-217) [ 310 0 R /XYZ 99.933071 458.126671 0 ] (cb9-218) [ 310 0 R /XYZ 99.933071 439.179649 0 ] (cb9-219) [ 310 0 R /XYZ 99.933071 420.232628 0 ] (cb9-22) [ 99 0 R /XYZ 99.933071 195.855959 0 ] (cb9-220) [ 310 0 R /XYZ 99.933071 401.285606 0 ] (cb9-221) [ 310 0 R /XYZ 99.933071 382.338585 0 ] (cb9-222) [ 310 0 R /XYZ 99.933071 363.391563 0 ] (cb9-223) [ 310 0 R /XYZ 99.933071 344.444542 0 ] (cb9-224) [ 310 0 R /XYZ 99.933071 325.497520 0 ] (cb9-225) [ 310 0 R /XYZ 99.933071 306.550499 0 ] (cb9-226) [ 310 0 R /XYZ 99.933071 287.603477 0 ] (cb9-227) [ 310 0 R /XYZ 99.933071 268.656456 0 ] (cb9-228) [ 310 0 R /XYZ 99.933071 249.709434 0 ] (cb9-229) [ 310 0 R /XYZ 99.933071 230.762413 0 ] (cb9-23) [ 99 0 R /XYZ 99.933071 176.908937 0 ] (cb9-230) [ 310 0 R /XYZ 99.933071 211.815391 0 ] (cb9-231) [ 310 0 R /XYZ 99.933071 192.868370 0 ] (cb9-232) [ 310 0 R /XYZ 99.933071 173.921348 0 ] (cb9-233) [ 310 0 R /XYZ 99.933071 154.974327 0 ] (cb9-234) [ 310 0 R /XYZ 99.933071 136.027305 0 ] (cb9-235) [ 310 0 R /XYZ 99.933071 117.080284 0 ] (cb9-236) [ 310 0 R /XYZ 99.933071 98.133262 0 ] (cb9-237) [ 350 0 R /XYZ 99.933071 799.173057 0 ] (cb9-238) [ 350 0 R /XYZ 99.933071 780.226036 0 ] (cb9-239) [ 350 0 R /XYZ 99.933071 761.279014 0 ] (cb9-24) [ 99 0 R /XYZ 99.933071 157.961916 0 ] (cb9-240) [ 350 0 R /XYZ 99.933071 742.331993 0 ] (cb9-241) [ 350 0 R /XYZ 99.933071 723.384971 0 ] (cb9-242) [ 350 0 R /XYZ 99.933071 704.437950 0 ] (cb9-243) [ 350 0 R /XYZ 99.933071 685.490928 0 ] (cb9-244) [ 350 0 R /XYZ 99.933071 666.543907 0 ] (cb9-245) [ 350 0 R /XYZ 99.933071 647.596885 0 ] (cb9-246) [ 350 0 R /XYZ 99.933071 628.649864 0 ] (cb9-247) [ 350 0 R /XYZ 99.933071 609.702842 0 ] (cb9-248) [ 350 0 R /XYZ 99.933071 590.755821 0 ] (cb9-249) [ 350 0 R /XYZ 99.933071 571.808799 0 ] (cb9-25) [ 99 0 R /XYZ 99.933071 139.014895 0 ] (cb9-250) [ 350 0 R /XYZ 99.933071 552.861778 0 ] (cb9-251) [ 350 0 R /XYZ 99.933071 533.914756 0 ] (cb9-252) [ 350 0 R /XYZ 99.933071 514.967735 0 ] (cb9-253) [ 350 0 R /XYZ 99.933071 496.020714 0 ] (cb9-254) [ 350 0 R /XYZ 99.933071 477.073692 0 ] (cb9-255) [ 350 0 R /XYZ 99.933071 458.126671 0 ] (cb9-256) [ 350 0 R /XYZ 99.933071 439.179649 0 ] (cb9-257) [ 350 0 R /XYZ 99.933071 420.232628 0 ] (cb9-258) [ 350 0 R /XYZ 99.933071 401.285606 0 ] (cb9-259) [ 350 0 R /XYZ 99.933071 382.338585 0 ] (cb9-26) [ 99 0 R /XYZ 99.933071 120.067873 0 ] (cb9-260) [ 350 0 R /XYZ 99.933071 363.391563 0 ] (cb9-261) [ 350 0 R /XYZ 99.933071 344.444542 0 ] (cb9-262) [ 350 0 R /XYZ 99.933071 325.497520 0 ] (cb9-263) [ 350 0 R /XYZ 99.933071 306.550499 0 ] (cb9-264) [ 350 0 R /XYZ 99.933071 287.603477 0 ] (cb9-265) [ 350 0 R /XYZ 99.933071 268.656456 0 ] (cb9-266) [ 350 0 R /XYZ 99.933071 249.709434 0 ] (cb9-267) [ 350 0 R /XYZ 99.933071 230.762413 0 ] (cb9-268) [ 350 0 R /XYZ 99.933071 211.815391 0 ] (cb9-269) [ 350 0 R /XYZ 99.933071 192.868370 0 ] (cb9-27) [ 99 0 R /XYZ 99.933071 101.120852 0 ] (cb9-270) [ 350 0 R /XYZ 99.933071 173.921348 0 ] (cb9-271) [ 350 0 R /XYZ 99.933071 154.974327 0 ] (cb9-272) [ 350 0 R /XYZ 99.933071 136.027305 0 ] (cb9-273) [ 350 0 R /XYZ 99.933071 117.080284 0 ] (cb9-274) [ 350 0 R /XYZ 99.933071 98.133262 0 ] (cb9-275) [ 390 0 R /XYZ 99.933071 799.173057 0 ] (cb9-276) [ 390 0 R /XYZ 99.933071 780.226036 0 ] (cb9-277) [ 390 0 R /XYZ 99.933071 761.279014 0 ] (cb9-278) [ 390 0 R /XYZ 99.933071 742.331993 0 ] (cb9-279) [ 390 0 R /XYZ 99.933071 723.384971 0 ] (cb9-28) [ 129 0 R /XYZ 99.933071 799.173057 0 ] (cb9-280) [ 390 0 R /XYZ 99.933071 704.437950 0 ] (cb9-281) [ 390 0 R /XYZ 99.933071 685.490928 0 ] (cb9-282) [ 390 0 R /XYZ 99.933071 666.543907 0 ] (cb9-283) [ 390 0 R /XYZ 99.933071 647.596885 0 ] (cb9-284) [ 390 0 R /XYZ 99.933071 628.649864 0 ] (cb9-285) [ 390 0 R /XYZ 99.933071 609.702842 0 ] (cb9-286) [ 390 0 R /XYZ 99.933071 590.755821 0 ] (cb9-287) [ 390 0 R /XYZ 99.933071 571.808799 0 ] (cb9-288) [ 390 0 R /XYZ 99.933071 552.861778 0 ] (cb9-29) [ 129 0 R /XYZ 99.933071 780.226036 0 ] (cb9-3) [ 99 0 R /XYZ 99.933071 574.796389 0 ] (cb9-30) [ 129 0 R /XYZ 99.933071 761.279014 0 ] (cb9-31) [ 129 0 R /XYZ 99.933071 742.331993 0 ] (cb9-32) [ 129 0 R /XYZ 99.933071 723.384971 0 ] (cb9-33) [ 129 0 R /XYZ 99.933071 704.437950 0 ] (cb9-34) [ 129 0 R /XYZ 99.933071 685.490928 0 ] (cb9-35) [ 129 0 R /XYZ 99.933071 666.543907 0 ] (cb9-36) [ 129 0 R /XYZ 99.933071 647.596885 0 ] (cb9-37) [ 129 0 R /XYZ 99.933071 628.649864 0 ] (cb9-38) [ 129 0 R /XYZ 99.933071 609.702842 0 ] (cb9-39) [ 129 0 R /XYZ 99.933071 590.755821 0 ] (cb9-4) [ 99 0 R /XYZ 99.933071 555.849367 0 ] (cb9-40) [ 129 0 R /XYZ 99.933071 571.808799 0 ] (cb9-41) [ 129 0 R /XYZ 99.933071 552.861778 0 ] (cb9-42) [ 129 0 R /XYZ 99.933071 533.914756 0 ] (cb9-43) [ 129 0 R /XYZ 99.933071 514.967735 0 ] (cb9-44) [ 129 0 R /XYZ 99.933071 496.020714 0 ] (cb9-45) [ 129 0 R /XYZ 99.933071 477.073692 0 ] (cb9-46) [ 129 0 R /XYZ 99.933071 458.126671 0 ] (cb9-47) [ 129 0 R /XYZ 99.933071 439.179649 0 ] (cb9-48) [ 129 0 R /XYZ 99.933071 420.232628 0 ] (cb9-49) [ 129 0 R /XYZ 99.933071 401.285606 0 ] (cb9-5) [ 99 0 R /XYZ 99.933071 536.902346 0 ] (cb9-50) [ 129 0 R /XYZ 99.933071 382.338585 0 ] (cb9-51) [ 129 0 R /XYZ 99.933071 363.391563 0 ] (cb9-52) [ 129 0 R /XYZ 99.933071 344.444542 0 ] (cb9-53) [ 129 0 R /XYZ 99.933071 325.497520 0 ] (cb9-54) [ 129 0 R /XYZ 99.933071 306.550499 0 ] (cb9-55) [ 129 0 R /XYZ 99.933071 287.603477 0 ] (cb9-56) [ 129 0 R /XYZ 99.933071 173.921348 0 ] (cb9-57) [ 129 0 R /XYZ 99.933071 154.974327 0 ] (cb9-58) [ 129 0 R /XYZ 99.933071 136.027305 0 ] (cb9-59) [ 129 0 R /XYZ 99.933071 117.080284 0 ] (cb9-6) [ 99 0 R /XYZ 99.933071 499.008303 0 ] (cb9-60) [ 129 0 R /XYZ 99.933071 98.133262 0 ] (cb9-61) [ 164 0 R /XYZ 99.933071 799.173057 0 ] (cb9-62) [ 164 0 R /XYZ 99.933071 780.226036 0 ] (cb9-63) [ 164 0 R /XYZ 99.933071 761.279014 0 ] (cb9-64) [ 164 0 R /XYZ 99.933071 742.331993 0 ] (cb9-65) [ 164 0 R /XYZ 99.933071 723.384971 0 ] (cb9-66) [ 164 0 R /XYZ 99.933071 704.437950 0 ] (cb9-67) [ 164 0 R /XYZ 99.933071 685.490928 0 ] (cb9-68) [ 164 0 R /XYZ 99.933071 571.808799 0 ] (cb9-69) [ 164 0 R /XYZ 99.933071 552.861778 0 ] (cb9-7) [ 99 0 R /XYZ 99.933071 480.061281 0 ] (cb9-70) [ 164 0 R /XYZ 99.933071 533.914756 0 ] (cb9-71) [ 164 0 R /XYZ 99.933071 514.967735 0 ] (cb9-72) [ 164 0 R /XYZ 99.933071 496.020714 0 ] (cb9-73) [ 164 0 R /XYZ 99.933071 477.073692 0 ] (cb9-74) [ 164 0 R /XYZ 99.933071 458.126671 0 ] (cb9-75) [ 164 0 R /XYZ 99.933071 439.179649 0 ] (cb9-76) [ 164 0 R /XYZ 99.933071 420.232628 0 ] (cb9-77) [ 164 0 R /XYZ 99.933071 401.285606 0 ] (cb9-78) [ 164 0 R /XYZ 99.933071 382.338585 0 ] (cb9-79) [ 164 0 R /XYZ 99.933071 363.391563 0 ] (cb9-8) [ 99 0 R /XYZ 99.933071 461.114260 0 ] (cb9-80) [ 164 0 R /XYZ 99.933071 249.709434 0 ] (cb9-81) [ 164 0 R /XYZ 99.933071 230.762413 0 ] (cb9-82) [ 164 0 R /XYZ 99.933071 211.815391 0 ] (cb9-83) [ 164 0 R /XYZ 99.933071 192.868370 0 ] (cb9-84) [ 164 0 R /XYZ 99.933071 173.921348 0 ] (cb9-85) [ 164 0 R /XYZ 99.933071 154.974327 0 ] (cb9-86) [ 164 0 R /XYZ 99.933071 136.027305 0 ] (cb9-87) [ 164 0 R /XYZ 99.933071 117.080284 0 ] (cb9-88) [ 164 0 R /XYZ 99.933071 98.133262 0 ] (cb9-89) [ 194 0 R /XYZ 99.933071 799.173057 0 ] (cb9-9) [ 99 0 R /XYZ 99.933071 442.167238 0 ] (cb9-90) [ 194 0 R /XYZ 99.933071 780.226036 0 ] (cb9-91) [ 194 0 R /XYZ 99.933071 761.279014 0 ] (cb9-92) [ 194 0 R /XYZ 99.933071 742.331993 0 ] (cb9-93) [ 194 0 R /XYZ 99.933071 723.384971 0 ] (cb9-94) [ 194 0 R /XYZ 99.933071 704.437950 0 ] (cb9-95) [ 194 0 R /XYZ 99.933071 685.490928 0 ] (cb9-96) [ 194 0 R /XYZ 99.933071 666.543907 0 ] (cb9-97) [ 194 0 R /XYZ 99.933071 647.596885 0 ] (cb9-98) [ 194 0 R /XYZ 99.933071 628.649864 0 ] (cb9-99) [ 194 0 R /XYZ 99.933071 609.702842 0 ] (configure-default-service-account) [ 53 0 R /XYZ 84.259843 561.590928 0 ] (configure-etcd-user-and-group) [ 45 0 R /XYZ 84.259843 283.686971 0 ] (configure-kernel-runtime-parameters) [ 45 0 R /XYZ 84.259843 539.222079 0 ] (configure-network-policy) [ 67 0 R /XYZ 84.259843 599.840928 0 ] (contentsbox) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (create-etcd-user-and-group) [ 45 0 R /XYZ 84.259843 169.788971 0 ] (ensure-that-all-namespaces-have-network-policies-defined) [ 67 0 R /XYZ 84.259843 564.092928 0 ] (header_bottom_text) [ 6 0 R /XYZ 85.009843 422.718898 0 ] (known-issues) [ 45 0 R /XYZ 84.259843 799.370079 0 ] (overview) [ 30 0 R /XYZ 84.259843 272.631496 0 ] (reference-hardened-cloud-config-configuration) [ 534 0 R /XYZ 84.259843 537.158799 0 ] (reference-hardened-cloud-config-for-red-hat-enterprise-linux-8-rhel-8-and-ubuntu-20.04-lts) [ 557 0 R /XYZ 84.259843 511.863778 0 ] (reference-hardened-cloud-config-for-suse-linux-enterprise-server-15-sles-15-and-opensuse-leap-15) [ 534 0 R /XYZ 84.259843 397.010799 0 ] (reference-hardened-rke-cluster.yml-configuration) [ 77 0 R /XYZ 84.259843 191.441066 0 ] (reference-hardened-rke-template-configuration) [ 390 0 R /XYZ 84.259843 518.211778 0 ] (set-automountserviceaccounttoken-to-false-for-default-service-accounts) [ 53 0 R /XYZ 84.259843 522.092928 0 ] (title-block-header) [ 6 0 R /XYZ 84.259843 502.818898 0 ] ] ->> ->> ->> -endobj -4 0 obj -<< -/ExtGState << -/a1.0 << -/ca 1 ->> -/A1.0 << -/CA 1 ->> ->> -/XObject << ->> -/Pattern << ->> -/Shading << ->> -/Font 632 0 R ->> -endobj -5 0 obj -<< -/Filter /FlateDecode -/Length 2586 ->> -stream -xœµš]c· †ïý+|`J¤¾€"AŠAS À¶APôb3Ø$¼6½èßïËcëÃãw?²iƘ‹Ô9⑨‡¤ìx¼^DüiCk½;?½9½=I¨Yðsô¸kê¿á·æs«!Ç^Z>¿qÖP{Nµœ/'üyÖÜ»_Nß~vú÷ý5ñœBï%Å|þ嵫LCmµJ&ª•D%ªœC­)¢*1à{g*˜‘«%{TU˜a½e¦‚–R#ª3ÔjÔGU‡©Fc*˜{®DvÄ$Bu0DÌ4]Ì¡`Ò Ó¥Jë©S]….ÖÄtª¡T‹92]¥”ܘÎ`Kî™.ÖÍ„é`‹i«LW`‹ªƒ-©Ueº -[f§]ƒ-ð®Fu°EŠD¦ërofTס“VÛ£.I¹© -ÓÅ2|S©; -‹T*Ñ%ØR$wªƒ-Y%1ÂËéÀ‰¬µ5²[“ïnL—aKJÕ¨¶`ñR-R CvPp.1¤Zƒõd ©i°fQ µ¬VeH=+½2¨DèRd P-Ù -#FØbعT[´c&Ø‚Œª°%™2(hkðÆnMzd,PðÖ$Æpa¦0(ˆ«­c¹p¥ÎX `®‚Œ -èb»WÆu5—ÄX À®Z+Œ -îªEa,0pWWRl3€ŽAcSÆw5Je,0pÛ(1`Uü°0ÀWBjM ÜMMŒ±ÀÀÝT¨®ìAe,0p7e<‹ÜànÊ’ ÜM– -c%€?B³pÉ<€-,câ=tI p³–0dp7v„?²§A«[/ŒÜ ËGu=ÄŠ'û³b)± ƒ»TÊXÁ]à³1€ñ51dp7*"5Ó»1ua,Èà.ò†ÌXw câ[ˆ×¥º¤·ÆXÁ]`)1 -i ÕÕGŠŒ€qä¡ŒÜExïŒÜ•¬ÊXPÀ]1¤=T[´%Æä5ÐaÛ’=d5H)˜Ü•ˆf:pW¤vÆ‚îzØd,(Õ©1ÀÀ€,81”f¡×Z JèòQ] hgÆ‚*d ‹ê`‹Uc,@bºöÆXPlÑ” j‚-H „‘}Óò¶À]h}`°™"cHZ7c,¨¹ ì2]I¡Uàšêt±2Ôj¡82 – - ›ž±€ž.1À è¢1Ô[°U ŠBKÈÌ™.Â$ÙŒÈt¡‹‘± %Ø‚l± ¥†*³c"4jIL5Ó™xÙ š•à©"c¶ *M’ê` 0ÁX€õñb;3 øA±‰2‡ê`‹fc,h-y¹Ù Zƒ-Ic|ÓëÍÊXж ãc,èR‚§ÉŒÀ8t"Œ=6Ô›É °/½ÞlŒ]õ&J<²o»¯7+cA7Ø’‘¢PlAÒÔɾ“Po‚NLW`KêÆXÐKñz³1 Í@½‰Œž 4¯7+=.ws™˜ÜEfZ Ð@½‰Ì–ÕöðæŠüŒ+›WœžЛ[é‘€½ˆV•žàë5=4€™¨,[¡§«PF¡Çþ"³&\OŸö#ªw¨ˆ™­úCvl Cv?kŽÕMßÑíz>ö˧Ï_!øžüÏé‡ÏŽ“²ñŠp~µâ…uñs8Q“ó›]nà† &´v¾@®’o÷¹¨ñ‹2òñÃhC™ž±]zêÑæcƒ¡¼»ï4üÔó㛇œ/_žì*}á«…º#\7!µÏç ™êùå›ÓçúûŸ¿ýê/X±—?œþù¯ðá’¯ñÛf©þËtëûûLÿÛS9"z<ª:T¨rüØ[>¨Ïg?{Ãâ˜îxLP<¿ãÊã°üíìu;?/©=º®o[ÿ·ÇB ˆÀþŒžµ\Ϻä# ççb`¡øÍ£L)váí­Ì{¡ôLÏdO§ÏëÏ{¢€•ÛÛ>o®y¼Å>*ëú<.ó„4+–ûîu`"Ä•%>ŠýéÕåUìçÅ-è­U¦)Hªû®ÚÞwÝp]øAqTÙ„ùQ(ÛpKœlؽ¾˜SW–pš¾O]Ü–ôMu[Î÷ ŸNVŸK/Tê§TãVcvŇL7£ú”æÑs.ˆÍu"¢§¾qkÃbNý¶jžë_…m8’Í~}ô#¢§íw»[ž+Ug¯¶.ícæ—)Ëյ͉'²cŽnÒê€m{Dt¼½_±ÖH¼–ߤáÓ y©=ˆ/§\Iï¼X!y¸Þ.Ôí¾SÜdùË’"?½M¶æy¦ä¯¨ÈSêä9Þ h—yµ³Zkß›S|?I»¼Lê›huõݦ+Ö:}vŸ¯Ø„ôwèÎÞsÆî¤Û”mòÚ‡y© {qœ^7ÅŽ°1?“•»P†n‰ótåËxÙ™çVÍ2+qs÷<ï»Äe²ó²‹ë\ñè¥ë´wÞwóù!|:›=»ë·1È'™Ñn ¶ —a€Â/Œ^ñØ­“NþqÜx»6‹¸O§ŸNG˜KšS÷Œu*®=[Šçÿ¢Ï×øýùdçoFêãI&-Òdëxdþ¹]ûH±Ž7„sÛlÂdö—8¶Õ7o˜q<¥×¶.—öÑ3N‚Æ©Ý…yÍá&NÓuSÔmfò{„OÏfq&2¿umVÀ¾6u<†Q/‡Uhìƒf[:]!ù¼Ý-ïºB:´–¶ ÑØoŠæ6`_–Ý™é¹2§Þí·})ÔÇÚ[áM]:[av¿õO±uA -¬{c¿±Å}Ð~g1ú“·×o^÷›Éeì—N´Oée—Îsé—êÑ,÷Ísôì±y× Í{é¶÷t&9kk¬ÔÂî»nÒ¶.»uÈû 6dwƒŽ@1q™ Ù¼c6•ÛecZŽ'nt–êÌ)–ô§}¦ó¾*só¾*塯ßwD¼™-éL€òîÎu=¬ž3—lDVö«§ÔVO²Ç±í~ìmÏÛcØ2¤¬ÌTÖeù¡ç'ñïSö=Ða½Æë8шm–!P*z|ÿIZ)`ˆùGXÕÚvÄñ«¯yÿ™ÇãÉÆ8óØGÂrS•ý›lÜ„ ýÊíVã[nü².ô‘~ݼóÔ‘¿Åã« ø¹ìÌfõRÑ¿¹¤žô6ÿÀµJ¤G<ÑîŽNŽ×ÿéò² -endstream -endobj -6 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 5 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -7 0 obj -<< -/Filter /FlateDecode -/Length 3524 ->> -stream -xœí]Mo¹½Ï¯èóîå÷ä°Ø É"°€³‹ ÈA;°wHìò÷Sìi²^M“’5’ mKî~îbÕ«G²©i^ý}§é[rzM)Çà–óÃéóI­Ñ+ú³µ'ÛË?Òþ5Þ­ÞDïó¢£^ULÑ©åᤕ]­ ÎGÄï%®vÿù›Ó¿ÅEš“O†½^uÒZ™Å‡5d“µ]¾|üŠÖZ«5”?q9|ùõôíõ}ùõ?§Oßl¡‡hÖ¬¼×f1&®1êàÝ.e‘’hâ%ÜÐpk'óÍâ{¸|‘Iì/bœŒQÂô”öæB­¾ƒ6Ççâ!¬–mžØ -{¤Ã½5öѳˆ‡ñ’GD\ãÄPkÍжé&XàŠ­°GÎT?ß²ßH¶à!J‚¿Æ±d¨'ÊQÁƒðx^µit"š}#‰ö¥øü…îHºS¢S¢S¢¯¶Ðu¡?‹2.¥èuEüZ¢ÖEXMÃEŒ ‰Žâr¹³‰Ž¥D“¯Û+R¢ ¶@Ø#%ÞKƒQYŒK‰2ŽbL°•¨GÛ¦›`Ä9zôÈ™êç[öÉ<ÉC ã¢dòj½âÁ xœ†‘£D³êñÀ¨Œ‡q)Ñ [k\ÒìQ ‚=àA#æ QÈT?ß/‘(Gó0îXŠ¢š‰)Ñ)Ñ)уDo½E‰bOdô¡Ñ”è”è”èI´/Åç/tGÒ}­…®#ƒ=‰.¤è´î΢¿–¨SVÓpãB¢€ƒ¸œÊM$p,$ꜯÛ+÷‡-öH‰×‡ÒT”àB¢€ƒ¸À£@=Ú6Ý#ÎÑ£GÎT?ß²ßH¶à!HvÁ.J&­é Ñ+Ì€ƒ<äƒDW=•ñ0.$ -8–4{¨àAxa´èÑ#gªŸï—H”£yHw,EQÍÀÄ”è”è”èA¢7Þ‹ -‰bOdô¡Ñ”è”è”èI´/Åç/tGÒ}µ…n}‰2.¥HÅÞE?H”Úé^i0.%Ê8Š+¤Î&8õ”?xk8à°ÒJ ¸(à(.“:›HàXJ4¸î#Ý>À{ üÆ}è_è¿p)Ñ:CàQ m›n‚7ð`8{äLõó-ûd ¼ä¡ -†qQ2©óH÷fÀà±óH·¹ËCð%U¢¶Ö¸¤Ù£@zÀƒFŽtC¦úù~‰D9š‡tÇRY&¦D§D§D½õ^%*z.¢ˆ¦D§D§DßH¢})>¡;’î”è”è”è«-t“ïK”q)Ŭú³(≦ÀNð¢$~õ †qWJM$p,$Œí>Ò l4Áð÷¹€ŠÒ\Hpx¨EÛ¦›`Ä<ÎåLõó-KÉ<œ¾ýî¯?ýýý÷ |:ýãwJéðû.~8ýá™þZã°3ÊúÅê-ACãñEÆ£-;,¯mÜÇÔâ.Ë34žn2]°Ö,îò{–‡Æó ÆK.ˆ-µøË/ο{¾qÖ`¢Îz —'¡‡ÆÝm½w†DœœÓÛèü©ž¾¬pÐ5î.è;ú?É×’þ¬uÉnÆÿø·?ýüý_ظ JyC_š¾¢R.ñ±ßu½ÖšDtë"²ÅhÞ úýø¼Ÿß=fX -躓®Ú¹£óÝv±©,µûþôåØ}º¼fýÞ¾J6ÊG4×k-}9·ÛsüZKŠ*!6ú•F§žÙïƬZM¦R"ã«ÍFû²(ñ*'å -Õ¥|üjTûXéw|ª3MvN%e®zñíûßÿù§÷LK ‰ºæx´h%‹ÔXš}w×v¯é.)Ü -œ.USZÂûùÝžJ³··¥ÔYWÆüÿsJe/Ž)Ý+d뎻t¥T:½Á®1[oõ^z-¤¶©)@õ»ý<@z5÷cb1.?GmVs9ù»ýšòúGVTÁJ.j_ýù±¾xO A·õ[ð¿õ cÿÔ ÃØvmä ŒæµJÄjâ,¹ìÌUÏ%ö’Ø‚ƒŠ.ÕÁèÓS%ãt\õÓækŸ½’y’JƒáÖî1zäDø¶ôпÚN•Qb/#ǹ{ƒøÄ<üºñüšDÎ2¶¿­^vZ´E¼Ûå÷‹j’ŽÛ/ÉÛ¾ü¨kœèÉ¡Í´ðwÉ“‚h©³71,_>ž~¦{Ú¯hRúLcàJÞbŠ…³z’cò¥Åq¡ói÷®—òw¼¾"“û]…q…àH«çî:åªì†Tž÷ÿïöv{º¶ÔAI–‘pk[°_8Ýæ|Á·6y_6ê²tÜÛ\—Ö/ûPûòñ°Äü|ò‰Š—ˆ”eº1§8­óÆ,º|ʆ/žî°ƒ -ÉÔ|ë-Cz\z~Ø ×Vå;A4"ZMCÝv.Nöö—õîÉ*z ûJƒÅåˆ`ræ¯aºM ™úp„é6Ð|%\ÀýP5[N·»bt¯jËýñUKëWUçûfܶÞøM¹Þ×Ëèv†èkÓ€)Å{qÜéÊ p‰¾l§\`"Á§Õîg¡uŇÚg٠öl/|ÖVè w “–rmkÚanœغŽ©Ó@mk"Ðùx>‘[{ÞwQ£fjvj˜…Nå†úÚ²âO¨ìMÀwªšvDf{X3kÚÁT ɵv¹¶ë@çS·ÝnÍ7¦bk•øÒ\3Ï]áR·©%¾ƒm9ÚQ&îá´Ü`šÊ¡0P¹V êÆÀó©Œ;øþäc§µç±BùZzZ°Ûस^©¦Š¤Y±rO^¾_† -ßÐ2òlÚµ’Š ÏØIÑfƒe’mÔq½ h¹-¤KÇØjó¥“ê´/ƒnkÝ2&PHà1×î•Aª Ö­ê\†°šŸ6V"¨j'Èþ•ò½€¹Ÿ¾I•Š½: ÊÝ7» ‡6vÞ#ã:¸Ïj¾‚çmbs×p)ýT0lT›íØ€Ü1æaô~ƒ«ZÛè¤iUYd½÷|úí´MsÆz“ËÏ6,Õ´Ü8ÕË©Íôõ¯“[Þ×ÔË> -endobj -9 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 671.061024 504.679134 638.361024 ] -/BS << -/W 0 ->> -/Dest (overview) ->> -endobj -10 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 489.361263 663.111024 496.429134 648.111024 ] -/BS << -/W 0 ->> -/Dest (overview) ->> -endobj -11 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 626.361024 504.679134 593.661024 ] -/BS << -/W 0 ->> -/Dest (configure-kernel-runtime-parameters) ->> -endobj -12 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.881526 618.411024 496.429134 603.411024 ] -/BS << -/W 0 ->> -/Dest (configure-kernel-runtime-parameters) ->> -endobj -13 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 581.661024 504.679134 548.961024 ] -/BS << -/W 0 ->> -/Dest (configure-etcd-user-and-group) ->> -endobj -14 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 146.805601 573.711024 177.256773 554.961024 ] -/BS << -/W 0 ->> -/Dest (configure-etcd-user-and-group) ->> -endobj -15 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.881526 573.711024 496.429134 558.711024 ] -/BS << -/W 0 ->> -/Dest (configure-etcd-user-and-group) ->> -endobj -16 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 536.961024 504.679134 504.261024 ] -/BS << -/W 0 ->> -/Dest (configure-default-service-account) ->> -endobj -17 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 146.805601 529.011024 197.845152 510.261024 ] -/BS << -/W 0 ->> -/Dest (configure-default-service-account) ->> -endobj -18 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.893245 529.011024 496.429134 514.011024 ] -/BS << -/W 0 ->> -/Dest (configure-default-service-account) ->> -endobj -19 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 492.261024 504.679134 459.561024 ] -/BS << -/W 0 ->> -/Dest (configure-network-policy) ->> -endobj -20 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.809749 484.311024 496.429134 469.311024 ] -/BS << -/W 0 ->> -/Dest (configure-network-policy) ->> -endobj -21 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 447.561024 504.679134 414.861024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke-cluster.yml-configuration) ->> -endobj -22 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 234.738707 439.611024 313.229430 420.861024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke-cluster.yml-configuration) ->> -endobj -23 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 489.877620 439.611024 496.429134 424.611024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke-cluster.yml-configuration) ->> -endobj -24 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 402.861024 504.679134 370.161024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke-template-configuration) ->> -endobj -25 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 484.970394 394.911024 496.429134 379.911024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke-template-configuration) ->> -endobj -26 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 358.161024 504.679134 325.461024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-cloud-config-configuration) ->> -endobj -27 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 210.896178 350.211024 288.867613 335.211024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-cloud-config-configuration) ->> -endobj -28 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.993831 350.211024 496.429134 335.211024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-cloud-config-configuration) ->> -endobj -29 0 obj -<< -/Filter /FlateDecode -/Length 4932 ->> -stream -xœí][‹$»‘~¯_Qφ“G—Ð ÃcÃ,Œ}°Y–}˜)Ÿ¶15†±ü÷÷‹¬”ʌ̪êžðNSÓ•!e(®ŸBÊKÛ³ÁÏwÿe²SÎ%E:_>>ŸÌ”‚Á¿¹Çp07¿Ç'…³5”¦`KÌáü‰ >M©—âùz«CÙýzúãÏNϱg7• ç¿ÿÈM䧔S2Ai*h2Ö+M!L)¹•¦h'¨XŠÖ1B"GÛ¦1¨ä 5A r.+MbxJÖo› -ÄpÉ’Ö1l Ii²rXgŒÚA ‘wJ› S„Ñ£ÖæìsqEmKh³ÉimÞO1‘ Vk+SŒ1d­ K(Æjm²Kd´6ÈB>'­-B½QÛ ‹ËÉkm ²8X§(m² º²ÚYL4Vk+~ -%©mm&§¼ms&L!{o´6k§€Øôj2 -NŠIis%šPÔ6ȼqZ›‡,„­ 8|ÊYÉVÇÙí·Z[€,Î%RÛ œ¯!€‹Å¤¨A2hb\Ò0À¥4Qq¤€Ë~¢LYC—ËD)y \ Å’4ðÆ¢ÍY ¼,¢†ÞBBæªmÅÒ°À;È‚xYy <Ж-xÀ-™b5,ðÀ[26jXà¸ÓhXà¸>GÒ°ÀrJEÃÌõÀN <@éž4,ð@]¢Ó°Àv=å¨aîz²FÃîz3Õ6È‚Ö°è8y›½†ÜõÖ$ ¸‹4rÀ«â0jX€X™\ÎFÃîºlHÃ)@mKrÐkX@À] ‚6sw]0NÃî:rQÃÊü1j5Ü„½€,X 0ߣÍy Àlr(K4,À][0ý)9 ´šl.QÂܵîSÛÊd,®ä-¬<Ù‹†¸‹‰ÊkX€»€Ï¬a0~ÂŒê4,À]ë1SkmÀ]ëŠÑ° wQ7 “E aæ·É„®ÚV&SrÖ° wKNÃÌ“É@µ-M$«aÀx2¨C5,ˆÀ]LïEÂÜ5Á{ "p×ʵ ²øì4,@]ƒ6¤­’Ó(V'ã€Zp×XxXkî“Š†¸ËÓ¦†11Áe à„*ØiX3M%¥¨aA,m(ùÔ¶8á8hX ¥¶AJ¤a -ë©ø’5,H²xç4,H² dаÁT,ªouyY.êú€ *E €dS.D¤±ƒUµ¶è¦œ×j[F›M¤D¦b«aÖRSFÒkX€Ÿ¸\Ò°b Í’†©@¤Š†˜Š¦ìP™km² ÈÖ°•.Ú¬Õ° ;È‚jPÂì2V™ÑhX€kI˜Zk#Ãë̬aA¦8q©¨aÒ+M ¥ÚYÀ?¼Ø`ñƒÅ&–9jdñ4,ÈÙñr3kX3dqÆiX€ØäõfÒ° È‚ŠOÂbâÄe²†€q´£aA±ëMG /y½™5,(Þ`½‰%ž’·ÅG^o& -A–€Emƒ,(šŠ’·À$¬7NZ[„,®†%F^of Pf`½‰Š^ÝȼÞLêvp7 “ÖÜEe5,ÀÖ›¨lµµ½ð†„úLo̼âÌêŽôÌ­ê–öb¶JêžøòIÝ4€˜XYæ¨î`ZE£5ê¶þ¢£ &ÜvŸäUD8$Ì™9±’iA¨îÛš£wó;Ýnûcÿóéû˜|ÏþÇéågóNYý±uOq^X#ïbqþþtBI4qîz+é×S$6ØšÎrÖÞð0)TWû^N‚îP#;9n¯#=¢Ž˜ébD@Bí-äÔ‹ÔGЯ ŠÎ5cïÝF¨^ònt¡Ïu¤7íňÂRº½/§?ÂõŸN½•Àtö.Í{jÖ²DsÀà$öo*Ýÿ3βŸÉþŸÅ4Pvûe -ÄÄû1ÊN,”1ŒYÓ1›w¯sõ¹¥±`wbAÐy£ÅÂ2"ûK‹…¸ qŒy+ºÅ²¤õn#Ô!üN,zÓ^Œ(,¥Ûûòÿ9ùÉI"}öãQK˾ÕGgVõÑY¾Z!‡Qý#è×ÞÑGgv‚Å“°±ÖOõà¯ýüÒðf oõÓ™Ué,_é¯A£Æßà/=^ïè£3;ÀC鯷âa«° úà?úV?Yõ—Îò•þä0jü þÒãõŽ>:³|,¾¾|}?ü=ºàsˆ -¾‹Ö·ãû Ï~~iø9зúèÌ|ÿú9ôùWЯ;þ¼£Îìß‹/‡ïƒ¿öóKÃϾÕOg¦àûÛý5ȡ׃¿ôx½£Îìߥ¿¾¾þp_¯k÷è[ýtf -¾¿Ý_ƒz}1øK×;úèÌð]°x¾oך¿øádãr7\<‹”ÆK`¾]é‡O§ïñ_øïw¿<ãàåô?ÿaŒOÆÐG|.ƾ¸àñ´àðÇ!ã2ño‹Oºõ ü ·~Pb>ŸÏ›ûűO¥ÍüÂ2–¿ñš¿—[Ÿyì·1?ºËç¶>e7,Ç—~AÈ`ÞöÆsîG?ÿßó¿Ý±“5ÒPl©ãl“¼èPåv7þ3ïe¬PÛ!½,ÒÏ™ù–*Ûí·ÇxîG|ª®ñÆ#dÁ;wÝÃ⫹oè6ªýù˜ªÏ>,2,>¯ã 6þ¸âÏW:ð1kl¹ýfº›ùÚÓmìé¹Òb—ê3©ËPõ¯6£gKŒ„ Û¶ÅqÕ+myÍv«¾}ã×x¥‡•Öì(mNKÌ¥1ÖC<´ ­mÃ>ŸÇ ÝþÍ6«¼j}„Ïæ­9ðããúιTóØè>pqÌë\tÕ§‚ÞdüthŸ¸¶¯±\1©âëu¹m>Þde¸¿¹p¼ÞÆùàñÌ?ÀÌTF3𳜧 éb|+Åž5®Õø¾•øH"–ê8³Ý>lc0h¶Éº/gº-VßäÝúD‡¤qæ|®¾”ñp„W/;y¹ø±}_åb‹cc†¹¨ê æeé±>˳²ÝÆÿÚ"®mQý%ã¾ùøhnXÅÉ£8 âºÕ»%€wX#Xž¨Þ×’WÕ_ò9òÝyl9Ï̙۹á‡æ{F›áf?SÅwÓj¤YwêãÍ·óy¾Çõ|n×jÜ—Q^mn“µƒ†× Ÿˆ©¹ï#sËË¡ÍâÚf¿ƒG5ŠýÓV×·l\ɲÏëßðÙõц‹}Û`#ÏÆWwíÈÿ-rRìËe·‘ó‰zbLiãœö•V›ÚeÃû¡8¯cÐîÒfi3Æ6Æáßþy‰‘‡2”µ _¶Õa]ˆ”ø¬óÔl§Z[ -_ÈZ£axêsî¬ó2×zyXóp܃æÊ}üÛž‡×¯mÞü¼Ìœ¿ûõéûÿ\Vу-æ«ÉΖó§Sà‡~J0ÅHú•2+%›pþçé÷,Ë6t4'[%WÈMü|›ÏghµÎ·²[SøøñΒ¯ë:;€•ËÑ[#éOë,ØÃG:»{k½Üç³u>ÑKŸ+‚\«\ºÿk¬µ'C𨽢ö"òSñ9/éo²ßœ“¼óo±×ú8šlòÞeö=&ÀF6NÒ7º|:~ðoÞÛ"3EÀ ß}W&FÓn@zÅ)÷îY*É*çöÅ*7+¬'à0óä‹ã›>Í„<̆6®uø›GÚ!¯›MǼ{ÿî7x×Ó*öóæ€})¤W†‘¼1›xß­eSq“->ø(éÏgS¡Îæ0Úä^Í0G˽mZŽ£È¦ûuË „ˆB.ä]áõòa>“Qé‹Ìá^ÔÁsX÷L~ìótÛ¿Yl.gbó²'tˆIú›lPQÓ‹½ Y#¯¯c¸½=±‡0îm<¬‘¨ŠºÄg*ä$]A"Ù1µ‰Ùšb ùùŠ-=ÎO‹=Ñù]}”õ6;ÓÐÂ5lÈr&ã7§çÉü&õJli#ˆŽìw²Í½o¨ÑÐæqç^S?Ýt`¯=mCPÛZ%1t -²k¡ë¬– ÄËÊŠ­y«oú„};ôU9 Ãï·jRñcÄBæÞæû”|,qÄ]û”ŽVrâDH¦8–.Ù &¤¤™žå'é -Ïc‰QFð¡ïmÔ§YɺÒf-D”À^HÆdå eGúÕéõf¿/"ÇšLmÐÞ¨WImŠ1µžNõóaÛœãÏ|P“·[¨ñò"÷|+rzjôÒ‚†£‚šûiK‡ ¥J­Eƒ ÐL¥5Ž©Ê—ÓªYf³j¥ÔjŠNý‹´t^iF Ò+qÓ—ùÖ¯UK¾@A†sêÚ™Þ³Õ’Y¡Eyv£Rïé+m;6c×Ùö,Ô .H앩駅MÏWáßkrGݲÔz ;6÷וF~0‰ocšÿÉ1Cˆ_"ž(‹-Ž§Ï9ÞóØîlÔ=9’å½HïM²ü·„t®h(÷«úw†ôÓŠQUzÎÊ–N‡¹í–ÕüÇ$ðoÙØi‡‰—Šü·c<½™_yŸÌÎ~sÜì^½ÿ?† <û -endstream -endobj -30 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 29 0 R -/Resources 4 0 R -/Annots [ 31 0 R 32 0 R 33 0 R 34 0 R 35 0 R 36 0 R 37 0 R 38 0 R 39 0 R 40 0 R 41 0 R 42 0 R 43 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -31 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 445.281496 409.271317 430.281496 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://releases.rancher.com/documents/security/2.6/Rancher_v2-6_CIS_v1-6_Hardening_Guide.pdf) ->> ->> -endobj -32 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 418.281496 162.214732 403.281496 ] -/BS << -/W 0 ->> -/Dest (overview) ->> -endobj -33 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 403.281496 332.953990 388.281496 ] -/BS << -/W 0 ->> -/Dest (configure-kernel-runtime-parameters) ->> -endobj -34 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 386.659914 295.486949 371.659914 ] -/BS << -/W 0 ->> -/Dest (configure-etcd-user-and-group) ->> -endobj -35 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 169.605601 388.281496 200.056773 369.531496 ] -/BS << -/W 0 ->> -/Dest (configure-etcd-user-and-group) ->> -endobj -36 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 367.909914 319.724254 352.909914 ] -/BS << -/W 0 ->> -/Dest (configure-default-service-account) ->> -endobj -37 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 169.605601 369.531496 220.645152 350.781496 ] -/BS << -/W 0 ->> -/Dest (configure-default-service-account) ->> -endobj -38 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 350.781496 255.796275 335.781496 ] -/BS << -/W 0 ->> -/Dest (configure-network-policy) ->> -endobj -39 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 317.031496 400.593883 302.031496 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke-template-configuration) ->> -endobj -40 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 302.031496 397.449596 287.031496 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-cloud-config-configuration) ->> -endobj -41 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 233.696178 302.031496 311.667613 287.031496 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-cloud-config-configuration) ->> -endobj -42 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 293.575760 126.231496 442.798661 111.231496 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/1.6-benchmark-2.6/) ->> ->> -endobj -43 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 111.231496 285.368241 96.231496 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/1.6-benchmark-2.6/) ->> ->> -endobj -44 0 obj -<< -/Filter /FlateDecode -/Length 5113 ->> -stream -xœÍ\Y«É‘~?¿â<”Ê%r30Í ƒ è±hc?¨[Ý WÛþûþ¢*—¨ª8ÛUãÛ-Ô:¹TdDäKe•;[üyçð¿BΔRs¢óåËéo'kr´øoé±¹Xš¿ÅßÏÎR6ÑÕTâù B6¹FŸÓùå„ÿí.e÷—ÓïuúëvŒ;{Skò.žÿþ7Q0¹äl£ÒTÑd]Pšb49û˜”¦ä –X«Ö6b&OǦ 6¨–¨5 ò¾(MlÊ.›*ØðÙ‘Ö6\Yir|8o­ÚF,QðJ›‹&AèIkóΤR}UÛ2Ú\öZ[&erÑimÕ¤”bÑÚ¼ÄjÖÁKtDVk/JÖÚx )Xµ ¼ø’ƒÖ–Á‹‡tªÒVÀ ¬«¨màÅ&ë´¶L¬…Hm«h³%—c›·ÑÄ‚ÕÚœ3¶Ô6ì(()e¥Íƒ—dcUÛÀK Ökm¼þhmÀ‰r)Ênõ¼»ƒ Nk‹àÅûLjxò5ð ¼Øœ4À2ŒKøœ UOø * -øR å4ð5J5k8¬C›w ^"% ‚/„«¶—PIÂàÁ :hXxñ4,@[‚µhX·d«Ó° oɺ¤aAà‚M«aA→HÂȅ)U 07;5,]l÷¬aAꆘ¼†°¨$ p7³p7ŒTÛÀ X࣠® ¸œÍpÛÈkX­ØaÒ°¶b|)VÃîúbIÃîú  ¶eƒ=4, à®X‚æ¹»>Z¯aw=ù¤a Ŭ w!QÀ VÃø{´ù a&3a‰†¸ë*ÜŸ²§VÆ•š4,ˆÀ]W >µ­—!qeßBÊÆ¥T5,ˆÀ]8ª aAî>‹†Àxê5,ˆÀ]ੵ6à®óÕjX»ˆ¢†0ãhXÿfœ…éªmÕØZŠ†¸ XòÀ [€$j[60$§aÀØXÄ¡$à.Ü{Õ° wm AÂܵ„°Gm/¡x ×  ÛVÙÓVõ@ -­ ¸k4¬µw­ÍUÂÜe·©aAÊLðEÃ0h{ R!SsN¤jцOmK×QÂlAˆ€,µ ¼P& X›jÑ° {ð¼×° {ð‚AÃD¦:Dßjz^`.j~@à‘¢†@2S*‘†9&äcªÖ–¼)p­¶´¹¬aAÎdàŠ†È¥LÁ¦×°o8\Ò°l Í‘†¹‚l àŠLñˆÌµ6^dkX€HmÎiXP~>ýé×Ø?X`þ"ŸŒŸÖ߈Ԭ¿àwj1)þý>üí ȶVU±VX×T”÷×ÈA߈çF’ã\ç$ÌCß·y–’Æa.€5×5¡:̵Œkãy.æ1Š9é3ÿ=ÎY¹àVB±Êœ<Çà/«<lK‹h+iòŽ¯ÍýoÊ&ô×ôÎöÕïûúã”;u{lû·÷[ø²7m‹Z×ärãQ¬íQý/´,äØxŽÔäy4aÓR~KŸÚtœ»­oå¾®ç¦üÊGâœëy]m×ó¨bžs¼N'·ÖèõÎ!ê´w5žå^È÷x\ï÷ßáî(#l[ѤfãWwØ9;äR\ó7~u:ÃE>?æU²9Fã9ý„lL¨žÃuÄ`¶"ýÚ{Kû]W1ŽK³ñ»•æûß~øÍw¦»æÑM³¹žE|~ ×Ëuš.+JóhÛc3F@Ìhëf`×~K±%¸O7[÷—¡¶,ðª§Õ €÷ñÊØ­sÌíµ3ixÐQSmåGY‘f Oy[cpˆ§;»!ß2‰a‹°¢PàÀ‘{Jñî û‚h?U)#6{4..SL ;~èS"§îѧǼñårmÔÄô:µ`¿Xu7 B£ë*#ÏyTq%ÞUY‡z(œìÖ½xÑ;7!±DB¸Ž%7]D<„ -æŒP¶¹dûýêj9 [\îEÛ#™–çfWîsÕÅ -^%ýÿszÿŸÍä¼ÉÁ*³CúåFÄjÍ’þÂç9*,*žÿyúݵ|IFöbš Ûê¶cýõpæfZóù>=èa„ÀóUöÇÐ<RJNd‡(t‰9¼ Av®þv -çó!\º9 ˆûP•±Æ/¾QìÅ.ƒ¯ä™Ÿñ²9y§Ü÷Þ¾ºåÃ7¡R6›Äxjשݧ…Y1[=„î¡ëHknê jiæ!ý‘©/ÿþa¥»ÒúõþKÁ‡œHÈâß,»MjW.8}Kn|”à˜*Iëé5Ý°Q®ÏAqË? y­1ÓÄ摲‹Ô}ЫÀGû¼Ì¥½ùMÊÝÆåÉÇLÕnÊ1äø6ÐK;½}‘ž²ß¤R=ýì÷ú$üã®û~ùþºÍoôÖôy¿üŽ¾o”Z›ø+ä'm1A¦€[,߯m›~Žõ+{±ÏÓýa/ÍüDÄJ™IÊì€Ý#Vk)ìZüÓì³ðaç:çžh÷­Ûpr\Š)r(ãsv¾HÏ´yç-®ÿn­«µ Kô]ÚQŒi–Û#‚1®µ]iç—Ýn–}­k_{õ¼Ï[pµ©8—å¸á…a–¸3LY'Öêdלó²D]Kã!C·ç|}Û »Ýá#_%SÎXe¶ÝÚùPïõüù1o»EÀéàw£ÉÃq^ˆ/­×ú¶YU¹äãèãÐî8Žµ²æ$ ¨ qÎ0ŸX JÑéé1o\uÏ@ÝMôa·>žJôШ‰ÚõÊü@Ž»¨ä‰Ë*x¼»ï•Dy‘‚†ù’¬5! ç£á>®E½b–xFªäé1ÿ•P¿ë*“qéBüòÙî¡®€áº_ï±Ã5‰X!íÚ; ¯‘Ò3ø>¢ŸsÄËðý׶Xº/~ÇøÛ´u}´ËËüG;ºb‰OEøñ:ÊüJ eîÁccÞÚ&¿¹€IW°k2‚*7Å^¦,JÞ˜’6 䎲¸Š›ˆ€Æ;öRµÏ í·¨:tC;(»TÃçíðµWžkNuðkv£lßëôÅnjõÔæ5øÁ“ÝQ<UŽgÉç¡ÅõÝpäWn\:Öã ù^å÷Š2ö¡çJÉQ{Ψ>#“ÇñÇ5oêÅÖÙï#d·áÑÙYF¥Í})}Wó}¹ö,÷(ÖGNá«ËA‘}Øo)‡•qG{9Žç÷& 5ÒîÛ'j:·¼¹S!MŽ¯°?âTž_ÂÔÖqÇþ"¿gS¨†p…mŸõ®,OolIŒ{mÑóM›¢ƒîvådiêyÎZBtœëZIßµgiKl§aüyÎVão¿ŸârøædØÉT\ª”xÞÖyß– ªÐ‰|V®qóìü`„å\3¿Y8-ã¯&aÄ >p{vÈ?oã×>»]Ÿ·•–qs²ÅŠ¸†¡…²ÛÏ­>ŽÞ`½âÃKÿÆËÑhnÄ?ü…ŒÐI÷@üs0%Ú´ûÞˆb­ë7bê­ûÆãF‹QÖ–ùfäJ@à7½ÛÇÃve‹k/ÉDý—m¹¢½¥xõ6÷θÊc…öÙÛ^ìÄÛsàS‰Tb•¯“Žmu·ËmtXÆxAóìv/Cnˆ)ûúÈPÍüôµrÐ Ûðïµ·Çv¸é?Ióþ¤clù¥oºÿÖšâ^ýˆ›¼ü°Ó=ÿIËKønù‹MÅ8Ï_j;ó‘`–g=ó‹ÿÅ$MÜÝb®Œ\lgÚ•]ÍÉòWpø ‹:äEë¿ÚÛD!dRx–ÏSÙ…\|¿˜ä—¿ÂF -¡¬ÌÄöÓŽ¹È Ðì´ËÉÁòê¾gˆÆ¶Ÿ³Þ湟ØWÃãcƯ#ÞñýŠmw½±BÄÈäŽd^½5®‘ùoÅ„v•+1užAœ’ì;'œï’ùAÛ$Æ#ÑŠÛM²§ÎãO‚ Ñ¥I¬KÑ9¡þÞ×g¡Î[ÄË · {ê‹JåKõ©ºt¥vZLÕA½çP =)¤ËI!¾³}j‚2G»Ðšï±2•nH4úÕÞO!]Nj¿6[šÊ£W™Ck—üdešz(Cð -í"ªq™¿Y)öˆ ýç–̯j]ÃóO»ñâåiìÄI~9ŬôŽ+lì¦'‰AÌ;ÈÅN{™TÄ7MTØšÆ7àÿ¯P•‘gùÁ.Óõ äÕ¼Éå$È[!Iz¨Cšøȃ;P1wÎÃf¥¼á)ýtGï!± UˆLÐùÅ1uà±%»auƒÌÖå3°Rmg·›ä8LùeCž|ƱU£7KN˜{óNrØù"ÉyhÜñ×f¿c^aóxYíÉlú¥ßd’½ÞnÞL'c…Iž0ú²ûnèÄ_Ðì?ç&Ó<îåôÓiqs>DÏ)H -Ž?ž&£%wþ'úüÿÿDçýk«÷v …غ!KOÆÇc˃dþôf'Žm#ˆžìO²+³oìÖ0ü¸÷¯i·Ø{O7ÔVIŒS†‚ì‡é._¢’‰7ˆ—G 󵺙{½ }q| „¿óҹ…\.Ël Ó%ŸÅ”¸â®Ó¥£•¼ˆ 9).Å ëälÃ&¤¼É=óORï%.DÁ—a¶Ñt³rêN[V!º òBNÌ5ò9WÝð£0ýêíõÕzo,§¾˜: }P_$u,Œ©ðtjX.Óörøœ°œ7ê›wJhÌÄÞ #È™[c†´¹6*¨ek¢¼)uÚæ¦ÝQ ¸ÍvÚ˜1wžRÖŲ¬¸¨RÊ#¦˜ÔŸ¤¤£ÔÊb”ZI‡¾ËKÂæ'­G=Òœó\=G,YZ’£•fÏÐiÇ{ÓöÞÝÛžÅ2h2’fdjç°xèù*ü{ÍÞQ_ôÕzm*®Ìï[ÁQ“ËÁ‡å“嶤 !þPE¦"JO¹]ó8V6zÍCÞ êà"ÍŽ?>¯³ð‚†z¿ALÕ?L¯«V]ÒsPJ:æŽ%«åëÃø¯vÆeæT‘?68è-üÔlõ -–¨xmÿ âóP` -endstream -endobj -45 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 44 0 R -/Resources 4 0 R -/Annots [ 46 0 R 47 0 R 48 0 R 49 0 R 50 0 R 51 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -46 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 191.048717 658.622079 244.661998 643.622079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/admin-settings/pod-security-policies/) ->> ->> -endobj -47 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 439.875057 101.433071 421.125057 ] -/BS << -/W 0 ->> -/Dest (cb1-1) ->> -endobj -48 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.928036 99.933071 402.178036 ] -/BS << -/W 0 ->> -/Dest (cb1-2) ->> -endobj -49 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.981014 99.933071 383.231014 ] -/BS << -/W 0 ->> -/Dest (cb1-3) ->> -endobj -50 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 383.033993 99.933071 364.283993 ] -/BS << -/W 0 ->> -/Dest (cb1-4) ->> -endobj -51 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 364.086971 99.933071 345.336971 ] -/BS << -/W 0 ->> -/Dest (cb1-5) ->> -endobj -52 0 obj -<< -/Filter /FlateDecode -/Length 4935 ->> -stream -xœå]K$7r¾×¯¨ó¢HFð:ëÃ.0 2¼‚£Þé]#Zü÷ý3ùĘ̀ª¬™–gO£»*#øˆ ¿ˆ ™9îjñóßÌÎä\Räë˯—ß.Ö¤`ñ¯–Ø\TöøMáê,'\‰9\%“Jð)^?^ðgw9ÿxù·?\þk[Ç]½)%z®ÿø ,&“rJ6(¬–u¤°B0)ùVt·XŠÆ‚!±ç#+A .9h,ˆÁÞg…•!qrtdˆá“c1\ Ia9 9œ·VåAËL^á¹`"”5žw&æâ‹ÊKà¹ä5‘‰‰]p¯˜cÈ!K(Öi¼Y‚c¶²0å¤ñ"d¡HVåAŸi¼Y<´S^†,°®¬ò ‹Öi¼B&”̬ò -x6§|äyLÈDVã9gl“Tf)&…ç!K´¡¨<ÈÈzG…ñ£ñ€RÎÊlõ2»É’Óx²xŸXåA ¾†>B›¢˜AFpIß’áâYŸÉp次€ÏÅpJ¤Á€/Áp,Iò<ï4 YG ÈAÆÌUy… -kX@² €†DÅ3iX@@[†µhX@€[¶ÅiX@À[¶.jX@\ˆi5, .åÈ ¦T4, `.;5, €.¦{Ò°€€º¢×°€»Ä9jX@À]bg5,`à.jª<ÈÖ°èhÈeÒ°€»älÒ°€»˜F^ÃŒªF `+Æçl5,`à®Ï–5,`à®Op*/ÌAÒ°€»>à4Ï ÜõÁz ¸ëÙG `(øcÕh¸  z8Y°À߃çIÃ4f< p׸?eN­ŒË%jX€».cøT^1.Aãʼ…–‹±hX€»pT¤aAî>³†Àxê5,À]GðÔ¸ë|±à.↠aÌÁ8À¿gaº*¯[rÖ° wK^Ãxac3Då%CrŒEªaAî½ "p×" "p×2•Y({ ×€‡i«Ìi«Æz …ÆîZ‡ÖxÀ]kSÑ° wÅmjX“|Ö°DÁ^˜ٔ”¢†±Xðò©¼hp4,H„ÈRy…kX€ÀÚ*YÂä! y¯aAò!ƒ†ˆLqˆ¾Õô²À\Ôü€! "E €d&f RˆÈÇ U½É p­ò2x.iX¸b§ar)“1é5,À —4,€à9Ö° È‚©¢a\‘É‘¹ÆsA¶†ˆtÁsNÂì! ¢A ²ÏÈ2£Õ°¹$T­ñØJž™5,È„Š`š ÓRª<Șаã#ÉvаÉ’M¤9*²P` rö’nf r†,Þz `›’o& Š…,ˆø4,(6 “5,Œƒg­†Åe䛞5,À¼”|3kXPÈ"ßDŠ§ÌÛBQòͤaAaÈ¢¨<È‚ ©(ó˜„|è¤ñ"dñ…5,(1J¾™5,@˜|½º$%ßLêrp7 “Æî"2àù&"[-··ÞŸéÌ,gVW , 7À·ªKØ o•Ô5 ð•I]4€˜È,sTW àVÁtV]6°À_Äc|À„eõi^¢Š0‡Ÿ™“ÜdÁ´`D÷=çÅèF±e}ì»|ûÎ÷ú·ÿ¾¼þ¡®”µã'Ž5±Fü‹/oà•©íq]MB‰#BÀO©#B¤2„t€ÅuÑM„„+†»-×þÅmE†Ä"³C0SïŒ7ý"Up+Šª¬O×y#Y‡Ž¼ä©Ù–Rf!¯CÔ†ïNUìÙJo®Ù¹ã¡%UÚç+½¹nü¬&¡‹IMHhàÇ%hÒT{²Î›kvîw(I“õé:o®× tb2Àe6?x5¸NIUìÙJo®Ù¹ãÉü4iŸ¯ôÒaøû/ˆ7–½Žx­ÓènåÇ_/ß~ÿ/?ýû»?¢Í_/?ÿ“µ˜d6ü‚OÆ'"G$‹–_:òµz¤LX¿{üƵ>q—µœ—úø^Özei³¶œÓBÏk¹<ÚYxßýÇõÇ?Ï÷ ËNêL»{±¥úÒì±"Ýè‚>T³í‚ÞcE7½*¿¿Ý£ä…äØe½GŽk¯yÕǪ‹¦Ÿ°ò+-Lº?èh!¸ƒÓx# eØŸ·]ò6yi²)ËPÂ?ÿ(¶Œ¸IB;qbañl²ë ˆ¯¶n‘ ë²Æ'™þjñP,rLܲ¡â%&G e r¬Ã4­×E;ýÒ!6]¥Ý(éÛŸþò¯?ýéû1)´ ²˜»âñ›º–úÄÛÎ7˜P–¼²ß5½ŸoÎÍ -—…Î5–ð$k‚Q–‡Úˆ¨¦ÓÙˆ‘lTþtÿ¥sëuQI¿tº¼².[ë2Ã*Ìøï¾quŽ¯Ÿmª¤åº~.àV§™w*¶Ä²ët™’K;În?§^¯{¹n ëu\?wå¸ÉÛäŒ{yƒŽŒK&¨šb³—<æþ Ÿ«ó¥}È‘q8ÄQô>Èq§Äðô ;Ì<[d-t×éÝA_ûíFg—ß.‡Ûjmçñåš0…Hšá¡L²j:©~õeª“wFÈoo·î)|wÿŽà9`'™e:Ölëís¦Ô©)ªO-9ÏkA²TŠ¯²ltj«óõL­œ ê†5´»íÅ>·î p+÷Ëhƒ_·Ÿ½îÞ›i†‰×Ú Ãpš©ö2Ítý¶Ÿ~ý~ª÷ÀgÏÉ1 ›ÐCuÚ”W/”Ö@9MAZjX® wä?à7«!¦‡;Š1æ]§Õ¶@Ñ-ÑY5õ÷K-u¦×C›ä­‰TÏcÛ¬26­¬²å>¶Y`ÔÁ‹mÛ¼}ËY ®g tYxMný6r)M?÷bðzöÂcÂÞÐAxÞ{Àeï异"C^ëGUŽÄ yØõ-9¦ ¾Zÿ_õ€ÐËÆÈ"û âû£xð\•/ _=ô²}Î0Þ‘ÁÞ¯–=n;®PykD9R±×({G7î©*_z†¸š‚èÕ«ƒÐðûDØW‚ÁӮџOÅÆ·ÛÍÁ°e@‘Òî­#9j†éž—ÓÉ&KîÁȬóÕ JèvuZw‡îÉ¡ŒÙ Lám';~""?¶Ó†îÎŒõmÆkŸ‰Ã“#päµúwƒ#ݤHvl%ô­‰÷9“:Wçë1)†!''eG“Út2'Ÿ`V‡¶ž2«Cí'ÍêØû³ÚÄ£e»XÕ/§è´ÆbÅ›kÑ©SŠ·ÆÇsôæ”zŠxD’»Ö#âe‰#†xýR¼ðµä¼ïÝïþôÓ»iY'm}5—–©îòÄ×ÉE<¥*©ãðφGªªzö1Åâ=¯×³jÙs•ÁwžÄ—Ӳ촧b8(ŽPò© Û(a½tAö³© î¶·ã€ð…¦5Û°cyº.ͧIÝ”å­ÙZß - kŸM®]™ê#èžæ>ÿ{FÿËToÍêk–ßäÕ¶‚í[tµÞë(×ï7ÜÕ!ïu(²4­óþuÅ^úíò4™óì‹ÖûòC/u¥%¯ò½L¾,¾îfÕ±‰k›Í_º­ÎÊW{Ú–¨”uw·åBÜënèc–¿ÙV×áYž±eÁp’Ïf e²m»µåù6Öúj6þ2hµüë®­»ã{Wù`wŸ;‡Öy£nÍÎã§{šVÿzÙ @OãÂS_³'Œ•°µÕ«\ÓÜi{c­Î^†Å.îé/Úƒþ49?C§í(íìòwÇÂgÆQY] ÓüSVôfüGµçfËû•ÕI¯mõµé£ÚÂì[V ÜØË _³ŸOŸ¥»C¼¦Ûæ}™ïêìà/ºnÊVÎùøH;–Ò£´ûšôxw¾ÜÀõekÏNñÂ#ÜF¾vó½áÂÆœíGj:†Ïã‘'»pvã‹ë87úûôÍ«ó1ò\’Wä{µÍ«æÞG‰ÕvάÖGÙ%€&²&ÇΉiU=¬þ£û${\EÙG²£¡ËÕæñ"ã½ñKv_¿÷=Ý[ý†½owqÆxÌu«]¾<´£b£IÙÉÓ¨G™ž3Šòô-µ7Ák71á];Ýï‚Õ±½;þ0s:•W[¹±®›äñi[ÏØ‘ÍÆ#•H¶uOÖùÒ‹p}_$ar#–¥õ˜L5írúæôƒÝþ%µ¼§â?á$Á’þ¯u>ìöhú–ìM%$‡!‹·J8µ°G9’g¬£RûÄ•h ]yüâX¬ZÙ±R5kìüfs¶²/È1“¾Ùîqê·³APÆïíÎmœyZ8û²…Ô9Íè)m«?帅ÂýiÎ:}‘{–§„ä ¿‘u) µ>TuyØÓgKŽí=žÑcúX»Ÿ®¼¹ý9fªºõ¹ryüÞ…|Í#Oå='ë|=f"¯ñ € ùÿ_Þ[Þ³UÂɼ'´¼çXûTÞÃ-ï9Ö?÷ü°>œÂUš,¦pe~ö¸ÛèÃ"L²…»Ç'ø®n÷´Ü6"F“òú?9#Š»/Iž˜WS°Ýêo+–a¬zÔ”j—nù5µÚ¯ÖjçoåÔ†ì¾P[M­!Ûç»õœªS_m•ÛN²|øn¯þß.±¾±ÁÕ·öXD,ÎËký®‘Xi*WyKD¶Äý!ZWä®7jÖ—!þÖKÉ_¬¼2I^—Q‡c¾XËÿVê׋ d\²$·¼ËÌVröíb?^ä]v¬™ÇÁ˜d!®_mo‹]ÏÜíå"Ï•}I¤vý:ãPÏôdùjÔ­š¼c€m+Dõ)û¥c…ˆšÑÉr÷Ö¸•,ïD¬µ^Å.JˆMfGƒ<— ŽŠÉK b8íÔÝ {nò8yLW]Ä.ú¬:7 + /êO_.è–öÔ*UÞBÖšjÚõ¶Óhªtjh%û€p'…ôrQˆßØÖ4c0;5?ν4Câ^®´r -é墖kLúH¥^*ª¥i~ˆ2LrW¼B{™Na%yÁé4G,µ¯[²<ÙÇZ`×!Sw_.ûLä—”Òa`… Íôf"Mívr¶Ã^Ù̪*LÍF“7aÈß*B§ -òÔ/Ž»IIl5®F'27;y«¤™;ê°¦Çi”Ô%Kbt £íl•òº½t×؆:©l¢§ÒÄêà±%»nu,ÖôÓ±r&Ú&ºäÐMùã†<ä }ªÛ;‹n2÷ÐÛäرóãLN}ÄåævÃÄÉæñ¥:6Þ“Åôsëd%7læÓ;›ˆC0€Â ýXÉm¶vt’“Uíë˜dšÇ}¹üýRÝœ§à%œääM{s´ä®ÿƒ2Æï^øú®½ÚeñãÞv´ð[7äÙ“É{YóI²<(Óˆ}ÚLDÏì²Ë£lhÖÐý¸¬²¥MªÛì­¤ëê:w&†¡Ã‰ì»ézG“fÂâËN‹=ùܱ{¹¤vs²`(/jRáb¾\æÁ£á’¯S“¸’¢Ã¥ƒË~ªˆ‹¹Q\N–!ÙFL1ȹ“®z‘Ÿç¡õ¶ùb -#ä’‡››n´zS„À4_Ì ³›;-y¡?yz}ö¸¯"Ç6„Ú¡½S?ÎÔ~cB퀧S©^Æíe÷9Thl“wh¨·EÓÜ£䌩1B Þ\Ì6:Qó¨¶s§Üh›N›£èpšm´Þbj2ŵZSK½ã¬j)õ˜bPÿ>k:Ì£Ò•æQ‰‡²Ònóx=Z¢…ٜӸ;;JöX2+´8×îT%©ÑŽ}ó¶ïæm¯Ómð$ŽÈÔŽjáPò“ðïSæŽúæ­Tû©+.—¡É+D\"Oõýö6Ç ayEiâ<-q<]çþšÇqe£­yÌ=a8 Ùää*ÐEøFy̘šjÿ‹^­Xõ–žS€²¤3`îøž£úªjü[vúe’TQÞLOôfy¡n²ú{\Þ,ÔŸÿa#À° -endstream -endobj -53 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 52 0 R -/Resources 4 0 R -/Annots [ 54 0 R 55 0 R 56 0 R 57 0 R 58 0 R 59 0 R 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R 65 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -54 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 751.923057 101.433071 733.173057 ] -/BS << -/W 0 ->> -/Dest (cb2-1) ->> -endobj -55 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 732.976036 99.933071 714.226036 ] -/BS << -/W 0 ->> -/Dest (cb2-2) ->> -endobj -56 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 653.081993 101.433071 634.331993 ] -/BS << -/W 0 ->> -/Dest (cb3-1) ->> -endobj -57 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.134971 99.933071 615.384971 ] -/BS << -/W 0 ->> -/Dest (cb3-2) ->> -endobj -58 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 615.187950 99.933071 596.437950 ] -/BS << -/W 0 ->> -/Dest (cb3-3) ->> -endobj -59 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 596.240928 99.933071 577.490928 ] -/BS << -/W 0 ->> -/Dest (cb3-4) ->> -endobj -60 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 280.647907 101.433071 261.897907 ] -/BS << -/W 0 ->> -/Dest (cb4-1) ->> -endobj -61 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 215.950885 101.433071 197.200885 ] -/BS << -/W 0 ->> -/Dest (cb5-1) ->> -endobj -62 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 197.003864 99.933071 178.253864 ] -/BS << -/W 0 ->> -/Dest (cb5-2) ->> -endobj -63 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 178.056842 99.933071 159.306842 ] -/BS << -/W 0 ->> -/Dest (cb5-3) ->> -endobj -64 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 159.109821 99.933071 140.359821 ] -/BS << -/W 0 ->> -/Dest (cb5-4) ->> -endobj -65 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 140.162799 99.933071 121.412799 ] -/BS << -/W 0 ->> -/Dest (cb5-5) ->> -endobj -66 0 obj -<< -/Filter /FlateDecode -/Length 4904 ->> -stream -xœÍ\[¯$·q~Ÿ_1ÏDñR¼† ‚pÇ Ià‡³ã=6‚£rò÷ó»IVw×ôÌ9+[ÒB»ÓUMv±._«/îjñ燿 -9SJ͉®·.?^¬ÉÑâ¿vÆæ ±ÿs¼:KÙDWS‰×˜²É5úœ®oüµ;”§¿]þýW—ÿÙŽqWojMÞÅë_¿0‹‚É%gV˺ °b49û˜VrK¬UcAŒ˜ÉÓ‘•!Õ5Ä ï‹Â*#PváȪÃgG b¸³Ârr8o­Êƒ –(x…ç¢IPzÒxÞ™Tª¯*/ƒç²×x!˜”ÉE§ñªI)Å¢ñ²ÄjÆ‹%:"«ñ …’5^‚,!«ò ‹/9h¼ Y<´S^,ð®¢ò ‹MÖi¼L¬…HåUðlÉåÈó6šXB°Ï9á›Aå!¢`¤”ž‡,Éƪò K Ök¼Y4p"†\Š­ž£;Øà4^„,ÞgRyÆ×À'ÈbsÒ d—4 ð9ªž4ð%*T4ð¥Ê9h0àk4”jÖp Xžw Y"% ‚ƒ,„ÈUy%TÒ° xÈ‚4,²x - -Á[4,€[²ÕiX€·d]Ò° p!¦Õ° qCI¤aA䕪†˜€€.Â=kX€º!&¯aAì*IÃ‚Ü ä¬†Ü #UdkXt4Á• awƒ³YÃî"Œ¼†°ª& à+Æ—b5, à®/–4, à®ÏH*/Ä`а€€»>b Zæîúh½†Üõ䓆pü±j5Ü„D½€,X ïÁóAÃLf<Ê "p×U¤?%¦VÆ•š4,ˆÀ]W`>•WËи·Ð²q)U "p‰*hX»€Ï¢a0Þ £z "p×djÜu¾Z "puCÔ°î` - ߌ³p]•W­¥hX»€%¯a²°±H¢ò²#9 ÆÆ¢Õ° w‘Þ«† ¸kc$à®%”=*²„â5,@]ÂV‰i«Æz …ÆîZ k<வ¹jX€»œ65,H™ ¾hX ª`¯aA*djÎIÂT-x(ùT^28Žd Bd©<ÈB™4,@amj¨EÂì!Kð^Âì! J P˜êP}«ÛÈwQ÷YP)jX$3¥iXcÂ~ ZÕxÉ›’×*¯€ç²†9“A*v`/e -‚^üárIÈž# r…, ŠLñ¨Ì5žƒ,(²5,@¥ žsYP jXP|Á.3Y ¡±—„ª5YÞg -%Ã¥¢†ì4”*²&4,€}x³5,Àæ›MlsTd ‘4,(Åóv³hXP -dñÖkXßäýfÖ° ZÈ‚ŠOÂj“á2YÃÀ8xÖjXP]Á~Ó“†ˆKÞo j°Øob‹§Äm ‰÷›YÂJ%¢DQyESU☄ý&ÐIã%Èâ+iXPSâýfÑ°eö›¨èÕ–@áýfVÛÀÝXL¸‹Ê4iX€ì7QÙj{{ àõ™Î,¼ã,jÇÀz#r«Ú2°À^d«¬ö ,À—÷HjÓbbgY’Ú5@ZÓYµm`¿¨Çè€ K÷I¶¨Ü!#g–Ì‹¬ Bu?öó´pç´¥?ö×?_¾}Aò½þù/¯¿j²þÇÁù¥¶±ö¾ÂÞÀ+Óæƒ[0 -‡Â‘Ê"B b1rbò¸µíÆb"#áÖëøá¶BCf–Ú¡œik£Í…±Yp°VHº´ïôSH{TýwŸ.À˜¥¿™®B‰H‚y¼½~úáòíwÿúýüî1ç§×ËýŽ£µTð?áw^~£>kÇí߸òp.}^híw{ÿaÛί ^Å\i™£ÑV:µóóÇë§ß^þé”L,vJõÊ-’E—ŽÆ„<~EêmkÌý¿{ȇ´¿*;Q”6Ð2¶ÿžSâE¼¥ƒ%b¿êbqPʬâníòýþáûùnÚ¥éªëƒ#ôÝjŸ~J»9×í4ô:|eë"˜»/ì3ìá)Ù„>Nsò%š’–Q7&M\ÛdÊ9 û·ž]A³è{Gü¼MÎwit nà±–( -‰pmùmñâä9¯©éÝC~^= qk=Ó“ýý|žô· Ïm°Øâq+Ö>†ÓûéìlNwñðóÄÏv\–%6 ì$p7ösââ.m¾´^㶪b]¢sÒ¸YÖ“Qá¾â?ÙèØ! ‹i .ýèÝcþ>ŽDýª‹©Æ¡ qÈ‹æÌ“XÝäüo¾qqý—Uè–ß^3Óú»yázž]iÿê -ÁöàXŠ}¤çÓS¿B½Û« ŠÎ°eEfä ;vEå¡<7æ'©`~ -W˜‚c[Ž±³ -¾u1_\Íã5÷§ŽÜ榶ÍØL½`Aw‚ûãS1ÖåÈÛÃøî|ÍÙìêT¿žŽôíîîõ<¶æÜšNéž¼;Çb 0½x»Wo›“±íó-ú -Ïq¹zM‹ôe]mÙ…`?îÚÈëñëC‰¬Ön3—3-¼g¡ZFð÷páéHë>ȉ„þ$æö[úµÆú?ó˜OÜöI1æªi€óÉ—–3¦?ä{€»s'–»ƒØ$˜ÒêŠøñ䘟;ƒLy!š§àmÒa£›×N3(Ðþ©%–å÷ËÎUòt!¿%½àž»ñœ´&0ûªB„r±0ùv9Íä·‡£GÈG»/F§lj¢\µÑO@LÈÜ­*lÆãø®½º‹z¾˜BµÏ§¸'Çür\”FÇj½^䄧ñÜÁQR°û„í¤#4æÇ8¨À¤U¹Þ@Ü=Ê)ÅWÜ£-ãXï1ìxtÞ#kÙ†ÝýG¸ö¹ªÏ"Ö­Œèþ|~E hBŠ®M?yÁêf¤ô0SXoàÍѺëŒU…§VÄ7ý(–Ô=²„†o²Ö{úxXo¤„aˆlÞØ=<Ý…‘ˆ 4U[ø±$nƒÛäã#ynÌ/F…Ÿº×nš¥ÀS²™pS`} —ݼpi-s®ü‘=o»Ðx¢@ôÄ;/ËN”ÿa®„:]u!YmtÔ7â>W¾ÇuPh·ÏZuzî[O ùå¸V¥¶O¨z†Ú¤qYôÈ­ô¦=S·=¤ªôBànã[ïvµ÷×ëãE¸šh_·v]y_öûEÎëyaÒ['û¶vqV÷ñ­îâÏC)>jó”¹EÝÝÇ÷ÖÖÆêè!õž’è¥éGŽXç‘-´"úO}®zoõké°êíŠÛ¿mÞçWN~¿rZ;_Ý>q•¤7Hõ»”ßçägK×yB B'æzYWW‰ëú¿Ðro*vÍÆÞ…“«í Æ]Άy|·3(,ÓZé_Y)N9¢°ÀÆZi‘#v«æ9¦óµ;8Ý*wôW÷úëw{F3Më7Ë×íZåF³«Þâ<óŒ»=ïÓï8?¯¼îÍ_ûvù»ÿ¬±ÞlÑeŠrÜ™~¢Óôó¬Üý–É£¨º›Èé‹wõq¾qz_çc…OÏ[3â‘æT?á Ÿ4å¶ñÜd“>MbMÒ‡ìŒãƒŽËÎ9.n+íe½Þº¹žv~RÖYæøv íWúÝùr×ë¸EF§:‹{ÅGþ¡ îWú ™W„®Šw>%ã{èëe磯sÞ38ÑóVN©c™Åú5c<`ÿÆ¿ßëKŠßt,í7í†ÏI¿‰bMNøÄóÙºÇeÓW×ÝmÆ“¼Û£ë%ٺω_•ÕïÄÅ•Æ掘>™6¾¹u\œêÉïõ´Ñs~[æšö1¿yB¢ûPÝ®ç)]½ê]—¥Ñ{Ͳ®µÏÝ}í<–ìxŠc`å¾FQõyªCRuØ}&O=t’¶÷Ÿ·2FQ }Ü–ìâIâÇÞ7ÇÚøÐÁ÷ow|ó ÿŸö8ÕcÚëñ¡ÝW_“¾BOaêvM{?>`X™¾µÁõ{¾“u½i»—gt8p¢ÉyªÃrðÅî{UØ]bÓ×úÆëÖ'x¿ÉXë×ø²ŸíSõÐSyAØkøÝùÛmÊÝ÷ -“NsF>䌑3“Àé_]·´ÒÅÓ\ÒfÍæ^V¹nšÌëµfÜä ü"E%ç‚"¯µäâû¢³lEÿíŸ/ßþÃÚÔs9g|¬ü¼ý -΄Ø^Àô7~1¸òm´ëÿ]þðăwrš¨ú%"7{§ªèé«1;óÞIœì÷#³Æ]åê5à®ìø#}WÎßæ=Ù뽚ÀŸ}|éêtŸ!ý¸Šë ì;Õß1çmrÒ©ßÉy¢ð;GÅ„øåEA·ßÉièÔïÎkÈ\⾆ìvÝå¶Øå°o9Á’}_eÔZú!ý<ÙW¢{^w9èÝ5æ™î`-eO|×ÿ´~¾NqcƒµŽÖ¿Úä·Ž«Ñö®Û=ãa=DdrÎÎ;e=ŸL«3䣦íÝÊ/'ç~Ðk+tR¤ÏœêÞk²ÞóEÚå5Y+ ÝËú³ã”Ü3žõX”ìC5þ9Ÿ©»è´Qè ¯'ëZÛ1o·×ÙÔëÏÔÖZ/ ç•Ý~ëÐýîq%j‡ÑÿðvÓkÕô´Ñ«Àî;ºJ]å)»ì'lzœëµn½W™„Nvcn¯k ÷ªû›Ú/{ðÔþÙYÕ郇þfÏñ¢×¦ù·´ëã}O9ä™G±ð“î)º>Å}}ÕëƒÃ^jßwuþöÓ\Ïݾ¶¨w‚óó[±ò]@þXFàwòèÑ“àÏù¹§ñ(¸Í¦xWK¹sëÐÞMáÎên¸§·gÁoÂ5æöàÎ3áÁe¸fÄÙ{ùî¾ñtI|L·ü¶C¾ÄÝÅúú› öVô®$?ºÖi º¹VEFö¶}LjßÃ'˜²Iw º™æTog¥þý²ì NÚ+‘Çôÿ(½{{° á=d¼ç¶^”ÐZ6òÿ~}‰.G,4Ôj*!Žå«{#öžò Ô&’_Ýî±í­±1%=«=éc¨f~áT35ß•eµø®¢ŽÚѺ„Uý½nÒ­ç]Wk¡«–ß·¥%<ÝZ‘ø}¥¼¾Š3dùrPÿ—Ô^xví£6ã<뚀_¬Ïzå§m‹ 人]S»ÞÙ¾%öã8‹ÿÉòGømófy°ž¿3öŠ•Lê Ú>d¹ø~0Éoþ)døþI2ןvÌEn{§Ý.žW÷g~iù9â#pÉ>ÇÇ>ŒŸ™';Þ˜k/©.Vˆ™Ü‘Ì«·Æ­dþœ¶ÇëQ¢ÄÔeæçlÆ„$ÏÎÉ.XAŒG¢—›dO]ÇŸ_ªK“8D—ªsÂüý\Ÿ…9ψ· .öÔ7•ÊñéSuíz;hAU5ö3‡AhØI!Ý. -ñÛ§&sð…Õ|ØÖ[‰ÆyµŸ§nõ¼u¶8,•ÇYe­]óS”éê¡ Å+´›xÐ)sº1bCÿ¹%'¶@·µ-Âoüx»D‘8Éo—˜•³ãÄ -ŽõH bÞA.vúˤæÒU…Ðì4~‘œÿ^ "*#OûÁos¹><šáØä­’$= Ô!MŽòžÏ ;ÎÛP…Ê=×.ƒÔ-Ù ¯d†°®Ÿ•’h»¸Ü$ÇáÊoò”3ŽPv\,9áîqÌ;Éi`ç›$çaqÇ_öòŽy…Ïwâ­%6Ú“ÙõK¿È${;²Ý¼˜ NÁ -“u¦t~úÞ‹8“âP\°NÉ6b²CÊ‹ Õ³ü$MøZâ@”|&fš•SwZ[…8%prbrò¢u#"ô‡Ãë«í¾Šœz0u@û ¾IêXSàéÔÐÓöpäœp僼SCc® b/Œ"g†Æ,-hs }TP˶žåE©Ó6í‰bÀh¶ÓÆŒ¹Ë”Öa]-mÅEÕR5ŤþEj:J« %Fi•t8—çíoTKa@Qºsž«³óÌQK…–äèA¥yfè´ãµi{ížm¯b4I³2µsX<œù!üûHì¨ !í¬þ§u4\™ßB¢æ¬}hŸ‡¶%%`ñþ2Ñâx÷˜óžÇ±³Ñ{òJ0‡¡lvü¡o]„70êc†˜ª\V­º¤÷)@iéL˜;¶ïÚ—^ñßÚ؇™·ŠüaçÀEoáïQfëô.hÝ´NÚŸÿ~çɲ -endstream -endobj -67 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 66 0 R -/Resources 4 0 R -/Annots [ 68 0 R 69 0 R 70 0 R 71 0 R 72 0 R 73 0 R 74 0 R 75 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -68 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 748.173057 101.433071 729.423057 ] -/BS << -/W 0 ->> -/Dest (cb6-1) ->> -endobj -69 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 729.226036 99.933071 710.476036 ] -/BS << -/W 0 ->> -/Dest (cb6-2) ->> -endobj -70 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 710.279014 99.933071 691.529014 ] -/BS << -/W 0 ->> -/Dest (cb6-3) ->> -endobj -71 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 672.384971 99.933071 653.634971 ] -/BS << -/W 0 ->> -/Dest (cb6-4) ->> -endobj -72 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.490928 99.933071 615.740928 ] -/BS << -/W 0 ->> -/Dest (cb6-5) ->> -endobj -73 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 287.395585 321.344928 323.513505 306.344928 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://github.com/projectcalico/canal) ->> ->> -endobj -74 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 291.344928 111.294999 276.344928 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/) ->> ->> -endobj -75 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 308.171464 172.723346 401.561844 157.723346 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://kubernetes.io/docs/concepts/services-networking/network-policies/) ->> ->> -endobj -76 0 obj -<< -/Filter /FlateDecode -/Length 5549 ->> -stream -xœí]K9r¾×¯¨óÃá#øx`ÃØd`¼Â,Ö‚­iFËÀ¬þûþ‚™|eFUVI½Û}°©3ƒI22#H&Ûœ5þ~gð_"£RÊ1ÐùòåôÛI«è5þ”'¦›Rü3þE6š¢ò&‡äÏ_˜à¢ŠÙÛÎÏ'ü·¹>ýéw§ÿžë˜³U9küùoŸ¸ˆœŠ)Fí…¢Œ"mœP佊Ñú £ðŠ9KE`ÃG²´/Š`ƒròRØ k“P”À†£hܾ(ƒ  IE`Ãd…"£Á‡±Z‹e`D9+”¯„¤2kTHÙf±,¢ÌD+•9§B$ãT–UÁ'©ŒÀ‹ÏÚHe¼xC¤¥2ðB.E©,€œËÀ‹MÑIe¼XH' e ¼Àº’X^tÐF*ËNùœˆÄ²Œ2bÚ—Yí•OÎi©Ìåa›N,ƒGAI! -e¼í³X^¼ÓV*sà…ðW*NxS¼Õ²w;íŒTæÁ‹µ‘Ä2ðåK`xÑ1HRŒKØeKØä%J -Ø”Åè$°Ù+ -9J8à´A™58 ^< œ/ÏËÀ‹Ë$a³àHXàx±ä$,p@[‚µHXà·¤³‘°ÀoI› aà‚M-aâºHÂÈ…)e 0×;%,p]¸{”°ÀuVÂØu”‚„¸ëÈh ¸ëjŠeà,aÐQ9“œ„ÜuFG ¸ 7²@« -v$,€­(›’–°€€»6i’°€€»6bË¢‚: ¸k=^A¹»Ök+aw-Ù a E´ w!^À VÂŒ÷(³NÂ4¦, `Ui ¤Ê€»Ú@ÃRpWë˜%,À]6%,‘ 6IX¢`+aAH¤rŒA‚5Êò‰eAáÞKX5%–Š$ak•]ND ^œµD ^2HX€ˆ@eƒè[LÀ ÌE̼ R”°H¦R&’° ú€| R•Ê‚U)®Å²„2%,ˆ‘†b#ar)•àôà‡K€ ”’° fðW‘°C‘J‘¹TfÀ ‚l é¢Ì ’/ˆ%,H6!Ë ZÂŒÐÈ%!j©Œ4ç™I‚DAq¨(aÜ™&R,/€  N¶½„H~l"ÍËÀ‹ó$aAJ–ÓÍ$aAJàÅj+al“óÍ(aAÖàŸ„YÅa²„€q”i-aA6 ù¦% à—œo& ²ÓÈ7‘â ~›]à|3JX ¼x„(bxAД¿&!ß:Ie¼ØLä8ßL Ì@¾‰ˆ^œHœoFqº¸ë3I*î"2 àù&"[)·×^ŸÉ…‰3Î$Îh@¯ÇØ*Nh`/F«(Îh€/çHâ¤ØDf™‚8k€a…F‹Óø‹xŒv˜°Ì>STæ1f¦È/™á„è¾åý1wå±e~ìo9}ÿ„Á÷ü—ÿ9}þ]™)« @ÝQàÄÚ³‹Ø u9!"RìºÎ äçS ׆ÌLÖgg¸"›ßB¼œ²¡›l²Ï3áÅBº3<0­ÔÎÛ@½Œ¯2ПA'Ä›[:bÅötïq¢†±íFï¯ó<’û«ýu!‰r¾œþÿ65Ó - ®RtJð|°ÌK±Ta¥ÆÜ•n0̬“˜¬t¤ýøÏû‹É`?Oeˆ³h½!cünÏB’´%Î`®À@‡(C3€µ;V’`:@§/€ nC7HCÚӭlj:€“  ‘û«ýu!‰r¾ü¿ÿ½®ÿ‰ø‡4ÐØ@W=Ò±#"-ª.ùU•ŽüC0’s»0Ìêâ·è¹]zšúóÈ„qå‚Ìä㕾ŠÉ ÊLÒ W)%Dc£t5òo¢Ezo¥ⶋi츋IäöñJ/Á- 3ý§÷'³ËBz…Çd 6,B,8ñ¬öû/§ïú·_þüîŸQ÷ýçÓ‡}ƒ úˆ­=ç%îÇÿ<¿ÿÃé_ÞóË!:sˆŸ]D’Ás;9€Ëé°×œdeuY‘ëòx´Ê‘4\ùsÞ_4¯ÐÊrj¸([N€`:#µÛ È×Î9µ[$Æ•ëY\¿üÇùýOƒ¸kQ‚¨À®Â?4€Óô en¹§Ï‹8 âô—&Φ²YS<»Žp; )íF-áúW\§åžÙ)4µY¯¹ìãJÉ»•µ4?Kqß6ÓË«Å…}Ù²xÑ ¦…a汈"­¯n×~õÊËÚW)¯}æ.ÂÒ¯[®‹ãò~´¾_áÏö÷«ïÎí3Z-ÏM¯pyˆsœþ"ñÖIĨA«}´Ê7€øæJµ×EøíÖ8ßØMÈWüc`–ìÕè¿3ÛŸ²¨2ƃŒÎÈÑ•I¼Ö~$«û꼶°¿¼‚™œÏVvîâ«'¸Ì–µ­XdX®+\é¦ -ój繦áùÈ´é¼(ç×j—Y40"ÔRw.~¸ëϧõ§íoAŸ†·¨õžŠÿ,×k¹ùØ߸• onj›<…B²ïY[¦9ƒ1<‹]²GÎwg·cP<ž’yÙ $qÇC£1Àc¢wvÓÁ]Fcx -Îè”P»û -£qn.£Ïýíh5ŒwWðÅ"ßÂà–4Oä‚ˆàˆŽ¬á®*oÇFŽ€€0ŠÆ°Œƒ³¸¹®;ý´U1k®ænGÙ© òÂ:1Fó~ Qe=à¶î«óv”À›b`*O$%Tû/?üykp:!gœý0‚. Š=h/y–¤õFhïØ«3Á•yQ¨]ߪ™Òjf”.ÓìµÕÔjdPLñóæ;£ÂÕgesCPï"ã t„KžÝ¿mmwUy3Æ檦¢¾âñÝ ’ðüÏÜòmD€*C|™'s"oW4:HûÎ:oG܎ǃLÚ½ oÏ~XÕÕƸÕK\Tøyãeaö¨>v8…²qùÓž‹ãàÐ’jhCª}‡ ,oW$¢ä% Ÿ úŠ©QV&[^WCò‘+«S·-í®*oÇÐ 2õ_ÒЦF?Qø’=¯Æ†ŸŠDj@‰œÍBWñ"¢¾÷¼ðí­BRo|8Râ]UÞŽSâ¥Zðò’JœmöQ}ÞœG¼8º¯OÿÀ°«ÓSsF*€14ÞsßWåÍ(™ †0fzIOý0Bý7xiØ÷í_óR²ÀxÞIiïöÒûª¼º2!Ì–_PS£{é¾þc^º¯ËK‘\)Þ&Íûb¬J¢æe¶ÛZ¾¯ÎÛQs Þ—žb~I5OîB·kÓëªG½£ó{ü¸mûþ¯úqäÏ‚îœùc½;Òð]UÞŽ‚oAñ>¾¨O~…ïêW>ìC¡×Vžw…²Ò‡ñø®ø¾*oFž÷®¦d£~AuÎ>®Î}ý¦Ît§ÛÚÿf¹´¯9{Ÿø‹¼¸ -gðÊEëÊÔ°¢V~úµ,,k«eµ*¬+Y~ûÝ°j—W”WçBféôeÕl@Çû"4ΞœrËâ³Í¼ÆoËRÌ~öñ:ÿ£¼¶4ë‚müΪ¹¶|S:=‹t]0|lÍv­×2»QÅÚ&«Ô˜ªæL,øõ‹§Ý{lM¬.f^±Ïä†ÊÅ$}7›²œº‰j×ßc³´ïÙ̦y.«ð'M&„²-VZÛ´Îëš/OV~'©íMgõÔ²^mûú´þ´¬¡·õò7k^ã”ÏŒlßUÚGàC5‰Á>Î&´Gª[&‰ Eé”’ok“T÷Ô½ -u/B]ÏO3rúúŒ_§Ó×½…—aÝ‘‰cg‚`â0ɺûÂè;ë¼ö°ÜVûC >¯š±fqS_ž%¿_Ü•Ñ°¸ý±©m®?]‹‚Ñà]üýÀÑFÛ~ƒxç^ê™ñ  ƒP!ZÀ}u^bÏÖËFhüŲç¯ìeÈNFqÖÜôãéï¾¾¸\¶]!¡ž§Ý±œ6äaWøÝþ=SâÉ0X½oi“Ç„G“ò–cb¶’ëBz[ŠóÃÛ - }iíjH#T9Á!Ý’ÂÓ~åy·{äÍ å[¸°´§Uä§ìZ 3½òPûjïÿ‘ëH<ðU9^–$à—©—iKÛ€±4À?‚`äÙ¼¨®UÙúu„wÖyí¤ókyÄËü‘¼U½º«ÉA€ú×2°,×OúÁÍ ]×ÌÛ¤uÓGPåRú˜4T>¿NQùå°vs™}móé¨vˆ*ŠYª}ĸÈ‚%Vã¾>+ˆ²‰âR'kNî•·†·pÜYçí˜(ñ¹†Ïèx¹¹‡¹Ñ¿+qûþOyÄôÈñ·Õ$N˜=\çµ'ÌLãwVÍÞVµäŠ «#{ñ<+Q2ð§=½ÌP\&48š©à½×’>2:Z³ÿ6Oú4LrÕ9Ø6Kpk¦‚?£ï–Ìä©F…ºÏÕ -æ3¶3òÄÌç= íÍd,ûÊ·mZþ&›ÐNÚœLØFر)¸s²¾šãír¯óº&lùô•ßI/ù=Îótxëb&³í[œHÛ Qu‹úM¬V3hƒT§µùÞ<‹£ÝŽœ‚BÜëRÛÿáêbD|nSØŽµ¶NËùîüõu›öä6í'Æî6„7©ÝÔ.‹²ìéoÑnïWsèßlÞâûw?¿ûý/ïÆÙØM”²é7Q÷r„âõÐŽ¨ök§U´ýÚð*AYÚ¢VU>·x5Ô¹ZÙf>pj@·³½j£Œ(ÂÇ8—¹uÄnEMHN¨œÂ! ÈÃu^EljüN”§ÿ?ëésº*Æ:ÀÝ7¸ñ™w&diÛçÑç|ãçlíó8-¦}´¡ý¼¬lOÏÓ; Ÿ×ùj õ]öŸ[iöÈ]ƒGï-zx P2õ÷߇®¡ÇŠ…çÜåÑ_EŒÛÈõ¢ÛrBái‹Üvy‡*çÒžú/ïtS>q+ÖU °ªkz=-:°•ç§Îk}Oýq£ûËb£’MÜ~Ïÿ|ø`á, üS]®ª1¼î²­mKX›Å[þý_OßÿÓê¸S_N…àåÃ{= ÂYÃß”ôg>8gþèOdx¨îwåûç©™ñUŽýð†4»œ‚Æ›viÝË,ÙúﲕY×Qá+öAºmhËj´b„Õ-¯’°âk}³ñ·ê».I[“ò¶G‡¡€ð ’OÑ@M¹)>¨ÌûNÅtìá*¯;«»“-Ü9sÃY(ª¯Î_nxSv‡ Kö×yå%û‘_ssÐ~|pæO×Áñ×¥›ÆET¨ˆ7ŽŽíÃøaGåU¯׫GÔtû2xÍ°½gàšÊqÛ¾Ÿ÷Ô•?‹0QÂ:þ=t±Z,߬!ŠÝÂ캓¥ñòi'~àh95Д“cuHÊX>Zþ€,ÏrúALÚ!][ÅmŠ€ÌùJÍr ÿoí)þ$ÍÇöò‘EãÍúü|9ñŽ¡‚@9O[2¯ÏoÉÏ'>O2QG’2×KÝÚ"Ó€©Ò.'x‹ÊÛ'Wz½ì‰ºëɇ¯oÃõ}­ÆçÜ‘nSu夷¥cˆšÁìÉüöZ™•ÌgÒ'åÖ»ÐX)#b%öi|¶7Ø+’ÓÑï‰z讓ù³„ú¬m—¹éÄÆú(:3¨¿>kã Î[ÄË Ýº-õY¤òIص©*] Þ•æ¦r£úúdS5= ¤ËI ~§kÓe¶òAk|žèBLÕ¨=—ësérŸ[[óMS±=•zÕ\%ßYé¦îR¼@» S+‘Ã÷ÁG0œÙª‡‘XU×: vcwÄËÉSóÄN~>ù(<í;Vh_Mo$º¡ÝFNºÛK§ò1'Ë\³Òø4Fþ -ߨŒ<åÂP3)žmíw½öÍFž…4ÒCC’$ÀßñšmÇØlv”—IZxžA·=Ý$6Q‘ täº5xÌdÓ¬®‘ª|VŽD]™àcáÙ7S~žÈOß\ÕëÖY0ƒ¹ûÖn'‡†Ï#96>·ñÛÚl¾/e`£-™M?ÕN:™?«æÓ:ˆ1€B'w}.äê­ øW~ÔËîdÒˆ{9ýõT†9ë¼å0?ðáTnŠ– ’p}þþý׉Îïêñ¢Ë8nuC Û°u"#ÿnt'™WH%6·ˆ–Øïd“ú³¾ZCÇùÄ™¸¡quݽ>i‚šV:}—á@¶Ít­qƒdü âe#ÅÈ|«nú€½ÜºúrèÆñÁ´•+>Wrà·©—¹>$Ÿ‡&qÇö!¥d‡Š¸ÅíÐaîœMl²AŽ4Ñ3ÿ4ªÂq_ÃÍFð­ëeÔ‡Ù±éJ+o1<‚Ø7cüšÑÛÊ?Ó_í^߬÷•åP=€© Úõy¤¶cj<™êÊm˜oÛ˜ãÊQDÕy»„Z[nð=ׂœî=´ éf´Ñšzµõ?vJ•6uZŠ éJk-ÆÊSX«U±”7N¢”b‹):õ¯£¤ý¨•&D?j%ìžåvëˆ×¢%× ?šsìo§û“-–L-Œµ•ú“®Òö}ÓÜwmÏÃkPg$ôÈT÷j~÷äWáß×øŽ8Á,=Uÿ– “úܨy+µuåw¬é0„ø×dDJÃÇÃunÏyìg6êœÇØÔ¡È9 ÿ¶<™…gäã‚¡©ú›ôäjY‹¯ô˜„)sûå×Î -;í6rªÈ¿ÍqЛø—ºDmäo±ž¦©“ò÷ÿå·l -endstream -endobj -77 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 76 0 R -/Resources 4 0 R -/Annots [ 78 0 R 79 0 R 80 0 R 81 0 R 82 0 R 83 0 R 84 0 R 85 0 R 86 0 R 87 0 R 88 0 R 89 0 R 90 0 R 91 0 R 92 0 R 93 0 R 94 0 R 95 0 R 96 0 R 97 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -78 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 715.084475 101.433071 696.334475 ] -/BS << -/W 0 ->> -/Dest (cb7-1) ->> -endobj -79 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 696.137453 99.933071 677.387453 ] -/BS << -/W 0 ->> -/Dest (cb7-2) ->> -endobj -80 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 677.190432 99.933071 658.440432 ] -/BS << -/W 0 ->> -/Dest (cb7-3) ->> -endobj -81 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 658.243410 99.933071 639.493410 ] -/BS << -/W 0 ->> -/Dest (cb7-4) ->> -endobj -82 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 639.296389 99.933071 620.546389 ] -/BS << -/W 0 ->> -/Dest (cb7-5) ->> -endobj -83 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 620.349367 99.933071 601.599367 ] -/BS << -/W 0 ->> -/Dest (cb7-6) ->> -endobj -84 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 601.402346 99.933071 582.652346 ] -/BS << -/W 0 ->> -/Dest (cb7-7) ->> -endobj -85 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 582.455324 99.933071 563.705324 ] -/BS << -/W 0 ->> -/Dest (cb7-8) ->> -endobj -86 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 563.508303 99.933071 544.758303 ] -/BS << -/W 0 ->> -/Dest (cb7-9) ->> -endobj -87 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 544.561281 99.933071 525.811281 ] -/BS << -/W 0 ->> -/Dest (cb7-10) ->> -endobj -88 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 525.614260 99.933071 506.864260 ] -/BS << -/W 0 ->> -/Dest (cb7-11) ->> -endobj -89 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 506.667238 99.933071 487.917238 ] -/BS << -/W 0 ->> -/Dest (cb7-12) ->> -endobj -90 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 487.720217 99.933071 468.970217 ] -/BS << -/W 0 ->> -/Dest (cb7-13) ->> -endobj -91 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 468.773195 99.933071 450.023195 ] -/BS << -/W 0 ->> -/Dest (cb7-14) ->> -endobj -92 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 370.326174 101.433071 351.576174 ] -/BS << -/W 0 ->> -/Dest (cb8-1) ->> -endobj -93 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 351.379152 99.933071 332.629152 ] -/BS << -/W 0 ->> -/Dest (cb8-2) ->> -endobj -94 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 332.432131 99.933071 313.682131 ] -/BS << -/W 0 ->> -/Dest (cb8-3) ->> -endobj -95 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 294.538088 99.933071 275.788088 ] -/BS << -/W 0 ->> -/Dest (cb8-4) ->> -endobj -96 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 275.591066 99.933071 256.841066 ] -/BS << -/W 0 ->> -/Dest (cb8-5) ->> -endobj -97 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 298.350419 124.541066 391.740800 109.541066 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rke/latest/en/installation/) ->> ->> -endobj -98 0 obj -<< -/Filter /FlateDecode -/Length 5345 ->> -stream -xœí]]¯ä¸q}ï_ÑÏ–ËâÈ"A°ñbd‡;íÁã<øïç%~H*µÔ3÷"ýàìL«È"Kä©Ã")Qæªñ烿•RŽ®·/—¯­¢×ø¯äX\”äŸñôW£)*orHþú….ª˜½áúzÁ_«Ë1ûë忹üÏRÇ\­Ê9Xã¯ýÄIäTL1j/$e$iã„$ïUŒÖ!)…[ÌYJ‚>’¥mR„”“—’`Y›„¤3Eã¶IfØhHJ‚&û($ ;ŒÕZLƒ!šÈY!ÍxÐèAJ³F…”mÓ"ÒL´Ršs*D2ÞHiY…|’Ò¶ø¬”æa‹7DZJƒ-äR”ÒlqÁi1 ¶Ø”a‹Eëd!-Á +‰i°Em¤´ì”ωHLËHÓ)¦mšÕ^ù䜖ҌQØtb< -¢faKÐ>‹i°Å;m¥4[¤4ð„w1%Á[-{·ÓÎHi¶XILƒ-è|‰l€-:‰àAŠyI⣢lI"›œ¢DIb›²¢D6{E!G‰œ6H³F"§a‹§ 13°…à¹blq™$.p¶ ƒÄÎÁKNâ¶% Eâº%Ä|KÚ‰ fj‰ ×¥@8P. ”%.pà\ÀtáîQâÖu>X‰ h×Q -8ð®#£%. ð®sÐÓ` ,qØQ9“œÄÞuFG‰ ¼ 7² Wp$.V”MIK\@à]›4I\@à]1ˆiQÁÄÞµ· Üà]뵕¸€À»–l¸@Qà-Fà]Pˆ—ÃØ‚•¸ã=Ò¬“¸…)‹°DâÞ5ßàÓ`+eRxð®Iè>1-+Ñâ‚ߢ•• !K\àÁ»¨œÄ¼ úL€ãFT+qZJļ‹¸ÁK\8(ƒ@âŒoÊh@WLËJç”$.ðà]Ð’•¸£°Ò L"¦E ‰ @ÆJ#•¸ €w1¼g‰ xW{ç$.à]M{Ä4Øâ’•¸q ÒංO#XUÚ‚)¤4ð®6èa) ¼«uÌð.›„È›$.€ -Q°•¸ $R9Æ qAÈiùÄ´ pí%.ˆÊÓ` E’¸µÊ.'‰ ¢…-ÎZ‰ ¢…-$.@D ²Aô-N` à"ζ R”¸L¦R&’¸ ú€ùZUJ V¥ºÓÒL”¸ FRŠÄ˜K©§—¸¯8\’¸f ÍÄ1øŠÄŠT²ˆÌ¥4[dK\€HiÆH\,lA4(qA² ³Ì %.À¹$šZJ#ÍóÌ$qA¢ 8T”¸n‚™&˜RLƒ-  ‰ Ð?<Ùö`òƒÉ&¦9blqž$.HÉòt3I\l±ÚJ\lò|3J\5lAÄ'qAÖAq˜,qhiZK\MÂ|Ó’ÄðKžo&‰ ²Ó˜obŠ'ømvç›Qâ‚L°Å#DÓ` ‚¦,ø-8 óM°“”`‹Í$qAç›I℘o"¢—Ï7£¸\ÞõÌ$¥w™‰ pù&"[in¯A¼>">“Ï8“¸b A½c«¸d Á½­¢¸f A¾‰OŠä '5ÝßLAˆW4ƪÙI¿Iéȳ1>c&rm? ›:y2êÃįŒ´¨ÏcšŒ_.ÈF>®ôMF®xgÛÎ?ýrA46í…ëÐvI,b+ º¿|¹üøÓ¿ýú¿ÿ'”ùËçˇ1À -J“áù_3Ënóïžcâ·®ËïÛ|Íùý$«º>Ϻ¾ëp––Ž²Í\ŽS -sY«|hÖéßÛ²~ût=×]òÛz/¿ý¯ë/¿Ûi‹<¶E¹ýЋ¦Ùt¾åzk»&Üúo‡|öþOsZ˜õj^?ÝÖÂô¡©µÛ˜ÌëŸä¬ª¶&œ[É÷»ýç_x³¯¨ÿû¿\~üÇc1“T„W¤È«–„)_ȃø•7 sN¿ÿvùà ˆ¥Ü…˜1ÍHü -œWhE¸qCñfY !ÞEʲ«®çv.Pú4µimë?3ÿ®3\xâB Ë¿‡~û<÷Mô¸®Ï³nìPuÇÏÐiÞåvë! Aé¯f`W“•Õe׶QÎÃ*G„ãÊ×íFŽý«¡ÁIÙòtƒ™Xh_+Ÿ:·]òÒ§ÜÇ¿þç~ýןz—¶Ÿ)…Û×Ì´R\¼öÚp„/xN Âû—Öû2çß6âã:OÑŠ£ÙIßkÅŠxó§ ©úe úJ!z šZW¿y+ -P³ÃhÖ6¬½µxL˜ê«¬ÇuL¬·dFï»M%ožPóæaPÑ’7.¼ -ÓÐŒi¿{÷À«Ty - ë<´Ûž+Z×9…š{¼á/Žž4g~ -1dïŽ0}Jåy Mnì›(G•‹µz¦è—-Õ³ÛœžOãhãJ¼[ o·þ¨=ÏéŒî7m ÜÚE7"~42ÛòH E~)„WžP{Nçiøøkc¡¿³„°j´“[§¼Q(¾Ïg´"ï9š&D!ÞâÇYîwÄ9çéˆXž5‚-Çïø¾Áo L½Ývö3}WÐ{Ö¾0ÜÏ:Íïس¾§ƒE‡uÒ…?ÉóEâ¯LÌ1žŸ/žÔyˆfŒÈ.R’¹âï±àXð¤;Ôµ¡–ÏÝw‰Íýœ J~ÈÞHü:‡½Æ¡¶îˆjÏ©< Œ½-kéüÖ”8äiÂi|·2~.Öèº.*û00Ⱦ6oƒk²^ÒîÑʾ~&¥3¿‚ è;Ì~IËÆ{JüWˆç»þ”Êót½ç‡ãlŠyg+wÕÕ¾G3´N;†ÕVŬySiYñƒM'~¿0èடŽwÞí1œSyžNˆö¬÷Qö¿:„œA:Z58Œ[ºX!ŽK -.þ|Þ7åîv\‚Ó‡}*Ïúiã˜zîœÎÓtFø¦D}ÜuveH¸ýè]Uò¡Ž|zµDwà{Æðë¶Îf¡¼C -¶V+Ã/%AûÄÝX[^ˆ¡’~yf@Q0VigÈOÏ,ú)M9Oê<ˆlà·4sðï ¢E% >CÓú$‰íkeVr‚Ü=©ìÑŠA¨ë`|zf‚ö€E«œ‹‰ÖÍY†±O‹ïÌE^J+¯ñs#ÑaÒq°s:Ï0ñß1}W–ZTr -`uj:.LÅóÑÄ™MÝÇ€³Ñ>¸d˜d¬–ôï•!>pŧÑNê<à8ÞÁyy¯ú·¨äà6 sG ¡PÞ­A¨íb#n´Ï@løFÿ.ÄRTÉdïø1Hƒ;˜ž€»±s:O1~-™Ÿìó;{õo±e%e(y8˾ý`šÊ+m3˜­ ÇÀËè6³•´O´•bеIl;ƒiä˜=Yë¦cv2EŽ€wRçy€WÎ1|†Î;oQÉwï;ÙJ›8i£}ˆÆ+ŽÞÉú÷pD™_&O¶œ¿DñˆçtžˆžÄø°ó$ÂÛqQÉwñáÁ·GX[+Ž9p`Ðö™ÁwþFÿ.ôxÔÖ1”§XÎBïœÎó@/"X°`nù©¢7‚Þ¢’½øN|X•†Í -»ÚSû|žûnly€í¸Ñ¿ ÀÄk-|rºL+”æòÑÎÛI§`Ò4ääÞ“û–•|—L+§IëãÇË©ÃjضŽc`¹XèHGAû °ú¤Fп³—P.Ë©!gŸ²>©ó<À²üz‚õvgK÷m€µ¨d¬Ô7T¶e©/ oµO-ê¶EaAÿ\F_y>Zèìc·çTžFž_.ð˜Õ¼'Œ•­vîËÓJ§7G·åChàFû „nõïA( -|tÎ#\tNçy@y˜ˆÙ¼+-*)¾[=âññlJ{ˆ…6Ú²ÐVÿ„?OËG;”ó—£Füs'ÔyeÍÞj>ôöý ´¬¤@hx4öÌörç mYpÐVû1jç×ù«úŒy}^¨ÝÕVÙxe’‹|–ÛYXÔyXY~iaüØÈÁjQIÞ^¾ R›r‚ÔFûAHmkR„)ÊöüåIçŸø…ã÷„Ô¢’ñ §qé`|.sñ~Û‰€¼ÏͶuîÏ̶Úg`×7œ¶úíÉJZ-‚ÔxQ†°5FÔt~·ò¤Îó@/ò žB~W6[Tò¡®Fí½V½~R}ý¨ëy mê}dßr«ýؾ¥ '"ËåˆbàÆ•“\tvѽ¯uRçiÀÆG=(“ùpîwDÛª–q‰añª};Ä:Ó •=Bu‚úc\'°DÙÏó ‘_êse—1ù<;Ú u˜åIÅÎíu_ûÉ";çq•£¼"RÀâ&säÃr¥FÖÇ•ÙÃîr=ÔÃÔýÝN€³ó±VS¥ûÁ=sPU)yÃtš…“ŽÜÊs?ÔU Òƒ-Ÿ6ÍÿõÊaͦدWcù‹>WLÌùÍ¿˜¯|@tÒŽLmnSÈ\w4Ëw¾¶\ü7Dš¿–À'e—î/æüÓik_.&ò=²÷LŸ1ÑEÌ›0kñë…?cC‚˜¨Es$fáüS·²È´£mªìŽ‹*¯sòéóÏvO9¬DÏNQï†õ}UsÓIc¶^1v§Š!4ƒÙŠ_Ëé@f󧀒róUh¦øPm†°HcÞ^`W<ó®Dú­PÕu±¥já£ã[Ó….l¦Mg†î¯ymºóžðvAµn-}¥ü’ZTm]«›Ì Få&õ5gëjý$ˆnAøƒ®E:³¥½ÆǸOÂTD-_®ùÑí"æ›Kó­§bË•ºj®-ßMéPw©5¼ +m4K#ÛlðíêÏ¥8pÔ¾ÖiÀÝoŒ9´¿^|rûΞòVè†r›8鎗.©6\³Êølþ{¢ -ߤÌ<凡)>Ⱦ_õJØ7›xÙH£<4Ö!©8Ê7)ÊŽ±avl/>êl›ŸI·ån-¶M6ÈcêVthä±›†º&f -«íÓ¸rêjºÛľAùu!îvúæª^·Ê‚àî[¹]w¾ŽâØzÜðW š½­ÜóUx+­Å ýT+éb^í¨ði• ÂnH¡‹;¾qõÖÆNü8WýÙLqo—¿\Ê0g·…gø#;c´d®Cžßáÿÿ¾Ðõ÷õT÷i·º±…mܺ#’-ó'Úª°¹Í ´4Ð~›ÔóúŠ†6Ž[;ð5uu݉½æ4AMK…¾·á ¶ ºÖ¸¡eüámÕŠ-ùÞ¾éötéêÍñ,‚¿P­ÂÅx;¸L=Íõ!ù:‰+ÎÚ‡t¤’q1ŠË¡ÂÜ-[˜É€+iMÏöÓؼ…2^ a_ºžF}˜‹®²rC„Àn¼ &3VšöF³{}w¿Ï&‡ê,mÔÞ¤¯£´ÝKáÉRW.Ãò²9®LR«óöje¹Á÷\ rºkôЂ#Fiêjs?VJU¶¨´. ÓUÖJŒÕ¦0«Õf)wœÄVŠ-¦èÒ¿Œ-íÇ^ièÇ^ ›¼\nñZ´äZäG8Ç~wºçl±ddaÔnRê9]•më¦eÝu´½·AÝÐ#SÝÕü&ç7ñß·øŽxf·”«þ)+&õï ` &ÃgU–OÛê8„øëd‘Ò°Äñ°Îý5íÊF]ókBw(rNGÃ)–MxEB>NŠª0–Õ²oé±–t:ÍmOXwí°Û²°Ó.ùfË¥uô&þ–^Ô;'®\,”?ÿsp’o -endstream -endobj -99 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 98 0 R -/Resources 4 0 R -/Annots [ 100 0 R 101 0 R 102 0 R 103 0 R 104 0 R 105 0 R 106 0 R 107 0 R 108 0 R 109 0 R 110 0 R 111 0 R 112 0 R 113 0 R 114 0 R 115 0 R 116 0 R 117 0 R 118 0 R 119 0 R 120 0 R 121 0 R 122 0 R 123 0 R 124 0 R 125 0 R 126 0 R 127 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -100 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 289.074296 784.370079 316.109452 769.370079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rke/latest/en/config-options/nodes/) ->> ->> -endobj -101 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 650.584475 101.433071 631.834475 ] -/BS << -/W 0 ->> -/Dest (cb9-1) ->> -endobj -102 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 612.690432 99.933071 593.940432 ] -/BS << -/W 0 ->> -/Dest (cb9-2) ->> -endobj -103 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 574.796389 99.933071 556.046389 ] -/BS << -/W 0 ->> -/Dest (cb9-3) ->> -endobj -104 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 555.849367 99.933071 537.099367 ] -/BS << -/W 0 ->> -/Dest (cb9-4) ->> -endobj -105 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 536.902346 99.933071 518.152346 ] -/BS << -/W 0 ->> -/Dest (cb9-5) ->> -endobj -106 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 499.008303 99.933071 480.258303 ] -/BS << -/W 0 ->> -/Dest (cb9-6) ->> -endobj -107 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 480.061281 99.933071 461.311281 ] -/BS << -/W 0 ->> -/Dest (cb9-7) ->> -endobj -108 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 461.114260 99.933071 442.364260 ] -/BS << -/W 0 ->> -/Dest (cb9-8) ->> -endobj -109 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 442.167238 99.933071 423.417238 ] -/BS << -/W 0 ->> -/Dest (cb9-9) ->> -endobj -110 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 423.220217 99.933071 404.470217 ] -/BS << -/W 0 ->> -/Dest (cb9-10) ->> -endobj -111 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 404.273195 99.933071 385.523195 ] -/BS << -/W 0 ->> -/Dest (cb9-11) ->> -endobj -112 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 385.326174 99.933071 366.576174 ] -/BS << -/W 0 ->> -/Dest (cb9-12) ->> -endobj -113 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 366.379152 99.933071 347.629152 ] -/BS << -/W 0 ->> -/Dest (cb9-13) ->> -endobj -114 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 347.432131 99.933071 328.682131 ] -/BS << -/W 0 ->> -/Dest (cb9-14) ->> -endobj -115 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 328.485109 99.933071 309.735109 ] -/BS << -/W 0 ->> -/Dest (cb9-15) ->> -endobj -116 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 309.538088 99.933071 290.788088 ] -/BS << -/W 0 ->> -/Dest (cb9-16) ->> -endobj -117 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 290.591066 99.933071 271.841066 ] -/BS << -/W 0 ->> -/Dest (cb9-17) ->> -endobj -118 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 271.644045 99.933071 252.894045 ] -/BS << -/W 0 ->> -/Dest (cb9-18) ->> -endobj -119 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 252.697023 99.933071 233.947023 ] -/BS << -/W 0 ->> -/Dest (cb9-19) ->> -endobj -120 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 233.750002 99.933071 215.000002 ] -/BS << -/W 0 ->> -/Dest (cb9-20) ->> -endobj -121 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 214.802980 99.933071 196.052980 ] -/BS << -/W 0 ->> -/Dest (cb9-21) ->> -endobj -122 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 195.855959 99.933071 177.105959 ] -/BS << -/W 0 ->> -/Dest (cb9-22) ->> -endobj -123 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 176.908937 99.933071 158.158937 ] -/BS << -/W 0 ->> -/Dest (cb9-23) ->> -endobj -124 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 157.961916 99.933071 139.211916 ] -/BS << -/W 0 ->> -/Dest (cb9-24) ->> -endobj -125 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 139.014895 99.933071 120.264895 ] -/BS << -/W 0 ->> -/Dest (cb9-25) ->> -endobj -126 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 120.067873 99.933071 101.317873 ] -/BS << -/W 0 ->> -/Dest (cb9-26) ->> -endobj -127 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 101.120852 99.933071 82.370852 ] -/BS << -/W 0 ->> -/Dest (cb9-27) ->> -endobj -128 0 obj -<< -/Filter /FlateDecode -/Length 5053 ->> -stream -xœÍ]ˮ丑ÝçWäÚ@Ó|_À  Ïb xc -mØ/ªsªl·hÏ¿ïJ‘¢’‘)*ëfµªPu¯ 1Dž>D™³Æßï þKdTJ9:_~>ýrÒ*z?SŽÕÅ$þ#þE6š¢ò&‡äÏ?s‚‹*foc8¿ðßÍe›ýíô§ßœþo­cÎVå¬ñç~f9SŒÚ ¢ ‘6Ny¯b´>¢`1gI3|$K½( ÊÉK"˜AÖ&A”`†£h\/Ê0ÃFC’f˜ì£ 2v«µ(ƒ!šÈYAf¼ -¨ô ɬQ!e›EY„ÌD+ÉœS!’ñF’eBðI’lñYIæa‹7DZ’Ár)J²[\pZ”Á›¢“d¶XÔNd ¶]I”Á´‘dÙ)Ÿ‘(ËéS/³Ú+ŸœÓ’ÌåM'ÊУÐH! -2 [‚öY”Áï´•d¶þJ2ð„w1%¡·ZîÝN;#ÉÒ0XUy0ÕÖønPç8³aêù•œ¶.dr%ŸβïïL—!YoÃ6ð–Y¯=P¼oдI¬ÎÔòúM$âùÇщ…Aã×yá$½x«B¾ -x_éd(u6m±R§=DãGïdý‡ &ïn"Ø,ÙiÅ- ŽéˆA!¹WqUÈWq·ó]"¬ÞŠ=“Z½ö¾Y­^ÿ!ôRäµ^G²1)“xWèôÆt=ÞÇ•LFøüBè­ ùXøªó<Ãhž"½‘µS«eÖ±N½–)×ØßoÒ»,Ó³ÂêcÓ-.òŠZâŠîŸc›7§ýYÆ› hðf&«yPÐ0Ý‚î¦R°¼ÀmCV”ÉÙï ÎqÀkp$‹˜ã•à]ò x+OÚµÓ^åqͺoýÀ Qå@1'á¶ÛÀ¾Ónü^ÿphÓN»nð¿2:N›-wLç8Àõ¼È‰@9¿¸«B&à:„¶t]ÈjÙôÒ¸uM¨éSÞ ¨ b0RuA 6ouz“ƒxçI¯=T–ŒIGI¿ êLízt1Ìw‰×Íw@wLç8Ðü2@D$ÿJè® -™&„¨qþ%n¥ŠÂ’Ò2e &¤5—±A”³ÄK”€¥`ã6T wÚ#Pm Þ×P^j¥ÖTØ„lJ y(Áåk Ñô4oô²c:‡,iÞ?¨ù½—×Av]ˆ&TFô äbßÛáU«Wáîšá"l9x¯íïŸðÞz1§sǸFWbuŽ*‹ÃÙäý~Pío‹@‹zsSèj¢Ü4½¿L}¹`Ü+/Óä}Y{¦É{í}Óä½þ>WÌ´KÓ+c`ÞiKòcŽé‚¡Cö¨o ÁU¡¥hñfL^ùG¿ç6©eR§·hÏlf¯½o6³×ß¿S -7Rü®§£éýÔ¨£v[NxPç8`åM½>Yz‚/Çðª7ú–ÿŠÃl†?5Z*ØMð5,Ù•x×)ò4¦6ÄÛv†â˜ÎaÙk8q‹ßžØDõ4#­ }­SìËÚã{í}N±×ßç=îšAZ–Æâ Îq È/&ŒªŸØ‚õ<W…®œâC'w3ÅB‚úawØÛ²ÇöÚûÜa¯¿ßz$lH–×~øµ°ëëHa:¦s˜zN νr»éº:R8CÏ0+P w ·k<ùnn¦¯ûçØ3}ÝkîõŸ€4¼7À`üø*ö Îq ÍgÃèÌGl¼Ò«BVÎ> +ÓÒ*Š°úÒN\×n±´¼ÙÙu/„*zkÍŽmªƒ:‡BÐüvôt’ηsÂëB_öeí‰{í}q`¯¿/ & ?o ŽuŽAǯ¹‚¢o ÁU¡¿nØÛ²'ìµ÷Žþ~§‰û(‹î¦Íø#ƒ:Ç©çwÞ¢1[¿†ß"[ßt×kdwàºâ—ÛÕÞ&õ~|;Xoé=·oœ×?üÚ ÎqÀØÍCû•›±Ö…|ý;hýýö¼ƒÖkï{Mа·$d‚²ÓÉ (ÜÍå­‰ÖAÀ(j~gÎzýÊͤëB^ûZ_Ö=Ž@H¤tðõO< ¢±~£yuŽÓ¼|æ@BÑO,f?׬ ]í6(¥‰YV±Ì­ƒù$Âöm`ïíÙÛôÚ¥V\ãÞª…‚µöó}WÚ¹ÃY×…ÖEÞŒWqmoåM¸ŠÃ»ôÞ¡@$«œMŽ÷õuŽÓe<‡1å'¶,=ßeV…¶c´ÛÍ9·±WݳüÓ%Ÿ¿Yo]-gíØÊ´lFêmݳë®×^jªƒ[àíñ|äÙó)b)ÙÍíFƒ:Ç[â}ÚšÒ•àæ` ý2ó:¬M×¾o9ý§ëõm:çµ³#u`P{™ïóÓœ—®ùË= ® C¦È?]ïåô|0—UòñŽ6ð‘þ2nså¸]w6Ÿ¦æ]ŒnÆt‡4ÊèøÈl ߪ¹vÁ©)ÏÞ–í–iSWܦm¸ÌÏR˜m–³½m•á‘ЪÆ -iƒ:LJãwÕx#Âãù7C«Ë&´J³áÚÌåÝá)hÝñ™ÉÙ„ì1¨sxxv~)DyÛÌó=îÍ öÎ%>ÒÐð:0ñ~‰ÞނǘÎqà‘ø-ƒj¹ï\~õæù'w§iQ÷#/Oã/Z ê¦i³æwšH‡o¹‘e]èÇvx+¼uÞ>V7×µs?·;Bw—Ð۲縄^{ßq B阮17|{ïšdg“• q:{txPç8vüfSˆá•[\Ö…|Û³Šú²÷œUÔkÌ|/K'½þ£wÙ3á6gÚqVÑ ÎqçÙyÃÜWž¾¶.äÕgõ¥íÙgÐkïÛgÐë?„z(ÛÀçžU4¨sˆ%~K‰¿ñJˆ­ -ùuÎ*êmØå|;íη¯g¡€xïœ?«hPç0À›ŽÞ6‘½rwÝM)Ç8­H0jÏj ¾o+‹pƒüã|ê9é|bà #Èö¬õ -ÁÍ,ˆ›ìÏ?››3¸Ya~.ø§øs‡ünñ– þBé‡ÿþñÏø]3£à¯Ë[þøKÍÅÿ#Ió'¢xCÀÔíÅœÿ—©¡~>° ?#÷²ë·Ûô”ÌÛyo“ßNüí>’†ÙÁdNœÕõ^¼%ê&í.Œ*ßæt^éù×\oî|ùÕúò4¬ï‹o¿/WŒÝkÁB"4ƒé“ùé5å×d>:6)7_…jŠÅf$.7¤6ïrÃEq3™wv,‰¾OÔMqK2€\òÚúk®mŽÄjz[u¦iþ’×Ʀ9%^N(Öݦ¾‰©üÕµr«R»V×4וkª/9kƒPm'!ér¿ÓåÖ ªSâGmÄ)sYMLÿ ñkð -endstream -endobj -129 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 128 0 R -/Resources 4 0 R -/Annots [ 130 0 R 131 0 R 132 0 R 133 0 R 134 0 R 135 0 R 136 0 R 137 0 R 138 0 R 139 0 R 140 0 R 141 0 R 142 0 R 143 0 R 144 0 R 145 0 R 146 0 R 147 0 R 148 0 R 149 0 R 150 0 R 151 0 R 152 0 R 153 0 R 154 0 R 155 0 R 156 0 R 157 0 R 158 0 R 159 0 R 160 0 R 161 0 R 162 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -130 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-28) ->> -endobj -131 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-29) ->> -endobj -132 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-30) ->> -endobj -133 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-31) ->> -endobj -134 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-32) ->> -endobj -135 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-33) ->> -endobj -136 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-34) ->> -endobj -137 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-35) ->> -endobj -138 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-36) ->> -endobj -139 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-37) ->> -endobj -140 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-38) ->> -endobj -141 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-39) ->> -endobj -142 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-40) ->> -endobj -143 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-41) ->> -endobj -144 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-42) ->> -endobj -145 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-43) ->> -endobj -146 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-44) ->> -endobj -147 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-45) ->> -endobj -148 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-46) ->> -endobj -149 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-47) ->> -endobj -150 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-48) ->> -endobj -151 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-49) ->> -endobj -152 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-50) ->> -endobj -153 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-51) ->> -endobj -154 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-52) ->> -endobj -155 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-53) ->> -endobj -156 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-54) ->> -endobj -157 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-55) ->> -endobj -158 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-56) ->> -endobj -159 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-57) ->> -endobj -160 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-58) ->> -endobj -161 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-59) ->> -endobj -162 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-60) ->> -endobj -163 0 obj -<< -/Filter /FlateDecode -/Length 4897 ->> -stream -xœí]]¯Ü¸‘}ï_Ñφ!YÅ/`0ƒÍCd`Yìû`ßµ,î˜äaÿþžR‹ժÝ;cØÓªb‰%òð°H•$w¶øóÃ?™É¹¤Èç—ŸO¿œ¬IÁâ¿©Äê`RÿS8;ËÉWbçŸE@ɤ|Šç×þ¹:ì‹¿žþýw§¿¯mÜÙ›R¢wáüO¢b2)§dƒ¢*PYGŠ*“’QQEgp‰¥h*¸{ÞªÜà’ƒ¦‚ì}VTn'G[U>9ÖTpÕ•³ðÃykU±Ìä &¢Ñ£¦óÎÄ\|Qu :—¼¦#21± NÓc YÓ1| Å:MàKpÌVÓÁ¦œ4]„/ɪ:øâs"M—à‹GëE—á ЕU|±Ñ:MWÈ„’™U]Îæ”·:oƒ ™Èj:çL6IÕaD¡“bRt¾DŠªƒ/¬×t_4x"PÊY­^F7Yrš.Àï«:ø‚Î×ÀGøbSÔ(#È/iàS2\à´™¼ëƒõ0x׳(ücÕh¼ - z8_@°`¾‡Î“Æ8™ñK4.à]W0ý)cle\.Qã‚ÞuݧêŠq -®Œ[´²q1 xi\À» Ï¬q8Þ`Fõð®#ÌÔš¼ë|±ð.↠qà` 0¿g]UWŒ-9k\À» %¯qfac3˜DÕ% 9 @ÆÆ"Õ¸ ‚w1½ "x×" "x×2ÂU_({ ×@‡a«Œi«Æz0…¦ïZ‡Ötà]kSѸ ‚weÚÔ¸ &ø¬q4ˆ‚½Æ1³))E b±Ð!äSuÑà8h\,”¥êà 'Ö¸µ)T²ÆÉÃò^ã‚äá B ˜â}«Ëø¸¨ë†/ˆ5.“™\˜5.H!b=†VÕtÑ›œ@ת.Cç’Æ)±ÁTì4.ÀZÊd z @ðFÂ% àtŽ5.H¾`¨h\€©ÈdÈ\Ó9ø‚ [ãDºÐ9§qAöðÑ ÆÙg¬2£Õ¸34Ö’hjMÇVÖ™Yã‚ÌÑH¨¨q† Vš`JU_@ d±4.Àâ‹M,sT|¡Àäìe¹™5.Ⱦxë5.6e½™4.(¾ âÓ¸ Øh$LÖ¸4µ—±Þô¬qÆ¥¬7³Æ…,Ö›Xâ)ã¶P”õfÒ¸ 0| QT|AÐT”q NÂzì¤é"|ñ…5.(1Êz3k\€0ëMDôê–@–õfR· À»¡€™4x‘iÔ¸Xo"²ÕÖöÄâ3]™eÅ™Õ ê ˜[Õ- îÅl•Ô= ò•5’ºi7±²ÌQÝ5À´ -¥³ê¶ÿ"ã '\vŸú-ª8$Ì™9ÉE FtßÖK1ºQì²?ö¿ž~ÿ“ïù¯ÿ<}þÝ´SVÿ8€Ÿ8ÊÂ:ÈñÝ º2ÓéíÌ$äÉC8øq"•ÅIZœ7ÝÄILŘn˹ýpk—á±øìÌLWÆ}½X)8tEÝ×Ý6_ËWªõbøaù€àíŒ|Úý`œ6c=m`v®î4ùž¢"‰"JQñˆz-¯ -ÿøî„a4WÙ¡m‡ß˜ÍëíùÝϧßÿôŸÿöÓDµï>ŸÞ/áÝß¹[ÿÿ¯ó»?Ý©1„u)€VuÈiÇËiø¿Ýü«mΗßá#þ¦ùw€œ¯ô³M(?Üw‘‚bÔÌ['Ü?4žÆ%–É[ãð\Ø!´QÌé.Æ^êÿû5’r0Œ°.ƒj%pý𙼠|ÏÀVÒ.„=s«:ä´gœ}ž±S±4Ë+ÞÆÂŒ¹¸`•&A1QÊÌ[Gp×@»1ÁÝÛ­¹ìtw W›¶”÷³Aè¤IïoÐæ0ÈÃÊ ‹Oð‡{"ôÖ•¼ï0Öc+”SWºæöbUø”.¿+o6~ý0—¿”$ÔÍSH¾¹Ž‡Ð½lœº ùÆz  ‹x=è[û{àE„.ÑA™¢5C^Ð#ðŽÙ¼Xé{†›¼ù¬8S$›e°:é -¬õÿ;p¦5 +@ >â6?í,À@©¹ÂkÓ¹aŽTà¹ì¸¡ãÒÃγ9Nç&¹ERðÏd¦U%ïkfçkŒ!äF"gt\QÎ÷xrÃòÍÉ_V¬Gf·@[û{ ‘eiXl–[ËÖ°l]ºG ³9 ˆ¼ì§ Ú’ž¢u%ïûÑ+lçùøÃÌ.º8€ùÁû>*uÝâAevìLw³‰hÚ-½Û½ƒ6Çé^§À°¼#vv·¬(ÐwÁ]Uú¾>jt2OÎ^M5ù¸ŽTj¹Íä®üudó.è6ã -øÝ*¾>ŽX¦”%–-Ö­õÒR¸É˜ÄŒ@%ÈÆ}ÎÓ&ô}¸ÙnrGWndnýgü?^F¹Ïø‹ß^ä/Ç×r)ëgV °™Ïóq.Ë—òõœÇN Sõ.ç";Ÿ#ÎuÕr°u¬íçqÿ(ìñëûÄi½ç’Û‡1›ãÀ+ƒb̤Âá[u×.8uõùëºiYoÑü›üìÃË|-•Ùf½ø{ß×ðÈIö×åÞÉ8<ÆlÉ]Ê®PÒÙâí#n†V—‡ÐªÝ†c7O(_Þ-˜sLŽ^n›ÃcÐæ8ðð˜ü²Çü§O&oq_q²yľÆäBè_秌 –ÛfY²ÝÁcÌæ8ðrƒ -¡Q¹=¹üêÝó%“Ü®d(Ë=AD[Þ8›¦ˆû];fsœ®M’èœ8ê#ÿ9«–u¥„‚rÏ)]­\x‘q¿½ÞXÄ.»ð[_ï’,÷Ž¶Ö#û¨VèœÏ^«½ÀuîŠooó çlb°Q¶c$Ù/æ¡G³9 €ÙJ¦”• ÿçíª¬+ÚU¹è€ÊóÌœƒR÷c€.7¸¶ÖC7©Ú=®­ý½»£ìÈ`±O’ÆÂIìqÓ=÷{€´9à<&oò9„gnUÉàvß}ç8¥vǨÔöbË6àÖzbÀ7öw!F’]­dÁ‡+ bã#ˆÙbAD èÞgBlUÉ%~ìÜA^&ß­{&ß­õ¾ÉWi/Þ^Ä9ÚOOyN>>Þ˜Íq€'iˆ!£sž ¼U%_¼/œd;(m|z¼7Ýic=D º)²Sìï2 ŒÄ|~GÜ Ía€,‚_ÏLYWòk¥¾m½Ø“û¶µÞ—ü¶µ¿½€s Öó9GSˆ¦äé»È39ðäá­ŒUü×LãXŸtŠæ>]Ý]«Lv+u#í¹ó¿­ïÖÙÀh>³ß‘¼1hsœ. "DOå’U%_ž¼±=ßžä­õ¾ä ÅþNò†¦éi¿ûϨŒÙgÈ]*t!ÿèšÍõ¹Kš+›0¾íò²·×÷x.YÙ[ëÖ•^Ù䧠µî>س¼e"E»#ÍoÐæ8°ò8OBäû-a¿ªôØi~[_÷¤ùm­o§ùÅ( ÀÖ—i~ƒ6Ç[–@ -Ö¿¥ù=ÊĈ(BÏ–‰:ƒ6‡C²òTŒö·4¿½i~M!æi~ƒ6LJoûr]É·Í’ÚÖ½'Kjk½/Kjk÷À‰s‘cÏ’´9àHW.Ïܪ’ggImkÛ“%µµÞ—%µµ¿ 1–w«i•!W•´Ü¯Ú9EaÂxGwÕÉ|uíUbƒ¯ hÇ„…jQü}<‡b¦¯ÙdÅzdI²@SQíïä–ù¸Kñ‘ç÷‡øDüè–ò Íq@šå (yÛþ3Aºªä«p©[—ŸÞkwë]z£¯ #Ïò¶L@Rñø1L—ÛÐ[ë˜.7¢û{0-²ÐDa|Ù3hs˜N¯Bv‰Ñ>Ï| Úº–UVÜÌŠ-¬\‹IÙ./YìÙux/Pq`ÏKAó¡½Ä`É÷K+%ÿBdåÃ=’¼0uG0—ÿeꨟO\#×(cíòE-;‰e•{-~=ÉÕX#øóƒbÎ?m;—¤z]É^Àˆ ®KR0vþYÚÉ)ÔŸ>Ô«ûPÍ„>äá„z$ؽT¬aÝV,WoØ^ÄòUºlh>ŠÍ•«Ï.'ä¾ìrÂÅð¡X’áaØ -mWÝ"–×±Ö²¾ý,­Ï!l®÷Mçºî¯e}êºóžðå„jéZúªJå[XõTµu½m2êœ*MjÉÖ!ÜúI½œáw¶žk×ô]¯ÉE.Â\Ä­\©åÑËI-7Ÿ-´žJ­T^LKmùÅ•ê”[Ã+²©fi’ÏlvcÄRý¹GéÚ×6w¸ñáËIÿ؈_O!)¥ÃÂ6TèõBêÎÛÄÙ.xY¤ò–êË/ Í*“ï1È¿ªM*Ì3ýpÜ %ÓËÑR‰ŒÍ&^7R/uXk¹óæ6Rœ;¥†Ù¾½\¶Jy!ÝVºµØJÚ5Y'O¥º'$ÕÈc-v uM,VÛ§qe/´Õ T·ˆCƒòëJ¼øÚP ¶U]÷ÐλˆcãÎ×^œZ;ù@Nó··Ã|¾L_‹ú¹V²ˆ½m³ÝRY'\),â…F_'q­$z¯?—A¦Í¸/§¿¦iÎSð³Grò½·>ZrçÿE™?áïÿœøüçú‘Ë<îmc ߸u%îg2ù:hË×B«° ›Nè¹£ýEìòR6T4´yÜûŽ¯y1· ±×’®1¨kÚ^–6ìľA×;êZ&ܾ\µb d¾´o– ûrHõâP ɧiªW8è/‡yÑÑ2%Ÿ»SâHŠ.S:´ì;Cô'ÅaWaY<[¹)€ì+iM/þsßrKª?èÂ9¤EÇË4ÛŸºÊ¦«èŠ ¦þ ?±ÜõYÎUVþ(N¿yx}q¿Ï.Ç:DÚ¨½I_{i»0‘6ÂÓ¥4Æõa›shzym¼K µsQ7ö¨9ËÐXB ^ôí¤y1› „¾R®²U¥u¢ht™­²vÆT}Š³Ym–銳ÚJ©Å‹ôo}K‡¾WZ#†¾W⦬œ·Îx-Z¢…Îi¹:»”l±dVd±·nR^JR•mëæuÝu¶=w—Á‹#q‰Líb6%ßÄo;›«Ý­Tý3íh¸¼|’ 5»Dr?EÂ#8„åC™‰s·Å±ÛæþžÇvg£îyô5¡; Ù䀷.¼BQ+ºS½Î_3ÓÍäi[å’ö5€²¥³ÐœÝìèLLÆóÆN;L²T”ï£ËVVÎòY×dºÅ3mµÐjûdúóãÉó… -endstream -endobj -164 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 163 0 R -/Resources 4 0 R -/Annots [ 165 0 R 166 0 R 167 0 R 168 0 R 169 0 R 170 0 R 171 0 R 172 0 R 173 0 R 174 0 R 175 0 R 176 0 R 177 0 R 178 0 R 179 0 R 180 0 R 181 0 R 182 0 R 183 0 R 184 0 R 185 0 R 186 0 R 187 0 R 188 0 R 189 0 R 190 0 R 191 0 R 192 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -165 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-61) ->> -endobj -166 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-62) ->> -endobj -167 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-63) ->> -endobj -168 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-64) ->> -endobj -169 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-65) ->> -endobj -170 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-66) ->> -endobj -171 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-67) ->> -endobj -172 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-68) ->> -endobj -173 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-69) ->> -endobj -174 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-70) ->> -endobj -175 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-71) ->> -endobj -176 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-72) ->> -endobj -177 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-73) ->> -endobj -178 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-74) ->> -endobj -179 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-75) ->> -endobj -180 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-76) ->> -endobj -181 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-77) ->> -endobj -182 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-78) ->> -endobj -183 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-79) ->> -endobj -184 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-80) ->> -endobj -185 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-81) ->> -endobj -186 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-82) ->> -endobj -187 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-83) ->> -endobj -188 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-84) ->> -endobj -189 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-85) ->> -endobj -190 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-86) ->> -endobj -191 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-87) ->> -endobj -192 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-88) ->> -endobj -193 0 obj -<< -/Filter /FlateDecode -/Length 5033 ->> -stream -xœÍ]ßä8n~¯¿¢žX$R¿€Å>,’‡pl䡧³s‡ 7À^òïç£Ë’e›UVõôžAw—)Q¢Eê#)Ë*wµøÿƒÃ¯ÌÎä\Räëëo—ß/Ö¤`ñoª±º˜Šÿ‚Ÿ®Îr2Á•˜Ãõ7!P2©Ÿâõí‚_›Ë¾úÛåßÿpùŸ5»zSJô.\ÿñ«1™”S²A)*(²Ž”¢LJ>D¥(:ƒ[,E+‚!±ç}Q‚\rЊ {Ÿ•¢ 1ˆ“£}Q>9ÖŠ †+!)EÎBç­UË ˆe&¯”¹`"=jeÞ™˜‹/jYB™K^+#21± N++&ƲVÆ%ë´²Y‚c¶ZdaÊI+‹…"Yµ ²øœH+KÅctŠR–! ¬+«eÅFë´²B&”̬–”Ùœò¾ÌÛ`B&²Z™s&À6I-ÃŒ‚’bRÊÃ)‘¾ñ$ È:”y§YÈ8jH@²0f®ZY¨°†ä! *hX@Y<“†´eX‹†¸e[œ†¼e뢆À…˜VÃâRŽ¬araJEÃæ°SÃèbº' ¨K!z °Kœ£†Ü%vVÃîS-ƒ,0` €Ž†\& ¸KÎ& ¸‹iä5,€V ì0jX[1>g«aw}¶¬aw}‚ PË’Á$ ¸ënAóÜÀ]¬×°€»ž}Ô°†b€?V€»€ ‡«aü=Ê3'¹É‚iÁˆî[αT£;ÕnëcÿøÛå/p¾×¿ýïå릕²úßÁø‰£$ÖA¦ˆ/P7àÊLÍ1íÌDäIBø"•EHXœÝDH¸b¸ÛrmÜZdH,2;3Óqß/2UQÔe}šç£d¥Ú/¦ÒoÈt- ÄÈȱ§ÌNÔ'Y>BRt$±@D-*^°Q¯åUåŸ?]0æîo+´í²“Þœ1¸Þ^?ývùã/ý·_þågtûéëåóÞýôƒ»÷÷?¯Ÿþô ÄÖ¥lXõ!Íò ~ò­©ànÛuÀg4ÒšÎÈÆCÜ× ðüÚµór«ËZÝÚ&êsé>׺Û~ß"#µ؆í0Ê-ºÿúé€8†@n`Ï<0¼ŒPá@ýŽ½ÞÂtËy{;ÿüim¹9F˜ óÎò–œ?0ö1–wû÷°qdîfôÛøM#ÖŽd3}Õædӿ΃^Ïh!«ÅR·¡Ø%ø¢´w^`BRâ¬pÜMg? -¿ÜJ¸cDˆyÅßJ ?ÌÐS,GF4Æs#Â|öÐŒ@†÷Ѫ“Ï=$|YcõdT~vCáf`“Ë;P12‰­ú¨ôul`òÜ‹™sP¸G 9QʼÎÉýjm´w ,"ÎË‘ŠŒV€‘Òë#ã9%yð„ÅO[u2d`Õ7õqS­7`pÉìú>6¸Î`vÜ#—¡Vã'ß3¸,Éa±"6'Ùàì!¢ñœÆ༬¨ Û’¾£Á­;2¸M°Ê‘‘pœž -Ũôvlb "î¹GLl1ð=ÿ#_ŸiWWUHu”9òœÇÄ|œæ_ùž˜¶îdr%/ -fùw¦yZŒ ‘Ž ¯@m.¯qŒ€<äwPmVGà3õòd(1ËÊ&ò„ÂÉÇCÃã9áÉfy$“¿§á­:ù&ÃûF'Û™ÒN¦CCì iÇ=bˆ.‰ÞÙ)ü0NÞÝ% YöÓÎ!NG†8ÆsCL -BŒ™¾§!®:ù&C|Úù.Ö^Šc ìtÇ=â|;Ãßñ?4½œä)†<¡ŠEdY»lyC,§1<Ù–]Að¬Þn)µÃl8mé)ÌËUÆXØØ"[6?Z{‚Ù˜ýôxt9qçÝG#9ìúyfsÏýÜ"¦r—ÂbB"GYž¹»„1–ó˜R’mä ‘ÚGšÒªÑimçNŽ×¼l$q6§´uh.Nž/!t ÷ÈtŽkß{ãßKÎ&e©|tArç4æÂVv}Yy[áãÌeÝh‰T¤á‡¹Õյ븉bZô2¾¢½—ì™åÉ=÷s˓ʸ<À*†ç*¸– :£)ü ÏyÌ@îÈç>ÒüV~)‚ª¦Ç÷l3§]Ý—±çµúwr=“;í¹ŸË”Qq?­cɯÝ_Ý$Iö™G+ûÉ“-wd’c<ç1É_Ô®šäd -›|Š¿(Iu¿µDÛN²»†W°×>ʽà匼ÝF<½‘—l‚%©kŒç<ê’mŒ!cŽ|$‚¬ýüãÇ£ÂïûŸ[ø»çA’×ûp‘4þ¹ YLjús4…hÚÉúØž†XNcNÁÂz|ØF2lNëF{sšLh<ëêá}›Çaôï¹GÌfY@Øó?Zˆ h£Dž^suƒ<ç1y+#YüÈMeëFWk¾Û]e_:wRƒìã%½.ß÷õL¿ç~.Wîôéè$0Têcö0‰ ¢õt„Nc,ç1² ÔH¤£“0´ðv›%½eVÞŒâi¬Øµ¾ÞÁ dÌòµ»½Uë­s‡K}ƒ<çQ¡œí`‹¼"?ô¬(.Ÿ)ÏKwZ,’;‡bÿ냨áþ#‚v×òô}©rtáî.6J붨þËçÖu8Vg~ßv‹±vFTä`ïáåX^*Éò.ø‘ñœÆˆ¢•÷"§34ž5"î -¦n°ç•ùíæ¯ÛC†.ßÙ•«†TTqÇvím‰~ k›÷b‹°^÷m4yjÿ_×2·{ò÷Œ,ºhÔÊÆ tËÅ!Õ ÏyŒL„$f§#•9yç.mSœÐü.…ã[½jÓõlˆsïDs›áöÓ”õº6,÷ei¿ÑJgei¿)2w<¥£…=¿{™ï«tíûu;§º5Ò ½ñoÊwô/ ‚÷OûTã× “åÔ…­»mòHä‚ Ažóf×M’súv‹!×Ñ#ȸ4g÷H*Ÿ+*Ö>«rï±1ÊSë1X^NNÁÔˆ‡È2Æsfq·àÛ¸5 ÝAÝLÜEó›YÙBÕ÷E¹¡ÒÎEu«ÇŽõ06"ÍÐFñÓAJD4½¨ÿX…c<§Qa²>ر-OÕók{ßz5Õ0pûœHBÄÝs þÁÎËWUîÚz*ŒDtel £\ä@¯XèpŒå<*”÷j3ºxô7rÚ,PŸ§ðýEéÄÞÏ$;åˆo§c%ìÏy†6 #‰x¯ µë†´ö }¤×Ú<]½–è»6t…FÙ3'§NÉKÏ9PpG)ÇXΣÎ,»Ý¬Û74Sº` Zwg…œx(ɉ £ÛdyN3Œy:9‰äXËwÍ -ê°~~É{·GàE±ònÖ´$€»¬¯ßw ¿1u.rÌW*EN× ‡ÏÑyΣ’m”½K9«uªšºÇ¥“âÂ^AMÁ›%Ëâk›} ¸É´š¢7ÐØ?‚mFóèqlÇ{(ß=¸üqNt£b9‹°¸HW²l|LÓñÔûÃAdðùÛ ã%ÊA@N€òÁxB>\?ä9ò³ìâv”Þ‡(áIñi3>vÙcŸÍõòºVön­åŽÇÎã‚'c,§Q`±²‹šm|ÇÛðÊßm¶GèЕ}€‹+ÆÅ$‡z‰e8Ž¼î Ïy†˜dsqLñ§Vl¼®ËZÛuÌn>µùWö©sK¹­R“~÷Ño›Kw½aa /ˆ’Æ_ä9²ƒ8Ä}ß|UÈp9¼¼øÈOœ³0Èsž¡Î²VN“ý.Ðe_î? X#…zdd_Èî¾@ôÄÙ*ƒ<§QÉt¬KœùGŒê¤›ýK.“$F¨ñC§‹¿Ìçé!LµS¶VQVŠ_SÀa•ƒñžH=mîê6§»É¸Æy@#<…‘/Ò |õ,O?äìIÆŸÿõ—ÿøó?-Ã(É—<ô—WlÏ×GJñrûûºº{3Ñ©žl\̳ÙÎõ„Ú³_;=÷µsXå_o«²Sy±Ü¹Ûc³©NèúŠ7ÞzPÒü¾ávø¿ÄéìS7mã}”,Ç#ë‘s‡¯rÞj¶Ä®·›È]ïpN_+ò{«%¿A²rø¸,µOêè/æú¿OŠúí";æäÅ’oß -`'²¬¶lÉoùVVÈpS~,Äù£mmÉã¹ íHLÙÖ¤`ìü±´Æ)Ô>Ô»þPÙäÈÙU¯Ävo+DpF·'ËÝ[ä$7²@R64_Å& -pª4âÒ ÷u—ÆC²l\ˆaO´]w YÀªu}ûXšÎAl¢÷Cç:õ׺>uê|D|½ [ÚRßTªœç_›ª£ëm£Q'TiÔPk6…pÓ“Bz½(ÄlmZ¢«VÞiMNE¾s5$nõJ­§^/j½¹µÐ4•Z­¼°–:ò‹(‹©Sn¯Ð¦1š©I¾*¨›#–êÇ59Šª®‘-vãwÄ׋ltÛ‘ß.!)µÃ‚6TÓë‰ÔµÛÈÙ.ö²PS®C…©Yir¦¬ü¾AEhTAžéƒãfR²ˆ´\-ÈÜläõ õôØP‡µƒ’ÜŽŠ¶Sj6Û—ËV©/ Ûj·[Q»!ëè©Tñ¤x¬É®Y]# „ÕñiXÙmÝ-äÐLùmE^ä mªÛ:‹®3÷ÐÚ]ȱaç[ONMãNùnò¶v;›¯Ä×ɱñ–,¦Ÿk' Y¶rVóiuÄE0€ÂB^`ôm"×ÙÚÐI^P«—I¦yÜ×Ëß/“›ó¼Äª‘œ|gE-¹ëÿ¡ÎŸðóß¾þ¹’|óãÞ6´ð [WäÞ“É7åA²|ãQ%¶iÓ=w°¿]^ê†j Í{ßá5/ìvöZÓ5u­´'†e ;²o¦ëu#_7£Ø™oÕÍâ°o—ToN^T‘㵫T¸èo—y)£Å%_»&q%U—ŽRö#.úFqÙuXÉVbŠAö´¡ù¹WI_ÝEFÈ%-e¼¸Ù¾éJ›î¢«‚˜ú‹¾av}§e%"ô»§×7ë}9Ö ÔíúÖSÛ µžN¥é2®/›Ï¡é=¨:y—jmQ7÷¨9ËÔXB ^]ô6ÚQóÂ6W}§\i«N«£hpš­´Öbª2Å™­ËtÇY¥ÔbŠ…ú÷~¤C¯•6ˆ¡×JÜÕ•v«Çkѵ(ô朖»³KÍKf…{îFå¥&UÚ¾o^÷]½íµ» ^‰Kdj¶°«ù.ü{ÏܹñlV´Zõÿ´¢áòòµpÔìyš¾)Òæ!,_ö“8wKOó<^óدlÔ5¾'¨Ã0‘MN¾óSá 帠kª~¨Î&»þ•[zn”%æìnEgúÒ7ü›vÚe’TQ¾ã‘$èÍòÕTÉ:u‰Ç­÷6ÿ?ܦ°¯ -endstream -endobj -194 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 193 0 R -/Resources 4 0 R -/Annots [ 195 0 R 196 0 R 197 0 R 198 0 R 199 0 R 200 0 R 201 0 R 202 0 R 203 0 R 204 0 R 205 0 R 206 0 R 207 0 R 208 0 R 209 0 R 210 0 R 211 0 R 212 0 R 213 0 R 214 0 R 215 0 R 216 0 R 217 0 R 218 0 R 219 0 R 220 0 R 221 0 R 222 0 R 223 0 R 224 0 R 225 0 R 226 0 R 227 0 R 228 0 R 229 0 R 230 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -195 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-89) ->> -endobj -196 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-90) ->> -endobj -197 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-91) ->> -endobj -198 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-92) ->> -endobj -199 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-93) ->> -endobj -200 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-94) ->> -endobj -201 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-95) ->> -endobj -202 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-96) ->> -endobj -203 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-97) ->> -endobj -204 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-98) ->> -endobj -205 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-99) ->> -endobj -206 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-100) ->> -endobj -207 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-101) ->> -endobj -208 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-102) ->> -endobj -209 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-103) ->> -endobj -210 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-104) ->> -endobj -211 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-105) ->> -endobj -212 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-106) ->> -endobj -213 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-107) ->> -endobj -214 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-108) ->> -endobj -215 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-109) ->> -endobj -216 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-110) ->> -endobj -217 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-111) ->> -endobj -218 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-112) ->> -endobj -219 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-113) ->> -endobj -220 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-114) ->> -endobj -221 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-115) ->> -endobj -222 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-116) ->> -endobj -223 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-117) ->> -endobj -224 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-118) ->> -endobj -225 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-119) ->> -endobj -226 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-120) ->> -endobj -227 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-121) ->> -endobj -228 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-122) ->> -endobj -229 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-123) ->> -endobj -230 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-124) ->> -endobj -231 0 obj -<< -/Filter /FlateDecode -/Length 4460 ->> -stream -xœå]M$· ½÷¯èsËI}Frˆ0²p9¬Ç^Á8€Cþ~«K*U5»»f<›T’Ìv)©X¢øHJ*M8{ü|ð_‘àJ©9ÉùéÇÓO'ïrôø7•X]Lì¯ð›ã9xÉ.†šJ<ÿ¨Î.×H9ŸOøos9>ýñW§¿¯ë„3¹Z…xþù;e »\röÑ`U°|`ƒ£Ë™b2X)8†^µx‰\É€k“WÀ Ù‚œÅÁ K¹£·°ï4\²°b€Ä‚\! LŸ"W‘¹Å A¶…ˆtÁ Á‚BÑ …… -²Ìä-,€‡F.‰®¶xâ5Ï,INCE `&È4”&²&,,€~4ÙŽ ùA²‰4ÇäAŽbaA)¤éf±° ÈBž,,ÀØÔ|3[XP=dAÄgaAõÉi˜la`<ï-,¨¡ ß$±°v©ùf±° ²G¾‰Ï°ÛÊIóÍlaAÈ¢˜<È‚ ©v LB¾ t²x ²P jJšo f ßDDoN Í7³9]ÜÈdñ€»ˆL“…¸@¾‰ÈÖÊí=€7fÄg6³hÆYÌèð­æ”öÂ[esÎÀ|5G2' &2Ë’ÌY¸U0ƒ7§ <ðñ˜\aÂeöiœ¢J>³d}È -³D÷=çXŠñb—ù±Ÿ¿?}úÎ÷üý?N~5Í”µŸ€ÁÏ’4±Žj"T¡nÀ•›š(:¸‰(“„ð5uTˆ\!`qžtS!áŠánë¹ k‘!±ÊÌLO&ã}‘)¨Š“-ë‹ë¼•¬Üî óCú€àí4‚…k& bAŽ=M`¢¾°Ê[HŠi,PŠ+)v õzYþüÝ f4ßþ2CÛ/¹áÍKþüîÇÓ§_ÿù_ÿîsÜö݇Ó_~­áÝgŸ„|¿¡¿÷ˆsññKš|‹ß2O—ϘçëFÏ­þg=¿ûâôÛwëÞF~.ÍðÈ:Mè‘ÑDÿ@Aûª¼JAC/È6€!PùxzI›>oºˆ ->ÐLŠ¯=ø@ç %?PÄÎ:‡Ñ¢u$,ˆRÂGSEîÍDâûù3¬¯U¸I=sy$³žçvè鎚¿(vVÐô«L!ó}5í«s5!"aòoc1>-ßµ››µôÏf!~±–V¾£Ù7³Êòšß­‹ç2F[“šQW>¬ëäâ4«»µ—‡: Çºn§ÉÜŸ­²²]vjç:÷a9Ó¹èþávÿ¬ä°j‚ï/ žY±c*Уº¯ÎqjÖÅèˆÌº…åíà‰Ãàé δ§;q3HÜ_)ì–ÒãPW†AççÁ6í7¬ÚGÚ¨ë{lånƒêêÙ7}ÂÍU4!¨H½¡qv¨–¨}u3 H³hÜÙ÷GsPô“;ÎFKÐØåL©"è’ œÍÎ:Çér䙞ƒ.xÌ°ùßÌêT²Ë§ ÿ¦Šu:‹è<ÊÞigã¨X׳t¸|<«zMpK% >#e€ÙTH¡ûôH%ûêG%ÀS*ü*•h÷{cAl¯Û#qÔñݬÒo‡¶ë`=Ö÷˜Ú´•V²Î»Msª19A¿¤úHiûêFiº§¡¤Û¯³£{JÓ¸§?é-Hw%Ñ´€±W!;ëG! .,~{+rQÊo¢¤ž.«…Ðy¦úH!ûêG!Q'gáë«2›`ÌX¹IETåØž„;.:OÄ_ŸT*= -™wÖ9N—gÝ|—%½ÎîMð8}&‹j¦˜ÊʇvähÝÞ6ñ]Õ¶Þ+^·•KæóÞ2òu9nå¦Ã4oÿÜ–áÍDÅÓ{¶¾ƒ£¹nçx\ŠKÑ')ˆh¼+:¸bľ:‡°âuµÞë.Ó·°£ò·?‚¶F©:Ïpkr€aOØ †0Ô»5PnD¸‚\¼âZ0½8&MÀ¨yg㨙ຘJŒ¯s˜§k|èªúæ6üëþ¿ÜÆYjt‘‚èRóýnÞWç8Ýá®j„¼=üçʇ4}}›t|‚¿fU›é;Ù9ï|uŸ;.%n,R¶SÉÁr[Û©¸Ñ ½_ÊMhÀ—_šÛRÙÇ„êö|0´Óýú¬Ó¦Œ0;‡òh¢agã Fݘ ‰mómÀRªåçû¢ÃƧO‹å&Üô±RôÍ T÷O/ì¬sEDgAøö6 }/ò±ü( -í(qÃOFݸ˜dÚdX«îo¤ôHU;ëGUºº ëxÝbߘεUÓ -d­Ø{.Oï7±ïØn‹_oø[äŽ(!!ºöÕ9Žº¢RóÀ²~AôzÓÂRtúâOxÁ¢ÒÎ:ÇQ™¾“諾ÚõV*½’™áݲª¯Žfݺ#aÚøy¿»÷Õ9Lw'¯;ߧ·$ß$"½¡Ò°(4©%‘èv§Â°ÐÛ?oDzw£ÎrDo&)$“L‘ÅÞ­`;ëgp蔋„·±ÅGKSÛýRƒØ6ŠØÁQF -öê;ëG Q·fÀ÷DZÑ-dú2¿Ù¨@7¾—ù·Ag¹¡–¤[`òiÿœÉÎ:ÇQKQ¨GíW.0ÕµºËÝòP éMAénö½1ÜÎ:‡éæìu? !IúŸˆáàkO1M}º3†ÛYç8*Óô·~»Iã×ÆpYHR¸ÐþngãtwTàÌ¥¾Ñ¢ÒÿM —“niÐ÷Ø÷Çp;ëgpÝáõ(£ÆpÓ[˜‘³¾¶7†ÛYç0j(ÓÛܬGíü·Äp½B’TΤïÐÎï‘ÞÙe_㨅u'šq¯ÞÑ׎¸CìºÀ\Ûsò47àŒn¬KÇÍBÍÍ}˜þ-ö¾ÑÓKjH¬GXIJ¹>z·f_•ã¨=*ˆ4ö-w6[}Ú]—ôÅÛ ƒž§3(Âec÷ݾÛWç8WtßYà¼Ïfd^ˆÓqÍ7^™<Æf©~Ç«PC–…ˆ{lö†¬ö´}ßnÖ´¬û¶6ÛnËoüz/Ȧ^hõëmW ·Š'Ð÷ª÷î[ÞYç0âzÝý&>í[iÝ»ŠóµêÇ-â{^áu£¦bj¨. ™BÇT=N-ÕÈô²¯ÊqÔºC.å´o dµò4®8 ßGú­¼« -„¡Z%îcgãtmTà†¸¯\ Ákèöð y—‰ uᵄtÞonžl»+ãf®e+Y'¬”ÄŸñ-"êÓC îëxW•ã¨¸è>.=-jŸõÜ™\z4SqïjŒÌqÿŽãuÓÓ‘!K‘}+ -ÛìrŒ%^¸öýÕ|–†›?ó4ñ‰ˆk¶¡7yh·“‹_ÆËBÍ¥uL³Ñô<'ýÿ±Sy¦/AúÒ3Ù–«å&j›¼î¤‘ž:êˆÕAòRvè®s³c…âò -º½tï±u貞kOAªƒÇšú¨ëd…°Ö?+G¢oBàv 9ö¡ü¼"/rÆnªÑ÷›¥0 ÷ØÛ]È©cçóHÎ]ãAØëòöv‡1߈O“c“-Y‡~i7YÈä»·[n6Á - yÑç‰Ü¬µ£“®0·¯‹‘Y÷éôÃirsÄ‘4Mô¼Ø1Z -ç¢ÌøýÛIÎ_¶Ê.~œ|G êغ"žLO/;ÉzÚx#v³ˆ$ì/äP–²±†îlj¼–¥º_€½• ACçŽÄ¸ôá@¦>t)ðÐ3ññiÓ‹=ù¥ºYöå’ÛÃá6¬GÛ5©p1>.ËÂãÅ%Ÿ‡&q¥E—®ÐPc£¸nXÉVbê€oÒ»^å—Q¬÷.†0B/yáÉâfǦmzŠ¡B`/Ɔ%Œ7­+y ¡_m^¿Xï³È©Y€R;´wêóHí¦Ôx6•§Ë´¾ì>‡§ãšñ.=ÔÛâÁö¸9‹i,¡…¬.Æ1:PËRm.Ç›J£­nÚE‡ Ð|£õs“)ÍÕZ·LO\Ì^Ê=¦X¨?Œ=G­ôNŒ£VÒUYm·y¼-q€â8œóòt~)ÙcÉbÐÒX»Se)Év}oYß»yÛóð²’–ÈÔ/ÕâUÉWáßklçRg3{`•j?ÓŒF(Ë‘žpÔ2O¥Å—”€!¢mg)ÃÇ‹ëÜŸó¸žÙhsã 'Ì>ý{;¶Ï`ÔÇŒ¡©ö·xìjºÛx¤—u€1¥³Àœ¿šÑ™þàþÍ;ý2kª¨_Eß --E…Ï>˜S<ÓTK\MŸL?ÿ.5 -endstream -endobj -232 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 231 0 R -/Resources 4 0 R -/Annots [ 233 0 R 234 0 R 235 0 R 236 0 R 237 0 R 238 0 R 239 0 R 240 0 R 241 0 R 242 0 R 243 0 R 244 0 R 245 0 R 246 0 R 247 0 R 248 0 R 249 0 R 250 0 R 251 0 R 252 0 R 253 0 R 254 0 R 255 0 R 256 0 R 257 0 R 258 0 R 259 0 R 260 0 R 261 0 R 262 0 R 263 0 R 264 0 R 265 0 R 266 0 R 267 0 R 268 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -233 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-125) ->> -endobj -234 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-126) ->> -endobj -235 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-127) ->> -endobj -236 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-128) ->> -endobj -237 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-129) ->> -endobj -238 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-130) ->> -endobj -239 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-131) ->> -endobj -240 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-132) ->> -endobj -241 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-133) ->> -endobj -242 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-134) ->> -endobj -243 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-135) ->> -endobj -244 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-136) ->> -endobj -245 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-137) ->> -endobj -246 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-138) ->> -endobj -247 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-139) ->> -endobj -248 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-140) ->> -endobj -249 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-141) ->> -endobj -250 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-142) ->> -endobj -251 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-143) ->> -endobj -252 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-144) ->> -endobj -253 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-145) ->> -endobj -254 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-146) ->> -endobj -255 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-147) ->> -endobj -256 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-148) ->> -endobj -257 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-149) ->> -endobj -258 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-150) ->> -endobj -259 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-151) ->> -endobj -260 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-152) ->> -endobj -261 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-153) ->> -endobj -262 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-154) ->> -endobj -263 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-155) ->> -endobj -264 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-156) ->> -endobj -265 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-157) ->> -endobj -266 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-158) ->> -endobj -267 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-159) ->> -endobj -268 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-160) ->> -endobj -269 0 obj -<< -/Filter /FlateDecode -/Length 4166 ->> -stream -xœí]ßo$· ~ß¿bŸ D‘DJ¢€"AûÐ)ô¢-úpç»KPø -\úпgWÍ.í•7N0hkÃö)ÍpHñ©_Gï/~ 'RKæãçÃçƒw%y|-%6 û;ü”t ž‹K¡fIÇOJ âJM±äãã¿..Çâ‡?ÿæðÏmpŒ®ÖC:þôAYL®H)>¬ -–d°Rr¥Ä” V¯X«Å‚©päkV\%Y,ˆÁ1ŠÁˆA\]³*Ĉ%°Å‚¡¦b°‚‡!zoò ˆg¦hðBrJÏ/—¥Æjò -x¡D‹Gärá‚Å«.çœÄâ1dIÕ‹— K -ÌÞâA&)/CÊäMd‰RÈâÈ¡jð² u‰Éƒ,>û`ñ*¹T…ÙäUð¼¹æEŸ\"oñBp m“L< -FÊÅàEÈ’}ª&²$òÑâda|[<àD¢"bxkTï&OÁâ%Èca“Y`| b†,¾d àANqÉ€XŠãÙ(äXX,ˆR—B ÄšçZ, À‹Áò%q¶€dax®Éƒ,TÙŠ,, ‚,‘ÉÂÚ2Z‹…¸e_ƒ…¼e²…À…˜ÞÂâ’d¶°€¹hJÕÂæ°ÓÂèÂÝ‹…Ô¥”£…Ø%–law‰ƒ·°€»D¨iò °…@GGAÈÂîRðÅÂ°ªC;Ì ­¸(â-,`ànÏ0p7t&¯8ø YXÀÀݘð -VÏ ÜÉG ¸9f ÐPðÇ›Ñp’ìp²`-,@^$ p3–XX€»¡¢û3|hå‚ÔlaAîùL^u¡@ã†ßBË.ä\-,HÀ]tTdaAî>ÅÂ`¼C-,HÀÝ@è©-p7Äê-,HÀ]Ä ÉÂ4XX€þͦkòªóUÄ‚Ü,E Ð ;/@“WR°°`ì<âP 2pÝ{µ° w}"²° w=#ì1y…$ZX€¸<¸­áÓV@ -‹Üõ¶xÀ]ïKµ° wµÛ´° %D±°:DÁÑ‚,ìj)Ù‚\=xùL^v¸NBd™<ÈÂ…-,@`í*U±° DÈB1ZXP"dAÈ`a"W¢o3=€,h.f~À‘¢…@2'•Ù‚’2ò1hÕâå褮Mž€Š…¥°CW,,@.åNoaÞi¸daÄ/°…¥B¸Š…芜DDæ/@Ù Ò/ $BDƒHd™Ù[X€¹$TmñØkž)g§¡¢…pdš@J“YÀ>šl' ü ÙDšcò %¶°@$jº)ˆ@–装h›šo ª‡,ˆø,,¨>; “-,Œƒç½…5òÍÈÀ/5ß *yä›Hñ ¿­”5ß,T†, !ŠÉƒ,šªá·À$ä›@'‹—!K¬laAÍYóM±°aòMDôæ€h¾YÌáànª@&‹ÜEdš-,ÀòMD¶VnＩ >³™¢§˜#ЛзšCØ‹Þª˜cà«9’9h1‘YJ6G Э‚¼9l࿈Çø -N£OãUFs(è3¥èKV¸#ºï9ÇZŒž(vûé‡Ã—oÑùø×áão–‘²öÐø‰³&ÖI]$V˜på–Û1 ÜBäEBxO¢ÔUÈX<º©èŠÑÝÖcÿ¶"Cb•9 ˜YÞŒÇç"S0e[Ö×y-Y©=î‡ôÁÛ2 -å0'X^c/˜ƒ¨/¬ò’âA d”¢;áˆz=o -ýæ7:?þ4BÛ/¹Ñ›3”ýñͧ×ßÿõOßÿák<öÍÇÃß~«áÝW_„gþ¿½f9ýMïðS.háü7Ÿéø Þßo¾9üþÍV¿ÈÈÁ^2 U0Þ6×&™«r—I~ K ¿DÈ…Ðä.K¤4hú¬á®]Êä­Uè¬y•G‡Á.ïÞeÚΈÐÄ×Z—IDÑ1癬³‹ NGª‚Æ^Å9L•÷°šçÒy¸lïÙ&®õûç8ÜÃ6âÅÒzº/‘~¸e¼¹:û1ò¢Èý¯ëO7ý)u”LÕ/£îD´duÏ«~®Î~T_t¸=U_ÇohU}óƒt¡öî7£iÿz0îõqíˆxè”6¦´(fV$tHµ\%Z"üçm8Ue7&Œš™á±Èèîòž:¨QM‘æ¼C'©˜£<"]‰ À¹¥Ø¹*ûQ,2§N•¾Šoˆá¶juì°0#ùФºš­?¯Ú©*ûQ­Î@èL©¶©ró×V]FöDΑ–íàoúûdý(¯ŸRÎBSÊ[ÂE:á%]8ûs~xÑŦõ~ ƒÃ’¶û¶xèÝ6æYžñþ\¶<óÜvÏçëw…ëz¡Õ×Q&~¬¤è¨‰Žˆ-^ ñÍf1Wg7ÍBg¤% ›ó©¦ÞE•M­åÚô1¯Ýï%„]åI'çnûÌÉ~Þ†öÖ6!€ØIŽ:`]u¢[ºaÁ¹*û1`†KŒOp“6Eò TÞÒŸêȉ˜·Ìò@CÈÃ9ð­ c²Î~T›tP ½c½/Fz"óÞ@$ ~A'Þ~ÒHÊäHtDs: ž«²]¤U8OzÏ30EƒÙ¸>“ˆËÉg8EužÃ2Õñ¼fçêìFµÔ¯ëæîñž1‰o/Z®l=f÷‡Ù8«¸Ö ‰Ë<¾±OÖÙO#ˆ€|Š’Ò}à×7lC‘®ÆÉžd¾¯›¬³C%àvMPÎÿh_‡.ÀéÊaâéHq®Ê~L¬ÄI"ÏùbºqëÀúÞ•»µ{ªßcÑõçcï÷&ëìFÍÉ¢#>ÍM*\Í°ý×÷{I^e^I‘w¬ ·àt²Î~.àÃ÷MNL÷{O6Ûÿã5cFºxŒ@ƒ ºcç–êçêìGõI©™è¾I½©ž¬ÁÞÃÅõe¨áϽÜåR: ²>·Óžš— -Ûç==‘¡ tu CHó€;Yg?×ÝU¾ê&•ÿ®ÙªnÍ‹1øî\Ý4‚ìueñ² mO€›Cvˆ]ªÏH¼UÓ-ÕOÖÙêI—F1‡;§À}Êÿ®–* ~×A—¯ïËO¦ ˆI\,HôÂô¬Û\•ý˜7é2«ÂÜÄÑü¬[Î:cî#Õ=sº«6ÜZò5Yg?Ê…QÔžœ^yY·pá-Hx{L¤'`ïí/3›–k–ÅeÀìp×dݘ»xò‹H_u6î0)Ñ–g-†YàÔ4zRçsÊÐæìhÈ\•ý˜JW! =7<ýZóf,GQHâ1ꆒè—M8Ïkv®Î~T›¤‹Ô¹I•{¾ì ëuàÝ„/Æ ³}¼(·õdYÛ¨Y'ýuŸ®.—Déf’5We?&]&àõ ƒÒõXá“ž¡{ÄÝ£¢ j -Ú»ÜÒâ\ݨQ–½¦¤ÜåÆBIʃŠ/¦$¯†g‡~b2|ð>{Û,m†9‹n“éâ¶[ óçêìÇ,¤k¹4L»/y¼ˆžÌI­_XO2¨!‚Ýâ$º/éy-OUÙ’“"žärßTàÕÂÈçÚlÖmyA·cÎF/sUö£LÑEJÊ-V~fk…ªgÝË<ÝZ§ªìFÁÕëZ%öù¾ü—´Öª ¹,›þgw£MÖÙ:I—å’±ª¿ Á6;f7mT†ÐH3YÁ"çA··L0Wg?&HŠp÷uZtœíôl…ë[5fÝ;ULÖÙÂE×êè (¯£p¹KÑ•]M ÑÍôøä\•Ý¨yÙ^ - ¿lþÇ üî¼}“?Ò²‡!Á¸ÿ»kìf‘ºÞÞö)Þ`ÕD>« KÁ¨+‹XGfõÔ}ñ¯ÿøý_¾ýÝúâºÈ)~850ŒïÎù±6 ¢H=¸…¯§Æµ”ÓF&g€=—Szz{.[NƒéKÙ|BÍB–2Ë ⧞ˤáYù<æY>|u©þχ¼œš–““|^úâ*zÐSŒ/õ¨'uˆ'MÝaQP8>Qs9òs/u>£Òë±U:¼¸˜c¼8—ÿ¼êÓ!À)ôµíΓó YG%/É=O 2ºŒ8IVâù£ï÷Ò™ƒ Ú\·¸zY’’óçµßœRûS{­ŸZ5=çA§ûÛ•¶ÝÓƒ "jæpMÖ·÷.œÉz&£8:_å.JÊMf×òXv½áZñ&Yg+Wbº&úáq+Yw·²±¬Ýæ vÑGÕ…Áü­l,ƒ9Ÿ#>ðXº¤>šT= ®Ýªi7úN£A¨Ú©©•ìán'ƒôp0ˆ_øvkt:°šž§s"JkHÜËÕVÎ =Ìr绥n©ÒKÉZµ6ͯ¢¬M¤+Þ -::S‹2;øˆ§öqKÎjfk$k»‰Wćƒ®A¹"?R1J§+|jMo$ÒpßN¿¶—•ª tNŸàš¦§‘èïT¤NUäY>îMJOZ¯Ö‡¨ovòVI#=wÔaKËZvPW(¥·ÙQ_A¼Q^A·—îÛP• ôR›x -R<¶äÐ[]'+„5ýt¬‰¾ Ç­äÔ›òㆼʙº«&ß–ÃÐÜS¿ïJÎ;Gréz´nä1„¾Û½~¶ÝÏ"çæJíÐÞ©#µ¿˜R;àÙTZ.óö²÷9´ì„oλj¨ß‹ߣ䬮±†¼¹Ûè@•µÚ¹@ʶyhë(:\€æ­ß±4™ò¹ZSËòÆbj©ô˜b¥þ8j:VéJL£UòUY½oëñz´D=Jcs.ëÛùµd%Å å±v§òZ’íúÙ¼}vëmÃkð*H^#S¿VKW%ï¿{|çTçbôÀ*Õ¾— ëtè¨9Š´ü/9CX‰-,ÃÇ‹ë˜C<ËPKÞ Ÿ,ßÿ§ö$C -endstream -endobj -270 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 269 0 R -/Resources 4 0 R -/Annots [ 271 0 R 272 0 R 273 0 R 274 0 R 275 0 R 276 0 R 277 0 R 278 0 R 279 0 R 280 0 R 281 0 R 282 0 R 283 0 R 284 0 R 285 0 R 286 0 R 287 0 R 288 0 R 289 0 R 290 0 R 291 0 R 292 0 R 293 0 R 294 0 R 295 0 R 296 0 R 297 0 R 298 0 R 299 0 R 300 0 R 301 0 R 302 0 R 303 0 R 304 0 R 305 0 R 306 0 R 307 0 R 308 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -271 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-161) ->> -endobj -272 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-162) ->> -endobj -273 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-163) ->> -endobj -274 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-164) ->> -endobj -275 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-165) ->> -endobj -276 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-166) ->> -endobj -277 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-167) ->> -endobj -278 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-168) ->> -endobj -279 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-169) ->> -endobj -280 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-170) ->> -endobj -281 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-171) ->> -endobj -282 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-172) ->> -endobj -283 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-173) ->> -endobj -284 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-174) ->> -endobj -285 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-175) ->> -endobj -286 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-176) ->> -endobj -287 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-177) ->> -endobj -288 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-178) ->> -endobj -289 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-179) ->> -endobj -290 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-180) ->> -endobj -291 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-181) ->> -endobj -292 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-182) ->> -endobj -293 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-183) ->> -endobj -294 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-184) ->> -endobj -295 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-185) ->> -endobj -296 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-186) ->> -endobj -297 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-187) ->> -endobj -298 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-188) ->> -endobj -299 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-189) ->> -endobj -300 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-190) ->> -endobj -301 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-191) ->> -endobj -302 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-192) ->> -endobj -303 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-193) ->> -endobj -304 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-194) ->> -endobj -305 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-195) ->> -endobj -306 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-196) ->> -endobj -307 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-197) ->> -endobj -308 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-198) ->> -endobj -309 0 obj -<< -/Filter /FlateDecode -/Length 4937 ->> -stream -xœÍ]M丑½ç¯¨³¡I?ÁÞÃðÆ6f±n졧IÁ(ÜbÎRÌðÑY7&E˜áròRÌpÖ&!)Á rÑИ”a†ÆII0Ãd…$£a‡±Z‹i0D;GVH3^4zÒ¬Q!e›Å´ˆ4­”F¤BtÆ)-«‚ORšƒ->k#¥yØâsZJƒ-ŽR”Òl¡@ZLƒ-6E’Ò"l±h,¤%Øt%1 ¶è ”–IùœœÓ2ÒtŠiL³Ú+Ÿˆ´”fŒòÀ&‰iQ褅4 [‚öYLƒ-ž´•Ò¶8ü•ÒÀžbJÂhµ<ºI“‘Ò1-+Ñâ¸E++B–¸Àƒw1Q‘ļ úL€ãfT+qïÂL-¥wÍZâÞ…ßà%.” qæ7e4 +¦e¥sJxð.hÉJ\€YXé&Ó¢ŒÄ c¥á‡J\À»˜Þ³Ä¼«=‘ļ«Ü1 ¶P²À¯A†­0¦á¬*mÁRxWô°”ÞÕ:f‰ x—§M‰ BdMÀ@/ØJ\’S9Æ qAÈipùÄ´ pí%.ˆÊÓ`‹‹Nâ8Ö*SND [ÈZ‰ ¢…-p$.€G ²÷-†°pã[à)J\&S);'qAôñZUJ V¥ºÓÒL”¸ F§0‰ K©„A/q^±»$qÌ@šqÄ [0T$.ÀT¤’…g.¥Ø'[âxºH3Fâ‚da ¼A‰ ’Mˆ2ƒ–¸34bI4µ”æ4Ç™Iâ‚ä‚bWQâ Dš`J1 ¶€&$.@ÿp°í%.@ðƒ`aŽ˜[È;‰ R²n&‰ R‚-V[‰ €MŽ7£ÄYÃx|d»É€Æ‘¦µÄÙ$Ä›ÖI\€qÉñf’¸ “F¼‰O·™Ç›Qâ‚ì`‹‡‹"¦Á8MY·à$Ä›`')-À›Ä9Ž7“Äp3o£—Ç›Q\.ïú f’ÒÀ»ðLƒÄ¸@¼ ÏVŠí5ˆ×Gøgrbâˆ3‰+Ôë1·ŠKÜ‹Ù*ŠkäË1’¸h3Y¦ ®`ZE¢Ñâ²ÿÂs'ÜWŸú%ª8DÌ™)òMf ï¾Æ-}!Û}}ì¹ýö&ß—¿üóöù7e¥lùk~rkÏCÄft7èJ•â:Ú¨"tÅBø6"æf¤->ÝØHLŘnóKýŬM†Ål³3SîÌõõ"R0è* -²­‡uβ•–z1ü>Ày+«P³GÏ'ÄØe³3õ Ê–¢"örQ¶Ìðzµ[eþþà ÃèQý}…¶^vvc6wh\«_>ü|ûíúïþó{Tûáóíã·ìÞ}÷éþg}x»Ú½²ç‡ÿÓ]î=~Â#Ò¼}ÈÍýóçV†gâvßýߡßßþãúw;¸\¸ƒ0ÛzÔž4üœÊ»þ×hoD‘p¬à€Ìµ÷O]›>ÚÒűM)lú"ßuÜÒ'4D?Möø¿öUZ÷™çzÄ^Â@„[¢sΘÅ0Á×¥'½4§r™^‚‡Ž Ma¦º‰×kóÇ6T\÷{/7–á8šã´; G&”ý–Ó¹NÓ"^±Ž¬žCS›®I»fïGCmöGšËÎ"ûüå~m -à“âçÞÿv¿3æt®Ó‘ÆÁvžþ78¯ô¡ŸPɯIS\vly*mnò®@Ò¢‚¡·CEb3#”ƒrh÷ðt„Îé\–£2T äJ XÛfÓv3w-òÏëÎ\ØÒð_4÷®2ˆÙ†¡¿mÄúem~ÊçœK^Ð.Ðx¢‹(&·m&Ö§ŸpZ¦v~ü– ÏüåiiÔQ“yœIëÑ,äÇÀpPø×5Hü†® ­/ÀÑÀ}êÒÃónr¡VóZFí`5¿EÐß8E¥`Ë#™dËN÷,Š™Ô¹°,fád1Ÿ ¬U¡ûi¬`–)Î -Îð§MÈúy¾r¤k™Ï€”Lqgôö¶ç`˜ÊR³ÒýÍÀ0&f+êïÁ’2¶<õµü ðñ0l†s:סçEvø‚ùL® -ýØ·5vò,ÍF¶…YŸy Ö_ÌE½WaÅ%4ˆ?…ô}·•ñ&wûÒÄ>Ÿ+ÏôF퉶vì0P²ä%ý=Hƒ’)ñó`Ž”6â3ŸoRç:Ž¼I5ºp*³® -teÖ~yaÚ“ë{Γ͵§üÀ²7 éï*%¼PœÕœÎe@Å#.EÍ;¦ÏÕºÐ_Ͳ¼àºk7ŒµÒ¨} J‚þ”°qÍüg=¿Ië@ cÈ&ïÏ„ÒªÐwóÓÙÓdçùñüFícžŸ ¿C⽪ABbÍÛa4"‹g0œÓ¹ =¦õìá9 ÃU¡_ C·÷è&Œp\‚•éÀ„ßO0è‚dËŸ{qÆ+~.⌠=Ñndo@M’þ<á’‘›Ÿp'u®OÞúåFÿ™ð\ÚÇÇVþ,n¬ëд;hœvGý=@1Ñjãx qšïæt.(¯á&Xü6±=bPëBߨÿ®­>Âu£ö1®ôw éQFFµ¼å”´r¼‡ëÙ3ÈIë@“_Iˆ³'¶—ÌCsUh¿È\×ò¶žÝv­o ·iöë~Î}įsâ" -Ú3Ü×–ºý=€9t¦ Éêù““:טgi :•ûV…žºÊ7•Á†ç@ë€2hÏ­=ô÷€¼âWoŸ÷Ú&u®4>@g~¹úD ­ --@ËkÆ*“çÂnF`µi¯m¬ëˆ×6jóÚý=@e>žÂZ£çŸ¿Nê\PAóÛuå$†óµ.ô]€Z¼¶°Éó:Ï>£GžÈŽÚÇžÈ -ú;P &€|\Ù¦> µIë@øuç̙ܵ.tõD¶wâ»ÇVÕû²O<¶y  6Ú }h£þп‘â„ù×Xæt®4ϯDLègmUèQ ç±æƒ5ñÁFíc>˜ ¿¯ÀÛšµÍa~¡cRç:ðJ<ÅCûÌ-KëBß ¯í–ëe¡£ÛÚTË8¦‚ƒn™c´ùÈ2Ǩ}l™CÐßf6¨Ø–7ͧyoNç2ÀŒšßú±^ŸÉ{ëBß ÌØnšùƺ0ߨ}Œùý€ÁÕS:øÀˆ¤RˆË>À&u®0~ 7¡ê3Ò¯ = °m`±°Ù÷ K+Ö›~c´iâYj};jϯ=‹ô÷€ç`¶M”0Û¤Îu€çÙEˆ)Ÿ¹åh]èxý;…}Dú5!CÇlC݇˜mÐ>Èl£þÀo)çÅæŸLê\`‰÷Ok>”ñD€­ -ØñP¡­o5y.0j{. èïÁŠÏ÷ñù|‘œùÈ%ž½×?©sX¥rNñ!®çÁj]èaXýú!B·-h´õyˆÐŒQ{&Dh!† ¿È\€ ÆgÔñàgܤÎuIìãpÔt& W…Nòäg mEl´ç9 6ßoÔžòÞª÷'èïÎñi¤ÙÀí›ÝœÎu@çÙ@tsæV£u¡ïÝW„ à[nÐ>¸Qp—ØLykf6\˜Ô¹à¿«f(žÊr«BË~O–rwÕëÓÂ…±î#á¨},\ô÷díÊ™•³á¤Îe–5¿…æt8sCѺÐY€}M¸0Öx$\µ… ‚þ¬²ÉÊ„ÈÇÜòÉôDTN>Ý…Õ¤Îu`Eü&XˆáÌmDëB ¬Hà£í¡,¹¹üe!J¾í›ùßÎll.ûhíó€¡=µg(™l(hI’eØœÝ G“:פç©æžÊt«B?~»Í§õ®ÞúNÖ¡íácG6ÚÇ6 ú{@âO‰døœÑÙ·²&u®¤Äo‘ñÙîgiUè! -œç;îZä_8)l⨚Î-m;ò>Ö¨}ì},A€™=O|–ñìëIË°©k"èáÌýG›RËôªõ°û¶À°„5ó®ýôñ-–Ì9òäJP?öèJ*`»?>Î-F´Êïéó".âÊþ´ä -¶§Yž`«Ø)žêûb6§è2†Âã¾à™Î9ò—PŠkùŒo†Ì÷ÿõÃÿþáw 2„[³?á'q;áÿ>:÷vädòXHéNH%Sz â‘å~9†e?>òü¼ÞI,yÊI¹øAy&?òø®®p×­o4ûBhÛæÿåÊó¦|gD‡âŠgÞN柚á2'>Ÿžà”=šÛ”2/_Ð,Ÿoû¥æâ!Òü‘ÞBQº£¿xäÿ¥tÔÏ7>§‹ï‘Ç×ýëKºˆyrÛŠßnüõ-'ˆá^ÙI1 ¿êZouÛÈ^ÁzQåmNòJ?~͵pò˯Ö/wÃú~QãSÑùå†åŠ±{¯XB3˜QÌwOå!æ$“¢ÇU¨¦ø°Ø a+Ðõy[Mñ©˜#ô&ô£PwÕ51Ÿž»äµõ×\ûÂjzßt¦ëþ%¯]wî _o¨–¶Ò7QÊßMZŠZZá"£Î¨\¥~ÉY;ÄÕ~D¯7Aø^Šæ¨ ¦w½Æ_Ÿ¸ Ó$Wóå%Ÿ z½‰ù¥ùÚS±æJM5/-ßLiP§T^•6zH#’±#š–_×âÀ=°ôµNnì |½ñ«!ƒøíæ£Û7®Ð~^/¤®Ü*Nºá¥IcZš -Cs‘ñÙýüï*|•2ó”_Œ«âïo´«V Í*^7R/•uœÔ¼Ü R”cÅlß^&i!?“nÍ][l%횬“Ǽ˜Ç$UÉc-6uU̶´OåÊ^¨#P]û -å·•¸ÙéëPõºVLw_ËmâP¹ó­ÇÚã†?¦Rí­åv˜_„¯ebs[1C?-•41ŸT¸À§VÖ ›a …&n4úVÄËh­ìÄOË—_Û “fÜ×Û_oeš³ä-ûå ¬÷–ÌË¿ç÷øùÛͽüaùÅ}·º²…­Üº÷3I2MŠùË’‹°›Nh]GûMlRËë4ÔyÜÚŽ¯]S×Ø—œ¦2¨©©½Ð·6ìĶB×êZÆï_7­X™¯í›6aß/i¹9^ÙåϘ,Vᢿ\¦–FmJ~éŠÄgmS:RíqÑŠË®ÂÜ,[™É€ì+©MÏö»¾+ˆëê.:7‚/©¥¹6ÍöE/²r]¸ÀÔ_ôó6ƒVV^Ù#ýîáõÕýþ09,#€¥•Ú«ô­—Öci%> -endobj -311 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-199) ->> -endobj -312 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-200) ->> -endobj -313 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-201) ->> -endobj -314 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-202) ->> -endobj -315 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-203) ->> -endobj -316 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-204) ->> -endobj -317 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-205) ->> -endobj -318 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-206) ->> -endobj -319 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-207) ->> -endobj -320 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-208) ->> -endobj -321 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-209) ->> -endobj -322 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-210) ->> -endobj -323 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-211) ->> -endobj -324 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-212) ->> -endobj -325 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-213) ->> -endobj -326 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-214) ->> -endobj -327 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-215) ->> -endobj -328 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-216) ->> -endobj -329 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-217) ->> -endobj -330 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-218) ->> -endobj -331 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-219) ->> -endobj -332 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-220) ->> -endobj -333 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-221) ->> -endobj -334 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-222) ->> -endobj -335 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-223) ->> -endobj -336 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-224) ->> -endobj -337 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-225) ->> -endobj -338 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-226) ->> -endobj -339 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-227) ->> -endobj -340 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-228) ->> -endobj -341 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-229) ->> -endobj -342 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-230) ->> -endobj -343 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-231) ->> -endobj -344 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-232) ->> -endobj -345 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-233) ->> -endobj -346 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-234) ->> -endobj -347 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-235) ->> -endobj -348 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-236) ->> -endobj -349 0 obj -<< -/Filter /FlateDecode -/Length 4965 ->> -stream -xœÍ]Ë®ä¶Ý÷WôÚ€i’Å'`xa$‹p#É ‹ñõŒà:€E~?§Ô"E‰Õjêº ÈÏ´HY"OU‘”d®>7ø+9£RÊ1¸ëË/—_/ZE¯ñßTbu1e‡ÿ£¿í¢ò&‡ä¯¿pE³·1\_/øksÙ½üý³ËÖ2æjUÎÁýí#g9R1Ũ½•‘¥ YÞ«­BV0 -·˜³”5|tÖõYj¸œ¼”5œµIÈJPƒ\4Ôge¨a£qRÔ0ÙG!Ëhèa¬ÖbÑΑòŒW¤'çļŒ C¯JyÁªA×b^Bž‰ÄèL±‘¸±”J˜ô€à»K@ ä'qAÌÐSEâ˜"•,†%DÓl}:ávàS8‰@™ Î+X³ò~·F&Tvpc^RÕˆþ¼~æ1‘7ùÀ0"s8«pêD @U`ý0ƒäãfð_ÖÀãv*`~x0¶–xq­Í0-LK !ôÂÀ@ç"ÂA÷Kâ;ÀŸÂ»Ô9chüc ~Dsƒ2§B-D›˜èæ6–,¼ ˯UÈ2½ôBxoÃz¼$¿8¾lt9˜…ÿÏû.>‚ȘÌy ÃeY}>؆|h ²µ1sŸ6é¡Ò öJµ1¨4CÝI@!QLnÛ1ï[;Ýû§M¿¤Rw¢‡.ÊzÚ(Œ:j2@4&sEÞaòY[D2ÅÏÉ^yÙ81ã,³R¨…N7<‰c¦ ‡õŠ_™h -W÷GgHä4ƒcy™ÍæøD'w]éÄZðVÓã:ÙÛ]  íë|8ß ;•™×§é‘ùÎ^eÖ9dI~Ç4ð~r2ìl`dIiÐ~´p3(sàX˜22|®á‰ÀYUú¾ Shgæª}0œèa—@ÚYhç±ñ°Z^O‚ô˜ì´n ÔKwùQë ï€‰76¢s,ï§ÃC0ÉœL¼AÊ; -I6/7¢vkRãSlâæêªnÃ?»ªyãƆ7™2^áƃ3› }áu"¢DIAzX„n4t¡ù·y%6LŽŽ‰à¢Œ*=|ðrc2ç\„ö!$’!7C¦x&Î QÏv‰$n®Ûåšç@OÏDë| Í ŒqXÆp˜˜­$=5Í{ÏÆ&+ÉïÄy}Ú Ô˜ÌiÅ”’ɈäX‰W6o\Ù†ÃVœöa.Ö+õ¬€6 œµ¢GÓKN/O0~îŽñƒ•T)XÞ5 ™·/ï9ìâfLä<°±0ÕÉÂZß5}•g>¬á± -¿‡]áuƒ{ÑQRÆÞŽ#Œ§ƒ2çé~Ï»?p”òÝØu¥íì¯TŸç¼5VÙ7´oeÀ‰­Nh/=âÄ.K¹‚üŽ @ŠoðGDƒ2çRäSË~à3¡´ªô‹ˆúvŽDD½ô±ˆH¸ËˆˆRRÁëAçèxç'=Ó˜ÌiÀÄ;Z)j>:ÿ<0­+}ß‚EZ›)ÉfÁ¿ÄõzëØQ.[Öc{ÍÃoYí¥Gà—Ì…kI~~d„x‹ø´Èè®á Ìyà‡¹iÈ&ïŸ ¿U¥ï[³÷ÃƹuÍ&Ÿ=Ÿ>[ÌSßÖ‘­Æ^úØ^c/¿ (â#ËAóѲa@ÉœPÆ<{ æ3µªô}Ã_+#Xx¬=1ñi1œ•¿^¬kûÀ:éƒëåw¼/ØUÅvÑsƒ2çŸáó !î3¶ªtˆ±¶[žù0sumV'}Xü^xîxUArØ”9 °¼†é¶ø%“x°Ö•ŽkÒ†½ê¶©>¶´ø]½Gü®^ú˜ßÕËïÍ£Ž?!ÀeÑtèlƒ2ç?½”ùÊ.Þ¶U¥ÏÛTV7.ÿ l6Qz}l¢ôÒÇ6Qzù]P:À†dõôôqvцG+Ûƒ2ç¥çÔ@ôT\Uú¾5qè½UriE\*ûa̯k8©Óë1#.kë½ô#.«ëB¯˜õÔªžë§{»wüd!?{müø²í Ìy ɯ…ЙŸ®"$W•N<Ùòcl๠'êÈ᯾­Ç0[|Ä^zf‹—(´Nwá”ùí$Ö­?×øNc2§SÐüpåô"ŽçÁi]éœîE©£°Z€Ñ·y$xè¥BëwaLPˆ¦=]@>°ý$“9¬ˆŸ|qàögÂjUik"¶çêc#íq™_‰žÛCyí€wº9„ÛK;„+Èï,‚À³T0×å ÊœpžŸN‰ÐUÞLkÂo7’»ÝÈ(`®qÿÇ9i¥ÔÞ†s|vV[tÇ°©”9Ï%6¾sÔm;D›Èª}Ôëy›ÊÍŒ^©·;XÙ `³üÐúèÙè1‘Ó UÔü´†õZžMo|pU銾ÇgWs.º¯ïȹè^úعhA~‡‚á(|àQ߃2ç ?¢›Ð´¼q[•ù\h¾ÜVcÞöÕW¬›?WôÒÇâ -A~¦Ü&Jv|õxPæ©ùmˆ÷ž‘ªV}{Ì¢û±ñè}§öœn<ƒíºí@°Ù ãJé£Ke½ô±¥2¡õ»ÁæôO‘_â1êûÊœBizñ›REùÆËØúûÂñ± Nícxa"‹AY+µçN&ô¬1Á…4>Dƒ2ç"b -eïú‰åºÒ»lñ¶sÃÍ Á¾#'{éc'ù½÷:8~¹e6 ™ÑØdLä›ò{ ü>*閭ÌÈy “øÁCñ©ì³ªtµ×½'âÑÐ7Àéj=œNú pzù=à`l3Ì+¿lØpÉœ:YóSN‡gKXWú~k˜æý’»¯£6\};G W/}Ìp ò;`Ê&+,ð`ŽîŠ ÊœLÄÏuLÕg‚iUéa0}ùÖºy¿P§Á‘½^úØ^ˆ ¿3‡:lÎîÀA™óÀ̳u†ºOå¬U¥»0»÷š¤qæéÚ:ò|W/}ìù.A~Püˆl¿"rt)oPæ<€Jü ¿–û™€ZUú&@½™»öé´8²ØK[ä÷ –™r=¼m`T²§GH9 Ц·žš÷E>7°:qY–k^´ðÖfaùø۳Ña³róÝüÖU„VüîF>È(¨}×k†‡Etû¤£øNÒ«Ù¼”»7Ì÷ÄöXñç–(]§³¡‘ßP̽ùõ_¿ÿÇ·Zz“ÐKö#þOŒ[üûCóÜQäùÆ=Xzòv¤p*ÇG SÙq½•ãt_Ú·¹;• øiæ2ü‚f´aPŸÉsß´æÅ7Ýèò±ëþ_/azC¶™¾’ Ãäif>YáqòWùíÚäLén3u¹Þ‘œ>>õk-Å#Ió'*xÛvŽöb.ÿë4P¿\8 -ã{d@ß¾£§d~ÙÝ6ùõÂßrB2< ;˜Ì‰óO]ëâS›´ðATy[’¼ÒóÏ\+'_~Z_î†å}ãw:óÞrÅؽ5,$B2˜>™ï³mNæ7Ò%EóU¨ªÀŸÏ5q©Ðµe— -Á‡É¼—º$ú>Q7Í-Éü†ÄRÖÖŸ¹Ž9«êm×™føKY›áÜK|¹ YÚ¦¾Š©üÕ—RUé]°ZI£F©\S})YÄÕq’^.BâçºTÍrÍoFßKLH®–Ë¥œôrË͵ù:R±–J‹h.=¿¨²@Ríx!mê£95òåš9¢©ü\'2Ö}ÜØ.ñ姻ä׋Bi¿p…özm"5õÖ䤼,©1•®ÂÔ,iüæqþûF¾¦2óL?Œ«â¯,WK#<7kòº“ÚôPYÇI=À¯5]*ꎱb¶í/“´PžI·–®=¶Jmº¬I¹¨Ç$UÉcl*êj2SXéŸÊ•m¢.J ¹%ÙW(¿®’=}ª^×Æ‚iàîk½Kr¨ÜùÚ&Ç:â†?Qõ­õ6˜/‰/“asÛd†~*,ɼ”XàSkÅ@ -KòB£¯Sr™­•øa‘òs™d’Å}¹ü|™Ìœ%oÙe døËF­·d®ÿC™oðÿ¿/îúmy•þÍŽ[]ÙÂVn]%·–Œ¿ƒ—“ù»x%±N›&Ѻ†ö—d“–²¾ ¡Úqk¾v‹¸^ˆ½”4•AMÍmýÒ‡M²­Ðµ†šžñ;‰/›^¬ŽÌï›Å`ß.©Üðü†¢.ÚÛÁeZòh1ÉצJ\qÑŤ#×ÙFm¥¸lÌ‹f+5m#µëY×q[ÍEãFð%-yn1³mÕ%mº‹¦\`j/ÚŠiÍ+}¥ß<½~÷¸Ï*‡28µR{M}mSëqj%<9•¦Ë°¾¬6‡¦õ¡2y—ªuQ3÷¨:9ËÔX\ ·ºh1Ú¤¦El.àÛF]I[5Z E¥ ¤é’VkŒE§0‹•n™î8‰½«O±¤þÜö´oG¥v¢oG%te¹Þbñª·DÕò-œãrwz)Y}É$¤…Vº¦º¥$•´¾m·n»XÛksnQ$,ž©^Ä|WòMü÷–¹s“Ù¬H¥ÊŸiEäåã30ÔÎD²4}OX§À!Ž? ]j–8Ëì¯yô+eÍ£m á‘Ž†¿ -«ðŠŒü8£©ª|5ZãǺ„[:ÖÂ’ÎBsº[Ñ™> Šÿæ…z9Tä/;½‰?`µ—x¦¥–´Z>™þü©(Þ5 -endstream -endobj -350 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 349 0 R -/Resources 4 0 R -/Annots [ 351 0 R 352 0 R 353 0 R 354 0 R 355 0 R 356 0 R 357 0 R 358 0 R 359 0 R 360 0 R 361 0 R 362 0 R 363 0 R 364 0 R 365 0 R 366 0 R 367 0 R 368 0 R 369 0 R 370 0 R 371 0 R 372 0 R 373 0 R 374 0 R 375 0 R 376 0 R 377 0 R 378 0 R 379 0 R 380 0 R 381 0 R 382 0 R 383 0 R 384 0 R 385 0 R 386 0 R 387 0 R 388 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -351 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-237) ->> -endobj -352 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-238) ->> -endobj -353 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-239) ->> -endobj -354 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-240) ->> -endobj -355 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-241) ->> -endobj -356 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-242) ->> -endobj -357 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-243) ->> -endobj -358 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-244) ->> -endobj -359 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-245) ->> -endobj -360 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-246) ->> -endobj -361 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-247) ->> -endobj -362 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-248) ->> -endobj -363 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-249) ->> -endobj -364 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-250) ->> -endobj -365 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-251) ->> -endobj -366 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-252) ->> -endobj -367 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-253) ->> -endobj -368 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-254) ->> -endobj -369 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-255) ->> -endobj -370 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-256) ->> -endobj -371 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-257) ->> -endobj -372 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-258) ->> -endobj -373 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-259) ->> -endobj -374 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-260) ->> -endobj -375 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-261) ->> -endobj -376 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-262) ->> -endobj -377 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-263) ->> -endobj -378 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-264) ->> -endobj -379 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-265) ->> -endobj -380 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-266) ->> -endobj -381 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-267) ->> -endobj -382 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-268) ->> -endobj -383 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-269) ->> -endobj -384 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-270) ->> -endobj -385 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-271) ->> -endobj -386 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-272) ->> -endobj -387 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-273) ->> -endobj -388 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-274) ->> -endobj -389 0 obj -<< -/Filter /FlateDecode -/Length 5119 ->> -stream -xœÍ]]¯ä¶‘}ï_ÑÏ,ó£HE€» d3ðb×ÈÃLg&Ap'€³ùû{J-~Hªn±¯ÛpûúÞiYd‰<<¬")µ=ü|cñ‡ÉNÌ9E:_¾ž~<™)ƒÿæ«‹9ùøMál ¥)Ø9œ¿ŠÀ§)åàR<¿ðgsÙg;ý÷¯NÿXëس›rŽÎ†ó??Kù)qJ&(IIÆz%)„)%¢’í„[ÌYK‚!‘£}R‚”9hI0ƒœc%‰a†§dý>)à —,iI0Ãæ”$k`‡uƨi0Äy§¤Ù0E4zÔÒœ"g—Õ´„4›œ–æýÙ`µ´<Åki[B6VK °%X"£¥Áòœ´´[|ôFMƒ-Ž“×Òlqh¬¤1lºXMƒ-&«¥e?…ÌDjZFšáÄû4gÂØ{£¥Y;`Ó«iQ褘”4[¢ YMƒ-Á§¥yØBøÑÒÀÁ'fe´:ÝÞx«¥Øâ\"5 ¶ ó5p¶˜5 -Àš„—4p)M”i$àØOÄÄ 8Î¥ä5p9LsÒxÀ‹4g5"ð¶Šx [#WMƒ->“ÆÞÁdиÀ{ØâÈk\àÁ¶´h\àA·d²Õ¸ÀƒoÉبqáÂL£qãzŽ¤qåJYãÎõàN D§qízâ¨qïz²FãïzM5 ¶À€'oÙk\@à]oMÒ¸€À»FNãôêF €•É1 ¼ëØÆÞu S€š–&ŒA¯qw]À-h37x×ã4. ð®#5.P&ðQ½ð.($èîlÁj\€ùiÎk\€Â&·Dã‚ÞµÓŸ2¦ÁV“å5.à]Ëè>5-O6¡Å•q‹VžlŒYã‚ÞÅDå5.à]Ð'k\ŽŸ0£: x×zÌÔZx׺l4.à]ø AãÀa²p4.Àü6Yèªiy2™Yã‚Þ-9 0 O†Á$jZš$«qÈx2ðC5.ˆà]LïYã‚Þ5Á{ "x×Ü5 ¶xvÀ¯A†­2¦á¬NÆ)´4ð®±èa- ¼kLÊDð®L›Ä$ÇÀÀ ^°Ó¸ 2M9¥¨qAÌipùÔ´8á:h\ ”¥¦ÁJ¤që)ûÌ$[¼s$[à2h\`ÊÞ·ÀÀE¶ÀSÔ¸L6q&Ò¸ …ˆx ­ª¥E7q]«iŒ4›4.H‰&LÅVãÄRcÐk\‚ŸÄ]Ò¸f Í’Æ)à  0Mìà™ki¶ÀÉÖ¸ž.Ò¬Õ¸€l7¨q;F”Ƙ¡K¢©µ42g²ÆLqWQã Dš`J5 ¶€&4.@ÿH°4.@ðƒ`aŽš[| ˜„›¬q3lqÆi\lJ¼™4.ȶÀãÓ¸ ›8‰›¬qhiÆh\-#Þt¤qƥě¬qAöñ&B^ô†1VeJ1>Løu×âæEºà7]¯‰yøõýJˆÕŠóÍûJì_•gµí•nÐyY€ÄERÔeun1þ?>¬‘Áa"¸èè+¡âÐäÀ4¦ò.0ýʘl‚ÃúD ­Êœ1T°#X*¸¹tr»ü{ÒZYõ.ï«@Q…àNyE „û[ülë(ùÁÍ6¬geÅ¡˜ã ê¼ Žs"ìFOÚ'i]è=RälYˆÜ‚¢^öes½¤$V2˜láŲcðÁyö>1mk}Œñ .Mÿ.ü]È웯›;™’‹ñ~c:¯?€ÆÌ3yl]èÜÌ܈¬B:RS`XanäýÙå¡îßÙu >žc€µ»_F×Û”Ú*ePnÜ—Rî’ðGŸÍ¼k2â-sÉ1×d’MA¸Kî™\úCUËÒø—Æt3ìæÖ6=îëyÄEÛk?æ£)wù8ÄXBõlX–o%hg‰´ï#lHåeædu Õæ¤lŒ›Sޒ瀙0Þd“bSiŒmˆ§ÆVœ¤ñ1>¨ó:]àâLÊù™c|]è“»²óŸ÷õ<â@ïµó ÷ú”[•ñ½òåt˜ÉžU"’5×QçzPçu`&{ß²YÄÏ„ÙªÐzhÙnÚð]'|êàV¼·ÉoÆ`Ø\齸Ò{íÇ\iEÿÎz€‹sÅ6Ùsbá"ÓÖ†T^j s`ˆ‘½ -µâô>±Û ‡OHÿ×…uµ<øÞ)Ú#€—-'k8[Eÿq¯Âq’Œy—Š’œû´²lxc:/9%ÆS¯Î6«èc‹kî…âÏŠÚ¸_yÏÁ8Ÿ8:Ù4vrvÌ’‰G‹xƒ:¯Óe;ÌiOœ Ö…Î]o–îû´t/+F^àpéÆ“\‡«1RÎGÅ—ùüàú²Ÿ‡“lfï-?äû‰MÒ´Ú̓åæÃÒ¬éß™R¼çɺù¼ŠÝ”ÃÝöˆQu^žAv±ànågÂsUèÏ Ïº`¸]®îX­.HÇկ˯6ûF^¢-šOìïñÈp0&ñg}P´Z˜$@ð6ìºíÈpé<ËþóøfÉ Îë9É¡úDñ©<»*tåe…Ö@Ãaú¾Ìc§Z€“MFoïµGœêYîõ÷´<óüì,ÚJfࣜA—“ŒUNFžíxœÖ…ŽnšÜ„\Iï®ç˜;^{ÞÛv ˶ò¼×™˜œõµŽÝ½–¹Ëk‘è^éUÌò¦«¦^v5@Ý‚Ž×ê»ÿüþþðï­Wý¼P#Ë×Ãr][;^¯ ZÙ}ZfÉ“6ùø*“|R¦ûŒ_.ˆÁÿ%¿)³Qñ½»òã~V£²Áÿ±ëåÐén7n¯í—‰«¦à8ÁŽD¡´ølÕ§fA¸Ùë–hw¸è Ü];º`ºc ‹^ìÊé졸ä·Û–ÐZášo.Ç.÷Á­ŽÒ;’-v§ÀY›všÇ`\l»,6çÎNZþU²®ÿ:v¯÷<ÛNK{}Yìs×{ñùz¯Tú!´òB×vÖ¾¯_J»W›‹Þ’ö`»¹m»i}>—»é¯Ñû¬¶Ú®=KÿæÖ¶=NkþÒKùó=ÅVGÅ‘]0Öµ+äÚ/K?ͶÔ1{¿Î»íG;Ü¥¥Þ¸ô‰i˜*ØYã¿ó¯¬ø_¿=}û›e"\Օ䀽ÿzr>"ÀŽ9†^þ&OôfÙÙ>ÿëô'™n õ®fÖv+«bú[Ù±îÜoK›–¾¸®Z,¿—ý¸.ãäh Ýâ„-nÞÃauL—:/ f¬èžxe\ËQ\¸ÙûpQŽ©üÒŽS¨æÊNw‚î8Å ““6rþ|tKjPç—nšv˜n8*`üFÓ4¿Rfÿ¯ÞØ^¢Îèõö^C·íüQoþ(> Üótæ1Xi¼NÓ˶R`G¡áÊ$gÒ8C“³4Ÿºß0c:¯Ó4² âðI?¶‹Hº­À­o¾ÚyòfÕhGÀâå¡0þr}¹Q¦r˜b•w;J*“wökÑV9@{°VÉÁ'Ø4àØJK¿2! ßk—±|wk9lî|{'˵5›u±n·~î÷ õa SG’‡×\ºY^çq4 Æt^gÈVûèõcj´ÙÓ¯çwvká ün5ð&¨Ÿ¡þÀéƵ™nÉîµ Øl†| ¬6ÔÂ:ÿx»Ü`ØŒÙÃEvr—fbÁáCc:/­$[V½¿Á°ÊZøÈš÷#þÁ¬Gûòf˜öP,6|<Ú”ÁíÙl}4›ÛÛÒiB{í¡-º%´×ßa•ä¼¼À(Ø0Îqƒ:¯DÙœ2Y^ÚòKqžŠ·Nj?å÷{²ƒÅêV÷ë>á^{”Ñr¤”5ýw‚’ò$ïIÓ}ò±3óƒÙ÷A9¦ó: Œó›Zä-QÏåj7¯÷˜îmdú–°“ÁéÙÜÔüÉãIÈ©hÀ¯cäþj?”a'ObYÊ&ŽŸËÔyØɆ”'²7¸ðË.ýyí -Ÿ8‡ DiÙßA¾t^Ñ~Óç8\é¹iuã÷Žç¤œåqÛhäȯ™H^ùqä› ê¼ Øb -ÏÉZýPÝûöùÖ…>ÌbûvmCw_×#G…÷ÚÞë?Æ4,K‘qèÎÑGÆT^`^¦chß8²÷øÝîÜåºð±S›íÔå^{æò-h*¨ú² rc‰‘åá$t[v£KŒC¯ÓÑR点Á$7–Þ‰bˆbûèÂ÷ Îë4 cR`T}ã0E·ðÝ6Ì6óî§ç.xÏïŸqìyC/ÓäÙ -K&Î7ŽCÝ@cvaÊYÞ+xvòÖ¦åÍE÷fLçuš†Pž5òjÖ›g pöÔ>¯½¾:¥÷Ë+7’íÒ®êù}ÜDÀG5ô)\7Ë=¿0˃Á'ygÐèS˃:¯’Š•ws=ó8áºÐz–¤b»üÑŸñúKçú‡&_­Ý™@n Ì‘³`í4×ÞÎãèµåÚkD¯-vVj/»|òøˆpÇýÔy@f#/hƒúS_ÒZ©Æ¥fê·z–ùÖ;D†÷(ö6Ýâ!‹¤)¥lçGç#""#;jO*½LÇ[ce~å˜nì]÷=ÿ Z$±©eõÌá“žê"ßWv´Rœ_,³Æ^½î™~–ÀoÙÖªv<\^¦59Ê/³<•þn0‡Yà±;U_‡u¶›×O­Må9ž–×3ÏËI^‹pùhjYdëæ"»€¹Ò”·9}˜Ìò±r–#X×ðÇcÓEMF¹øxåJ°{­XB3Ú½XîÞLvË×*ðä—«XM­æ*lRŸ·ØŲ¤Ø„a/4]uM,+F%¯«sís«é}ÓÙ®ûK^—ºî¼'¼œP­ßJßT©¼Ì½UZ×™*óQ¹JCÉY;„j?)¢ËI~cJфάé]¯É+q¯B.@¢š/—|ŠèrRó-¥…ÚS©æ⦚KË7SÔ=׆Wd—îI€$ßÓãËǵ8J”¾6ÜáÆí„—S :›øí’’;4®0¡@¯ú®Ü*fÓðÒ¤²Å|ý„¡YdòBQù{¥ŠP¥Â<óKRòRàvÕ*‘±YÅëFêå±²i- n´ÝIQvJ³}{ažVò éÖܵÅVÒ®É:yÊÅ•+{¡)F º&Êo+q³3Ô¡L­,Úî¡–Ûırç[/NµÇ­¼á¹Ú[Ëí0_„—yb£­X Ï¥’&–‡Ì -|je°RhâF£o³¸ŒÖÊNî—mi3îåô·Ó<Í9œ¸ÖÑ[ù‚Þ[²ç!Ïïñû÷ÿPÞ{Ç©lá*·®ÄýL&_oÃbùº›"¬Ã¦:êh¿‰-·¼¡ ¡ÎãÎu|MMÝ4b/9meP[S{ahm؉]…®³¾k™pGxÙ´bud~jß´ ûzéËÍIh ïV.Vᢿ\rKómJ>wEâJ²¶)©ä:E\ô…ⲫ07ËVf - ûJjÓ‹ýÔw…ìÅô!—¾¥Q›fû¢‹l¾‹. \`ß_ô“í+Í+{£ß=¼~r¿/&Ç2DZ©½Jßzi½1‘VÂÓ¥~¾ŒëË:çø9ò,ƒ·µP-ËwcÏW'§ æZÐê¢Çh'妶d}¥Td«JËDQé2SdµÄTlŠ‹Zi–ùŽYm¥T}Š&ý[ßÒ¡ï•Úˆ¡ï•¸Ë+å–¯zK¾:@¡‡sjwgZÎêK²"‹½v•RËé‹l_7­ë.³í¹» j†Ä晚¦v9ßÅï;ê£UZ®ò3¯hXnï”ÇD-¯ô‘u\qw8FpÉ7½$ân‰ãaûkû•²æÑׄî˜È{“¬|á£nÂòqBWTù2H]M•[z¬”%FsûáæoüÂËÂN½L*ÊüÉÓ\Ìò½DÉXu‰g^jÉ«å“ùçÿ—•Bû -endstream -endobj -390 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 389 0 R -/Resources 4 0 R -/Annots [ 391 0 R 392 0 R 393 0 R 394 0 R 395 0 R 396 0 R 397 0 R 398 0 R 399 0 R 400 0 R 401 0 R 402 0 R 403 0 R 404 0 R 405 0 R 406 0 R 407 0 R 408 0 R 409 0 R 410 0 R 411 0 R 412 0 R 413 0 R 414 0 R 415 0 R 416 0 R 417 0 R 418 0 R 419 0 R 420 0 R 421 0 R 422 0 R 423 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -391 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-275) ->> -endobj -392 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-276) ->> -endobj -393 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-277) ->> -endobj -394 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-278) ->> -endobj -395 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-279) ->> -endobj -396 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-280) ->> -endobj -397 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-281) ->> -endobj -398 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-282) ->> -endobj -399 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-283) ->> -endobj -400 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-284) ->> -endobj -401 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-285) ->> -endobj -402 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-286) ->> -endobj -403 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-287) ->> -endobj -404 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-288) ->> -endobj -405 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 443.811778 177.650223 428.811778 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/installation) ->> ->> -endobj -406 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 415.114756 101.433071 396.364756 ] -/BS << -/W 0 ->> -/Dest (cb10-1) ->> -endobj -407 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 396.167735 99.933071 377.417735 ] -/BS << -/W 0 ->> -/Dest (cb10-2) ->> -endobj -408 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 377.220714 99.933071 358.470714 ] -/BS << -/W 0 ->> -/Dest (cb10-3) ->> -endobj -409 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 358.273692 99.933071 339.523692 ] -/BS << -/W 0 ->> -/Dest (cb10-4) ->> -endobj -410 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 339.326671 99.933071 320.576671 ] -/BS << -/W 0 ->> -/Dest (cb10-5) ->> -endobj -411 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 320.379649 99.933071 301.629649 ] -/BS << -/W 0 ->> -/Dest (cb10-6) ->> -endobj -412 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 301.432628 99.933071 282.682628 ] -/BS << -/W 0 ->> -/Dest (cb10-7) ->> -endobj -413 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 282.485606 99.933071 263.735606 ] -/BS << -/W 0 ->> -/Dest (cb10-8) ->> -endobj -414 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 263.538585 99.933071 244.788585 ] -/BS << -/W 0 ->> -/Dest (cb10-9) ->> -endobj -415 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 244.591563 99.933071 225.841563 ] -/BS << -/W 0 ->> -/Dest (cb10-10) ->> -endobj -416 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 225.644542 99.933071 206.894542 ] -/BS << -/W 0 ->> -/Dest (cb10-11) ->> -endobj -417 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 206.697520 99.933071 187.947520 ] -/BS << -/W 0 ->> -/Dest (cb10-12) ->> -endobj -418 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 187.750499 99.933071 169.000499 ] -/BS << -/W 0 ->> -/Dest (cb10-13) ->> -endobj -419 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 168.803477 99.933071 150.053477 ] -/BS << -/W 0 ->> -/Dest (cb10-14) ->> -endobj -420 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 149.856456 99.933071 131.106456 ] -/BS << -/W 0 ->> -/Dest (cb10-15) ->> -endobj -421 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 130.909434 99.933071 112.159434 ] -/BS << -/W 0 ->> -/Dest (cb10-16) ->> -endobj -422 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 111.962413 99.933071 93.212413 ] -/BS << -/W 0 ->> -/Dest (cb10-17) ->> -endobj -423 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 93.015391 99.933071 74.265391 ] -/BS << -/W 0 ->> -/Dest (cb10-18) ->> -endobj -424 0 obj -<< -/Filter /FlateDecode -/Length 4795 ->> -stream -xœÍ]ßä8n~¯¿¢žX$R¿€Ã>,’‡pl4ò0Û7s‡ 7À^òï磫$Ë6ÝVÕô<ž.“¢D‹Ô'Š–UîjñóƒÃ™É¹¤È××_/¿]¬IÁâßTbq1±ÿŒß®Îr2Á•˜ÃõW!P2©Ÿâõí‚ÿV—}ñ·Ë¿ÿîò?Kwõ¦”è]¸þý‹°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑÜb) j„Äž·¬5¸ä ± {ŸV†ÄÉÑ–U †OŽ5Ôp%$…å,ôpÞZ•E,3y…炉èô¨ñ¼31_T^Ï%¯ñˆLLì‚ÓxÅÄCÖx ]B±Nãè³ÕxÐ…)'¡ E²*ºøœHã%èâÑ;EáeèïÊ*ºØhÆ+dBÉÌ*¯€gsÊ[ž·Á„Ld5žs&À7IåaDÁH1)<]¢ EåA—@Ök<‚.Œœ”rVF«—ÑM–œÆ ÐÅûÄ*ºÀøø]lŠ`Á% |J†‹g |&Ù³†>Ã)‘¾ñ$ È:ð¼Ó€€,t 5$ ]#WåA*¬ayè‚AϤamÞ¢anÙ§aoÙº¨ap¡¦Õ°€€¸”#kX@€\¸RÑ°€€¹ìÔ°€ºîIÃêRˆ^Ãìç¨aw‰Õ°€»DTyЬaÐÑˤawÉÙ¤aw1Œ¼†°ªF à+Æçl5,`à®Ï–5,`à®O˜T^2ƒ¤aw}À-h37p×ë5,`à®g5,€£àU£à. $èátÀjX€ùc•­†˜¡±–DWk<¶²ÎÌdŽFBE 0L°ÒRª<è˜Ð°ö‘Åvа‹,6±ÌQyÐ…kX³—åfÖ° gèâ­×°¾)ëͤaA±ÐŸ†ÅF#a²†€qð¬Õ° ¸Œõ¦g 0.e½™5,(d±ÞÄO·…¢¬7“†…¡K@ˆ¢ò  ‚¦¢Œ[`Ö›@'¡‹/¬aA‰QÖ›YÄXo"¢WSYÖ›IMwC2i<à."Ó¨a.°ÞDd«­í-€7$Äg:3ËŠ3« è ˜[Õ”öb¶JjÎÀ|e¤&  &V–9ªYL«`:«¦ ,ðño0á–}êSTî0gæ$7Y0,Ñ}[sÌÅh§Ø-?ö÷¿^~ÿ“ïõ¯ÿ{ùú»)SVœŸ8ÊÂ:Èñæ\™©:†¡™ˆ,óQºRmÃËoˆ´L#<`d,±§üe§éc¡'Ú‘H ¢/ȉဘ×ò¢ðOŸ.D÷æoùÙvÙ©¹œÑµÞ^?ýzùýÏÿùo?ÿËOhöÓ×ËË$¸ûñWÿþ×õÓß©Q–ĦZÔ)ÕpºUÜýo<®N&"„«Åm«sùqþçOK`ÍÎ×Ð’ŽLÑ‹;°Ú˜ÈSfûÖ -A‚—ck=h=Ä{Ö¥_´ñÒ[ÍßþVkr¾ÿýºâÇû_¾óÞo‘ò´üqÛôž¥‹l)èëDˆF¦ÃSÊœÆÖX#`™„ÞwûAãË(°ìeB]4:ÝWXÁ"û»EÓÒ B˜=!ÄáñpÐ¥Ìëí;/Ñ—HФîXÒ³˜pC$Mþ³üî8â@™ƒÊ-eNDÓ’ì}Ç“9ãÉ=“·O ÌóŽ·htvüìpk(j×Õ!Ó²®Ž kNvX•’¢Ù¡{J¦‹KŸ½"=Ð/äV ˆÖÞí—õ=7¨Ý8mD<•#û€ÓŽÉœÇi“<à Åú¤Ó.}éŒ2!fuXžPsÞÐ#©Vx^rš^Ç. D$„’ŠôˆËÂ!y´Z¯|±6ÚDͲ`+6ËC–$”tzß9ÇdNãœ^²h¶¤Œ²—•¾tSbš®Ã2úÃmšŸh+¤lÁÜ—¥ü$“ŽÝbF­¾ÇN9»ôVzÄ)g§Vz«¬ÂÔ?¬æÕ]å!zv¤^½,ôlšr–ïºë ÌyÜÕÇ)º*X:Jkd9aÛj¢ï#ʇ]Rpµ,iͨ¯]îDÛê~ìºÓF(–ÄíVzÄuÑs!e›4ù~ÆhaÀ¾ËÊ3£Äœâ5›Xˆ$ ý¾ÃŽHœÇ]å¹³<¨Éº»F}âñqZ¸ä®œ²qIJ=è˜1™ótMÂDbÌ´Ó5Ýhî>ûØE2aµÆ«£µF):_—‘Ôb¤½W®ò?ßGh­ÿ—¹LOŸcoÍÙ×KwÝ«åŽ8«è*„U;ëzjý]d7ývhµ‰Øî×®¡ÌÆùr’|»¯ëóÜô£cxfÔA²û5ZÊ£+Ú!‰ó"fK@§<´¢Ìy·…øš£™nò(Í?&ržŽ‘-L!{ÞñЧ2SËJ_öÎý‡HÎÉ "ä‹ÒÆ.ÐÈ£IëØß?¶9´â˜ÌiÌ,€ÑãÓ¶‡÷1,é·¡´y Ë¯Ÿ¦µðÖwa>¿“|¬îrœïž-nu|äÙâVú±g‹J=•²QzIè6Þ ÊœÇeå…‹Œø÷{n½Y6òM.ûÀʳÔB%~éºlõöŽ]:3½Æšé—ŽI|)Mþ±\y`¸ÙÛéõS[({äÊc2çqå ÔHô]ÑwÑÈË=QÐŒQÿÞ7© ÕámcÛ¶Ž7A¤Ø@“Ù@$ï -3sZë»iyJ^uòs’—ÒÝaÎmPæ<î%ï²Û"¯G÷Z4òž{õ(÷Œ›uŽ²ióØÍ:7ÙH¸Ù¼KNi}ßÍŠµà½³ã‹ÕA™Ó¸Y´ò¦ØtªÀ÷s³e#/¥ƒŽ\ƒä{\$¥•cÇšÑo+ýl -*:¬¯$HCLòîÃûÛÞÇdÎã6$ûôñ÷® —•¾t˃q„µ{lÔ¬Ÿ".ëƒö4Ì‹‡­ÎlÇÙJ7§“·²øþìSj «kà´³óL‘fí‰zW¿,ïv’¿Ë8Þq^–·ÏS” lòR®CTx´1gPæ<Îäu…]?Òy•Ng•§‰¯z¾t¹Ðxó6­î%Db”Í~Ö£½œ,gˆ‡†“9a³L¾ÞyTþ1“Ù¢‘ï6™mZyh2ÛH7\±[¤lW+Ú†/ëÇîa¤;.W€ÎOo*èX^r?Jh ÊœÆå’•D¡vKžv¹e#/Õdý&é¯3¾ô/¤Wg§Û¶óˆÓm¥GBóya ´¾3I!L36†Ë>¸’8SÉ››Mï=àÛéÆ2Çg’]Â%9>Ív̘Ìyº&È””ryüQ´ï¶w48|]ÍÅë­'¼k»Û:V‰Âõ.(f}Œ®·Çìm[ñ„¨—öLeS›œ`4<"F$Îcö,›à¬÷Јó0%y4š”9M×äé\’C¿u7À&0ø¬Dª÷%ÙîÛÃè)ç"GÉzøéÄ;NG¯Éœ§ËIòI_}Øþ~ݶ£XOþG€E7ËYnÅEÌ4(s·ÓnÁDò)ÍðÄxj‰ÕáMR;ã/ÊQ)NŽÈ^ç ʜǼYöÏ:J7þvͽz+¿Íô¬ÙVÁEu‰Ý]:=^,ËyVƒSûÄiŒY¬ìO…Çí<Þ™Ú‹+Æaf†j^¶ñg9ù ceÎÓ5$ûDcŠ;Øw=ºïyý^I{RÞ?Ñ® ðõù¯+Ù>Y®Œ…Å6‰Ãí…ÑE¾£ã`Hâ<ÆÂPöÁq G*å¼¥ÑwPæ<]“e¨œqyŠeJ$§˜¬ÔÉs›ƒ.“9M—O‡i¹Ä™wžðü#bÜG±§ZëÏ÷s¾þÙ+•b -#¶éOk&:,r`‘© ÕS°®nuî”ô|¼wy¶9àŸòuz»:É™xÒÑ?ýëÏÿñ§š;šàþ ~³t þþÒe%’t€tFíÄš}@9Ù®”k r+'ôP;<ÝMeã-;Bñ^¦=|Âo¹— ][ñ¾QÀvº|ùqÝý¿]ât&£›ÎåµqšçK–cTÌQòRn–ó ‹‰{w»©ƒÜuGrúºƒßZ)ù$+‡"KÊx2Gq/ÿÛd¨_/£PîQ|ývZ¹È²è\“ß.rZ=+dL+~,ÄûGÛê’Çe+Ú+°"™².IÁØûÇÒ*§P?úPïFäC“Se3U½ß½5¬!Ý–,wo»“åÄÿlè~›*!VAœ+ä¾ì\á,xH–ÜÿL [¢íš›ÉrQ-ëÛÇÒlbS½ï:×™¿–õ©3ç{Ä× š¥5õM¥Ê9㵪ڻÞ6uJ•F µd37;)¤×‹BüÁÖª ¹Æï¬&§µÞˆ¹:·r¥–SH¯µÜ½¶Ð,•Z©<‹–Úó³*³«Sn¯Ð¦>ºS“|…I7F,ÕKr T[cá1ûß_/²%mC~»„¤”3VØP]¯'RWo#g;ûËL•µÉí†f¥ÉY—òÿ *B£ -òL7—’ój竹›¼ì¤žê°Ö²Îm¨¨;¥æ³}¹l•òº­t뱵벎žJUO@ªÇ’ìš×5²@Xퟆ•=ÑV%ÐÜLÍ•ßäYÏІj°­±è:w­Þ™v¾õäÔ,îäðá¦o«·óùJ|&6^“Åõsmd&{Ûf»¹±Ž8+P˜É3Œ¾Mä:Z:ɪÔó ÓfÜ×Ëß.Ó4ç)x‰f#99K¿–ÜõÿPæøýï _ÿTo½ÍãÞ6´ð [ä~&“o^Ƀdù&–Jlæ#zî`&»<— ÕÚ<î}‡×<‹ÛØkI×Ô5nO svdß\×;êz&¼C|]õb d¾Õ6ó„}»¤zsò¬YŽý­Zᢿ\æ™Gó”|íªÄ•§tpÙw‚¸è+Åe×`™5[¨)Ù7Òº^ôçÞ$mu]!—4óxžfûª+mº‹®B`ê/úŠÙõ–…>ŠÒO¯o¶û]åXG€P´7ê[Om7&Ôx:•¦Ë¸¼lsM[êà{¨ÕEÝØ£äÌCc-xqÑûhGͳؽ@èåJ[4Z'Š ÙJk5¦ªS¼‹Õn™î8«½”ZL1SÿÖ÷tè­Ò:1ôV‰›²RoñZ´D- -½;§ùîì\²Å’Y¡Å^ºQy.I•¶m›—m×ÙöÚÝÏŠÄ92µ³XØ”| -ÿž;7™Uö@+U¦Œ†Ëóq瘨Ù%ò4}ƒÍ1CX¾„$qîR˼ŸóØf6jΣo æ0Ld““ï"ÔUx£3ºªê÷êb²÷^¹¥Ç:@IéÌ0g7é˨ðïžØi—I–ŠòÝs$Ao–¯ÌIÖ©)ž)Õòy‘>™~þ0_Î -endstream -endobj -425 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 424 0 R -/Resources 4 0 R -/Annots [ 426 0 R 427 0 R 428 0 R 429 0 R 430 0 R 431 0 R 432 0 R 433 0 R 434 0 R 435 0 R 436 0 R 437 0 R 438 0 R 439 0 R 440 0 R 441 0 R 442 0 R 443 0 R 444 0 R 445 0 R 446 0 R 447 0 R 448 0 R 449 0 R 450 0 R 451 0 R 452 0 R 453 0 R 454 0 R 455 0 R 456 0 R 457 0 R 458 0 R 459 0 R 460 0 R 461 0 R 462 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -426 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb10-19) ->> -endobj -427 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb10-20) ->> -endobj -428 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb10-21) ->> -endobj -429 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb10-22) ->> -endobj -430 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb10-23) ->> -endobj -431 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb10-24) ->> -endobj -432 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb10-25) ->> -endobj -433 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb10-26) ->> -endobj -434 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb10-27) ->> -endobj -435 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb10-28) ->> -endobj -436 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb10-29) ->> -endobj -437 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb10-30) ->> -endobj -438 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb10-31) ->> -endobj -439 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb10-32) ->> -endobj -440 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb10-33) ->> -endobj -441 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb10-34) ->> -endobj -442 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb10-35) ->> -endobj -443 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb10-36) ->> -endobj -444 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb10-37) ->> -endobj -445 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb10-38) ->> -endobj -446 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb10-39) ->> -endobj -447 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb10-40) ->> -endobj -448 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb10-41) ->> -endobj -449 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb10-42) ->> -endobj -450 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb10-43) ->> -endobj -451 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb10-44) ->> -endobj -452 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb10-45) ->> -endobj -453 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb10-46) ->> -endobj -454 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb10-47) ->> -endobj -455 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb10-48) ->> -endobj -456 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb10-49) ->> -endobj -457 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb10-50) ->> -endobj -458 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb10-51) ->> -endobj -459 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb10-52) ->> -endobj -460 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb10-53) ->> -endobj -461 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb10-54) ->> -endobj -462 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb10-55) ->> -endobj -463 0 obj -<< -/Filter /FlateDecode -/Length 4641 ->> -stream -xœí]Ï中¾×_Qç«H"õ Xä°x9$@l4r˜®Ì$xè °É!ÿ~>º,Yv±Úêžšy>¼iLW™”,Z¤>’’¬vg‹Ÿï~ev&ç’"Ÿ/?~>Y“‚Å¿©ÄêbbÿÿS8;ËÉWb矄@ɤ|Šç—~m.ûâ/§?ýâôÏuwö¦”è]8ÿë“°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ+¬ 1ˆ“£[V>9ÖXÕ–³ÃykU±Ìäž &¢Ó£ÆóÎÄ\|Qy <—¼Æ#21± Nãc Yã1d Å: KpÌVãA¦œ4^„,ɪ<Èâs"— ‹Gï…—! ¬+«<Èb£u¯ %3«¼žÍ)ßò¼ &d"«ñœ3¶I*# -JŠIáyÈm(*²²^ãdaüh<àD ”³2Z½Œn²ä4^€,Þ'VyÊ×ÀGÈbSÔ #È.iàS2\• ®ƒ†É‚Y*²pb X›B%kX ŠFÂd ãàY«aAqù¦g 0.%ß̲È7‘â)ã¶P”|3iXP²„(*² h*ʸ&!ß:i¼Y|a JŒ’of f ßDD¯N dÉ7“:]Ü Ȥñ€»ˆL£†¸@¾‰ÈVËí-€7$Äg:3KÆ™Õ è ð­ê”öÂ[%uÎÀ|%GR' &2ËÕY¸U0U§ ,ðñß`Âuö©Ÿ¢Š0‡Ÿ™“Ÿž¾GQñuô«ïÜ÷ãŸ\æÏÏó'Ú îú½~rÞ”ñ·¼æÏó§Ÿ?ѳü©«GWZãÃ1ðÿÓæ~(ãþö«¿ž?üöôëkí!ßg„zèB›côe,; -«ò.… =#{E@‡Àçqz¾tºF{×ýíþÖ•Ý蔟Ág]æzl)å UˆÂž*†jFˆþ‘ÁÜÝUÜéÄ ‚E¦v¤d›Ž«sœ®A.㙼°Òî;¢¨6ú+-ÌÖ(èÐÐ&ÞZn¸|9zõ#¢•UÐçµÑ²m¿G»Æ‹·2M÷ÚÈ1=óL UÖ|º!Ù÷9R±Ããn¤ÆqŒ+É"¼ƒÛ¸ËR$oŒÅ˜J~¯cÆê¦k¼äahùÛ{½Ã{¼÷=o,KUHÏ“Dñ‰ RN¤;;]>Xç8]ŽLÅ’“%ÓG9äÝa¿uʯÁ‰®™LÌH`<Þ,èö3Vç8Š‘u ™øË‹”vpûÿ:úõÙ!!<|Ëx«sµ&@rˆ1ÓÃÔú­bŸ“ÌÎÈÌÛ cªqÕȪwv…Ò½§w Àfçe²îvzHÞé—±*ÇéèÌè©wLoSÝ¿)ëH6;ÚÜôé=®ûõ¶ÄsF_”¶îAQ6ÎO«BÃʪre™‚ƒ,ûÊ|£rcFˆ™ùM#“rm§Ð¼¯ÂÆYîTîU•w_õÖ›TlA•ÛÚOÒ›Îmë­þñD2”efU¦j òG ô=ë«sóI²] ü¼ß|V<õcŸ:_uéè5ÝûÊõÓ¢"`Fiç..d8°`#*yY@œÑ^WíXè–­¬ÄZÙAøfÕ¾QÕ2i‰°1¸M£Oïœù”ƒò|Z—oõÓR¯¡Ò¥K_ö -0o¦]¯Yy†]„’Ýb%r*Zí–H!†ÏJO¨&ÊÈð ®ea“l5v»sƒuŽc¢»Ž|á+¢Ïº‘§á‰¬WÇ'ÛI²rï}g¶àÕmí÷çв¿Î¥ha¿Ã9ô`ã˜K€s-ã×õMŸ¾ß佩SÄ&÷Pj‹Du~8¬è5'7Ýg‹l—}ÄBÇ8Œöh•gØG¬é -–= ·µzЃ!e›´ú«tµ´>mÕM8âGÒ¡Ç1_Ù*²ç;hw'!å,oO8Fô*/D$›SîuÌXÃtM°p\ßî-^íÇ)a3-2À:kµåÅ:u¢wyMz‘§ u${?)…½•úÁ:ÇérÙ웑¯"ªlB–½ø{*«s¡F¢÷[ý#GÅä‡*mµuOmýÑÏ~æõŠ|^OWLa‰Ÿ“–pMXFÆvš6/fÞ>àkêÎÐ’s‘c>{y××Ûéýè×_«su“tŒ„Á_®î·©¿Û:´â©|6KTÛ™­–Ë>o» ¤rW~Tí›O·¤v+ëþ‚޲ᶶ½>e–³lŠ‹trT@Î~wÝx°ÎqÌ/ˆ+Ì1é oòžÿŒÏxõ>ã?¾{¡?_¯·t)ëg”  ƒ¿Ì÷yžËòµ|½çÊÉTþÇë½ÈÎ÷ˆs[µ\–é 0?Ëûyƒ\wÐ(Ê«åNv6›ÃXã˜C–íºÝ¢šÃ·Rכ̩kÏoÛ®ižÔ¿“Ÿe¸ÌÏR‘­¦VyOÖ;æýËršÈ¸yŒÕ9Œy+€ÙF}aôý#n6­Î\vM«ªíyÉ–Žï2-Ý<Š+ÆÅ$ ›Ç`ã˜Éߘ¢¾hûþ÷@góLìÎ¥0:Ô—Âa| g°ÎqÌ#ˆ3„¸:zC=_âäî¨VŽ2->2Bîà³i:uéuÕŽÕ9Žj³ì®•³å¾aÒºnt2¡Ðe› Ðý›¶)ºOl·S¿ImžŽ‘ öoeÙßíZ N—Š×jl3´rdŸó2'yÛz5Ìë‚m2ùŽ6%¤Mç”A´H¬÷ìw¨ÊaÌw:È%Î|guù!3l›VV³,Õ cg˜Ãé»rãnJåóYFHñ`4ÓŽä#ý JMƒ»Ev46ɨžôsv›“uD3q~¦ˆÈÁÈ!æ²ÛšemVÎýEüðûÿü»ÿéB˜´ÿtÅjã ¿ëLT’þ”_à:;5•“Yª<ÏLÏå„êÖöt˜ÊÆÙ?ĹL™‡“›)º¶âµn]$œ_ÚÛvÿϧ8;禳Gmœ"é’å¨H#®(•³œu—-±«Ýí¦rç;5§#Ýn¥ä7HV~•E¾IýÅ\þçIQ?äÍ3yF±Òë‰Ìv"# [òËINäf…Œ Ä’…8µí^²y`C»J’)Û’Œ¿–vs -õ«õi¤~¨ÕdàÉæÇz%¶{mX!¢ft·dyz /|%Ë©æÙÐ|›(!V™A\nÈ}Ùå†KÅ]²ìªXˆá–h»æ²œzSËúöµ4ƒØDï»Îuê¯e}êÔùñrB³´¥¾¨T9K¹Þªö®·FP¥QC-ÙÂMO -érRˆßÙzkÄήñ;­É‰”Wb®†Ä­\©åÒ外›ïš¦R+•—ª¥öü"Êbê”[Ç+´©fj’?ÓÐKõëšEU×È »ñ7ÄËIväÞ_N!)¥Ã‚6TÓë‰ÔÝ·‘³]ìe¡Êán×oš•&çùÉï+T„F䙾8n&%Û—«¥›¼î¤žê°Ö²¨än¨¸wJÍfûþBª ”Ðm¥[­¨]—uôTªxR <Öd׬®‘Âjÿ4¬ì‰¶ -ærh¦ü²"/r†6TƒmE×™{h÷]ȱaçKONMãNXmò¶ûv6_‰—ɱñ–,¦Ÿk# Y¨j>­±Ž¸PXÈ Œ¾Lä:Z:ÉÖèúudšÇ½œþqšÜœ§à%Úää¼ð>Zrçÿ ÌoñÿO|þ]= òêǽmhᶮȽ'“¿.‘Éò×&*± ›Žè¹ƒý…ìòR6Tkh~Üû¯y©n`¯%]CP׸=1,}Ø‘}3]ï¨ë™ð -ñ²éÅÈ|©n‡}½¤úph†ähÓ*.úÇÁe^x´¸äswK\IÑÅ¥ƒË¾«ˆ‹þ¦¸ì,‹d+1Å ûFZ׋üÜ«‚¤­î¢ #ä’/n¶¿u¥MOÑALýEcv}£e%"ô»‡×ë}9Ö ÔíúÒSÛƒ µžN¥é2®/›Ï¡é¨:x—j÷¢nìQ r–¡±„¼ºèm´£æ¥Ú\ ôr¥­­Ž¢Áh¶ÒÚS•)ÎÕj·LOœÕ^J-¦X¨ÿè{:ôZiz­Ä›²rßêñZ´D- -½9§åéìR²Å’Y¡Å¾v£òR’*í¶m^·]½í¹{ ^‰Kdj—já¦ä»ðï=cçZg3{ •ª?ÓŒ†ËË‘ÎpÔìÉlš„;9F`ËZHœ»)Ž7×y}Îãvf£Îyô-A†‰lrò÷Öt^À(ûŒîVõo±éÕä=å‘ÞÖÊ”ÎsöfFgúƒ;ø7Oì´Ë$©¢ü}-’ 7ËŸIÖ©S<ÓTËójúdúù/je( -endstream -endobj -464 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 463 0 R -/Resources 4 0 R -/Annots [ 465 0 R 466 0 R 467 0 R 468 0 R 469 0 R 470 0 R 471 0 R 472 0 R 473 0 R 474 0 R 475 0 R 476 0 R 477 0 R 478 0 R 479 0 R 480 0 R 481 0 R 482 0 R 483 0 R 484 0 R 485 0 R 486 0 R 487 0 R 488 0 R 489 0 R 490 0 R 491 0 R 492 0 R 493 0 R 494 0 R 495 0 R 496 0 R 497 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -465 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb10-56) ->> -endobj -466 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb10-57) ->> -endobj -467 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb10-58) ->> -endobj -468 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb10-59) ->> -endobj -469 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb10-60) ->> -endobj -470 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb10-61) ->> -endobj -471 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb10-62) ->> -endobj -472 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb10-63) ->> -endobj -473 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb10-64) ->> -endobj -474 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb10-65) ->> -endobj -475 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb10-66) ->> -endobj -476 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb10-67) ->> -endobj -477 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb10-68) ->> -endobj -478 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb10-69) ->> -endobj -479 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb10-70) ->> -endobj -480 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb10-71) ->> -endobj -481 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb10-72) ->> -endobj -482 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb10-73) ->> -endobj -483 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb10-74) ->> -endobj -484 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb10-75) ->> -endobj -485 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb10-76) ->> -endobj -486 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb10-77) ->> -endobj -487 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb10-78) ->> -endobj -488 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb10-79) ->> -endobj -489 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb10-80) ->> -endobj -490 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb10-81) ->> -endobj -491 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb10-82) ->> -endobj -492 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb10-83) ->> -endobj -493 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb10-84) ->> -endobj -494 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb10-85) ->> -endobj -495 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb10-86) ->> -endobj -496 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb10-87) ->> -endobj -497 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb10-88) ->> -endobj -498 0 obj -<< -/Filter /FlateDecode -/Length 5126 ->> -stream -xœå]K䶾÷¯è³Ó|_€áƒ‘bÀŒ,$‹f;»6‚ÙvùûùJ-R”T=¢zç! »ØQ‘%–È_UQ”dο6ø/‘Q)åè|ù|úí¤Uô†³ƒ¡ø'ü‹þl4EåMÉŸ?³ÀE³·1œOøoqØV<ýõ«Ó¿ç:ælUÎÁþý#‘S1Ũ½P”Q¤Š¼W1Z„¢`.1g©føH–ÖEfPN^*‚dmŠÌp[e˜a£!©f˜ì£Pd4ì0Vk± †h"g…2ãU@§©ÌR¶Y,‹(3ÑJeΩÉx#•eBðI*#Øâ³6R™‡-Þi© ¶KQ* °Å§Å2ØbStRY„-½“…²[€®$–Á´‘ʲS>'"±,£L§˜ÖeV{å“sZ*3Fy`Ó‰e˜Q¤…2 [‚öY,ƒ-Þi+•9ØBø+•'¼‹) ³ÕòìvÚ©ÌÃk#‰e°ƒ/1€ °EÇ Qfb^’8Àƨ([’HÀ&§(Q’XÀ¦¬(F'Ñ€Í^QÈQ⧠ʬ‘ˆÀiØâ)HLà l!Ì\± ¶¸L8 [PAâç`‹%'qÛÐ"qÝ’ÎFâ¾%m‚Ä„ 3µÄŒëR ‰ (PÊ8p®wJ\à@º˜îQâÖu>X‰ h×Q -8ð®#£%. ð®sÐË` ,qØQ9“œÄÞuFG‰ ¼‹id%.À¨*à0H\¬(›’–¸€À»6i’¸€À»6ˆeQa:‰ ¼k=.AòÜà]뵕¸€À»–l¸@Qà-Fà]Pˆ—ÃØ‚•¸þeÖI\€“)‹°DâÞ5îO˜Ó`+eRxð®I>±,+Ñã¼E/+B–¸Àƒw᨜ļ úL€ã<ª•¸Àƒwƒ§–ÊÀ»Æf-qï"nðÊ ¸þM èŠeY霒ļ Z²À +À$bYT’‘¸d¬4âP‰ xî=K\À»Ú;'qAïjBØ#–Á—¬ÄˆkP†i+Ìi«J[0…TÞÕ#,•wµŽYâ‚Þe·)qAˆ,°Iâ¨[‰ B"•c „¬Q†O, -Ç^₨!ð ,± ¶P$‰ X«ìr’¸ ZØ⬕¸ ZØ‚AâD*Dßbz[1? Ø‚HQâ0™J™Hâ‚èò1ôªT¬Jt-–%”™(qAŒ¤àŠÄÈ¥T¤—¸¯8\’¸f ÌÄ1ÃL‰ àŠT²ˆÌ¥2[dK\€HeÆH\,lA4(qA² YfÐÀC#—DWKe¤9ÏL$ -ŠCE‰ 0Mi‚)Å2Øš¸ãÃɶ—¸É’M¤9blqž$.HÉrº™$.H ¶Xm%.69ߌd [ñI\uP&K\G™Öd“oZ’¸ó’óÍ$qAvù&R3E¾ÈŒiAˆîkÎ1Us7ª]×Ç~ÿåôÍœïù—ÿœ>}5¬”•¿àw8±ö# AnÑ;uƒFdIH1îæ á87âýÈ/‹eÐ*FyðòˆÑ‚Mæ¬ _µuæ-€CNÁ–MÀ:t§« ZÐîè 7ÜУ!Y·Ž:ÆËd‰&yÅaÈ‚{]h§ÎqŠÌÛ’³ú-ùrnÄûXK€.{Y8Òn[·»Ížiˆs}p‚vÏUON[h=ßb@4™‚ËÐG6mtVžbŸÎq€ùöˆÇ¾%gF @,‘5ÑYšXqBÝ€4,@J×rï¶Ø®áÊ•5»¸r¥½“+WúõúÏPû綛OœÎfö°hŸÎaÀky Íæø†àÑÆc+p.Ýø®sÝΞ˜s­½/èZ!h\óS„"oH†CVУW€V²[PìÔ99,FWågàÑîdxÞèÀ…ÿG>pàNÛ#pCØfƒÐÖ6üð®´Ë_#AŽse'lù¾U$âuWŠ¼ËðBÉÓàéÓ9xøþ7ß0J¯ žY£³tùÃ<§ÄÇ]éGË>«¶n­”Ø0(™ƒV‰‡Æl wŸÎq†;ÂÍú’{K·53¢þÕÊÈ2¦Zr§ê|™ëkb¢µ­Û¢|“Ž’vODæ³6H&©õ´p{O»¿ù ßý²!#Ê$C›îÓ9 ¤y÷Y2³ý !=7b€ô‡…ô Dýæ›,pÞµ`Ó ¨µÛpQå@1KÚ=pŽ|2ñ0 ­/á,ÂÞnÐ÷&:DZEà,b‡WtÄóF_zfŠÉÖíîI(ÖÚ5¢{x:¢s.)c¯»xzï†têHžoš"ͯ ¤Y£í2J{„o†Õåƒvq¹#Òu[{ÒµvÓ7à[éß·Hâ`’K|_}xf@ƒ½Þ‚bŸÎq ùaHáU9mÖèû&òî±v@ ù?%r6 çÚ†šÕÊðfÀ$h÷@͛؀©õtå·ê2ËBH¼ ³”à}uÀéúaÖ§s˜‘æ=šŸey=˜Íxàርϳbkí}Zß 12Neó-ÞYl1nÝ«èÔ9ÄÐeÆÙäý~ˆm@*$\~ôÎ. µØM²JìäXé²ÇKá --ÝZáç*L š“ÑÞ ²Sç8ƒëáV³Gèûšü1kôý2~xh¢¦Ë"¦°c$µ\4ioÝ—Ey©½IÕ½ø±¶qÏâÇZ{ßâ‡ÐCwEZ🊟žtÔï;uŽaÞ&ë“¥;øé~Ï-öí®¦’%ÆšeàÆà{×Û¼¯[¼É[|×^²¹?ÓëÔ9Ì { 'jñÛ[n5šñ²[ßÖmíY6XkﻹÖß·ûÍã¬9ÐðMÎü a’:Ç$?â—§>Ãæ¢nš7:ƒ`~bõÊN·˜–+]-OUWzéZ¨hvs¬íºÅVž0 6$«ûÙªSç8Ðð, ν)[ÍŒxa¶Zµµ‹­VÚ;Ùj¥¿“­æWÊÞøþÀ¿Sç8ä7˜èÌ/‚xE¶š5: Œk¢¡–Â~eIµ¹ ÔFW%Q¨ ‚]ÀøÒœãÒŠ×P~msG"PùµvO"0Ývzl”3¿ÄÇZà ‰oâ7úlA¹Oç0PšŸAÞWózPž7:Û<¹„¬p[©ÍgW,ÜèÕü×=½àÙ³áE1Þôå¼p…·\x0A!wÈ:ô»ðNã€ÌñãbDæ-7=ÎxY¾nk _kïsáký}¼‡¼BQ7¼ÒÂ\wÚ> É>ã@ÒóÃjјgØÐÔÏ{³F%Þ«;4—;ÛDnæÃÀ! oetüÊl ¯5\»àÔ´g—m»)#uÅÚцËx-Ë|0mÙzã=6Uc…ÔNãÀÃq ÌÙŽûgÜ­.›Ð*ÃöaÜú÷í 0Â]кâw&gv°G§ÎqàáÙù¥åM>÷ϸgt6/±çp.)ð+ ¿Ê’ø~‰_½½>ãÀ#ñOÝrÛ¹¼ùð|‰“»1´èûŒLÌSÿ«R:u3´Yó3H¤Ãkn«™7:{á]ólbY¹®™ 5«ÙíÚÐò5)ëBÓ›wÖ¶l߈™×Ú=7 5¿ÍÛXdºB똼bm|{ë˜dg“• qx=¶&d×|ûgÀ:Ç°ã'—B wlþé^Õž7Òµ^Z¯é~xbÝÞ­ÅÊLз9“ï_›îÔ9Î{v—0÷U9jÖèË®M¯Ûº9ÜüáléþµéNã wâ'xøKo¸°67âíצ×öìY›^kÿÿ®Mƒ÷TöÙ0zˆ?EÃkÚŸžB}:‡™BÃ+´MÄDËý4 +ê†D r7^ËR'ÑÇÄ|'.`Xu–çnžñY>á4Û9Ö½‡K¸ÀÍYIü€&&öz”:£Gâg4žyñ›sá§ñÝéHõyßïüGÚ¾±½N€Í*x,ß,~6‹7y3®Ãx¥áâ&òS~La‘¿3À0þþÏ?ÿíÇ?4ëþJLœr†7$…Å G¾pFIÉ*®T<ÔcFrë±Ü?”û×û"CÝ0&a¬“ÇÝŒ«£\Ç7m…«nyFn°åãwËîÿí†ï\˜á[G: i@æaˆ¹¹?ó™¿­‘´ùÝm†2çšÃ'$«µøˆ4hŠ·! ÃÑŒõêóÉ€ãø™I®_€Óƒ˜·.Å'þ b„–¶SÌÂñW]ÏśIJ ˜8ª¼¬é¼Ò㯹žÜùò«õåjXß5þ2?ÞPŽ»×†!4ƒY‹ùê1G1E1)7…jŠÅf§R[w:ᤸ)6N7B¿ꦹI̯+.umý5×1‡°šÞvi†¿Ôµ±Χ„—šuKé£(åo·•S•Þ×™kŒÊUêKÍ: TÇI]N‚ðk]NÍQ-oF¿€s¦$ªõr©'ˆ.'±Þx6_G*ÖZiRÍ¥ç'S&¨»T;^ }4J#¶™#Ú•_çâÀ#PÆZ§7v%¼œø!•øñä£PÛO\¡}^+tÍy«8é /“”·Ì_ÃÔ,2þ~ÿ¥ -_¥Ì<Ã/†*¤ø@ÓÑÔÏÍ*žwR+•uHêN¨ÍJŠsÇX1Ûö—IZ¨Ï¤[k×›I›.kä1ó˜¤*yÌŦ¢®Š™ÂJÿT®l…ºæ&±¯P~œ‰';}ª^×Æ‚iàîëy'q¨ÜùØŠcqÃtªöÖó6˜/ÂËàØh)fè§ÒÈ$æèµÀ§6Ö'Ã@ -“x¢ÑÇA\fke'~IJü:M2Éã^N¿ž7‡Ór®œáï¶Ñ’9ÿu~À¿èücù ÎÕ[]ÙÂVn‰[OÆ_³MbþºmÖiÓ-5´?‰Mšêú‚†êÇ­møš&u={©i*ƒšZÚ -ýÔ‡ØVèZãšžñO/‹^¬Ì—ŽÍä°¯‡®\¿Dƒ?¥T¬ÂA{98LS™›\ò¹9%Ž¸êäÒQJ¶QÄA{R6 æɲ™™ ȶ‘Úõl?µCá¸­æ  #øÐMe4¹ÙöÔE6\ES!°kÚ“iÍ3{£ïž^_<î£É¡Ì–Vj¯ÒÇVZ/Œ¥•ðd©Ãü°ú7¼£¥LÞ©‡ê¹\3÷\ r¦©1…4;h1ÚHÓ¤6Vðm£Td³F‹£¨t™.²zÆXl -£Zé–ናØK±Æ“ô׶§};*µ};*aU—Ï[<^–\ €| ç8]žjÖX2 ²ÐjW)M5]‘­Û¦yÛÅÛž›Ë É0E¦zRó«šwñß=s窳X=j•¿ÃŠ†IÓ'äà¨ÉDÇ·è8ÜI!€Cˆ?ì)5K»už^óX¯l”5¶% ‡"çt4ÀÛ Q· šS=Ž_ß“Õø©Sá’öu€°¤3Ñœ^­è øÆŸqa§FN}>êSâÏGmÄ%ža©å2[>þþÆky -endstream -endobj -499 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 498 0 R -/Resources 4 0 R -/Annots [ 500 0 R 501 0 R 502 0 R 503 0 R 504 0 R 505 0 R 506 0 R 507 0 R 508 0 R 509 0 R 510 0 R 511 0 R 512 0 R 513 0 R 514 0 R 515 0 R 516 0 R 517 0 R 518 0 R 519 0 R 520 0 R 521 0 R 522 0 R 523 0 R 524 0 R 525 0 R 526 0 R 527 0 R 528 0 R 529 0 R 530 0 R 531 0 R 532 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -500 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb10-89) ->> -endobj -501 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb10-90) ->> -endobj -502 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb10-91) ->> -endobj -503 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb10-92) ->> -endobj -504 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb10-93) ->> -endobj -505 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb10-94) ->> -endobj -506 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb10-95) ->> -endobj -507 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb10-96) ->> -endobj -508 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb10-97) ->> -endobj -509 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb10-98) ->> -endobj -510 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb10-99) ->> -endobj -511 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb10-100) ->> -endobj -512 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb10-101) ->> -endobj -513 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb10-102) ->> -endobj -514 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb10-103) ->> -endobj -515 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb10-104) ->> -endobj -516 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb10-105) ->> -endobj -517 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb10-106) ->> -endobj -518 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb10-107) ->> -endobj -519 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb10-108) ->> -endobj -520 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb10-109) ->> -endobj -521 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb10-110) ->> -endobj -522 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb10-111) ->> -endobj -523 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb10-112) ->> -endobj -524 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb10-113) ->> -endobj -525 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb10-114) ->> -endobj -526 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb10-115) ->> -endobj -527 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb10-116) ->> -endobj -528 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb10-117) ->> -endobj -529 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb10-118) ->> -endobj -530 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb10-119) ->> -endobj -531 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb10-120) ->> -endobj -532 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb10-121) ->> -endobj -533 0 obj -<< -/Filter /FlateDecode -/Length 5072 ->> -stream -xœÍ]K$9n¾ç¯Èó£Õƒ’(À˜ÃÀ>x5°pc oÇêÜ©]=f}ðß÷GeèĮ̀êšìêªÊõ Dê#))TîlñõÃ&g˜KNt¾ü|úådMŽÿjŽÕC%ÿ ß9ž¥l¢+‰ãùgIÙä}Nç¯'üØ<ÎÙ¿žþãw§ÿY—qgoJIÞÅóß“9gRɺ b49û˜Rr],E#˜ÉÓž”ÁŽ l÷¬lÊ.ìIløìH# WbVH΂ç­Ui`įÐ\4 ƒž4šw&qñE¥eÐ\ö-“2¹è4Z1)¥ÈÀK,Öi´^¢#² ¼Pà¬Ñx )X•^<ç Ñ2xñ¢Ð¼@»X¥›¬Óh%˜X˜H¥Ð,gÞÓ¼&rV£9g"t3¨4Ì()e…æÁK²±¨4ðƒõ-€—FNÄ™•ÙêevœF‹àÅûL* ¼@øø^lN`Á% |ΆŠ' <CL¬¡€çb(ç Á€/ÑP*YÃ`hÞi@,x‰”4$¼f®J/¡†Áƒdа ðâ)hX€¶mÑ° nɧaAÞ’uIÂÀ›VÃ‚Ä œHÂȅ* 07;5,]L÷¬aAꆘ¼†°ˆ“†¸ÈY ¸Jª4ðÖ°èh‚ã awƒ³YÃîby U=L@WŒg¶p׳% ¸ë3L€JËs0hX@À]ÑÍrw}´^ÃîzòIÃ(ŠþXÕîB¢î€¬†°÷ ù a*3n‰†¸ë -ÌŸ2§VÆqIDà®cˆO¥ã2F\™·eãR*Dà. Uа wŸ¬a0ÞÀ¢z "p×XjÜu¾X "p~CÔ°ê` `ߌ³P]•VŒ-ÌDà.`ÉkX+l,ITZ6P$§aÀØXø¡$à.Ì{Ñ° wm AÂܵ·G¥—À^Ãø5 aÚ*sΪ±H¡Ñ€»ÖA ¸km.$஘M R–Ï€A/ØkX˜LÉ9iXŠ .ŸJKÏQÂl‘Y* ¼P& àX› -kX=x ÞkX=xË a<S¼o5</P5> ðOQà ™áB¤aAŽ ñFU£%o8®Uƒæ²†9“)v –2ŒI¯aÞˆ»¤aØÍ‘†¹€L `Š {xæÍ8ÙÀÓÍ9 ؃xƒ°gD™ÉjX XC­ÑÈJœÉ0%#®¢†˜&ˆ4”* ¼&4,€|$ØŽ øA°‰0G¥—IÃf/á&kXÀ ^¼õ@7%ÞÌ ^àñiXPl2â&kXÍZ ŠcÄ›ž4,À¼”x“5,(Á"ÞDˆ§ÌÛ’Ä›YÂBà%ÂEQiàNSQæ-0 ñ&ÐI£%ðâ iXPR’x“5,€›x½º$Àofu¹¸ I£wá™& ð€xž­Û[oÌðÏt"KÄÉêŠôFØVuÉÀ{a­²ºf`¾#©‹`‘%'uÕfDgÕe ü…?F;L¸®>ÍKT êa39K' ¦Á»ï1ÇÈnd»®ýý¯§ß¿Àøžÿú¿§×ßÕ•²öå ü’ÖQ¦ˆ/7àÊÔêÁ™Epa™+‡`ð=e„‰\“°¸,º “0Å0·åÜ?¸5ËàXxvpfjÏhn‘‚ƒ¨BÒy}s™âuŒQ;à1úÕ ÁCþÁF«{´Ð‡ìÜð4L·o/ôQ܆ް•Ð Mœ½XLo«—ÁT±«¯o,òœ¢!ñ³r…âÅ.lQXZeþáÓ µ4]ýîßð”ƒëíùÓϧßÿøçÿñ_@³Ÿ^OŸÿI\çï¿sïýý_çO¸ÍL1€Æ–TÖĸ°D£Å²ìð ¼dÙ¦z¤ÇÊ’gÙ.‘¥ã}N¨ -”À¥¬•>0ÂDÉxe!y_¾õ¢öšª~ò€V„¿®•*”}JTÿX™çQý '!¦Äá±ê?Pm8ÿ)X±¯*ý¿4™4–üÒŸË5_m'ÍÕgð·îÿ~ü¦1çÝø]FßzÒµO×pƒ<õ¹õgá¿/šÄ1Õˆ6zý®ŸýR~îo5cìêsü6tCîì1ïy«y}E;c­¦{µ=» :/|ðZNm<;/‹<%Ý-¼Ê²ùõ ÇÂÏM$Z£Ð­±94n<ä"KïaYª÷ á^§>HÞ/wÇÇïæó4Æ­Ýú]†^·ö{ž©ÏuoàËÂ×ëØN°2?^‡ìâ¤7û¶¯O·²ñóÕÛóú°¯‰ÿ8vÛ$Ÿ½ÜKÚéZÓ…/é-ó³Ï1VÆc;?ÜЪ——I?—¹Qug¯& zehÑóm}Ž/˜ÖÆu…]“^·9~o¾4|íÜ·½MùiÁ°8Æ£cnës‡×zÓñd/Çã|ÎÐECQ–…Z+·ý{Àg˜GèòN`ŠÕî¹x-¼Ù¥$§ð‰YiICé· È¡Q¦m«…'5Àá;Ø2ç¹ [5Ï¿waN¯Wª'y‘2×w½„¥ìÌÄxàòáÕûu¥Ÿç¥„í²Ã´MÔgÜ´ì°]f8¼¼€Q3¦Ê'…››O^ÞÍÎEnuH%>:›~¬Èóˆ;'y“Þ“>?fQsÝHÛ$ŸWÏçÍœ‡Ä½¬uË{¡¬Ô}S˜ŒÀÝü¸+ó4âd+Æ€ éG{?FœëF>7Û÷ ¼¼Jgà·/ßøØ.XvÛ{c]zw’¿¾èëû´GE|°ÌóˆXÞ'v.ýxn_ãm^¨²ºrE6[‡f_·kfîÍ>Y^»f/QC1–\}[ó¾hŽ•yÑD/×0 *REsd– Ò.!Ë‹žëÊ.cO6w_>¶Àáèñ‹éœÓ¾¶‡Kò®±EnJx__¦—–”òrÓoT7¯Q£Ño¹”´u-—©à6Û'­·”ïGšíºß7Sn;Åò¦ƒ½…vrS - ÙÉEVÉ ¶#ŒÕƒ)u¬ÌóL)¹qIÞŠÖ¿Ï]Wúyú—Žº—³²ïj|RiRõ]éÖÇ•¿ä7ªô—;t]y˜EúÞÉ4÷õF/ztå`™§QžbÅ´Ë];¨<ëJ;®Î'1m[ÚàÍv;ê ¹•³~nRÚ~¬Z#îÙ—>‚Âr qÔÊ¿LhXÆ ý~RÿÆ:Gqr÷!DŽûiË<òy9ÜàÉ@®;F`·îá¶3û¶èä<S@ð²fæó‚GÄ?|½}ñÆkPEœ5Yÿ\Žï{DÃ^†8XæyÄñž‘;päVù-‹À‚Íèb/k,Ù©…ðe^±ë¥±ùTOó{dŸÓÑ ‘%2>±/é¸98XæyD–Å|ɽ;#²¾?ESóÉš­;7§Mâ¬b¹ùrzá"×_Ök’Žž>Xæi#õ~"û¾É´_Ø{p[çþžô®e#•?-—®äˆ~„úVÇ2_õÒEñ0˃‘o§ˆö×fœÝæš -á±÷ˆ*å¶åÀp •,Ý:X$[‘þÑFàf‹Ö/Ûõu°¦£rTâŠH¶hvêxm¶,ƒê–WÅ%Ïvcò‹Ç$®ç·ÃÿË)Õ ²\½$Ñ&®‡^Yî´3rj!—³\Ê%{…® ·«äÎ7JÖ»§é¹ä'’¬ÜP)S¨Šc~XòÿRõóI\écÝ/¬WÇÚš,¸Mþz’«ƒII&çÃ$KâòÑöºÈõ“U-íL€ÉÞæ ÑØåã8“&[b×>¶ÞHùØŠÉ•NdûA±z©Ñµa%%“Û'Kï­qK²D lÂò”:+15ž‘8*¤9ï¨p|˜,û/#1îíÔÜH–ÛZ^ß?–.s$vÖç¡s“ø[^¸PþPâå„fÃ6õ«š*—¾¶ªÚèzÛÓÂÄT驱åì¡.'%érR¿³­j‚0;}’šïû÷܉z¾Òò)I—“šo©-vIåž‹GÑÒF~°2qä>ðJÚe:¢˜å>ùiŽÀHù&‡99‰š¬-Ozãw‰—â1Ú%=ŬäŽ+llª7'†©ÞžÌvèËH…ó³ ¦fK“‹Çäç*bOä©u•’ËÇÓhDæfO^Òœž:ê6ŽòÈ; —˹ëì<^Ž­’_@·çî#¶J†lJÏ¥±' ÕÁcìºÖõd°6>+çDÛ˜@s#9vUþºJ|Æ>U£í%7©{ìõŽäÔ±ó뜜»ÄÜÙùíõN:ß/Õ°Ñ6YTŸ[##YV°šúôƦÄÁ@a$ýZ“Ûlíè$¯_¶c’i÷rúÛ©š9¢¯5'ÏÞ’;ÿòüßÿ}¢óÛMzW;îmG ß±u•<[2¹Ÿ&˵ø-±O›)ÑÓû#ÙñÈ›6t;îý„×4ŠÛì-§ëê:uNŒc §dßU×»0L¼“xÙŒbwd¾U6Ã`_C뜕CÓç®Pò0wˆq–JÚå•z›ÅëÞRèPœÕ9ÞÙ‘³û’¬¤¥¹tO¥‘3´´}Û´n»YÛóÔ Œ¤á™ÚQ,îr¾ ÿÞ3w®e6«Z®öUW4»ga¨ÉåàCýsB–S†ÜŸ‰§%Ž7—¹¿æ±_ÙhksK‡¡lvò‡¡t¾‚P¦ªÚÒ‹«vém ,é ˜³»ú—AðoYØéYBEùC@rÖ“Yþ~A¶N]âY®/›—Oê×ÿû`ª´ -endstream -endobj -534 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 533 0 R -/Resources 4 0 R -/Annots [ 535 0 R 536 0 R 537 0 R 538 0 R 539 0 R 540 0 R 541 0 R 542 0 R 543 0 R 544 0 R 545 0 R 546 0 R 547 0 R 548 0 R 549 0 R 550 0 R 551 0 R 552 0 R 553 0 R 554 0 R 555 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -535 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb10-122) ->> -endobj -536 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb10-123) ->> -endobj -537 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb10-124) ->> -endobj -538 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb10-125) ->> -endobj -539 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb10-126) ->> -endobj -540 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb10-127) ->> -endobj -541 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb10-128) ->> -endobj -542 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb10-129) ->> -endobj -543 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 329.565778 101.433071 310.815778 ] -/BS << -/W 0 ->> -/Dest (cb11-1) ->> -endobj -544 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 310.618756 99.933071 291.868756 ] -/BS << -/W 0 ->> -/Dest (cb11-2) ->> -endobj -545 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 291.671735 99.933071 272.921735 ] -/BS << -/W 0 ->> -/Dest (cb11-3) ->> -endobj -546 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 272.724714 99.933071 253.974714 ] -/BS << -/W 0 ->> -/Dest (cb11-4) ->> -endobj -547 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 253.777692 99.933071 235.027692 ] -/BS << -/W 0 ->> -/Dest (cb11-5) ->> -endobj -548 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 234.830671 99.933071 216.080671 ] -/BS << -/W 0 ->> -/Dest (cb11-6) ->> -endobj -549 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 215.883649 99.933071 197.133649 ] -/BS << -/W 0 ->> -/Dest (cb11-7) ->> -endobj -550 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 196.936628 99.933071 178.186628 ] -/BS << -/W 0 ->> -/Dest (cb11-8) ->> -endobj -551 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 177.989606 99.933071 159.239606 ] -/BS << -/W 0 ->> -/Dest (cb11-9) ->> -endobj -552 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 159.042585 99.933071 140.292585 ] -/BS << -/W 0 ->> -/Dest (cb11-10) ->> -endobj -553 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 140.095563 99.933071 121.345563 ] -/BS << -/W 0 ->> -/Dest (cb11-11) ->> -endobj -554 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 121.148542 99.933071 102.398542 ] -/BS << -/W 0 ->> -/Dest (cb11-12) ->> -endobj -555 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 102.201520 99.933071 83.451520 ] -/BS << -/W 0 ->> -/Dest (cb11-13) ->> -endobj -556 0 obj -<< -/Filter /FlateDecode -/Length 4911 ->> -stream -xœí]K$Çq¾÷¯è³¦23"_Áa,@/hX f› ÆP¥ƒÿ¾¿¨®|TUtWõp(¶ ïb·»"ò•ùEä³ÝÙâïWÿev&ç’"Ÿ/?ž~:Y“‚ÅŸ)ÅâabÿÿR8;ËÉWbç…@ɤ|Šç·þ[=ŽÉßNÿù›Ó_—yÜÙ›R¢wáü·„ÅdRNÉ…UÀ²ŽV&%¢ÂŠÎàKÑX#$ö¼e%ˆÁ%1Øû¬°2Ä NŽ¶¬1|r¬± †+!),g!‡óÖª<b™É+<LD£G特ø¢òx.yGdbbœÆ+&ƲÆcÈŠu/@–à˜­Æƒ,L9i¼Y(’UyÅçD/AÖ) -/CXWVyÅFë4^!JfVy<›SÞò¼ &d"«ñœ3¶I*= -JŠIáyÈm(*²²^ãdaüÕxÀ‰@)g¥·zéÝdÉi¼Y¼O¬ò ”¯!€Å¦¨Az\Ò0À§d¸xÖ@Àg2œ9k(às1œi0àK0KÒp€¬Ï; ÈB–ÀQCr…ÑsUd¡Â‡,H adñLЖa-à–mqð–­‹bZ ˆK9²†È…) ˜KÀN  ‹îž4,  .…è5, À.qŽp—ØY ¸K„œ*²À€5,:r™4,`à.9›4,`à.º‘×°Z5°Ã¨alÅøœ­† ÜõÙ²† Üõ .@å%ƒ>H0p×¼‚æ¹»>X¯aw=û¨a Ŭ w!A VÃø{ð• žƒ†É‚Y*²pb X›B%kXc”­†ðÐK¢©5[gf 2G#¡¢†è&i)UdLhXýÈ`;hX€Á›æ¨<ÈB5,ÈÙËp3kX3dñÖkXÛ”ñfÒ° XÈ‚ˆOÂb£‘0YÃÀ8xÖjXP\ÆxÓ³†è—2Þ̲obˆ§ôÛBQÆ›IÂÂ% DQyASQú-0 ãM “Æ‹ÅÖ° Ä(ãͬa Œ7Ñ«SYÆ›I.î†dÒxÀ]D¦QÃ<`¼‰ÈVÛ[oHˆÏtf–gVg , 7À·ªSØ o•Ô9 ð•1’:i11²ÌQ5€[ÓYuÚÀñ®³OãU„9$øÌœä% º#ºocŽžŒn$»Îýíϧ߾Àùžÿü÷Óëo¦™²ú×Áø‰£ ¬ƒt_ nÀ•™Šãi  …' !à{òˆ©t!`qžt!áŠán˹}qK‘!±ÈìÌLoÆc½)8¨Š¢.ëÃy>HÖ¡dΪȀsl¤´*ž¨½afúø–*îͤJûx¦’–jÅ€6 ÍŸ?!*…dÙ€+~uõÁ,!)*’8+"/~ `ƒ…åEâo?Qsõ×Ùïö8ÈH‰Ñ¸Þž?ýxúíwüïþí[Tûéõôùk ¿ùÊÝùäæÏ|ý PNpK¿^?ÝLÇHÄŠ-Ýÿâü݃îçtN¡ù^F˜ó€ 0þæOçO¿;ý맥Vr0ŒðMCpGÄ};Š<–å]Šü%ô‡?‚`‹ÿ0ýmò\æϸÒsPô–ÚÙ¾à3¬ÊJ«:F]#£Ëø7¿Ã¡OÕ:(]m’ ¬Àˆ!!dÙ1ƒyžÆ>0ŽÃPJuªÔÎØ:`U®Ð^#…UeHžÔÓ-”55ôMÁð®è¢5âJ0‘Å}ÿÍNnYþ`æ”Ü–ökGèI”2¯›åó`r“ÙæõmL‡äiAs’µ;èß1cyžÇt0j÷LÞêØRûdýä/³™6¿ æ‘W}úËžš=;™aÉ+Ž‰Ÿæ ;%÷#!YgÂCRòsºÇëêýoKDH“#+aBÜž:–åyL%ÉâU(Öë¦2ô©æ^ªùØMî«Å‰_†}ÉjQÙh›†Ï2Ô*6˲’ñÚ.ÀËó4Mïe~Õ–¤G±{{Ÿº'ÞþÔôcÏn|TGÐ"Š±g„ÁpO[½|Ë–24ú¥›KC›´­§}æ2 åT¡]ü+þåë÷Jw|.ÆtTyñV—…ëì$›ÖNew…Ûsó<¡É¸†œl Ð ÍÙý>ÌÎJ2e´,l UÃ÷CHs´¡ç£ÆÑòU#| #4¬Z Mõ òŒetlÛ¬”$ÆcÚÜ@Òæ½aæÁ<Ïc4²â*Kù#ŒfQØÂhª‚.7 ¢*ÄÊL÷ dRêo­üIö5ʽ®Œ‹ï@œâQ—€²ò0Ͼß7€cyžÇÜiˆ1ÓGÀ¢°_ÌÖ‘ -.jåjÞƒ*n…Z‹ÁwYMˆÜ *}N2.k²…$;æ@{Æs,ÏÓì5ÊÊÿôXöKæŠ&Å®b’Š EÜÒ€*}mx· ì×3Dxq“£—U -Ñ°Ÿ¦½ïâÁ<Ïcˆn7{xÞ0ÄEaG qÏèšÑ@ó+¥¦eY•ÿhm0-Ëqs~·’©^8¬‚ý5¢ÖúÜzœ¸ž.7 ’²q~Ú½áeÁ-m÷ñ`žç1È k:ËGä¢°Ñ e -p1;N#ºõˆknÓTâEI—çùÜq„%ŸëÜ #¾[QÄØ=ËJæ¢í¹ÈRí}#8–çyŒ ÉöìÄñCPiQØ•V“ð‹ÑÍ#F°îÉvv©«xkí*mØ1Aw]Ùh\ãü÷ZÆ[²…ÛïÙîgü¡¤~Î&+‹¦$Ûâ-´7çu0ÏÓ>[Ùfå<ƒjø*w”±˜ñ^»±µk^R5°°uÁÍ5S n=åà—õ´ç—!_5Œ¶Oíê­‘i­Ú®ýqØÀ2¸Ä €®ƒµoÿý»ÿúý¿ôv¥©#Šë½~†ù98i»ëw Ïà¿ô¸—Ó&_ŸÑ6“„7E’}ý18H²išƒ—š^åÍ¥U¯%:{¥O47K(Ýåe§&ŽQvÒqÎJMò.RÒTj¸>·6H«wM;5Í\Ö5ù¹…Bê-;M~…Y«cë¾VM_ß_ÖO…îæOûå*¥H&åJúÆ¿ÌoÅs;ÍoAòFa.o–#Ìõ]{’dógêrP’R}ò=L_Éf.'ûw)¦=|0ϯ D¡ÉËÉÀîr7'ߨ¬:lói .wãݯ—f½iòñ2l¼-ò2ź×äÇòüÚMÞåM²Ç“Ð3î¯JËû%ÖÖ&žšþð*ãR {ËQr, ÉþMY ¶†egºÛSϱ<Ï£ž’LÁОoôˆqÊNl¬‰d³ì¬[úyè5Íý ³ë%ªÅ¦“›±ìÞ²rß¹°•æ–ÂG‹a„/ç"'0cÙ¢;–åiÔ½|Í…õí%w÷í¨?fà -äW•|þúþ¸4Ľ·í0Ø–}S™Äh:?Œ8 -®ó<:ÎÀ¹HïØNv\‹JŽŽTeÛ¿³à·Íÿøäç ËùÜÀÙÛT|,Ïó¨XÎø'Äž76„½ ƒûËÖoN>s EÖÓwGz_óKaîö¾Ì&Ã~œœìD²³»›hæyÕ)Ç-ô­Íç, ;¸ùið¹ÛümãîÑ\Þmi»¹\ac‹œj|ß»x‹¸¬Ø‹–~ÿŸž!ŸÞUïRè‚Æfëè=Œ/9î-?Ìó4]*yqÀr´ùÃÑe¡ŸÇ¦Y©àhx9û¶ôý=ÝÔ·¹ÛÔèýØîû;|Ýx0&D©VÕO·opÚ3žcyžÇx®q¾»±µî}Ƴ(´áê8_më¤Ï -o.Û>~ÄÝrœ./ˆQ©{ß´ú¸g›û -÷=ÛJþ— KŸ mµ7æ9’œ=”I1w ê¼G¼œP-­©o*U®…®EÕÖElVi4U5Ô”M!Üô¤.'…ø•­E3”ÙøƒÖ|Ûœ«!qKWj:…t9©éæÒBÓTj©rÏZjËwQº©Sn ¯Ð¦6š©I~qbè#–ê×%Y7]Û<Øß/' ­xC~;…¤¤+l¨¦7i(·‘³íöÒ©)צB׬4¹šPþ¿BEhTAžé‹ãfRr½hê•Hßläe#ôØP‡µ¡–ÛPQvJÍfÇörÙ*ét[êÖb êÐd=•*ž€T%Ù5«kd°Ú> +G¢­B ºNÍ”ßä.gh]5ØVYtƒ¹‡Vn'džo#95;¹+¶ÉÛÊl¾/“cã5YL?×J:YÖÍ«ù´Êb  ÐÉFß&rí­ ä÷/ê×ÞÉ4{9ýå4¹9OÁKDÉÉÕçc´äÎÿƒ4¿Ã¿ÿ>ñù÷õ®Í«÷¶¡…oغ žL~(#$ËgTbë6ÑóûìrOª54?îý€×ܳÛì5¥kêw$†Þ†Ù7ÓõŽ†– wˆ—U+¶@æçê¦;ìë#Õ—“™¹¥µJ…‡ñuð˜;ºK>EâI’v—.û!#ÆBñ8TXºd 1Å ÇJZÓ‹ü<ª‚¤®áa#ä‘:»›‹®´é-†$i| f7VZò(B¿»{ýl½Ï"ÇÚ„Ú ½QßFj{1¡6ÀÓ©4=Æåcó94M4ÕÎÛ[¨•ECߣäô®ÑC ^<Œ6:PsÏ6'c¥\i‹J«£hpš­´Vbª2Å9[m–é³ÚJ©Åú—±¥Ã¨•ÖˆaÔJܤ•r«Çkѵ(ŒæœúÛÙž²Å’Y¡Å1w£rOI•¶­›—uWo{^ƒ» ±G¦¶g ›”ï¿÷ôkžÕì–ªþf4\î·SÃQ³KäiúÁ1›c†°üfDâ> -endobj -558 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb11-14) ->> -endobj -559 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb11-15) ->> -endobj -560 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb11-16) ->> -endobj -561 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb11-17) ->> -endobj -562 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb11-18) ->> -endobj -563 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb11-19) ->> -endobj -564 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb11-20) ->> -endobj -565 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb11-21) ->> -endobj -566 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb11-22) ->> -endobj -567 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb11-23) ->> -endobj -568 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb11-24) ->> -endobj -569 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb11-25) ->> -endobj -570 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb11-26) ->> -endobj -571 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 444.418756 101.433071 425.668756 ] -/BS << -/W 0 ->> -/Dest (cb12-1) ->> -endobj -572 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 425.471735 99.933071 406.721735 ] -/BS << -/W 0 ->> -/Dest (cb12-2) ->> -endobj -573 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 406.524714 99.933071 387.774714 ] -/BS << -/W 0 ->> -/Dest (cb12-3) ->> -endobj -574 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 387.577692 99.933071 368.827692 ] -/BS << -/W 0 ->> -/Dest (cb12-4) ->> -endobj -575 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 368.630671 99.933071 349.880671 ] -/BS << -/W 0 ->> -/Dest (cb12-5) ->> -endobj -576 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 349.683649 99.933071 330.933649 ] -/BS << -/W 0 ->> -/Dest (cb12-6) ->> -endobj -577 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 330.736628 99.933071 311.986628 ] -/BS << -/W 0 ->> -/Dest (cb12-7) ->> -endobj -578 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 311.789606 99.933071 293.039606 ] -/BS << -/W 0 ->> -/Dest (cb12-8) ->> -endobj -579 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 292.842585 99.933071 274.092585 ] -/BS << -/W 0 ->> -/Dest (cb12-9) ->> -endobj -580 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 273.895563 99.933071 255.145563 ] -/BS << -/W 0 ->> -/Dest (cb12-10) ->> -endobj -581 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 254.948542 99.933071 236.198542 ] -/BS << -/W 0 ->> -/Dest (cb12-11) ->> -endobj -582 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 236.001520 99.933071 217.251520 ] -/BS << -/W 0 ->> -/Dest (cb12-12) ->> -endobj -583 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 217.054499 99.933071 198.304499 ] -/BS << -/W 0 ->> -/Dest (cb12-13) ->> -endobj -584 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 198.107477 99.933071 179.357477 ] -/BS << -/W 0 ->> -/Dest (cb12-14) ->> -endobj -585 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 179.160456 99.933071 160.410456 ] -/BS << -/W 0 ->> -/Dest (cb12-15) ->> -endobj -586 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 160.213434 99.933071 141.463434 ] -/BS << -/W 0 ->> -/Dest (cb12-16) ->> -endobj -587 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 141.266413 99.933071 122.516413 ] -/BS << -/W 0 ->> -/Dest (cb12-17) ->> -endobj -588 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 122.319391 99.933071 103.569391 ] -/BS << -/W 0 ->> -/Dest (cb12-18) ->> -endobj -589 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 103.372370 99.933071 84.622370 ] -/BS << -/W 0 ->> -/Dest (cb12-19) ->> -endobj -590 0 obj -<< -/Filter /FlateDecode -/Length 3169 ->> -stream -xœÍ›KÇ Çïó)æÀe²ÈzFrˆ0"8H‚äd#XsÈ×ÏŸ=]îå®ve«é&ëÁ®býŠ¬éå+áç+ÆU9ÔÚJÖëÝûˇ …’ÿ¶‡›Mý=~Kº2i ‰[®éúÞRBi)–|½¿à¿ÓíZüþò×ß]þs¬Ã×ZË‘Óõ—·¦R ¥–BÉQ5¨ˆÅQ¥J‰);ªÌØš§‚©hÔ‡ª3´Õä©`†ÆXU…¢…塪ÁŒXX=Ìà–Š£b‚‰\ !U‰ŽŽSÈôìé"‡\[l®®@Ç%z:‘‹rbO×BÎ9UO§°%5bO—`KbUòt°E¥O—a‹d!W[b-âé -l‰æè*lwUW[({º&!µªêêtTK}¨‹”Bª"äé˜C‚oŠ«ÃŠÂ$åâè"lÉ”š«ƒ-I(z:-ŠON$)µ:«5Úêöt ¶ÄXÔÕÁL¾G€˜a •ì!+(—<ÄR‚¶¨b• U«GX[ÐRÄÃ@l)hnÅã€CÙlIš=ÃÅÊuu°Ešz,[PÀcl‰* ´Ux‹Çn•{,ðV‰³Çpa&y,WjVä•šÇsìôX €.–{ñX  ®¤=°+Z³ÇwE™<(¸+‚š®¶À=€ŽA¸ŠÇw…©x,PpË(z,À¬øaöX_ ±VòX àn¬¤ Ü[€«+kP<(¸ÁÛ¹Áݘ(z,Pp7jÌ à(ü!7w䇰€õX€ýº( ÐXˆK<$p—¶?gMƒVkË ¸ËÓçêZà‚wÖ-F9pÎÍcAw±Q‰Ç‚îŸÕc°£F ÜeÁNíéÀ]Ž<$pqCòXwŒÀcö·À×uu-P«ÕcAw¥è±»p  -’¸ºàHì±0„8ÔcAw±½7Ü¥$â± ƒ»¤{\l‘= ®ËÖYÓVEÂӻĘaOî•æ± ƒ»¶mz,Èűz,€QpôX«†VJöXA‡ÏÕå€ûä±  Èru°E‹z,@`š´ê± DØ"1z,(¶ dðX€ˆ 4Fôí¦°îâæ -[)z,ÉBmª JÊÈÇ0ªž.ÇP píê*t\<”¢[1{,@.*½Ç>X¸ä±f@Çê± 4Ø‚¥â±[Q¨‘¹§cØ‚ Ûc"]è˜=Ô[ z,¨±"ËÌä±;4rI µ§S²<³z,¨šƒ…Š °Li‚”®¶ 0?–l'H~l"Íqu°E’z,¨5ZºY=Ô -["EðMË7‹Ç‚F°ŸÇ‚F9X˜ì±‡ŽÈcAãŠ|3ªÇ¬KË7«Ç‚&„|)ž³n›dË7‹Ç‚¦°%!Dqu°ASsÖ-˜„|tòt¶Ä¦ ZΖoV3o"¢wªå›Å=.wS™<¸‹È4{,À òMD¶^nOo*ˆÏ|eµŒ³º'ô&ì­î‘½Ø­Š{f@€¯åHî¡ÌDfY³{j€mJ&÷Ø€À_Äcú€ ·Ó§õˆ*à -öÌZì!–…"º9Ç,&»ýòÓåë7Ø|¯?ý÷òîwÛIYÿa8¿h¶Ä:Ù‰ Ó \…­9xE1`V[¨f! ü”:fDiÓH÷C73[1¶Ûv|4›ÍŒ`f{2]ûE¦À˜*ɾ­/®ó¹l•Þ/–ÒoÈäm½‘»"ÇÞ0S_XåsXŠŽ,È(%-;± õ’ -óú‚e´w;¡·‹ÝØ̓éúúýåëþþ—þô º}ýîòßéüÞv’W_±Ýÿˆëº_§Û'ó~ŸoŸ)îŸè‰û5n¬ït*kíiezù­Ï½lêívÐŽ¾Û¯÷O¦ý¾mý¼Ý?G_¯þy}ýí寳ˆ¼_òa(“…cØóŒ$ONüóª|ÒÄÿó, w¾·qÜFæñ¶C"ƒ´½éЖU'9¹Â2µ}ªF™甎O™ÓÍÿšîpp£zœúq¯³î˜æÞ¿z™>Ã¥{]žß®åø,^ÈÒò<}œÓ»ýÚux0&µ—+vƶŚO:üóª|1d «SÅ®ÇÏ!ó‰ÀýlÛivB™ƒFÄêú±z^/gˆBF•HŸ -ÇÆV*ŒUrwòÖóê9‘¢¯£ÃÅÓj)Ƕ1^ØçXùåØïõùdS_µçMñ1Jõþ%L—·GWlFxT³4ºFK9#miúÓù¼:_ŽCû¾&5ŠŸÃ!¶©77ÌŽÁï“$;ÂËÑÎ9œ`’”«'twç<•{̉·ö|'¨–ç4™ˆ:lìµY²ô´<¯ÎãÑÐm+ŸÁ Ž¨tŠÖý÷ENàÄ -ö;Úãã$÷v(}Ä‚®6±oÎwÒ çÒEw¶ñ1ÛÒãÏ9H÷+þY Èu|û»²W±ZÄ!vXõ¤ã?³Î—ãøჰ½Má:þç Ê“‘œ øÌž·áó"Zƒ×Ñ^šŽÓŽOÛÞŠßîß,õºc|¿‹”„¡$#M±À×Øá -ò‘Éx&Ó ®|JÒm’ó>»ùŠ&í}(øÂPmÅŽmR¿ùóûîsRßâ·ÚàâóÇ=S1:›{ø>·ÙÊÙVû깕3y‰{ÏP6ã2É{™¶ghÛ^&-}å=K£Å–·¯ÎÃÿá’·#l޾Ƥ\BîVí[§ 6žv$bÇߢ܇›·âë#5··Ã>ŒRû cdß!Û÷··É–›½ü‡m¢Þ_¸i°g´eu{¹ƒ6qýfŠï/ör:bÕŸ)6á~I£-l>ñ$»–Jhç’’í—m4.©_ÆÔŸÆê§^Í]•z!ÙŽo;BÔÌüPlOOw±½ Uƒìwy˜‚¦ álPײ³ÁYñ£bZ„é¡–î¦Ø2º^6ŽË6æÂaú:t¼L/Ë2O ï.èVÎÒ{Wj¯eô¦úèF2YŒjCšzÉ1!:æÉÝ]áWÔ›VLæÐ/³f_nÝ„µ;’Žr­—sDw·ÜÞZ3UF©:«¶>òÓ”éêRÇÀ;²mŒvi±7>—5BÒ/âl3Ðçšêâ7ñðî’t¬Ä)¾¿¤â”N“”ºë­BYÚâJÓ_¦´Ô>TXš]f_ Øÿ7T¤!5òl¬Ã¥ìë½y7;±µ9ÄÇAZåyPG½`-³ì2\\ÊðÙu¼—8å º£ô±ƒt²E^Z7Ï 5àqóðº!6„õñ¬\…Ô@wSœ†+ßÄÓÎ4–j¢ÑYæÅÝÓhwŠó`çý*.cÆÙ¾«öŽvŸï»mcÓ³Ø\¿öN¦8ÒØífg‹p(LñÄèý&î«uÐÉÞ?í—s‘y;îÝåç˶ÍEIÑç,l¯­Ñ_ÿ‡2ßâ÷ß½~׿ëºíã‘-â`ëA¼îdö¢j}¦Ø^\í±laÔûSÌu–MÝÆ>ãÂkÕi‚½—äAPÚU˜æ.â8\7²,#“žÞFq2¿vnæ†}»•þpvÆdß’v«p³>nëÔÉÜ’¯K“¸³¢sK‡VãR7k£¸]:lÓ²ƒ™æk'cèÍ~]§B¬¯åf #ìV¦Nç6»6ÝeÛS,EËz³6¬¼vÚö8FòòúÕ󾛜û -0é@ûÞ¯Òñ`&Àó¥²ÝæãíØsäj7}ñÎmɲöd9siÌÐB7«.Ò:«íÒÚ©vÙ¡Ó¾Q \@F]6Z,ݦ¼Wëò=quG©Œ˜bJ^G:­³21­³’”µvûŽ7¢%PZݹ̧£YrÄ’Õ‘åµöê,)]ö°o=öÝwÛëò: É32¥Y-=(ùIüû”µs«s:=ðJõŸíDƒë|;µr‘(ÛüPÍ Q{g³h]Ž8^\çé3‡'ýÌcí ÓT„ -ÛŸnù&ÜCÑ>®XšêÖåWkä>ÒËàÑ×l*ìDg{wÿöƒq[,U´?Õ z«½a\ˆÝ#žíÌ.ŽO¶ŸÿÒ³ -endstream -endobj -591 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 590 0 R -/Resources 4 0 R -/Annots [ 592 0 R 593 0 R 594 0 R 595 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -592 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb12-20) ->> -endobj -593 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb12-21) ->> -endobj -594 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb12-22) ->> -endobj -595 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb12-23) ->> -endobj -596 0 obj -<< -/Title (RKE Hardening Guide with CIS v1.6 Benchmark) -/Dest [ 6 0 R /XYZ 85.009843 502.818898 0 ] -/Count 14 -/First 597 0 R -/Last 608 0 R -/Parent 611 0 R ->> -endobj -597 0 obj -<< -/Title (Overview) -/Dest [ 30 0 R /XYZ 84.259843 272.631496 0 ] -/Count 1 -/First 598 0 R -/Last 598 0 R -/Parent 596 0 R -/Next 599 0 R ->> -endobj -598 0 obj -<< -/Title (Known Issues) -/Dest [ 45 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Parent 597 0 R ->> -endobj -599 0 obj -<< -/Title (Configure Kernel Runtime Parameters) -/Dest [ 45 0 R /XYZ 84.259843 539.222079 0 ] -/Count 0 -/Prev 597 0 R -/Parent 596 0 R -/Next 600 0 R ->> -endobj -600 0 obj -<< -/Title (Configure etcd user and group) -/Dest [ 45 0 R /XYZ 84.259843 283.686971 0 ] -/Count 1 -/Prev 599 0 R -/First 601 0 R -/Last 601 0 R -/Parent 596 0 R -/Next 602 0 R ->> -endobj -601 0 obj -<< -/Title (Create etcd user and group) -/Dest [ 45 0 R /XYZ 84.259843 169.788971 0 ] -/Count 0 -/Parent 600 0 R ->> -endobj -602 0 obj -<< -/Title (Configure default Service Account) -/Dest [ 53 0 R /XYZ 84.259843 561.590928 0 ] -/Count 1 -/Prev 600 0 R -/First 603 0 R -/Last 603 0 R -/Parent 596 0 R -/Next 604 0 R ->> -endobj -603 0 obj -<< -/Title (Set automountServiceAccountToken to false for default service accounts) -/Dest [ 53 0 R /XYZ 84.259843 522.092928 0 ] -/Count 0 -/Parent 602 0 R ->> -endobj -604 0 obj -<< -/Title (Configure Network Policy) -/Dest [ 67 0 R /XYZ 84.259843 599.840928 0 ] -/Count 1 -/Prev 602 0 R -/First 605 0 R -/Last 605 0 R -/Parent 596 0 R -/Next 606 0 R ->> -endobj -605 0 obj -<< -/Title (Ensure that all Namespaces have Network Policies defined) -/Dest [ 67 0 R /XYZ 84.259843 564.092928 0 ] -/Count 0 -/Parent 604 0 R ->> -endobj -606 0 obj -<< -/Title (Reference Hardened RKE cluster.yml Configuration) -/Dest [ 77 0 R /XYZ 84.259843 191.441066 0 ] -/Count 0 -/Prev 604 0 R -/Parent 596 0 R -/Next 607 0 R ->> -endobj -607 0 obj -<< -/Title (Reference Hardened RKE Template Configuration) -/Dest [ 390 0 R /XYZ 84.259843 518.211778 0 ] -/Count 0 -/Prev 606 0 R -/Parent 596 0 R -/Next 608 0 R ->> -endobj -608 0 obj -<< -/Title (Reference Hardened cloud-config Configuration) -/Dest [ 534 0 R /XYZ 84.259843 537.158799 0 ] -/Count 2 -/Prev 607 0 R -/First 609 0 R -/Last 610 0 R -/Parent 596 0 R ->> -endobj -609 0 obj -<< -/Title (Reference Hardened cloud-config for SUSE Linux Enterprise Server 15 \(SLES 15\) and openSUSE Leap 15) -/Dest [ 534 0 R /XYZ 84.259843 397.010799 0 ] -/Count 0 -/Parent 608 0 R -/Next 610 0 R ->> -endobj -610 0 obj -<< -/Title (Reference Hardened cloud-config for Red Hat Enterprise Linux 8 \(RHEL 8\) and Ubuntu 20.04 LTS) -/Dest [ 557 0 R /XYZ 84.259843 511.863778 0 ] -/Count 0 -/Prev 609 0 R -/Parent 608 0 R ->> -endobj -611 0 obj -<< -/Count 15 -/First 596 0 R -/Last 596 0 R ->> -endobj -612 0 obj -<< -/Length1 10704 -/Filter /FlateDecode -/Length 7088 ->> -stream -xœÕz xTE¶ð©:·nß^Òéît'„ìKga дaS ëŠ&„MH ,b` -„ - ‹#"$€#‚ˆè¼qG‰¢cDfu&&•ÿÜÛ ßÌüÎ|ßÿ¿ïÝ“[ë©SU§ÎV ‚(Q %;{äÝ«'ìÝï¦ÖðýúgófDª§P}ꀜ{Ft}63ƒê›pû€£ú&nÏz ÉGýUwß1r 9+m Õߧú÷÷Œèž:ío/.`µT=qz^1~¢¢± ¨¾}âÜÙÑ0%"ÀÒƒpdañ¤é3{ÌJ‹‰¦þ=“òJŠÁDñÕm“œ_ø×{ºeQ±À“0¹ /?ºcî(Û‡ú3&Sƒý¤JõÙTŸ<}öCkw9ßàj{ðÁ¢‰y û,™Gãi>¨ž÷P±²’@}¨=#ozAâ7·ˆ¥ý³ ÅE%³[>„qq_êýų -Š{›þLEåm1tÞñûkÿðùû|Qú2>øfK¹žÿa{è‹ÐSŽ6E¨:?5ààhœiºŒ ô,ôl¹hŠ0(µî5Zî…d@£ÎÁ‰(짡ØÙj´Ž4±‘ª‘þ…ÜE¬*¢¦p®øçûéÉ)ìŸ>ˆ†yª[ºÙ&Ótv9XË'-z¯r -Û0ù -ø÷a[Y„À§?臨Uÿ=±Έ*(1Êv8£çj”¨ -œá÷ùë"ËŸÿÃñ«h¼Êÿ9Ûå -”y!Ì$3•˜iФ²pÁaž 'ŒþmpXÏéDëíÊç~<ýÁÁTï E=©oŸrnSWÀhC"ôvSÊŒ\Õ”G(FÞ n ¸úà #  -` -Lƒ"˜×ÒÒ£/a Œá„1&Æt˜ÕÒÒrù_Á“»ÿö2©BAΠK´£†{å×ø†‰~slL×.?«F;LÑ5S0?º¶¥%gŒ&ÆÖˆðôj5Š7îÓÖùi×.CrÆD×4÷ï×Jµn?j1†Šzš©½?£OŸ´FxéoPnMôÄÉÑ9‹ëõ˜£ WW¨å‹|-?Jltãß½ø·Tü¡ -¿·ãw¯Kü«ÿbÇ?Wá5/~ûØâ[‰W«ð›*lhįñO¿ê…_öÅ+¿HÅÏëGˆÏ«°žëGàåϺ‹ËøYwüTâ%~’ŠpãÇUø‘Ä]øÁB¼tß—ø¡¿·/^ ..Ä ðüïÂÄy‰¿ Ãÿ’ø[‰ïJ|Gâ¹*|ûl¤x[âÙH|+ÏH<µÔ)N…ãÁxRâë#ñ„Äã_“xLâ«J<"ñ°ë–yEÄÚCGD­ÄCÇ‹CGðÐ"åà+^qp¼¯ú”W¼x@âËU¸_âKk$¾(q_>¾`ǽ{¼bo>îÙí{¼¸Û…ÏÓ¢ŸoÄ]Ÿ“¸Sân—øì6»x6·Ùñ™|¬&”ê*Ü*qËÓ6±EâÓ6ÜüT¨ØœOmrˆ§Bq“7ZpƒÄõUb½Äª¬¤A•U¸n­]¬Kµv\ÓˆO®>"ž”¸zÕx±ú®^¤¬zÂ+VÇU>å />.qåŠnb¥ÄÝð1Úæcw`År«¨pãr+–SCy>.#N-óâR'.‘øè#Nñ¨ÄGœ¸Xâ"‰e}-¿^¸PüZâÂ…¸ KGzD©–8_âCvœgùœ#qv#–4â¬FœÙˆÅ‹$Îø` N“8ÕÙWLS$N^ˆ“¨R(±@b¾Ä‰'HÌë…¹x¿ ÇK¼Wâ8‰cÇXÄØFcÁ_‡Š_¥âh‰£hæQ}q¤G0‡Ñ‡»qØà 1LbŽï‘8ôn‡*ñnÞ%qõ ‘8xC ÂAbà‰ÙUØ¿ -ûI¼“ww6bß#xÇôIÌ’xûm.q»oë(nsaŸÞ¢¯%{`/‰™oíé·6bÏ ‡èéÆŒt«Èp`º{DbZ¦Þb©o±bJw«H ÀîVìÖÕ,º9°«»¤bçN^Ñ9;%»D'/&»0)Ñ+’îÀD/&x­"!½VŒ—'16chŸ1.ŒÎǨFŒ¤-DæcD†Ã%†5bǾJ•P‰ò1„8"1˜‡¢G¢[bD!¸$:i¯Î¾èXˆùh—` m„m F«D‹Í5BÓ$šÜ¨æ£B -I€©%Y-‡à]‘9$²Z–¿ôqÖùÃÿÓ ø—OK‡Z8KpvÃf¶“jzÌ0“Zªù~X -s¨åuv–Uð®Ô¶®ÁyÂ,‡³¸[6Ò¨à’àp„D#“¹Y¦‰º2T9  Wj•+‰ôTJ”sJ®RÂÒp›-vÒ›‰oP|s¢ –}B1Áaü -Óð¨ÒO±Ã'xwÃç4‹î;ÏR”±¼w-¸Y”ñR>œZN‹s°‰ ˆúϱ-ì<­î0{.ÂTø@ØÂ.Ò¾ÎÂ÷ð(ŽäeäžÓx!­ÿ4Ñ:Gã7A‰â"³€ä©VOsM0Òì*.p Êhæ‘°]­Uݦ8šEçØNö:kP×B5œÇûp&~È–*qÊ.e ¬òssaÑÞ¤Q Ù|Ú»¥:u>OÉe»á+%×4h¿¡ïˆæ<À‡ =f;Jï<ÕA{êÍ–b­Tï€s¦ÁJwOL i×@qH:L¥R)ìƒýЫ`Q2ö«ößÓÈÍʧ´çUìqþ=œÃ~…*Wõø‹Âت_8¹ rþc`v6–`9Á‡ÜÌ»o8OýsPî#xNù«OpB4«^‚§ ¤©—i…¿Ñfjû´kæÅ–µíP\Êö¤hËyå,:(Ž¦†[ {CJÏ4O܇çÏKIx³*ãÄE°€Ïçé‹Õ -¯‹MPmÖ¢Ôp„(fu\R8rL!ûnÛpÒO$õzÃ…†Š]ÆƲ¨ðñ=cœ"Ý›æŒñÄH6Xndo±ÁMÛw+%k6^ÜM1ખËJÉŒâaŒ/>H…€e6¨V«Ãƒw8ªm±O†¯ôÚbÍá¡‘Aáæ¥E7\o¨¿õ×ë›êWupº2S|nÒLvŽŸÃsÊYqV5mÜÉdzñ,Võ¸ƒÓR3Ò{$0OBB\,ÇŒŒ´Tð¸!.œˆI æÛ—oݺœ^f¾ë©»Þ<Ø{ÿ´O™×>“Íò*Ëaaw=…½o{æÈ‘g¶æókãä_ä·¿/¿ýú ù'ö8›Ï&°‘zTKñ¸Ò@<ÔÀIQ꾸P¨´˜+]‹Y¥å…(§UãA¡QìáÁ"4¼›Â]J m+µ‰Xè ÉÌÌÔJ›t\ÍLÙËh Îcùq7 -^Z³Ç­š‚)UâbØZÖïÙ§Ÿ~Ve×=ùä:iåÊ•ÆE *wÈk?6ÉÏ4\¾båR^(o/š5³xçñ—*¶¹£Ïnxó:Æ’–Ë"‘N!2|ž±ï³T:Ù3®W†<é\ÙÑ)nGG}‰ ßµ}5å@`XT§åé\mådFOªÞ`«H,¼òH ÈkÌÁà‘+…S¿Y"_³elIJoÄ„‹Ü/OË÷å%yúþÎȶ²Il2Û:€V¥sqqÑ -Ý|µRá•°X«T^°f6a8(6}ANž¼Á­”ýQĬ 8gë{kš;òÓÍ™üoM·‹‹»eöîæË»[÷¼‚ö™¾ha¬Ã*5×3Î}žJû“ÚÊHáÎJZ‡P«#’&jªo:ycïòÂUgffŠ×“ãT•¶Ý*!íù ¼.r×ùEµÜ&ç°ìþ5ÌTTÜ´B^•ß° 暶ë"{rgsÙˆQl#›Îf°³ÿ@®|Gþ—ü|Çëçèmð ‹Ï­UòXlQi÷âV3kcý úÔS!eŽÁR:gš3ŽÒ¸3oñ?¼õVsì[âbófžßØYçøi³µD¡Û!x‘ëäÈ" ©qèº †nÖùR„Oäˆ\±JlªN¨½ÆέtÄTÃb|ëKÒœB&§ª -t*BáN!”¾šàˆð¢YLQ5}"«ÿ蜺¤÷iH ÉÔ-‡É¡|crˆÖWùflì~ i7¾5nÎnÖ‚y’HÒzò ÑCÀ³ÅÚ(>‰ÏåóÄ£|¹X¥­ãOiW¸GE³0«aj´˜:`’è¬v2e("CM7¥Øî@ŸÒ_øTŸÉg›€¹Êd1É4OÛVà -ñ„ºÊ´Ê¶ É–šâ+¦7ð Óïñ=Ó—ø•ò¥ø“úþMü]í2~&ŒŸIŒ`1Œþ .oaJsv”ß7§é¼®àóš6]æï6ߢßvu.$.A8¼äK7k&´¨NTP8ûRàAÅSivW,¶*BE'Ù…`»°„†*Î,·%ܦD‡}R7N¿‘è£]W¦~ú™GÄ;_¤Î»¾1‚ ®¢Iñ€‡¹y0†(^ð2/OÀD5Á” %˜£#3XÏfÙ|²˜£Ìó‚–«ËMÔ ¦¨ñ0ž‡Å!Iwœ.UѺéaN¿IÃÇï(½ýܥׯx裷؛ šm®k*+×ð£Á«-'³²ª Íââ{ï?~˜ßÓ|µüÑG—ê’SNÖé`"üÚ×'ÀÆíV©™¹É£¢"ûZ¬‘QŠ‡ç÷º•N¥ÖyÉ %EZ¬Qa&ˆ µw5…ºc“lhj¨'/`ðE÷ †rœr¶‰–]çJkFÌ L&æŒJîž|O2úí—á%BB ƒ›˜Üê3Ò{ħ¥+KÞ~`ÇËóv>üÙïåÇòÊÔo•6Ìzáhù¦ÒÏÞb!ßMù@l£gÆ¢¹ ¢B;_:xé)ÝÛ?{ù¯g,ˆêÐõøžSõ º,”Ò¾»Rlc/õ%†FYCÌvx>D­³;£—E¯‹«u® ±Av0kÖ(ÔÜýèô߾Кê4Žþdýõ&²D§tçGâ û¿))‘)Q)Ñ)1)±Y‰¾_¤/Êí‹ñÅæDäDæDåDçÄäÄæ$'.(,*.Y»:±:ñZbdÛжAmr#s£r£scŠ#‹£Š£‹cE.ŠZ½(¦ÃMž5½§3.ì~,q*#-F‹XÕ”®»Ù`~쓽‹‹6ÖÕÖf]¾÷lóŒ?·>÷àÈ‚cãþz§–N(¹t ù®æÅ» óNl{õ¸«lE·n»›t™Ùr¯¯B!ËËØrž,`¹¥Î©Ô…ÔêÎÉÝýÉ9Õ·9'‡¼~ÕñÝÕŸ50̶(luXu˜`í9­ÕIŶ:)¼2ô霗Oz9çé¡wïß,ßc]™:j›’¾·sçËçÎ]îÜyw|<»B6ëG'H«RÆ©np6ßê íXvwÐVÚkÙzR.Ðø§ËÚ_×Øëtdº­¿~²ÁqòjÊÁÜÈE‘Õ‘¨;õ´V¦q§ÃEŒbmQ --·ÕÖöziÁÙ––³ ^j>ýÜš5»v­Yóä÷ÿ½aW~ëÇ4‚~yÒsöÊ•³ôBëºÊˆ[nƒb_zóôG}-?“_ED¾Ü¥Ó«¯Ý;qëÅ)¬×„‰»uI>LñL>ÉLôð…¢ÐÎÔr»³Ö¶Þ¸CuMÏvë6þzƒáiÝNÙ¯ý¹#Œsú—L…4§¡YJ~í‚•{ëêú¾<çÄ)¾½ù>¾eë–cÛ›ËUwó–‚üouþ Éç«úï¬&è쳫ǔ—à(LS [s4õ1¢‡zŠ|V‡ÙgÎ1çš‹ÍÂ"h"O܉Zz”Ü«U÷WmûØLô,d»’ÿ¹¿? -ëÛ¾Ù7;üúêð6Ýi= p yûDž,:k£y!Ÿ¤•—„Wˆ'´µ¼J¬×žå.ÝËs+yø$LTtß™üùd̵UàR¥Bõ×ûœlH5Ó”Ÿ_¿:¾šàµøOâ[âÍtºÆyzÚŸñO‡íñvrÿã¼x¬nÖœU;ëfÍ{|g]]VÍü‡÷`Å‚¹ß}¦ý3›õ£ç[¶=õÚ³ÍåJî¾IÜ<ÚAÅ÷7IÞÑ,yõm’w ×ó®‡ÿ\ö<ÿÙ£‰uÑó[£9†æ†æ©u.¨³ÕºVv0»‡¡ËÓ¿ƒa|Z­6츬ÐR(UËLeZ™¹ÌRf-µ•”ÙËËeÎRWuèµPçÍ÷¹ÄTÝÎø-9/Y·wOåÚ½{×^c.yõÚŸå·Ì‰Ÿ\9sæÊ—ožþj³|S6ÈoÈôd’…q³[ýœÁÁ´B'ÐõQµÒIZ±Ü^k>j²¨hÙ.Ý”úA6òÂÛºQ<´5HçIÂÏ‚ƒ£uÙü\]]¯ÃKƒº…ã—óì±æýωBÐlEäËNÓl‰p¥5ÞÑîŒø)Ü!W¡¸—y*:è>Î[ûS¼3CZæÓYÌe¢œoäÄ:ËQ~T¼ÃOã»"Bp3ªŠUX4«™2›‡‡b°ÒQ„iaf·ÕcÓïƒq<c¯ˆUcM^-Ño‰±ÆÙ2‘nÎZ¦-ÅžÍb¶âSúúïÎZ?s?K?«Ïî³ë§8šç(ÃÄpu¸)Gaieù¬€OÅeª˜ªN5Í0çY'ÙŠìs`›ÏâCÊB:Ý2õaS™é!m¾¹Ì\j™k]h+×oóöõ°ž­ãkq³ò”Ðï¡5_÷*ÛVûNØɶóí¸GÙ#žWŸ7íѶÛ^´¿Â_ÂW•#¢Öüšý$ßVÞó ‰2þXbÜèÚ/>¿ôÅçµòÃKþË%’*œª¿?VcUÓTã×[ŒÓÿ#­˜I©RŒÿQëeÁTó—9h,»µŒíÚ•veØÐÖ² -nVwBÃ|˜S`L†ÙMñìDH¦<RÒ¨40¢¡/á̆zgAäÁtèB­ƒ`áw£Òð A4 ¿A«Ä¨P^@cæRšO˜–_0kÆYGÒLsi.ýò„­¯#Æü{3ö£ÒT7æÆDÂÍ3¨#òŒE•”΢;…ð¢i|ÍžgôýœÎƒJ ­ˆî0ZõYK·È ”Js§‘h?ªmŒÿ;hy_ÿžå>\ÿB¥¥ÅÀ4¾ ø ¼ÈøN£í{#×{l7F¶þå‚“€¿ø3º£†Rê…J“t¢“c´¶n”¦¯­°¥Ù0ÒÁp¥C ä7Œ¢5²ÖUXhTàý²ï öófπƗ`|ýÐV2°§Ìš¡#Ó:V1¾®`ú®ø}:&_Àß;xZ9ñ‘þêéÿïçß›³euûQ-›Zk?ϵ+Ÿ7Òí­µŸåÿ¯ŸÖuúi]F~â?£ó‹0ûÝT;O§z¾õtÏßL¥%ï?£ù/F¥Ëz'èL’ªK÷@’äa$Ç£`,É1‡êM0´ Õ: ¡ -a§“Ì PiÔ@²ººh4z˜‰Âp’þQV¢4–´nAÑiÕ7òVTÓ©£A]Ô•vÔ•V꺾™Hß²‰ú³1Ÿ…toQ¿‹ÀFº—Ce}îcn»1w Y‘Ñà€_8a ËXM±w«Ž®uà6tôá¼Yy`IÞ¬é3`É„YyS`Éļ%”N.˜EéüYÂ’IETž4«`,™œ7ƒp&L –iy3ò`ɃyEÑzJº¾dzÞìÉ°dÆ4½¥hRÞtX2kÎ Âœ]8c¥“uúíìAûsÑ×õGø’vÕǦõFþ™Œ\qѤ—Û?mu6¹µ<ê€èÎý’2ÎkØ߶r;7µkWŸõS™ß -ð£n“ûÜšD»´ÜØ1ürcS -endstream -endobj -613 0 obj -<< -/Length1 6944 -/Filter /FlateDecode -/Length 4153 ->> -stream -xœíYyxSו¿÷>Y2‹^dØ,KX€ñ¦'É»,[²v¯’ñ"y‘w[ÞmpÌbÌR¦Ð´$L“&¡”5àIø†¥_L iBšÌ4ÌŒÓL ’&:i:S:´CÀOs’Οó}3ïù¾ûî½çÞsÎïž{Îy2¡ùh1Ho..r78Ö#$‡Þ³³,o…EéƒöŸ \äLRU „!„-ЮkèôöÄž—ïF(d>´w¶t 5÷6°"$2">ÞÚäm” -E7aìm+t„Ê™=0_mEkçÀÃ}öch"Ä4vt7xÏ|ñÚË°Þ£Ð÷R§÷á\€:à}•§ËÛÙ$W¤³-Ž‚¾§{ºûü¯ È÷<GT:>?¦­]õ_ˆa>‡ôÁ±¿;Gë->å¿ÁÝbÎ1Ç¡)B.˜'rGP£@è¿B™süJ3¯Ë|Ïe¤œÁ< e9B_#Ú0Nü( -ÆR}ö@kŠG‹¡i~Ó—Õe*B¿D11±AºI BKLj‚œá5à¿Ì˜>}ûuÉñ[¨Šlªï-?G”(ý­åW@:¬Gq$±¤¥õhôYÉ&(±HFQÙ€¬èýýƒ.rèšP#@ñä<* ௠=Ú/¡t -¥+‡þçƒï¯ÁxŒ‚í:˜gGédŠ%G±c(šG’ïÐnšÿ1Xç̯åëxÒâ±Iø÷ž¿}å¿A˜€,!ÐŽð%0èðz¨ç¢$ܤÐo!)h>£FóñoQaÑüÿ‰ÿýï¾ðf´J ÿþSäü6:8ZAa DÁóµõ@½-„žPXg5J„“dêbTŽ†b–ÅÄúýˆžQ%œ–d”‹ì¨•"oÌR:â¿vÿýÉ Ÿœú†ùkWZƒ,ü½Õ ¸¡Sè"ŽÂɸïǧðÂ’œYw%yŒ¼F>#_1ó=SÉ4Ã=Âì…ûæ"s›¹-X.¨ô -ö N~õ€ûnHJˆ'd_È[!ׄ«…a›p¯ð„ð%á¢y¢,QŸhTtNÄ…. Í - íÝzðÿÀýBè¯à¾2ëþvSNÊÐ&f)µH‘\¹i~N³ôí·a¤ÃoÂÏ@¼Y€P4Ѩ#RY±ˆ£"$bù%_o[mO¯¯¥ÖƒcGOâxïœ6îòÁýÜïÚçÀL̬ -Ì”h´5QjØèq%·øz{jÛz}ugæ´céþƒXÙ6Ç˽r”û¸ ÝÏù{Ñsh˜Ê“ÊÈ#ŸëɶsÆal5[ð?âÇÁÖ?ø#EkýwÐq´AT‰ŽÓ¨µ¬*Z%”+ÒÅrùb¡B*UÐBO‡ò1Ð.A(b’46N£ˆ‹“Ç -…â¨èhV¥Õjt=ü §çôà³Áh–HÌFƒYB–½þúÉSo¼qª|Èf*¯Úh³m¬âÀÆ?#ïÑ ©aÃåáqg=dÄ9‘~#…’(ò œB¤ )'‘\ä.ÉÄ2Í 6䘌v¦B0ÒÔ2Ô0\+¸§vìÍϵ—™{6ö5: 9r©¾)þ»d!ùÅ5 ‹ˆÄ|EUQ²¼ -”\Ãjµ“ü #ÍÆÖœžëP·ZÛ¿Z!r™LÖÆ&É"OM¾^çU’¡Ö—FW²°Æ‘îVw7g9¬ö<ÎÔiôz-ày © Ð\8Ñ(UƒÙpV,Ë5‘X¶ -×v66V è6ÎÇ¿ãE‡ã Ü*³"x:ùWêÇ°L¬û>y©˜á22;,©R;[´Ý¥mÛª sé‹Š‹ 5Eëð‹dçõ˜ÕùÙ–¾¼Žªt}maa»? ²Q:ò ¼v“LA lâ4 eªh‰ü!0ÞI~øý•XâËkÙ¤ñ9Z1ªï™ëzÑ‘TQjr'vPgŽô;ôž#ƒ¸»½äq¹™uÔ–Û -+©V¡à»ä]šGa,Ç2 ßå¾ð`)7Œ?$L …ÔgùÿdÕ- Ù_µÒ©‹ž–P­MUP‚ -…Ó¢îÉÒú_ªN«³ê×;ýz‹/(6å;ó ÎOjÜU;þ‘+K[βåÚŒrqd­!׫Ñxs uäåéÒŒùiçóóõ……zƒ™ÇËOÔ`GRj½ZÊH9%Ç4nÊ)¢Â¼jÕ9‹ˆ¸>ËЖµ¹ï‘Þ_us_¨k|Ëar–îÖm:Mm·ûvüü¥ácëºÊö9ÍvO…ÅZAO.R!C‹Eáˆæ¯ðÄ'AÀ‘l#/CÏ\„VÊDòH6R®dü{n<­î^ÛݺÒááýûÉËö¸Ž;D1… ¤ƒ5¤H«a”…OâºròD*1‹ÇsVj[Lܽ;æF•!•ãžÒ›Ly?¹á(6üì0öªlk4e)d,©X¥+‰X™¥ÑdM|¤OÏ0ܦ{ï¿C~MÞ@Ëè©£øHä³\F*/ÞsÐÛ÷¬²sY§Ã:—7`uÀ«òÙ>ïA>ÔÓÕ”c4nq9· 9î#Í ?ÕåIÐe.ŠœÖDÎ%gñ“'*+Oô]¾ü7?Ú»kx˜Œ•ýÌç;à$cûönÛ¾oâ*•VÁoB¾ ~0RL%ø•p™x”ãðRn7$Ý¿·ÿÉNÎ؃´—€ã¨¨ÁgáKÜ»·o»É˜í6î- hý[ò:oé³Õc?D‚æÂãAª—uY­ë †õÖæó¸/ð`¨ç ÷Puõ!o÷sJ|3Û`ØRZºÙ°±ÕfZ·× º7uw5e'ÍÀdÇ ‰ - OÒìæ¸Ý¸45ã,îM2Æý3Vg Q˜A#Pº) /3]KÇ£‰d˜_³ã·¹Oa¥˜~îMœƒuÜ°Ô«ØÀåŽñø)Ìÿ Œhô/Üi·Í]Á{Y@K&ç—Äá,éusû`Å.7‰ŸxŸœ™¸HÖÀ^Æû¿&ÿˆ-@+à«,næQÊyï? #^ŃSÈã9TO‘°Ù(~"÷QŠûHKËw‘qØéÚÂÍ°1h-+xý"j(rjôb!X 5²bpã.÷§6=cùñ®]ø‰~î#2ÖWµ5/{›篆ùÑ/œÎŸ™H˜á¯u“-“I~åÄUB3}ÿ îú?S2{f$€”?ÿr¹T -%ÁfÖÄÅ­˜±ÎÄ|«±`¹‚<5Yï‚.4¢NF*([/¹Ýïm…ðÊܺü™»ÉýÇtlº>#6]ŸX -m~„9G'3–™¨LÃÈiöÊÄ4fT•ä›]ür§æ·K%í¹3ðr¯ã~3YEQ|¢.+C¦v 1S|“>»yÿj ꛿¢GŸZ|ØË¢¿;–ÝÜb¶[í#…)ë4jWŠÊ©J)M™öíÚ–Ÿ?RV¶ÍTœèÉËó$$xò žÞq÷À%<ÈñrÞïÎû‚6iò-ý†úcNÀå$q?aV5uÎò@ôtÜ{ g ÿ_aö-öx¤˃j:ù,QÜû‡’土ËZ„/b7Ÿ‹u¤<••@n®Ëm‚/^»vÚ_ PÕOQ±" ÉÝMuúšêrÉO…‘½‡­ø¨¬Ú^ëi³™U[,ê÷ 2›Õ¬™Ò:°_ÁS?Y¼_ázñãü·UýP+:Âÿ&Hß1ä|G‚ïÞŸ¾Ã*:| åèNð=ÍÅ‹‚ïB´Ç"êF=hõ¡6Ô‚ZÑÄBJæK rAOÔÅ~|¿ÒÌèG»’`í6žb=ªG‰0«uB/ík†÷õ°n#Ì‚¾Ù¼Vß×.ž-@ß¼@o M¨e>@ÆÙ””B œ“áVƒ i…~šv6]Â}t_gáò¿Lÿïó |¡ÌEKa°Ñäp¡P$àéý³Æ×?†jlÚ BËáé…9³iáúo/;Ðe -endstream -endobj -614 0 obj -<< -/Length1 7204 -/Filter /FlateDecode -/Length 4875 ->> -stream -xœÍ9kXUוkŸuö¹ï÷Áå}.y\¸€¢^ j  A#§òˆH Q¢V1iK'!ÄCmjhjRÔ4ôkÓ6µ~ÑIk-ÓvZ˜~6“8Ì:ç¢1™t:ýúcæ컟kíµ×^{íµÖ`B…ž¥KKVõV>×QD£‘Ë -—,†…—©ß@ýµËŠî*.ܱò>꟣¾gYñºÅEY[>¢þ¯©¿ø®bŸ¿ÑrÁÀ©¿~KcE –Šfl§þ¾-;Ú=§—{6€î ÂÙWÓRÛ¸=;o+Á‰|³¶¢­4” ¢úÆچΚ‰ß‡¶~ `¨«®¨Šz¬üÍM%øœ:0p%õ«¨_×ؾ³+ßBøuY{Có–Š%Ë?¤ù8ÛX±³ELdì ÂSEcõ¬èyIa÷+<·4·µO_†{hýxÞÒZݲrÍSY$œã”?EV½ÃÃ÷žÝl)ø3ÄhAùÞ}¿ÿ€RÿêxØór”xAê&\­Ê†òÑ5_Kù9Ê•”û‚073ÖLy€rÑmóçAPA‘²ƒJ¦Ö"´Pí¦sG:÷tȇEP›¡j ž`; szš0ÒTH± -ÙBmЪ@¦ßû¼ôßôìÖ÷²Fâ" - R=Ö!Á»¼j(°¦Ôó£ ±i©Ÿéz¬Ï ™:=ÃÓÓE¥bß0Ä#‡Ð«½q£ 8š–º¢¨Ô3tnIá Õ%å…4V\JM¥GÃ4¾¤P…)‹q/ý–—y¶Ôy²>—ÿµ:? †…=öWegÇoÊxò&~r¿aÂÁgæƒ2žŸ=±?û0>»G<ñL?±OÄgð¸Œ_÷ã1'>=°Œ?-ãÓÃÓÓâÀS‹øÀ2|jö?éçýÛðI?>!ã?OàÑ |\ÆÇBðk^ìsã—eü'•±WÆžj+ïéÂGŽ8ø#.<ò°•qàÃV<\Íûñ!#:x†’ñà2~ð Ü#ø’—(ÃñK^Ü¿ÏÂ÷‡á> îíÃeì&zÝ~ܳÛÎ÷Tán;>Ї]Ux1î"Š»üøEBø¢;ݸ“vVa‡Œ;dlo3ñ6¶nOç­2nOÇ‚¶ta³›Ïð&ÊxãlÜ#6Ô{yC6ÄmÛp+u¶žÁúÀ´ëhN]ÖRU[…5´RM4V[±JÆ-2VÊXÑ…å2Þ[…›îqðM2ÞãÀ~ÜÂ7¸°Ô…wC¿[Æõ~\'cI±—X±Ø€k׸øÚ>\Sdåk\XdÅ»d\½ªŠ¯îÃUU¸RÆ2Þ¹ÜÎïìÃåvüÂ.›À¥]¸¤ÐΗta¡ïpã"3.ôãÎ/0óù2Ì3ð3Î3`~^(Ïïü¹žŠy{Ĺœsç¸yn.Î!ææ¸1'ÛÍsV`v–ƒg»1ËþL7÷Wa&3ݘás\<£}nL§¡ôhLKµñ´L˜šbá©6L€z)³Í<Å‚)ÃìõfKz>ÛŒ³‡ÙžÀÝbò -L²`â -Lˆwó„xôRå]ñ$šx7Æ9pV¤ƒÏòc¬'›Ç.@{²1&ÂÁcÂ1šªh£%Ê‘Œ·óª”»¡ÃÃl<ÜŽáAžÂ´fÃ0…§»E·CW ‹È¹úЙ‹Ž\´“÷°O`ˆmV?·M • V?ZÌ¥ÜÒ…fê™KÑDdLÑh4X¸1 ÔÓ¸VcäZ jƒ;ÔÐ5FÔ(;Ô‰×qIÒ0 œ¹ù0ƒ@˜(ö! -&ŽÑT"#,!Ù0«Ú„¥ü¿þàÿpí(–ÅâØ)v^"Lãl5l‚]x™ ‰”Ã(kÉCY¼I Ö&èÃð,Á²àTL¦¬@+oRÔî8Jõ~8Žoâ^r]èÞz€\£þ^ò}ß…ðœ…!¸Á¼4ú:ó²Lèg>¢z0}ìÓ³<¨Ä7a-q{ ŽÃ°ãL‚6ø%xFá-ç—øcpŒÊKü:lgJ¬% KMí¬&`= ¡ò5øwl >Á¾¯J5ÂQáu±\M»hƸ°IØ%…Q±œ„«’ƒÖÛ G¡™vob®¸>‚ ?O¤œÄ~$˜ˆo-¼Å/ óa—Â%qу…0Ÿ0cá¼æ˜df>Ín>N³Û§ ¿ý_„.Hý+iõ§ÒÈßN,ŸmüÜÔLéì‚vþšÕ´ˆÒn|Nœ-V‹_¯‹×y2™ ¥H÷©éçÀX;‹Ç…«SX_$ñ0­š Td°X[,K:¦ Wå+„K1šé÷%è˜^AËßX÷5¶Jþk–¥WqÅDÂÕ@xÀ(t€Ø!q¦Ak½X0æWf] γ}2—²˜(Y~Q¡¡Äyâé†zøv`Å*".j8Y.2=õ:Ôk©ãÔÔ±°JÇuZ¤F/:¡w¬7V i‹W é×l,}…¨æn Í# -®MŽ…äePÌ¡±ò74ô³jßP›3ÝàȆY,jQâf!ƒeš ­˜EQ/æ°|!_“£mc÷ {XpšÌePÆ0–QBJBÝa*ŸcõìËò;›äû6Êïˆ?œz·O¤°írB·{ú=q=i["´¸œB¨#Í«-„›ì¤¦^; ˆúèî¨æØ–p}·®ÙÔ"5…·&%ÚoÕ%Å»Bc¬®(œ•d›»f É£ŽòÚ±cÖñÇm¡3;5¿AÍ™Š¶–“Ÿåw…ÚâfIN‡+4Ôår:ÄØY ‰6—+Ë?'';!‘ÉWû™ø^íϪ7”ok-Þ·ûôIùÏòŸš¯Öí¸Òи³h¥Ð³îLÿO~ïK—}ÇÒõE Ödf,~÷Äo®æd³Ì’µ[6ÜQ¥ì•bn±ƒÎUÕ Ih«èÙbAFš129B§rcÌŸ;£¤üŽ¼_Œ¢üdY‰X•”|™tËò"FYákñÕEzá€Xˆ3:MŽÃàì1Ùê M¸U1E:]`L7»"\Rr$q7F÷*(¬1ëoÇ=PÀpÞxÞtÞ|Þy>‚—Ý’_–lÎXõîä¢*G¸ºæâùwòÛ,ú?ÿÂìS:vôñê‘m m-Õ; ¹³g“1Ž`!l)üŽ}æ/dõí»¶ÓyÒëR•£²á`°j°‡Y{Ìí:ÖËmf²CàÒê%YÖ‘É‚P]ý“$E?YåZ8ÕË”Kº–•å$v„÷X©<(_–¯3ëàµþm±Oü~åèd7î]ýã'F3HJd˜Ä#R7èÀ³N£^Û½’УoÑÀ\ô³²¤²¦rlc~:·ŸeØi!º‡ŠU§|E(d “k”GY óÊ—ÅòÉîÑQ¡ù$ÌìííÍ@¬V§å§§§WËÄ^h×KZz‡J:æ’Ò!Ǩœƒr!Û=vmÆ Ý4Ò´2™\úil±§È•—³-r);'Ÿ•ß¬’/KŽÉó¬K¾wj±`,—KnZ¢¥´¶ ¼ZÖƒíÚ½Ö ]˜U{¤ -“TÓ–§*#í‹0ËI¶‘í•ßV¯­üƒ—/Ôçö…ù"}Q¾h_ÌBíBÝBýBÃBãBÛÂ…ö…Ž…Έ2FfÆï²Ùâlê=rÞÞÈrª@á…Øš;m¾žà÷'½Ÿä÷'\¿ÖTX؂חþH¡¨à|WÉú]S™Âù]ëKº¦2ÅòöÅ«ÖÐÞŸ£½ûhïvÈ„é™À¤:Û!c¯ž .ÈULlŽC1>“cª¾1¦ìiFÀ7Q´HeDôÉW–¬¾ëù -‹‹éÛúÔ1áí©dáäëy`j·X>XVó–r§+éN7Óª…¸€]ªsBå³Õ­s…¤£Ë˜ìV-Lð[Ç3>±ÄÙ ,xa­7 „ºú;ë·îÜ9ÕËÂÞgaòïÆß—‡G^yæ™W”,c6_~]þ€Òël>­Ý'¯;hmÅÎåbn·s‡Ì­ìQì"çW­]†béüþ '×n7uÞ[{±Yr ,û̾K¦7>½oÆԱʛ¦®jXÒVTÊ?•/_“^Æ>¸eéÅß îV-]rÀav N žœ¤ãj4FŒ\±MnÐ^I·B¨¸;£{UÙá·]¾Ì¼)/t´ecJjê¢ MS“bù©òú¨HåôéRhÅDx&0ßd̆˜h­NÐè£c¢½:½¨7DLjÌyØÑ鮳1òMb—v£€Îgæj8gl3¨–Hxí–»—¯ýpòù7ôhï—¿?:5¾bÑèÓï -â([@k¨±@ùgoc«žõ•›˜KWR½“A=ðrí6uA’Ä-'œ0À¼…wݵA±ä1_­ø:; ¼3UÒY÷ðÂÞN”Õ\¤5‹hŸÝ´¦õ¯D\äŒJ‰Œ:Í(x9B/“(œg2ѧ¡õCƒ^ë“h\ Ńá8m©ÄXYÀ…|ôÀE£´Öá:qƒ¤Uâm%Ú¶ÛÅnù ùkò,ù?®‹å»éíÄÂ<¹T|Œx²@G`‘Þ ‹¤Ñ"×XÌ°˜Œ&j™Ì& ½×dÐYÁÀ{ðQ³¡×j6õ: Ak-™Ÿ•øQÝõ¦ÀB>—q%+ -ª¸'NŒËQ"‚\âÕ+üpÞ3;þÔ±±$Å%¯§@þ̲;û½:¿¹@\D&îmz;jÈG!0‘Ø„WÕ¿ +mêÛháW3m„Ùô" ¶ÅÛÚÜÌ:Ó– %Á¤‹-Ð ­PµPíD1 ¶@2Õ~È ”E­JÂðÀbÂi§÷n;aWC4ÒËÕË¡‰ðÓ©µ(yÈçݤզöª©®¦9;¨¬"L=Rk+QX÷Æ­ *µ*¦‡Ú -}Qi¢²…p*‰n=áyh~3­[¡Â>K§X¥¢PXEXM”Sn 9 Ïs s½ÊiÑoVW÷¿Yû) -7çßœvk6úÓ¿PÿŸðySWNI˜žV1Uø»šÕÿÜüßZ+ã­ò'6‹¢Ä žƒ9Ä£uSYH‰ÁR: +)1XM<2X¨ÜD‰Í¬¦'¿ ¬péÊâCTyžY'_TxBEkE%dlélm€ŒÚÖêmÑPÑìQÊö&Ò‹OÏV¿ÿ}ø -endstream -endobj -615 0 obj -<< -/Length1 16324 -/Filter /FlateDecode -/Length 11654 ->> -stream -xœÝ{y|TEöo-wëÛ{§;µ³4€!d# Kš° aß¾*‹ÃÃ/$@D0M%2 b†1ЀD@‰ Š ê€ý9€âºÝAð7¾÷Çûã}>¯oêÞª»œ:uΩs¾§î Â!ì[P0lÀ³“ö¼„P7?œêݳW>‚F(/ Ú½ÚsqÿEОíØÞCŸÈœ1ù´WB;ÐДôYŸ~˜ŠÞ íá“çLœOºÑx„hh/œ¼xaì5Ÿl@Hÿ_‘Ô©ó§ÍMf]‚ëoÁõ=Ó&.˜dØP· m˜6{ÉÔU—v2=ƒP‡NŸ2±8Ú7a.Ðo׳¦Ã ƒ—¦C»Úm¦ÏYøŒÃ‘zhC÷™=oòDݬЄŒàD㜉ÏÌ?Æ_õ2Îÿ܉s¦DÝÌ‚ñvçΟ·`áýÏÑhèÿC~}þSSæçž< -üÓ’#—WÿÕð¿¼:ÞÜåŠQÿ]øî¥R~ürgø^„X´pNš -MEcƒÿà9y‹:çxö=úë¡™ˆr‘¨µ ² j €ÄØ !œÃÏò«b†X MWàH?AS‰ ÐK:ª„þ~ù žÚ«yQ,Z.Ù™o•çà+¾tÿR—é­wrî„sD€ž†@@i€%Î[à|-”1ÐޥꛠC¹í©PJ Ü†²ÊÚà¹Êàùr(«¡¬Ü7û,z˜[h{‚ô˃4N@i¥ÏÛ§‚÷ñú'P†ï]Äï ^« žK²)Àÿ/ôG|6xÕ:vìD%èt-ƒí¸Vö£C¨Õ 9P_‹ÐZ<5¡ÕÐâ×dw]ù_D££Pì°ÇÚQ@´£Î¨3êŠ -Ñ`4‹–ß¿Ï­ ÎvGýÐàÙñ÷ïß¿òëíØ‹ö{S–DŒÚÇZöOañ>ïãE±ïŒKnÿ«f¬EŽÝ‡ï3.‰­¿p‘)ŽÜ'Fí£eŸàq_þ­‹—“Û÷\[½zÉöšÐN-‚*oÁi8ß«§v÷ºOôÀ_á„}±“§Ç®³¬s箳LÉM>HVbÇþÍcÅîQت…ýJ$`;bPÑö6dƒ½U«[´½m…½I«÷Û[ìîÁF´ZP#Åz”{U£§ÓîR ö²V—´{D­.hç©v†hg°w$£ŒÑ–åÔÏè=Fï¦ÓŸÓ;ËéO·+ÄŸýé¨pûÖHñv½½R¸u3A¼5’Þò -7è¿Lÿ}—þ˜Bÿ›Ñý>Þ°Óïªh3°ØÌhsýýsÞû·½éõkÅâõ*z­˜þ‹Ño¾Ž¿aôëHú£WgÑ+Œþó0½üpñò]úpz©Š~ÉèEF¿øÜ!~Áèçz¡Š~ö©CüŒÑO×ëÅOô“åôã\Ú¦\zžÑþ®Š1úw•žcôCFÏ®³Šg£èßB錾_EÏ”{Ä3ŒžfôÔrú£'}—Ñ[b#£Ç=Æè;ŒzGíôˆ6¼}Xl`ôí·ÆŠo¦o¯Þ:ìßKßò -‡=ô£«h}ewñMFëàPw—¾´0úz1Ý_Lÿj¢ûlt/£¯1o ý £{ý³Ö2úên“øj:Ým¢¯ì²Š¯´¥»¬ôO;“Å?-§;“éÝÁèËŒÖlkŠéö—,âöpú’…¾¨ÒmŒ¾¼ÀèV#­ÞÒA¬ftKêƒþ}U´êùÃb£Ïƒm=˜>¿RØü¬GÜ<–nö -›}ŽÑÐÞx˜>ë¡• ŒÊîtŒvƒ®×Ó -8QQLËAhåºÎJ×2ZÆh)£kJ¬âFK¬ô¿]Íè¬ùâ†ÒUŒ®|†®øýrq£¿_N—»èï]f¢K}šÑÅŒ.Zh™é¢zŒ¼„…ºð¨°ÀFx…§}’ÑùŒÎ›;TœWEçÎi+ÎJç´¥³•Ng2:#N¿K§¦SÂh1£“'¹ÄÉŒNBq’‹Ndt£ã7J/Ž3ѱÅtÌ{t44FÛé(=‹.²ÓŒgô‰Èpñ‰t:ŒÑ¡Œaôñåt0£ƒìt £p²8€Ñþ‡i¿¶´oa˜Ø7›ö°‰…a´O¯0±£½¡Õ»˜@«à0íF{‰žÙ´G¾Uìa£=ê‰×«ò»›Å|+ͯ'Zݽ&±»™v¯ÇG¡åÍ3ˆ^õÖã•ÐÊ3èÄ<Í«Ç^o±ÐÑ®ÀB×»´ £ÛÒ\Fs@À9Å´SZ„Ø©Íf4+Ù.f1šÙvL;ö£pÈ`4nLg4 .§EÐÔšµ”0ÚA*v8L“Û‡ˆÉvš\Ox·í-V±}mÏÙ­’óˆIŒ>w>æ¡íH®ØŽÑ¶Œ&2š`¦žÐ|ÑÓ‹¶1S7£ñf³Ïh\l²·œÆ&Ó˜~Ô=»f4 -dÅh$h%2œF0Îh£N à, ¡Žd14Ÿ:ìÑ‘Lí÷…Ø© ž·1j…‘[ó©z°X©% ;³É šÍÔɨŠ&5dgÙUjÙ :jඕ-èUa$*£ºPªX¨Ì¨¤%FE;¥08z‚C²Hr)p2EŠëqqÉzœôÿÏý¿fàÿò°°P/µx#9v¬G¥d ŠÖ­Ç8òûêñ¼‚ú.ä‡ýjô#VéIœ 5Àº€Ëâàl%Ú¦=YI¿A‹_¸és¨}ƒs(<‹?BqøP[ûK/ð :ûe´á<íįÅeÐç<´‚À‘ ÊhhêT -Û&´̓:çl5ð@åè&ÚB®¡QP?ø-®)ÐnB·R-éJ¦Â}'€ÚV´¯Tµ@@ʺ,6‘$ zF€Ð$´Ml·pyÀ±Iü® -ÕKvÙ £àrÛ…ßÂid úž_††Ñ1ôIú9.ÜÂÓôª$ˆN@3ÑY±I²£JÙ*¥©x‰0AÛ–ññ‘§… ¸]š“èhÇgÛ´#t€ Š9…sÛ´}e`/YÐô.È}#a¸P@óàÊ2¡?Ú‚vÀ“‰ „æÑLè}Z&®l¨¶dq=­úš4p銶‘©¸¸½ ÒœG{¢lè#Z¼JðàÉËѱ‰Ãɇ~ ¨ŒÞƒw“'Èl²…4R+íJGÑù´„n& 녯ĞâÄ¥$i¤tLŽ’»ÊÓ`+ n?(=•µÊ1å{]¨nšî9ÝÛº·Õ6ªW­Qoè“õSa{ßÐ×0Ë°Ïpݸ¶ƒ°]€ížñžÉbJ7½búÞœa^f>kQ-s,²|fµ[Ÿ³6ØzÚ–ÙÞ A!…!ÏÚE{ž}±ý-ÇDÇÇ¡ýB7„þÛ©:ŸDZ¦B@'ÓèPŒ×LK¥5¨L¨Œ©„TËù.Íé99i(媿9gX©›Z3n"œq¼ì8sFljy’½€§´Ô3°ª!´–‚}ΖQ4„!¼VZH ’ (–óÍ_4§!ËM ˜Š­nm£+^;÷[ãE³N$| 6 HƒúzS£œrEá"v9t!f½Õ㊎ŠŒs††Ø¬³ÉhPuJ€GHöXËù÷œÝ.ç»tñwáûÆôætèf„dÐ GœVÜ!n(q™qZÁnšAöL¿5£fÆMöÍ°»ìÒôš·glŸÎn7ŒÝÆ)Ó±rkØm€dñv6q+Û_Í&á—x©Æ·âí-'Ù~uûHV"!#JAÅÞ¥m$· <ö‹3{ÊÛlˆi[é²—‡lÀ•¦8—EÚ(íE¤—"CĶ±&œËi¦rªå¼³K³3ÝߥÙâ¿zÆónzãÛ7,7Þ½}Ãj˱æXmN(9 Ȭ¬ÌŽ ‰VIrØCaX ‰™¡¡NkBBfǬìÌ '4pÒê°K290÷w󿜾hÎùb<õÔ¤â3“ß;ýÑ‘bžãuï$«…ØÌNƒÑ`r†äö¤C’òÈôŒŽ™YbZ¶c”ŸmÐIX”ÍiÆ0”Dãå4Ÿ->ÎãsUÙªe#j‹e‰Æ!;).¬m¤9NQÚÚâ"£%²ƒÍÑÉòEs£¿Ñòn@DÜ641ypé½{õöÿ—öbuæ¤BÚ([LßAõÁAüŽ¨ŽŒÇ˜Tbb·§;Q¹!nJšAqa‚5dÁÚdÍæ@ÖÙYYÙÄk(ÞÙ·_mÓÀ¡‡f~ʾ^Wžžzò/ÝWt^Þkâüܬ1£ß{á±1í#ÝZÞr.­jzlœøò¿‹¾šSÐß_{½¸ÏºÞU¯…9ßLôŒÐyÕÙÂ?Žñ sšF„‰…Ö’Í»ØÞĨL¿"€Sá.¥Qs(¤!èJ¸–ýHI6˜‰àÕ¤mè“QFÔ&¡0ÕO¥7[¹uY¸W“ˆÃnsºHfG[6]T²zuIoófŸd»Ê:ýËýú>þK¸±èŽº‰­teNWÆz… 6=Ðõ7§ÿB7#Ôæ°ÙeËìHðf P—lͬ˥ˬKó¿ð»W®âwùZÑ.ÒWH»±¢Þö£Æ”a„ -}Q´^T±©Ê¦Tª“¬Anôºp£dÓ\ôPœšSàâàgù h§bQ”= 1ÑãÉ­VĶâ©9l߶/Oe[sðàx°péØñIgX)^rfÒñc“Ïà%¬ô pV„ž–›`^'zíèe®_–©D(J‰P%ç<ô¨€2âqÐ%8W+y ¿Ízìf=ñ[ ö¬å•ZüHr»L~Ä)à9¼zúG´C¢ŽFhçüû­"t€ŸÆ·ý'¶NÆ)gÙ*¼‚ë¶_¢sÈj°kª!F`¼xºÆÅD×òY½î½“ò+œ÷ZêÐŒ&E gn|ÍïÇiì,Œsêý|![³/à‡€a".Êo`e+–4øo›~žw–ÀlïZÓ#'rcÕAÿ¤³ù\_X•«ºÃ)Å¡Èx³+.¦ ŒèXÀ™[®Þ„ãù©ÞSP -N!)4EHS¤9EIÑ¥¨)ú<”‡óHÍòÄ<)OÎSòtyjž~„‘Aê ýx4'ãÕñúíh;ÞN¶ÒØ.n—¶ËÛ•íºíêvý^´ï%{é^a¯¸WÚ+ïUöêöª{õGÐ|„¡G„#âéˆ|D9¢;¢Ñü3Á®èxa¼8^/WÆëxÇ¿E¨ÍX<k¾"àuCâ‰ÕbËH‡xMµ½[;ƒ¯ Èê4xPN§~%ëËË×o(/ßðý­[ßó&¹‘=xpv§ýÉ6ö!;ÅN³q*ÎÂÙ8µ†=ÃV±?°gpþ=^Ë@s·!ò€æT”àµSŸ@|b™Œ|:Å-¹(èE*lôkJO¿ JHõÄYÅLO7Sl¹ì5<ø4Îò¿W+,ê_ß÷nS-Ÿ‘[€ªt…÷¶CQQÃÂ#¨3Ò#Ib¾ÅúŠÑg¯ ‹J°êrÆ[h›h‹z:z|m°»›ï<®Øêjé#ã=ÐsfÊÎjS™Y /û\I‹ÈkþE‡qXfqÁ¦•£ß›?íäÄϱ~dq§¦ÚÚÚ¸C·¥¾AË7ä÷x?-ýÚÛŽ.ìÎÓÑZ°Ålà·-ê ü†øj¹n—Ñ'Å”ÇîŠò¹«¤jÇ«íBCµ‡»,.c×Å´~áóÍVÍu4ƒirãäP\›hCŠ`Lugþ\0M•MÛØ··¦}ÿ·oÚœÊìXµþÕ»–ÎY°¬mÛC±±Ÿ¿±l'›o ó@‡1pÌ¡‰¯¼qø•Ç·‚íÖ²3»q½™ýûAzÚ¾5û=F«‡ Cm€;·îSc|õϤ)_h•¥ÚïB ÆxYŠÂ!1îhüW›~ØûÊ¢AƒT¬n¢1È“;&!ŠKÅñÁ8ŒÒKy}7µvî§ßݼÈü?b7¶÷ÝÌ.®Ú¼yÕš²2ñÀ!O"»Ä¾)žÅ~ú÷Žáx)^Ó2ûзþú—×öDËA›Ü—J(Ó.z%Ô#ˆB¾ò¤"ÅŽG12pÚ¨…Ë”›Í­“„—‘ñ Iè8®œîi¹þQZ2ŦáwW‰I|m5Hd æsÝ€²{x=aäK”|®dŸ ¼nâ«©a†6¹m\fËO]渘TPZ³¦5.‹VSç­ÌCð´â>ƒöß΄<„*ÉôÒMÏ—”mzž^µñÇÏý¸qUÕvÆ®^e÷·X¹déÊË–¬$'|ëÖUû*Önw`Åþsçö¯8÷ÞöÓW¯œª9…'=óûß?³tå*nû+`DÚˆÚ ÎÞø ùBÿl1–6X|ñ’/ª*¾Ú"Q¯s¹Š¯Lnn %`·4 ·2£¡šíÙ¬¿q`“kãÚ õ¯1V}xLˆBDdr…ŠéQi:ä²¥ÆÁ¼LÆmv6k1'›Á)@[+ž¸‡&J\~òû}U/íþþÛçV¯ÚÌúâC_ÝY½zÓ+ì6û™S-—­n-™ÊºÍ_þdñ®wßXû’=ôlÍŽ¾Êa¤ÁG¼ᯣý¡>j|Ý°ßâÓUÑêˆtJ“R#8k¿¸6°üTˆÆîXðdΠ‹ãh#+[H°c4»ÌŽã<=zÇ€¾µÃO46ž(zµ0³];\…ç㹸º]»³]½2>`cz»‚ÞׂĆ%ÖÖë|º2ä3KȬ“iº!MvYRmÀ†Ÿgñ€™{Ò³¬–„8·5$ÀKœ¸úÔ}4¡Ç=Ù[¬–Ûu÷ÁýîÝ{ZLa›ÙJ@1ÏC?˜¿/Öó¿7Tâø•)>á¯*ui2„ž ƒ¦’ÆÆÊx`Á¼œ ‹[†‘ -O‘#þ§y,)¨m¹¢a˜¥BPÖ¡6Þ eá¯0!4²ÕÙÀ&ê¾F経!SZ^8Ãéõ©mÉÖòa0è(uñÆ…G Pá‹zDO®Èp—e…¤K™..£FN<¨,v>°tñ¿Ñ˜Õ' ~üÌbv•5b/Ž·gðãµãOœ8òn)í>Ã[—õ€}xW\fÖÙ…ì,¨íkŠ‰Ãwq ãí£I â#P™*ÁXÅ4ÀùÔ€5çÔå*TRCr÷{÷Ú3¤Ý™3-ŸðU­m¤øn9Ù’Ã¥x -¨JUðSª¦Á¾.›ºd”­3ð|2@”¾ôb„ô'îíÊNâÿqœÃN‚,ï­¨­V¡ ¯§4ª>oœâÑ)0e@KE$”¢ªÄ×µ$ü ã–£WÂS;Í“ÌnW³[‡`Ø -¹s7ILôGCî~D:Ú ›ã’án>Ÿ³}ŠebÒd+JU[]F€ßœ‡ðªÀ¹tb®Á' “Û‘¹“<‚“™ò"²TVÃ$`X*À…Ò< O—x>Ánaç.Çß9~1å¸Øt/Cøàn’ðÁ½ ðd‹`¾ŽiÍN½!:Ÿ™úUæê0”nÈ’Òm™aS­0"·ffBªÓµ¥[«KK«·–~ôSKËíŸü\_Ã…8‚}ÍÞdõìkŽ ÙR¶2²2\ŠW²•Ð;÷j‹4?‚’¼aØg”¬ÌfQ" 1ÝØÉŠ\º {`F©%gø¡à"püW^¯xqÛzÖ¸‹ »ïúi1¥åìæ’’M»®|~ñŸ-»yô¯ä_@!³Íôv‚@LUÉC*zæKrPÁáÓ°/Ó ¢5T'Q•ZQ”‰ªáé‚5Í®º ©Ñ§ÈY²æXƒëh]8k|õ±5†ó©g -.™…†:µ•W·–mhуA;ç9óÙþË&¿s×ÜÆÒË-^ܸÁ÷,yÇ>o›Ž—¼<¶e­Øôåß7&#[nüaͪµÁxØt—ˆþËÛÕh &½ÇãRtDV=11®|Uãuû+a>+~½"ø-ÙÁº]^³P'½IêÐÉhWž¤øµ$ů¥NqHqø,ÞýìH²Wþì¯ -£€i¥@ëð›"§€ºJ„¿fkÔr€‚5.ÀM&%¬è‡$û/+%¡’[ÿ>’ñLó¨Ò#’ò~pý`‹pî^ -›3x0_ku -7Èmi*Üéñêð!tPÀ]hëº{¿}ºaýö™†.:6çí4ž³ÄqøVvXšÊÊð3Á5 ·°Œã[4×ûx–ˆXC˜YFuar…-®<öHtE›[u˜‡ á&U2ôŒ$G·Ç,7¹ëJ¬©§4^½é¿Ìà­9š.R½Ñ©1©±©q©ñ5¨×µF¿#´ÆYV^a‹Ý­¹}v&‡—Útæ ~FfÂ/kX|Ö“†®/ÿqéì-¯áƒ;¿¾òÏïßû÷OxͦqGGO=\T~¢kB,Éxrþ”ùj׿eÕ®âñïì8|,zÍ’¬Žõ‰‰C†¤oâè)è F…R½a­0ë*Õæ†È‘ÍÖ;Ì IÑ0²ôôÀ’¤{­|¢5wn±¶J£-•&$r3 Ue%kÖ­[SRÖr¥ý SOÿëÚ™âêõõ$åã Ÿ7ÿâyfH@áïÙ·ìĈ¡ëùËjŒŽ?|8JñFÊG-¨<ühhµTèxZe3 © Âr³±5«²°›|5á?€ôV9 -ž¾/a~ö9ö`áñ­ 7ýË¡Ã{&léž“”„;a;lkÿN÷Ü|xörçn\.Ë€›Ä¿ÁÔä""{D…É^¡T›ð Ô)*ímµéûp¹p…?*Ð5ÍðHèÃ/é¨úúÕÅg®]?=e+3—•””——””Ñs¤ÇÏÍ뇎À±[qö¦ÿøÂç›>¿ÐÊOÈÅüôò&@&ƒÕrÝz)´KFüNxEHƒ±:š’(‹.TB…Q6K·ÆÀkI.ª«Ü-7ohI Ì› |­’Ê#ŠüWŽ¼VttÎŒã£Ù=vÇþðñOõÂƲ’=2n”ôæ{rÞáåàlÀ^ö副îÛÆõ+ä‚:‚Ô,Ø ÔI¸½`’Ž©$DF:Q2šõv¾¶¾9ø¦>/–¤Û¸ä\mŽoc¸á ‰ç&À«ØŠzÖ„“^}]²W?>mr¥?…ž«Xÿš&ÖG°çkob¸Á™¨£¶òP]…+´ÂLa¶ºÌÇÛ{,1I†Þ’Í×&kcPw–«WÕÒÄs¾œ_[wˆü‚º’‡TëÌÌ £JW•n¨,[QZã›þ»†Nªéñ|Yû-s¯_oœ]•ROrÎ~òÉÙ³Ÿ]`ýÌYסýŽ}Êòqcp.–±‚s‡Úˆ‚²‹Ôd×Þ¦3Qþ-j…µÁðdêkÔIú;Ÿ‡Ú4ìÒ¬¡€íMb€UmáJsBäÁ™O®*¯¯OÛ½àϯ’º–¾¤Î·áÍ?·”Jö–&k»ï‡þzB:dD¼áJ24ª/ c©h=–Äþ¨Ÿ*›@YWïË Š¶¤‹8Ãá<ÿ žãŸ‹ç°Ë8¦¾^˜àO©¬¤ù¤_tC‡Øpj~ŒÈŠ:y£E„Õ -×QK£ézL9nÓ+DGÌ‚dB—ƒA“Õ: Ì)Q{9g—ëùÚ'Eì8¾õúŸØõQ0¥ìô«Ê{Ç?Ú¶\úÓ5.ÍfÝ~ͳðæ*Î@¶ê”\8!v:eÈÎD§‚N@7$öz!˜Åê°$õD½ ïæôÖ¬íªóÑ -B+œÐ²Z™cˆ!Q”P&†)™¤£˜¥©dY,‚sU”pðab¸!‡)mi¢ØVîL; ÙbŽœ­ô¥…òHi¤<‹ÎfH3ä%ôq‰´DŽúUnÛÜ€¯üƒõÆì2›´¶A²û—áslTKÉ_Åò`ì50vUŠ‰'@…¨Zg8AëDlHÒ÷×ñ÷Éþ÷KAZŠ§}¡"…HÃÅLVŒùɵ?½Á>×pÕï~¿íü1úó³@wçó,ýÑ›f3ëEGJ²ÃPK"…[dd5+¤ÖæQa"z¹yÐ5@еjA7âþÑN#ý]üpÂ…ÚåêMíÅÿ Äëí*¤Š©Rªœª¤êRÕT}^hž3/,/ÿ¼éã ÛØßÙ?Á¢>ÂÉ8»p>wØpaôËã`†7êAl0UããôX4ÄÀÞZ4|!À¼} -=FÄ`†§ä¡ð›žZ( à…Òõ-§$µrèö.»ðàÔ|+8GkÙ!FxZC,ÀÑÄÒ`®Ž<~,ZÃ+½¹tû%6?ÌÑñ?êÈ!mï_ZR²vmII)‰H®žrêÚ¿Nh9x$q ·¬:gC|Ź#†Vþ|çL°„Õ #+š‘õY©ÏÔ ;&«’„ðiÚZ°pþ}nè©^Y°Å‹bq¬5¥â KªÕ‹¼¸‡ÅkŒãÁ–ÁVÛXœð+Kq -«;/-ÜSr;²ºçâ,º¤ÃcŸ¾ßò‘0áó§WÄ·ávÙ–"7‰h‘׫åáÎ`î„|´__·)bÛò]‡Ýî†ÌÌÉ#W¦Ș/®¹+âK0Ó¼a&QÌYñ1¥NUô:Fi±™EÍ7»4j¼¿øxÄëYÉT¼ƒ·èìå³µjˆ/±c•-5¿›´i×2¡w âÓHaܯf}µŠøŒï S_›õÕÕ`¦Þšªs<€† ÇëgÍÿúƒÓ^}rÏn¼›O{>éÉ’{;vO,¾ÌuY #œ#L€\|Š·Ä Hüœüb˜„$ìDHâùß1n|µQ(D½dèÞÀU¿‘òíO$|…ÑÄLª‚H<„ðµD-tÁ½pÁöÔwɇÑ=÷jZstàÅ€­Þ^z›ªNå+ŸzØÒO’“'éÎ@’î $é·’ôc¿•¤ßüí$=·&ëÖF+ô:Aƒbp4‰ QB¸¥‹T#ô1†”€Û‘±ÔNv+í 9$KÌ‘rä ¥‹¡—R¨+Ð÷5 WŠt#ô# SÉ :U˜!ÎÔMÑ/&KÅg”ßéžÒ'›ÕpÂÂÅ5FM"Ib7ÒMì¢öQGª3É qººŒ¬ K„ÂRñwê -5 äÖšÓcžÓׯyqÇñ:vwßëû÷Ÿ&ù- tí½2µe+×jd HR^õö "K€@™ …'˜: ȇu§:u*æ½ -²Õ9A²ùª,`A“`*Ç)þÆ N¹šîüšÿ– kòtÚ‘§âòb¼D–d ¼¨5W討†« -ŒPGÜZ¢ "Œcóp}«g›p=›÷>n‡… -ÿj©Çï°<Ò‡„±Yøy>Òé=VÁHm¨ÆÛËè4¹Ùè‰ÎÀ—Õi¼ç4[?X-F“Ñì4™Œù–#2é*äãVÓ±«Ålä+{6ÅVhêÂìÀ—ÚÜÊydyóÑ‘jK„,ëdÓÞP -–Å¢X[¸!ܘ`J0w²Z m#*_/”dÂ×Î4x™ÍÇ ¸¾CïÉñÖÙ<çñ1½Ù ã¸?î}O_t„ÝÉ8pë¸Fa 3ÍGztj&–.òFQ>'ᨲä¥P$àB$õR¹G .4ÿú#‚ýΨ#‹vRzѾÊZ¤L¡3•Et©¢ŽåŸpÐã®Ðä–ådµÿ0YÝò¼0i·ÿ‹ªZê..²ýô&`Þ6hª73ÜìÐK]„Å­cã(Ò×éP~WwÒQò†Ç SÅ6¡á(ZCˆņwUÍ¢çwX; -~Ôjñ²[ðú­lò ©¢æ=ùÌF[¨äp—¬<ÐÓ›ÞŸ_UÑ­Ûú™¯þì-¨xbôÜy£ž¨hxvóÅï} +Ÿªúáâ¦Ê!Ü3 è΀ûbáùyÐïDíÚ¯é Õ¨p -ஹPò¡ÌC³à:ï<5O£™\d ìGžo}ºõYüýOѨ_ÿ'tðÇ%Nîß×îÔ®_(W©ƒÿÇ­ùÃ'¸n\šÞbaß$ˆ5^0ÊD¹°ïFÝG€þ`Ÿ&h4xQAgðtÏ‚þàoAãï~vàz7xFÒ®'L|jâ$”:yÉS³Qê´§¦ÌB©³'΋åû…sA¯>­ýþÐZD -endstream -endobj -616 0 obj -<< -/Type /FontDescriptor -/FontName /GUHWDL+DejaVuSans -/FontFamily (DejaVu Sans) -/Flags 4 -/FontBBox [ 0 -14 472 560 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 560 -/StemV 80 -/StemH 80 -/FontFile2 612 0 R ->> -endobj -617 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /GUHWDL+DejaVuSans -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 317 ] 17 [ 317 ] 20 [ 636 ] 25 [ 636 ] 37 [ 686 698 ] 40 [ 631 ] 42 [ 774 751 294 ] 46 [ 655 ] 53 [ 694 634 ] 68 [ 612 ] 70 [ 549 634 615 ] 74 [ 634 633 277 ] 78 [ 579 ] 80 [ 974 633 611 ] 85 [ 411 520 392 633 591 817 ] ] -/FontDescriptor 616 0 R ->> -endobj -618 0 obj -<< -/Length 744 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -30 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0003> <0020> -<002b> <0048> -<0044> <0061> -<0055> <0072> -<0047> <0064> -<0048> <0065> -<0051> <006e> -<004c> <0069> -<004a> <0067> -<002a> <0047> -<0058> <0075> -<005a> <0077> -<0057> <0074> -<004b> <0068> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0046> <0063> -<0050> <006d> -<004e> <006b> -<0052> <006f> -<0056> <0073> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -619 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /GUHWDL+DejaVuSans -/Encoding /Identity-H -/DescendantFonts [ 617 0 R ] -/ToUnicode 618 0 R ->> -endobj -620 0 obj -<< -/Type /FontDescriptor -/FontName /BNVYMD+Poppins -/FontFamily (Poppins) -/Flags 4 -/FontBBox [ -27 -260 563 736 ] -/ItalicAngle 0 -/Ascent 1049 -/Descent -349 -/CapHeight 736 -/StemV 80 -/StemH 80 -/FontFile2 613 0 R ->> -endobj -621 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /BNVYMD+Poppins -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 266 ] 10 [ 158 453 453 ] 15 [ 197 550 209 ] 19 [ 627 319 574 588 628 627 634 545 630 629 212 ] 36 [ 673 612 771 706 512 503 777 691 245 ] 46 [ 598 431 ] 49 [ 702 785 578 ] 53 [ 607 586 540 674 675 975 ] 68 [ 675 675 606 675 619 328 675 639 245 247 514 245 1029 639 639 675 675 372 521 363 639 560 819 478 562 ] 206 [ 379 379 ] 209 [ 411 ] ] -/FontDescriptor 620 0 R ->> -endobj -622 0 obj -<< -/Length 1248 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -66 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0003> <0020> -<002b> <0048> -<0044> <0061> -<0055> <0072> -<0047> <0064> -<0048> <0065> -<0051> <006e> -<004c> <0069> -<004a> <0067> -<002a> <0047> -<0058> <0075> -<005a> <0077> -<0057> <0074> -<004b> <0068> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0046> <0063> -<0050> <006d> -<004e> <006b> -<0016> <0033> -<0017> <0034> -<0018> <0035> -<001a> <0037> -<0015> <0032> -<0013> <0030> -<0032> <004f> -<0052> <006f> -<0049> <0066> -<004f> <006c> -<0033> <0050> -<0056> <0073> -<0053> <0070> -<0024> <0041> -<0031> <004e> -<005c> <0079> -<0037> <0054> -<0010> <002d> -<0045> <0062> -<0054> <0071> -<000b> <0028> -<000c> <0029> -<000f> <002c> -<001d> <003a> -<0039> <0056> -<001b> <0038> -<0027> <0044> -<0029> <0046> -<00d1> <2022> -<005b> <0078> -<003a> <0057> -<0038> <0055> -<00ce> <201c> -<00cf> <201d> -<004d> <006a> -<002f> <004c> -<000a> <0027> -<001c> <0039> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -623 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /BNVYMD+Poppins -/Encoding /Identity-H -/DescendantFonts [ 621 0 R ] -/ToUnicode 622 0 R ->> -endobj -624 0 obj -<< -/Type /FontDescriptor -/FontName /MQMIVM+DejaVuSansMono -/FontFamily (DejaVu Sans Mono) -/Flags 5 -/FontBBox [ 0 -14 584 759 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 759 -/StemV 80 -/StemH 80 -/FontFile2 614 0 R ->> -endobj -625 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /MQMIVM+DejaVuSansMono -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 16 [ 601 601 ] 29 [ 601 ] 36 [ 601 ] 54 [ 601 601 ] 66 [ 601 ] 68 [ 601 ] 70 [ 601 601 601 601 ] 76 [ 601 ] 78 [ 601 601 601 601 601 601 ] 85 [ 601 601 601 601 601 ] 92 [ 601 ] ] -/FontDescriptor 624 0 R ->> -endobj -626 0 obj -<< -/Length 674 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -25 beginbfchar -<0048> <0065> -<0057> <0074> -<0046> <0063> -<0047> <0064> -<0049> <0066> -<0044> <0061> -<0058> <0075> -<004f> <006c> -<0056> <0073> -<0055> <0072> -<0011> <002e> -<005c> <0079> -<0050> <006d> -<0042> <005f> -<0053> <0070> -<0052> <006f> -<004c> <0069> -<001d> <003a> -<0010> <002d> -<0051> <006e> -<0036> <0053> -<0059> <0076> -<0024> <0041> -<0037> <0054> -<004e> <006b> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -627 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /MQMIVM+DejaVuSansMono -/Encoding /Identity-H -/DescendantFonts [ 625 0 R ] -/ToUnicode 626 0 R ->> -endobj -628 0 obj -<< -/Type /FontDescriptor -/FontName /VZSVIB+DejaVuSansMono -/FontFamily (DejaVu Sans Mono) -/Flags 4 -/FontBBox [ 0 -235 585 699 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 699 -/StemV 80 -/StemH 80 -/FontFile2 615 0 R ->> -endobj -629 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /VZSVIB+DejaVuSansMono -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 601 602 602 602 602 602 602 602 602 602 602 601 602 601 601 601 601 602 602 601 601 602 602 602 602 601 602 602 ] 32 [ 602 ] 36 [ 602 602 602 602 602 602 602 602 602 ] 46 [ 602 602 602 601 602 601 ] 53 [ 602 602 602 602 602 602 ] 60 [ 602 ] 62 [ 602 ] 64 [ 602 ] 66 [ 602 602 602 601 601 601 601 601 602 602 602 602 601 601 602 601 601 602 602 602 601 601 601 602 601 602 601 602 602 602 602 ] ] -/FontDescriptor 628 0 R ->> -endobj -630 0 obj -<< -/Length 1500 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -84 beginbfchar -<0056> <0073> -<005c> <0079> -<0046> <0063> -<0057> <0074> -<004f> <006c> -<0012> <002f> -<0048> <0065> -<0011> <002e> -<0047> <0064> -<001c> <0039> -<0013> <0030> -<0010> <002d> -<004e> <006b> -<0058> <0075> -<0045> <0062> -<0052> <006f> -<0051> <006e> -<0049> <0066> -<0059> <0076> -<0050> <006d> -<0055> <0072> -<004c> <0069> -<0042> <005f> -<0020> <003d> -<0014> <0031> -<0053> <0070> -<0044> <0061> -<005b> <0078> -<0015> <0032> -<0018> <0035> -<0003> <0020> -<0016> <0033> -<0017> <0034> -<004a> <0067> -<0005> <0022> -<004b> <0068> -<001d> <003a> -<0036> <0053> -<0024> <0041> -<0037> <0054> -<0039> <0056> -<000e> <002b> -<0006> <0023> -<0004> <0021> -<0007> <0024> -<000b> <0028> -<004d> <006a> -<005e> <007b> -<003e> <005b> -<000d> <002a> -<0040> <005d> -<000a> <0027> -<0060> <007d> -<000c> <0029> -<001e> <003b> -<005a> <0077> -<0031> <004e> -<0033> <0050> -<001b> <0038> -<002c> <0049> -<0028> <0045> -<002e> <004b> -<000f> <002c> -<0035> <0052> -<0054> <0071> -<0027> <0044> -<0026> <0043> -<002f> <004c> -<002b> <0048> -<003a> <0057> -<002a> <0047> -<0030> <004d> -<0019> <0036> -<0032> <004f> -<003c> <0059> -<001a> <0037> -<005f> <007c> -<0038> <0055> -<0009> <0026> -<0029> <0046> -<005d> <007a> -<0025> <0042> -<0043> <0060> -<0008> <0025> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -631 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /VZSVIB+DejaVuSansMono -/Encoding /Identity-H -/DescendantFonts [ 629 0 R ] -/ToUnicode 630 0 R ->> -endobj -632 0 obj -<< -/GUHWDL 619 0 R -/BNVYMD 623 0 R -/MQMIVM 627 0 R -/VZSVIB 631 0 R ->> -endobj -xref -0 633 -0000000000 65535 f -0000000015 00000 n -0000000235 00000 n -0000000355 00000 n -0000027529 00000 n -0000027662 00000 n -0000030321 00000 n -0000030520 00000 n -0000034117 00000 n -0000034467 00000 n -0000034601 00000 n -0000034737 00000 n -0000034899 00000 n -0000035062 00000 n -0000035218 00000 n -0000035375 00000 n -0000035532 00000 n -0000035692 00000 n -0000035853 00000 n -0000036014 00000 n -0000036165 00000 n -0000036317 00000 n -0000036492 00000 n -0000036668 00000 n -0000036844 00000 n -0000037016 00000 n -0000037189 00000 n -0000037361 00000 n -0000037534 00000 n -0000037707 00000 n -0000042713 00000 n -0000043017 00000 n -0000043267 00000 n -0000043403 00000 n -0000043566 00000 n -0000043723 00000 n -0000043880 00000 n -0000044041 00000 n -0000044202 00000 n -0000044354 00000 n -0000044527 00000 n -0000044700 00000 n -0000044873 00000 n -0000045135 00000 n -0000045395 00000 n -0000050582 00000 n -0000050837 00000 n -0000051092 00000 n -0000051225 00000 n -0000051356 00000 n -0000051487 00000 n -0000051618 00000 n -0000051749 00000 n -0000056758 00000 n -0000057055 00000 n -0000057188 00000 n -0000057319 00000 n -0000057452 00000 n -0000057583 00000 n -0000057714 00000 n -0000057845 00000 n -0000057978 00000 n -0000058111 00000 n -0000058242 00000 n -0000058373 00000 n -0000058504 00000 n -0000058635 00000 n -0000063613 00000 n -0000063882 00000 n -0000064015 00000 n -0000064146 00000 n -0000064277 00000 n -0000064408 00000 n -0000064539 00000 n -0000064735 00000 n -0000064994 00000 n -0000065225 00000 n -0000070848 00000 n -0000071201 00000 n -0000071334 00000 n -0000071465 00000 n -0000071596 00000 n -0000071727 00000 n -0000071858 00000 n -0000071989 00000 n -0000072120 00000 n -0000072251 00000 n -0000072382 00000 n -0000072514 00000 n -0000072646 00000 n -0000072778 00000 n -0000072910 00000 n -0000073042 00000 n -0000073175 00000 n -0000073306 00000 n -0000073437 00000 n -0000073568 00000 n -0000073699 00000 n -0000073928 00000 n -0000079347 00000 n -0000079784 00000 n -0000080022 00000 n -0000080156 00000 n -0000080288 00000 n -0000080420 00000 n -0000080552 00000 n -0000080684 00000 n -0000080816 00000 n -0000080948 00000 n -0000081080 00000 n -0000081212 00000 n -0000081345 00000 n -0000081478 00000 n -0000081611 00000 n -0000081744 00000 n -0000081877 00000 n -0000082010 00000 n -0000082143 00000 n -0000082276 00000 n -0000082409 00000 n -0000082542 00000 n -0000082675 00000 n -0000082808 00000 n -0000082941 00000 n -0000083074 00000 n -0000083207 00000 n -0000083340 00000 n -0000083473 00000 n -0000083605 00000 n -0000088733 00000 n -0000089212 00000 n -0000089345 00000 n -0000089478 00000 n -0000089611 00000 n -0000089744 00000 n -0000089877 00000 n -0000090010 00000 n -0000090143 00000 n -0000090276 00000 n -0000090409 00000 n -0000090542 00000 n -0000090675 00000 n -0000090808 00000 n -0000090941 00000 n -0000091074 00000 n -0000091207 00000 n -0000091340 00000 n -0000091473 00000 n -0000091606 00000 n -0000091739 00000 n -0000091872 00000 n -0000092005 00000 n -0000092138 00000 n -0000092271 00000 n -0000092404 00000 n -0000092537 00000 n -0000092670 00000 n -0000092803 00000 n -0000092936 00000 n -0000093069 00000 n -0000093202 00000 n -0000093335 00000 n -0000093467 00000 n -0000093598 00000 n -0000098570 00000 n -0000099009 00000 n -0000099142 00000 n -0000099275 00000 n -0000099408 00000 n -0000099541 00000 n -0000099674 00000 n -0000099807 00000 n -0000099940 00000 n -0000100073 00000 n -0000100206 00000 n -0000100339 00000 n -0000100472 00000 n -0000100605 00000 n -0000100738 00000 n -0000100871 00000 n -0000101004 00000 n -0000101137 00000 n -0000101270 00000 n -0000101403 00000 n -0000101536 00000 n -0000101669 00000 n -0000101802 00000 n -0000101935 00000 n -0000102068 00000 n -0000102201 00000 n -0000102334 00000 n -0000102467 00000 n -0000102599 00000 n -0000102730 00000 n -0000107838 00000 n -0000108341 00000 n -0000108474 00000 n -0000108607 00000 n -0000108740 00000 n -0000108873 00000 n -0000109006 00000 n -0000109139 00000 n -0000109272 00000 n -0000109405 00000 n -0000109538 00000 n -0000109671 00000 n -0000109804 00000 n -0000109938 00000 n -0000110072 00000 n -0000110206 00000 n -0000110340 00000 n -0000110474 00000 n -0000110608 00000 n -0000110742 00000 n -0000110876 00000 n -0000111010 00000 n -0000111144 00000 n -0000111278 00000 n -0000111412 00000 n -0000111546 00000 n -0000111680 00000 n -0000111814 00000 n -0000111948 00000 n -0000112082 00000 n -0000112216 00000 n -0000112350 00000 n -0000112484 00000 n -0000112618 00000 n -0000112752 00000 n -0000112886 00000 n -0000113019 00000 n -0000113151 00000 n -0000117686 00000 n -0000118189 00000 n -0000118323 00000 n -0000118457 00000 n -0000118591 00000 n -0000118725 00000 n -0000118859 00000 n -0000118993 00000 n -0000119127 00000 n -0000119261 00000 n -0000119395 00000 n -0000119529 00000 n -0000119663 00000 n -0000119797 00000 n -0000119931 00000 n -0000120065 00000 n -0000120199 00000 n -0000120333 00000 n -0000120467 00000 n -0000120601 00000 n -0000120735 00000 n -0000120869 00000 n -0000121003 00000 n -0000121137 00000 n -0000121271 00000 n -0000121405 00000 n -0000121539 00000 n -0000121673 00000 n -0000121807 00000 n -0000121941 00000 n -0000122075 00000 n -0000122209 00000 n -0000122343 00000 n -0000122477 00000 n -0000122611 00000 n -0000122745 00000 n -0000122878 00000 n -0000123010 00000 n -0000127251 00000 n -0000127770 00000 n -0000127904 00000 n -0000128038 00000 n -0000128172 00000 n -0000128306 00000 n -0000128440 00000 n -0000128574 00000 n -0000128708 00000 n -0000128842 00000 n -0000128976 00000 n -0000129110 00000 n -0000129244 00000 n -0000129378 00000 n -0000129512 00000 n -0000129646 00000 n -0000129780 00000 n -0000129914 00000 n -0000130048 00000 n -0000130182 00000 n -0000130316 00000 n -0000130450 00000 n -0000130584 00000 n -0000130718 00000 n -0000130852 00000 n -0000130986 00000 n -0000131120 00000 n -0000131254 00000 n -0000131388 00000 n -0000131522 00000 n -0000131656 00000 n -0000131790 00000 n -0000131924 00000 n -0000132058 00000 n -0000132192 00000 n -0000132326 00000 n -0000132460 00000 n -0000132594 00000 n -0000132727 00000 n -0000132859 00000 n -0000137871 00000 n -0000138390 00000 n -0000138524 00000 n -0000138658 00000 n -0000138792 00000 n -0000138926 00000 n -0000139060 00000 n -0000139194 00000 n -0000139328 00000 n -0000139462 00000 n -0000139596 00000 n -0000139730 00000 n -0000139864 00000 n -0000139998 00000 n -0000140132 00000 n -0000140266 00000 n -0000140400 00000 n -0000140534 00000 n -0000140668 00000 n -0000140802 00000 n -0000140936 00000 n -0000141070 00000 n -0000141204 00000 n -0000141338 00000 n -0000141472 00000 n -0000141606 00000 n -0000141740 00000 n -0000141874 00000 n -0000142008 00000 n -0000142142 00000 n -0000142276 00000 n -0000142410 00000 n -0000142544 00000 n -0000142678 00000 n -0000142812 00000 n -0000142946 00000 n -0000143080 00000 n -0000143214 00000 n -0000143347 00000 n -0000143479 00000 n -0000148519 00000 n -0000149038 00000 n -0000149172 00000 n -0000149306 00000 n -0000149440 00000 n -0000149574 00000 n -0000149708 00000 n -0000149842 00000 n -0000149976 00000 n -0000150110 00000 n -0000150244 00000 n -0000150378 00000 n -0000150512 00000 n -0000150646 00000 n -0000150780 00000 n -0000150914 00000 n -0000151048 00000 n -0000151182 00000 n -0000151316 00000 n -0000151450 00000 n -0000151584 00000 n -0000151718 00000 n -0000151852 00000 n -0000151986 00000 n -0000152120 00000 n -0000152254 00000 n -0000152388 00000 n -0000152522 00000 n -0000152656 00000 n -0000152790 00000 n -0000152924 00000 n -0000153058 00000 n -0000153192 00000 n -0000153326 00000 n -0000153460 00000 n -0000153594 00000 n -0000153728 00000 n -0000153862 00000 n -0000153995 00000 n -0000154127 00000 n -0000159321 00000 n -0000159800 00000 n -0000159934 00000 n -0000160068 00000 n -0000160202 00000 n -0000160336 00000 n -0000160470 00000 n -0000160604 00000 n -0000160738 00000 n -0000160872 00000 n -0000161006 00000 n -0000161140 00000 n -0000161274 00000 n -0000161408 00000 n -0000161542 00000 n -0000161676 00000 n -0000161906 00000 n -0000162041 00000 n -0000162174 00000 n -0000162307 00000 n -0000162440 00000 n -0000162573 00000 n -0000162706 00000 n -0000162839 00000 n -0000162972 00000 n -0000163105 00000 n -0000163239 00000 n -0000163373 00000 n -0000163507 00000 n -0000163641 00000 n -0000163775 00000 n -0000163909 00000 n -0000164043 00000 n -0000164176 00000 n -0000164308 00000 n -0000169178 00000 n -0000169689 00000 n -0000169823 00000 n -0000169957 00000 n -0000170091 00000 n -0000170225 00000 n -0000170359 00000 n -0000170493 00000 n -0000170627 00000 n -0000170761 00000 n -0000170895 00000 n -0000171029 00000 n -0000171163 00000 n -0000171297 00000 n -0000171431 00000 n -0000171565 00000 n -0000171699 00000 n -0000171833 00000 n -0000171967 00000 n -0000172101 00000 n -0000172235 00000 n -0000172369 00000 n -0000172503 00000 n -0000172637 00000 n -0000172771 00000 n -0000172905 00000 n -0000173039 00000 n -0000173173 00000 n -0000173307 00000 n -0000173441 00000 n -0000173575 00000 n -0000173709 00000 n -0000173843 00000 n -0000173977 00000 n -0000174111 00000 n -0000174245 00000 n -0000174379 00000 n -0000174512 00000 n -0000174644 00000 n -0000179360 00000 n -0000179839 00000 n -0000179973 00000 n -0000180107 00000 n -0000180241 00000 n -0000180375 00000 n -0000180509 00000 n -0000180643 00000 n -0000180777 00000 n -0000180911 00000 n -0000181045 00000 n -0000181179 00000 n -0000181313 00000 n -0000181447 00000 n -0000181581 00000 n -0000181715 00000 n -0000181849 00000 n -0000181983 00000 n -0000182117 00000 n -0000182251 00000 n -0000182385 00000 n -0000182519 00000 n -0000182653 00000 n -0000182787 00000 n -0000182921 00000 n -0000183055 00000 n -0000183189 00000 n -0000183323 00000 n -0000183457 00000 n -0000183591 00000 n -0000183725 00000 n -0000183859 00000 n -0000183993 00000 n -0000184126 00000 n -0000184258 00000 n -0000189459 00000 n -0000189938 00000 n -0000190072 00000 n -0000190206 00000 n -0000190340 00000 n -0000190474 00000 n -0000190608 00000 n -0000190742 00000 n -0000190876 00000 n -0000191010 00000 n -0000191144 00000 n -0000191278 00000 n -0000191412 00000 n -0000191547 00000 n -0000191682 00000 n -0000191817 00000 n -0000191952 00000 n -0000192087 00000 n -0000192222 00000 n -0000192357 00000 n -0000192492 00000 n -0000192627 00000 n -0000192762 00000 n -0000192897 00000 n -0000193032 00000 n -0000193167 00000 n -0000193302 00000 n -0000193437 00000 n -0000193572 00000 n -0000193707 00000 n -0000193842 00000 n -0000193977 00000 n -0000194112 00000 n -0000194246 00000 n -0000194379 00000 n -0000199526 00000 n -0000199909 00000 n -0000200044 00000 n -0000200179 00000 n -0000200314 00000 n -0000200449 00000 n -0000200584 00000 n -0000200719 00000 n -0000200854 00000 n -0000200989 00000 n -0000201124 00000 n -0000201257 00000 n -0000201390 00000 n -0000201523 00000 n -0000201656 00000 n -0000201789 00000 n -0000201922 00000 n -0000202055 00000 n -0000202188 00000 n -0000202322 00000 n -0000202456 00000 n -0000202590 00000 n -0000202723 00000 n -0000207709 00000 n -0000208180 00000 n -0000208314 00000 n -0000208448 00000 n -0000208582 00000 n -0000208716 00000 n -0000208850 00000 n -0000208984 00000 n -0000209118 00000 n -0000209252 00000 n -0000209386 00000 n -0000209520 00000 n -0000209654 00000 n -0000209788 00000 n -0000209922 00000 n -0000210057 00000 n -0000210190 00000 n -0000210323 00000 n -0000210456 00000 n -0000210589 00000 n -0000210722 00000 n -0000210855 00000 n -0000210988 00000 n -0000211121 00000 n -0000211255 00000 n -0000211389 00000 n -0000211523 00000 n -0000211657 00000 n -0000211791 00000 n -0000211925 00000 n -0000212059 00000 n -0000212193 00000 n -0000212327 00000 n -0000212460 00000 n -0000215704 00000 n -0000215951 00000 n -0000216085 00000 n -0000216219 00000 n -0000216353 00000 n -0000216487 00000 n -0000216662 00000 n -0000216816 00000 n -0000216931 00000 n -0000217097 00000 n -0000217286 00000 n -0000217415 00000 n -0000217608 00000 n -0000217781 00000 n -0000217965 00000 n -0000218124 00000 n -0000218303 00000 n -0000218480 00000 n -0000218672 00000 n -0000218890 00000 n -0000219102 00000 n -0000219164 00000 n -0000226342 00000 n -0000230584 00000 n -0000235548 00000 n -0000247293 00000 n -0000247525 00000 n -0000247941 00000 n -0000248738 00000 n -0000248888 00000 n -0000249117 00000 n -0000249649 00000 n -0000250951 00000 n -0000251098 00000 n -0000251339 00000 n -0000251712 00000 n -0000252439 00000 n -0000252593 00000 n -0000252835 00000 n -0000253429 00000 n -0000254983 00000 n -0000255137 00000 n -trailer -<< -/Size 633 -/Root 3 0 R -/Info 2 0 R ->> -startxref -255224 -%%EOF diff --git a/versioned_docs/version-2.6/security/hardening-guides/rke2-1.6-benchmark-2.6/Rancher_RKE2_v2-6_CIS_v1-6_Benchmark_Assessment.pdf b/versioned_docs/version-2.6/security/hardening-guides/rke2-1.6-benchmark-2.6/Rancher_RKE2_v2-6_CIS_v1-6_Benchmark_Assessment.pdf deleted file mode 100644 index a99a551e3d6..00000000000 --- a/versioned_docs/version-2.6/security/hardening-guides/rke2-1.6-benchmark-2.6/Rancher_RKE2_v2-6_CIS_v1-6_Benchmark_Assessment.pdf +++ /dev/null @@ -1,14240 +0,0 @@ -%PDF-1.7 -%🖤 -1 0 obj -<< -/Type /Pages -/Kids [ 6 0 R 8 0 R 38 0 R 76 0 R 114 0 R 152 0 R 186 0 R 222 0 R 258 0 R 294 0 R 306 0 R 313 0 R 317 0 R 321 0 R 325 0 R 329 0 R 333 0 R 338 0 R 342 0 R 346 0 R 351 0 R 358 0 R 398 0 R 423 0 R 462 0 R 486 0 R 490 0 R 492 0 R 495 0 R 497 0 R 500 0 R 502 0 R 504 0 R 507 0 R 509 0 R 512 0 R 514 0 R 517 0 R 519 0 R 522 0 R 524 0 R 527 0 R 529 0 R 533 0 R 535 0 R 537 0 R 542 0 R 544 0 R 547 0 R 549 0 R 551 0 R 554 0 R 556 0 R 559 0 R 561 0 R 564 0 R 566 0 R 569 0 R 571 0 R 574 0 R 576 0 R 579 0 R 581 0 R 584 0 R 586 0 R 589 0 R 591 0 R 594 0 R 596 0 R 599 0 R 601 0 R 603 0 R 606 0 R 608 0 R 611 0 R 613 0 R 616 0 R 618 0 R 621 0 R 623 0 R 626 0 R 628 0 R 631 0 R 633 0 R 636 0 R 638 0 R 641 0 R 643 0 R 645 0 R 649 0 R 651 0 R 653 0 R 656 0 R 659 0 R 662 0 R 665 0 R 668 0 R 670 0 R 673 0 R 675 0 R 679 0 R 681 0 R 684 0 R 686 0 R 689 0 R 691 0 R 694 0 R 696 0 R 698 0 R 701 0 R 704 0 R 708 0 R 713 0 R 717 0 R 720 0 R 724 0 R 728 0 R 732 0 R 736 0 R 740 0 R 744 0 R 748 0 R 751 0 R 755 0 R 759 0 R 763 0 R 765 0 R 769 0 R 773 0 R 775 0 R 777 0 R 802 0 R 813 0 R 817 0 R 821 0 R 825 0 R 828 0 R 836 0 R 853 0 R 855 0 R 857 0 R 859 0 R ] -/Count 142 ->> -endobj -2 0 obj -<< -/Title (RKE2 CIS v1.6 Benchmark - Self-Assessment Guide - Rancher v2.6) -/Creator (pandoc) -/Producer (WeasyPrint 54.3) ->> -endobj -3 0 obj -<< -/Type /Catalog -/Pages 1 0 R -/Outlines 1003 0 R -/Names << -/Dests << -/Names [ (TOC) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (api-server) [ 486 0 R /XYZ 41.740157 784.430079 0 ] (apply-security-context-to-your-pods-and-containers-automated) [ 859 0 R /XYZ 84.259843 324.051496 0 ] (authentication-and-authorization) [ 696 0 R /XYZ 41.740157 784.430079 0 ] (cb1) [ 317 0 R /XYZ 84.259843 529.251496 0 ] (cb1-1) [ 317 0 R /XYZ 101.433071 527.554475 0 ] (cb10) [ 321 0 R /XYZ 84.259843 180.731993 0 ] (cb10-1) [ 321 0 R /XYZ 101.433071 179.034971 0 ] (cb100) [ 556 0 R /XYZ 84.259843 700.370079 0 ] (cb100-1) [ 556 0 R /XYZ 101.433071 698.673057 0 ] (cb103) [ 561 0 R /XYZ 84.259843 643.970079 0 ] (cb103-1) [ 561 0 R /XYZ 101.433071 642.273057 0 ] (cb106) [ 566 0 R /XYZ 84.259843 590.570079 0 ] (cb106-1) [ 566 0 R /XYZ 101.433071 588.873057 0 ] (cb109) [ 571 0 R /XYZ 84.259843 519.320079 0 ] (cb109-1) [ 571 0 R /XYZ 101.433071 517.623057 0 ] (cb112) [ 576 0 R /XYZ 84.259843 459.320079 0 ] (cb112-1) [ 576 0 R /XYZ 101.433071 457.623057 0 ] (cb115) [ 581 0 R /XYZ 84.259843 373.070079 0 ] (cb115-1) [ 581 0 R /XYZ 101.433071 371.373057 0 ] (cb118) [ 586 0 R /XYZ 84.259843 294.320079 0 ] (cb118-1) [ 586 0 R /XYZ 101.433071 292.623057 0 ] (cb121) [ 591 0 R /XYZ 84.259843 219.320079 0 ] (cb121-1) [ 591 0 R /XYZ 101.433071 217.623057 0 ] (cb124) [ 596 0 R /XYZ 84.259843 125.570079 0 ] (cb124-1) [ 596 0 R /XYZ 101.433071 123.873057 0 ] (cb127) [ 603 0 R /XYZ 84.259843 799.370079 0 ] (cb127-1) [ 603 0 R /XYZ 101.433071 797.673057 0 ] (cb13) [ 325 0 R /XYZ 84.259843 547.070079 0 ] (cb13-1) [ 325 0 R /XYZ 101.433071 545.373057 0 ] (cb130) [ 608 0 R /XYZ 84.259843 685.370079 0 ] (cb130-1) [ 608 0 R /XYZ 101.433071 683.673057 0 ] (cb133) [ 613 0 R /XYZ 84.259843 575.570079 0 ] (cb133-1) [ 613 0 R /XYZ 101.433071 573.873057 0 ] (cb136) [ 618 0 R /XYZ 84.259843 485.570079 0 ] (cb136-1) [ 618 0 R /XYZ 101.433071 483.873057 0 ] (cb139) [ 623 0 R /XYZ 84.259843 391.820079 0 ] (cb139-1) [ 623 0 R /XYZ 101.433071 390.123057 0 ] (cb142) [ 628 0 R /XYZ 84.259843 279.320079 0 ] (cb142-1) [ 628 0 R /XYZ 101.433071 277.623057 0 ] (cb145) [ 633 0 R /XYZ 84.259843 185.570079 0 ] (cb145-1) [ 633 0 R /XYZ 101.433071 183.873057 0 ] (cb148) [ 638 0 R /XYZ 84.259843 110.570079 0 ] (cb148-1) [ 638 0 R /XYZ 101.433071 108.873057 0 ] (cb151) [ 645 0 R /XYZ 84.259843 799.370079 0 ] (cb151-1) [ 645 0 R /XYZ 101.433071 797.673057 0 ] (cb154) [ 645 0 R /XYZ 84.259843 332.176036 0 ] (cb154-1) [ 645 0 R /XYZ 101.433071 330.479014 0 ] (cb157) [ 653 0 R /XYZ 84.259843 535.251496 0 ] (cb157-1) [ 653 0 R /XYZ 101.433071 533.554475 0 ] (cb16) [ 325 0 R /XYZ 84.259843 195.929014 0 ] (cb16-1) [ 325 0 R /XYZ 101.433071 194.231993 0 ] (cb160) [ 656 0 R /XYZ 84.259843 463.070079 0 ] (cb160-1) [ 656 0 R /XYZ 101.433071 461.373057 0 ] (cb163) [ 659 0 R /XYZ 84.259843 388.070079 0 ] (cb163-1) [ 659 0 R /XYZ 101.433071 386.373057 0 ] (cb166) [ 662 0 R /XYZ 84.259843 298.070079 0 ] (cb166-1) [ 662 0 R /XYZ 101.433071 296.373057 0 ] (cb169) [ 665 0 R /XYZ 84.259843 219.320079 0 ] (cb169-1) [ 665 0 R /XYZ 101.433071 217.623057 0 ] (cb172) [ 670 0 R /XYZ 84.259843 700.370079 0 ] (cb172-1) [ 670 0 R /XYZ 101.433071 698.673057 0 ] (cb175) [ 675 0 R /XYZ 84.259843 565.251496 0 ] (cb175-1) [ 675 0 R /XYZ 101.433071 563.554475 0 ] (cb178) [ 675 0 R /XYZ 84.259843 158.254475 0 ] (cb178-1) [ 675 0 R /XYZ 101.433071 156.557453 0 ] (cb181) [ 681 0 R /XYZ 84.259843 252.651496 0 ] (cb181-1) [ 681 0 R /XYZ 101.433071 250.954475 0 ] (cb184) [ 686 0 R /XYZ 84.259843 246.470079 0 ] (cb184-1) [ 686 0 R /XYZ 101.433071 244.773057 0 ] (cb187) [ 691 0 R /XYZ 84.259843 466.820079 0 ] (cb187-1) [ 691 0 R /XYZ 101.433071 465.123057 0 ] (cb19) [ 329 0 R /XYZ 84.259843 580.820079 0 ] (cb19-1) [ 329 0 R /XYZ 101.433071 579.123057 0 ] (cb190) [ 698 0 R /XYZ 84.259843 595.251496 0 ] (cb190-1) [ 698 0 R /XYZ 101.433071 593.554475 0 ] (cb193) [ 701 0 R /XYZ 84.259843 529.251496 0 ] (cb193-1) [ 701 0 R /XYZ 101.433071 527.554475 0 ] (cb195) [ 704 0 R /XYZ 84.259843 799.370079 0 ] (cb195-1) [ 704 0 R /XYZ 101.433071 797.673057 0 ] (cb198) [ 704 0 R /XYZ 84.259843 429.479014 0 ] (cb198-1) [ 704 0 R /XYZ 101.433071 427.781993 0 ] (cb201) [ 708 0 R /XYZ 84.259843 799.370079 0 ] (cb201-1) [ 708 0 R /XYZ 101.433071 797.673057 0 ] (cb204) [ 708 0 R /XYZ 84.259843 448.229014 0 ] (cb204-1) [ 708 0 R /XYZ 101.433071 446.531993 0 ] (cb207) [ 708 0 R /XYZ 84.259843 112.087950 0 ] (cb207-1) [ 708 0 R /XYZ 101.433071 110.390928 0 ] (cb208) [ 713 0 R /XYZ 84.259843 658.970079 0 ] (cb208-1) [ 713 0 R /XYZ 101.433071 657.273057 0 ] (cb211) [ 713 0 R /XYZ 84.259843 307.829014 0 ] (cb211-1) [ 713 0 R /XYZ 101.433071 306.131993 0 ] (cb214) [ 717 0 R /XYZ 84.259843 643.970079 0 ] (cb214-1) [ 717 0 R /XYZ 101.433071 642.273057 0 ] (cb217) [ 720 0 R /XYZ 84.259843 490.251496 0 ] (cb217-1) [ 720 0 R /XYZ 101.433071 488.554475 0 ] (cb218) [ 720 0 R /XYZ 84.259843 429.304475 0 ] (cb218-1) [ 720 0 R /XYZ 101.433071 427.607453 0 ] (cb22) [ 329 0 R /XYZ 84.259843 229.679014 0 ] (cb22-1) [ 329 0 R /XYZ 101.433071 227.981993 0 ] (cb221) [ 724 0 R /XYZ 84.259843 305.570079 0 ] (cb221-1) [ 724 0 R /XYZ 101.433071 303.873057 0 ] (cb222) [ 724 0 R /XYZ 84.259843 244.623057 0 ] (cb222-1) [ 724 0 R /XYZ 101.433071 242.926036 0 ] (cb225) [ 728 0 R /XYZ 84.259843 455.570079 0 ] (cb225-1) [ 728 0 R /XYZ 101.433071 453.873057 0 ] (cb226) [ 728 0 R /XYZ 84.259843 394.623057 0 ] (cb226-1) [ 728 0 R /XYZ 101.433071 392.926036 0 ] (cb229) [ 732 0 R /XYZ 84.259843 583.970079 0 ] (cb229-1) [ 732 0 R /XYZ 101.433071 582.273057 0 ] (cb230) [ 732 0 R /XYZ 84.259843 523.023057 0 ] (cb230-1) [ 732 0 R /XYZ 101.433071 521.326036 0 ] (cb233) [ 736 0 R /XYZ 84.259843 384.320079 0 ] (cb233-1) [ 736 0 R /XYZ 101.433071 382.623057 0 ] (cb234) [ 736 0 R /XYZ 84.259843 323.373057 0 ] (cb234-1) [ 736 0 R /XYZ 101.433071 321.676036 0 ] (cb237) [ 740 0 R /XYZ 84.259843 174.320079 0 ] (cb237-1) [ 740 0 R /XYZ 101.433071 172.623057 0 ] (cb238) [ 740 0 R /XYZ 84.259843 113.373057 0 ] (cb238-1) [ 740 0 R /XYZ 101.433071 111.676036 0 ] (cb241) [ 744 0 R /XYZ 84.259843 328.820079 0 ] (cb241-1) [ 744 0 R /XYZ 101.433071 327.123057 0 ] (cb242) [ 744 0 R /XYZ 84.259843 267.873057 0 ] (cb242-1) [ 744 0 R /XYZ 101.433071 266.176036 0 ] (cb245) [ 748 0 R /XYZ 84.259843 133.070079 0 ] (cb245-1) [ 748 0 R /XYZ 101.433071 131.373057 0 ] (cb246) [ 751 0 R /XYZ 84.259843 583.970079 0 ] (cb246-1) [ 751 0 R /XYZ 101.433071 582.273057 0 ] (cb247) [ 751 0 R /XYZ 84.259843 523.023057 0 ] (cb247-1) [ 751 0 R /XYZ 101.433071 521.326036 0 ] (cb25) [ 333 0 R /XYZ 84.259843 622.820079 0 ] (cb25-1) [ 333 0 R /XYZ 101.433071 621.123057 0 ] (cb250) [ 755 0 R /XYZ 84.259843 369.320079 0 ] (cb250-1) [ 755 0 R /XYZ 101.433071 367.623057 0 ] (cb251) [ 755 0 R /XYZ 84.259843 308.373057 0 ] (cb251-1) [ 755 0 R /XYZ 101.433071 306.676036 0 ] (cb254) [ 759 0 R /XYZ 84.259843 159.320079 0 ] (cb254-1) [ 759 0 R /XYZ 101.433071 157.623057 0 ] (cb255) [ 759 0 R /XYZ 84.259843 98.373057 0 ] (cb255-1) [ 759 0 R /XYZ 101.433071 96.676036 0 ] (cb258) [ 765 0 R /XYZ 84.259843 715.370079 0 ] (cb258-1) [ 765 0 R /XYZ 101.433071 713.673057 0 ] (cb259) [ 765 0 R /XYZ 84.259843 654.423057 0 ] (cb259-1) [ 765 0 R /XYZ 101.433071 652.726036 0 ] (cb26) [ 333 0 R /XYZ 84.259843 446.073057 0 ] (cb26-1) [ 333 0 R /XYZ 101.433071 444.376036 0 ] (cb262) [ 769 0 R /XYZ 84.259843 485.570079 0 ] (cb262-1) [ 769 0 R /XYZ 101.433071 483.873057 0 ] (cb263) [ 769 0 R /XYZ 84.259843 424.623057 0 ] (cb263-1) [ 769 0 R /XYZ 101.433071 422.926036 0 ] (cb266) [ 777 0 R /XYZ 84.259843 768.620079 0 ] (cb266-1) [ 777 0 R /XYZ 101.433071 766.923057 0 ] (cb266-10) [ 777 0 R /XYZ 99.933071 596.399864 0 ] (cb266-11) [ 777 0 R /XYZ 99.933071 577.452842 0 ] (cb266-12) [ 777 0 R /XYZ 99.933071 482.717735 0 ] (cb266-13) [ 777 0 R /XYZ 99.933071 463.770714 0 ] (cb266-14) [ 777 0 R /XYZ 99.933071 444.823692 0 ] (cb266-15) [ 777 0 R /XYZ 99.933071 425.876671 0 ] (cb266-16) [ 777 0 R /XYZ 99.933071 406.929649 0 ] (cb266-17) [ 777 0 R /XYZ 99.933071 387.982628 0 ] (cb266-18) [ 777 0 R /XYZ 99.933071 350.088585 0 ] (cb266-19) [ 777 0 R /XYZ 99.933071 331.141563 0 ] (cb266-2) [ 777 0 R /XYZ 99.933071 747.976036 0 ] (cb266-20) [ 777 0 R /XYZ 99.933071 198.512413 0 ] (cb266-21) [ 777 0 R /XYZ 99.933071 179.565391 0 ] (cb266-22) [ 777 0 R /XYZ 99.933071 160.618370 0 ] (cb266-23) [ 777 0 R /XYZ 99.933071 103.777305 0 ] (cb266-24) [ 802 0 R /XYZ 99.933071 799.173057 0 ] (cb266-25) [ 802 0 R /XYZ 99.933071 780.226036 0 ] (cb266-26) [ 802 0 R /XYZ 99.933071 761.279014 0 ] (cb266-27) [ 802 0 R /XYZ 99.933071 742.331993 0 ] (cb266-28) [ 802 0 R /XYZ 99.933071 723.384971 0 ] (cb266-29) [ 802 0 R /XYZ 99.933071 704.437950 0 ] (cb266-3) [ 777 0 R /XYZ 99.933071 729.029014 0 ] (cb266-30) [ 802 0 R /XYZ 99.933071 685.490928 0 ] (cb266-31) [ 802 0 R /XYZ 99.933071 666.543907 0 ] (cb266-4) [ 777 0 R /XYZ 99.933071 710.081993 0 ] (cb266-5) [ 777 0 R /XYZ 99.933071 691.134971 0 ] (cb266-6) [ 777 0 R /XYZ 99.933071 672.187950 0 ] (cb266-7) [ 777 0 R /XYZ 99.933071 653.240928 0 ] (cb266-8) [ 777 0 R /XYZ 99.933071 634.293907 0 ] (cb266-9) [ 777 0 R /XYZ 99.933071 615.346885 0 ] (cb267) [ 802 0 R /XYZ 84.259843 607.293907 0 ] (cb267-1) [ 802 0 R /XYZ 101.433071 605.596885 0 ] (cb27) [ 333 0 R /XYZ 84.259843 239.326036 0 ] (cb27-1) [ 333 0 R /XYZ 101.433071 237.629014 0 ] (cb270) [ 813 0 R /XYZ 84.259843 565.251496 0 ] (cb270-1) [ 813 0 R /XYZ 101.433071 563.554475 0 ] (cb273) [ 813 0 R /XYZ 84.259843 233.057453 0 ] (cb273-1) [ 813 0 R /XYZ 101.433071 231.360432 0 ] (cb276) [ 817 0 R /XYZ 84.259843 607.820079 0 ] (cb276-1) [ 817 0 R /XYZ 101.433071 606.123057 0 ] (cb279) [ 817 0 R /XYZ 84.259843 256.679014 0 ] (cb279-1) [ 817 0 R /XYZ 101.433071 254.981993 0 ] (cb282) [ 821 0 R /XYZ 84.259843 643.970079 0 ] (cb282-1) [ 821 0 R /XYZ 101.433071 642.273057 0 ] (cb285) [ 821 0 R /XYZ 84.259843 258.881993 0 ] (cb285-1) [ 821 0 R /XYZ 101.433071 257.184971 0 ] (cb288) [ 825 0 R /XYZ 84.259843 607.820079 0 ] (cb288-1) [ 825 0 R /XYZ 101.433071 606.123057 0 ] (cb291) [ 828 0 R /XYZ 84.259843 514.251496 0 ] (cb291-1) [ 828 0 R /XYZ 101.433071 512.554475 0 ] (cb294) [ 828 0 R /XYZ 84.259843 174.360432 0 ] (cb294-1) [ 828 0 R /XYZ 101.433071 172.663410 0 ] (cb294-10) [ 836 0 R /XYZ 99.933071 723.384971 0 ] (cb294-11) [ 836 0 R /XYZ 99.933071 704.437950 0 ] (cb294-12) [ 836 0 R /XYZ 99.933071 685.490928 0 ] (cb294-13) [ 836 0 R /XYZ 99.933071 628.649864 0 ] (cb294-14) [ 836 0 R /XYZ 99.933071 609.702842 0 ] (cb294-15) [ 836 0 R /XYZ 99.933071 590.755821 0 ] (cb294-16) [ 836 0 R /XYZ 99.933071 571.808799 0 ] (cb294-17) [ 836 0 R /XYZ 99.933071 552.861778 0 ] (cb294-18) [ 836 0 R /XYZ 99.933071 533.914756 0 ] (cb294-19) [ 836 0 R /XYZ 99.933071 514.967735 0 ] (cb294-2) [ 828 0 R /XYZ 99.933071 153.716389 0 ] (cb294-3) [ 828 0 R /XYZ 99.933071 134.769367 0 ] (cb294-4) [ 828 0 R /XYZ 99.933071 115.822346 0 ] (cb294-5) [ 828 0 R /XYZ 99.933071 96.875324 0 ] (cb294-6) [ 836 0 R /XYZ 99.933071 799.173057 0 ] (cb294-7) [ 836 0 R /XYZ 99.933071 780.226036 0 ] (cb294-8) [ 836 0 R /XYZ 99.933071 761.279014 0 ] (cb294-9) [ 836 0 R /XYZ 99.933071 742.331993 0 ] (cb295) [ 836 0 R /XYZ 84.259843 455.717735 0 ] (cb295-1) [ 836 0 R /XYZ 101.433071 454.020714 0 ] (cb30) [ 338 0 R /XYZ 84.259843 628.970079 0 ] (cb30-1) [ 338 0 R /XYZ 101.433071 627.273057 0 ] (cb33) [ 338 0 R /XYZ 84.259843 315.723057 0 ] (cb33-1) [ 338 0 R /XYZ 101.433071 314.026036 0 ] (cb36) [ 342 0 R /XYZ 84.259843 715.370079 0 ] (cb36-1) [ 342 0 R /XYZ 101.433071 713.673057 0 ] (cb39) [ 342 0 R /XYZ 84.259843 398.176036 0 ] (cb39-1) [ 342 0 R /XYZ 101.433071 396.479014 0 ] (cb4) [ 317 0 R /XYZ 84.259843 197.057453 0 ] (cb4-1) [ 317 0 R /XYZ 101.433071 195.360432 0 ] (cb42) [ 346 0 R /XYZ 84.259843 799.370079 0 ] (cb42-1) [ 346 0 R /XYZ 101.433071 797.673057 0 ] (cb45) [ 346 0 R /XYZ 84.259843 467.176036 0 ] (cb45-1) [ 346 0 R /XYZ 101.433071 465.479014 0 ] (cb48) [ 346 0 R /XYZ 84.259843 134.981993 0 ] (cb48-1) [ 346 0 R /XYZ 101.433071 133.284971 0 ] (cb51) [ 351 0 R /XYZ 84.259843 520.070079 0 ] (cb51-1) [ 351 0 R /XYZ 101.433071 518.373057 0 ] (cb54) [ 351 0 R /XYZ 84.259843 203.073057 0 ] (cb54-1) [ 351 0 R /XYZ 101.433071 201.376036 0 ] (cb54-10) [ 358 0 R /XYZ 99.933071 704.437950 0 ] (cb54-11) [ 358 0 R /XYZ 99.933071 685.490928 0 ] (cb54-12) [ 358 0 R /XYZ 99.933071 666.543907 0 ] (cb54-13) [ 358 0 R /XYZ 99.933071 647.596885 0 ] (cb54-14) [ 358 0 R /XYZ 99.933071 628.649864 0 ] (cb54-15) [ 358 0 R /XYZ 99.933071 609.702842 0 ] (cb54-16) [ 358 0 R /XYZ 99.933071 590.755821 0 ] (cb54-17) [ 358 0 R /XYZ 99.933071 571.808799 0 ] (cb54-18) [ 358 0 R /XYZ 99.933071 552.861778 0 ] (cb54-19) [ 358 0 R /XYZ 99.933071 533.914756 0 ] (cb54-2) [ 351 0 R /XYZ 99.933071 182.429014 0 ] (cb54-20) [ 358 0 R /XYZ 99.933071 514.967735 0 ] (cb54-21) [ 358 0 R /XYZ 99.933071 496.020714 0 ] (cb54-22) [ 358 0 R /XYZ 99.933071 477.073692 0 ] (cb54-23) [ 358 0 R /XYZ 99.933071 458.126671 0 ] (cb54-24) [ 358 0 R /XYZ 99.933071 439.179649 0 ] (cb54-25) [ 358 0 R /XYZ 99.933071 420.232628 0 ] (cb54-26) [ 358 0 R /XYZ 99.933071 401.285606 0 ] (cb54-27) [ 358 0 R /XYZ 99.933071 382.338585 0 ] (cb54-28) [ 358 0 R /XYZ 99.933071 363.391563 0 ] (cb54-29) [ 358 0 R /XYZ 99.933071 344.444542 0 ] (cb54-3) [ 351 0 R /XYZ 99.933071 163.481993 0 ] (cb54-30) [ 358 0 R /XYZ 99.933071 325.497520 0 ] (cb54-31) [ 358 0 R /XYZ 99.933071 306.550499 0 ] (cb54-32) [ 358 0 R /XYZ 99.933071 287.603477 0 ] (cb54-33) [ 358 0 R /XYZ 99.933071 268.656456 0 ] (cb54-34) [ 358 0 R /XYZ 99.933071 249.709434 0 ] (cb54-35) [ 358 0 R /XYZ 99.933071 230.762413 0 ] (cb54-36) [ 358 0 R /XYZ 99.933071 211.815391 0 ] (cb54-37) [ 358 0 R /XYZ 99.933071 192.868370 0 ] (cb54-38) [ 358 0 R /XYZ 99.933071 173.921348 0 ] (cb54-39) [ 358 0 R /XYZ 99.933071 154.974327 0 ] (cb54-4) [ 351 0 R /XYZ 99.933071 125.587950 0 ] (cb54-40) [ 358 0 R /XYZ 99.933071 136.027305 0 ] (cb54-41) [ 358 0 R /XYZ 99.933071 117.080284 0 ] (cb54-42) [ 358 0 R /XYZ 99.933071 98.133262 0 ] (cb54-43) [ 398 0 R /XYZ 99.933071 799.173057 0 ] (cb54-44) [ 398 0 R /XYZ 99.933071 780.226036 0 ] (cb54-45) [ 398 0 R /XYZ 99.933071 761.279014 0 ] (cb54-46) [ 398 0 R /XYZ 99.933071 742.331993 0 ] (cb54-47) [ 398 0 R /XYZ 99.933071 723.384971 0 ] (cb54-48) [ 398 0 R /XYZ 99.933071 704.437950 0 ] (cb54-49) [ 398 0 R /XYZ 99.933071 685.490928 0 ] (cb54-5) [ 358 0 R /XYZ 99.933071 799.173057 0 ] (cb54-50) [ 398 0 R /XYZ 99.933071 666.543907 0 ] (cb54-51) [ 398 0 R /XYZ 99.933071 647.596885 0 ] (cb54-52) [ 398 0 R /XYZ 99.933071 628.649864 0 ] (cb54-53) [ 398 0 R /XYZ 99.933071 590.755821 0 ] (cb54-54) [ 398 0 R /XYZ 99.933071 571.808799 0 ] (cb54-55) [ 398 0 R /XYZ 99.933071 552.861778 0 ] (cb54-56) [ 398 0 R /XYZ 99.933071 533.914756 0 ] (cb54-57) [ 398 0 R /XYZ 99.933071 514.967735 0 ] (cb54-58) [ 398 0 R /XYZ 99.933071 496.020714 0 ] (cb54-59) [ 398 0 R /XYZ 99.933071 477.073692 0 ] (cb54-6) [ 358 0 R /XYZ 99.933071 780.226036 0 ] (cb54-60) [ 398 0 R /XYZ 99.933071 458.126671 0 ] (cb54-61) [ 398 0 R /XYZ 99.933071 439.179649 0 ] (cb54-7) [ 358 0 R /XYZ 99.933071 761.279014 0 ] (cb54-8) [ 358 0 R /XYZ 99.933071 742.331993 0 ] (cb54-9) [ 358 0 R /XYZ 99.933071 723.384971 0 ] (cb55) [ 398 0 R /XYZ 84.259843 379.929649 0 ] (cb55-1) [ 398 0 R /XYZ 101.433071 378.232628 0 ] (cb56) [ 398 0 R /XYZ 84.259843 165.485606 0 ] (cb56-1) [ 398 0 R /XYZ 101.433071 163.788585 0 ] (cb56-10) [ 423 0 R /XYZ 99.933071 666.543907 0 ] (cb56-11) [ 423 0 R /XYZ 99.933071 647.596885 0 ] (cb56-12) [ 423 0 R /XYZ 99.933071 628.649864 0 ] (cb56-13) [ 423 0 R /XYZ 99.933071 609.702842 0 ] (cb56-14) [ 423 0 R /XYZ 99.933071 590.755821 0 ] (cb56-15) [ 423 0 R /XYZ 99.933071 571.808799 0 ] (cb56-16) [ 423 0 R /XYZ 99.933071 552.861778 0 ] (cb56-17) [ 423 0 R /XYZ 99.933071 533.914756 0 ] (cb56-18) [ 423 0 R /XYZ 99.933071 514.967735 0 ] (cb56-19) [ 423 0 R /XYZ 99.933071 496.020714 0 ] (cb56-2) [ 398 0 R /XYZ 99.933071 144.841563 0 ] (cb56-20) [ 423 0 R /XYZ 99.933071 477.073692 0 ] (cb56-21) [ 423 0 R /XYZ 99.933071 458.126671 0 ] (cb56-22) [ 423 0 R /XYZ 99.933071 439.179649 0 ] (cb56-23) [ 423 0 R /XYZ 99.933071 420.232628 0 ] (cb56-24) [ 423 0 R /XYZ 99.933071 401.285606 0 ] (cb56-25) [ 423 0 R /XYZ 99.933071 382.338585 0 ] (cb56-26) [ 423 0 R /XYZ 99.933071 363.391563 0 ] (cb56-27) [ 423 0 R /XYZ 99.933071 344.444542 0 ] (cb56-28) [ 423 0 R /XYZ 99.933071 325.497520 0 ] (cb56-29) [ 423 0 R /XYZ 99.933071 306.550499 0 ] (cb56-3) [ 398 0 R /XYZ 99.933071 125.894542 0 ] (cb56-30) [ 423 0 R /XYZ 99.933071 287.603477 0 ] (cb56-31) [ 423 0 R /XYZ 99.933071 268.656456 0 ] (cb56-32) [ 423 0 R /XYZ 99.933071 249.709434 0 ] (cb56-33) [ 423 0 R /XYZ 99.933071 230.762413 0 ] (cb56-34) [ 423 0 R /XYZ 99.933071 211.815391 0 ] (cb56-35) [ 423 0 R /XYZ 99.933071 192.868370 0 ] (cb56-36) [ 423 0 R /XYZ 99.933071 173.921348 0 ] (cb56-37) [ 423 0 R /XYZ 99.933071 154.974327 0 ] (cb56-38) [ 423 0 R /XYZ 99.933071 136.027305 0 ] (cb56-39) [ 423 0 R /XYZ 99.933071 117.080284 0 ] (cb56-4) [ 423 0 R /XYZ 99.933071 799.173057 0 ] (cb56-40) [ 423 0 R /XYZ 99.933071 98.133262 0 ] (cb56-41) [ 462 0 R /XYZ 99.933071 799.173057 0 ] (cb56-42) [ 462 0 R /XYZ 99.933071 780.226036 0 ] (cb56-43) [ 462 0 R /XYZ 99.933071 761.279014 0 ] (cb56-44) [ 462 0 R /XYZ 99.933071 742.331993 0 ] (cb56-45) [ 462 0 R /XYZ 99.933071 723.384971 0 ] (cb56-46) [ 462 0 R /XYZ 99.933071 704.437950 0 ] (cb56-47) [ 462 0 R /XYZ 99.933071 685.490928 0 ] (cb56-48) [ 462 0 R /XYZ 99.933071 666.543907 0 ] (cb56-49) [ 462 0 R /XYZ 99.933071 647.596885 0 ] (cb56-5) [ 423 0 R /XYZ 99.933071 761.279014 0 ] (cb56-50) [ 462 0 R /XYZ 99.933071 628.649864 0 ] (cb56-51) [ 462 0 R /XYZ 99.933071 609.702842 0 ] (cb56-52) [ 462 0 R /XYZ 99.933071 590.755821 0 ] (cb56-53) [ 462 0 R /XYZ 99.933071 552.861778 0 ] (cb56-54) [ 462 0 R /XYZ 99.933071 533.914756 0 ] (cb56-55) [ 462 0 R /XYZ 99.933071 514.967735 0 ] (cb56-56) [ 462 0 R /XYZ 99.933071 496.020714 0 ] (cb56-57) [ 462 0 R /XYZ 99.933071 477.073692 0 ] (cb56-58) [ 462 0 R /XYZ 99.933071 458.126671 0 ] (cb56-59) [ 462 0 R /XYZ 99.933071 439.179649 0 ] (cb56-6) [ 423 0 R /XYZ 99.933071 742.331993 0 ] (cb56-60) [ 462 0 R /XYZ 99.933071 420.232628 0 ] (cb56-61) [ 462 0 R /XYZ 99.933071 401.285606 0 ] (cb56-7) [ 423 0 R /XYZ 99.933071 723.384971 0 ] (cb56-8) [ 423 0 R /XYZ 99.933071 704.437950 0 ] (cb56-9) [ 423 0 R /XYZ 99.933071 685.490928 0 ] (cb57) [ 462 0 R /XYZ 84.259843 342.035606 0 ] (cb57-1) [ 462 0 R /XYZ 101.433071 340.338585 0 ] (cb60) [ 486 0 R /XYZ 84.259843 565.251496 0 ] (cb60-1) [ 486 0 R /XYZ 101.433071 563.554475 0 ] (cb61) [ 486 0 R /XYZ 84.259843 343.504475 0 ] (cb61-1) [ 486 0 R /XYZ 101.433071 341.807453 0 ] (cb64) [ 492 0 R /XYZ 84.259843 241.820079 0 ] (cb64-1) [ 492 0 R /XYZ 101.433071 240.123057 0 ] (cb67) [ 497 0 R /XYZ 84.259843 178.070079 0 ] (cb67-1) [ 497 0 R /XYZ 101.433071 176.373057 0 ] (cb7) [ 321 0 R /XYZ 84.259843 565.820079 0 ] (cb7-1) [ 321 0 R /XYZ 101.433071 564.123057 0 ] (cb70) [ 504 0 R /XYZ 84.259843 799.370079 0 ] (cb70-1) [ 504 0 R /XYZ 101.433071 797.673057 0 ] (cb73) [ 509 0 R /XYZ 84.259843 670.370079 0 ] (cb73-1) [ 509 0 R /XYZ 101.433071 668.673057 0 ] (cb76) [ 514 0 R /XYZ 84.259843 560.570079 0 ] (cb76-1) [ 514 0 R /XYZ 101.433071 558.873057 0 ] (cb79) [ 519 0 R /XYZ 84.259843 481.820079 0 ] (cb79-1) [ 519 0 R /XYZ 101.433071 480.123057 0 ] (cb82) [ 524 0 R /XYZ 84.259843 406.820079 0 ] (cb82-1) [ 524 0 R /XYZ 101.433071 405.123057 0 ] (cb85) [ 529 0 R /XYZ 84.259843 316.820079 0 ] (cb85-1) [ 529 0 R /XYZ 101.433071 315.123057 0 ] (cb86) [ 529 0 R /XYZ 84.259843 110.073057 0 ] (cb86-1) [ 529 0 R /XYZ 101.433071 108.376036 0 ] (cb89) [ 537 0 R /XYZ 84.259843 772.370079 0 ] (cb89-1) [ 537 0 R /XYZ 101.433071 770.673057 0 ] (cb90) [ 537 0 R /XYZ 84.259843 535.623057 0 ] (cb90-1) [ 537 0 R /XYZ 101.433071 533.926036 0 ] (cb91) [ 537 0 R /XYZ 84.259843 298.876036 0 ] (cb91-1) [ 537 0 R /XYZ 101.433071 297.179014 0 ] (cb94) [ 544 0 R /XYZ 84.259843 200.570079 0 ] (cb94-1) [ 544 0 R /XYZ 101.433071 198.873057 0 ] (cb97) [ 551 0 R /XYZ 84.259843 799.370079 0 ] (cb97-1) [ 551 0 R /XYZ 101.433071 797.673057 0 ] (cis-v1.6-kubernetes-benchmark---rancher-v2.6-rke2-with-kubernetes-v1.21-up-to-v1.23) [ 306 0 R /XYZ 84.259843 784.970079 0 ] (client-certificate-authentication-should-not-be-used-for-users-manual) [ 696 0 R /XYZ 84.259843 684.651496 0 ] (configure-image-provenance-using-imagepolicywebhook-admission-controller-manual) [ 855 0 R /XYZ 84.259843 684.651496 0 ] (consider-external-secret-storage-manual) [ 853 0 R /XYZ 84.259843 568.851496 0 ] (contentsbox) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (controller-manager) [ 653 0 R /XYZ 41.740157 784.430079 0 ] (controls) [ 313 0 R /XYZ 84.259843 570.381496 0 ] (create-administrative-boundaries-between-resources-using-namespaces-manual) [ 859 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-a-minimal-audit-policy-is-created-automated) [ 698 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-a-unique-certificate-authority-is-used-for-etcd-manual) [ 691 0 R /XYZ 84.259843 652.220079 0 ] (ensure-that-all-namespaces-have-network-policies-defined-automated) [ 828 0 R /XYZ 84.259843 318.510432 0 ] (ensure-that-default-service-accounts-are-not-actively-used.-automated) [ 775 0 R /XYZ 84.259843 206.451496 0 ] (ensure-that-encryption-providers-are-appropriately-configured-automated) [ 643 0 R /XYZ 84.259843 220.970079 0 ] (ensure-that-service-account-tokens-are-only-mounted-where-necessary-manual) [ 802 0 R /XYZ 84.259843 449.446885 0 ] (ensure-that-the---anonymous-auth-argument-is-set-to-false-manual) [ 486 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the---audit-log-maxage-argument-is-set-to-30-or-as-appropriate-automated) [ 586 0 R /XYZ 84.259843 464.720079 0 ] (ensure-that-the---audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate-automated) [ 591 0 R /XYZ 84.259843 389.720079 0 ] (ensure-that-the---audit-log-maxsize-argument-is-set-to-100-or-as-appropriate-automated) [ 596 0 R /XYZ 84.259843 295.970079 0 ] (ensure-that-the---audit-log-path-argument-is-set-automated) [ 581 0 R /XYZ 84.259843 558.470079 0 ] (ensure-that-the---authorization-mode-argument-includes-node-automated) [ 519 0 R /XYZ 84.259843 652.220079 0 ] (ensure-that-the---authorization-mode-argument-includes-rbac-automated) [ 524 0 R /XYZ 84.259843 577.220079 0 ] (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated) [ 514 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated-1) [ 724 0 R /XYZ 84.259843 520.970079 0 ] (ensure-that-the---auto-tls-argument-is-not-set-to-true-automated) [ 681 0 R /XYZ 84.259843 408.051496 0 ] (ensure-that-the---basic-auth-file-argument-is-not-set-automated) [ 486 0 R /XYZ 84.259843 528.904475 0 ] (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated) [ 668 0 R /XYZ 84.259843 133.670079 0 ] (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated-1) [ 675 0 R /XYZ 84.259843 313.654475 0 ] (ensure-that-the---cert-file-and---key-file-arguments-are-set-as-appropriate-automated) [ 681 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated) [ 628 0 R /XYZ 84.259843 464.720079 0 ] (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated-1) [ 728 0 R /XYZ 84.259843 670.970079 0 ] (ensure-that-the---client-cert-auth-argument-is-set-to-true-automated) [ 681 0 R /XYZ 84.259843 538.851496 0 ] (ensure-that-the---encryption-provider-config-argument-is-set-as-appropriate-automated) [ 638 0 R /XYZ 84.259843 295.970079 0 ] (ensure-that-the---etcd-cafile-argument-is-set-as-appropriate-automated) [ 633 0 R /XYZ 84.259843 370.970079 0 ] (ensure-that-the---etcd-certfile-and---etcd-keyfile-arguments-are-set-as-appropriate-automated) [ 618 0 R /XYZ 84.259843 670.970079 0 ] (ensure-that-the---event-qps-argument-is-set-to-0-or-a-level-which-ensures-appropriate-event-capture-manual) [ 751 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the---hostname-override-argument-is-not-set-manual) [ 748 0 R /XYZ 84.259843 333.470079 0 ] (ensure-that-the---insecure-bind-address-argument-is-not-set-automated) [ 561 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the---insecure-port-argument-is-set-to-0-automated) [ 566 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the---kubeconfig-kubelet.conf-file-ownership-is-set-to-rootroot-automated) [ 708 0 R /XYZ 84.259843 603.629014 0 ] (ensure-that-the---kubeconfig-kubelet.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 704 0 R /XYZ 84.259843 214.987950 0 ] (ensure-that-the---kubelet-certificate-authority-argument-is-set-as-appropriate-automated) [ 507 0 R /XYZ 84.259843 164.720079 0 ] (ensure-that-the---kubelet-client-certificate-and---kubelet-client-key-arguments-are-set-as-appropriate-automated) [ 502 0 R /XYZ 84.259843 277.220079 0 ] (ensure-that-the---kubelet-https-argument-is-set-to-true-automated) [ 497 0 R /XYZ 84.259843 333.470079 0 ] (ensure-that-the---make-iptables-util-chains-argument-is-set-to-true-automated) [ 744 0 R /XYZ 84.259843 544.220079 0 ] (ensure-that-the---peer-auto-tls-argument-is-not-set-to-true-automated) [ 686 0 R /XYZ 84.259843 416.870079 0 ] (ensure-that-the---peer-cert-file-and---peer-key-file-arguments-are-set-as-appropriate-automated) [ 686 0 R /XYZ 84.259843 708.470079 0 ] (ensure-that-the---peer-client-cert-auth-argument-is-set-to-true-automated) [ 686 0 R /XYZ 84.259843 547.670079 0 ] (ensure-that-the---profiling-argument-is-set-to-false-automated) [ 576 0 R /XYZ 84.259843 614.720079 0 ] (ensure-that-the---profiling-argument-is-set-to-false-automated-1) [ 656 0 R /XYZ 84.259843 633.470079 0 ] (ensure-that-the---profiling-argument-is-set-to-false-automated-2) [ 675 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the---protect-kernel-defaults-argument-is-set-to-true-automated) [ 740 0 R /XYZ 84.259843 389.720079 0 ] (ensure-that-the---read-only-port-argument-is-set-to-0-automated) [ 732 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the---request-timeout-argument-is-set-as-appropriate-automated) [ 601 0 R /XYZ 84.259843 239.720079 0 ] (ensure-that-the---root-ca-file-argument-is-set-as-appropriate-automated) [ 665 0 R /XYZ 84.259843 389.720079 0 ] (ensure-that-the---rotate-certificates-argument-is-not-set-to-false-manual) [ 759 0 R /XYZ 84.259843 389.720079 0 ] (ensure-that-the---secure-port-argument-is-not-set-to-0-automated) [ 571 0 R /XYZ 84.259843 689.720079 0 ] (ensure-that-the---service-account-key-file-argument-is-set-as-appropriate-automated) [ 613 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the---service-account-lookup-argument-is-set-to-true-automated) [ 606 0 R /XYZ 84.259843 164.720079 0 ] (ensure-that-the---service-account-private-key-file-argument-is-set-as-appropriate-automated) [ 662 0 R /XYZ 84.259843 483.470079 0 ] (ensure-that-the---streaming-connection-idle-timeout-argument-is-not-set-to-0-automated) [ 736 0 R /XYZ 84.259843 614.720079 0 ] (ensure-that-the---terminated-pod-gc-threshold-argument-is-set-as-appropriate-automated) [ 653 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated) [ 623 0 R /XYZ 84.259843 577.220079 0 ] (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated-1) [ 755 0 R /XYZ 84.259843 614.720079 0 ] (ensure-that-the---token-auth-file-parameter-is-not-set-automated) [ 492 0 R /XYZ 84.259843 427.220079 0 ] (ensure-that-the---use-service-account-credentials-argument-is-set-to-true-automated) [ 659 0 R /XYZ 84.259843 558.470079 0 ] (ensure-that-the-admin.conf-file-ownership-is-set-to-rootroot-automated) [ 338 0 R /XYZ 84.259843 157.876036 0 ] (ensure-that-the-admin.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 338 0 R /XYZ 84.259843 471.123057 0 ] (ensure-that-the-admission-control-plugin-alwaysadmit-is-not-set-automated) [ 529 0 R /XYZ 84.259843 280.473057 0 ] (ensure-that-the-admission-control-plugin-alwayspullimages-is-set-manual) [ 535 0 R /XYZ 84.259843 202.220079 0 ] (ensure-that-the-admission-control-plugin-eventratelimit-is-set-manual) [ 529 0 R /XYZ 84.259843 502.220079 0 ] (ensure-that-the-admission-control-plugin-namespacelifecycle-is-set-automated) [ 544 0 R /XYZ 84.259843 370.970079 0 ] (ensure-that-the-admission-control-plugin-noderestriction-is-set-automated) [ 554 0 R /XYZ 84.259843 183.470079 0 ] (ensure-that-the-admission-control-plugin-podsecuritypolicy-is-set-automated) [ 549 0 R /XYZ 84.259843 295.970079 0 ] (ensure-that-the-admission-control-plugin-securitycontextdeny-is-set-if-podsecuritypolicy-is-not-used-manual) [ 537 0 R /XYZ 84.259843 736.023057 0 ] (ensure-that-the-admission-control-plugin-serviceaccount-is-set-automated) [ 537 0 R /XYZ 84.259843 499.276036 0 ] (ensure-that-the-anonymous-auth-argument-is-set-to-false-automated) [ 720 0 R /XYZ 84.259843 720.651496 0 ] (ensure-that-the-api-server-only-makes-use-of-strong-cryptographic-ciphers-manual) [ 645 0 R /XYZ 84.259843 547.576036 0 ] (ensure-that-the-api-server-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 317 0 R /XYZ 84.259843 352.457453 0 ] (ensure-that-the-api-server-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 317 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-audit-policy-covers-key-security-concerns-manual) [ 698 0 R /XYZ 84.259843 418.457453 0 ] (ensure-that-the-certificate-authorities-file-permissions-are-set-to-644-or-more-restrictive-manual) [ 708 0 R /XYZ 84.259843 252.487950 0 ] (ensure-that-the-client-certificate-authorities-file-ownership-is-set-to-rootroot-automated) [ 713 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the-cluster-admin-role-is-only-used-where-required-manual) [ 775 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-cni-in-use-supports-network-policies-automated) [ 828 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-container-network-interface-file-ownership-is-set-to-rootroot-manual) [ 333 0 R /XYZ 84.259843 586.473057 0 ] (ensure-that-the-container-network-interface-file-permissions-are-set-to-644-or-more-restrictive-manual) [ 333 0 R /XYZ 84.259843 763.220079 0 ] (ensure-that-the-controller-manager-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 321 0 R /XYZ 84.259843 351.131993 0 ] (ensure-that-the-controller-manager-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 321 0 R /XYZ 84.259843 736.220079 0 ] (ensure-that-the-controller-manager.conf-file-ownership-is-set-to-rootroot-automated) [ 346 0 R /XYZ 84.259843 290.381993 0 ] (ensure-that-the-controller-manager.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 346 0 R /XYZ 84.259843 622.576036 0 ] (ensure-that-the-etcd-data-directory-ownership-is-set-to-etcdetcd-automated) [ 338 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the-etcd-data-directory-permissions-are-set-to-700-or-more-restrictive-automated) [ 333 0 R /XYZ 84.259843 409.726036 0 ] (ensure-that-the-etcd-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 329 0 R /XYZ 84.259843 385.079014 0 ] (ensure-that-the-etcd-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 329 0 R /XYZ 84.259843 736.220079 0 ] (ensure-that-the-kubelet---config-configuration-file-has-permissions-set-to-644-or-more-restrictive-automated) [ 713 0 R /XYZ 84.259843 463.229014 0 ] (ensure-that-the-kubelet---config-configuration-file-ownership-is-set-to-rootroot-automated) [ 717 0 R /XYZ 84.259843 799.370079 0 ] (ensure-that-the-kubelet-only-makes-use-of-strong-cryptographic-ciphers-manual) [ 769 0 R /XYZ 84.259843 745.970079 0 ] (ensure-that-the-kubelet-service-file-ownership-is-set-to-rootroot-automated) [ 701 0 R /XYZ 84.259843 394.260432 0 ] (ensure-that-the-kubelet-service-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 701 0 R /XYZ 84.259843 684.651496 0 ] (ensure-that-the-kubernetes-pki-certificate-file-permissions-are-set-to-644-or-more-restrictive-manual) [ 351 0 R /XYZ 84.259843 362.223057 0 ] (ensure-that-the-kubernetes-pki-directory-and-file-ownership-is-set-to-rootroot-automated) [ 351 0 R /XYZ 84.259843 675.470079 0 ] (ensure-that-the-kubernetes-pki-key-file-permissions-are-set-to-600-manual) [ 398 0 R /XYZ 84.259843 324.635606 0 ] (ensure-that-the-proxy-kubeconfig-file-ownership-is-set-to-rootroot-manual) [ 704 0 R /XYZ 84.259843 584.879014 0 ] (ensure-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) [ 668 0 R /XYZ 84.259843 294.470079 0 ] (ensure-that-the-scheduler-pod-specification-file-ownership-is-set-to-rootroot-automated) [ 325 0 R /XYZ 84.259843 351.329014 0 ] (ensure-that-the-scheduler-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 325 0 R /XYZ 84.259843 702.470079 0 ] (ensure-that-the-scheduler.conf-file-ownership-is-set-to-rootroot-automated) [ 342 0 R /XYZ 84.259843 221.381993 0 ] (ensure-that-the-scheduler.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) [ 342 0 R /XYZ 84.259843 538.576036 0 ] (ensure-that-the-seccomp-profile-is-set-to-dockerdefault-in-your-pod-definitions-manual) [ 859 0 R /XYZ 84.259843 604.851496 0 ] (etcd-node-configuration-files) [ 681 0 R /XYZ 41.740157 784.430079 0 ] (extensible-admission-control) [ 855 0 R /XYZ 41.740157 784.430079 0 ] (general-policies) [ 859 0 R /XYZ 41.740157 784.430079 0 ] (header_bottom_text) [ 6 0 R /XYZ 85.009843 392.718898 0 ] (if-proxy-kubeconfig-file-exists-ensure-permissions-are-set-to-644-or-more-restrictive-manual) [ 701 0 R /XYZ 84.259843 263.460432 0 ] (kubelet) [ 720 0 R /XYZ 41.740157 784.430079 0 ] (logging) [ 698 0 R /XYZ 41.740157 784.430079 0 ] (master-node-configuration-files) [ 317 0 R /XYZ 41.740157 784.430079 0 ] (minimize-access-to-create-pods-manual) [ 775 0 R /XYZ 84.259843 307.251496 0 ] (minimize-access-to-secrets-manual) [ 775 0 R /XYZ 84.259843 523.851496 0 ] (minimize-the-admission-of-containers-wishing-to-share-the-host-ipc-namespace-automated) [ 817 0 R /XYZ 84.259843 763.220079 0 ] (minimize-the-admission-of-containers-wishing-to-share-the-host-network-namespace-automated) [ 817 0 R /XYZ 84.259843 412.079014 0 ] (minimize-the-admission-of-containers-wishing-to-share-the-host-process-id-namespace-automated) [ 813 0 R /XYZ 84.259843 388.457453 0 ] (minimize-the-admission-of-containers-with-added-capabilities-manual) [ 825 0 R /XYZ 84.259843 431.026036 0 ] (minimize-the-admission-of-containers-with-allowprivilegeescalation-automated) [ 821 0 R /XYZ 84.259843 799.370079 0 ] (minimize-the-admission-of-containers-with-capabilities-assigned-manual) [ 825 0 R /XYZ 84.259843 315.226036 0 ] (minimize-the-admission-of-containers-with-the-net_raw-capability-manual) [ 825 0 R /XYZ 84.259843 763.220079 0 ] (minimize-the-admission-of-privileged-containers-automated) [ 813 0 R /XYZ 84.259843 720.651496 0 ] (minimize-the-admission-of-root-containers-automated) [ 821 0 R /XYZ 84.259843 429.281993 0 ] (minimize-wildcard-use-in-roles-and-clusterroles-manual) [ 775 0 R /XYZ 84.259843 423.051496 0 ] (network-policies-and-cni) [ 828 0 R /XYZ 41.740157 784.430079 0 ] (overview) [ 306 0 R /XYZ 84.259843 704.822079 0 ] (pod-security-policies) [ 813 0 R /XYZ 41.740157 784.430079 0 ] (prefer-using-secrets-as-files-over-secrets-as-environment-variables-manual) [ 853 0 R /XYZ 84.259843 684.651496 0 ] (rbac-and-service-accounts) [ 775 0 R /XYZ 41.740157 784.430079 0 ] (scheduler) [ 675 0 R /XYZ 41.740157 784.430079 0 ] (secrets-management) [ 853 0 R /XYZ 41.740157 784.430079 0 ] (testing-controls-methodology) [ 306 0 R /XYZ 84.259843 169.826079 0 ] (the-default-namespace-should-not-be-used-manual) [ 859 0 R /XYZ 84.259843 193.251496 0 ] (the-v1.5.1-guide-skips-5.6-and-goes-from-5.5-to-5.7.-we-are-including-it-here-merely-for-explanation.) [ 857 0 R /XYZ 41.740157 784.430079 0 ] (title-block-header) [ 6 0 R /XYZ 84.259843 502.818898 0 ] (verify-that-the-rotatekubeletservercertificate-argument-is-set-to-true-manual) [ 763 0 R /XYZ 84.259843 187.970079 0 ] (worker-node-configuration-files) [ 701 0 R /XYZ 41.740157 784.430079 0 ] ] ->> ->> ->> -endobj -4 0 obj -<< -/ExtGState << -/a1.0 << -/ca 1 ->> -/A1.0 << -/CA 1 ->> ->> -/XObject << ->> -/Pattern << ->> -/Shading << ->> -/Font 1024 0 R ->> -endobj -5 0 obj -<< -/Filter /FlateDecode -/Length 2633 ->> -stream -xœ½š]‹\¹†ïûWôõ‚µ%Ué Ã6!†…, „r±ì]Â8àÍEþ~Þ:Ýúèé×k‡xèñ9U:R©ôT•zâYðó"âW³ZëµØùñÝéýIBÍ‚G‹››Cý>5Ÿ£X 9öÒòù ´†Úsªåüt¯g·{ó§Ó_¾9ýëö™xN¡÷’b>ÿöÆU¦¡¶Z%U‡J¢UΡ֔ Q•ðŠ½3ÌÈÕ’Ý«*Ì°Þ2SÁ K©Uƒj5꽪ÌT£1̈=W¢Š;b¡:"fšˆ.æP0é…éR ¥õÔ©®Bkb:ÕPªÅ™®‡RJnLg°%w‰L—aKŽfÂt°Å´U¦+°E‹ -ÕÁ–Ôª2]…- ³Ó‰®ÁxW£:Ø"E"Óu ¹73ªëÐI«í^—$‡ÜT…éb ¾©T‡…E*•èl)’;ÕÁ–¬’˜Na‹á‡éÀ‰¬µ5²[“ïnL—aKJÕ¨¶`ñR-R CvPp.1¤Zƒõd ©i°fQ µ¬VeH=+½2¨DèRd P-Ù -#FØbعT[´c&Ø‚Œª°%™2(hkðÆnMzd,PðÖ$Æpa¦0(ˆ«­c¹p¥ÎX `®‚Œ -èb»WÆu5—ÄX À®Z+Œ -îªEa,0pWORl3€ŽAcSÆw5Je,0pÛ(1`Uü°0ÀWBjM ÜMMŒ±ÀÀÝT¨®ìAe,0p7e¼‹ÜànÊ’ ÜM– -c%€?B³pÉ<€-,câ=tI ÐYHHK 2¸;ÂÙÓ Uˆ­Æ‚ îƆ壺bÅŒ“}‹Y±”ÎXÁ]*e,Èà.ðÙ Àø€ˆš 2¸‘šéÀݘº0dpyCf,€;„ˆ€±ñ-DëR]Ò[c,Èà.°” …ƒ4„êj€#EÆÀ8òPÆ‚î"¼wÆ‚îJVe,(à®Òªƒ-Úcòè°mÉžF²$LîJÄ -3¸+R;cAw=l2”ê‚Ô ``@œ J³Ðk-Œ¥ tHù¨®ÜgÆ‚*d ‹ê`‹Uc,@bºöÆXPlÑ” j‚-H „‘}Óò¶À]h}`°™"cHZ7c,¨¹ ì2]I¡Uàšêt±2Ôj¡82 – - ›ž±€ž.1À è¢1Ô[°U ŠBKÈÌ™.Â$ÙŒÈt¡‹‘± %Ø‚l± ¥†*³c"4jIL5Ó™xÙ š•à©"c¶ *M’ê` 0ÁX€õñb;3 øA±‰2‡ê`‹fc,h-y¹Ù Zƒ-Ic|ÓëÍÊXж ãc,èR‚§ÉŒÀ8t"Œ=6Ô›É °/½ÞlŒ]õ&J<²o»¯7+cA7Ø’‘¢PlAÒÔɾ“Po‚NLW`KêÆXÐKñz³1 Í@½‰Œž 4¯7+=.ws™˜ÜEfZ pƒz™-«íàÍùW6¯8=1 7#¶Ò#{­*=3À×k$zh3QY¶BO V¡ŒB üE>fwL¸œ>íGTîP3[õ—ì؆ì~Ö«™~ Ùå|ì·_NßþŒà{þåߧ·ß'eã'ÂùÕŠÖÅÏyàDMÎïv¹‚˜ÐÚù rý”|ëçz¢ÆÊÈÇ£ ezÆvé©G7š †òæ¾Óð¯žï/î^ü~>^=œì"}á«…º#\7!µÏç ™êùáÝéÛ~|õý«¿bÅÞžþöWøpÉ7ø4ã(ùq ˜§G\–Ë}vª>XÔÅ—6hkx&CfhgÿÀ5TX1É®{óòïç‡×0 -™Îf”ÛåÒ¯éýÚ[|úËl‡îeŠø?yßåò9ƾ^ã^mÁô^ìû×ÞÞ.zè]Ž¾>fWyn—æËØÇ{mïzŒ—·ùÉc~.ýÿð€eºx(|\Px†€€Ý6‡÷Í¿úŒ&nkík{Däëq/ß×Ñ}WÇ玂q9.€xð£iÄ6DäƒNsw“×øÓÿüÓÿÃMƲ°å¿Èÿ7Ë~í?ÕR¾?•#3‰GuŠ*Õš_]A}múÙ³!¤JÇÒÅc²ãùO‡þïg«ë÷âGž^¾$Øn®íß‹À!“ðù÷ìërf'‡‰I~.ÞŠŸ5Ý‹òô™b^/eö…:=“=ž"¼¸?o‰B\®—}v®y\‚ e=ŸÇcK‘.¦qçûà20âÉïÅþöâUìçÞ-èõ®LSPô)\ÚÞvu¸ü¤8ªlÂ|/”m¸%N6ì‰^'Í©+K8Mߧ.nË?Ú¦º-çÇ„' «Ï¥OTê§m£«1»ÈF†L7£ú”æÑr.ˆÍu"¢Ç¾Ñµa1§~[5¯Y.Â6Éf»>ÚÑ㉶»ö–çJÕÙª­Gû˜ùeÊrumsâ‰ì˜£«´úyÛ—·ââ+0Ö äò›t'|h¿±‡ýÅÛë«×ýjr;À¥íSú´K狹tKõ¸-··3æèÙoÆæ]34ûÒmïéLrÖÖX©…ÝÜì>ºIÛzìÚ ïƒÚÝ :ÅÄd2d³Ç:l*×ÇÆ´oÜè,Õ™S,é¯ûLç}Uæ$æ}UÊ][ïwD¼™-éL€òîÎu½¬–3—lDVö§§ÔVK²û±ívìmÏÛkØ2¤¬ÌTÖcù®åñïKö=˜b­ÆÏq:Û:ôC >bôø;.i¥€!æ_ÅUkÛqÉï~æãç'÷§$ãüd ËLUjô¿Èã&> -endobj -7 0 obj -<< -/Filter /FlateDecode -/Length 3679 ->> -stream -xœí\[ܶ~Ÿ_1Ï¢ðvxŠ 61  AчõÔNPìpúпߒHJœ3»ã]7vÇÆÚÒ§#òð;7’’Vþ~©ñOtzŠ1‡Ãûƒš)ü™%º“ùòøá ¹‰L JGô¤B NZÙÉ:ï(pü¾ÇUûvþӇߺ›4#EƒCÒ“ŽZ+s$?ùd’¶Çßß>B:0i5ùü'÷¿ÿrøêºù÷áÝóÐ}0SRDÚ SÚ“›‡ A¢ÏÃõ·¦áh¾¶Ãñu¸í&['ü¦†£1¦w¸íµ 5Ñ­Ÿ÷~²­m†ÇÖJ뇫4ׯ¡§n< Ïä8b‹ÃBUºõØ¡ž·m†w¸j­´Sc¾{½¹±;;„Þž¶8wh¢Ìíì`ÎØáiÒ¦Ú¡ôCì ÏØA÷vÐn‹s—n=vhg}ÆšÛA™j‡ÒccjÌ÷ !šlþsÜ\Ñ6š‡3¡{>9Î-q Ñ[ˆÞBt¢×VQ¢\“~ô~0¢[ˆÞBô¢/¢ãP|úD÷\èÞBô¢·}¶‰n ãmxŠÉŒ«(Çw!#›M³›Þ‡hÃyp% PÓ(f¸54¥AˆZ¸om¥öhM*ÒL?†v®Áð.D΂‹õØ¡ÄÛ6C‚;\µVZˆ6¦Æ|÷.ÍÝÙ!ôv(ÓðÎeÒä÷!ÚÛÁœ±ƒáv»µ€ÚÛ¡á]ˆ2œ¹4ë±C;;è3v`QGÏC´15æûB”æáLèžÅΛ™%n!z Ñ[ˆîBôʵh¢\ó~ô~0¢[ˆÞBô¢/¢ãP|úD÷\èÞBô¢·}¶‰®óã*Úð>I«(Ç·!j]`³ivSûe8.—›H¬ã>D#•í•>D#Ûi=‚xÚ»FC{×hx¢ çÁÙVC‰·m†s¼ž÷ؘóÝëÍÝÙÁ÷v(ÓðÎeÒdv!º±ƒ9c†#ìC4©‘Ú§á}ˆ&¶µÖ\ºõØ¡ô;hn³ QÆÔ˜ï Ñ6š‡3¡{>;of–¸…è-Do!º Ñk×¢*#Èx3¹(y^&N.F¬ ÜäBRJâ0ªÉ!ÄÄaôQdt¦ü2€$N$pHðtØÌGCÒÐÙXga‡ ~¦Ià tÆ°’À!Á‘m‚' -Âw&x˜õ‡äò‹,A+CB[¢8$2“¥ˆð‘d ³KVIzèì”r‡ùå´$ƒÏ©Ž´Äa„ÎÚ'q˜S¦2) -9’›L²Áz¥&ÂL’ñ“ ¤“À¡×f2ž’8ôHx†0õ8ô:»`“À¡·ÐÙFmá](1zCï ³NˆTI:kÓ 2•VAðgï1£Iù IÆO:r‡¨Ó:X%Cœ´wJKFTHôD‡y“Ý‘“jŠOÐÙ¢t Ì•´¤ÀaPÐYŸê VÑI5Î>©”´TSê Š)I5CFy©¦ÔA´•jJ@T$pPå(”d ³C&“8D„§Z-qˆ:¨ )©¦ÔA¥)D!'ÔA¥^ß|Èñù‚Í#Ë^2mº¾ùˆÅÞ%Ó~HóÈÿ—L{w}ó)‡ñ‚ÍGsÑ´}óæ݂~™'ì7XPc­‹vnüÛ¾ùú›Ÿ[ãÆ+,HðƒjD(¡.¶cZc™¥2­ç ¸œö ‰É87lNJYü¯P-)­#ɵ1-˜y :Z;¦¥óù¸(³Þo¨)瀹78Æ%Ì­æûÜÛEN«åKë5&_ú¨úP¯O¾'ëdârm–»Ã}§UŸ7OÑ[$¯«d™¯ŽZŽçþs[v=‹±v|RK¼ãþúÂ62ü<œ»w—tïkXõPÝt¶Å.žñ±òiõÚÏŠÏ|2]° -˜ùvwëØ‹Ãj‡Un¾¯È¾ÛpnV½1Esy—`ÝÌÆâÉÖ|çÃLïcJDx¦Pl™kU°ÃÊàpÍÚ%æëž9MÚ8¥mäÙŒe™Ù‰OÚÞ½]‡¤¬×ß1-ëO­ò~ÕƒÝ[­TÜ踊D'zæ6;kX~Ô:Æ:fÕÎkŸ%°|sÎÙО95yõfá™b ’šVG›9=‰zÇü¢d§÷Ö'è3ñ ³$¶š´hÕáÍzïšpŠ¯\礶|–$3ÿdøç{>{V»$ÛÓ?=Lkrbþ^ “[Ý'ˆ|ú-ŸËkaþ$r@~’beŸ ÿ'>ó¤!qš·zN%?ù¸9"¯Íµlïø<öžÇÌ&¢Y·y¹µõg1ÈoŽèOpàSÚÎ]¶þ>ø(õ?è-—7¶™Kó¹»gÊåeáç™>™8ÞÖg‘Ó°åôÇQÑ¥¹ß›OÌöϳ­³"¿fËï3Ù-¿¡#ÛíôLk°íΗ[¹cü– ‹ÂáÌñïuÃÉœXE´¶çK„šÃŽÛP“-›»ÎõÍ-<È~Ÿ®^˜—åÿígâ ¥'»åô)õøJ;z{i>åžo¯m”‹]`9È1ìÔÚ/þ[ìIe_æ%ëíÝâ¿Å‡—¾E.ã–ËÿE½]A”§ h~L}$w‡§È??Y>üö‘<Öìü?¿Þª™H'HŸyv¼öžŸäkáÁk~dÝowy÷:X>«íû?êùÅSŸI¬î0‡~Zpã˜KÅv<·S¡l9¬{â³9C3áuÏЮºTþ µNrlV¸'‰ã¾f7b@»Ðd]:„곜/Õ@>'Ý*]ëPFÃC*êå$U“GëêuÎ)¬ðSs%UQÝ5˜ª+ßwpÓ“j¨æ÷±WÐkæîTÛm°¯¹óžÃ¡Z\{WÑê€g>_ÀÓ\ØÜήK' 6ªV»Ö›bH - niô~†K´Öì¤Sª‡-ÈF÷tøõ0—9cɤü]«_†çS/}üd^áç_wü¡¼à·Ôq£j¶05·v0¯dÆØb‹°1©‚5lhKû Ö±ÉRñ†ZÇaùÚµÛUKìER× ªëURãÁ¦º®Ñ–1CxÚ°X'2j›V°—S[‡nò[—U+œðáà4¶k¶•ä#kgY´•t\u†ÝˆÞ(NY‡©iÖ©™’wR©Ïú;n -›ûb'l‘Om»æZ™åMlqšÝìt×°Ó¼ÓÔé3Púêðú`»¯*û­©½¢÷­ËhMxcÔΧ¾?­5ÇóI ÞÆPm˲سu’ÓB£M-\wÂ}”¡±Ý¶ -ïԬ봊š.ò§@«-†¢“_o+´Ì#ŽC–BS4ôWÎ4q«T‰[Åïds»¥âÕÙ’­ âîÚèT“¬sÉ8À<¿»¢®IÚ‚íûv}ߥÚÙ0\SÄ·™©j·ÑNòªüwMì ß»IuÛ#:Öm åó—·Á{Lyº½GY¾Àr‘í—<ùye¿MR6PxO0íV ;£Âý!,5L¾ÀšZ¿q?s[RÃ!=€³ßŒßË·“]?ýŸw‰êiÈKEJÑÙ<éE'Ô™×ôi·öã½w -endstream -endobj -8 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 7 0 R -/Resources 4 0 R -/Annots [ 9 0 R 10 0 R 11 0 R 12 0 R 13 0 R 14 0 R 15 0 R 16 0 R 17 0 R 18 0 R 19 0 R 20 0 R 21 0 R 22 0 R 23 0 R 24 0 R 25 0 R 26 0 R 27 0 R 28 0 R 29 0 R 30 0 R 31 0 R 32 0 R 33 0 R 34 0 R 35 0 R 36 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -9 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 671.061024 504.679134 638.361024 ] -/BS << -/W 0 ->> -/Dest (cis-v1.6-kubernetes-benchmark---rancher-v2.6-rke2-with-kubernetes-v1.21-up-to-v1.23) ->> -endobj -10 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.750423 648.111024 496.429134 633.111024 ] -/BS << -/W 0 ->> -/Dest (cis-v1.6-kubernetes-benchmark---rancher-v2.6-rke2-with-kubernetes-v1.21-up-to-v1.23) ->> -endobj -11 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 626.361024 504.679134 593.661024 ] -/BS << -/W 0 ->> -/Dest (controls) ->> -endobj -12 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 485.690364 618.411024 496.429134 603.411024 ] -/BS << -/W 0 ->> -/Dest (controls) ->> -endobj -13 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 581.661024 504.679134 548.961024 ] -/BS << -/W 0 ->> -/Dest (master-node-configuration-files) ->> -endobj -14 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 485.521907 573.711024 496.429134 558.711024 ] -/BS << -/W 0 ->> -/Dest (master-node-configuration-files) ->> -endobj -15 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 530.961024 498.679134 507.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -16 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 510.561024 491.179134 495.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -17 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 491.361024 498.679134 467.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -18 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 470.961024 491.179134 455.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -19 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 451.761024 498.679134 428.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -20 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 431.361024 491.179134 416.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -21 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 412.161024 498.679134 388.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -22 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 391.761024 491.179134 376.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -23 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 372.561024 498.679134 349.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -24 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 352.161024 491.179134 337.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -25 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 332.961024 498.679134 309.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -26 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 312.561024 491.179134 297.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -27 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 293.361024 498.679134 269.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -28 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 272.961024 491.179134 257.961024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -29 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 253.761024 498.679134 230.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -30 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 233.361024 491.179134 218.361024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-pod-specification-file-ownership-is-set-to-rootroot-automated) ->> -endobj -31 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 214.161024 498.679134 190.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -32 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 193.761024 491.179134 178.761024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -33 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 174.561024 498.679134 151.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-ownership-is-set-to-rootroot-manual) ->> -endobj -34 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 154.161024 491.179134 139.161024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-container-network-interface-file-ownership-is-set-to-rootroot-manual) ->> -endobj -35 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 134.961024 498.679134 111.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-permissions-are-set-to-700-or-more-restrictive-automated) ->> -endobj -36 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 114.561024 491.179134 99.561024 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-permissions-are-set-to-700-or-more-restrictive-automated) ->> -endobj -37 0 obj -<< -/Filter /FlateDecode -/Length 3697 ->> -stream -xœí\YÇ ~Ÿ_1Ï\.’Å:A@$±ˆ~0ò %ÁÊ€œ‡üýÝ]GÏtsö VÂîv³¿f±ø‘¬ª¾àèåÿ· ¿r—sI1OÞ%öòoB¬v¦Ã?ÊÏø 9=›ò‘;_°?Œò˜]œÅw"ÎWă–»ÃOß~ßVGt¥ÄÀþøÇ»’‚‹™³ÉÞÅ” ‰hA -º(b"ær -à÷!àÅ^ödb@ „¾X±˜"˜“±xŸ-ŒØŒP|20$6CÈ&&ˆÍ>%-ŒØì}41ŒŽ ±g “çLL ŽS! Lò‚AôFì@ŠŽ#ƒ‰Éb3gïè,6³/&¦ˆÍ²7B ½ØL1y#~ЋÍX¢‰±1#~P²”ÉÄ Øì3#ÆÄfPŒøAŠ.”àML@r”c&»J61ƒ©ñƒÑ»9#~0ŠÍœÙÄ$±™!#~0‰Í!‰Éb3%,FüHª ÆC±â§ˆÍBˆ…!/6—bÄy±ÙË@ba@lö²?Rå•mLt”g#ƈP0>d#~ˆ²£$¿-LŽbÄlÄT'G\Àƈ͌>1FQl¡d#~H†@¢”MŒŒD>e#~Ä7Ž$ÉlŒØ ‘³c$àDOÈVüÈ8(1D&È8ˆ…1ñ#©ÃœÁÆDÁ€OFü1QIFü%Ÿ³‰‘q¹¤dÄOq£›s2â'È8( 6Fl& dÄXqPj4&#~‚Œƒ lŒØ Þ'#~‚Œƒè±Ø˜ì pNFŒÉ8àd¼LɈ™¦ -¢‰N:ÅɈ–qP´#6GOш1–qP†JŒFü°Œƒ2#6Sñш–qŠ›e0ŒFü°Œƒ²›¢?,ã ÌG£›½T<#~dž%Óü.0Ë”c™ æIÀe Ke2øie²V;¢y=/fþøõðÝpþøë¿ï¿™–5QL*ž%Ž,£blKÍê.—E‹¨­Í -¤ÉcLUÏJ>÷¬Ÿ$•£72œ4ÈE™ä"\È%Ì;ºÈDé\:4|:Œrb)³M÷ O]Kk1R©è±S]zZ9¡ËÕ9²F¸ƒÌ$ºµ¸’ò¨7<ȇÞ-žÚö÷iÍCÜá!®yˆ|._…Lv%œKÏxÀyv>4–£"—§èsŠ>§è“&º«;DÓ-ô!ÝWÓK9ÍÄaż#mÜZëgÁ‘œ_2Ÿ¯®…)‚³‰‰Ž(fF“ÑçŒ`a²^ÀMÓå=LÑ Ê˜¸ìc’÷.`H˜-Lt!Ä8ÝlØúsŒ&&K’úHÑÀè À8ݬÚÃè  -<]€ÛÅDÇ9\¤€ŽSf0¸H!ëM¸-  œÀ_)êMÊÐà+ŨÃe‹/©½„‡…É.'´¸ÈÁ%B,EopŒ&&º”"’ÁW–:JFoð•½$"zH_‚Ë!\ˆ—c0¸È(5(GŸ- i-Ξ ¾²”ÌBÞƒÁWÁ†’ ¾2K­KTØà+st¥pAƒ‹õ@ÊÅâ"ç©ähaôƒ™,¾²Tdq€·øÊIg)Y|r€)‹ Ü!”2:H³…pP(ƒ¯"›,½70H)q6ø’¡Bo~H 20:¸f`4¸(y -§%$GÈóÅâ= “£|Ix9’AÃ|•ÈŽdL_” @,.’LûˆÉ[\drRå1™˜¢7°1X|Öê_ iìi¾K¹JŽ¥Î³Á†Ì†È±Ì©Ñ C@Åq.¾˜ dõvÂ9iwúìd¸ ‘.q!Âp>1á¼ ß½ðqãП_ÊüÌT‘é¥ÞZF¯£³—bã‹zâõ‡ÃwúþÕ÷?ý ]yýþðó qxŸ·/ÿy|ýêð—×¢ü¡ê£Ì;?zhdèÿtêe®¨?=^½>)€ŸP}"“Z–z¼ú ‘oQûTõ)›ÔªúðxõL`RûTõ2w³¨Uõñ~ê!.:Æ© tq‰º°&/“(màÕ_øÛ?~-ò ؉¥êù ê£Xª>=¡â{´3Kõ¿Yéß{\§'1}C:Yü£L3w+&œýÕÇe€Ù{Kûúd#³.çc,KìðV~¼½’-ý;Ét©¹`ôo=Gÿ6Ù©ëW¼gœe|Zôëþ›Ù¦°Ø£¶Mmêù´è;ÍòéœØí¨6ªž-ûà—sÙìóm¿êºÝ±êK/v`X|Uí÷Có⇓©W#¯ônroÄQ˜mšì<ͺ´‰\|] ôæl¿ôÓÀƒï~Ÿ|~êçW.&;Fû n hNÍ?û%.õOÝÙôe<÷emŸkŒñlk¹ÉîÒm¼§2W×9M_§7αé|\~–[Ë,¿ÉšõÌo·ácz@ø:ù6|Lþ‰]6q“»ß™_JÒ¿øJ<Œçþ2‰'³=‡×‰._+Ñ_U6ž;óF$éâþ:IonT¿×@ÔýÉïﻟÁ}¥ª¾ùª2Ötjºpê½3»ËçÏØ ·Ò®ÃÛÿå`¸YV›ŽÄsG^Ïþ›Éå>sžÓmˆÄw/!¾XÊ-~…î·ÌŠ§éù9=l Ò }QnÓ¹o?/·úÀü•ùÓt-ë+á6çL¹MÌ”°»Âª1s£ [âìq±UÿÚ,ö´øääo’¿3O¦ã¹¿ ŸÒ¡ÝeU%ôFÁV¦=~:0L}ëµi`;Oôú×—JàñÜaXg]zq&¼ÄòbŽ$U^MÇ—uúcØ[îÁÖû¼‰Å2}@)ËìôÊ›K…1Å陦Ãï÷hçR!—¦§–t -Ž¡ÄùÁ¨§––Öõy50•¹ÞòÐEA"Þ\ /K¥6ÔòQ-}Öô£ó·†???ðá ßRô;>àxÞ±~nè\|'Ô—úÖûJôÙšû‰U¸lú¦+@6Ùé¤okœ!IŸ¼›7KS®„Λúì_?Ÿëi4ç5Ö=M„¹á ¡œáR¬½×—{f1O_ˆ¢e/6Sô‰ã&ì -Èí -û‰WŲ²„|)ôCs],yY*ÛfiœvÓG×Á@Åbè´„§ƒ4KçÒ»M)B“aõ®~¯§Ú7Uš”+²O"}SæBø­¯ªƒÙŽ¬¡Ë‹0×@ - W*nCt:lâmܘJ •û©¥z¾›ÒCrsü†lòÑ"•2q7삧º¹Ge ríó7x!<´î\ˆïœ6ÐÜk…çz£½Mœ}—.M¹º*B“éãwú{.ܤZy¦ }Öª‚q¯7¢¹ÙÄk'òتNØò„Ô±ƒ» ¥³£¿d±µ×¢ÛÐÍc+éà²AžJ5O‹T+k1´¨kb-aÕ?­VŽB_溘[(ß­ÄÝNn©*Á^‹0„;7½][í¼Å©1®Ü«èã@b¾ -OÓÀÎÅú¹6ÒÅèÛhׄÝ0) -]ÜËèÝ$®ÙÚªè;ËfO²­÷tøí0 sHŒEß´Õo.­_ãƒãóJ~þuÇ¿×GËçq\¿Ç·´­¶®ÄãH†H•Â«bÄÒ„-m!†¡ìw1äŽå mGêuè§û^Ø+Z…vtr÷á ƺ4x† áéÌ‹m"óTnú€=ïRíœ4Cú©¢j•ìŒÝ‘ÝÜQ’ƒJÙShÒA?b8œ(;£RÙ,ݲ•™c#Íõj© mkئºKýXèÃ쨺ʦ^ ÃÉVŠŒ–•=F?:½žÌûbr¬ ÒVÚ›ôn”¶Ž©´¼m)M»q½ÛÆ:êNMÞ‹†Ü£6Éé©Ñ§aµ3Æè Íý´Àc£¡ÊVÖ¢• }§ºÊšÆTmŠËiÕ-Só¦—R›Stéo£§yd¥9‘GVâVõ͖֯¨M€x çÔ{ç;²Í%ó†,Žg7ièHª²Ë¶Ãºí:Ú‡n„nHì3SßOã ä£êßcrgóÁó-ÔêòävYÃëÇR!Ò±èt'Ç(5D:"«æ‡ë%>Ǿ€ry™¤^@[}k\¿¡ o;&ÜÉrýÀ jyë~ç4}y£KsÀî[mÛ¯ £åcÓU¢¶›t©È%ëÛå>K’H~çÎq¼¸öãíP -endstream -endobj -38 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 37 0 R -/Resources 4 0 R -/Annots [ 39 0 R 40 0 R 41 0 R 42 0 R 43 0 R 44 0 R 45 0 R 46 0 R 47 0 R 48 0 R 49 0 R 50 0 R 51 0 R 52 0 R 53 0 R 54 0 R 55 0 R 56 0 R 57 0 R 58 0 R 59 0 R 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R 65 0 R 66 0 R 67 0 R 68 0 R 69 0 R 70 0 R 71 0 R 72 0 R 73 0 R 74 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -39 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-ownership-is-set-to-etcdetcd-automated) ->> -endobj -40 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-etcd-data-directory-ownership-is-set-to-etcdetcd-automated) ->> -endobj -41 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -42 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -43 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -44 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admin.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -45 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -46 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -47 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -48 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-scheduler.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -49 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -50 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -51 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -52 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-controller-manager.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -53 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-directory-and-file-ownership-is-set-to-rootroot-automated) ->> -endobj -54 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-directory-and-file-ownership-is-set-to-rootroot-automated) ->> -endobj -55 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 473.570079 498.679134 450.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-certificate-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -56 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 453.170079 491.179134 438.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-certificate-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -57 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 433.970079 498.679134 410.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-key-file-permissions-are-set-to-600-manual) ->> -endobj -58 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 413.570079 491.179134 398.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubernetes-pki-key-file-permissions-are-set-to-600-manual) ->> -endobj -59 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 394.370079 504.679134 361.670079 ] -/BS << -/W 0 ->> -/Dest (api-server) ->> -endobj -60 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.910335 386.420079 496.429134 371.420079 ] -/BS << -/W 0 ->> -/Dest (api-server) ->> -endobj -61 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 343.670079 498.679134 320.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---anonymous-auth-argument-is-set-to-false-manual) ->> -endobj -62 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 338.270079 491.179134 323.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---anonymous-auth-argument-is-set-to-false-manual) ->> -endobj -63 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 304.070079 498.679134 280.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---basic-auth-file-argument-is-not-set-automated) ->> -endobj -64 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 298.670079 491.179134 283.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---basic-auth-file-argument-is-not-set-automated) ->> -endobj -65 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 264.470079 498.679134 241.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---token-auth-file-parameter-is-not-set-automated) ->> -endobj -66 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 259.070079 491.179134 244.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---token-auth-file-parameter-is-not-set-automated) ->> -endobj -67 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 224.870079 498.679134 201.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-https-argument-is-set-to-true-automated) ->> -endobj -68 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 219.470079 491.179134 204.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-https-argument-is-set-to-true-automated) ->> -endobj -69 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 185.270079 498.679134 161.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-client-certificate-and---kubelet-client-key-arguments-are-set-as-appropriate-automated) ->> -endobj -70 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 164.870079 491.179134 149.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-client-certificate-and---kubelet-client-key-arguments-are-set-as-appropriate-automated) ->> -endobj -71 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 145.670079 498.679134 122.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-certificate-authority-argument-is-set-as-appropriate-automated) ->> -endobj -72 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 125.270079 491.179134 110.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubelet-certificate-authority-argument-is-set-as-appropriate-automated) ->> -endobj -73 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 106.070079 498.679134 82.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated) ->> -endobj -74 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 85.670079 491.179134 70.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated) ->> -endobj -75 0 obj -<< -/Filter /FlateDecode -/Length 3501 ->> -stream -xœå\M䶽÷¯è³Ó,²ŠdHb øä°Ûޱ̰sÈßÏ£ZüènMõÌtO‚Mv1»R©T¢Þ#«)ihïñ÷KÂ?…É•¢9ñþði÷ëλ,“åð÷ø™ÿeWÏŽeO>—Bðœ÷Ÿæ f Jqÿs¹bžÃ<î~øb÷Ëv,ÚGç½R ûß>Î.™]& -¹<íRpcAÃ%¹ÌÅûü´‹—SŠ™ —âráÓÓ.äÙ (?íCÞ•@œÄòI®D-A Ÿ\‘BÊ–¨ÈIÄÀ"Ú¬¬Á Ø;ðŠ îÈi¤$$Á)z!YJqšJ,”ØiI™- 3့0ÌÙù9[`ükaXÔù䃷0Tq>I†Á£Íš4ŸH50 Ř’a uãƒaâ(û¨F‘•’ÅÀ0ÄìÐn"ƒ‹ÀÑáz\ «ƒ_aƒ‹ ârY&r¡xÉ mVŒ Ã]¤½ÅEV#h·0,⢰J.æÕÂP³YØ~Ú'úè‚aôê8d$ÇÄ1KÃÈ1Rs00Œ!;.AŠaŒÑ!m uX>ê„ #‹“˜S60ŒBN$y6ús”ì$3{ Ä6kÈÙâ"©KD- ³ „âÖ-. -¹Ä¹$ Ã’]J ÃÕðÑèRaQ CU—=ÒÁ{AQFÁ00d"—#2½!c3 ª!‡è2$r@›rW(¢Œ>U`¡‡‘!36Y‘ ‰®¤ØÂPb.¡#>IœzFÅ0|PQÜñÇòɉ.©Åê H÷ÉÂuP!’‚Åê *† !:»ó€Y .Äç‘Xƒ¡;H€R .uÙ£´X>ÉyAƒ#ŸÔÒª‹£ Ñ«Âìˆc¶Ôªˆw”jEµ|ê²gK¯JB›µK¯ -dz€˜²ô*r¡ QÄÒ«Æ·^,½Š"àB† -°0„TŠnhôgV¾¨¥W1°jO´ôj‚VGÍÈ–^MÐê1¡\@Ž¸ˆánéÕ­Îa .´:“’¥Wчî#Ñà"A«³ˆZzBÞA‡dp‘ Õ!’¥W´zUá–^…8rè<ÑÒ«)''HT–^EJu‚”iéU'TX*;ÈùbéÕì=ê2&»ÖôÕ'—¢Š¥W3—éÐÀ0cJû–^ÍmV¨-C(U̧£·ô*&¸¨Ë$†è˜Qk¶ô*fܘRoéUL0§Fo4¸È óe…ha˜0_†²Q‹‹Œù2“Xz5gÌ—E5X\Ì—¡$,½šmÖ”,½šóeoéÕ‚:¨¨Ü–^-¨ƒ*H †uP³'K¯ÔA”¶ôjVGç)–^-Ðê˜TK¯huT&±ô*d(æÔUÐ>Ðê YzÌ©«2| Õ±ë-½Z Õ)Öfø@«“Ôäkø@«S®‰ÃòÁ_k§7| Õƒ¯„Y>˜/‡z±§}Z= XzU¡Õ‘åÕÒ«¨J˜S‡héUèPW³¥WZ#Ù[zU¡Õ1*¢¥W¡g1§–léUL 0§f²ôªB«G…<´0„VgŒ.K¯*´:æ!ÁÒ« -­ŽùƒXzU¡Õ¡ûÕÒ« -­Î…£¥WZ:;YzU¡Õ%†êÓ>ä=&̵oX‹ƒÞcÆ,IØr"L™s"{‰•Ðne$NË)`ÒLÈÑç@®kÐ ׂZ„Vh«Jçbä í0¼eÛû¸¾þÛO»¯Þ#7íúçîá‹u¥ýòÐÞíô¸|¯Ðd‹FôUÛy]šŒtôîÓî«ßóÝ7?|‹y÷°ûÛïêê)~_ÿ}ÿî»Ýß!øKÃ'ÌbðP6į¨Š';|º!|ÊÇõ«· QÕðå†ð9šÔÖðúúðŒ~Ú[ÂC]£öÃëÃK$“ZÌS)ÞÊÍ¢¶†—ׇO1›Ô–ÛzŠ¦Im ÿþõá1ë6©-·åœ edQ«·åÈq“Z½-)h}.s%ü ÔV‰aQ«·Q‹â›Lnõ‚ÛÍؘ¡5‡å oúRÓoÇfüT‰ ëÿ¼÷€¡,Ïæ¼ –IÝÆñúˆ¶¤†óqûĶúàÔågñ)ã8æÇXüÿ㈱ì‡ã5ëyõQxÝçÉýæÐ/Æ­ù®qÛúÃÝóccºå‘~ s}Vý9~Ë9q_c{Ñ,?÷ãQë·R×y¼Ó<÷&×5ÙeÎóq½~|NùÕœÊ9–ÿ%Nëw×9½ÓùN»þÿñÆçÊåëø¤úŠOúº>¬lD³ Uõ5ž’ŠP]esY%ä´¼ø³ûå9.õƲF———{öT¿²Rx?ñrÏzõú2]<0M+P ™££ÅŽ5×WgëóÒïþôíŸÿúýx^º<³úxìg½Ž¦ãúplêÔGÛš€ Þ—1ÿa‚NŽ9 õ¿y}kÑ#z´/Ч©¬Û[}*¬ým¡q¢°ÇoÏ„¦ö´õ´~m|]¾‘òëNP2.õµDÂ8²Ô×rCqy±ïë7Xɧõ½®úZׂ6íŸ8uùæù×îµ~íDÒÂûã7ÒÓÎê|ÐþiGË;áõzEý­nÁ\_9?7?‚z­oó_˜¹¾‚ònZu[hèßmqj”v«4ÏNwž6L‡Ý†ñKßB×ïWúñ‰µàÊj,­#q÷Óæ·a:ì6ýÖhÒ™ÊÝ«ŒSµ!?š2ºz,ø Û‚ÑjEšxœvÉǶyjN•Æµ/S¿ Æîæ óãNò†·Œ\á¥u½Ù§¸Ý\üè/ÚKƒ*Q·¥}âúï1UH·Ö̳lÔF¨˜÷ÆEêØìæSf{êY‡· ÎÃw‚«~‰/ìˆ]ü†MºÝ»#vb ›ìY[ój’êÉãÔL½×usMa Ÿž+g£oÀå†YzW~<1vJªèìíb‰¦î.=î0§ž;gsîŒ×_ÐnbÔ4õùf<,…Ï͵ë—v‘a¾W»q±É8†¤0Ì#>.æ6Z{v"Õ¾9ÙVÅ=ì~Þ-e.Äú¹!ºgýV)žH/Úÿ >ßáç;Þÿ¥½}¬ãÁ÷lzn=1Ï•,„Ø(¼jA»±›ÉxJûÃLeøJë ½Ž‡0åk§û‘Ø›'õ Jýèl”ád½ëÖß1Ãx8C± ™[¹û¸ÛÍùú{&ö½CÕùv°[Ʊ8Jò~ -‰½ê:J:Žr˜NÄλÓu´ì¤™µCÎéÐ×öóLE¬×šv&Qwã8Æ£ÌΡ›m¹‹É…i:™é$0Ó|Q=iÏF£_=¼næ}mrj# Z{jïÖÇÙÚo¬Z{ÂÛ¶Æe7îöš—_QÒï@¨ÇŠÓØ‹]䌡1¤ŸìÌ}t²–qÚê óE¹ÙN.Ú -EOõ N³õˆ¹µ)­§5X–;.›(å®)†õçi™Yé ÊÌJºð­q[Åëj)v$swÎãîüðìZ²lØÒ|v·òðŒÍvym>½v«¶ûé6x4$ eêÇiráùªü÷š±³ù†ö–×Éò•¾¬áQ¨™r q¯Uî”ú«‚0X꬙˴^òâsì”Ëe’¶€2_ t`Òë¯7øôTq@¯˜B­¿€ì‰ÓÔoÞÒËxòã¯í÷飋˟u•¨ïæ:U-õ³;_0È•²§Íõ"ÊKaßÿáà£% -endstream -endobj -76 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 75 0 R -/Resources 4 0 R -/Annots [ 77 0 R 78 0 R 79 0 R 80 0 R 81 0 R 82 0 R 83 0 R 84 0 R 85 0 R 86 0 R 87 0 R 88 0 R 89 0 R 90 0 R 91 0 R 92 0 R 93 0 R 94 0 R 95 0 R 96 0 R 97 0 R 98 0 R 99 0 R 100 0 R 101 0 R 102 0 R 103 0 R 104 0 R 105 0 R 106 0 R 107 0 R 108 0 R 109 0 R 110 0 R 111 0 R 112 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -77 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-node-automated) ->> -endobj -78 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-node-automated) ->> -endobj -79 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-rbac-automated) ->> -endobj -80 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-includes-rbac-automated) ->> -endobj -81 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-eventratelimit-is-set-manual) ->> -endobj -82 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-eventratelimit-is-set-manual) ->> -endobj -83 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwaysadmit-is-not-set-automated) ->> -endobj -84 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwaysadmit-is-not-set-automated) ->> -endobj -85 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwayspullimages-is-set-manual) ->> -endobj -86 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-alwayspullimages-is-set-manual) ->> -endobj -87 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-securitycontextdeny-is-set-if-podsecuritypolicy-is-not-used-manual) ->> -endobj -88 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-securitycontextdeny-is-set-if-podsecuritypolicy-is-not-used-manual) ->> -endobj -89 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-serviceaccount-is-set-automated) ->> -endobj -90 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-serviceaccount-is-set-automated) ->> -endobj -91 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-namespacelifecycle-is-set-automated) ->> -endobj -92 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-namespacelifecycle-is-set-automated) ->> -endobj -93 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 473.570079 498.679134 450.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-podsecuritypolicy-is-set-automated) ->> -endobj -94 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 453.170079 491.179134 438.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-podsecuritypolicy-is-set-automated) ->> -endobj -95 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 433.970079 498.679134 410.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-noderestriction-is-set-automated) ->> -endobj -96 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 413.570079 491.179134 398.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-admission-control-plugin-noderestriction-is-set-automated) ->> -endobj -97 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 394.370079 498.679134 370.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-bind-address-argument-is-not-set-automated) ->> -endobj -98 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 373.970079 491.179134 358.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-bind-address-argument-is-not-set-automated) ->> -endobj -99 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 354.770079 498.679134 331.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-port-argument-is-set-to-0-automated) ->> -endobj -100 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 349.370079 491.179134 334.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---insecure-port-argument-is-set-to-0-automated) ->> -endobj -101 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 315.170079 498.679134 291.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---secure-port-argument-is-not-set-to-0-automated) ->> -endobj -102 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 309.770079 491.179134 294.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---secure-port-argument-is-not-set-to-0-automated) ->> -endobj -103 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 275.570079 498.679134 252.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated) ->> -endobj -104 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 270.170079 491.179134 255.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated) ->> -endobj -105 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 235.970079 498.679134 212.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-path-argument-is-set-automated) ->> -endobj -106 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 230.570079 491.179134 215.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-path-argument-is-set-automated) ->> -endobj -107 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 196.370079 498.679134 172.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxage-argument-is-set-to-30-or-as-appropriate-automated) ->> -endobj -108 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 175.970079 491.179134 160.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxage-argument-is-set-to-30-or-as-appropriate-automated) ->> -endobj -109 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 156.770079 498.679134 133.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate-automated) ->> -endobj -110 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 136.370079 491.179134 121.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxbackup-argument-is-set-to-10-or-as-appropriate-automated) ->> -endobj -111 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 117.170079 498.679134 93.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxsize-argument-is-set-to-100-or-as-appropriate-automated) ->> -endobj -112 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 96.770079 491.179134 81.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---audit-log-maxsize-argument-is-set-to-100-or-as-appropriate-automated) ->> -endobj -113 0 obj -<< -/Filter /FlateDecode -/Length 3643 ->> -stream -xœí\Ý‹ä6ï¿¢ŸqT¥RI‚¸ƒ»K.°‡p3ÍnÂ1ØÜÃýûW²-©Üm—w¦½ù€™¡gìòÏ¥R}J²Úpvòû%ÈŸD0¤”#Óùòáôñ䆜üŒˆÅÉxù{ùè_€8”»}:‡—1ƒ?ÐtNOä'!§²âòtúá‹Ó/ë¬àŒCÎLÁ}§!‘N!YäŽ)°É8° ˆˆ"Û†€yƒó&D`òè²…‰=ƒ‰A³sɈÌÙEãEf dbHdv1:¶0"³slb!ûà‚…ICHL&†i1{G&:Á :Ãw ò8€‰I"sHÎÞId.›˜,2“OÎp1t"³çè ÿA'2cf"3"eÃP¢4@ð&Ef—0>†^dvÙðô;6ü'HÙƈÌR ÙðŸ uPN#þ¤Êx”mŒÈì$ãþ#ã,æ3Ý`æ!ÿÊ4¡ˆ'—Ä-‹%É3“%[ëèi2óëO§¯`pçŸþ{zÿÅ8­a)» pR¹Me4©;]&-¶6+FgŽ•Ï‚>õ¬ß$™£7¢nRta&±7tqóŽÎ2Pº¦ª†/'M÷AÒlã­è±si-²Ï­;Õ©—…:½(Gæ7t‘DC·Ô y㪂]õ^µ¨4µ®ïËÒ¼a^ÚÃ5}á2iÈtM½²nØAÑÓà¨Ùan‘K¹µC§.íÐé“€®éÊ¥U‹ ê°a­÷ªE¥©u}_NnȾüœoìU½ù°ºÛ¡¸ðfe‰×} Ñ×½ Q.?ñ|{ðŒÕ.{Ï+=z Ñ×} ÑÏ¢ë¡øüîV辆èkˆ¾†è]ÝÅ"@£j®[–ršˆjƼA׌V­õ»àì‡ìR¼ž]‹¥¼ÌÓÇÓM >Gd00 ¢L>X˜Tðø@k S?-tn`¢+¢B†ÌQlÆ噡!såEpÞ2GQvd¤hÈ‘†ÄÁ’Ù»!±h‰- ²êlaHâ-´·‰ÉrÁ{J¦¬¥ÈDÃ0FΖÌÓsŠ–ÌÑèÑ‘%s”D›Á’9…Á{IÉ–ÌŸR`KfI¦D.zæÉù–œ·1y;±“ a'vJbÚ‰„qˆ;±“¼âNì$)hi'v…!íÄN -Rwb'É÷;±“$G¹ØçÜNì$ÉQ°;IrìÄN’…;±“$GáNìˆ$ƒß‰,9ÊïÄN–E;±“%GÑNìäòw'vryˆ»;YrïÄŽÓ¾§c#yu'vr !îÄNfÉ«;±“YòêNìä2ŽÙ‰,9ÊíÄN–åvb'KŽ‚ØÉ’£`'vÀI’ÂàP{ÝDÏͲ|.¨uy‰„Á­®ËkxX‡o.Ì\¹ô×·§ßÁ^Ê/í°|9ûòè öòñw°—eÚÂ>¼œ=a0M[ØÇ;ØÇdšöñ>Ó¦iï3mH´kÚ˧±ž÷òØc,ûSËL×;€±7ÿîÿú^5p¹Ï¸ÑÙu/{oÖå¾´À¬Ë}®™Ø¬Â>½œ};°.÷y~f;° -û‡;2¾C;²nù¯»=ÎnãÖHWuƒ£ˆàÖy“| ¸¦ú_F­Þ9L2¸Z`ù”c¹FiºdÎK’2m´#·ŽŸz_ ‰ÏxÌ~¼^î“>–mÀã5œÛ›ùŽm3½â@a.ÏQž¯ëûYûùæ¶æã£ö‡fo\×iÙ-Û•Zôè¤ßH]ÓüäS乘|Å)Ò‚ïª}¶OÕWÑGžõ0Ó`îÃxÎÊ6ÐíGïgº|èÝ|Ýc»ªËvª™zäk=dŸd²±kŸÇ?˜}Þ}ƒüõÌ£ä#=¼›ìVÛ:&Æ,Ý!\ëîÓcqj㎻u÷mx9Ö†UW£ª]´]ã†M`îÓ¯…ŸoßE~Ô}|¾ùZ¿¿e 7 mg.³…ƒc5Îù×llÄÞŠ¯yiy@éã¨8¿õSÇx­ãcjï¡~P–öý€Žwžõ~Yê·]£Ï™‡ë”ÞŸc¬Cå!Ö¾]Âo”‡éÏ` Âkd‹÷Æ5<}·Lúur™uRç8]×¹o¤Õ1Pì¹µÚqºg¶åÃo5¶¡x­¿ß'וå;c›bÇø2;~ª½Üu¿Wìµ5ÞxŽÞÆÚ8ÓÖl¯ûDÑÔ›¿ÖÛAöàð)ctL\y}ù?ëÄ_ÏæãñÞÙVZ‡µ‹qD‰Ôã¢ÞO¹·Óæf•o£®øÊxïÃÜ¿ÙÞc?Æy©Ït­ÏŸ.ŠÇ•;új7˜ú1öñ½²]_.„@óã„åBêlÅžTª2Æ‚ò~©|-Ð(ÌC½nuž“ž@m¯äðüÿàMÐÑëf­ÃmD´lªÂ>LÚø¬ÎXJ@â»ÉGÚ€’§é¢¯6ÉÊ·j"ÝŽTëNÙû]÷=íƒ:Ot¤î+ÚoÖ|gÿÑK þuÚ¨äYLm³š^ß>õÿx -ek¸@Ê; \y/‹§që$–€…~qé8aµŒÚ†óÆ­ãk«>6Ôü&«òÖ(oÚ˜^s¥Nfü´Gà鼄§ ì…!LGB.ïø¹&?‰ésýªù‚Leÿ̧‘ q>tA6Úå¾|Eâ -éCÝR¹1/_r˜1ÔÞ”ûC½ÍO1Šõ¬ÂÔð -Qîd¸%—Þ—oÔLä0¾–ÉÏgÜD \ebgHÛöwÉ’~1Üj®“%.sÅb;ÌÍæ»èZu Ì_±•9-âå$ÍúkêÓ*¡Ñ°j·¼$§Ê§„Ê*²„šVHåë)7Ä/]eMbÌv]Y ‡4Su$j¸\q+¤Ëi7s ÍR±¡R¿5WÍwQº«ûÔ¿Bu4S%M<©S(Ûc«4™‹ª­]R~ƒ7ÄË©äòÓ)Ätè¹Â…êzšèßFN®ûK§ÆTUÅÐhe‹]ù;¥ŠÐ¨%óŒeO Tú¬7Rb³‘—JÒtnY‡Ö4;V© bl>«õ%“Ž|Iº Ý4¶ *•)ú¸»²æ–<–dh^×È%…Uý´\©‰® -!Íurh®ü´ w9C UqöÚƒr÷Ðøv2·Üù¤É±Y¼¼å®v¢×V>_‰—±°Ñ5¹¸~ªt2ºVízcŠØ“¤ÐÉ=>ä­-;AÎí°ÙZŽœ~>e}À\¾ÞZ^t´üîœÿ'˜7òùωÎÿ¬Ûǧ:^^‚7·-·.Ⱥ’!újÂ]2bnÄ6Šˆ¤Ò~'CêØP½¡ÕqD•¯©ßîzb¯HhÚUM ]‡ŠŒÍu¼ÒL0ˆ—+-¶Ì½¶é{:õµsÒŒ/ïªRɉ¦~Í÷’|V,å¬@{I‡òæ@u£œh¦rªÌ]²…˜Å!u#MõE~Ò¦ð¥-u¢†åÔ÷kÔˬf]ic/„@ÝL°`L Í yV„~qxÝm÷Yd®P¨-µ7ꓦ¶ŽjKxëT?žòò´Õ.'5x»†/¯bÏ·AN>´ Å‰öQEMý¶t£Ti‹Fk¡hé·oÍ„Î1V™x¾­ªeìqZÕRlcŠNýYk:h«4%m¾Á¾µâµÑ’o  Ý9öÞ¹ŽlcÉ´Bc}w£RGúJ»m›–m×j{VÝ .÷‘©ë·…ä‹òßKbgusùj±<©-k¸ò†Rˆý9—áNb–R¾T"%µ^òì{ì”Ûe’º€¢[sȤݻâo"<É…¼A±š¿ê¾q[v«]zž6¿¹¶þU?øù ã*Q;eªr¢ò$Až!º'Üéf)ìûÿ¿^`è -endstream -endobj -114 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 113 0 R -/Resources 4 0 R -/Annots [ 115 0 R 116 0 R 117 0 R 118 0 R 119 0 R 120 0 R 121 0 R 122 0 R 123 0 R 124 0 R 125 0 R 126 0 R 127 0 R 128 0 R 129 0 R 130 0 R 131 0 R 132 0 R 133 0 R 134 0 R 135 0 R 136 0 R 137 0 R 138 0 R 139 0 R 140 0 R 141 0 R 142 0 R 143 0 R 144 0 R 145 0 R 146 0 R 147 0 R 148 0 R 149 0 R 150 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -115 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---request-timeout-argument-is-set-as-appropriate-automated) ->> -endobj -116 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---request-timeout-argument-is-set-as-appropriate-automated) ->> -endobj -117 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-lookup-argument-is-set-to-true-automated) ->> -endobj -118 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-lookup-argument-is-set-to-true-automated) ->> -endobj -119 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -120 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -121 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-certfile-and---etcd-keyfile-arguments-are-set-as-appropriate-automated) ->> -endobj -122 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-certfile-and---etcd-keyfile-arguments-are-set-as-appropriate-automated) ->> -endobj -123 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -124 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -125 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -126 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -127 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-cafile-argument-is-set-as-appropriate-automated) ->> -endobj -128 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---etcd-cafile-argument-is-set-as-appropriate-automated) ->> -endobj -129 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---encryption-provider-config-argument-is-set-as-appropriate-automated) ->> -endobj -130 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---encryption-provider-config-argument-is-set-as-appropriate-automated) ->> -endobj -131 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 473.570079 498.679134 450.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-encryption-providers-are-appropriately-configured-automated) ->> -endobj -132 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 453.170079 491.179134 438.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-encryption-providers-are-appropriately-configured-automated) ->> -endobj -133 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 433.970079 498.679134 410.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-only-makes-use-of-strong-cryptographic-ciphers-manual) ->> -endobj -134 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 413.570079 491.179134 398.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-api-server-only-makes-use-of-strong-cryptographic-ciphers-manual) ->> -endobj -135 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 394.370079 504.679134 361.670079 ] -/BS << -/W 0 ->> -/Dest (controller-manager) ->> -endobj -136 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.970394 386.420079 496.429134 371.420079 ] -/BS << -/W 0 ->> -/Dest (controller-manager) ->> -endobj -137 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 343.670079 498.679134 320.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---terminated-pod-gc-threshold-argument-is-set-as-appropriate-automated) ->> -endobj -138 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 323.270079 491.179134 308.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---terminated-pod-gc-threshold-argument-is-set-as-appropriate-automated) ->> -endobj -139 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 304.070079 498.679134 280.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-1) ->> -endobj -140 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 298.670079 491.179134 283.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-1) ->> -endobj -141 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 264.470079 498.679134 241.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---use-service-account-credentials-argument-is-set-to-true-automated) ->> -endobj -142 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 244.070079 491.179134 229.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---use-service-account-credentials-argument-is-set-to-true-automated) ->> -endobj -143 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 224.870079 498.679134 201.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-private-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -144 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 204.470079 491.179134 189.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---service-account-private-key-file-argument-is-set-as-appropriate-automated) ->> -endobj -145 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 185.270079 498.679134 161.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---root-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -146 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 164.870079 491.179134 149.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---root-ca-file-argument-is-set-as-appropriate-automated) ->> -endobj -147 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 145.670079 498.679134 122.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) ->> -endobj -148 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 125.270079 491.179134 110.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-rotatekubeletservercertificate-argument-is-set-to-true-automated) ->> -endobj -149 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 106.070079 498.679134 82.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated) ->> -endobj -150 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.786556 85.670079 491.179134 70.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated) ->> -endobj -151 0 obj -<< -/Filter /FlateDecode -/Length 3770 ->> -stream -xœí][ݶ~?¿â<Ã!gx‚-Ð61P  ä!èƒ}j'(v8}èßïPâe(QÒîžuZ4Zcmé;#ù ¿áEÒ1\5ÿùø¯€ BˆÞáõöxùtÑÊ“æŸÉ¢;™>þžå犚ÌÕ墉`¯Ð\*jƒ€vö¶ÁÎùRˆ„øâò‹¼ÄÅæ@\Ò`.ÉkKØy¨ÅƒÂ5X}Þ.í•i <¶Bª;(Ö¢n½É¦<±‚\%ÚTëæ±C½,ÛŒ˜•0´2ª¿FÒç®Î2¾’þÐÓïh‹.ÂG\ýfƒþ†ƒV+ýÙ€Òo6è7=ý€K\ô`áQ¢=ý0¦¿ÁµéÂ_#iÈóÕmú¹®~ýéòÕ;PúúÓ¿.¿˜„Ù±T[ò8é¦ì$2§O9þîåèÒ¿®ž!Ç®-]ËÝ 5§ OAž‚|AŽ…÷ÜÙê–LO9žrüÝËñ9ÓU_ -1F 96¬-‡p„‹rŠ ‡…ÁÕ*˜~®¿~èlÐ(bÓ=› ¼õ–ÂŽ ¡Š)è -möl¼BÚàŽMŠ¡qýžMTº¸cœÒ^»;6ÑsÃ!‡†{Õ‡†;ò‡†‡æàz ql´öÁã5¤£i‡cÑiMCëySdÚgÒæÊ`”x7žt ‰¡Ã—ãŒAÓœtQXKýÄA @Âq7ؘ Kz~èqÓJi™8\e;vÙNàÝ`#p1NªeÙfHp‡c+¥ylLùîë-ƒÝÅú8ä1@à]—Iévvq0q¸Uf5ê˜@Ã8ÐF¨C.»tóØ¡]`# ã«qG05æûŽÑšÇ énK±cQDâ”è)ÑS¢+‰¾pó¤“hWó®õnТS¢§DO‰~&‰Ž¥øü‰î–tO‰ž=%z×DW,iy!ц۶[×m¥lᢜõ‹¸(-ǵq:øÅÚÚ)BJlÛP½q°gã•Ãˆ–vl¬U^ƒÓqÏ&*OÖ‡½:#© )Ò^‰ÃDì^™¼Úê½:;«bÚÍÙ«3÷u èh¯ÎÞ)í\0ndž“ -@L[;6Agwl"*c¬Å°mƒZ+㉌ٳqŠãêâNŒ²>¿SgäþŠÖhÜ©3N³ ìÕÙjE¸Wgë…@n¯ÎœJjowbŠØ¶W7m•?Ð:­üvÐqv9ÐNYv_;胊ÚÁ6Ðöµƒ‘óÖv0zî«ûÚ!mh‡8G™íç(s âe´Cœ£ìvˆsh‡8Gáv8§*:ЃŠ´Cœ£Üvˆs”;Ð9ΫÚ!OÊh‡»÷Õ}íPðuàÜ´‰œW´C1ªx Ç9JhÇqŽ‚í8ÎQp îÊÊhÇqŽ2Úqœ£ìvç({ Ç9 -´ã8GáH;Ë-y1Y[ò¬ƒñž|gNcó§lÊ;•¬UÞ­I\Úöl'tøêæ/Ïõ`Ž,ðþöoÃÅ*Ã1s†ãn­â™+qoSౕR=zN_n5Gh7Gx·Vxwó¶zìP/Ë6C‚;\̆Û=àÆÔ˜ïþ.° ¶Œƒ ¶¼Üðî©9Nõ«µÊ"f# 7¼FX­U¼1Ã8˜8˜>yå py·y”h؈È8ØõÍàÆÔ˜ï;6åEk7¤»-EÉ¢ŒÄ)ÑS¢§DW}á¦|'Ñ®=]ëÝ E§DO‰žýLKñùÝ-éž=%zJô®‰®XÓz_÷-ºMyN´¾-™7ñVÎzS^\”ä˜sG·¸öÔ=C8¶qZQ{oÃÆËc -G¾–¢ârƒ”ítÖ4´~Êö€qx+Sà]v ZÜŻ×Y3hÝœtu·¹j¸ÈwA·§ˆ%*n7JÜ™ Žº•Ò<¢©ùAÔ¯¡ZÞeM‹|'9‡ \tiáQ¢}`#¯­—y:ÆAzÇö€hÍã†t·¥(q‰S¢§DO‰®$úÂíN¢]{ºÖ»A‹N‰ž=%ú™$:–âó'º[Ò=%zJô”è]]±¤B¢ LPk}[0oᢜõö€¸hó5»à­²¯ô¼Ò)ðJ_ˆ¬Øƒ×"C ¼¬ß-2ên›alÜÿ^‹ŒÀQ=x-2{È!|Èa´tÈaD8ä0¢s¸ÜbÁÂk‘õ3_‹ü4øèo/àòwQ¹é!Ó#‹\vi5×ùíãå«7þî/ûž[òöãåǯµä_›~¿ùûõí›ËŸÞ²‡qv'`´%±3{Prñ‡oß|ûÃw¯ëÂpÀñõ\tLyî&‰ «q)‰SE/o‡Å¨ÂÚq‹ôpûçu,­§Dã¤SV:váïpáøóÏëÂ}R¸ïqO_|2¸d4½&ž6+ãýËÛÁéïIÑx¢‹e;ˆS=']CœÛq{y;BðOÒÆ.bš -?ÛÅMé0° y‡C¤^Ëãû±NkL¿ïù—'ÍÈ8¥?Îç”ÓOAg_c8O!—;ùq¯ÏçÔ>.ÞåöÙÅ¿¸Ë—]òõì8 t4“AðY=EM0;EßÈ6.Wrº òHTîÖì¦ëŠmQ¢Û#ÃÚ´²ÛV"}b‰.Í6¼JlJ ¢Ç@#©^ûáã¾Î×ßv®Ýém«^ƽnú\ÚZ™=ÚñDxí-»„“$üE½o\n°"¥v±{mõ»–1%§ã¸æ2C+ïµ³åŒ3 ?¯šYÓ›I8æÔ½2§‹ºN£”o#Æ‹8ËI¥úþïp¢2cýëêZÿ»7Ï ËûÍsÏ·Ó›X·OÏQ¯cJ›·ã‡ß(Æÿ³ùh—7·äí•â‘æòãxÄß ÿ·¹ŒÇ×­1öÝËxÅ<¯¢|ÝÔfìën}o9縵žÍ¢oOí,³u?笷¡ñYr`šp–kµm\ËIkâÔ!×9¯rögÊG; n1é\4Jv”Iô’ˆÅlx™Üw úǨœ½áÓ”gö,Ò5h#cM~F´—™ì9Ä„\?VÊ^ü„Þ"m¦v¸{{ÌÑN!ºLu?ÎJéeú­‹û½h†Äto;šw®‰ªæµ¨Ø­å£‰ ],Ñ+ѱ¹q[”Gºžð\—Už{µœ˜î¦,æwKž^iýñ">Ê":®{ãr.'Ç”u\æN¹0—½ž×Óó]Ù ôÄä¦{HÁ‚ôõÊÇô>útÛéòËSLR||L_Ù‘î,%¥ocl½qk){Owæ`g›‹Ìw°Òw>Dï- uÈ‹[m>Ì¡®agJ SgËpEB,Ó‘²aT(}/ºbúìC›vLû%%Lçä“p“—ê5ñäãÑÔÂäiÇÔ]DB­åS Üå¬ .Ê)ÕjS÷Ó…§.6áˆNïs[œ¾{Àå'üJ^qí\0%v0± ×K§ÿ=æSµÊÿ¡Lúv ˆ¯óÿ6#N²ý¼Ãüxž¦¤/[EóÃ쌖ð‡>Ýy[ÈåaðC8ùPײ°>L^±ÛlzZaai©Ü…X g0¦á ]Oå2;§&SÎ’fǯt°†SëÓÃ-3ÌA  l>sµ*äJl¢´m¶ a°Z€´µp×`Öe,¶¦ÆskÕ%u Â_láÜovk—èÃM_öQŠ*ì]1+*+JŲkœPzRd~©KÑÈÁ¬Ÿ‹¨2JGÂj‹Ýº]†v¹4ª‘òÕ*´Kca¾U¥uu*ñlâ(£œ&Ä)h[{Ø¥”XO÷Yç4é6ðvIyg?\Ȭ©å -ž×Ç5hE¹ºõ—†úP¨bi,ÝïLÏ©‚*š2Ït¾tJò¬9IÚ¬pO’Ä]Í:8bÐ7[Ax_û¬ä ‚ا¤[­+c*(xzeªíjòèa¨½®Â)…~j®” .•`w ¦Ú•:¸Õ“ªT¹³gDw§Znƒ]Íö5âà°6¢NôùÞ¦ —pêú¡8i°Ñu´kÎØ*ÆI¡Á->LpQkÍNc=l"¸·ËÏ—i˜3–LLOšZîÓýclpý7Û¼áß^ðú×òüÏ<Ž]³…©¹µƒåHfŒ-!<„ÓÓ6¬² A‘ö ¡ÙRé u7Fäkl—ë–Ø‹%Ô -õS RãPÀ¦v]V0C;àmÁbÈÜ›6`ϧ¶4ŽÝX¸Ö•Ndsø4´Ïl’¯¢H>K¦mH‡ô“âB>‘…ò©p[ͺj¦)TêSýQ†Â&_âDL#Ò©mŸafeÑ›Z!LÄÅ]ÁÒiìê3¨ô‹åuwÜs•]Q@Bkj¯èƒDkÃZÞµÓ©ëOë˜c¯é¤ˆ·1T˲B{¶Nrš4ÚÔ»ÙGÚeÙ€¤S,Xç´ 5]ØúÅSÔJô¥N._Vh™Z†,ù:§hèÏ’i’Q©$’ŒŠ[Ù¦rˈWgK¶N€HvgßZ§›eK†æäÕÅfi ¶ö½ï2Ú^E3°Uĵ™©n—ÑÊòEùï%Ú>—4²ê¶G Ôm Í5‚·Æ^cšîç8‡pCxÕŒAì—<ûšý ”õ6IÙ@‘ž ½B`­öÀým£ -?aûˆ¢òsç—E=lÒóØ|ôxü™U6?Œ?íÕSŸ–Š¦úCz&¼ÞØ·«­°ïÿr¦¹¬ -endstream -endobj -152 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 151 0 R -/Resources 4 0 R -/Annots [ 153 0 R 154 0 R 155 0 R 156 0 R 157 0 R 158 0 R 159 0 R 160 0 R 161 0 R 162 0 R 163 0 R 164 0 R 165 0 R 166 0 R 167 0 R 168 0 R 169 0 R 170 0 R 171 0 R 172 0 R 173 0 R 174 0 R 175 0 R 176 0 R 177 0 R 178 0 R 179 0 R 180 0 R 181 0 R 182 0 R 183 0 R 184 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -153 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 790.370079 504.679134 757.670079 ] -/BS << -/W 0 ->> -/Dest (scheduler) ->> -endobj -154 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.518001 782.420079 496.429134 767.420079 ] -/BS << -/W 0 ->> -/Dest (scheduler) ->> -endobj -155 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 739.670079 498.679134 716.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-2) ->> -endobj -156 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 734.270079 491.179134 719.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---profiling-argument-is-set-to-false-automated-2) ->> -endobj -157 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 700.070079 498.679134 676.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated-1) ->> -endobj -158 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 679.670079 491.179134 664.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---bind-address-argument-is-set-to-127.0.0.1-automated-1) ->> -endobj -159 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 660.470079 504.679134 627.770079 ] -/BS << -/W 0 ->> -/Dest (etcd-node-configuration-files) ->> -endobj -160 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.154476 652.520079 496.429134 637.520079 ] -/BS << -/W 0 ->> -/Dest (etcd-node-configuration-files) ->> -endobj -161 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 609.770079 498.679134 586.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---cert-file-and---key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -162 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 589.370079 491.179134 574.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---cert-file-and---key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -163 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 570.170079 498.679134 546.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -164 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 564.770079 491.179134 549.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -165 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 530.570079 498.679134 507.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -166 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 525.170079 491.179134 510.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -167 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 490.970079 498.679134 467.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-cert-file-and---peer-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -168 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 470.570079 491.179134 455.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-cert-file-and---peer-key-file-arguments-are-set-as-appropriate-automated) ->> -endobj -169 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 451.370079 498.679134 427.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -170 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 430.970079 491.179134 415.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-client-cert-auth-argument-is-set-to-true-automated) ->> -endobj -171 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 411.770079 498.679134 388.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -172 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 391.370079 491.179134 376.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---peer-auto-tls-argument-is-not-set-to-true-automated) ->> -endobj -173 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 372.170079 498.679134 348.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-unique-certificate-authority-is-used-for-etcd-manual) ->> -endobj -174 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 366.770079 491.179134 351.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-unique-certificate-authority-is-used-for-etcd-manual) ->> -endobj -175 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 332.570079 504.679134 299.870079 ] -/BS << -/W 0 ->> -/Dest (authentication-and-authorization) ->> -endobj -176 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.482112 324.620079 496.429134 309.620079 ] -/BS << -/W 0 ->> -/Dest (authentication-and-authorization) ->> -endobj -177 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 281.870079 498.679134 258.470079 ] -/BS << -/W 0 ->> -/Dest (client-certificate-authentication-should-not-be-used-for-users-manual) ->> -endobj -178 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 261.470079 491.179134 246.470079 ] -/BS << -/W 0 ->> -/Dest (client-certificate-authentication-should-not-be-used-for-users-manual) ->> -endobj -179 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 242.270079 504.679134 209.570079 ] -/BS << -/W 0 ->> -/Dest (logging) ->> -endobj -180 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.494564 234.320079 496.429134 219.320079 ] -/BS << -/W 0 ->> -/Dest (logging) ->> -endobj -181 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 191.570079 498.679134 168.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-minimal-audit-policy-is-created-automated) ->> -endobj -182 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 186.170079 491.179134 171.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-a-minimal-audit-policy-is-created-automated) ->> -endobj -183 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 151.970079 498.679134 128.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-audit-policy-covers-key-security-concerns-manual) ->> -endobj -184 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 146.570079 491.179134 131.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-audit-policy-covers-key-security-concerns-manual) ->> -endobj -185 0 obj -<< -/Filter /FlateDecode -/Length 3781 ->> -stream -xœí\ÛŽÇ}çWð9€Ç}©ê `H€$¶€1 ÀAV´d#XóßÏé™éî²YË]Rk'¦dÊ35gºN×¥/5CÚ½Áß/,þId§”r ´?|Ü}Ú™)²ÁŸ±9™/‡ü¢›²a¶nÒ²ËÖï?î¬A«hÔ'!~„8Tqô]B¬Hñ÷Øý,o ¹+·t1ZŠf²ÇâmkÞNt*l:;!¶&N®7,ä¹7ÒÔYk+Zp҃슫˭q Ý5n¤Q¶íF–•bÛÛhúº‘†vÞp–þ•æO[ó>‹ÁåL'ÂùÝów¹5“¡fþUµnh~wÆünk~KÇrÁB£”nÍoÇæïâÖu¡¯ihç²1ûògzðË»/ìdö?þ{÷ásbn¬Ôzòqœ¤gÓNÚOúàžŽ÷tüݧc(âþôàé¸é˦çaЛ{BÞòž7IÈqâ=wµz.MïéxOÇß}:>g¹jml›Ç.·>ˆžÛô”\´SrؘÝÃA9›ý/ï74à'“&S,;‹aÆÌä ·VÀ3G²FÁ Ú¯cÀ™¼3YÁ$pöÁª˜ Î.“4 8; ãyŒC¦;RÇ€³‰ÑcÁÙ˜ b†½ìÙ°† “MTŒw“ÙÒ0 hTâÇ2*°U1 ÎœŒQâÇ18³É*&€3ùd”sHaäF4Jü¸Î.“ÀÙ9ÊJü¸Ζ½ŠÉàl’ËJŒá"0Öf%~à¨Éd2*¦ –)àš† “‰9©ç€q1+ñã]šLà•øñœ9±Š!pfKY‰OàLäU ƒ³0·†golVâÇpvÞ¨L(ÆrÎJüøÎ&%“ÀÙ`4SâÇ'̨™‚ŠÉ~Ê)rRb A -Œ¡¤ÄŒ)GïU VI9„¾†gÎVÅ8pfŒŠJŒaÎÆÄM9)ñCœ}L*†ÀÙ›˜”ø!gL*†Áƒ]RbŒœ‘¨I‰ -àŒ S1±,_ÆÁaJVÅ$ LJüPÊSŠ>G%~(ó”BH†Ë’‹sŒJü0–n‰]P1œ‰9*ñÃœ}"ãÀÙ[•cÎŽ0%hÚhU ³5&*ñÃÎÆeÃ7Vž¸§è=Eï)z’¢/|t¶IÑ óMïàG÷½§è=E?SŠŽSñù Ýs©{OÑ{ŠÞSôª…®ÜÓ—¦àœ¡¸}ÂK1§Qì{æsrÙÐé#6qײ%»óíö:–ÒvÔd Í%¸ÜÊ#CL)Ájåýû÷Á™Lç:gÄQ -Æ$sg¦`4Îœ æÑ8'p&ßË##L¶¥”g³Æ9ãлèÎÉ€3 ¤pNœí%ÊÆ‚³ Þ(œÑF)÷%¯p—)öAã\JC9§¤qö„˜Æ™¸”3iœ¹” =E3Ç)ÆdÚ£±&”²¡c—4Ì\64L,eCË­t?¤R6 ¶=zbbyµ;Æ9ƒ3q/-1¥lˆ4U8cü†œS8g νS8gFç{y„q¾”“Q8cxÆy¯qöàlB -ggc}Ö8SÄ,ËÙiœ¹¼‚aˆ4Îœ§€#jœ1a‡˜Ùhœ±† ‘Œ×8Gp‰ƒÆ93Ò=iœ83¢UãœÁ™%%¿¬ÁÄ(ôºüÖdmVFÌÀ í9:eX¼=¦ e\(3 -fAêÏÇ*è¸ö,fDQ{v9 –!<œÇð³µçOƒKz»ƒ–¯{„=,4Ì1#4¡Èk÷o?î¾|ó—oÿú÷ïЗ·vÿø -§õã¿þçþí›ÝŸßBƒ¢#/*ò>ø0YW*÷{ÄÙÐ"ñ‹Š?~óæ›ï¿½­ -‡\¦Ï¬"¹É}^ÞO?­‚®P‘ÊÄúYU™‹ÜÍW¨À”t‰»¯PÜEîW¨À¤y‰»¯P°n¹ÄÝñ2ã“ÖD !éåýˆñ²ä»BEr—%_¾BE¼,ù^®"ÛË’ïÝ*ÒeÉwبx*žÊ;¬TKêŒñ±‹ãñ߃1ìðacè=>i9ž¯ÙåÅE^d.¬x\£ŒÏŸÒÆz†/Ãq‘WÜ|_Å~Xu¬i‰åõÒ,¥Ê Yë€Ë2{.36’è„ìŒK‹B‚ÐJ|&õ®ÜÈVL1@¹ÖûÙŠÝ@\;êýÖ?¼ï÷Ìx/ðf5JXÛ*ç³~:âŽôÇåžrlój‡zÍuÇ•ÿ5§&ÕÐQºØ¶êŸyÆõ|ígslî ìçhµ÷Êoîk „´9h<@`¢ óoÔÇ¥¿kBÔûªó°ÄÀ¬÷Ðý¬ùu¾ß­ŸbƒŽeªý±oäì¼îŸµoî° 5þwðàôAeµïÃØÖ³ÍÞu[Îý âÚ¹¸xõµ^zwCŸþ²ûªñð*¹Ìtl××ÍeÜýÔZéð:û”6^ŸŸr}“>XoWŽÕ_ÏÌùî_ͦÙôeûÜ_sžÞ=µ^›«Ûÿ?±ñ:ùøØ®Ÿ-߇Ŵ§Š³Õɵ8qÖ1Z'#Ë áùj߸ÚUgµšµ‘QkÙj113lfó9f0û\¸ëšËˆë¬Zq·ËÌdÎînÙÜ´²Ô²N›IgX¹¯Ú^–[ŸmçêÏãjÖÅ—ܱ½\õÍCáóµË_­±úµùú:"ÜÐw|v\}nì»3«©v­Œâf0S^™r–«%uf;Hª¶‹Ç¶»OÊ;&gv¤Õ'7ªUŸÔûæ1bõEÛY~Xbq–‰ÕÁÆ®7§êì}#;†³»ÂjÇtc;ÊÑj϶Š³‹¶«Óf÷ÍêIŒYs<Æž ³žŠ«¶vÝV×iª=ÓÉð3ør}WŸúE‘òBJ -‰mùî}ù -­‹a~‡e÷ó%Ò—X^›ßR)‹7±Í@ŸyKeÕ^^ã±Ê#A4¹¾ ã¨,g¢çáÆó²n)a֦°ò}uAœ¸»Ö…_3]¹ö¾‡_ù¿bM”¹[‡½æªõxn —ré¨ýºx|ê´õ¡¦ÖécèO;NXöR¾´mà0òÄ1æÒgùžãv!¹ê;;[ÛîÏÜ:ÿÖ맆ZþÕLìmùiå·aÅÉŠ_ž&ÜÙLSq@y%ÒN¼A e|,~„ësýfíFLóO»\$.Âõд¶ÈV`“vÖ—7Âóû7ËanC¸–w½úý\oóK»zVaQ<âÎ`OÅ¥÷å ‹NÀÂÁ¯g¡QáP9CØ$‰í öŸc4B>¡®‹‘—¹b];ÌÍçÞuêÒtV¸¿bK½é"áaµþXú8”:Ûd®Z×™&ó‚TnR®Èæj~ˆÊÛø'Â/Lmº¼VÜ® ¯a[¸ -S $j¸\qÑa7Ä­­qóTl¨ÔoÍÕòJuŸšá²ÙF«ÃÄ£8µÆ×í8T_Ï?ã´¸ §nveÜ9?î8ÐÜÇŠòÓ+§B/Úmâdz¼tiLÕTHÍ*+¯8•—¡‚›´Œ<óAù‰0[g]IÉÍ&ÞIÊCuhdK±c…¹Ê«þDŽ¶“àË ÛÐÍb©0™Ç\é•Aª [±mQ×Äe«öic¥šJ꺘[(?nÄ'·TE°WeÁŠpçÖn‡6v>Jql·XÏÖNôy ˆ˜¯ÂÃ<±Ñ±¸„~ªJº¸üÄY Ÿ¦L;1 -]܇ÑÇY\³µN6çvØ“l4ãv?íæiÎyv¹|›¯ü²Ëö«Bvÿ`Þàó¯íÿV_%^æqgÚháÚغË™Ì9_]ø¤¸¼¸[…-m„Бö»Ø¦Žå mwNŒ×Ôo7}`¯HÛFPÛ®J!w -±k¡ë¬–aEx8²b[È\ë›>a/§¾vj|y)½²Â‰ìNS¿æû”¼Mâ¬@û”Ž«äÄ8‘âT(ÌÙ†f H©¤™¾ð'é -_t‰±Œ(§¾_£>Íʦ«lî…€7“Ý4LV*Í>Ò/N¯«ý¾R5Š´ íMú(¥­cEÚ¼±Ôϧa{Úæ¿/'5y»…Z[^äžo‹œž}iA›£Bšúm+€¥Rª²Ò:Q´á2Se­ÅX9…õ¶j–¹Çih¥ØÖ]ú“´4K¯4#²ôJ8Á–vëŒ×VK¾-€X†sì½3ÙÖ’i òî&¥ŽôUvª›¶ºël»Ý N$ô•©é·ñ òEãßKrgøòµ)ØÔÊ5Ùèß—ä2)Œ!T~Î0Rõ’gߣPNË$µ€"5ÁØ´{-âí …G\ÈO_M­ßì=s[6Ã.=Ïg¿Å4~cÜO~ýÂó\%j§±l9§ò 2“äÙFsæ™×ÃI)ì»ÿ¬Áœt -endstream -endobj -186 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 185 0 R -/Resources 4 0 R -/Annots [ 187 0 R 188 0 R 189 0 R 190 0 R 191 0 R 192 0 R 193 0 R 194 0 R 195 0 R 196 0 R 197 0 R 198 0 R 199 0 R 200 0 R 201 0 R 202 0 R 203 0 R 204 0 R 205 0 R 206 0 R 207 0 R 208 0 R 209 0 R 210 0 R 211 0 R 212 0 R 213 0 R 214 0 R 215 0 R 216 0 R 217 0 R 218 0 R 219 0 R 220 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -187 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 790.370079 504.679134 757.670079 ] -/BS << -/W 0 ->> -/Dest (worker-node-configuration-files) ->> -endobj -188 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.214534 782.420079 496.429134 767.420079 ] -/BS << -/W 0 ->> -/Dest (worker-node-configuration-files) ->> -endobj -189 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 739.670079 498.679134 716.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -190 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 719.270079 491.179134 704.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -191 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 700.070079 498.679134 676.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-ownership-is-set-to-rootroot-automated) ->> -endobj -192 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 679.670079 491.179134 664.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-service-file-ownership-is-set-to-rootroot-automated) ->> -endobj -193 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 660.470079 498.679134 637.070079 ] -/BS << -/W 0 ->> -/Dest (if-proxy-kubeconfig-file-exists-ensure-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -194 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 640.070079 491.179134 625.070079 ] -/BS << -/W 0 ->> -/Dest (if-proxy-kubeconfig-file-exists-ensure-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -195 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 620.870079 498.679134 597.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-proxy-kubeconfig-file-ownership-is-set-to-rootroot-manual) ->> -endobj -196 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 600.470079 491.179134 585.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-proxy-kubeconfig-file-ownership-is-set-to-rootroot-manual) ->> -endobj -197 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 581.270079 498.679134 557.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -198 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 560.870079 491.179134 545.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-permissions-are-set-to-644-or-more-restrictive-automated) ->> -endobj -199 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 541.670079 498.679134 518.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -200 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 521.270079 491.179134 506.270079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---kubeconfig-kubelet.conf-file-ownership-is-set-to-rootroot-automated) ->> -endobj -201 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 502.070079 498.679134 478.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-certificate-authorities-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -202 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 481.670079 491.179134 466.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-certificate-authorities-file-permissions-are-set-to-644-or-more-restrictive-manual) ->> -endobj -203 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 462.470079 498.679134 439.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-client-certificate-authorities-file-ownership-is-set-to-rootroot-automated) ->> -endobj -204 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 442.070079 491.179134 427.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-client-certificate-authorities-file-ownership-is-set-to-rootroot-automated) ->> -endobj -205 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 422.870079 498.679134 399.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-has-permissions-set-to-644-or-more-restrictive-automated) ->> -endobj -206 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 402.470079 491.179134 387.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-has-permissions-set-to-644-or-more-restrictive-automated) ->> -endobj -207 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 383.270079 498.679134 359.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-ownership-is-set-to-rootroot-automated) ->> -endobj -208 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 362.870079 491.179134 347.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet---config-configuration-file-ownership-is-set-to-rootroot-automated) ->> -endobj -209 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 343.670079 504.679134 310.970079 ] -/BS << -/W 0 ->> -/Dest (kubelet) ->> -endobj -210 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.214534 335.720079 496.429134 320.720079 ] -/BS << -/W 0 ->> -/Dest (kubelet) ->> -endobj -211 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 292.970079 498.679134 269.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-anonymous-auth-argument-is-set-to-false-automated) ->> -endobj -212 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 272.570079 491.179134 257.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-anonymous-auth-argument-is-set-to-false-automated) ->> -endobj -213 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 253.370079 498.679134 229.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated-1) ->> -endobj -214 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 232.970079 491.179134 217.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---authorization-mode-argument-is-not-set-to-alwaysallow-automated-1) ->> -endobj -215 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 213.770079 498.679134 190.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated-1) ->> -endobj -216 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 193.370079 491.179134 178.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---client-ca-file-argument-is-set-as-appropriate-automated-1) ->> -endobj -217 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 174.170079 498.679134 150.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---read-only-port-argument-is-set-to-0-automated) ->> -endobj -218 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 168.770079 491.179134 153.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---read-only-port-argument-is-set-to-0-automated) ->> -endobj -219 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 134.570079 498.679134 111.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---streaming-connection-idle-timeout-argument-is-not-set-to-0-automated) ->> -endobj -220 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 114.170079 491.179134 99.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---streaming-connection-idle-timeout-argument-is-not-set-to-0-automated) ->> -endobj -221 0 obj -<< -/Filter /FlateDecode -/Length 4009 ->> -stream -xœí\[«$¹ ~ï_ÑÏ8¶,ùË@¹-²0‡‡ÙÎL–pfa“‡üýÈÕ¾¨ª\ê>§Ï²/}†3§J%[²>I–]w¶üï׎ÿKèLJ9<_¾ž~:YÉòϱ:Y.Ç¿òŸsÑ”Ö>1dƒ²ó篒’ Wò “Ó ²èååô·_~œwåÎÞXçÉãù?Ÿ%KD]Îɳ$F‹Ya &Ft.³d0ɲ⨰ð¨ Á1‹³h‘wŠ2ÎY†|ŠO0Ùf$RxLö@Ó’ÉÁS¶ -G“³ ”v+ëbMŒÆ"ƬÀéˆ/D—‚‚§£llN4}”³‚¨‹Î°<HyHÆ1î^Ó'y¶x¡Æ“ x„ ašÉ@pÞ+ú€uRB«`ʺïˆ]^áqÞ°:ä}ÀeãCVÁ€ŒÏ!DSðÎ øˆš>>$›¬‚) 7cŠ -¦€Ù‡:júòlISÆÁGET0e0 ±§¢¦Oô†!§a³ ŒhÒ0MdŠ•µ$Ɖ’Ó%¢Ó0ÍÑDÖ&)˜zë9ar–Rôñ6sƤ -¦ìƒœ2!$S¶ ç̵<æ)pFTx¼çœéSV0õ>›ìlÖò˜GâœY´VxÈqÎ$ξOäœ .h6 È2€‚©–sf` i)qÎ,€Â“‘s&2ÇCt¢¡üÄóþà!ºÏjôa2¢gˆ>Cô¢?SˆÎCñõ…îQè>Cô¢Ï}¨Ð•·¨#¿ºA4¿qqDýìohˆFËr=ä¼Y[S@c¦¶=9åa,S}›aÎL&ˆ6*<‰/øìÛïœ'/[éVÓ¹l]3“Uteë:g›CÙºN²¢spe +:— Û:eEçdfMgN*уMšÎž] -IÓ}‰2HšÎ˜MÎ.µmà)•­¾„m;gÊÀ¸@̦ñ$äBÛJŸòD4Þ'h[bSžd ¦¨éœ‚agî·#¦<œJ)'43ÇNBžmy¢-±ã (:Çr{b -ŠÎ±Ü^CÄ è¡ÄÏlŠÎ‘ÓŸ2l -çر¤éŒ%vl"Mgä؉QñÕÈÉ ZT|5R‰Ë¡¡ðŽ•øb Û(¡_1–ØáKšÎ©$Ôh½¦3稜|ßržò”©&ZðŠÎ‰+G!yEçd9v°LN -+±c-(:'àØq­ŠÎ 8v¬Ðtæ —stMgϱ:MgäØ Þ:ÅWqìðœí¶¾ºÝNØO†2MÌö“Wì4g¿gC9EÏ$ŒrBÐWõt*ùk²^Ñ·uvâ‰'õ ¯êlArŠìF{꼪³3;”¨=Œ^ºDžEzE9ôÔU}'è«:[ÐE…,$®¨(û†©%}Œ^H–šÛû²Æp 5µêô•ËDwuö8À$iWgg—§8äò‡Zõ -ºpi!qE]áàpÑG/$ -KÍíýÀ†²Í׃Ð=Å•Ï}†è3Dw!úÆ åUˆ®4_>LFô Ñgˆ>Côg -Ñy(¾¾Ð= -Ýgˆ>Cô¢º«G½µe³eµ£œÑ‹á5ó]v´ßR­Ü™abÅÓvy1•­E\¶òŽxx™ž²µËÊO°&¦DËSÕ‡<Á„E…'–- Š.i<É ã¾<{Ä“Ðøiy²øˆ'[Éá²5}ÈŒ‹ü?ó8ËAmc"P ÍLìý1Ú¤Xš½Òö±ëÛ1SyÂÁ*¶fÏ ÃÁÃÖØÛýá%b?&‘¡BÛ½È ¹iÊ}¸óÓäÒï>žòõ”|žm ìf<.Ëñn³<üzúÍoÿôíŸþögÈÇ/§¿c-cfñ¯ÿðóÇoO¿ÿÈ^+"p¸C> "sZ¿C½]DyÒïÞ.‡É"â"¢¿ îôvtÜùœ—îûûûD¸PßÍ -Ë$ï0dw&ŽEȾ¼¢P„|û‡?ÿñ¯ßm„„Ç"ƒòíà{PD¹{v GE¤ÛÁ÷ ˆèo߃"’»|Š·ƒ/ÜŸ¥6n›}™ñÏÙ–7Pn»íy*ó8îAc-b>¨càY-Ox×¹¯¼pÁÙyÿ‘] ñò—+&[nô$k‰Ï‰eS9æ똮׈Ûà÷ü‹×ã­òpÓå—üµ-Aåáë®Çå:~þá›+}ás•çK½ÞÚä*¯^+í‹nEÖB/ú}ªºÚÚŸúáeð—¿Ëu¡{ѯS•S÷Ü®å­ÆaØbKË㇪cësÑÅØ‹Ì‹Ú/òZkÕï£OïŒQÕQb±ØþRñ‹õ:]m¿Ø.Ô¶u¬…·c†ì¥÷‹c¶6}'¬ryäBÅêû÷ŪÐ]k -ñ‚ÃŽ &åz1EŸ8ú{5nÄð —bOßìç*&xõ ݆Àk9Ômxyg&a7raõ÷ŸÁW¯4¸öÙú_þ~Yë´Ø¬ð~ªú_ZL©v [;â½Vãó›í…¿ˆXXø'v[èAô÷ؼÝ7|yÄJ÷ ¶}·|ëD>k±´øŠóŸ>ܺ-]ÍJ_Ò¾”×6Þ͇E–â—;ÿUscycumã­/ý¼ÑgÞ'Ïú²Ä¿íøÎ~ ë–-NŸÀKÅ)¬yßœKnä[™[ZmÔüìÞqkË·ÇgyëõF­R—E€ÏÈ‹Zoñ—|õó{±ñ˜Àç®ÄÍ¢;<Úlæ%^bn„;qüåj„­À‹ò­z¥®7Þ#ŽnaÒçCW}·ùÁ¤í6Mî­=Ö5‹®­ßV+‡J5íÒöSWÍkc¾Uí·v\üè"úh>^1Ž5¨s`°>Jvcë$ àZ€]“.Èä#&§]0´£‡õä7>DAŠõñvE¿[Ummßs‰õvö¤–­ßÅbÞ­ ¶ãkÉ앹EÛqZÍêÉÕä¹­ÌçÕ^ë£í¬È•áQrN³Wr[{½Oe^²¶:oÜQ|d’ZÚÇÎÃ7Õéd)1ÛŽåĬÙ -¸+ù·­ÌMÐ4™¤6¼©LŸ–·¶±Óf‘*‘@‰Ü¥zÍeÓæ‹@êÒ® 4ó@ñŽ’A¨w§öm™°-"—ü¶¨Ü-ØÛTQ òií Ý›—ˆ ÐYAøšð«ÿíþB$¶d¹óx6teUgby9.,7ËO?ÞÃRÀˆÙ›¸Ü/†\fîƒâUzyðÀ)7P–WNëÞ^qÁèiêwK©úùššï•Y.WŸ[ÌŸ'™¤UÎö½ænºrí³íëø•ÿ~¹ÂZè€~œ³5ÔõÞvµ÷ßÖBŸwC¿¿²»UôÓ‰‡*³0bΖ%y\>rÉÄ…~¦hX» açk»óAÓå[—?u®úùKkʇž¯ßÆ'•ÿzßêëéú@ùa¯0t=b2 £-ù…¡Ï'džàNr!ÖCÛû*ïˆmh—“+w綜¾<Óq=̽sß_¶)•ŒöÔšùk‚vVá*xBä–ÁíÉeôåé½+™A d|= ] -Mg&ŽQòŽGÛdç­ Òžh…¸Aæ¸Ìúaî˜3±«.Mçü¢€S#^N,Öo©/S*¸Nƒf]°æ…R¹S©qv@°ã4!•GávÄ_ÛÖuy˜©_¨Aû&X¹oQ:_n|Òå4嫽QG*v®4šæfù¡Êpõò2Ü1m±Q¥ršx§Îúv¸&‡‚@ú¼X!¡;ˆ—SÉ;;òˉℛF®°Ô\O½è·““þ2¨15Sqh6Z8,ÿ_SujÉ<ËA¹ÁêZòl)±ÙÉk#IzèYg(߇u;*÷c÷Yi/.u&ü%évîn±U˜LÐcnê•$Õ“Çšìº×urIaÍ>=WJ¢mJ°¸A¦îÊ/+òГz¨²³7aÁ w§Þï ‡ž;_$9vÄ]ù¾@×·÷+|¾/ËĆ[rqýÔ„ 2Ø>Û a‚8ã¤0È#¾,ä­=;¹œûá²ÙŒ{9ýpZ¦9ð¹çøåãÔ-x‡…z_^ÄžïEÎQZàêDú¨ ¦Ñ¬2Š¶Ú&Šž.˜f­÷›N¡6kfYFœ¦VŠ½¦Ô¤¥I¢ÒH•°ã-ý¶¯WK¾@$Ý9ŽÑÙÁÙkÉ4¡ÙºSqpúFÛËƵì6ÛžÅ0p(FejG3Úq¾)ÿ½%v¦O;θVÛ#.õm [>ì¢Î¥Ü)ß:ûò©jŠ˜Ä~É«Ûè(ûm’¶"%1¼h÷6:ö·^øB¾}AtU_«9h–ítH¯3Àá ógS½ñõm£e—¨ŸÆ²T¤œÊ—Wl*ï­¹hö)¿ßm…}÷½=÷j -endstream -endobj -222 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 221 0 R -/Resources 4 0 R -/Annots [ 223 0 R 224 0 R 225 0 R 226 0 R 227 0 R 228 0 R 229 0 R 230 0 R 231 0 R 232 0 R 233 0 R 234 0 R 235 0 R 236 0 R 237 0 R 238 0 R 239 0 R 240 0 R 241 0 R 242 0 R 243 0 R 244 0 R 245 0 R 246 0 R 247 0 R 248 0 R 249 0 R 250 0 R 251 0 R 252 0 R 253 0 R 254 0 R 255 0 R 256 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -223 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---protect-kernel-defaults-argument-is-set-to-true-automated) ->> -endobj -224 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---protect-kernel-defaults-argument-is-set-to-true-automated) ->> -endobj -225 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---make-iptables-util-chains-argument-is-set-to-true-automated) ->> -endobj -226 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---make-iptables-util-chains-argument-is-set-to-true-automated) ->> -endobj -227 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---hostname-override-argument-is-not-set-manual) ->> -endobj -228 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 705.770079 491.179134 690.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---hostname-override-argument-is-not-set-manual) ->> -endobj -229 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---event-qps-argument-is-set-to-0-or-a-level-which-ensures-appropriate-event-capture-manual) ->> -endobj -230 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---event-qps-argument-is-set-to-0-or-a-level-which-ensures-appropriate-event-capture-manual) ->> -endobj -231 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated-1) ->> -endobj -232 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 611.570079 491.179134 596.570079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---tls-cert-file-and---tls-private-key-file-arguments-are-set-as-appropriate-automated-1) ->> -endobj -233 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---rotate-certificates-argument-is-not-set-to-false-manual) ->> -endobj -234 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the---rotate-certificates-argument-is-not-set-to-false-manual) ->> -endobj -235 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (verify-that-the-rotatekubeletservercertificate-argument-is-set-to-true-manual) ->> -endobj -236 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (verify-that-the-rotatekubeletservercertificate-argument-is-set-to-true-manual) ->> -endobj -237 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-only-makes-use-of-strong-cryptographic-ciphers-manual) ->> -endobj -238 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-kubelet-only-makes-use-of-strong-cryptographic-ciphers-manual) ->> -endobj -239 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 473.570079 504.679134 440.870079 ] -/BS << -/W 0 ->> -/Dest (rbac-and-service-accounts) ->> -endobj -240 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.986019 465.620079 496.429134 450.620079 ] -/BS << -/W 0 ->> -/Dest (rbac-and-service-accounts) ->> -endobj -241 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 422.870079 498.679134 399.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cluster-admin-role-is-only-used-where-required-manual) ->> -endobj -242 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 402.470079 491.179134 387.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cluster-admin-role-is-only-used-where-required-manual) ->> -endobj -243 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 383.270079 498.679134 359.870079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-secrets-manual) ->> -endobj -244 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 377.870079 491.179134 362.870079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-secrets-manual) ->> -endobj -245 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 343.670079 498.679134 320.270079 ] -/BS << -/W 0 ->> -/Dest (minimize-wildcard-use-in-roles-and-clusterroles-manual) ->> -endobj -246 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 338.270079 491.179134 323.270079 ] -/BS << -/W 0 ->> -/Dest (minimize-wildcard-use-in-roles-and-clusterroles-manual) ->> -endobj -247 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 304.070079 498.679134 280.670079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-create-pods-manual) ->> -endobj -248 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 298.670079 491.179134 283.670079 ] -/BS << -/W 0 ->> -/Dest (minimize-access-to-create-pods-manual) ->> -endobj -249 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 264.470079 498.679134 241.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-default-service-accounts-are-not-actively-used.-automated) ->> -endobj -250 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 244.070079 491.179134 229.070079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-default-service-accounts-are-not-actively-used.-automated) ->> -endobj -251 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 224.870079 498.679134 201.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-service-account-tokens-are-only-mounted-where-necessary-manual) ->> -endobj -252 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 204.470079 491.179134 189.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-service-account-tokens-are-only-mounted-where-necessary-manual) ->> -endobj -253 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 185.270079 504.679134 152.570079 ] -/BS << -/W 0 ->> -/Dest (pod-security-policies) ->> -endobj -254 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.454036 177.320079 496.429134 162.320079 ] -/BS << -/W 0 ->> -/Dest (pod-security-policies) ->> -endobj -255 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 134.570079 498.679134 111.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-privileged-containers-automated) ->> -endobj -256 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 129.170079 491.179134 114.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-privileged-containers-automated) ->> -endobj -257 0 obj -<< -/Filter /FlateDecode -/Length 4115 ->> -stream -xœí]K#·¾ëWèlÀ4Y,¾cHb/ ðÁÈaVÙµÌXç¿Ÿ"›b7›Òhv,ÜckÕý©šU¬UER-I%ý÷µ¢<*á}pÏ—§Ó§“ÎHúKÝIzùzðÿ”r"^­ým (}~â¸õÂ.ð#Áþ -ÌZy<ýøÕé×qSꬅTÚh<ÿö‹8N…àÕ¾ˆ§n¡“&"V8‡JÙ}‘ÂK2'"Ô+ð``_DIÞ­&Æ(%‰"ÐÞÍd¬2 1´50³¼V› '2EÒ?‘¡a%•s0³ˆ.LèT†^pÊÛ ŸÊ!ƒ0³Ç¡À„0aT9%HŸ´J©KBïzf×d…3™ @#اÁ°Jë‰= •ïQN8%[„V††üDFiAæ=±TÚ+'œ¡ƒµnÂ)h%´Ã™=Ú 4ÒË §€Z sÞM8 ÂP¨ãÌc„ÑäÉ §Äƒ0nÂ)‘) TœÙã´ JAÍ8uAXbÔÏ8õFD/Ï’%JJ—ˆjÆip‘5~©–š&e©‰=ZʘÆ„Sƒ”2Áú §äÊ™ÁÍò˜:4–2âDFkʙڇ §Z” ³<¦ÑPÎŒVOdŒ¢œi(ûÎdåLPvæC‹ôB˜pª¤œiÉC3K9SS¦ŸÈxJJÔ3{¼§œg™€”3‘ØØ—A))g*ª*3K9Ó;9áPÎ4ÎM8EšI€¯göRÎŒl"£%åLÜ„SÔ–r&JœÙƒ ´£ ->áÑSÎôTZ&2)gÀ™=VRΤì:á­±âú5§yª5˜žR-%¥YLLz™f„}³\ÚŒ¥—Iäo?Ÿ¾y ©Áùçÿœ>~•¦“ÖÑ´DCf#e,Y§TÚŠÕ §É" •ŠÛ†S:lí0|é¿š’î"`Qœ© ni†P¥)Á Ъørb¸Ðf»תµR5J^YšÙÇÐ ïãsNÖ¸’Ф«FŽ6ûbÛ0tp‡ëÖJÕÈ<5ö÷¥çAíð z¬YãÝI— Úñ;<4\ƒ@¬ô4Ê®ùÐ+ûp½ÅÂÈ¿a‹¥“¾‹ÅÇ{•‡á]…ñ gÉ¡ÃוÇÓÅAÆcxWyÎj†G9¨GLqWy¼ãïåsÜ·VšF/k=bö5´Ëx ï*ÃYÍ`;Ôò¶aèà—­•¦±yjìïÞnNvǃëyÈu€áÝ Âm*ÏŠØá8~Sy¼WCÔªç!׆ó!Ý4vhǃÚáFí=×Ø<5ö÷ ¶XXožvBw?9î&7š!z„è¢÷n±t!Ê-ñã·È{!z„袯¢ãP|þDw/t=BôÑMtùGPµeo÷3¼mÝu[)»xkg»ÅÂ.ŠËq̹£ÿ,,-ÙÙ!;2^„¶ôË@ÝëÙ—Ñ4òGºÖÛÌp¶= •€Ñö@'m†Ò·l3Þ˜fx—ƒ¿wÔáë¬INlJøE ï²&ÃY¾ VÖûÂ8Ê69,Ï ÷­•ªQ‘Ëj‚hr¸‹WþB—8ù ,ç1­jyë0tr‡ËÖJUɼ5öù¥çÂípáz.rc86d‰ÚdΰÃÀMæ$ª1j Õs‘Sí¦µC;.ÔŠs¡6Ù“ykìólðî<íÄð~LrœÓqÄê«G¬îÆêû}¬2SÖ°ƒ>Áz묯¬;1ùüIð^ ±zÄê«Ÿeüi°”ýó»SX¾8,œ­¶BÔÆ’Â@kaZßãùÝÓé›?}÷ö»¿§…÷»§Ÿ¾%W =l|¼ùçùÝÛÓ_Þ‘‚窰ÔÿT¸ûU@üÉ+« A…7¨ð÷«Ð”à•U8}Ýá~÷x^Y…ó7Ñ}£ -eó7êYRB-ƒSAr‚Ž_,•¼ýë÷ûÇ%÷÷øۂï*¬¹-ønTѹŠüã5¢:ÛË×uW]îpKl¼@…;„ŸOEç*+¬3õÙÛøq×¹«=ôýýh®fªg¨àý@Eq§ÁX&QÈ:‚–±È¨^c|%oØ -‰”÷ÂÓ#ÖjÓž¥NÿK¼HiT~.çÿ¢g¿È² ß·s$[Ñ1Y»<Ò1,m¥ëèC¾Æ¶×R{ÈôR»Æ,m¤ëZ»Io‘{höĶÒq‘Á¥]›ßgýÑŽ\ÖÆ~Œ_@Ù}gt¶–~${m³È>pY¯Ê¶H&W|f›=’ì(ç[Ò5˜m÷‹åej+Ò¼©³uóý#óÜøÑñÙN}i×¾ü¹ ñ¾æ)—îÌåÌw Ö¾«6ºÅ¶ÔNÔõáÿ§€^àœ[ÿp›õV—Gjça‰±äã(–ç$ã—qŸ!û:ù¸´Q|YíúÒ®}ùy8ÒñËeç…ß—£RsÒÃ=Ÿ·Ïä—ø^ñÜ/_ÖØ]Û Õ2.uÄ©]m¢ßÞ ¦¾ê‡¦3ù­ØlØøÎ훹ÝÚ‘]øP O%&æœÄ/½R÷ßYœÈÇå~Oç¾÷wúÕ„k5øò…ùõÿ%yf[ÌÝENë0ºµÿn⢭˜”Š÷¥¶®í'bÊ÷JÁ×ű¹ãhw:®Úà›“ÃtÐÛÊËægÌûì›3ôh&„-ó¯GS3.a#¨TÙÒ^Ñ¡ó#;'9ë3:ðóT«wW5Vv«×çúÏZjŸ¯Ìëà(m…¾r¼xÐ%¿(ýðáÖ(³fíŸgû}q׶¥ - e’”R”aŽàKÎQ(ë°:;¥´YGÑlJ¶Y®¬:õK-y²‹ŒõúäJgJN$ÊSe^[s/´‘QuÝÜ&sHž7×¹ŸZ;,·d[?]Xn¯vMëôj®{ÍŒaÜ™ºU´òZV"Áõ>KÇïÛ¨M~PÌ–‰_k¦+`úwóÜ`×v"K§Kz^w„wš•doœttmàÇA,[0íG‘_Õ§ª,ø‡mŽ…2óYy¹D`ªy¯°÷"rÔö(Ÿš¸»óðF›oó¨0²í@ËÅ×g_S'¢]­:×3³F‹BóÒQü|Û¸V–¦«IBt~ÚZ\åaXÃëCvŒn)j­k=[Â\»âØ@«vX¡.[Cõµ‡å9õ‰ ¬õ–Q4vq¦É…® ¦™sW›ò‰ÕKË%őݬBöXRza…A.Ó²ís”(èö;bg»}ÿòƒ3du¼[Û[oT\D - 8›nð>ýz‹Hì¸ Z¸twì?£IïÜĵÇÈ«É»éËBóöVô¨Óf8FÓrúÃ’y)HÛ²v0Nóø‚’Ú ¿ï[Bà“,%Wó‘ ¾”ñíxTRa'jûf•ÀØð¨}¨[E›7‹>Œ§‘G"Ę’ñ'.4¦Ÿ§/\ÂÏÆ ²Îz(Ü©ämuÞ¹4ýJÙ§*•¸LŠø[Ï˯š±“,¿¼sõtR!ÞÿhT³LÊÌ~$êƒÀŒX>pŽ`>”µ-¬Ÿ)¨Øå¤t¼'e%©MùrjãæC0¥7ñzS.ÓK"‡raQ<éJ«¶pì}¼i‰ã…Îg¶šbl±™ÀÖ rÙÖ`»ð*¬´d Ù‚’©k0Åe(²PCå\C3»N1ú‹,8Fç ¼œH­^£CTÅ xdÅ43*TÔÉJVžP¼h~-KÓHdÖ×kP~Í%nÝç£*ŠÜºœ†r¹5S™rUÊ·KCñ|3¥ õx¯Å>–|”QJìTI]{8Þ T¹–žØ€—SÌ;øñdÜ@Ú´\!MzÔ¬Ý -{ÙÆKC/®¢Ð,X|w=þ»¤ -SјyÒA|oT•øYSc³Â½“8nkÖÁ‘â/û© Jmǯ ø‹æ8ù˜t«tõX‡2—1<ÝSš¶5yô°ª£®Â1…ÿÔ\ÉAYŒ u 6u(?vp³ÓÔP¥Á^”Yņ»©í6ØÖÜùÈaWWk'Z°lÌð’ -®á8ô}QÒ`µÚ5e l†QRhpK£ .ÑZ³“ -¡¶ UÜËé—S*s  „xW±Ž_ÍÜM½Ôù¿$ó–ÿ>áùïåÎÅ¥Žƒ¬Ùjní`^Ét¡ð* *XÆ€,í7Xù&kÊh¨u€åkl—Ë–Ø‹¤ªTÕW9hš uèÆ_zlž1ð²òbȼ”›V°—S]:'ãoCžë€Š'¼;têÛkº•ä3k’΢h+éô*»Nx£tʆfYgf\Iu}´9:êb'lOu{ [™åM,õ‚‰ b£êFÅ•†ÎžÑw‡×‹yÏ&Û­©½¢­‹hMxcT§SÛŸÖš£ÓÏŠ–àmªmi{ºNrZh´©v'|Œ2Է˲€áJ±`ÒR(jº L¬¶èŠM6_VÜ’zì‡^ruNÑÐ_¸§ g¥:ÑpVìF6¶[*^-é:2|8»Ö;Ù$ë\Ò0˯®(6I]°­nìu—j{fÝÀfˆm3SÙ.3É»òß=±3¼Ùq$Õm(_·5dü±Gå4èsˆÓo-åø#£Æ¡gû%Ͼf¾²Ý&)(\“ŠŸXÑZ:EãmÇ„Gz!\5•?´sYÃ.=Ï»òßšª…Ο·H»DõÔÅ¥¢ >þ4±ôäA9¹³§yÙl…ýð?Óë -endstream -endobj -258 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 257 0 R -/Resources 4 0 R -/Annots [ 259 0 R 260 0 R 261 0 R 262 0 R 263 0 R 264 0 R 265 0 R 266 0 R 267 0 R 268 0 R 269 0 R 270 0 R 271 0 R 272 0 R 273 0 R 274 0 R 275 0 R 276 0 R 277 0 R 278 0 R 279 0 R 280 0 R 281 0 R 282 0 R 283 0 R 284 0 R 285 0 R 286 0 R 287 0 R 288 0 R 289 0 R 290 0 R 291 0 R 292 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -259 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 790.370079 498.679134 766.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-process-id-namespace-automated) ->> -endobj -260 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 769.970079 491.179134 754.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-process-id-namespace-automated) ->> -endobj -261 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 750.770079 498.679134 727.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-ipc-namespace-automated) ->> -endobj -262 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 730.370079 491.179134 715.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-ipc-namespace-automated) ->> -endobj -263 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 711.170079 498.679134 687.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-network-namespace-automated) ->> -endobj -264 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 690.770079 491.179134 675.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-wishing-to-share-the-host-network-namespace-automated) ->> -endobj -265 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 671.570079 498.679134 648.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-allowprivilegeescalation-automated) ->> -endobj -266 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 651.170079 491.179134 636.170079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-allowprivilegeescalation-automated) ->> -endobj -267 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 631.970079 498.679134 608.570079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-root-containers-automated) ->> -endobj -268 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 626.570079 491.179134 611.570079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-root-containers-automated) ->> -endobj -269 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 592.370079 498.679134 568.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-the-net_raw-capability-manual) ->> -endobj -270 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 571.970079 491.179134 556.970079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-the-net_raw-capability-manual) ->> -endobj -271 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 552.770079 498.679134 529.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-added-capabilities-manual) ->> -endobj -272 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 532.370079 491.179134 517.370079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-added-capabilities-manual) ->> -endobj -273 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 513.170079 498.679134 489.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-capabilities-assigned-manual) ->> -endobj -274 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 492.770079 491.179134 477.770079 ] -/BS << -/W 0 ->> -/Dest (minimize-the-admission-of-containers-with-capabilities-assigned-manual) ->> -endobj -275 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 473.570079 504.679134 440.870079 ] -/BS << -/W 0 ->> -/Dest (network-policies-and-cni) ->> -endobj -276 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.970394 465.620079 496.429134 450.620079 ] -/BS << -/W 0 ->> -/Dest (network-policies-and-cni) ->> -endobj -277 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 422.870079 498.679134 399.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cni-in-use-supports-network-policies-automated) ->> -endobj -278 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 417.470079 491.179134 402.470079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-cni-in-use-supports-network-policies-automated) ->> -endobj -279 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 383.270079 498.679134 359.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-all-namespaces-have-network-policies-defined-automated) ->> -endobj -280 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 362.870079 491.179134 347.870079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-all-namespaces-have-network-policies-defined-automated) ->> -endobj -281 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 343.670079 504.679134 310.970079 ] -/BS << -/W 0 ->> -/Dest (secrets-management) ->> -endobj -282 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.962581 335.720079 496.429134 320.720079 ] -/BS << -/W 0 ->> -/Dest (secrets-management) ->> -endobj -283 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 292.970079 498.679134 269.570079 ] -/BS << -/W 0 ->> -/Dest (prefer-using-secrets-as-files-over-secrets-as-environment-variables-manual) ->> -endobj -284 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 272.570079 491.179134 257.570079 ] -/BS << -/W 0 ->> -/Dest (prefer-using-secrets-as-files-over-secrets-as-environment-variables-manual) ->> -endobj -285 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 253.370079 498.679134 229.970079 ] -/BS << -/W 0 ->> -/Dest (consider-external-secret-storage-manual) ->> -endobj -286 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 247.970079 491.179134 232.970079 ] -/BS << -/W 0 ->> -/Dest (consider-external-secret-storage-manual) ->> -endobj -287 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 213.770079 504.679134 181.070079 ] -/BS << -/W 0 ->> -/Dest (extensible-admission-control) ->> -endobj -288 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.506282 205.820079 496.429134 190.820079 ] -/BS << -/W 0 ->> -/Dest (extensible-admission-control) ->> -endobj -289 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 163.070079 498.679134 139.670079 ] -/BS << -/W 0 ->> -/Dest (configure-image-provenance-using-imagepolicywebhook-admission-controller-manual) ->> -endobj -290 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 142.670079 491.179134 127.670079 ] -/BS << -/W 0 ->> -/Dest (configure-image-provenance-using-imagepolicywebhook-admission-controller-manual) ->> -endobj -291 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 123.470079 504.679134 90.770079 ] -/BS << -/W 0 ->> -/Dest (the-v1.5.1-guide-skips-5.6-and-goes-from-5.5-to-5.7.-we-are-including-it-here-merely-for-explanation.) ->> -endobj -292 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 481.202815 100.520079 496.429134 85.520079 ] -/BS << -/W 0 ->> -/Dest (the-v1.5.1-guide-skips-5.6-and-goes-from-5.5-to-5.7.-we-are-including-it-here-merely-for-explanation.) ->> -endobj -293 0 obj -<< -/Filter /FlateDecode -/Length 2240 ->> -stream -xœíZÍ«7¿÷_Ñç@:ªÒ7Ã’M kxò¿æ9aœ=äßOI­*UOkzüìä”g3žÖ¯KU¥úRIc˜ ýýèŸä`I)ÇàæõÃôq2Kô†þTŠÝ ¾~Ký7D\²ñpi 3ØùƸS›|!80m‡CˆÌDÃ?}5ýª§„ܨ)&NÑ,p ‡ÂwEæ:)L\°3VxîLD0µÒM¡«^ŠÂ‹U)pƒA¡îwhÔ¼qdY Cç!òº‘†vÞé¬ý«ÍŸöæþ -V!B¯³;€;óã ówÌbœ˜¿‰À¡ùñ†ùqo~p׸Š`%Q£{óÃØü–¥+yÝHC;¯”Ù–?óñá·Ÿ§oÞÁbæŸÿ?½ÿª&æÎJ²’ã$½™vÚ~Ú/éø’Žût åOœÏHÇÝZv+ƒÕ¼$äKB¾$䟒ãÄ{n·z+M_Òñ%ÿöéøœv 2‹ž9;6¨•Cº‡+>œCf0ÛŘìÑÍ¿=íh. 8£I‹µ@ïNh¼[,}8¡ f±!“ÏhÂbS¦E›3ò™>1-ºxB“ÜâœÇ|¦O6‹ Î:<£ ‹KÖfs›)Ó½AgOôA“dmBnñÎx{¢¢Y¼Ï>éƒañ1L'4–t¦Çx¦¥tÊÿ“CGÍúO|Š¾Ô -—àL–m -'1F.—œ½M“–x'æ1º%Þ‰yLÛ}ÑYÌc -K¼ó”£K¼ó˜)çïļ¥Ê•îÄ<-iIwbÞ»;1o‘v®QÌ·"4ª\žj„11E’h3õªí*—4¹“o×sƒûqðêÛ‡ B»þ s°‘R̆æLÆÊÖ̦oÞ|ÿã?ÿý–Öòð~úÏ+cÀÑ'ÒÇ¿þïüðfúî$œÈÈ›ˆLÂhl ŠHñj2aŠˆüð懟~üsE Rëñ‹ ŒÂ¿V„…âé犸çëÒ-¸-ªîø:чRÞQ€Yc¾aÚ7å¸#Üûí½¥ÇmìVú„ö]hæ#u7›”CŽúÞ%ã\D0ã…*Eê·k -…¦DÚñq{.ïªb4.÷ÖE‰ªtù+_ç<«¹~[ˆçÅÆFëõ‚:mžÍ@l¤òžõ«r™§oÆQ||{]ß,jƒÕ9®­—å±®d™Çí«ÒÃ&[¹žÉ%Ôh¹ßøæ›fƒªGêv¨k-ö|ìößaLÓt®º³í̶¦ú¾Ù–ûî©ûoUv Ý?,§ðªtí¹ÊxzáUlÀæ÷¢Cnº¦Mób_U¾ëÞ¿¢cåsjS{mÓMî¶.‰[ŽSìvýB_»à¹/› tm-¶­m ¯|$>nº2 ¥s³§ûî¯3¼:ÚQ -KTyçznå8•K—¶6²|Ï6dŸÇ{ö+í€9·_ÓÍ^Åö½¸ú”\–Ø~ìöª9 xTœcUçךÔé?%ŽÚÃûõ¿¦6/)$O *õ‘Úê!J¿CíÎ'»Æl—X;š²7QoM-(õú㎦I/-!œlrÄRºì²ÛEë‡{\9›áÓV®¤t‘ipÝ»š+·½¶¸47.ëÐ\ò¨B¨¼{ÚèJuØ¥ÊûÍí¯!T‰jÏâvèî+{ouÉÚÃgÇßw×±>N—¢¼­o[á%ù8ùT´Ó¡ÆèÍõHñŽt&¨øL{4i²ï ZæSëïÄ…ªýtLG0 9¹ö»²4ú­?¢Ó?5'­·XüöDpBt˜Îþ¡4|G˜ŽXø‰pÛ£^˜P°u[®j®(méÕ¶Ç,Ì lèy5e¾çiv+)È£’›àH3á²úr¡·ÁäŸÛFAT¡Cb°3tš¶3ìïÂ`ý4J\‡)/3Ó¢<Êy¦€¢º6(÷3-FåÎ3pH¬½F/CA0dë¢Ì*¥² ž)Å!Nü4€ÊUÙüÚ0kGΔ÷ÊktÆl`â@rB—™n­Ó®qóâ©(T©OÍlù®Jõrßq«6j(•‰‹‚±ü¸‡CñûÚ$7xשÔ|™|Pû^+ŒçÐÓ U|N¦ÇKGcbSQj2FjWþÝJ…´TžúP®èQRrSད4¤ê¸‘ÀÅN«ÌUö€ïdô¥è -µXl‡*“)'WþCC¢$ÏÍÉï_öpööXªO -endstream -endobj -294 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 293 0 R -/Resources 4 0 R -/Annots [ 295 0 R 296 0 R 297 0 R 298 0 R 299 0 R 300 0 R 301 0 R 302 0 R 303 0 R 304 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -295 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 790.370079 504.679134 757.670079 ] -/BS << -/W 0 ->> -/Dest (general-policies) ->> -endobj -296 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 478.142757 782.420079 496.429134 767.420079 ] -/BS << -/W 0 ->> -/Dest (general-policies) ->> -endobj -297 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 739.670079 498.679134 716.270079 ] -/BS << -/W 0 ->> -/Dest (create-administrative-boundaries-between-resources-using-namespaces-manual) ->> -endobj -298 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 719.270079 491.179134 704.270079 ] -/BS << -/W 0 ->> -/Dest (create-administrative-boundaries-between-resources-using-namespaces-manual) ->> -endobj -299 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 700.070079 498.679134 676.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-seccomp-profile-is-set-to-dockerdefault-in-your-pod-definitions-manual) ->> -endobj -300 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 679.670079 491.179134 664.670079 ] -/BS << -/W 0 ->> -/Dest (ensure-that-the-seccomp-profile-is-set-to-dockerdefault-in-your-pod-definitions-manual) ->> -endobj -301 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 660.470079 498.679134 637.070079 ] -/BS << -/W 0 ->> -/Dest (apply-security-context-to-your-pods-and-containers-automated) ->> -endobj -302 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 655.070079 491.179134 640.070079 ] -/BS << -/W 0 ->> -/Dest (apply-security-context-to-your-pods-and-containers-automated) ->> -endobj -303 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.673228 620.870079 498.679134 597.470079 ] -/BS << -/W 0 ->> -/Dest (the-default-namespace-should-not-be-used-manual) ->> -endobj -304 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 477.090267 615.470079 491.179134 600.470079 ] -/BS << -/W 0 ->> -/Dest (the-default-namespace-should-not-be-used-manual) ->> -endobj -305 0 obj -<< -/Filter /FlateDecode -/Length 4758 ->> -stream -xœ½\ß$¹m~ï¿¢Ÿ l$ê'`H€ÄÈÁ6|öÂ~ò°×¹±Ì8çÁÿ¾?ªJUÅ®î™Ùd³ÓEªX¤H}¤XÕe¯?Ÿ,þËÞ.9—ýõöõòóÅ,)ü«#¦ƒÊþ¿)\­ñi ¶Ä®_™@iI%¸¯¯ü·;”Ã_/þÅåoó9öê–R¢³áú÷Ÿ˜åiI9%VËXRX!,)¹V´ L,EcA¼óGV‚¾ä ± †w.+¬ 5È'KGV.Y¯± †-!),k ‡uƨ<(b¼'§ðlX"&=jk(àrY|J¤Á€+añ±$ ÈXðœÕ€€ t >jH@ºx¬\•]¨x ÈA а€º8OÐÖ#Z4, À­7ÅjX@À[olÔ°€¸PÓhX@@\ÊÑkX@€\„RÑ°€€¹ìÔ°€ºXîIÃêRˆNÃì’ÏQÃî’·FÃÜ%™*º €5,:.d3iXà»dMÒ°Àw±Œœ†ðê‚8Œ V—³Ñ°Àw]6^ÃÜu )@å¥k4,ðÀ]`‚–¹».§aî:”øcÔj¸  z9]° ߃çHÃ[Ê pפ?eM­›KÔ° wm†ûT^YlÂŒ+볼؋†¸‹DEà.à3kXŒ_Q†¸k ™Zãw­+FÂÜEÝ4,@8,€†Èo‹5]•WSrÖ° wKNÃdáÅd ‰ÊK ÉjX0^ êP "pé½hX»&iX»Æ£ìQyÐ…²Ó°u xX¶ÊšF±º¤ÐxÀ]cáaÜ5& "p—Ó¦†11Áe  à‚*ØiX³_JJQÂX x(ùT^\p4,H„ÈRyÐÅ'¯a -ë¥PÉ$]È9 ’ƒ.(4,@E°‹ê[Ý@„‹º?ðЕ¢†@²%ï5,H!b?†YÕxÑ-9®U^Ï& Rò R±Õ°{©%cÑkX€_¸\Ò°j€g½†©@, Š–ìP™k< ]PdkX€J3kX}\¸TÔ°Ë;M ¥Êƒ.€ àÞl °ùÁfÛ•](x rv¼ÝÌä ]œq 6y¿™4,(º âÓ° ˜¸p™¬a`l<»_gº°c’³·O†”€ýTBA·Ú§‹£Þc_àBÍ Jýlo,v¿ÎtaÇ$goŸ.Œ{c%ZÎùÕ>]´õû¾ çÚù/`eí~éÂŽIÎÞ>]:ïÍ9GUûtÑbÔ{ì“AÜöÄú£‡ô!çtýìÖu§‰„1+ûmöÈ —öÜ_oþÌôƒ=º°f.òÝö=H÷O!éÍŸìÑ…áãña|‹Vúë>~hx:Óöéš¿t‘ïôפéñW¦¼¥Åë{tagx/üõQ¼ $ýuµü0Óöéš¿t‘ïôפéñW¦<¬Åë{tagùKøë£ùËö¼#ý%éÒ/z¾›éûtaÍ_ºÈwúkÒƒôø+S]¡Åë{tagùXøë›åãÂ3xOèBÎ)Þ›kfƒê]í]Zƒ`Ó4êÝÙy²î>:jÙo¢­Ó…í¬Ó%Ô:¡•^K úë_?°Nv’«…ˆo—«'ïÝÇJ-Nô£}º°÷tÉóÞ¤•^9NÞÓcùuº°“Ì-½÷Í2÷ä½ûÈ©eƉ~´O¶óž.ùcÞ›´ÒëÈÉ{z,?°Nv’Ç¥÷¾YŸ¼7å÷;ûÍ;ô£}º°÷tÉóÞ¤•^UNÞÓcùuº°“¬.½÷‘¬~l‚ýëç‹-‹3ý ²OãP@wá(Y—¯Ÿ¿^¾ûþßÿã׿ÿáj¯Ÿ_.ÿùKc\ÄïÍŠ¿údÉw¯ 7÷‘=~-~ËJs?ž×1>à7ãƒÝh|œ¶¿B–kcÛ8\Ïÿ¸~ægL¼_åøŸÖñÖ¬)l<å¼ví®kXéR_>Ÿuv›ÎmL=çË&ç¶éÌrÿëúùû§æ“?Ž ­søι™æ¹Ùà7Þ&/ÐvŒóƒ{pnüÕjEÜÔ“ÞÁ,±?!Öôæ9}Yç¡Îïæÿã°ÿŽëoüísøÂ6®2ø˜ýż:f³ƒ8@seèÝç'n>p«œÊg9e\³êr[ÇÊk³¾!o1”×óCZíÿ·Ïü eƒ?üúòÝ¿lëeš ·¸BÖ”Šü_0ÅHú+?ªYJ6áúËï.À1Õ“9Õ‡5gíª§ÃjÑ6…¦üüg,¥yÜnËæÁ\#û$\ˆ‹qVD½y†·1rüæé:Þ­ãµÕ«xXDR÷p²ƒîÞòM»bÔtâqŽÚ ÷PaåÕqq=¯Û2„ÛiDMú¥úÌc(~˜!DçR’ô7G”t]’æ"ªÚÞ4 U;øï—ÍG_Vk¼ÔqÏ®~î¯ÛçêüH%9I³}“˜Sûì;4ÅËGìÞ|OÛjkJð[|·1Vã¤ÅH‹áÛX“êõdœæÓ5ZöqÎhXõ o´Ùm¬k»ÅzÓ[¬Í¾ß2ÄËvMå‰Ø×ä™uµ5y67d÷s£ùoŸUêÚN£j¶ÊÂF‘Ì˘»zn:ÉDAÌýf«VÑÜ‹=)«al»nHG¬lsYõoUA³ó4¦ˆ1•·ùòc>ºóÎæ¦ï‹8oÓã±O±ô$Ã?ZÏOú°ùŽç¼þÝÅsåo¶nåÙ¼rNÏq›úºz™×jÛñÒ*ƒ;úpSàqþ¾oM÷0ãY ±V’ˆI™§·qÍ·UvÃË6þ¤²ë2ßãûg°üålÞ¼=ÌÛNm¾;£ýNæÁzßaǺ;0»Ùí -àûÿ>ÚàF÷4” îÉùrwrþë¤ýuè®_r\ýЃ¾÷çüTÎGôñftøŽó÷fÿžÊÿˆžü0±ÙzYG=ŸŠŸS™ÑMÄFü¿Ä`ð‡ëì|5w(NeѬ·ûYÄ#Pð›Ä5®´¯evû•ú—±í4î>?´Óø‹ÑÞ´AT¿¼^X!âÌhd¶Þ,v#óKÌk¶£ØU ±é âèåØ!pœølÉb8¸Ü ;ßôáYûXºÏAìªË©³Âým¬KÂgÄÛ—¥=õU¥ò«“š¨6»ÈíFB©Ò©¡ìñÝO -évQˆŸLíáÌÎ^s­Â€÷¶@ò}\iãÒí¢ŽÛ¤…î©ÔGåqji3?T¡Îݼû´›x–4ñ[Å1Ô>ÎäÈh¾6YÄ;o¤ ¿^BRF‡&´Ð“Dr;9›/ƒšr›*,ÍFã¯ïóÿ+T„Ne䩬ï!ůàGã"¼6;yž$Iu¼6Ö§1VL—M©Ç¬œ/›2žA·î36QÅ” :ßjí¢c™l{Ôu2CX›ŸŽ•’hš¸Ü ‡ʯyèúRå;q1Zî¡ËäرóU’S÷¸å÷©t}»\óx«‰ÍïÉú¹]déÙn\L‡b…A0úZÉmµvtâwD¶c‘i÷vù륦9GÁñ͈H–_&+/‹­•¹~ßÿ¹øëoÛ7Ö<îLG ×±u"ËLÆ/“ÌO’ùå’Ø— :/`mcC‹†žÇxíÇéf{i;‚ÚΕÄ0æP]]gIÌL8!Þv³Ø ™úf$ìõšq¸ ñ›LšVüí¡ óàÑHÉW!G> -endobj -307 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 740.570079 409.271317 725.570079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://releases.rancher.com/documents/security/2.6/Rancher_RKE2_v2-6_CIS_v1-6_Benchmark_Assessment.pdf) ->> ->> -endobj -308 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 311.068925 661.574079 469.171708 646.574079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke2-1.6-hardening-2.6/) ->> ->> -endobj -309 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 646.574079 183.998124 631.574079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke2-1.6-hardening-2.6/) ->> ->> -endobj -310 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 351.653153 220.574079 461.230790 205.574079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://www.cisecurity.org/benchmark/kubernetes/) ->> ->> -endobj -311 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 205.574079 165.615800 190.574079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://www.cisecurity.org/benchmark/kubernetes/) ->> ->> -endobj -312 0 obj -<< -/Filter /FlateDecode -/Length 3687 ->> -stream -xœí[K$· ¾÷¯èsË’H½Ã@$FŒ8ˆ“E|rX·=6‚Þëü÷óQU’XÝìÏ:0|Èf¶EêAññ‘RU‡³ÇÏG*Wk+™Ï—w§÷'ïJòø×{ý%~K:ÏÅ¥ÐrMçwB âJK±äóõ„?7MÝýzúê7§Ç„st­åÒù‡o…ÅäJ-Å'ƒÕÀò VJ®”˜²ÁÊÁa‹­Y,ˆ‘ -G¾gˆÁ­&‹18Æj°*Ä .îY bÄØbAŒÐR1XÁCŽ½7yÄ3S4x!¹ ¥g‹ƒËµÅfò -x¡D‹Gärá‚Åk.眪ÅcÈ’š/A–˜½Åƒ,LµX¼ Y(“7y%ÖB¯@–í4ƒW! ¼«š<È⳯‘K­2›¼ž¯¥Þó¢O.U"oñBp ¾I&#åbð"dÉ>5“Yùhñ²0~,p"Q©ÕˆÖ(ÑMž‚ÅK%ÆÂ&²ÀøÄ Y|É ‚œà’…±Ç-²±’ãÊÕBX›ãRÈ‚Ø’ãÜŠ…äx1X@@²$ÎP€,ŒÈ5y…[X@² ƒ…D%2YX@@[†·XX@€[ö-XX@À[ö![X@\ˆé-, .ÕÌ ®Ô,, `.;-, €.½XX@@]J9ZX@€]âš-, à.qð0p—#Md[Xtt*YXÀÀ] -¾XXÀÀ]„Q´°VuðÃla|ÅÅZ½… ÜÕ³… Ü)Àä‡$ ¸¶`enànL>ZXÀÀÝÈ1[XGqÀoVÀ]@H²ËÈ€µ°ù¼H`2Q–XX€»¡!ý1 ´r¡¶laAî† -ó™¼æBƸ…–]ȹYX€»HTdaAî>«…Àx‡Œ-,HÀÝ@ÈÔ¸bó$à.ê†daÜÁ ¿¹àáº&¯9ßjµ° wKÑÂdaç+ÄäG -ŒGjaAî"½7 2p×'" 2p×3Ê“Y¨F P×€‡°5bŪóHañ€»>À¸ë}idதM rB¬@@‡*8ZX+»VJ¶° 7J>“—ÚÉ‚âAH€,“Y¸°…(¬]£V-,(²PŒ”YP2XX€ŠÀµ€êÛ<@¸‹y>`È‚JÑ ™«Ù‚’2ÎcЪÅËÑÕ¸6y¼P,,(…Rq°°g)Wôà”K@ ð[XPdA¨XX€TäjDenñdA‘ma*]ðB°° FÈ‚jЂ+N™Ù[X€ ³$TmñØË9³ZXP9;)-,@˜à¤ ¤4y0aaì#‡ída?8lâ˜cò %¶° Ö(ÇÍjaA­%úha|SΛÅ‚æ! *> šÏNÊd ãàyoaA çÍÈ .å¼Y-,häqÞÄψÛFYΛÅ‚Æ%¡D1yES3☄ó&ÐÉâeÈÐr–ófµ°eΛ¨èÍ+*çÍb^wS2Y<à.*Óla8o¢²µÎöÀ› -ê3›YåÄYÍèMÈ­æ•ö"[óÎÀ|åŒd^@Lœ,k6o VÁ Þ¼6ðÀ_Ôc|‡ Û퓾¢Êp‡‚œY‹l²!,Õýï©W\aÑYdž®°¯(æ²\!éîÑökѯÊh¯Ùž½×ûä#ú/ì“ÇíÝÇáïÞœP¬o -óY­ˆŠ5¢ôFMöæÝéãÏÿðÇÏþò%Š¬7O§|‚ -g­ñ‹" 9Õó¿oÑ®øMøÌø¿lôÑÏÓÞÎûgá×5¾ÏU7zçcžtÙ?ó>ïXÃo}eŒ¬3æL£oÞ~õ|½_\ŸùëßçWûé<>Ê”’’£ÜÎñé?Ïo> Ç¦õ(ª›jñkyYZÔ0Uöÿ÷¾]œ§öÍcõöÏ»¨©­±·[|•ÉFߧMÞžÛnÀIü¸_z»TÝ×y ráu=<)Uïæº8¸Îe—­Ý˜¨­¹úsmÃÜCOCSç¼Ö S^î8\Nd—ùº ‡k~‹ÆÓ÷ºÛ˳:ä[N]yµfØæ›®Ÿw÷¯»|C?—­}”Ô<ùh£CÈÕZS¯cÞÂEé^ûàÎï²½‡Mça÷¯ƒ<ï[ùV/c¿]NµvבZËôÙº‡ýF Œ¾oU Œ>®ôþy×eVsº?†};ÿÉàhúp¸ïYýÕ;¿úzŸGëàø¸oñ›_‚ié¢Q:ö}Å^g|ïk¦¬t¶÷•8›:¨ æoå>êøN/òh!”Ö!ßëåÛ=^ÓŠ§>ßÓ6ÏïßÈK4}®¿~vúø·{2=ÌÙ\m|“Ê/5¹ší²ý*¯á´V}:ÿxúÛë²sôõ¹ôì¿^>;ô0ô:sƒÊ)ÓÏþG9Âë´=ýøÎý±¬Çé.Ül¨Ûy.ñ³ú>ŒWúf(Ê×TBÒôWëû0ÍóúVñ=rØOÆ]Öš;rÌÈ—E›6º¼6îã]N~—ïí¶÷U>Óe{¦9Ä Zç°Ç¼Ïó´°gÔ°·Üö -ö&²w s„xÆÙgZ–K,›Àæò”½ãOǪov<þýä÷òRù˜ÜD¸HóçX²éÕíöÿ<ílí-[àóáËý¹BIX6%y®[S¨ê1ÅôÆ»¼à}÷sûêÓÎ’#¦”Š±íÈŽ[‘gp& åâíÑJLx+Þ:âO9n¡_Øx4NûO[úz¼¹ Ÿooúz†[æá¸Ô÷Æ>½5åûSîÏ“C§Èã$¢¼t|Îì$C•v\©žäÞf3]èÊç#û«Úïg¯ýím//tÉÃüíÕnÕØûo…Ô;87»<ܹ¿ié;¹ÆÑXäëIÞ´eƒÌ¼bâ²÷~ÎÅa†Ø ]N^Ün{"‰øýãŠCR mìFƧ1Lž€²¨?Ü6ˆ™Ã=Yvï]ØÉò¶ru´·ò%å!3ˆkBÖ}ׄkà‹ä@^Ó=Ñ«å9ò'ÈÛ-Suy§èZuA™ôE™ó9âåÐØï¢^Mª¼#9¦Ú~ÒH Õ&5žÓ <íd.'ƒø‘S3Œ9ùÊjò¦ÉF¬Ã‘xök£ŸAºœÌ~ûliZªÌ^u mCóK”åêT§â ÚEåª"_¿P1"·ØÚœÅÃÖr¾ÙM°Bw/'¤¾#_O©½Ó -Ÿ†ëi"©y'¹úå/‹Šôº« -¡9hòœ^þnP‘&U§<]JÞµY­µˆÄæ$•¤éy¢[\V_¥®PÊôY­¯P½Ñ_@wöž;P•Ê½´!ž€Ô#9L¯›d°¡Ÿ‰•šè‡ò`p’Ótåë¼äL3T“Ÿ‹å Ü=Íy9Oì¼jr™òâÔ”wΫ|~/=±ñ-Y\¿ŽE9ú™íÖbŠ¸(,ò‚Ñk'hè$_WY÷rúþÔÓ\¤›<šBÉ[èPy…óèó9~ÿuâóãQÕ–ÇQõãÄÖYg2ùÖHý‰dùÉ Î°QÄÈ -ö9ÔÕ7 o˜y®=²íYmƒ— yU¦~ Kw=?ÿ>$vÌËx«×øé·#¡®W5‘¨9ŠÔ¿}ëkÎòê”|¢pU×%¯óüýÉý-ɸ?Ñ+ÁŽ‰| ò=j[„+íe†šj|ÇÚÖ¼¹¥×)àá;¯ö£šW˜ý’h6‹å{³$Eo•¯ûo?Ié×@ép}Òþ %ù{º -endstream -endobj -313 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 312 0 R -/Resources 4 0 R -/Annots [ 314 0 R 315 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -314 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 450.884598 712.370079 494.898026 697.370079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://kubernetes.io/docs/tasks/tools/) ->> ->> -endobj -315 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 297.976884 697.370079 309.064286 682.370079 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://stedolan.github.io/jq/) ->> ->> -endobj -316 0 obj -<< -/Filter /FlateDecode -/Length 4307 ->> -stream -xœí\K$· ¾÷¯è³Ë”D½Ã‡I#b`ŒÆ]Ál€uùûù¨Ò«ºÙÝ3»ëرÇãÞ®"õ (ò©®’=þ>µø'³59—ùxz{xw “á¿ZbwSÙ_ã“ÂÑ'l‰9ß -Á'“Jp)øçìv-þxøæ“ÿöuìÑ™R¢³áøãka±7)§DAa°Èz…‚IÉ…¨°¢5b) b„ÄŽ/Y bpÉAcA v.+¬ 1<'ë/Yb¸dYcA [BRX– ‡uD*‚³w -Ï¡ô¨ñœ51WT^Ï&§ñ¼71± Vãc Yã1d …¬Æ %XfÒx…}N/B=©<Èârò/Aí…—! ¬+«<ÈB‘¬Æ+Þ„’™U^rÊ—³†ÀxƒÕiX€»Öc¥ÖxÀ]ë -iX€»ˆ‚†0chX€õÍX‚骼b¨ä¬aAî–œ†X… e ‰ÊK†d5,BªaAîby/Dà.ï5,ˆÀ]b„=*²øì4,@\ÜVñi«†BãwÉb†5p—( "pW–M b‚Ë@@ƒ(ØiX3›’RÔ° !ŸÊ‹÷AÂD @–ʃ,œXÃÖ¦ø’5,H²xç4,H² dа)Ñ·š@˜‹š0dA¤¨aÌä¬aA -ù´ªñ¢39®U^Ï& Rbƒ¥ØjX€\Êd8½†x#á’†<ˤYà*`)2Ù!2×x² ÈÖ°‘.xÖjXdA4¨aAvYf$ °B#—„ª5“ä™YÂÌÑH¨¨aÜ™&RåAÀ„†˜I¶ƒ†H~l"ÍQyÅÖ° g'éfÖ° gÈâÈiXÛ”|3iXP² âÓ° P4&kXHÂb3òMÇÀ/%ßÌOÈ7‘â)~[|”|3iXP²„(*² h*Šß“o4^„,®°†%FÉ7³†3o"¢W·²ä›IÝ.î†dÒxÀ]D¦QÃÜ ßDd«åöà ñ™ÎÌ’qfuÇ€½k«ºe@À^¬VIÝ3 €¯äHê¦ÄDf™£ºk€eLKê¶ñ&l»OëU„9$¬™9É Ü‚ÝœcóWŠmûc?~øì‹ïñûÞ|RwÊúŸØF'Lc¶]Ál¿=@!H×Æ…ürFdzFö€é^àωu×¢O‡…Œ´è`Åb÷tdƒ¹Ò—î×öÒ‹l õ´Œd!?, #=#‡0ËŽîV"Ïv'yŽäq¡ÎAÏ®¦v4ýžß`¢ß­mLmT–5q:ăÆg©ªÈ\¦2çÚbui{—(!†h÷s©–É^§h™e­ÿÇEëx­¯äj²Ë“nCim( ÂíÉ’>÷²¸º îlh’÷6´Ò5ukÝYYWʺ åņ2JŸ“CœeGw+qµ¡¨ÚРÎAÏ®¦v4ýž^<÷gñ\ Õå¿ãå…‚²;íŒ1¼Õòª“íô¶èþÅù^œï7æ|QþKÇË‹g8ß*ð2âU;Ë~q¾ç{q¾g:ŸêdÏ 9¯8ä‹ã½8ÞoÌñžr")4ž£üRβ16q™š4âRv‚/¶g•ïSçv&*{ë.p9Ž =/µÞšºUÁ»~ƒEÞKÎGUÖg×ùè²Êi>E¶»~]Q…¼Wø®tOšûK!#ñÓ…¼Wøã -9­-xYëwZ›<À”3å¨Zèë|ôY_û:Òd}v.k)Öh‹è “¯ðGœüËíÈß½:À}ë3ø.l˜sýµR"Jò‹ËñÕÛÃg_þáOüë×ø«7‡o?'²ŒÝ¾É×ÿ £¡ñ œñå‚­Ò¿øûñÕ—Wú”{FŸÒ‹[U€ qÁç„Ïî[óµË´Ñ{9éÒõ²oš(qëÖ¬Qã±ÅOçíbßÎEyÌÄùÜ¥X»Úå/>µŸK¯í;¶ïN[ïR¶JùÝFïR_Ð[Y¾—ïS+'_Ú e¯à ×ʤ}áIYŽM#]3ñ\{û±H¼Óâ34ÈçìRTé©õ÷ZÓ¤è¼* ŸitÕÊ¢ÍÞ†Ü[ùY)mZ´44G{Úh; »Ùx©Ñšæ†ÖʬCßm37f‘%ТíÜféÔ4›*ãN‡ˆj8€0|Á™6û—Y‘öì?.Û‰^~1”§Ò.Ûé3Ü5Þ’Ã;º&ͱT½(¾¨È&O'b¹s±(mW¸Í¿íê!SÏÜú¯ã—~½c-ú¶Ó#d~8Ìqyð{ì¸èï K*­õ}éEK;õú¦nù|ü¡·±zwlôZÅåN÷ezP`9ì`yC¶ŽµÍV®Öù®µCÍvº.ÞLÝK¿›?M½žûŸ<©\}¦ù²Ü׶NÛÜXwS_ñB_}>픡Ѷ1ðë6¾Þok7«>ä:Šn²ÛsêÈ™§.延÷ºùc³KÛëùŽs:r‹¾Ãiê•ßÜ{‰gcß°fúߊ¿UC ?"u„ ¾xyô P4ÎrfFæd¤s¢5,yn÷ŠZ4bœ<ƒñ¢#'Ï T2ñyˆb¹÷º©fÜZ†¼ò³¾õÎoËŸ¿üÛ_¾Z"– å}^î7Å_íDÂ\›‚wglµµS§},Wí;Ï~F¹á}Ù™t˜@[ -Û÷ÂëåF=;ûÛÌþ²½QçõäO7h¼‡¥®úÙ\¢ÑGp1ÇH‘§Ë]–6ÏôÞï§ëìÇ:ä¤+¦Ì²»SCzD0»„óž)?­ÎÏmÊSÞhMN¡¸¨šòP¹ß›ËŒˆö¦Šb}z„Òê¾éîÅtAäVa)Èóx)ëÔ4ÁµµcBw1¿×\Æ ò”z‘çAÏ:­úÜ .÷‰—7ðMòËMcybÉÊ~ -ø ìnÙ =,örn¤ÛÑŒÌ÷0Pa¢CÝĶóú¾E{<¿—å6. é¶™ý-Ûv—f¼UAlµÍ«T›Ì4³ Õ.Ù”$iÝwxË.#•nckKºc–O«òK³Ê¸½a•3>üB×'$›QÞ*(1ºrw£¢~‡™¾äïWTšW•~»zþþ—1¢ˆ,¹¼oR,¹þSóöÚfïs ‡øÓuÞÇÏÁ“¼t³óç÷ËÁ/Ûy^ž¢¿&Ççà—mÿ¶rð”ÏÇÿŸƒ·ùò„Fsç>4izn~Ko™Îõ¶Ïãé7”›g_ÎtñœÜÔUtÓ,6˜Ìõ'X.É„ºÙyÇ4ŸXç—cš’6F¦–_˜æ/ -A~®²…±Í´ô”W0Š¼^«’ÿ‡ÛlûNomgÈ›cÍ:ÌrIò2èµMB÷zó0Û’1y(E¶v|l VY¶—J+Ó½mI~FÒÒŸštQóΞ ¿ÙPBènl,ži$ý=Aªýµ‡bj2Ô<×þdlŒ!ô1\Lå»C¬/6Ûz¸Ål¬“Ó¯ŽQž[“CŽò25’BýÙªl{¼R³žön”jLj‘œ,"o•ogŒ-7­ü»:éo¶°ý׫zäUrvýf’rä+dæ¹ýu‡,ÄvI£-¶c7­ÓN9y´ô¬$„ÚåÜrXÙ.]裑ú¡W“Wq™z!__FÝ:Vˆ¨í%YF±‘åجl|»‹C”»Ì Îy-;œï’­§….‰´t7ÉŽ»~Õ_”ØÖqG-ÜÀÖy]ÉäøÂüD²g؉Ãm¢ãö'ÙæY6tkë¸s ^ó¬NØ{I;ÔîJ S‡ Ù ÓuÖ/š 7ˆ§3-Ž@æCçf.ØۭQ9;£K…›u8¸Í“çç’|\šÄK:¸ì–Š¸YÅíÒa™’íă\;ªùy -/}-7K!·~òx.³kÓVG±Aì×›µa¶k§e'"ô{»×Ï{9vê€öA}\©c`B€§S}½ûÛ±æøº Ñwjh´åßó#È™®1C Þݬ6ºPó¬Ö -„µSî´]§}¡puÚh1u™b«ÖÕRGœU-¥SLꫦÃ:+C‰a•xQVÚí+Þˆ–ü€ÂjÎiŽŽfÉKf…×ڃʳ¤ï´Ë¾yßw_mË0x -gdJ³Z¸(ù^ø÷>¾£¾X£•êuwÄæyfj¶É;_¦#0„å$¿ÄyÙ.yv÷=±fí ÓaØ{JVôÖEx£Üg,Mõþõj…Ô!=OW_R_ƒ²õD׺‡X7‰Æ­<¾áäg/Ao–s'Ýx-*î¶Oêß»¨ã -endstream -endobj -317 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 316 0 R -/Resources 4 0 R -/Annots [ 318 0 R 319 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -318 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 527.554475 101.433071 508.804475 ] -/BS << -/W 0 ->> -/Dest (cb1-1) ->> -endobj -319 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 195.360432 101.433071 176.610432 ] -/BS << -/W 0 ->> -/Dest (cb4-1) ->> -endobj -320 0 obj -<< -/Filter /FlateDecode -/Length 4053 ->> -stream -xœí\KÇ ¾Ï¯˜³—«Š¬è bÄA É!Èa5ÖÚVäò÷ó±»^=Íy¬´¶¥ÄZŒfš¬‹E~ErºÇ-þ¾tø/³39—ùxzwx°&‹K‹ÍÅÂþ¯ŽÎr2Á•˜Ãñ(™T‚OñøtÀg—só§Ãß¾8ükÛǽ)%zŽ?¾“I9%VË:RX!˜”|ˆ -+:ƒ%–¢± FHìyÏJƒK b°÷YaeˆAœíYbøäXcA WBRXÎBç­UyÄ2“Wx.˜¥G特ø¢òx.yGdbbœÆ+&ƲÆcÈŠu/@–à˜­Æƒ,L9i¼Y(’UyÅçD/Aí…—! ¬+«<Èb£u¯ %3«¼žÍ)ïyÞ2‘ÕxΙÛ$•Â&ŤðÃ)‘¾ñ$ È:ð¼Ó€€,d 5$ Yž«ò Ö°€³†ÀxƒÕkX€»ŽpRk<à®óÅjX€»ˆ‚†0ãhX€óÍ8 ÓUyÅØ’³†¸ Xòà66IT^20$§aÀØXÄ¡Dà.Ž÷¢aAîÚ@¤aAîZFØ£ò e¯aâðඊO#X5Ö)4p×:ì°ÆîZ›Š†¸+Ǧ†1 Ág  Aì5,ˆ™MI)jX‹!ŸÊ‹×AÂdA€,•Y8±†¬M¡’5,H²÷$Y2hX€ˆÀ‡è[M ÌEͲ RÔ°HfraÖ° …ˆ| ZÕxÑ›œ×*/ƒç’†)±ÁQì4,@.e2œ^ü‘pIÈžc R,p p™ì™k<YdkX€H<ç4,Ȳ Ô° ûŒ,3Z pB#—„ª5[É3³†™£‘PQø 2M ¥Êƒ,€ °?’l ü ÙDš£ò Ö° g/éfÖ° gÈâ­×°¶)ùfÒ° XÈ‚ˆOÂb£‘0YÃÀ8xÖjXP\F¾éYÃø¥ä›YÂBù&R<Åo EÉ7“†…!K@ˆ¢ò ‚¦¢ø-0 ù&ÐIãEÈâ kXPb”|3kX€0ù&"zµ$%ßLj¹¸ -Iãw™F p|‘­–Û[oHˆÏtf–Œ3« è 8[Õ’öâ´JjÍÀ|%GR‹™eŽjÕÇ*˜Îªe üE<Æ;LX«Os‰*ÂÎÌœd‘nÁˆî{Î1šÑ…fk}ìÇï_=àð=~ÿïÃãK¥¬ý9Kk†=à |ÁvDß˧ˆs‘ ‚=§­LšÊÊk‘M„Âq'ÿŽûI!¨ˆŠÞȃ¢äþ„£ÄÖy<Ëè°\+>»IÖçwº)tBLPŽýƒÓ¥Jš'ZR¥}~§—AÂr’Pãöþßjü²0ææÜ/ä­Æ?••"en¬óÞm¥÷vzñ}Ÿ'¾ÛJïíôÒî1ì·¯8¬×/ -âqR"N,£˜üúÝ᫯ÿÇ?üå[Œùúñð÷ßHQÎZΑ ^òŽëàVÐ $ÁgÆëqmÃùÕ?Ž¯¿žçt´‚Áí|2…ûnmþ»×‹¾¨ãe’ŠS#ðAÖˆ1Ž-d&„“†Ÿ×ãc,¸* @.€Õ*^b'ˆÈz-ï4ÎmòUýÒQhBËV8ÄE>,ð§¯ÿúço¦ áÕ—NÞýÙ{ZßÝw7Ú4Ýv“¹Šñv’«_Nv±”êa¾>«†Ð{8½ÇÕ|®s»úë{£Wƒ’¶bTüf¥‹bBRèµ-Çi]mìT ²ÒÅ—W5Ôe,[Çvc~8kCÓM†&7UcÇü|«Ô÷8d–ël«ùߣä4+YôÚÇœôDCÞe=§I¾ú¾“aâ-úã3ÝÇ¡ßYïm ¹vá§õe'ͺi}ìq»OMo]ge‹{´UεeßÀâx²®i}çÓW§4MãË7&qƒMþb…¶Çu¼†Yó8‘$哯öãô]ä:V¼*Gr—ä°c-‹67¸B³"›|½-ûX”±ÞÝì“÷Õù—õË<³WVKèž6¼Högé[×=M^u>Å ÚÜ{Ï›ÆY>_Õm9_hc̈+½YkÕ½=Ý–½ëaîÛÆuu •îê8PÖ1k»¥Ï›:Ž­¶Ótñ8t/ó®^8ôzîµòUs(äzë´îó×ôEn§¯¶ŸnÈÐÖèêøm]_›oF]W}È7ä]ewç6ÔÐ6]Êû2ÞÛêÕ.¥o·E7Ÿ ûSa7ÏÙ‰ {šoI»Ç«úát¦Ÿ†ÎxqãÈ÷c‘°ªrÄôÆa´oD9wöù 0ò§p('“ñz«8Ëv†vˆµ¬šG;¤ÞÔ-¸8]‰ÈåK‚³éV€«¡Òb$íxëÇô¥1=Y©ÁÂ.´1†´b¤mŽ–qÞÁ¾‘%ŸÑº£¶#uÐa„ ñÄkíz?7Ö¹:ç~¼Þçíàg­¼‡©¯kXwjrãÙ †.Eä’ &Ÿ‡}q‘âÒuC¿¯Ï/mè]^v„¬™\Ô#y«lÓdB—L¦]ÜÛš@߾ɼG$8s–%t“zÜŽÓÍÄ^Y3Ö§cg‹Á¶s¸·Š«¼ÙšYçŸé¡©¹RÜÒlío»—n²ìÙ «)8¥d³H7LöÎ>ŸŽÉR6”¡v§šìçƒ:Ÿ£{]0;¹EÇ/e"·Ü‰#¨Ü2»ûú|:f'·ú1yû ¤|!ÔY3‹Š ¯ÔêˆèËòµ]*^­‡ø±Ïõ‚MæI—²I¥X&7½Émg“^«–É YMÀ[’/ÕýÍHòÎ>ŸZÅ,0_³ Í69Ðôû§­Ówz³¨©–Ð-îqÛ¶;¼ßÒ÷ó>Ǻä6ãËÖõòeØí„×,+:׬ö"'«|«}Õ°îëò©Ù•ÜtÅ®ÆÞêûzG+&H±Q徚jÝÙ_kª¯žQSeVòÅšjûa»þ^g©gEŽUîEoSýõÞZê¬ÓQŸoõ{×*h/WáLrßâÆë?¬Â¹çyÎÃ%9ìXˇU8÷cÿU8S>_ÿg_á¬ûÑå©•½åµñ¡AÓ+Ÿ×ô†4èLoÛ*©ý¨|^]¿ßÙÍ,ÓGUIstgc?§JšsB!~•ôÎ>¿tJÔƒ eH¬Õÿ,UÒít/S%UÆüµJz¡^qá–‡ÅcŠÍò„Ãæ–;y¯ßôp_Ÿ_ÚЇ¼r‡ ¦-é×*é'P%¥¼žŸòÙ?ÌnvÁDI¾ƒ—*÷‘K009Î7ï˹¯Ï§c¢Á™L΋?ôùÝì‚9.Q0Ë å÷VKïìóé˜cB(#OÂäϧZê,ôé ->òÏX.=›õ¬ªõm½‹;ô Až¢ÉÁåé¦ðn-7›Ü0Žý-äí&Ò£;»as«B )?ß@ùèa -Øy - ‚þíªGWS*=yl,Õ[ñ§z)©Ô6M§S -ÓÃüš¾,©“­{ÐRÚÇÕ„î{Š?íO´=uoiÈ2ßCÝ»“í)áfü+)U_ChkØmåûC\žÞqËœ6b÷½üÄÃ1²!Ù›r”'†²%î÷׺EÙîx¡çòÃï{«ú[VŸ•G§ÖÒ˜.jû÷˦¿;ÀúLl5©å¹v»³oƒütß5`…Ì< -[7ÈB¬m‹]¯“5Ú ˜L9o‰ ÄÖ£˜&iÉúQJ¢£hÝäy¶­-O\¬+DôŒnO–Õ#1«dùmˆl¨^Å.JˆMfÇ€<·ŽŽ7ÉŽìD {¢¦dÏM'ÏvÕÅAì¢ÏªsÓö·¶>MÛyx:`Z:§>©Ty"½ Õ´ëm§Ñ$TéÔÐZö á¾O -étPˆ_Ú64c3;Ú5y®o%æfHÜÛ•ÖN!j»:Zè;•z«<º–¦ù!Ê0uÊ]ñ -í4œ“üØÍä#–ÚÇ-Yj‘}¯mžìÆŽÞ‘Ÿ!)­ÃÀ -šéÍDšÆíäl‡½ jÊMUpÍF“§¢äÿ*B§ -ò,w“’/tÆÕ˜D|³“·Jšé±£kpœFÛI]r¿4íè;[¥½€noÝ5¶¡N*›è©4ñ¤:xlÉ®[]' „5ýt¬œ‰¶ é9tS~Ú‡œ¡»j°}²è&s}ÜAŽ;Ÿfrê;îä1Õ.ow²ùF<-Ÿ“Åôs›d½í§Ý˜l"Á -ƒ<`ôi!7oíè$?½Ó>'ÓNÜÓá‡ÃrÌy -^‚ôHN~uaŽ¼Üñ?hó5^ÿ<ðñ›ö˜ßzŽ{ÛÑÂwlÝç“L~£'ßI–ßìiÄî6Ñóûƒìòhš5ôsÜû ¯yt·Ø[K×ÔuîL C‡ÙwÓõŽ&Í„+ÄÓ™{ ó±{3ìõ’Úâ¤)ˆ6©p1/—yðhÉÇiH\IÓq¤ƒË~ꈋyP\N–!ÙFL1Èy’®z‘Ÿç­ ™kº˜Â¹¤ÁãqÌÎC7Ú²Š© B`š/æÙÍ“–<ŠÐì^½ïUäØ<@¨Ú;õi¦ö… µžN¥å2n/û™CK²Ùœwh¨E“ïQr†kŒÐ‚7³NÔ<ºÕaž”m3i;(:\€f­˜šL±vkjYVœU-¥S 곦ü+]‰aÞ•¸k+㶯GKÔ 0›s«³£e%³B‹sïNåÑ’m?7oçn§íqZAâˆLíèv-?ÿ>ÄwÔAµVío©Ž¸<ŒÇAÍ.‘§å·mŽÂòs5‰óT.yvŸëõ“}•¤ÕOæ™°†‰lrò«•ºO`”ÛŒi¨ö‹–z·bÕ%=OJyhÀÜþ±ÝågË– –"Q¿L’*ʯ’½ò¼KVŒw)¥Mùdùû/|¥ìQ -endstream -endobj -321 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 320 0 R -/Resources 4 0 R -/Annots [ 322 0 R 323 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -322 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 564.123057 101.433071 545.373057 ] -/BS << -/W 0 ->> -/Dest (cb7-1) ->> -endobj -323 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 179.034971 101.433071 160.284971 ] -/BS << -/W 0 ->> -/Dest (cb10-1) ->> -endobj -324 0 obj -<< -/Filter /FlateDecode -/Length 4066 ->> -stream -xœí\KÇ ¾Ï¯˜³—‹U¬ø "ÄA ÉAÈa5ÑÚVäò÷ó±»^ÝÙ]­l%–«&ëÁb‘_‘œî¦£ÅÏׄÿ2“ɹ¤ÈÇÓûǃ5)Xü[Zl.ö÷øMáH–“ TbÇ÷Bðɤ\ŠÇ‡þÛ]ÎÍûêð¯m::SJtŽ?½{“rJ6(¬–%¯°B0)¹V$ƒ%–¢± FHìøœ• —4Ä`ç²ÂÊÃs"Î*Ã%b1¨„¤°ÈBrÖª<b™½SxL„Ò£Æsdb.®¨¼%§ñ¼711ÒxÅÄCÖx YB±¤ñd Äl5daŸ“Æ‹ÅGoUdq9y— ‹ƒvŠÂËÖ•Ud±Ñ’Æ+Þ„’™U^Ïæ”Ïyβ÷Vã™Ûô*…MŠIá9Èm(*²oÆó…ñ£ñ€Á§œouâÝÞzÒx²8—XåAl¾†.B›¢ð #¸¤a€KÉpq¬€ËÞp次€ËÅpJ^ƒW‚áX’†ÞxŽ4 ð²Žx‚, ÏUyÅÖ°À;È‚xY{ <Жa-xÀ-ÛBxà-[Šx.Ä´x ®Ï‘5,ð€\˜RÑ°Às=°SÃÐ…»' öÓ‡oîpøø÷áþ«¥RÖ~ÂÒš_Á \ÁvDß„åûˆs‘ ‚=¥­LšÊŠƒµÈ&BḓÇóI!è÷Ûi k¾YÆk_VH€ˆç( -1mð¸ŒÎKU‰hÈúä>WEN[ʱ k -ç:Òd}rŸ—5o1C)å½ÒøSYhDòv³×* jIæ½Ì{«…ÞØçÅw}ž÷V ½±Ï‹ËZÄ3JäpÃæ_küÂê»Ñ‘¬G¼×ãØt1-]6àJ(:‰úÄ./(°5N2”ˆ6¾8‰èpL#·¼iüÛ×îUŠõ{£~I>4ñ‡‚5Ž¯ß¾ùÓ«¿þù»ï±™¯ïo~ƒ¦wÖ†ðíׄÏÁíþ¦õ/ýãzÇúõ3Ÿ*/V^£çJ‡Œ¡~f®ï·mÛ¸mžF9y¿ýûñõ«Ãï^(A«&ãqò~¢„-ñò‰èðÕïÿø‡¿L:ôAÖ&c˺dnüÒJCœƒ’ÛËZekèb™šA¡€4k7á›eM]Þq"ÑkÖ‰¤Â1܈¯Øóm]>Æž?…Ë÷Mìø£mBµ†bœäê—“m U“ï¥ÈeÕ6c§¿¹Ù3búæ´õ—æ¡Ú•´Ûâ·ÍgÖë3zk[Çá¸åqšüî¾Úl¨}üÐG³Ù.¯ýe¼~Ký‡,rÝõIc½íýdó·¨ÕÍj}3I±x—­3Æ­æ4) -Uiƒ¢©¼Õè@ì˜$Çiýµ“–ºæì–ÖÇn¨ÐdL•V5×µVFûV1¡ã,˲æ¦í†,§3™u(©ïE:RÅŠB÷ëx ]æq¢—º‡|·v>N³˜Ež¸¬ñ’%>&‡kYô²±¦EÙä;ÖhÙÅ¢Œ½ m¨HL³— =sY¿ÌsWé±^ÛIß4÷¢iœåó%ÝzÚ¯?´1fÞì®êÞž®ËÞõ0÷mãR]C¥SÀ¿ŽYÛ-}ÞÖqlµ¦‹û¡{™wõ§¡×½ÿÉý¡ _–ëe¬Óº7ä.êËŸé«í' Ú©®ßÕõµùä$¿«í¨úk(ºÊN{jÈ™‡.åï2Þ»êÕ.—¾q+§á³]>Z÷µéQÔ¥µKIk³ök†ÿÍX°4|É&zHUŽ˜Þçã•PãÆ>ÏJE>EÁPVF>â¬d,[vA.Õ­oGÉÛºNIJE¾ÛM÷f -sÅxŸ/鼕/`Ú˜wCZ1À}ˆÝÃò¸¥odÉ;ZwÂvð z åûœ¯µëýh¬s5ôóñzŸwƒ?±ò4Ö°:å6ÜãƳº¡³ó&—ìaB,_´¸L.\1ôûüÒ†>äõ9wÊ5š¶Ê6M&ô˜É´ëi[èÛg'‡Q¶¿mшVw¦ÚÌ£ã_‹‘¶íébîo·¦Òù»µô15wØe¶6öç"˜$Cq)épI&8â|-Í»±Ïçc’8}Æ–ÐÅïó@ŽÿOYÃÔjÆO)‹Í„cMM‚] ÿæDr“ÆøÇR¥H"·Ê)»I/I0os¹%’‘ŒË7iÝçÆ>Ÿ[™$0_ò¢¥|¶/Ç…eìÊw#¹ÞzÃâ- o(ÿ­É‚ªÝÓv^0ô‡>Á6s¸d›/_¼ÛNxÉ.£Ü϶Ú¬M‘ŠÑ·tùܬRnÒº`•#ÅÓ1ç†zQLˆ€ªÄ·•áŠýR‚»¦Ò2«ôÍìùwÛYzYnä@ VÉ)¦rÝ­¥·e̳bm+æê¼—/£%—MÞøóóÊhçã<­Œ–"?&‡ky^í|ì_W-åýúÿçËhu?º<¡ÒÜÞ‡M/¯]ÒÛr×àFoÛRœý•×äÎÄ­.žR^ËQÊ^îu½¹¼vcŸ_:Åëa´c,{gõªÃK—׶ӽLyMóKyí‘$ùC/qkDL} pžb³“û!/úm}~iCïòÊ ·^vÐ})¯½PyÍçõÌ’Ïîîc]E7ÍBòÌÙrSÒÍe¶û|>¦éIî5þRfûù]åÙe¶åžôX(ûŸ±Ì¶ôR9£”Ü\ -ý¼¾ÂÜØçs+h ë’ýœ·Õ=»vö}½>¬3y)ÊÓ­õ}Ç®6¹²Aç7â·[3´»³tkýE¾WKòt­Ã>”$Ï«Ž¦íom]š¶óñtÀ´~O}P©ò\ªi×ÙNó“P¥SCkÙ7„û>)¤ÓA!~mÛÐŒÍìüi×äéÈ•˜›!qoWZ;…t:¨íêh¡ïTê­òèZšæ‡(ÃÔ}îŠWh§©pœä•A“Xß>nÉRgì{-7Ð×-®;ˆ§Ž>#?BRZ‡64Ó›‰~·“³ö2¨)7UÁ5Mž-“ÿW¨*ȳ|[©© 0_IÄ7;y«¤™;ê°¦â4ÚNꢔºÍÎú¢l•öº½u×؆:©l¢§ÒÄêà±%S·ºNkúéX9mÓ rè¦ü°!9CwÕ`ûd‘&s}ÜAŽ;frê;Nò°o—·;Ù|#ž–ƒ÷d1ýÜ&d¹I¼™OŸl"Á -ƒ<`ôa!7oíè$/0j‡“i'îéðãa9æœNòÌèIÞ]1G^tüÚ¼Âï?|ü®=,¹žãÎv´p[7äù$“7åÉòæ£Fìn3O°?È”GÛЬ¡ŸãrÿxÚѤ»ÀÞZRGPêÜ™†'²ë¦ëÈOš ˆ§{ ó±{3ìõÒ·ÅIíR³mRáb^.óàùq$§!q%MÇ‘.»©#.æAq9MX†d1Å çIºêE~ž·ÂË\ÓÅFÈ¥<Çì> -endobj -326 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 545.373057 101.433071 526.623057 ] -/BS << -/W 0 ->> -/Dest (cb13-1) ->> -endobj -327 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 194.231993 101.433071 175.481993 ] -/BS << -/W 0 ->> -/Dest (cb16-1) ->> -endobj -328 0 obj -<< -/Filter /FlateDecode -/Length 3986 ->> -stream -xœí\Ko$·¾Ï¯˜sÓ$«ø$AŒ8ˆEr0rÐŽW6m€uùûùŠÍWÏ”F£]ÙVKfºŠb±øÕcºÇ-~?sø—Ù™œKŠ|<½?|8X“‚ÅOm±»¨ì¯ñ—ÂÑYN&¸s8¾%“Jð)øwv¹68üý7‡íû¸£7¥DïÂñ‡wÂb2)§dƒÂ*`YG -+“’QaEg°ÄR4ĉ=_²Äà’ƒÆ‚ì}VXb'G—¬1|r¬± †+!),g!‡óÖª<b™É+<L„Ò£ÆóÎÄ\|Qy <—¼Æ#21± Nãc Yã1d Å: KpÌVãA¦œ4^„,ɪ<Èâs"— ‹‡vŠÂËÖ•Ud±Ñ:WÈ„’™U^Ïæ”/yÞ2‘ÕxΙÛ$•‡…MŠIáyÈm(*²²^ãdaüj<àD ”³rZ½œn²ä4^€,Þ'Vy›¯!€Å¦¨AN\Ò0À§d¸xÖ@Àg2œ9k(às1œi0àK0KÒp€¬Ï; ÈB–ÀQCr…qrUd¡Â‡,h adñLЖa-à–mqð–­‹bZ ˆK9²†È…) ˜KÀN  ‹ãž4,  .…è5, À.qŽp—ØY ¸K„ž*²À€5,:r™4,`à.9›4,`à.Ž‘×°»j`‡QÃØŠñ9[ ¸ë³e ¸ë\€ÊKg4,`à®X‚æ¹»>X¯aw=û¨a Ŭ w!A VÃø{ð• ®ƒ†É‚Y*²pb X›B%kX­%ªsHð™9É" Ž#º9ÇlF4Ûêc?|wøüÎ÷øÝ¿÷¿©•²þë ÔÖ {Æ1ðÛ};,Ÿ"Ì*{N[™4•)” ¶"›w'?ÇË7;I!¨ˆŠÞȃ¢äþHxvmóyêè\ ^È1p¶§¨Ïîó¤È A9Ž7N—uªhwêH“õÙ}^\V$pí¶”rÃÞ?Õøe7΀c7 ùTãËB%ðmÞ[-ôÆ>/¾ëë¼·Zè}^\Vq~^:‡[Ðé‰Æ/¸ù—û»7DÛ§ñ¸l3Ü©G\‡ñæýáó/ÿð§?þõk,üÍýá›ßJÅÐZÎaþä×Ám4¸ -4ôï÷[Î_üãøæËu>øx”€Hu?ŸLá¾ÝšÿþMU u%ÙDeHi1ÆqÌ„ØtQðóz|Š–›r±$*X¡â%°ƒ·FJnùBãÜ'ß1.….´l…CÐæCÝ€?ù·¿|µl@_|æäÕŸ½¦íÕ}ûD›®Ûa"W1Þ.rËÅ.êç°^ŸUÃ@^=\^ï6cð¹ÍíÚkl¯Þ JÚŠQñÛ.Š I¡·¶Ý9Nà 4×:h}¾Ö§¶?᯴×8ç’ë¡«&o«·½_dí2È°­oܯQ¯ò¶ãp‹ÒÓªôoº^VÅ©—U_}^¹v’6¤íÏ.: -¡]Û=mŒÝ{_Wj´¦·¡³2ûØ·Øs^öw‘¥ê©ëºÆé–õûúÜŠ€b—ûm¼Žë8‘$#”O.Ç{Èm¬xUù€M—ÃεT½ìliGE6ùô)Zö±(cW `Ýzûoó×õË<í¼Õ}ñmO»¾Ý<²?µo[÷ -Ü]^u>å ô¹/ÏÐ2N}U·é|ý¡‘Þí®éÞžž–}èaíÛÇum îÚ€ômÌ°àÍÛ6Žm¶Óuq?u/ónçiêõüüÉ'ѡ̳,×u¬Ó¶7Î_ÕW¹ÐWßO7eèktm ü®­¯Ï'ú®µsí ùŽ¡›ìî܆:næ©Ky­ãá±è1œ¦¾øþÚšH>0ß­iÃy®Ö3~P±!I9B ã´ÆøDlpcŸŠŒ°@>ò˽Uº¡»îrÚ–vñ¶mÁ£Ó•ˆ¼NêþgÓm Ô ÙÌØËpféÉJYö¡y7¥Ãêsô`†óYЖy»,ùŒ6Wwh“#Üiˆ^o7ú¹¹Îí@]Ž7ú¼›üyÀïnéëæ¶Ã¶иóì#†ž²É%Lj4H¼”¨®úm}~nCŸòBç28=þµÊ6-&ô˜Éôë «ÎÌ,ž©`W[¿™{§˜êÛý6þ™cLÍ”ãžfólÿºÌ[7O¶l\¬Y·Gî%÷„IÿªyÞØçÕ˜'C4ÊPŸSÍóÓNýë3ï-ìk&x÷…š<ªC1œ܉š.úN­iÙ.- ÇB}¥– 7,ÉGàg“^+&pÃì,É¢þÉáÆ>¯­  7q\±XñÁ»õnÙýÓÞº½[Ø’è Ë¿ß·ê÷ôËyŸa]˜øšu½|•j?á5Ë -1v+Aè#.ŸH^5¬Ûº¼6»‚Ò®ÙÕÜ[}_o¨~È-#æø¶’ÓÛYòøÅ–z»»}ûQŠš±~;Á› u¼^¢º¹ü„Ë|³èñZ¹©ÊqQzì¥I÷ò¥£˜Éä݉ý¸ÒÑå8Ï+%W“Ãε|\éèrìÿ¯ÒQ¢óõÿÏ—ŽÚ~ yB£ùó34izIéªÞ¹Þöå'û ()¥ÎÖøœ’RöÎdOrGÛÍ%¥ûüÜ©Ìpà™¢±LÞê™öK—”öÓ½LIIó×’Ò#Éå#†Î gD4| r7fÎõÎÂë†~[ŸŸÛЧ¼ò(…ì ÿµ¤ü£¼ùyïï^›™?b¦I€½Þ+qsiéÆ>¯ÇLs”ûCKj--=âÙ‹|X‹Ëô––ö“^½O%P7;²Ùø˜Â“€û¼¶@‰éšÅŠ/þäûUÚ8?UIêeä}†5×Û‚¯˜óËײÎf<³å¯Û·) gòàC./÷ñ~²Éözy×o¿£îèÎîµÛ«CÊ÷”fY’ܤþ˜îü» \ËÆ<Ž¼DQlVY*H¥µé:_²Ÿ‘µ´Ì§f]¶é¿gÃ÷¬ ÝêÀ5ÑŽ¬¿gHu¾»¶o§™MîÆ¿’5„¾†‹­üpˆõ Wº³1çå©ücdC²7å(ydKìúÖ¹ªlw|¤gý.ƒ£Uûz+O<ÊÓ.Ûw,­ý‡ºéï®°‰½ÜTE¶•œ}¿˜ä‡ƒ<ŠÎ -™yÖ¬ž ±½µc,v£Öi' r2å¼% ϶·³N&8¸½õ¡¯Fú‡ÞM`ÛQ½I~›X!¢gt—dY=rÉF–Çù³¡v‡(!v™AœòÚv8;>Ivdb¸$ÚeºIöÜåqòø×P]œÄ!úª:·lo Çëo"ž˜–Ω*U"îCuíz;h´U5ô–cCxì“B:âg¶ÍØÌÁ_vMÅÚˆ¹v¥·SH§ƒÚ®ÆN¥Ñ*Ï®¥k~Š2MòP¼B;-µä$ßO²œKýíž,e̱×ñ´-˜GwO¸¾ ?BRZ‡‰6tÓ[‰´Œ;ÈÙN{™Ô”»ªp4;Md‘ÿT„Aä©o“’ n^ÍIälò^I+=ÔaMŽÓl»¨Kî¥ :ÆÎVi/ ;Zí¨‹Êz*]<©{²V7Èa]?+W¢íB`ºIÔvä)gG5Ø1Yt‹¹‡1î$Ç+9wòdáwŒ»Ø|'žªcãs²˜~î“L²·ÃÛÍÉâ  0ÉF*¹ŸÖNòm)ýí> -endobj -330 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 579.123057 101.433071 560.373057 ] -/BS << -/W 0 ->> -/Dest (cb19-1) ->> -endobj -331 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 227.981993 101.433071 209.231993 ] -/BS << -/W 0 ->> -/Dest (cb22-1) ->> -endobj -332 0 obj -<< -/Filter /FlateDecode -/Length 4004 ->> -stream -xœí\K·¾Ï¯˜s3,²ø $AŒ$ˆ!99¬ÇZÁ:€Cþ~¾bóÕÓµ³³Ò -PHfºŠb±øÕGv÷ÐÙâï„ÿ2“ɹ¤ÈçËO§ŸOÖ¤`ñ§–Ø]Tõ7ø—™,'¨ÄÎ?‰À'“Jp)žŸNøïêr-þtúÛ¯NÿÜס³3¥DGáüË{Q±7)§dƒ¢*PYòŠ*“’ QQE2b)š -f„ÄŽª3¸ä ©`;—U†žù£ªÀ —ˆ5Ì ’¢" ;ÈY«ê`ˆeöNÑQ0NšÎ‘‰¹¸¢êt”œ¦óÞÄÄHÓc YÓ1l Å’¦ °%³Õt°…}Nš.½Uu°Ååä5]‚-Þ)Š.ÃDWVu°ÅFKš®xJfVu:›S>êœ &dï­¦#2±éUV&)&Eç`K´¡¨:ؼušÎÃÆ_Mœ>嬬V'«Û[Oš.Àç«:Ø‚É×ÀEØbSÔ +È.iàR2\k à²7œ9k(àr1œ’×`À•`8–¤á€·# ¼…-£†ž` cåª:Øâ kXàlA ¼‡-Ž½†hˈ <à–m! <ð–-E <fZ <×çÈx@.B©hXà¹Ø©aèb¹' `Âvú´QE„CBÎÌIY°,ì~ì9f1ÿL±í|ì—N¿~@ò=ÿð¯Óã¯êIYÿK µ4ƒ_a¸‚é`ß„áûˆ fµ †½¦¬tšÊ4ŠƒíMŒBº“?çã—¥0TLEm샢ìý±!íÚúòÔÖRìœäÈiê«ë¼hr#(çñ…t[§‹Ö~Wm}u7²uúQˆ ˜÷ûõÎ:oî×µß{ýzg7÷+R¿al·-ßï×;ë¼¹_×~ïõëuÞÞV¤˜˜m)å¬z©ðÛ‚Õì;ñû|©ðé{¿Ø“Áé8`dØÎ"/Áy ^ºXúºoh®5Nv+e|qÂî²±/GÌ­…óî„D߬Øî!Kò¡[NJ n.œßýtúõ¿þëŸÿô âíÝãéÛ/­ á«/H>ÝÕgÚ>éûÊüýüîëÓoßͼ+fa9ØŬq¹,úzÏáàrµëëßýá÷YìÂÞ{Õåó"›ep¢Üú¦öÛg—k9oe&–¿Ûäâ¹>È[Y—qõ¶ÓV/S&åCØê€îl×(®|#v¼om_®ÚlmpiíÇiG•¡©?WÆ­ÿ:~·¥Êc»¶Í6ÞÚà4ç¥Öå͇Ýö-nô·‹þ&k}×U¢Œ{¶yÓ·þzü¡·±¬è:ß—ö™¦ïíåeÛ‡Öº½]jchrjmç·6[¹Zç»ÖŽmk¸ûâqú^úÝÖÑôë~ÝÝôE¢+_8nqÞ}»Ä:°}— ]APùÊ,]d“%ßÑ ôÎ:Äš>EÎôD&ƒ•8«'͸O’Ì× ñùN„àR -Øòï;Ù–dKºJ†ÛgžýTÙí~œ·r‰ÈQú¡Ç¯^ª]Lˆ¸HJíà•Q¹¥<ѯ;È“›£© qYäe^ócO.K"ºm§ÏÉx¹k;}õ¡äÄË ·²ãûȉ|ú™ÿORžqkZÝúíjÉÃ~Tƒ¸tÂÑ=qÙ@±Z±š{ÉImó@c;ÍÕuoG4ê½è€6ãÇc;¯#r¯ù;>šhÛþß"œ®Çÿ_O4Ú| {B“¹ëµ3e· ¢øÊG¯! ¡>}Q`0,“Û¡›äÎ:Ÿ  õM¤ø Ⱦ“{ˆÏÛtËw÷p Ñ”OF¥¿—‰ˆ1sJíψˆ°ÑiXÅèí_½î}¶X®9ÿ±aK¿Þùë昳¿ókl–Çæb½óÅh3dyüú{gφÀæúø-¦Ÿ>!Ýwòê´/×°èiz¡„vZ¸èz¹QfÛ<¶7꼟zj¡7Éʾ޶LÛÒѦQ®ëz›ÏÞ¦Sc[~s™lI*åtmQvÊÆË.tÂ#™8&ky ¢ÈÇWVjûÜâÉ¥/yÕ…Awå ©Û‹ç¾:s ÷S¬¡bùÖ²’–ü3qc÷±{$ÓWç\4uöa®ëú]¾QŸ}¼ ßÛ²ÛF­×´®÷»b³Ô(žÍwÉŽÓ–@W±ÙÏ3[lf%.Ù”ˆ¥«oÅe,µó¨z‹ã…°¼¯Êg•9ÜŠÊ™x‡Ÿ¾iÏߥ€FC矱½ÊËã|Ã;/yÁLJÿú gºzÈfdhR^¼õPJò¬ês!æÞo0H§È±¼ŸûÆ7´ÇŽ¼qN×C“öœepë÷“«Ö0í<ïqãS"wW¼½×ø¸k\½ò›¾?YÛï|r±g··)-9Õ1¦òçS¬Ï]S}÷ÆÆlÈÉ˹gท¹ÒfyfÛ3õ©£êl:?S³¾Òüó(ÕÞr¶òâ“<ô¾½½\´ò?×Iÿ [{6±oæ뉶Š³ëSüt’7RY3ÏÄ"l_íh‹i0tÙK>™r] ʶ¯óBVÙöÕ…>©z5yR˜m/äë³²[ÇŠ5#Å2zk¨‰å­Þl|»ŠÃ9¶ÂÙ ¯egƒ³â‹b€Ã" G¡]º›bÇÝ’·@†ëâÓW×Ñ2ý½,vxî.áå„nýµôI•Ê»„½©î]g‡Ì/F•! ½ä˜ó¤ˆ.'Eø…íM3&sè—Y“726aîÄ£\éåÑ外k­…1Si”ʳj鞟¦ÌP÷y8^‘]–“º$?S°¬ëû×½XÆ\c·4ãÆ„—R ÄO§”Òab… =ôV¡_Úâlg¼LiÊÝUXš]&ϳËÿT„!ä©_ˆGH E˜W³Y›C¼wÒ*uXóqšewQJ#fW(åtGéá±tqÙ"O¥›' 5Àc/¦uC,Öý3°rÚnº›â0Bùi'žv†±TƒEZÂ=Œv§8ì|ZÅiÌ8É FÃÞÑîó]x©‰¯Åú¹w2ÅÎŽl7;[„Ó0€ÂO}ªâ¾Z:É&ô¯s‘i÷rúñTÓœóÁÉACô$ïˮ̋ÎÿF™¯ñï'>ÿ©¿ ±åqgZ¸­;ñšÉä×òbùµ….Ëf:^`Š)ϲ¡GÃÈãÎ-xͳºÀÞKÒ@PÚU¦±¡ëÈ/ž 7„—+/"ó±s3övéûàä‘!yµ§[…‹u8¸ÌSçgJ>/MâJŠÎ”-»¥".ÖFq¹tX¦e;3% ×N†ëÅ~^§B”Ö‹…FÈ¥Ÿ:živmºËê(–" À~½XfZ;-;{£?xy}ô¼7“c_"Ð>¤O«t L¤ðt©¯—q9rŽ¯gE}ñN¶ü²öü 9siLjÁ»‹5FižÕZ°vÊ]¶ë´'ŠÙ.-¦nSlÕº[ꈳê¥48Å”þ¸z:¬³2œÖY‰‡²ÒnÏxƒ-ùA€ÂÎiŽÎÎ’ƒKfE×ÚCʳ¤ï²cß¼ï»gÛó2 ž†ÄÉLí¬%?ÿ>dílu®N"´Rýo=¡<_iD¢fJÞùú+U6Ç aù¡Äy9.yuÛç'ÇS’~~²ö„é0ì½M$¿7¦›ðEyY±4Õ‹L¯V¬:¤×9àÙwC -9ª?8SÔê!Ѹ”;„N~_Ê éÍò³ÉÒóh<ìŽOêßÿÖç›O -endstream -endobj -333 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 332 0 R -/Resources 4 0 R -/Annots [ 334 0 R 335 0 R 336 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -334 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 621.123057 101.433071 602.373057 ] -/BS << -/W 0 ->> -/Dest (cb25-1) ->> -endobj -335 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 444.376036 101.433071 425.626036 ] -/BS << -/W 0 ->> -/Dest (cb26-1) ->> -endobj -336 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 237.629014 101.433071 218.879014 ] -/BS << -/W 0 ->> -/Dest (cb27-1) ->> -endobj -337 0 obj -<< -/Filter /FlateDecode -/Length 3870 ->> -stream -xœÕ\KÇ ¾Ï¯˜³•«Š¬ ø "ÄA ñÁÈAkm«rùûùXÝõèiÎcW«@Öb5Ód=X,ò#YݽîhñóÊá¿ÌÎä\RäãéãáÓÁš,þÕ›‹Êþ¿)åd‚+1‡ãG!P2©Ÿâññ€ÿÎ.ç懿9ü{Ûǽ)%zŽ¿}“I9%VË:RX!˜”|ˆ -+:ƒ%–¢± FHìyÏJƒK b°÷YaeˆAœíYbøäXcA WBRXÎBç­UyÄ2“Wx.˜¥G特ø¢òx.yGdbbœÆ+&ƲÆcÈŠu/@–à˜­Æƒ,L9i¼Y(’UyÅçD/Aí…—! ¬+«<Èb£u¯ %3«¼žÍ)ïyÞ2‘ÕxΙÛ$•Â&ŤðÃ)‘¾ñ$ È:ð¼Ó€€,d 5$ Yž«ò Ö°€X¯aw=û¨a Ŭš w!AO VÃÄ{ð ŠFÒd ãàY«aAqõ¦g à—Rof -YÔ›(ñ¿-¥ÞL†,)ŠÊƒ,HšŠâ·À$Ô›@'!‹/¬aA‰QêͬaÒ Ô›ÈèÕ#,õfR €»¡™4p™iÔ°¨7‘Ùjµ½ð†„üLgf©8³zb`½±U=2°À^D«¤žX€¯ÔHê¡ÄDe™£zj€° -¦³ê±þ"ã&,§OóU„9$ÄÌœd‘nÁÈî{Í1šÑ…fËùØo¿¾}‡à{üå?‡‡oêIYûq0~â(…µÔ rì‚ý^™:£ÂD²í%‰„ð9}DˆT†°¸º‰Å·åØ¿¸­ÈXdvHfêÊx3/J‡½¢¨Êúä>/.+IÞ²·í¼ª·ß”Žäßqÿå†@¸û…¼Õøe…Ö†„ ÞºÌ{¯…ÞÙçÅw}ž÷^ ½³ÏËËŠhŒì†,ß±ù·) Í(íïòVãr±x{@¹æm¿?ñj\ΖI‰¯óùøöãáÛ7úËŸÿþöêíÃá§×ÈF¿îì3|÷ʽ–: á0/߃[?ãúÙèÁZÎK[¾å÷ ú©×;úÚV>+‰)¯íä³õ“ÏN;y¤½ðƒó‡Ó:O£½ÛÊ\ç -Cö.‹ŒKë<§Á¯cÅ©ï´Ž:Ç?oßÜ¥û2ëþ§×ûe»ŸuUØ÷P=Ojž—l'5åµÏé»E¨¸Jg1Äb¹"’ˆUZÕD2ÆÃ2žûy?lyy=ûÝSÕB«<±ªíš1]’ÃŽµÈVôõɶxÙJU6¹-µ”±E6Zû3¼n‚²†n.e|¯<·È«ÞíÃ:λ½™Üšçi¦>™ùÃUýæsl÷f²—‰Þ¿»é{XÖ¦½©ë_ûJSù=_G§­ëè:*ëü2žª§¶î‡µÝ»•G¿nûnøO³Á¶×l©¬¼«:óö\guÝÍ.éòŠí†|žž±žæûµ/ï÷,<ÑžŸggWõåwú*kß¼]‡ÌÇmÏW_²§%ÌÔÐRÆœ]þ÷«¼vµÝ‡3_‹Ëš›>»ÍÞ‰«®É²Ú¸\×9N‹¬õzÓïª.[],˜=éxÂÔ?¾­ÉBAnrÕ4ŒQÈ¡Ž/‚cÙÔ²sÂñÔ>ÏJÝÖ<Ãx9µÃz /§(]mÉHqΓn³.ªé—ŽB¯3,™ˆÔÈÛš}üõÍ?þöý”}ôl"µ¬aº^qɲ] -(ã·“,пŒS¡ GõõÛCy…|úwSûn-»t~hYÃú9ñZ»ÞÏyÝ×û||·šàÈ>¶ý7RdK{™z»ÆkcvÝ {V)û©Ú8±ËÁŽ£š?úÕimÁd-=ÙØs¹A[äVÈÙ¤5O½àD3X˜] ‚N×èÎ>Ÿ“¼_Ê|Í—ì»{þºÁâõ6vò‡fëá̇mm°yn·nöå»ì}¸foÝ–ÒpÂêm³½­_í,Wiö¶Æ¦”“;›ðš­ÉÝâÕnPœê1Ê S»¯Ë×fiÃ5Kûl+S­áŽrJnK£Éžï¯eã®cy±»šÈÆøεDï¬-#!®IÇÃ4oKdÂ4^<ó¬Ó~̹níÉ{óEêØ@³â«®åNFZ~7µw+–ì–Öåé±`*lê‹;NSÂøâÅpˆEN'^1¼çiÅp´á’Ÿ] ïÇþ* ÅÚÜ5ùVÖ=Ƽª[¾þÐƘ ú¸Ò§œ¥§§Û²w=Ì}/¼HÁcǘµ9ó¿óÂÅù«úásýHZm?Œµ´¸Ú××yv˜7ãÝÚ¯\?»³ùŸR8%yº¥,§®ˆã¾p‘»nWñ}¾šÂ)Õ§äRvñ NÛIî)œÚ<¿û‚)ni­MOaü­Ñ‚=KŠ›Ù¡‰ÏÇkóøieÒÉ»ïîOåévIäÿ±ÛNz-=N97“ç¡éW¼é–÷õùÚälý5•H;‹<·¢]‚··˜žàÝSš¹©o:ã}¹²-ËÔ—móå˶í„×ì2ÇÒlŒ}Ylè†YÞ×嫳ÊL׬r¤ :æÜs?Kž%“ÇÚÊýXú`_ß DžþÓª§.g÷ó¥è¾Ä{ù"ªÈƒ‘ï{^µçRkýa}¤( ‚<Ò Íäé ¥îË7›ÜðÝýóLíI£;{Ða ‰EþIò’‚‡‹–$ß]Dÿa Ún­<–ëaF´žYHòÒ]i5[ßÔ춵A¯û?Ø~¬‚j«Ç“º?»!ؾk7úüz°ºJ®ÒÇouß$Ϧ*k’V×°ÛÊO‡X%uõu³q^Þ7D…bHö¦r²%vmë\U¶;^èYßÒüÔ[­/nZy—Cžã]Þêœ.ÖöŸê¦„W²‰ÍëKV¶’³oƒüx—ìX!3g¾AâúÕö±Øulh´T2å¼%àß®_€HLX¾¢Ž£hÝäáG¶­ÕÇÿ–‰"zF·'Ëê­q+Y^T̆֫ØE ±É âç¶cÀÑñ&Ù‘ˆaO´Ótƒì¹ÉãäÁö®º8ˆ]ôYunÚþÖÖ§i;¯OLKçÔG•*5Yªi×ÛN£I¨Ò©¡µìÂ}ŸÒé _Ù64c3;Ú5yÈ|!æfHÜÛ•ÖN!j»u´Ðw*õVyt-MóC”aêrô2í4Ù$o^O>b©}Ý’å ­ïµÍ“Ýøñt@ˆáùñ’Ò: ¬°¡™ÞL¤iÜNÎvØË ¦ÜT×l4yDWþ_ "tª Oý⸛”$´ãjL"¾ÙÉ[%ÍôØQ‡5 8N£í¤.—R·ÙY_.[¥½€noÝ5¶¡N*›è©4ñ¤:xlÉ®[]' „5ýt¬œ‰¶ é9tS~܇œ¡»j°}²è&s}ÜAŽ;grê;îä‰.ow²ùF<ÕÀÆçd1ýÜ&do{´“MÄ!@aŒ>VróÖŽNòxû:œL‹¸§Ã¯‡æ</n‘œ¼8g^îø_´yƒßøø}{æ|‰ãÞv´ð[7ä9’É ãùN²¼@ÞˆÝm&¢ç öÙåÑ64kèqÜû ¯yt·Ø[K×ÔuîL C‡ÙwÓõŽ&Í„+ÄÓ™{"ó¹{3örImq˜†äm…&.æåà2|œ†Ä•4!\öSG\̃ârš° É6bŠAΓtÕ‹üY}¿9>©?ÿz÷ -endstream -endobj -338 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 337 0 R -/Resources 4 0 R -/Annots [ 339 0 R 340 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -339 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 627.273057 101.433071 608.523057 ] -/BS << -/W 0 ->> -/Dest (cb30-1) ->> -endobj -340 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 314.026036 101.433071 295.276036 ] -/BS << -/W 0 ->> -/Dest (cb33-1) ->> -endobj -341 0 obj -<< -/Filter /FlateDecode -/Length 3869 ->> -stream -xœå\K$· ¾÷¯è³Ë”D½€À‡IEÄÀ"99ŒÛ;6‚Ùëò÷ó±J¯êb?fgÖx³ÝEJ"EQIUÕØ#áçs‹ÿ2[“sI‘§÷‡2)þ--6 ûkü¦p´ÄÉ[bÇ÷Bðɤ\ŠÇ§þ;»œ›?þñÙáßÛ>öèL)ÑÙpüñ°Ø›”S¢ ° -Xd½Â -Á¤äBTXÑL±5BbÇ{V‚\rÐXPƒË -+C ÏÉú=«@ —,k,¨aKH -Ëô°ŽHåAböNáÙ`"Œ5ž³&æâŠÊKàÙä4ž÷&&¶Áj¼bbŒ!k<†.¡ÕxºËLº°ÏIãEèâ£'•]\N^ã%èâ`¢ð2twe•](’ÕxÅ›P2³Ê+àQNyÏsLÈޓƳÖø¦WyØQX¤˜žƒ.‘BQyÐ%xrÏCÆÆNŸrVv«“ÝíÉ[ ‹s‰UtÁâkà"t¡5À2‚K¸” Ǹì gÎ -¸\ §ä5p%Ž%i8àɂ笞 Kà¨!·Ð…±sUtñ…5,𺠆ÞCÇ^ôex‹†pËT¬†xËd£†€ 5IÃÄõ9²† W*x`®vjXàºØîIÃÔõ!: <`×sŽxà®gK0p×{ôTyЬaÐÑx›½† Üõ–’† ÜÅ6r`U ü0jX_1.gÒ°€».kXÀÀ]—T^2؃^Ã)h‘¸ë9 ¸ëØE à(øCj6Ü„=€.X ïÁs^à fÒ pׄ?eO­ŒÍ%jX€»6cùT^16Áâʾ…•±hX€»T^ÂÜ|f €ñÕiX€»Ö#Rk<à®u…4,À]ä Aøƒ±H4,@|3–àº*¯*9kX€»€%§a¢°¡ $QyÉÀ‘¬†cCÈC5,ˆÀ]„÷¢aAîRð^ÂÜ%FÚ£ò ‹ÏNÃä5àaÛ*{ɪ!¤ÐxÀ]²XaÜ%JEÂÜ•°©aALBpYÃ(h; bfSRŠÄBà!åSyÑà:hX„ÈRyÐ…kX€ÄÚ_²†ÉAÉA¤  #0Å"ûVËèwQë†.È5,’™\˜5,H!¢ƒU5^t&'ÀµÊËàÙ¤aAJlŠ­†¨¥LƦװo$]Ò°j€gYÂT  ¶Š†E&;dæÏB$Ù ÓÏZ ²ƒ.È5,È.£ÊŒ¤a"4jI˜Zã1I™5,ȤŠ`› ÒRª<è˜Ð°ë#ÅvаÅŠM”9*ºøÀäì¤ÜÌä ]9 à›Ro& -Ad|ŠFÒd ãàiXPlF½éXÃìK©7³†ÅêM”xʾ->J½™4,( ]R•]4eß“Po4^„.®°†%F©7³†H3Po"£W²Ô›I=.î†dÒxÀ]d¦QÃ\ ÞDf«Õöà ù™ÎÌRqfõÄ€½±U=2 `/¢URÏ à+5’zh5QY樞 ¬‚iI=6 à/ò1ÞaÂzú4QE¸CBÌÌI&Y°-Ù}¯9F3¡Ùz>öã÷‡/|ßÿçðøÙrRÖ~,œßs\ -kàt,®`½Wf‘ÉAÙ€:eÑ -~LQ"•¡¤,ÖC7Q¡á¶û»U‹ÎÉÌ23ÞÈE©`±V>ªº>»Ï«ëꤶ@ÍE<ËuEUòVã›Úyùwܹ¡$ -Ôû•¼Õøu•Þä"­rïõÐ;û¼úªÏrïõÐ;û¼º®Y€^²ÌÛ‹«ñ§òДÊýJÞjüŠJî!ö÷oÈ%Öûñ8ù"ªCf€ñöýá‹7üóŸþö5Vçíãá›ßÉ™!g"9?”Oâ~´ú=8üãíw_þóøöÍ,GŽ]#±‹e+G†F)· òÚ]®—!¿]EÉ5WñüXÅò½\•üv+ßq•ßÜ ÏÿÃÛeÇûÝQI:&ä=låÐѺ€+Q %Ö„ÏíóQx[Á3B©ù_œ”&È7© ÔÜ!7©«iú¥õ¡ë‹ÊˆgŽùË›¿ÿõ« @Büòs+Ÿiýdž®WÃ_"¡Ñ¦ ¹÷FÈŠë8â}\e4ay|^]P¾»‡©½,{™ôƒÎUÇSýœx­]ïg‡ÜuîÇë}Þ ¾ •×úžõ[tSÆj²ú˜µ §Öæ‚ –b¸,áÛ2V¶ ®¸é‚÷õù¹]°ëë¬ÜðM[WsÁ¶dÍT¶ËÜQaA–íÒw37ó»iùËä*Ýü=ö^ ¹°†Ü095溊¡"2OˆÞ‚X˱¢Ñã²¥vñWnc90>*r.¡ä·å÷(b]LË1ËU—¹³ÏKRœO^n9º¹è9ô0mLwöÙ6Þw·ÛÐ@ÝëÎ@«ozž¼°yîã¶m·ÉéÀ÷jú>Û=ÌyÅ›»§¦‘ë´xݽ¹ÔXýX½9+žÌ¦”“=xÍ“å®jõÊŒ<=<ßpäûºüÒü}Í_ìª7 9¡I¯~9ù†Ü¾µr²]Tß°¼"ïæ37Ÿ$lh»Ý3¡È)ÿ]òá¶oF¾¹¡·¶-øÆ-oÉÛÞ{ùåÈ·úH¸T64ô^ü½åŸq;µŒÐ縌ËgsŒC׎ö÷,ÏË°xDšÑ3”³æ´þ®•Æ°p«.fZרǡæ-•všâÖ©¥5£8sÊͺô*fŽw§k:'¹k=£ÕúT‰Bn#ªìÇé«ØªªxM`Ã%=^\;ïÇþmÏrÇ;ÿ_}õLcLuÿÅÑv ÓŒJWí•ì™½žSµ†’ŒEýYÊÑËã"˜pr7Âæ}~î’¡Ìhåy+,­ý„UëVÈ=Uk“ó[­V#üÝrD*Oæ˜Pùßp½;ûüÜ®7ô•[­ì]ÈÕzn²_®^ÁöE_–Ÿ´š)ÉíöT~Êjv+ôZ È7÷o9f¼…fwöù¥UɦkžµT‡çÕfÛÈçg"»Luï1=S½£º]cèY•Ùx~«K×ít,]©Fô›òÉeß|ýÚt+ðª_fn>æ1²øÐ-·¼«Ë/Î+K¾æ•#óÑ1çŽò&3ŠX¨gT™åÿ ÂlmÎú4Oîdª6}ïD›Js[YŽy-2î®3婬±ßTûíOæ»C{Þë׈…œÜyvåÇÕˆûqžW#Ÿ/éñâq?öo«FþžÍÿW_#>ûëUû¤}^Tc.Ï°m¼Qd~]òJ£¹˜dóôÌXz7›Ürû'ÌÚ³G{öèÉ6wÀòv§ÏHük&y òRæàÞ­Ù­­kë`K‡iûX׫L1§Å›¶ß-mÎúIß»z@uÿ¶½õ¸î¡»Ž5†Å!kžTå=Ô%9 ßÜŒßö°ÝúB÷ëR‹Àe»¥üpˆËývyÁƒ"\ÌÉ G/kSŽò@q&ÏýI!»Û/ô\Þ›ýÐ[ÕWiIÞ®‘'«×÷l§‹Úþòèï¦ØĘ–×Þh!g×.ùé ¯=²BFBáî$ ±~¥>Û,í„L.™rÞyÕ¯#¢Jò´~u¡ÍFú‡ÖMn‰3µF~y s¬Ñ3Ú=YfOÆV²¼:š¯W±«bÓÄ1 ÏmÇ€£ãM²õ4ÞH“¸AvÜô±òªA7]Ä®úl:;-këÒ´œ×ˆ§Äúsê“J•ÖÚPͺŽ:ÍOJ•N ­e_îë¤N…ø9µ¡‹ÙùÓªÉcÿ+17GâÞ®´v -étPÛÕÑB_©Ô[åѵ4ËU†«ûÜ ¯ÐNSÖ™ä]øio_·dIŠúZË­èºcëâé€Ã;òÓ!$¥uXA¡¹ÞLôÓ¸œiøË J0^¿ÉýÆJ“‡¦åÿ*B§ -ò,_,w—’Êo\ !²7;yk¤™;ê°fËi´ÌeSê>;ÛËfRÚ èöÖÝbêd²‰žJSO@ªƒÇ–l»×u²@X³OÇÊ™HM ˆäÐ]ùiCz†¾UuaÑNîú¸ƒ;v>ÍäÔWÜÊ[,]ß>îäóxZŸ“Åõs2ÈŽz´Â&âP  0ÈFŸrÛ­äÁ öul2-âž?–0ç|prª½•—2çÌËÿ‹6oðû¯¿jo¬qÜQG ×±uCž#™¼ÂŸï$Ë+ýØ·ÍDt<Áþ Û<Ú†æ =Ž;7á5î4€½µ´AmçÎÄ0l8‘]wÝåi§n™p…x:³bOd^º6#`¯—¾Mb¼¼?Ò´ÂÅ<\æÁó#$§!q%MGH—ÝÔ󠸜–¡ÙFMqÈYH7½èÏóRx‘5]Li„\úÁãfç¡m™ÅÔ)°Ÿ/æÙÎBËFEéÞ^/^÷ªrl;@¨Ú;õi¦ö‰ µžNõËeÜ^ö˜ã—[mó õ±ü´÷|OrÆÖ©o.f¨yt« Â,”m#´Š Q£õSÓ)ÖnÍ,ËŒ³j¥ÔsŠAýa¶t˜W¥1Ì«wmeÜñz¶ä{fwNcv4Zö\2+´8÷îT-}£íeóVv‹¶Çi<‰#3¥Ñ-ìZ~þ}ÌÞQ_ÄÐZµŸåtÄæñÞ5Ûä_þå!,o³'ÎÓqɳû\??ÙŸ’´ó“Y–ð÷”¬üQ+]…'0ÊmÆ4TûƒWz·Bꔞg€‹/ ê¯Í,Õd¹S²õË$¥¢ü#/Io–¿½Hf9:mŽO–Ÿÿ`M“ -endstream -endobj -342 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 341 0 R -/Resources 4 0 R -/Annots [ 343 0 R 344 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -343 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 713.673057 101.433071 694.923057 ] -/BS << -/W 0 ->> -/Dest (cb36-1) ->> -endobj -344 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 396.479014 101.433071 377.729014 ] -/BS << -/W 0 ->> -/Dest (cb39-1) ->> -endobj -345 0 obj -<< -/Filter /FlateDecode -/Length 3885 ->> -stream -xœí\KÇ ¾Ï¯˜³—‹U¬ø "ÄAÉÁÈa=ÒÚVäò÷ó±ºëÑ3Ü™ÞÕ*’ãh±Úi²,ù‘¬î:Zü|Mø/3™œKŠ|<½?|8X“‚Å¿ÚbsQÙ¯ñ›Â‘,'¨ÄŽï…à“I%¸üwv978üí«Ã?·}èèL)ÑQ8þüNXìMÊ)Ù ° -X–¼Â -Á¤äBTX‘ –XŠÆ‚!±ãKV‚\rÐXƒË -+C ωü%«@ —ˆ5Ä ’Â" 9ÈY«ò ˆeöNáQ0JÏ‘‰¹¸¢òx”œÆóÞÄÄHãc Yã1d Å’Æ %³Õx…}N/B½UyÅåä5^‚,Ú) -/CXWVyÅFK¯xJfVy<›S¾ä9LÈÞ[GdlÓ«GÖ°ÀraJEÃÌõÀN <@îž4,ð@]¢Ó°Àv=ç¨aîz&«aw½GO•Y`À§ì5,`à®'›4,`à.ÜÈiX€]5°Ã¨alŸœ­† ÜuÙ²† Üu !@å%ô0p×,A‹ÜÀ]¬Ó°€»Ž]Ô°†b€?VÍ€»€ §«aâ=xÎkX€ÁŒCZ¢aAîRAøS|he(—¨aAîRÆö©¼b(AãŠßBˆb,à.•×° wŸYÃ`¼ADuà.yDjÜ%W¬†¸‹¼!hXs0„@ÃÄ7C¦«òŠ±%g p°ä4,@66IT^20$Ò°`l,òP "pá½hX»6x¯aAîZFÚ£ò ‹ÏNÃä5àÁmŸF²j¬Rh<à®%ì°ÆîZ›Š†¸+aS˜„ಆÐ vÄ̦¤5,ˆÅ‚‡”OåEƒë aA² @–ʃ,œXÃ$Ö¦ø’5,H²xç4,H² eа)„ì[- ÌE­² SÔ°HfraÖ° …ˆz ZÕxÑ™œ×*/ƒGIÂ”Ø “†¨¥L†ÓkX€7’.iX1À#Ö° ÈWÑ°¡Èd‡Ì\ãdA’­a2]ðˆ4,Ȳ Ô° »Œ*3Z ¡QKBÕ­Ô™YÂÌÑHª¨aÜ•&RåAÀ„†Ø)¶ƒ†(~Pl¢ÌQyÅÖ° g'åfÖ° gÈâ¬Ó°¶)õfÒ° XÈ‚ŒOÂb£‘4YÃÀ8xÖjXP(£Þt¬aüRêͬaAñõ&J<Åo‹Ro& -C–€EåA$MEñ[`êM “Æ‹ÅÖ° Ä(õfÖ°iêMdôê‘@–z3©ÇÀÝP€L¸‹Ì4jX€ Ô›ÈlµÚÞxCB~¦3³TœY=1°€Þ€ØªX`/¢URÏ ,ÀWj$õÐb¢²ÌQ=5@X“¬zl`¿ÈÇø–Ó§ùˆ*ÂbfN²È·`d÷½æÍü#Í–ó±Ÿ<|s‡à{üñ_‡û¯êIYû!¿ç(…uqÛ ¸2u8F"YÊ”* ä{F!•!"×#7Á¶ûÚ - yEbB*S×Åó´¨å£*éS»¼¸¤r÷ò–çy]Qe¼Õø¦t^þ/?Üö y«ñË -9LM*¦¸Î»×/¾ëó¼{-tgŸ—’Õ÷ýtrü‚â÷È$§Iä"g6àJî<‰úÄ./!©5Nj©ˆV¾8É=‘PØ‚ZbÓø·oHCÖé—;\ý’|hr#cfèÖÙã›÷‡oþôê¯þî5¦}søþ7Ö†øí×$Óò—yºþûñÍ«+sˆqQ -È­6sÈ°Ö/à u°¼NÑh6ã#~éíòÙÝMí¦.“aÐù~ñ´þx­]ïGc^þ¿ùr¼ÞçÝàSXy­ïY¿*›2V›«¹¶áÔÚ|[•ù»7[£Êr|_ZîÒ tAÚ|Ã÷uy–¾ ùuq‘+£ºJ(h¯ÚŸ¶YMy¡]ßë›B´lbow¾ msÜde2¤»¾9=·•ÕÅuYñ8¡+±Pµ³uu=¯~ÿÇ?üeZƒA¬!x™_æ_Z?'1t¹‘³\á ý¾úÛâmcR'÷0ŠœžM*óÐ[Ý (æfŒ‹©ÖØW-jgŸ‰qŸØHêýÇ-ËÞMnëÎþ6·|{»à©YL8ƒ´n‘aù¼dOýÜæùµÖÍp=Ш÷°åÖPDên™Þ¾>ŸÛô†¼Ìò,¨³z^¨Uµ½¶8Ë» | t'†Ù§ò߬ ·“^«;"Q3yÒ“‘×Æ[¨¶³Ï—VyÈS>W,¬V¤çn³,«WÄ#‡U¬³ÙŽŠz‰Ég•mãù­,]¶sði±ß6cŒ×lóåëáí„Wí²¸fc2 7vöuùÒ¬2És[åy&õŒÊ6glÄF=¡²ýaÔRÿÛÕíY}xwÖ§Y}O¦šÕ﯌Sš·àû6ÂTåî©jgÍ ´hg":ïå+Ty‹6>ü¼ -õrœ§U¨ò Ö#r|t…z9ö¯«BÍñ|ý¿ø -uÝ.ÏZYÖß5ÚUýäsý|Úʵ Hmç{Jå*½Ù,ð³ÿþïÎ>Ÿ»|èa´D(K -/÷ +×í$ÿ¿ó»¿‚-òæð*?¡‚ÝÙçs›à7×·öäͼ_Pûz}˜1,,y˜<ÊÓ³‘}on6¹±—OR¶GUŽtöhȶdÁò³Ï¨7Qâ&yð÷±‚Ž[ŠjZƒ„0;¤+ð—)Õ]SBײ›#Í~ëáÝÒN¼¼&-Hß/jºëIË” Å‘Œ´ÀPç»[Áû4‚Üfü– LòldY=¸®áb+?b}ˆê‹L6fCNÞt>F6^ö¦åÁùl=÷gŽ¨*šÞ³¾þ¡·Z_·ò™¼A°¼O>]¬í?ÔM™MlYq}½ÓVrvíbòz/+dynr'YˆëGÛÇbê™z£8É”ó–ˆ+vý8Òy 6ËG9HýCë&]³m|}ðx™X!¢g¤K²¬yñJ–W¤³ñëU좄Ødq ÈsÛ1àèx“LÞNÄpI´Ótƒì¸ÉCòJMW]Ä.ú¬:š¶¿µuiÚÎkÄÓÓúsêƒJ•3ÛPM»ÎvšŸ„*Z˾!Ü÷I! -ñkÛ†flfçO»&¯·,ÄÜ ‰{»ÒÚ)¤ÓAm·ŽúN¥Þ*®¥i~ˆ2LÝç®x…všJÞ$ßù0ùˆÜÙkû0“¥ºê{-Oݬ[0\wO„¾ ?BRZ‡64Ó›‰~·“³ö2¨)7UÁ5M^ÿ¨*ÈS?ÈãJÔ˜¯Æ$⛼UÒLuXÓqm'uQJÝfg}Q¶J{ÝÞºklCT6ÑSiâ HuðØ’©[]' „5ýt¬œ‰¶ é9tS~؇œ¡»j°}²H“¹‡>î ÇŽ39õ'y[«ËÛÇl¾O5°ñ9YL?·IÙÙíÆdqP䣕ܼµ£“|Eû8œL‹¸§ÃO‡æœNRâèI^>ž3/:þm^á÷>~×ÞvY⸳-\ÇÖ yŽdõé§dùêŠFìn3O°?È”GÛЬ¡Çqç&¼æÑÝ`o-©#(uîL C‡ÙuÓuä'Í„+ÄÓ™{"ó±{3öréÛâ0—÷¤šT¸˜—ƒËbj2ŵ[SK]qVµ”zN1¨?Íšó®t%†yWâE[·E¼ž-ùž…ÙœÓX-{.™Zœ{w*–¾Ñ.çæíÜ-Ú§eð$ŽÌÔŽná¢å³ðï9¾³ô9;‰ÐZµŸz:By¼Š@Í”¼óõ+¿lŽÂò­ ‰ót\òä>×ÏO.OIÚùÉ<¶Ã°÷6‘|y›.Âå6cª}±›Þ­XuIOSÀ£/Ú^2ät¨~{O½A[‰úe’RQ¾¬ËKÒ›å;F’%ýΧ¡øÍñIýùjæÓ -endstream -endobj -346 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 345 0 R -/Resources 4 0 R -/Annots [ 347 0 R 348 0 R 349 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -347 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb42-1) ->> -endobj -348 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 465.479014 101.433071 446.729014 ] -/BS << -/W 0 ->> -/Dest (cb45-1) ->> -endobj -349 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 133.284971 101.433071 114.534971 ] -/BS << -/W 0 ->> -/Dest (cb48-1) ->> -endobj -350 0 obj -<< -/Filter /FlateDecode -/Length 4178 ->> -stream -xœí\KÇ ¾Ï¯˜s—«Š¬ø "ÄAÉÁÈAK6‚U9‡üý|¬îzt7ç!imËH¼^í4Y‹üHVw;[ü|áðOfgr.)òùòîôþdM -ÿÕ›‹Ê~‰ßÎÎr2Á•˜Ãù(™T‚OñütÂ?»Ë¹ùÓéï¿9ýkÛǽ)%zÎ?¾“I9%VË:RX!˜”|ˆ -+:ƒ%–¢± FHìùÈJƒK b°÷YaeˆAœYbøäXcA WBRXÎBç­UyÄ2“Wx.˜¥G特ø¢òx.yGdbbœÆ+&ƲÆcÈŠu/@–à˜­Æƒ,L9i¼Y(’UyÅçD/Aí…—! ¬+«<Èb£u¯ %3«¼žÍ)yÞ2‘ÕxΙÛ$•Â&ŤðÃ)‘¾ñ$ È:ð¼Ó€€,d 5$ Yž«ò Ö°€X¯aw=û¨a Ŭš w!AO VÃÄ{ð ŠFÒd ãàY«aAqõ¦g à—Rof -YÔ›(ñ¿-¥ÞL†,)ŠÊƒ,HšŠâ·À$Ô›@'!‹/¬aA‰QêͬaÒ Ô›ÈèÕ#,õfR €»¡™4p™iÔ°¨7‘Ùjµ½ð†„üLgf©8³zb`½±U=2°À^D«¤žX€¯ÔHê¡ÄDe™£zj€° -¦³ê±þ"ã&,§OóU„9$ÄÌœd‘nÁÈî{Í1šÑ•fËùØߟ¾|à{þþߧ·¿©'eíÇ!A¨­ö 7ðÛ};,Ÿ" -Ì*û¶2i*C(\ÙD(„;ùï|ü°‘‚¾ÜN ›z\Æ{ŸWH€q” -ö’7.óëèŒyQ´x äCÖîsWä„´¥œûwO¡ó¼³ŽŽ²~pŸg—5 £®GüÀæßküSYh‚þa!ï4þ©,é"ú2ï°6‡x#°¤¨&úh§gß÷yâ¡%UÚïôÒAöw¯NÈ&–;ñ<)!Õ#7@Ðxõîôå‹?üé}‰1_½=}ó[„¸lmø¿H¨ŸÉ!t½|NRÉÉâr„'ô·Òæ«œ_½˜çôr¨V¤|ÝÎ)Ó¸ï–æ¿UuFÔ“¬RüšP}L5åË\IR'5`—O±ãUÅXJ¬ÏPñ’â!ncu–zç6ù¢Š~é(ôÈ€ qHß|¨ÛðçûË×Ó6Ø×Piøê ‡ÏÁïþ¦å¯ûî~ÇÒò™/+/®¼FÏ+e;W¯ßnÛ¶qÛ<þ|òv»è&}Õ’] [¦ÜÍ4-fp LÞšr‘µ®&œ«.ŽfŒZµ^» oÙ±“CÅ*Qfx†ëó;~¬ËggÇ冲M¨Ö€8o'¹úådé/2nç³j¨”±‹»¿—Å&|ó·õ—æaµ+i[¡ñÛæ3Ëõ¾¶õo&ÿj¶Æ<ÍVhmþ…ÿ×¾~•Ol—/CŽÑ]o2fkËÓø«l\–þbóÝÞQi˜UúͼS¯·ë+[WÑ5qYK•â²]eo›·ZH³Žy°’fE:OúY4?¡ÛF[vì\G†Ëf=)ägÐâ@‡yœHr’!wËŽãÔ5¶]‹U'7ä ò×ä°c-u§ÛúDç~AAE6¹k-#ƒVÆn¿"©›­|Xx³êº~™çõJëµôí†EÊþÔ¾ëºg„nòªóm,y¥­s˜ںǘ7uöëmŒÕ¶›]éS’T ¹Ü—½ëaîÛÆuëVº[Çp/c®íxMÖê8vµ¦‹·C÷2oÛ&OýÓæ +ßïýiÐjSgi¯³ŽhQúâï›Õ?6è·G¾¥m]Ë›eïnÏËHû¶ó.¾>!eº©™äتférG3äî„êû|TbÿSiæl2joõ ·A¹%#_ŸDÊ0—ùÝ$ßLÉ¡˜ï’N‹í¡¼˜š|öS¢Z·¿Lr„Ao iKfyâµv½Ÿó.®w¯÷y3ø.¬¼’¶ýªlÚXMo§Äáñt–s1%ÀVãÏX™m'½•Òy^j1ú‡k³û|nI­ÜŽ¾á/ÿ/ΰf9 ¹aÍÏ_œm'¼iÉ’x-VùhqöX—ÏÍŽ£Ü2¼û¿TqC4>…ìù±â¬al³ç_C‘§¾- oéꜚª©ìãEYL³*ë®Ò4³ÝZ=¢]¦Üi°—³(ΜܿLËï’üŽh ïLëc‡­6ë¸ahª'ðeô‘‚ì´£n»û 5nròDmãã -¹ã8u]¯‡oòÄÃ9>¹;Žý¿UÈ¥´_ÿ¯¾³cÌ}!·÷Á‘I/s-ôªëp¯°Jå »¶·nÈÓÖëÖõ,™ôœEOxTvíÓºÏkÑg¿[tTí¬"ç-ùäé•àRvñVáWí=Ž1Û!Ô¿öVíLX·2>2ª‘QÔÐ)g¶çìÈÄåV< -^VQ;`÷”âÃûü -MÁYÞ…»‡Íæ7ŠÓìvGß.ïq„÷‡ÏÍwŠî³ûù÷þÔ}ùí±ï5?øæ÷ã÷ë׺¬›>û5ít³_û~mË]\§Ê¥rF¢é$u8Ç"O“fyô¶?Ôåó1ã‚xMN^×Uqáµ²õi»8žÏ·^®Ï«B ö‚¼/ƒËÓã¯]‰w›ÜQÚñaÙ–ÒœÝ.ؽ¹•ä 8•$Ïv_;x­Uk‰VËѸ\Q\ËÖ2¾•µM;’˜J¿^ú®e_+G{ɵ!ˆï Ý÷ûÚ£¨¿vŒÝJ6¹!TK¹Ë(Ÿ7ãß(EûB[ÃWû­|Šõ=WßU³1çåeösd#©QBU–åb׶ÎUeÃ%ôžõ+Þ÷Vë·XyQP^Y¾2`ºXÛ¿¯›þîä -›¸T]nyƒ×V²dÉ{òÓIÞàf…Ì< ï…¸~´},T¢~G»’)û–(MìúqLÒ¨C[ô­›ÍäÔwÜÉ y]Þ>îdóx©÷d1ýÜ&do{´“MÄ!@aŒ>UróÖŽNò%#íãp2-â^N?œj˜ó¼$¯‘œ¼_>g^îü´yßžøüu{¡i‰ãÞv´ð[7ä9’É·‘äÉòí$ØÝf"zž`]mC³†ǽŸðšGw;€½µtA]çÎÄ0t8‘}7]9Lš 7ˆ—{"ó©{3örImq˜†äU¸&.æåà2|ž†Ä•4!\öSG\̃ârš° É6bŠAΓtÕ‹üyßW‘có¡vhïÔ§™Ú&Ôx:•êeÜ^ö˜Cõ™«æ¼CC},š|z’3\c¤¼¹˜mt¢æÑmmæI¹Ñ6“¶@Ñá4Ûh}ÄÔdŠk·¦–ºâ¬j)õœbP˜5æ]éJ ó®ÄC[·E¼ž-QO€ÂlÎi¬ÎŽ–=—Ì --ν;•GKj´ãܼ»EÛó´ ‚Ä‘™ÚÑ-Z~þ}Œï¨o”i­ÚO=qy¼Œ@Í.‘§ú­n6Ç aùbŽÄy:.ùà>·ÏOŽ§$íüdž Ûa˜È&'ßϧ‹ðF¹Ï˜†jßݧw+V]Ò‡)àê»Ôúûõ šêƒfõ¨_&)åûØH’Þ,_#“¬þU=BáÍñIýù/ÂŒL{ -endstream -endobj -351 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 350 0 R -/Resources 4 0 R -/Annots [ 352 0 R 353 0 R 354 0 R 355 0 R 356 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -352 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 518.373057 101.433071 499.623057 ] -/BS << -/W 0 ->> -/Dest (cb51-1) ->> -endobj -353 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 201.376036 101.433071 182.626036 ] -/BS << -/W 0 ->> -/Dest (cb54-1) ->> -endobj -354 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 182.429014 99.933071 163.679014 ] -/BS << -/W 0 ->> -/Dest (cb54-2) ->> -endobj -355 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 163.481993 99.933071 144.731993 ] -/BS << -/W 0 ->> -/Dest (cb54-3) ->> -endobj -356 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 125.587950 99.933071 106.837950 ] -/BS << -/W 0 ->> -/Dest (cb54-4) ->> -endobj -357 0 obj -<< -/Filter /FlateDecode -/Length 4592 ->> -stream -xœå]M$9n½ç¯Èó£•DRÀ` ìx а ºkºw±¨10ëÃþý}Œ )̌Țj#w¡*3H}0$ò‘T(Ôáìñó]ÀŸÂÁ•RsâóË/§_OÞeñø7•X]LìŸð›åÁq9GI+‡[¬ÕbA ÉyËʃk‹18Æb° -Ä Î¶¬ -1bl± F¨’ Vð#DïMñÌ ^—0èÉâÅàR©±š¼ ^ÈÑâ¹”9H°xÕ¥”¤X<†,R}°xY$0{‹Y˜J¶x ²P"oò K,™,^†,£S ^,Юbò ‹O>X¼JNja6y<_rÙò¢'…È[¼œ@7ÉäÁ¢0I)¼Y’—jò ‹ ãÇâ'„r)†µFµnò,ž@–3›<È‚É· &Èâs² ä—, ˆ9;®‘-ˆ…. -ÄRçL Ä*ŽSÍàÅ`yÈ"œ,$ Y–kò U¶°€"dA ˆ Kd²°€€¶ m±°€·ìk°°€€·ìC²°€¸Ó[X@@\*‰-, @.T©ZX@À\vZX@]˜{¶°€€º$)ZX@€]â’,, à.qð0p—5Md[Xtt -YXÀÀ] ->[XÀÀ]˜Q´°³ê ‡ÉÂ芋¥x ¸‹g ¸3\€ÉË6H0p7 -nÁòÜÀÝ(>ZXÀÀÝÈ1YXEqÀoFÀ]@ˆØádÀZX^$ И‹K,,àn¨p†M­\(5YX ÀÝP0}&¯º1â†Ýb”]H©ZX À]8*²°@€»€Ïba0ÞÁ£F ¸žÚâwC¬ÞÂî"n  . °°þÍÕ5yÕùZŠ…Ü,E à…/@“—)XX0vq¨… ¸ ÷^-,HÀ]/D$à®g„=&²P‰ ®fkØ4‚Uç#Ââw}À [<à®÷¹ZX€»ê6-,HY ±XX¢àhaA*ìjÎÉ‚T=xùL^r¸ ²AY&²pf X»JµXX#d¡-,Ȳ d°°«Ñ·™@¨‹™0dA¤haÌ•ÊlaA–„| £jñRt%®M^/d rfW,,@.å -ŒÞ¼ÓpɈ^` r…,0 àŠ\‰ˆÌ-^€,²-,@¤ ^”Y ZXPbA–™¼…ðÐÈ%1Ô½æ™Å‚ÂÉi¨haÌ™&ÒäAÀ„…˜M¶ÅÂ$?H6‘æ˜<ÈB”5Ý,”Y¢@75ßÌTYñYXP}r&[XÏ{ j(È7#[X»Ô|³XXPÉ#ßDŠgØm¥¤ùf¶° 2d„(&² hª†Ý“o,^‚,±²…5%Í7‹…3o"¢7—Šæ›Ù\.îJ2Y<à."Óda.o"²µr{à•ŒøÌfÍ8‹¹bà½ßj.x`/¼U6× <ÀWs$sÑb"³,É\5€[3xsÙÀñ.«OãU‚:døÌ’õ&+Ì‚Ý÷œc)F7Š]ÖÇþþ—Óï?Áùžÿò¿§¯¿›VÊÚO€ò'M¬EM$VL7àÊMÍ1&:¸‰È“„ð-uTˆ\!`q^tS!áŠánë¹ k‘!±ÊÌLwÆc¿È¦Š’-ëÃuÞKVjýÂü> x;Ã$àí2Ìí \W±õÁ*ï!):ÒX ¡Õ¨Ø ƒ@ÔëyUøN0£¹ûË -m¿ä†7g nôç¿œ~ÿo?þç¿ÿù'tûáëéã÷(ª¾Ž~ø.໤Ë'—ùóëÕõÌ—<_¿Ì×qþ C[3ëBÓï­ÎuÛSßÿ}þðãéÖãŽL¤áæo•H]ì݉:RáMÓô-f9'Â0„+7fçsL`(|­Qˆ0Þ•cUžf\A#‰€J…]µíjõ“¦~e­ªMµÃÏ7þ_±§jz­wÂU}Óý=Vçy†yDdŠþ–®-Cºù„ð—ëè—ïú¤A-[®QãëÂoÓÂ|… Ÿg~^£ÆÔæ5ÊÈ}4ê(4—“—-5Þ(û›dÛC2ýü4È>´y«¿^¦\ÝG¸¡²HÝcITý™k†S{>ì`çQÙ¬ËúR}|\eÿŸv5R5þyàë ƒæAã/C95ü†E;ÏßrΡh8^‘øõ·j<FÍ_Ñ-òÞGü³>Š+Aýú9…>ß%Í2îÌÁ:Ï34ÈÄ<}$ü&w"Ãõç¢dß 2t6èõÔ®i£Ç2èóØvZÞ lUÖ[¶tÅ‚.¿þå†jè -ofFŠxÐbÕxµÐ§Dº¬Z³˜„ü• -‡3bºÉ½ˆöX•ç˜ à“” -í'bMﮣØ7F·±d]ý™VöbÀPdîÞè«ó4ëOÖK¨”oéÝm8êÃ(ëámfÞ#¾+ØYåËæÀ]IQ—ËýÎxß/ú†yÝ âôl†%:/œ´¹û’«ó|+$º&ܬ»*Ay°*Ù¢¾é dí 6|’˜¦Rù*Jû4¨S[2‘¥ýqéäV#ÅPÇï·²K1þÓ ~úx1î{Hbz¢Óî©õÇk¹&¾m 9GÑÒi󗇵ʮ]«ó<8”u×WæĦÒõIºòLw»#}~|©áª£Ã„ëüµ~¬t +ê-å³'¯ úŸ¸ìcÙÝ¢ï…eÈU\3DzƒužË8„‘¬‹¼–M.m\žýlDÄGW®Êôö. ØÕ±ä -{×rÓúúZ¾ëE—n‘úxÿ×x9àx¸`¤Ué>*Ø£§r†*…„”&í©ß¡*Oƒe,ÀÞ*"ö³ƒÍ$À/Ñ}p¬Ï!×wüÚ©ŸŠó! E£¾Îqü|c²Ú€!ÞE¯ûEß ½ŠîÄQD7JyoÅù`çC/ñ€Ýˆoöj>üÁ¢~jT5£EÊP˜QæryÎñ§%i{²Ewö$Ö]8{“}¿è{M¶0Æ3¦ýçׄ0fo­ö`'œlQj"²'»»€ÁõÐýýápì³îâ(vèƒAè–˜õe¯÷Ê>u› U[]î—Ç9ù^uOxÀœ–ø™)ˆÃ$B½ß6b@j}” -C3ê«Æ]ù§÷…X÷7½M&æâ³Y_ŽŒAÊj๖[|ñò¼{/ª<í-Ú¶¾ºA÷ãËÆL>Ô:p¹q Ê\ª9¾¤s°ÎÓ„¢oMùª/Ÿ˜1︌³vx—È—iOÌØèÇ17¿NÍŽÃɦÍ#2 €²­/–œبGÕ—ñbÔÆƦǪŒôQGgO »E߃Z…ãDo5žcªðÂxßÖy¾ˆ1{ÝI‘%|³ˆqÝÅã㶾f›¼ŸñF]®¾¦ú6ô±0¦hÖÿfqçùÝúpÔ´DpÆ‹š†nÛÆ#Q Ýù$ºæu7ôÓw­ê`§A謻k º¾ñÔãMŽqÝèÇY"/+ŽS^>«_S¹Ýô FTŒvCû§—X0ëVíofKJ³íUêΚ[qYæè(B>§º¤›<öÖŽUyõõ“¹TûùÇõÞâýg¨9éƒh}vz‹ˆ¦·¤îÙ±:Ï3hE^{=FàmvÕè7±ÙMÙì¶öÿ…ÍÞ”y£xúΚPÖwfóǪ<Ú•éÍWÒcIæ î+„Äiÿyàý¢ïÌÝÑškHt|Eý`ç ™‹(ÂAÁìç“¥7Ëö³å¤Ùªô{|Èâ»Í®»½^„Û¨HҷゾyºÖy*ºo'†¾Yî²îâñÜŨÿÀj·Q{Ú¾¹§4KÞò†þ'ó¦»Uv[Ÿ(=§éÔŠ”Þ(°«½4ëY¹ŽyÚßXøZÉÎÁb…ù8'2Z:šƒ-϶m<’ƒXmõ<½O:ê©î­l¬ó4 P½n™cŸn¼jó¦xnÝèã(¿6Üëi©¡ºç§Ãr0Ú9Våy&…h2Ýx.y7Ú©Œ&c­,»ÑÎý¢ïíT=±¬ÆÄú`ªê²ûè`ç‹vjÑm[z–Œ9uèjÒ÷øu¶‡´ä ØÈD›íÄ>þhv-Þ¸5¦ãø²×á\eÛî¸Å¯ö<ù²€î‘‡˜ÛÖéë!Lb–ån´EzÔ#P„¬¶Z>x oÕ p·Þ¹€- Ñã½[ô7ïOó+þYÖg¿t%”áÄ€.În‘1¦16Ÿ/œÃ•±ªý¥yN äôlO*çiù7ëqjo?þé_ÿå?Æìã¿\æàò¾§Å¶ôZwýé+£ú>Y¨s™iÇô¼Ó.ÍOZU“Ywø]ÞGÓrªÝú9Ù±êÓ× *_=ô›\“_Ozè%dø¯x¬Äù«ïm飻+Ú l,»z]’Äùùkí“´¯QÚÝh}iÕô0ÝÕ®Ô.DÔLaKÖ»GŠ1“õàÐâh¾J]IMf—y,»4¸TÜ%k»eKôCw Y_ŠoecÿZûœƒØE‡. ÓßÊÆô*?SAÚ×p1„zI 7;6ÝhÓ] EÓx16¬kŠK[u%!ô›Íë7Ïû,rj Ôíú:Rû)µžM¥é2­/»Ï¡é$€f¼Ëõ¶h°=êAÎbKhÁ«‹QGjYªÍdì”mÕis.@óÖ[ÌM¦4WkÃ2Ýq1G)÷˜b¡þuig¥¢Œ³’6eµÝæñz´D=’Qórw~)ÙcÉbÐÒX»Sy)I¶í›×}7o{nƒAÒ™ú¥šlJ¾ ÿÞb;—:W+V©ÕêH(Ë©‰pÔú¦X¤é?Âð%%=EÏ2Î\†å’‡ëÜ_?Ù®’´õ“±'L‡c"Ÿƒþ—&¶¯`Ô}ÆÐTûïNìjºCÛ¸¥ÇÀXZ`ÎoV‡¦3íño^$ê—YSEý/,HƒÞ¢'ogÌå¢i EVË'ÓÏ?è“I -endstream -endobj -358 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 357 0 R -/Resources 4 0 R -/Annots [ 359 0 R 360 0 R 361 0 R 362 0 R 363 0 R 364 0 R 365 0 R 366 0 R 367 0 R 368 0 R 369 0 R 370 0 R 371 0 R 372 0 R 373 0 R 374 0 R 375 0 R 376 0 R 377 0 R 378 0 R 379 0 R 380 0 R 381 0 R 382 0 R 383 0 R 384 0 R 385 0 R 386 0 R 387 0 R 388 0 R 389 0 R 390 0 R 391 0 R 392 0 R 393 0 R 394 0 R 395 0 R 396 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -359 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb54-5) ->> -endobj -360 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb54-6) ->> -endobj -361 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb54-7) ->> -endobj -362 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb54-8) ->> -endobj -363 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb54-9) ->> -endobj -364 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb54-10) ->> -endobj -365 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb54-11) ->> -endobj -366 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb54-12) ->> -endobj -367 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb54-13) ->> -endobj -368 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb54-14) ->> -endobj -369 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb54-15) ->> -endobj -370 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb54-16) ->> -endobj -371 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb54-17) ->> -endobj -372 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb54-18) ->> -endobj -373 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb54-19) ->> -endobj -374 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb54-20) ->> -endobj -375 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb54-21) ->> -endobj -376 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb54-22) ->> -endobj -377 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb54-23) ->> -endobj -378 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb54-24) ->> -endobj -379 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb54-25) ->> -endobj -380 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb54-26) ->> -endobj -381 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb54-27) ->> -endobj -382 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb54-28) ->> -endobj -383 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb54-29) ->> -endobj -384 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb54-30) ->> -endobj -385 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb54-31) ->> -endobj -386 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb54-32) ->> -endobj -387 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb54-33) ->> -endobj -388 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb54-34) ->> -endobj -389 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb54-35) ->> -endobj -390 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb54-36) ->> -endobj -391 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb54-37) ->> -endobj -392 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb54-38) ->> -endobj -393 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb54-39) ->> -endobj -394 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb54-40) ->> -endobj -395 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb54-41) ->> -endobj -396 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb54-42) ->> -endobj -397 0 obj -<< -/Filter /FlateDecode -/Length 5075 ->> -stream -xœÍ]K$¹¾ç¯È³ K"õs0`/ž~>ür0KôÿJ‰ÕEaÿ„ßèÖp\¼Í!ùãÏB ¸Äì] Ç·þ»¸œ‹¿þû7‡ÿ]×±G·äœõǼ‹i‰)FãVËXRXÞ/1:V° n1g1|dÇ׬18'¯± ;—V‚ÄÑÒ5+C -k,ˆa³ -ËÈa1*‚fr -Ïú%@éAã9»„”]Vy<Æ#ZBdë­ÆËKÁ'ÇÅgc5ž‡,Þ2Y˜RÔx²P £ò ‹K‘4^„,ÚÉ -/AXWRyÅc5^¦ÅçĬò2x&ÅtÍsÆ/>gíâa›¤òàQ¤žƒ,Áø¬ò ‹'ã4AÆÆNxŠ))ÞêÄ»ÉÕx²8YåA ¾†.@ƒð EpIÃãÂÙ±.щ“†.å…c$ \ö ‡5 cÁsV2ÅsЀ,dax®Êƒ,”YÃr4, ‚,ŽIÃÚ2¬EÃܲÉVÃÞ²±AÃàBL£aq)Ö°€¹0¥¬as Ø©atáîQÃê’NÃ짠aw‰­Ñ°€»D¨©ò XàãB6‘† Ü%k¢† Ü…9 0ª ì0hX[Y\JFÃîºdXÃ)@åÅ>H0p×yÜ‚6sw7NÃî:vAÃÊü1j4Ü„x=€,X 0߃çHÃ4¶8„%xà®Í˜þŸZ-6å aîÚ„áSyy±WüZ^lYÃÜÅDExà.à3iXŒ_0£: ’l{ ü ÙDš£ò yÖ° %'éfÒ° %ÈâŒÓ°¶)ùfÔ° È‚ˆOÂlÂ"a²†€qðŒÑ° Û„|Ó±†ðKÉ7“†™ òM¤xŠßf -’oF 2CEåAMYñ[`òM “Æ ÅeÖ° ‡ ùfÒ°aòMDôê’@’|3ªËÀ]ŸL¸‹È4hX€ 䛈lµÜÞx}D|¦3“dœI]10€^¹U]20À^ÌVQ]30_É‘ÔEˆ‰Ì2uÕÓ*˜Ö¨Ëø‹xŒ¯0á¼ú4/Q˜CÄœ™¢Üd†[0¢ûžsŒbt£Øy}ì=üö3&ßã_ÿyxýMY)k?ÆO$±öâ".c¸WKiŽŒ"Àõÿ)BÀgêˆ1!-`±.º‰˜Š1Ýæcÿb×"Cb‘Ù"˜)wÆs¿È,†Š‚.ëÃuÞIÖ¡£€œ+3PĬ”$F€Ž­ª×uÞ]¯s¿“’Y®óîzE ƒ¸Â9»Ò‘…ÓÊÊ(EU±{+½»f率–Ti¯ô^ÒRS#)/Ž#âRDû‘ZJ ¸¯L²>Xå=$EG¿”¢ìd¾ˆ#S3¼*ü»O@íþüT¡_Nr#e(ÖûéçÃoÿýÇÿú?ý„n?½þü½1þ³1ü¿§¾³¸æ×ú™ðù?ÇO?ÞéCâdIpUÒ¬¡íêÉ/lØy¥º÷“òÉõ3n6ë0ãYÉŠÒ ©ð‰I¤w±CR ’" e¸n’ó ÝáÓÐ=—ñn³B¼«‡x×ÝØ—]’Ry¤Ä_KÏR I~ÿimá2 ð:˜R`¶%d¼ëûª<åßÂ2b‘Ш¾ÐÌcñ" d&ÛU›Åxk3NJâ–Í—v{ÂUcÝBýü²Ë–&/¸jrÅ»’Ò¢w¥:nÉ¿^!€ç˜œyOè\7Zœ¥JmÒ‹ünahæÅdYyVZ݃xâ'Ùäµú[ö,‹9¡ŽZ»&ܸ$ÏNñkÎ ´*¿å»l3°ƒåòNXù”¶&¸Tê³Ì}[ø¼”íéI B\rà˜Õú~ÊcKë%`Ñ%ø¾Ü -~·ÐLFŸ¬¿r“"· ^¯ÛðqLþmŠõöLdW) ㇎ä’-Ôû¨´¯Î‡ ãd‰7>· F‚SH‹Aó²F·êä«Ài†‡«Vw…c=šSêo‚Óæ”Ú±¬mp(ÏCxJ‚‘@=wÿ²Ûˆ™“×êï—`a„lèÒxöJʪ£ôÔýg¸’Ù©µ¿Åô0CÒãò²Ø›ãìojk74ŠÝŽUÚ #X¼nã!hL²Þ˜‘ðHÄæy ÷Õù0Ðèä!ºÍñqh|*GÖºîô}ÒÖë6Ë[•ú¾&­m1©&­¾ÉXenîqi>‚âÉJ®r„Qd"¿9±î«òqŒG6’ù‰yõyãYu:ŸÖ%GÖ|5,²Í#22©£w‹ È>·²°}U>ΰÈf1Ù]‘¾a¸³îdµ´©«=¸…©»Ý¯ö]U>ŽÚ# ß‡èSÞu£+¨ÔàHW}ŠòZ6xÙ“RÙÓq_÷ûê|åËæÞd3ÅoióëNÞ+Ä¿nõ±_©ÿ@ˆ¯Ô~8ÄF‚â?wÿ#ÄWê?â?'ÁñµûG¸*+›ñüûÈ -Ë iä3×߇Ö;žiAùiû Yaã²è ¦uè§FgJhž´i…å9 H^¤²!nÚíÑG M’L¼‹}xëdHÙ¨#¼%—ÕDöN÷ý}ö!Û¥¡Ý¬{ð¦òÅyy¦þÔèÈÆjgSÔ½}xñãÙ=i!-ý˜´y¥Ûí´ ›'ƒ÷ñž}ìJÁ9ÖTß_§à—ÐtIÁÉUÙ¯©,×Þ3öUù8a†C&’}wÄ« „°ÊºóVí½« ;ë|uyÙh„¼)ÿ Äu§ï³ºpÝæc« Jý¯\] @”dÃãîÕ…}U>ŽñDy‡3rÐ}íϪÓÇW(%DN&`Ì÷¦¹ûª|˜a‘È ó¦¼’ûí2­u'Û« liAôŒÙc÷̳¯ÊÇQ;ÀÄ’KÞ¿ãêºÑj&y5.hÊ…¼pfË[+ ;ë|E{ dö0UE¯¶Å}_·-íØü8Ùöªƒ½»ÇæÉëúöµýnå¼c×ÂSRL«JýíU‡>Ÿ*µëc- ey|å^ëã­ýV]?òÈKhÛ»"FŽzû~®œ  >æQâãÖÆ¢ûE¿Âä×½ÈJ„±ì¶w:Ý/ú”@÷¶M{¨:–Wò²ä²§ ÷UyIß6Ê[à !–¾áéÉt‚uÆRߌ¥mon»·ÂOÏPŸ+/9PØ7áî«òaæï…ˆìݽžÛFÇÒ_¨…ãü²P–Cûå„ZÑÙ²ŠZ–«‹µÏ6€ÜЫ/‹­]¡øÙ¤lAµ/Íè®èµ¬{™&¯Ú†´Õúᆒiô)õ¨Ö•øQ®Ï†Š2í:Ÿ‘•_'¹hjßTä½réÙ×¼Ò.·{­.½GõnVýŸW«Ekí´žåú¼Ž\GÝ|‘I»v=B“»nìõI,JˆœŠUe¢¦ý:7I"S›“ævÉ«ôr\Óu;]Ó\Û -÷ä@G·ä0u®¬sd³$m¾œÚ–c»¹ÒuÛÅr|{íl•ÃJš–û—~>Wz¨×¦ÊÆJã—R·Þw“½ôáîô§Xjë[úÔî{´yW·îòþ}k£zAi'Tz˜YŽ³³,oñÛ+OÒ·Ò³u~íا‡ùYŽÅ¼­~#Épõª‹p}𶵅à-=HãÚO»{ø2¾ͯ†‚Üb Æn.ÎÜ/ú^«EÙÇ%3Ð3arÎrÈ}ÉöÕùx«2rf‹õ¨}c[XKmJ:]k}aD½|+×õn¶w™C¶°ë¢¾¿Xp6—9æ”ú‹¬©¯â\´5gk½½;yn¯ÿ òÛÞ?oÈ~™-ÎzÒ]/á+Œ6»# S¡£­G¤ûªüÚ8½ŸÌvÎ}Ëé-ÓYíL£‹¡ñ“:ªgãD4eA,ámšŽÚéJÜ,²¡´–…^/'íP¬ÔФŠ-— ¢å)-1•ð®$zé¼QBÈ0"”P3O‰k®eZØf×ácOà^FÙC«ö——›M ÃÚ\žÆ÷–šhË{ŸkÈv!ïªý–Lò¬Âå\WXÊ=üp9”¿ÊG´å\LRys'É1†‹@`ÌGÙJ” q?ªÆeÛãšå¸ñ_z)ù$#‡’Š#•¡/jù_Ê ÿ|WqCËuËiÁ¦“kƒüvÓ¢Y!3„yƒ,ÄúÕô¶qº Ú È€yï²$BS¿Ž$F„èÛÝH}ߪÉM²Þ®ÄÎ+DÔ öš,wo[Éò†QZ¨^….ŠMfYfï ò\v48*n’-™‰è¯‰fên7y¬œÐØU±‹>«ÎNÃßʺ8 ç=âé€né’ú¦RåœßÖTÓ.2ÞF£I¨Ü©¾•ìÂ}œÒé ¿3­iÆ`vþ4jrZ♘š!q/—[9…t:¨åjk¾Tì¥Ò¨š›æ‡(ÃÔ)uÅ+´Ó´åOL>b¨}]“e­¥µI“ݸ+âé e®Èo•Ò~`…ñÍôf"Mívr2Ã^ñ@U\³Ñä¬9ùÿ ¾SyÊËݤä¼Èq5:ßìäµ’fzè¨Ãš,ÇQvR—±Ûì¬/›ŒR^@·—î[Q'•Mô˜›xR<ÖdÛ­®“š~:VÎDÓ„@wƒì»)¿­ÈCNß]Õ›ÞY°“¹ûÞî ‡Žo39ö·røg—··;Ù|#žÊÄÆ—d1ýÔ:dÙ™Ņ̃w6‡`…A0úVÈÍ[;:É4h_‡“i3îéð·C™æyÙ€äÚÊYÖsäeÿ‡2?â÷ï>þ©žxžÇéhá:¶®ÈóL&ù í$Ë_BhÄî6ÑñûƒlÓ(ë›5ôyܹ ¯yT7Ø[IÛÔvîLôC‡ÙuÓ•E£¡‡xºÐbd¾vlÆ„}¾¤vsè†äØÍ&.æÛÁe<SòqjWRtLéಛ*âbn—S‡yH¶S r^äçy(Húš.¦0B.iðxL³sÓVîb*‚˜æ‹¹a¶s§y%"ôÓîõÕã^EÍ„Ú¡½Sßfj¿1¡vÀÓ©T.Ãú²Ï9T^žmÎ;4ÔÛ¢É÷¨9Ã5FhÁ«‹ÙF'jÕj?wʶê´M.@3Ö[ŒM¦P«5µ”;Nª–b)õo³¦ý<*]‰~•pUVÚm3^–¨@~6ç8îÎŒ’=–L --̵;•GIj´ë¾yÝw›mÓmð$ŒÈÔŒjþªäSø÷Œïœë\¬Dh¥ÚOY±i7Œ‰šm$Gå/H™‚•ýÅHÉ#§i¹äá:÷×O®WIÚúÉÜ“•™D&Zù[`ºo`ämÆÔTû;azµlÔ[zLÊòЀ9sµ:Tþ þÕE¢~%U”¿ýDô&ù“ÑX}ƒ,¡„ÕòIùù¹þP -endstream -endobj -398 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 397 0 R -/Resources 4 0 R -/Annots [ 399 0 R 400 0 R 401 0 R 402 0 R 403 0 R 404 0 R 405 0 R 406 0 R 407 0 R 408 0 R 409 0 R 410 0 R 411 0 R 412 0 R 413 0 R 414 0 R 415 0 R 416 0 R 417 0 R 418 0 R 419 0 R 420 0 R 421 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -399 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb54-43) ->> -endobj -400 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb54-44) ->> -endobj -401 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb54-45) ->> -endobj -402 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb54-46) ->> -endobj -403 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb54-47) ->> -endobj -404 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb54-48) ->> -endobj -405 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb54-49) ->> -endobj -406 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb54-50) ->> -endobj -407 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb54-51) ->> -endobj -408 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb54-52) ->> -endobj -409 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb54-53) ->> -endobj -410 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb54-54) ->> -endobj -411 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb54-55) ->> -endobj -412 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb54-56) ->> -endobj -413 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb54-57) ->> -endobj -414 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb54-58) ->> -endobj -415 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb54-59) ->> -endobj -416 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb54-60) ->> -endobj -417 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb54-61) ->> -endobj -418 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 378.232628 101.433071 359.482628 ] -/BS << -/W 0 ->> -/Dest (cb55-1) ->> -endobj -419 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 163.788585 101.433071 145.038585 ] -/BS << -/W 0 ->> -/Dest (cb56-1) ->> -endobj -420 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 144.841563 99.933071 126.091563 ] -/BS << -/W 0 ->> -/Dest (cb56-2) ->> -endobj -421 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 125.894542 99.933071 107.144542 ] -/BS << -/W 0 ->> -/Dest (cb56-3) ->> -endobj -422 0 obj -<< -/Filter /FlateDecode -/Length 4555 ->> -stream -xœå][¯ä¶ ~Ÿ_1Ï¢H"© ä¡@[4hŠX´‹>lNv'’>ôï÷£Ç–e›3öLN‚ºç̘ԅ¦¨¤,kÃÙãç³€?…ƒ+¥æÄç—O?¼Ëâño(±¸Øßà7Ë9xÎNBMEÎ?*²ËUbNç×þ¬.û⯧üîôïepŽ®Öƒœþ¨,&—KÎ^ VË2X".ç(É`¥àp‹µZ,ˆ!™#oYbp-b± ÇX VÄ9ЖU!FÌ-ÄU²Á -r„è½Éƒ ž™¢Á ┞,^ .•«ÉËà…-‘K™ƒ‹W]JIŠÅcÈ"Õ‹'E³·x…©d‹— %ò&²Ä’ÉâeÈ¡jð -du“Y|òÁâUrR ³É«àù’Ë–½8)DÞâ…à¶I&3 -ƒ”²Á‹%y©&²ùhñ²0~,pB(—bÌÖ¨³›<‹'%ÆÌ&²`ð-ˆ ²øœ,À rŠKÄœ×ÈÄBŽ  b©Žs& bÇ©f Èðb°€€W pWݦ…)+!  C-,H…]Í9YXª!ŸÉK×baAö  ËäAÎlakW© r„,£…9B„  "p5 ú6ÓÈs1ó†,ˆ-,’¹R™-,È’A«/EW2àÚäðB¶° gvpÅÁÂäR®`Ò[X€w.YX1À laA®SŸ"W""s‹ ‚l é‚‚…%BDƒ”Xe&oa<4rI¨Úâ±×<³XXP89 -,À4A¦ ¤4y0aaÆG“m±°É’M¤9&²°…¥DM7‹…¥@–装°MÍ7³…ÕCD|TŸœ†ÉÆÁóÞ‚ -òÍÈ`^j¾Y,,¨ä‘o"Å3æm¥¤ùf¶° 2d„(&² hªÆ¼&!ß:Y¼Ybe jJšo f ßDDo. Í7³¹\Ü• -d²xÀ]D¦ÉÂ\ ßDdkåöÀ+ñ™Í,šqsÅÀz¾Õ\2ðÀ^x«l®x€¯æHæ¢ÄDfY’¹j· -fðæ²þ"ã &\VŸú%ªsÈð™%ëMVL FtßrŽ¹])vYûùûÓçà|ÏßÿçôéwÃJÙô`üÄIkÑ)+†på†æÜ@äABøH"×YÈXÝTH¸b¸Ûzn_ÂRdH¬23Ãqß/2…€¡¢dËzw·’•¦~1ý> xC„ŽáöÃaý9ö°€Ù‰zg•·i,PŠjTìÄ„@ÔëyQø÷ïN˜Fc÷—ÚvÙÉ oÎPnôçw?ž>ÿËWÿë×ß ÛwŸNï¿@QõuôågßÅŸqü”Ë'—Ëg+·¢K?óŠÿ2~®ø]ê¶m®ÛþÇÏOËϾnëó“-c+;ɶí·ë¶¬‹:ë{Zéf}ïë{“—N¯;÷=\ÿóüî«ÓÞ-m³ˆc²0äӀǢ˜vÓšÕxȘ FfŽ`AiÃòÁ÷¼Ôÿ¤Ã6>ùš.BÉ|­¸ŸLˆ„3\ñŽ6Öy}"?AŠCû Vý鶷‰·“e¢éD=–D·Q@Ä óž^UyÅd]èÇ]Ëí!^& ,Kk¬;|wE¡EÃM8{]vÓ;áª1ëm«ó4*š£¡[äv{*Ý|BøÀóuôów}9xó5p|šù²òp D¾ù+¯9´ùx܇e;ôÞªoóZ­LYÝG°MV– ÞòÌÈAHÓ¬=ÿu°Îó˜,²@OAGßm²ÿçŸÍŒÔŒ¿ëøªdÐTãy PŸ…éâq¹Ë?Ç„,ʆ3qÖ}´©¬ó<ªÉp’R¡Ç܉t×ßv=Aö8s úê{×´±céì¹o;-?oŶ*ëµ¹t%“.¿þåŠi”¬+tºúztÆ©ñ4f¡;J¨”ï›1y¤SQ8 ¬nr/¢=Våy|%ûös±ÉîÖQìƒÑ-Qq!^žÔÅUd8Õ=í«ó<ê]…[©wÃÑz=«E²eñ­`g‘2ÛŠO²Š®Cî}ß,ú Ô¼ì¥BÄ'.g†Âü°"´'Ù±:Ï·"ŠוìuCßžIPîf•lQßô²ô›>tIÌdRy¥}èÌiZ5‘¹ý#Ë¥R süb+{¿ã?Œ‘à‡®c©ôm—Ü8«¸Ö'W¤<áìÍ‹ƒuž‡87‘U‰Øk˜“Ò®…<ƒâ®w¤Ïuƒ/5¬:zß ¸ŽßfÜ2ÔkÆgéV¥ä©ìbÙí¢o…eœ @&ñq,;Xç ±LúJ‰l›Õ#X6¸´~…ö[#">²¸²*³~œ!v5,Yá`ëo-÷úQÔJ¾k}Üê#õþþ×xÙáwx8c¤UqÑ=öc=ÔB‚£I{æw¨ÊÓ`™x`oÄ7ûñÁfà—èþXÖý ËÆ~íÔOÅù…¢Q_Ç8~k–è&°Äºak½n}+ôÆXÅ„„mnŠRÞ[q>XçùÐKD©‰È6$*ø‚EýÔ¨jD‹”¡0¢åò˜ãKÒW;‰ÓýüAöûfÑ7ìªïLÅüµÖ„0fo­ö`çìäuïçðž5ØÍt®‡>êï—‡cŸeG±CŸ…öĬ/{½WFð©Û‡­Úêr?ÞoÈÉïôªïŠŒi‰ÈLAXö˜Æ€Ôº{ͨ¯“0îÊ?¼GȺïñ1 –˜‹Ïf}9¢ƒ”Ís-×$øââåy÷^ÔxØs¸m)|9ººÀ‘’n ð7²¾è6lø¾­öCUžGíE£Ô¶—÷Ûƒq[=Þ·Z÷—Co}«¸ ðë|]ØèvÑ7߸š9"Q*ÇS…Qrà=|?Xçù"F}»E8—j?y‹ˆqÙÅý㶾f›¼ŸñF]®¾¦ú˜ ºeƒ S4ëÿjqçù]‘úpÔ4Gp†EM] ·m㞨)'}”ªïÙž£¾å}ÞÝ~°ÎÓ t.úÖëQoç—¾Í)ò¼â8äå£ùM&·›Äè]Ð׋ÑÃnhãðrFݪý«M9¥Ùöª -ug͵¸lx?N(g:«-é&½u€cUžÆüÊð–-é(&–—­½ç÷BâT†Ó-ˆhx{òö®÷cužGi¤{4¬zÃ9»lôט³ÛFíß`Î^—ycxºC2×èp0¬Êó˜¨kƒ”öC¥ÛÁ|IúJdÐWa÷bçÛEß*˜/h£zÞ»>º¢~°Îó…ÌÕëööÉ~q`˜éÓÌöãÌIã¬Òïñ®ßæì²Ûõ"ÜÚDj¨.@¡ÐÈaè>XçiæP%&z²m¼Eî²ìâþÜŨÇj·Q{ؾ¹g4sÞò@ÿ“yèî{“ÝÖç#FÏi8Í–þ˜® -ìÒ¬då:æaCpᵑ݃ŊéH’ÑÒÑl~–°m㞬2ÚŒµ²ž ça£žêÞÊÆÁ:Ï -¢â^y­ë¡xnÙèý(¿œ¸›aÑ“ñjLúÚüÁhçX•ç”¢;Éô ¢¢Ø¦³ rÜvnýêøf<= kÃúHq•„ÒFÐÄÙ-²#Æ Fs‰òVÑj?jW¬szl(RÔa)ëIªí¯þøç?ý­P`²ñãÅ|/¯à{šMX¯uã¾u¦¯¤„:–6]Ž›uÒø°F1u“Ðå•-§«sú©miyþtáaˆ–M—ß¡­ñûÐÎض䱿ºaH-CWû.üEû2öÝÉ3´!Ý=ÈtmêMCùÓ) §Æ„áä0Ÿ†èª=èÉif“=qÌ4taPv8_©9ÈúS+¥AòzlÛe5?ýÅXþ§aÐ<©çWýëܹœ§èr‰ÓÅL~=éyšl×ñ Y‰ãWßÚÒgQ+Ú æXvu]’Äùñkm“L_£Lw£õeª¦çœèfšéJçÁ¥cƒˆš)lÉz÷ˆRF²žIZW©‰"i’ĹAîËÎ ÎwÉêg¢l‰¾ën&ëáSÙؾÖ6æ 6Ñ{Õ…nø§²1wÃy‹ørB·´¦¾šT= qjjÒ.¢Á‰FPµQe*Ù„Û8¤—“AüÌOMk˜ÑøݨéyRb™ ‰[¹:•3H/'³ÜØš´‘Ê­T™«ÖIó³(³©SiŠ7hƒŽFjÖC–»9âiúº$'i¬‘îÍv7Ä—“nìÚ_O’Ò2c…—Éôz"uí6rñ³½ÌÔ\&UajN4=Gÿ^ BU‘gø¸™”ž¨5_ÍèÜl䥒zzj¨Ã–ç¹l§.ä4Íf{}…âò -º­tÓØ‚Ú©¬£ç:‰§ ÕÀcIÍêY!lÒOÃÊžè'!ÐÝL–fʯ ò,§´©*¾u–BgîÒÚÉ©açkOÎmăÖämív6?_ÇÆk²š~™:™ÉúÐn2ŸÖYGœ(ÌäF_ò4[:é‘ÏÓ×y’Y÷åôÃips‘$j8˜(èiŸ}äÎÿE™¯ðû¯Ÿ¿žŽ—ºøñèZĆ­ rïÉôlèr¬gEOÄ6m:bäögr(sY™¬¡ùñ;¼æ¹ºŸ}*‚†Æí‰2ë°#Çfº1P§¹A|Yi±2¿tlf‡}¹¤éæÐ éÁd“T¸èo—eæÑì’Ï]“¸Ò¢³K—cW}£¸ì:¬³d 1Õ ûNšêU~´¯î¢ #ô’fÏn¶oz¢ wÑALýEß0‡¾ÓºÇúáéõ‹Ç}9M3@© Úõµ§¶Sj<›JÃeZ^6ŸCÃë÷Óä5ÔÚ¢nîQ ræ©1‡¼¸èm´£–¹ÚX@úNy¢-:Eƒ ÐüDk-æI¦4V›Ô2Üq1µ”[L1Sè5-ý¨4%J?*iSVÛ<^‹–¨@Ò›sžïÎÏ%[,Y Zêk7*Ï%i¢mûæeß“·=w·Á³ iŽLý\M6%¿GæÎ¥Îj%Â*µX e>ŽZ_6Ñ…O wJJzº‚“œ¹tË%w×¹½~²]%™ÖOúž0Ž‰|ú¿¥Ø"¼‚Q÷]SÓÿ¤bWÓ— -Œ[ºOÆòÐ s~³:4—ã"Q»Ìš*êÿŽAô=Ô;û`. K(y±|2üüð½ª+ -endstream -endobj -423 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 422 0 R -/Resources 4 0 R -/Annots [ 424 0 R 425 0 R 426 0 R 427 0 R 428 0 R 429 0 R 430 0 R 431 0 R 432 0 R 433 0 R 434 0 R 435 0 R 436 0 R 437 0 R 438 0 R 439 0 R 440 0 R 441 0 R 442 0 R 443 0 R 444 0 R 445 0 R 446 0 R 447 0 R 448 0 R 449 0 R 450 0 R 451 0 R 452 0 R 453 0 R 454 0 R 455 0 R 456 0 R 457 0 R 458 0 R 459 0 R 460 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -424 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb56-4) ->> -endobj -425 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb56-5) ->> -endobj -426 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb56-6) ->> -endobj -427 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb56-7) ->> -endobj -428 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb56-8) ->> -endobj -429 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb56-9) ->> -endobj -430 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb56-10) ->> -endobj -431 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb56-11) ->> -endobj -432 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb56-12) ->> -endobj -433 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb56-13) ->> -endobj -434 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb56-14) ->> -endobj -435 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb56-15) ->> -endobj -436 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb56-16) ->> -endobj -437 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb56-17) ->> -endobj -438 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb56-18) ->> -endobj -439 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb56-19) ->> -endobj -440 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb56-20) ->> -endobj -441 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb56-21) ->> -endobj -442 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb56-22) ->> -endobj -443 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb56-23) ->> -endobj -444 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb56-24) ->> -endobj -445 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb56-25) ->> -endobj -446 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb56-26) ->> -endobj -447 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb56-27) ->> -endobj -448 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb56-28) ->> -endobj -449 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb56-29) ->> -endobj -450 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb56-30) ->> -endobj -451 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb56-31) ->> -endobj -452 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb56-32) ->> -endobj -453 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb56-33) ->> -endobj -454 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb56-34) ->> -endobj -455 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb56-35) ->> -endobj -456 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb56-36) ->> -endobj -457 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb56-37) ->> -endobj -458 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb56-38) ->> -endobj -459 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb56-39) ->> -endobj -460 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb56-40) ->> -endobj -461 0 obj -<< -/Filter /FlateDecode -/Length 4707 ->> -stream -xœÍ]M$9n½ç¯Èó£•DRÀ`lïáöaàCOm×.Œ³>øïû12ô•¬ŒÈìêÙêBVg¢ÄÈ'R¡P…³ÇÏw¿ -WJ͉ÏO¿ž~;y—ÅãßRbs±°Â'Ë9xÎNBMEο*²ËUbNç—~½ºœ‹¿œþó§ÿÙÊ„stµ¦äü·/Êbr¹äìÅ`U°| ƒ%ârŽ’ V -·X«Å‚’9ò5+C ®E,Ôà‹Á*Pƒ8ºfU¨s`‹5B•l°‚‡!zoò ˆg¦hð‚¸„NO/—JÕäeðBŽÈ¥ÌA‚Å«.¥$Åâ1t‘êƒÅè"Ù[<èÂT²ÅKÐ…y“]bÉdñ2t‰èjð -tu“]|òÁâUrR ³É«àù’Ë5/zqRˆ¼Å Á l“L< -ƒ”²Á‹Ð%y©&ºùhñº0~,pB(—bxkTï&OÁâ t‰1³Éƒ.| b‚.>' àANqÉ€˜³ãÙXÈqáb¡@,ÕqÎdÁ@¬â8Õláù^ ‡.ÂÉB -Ð…á¹&ºPe (B°°€ºD& hË° p˾ xË>$ € 5½…Ä¥’ØÂ䔪…Ì%`§…Ð…»g ¨K’¢…Ø%.ÉÂîoaw‰ iò  Ø ££PÈÂîRðÙÂU;LÀV\,Å[XÀÀÝX<[XÀÀݘ1˜¼ìàƒdaw£à¬™¸ÅG ¸9& `(øãÍh¸ ;€.X 0߃ÉÂTæ" ¸*¦?çV.”š,,àn(>“W]ÈèqÃoÑË.¤T-,à.&*²°@€»€Ïba0ÞaFp7fj‹Ü ±z ¸‹¸A,,€9¸€ÀÂÌo.x˜®É«Î×R,,à.`)ZX€YØù$1yÙÁ‚…cç‡ZX€»˜Þ«… ¸ë…È‚ÜõŒ°ÇäA*ÑÂÄ5àÁm ŸF°ê|RX<à®a‹Üõ>W pW§M RVB,@A‡(8ZX -»šs²° UB>“—®Å‚ìA@–Ƀ.œÙÂÖ®R-ä](F r„.,,@Dàj@ôm¦Ðæbæ ])ZX$s¥2[X%!C¯Z¼]É€k“WÀ Ù‚œÙa* —rNoaÞi¸daÔ/°…¹B¸Š…˜Š\‰ˆÌ-^€.²-,@¤ ^”] ZXPbA–™¼…˜¡‘K¢«-{Í3‹……“ÓPѸ 2M ¥Éƒ.€ 0>šl‹…H~l"Í1yÐ…„-,(%jºY,,(ºD-,€mj¾™-,¨º â³° úä4L¶°0ž÷ÔPoF¶°~©ùf±° ’G¾‰ÏðÛJIóÍlaAeè"QLtAÐT ¿&!ß:Y¼]be jJšo f ßDDo. Í7³¹\Ü• -d²xÀ]D¦ÉÂ\ ßDdkåöÀ+ñ™Í,šqsÅÀzs«¹dམ­²¹fྚ#™‹P™eI檦U0ƒ7— <ðñ_aÂeõi^¢J0‡Œ9³d½É -·`D÷=çÅèb—õ±¿ýåôÇϘ|ÏùßÓó–•²ö`üÄIkQ‰à ¸rKuŒ€ ­Qጪ!|DF•Èu(‹ë¢›*‰©Óm=÷/a«24V‚™åÎxn™BÀPQ²u½[æt}„hù{ŒÁo:I@ÿ™ýzPæÝûunwê$C×»eÞ]׼ąáöÜn¬¦’{…wµ#ýw¾þ²£$ÂùãJî~g%© ×éÅ™—bŽv1ó«Ñɤê"ï1æhH£Õ„RT£Îî€läež7…ÿáÓ @¿6y†Ð/'½o2ú6úó§_Oü×ÿãßþôšýô|úù{Ì1â>Ïøä#ˆÎ8þð]h¥ >þRNùúY¾ë*xXË*M›Æû_~ø¯ó§ßÖN×â>"ÁÙj§ -IºT(ùò?ót½Síòø„5Ü»ªÖӥ̞×&Íã%ìéŒ8 ÊhìsU9=¯•Ý®‚CÐåBy=(?¯:´[=Ré3œ€x˨ -C _Öá[«D ´|úp®Ã¦±¸ºÜú?~Ú|ÇÈß`u~Ç5n–|È# —6 ˜÷µÖ32Gä㙘w;(óñœ).²|Xe0½V>ÃZ~Áçiu’çõÿ²k9As”à ¦©m#?³¼ ƒ ˆK^dÒcr^ÞwÞˆ¸4èÚEyK¯Õ{[tÅ`¹Šjb2êäúFÿ}A7 —2÷•Ã ŸÑNørH× ¼ ]W0º¼vÖ@jÞ¤Ë&DH–ìî¶w“ù -~g§Ôǘu’éÎ1jJä‹&z›J#^ë9<Í.qU[w…uÒQ·=`R³;\ÕyÄôã²…æ-ùu†¸Ë–Ùbo’œá­¹¹€|%yk\ž÷ZŸ\è¡Ö!$¹ølÉ·ñ“r<ɺìQ–å[Ÿì×Åy1•¨K(×u}ž¬ÿóN/±ZF(dÚþF+]!’½Ú’>jÙ´ý·"‹”DÕ‹¾[lQtÝ£ú¢H¬âœd=É|¼Ø"êª!Ú¯ùat[éâ0 Xꨓ4,ý²§iW}ddÔzúÔUª¯©Zò{­û<¸ÀŒMé9´>˜Ýšè;AæC½5¦!Ï: îi Õ-û¨Êƒ¤ìjâ\My9ÒË~ƒ ˆ^ÞÐàûåVðÙƒ³‘¢]×t0’›öºŽžÖþ2¦Ú–{¾†I5ÅB.}¤O‚Ãwn¡ÒA™Ó-iÝú¶ Fw‚S*Îcb¢øª‘¯§®j=—õ¸ÎߧÕÒy]wêà´lIé! FJõØýë6Af.bÉ›XÂ"æ_ÏQ Êò¸@=tÿ®r¦ô·˜fHº__V{‹\åÍÞ: j7€cÓŸŽBãˆ¯ë¸ u'GfÖ'‘’à‘ˆÏë4“ù8Ш;Ât E¹ï„Ê‘Àn}ŸöºÎûXC^Öäµ­/­Ék[Ì°èå*å|úp¸³md³Öiv;PØ•uÛÑÑn?&òqº=úKú¿cÊ»­t•Ù]OÓó²›Ot[)Ëf¬Û}Læãt¾èC~L»õ[Úü¦‘÷ -ñ¯k½/Ä7äïñ é»CüG4!þc÷?B|CþÎÿ1 FˆoÝ?ÂU]ÚÌ—Ï=+,oh£ÿ×õs×zÇ£3-*?lÓ³ì4˜–¢)¡yÐ>¦–Ç4OÏw샎ôGQ -šL¼‹}H‡ ©zs„÷´]Md‰¶ï³}Ͻ[mÞÕ@£DÑ'펾Cɶ÷±QñÂñAû„´Hôs±æ•ntÐ>’îzN"ù–}JÁ¹ÖLßߦàW@™Šîž<ëFëLË&åÛqÆ!‘fd}!4s²c¼{V¨`¬OËÃÓƒ« e>Lw)ŠÃÇô½Ûß/AÜ6ú>« ×uÞ·º`Èåêrˆà/‡WŽ‰|ãA÷ŠEä÷4žM£÷¯.è›™!';šæù8Ã"@ì*˜€¿a¦µmdu€ìôÌâÃ3Ï1‘Óíújˆ$︺°­ô@7kÎã냩TW¼·’pPæÃt´x dÄ7{Wäf—Ü÷ëö¥;"‡mo8º#r쨼–Ïí³—óŽ] i1­Zòû«}>5¤×ÇZï¹W|WÄÈQß¾Ÿ×N Gj­/ˆîm,º]ô+L~Û -Ãb*Ñï+t³èC -ÝzBß-×Ó7‚¾Jçü_hOÃC"ï¡é;Æž„ä;Úž AgØØ4ðl\ËËÈXÚ†ç¶óp/ü”ªGE-o'Q:6áù0ó@òúžûr&Ò­}3,¼Ô†µ½´6”ÎóÛhœœÏQOÒ~ü§ùçŸßšaq´­Ý/°§¸´ª¼t)#­L¼ÌHáÏö°éKÿ˜‹«O zpPæï=pÒõEr‰9Øž´{âÖ¯¼ó˘tní_îÉZ è¿y•_éO[¹.ßèSPÑxaúλ/=]:íÂm7löݪ‘L©gÓré•ç­F<ñú‹<´´zQ,™/ƒdå¥7z«¾QW[]y•*‡ø†±×ìb^Þûc=Ï)e}÷ý¶­ù{›úxíNã1€TxãüŸ/æ3w¾ŒíÒû¸vRsªÕF¨‰„&4Êë÷­4VÓkY‘ŠŸ—¼˜ñhT÷ÍÀÓtûŶÑÅeß@³ k_GKO$CbáõEö›#|Pfwˆo¼ú-@MO߸5ÒŸ¿ øÌ›þ_Ë7úR¾ÉæW<Úêrœeªã¾Ù³ݲÍnwy!¥åj“m. ¸ÚdY´¹¶KDUŠîÈÙ4xË.˲Yy±1*Ç‚£c"Í*K¦[Vy3¸l=÷ÓzpB–³>–уkŠ„2ÃÐûk·ÈNÿ4›ºNYÎáUŠ°5;T©'¦êæsÖÖCFÞ2ºøåŒAÖp-DU¯uvè\⺖iÆ. ºdþbœj¸üåRNߘY 7­†ü¼¾Aã×p1]>pÓZOi.¶¶÷y5ø§5¼,¯ê—µíI^¨ßƒ´{øáõPþvJË9a94ͧ²ì-zÆ•Ó„s=ëãªâ‰û9aéìp~Cr9‹ö·^JƒäõÄ:=­hÚùb-ÿÛ2è¿žôuå5d_Ž’ô ¹Äv1È/'=J” 2³‹ÉJ\¿ú^‡V°ÓžÙÕ×%áb~ýZ{åêw—¯QÚݨ¼41=âEüv¥~piØ B2…k²Þ½wa%ë‹`ÅÑz•º*’šÎšÊõ -y.;*‚»ä@~"Ê5ÑOÍ ²¾ÌÚÊÆþµö1±«>w]˜†¿•yÎ[ħš¥×Ô“ª‡@¶ªZïFßi4)U;UZÉ> ÜÇÉ = âw¾UÍÌΟFMÒºK3$îåj+gžNf¹µ6é#•{©2Dkëù¡Ê0u*½ã ÚÒG+5ëùÒ“xj_·ä¤#ÐÆÚ—Énâñé„)†¯È/'ÉFiXᥙÞL¤©ÞN.~ØË "6]» -®Ùhz‘þ¾@…tª"Ïò%p7)=Ll\FÔ7;yÛI3=uÔa«çQvê®s·Ù¹¿BñFyÝ^º÷؆:uÙDϵ©§ ÕÁcKÝê:Y!¬õOÇÊ™è›hn¥›òˆ<ô”îªâ{c)Læ.½ÞAN;_frî#ôd¸®o¯w²ùF|Z&6~MVÓ/­‘AÖÕÿf>½±‰8( ò€Ñ—…ܼµ£“žvݾ'³fܧÓ_OË4I¢&ȉ‚t:G^áü(ó#>ÿ}âóŸÚÉZ—y<úŽ±cë†<Ïdz,v9HÖc²±»ÍDŒ<Áþ ‡2ÊJ³†>Ç8á5q?€½• ACçÎD}8‘c7Ýhê¹A|zÕ‹=ùÚ±öå’ÚÍ¡Ò3ÙšV¸˜o—eðhLÉç©J\iÑ1¥kþ'A\Ì•ârj°Í6jªAÎô®Wýy -Ò¶¦‹)ŒÐK<Óì\u£-w1ALóÅ\1‡¹ÑºÑÇPúa÷úêq_UN͔ڡ½S_fj¿1¥vÀ³©´\¦íeŸshY=jÎ;z¨×E“ïQr†kŒÐ‚7³NÔ2ÄÖ27ʶi´M.@óÖkÌM§´ŠµnY½”{L1¨{ZæQé(󨤫²Zo›ñz´D=’Ùœó¸;?JöX²´4Kw*’Ôh×mó¶í6Ûž§Ûà¡H‘©brUò!ü{Äw.2¯V"¬RígY eœE©'†L‘–?/âKJA÷° %Ï\¦å’»en¯Ÿ\¯’´õ“¹% ‡c"Ÿƒþ¡[…0ê>cªªý[LŸ,·t_ËCæüÕêÐò—–E¶e‘¨_fMõƒ½EÏ3Ï>˜ËEËJÙ,Ÿ,?ÿ}èÉé -endstream -endobj -462 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 461 0 R -/Resources 4 0 R -/Annots [ 463 0 R 464 0 R 465 0 R 466 0 R 467 0 R 468 0 R 469 0 R 470 0 R 471 0 R 472 0 R 473 0 R 474 0 R 475 0 R 476 0 R 477 0 R 478 0 R 479 0 R 480 0 R 481 0 R 482 0 R 483 0 R 484 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -463 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb56-41) ->> -endobj -464 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb56-42) ->> -endobj -465 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb56-43) ->> -endobj -466 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb56-44) ->> -endobj -467 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb56-45) ->> -endobj -468 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb56-46) ->> -endobj -469 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb56-47) ->> -endobj -470 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb56-48) ->> -endobj -471 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb56-49) ->> -endobj -472 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb56-50) ->> -endobj -473 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb56-51) ->> -endobj -474 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb56-52) ->> -endobj -475 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb56-53) ->> -endobj -476 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb56-54) ->> -endobj -477 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb56-55) ->> -endobj -478 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb56-56) ->> -endobj -479 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb56-57) ->> -endobj -480 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb56-58) ->> -endobj -481 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb56-59) ->> -endobj -482 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb56-60) ->> -endobj -483 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb56-61) ->> -endobj -484 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 340.338585 101.433071 321.588585 ] -/BS << -/W 0 ->> -/Dest (cb57-1) ->> -endobj -485 0 obj -<< -/Filter /FlateDecode -/Length 4634 ->> -stream -xœí]KIn¾×¯¨óÆ Ìaµá×ð‚÷°ðAª‘va´ h}ðß÷ÇÈŒf&ëÑ­¶¦Õšª$ãÁ`ÉȬ”;[üùÁá…œ)¥æDçËçÓ—“59Zü×Zl.ûüÍñì,e]M%ž?3!d“kô9ŸNøßîR6:ýùw§ÿÞöqgojMÞÅóß?2‹‚É%gV˺ °b49û˜VrK¬UcAŒ˜ÉÓ‘•!Õ5Ä ï‹Â*#PváȪÃgG b¸³Ârr8o­Êƒ –(x…ç¢IPzÒxÞ™Tª¯*/ƒç²×x!˜”ÉE§ñªI)Å¢ñ²ÄjÆ‹%:"«ñ …’5^‚,!«ò ‹/9h¼ Y<´S^,°®¢ò ‹MÖi¼L¬…HåUðlÉåÈó6šXB°Ï9a›AåÁ£°I)+<Y’UåA–¬×x²þh<àD ¹Å[={w°Ái¼Y¼Ï¤ò 6_CŸ ‹ÍIƒxa\Ò0Àçl¨zÒ@À—`¨PÑPÀ—j(ç Á€¯ÑPªYÃ`xÞi@,d‰”4$²#¨¼làƒAÃîúˆ%h‘¸ë£õpדOÀP ðǪÙpõt²`5,@¼Ï 0˜ñHK4,ˆÀ]WþŸZWjÒ° w]Áö©¼j\†Æ¿…–K©jX»TAÂÜ| €ñÕkX». Rk<à®óÕjX»È¢†0ãhX€øfœ…骼jl-EÂÜ,y …-@•— ÉiX06y¨† ¸‹ð^5,HÀ]Cа w-!íQy%¯aòðඊO#Y5Ö)4p×:ì°ÆîZ›«† ¸ËaS”™à‹†Ð ö¤B¦æœ4,HÕ‚‡”Oå%ƒë¨aA¶ D@–ʃ,”IÃ$Ö¦†Z4,Ȳï5,Ȳ eа©Ù·Z@˜‹ZdA¦¨aÌ”J¤aAŽ õ´ªñ’7%®U^Ïe r&ƒPì4,@-e -œ^üátIÈž# r…,p ŠLñÈÌ5žƒ,H²5,@¦ žsY jXP|A•™¬†ˆÐ¨%¡jG–ëÌ¢aA¡d8UÔ°n‚JH©ò `BÃìÛQÃ?(6Qæ¨<È"iXPŠçr³hXP -dñÖkXÛäz3kXP-dAƧaAµÉpš¬a`~Q@Vjg®á³îWLêMêþ»ó}w¾0çKü_>¿<Ãù„ÀrÅB[Ëþî|ßï»ó=ÓùT'{^ÊyÅ!¿;ÞwÇûs¼ç¤œ( -M Ä7ÊŸ»ÆEÜlM+éì«ñÉóòZU¾¤ÏíJ”Ö}¤z_ôºÔgÚImæu¯õ!©²>»Ï+É:u”a€Àç¨ëõÁ>¯®W9ï£z}°Ï«Ëš‰’Jävû© -y¯ñ]éò©£Å'm^ëL¡˜¸¿"ì£^QèãÁßÞà)íé:|V2D¥ÝäÇu2ßÛ8¿û|úýÏÿô/ÿüï¿`·Þ}:ýåGk,¶®=ÖbámøàÏËr’µT¬àÇÚ¿ÿôŸçw?ŸlÆŽ§ô~˜—Â30¶æCáé·3þôƒûqýNëìe¡E·~¦õ³Óã"·…£Yú°Ð±íú@_ÛbŠö·µëcûÝ\X1+aÐËV†Ñ¿ÓÅ<­íeŽÛdNS^¹Ž&Ë㺮{]S]ä¢OÛ9x|û?„þÜvݽŸ½¬»Ö©“œ–Ž ‚ës†ubâ±Ö¹±÷ëqœøÖ?Þt§éàýÔ+ä»%Gö×äX÷€ò¢÷n+m<«Êƹ%K€DeìÅVç˜[Û›:Þ{ST½iå…Eʳ-Óšìi•½®ŸI_G³­Þæ“°¿Î·ê’¯ŸÛ­×Ívi§ÿaÊçüMÝ×½~ÚX—þ®ÏG‹o,þ°Ú„ß®›—­£¯¹ÛS^>ÛxWûZ÷yøçð/]׌7±Û]måÿè¶3𧯫leuNù^›žæMYw¶¸rwÝTvëöë˜Òºþñ]‹®aä™L¨-_áGØ|q>ö0$5ù¹}^”ã¬Øx~ÈëÅDžoJ[m-–öQÙQŸuQ͸t!yKág}X‚ô¿þüÿö'¤Ù¼ÖpI—¶ºûˆ0µlÂÕ k2¶òãI» Ha®‡€ºÒoëQ8Å„‹¬Œ?ýt§7¬ƒŸéKéšTwú£ê­l]ZÿŒ»æzP~`Ìö¸<'o74µÐú}7Çâ8b>¹si'ÏÚw©-íõShÏ®§ÉzWûü'”—òÕ•ÞéÏ»Üþõ´OÅÃã -!ôÞ¶ÓXÅnÜœ  2Rföɲ) -d¼k ;‰³HcÇ®}ƒT¶C}·¦5-¸•¦ºÝ»ôà«R×}:=B™Li/(Jµ·ÿÐÆœaÌf§Æz’w;a%$+Î…âò×%¬Çq685 Ü#úrMŽ¯NXc7‹¬k¢òII ö‰F^Û®:™Ö°¶½–|îšÖåË{¿Ž)<`$"ai}ÆÞÞÔ%í×;tØY‘P¶½±ËgO˜£Ÿ Ý>áÙ¬ûFòݬ°ëgדv Ïë 7u“ºyS……ýŠ…›:+{ýö……î_Q`FôSm®cäТÆì§E SoÉlí¡Âäæ˜D»1ŸS<$>íhîáâáÁ>o¦xàá}Ž ð·)¶¾^ñp÷9Åé.Žý¿¾x¸®©·W<¼@û¢xx‘öEñðjÚÅÃm;}yñ âœÜ„œ¢ój¹À§…ñÃ6ðn’Öp-=&züƒÊÊ?]ÚMÚ¼ý -æè:¨yþQp´Õ;@ø`Ÿ¯¹­ñÿ‡ÐÇ-<´ïe(ìE×λVo£¤xä(CÖ6›P¹Ò>mmɾ¶¶óäN_ÂývìÁ [Ϧݽ‰qíÖ¾Ûm±é–Ý›Ì3io ˆ´[yX¬–¬{ÓµF¾ µð–Í–à»ýEtòŽ -Ñ›}°Ï[³YþaÏ ›(ïwŸ;;q=…~¿|?Ðc¿Ó·Òi׎¯[:µöù0yÞ‹vaýûëúIëw_âê·Šr?Êsí­ŸRÞ{û+Vó° ~UÅ¿J¹muõy#V7ÅÎî¦Õ…­µ TÛ¡#=÷§e¢{ôìëºõt“¿2Ûßôù(,k‘ZÂ3×: -½~µíór‹œc 9Þß°Ô2±.°ePõw-õ±>oÎRëm| ;˱¹[ÞrLq´ ÃNkÖõJù\“²IkˆÂc¶,§õ­,µÚülK}°Ï[³ÔêocªÝYÔ‡ù}9ðx™–èúú»tEŽµKQ¼{Z“¾­|ûì<þºÓÍÌd+Ó«œ¹¢÷ÉXŸÃÝüòÁ>oÎ*oç—·ò´}$W£æ¾‚g‡6Ò -hg)Yï3n¢ºìÊÎÆàÍc}ÞÜÎÞÉáº&/"Jî´É¾Ã7Fõ¨E('viµîÃ7+øÇ8ßxiÿ¢ªI[K“}.ôÊú -ö «ÝáK|År;¦ÜÈa%>w eÝŵO’2êv:³1â¹~7Õ=;}¬Ï›³ÓÛÜ!vÈÓñtÒn/Õ'4w¼]&Hõh#û¬ìj<ÛU©Ä:y2CûìOø ߟa·¾_—¬¯?kúeýµŽP|Œü¶•]/&t·É‹9ü2}{Ø„!ù5Ÿ¡œ= £f~3Ƶ£&ÿq’ñ8rZab½WËî9ž¼¨k›~D%î/Žûjë½Å~ÿK>ÒÜîiÕ…îÇ={;`×ûÄòd›o½Ý{scü÷;Çb_Ãa+¿œR{Ë‹koú²|ûÖó«@ω ßZÎõÌo–)6ë[çš²ÝùJÏöÕ/£ÕúNU˯YãWì,/\kû/mÓ?Ÿ\%Ãúoj´÷ÚFÅ=ùéÄï¿$…L4Ÿö¸CfâúÕŽ±È‡G:í\äÚìZ†hìúu>aÂgmËWûj¸ìÝø½$d{£ÐÞ̱L¬Ñ3¹#™Wo[ÉüÑbÂz•†(|Ïiç€$ÛÎgÇ»d¬ Æ#ÑŠé&ÙS—Çñ;§†êÒ$Ñ¥êœØþÞÖg±·ˆ—¦ {ê“Jå7ö¡ºv½´ „ªƒ{˱!4öI!ñ/±Älš°™ƒ/vßÿ´K7$íjo§.'µÝ:Z;•G«2»Ö®ù)Ê4õP†âÚE<…•ù¥ÈÂGlè_·d~¸hìµ-Ânüx9!ÄÐÜ~}wl'VØØMOƒw‹ö2©¹tUÁ5;ßžÃÿ_ "*#Oûâh˜¿k^ÍIØ7y«$IOuHÓ€£<Û -u¹œ‡ÍJ}¹b•ö º£õÐ؆*T&è¹vñ¤xlÉnXÝ 3„uý ¬”DÛ…Àt“‡)?mÈSÎ8\5Ú1YrÂÜãw’ÓÀÎ'IÎcÇ¿ÎlÈ;Æ6߉—ØhOfÓ/}’IövD»9™ NÁ -“> -endobj -487 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 563.554475 101.433071 544.804475 ] -/BS << -/W 0 ->> -/Dest (cb60-1) ->> -endobj -488 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 341.807453 101.433071 323.057453 ] -/BS << -/W 0 ->> -/Dest (cb61-1) ->> -endobj -489 0 obj -<< -/Filter /FlateDecode -/Length 4569 ->> -stream -xœí]Kܸ¾÷¯ès€åòý‚=H‚Ù Œää`׻Ƽ9äïç£Z$KT©Åî™±åͺўV%òãÇâC¢:K|¾Qø/Z%bLÁÛóÃÇÓ§“ÁIü›B,.&õk|ƒ;+iƒp*ùèγÀ’ÓÁŸOø¯»¤ÁOÿÍéßË8ê¬EJ^+wþù‡¬²F„‚tŒ*A%•aTΉ´óŒÊ+[L‰SÁ ¬¶kU€6EÇ©`†Õ:2ª3Œ ʬU fè ,§‚*¹À¨”„JKÉê`ˆ´ÖhF§œð(tÏé´>&X]€NÍéŒ>Xå§KÂ{ï"§³°Å%©8ƒ-NY+9l±&Nça‹ñF²:Ø¢c0œ.ÀÒIŒ. +²:Ø"½Tœ.áR´–Õ%èd q­ÓÒ ‘œN)á€MÃêТPI>0: [¼t‰ÕÁg¤æt¶X|8x™#ÓZunÝFÅélÑ:XV[Pùh[dð ‰ÌKè„MÚr$ £6Úȱ€ŽIØ G:9a} -© ÓŠ##a‹³žc£`‹EËeu°Å$ËqÑ°8.0¶hk8.0`[ ´p\`@·V&ÅqßZ©<Ç„ 3%ÇŒk¢·P. ”8.0à\îä¸À€tÑÜǬkœ×Ю±Ñs\`À»Æ*Éq¬¶À€…QÑp\`Á»FÉÀqï¢iŽ P«8ô+BÇ(9.°à]¥å¸À‚wu@Àê‚@4Xð®v¸®çïj'5Ǽ«­ö(ü#Yo¼ -q¼;[@° ¿‡NŽ ˜ÐpK8.pà]•Ðý1ml%TLžãÞUÕÇê’P%δ[”²PÞ'Ž x•á¸ÀwAŸ‘ãp¼@ª9.pà]eÐSs:ð®ÒIr\àÀ»ðÇ€ƒPp8.@ÿ&”tY]2ÅÈqï‚–4Çè……Œ`V€¤8. ?”ãÞE÷ž8.ðà]éŒá¸Àƒw¥…ÛÃê`‹‰šãø5СÙ2mΪLÁéÀ»R¡†9xWÊ8.ðàÝÜmr\àCèÈq ð‚5Ç>Z‘Bðø$¡ƒËÇê¼Àµã¸ H(‹ÕÁ,Çp¬E2)r\4l1Zs\4lËÀq<‘¼ovx[v|`a Ž ’ô"»É€Æ¡“’イ"Æ›Úr\€v™Ç›‘ã‚d$Æ›â1í6ŸÇ›ã‚da‹ƒ‹Âê` œ¦Ä´[pÆ›`'Nça‹N–ã‚ä}oFŽ àf`¼ žˆy¼Øéð®K`&NÞ…gê9.ÀÆ›ðl¹±½ñºÿŒWÆ<âŒìŒõ:ô­ì”÷¢· -ìœùæ1;i31²Œž5@· -¥’ì´ÿ³+N¸Ì>Ñ)*8ô™1ä›LhÞ}s´`f#Øe~ìçOß¾Eç{þñ?§¿™fÊÊGÁA˜B[øWh:¡º¼o…Û7­Ä$´“…0ðž8Ùˆš‘ -´8Oºe#Ñýåçõ…å0üõ”½©Ù'ôâ—ìAÞZa G _§¹AbñQžÑ`‰Áz[0&éÌN€üJi÷æ ÎV\æ@ë%1ý%ÃV»ó›§oÿüêoùþ5`ôæÃé¿•Òþ ¥‹ß}£òo7ÿ¯Ý|í‘….Œ¿üU9L§³ HkçpEöa™NŽWÒp9î*ã ˆ]Eb©eKÐQjêCFz›`&ÝV-§´JX¢k¡+ñJ«1áô÷Pò|C^És±=ºb+«z;Ržd‰Äû°ŒÕÈ5MFŠMÙ¯»ŽÜ±8‡C®½Ê¡Ûmþب~2ªÈ½ôH^!˜¤YyÞuùG’W×”Wù¸ënéúUs<-Oeð~¹cq‡ÜëNjá¯Þ´o—½4íi÷¼ì ¬zê¾'/HëN¦'ï¬ó3ž8Ä~iÿÊY,ù?tiõÈ÷-ÜU÷¨ }ÁΨŠwpéXœÃ!rÇ5]É–Ò£½×Öð„p-õEÏ÷ö)>Ü—çóyµæ‘ª‰ï©´H2êçh(4×}KegԽݘնsiæ0~ÇÕ83†ß3“Y˜­ùÍRÃeæ}•ï•Å¯žÿئ-O ´Ê§wÒÏÕnd ™‹í™ãm;>ã=ËodFå8>Øórôçó“7JüÀü nÒïÍ - Æ9Bï_Lâ“[g¸Ÿ‘~™­c±Ä'ÌÛuTÚí!v,Îá{ݼyW¾Ø"a–éÝT£·Œr>W‹ë|Öç^_·Ä?tZHg}FÏUDÆ9"í×»¾~ØÖruDµ8sû΢Á8‡CÜu¿rk-‡ ¡{‚æ5ºÍ­GMÝ{ÔñÖæúÑ;¹Þ…ñL^ÁÓvW<·°J²~=:C>çp¨Ü™‡<*§]… !û†ÞýEZIÇ•›üH×­w¼ Æ9w|ÄÃpÆ/¼UÜÑ‹»;öeÆ9JÝ ßXJÐßkcŸEÙjW/7sÒõòý&s÷–¤Gg«Én²ºC¬³I“Yíi÷ÃÖ&÷9¾zÿtÔ­øöÅËpÅæöñø`œÃ¡xgŽó‹î ýzùúän¢ñŽùÌÁ8‡CãÎ|æ­üø„Ÿ…¿‹Œ®v¨žÂÝnE!÷5 Ò"sÞÓþÔ™£ûP»§\F׈™›ô õ¬¬Ü[‘Œs8Ä^÷UuܯÝ/‚¼·L/\Â{îAÞýhBÜ»FüPOæ"G8çhô_Þí„\!㊿ù%Pfⶭ«Öxóúÿ"›˜_ÕµnoÝ|0Îá98¿ymwD~›A˜¿Ì3‘W}µ~…‘™á®º~‡ñ–_»ñü̪%¹vϱ·Çß±r0ÎáPóëƒá dö†é08OéÉœãèhe0Îáûë<å¢eí¥µ…Ê=˜÷¡Gh·¿sc0ÎÑvžÅ¹“ ú§G§Ñ„’«ÕHž7þ[ÆÍík:ƒq‡ÄŸðP¼ñ„úŠÄ™o;ÔçÝÉe§òì÷®v1ë9\zaŽ¼ã½Dƒq‡Ìuo%7ŸÙÙë=\û~®aú-ò²§=]þnÕ|el¼Wƒæ9!µ×tÐVEüeMç,·ž†àFk[¼¸…¾;öCÆ9ú÷Cömú{hŽågö^ÈöηW]ÝñÌö`œ£¡0Žù‰Ÿkí6_ÈõšÆÉsóY–¯ÙÝ@§¹ýÙÃÁ8‡Cç|Ç/Ê+ÏØr6üÜÕZI7ë][Vœ¿N._%g_³÷)K/ÎcÏݾËg0Îá°wÝ;œÊ-Ìó¸¹,ßµ2Ôš”gñèÞw¿»rßÅS?B'#óÕ~ánW»rSVc(S1ÞâÖêÈÄ }úe·OZÛRÇQÌ*Ðè[•6ûë;žÆŒs8TîxÏþÆëÈÜ|²E߆‚[ß«µé7ô¬Ú¿®GöktÜÖíï*xÛZã"Ó»!ã ‘igš>Û¯ÞÏhÅ]tMÐÊÅèxkwöní_©avôN=·®ÕpÏ_¬ø¸[#Üzï]ïoxâ}»Ow<Ë=çpø¼îC®êbcGíþSórwƨ¦ñÀ×÷bî­XâpÉì Q¬Q{vç/6˜“z‹ƒoöŒs8dî¬{¿@_~䙯Âæ1{ÇÛ(ã³×ýÏUë¿ge‡%ÙÇ0Ó•“§Øwëè‹Ï4&}=Ÿ*й|>Qt*’ã6*wƒì o}8G9ââœ?=´üŒ)7BäƒqM<Ãq´)ä³d2”^ýáOü+Òô‚ü.i/——äë‡Ë«lóu.Ž©ŠU©æËKõm,Ô6;—SQç"»T‰œ—XòßËkq3.ƒÚ,×9¬¿|§´æßS:±PÏœ_®þ?¿¶,\ô‹ôÝœ7±gJÑ{(ðM«ªütòÓ¹Hj:Oz Uu><÷ì­0¹nÒ9ŸÅ¥±ªTš -[7bNGª¡æSˆe>˜0Ju9¢˜\Ìá?M•þñ¤’¹ü31\N •“8êrÑħ|b¬eÄÖ -=(ÎÂù§¬iYUVÙø5ˆÔ‡4NÈùgª‰W~jWî&Çw%Z>ÉÇÊÈLgÙ\2f„ˆéÕZœï^ -5‹ó©»Q˜ùÊWSœ/6CØ´4lK°EÜ+#‰Ð­…’d×ÄÚ{T>¥­oÂj:-:Eª¿„ÕTç5áà ٚ^úÈJóYŸ%©RºZV™!F¥*u%d­[ë‰=œá7²$mQ™UOj-Ÿ˜vÆ$[Ã¥Ž=œØpsj®ÖT¨¡b‹šJÉ7SÔM¬ÏȦ2š¥!#NÚˆ4åçRìs ”º–‘àF¯„'t1v%~<¹À„v+¤+УBCÒ­â(^š4Oê]~¡iY>o*ÿ¡ -W¥™y¦ÊVHå3ãÚUË$·Í*^•ûÊ:–+eC KŠK…P1KËKEɄϤ[C×[HI‘yHżLR•<–bUQWÅ™ÂJùT®¤BYŒ@vMì*”âf§«M`/™yEàîjºMì+w>Rq¨5®ò€ÕÞš.Á|>L›íÅú±dÒÄZÖÞ®eF„Í0B7}œÄ¥µVvšÞö>ÿlŒëqN?¦nN§~x£òy¶ÔóRçÿ"Ì+|ÿu²çïËj—~\ËʺrëBL{²|úyçÓЋ°6"Ô–Ð~«Øº‚†ÚkMøÚ¶è²{ ©*ƒªª¥B×ʈu…®V†”Œ»"|èJ±:2O­›Öa_.M¹9dcòÑ{Å*\ÐÛÁel:Óºä3IW9hëÒ¡µšDÄM—$ÃÔ,[˜™I3©EŸí·´*L΋\7"_š¦³­›¥IÙt$\`C/hÂVÑLÓÂÆ軛דë}6Ù—¥•Ú«ô‘Jëei%<^j¦K¿¼¬}Ž9ç‹Òx[ Õ´ i{¦:9­i4×Â..(F‰4¶hsG3µE¶È´t•. “EVS Å&?G+Å2ÝqdK)TŸ¢I¢%íh­ÔBt´Vü*lN·ôxÕ[2ÕrΡÝl!«/™§±«Ô¶¦ÈÖyÛeÞ¥·=“Û°Íß²ž%)ó'4'T‡°ÆÈ €· ¡Hû -’Ôã|Z+-Iö–n+fz¨Ñœ\ÍaæéËi’¨^†> -endobj -491 0 obj -<< -/Filter /FlateDecode -/Length 4479 ->> -stream -xœí]K#Ç ¾ëWèÀåz°^€áC€$ˆ1°HFcy×F0`ç¿Ÿ¥zu7%µfÇ­í]ìJMÖƒÅ"?’%M9jüýÄà¿DF¥”c ãé‡Ã­¢×øSZ, -û+ü‹þh4EåMÉ`‚‹*foc8>ðßêqnþ|øÇïÿ^ö1G«rÖøãOo™ENÅ£ö+ƒ¥XÞ«­+…%æ,± †diËŠƒrò bµI`%ˆá(·eeˆa£!‰1LöQ` 9ŒÕZäAMä¬À3^(=H“„Àx…ˆj%,ðÀ]ã©%p×ج%,ðÀ]ä ^˜ƒ2H$,@|SFÃtE^V:§$aî–¬„ˆÂJ' ‰È‹ -†d$,+"?“™‰+Î$žh@¯Gl 4°Ñ*ŠgàË5’xh1QY¦ ž ¬‚i´xl ¿ÈÇhƒ çÓ§ùˆ*À"bfŠ¼È · d÷½æÍÜ…fç󱟾;|ú„à{üî?‡w¿+'eí¯A‚PZò+¸ÍØnìÛ`ù. 1.™AŠ%„€/éÃBÄ<„4€ÅzèÆB"üñŸãöÍBr΢£7ê¢Àg .¯ëü@¢2:¡(F}`9f‘ïîsSäˆ !û#Ë:T5Ï;t%ÉzwŸ×—%Jh$œËye!o4~ÝÍóÂo„yh6À!dYwöye‘]w—Ì[ÀÛI( €óÄÀ›¸+»³Ë+ -¬QÌ#; hã²åh -ˆD¤iÑø÷oÖ*Åù̾?Nâ#¿3”Ößüpøô/_üý¯_~û{óîðõgZSøüïïê멾¦ó«œùx~·£}ãûú:·ÉÓ8ï–í-ø|Zí¹ -Ç`úFŽ‰×Úõ~f’ç›­L‹>oßøÊ»´–|a¬¸\Së_ä¾KŸÿóøæ‹ÃÞ, ­TKg¨ql”9b^µÍ}]ÞÇ6“Ìy—IÖm6Ó6¯ÍM»av ól[”–[Õ·p½½z2‡¶uOÕÄN“¹=ÕyÞŽ±›‰vóÒ“‰ºå˜w™œ°ž›î9™¹?½¯;ÊfÊŸ*5£ãŠàOá–îìóh†ŠÊþª¥þ*Àê‚G®,íÿã‰,Ô XÔ¤Jľåwöy8 ¥ëXz§ÖX îøoiÁ{xÚë åî8¿³ÏÃYçÏ”|~¤™Á«z܇ˆôSz¹;Òïëóp–z#'ý(ñç—éí”W«²N–6¯ZèÎ>f¡öF.úBܹjm¾ýéwGz;å”üq`J壭ëÖ¹¯ÏÃYçõ<ôÅuï/0Òßíq Ò[M¿³ÏÃYêœô£ÄŸWò°GŽôéþš~gŸ‡³Ð¹hšl»Ø1²âDß¡º»%ß³•Vwà·¨~_Twúþú}gŸG³Dw#çü%Öï?‡w}€îÜýµúÎ>g•7ÎMV-}®µöÂ6Nó–qº|þ¾³ÏÃYéõ<³[ÊJƒ hƒµµö{v¼Í±–¡[Þºzñ[Yô·ø÷nE³Î0zßzhí÷[ài5ÞE 9ã~ Ý×çá,ôÎ3Ï ÓæH¸Á­K–x­^ÞKsŽ¹Îrhm­¢:½à³ö}Íi_®ù8Ö"åûXãÚºÛ7=zŸU>[¬‹6®ZQ:¿^g½&)_\å¯/ ¶KÙ/a%¹û±rgŸ‡³Î9ç:ûUÖËwŽõ^á‹œ?'ßyâ¾³ÏÃYäürµ]«W²ûa˜„…/ÈçoÍuÞ‡/TBkÌì¸8{×&ÿÛVK?Ÿ×^°ÖœiîìópÖz=×üh1í’5ï¨Å×'X Oü $[§y¤!ìvvÑë[ß•ß×çѬÓïÌ=/dïÒ Mߥ ¹cÉ&MÓxmOúñZ²*µw[Kœ~ºÆóå&Æ›ÊZ¾øãŸÿô·i-¿ù\rÊ… ÿ$0Ýöu,e1ƒe;·õ‘=³­ñü¼¡×¶Ýâã”Ͼ=óÛ\gëëÿ1ÎÀeÖã$ƒ«}|}ÕuÌX_ý¹÷kk)ýÌÊ+šìaȽ);°KåiV9kY³ìj5}6=4Æ3Ï~ª³…:M˜Ç|çNv|?NßÕðç8k…Ç3ßnÇ Žx”/(ÙŽ³Ðb(¸&ß©#Ë¡ÇZÊ.µõ±æíÙ3Ùø¢š É†,Œ],2Ÿû—]WÇzš-mXÅÚ¶ê¤[ƒ©m™t[»©ûàjÿÊïV÷TÇœ<€û5½7k+}úÞ^ÕeZ¯·ë0Ôu5¹NuoôùµÌ]õÒ-œª|“NúºOuLiOªܪŸ¶•q§=]zøÐÛ¹ã×Só¢Iyé‰lkeâ5ÝD½ÑM¨vÚôsšöæÂÚšžÖ¨12¸³\¦Ù·kùCC3íGí ¿"XË9Š=<-u_Úö5WÔz±·Ôt*N„:ÞÛêÛÕæZœ¾ 3»ÖY™££Ÿ¼'¼ÿ¾ù«>ëA»¥×ûÝå[¤=Ÿ¸V›I²ù 3¼òHs-#Ç5æˆ1úI‘Â^×mlmGT1æê˜1¯Æ´UÎÙ—n~ L{\ù I¾Ũ½™wíìó¢S}Å„ËwyG“÷0s1á*æ¿.æLs§)99oÂÅ s@ -Ê—½¬&üzuœÐº\é×ǵ.+ð…qý»Ïoô¦TnH -á’T7úû ²ÇÞIýK Z¥©~ǘåòAòÁ]ÑÔ‡ø¡·ë'Wná[^ }¾ Ê ñâJoôçÝ&8ûzÚ§dáq‰]·SŸ§Ý¸:׺pq*xº±T° vß,ï"iu—Ñm¢Ç7Ìe¾Ëi5iñöK@˜B5þJßÃÉ7§\Â}}¤íxÈ7§\ÁCý4‡Âeñ²8f0+O\˜º‘ò!—žôÖÜè‹/º¬Žg7‡qŽ9z±¼ß^3™köÚm1Žd½$³½ÖįØi“ôò ììS4« ¯Ùj±Ù×aßQÞÎ>g«É\³Õ[&u~åëý¦pçZ{…×NŸ–}^n’zóœÕ÷·¿ª÷E]õžï™KÞ¤éÚ¤nŸ7›Ü0Ç:¶W¿Íê²etÁ|Á¹KHË‘¡D¾ìRl±oÏ9‘©Å® çCWeØãûñj®mZLšz]Z¡[âS-ÔKñšÏtÛç¦\+èÍW™¯T•x*ÂûøW6ú|[Ãf+<„r¿)wœê”±| ú1â3¤ˆ´+ñÝxŽLÛ:S”mŽz–«ãì­êmòš/˜åËÏWÍOµýeÓ8˜LŠõ_NdËÍϺ‘Rû5ùùÀ7“@&Ǻ7ÈL¬ou y‚]ÑNݨòº%⨮oÇQ2×ó[ëÛj¸¿oÝøF6Ò­‘+w’'ˆèÌ–Ì«×ÊT2ßžž”«O¡‹ÂÅe'Žin;o’ÓÑo‰zšn-5y ߶ÙU±‹>«ÎLÛßÚÚ8mç5âé€iÝšú,RùÎæ6TÓ®Õæ&¡r§úÖ²oõ}H§ƒ@üD·¡ ›ÙùÓ®ñÍ—gbj†D½]níÒé ¶«£ù¾S±·J£knš¢ Sw©+^ ¦["ÿ:ˆÉG´ko—dþ¡ïµN“ÝØ ñt@ˆ¡ ùùà£Ðڬо™ÞLtÓ¸œô°—A©© -®Ùh|o ÿ† -ß©Œ<å¡nR|÷çx“°ovòRI3=tÔ!I†âh;©ËÄØmvÖn¡=ƒnoÝ5¶ N*›è17ñ¤:x,ɦ[]'3„5ýt¬œ‰º éÙwS~^‡œ¾»ª×}²`&s÷}ÜA;Ÿgrì;nø"×.ow²ùF<•ÀFk2›~j“ ²Õ=ÚÉ&â  0ÈFŸ ¹ykG§r+B};œLŠ¸§Ã÷‡æ¬ó–Oƒ3|/ùœy™ãÑæ üû׎_¶‹0ÏqÜꎶcë‚åL³õÇÈ¥"ÿ/ÇIoâ_?µ‘¿eÇG(ß,ŽOÊßÿƒ’…g -endstream -endobj -492 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 491 0 R -/Resources 4 0 R -/Annots [ 493 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -493 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 240.123057 101.433071 221.373057 ] -/BS << -/W 0 ->> -/Dest (cb64-1) ->> -endobj -494 0 obj -<< -/Filter /FlateDecode -/Length 4604 ->> -stream -xœí]]¯Ü¶}ß_±ÏÂðû(òP -j4EíCÑ{m'(n -$}èßï¡V$GÔhÅÝ{mËi|±ÞÕ IÈÃ3CŠÕYâï+…ÿ¢U"Ƽ=_~<ýt’"8‰SŠÅÁ¤þŸàÎJÚ œJ>ºóY`‚ÉéàÏO'ü×ÒäO§¿ÿæôïeuÖ"%¯•;ÿü>«¬!† £JPIe•s"í<£òJàSâT0ëíZ`†MÑq*˜aµŽŒ* cƒ2kU‚:(Ë©`†J.0*%a‡ÒR²:"­5šÑ)'<*Ýs:­„I'V SAs:c„V9Åé’ðÞ»Èé,lqI*Nç`‹SÖJN[¬‰ÓyØb¼‘¬¶è § °E£v£‹°芬¶H/§KF¸­eu :C\ë´tÂEc$§SJ8`Ó°:ô(4’ŒNÃ/]bu°Å©9-œ<áLˆ‘é­:÷n#ât¶h,«ƒ-h|Ž´‡-2xŽЃDæ%ŽtÂ&m9ÐÑmäX@Ç$l†£œ°>ŽŒTÐiÅ‘°ÅYÏ1Q°Å¢ç²:Øb’å¸ÀhØ‚[´5°­Z8.0 [+“â¸À€o­TžãÂ…™’ãÆ5Ñ[Ž (PJp®wr\`@ºèîãÖ5ÎkŽ h×Øè9.0à]c•ä¸À‚wANV[`Ž ÀŽÂ¨h8.°à]£dà¸À‚wÑ4ÇhUzŽ €¡c”Xð®ŽÒr\`Á»:À°º РǼ«.óÜà]í¤æ¸À‚wµÕžãE€$ €wA!Ž` –ãø{è´á¸… °„ãÞU îéÓ`+¡bò8ð®Šh>V—„ -¨q¦ß¢–…ò>q\àÀ»pT†ãÞ}FŽ ÀñUs\àÀ»ÊÀSs:ð®ÒIr\àÀ»ˆÇ€ƒP8.€Jº¬. ™bä¸ÀwAKšãxa!#˜„Õ )Ž @ÆB"å¸ÀƒwáÞǼ+1xð®´{Xl1Qs\€¸:t[¦O#XRƒ)8xW*´0§ïJǼ›Ý&Ç>dŽÀ@(Xs\à£)ÏqO:„|¬Î ;Ž ‚„À²Xl±Ár\€ÀZ$“"ÇAã5ÇAÄ  "I!úf‡°paǶ Rä¸L&b²–ã‚à<Æc¨UN絈tÍê"t*p\‚pŊ㌥DD§ç¸/r¸ÄqÌ€NYŽ B‚-è*À‰¨™s:[ds\€H:¥8.ˆ¶ ä¸ êˆQ¦—ÀCc,‰ªætVæqfä¸ Z/r¨Èqº Fš`JV[@ }ò`Ûq\€Á›æ°:Øbœå¸ F‡›‘ã‚a‹–šã`37ÇIÂD|$éE“9.C'%ÇIEŒ7µå¸ý27#ÇÉHŒ71Äcúm2>7Çɇ…ÕÁM‰é·à$Œ7ÁNœÎÃ,ÇÉû<ÞŒ ÌÀx=;%óx3°Óà]—ÀLœ¼‹ÈÔs\€Œ7Ùrc{ âuñ¯ŒyÄÙ êuð­ì”÷Â[vÎ@‚|ó‰4€™YFÏÎÀ­B©$;m Á¿ˆÇ슮³OtŠÊ>3†|‘ ÝÂ"º¯cŽ–Ìl$»Îýüýéë7p¾çïÿsúð›i¦¬ü)Sj‹ø -Ý@'4·Cô­pùÆ£¡•˜„v²>’'R3RçI·l$Ü_þw^ÿXXÿ›Noêé¼øõôèpÀxÊ­1|æ‰ÅwfyAƒóIàm=Ò˜¤3;rˆ+¥]$þÝë€:[q­‡Ä|øK †­vç×?ž¾þó«¿ýåÛï£×Nÿø­DøÍW -ßÖ^¿]˜ß¢ÿ ¤àwœûè£ùß6¨­Öy+/ÌåoqOEmÇ/îÍr §$tȧü\šëÎuh­=‰ÚÈã´…d6ò1Ï¿ïát,Ïápz;|[ùEÚLÍø•][v8æ|OÚ´ÆH’mú3³Äʈ¯«ý€†Lÿ³¤ïÔoÓ]ß»kȧôë‘€IZ¡•ûþl,ÏáÐt;ʺåόԣ¤G”"-J}[ßê–åä|•ÅRkyEQðaYFÏz«ÁÀ—²:DgFÝ„ìEk´ÇÔžÖ³ä;f uYÚ–ëwË?ëE #u0ÏѪoG^=§õhËÜ`Ôí4¹µ ZVçÏÜ*Ú3-Ý·"A÷s¢°)ö6W„rß+;¶†ÀÔ–UŽœë¡¡üiÔ—o¨Äiü6Çò‰;‘âgžè¨<Òyå…®ä+½¦xÈž§:däèay圛ˆíÑE¼û½/«ylD–·Q½Üfó¹·cÇ#OßÝBõ³QE®¥Gò -Á4µËzêÑÈù‚:îº,¯c5'3âý[ÌiCxï÷;–çpÈÝ™ 4:èH“xiêi÷¢ ¬rŽÌ¨'{YŽþtqòBI˜ÌÆEú½Y¡Á<‡Cèã÷Ó‡øäÞîäŸ_fïàëHL˜ÒQi··*|,ÏÑëvV4Þ‹¼¿µè®½g”ó©z\³¾ôýuGâC§…tÖgôÜFäXžÃ!ò˽¿~ØÞrsDµ8wÿÚ¢Á<‡CÜØ#1ý½v6„® zóøZÉÍûGoåzÆ EÏ[]ñÒÀ*ÉýëÑòÁ<‡CåÎ<äQ9íÆ(lÙwx÷ÒK:®ÜâGOï[®xÌs4$úñ0œñ ïxqÿÀºÌÁ<‡Cé`ÜXjÐÊÕ&«µãe«U½ÜÌÉÀS3«•RÞ|2‡ÎNWû.Œ­ÝšNõîù¨[ñíG¯Ã »ûÇãƒy‡â9ÎϺ‚öËåëG»‰Ææ3ó;ó™÷òã3V`|þ.2z°Cõ”îŽréSu7Ÿó½´sOëSgŽ^¬CíŸÜ¼GÈܤOhgeåÞÉÁ³=²Ö{¸õç·F.>E^Ö´§ë÷VË×QÆÆ{5è9'¤öú‚Ú«H¼¬éœåÖÓÜhm‹7ЗX9˜çhèKƒë!û>}È54ÇŠ3û(d{åÛ -xf{0ÏáP8'~ªU´Û|!o¿róY–/9Ý@§»ÿÙÃÁ<‡Cç3bÇÏÊ+/Øs6âÜÕ½’nÖ»ö¬8œ\¾M0αfS/Îc/Þ¿Êg0Ïá°w;:œê-Ìó¸¹.߶:ÔšÔg‰èÞu¿»zßÅS?B'#óÕzánU{ç¦_7¹SC£Å­»#ƒöå—Õ>imKG1wFߪDüõwóÎÁ¡Ë;óD§"Ùh¢Bq7ÉòÖÛR”ÍÎù¯‡–Ÿ1刼%¬‰g„z6…¼‹J†Ò«?üé%Pš^ùþzËøÚE¯¯ƒœ^·ì¯Ç¹J'Ȩ›ëë#m,Næê`ÜD=¹Ù®’ó`"__•«ùÚ|ÓmלÖ_?SYóï©œX`6Ÿ/OöäüùýpÕ/Êwó¹‰=SŽ\ƒ+×°jÊŸN~ÚHM»ÂI7¥ó¶±go…Ém“Îy¢(U¥éÔTÙ꼑sÚl÷§šjÞWæ-ùòvL×ÍyÉÁœþ§©Ñ<©dE®ÿL ×½2å$Žº4ñÓ)ï•j±ÅˆxPœ…óOY˲ª$¬² ø5ˆÔ§4NÈùgª…W~jW®&çw%[ÞÃÆÊ’ÈL»¸\OÌ‘Ó«µ8_½jçýf£0󑯦8_l†°hiÚV`˸+VF¡[ %9]k[ìQy²Zu¾ «é´êiþ’VÒœ·„—Nkzé+Í»\–¢JíjYe†•ªÔ•”µAlm'Ft91¯d)Ú¢1«ž´ZÞ+ì*ŒH¶¦K%#ºœØtsi®¶T¨©bËšJÍ7SÔM¬ÏȦ:š¥!o Múˆ4åçRìs ”¶–‘àF¯„—\Œ]‰ŸN.0©]ã -é -ô¨Ðr«8ʆ—&Íýõºf‘å–òÿWªpUš™gú¡l…TÞ-­µ“ä¾YÅËJ¢r_YÇr5 lhiIu©*fi}!šbÒgÒ­©k-¤¤Êˆ<¤b^&©JK±ª¨«âLa¥~*WR¡,FàtMì*”Ÿâf§«]`/'óŠÀÝÕr›ØWî|¢âP[\å­ïª½µ\‚ù"¼LŽÍöâ ýXNÒÄZVo×NF„Í0B7}šÄ¥·VvšÞ?ÿlŒó¸—Ó§ÉÍiãtÂoTÞÉ•F^êü_¤y…Ï¿NöümÙ:ìêǵ¬l¡+·.ÄÔ“å}¿ã 8ï^„µÛ¡¶„ö›XÅ–Ö4T?®5ákÛ²ËFì%¥ª ªª– -]«C"ÖºZR3î†ðÒÕb džÛ6Ía_M¹8œÆäMçŠU8 —ƒÃØt¦¹ä3)G9iséÐZM2 CrÂÔ,[˜™IOR«>ÛoiS˜|.r@ˆ|hšÎ67K‹.²é*H„À†Ђ­¢'M {£î^Ïn÷Ùd_z@–Vj¯Ò'*­–¥•ðx©™ýò°úsÎ¥ó¶ªeÒ÷L rZ×h¡…]PŒilÙ掞ÔÙâ¤ÅQTº€LY-1›üœ­TËtÅ‘­¥PcŠ&ýÖ´£­R+ÑÑVñ«´¹Üâñj´djä(œC»:ÙRÖX222OsW©m)M‘­Ïm—ç.ÞöL.Ã6C|‹LeËæV)â¿GúÎ5O7Á¥*Ó숊m³M8j«‚Ñæœr¸½‡Ø¼v°‘L—ÜçöüÉz–¤ÌŸÐ3¡9„5F¼m˜ðEÚW¢žæ}JùlI²—t_0ÓCæäjvÈ3O_N“Dõ0ä¡¢KÓ†Õ2Ƽa{Š.š¦P.‹é“éïÖݶf -endstream -endobj -495 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 494 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -496 0 obj -<< -/Filter /FlateDecode -/Length 4520 ->> -stream -xœí]KäȾç¯È³‘ãAÆ0æ°À®ámìÝÃ`Ýé.‹cü÷ýQR<$1•Êìšvv{ªP•)2 ù‘© Ù³Áï7ÿÙ!¥/?ž~:™!²ÁÏXbq1²¿Ç_ä³5¶9$>ÿ(‡˜ÙÅp~=áßê²/þzúß_þº¬cÏnÈ98Ëç¿}ù!¦ +¬ –±^a11: -+ØCÌYcA ŽähËŠƒrb1ȹ¤°Äð­ß²2ÄpÑ’Æ‚6sTXÖ@ëŒQyÄy§ð,JÏÙ!¤ì²Ê‹àÙè4ž÷CˆdÙj¼<„8i<‚,œÕx YØYȧ¨ñdñÁ•Y\Š^ãEÈâ ¬ðdu%•YL0Vãe?pND*/ƒgRL[ž3 ´lÊAÃîÚ„éSyy°WüZlYÃî"Py ¸ øLãDT§aw­G¤ÖxÀ]ë²Ñ°€»ÈXØÃ`‘hX€ø6XÓUyy09% ¸ Xr -&IT^`HVÀñ`‡jX€»ïYÂÜ5콆¸kiʃ,>9 ×€·U|Éê`BãwÅ k<à®11kX€»65,Q.iXdÁNÂhÈ1 B6à!åSyaÀ5kX  ÈRy…"iX€ÄzÈ>' ¢ƒ,Þ9 ¢ƒ,H4,@F0d‹ì[]@˜‹º> È‚LQà Ù2‘†‘ÖcÐªÆ nHp­òx6jX# ÅV쥆§×°?Hº¤aÄÏ’†1C¸Š†ECrÈÌ5ž…,H²5,@¦ žµ$Y jX\Â*3 ¡±–„ª5Yg& …ARE à&Xi)UdLhX€ù‘Å6kX€Å›Xæ¨<Èâ™4,HÉÉr3iXdqÆiXÛ”õfÔ° È‚ŒOÂl i²†€qðŒÑ° Û„õ¦# à—²ÞLdo°ÞÄOñÛ샬7£†™ #EQyISVü˜„õ&ÐIãÈâ2iXCõfÒ°iÖ›ÈèÕ-$ëͨnw9™4p™iаXo"³ÕÖöÀËù™ÎL²âLêŽô2b«ºe`€½ˆVQÝ30_Y#©›+ËÔ]„U0­Q· ðùm0aÚ}ê·¨Ì!"f¦(ƒÌp Bv_×­˜¿RlÚûÛŸO¿~à{þóßO/¿wÊʯE‚0–&äWp—1ÝŒìÛbøMCïfü%„€Ô!bnBZÀâ¼é&B"üÉÏyûf!9ÑQë¢ {)I6õ$[',Š±>p3›Èw×¹)rD†ÏõÕemªêûmºÒd½»Î›Ë*°€%4ξ_ˆ&ä­Âo;ù²Øœm/ËxD74º< qÀ]™ëUÞP`ƒ•1R­€2>; MÀ,* - -ÿÇ»Pj–bÚ¯—øH–,¢Žãó»O¿þýwÿóÇ?|É|÷rúá7+Æo¿±òz™^™§W‡Vd‹–㈖<¡Ó˲õ¼(›–t*×/ËkŽ­žôÅnY®´ÃfUÏëåè}7à‡é½™Ë[3ý•~k{¼ê§k£ŽU§ènlon¿ÊßñF™B¹þöÿÎï¾;ý绥‘ËŽÉg% Gª#d×.Uù»ü9Ì1ç=s¬*þЦ¤NQèx+3)êÍ}Zßþ6SïNýÎôÖ:}™N¾µË,Ìéý$g©[ÛJK¹Ör—6«Ë®ê¯õT^«NèX;馥¹?Õ8åC™bjˆ‰Ù$'¹Ø®u¬ól扅ñž}næâeùZí²ØÂèíÅná¢ç×6.ú|÷ó{·-ivÙCÖç°µ^ž°jwã_:l"7K ,)û¾e«ót–I»Èùsò:ÛÔYIriÛ^­ó±³fžy×f9_i«Œéeie·’ŒÞ¢*b[ÅKÖQdœpk¯F‡^ÅKÞoëì iK(Ûì±:Og³ûÉçÆû×I_™Ã+ÉÚ”\d‚>q¶Ç52¹¹\ÞA§äãµ|ïgÛYec?·lºØž_¶y—_]CÒ–NRæ%ÙßÙ·ÊcužÎ*÷sÐõl×™KmÆú¼ïvì/wí¬Pæ«Eè»ô¨[¥3÷cåÁ:Ïf•n?ó\g”¶›éµÅ}Ýøõ€‡v–ΗOõÈ+–Úe•r/\*Ü´ÔcužÎRod¢ÿxuÅ#W–ö¯ñÄ+Ú円1Xߊðë<…ÞØô|dƒòÜù%3P<íŠu¦ûWòëÚìZïò¼¯ºOâyž_¢ú}Qø$þ`§³Ä9ç׸~ÿ9¼ë3Dpâû×êëŽÑÂJ™—ÎÚk†0õ7es¹ŸxæÛ¹ýM¾’-èºOf­Ÿ±­Sýÿ¤íé»z•æ~»qqÆtr”1{ŠÓkûbóZ[êU/Óu-èÂÅ̤ã—:\Ïc•)4ߨóf›üÕ3u=¹µžê8Ëœw6ˮɭɴD°=ôZ#×Ê·¸ÓEZŽÏÚÝñZg•¥_ßÚºÍüx‰œc‰Ž»ËÖyèt•7 â\Ï[ʘa®úW0ÅlÖ [Ü¢ xÓ$\í0¤5rFáªÃVKÔš$”¯Yî·ë|8à"*íòË·7jSö ášT7ês2cî´úc¨]¥>| ÍñÌlâàw4õ9¾Hu`üäÇìä0¯´/ǸBy!^éú2{ÀïÞNû”<.Bê¾r÷5àý¾ÖɨŠS™–, -jú)œ?,ƒí"ò×’šmÒ #g9‚tÕéèíW€0G[@½;vóÆÁ:O²¨ix˜xÍûƒÍì}iáC»®¯~é¡¦Û Z{o¡/nœXm÷m6w®,›ûÅW©ëÝdMåú‹³n·;³ ÊY¢ìÙÞ:ÍëX•'±Úî49KôºÙÞmEßÏûEFßÌr°jb›ºs«o¹¡´’ÛãÙÎvuºÜGѤ<ÑÃ't`[”C05›v'4µsjì´ôñ%åÌÝb?Ïe -²vËŽšnÏˈ’îŽHº47OtW—ˆJ‡.¥µm‰æÞÏ~))ûªýePC9Â"»žÊŸNa<ÐÕŽ‡z›ëä©gäÊ^æ&ŸåÙd<Ù2uvT¶=_©9>+å§ZJþƒdäDu9MwœÚþb.ÿÓ8é?ž¬l¾˜É¦§G˜‘Œ`ÌkòëIuA -™¨m.Ü q~kj[dë^E¡]€ qÈë’ˆf~Û64$„Lo—ÑH}.ÕäR2¥áœ:Vˆ¨ì–,£7ƒÉò¸4øù*TQ$-­ÄÖ õe[ƒ­âM²õ¦#ò–hºîÙQ‘ÇÊñÒUu¡«è½êl7ý¥¬‹Ýtî/'të×ÔW•*)(Mí:Si¾*W*—’uB¨Î“Bºœâ7¦4M˜ÌÊïfMŽzžˆ©Õr¹”SH—“ZnnëLÅZ*µª¹h¾‰ÒLݧªx…vé6ý¢<ÿ¨óãËÛ%Yö²ê\›ÔÙÛ/'„Ú_O•ÒÜ°Âp1½žè»v+9™f/*§{Nïàš…&åÊÿ *¸RyÆ7–ªIÉa×íªu"¾YÉK%õôPQ‡4 XŠ­l§.cµÙ^_6¥¼€n-]5¶ v*ëè1ñ¤*x,ɶZ]% „ýT¬ì‰¦î™«)¿.ÈMN®®Ê¦vlgî\ÛmäP±óµ'Ç:ãVN.¯òÖv;›/ÄËØhMÓO¥“Fv¦F»ÖYGl‚¹ÁèëH.ÞZÑi<Íh~ÛœL‹¸—Ó_Nc˜sžì=¬ìãòZ{þÊ|‡¿ÿ?Ñùåäç)Ž;SÑÂUl]ûH&mJÉò§B¬nÓu°ßÈ6µ²\¬¡Æqç:¼¦VÝ4`/%mEP[¹=‘›;²«¦ë¬ï4Ã;ÄËJ‹5‘ùÔ¹i{ºôep²‹&g†©pÑ—©ñ| Éç®I\IÑÒÁ%×UÄEß(.»s“l!¦dßIU½ÈOýTxé«»èÒ¹ôG-ÌöMÚ8Š®R`ß_ô “í;Í y¡v¯Ož÷YäP<@¨Ú+õµ§Ö µžNõãeX^Ö˜ãÇ-§â¼MCµ-ßùž¯INs–ZÐ⢷юšZµ¹÷R¡-:-¢Âh¦Ðj‹±ÈæjE-㈓ª¥XsŠFýK¯iîg¥*‘ûY ›²Òn‰x5[ò5âÞœci%k.™ZèkW*µ’¾Ð¶}Ó²ïmÏÝ0¨ ZfjZ5Þ”|ÿñ©Îj'B+U~Çݛڳ¨ÉFïüøøK“B†<Á(Rê¶K¿²Ý%)û'}O˜Ž¼7ÑʃLu^ÁÈ·]Så!§zµlÔ!ݧe{¨ÁœÙìO²wÞÆM¢ze©(®ô’ô&yÞV4V¿çC>Åõ‹í“ñ÷Ÿp ð¿ -endstream -endobj -497 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 496 0 R -/Resources 4 0 R -/Annots [ 498 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -498 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 176.373057 101.433071 157.623057 ] -/BS << -/W 0 ->> -/Dest (cb67-1) ->> -endobj -499 0 obj -<< -/Filter /FlateDecode -/Length 4623 ->> -stream -xœí]ˮ丑ÝçWäÚ@Ë|_ÀÀ žã(Ø Ã‹ª¬®6·h{áß÷¡$R!*¤dæ½U¥öt]d¥AR!òð0¢’úªð÷Æ‘ôc -ž®·/?]Ԝ¿1ÅêdT‡OpW­( N'ÝõÇ,°aÉ™à¯/üלòä/—?ÿêòë<új†”¼Ñîú÷ﳊìbÊ ª•ÒVP97„`œT^¸Å”$Ìp mUfPŠNRÁ 2& -ª3,m·ª3LÐ$©`†N.*­`‡6J‰:¢ˆ¬tÚ •î%уÉ$Q ÓÁH:kH;-éÒà½wQÒlqIiIç`‹ÓDJÒÁ²1H:[¬·JÔÁƒ•t¶ÔNt¶]QÔÁå•–tÉ.E"Q— S1Ä­Î(7¸h­’tZØ´¢= -䃠3°Å+—DlqVIga áOÒ'œ 1 -½ÕäÞm•Õ’ÎÁc‰:Ø‚Æ—ÀxØ¢‚—(=hȼ$q€ a dH"í@‘¢Ä&¦B° ˜äò)H<`•†Îh‰¬‚-Ž¼ÄVÃBÏu°Å&’¸ÀØ‚X [ Y‰ ,Ø–€‰ ,è–TÒXð-)í%.° \˜©$.°`\=I\`A¹€R’¸À‚s-¸Sâ ÒEwX°®uÞH\`A»–¢—¸À‚w-i%qw­ENQ[`‰ ÀŽƒÕÑJ\@à]«U¸€À»èFFâ´êz‰ €•ÁĨ$. 𮉊$. ð® D]ЭÄÞ5· Üà]㔑¸€À»†Œ—¸@À?JôÀ» '»°+qÆ{茕¸… n‰Ä¼«†?¡Oƒ­“—¸ÀwuDó‰º4è€ú-jyÐÞ'‰ x••¸ÀwAŸQâpü€ÕH\àÀ»Úb¤–tà]m’’¸Àwá78‰ ‡AøãÛ  +êÒ RŒ8ð.hÉH\€QxPL"ê i‰ @ƃ‚*qïbxOxð®rÖJ\àÁ»Šàöˆ:Øb£‘¸~ tè¶BŸ†³:(¦tà]¥ÑÂ’¼«THxðn6%.ð! L”¸ð‚Ä>ÒBðø¤ ƒË'êü€s'qAP8P–¨ƒ-Hâ8ÖC²)J\ l±ÆH\ lË q<‚!ixßâô¶.âü€` ‰ ’òCv“%.C§”ÄIGÌ7 I\€~™ç›Qâ‚d曘â ý6YŸç›Aâ‚D°ÅÁEu°NSú-8 óM°“¤ó°Å$’¸ yŸç›Q⸘o£C1Ï7ƒ.ïºf’tà]x¦^âœ`¾ ÏVšÛ+¯ ðÏdeÌ3Î(F ¨×alC -Ü‹Ñ*ˆ1òÍs$1h31³Œ^Œ`X…R+1l À¿ðÇhà Sô‰‡¨<à0fÆo2¡[¼û:çX’ÙdS|ìï?\~ýƒïõ‡\>ýjŒ”•? aLMÀ3ºIhnï[ãö­XÝ0 -i´>“'Òb¤-ÎA·l$†¿üïº=XYÿîè·ï.èºSØÐç²1kò9RütD´Ð»/¿þö?ÿ_üUþîÓå/ÿ‘§èJQTà5|ò7Ξdh$ÈþŽ ŸOSŠ¿ùëõÝ·üzhH4¡Ãа¾^¾„þ8%ÿÝ»±Öl­µçcª5ð’јù£ÿÅ!_ -£«è³¼¦žçêÅ-À7Àýaþl2—¢ƒÀ V´©s*Ÿª¢žjëŠÕ¹10w'ãÆ&øïoÿô?`MàÜo¾ÑùÛ4ßaúVvúÆ<N%>ï§ã|þT95éòyÈí1çù°èŒaéìüù8Ó|œËOK:ú>Ãa>žïæs ™ï£¤¿Íç~¶ÞÚʪMª)³\74e‡’^Æ\†BAP°â¦2—‚®3ÏIP·˜ f­k’æVr·¹æÓ}Z·`O×Ö ë2²®´hmA½èéöåWy¾gÈr󵺑ðà½êé»–¯ÖyžGäRFµãýRiá:›‘AÉÜEj_žÓ!Õó£mÔäÜA[mZZB×!ò^ã¶q48Öcº‘¥å²¾Rƒy©}yN‡ÔxÌ©ªAÔ‡å8×àŠ/ý4º¾}+íØÑ ½"⥧öÑ—µ¯öžRÎǦ¾hñLÖ6í rñ f0Ê{×½ìÌs6TšcÿòÈOkGrqÔló|¶IÃQ@ R‚œ§  ràNËó8ßtæ9]ËÞñáJMÞØ(ÙÔfî;æ6nuäl[O³Vš‘óØÜ×0é3qî£ÏæÿÀf5~4ž·òB<öµ*j~qïw[8åÀ‡åü\šëÎ5h­=‰Û(ãtñÆ(‡fb~¢z§}yN‡Ócn3vhÖf³¿]Ú¾ÊKcOë RÚb¤õÊvdzf–Ú3ÖÕ~À=G¡ÿë;õÛ6÷÷qòú´Ùa=æ1)ê —tæ9šŽ½¬£ñlÅH-JZDiÖ¢|lk[ýÓºœq†XX,±Y.GÁ§u-ëmæ’_ÊjuorÏ[ã=¦ö´–%? -s­ÛÚ¶\¿»ã3Òõ"µ/ÏÙjïDöNkÑ6ÝôqšÜÚ9Hkògnã…–n[‘¡û5^Øè{Û ¡Ò÷ÆŽ½Y0·¥E•c×zj6/#Ñr¯/?"ãcÍC$væ9ïxŠ_9ÖQy¤•Wº’¯ôš&j·¹‡rÍ[òÊ5wÛ¢‹î+ô6¼Ü•Q½ƒÜ'¢yN‡Ücßñ̼#T¿Uì^Z$oÌýPZ×S‹Fi,¨ó®æ Í&&Ó1úÛÅç¤ Tz¹}yN‡Ü;Ñ@Û ƒÏ4Ù(ÍGÚ{ÞÀô¸¶i¥v$oŸjøp£wa·×³žùÄ~mÿÆ_dQˆUYOYÜ¢žE7žñGmz‚Kûòœ ‘tÇmâ"µöøèµ7CaÅk}5ò½ÿ™?‘i·‰fšÏ‹Tb¾'Z2©hò¡C¤væ9RýÕ¤#äGƃœ×öø¥y2M#ðó>ݹQ¾ïµì ’Å$“Ü89¿‡Ê¾<§Cå_ôŠm·òûzP½3+úy |™á ¾ìËs:dÞñ5ßræÒò‡f7oh†Žq¥Îi¼ƒÒÅ—ìžuæ9J]ßJÇÏ>÷þnP÷ r¹Ç·~~äÌãóòÎ<§Cá±o9®$)ëh¥¨6_ïêçtR‹ sø»sf…Ù‹o–.‘÷ÍuŸ\Ïq×(=€óiã´±Ú½™Œc±ÈÞ‘¹3ÏéÐvÇg|æ ‹¨œÇ{[Žþr~òBÃã+Á;óœ¡Ï?Oïâ“G#ÜoÈ?ÿž½c±Ì'Ì/~š¨»‡Ø¾Ë£!|MÐûç×Jî>?ú ¶«0ÞÈ+xÝꊷövPÉž_÷FÈ;óœ•wâgå´ƒYX²Ý?K/i¸r—ùsëÎoyΆÄpÇG< gü›÷Š'FñðĺÌÎ<§Ci§ßXjÔæ “ÍÚq†²Íª^)rÒñÖÌf¥ƒR‡oæðètµï&ØÚ¬éÔ_º ß~ö:ÜA1=>ïÌs:߉q~Õ´?_¾~¹»h|"žÙ™çth¼Ï|”_±ã‹ðw‘ñ§€ ªÇt”Ëߪ;|Ï÷¶\{\Ÿ:sôjjûæ^ç3¢Àb“>¡5©{O$;óœ ±ñØWm߯•Z÷« ï½0 -—tÌžg÷<š·Ã®?4²Xd/;óœ_ßmß…Ü ãÀßü(³qßÖMo|øùÿ"ÿ0ÿÐvÔDîÞsóÎ<§Cdg|óhuDþ½Ê0„w"}µö £ᮺv…ñž_»óþ̦'¹åÞbmO|bdgžÓ¡æ—wÃWÈlß ÿÚ= 3NY̱w¶Ò™çlˆM¿Ä)W=ë^Y{¨ìXƒùÚw~”Å{Ÿ4væ9B＋ó$´oŽ³ ­6O#eÞøÿØ3vH?ÓéÌs:$Þñ Oůѹ¯Èl‘Ùþn`^\V*Ï~ïf³™Ó¥ÏÌ‘Oü&QgžÓ!óÎso­vßÙîYëÝÝúó¯F®>E^Ö´§é{¯åë,cçw5ø5G¤¶ú‚Þ«˜¿lxÌrïmi¶¶Ç‹{è{b=dgž³¡O«Î‘m§>å"šs9š­²¿ô­Âð»yó€àÐbÎåÍ[¢Ó‘íEP±w7ɨmw.˜¯ž7\Ð; È¼k¨WD¥7ÚØÛÀ|?Å §_›~_jüýF?çêÇ]Æ”é÷¨ÆÆæžšÚ3Ç\ÍSU*[b×å%Ô¸‰@†âÇÍiýôËšÇrbñ(çëeï1çÏoü…I¿*ßÍ×föм¡A½WîaÓ”?]ü¸iŒ7S>ÚäE¯ž›Û&]óF5QYÒ¥éôXÙúº“sÜõ§šjÞ¢Uå]ÛòŽ=cÓò“9ýOc£ÿxщ†\ÿ™ ¦íÕ(Ž¦œ,â—KÞN“1aˆígá|¨jY¤KÂ*»]òO˧6¥uƒšS-¦ Bäôz+Îw¯=‹ó–¤q°ó™¯¦8_l†p)xÚ¥À%ã]±¶Š ÝV¨Ø屡bÎ[Xժ󋰚ΫN³æ/iM`Íy$¼]pYÛJ_DiÞ±Ujר*³Ì¨T¥®¤¬ BµÑí"¿Q¥hBcV=kµ¼Ô$ŒHTÓ¥’NÝ.bº¹4W[*ÔTqÉšJÍ/¦,P·±V¼ ëh–†¼Ç2ë#ʖõØç(m­"ÃÙo 1´¿\\R»…+”+ÐãBËʭ⨼,Òü^èt„®Ydy3žüÿD®J3óŒš*¤ò†ZËÙr‘Ü7«x]I\î+ëTyK½‘¢ì*fy}騄ô™tkêZc+)«2&©˜—Iª’ÇZ¬+êª8SX©ŸÊ•\¨Š¸Ü"vÊ/+ñb§«]`/óšÁÝÕr±¯ÜùÂÅ¡¶¸Î»£U{k¹ óEx6jÅú±\dUG»åbL¸RXÄ ¾ŒâÒ[+;åŸËáÒɤ÷vùÛeæŒu&áÀ[7ûäž—¾þi¾Åç/týCÙ]jǪla*·®Ä|$Ë[CÇNqÞ*ºk·aBCŒö±ŽKZWÐPÇñüKé¡‘åìj!ö’RWÕUË…n©C&6ºF[V3î@xkj±:2¯m›eÀžNm¹9\Ææ}ÉŠU8á·ƒÓ¸èì2$_Y‘8ËI—!Z2,#Nx¡8eL‹e+33 ùEjÕgû‰7…Í×b'ÌȧvÑÑ2Ìò¢‹l¼ –.°å'¼`Òü¢ie`ôÓÝëÕí>›ìKÈÒJíUúÂ¥õƲ´ž,µã©_ŸÖ1ÇŽ?í_:ïRCµ,Ëúž­NÎÒ5ׂV'£L—lsÇ/JE¶ºh(*]@¦Š¬–ŠM~ÎVªe¼ã(ÖR¨>Å"ý¯iÇ[¥V¢ã­â7iÇÝŠÌ,²âõp8‡åîÔ’²ú’Qyž»JiIi‹l{mZ_»Œ¶Wv´âÏT-ÙÜ&åSü÷Lßw”R•¿1:¢ã²#jÒÁ{MÙ݉ރC(ï’(²pÉÃyŽã'Û(I‰Ÿð+¡9²V ¼í˜ðEº¯`E½Ì[YÊÙ’oé± -ÂC Ím÷‡´ƒã•c¨ž†> -endobj -501 0 obj -<< -/Filter /FlateDecode -/Length 4435 ->> -stream -xœí]]¯Û¸}÷¯ðó«åÇð (ö¡@[4è tŠ>Ü8¹»(n -dûпßCJ$GÒX–›Ô7»¹plÍÔpxxfHË’>*ü}«ñ_$=Ę‚§ãéÃáãA Á)ü+%fEý¯àŽZQœN>ºã‡,°aÉ™àOü·8äÅŸ?~sø÷¼Ž>š!%o´;þò>«È!† œ JP)m•sCÆyAåõ€.¦$©`† dh­ -0ƒRt’ -f1QPE˜a)h»V%˜a‚&I3trAPi;´QJÔÁEd Ónðpº—tF>&“D]€N#é¬| í´¤Kƒ÷ÞEIG°Å%¥%ƒ-N)I[ÈÆ é£ÄàøÕH\àÀ»Ú"RK:ð®6II\àÀ»ÈœÄ€Ã ‘H\€ø6h芺4¨£Ä¼ Z2 -*‚ID]$-qÈxPÈC%.ðà]„÷$qï*g­Ä¼«i¨ƒ-6‰ ×@‡i+Ìi$«ƒ2` -IÞU#,éÀ»J…$qïæ°)qY`¢Ä0p@l$.𑆂—¸À'R>Qç;‰ ‚‚À²Dl¡@ ±’MQâ‚``‹5Fâ‚`` R‰  I#û—°p×[)J\&b"’¸ 8õ¼*é¼b]‹º„@B±–¸k©!bÒK\‚rº$qÌ€N“Ä!ÁL‰ Š†h™K: [dK\€L:­%.ˆ¶ ”¸ šˆU¦W Bc- WK:Ry%.ˆä‡œ*J\€i‚•&˜RÔÁЄÄŸ¼Øv`ñƒÅ&–9¢¶XGÄhòr3J\#l1ÊH\læõf¸ )Ø‚ŒO₤üÓd‰ @ãÐ)%qAÒëMC`^æõf”¸ Y…õ&–x¼MÖçõf¸ lqHQDlAÒ”„y NÂzì$éF+NwŸø•bf ¹“ Ó‚Ý·5G/fÏ÷Ç~ùéðÝ‚ïñ§ÿ¿);eõO#A(¥ ù¦In‡ì[£ûM‡˜3Ie aà-u²!u#5hqÚtËF"üåÇõ‡™å0üu9½m§Oˆâãé1‰“B† `E,_—_Yå VX!Úz”±ÉdväW*šþý›€:Y1î¶Cf>â%ÃdÜñ͇Ãwyõ÷¿þð0zóxøÇïõý·ïÎŽïtšŽýt§c7½§µ ƒ¨È/Ê…ñ]Mí¢HyU9=ÎÏStvÞF³%1;ÃÜ®\Þ3›N“,õúõ\µŒÉ¶˜E»µ/†•¯õ™®ÙVë±¾Ó[Á7¼³S»¬ûþŸÇ7¯x3\ÉB ऩ1ｿ€Ñ}U>£ŸšXÒn@ó*š h™ mØP>tH¹8þäžkÚ, «SÀ™ÅûrZÅéåÆ×LŽaFŠ‰ábr?}—÷ß+Õåår®É6‘·³Î½Ak -ì¿eÿ½›|ù¶ûÐæO;½Þ->/ü~O~1κõ že¬*UMMŸ3Æ—u–4]èsÙþÛ©í´¶¥ö­—õ­Í‘EÙb›{ZÉt6»šwÖ¹;TÒ&#Re¸Ó|ô>% m!s…€Àõ -ìE@eϳIÃ’UÕ åË:Õw³DìuŦ–”œA¤»'÷Õ¹;Dn§ÍËoYšSG‰¥E+î - ¹Ä¸Ó -£owŽþÆ/ÓÒR†§m‹Y³Ì Cak+ÎíZq(ÍûJ‹úK?Õ÷æ“ÓÆ<¡ø†¿ÓfïéàîUÎÎ:w‡Ïír5‹ØÙpY±°±Ì( µ` )†7–]â…ïÕX’pÉYëK`Ûs.Ïm6ÉÌix¶˜‘@)ï²l"sg{C¦ÙÎ0?G,ÞÅéó®À^D,c¶ç’û1»¯ÎÝav;ÿ\ÍþeÞWÇðL¾¶‡%gžVaN`&#¬nŸÁ>·´ïaÂÎÿk—à“ö|’’œÑ”·ä·Q¹¯ÎÝ¡òÂæb´ÛÈÅ>b<ï»»Ô Ù>|f†¾ÊgPoàÊ}uî•Û™ç2£¼f“üëâ¯fè³nèËHµ,«Ì—cJùKHÝYçÞj/d¢¿ -¾:3#Ïì¿Ù™x¡,‡T4”`})Âï¬sw½°ïyËå-¼ó[f Ì´3ètׯäwÖ¹;t~¦üó…fÏ:ã¾D¤gùåîH¿¯ÎÝ!õBNú"ùçëô¤®¿*ig{C(]ÈEo䫯ïø-ÒïŽôÄ÷2w~Û¹³ÎÝ¡s;½yÝûFú[®¨úÜ‘žÜõkúuî©rÒÉ?Ï4Ãî9ÒÇë×ô;ëÜBwæ¢üZ úýœï£W¼Îò¼ÂWì›x7ÀoQýº¨înø&~g{C¢»s~ë÷Ï1»¾@wöúµúÎ:w‡Ê {ž/’kži6íŒàüZ‘ª7®5Y^Öòà ßrº¾{ßYçîPzáúÏåï+…åug¦rÉ:{ýÛ3^ÄmQïðzÜ¿¢Yf­îN?Ôòû8çãõu‡gÊ~´¡ûêÜB¯ÜódœÆ#኷Î!qk½¼ƒ—^rŽ¹ÌrV×ÖŸ‰êþ†ïÚwÖ¹74ú}¹æ³\å¨?箵?wÝüì꼌¢é·ggÛYöIÊ…kío‰—~3wŽ+½½ž+wÖ¹;t^È9—¹Ø¯r½|e[Ÿ´"<ƒÈ~_´³ÎÝ!òB~¹‰æÕìþ&“¸ð†|_5×tK>³ºÿŸ¿ŸAë {š;ëÜZ/übý¥rÚ94ïX‹/w°f3ñ‹Î ¡ç‘š0Úɧ. sg{CgØ™{žÉÞ¥š6Jgr±‹¿e«cÒn+“û’†²ö®}i‡ðà`Éç{¶†|Ë[mµ‰¥/¯þøç?ýmùûü¼/ÉraÝòBØÕú2··ÇlßXÖ…<;k?Çã•|*+¯G==Žï¹Þ˜L²Ó$¯¶p}m)ç:UÞžt“-õÜåX×1ãöŒÈßåÖÈݺ¾ââÆž¼Ÿä'f©Só}8µ®ÛïÍêÔQñÝóuDJ™e;½ã‡é¸ØÛ¼¸ßKQ-½¤ÀÞ†Xo;›šU›Ót6?ÆÏÚ§0ødóý£§öíÔsçó9Foï¾[·ãm¾¹Z¾ïº5ycôTñÖ–Ñž³Cõ¾Ì¼xÊý9B°-ßÈÙ+2> m—y™ÆúMS[|¾u˜÷“/ªMnŠ¨±ûÚù ajwò_GÛTV°¿Ï©>ŸœW3Ä•¶¬lŸÅ±A즓Zú¡Ø8ÙPûSd’}®Û㦲±ë—6ÕQÚáü’úg©ß´ð5g€ÜŽÖ½Ïõ|ņǩ>ÃJõ]ž/6¿Ÿ¾Ù³é;³ô]™Ùf>^ÎvžåÐ¥_k™GÆí-'mÔ§§Ñ/c¶Æ|ç™/¸~bÚÅåòô0Çc)ËúR²%ÅÚ®ý¨œPÇÁlú‹VX[Œ«®ç¸€‘<Þ®Î{5úAÙ¹—ØköW,3¼{ØFâι·ÄöF‹~¥²nÓ_^š›×Çäù<â‘°qqÅê6·óxÖ–ŸÏ{ΑuÎsÙrÔ^èvöÓ¨M®xþYìc~Ü>¿Fþ¼0` ÔsBªÁ -©èëév†Á¡5çòíd£Ó‘ݱ­ .¹°XßK±Þ‘ð˜ÿ–9~ï šÌÏ1±ñˆœžRÈ·þ”òà¯sœ4mŸýȹvÂRž×-_NS×}^y¬Í߉Ã*Ö -ï{6†i”›–c0žòëØ[Î÷0 Éiέý ^m}¨¿O«¡üxðå6¶ºÜÊ\ù8h“Ÿurô4äÒ1ß:7*Kº.ÎÖÇ35Ëb>¶RÓCcT¾|¾‡ðøDv0•ÿXýÒL|M,ËTGSºøéðA‚˜¨g§ÄY8}T­-,^ÍBvÂB7 iYÒºzN–[W?W{“ë»Z-ßx•T-dË­GÇ BÔôz-νWƒžÄù!)q°Ó‘o¦8_m†°7H¼lo°W¼(ÖV1¡[ ;]ªöè|Síæ:ß…Ítî:͆¿–5 ç–ðtÀiíRú$Jó£jSÕ»F5™eF¥&uµdjã$ˆNAø­ªM³éÙ¨å\ÂXD­\ªåÑé –›Zsm¤B+{ÕT=ßMéP·±9^ت1ä§>±9‚ôÞÔqàâ¼jc­"ÃY O„Z‰Ÿ.¥]ç -å*ô¸Ð²v›8ªŽ—.Í÷Æ?ajVY¾=pþ¤ -פ™yÊM Rùßý¨Ÿ$ÏÍ&ž;‰Ë}c’< )ô²Ì]:„†Yî/•P>“n+Ý<6“2—1yHÕ¼LR<æbÝP×ęªWr¡ªFàt]씟fân§kS`¯'óšÁݵv»Ø7î|ââÐF\çûµ7{[» óUx*–â ýXOÒÅFµh×OÆ„Ý0Bw}*â:[;•›~Mû$“"îéðó¡„9cIøà­Îá™—>þe^áõ¯¨÷»ã¸Q-LãÖ™˜G²ü°ª¸Sœ^U…mÚ0¡!Fû]¬c/ë*Z7†ñ5õêª{-©ƒê¦åB×}ÈĦA×hË<ã6„§…["ó©cÓöxhkçp›ï”^­Âïc×Ù’¬Iå¢=¤CK†UÄo‡ì„©[633’Ÿ¤¹>ÛO|(l>;`iD>´]G=Ìò¦«¬ô‚A -lùo˜4?išÙ#}óôúäqŸLöudi£ö&}âÒÖ±,m„'Km9ôóÃsì1ÔÉÛ=ÔÚ²lîÙ–äô©ÑS špŒ2iìÕ¦ŽŸ”ªlvÒ(]@¦ª¬µªM~ªVÝRzE/…–StéÏÜÓŽJs¢ã£âWes»5âµlɶÈq8‡Þ;ÕK¶\2 -2Ïk7)õ’¶ÊÖç¦ù¹k´=²nP7Ä÷ÌTõjnUò&þ»eîŒu;R©úWvGtìOˆ@ &¬±å¡Ÿ*z¡üܦ@‘m—\]g{ÿd½KR÷Oø™0Y«‚Îo•Mx‚"]V°¦ê£]åjI‰]ºÎÂöP§9µÚ*Ïï+ß#—M¢vòR1?®Óæ¤7槌¥å¯M}ùÊ”oŸ”¿ÿ²!&¹ -endstream -endobj -502 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 501 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -503 0 obj -<< -/Filter /FlateDecode -/Length 4760 ->> -stream -xœÝ]ÉŽäÈ‘½ÇWÄY@S¾˜o€Ð‡FÂ4Fƒi 0::TEuJ²´æ0¿?Ïô…Nã‘YUlU"+ƒf¾ÝŸ?3s2H}UøùNã¿Hzˆ1O×ÛÏ—_.jNáßXbv0ªÄopW­( N'ÝõgØ0„äLð×× þëÛ⯗?ýæòy}5CJÞhwýçO¬";„‚r‚*A¥´TÎ !ç•×N1%I3\ CKU€”¢“T0ƒŒ‰‚* KAÛ¥*Á 4I*˜¡“ ‚J+Ø¡R¢†("kvƒÇ {Igôàc2IÔèt0’ÎÚÁÒNKº4xï]”t[\RZÒ9Øâ4‘’t°…l ’ÎÃë­u°ÅÄ`%]€-£“]„-@Wu°Ey¥%]²ƒK‘HÔ%èT q©3Ê .Z«$Öƒ6­¨ÃŠÂ$ù è lñÊ%Q[œUFÒYØBø‘tà gCŒÂj5¼º­²ZÒ9ØbL Q[0ù[Tð` ÌK˜J†$0Ñ)J,`b(+Ñ€In Ÿ‚ÄVi茖ˆÀ*ØâÈKL`5l!¬\Q[l"‰ ¬-( qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ],÷ qëZçÄ´k)z‰ ,x×’Vx×ZÔu°–¸ì8X­ÄÞµZ‰ ¼‹ed$.À¬À¡—¸XLŒJâHâïš ê€5h%. ð®q8ÉsƒwSFâï2^âeÿ(1ï‚BœÀ¬Äð÷Ð+q ‰ xW'¸?aMƒ­“—¸ÀwuÄô‰º4è€Ö-FyÐÞ'‰ xŽÊJ\àÀ» Ï(q8~€G58ð®¶ðÔ’¼«MR8ð.â'qà0hÀ¿ Zº¢. *Å(qï‚–ŒÄðƒŠ`QIK\2âP‰ ‰ ’ò‡É€Æ¡SJ₤#òMC`]r¾%.HV!ßDŠ'¬Ûd=ç›Aâ‚D°Å!Du°ASÖ-8 ù&ØIÒyØbI\¼ç|3J\€0ù&"zqK r¾Äíð®K`&IÞEdê%.ÀòMD¶Rn¯@¼. >“•‘3Î(î(P¯ƒo· ¸Þ*ˆ{ -äË9’¸i3‘YF/îÀ­B©•¸m À¿ˆÇhÁ ÷ݧv‹Ê>3>É„eAˆîKÎQ‹Ù•b÷ý±þõòÛp¾×¿þïåå7ãNYþÑ¿%ωµã%b¦t5ŒÍLÄÚ†ýûž¨Â&„TMÔ ÅiËM„#†³M×òAÏ †½l±F(3žµÝ"OИ(ëEK­òî–jL¼0"·¶_¬DÉƽ»ÖYþw]~Ø1y¹ÐoÀ2F Ž N4öh¥w6Úæþp )ãî9o…ÓFjw[C˵1ùÁ*ï…„–G)› {$Ð r E³Âÿöáršº¿ï{—Cm]¶~”0ÆF]?ü|ùíþð?ÿõÇÑ퇗˟·e"üï¿ÓøL·û_§ïrg§c¿¹~øa£?°¯JœšÌûã.ÔÔ ZU§®Ò$ßnÖØöÅAX6ë^¾ß© ÇÙ¼÷+6íTÇ¢GÖ§P>¢ÿ]jfb³«ÿPÝ#¯D?u鯇Ӽ™íàÝü¸ø~øýüá¿›Åg"ZüÄ­røE8Â÷Ï-á½àû±cË_¸Ì4µSÃÛ ‰7ºNÇEþùûlóŒÒžNK¨d4q(·Iƒë¼…¸¿ òå– Tïó?®°H/B³2}WæÖ•Õ+åûºú©ùœëPWvjƒÏC­0C–Zï±ÝÛÓ؛뚩_«§ãPuoÔú˜Ç£Œ³ÈÇöPz¬ÎIPZÍMm ôa„ýÔèn_ÇÙp¼|·Î†…éÂ霻÷2cC^ 4±`­Y2!!+r1è®Ã-&4”2^’ý‘Ôö˜ð`“`¬0¡ñv c……L÷·cÑט!ýñþy!Ÿ~‹œºr|xF¦:Ÿ–1M9;ý~žþÒô™Ûo½õ׊›~×°\n3÷Û1÷³™@A|ÛNŠw(¶Qw¬ÎIPWÍŽquvŽ¶ÃùÊbãÈ·>±ó' ž1­éüKncLm2Bò ê†]?-g~V§aÕ…æ$ÕÏ<‚3¾ôwïúþ³´bG‡ö‚ôÆ‹ç•6ÚG_×¾²zr;Ÿ»ñ¢™ÌmZAeñò&»_¬s:TnÇ—Ïf×ky |¶(Ó¢€:¤¹NF@áÀµ™ Oðͱ:§›Ù.ä­ñ’ÝhòÚ1·y¦º˜=ÝÌÒ„®cy­½4»fÏÖÿÔd5~Ž´¶ná…¼·²Â=µ¿¸+ÈÍœ²öüœÊcç:´–•ÔÚ(ã´FcÄ<#ߧ¸‡ÓcuΆSÚŽà¾C7s6ÅÛyñä{úHÒ#}T¶êϺ,õˆ¯kwf{‹Íú£f픿¶;¿Ï÷¨O™õ¨‰˜Û/9XçthÚŽ²¶üÙŒ‘z”ôˆÒÍŒ¶¾­Ÿõ—y;c†˜Y,5Yn‹‚—y=ë-ò)†ïvžKú²ž‡ìEkíŠ)+­gÉÏB®u›Û6^°\ñÏÔîÒEê±:§CêÎÎ^Çi=ÚÆM7½]†g›÷i«×ŒfZ¸¤üQØ{Û;B¥¿ ;v®±ŒezT¹¦¯§²ù$¶QßxÇ›·‘x¬Îé¸)~㽎Â#Wžér½¼jº]»Å9ä>oÇ×_wY öÑ«ˆÔÈìŽÊ¨^Aî»ëœ ¹n;v<óުߌªæ\z$/ÜÆ¡4§’/(yWw…f±'sÀû»sRˆƒ¶Ä·m#÷`Ó!wg7Ðvèh3ÍÆK·žv/¸_±íf©÷äýUnŒ.ì²?뛘ØÏí_Ä‹Í.Ĭ­«,›îQßìn<:zœKÖ9"wâÑn_¤Œ^ë½Ö2”†£ÚQŸy¾¿ò+2ò»™æ #µ‰=µ’Š†o¼ßFê±:§Cêv¼zi£‡üÜð`ËkküÒ]™¦ ü|Lwn”¯G-+¨lö$“;vÏâÁ:gC¥ß‰E÷Pôèl¼WÜwÕk÷<þ*P.#Ó›Çùò`Ó!s'Ö|ÏÌåÊÊnÞÑÝ=–çAð -Jk,y8#:Xçt(=v§ãϽ¿‚û;{Ÿ@î#çøÞ×|xVçlh ;1ã3Wàš•óÄ`ïËÑ_/N–Ìãw‚¬s:„>=ýìß¾ú×\+ˆmbB~œÊø…è=Ä«s:ÄîÜÑøÄ·²¾ô7UÊr¾ÖŠëbÖ÷¾¾šøЙA9òŒžmD«s:Dþz¯¯ŸvµlfT+ˆKß[t°ÎÙ}+¦¿–#÷}|þ^ÉÕëGŸÔò.ŒwŠ -ÞvwÅ{G2*csýúèùÁ:§CåÎ>äY9m# ;„ì¼ûY%W®ñcl¯[¼ãí`Ó!q'F< gü‹¯Š'¼x|â¾ÌƒuN‡ÒƒqcAR‹o˜,îoP¶¸«WÚ99ð­™ÅJm~3§Ý.öÝ[»{:õç·£nÁ·_| WPœÏÇÖ9ŠÓÎç7½ƒö×Ë×Ï w é‰ý̃uN‡ÆýÌGùñ w`|þβö*`‡ê±Üí¶ßªÛüžï­ö=ÞŸ:qôì>Ôþ›{¯¥foÒ'̳¦Ý§L¬s:ÄnǪý÷k¥Ùý&ÈkžÇ·ø^fûŒÉ'÷<šP÷€]GâÐÔìEFà±:§Cà·CûïB.±o~ ”Ù¸nëb5>|ý‘5>ä××DMäö®›¬s6Djupƒsëö~ |˜~…/Enký%Fa‹»èú[Œ×Û•/Ð,–’«çðäÍ=?N#ì¿–!:›§Œ¬ìÙF~ÌàòñÎWÝ=Nyþ$B4Éï´ñŠ@“RàGè¯=‡Ðüt÷ƒ÷o1ß¿ã:>CÂßyÊÆ;s´šžBwÿNìøüÂ1b›†rœ>â{üÎåîþ3«EÏ2äèfô%\ÖßËS_ýÔNÌS<õÇ~‡ëó]‡ÓScgí»©ïÆšž«XÎÁåsXLå/?¾B¯R>Úð;¯žËs“®ü -Š¨,éÅϬ{n'B*•ÆÝë›Öà¨Ikì•ß -¤¢÷àâ÷ŸŠÍvÉÃu¶÷O–»$yÿ¤í Ó1µ*hàmÅ„W(Ò¾¢iêuzI\-)ñ”a{¨ÒœZìïÁ÷ÇM¢r8Uä×Þ[z#¿­7(-n_Êò³í“ñçÿÚu› -endstream -endobj -504 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 503 0 R -/Resources 4 0 R -/Annots [ 505 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -505 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb70-1) ->> -endobj -506 0 obj -<< -/Filter /FlateDecode -/Length 4338 ->> -stream -xœí]Kܸ¾÷¯èó«å£ø‚=H‚Ù Éa‘ø½ÞE0àÍ!?%‘,I¥–ºgÆ‘´»UÅ¢ŠäǯŠlµ¤Ï -ßjüIw1¦àé|yúpR]p -ÿú“ƒ^ý#^Áµ¢Ð9|tç÷Y`C’3ÁŸOøovÈ‹?žþñÍéßS}6]JÞhwþíç¬"Û…‚r‚*A¥´TÎu!ç•×š˜’¤‚.¡¥*À JÑI*¸AÆDAᆥ íR•à† š$ÜÐÉA¥üÐF)QG‘5‚N»Î£Ó½¤3ºó1™$êt:Igmçi§%]ê¼÷.J:‚/.)-é|qšHI:øB6Içá‹õV‰:øbb°’.ÀƒÞI‚. +Š:ø¢¼Ò’.ÙÎ¥H$êt*†¸Ôå:­U’NëΛVÔaFa|t¾xå’¨ƒ/Î*#é,|!üI:ð„³!Fa¶š<»­²ZÒ9øbL Q_0ø_Tð`u™—$0!t” I$`¢í(R”XÀÄÔQV¢“\G>‰¬ÒÐ-UðÅ‘—˜ÀjøB˜¹¢¾ØDX_P@âká‹!+qÛÐ"qÝ’JZâ ¾%¥½Ä„ 7•ÄŒk£'‰ ,(PJXp®wJ\`Aº˜îAâ ֵΉ ,h×RôXð®%­$. 𮵰uð–¸ìØY­ÄÞµZ‰ ¼‹id$.À¨vÀ¡—¸XéLŒJâHâ êB‡9h%. ð®qh‚¹Á»Æ)#qw /q€Ò”˜ €wA!NNà VâÄ{茕¸•ui‰Ä¼«Ÿ0§ÁVŽÉK\àÀ»:bøD]êt@ ó½Üiï“ļ‹@e%.pà]Ðg”¸ß!¢‰ xW[DjIÞÕ&)‰ xyƒ“¸pè4‰ ß:­]Q—:•b”¸ÀwAKFâDáNE0‰¨ €¤%.w -y¨Ä¼‹ðž$.ðà]嬕¸Àƒw!íuðÅF#qòè0m…9dµSL!éÀ»Jc„%xW©$.ðàÝ6%.ð! L”¸vÈ‚Ä>R—Bðø¤ CÊ'ê|‡c'qAP8P–¨ƒ/Hâ$Ö]²)J\ |±ÆH\ |AÊ q2‚.idßâò¾.âú€à 2E‰ Àd]LDç±C¯J:oº@×¢.B§ƒÄ!P‡P¬%.ÀZª‹˜ô€à»œ.I\7 Ó$qAHðSEâ„¢.dæ’NÃ$Ù Ó…Nk‰ ¢/È%.ˆ&b•é•ÄˆÐXK¢«%©¼ÎŒDò]N%.À4ÁJL)êà hBâŒO^l;‰ °øÁbËQ_¬#‰ b4y¹%.ˆ¾e$.6óz3H\|AÆ'qAR¾Ëi²Ä qè”’¸ éˆõ¦!‰ 0/óz3J\¬ÂzK¨£Ãh=dî#^b1LÆ_¿?}÷—Wÿë?F¯ß~úRô³R.~ÿ­ÎŸÝø>»ñØi& ³2~x×¹ÌLG H¢±\‘½›Ö“íJ.—çõù™Íìœ _³-6f,Sê¢iÊŽ6jx•º}}»ÌêŠÍ¶Öë§ïåüÕwÍÚ®†þ¯çºdÙ÷ÿ<¿~uúÃë)ûÌtaÞìDÆœyí*n÷™<·/W,s¯Á•ûé0˜<„f*šA°òÅžéJ¹jÇá÷F€·aC¨Ý¨»#©®0mSV[u­AR/¡õÇBe¨w2Öz6US Ùb±™3uÏ óúߌu§¥/u¥–m«sdV¶÷ƒÍ½Õxï`Ä}6‡CåFÖXî2½§Ä´kÈ\ 4»[P°…=Wó†9«ªY0”Ïmˆ¯Ûfk;®ë}ÚXÇXu;Oî´9"íõ ²öò–éÌöTzÝœ»ØʳÅèÛ£e„ÅÕ;ÏÜf³fž†ÂZWœúµàPš¶•föó~*ïµOÊîÀ|¯Húõ8nÙ^âÞÎN›Ãáóz¹‹Yì\ûjNZilíÕ:.òxóñ½K.9k} ¬q6÷/V˜“g‹ ”ÌÖwˆ;m‡Ì/½_ –ygç“È€e̲/±wcvŸÍá0{=ÿ\Ìþ{¾EÙ`Iñ'0Ó•oLžâß­ß¾ø.Á -“RË')¹Îõ_Ël r§ÍÑPI»˜³Ñ®#·v¹Ë×½w™¡oêÇTÚÛ¹r§ÍáPy=óœg”·ì“^üuÇ }Ö=ý¤²¬rïÕl;m‡ÔLô‹à«•¹qÉÇ™‰+e9äÞk5vÚ¡ûž÷ìQÞÃ;_3ƒÝ׸9uûJ~§ÍÑÐé^(ÿüD3ƒgq!Ò;{ûuë;m‡Ôœô“äŸÏ?Ò»{~ôò? z„ÞÿS k¼sóõ_#ýþHo¿š}§ÍáÐy=½{ÝûFú{®¨zéHïÕíkú6GCªßÈI?Iþy¦vàHïííkú6‡CèÎ\”_ 4û½Bá þ{µž¯Ø7ñn¯Qý¶¨îïø&~§Íḑs~Žë÷—˜]#‚ßñó6‡CåÆžç'É5Ï4›vFp~­È‰ ÕW®5™_Vóào9Ãß½ï´9JÃÆõŸóß)ÌV -óëÎLá ’´zýÛ3^Ä}Qoñz·E3Ï0ªíÎ~(å÷#pÊÇËëWÊ~´¡ûl‡Ð÷<§ñH¸à­5$^[/ïà¥O9Çœg9‹këW¢z¸ã»ö6‡Cã¾\óY®rÔOGãÚµök×ÍO®ÎË({¶ZϼMR¾(\kOŒØúÍÜ*WÆ;¸rŸÍáйóWéÔzöË[/ßXדV„+7Úºã÷E;mŽ†È¸‘_ÎF¢öê•ìþ.»r÷[òA~ÕÜÍw_:ü/àWÐzÇžæN›Ã¡uãëŸ*§=åNL³Þd&~Ô´‚ΖGjÂh'œÚBç>›Ã¡sgî¹’½K;4u”Vr±Íß²•1©·tÍmI]¿ö.m©‡èÁÎ’ÏÏKIzmµ‰}[^ýñÏúÛü÷ùy_’åÂÃ/AàSmÇÔ×3²oCYòÌ,mŽò±¬¼ôônxÏvCF0»±®KáäQ7ž+—Ï÷ Ú¹[ l³Î]¤ïêFË»ñ§ÒJ§¦{djôV7r‹û3ú’·ðùÌÊдLlÇ—c-°­¼õÙFͬªš–§|ÞÒ~lºŸ´9„Î'›ŸŒ4¶ÙŽ£ä|>Ç0ZýH½]Öãm¾mx~4Ͳ5¶khsßî‚ïÇû‡ŠœË÷‡NGv»ãJ4›E6xeysär‹ásþ›GkªÌ&³ñŒ¤‹Ð<ëÄÉ•»Ëüœ'RYÜá3šk0tÖÝØ$,¯t³Aþ† Ÿ&N™HùÝz6yÒ 7µkÙyÑ=ó0çe€Ÿ¨}ýn<7ó§LÊÚ†òÓ²´Ê'ßß—^÷Ï&Q>vÚ䇗=u6M:ç{áGeI—¡Ó}gëóŠeÿÈ·µÔø8• “ -0<"ŽŒå?ôƒþ3ž:_æxÿÄ&Õ‹£)MüxÊOì"ALÔˆbCœ…ãGUëBD43ÙÑ3ti^ÒºrGmFNÖ•Æ•Öd{WÌòÔI•B¶¿—øpbAK¯—âÜzÕéQœŸz;;ùêŠóÅg[…Ä˶ -›á¦X[Å„n)TìtMl¨ø£óS2j×ù&¬®ó®ÓløKYØp^^N8­KEi~ÖR©ªô®QUf™S©J])Y„ê8 ¢ËI~«JÕ„Á¬z6jù‰ƒ0 Q-—J9At9‰åÆÚ\©PKÅfšJÏ7WÔm¬/È.,€‡üG6G”-§bŸG ŒµŠ 7f!¼œbh!~<¹ ”v+”+ÐãBËê­â¨^š4ßpgø„©Ydù~ÿùÿ*\•fæé?hªÊÏìhGí$ynVñ´“¸ÜWÖ!©4…V–u—¡b–÷—ŽJ(ŸI·–®=6‘².còŠ{™¤*yLź¢®Š3…•þ©\É…ª8Ó5±«P~œˆ›Ÿ®NU€½œÌkwWëmb_¹ó‘‹CqÀRý­õ2Ìá¥l4gèÇr’&6ªF»v2&lŽš¸Ñèc/.³µ²SÇÎñc›dRĽœ~=õaÎXg>x«óóÄxæ¥ÏÿE™WxýëDçÊ,†8nTe S¹u"æ‘,?}2îç§Qa6LhˆÑ~ëØʺ‚†Ça|MÍ\5b/%ueP]µ\èZ2±©Ð5Ú²žqW„—Y/ÖDæ©cÓöphKãp›}R¼ÂocÓÙ’ϬJå¢-¤CK†â€WŠCvÂÔ<›¸™ÉOR»>ûO|(l>;`iD>´MG-Ìòª‹¬o+‚Øò^1i~Ò4ñGpúîéõäq]öedi¥ö*}äÒÚ°,­„'Km觇5æØs>(“·õP­Ë²¹gk’Ó¦FK-hrÀ1ʤ±™?)Ùä¤%PTº€LY­1ŸühVº¥oq{)Ôœ¢Iå=íø¨ÔNt|Tü¢l®·D¼š-Ùš9çÐZ§ZÉšKFAæ¹u•R+i‹lynšž»DÛ3k5G|ËLU3s‹’wñß=sg°™íDH¥Ê_¿;¢c{ä5é`íŸâ­¢÷àÊb ÙvÉÍ6×÷O–»$eÿ„Ÿ ÃÑ‘µ*èü> -endobj -508 0 obj -<< -/Filter /FlateDecode -/Length 4801 ->> -stream -xœÝ]ËŽ%¹qÝ߯¸k“âû faÀ2<° а†ÕÕS„#/üû>ÁL’‘ÌÈGUW·î¨ Õ73‚ yxÁ¼ÅÔw…Ÿï4þKNO)åÜýù—Û¯75E¯ð¯¤XÝõOøþ®•‹“×9$ÿ…6N1{Ãýó ÿ ·<ùçÛŸ~wûŸu}7SÎÁhÿÛϤrvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS@§IgôR6YÔEèt4’ÎÚ)D§½–ty -!ø$élñYiIça‹×Î)I[œMQÒØbƒU¢¶˜­¤‹°Å w² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Å´Õå'Ÿ¬U’NëÉ›VÔaFaBt¶峨ƒ-Þ*#é,lqø‘tà ocJÂl54»­²ZÒyØbLt¢¶`ð%0¶¨$ -À šˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lq˜¹¢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.¦{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]k‘SÔÁXâ°ãdu²8ð®Õ*J\àÀ»˜FFâŒê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„9h%.pà]ãÑiå﯌ļkœ (øG‰Þxâew¶€`%.Àz± °ÉÀ-‘¸ÀƒwuÆò'Ìi°Õ¤Sxð®N>Q—'Ñã¼E/O:„,qïb¡²xð.è3I\ŽŸ°¢‰ ê.º\÷×Q;¨­eÈÙÐyÛû´ 6/Ÿá5ÍK·—¶îK}n®OWÈ>ÏýTôžõe`}Ãõ?Ï2]Ë@z÷´†fIËÚ¢ÕaÙM)fGmó25Úší8ëšï34þ¾2€šûAÙuŽãÝlªØæã¦{[WØÝ™†RùTíI?ùÍÜ;Àyé Þþ¨öñÃû.uÛ|óePŸUœVµ>LB¿,&ö§eõçAØ­’Ê´}8T8{s9eÈÓRW^äÇå›"á& -åú—Nr# í¹‡°gÕI~àEbì¤ü´ˆøÚsK«ü…2Ëãlçƒ=è©y¢-×CóÄaõñ‘ ƒ=KÞy1YË.´ßÙ²·DqözŸž° óBÜméI~=p5¢îwë}—ÖÆäàãÔg6‡uTšë¿ëñÓ“RïÁEAtù ‘àǵóT‰².È-,Xüëâ»Ç­ûOÏ,3=*-³}‡­ö•Ôl -ô ´vB„ó|I„õ5øÐ"¬>àCõ4.wò,t/"›{³0®gë¼4.×5ÒJ³~¨s^:×u5FXÒWè‰az`Œ*oœ?â}l“^ʾ>½Í˃­!Öls`  -öK0&àßM9ûõPá!þ±|-Xö -—Ú%çÎð-Ï£áßiu„ÿ†U3|Æ5Nt ©žæë|ùmr7¤£ûâš-y>v1,]~?-Ÿn¹¦ò9G«ó{ÆÃ|tÃüîœ £®ø ‚è›ehT8cÝ‹yuÝlðuv¶Æ®;– D Ì901§xµ#K.eÌÛjÆÀ·#¿Êó3CVÛT¸Š„W¶µþ‹L­ó¼‘½ŒfÇÓR]ç:KÈpÙœ"õZž‡Cj8æG; ‡­Áyó¶ÕA›‘–ÐõNˆ| ¯qÛ8<›1—‘¥å²¾R£~=R¯åy8¤¦cNU¢>öëy”ñe˜W×÷¥;´7¤³U¼Î´bŸû¶öµÙSËù4ô—ëžÉÚ¦Tv_ј0)í©y1Ï£¡Òû—o©öâ'ðÙ& Gå<wFÖ›×óÍÅ<7²'>‹4Û*9ô&ÍóÌ¢Gi…Òl”DPzxE¼Ú^É[óóh9¬‘Æó6^HǾVCíÀ/þi¹•S|XÎÏ¡Ôw~@k›IÜF§Ýsô”D_%?Ãéµ<‡Ócn³vh6f‹¿]ǾÉŸíƘ£ ½²ÝõlˆR¯¬u|gf³ÅæŸcs§}Ú¡}Ÿf¯O›Öc“r×öK.æy84{YGëÙŠ‘F”ŒˆÒlDùÚ6Žú˺œ!VË,Êå(xY—1²Þ&|øa²E /ûqÈ™·viÿò“k=¯m+Ï«öÖg¾Kw©×ò<RÃÉÎÞÀi#Úʦ›>NC£Mû´†~—Q1Aiá‰â{xaÅ÷¶3B¥Ï';ñ%͈*ÏêzS4/#1p¯þ6 •ïs"ñbž‡C≧øwÞëh<2¬Ê+]ÍWgÍ°k·iC­óùòÆç.Ď蟻|?¤wLvawTFõrß°ûw1ÏÃ!÷Øw|ä¼#T1ªX[F$oÌýP·î§ÒZÐâ®á ÍfOæÂêºÏébš´uô]ä^ËópÈ=Ù ´:x¤ÉVi¾ÒžyóÛa”Æ•||ªqàÃïÂnë³ùÄamÿÑóîUYOY<¢žín¼Å ù \z-Ï£!2žø£Ã¾Hë=¾zíE(Œ£x¯¯V¾§ßø™y›ÝLóu‘™ï©Í”U2ô·Q‡H½˜çázì¯^AZY!?1伶Ç/Óiw°¿Ý§{l”ï{-;¨d{’ÙO¾çg¨¼–çáPyâ‹ž¡èµ£ñ^~ßTï}3î7òdÆ7ðåµ<‡Ì_ó=#— (UtóÞˆfèðÏ„à”v_òrDt1Ï£¡4]û¦ãW½¿‚Ô½¹¯iã{??JæõqùÅ<‡Âcß²|“¤~VÚÕæßw K:iÄ…þ4ff»0{û›u„ëÎû¦Þ‡úŽüÚØ^äÕ•ùbž‡CÛ‰Ïø…Að8>Øûrô·ó“w_ÿMð‹y¡ož~‰O^»ÃýŽüó9;vË|B:ñªüìb¯åy4Äæ“o4¾yßà/U^å|«7ø¬ïý|=3ÿЛIy=‡ˆ¼˜çáùÛ}¾þ°³å0¢ÚAœ{ýw‹.æy8Ä]û«˜ñYŽ¸¿ôôöïJî>?ú¨¶ßÂx'¯à˾]ñÞÀ*Ùóë«;äó<*Oö!•Ó¢°KÈ~ÅêþUfÉÀ•»üÈŸ[_üÆÛÅ<†D­NœÄ‡!ðiqmÿi9i2zŒ¨÷tÒoò:±ƒ+6O“œ@±‹°=âð®‡×'' HzÅŒMwÄåH§²î›@çô™å/ïËaoaù›—ål>¾²“H;ŠyISÏ[(™»¹¶œV*g/,‡;•CŸò,/‡-Õ [®Ç³ÿJ}ô¥”Ÿž’°ƒ¨Zù‡¹µ6øÚ†ÍPþz å„a]N™W!MÚÐkhîÁMtn^Ìw:Õ8)ët:]:[ßwr–—÷üÚRÑÿ):⟎w.CËo–ô¿–Aÿ妳›¨ÿ‰)æwo¨"N¦Þtñç½{Å b'ñ¢˜„Ë¥je9]6Ù37NyL‰ B-—¹NGÌ—Æ×ÖP~_³Ñ™¸NÕD¶œ -;W,‘3è­˜Z¯&½ˆéý5i²Ë]h¦ÐaLMØ t S’kérM'ˆžobº¥4ßF*¶T©g͵ç»)ê6µŽd¥i¤r±9¢l½\‹é,Ö6ÖtPÝ2}êváó KŒÛˆ?ß|RûÎÊWèq¡eå6qR/]JÏÒæ+LÍ*£“›éÿ™*|“ó” í¤èôõ~×+¡¹ÙÄëNâòÐXÇI= ]ìiYwéfy餄ôDº-u뱕”u“Ç\Í#’jä±놺&& -«ýÓ¸’ U5Õu±oPþ¼w;}›ª^µÊ‚fp÷­Ü.;?sql#®é(ýfo+—a¾ -ŸËÂæF1A?ÕJºØ¨¶Úõʘ°RèâN£Ÿ‹¸ÎÖÆNôz°zÙ'™´â>ßþr+Ë -C'.«éÍ0ÜóÒ÷ÿCšñû×›»ÿ±E>¯ãF5¶0[Wb¾’Ñ{ÄÒE1½W¬ -Û´aBãíw±N=­¯hhë¸1Œ¯]Ï®:±×”º1¨nZ.ô½™Ø4èmYÏøáóЋ͑ùÒ±é ö|kkãèìH:ľZ…Þܦ®³}I¾³"qGIû’­3,#nx¡¸eænÙÊL$¯¤u=ÙïøPXª‹Ý07‚nm×¹¾Ìò¢«¬´‚% lù /Øi^i^Ù#ýæéõÅ㾘ê i£ö&ýÌ¥­a$m„'Km¹ ëÛ¶æØrÐb¼½‡ZY–Í=Ûœœ>5ºkáV7£Lšz¶%畺*[UZŠF©*k%ÆjSX²Õn)-Nb/ÅæSté_xO{>*­=•°IKåÖ¯yK¶9@žÃ9öÖ©ž²ù’Iž»I]Oi«l[·[×]WÛ;k†ë†„žÍoR¾‰ÿÞ2wÄ÷H©êOÙÑ©¿¼ µÓÑ[ÞǪRàG¯ÔŠ.±í’Wç9Þ?Ùî’Ôý^†crÖª¨éͺ² Ÿ¡Èç -VT}뮜-+±I¯ëa{¨ÓÜömåÕŠe?³lµÛH¡"½IÕ’Ó›èpQÉo—(_"K«í“òóÿ“ôvÑ -endstream -endobj -509 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 508 0 R -/Resources 4 0 R -/Annots [ 510 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -510 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 668.673057 101.433071 649.923057 ] -/BS << -/W 0 ->> -/Dest (cb73-1) ->> -endobj -511 0 obj -<< -/Filter /FlateDecode -/Length 4388 ->> -stream -xœí]K·¾Ï¯˜³Ó|_@àC€$ˆ $‡ ‡ÕZ²¬Ø9äïçcOóÑìêiÎìJjIÖb4ÓU,v5ùñ«"»g¨Î_+üH‰¢wt~üùôËI -o%þM%“ú^Þž•$/¬Š.ØóÏI`¼ðÑjïÎO'ü׶şNÿøêôŸ¥:k£ÓÊž}›Td„ÞK˨"TRFe­ð^[Ǩœ¸Ä9Ü°ž4­UnP –SÁ Ò:0ª7 yeÖª7´WÄ©à†ŠÖ3*%á‡ÒR²:8"‰ŒftÊ -‡FwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¾XE$9|!<§sðÅ8#Y|ÑÁNçá‹FëDFà ÐX|‘N*N°1±º >¬uZZaƒ1’Ó)%,°iXF:ÉyF§á‹“6²:øbÔœÎÀ§OXãC`F«N£ÛH£8…/Z{buðÏ1€vðEzÇQFH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHøbÉqL`|!Œ\V_L$Ž Œ†/(Àq1ðE“á¸À€m há¸À€nIFÅqß’TŽãÂ…›’ãÆ5ÁÇ” (EŽ 8×€;9.0 ] wÏqëë4Ç´k(8Ž x×’x×X²:øs\vFÃqw’žãïbiŽ Ы8t+B‡ 9. ð®’8. ð®ö¬Î ŒAÃqwµÅ%p‘¼«­ÔxW“v(ü#Ùl¼ -±|:_@° ÞC§ ǨLh¤%Xð®Š̘[ ¢ã¸À‚wU@÷±º(”G‹3ã­,”s‘ã ÞE 2Xð.è3p\Žˆ¨šã ÞU‘šÓw•Ž’ã ÞEÞ`9.„BÀqâ›PÐeuQÈǼ ZÒ - À$¬Î Iq\2y(Ǽ‹ð9.pà]iá¸Àw%!íauðÅÍqòè0l™1dUH ¦àtà]©ÐÃœ¼+¥8ðn -›8Ÿ:p\²`Íq $¢÷Žã%tHùX8¶x e±:øBž8.@b-¢‰ã¯á‹Ñšã¯á RŽ ˆ¨}³Óø¸°ó‚/È9.“‰‰8.ðÖa>†VåtN‹àA׬.@§<ÇÞ“@(V`.%=Ç x‘Ò%Ž àtŠ8.ð¾`¨p\€P$‚FfÎé|A’Íq2]è”⸠hø‚lã‚ f™Nr\€¹$ššÓ‘LóÌÀqA 'RªÈq† fš`JV_@ ÒdÛr\€É&›˜æ°:øb,q\‚NÓÍÀqAðEKÍq°™æ›žã‚(á 2>Ž ¢t"¥É€Æ¡“’エ曚8.À¸LóÍÀqA4óMLñ˜qKóMÏqA$øb‘¢°:ø‚¤)2㜄ù&؉Ó9ø¢#q\KóÍÀqÒ Ì7‘ѳK!Í7=»\ÞµÌÄéÀ»ÈLÇ8À|™-7·— ^ë‘ŸñÊfœ]1 ^‹ØÊ.Hp/¢•g× $È7Í‘ØE¸‰™epìªÂ*”J²Ëü‹|ŒVœpY}j—¨àà3ƒO1,Ù}™sÔbf£Øe}ì×Oß< øžüïéÝWÓJYþSH¦Ò„ü -Ã@Gt·Eö­pùÆ¡£•˜„4yï±INøXT ÅyÑ-9‰ð—þמÃñWÓéM9}D¿œ><°Jê«€éë´6Øx|£É :,19B´u(c¢NìÈ!¯”´(üû×'uöâ²Z÷/1&mϯ>}ó—ïþþ×ï_F¯ßþù;)­ýök…w -—w {:™›ßý,SuDó»ïlæz®ZæÝòÝêùý¡©ÏW;«ª­•¼OZ6þ=2¾†¥½ú¡©3¬¯k:¯Y^CñóÍåÒT§òï½ ¿ý×ùõw§?¼^sÊV'`RJnKmw°›æ]ÏåâûåH#oçÈA›£!Ó\Ï;'î˹3—Ü›Í ÷¾›{¸}ey⣇yVô°Ýóe–ÑùRxþ¡Cy¯ÏèhGU“/ëvÍÒw6ùz¹ÙÚ/n¡ÏÜ~÷eÐæpèÛÉ!·Æt›™qÑtŒëú,dkö¸‰B{ûÜ{Ðæp(Ë3×õ«!C=z˽—M¾ë{Í9iYï‚ë>Ùwí¾‹§~†ÞÌÌ{,ô±wnòݘ–©˜lqëîÈÄ }ýùQŸ¸ö¥Ì£˜»@eŒte'?š±·¯ÉÜΈƒ6‡CåNÖ˜îqÙ{ωi×¹B€¯v· `™=7ó†žUe×íýÈþ^@;oëæv­nòigCw<9hs8DîÜ…nî,¯îH7™ÑŠ»Ú{‚$³ãUï›ÁÞ¿ÒÃìì½ÍܺQÓçºAa©«»G¸âÐþŽwÿ¬H¿öäº6É«ýZ‘ºè¯Äñf-qt†3hs8|^Ï!W}±ñ8mwŽ›iì­•:ùþ^<~{+–8\¶¬õ!°Öú³»~Á3§m³Å„Šzï.â ÍÑiwî{¿‡X~ä•O"æ1ÛÜÅÆì˜Íá0{=ÿ\þ{î¢ì°${ÇÄ2ÌtåŽÉsü»õè{_%ØbÒæ©Æh…nËì¡rÌæp¨ÜYż÷» ¿­½/ú¦vÜ@e¸ƒ+Çl‡ÊÁoØÜñ…Ï‹¿î¡/º¦Ï#ÕÝñí›A›£!Õíd¢__mŒÈçL>ÌHÜ@h“CŽ>«1hs8„î¬{Þ³Fyïü– ?ãæìí3ùA›Ã¡ó=域hfð¢#îCDúpû“ëƒ6‡CêNNúIòÏçéýß´9BýýߺÆ;7?ßñ[¤ŽôÞÜþ4û ÍáÐy=½{ÞûFú{ž¨zß‘ÞÛÛçôƒ6‡CêNNúIòÏ °#Gúpûœ~ÐæpÌEÛgºï+džh¿¯6ñUs'ÞÎ=ð[T¿-ª‡;îÄÚ ‰a'çüçïïct}€îø–ù ÍáP¹³æùIrÍ ¦ÁÞ>+²@âCƒê+Ïš¬~T1–»œáŽ{ïƒ6‡Céد­¾Y¾ñ+:sñÚ|þíŸj}‘?àõn|FÓgÅv°rùq.ùxýÜáB›o #tÌæp½qͳá´6®xk ‰×æ˼ô)ç˜}–³z¶~#ªÇ;îµÚ q,×|‘§ÕóѸõ¬ýÕßðÊOç%Íß=Û¬§¿&._džµ¿'Fì}gn‹+£¹+m‡ÎÁo¥é¿òûÁÖy7yÇ÷‹m‡Èü²ë‰ý_i¼“îüúÆ-ù`ûÔÜÍ¿¾tøoÀ´¾šw7ðkmÚ]&XšÍ -Dw‹ì r½µBÞ àœþzȹk-Ò¶¦&œ’)ú´H‚ØwüóŸþÖ@,ýȨ~{ùIßK(Ãg‡šÒ¸Ëqjª 2i)$Îe¦'ÍÄu!-;ujÎK·ÈùÇ^Ò{ª+•§w—îIrʺËkªkþ<Õ2DæóM“ÚÔ-)áºèõÛùÜ?S¶¹†ü }\uå/'7íj£¦Í¤ Bé´õéÙ‘0©oâ9í¤¤!•»NM­Î–Ó†±¿”Ró²2m+—¶ºl0ÛÌå™:ýç“Š$Rû'¸ì÷('qÐù ŠŸNi¿ObÄDBŠ“pþ(K]¤rÁ"{ïzû’Æ -9Œ¥rcóGmóÕ${›ÍÒ>,$s!3íDr91#„¥Skqºz)Ô,N{¦aæ#W\±.û a­Ú²µÂj¸+VF6B»ÊætU¬)û£Ò[¥é\×Û¦SM÷ç²Ú7ÝyMøxÂiM/}b¥i§Æ\Un]-‹Ì4NÅ"µ¹dé*ýĈOŒðk™«&tfÑ7½–ö»ºC•r1—cD'¶Ü\›-=åK©PMcnùêJ…º ¥áÙÔF³Ô§M ›1"Mþ¸»Ô¹¯ehp£WÂÇB ­ÄO'ë™Ò¶r…´z­Ð4õq/Uš~^àò C3ËÒnAéÿ UØ"MÌ3}PT •vüªGõ$ilñ²‘Z¹+¬C\ (òµlÓ\Êû‚Ù¶½ 1åé–Ò¥ÅҦɹٽDR…<–bUPWĉÂrû®l…2;ÓU±-P~Zˆ«Ÿ¶ U€=ŸÌ©î¶Ô[Å®pçS+ö¥ÇUÚ¾­ø[êm0Ÿ…S`£^œ òIªXËíêÉau ¤PÅ•FŸ&q­…¦ßòž?ÖAÆEÜÇÓO§)ÌicuÄgTڴͼÔù(ó^ÿ>Ñùû¼ýÕ%ŽkYØBn]ˆÛH–ö®ƒâ´—u–aÓ55´_Å*Ô²6£¡Äq­¾¦j.+±ç’ª0¨*ÚVhk6b] «•iZÆ^>v­X™çöM Ø—C“/§1iã´ìÚËÁa¨:SCò¹©G©h éÐ’n qÐVŠÃ愱z¶p3²=Iiúä?µ]aÒ¹šƒ&H‡¦ê¨†Ù¶ê,›®¢)‚Ø´mŤړƅ?ŒÓw¯g÷ûì²Ë# I µéS+-–¤…ðx©™Ýò°ÄsNyðÖ*u™fì™’äÔ¡QS Z´m¤¡šÍl{RʲÅIs (t™Ì²R£Ï>¹Ù,7ËtÅm%_rŠ*ý©miÛöJiDÛöŠ[•MõæˆW²%S ÛÂÙ׫“µdÉ%#s­u‘R-i²l}nZž;GÛssTq53•ÕÌ®JÞÅ÷Œ‹M·Á•ÊÓêˆ -uÃHjRÞhsŽ)Ý ÎC(mãì)4Ë%7Û\_?Y¯’äõ“öLèAÆH¯€· ž ˆûŠ¦ª§y¯MÞ,Jö’nkfy¨Òœ\­aæeÍi‘¨ú4U´qÚtY†6÷R±ËEÓïö.—O¦¿ÿÌñvf -endstream -endobj -512 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 511 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -513 0 obj -<< -/Filter /FlateDecode -/Length 4851 ->> -stream -xœÕ]Ë®$7rÝ×WÔzåðý-<†4<‹ÝÕÝ30® h¼ðïû“HfTVÞÛWrWUÁG<<Œ`f²ôUáßwÿKN/)åÜõöóå—‹Z¢Wø¯¤Ø\õOø‹þª•‹‹×9$ý™6.1{Ãõå‚ÿM—<ùËåÏ¿»ü÷6¾š%ç`´¿þý ©œ]bŠQyA•¡RÚ -*ï—‚*èMÌYRÁ q{U„.'/©`†3& ª3¬‹ÚîUf˜¨¤‚:û(¨´‚Ú(%ê`ˆrÎA§ýÐéAÒ½„”Mu:¤³v Ñi¯%]^B>I:[|VZÒyØâµsJÒÁgS”t¶Ø`•¨ƒ-&E+é"l1è,èlº’¨ƒ-*(-é²]|NΉº J1íuFùÅ'k•¤ÓzñÀ¦u˜Q¤-Aù,ê`‹·ÊH: [þI:ð„·1%a¶šÝVY-éá/UyMG{Ë=Ï—¡×¾êZÞ)ŸÏwÊŠ5ýת LwTÖ­ê¦4Zoíò7 -cªN­>mëêyÕëT£ÿº®~zS?[yz´Y«þóúáÇË?lWâ¤8^!¢÷êFÏeùŒþÐÌùš½ ý¶»]œ†ˆui¢6¬ŒiûÐ&V^¾PSSÀ÷Š5¥_²UMÉ€S£M*mùñŸþåŸÿµÁH¦ö¦ú#A6õvlmíñdÛš+&¦ekãz½“×´ î%]+‹¥ëФò1MüçQf¹6[»ÊôPCî⨫ä£r>²ºÔªïíª¶–²k‹íz²'®ÓâT·ÞíÙLêmï´ÒK;BdYO2«š¾¤iV}ùÕ§5 ïÕžN±^Lk/©[mQ¨M ¼ ­à¼YºßUÛ`ëˆc™DÕ”£?ïË –‚Aºá°/§´½N(í;²Nò;Ôh Þ%ë l£OA9xJBÙ+òG™[$dQ?[ú¼U}¨6ù¶¨¬ß[[ÉGÚÒ~ª'TÛsý r; -B[š¯l6™Z_E:]=¥k‹Ÿ¯z]Ëÿ4ìÓæ°ïýÜ?¥¬/kþ¶¸RÙ4ÃÖYU1a¶ín³´ÛÑÚÜð×ÏRÞ—Š¯:Î-ú,•ûšØË7¨µ”Ýöá<ŽÝ¦0æF7=ì õSœû©·3Nø¸ƒ§-+½oã'2aÅxgÀ-ûy΀­O’Ðθæ-ß+–ÊÜNc|z?2»yþ"+m>ì§<÷Óà<Îw3×­z]ÇÐÖv÷v6>ªØVunùO£Ý¥/ؼ¡´¥ÜÊcmœ[Õ.¸\²çÛÆ¬ÞÍ¿iœË°AhüÚgæK-íÿìúßO|×8svmRK°%H¢›€&ióÐ<™çM¡ê;ú~¾Û›Ý’%:…žšãwŽÞJwÌQZán…9À¦õñ‡óHõ:!µ£0V×À×¥#uçr (uKÎ>E=Ux„RO7‰VÄyNn*žËól(õtÏù>J;¯›ésBšnAÍÇõûN^ÿºÜMéèº8P5ϧ¡3†¥³õïsýtõ;•Ï™ô·Z×¾çNì´ý4ma=Ú%ô!vѺhÝŒ;FݹðáZOÞØ*9õ&ͺ ´¹á;žf£TAyèÅÇ}Gã­ù?±¨&l‘Æóv^˜bè™{Žâc¹S|XÎÏ ¡o|7eœoÌѳ}‰^Éy„Ósyž §ñ؃ۭ|?¥úÛmìwÏ~„yí™=A—÷™½²»ëÙ¥žYëøÃóó-|þ96wú§Ú÷yõúÚ-ÏM‘yLgÂ:™çéÐtìe­gFšQ2#J«Ýn GŸÖHîowË,Êå(øº-cf½]< ùð­¬ ÑĨ÷âGÞŸ1ãQÁóšc­ÛÖ6~sp‡T¾Kw©çò<RììMœ6£­lº½uziá¾ß{xaÅ÷¶+B¥Ï÷¢`nËŒ*ÏêzS4‰Üë£w¬Ry/æ‰çò<xŠÏüÄểmÖL»v»6´:oç7Ÿ»ƒØ]âÓ§{^>»;*£úrß°ûw2ϳ!7ûŽÏ¼ƒw„êoFkˌ䂹ê¶ý4£QZ zÜ5Ý¡ÙíÉœXýÓð9]L‹¶ŽžØxð ö¹ñîÑê©<¶ ±)ëà.Ë!‚gԳݷø£É½žKOæy:D>ðG§}‘Þ{|õº¡0Žâ½¾Yù>þƒß‘yðòÄæç¯Tæ{j³d• ½czŒÔsyž©Çþꤕò3ãAÎk÷øeº3íVà·ûtÏòû^ËT²=Éì_‚óG¨<—çÙP™ø¢PôÚÑx/¿ï ªïDEÿ(—‘™õëùòdž§Cæ_ó=#—(UtóÞˆžÞT}ßAéð%OGD'ó<JÏ=éø«ÇÞ¿‚¼}¹¯iã{ß?ÊáõqùÉJ÷U–=aÕ_Ÿn¯–÷?ê«Áå5Ô¼ÊM}õ´¿6\¿Ã T•Ø”à ÔúJ5{ý·—ðúwoƒomØ å/—PN‹ÓåÄP°J:RüÜbilò•N¨KÊ:݆N—ÎÖ×;9ËAì¿ôTõlvEǵÒQ}ëÁí좦ÿ¥ úÏÝBý_Î\(ç(«"N¦] ñË…ÎÑv‚عqpÃ1 ëWÕËrºŸÑd7m\òœÒúEÕ¯ã°zyeýj|k å÷-oæTKdË _kÅ‚9ƒÞ‹©õjÑULg‘§ÅÖ«ÐM¡×6»pèxÚQàÈøP¬­bB¿*VÝ×ìÑtveïº0„ÝtÞuš Kk"Î#áí‚jí,}¥tr+ªõ®Q]f™Q¹K}KÙÄõqD·‹ üNµ¢³ëÙ¨Ñ9’«05 ¹ž.·t‚èvÓÕÒ|©ØS¥‘5·ž¦ ¨Ó÷e7v J¤W`sDÙöu+¦3<úX«ÄpcvÂÛKŒÛ‰_.> -©ýà -åô¸Ð²r»8©—!%~ý†©Ùdt -ý¥ -ߥÄ<å‹vRt’渕ÐÜìâm'qyè¬ã¤Ð.Ž´¬»tŒ³¼¿tRBz"Ýžº÷ØFʺŒÉcnæIuòØŠuG]…µþé\É…ªê†Øw(¿lÄÃNߧªW½² Ü}/wˆCçÎ.Ž}Äé( Öˆ±†ù&¼•…ÍÍb‚~j• 1«ÓàÓ+cÂaHaˆ¾q›­è§Ú×1ɤ÷vùÛ¥,sÆzCg3«é”oîyéëÿ"Íøû¯‹»þ©+¹®ãFu¶0[7b¾’ÑoB¤“búˆ&ìÓ† c´?Ä:´¾¡¡¯ãÆ0¾v#»ÄÞRêΠºk¹Ð>dbÓ¡k´e=ã„·©»#ó­c3ìõÒ¶ÆÑ)t i³ -¼9¸LCgÇ’|eE⊒Ž%ZgXF\ðBqÉ*Ìò™H^Iïz²ßñ¡°T»`n]Ú¡sc™åE7YiKØò ^°Ó¼Ò¼±G0úÍÓë›Ç½šÚ i§ö.}áÒÞ0’v“¥¶\†íe_sl9’¡MÞÑC½,ËæžíNΘõp› ŽQ&M#[Mày¥®É6•¶…¢ÓdªÉz‰±Ùj¶Ö-¥ÅIì¥Ø}Š!ýïiÏG¥w¢ç£vi©Ü¶âuoÉvÈs8ÇÑ:5Rv_2 ²Àsw©)m“íëvÛºÛj{eÍpÃ0> -endobj -515 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 558.873057 101.433071 540.123057 ] -/BS << -/W 0 ->> -/Dest (cb76-1) ->> -endobj -516 0 obj -<< -/Filter /FlateDecode -/Length 4351 ->> -stream -xœí]Mܸ½÷¯èóË%YÅ/ ØC€$ˆ‘ ²€‘‚Ƴö.‚qoùûyT‹E•Zê¶Ç–íõ`<­*U"_)©iÎ?ßüÙ¨Sð|~|{zwÒ*8C‰ÙÁ þ¿Áæ œI>ºóÛ,  Br6øóÓ ÿu‡mñ§Ó?¾9ýgncÎV¥ä­qç__g“ -1íU‚JTΩ¬ó‚Ê…KLIRÁ ØòRà§è$Ü`k£ Špƒ8ZªÜ°Á°¤‚&¹ ¨Œ†Æj-êàˆf&+èŒSî%5ÊÇd“¨ Й`%‘ò3’.)ｋ’Žá‹KÚH:_œaÖ’¾0Å é<|!OZÔÁIº_,Z' º_€®(êà‹öÚHºDÊ¥È,êt:†¸ÔY픋DZÒ£°I¢# -䃠³ðÅk—D|q¤­¤#øÂø‘tà G!Fa´Ú<ºI“‘t¾XXÔÁt¾ÄÖüDA*ó’Ä6ÅɲD6’âÈQb“âH¢›œbŸ‚Ĥ tÖHD@¾8ö/Œ‘+êà %–¸€,|A‰ ˆà‹e’¸€À¶ ´H\@ [ÖÉH\@à[ÖÆK\@ \¸©%. 0.EÏ(PJ8—ÀHÃ=H\@`]rÞJ\@ ]âè%. ð.±Ñ0x—–¢¾À€™H0x—Œ0xÃÈJ\€^UÀ¡—¸XQ6F-qƒwmÔ,qƒwm@uAa ’Ä Þµ— Enð®uÚJ\Àà]ËÖK\ (ð³ð.(ÄÉé|ÁJ\€x%‰ P™²HK$.pà]“þ„1 ¶R&&/qîuI™€Æ-ZYï“ļ‹@E8ð.è3J\ŽWˆ¨VâÞ5„H-éÀ»Æ&-qï"opÊ ¸ñM 芺¤tŠQâÞ-Y‰ …•Ž`Q€d$.+Æ N¸¬>µKTpˆ™1ä‹LŒì¾Î9¦b´Rì²>öëϧï|Ï?ÿ÷ôæ›a¥¬ü$CiF~…a`ºÛ!û6¸|òèh£!ÂÁ{l²!MNÐâ¸è–DøËÿÎË3ÏáøÃé©ž>!Š_NŸyd|DL_‡µÁÆãM> Ã“#D[2”lf'@y¥æYáß¿<¨£—5Ðzظx‰É0[w~ùöôÝ_^üý¯?ü½|súçï´æ×Z»øý·&vãßrüf~ìÂå/£Ùwe»²f¥|Ñ»yÕ>5Ÿ‹ weÇ:0?Öy%Ñ¥¦ÌXo–/|kt¥\µ3Íù^-ýœÙ¼žôƺµëK+u…yû{c¿ÿ×ùå‹Ó^ÎA8d¦9'2H9¬ÛÀí>“÷ÁísÀÓÜkp½vWà=èK÷Ó¼¾›º³NkMÓ¹¶üù Ã­¨{|Ÿ¡"Ã1¯Ôp!IÔzå‘Tý{|s{óÓû£nÁ·ÏÞ†+(¦Ûgã;m‡âë¹èÝ97²¯¯ïAî*ïXÓÜis84n,fÞÊ=oÉõ>Ù«©ŽÕC¹ê­(ø¶ÖÑŒÈ|î< -iährKn¯<Ýs±ÍÒ¤OègÃÚo!vŸÍá{=Wµq»w? ò„(\Ê5þ܃¼ûÑÛ~íÉC©Y‹Ü‹À6GC }ú<Ôuõ,q%ßü(£¸îëb4våjüèì¶8‘¦ü0?6 ³£-Dî³9"w®oÚ®gÃÔ³&?}Æ_¿Ý+³\­¿½(¬pWï|YËk»•pס ÷ÃŽ¨®¼_þÒÜ>Ïšr¾¸¶þHMg¬J:Z½‰š}6‡CÍý7²g<ÓÏ=;nrÀN×®˜×s¼™×“íj¯§ 9mNYmÖxUºWÒÍÙûœµè?ùعNIÍšãÞÙÊN›Ã!ö·uÊÙÈÚªk •›k÷¢]F(7ë{ï4î´9Bùznx/ÔÞ(-ÿ0¢°»)óÆ×82VH·ßÓÙis8$n䄇â÷ˆèm®Øø""ó¡C=ÍÐåºJÞ;|¦æ¯Ë¥gæHwGî³927î{›&7æ’[³™Ý½ïÇn‹<óÑÃ8+zXïù:Ëè|©<ÿС¼×t´£ªÉ—m»f:›r½Òlm×ÐwÇÓ;m‡¾ÏCöcúÏÐ+Ïì³õ'ßd:}ûÜ{§ÍÑPèöå‰ë)Úu¾ÐË{Í9W$ÿ¼óÐt6÷›CT†8ø­î6‡Cç{䎟”W>àÈYÉs÷JºUï:²âøë.¿3yrÊÅeì¹ÛŸòÙis8ì]χv ã:nnËWSZÛ´gÉè~ê>wí¾‰§~†ÞÌÌÏ wO‰wnÊݘ–©„lqíîÈÀ }ýåiŸ´ô¥Î£„»@uŒte?š±·¯ãŒ¸Ïæp¨ÜÈ»—®Ú»÷Æ´»Þl±·¡`/ -{®æ =«ê®=Úû‘ý½€vÞÖÍíZÝàÓÆ<Æßñ4äN›£!ÒoÜ…nî,/îH7™Ñ‚»Ú{‚¬g³ãµ§³7{ÿJ‹³÷6sëFôþÅ‚»{„ íïx÷ÏŠôkO¾k“²:Я™‹~=Žûf-qï g§Íáðy=‡\ôÅʵ[¯¬–§k¯­Õ:åþž={+–$\ -Ï…?+ÖZ6×/V˜³Í38Ù­»ˆ;m‡ÌûÞÏ˼²óYdÀ2fÛ·möbvŸÍá0{=ÿ\Œþ{î¢l°¤xÇÄ ÌtåŽÉûøwë=Ðg_%XaÒ oƒq§ÍÑP6V1¿è÷W?"CßÔŽ+¨¤Û¹r§ÍáP¹ó ›;ÞQø²øëŽúA×ôWzÇÛ7;m‡ÔLô«à«•¹ñœÉlj+·sÆN›Ã!tcÝó _võuf+_l¥oŸÉï´9:ã3域ifðœßwõ‘>ÒíO®ï´9R7rÒÏ’¾üHïxh§Íáú<_kyóó¿Eúý‘þŽo(Úis8t^ÏCïž÷~‘þž'ªž;Ò'}ûœ~§ÍÑš6rÒÏ’>Ð;p¤Ow|æN›Ã!tg.Ú> Ô½¯Px¢}_mà«æN¼{à·¨~[TOw܉ßis8$näœ_âüý9F×Ljàw¼e¾Óæp¨ÜXóü,¹æ¦¼}Vd†Ä‡ÕWž5Y|¯b-Ë»œ?Ž{i‡u.ïe‰ÍÖš›E6¸ÜÈ£l‡qÎ?=Ôüˆ1DAåMt)ž‘r -yß™ ­üóŸþÖ@+­Ž}}ù«ËKøìñ‹¦&9Î]?<^‘Ë _ÇSàvZ6ÏØvîÒmz|½1ÿÍuåòüæÒ=YnsYùê?õÄñ|C7ænÍ]rÑÏêw㹆:\s åÑ‘´èÊw'?ì¡d†}ô´ÊؼÑîÙ³¢Ü7éœ÷mŠšØ”®3Cc›óŠå°=ñ»ZjܱXçM óV—팛ƒ±ü»¡ÓßžLb•Û?ÅewQ=ˆ£-“øé”w—eAÌÈwŠ³pü¨k]lJÁ*{ß•ú’ä”?¦Z9¹òѺr5ÙÞ³¼ëëRˆ†}o.'„°ôf)ÎW¯•Åy‡Þ¨h<òÕç‹ÏNr[vªp2ÜÒÐ-…º9Ý$¶\ü1yG·Út~V×Û¦3M÷—²64ÝyMøxÂi©—>‰Ò¼/h©ª´®ÕUFS©J])Y;„k? ¢Ç“ üV—ªYõM¯åÝÕ.ÂX€Äµ\*åÑãI,7ÖæjO…Z*N¦©´üäÊuŠµáÙÐF£4ä-Ç›1¢©|œ‹}îÒ×:6¸± áã !†⧓ Bi7q…vz­šz«8ê /“4¿Psù„¡Ydyoªüÿ…*\•fæ>®ÊûËMGÓIòجây#µr_Y‡¥0¦²Ms™*fÛö2Q å3éÖÒµÅfÒ¦ÉyHŽLR•<æbSQWÅ™ÂJûT®l…º8ÓMbW¡ü4O~º:Tör2o¸»Zï$ö•;ŸZq¨=nòfÕßZoƒù"|÷â ýXN2‰­®Ñn:Y#œ)Lâ‰FŸq­•†oŸ?NƒLŠ¸§_NC˜³älÂO&ï}Ûf^æü?”yߟøüCÙlíÇ­®la+·ÎÄm$Ë;¥Çâ¼szÖaÓ-7´?‰MœÊº‚†Ç­møš's={)i*ƒšªm…njÃFl+t­¡¦eÜác׊5‘yß¾™öåÊÅá4”·é+^á ½ÆIGSH>7Uâ(B:´lC´•â°9aš<›¹™Ùž¤6}öŸÛ® |®æ I#ò!M:žÂl[u‘ WÑA -LíA[1›ö¤iæàôÝÃë½û}tÙ—¥•Ú«ô©•Ö ËÒJx²”†C??¬1‡Îù  Þ©…j]ÔŒ=ªIÎ44¦Ô‚g-FiœÌÆ®=)Ùì¤%PTº€LY­1ŸühVše¸â(¶R¨9Å$ý¥mi×öJmD×öŠ_”Íõ–ˆW³%ª kᦫÓSÉšKFAæ[ë*å©$ÙòÜ> -endobj -518 0 obj -<< -/Filter /FlateDecode -/Length 4836 ->> -stream -xœå]K$9¾×¯¨3Ò&~?¤Õ±ÄJ#8 35Ó Ô‹´pàïó…3mG:#³²{zFµbZ=UáGØþüEØéÌÖW…Ÿo4þKNO)åÜõöã始š¢WøWR¬.ŠúüFÕÊÅÉë’¿þH§˜½‰áú|ÁÃ%Oþ|ùË/.ÿZçÑW3åŒö×"•³SL1*/¨2TJ[Aåý£ñAP=¡‰9K*˜á£3n«Š0Ãåä%ÌpÆ$A•`†uQÛ­*à µ“T0Cg•V°C¥D QÎY#è´Ÿ:=H:£§²É¢.B§£‘tÖN!:íµ¤ËSÁ'Iç`‹ÏJK:[¼vNI:ØâlŠ’.À¬u°Å¤h%]„-½“]‚-@Wu°E¥%]¶“ÏÉ9Q—¡S)¦­Î(?ùd­’tZOØ´¢3 -ƒ¢ 3°%(ŸElñVIga‹Ã¤OxSf«¡Ùm•Õ’ÎÃc¢u°ƒ/1€ °EÅ QfÐD¼$q€‰qrÙ8‰L²“K.I,`Rž\ŒV¢“ýäBŽX¥¡3Z"«`‹wAb«a‹ÃÌu°Åf'q5° $.°¶g%.°`[´H\`A·Ne-qß:¥ƒÄ„ 3•ÄŒkSpXP. ”%.°à\ À‚t1ݣĬk}0XЮu)H\`Á»Öi%qïZ‹œ¢¶À€'«“•¸Àw­VQâÞÅ42`T'à0H\¬L&%%qœÄ¼k"\€¨‹æ •¸ÀwG$Ï Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\± °É ,‘¸Àƒwu†ûæ4ØjÒ)‰ ƒ™$x‘i¸Xo"²•Öö -Äë#â3Y™hÅ™Äêõð­â–÷Â[EqÏ@|i$nÀL¬,Sw àV¡ÔJÜ6Pà_Äcnà óîߢ -€C„ÏL‘™1-¢û¶æèÉìN²yìß¿üò=œïõïÿ¹<ý¢ì”Õ¡¤vˆ¯0 LÆp{DßÍ·@vŒÅ4Xö¢ÄTmÌÝ, "\¶ÙÈ,8<úwÝ~YÙ -SÉXäÆJ(ÐêßÁ)¨¥bpO)ÝaŒ!/Ùm}qž»&GÄùÚ¾hÙÖÞG¼ÞÞI’­/Îóæ¶zZ{s]ïÎà'~ÛÁïõÒtØÖ‹IˆÞÁ=›é¶­³2  ¨Ã¢ÜîˆlÓ-¹~fò ³¼¡Á - xD„il6äAA‹Xû(·Jü«wébżOß.™ùˆé4œ£ñ×w?^~ùûïÿüÇ?ü¾{ºüõ[¥¼ÿîMŸfùLËgœ?]½ó§AéîÙ¹Eÿ´Ö×ôÊΟ`ÀòëÂPîuŒ6Ô2ÛgÒ1[ÔGü>ÍuŒ6´ôdïmÑvÖ¼'û¡¦oå<Éù\ìí/m5ky-O«ïþv}÷ýå×ïÖ +«¥—áô\–ÏÁé—€gÎGðl¨Ã—{×5Hݶñ ÃOlˆü:­QÇ¡ó~†Ý`ñl -Tè|c/ÓÕt-ŸfvØÂv•çƒŒ_tGpÊjíÛ]¡jd(Ò£XžnÁŠû±x2Ï£«÷C4¾!‰iýùhÑíí§Ú;aba¬´]P”æÏÝrÆ6ñú¤ûÞ¿ÞAlجmß#Jm_Δ'ó<:Ý1WÖv½gÅK ?S.kny@׋ËRB¿±¶Œ3hÚDö€ÑÓÆ_Jeë‘çò<"ï—ÃH´^F¡C¯ä0‰ _ 4¸A7òp-OsnÚrfãE>»6Áßz6P$óåfíZ{Øè•› -ÞåÏsy­Ç±æÏ–ÓöЬ·èÚÌ’ay·š‰_uÉè4=ŽÔ£môê:Oæy4tš“±çNôî„^m£´‹—lâ×ÄÊ«cÒ6.©-y* ïÚ–vÉöø ªÒV›TÚòýo~÷Û?±¶þÖ4S:Rõ‡êaaWkËÚÞî7Ⱦ9­4;k;çë|IÛVŽ/ð{ºîs¨ïðù±—Y®ÍÚ®2K_Ü÷ºJ>?ÇÄ­.5ë[»âŒúS]šx—DÜzI.Ðè²Ñ[,©=GY-[ªÀmfè‘ÖRÅz ÍùÔm±:,æn§¥3ÙÒù½ÅNë«-TÇb'ÊÑ·åK7·èÕ¶œ2ªµgCiÛ‘tæO¶Cõ¶P®GxžA‚mt.(h eϨíe®QØûšúÙÒçmчÅ&_ýÂü½¶•ìq±§-íOËxßj¤°\ í(H¬ižØLhÄ\ß9,éªÿò‹^/åèöÕhb§ïÓØ?¥¬Osþê©lšóŒX0aÖí®3¬ÙQÛ\ñçÏRÞ§_Ë8·™n—>Ùékb_q æ~Pv݇ã86›BŸmÜt·,zÐOtnlÝO­qÀÇžÖŒÖõ¾ŽŸÈb Æ{­ùÀsöª}’„vÆ9oe»"[05sÑÈÒk†žù‹s׊·ŽúÍlæ¶^·Cë/Ó?¥¹C]´%LÜbü<¿Í±íXé­m7Žµý¶æÆ1rÉOÁ–Û't@‹Ds7:™çU7±Þ0òÍ^„éÉ{Li1*S}\déJ,`˜a·ÂÒᯡ™ÎY8[]j^äÇå›'p…rýÓwwr»TNL†°gÕü>LÙcì¤ü%ÂG¢Ìò0‚óÁôÔÑV&_Ôj#÷‚EÒ‰ö;[NY-Ùz·÷é|,:/ÄÝ–ÞÉO£ΰæízß%ƒ—ÂŽcœúÌFã°®1 yÊ;Ìt. -bOÁŽÿ°HVA¢Ý ü¶8Ït¶s¨´Ìö"ôtTf&5AÞfdºs#÷\žY6>ôåäÝ.ª÷ä’Ö ˜qQ¸vQËâ/Ìyf>°YØŒ;[ÇûïΣ+À «!'.nÛ/¡G×&Ô妜}Šz¨ðYÁÅŠ¯Â¹ ±“y YtøY÷nÉ4ö©AýûùûFî‡ÛLnHG×%èYò|è:cX:»ü~\>ÝòÊçì÷µ|Ñ·<À¶O†-˜{]t¬º"èìMÖ“yuÝl:-|€ºáÀfC›Ýö?»YÞF/®Ëø·²Î#á…mmËÔº5³ÎózDªÍ)Šv‘š;×YB†Ëæ.RÏåy4¤F}Ìv@Nì¨ù9ß\•ÐÀoFœF––ËúZHƼ©'ó<Rí1§ªQ,Ö›·à_†Ù»¾ý(íØ1 ½!yñ:ÓøÍ߯e_›=µœC՛ћvPÙcEc¤L´wãË“y•Çñåáñ§ñØÔÑ­v¶C³á¸½Ãn 1 )QÎÓn?UÜÙø -¾9—çáFöN W{òƼ¤p’Ø,[À›ÓÆuô4¥”‡n¡Ðš¶íB¼6ÿ¶ª k¤ñ¼öŽ•ulN¿ßA.?$²Ãr~®½{ã¶Ú(ã´GcŽžÕIôˆý=œžËóh8MÇÜÆw°[¤~‰·ëØoŽ/„ŽÑ÷Œ‘à«NÌï¬RÏø:~`<¢±:ÉÏæNû´Cû>ÎQßÞ¢Ä"¦³ˆNæy84GYGþlÅHwž±(qó ãòƒäñvc±ÌV¹Oë2FÖ)ÑÒp`²­@Ÿö×!÷¢5>cÆ£«Èk\kÝÖ¶Í7îöž©ˆ/Gê¹<‡Ô;;{§h›os§ÙÝSGZ¸W÷QX‰½íŒPésï ¤ÍŒä¶Œ¨ò¬®7ÑÖu¡ ›é¼ë4þšÖD6œGÂÛÕÚQú,Jé­Ôµ¨Ú»F5™eFå&õ5e×ÆIÝ.‚ðU‹v̦g£Fïöœ…©ɵt¹¦D·‹˜n)Í·‘Š-UêYsíùnJ‡ºM­ãÙ=øé^°9¢lýºÓsˆm¬Ub¸1áíã6âç‹Bjß¹Bù -=.´¬Ü&Nªã¥Ki¯dþ†©YeôfDú¦ -ߤÄ<å‹v RôvÓ~Õ+¡¹ÙÄëNâòÐXÇI= ]ìiYwéfy餄ôDº-u뱕”u“Ç\Í#’jä±놺&& -«ýÓ¸’ U5Õu±oP~^‰»¾MU¯ZeA3¸ûVn‡ÆÏ\ÛˆkzUm³·•Ë0_…·âØÜ(&è§ZIÕ¼]¯Œ »a ….î4ú\Äu¶6v*ï‡X¾öI&yÜÛå—âæŒõ†ž‰ VÓ›×y䥯ÿEšïñûÏ‹»þ¡¾êsöãF5¶0[WbîÉèït¤“bú»Uئ Çh¿‹uêi}ECóãÆ0¾v=»êÄ^SêÆ ºi¹Ð÷>dbÓ k´e=ã„·¡[ ó¹cÓö|ikãèé>zIlµ -¼9¸L]g»K¾²"qEI»K‡Ö–¼P\² -s·le&’WÒºžìw|(,ÕÅ.XA—¶ë\w³¼è*+­`I[~Á všWšWöF¿zz}ö¸/&‡:HÚ¨½IŸ¹´5Œ¤ðd©-—a}Ù|Ž-ÂÕÉÛ{¨•eÙܳ-ÈéS£‡nuÁ1ʤ©g[x^©«²U¥ÕQ4º€LUY+1V›Â’­vKiq{)¶˜¢KÿÁ{ÚóQièù¨„MZ*·z¼-ÙyçØ[§zÊK&Axî&u=¥­²mÝn]wõ¶WÖ × =2U=›ß¤|ÿ½fîÌy†)Uý)»#:õ—cÃQ;­±åï©8ÄÑŸ¬ˆ.±í’ç9Þ?Ùî’Ôý^†crÖª¨é/×É&> -endobj -520 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 480.123057 101.433071 461.373057 ] -/BS << -/W 0 ->> -/Dest (cb79-1) ->> -endobj -521 0 obj -<< -/Filter /FlateDecode -/Length 4387 ->> -stream -xœí]Mܸ½÷¯èóËåGñ ö bdƒ,`$‡ ‡ñ¬½‹`À›Cþ~Õ"UM•ZjÙcËöz0žVI•ÈÇÇb‘jš³ÆÏ·ÿ%2*¥ßžÞ´Š^ãßâêbPÿˆßèÏFSTÞäüùm¸¨bö6†óÓ ÿu—<ùÓéßœþsÇœ­Ê9XãÏ¿¾.*r*¦µT*mœ ò^Åh}TÁ(öëϧï0øžþïéÍ7C¤¬þ8Cj‚…n`3šÛÃû6x|ÐÐF B,„{ò#bžŒ4 Å1èVŒÄðWþç®,‡á?·wíö£øåöèÄYÃð¦¯ClY|g–h°Æä£m@—ma'@~¥¦«Ä¿yPG+.1ÐvÉÌÇx‰É0Y~ùöôÝ_^üý¯?ü½|súçï´öáûo þRºüõ~ü›ç²µóqLÿfÔ…I×Ò×4µìÈÒ„ëüôØÝßLùŽ–¾Ù5ê´ÓèËïê½Kš×,Ý#>gvÿ7×vX=>oICóz˜ÙÏëhüÜò±ç WR½~ÿ¯ó˧?¼¼ÞàÀ#':G a«Û²¼VŸ¢˜ÚÞ‚h­NÖ|ÆU}tøvÛÛ{ž±¦-ð-P«å¶¿®{îŸ.ϵÁ‘¿ƒó „˜ÝB!œõ ê.-k\÷—ÆÚ,i˜NjñG†”®eÙu­ÚP¯Y Z™ârÂ}_OåRWnO~ž“æú*bkàdZóP—¶=ÇÚÝ=:oÌs8´ÑMÎk5×óëyË7DÕ>þø±‡¯ÇÑw•¥çHÜîR, ”9ƒeÑYæ·º-ÏázÛq|o>Ù2‚>ÿ|™½c±Ì%,!0›ŒõkˆÝ–çpˆ½íGÞ¼ô®ãá{Zôž)ÎÇêqϺ¿·Èˆ´Ì?ôViO¡ ç&"7æ9"ímŸò¸\ràÞrsFµ€8æ#jRÖP"ZCܶ<‡CÜm¿²Íj-[aôìZ|@åÃ8§dУ¦¶JÏ[K÷ô¯.-ø^Ñ Ït ðŽüBÊ^ùbk¨Ü–çp¨\ B•ÓnÌÂ6!ûŽÑýYzIÇ•‹ü8ù{¾,§4,iÞFâ¶<‡CâŠxÎøÂ{ÅŽQÜM>àf”nÌs4”º~c­Ab¾WºnaÞ:m¶PGð(£ˆ·rCÅ›®5Xy®<ÿС¼×Wtð^ÅüeËc–KoCH³µ%^\Bߎýó}÷Cö}ú{hŽågö^ÈòÎ7…Aß?÷Þ˜çh( Ûüĵ‹v™/ô|Mã‘Ýsñ]–ÏÙ]@§»ÿÍÃy‡Î÷ð?)¯|Àž³àçÎÖJº¨wëYiüõ—ß+y|Ê:ŠËØó÷ïòÙ˜çpØ»íõÇ8n©ËWSZËê³zt?uŸ»z_ÅS?Cg3óÙ~ánW¸rSWc8S ÞâÒêÈÀ }ùu·OžÛÒæQÂ*Pë#]ÚÁÖ÷ÇëïboÌs8T®xÝ{W|UbëÍ{ -¶" ²ç¢ßгªîꃯGök|ÞÖÍí®¾¨àaêKˆŒ;vCnÌs4DÆ•Uhþnl¿zÃ<£wñ5AÒW³ã¥ÝÙ«­£…ÅÙ;÷ܺ^#½1ããnpÆ¡ýŠw¿W¤=…®NVÞx_ÇãŽw¹7æ9>oû³¶XØQ»þÖ¼^µ2åö¾Ú{/–$\ -ûŸkÜžÕøÅsro± ²][EܘçpÈ\Y÷~†±üÈ‘Ï–1Ë߶يÙmy‡ÙÛþç¬÷ïYEYaIqÅÄ ÌtcÅä}ì»w ôÙ£ Lšôýo0nÌs4T¦•(æýþêGdè»êq•î~®Ü˜çp¨Üø†ÍŽw¾,þÚÑC?hL©;Þ¾Ù˜çpH]ñD¿ -¾Zè‘kßøQzâBÓýßœ±1Ïáº÷<È÷]}žŒÎ¬ïŸÉoÌs4tægò??SÏà9¿ïê9Fúìîß¹¾1Ïáºâ“~–üóåôyÇÛ@ó¡ÏóÍ–wïïøm¤ß>Òïø†¢y‡Î•oCß;ïýGú=;ªža¤ÿq<#z4ª÷å|¢äMbÇm4x®&YAãüpŽzÄŹüôp #Îœ UÆué G’r,gÉx½øãŸÿô7¯òº³}}ùrËf.|øEu¹p¹.Õ1„½M }C?¼&]‰äB"~ß”*»T½·—¿¥¬’žÞ\š ÈmI.¿CYãç¡œT‡Øñ~N%9Ç!^ôWåûñÞÌž¡ Ïž¡†ôó¬)ßÂp.’ÎÆÓ!)cËá¹ç@Ê•¶ÉçrSÒŽLm:3T¶9/äŽ~×R§ër0a9”êrD1»Ó¿ýíÉdR¥þ Y\N Õƒ8Ùz1‰ŸNåÄXÄ„ùøFqŽu+‹LMØdàܨrŸÒy¥Ç¹î|ýh}}š’ß×lå$Ò5‘β¹ÜX"g0sqyz­Ì(.§î&åÆ«ÐLñ¡Ú áT ñ´SSÆU±qš ý\¨Ùí&±¥j)§´µª “°™Î«Î°æ¯imdÍyKøxÂm]/}¥å¬ÏZT­]«›Ì1£r“úš²5µvD'Aø­®E³éY«•Ó.ÂTD-]®éÑãIL7–æ[KÅ–*MYs­ùÉ” ê.µŠdCÒXŽg}D»úñZJ Ô¶Ö‰áÆ΄' 14?|Rû‰+´¯ÐãBÇÊmâ¤'¼LÒ²Ññò ]³ÊÊySåÿ Uø&-Ì3|0Ô UÎŒ›®¦›”¾ÙÄוÄ塱I5`(NiYu™fy}™¤…ô…t[êVcWRVeLs5¯T#k±i¨kâBaµ~Wr¡®Fàv“Ø7(?]‰';}ëª{½Y0 ;‰CãÎ'.Ž­ÅM9°ÙÛÊe˜¯ÂÇa`£^\ ŸêM&±Õm´›nÆ„“a …I<ÑèÓ ®½µ±ÓðÍñãÇ©“I#îãé—Ó0ÌYçmƇàL9Ï–{^æü?¤yߟèüC=@í2Ž[ÝØÂ6n½󑬜~ž6ŠËièUغ Zb´?‰MšÒúŠ†6Ž[Ëøš¦ìz"öšÒ45MË…~ªC&¶ ºÖ8V3þ†ð±«ÅæȼoÛLöåÒÕ‡Ãm\9z¯Z… þ8¸L“ÎMCò™‰«’tÒ¡%Ë2â‚ŠKvÃqi{°"m„'KÝp®/Û˜ãÎå¢vÞ©†ZYŽõ=לœ©kL®]]pŒ2iš² <¿)UÙÕMë@Ñè2]e­ÄXm -c¶Z-Ã'±–bó)&é/¼¦=o•V‰ž·J˜¥-åÖ¯yK®9@žÃ9NO§§”Í—L‚,ðÜMJSJWeó{Óõ½ëh{fA“!aòLõ”ÍÏRîâ¿=}ç’§‹DH©êÏ1i:r5™è¬;çâî¤À!T”X¸äî<·ã'ó(IŸð;¡99§£ÞLx‚"¯+XQOãi­r¶¬ÅGº¯„ðÐDszrÊ!Í!HÔ.c™*ú<Û­S*ÇÖGmÄpÑ°3Ò]…O†ŸÿÈÏîR -endstream -endobj -522 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 521 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -523 0 obj -<< -/Filter /FlateDecode -/Length 4804 ->> -stream -xœå]K$9¾×¯¨3Ò&~?$´$@¬±ÒˆÃLÍ4õ"-øû|áLÛ‘ÎȬ잞¥FL«§*#üÛŸ?G8ÙúªðóÆÉé)¥ƒ»Þ~¸üxQSô -ÿJŠÕEQßè¯Z¹8yCò×H`ã³71\Ÿ/øo¸äÉŸ/þÙåŸë<új¦œƒÑþú¯O¤rvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS@§IgôR6YÔEèt4’ÎÚ)D§½–ty -!ø$élñYiIça‹×Î)I[œMQÒØbƒU¢¶˜­¤‹°Å w² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Å´Õå'Ÿ¬U’NëÉ›VÔaFaBt¶峨ƒ-Þ*#é,lqø‘tà ocJÂl54»­²ZÒyØbLt¢¶`ð%0¶¨$ -À šˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lq˜¹¢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.¦{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]k‘SÔÁXâ°ãdu²8ð®Õ*J\àÀ»˜FFâŒê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„9h%.pà]ãÑiå﯌ļkœ (øG‰Þxâew¶€`%.Àz± °ÉÀ-‘¸ÀƒwuÆò'Ìi°Õ¤Sxð®N>Q—'Ñã¼E/O:„,qïb¡²xð.è3I\ŽŸ°¢‰ à7-r)ϧ®×~ÑÕ¼C>ŸwÊŠKúÅÆšÿÅe)f÷Ò­×6ú<=ý*O­?­íhö õ5ûj¿¾_úïÆúò=¥ýö¯×wß]~õn º-Í ƒ#l´KÎÝÁé¹,ŸƒÓ/ÏœOÁ3õn¬CX‡‡†®tyb]ûiþôf‘-Ý¿Ö:‘Á%³¡l0j`fSƒÁöëœR2 é®Q•¥wÙÜãË“y ˆˆÜ‘8A]†º‚$ Ã(-éÿ7Äó…f×@ªŸ7›vPiÝ‹Wñ“y•î?~U\óF³éäòÝPª$¾g¨öëúÅYXË©³¥Ù·ƒÒî8ºìdÒŠ|¥çò<J̆”¡7<:'8Æ#‚jú3#^ëmhÈø¨¥c¶¨ø}øUšY£5ïÉ~¨éÏ#pÍÇn(o¡Ýe<Ðsy¡'ýÌ:‚ŒÓøJ¸á­=$~b£4pË^úš}ÌÑËq#ZwVu£^›ŸÌóhh4ç|ͷ౺R~Gt{;äüÙ‚^4ZÛEiþÜ-gl“ä/þëk׈ ›µí{\iì˹òdž‡CçŸsôÅþ/ãå–õYá"ÙÆ$Ý`N©Ü,=Fä¹<‡È;þå0­W¼ûWq˜Ä…¯ð †8{ÃÃ;‘ÐÈ™ùìÚøÛhéËÍÚ´¾bGódž‡C뱯ùÕrÚšOÄâãÖj&þ¤3HF§í~¤víl£WwÐy2Ï£¡Óžô=w¼wi‡¦ÒŽ/F\²ñ_+¯ŽI»ANmÉS‰½k[Ú%»—léð¾¶Ú¤Ò–ï~ýÛßü‘µ¥ð7íK2_Xt¦֎µ­}Í Ûæ´>Ò̬mœ¯7ò%m‹ï{º¾ÞP¿áóc/³\›µ]e†(Û÷ºJ>¯Ö;YjÖ·vÅñ§º3ñî,h¸õ’\ ‘e#·XR{Ž,²¨ÞÐ/,3a–©ó5ïÖJÅZŸæ2Õm±8,¦n#}ÙÒó!‹ÖW;¨ŽÅF”£?nË –OÑým9eDk¯†Ò®#;è™ÙÕÛBý·Ýyæ¶ÑƒA9²PöŒØ^æÔÏ–>”S;ŠM¾®ó÷ÚV²ÇÅž¶´?-c}«Âr-´£ °¦yb³ ys}³Ç°¤«ë–_ôz)ÿC·¯z;}ŸÆþ)e}šó×u‘Ê.w4Re«¥^Öî:»šµÍOqþ,å}ZðµŒs›åv链&Öñjîe×}8Žc³)ô¹ÑÆMwûÁžýäÕØO­qÀÇžÖlÖõ¾ŽŸÈ`ªÝ š™kÍZž3Wí“$´3Îy+ÓÙ‚©™‡F†^³óÌ]œ·VœuÔoFšÛ+^{êد}¡–¹ã?ôv•¶.óBü2}ZºHýü4Ú|Ø^D…ëöÇúë¶æÓÑwòÉOÁ–#=ôÐ -Js×w:™çU«ÞÐiòý Pž’÷ Ñi*ô0dºÒ s0æAØ­0¸‘ô@ÂPá_†-æ”åE~\®±yòQ(×?}{'·Kå)žö¬º“߇){Œ”¿¸.ƒ«éO”Yu>؃ž:ÚöäD¡6r/¨N´ßÙrZNî¾¢÷é™-t^ˆ»-½“ŸFœaÍÛõ¾K3.9¸*Ç8õ™Æa]£ó/òTp˜é\Dwß,䢕ci÷œÅ­3FOAfzÞh¨´Ìö" t|{&5AÞfd:&“y$ˆl|ÊÓ »|¨ÞÓ’´xÆr½D-b˜ó6Ì|`³4°Wëó{Q‹Ï¢Zâ²TûÅ¥àˆÚ¸$šÜ”³OQ¡)ºX‘áU8·av2Ï£¡ +ÞšîݲiŒSÿ÷ó÷Ü·¡ÜŽ®ã쨕<ºÎ–Î.¿—O·|§ò9ãýTëÏ/¸S9l¯ [4÷6Â"=f² èìMØ“yuÝlzjíuà ‚͆÷p@îÌfz½¸.ãKÜê:„¶µ…³u gçõˆT›ƒ.G'¤cî\wöhêÉ<†Ô¤ùÑȉ5_óÍW «C„g‘¥å²~*¤Òc§/EêÉ<‡T{Ì©j@óïæ­:Æ—a^]ß~”vìÐÞÎVñ:ÓøÍáŸÊ¾ñ`wÙ"J[{¶6í=aҸИ0)í‰GLNåy8Tû—‡Ç£ÆcUG·âٮ̆ãöÃ1Ä4¤D9O»EU9podã+øæ\ž‡Ù;>\íÉ[%…ÃÆfÙ*Þ{°¢¬|ÃÍMÊC·Z(Žm;¯ÍÿE5ÃCÜ9ÎHnˈ*ÏêzÃC™™{}'ŸÌópH¼ã)>ò¡Ì‘ë¬Ù9Â<"£íPÞAÞx‡eƒØ×>¬Y=7|8¿b÷ïdžGC®VÇÎã#oá}ñg€Ï>QÏQ·î§ŽÒbЯáÍfSF^þ¿_Þò=FÙ{zË^ò:±—F5¼ÞMržõ–çö%8W=¼xh}WEÒëÝmº 0Gz#ÚÞ=QóiŽyõrì‰ÖÛÏ.G> íðm^ÒÔ{©ì¸_;æ¶ß«Ç£Ê}ÕåhZ9>•g¹iG3U¿‹;ôªÚñ/SŪùH;~ÖÊ?8~ØÚàk6Cùã%”·ûéò†WÒ¤ ½þÜdilò•Þ(˜”uº.­¯;9Ë‹ól©–wé+z½.½Z± -¿XÒÿXý‡‹În¢þ/gzË{¯U'S/ºøùBï=w‚ع~0øŽ˜„ËWÕÊrº3®²H8NyLiý¤–¯ý02Ýž¿_[Cù}ÍFï£sª&²ålsÅ‚9ƒÞŠ©õjÒ‹˜ÞŸ&»\…f -[jÂ^ ãi{=ã]±¶Š ýV¨Xu]l\µGÓ»F[×….l¦ó®ÓløkZÙp oTkGé³(¥7V×¢jïÕd–•›Ô×”m@\'At»ÂoT-Úa0›ž½÷s¦ -$×ÒåšNÝ.bº¥4ßF*¶T©g͵ç»)ê6µŽd7v`?ÒÃ`sDÙúu-¦3äm¬Ub¸1áí‚%ÆmÄÏ…Ô¾s…òz\hY¹MœTÇK—ÆT» -S³Êè­‰ôÿL¾I‰yÊí¤èͧýªWBs³‰×Ä塱Ž“z@»ØÓ²îÒ16ÌòþÒI é‰t[êÖc+)ë2&¹šG$ÕÈc-Ö uMLVû§q%ªjªëbß ü¼w;}›ª^µÊ‚fp÷­Ü.;Ÿ¹8¶×ôÛfo+—a¾ -oeas£˜ Ÿj%]lT[ízeLØ )tq§Ñç"®³µ±SyÄòµO2iŽ]þ~)Ëœ±ÞÐÙä`5½•{^úú¤ù¿ÿ¸¸ëïëk@çuܨƦqëJÌW2ú餘þ¦G¶iÄÆ1ÚïbzZ_ÑÐÖqc_»ž]ub¯)ucPÝ´\è{2±iÐ5Ú²žñÂÛЋ͑ùܱé ö|ikãè”5½@¶Z… Þ\¦®³}I¾²"qEIû’­3,#.x¡¸dænÙÊL$¯¤u=ÙïøPXª‹]07‚.m×¹¾Ìò¢«¬´‚% lù/Øi^i^Ù#ýêéõÙ㾘ê i£ö&}æÒÖ0’6“¥¶\†õe[sl9’\'oï¡V–esÏ6'§OîZ¸ÕÇ(“¦žmIày¥®ÊV•Ö…¢ÑdªÊZ‰±Ú–lµ[J‹“ØK±ù]úwÞÓžJëDÏG%lÒR¹uÅkÞ’mçpŽ½uª§l¾ddçnR×SÚ*ÛÖíÖu×ÕöÊšáº!¡{¦ªg󛔯â¿×Ì9Ï°!¥ª?ewD§þâl,ÔNGklù[h*…qôç,¢Kl»äÅyŽ÷O¶»$uÿ„ׄᘜµ*jú«v² ÏPäû -VTý‹wr¶¬Ä&½¬„í¡Nsj³;Tþ¬QÙã,›Dí2R¨HÅÌ’Ó›è¯D¥åg°é@µ[mŸ”Ÿÿ}XµÁ -endstream -endobj -524 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 523 0 R -/Resources 4 0 R -/Annots [ 525 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -525 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 405.123057 101.433071 386.373057 ] -/BS << -/W 0 ->> -/Dest (cb82-1) ->> -endobj -526 0 obj -<< -/Filter /FlateDecode -/Length 4482 ->> -stream -xœí]ˮܸÝ÷Wô:Àhø(¾€`’ F&ˆ#YYØwì×›!%o´;ÿú6«È!† œ JP)m•sCÆyAåõ€GLIRÁ ÈÐZ`¥è$Ì c¢ Š0ÃRÐv­J0ÃM’ -fèä‚ Ò -vh£”¨ƒ!ŠÈA§ÝàQè^Ò=ø˜Lu:Œ¤³vð´Ó’. Þ{%Á—”–t¶8M¤$l!ƒ¤ó°Åz«Dl11XI`‹Aé$Aa ÐElQ^iI—ìàR$u :C\ëŒrƒ‹Ö*I§õà€M+êТPI>:[¼rIÔÁg•‘t¶þ$xÂÙ£ÐZMnÝVY-él1&¨ƒ-¨|‰Œ‡-*x‰Ђ†ÌK˜J†$0Ñ)J,`b(+Ñ€In Ÿ‚ÄVi茖ˆÀ*ØâÈKL`5l!´\Q[l"‰ ¬- qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ]4÷ qëZçÄ´k)z‰ ,x×’Vx×ZÄu°–¸ì8X­ÄÞµZ‰ ¼‹fd$.@­À¡—¸XLŒJâHâ.@Ô…mÐJ\@à]ãðRÏ Þ5N‰ ¼kÈx‰ ”ü£Do¼ -q²;[@° ¿‡ÎX‰ Ø`à–H\àÀ»:¡ûÚ4ØjÐ1y‰ xWGTŸ¨Kƒ(q¡Ý¢”í}’¸ÀwÑQY‰ xô%.ÇèQļ«-zjIÞÕ&)‰ x~ƒ“¸p4‰ п Zº¢. *Å(qï‚–ŒÄè…Á$¢. ’–¸d<(ø¡xð.º÷$qï*g­Ä¼«n¨ƒ-6‰ à×@‡f+´i8«ƒ2` -IÞU5,éÀ»J…$qïænSâ²ÀD‰ `à/ØH\à# )/qO -:¸|¢Î¸we‰:ØB$.€c=$›¢ÄÁÀkŒÄÁÀ¸ À#’†÷-` à"Ž¶ÀS”¸L6ÄD$qApã1”ª¤ófˆt-ê"t:H\ 芵ÄK ^âüÝ%‰ `tš$. ¶ ©H\€®hˆž¹¤Ó°N¶Äðt¡ÓZâ‚h` ¼A‰ ¢‰ez%qzhŒ%QÔ’ŽTgF‰ "ù!»Š ™`¤ ¦u°4!qê'¶Äü`°‰aŽ¨ƒ-Ö‘Ä1š<ÜŒÄ[Œ2›y¼$.H -¶À㓸 )?d7YâÐ8tJI\tÄxÓÄh—y¼%.HVa¼‰!žÐn“õy¼$.H[\Q[à4%¡Ý‚“0Þ;I:[L"‰ ’÷y¼%.€›ñ&¢Á -ƒ#ô¶al2™9ø•Š÷ê NV\æ@ë-3ý%ÃdÜùÕûÓ÷~ñ·¿üø0zõîôß*x?|§ñMï.ß.N÷¯ñy˜d€ˆó—k“}.=ÉÍ6LßS\Œ|êu‰ç¦0ÎMa§´©Ñ×<Íœ·z‡4í:?ëç|j:QΧæÿФõnÞ‡ü —ëlBþÔgÒ,œ›Ã—²¼ö,ZýðÏó«§ß¿Z‚lô¤¶Í7M¥™Ãø)œTãÂ~wÀJ š¦æã.+äûvN—Ú‘JÃîO ´Î§wâBûh›ý¿îž¹3Îáжã3Rƒ„R‚oÖ5/M§Çû¸ýéüä „2?0opÃCú½9¡Î8‡CèußñÉ|ÒÓƒ>ÿ|­c±Ì'ÌË­&jãöÛçpˆ½îGÞŒ¼+õv™ÞM5zË(çSµ¸Æg½¿µÈˆ´Ì?tfPŽ|FÏUDvÆ9"íuŸò¸\ràÖruDµ8æ#*ÆiŽ¹ÈÎ8‡CÜu¿rk-Gœ a#•²FCŒZÔ”ZiyksýèÍ¥ŸÃ+Ðjã™4ÃcÙ"vï ygœÃ¡rgò¨œveÖ…ìz÷gi% Wnò#_¶Î[1ã¸}î:ûâ‰;>âa8ã+owôâÄÖ¬{QÚçh(¥N¿±” 1ßkc“GYíÁ¯íÚhzùŠœR›¯Yz|¶Z3„(Ù&ÃfµÇÝ­­q?Ï«>u+¾}ö2Ü@±½}<Þçp(Þ™ã¼×ç#&ûùúän¢ñŽùÌÎ8‡CãÎ|æ­üø„Ÿ„¿‹Œ¯6¨ÃÝnE¡À·5 Ö"sÞãæÔ‰£›PUÃÓkDÄæ&}B=kR{+’q‡Ø뾪‰ûµûY÷Zè…K8fÏ=È»MˆÛaWêØ\d/;ã îóû¡®Ig…Œ+þæç@™Û¶®ZãÍëÿˆœýÃüz{ÔDnoݼ3ÎáÙ9¿ymwD~K,L¿_+ _­]af¸«®Ýa¼å×63á®AAkÇÇÚÛãîØÙçp¨¹-{Á3íسá¦Ñlt|Ƽæñn™Žf{F”ñôšQRˇ+[øZI3f_íU×ÇKÏÞ:ç)›sì­tÆ9bÿ?O¹hY{im¡²cæ}h—êÙ|cïJcgœ£!Ô＋s'ÔÚ`ïìŒ(lV#eÞø[Æíík:q‡ÄŸðP¼ñ„ûŠÌ™¯ÔçÝÉe§òä÷®v1›)\zfŽtwpd_œÃ!sgÝ[«Í¶{özw×¾Ÿj˜Š¼ìiO—ï­š¯£Œv÷qáù× Ê[}AoUÌ_6|Îrëmi´¶Å‹[è»c?dgœÃ¡¯s?dÛ¦¹‡æX~fë…lï|“Qîxg»3ÎÑPúüÄOµ‹v›/ÔzMãå¹ù.Ë—ì‡n ÓÞþîagœÃ¡ó ¾ãgå•Ør6üÜÕZI3ë][Vœ>îòYȃàS–^\Æž»}—OgœÃaïºw8–[˜æqsY¾™ËÐVžÅ£û©¹nÊ}OíÌWû…›]AâÊMYáL%x‹[«##ƒ¶é—Ý>imKG «@½?©´Ù_ßñ6vgœÃ¡rÇkl¡†­JÜÛ§Ýõf‹¹ ·þ¨Ö¦ßвªjʃ¯G¶k|ÜÖŒí¿Uðzn›?NuÇnÈÎ8GCdÜY…æïƶ«7Ì3Zq_$µoíÎÞ­ý+5,ŽÞ¹çÖ´éý‹7k„+mW¼Û½"íÜ“oÊdç÷í~<Þñ.wgœÃá󺹪‹µûoÍ«Ý£šÆƒ\ß‹¸·bIÂ¥°/üY±ÆíÙ¿Ø`Nî-öþ®__œÃ!sgÝûúò#Ïì|°ŒÙ{~‹òËü1ÊxÝÿ\µþ{VQvXR\1q3]Y1yŠ}·®>û,Á“&uûŒqŽ†ÊÔ÷Ë“_çû«Ÿ¡o*Ç TÚÛ¹²3ÎáPÙù†Í=¿ùUñ×-ô£Îéo õŽ·o:ã©;žè7ÁW-rï7?IKÜ@h¼ý—3:ã¡;óžù½«oÓ3¨è|9{*Ò¹|îPt:²c4*$wƒì p}èF9ºâœÿZˆù [Îļµñ §‘RÈgÄdH½øßþøW©üj‰y{y‘ë²p†kÏÃå…|Ÿ‹jbè2Ì€~|%¥TÏ¥jÜØUæâ¼T…š¶øäïË™ ¹X/ÐÉr“ÃúËgLkºÓ‰–S~™4rüü³¹á¢_¤ï¦¼™=cŽ=C>¥UU~8ùñ¼#=žy§|´É‡âž= 6×M:ç3–¢²¤KÕé±°õy#æx”ð‡j:]XåóaS—£‡ÙÍþÃXéïO:ÑË?Äå$P5Š£)7³øñ”O‚%ALà¾NqN—ª¦Eº¬²ðlRÒºAM—©&n]¹4®Ã³$UJר*³Ì¨T¥®„¬BµžÑÃI~§J҄ʬzVkù$´‹0 Q —J8AôpÃM©¹ZS¡†ŠsÔTJ~6e†ºµàÙXF“4äãÁYQ¶\.Å>×@©knÌJøpBC+ñãÉ!´›¹B¹=.´,Ý*ŽjÆË,Í‹Ê—+4Í"ËçHåÿ/Táª43Ïx¡©B*Ÿ7ßÍ™ä¶YÅËBâr_Y‡¤Ðæ°¬¸t³¼¼tTBøLº5t-±…”“‡TÌË$UÉc)ÖuUœ)¬”OåJ.TÅd7‹]…òãB<ÛéjSØKf^3¸»šî,ö•;¹8Ô×ù`¿joM—a¾ÆŽZq†~,™Ìb£jo7gÆ„³a …Y<Óèã(.­µ²ÓøkïÓåÜȤ÷áôËiìæŒu&áÂ[Ï©åž—>ÿa^àó¯,£]úq£*[˜Ê­ 1ïÉò©æ±SœO9/ÂÚl˜Ð£ýY¬ãÖ4Ô~ÜÆ×4GW3±—º2¨®Z.ts2±©Ð5Ú²’qW„M)VGæ©u3wØ—[[ÙØ|¤^± -7üqpg»ä3Kw9èÜ¥CK†EÄ O·,Ã4[¶03’gR‹>ÛO¼*l΋Ý07"ßÚYGs7Ë“.²ñ)X¸À–ßð„IóLÓÂÁ軛דë}2Ù—¥•Ú«ô‘Këƒei%Òm LÍ4§V³Cv°Ó4æ8IToC*º4Ç­bÌÇÑ¥Åé¢Ë*ôbúdüû˜ã -endstream -endobj -527 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 526 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -528 0 obj -<< -/Filter /FlateDecode -/Length 4446 ->> -stream -xœí]KÇ ¾Ï¯˜³·ëÁz†’ FÄ€Œä±d#XsÈßÏÇêzu7§§gµ²GN´ÐÎ4Y‹üHÖîÔê³Â×çß"é)ÆIXà€»TVÂÜ|F €ñ"ª‘°ÀwµE¤–xÀ]m’’°Àw‘78 `“F aâÛ¤LWä¥I¥%,pÀ]À’‘°QxRH"òÂCÒŒ'…‘ç'<; ‚Á²Dd¡@ ±ž’MQ‚` ‹5F‚` R LI#ûËÈsë‚,È%,’M1IXœG=­JûùÇÓ¯|Ï?þûôö³|RV¿4„Üš_Á LÂv;dßË·,KÈ¢A²»ó´!u±4€°³±Xxüï¼}³¢²°èJÈsõñxÍ{òè„2á(Ùe½»ÏM‘r‚tno´,k×Ñ8oW’$ëÝ}^HÖ®#`#@Þ!Å<®×ƒ}^\¯ã¼Gõz°ÏKÉj›Ÿ$nš·!(œ‡â.þAÔ;»¼ ƒñ$È=ÚØd8~Qù(Z4þÝ« ´H1ŸÒ·ÇA|dt¡Ñ¸ó«w§/þüõßþòÍ·ÐÓ«·§ï¾TŠüWŸk~}[^/å5ίæÂüo{{*4ÝiΕ×Ò^ÙÞoÑ·Ìåê~Ù¯ÍízÿÚ¦É÷z–­>ç÷¯ËUO»b¶C!À ÊåÚaß:õy8ë¤]ü¼KÓ¿ñ0·Çýa^IæÑ8°ÏÃYê„ô“ÄŸò°GŽôCZ©h2š"Ñ- =Öçá,ô`.»&ë.6Œ,8Ñv¨ìnÎ÷L¡•øT¿/ªu °Ï£Y¢¹‘sþ‹÷á]¿@7öþJý`Ÿ‡³Êý\óÓÄšò¦ƒ¼Y©^YâëÁªW9­è…uœê-M¾+VÚsFJnr9$ß²Òc}ÎJ÷óÌf)+ npÌwL ÖTÛÙñ:ÇZ†fyëêÅmeQ?àÿÛãÍ:Ãh}꡶?nK<¦ÕxW-´çŒÇ-ôXŸ‡³Ð;Ï<L#á·®Yâ^½|—>åsåÐÚZ¯Du«î¯Ëöy4k´ÇrÍ—À±)?Ä×ÖíìªÏ*ŸÍÖ‹Ek[¬(ίWÇY¯IÊWùëscăÍRökXiíýXy°ÏÃYçœs‹ýOÖËwŽõAá‹X~ðÄý`Ÿ‡³Èùåj'šVw²ûga˜„…Ïȳ5¬êì _©„Ö˜Ùpqô®Mþ·­–>ž×^±ÖgœiìópÖºŸk~²˜vÍšÔâ묅'þ¢$['õ¦†ZW•Óà2öµõÕ¶oçùu¡ÛA×¹mñÏqÏ·:(û‚¹,¿^V:sƒÿ¹îïy_žþ¼Zë/ûA•§è°­Å_/ÓzN8Ë¥«ý]fÛš³•Áü ›‘ÿ¦è°ŽA¿;ËmÛšK¶ †±ë:ª¯—}Ïã­ì½Fþ+:3›£¢³Ëõ=áýwÕŸUÇåã~PÚW{^ûpË sŒþêÊzË~µÓǢìS7ØkÑÝ2. ~U±h_W´±¯‡Ôƒæqs¡ûö ÇØrG\y»k\ó¡u-}¿ÇÕYþ5Ć^/u}š}=…´Òûº[aBÅýuF”›¼Í¿´ÏNG™nnf¤û<ëý_0uM^ä#Ñ9¸«˜Šf7^—¹ºÂÂNÍ›puÂä‘œóWÎat((jÚ‘ -}\cÓä<‚0®{ûÕÞó§õ½¿&ÕþÎOÉaï¤þ9˜­xw`Ì|!ÙØÑÔÞaòX–Õj;wG™z`ýdó§RùzÏÐ>ßÍåùpu¥7úóîk¬y9íS4𸈚憺4ìÆî\Ï(©‚‡{ ÝWR•Ôê7[NÕ”»”$îõ ÷ YËYSµ#ª£ê¿ûr¹Ò,a]é+Ù+®æ€3¬ºöSjls¢²[dE$HÖ¡½ÿ°"k;Î"ÑÉA{W—®É¡úZžWdmÇž-Z}¤$¾·}œä|W÷a­Ÿ_'‘W”œ:\ïc“É«E’=û^—¿€i­§uA÷ú²œ¿Í3س3}M’¼/—ð«Ý$þh1¢F»•/ì­Iot¶ò7s,â«7™×:"J;Ð9ê~.¬ªOÔ1ÃÒÇgäQ_B|¶½Ýµò]‹µÞS$¾’)E«èxap°ÏÉZ®CÔþ* –¾\a°÷žÂàŠT‡ ƒmÿ/ ®kêñ -ƒgh( ž¥ý¡0x1í…Á¾ÞU|[î³9ÇwE§ãpµFC’›MnG-A¶—œõꈚ’/Áµ^1Q -|oŒT•pÈܦý¢Þû9³ïÓP½¤Ò¦ÆÚ!j9@ÉmjœÍ½%–åøšfºiyë¿xµ94Ïó•Cæœï ±¥¿“›µ5¸º†¯Ö[ùþäóH:߃§|œ´á‹rÏž&>£éÌ÷.EeI×­ÓYÙú|¥g¾^ø}kÅßAR| !_@•·v|(íßçM‡²‡&_ |;¨ÊdX´[“ŸN|;, d¢^-Ý 3±¼Um,Ò­øª´ Âc˜Òº%B*o{…ÆqIl­«áþ®vã[{HÕF6ß[3O,ÑÓë-™W¯&]È|ÃnœlyòMÆöFìÒضØ;Þ$k«¢ÛÕ0]'ªòh¾‘­©Îwb}T¶¿¶5aØÎ=âå„iíšú$Rù^Ï:TÕ®Qf¡R£ºÚ²mµ}H—“@ü\Õ¡ ›Ùøîñíh31VC¢Ö.ÕvérÛ•Ñ\Û©ÐZÅÞ5UÍwQº©ÛØ/Ð.Ã)Fà+ÃQ¶¾]’¹Àn{­â`7fC¼œbhC~:¹ ´v+”«¦7í0n#GÕí¥SC¬ª‚kVß-Åßg¨pÊÈ“ßhj&Å÷Ãõ§> ûf#/•4Ò}C’4 )ô¶ƒºtÍfG}騄ö º­uÓØ‚:¨l £ìWmhßÀcIÖÍê™!¬ê§aåHTUL×É®™òÓ‚ÜåtÍUj“y=˜»kãv²oØù4’CÛqÍ—ý5yÛ¸ƒÍWâ%6Z“Ùôc¤“jÑ®O6»`…Nî0ú”ÉÕ[:åÏΖ·Ýɤˆ{9ýtÊaÎXg¸€óHÃò>$}þÚ|ÿÿ<Ñù›zYÚÇjha¶.Èc$ã›ÎãA2ß|^‰Ím¢¡ö;YÇÞÖUkhqܘ¯©wWØkKÝT7îHt]‡Ù4Ó5Úšq;ÄËJ‹-‘ùнé{~´uq\Šò5{U*<ŒËÁcì<ÛCòyOÜ´‡tpÉ ñ0ŠÇaÂÔ%[ˆÉ9NÒTÏòÓ¸–ç†4‚mçQ³ãЕ–W14A -lLJq`Òã¤i! ô³Ýëƒ÷½ˆì«0µA{£>Ô¶0¦6À“©6?úåc‹9öÌÕy»†ÚXvð=Û’œî=µ ÅÃh£5ön¥'¥J[LZEƒ ÐT¥µC•É—nU-yÅQÔRh9E§þ4jÚ»Ò”èÆ]ñ›¶> -endobj -530 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 315.123057 101.433071 296.373057 ] -/BS << -/W 0 ->> -/Dest (cb85-1) ->> -endobj -531 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 108.376036 101.433071 89.626036 ] -/BS << -/W 0 ->> -/Dest (cb86-1) ->> -endobj -532 0 obj -<< -/Filter /FlateDecode -/Length 4629 ->> -stream -xœí]M丑½ç¯È³‘ùü  Ø‹¬;@c}X졦zƆQ³Àxû÷÷QÉH*”bfWwkîBu¥"ø"ƒ!¦¨¯ -?ßhüIO1¦àéúúÓåç‹š‚Sø7§¸¹˜Õßá7¸«V&§“îúSØ0…äLð×· þë.yò·Ë_~sùŸÛ<új¦”¼Ñîú²ŠìbÊ ª•ÒVP97…`œT^O¸Å”$Ìp mUfPŠNRÁ 2& -ª3,m·ª3LÐ$©`†N.*­`‡6J‰:¢ˆ¬tÚMî%Ñ“É$Q ÓÁH:k'H;-éÒä½wQÒlqIiIç`‹ÓDJÒÁ²1H:[¬·JÔÁƒ•t¶´Nt¶]QÔÁå•–tÉN.E"Q— S1Ä­Î(7¹h­’tZOØ´¢# -䃠3°Å+—DlqVIga áGÒ'œ 1 -£ÕäÑm•Õ’ÎÁc‰:؂ΗÀxØ¢‚—(#hʼ$q€ a¢dH"íD‘¢Ä&¦‰B° ˜ä&ò)H<`•†Îh‰¬‚-Ž¼ÄVÃÂÈu°Å&’¸ÀØ‚X [ Y‰ ,Ø–€‰ ,è–TÒXð-)í%.° \˜©$.°`\=I\`A¹€R’¸À‚s-¸Sâ ÒÅpX°®uÞH\`A»–¢—¸À‚w-i%qw­ENQ[`‰ ÀŽ“ÕÑJ\@à]«U¸€À»FFâôêz‰ €•ÉĨ$. 𮉊$. ð® ˜D]˜0­ÄÞ5· ÍÜà]㔑¸€À»†Œ—¸@™À?JôÀ» '»°+qæ{茕¸…Mn‰Ä¼«¦?aLƒ­&“—¸ÀwuD÷‰º4é€Æ-ZyÒÞ'‰ x••¸ÀwAŸQâpü„ÕH\àÀ»Úb¦–tà]m’’¸Àwá78‰ ‡IøóÛ¤ +êÒ¤RŒ8ð.hÉH\€YxRL"ê i‰ @Æ“‚*qïbzOxð®rÖJ\àÁ»Šàöˆ:Øb£‘¸~ t¶Â˜†³:)¦tà]¥ÑÃ’¼«THxðnž6%.ð! L”¸Nð‚Ä>Ò”Bðø¤ ƒË'êü„k'qAP8P–¨ƒ-Hâ8ÖS²)J\ l±ÆH\ lË q<‚)ixßâò¶.âú€` ‰ ’òSv“%.C§”ÄIG¬7 I\€q™×›Qâ‚dÖ›Xâ ã6YŸ×›Aâ‚D°ÅÁEu°NSÆ-8 ëM°“¤ó°Å$’¸ yŸ×›Qâ¸Xo£C1¯7ƒ.ïºf’tà]x¦^â\`½ ÏVZÛ+¯ ðÏdeÌ+Î(F ¨×anC -Ü‹Ù*ˆ1òÍk$1h3±²Œ^Œ`Z…R+1l À¿ðÇhà Kô‰‡¨<à0gÆo2aX¼ûºæhÉìN²%>ö¿^~û‚É÷ú×ÿ½üø›9RV~4„95Ï&¡»¼oÛ·Ž\ž—3GeaßY² !55Hq ¹e1ùå×퇻aöw·µc -ªpgg!‰&fzG£·ð‡°Íéô¹l,ô|n`Èp@õá§Ëo¿ýÓ¿þË|”|øñò_¿ÃˆJ¹ïñ Â%|&L˜Z–Ï!ûO9ò°\»¬Ëòsšßÿ÷õ÷¼N“Ý)»··uæjôÇ%ù?ÌMgkoøLKËa -BÖyªŒ´@Ìò)m½61n. îË~“§ŒkÜ¢M»S©|iŠz©­«A‡hлqs7üÛ·ÿùïfÝ ^ÐÌú÷ßh|vfùKaý—¿K7¬iüúwMãÜšöuýÛ鋼”í0¨ºÑÚm}Ö·zj9Q®§ÖÿÚ•õãŽ=¯Ë½*Ûì¨6­e–D«.5›Š>—cè¶>÷Âò¼¶vªéŠ½J¾n›5K›Ï×"z3¨V,‚ïá±a*µðÌsü6³1¸ð›!•+Fuë r.Ö4sú£¾ñ·%,çrœí°P0üÂòèVïë¾®|–úê˜èÆ&o‡9oÁP%ï]ÎÖð…îv%ä°²skksÒÎ#†V²Ž³5[Âƪ-9,Aº -ï1¶Q¶ò/<`£)`~0ÏI0_9Nì=Ì—~¯øØÁ‰Fëkô~Y>oäëo•S—._‡Ü#kžï›Î–ή¿׿´~Îå'†õØØè°ËÇ£ÛÔÍœ›‹¬Ú¤º2K½ý¼JúÔaa[”ŸåᦲÛ~ucyN‚ºf6<ï{¨³·h»7ÓÎ-Ÿ˜®cß žü¤µö^¸-#ë*»•ÔM?ÏÐ]Ïßäù!Ë­u #áÁ{ÕËßZ¾ºÍó<"ÙŒRìx¹ƒT׸ÎfdP2‡HËs:¤úûüh;䄆š‚¼™;h‹ MOKèz'D>ÂkÜ6ŽÇFÌ0²´\ÖCjˆ#u,ÏéšîsªêÅV³÷ÈùÒ/³ëû÷ÒŽÚ+ÒÙ,^FÚl}Yû6^ùÇ®½¨y&·6ɨ´ÍW4ÆOÊ{è_æ9*í}ÿòžŸÖÏäâ¬Ùçøl“†£ [—Qkèc…÷zÖ<Î7ƒyN׳>‹šÔY²kÍ»a¤%=¢´ÚÄÆ9‚\7Gr»²Xb«\Ž‚.vسÞf= ùð¥¬Ñ™Q÷Ö!GÞ1u¤õ,ùQXk‰±R©Ä£tƒHÌs6¤ÒAd¯ã´msÐíàiÍüt=k\{ú`¼ÐÓ}/2tŠ6ûÞvA¨ôwcÇÞ*˜ÛңʱºžZÍï ‘{}qì±à`žÓ!ñÀSüʱŽÊ#ݬ|£+ùʨé¢v›{(u¾Ž!¯&³Al®þ¹ËïºôÄdÑQÕ;È}"ú7˜çtȽï;ž9‚wÕŸŒ*v/=’7þ*OJwÛ|ÎüQ[ -Þ!w,Ïé{ |ð9òˆ7ð«Ù»qÁ=êYtãԩǹt0ÏÙéüÑ..R[Ï^{+ÆQ¼Õof¾—_ø™y›h¦ùÌHe¾§6SRѨ£}@ƒyN‡ÔûþêÒæò#ãAÎk{üÒ=™¦;3ðó>ݹQ¾ïµì ’Å$“›Ü¼8?BåXžÓ¡òÀ=BÑ£½ñ^~ߪwVE¿ ”ï 3<Á—cyN‡Ì_ó=W.(huóÞˆfè˜wêœÁ2J}ó%‡WDƒyΆR?¶Óñ³¯½?‚;Ô=ƒÜGîñ½Ÿyóøº|0ÏéPxß·œw’”}´RT›ïwõk:©Ç…5üáš™Eaö⛥‡Kä}Sï“û9=28ŸvÞI«Ý[Éx‹™óœm>ã3OàXDå<>Øûrô—ó“wß >˜çt}þyúŸ<á~GþùuŽ±ù„ù&jã;˜çlˆ ;EÞø¦ÊC«œ/5â:Ÿõ½Ÿ¯æ:3)G>£ç>"Çòœ‘¿Üçë§-wWT;ˆsï-Ìs:Ä}+¦–#FCøž —ç÷Jî>?ú^mwa¼“Wði»+ÞÛØA%{~=!Ìs:TÄ!ÏÊiwVaCÈ~`vÿ,£¤ãÊ=~Œü¹õàŽ·Á":Ù Í+0“ÀpûúóòñkþÙ'7ŠÌGÚx5@[ -ùmý{ow5?,«‹åÝË›æ7óøå:7˼ßQ«õݞ˛æ·ÂÎpYg¨™rs.]Ó-«’ò]A5¿!6CyöÐsZ¿üÖW~ûµœXxc­/{ó9Þ˽¾2ü¦|·ÖÍì¡õmµõ\¹‡MWþ|ñóÉz>}He¯Ãäã ¯ž&›û&]óiQYÒ¥ëôÜØúº“s>Ôñçšj=çQ壟ò±Ë!ìbMÿóÜé?]t¢)·f‰åL65‹£)MüvÉgò‘ &šÌ 8 ת–Eº$¬²×K~=sêSZWÞš¯S-<¿Þvùh\¹›œß•lù¬R%‘O X*„ÈéõVœï^Mzçs ãd×+_Mq¾Ø a+xÚV`Ëx(ÖV1¡Û -«®‰ {t>§6oÂj:o:ͺ¿¤5uç=áëÕÚ^ú&Jóij¥¨ÒºFU™eF¥*u%eíªý$ˆ^/‚ðUŠ&tfÕ³^ËgÒ,ÂX€D5]*éÑëEL·–æjO…š*¶¬©´|3¥AÝÆÚð‚ln£UòA­lŒÀ]1¥¸Øç(}­"ÃÙ_/˜bh#~»¸ ¤v+”+ÐãBËÊ­â¨^š4ïø_>ahY>Ñ#ÿ¿P…«ÒÌ<óMRùTžvÕ*Éc³Šo‰Ë}e’Z@ShiYsé*fy{騄ô™tkêÚb7RÖdLR1/“T%[±®¨«âLa¥}*Wr¡*F º&vÊo7âf§«C`/•yÍàîj¹Mì+w¾qq¨=®óKÕÞZ.Ã|¾Îõâ ýX*ib£êl×*cÂfH¡‰¾Íâ2Z+;åccËÇ6Ȥ÷õò·Ë<ÍëLÂou>1{^úúHó-~ÿ~¡ëŸË5Ëj5Pdá’‡óÜŸl£$%~ÂkBwLd­ -xÛ1á Št¬`E½­çáÉÙ’oé±ÂCæ¶'¶ÍGnϱÌ9HT/C^*º4ŒªbÌ%´> -endobj -534 0 obj -<< -/Filter /FlateDecode -/Length 4510 ->> -stream -xœí]Mܸ½÷¯èó«åGñ ö bdƒ0²‡ ‡q¯½‹`À›Cþ~%‘,I¥–ºgìh¼ö Ý­*’*_ÙjIŸþ¾Õø/’îbLÁÓùòáôñ¤ºàþõ%&½ú5^Áµ¢Ð9|tçY`C’3ÁŸOøovÈ‹?ž~üæôïi}6]JÞhwþõ]V‘íB A9A• RÚ -*續ó‚Êë]LIRÁ ÈÐR`¥è$Ì c¢ Š0ÃRÐv©J0ÃM’ -fèä‚ Ò -vh£”¨ƒ!ŠÈA§]çát/éŒî|L&‰ºFÒYÛù@ÚiI—:ｋ’Ž`‹KJK:[œ&R’¶AÒyØb½U¢¶˜¬¤ °ÅÀ;IÐEØtEQ[”WZÒ%Û¹‰D]‚NÅ—:£\碵JÒiÝ9`ÓŠ:Ì( ’‚ÎÀ¯\u°ÅYe$…-„?Ižp6Ä(ÌV“g·UVK:[Œ $ê` _bãa‹ -^¢Ì .ó’Ä&„Ž’!‰L´EŠ ˜˜: -ÁJ4`’ëȧ ñ€U:£%"° -¶8òX [3WÔÁ›Hâk` -H\`-l1d%.°`[Z$.° [RIK\`Á·¤´—¸À‚pa¦’¸À‚qmô$qåJIâ εàN‰ ,HÓ=H\`ÁºÖy#qíZŠ^â Þµ¤•ÄÞµ5El€%.;vVG+qw­VAâïb‰ 0ªpè%.V:£’¸€À»&*’¸€À»& ˆºÐaZ‰ ¼kº Enð®qÊH\@à]CÆK\ tà%fà]Pˆ“ÓØ‚•¸ñ:c%.@cAZ"qïê„ð'Ìi°U§cò8ð®Ž>Q—:àqaÞÂËö>I\àÀ»TVâÞ}F‰ Àñ"ª‘¸ÀwµE¤–tà]m’’¸Àw‘78‰ ‡N#¸ñ­Ó -Ðu©S)F‰ x´d$.@îT“ˆºÐHZâq§‡J\àÁ»ïIâÞUÎZ‰ y±í$.Àâ‹M,sDl±Ž$.ˆÑäåf”¸ FØb”‘¸ØÌëÍ qAR°ŸÄIù.§É€Æ¡SJ₤#Ö›†$.À¼ÌëÍ(qA² -ëM,ñ„y›¬ÏëÍ qA"Ø⢈:Ø‚¤) óœ„õ&ØIÒyØbI\¼ÏëÍ(qÒ ¬7‘Ñ‹[1¯7ƒ¸]Þu Ì$éÀ»ÈL½Ä8Àz™­´¶W ^ŸÉʘWœQÜ1P ^‡Ø*n(p/¢U÷ È7¯‘ÄM˜‰•eôâ®Â*”Z‰Û -ü‹|Œœ0ì>ñ-*8ÄÌr'¦!»¯kŽVÌ®öÇ~ýùôÝ‚ïùçÿœÞÓ?¡/Mȯ0 LÂp;dßÝ·ÝæLRÙBxOlDHÍH Z7ݲ‘ùßyùab9 ÝŸÞÖÓ'Dñáô o£±„#–¯s‹o¬òŒ+,Žm=ÊØd2;rÈ+M -ÿþÍ @­ö@ë!3ñ‹a2îüæÃ黿¼úû_x ½yúÇïõý·:¿ûé»3ã{߆Ïø¨\)߆2å¸ÿü0”¥wx•v.£,±ºïÇ÷± Ö½*ïºÔìsn<·aåK}¦+åj=Íúö–Ù!Õavj7êüÔ¶R¶·Mj+LûTêkóý?Ïo^þðf -®>ãÀ•0D¹ÌWñ¸¯ÊSðø)`ˆåë^usæY­—ÐR¶AtÓ[ÚULV†1N‡³óŠMf‹ˇäá3m‡~!÷Ev1~¶ìÝŒåÒsõU†eÞ‰¿•'wÖ90±ºŠLÝÈÂ)e”+aº)Bv¾G˜¿Š<“ÑÃ0êù}mäË9ç¶T’˜¡|®/èà³J·~g;«<Ìê”þzÁ¾5R\CŸ­GyÙ‚3n }ûê}t•çóUB×o/”îãºy -R8¼¢~‹Yâ‡_RÑäEÈuî«s8îK ×Uß2¢ =sïç Æ·nf G¸ÀÓ/7 ]AgK KkÐ_ð~ ûêOÈÿ¯¼òŒ3g%Ï­õÌì}>³âørÃk"BNY¢¸ˆ=Ã2½½«–uŽ†=s=;ìý–ý÷Óè˷͇Æ0–Œî§Ùç™ß7ñ4_ž³eù •¡ÞËXëÙT L%d‹¥Îœ©{·ÿvl;-m©ë(µì[#³²½lî­ÅkcogÄu‡Ê¬±0Üe:zO‰i×¹@@hõnAÁ^ö\Í欪fþ`(Ÿ×!¾n›­í¸®·iccÜ<¹¯Îáy=ƒ¬^~Ë2ÙžJ¯›s[yÖý‘ñµ}»sô¯Œ°¸zç™ÛlÖÌsÃPXÛŠS»JÓ¾Ò¬þÜOå½ú¤ìÌ÷Šô ¿ÇÙVâÞÎÎ:‡Ãçõr1³ØYqY°pe¥±µcTÛ¸ÈãÍÇ÷f,I¸ä¬õ9°ÆíÙÜ¿™Óòl1#Rþ6î*2wÖ92íõ óSÄò#ï켈 XƬ¥Û1»¯Îá0{=ÿ\Ìþ{¾EÙ`Iñ'0Ó•oLžbß­_€~ò]‚5&mù$%×¹þk™-Tî«s8TnìbÎF»ŽÜÊWÜ_÷ÞWú&?® 2ÞÁ•ûê•×3ÏyFyË>ù—Å_wÌÐgÝÓ—‘J,«Ì?3Ôò[HÝYçhH¥Lô7ÁW+3rã:“Ï3WÊrȽ×jì¬s8„nì{Þ³Gyï|Í v_àFîö•üÎ:‡Cç'Ê?_hfð¬3îsDz–_îŽôûê©9é‹äŸ/?Ò;uûI;ë ¡n#½“wn¾¾ãk¤ßé½ýZöu‡ÎëyèÝëÞ/0ÒßsEÕ§ŽôÎݾ¦ßYçpHÝÈI_$ÿ<Ó ;r¤·¯éwÖ9Bwæ¢üZ Ùï -Oð«õ|ž‰wã|ê·EuÇ7ñ;ë ‰~#çü×ïŸbv}†îííkõu‡Ê=ÏÉ5Ï4›vFp~­È‰ ÕW®5™_Vóào9ýß½ï¬s8”n\ÿ9ÿÂl¥0¿îÌÎ A«×¿=ãÕAÜõ^ï÷¯hæF­»Ó¥ü~NùxyÝá -BÙ/‚v#t_Ã!ôÆ=OÆi<.xk ‰×ÖË;xé%ç˜ó,gqmýJTw|×¾³ÎÑÐöåšÏr•£~:×®µ_»n~ru^FÑøÛ³Õvæ}’òEáZû{bÄÖoæÖ¸2ØÛ¹rgásç¯Ò©yö··^¾±­'­WyÇï‹vÖ9"7òËÙHT¯^Éîïâ°+w߸%äWÍÝ|ë¥Ãÿ~­wìiî¬s8´nübý¥rÚSîÄ4[áMfâgA+7âjy¤&Œv²Á©­;qí«s4tƹçJö.íÐÔQZÉÅ6ËVƤÞ~0÷%uýÚ»ô¥ƒ%Ÿïíó£´Õ&ö}yõÇ?ÿéoóßçç}I– ¿c+Q¶ÕþLmn±#Û8”u!ÏÐÒ×áx!Ëf=AžÓ^º°vëþ,Ϲ›=ä—òÜ~¶£åüãyíðyúKé±^ïÓ]NŒÜ‰Ùo†ÆvX_Y¬eç|?¼Ìeì+vÄV¾ï×ezÜf}ó£z›ßØ9õ +£]ê2ÌbíÇnyÞŸ”Ÿù‘l~>ÇØëŠ=¹­±oý}–íx›o^›°l§·ûa´Íõö]³#?ÓC¶C ïÙ/K&†¹%Ø–”ោ¶,·6§Øl¸Ìc›ÇÏ\Ê8Œ6¹1†Ï_Ù -­l¹8à´·=ï^îG?ö¥ÌûfGË-†ó 9ÅX®D6Wp0¶ÿ¶ÙWòŒßǹú¶Þ õKäœ`U˜0Ó~÷J1;JŸ žÂðÞ~½8Øߟ§Ô«ø—}yÉ(6¿™çãXmm˜Œ›nöƒ_¯ø 玪 3€¬ªnyNìêÚ`'LÉpl—^qBãd6”­„G ªÙ6q@Ú­ÄSTÚÞá3wLcEΈ… ·˜påÏé°>ƒ×í•w•ŸÚ‡~gº¶Ys©×ã=‚ƒƒ'Ë÷hNGvËáš@mÙÈ—–7(.·ù=ç¿yBÔM懃ÙxFD)äûiKIC&ión S=âÉøÁ%¶à0±ä"eÒ‘¬ê$ɧ ~·ã¼ê]ž¹!6Ô±}v .eš‡qÈ.ϳö¯gíCùÉlZ åÇ“ïï ¯ûçƒ(;mòÄΞ:›Ç&óý裲¤ËÐéÞÙú¼R³ìÚÇZj|›ÊgÉ7æÓÆÆòûAÿ€\†:_²—þ©IªGSšøñ”ŸšE‚˜¨¥@â,?ªÚ2}3“]°*]š—´®ÜÕš¥]Ö•Æ•Þäú®TËw3'U -Ùþ~ÞÉ!jz½çÞ«Nâüä±ØÙñÈWSœ/6CØ$^¶5Ø*nŠµULè–BÅN×ĆŠ=:?©¢ºÎ7a5»N³á/e‘º™]ÂË §µsé£(ÍÏ;*MïUe–•ªÔ•’u@¨Ž“ ºœá·ª4M̪g£–Ÿ1cÕr©”D—“XnlÍÕ‘ -µTlUSñ|3¥AÝÆêxAvaK“¥ÈæRSÆ‹s\ÇZE†³^N1´?ž\J»ÆÊèq¡eíVqT /Mšo$6|ÂÔ,²|Ïýüÿ@®J3óô4UHåçf´£v’<7«xê$.÷•uHò€¦ÐÊ2wé*f¹¿tTBùLºµtõØDÊ\Æä!ó2IUò˜ŠuE]g -+þ©\É…ªÓ5±«P~œˆ›®NU€½œÌkwWÛmb_¹ó‘‹Cq‚Rí­í2Ìá¥l4gèÇr’&6ªF»v2&l†š¸Ñèc/.³µ²S—Íñc›dRĽœ~9õaÎXg>x«ó3½xæ¥ÏÿE™WxýëDçÊC$†8nTe S¹u"æ‘,?2îç'Ba6LhˆÑ~ëØʺ‚†Ça|M­ºjÄ^JêÊ ºj¹Ð52±©Ð5Ú2ϸ+ÂËÌ‹5‘yêØ´€=ÚÒ9œÆæÇ«pÀ»ƒÃØt¶…ä3kG¹h éÐ’aqÀÅ!;aj–MÌÌ€ä'©®Ïö -›ÏÅX‘mÓQ ³¼é"ë{ÁŠ ¶ü€7LšŸ4M쌾{z=yÜG“}™YZ©½J¹´v,K+áÉRÛúéa9öœÊämªmY6÷lMrÚÔh©M8F™4¶jcÇOJE69i •. SEV[ Å&?V+né{E/…šS4é/ÜÓŽJu¢ã£âes»%âÕlÉÖÈq8‡Ö;ÕJÖ\2 -2ÏkW)µ’¶È–ç¦é¹K´=³nP3Ä·ÌTµjnQò.þ»gî uf;R©ò×ïŽèØ»„@M:Xcû'i«è=8„òÃE¶]rsëû'Ë]’²ÂÏ„áèÈZt~&ºlÂ#i[Áš*ÏK—«%%vé6ÛCæÔbw¨(nÿ¥[¿ITC^*æg`ÛœôÆüèΠ´üSÞ#Ž“í“þïz7«i -endstream -endobj -535 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 534 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -536 0 obj -<< -/Filter /FlateDecode -/Length 4427 ->> -stream -xœå\K$· ¾÷¯ès—õ ^€áC€$ˆ1°HA³½;6‚q;‡üý|T•$V»ºgv¼ÀÏN©EQIU•ìÙàç ‹2Ù)ç’"/?ž~:™)ƒÿj‰ÕMe‡ßÎÖPš‚-1‡óLðiJ%¸ÏO'ü³¹•ÅŸNÿøÝé?ë:öì¦R¢³áüóGf‘ŸRNÉ…UÀ2Ö+¬¦”\ˆ -+Ú C,EcAŒÈÑž• •4Ä ç²ÂÊÃS²~Ï*Ã%K bØ’²rXgŒÊƒ †È;…gáô¨ñœb.®¨¼žMNãy?ÅD6XW¦cÈ K(Æj¼Y‚%2²ÏIãEÈâ£7*²¸œ¼ÆKÅA;EáeÈëÊ*²˜h¬Æ+~ -%©¼žÉ)ïy΄)dïƳv -°M¯ò°¢0I1)<Y¢ EåA–àÓx²~4p"ø”³²Z¯no¼Õx²8—HåAL¾†.B“¢XAã’†.¥‰Š# \öeÊ -¸\&JÉk0àJ˜(–¤á€7•'Ü ’!²Td¡D °žŠ/YÃ‚ä ‹wNÂä B LÅ"úVÓÈsQó‚,ˆ5,’M¹iXBD>­j¼è¦œ×*/ƒg“†)ÑWl5,@.5e,z ð‡K@ ð,iX -dÁRÑ°®hÊ‘¹Æ³A¶†ˆtÁ³VÃ‚ì ¢A ²ËÈ2£Ñ°¹$T­ñÈpž™5,È'5,À2A¦ ¤Ty0¡a懓í a’$›HsTdñ4,ÈÙqº™5,Ȳ8ã4,€mr¾™4,(² âÓ° ˜8q˜¬a`r¾™4,(YB•Y4eÝ“o4^„,®†%FÎ7³†3o"¢W·2ç›IÝ.î†dÒxÀ]D¦QÃÜ ßDd«åöÀâ3™9ãÌêŽôøVuËÀ{á­’ºg`¾œ#©›™eŽê®Ü*˜Ö¨Ûø‹xŒv˜0ï>É-ªsHð™9ñ –!ºï9Ç(毛÷Ç~þþôåœïùûÿžWwÊÚ…ñ{ŠœX^®`ºWSmŽ`"ÖvìX@È÷‚*,B*CD P\¶ÜXD8b8Ûrîv-0äe‰-B™:.’Ý"O°˜(UIŸ[å•$ -"X&rQDn÷+õÎ:¯®UÙï½j½³Î«ëŽ83{C÷ëõÎ:¯®WÙï½z½³Î«ËšMÍÉKYÍ'<‡&ä­Â7¥óüßyqkù# Þ÷kí¼]BY€;+½¢Ð{ðýý»¢Œù w.lsrˆàLÞýxúò›?þùOû³õîñôϯàúȘË0tÁ5þÚ_ÿëüî›ÓÞÕqú>ÎÌ›¼ y?Á¬ ð3y—#M¡šgVy‘­-ŠÀàp`ä“/Ž4x]¤Ö°î­v¨õ:+©ßZš¸¬6Â:S•õ—oþþ×o…²øi…¯¿°_±²æ¿ÁÎôà—ûˆ¿U}×;D4b -§ê›¹³´ƒfa:K_e¡·ë|A8‚›¤´¿¾Q1 ïoÅxMªõ‘Õ€ªZ}z€Ms˨ÂmÖGG¢?Ð}d^®7}T-’èOÎ\ÜÈ³Ô eO»cüHU8«1íóµÏ»™P^LWGz£>Ïž 5Â}-í2)DãË ; EÌÆa_€”]¼&áuŒèºß -üâgaéò ÀÛð»ôY¯™ÆÙ/²G×d°ëY}†Yι,ƒ½_d§ù~G_Ê2ŸA“ÉTàŒ«bš[÷]ëÅ=Ûg9—~ý|Ý­ûai×ö¹»GI*uæãK•GêbqÝY¹^x?_»vßø—eœB'u,¢Ï¦W.3cþ÷büé~Wõ—6ãMN7ÆSéy3Õ)~}¿–ËVËæ=7,¬É®-¯ZÏôeéEußüx²Hûì-ââº×n[¶=ï¯ð3¼};uÔ‹l¡Êw$G²×ä0M[‹ö—•3¯€ÅCîeãg¹ÑbN¥íyå®Ã’±Ç*äð…-À]š•-2…†ãóu[M,¥Q¶®°Åš56 ÒÆQ-ª•yrÔ  ]ÖH¡Yãr_-—ævjûï‡|Öê¾lõSÛú8×·­?Z"Ï}T›pëqW´6BŽ6æfOiþ[Ûû¸Ø×2϶Õë«J×5#phv`š nç±Ë´y³C~x”=y»ÕSgÚØÇ{bñ¬:¬: c®›ÖÈ¿´+íÝHô–È=—íèNÂ.Ìm5þ4·ÛѬÙOkû¶^üníF‰¬×Ø‘Žø ʶùm:{ö%º7d?î»—½ŠæÂ?šîÁû¼\Öº’ž´–Yú·VÎù¡þÂή~aûà鮑ÌòË:~í9íŽ=§ÍØŸ“2’õS.u³áîœñÎ:o&i$—‘ó§lãgJ×¾^Ò¸o÷9Iã©îN÷õ?=i¼®©·—4¾@û"i|‘öEÒøjÚIã±þ¢I#%ÏP€ÏKS =ëI#¿a4¸M{È×Bå<»…z-eÝêQ$~·’CNŸfW#ÆÛÚ5C×-õ8N£‚ç7\3÷iiÔ¾U(W]ý¡1]“ã“Ó¨}ÛÕ–‹§—¶ôð—–P³‡"®e¯¥D›0»ÖaY›§ -nÓ†«64BÊuš·ã®càþ?ˆ)­CÇU.Ãí‹è{¡u,òÎën¡mtÔÂJ/tì“i:½^Z{¤#~;l££7•þš_)¥=Ô™Ûêì×O¯¬·¦™äM_;ß$æÏ,QOcÛ¥6‡º"MWŸ3M–©¡–æ÷šÚb¯½ï}u\Ì¢e^V>íÙ©yÜûƒáã­4.¿éç9i\âï'b}Âywwg7“Æ¥úùÔi?S·îðõÒ¸}»ÏIã®Huw·¯ÿéiÜuM½½4îÚiÜ‹´/Ò¸WÓ¾HãŽíôåiœŽS™¦œ-–ˆþºBÛÓË* ð×û}àÌß~ Ói]íW€_Ö]@-ð{×&;~ßðï¬ó)o‡üx˜á¡yaÕÕE›•ùx}…¶6‚ÑÛ)òæù]#Ö}¬ÓßQ¶§Ân7}5úìâ[ú¹áµ¿[{ß"‹Ý×õn¶Òvÿ:º¼b±üyãl}.g˜ùüv÷±ÅÞWçXì{þJúšÅþf,í~´ÍÅ¡mGÒ4ÒŠâJ´Ý…Ñ -Òõ -²t»éði‹5Mœœ¥Ltëm³ûê¼»íHËoôØmMÜæïÆNlKœæë}ùítÚ”[6@퇥ÎûÁsN”óËï‡å/-×ܾŒ>Wlö•@ÏÖfëw‹¦é- ð½XŸ€AñgÇVw_7buCl>8âÀêüÚÚº¿Y4I -/‹æ‹àm|S}šéÄì¥uóöÏÚÏÍ©ôB{¿ŸùU²úÖѽ–ð̱öíyüT‹¥Éñp`©q`gË ânZê}uÞœ¥æc|ôËIÃjšåÍ›={ ÚÍ´f]¯d‘ÏÁ5)›´† VÌÝ–eõ¶>›¥û|K½¯Î[³TƆCS5“z/ÂÒ‡ `ÆÙ½¾þ4]‘ccîÝÔ…oK­ÊGŸW¾]Xþa£/¡ÉZ¦n–ß-ߧ¥€ ¿ÎÁfñ¹[·Å›En˜^ ½÷›œíæ[™ut^øå—Ä'^ ÿ¦’ø[Îk±¹û8ïˆØe œßâGF¾mQñ¨{yÏÖµ˜^<†é–G0í‘€|\W”™îú£N³zÓkû(¬ö÷°ä³z„ÑÛ?x,ÔÇÚvSùÓ)Öï’m=›ÂÄ×& »q;âåC;òÓ)$¥tX8£ì‰^´ÛÉÙ {Ô”›ª°4¿÷æg¨ÊÈS/,u“â3ÆÝè„×f'¯•$é±£i°”FY¡.þ˜Åïèh;¥<ƒn/Ý5¶¢ -• z*M<©k²íV×É aM?+%Ñ4!ÐÝ ‡nÊO+ò3ô¥Lï,Zaî¡·;ȱcç“$§>ã–àèòöv…Í7â¥:6Ú’ÙôsëdéÞnt&ˆC0€Â }ªä¶Z;:ñ¡‚ír,2Íã^N?œª›s>8~~½åó¤däeÏÿC™oðûï¿mÌ~Ü™Ž®cëŠ,=Ÿ>˜ï$ói„Ø— :°?È6²¡YC÷ãÎ ¼¦QÝ `o%mGPÛ¹’†ÙuÓuÖ Í„âe£ÅÈ|êÜ ‡=ßú68~ÎG_4©p#‡ƒÛ> -endobj -538 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 770.673057 101.433071 751.923057 ] -/BS << -/W 0 ->> -/Dest (cb89-1) ->> -endobj -539 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 533.926036 101.433071 515.176036 ] -/BS << -/W 0 ->> -/Dest (cb90-1) ->> -endobj -540 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 297.179014 101.433071 278.429014 ] -/BS << -/W 0 ->> -/Dest (cb91-1) ->> -endobj -541 0 obj -<< -/Filter /FlateDecode -/Length 4620 ->> -stream -xœí]]¯ä¸q}ï_ÑÏ–æ÷ìC€Äð"²À ~òp§gÇFp×ÀÚùû9T‹d‰*Iì¾wf´Žç¢§[U$U"O)JTW‰¿ïþ‹V‰SðözûùòËEŠà$þM)“úG|‚»*iƒp*ùè®?g "$§ƒ¿¾^ð_wH“¿^þø›Ë_–yÔU‹”¼Vîúן²ÊbÒ1ª•T†Q9'BÐÎ3*¯.1%N3\°Ú®Ufا‚VëȨ"Ì06(³V%˜¡ƒ²œ -f¨ä£Rv(-%«ƒ!ÒZ£r£Ò=§ÓJø˜tbu:4§3Fø`•Sœ. ｋœÎ—¤ât¶8e­ät°Åš8‡-ÆÉê`‹ŽÁpº[4j'1º[€®Èê`‹ôRqºd„KÑZV— “1ĵNK'\4Fr:¥„6 «CB#ùÀè4lñÒ%V[œ‘šÓØbñÇéÀ΄™Þªsï6Ò(Nç`‹ÖÁ²:Ø‚Æç@{Ø"ƒç(=Hd^â8@‡ lÒ–#°ÑFŽtL†`8ÐÉ ëSàxÀHV [œõ[,z.«ƒ-&YŽ Œ†-HÀq1°E[ÃqÛZ …㺵2)Ž øÖJå9.0 \˜)9.0`\½å¸À€r¥Äqçp'Ǥ‹î8.0`]ã¼æ¸À€vžãÞ5VIŽ ,x×ädu°æ¸ì(ŒŠ†ã Þ5JŽ ,xÝHs\€VÀ¡ç¸X:FÉqïê(-Ǽ«\« }Ðp\`Á»Úá8Ï ÞÕNjŽ ,xW[í9.PøG²Ñxâøp¶€`9.€¿‡NŽ P˜ÐK8.pà]•àþ˜> ¶*&Ïq愈æcuI¨€gú-jY(ïǼ Ge8.pà]Ðgä¸/àQ5Ǽ« <5§ï*$Ǽ‹¸Áq\8…€ãø7¡$ Ëê’)FŽ x´¤9.€2‚IX]’â¸d,$âPŽ Z‘Bðø$¡CÈÇê¼À±ã¸ H(‹ÕÁ,ǬE2)r\4l1Zs\4lAÈÀq"‘¢ovx[v|`a "EŽ Àd"&k9.Îc<†Zåt^‹@׬.B§Ç!XW¬8.ÀXJDtzŽ @ð"‡KÀ è”å¸ $Ø‚®Âq\‘ˆ‘9§S°A6Ljt¡SŠã‚¨a ¢AŽ ¢ŽezÉq<4Æ’¨jNgegFŽ ¢õ"‡Š ›`¤ ¦du°4ÁqÚ'¶Çü`°‰a«ƒ-ÆYŽ bÔy¸9.ˆ¶h©9.6óx3p\$lAÄÇqA’^ä0™ãÐ8tRr\TÄxS[Ž Ð/óx3r\ŒÄxC<¦ß&ãóx3p\,lqQXlAД˜~ NÂxìÄéûD§¨<àà3cÈ™Ð-,¢û:æhÉÌF²ûüØ_ÿtùí œïõO»|þÍ4SVþ„)µE|…n šÛ!úV¸|ãÑÐJLB;YŸÉ“©©@‹ó¤[6î/ÿ»®,,‡á?N§7õô ^|:=Ü'‚’`,€1|æ‰ÅfyGƒ%Gð¶iLÒ™9Ä•Ò.ÿó‡ €:[qŸ­‡Ä|øK †­v×?_~ûo?üç¿ÿáGÀèÃçËý“”ö')]üþ;•»ù»^»pÿb=β>O"éÝFRŽµKYùîó8¿üVòûÿ¾~øáò/–5Epwx™\6å^¼Û¾cYÞÒ¾_¢Y1ÜiÖÒTîÖšª¯JuÖ«–~ÕtŠ4Ñ ‡œÇÀËÏø‘æãò缺kâ°Îëä²|i„0ú¼ü®e¿lÀö6§ö»‡½}YC7×ë Z»µ‘ižçž!g³#Šyþ襃yÎSŒ5öpÚ·ß„EM~ßZÛWy‡ãEþŽªìÇ{[M´Ôa¤ä?ÂTí?·íóUúº-Ëœ0܇ö?KúNý6Ýõ}ÂGçOyðš•À0 ÑÊÆCŸ6˜çth²»¬·ç̌ԣ¤G”"-J[ßꟗåä|•ÅRkyEQðyYFÏz½½SÞŽÍjY¢3£Vfû©•¥Üº™S{ZÏ’ŸH¾ñ´×n9gÕB©q¤Žå9R÷îžÓz´en0j?Mnmƒ–Õù3·ŠöLK÷­HÐý–,̡ۨÜ÷ÊŽžÝISšUŽœKϽÊ.uY^}váߪÛ@" úò„`œ&qö‘8–çtH<ˆK+\[ê#ÃGz›c&ÝV+3ƒ)}iñÎ+/t%_é5ÅCö<Õ!#G#È+çÜDl.âÝèíxY•Èc#²ÜG5\ÝâÀáÎ`ž³!WïÇŽÛ}þܨ~3ªÈµôH^!˜Æ¡vYO=9_PÇ]·åuÔúxÀûësÚBÐàýrÇòœ¹ûqjQt±à4Ò$^šzÚ£h G+OÝ{ò‚´>ædZqŠ.Ìú|Æ“˜Ø/í_Å‹dbQV|ßÒí"¸G=™Ýx&Õî .Ës:DģݼH­=꽶F(„£h­/<ßË[b¸oÏç=òVS™ú #•ÄžJ‹$£Î7Dö‘:–çtHÝWG6yÈO„)¯mñKißµ,ãŸéÎòí¨…G¥!s’É 7 ÎP9˜çl¨4±èŠm÷ŠûFP½1*úu |™æq¾Ìs:dÄšï9r@ùC£›÷F4AGö¸çAðJ[,9<"Ìs:”ÄŸ_kìý%Ü¡îä>rï}ÿÈÄÇÇåƒyN‡ÂýØRÙu/³Úv®ÍœÆÏé¸gÆð‡cf2 ³5¿YZ¸Ì¼¯ÎûäbŽÃH ôʧ]tÒÏÕnd,™‹õ̃yΆ6{3>sŽÌ¨œ'{_ŽþzqòBI˜,ÄEú£Y¡Á<§Cèó÷Ó‡øäÑîw䟿ÏÞ±Xæeî:*íŽ;–çtˆ=XÎø(òv¾·è¡}d”óµz\³¾÷ýuKâC§…tÖgôì#r,Ïéù뽿~ÚÞ²;¢âçäãk‹óœ qn?®Üº—ÃΆÐ5A/ϯ•Ü¼ôQ®Wa¼STð¶Õïl ’Ü¿!Ìs:TÌCž•ÓvFaCÈ~À»‘^Òqå&?ÒûÖƒ+Þóœ‰1âi8ãï¼W<ãÅŸX—9˜çt(ŒK Z¹zÂdµvœ lµª—›9xjfµÒAÊÝ'sèìtµïÆØÚ­éTŸÞŽºß~ñ:äQìåããñÁóQ~|à -Œ¯ÂßEFïv¨žÒ=P.}ªn÷!ß[;÷´>uæèÅ:ÔþɽÁ{DžÌMú„vVVÝ‘Ìs:ÄîǪýóµ\ë~ä½0^¸¤#ö<ƒ¼çÑ„¼vÄ¡žÌE#p,ÏéøíãÐþYÈ2vâÍo2·m]õƇïÿóˆ ->̯ŒÊZwtß|0ÏÙç7÷VGä·ó„ùÃ<¹«õw™îªëWoŵÏϬz’k×ðk{Âë óœ5ÿx6|ÌþÙðoÝç)™s­ æ9bÿ1O¹èYGem¡r` æshß@h|ü}ƒyN‡Ðƒgqžä‚þéÑi4¡äên$Ïÿ{ÆÆË®äã÷tóœ ‰ñ &<o¼Á£ÓX‘ØÂ"ó¥C}^\V*Ïqïj³žÓ¥/Ë‘ñ‰w æ92î{+¹ùÌöÈZïáÖ÷s ÓO‘—5íéþ½Õòu”±ñ^ zÎ ©½¾ ƒö*/k:g¹õ47ZÛâÅ-ô=±r0ÏéÐ7¸²ïÓ§\Cs®8³B¶W¾m ð‰g¶óœ…cqâ×ZE»Írÿ ›Ï²üšãPI>þìá`ž³¡3½!vü¦¼òŽ=g#Î]Ý+éf½kÏŠóÇÉåÛãkö1eñâ<öÌã«|óœ{ûÑáToažÇÍuù±Õ¡Ö¤>KD÷©ûÝÕû!žú:™¯Ö w«‚Ø;7ån e*&Zܺ;21h_~Yí“Ö¶Ôqshô­J[þ:=ñ4ö`žÓ¡ò j|÷·0î#sóÉý -}¯ÖfÜгjÿ4z?²¿@ÇmÝØnñ®‚—Ö7ùÄjÈÁ<§CäØû̧{býÝ­¸‹Þ´r1:ÞZ}Øú;-ÌŽÞiäÖõîù‹¡÷ŠS}Ëû[žx_øñç½ ‚CS:—÷â‰NE²µDåa’ ®7¢(Û9\ó_2?£Ë#TyX¯ül -yß” ªþõ÷¿ûªéå¹?ÝÙ܃šû t§Ôûûq¦Ÿ©’Tq´÷îÚXÂòxSCäJ¿Sœ§_ò÷ý•y”w‡7-TÉiýý3•5ÿžÊ‰Å1ÏçËÓã9~¥I¸ëå»ùÜÄž© G®Á•kø¾oÊ_.~ÚHMûÀI7¦óF±Wo…Ém“®yß¡(U¥éÔTÙ꺑sÚ^÷—šjÞqWæMøòL÷íxÉÁœþ—©Ñ¾¨dE®ÿL÷Ý1å$Žº4ñë%ïŽj±µBŠ³pþ)kYV•„UvÓ‘ú”Æ 9ÿLµpãÊOíÊÕäü®dË»ÖXY™iß–û‰!rzµç«—BÍâ¼Ãlf>òÕç‹Í¶-MÛ -lÅÊH"tk¡$§kbm‹=*ïHV«Î7a5V"Í_Òê@šsOx»à´¦—¾²Ò¼¯e)ªÔ®–UfˆQ©J]IYÄÖvbD· #üN–¢-³êI«åÝÁîÂX€dkºTÒ1¢Û…M7—æjK…š*¶¬©Ô|3¥AÝÄZñŒlª£Yò–Ù¤HS~.Å>·@ik nôJx»ÀÅØ•øõâ“Ú5®®@ - )·Š£lxiÒðß¡kYÞ[)ÿ§ -W¥™y¦ÊVHåýÑÚQ;Iî›U¼¬$*÷•u,Wʆ––T— -¡b–ÖÆŸLúLº5u­±…”T‘‡TÌË$UÉc)VuUœ)¬ÔOåJ*”Åœ®‰]…òëBÜìtµ«ìåd^¸»ZnûʯTj‹«¼Ù]µ·–K0_„·É±Ù^œ¡ËIšXËêíÚɈ°RhâF£¯“¸ôÖÊNÓûà短“q÷vùóersÚ8ðÕ÷n¥‘—ºþ/Òü€Ïÿ\ìõe³°»ײ²…®ÜºSO–wúŽƒâ¼ówÖnC„ÚÚob[ZWÐPý¸Ö„¯mË.±—”ª2¨ªZ*t­‰XWèjeH͸á­«ÅȼµmšÃ¾šrq8ÉÛÌ«p@/‡±éLsÉWR$ŽrÒæÒ¡µšdÄ-‡ä„©Y¶03’ž¤V}¶ßÒ¦0ù\䀄ùÐ4mn–]dÓU$ = [EOšö0F?ݽÞÜî³É¾ô€,­Ô^¥¯TZ/,K+áñR3úåaõ9æšJçm5TË2¤ï™ä´®ÑB »8 %ÒØ²Í =©-²ÅI‹£¨t™,²Zb(6ù9[©–éŠ#[K¡ÆMúgZÓŽ¶J­DG[ůÒær‹Ç«Ñ’©£píêdKYcÉÈÈ<Í]¥¶¥4E¶>·]ž»xÛ+¹ Û ñ-2•-›[¥|Šÿžé;÷<ÝL—ªüM³#*¶í5ᨭ -F›kÊáNôbó¦×ÁF2]òpžýù“õ,I™?¡gBskŒ -xÛ0áŠt¬ E½Î;“òÙ’d/é± -`¦‡ÍÉÕìfžÈœ&‰êaÈCE—¦-ªeŒy‹ö ;]4ݧyYLŸLÿl²Æ -endstream -endobj -542 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 541 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -543 0 obj -<< -/Filter /FlateDecode -/Length 4520 ->> -stream -xœí]K#Ç ¾ëWèÀíz°^@àC€$È"b`‘‚fåÁ8À:‡üý|ìîªb·ØRK;Þhmï`vÔdU5‹E~$K­’=ü|añ_&;ä\R¤ãé‡Ã‡ƒR0ø7¶X\Œìoð›ÂÑJC°%æpü > ©—âñå€ÿV—²ùËáï¿9ü{ÙÇÝPJt6|Ï,òCÊ)™ ° -XÆz…Â’ QaE;`Š¥h,ˆ9:g%ˆA%1ȹ¬°2Äð”¬?gˆá’%1l IaY9¬3FåACä³aˆPzÔxÎ1WT^Ï&§ñ¼b"¬Æ+CŒ1dG%c5^€,ÁYÈç¤ñ"dñÑ•Y\N^ã%Èâ ¢ð2due•YL4Vã?„’‰T^Ïä”Ïy΄!dïƳv°M¯òàQX¤˜žƒ,Ñ„¢ò KðÆi<Y?8|ÊYñVÇÞí·/@ç©<È‚Å×ÀEÈbRÔ 40.iàR¨8Ò@Àe?P¦¬¡€Ëe ”¼®„bIxcÁsVo K ¨!·…à¹*²øBxYÐ@Ãï!‹#¯aÚ¬Eø%S¬†xKÆF <b <×çHx@.L©hXà¹Ø©aèÂÝ“†¨ëCtxÀ®§5,ðÀ]OÖhX@À]ïÑSåA°†@ÇÁÛì5, à®·&iX@À]¸‘Ó°«:À£†°•Áål4, à®Ë†4, à®K*/ ðA¯aw]À´È ÜuÁ8 ¸ëÈE `(ðǨÙpôt²`5,@¼Ïy 0Øà–hX€»¶ ü)> ´l.QÂܵ˧òÊ`4®ø-´<Ø‹†¸‹@å5,À]ÀgÖ°? ¢: p×zDjܵ® pyCаæ0X$ ¾ ÖÀtU^LÉYÂÜ,9 …“$*/ 0$«aÀx0ÈC5,ˆÀ]„÷¢aAîšà½†¸kiʃ,>; ×€·U|Éê`BãwÅ -k<à®1©hX»65,ˆ‰ .kXdÁN˜i()E b1à!åSyqÀuа  KåAJ¤aë¡ø’5,H²xç4,H² eаÁP,²oµ<€,0µ> È‚LQà ٠‘†)DÔcЪƋnÈ p­ò2x6iX ÅVÃÔRC†ÓkX€8]Ò°b€gIÂT \EÄ¢!;dæÏB$Ù ÓÏZ ²ƒ,È5,È.£ÊŒFÃDhÔ’PµÆ#ÃufÖ° S8UÔ°n‚JH©ò `BìÛAÃ?(6Qæ¨<ÈâiX³ãr3kX3dqÆiXÛäz3iXP dAƧaA1qà4YÃÀ8xÆhXPlF½éHÃø%×›YÂâ êM”xŠß¹ÞL‚,)ŠÊƒ,HšŠâ·À$Ô›@'!‹+¤aA‰‘ëͬaÒ Ô›ÈèÕ-ÌõfR· €»¡™4p™iÔ°¨7‘Ùjµ½ð†„üLgf®8³ºc`½±UÝ20À^D«¤î€/×Hê¦ÄDe™£ºk€° -¦5ê¶þ"£3L˜vŸäU„9$ÄÌœx’nAÈî[ÍÑ›ùfÓþØß¾|Bð=~÷ŸÃóoƲúc‘ Œ­ ùÜÀ,w@öm1}ñÒq`fb !à=}XˆTº°8oº±üïxþb!9gÑÑuQä½€ÌéÆ| Ñ8:¡(F}à8fv‘oîsUä„ ¡Û «ËÚU%ïÛu¥ÉzsŸW—•! -%4Îź’*äµÆ¯»ø\lζWx>£n°ºÅ ¼ªäÜ•¹ÞØå6¨Œ‘jE´ñÅqhÞ ¨0´hü»· Ô,Å´Þ.…øH–,¢Ž Ç·?¾üó›¿ýåëo°˜oŸÿø-â—™~‰¾úÂâšž—ƒ›¯ãüwnÒ|§¿×lºÚž_ùú-~µ£Ö7P¤9[âjå²UîëópVI—‘tµÚmår_± Ü \à1Î -e~¶}“7¬2Ü•ûú<œU^N;×é¤+½¶¸Ÿ7~Ýá¡ÂÒÃéc=rÃRERÉOVÀ¥âUKÝ×çá,õJ&ú‹À« \YÚÿÇu u"‡44ŒÁúZ„ßÙçÑ,Ô]Î;/áÔY  ­ø¯™ÁGxÚ†uúÛëø}Î:¢üó3Í ^Õã>A¤w"¿Üéwöy8K½’“~–øó ˆô"¯Ü»g¿³ÏÃYè•\ôNܹhm¾ýéwGz/÷2ù¹¢<> rÑ:wöy4ëô—óлëÞŸa¤¿Ùã>A¤÷þöš~gŸ‡³Ô+9ég‰?¯äaé}¸½¦ßÙçá,tg.š»&Ûûs±¯Þ¨u‘çx%Þ†ó -üÕoŒêw¼¿³ÏÃYâ•œóçX¿ÿÞõ "8™Ûkõ}Í*éÊžçg‰5¯äM;#¸|Vda‰Oª/gûrÍWyÊÑ~¼5®­»>éÑú¬òÙÅÓylEyú»9ÎzNZ¾¸Ê_ïgì–²oae0·cåÎ>fáJιÎÅ~‘õòc}TE¸a‘ò}ò;î;û<œE^É/W+Ñ´z!»¿ Ã4,¼#”OÍ5Þ‡7*¡5f6\”Þ¥}èäUëòK^»a­wìiîìópÖz9×ül1mËšwÔâ묅'~RڰΞGZÂjŸ‚¹fûú<œuîÌ=7²wm‡¦­ÒF.võƒluMÚ'ny.ekï:—v)>Ž‰øT/ëò8—7øÓÿ*?‰B“uZ‘ ´–B¶6Ÿ¥Ì=v°ŒSÛØCë\§ë3úÜ–ù:§½tã¶ýY™swy(žÓy|–£çüó}ýôºéòi×VîRb”Jd½y;ÏÏLói²×yʼnûMÕò,ûiæ? ]ˆùrÇ®«ªCäs$ôž„L×÷xîsš<ÙÆyjQΉ‰Åó±tuN¡Î…ï1ë?5ôíù8Ñó™ |.Øù8Mï4/Ê$kCÓç2ê°Îo´‹É¿Ùø|¸hÈÅ¢Œ=ÙssiŸ}=XÏžÿžêzÌ2…5¦×u®,¥Þvœÿ¼Þmýëº+óm ¶vÑó‹é~S^1·«Ñ-Ì|;ÿ®ËWs ݇µ~ƱÞOýkôíùiÓÎ6á–ó£”r›õ‘¦¿ýŒ“üã}j¿æº®›Bµ#|\èp½ŽM¦Ø}£­›íòc/é)­õÔæ™Vö±aOm×+‰9Ò¬ƒç¥–˜8·•6o$®IL›Ú.ü/˜%N­u¦që;û¿Äué+4÷™qï’ÞÊ™}Í:oc„ÝÖù¤~ßsœ½c­½$'‡³”sÂÙ%NT¬9û 6€âÇ -øŒ;”‹îjf´³Ï]'/¼bJš¼|Rp_ý­lërËV˜a}Z„Í–ˆ$‘Ï/[Ýp‚n‘ØÖp^?´zy\çË ™¤Œž¿ºÒ›òxè_Œ[R]éâPÖNë?¾Å¶J$ÃŽ1Çót)DASŸâci;æO~<ÜŠú¹Cû|Ä#”ÓæL¯ôçÕ~x÷zÚ§ìàq™b\¶Ó >T}ù^ëÔ^Å©b, XÕdž›ðn™|,B¿•ä'Q|hjáã W7½} j¼ÔH·çñë}¤Dlx…\ÂCó$Cþ†wÑÊ3Ÿ·=´}¸ßèc­i‘oUÈ-±õã2Sê ÊÕZ‚ˆm¼õ½*}ñxËjSöl ncsC–ȵ¯w“•Öë×Ñå†Åòùd“õ¹œ‡<{wÙb÷õy‹íbÇnZì/ÆÒö£­5ÈÌ.ÀmƒÒÔóå1—•p;×n-·=¯3Æó•“]ßq…µßÌ'«¥€ž!ðÉ–9Ø,jkæzµÉ묺9?ëhWÇ{-Õ‡!ù+|FÖ„’øÂ-ݹ÷SȲs-ââToûZñ?Uæ6Uç¢Öm5Þ\»ÖzkÔÿ\#Œu@™è®íKˆPEa{}âžæu;-ë’6þ…ڻ͡žºRΖòÃ!Ž'jÚñTeó`íÂ1Òàymcø4NO¶.•m=Ç/«øÐZÍß_aøHk>Ît\Zy1·ÿ0.ú[P -™ <¦³æÍHFÆÖä—×)d¢¾‹v…ÌÄù¥ic!)p+Ú œ†²n‰læ—}çŽãôôÒ…:îj7>’LmäÇS§+DôŒöœÌ³7ƒÉü} yðóUl¢pî߈}@’mû€½ãU²õFÃ9шÛu²£*åó}›êb'6Ñ¥ê¬XþÚÖ%±œ—ˆ§në×ԕʧÄסªvi4/„*j˶ ÔÖI! -ñ S‡&,fã‹Uã³v'b®†D­]©íÒé ¶›G m¥Rk•{×R5ßEé¦ÎœÛ´“ØÝNü4ÂGŒ¯/—dÞ@mkm²°wF<bèŒürIi:V˜PMO½·‘³éöÒ©)WUÁ5+O*åÿ'¨ÊÈ3¾°ÔLŠOîWý&웼T’¤Ç†:¤iÀRêm…ºlJÍf¥¾l6J{ÝÖºilA*ôTªx R <–dÛ¬®‘ª~VJ¢©Bàvš)¿,È]ÎÐ\5˜v³h…¹‡6n'dž/’œÚŠ[>:ºÉÛÆ6_‰§1°ÑšÌ¦ŸëM:Ù™íúͱ Pèä£/#¹zkC§ñ™ùew2-âžßÆ0ç|p¼Á½åoB™—=þmÞà÷_:~]Þâ¸3 -\ÃÖYF2þÞœ¼“ÌߣS‰ÍmÑ‘€ýN¶¹· ÕZwNà5õî¦{mi‚ÚÆ•ÄÐu(È®™®³^h&\ žVZl‰ÌÇ®MØÓ¥¯“ã­J>´¹J… 9\æÎó=$Ÿâ¦=¤ƒKNtÄ…—↥K¶“ RÞ¤©žå'¹¼Ñ&/DÁ—¾ó¨‡Y9t¥³M{y!&+oZò(Bßí^½î³È±zS´7ꋤ¶‰1µžNõãe\^¶˜ãÇ}½ê¼]Cm,/|Ï·$§»FO-hq!mTPsï67ò¦Ti‹›Ö@Ñà4SimÄTeŠs·ª–qÆYÕRj9E§~/5äª4%¹*ñ¬-[#^Ë–|K€‚4çÔggzË–Kf…eïF¥ÞÒWÚù½iyïmbÔ‰=35½[8kyþÝã;SŸÕN„ÖªþŒ»#6÷Ãê¨É&ïüøýƒ&Ç !þ -™DYl—ÜÜçòþÉù.IÝ?‘wÂr ä½I–¿IRáŒr!†ªß2©w+FÒm -P¶‡:Ì™³Ý¡ñ«ÄÆ-Îq“¨]&.ù›='½™¿ð(«?¦Äo¿¾[lŸŒ?ÿô¤o” -endstream -endobj -544 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 543 0 R -/Resources 4 0 R -/Annots [ 545 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -545 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 198.873057 101.433071 180.123057 ] -/BS << -/W 0 ->> -/Dest (cb94-1) ->> -endobj -546 0 obj -<< -/Filter /FlateDecode -/Length 4598 ->> -stream -xœí]]¯Ü¸‘}ï_Ñφá÷°ÈC€l°ƒ$ÈFò°Ø»=N°¸`²ùû9T‹d‰*Iì¾×¶œ/Úݪ"©yxX,Q¢ºJü}§ð_´JĘ‚·×ÛO—Ÿ/R'ñoJ±8˜Ô?àÜUI„SÉGwý) L!9üõå‚ÿºCšüåòç_\þ¶Ì£®Z¤äµr׿ÿ˜UÖˆCŽQ%¨¤2ŒÊ9‚vžQy%p‰)q*˜á‚Õv­ -0æè8Ì°ZGFa†±A™µ*Á ”åT0C%•’°Ci)Y ‘ÖÍ蔕î9VÂǤ« Щ 91«œâtIxï]ät¶¸$§s°Å)k%§ƒ-ÖÄÀéÇÚÃIèàò±:/pì8.Êbu°ÅËqk‘LŠ [ŒÖ [à2p\@$ï›ÀÀ…XØO‘ã0™ˆÉZŽ ‚ó˜¡V9×"Ð5«‹Ð©ÀqAV`(V`.%":=Ç x‘Ý%Ž `tÊr\lAWá¸C‘ˆž9§S°N6Çðt¡SŠã‚¨a ¼AŽ ¢Ž˜ezÉqFhÌ%QÕœÎÊ<ÏŒDëEv9.@7ÁLLÉê` h‚ã´Ožl;Ž 0ùÁdÓV[Œ³Ĩót3r\#lÑRs\læùfภIØã‚$½Èn2Ç qè¤ä¸ ©ˆù¦¶ _æùfä¸ ‰ù&¦xL¿MÆçùfภYØâࢰ:ا)1ýœ„ù&؉ÓyØ¢“å¸ yŸç›‘㸘o£gC1Ï7.ïºfâtà]x¦žã`¾ Ï–›ÛK¯ ðÏxeÌ3ÎÈF $¨×aleCÜ‹Ñ*°1 òÍs$6h31³Œž`X…RI6l Á¿ðÇìŠîÑ'¢ò€CÀ˜C¾È„naáÝ×9GKf6’ÝãcÿËå—ï1ø^ÿò—O¿˜"eåOÁA˜R[øWè:¡¹¼o…Ë7 ­Ä$´“…0ð™<Ùˆš‘ -´8ݲ‘þò¿ëúÇÂrþÃtzSN(N®¦ÐrÄôuŠ ‹Ìò†KLŽ0Úz¤1IgväàWJ»Hüëwu¶â­‡Ê¸b>8“a«ÝõÝO—_þîû?ýá÷?Fï>]þû?¤tîWß©ü­»ïpÿ–æþ Ç^>ïï¿WòùSå¶K—Q7êãœçCÓiMÒ™ùóqþ¶óï\~jéì°1οçk°ó13_GI›ý2m½þ´–U›dWf9oèÊ%ý¯þçúîûËoÞ-¤),qM™›v±7–å5Ø{CÈU«Á »˜3K¬ÙOËz´s¹Û\ï‰è>-Û/C'G“kÛ…eYWÚ³¶ŸjzûaÝÊQÚï\ƒ ¾ô÷±õí[iÃŽíéd /=m²Ï~Yûjï)å|ìêË6¿diÓ*›Ç¨Á/Rsèdæ9*÷½Ë=/­ÉÙQ³ÏÃðÙ* Eíø<•·Z6>Á7cyÎÖ²úÀ‡+5y#£dW›¹ïèÛ}t«#gßzŠ´ÒŒˆœÇä¾ö ¿ãÜGŸÍÿÌiüi4oå…¸ïkUÔvüâÞo ·pÊŽKù¹ 4×ëÐZ{µ‘Å©nޘ͑Ž˜oPàt0ÏépºïÁ­ÆEÚlö·KÛWy‡cnìé=A›Öé½²Íñ¬›£ŽŒuµPÏ‘é–ôúmºëûx÷ú”æYOIÚ± É`žÓ¡ißËÚόԣ¤G”"-JǶ¾Õ?-Ë™fˆ…Å™åR|Z–ѳÞj>Àùð¥¬Ñ™Q·æ!GÞí1µ§õ,ù‘™kÝ–¶åúÝŸ5Õ"u,Ïéz×ë8­GÛrSûirk´¬ÎŸ¹U´gZºoE‚î×xa“ïmîå¾WvlÍ‚©-=ª9×S³ù $R¯/ßqŽÓ]Â}$Žå9ͧø•c•GºQy¡+ùJ¯é¢v«k(ç¼!¯œs±=ºÈè¾@oÇË£ÑQÕã‰Oì—ö¯üE…X”µs—eÁ=êItãÔø'¸t,Ïéyàvq‘Z{tôÚš¡Ž¢µ¾ùÞãwd:ä­¢™ú3#•øžJ‹$£Î+nö‘:–çlHµûþêÒ¦ò#áAÊk[üÒÝ™¶;#ðó>ݹQ¾íµð¨´$&™œpÓäü•ƒyN‡Ê_ôE¶Æ[ù}#¨Þ˜}(ß@¦yœ/󜙾æ[Î\PþÐìæ­MÐ1­Ô9 ‚7PÚ|ÉáÑ`žÓ¡tlãgŸ{w¨{¹\ã[ß?²ññyù`ž³¡Ðíû–ÓJ’²Š–‹jÓÕ®~Nǵ83‡?œ3“(ÌV|³´p‰¼¯ÎûäzŽCO ôʧwÒÇj·f2ŽÄ"GGæÁ<§CÛÏøÌ8Q9ö¶ýåüä „šÇ‚æ9BŸ¿Ÿ>Ä'F¸ßþ5{Çb‰O˜Ÿ£ÔQiw„ر<§CìÁŠÆG‘÷žSyh–ó¥z\ç³¾õýuGüC§…tÖgôì#r,ÏÙé¿Ýûë§í-»3*q^=¾¶h0Ïé7öTL/‡†Ð5AïŸ_+¹yÿèƒ\¯Âx#¯àu«+ÞÚØ@%¹=!Ìs:TÄ!ÏÊi;³°!d?0º–^Òqå&?ÒûÖƒ+Þóœ‰>âi8ã_¼W<3Š?±.s0ÏÙPýÆRƒV®ž0Y­'([­êå"'OͬV:H¹ûdNWûnŒ­ÝšNõñõ¨[ñíg¯CÅA=>Ìs:Ä8¿ê -Úo—¯ŸAî&Ÿˆgæ9â™òã+V`|þ.2z°Cõ”îréSu»ÏùÞÚ¹§õ©3G/Ö¡öOî Þ# -$6éÚYYytGr0Ïé»ï«öÏ×r­ûU÷ž…K:bÏ3È{MÈ;`׈H,rcyΆÀøõýÐþYÈ2vüͯ2·m]õƇïÿo¼ªù‡ù½ÕQYëŽî›æ9"ã›{«#òëÃüaž‰ÜõÕú;ŒL„»êúÆ[~íÆó3«žäÚ5¼ÅÚžøÄ:ÈÁ<§CÍ¿Ÿ _ ³6ük÷€Á8e$1Çñ×ð å9bÿ§\ô¬£²¶P9°ó9´o 4>þ¾ÁiÊ•ò¾“Ü\÷æžî2ä´þþ™ÊšOåÄ2ߙϗç69~5Üõ‹òÝ|nbÏT†#×àÊ5¬šòç‹ŸvRÓ.qÒG¡tÞFöê­0¹mÒ5ïJ¥±ª4š*[]7rN›ïþ\SÍûñʼE_Þžé¾Y/9˜Óÿ<5úO•¬ÈõŸâ¾w¦œÄQ—ƒ&~¹ä½S-#¶pÅY8ÿ”µ,«JÂ*»hƒH}Jã„œ¦ZxÞŸãþS»r59¿+Ùòž6V–DfÚÕå~bFˆœ^­Åùê¥P³8ï?…™|5Åùb3„­@KÓ¶[ÆC±2’ÝZ(ÉéšXÛbÊû•ÕªóMXM§U§Hó—´:æÜÞ.8­é¥/¬4ïzYŠ*µ«e•bTªRWRÖ±µÑí¿“¥h‹Æ¬zÒjyï°»0 Ùš.•tŒèvaÓÍ¥¹ÚR¡¦Š-k*5ßLiP7±V<#›êh–†¼¡6é#Ò”ŸK±Ï-PÚZF‚½Þ.bìJürqIíWHW G…†”[ÅQ6¼4i~jùþ ]³ÈòÎKùÿ;U¸*ÍÌ3ýP¶B*ïžÖŽÚIr߬âe%Q¹¯¬c¹P6´´¤ºT³´¾T”LúLº5u­±…”T‘‡TÌË$UÉc)VuUœ)¬ÔOåJ*”Åœ®‰]…òËBÜìtµ«ìåd^¸»ZnûÊ/Tj‹«¼^µ·–K0_„·i`³½8C?–“4±–u´k'#ÂfH¡‰¾LâÒ[+;åí½ËÏÖɸ÷vùëeæ´q:á‡7*ïìJ=/uýÒ|Ïÿ^ìõ÷e+±û8®ee ]¹u!¦#YÞ<Šó¾àEX» jKh¿‰Uli]ACǵ&|m[vÙˆ½¤T•AUÕR¡kuHĺBW+CjÆío]-VGæµmÓìû¡)‡Ó˜¼ ]± -ôrp›Î´!ùJŠÄQNÚ†th­&q@ Å!9aj–-ÌÌ€¤'©UŸí·´)L>9 nD>4MgÛ0K‹.²é*H¸À†Ђ­¢'M {£Ÿî^¯n÷Ùd_z@–Vj¯Ò*­–¥•ðx©™ýò°Ž9æšJçm5TË2¤ï™êä´®Ñ\ »8 %ÒØ²Í =©-²ÅIË@Qé2YdµÄPlòs¶R-ÓG¶–Bõ)šô¯´¦m•Z‰Ž¶Š_¥Íå–¯zK¦:@ŽÂ9´«“-eõ%##ó4w•Ú–ÒÙúÜvyî2Ú^ÉeØfˆož©lÙÜ*åSü÷Lß¹çé"\ªò7EGTl›ob ¶*m®)»;Ñ{pˆÍ[bI¸äá<ûñ“u”¤ÄOè™ÐÂ#ƒÞ6Lx"+HQ/ó¾¥|¶$ÙKz¬˜ðP£9¹Šaæ8æ$ª‡!O]š6°–1æ ܃Tl¸hº!v[„O¦¿úظK -endstream -endobj -547 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 546 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -548 0 obj -<< -/Filter /FlateDecode -/Length 4449 ->> -stream -xœí]K$· ¾÷¯ès—õ ^@àC€$ˆY`‘‚ƽÁ8€Cþ~>©J«ŠU]Ý;»é]{³ÓEJ*Š¤>’êjµ>+ü|¡ñ_$=Ę‚§óåÇÓO'5§ð¯´˜]öüwÖŠÂàtòÑ̆œ þür‹KÞüåô÷ßœþ=ï£ÏfHÉíÎ?—Yd‡CPN`%°”¶˹!ã¼ÀòzÀS’XÃ2´fˆA):‰1Ș(°"Ä°´]³Ä0A“Ä‚:¹ °´‚Ú(%ò ˆ"²Fài7x(ÝK<£“I"/€§ƒ‘xÖ>vZâ¥Á{ï¢Ä#Èâ’ÒÏA§‰”ă,dcx²Xo•Èƒ,&+ñd1ÐNx²À»¢Èƒ,Ê+-ñ’\ŠD"/§bˆkžQnpÑZ%ñ´|ÓŠ<¬(Ég ‹W.‰<Èâ¬2ÏBÂÄN8bV«É«Û*«%žƒ,ÆyÆ—ÀxÈ¢‚— +hȸ$a€ a dHí@‘¢„&¦B° ˜äò)H8`•Ïh ¬‚,Ž¼„VCÂÊyÅ&’°ÀÈ‚X Y Y ,Жà-XÀ-©¤%,°À[RÚKX`¸SIX`¸6z’°ÀráJI ̵ÀN ,@Ë=HX`ºÖy#aìZŠ^ ܵ¤•„ܵ=EdKXt¬ŽVÂîZ­‚„ÜÅ22ÀªüÐKX_LŒJÂHÂ ò€5h%, à®q˜‚¹»Æ)#aw /aeþ(1îBœœ@¬„ˆ÷à+a Ò pW'„?aM­“—°Àwu„ùD^t€Æ…u -Úû$aî"PY pð%,ƈ¨FÂÜÕ‘ZâwµIJÂÜEÞà$,€;  €„ˆoƒVp]‘—•b”°ÀwKFÂDáAE ‰È IKX0òP ÒBðø¤ÀCÊ'òü€k'aAP 8@–ȃ,HÂ$ÖC²)JX d±ÆHX dAÊ a2‚!idßbyYà.b}@™¢„@²!&" ‚ó¨Ç U‰çÍàZäEðt° Šµ„¨¥†ˆE/a~Èé’„ø!ëŽã¥eå½›ùmW 1L™vú}·x½PúUgò #ënè¥#Tçn—N…ñœšxÔÇ«¯³ƒ/û´ù¨Iž$Œÿí4vZËRçÖîËæÖÈ¢m‘ƒ-<­¾]2ï»ßŠŠû<šS¢öÙóJªðv™[/;d¸Ä,Î`±Yò2÷ÀÂÛñÌ•§7xÁQ¨Ð¹ôÆëKHU }0/_ö©ºãÙäg¼"S [ioGɃ}Î#i'›–¿íViVòŒ·Ä®À<—vZÁúö õw,Üúð6L¾åªY&†ya+ÎåZa(ÍçJ‹þK=Õ¿M'9 ?IBÑ ÿK{A\÷¤Ð¡L*š\ ìûç±>çŸû äÊ‹ØÙü²úÂzÑkÄbxCÙ¥¿0ûÞìK’_rÔú¾ÆåÙJr›LÈÉ“Åì ”r»ï™Çú<œgîg˜"–7k󬣜T2Õ>¬ )Þp­dº±üú$2`ÑgMÏ%ûìÁ>æ³f?ÿ\­þeÞWm¸‘¯AÉYF §*Ì Èd„ÒöäsKùž&ßùml ©éù$%78£)ozî{å±>ç•û9èÒÚÍr±[Œç}×c×Tÿ^ãÆÏ¡oÒã†Wº;°òXŸ‡óÊýÌs™Qjfé¥Ç}ÞøuÇ -]l‰¾ßŠÜðT–Uæ4±¤üUO=Öçá<õJ&ú‹À«¹±ùþqW¢ì¡–劆¬¯Eøƒ}ÍCí•}Ï{ö(ïÁ_3a¥mx§½½’?Øçá¼ó域hfðª+î#DzËòË£‘þ`Ÿ‡óÔ+9é'‰?¿€HÏòÊ£ûõû<œ‡^ÉEïÄ›îø5ÒŽôÄ÷2¾Ûy°Ï£y'íç¡w×½Ÿa¤¿çqªéÉÞ^Óìópžz%'ý$ñç•VØGzr·×ôû<œ‡ÌEù³@õý9ß­W´Îò¼‚Wìx7Yàר~cT¿ãøƒ}ί䜟cýþ!V×GˆàNÝ^«ìóh^é®ìy~’XóJ«é`çÏŠÌ<ñ‰yõγ&˧ÀZ|å]NwÇ{ïû<œ—^yþsù!…E¥°|îÌTÌ Ùƒ6Ÿ{ŧƒ¸,ê~ŸW4Ë £õ=¨‡Úþ¸ÎñxýÜᆇ²öÐc}ÎCoÜód˜Æ#á -·¶€KŸrŽ¹ÌrVÏÖoEõ;Þk?Øçá¼ñX®ù*O9ê÷÷Æ­gí·ž›Ÿ=—½húàÙæ8Ë9Iù¢ð¬ý=1âÚ涰ҫ۱ò`ŸGóN%ç\æb¿ÈzùƱޫ"ÜðÈ;>_t°ÏÃyä•üra‰¦Õìþ. “°ðŽ|?5×xKÞ¨„–˜Ùp‘¯.és'¯Z—ï­Ú o½cOó`Ÿ‡óÖ+WÿT1mË›ÔâˬÙJü¨+hÃ;{© ÖN68uÍ;õy8ï<˜{ndïÒM³ÒF.võ³lÕ&íàŽ<—4”ڻΥ]Bƒƒ%ŸOÅ̇:ymµ‰e._ÿáOüëòóùy_’åÂýÓ «Íe.oY¾±­ yuÖyŽ×+úÔ6ó ôœòÒ…Ûöfy¾]#~ÖýšžÇÏrô|º¯_Ï?%=õ+ú<¤@Ϙuf-ßGÀµïs+òqÝ\&ù.‹~Ï#¯´¿Lscz(ò³q«.ÐÏ»G`úP]ïE–Üç2®fí§)z>·¨ò G6Ÿp[ç6ÙÓ•yLú+ç7¬Çñ6ÿ”]ÓôOÓX~W$Zr¨>—¢¯:¿âãdËGÍzEÆ'aìâ×iì_ìð<õÄý•ùj˜ÚN:©2ë[–©úøøùRæ»Ó¼¸2oôÞÖzîOk_šõ™ùÒ®žÝR¥û»_m¦˜ýš='YÊu/£5½¤I^×u²Ô›žôd™¾‹îžûXÕîs,ézËk!ëÁ\êšé2ŽÑÉÛt¼«£°Ò‘ïsÏú)ºN]ç›ö¯mž»¼=_åÒÜ~5[qÌw<Ó ç7Òj†Sðôi¡_ÃçWgøl²løSÛœôXôêº µ’b×4.Ǥ]}•Óãæ>ÖbS‹µGÌ7TŸÇLVÕ¹Úäomü¸˜;‹Õõ¾Åïjûz¿Æªk8%c”*gût}ïêj…ßïo›}}—ºN÷ߌG¯1çòžWÂ>ÛcDóñÉî×1pW+ì·›ërwœD‹qÆ|dŽÕ5&#c}3+s.ŸëŽì˜ºV:\mr¥RXjW†;çŸe)Ðç‡!óJØx6(RÈg0JérNKÌw9-®ÛµxíÇðc}_n-­®i´cn;Azƒ² Îër˜…ÿç¢3ÝsçØ_/Ãj¹ßÓd’ËÛø;!¦Í¡~X<­LùÓÉ—óDu9SZù8h“¿tâŒ\9{tHç|†iT–t5.ÊÖçžå«:~j­¦oïPù@ï|˜ëøÕìbjÿS1úÈäá•5w/'í«BŽ¦^tòË)Ó dÔJæ 9§—ª…×,hÔÃaHË–ÖÕ“YÑa]}i\Mîïj·|&©ÚÈ–3 Ç DôôzMγWƒžÈùÛ*â`§+ßDq¾Ê bxÛ>`ïx•¬­bD·&*v»N6TåÑùtã¦:߉Mt®:ÍÌ_ÛšÀ̹G¼œp[»¤¾ˆÔ|F~ªjרF³L¨Ô¨®¶l¡f't9 Ä/Tš`ÌÆgVË' ÄX‰Z»TÛ ¤ËIl7暥Bk{×T5ßE鮞cú6í -ó¿~‡­eëË99×|ÍÖ*2¿1+âå„C+òËÉ¡µëX¡\u=N´lÜFŽªûK§æ#ôÆWXš•–ÏiÍÿPá5#Oy¡©¹T>k¹_õ›äµÙÈs%qºo¨C’4…Þ–©K‡Ð|–ëKG%´Ï ÛZ7ͨLeŒR/ƒT9Y7¯kä aU? +9QU!p»NvÍ•_fä.§kKÎ^oæ5sw×Æídß°ó…“C³¸Îg7yÛ¸Ìç+ñR-ÉÙõc½I'Õ¢]¿#vÁ -Üaô¥ëjmèTN`š^öE&EÜËé‡S sÆ:“ðÂ[¿‚g^úü_´ù¿ÿ:Ñù›zððÇjha¶ÎÈ<’åo ŠÉù[„*±-F4Ä`¿“uìm]õ†ÇaxM½»êÀ^[ꆠºq9Ñu2²i®k´ešq;ÄËB‹-‘y_Ûô€=^Ú:9ÜÆæ#««T¸àÓÁeì<ÛCò™ ‰«Ü´‡tpÉ°Ž¸àƒâ’Ý0uÉfbf‡ä7iªÏò7…Í÷b,È—¶ó¨‡Y>t¥•Y°&H-¿à“æ7M3y¡ï^^ïm÷Id_W@¦6hoÔNmËÔx2Õ–K?¿l1ÇžóE]¼]Cm,ËÖžmIN_=µ Ù÷QF½ÛÔÀñ›R¥ÍnZEƒ ÐT¥µC•ÉOݪZÊŒ£¨¥ÐrŠNýkÚq«4%:n¿j›Ç­¯eK¶%@Ž»sè³S½eË%£@ó¼w£Roi+m}ošß»FÛ3›uA|ÏLUïæV-ï¿{ÖÎØg±!µª?ewDÇ~T?5é`-ß¾¨¢÷ÀÊ_ (²í’›ûìwIêþ ¿Ì1µ*èü=š²/`¤ë 6TýŽM¹[Râ”nS€°=ÔaN­v‡Ê©•·›Ë&Q» ¹TÌß›hsÒó×=¥åwWó6‘mŸ”Ÿÿ_ø¥¤ -endstream -endobj -549 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 548 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -550 0 obj -<< -/Filter /FlateDecode -/Length 4730 ->> -stream -xœí]KäÈq¾×¯¨³€¥òùŒ=°/,à ìƒáÃLÏŽ¡WÀÊÿ}A2LUÓ3Õ<ž.Fä#˜ùåI©ï -?ßiüI1¦àéþòóí—›‚Sø7–XŒêñÜ]+ -ƒÓÉGwÿ™6 !9üýõ†ÿºÃ¶øëí?sû벎¾›!%o´»ÿí'V‘B A9A• RÚ -*熌ó‚Ê맘’¤‚.¡µ*À JÑI*˜AÆDAa†¥ íZ•`† š$ÌÐÉA¥ìÐF)QC‘5‚N»Ácн¤3zð1™$êt:Igíài§%]¼÷.J:‚-.)-élqšHI:ØB6Iça‹õV‰:Øbb°’.ÀƒÑI‚. +Š:Ø¢¼Ò’.ÙÁ¥H$êt*†¸Öå­U’NëÁ›VÔaEa’|t¶xå’¨ƒ-Î*#é,l!üH:ð„³!Faµ^ÝVY-él1&¨ƒ-˜|‰Œ‡-*x‰°‚æ%‰L%C ˜hŠ%01 ‚•hÀ$7OAâ«4tFKD`lqä%&°¶V®¨ƒ-6‘ÄÖÀ¸ÀZØbÈJ\`Á¶´H\`A·¤’–¸À‚oIi/qáÂL%qãÚèIâ Ê”’ÄœkÁX.–{¸À‚u­óFâ Úµ½Ä¼kI+‰ ¼k-jŠ:ØK\v¬ŽVâïZ­‚ÄÞÅ22`VàÐK\¬ &F%qwMT$qwM€ uaÀ´x×8œ‚ä¹Á»Æ)#qw /q€2€” €wA!N` Vâø{茕¸ a‰Ä¼«ÜŸ°¦ÁVƒŽÉK\àÀ»:búD]tÀˆ ë£)èò‰:?àØI\”%ê` ’¸õlŠ[¬1[2H\€ˆ`HÑ·˜ÀÀE̶ R”¸L6ÄD$qApùFUÒy3ĺu:$.¸b-qr©!bÑK\‚8\’¸f@§Iâ‚` –ŠÄpEC4ˆÌ%†-²%.@¤ ÖD[ J\MD–é•ÄðÐÈ%1Ô’Žç™Qâ‚H~àPQâ,dš`JQ[@`~8Ùv ùA²‰4GÔÁëHâ‚ §›Qâ‚a‹QFâ`“óÍ qAR°ŸÄIùÃd‰ @ãÐ)%qAÒù¦!‰ °.9ߌ$«o"ÅÖm²žóÍ qA"Ø⢈:Ø‚ ) 뜄|ì$é>]jfb·«~WÝ#¯D?wéï‡Ó¼™íàÝü¸ø~øý¿üáß›Åg"ZüÀ­røE8ÂÓçÀ–ð^ðtìXÇòO\fž€Ú©ámÐÄ]§ã"ÿø}¶yAiÏ2§9¤!IEáÕ. ž¬ó9Äý%X/·ìРÂ\[=ϱ™ç<,ç~š‰åjsa¹²òj¤N_V©©«0“Ö®û³¾áÉÜN”û)ý¿tm}Ú°çe:ׂñ £ø†ò˜|û¾`…!%ßYaeõ„iõ87ýb…1CÒ¼²âxnëÕEˆ´2ø®Ã½Õeáå•‚ÒhŠD«ëd«­.ƒŒegu_dº¿a‰8Ñט!ý~ú¼’Ï¿EN]9><#sUgLSÎοç¿4æö[ðµ|ñÜÞ¢ÍÜoÏ !—ß@¥‚ ¾Ž“â¬wuçê\uÕlowQg—hÛãœqäS£û´œAÏ*Çå,ä6Æp9#$Ï ®ú‘g»™_Ôù©A–›û:„ÏUOKûjYçyDÖ6Šïw*×YF%sˆÔsu.‡Ô¸Ï¶CN¨¨ÉȹƒÖZÍ´„®7Bä#¼ÖÚÖ¢Á5+æ4²´ÜÖWCjJ#õ\«!u¼}c©ªCT?ò.øÒOÞõígiÃŽíéÏ+m´¾®}eõäv>vãE52YÚ$£ÒÖXÑ?(ìa|y²ÎåP¹_îÅi½'½f_Gà³U™Ô!¥Ï -û¬1sàÖÌÒã|s²Îåfö †kòÇâ%»Ñäµc^&ï溜½ÌžnfiF„™sRÎÃËN̳õ?4YM¿7ÐÔ-¼÷c­‚ÚŽ_Üû äfNÙ‰a[~Îå±ë÷0ÊJjm”qZ£1â‹h‘ï};Âé¹:—Ãé~·òz¹×3âWusÙáXò=}$Hi‘>*Ûôg]–zÆוuÐFŽÂú£f픿¶;¿SÔ§Íë5“¢sû%'ë\ M´eíù³#õ(é¥Õj³Eë|doKM–Û¢àÓ²žõVù€Ãç¶:D3£nå!GÑZ»bÊJëYò£k ;´[þ™Ú]º“H=YçrH=ØÙë8­GÛ¸év°¯>îƒcf ÿγb¼0ÓÂeÊ·ˆÂÆØÛN•þ®ìØÊ‚[[zT¹¦¯§²ù $¶QßÌÇÐö‘x®Îåx)~㽎Â#W^èr½¼jº]»Õ9ä>_Î!¯¿‚³Bl®Æ»/ÐÛñòÙÝQÕÈ}b÷ïdË!w?v¼òÞª?U͹ôH^!¸Ci9N=%_Pò®î -ÍjOæŒ÷¯1'…8hK|‹ÉrÏÕ¹rÝÁn íÐÑfš—n=í?ÌUö=÷¨ov7ž‰GyœKOÖ¹"âÑn_¤Œ^뽶2”†£ÚQ_x¾÷¿ò+2òV»™æ #µ‰=µ9wÇÑÉ:—Cê~¼zi£‡üØð`Ëk[üÒ]™¦ü|Lwm”oG-¨lö$“ܘœ¡ò\Ë¡ò =BÑ£³ñVqßTodE¿”o 3=Á—çê\ ™þ Ö|ËÌåÊÊnÞÑ :Æ;u.ƒ`¥¾Æ’§3¢“u.‡Òsw:~ñÜûK ¸CÝ3È}äßúú‘§Çóò“u.‡ÂýØr¼“$ßG+íj·÷»ú¹œ4ãB˜37»0[û›y†óÎûªß'ïç8Œò -hù´‹Nú½Ú­LÆ7{‘g=óÉ:—CÛAÌø̸fGå:1ØÛrô׋“7š¿üd«!4<=ýŸ<ºÃý†üó÷¹:dĆ&&äGtŒ_²=@ìÉ:—CìÁ"ï+|Så¡,çk­¸.f}ëë롉”#ÏèÙGä¹:—Cä¯÷úúeWËnFµ¸ðø½E'ë\qç¾Ó_ËwCÚ{‚Þ?¯äæõ£j}ÆEŸwwÅ[G¨l®_ŸÝ!?Yçj¨ŒûWå´,ì²ðî_d•t\¹Å±½n}òŽ·“u.‡Äƒñ2œñw¾*žðâñ‰û2OÖ¹JOÆyI­¾a²ºw¼AÙê®^içäÄ·fVw:(µûÍœvwºØ÷"ØÚÝÓ©?~>êV|ûÅÇpÅáñ|üdË¡ø`ó›ÞAûëåëg»‰Æ'ö3OÖ¹ÓÁ~æ£üøw`|þβö*`‡ê±Üí¶ßªÛýžïKí{¼?uæèÅ}¨ý7÷N^#JÍÞ¤O˜gMêèŠäÉ:—Cì~¬Ú¿VšÝo‚¼æo«ïe¶Ï-|yÏ£ uOØu&MÍ^äiž«s9~û8´ÿ.ä -;ñæ·@™Û¶®VãÃ×ÿ7YãC~%JÔDîèºùÉ:—CäÉýͽ»#øÉâaþ¾¹«õW…î¢ëï0ÞŠk7¾?³ZI®žÃ[ÜÛ“ž¸òd«¡F«ÿÿrøšý—ÿõØÞ¨üq~8zp˜]çøåÑéØ}“á7F?\ÖO¿åÙ·~n'æ)™ûãH‰ëó}²ó³sí»¹ïÆšŸZÎÁåsXMå/7?¾C/FR>Úð›ïžËs“îü"Ž¨,é{5èY̯\Œƒ|1…ó]„µAjËÖkÅC±¶ªºµP5ÝU±¡læWô”¡óUXLo‡N7ÓŸËšÐLçžðå†nm/}¥ü¢·ÜT]£ŠÌ6F¥"u¹d™*ó$ˆ^n‚ð;•›&LfÑ7³Æ¯Ë™„1‰J¹”Ë ¢—›XnnÍ•™ -¥T¬USùjJ…ºeàÙ8F³4ð;d›5¢lþ¸{ž<×*6¸1+áË .†Vâ×› BiW¹B¹ ½Vh›v‹8ªŠ—*廩§OXšYÆ/áÿ'ªpEÊÌ3~ÐT Å/ ªGµ^›E¼¤Vî ë4šB-Û —¡`¶/•PžI·”.#¶6CÖÈCÊæ1IòXŠuA]3…åñ)\Ù -U6ÝU±+P~]ˆ«®,U§Jg^7pw¥Ý*ö…;_[q(3®ùíOÅÞÒnƒù,|õb†~ÌT±QÅÛÕÎa5 ¤PÅ•F_Gq^­…ø¶ùc]d’Ç}¹ýù6º9¤†_èá­æ—¶‘—¾ÿ/Êü€ß¿ÜèþÇüöœÉUØÂn]ˆ[OƯ¾'Åü*Ü,,˦jh¿Šu¬e]FCñãÆ4|MµºªÄžKê ºh[¡«c؈M®Ñ¶·#|éF±2Ÿ;7ÕaO‡6Ÿ¿“„ß»”­ÂA{:8ŒUg«K¾7M∋V—-™¦"ÚFqØt˜ªe 3m'eèÙ~j§Âr_ÍAFð¡­:ªn¶m:ËƳhŠ ¶íAÛ0é¶Ó´°G0úéåõÙó>›ìó -`i¡ö"}m¥åÄXZO–ÚñÐ/‹Ï±ã‹<òâ­#TÚ²ÍÚ³%È©K£†´8h1ÚHc­6pm§”e‹N³£(t™Ê²ÒbÈ6ù¹Z–ñŒ£8J¡ÄUúçv¤];+e];+~U–ÛͯDK¶@®…s¨g§jÉKFAæÛÚEJµ¤Í²uß´ì;{Û{sT ñ52Uµš[•|ŠÿžY;Sn'B*•ÆÝëûæà¨Ikìߤ¢÷àâ·ÀŠÍvÉÃuö÷OÖ»$yÿ¤í Ó1µ*hàmÄW(Ò±¢iêu~UŸ\-)ñ”a{¨ÒœZío÷6ÇM¢r8Uti|g«Š‘ßY”·‹Æ×mÐbûdüù?ê¬' -endstream -endobj -551 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 550 0 R -/Resources 4 0 R -/Annots [ 552 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -552 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb97-1) ->> -endobj -553 0 obj -<< -/Filter /FlateDecode -/Length 4426 ->> -stream -xœí]Mܸ½÷¯èó«åGñ ö bdƒ0²‡ ‡qïzÁ8€7‡üýd‰ÀÀY°‘¸ÀGêR^âŸtHùDïpì$. -Êu°…I\€ÄºK6E‰ ‚-Ö‰ ‚-H$.@FÐ%ì[\ÀÀE\lA¦(q˜¬‹‰Hâ‚à<Öcðª¤ó¦‹t-ê"t:H\uÅZ⬥ºˆI/q¾Ëé’Ä0:M„[0U$.@(ê¢Af.é4lA’-q2]è´–¸ Ø‚lPâ‚h"V™^I\€µ$\-éHåuf”¸ ’ïrª(q¦ Vš`JQ[@`|òbÛI\€Å›Xæˆ:ØbI\£ÉËÍ(qAŒ°Å(#q°™×›A₤` 2>‰ ’ò]N“%.C§”ÄIG¬7 I\€y™×›Qâ‚dÖ›Xâ ó6YŸ×›Aâ‚D°Å!Eu°ISæ-8 ëM°“¤ó°Å$’¸ yŸ×›Qâ¤Xo"£·b^oq»¼ë˜IÒw‘™z‰ p€õ&2[im¯@¼. ?“•1¯8£¸c @½±UÜ2Pà^D« î(o^#‰›0+ËèÅ]„U(µ· øù-8aØ}â[Tpˆ™1äN&L Bv_×­˜])6ìýöËé»ßó/ÿ9½û¦ß)+ B_š_a˜„ávȾ5ºo=HÑäÀœI*[ï©“©©A‹ã¦[6á/ÿ;/?L,‡á¯ûÓÛzú„(>œ>o!óÈøˆX¾Î-¾±Ê3¬°8B´õ(c“ÉìÈ!¯T4)üû7'u´bØ­‡Ì|ÄK,†É¸ó›÷§ïþòêïýá5`ôæÝé¿SXKáuùþ[?¿ßãðŽõ§Ê;u.—¡AæÜðžåµ|©Ït¥\­˜‘eoÛ9œTçç¦×nÔù©m¥lo›ÔV˜ö©Ôßl«ôeVFë©]î’“ÏQ§†×]íªïÿy~óêô‡7SÐõ™è:$FS$ÚÀé¾*OÁéÇ€'–µ×àI3·ïÎŒï±Á« ùd(ŠÛF^†²óaíe_Ü´a˜wä ¨òYXŽmWq¸³ÎÑ€ˆUÑ5$‹4˜¬ŒdœŽhé9 -›Ì™3ÔÃiÚýÂê ùïøÙ²w3–K— µ½!wÖ92é*GöÜ7ò!…%RÊ(WÎtS„ì}?Ž0y棇aÔóûÚÈ—sÎm©<ÿ0Cù\_ÐÁg•nýÎvVy˜Õ)ýõ‚}k¼¸†¾–RÞX÷…¾}u‡¾rmN3t}yÑt×ͳÂáõ[ÈR>u¡Š&/“¯£p_áp_žX¸®zø–e虣x?_0¾u3[8ž~¹y¨ŒNÓò@ -±Cð Þo sg£¡Óëžþ -RYV¹÷š¶u‡ÔLô‹à«•¹qɧ™‰+e9äÞk5vÖ9B7ö=ïÙ£¼‡w¾f»¯q£xûJ~gáó#åŸ/43xÖ÷ "½Sw\½¾¯ÎÑê6rÒÉ?Ÿ¤wöö+’vÖ9B7rÑ;yçæë;¾FúÝ‘Þ¹Û¯fßYçp輞‡Þ½îý #ý=WT}ôHo_Óï¬s8¤nä¤/’ži†8Ò{uûš~g£!ÔïÌEùµ@³ß+žà¿WëùŠ}ïÆøÕo‹êþŽoâwÖ97rÎÏqýþ1f×'ˆàþŽ_šï¬s8Tnìy¾H®y¦Ù´3‚ókE&H|`¨¾r­Éü*°šo|Ëéïøî}gátãúÏùïf+…ùug¦pÉZ½þí¯ⶨŸðz·E3Ï0jÝ~(å÷#pÊÇËëe„ö‹ ½ÝYçh 7îy2Nã‘pÁ[kH¼¶^ÞÁK/9Çœg9‹këW¢z¸ã»öu‡Æ}¹æ³\娟ŽÆµkí×®›Ÿ\—Q4þölµyŸ¤|Q¸Öþž±õ›¹U®twpå¾:‡CçÎ_¥Sóì—·^¾±­'­WyÇï‹vÖ9"7òËÙHT¯^Éîïâ°+w߸%äWÍÝ|÷¥Ãÿ~å¦[wìiî¬s4´Æ_¬¿TN{ʘf+¼ÉLü¤3h-Ô„ÑN68µ…Î}u‡Î¹çJö.íÐÔQZÉÅ6ËVÆ¤Þ 3÷%uýÚ»ô¥ƒ%ŸŸ>‘o@ëµÕ&ö}yõÇ?ÿéoóßçç}I– ÷²žÇaWíËÔÞ7²}CYòì,ýŽò±lÖä9å¥ k·îÍò|»Düìû¥<·Ÿíhùþx^;|žþJz¬×ûs“æÌ>³“}„¡OvôCÛ}smWúÄúßÛ훯ŠXÏó{`m©æï\'Û¤.Ã,Ö~욟ôÉå;Ûü™Ò§j>Çè·þ¾ Ëv¼Í·WÎðX¶SýNc[þª1®Ù¡Z_z÷á0·Ûò£\¼"ã“Ðvç4ÔÏýëû˜Ûzà8m5?¾(6¹1ÅækçÇ1 c»£ÿŠýÈö—y0ü•ã[±5¦waË\»æOo,Û‡¾×öz¼™ppðžsùfîÑéÈîM\ãØf‘°µ¼“q¹ð9ÿÍãRP4™Ÿ"fãY¥o¼-qwÊŒQS÷ðÙ®±#.³K*ǧ±Œk®ÏÃÞßW pˆ†Qw×ë‡Ë³áLƒÜÔùÍàä—¸ïÏ7â·‡YhCYÛwlx ÇúP~¹˜Cùáäû›ÈëþA"ÊÇN›ü¤±³§.#-¤s¾q}T–t:Ý;[ŸWjöÏgûPKlSù).ùþÃóÜØÁXþC?èïV2J 鯤zq4å ‰OùñZ$ˆ‰Z4ÚgáøQÕ¶p™™ì‚ä,ti^Òºrûk­++½Éõ]©–o{Nª²ý¿‡ BÔôz)νWÅùe±³ã‘¯¦8_l†°5H¼lk°UÜk«˜Ð-…Š®‰ {t~¤EuoÂj:wfÃ_ÊšÀ†óšðrÂií\ú(Jóƒ‘JSÅ»FU™eF¥*u¥dªã$ˆ.'Aø­*M³êÙ¨åÇK ÂX€Dµ\*åÑå$–u¤B-[ÕT<ßLiP·±:^]X–ò3ÙQ¶|œŠs"RÇ!¹áÆ,„—B -Ä'„Ò®q…rz\hY»UUÃK“æû9 Ÿ05‹,ßœ?ÿ?P…«ÒÌ<ýMRùí¨$ÏÍ*ž:‰Ë}e’< )´²Ì]:„ŠYî/•P>“n-]=6‘2—1yHżLR•<¦b]QWřŠ*Wr¡*FàtMì*”'âf§«S`/'óšÁÝÕv›ØWî|äâPG\秥T{k» óExéÍÅú±œ¤‰ªÑ®Œ ›a …&n4úØ‹Ël­ìÔߊsüØ&™q/§_O}˜3Ö™„Þêüð/žyéóQæ^ÿ:Ñù‡ò´‰!ŽUÙÂTnˆy$ËŠŒ;ÅùÑ‘EX§ b´ßÄ:¶²® ¡Æqc_S«®±—’º2¨®Z.t͇Ll*t¶Ì3îŠð2óbMdž:6-`‡¶t§±ù9%Å*ðîà06m!ùÌšÄQ.ÚB:´dXEðFqÈN˜še33 ùIªë³ýćÂæs±–FäCÛtÔÂ,oºÈú^°"H-?à “æ'M{£ïž^O÷Ñd_f@–Vj¯ÒG.­ËÒJx²Ôö‡~zXcŽ=çƒ2y›‡j[–Í=[“œ65ZjA“ŽQ&­ÚXÀñ“R‘MNZE¥ ÈT‘ÕC±ÉÕŠ[úGÑK¡æMú+÷´ã£Rèø¨øEÙÜn‰x5[²5rΡõNµ’5—Œ‚ÌóÚUJ­¤-²å¹izîmϬÔ ñ-3U­š[”¼‹ÿî™;CÙN„Tªüõ»#:¶ç3!P“ÖØþ‘Û*z¡üÔÄ@‘m—Ü\çúþÉr—¤ìŸð3a8:²Vž.›ðEÚV°¦ÊƒÕåjI‰]ºÍÂöP£9µØ꟞Û÷ÑoÕ׊ùaÙ6'½1?ã3(-oõ÷ÛD“í“þï;#³K -endstream -endobj -554 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 553 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -555 0 obj -<< -/Filter /FlateDecode -/Length 4783 ->> -stream -xœå]M丑½ç¯È³‘ùý æ°ÀÚðÀ6v€Æîa±‡êì)F±þû~A‰dˆŠT*³«Û9Þ.TW*‚Aòñ1‚RRú¬ðóÆÉé)¥ƒ;_~:ý|RSô -ÿJŠÕEQÿ€ßèÏZ¹8yCòçŸH`ã³71œßNøo¸äÉßNÿó«Ó_×yôÙL9£ýùo?’ÊÙ)¦•T*¥­ ò~ŠÑø ¨‚žÐÄœ%ÌðÑ·UE˜árò’ -f8c’ J0ú¨íV•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚO$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ƒÞÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓVg”Ÿ|²VI:­'lZQ‡…A -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) ³ÕÐì¶ÊjIça‹1щ:Ø‚Á—ÀØ¢b(3h"^’8ÀÄ8¹lœD&ÙÉ%—$0)O.F+Ñ€É~r!G‰¬ÒÐ-U°Å» 1Õ°ÅaæŠ:Øb³“¸ÀØ‚X [Œ³X°­Z$.° [§²–¸À‚oÒAâ Â…™Jâ Ƶ)8‰ ,(PÊXp®wJ\`Aº˜îQâ Öµ>‰ ,h׺$.°à]ë´’¸Àw­ENQ[`‰ ÀŽ“ÕÉJ\àÀ»V«(qïb‰ 0ªp$.V&“’’¸ÀwMRNâÞ5K€¨‹æ •¸ÀwG¤•¼k¼28ð®q&H\ Là%zà]Pˆ—ÝØ‚•¸ë=tÆJ\€Â&·DâÞÕËŸ0§ÁV“N9H\àÁ»:aøD]žtD ó½<é²Ä¼‹…ÊJ\àÁ» Ï$q8~Šj$.ðà]m±RK:ð®6YI\àÁ»ð¼Ä€Ã¤áH\€õmÒ -ÐuyR9%‰ Â?“•‰"Î$î(P¯ÇÚ*n(p/V«(î(/ÅHâ¦ÌDd™‚¸k€eJ­Ämþ…?æ6œ0ï>ñ-ª8D¬™)R#3¦…ƒwßbŽžÌ^I6ïýíO§_¿`ñ=ÿéï§×_•²ú£~ëB ¬1ïB6ã ¾šJy&œmC YÉCFÄÜÔ ÅeÓŒÄRŒå6ŸÛ½6“ÍÎLi™[Õ‹PAc¬lm½;ϻۊÂá3xxoëzE#o%¾i¥çí‡F"¾ê páŒ#Š=šéÞú?>œÀÜó®q83l‚¾ xôÃO§_ÿ›ßýö¿~Àh}x=ýï·JÁj8ÍJ¹„_|vüæå/];èã|í»{Mþ)Kó:ç%m€ú¼äñóuÑSºåšäE¯—ò?Îù=×ÿ8Ët-éÝË’NÏö”´v¶ÉÑ5L §VvmGZÚç¿¥<”ïÓlS©G}÷çß_é¿Ìûº¬Ta—. Ý|ŸYSÈÌË’VÍÝ ìº K·~ìÝ×Ì_Ì­eîM­é[Sã’¦Úåò©Zú­ÝXºÕ÷!ÔKýT^±eiŸëáßï.ú¸î¯ÒW¯K—¿,eé¹Übgm‹gý•„>ˆ³ µíE¶À­•Ÿ†¶»ÞµÞ»š¾Ö±Øb‡~¶~€ïScœZíçÝþ³ÞnÉ~?S·iÝÕë絕ò÷òvÛƒ°gÝãXß_–éüi.ã??jµm±‚£kl¡VÚ—CPb<ü…4(NÇ÷æyh[Ø-BäˆöN6ò´á>©ŒÈiCÉ®Ö:wM»ÔÖ·µn›ÃÂaTaèßÿßücèÂXþ»o4Aá2ÿ-C^p¹ø;ÂÕ -áVªL{.C…T‡ZÊ™a»Ô•ù~¹Æfø•¸ˆB¹þõ»¹áÌÓFe׬º‘Áxö;)?M_{ni•?Pf¹è|°;=5¯!Ëç¡Ž™HY}|äÂ`Ï’w^<Ö²íGÌI9'ô>mK£ó =ÖûŽF¼„PåÝzß!$FXåB¾SŸÙhìÖRiþÒU7É8ºí. -¢Ÿdˆ?®}¥º@ÕxE i!ã¸!GC7z2m©•–Ù~… íÌ/¤F7+v¬îáÁ<Ÿã–~>¤}×ùP½ÐÒµŒ±YÆ<®Ç~‰õŒóq=»êŒtƒ¾ÍTÓ¹¶,—v[Ÿ Œ/k9I®§ÕÊz½bÏenkÃøGÆ*1Ä¿ªO^¾;>»Ðà½ÙÕfNœgNq§õ0»6®ž0³Ü”³OQîÍ,L:K¼ÂGš$·fÖÁ<Ï6³lT{3«­Efø×hÓ56{™?oäËo“»!]§oÉó±ëŒaéìòûiùë–ÏT>gÿ¯µËœûZf­wœ)±¦¿‚ºùnjA=èƒFÑžÞ>êŽåyÔu³éáŒÔÙ5Úöø°ô|fº×õ–0Ç°Ñ‹ë2æ†EVGPwý.¯G~•çG†¬¶;q w¶µm!,2µÎó8"{ÍŽ—ëHuªs%d¸ln!õ`žgCª3ûühäÄŽšŠ¼yCb‹ ÍHKèz'DÞÃkÜ6ŽÏfÌadi¹¬¯†T«ïGê±åy:$ÞðÿÅ{G†Uy¥«ùê¬ví6m¨u^Ž!o¼{³Aìˆ.¶º¯Ð;ðòÑÝQÕ2rûó<rþïøÌ;x{¨þlT±¶ŒHÞ ˜û¡nÝO#¥µ Å]Ú͞ÌÕ?tŸÓÅ4iëè)“È=–çé{c7Ðèà‘&[¥ùJûÿæû‚GÔ³ÝGüÑààÒcyž‘7üÑa_¤õ_½®E(Œ£x¯¯V¾—_ø™y›ÝLó…‘Ê|Om¦¬’¡¯ªì#õXž§C꾿zie…üÄxóÚ5~îL»øqŸî¹Q~Ýk‘QÙž$FíÐ3pó<*ã _ôŠî÷òûŽ úJTôË@ùdÚûùò`ž§Cæ _ó=#—(¿+ºyoD3tøË3!ø -J»/y8":˜çéPzìIÇ/{ ¨{¹÷´ñ½ïÅt\~0ÏÓ¡pß·,O’Ôçh¥]mþ¼kXÒI#.Äð7cf¶ sm³ŽpÝyßÔûàó7=:8ŸÞɸW{-’Il/òèÊ|0ϳ¡-Ýð¹ÇvTžÇ{_Žþz~ò„ÚûŸ?˜çéúøýôC|rï÷;òÏ¿ç츂XæÒD嶷{,ÏÓ!öÆ÷"ï+|Så®(çk͸Ág}ïûë‰ù‡ÞLÊ»@èÙGä±ö­˜ñ^Ž¸Ÿ zyüYÉ«÷>ªíSïä|ÞÓïí\A%»}t‡ü`ž§Cå}Ègå´(ì²ïXÝ¿È,¸ò*?òûÖŸx;˜çéxÃG|Îø7Ÿ¬â<—y0ÏÓ¡ô ßX{ЩÍ7L6ÏŽ3”mžê•vN|kfó¤ƒR»ßÌá»Ó;‹`ëðL§þôù¨ÛðíïÃ+‡d)u@~4ӳḜUúž!ù»>DûË¥ìGÀËùÃrœjô9ïé8ëäub§³6ÞLrrõ°™í9žg=Cº>EÒ{”,»4¹éèák§Ñ˜çoêåðAú¦Cù&a=´0/÷gè>M^ÒÔSlÊ°ÎCÚu\ε¬–m–ËÁ€y–—Ãñê©n‡"ø.õÑ÷ô)?Ý{fC¶òwÎÚlmðµ ›¡üùÊ1Úº¼JA…4iCïZ:7Y›|¦£»“²Nסӥ³õùJÎò†ªŸ[*ú"Eï± 3ÌËÐò‹%ýÏeÐ:ÑÉ~ÔÿÄó fT'S/ºøíD/r‚Ø!ô>(&áòQµ²œ® ›ìfSSZ?©åcn…Óq<óGãkk(¿¯Ùèàg§j"[Ž>ž+„ÈôVL­W“^Äô’¦4Ùå*4Sèð¼&ì:ž¶Ø3Þk«˜Ðo…ŠU×ÅÆU{èÌú1·1§£1«é¼ë4þšÖD6œ{ÂË ÕÚQú&JéÕ0µ¨Ú»XŽªÌ2£r“úš² ˆkã$ˆ.'AøªE; fÓ³Q£ögaª@r-]®éÑå$¦[Jóm¤bK•zÖ\{¾›Ò¡nSëxAVúh‘Fzë›#ÊÖkq ¨c­ÃÙ/',1n#~;ù(¤ö+”¯ÐãBËÊmâ¤:^º”žP˜?ajVONÿÏTá›”˜§|ЮAŠ^1Яz%47›xÝI\ë8©´‹=-ë.cÃ,ï/”žH·¥n=¶’².cò˜«yDR<ÖbÝP×ÄDaµWr¡ªF º.ö Êo+q·Ó·©êU«,hwßÊíâиó‹cqMï‹hö¶ræ«ðR67Š ú©VÒÅFµÕ®WÆ„Ý0Bw}+â:[;Ñ;ðêÇ>ɤ÷rúó©,s ¬¦×qÏKŸÿ4ßã÷/'wþC=o^Çjla·®Ä|%£—奃bzy^¶iÄÆ1ÚïbzZ_ÑÐÖqc_»ž]ub¯)ucPÝ´\è{2±iÐ5Ú²žñ;ÂËЋ͑ùܱé ö|ikãè¬_zSCµ -¼9¸L]gû’|fE⊒ö%ZgXF\ðBqÉ*Ìݲ•™H^Iëz²ßñ¡°T»`n]Ú®s}™åEWYiKØò ^°Ó¼Ò¼²G0úáéõÙ㾘ê i£ö&}ãÒÖ0’6“¥¶\†õe[sl9·NÞÞC­,ËæžmNNŸݵp« ŽQ&M=Û’ÀóJ]•­*­ E£ ÈT•µcµ),Ùj·”'±—bó)ºôϼ§=•Ö‰žJؤ¥rëŠ×¼%Û Ïá{ëTOÙ|É$ÈÏݤ®§´U¶­Û­ë®«í™5ÃuCB÷LUÏæ7)â¿GæŽø)Uý)»#:õ7Ô`¡v:ZcËK‡U -âè½qÑ%¶]rwžýý“í.IÝ?á5a8&g­Šš^-›ðE¾­`EÕWKËÙ²›t_ÛC涯L)ï-û–e“¨]F -éuÁ–œÞDo9ŒJ~…J9Â6¬¶OÊÏ?h  -endstream -endobj -556 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 555 0 R -/Resources 4 0 R -/Annots [ 557 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -557 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 698.673057 101.433071 679.923057 ] -/BS << -/W 0 ->> -/Dest (cb100-1) ->> -endobj -558 0 obj -<< -/Filter /FlateDecode -/Length 4384 ->> -stream -xœí]Mܸ½÷¯èó«åGñ ö bdƒ0’CÃxÖÞE0àÍ!?j‘,±K-uÏŒ­±íA»[U$U"_UQê¦>*ü}¯ñ_$=Ę‚§ãý‡Ãǃ‚Sø7–˜Œê×xwÔŠÂàtòÑ?d CHÎ|8à¿î8üã»ÃæuôÑ )y£Ýñ·wYEv1åU‚Ji+¨œB0Î *¯\bJ’ -f¸@†ÎUfPŠNRÁ 2& -ª3,mÏU f˜ IRÁ \TZÁm”u0DY#è´<:ÝK:£“I¢.@§ƒ‘tÖ>vZÒ¥Á{#Øâ’Ò’ÎÁ§‰”¤ƒ-dct¶Xo•¨ƒ-&+él1è$è"lº¢¨ƒ-Ê+-é’\ŠD¢.A§bˆç:£ÜࢵJÒi=8`ÓŠ:Ì( ’‚ÎÀ¯\u°ÅYe$…-„?Ižp6Ä(ÌV“g·UVK:[Œ $ê` _bãa‹ -^¢Ì !ó’Ä&„’!‰L´EŠ ˜˜ -ÁJ4`’ȧ ñ€U:£%"° -¶8òX [3WÔÁ›Hâk` -H\`-l1d%.°`[Z$.° [RIK\`Á·¤´—¸À‚pa¦’¸À‚qmô$qåJIâ εàN‰ ,HÓ=H\`ÁºÖy#qíZŠ^â Þµ¤•ÄÞµ5El€%.;VG+qw­VAâïb‰ 0ªpè%.V£’¸€À»&*’¸€À»&Àˆº0`Z‰ ¼k.AòÜà]㔑¸€À»†Œ—¸@À?JŒÀ» '‡°+qü=tÆJ\€Æƒ°DâÞÕ îO˜Ó`«AÇä%.pà]1|¢. : Ç…y‹^´÷IâÞ…£²8ð.è3J\ŽàQļ«-<µ¤ïj“”ļ‹¸ÁI\8 €ÄðoƒV€®¨KƒJ1J\àÀ» %#q¼ð "˜DÔ…@Ò€Œ…8TâÞ…{Oxð®rÖJ\àÁ»Šöˆ:Øb£‘¸q t˜¶ÂœF°:(¦tà]¥1Â’¼«THxðnv›ø&J\DÁFâiH!x‰ |RÐ!äu~À±“¸ ((KÔÁ -$që!Ù%.¶Xc$.¶ d¸Á4¢o1=€-€‹˜lA¤(q˜lˆ‰Hâ‚à<ò1ôª¤ófˆt-ê"t:H\ pÅZâäRCĤ—¸?äpIâ˜&‰ B‚-˜*À Ñ 2—t¶ È–¸‘.tZK\ lA4(qA4Y¦WÀC#—DWK:R9ÏŒDòC%.À4A¦ ¦u°4!qÆ''ÛNâ$?H6‘æˆ:ØbI\£Ééf”¸ FØb”‘¸ØÌùf¸ )Ø‚ˆO₤üÃd‰ @ãÐ)%qAÒù¦!‰ 0/s¾%.HV!ßDŠ'ÌÛd}Î7ƒÄ‰`‹Cˆ"ê` ‚¦$Ì[pòM°“¤ó°Å$’¸ yŸóÍ(q 䛈èÅ%˜óÍ .€w]3I:ð."S/qo"²•r{âuñ™¬Œ9ãŒâŠõ:øVqÉ@{á­‚¸f @¾9G `&2ËèÅU¸U(µ— øñqÂiõ‰/QyÀ!ÀgÆ/2aZ¢ûšs´bv¡Øi}ì·_?ÜÁùùïáýwãJYùÓÆÒ„ø -ÓÀ$ ·Cô­qùÖä£cÎ$•-„·ÔÉF„ԌԠÅiÑ- ÷—ÿÏ?Ì,‡á¯ÇÓÛzú/~:}^4Bä‘ñ‘¾ö_Yå VHŽàm=ÊØd2;rˆ+Í -ÿþÍ@¬8­ÖCf>ü%’a2îøæÃᇿ¼úû_z ½yøç·xÅ¿×ù3MïazŸäÎÞE~^Xe¥œ›ê+ÛêåWÑ×va«‹ÌÏÛ¸Ú¾"{ÛÚ¨ç%VîŠv‹Mã5Ø®ÍÒÜÝ·s›|ùoOu ¶6ÕônºëŒå?þëøæÕáoæ #Ó„9)E¸Eʯàv[•Çàö9àŠ4÷\‘ý¯íg݃KÚRŽÙs ìn‡ên°kÞ_2üòÊüµøÛXgoDvt ŸM®kç ºµSPÀGýS£ÌÆe[ÏfcW®:®Þ!j[é-/³"!g×¹­ÎîI9±º$Ólh#«s¶¦—_•Úskä¤Q§ó-¥=ÝìëWé»÷~&¹v Äy¿¼Ûy}ý3^&¿PÂ9$qIE£VQ³­ÎîP³ø½c½Ýsç™8ïñžãÆ°ÓÁu#ú~ÞN®WG=5äð€²ÖYâÕž;5«c:dÒ¼ÍÏ>Þ±sÝ_à¹ÆmÎX6ÖÙb/Ç~<Ä.£Xzpä83u¨0¬|©Ït} é8ß -(ãuØ(j7é.!Ij+̯©Î¬µ¶–P©ÏÑõ4h—jZœç FS$ZAèÆ:{C¨¹ÞÊu4JÏßM(¼oå³Læ¯qf, ±Åw.ßΈq\š¿ŒÄmuv‡Ä•˜pW¼ñÎcEf‹ˆÌ»õè4mO×UâÞñ³eïf*—ž™#Ý ¹­Îîy9ÄfB.¹–Íl}?0y棻)+º[ùšet¶Tž¿ëPÞë :ø¬bñ²á –¡«S®WÊÖ–xq }l)0ß÷†¾muv‡¾•riNóÈü«ó¦Û¸®B–²Ç%Zu}î½±ÎÞPh·Å‰…ëúÕM#zÍ—E¾Pç74îÙ9iÞîŒë^lº€ÎRˆƒ¶üÚ -÷Æ:»Cç#bÇÏÊ+O8sâܳ{%ݪwYqz¹Ók&BLY¼¸Œ=émÍ[6ÖÙö.G‡c¿…i7÷åÛևưþ,ÝÏÝç®ßWñÔgè,3ï±P꽌µ‘MËÝÎTB´¸twddоýé®LÆroKÍ£„»@uŽteG;ØÜ[ô×ñFÜVgw¨\‰ ÃÝÏGï1>í2ÏZ½kP°…=㆞UU×ü~d/€çm]nÇu£M+y ©ëyrc½!’VîB³;Ëgw¤YdtÆ]üž ©Yv|6úvãè_a1{ç‘[7kú˜À0Ö¶º{„gÚßñîŸéמ|×'eu _+Ò'ý²'¶–¸5ÃÙXgwø¼CžEç;—îÎI™ÆÚŠQmã^o>¾WcIÂ%g­O5nÏêúÅsòh1#’Y»‹¸±Îî¹rßû|ùžWv^D,c–ÝÅÞŒÙmuv‡ÙËñçÙì¿å.Ê -KŠwLœÀLî˜<ƾkï>û*Á“:öTcrƒoˬ=A¾­ÎÞPéVV1»Ñ®#·ôÄË·µw™¡¯êÇTÚë¹rcÝ¡òräù˜/(|Yüuà }Ò5ý¤Þò œçÿ -γ u%ý*øjaF®“ßXgoèôϾÐÈàIgÜ'ðôÞ^ÿäúÆ:»CêJLú"ùçË÷ôþ†om¬³;„Þþm K¼sõóß<ývO¯š}cÝ¡órzsÞûzú[ž¨znOÔõ9ýÆ:{CjX‰I_$ÿ<Ñ Û±§öúœ~cÝ!tc,ÊŸ꾯Px‚_mä+v'ÞM#ðÍ«_çÕà wâ7ÖÙWbÎ/1ŽÙõ)<ø ß2ßXgw¨\Yó|‘\óD³i£çÏŠÌxÇP}áY“þ)°¯ÜåŒ7Ü{ßXgo(Û~…èì›å ¿Òa -gŒ Åçßžðé n‹ú¯÷Û3š>¨u7öC)¿s>>îp¡ìA›º­Îîzåš'ã4î Ïxk ‰—òå ¼ô’cÌ>Ê9{¶~Á«Çîµo¬³;4n‹5Ÿä)Gýx4.=kñ7¼ÊÓyEÓwÏÛé¯IŠ…gíoñkß™[äÊxWn«³;tnüV:ÿÒ¯._¾²­Ge„2"Ó ß/ÚXgoˆL+ñe7ë¿Òx#‡]øõkâAþÔÜÕ¿¾´ûoÀ/ õ†5Íuv‡Ö•o¬¿TN{Ì/1uÞl&~Ò´€ÎGjÂh'œZCç¶:»CçÆØs!z—Vhê(-Äb«ße+còzúíþà`¾syï”ètd[ÔX-²Òï生ß?濾cýÔ£¤4äM;m<Â)Q -yŸ‹Ü‘¯þøç?ýudþYbóîô#à§àŸ=^˜ÖŸŽ3ÄG'“OÓTfü9ãB 'Jp#Üs禚š~*¿ç¶ryz"ô,7¹¬?½Æ¶¦Ïc;± Þt¾q,bä“~Ö¾›ÎÍìÛpìÊWoÒýP~<øqÏ=îÛ¥|´É{= 6M:æ}b¢²¤ËÐé±³õq¡æ¸êÇZjÚ!UåMÓò†9§íSÙÁTþã8è:Ñû?O‹Ón†jGSšøáw³$ALðGÅY8}Tµ-0†éd÷`—0¤¾¤ue· jãÖ•Æ•«Éõ]©–w!U -ÙqŸÓ‰!jz}.ÎW¯=‰óŽ q°Ó‘¯¦8_l†°5H¼lk°U\k«˜Ð ;]*ö較Tí:ß„ÕtÞuš )kÎKÂûNk{éƒ(Íû–¦JïUe–•ªÔ•’u@¨Ž“ º?ÂïUiš0˜UÏF-ïætÆ$ªåR)'ˆîb¹©5WG*ÔR±UM¥ç›) ê6ÖŽdcMÒ·8fsDÙòq.öyÊX«Èpc΄÷¸:?\J»ÆÊèq¡eíVqT /Mšäô S³Èò^8ùÿU¸*ÍÌ3~ÐT!•÷³jGí$ynVñ¼“¸ÜWÖ!©4…V–u—¡b–÷ò'¡|&ÝZºöØLʺŒÉC*æe’ªä1늺*ÎVú§r%ªbN×Ä®Bùa&nvº:Uör2¯Ü]m·‰}åÎ.uÄuÞœ¬Ú[Ûe˜/ÂûѱQ/ÎÐå$MlTõvídLØ )4q£Ñ‡Q\fke§ñW±§m’I÷þðëatsÆ:“ðÁ[÷Úä‘—>þe^áõï*›;ü¸Q•-LåÖ™˜{²¼3sÜ(Î;5a6LhˆÑ~ëØʺ‚†êÇa|M­ºjÄ^JêÊ ºj¹Ðµ>dbS¡k´e=ã.ï»^¬ÌcǦ9ìÓ¡-‡Óؼ-X± -ürp›Î6—|dMâ(m.Z2¬"x£8d'LͲ™™ü$µë³ýćÂæs±FäCÛtÔÜ,oºÈÆ«`E[~À&ÍOšföFß<½=î“ɾ̀,­Ô^¥\Z/,K+áÉR;úùaõ9ö˜Êäm=TÛ²lîÙä´©ÑB špŒ2ilÕ¦ŽŸ”ŠlvÒâ(*]@¦Š¬¶ŠM~ªVºe¼â(öR¨1E“þÊ{ÚñQ©èø¨ø³²¹Ýâñj´dkä8œC»:ÕJÖX2 -2ÏkW)µ’¶ÈÎÏMóso{d—AÍß"SÕª¹³’7ñß-sçT§[‰J•¿quDǶ"5é`=¦îDïÁ!”7)ÙrÉÕu.¯Ÿœ¯’”õ~& Ç@Öª · Hë -ÖÔô“¤\-)ñ’®ëay¨Ñœ:[7«ïÆE¢zrªèÒ¸¥°Š1o©”—‹N¿Å9[>ÿþ69‘ -endstream -endobj -559 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 558 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -560 0 obj -<< -/Filter /FlateDecode -/Length 4776 ->> -stream -xœÝ]K#9r¾ëWè¼Àäò|ƒ9° ¼†h؇…Uê®]5f}ðß÷Ìä#™¡TJ]ÝÖ¸ j)#ø’¿2S”>+üý ñ_$=Ř‚§óå×Óo'5§ð/§X]dõ/xwÖŠÂätòÑe SHÎ~?á¿á²Oþ~ú÷?œþkGŸÍ”’7Úÿö…Ud§CPNP%¨”¶‚ʹ)ã¼ òzBS’T0Ã2´U˜A):I3Ș(¨"Ì°´ÝªÌ0A“¤‚:¹ ¨´‚Ú(%ê`ˆ"²FÐi7ytº—tFO>&“D]€N#é¬| í´¤K“÷ÞEIG°Å%¥%ƒ-N)I[ÈÆ éI\àÀ»pTVâÞ}F‰ Àñ<ª‘¸Àwµ…§–tà]m’’¸Àw78‰ ‡I#¸þmÒ -ÐuiR)F‰ x´d$.€žT“ˆº0HZâñ¤‡J\àÁ»pïIâÞUÎZ‰ ½ý!? ð[ò¼°6ˆyÛã ¾šry„&‚mÈ-„äa#BjFjÐâ²éÆFÂÃݦsý ×&Ãb¶Y#˜É-£U½X*hŒ•õ¢­wçùp[-Ö9‰½­ë¼•ø¦u–ÿ·n‰µP¯Ç¬#xÉÆÍôFoýwŸNŽª»Á?´Ë©X¼#ÌÐ&ž?ýzúãÏÿðOÿø¯¿`ì>½þü#¸ŸðÂÜÑî§ôËg–½Î׈…•‰óg§—w¿¼¹SŠâœ“AÑ’}•¯7ò%-ªÊ/ºlËæ4ä·uäôn„2Ëiþ̯’¾Úê[{²]¬éÊWKúbÇbw®‡óþÇùÓχú;õýýç¾Û̺辩E¦^ÑÝÔ™º|ªuiîÎsùi6Ì/ùÞF,ã¼Á»Øbk·pøü6—§?oËk úÉ;l›r²ý¥+}nËž>\³Cµ¶pW×öq·î;Ñ6Þiõ -³0 eÏme®!ØàÇýlùýRÆc± F¹Ô>—¶²=ZÚÜþ¦Ùö´¼{¹N%Í[7 ÌRßK¾ÎúË õ¬w‹^/å¿6û´Ùíû8öO.ëË2J}4O‡y -,˜0ëvs*u×悧0¿çò¾,øê§--}v¹Þ×L?®à@Íý ìºÇq¬6ù67ê¸éf?Ž~2jì§ÚÎ2æfivK6­©­µ—–y'ÓÙ-*›_«¹çº¾Šëök½Û^Þ`^µwæõ¼)sïï?eh[HÁÛáÙòî)–ŽÆãÄÆ€;§yož‡ÂÅGN†×÷hïd“áõ‚\•°¾Ý8N*µÎ]S/µuÕÞh±$q€söœÿüó¿ýËŸzÏÉ°Z¼&]Š‡,Ó¦ó:ó \­Á¿J¼36T8SÙâ•óð.u¥E¾_®± Ñ?.‚P®{ûéFn,¹x;ÙûkVÝÈïü”ÆNÊŸýné¹âè”™ïÔ’óv§§fâY>uÌ««¯9?سä'ùZv ýdó¶ /!è}¾y€ÎÃ2ú±Þ'=p”Öû±b ‘àr÷qêR7»uTjTËsRä)kx\äÅø•½{];ãU€d¯=Û ‚oÇ%Þø*ͳý -¢ìBj+`ÉòRw—æùšÅ÷àCëiÕKïöŽ3‹6«a– -¼Ja;‹‹¬¼o°6Fÿ/6‡™_äsÈPÂïAg÷ë©×zÉ{ç¤ÜÎ+†C r¯ô8_‚ÈŒï(<ÆÒ”’‹Aîaœ8üŸñꔟŒ¦Htãó<Æ)?sãÕ+˜á}À‰.¡õËüy#_^UNC:¾Îá×’çµéŒéÒÙåõyy§å3—ßóð÷òŠ?vPÊ,õ†¡ì:Ÿ¯ !DA?…Fñè>êŽåyÔ5³ƒÚE]£ÞÖ=Yõ²ô|êtoëdðäÈ5®G¡”1/6ËþMǾ ¯Û‘_åùÒ!«.."áζÖÅbÙ Yçy‘­ŒjÇËRcã:ËÈ dn"õXž§CjÚçG; '4ÔäÍ[[mFZB×!ò^ëmëÑàºsYZ.ë{!•]ñ½H=˜çÙêÌ>§ªQ¯]Tù2𥟽ëÇÒ;´W¤w^¼Ì´l}_ûêì)å|ú‹Zd²¶é -*[¬hŒŸ”ÁŠèV|y0ÏÓ¡r?¾Ü‹ÓFO.zÍ1Àg›4°®º:²î¾9–çéFöF WzòÒyÉ¡7ya†4eôt7J "8ßÈPoÝ~È£ù_»U_#­Ï[y!îÇZµ¿¸—+È-œ²Ãöü\Ê}ç´Ö™ÔÛ(ã´EcÄTD~úNåy:œîGpßÑï†,ñvû*p,ùž1¤´ÅÈ•]õgÃ*õˆ¯ëwfhŒþºùGÝÜ©ïvhßç9ê+7ÞF4ù.bRtl¿ä`žgC“ß²öüÙŠ‘F”ŒˆêöÿV¾mõ·u9ºÛ̌җç‡<£o»Â b _ÊÍŒzmr+ZëgLi#K~ÖZ—µmùžÔÿìû]º£H=–çézcgoà´myÓMï§áÑæ}ZïeTŒFúínçØÛΕÞ7v\[÷¶Œ¨r]]­æ¯ ±úøÑö˜GÞGâ±Ñ:Üå™|¥-–<¼":˜çéPzìIÇo¾öþP÷rïiãGß?ŠîþuùÁ4:»3í*o&¹·ò•óíégg=Þ¶þVzâãâkm€ªøÀÆkßI7_æïèåÀ~Þ1Ÿ T“T;c,-iÊwÙó0ÍÃZñY'*‡èäïµ—ÃsÞf~ay>¬f9§?§ã¥êÁO&Ÿó¥æ‹–³!Våï®TÛàJ6CùÛÉçÃGu>€Zù8iÿPqö4Y>1ƒ.µ¤ËÐéÜÙú|%gþ]ßj*þ"ŧóɯyhû‹%ýoyÐ=éD÷>˜,˯²8šrÑÄï'þYÄDít³b.U-‹t=,­È.`Õ0¥1¥u“Z>¶ÕøKùóGãJk8¿+Ùø¸LR%‘ÍFÎ Bäôz+æÖ«I/bþi‹8ÙåÊWSø0›*lRŸ¶Ø2Þk«:¡Û -UW]*öh> -¹voÂjzßuºþ’Ö„n8÷„—ªµ£ô]”òú¥¨Ò»pEEf;£R•º’²ÕqD—“ üA•¢ ƒYõݨñ±Ä³0 QM—J:At9‰é–Ò\©PSÅ–5•žo¦4¨ÛX;^]ºSÿVO7G”-×b>!¯ŽµŠnÌFx9ÁÅÐFü~rAHíW(W × mWnGÕðÒ¤|Ÿbþ„©Yd|¨+ÿ?S…«RfžüAS…ÌÜ®Z%<7«xÝI½ÜWÖ!©4…–¶ë.BÅlß_:*!=“nM]{l%íº¬“‡TÌc’ªä±늺*f -+ýS¹²ªbªkbW¡ü¾7;]ªNÕʼîàîj¹Mì+w¾÷âPG\ó)ÛÕÞZn‡ù"¼dÇF£˜¡K%MlTõv­²NØ )4q£Ñ÷,.³µ²ÿrPùØ&™äq/§¿ž²›3Ö>±Î[Í?ÑG^úü?Hó3^ÿy¢óŸÊ)ų7ª²…©Üº÷žŒb(óOa6ÐPGûM¬cKë -ª7¦ãkjÙU#ö’RWÕUÛ ]ëÃNl*t¶]ϸáeèÅÈ|íØ4‡=_ÚÒ8>{Ï·.Vá¢o.cÓÙæ’Ï]‘¸â¤Í¥CK¦Ëˆ‹¾P\v¦fÙÊLd_Iíz¶Ÿú¡°\WwÑ…|i›Žš›í‹.²ÜŠ. B`Û_ô“î+M+{£ž^_=î‹É¾Ì–Vj¯Ò÷^ZÆÒJx²ÔæK¿¾¬>ÇæƒêÊäm=T˲Ýܳ5ÈiS£…´ºè1ÚIc˶$p}¥Td«J‹£¨t™*²Zb(6ù%[é–Üâ(öR¨1E“þµïi×JíD׊ߤår‹Ç«Ñ’­ëáZëTKYcÉ(È|Ÿ»J©¥´E¶­›Öuo{îšAÍß"SÕ²¹Mʇø#].¥*ywDÇv®?5é`Í?Õ¨¢÷àâ_Û »í’»óìïŸlwIÊþI_†c"kUÐü£›² ïP¤ÛŠ®¨òƒœr¶¤Ä&Ý×ÂöP£¹íAóùW×òžeÞ$ª——Šü#‹–ƒÞÈ¿ ”–šçm¢¸Ú>Éÿ 뎧S -endstream -endobj -561 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 560 0 R -/Resources 4 0 R -/Annots [ 562 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -562 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 642.273057 101.433071 623.523057 ] -/BS << -/W 0 ->> -/Dest (cb103-1) ->> -endobj -563 0 obj -<< -/Filter /FlateDecode -/Length 4418 ->> -stream -xœí]Kܸ¾÷¯èóËå£ø‚=H‚Ù É!Èa¶×ÞE0àÍ!?ÕâCR©ÅÖÌØòÃv·ªXT‰üøU‘R7ÕYâï[…ÿ)BôŽÎ—w§÷')¼•ø7”˜ êñòö¬$yaUtÁžß%ñÂG«½;?žðßì°-þxúÇ7§ÿLmÔY‹VöüÛ›¤"#|ð^ZF¡’Ê0*k…÷Ú:Få”À%ÆÈ©à†õ¤i©òpƒb°œ -nÖQ¸aÈ+³TE¸¡½"N7T´žQ) ?”–’ÕÁId4£SV84ºãtZ ¢Ž¬ÎC§¼ætÆçIYÅé¢pÎÙÀé¾Ø(§³ðÅ*"Ééà ™à9ƒ/ÆÉêà‹Þp:_4Z'2º_€®Àêà‹tRqºh„ˆÕEèdða©ÓÒ -Œ‘œN)aMÃê0¢ÐIÎ3: _œ´‘ÕÁk¤æt¾þ8xÂ3ZuÝFÅé,|ÑÚ«ƒ/è|Ž´ƒ/Ò;Ž0‚Dâ%Ž´÷‚¢&Žt0‚Žtˆ‚¼7 èh¹è90RA§GFÂKŽc£à aä²:øb"q\`4|AŽ Œ/š ÇlK@ ÇtK2*Ž ø–¤r.Ü”0® Ž8.0 \@)r\`À¹ÜÉqéb¸{Ž X×X§9.0 ]CÁq\`À»†”一À»ÆÀ’ÕÁ˜ã°£0*Ž ¼k”ôxÃHs\€^À¡ã¸X:ÉqwuÄqwµG`u^` Ž ¼«-.‹Üà]m¥æ¸€À»š´ã¸@àÉfà]PˆåÓø‚å¸ñ:m8.@eB#-á¸À‚wUDøcÆ4ØJ¨Ç¼«ºÕE¡¨£×5Ðrظx‰É0i{~ýîôÝ_^ýý¯?ü½~{úçï$æRx]¾ÿV¥ÏoÇ÷p}ÇüS¦•:›ÊÐUfíõ=ÉKùlßèr¹b˜‘e?ÕsXÎæMÕ+;êÜÔ·\vð«ËO¯)Û~»Ùõ^fu«jõ²Þ×ëHzkf>éñz{A5½¦$“¦Ö¯ä÷ÿ:¿~uúÃë)ð†lô>¬ö™<«/QLmo@´À"7Õ®(2·ìš¬ËÝGž‡PÛÅog]ùÐÔ×@c€u†‡ä}Jèøwa| S{õóÓ!7…ׇhCÂiE$¥pÔvÃ6G1fU7‰ö^ÒqÓ–d_ YïAî*M¥Çt˧ìð6ûl‡FºÉ©wóã BJïŽöž{ÙOµŽ9ª‡rwÔ[PÈðm©£‘éÜiš‘£]r{áéÙ9W[“Ï´M¤Hº-ÄöÙ±·U¶{÷£ ï‰Â¹\ãÏäíGl;üêIBUM)ûØgs8~ü<ÔÎêY ãF¾ù1Pfº¯‹Ñ8+WâÇÌn‹uÍÓ Ö ˆ¬Ù@d§ÍÑ©oç”%*éYÏúÚ³*­Súñå¶{e’«]¦=5AÍtnæËZ^›£¥›½ÏG’­×@-ïçw3µO³¦”/*½‚š&SZD´ÜDMŸÍáP³‘û½iZ{Îi-ÏÌçž3nrÀ™ŽÚÙG–½Ö“ìJ¯ÇŠœ6§,6k¼:çNµœ³ÏsÖ¬ÿè#`¶´ÊsÍÂcïl¥ÓæpˆýºH9Y[u­¡rs¡q/ÚWÚ,7JZQ ÚBhŸÍáz;7ÜË¥7rË?Œ(¼ÔòIÆóÆ—82x$yÿ}N›£!Ñl䄇â'Dô6Wl|a‘ù0C=M™ëuå¼wølšw=–‹/Ë‘ÆÜÏ‘6‡Cæí¼sྜ›1sÉ­ÙLwï»±‡ÛW–'>zgEë=_f3_ -Ï?ÌP>×gt´£ªÉ—u»fég6ùz¹ÙÚ/®¡ÏÞÿ¥ÓæpèÛÈ!×Æt›™qÑ´ëæYÈÚìq…áþ¹w§ÍáPØ—'f®›¯†tõè=÷^VùB.ïi\šsÒ´Þ ×}²y(Njî7û ”!ï¶V¸;mŽ†NzBîøQyåGÎJž»¸W2[õ.#+Œ/{}MäžÉ)sç±gîƧÓæpØ»íæÇuÜÔ–?Õ6ÔºiÏœÑý<û›Ã¡r#kÌ w™öÞSbÚ-d.à«Ý=(èE@fÏÕ¼aΪrÖíýÈù½€vÞ6›ÛµºÁ§y íx²ÓæpˆÜ¸ ÝÜY^Ü‘n2£wµ÷INfÇ‹Þ7½£‡ÙÙ{›¹ÍFÍ<'Ð -K]³{„ ßñž?+2_{r³6É«óµ"uÕ¯ÇqÛ¬%öÎp:mŽ†O{;‡\ôÅʵó»sÜLckŨÔqáû{òî½XâpÉ<þ¢XkýÙ\¿à™Ó¶ÙbBE½u±ÓæpÈܸïý±üÈ+;ŸDÌc¶ýÊM/fûl‡ÙÛùçbô﹋²Á’ìË0Ó;&OñïÞ{ /¾J°Æ¤ÍSÑ -;Ü–ÙBeŸÍáP¹±Š¹÷» _×ÞŸðE4•NÞÏ•6GC¥ëü†ÍŽï(|^üµc„>ëšþ -Rw|û¦ÓæpHÝÈD¿¾Z‘Ï™|˜‘¸‚Ð&‡ì}V£ÓæpÝX÷ܳF¹‡w¾fÝϸ¹pÿL¾Óæpè|¡üóÍ žuÄ}€HïåýO®wÚ ©~#'ý$ùçóô~Ç·:m‡ÐýߺÅ;w?ßñ5ÒwGz¿ãWŠ:m‡ÎÛyèîyïgé÷Ü?§ï´9R7rÒO’ži„8Òyÿœ¾Óæh ¹hû,Ðìû -™'Úï« |Õ܉·c|ê÷Eõ°ãN|§Íḑs~Žó÷—] ‚‡ß2ï´9*7Öåsžå,ž­_‰êqǽöN›Ã¡±/×|–§ÕÓѸö¬ýÍßðÊOç%ß=[­g~M\¾ÈíW“ öêþÓ߈¥Õo®?é{ eøìðBSw=NM5@&-…ıÌð㤙¸®¤e‡îOÍyí9þØKzOu¥òôöÚ=I®SYw} uŸ‡zB†Èx¾aR›º%%\Wý¤~;ž»ñg¨Ã6פ‹®|rÃÞKjØOº ”Nôž “ú&žÓ~OAR¹ëÔÐØê¼b9lkü¾”w:–ióôñÕuäæ`,ÿ~èôw'I¤öO„qÝ•Tâ óA?žÒ®´Äˆ‰„î'áøQ–ºHå‚Evïzç%rüKåÆæÚæ«Iö6›¥Ý‚HæBfØ/çzbFK§–âtõR¨Qœvö ÂŒG®¸b]öÂZ!µek…ÕpS¬Œl„v)”ÍéªXSöG¥àJÓ¹*,®·M§šîÏeµoºó–ðrÂiÍ\úÈJÓ~¢¹ªÜºZ™iœŠEjsÉÒ!Tú‰]NŒð[™«&tfÑ7½–ve» -C•r1—cD—[n¬Í–žò¥T¨¦1·|u¥BÝ„ÒðŒlh£QêÓVåÍ‘&œŠ]êÜ×24¸Ñ áå„C ñãÉz¦´­\!m†^+4M½EdÅK•¦Ÿ¸~ÂÐ̲´§UúÿJ¶Hó H¥}éêQ=I›E…+[¡ÌNàtUl ”'âê§-C`Ï'sª»-õV±+ÜùØŠ}éq•6,þ–zÌgáel4'è‡|’*Ö²D»z²FX)Tq¥ÑÇAœGka§á×åÇuq÷rúõ4„9m¬ŽøàŒJ{涙—:ÿe^áõïÈ›´]㸖…-táÖ‰¸di‡õÐ)N;®ga6PSCûU¬B-k3J׺ákªæ²{.© -ƒª¢m…¶¶a#ÖºZ™¦eì áeÖŠ%‘yjßÔ€}=4ùâp“¶÷Ë^á ½†ª35$Ÿ›*q”ŠÖ-éÆm¥8lN«g7 Û“”¦OþSÛ&«9hÒˆthªŽj˜m«Î²á*š"HM{ÐVLª=iœøÃ8½{x=¹ßG—]IZ¨½H[i¹°$-„ÇKÍp覇%æ˜s:ȃ·¶P©Ë4cÏ”$§šZÐä Åh# Õl,`Û“R–MNšE¡ Èd–•}öÉf¹Y†+l+ù’STé¯mKÛ¶WJ#Ú¶WÜ¢lª7G¼’-™’Ùξ^¬%K.™k­‹”jI“eËsÓôÜ9Úž›Ë êˆ«™©¬fvQrÿí;W›ÙJW*ÿ «#*ÔmM¨Iy£Í9¦t'8¡´Ù¸§Ð,—Üms{ýd¹J’×OÚ3¡;#½ÞV\x„"n+šªÇay³(ÙKº¯˜å¡Jsr±:d„—5‡E¢rèÓTÑÆakp0È£òR±ËEÃjátùdøû?ÿ,„ -endstream -endobj -564 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 563 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -565 0 obj -<< -/Filter /FlateDecode -/Length 4795 ->> -stream -xœÝ]M$¹q½×¯¨³€Mñûö`À¼° /0°‚=5ÓŒ^+ü÷ý‚™$#™Q™Y½½ã’wÐ[•ü’/‚ÌL–¾*üûNãÉé)¥ƒ»Þ~ºü|QSô -ÿ•«‹¢þÑ_µrqò:‡ä¯?‘ÀÆ)fob¸¾]ð¿á’'»üñ7—¿­óè«™rFûëß¿’ÊÙ)¦•T*¥­ ò~ŠÑø ¨‚žÐÄœ%ÌðÑ·UE˜árò’ -f8c’ J0ú¨íV•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚO$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ƒÞÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓVg”Ÿ|²VI:­'lZQ‡…A -QÐؔϢ¶x«Œ¤³°Å៤OxSf«¡Ùm•Õ’ÎÃc¢u°ƒ/1€ °EÅ QfÐD¼$q€‰qrÙ8‰L²“K.I,`Rž\ŒV¢“ýäBŽX¥¡3Z"«`‹wAb«a‹ÃÌu°Åf'q5° $.°¶g%.°`[´H\`A·Ne-qß:¥ƒÄ„ 3•ÄŒkSpXP. ”%.°à\ À‚t1ݣĬk}0XЮu)H\`Á»Öi%qïZ‹œ¢¶À€'«“•¸Àw­VQâÞÅ42`T'à0H\¬L&%%qœÄ¼k"\€¨‹æ •¸ÀwG$Ï Þ5^‰ x×8$.P&ð£ð.(ÄËálÁJ\± °É ,‘¸Àƒwu†ûæ4ØjÒ)‰ ƒ™$x‘i¸Xo"²•Öö -Äë#â3Y™hÅ™Äêõð­â–÷Â[EqÏ@|i$nÀL¬,Sw àV¡ÔJÜ6Pà_Äcnà óîߢ -€C„ÏL‘™1-¢û¶æèÉìdóþØßÿrùí œïõ/ÿ}yýMÙ)«ÿ4„’Ú!¾Â40Ãí}k4߇Å2öHZª4æn” .›ldÜýwÝ~YY -CÉTäÆ:(ÐÚßZ -»æzÁ<¥t‡E0Ö†|d7õá<‡&GDùÚ¾hÙÖÞE¼ÞÞG’­çùp[±üÆÕ#À\×+y”øcŸÕKëáM½žÌ€;ÆžÌôÁFÛ6Q2 BP„F»ä@µi‚–?3ùÁ,h°Âòñ`@› ùO"V>Ê­ÿÓ§ ¨t±bÞ¥o—Ì|Dt®Ñø맟.¿ý·þóßÿð#øéõò§ß)åý÷ßi|:7z– ‹ì3þÒ"_ÒÑÞrËóµëµ_t5ïÏç;eÅ%ýë¢ L·WÖmÑ i´^Ûåo´ŒYtjþói]WË«ÞY§êý×t˧7Ëg-O÷6kõýŸ¯Ÿ~¸üó§5àÊ:©NcI¯°¢÷ê£ç²üŒþÐÌyš­ ýº»]†ˆui¢:¬ŒêiÛÐ&V^権)à{ÅšÒ.™WÓ"Ôh“J[~ø—ýý°¶`1R ©}‡i‘x®֖µ½ 6žì›ÓÂkbjÖvÎ×ù’¶BÞݶeS¶u”i`t}¯£ÔGé^X>5çkå×´·^W±ÕËm®õ{mƒSx§S?+P–qÌ6V~±sé¯Òv²á6OA¬5æj/‹„U–îD-åÛ¥|é7Â×\žþ²-'XZ¦Ñ­€m9¥íµ?Cé›=;¾Þ±Cõ¶P·öQß›yn¶Ñ-¡ b¡ì½Ì5¾:¶¨Ÿ-}Þêø.6ùJ÷ó÷ÚV²ÇÅž¶´Á`±=/ŸAnGÁTMóÊ0n–úlÒuÑSºê–ü¢×KùŸ»}Úìö½û§”õu™+µ>7ωy,˜0ëv—y¥˜µÍOqþ,å}]ðÅædÉ×æ•Ü×Ä+¾â@Íý ìºÇql6…>7Ú¸én?øq¯ŸâØO­qÀÇ<Õ¶–9Eö¾ óijv¥µ­ZK|×ëtË|Ýr\ç7âðÏN-¼Øº3׬çJo£[·ë~[B_ºµc’6‡~ýdžw-@>Уûf¯wSB`o”èÒ ´ýÚUŠ*S…¹§yîV˜bÚ¶*œé‹EeÕµçE¾_®±yòQ(׿~Û¥r¯+„{VäÇ">{Œ”¿8Ü!ò'Ê,‘8ìNOÍd³|êà±m¡6rÄú'ÚïlÙÓ¥ý­wô>ÝÙDç…x·¥ùiôÀ-Ö|\ï»d0ã’ƒ›ÝÇ©Ïl4vëCS™§²’ÀEA FɹûÏk¼ -Šì½@gHг™îÊ •–Ù~‡‰ÉR£got´ Ü>žÌó$+œÆ‡_wøP½Ë™?ÕÙÕä ,¼—95m[☵¼×÷ýy4¹öÐÔ»‹-nŸ£i Ï -Š’J‹%lö)ê¡Â=$y -¬gTxNnçœËólHòtWë>’÷šá3® kÐú2ßÈ—¿&wC:º.AÎ’çs×ÃÒÙåïËòé–ïT>g»oå{~Ç‚ÃZ渡p[÷Ù½ýO=."ZB£è6È>êÎåyÔu³é™ßÔÙ5Ú*;e´$)=ÏwÑ^×#XÂoÃF/®ËXí<~Ðnåy$<ØÖ¶ «[2O?ŽÈ^F³ãe©¾s%d¸l‘z.ÏÓ!5ìó£;j*òæM„-‚6#-¡ëÿhÿ\BßC?,-—õÍããH=—çéš÷9U ˆúÌâ¼—/ÃÏ}ø(ݱc@{C:óâu¦ûÜ·µ¯ÍžZΗ¡¿\LÖ6ɨ =V4&LÊD{_žÌól¨ ûñå^œ6zrÑkŽy>»w£gu-®?7+å°þ¼;²æq¾9™çéFö †«=yc^rèMš;¦n©Ž·ßêèéíz”òÐ-õÊvÞ›ÿ3[Õ„5ÒxÞÆ i?Öj¨øÅ¿ÜAnå”–ósEèá-Çj£ŒÓ9zª*ÑËG8=—çépºÁm|‡fc¶ÄÛuì7wÝCÇÀè{ÆHÐå-Fƨì®?V©g|¿=>YÀçŸcs§}Ú¡}_樯ÞÒÚ ‰ELg9™çéдeíù³#(¥ÕæŽ ?øHo7Ël•ËQðº.cd½Íz@ŠákYãNáëýuÈQ´ÆgLH@ˆ¼ÆµÖmm[¹ósÇ?G¾Kw©'ó<RãÁÎÞÀi#Úʦ›ÞOC£Mû´†þ–Q1AiáÞÜGDaó­Õ¡ÒçÆŽ{«`nˈ*Ïêz×jþyÔGo·¤òFÂ>Ïåy:$DŠÏü¬àȉuÖ »v›6Ô:oç7Þ•Ù vD—øÜß–—ÏîŽÊ¨¾ƒÜwìþÌótÈÝŸyoÕ¿U¬-#’7æq¨[÷ÓˆFÉ´u×p‡f³'sÆû÷˜ÓÅ4iëè©ŠäžËótÈ=Ø ´:øJ“yiîi¢ùŽí0J£'ïjìÄp%º°Ûúl`1ñæÖ¡<¶ ±*kç.Ë.‚GÔ³Ý÷Ä£I=Î¥'ó<"ÓA<:ì‹´ÞãÞëÞ -…qïõ•ç{ù¿#sðØúêç¯T{j3e• ½Ý·Ôsyž©ûñê¤ù…ñ çµ{ü2Ü™v;øý1Ýs£ü~Ôr•lO2ûÉ—Åù*Ïåy:TÄ¢G(zt4>*î;ƒê;«¢ ”ßAf|_žËótÈ<ˆ5?rårå­n>ÑÃ;‚σ`¥¹Ç’§WD'ó<Jó¹'õµ÷¯àƒ7SÏ ÷‘6~ôý£l_—ŸÌót(Ü-Ë“$õ9ZiW›?ï–tÒˆ køÃ53Û…¹·¿YG¸î¼oê}çó‡‘@ÒûÒãý ÖŽ;hc{‘g=óÉS'oï¡V–es϶ §OZ¸ÕÇ(“¦žmIày¥®ÊV•VGÑè2Ue­ÄXm -K¶Ú-¥ÅIì¥ØbŠ.ý+ïiÏG¥u¢ç£6i©ÜêñZ´d[ä9œcoê)[,™Y๛Ôõ”¶Ê¶u»uÝÕÛ^Y3\7$ôÈTõl~“ò]ü÷ž¹3çv"¤Tõ_ÙÑ©$Gít´Æ–߆S)pˆ£Ÿ÷ˆ.±í’‡óìïŸlwIêþ ¯ Ã19kUÔô+² oPäc+ªþ œ-+±Iu€°=ÔiNmv‡ÊÏ<•½Ë²IÔ.éC¿êf)èMôc4Qiù,XÚ&zYmŸ”ÿ çÖë¸ -endstream -endobj -566 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 565 0 R -/Resources 4 0 R -/Annots [ 567 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -567 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 588.873057 101.433071 570.123057 ] -/BS << -/W 0 ->> -/Dest (cb106-1) ->> -endobj -568 0 obj -<< -/Filter /FlateDecode -/Length 4402 ->> -stream -xœí]Kܸ¾÷¯èóËå£ø‚=H‚Ù Œää0î]ï"ðæ¿Ÿjñ!©Ôbk<¶Æk7Ú#U±¨ùñc‘”šê,ñùVá¿@J„½£óåýéÃI -o%þ )&'ƒú5¾Þž•$/¬Š.Øóû$0^øhµwçÇþ›¶ÉOÿøæôŸ©:k£ÓÊžý)©È¼—–QE¨¤2ŒÊZὶŽQ9%p‹1r*¸a=iZª<Ü ,§‚¤u`TnòÊ,Unh¯ˆSÁ ­gTJÂ¥¥dupDÍè”…î8VÂ…¨#«óÐ)¯91ÂyRVqº(œs6p:‚/6JÅé,|±ŠHr:øB&xNçà‹qF²:ø¢ƒ7œÎÃÒ‰Œ.À +°:ø"Tœ.ac bu:|Xê´´Âc$§SJX`Ó°:´(T’óŒNÃ'mduðÅ©9/„§OXãC`Z«N­ÛH£8…/Z{buð•Ï1€vðEzÇQZH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHøbÉqL`|!´\V_L$Ž Œ†/HÀq1ðE“á¸À€m há¸À€nIFÅqß’TŽãÂ…›’ãÆ5ÁÇ” (EŽ 8×€;9.0 ]4wÏqëë4Ç´k(8Ž x×’x×X²:øs\vFÃqw’žãï¢iŽ P«8t+B‡ 9. ð®’8. ð®öèXhƒ†ãïj‹[àznð®¶Rs\@à]MÚq\ ðd£ð.(Äòá|Ár\€þ:m8.@fB#,á¸À‚wUD÷Ç´i°•P!:Ž ,xWT«‹By”8ÓnQÊB99.°à]tT†ã Þ}Ž Àñ=ªæ¸À‚w•AOÍéÀ»JGÉqï"n°B!à¸ý›PÐeuQÈǼ ZÒ 2€IX’â¸d,$âPŽ xÝ{ä¸Àw¥5†ãÞ•„°‡ÕÁ4Ljk C³eÚ4‚U!5˜‚Ów¥B s:ð®”>r\àÀ»©Ûä¸Àù$Ðã8(kŽ \ ½w¸(¡CÈÇêœÀ¹å¸ÀK,(‹ÕÁòÄqkM x _ŒÖx _2p\€ˆ@D…è›ÀÀ…|A¤Èq˜L„HÄq·ã1”*§sZºfu:å9.ðžºbÅqÆR" Ñs\‚)\â¸n@§ˆãá š -ÇèŠDЈÌ9‚/²9.@¤ R _ r\tÀ(ÓIŽ ÐCc,‰¢æt$Ó83p\ȉ*r\€f‚‘&˜’ÕÁÐǨŸ4ض`ðƒÁ&†9¬¾K„ Óp3p\|ÑRs\l¦ñ¦ç¸ Jø‚ˆã‚(Ha2Ç qè¤ä¸ ª€ñ¦&Ž Ð.Óx3p\ÄxC<¦ÝFãÒxÓs\ ¾X„(¬¾ hŠL»'a¼ vât¾èHDçÒx3p\€0ãMDôì”@HãMÏN€wm3q:ð."SÇqN0ÞDdËí%ˆ×zÄg¼2¤g`g $¨×¢oe§ $¸½•gç $È7‘ØI¸‰‘ep쬺U(•d§ $øñ-8á:ûÔNQ9ÀÁ£Ï >ÝdD³ D÷eÌQ“™•d×ù±_>}÷€Î÷üóOï¾fÊòG!@Râ+4QÝÑ·Â퇊VbÒà!Üc“œð±:©@‹ã¤[rÝ_úw^L<‡ã¯‡Ë›rùˆ^|¸<Ü$*+å0|æï4ùˆK ŽÐÛ:¤1Q'väWJš$þý›€:zq-§ûè/1&mÏoÞŸ¾ûË«¿ÿõ‡×€Ñ›w§þNJk¿ÿVá/Ñõ¯Ȳ·ø†Q>¦K3wÅ槪WvÔeÛ™+yù1ý»Q窮ø1êè2Ë[U{þØQGvšvÐç{4ÓüîòYMïÛ^RZ¯¡ä÷ÿ:¿yuúÛ)ˆ†Èò -{ ZQ ÚÀ]ŸÉSp÷pÃ0õÜ̬ˆõXô±©ÞYu|Ë1ñÉU2­ŽõkÚ·×ê q/ÔÖ Ó¦áîiLŸîe€s?m³XøÁ5¿î&ËC2ͶgbÃàÔÛÀd§ÍÑ@‰O Ðr ‡)ºR-w!»AÆ¡Ÿ¤•ÌˆrÑïªKÃPÄú)‚ºÄ>›Ã!‘nòãq8ã o;ºpUÁ~”öÙ¥Ac.Aj¯0­á¶v²®ôàžGQ[ËïfµùÐä× c@vFˆä}J-©øwa| S{õãÓQ·àÛg/Ã×°’Ò”ð¨íŠûl‡âÛ±èî˜Ùo¯÷ w ºÆ“6=ćÓ|ÍM4vÚ úv z7?ÎÑxO¬÷)ø;ËÞÖ<æ¨ÒÝ‘oA!÷%¦E¦k§VhFŽ6vÉí…§g×\El;Óò0‘"é¶Ûgs8ÄÞŽUuØ®ÝÏ‚¼¦Îéö o?š`ÛáWOª› ÉnöÙŸ?µ³|ȸo~”™°îë¢5ÎÒ•þcf·É‰5>L<EdÍ"ûl‡ÈÎùM=«Y_kV¥•C?~Ýv­LbµË´¦¸éí¢s3_ÖâÚÙ4¸¡`î‡Q]x?ÿ5Sû4jJñâÚü£iâ8¥E”AË-ÔtÚ 5f#ö»±p1á™ùØsÆMC 8Óµ3æåï¦ù$»Rë±"§)‹Í¯r %³1ûRkêìÁÛgE&H|hP}ãY“Åï*æÖ²±Êw¬½wÚ¥}¿B´x³|åW:tæŒ;Çúc>Ôú"Ä÷]ÿˆfaÛÎrÈéû8åãås‡+mÞêFhŸÍázçœgÃimO¸à­5$Þ/wðÒKŽ1çQÎâÙúÚ«¿÷ÍðgmÚ·(Xšm8 -7“l n¹iGÞúâœ>sH¹KƒHæšpF€Hѧ=f„^ýñÏú[¡ô3dú§ëþ]_²Â±Ã°1îzžŠlx-KqL3ü|Y®¦k%ó„ak¯U"Ç×ÁÓß”WJOï®°IrÒºëwÈk<ò ™Çë Ý^‚Nª’«~’¿¯Ýø3äa›{ÈÚÅEU~8¹a¿$5ì™']J§MuÏŽ„IuÏi¦ ©\uj(lu^±¶"þPR»Ë´aaÚ¬êºuqs2¦ÿ0Túû“Š$Rù'B¸î$*qÐù¤ŠOi'YbÄ„1u§8 ÇCYò"•Ù¼êEœ§4VÈñ0–Ì͇Úæ»Iö6›¥~HæDfØãæzaFK§–ât÷R¨Qœvã ÂŒg®¸b]öš!µik†ÕpS¬Œl„v)”ÍåªXSöG¥ÝÛJѹ*,®·E§šêÏiµoªó–ðrÂeÍ\úÈJÓ 9«\ºZ™iœŠEjsÊR!Tê‰]NŒð[™³&TfÑ7µ–vR» -C•t1§cD—›nÌÍ–šò%U¨¦1—|u¥BÝ„RðŒl(£QêÓöâM‘&NÅ.Õ@®kÜè…ðrBC ñãÉz&µ­\!m†^+4M¾EdÅK•¦¯GhšY–ö¡Jÿ_©Âibžá@QTÚK®žÕ‹¤¶YÄÓBjå®°q% È×´Mq)ï fÛòRA2éé–Ô¥Ä&ҦȹٽDR…<¦bUPWĉÂrù®l…2;ËU±-P~œˆ«Ÿ¶4U€=_Ì©î¶ä[Å®pçc+ö¥ÆUÚ°ø[òm0Ÿ…—¡c£¹8A?ä‹T±–¥·«k„Õ1BW}ŵv~A~<¬Œëq/§_NC7§ÕΨ´Ïmy©óÿæ¾ÿ>Ñù‡¼±Úµײ°….Ü:·=YÚ=tŠÓ.éYXšM#ÔÔÐ~«PÓÚŒ†ÒkÝð5UsY‰=§T…AUѶB[Ë°ë]­LS2ö†ð2+ÅÈ<µnj‡}=5ùæp“¶äË^᤽œ†ª3µK>7Yâ,%­]:´¤Cœ´™â´¹`¬žMÜL€l/RŠ>ùOmU˜t­æ¤ #Ò©©:ªÝl›u– wÑ$AlÚ“6cRíEãÄÆéÝÍëÉõ>ºìr HÒBíEúØJË%i!<^j†S7=-}Ž9§“Üxk •¼LÓöL rjÓ¨¡MNZŒ6ÒPÍƶ½(eÙ䢹£(t™Ì²’£Ï>¹Ñ,ËpÇ-%_bŠ*ý¥-iÛÖJ)DÛÖŠ[¤Mùæ¯DK¦@¶…³¯w'kÊKFæZë"¥šÒdÙòÚ4½vîmÏÍmPuÄÕÈTV3»H¹‹ÿö´«Íl&‚K•?Ãìˆ -u+RtÔ¤¼ÑæS¸œ‡PÚ ÜSh¦K=²œ%Éó'í•P‚Œ‘^o+.>ÿß õT -endstream -endobj -569 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 568 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -570 0 obj -<< -/Filter /FlateDecode -/Length 4837 ->> -stream -xœå]K䶾÷¯ès+|?€À‡I# b`‘Œv{wbãvùûùŠÉURkƳ‹6²ƒÙnUñQ$¿úX¤(¾*ü|¥ñ_rzJ)Çà®·.?^Խ¿’buQÔßâ7ú«V.N^çüõØ8ÅìM ×ç þ.yòçËßuù÷:¾š)ç`´¿þô‰TÎN1Ũ¼ ÊP)m•÷SŒÆAô„&æ,©`†Î¸­* —“—T0ÓU‚ÖEm·ª 3LÔNRÁ }TZÁm”u0D9g Ó~ -èô 錞BÊ&‹ºŽFÒY;…è´×’.O!Ÿ$ƒ->+-é'çD]†N¥˜¶:£ü䓵JÒi=y`ÓŠ:x)DAg`KP>‹:Øâ­2’·Ižð6¦$x«!ï¶ÊjIça‹1щ:Ø‚Á—ÀØ¢b(4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ๢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.Ü=J\`ÁºÖ#qíZ—‚ļkV8ð®µÈ)ê` ,qØq²:Y‰ x×j%.pà]¸‘‘¸£:‡Aâ`e2))‰ x×$å$.pà]1ˆº8Á­Ä¼k<š ÍÜà]ã•‘¸Àw3Aâeÿ(1ï‚B¼À¬Ä˜ï¡3Vâ6„%xð®Î˜þŸ[M:å qïê„áuyÒ=.ø-zyÒ!d‰ 04gv“_œç®ÉB¾¶/Z¶µw¯·÷•dë‹ó¼¹­´ÌÀçº^w¿íà³z1Ïoëu`Wà;ÆžÌôÆFÛ^? BPÚæD¬LŒ–&h{a–74Xa9ø0 ͆æS$VBÊ­ÿöÝÔºX1ïÚ·Kf>"<©Òøë».¿þÓ7ûËŸ¿ß=]¾ûR.|ý•Æ§7Ë'ÌÇå»_ëÜÓð™Öé0˜Ê}Âõ"w~NªËaâw·Eÿž– KYjþõvÈSmZl4”mÖˆ´Çoš?wËÛÄë¯íº1ûÞÏu”tn·uû6uuµÏXÓ}ýë»o.¿{·\Y7À9,s:GáÁ!FÏeù9ýÐÌù 4k·kµ3\‰Aÿy6T šf;¼Žéjº–O³ú?Pçy>u} ÙB£A!ï”eh½¸,%ôkËè>kÈÊp¤ûH•iY‹µEd‡x<™çщµü"Ç‘h½:ŒBCÃk L"Â#ÜÖöðúi…Áf3Ø®91m ³‘"÷®(ÔǼÁß>§×î Õ6*ôÊM… ï±çÉ<‡VwÈŸ¿XNÛC³Þ¢kã%i]×Ê¿¨í ³“Úa´3–¾ê:Ïåy8tž <Ç qàÞ«m”v1â’MðšXyuLÚ²œÚ‚Õœbmi—lÍj XŘTÚòÍïÿø‡¿²¶þÖä)©í»­3-ìkmZÛÝç²sN‹µ1¼´¶w¾ÞÈ—´Û9‚úw[îÌë|.êå–:üð¶|jÎ×ì­io½®bSÕ›u:n·¿¤+cqªó-ïüï†>U`2ã˜Í‘Ù¡zß•>!n³gê°TVuÅ8…lé,ÊR—]úΗþ$ÌÍåéÛr‚¥Z: °-§´½ös(}s`رCõ¶Ð8´öј˜Ù_ÛèPHP΄,”=c³—¹ÆZÇõ³¥Ï[ßÅ&_gù{m+ÙãbO[Ú¿`³Øž—Ï ·£`­¦ybxoñÂ\ß',éêlå½^ÊÿÐí«±ÃNßÛ±JYŸªõ¹ÙWfÿX0aÖí.þ¦˜µÍOqþ,å}ZðÅ|µäkþ&÷5q¯8Ps?(»îÃq›M¡ûF7Ýígõ“û©µ³òDêxâý_êaxö¦·I²·óç6‰×Öœæ9¯ÕþIB› ûÔŠ¯F ´•/ËgõÃÌƶ¶mž»Žú0n|ñCƒâËÔ¶lì©þ«{ÁR±Ó3ž³Û>ÑúÈ:·²¶eæÕ5/Tn#gýlÙØ£ƒ,Xî™»‘ÍÉ<¯Ú^}ÃÆ7{p'ïá®bHSÜx\.éJ lÊŸa·ÂäÑ!…¡Â™ªY`ZC›¼ÈË¥yÔ\D¡\ÿôõÜ.•“=!ìYu'¿Sö;) :†@П(³šu>؃ž:Ú‘äá}¡6r/Xîœh?ÝJH:ÑݼWô>ãBç…¸ÛÒ;ùiôÀÖ¼]ï»dàqÉ!¤8Æ©Ïl4ëCs™§èìˆ1§@ÆX«ÐîuÛ ‰NFf:ƒ4TZ¼}‡½Ô)V»w‹á\žYâ5>ô„é>¤›úqù ƒ‡aco«iåy\ÿA­6<mÿnÍŒ€z­Þïo³½ÅŸz¿íó”?䫺q+ng“£-•ÏûÛ;ð¥æ'±\%Àã¾´âŇ’4Ú_Ë>E=TxèG9VŸð*œÛÈ;™çÑü(h}äG÷î#5Œ×åÉûùûFî‡{cnHç_ý¸äùÐuƬ}©ü~\>ÝòÝÏ>ÿÅgÞßtÿmeŽÛI·uŸímÐ@¡"èì â“yuÝlQ7°Úf#¾2ã 6ùÛèÅuŸãÜy$¼°­mÁ½ÈÔ:ÏëÙËhv¼?@ªë\g .›»H=—çáŽùÑÈa±À/ù¦°„~å4²´\ÖCj4/Gê¹<‡ÔtÌ©j@‹VçÍDÆ—aµ¾Ý(íØ1 ½!ÍâÕÓøMë/e_óžZÎÇ¡¿êMôM;¨ì±¢1aR&Ú»ñåÉ<†Êx_žÙÏz`ûQŽÛ¹Í·ºÁן{ëÆ;#ÍËùædž‡Ù;1\íÉ›%‡Þ$ß¡:«­ãèi6J "(Ý ROlÏåµù?°UMX#çm¼°sân¬cäÿ~¹üpËN Ëù¹"ôî çj£ŒÓ9:AŸèÁ×{8=—çápzÁmæ¾²ÄÛuì7Ç.BÇÀ8÷Œ‘àkNqî­RÏÌu|gf[1Òˆ’QZmŽ£pùaŽäñvc±ÌV¹Oë2FÖÏVÑÒpг­@Ÿö×!÷¢5î1ã~æ*ò×Z·µmó­Ë¤ò]º³H=—çÑšîìì œ6¢­lºéã44Ú´Okèw„‘îL¾EVbo;#TúÜر· -涌¨ò¬®7<,šxÔwòhóÉ<‡Ä;‘â#9±zÍÎ}—m‡òòÆ{2ÄŽèÚ9:ý¶.ì=#òòÝ¿“y¹Ç±ã#ïà¡úg£êôcCêÖý4¢Qš Úºk¸C³Ù“91û§sº˜&m)¹ƒÜsy¹wv퀾Òd³4ŸiïEóÛa”Æ™|¼«qÕèÂnë³ÅÄ{÷ȇ?Ê:¸Ërˆàõlwã5ñhʯàÒsy ‘ùN<ºó”äj_do…Â8êè Ï_ô™;-¬îq~¤f{j3e• ½Éá©'ó<RãÕ3H+3äGƃœ×öøe¸3ífàÿ¿‡QwPÉö$Ï>°|2ÏáòN,zE_ôqÞ¢zgUôË@ù2ã+øò\ž‡CæXó-W.'Pþ¢ÕÍ[#zxHôq¼ƒÒKž^Ìóh(ÕêÜQÇϾøþ¾ólò辤opéÛå]TÑcļ§w&¯{µUÃÞÝ$w V"o_ÎsÕà ‘Ö§•Q$½„Þ¦«¢r¤÷¶íU6Ÿæ{Ñzy˜‹öÁË}æ0_S7¶‡uó’†=¬UlÈ-þÕ‡ÐÊyçúÚÓÌ,$7í¡NÕŸ+Š=@«ÚCd¦<`«æ‡Æ–çVå<¸ØÚàk6Cùã%”wêòZÀ †^T n²46ùJï=LÊ:]‡N—ÎÖלåõþ?¶TËÿ½˜^Y†–_,é,ƒþÃEg7Qÿ—§zËÛ¹U'S/ºøùBogw‚عþhð1 —¯ª•åt{Ò¸Ên Ô8å1¥õõíiìqd:¬=5¾¶†òûšÞšçTMdË{ãæŠ!r½SëÕ¤1½á>Mv¹ -Ízª {Ž§íöŒwÅÚ*&ô[¡bÕu±qÕMoDm]º°™Î»N³á¯iMdÃy$¼]P­¥Ï¢”Þ«]‹ª½kT“YfTnR_S¶qmœÑí"¿Rµh‡Álz6jôvÒY˜*\K—k:At»ˆé–Ò|©ØR¥ž5מï¦t¨ÛÔ:^ÝØ#û‘þdóeë×µ˜ž>oc­ÃÙoL1n#~¾ø(¤ö+”¯ÐãBËÊmâ¤:^º”¢¨ù\³ÊèÝŽôÿL¾I‰yÊí¤èý¬ýªWB¾ÙÄëNâòÐXÇI= ]ìiYwéfy餄ôDº-u뱕”u“Ç\Í#’jä±놺&& -«ýÓ¸’ U5Õu±oP~^‰»¾¹ªW­² Ü}+·‹CãÎg.ŽmÄ5½l·ÙÛÊe˜¯Â[™ØÜ(&è§ZIÕf»^vÃ@ -]Üiô¹ˆ«·6v¢? R¿v'“fÜÛåûK™æŒõ†ž–VÓ»ãy䥯ÿEšoðû¯‹»þ¹¾¬tžÇjla·®Ä|&£¿4’NŠé/Tas&4ŽÑ~ëÔÓúŠ†6ÃøÚõìª{M©ƒê¦åBßû‰Mƒ®Ñ–õŒ?Þ†^lÌÏ›>aÏ—¶6Žžû¦×ÜV«pÁ›ƒËÔu¶OÉWV$®(iŸÒ¡u†eÄ/—¬ÂÜ-[™I€ä•´®'û -Ku± FÐ¥í:קY^t••V°$-¿à;Í+Í+{£_í^?{Ü“Cõ’6joÒg.m #i#> -endobj -572 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 517.623057 101.433071 498.873057 ] -/BS << -/W 0 ->> -/Dest (cb109-1) ->> -endobj -573 0 obj -<< -/Filter /FlateDecode -/Length 4380 ->> -stream -xœí]Kܸ¾÷¯èóæòQ|Á$AŒlFrr÷ڻƼ9äïç£$RUj±{<¶l¯ãiU±¨ùñcñ¡¦:Kü¼Pø/!Dïè|yúp’Â[‰CŠÅÅ ~…_oÏJ’VEìù}/|´Ú»óã ÿ5—uòÇÓ?¾;ýgi£ÎZÄè´²ç_ß&áƒ÷Ò2ª•T†QY+¼×Ö1*§1FN7¬'Mk•‡ƒåTpƒ´Œ*À C^™µ* íq*¸¡¢õŒJIø¡´”¬ŽH"£²Â¡Ð§ÓJ¸udu:å5§3F8OÊ*N…sÎNGðÅF©8…/VIN_ÈÏé|1ÎHV_tð†Óyø¢Q:‘ÑøtV_¤“ŠÓE#l D¬.B'ƒk–VØ`ŒätJ lV‡…JržÑiø⤬¾X#5§3ð…ðÃéÀÖø˜ÖªSë6Ò(Ngá‹ÖžX|Aås  |‘Þq€$/q ½5q$ ƒ(p, Cä½áh@G+ÈEÏñ€‘ -:­8"0¾Xr_-—ÕÁ‰ã£á p\` |Ñd8.0`[Z8.0 [’Qq\`À·$•ã¸À€pá¦ä¸À€qMpÄqåJ‘ãÎ5àNŽ HÍÝs\`ÀºÆ:Íqí -ŽãÞ5¤$ÇÞ5–¬¾À€…QÁp\@à]£¤ç¸€À»hFšãÔªÇÀŠÐ!HŽ ¼«ƒ$Ž ¼«=ºVçڠḀÀ»Ú⸞¼«­ÔxW“v(ü#Ùh¼ -±|8_@° ¿‡NŽ ™ÐK8.°à]Ñý1ml%TˆŽã ÞUÕÇê¢P%δ[”²PÎEŽ ,x•á¸À‚wAŸãp¼@ª9.°à]eÐSs:ð®ÒQr\`Á»ˆ,Ç€ƒP8.@ÿ&”tY]2†Àqï‚–4Çè…… `V瀤8. ‰8”ãÞE÷9.pà]iá¸Àw%!ìauðÅÍqâèÐl™6`UH ¦àtà]©PÃœ¼+¥8ðnê69.p> tภ-DÁšãHDïÇ.Jèò±:'pm9.ð Êbuð…gjÒ–çøá_ç×/Ox½ÐUŽGÇ%FUfs}&OÁÜs@ CÔ+P+ÅÖT½YW{Ó”Æ^&à?«¼¦ôI^ªû²„Ã`c›ªSUu¾YÃ`aó¶ªb;é¶à7òÊÏ:ù˜íoÎK®a¨ªgia¼l.<<Ó¬{[šHÇC¦>ó*>;mŽPŒ|®"ô©dÒ"0§UÏO>_gëØ@¬)ôHiX‡˜¶{ˆí³9bé:§ÞŠ¼+ô¦áá[j´›>a‹«Pú´¾d‘UˆA³Ä ¡ç:"ûl‡Èëåq¹äÀ­¥AgW¯]…ˆ’„Vˆö×gs8Ä]+ËÐ!—²fzϦÆT>Œ<@´¨ÉµÒòÖÖ=í›±Ÿ#*Prã™>KÀ£RÏq!E+ì±TvÚ •úz,yXN»2 -ëBö ½û³´’†+·øQÏñžMK! ÓôבØgs8$îĈ‡áŒ¯¼UÜÑ‹ë9ìGiŸÍáPÚ7æ¤*ö -Ë®k§ŒrîyÕµ\Pñ®©Í‡*¿ -²3B$ïSjIÅ¿ ãkXÚ§IÕ§¢nÅ·Ï^†(·Ç;m‡â9Î{c>ªdß _߃Ü-4š;æ3;mŽ†F³3Ÿy+?Þ=³ü‰ø;ËÞÌy´¨ÒÝoA!÷%ªE¦{§Vh&Ž6vÍí…§;ˆL57é"êY‘Ü[•ì´9b¯Çª:ì×îgAÞÓ çt•?÷ ï~4Á¶Ã¯ž8ÔTs‘Ýì³9?j›|Vȸo~”™°íëª5Þ¼ø¿È9>LÛèƒ"²{«æ6‡Cdçüæµ­i7šŸ~Ý~­,bµv…‘™á.:×ø²×63á¶AA뇞P]x?ÿ5Kûa+ŠÞž¤*ŽëÝkÑis4ÔÐýkÙ žiÇž 7 1`£«gÌË=Þ-óQÕF eu~Í(©åÕ/õZI3focÖ¬ÿì- sž’ª9ÇÞÑJ§ÍáûÛ<å¢eíåµ…ÊݹÆ{ѾÐj¾±w¥±Óæp½ÞË¥6rÉ?L(lV#yÞø[ÆÃík:6‡CâNLx(ÞxB^ÇŠ•/,2Ô§­Éy›ò÷®¶0ë)]|^Ž´òvŽì´92íκ·ªb3f,¹7šé®}7Õpý›åyC{ÿnÕ|e´»3Ï?4(oõu«ªâe]ÏYúÆ&?/7ZÛâÅ-ôݱ²ÓæpèëÜÙ¶éCî¡9VœÙF!Û;ß6Pho{wÚ…}qâ§ÚE»Ír½¦q©î¹ù"Ë—‡n ³Zoî~ó«Ïæpè|BìøYyå#¶œ8wµVÒÌz—–¦_;þ.䞉)s/ÎbÏÉÛwùtÚ {îzt8”›ŸæqSY¾™ËPëªÏ7[Þ¼¿ã·ž¾»§w|CQ§ÍáÐy=ýt߀{üžþžUÏÝÓG{û˜¾ÓæpH݉I¿HþùH-ìÈ=ý߆Ùis8„vÆ¢õ^ æ}…Ìõûj_U+ñvªßzõÝ^ýÕt>†·¨4kÓùDÁªP·Qà·›dmëÃ9òçôÓÂÉM8rD:ׄ3EŠ>%“àóòþÓß*ø¤¯†ÐoÇ/b7¾â³Ã/ cÜxŠjX"Ty™úá+%rõŒUc‡©îTœcUÈéô7å•ÒÓ»:I®SZ7þyMŸ‡|BŽL÷KðLöð‹ü¨_äo§{Wþ yØêòòg\U凓ÎERÃÙxÒ¡t:<÷ìH˜T7ñœÎb -ÒÊU§†ÂVç ËáÈá%Õt -±L¦C©Æ#Š«‹)ý‡¡ÒߟT$‘Ê?‘Áxb¨ÄAç‹YüxJ'Æ#&ð\§8 §²äE*',² 8Ջئ4VÈéc,™›?j›Ÿ&ÙÛl–Nò!™™á,›ñÆŒ–N­Åéé¥P“8º„™®\qźì3„s†T§3œ wÅÊÈJh×BYÝnkÊþ¨tJ[):7 ‹ëuÑ©ªúsZí«ê¼&¼œp[ÓJYi:ë3g•KWË"3•S±HmNY*„J=1¢Ë‰¾9kBe}UkéÄ´Q2¨¤‹9#ºœØtSn¶Ô”/©ÂlsÉÏ®ÌP7¡<#Êh’útŒxÕF¤É—b—j ×µ nôJx9¡‹¡•øñd=“ÚÎ\!m†^-4U¾E䌗Yš6…ŸÐ4³,7•þ©Âibžáƒ¢©tfÜ|5ß$µÍ"^R-w…uˆ+E~N[—ò¾`¶./$“>‘nI]Jl!­Š¬’û˜ÝK$UÈc)VuEœ(,—OáÊZ(³¸Ý,¶Ê ñì§-M`Ï7sª‚»-ùÎbW¸ó±ûRã*Xü-ùV˜ÏÂËбQ+NÐù&³XËÒÛÍ7«„³c …Y<Óèã έµ°Óð óÓǹ‘q=îåôËièæ´±:âƒ3*g[G^êü?¤y‰ßŸèüc>@mìǵ,l¡ ·.ÄuO–N?âtz–fS 5U´?‹U˜ÓÚŒ†Òk]ñ5Íær&öœRUE[ í\†•Xèjeª’±W„—¦K óÔº™;ìñÒä‡ÃmL:z/{…‹úqpf™»äs•%®RÒ¹K‡–teˆ‹:S\V7Œ³g 7 ë›”¢OþS]&Ý«º¨ÂˆtifÍÝlu– OQ%Alê‹:cRõMãÂÆ黛דë}rÙ夅ڋô±––KÒBx¼Ô —nyYúsN¹ñÎ%Tò2UÛ3%È™›ÆZÐâ¢Æh% ³Ù”ÀÖ7¥,[Ü4w…. “YVrôÙ'7™åbž8°¥äKL1K©KÚÖµR -ÑÖµâViS¾¹Ç+Ñ’)­áì秓sÊKFæjë"¥9¥É²õ½iyïÜÛž«Ç Ù7G¦r6³«”wñß=mg´if"¸Tùg˜Qa>r5)o´9ÇîçÀ!”÷ªé’›m®ÏŸ¬gIòüI}'T‡ c¤WÀÛ† PÄ}E•ÕãtZ+o%ûH·3=4Óœ\Ía¦)Ëa’¨\ú4T´q8¶[†Ž­÷R±ÓEÃ7¬˜ÅôÉðó6´îê -endstream -endobj -574 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 573 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -575 0 obj -<< -/Filter /FlateDecode -/Length 4842 ->> -stream -xœí][¯ä8~ï_ÑÏH|¿Hh±ÄJ#xXñp¦gfWè,ÒŸ¯œØ®8•tΙ3«F0£™îTùR¶?Uvœ´¾*üýJã¿äô”RŽÁ]o?^~º¨)z…?%Åꢨ¿Å¿è¯Z¹8yCò×I`ã³71\Ÿ/øo¸äÉŸ/ýÅåë<új¦œƒÑþúϤrvŠ)FåU†Ji+¨¼Ÿb4>ª '41gI3|tÆmUf¸œ¼¤‚Θ$¨Ì°.j»Ue˜a¢v’ -fèì£ Ò -vh£”¨ƒ!Ê9köS@§IgôR6YÔEèt4’ÎÚ)D§½–ty -!ø$élñYiIça‹×Î)I[œMQÒØbƒU¢¶˜­¤‹°Å w² K°èJ¢¶¨ ´¤Ëvò99'ê2t*Å´Õå'Ÿ¬U’NëÉ›VÔaFaBt¶峨ƒ-Þ*#é,lqø+éÀÞÆ”„Ùjhv[eµ¤ó°Å˜èDlÁàK `lQ1H€4/I`bœ\6N"“ìä’K ˜”'£•hÀd?¹£ÄVi茖ˆÀ*Øâ]˜ÀjØâ0sEl±ÙI\` lA‰ ¬…-ÆY‰ ,ØÖ-XЭSYK\`Á·Né qáÂL%qãڜĔ (e‰ ,8ׂ;%.° ]L÷(qëZŒÄ´k] -Xð®uZI\àÀ»Ö"§¨ƒ-°Ä`ÇÉêd%.pà]«U”¸Àw1ŒÄÕ 8 +“III\àÀ»&)'qïš êâ„9h%.pà]ãÑÉsƒwWFâÞ5Ή ” ü£Äh¼ -ñr8[@°ÀßCg¬Ä(l2K$.ðà]áþ„9 ¶štÊAâÞÕ Ã'êò¤#z\˜·èåI‡%.ðà]8*+qï‚>“Äàø ÕH\àÁ»ÚÂSK:ð®6YI\àÁ»ˆ¼Ä€Ã¤H\ÿ6i芺<©œ’ļ Z2À O*ID]œ$-qÈxRˆC%.à]¸÷,qAï*o­Ä¼«ÂQ[l2 ®ÓV˜ÓV'eÀ’¼«4FXÒw•ŠYâ‚Þ%·)qAˆ$0Iâ8! -6„ä¦c¸ dB>Q&\{‰ ¢‚Àƒ²DlqÑI\€ÀzÊ6'‰ ¢-Ö‰ ¢-$.@D0eè[\ÀÀE\8Ø‚HQâ0Ù”²sD°C¯Jº`¦A×¢.A§£Ä1º ®XK\€µÔ”0é%.ÁO.I\3 ÓNₘa ¦ŠÄpES2ˆÌ%†-²%.@¤ Ö$[ J\LÂ*3(‰ ࡱ–DWK:§h™$.H.L*J\€i‚•&˜RÔÁЄÄZl{‰ °øÁbËQ[¬w¤dh¹™$.H ¶e$.6i½%.È -¶ ⓸ «0Q˜,qh:¥$.È:a½iœÄ˜—´ÞLd«°ÞÄO˜·ÙZoF‰ ²ƒ-!Š¨ƒ-š²0oÁIXo‚$]€-&;‰ r´ÞL ÌÀz½¸%h½Åíð®Ï`&IÞEd$.ÀÖ›ˆl¥µ½ñúˆøLV&Zq&qÇ@z=|«¸e À½ðVQÜ3P _Z#‰›0+ËÄ]¸U(µ· øñ˜Ûp¼ûÄ·¨àá3S¤FfL ‡è¾­9z2»“lÞûç÷—_>Áù^¿ÿ×åÓ/ÊNYý« ”Ôñ¦Énè[£ù6`Q‹i°ìE‰©Ú˜»YD¸l³‘Ypxôçºý²²¦’±È•P Õ¿«¥bpO)ÝaŒ!/Ùm}qž»&GÄùÚ¾hÙVÖG¬^ÖI‚­/Îóö¶ÒBÔ{„˜ëze#ï$~ÛÁïõR$¸­kkôæ€lìÙLol´íEƒPN<œ‰‰Ò-¹~fò ³¼¡ÁT "€46ò  E¬}”[%þõ» Èt±bÞ§o—Ì|ÄtÎÑøë»/¿üÃ7ùÓ¿ß}º|÷+¥\øú+MŸŸ–ÏÛò™æOÌÄów|íéÝ"Ó]æýò¹¤W¶ç[å]êòµŽ°Î×êö=MÓì{šm«×åûÓRϧu9$¯ö6[oK¹¬îãÒ^³È–:J™™]K^—™=C†ÚŒÿ|îu·ö˜¯ÿv}÷Íå7ïÖ@++¤h´éŽ•[¸‡ÍsY>›_’9Br„bë¶Þ…­KüÜ{:<Bí]!W†M*+®‡»æ'Ý›L¥f{S¨AT0|böëúÅ)XË©S¥Ù'C”î"-€sÙ#pÒŽ‚ÕCŒžÌóh ÅJþ¥ )CnH,tBpŒ FÕôgF¼Ö1ÚÐ7QKÇlQðïÓ@®Ò,¨Èí¬yOöCMk2vCy»µîå=—çáê^Æ£ŒÓ¸ÜðÖ?²Q¸å /»ÄÇæý1Äq#Zw\ºfÁãYŸ~2Ïáñ\ ù±8ÎHnˈ*ÏêzÃÆ™G}'ÆžÌóhHÌw"ÅG>l8rb5;GsGd´Ê;ÈïÚlûÚ‡käñ†ÏµæWìþÌópÈ=Žyï‹?Úzö)q‡ºu?h”|A[w wh6{2'¼î1§‹iÒÖÑ©Œ;È=—çá{g7Ðèà+M楹§½ Ìwl‡Q=ùxWã †+Ñ…ÝÖg‹‰÷£vüüm(ëà.Ë!‚GÔ³Ý×Ä£9¾‚KÏåy8DÞ‰Gw´[틼Qâÿ¬~Y¤jÅ‚Om¦¬’¹{<çl¦GêVÇ!ë°'ùQ!§¶=ŠnN»'ü¿÷ÎûØz9^Mkè²Gækêªö H^ÒÔóQìèz;²½E¯GŸËY©å˜u9îœg¹i¨~ª/(ö0‡jDzMyØCÍG¦—S«òŽÒ·6øÚ†ÍPþt å튺¼aWDZ†^Á n²46ùJotLÊ:]‡N—ÎÖ×œå‡ ~j©–ß2PôzczµeZ~±¤ÿ© úÝDý_ž-)ïWEœL½èâç ½wÞ bçú*wÄ$\¾ªV fÝÀ°qÊcJë'µ|íÅÐA¯ù«ñµ5”ß×lô>@§j"[Þˆ7W,‘3è­˜Z¯&½ˆéÝýi²ËUh¦Ð1æ&ì:ž¶Ø3Þk«˜Ðo…ŠU×ÅÆU{4½ëµu]èÂf:ï:͆¿¦ÅêÖœÞ.¨ÖŽÒgQJo ¯EÕÞ5ªÉ,3*7©¯)Û€¸6N‚èv„_©Z´Ã`6=5zïê,LH®¥Ë5 º]ÄtKi¾Tl©RÏškÏwS:Ômj/ÈnìÁ±H?FÂ戲õëZLÏCµ±V‰áÆl„· \ŒÛˆŸ/> -©}ç -å+ô¸Ð²r›8©Ž—.¥5éü S³Êè­•ôÿL¾I‰yÊí¤èͳýªWBs³‰×Ä塱Ž“z@»ØÓ²îÒ16ÌòþÒI é‰t[êÖc+)ë2&¹šG$ÕÈc-Ö uMLVû§q%ªjªëbß ü¼w;}›ª^µÊ‚fp÷­Ü.;Ÿ¹8¶×ôáfo+—a¾ -oű¹QLÐOµ’.6ªy»^vÃ@ -]Üiô¹ˆëlmìTÞß±|í“Lò¸·Ë—âæŒõ†žU -VÓ[ñy䥯ÿFšoðïïwýc} ëìÇjla·®ÄÜ“Ño¨¤“búM•*lÓ† c´ßÅ:õ´¾¢¡ùqc_»ž]ub¯)ucPÝ´\è{2±iÐ5Ú²žñÂÛЋ-ùܱé{¾´µqôÔ½À·Z… Þ\¦®³Ý%_Y‘¸¢¤Ý¥Cë ˈ ^(.Y…¹[¶2“É+i]Oö;>–êb,Œ KÛu®»Y^t••V°$-¿à;Í+Í+{£_=½>{Ü“C$mÔÞ¤Ï\ÚFÒFx²Ô–Ë°¾l>Ç–G”êäí=ÔʲlîÙäô©ÑC ·ºàeÒÔ³- <¯ÔUÙªÒê(]@¦ª¬•«MaÉV»¥´8‰½[LÑ¥?ðžö|TZ'z>*a“–Ê­¯EK¶@žÃ9öÖ©ž²Å’Iž»I]Oi«l[·[×]½í•5ÃuCBLUÏæ7)_ů™;sža'BJUÿ–Ýú‹ËᨎÖØò[t*…qôs"Ñ%¶]òâ<Çû'Û]’ºÂkÂpLÎZ5ýª lÂ3ù¾‚UqPΖ•Ø¤—u€°=ÔiNmv‡ÊÏJ•ý˲IÔ.éÙC¿"g)èMôã7Qiù}!e+hµ}Rþþ¬‚y -endstream -endobj -576 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 575 0 R -/Resources 4 0 R -/Annots [ 577 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -577 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 457.623057 101.433071 438.873057 ] -/BS << -/W 0 ->> -/Dest (cb112-1) ->> -endobj -578 0 obj -<< -/Filter /FlateDecode -/Length 4395 ->> -stream -xœí]K¹¾ëWè¼€¹|_@°‡I#Ä€‘‚ÆZ{Á8€7‡üý|l5bW«)ÇnÛëÁxÔU,v5ùñcñÑ¢:Jü¼Pø/!DïèxzøpÂ[‰SŠ‹‹Iý -¿Þ•$/¬Š.Øãû$0^øhµwÇÇþë.Ûä‡|wøÏ¥:j£ÓÊ}›Td„ÞK˨"TRFe­ð^[ǨœxÄ9Ü°ž4-UnP –SÁ Ò:0ª7 ye–ª7´WÄ©à†ŠÖ3*%á‡ÒR²:8"‰ŒftÊ -‡BwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¾XE$9|!<§sðÅ8#Y|ÑÁNçá‹FéDFà ÐX|‘N*N°1±º >,uZZaƒ1’Ó)%,°iXZ*ÉyF§á‹“6²:øbÔœÎÀ§OXãC`Z«N­ÛH£8…/Z{buð•Ï1€vðEzÇQZH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHøbÉqL`|!´\V_L$Ž Œ†/HÀq1ðE“á¸À€m há¸À€nIFÅqß’TŽãÂ…›’ãÆ5ÁÇ” (EŽ 8×€;9.0 ]4wÏqëë4Ç´k(8Ž x×’x×X²:øs\vFÃqw’žãï¢iŽ P«8t+B‡ 9. ð®’8. ð®öèXhƒ†ãïj‹Gàznð®¶Rs\@à]MÚq\ ðd£ð.(Äòá|Ár\€þ:m8.@fB#,á¸À‚wUD÷Ç´i°•P!:Ž ,xWT«‹By”8ÓnQÊB99.°à]tT†ã Þ}Ž Àñ=ªæ¸À‚w•AOÍéÀ»JGÉqï"n°B!à¸ý›PÐeuQÈǼ ZÒ 2€IX’â¸d,$âPŽ xÝ{ä¸Àw¥5†ãÞ•„°‡ÕÁ4Ljk C³eÚ4‚U!5˜‚Ów¥B s:ð®”>r\àÀ»©Ûä¸Àù$Ðã8(kŽ \ ½w¸(¡CÈÇêœÀµå¸ÀK,(‹ÕÁòÄqkM x _ŒÖx _2p\€ˆ@D…è›ÀÀ…|A¤Èq˜L„HÄq·ã1”*§sZºfu:å9.ðžºbÅqÆR" Ñs\‚)\â¸n@§ˆãá š -ÇèŠDЈÌ9‚/²9.@¤ R _ r\tÀ(ÓIŽ ÐCc,‰¢æt$Ó83p\ȉ*r\€f‚‘&˜’ÕÁÐǨŸ4ض`ðƒÁ&†9¬¾K„ Óp3p\|ÑRs\l¦ñ¦ç¸ Jø‚ˆã‚(Ha2Ç qè¤ä¸ ª€ñ¦&Ž Ð.Óx3p\ÄxC<¦ÝFãÒxÓs\ ¾X„(¬¾ hŠL»'a¼ vât¾èHDçÒx3p\€0ãMDôì”@HãMÏN€wm3q:ð."SÇq.0ÞDdËí%ˆ×zÄg¼2¤g`g $¨×¢oe§ $¸½•gç $È7‘ØI¸‰‘ep쬺U(•d§ $øñ-8á<ûÔNQ9ÀÁ£Ï >=dD³ D÷eÌQ“™•dçù±_>|ÿ€Î÷øóï¾›fÊòB€0¥&ÄWh:¢º-¢o…Ç7­Ä$¤ÉC8xMrÂÇê¤-ΓnÉItéßqùáÂs8þjº½)·èŧÛ2´J@¾NsƒÇ7š|D‡%GèmÒ˜¨;rˆ+%]$þýë€:{qž-—ûè/1&m¯ß¾ÿËË¿ÿõÇW€Ñëw‡þNJ -?¼Pøkýù/¹ùï|fêl–å´vþ—2 ©æA³LUY±ó—¦ÚM¶[þ´ù÷þd›ä×iþüî2­–ó3Åê_ñ]7é³}£ËéŠ]ó\ô†)‹ÖæmÕ+›t?üëøúåá¯/Á6E çæN&JŒÀÌ>ÇLž‚Ïç€%†³W`yätSõsõeû©J ^d}õ‡ZÕÙþ£Ã¹†=í ¾.Gº y¡M˜–Ãñiä{¡c6»CèõØñÉ|2Òƒ>ÿ|­ƒG¬nbÂ49«ƒÒv±ƒ6{C¬¾GÞŒ¼+Ÿô¦ãá[jô–QΧjq]ÌzkYAdZ-¤%—Ðs‘c6»Cäõ˜r¿\²ãÖruDµ‚¸&F”$¦¹Å­ùÈA›Ý!îz\YF¹”5Ó{v5>¡òaS6У&×JÏ[k÷´oÎ5øQ’+Ï´£]ãÂáYòA›Ý¡rcr¯œve6„ìz÷gi%W®ñ£©ñžMB˜Û¯"qÐfoH41ân8ã+owôâ¦Æ€ã(³ÙJãÆ\‚ÔÄ^ᲆÛÚ)£…܃{Em-T¼ëjó¡É¯­V B$ï“nfµSí/| —öi^õ©¨[ðí³—á -ŠííãñA›Ý¡xcŽóÞ˜Ù7È×÷ wwÌgÚìó™·òã¶_|þβv°Cõ”î†| -¾-y4-2Ý;µB3s´±Kn/<=¸FDÍܤ‹¨gErkErÐfoˆ¥ë±ªÛµûY÷ÀôÂ9]ãÏ=È»M°ðk$¥f.rc6»CàçCm—ÏWâÍÏ2Ö}]´Æ›×ÿWYãÃô2\PDvkÝ|Ðfwˆœß¼¶;"í)÷ó¯Û®•‹X­_adf¸‹Îu¾¬ÅµÝL¸íPÐûñ±ööл mv‡šûײ/x¦{vÜ4Å€®1/÷xw™jöM(kóëFI=.|i×Jº1{³fýgoƒó”¶™s­ Úì ±ö·yÊ‹–µ•×*ö`Þ‡ö„6ó£+ƒ6»CèõØð^.(µ‘KþaFa·ÉóÆ·Ø2Vho_Ó´Ù7bÂ]ñÆzô6Vl|a‘ùС>íNÎ;•ç¸w±‹YÏéâ3sd¸ƒ#Çlv‡ÌuoÕÄfÌXrk43\ûn®áö7Ëóžöxþ»Vóe”Ñï>Î<ÿС¼×gt´­ª‰—u;g¹ö67Z[ãÅô¹;öCÚì }np?dߦw¹‡f_qf…¬ï|[A¡¹}ì=h³;ŽÅ‰Ÿjí:_ÈåšÆ©¹çê»,_rº‚N{û»‡ƒ6»CçbÇÏÊ+±å¬Ä¹‹µ’nÖ»´¬0ÿÚóï…Ü31eîÅyì…Ûwù Úì{ףéÜü<›ÊòM-C­›òÌÝOÝç®Ü7ñÔЛ‘ùb¿p·+ˆ]¹É«1-S1ÑâÚêÈÄ }þy·O\úRÆQÌ*Pi#]Úɦí­õ×þŽ·±mö†J¿5vï]µ«÷öiw½Ù¢oCÁ(2{®Æ =«Ê®<ÚõÈ~- ·uc»‹ï*x¨­q‘w솴Ù"7V¡ÛwcûÕ›&2ZpW»&Hòbt¼¶;{³ö¯Ô0;zo#·®Õpï_,ø¸[#\ph¿âÝïéçž\W&o¼¯÷ãþŽw¹mv‡Ïë1ä¢.VvÔn¿5/7gŒJ'¾¾/vàÞŠ%—̾ðgÅZëÏæüÅ -s¶ÑbBE½µŠ8h³;dn¬{?C_¾ç™/"æ¿°ª}Ûf³ƒ6{Ãl¸.Zÿ=«(,É®˜X†™®¬˜<Å¿[×@Ÿ}–`…Iƒ¹ý ÆA›Ý¡rcó«~õ2ôM市J{WŽÙ앃oØÜó‘_ÝÑB?êœþ -RïxûfÐfwH݈D¿ ¾Zi‘[ßøIZ"Ð(oÿæŒA›½!4nÌ{îäû®¾ÍÈ`æö‘ü ÍîÐùLñç<ç÷]=GOíí;×mv‡Ô˜ô‹äŸo §¿ãm A›Ý!ôy¾Ùòæý¿õô×zúWóyÞ¢"­Mç«Bs¼Fäf’ .ãÈGZÓO17cË!˜é \Ž)útvL‚ÔË?þùOk •^"ÕoϯlŸ·Èà³Ã/HÀ¸óu*ªi2Qå E觗Osõœ«ÆNƒâTœçªófÞô7å•ÒÓ»3t’\§´îü;å5žò ™¸æû%ÒHöé òýY‘¿ïÝø3åa›gÈ¥qQ•n:IMgáI„Òé°Ü£#aRÝÄc:{)HC*Wš -[W,§#†?”Tó©Ã2D˜¡:IÜ\Ìé?L•þþ "‰Tþ‰ Î'„ÊIt¾¨âÇC:!–1a”3(NÂù£,y‘Ê ‹ìžõ"ö)rþKæÆæÚæ§Iö6›¥“{HæDf:»æ|cFK§–âôôR¨YœNÙ ÂÌW®¸b]öš!µik†ÕpS¬Œl„v)”ÍíªXSöG¥SÙJѹ*,®·E§šêÏiµoªóšðtÀmM/}d¥élÏœU.]-‹Ì4NÅ"µ9e©*õĈNFøBæ¬ •YôM­¥ÒÎÂD%]ÌéÑéÀ¦›s³¥¦|IªiÌ%_]©P7¡<#›Êh–útlxÓF¤É/Å.Õ@®kÜè…ðt@C ñãÁz&µ­\!m†^+4M¾EdÅK•¦ícçOhšY–ΗJÿŸ©Âibžéƒ¢©tF\½ª7Im³ˆ/ ©•»Â:Ä•€"_Ó6Å¥¼/˜mËKɤO¤[R—»6EÖÈ}Ìî%’*äq)VuEœ(,—OáÊV(³¸]ÛåÇ qõÓ–¦ -°ç›9ÕÀÝ–|«Øî|lžԸJþK¾ æ³ð4ulÔ‹ôC¾IkYz»z³FX)Tq¥ÑÇIœ[ka§éä絑q=îéðËaêæ´±:âƒ3*_ÛF^êø?¤y‰ßèøc>0íÜkYØBn½·=Y:í< ŠÓéçYXšM#ÔÔÐ~«PÓÚŒ†ÒkÝð5UsY‰=§T…AUѶB[Ë°ë]­LS2öŠðÔ•b džZ7µÃ>_šüp¸IGíe¯pÑ>.CÕ™Ú%›,q•’Ö.ZÒ!.ÚLqÙÜ0VÏ.ÜL€loRŠ>ùOmU˜t¯æ¢ #Ò¥©:ªÝl›u–MOÑ$AlÚ‹6cRíMã…?ŒÓw7¯'×ûì²Ë- I µéc+-–¤…ðx©™.ÝåeésÌ1]äÆ[K¨äeš¶gJS›F -èâ¢Åh# ÕlN`Û›R–]Ü4w…. “YVrôÙ'7›åb™ž8°¥äKLQ¥¿´%mÛZ)…hÛZq‹´)ßÜã•hɔȶpöõédMYbÉÀÈ\k]¤TSš,[Þ›.ï{ÛcóTq52•ÕÌ.RÞÅ÷´³M7Á¥Ê?Óìˆ -õˆQtÔ¤¼ÑæS¸œ‡P:øÛSh¦Kn¶¹>²œ%Éó'íP‚Œ‘^o+.> -endobj -580 0 obj -<< -/Filter /FlateDecode -/Length 4763 ->> -stream -xœå]KäHn¾ç¯Èó£ã,æ°ÀÚðÀkx€†}XøPݽ £ÆÀ¬þûþR<¢”ªìšv6Ü…êL‘ñ`Œ/ÈPH¥¯ -??hüIO1¦àézûåòëEMÁ)üË%V™ý3~ƒ»jEar:ù访0Á†)$g‚¿¾^ðßpÙ½üûï.ÿµ®£¯fJÉí®ÿÌ,²Sˆ!('°XJ[åÜ‚q^`y=aˆ)I,ˆáڲĠĂdLXbX -ÚnY b˜ IbA \XZAm”yDY#ð´›<”î%žÑ“É$‘ÀÓÁH ¬Òà-UÅ‘—ÀjÈB˜¹"²ØDXYP@Âk!‹!+aÚ¼E ¸%•´„xKJ{ ,b* ,×FOX@.\)IX`¹Ø)aèbº ,P×:o$,°€]KÑKX`»–´’°€€»Ö¢¦Èƒ,p` €Ž“ÕÑJX@À]«U°€€»˜FFÂXu‚z à+“‰QIX@À]IX@À]°ˆ¼0aZ ¸k† ­ÜÀ]㔑°€€»†Œ—°Ž2” w!N V¬÷à+a›  pW',ÂœZM:&/aîêó‰¼4é óZž´÷IÂÜÅBe%,pÀ]Àg”°?aE58ஶX©%pW›¤$,pÀ]Ä N¸äHX€õmÒ -®+òÒ¤RŒ8à.`ÉHX€UxRH"òÂGÒŒ'…8TÂÜÅòž$,ðÀ]嬕°Àw!ìyÅF#aâð0m…9`uRH!ñ€»Jø«THxà./›øÀ%,€€¢`#a4¥¼„>)ðò‰õ[Tî°fÆÀƒL˜„è¾æ­˜Ý)6ïýý¯—ß¿`ñ½þõ¿/_~—wÊÊF€Kâ+L“`n‡è[cø²(“Eƒdo*Ì݆ÔÄÒÂe›Å‚Çÿ®Û/+Y!* ‹ÚÈ„Æ“8ËK'ò«¼£À - <"B26^A‹È}­ -ÿñÃ`ºH1ïÓ×ËN|Ät‹£q׿\~ÿÏ?ýÛ¿üùgxà‡/—¿üA)çüAã“âüéÜò™¶4 ™¢Ržšn´Z7̟ʶz«º_–r¥}¿®'õ[å¼-Ÿ/ év¿¿,ý|nm“[·Yú®}¡I;ýÔ²·…Ö•û2¬³´GÃÍÿqýðÓåOÖΗ³¦Ùùx#Ù¯¹‡þz®Ê×øëoá¦)¹iUåmmÖ¬ÎbbêLTLùQp§¾NgBíÞ[¦„éܶ¸µïxï1½7û¿™V²{ò]¥âlHØŒ¦HtÇ?OÖy6EfÊCcÓd±bÈ$ª…ëò§3 m±Àʲ£÷t õ8à|¯³jÇmƒ=Ë–§dî!åÉ:Oç‰tŒ•Tëv^·ò ïaIÿ-f×7X¾u<ž]¿OÖy:¯<4¿O¬y§Ùtr¯^ªO|é¼zhÅYXÚ)³¥Ê·ã¥-d¤ä&——ä{^z®ÎÓyéqœY=eÐàÇ|Ãêð`ô RþŒÅK£ ÕóÆÔÅmeQŸðûå|:3FµîI=”òç=pÇ4´·ç¡¦ÅŒ§=ôdgóPs2Î,ì0­_ 7¸µç‰GÉò \úžcÌ1Ê¡Ñ[wVucßž•Ÿ¬ótÞx.Ö|++å×xãèÝÎu†x6{/­íâEqþÜmg“/ñë£kăÍZö]¬t`å¹:OçwbÎ1û™/¿±­¯Êw<²Û’<»ë~²ÎÓyäør°DÕêAtÿ†IXø@<˜½aȳ78¼“ ˜Yq±Ÿ]›øo›-ýv³VöVûÀžæÉ:Ïæ­ö8Öün1mÏ›OäâãÖj&~Ó´ã-ŽÔk'œºççê™fùɤ¥}»hÛyîcÑkëÓ¶oùØ?²m'ºhÍg É㞪…µZÇÇ6óLdãG„¼"ã“Ðöìukoh^Ô<ˆõlùóV¬¸Èä -¾ÏßËXY -­l?÷ãÙÓòéåqdÏ)e¾tž\#¹¿9XÊ•uÈ-|½´ÿ±ÉW¢Y÷NúÉm}žë—uŽÛÎw(bAŸ¥ßnÜyö¨NŽ2æâOaþÌí}^ük±s©uöȺfqÅ–¬ìZ‡£«L¾Íj7ÝäéÉŒzªã ƒìøÓ‘öÐh‰†9å:Da\an« WõÉØúÌ6uk_“úǹöÑCÑÆ·^Ö¾Yû-ó \¿´ëÜOé»øýçVWÒg)[ôÉcßè¥ïÿe‘g©_õɆøÒÆíLÓqælvp¶éRzÄ懺ô]¶Zg k¬¨}˜“óÌìȬûöŽäüò©½OrLèù|³Í‡êø±1$ÊænLx²ÎCGß1tU^¢):U‹Á`6Á˜hê»]°4a·Ãäó#AC‡¶:j°™úq»Æ¦É¡E„vÝ—ïÔ2ðstÞïIu§¾óSr°T?Oø!„v'Ų́“óö@SGÛ¹}bT,T-÷†DñÄøÉæçEøìüÚç§&¡<vGz§>[˜gÍûiŸ¢ÁŒ‹„íØO]ê¬qØטÔÈ8< XäÅ4Æ, ßs«ÛîÍÛ ”ŸCNüÄßÐiží;@´. Æ>¯U »‡€NÖy’ä¸â!àöÕK¿œ‰ì ¯Ž/ 7p[x5l•6>Î}©\è›ÍÀm–š¬Ÿ÷IÀâ‘OV ³¿åÅ[>¹Ù£¨åûwÉÅ ‡ý1™â[Nùs[‰'ë<›?FÈzà÷nfUÌ*iÔËü}Cwà :Êñu˜CÊ\çcãÓ•³Ëï§å“–ïÜ~™ßjûÃ#óuçÌ8çó‹>4~®Î“x]ÛêC¯nln GÏÜfŸ ~Sú7Ìß‚ÜokÝ(›aë:{äÞ -²÷tCú³‡vOÖy:OuÇøx°¾~Ï·¥%oX¯<ëYZnë›yªo÷ÔsužÎSã1¦ªÁ£>¶ïó¦g‡—~^]ßßJ;r Þ^=½[ÅËLëo›+ùÆ#ïîÓ ¯r#ÓŽW¶XÑ?)ìÝøòdgóÊt_>üÌßN^Bžíì<¦zJëÔvw,›ôÛñæd§³ì®hòÖ­’Â1l¾ñ´ºÕ;ZOwVZ<‚ëðM+Þ´®{ÖÿØe5Ãã/}ÝŠ {r‡>6Gª_v<·?^³Ãöø\<ôî-ï"£ì§-#~ïCä×µÝóÓsužÎO#¸ÍÚÑ?v´ÄÛÅö›ƒ¾ùÀ¸öŒ‘àCìd©gÖº~gfó@bÿD7wê§Æ÷iŽúöŽ^¥.b:{ôêd§ó¦ã(ëh=[!ÒTrÜ<ðzrÃÙÇÛÅR—åö^ðÖ‡þ¤~8jZ3Ð/ûyȽh­Ÿ1ã¡ÉUä5æZ·µlóƒ‚;žÚïÒõÔsužÍS¡…ãÔuµÑÝò®›>.ÃææZÿ‹YŒL-Üâ{0,ßvvQés÷¹×qJö²Œn庾¾î¼êÏË[ƒ‚ƒÕœã·¶E§c÷¢êw‹Üq·²ý¾}©ÊU/²YïУI~]¸W¯Jß°µ·?o>Ïñ—^ °éslU"$ÕŽÈ¥¥LÙ×ïñÔÃ+Ë¡œr¨ ïñ/7ïó¡ƒ4ÓM=p¥Ú=)¯º£jª0ù(›šT,÷´Ví*ªcpe Sþzñùmq:¿1Tù8iï¿zš,ÛéZä7ÍYÒÅt:+[_wjæ±ÿZK-ïfWüºV~Ußüâöîb)ÿk6ú/ÍGÕŒó{”U&GS.ùõÂïÑ&LÔŽêÝ!3qùªj[üf v¨†)%­›ÔòµäóWãÊh¸¾+Õøýf¤J!›ßð5w,QÓë-™G¯&½ù]äq²Ë•¯¢ðMøJl R_¶5Ø*Þ%k«:¢ÛU×]#*òh~weUoÄ*z¯:Ý™¿”5¡3çñvA·v¤¾ŠT~riªhרJ³P©R])Y BÕNévˆ?¨Ò4Á˜•ßYß#9cq$ªåR)'n±ÜÒš«– -µTlUSÑ|¥¹ºUñíÖ¡ üǺ9¢lùº&óÉÐjk;¿1âí‚%†6ä׋ Bi×°B¹âz=ÑvíVrTÍ_5Ä¢*LÍBãƒ.üÿ ®RyòMÕ¥øMšíªuÂs³’×Jêé¾¢IÐZÙN]:„ê³½¾tTByÝZºjlEíTÖÑC*â1HUðX“uõºJf+ú©XÙUÝ5²«®üº"79]ªNÕμîÜÝÕvÙWì|íÉ¡Z\ókQ«¼µÝÎç ñ–6Éìú±tÒÈFÕÕ®uÖ›`…Fn0úšÉe¶VtÊï!X¾¶I&­¸·Ëß.y™3Ö>iç­æ·|÷‘—¾þÊü„ßÿ¼ÐõÏåµ’ó:nTE S±uEîW2þ›ñ$™ÿFD!ÖiÓ u°ßÈ:¶²®xC]ÇéðšZuÕ€½”ÔAuåöD×tØ‘Mu]£m§w@¼ Z¬Ì×Ú¦-Øó¥-ƒã3ƒüBÒ".úáà26žmKòµkW\´-éà’é*â¢o—]‡©I¶“²ï¤ªžå§Þ–ûê.º0‚/mãQ[fû¦ -¢+‚Øö}äûNÓJA臧×WÛ}Ù—ÀÔ -í•úÚSëÀ˜ZO¦Ú|é×—uͱù€]™¼MCµ-ÛÍ=[ƒœ65ZhA«‹ÞG;jlÕ–®ï” -mÕiY(*\€¦ -­¶ŠL~©VÔ’GE-…S4êßzM»Þ*U‰®·Šß”åvËŠW£%[ ×»sh£S­d%£@ó}íJ¥VÒÚ¶oZ÷]VÛk7 j‚ø™ªVÍmJ>„̹ΰ!•*?ywDÇö"f,Ô¤ƒ56ÿm-½†ÿy„@±Û.ysãý“í.IÙ?é{‚9&²VÍ%MáŒtŸÑ5Uþ‚š\-)qHoS€°=Ô`Nmv‡òŸÉÉ{–y“¨^Nù¯bYz#ÿ1 ´ü¤d~7üjû$ÿü/< ^Š -endstream -endobj -581 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 580 0 R -/Resources 4 0 R -/Annots [ 582 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -582 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 371.373057 101.433071 352.623057 ] -/BS << -/W 0 ->> -/Dest (cb115-1) ->> -endobj -583 0 obj -<< -/Filter /FlateDecode -/Length 4538 ->> -stream -xœí]Mܸ½÷¯èó«åGñ ö bdƒ,`$‡ {ÖÞE0àÍ!?j‘*Q¥»gÆ–x0žVI•ÈÇÇb‘jê³ÂÏ·ÿEÒCŒ)x:?|8}<©!8…cŠÅŨþ¿Áµ¢08|tçY`Ã’3ÁŸOø¯¹äÉOûæô¯e}6CJÞhwþõ]V‘B A9A• RÚ -*熌ó‚Ê똒¤‚.¡µ*À JÑI*˜AÆDAa†¥ íZ•`† š$ÌÐÉA¥ìÐF)QC‘5‚N»Á£Ò½¤3zð1™$êt:Igíài§%]¼÷.J:‚-.)-élqšHI:ØB6Iça‹õV‰:Øbb°’.ÀƒÚI‚. +Š:Ø¢¼Ò’.ÙÁ¥H$êt*†¸Öå­U’NëÁ›VÔ¡G¡‘|t¶xå’¨ƒ-Î*#é,l!üH:ð„³!F¡·šÜ»­²ZÒ9ØbL Q[Ðø[Tð  ™—$0! ” I$`¢(R”XÀÄ4PV¢“Ü@>‰¬ÒÐ-U°Å‘—˜ÀjØBè¹¢¶ØDX[@âka‹!+qÛÐ"qÝ’JZâ ¾%¥½Ä„ 3•ÄŒk£'‰ ,(PJXp®wJ\`AºèîAâ ֵΉ ,h×RôXð®%­$. ð®µÈ)ê` ,qØq°:Z‰ ¼kµ -xÝÈH\€V€C/q°2˜•ÄÞ5Q‘ÄÞ5C€¨ ú •¸€À»Æᤑ¼kœ2x×ñ(øG‰Þxâdw¶€`%.Àx± °ÁÀ-‘¸ÀwuÂð'ôi°Õ cò8ð®Žh>Q—PãB¿E-Úû$qïb ²8ð.è3J\Ž0¢‰ xW[ŒÔ’¼«MR8ð.ü'qà0h8`|´tE]TŠQâÞ-‰ 0 -*‚ID]$-qÈxPðC%.ðà] ïIâÞUÎZ‰ d‰ÀÀ^°‘¸ÀGR^âŸtpùDpí$. -Êu°…I\ÇzH6E‰ ‚-Ö‰ ‚-p$.€G0$ ï[œÀÀEœl§(q˜lˆ‰Hâ‚à<æc¨UIçÍèZÔEèt¸ 0k‰ 0—":½Ä ø!»KÀ è4I\lAW‘¸CÑ öëϧïÞ`ð=ÿüïÓûoÆHYùÑpÆÔÿ -ÝÀ$4·ƒ÷­ñøÖ£¡õ0 -i´Þ“'Òl¤-NA·l$†¿üï¼þ°°†ÿ8ÞÞÖÛ'Œâ—Û笒Û*bú:Æ™Å7fyFƒ&Gm=ÒØd2;rð+-ÿöõ @¬¸Ä@ë%3ã%&ÃdÜùõ‡Ówzõ×?ÿð#`ôúýéï¿Qʹï¿ÕøKtùëò“ì-~ã$ŸÒåÈ]ÍónÖk7éJÞ&ŸKe…)ýûIç]ɧ¦¿vúk–åÕg(÷|@9Š=OkSXÞSMåæ<‹|“môД£ç2é‰ÉË÷~™ß¨é9ÓÚvm¾ÿÇùõ«Óï^/Á8z¨3N(e–ºŠß¾,OÁïKÀÓÝ+°]5‰ÏBúÉbÏÒÂx_VfÉ_ꩅ☾¤)ÝB-m,ÏQë£ÈÍ|­dØæÈþB‚#¢Þïà¶3ÏÑ€‹ÙÕUµ :¦ 7¬Fõ\Ë™$¬^Ö45Ähkm¿DEEvAZ£—ZQ½G™v}?ëçûÔr¢|Ÿzÿ‡¦¬ù~NwÁ-êØ@påY6iogÒÎ<‡C$]åÒZ‹®©=>tµC±›[ª¶ «õÅ°÷fo¨;6Ÿ·È«= °4/‰Tæ€bâ™T4y2t©}y‡ÔëÎjÒÆò'ƃœ×¶ø¥´ŠoZVïwèŽòm¯e•³[IÉazª)‡®£²/ÏáP¹ã‹î¡èÖÖx.¿¯ÕS¢/å22º/;ó ™fÇ×|ΙKÊošÝ<7¢:òˆ{o tö%»gDy‡ÒÿóSͽ_Á êîAî-ÏÈg…mu–UþÚ湺<×& -ÝíóòÎ<‡CáußRÓ„ºÉ‡Ô¶ùKSmæ4~J'µ¸0‡ß3³(ÌVp³´ðÈ]V¸ï»¹\jg* ÿ‰¦-O ôΧwÒj·f2†…"»Gæ¾<‡CÛŽÏH J ¾]·¼Q9Žö¼ýéüd¡–ùyS1ÒïE…:ó ¡öºïød>¹5ÂýŒüóuöŽ Ä2Ÿ0oq1Q·‡Ø¾<‡Cìu?òfä]A8_©ûf¤«Eo™å|ª×ø¬÷÷– D2ÿЙA9ò=×Ù—çpˆür×Û[®Î¨6Ç|DEÃVÜ‹Evæ9â®û•[k9b4„ÍTÊ 1hQSZ¥å­Íõ£·j½㙼‚§m­xn@F%±õëÞygž£¡’vâGå´+³°.dß0º¿H/i¸r‹‰¯[wîzëÌs8$îøˆ‡áŒ¯¼WÜ1ŠÓ{3;ó¥~c©Ab¾×Æ>Ž²v'—9iFùŠœÒšoXyaÆ'áï"ã«€ ªÇt7”[Q(ðm-ƒõÈ|ïqêÄÑ‹}¨e³¾ižsgȱؤOhgMjoE²3Ïá{ÝW5q¿u? òÞ£pIÇì¹y÷£ y;ìêñC‹Ev#°/ÏáøùýP×”³BÆós ÌÆm[W½ñæõÿ DÎþaþJ‘¨‰ÜÞºygžÃ!²3¾ymwD~37L¿~¿U¾Z»Â(D¸«®Ýa¼å×n¼?³êIn~†çØÛãïØÙ™çh¨ñ÷¯e/x¦{6Ü4ú€ŽGÌë=Þ/ËÑl/Ј2^^3Kjùpe _+iæ쫽êïûxéÅ{@gœÒ³˜cïl¥3Ïáûÿ8å¢g핵…ÊŽ=˜÷¡}¡,ÞØ»ÒØ™çpÝyçN.hßgZ­V#eÞø_ìHŒ·¯étæ9w|ÂCñÆFtî+2[Dd¾iPŸw'—Ê“ß»ÚÅl¦tée92¨Û9²3ÏÑvÖ½µÚ|g»g¯wwëû©…ùo‘—=íéòw«åë,cãK5ø=G¤¶ú‚Þ«˜¿lxÌrëmi¶¶Å‹[è»c?dgžÃ¡¯s?dÛ§¹‡æX~fë…lï|Û@áïlwæ9 -ûüÄOµ‹v›/ÔzMãÝsó]–/ÙÝ@g¼ýÝÃÎ<‡Cç|ÇÏÊ+ÏØs6üÜÕZIõ®=+N¿îò»Á§,£¸üVêö]>yŽ†½xÝ;ë-LqÜ\—oç:4†Õgñè~j>7õ¾‹§v†Îfæ«ýÂÍ® q妬Æp¦¼Å­Õ‘‘AÛòËnŸ´¶¥Î£„U ÞoUÚ¯ãocwæ9*w¼ÆöjتĽcÚ]o¶˜ÛPpë÷jmú -«¶ß‚Æ×#Ûµ>okæv‹ï*x3÷ÆMDÞ±²3Ïá¹³ -ÍßmWo˜g´â.¾&Hj1;ÞÚ½ÛúWZXœ½sÏ­é5Òû+>nÖWÚ®x·{EÚØ“oêdç÷+ãøïrwæ9>¯û«¶ØØQ»ÿÖ¼ÚÕ2äö^ìÀ½K.…}á/Š5nÏnüBfÎĽÅÎoöëÌs4d¦uïËÙù"<`³w|egžÃaöºÿ¹êý÷¬¢ì°¤¸bâfº²bòûn]}ñ(Á“ºÛß`ìÌs8Tö}óä×ùþê'dè›êq•w|zgžÃ¡²ó ›{¾%ò«â¯;zè³Æô+RœÎÊê\>«(:ÙÑž»Ivи>¨£wqÎ?-Üü„3ÇaȇäÚx†ûH)äse2¼^ýþø ƒWÞmÞ]^D¸~ñÙã÷áòUàù:WÇ8Dê2LB?n©.Äs!76u®²KÕ«i‰:ÿ½|­x®òKd¹Éiýåw,kú<–KsN÷ËðËùó×>†‹~Q¾›îÍìËpìÊðŸVMùñäÇ3’ôxNžòqÐ&¤{ö4ØÜ6éœÏeŠÊ’.M§ÇÊÖçœãñÃkªéDb•)ÌT]Ž+fSúc£8éDC®ÿL—ÓCÕ(Ž¦\ÌâÇS>=–1Ñ`:ÅY8}Tµ,Ò%a•=€sÃÚ”Ö jú˜jáÖ•Æ•§Éù]É–Oõ!UÙñ\›Ë!rz½ç§WƒžÄùÞ8ØéÊWSœ/6C8H<í\àœqW¬­bB·*v»Yl¨Ø£ó‰mµêü,¬¦óªÓ¬ùKZXs^>œp[ÛJEi>÷³Ujר*³Ì¨T¥®¤¬ BµÑÃI~«JфƬzÖjùô´‹0 QM—J:AôpÓM¥¹ÚR¡¦ŠsÖTj~6e†ºµâÙXG“4ä#ÅYQ¶|\Š}nÒÖ*2ܘ•ðá„!†VâÇ“ Bj7s…rz\hY¹UÕŒ—YšE.ŸÐ5‹,Ÿ=•ÿ¿P…«ÒÌ<ãMRùü¸ùj¾Iî›U¼¬$.÷•uHªMaN˪K‡P1ËëKG%¤Ï¤[S×[HY•1yHżLR•<–b]QWÅ™ÂJýT®äBUŒÀíf±«P~\ˆg;]íª{¹™× î®–;‹}åÎG.µÅu> °Ú[Ëe˜/‡q`£Vœ¡ËMf±Qu´›oÆ„³a …Y<Óèã(.½µ²ÓømïÓǹ“I#îÃé—Ó8ÌëLÂou>Û–{^úü¤y…ßžèüC9Lí2ŽUÙÂTn]ˆùH–OBâ|2zÖnĆíÏbç´® ¡ŽãÆ0¾¦9»š‰½¤Ô•AuÕr¡›ë‰M…®Ñ–ÕŒ»"|hj±:2Om›yÀ¾\Úòp¸ÍÇð«pÁ—qÖÙyH>³"q•“ÎC:´dXF\ðBqÉn˜fËff@ò›ÔªÏöo -›ïÅ.˜‘/í¬£y˜åEÙø, \`Ë/xÁ¤ùMÓÂÁ軻דÛ}2Ù—¥•Ú«ô‘Këƒei%Â>(-†‹.ï]/Â'ãÏ¡í> -endstream -endobj -584 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 583 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -585 0 obj -<< -/Filter /FlateDecode -/Length 4553 ->> -stream -xœí][¯Ç ~ׯÐs€læ¹A -´Eƒ¦h£}ú`Ëv‚â¤@Ò‡þý~œÛîR«Õñ‰+§ñ¬]r.ç#9+ôYáïsÿ"é)Æ›)%o´;ÿüŽYd§CPN`%°”¶˹)ã¼ÀòzÂS’XÃ2´eˆA):‰1Ș(°"Ä°´Ý²Ä0A“Ä‚:¹ °´‚Ú(%ò ˆ"²Fài7y(ÝK<£'“I"/€§ƒ‘xÖN>vZâ¥É{ï¢Ä#Èâ’ÒÏA§‰”ă,dcx²Xo•Èƒ,&+ñd1ÐNx²Àº¢Èƒ,Ê+-ñ’\ŠD"/§bˆ[žQnrÑZ%ñ´žlÓŠ<¬(L’Ï@¯\yÅYe$ž…,„?‰œp6Ä(¬VëÛ*«%žƒ,Æy“/!€ñE/AVÐĸ$a€ a¢dHíD‘¢„&¦‰B° ˜ä&ò)H8`•Ïh ¬‚,Ž¼„VCÂÊyÅ&’°ÀÈ‚X Y Y ,Ж`-XÀ-©¤%,°À[RÚKX`¸SIX`¸6z’°ÀraJI ̵ÀN ,@Ë=HX`ºÖy#aìZŠ^ ܵ¤•„ܵ5EdKXtœ¬ŽVÂîZ­‚„ÜÅ22`V'Ø¡—°¶2™•„Ü5Q‘„Ü5.@ä… kÐJX@À]ã0ÉswSFÂî2^ÂÊüQb4Ü„89€,X àïÁ3VÂ46„%8à®NpšZM:&/aîêˆéyiÒÖ-´Š…÷ê0-RÌûôív1†s4îüêÇÓþúoùæ[Xà«÷§ï¾TŠÞõ¹æ÷Kywó;ï"»rM4¿;ÀùB{ƒW,t©Î»Î×®ðjÝU=—®´Jù"c­Ï¼&ÇZþØe­õqÙË—±h½”Ñ]8áéåujÿq)G“o-»ÆZåz]ôwtùšË~õó«¯O¿µ4ºœ-ÍF‡@ØhŠD7ìôX•±Ó_ÂâŠ[°+–jéno°ÎÃY*íãè'‰?¿~7¯‡à)RRÑp2¾o¡Çê<œ…ÞDŸ‰;»Ö&áÛožþ¸§BJÞÂŽ1oÇî[ç±:gûqè]šþ•{ú»WÜGðôFݟ׬óh–jnĤŸ$þ¼Ð -{`OoìýýÁ:g¡cÑØ5Yg±adÁ‰6Cevs¼g -­ÌÀo^ý>¯nÜýùûÁ:g‰7bÎ_cþþK¬®áÁãý¹úÁ:g•7ö]æ]½…}éyüYf·’×Ì6“û¥búíËê1Gö3fïÈ).å7R¿oåx*X7y›¿ºÃ‡S É47㩃užõª ¤\ÿ²Lš¢sXÖb •—û:IÓ>†`cž„«&Ð’Xu8Cú× #ú~»~ÝyÜ¡]÷þ«µ)æÓ:¼¿&ÕúÎOÉaî¤ú9¼Y…Ÿî@›ù ,rÞîhjo+tL*ê µ™»#É:0~²ù[éü ÝghŸÏfò1ÏÓ>ñì¬y9íS4Xq‘zìÛ©KÃlìöµNdœâó °È‹)< €cP²íµào\ñiG‰ÏYušWû ä# -¨ñ‰V„”äæCƒu$±lxÙ¦wð°¹ayeíÙĵ­ÅŠSÿ½•“±U²ÚÚ˶ší.óeÆ ÷,³Y](a+aÕh™%üÍÅ#î/9t¶êp×*S¨æ”?ø•ÁcuÍ*“Ö{VyëqP›ýš¼ž¯7t·zÄE«r|æ`.×yÓyÆ åly½-ïT®¹ý9?–ûr n wŽµ¥¹ukGÆèû-Rm>D³÷ìDëŽ~ìõ`‡³T¿ve9¡[ͧü`W²†ÅZ––Ûúh–Ìý–z¬ÎÃYjÜÇTµ²¨7CŒ÷z…—«XôåféŠ+ko–>xñºÒÆÏK¾õ‡ÆÝÛ•¾êƒðLW¬²ÇŠÆøI™`oÆ—ë<šUjµ`>û›sëçüÃöÎä®}Òn0™f*A®Óž{UìSûm9w*8hÁ9>÷/:‡c¬Ú|Þ,rcújRµ=–ç¬WG!-ó.4ÉÎÛˆ”Yzà3Ú®e]æݼ/ Ëæ«ñóÃ[6ƒÙg´G‡eSÖÔlmxÐÐ6X˃ƒºI›3·²)žX¤™nÚC¡a¿Á«ážj›èæuÉø.ªlÒ¯ÚßyðÑÆàê6SùÓÉçóu>sVù8iÇҟ=M–ç‹0òY…–t:•­ÏWjæ£üj¥ÊéþŠüåÃóÔŽ7¥üOyÒ<éDë??}Ì'q«LŽ¦ÞtòÓ‰Ob'LÔaÞ 3±\ªÖéöD´Ò.©0¥uId˜ª\öǦœvΗÆÕÑp}W«ñ y¤j!›Ïˆ›;ˆ¨éõ–Ì£W“.d>Í>N¶Üù& -o°6boƲ½Á^ñ&Y[5Ý–¨†î:ÙP•Góé§Mu¾›è£êô0ýµ¬ Ãtî/'tk×Ô'‘Êghצªvj4;•ÕÕ’mB¨Í“@ºœâçª6M˜ÌÆfO"‰±µr©–H—“X®´æÚL…V*öª©j¾‹ÒMÝƦxv>Zøç9†5¢l½\’ùéu›k»1âåCòÓÉ¡´ëX¡\5½‘h‡v9ªn/bU–f¥ñ9Žüÿ ®Qyò…¦fR|k¿ëðÚl䥒Fºo¨C’4…^vP—¡Ùì¨/•PžA·•n[P• ôªx R <–dݬ®‘ª~VŽDU…@wìš)?-È]N×–ªS­3¯sw­ÝNö ;ŸFrh3®ù`Ý&okw°ùJ¼dÇFk2›~¬t²QÍÛõÎb  ÐÉFŸ2¹®Ö†NùÄŠrÙ™äq/§NÙÍë ?EõVó9ñcä¥ÏÿA™¯ñúç‰ÎßÔƒIg?nTC Ó°uA=ÿªHxދȾ®¦6hoÔ§‘ÚÆÔx2Õæ[¿¼m>Ç懧uñv µ¶ì°öl rúÒè¡-nF¨±W+ÜØ)UÚ¢Óê(\€¦*­µªL¾T«jÉ#Ž¢–B‹):õ‡QÓnœ•¦D7Ίߔåv«ÇkÑ’mÍ9ôÑ©^²Å’Q ù±v£R/i+mÛ7-û®Þö< ƒº ¾G¦ªWs›’Ï¿笹Îj'B*Uÿòý(o8jÒÁ›MEï!Ä?°(Û%w×Ùß?Ùî’Ôý“±'LÇDÖª ùwödžÀH·CSõ7øäjI‰CºOÂöP‡9µÙÊ?´”÷ó&Q» œ*òïªYz#ÿLPZþ9o£ÄÅöIþû/Q“Õr -endstream -endobj -586 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 585 0 R -/Resources 4 0 R -/Annots [ 587 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -587 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 292.623057 101.433071 273.873057 ] -/BS << -/W 0 ->> -/Dest (cb118-1) ->> -endobj -588 0 obj -<< -/Filter /FlateDecode -/Length 4632 ->> -stream -xœí]]¯ä8}ï_ÑÏHüQþ’H€Xb¥< îÜÙY„î"-<ð÷9NÇNÅ©$î¾wf²°sÕÓ*—S±ËŽë«ÂßWÿEÒCŒ)xº>ùᢆàþ)£ú|‚»jEap:ùè®ßg CHÎ}¹à¿æ'¹üåg—.môÕ )y£Ýõ_ßfÙ!Ä”T *¥­ rnÁ8/¨¼p‰)I*¸áZ«Ü ¤‚dLTnX -Ú®U n˜ IRÁ \TZÁm”upDY#è´< -ÝK:£“I¢.@§ƒ‘tÖ>vZÒ¥Á{#øâ’Ò’ÎÁ§‰”¤ƒ/dct¾Xo•¨ƒ/&+é|1($è"|º¢¨ƒ/Ê+-é’\ŠD¢.A§bˆkQnpÑZ%é´°iEZ*ÉAgà‹W.‰:øâ¬2’Ο¤O8bZ«É­Û*«%ƒ/Æuð•/1€ñðE/QZÐyIâÂ@ÉD&Ú"E‰LL…`%0É äSxÀ* ÑX_y‰ ¬†/„–+êà‹M$q5ð $.°¾²X°--XÐ-©¤%.°à[RÚK\`A¸pSI\`Á¸6z’¸À‚r¥$qçZp§Ä¤‹æ$.°`]뼑¸À‚v-E/qïZÒJâïZ KQ_`‰ ÀŽƒÕÑJ\@à]«U¸€À»hFFâÔêz‰ €•ÁĨ$. 𮉊$. ð® èD]ЭÄÞ5— õÜà]㔑¸€À»†Œ—¸@À?JŒÀ» '‡ð+qú{茕¸™ a‰Ä¼«º?¡Mƒ­“—¸ÀwuDõ‰º4è€Ú-JyÐÞ'‰ x••¸ÀwAŸQâpü€ÕH\àÀ»Ú¢§–tà]m’’¸Àw78‰ ‡A#¸ýÛ  +êÒ RŒ8ð.hÉH\€^xPL"ê i‰ @ƃB*qï¢{Oxð®rÖJ\àÁ»Šöˆ:øb£‘¸q th¶B›F°:(¦tà]¥QÃ’¼«THxðnî6%.ð! L”¸ˆ‚Ä>ÒBðø¤ CÈ'êü€c'qAP8P–¨ƒ/HâÖC²)J\ |±ÆH\ |AÈ q"‚!iDßâð¾.âø€à "E‰ ÀdCLDç1C©J:o†@×¢.B§ƒÄ!Ѐ®XK\€±ÔÑè%.Á9\’¸n@§Iâ‚à šŠÄ芆h™K: _dK\€H:­%.ˆ¾ ”¸ šˆQ¦W ‡ÆXE-éHåqf”¸ ’r¨(qš Fš`JQ_@ ~ò`ÛI\€Á›æˆ:øbI\£ÉÃÍ(qAŒðÅ(#q°™Ç›A₤à ">‰ ’òC“%.C§”ÄIGŒ7 I\€v™Ç›Qâ‚dÆ›â í6YŸÇ›Aâ‚DðÅ!DuðASÚ-8 ãM°“¤óðÅ$’¸ yŸÇ›Qâ„o"¢§boqº¼ë˜IÒw™z‰ p€ñ&"[il¯@¼. >“•18£8c @½}«8e À½è­‚8g @¾yŒ$NÀMŒ,£g ЭB©•8m À¿ˆÇhÅ ·Ù'>E净>3†|‘ Í‚Ý×1ÇœÌn$»Íýë»ËÏŸÐù^¿û÷åãÏÆ™²ò§ Œ© ñšI¨n‡è[ãò­GEëaÒè!|Ä&;Òì¤-N“nÙItùßuýcá9ÿf<½­§OèÅo§GƒÆS®åˆáë87È<¾Óä Î'Aoë‘Æ&“Ù C\©h‘øWï.êäÅm´2÷Ñ_b0LÆ]ß}ùùï¿þóÿð `ôîã寿Pàè_~¥ó÷3>öö›â$ó·oŒC•Éz=§¯iÜô Ѥû8ÙÁÆ‚éÕGüŽHóþöÙÖ,ýª¾3[§–ù«éšdü´ç(ß5ï§[ºRÅ?zžŽÓü»æ1‹žØï0—]õ³\k|üÛõÝ×—_¿[BoŒGGèQ&¬˜çÐÚgò´~ -bp»Ò¶òF öûy®ø*o@¼°KÑû[EQZ¤Øª6žçíóPTÅù¨9o|ÄNý¶Íõ}ÀÇäÏ/E(å¹õBc[¦Ht€¥N›³ ã›=4Ñ·¬F\S꜎Z”´ˆÒ¬FK~I¨õË|²]¥°4×¼æ(ø¸Ì£¥¼ÖßѶ¡²šWƒèL§•Ö¾óÒnÝB‹©-­¥ÈÌ®Aj{C¨ô½ò£åA·DÒ˜¦E•cç2S«¢¥.Ëk‡]ø·ê6ÈC¿ûlN‡Äƒ0±Ô2ëkM½øÈls̨۪崑W©ñ¦W^èŠ]i5¥‡lyªAFŽzWι‰Ø]¬w_ ·áe]"°rÕÈÃÀîqN§Íé»;n·ùs£úÕ¨b×Ò"y…`‡Ò²œZ4J}At=/¯£–ǽ¿™cN -q@Ǽ?@n§ÍÙköãÔ:¢hbÁq˜ÉziÞÓE9XõÔmO^ÖÆœB-ŽÑ…]ŸÏzû¥ÿ«x‘MA,òj‘ïçt»nQϦ6‰G½ŸK;mN‡Èƒx´™©¥Ç{¯­ -ã(^ê‹žïé51Ü—çóyídИæS"•ÅžÚ IE“§Í÷‘Úgs:¤îÇ«=H{ÈŒ9¯mñK©ßÔ¬Ð?ÓåÛQË*Ù”drƒçG¨ì³9*bÑ#Ý[o÷õ zcTôã@¹ŒL«îçËN›³!ÓÄšo9ré@ù]£›·F4CGîqÏƒà ”Î±d÷ˆ¨Óæt(=ˆ??×ØûS ¸AÝ#Ƚçßúþ‘u÷Ë;mN‡ÂýØRÓ„º§YmšJ3§ñS:©Æ…1üᘙÍÂlÍo–.3ï«óîÜÿjùOœaÚŠJ à|ÚD'í\íÖHƲ¹Èîž¹Ïæth;ˆ¹ÇfT΃½-G¾8YF(±80?~†‹ôG³B6gC(=~?½‹Oîá~Cþùßlˆe1a^ m¢6î±}6§Cì~y7òv¾·è®½g”ó¹Z\³¾õýubñ¡3ƒrä3zöÙgs:Dþxﯟ¶µìŽ¨6ï_[Ôis:ÄíÇ•[÷rÄÙ¾&èéñµ’›÷Þ«õ*Œ7Š -^·ºâ­#•ŽÝ¿î!ï´9*ÝÁ<äY9mgÖ…ì;z÷OÒJ®ÜâGÇï[w®xë´9bÄÓpÆÿx«x w¬Ëì´9J;ãÆR‚¤V—¬ÖŽ3”­VõJ3'̬V:(µûXŸ®þ= ¾6k:õ‡×£nÅ·Ÿ¼ 7PïwÚœÅsœ_tí—¯Aîýó™6gC£?˜Ï¼—_±ã³ðw‘ñ»€ ªÇtwäËŸª[=*Yò`-ÒMOØÙ‰£ëPÛ'÷:ïy67éêY“:º#Ùis:ÄîǪíõRí~ä= ½pIÇüyy£ ¶~õÄ¡žÍEv#°Ïætüòqhû,ä -;ñæ—@™Û¾®ZãÝ÷ÿ79LJùåsQ¹£ûæ6§CdçüæÞêˆü—0}„g"wcµö£0Ã]uí -ã­¸vãù™UKró5¼ÅÚžðÀ:ÈN›³¡&üôlø™í³á_ºtÎS6çØ;Zé´9bš§\´¬£¼¶PÙ±ó1´o ÔÝÿ¾N›Ó!ôàYœ¹ }ztMhµº)óÆÿcËØ@b¼ÿžN§ÍéxžŠ7^Ñ£óX‘ù""ó©A}^\V*Oqïj³™Ò¥OË‘ñwuÚœ ™ñྷV›Ïl÷¬õî®}?Õ0ÿyYÓžnß[5_GïÕàç‘Úê :x«bñ²ás–[OCH£µ-^ÜBßë!;mN‡¾Îõm›>åšsÅ™m²½òm…<³Ýis:öʼnŸkí6_¨ý×?n>ËòcŽC7Ðïö°Óætè|EìøEyå [ÎFœ»ºWÒÌz×–§SË· Æ)ÖlcÊÒ‹‹ØKêþU>6gÃ^ÚÇr ÓiíKG wzߪ´Õ_§žÆî´9*¢Æ7 ã>27Ÿl1÷¡àÞ÷jmÆ -«¶oAã÷#Û{|ÜÖŒíï*xš[ã&"X Ùis:Dö½Ì|¼'ÖÞ½a‘ÑŠ»ø=AR‹ÑñÖêìÃÚß©aqôÎ#·¦ÕHÏ_t½TœsèkÞßÚñÄûN?þÀ³Ü6§Ãç}ï1ßZQ{üÔ¼:œ1ªy<Ëõ½X{/–$\ -ëÂ?)Ö¸?‡ó•9¿™ö²•è\ÞK(:ÙÖŽ‡IзÞH£lGqÍ-¼ü„+paÈ›ØÚxEXH)ä}_2œ¾þÍï~û'§ñµÎßÞ–ÝÂíÛ«Ç}üí8wŒc‘êÞ^M± 'J«-Ý­VÓÄ`þ¾½Ì1ÃôŠK¨rZûŒyM¿Ç|b §óå7Ù>¿l'Üô‹üÝtnæϘ‡c×àÊ5¬ªò‡‹÷0Òã>vÊ#À2y£Û«§ÁæºI×¼oRT–t©:=¶¾nXŽÛÿPSM;«¼‰`Þ@ê¶0;˜Òÿ0Vú÷hÈåŸÉᶻ§ÅÑ”ƒYürÉ»»’ &L§8 §ŸªæEº$¬²gplR›ÒºAM?SÍܺòÓ¸r5ÙÞ³¼ë©’ÈŽûÎÜN,aéõZœ¯^ zçrã`§#_]q¾ø áœ!ñ´s†³á¡X[Å„n-Tìt³ØPñGçÕjÑùYX]çE§Yõ—´&°êÜ>_pZÛJ_DiÞ—³dUJר*³Ì©T¥®¤¬BµžÑóE~¥JքʬzVkyw³›0 QM—J:Aô|ÓM¹¹ZS¡¦Š³i*%?»2CÝÆZð‚l,£Iò–߬([~.Å>×@©knÌJø|AC+ñËÅ!µ›¹B¹=.´,ß*ŽjÆË,ÍCÑÛ/4Í"Ë{CåÿoTáª43ÏøCS…TÞßm>šO’Ûf/ ‰Ë}e’J@S˜Ó²âÒ!TÌòòÒQ é3éÖÔµÄRVdLRq/“T%¥XWÔUq¦°R>•+¹P'pºYì*”_âÙOW›*À^Næ5ƒ»«ùÎb_¹ó…‹C­q7ë«þÖ|æ‹ðyìبgèÇr’YlTííæ“1áìHaÏ4ú2ŠKk­ì4¾ï}ú972©Ç}¾üý2vsÆ:“ðÃ[÷žå‘—¾þi¾Æçºþ¡lvvëǪla*·.ļ'Ë;•ÇNqÞ¹¼k³aBCŒög±ŽsZWÐPûqc_Ól®fb/)ueP]µ\èæ2dbS¡k´e%ãv„ÏM)Ö@æµu3wØ·C[.§±y›¼âøåà0Î:;wÉW–%ŽrÒ¹K‡– 3Äχì„iöláf$?I-úì?ñª°ù\쀅ùÐÎ:š»Yžu‘WÁ’ ¶ü€gLšŸ4-üœ~¸y½ºÞ'—}iYZ©½J_¸´^X–V“¥v<ôËÃÚçØk>(w.¡š—emÏÖ gnshA‹ŽQ&³Ù”Àñ“R‘-NZ:ŠJ©"«9†â“ŸÌJ±ŒWÅR -5¦˜¥ç%íx­ÔBt¼Vü*mηôx5Z²5rÎa¾:5§¬±ddž[W)Í)m‘­ÏMËs—ÞöÊ.ƒfGü™ªÙÌ­R>Ä´›M3!¥*ã숎óö è¨Ikì5åp'z¡¼iw È¦Kî¶ÙŸ?YÏ’”ù~&TÇ@Öª · ^ HÇ -–ÕË´³ªl–”xI÷€0=4ÓœZÍÙÁNS˜ã$Q= y¨èҸŶŠ1o1”§‹Æu]Ëé“ñï¿ÇçÙ8 -endstream -endobj -589 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 588 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -590 0 obj -<< -/Filter /FlateDecode -/Length 4360 ->> -stream -xœí]K#·¾ëWèlÀ4Åø ²ˆƒX$‡ ‡‰¼k#˜ °Î!?_±›n•Z-íìF»ö f¤®"ÙÅbÕWU”D™£Æï×ÿ•RŽŽ§w‡÷­¢×ø)-…ý=þ¢?MQy“CòÇwLpQÅìm Ççþ­.Çæχ¿}uø÷²9Z•s°Æ~Ã,r*¦µX,mœÀò^Åh}XÁ(L1g‰1|$Kç¬1('/± Y›V‚Ž¢qç¬ 1l4$± †É> -,£!‡±Z‹<¢‰œxÆ«¥‰g -)Û,ò"x&Z‰çœ -‘Œ7/«‚O ‹ÏÚH<Y¼!Ò²KQâÈâ‚Ó"²ØÄ‹ÅB;Yà%ÈëJ"²è ÄËNùœˆD^O§˜ÎyV{å“sZ⣋<Èâ¶ÏA¯ÄNxS¼Õ²w;íŒÄóÅÚH"²`ñ%°²è$€)Æ% lŒŠ²% lrŠ% lÊŠbt Øì…%pÚ€gNCOABg ÁsEdq™$,p² „ÎAKN´%X‹„pK: ð–´ 8.ÄÔ8 ®K$,p€\˜R–°Às°SÂÐ…»G P×ù`%,p€]G)HXà€»ŽŒ–°€€»Î¡§Èƒ,0` €ŽÊ™ä$, à®3:JX@À]¸‘•°«ª`‡AÂØŠ²)i ¸k“& ¸k#B€È‹ ->è$, à®õ˜‚¹»Ök+aw-Ù a E´˜ w!^N VÂÄ{𬓰ƒ)‹´DÂÜ5áOði •2) WEŽÈò±=1²¬]Uã}»®$Yoîóâ²¢þEÉê‘p.ï+ -y­ñË.>›³íežOÑ Ô€ËìâI»2×»¼ À|¤Zm\¶š€7(*4-ÿöõ(5K1m€·ËA|$KQÇúãëw‡oþôê¯þî{,æë·‡¿ÿF£büökçéÑûéÑbÞ¢õˆqDKÓéí² ¼Ú®õ¾P˜iÿÄ_:¯õyÓùÆϼÚwÕ¯È&Uç4ËXû¹1¾OK™Ûýt—ÓÛ.{¯Ý×.ÇokY0ž×‚.æçôtÞG;<þãøúÕáw¯—fX -“Û,w_—±Üa°9o,+u\¨¶Èi¹ØÍ„Ei}«ñ®¸?ä2ÈQLeÈdŒf ¹É J»g•´‰•M›óJ›a¥×÷eã×:Xº?}¨G^°Ô!µäׯáR᪥îëóp–z% ýEàÕ\YÚÿÇ/XèBjR%X_ðûú<œ…nç[8uVH+þkfðž&[§Õ·Wó;û<šuÚ”~¦™Á‹zÜ'ˆôvÈ/÷Fú}ÎR¯ä¤Ÿ%þ|ù‘Þy¥±*ëdùõ–m Ý×çá,ôJ.z'îlZ›„o¿Fúý‘~ÜÊäwo¤òŠû¶uîëópÖ¹‡Þ]÷~‘þfû‘ÞéÛkú}ÍRÝ•œô³ÄŸò°ŽôÎÝ^Óïìópº3M]“íŹÐW¯h}Èó -^q¾W_$œWàר~[Tww¼¿³ÏÃYâ•œóK¬ß?†w}Šžn¯Õwöy8«¼²çùYbÍ yÓÎ>¾Qda‰OƒUo¼Ñ¤É_Ç©ÞråUNºãµ÷}ÍJi;Ï\¿-§i}c¡c x°¶ Ú~ÏŠßûÖ QýþÞî¯hÖFë»Sµý~ \â1­Æ»h¡=gÜo¡ûú<œ…Þ¸ç9`Ú Ïpë’%nÕË;pésÎ1×Y­­õBT§;^kßÙçá¬q_®ù"oq4nkë®ïôh}VùìâÝylEiz¼8ÎzNR¾¸Ê_ïgl—²_ÄÊtVîëópÖy%ç\çb¿ÈzùƱ>¨"”-Ò¯“ïÜqßÙçÑ,Ò_É/W+Ñ´º‘Ýß…aÞ‘Žïšk¼5_¨„Ö˜Ùpqô®³üï¼Zúx^{ÁZïØÓÜÙçá¬u;×ül1í’5ï¨Å×;X Oü¤tÁ:{i«]ôúšuîëópÖ¹3÷¼½K;4m•.äbŒ%gùkÆ«kÒ>×ÈsɪÔÞu.írøПŸŒ36•¹¼úýÿð—ñ“(4Y§ráö<²•B¶6Ÿ¥Ì=v°ŒS[ÙCë\§ë3úܶU‚4Ö묫©ý´ÿ:ä¼óž¬¯} üôœÂ2‡÷nºGiçõr'KO2´ùÌ2–û†ªë]Êu£r—qv9÷:‡2ï3ûu¯ƒÌ¡ó˵›ÿüÐ~¾f¹«®«n5tc½Ž24ýÍíYçú4y½ ótÃbž1ª6ÏÓÍkîßc^'^³ÎÇ Ž?EÏ'5£çyMs.óÞ#wIÝç²Ð¯©|QOì -šlÈÂØ“í/m²Ûr×5ëÙñãiæ‡Y&_#L_Û²fvÖùܶÌ?Í6|ª9Ç|-Ì£ØDmóv𧖋L÷›r¹]„³¿LµÏcj¿©{·ÖOëÍÔ¿FÚâ‹OÕ¿f›°Ëy/|Ø s®ö§ÇþQÇÙÏÓ€Í_e]3Žùj3Žh·Ôáz›L¡ûF[7ÓåoéɯõÔæWöqÁž–¸x ÷âaÇÂæg~ÐKæ§ñ;V ºÚLóÛÔIu²À´m<«v]æžf[0O_½¶ÙšWB­¸œ—•døAΨ’ó*¸òñq> e¦½šQíìs×çâ_0•òýÓ»Y%ïáÊòayYÖeš©1¤Ó"\¼aH.ùt©Õ 'âšfÔ»nkß}ÀEÆõo¿½Ò›R9’-„KR]éïƒÊk'õ/ Í*õ;Æ,§’nCSŸâãl;æO®=ÄÇ°Ü¡}>€Ê ñâL¯ôçÕV8ûrÚ§dáq‰nlÛ©>Œ½}¯uI ãúåEA,8Éa0‘Erè.%|ç i™ùð¸ÕM‹·_ÂlB5>¶”¼W_ÖØÙçAJˆ‡|<Ò¶p+{Ö–M\ÚÚZxÜXü¯è­ýXŽ­ÊÕ6F\ŽÕlw™»,3G³e™ÍêâšýœJ–y–VI*gŸ¢YÝpe•ßÏ'EŽÞó mÉ›48ÔLñj“+–W5s~ÎËѬŽ©Y*CòÑà.Áîáj‘OÓº¤:ûfrn3gm6LUŠ =«j;ynSU>T-cš3þšuõÏÙl©4òD·­š@# §éÙ¯}š—íT³ëÕøK›C=×"Ÿ-åûC('Ùr:¨IËLJ)ÇküH|ªœ#S—Îe›ã…žåÐõ÷­Õ|»æ£YùX¾éöábnÿ¾,ú»ƒÉ¤Xÿe+¡œ™¬ ±Á¯ÉÏ>3›2Q߸BfâüT·±È´íJ;_£Êë–/=?í{ ŒhzÎRêl¸¿¯Ýø,3Òµ‘+§yM7ˆèÌ9™g¯•™É|îxRn¾ -MÎ’±HcÛ>`ïx•lœˆþœ¨‡Ûu²¥*ás*›êB'6ÑGÕ™aùk[‡åÜ"ž¸­[SŸE*Ÿv\‡ªÚµºÑÜ TnT_[¶¡¶Nétˆ_ë:4a1X5>3r"¦jHÔÚåÚN b»y4ßV*¶V©wÍUó]”nê.5Å ´Ó°Où‹Ñ®>]’yÛ©­µNƒÝØ3âé€CgäçƒBkß±Bûjz#Ñ ã6rÒÝ^:5¦ª*¸f¥ñ‰{ü‚ -ߨŒ<剡fR|jf¿ê7aßl䥒Fzh¨C’ ÅÞvP—‰±Ùì¨/“´ÐžA·µn[P• ô˜«x R <–dÓ¬®‘ª~VŽD]…Àí:Ù7S~^»œ¾¹ª×ífÁ æîÛ¸v>äØVÜð¨MÞ6î`ó•x*Öd6ýToÒÉV·h×o6»`…Nî0ú\ÈÕ[:•æ§Ýɤˆ{:üt(aÎ:oy+$ ÐŒË£ìÌñ¿hó -ÿ:Ðñ»z„äÇ­nha¶.Èc$ãïH;Éü}•ØÜf Z`¿“Mêm}µ†Ç­ðšzwݽ¶4 AMãŽDßu8m3]kÜ ¿A<­´Ø™]›°§KW'Ç›:|øh• -ãtp™:Ïõ|†Ä7í!\²CG\Œƒâr¸aî’-ÄdƒoÒTÏòÓ¸¼%1^ i_ºÎ£fÇ¡+­Ìbh‚ØãÀdÆ›æ…<‚Ðw»×¯û,r¨ÀÔíú> -endobj -592 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 217.623057 101.433071 198.873057 ] -/BS << -/W 0 ->> -/Dest (cb121-1) ->> -endobj -593 0 obj -<< -/Filter /FlateDecode -/Length 4598 ->> -stream -xœí]]¯Ü¸‘}ï_Ñφá÷°ÈC€l°ƒ$ÈFò°Ø»=N°¸`²ùû9T‹d‰*Iì¾×¶œ/Úݪ"©yxX,Q¢ºJü}§ð_´JĘ‚·×ÛO—Ÿ/R'ñoJ±8˜Ô?àÜUI„SÉGwý) L!9üõå‚ÿºCšüåòç_\þ¶Ì£®Z¤äµr׿ÿ˜UÖˆCŽQ%¨¤2ŒÊ9‚vžQy%p‰)q*˜á‚Õv­ -0æè8Ì°ZGFa†±A™µ*Á ”åT0C%•’°Ci)Y ‘ÖÍ蔕î9VÂǤ« Щ 91«œâtIxï]ät¶¸$§s°Å)k%§ƒ-ÖÄÀéÇÚÃIèàò±:/pì8.Êbu°ÅËqk‘LŠ [ŒÖ [à2p\@$ï›ÀÀ…XØO‘ã0™ˆÉZŽ ‚ó˜¡V9×"Ð5«‹Ð©ÀqAV`(V`.%":=Ç x‘Ý%Ž `tÊr\lAWá¸C‘ˆž9§S°N6Çðt¡SŠã‚¨a ¼AŽ ¢Ž˜ezÉqFhÌ%QÕœÎÊ<ÏŒDëEv9.@7ÁLLÉê` h‚ã´Ožl;Ž 0ùÁdÓV[Œ³Ĩót3r\#lÑRs\læùfภIØã‚$½Èn2Ç qè¤ä¸ ©ˆù¦¶ _æùfä¸ ‰ù&¦xL¿MÆçùfภYØâࢰ:ا)1ýœ„ù&؉ÓyØ¢“å¸ yŸç›‘㸘o£gC1Ï7.ïºfâtà]x¦žã`¾ Ï–›ÛK¯ ðÏxeÌ3ÎÈF $¨×aleCÜ‹Ñ*°1 òÍs$6h31³Œž`X…RI6l Á¿ðÇìŠîÑ'¢ò€CÀ˜C¾È„naáÝ×9GKf6’ÝãcÿËå—ï1ø^ÿò—O¿˜"eåOÁA˜R[øWè:¡¹¼o…Ë7 ­Ä$´“…0ð™<Ùˆš‘ -´8ݲ‘þò¿ëúÇÂrþÃtzSN(N®¦ÐrÄôuŠ ‹Ìò†KLŽ0Úz¤1IgväàWJ»Hüëwu¶â­‡Ê¸b>8“a«ÝõÝO—_þîû?ýá÷?Fï>]þû?¤tîWß©ü­»ïpÿ–æþ Ç^>ïï¿WòùSå¶K—Q7êãœçCÓiMÒ™ùóqþ¶óï\~jéì°1οçk°ó13_GI›ý2m½þ´–U›dWf9oèÊ%ý¯þçúîûËoÞ-¤),qM™›v±7–å5Ø{CÈU«Á »˜3K¬ÙOËz´s¹Û\ï‰è>-Û/C'G“kÛ…eYWÚ³¶ŸjzûaÝÊQÚï\ƒ ¾ô÷±õí[iÃŽíéd /=m²Ï~Yûjï)å|ìêË6¿diÓ*›Ç¨Á/Rsèdæ9*÷½Ë=/­ÉÙQ³ÏÃðÙ* Eíø<•·Z6>Á7cyÎÖ²úÀ‡+5y#£dW›¹ïèÛ}t«#gßzŠ´ÒŒˆœÇä¾ö ¿ãÜGŸÍÿÌiüi4oå…¸ïkUÔvüâÞo ·pÊŽKù¹ 4×ëÐZ{µ‘Å©nޘ͑Ž˜oPàt0ÏépºïÁ­ÆEÚlö·KÛWy‡cnìé=A›Öé½²Íñ¬›£ŽŒuµPÏ‘é–ôúmºëûx÷ú”æYOIÚ± É`žÓ¡ißËÚόԣ¤G”"-JǶ¾Õ?-Ë™fˆ…Å™åR|Z–ѳÞj>Àùð¥¬Ñ™Q·æ!GÞí1µ§õ,ù‘™kÝ–¶åúÝŸ5Õ"u,Ïéz×ë8­GÛrSûirk´¬ÎŸ¹U´gZºoE‚î×xa“ïmîå¾WvlÍ‚©-=ª9×S³ù $R¯/ßqŽÓ]Â}$Žå9ͧø•c•GºQy¡+ùJ¯é¢v«k(ç¼!¯œs±=ºÈè¾@oÇË£ÑQÕã‰Oì—ö¯üE…X”µs—eÁ=êItãÔø'¸t,Ïéyàvq‘Z{tôÚš¡Ž¢µ¾ùÞãwd:ä­¢™ú3#•øžJ‹$£Î+nö‘:–çlHµûþêÒ¦ò#áAÊk[üÒÝ™¶;#ðó>ݹQ¾íµð¨´$&™œpÓäü•ƒyN‡Ê_ôE¶Æ[ù}#¨Þ˜}(ß@¦yœ/󜙾æ[Î\PþÐìæ­MÐ1­Ô9 ‚7PÚ|ÉáÑ`žÓ¡tlãgŸ{w¨{¹\ã[ß?²ññyù`ž³¡Ðíû–ÓJ’²Š–‹jÓÕ®~Nǵ83‡?œ3“(ÌV|³´p‰¼¯ÎûäzŽCO ôʧwÒÇj·f2ŽÄ"GGæÁ<§CÛÏøÌ8Q9ö¶ýåüä „šÇ‚æ9BŸ¿Ÿ>Ä'F¸ßþ5{Çb‰O˜Ÿ£ÔQiw„ر<§CìÁŠÆG‘÷žSyh–ó¥z\ç³¾õýuGüC§…tÖgôì#r,ÏÙé¿Ýûë§í-»3*q^=¾¶h0Ïé7öTL/‡†Ð5AïŸ_+¹yÿèƒ\¯Âx#¯àu«+ÞÚØ@%¹=!Ìs:TÄ!ÏÊi;³°!d?0º–^Òqå&?ÒûÖƒ+Þóœ‰>âi8ã_¼W<3Š?±.s0ÏÙPýÆRƒV®ž0Y­'([­êå"'OͬV:H¹ûdNWûnŒ­ÝšNõñõ¨[ñíg¯CÅA=>Ìs:Ä8¿ê -Úo—¯ŸAî&Ÿˆgæ9â™òã+V`|þ.2z°Cõ”îréSu»ÏùÞÚ¹§õ©3G/Ö¡öOî Þ# -$6éÚYYytGr0Ïé»ï«öÏ×r­ûU÷ž…K:bÏ3È{MÈ;`׈H,rcyΆÀøõýÐþYÈ2vüͯ2·m]õƇïÿo¼ªù‡ù½ÕQYëŽî›æ9"ã›{«#òëÃüaž‰ÜõÕú;ŒL„»êúÆ[~íÆó3«žäÚ5¼ÅÚžøÄ:ÈÁ<§CÍ¿Ÿ _ ³6ük÷€Á8e$1Çñ×ð å9bÿ§\ô¬£²¶P9°ó9´o 4>þ¾ÁiÊ•ò¾“Ü\÷æžî2ä´þþ™ÊšOåÄ2ߙϗç69~5Üõ‹òÝ|nbÏT†#×àÊ5¬šòç‹ŸvRÓ.qÒG¡tÞFöê­0¹mÒ5ïJ¥±ª4š*[]7rN›ïþ\SÍûñʼE_Þžé¾Y/9˜Óÿ<5úO•¬ÈõŸâ¾w¦œÄQ—ƒ&~¹ä½S-#¶pÅY8ÿ”µ,«JÂ*»hƒH}Jã„œ¦ZxÞŸãþS»r59¿+Ùòž6V–DfÚÕå~bFˆœ^­Åùê¥P³8ï?…™|5Åùb3„­@KÓ¶[ÆC±2’ÝZ(ÉéšXÛbÊû•ÕªóMXM§U§Hó—´:æÜÞ.8­é¥/¬4ïzYŠ*µ«e•bTªRWRÖ±µÑí¿“¥h‹Æ¬zÒjyï°»0 Ùš.•tŒèvaÓÍ¥¹ÚR¡¦Š-k*5ßLiP7±V<#›êh–†¼¡6é#Ò”ŸK±Ï-PÚZF‚½Þ.bìJürqIíWHW G…†”[ÅQ6¼4i~jùþ ]³ÈòÎKùÿ;U¸*ÍÌ3ýP¶B*ïžÖŽÚIr߬âe%Q¹¯¬c¹P6´´¤ºT³´¾T”LúLº5u­±…”T‘‡TÌË$UÉc)VuUœ)¬ÔOåJ*”Åœ®‰]…òËBÜìtµ«ìåd^¸»ZnûÊ/Tj‹«¼^µ·–K0_„·i`³½8C?–“4±–u´k'#ÂfH¡‰¾LâÒ[+;åí½ËÏÖɸ÷vùëeæ´q:á‡7*ïìJ=/uýÒ|Ïÿ^ìõ÷e+±û8®ee ]¹u!¦#YÞ<Šó¾àEX» jKh¿‰Uli]ACǵ&|m[vÙˆ½¤T•AUÕR¡kuHĺBW+CjÆío]-VGæµmÓìû¡)‡Ó˜¼ ]± -ôrp›Î´!ùJŠÄQNÚ†th­&q@ Å!9aj–-ÌÌ€¤'©UŸí·´)L>9 nD>4MgÛ0K‹.²é*H¸À†Ђ­¢'M {£Ÿî^¯n÷Ùd_z@–Vj¯Ò*­–¥•ðx©™ýò°Ž9æšJçm5TË2¤ï™êä´®Ñ\ »8 %ÒØ²Í =©-²ÅIË@Qé2YdµÄPlòs¶R-ÓG¶–Bõ)šô¯´¦m•Z‰Ž¶Š_¥Íå–¯zK¦:@ŽÂ9´«“-eõ%##ó4w•Ú–ÒÙúÜvyî2Ú^ÉeØfˆož©lÙÜ*åSü÷Lß¹çé"\ªò7EGTl›ob ¶*m®)»;Ñ{pˆÍ[bI¸äá<ûñ“u”¤ÄOè™ÐÂ#ƒÞ6Lx"+HQ/ó¾¥|¶$ÙKz¬˜ðP£9¹Šaæ8æ$ª‡!O]š6°–1æ ܃Tl¸hºõña>™þþ úó¸L -endstream -endobj -594 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 593 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -595 0 obj -<< -/Filter /FlateDecode -/Length 4598 ->> -stream -xœí]K¯É­ÞëWh`:õ~Á,$AIp0î] îÂ–Ç ‚3&Yäïçcu=Ø-v«%;òÌøàXj²ªšE²>’uZ%}VøùJã¿äô”RŽÁ/?œ~<©)z…¥Å⢰¿Åoôg­\œ¼Î!ùóD°qŠÙ›Î/'ü·ºäÍ_Nÿ÷«Ó?–}ôÙL9£ýùŸßËÙ)¦•X,¥­Àò~ŠÑø °‚ž0Åœ%ÄðÑwÍŠÃåä%ÄpÆ$• †uQÛkV†&j'± †Î> -,­ ‡6J‰<¢œ³Fài?(=H<£§²É"/‚§£‘xÖN!:íµÄËSÁ'‰ç ‹ÏJK<Y¼vNI<ÈâlŠ/@¬yŤh%^„,ÚÉ/AxWyE¥%^¶“ÏÉ9‘—ÁS)¦kžQ~òÉZ%ñ´ž<|ÓŠ<¬()Dg KP>‹<Èâ­2ÏB‡‰œð6¦$¬VC«Û*«%ž‡,ÆD'ò Œ/!€ EÅ AVÐD¸$a€‰qrÙ8 L²“K.I(`Rž\ŒV‚“ýäBŽX¥Á3Z« ‹wAB«!‹ÃÊyÅf'a5 $,°²g%,°@[o‘°ÀnÊZ ¼uJ ,b* ,צà$,°€\¸R–°Às-°S ÐÅrX ®õÁHX`»Ö¥ aîZ§•„¸k-zŠ<È–°è8Y¬„¸kµŠ8à.–‘‘°Và‡AÂøÊdRR8à®IÊIXà€»&"ˆ¼8a Z p×xLAŠÜÀ]ã•‘°Àw3AÂ8ÊüQb6Ü„x9€,X ïÁ3V 6¤%xà®ÎšZM:å aîêó‰¼<é ëZžtYÂÜE ²xà.à3IXŒŸQ„¸«-"µÄîj“•„¸‹¼ÁKXw˜4 ß&­àº"/O*§$aî–Œ„ˆÂ“J@‘'8’–°`<)ä¡à.Â{–° w•·V‚ÜUiȃ,6 ×€‡e+¬i$«“2@ -‰ÜU–xÀ]¥b–° w)lJX"L’°NÈ‚„!¹)Ç$,Y‡”Oä… ×^‚¨@ð€,‘Y\t ±ž²ÍI‚h ‹5F‚h R LY#ûËÈwëY)JX$›RvN‚èê1hUâ3¥¸y <%,ˆÑMÅZÂÔRS¢—°?Qº$aÄO; b†,X* MÉ 3—x² É–°™.xZKX dA6(aA2 UfP B£–„ª%žSTg& ’ ¥Š`™ ÒRŠ<Ș°ö¡bÛKX€âÅ&Ê‘Y¬w¤d¨ÜL¤YŒ2À7©ÞŒdYñIXU˜(M–°0žRdPo'aÖ%Õ›I‚lêM”xºÍ6P½%,Ȳx¤("² iʺ&¡Þ:I¼YLvä¨ÞL Í@½‰Œ^ÜHToFq»¸ë3Iâw‘™ pz™­TÛ+¯ÈÏdf¢Š3‰; -Ðë[Å-ìE´ŠâžøR$n@LT–)ˆ»«`j%n(à/ò1w… óîߢ -p‡ˆ˜™"M2cY8d÷½æÍìF³yìŸ=ýú-‚ïù¯ÿ:}øUÙ)k? Bií_a˜ s{dßÓ· ¡Ê¿"!|¤ óRë¦ ‰ðGÿÎ×o’Cp½QÚ È”nÔû‰ÊèE1êC1sˆ|wŸ›"GdùÜßhYÖ¡*~ß¡+IÖ»û¼–¬¶›5SSº­ÃBGÉ@.Mà®<áÎ.¯è -E'²˜€66B},eäëÊ-ÿöÍ P¥˜÷–û%yˆ ~óÃé×úæÿòço¡§7NßýF)¾þJãÕ¥ùÕûúš¯i´êcmÿ¡òãíu©ï±ŒÝ[´‡”î{\׶Î/û¾«}lå]–÷<$³žïÓéJÆWýÌêµÎ±µC±7ÿúùwA‡½‘Á\Œêûÿ?¿ùæô»7Kw*¹ûìø”Ê"_ ÞõÀc]>Æ?…ãå¼çxEi¤¼÷U‘ï†aÊ´õ÷ýêýJé7)¬Œ¬‡¡×ŽÐœ»¿® -ãyUynŒ×Þ“ƒ¯ûôù¨*OÆWÇÎײ´¹õû²¹õ²j[ä` O«¯E—¤¿gÜ‹Šû<›S¢¦ÜóJ×àí²´9d¸Ì,Î`±[ò²ôÀÂÛñÌ+ˆ Nïð‚£РsíÖתVú`^¾îÓtÇ=²ËÏxE¦6¶<ÒÞ’ûOç™û槈åÝÚ<ëã('•L­+CŠ7Ü*™î,¿¾ˆ XôY3rÉÃ>{°Ï³ù¬ÙÏ?¯Vÿ:ïk6ÜÈ׎ ä"#е -ó2¡´}ùüZ¾·Õwþ[[HjF>鲟¼ÑŽ6“÷½òXŸ§óÊýtmín¹4,Æó¾Û±«Ö¿—Õ¸é'ŽÐwéqÃ+ýXy¬ÏÓyå~æ¹Î(5³ôÚã~ÚøõÀ -]m‰~ÜŠÜðT–UÒƒ¯XRᦧëótžz#ýYàÕÆŠÜØ|ÿ¼+QöPËrH妬oEøƒ}žÍCí}ÏGö(Á_2a¥mx§½¿’?Øçé¼óåŸ_hfðª+î3DzËòË£‘þ`Ÿ§óÔ9é‰??ƒHÏòÊ£ûõû<‡ÞÈEÄ»îø%ÒŽôŽïeükçÁ>Ïæn?}¸îý FúG§úÔ‘ÞÙûkúƒ}žÎSoä¤_$þ¼Ò -{âHïüý5ýÁ>Oç¡sQþ,Pûû\Ö+Zgy^Á+ö—x_-ðKT¿3ª?ð—øƒ}žÎoäœ?ÅúýS¬®ÏÁ½º¿V?ØçÙ¼ÒßØóü"±æ•VÓÁΟYxâ[æÕ;Ïš¬Ÿëyð¿rúþö~°ÏÓyéç?×RXU -ëçÎLà '{Ðæóo¯øt—E½Çï‡ãÍ:Ãè}꡵?îK<¾~îpÃCÙÇ‚{è±>Oç¡wîy2Lã‘ð -·¶€K_rŽ¹Îr®ž­ßŠêü­ý`Ÿ§óÆc¹æ«<å¨?Þ·žµßzn~ñtyQýàÙæ8ë9Iù¢ð¬ý#1âÖ涰2¨û±ò`ŸgóÎp#ç\çb?Ëzùα>ª"ÜðÈ>_t°ÏÓyäüre‰®Õìþ! “°ð|?5×ykÞ¨„Ö˜Ùq‘¯.és'¯Z—ï­Ú o}`Oó`Ÿ§óÖWÿR1mË›Ôâë¬ÅJü¬+hÃ;G©¬môê–wëótÞy0÷ÜÈÞ¥šn¥\ìægÙšMú(4—<•Ú»Í¥_²ÓBè°¬ ­6©Ìå›ßÿñÿ³þ|>íK²\¸¿ïù!äësZÊ=âÉ9·õ‘Vi›ï|}E¯m{5èxÍNúšÛÏ{°,ï­û²¾õyÇVõy?Æ.|¯–»Xjæ÷yTÙJß0æ;V\±KÑþ!u®îïêšôeÆ%bV [vG34Ù¥g•k[=k_¯iMÛM -Ú1L³\–®ÁÚž´®.u¦¡N1ð¹%EçLY:g¸ÎÍV«û@÷¨–*§=\,ÂE½^£ê¼æ9—yïÉ´lC5æ²ÐY×Ì+Rü Ê™…±gï_zåðæ¡kÒ³¥×Kó *“oqfضØÌT׶eþtŸPeÏõ5Èó(>ÑÚ|`+ªg$óýæL¤¶kñÐW¾®ã¿òµìdC÷~­Ÿ2Ö÷sÿoiìù³ ë}Ù¼«X³97ŠóëøÌã,¹Oë×W®¬kB3ßü "‰²K®íØe -cmt»é!?PyOOq­§>ϸò Z"ã*n!â5öõå™>’0Ç8Û1G« üÙÇI†r­wõ–¯Ö6ºšj˲Îr½wÃÉ4Ϲ̳ͩ­‡×sZë¯éf÷Óªž¤ÀäxÿúöYžÞ°§ŸŒêt©#Ýÿ½œÃ•“ô,îžËaÉ(lÍÍî`Ÿ‡ŽðzÅäÍwyéDNïòGoÉ$ëÂP7xbÉÍl„Íæ€t–Â]Ýp,oIMûxíþ¸ÆæÉ\Da\ÿáë½]*§G‡°%Õþ>LÙÃvRÿ’F­R^`ÌòŠλ£©Ïñºów¶œ’J'F> }:+Ê qs¦7ú“õ€Ö¼žö]2XqÉ!µÙ÷SÏÙڽ׺qJ#€+Œ‚Xw˜ŠÚ<ëYd¢v+»¼ÎÞèøýL]¯ïZÖû€ÂoëÁ‘Ñ£§÷tpoò:±s(;þÝlrîš~®Ï(<ëÕé…KbHúÆ›°–1«H‡¬Jú#ݘïg=ê M˜³O[#™´×w¹¶i:e™_L5“kÑ­Ø FËQóL7=Kgö ,£Ð#¢š·Õv—–5­ÆßÉDûÚ©ùÊ”?žB90X—CãUH“6ô­2g”a–l“ÏtHqRÖéf:]”­Ï=ËwñüØ[Ñÿ ):±ŸNk.¦åµýÅè? HD€VsŒ˜¿JC2Ö¡_“_NôU*N £h7ÉD¬oUËé^ª6ÚA=NyÝSÕ·£ž¥hªjDh³¡þ¾u£#njl9äu¾±@DÏ ¯É4{5éJ¦¯£I“­W¡‹B©Ç€Ž·ŽŽ7ÉÚ*Fô×DÅn7ÈÆ5y4_ÞU±‹ÎU§™ù[[™9÷ˆ—nk×Ô‘J_‚цjÚ5ªÓ,*wªo-»A\·“@ºœâWª í`ÌÎgV££ÄgbjŽäz»ÜÚ ¤ËIlWGóÝR±·J£knš¢ W§r›va{>‘¾_‹­eÛÛ%™¶º­Ub~c®ˆ—BŒ»"¿œ|ZûÊ7×ãDËÆí䤆¿ *å8¿ÃÒl4:ˆ™þŸ¡Âw*!Oy£]w):L}\›ÐÚì䥒8=tÔq’´‹£-S—Ž±û,×—NJhO Û[w-¨LeŒs@ªƒÇ’¬»×u2AXÓOÇJNTMÜn}wå—yÈéûRõªß,hæî¾;È¡cç 'ÇnqM'ãwyû¸ÌçñR›[“ÉõS»É գݸ#Á -ƒ<`ô¥ÛjíèTŽ«oÇ"“"îåô·S sÆzCeg@R—Ç0ëó¿ÑæüþýäÎn'‹ÏqܨŽ¦cë‚Ì#}-X:H¦¯ kľlÑ8ûƒ¬Óhë›7ô8n Ãk7º«ì­¥îª;—ýÐ!#›îºF[¦¿C¼¬´Ø™µÍØó¥m“£šÎ¤oRá‚O—iðìÉg6$®¨ééà:Ã:â‚ŠKvÃ<$[ˆIÉoÒUOò;n -K÷b, K;xn„Y>t£•Y°&H-¿à;ÍošòB?¼¼>ÚîUäÐVQ;´wê §ö‰µžLµå2,/{̱¥Øl‹wh¨eÙÚ³=ÉKc¤nqÁ}”QÓèVx~S×h‹›¶@Ñá4Õh}ÄØd -µ[SK™qµ{N1¨ãšöÜ*]‰ž[%\µ¥q[ÄëÙ’í çîÇìÔhÙsÉ$ÐïÝ©n´´v}o·¼w‹¶g6 7 #3U£›¿jùþ=²væ>«©Uû)»#:ïâ@ v:ZcË׫ª0ÄÑ7dE—ØvÉÝ}ö÷O®wIÚþ ¿Ì19kUÔôE¹²/`äÛ 6Tû]¹[Vâ”îS€°=4`N]í•oJ,Ï=”M¢~©T¤/Fµ”ô&ú>·¨´üg~ÚF¹,¶OÊÏm -endstream -endobj -596 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 595 0 R -/Resources 4 0 R -/Annots [ 597 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -597 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 123.873057 101.433071 105.123057 ] -/BS << -/W 0 ->> -/Dest (cb124-1) ->> -endobj -598 0 obj -<< -/Filter /FlateDecode -/Length 4611 ->> -stream -xœí]Mܸ½÷¯ès€Õòûö bdƒ,`$‡ ‡ñØÞE0àÍ!?’H–¨’ÄîÛòf=O«Š¤Jäã«bI-Ê«ÀÏ7ÿ#‡¢wæúøáòñ"oþ%£úüz{•ÂøÁÊ肽~Hí­òîútÁÍ!-þtùûo.ÿ^Ö‘W5Ä蔴ןß%•ÑƒÞ ˨"TBjFeíིŽQ99àcäT0Ãz£ÌZåa†‰Ár*˜a” -Œ*À m¼ÔkU„ÊKé`†ŒÖ3*)`‡TB°:"ŒÑŠÑI;8tºãtJ.DY‡NzÅé´œ7ÒJNçœ œÎÀ…ät¶XiŒàt°Åèà9ƒ-ÚiÁê`‹ -^s:[z'2º[€®Àê`‹pBrº¨ƒ1¬.B'‚kv°AkÁé¤,°©YfÉyF§`‹6²:ØbµPœNÃNž°Ú‡ÀÌV•f·Zr: [”ò†ÕÁ >ÇÊÁáG˜ACâ%Ž”÷ƒ‰Êp$ ‚L0câ`¼× ¨hã¢çx@ ’h[¬qh [ f.«ƒ-:Ž ´‚-(ÀqÖ°EÍqÛ …ã º5"JŽ 4øÖé8.Ð \˜)8.Ð`\œá¸@ƒr¥Èqçjp'Ǥ‹éî9.Ð`]mâ¸@ƒvµ Žã ÞÕF -Ž xWkÔdu°æ¸ì8h4Ǽ«¥ðð.¦‘⸣:‡Žã`eP!Ž xWa8.0à]åáX05Ǽ«,.óÜà]e…â¸À€w•QŽãeÿ6ï‚B,À,Çð÷Ð)ÍqÂŽ ,xWF¸?fNƒ­¢ã¸À‚weÀð±º8Hgæ-zyÎEŽ ,xŽJs\`Á» ÏÀq8~€GUXð®ÔðÔœ¼+UXð.âËqà0HÀ¿ Rº¬."†Àqï‚–Çðƒ`VçIr\2âPŽ xî=r\àÀ»ÂjÍqï -ƒ°‡ÕÁLjk Ã´eæ4‚ÕA(0§ï -‰ætà]!|ä¸Àw“Ûä¸Àù$Pã8 -V¸`†è½ã¸ÀEB>Vç[Ž ¼€À‚²Xl1Þp\€Àzˆ:Ž ¼‚-Z)Ž ¼‚-8.@D0D‰è›]ÀÀ…]Ø‚H‘ã0Ù¢1xë°C¯r:§†àA׬.@'=ÇÞ›®Xr\€µÔ0é9.Á)\â¸f@' Ç>ÂLŽ àŠ† ™s: [ds\€H:)9. -¶ ä¸ ¨€U¦ÀCc-‰®ætF¤ufภ7¤P‘ãL¬4Á”¬¶€&8.Àø¤Å¶å¸‹,6±Ìau°E[ÃqA*-7Ç!À%ÇÀfZozŽ ¢€-ˆø8.ˆÂ )Læ¸4D°ÞT†ãÌË´Þ D-°ÞÄ™·Q»´ÞôD[,BV[4EfÞ‚“°Þ;q:[T4DçÒz3p\€0ëMDôlJ ¤õ¦gÓà]ÁLœ¼‹ÈÔq\€¬7Ùrk{âµñ¯ iÅØŒõZøV6e À½ðVžÍoZ#±I˜‰•eplÖnJ)Ø´ÿ"3+N˜²O4EåŸ|ºÈˆiaÝ—5G-¦7ŠMù±Ÿ¼|ûç{ýñ?—÷¿3eùG"@KÄW˜*b¸-¢o‰Ë× ÌÑ2vKÙtR«Q48'Ù’Qpwéßuýaa) ýayZ‰k^Ÿ7$gž„†µµ·Ò ­óùö`*¥Óc}o€Ö™€5ö˜À$&ßXå XÁ!$p(££JŠyàW˜Eáß½¾`6ÍVL‰Úr(µÍæƒø±b7Ê^_¸|ûçWûË÷?ë¯ß_þñ[ð,ÊK=ý -ýÝ72ó8ýµnú[äÙùs˜ÿš¹¬_Ês¹\×*^^ÊÃÛ´iÞ7møe[Y^¯á»^_¿ºüþuj©“ÜÜ;. -VñnÌ\a¦à@©_^ýáOü+é,1a[:g²+Ù_9É@(ädk²s*ƒkÏONf§XÄ*Í Ç¾[ì]ÑRFœ…«TÒc@ÚYç9(ýàLiÛtf´ô¨% -Ðû#$¦Ï+ùü»@LhŽ}‘¹Î›ªSŠ”›gÊXNÏõÞÎíG‚êwÍÍŒˆê¬Ù˜qí,²q{f¥ömæóú¦mŸËo «¾Œ ”ôÇE%ÿ¾º¾:'A]5ïêÎÊ<”{ÒÌ£dçžD÷~9‚ <é–L=¿l#é -§æ”UoÞ¬G~QçA–ÏÕ„¯UNKû‚çéÛYÛ(v<ì 5V®Ó &ªC¤öÕ9R•ØçGÝ ÇWÔdäÜaÖZ4‡®Bä-¼Fm£h°dÆt#Kòm}.¤¦»o·"µ³Îéª÷9U4ˆzC⼇†/Ý2}¹QÚ°£A{A:ñây¦ö™Ïk_™=¹·M11üdÓ*k¬¨°ªÊëÃø²³ÎéP¹_îÅi­'g½f[‡á³UŠÓ Åóu2 -n¬»ƒoúêœndb¸Ü“ÄK6½™æŽzœ¼ÛÖúuô|Íz4ÕÑi®aݧ´jÖ¸„ž}DöÕ9"¿Þûë§-»+ª ÄÅÛŸ-ê¬s6Äù¾oÅ´÷rØl}&èáþg%7ï½ë§0^(*xÞÓ/ð¨ôäþuo†¼³ÎéPy‡<+§í¬Âº}ƒwÿ$³¤áÊ-~ôô¾uçouN‡Äƒñ4œñ ŸwxqÇs™uN‡Òθ1÷ }CÄÆs>e«§z¹ÌIÇ·fVO:±ûÍš.ö=2¶6ÏtÊ·ÏGÝŠo?yn 8Þ¾ï¬s6‡ƒç}‚öëåë{»…ÆpG>³³ÎéÐxϼ•ŸñÆgáï,£wTånh—~«n÷{¾õÜãó©3G/žCm¿¹×y(ܤ‹giÄÑÉÎ:§Cì~¬Ú~¿–Ý/‚<ò.¬Õ÷2‰=÷ ï~4¡n‡]=qh ¹ÈnöÕ9¿|Ú~r…ŒxóK L‡m[W³ñæûÿˆ¬ñazù{ÆØ£ûæuΆÈØ™ßÜ{:"½CÕÏ¿Ìw"wcµö#“á.ºö ã­¸vãû3«™dë5¼Ä³=ñŽç ;ëœ5¿~7|Ì·H~±Й§Œ$çØ»Zé¬s:Äþš§\̬£¶¶PÙñ æ}hß@¨»ý}uN‡ÐƒïâÜÉí·GÇÕ„«»‘±Ò´1]n*÷®E¦‰Q±Hm.YÄ”qbDFøÈM fÑ“QKÛûLÂdJ¹˜Ë1¢Ç [nnÍ–‘ò¥T¨UcîùjJ…º¥ãÙØG³Ô§=oÉ:\Š]<Ö"ܨ•ðñcV⧋õLi[¹BØ =*Ô¤Ý"¢â¥JÓw¢¦O˜šY–6GIÿOTa‹41Ïø!íb sô¨ž$ÍÍ"^v•»Â:†ëi|-KºKz_0KûKÁ”O¤[J—[HI—¹Ù¼DR…<–bYPWĉÂrÿ®¤B‘ÀéªØ(?-ÄÕN[¦*ÀžOæ$»-íV±+ÜùDžŒ¸L»U{K»óYø8:6ÓŠôC>I+Q¼]=VÃ@ -U\iôiçÙZØ)íÀ›?ÖIÆyÜÇËO—ÑÍ)mUħeÚ|‘F^òú_”y…ß]Ìõû¼ÛÏäÇ•(l¡ -·.ÄÔ“¥­zC§8mÝ›…eÚ¡2„ö«X†ZÖf4?®ákS«‹J칤, *‹– -míC"VºJjÒ3vGøØôb dž;6ÕaO‡:_N£Ó>QÙ*ÐËÁa¨:]]ò•4‰£T´ºt™Þ®M*â€6ŠCrÂX-[˜™IORº>ÙoèPèt.r@ˆt¨«ÎT7K›Î²ñ*H„ÀšІ¤' {£ïž^Ï÷Ùd—g@’j/Ò'*-–¤…ðx©Ýò°ø}MyòÖ*mi2÷t rêÔ¨¡…YPŒi¨Õæ–žÔdÙâ¤ÙQº€LdYiÑg›Ü\-wËxÅí%_bŠ*ý‰ö´¥£R:ÑÒQq«²©ÝìñJ´¤Kd)œ}½:QK–X202Gk©©%u–­Ïm–çÎÞöJ.ÃTC\LE­fW%ïâ¿{æÎT§ÉDp¥òϘ‘¡îGm¤×J_c -w‚sà“v­õ&tÉÍuöó'ë,IΟÐ3a8£µðxÛ0á Šx¬ M=Í[ òÕ¢`/é¶`ÒC•æÄ*;4î^>æ0Ç$Q9ôi©hã¸Ç¬!í±ì¿]Úø"½HŸŒ?ÿ4Ói -endstream -endobj -599 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 598 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -600 0 obj -<< -/Filter /FlateDecode -/Length 4420 ->> -stream -xœí]Mܸ½÷¯èóÖ’¬âì!@ÄÈ1`$‡EvÛ³‹`À›Cþ~%ñCR©¥n×´»UEREòÕ«"[-é³Âß ÿë.„èŸ/NOªóVá__brЫ_áåíY+öÕÑ{þä;­ñîüx³öøãéßþ=­£Ï¦‹ÑmÏ¿¾O*¦Îï•T*¥IPYÛyo¬TNwèbŒ’ -fXφ—*38+©`U€Ä^ÓRa†ñš%ÌÐÑzA¥ìÐF)QC3A§mç0èNÒݹMu:í¤#êœgmµ¤‹sÎIÇ°ÅF¥%…-V3+I[˜‚—t¶#%ê`‹ ž$‡-£]€-@Wu°E9¥%]¤ÎÆÀ,ê"t*ø°Ôe;ˆ”¤Óº³À&‰:x&ÉyAg`‹S6Š:ØbIIG°…ñ'éÀ–|‚·šäݤHK: [Œñ,ê` &_bã`‹òN¢xP—xIâã}ÇÑ°D&PǃÄ&ÄŽ½'‰L´»è% ¥¡3Z"R°Å²“˜€4lax®¨ƒ-Yâ2°$. ‚-†IâÛ2Ð"qnYE-qoYi'qpa¦’¸€À¸K\@ \@)J\@à\wJ\@ ]¸»—¸€Àºd‘¸€@»ÄÁI\@à]b­$.`ð.jŠ:ØK\vìH’¸€Á»¤•—¸€Á»p##qfµÄÀJgBP0x×Å0x×x„Qç;ø I\Àà]cÑ)rƒwUFâï6Nâ¥ÿ(1ï‚B¬œÀ¬Äˆ÷Ð’¸ui‰Ä¼«#ŸàÓ`«N‡è$.°à]0}¢.vÚcÄ¿Å(wÚ¹(qï"P‘ļ ú €ã;DT#qïjB¤–tà]m¢’¸À‚w‘7X‰ ‡N#¸ñ­Ó -Ðu±S1‰ ,x´d$.@îT“ˆ:ßHZâq§‡J\àÀ»ïQâÞU–HâÞUŒ´GÔÁ -Fâä5ÐÁmŸF²Ú)¦tà]¥1Ã’¼«”8ðn -›8Ÿ&H\;dÁF⸋Þ;‰ \TÐ!åu®Ã±•¸À+,(KÔÁö,që.R x[ȉ ¼-H$.@FÐEì[\ÀÀE\0lA¦(q˜¬ ‘YâoÖcUIçL<èZÔè´—¸À{ÄXKuN/q¾Ké’Ä0:Íø[à* uÁ 3—t¶ É–¸™.tZK\ lA6(qA0«L§$.@„ÆZC-éX¥uf¸ °ëRª(qÜ+M0¥¨ƒ-  ‰ 0?i±m%.Àâ‹M,sDl!Ë„`Òr3H\l1ÊH\l¦õ¦—¸ *Ø‚ŒO₨\—Òd‰ @ãÐ)%qAÔëMÃÀ/Óz3H\Ia½‰%žà·‘\Zoz‰ "ËEÔÁ$MQð[pÖ›`'Iç`‹‰,qAt.­7ƒÄH3°ÞDF/n „´Þôâvx×F0“¤ï"3uàëMd¶ÒÚ^x­G~&+CZqqÇ@z-b«¸e À½ˆV^Ü3P ß´F7 `&V–Á‰»«Pj%n(ð/ò1^p°ûÔnQ9ÀÁ#fŸ:áŒì¾¬9j1Z)6ìýúóéû7¾çŸÿszø®ß)Ë B_š‘_Á LÄt[dßÝ'‡ÔæDRÉBxOd„ÕH Z7Ý’‘éßyùab9 ÕŸžÊé#¢øpz·ÑXBÈË×¹Å7VyBƒGˆ¶e(šÄN€òJœ¿}PG+†=Ðrؘx‰Å0{~ýáôý_^þý¯?¾Œ^?œ~úùáe~x¡ñ™ýø†w¨%ŸskÇwß”eÌ£B~SÛÁ:T‰Í^YnñBy‡wEµí¾}7=çÜ–|Þöœ©Ïõc;ü¾–ëmõÕÎ"÷³:¹¿N°/—}뺬ÿáŸç×/Ox=TŸeö€â”” }0vƒûª| -?ô°d½½2´aZ Hðe:¼Xá?èm$y)Ÿë7:²Ùž·K›&uøôÐ5uÊ}‰+m­Af«­ËÔÅú1‚=6LûXÝ@†`ÚiϤ†d&*¬ôiƒ;ë „Xí\EaFMÑrÓŒ6虣x?Y4dkg¶´HzBh—çä9+è¤JwHqèÛBç¾:‡C'_åÈãòÊzÎ ½jæ%Üç÷¹g…ñe‡×DŽyÆ2»O-Š<‡p{Mº—¶DBè—÷×±·¯Îá°w=5ìÇ-ß»q,ßÖ14¦Ϝν›}žû&žÜlžu“òÍ°PêAÆZϦJ`*!UÌuæLÝ3è¼ý·cÛqiKî[9¯FÏËöv4¾·¯ÃŒ¸¯ÎáP¹‘5f†»LgïSbÚ5d.àk½[P°™=Wó†9«ªÙx4(Ÿ×ávÑ6[صºÞ¦EŒQ·óäÎ:GC¤¹žA–Q~Ûd:y–šÌhÁ]Ͳ³çÊfi¼˜}Ú9ûWfX\º·™ÛÌkæ9iPXÚ -S»ÊÓ¾ò¬þ|œò{“¼5 Ç±ißùZ75ܽÂÙYçpø¼žC.æb; .3®¬4¶¶‹Jy¾Ûù½K.[ÖúXkíÙÜ¿XaÎ6[LHà˜vš¯#s_Ã!óz†ù9bù‘wvžE,c¶æ’û1»¯Îá0{=ÿ\xÿ<ï›mVKó²Å’“Œ@«0+0“V·O`ŸÛ÷fÄÎÿk—`…I©æ“mgûïf6P¹³ÎÑPI»˜³Ù.3×|Í1ÙWÿ¶÷.3ôM㸂Jº+wÖ9*¯gžóŒò–}ò¯‹¿îðÐ'ÝÓ_Aj“U¦ŸPÀ¥Ü&R÷Õ9R72Ñß_­xäÊþû—õÄ„69äÞë5vÖ9B7ö=ïÙ£¼‡w¾e‚§ÉèduûJ~g£¡“?SþùL3ƒ'õ¸/é¹É/÷Fúu‡ÔœôYòÏ×éÙÞ~EÒÎ:‡CèF.z'ïÜ|}Ç·H¿?Ò·{™;¿íÜYçp輞‡Þ½îý -#ý=WT}îHoÕíkúuŽ†T»‘“>Kþy";p¤·tûš~gÃ!tg.Ú^ 4û±Bæ‰2Cãì¶ßÄÛq¾EõÛ¢º½ã›øu‡Äœók\¿ïú<ܾVßYçp¨ÜØó|–\óDÞ´3‚·×ŠLø¦Aõ•kMæW•';òxåñQ`:ÃÓþ•zjZž“­—ÁsµMs›¼ï\¤t¿ùÑ&ÊãîÒ9&DZ~·lÇQºcºá÷²5ök§~¬®Ø2X³CÕ¾LÆ#Í‘üI°-ÝøÝ)6. -mø­mNñXÇ:3¥÷KÆÁh“ÍQbøœûšìa_ËöýOçq£íq|wr?z\å2O”|b8ßGŒår4³£^í¿­öåÜbeìi>>}[ï‡ú9Z¦¶‡_öeÏÛô»÷GÕØ‘ûœñä‡÷ú‹ÅÁþþ<¹^ñ?y¬ÙŒ5ŒCñG=ÿÖ_çs›ÏÛã;õãÍÕ1²‹1j9%Ûªïe\dü—Ï¿îëó>åñìû£Gû3ŽÇw=âÈıýlG²émµµ·?cìá÷Nç/g+ãäã´àç¡Í$ïsåæö}×9B§ÛIN7ÙÀÆ‘3 ,¾o/é-Z³6ÝÞ7Xš»U–üd³ÈF:²¼·e¾Cä9ýÍóÚA4™ž+CጵGŸnÅ*ÅäćæýÀ[zœSãn¢_ÉOJìÎq:Ïeà ÅGG?ÏsŸÞÉU¿HXKrS8¼Á‚kãgõ3ófœ’KÅå¤ý+‘n)]Fl"m†¬‘û˜ÍK$UÈc*ÖuEœ(,OáÊV¨²8]Ûålj¸Úi‹«ìùdN7p·¥Ý*v…;[±/3®Óýó‹½¥ÝóYxéÏÅ ú!Ÿ¤Š*Ñ®ž¬VÃ@ -U\iô±go-ìÔß²müXLŠ¸—Ó/§>̲&âƒ#Óf^úü_”y‰×¿N|þ1ß|ˆãF¶0…['â6’¥‡‡…âô0±,,nÓ 7´_Å:Ô²6£¡Äqc¾æZ]UbÏ%uaP]´­ÐÖ1lĦ@×hjFÆ^^f£X™O›°‡CÊÃi(ݹ>[…ƒ¶;8 UG5$Ÿ›&q”ŠÖ-›¦"ÚFqØœ0VË&f&@¶')CŸìçv*(«9hÒˆtHUÇ5̶MgYß‹¦R`jÚ†Y·'{£ïv¯Ož÷Ñd—= I µéc+-KÒBx²”úC7=,1‡Îé ;o¡Ò5¾G%É©®QS ž´m¤¡V Øö¤œe““æ@Qè2•e¥EŸmrcµ<,}ƒ8J¾äUúK;Ò¶•2ˆ¶·(›ÚͯdKT ÛÂÙ×Þ©Z²ä’A¹¶v‘r-IY¶<7OÏ£í¹éWC\ÍLU­f%ïâ¿{|g¨3Û‰Jå¿~wD‡úÄjÖž õaUÁ9p§çhyÍvÉÍu®ïŸ,wIòþI{&LGÇDÊëô8]Ù„G(ⶢi*?jW®•Ø¥Û@ت4§»Cýóûï´úM¢r˜fbÒãS)%½!=õÍ+-…“.UçÉöIÿ÷?´rx -endstream -endobj -601 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 600 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -602 0 obj -<< -/Filter /FlateDecode -/Length 4762 ->> -stream -xœÝ]KäÆ‘¾×¯¨³Ñùˆ|‚ ¬Ö‹0X fzÔŒ–iþûþ‚Å|0|TMÍ åiôt1"ÁÌ/¿ˆ ²H}UøùFã¿Hzˆ1O×—Ÿ/¿\Ԝ¿±Äì`Tÿ€ßà®ZQœN>ºëÏ,°aÉ™à¯oü׶Åß.þÝåó:új†”¼Ñîúë¬";„‚r‚*A¥´TÎ !ç•×N1%I3\ CKU€”¢“T0ƒŒ‰‚* KAÛ¥*Á 4I*˜¡“ ‚J+Ø¡R¢†("kvƒÇ {Igôàc2IÔèt0’ÎÚÁÒNKº4xï]”t[\RZÒ9Øâ4‘’t°…l ’ÎÃë­u°ÅÄ`%]€-£“]„-@Wu°Ey¥%]²ƒK‘HÔ%èT q©3Ê .Z«$Öƒ6­¨ÃŠÂ$ù è lñÊ%Q[œUFÒYØBø‘tà gCŒÂj5¼º­²ZÒ9ØbL Q[0ù[Tð` ÌK˜J†$0Ñ)J,`b(+Ñ€In Ÿ‚ÄVi茖ˆÀ*ØâÈKL`5l!¬\Q[l"‰ ¬-( qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ],÷ qëZçÄ´k)z‰ ,x×’Vx×ZÔu°–¸ì8X­ÄÞµZ‰ ¼‹ed$.À¬À¡—¸XLŒJâHâïš ê€5h%. ð®q8ÉsƒwSFâï2^âeÿ(1ï‚BœÀ¬Äð÷Ð+q ‰ xW'¸?aMƒ­“—¸ÀwuÄô‰º4è€Ö-FyÐÞ'‰ xŽÊJ\àÀ» Ï(q8~€G58ð®¶ðÔ’¼«MR8ð.â'qà0hÀ¿ Zº¢. *Å(qï‚–ŒÄðƒŠ`QIK\2âP‰ µ)*8øÌø$–!º/{ŽZÌ®»åÇ~ýÛå÷ïá|¯ûÿËëïÆLYþÑ¿%ÏkÇKÄ$L7èj›#l0kö?l ì{  -›R5Qƒ§”›G g›®åƒž {ÙbPfAc¢¬-½·ÊÓ-Õ˜$xaDnm¿X‰’{…w­³üïºü°c$öåB¿Ë8vp¢±G+=Ùh›ûGÀMwÏûV8mlí®`khÙ£6&ßYåPØ#Âò(e“ašÁ^BѬ𼻀œ¦îoyïr¨­ËvÃÆب뻟/¿ÿïïÿïþôº}÷zùË·p[F!ÂÿîÏôrûëôMîìtìñ÷¯×wßoôöU‰·&óþ¸ 55ƒVÅ©«4É·›56}q–ͺ×ïv*ÃãñnÞû›vªcÑcקPÞ£ÿ…¸? òå– Tïoó?b ŸsjVdœ¯6ÌDÅNèV¤êê˜éoœ×å¾ÕÊjÎr¦äR?t:;_ݽåX/ëZsCX>>õ8Èö1ã1…aŒÇì„U9 ‚«Õ ° ?!Çn¼$·Îp…½Â½œ»õ2c8öP41[­Y²a§ãbÐ]‡[ìf(¦ÂÞhŠD;Ð8Xç$Ø(ìf¼ÝĆëVÞ -N4F_c†ôûÛç…|ú-rêÊñqà™ê|¨:cšrvúý8ý¥é3·ßzà/ }[Y¯´™ûíÙ¬0Ý -êñ­8)¿çSÖ9 êªÙ1n¢ÎÎÑF¯ó‘¤ì;^¦‘Oîu>ƒ žq«ÒùŒÜƸ]ÉÉ3¨«ž>,g~VçÇYnêë0îRËH‡ŠšŒ¼‘;h‰ ÅLKèz"ïáµÖ¶ ®Y1‡‘¥å¶¾R­¡»‘z°Îéj·9UuˆúP?óÎøÒß¼ëógiÅŽíéÏ+m´¾¬}eõäv>vãE52™Û´‚Ê+ãe‚Ý/Ö9*·ãË­8­÷ä¢×ìë|¶(Ó¢€:¤¹NÙ­f\›Ùðß«sº™Ý‰áòH¾4^²M^;æež_XÌžnfiB×±¼Ö^›LØ£õ?4»š.'ÑÖískÜSPÛñ‹{¿‚ÜÌ)1lËÏ¡¦É—”s‚2Nk4F|3ò½‡{8=Vçl8¥íná;ÚlÈoç¹_äŸ|Å@ï{úHÒ#}T¶êϺ]ê_×ff¨þšõGÍÚ)mw~oQŸ62ëQ1):–/9XçthÚŽ²¶üÙŒ‘z”ôˆÒÍŒ¶¾­Ÿõ×y;ã1³Xjv¹- -^çmô¬·ØH1|n«C43êÚ>d/ZkWLYi=K~öZ/sÛÆ‹+þ™Ú,ÝQ¤«s:¤îdö:NëÑ6&ÝôvžmÎÓþfÅxa¦…ËÄψÂÆØÛÞ*ý]ر¶ nméQåš¾Úͯ ±úøfÒ8Þ¸ÄcuN‡ÄHñ+ç: -t^y¦Ëõòªé²v‹sÈ}¾C^ÕnØ]•¹Yyjd²£2ªWû@öï`³!×mÇŽgÎàm¡ú“QÕœKä‚Û8”æãÔ£QòeßÕ]¡YädxWcN -qЖøŸmä¬s:äîdm‡Žv§ÙxéÖÓîE·+¶Ý,õž¼¿ª±ÃÑ…]ög}û¹ý‹x±ÉBÌÚڸʲ‰àõMvã‘xÔÑý\z°Îé¹vy‘2z­÷ZÛ¡4ÕŽúÌó½ÿ_‘é·ÈfšÏŒÔ&öÔfH*µw»ÌÁ:§Cêv¼zi£‡üØð`ËkküÒ]™¦ üxLwn”¯G-+¨lr’É nÜœï¡òX³¡ÒïÄ¢{(ºw6ž÷AõÊ®è·r™ÞÜϗ뜙;±æ3w.P~×îæÙˆnÐ1Þ©s¯ ´Æ’‡wD뜥Çîtüì{ïÏàu ÷žs|öõ#îß—¬s:nÇ–ã$ù>Z)«ÝÞïê§rÒŒ {øÝ=s“…YËoæΙ÷E¿Þϱ äÐòiô¹ÚµŒor‘‡=ó±:gC[؉¹×dT΃=—£¿\œ,#4˜ûï?Xçt}üzú!>¹7ÃýDþù÷\+ˆmbB~DÊø%ç=Ä«s:ÄîÜÑx/ò¾À7UîÚå|©×ŬϾ¾šøЙA9òŒžmD«s:Dþv¯¯Ÿvµlî¨V—î¿·è`³!.ûVL-Ğ´÷½ü^ÉÕëGÔò.Œ'EŸvwų#•±¹~}4C~°ÎéP¹“‡<+§mìÂ!ûïþYVIÇ•küÛëÖïx;XçtH܉OÃÿæ«â/¸/ó`Ó¡ô`ܘGðÀSË£›Ñ—pYû-OrõS;1OñÔû®ÏwNO‚µï¦¾{hz®b9—Ïa1•¿\üøŠ=¾æGù8hÃï¼z,ÏMºòk%¢²¤óÔéq°õu¥æøöÄ_J)þ"ÅïXâ÷kŒSÛLå'ýç‹N4ðøóʽ½üLâhòA¿]øåw$ˆ ó ˜…ÓGUÚ" ÙËe| z_ÒºAMSiœÄxûh\>®ïr5~)©\ÈŽå¿u,QÓ류Ï^ zó ã`§#_Lá‡VamÚ²µÁZqW¬­j„n)TMwUl(Û£ù…3eè|ÓÛ¡ÓÍôç²&4Ó¹%|¹ [ÛKßD)¿¶,7•Gר"³Q©H].Y&„Ê< ¢—‹ üFå¦ “YôͬñË_n˜D¥\ÊåÑËE,7µæÊL…R*Öª)|5¥BÝÆ2ð‚l£Iø¨ÍQ6œ‹=Ï@žkܘ…ðåC ñÛÅ¡´«\¡\†^+´M»EUÅK•ò½©·OXšYƯÎàÿoTኔ™gü ©@Š_Sj'¼6‹x>H­ÜÖ!i4…Z¶.BÁl;^:*¡<“n)]Fl&m†¬‘‡”Íc’*ä1낺"f -ËãS¸²ªlº«bW ü6W;]YªN•Î¼nàîJ»Uì w¾µâPf\󻌊½¥ÝóYø2:6êÅ ý˜;©b£Š·«5ÂjH¡Š+¾â¼Z ;ñûYóǺÈ$ûrùé2º9„ù†_Oá­æW󵑗¾þe¾Çïß/týS~ÌÍUØÂn‰[OÆqŽÅüb×,,˦jh¿Šu¬e]FCñãÆ4|MµºªÄžKê ºh[¡«c؈M®Ñ¶·!|éF±2Ÿ:7Õaßm>9~ÿE([…ƒötp«ÎV—|mšÄ­.Z2ME´â°é0UËff2 ÛNÊгýÔN…徚ƒ&ŒàC[uTÝlÛt–gÑAlÛƒ¶aÒm§if`ôÃËë“ç}2ÙçÀÒBíEúÖJˉ±´ž,µã¡ŸŸcÇ×RäÅ[G¨´e›µgKS—F -hvÐb´‘ÆZm*àÚN)ËffGQè2•e¥ÅmòSµ<,ãGq”B‰)ªô§v¤];+e];+~Q–ÛͯDK¶@®…s¨g§jÉKFAæÛÚEJµ¤Í²eß4ï;{ÛksT ñ52Uµš[”|ˆÿY;·:]&B*•Æ숎õíipÔ¤ƒ5öÊoúQÑ{pñ;MÅ&]rwíüÉ2K’ó'mO˜Ž¬UAo+&¼A‘öMSoÓ‹çäjI‰§tßé¡Jsj‘ßm=æÇ$Q9 ¼UäWÙ[z#¿7(-¦‹Æ/d¹YúdüùJk -endstream -endobj -603 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 602 0 R -/Resources 4 0 R -/Annots [ 604 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -604 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb127-1) ->> -endobj -605 0 obj -<< -/Filter /FlateDecode -/Length 4327 ->> -stream -xœí]Kܸ¾÷¯ès€ÕòQ|Á$AŒlŒää0î]ï"ðæ¿Ÿ’H–¤RKÝ3ãhl{0žVIɯ¾*²õÐg…Ÿo4þ‹¤»Sðt¾|8}<©.8…}‰ÉA¯þ¿Áµ¢Ð9|tçY`C’3ÁŸOøovÈ‹?žþþ›Ó¿§uôÙt)y£Ýùן²ŠlbÊ ª•ÒVP9×…`œT^wèbJ’ -f¸@†–ª3(E'©`U„–‚¶KU‚&h’T0C'•V°C¥D QDÖ:í:A÷’ÎèÎÇd’¨ Ðé`$µ¤–t©óÞ»(鶸¤´¤s°Åi"%é` Ù$‡-Ö[%ê`‹‰ÁJº[ F' º[€®(ê`‹òJKºd;—"‘¨KЩâRg”ë\´VI:­;lZQÂ$ù è lñÊ%Q[œUFÒYØBø‘tà gCŒ‚·šìÝVY-él1&¨ƒ-˜|‰Œ‡-*x‰àA]æ%‰L%C ˜h;Š%01u‚•hÀ$בOAâ«4tFKD`lqä%&°¶F NvŸø•bf ¹“ nAÈî뚣³+ņý±_>}û€à{þù?§÷¿éwÊÊF‚З&äWp“0ÝÙ·F÷­Iõ9“T¶ÞS'R3RƒÇM·l$Â_þw^~˜XÃèOoëé¢øpz8qRÈp¬ˆåëÜâ«<£Á -‹#D[26™ÌN€òJE“¿{{PG+†=ÐzÈÌG¼Äb˜Œ;¿ýpúöÏoþö—ïŒÞ¾?ýã·JÑOJ¹øÝ7:vãßñØÇN3Y˜•ñÃ_ËÌtŠ$ËÙûi;¹^iÃåò¼=?«3;çÂÇê–:f,SÚ¢i›ÊŽuÔð[Ú^ŒA¶í2k+¶ºµ]?ý[Î_m׬ïjÿz®K–}÷ÏóÛ7§ß¿‚°ÏLæÍNdÌ™×®âv_•§àö%àŠeî5¸òix?“§ÐŒSE3HV¾ÔgºR®Öãð{'@Œ×aS¨Ý¨»#©­0íSu«­¶Ö ©—Ðz¨ËðÌ;÷lHŠ¦H´ÏuŽP¬ž®"ôN"¨³QFþaDá…!òa4¾DÏXA¢m´——´X7älì:÷Õ9é*W‹7žÎóf‹ˆÌ‡ê1hÚýÒ!¯ØÆÏ–ý5c¹ôÂéîàÈ}u‡ÌëIgÏ}%1 K¤”Y®œé¦Ù=û~œaþ[䙆YÏ×f¾œsnKåù‡Êçú‚îU,YÎv.’k?›lï¾5^\C_K)o…áŒÛBß¾:‡CßF¹æÓ<-ÿâ¢é>®›g!kKÇ5uûú{g£¡ÐìË ×Õ¾eFzæ(ÞÏŒoÝÌŽp§_oº‚ΖRˆÒÂàý:÷Õ9:Ÿ;þ_yå=g%Ï­õÌìïܳâøë†ß‰<9e‰â2öX¦·wݲ³Îá°w=;ìÇ-ßãX¾kch Ï’Ñý8û<÷M<ÍWèle>ÇBe¨÷2Öz6US Ùb©3gêžAçí¿ÛNK[ê:J-ûV}dV¶·ƒùÞj¼Žw0â¾:‡CåFÖXî2½§Ä´kÈ\ ´z· `/ -{®æ sVU³ñ`(Ÿ×!¾n›­í¸®·iccÕí<¹³ÎÑi¯gu”ß±Lg¶§ÒëæÜÅVžudü]̾Ý9ûWfX\½óÌmæ5óœÀ0Ö¶âÔ®‡Ò´¯4«?§ò·ŽIÙ˜ïéA¿Ç-ÛKÜ»ÂÙYçpø¼žC.æb;×¾š“V[;Fµ‹<ß|~oÆ’„KÎZŸkÜžÍý‹æäÙbF%³õâÎ:‡CæÆ—Þ/˼³ó*2`³ìKìݘÝWçp˜½ž.¼ÿžoQ6XRüÆÄ Ìt哧Øwëw /¾K°Â¤ÔòIJ®sý×2¨ÜYçh¨¤]ÌÙl×™[»ÜåëÞ»ÌÐ7ã -*íí\¹³ÎáPy=óœg”·ì“^üu‡‡>ëžþ -RYV¹÷j¶u‡ÔLô‹à«ܸÎäÓxâ -BY¹÷Zu‡Ð}Ï{ö(ïᯙÁîkÜœº}%¿³ÎÑÐé^(ÿ|¥™Á³zÜ'ˆôÎÞ~ÝúÎ:‡CêFNú*ùçóôîž;‚^þ– Aèý·]㛯ïøé÷GúxûÕì;ë×óл׽Ÿa¤¿çŠª—Žô^ݾ¦ßYçhHõ9é«äŸgò°Gzoo_Óï¬s8„îÌEùµ@³û -OðûÕz¾bßÄ»q¾FõÛ¢º¿ã›øu‡Äœós\¿¿„w}Š~Ç=æ;ë•{ž¯’kžÉ›vFp~­È‰ ÕW®5™_Vóào9Ãß½ï¬s4”†ë?ç÷)ÌV -óëÎLá ’´zýÛ3^ÄmQ?â÷ýþÍ<èuwŽC)¿S>^^w¸‚PvGÐn„î«s8„Þ¸çÉ8GÂo­!ñÚzy/½æsžå,®­_‰êáŽïÚwÖ9÷åšÏr•£~:×®µ_»n~ru^FÑxïÙj;ó>Iù¢p­ý=1bëž¹U®Œwpå¾:‡Cçλҩ엷^¾±­'­W´uÇýE;ë ‘q#¿œÍDÕ+Ùý]våé·äƒüª¹›Ÿ¾tø;àWÐzÇžæÎ:‡CëÆ믕Ӟò$¦Ù -o≟ԃVÐÙòHM˜ídƒS[èÜWçpèÜ™{®dïÒM¥•\ló^¶2'õ‘®¹/©ë×Þ¥/õ#ØYòù}) R¯­6±ïË›?üéߟŸ÷%Y.\?÷\Ûj¦6·Ø‘mʺ=´ôu8^ÈDzËø0´5 :ó <{ ½…ÓËží2wåù½³Ãù{ûœšîr©ÑŽÚæà »†Úò¡î‘ri½ëG÷”L±²;>ûƒ\÷ 1KY=§Úhçò”Û¾Œ–ûÑd?±5„Î'›ßz4ÚjÇYty„ã8Šý“–íx› ž_;³l§ïC]ß÷¹`÷‡ñÙÃÁ¡!çò³ß£Ó‘=ʸ’Èf‘ ÎX>ø¸<>øœæ¤Ðú†&óKÇl<#¡"tÏ:Ñqòp™Ÿ²s”…>£»Sný8 ‰9XË”aÖ# ß±ésP‹sä¿vt‚<9ÉrS‡–M™çàmð2ãt^XpçëÛwã¹™=ÅkÊmci1•O¾æ¼îß;¢|ì´É/&;{êlž›tÎϹʒ.S§ûÁÖç•šýëÜ>ÖRãÞT~éK~àÿðú7v0–ÿØOúx*u¾øfÿ6&Õ‹£)MüxÊoã"ALÔ|Cœ…ãGUÛB´33Ù‘1ti^Òºò´lF*Ö•Æ•Þäú®TËOI'U -Ùþ9áÉ!jz½çÞ«NâüF³ØÙñÈWSœ/6CØ$^¶5Ø*nŠµULè–BÅN×ĆŠ=:¿£oÂj::ͦ¿”5Mç5áå„ÓÚ¹ôQ”æ÷(•¦ÊèUe–•ªÔ•’uB¨Î“ ºœá7ª4M˜Ìªg³–ßF1cÕr©”D—“XnlÍÕ™ -µTlUSùfJƒºuàÙ…Þ_ÑÈ|DÙòq*öyÊ\«ÈpcÂË !†âÇ“ Bi׸B¹=.´¬Ý*Žªá¥IóÃt†OpÍ"ËÏòÏÿTáª43OÿAS…T~G;j'ɾYÅÓAâr_Y‡¤ÐZY6\:„ŠY>^Xû å3éÖÒuÄ&R6dLR1/“T%©XWÔUq¦°2>•+¹P#pº&vÊq³ÓUWØËɼfpwµÝ&ö•;¹8Ô×ùå*ÕÞÚ.Ã|^úÀFsq†~,'ib£j´k'cÂfH¡‰>öââ­•ú§qŽ›“I÷rúåÔ‡9cIøà­Îï -ã™—>ÿeÞà÷_':_^N1Äq£*[˜Ê­1dùÍ’q§8¿i²«Û0¡!FûM¬c+ë -j7†ñ5µêª{)©+ƒêªåB×ƉM…®Ñ–Œ»"¼ÌF±&2O›°‡C[:‡ÓØüZ“bxwp›Î¶|fMâ(m!Z2¬"x£8d'LͲ‰™ü$uè³ýħÂæs±–FäCÛtÔÂ,oºÈú^°"H-?à “æ'M{£ïv¯'Ïûh²/¥•Ú«ô‘KkDz´ž,µý¡ŸÖ˜cÏù 8o¡Ú–e¾gk’Ó\£¥49àeÒت?)Ùä¤%PTº€LYm1›üX­ Kßã(ŽR¨9E“þÂGÚñY©ƒèø¬øEÙÜn‰x5[²5rΡõNµ’5—Œ‚ÌóÚUJ­¤-²å¹izîmϬÔ ñ-3U­š[”¼‹ÿîñ¡Îl'B*U~úÝÛ뜨Iklÿ†n½‡P~Éb È¶Kn®s}ÿd¹KRöOø™0Y«‚ÎïZ—Mx„"m+XSå=ìrµ¤Ä.Ý6ÂöP£9µØê_¶Ûo<÷›Dõ0ä¥b~·¶ÍIo̯ JËû¬ùzÒéNXÿó?u ½˜ -endstream -endobj -606 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 605 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -607 0 obj -<< -/Filter /FlateDecode -/Length 4832 ->> -stream -xœÝ]Mä8r½ç¯Èó£å÷°˜ƒ{áÁîÂ4ìƒáCuõô.Œšf}ðß÷ J$CT¤¤Ì®nçLª3Á ùøA©(}UøùNã¿äô”RŽÁ]_¾ürQSô -ÿJŠÕEQÿˆßè¯Z¹8yCòןI`ã³71\ß.øo¸äÉß.ÿñ»Ëß×yôÕL9£ýõ?‘ÊÙ)¦•T*¥­ ò~ŠÑø ¨‚žÐÄœ%ÌðÑ·UE˜árò’ -f8c’ J0ú¨íV•a†‰ÚI*˜¡³‚J+Ø¡R¢†(ç¬tÚO$ÑSHÙdQ¡ÓÑH:k§öZÒå)„à“¤s°Åg¥%‡-^;§$lq6EI`‹ V‰:ØbR´’.ƒÞÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓVg”Ÿ|²VI:­'lZQ‡…A -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) ³ÕÐì¶ÊjIça‹1щ:Ø‚Á—ÀØ¢b(3h"^’8ÀÄ8¹lœD&ÙÉ%—$0)O.F+Ñ€É~r!G‰¬ÒÐ-U°Å» 1Õ°ÅaæŠ:Øb³“¸ÀØ‚X [Œ³X°­Z$.° [§²–¸À‚oÒAâ Â…™Jâ Ƶ)8‰ ,(PÊXp®wJ\`Aº˜îQâ Öµ>‰ ,h׺$.°à]ë´’¸Àw­ENQ[`‰ ÀŽ“ÕÉJ\àÀ»V«(qïb‰ 0ªp$.V&“’’¸ÀwMRNâÞ5K€¨‹æ •¸ÀwG¤•¼k¼28ð®q&H\ Là%zà]Pˆ—ÝØ‚•¸ë=tÆJ\€Â&·DâÞÕËŸ0§ÁV“N9H\àÁ»:aøD]žtD ó½<é²Ä¼‹…ÊJ\àÁ» Ï$q8~Šj$.ðà]m±RK:ð®6YI\àÁ»ð¼Ä€Ã¤áH\€õmÒ -ÐuyR9%‰ Â?“•‰"Î$î(P¯ÇÚ*n(p/V«(î(/ÅHâ¦ÌDd™‚¸k€eJ­Ämþ…?æ6œ0ï>ñ-ª8D¬™)R#3¦…ƒwßbŽžÌÞH6ïý㯗߿`ñ½þõ.ŸWvÊêø­ XÃ;Ä<0ã ¾šJy&œmC YÉCFÄÜÔ ÅeÓŒÄRŒå6_Û½6“ÍÎLi™[Õ‹PAc¬lm½;ÏûÛ -?+1¼7^/f£häAâCë,ý»n¿ ÷X¨7äpÆʼnƞÍôŽFoýO.`îy×8\6A_<Œ úáçËïø—ýã¿ýˆÑúðùòŸ ¥\RŠvkèÓQ¹Wü:È–ï1»ÇŒÒŸ¾ÿ¯ë‡x=´É"Ƽ®‡Š†ã¬ +’Š#Y)öã\]Iƒª,}¾.ú°˜›|îß‹ÎÎæ¸ØÓ’¬˜ÓóòäfP¾–æs·ƒöoK}n®®‹žÒ-×$/z½”ÿ±Û§Í¦{Ø0ä±{JQ?ÍÙu­E»—¥H=›^ªeͦ”ѪfÔ&§¥;âüYÊCù>Íæ—zj>»tÉ®ÖTwEš»AÙuŽÃØlZlX ›îö»´×MôuÝO­q€Ç 8Õ¶’ÍÅÞ—:´­]im«^ÚF¥ŸÇþ©°b}Y®ÃR7Ål`cAvµŒ“_Æg¿OìØ'4eJÙÌöÒGî™õM­:†¯ÝÖÒ_¡÷q¹Ž=﩯¿v?·Ý¾ó<åÅN3—µj×2'nQ„_ì_åq;X¬}Yi¥ŽOï:´ºkž:†ËœÒz·­ˆ¶8ñWÿçe³mÉ…ƒhlYÅh ñŸñpÍÒD"8c+ß½yò%–…-BŽöN6 -j੪Œ u³ú¹ZëÜ5íR[ßìE¬é°FUÃ?ýðïù3[ eúï¿Ó“×ù³,|…‚—ë€ÏynV^eÚÞ*œWŹœû´Ô•ù~¹Æf¸ð¸ˆB¹þó÷¹7Ñžp·¬:È，”Ÿ-_{ni•?Qf¹Ýê|°;=5Âò}¨c&V¹0سä‰q-;Ñ~„÷´÷Aqà½OwÐyˆ…ë}G£¾BTøn½ï®˜œ·}œúÌFc·.JsMoz¤ðm‚‹‚è’’Ïè?®ýºJöÕ hnkऺuOéžZ¦Ýë¡Ò2Ûo!ÜùJj`ÔTXíˆÏåù’àkð¡E”¼Ã‡ê…/ñ;³i™q.ÜžM·|z³|ÖòŠ;Àò}Ò™-Ô¯^>‡Ù_å³ º|ƒÎ®ÛÔÚ8¶YoóZ3£Q1[¾¬ÏdT¢žŠ0C»Ì„ }€Ê“yž•ÝlëöPù›DÔyö´Ñï±gcƸ¸Ñ54à칸£…5åðÃ` rö)ê¡Â=ætJW¼y&x‡É¹ŒžÌó$mÌIOì`´Íôk#Nt þ_æïùòÛänHG×Å©_ò|ì:cX:»ü~Z>ÝòÊç«û·òµþÐY²•YëCÙmß@ AôÌEÛãû¨;—çIP×Ívjuv¶ÊxãZC[¥çù:ôy=‚ž¥õ(Ô2æÝ°EVGP3Vþ¸ùUžŸ²Úö×Y$ÜÙÖ¶µÈÔ:ÏãˆÜ®*îe©¾s%d¸l‘z.ÏÓ!5ìó£;j*òæ½Ø-‚6#-¡ëy¯qÛ8<›1§‘¥å²¾Rc¸©çò<Ró>§ªQ™ú2ðe˜W×÷¥v hoH<ÚbŸû¶öµÙSËù4ô—ëžÉÚ&•¾ûŠ ¸2Ñú—'ó<*ý¾¹ç§+¹¸jŽy>Û¤á(pR¢œ§" qà­‘5÷óÍÉ\íÉW¶J½Is‡î–òw3zšÒ‚ÊCwZéVPÛe{4ÿGÕ„5ÒxÞÆ iß×j¨£ô—È­œ²ãÃr~®o½ mM‚2N»7æèi›DÉáô\ž§Ãé¾·Y;ønÈâo×±ßìÌ„Žqí=A—·½²›ëÙ¥žYëøÎ̸7ÇçŸcs§}Ú¡}Ÿf¯¯>°Aó˜”;·_r2ÏÓ¡ißËÚ[ÏVŒ4¢dD”f#Ê׶qÔ?¯Ë)be±Ì¢\Ž‚Ïë2FÖÛÄ’_ËMŒz+9òÖøŒi3mdÉOB¬õº¶­Üé¼±>¾Kw©'ó<RÃÁÎÞÀi#Úʦ›ÞOC£Mû´†~—Q1Aiá^ô{xaÅ÷¶3B¥Ï·¢`nˈ*Ïêz(š¿DîõÑ_=$sxädž§Câ§øÿ¼×ÑxdX•Wºš¯Îša×nÓ†Zçë9ä÷d6ˆÑuëÞÐÀËgwGeTß@î»'ó<r÷}ÇgÞÁÛCõ£ŠµeDòÁÜuë~Ñ(­-îîÐlödάþÝçt1MÚ:zŠè¹çò<rví€i²Uš¯´GÞÀ|Çv¥q%ïjìøpÅ»°Ûúl`>qXÛ¿ñÙ.Ī¬»,»QÏv7ñG£ºŸKOæy6DÆtØi½ÇW¯[ -ã(Þë«•ïåW~Gf@Þf7Ó|e¤2ßS›)«d课ö‘z.ÏÓ!uß_=ƒ´²B~b<Èyí¿ w¦ÝÎ -ü¸O÷Ü(¿íµÜ@%Û“Ì~ò%8?Bå¹™çéÐvà3>rŽí¨<ö¾ýíüä÷? ~2ÏÓ!ôñûé§øäÞîwäŸßæì›™OHgyù[Óyž ±ùà‰Æ{‘÷ þRå®(ç[͸Ág}ïûë™ù‡ÞLÊ»@èÙGä¹E˜*\K—k:AôzÓ-¥ù6R±¥J=k®=ßMéP·©u¼ +}´H#½lŒÍeë×µ˜Î™mc­ÃÙ_/XbÜFüvñQHí;W(_¡Ç…–•ÛÄIu¼t)ÝM›¿ajVJMÿÏTá›”˜§|Ñ®AŠN–ïW½š›M¼î$.uœÔÚÅž–u—Ž±a–÷—NJHO¤ÛR·[IY—1yÌÕ<"©Fk±n¨kb¢°Ú?+¹PU#P]ûå·•¸ÛéÛTõªU4ƒ»oåvqhÜùÆű¸¦×4{[¹ óUøZ67Š ú©VÒÅFµÕ®WÆ„Ý0Bw}+â:[;Ñ«Ïê×>ɤ÷õò·KYæV:­3XMo½áž—¾þ/Òü€ßÿ¾¸ëŸë1ëó:nTc Ó¸u%æ+½#-Ó;Óª°M&4ŽÑ~ëÔÓúŠ†¶ŽÃøÚõìª{M©ƒê¦åBßû‰Mƒ®Ñ–õŒß¾½Ø™/›¾`Ï—¶6ŽÎ¥ú«U¸àÍÁeê:Û—ä++W”´/éÐ:Ã2â‚ŠKVaî–­Ì$@òJZדýŽ…¥ºØs#èÒvëË,/ºÊJ+X¸À–_ð‚æ•æ•=‚ÑO¯/÷ÅäPgIµ7é—¶†‘´ž,µå2¬/ÛšcË!uòöjeY6÷lsrúÔè®…[]pŒ2iêÙ–žWêªlUi](]@¦ª¬•«MaÉV»¥´8‰½›OÑ¥ã=íù¨´Nô|TÂ&-•[W¼æ-ÙæyçØ[§zÊæK&Axî&u=¥­²mÝn]w]m¯¬®ºgªz6¿Iùÿ=2wÄw/H©êOÙÑ©¿˜ µÓÑ[Þ5«RàG¯ ‹.±í’»óìïŸlwIêþ ¯ Ã19kUÔôÖ`Ù„7(ò±‚Uß(,gËJlÒ} lušÛ¾)£¼6²ìg–M¢v)T¤·ÄZrz½Ü.*ùÍ岸Ú>)?ÿu…¢O -endstream -endobj -608 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 607 0 R -/Resources 4 0 R -/Annots [ 609 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -609 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 683.673057 101.433071 664.923057 ] -/BS << -/W 0 ->> -/Dest (cb130-1) ->> -endobj -610 0 obj -<< -/Filter /FlateDecode -/Length 4388 ->> -stream -xœí]K·¾Ï¯˜³Ó|_@àC€$ˆ $‡ ‡ÕZ²¬Ø9äïçcOóÑìêiÎìJjIÖb4ÓU,v5ùñ«"»g¨Î_+üH‰¢wt~üùôËI -o%þM%“ú^Þž•$/¬Š.ØóÏI`¼ðÑjïÎO'ü׶şNÿøêôŸ¥:k£ÓÊž}›Td„ÞK˨"TRFe­ð^[Ǩœ¸Ä9Ü°ž4­UnP –SÁ Ò:0ª7 yeÖª7´WÄ©à†ŠÖ3*%á‡ÒR²:8"‰ŒftÊ -‡FwœN+áBÔ‘Õyè”לÎá<)«8]Î98Á¥ât¾XE$9|!<§sðÅ8#Y|ÑÁNçá‹FëDFà ÐX|‘N*N°1±º >¬uZZaƒ1’Ó)%,°iXF:ÉyF§á‹“6²:øbÔœÎÀ§OXãC`F«N£ÛH£8…/Z{buðÏ1€vðEzÇQFH¼Äq€ö^PÔÄ‘€FP À±€Q÷†£­ =ÇF*è´âˆÀHøbÉqL`|!Œ\V_L$Ž Œ†/(Àq1ðE“á¸À€m há¸À€nIFÅqß’TŽãÂ…›’ãÆ5ÁÇ” (EŽ 8×€;9.0 ] wÏqëë4Ç´k(8Ž x×’x×X²:øs\vFÃqw’žãïbiŽ Ы8t+B‡ 9. ð®’8. ð®ö¬Î ŒAÃqwµÅ%p‘¼«­ÔxW“v(ü#Ùl¼ -±|:_@° ÞC§ ǨLh¤%Xð®Š̘[ ¢ã¸À‚wU@÷±º(”G‹3ã­,”s‘ã ÞE 2Xð.è3p\Žˆ¨šã ÞU‘šÓw•Ž’ã ÞEÞ`9.„BÀqâ›PÐeuQÈǼ ZÒ - À$¬Î Iq\2y(Ǽ‹ð9.pà]iá¸Àw%!íauðÅÍqòè0l™1dUH ¦àtà]©ÐÃœ¼+¥8ðn -›8Ÿ:p\²`Íq $¢÷Žã%tHùX8¶x e±:øBž8.@b-¢‰ã¯á‹Ñšã¯á RŽ ˆ¨}³Óø¸°ó‚/È9.“‰‰8.ðÖa>†VåtN‹àA׬.@§<ÇÞ“@(V`.%=Ç x‘Ò%Ž àtŠ8.ð¾`¨p\€P$‚FfÎé|A’Íq2]è”⸠hø‚lã‚ f™Nr\€¹$ššÓ‘LóÌÀqA 'RªÈq† fš`JV_@ ÒdÛr\€É&›˜æ°:øb,q\‚NÓÍÀqAðEKÍq°™æ›žã‚(á 2>Ž ¢t"¥É€Æ¡“’エ曚8.À¸LóÍÀqA4óMLñ˜qKóMÏqA$øb‘¢°:ø‚¤)2㜄ù&؉Ó9ø¢#q\KóÍÀqÒ Ì7‘ѳK!Í7=»\ÞµÌÄéÀ»ÈLÇ8À|™-7·— ^ë‘ŸñÊfœ]1 ^‹ØÊ.Hp/¢•g× $È7Í‘ØE¸‰™epìªÂ*”J²Ëü‹|ŒVœpY}j—¨àà3ƒO1,Ù}™sÔbf£Øe}ì×Oß< øžüïéÝWÓJYþSH¦Ò„ü -Ã@Gt·Eö­pùÆ¡£•˜„4yï±INøXT ÅyÑ-9‰ð—þמÃñWÓéM9}D¿œ><°Jê«€éë´6Øx|£É :,19B´u(c¢NìÈ!¯”´(üû×'uöâ²Z÷/1&mϯ>}ó—ïþþ×ï_F¯ßþù;)­ýök…w -—w {:™›ßý,SuDó»ïlæz®ZæÝòÝêùý¡©ÏW;«ª­•¼OZ6þ=2¾†¥½ú¡©3¬¯k:¯Y^CñóÍåÒT§òï½ ¿ý×ùõw§?¼^sÊV'`RJnKmw°›æ]ÏåâûåH#oçÈA›£!Ó\Ï;'î˹3—Ü›Í ÷¾›{¸}ey⣇yVô°Ýóe–ÑùRxþ¡Cy¯ÏèhGU“/ëvÍÒw6ùz¹ÙÚ/n¡ÏÜ~÷eÐæpèÛÉ!·Æt›™qÑtŒëú,dkö¸‰B{ûÜ{Ðæp(Ë3×õ«!C=z˽—M¾ë{Í9iYï‚ë>Ùwí¾‹§~†ÞÌÌ{,ô±wnòݘ–©˜lqëîÈÄ }ýùQŸ¸ö¥Ì£˜»@eŒte'?š±·¯ÉÜΈƒ6‡CåNÖ˜îqÙ{ωi×¹B€¯v· `™=7ó†žUe×íýÈþ^@;oëæv­nòigCw<9hs8DîÜ…nî,¯îH7™ÑŠ»Ú{‚$³ãUï›ÁÞ¿ÒÃìì½ÍܺQÓçºAa©«»G¸âÐþŽwÿ¬H¿öäº6É«ýZ‘ºè¯Äñf-qt†3hs8|^Ï!W}±ñ8mwŽ›iì­•:ùþ^<~{+–8\¶¬õ!°Öú³»~Á3§m³Å„Šzï.â ÍÑiwî{¿‡X~ä•O"æ1ÛÜÅÆì˜Íá0{=ÿ\þ{î¢ì°${ÇÄ2ÌtåŽÉsü»õè{_%ØbÒæ©Æh…nËì¡rÌæp¨ÜYż÷» ¿­½/ú¦vÜ@e¸ƒ+Çl‡ÊÁoØÜñ…Ï‹¿î¡/º¦Ï#ÕÝñí›A›£!Õíd¢__mŒÈçL>ÌHÜ@h“CŽ>«1hs8„î¬{Þ³Fyïü– ?ãæìí3ùA›Ã¡ó=域hfð¢#îCDúpû“ëƒ6‡CêNNúIòÏçéýß´9BýýߺÆ;7?ßñ[¤ŽôÞÜþ4û ÍáÐy=½{ÞûFú{ž¨zß‘ÞÛÛçôƒ6‡CêNNúIòÏ °#Gúpûœ~ÐæpÌEÛgºï+džh¿¯6ñUs'ÞÎ=ð[T¿-ª‡;îÄÚ ‰a'çüçïïct}€îø–ù ÍáP¹³æùIrÍ ¦ÁÞ>+²@âCƒê+Ïš¬~T1–»œáŽ{ïƒ6‡Céد­¾Y¾ñ+:sñÚ|þíŸj}‘?àõn|FÓgÅv°rùq.ùxýÜáB›o #tÌæp½qͳá´6®xk ‰×æ˼ô)ç˜}–³z¶~#ªÇ;îµÚ q,×|‘§ÕóѸõ¬ýÕßðÊOç%Íß=Û¬§¿&._džµ¿'Fì}gn‹+£¹+m‡ÎÁo¥é¿òûÁÖy7yÇ÷‹m‡Èü²ë‰ý_i¼“îüúÆ-ù`ûÔÜÍ¿¾tøoÀ´¾šw7ðkmÚ]&XšÍ -Dw‹ì r½µBÞ àœþzȹk-Ò¶¦&œ’)ú´H‚ØwüóŸþÖ@,ýȨ~{ùIßK(Ãg‡šÒ¸Ëqjª 2i)$Îe¦'ÍÄu!-;ujÎK·ÈùÇ^Ò{ª+•§w—îIrʺËkªkþ<Õ2DæóM“ÚÔ-)áºèõÛùÜ?S¶¹†ü }\uå/'7íj£¦Í¤ Bé´õéÙ‘0©oâ9í¤¤!•»NM­Î–Ó†±¿”Ró²2m+—¶ºl0ÛÌå™:ýç“Š$Rû'¸ì÷('qÐù ŠŸNi¿ObÄDBŠ“pþ(K]¤rÁ"{ïzû’Æ -9Œ¥rcóGmóÕ${›ÍÒ>,$s!3íDr91#„¥Skqºz)Ô,N{¦aæ#W\±.û a­Ú²µÂj¸+VF6B»ÊætU¬)û£Ò[¥é\×Û¦SM÷ç²Ú7ÝyMøxÂiM/}b¥i§Æ\Un]-‹Ì4NÅ"µ¹dé*ýĈOŒðk™«&tfÑ7½–ö»ºC•r1—cD'¶Ü\›-=åK©PMcnùêJ…º ¥áÙÔF³Ô§M ›1"Mþ¸»Ô¹¯ehp£WÂÇB ­ÄO'ë™Ò¶r…´z­Ð4õq/Uš~^àò C3ËÒnAéÿ UØ"MÌ3}PT •vüªGõ$ilñ²‘Z¹+¬C\ (òµlÓ\Êû‚Ù¶½ 1åé–Ò¥ÅҦɹٽDR…<–bUPWĉÂrû®l…2;ÓU±-P~Zˆ«Ÿ¶ U€=ŸÌ©î¶Ô[Å®pçS+ö¥ÇUÚ¾­ø[êm0Ÿ…S`£^œ òIªXËíêÉau ¤PÅ•FŸ&q­…¦ßòž?ÖAÆEÜÇÓO§)ÌicuÄgTڴͼÔù(ó^ÿ>Ñùû¼ýÕ%ŽkYØBn]ˆÛH–ö®ƒâ´—u–aÓ55´_Å*Ô²6£¡Äq­¾¦j.+±ç’ª0¨*ÚVhk6b] «•iZÆ^>v­X™çöM Ø—C“/§1iã´ìÚËÁa¨:SCò¹©G©h éÐ’n qÐVŠÃ愱z¶p3²=Iiúä?µ]aÒ¹šƒ&H‡¦ê¨†Ù¶ê,›®¢)‚Ø´mŤړƅ?ŒÓw¯g÷ûì²Ë# I µéS+-–¤…ðx©™Ýò°ÄsNyðÖ*u™fì™’äÔ¡QS Z´m¤¡šÍl{RʲÅIs (t™Ì²R£Ï>¹Ù,7ËtÅm%_rŠ*ý©miÛöJiDÛöŠ[•MõæˆW²%S ÛÂÙ׫“µdÉ%#s­u‘R-i²l}nZž;GÛssTq53•ÕÌ®JÞÅ÷Œ‹M·Á•ÊÓêˆ -uÃHjRÞhsŽ)Ý ÎC(mãì)4Ë%7Û\_?Y¯’äõ“öLèAÆH¯€· ž ˆûŠ¦ª§y¯MÞ,Jö’nkfy¨Òœ\­aæeÍi‘¨ú4U´qÚtY†6÷R±ËEÓ¯¤„ÅòÉô÷лv -endstream -endobj -611 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 610 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -612 0 obj -<< -/Filter /FlateDecode -/Length 4803 ->> -stream -xœÝ]ˮ丑ÝçWäÚ@Ë|?€F/ xŒiŒÓ@a¼0¼¸•UecpÛ@{þ}Ÿ D2D…”ÊÛ·Ëiwáv¦"ø’‡‡”ÄÔW…ßhü/9=¥”cp×Û—Ÿ.jŠ^á¿’buQÔ?à/ú«V.N^çüõGØ8ÅìM ×× þ7\ò䯗?üêò×u}5SÎÁhýÛgR9;Å£ò‚*C¥´TÞO1UК˜³¤‚>:㶪3\N^RÁ gLT fXµÝª2Ì0Q;I3töQPi;´QJÔÁåœ5‚Nû) Óƒ¤3z -)›,ê"t:Igí¢Ó^Kº<…|’t¶ø¬´¤ó°Åk甤ƒ-Φ(él±Á*Q[LŠVÒEØbÐ;YÐ%Øt%Q[TPZÒe;ùœœu:•bÚêŒò“OÖ*I§õäM+ê0£0H! -:[‚òYÔÁo•‘t¶8ü“tà ocJÂl54»­²ZÒyØbLt¢¶`ð%0¶¨$ -À šˆ—$01N.'‘€IvrÉ%‰LÊ“‹ÑJ4`²Ÿ\ÈQâ«4tFKD`lñ.HL`5lq˜¹¢¶Øì$.°¶ ÄÖÂã¬Äl뀉 ,èÖ©¬%.°à[§t¸À‚pa¦’¸À‚qm -Nâ Ê”²ÄœkÁX.¦{”¸À‚u­Fâ Úµ.‰ ,x×:­$.pà]k‘SÔÁXâ°ãdu²8ð®Õ*J\àÀ»˜FFâŒê‰ €•É¤¤$.pà]“”“¸ÀwMÄ êâ„9h%.pà]ãÑiå﯌ļkœ (øG‰Þxâew¶€`%.Àz± °ÉÀ-‘¸ÀƒwuÆò'Ìi°Õ¤Sxð®N>Q—'Ñã¼E/O:„,qïb¡²xð.è3I\ŽŸ°¢‰ \@¥‹ó.}»dæãÓX¿~øñòëÿúþÿû÷?¾\þø­RÞ÷Ƨsó§a¹°È>â/-ò%í-·<Ÿ»^ûEWóù|Þ)+.é¿,ºÀtGeÝÝFëµ]þFaÌ¢SóŸOëºZ^õÆ:U￦[>½Y>kyº·Y«ïþtýðýå·Ö€+qRœFH¯Ñ{u£ç²üŒþÐÌùš­ ýº»]†ˆui¢:¬ŒêiÛÐ&V^¶PSSÀ÷Š5¥]²UMÉ€S£M*mùþ?þówÿÃÚ‚`¤@SûÓöýãü‰ø^™Ö¦µÝ >žìœÓbõÄ­í¯7ò%mƒ~èr*k†5õõ,›¡¼\ ¾Ëg\Ò~^ʺ-×µ¼/½îbÙýÂÊQK¾VÞ<Nuuà]]rë-+=À[™,M¹¶ËŸg-]®K+–^­-Q DãX+"˧ÖéÕ{[Z–æÞ„TpÚ,ÝçZÚa—ÑõÔóiîÅÒÓŸ¶åKA ÝhØ–£–vÍm.í>²ÎñŽª·eÕÔßfžy‚mtÃ)(I({Fy/sÚÞ×ÔÏ–>ouLÔ.ßk[É{ÚÒþ´ ùÆävLè- Qî(¦ë¢§tuÑó‹^/åìöisØ÷~ìŸRÖ2«ê¢ZfäK1 &̺Ýev+fGmsÅSœ?KyŸ|ùõ¬/}vÛïkb-_q æ~Pv݇ã86›BŸmÜt·ì{ÔOqì§ÖÎ8àcOa¿¹ŸZŸŸd<ÏY¯öUÚç:Ä~³lŒ¾t; N™ [ÜöcÞôc^ÚéwæëŸÊ›{ýSðõéëö·QGíµÖ íù{Í?•ÃFGÌÒN-Á -ÝŒ3I›»žØÉ؃žš |ù>ÔÁ£‘:BmäˆÎN´ßÙ² O;’oè}ºÎ q·¥wòÓè[­y¿ÞwÉ`Æ%×å§>³Ñ8¬k &Dž¢ç²1| ‡É\;5+GÓî9[猞îÈtu¨´Ìö"t*R£'@”ÓewþOæy’˜´ñ¡óþˆÕ‹L}½µÙ×èËá:Ø­iÈVµÃUÞ8|Äïضj˘¦û¶û˜nxˢczq¼ –“èܬð9ûõPáž=@¶`Ó«prð\žgó·áÏ ³føŒkœèŽ¼Ìß7òå¯ÉÝŽ®ãì–<»Î–Î.Ÿ–O·|§ò9ç~­ðÛ>ß[™ãFÔmÝg{ûzôNE=S‹FÑí³cÔËó$¨ëf}ˆ:»F[c®5Ï°§x©i= -Žï$¿ó.÷y$<ØÖ`×M¥už·#²—Ñìx9@jì\g .›»H=—ç階ùÑȉ5yóöÐA›‘–ÐõOºï"¡ß{9,-—õÕšããH=—çÙô1§ªQ™§ù2ðe˜W×÷¥;´7¤³U¼Î´bŸûºöµÙSËù4ô—ëžÉÚ&•¡ûŠÆ„I™hïú—'ó<*ýË#?m\ÉÅUsÌ#ðÙÞ ÂU¬ן{ñNãÀ½‘uóÍÉ,ççŠÐ»·ª«2N»7æèi¼D/ÑÜÃé¹Í^_½Y¹Aó˜Î>6u2ϳ¡){YGëÙŠ‘F”ŒˆÒjó GÖHîo7Ë,Êå(ø².cd½M< ùðµ¬ÑĨ{qÈ=oϘ6ÓF–ü$ÄZ·µmåþÓÎúù.ÝI¤žÌótH½³³7pÚˆ¶²é¦ÓÐhÓ>­¡¿eTLFZ¸Cø^Xñ½íŒPéscÇ^ÌmQåY]oŠæwȽ>z+*•7YŽ‘x.ÏÓ!ñŽ§øÌϘŽœXgÍ°k·iC­óvyã}— bGtíÝxùìêä¾a÷ïdž§Cî±ïøÌ;xG¨þÙ¨bm‘¼A0÷CݺŸF4JkA‹»†;4›=™3«÷9]L“¶Žží¸ƒÜsyž ¹éÎn ÐÁ#M¶Jó•öž70ß±Fi\ÉÇ»>\ñ.ì¶>˜O¼yz(íB¬Ê:¸Ërˆàõlwã-þh2séÉÏqר3@zv~¼ÔÚ±ƒ6¶yve>™çéÐvÇg|Ë8¶£ò<>Øûrô×ó“wšüdžgC¨Vo¿¡~ŠPÝâ~Gú÷œ ²?,'SEÑõžNL^'vÐUÃéÝ$w`Y_&ÚÝsÕÃqIë÷P$IoÓÕ}9Ò)n{o™ÏóÓdzyÕ™îd—'Ŗת©«ÚAyISßR*ó0g¹‹¡Ú«úõuçòÆÒòjsy=9ÏrS_¯¯B/ßû!ª½¶nÊ!j~Å™=ÑÊ?8j µÁ×6l†ò§K('êr*­ -ð« [ n²46ùJ§ &e®C§KgëëNÎrØÿO-Õrþ¿¢#é8È2´übIÿSô/:»‰ú¿œñQÎêVEœL½èâ× Õî±sý ;b._U+ËévîH•Ý@¾qÊcJë'µ|퇓ÐëVóWãkk(¿¯Ùè =§j"[N‘›+„ÈôVL­W“^ÄtÞ}šìrš)ôJröOÛ ìULè·BŪëbãª=šÎGm]º°™Î»N³á¯iMdÃy$¼]P­¥¯¢”NÙ®EÕÞ5ªÉ,3*7©¯)Û€¸6N‚èv„ߨZ´Ã`6=5:«t¦ -$×ÒåšNÝ.bº¥4ßF*¶T©g͵ç»)êtžË¾ìÆð‰ôlŽ([¿®Åt^Lk•nÌFx»`‰qñëÅG!µï\¡|…ZVn'ÕñÒ¥Îß05«ŒNz¤ÿÏTá›”˜§|Ñ®AŠNkíW½š›M¼î$.uœÔÚÅž–u—Ž±a–÷—NJHO¤ÛR·[IY—1yÌÕ<"©Fk±n¨kb¢°Ú?+¹PU#P]ûåו¸ÛéÛTõªU4ƒ»oåvqhÜùÊű8=UÑ×À0_…·²°¹QLÐOµ’.¦#œ*|ZeLØ )tq§Ñ×"®³µ±ýœHýÚ'™´âÞ.¹”eÎXoèÜ‘`5$Ï=/}ý;Ò|¿ÿ»¸ëïëÑ¥ó:nTc Ó¸u%æ+ýîH:)¦ß!©Â6m˜Ð8Fû]¬SOë+Ú:n ãk׳«Nì5¥n ª›– }ïC&6 ºF[Ö3þ@xz±92?wlú‚=_ÚÚ8:A…½­Vá‚7—©ël_’¯¬H\QÒ¾¤Cë ˈ ^(.Y…¹[¶2“É+i]Oö;>–êbÌ KÛu®/³¼è*+­`Ià[~Á všWšWöF¿yzýìq_Lu´Q{“¾rikIáÉR[.Ãú²­9¶7R'oï¡V–esÏ6'§OîZ¸ÕÇ(“¦žmIày¥®ÊV•Ö…¢ÑdªÊZ‰±Ú–lµ[J‹“ØK±ù]úÞÓžJëDÏG%lÒR¹uÅkÞ’mçpŽ½uª§l¾ddçnR×SÚ*ÛÖíÖu×ÕöÊšáº!¡{¦ªgó›”oâ¿·Ì9Ï°!¥ªÿÊîˆNý°o,ÔNGklùý6•B‡8ú ŽèÛ.y8ÏñþÉv—¤îŸðš0“³VEM¿Ä'›ð -E¾¯`EÕ_铳e%6鱶‡:Í©ÍîPù)¦²·Y6‰Úe¤P‘~yÍ’Ó›èc¢Òòy­/›í“òïºØì -endstream -endobj -613 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 612 0 R -/Resources 4 0 R -/Annots [ 614 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -614 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 573.873057 101.433071 555.123057 ] -/BS << -/W 0 ->> -/Dest (cb133-1) ->> -endobj -615 0 obj -<< -/Filter /FlateDecode -/Length 4396 ->> -stream -xœí]Kܸ¾÷¯èóËå£ø‚=H‚Ù Œää0îÝÙE0àÍ!?ÕâCR©ÅÖxlÙkÆÓªbQ%òãWEJjª³ÄÏ· -ÿR"„è/ïNïORx+ño(19Ô¯ñëíYIòªè‚=¿Kã…V{w~:á¿Ùa[üéôoNÿ™Ú¨³1:­ìùן’ŠŒðÁ{iU„J*è¬Þkë•S—#§‚Ö“¦¥Êà ŠÁr*¸AZFà†!¯ÌRá†öŠ8ÜPÑzF¥$üPZJVG$‘ÑŒNYáÐèŽÓi%\ˆ:²:òšÓ#œ'e§‹Â9g§#øb£TœÎ«ˆ$§ƒ/d‚çt¾g$«ƒ/:xÃé<|ÑhÈè|º«ƒ/ÒIÅé¢6"V¡“Á‡¥NK+l0Fr:¥„6 «ÃˆB'9Ïè4|qÒFV_¬‘šÓøBøátà k|ÌhÕiti§³ðEkO¬¾ ó9оHï8 -À‰—8ÐÞ Šš8ÐÁ -8Ð! -òÞp4 £ä¢çxÀHV _,9Ž Œ‚/„‘Ëêà‹‰ÄqÑð8.0¾h2°--Ð-ɨ8.0à[’Êq\`@¸pSr\`À¸&8â¸À€r¥Èqçp'Ǥ‹áî9.0`]cæ¸À€v ÇqïR’ãïKV_`Ž ÀŽÂ¨`8. ð®QÒs\@à] #ÍqzU‡Žã`Eè$ÇÞÕAÇÞÕ!€Õy1h8. 𮶸.rƒwµ•šãïjÒŽãE€$› €wA!–Oà –ãÄ{è´á¸• ´„ã ÞUáÓ`+¡BtXð® -è>V…òhqfÜ¢•…r.r\`Á»T†ã Þ}Ž ÀñUs\`Á»Ê Rs:ð®ÒQr\`Á»È,Ç€ƒPH8.@|Jº¬. -Cà¸À‚wAKšãDa!˜„Õy )Ž @ÆB"å¸ÀwÞ#Ǽ+­18ð®$¤=¬¾˜ 9.@^†-3¦‘¬ -©Áœ¼+z˜Ów¥ô‘ãÞMa“ãç“@Ž à @¬9.pDôÞq\ࢄ)«sÇ–ã/!° ,V_ÈÇH¬E41p\à5|1Zs\à5|AÊÀq2²ovz_v~@ð™"Ç`2"ÇÞ:ÌÇЪœÎi<èšÕè”ç¸À{ÅŠãÌ¥DÀ ç¸/RºÄqÜ€NÇ> Ž ŠDÐÈÌ9‚/H²9.@¦ R _ r\tÀ,ÓIŽ ¡1—DSs:’iž8.äDJ9.À0ÁLLÉêà h‚ãôOšl[Ž 0ùÁdÓV_Œ%Ž BÐiº8.¾h©9.6Ó|Ós\%|AÆÇqA”N¤4™ãÐ8tRr\UÀ|SÇ—i¾8.ˆFb¾‰)3n£qi¾é9.ˆ_,RV_4EfÜ‚“0ß;q:_t$Ž ¢si¾8.@šù&2zvI ¤ù¦g— À»6‚™8x™©ã¸˜o"³åæöÄk=ò3^ÒŒ3°+Ôk[Ù% îE´òìšù¦9»h71³ Ž]5@X…RIvÙ@‚‘Ñ‚®«O핛á´3oÌ-HMî¦=ÜöN™*äîyµ½\Pñ8ë͇¦¾²3B$ïSIÅ¿ ãk˜Ú«Ÿºß¾xò(Öòþ9y§ÍÑP¬o碻s>jd¿A¾ÞƒÜU4Ö|ҦNJàpZA¼Æ>›Ã¡ñvz7?ÎÑxO®÷1ø;ËÞÖ:æ¨ÊÝQoA!÷¥ŽfD¦s§QhFŽ6vÉí…§gç\El³@é"úY‘t[ˆí³9boçª:l÷î'AÞ…s¹ÆŸ=ÈÛ&ØvøÕ“‡êf)²}6‡Cà§ÏCí¬ž2n䛟e&¬ûº³r%~Ìì¶8ÑÔü0=„‘5ˆì´9"Mçú¦žõ¬¯=«Ò½l?þºí^™äj—iOq+ÜEçf¾¬åµ³•p;CÁÜ=¢ºð~þk¦öiÖ”òŵõGÓäqJ‹(ƒ–›¨é³9j6r¿÷.&<3Ÿ{θiÈgºvżœãqZO²+½+rÚœ²Ø¬ñ*w¯d6gŸç¬YÿÉG@ç:¥iÖ{g+6‡Cì×uÊÉÈÚªk •›k{Ѿ‚Ðf½±÷Nc§Íáz;7ÜË¥7rË?Œ(œÝäyã·82x$’¼ÿžN§ÍÑH9á¡xã½Í_Xd>ÌPFSæz]9ï>›æ¯ËÅ—åH2÷sd§Íá¹qß[5¹3—ÜšÍt÷¾{¸ýÍòÄGã¬èa½çË,cæKáù‡ÊçúŒŽvT5ù²n×,ýÌ&_/7[[ãÅ5ôíx&²Óæpèë|r>¦ù ͱòÌy²þäÛ - -ÃýsïN›Ã¡°/OüXЮó…\ÞÓ¸4ç¤i½®ûlóP¶¹ßìƒP†¼ÛZáî´9:í3rÇOÊ+pä¬ä¹‹{%³Uï2²Âøk¯¿¹grÊÅyì™ûŸòé´9öng‡C»ùq7µåÛÚ†Z7í™3ºgŸgí¾‰§ù ½™™/žž=ÄÞ¹ÉwcZ¦b²Åµ»#ƒÎëÏOûÄ¥/eÅÜ*cdVvð£{kñÚÚŒØgs8Tnd™á.ÓÞ{NLÛõZ‹¾½È칚7ÌYUÎÚ£½9¿ÐÎÛfs»V7ø´1±;ž†ì´9"7îB7w–w¤›ÌhÁ]í=A’“ÙñÚÓÙ›½£‡ÙÙ{›¹ÍF ÷þÅ‚g÷:¿ã=Vd¾öäfm’WækEêª_ã®YKìátÚ Ÿîv¹è‹•'jçw縙ÆÖŠQ©ãÂ÷÷ä Ü{±Äá’y.üE±Öú³¹~Á3§k³Å„Šzë.b§Íá¹qßûbù‘Wv>‹ ˜Çlû¶M/fûl‡ÙÛùçbô﹋²Á’ìË0Ó;&ÏñïÞ{ /¾J°Æ¤áþ7;m‡ÊUÌ/úýÕÈÐwµ#J/ïçÊN›£¡Òw¾a³ã…/‹¿vŒÐº¦¿‚ÔoßtÚ©™èo‚¯VFäÆs&g$® ÔÞÿÍ6‡CèƺçÇúꯙA÷3n>Ü?“ï´9:_(ÿüL3ƒ—ü²«—ˆôAÞÿäz§ÍÑ6rÒÏ’¾üHv¼ Ôis8„îèïÜý|Ç×HßéÃŽo(ê´9:oç¡»ç½_`¤ßóDÕ‹Gúpÿœ¾ÓæpHÝÈI?Kþù@#ìÀ‘>îø6ÌN›£!4væ¢í³@³÷2O´ï« |Õ܉·c|ê÷Eõ¸ãN|§Íḑs~‰ó÷—]!‚Ço™wÚ•kžŸ%×| ÑÔÁÛgE&H|hP}ãY“Å÷*æѲq—3î¸÷Þis8”ö} ÑâÍò•oéЙ3îüëùtPë‹ü¿ý3šy†Ql;Û!—ïGà”—Ï„¾÷ÎðimÚ»(Xš­8 -,7‹l p¹qGÞþâœ~æ0s#¾‘6Í5áŒä¢OûÌ$X½úãŸÿô·Vé‹ŸôOׯY»¾Ö‚Ï¿ ã®Ç©[†€RxŠc™á £2!^É0™'\[{%9¾€›þ¦ºRyz¼Â2Éu*뮿C]ãç¡ž¡7žo šÔ5Wý¤~;ž»ñg¨Ã6×nŠ‹®|rÞIjØ7Oº ”Nëž “ú&žÓ>MAR¹ëÔÐØê¼b9lGü¾”w(–iÓ´aÕuûâæ`,ÿ~èôw'I¤öO$qÝMTâ óA?Òn²Äˆ ³˜NqŽe©‹T.Xdp­q^ÒX!DZTnlþ¨m¾šdo³YÚå‡d.d†}n®'f„°tj)NW/…ÅiGÞ ÌxäŠ+ÖeŸ!¬R[¶VX 7ÅÊÈFh—BÙœ®Š5eTÚÁ­4«ÂâzÛtªéþ\Vû¦;o /'œÖÌ¥O¬4íš«Ê­«e‘™Æ©X¤6—,B¥ŸÑåÄ¿•¹jBg}Óki7µ«0d Q)s9Ft9±åÆÚlé)_J…jsËWW*ÔM( ÏȆ6¥>m1ÞŒiòÇ©Ø¥È}-Cƒ½^N1´?¬gJÛÊÒfèµBÓÔ[ÄAV¼Tizåëú C3ËÒ^Téÿ+UØ"MÌ3|PT •ö“«Gõ$ilñ´‘Z¹+¬C\ (òµlÓ\Êû‚Ù¶½TLùDº¥ti±‰´i²Fîcv/‘T!©XÔq¢°Ü>…+[¡ÌNàtUl ”Ÿ&âê§-C`Ï'sª»-õV±+ÜùÔŠ}éq•6,þ–zÌgáel4'è‡|’*Ö²D»z²FX)Tq¥Ñ§AœGka§á›çÇuq÷rúå4„9m¬ŽøàŒJ{ݶ™—:ÿe^á÷ß':ÿ7W»Æq- [è­qÉÒÎè¡SœvJÏÂ2l¡¦†ö«X…ZÖf4”8®uÃ×TÍe%ö\RUEÛ -mmÃF¬ tµ2MËØÂˬK"óܾ©ûzhòÅá4&mË—½ÂA{98 UgjH>7Uâ(­!ZÒ!ÚJqØœ0VÏ&n&@¶')MŸü§¶+L:WsФéÐTÕ0ÛVeÃU4E›ö ­˜T{Ò8ñ‡qz÷ðzv¿.»<’´P{‘>µÒraIZ—šáÐMKÌ1çtom¡R—iÆž)IN5µ ÉA‹ÑFªÙXÀ¶'¥,›œ4ŠBÉ,+5úì“Ír³ WØVò%§¨Ò_Ú–¶m¯”F´m¯¸EÙToŽx%[2%²-œ}½:YK–\202×Z)Õ’&Ë–ç¦é¹s´=7—AÕW3SYÍì¢ä.þÛ3v®6³•®TþVGT¨Û‘"P“òF›sLéNpBi“pO¡Y.¹ÛæöúÉr•$¯Ÿ´gBw2Fz¼­¸ðEÜV4U=;¹òfQ²—t_0ËC•æäbuÈ3.e‹DåЧ©¢Ã–Þ2„´¥½—Š].Þ\}˜,Ÿ ?ÿ,ö“ -endstream -endobj -616 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 615 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -617 0 obj -<< -/Filter /FlateDecode -/Length 4878 ->> -stream -xœå]K䶾÷¯ès+|?€À‡I# b`‘Œv{wlãNùûùŠÉURkfÇ‹6²ƒÙnUñQ,~üX¤(¾*ü|¥ñ_rzJ)Çà®·/?]Խ¿’buQÔßâ7ú«V.N^çüõGØ8ÅìM ×ç þ.yòçËßuù×:¾š)ç`´¿þû©œbŠQyA•¡RÚ -*蓼*è MÌYRÁ q[U„.'/©`†3& ª3¬‹ÚnUf˜¨¤‚:û(¨´‚Ú(%ê`ˆrÎA§ýàô 錞BÊ&‹ºŽFÒY;…è´×’.O!Ÿ$ƒ->+-é‹:Øâ­2’·Ižð6¦$ŒVC£Û*«%‡-ÆD'ê` :_b`‹ŠA¢Œ ‰xIâãä²q ˜d'—\’XÀ¤<¹­D&ûÉ…%°JCg´DVÁï‚ÄVÇ‘+ê`‹ÍNâk` H\`-l1ÎJ\`Á¶h‘¸À‚nÊZâ ¾uJ‰ ,f*‰ ,צà$.° \@)K\`Á¹Ü)qéb¸G‰ ,X×ú`$.° ]ëR¸À‚w­ÓJâÞµ9El€%.;NV'+qïZ­¢Ä¼‹ad$.@¯NÀa¸X™LJJâÞ5I9‰ x×DL¢.NƒVâÞ5Mfnð®ñÊH\àÀ»Æ™ q€2” €wA!^` VâÌ÷Ð+q -› ‰ Ê­ÿöÝdºX1ïÓ·Kf>b:ÉÑøë»/¿þÓ7ûËŸ¿ß=]¾ûRîéë¯4}Þ–OL½.-ßÃüéoý{K¿¤QvþÓ•_÷ ék~¿NÛÊ3˧^ê«u¿Ç¼Båà?Or·¤[Ê!ùÆ^¦«éZ>ÍìþÐíðRžO]¯ý¢ kûkZŸå²ZûÆvÇšæë\ß}sùÝ»5ÀÊÊh&ÚlGý4¯bò\–ÏÁäÏÅœ¡(ÀÃÇÁõf€á“ÜEZ>Gh{;ä©6-6è¢ÍÚ.Jóçn9c›xýÌ‹}ËðplhÔt#T7uuµÏ -M%C“î½”/Oæy4pbå~ÕóZíôXbhø…YEň®—¥¿±¶Œ#hÚDÚN}´­‚µ+­Žy.ÏÃ!ÒòåØÍ«C/44¼–Ã$.wsZi„Ö¶Î×ù’¶-Ó¢äà ,r¿è -ÒÉo=)_Wߟ(óÓ"¿å͈?åÎÀÝù]m¥Ÿ#îæ‘jAõXì^k–s¯…î™ê±’fÌc—_ÏÆðr]Z³x¼–«À•Æ1»"˧Öé‹o‹'Ââ‚ÀÛní¶[:mµ´Ý.mð꘽IåéÛr‚¥[tÜe[ŽZÚ5·¹´ûÈ„e;v¨Þ–•?nÔîyD -¶Ñ±§ œ Y(»Œ€<ç/hyZÊzÏ‘Ý}n>-¾¨6ùeþJÝ×ÔÆÒ§q)wñ_ÇÍ’V°¿#¿£ÞµÂN)ËÊöY\´ÁûØ~(6.6Ôö™dŸïöŸJÚÔõ£MÛ¥>òsÿ.µ{3Êuoc-¿ÔùÔÓWl¬™‰õê°ôy|»²çÐwqô]£fÝ_Þv:ÎHá ßìèl£®8½Íý0ÇFÌwù†ëv¬1¥wï×x,iY[Jl¢XÙµ•âüY#Þå Ö†1¤kw0Býïë¸W³”]ûpìïfÅ2Ã[i÷ÒÖõ¬s0öFl-²qvÛëÿtä7§7~Ç›ñöðÓx·b„ÏžÕÆÐÇÖçÌÚFmÚãè~›Ñ&h¡=¯ŸÑź¸ïè®ëª®yž\óC+ÆXåOÁ–tôÎ$mîÆÂ'ó¼êöÐÁ¾Ù›Ý”¼ÇƒàBã[WŠ`âÜ »æ€e«*ünØâiAv^äÇå›'p…rýÓ×wr»TÎ"†°gÕü>LÙ£ï¤ü%P–þD™å˜¿óÁxêh›/kµž{ÁùDû-gn¬–l½ë}:y -ç…¸ÛÒ;ù©÷ÀcÖ¼÷]2qÉ!D<ƩϬ7ës"OÑ)ÎìÁEA\¾"¿ë`bÐÛ½ }ÓYîL§&‡JËhß!Bt%5oü¹ÛQ'ó<Ȧ@ãCýª÷|ª6ëèb›”ãv͸¹z´ñ¨Ø&ë8‚«|³ º³½Ä7)j^³°„]–¯&vÝ‹Û9l{}ñvkA‡£±fù¢fXص±¶âeŒ%1€3t!ûõPáÑ8£ÓDųpnkødžGgΪî³{7'×¥ÌûùûFnHG×%\ò|`ãðtvùý¸|ºå;•Ïç‚/53ÿ†ÏZæ¸Ay[ûloË—Žv7¦>yØàdžA]7;éCÔ ·Â6·vØé—³·öØó縩{ /lk[p/2µÎózDö2šïš;×YB†Ëæ.RÏåy4¤b¶8ƒÔÍοìcøm¹ÓÈÒrY_ -©QÇ#õdž‡Cª=æT5 êCÿ>o<2¾ óìúö½´cÇ€ö†t6‹×‘ÆA|)ûÚè©å|üUellÚAe "~{7¾<™çáPy_:aûUŽÛ;óÉÓå<í&låÀ½ž ¯à›sy®gïÄpÕ“76KÞ¤±C7~V·îÇÞÓ¬—DPºiD7šÚžÌkó`«š°FÏÛxaïtõPÇÈ/þýrùq©–ósEèÝ# ÕF§=sôLP¢Gùïáô\žGÃi:Žà6s‡f}¶ÄÛµï7yBÇÀ8÷Œ‘àkŽï­RÏÌu|gf³ûÅÆŸcc§}Ú¡}ç¨o÷”<‹˜Î¥;™çáÐteÍg+FQ2"J³åsÛØëOërÊ -±²Xf«\Ž‚§u#ë‰öÇhi8:ÜV Oûë{Ñ1ã!ØUä5®µnkÛÊ}±ù9ñ]º³H=—çázggoà´meÓM§¡Þ¦}Zãû.¶ BO w.ß" -+±·*}nì¸s ¤QåY]oxü8ñ¨ïäaù“y‰w"ÅG>~ -³ƒÜWìþÌóhÈÍDZã#ïà¡ú³QÅÚrü Ð‡ºµŸF4JsA[w wh6{2'fÿÜcNÓ¤­£3'ÇÈ=™çá{g7Ðèà+M6Kó™ö^40ß±ziœÉÇ»1\‰.ì¶>XL¼÷`Õ°ãçoCYwY<¢žín¼&Íîå\z2ÏÃ!òN<ºóèíj_do…Â8êÿ÷ öŸ©,öÔfÊ*z7Í1RÏåy8¤Ç«gVfÈŒ9¯íñË™óKuvü¿{¼y•lOòìCð'ó<*ÑÇÁè=}Ñ'Ä_ëWË{˜7h~»¼1*zô¢÷ôƾäub/ jx¼›äüêñºí u®zx‰ÑúŠ¤WÅÛt5@YŽôvµ½ówôp•i¯Ê˜÷vʽ“å*r}{¤1/iê¹½âÒÙí¡åÁ—ú A9÷<ÄP$ȳܴÓT?KÔæ­R_yPÍ#°GZùã´6øÚ†MWþt åMº¼-V¬ ½NþÜDÏ=Å|¥·&e®]§‹³õu'gy ÿO-Õò^~E¯ê¥×4–®åKúŸJ§ÿxÑÙMäÿò¤dy‡¶*âdêE?_èêN;×·¼#&áòUµ²œnOoVÙ $§<¦´~RË×þˆ'@œ¿–3É-¿¯ÙèÝvNÕD¶¼Ým®X"gÐ[1µ^MzÓ{èÓd—«ÐL¡‡š°èxÚ^`ÏxW¬­bB¿*V]WíÑôÞÒæºÐ…Ítî:ͺ¿¦5‘uç‘ðvAµv”>‹Rzûu-ªzר&³Ì¨Ü¤¾¦lâZ? ¢ÛE~¥jÑÙô¬×袳0U ¹–.×t‚èvÓ-¥ùÖS±¥J=k®žï¦t¨Ó®û²{ :ÒÖ`cDÙúu-¦'h[_«Äpc6ÂÛSŒÛˆŸ/> -©}ç -å+ô¸Ð²r›8©Ž—.¥½’ù†f•Ñéÿ™*|“ó”/Ú5HÑ[TûU¯„Æf¯Ä塱Ž“< ]ìi™»tŒ ³Ü_:)!=‘nKÝ<¶’2—1yÌÕ<"©Fk±n¨kb¢°êŸÆ•\¨ª¨®‹}ƒòóJÜíôm¨zÕ* šÁÝ·r»84î|æâØz\Ó+q›½­\†ù*¼•‰Íb‚~ª•t±Qm¶ë•1a7 ¤ÐÅFŸ‹¸ŽÖÆNå]Ë×>Ȥ÷vùáR¦9c½¡'ƒÕô†wyéë‘æüþóâ®®¯çq£[˜Æ­+1ŸÉè邏búû U؆ Çh¿‹uêi}EC›Ça|ízvÕ‰½¦ÔAuÓr¡ï>dbÓ k´ežñÂÛàÅÈ|nßô {¾´µqô¬#½Œ¶Z… Þ\¦®³}J¾²"qEIû”­3,#.x¡¸dænÙÊL$¯¤¹žìw¼+,ÕÅ.XA—¶ë\ŸfyÑUVZÁ’ ¶ü‚ì4¯4¯ìŒ~õðúì~_Lu´Q{“>sikIáÉR[.Ãú²Í9¶<Xo÷P+˲±g[Ó‡F-Üê‚c”IS϶$ð¼RWe«JëDÑè2Ue­ÄXm -K¶ê–Òâ$z)¶˜¢Kàžö¼Wš=ï•°IKåÖ¯EK¶@žÃ9öÖ©ž²Å’Iž»I]Oi«l[·[×]gÛ+k†ë†„™ªžÍoR¾Šÿ^3væ<ÃN„”ªþ”ÝúK¸1Q;­±å睊8Äџƈ.±í’ç9Þ?Ùî’Ôý^ºcrÖª¨é/äÉ&> -endobj -619 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 483.873057 101.433071 465.123057 ] -/BS << -/W 0 ->> -/Dest (cb136-1) ->> -endobj -620 0 obj -<< -/Filter /FlateDecode -/Length 4386 ->> -stream -xœí]Mܸ½÷¯èóËåGñ ö bdƒ,`$‡ ‡ñ¬½‹`À›Cþ~Õ"UM•ZjÙcËöz0žVI•ÈÇÇb‘jš³ÆÏ·ÿ%2*¥ßžÞ´Š^ãßâêbPÿˆßèÏFSTÞäüùm¸¨bö6†óÓ ÿu—<ùÓéßœþsÇœ­Ê9XãÏ¿¾.*r*¦µT*mœ ò^Åh}TÁ(öëϧï0øžþïéÍ7C¤¬þ8Cj‚…n`3šÛÃû6x|ÐÐF B,„{ò#bžŒ4 Å1èVŒÄðWþç®,‡á?·wíö£øåöèÄYÃð¦¯ClY|g–h°Æä£m@—ma'@~¥¦«Ä¿yPG+.1ÐvÉÌÇx‰É0Y~ùöôÝ_^üý¯?ü½|súçï´öáûo þRºüõ~ü›ç²µóqLÿfÔ…I×Ò×4µìÈÒ„ëüôØÝßLùŽ–¾Ù5ê´ÓèËïê½Kš×,Ý#>gvÿ7×vX=>oICóz˜ÙÏëhüÜò±ç WR½~ÿ¯ó˧?¼¼ÞàÀ#':G a«Û²¼VŸ¢˜ÚÞ‚h­NÖ|ÆU}tøvÛÛ{ž±¦-ð-P«å¶¿®{îŸ.ϵÁ‘¿ƒó „˜ÝB!œõ ê.-k\÷—ÆÚ,i˜NjñG†”®eÙu­ÚP¯Y Z™ârÂ}_OåRWnO~ž“æú*bkàdZóP—¶=ÇÚÝ=:oÌs8´ÑMÎk5×óëyË7DÕ>þø±‡¯ÇÑw•¥çHÜîR, ”9ƒeÑYæ·º-ÏázÛq|o>Ù2‚>ÿ|™½c±Ì%,!0›ŒõkˆÝ–çpˆ½íGÞ¼ô®ãá{Zôž)ÎÇêqϺ¿·Èˆ´Ì?ôViO¡ ç&"7æ9"ímŸò¸\ràÞrsFµ€8æ#jRÖP"ZCܶ<‡CÜm¿²Íj-[aôìZ|@åÃ8§dУ¦¶JÏ[K÷ô¯.-ø^Ñ Ït ðŽüBÊ^ùbk¨Ü–çp¨\ B•ÓnÌÂ6!ûŽÑýYzIÇ•‹ü8ù{¾,§4,iÞFâ¶<‡CâŠxÎøÂ{ÅŽQÜM>àf”nÌs4”º~c­Ab¾WºnaÞ:m¶PGð(£ˆ·rCÅ›®5Xy®<ÿС¼×Wtð^ÅüeËc–KoCH³µ%^\Bߎýó}÷Cö}ú{hŽågö^ÈòÎ7…Aß?÷Þ˜çh( Ûüĵ‹v™/ô|Mã‘Ýsñ]–ÏÙ]@§»ÿÍÃy‡Î÷ð?)¯|Àž³àçÎÖJº¨wëYiüõ—ß+y|Ê:ŠËØó÷ïòÙ˜çpØ»íõÇ8n©ËWSZËê³zt?uŸ»z_ÅS?Cg3óÙ~ánW¸rSWc8S ÞâÒêÈÀ }ùu·OžÛÒæQÂ*Pë#]ÚÁÖ÷ÇëïboÌs8T®xÝ{W|UbëÍ{ -¶" ²ç¢ßгªîꃯGök|ÞÖÍí®¾¨àaêKˆŒ;vCnÌs4DÆ•Uhþnl¿zÃ<£wñ5AÒW³ã¥ÝÙ«­£…ÅÙ;÷ܺ^#½1ããnpÆ¡ýŠw¿W¤=…®NVÞx_ÇãŽw¹7æ9>oû³¶XØQ»þÖ¼^µ2åö¾Ú{/–$\ -ûŸkÜžÕøÅsro± ²][EܘçpÈ\Y÷~†±üÈ‘Ï–1Ë߶يÙmy‡ÙÛþç¬÷ïYEYaIqÅÄ ÌtcÅä}ì»w ôÙ£ Lšôýo0nÌs4T¦•(æýþêGdè»êq•î~®Ü˜çp¨Üø†ÍŽw¾,þÚÑC?hL©;Þ¾Ù˜çpH]ñD¿ -¾Zè‘kßøQzâBÓýßœ±1Ïáº÷<È÷]}žŒÎ¬ïŸÉoÌs4tægò??SÏà9¿ïê9Fúìîß¹¾1Ïáºâ“~–üóåôyÇÛ@ó¡ÏóÍ–wïïøm¤ß>Òïø†¢y‡Î•oCß;ïýGú=;ªža¤ÿq<#z4ª÷å|¢äMbÇm4x®&YAãüpŽzÄŹüôp #Îœ UÆué G’r,gÉx½øãŸÿô7¯òº³}}ùrËf.|øEu¹p¹.Õ1„½M }C?¼&]‰äB"~ß”*»T½·—¿¥¬’žÞ\š ÈmI.¿CYãç¡œT‡Øñ~N%9Ç!^ôWåûñÞÌž¡ Ïž¡†ôó¬)ßÂp.’ÎÆÓ!)cËá¹ç@Ê•¶ÉçrSÒŽLm:3T¶9/äŽ~×R§ër0a9”êrD1»Ó¿ýíÉdR¥þ Y\N Õƒ8Ùz1‰ŸNåÄXÄ„ùøFqŽu+‹LMØdàܨrŸÒy¥Ç¹î|ýh}}š’ß×lå$Ò5‘β¹ÜX"g0sqyz­Ì(.§î&åÆ«ÐLñ¡Ú áT ñ´SSÆU±qš ý\¨Ùí&±¥j)§´µª “°™Î«Î°æ¯imdÍyKøxÂm]/}¥å¬ÏZT­]«›Ì1£r“úš²5µvD'Aø­®E³éY«•Ó.ÂTD-]®éÑãIL7–æ[KÅ–*MYs­ùÉ” ê.µŠdCÒXŽg}D»úñZJ Ô¶Ö‰áÆ΄' 14?|Rû‰+´¯ÐãBÇÊmâ¤'¼LÒ²Ññò ]³ÊÊySåÿ Uø&-Ì3|0Ô UÎŒ›®¦›”¾ÙÄוÄ塱I5`(NiYu™fy}™¤…ô…t[êVcWRVeLs5¯T#k±i¨kâBaµ~Wr¡®Fàv“Ø7(?]‰';}ëª{½Y0 ;‰CãÎ'.Ž­ÅM9°ÙÛÊe˜¯ÂÇa`£^\ ŸêM&±Õm´›nÆ„“a …I<ÑèÓ ®½µ±ÓðÍñãÇ©“I#îãé—Ó0ÌYçmƇàL9Ï–{^æü?¤yߟèüC=@í2Ž[ÝØÂ6n½󑬜~ž6ŠËièUغ Zb´?‰MšÒúŠ†6Ž[Ëøš¦ìz"öšÒ45MË…~ªC&¶ ºÖ8V3þ†ð±«ÅæȼoÛLöåÒÕ‡Ãm\9z¯Z… þ8¸L“ÎMCò™‰«’tÒ¡%Ë2â‚ŠKvÃqi{°"m„'KÝp®/Û˜ãÎå¢vÞ©†ZYŽõ=לœ©kL®]]pŒ2iš² <¿)UÙÕMë@Ñè2]e­ÄXm -c¶Z-Ã'±–bó)&é/¼¦=o•V‰ž·J˜¥-åÖ¯yK®9@žÃ9NO§§”Í—L‚,ðÜMJSJWeó{Óõ½ëh{fA“!aòLõ”ÍÏRîâ¿=}ç’§‹DH©êÏ1i:r5™è¬;çâî¤À!T”X¸äî<·ã'ó(IŸð;¡99§£ÞLx‚"¯+XQOãi­r¶¬ÅGº¯„ðÐDszrÊ!Í!HÔ.c™*ú<Û­S*ÇÖGmÄpÑå”»«ðÉðóÐ%î¬ -endstream -endobj -621 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 620 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -622 0 obj -<< -/Filter /FlateDecode -/Length 4825 ->> -stream -xœå]K丑¾ç¯È³‘ù~ x¬;@c}0|èÎ¨10Þƒÿ¾¿ D2D…”Êêšv6Ü…êLEðŒ~¤(•¾*ü|§ñ_rzJ)Çà®·Ÿ.?_Խ¿RbuQØ?â7ú«V.N^çüõ'"Ø8ÅìM ×— þ.yñ—ËuùÛºŽ¾š)ç`´¿þý±œbŠQy•ÁRÚ -,ï§+è CÌYbA q[V„.'/± †3& ¬1¬‹ÚnYb˜¨Ä‚:û(°´‚Ú(%ò ˆrΧý ô ñŒžBÊ&‹¼žŽFâY;…è´×/O!Ÿ$žƒ,>+-ñ'çD^O¥˜¶<£ü䓵Jâi=yø¦y˜Q0RˆÏ@– |yÅ[e$ž…,?8ámLI˜­†f·UVK<YŒ‰NäA_B ‹ŠA‚Ì ‰pIÂãä²q˜d'—\’PÀ¤<¹­&ûÉ…%°Jƒg´VAï‚„VC‡™+ò ‹ÍNÂk -HX`-d1ÎJX`¶Þ"aÜ:•µ„xë”X.ÄTX ®MÁIX`¹p¥,aæZ`§„ ‹é%,°@]냑°Àv­KA ܵN+ p×ZÔy,aÐq²:Y p×j%,pÀ]L##a¬:Áƒ„ð•É¤¤$,pÀ]“”“°ÀwMDyq´8à®ñ‚¹»Æ+#aîg‚„p” ø£Äl¸ ñr:Y° Þƒg¬„hl2HK$,ðÀ]þ„9 ´štÊAÂÜÕ æyyÒæ-´<鲄¸‹@e%,ðÀ]Àg’°?!¢ "?“™‰VœIÜ1P€^Ø*n(`/¢U÷ À—ÖHâ¦ÄÄÊ2q×aL­ÄmüE>æ6˜0ï>ñ-ªwˆˆ™)Ò 3¦…CvßÖ½˜Ý)6ïýý/—_¿Gð½þåÿ/ŸUvÊêF‚PJ;äW˜&ÃÜÙ·Æðm Ek ’=T˜º¹‹¥„Ë6‰…€Gÿ®Û/+Y!* ‹ÚX Zý.‘xÍ{JëË`¬ EÉ.ëÃuäkû¢eY»Žx¿LI‚¬×y{Y±ÈÁ¢)&ï! -y§ðÛ¿÷‹Æ„~iÿÇÌ, -{¶Ò mÛLÉd„bPÚØDvL–&p)ô3‘¬ò†+,à‘”±ÙP,bí£Üªðoß]¦‹ó>}»dâ#§ÓŽÆ_ßýtùõýð¿ÿý‡áï>_þô½Rîóo¾Óôy[>ýüI»È~ùîÜüég.,´øM ]ªó©óµ_xµîPÏç¶âR~‘±Ö¸-Åä^Æ¢õZF£O/¿ªSûOk9š|BM¾ª×÷‹þnL—ï©ìoþ|}÷ÃåwïÖNWVK³Ó!6Ú%çîøé¹*_⧿„{æ|Ê=SWc5a5™®¨<1Õ~š?½Yh‹úWf]'2wÉÌtƒ F-.˜ÙÔ`nûmN)Ù é®Qu*K†wÙÜÃË“užÍ±r?ôÄ šu™×­â÷ó€¯Ò,¨ž5ÊYëžÔC-Þ×xì†öv=´§Œç=ô\§óГyfµ Ã4 7¸µç‰Ÿ˜•l9ƒKßrŽ9f9nôÖ¨nÔãkó“užÍ͹\ó-p¬FÊ/ñÆÑ»½ê ùlñ^ ZÛÅ‹Òü¹ÛÎ8&)_ò××ƈ ›µì{XiìãXy²ÎÓyçœsÌÅþ-×˶õE+Âd“tƒ9¥r³ôØ#ÏÕy:¼“_–hZ=Èî_…a¾",Þ0¬³78¼³1³á"Ÿ]›üo»Zúåf펷¾bGód§óÖã\ó›Å´=o>±w°V3ñ«Î Ù;mÏ#µƒµ³^ÝñΓužÍ;íÉÜs'{—vhš•vr1Â’MþšX{Õ&í9%Oeí]ÇÒ.Ù½dK‡÷µÕ&•±üðÿùûÿac)øMû’,Ö0½FYÈÖƳ–¹Ç’q.ë#ÍÐ:ÖùzC_Ê6Ï'Þg6ËJ&0—£¶‰ß±x.[Û(}èªÿûíz»È{«¸½”K<TüŸgÂ)5'®æâ%{ÒzµÞWSKU»±K¼ªS5º«vK™±Ží£móžœê(àªqL®Èê©uù¢éÛ¢°¨%p}Ðùë-=£²èÃ.cðúX,C–ý¸m'X:ÀE lÛQ˸æ1—qÉAϵÈr¨>–•>È3Ì<{Ùèa‘ œ Yh»Ì–<×/vÿ¼´õžÏ‚®sóiÑE•É/±.u]Ó‹MãÒî7KYAþ>CúìðA­|§´eeù,® Æ`uœF=êx -Moõ}è¿úq©Ã!÷ïZwYkÝÒÞç¥ fã51; KŸ·AGC_Õ÷Š˜>½íc,¾Tç~ìr¤K¯6ºp¤g‹ª¿Ýf_šó!fûÀtÃù ÚÕ<‰Ê—;\̯JY6æ‚¢Šµ]Ç[çvœ?K{ƒëZï„M}“ªDË#™QgG>Õd¯þÈm«û8×æ`þìΙ¥þÉvýc¯¿ƒˆUʳˆU®Wþr¨7·Ñ7U÷i­§b[½iÏGo£6²Y‹LD¾áÙŽú{,ªŸ’…ë)æA–9N®q¥ÆŠ1oÊOÁ–ã\ôÀ’IÚÜÍ›OÖyÕ¡º7L˜}“9QòS^L˜ Œ‹q]¡…%•³v;ÌKzeèpv–<ÖÄ9/ôãvÍ“¸ˆB»¾BÏnm—Ê\!ìIu§¾Sö°T¿$ŠÃ2ßh³<í|°š:Úòæ‹Çj¡f¹Ó'ÆïlyRNm¿Bûô¼”âîHïÔ'ëç¬y;í»d0ã’CŠxì§>3kö5.üDœ -3Ý‹‚¸Ô3€ÖÉÊ*¡·{Iú6 ¦'`3=k6tZfû¦PAÍ[sî ÔÉ:O²Ððrá¡zÏCÙÎVS £;3íä1<Å6cÇÙ[é›ÍÒm(¾™Qëš!ì²t5±óÎŽñkmÖÞÕ…è³QëêôÀ˜·ž¦Ü¡Ëž«ò$Û¥6þÈcö’ó¨…Ày„Z ‘âŒH|‰ÜPkIù -Z%1í-÷³OQ!=˜WÑG…sò'ë<‰ýb%Èzdÿ;·„[T«‹É÷ó÷ Ý·¹ÝPŽ®K2½ÔùÀÐÆ°rvùý¸|ºå;µÏ£ê×Êq¾ïh×Ú·…okížk§¦ÅƒÎò8YçI¼®‹mõ)Ôr ·<¹Y׬×mü·ÒÏ{ƒcmÛ# M­ë¼Þ#YŒ®r>ѱîìÑ÷“užÎSý1>ÚÁsb÷šoùp‡ä «CÊg=KËm}5O ñqO=Wçé<5cª<êCÿ>oý2¼ st}{+íÈ1x{ótÅëLã‡O¾–|ãƒ#þã ¯zf#ÓŽWö\ј0aýdï?Âv®Î³ye>Î/_ŽÇ6Žú°¿ Æí¶eÓ<%Êuڭﶂ•-›õãxs²ÎÓYöNW5ycQRx˜n½­LŒÖÓÌJ‹GPºmG·úÚîÖkë`«šá!2^·áÂÞ±ö¡Íƒ ïw<—ï{ìä°Ÿ«‡Þ=8Re”ý´gcŽÞI‘èUr÷üô\§óÓã n;ØÃ"~É·«í7ǧøžÝ{ÆLðUíì¬RÏÄ:¾3³ÙOä±¹Ó>í0¾sÖ·w€1³ŒéìÆ“užÎ›Ž³¬£x¶B¤;y•¼yàqòCŒäùvC±ÌV¹Ü }tVÊá‡Ûmúyr/[ã3fwŸ?º±2º•g}}Ù©ï—7E«yOo”K^'ö‚¤æw‹Üq·ºý¾}áËU/ÙYïУIz•¹MW¯Ê‘Þþµ·?OÇØL{€i6}É­–#0¤ªvÐ4/eê¾>;žÔŽæ,G“ꑲǿ))G6òL7í êw-ƒÚ+ý•#—j>öÁŽÏ´öŽKµ1ø:†)¾„ò&;]ÞfªBš´¡×_ƒ›èDPÄr-Ñ[ð¬ÓÕtº([_wj–—ÄÿÜJ-ïWô*Yz`1-¿XÊÿ\ŒþÓEg7‘þË9ÕòŽgUÈÉÔ‹N~¹Ð;¾@v®v½C&âòUµ¶è­EíPSKZ?©åk?`K7(æ¯Æ×ÑP}_«Ñ»×œª…lyûØܱ@DÍ ·d½šôB¦÷¤§É.W¡‰BÇ4±7èxÙÞ`¯x—¬­bD¿%*Ö]'WåÑô^ͦºÐ‰Mt®:ÍÌ_ËšÈÌyD¼]Э©/"•ÞÎ\›ªÚ5ªÑ,*7ª¯%›A\³“@º]âwª6í`ÌÆgV£w\ÎÄTɵr¹–H·‹XniÍ7KÅV*õª¹j¾‹Ò]ݦ¦xvc‡Ð#ýá6G”­_×d:«Ül­ó³!Þ.1nC~¹ø(”ö+”¯®Ç‰–µÛÈIuéÔ˜ªª05+ÞHÿÏPᕧ|Ñ®¹½å³_õNhn6òZIœê8IÚÅ^–©KÇØ|–ëK'%”'Ðm¥›ÆVT¦2F¹ŠG ÕÀcMÖÍë™ ¬ê§a%'ª*ºëdß\ùeEîrú6U½jÍÜÝ·v;94ì|áäØ,®é•­MÞÖ.óùJ¼•ÀæF2¹~ªt²Q-Úõα Pèä£/…\gkC§ò®„åkŸdRĽ]þz)aÎXoè,f°šÞ@Î3/}ýÊü€ßÿ»¸ëê+/ç8nTC Ó°uEæ‘Œþ^E:I¦¿_Q‰mÚ0¢q ö;Y§^ÖWohq܆׮WWØkIÝT7.'ú®CF6Íu¶L3þ€x´Ø™/µMØó¥­ƒ£S¥ô²Ô*.øpp™:Ïö|eM⊊ö®3¬".x£¸dæ.ÙJLrHÞIS=Éï¸),õÅ.XA—¶ó\³¼éJ+£`E[~ÁvšwšWòB¿zz}±Ý‘CDmÐÞ¨/œÚFÔx2Õ–Ë°¾l1Ç–#˜uòv µ¶,›{¶%9}jôÔ­.¸2jêÕ–žwê*mÕi  .@S•ÖZŒU¦°T«j)#N¢–bË):õ¯\Óž[¥)Ñs«„MYj·F¼–-Ù–yîαNõ’-—L-ðÚêzI[i۾ݺïm¯l® zfªz5¿)ù*ü{ÍÜ™ë ;R©úSvGtê/‰F v:ZcËßýR)`ˆ£?Ý]bÛ%×9Þ?Ùî’ÔýÞÌ19kUÔôÜd^ÀÈ÷¬©ú×ÝäjY‰CzLÂöP‡9µÙ*§ìY–M¢vi©H±ËRÒ›èD¥åç?ÐŽÜjû¤üü{ö -endstream -endobj -623 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 622 0 R -/Resources 4 0 R -/Annots [ 624 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -624 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 390.123057 101.433071 371.373057 ] -/BS << -/W 0 ->> -/Dest (cb139-1) ->> -endobj -625 0 obj -<< -/Filter /FlateDecode -/Length 4538 ->> -stream -xœí]Mܸ½÷¯èó«åGñ ö bdƒ,`$‡ {ÖÞE0àÍ!?j‘*Q¥»gÆ–x0žVI•ÈÇÇb‘jê³ÂÏ·ÿEÒCŒ)x:?|8}<©!8…cŠÅŨþ¿Áµ¢08|tçY`Ã’3ÁŸOø¯¹äÉOûæô¯e}6CJÞhwþõ]V‘B A9A• RÚ -*熌ó‚Ê똒¤‚.¡µ*À JÑI*˜AÆDAa†¥ íZ•`† š$ÌÐÉA¥ìÐF)QC‘5‚N»Á£Ò½¤3zð1™$êt:Igíài§%]¼÷.J:‚-.)-élqšHI:ØB6Iça‹õV‰:Øbb°’.ÀƒÚI‚. +Š:Ø¢¼Ò’.ÙÁ¥H$êt*†¸Öå­U’NëÁ›VÔ¡G¡‘|t¶xå’¨ƒ-Î*#é,l!üH:ð„³!F¡·šÜ»­²ZÒ9ØbL Q[Ðø[Tð  ™—$0! ” I$`¢(R”XÀÄ4PV¢“Ü@>‰¬ÒÐ-U°Å‘—˜ÀjØBè¹¢¶ØDX[@âka‹!+qÛÐ"qÝ’JZâ ¾%¥½Ä„ 3•ÄŒk£'‰ ,(PJXp®wJ\`AºèîAâ ֵΉ ,h×RôXð®%­$. ð®µÈ)ê` ,qØq°:Z‰ ¼kµ -xÝÈH\€V€C/q°2˜•ÄÞ5Q‘ÄÞ5C€¨ ú •¸€À»Æᤑ¼kœ2x×ñ(øG‰Þxâdw¶€`%.Àx± °ÁÀ-‘¸ÀwuÂð'ôi°Õ cò8ð®Žh>Q—PãB¿E-Úû$qïb ²8ð.è3J\Ž0¢‰ xW[ŒÔ’¼«MR8ð.ü'qà0h8`|´tE]TŠQâÞ-‰ 0 -*‚ID]$-qÈxPðC%.ðà] ïIâÞUÎZ‰ d‰ÀÀ^°‘¸ÀGR^âŸtpùDpí$. -Êu°…I\ÇzH6E‰ ‚-Ö‰ ‚-p$.€G0$ ï[œÀÀEœl§(q˜lˆ‰Hâ‚à<æc¨UIçÍèZÔEèt¸ 0k‰ 0—":½Ä ø!»KÀ è4I\lAW‘¸CÑ öëϧïÞ`ð=ÿüïÓûoÆHYùÑpÆÔÿ -ÝÀ$4·ƒ÷­ñøÖ£¡õ0 -i´Þ“'Òl¤-NA·l$†¿üï¼þ°°†ÿ8ÞÞÖÛ'Œâ—Û笒Û*bú:Æ™Å7fyFƒ&Gm=ÒØd2;rð+-ÿöõ @¬¸Ä@ë%3ã%&ÃdÜùõ‡Ówzõ×?ÿð#`ôúýéï¿Qʹï¿ÕøKtùëò“ì-~ã$ŸÒåÈ]ÍónÖk7éJÞ&ŸKe…)ýûIç]ɧ¦¿vúk–åÕg(÷|@9Š=OkSXÞSMåæ<‹|“môД£ç2é‰ÉË÷~™ß¨é9ÓÚvm¾ÿÇùõ«Óï^/Á8z¨3N(e–ºŠß¾,OÁïKÀÓÝ+°]5‰ÏBúÉbÏÒÂx_VfÉ_ꩅ☾¤)ÝB-m,ÏQë£ÈÍ|­dØæÈþB‚#¢Þïà¶3ÏÑ€‹ÙÕUµ :¦ 7¬Fõ\Ë™$¬^Ö45Ähkm¿DEEvAZ£—ZQ½G™v}?ëçûÔr¢|Ÿzÿ‡¦¬ù~NwÁ-êØ@påY6iogÒÎ<‡C$]åÒZ‹®©=>tµC±›[ª¶ «õÅ°÷fo¨;6Ÿ·È«= °4/‰Tæ€bâ™T4y2t©}y‡ÔëÎjÒÆò'ƃœ×¶ø¥´ŠoZVïwèŽòm¯e•³[IÉazª)‡®£²/ÏáP¹ã‹î¡èÖÖx.¿¯ÕS¢/å22º/;ó ™fÇ×|ΙKÊošÝ<7¢:òˆ{o tö%»gDy‡ÒÿóSͽ_Á êîAî-ÏÈg…mu–UþÚ湺<×& -ÝíóòÎ<‡CáußRÓ„ºÉ‡Ô¶ùKSmæ4~J'µ¸0‡ß3³(ÌVp³´ðÈ]V¸ï»¹\jg* ÿ‰¦-O ôΧwÒj·f2†…"»Gæ¾<‡CÛŽÏH J ¾]·¼Q9Žö¼ýéüd¡–ùyS1ÒïE…:ó ¡öºïød>¹5ÂýŒüóuöŽ Ä2Ÿ0oq1Q·‡Ø¾<‡Cìu?òfä]A8_©ûf¤«Eo™å|ª×ø¬÷÷– D2ÿЙA9ò=×Ù—çpˆür×Û[®Î¨6Ç|DEÃVÜ‹Evæ9â®û•[k9b4„ÍTÊ 1hQSZ¥å­Íõ£·j½㙼‚§m­xn@F%±õëÞygž£¡’vâGå´+³°.dß0º¿H/i¸r‹‰¯[wîzëÌs8$îøˆ‡áŒ¯¼WÜ1ŠÓ{3;ó¥~c©Ab¾×Æ>Ž²v'—9iFùŠœÒšoXyaÆ'áï"ã«€ ªÇt7”[Q(ðm-ƒõÈ|ïqêÄÑ‹}¨e³¾ižsgȱؤOhgMjoE²3Ïá{ÝW5q¿u? òÞ£pIÇì¹y÷£ y;ìêñC‹Ev#°/ÏáøùýP×”³BÆós ÌÆm[W½ñæõÿ DÎþaþJ‘¨‰ÜÞºygžÃ!²3¾ymwD~37L¿~¿U¾Z»Â(D¸«®Ýa¼å×n¼?³êIn~†çØÛãïØÙ™çh¨ñ÷¯e/x¦{6Ü4ú€ŽGÌë=Þ/ËÑl/Ј2^^3Kjùpe _+iæ쫽êïûxéÅ{@gœÒ³˜cïl¥3Ïáûÿ8å¢g핵…ÊŽ=˜÷¡}¡,ÞØ»ÒØ™çpÝyçN.hßgZ­V#eÞø_ìHŒ·¯étæ9w|ÂCñÆFtî+2[Dd¾iPŸw'—Ê“ß»ÚÅl¦tée92¨Û9²3ÏÑvÖ½µÚ|g»g¯wwëû©…ùo‘—=íéòw«åë,cãK5ø=G¤¶ú‚Þ«˜¿lxÌrëmi¶¶Å‹[è»c?dgžÃ¡¯s?dÛ§¹‡æX~fë…lï|Û@áïlwæ9 -ûüÄOµ‹v›/ÔzMãÝsó]–/ÙÝ@g¼ýÝÃÎ<‡Cç|ÇÏÊ+ÏØs6üÜÕZIõ®=+N¿îò»Á§,£¸üVêö]>yŽ†½xÝ;ë-LqÜ\—oç:4†Õgñè~j>7õ¾‹§v†Îfæ«ýÂÍ® q妬Æp¦¼Å­Õ‘‘AÛòËnŸ´¶¥Î£„U ÞoUÚ¯ãocwæ9*w¼ÆöjتĽcÚ]o¶˜ÛPpë÷jmú -«¶ß‚Æ×#Ûµ>okæv‹ï*x3÷ÆMDÞ±²3Ïá¹³ -ÍßmWo˜g´â.¾&Hj1;ÞÚ½ÛúWZXœ½sÏ­é5Òû+>nÖWÚ®x·{EÚØ“oêdç÷+ãøïrwæ9>¯û«¶ØØQ»ÿÖ¼ÚÕ2äö^ìÀ½K.…}á/Š5nÏnüBfÎĽÅÎoöëÌs4d¦uïËÙù"<`³w|egžÃaöºÿ¹êý÷¬¢ì°¤¸bâfº²bòûn]}ñ(Á“ºÛß`ìÌs8Tö}óä×ùþê'dè›êq•w|zgžÃ¡²ó ›{¾%ò«â¯;zè³Æô+RœÎÊê\>«(:ÙÑž»Ivи>¨£wqÎ?-Üü„3ÇaȇäÚx†ûH)äse2¼^ýþø ƒWÞmÞ]^D¸~ñÙã÷áòUàù:WÇ8Dê2LB?n©.Äs!76u®²KÕ«i‰:ÿ½|­x®òKd¹Éiýåw,kú<–KsN÷ËðËùó×>†‹~Q¾›îÍìËpìÊðŸVMùñäÇ3’ôxNžòqÐ&¤{ö4ØÜ6éœÏeŠÊ’.M§ÇÊÖçœãñÃkªéDb•)ÌT]Ž+fSúc£8éDC®ÿL—ÓCÕ(Ž¦\ÌâÇS>=–1Ñ`:ÅY8}Tµ,Ò%a•=€sÃÚ”Ö jú˜jáÖ•Æ•§Éù]É–Oõ!UÙñ\›Ë!rz½ç§WƒžÄùÞ8ØéÊWSœ/6C8H<í\àœqW¬­bB·*v»Yl¨Ø£ó‰mµêü,¬¦óªÓ¬ùKZXs^>œp[ÛJEi>÷³Ujר*³Ì¨T¥®¤¬ BµÑÃI~«JфƬzÖjùô´‹0 QM—J:AôpÓM¥¹ÚR¡¦ŠsÖTj~6e†ºµâÙXG“4ä#ÅYQ¶|\Š}nÒÖ*2ܘ•ðá„!†VâÇ“ Bj7s…rz\hY¹UÕŒ—YšE.ŸÐ5‹,Ÿ=•ÿ¿P…«ÒÌ<ãMRùü¸ùj¾Iî›U¼¬$.÷•uHªMaN˪K‡P1ËëKG%¤Ï¤[S×[HY•1yHżLR•<–b]QWÅ™ÂJýT®äBUŒÀíf±«P~\ˆg;]íª{¹™× î®–;‹}åÎG.µÅu> °Ú[Ëe˜/‡q`£Vœ¡ËMf±Qu´›oÆ„³a …Y<Óèã(.½µ²ÓømïÓǹ“I#îÃé—Ó8ÌëLÂou>Û–{^úü¤y…ßžèüC9Lí2ŽUÙÂTn]ˆùH–OBâ|2zÖnĆíÏbç´® ¡ŽãÆ0¾¦9»š‰½¤Ô•AuÕr¡›ë‰M…®Ñ–ÕŒ»"|hj±:2Om›yÀ¾\Úòp¸ÍÇð«pÁ—qÖÙyH>³"q•“ÎC:´dXF\ðBqÉn˜fËff@ò›ÔªÏöo -›ïÅ.˜‘/í¬£y˜åEÙø, \`Ë/xÁ¤ùMÓÂÁ軻דÛ}2Ù—¥•Ú«ô‘Këƒei%Â>(-†‹òþ5M‹ðÉøó_¤íd -endstream -endobj -626 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 625 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -627 0 obj -<< -/Filter /FlateDecode -/Length 4531 ->> -stream -xœí]K·¾Ï¯˜³·ù(¾Ã‡I#b@HFÒH²¬Ø9äïç+6_Ý]ÓÓ³Z+#ÇZ¬vºŠd‹?V±g8ú¬ðó¹Æ‘ôc -žÎ—O?Ԝ¿\bq‘Õßâ7¸³V&§“îü# l˜Br&øóÓ ÿ­.ÇâO§¿vúײŽ>›)%o´;ÿüŽUd§CPNP%¨”¶‚ʹ)ã¼ òzBS’T0Ã2´U˜A):I3Ș(¨"Ì°´ÝªÌ0A“¤‚:¹ ¨´‚Ú(%ê`ˆ"²FÐi7y8ÝK:£'“I¢.@§ƒ‘tÖN>vZÒ¥É{#Øâ’Ò’ÎÁ§‰”¤ƒ-dct¶Xo•¨ƒ-&+él1ðNt¶]QÔÁå•–tÉN.E"Q— S1Ä­Î(7¹h­’tZOØ´¢3 -ƒäƒ 3°Å+—DlqVIga áGÒ'œ 1 -³Õðì¶ÊjIç`‹1DlÁàK `Q—&àqaÞÂË“ö>I\àÀ»X¨¬Ä¼ úŒ€ã'¬¨FâÞÕ+µ¤ïj“”ļ‹¸ÁI\8L€ÄXß&­]Q—&•b”¸ÀwAKFâ¬Â“Š`Q&IK\2žâP‰ }»ÌGL§±8w~õãé‹?ý·¿|ó-øêýé»/•¢÷_}®ùï¥üuó_ÞEvå5Ñü×ÎÈÙüÆ"—ê¼ëz튮Ö]ÕséJ[¡”/6Öú¬kv¬íÝÖZ/{ùÒ­—6º '<½ü¢N½\ÚÑì[Û®†¾V»^ÿ]_¾æ²_ýãüêëÓï_-A—³¥t„¦Ht§Çª|N x¦´ ÏwÝíšÕÝmèãr8¤¡; 54Ç2ihguo£ -dÓ0](šÓL†&?Iª@³ J懬óhàD6¿‹Î— !¿¥RþCFqÆ­È÷ÃfؤZº{µ?XçáJû<úIòϯ™×Cp‰È8©h8ßGè±:‡Ðè3ygm¿ý¶Ò_釒·°cÌÛ±ûèåsåЭWVuûŒçíë<Åš/Ácu¥ü4®ÑíìªÎ*žÍèE§µ-(Šóß«í¬û$Å‹«øõ¹kĆƒÍÒö«\ŸÁ•Çê<:oÄœëXìÿ2_¾³­ÊeDÒøœüàŽûÁ:†Hº_®F¢yu'º‡I\øŒx0£a•goxøJ&´æÌÆ‹ãìÚÄÛlé—›µWÐúŒ=̓u­û±æ'Ëi×Ð| _ï`-fâGAWÐÙãHMídƒS·Ðy¬Îáó`ìy%z—vhÚ(]‰Å˜K6ñkÚ«cÒÞôÎ}ISνk_Úåðþpø‘?X¯MÌ}ùúúã_‡¾dþæ}É!Ö~–+ðºi}YÚÛ× ¶o.ëÏÎÚÏùz#/eû~û‚ý3ëÚ} §‘‹KùÒN¾SË*µm+—½tû³]¾ÛÔÊP‘gr°ü]í»-¿n˜må:÷¡ø§öCÕ }C=µ,Olëež¥ÚÓü¦&Ÿ,ŸúPl²eœœç{Ì>Ì>~»mÇ[þHì~ÛŽ*ýšý”}µcêóŠª÷eá#3ÏÁ6>~Á+2> mg¼¦¹~ÆÈûÒÖë‡Ýçæ]ñEµÉ••&v_s3.Bi·ø¯c¬”ìoxÒeÌìÜÞˆ·Ü–•í³¸6èƒÝ÷±]û!ÛXl¨ýÉ2Á¾ÅëÕý+Žçy3ø(õ×Zw[kÝÜÞûÒÆ0ÆK>Æ÷±ü÷²òÑê^{y:Ûû˜±TyÂKã²ëK·ñ¥À9=æ+>¨x»ÌXš£‘aìýà›Qÿn–Õ(…ËççK®rÙ¡Ï9PCÛµ¿un‡ùono…o]ëSWç¤ÊkÕžÏÂÚg{˜j¶W<Žc«{?Ü›?kÌnÖ“ÛÇþu+ã½c¿‡«ãV°ì -^שsâúWÆVíú5IóúÚZhvÛB¶jË”¾s³òí:ò ÎMÞæñ1H‡ÍÍÈï`g}ÔëC>×ìõiŠÎaÚˆ!_žNëtR×é9„Fó \½aò‚ùˆ„Õ ¿[mh´2ù~»1ˆó¸B»®Nß«µ)æsE¼¿fÕúÎOÉaì¤ú90[Êî@›ùÈ.rÞîxjoÓvL굑»#<вùóóüYâgxŸO‘ó@£Ïó>ñèW­y9ïS4˜q‘fíãÔ¥a4vïµN]Džâ“’y1Y1Lžo– þ"(¶×Ým Éç2%>euÓ<Û¯aä·Ï¤Æ˜×*ÐÍ·ú¬ó )pãÃÈge\çCõúú[qî~£B[Îäm65l)®gp•o¶ü®l¦4[îÀ$˜h“ o¡ CÑ0Y‚ŌŘ­Ù⑦”\ zuÃ=<&Άfl9åm¬óhxLVïáñÖ#«ÆY5Ü~=¿ÞÈÝê1­Êñu•J7]gÌPΖ߷å/•×ÜþÈ™kûr˜¯µÍõ¶Õeé³kù‚ £¡ÖyÔu³ùT Ô­lpõÁ#Ö qëù…·Å#án`-ë<‘½fÇΛēï\wô­¹ë<Rã>?ÚrBGͧüðYBÃâM”G‘¥å¶>R“¹©Çê<Ráš}RU+H½é¯çݱ0ý¼¼¾ü0]±c÷õa¯Sm|:þ±ì[¿³Ý½]ù«>­ßØÔ`ùm9A)8Œ s|‚]t:25,Þ,rz5ôÞ0sÖ«C}–Ñ9šä£Óm< ,>mìZlΛü¦½¹ -¯ý¼ÑlËÆ>3K{–J™Ó›·mã²lÜÖ ¾ߗż—f¹iH†|ԫ̓ƒ|¿×%/¸¨Åægkg3¹õÁÕ>l†ò§“Ï'çé|zªòqÒ†W?{šxß> 5|êž%]‡Nggëó•šùPúŸZ©rN½â£kùØÂùûᢔÿ)ú'hbÿç'iùLi•ÅÑÔ‹.~:ñ™â$ˆ‰úã¸b–—ªµEº=Ý«² 6Li]yˆ*/û#@NNæ—ÆÕÞp}W«ñYo¤j!›O;›o,QÓë­˜{¯&]Ä|.{œl¹òÍÞ€kÂÞ e{ƒ½âM±¶jº­P ·ëbCÕÍçx6×ù.l¦®ÓÃðײ& ù'¼œp[»–>‰R> º6U½kT“ÙÁ¨Ô¤®–lBmœÑå$?WµiÂ`6ý0j|¦æ,ŒHÔÊ¥ZN]Nb¹Òšk#Z©Ø«¦êùnJ‡ºÍñ‚ì2<&üEÃQ¶¾\ŠùInkܘðrÂCñÓÉ¡´ë\¡\…Þ(´C»MUÇK—†X]…©Ye|"!ÿ?S…kRfžüBSƒŸ*Ú¯úMxn6ñÒI£Ü7Ö!ÉšB/;¸K‡Ð0;úKG%”gÒm¥›ÇÒÁeƒ<¤j“T#¥X7Ô51SXõOãÊQ¨ª¸]»姅¸ÛéÚTås‹ÐëÛžqçÓ(mÄ5Ûìm혯ÂK^Øh-fèÇz“.6ª­výfƒ°RèâN£OY\gkc§|öByÙ'™´â^N?œò2g¬3ü”Í[Í'ž‘—>ÿe¾Æï?Otþ¦±9¯ãF5¶0[âq%ãïLjÅü}UØ¦Í 44Ð~ëØ˺Š†¶Ž3ð5õêª{-©ƒê¦…®ûp›]£íà·#¼¬¼Ø™›¾`Ï—¶vŽŸòá¬Õ*\ŒÝÁeì:Û—äóÐ$®¸h_Ò¡%3TÄÅØ(.‡¦nÙÂLäx“æz¶ŸÆ¡°|¯áb#øÒvõevlºÊr/†"íx16Lz¼iZØ#ýìéõÁã^Löu°´Q{“>ÒÖ1–6“¥6_úåe[sl~¸V'o÷PkËs϶ §OZÐâbÄè ½Z)àÆ›R•-nZŠF©*k-†j“/Õª[r£è¥ÐbŠ.ýaô´G¥9Ñ£â7e¹ÝºâµhɶÈp½wª—l±dd~¬Ý¤ÔKÚ*ÛÞ›–÷®«íyèuC|LU¯æ6%ŸÅÏ™;sÕN„TªþäÝû¡ÔX¨Iklþž1½‡UD 8l—Ü]gÿd»KR÷OÆ;a8&²VÍß'›ðEº­šªß&'WKJìÒ}¶‡:Í©ÍîPþÊ ¼™7‰ÚeàT‘¿!ÌrÐù‹M‚Òò{¡ßä÷AÛ'ùç¿Î—ò -endstream -endobj -628 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 627 0 R -/Resources 4 0 R -/Annots [ 629 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -629 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 277.623057 101.433071 258.873057 ] -/BS << -/W 0 ->> -/Dest (cb142-1) ->> -endobj -630 0 obj -<< -/Filter /FlateDecode -/Length 4620 ->> -stream -xœí]]¯ä¸q}ï_ÑÏ–æ÷ìC€Äð"²À ~òp§gÇFp×ÀÚùû9T‹d‰*Iì¾wf´Žç¢§[U$U"O)JTW‰¿ïþ‹V‰SðözûùòËEŠà$þM)“úG|‚»*iƒp*ùè®?g "$§ƒ¿¾^ð_wH“¿^þø›Ë_–yÔU‹”¼Vîúן²ÊbÒ1ª•T†Q9'BÐÎ3*¯.1%N3\°Ú®Ufا‚VëȨ"Ì06(³V%˜¡ƒ²œ -f¨ä£Rv(-%«ƒ!ÒZ£r£Ò=§ÓJø˜tbu:4§3Fø`•Sœ. ｋœÎ—¤ât¶8e­ät°Åš8‡-ÆÉê`‹ŽÁpº[4j'1º[€®Èê`‹ôRqºd„KÑZV— “1ĵNK'\4Fr:¥„6 «CB#ùÀè4lñÒ%V[œ‘šÓØbñÇéÀ΄™Þªsï6Ò(Nç`‹ÖÁ²:Ø‚Æç@{Ø"ƒç(=Hd^â8@‡ lÒ–#°ÑFŽtL†`8ÐÉ ëSàxÀHV [œõ[,z.«ƒ-&YŽ Œ†-HÀq1°E[ÃqÛZ …㺵2)Ž øÖJå9.0 \˜)9.0`\½å¸À€r¥Äqçp'Ǥ‹î8.0`]ã¼æ¸À€vžãÞ5VIŽ ,x×ädu°æ¸ì(ŒŠ†ã Þ5JŽ ,xÝHs\€VÀ¡ç¸X:FÉqïê(-Ǽ«\« }Ðp\`Á»Úá8Ï ÞÕNjŽ ,xW[í9.PøG²Ñxâøp¶€`9.€¿‡NŽ P˜ÐK8.pà]•àþ˜> ¶*&Ïq愈æcuI¨€gú-jY(ïǼ Ge8.pà]Ðgä¸/àQ5Ǽ« <5§ï*$Ǽ‹¸Áq\8…€ãø7¡$ Ëê’)FŽ x´¤9.€2‚IX]’â¸d,$âPŽ Z‘Bðø$¡CÈÇê¼À±ã¸ H(‹ÕÁ,ǬE2)r\4l1Zs\4lAÈÀq"‘¢ovx[v|`a "EŽ Àd"&k9.Îc<†Zåt^‹@׬.B§Ç!XW¬8.ÀXJDtzŽ @ð"‡KÀ è”å¸ $Ø‚®Âq\‘ˆ‘9§S°A6Ljt¡SŠã‚¨a ¢AŽ ¢ŽezÉq<4Æ’¨jNgegFŽ ¢õ"‡Š ›`¤ ¦du°4ÁqÚ'¶Çü`°‰a«ƒ-ÆYŽ bÔy¸9.ˆ¶h©9.6óx3p\$lAÄÇqA’^ä0™ãÐ8tRr\TÄxS[Ž Ð/óx3r\ŒÄxC<¦ß&ãóx3p\,lqQXlAД˜~ NÂxìÄéûD§¨<àà3cÈ™Ð-,¢û:æhÉÌF²ûüØ_ÿtùí œïõO»|þÍ4SVþ„)µE|…n šÛ!úV¸|ãÑÐJLB;YŸÉ“©©@‹ó¤[6î/ÿ»®,,‡á?N§7õô ^|:=Ü'‚’`,€1|æ‰ÅfyGƒ%Gð¶iLÒ™9Ä•Ò.ÿó‡ €:[qŸ­‡Ä|øK †­v×?_~ûo?üç¿ÿáGÀèÃçËý“”ö')]üþ;•»ù»^»pÿb=β>O"éÝFRŽµKYùîó8¿üVòûÿ¾~øáò/–5Epwx™\6å^¼Û¾cYÞÒ¾_¢Y1ÜiÖÒTîÖšª¯JuÖ«–~ÕtŠ4Ñ ‡œÇÀËÏø‘æãò缺kâ°Îëä²|i„0ú¼ü®e¿lÀö6§ö»‡½}YC7×ë Z»µ‘ižçž!g³#Šyþ襃yÎSŒ5öpÚ·ß„EM~ßZÛWy‡ãEþŽªìÇ{[M´Ôa¤ä?ÂTí?·íóUúº-Ëœ0܇ö?KúNý6Ýõ}ÂGçOyðš•À0 ÑÊÆCŸ6˜çth²»¬·ç̌ԣ¤G”"-J[ßꟗåä|•ÅRkyEQðyYFÏz½½SÞŽÍjY¢3£Vfû©•¥Üº™S{ZÏ’ŸH¾ñ´×n9gÕB©q¤Žå9R÷îžÓz´en0j?Mnmƒ–Õù3·ŠöLK÷­HÐý–,̡ۨÜ÷ÊŽžÝISšUŽœKϽÊ.uY^}váߪÛ@" úò„`œ&qö‘8–çtH<ˆK+\[ê#ÃGz›c&ÝV+3ƒ)}iñÎ+/t%_é5ÅCö<Õ!#G#È+çÜDl.âÝèíxY•Èc#²ÜG5\ÝâÀáÎ`ž³!WïÇŽÛ}þܨ~3ªÈµôH^!˜Æ¡vYO=9_PÇ]·åuÔúxÀûësÚBÐàýrÇòœ¹ûqjQt±à4Ò$^šzÚ£h G+OÝ{ò‚´>ædZqŠ.Ìú|Æ“˜Ø/í_Å‹dbQV|ßÒí"¸G=™Ýx&Õî .Ës:DģݼH­=꽶F(„£h­/<ßË[b¸oÏç=òVS™ú #•ÄžJ‹$£Î7Dö‘:–çtHÝWG6yÈO„)¯mñKißµ,ãŸéÎòí¨…G¥!s’É 7 ÎP9˜çl¨4±èŠm÷ŠûFP½1*úu |™æq¾Ìs:dÄšï9r@ùC£›÷F4AGö¸çAðJ[,9<"Ìs:”ÄŸ_kìý%Ü¡îä>rï}ÿÈÄÇÇåƒyN‡ÂýØRÙu/³Úv®ÍœÆÏé¸gÆð‡cf2 ³5¿YZ¸Ì¼¯ÎûäbŽÃH ôʧ]tÒÏÕnd,™‹õ̃yΆ6{3>sŽÌ¨œ'{_ŽþzqòBI˜,ÄEú£Y¡Á<§Cèó÷Ó‡øäÑîw䟿ÏÞ±Xæeî:*íŽ;–çtˆ=XÎø(òv¾·è¡}d”óµz\³¾÷ýuKâC§…tÖgôì#r,Ïéù뽿~ÚÞ²;¢âçäãk‹óœ qn?®Üº—ÃΆÐ5A/ϯ•Ü¼ôQ®Wa¼STð¶Õïl ’Ü¿!Ìs:TÌCž•ÓvFaCÈ~À»‘^Òqå&?ÒûÖƒ+Þóœ‰1âi8ãï¼W<ãÅŸX—9˜çt(ŒK Z¹zÂdµvœ lµª—›9xjfµÒAÊÝ'sèìtµïÆØÚ­éTŸÞŽºß~ñ:äQìåããñÁóQ~|à -Œ¯ÂßEFïv¨žÒ=P.}ªn÷!ß[;÷´>uæèÅ:ÔþɽÁ{DžÌMú„vVVÝ‘Ìs:ÄîǪýóµ\ë~ä½0^¸¤#ö<ƒ¼çÑ„¼vÄ¡žÌE#p,ÏéøíãÐþYÈ2vâÍo2·m]õƇïÿóˆ ->̯ŒÊZwtß|0ÏÙç7÷VGä·ó„ùÃ<¹«õw™îªëWoŵÏϬz’k×ðk{Âë óœ5ÿx6|ÌþÙðoÝç)™s­ æ9bÿ1O¹èYGem¡r` æshß@h|ü}ƒyN‡Ðƒgqžä‚þéÑi4¡äên$Ïÿ{ÆÆË®äã÷tóœ ‰ñ &<o¼Á£ÓX‘ØÂ"ó¥C}^\V*Ïqïj³žÓ¥/Ë‘ñ‰w æ92î{+¹ùÌöÈZïáÖ÷s ÓO‘—5íéþ½Õòu”±ñ^ zÎ ©½¾ ƒö*/k:g¹õ47ZÛâÅ-ô=±r0ÏéÐ7¸²ïÓ§\Cs®8³B¶W¾m ð‰g¶óœ…cqâ×ZE»Írÿ ›Ï²üšãPI>þìá`ž³¡3½!vü¦¼òŽ=g#Î]Ý+éf½kÏŠóÇÉåÛãkö1eñâ<öÌã«|óœ{ûÑáToažÇÍuù±Õ¡Ö¤>KD÷©ûÝÕû!žú:™¯Ö w«‚Ø;7ån e*&Zܺ;21h_~Yí“Ö¶Ôqshô­J[þ:=ñ4ö`žÓ¡ò j|÷·0î#sóÉý -}¯ÖfÜгjÿ4z?²¿@ÇmÝØnñ®‚—Ö7ùÄjÈÁ<§CäØû̧{býÝ­¸‹Þ´r1:ÞZ}Øú;-ÌŽÞiäÖõîù‹¡÷ŠS}Ëû[žx_øñç½ ‚CS:—÷â‰NE²µDåa’ ®7¢(Û9\ó_2?£Ë#TyX¯ül -yß” ªþõ÷¿ûªéå¹?ÝÙ܃šû t§Ôûûq¦Ÿ©’Tq´÷îÚXÂòxSCäJ¿Sœ§_ò÷ý•y”w‡7-TÉiýý3•5ÿžÊ‰Å1ÏçËÓã9~¥I¸ëå»ùÜÄž© G®Á•kø¾oÊ_.~ÚHMûÀI7¦óF±Wo…Ém“®yß¡(U¥éÔTÙ꺑sÚ^÷—šjÞqWæMøòL÷íxÉÁœþ—©Ñ¾¨dE®ÿL÷Ý1å$Žº4ñë%ïŽj±µBŠ³pþ)kYV•„UvÓ‘ú”Æ 9ÿLµpãÊOíÊÕäü®dË»ÖXY™iß–û‰!rzµç«—BÍâ¼Ãlf>òÕç‹Í¶-MÛ -lÅÊH"tk¡$§kbm‹=*ïHV«Î7a5V"Í_Òê@šsOx»à´¦—¾²Ò¼¯e)ªÔ®–UfˆQ©J]IYÄÖvbD· #üN–¢-³êI«åÝÁîÂX€dkºTÒ1¢Û…M7—æjK…š*¶¬©Ô|3¥AÝÄZñŒlª£Yò–Ù¤HS~.Å>·@ik nôJx»ÀÅØ•øõâ“Ú5®®@ - )·Š£lxiÒðß¡kYÞ[)ÿ§ -W¥™y¦ÊVHåýÑÚQ;Iî›U¼¬$*÷•u,Wʆ––T— -¡b–ÖÆŸLúLº5u­±…”T‘‡TÌË$UÉc)VuUœ)¬ÔOåJ*”Åœ®‰]…òëBÜìtµ«ìåd^¸»ZnûʯTj‹«¼Ù]µ·–K0_„·É±Ù^œ¡ËIšXËêíÚɈ°RhâF£¯“¸ôÖÊNÓûà短“q÷vùóersÚ8ðÕ÷n¥‘—ºþ/Òü€Ïÿ\ìõe³°»ײ²…®ÜºSO–wúŽƒâ¼ówÖnC„ÚÚob[ZWÐPý¸Ö„¯mË.±—”ª2¨ªZ*t­‰XWèjeH͸á­«ÅȼµmšÃ¾šrq8ÉÛÌ«p@/‡±éLsÉWR$ŽrÒæÒ¡µšdÄ-‡ä„©Y¶03’ž¤V}¶ßÒ¦0ù\䀄ùÐ4mn–]dÓU$ = [EOšö0F?ݽÞÜî³É¾ô€,­Ô^¥¯TZ/,K+áñR3úåaõ9æšJçm5TË2¤ï™ä´®ÑB »8 %ÒØ²Í =©-²ÅI‹£¨t™,²Zb(6ù9[©–éŠ#[K¡ÆMúgZÓŽ¶J­DG[ůÒær‹Ç«Ñ’©£píêdKYcÉÈÈ<Í]¥¶¥4E¶>·]ž»xÛ+¹ Û ñ-2•-›[¥|Šÿžé;÷<ÝL—ªüM³#*¶í5ᨭ -F›kÊáNôbó¦×ÁF2]òpžýù“õ,I™?¡gBskŒ -xÛ0áŠt¬ E½Î;“òÙ’d/é± -`¦‡ÍÉÕìfžÈœ&‰êaÈCE—¦-ªeŒy‹ö ;]”ïÍ(¿˜>™þþlÉ²Æ -endstream -endobj -631 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 630 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -632 0 obj -<< -/Filter /FlateDecode -/Length 4489 ->> -stream -xœí][¯Ç ~ׯÐs€læ¹E -´E¦h£}úp¢Ø ŠãNú÷ûqvç²»ÜÕJ>qe'>¥%gf9ò#9’Fú¬ð÷¥Æ‘ôc -žÎ—w§÷'5§ð/·˜]dö·xwÖŠÂàtòÑß1Á†!$g‚??Ÿðßâ²oþ|úǧÏûè³RòF»óÏo˜Ev1åVKi+°œB0Î ,¯L1%‰1\ CkV€”¢“XƒŒ‰+B KAÛ5+A 4I,ˆ¡“ K+È¡R"‚("kžvƒ‡Ò½Ä3zð1™$òx:‰gíài§%^¼÷.J<‚,.)-ñdqšHI<ÈB6‰ç!‹õV‰<Èbb°/@í$! ¬+Š<È¢¼Ò/ÙÁ¥H$òx*†¸æå­UOëÁÁ6­ÈƒGa‘|x²xå’ȃ,Î*#ñ,d!üI<à„³!FÁ[ {·UVK<YŒ $ò _Bã!‹ -^‚xÐÀ¸$a€ a dHí@‘¢„&¦B° ˜äò)H8`•Ïh ¬‚,Ž¼„VC‚çŠ<ÈbIX` dA ¬…,†¬„hK° ,à–TÒXà-)í%,°\ˆ©$,°@\=IX`¹0¥$aæZ`§„  wX ®uÞHX`»–¢—°Àw-i%aw­EO‘Y`À«£•°€€»V« awáFF¬ê;ôÀV£’°€€»&*’°€€»& ˆ¼0À­„Ü5S"7p×8e$, à®!ã%,€¡ À%fÀ]@ˆ“ÓÈ€•°ñõ[Tæ3càI&¸!»¯5Gkf7šûc?ÿxúê Á÷üãNo¿È;eåO#AÈ­ ùÜÀ$,·Cö­1}ëñÒp`fb !à=}Xˆš°8mº±üï¼~1“‚³èèºÈó^@ œ†÷åÑ E1êÃ1³‰|sŸ«"dé\_hYÖ¦ªþ¾MW’¬7÷yqYQÍ¡duH8ç÷…¼ÖøeŸ‹ÍÉöÏ'뫛ʠJÀ]˜ë]^P`…Ê©–G› ‡&à Š -E³Æ¿}JMRŒàõ²É’FÔ1îüúÝé«¿¼úû_¿ù‹ùúíé»ß!~©ñAôõ—×ôvþìÌtí§ç© ÓuŸy §»ö¿Žq™·qnzžøFµñ+/ÎÇS¶›Çvó±Y–2þl~A¾w·ÈÐË‚$ å Ox`0mÏ$Èã㺹޴úúŸçׯNx=7¶\~ŒÆfÙ(qÚµÏc]>Ä> ³LiÏ,ë2]æêËËÿ\êLqâ1½šo1±ŽW—š:)Ëôýz©f}Þt¦ì&ÞÖ§±ÊœÞÎM,Ëý¦3ñ…IöæTÜ+»ÚÒEÌ|üú¼”…ÝK º(.ò´î“Í[4X~ÃæV‹=ØçÑLEó!(õ‹u^BKØ^—kYá¨@O˜ìq Kfj—v éùÜR¾§ÉvzØ}šîsͦ‹íÙ…¯ßâW0ªmEJ™³&®Vö­òXŸ‡³JÚGÒÅjו‹mÅ\,p3p—ºq(óÙ"ôMzÜ°JwVëópV¹Ÿv.ÓIÝ­ôÒâ>oüºÃC;Kw—õÈ Kí’Jþd\Ê_µÔc}ÎR¯d¢¿ -¼ÚðÈ…¥ýg©WrÒO^ÈÃ8Ò[w{M°ÏÃYèÁ\46MÖ÷ç|[½¬õ.ÏËxÕ½ ï¦ø-ªßÕïxþ`Ÿ‡³Ä+9ççX¿ÿÞõ"8©Ûkõƒ}Í*éÊžç'‰5/äM#xÿY‘™%>uV½óY“åGÀj|å]Nºã½÷ƒ}ÎJ÷óÌå'sªÖ—8Ö}ØŒ:¤Tß+õ»2W;=\çaÓužÃ¤2$3ÔÍ!týÔ¼=ëE]FÏÔ~Í÷2ñÑ!>Y>fn’ÉNëâ<ßcÔ!§Xã-ŸÁÀç|­ÇQÓ¼F=e]íɤiCÕæ2Ó¯‘ýEÏ{óŠŒOÂØÙ>ÓØ?ÛÈÛi¬§ÞîšÎÍ›IE&7E—ØtÍsÌv¦q'ý5›Ú -òW{ÒÓšÙq¼ÞÞòXV–ÏâÚ`v_Çn©‡,ã$C™O¦Iò¹&{šÚÆÆ_ÊTl{ô¥No©½–æ½òkÝæXÆÏ÷|ÛÚÛ˜ãF·~¸‡åçËB·3yvu–ºË>jæëålÓa¶Õ‚C~gݬjùá(£.vz×aÌ\:ÝùN7=ÿÍH+MÆƧ¹=æ¶Ý\ræ º±Ë< -&„ñ¹ä£úJ+[[ø.÷¸b#¼þ®ø½õ ì\‡Ëõ®ò[îìmÜÏl&ô½¥m/l4ÓzùƒÚ^ÿ¸§·¨Wz£É÷'\YÛÉÞÝeÛŽ*þºNq®“â_÷ÄU£vçÂÇ¿Ìæb¦¹ôþZ°{õEõ¨oó| Êis5s<Ø箓)^0etUÞDCt.(á—ÍtYŽêâ²]Z5.Âæ “GÍç»-nøÝbC¤¦¤åK½û㛇LQa\÷öë+½)æC½ß’êJç‡ä°vRÿœä-mw`Ì|Þ09ow4õ1¾¶w`þdóá_|ÒÚç#0¡<6gz¥?¯ðÄš—Ó>E‹„”mßN]÷¥óý{-K§29`‘‹Àøý<¸Ïl»•4¯“R>T6ññ‹›foßÂäu5ƒ5W&Ø«<ûoôÌ?q𾶚~õ@ñAÈ|f^Úþbjÿ>/ú»“ʳþó6I>¡\e2â€[’ŸO|B= d¢¶×r…ÌÄ饪c‘®[7…v–†!-[¨Ôô²íï0z©)#)³áþ®tã“I•F6Ÿ7ÞX ¢§×k2Ï^ z"ó)ÿq°Ó•¯¢pFT‰m@êÛ¶[Ç«dmUGtk¢ên×ȆŠ<šO…­ªóXEïU§»å/mMè–sx9á¶vI}©|¶xªhרJ³P©R]iY„ê: ¤ËI ~©ÊЄŬünÕø„Ö‘‹!Qm—J;t9‰í¦Ñ\]©P[ÅÖ5Í7Qš©ó6Ú6íÒíþÙ’ÎG”-/çdÞ¦«k­bg7fE¼œbhE~>¹ ´v +”+¦×m7n%GÕì¥QC,ª‚kŸoÉÿPá*•‘'¿ÐTMŠÏ¨mWí&웕p¸Ê[Çíl¾/9°Ñ’̦ËMÙ¨íÚÍ:b  ÐÈ FŸ3¹xkE§|ìÈô²9™q/§ŸN9Ìë o{x•a~p¤>ÿm^áñ¯¿)¶ŽqܨŠ¦bëŒÜG2þµ•xÌ¿¾RˆÕm:¢¡öYÇÖÖk¨qܘ¯©uW ØKK]TWnOtM‡ÙTÓ5Úvšq;ÄËB‹5‘ùеi{¼´er¼ÃGý©pÑO—±ñl ÉçnH\qÓÒÁ%ÓuÄE?(.»¦&ÙLL6Èþ&Uõ,?õKÁÛýE—Fð¥m> -endobj -634 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 183.873057 101.433071 165.123057 ] -/BS << -/W 0 ->> -/Dest (cb145-1) ->> -endobj -635 0 obj -<< -/Filter /FlateDecode -/Length 4598 ->> -stream -xœí]]¯Ü¸‘}ï_Ñφá÷°ÈC€l°ƒ$ÈFò°Ø»=N°¸`²ùû9T‹d‰*Iì¾×¶œ/Úݪ"©yxX,Q¢ºJü}§ð_´JĘ‚·×ÛO—Ÿ/R'ñoJ±8˜Ô?àÜUI„SÉGwý) L!9üõå‚ÿºCšüåòç_\þ¶Ì£®Z¤äµr׿ÿ˜UÖˆCŽQ%¨¤2ŒÊ9‚vžQy%p‰)q*˜á‚Õv­ -0æè8Ì°ZGFa†±A™µ*Á ”åT0C%•’°Ci)Y ‘ÖÍ蔕î9VÂǤ« Щ 91«œâtIxï]ät¶¸$§s°Å)k%§ƒ-ÖÄÀéÇÚÃIèàò±:/pì8.Êbu°ÅËqk‘LŠ [ŒÖ [à2p\@$ï›ÀÀ…XØO‘ã0™ˆÉZŽ ‚ó˜¡V9×"Ð5«‹Ð©ÀqAV`(V`.%":=Ç x‘Ý%Ž `tÊr\lAWá¸C‘ˆž9§S°N6Çðt¡SŠã‚¨a ¼AŽ ¢Ž˜ezÉqFhÌ%QÕœÎÊ<ÏŒDëEv9.@7ÁLLÉê` h‚ã´Ožl;Ž 0ùÁdÓV[Œ³Ĩót3r\#lÑRs\læùfภIØã‚$½Èn2Ç qè¤ä¸ ©ˆù¦¶ _æùfä¸ ‰ù&¦xL¿MÆçùfภYØâࢰ:ا)1ýœ„ù&؉ÓyØ¢“å¸ yŸç›‘㸘o£gC1Ï7.ïºfâtà]x¦žã`¾ Ï–›ÛK¯ ðÏxeÌ3ÎÈF $¨×aleCÜ‹Ñ*°1 òÍs$6h31³Œž`X…RI6l Á¿ðÇìŠîÑ'¢ò€CÀ˜C¾È„naáÝ×9GKf6’ÝãcÿËå—ï1ø^ÿò—O¿˜"eåOÁA˜R[øWè:¡¹¼o…Ë7 ­Ä$´“…0ð™<Ùˆš‘ -´8ݲ‘þò¿ëúÇÂrþÃtzSN(N®¦ÐrÄôuŠ ‹Ìò†KLŽ0Úz¤1IgväàWJ»Hüëwu¶â­‡Ê¸b>8“a«ÝõÝO—_þîû?ýá÷?Fï>]þû?¤tîWß©ü­»ïpÿ–æþ Ç^>ïï¿WòùSå¶K—Q7êãœçCÓiMÒ™ùóqþ¶óï\~jéì°1οçk°ó13_GI›ý2m½þ´–U›dWf9oèÊ%ý¯þçúîûËoÞ-¤),qM™›v±7–å5Ø{CÈU«Á »˜3K¬ÙOËz´s¹Û\ï‰è>-Û/C'G“kÛ…eYWÚ³¶ŸjzûaÝÊQÚï\ƒ ¾ô÷±õí[iÃŽíéd /=m²Ï~Yûjï)å|ìêË6¿diÓ*›Ç¨Á/Rsèdæ9*÷½Ë=/­ÉÙQ³ÏÃðÙ* Eíø<•·Z6>Á7cyÎÖ²úÀ‡+5y#£dW›¹ïèÛ}t«#gßzŠ´ÒŒˆœÇä¾ö ¿ãÜGŸÍÿÌiüi4oå…¸ïkUÔvüâÞo ·pÊŽKù¹ 4×ëÐZ{µ‘Å©nޘ͑Ž˜oPàt0ÏépºïÁ­ÆEÚlö·KÛWy‡cnìé=A›Öé½²Íñ¬›£ŽŒuµPÏ‘é–ôúmºëûx÷ú”æYOIÚ± É`žÓ¡ißËÚόԣ¤G”"-JǶ¾Õ?-Ë™fˆ…Å™åR|Z–ѳÞj>Àùð¥¬Ñ™Q·æ!GÞí1µ§õ,ù‘™kÝ–¶åúÝŸ5Õ"u,Ïéz×ë8­GÛrSûirk´¬ÎŸ¹U´gZºoE‚î×xa“ïmîå¾WvlÍ‚©-=ª9×S³ù $R¯/ßqŽÓ]Â}$Žå9ͧø•c•GºQy¡+ùJ¯é¢v«k(ç¼!¯œs±=ºÈè¾@oÇË£ÑQÕã‰Oì—ö¯üE…X”µs—eÁ=êItãÔø'¸t,Ïéyàvq‘Z{tôÚš¡Ž¢µ¾ùÞãwd:ä­¢™ú3#•øžJ‹$£Î+nö‘:–çlHµûþêÒ¦ò#áAÊk[üÒÝ™¶;#ðó>ݹQ¾íµð¨´$&™œpÓäü•ƒyN‡Ê_ôE¶Æ[ù}#¨Þ˜}(ß@¦yœ/󜙾æ[Î\PþÐìæ­MÐ1­Ô9 ‚7PÚ|ÉáÑ`žÓ¡tlãgŸ{w¨{¹\ã[ß?²ññyù`ž³¡Ðíû–ÓJ’²Š–‹jÓÕ®~Nǵ83‡?œ3“(ÌV|³´p‰¼¯ÎûäzŽCO ôʧwÒÇj·f2ŽÄ"GGæÁ<§CÛÏøÌ8Q9ö¶ýåüä „šÇ‚æ9BŸ¿Ÿ>Ä'F¸ßþ5{Çb‰O˜Ÿ£ÔQiw„ر<§CìÁŠÆG‘÷žSyh–ó¥z\ç³¾õýuGüC§…tÖgôì#r,ÏÙé¿Ýûë§í-»3*q^=¾¶h0Ïé7öTL/‡†Ð5AïŸ_+¹yÿèƒ\¯Âx#¯àu«+ÞÚØ@%¹=!Ìs:TÄ!ÏÊi;³°!d?0º–^Òqå&?ÒûÖƒ+Þóœ‰>âi8ã_¼W<3Š?±.s0ÏÙPýÆRƒV®ž0Y­'([­êå"'OͬV:H¹ûdNWûnŒ­ÝšNõñõ¨[ñíg¯CÅA=>Ìs:Ä8¿ê -Úo—¯ŸAî&Ÿˆgæ9â™òã+V`|þ.2z°Cõ”îréSu»ÏùÞÚ¹§õ©3G/Ö¡öOî Þ# -$6éÚYYytGr0Ïé»ï«öÏ×r­ûU÷ž…K:bÏ3È{MÈ;`׈H,rcyΆÀøõýÐþYÈ2vüͯ2·m]õƇïÿo¼ªù‡ù½ÕQYëŽî›æ9"ã›{«#òëÃüaž‰ÜõÕú;ŒL„»êúÆ[~íÆó3«žäÚ5¼ÅÚžøÄ:ÈÁ<§CÍ¿Ÿ _ ³6ük÷€Á8e$1Çñ×ð å9bÿ§\ô¬£²¶P9°ó9´o 4>þ¾ÁiÊ•ò¾“Ü\÷æžî2ä´þþ™ÊšOåÄ2ߙϗç69~5Üõ‹òÝ|nbÏT†#×àÊ5¬šòç‹ŸvRÓ.qÒG¡tÞFöê­0¹mÒ5ïJ¥±ª4š*[]7rN›ïþ\SÍûñʼE_Þžé¾Y/9˜Óÿ<5úO•¬ÈõŸâ¾w¦œÄQ—ƒ&~¹ä½S-#¶pÅY8ÿ”µ,«JÂ*»hƒH}Jã„œ¦ZxÞŸãþS»r59¿+Ùòž6V–DfÚÕå~bFˆœ^­Åùê¥P³8ï?…™|5Åùb3„­@KÓ¶[ÆC±2’ÝZ(ÉéšXÛbÊû•ÕªóMXM§U§Hó—´:æÜÞ.8­é¥/¬4ïzYŠ*µ«e•bTªRWRÖ±µÑí¿“¥h‹Æ¬zÒjyï°»0 Ùš.•tŒèvaÓÍ¥¹ÚR¡¦Š-k*5ßLiP7±V<#›êh–†¼¡6é#Ò”ŸK±Ï-PÚZF‚½Þ.bìJürqIíWHW G…†”[ÅQ6¼4i~jùþ ]³ÈòÎKùÿ;U¸*ÍÌ3ýP¶B*ïžÖŽÚIr߬âe%Q¹¯¬c¹P6´´¤ºT³´¾T”LúLº5u­±…”T‘‡TÌË$UÉc)VuUœ)¬ÔOåJ*”Åœ®‰]…òËBÜìtµ«ìåd^¸»ZnûÊ/Tj‹«¼^µ·–K0_„·i`³½8C?–“4±–u´k'#ÂfH¡‰¾LâÒ[+;åí½ËÏÖɸ÷vùëeæ´q:á‡7*ïìJ=/uýÒ|Ïÿ^ìõ÷e+±û8®ee ]¹u!¦#YÞ<Šó¾àEX» jKh¿‰Uli]ACǵ&|m[vÙˆ½¤T•AUÕR¡kuHĺBW+CjÆío]-VGæµmÓìû¡)‡Ó˜¼ ]± -ôrp›Î´!ùJŠÄQNÚ†th­&q@ Å!9aj–-ÌÌ€¤'©UŸí·´)L>9 nD>4MgÛ0K‹.²é*H¸À†Ђ­¢'M {£Ÿî^¯n÷Ùd_z@–Vj¯Ò*­–¥•ðx©™ýò°Ž9æšJçm5TË2¤ï™êä´®Ñ\ »8 %ÒØ²Í =©-²ÅIË@Qé2YdµÄPlòs¶R-ÓG¶–Bõ)šô¯´¦m•Z‰Ž¶Š_¥Íå–¯zK¦:@ŽÂ9´«“-eõ%##ó4w•Ú–ÒÙúÜvyî2Ú^ÉeØfˆož©lÙÜ*åSü÷Lß¹çé"\ªò7EGTl›ob ¶*m®)»;Ñ{pˆÍ[bI¸äá<ûñ“u”¤ÄOè™ÐÂ#ƒÞ6Lx"+HQ/ó¾¥|¶$ÙKz¬˜ðP£9¹Šaæ8æ$ª‡!O]š6°–1æ ܃Tl¸(oh©â"|2ýýû„¸K -endstream -endobj -636 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 635 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -637 0 obj -<< -/Filter /FlateDecode -/Length 4603 ->> -stream -xœí]K·¾Ï¯˜s·ù(¾€@‡I#b@HFÒHk#X°sÈßÏGv“¬î®îé­ä‘m-V3]E²‹Åª¯ª¸=}VøùBã¿Hzˆ1Oç˧OjNá_i1»(ìoðÜY+ -ƒÓÉGwþ!lBr&øóó ÿ-.yóçÓ?wúϼ>›!%o´;ÿô>³È!† œÀJ`)m–sCÆyåõ€)¦$± † dhÍ -ƒRt b1Q`Eˆa)h»f%ˆa‚&‰1trA`i9´QJäAEdÀÓnðPº—xF>&“D^O#ñ¬| í´ÄKƒ÷ÞE‰GÅ%¥%žƒ,N)‰YÈÆ ñŽrRÉTû°2¤Xõ’éÆòë³È€E›5=—g©WrÒÏ~‘žå•G÷ëöy8 ½’‹Þ‰;7?Üñ[¤?é‰ïeükçÁ>f´Ÿ‡Þ]÷þ#ý=S}ìHOööšþ`Ÿ‡³Ô+9ég‰?/äaéÉÝ^Óìópz0åÏÕ¿Ïù¾zEë,Ï+xÅþï¦ø-ªßÕïøKüÁ>g‰WrÎ_býþ1¼ëDp§n¯Õöy4«tWöÿö‚OqYÔ;ü>¯h–Fë{Pµýq œãñú¹Ã e :l¡Çú<œ…Þ¸çÉ0GÂnmYâ^½|—>çs™å¬ž­ßŠêwü­ý`Ÿ‡³Æc¹æ‹<å¨?Ü·žµßzn~öt^¶¢éƒg›ã,ç$å‹Â³ö÷Ĉk˜ÛÂJ¯nÇʃ}Í:ý•œs™‹ý*ëåÇú ŠpÃ"ïø|ÑÁ>g‘WòËÅJ4­îd÷wa˜„…wäƒü©¹Æ[âðF%´Ä̆‹Ü»¤Ï¼h]¾çµÖzÇžæÁ>g­W>®þ¹bÚ–5¨Å—;X3Oü¤´a=Ô„ÕN68uÍ:õy8ë<˜{ndïÒM[¥\ìêgÙêš´Qò\ÒPjï:—vÉN ɇeymµ‰e._ýé/þûòóùy_’åÂù`ƒñpÈÕæ2—·Ç,ßØÖ…ìužãõŠ>µmU`ìc“ÇËÞáj¿ Ï»9ÆOm+/m)LcºšQ¬ûSmûf´ôCjô\ßÖ:5ß#[̨j ÌÂ×¼…û3kCó6åÚ -3®Ë &-W­*`£!&O`ýÔ¼}Ö”ºLðÓÔ=ŸsTù\)›Ïžæl§ÕvY«yŒ§q<ýn=Ž·ùЭ|°ëz5Íkœs™÷žHÃ6äP}.3}«=P-ð럄±‹Õ§±ž_™cë ·æ®só~Ò…ë–WâUìºv“7Q˜Æô×íej+ÈßlAOkf«U+V‰V;œ{$ÐfO¯n9wk¬ÝòÄ‘o6î_îY¯ŸF9ô4ËôVæõ4µgë7G ¦cÌßæ×Kõ¦ëÔßW»*:½­«÷ñ“lUF¿£óÅ< -j™]}†¥>ÇÌpB£jO—ÑVÆœ…­­gúáü÷“뤦Ï]v»)mÙ¼ B*6vsõÝ0¾öÏ“vûÕµ_Ã-Y×y]]õ95ÎMÙªÃ]=¥•žëÝd÷̇êúê>Ïnçsÿh2oØÓ<ú\·÷yĹmd¿hxç˜Î8F¸=•süV:Ë÷’æ×î5ÑëºPZ´gQzå·ï~]µ«‡|Nà³h²_†!ó—9,ÂÁ[XG*‡E£°7Ws؃}î:Âì“W×Ïc¢!:×–?zœÍ`Yë -!,Éaó†É#Ï/n8†PV‚Äžtõ†oŽklœÇEÆuO¯®ô¦XNÏö~Kª+ý’ÃÚIýK€]¤üîÀ˜å‹)Èy»£©OñÂó'[N‰Í'fÞ¡ý|V:”çÃæL¯ôÏ«\´æå´OÑÀã"!ÕÛ·SÇÛ½@å›éÊàp7çòÀÑéÈN´lHrµÉà¨åÞú´Ã³^œƒ˜¢ƒ&†Ìß=c#¼b òq­R˜séœÃšöˆ1¿Ë¹–òÖ¬œV)N1ĸyL(§Õ¸=å-5~”üoŠÉ%¾¤‘nZþÏr¯Vyq¹ß”ßòX2'—js¨ç¤WË¥üñäËÑú?¯|´ÉßOsFá—S×Îù¸ã¨,éºtº([Ÿ7z–oõù±µš¾èGå³ÿó¹ÏeiùÅÔþDzè? ,E¨S#ÚŽ_Ê¡ -í–äçSþRÈ(ÿÍAr&NoU‹t+Ž+í‚ð†´l‰¤¦·½‚ÎqIMØZg“û»Ú-–Kª6²å¸ØñÆ=½^“óìÕ 'rþb›8ØéÊ7Q2¶7bxÛ>`ïx•¬­bD·&*v»N6TåÑù ô¦:߉Mt®:Í–¿¶5-çñrÂmí’ú,Ró×iÔ¡ªvj4Ë„Jêj˶ ÔÖI ]Nñ U‡&,fã³Uˇ’ÄX ‰Z»TÛ ¤ËIl7æÚJ…Ö*ö®©j¾‹ÒM=o$lÓ.l—)äoêb>¢l};'状Ö*2»1+âå„C+òóÉ¡µëX¡\5=N´lÜFŽªÛK§æS!ÇwpÍJËG:çÿG¨pš‘§¼ÑÔL*ËÞ¯úM²o6ò\Iœîê¤M¡·eêÒ!4›åúÒQ í3è¶ÖMc3*S£‡TÅË ÕÀcNÖÍê9CXÕOÃJNTUÜ®“]3åç¹Ëéš«:Õnæ53w×Æídß°ó™“C[qÏØoò¶q™ÍWâ¥6Z’³éÇz“N6ªE»~3Fì‚:¹Ãès!WomèT›Þv'“"îåôý©„9cÉœGzæ:ëóÿÐæ+üþûDç¯ëåc7ª¡…iØ:#óH–¿`,$ç/«Äæ6ŒhˆÁ~'ëØÛºj -ŽÃðšzwÕ½¶Ô Auãr¢ë:ddÓL×hË4ãvˆ—…["ó¡kÓöxiëär)šO·¯Rá‚O—±ólÉg6$®rÓÒÁ%Ã:â‚ŠKvÃÔ%›‰™ ’ߤ©>ËO|)l¾»`iD¾´G=Ìò¡+­Ì‚5A -lù˜4¿išÉ#}·{}ðºO"ûê™Ú ½QŸ9µM,SàÉT[.ýü²Å{ÎÕy»†ÚX–ùžmINwžZÐì‚Û(£ÆÞmjàøM©Òf7­¢ÁhªÒÚˆ¡Êä§nU-eÆQÔRh9E§~Ï5íøª4%:¾*~Õ6[#^Ë–lK€7çÐg§zË–KFæyïF¥ÞÒVÚúÞ4¿w¶g6 ê‚øž™ªÞÍ­ZÞ…÷øÎØg±!µª?ewDÇþ­Ô¤ƒ5¶|Q«ŠÞC(×V È¶Kn²Þ%©û'üNXŽ¬UAç¯Ü•Ex#]g°¡ê×ñÊÝ’§t›„í¡sjµ;T¾s±> -endobj -639 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 108.873057 101.433071 90.123057 ] -/BS << -/W 0 ->> -/Dest (cb148-1) ->> -endobj -640 0 obj -<< -/Filter /FlateDecode -/Length 4595 ->> -stream -xœí]ˮܸÝ÷Wôz€ÑðQ|ƒYH‚™ Œddq}mÏ ¸àÉ"¿ŸCI|ˆ*Iìöµ-OÆömUñQ"«¨åUàó­Äžäà}p–®ï/.bpFàߘbq0ª_âëÌU -rƒ‘Ázs}Ú .åìõé‚ÿšÃ:ùÓåïß\þ½Ì#¯jÁ*i®¿¼*ÒƒóΠèTBjFeÌàœ2–QY9àCàT0Ã8R´V9˜AÁN3H)Ϩ<ÌÐä¤^«ÌPN§‚2Ǩ¤€R Áê`ˆ ÒŠÑI3X4ºåtJÖXƒN:Åé´¬#i$§ ƒµÖxNG°Å!9-F N[H{Çé,lÑV V[”wšÓ9Ø¢Ð:ÑyØtyV[„’Ó=˜à‰X]€Nxç×:%Ì`¼Ö‚ÓI9`S³:Œ(t’uŒNÁ+L`u°Åh¡8†-„§Oí¼gF«Š£[ -9-J9bu°Ï1€²°E8ËQFÐy‰ãåÜ@AGÊë=Çàø3ªâ¸À€w¥ÆLÍéÀ»RÁqïÂo0ƒ„Àqæ·A -@—Õ…Aï9.0à]В⸳ð <˜„Õ¹@’€Œ?”ã ÞÅô8.°à]a´æ¸À‚wÁíau°E{Åqüè0l™1 gu -LÁéÀ»B¢‡9xW8.°àÝ8mr\`](Ïq à+Ž ¬§!8g9.°A@—ÕÙdžã' 0 ,V[ÈÇp¬‡ ƒç¸À)Ø¢•â¸À)Ø—ãxCð¾Ùð¶.l|@°ž"Ç`²Á"Ž œ±ˆÇЪœÎªÁ;Ð5«óÐIÇqs4`*– –<=Ç ø!ºKÀ è$q\àlÁPá¸SÑàFœž]1 ^ƒ¹•]2à^ÌVŽ]3 ß#±‹0‘¥·ìª¦U(¥`— øþ­8aZ}ª—¨,àà0gzO2`X¼ûs”dz#Ù´>öËO—ï0ù^úÏåÝ7ãJYúH8cjž1 T@wxߧ¯-ÌÑ2vKÚX© Å( œÙ¢Q˜îâ¿ëúÇÂRúrY-0ÅÔëþQH¬­½™žÑèu³ÿîÕü2­mÚX6B;—30ÈØ0zõþòÝ‹?üé} \¼zwùÇ÷½¼ÆKøM˜.0™L¿]ô˜âZÃtl¢.ÊßÅ4?üóúêE]§Šavˆí²ÎX|3%ÿý«±étîf/il9 AèÐ\ Dhã´UµöY>¦­ç&Æ)À‰Áù!ÐW‘ô1’qv‚VíN©ò©)ò¡Ô&c"AèÊŒÝðçûËU7ˆ´‘˜¾äøVBfdìŽù·™ÿ>Æ®š»é/=ÎǪä‹åät¦Ñ‡’‡R~™v´Ã®Ë¥:ïÃd·Ðs¾$oíÈuÙæXÎç”’A¾‰miô¸3pÝ\c注ÁψfPûÑš5°1’ƒsÖT¸‡l‡ §¾’äéÚyN‡mزƒíþÔ¿ 'æÙáû0ý^Éço–S“.»Ø#sž×E§T•NÏß7ó_šÇòC…ç·óï»ãø %¾[ü·cjw,Û$š2S½íw)=:%tFP¼Ê“ŠÍ.ê:óœuÅlX¸‡:½D½[¶df·Ç¹åkf{·ìÁžx *÷ž[–u™ÝRVìM¯†­ó¼­e溺‘pã¹Êéo._,óÜÈRF¶ãa©ªpŽÈ  ‘Ú—çtH¥}~Ô r\AMBÞÈ´FЪ§9t="oáµÚ¶ ¦1ÝÈ’|YŸ ©FߎԾ<§CªÝçTÑ êuù[pÁ—vš]Ÿ¿—6ìhО‘^Íâi¤öÑçµ/žTΛ¦½¨x&K›6PY|E¥ì  #ÿ²3ÏéP¹ï_îùiíLÎΚm†ÏVijPƒÇçIȸѳZÜÎ7yÎÖ³úÀ‡K-YE½mkƱ£«è‘›¡dÕK3"bÇÚ»jäÞü¯«¨Æ.‘Vçͼà÷}­ŒÚ†_ÌÃr§ìø°5?'„ƶ3 ZóHªmäqZ¼1Šë£>ÞÖp„Ó¾<§Ãé¾·š;ärÕdįhú²Á17÷´ž …5FZ¯ls>k¢Ôž¹Ž[‰âÆUc'ÿÕÍù½™¼>©6X¯ò˜õ­—tæ9šö½¬½ùlÁH-JZDI±Zß«dš9²ö·3‹…*Ê­QðnYFËz«x€óáSY ¢#£nÅ!GÞZ=bòHkYò k±k¨H­Wéz‘Ú—çtH=XÙk8­EÛ¸è&÷ÓÄÞŽë´*~ç^Q–éé¶+tŒ6úÞzB(÷weÇV\ÛÒ¢ÊTuÝÍóH¤Úë‹÷©øñÞ‚]$væ9éÀSüÂk™GšYy¡KùÒ¨iVíVçP]óéA^{Ýe…Ø]íu—ï›ôTÉ:VGyTo ÷ŽÕ¿Î<§Cî¾ïxæ¼=T4ª¸«›í¡„àÚ¥e;µhäæ‚w5WhVk2³?Ÿ“0©KMÎÚ#äöå9rVuƒŽ:Ò¬féz¦=ò¦+¶M/µ3y{UcLJ½ ½®OÛÊ'¶KûWþbµ -±(kç*Ë.‚ﾦ¿Hw—öå9"üÑÏpÇÃW}E¦AÞj5S}Z¤šÊ÷”j«xŸÞ.R;óœ ©fß_íAÚ8C¾©x°æµ-~i®LÓÎ |¿Own”o{-¨¬Ö$ƒÌœ¡²/ÏéPyà‹¡èÖÞx.¿¯ÕQÑ×ò dš;ø²/Ïéyàk>gäÒò›¢›çFt…ŽñNÓ x¥Å—쎈:óœ¥}w:~òØûS ¸AÝ=ȽåŸûú‘·ÇåyΆB»ï[Žw’¤ûh¹Uíú~W;§ãzœ‰ácæjfk}3õpZy_Õ{çý‡ž@5Ÿ6ÞI»V»ÉØj-²wfîÌs:´øŒ÷\«VTÎãƒ=/G>?y¡æö;Á;óœ¡÷_Oïâ“[W¸Ÿ‘~£c±•OŸ¾V^*s„ؾ<§CìÁ·"ï3<©rS”ó¹F\ã³>÷õuWù‡F ÂèÙEdgž³!Ò}½××O;Zv#ª Ä©Ûï-êÌs:Äõ=Ó^ËaWCê{‚î¿WróúÑk±¾ 㙼‚»»â¹=€ TVׯ{WÈ;óœ•ëgå´(¬ Ù7ÌîŸd”4\¹ÉõuëÎ;Þ:óœ‰>âi8ãW>*î˜Åý÷evæ9J}§ß˜ZÄê “Õ½ãÊVwõr+'Oͬîtb÷Éœzu:Û÷ÈØÚÜÓ)ß|<êV|ûÉÛpÅêöx¼3ÏéP|°ÆùEï ýzùúän¢ñŽõÌÎ<§CãÁzæ­üøw`|þN²ú*`ƒê1Ý åÖOÕí>çûXêïO9zqjûä^ç5"_­MÚ€~–$Ž®Hvæ9b÷}ÕöùZ®w¿ò˜Y8¥«ì¹y÷£ y;ìêñCCµÙ‹ÀÎAv®/Â3æOÌ/ù^”oæº+{h~or>“ÎaÕ•.vÜBŽ;‰èɨ¸…àÕÒ cß„kÜ‘Â M2u[^7rŽ/~È©æ½EÜž)nÍ1mÔXÌé?Œþþ" ±ý#Lû¦‰QìU:(â§KÜ71è:ÅQ8ÿ¹,’)a–=‚UÝÚ”Ú bþráñEËÓOeÒÙÄü&e‹ûH‰ôøFÿ©bFˆœV®ÅñìÅ gqÜ{Ðz>²Ùc“Í–©N[ -,ÅR‹JhÖBQUWÄŠ’=2îU“›Îa6½n:YuJ«\Õ{ÂÇ ªÕ­ô‰•ÆÏRQ©u•È2]²Ô¤”¹C(÷#z¼0ÂoE*šÐ™Y_õZÜ7fú$ÊéBJLj/lº¹4“{ÊåT¾d ©å‹)êÚç†gdcÍR7S­ÆÜ•ú¡ÛØ©¯…¯p£VÂÇ ¦Z‰Ÿ.Æ1©Má -aôj¡®ÊÍb/ -^Š4>{2ýÂÐL²¸ëFü¢ -“¥‘yÆ’2¤âÎ9å¨TÇf/©–ÛÌ:ĵ€$WÒVÍ%˘­ÛKzÁ¤¤›Sç[H«&«ä.$ó"IeòXŠeF]G -K퓹²Šdª+b“¡ü´;Mª{ªÌÊ -î&—[Ä6sçS-v¹ÇeÜ)ۛ˭0Ÿ„ãÄF­8BߧJŠX‰<Û•Ê*a1 ¤PÄ…FŸFq­™âÖ®égdÜŒûxùù2NsJðÃjwõ«=/yý/Ò¼À÷_ºþ˜¶‘™æq%2[¨Ì­ q=“Å=`}§8î ›„yØTBEí±ô%­IhÈó¸R_SÉ. -±§”23¨ÌÚZhJVb•¡«¤®ZÆì›VÌŽÌÇöM™°§CNÕè¸Q² -õéàÐ.Sòµ*G1i™Ò¡%UeÄA](« -C±lafd]Inúh?Õ]¡c]ÕAåFÄC]tT¦Ùºè$Ï¢JX×uÁ$ëJÃÂÆ軇×G÷ûl²M# J3µgéS-Í'¥™ðx©íò0Ï9zÜq( ÞÒB¹,]=œ24ŠkA‹ƒ£•Ô—lsSWJI¶¨4M™. I–KtÉ&;gKÍ2ž±g[ÉeŸ¢H®[ÚÔ½’ÑÔ½bWic¹iÆËޒΩáìÊÙ‰’2û’ž‘Ù:w–RI©“l]7-ëN³íµ: *†Ø♊’ͬRÞÅ÷Œv×..UúŒ«#Ò—×0Q“tZékˆîŽ·Bq;TG¾Z.¹9ÏþúÉz•$­ŸÔ5¡;ÒZ8 ¼m˜ðE8VTE=Í{ÖñÙ‚`Oé¶`–‡ -Í­÷X·Å×,ÇE¢|èb¨h¸y©ð>nÞë¿-UܘH>,–OÆÏÿLW~@ -endstream -endobj -641 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 640 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -642 0 obj -<< -/Filter /FlateDecode -/Length 4399 ->> -stream -xœí]M䶽÷¯èsËü(~’ ‹8ˆEr0r˜mïØf¬sÈßÏ£$’%©ÔR÷Ì®5öî ·[U,ªH>¾*²Õ’>+ü}¥ñ_$ÝŘ‚§óåÃéãIuÁ)üëKLzõwxwÖŠBçtòÑ?d ]HÎ~:á¿Ù!/þtúçïNÿ™Úè³éRòF»óÏﳊlbÊ ª•ÒVP9×…`œT^whbJ’ -n¸@†–ª7(E'©àU„–‚¶KU‚&h’TpC'•VðC¥DQDÖ:í:N÷’ÎèÎÇd’¨ Ðé`$µ¤–t©óÞ»(龸¤´¤sðÅi"%éà Ù$‡/Ö[%êà‹‰ÁJº_ z' º_€®(êà‹òJKºd;—"‘¨KЩâRg”ë\´VI:­;lZQ‡…AòAÐøâ•K¢¾8«Œ¤³ð…ð'éÀΆ…Ùjòì¶ÊjIçà‹1D|ÁàK `<|QÁK€Ôe^’8À„ÐQ2$‘€‰¶£HQbSG!X‰Lrù$°JCg´DVÁG^b«á aæŠ:øbI\` |A‰ ¬…/†¬ÄlK@‹ÄtK*i‰ ,ø–”öX.ÜTX0®ž$.° \@)I\`Á¹Ü)qébº‰ ,X×:o$.° ]KÑK\`Á»–´’¸€À»ÖÂRÔÁXâ°cgu´x×j$. ð.¦‘‘¸£Ú‡^â`¥31*‰ ¼k¢"‰ ¼kB€¨ æ •¸€À»Æ¡ Rä琉ÄÞ5d¼ÄJþQb6Þ…89€/ X‰ ï¡3VâTÖ¤%8ð®NÂœ[u:&/qïêˆáu©Ó=.Ì[ôr§½O8ð.••¸ÀwAŸQâp|‡ˆj$.pà]m©%xW›¤$.pà]ä NâÀ¡ÓH$.@|ë´tE]êTŠQâÞ-‰ …;Á$¢.t’–¸dÜ)ä¡xð.Â{’¸Àƒw•³VâÞU„´GÔÁÄÈk Ã´æ4’ÕN0…¤ï*–tà]¥B’¸ÀƒwsØ”¸À‡,0Qâ8Ø! 6øH] -ÁK\à“‚)Ÿ¨óŽÄAAà@Y¢¾P ‰ Xwɦ(qA0ðÅ#qA0ð)ƒÄȺ¤‘}‹Ëø¸ˆë‚/È%.“u1I\œÇz ½*é¼éb]‹º„@B±–¸k©.bÒK\‚ïrº$qÜ€N“Ä!ÁL‰ Šºh™K: _dK\€L:­%.ˆ¾ ”¸ šˆU¦W Bc-‰®–t¤ò:3J\Éw9U”¸Ó+M0¥¨ƒ/  ‰ 0>y±í$.Àâ‹M,sD|±Ž$.ˆÑäåf”¸ Føb”‘¸ØÌëÍ qARðŸÄIù.§É€Æ¡SJ₤#Ö›†$.À¼ÌëÍ(qA² -ëM,ñ„y›¬ÏëÍ qA"ø⢈:ø‚¤) óœ„õ&ØIÒyøbI\¼ÏëÍ(qÒ ¬7‘Ñ‹[1¯7ƒ¸]Þu Ì$éÀ»ÈL½Ä8Àz™­´¶W ^ŸÉʘWœQÜ1P ^‡Ø*n(p/¢U÷ È7¯‘ÄM¸‰•eôâ®Â*”Z‰Û -ü‹|Œœ0ì>ñ-*8ÄÌr#¦!»¯kŽVÌ®öÇ~þñôõ‚ïùÇÿž×?¡/Mȯ0 LÂp;dßÍ·¤or`Î$•=„ƒ÷Ød'BjNjÐâ¸é–DøËÿÎËÏáøwýém=}BNŸX%UÄòuîñ&/è°ÂâÑÖ£ŒM&³ ‡¼RѤðÞžÔÑ‹a´2÷/±&ãÎo?œ¾þë›üíÛ§ï¯”óß|¥ñNqxwn|½ÑeÔÍÊ`нg6—œˆ:5¼îªW1ÙãX¾Øräz{Û±^Ç|)z¢Q—}hö&û‹>Óvh’ÐúøÙ²w3–K/ÕÖoþu~ûæôÇ·S°õè6$FS$ÚÀç>“çàóSÀËÙk°D¿:3vYX¤ q)S»<Ü8ô~^þ*r‡×Ã0äù}mØË9ç¾”óòsö0ë 4ø”Ò­ÝÙÏ*3›Ò^/øWÊ–©ä¯C/ï¾@¢œ,ƒæŒÛÀÞN›£+ kè›OV ]Ä'÷ã´|,3MëÈòZ¾Ø3ˆÚâÏ;ø¸ CP^³ANR]k¨ÙªëÂü}ûè}#ðŠú Ô¶ÑÒ줢ɩßuî³9 -é*VØ)ZnQ†ž9Š÷óã[7ó…#\àéyšðzfÎ -:[2HXÐxÁû-tî³9:Ÿ‘8þ¢¼ò‚3g%É­vfö>ŸYq|¹á5‘!¡,Q\ÆKôö®]vÚ{׳þßrÿý0öå»Ö‡Æ°þ,ݳϳ~ßÄ“Ÿ³fYß •¡e¬õlª¦²Åb3gêžAçõ¿ëNK_ê"J-ÛVçȬlï›{kñÚ¨Ûq§ÍÑPi6²ÆÂp—éè='¦]Cæ¡ÙÝ‚‚½(칚7ÌYUÍúƒ¡|nC|Ý6[Ûq]ïÓÆ:ÆØÛyr§Íáy=ƒ¬½üŽe:³ •^7ç.¶ò¬›#ãk1úvçè_aqõÎ3·Ù¬™ç†¡°Ö§~-8”¦m¥™ý¼ŸÊ{퓲;0ß(Òƒ~=Ž¶¡¸w…³Óæpø¼žC.Æb;+. ®¬4¶vŒjy¼ùøÞŒ% —œµ>Ö¸?›û+ÌɳŌJù;ëÈÜgs8d^Ï0?E,?òÎΫȀEÌZ•nÆìN›£aÖ^Ï?³ÿž¯P6XRüºÄ Ìtåë’çøçæþ=ŒØù¥v V˜Ô¶|’’ë\ÿ­Ì*÷Ù•»˜³Ñ®#Ǿé˜ì«Ù{—ú¦~\A¥»ƒ+÷Ù•×3ÏyFyË>ù¯‹¿î˜¡/º§¿‚T–Uæ÷`JùM¤î³9R72Ñß_­ÌÈ‹L>ÏL”J,‡Ü{¥ÆN›£!”6ö=ïÙ£¼‡w¾dÂL[A§½}%¿ÓæpèüDùç+Í ^tÆ}†HO,¿ÜéwÚ©9é«äŸß@¤·_‘´ÓæpÝÈEï䛯ïøéwGz§î¸¢}ŸÍÑÐé®ç¡w¯{…‘þž+ª>u¤wöö5ýN›Ã!u#'}•üóB3ìÀ‘Þ¹Û×ô;m‡Ð¹(¿hö{…Âu„ÆÑåßÄ»q¾Dõ£úßÄï´97rÎ_ãúýSÌ®ÏÁ½º}­¾Óæh¨ô{ž¯’k^h6íŒàüZ‘ ª¯\k2¿ -¬æÁßrú;¾{ßis8”n\ÿ9ÿÂl¥0¿îÌÎ A«×¿½àÕAÜõ^ûW4ó £Úîì‡R~?§|¼¼îp¡ìA»ºÏæp½qÏ“q„ ÞZCâµõò^zÍ9æ<ËY\[¿Õïø®}§Íáи/×|‘«õóѸv­ýÚuó“«ó2ŠÆßž­Ö3o“”/ -×Úß#¶~3·Æ•AÝΕ;mŽ†Î°óWéÔzö··^¾±®g­WyÇï‹vÚ‘ùål$j¯^Éîïâ°+w߸%äWÍUÝœ‡WVBÇÿü -ZïØÓÜis8´nübýµrÚsnÃ4[áMfâgA+èly¤&Œv²Á©-tî³9:wæž+Ù»´CSGi%Ûü-[“zÓ·Ü–ÔõkïÒ–zˆì,ù|GuÐA¾±¼6±oË›?ýåÏŸÿ>?ïK²\¸¿£RÈ…_µ-S[ÜÈþ e]ȳ³´s8Îò¬/öä›<\±¹ðܺD÷Qï˜. e)ŒuºæOšúÓKõŒÇ}]£ŸÙ†³_úwu­ç]û}iݬ%T¼~Xö=¤Ài†˜>°zÔÌËlS¼ô£{žû•T¾ç¥ÍÏ<ý²ãù\ö¯´´¿+òoó AóMç—õ”Qéýñ}ï_óéÓŠªµe2 -=†™#ø–>àŸ„º{´¦Á>·¯oc®ë£³¡Ã¼ûÂ14éñshýÕ#)ŒõŽýWüf¸ìE nã<àC±dÁŹWûÕÍÛnb‰‘k³kЛ•ó÷ç,ǃzlƒ¹´öÔ>¼°¾™Ùå¹ÒÖáëãËOgféß¾œÞ«íû¡Ë9z{7~ë)m­ãåÆrïXß_Çg˜÷ã#´³_§l%3•Ö×|éïd:TZNÆ&,Áwã­>ƒCmÎå[-G§#»shÈ›E6ðò>£ånçü7°­¨2?ãÇÆ3ÖË”B¾-®…2/äùhê.È€ÕŒ=[0X´Jc7£o…/Ý€#5Þ'0¿Û‚Ç·Yn*—1ŽôË9ÞŸoœ«|l'õ;6ç}ÃRÅnRí7˜i1”O¾¿Å³îoó¯|ì´ÉÏ:#ÐØ<6éœo+•%]†N÷­Ï+–ýÓ“>ÖRã•T~ÆB¾¿öð´%v0–ÿØú„A ²¾þá'ªGSšøé”~C‚)ˆÙ)ÎÂñ£ªu!u43ÙifèÒ¼¤uåæ´,b[W>WZ“í]1Ë7%&U -Ùþ¶¼Ã‰!,½^ŠsëU§Gq~€Pììxä«+Ο!l/Û*l†›bmº¥P±Ó5±¡âÎ7œ¯]ç›°ºÎ»N³á/eM`ÃyMx9á´v.}¥ù±%¥ªÒ»FU™eN¥*u¥dªã$ˆ.'Aø•*U³ê٨囿ÂX€Dµ\*åÑå$–ksu¤B-›i*=ß\iPÏIѺì²ڟˆÆ戲åãTœ“®:Ö*2ܘ…ðrBˆ¡…øéä‚PÚ5®P®@ -«·Š£jxiÒ|gªá¦f‘å[gçÿªpUš™§ÿ ©B*ßþ¾µ“ä¹YÅÓNâr_Y‡¤ÐZYÖ]:„ŠYÞ_:*¡|&ÝZºöØDʺŒÉC*îe’ªä1늺*ÎVú§r%ªâN×Ä®Bùi"n~º:Uör2¯Ü]­·‰}åÎ'.uÄu~–Aõ·ÖË0_„—>°Ñ\œ¡ËIšØ¨íÚɘ°9RhâF£O½¸ÌÖÊNý­óÆm’I÷rúéÔ‡9cIøà­Îæá™—>ÿeÞàõï¿-÷‚â¸Q•-LåÖ‰˜G²ü ·¸SœìV„uÚ0¡!FûM¬c+ë -j7†ñ55sÕˆ½”Ô•AuÕr¡k}ÈĦB×hËzÆ]^f½X™çŽM Øá-Ãil~Š@ñ -¼98ŒMg[H>³*q”‹¶-fˆ^)Ù Sólâf$?Iíúì?ñ¡°ù\쀥ùÐ6µ0Ë«.²¾¬R`ËxŤùIÓÄÁ黧׳Ç}tÙ—¥•Ú«ô‰Kkò´ž,µý¡ŸÖ˜cÏù LÞÖCµ.Ëæž­IN›-µ ÉÇ(“Æf6pü¤Td““–@Qé2UdµÆP|ò£Yé–¾ÅQì¥PsŠ&ý‰÷´ã£R;ÑñQñ‹²¹Þñj¶dkä8œCkj%k.™çÖUJ­¤-²å¹izîmϬÔñ-3UÍÌ-JÞÅ÷ÌÁf¶!•*ýíé)Ô¤ƒ5¶ ®ŠÞƒC(?Ó,PdÛ%7Û\ß?Yî’”ý~& GGÖª ó£ež HÛ -VUyì±l–”ؤÛ:@Øj4§»Cý³-ûoqúM¢zòR1?ÊÖæ¤7æ'ð¥å/-ò tßM¶Oú¿ÿMëÌ -endstream -endobj -643 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 642 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -644 0 obj -<< -/Filter /FlateDecode -/Length 4804 ->> -stream -xœÕ]Ko$I¾÷¯è3Òùˆ|Iˆ V€Xi‡»Ç!/Ò¿ÏYùªª¨ê²Ç;˜µ<îŠÈGddä™ÙµúªðóÆ?‘ôc -ž®·.?^Ԝ¹Äâ!³¿ÃopW­(LN'Ýõ&Ø0…äLð×— þY=ŽÅ_.ùÙåŸË:új¦”¼Ñîú¯'f‘B A9•ÀRÚ -,禌óËë CLIbA ÈЖ ¥è$Ä c¢ÀŠÃRÐvËJÃM bèä‚ÀÒ -rh£”ȃ ŠÈ§Ýä¡t/ñŒž|L&‰¼žFâY;ù@Úi‰—&ｋ ‹KJK<Yœ&R²AâyÈb½U"²˜¬Ä Å@;IàEÈëŠ"²(¯´ÄKvr)‰¼žŠ!nyF¹ÉEk•ÄÓzr°M+ò°¢0I><Y¼rIäAg•‘x²~$pÂÙ£°Z ¯n«¬–x²HäAL¾„ÆC¼XAã’„&„‰’! L´EŠ -˜˜& -ÁJ0`’›È§ á€U<£% ° -²8òX Y+WäA›HÂk -HX`-d1d%,°@[‚µHX`·¤’–°ÀoIi/aàBL%aâÚèI ȅ)% ,0×;%,°],÷ aêZç„°k)z ,p×’Vp×ZÔy,aÐq²:Z ¸kµ -pËÈHX€Y`‡^ÂØÊdbTp×DEp׸‘&¬A+awÃ$Ï Ü5N ¸kÈx `(ðG‰Ñpâäp²`%,€¿ÏX ÐØd–HXà€»:Áý kh5阼„¸«#¦Oä¥Ih\X·Ðò¤½O8à.••°ÀwŸQÂ`üj$,pÀ]má©%pW›¤$,pÀ]Ä N˜äHXÿ6iÓyiR)F p°d$,€žT’ˆ¼0Á´„ãI!•°ÀwáÞ“„¸«œµxà®"„="²Øh$,@\–­°¦¬NÊ)$pWiÌ°Äî*’„¸ËnS˜`¢„pBl$,𑦂—°À'B>‘ç'<; ‚Á²Dd¡@ °ž’MQ‚` ‹5F‚` B LI#úÓÈsó‚,ˆ%,’M1IXœG>­J[Tæà3càA&, BtßrŽ^Ìî›÷Çþõ·ËÏà|¯û÷åùgy§¬þh¿%ωµã%b¦p5åæduHS²€ï UX„ºˆ X¶ÜXD8b8ÛtmôR`ÈËk„2y\4v‹tY)õ.tWÇËXìÐ%Óiůc¬Ýò¹‰ÏúH½~í'ëïy©?xMêºm}=rHuž:_»Õœ¬êeÙ„¶j_k½P¨eæ¹ýõ§¥µGÞψǛ¹Á’åépœ«ò¦òŽë¢‰‹ð K@Ž(.Œ¦L½U¢»uÃraµxÌ°ˆ›él´ÍX`´šiŸûä.ÚxêFØúz\Ê׌»Nî``µ­fÈvÙæ(‹zhÆÐÂSÖ¦/jô×Á#çÄ›”Éúûö7¿ûíŸý™ˆ¾¹Ër±L,ù¸>‹™Ÿó˜þœÇ8#CïÔð1Dâ ¿U§À>ˬ}¬æ¨ƒ¹eõ >YçKœÒOðšSö}Kf¸äYç¿mÖ«µ®`®YõF×°£‹5ž·>`<°—f a¶çÊZí%p±“˜¥ÙÚ -!osp«lÅðyÕ<ïŽ0[1æ­¦C[9Yç£ÙŠñöÈV#ƒ+®gª»`žþ<ÿÍ+ü ÐÑøcTt+wÖ\èA›,—z.c ¥M×òVp•Í¾wÚ¬cÞkóUnyÔÙbü;Ö¨;ÖpΟ¬óA¬³‹ÍŒ>yí3a¥ú±k?ÓV3X7MKëØC¸6#kë[ÏpEÎbö©¯ž÷m{Ç*RÃ>þŽšÈÙ{Vq®ÎG³ -¾öp`¶j}¥Å½Ð·âǨåÜgµŒ•l³6âSXµéî[Å«SÒÆ“kÝÛ9+±¦á€ãƒ§3žídg%ö;ëSH®–³"¡ýÙ[ÏÐo^i£–-‰ãv?·kõà5JòÉÞ4çŸ{Ûæ)‡MÊp³‡­ã­m0¯Òªå›‡í¦€åñ<õÕÀíÍñY‘÷iÇ{œt:“—3j¶>ä>}ùeËjØ°‡KZm㥰´ÐŠ1µ]Zý홣œ¦)KX5Ӈ؟/>i>ÛMbìϾ™}·"$¶u]m˲&êˆV>µ!ðœ+pÙœg>Ö¨q~ÞÐKYCó¬°ÖsŽá;?ç ©Îm˜¥9)u¹HÎOžj^Ûåw›LjÙÞ¼‘ÕûÏyú+9ú)µ›Qíߔȭæ½ôR%x(Z¤™×4u›³÷¬-??/øUSÃhÕ#ÿf@/g«fqê¶ÉàFù}à!ƒ“w ºÿÛáÕè_Àܶ“5o‹l>Ë~ ‡SvOÕ3Ô¬§bmYGf^‚l|×+2> mÏÖÞÛ\Zï¾åò8v­ÖÎòÔL:yؙɲ§>×Ò8¸^+óÜåÈÛŒ%sçºùù©ÌdÒC¹2‡uûqGïf­›ºÂr[Uæj¡¶¹í_×z¶ŒiGWŒ<®Î£*6j—:XÏC“iØájz×]ïµ|“?¬æmgžë²­³‡:£ÎjŸnW\ʪËØøONyCéÏ×]ÌqÝõÔ¡,~#K]··¥0vY<›gF@ÐMñ}ÌcúcY+:—åuÀtö¯¼l)ŸËRñµ¥ÍìiÜœ5fþÃÜCVn×¾j¹8ûköµgåclx?¹õ×ú}¯1¼JÇCfÝ·-õس™Ò÷­Œ×—Ýèøô;èá¼~ý›7úõóœq÷îcøÊzý¢uõ6ýnð}_¿‡í¸´j‡ëº•ÝË‘‘/Lè‡Ï¢ù¢=’]äD÷ƒÏÕù_vµm_ŸÒd#ÜŽ~ý)°×ÓçÀË7Áq zï¶klšœÇCÚuÏwN§ ÅüÍï÷¤ºSßù)¡Š‘êïž9ßi3©à 451«­†8hñ6£OŒŸl¾aË· ß }þž Êû°;Ò;õyö³Xó~Ú§h°â"!ü>¶SwævÁù#°@j¢¨±D¾â‘é²Ó£c0¬ -jVóM{•ø†ð!ž¬óA¶jú¥˜¨Žð0™¶P¸&úËÕ5î·•WmEºÙ¡ÔfÓb½%³98sd«‡M‚°âýtǹnïÀ–ßÿ8wÙá‘GþŠÓl“NyD‰î]^9Wç£ÙqÌߘܵãÍÆŽäÐC¸‡6®éå·ÑiUŽÊVæçRç±óŒÊÙòû¹ü¥òÙ•åk{¾_ôuÞÚ\ÝZ¯ÑšònoL™fAü=y Êß»Sx²Î±º.vô‡Vg—Ö¶¹WQðV4?¬=/ggÞ`Z!eEáŸàŽÝyKxåXÛæR¡©e·[äà†ÍÜ]KMë¬:wûõdf©IããêVèxÁ®ZÞÿå­OÁÜ°bN[––ÛúZ– -{µ¥ž¬óá,ÕcªZYÔcÿÌ\àeÙzÿYÚ‘ceíãq¸3¥*}]ù6—»>¯ôE=2YÊ´c•=V4ÆOÊ{7¾ÏaßðeŸïÊwƒƒæãw=D§ãðÕåfCw‹Ü1™º•´ý>ÞU¯¾ÿ¶ÜmB“ü’A¯ùFbàïåïí5ñU7ë.E9Ⱥ͇IùP½\ÎW‹ÊUS÷¨²Š‹*óÔÌûUô¤Ú¡ø|5Gµƒx6¦›vDõ}Y¯ú!{ÆÆÒ_¾Ô®æ‹ Ã…€Ö~= äY\&H%#ÈcØLåŸß1¡ó{†”“6ü"«§‰ÏîBÈ藺ԾV­³²õu§f~}ã­Ty££â—<ñ >æ×=¥üyÒ¸èDS¾AÆ·’òÛ×T&GS:ùåÂoß#Ìßj>Ifbù¨Z[¤ÛM©J»ÔÖ%­›TùدSñfÛüѸ:®ïj5~+©ZÈæ÷Ì DÔôzKæÑ«I2¿Á0N¶<ù& -D5boƲ½Á^ñ.Y[5Ý–¨†î:ÙP•Góošê|'6ÑGÕéaúkY†é<"Þ.èÖ®©/"•ß›V›ªÚ5ªÑì TjTWK¶ ¡6Oévˆß¨Ú4a2˜5~ûÌLŒÕ¨•Kµœ@º]Är¥5×f*´R±WMUó]”nê66Å ´Ûpå0ð+Y‡5Â÷$ê<Œd¾éÖæZÅÁn̆x»ÀÅІürqA(í:V(WMo$Ú¡ÝFŽªÛK§†XU…¥Yiüîþw† -רŒ<ùQ×Æ§Þ ¯ÍF^*i¤û†:$i@SèeuéšÍŽúÒQ åt[馱uPÙ@©ŠÇ ÕÀcIÖÍê™!¬ê§aåHTUt×É®™òË‚Üåtm©:Õ:óz0w×Úídß°óe$‡6ãš_¦Ôämí6_‰·ìØhMfÓµ“N6ªy»ÞÙ@ì‚:¹ÃèK&×ÕÚЉ_[?öE&yÜÛåï—ìæŒu†o›x«ùÝ€c䥯ÿA™oñû ]ÿP_F3ûq£Z˜†­ òèÉò÷.O’ùͲ•Ø–Í@44À~'ëØ˺j Í3à5õêª{-©‚êƉ®ëp ›fºFÛA3î€x[i±2_:7Ýa϶ŽïÍðkŒªTx‡ƒÇØy¶»äëÐ$ž¸hwéà’*âalC‡©K¶“ r줩žå§q*,÷5< a?ÚΣîfǦ+-b(‚ØŽcäÇNÓBAè7/¯/ž÷"²¯+€© Úõe¤¶1µžLµùÑ/›Ï±ù’I]¼]C­-;¬=Û‚œ¾4zhA‹‡ÑFjìÕJ7vJ•¶è´:Š ©Jk-†*“/ÕªZòˆ£¨¥ÐbŠNýû¨i7ÎJS¢gÅoÊr»ÕãµhɶÈæúèT/ÙbÉ(ÐüX»Q©—´•¶í›–}Wo{†A]ß#SÕ«¹MÉ7áß[ÖÎ\gµ!•ª?ywDÇþú68jÒÁ›ßȯ¢÷Àâ—ª†ñ-G¯¯s¼²Ý%©û'cO˜Ž‰¬UAóÿ[AáŒtŸ14Uÿ¿ rµ¤Ä!½NÂöP‡9µÙÊ/×ÎûŽy“¨=Nù]ú–ƒÞȯJËß6ãûÄ·ÅöIþù/:Qõ% -endstream -endobj -645 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 644 0 R -/Resources 4 0 R -/Annots [ 646 0 R 647 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -646 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb151-1) ->> -endobj -647 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 330.479014 101.433071 311.729014 ] -/BS << -/W 0 ->> -/Dest (cb154-1) ->> -endobj -648 0 obj -<< -/Filter /FlateDecode -/Length 4571 ->> -stream -xœí]Kܸ¾÷¯ès€åòý‚=H‚Ù Œää`׻Ƽ9äïç£Z$KT©Åî™±åͺўV%òãÇâC¢:K|¾Qø/Z%bLÁÛóÃÇÓ§“ÁIü›B,.&õk|ƒ;+iƒp*ùèγÀ’ÓÁŸOø¯»¤ÁOÿÍéßË8ê¬EJ^+wþù‡¬²F„‚tŒ*A%•aTΉ´óŒÊ+[L‰SÁ ¬¶kU€6EÇ©`†Õ:2ª3Œ ʬU fè ,§‚*¹À¨”„JKÉê`ˆ´ÖhF§œð(tÏé´>&X]€NÍéŒ>Xå§KÂ{ï"§³°Å%©8ƒ-NY+9l±&Nça‹ñF²:Ø¢c0œ.ÀÒIŒ. +²:Ø"½Tœ.áR´–Õ%èd q­ÓÒ ‘œN)á€MÃêТPI>0: [¼t‰ÕÁg¤æt¶X|8x™#ÓZunÝFÅélÑ:XV[Pùh[dð ‰ÌKè„MÚr$ £6Úȱ€ŽIØ G:9a} -© ÓŠ##a‹³žc£`‹EËeu°Å$ËqÑ°8.0¶hk8.0`[ ´p\`@·V&ÅqßZ©<Ç„ 3%ÇŒk¢·P. ”8.0à\îä¸À€tÑÜǬkœ×Ю±Ñs\`À»Æ*Éq¬¶À€…QÑp\`Á»FÉÀqï¢iŽ P«8ô+BÇ(9.°à]¥å¸À‚wu@Àê‚@4Xð®v¸®çïj'5Ǽ«­ö(ü#Yo¼ -q¼;[@° ¿‡NŽ ˜ÐpK8.pà]•Ðý1ml%TLžãÞUÕÇê’P%δ[”²PÞ'Ž x•á¸ÀwAŸ‘ãp¼@ª9.pà]eÐSs:ð®ÒIr\àÀ»ðÇ€ƒPp8.@ÿ&”tY]2ÅÈqï‚–4Çè……Œ`V€¤8. ?”ãÞE÷ž8.ðà]éŒá¸Àƒw¥…ÛÃê`‹‰šãø5СÙ2mΪLÁéÀ»R¡†9xWÊ8.ðàÝÜmr\àCèÈq ð‚5Ç>Z‘Bðø$¡ƒËÇê¼Àµã¸ H(‹ÕÁ,Çp¬E2)r\4l1Zs\4lËÀq<‘¼ovx[v|`a Ž ’ô"»É€Æ¡“’イ"Æ›Úr\€v™Ç›‘ã‚d$Æ›â1í6ŸÇ›ã‚da‹ƒ‹Âê` œ¦Ä´[pÆ›`'Nça‹N–ã‚ä}oFŽ àf`¼ žˆy¼Øéð®K`&NÞ…gê9.ÀÆ›ðl¹±½ñºÿŒWÆ<âŒìŒõ:ô­ì”÷¢· -ìœùæ1;i31²Œž5@· -¥’ì´ÿ³+N¸Ì>Ñ)*8ô™1ä›LhÞ}s´`f#Øe~ìçOß¾Eç{þñ?§¿™fÊÊGÁA˜B[øWh:¡º¼o…Û7­Ä$´“…0ðž8Ùˆš‘ -´8Oºe#Ñýåçõ…å0üõ”½©Ù'ôâ—ìAÞZa G _§¹AbñQžÑ`‰Áz[0&éÌN€üJi÷æ ÎV\æ@ë%1ý%ÃV»ó›§oÿüêoùþ5`ôæÃé¿•Òþ ¥‹ß}£òo7ÿ¯Ý|í‘….Œ¿üU9L§³ HkçpEöa™NŽWÒp9î*ã ˆ]Eb©eKÐQjêCFz›`&ÝV-§´JX¢k¡+ñJ«1áô÷Pò|C^És±=ºb+«z;Ržd‰Äû°ŒÕÈ5MFŠMÙ¯»ŽÜ±8‡C®½Ê¡Ûmþب~2ªÈ½ôH^!˜¤YyÞuùG’W×”Wù¸ënéúUs<-Oeð~¹cq‡ÜëNjá¯Þ´o—½4íi÷¼ì ¬zê¾'/HëN¦'ï¬ó3ž8Ä~iÿÊY,ù?tiõÈ÷-ÜU÷¨ }ÁΨŠwpéXœÃ!rÇ5]É–Ò£½×Öð„p-õEÏ÷ö)>Ü—çóyµæ‘ª‰ï©´H2êçh(4×}KegԽݘնsiæ0~ÇÕ83†ß3“Y˜­ùÍRÃeæ}•ï•Å¯žÿئ-O ´Ê§wÒÏÕnd ™‹í™ãm;>ã=ËodFå8>Øórôçó“7JüÀü nÒïÍ - Æ9Bï_Lâ“[g¸Ÿ‘~™­c±Ä'ÌÛuTÚí!v,Îá{ݼyW¾Ø"a–éÝT£·Œr>W‹ë|Öç^_·Ä?tZHg}FÏUDÆ9"í×»¾~ØÖruDµ8sû΢Á8‡CÜu¿rk-‡ ¡{‚æ5ºÍ­GMÝ{ÔñÖæúÑ;¹Þ…ñL^ÁÓvW<·°J²~=:C>çp¨Ü™‡<*§]… !û†ÞýEZIÇ•›üH×­w¼ Æ9w|ÄÃpÆ/¼UÜÑ‹»;öeÆ9JÝ ßXJÐßkcŸEÙjW/7sÒõòý&s÷–¤Gg«Én²ºC¬³I“Yíi÷ÃÖ&÷9¾zÿtÔ­øöÅËpÅæöñø`œÃ¡xgŽó‹î ýzùúän¢ñŽùÌÁ8‡CãÎ|æ­üø„Ÿ…¿‹Œ®v¨žÂÝnE!÷5 Ò"sÞÓþÔ™£ûP»§\F׈™›ô õ¬¬Ü[‘Œs8Ä^÷UuܯÝ/‚¼·L/\Â{îAÞýhBÜ»FüPOæ"G8çhô_Þí„\!㊿ù%Pfⶭ«Öxóúÿ"›˜_ÕµnoÝ|0Îá98¿ymwD~›A˜¿Ì3‘W}µ~…‘™á®º~‡ñ–_»ñü̪%¹vϱ·Çß±r0ÎáPóëƒá dö†é08OéÉœãèhe0Îáûë<å¢eí¥µ…Ê=˜÷¡Gh·¿sc0ÎÑvžÅ¹“ ú§G§Ñ„’«ÕHž7þ[ÆÍík:ƒq‡ÄŸðP¼ñ„úŠÄ™o;ÔçÝÉe§òì÷®v1ë9\zaŽ¼ã½Dƒq‡Ìuo%7ŸÙÙë=\û~®aú-ò²§=]þnÕ|el¼Wƒæ9!µ×tÐVEüeMç,·ž†àFk[¼¸…¾;öCÆ9ú÷Cömú{hŽågö^ÈöηW]ÝñÌö`œ£¡0Žù‰Ÿkí6_ÈõšÆÉsóY–¯ÙÝ@§¹ýÙÃÁ8‡Cç|Ç/Ê+ÏØr6üÜÕZI7ë][Vœ¿N._%g_³÷)K/ÎcÏݾËg0Îá°wÝ;œÊ-Ìó¸¹,ßµ2Ôš”gñèÞw¿»rßÅS?B'#óÕ~ánW»rSVc(S1ÞâÖêÈÄ }úe·OZÛRÇQÌ*Ðè[•6ûë;žÆŒs8TîxÏþÆëÈÜ|²E߆‚[ß«µé7ô¬Ú¿®GöktÜÖíï*xÛZã"Ó»!ã ‘igš>Û¯ÞÏhÅ]tMÐÊÅèxkwöní_©avôN=·®ÕpÏ_¬ø¸[#Üzï]ïoxâ}»Ow<Ë=çpø¼îC®êbcGíþSórwƨ¦ñÀ×÷bî­XâpÉì Q¬Q{vç/6˜“z‹ƒoöŒs8dî¬{¿@_~䙯Âæ1{ÇÛ(ã³×ýÏUë¿ge‡%ÙÇ0Ó•“§Øwëè‹Ï4&}=Ÿ*й|>Qt*’ã6*wƒì o}8G9ââœ?=´üŒ)7BäƒqM<Ãq´)ä³d2”^ýáOü+Òô‚ü.i/——äë‡Ë«lóu.Ž©ŠU©æËKõm,Ô6;—SQç"»T‰œ—XòßËkq3.ƒÚ,×9¬¿|§´æßS:±PÏœ_®þ?¿¶,\ô‹ôÝœ7±gJÑ{(ðM«ªütòÓ¹Hj:Oz Uu><÷ì­0¹nÒ9ŸÅ¥±ªTš -[7bNGª¡æSˆe>˜0Ju9¢˜\Ìá?M•þñ¤’¹ü31\N •“8êrÑħ|b¬eÄÖ -=(ÎÂù§¬iYUVÙø5ˆÔ‡4NÈùgª‰W~jWî&Çw%Z>ÉÇÊÈLgÙ\2f„ˆéÕZœï^ -5‹ó©»Q˜ùÊWSœ/6CØ´4lK°EÜ+#‰Ð­…’d×ÄÚ{T>¥­oÂj:-:Eª¿„ÕTç5áà ٚ^úÈJóYŸ%©RºZV™!F¥*u%d­[ë‰=œá7²$mQ™UOj-Ÿ˜vÆ$[Ã¥Ž=œØpsj®ÖT¨¡b‹šJÉ7SÔM¬ÏȦ2š¥!#NÚˆ4åçRìs ”º–‘àF¯„'t1v%~<¹À„v+¤+УBCÒ­â(^š4Oê]~¡iY>o*ÿ¡ -W¥™y¦ÊVHå3ãÚUË$·Í*^•ûÊ:–+eC KŠK…P1KËKEɄϤ[C×[HI‘yHżLR•<–bUQWÅ™ÂJùT®¤BYŒ@vMì*”âf§«M`/™yEàîjºMì+w>Rq¨5®ò€ÕÞš.Á|>L›íÅú±dÒÄZÖÞ®eF„Í0B7}œÄ¥µVvšÞö>ÿlŒëqN?¦nN§~x£òy¶ÔóRçÿ"Ì+|ÿu²çïËj—~\ËʺrëBL{²|úyçÓЋ°6"Ô–Ð~«Øº‚†ÚkMøÚ¶è²{ ©*ƒªª¥B×ʈu…®V†”Œ»"|èJ±:2O­›Öa_.M¹9dcòÑ{Å*\ÐÛÁel:Óºä3IW9hëÒ¡µšDÄM—$ÃÔ,[˜™I3©EŸí·´*L΋\7"_š¦³­›¥IÙt$\`C/hÂVÑLÓÂÆ軛דë}6Ù—¥•Ú«ô‘Jëei%<^j¦K¿¼¬}Ž9ç‹Òx[ Õ´ i{¦:9­i4×Â..(F‰4¶hsG3µE¶È´t•. “EVS Å&?G+Å2ÝqdK)TŸ¢I¢%íh­ÔBt´Vü*lN·ôxÕ[2ÕrΡÝl!«/™§±«Ô¶¦ÈÖyÛeÞ¥·=“Û°Íß²ž%)ó'4'T‡°ÆÈ €· ¡Hû -’Ôã|Z+-Iö–n+fz¨Ñœ\ÍaæéËi’¨^†> -endobj -650 0 obj -<< -/Filter /FlateDecode -/Length 3465 ->> -stream -xœíœM¹†ïú:/`.É*~’ F6ˆ#{rp{Á8€7‡üý¼Õj~t«$µflGãõ3RW‘ìj²øT‘šn··x½pø“Ù™œKŠ¼?|Ø}ÜY“‚ÅÏTbq0©_ã7…½³œLp%æ°ÿ J&•àSÜ?ìðgu8ØýøÝîßË:nïM)Ñ»°ÿ娘LÊ)Ù ¨ -TÖ‘¢ -Á¤äCTTÑ\b)š -f„ÄžOU fpÉASÁ ö>+ª 3ˆ“£SU>9ÖT0Õ•³°ÃykU ±Ìä &¢Ó£¦óÎÄ\|Qu :—¼¦#21± NÓc YÓ1l Å:M`KpÌVÓÁ¦œ4]„-ɪ:Øâs"M—`‹GïE—a ¼+«:Øb£uš® %3«ºÍ)Ÿê¼ &d"«éœ3¾Iª3 -ƒ“¢ó°%ÚPTl d½¦#ØÂxi:p"PÊY™­^f7Yrš.Àï«:Ø‚Á×à#l±)jÀ 2Â%>%ÃųŸÉpæ¬QÀçb8%Ò0àK0KÒ8@ÖAç²°%pÔH@¶0f®ªƒ-TXcyØ‚ ˆ`‹gÒX@ -Ã[4p˶8Þ²uQc¸0Ój, —rdä•ŠÆs ìÔX@€.¦{ÒX@ .…è5°Kœ£Æw‰ÕXÀà.jª:ØÖX:r™40¸KÎ& îbyU?Œ à+Æçl50¸ë³e îú„ ê’Á$ îú€KÐ"7¸ëƒõ Üõì£Æ8Š¬š €»@HÐÓØÀj,@¼‡Î“Æ4f<ÒÜuáO™Ó •q¹DÜuçêŠq =®Ì[ô²q1ÜE "Ü>³Æ0Þ ¢zÜu„H­éÀ]ç‹ÕXÀ]ä AcÜÁ8$ ߌ³p]UWŒ-9k,à.°ä5 -›AU— Éi,ŒEª± ‚»ïEcAwm ÒXÁ]ËH{Tl¡ì5 ¯ÓV™ÓHVõ …¦w­Ãk:p×ÚT4DpW¦Ƃ˜Dà³Æh{1³))E±Xèò©ºhp4$ A²TláÄ X›B%k,H¶÷ ’‡-H4 #0Å!ûV—°¶ SÔX’™\˜5¤±C¯jºèMNÀµªË餱 %6ÅNcÖR&cÒk,à¤K `tŽ5¤[0U4 ™ì‘™k:[dk,@¦ s ²‡-È5dŸ±ÊŒVc"4Ö’èjMÇVÖ™YcAæh$UÔX€i‚•&H©ê` 0¡±ã#‹í ±‹,6±ÌQu°…k,ÈÙËr3k,ȶxë5À7e½™4 [ñi,(6I“5ãÐY«± ¸Œõ¦g˜—²ÞÌ -Y¬7±ÄSæm¡(ëͤ± 0l HQTlAÒT”y &a½ :iº[|a%FYofH3°ÞDF¯n dYo&u»Ü dÒtà.2Ó¨±Xo"³ÕÖöà ù™®Ì²âÌêŽzb«ºe`Á^D«¤îXÀWÖHê¦ÌÄÊ2Gu×aJgÕm þ"ã&wŸÆ-ªwHˆ™9ÉEL FvßÖ½)vÜûå§Ý÷o|÷?ýg÷þ»i§¬¾„©4#¿Â4ðÃ};\>E$ÆS`H‰…0ð1uĈTº‘Xœ7ÝÄH„?ùÙŸ~XXÃ_O§§vú‚(>ž‘aÞ°8rÆòumñU>¡Á‹#DÛˆ2T¼Ð .‡¼Òò¢ðoßìਲ਼Ç=Ðv8˜x‰Å0û°óa÷ýŸ^ýõÏ?¼†½y¿ûÛo¬åøò…“÷÷óûa~ÏÇ÷÷éø{ùªóûX¦ í¼_–ÇšÖÊî_2¼lCä'v ºZ®Õsƒ=ÿ8µiQç]×»0ëÎ]K9ÓVZ^S­?Ù}S?¾üûþÍ«ÝïÞ,mÊ>SƒÄ¸.úæ¶*OñÍÏá’XÊnqÉy˜Ý0Ìkw³ÔÝnážuˆòr¨Ú®‡×îP‡îíìb‡ÁÝÞÎçy×Û®.ÚÜË.JË6or9åz®NÏÁÍÃá©ÓQwSÙ¥¯N'ï˜OñšŸn¬soŽŠ•ÒEOýUÀêÌŒ\yÚÿg&žñPêXÄmŠØ×¢üÆ:wç¡|™¥8µf:âßÒ‚'Ì´3ÞÙÊÍq~c»óÎÏ”|>ÓÌà“θ/é‡ôrs¤ßVçî<õJNú,ùóõGz?ä•Î›b³—¢‹º±Î½y¨¿’‹>’;½MãÛ·H¿9Òû!§”¯Wrž¾*¸ìÛêÜw^ÎC½îý -#ýÍ3î Dzn_Óo¬swžz%'}–üùD3ìž#}¾}M¿±ÎÝyèÆ\4÷ž¬£Ø9s¢Ð<ºS¾çgÙ<ߢúmQìíë÷uîÍéJÎù5®ß?ÇìúœèöµúÆ:wç•Wö<Ÿ%k>ÑlÚÁ›—º•'¾¼Zû^k= k;u¶\ùŠ“ñýûÆ:w祗óÌæ)«<áXìLàkªå·Œx=ÇÚ†æyëÕK8µÅþ¿ï·¯hÖF«»±jùí¸ä1¯Ú;ë¡=gÜî¡Ûê܇޸ç90mŒ„'Ü:牗Ö˸ôœsÌu–Ãko=Õùßµo¬soÞÈÛrÍOÁ±)Ÿâkï®ÿéÑê¬òÙÉ{qÑŽf/ÊÇ÷³í¬¯IËWùëccÄ ƒýÒös¬dº•ëÜw^É9׹دr½|c[OZžñÈñ{ò;îëÜG^É/W#ÑzõBvÿ(†i,|D>8þ×\Ó­9|f%´ffãâ8»Nò¿ÓÕÒ盵g¼õ{šëÜ·^Î5Ÿ-ÓÎyó†µøzk1¿è Ò½3ô<Ò1F»P -öÚÿÊo«soÞ6æžg²wm‡¦Ò™\LXr’¿æ¡½:&¯ç»/R€ù!ÈÝo9¸<ÜÌÑâj‘+ý~zëG½b/¯uÇƹGãMÊcW(ïÑ‘\’Ü©$ùê÷üÃ_†Ž$LOiá[ò‹Ï¿˜ÇââS‘ÍÓ2— Ò) G$„ÉÝ¥óŽSMÊÉwiKÊóû#ÐEî¥l<þNmÍŸ§vr¼ù|Ó6˜ ¢ òQ¿h?Ìçì™ÚÃ5„z /×Cùq§»îÜtçµÙ8/fÙG6$cSör§_¶Ä®›:ÛíÏÔœh󱕚Ÿqcå¶w¹åñøœá`.ÿqô;WØHÿË´8>ÂNâìëA?ìäy$¬ˆñh£X„óGÛÚ1üJv]’)ë’êýB®´Æ)Ô>Ô«‘ú¡V“ûÄØÖB4Ý)u<±"DÍèNÅrõÖ¸Y,ÏtɆæ£ØL ±Ú aoDz½Á^ñªØ‘„áTh‡Óu±çj“{€[×Å.l¦]ç†á¯e}†ó’ð°Ãii-}P¥ò$‰ÚTí]o›Œ£J“†Z² ·qRD‡"|akÓŒÁlúaÔä~Ü£0WGâV®ÔrŠè°SËÍ­…6R©•Ê½j©=ßMé®N¹u¼"›úh–&yHÕ0G,ÕKq”¨cmóà7þDxØ!Äð‰øa’R:tVØP]oÒÐngÛý¥KS®]…©Yer7£ü=¢"4©gúู”Ü‘ÜúIdn6ñ²“FylÔa­§^vè.—RóÙ±¿°~RÊ t[éÖc éÐeƒ<•jž@ªÁc)vÍëšXVû§±rÚjN×Å¡¹òÃBÜí mªÂÙëÉ¢Ü=´v»86v>ŒâÔFÜÉíåÍÞÖîàóUx˜¯Åâú¹ž¤‹½mÑ®ŸlvÃ….î}˜Äu¶6:Mÿ[<ì“L‹¸‡ÝÏ»)Ìy -¾àC$'OK3/·ÿ/ʼÂï¿v¼ÿ¡Þž{ŒãÞ6ZøÆÖ…xŒdòl­¼Q,ÏÚªÂ6m¡çû]ìr/ª7´8îýÀkîÕm{-éA]ÓŽÂÐûpûæºÞÑÐ3á‚ð°êÅ–ÈR½8œ†äÆîjÆËÁaî:ê!y?4‰#)ÚC:´ì‡Š8ÅápÂÒ-[˜)9ž¤u½ØÏãPœk8Ò9¤®ãfǦ«lºŠ¡R`ƆÙ'- {£=½ž<î³É±Î‘6´7éÃ(m&Ò<]JÓa\¶˜C{9¨“·÷Pk‹†¹G-ÉéS£§¼8}tæ^m.Æ“r•-NZEÃd¶ÊZ‹©Úçjµ[¦+Îj/¥–StéÏcO‡qTZ'†qTâIYi·F¼–-QK€ÂèΩ_í%[.™Yk7)÷’Te§çæå¹k´Ý—ÁÝØ3SÛ«…“’âßcæαÎj'B+U_ÓîˆËýÔìyšžQjsŒ`Ëc¦ça»äæ:—÷ONwIêþÉx& ‡a"›œ> -endobj -652 0 obj -<< -/Filter /FlateDecode -/Length 4664 ->> -stream -xœí]K·¾Ï¯˜³Ó|?ÇI#b@ˆAÒX²¬È9äïç+v“¬î®éé]­í‘½^fºŠbñãÇ"ûeΟü“½Q9—ýùòþôá¤U -ÿÕ‹ƒªþŸÎFû¤‚)1‡ó{¸¤R 6Åóà ÿ¬yò‡ÓwŸþ³ÌcÎV•­ çŸÞ’Ê;•rJ:ª•6NP… R²! -ªhšXŠ¤‚!yë·ª3|ÉARÁ omTf8ŸŒÛª -Ì°ÉxI3L IP ;ŒÕZÔÁí½³‚Îáô(é¬Q1[D]‚Î$+éœS1yŒ¤+*Ʋ¤ó°%m$]€-Áx¯%lñ.'Ia‹‹N‹:Øbsr’.Á ïA—a ЕElÑQIWœ -%{/ê -t:§¼ÕYTÈÎiIgŒ -À¦uQ褘…-Q‡"ê`KpÚJ:[<þ$x"¸”³0Z-n§‘t¶X›¼¨ƒ-è|‰l„-:E‰0‚ñ’Ä6%å‹õ Øì”Ï>K,`sQ>%'Ñ€-AùX’ÄN謑ˆÀiØ|”˜ÀØâ1rElqÅK\à,lA‰ œƒ-Ö;‰ ØÖ-8Э×ÅH\àÀ·^›(qáÂL-q㺽Ĕ (‰ 8×;%.p ] ÷$q뺭Ĵë|Ž8ð®óFK\àÁ»Î!§¨ƒ-°Ä`GåLvxð®3:I\àÁ»FVâôª£ÄÀŠ²9k‰ y‰ X«âJ–¸ YØ⬕¸ YØ‚AâDªDßâò¶.âúÀÃDŠ€ÉT.ÞK\BÄz ^•tѪœ@×¢.Cg’Ä)y…©ØH\€µ”Êô€à…KÀ 茗¸ Ø‚¡"q¦"•-"sIg` ‚l‰ éBgŒÄÙÂDƒd›±ÊŒZâÌÐXKÂÕ’ÎkZgf‰ ²ŠBE‰ 0L°ÒSŠ:Øš¸ýC‹í q?Xlb™#ê`‹ ^â‚œ--7³Ä9ë­ÄÀ&­7“ÄEÃD|…ɀơÓZâ‚b2Ö›ÖK\€qIëÍ,qAqëM,ñ„q[\¤õf’¸ xØ¢ˆ:Ø‚ ©ãœ„õ&ØIÒEØb‹—¸ ÄHëÍ,q ¬7Ñ‹[™Ö›IÜ.ï†f’tà]D¦Qâ`½‰ÈVZÛkoHˆÏde¦gw 4¨7`n· 4¸³U÷ 4È—ÖHâ¦ÌÄÊ2Gq×Ó*”F‹Ûü‹xÌo8aÚ}â[TpH˜3s¢F 辯9F2w%Ù´?öÓ§/^cò=ÿðßÓ»ÏêNYû#l¤ iô¶-èí÷'8ËCĵ‘‰ ΈLWbšniA®q-¬»Ux911¢<°ƒ!Ä>,åXÕå*gÕÄšsjf“^XK˜øá„ÅV¤+q=í¨Ž ã(wˆGK˜t4zT5¼#ù÷rúý—1¼ RI˜Ö(+Â]`ÏèŠ ä ®LetµÁä2o]"áÐ,»=M}Í{ˆu²Týs#o.s:WIJ&&B‰”ˆà–bZ=·´øµ. 4ÄKq9‚Ÿ–^¡iS€–!¤„2R¯Å!´ôê¸C(ŠêÒÑèQÕðŽäßËËÀýU.EêôßyûC Yîц÷ò€¼:ȸ߸ï_ßËàû ¾Hÿ¥óöÇ#3˜·˜9b‰ì—Á÷2ø^ß#Ÿ8Èr^/ïeàýÎÞcBN, -•ó‘N”{ÚÂdnÒª.ýÙe£¥ åyUù”<û+QÚZ·Á—sÿ!¯K3ªîTøE½Á`Ý«­‹¢­Îóü¶Mç“huQ¯läÄ7­;Ô÷[#ë ±M½tê4èìl=šéÞnPýáÕ =Z¯ÃwñÊû\Ï_Ñe%‰öàϯޟ¾øúOùóß¿Eo½zwúç—Zƒ'5 Ö†6ÔÖßÁâ ¬C˜Žý»ù“'¥Åÿ¨iéÛ¿nú¯þu~õõÉSº_möù8d#Ç`lÃsÖe2omQýö³eù«ÏÍ—TÛüçï&Sí”–,÷o&yµ0 ò9-ª¨Ÿª›[G ô—QW+ƒô>Íé©<;ëgYõ@då½YÚÖlîò–Ÿ<›áµ²öZµ0Ltè¹Öæ±Ù µe³%Õ+qx¤§ñË4õØÍŸ0,oÇT&÷RÇôšgö$–Oo½ª/³âÜôÈÛŒ1…I ÓÚìºW©ŽÉ‹Tžù~[Ntt2†.øÙ–£çvMm®íÞ³#ÙkvèÑ–…?.cL ¶Ñ…_Q{¯Pö„üQæÉÃ×Ï3n‡/jÞVWœý“'t׶•ù;Êí¬˜iiÞ±ÑFú2ðj쮯ËÚµ,*3ÌeÖ–8|¸þí$3³lÑæÙ_5-kwÁÜO­Í oiú®å½ñ7ã 2ÁÁþ0z¯ýάÛvúŽØ24Üé©í–}²ÆM//.°êÌðGKßëfØ༱.Ÿ]÷˜UbÕ]_¹¯f[ã‰zÜúÙL2Þ>™óöùîP}ïÆØvøº¦3ú®Ž·'æûçòå®ÿÂf¬mf³ý¶‘ ëìívëÉyUÏ4G,9®ñä_ÕXËõÀÐxåJµèb°`§“°(Ò×I’ÅgÍó¤ˆwÇ”¥KCÐ^TdéTºFìW²öëÍøVëäš~h\èöÚLW,[7…lýúû†‡läò0OÙ—5 -cÁÑÔ W+,QéBU­*œ¦©œ:üæé› [åûåZ,÷BÄAÊ ï¾º‘Ûçz%bŒ×¬º‘kõBè’ò×à¨y®%ʬùûÝŽ§¦I`þ½ªcš–!ß41´ðp®¾ó2Ý49µÐt”×~O“Â2Ïß± -m>?¥çè:Ê„ñ"öû‘Ú-â2Ÿ±çBÄ" k½TkŒ‡Âzr·.R_|Ñx9Ž®™MÆe“ÄE—ÉžU‹@Ø] n·Á#]^èzËU¥•)®‘h 2ŠIzƒDåù˜5îÏÁ¥„é.Õ¯ù´*¶>Ò.Û‘¶ÎãÔ[j¦›¦V–¶•óFÀ^dº5CÌù¨¿—e¾&ïö\+?öU{ŽãQÁÞ;–ӄ帘ÞçEAÅy®Öl±îU)¶2–îa=¤ØpKWsG¬èb¨]¬ÌswX/zëëY¦§%NLž?aú,äiñ_N[Góo?§ù~Nÿfè¬eéÜüù~õ{Uö'1«~Ù8DF ]ØÞÂOºþÖfcà Ìs'fÓý‰×XÐkï¿3ðº§ÆFֶ盭ÓÖ^âQ±,ÙåsïÕKßX¡Á2[/KŽE¿ŒÌMÓÈ÷˜vmãšéwÑh·Ñé°œë•Ûûh<–çîÐè÷ùðžïÇR{|–½ÏÐs«};mèyxfßzÔ,|öz²sÃÇyi×!Ô†Á‡Áb•â#!lµÇòÜjã>‡ú%üÊ»kµïŽšÕ_ϧVÜêÛoQÔë[Íž™ WFHEŒÙŽ -iï<»F kÃãÐ$‘ymé y,ÏÝ¡m?füÅѶ^‹üósÌæ½=<îã<·Žyž· Íí´Ãµv”+eµ6½[ú±Y#4øð0Bæ¹7„¦1å'µB`¨ZÇ~ÍÎ+QÉž\Df^‰aôYa÷Ñö]Aêâ̓yî©7âÍu～ѳŠ™j/ÜBuCß•=ŸC¼#ôv·m½´N¿ÇÏ…•³I÷ÃÑWÐ:bÆ ½ªgŠníŒÌswhÝ3Ÿoû”Í=mXO¯÷œüz#ô -JG¬yxGé`ž»Céøô±Èb<@—J|:|õkŽH…™Å“ƪ¢³¥û“wQx0Ͻ¡0߈A%¾Z!íyêê>å'\A§}z™é¯s#Û«<aó`ž»Cå/³‡ù¬|ô»™é¯ 3<þ\ÐÁ s’ïéJK'ˆ.'1Ý\Zè=•zª<²–æùaÊ€ºËÝñ‚ìÂ’èÅ[lŒh×~.Åô|‰Þ×:3ÜØðrÂã7âú„§mê0¸B‡=.t¬Ü.ÎzàeHSn®ÂÐl2zB3ý;QEèRbžúÃø)zÊú8•ÐØì⥓¸›Û i§ö.}àÒÞ0’v“¥®ÆåaŸs\½ý¿ Þá¡^–ccÏõ g ZøÅÇ(“æ‘mNx¥¾É•¶‰¢ÓdºÉz‰©ÙçlÍ-µÅYôRê1ÅþÈ=x¯t'Þ+q“–Êm3^–\€‡s­Ó#e%³ ‹> -endobj -654 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 533.554475 101.433071 514.804475 ] -/BS << -/W 0 ->> -/Dest (cb157-1) ->> -endobj -655 0 obj -<< -/Filter /FlateDecode -/Length 4595 ->> -stream -xœí]K$9¾×¯¨3Ò&~„_Ú V€XiÄa¦w„z‘ü}¾p¦éŒÌÊêéjVL«§+#ü;>;Yúªðó•Æ‘ôc -ž®Oß_~¸¨)8…9Åê"«¿ÅopW­(LN'Ýõ{Ø0…äLð×— þ.ûä/—?ÿìòÏu}5SJÞhwý×GV‘B A9A• RÚ -*禌ó‚Êë MLIRÁ ÈÐV`¥è$Ì c¢ Š0ÃRÐv«J0ÃM’ -fèä‚ Ò -vh£”¨ƒ!ŠÈA§ÝäÑé^Ò=ù˜Lu:Œ¤³vò´Ó’.MÞ{%Á—”–t¶8M¤$l!ƒ¤ó°Åz«Dl11XI`‹Aï$Aa ÐElQ^iI—ìäR$u :CÜêŒr“‹Ö*I§õä€M+ê0¢à$-^¹$ê`‹³ÊH: [?’<álˆQ­†G·UVK:[Œ $ê` œ/1€ñ°E/QFÐļ$q€ a¢dH"íD‘¢Ä&¦‰B° ˜ä&ò)H<`•†Îh‰¬‚-Ž¼ÄVÃÂÈu°Å&’¸ÀØ‚X [ Y‰ ,Ø–€‰ ,è–TÒXð-)í%.° \˜©$.°`\=I\`A¹€R’¸À‚s-¸Sâ ÒÅpX°®uÞH\`A»–¢—¸À‚w-i%qw­ENQ[`‰ ÀŽ“ÕÑJ\@à]«U¸€À»FFâxu½ÄÀÊdbTx×DEx×L¢.LƒVâï‡&H37x×8e$. ð®!ã%.P&ð£ð.(ÄÉálÁJ\€ù:c%.@a“AX"qïê„éOÓ`«IÇä%.pà]á>Q—&Ðã¸E/OÚû$qïb¢²8ð.è3J\ŽŸ0£‰ xW[ÌÔ’¼«MR8ð.â'qà0i`~›´tE]šTŠQâÞ-‰ 0 O*‚ID]˜$-qÈxRˆC%.ðà]LïIâÞUÎZ‰ õ[Tp˜3càF& Bt_×-™ÝI6ïýëo—Ÿ¿Çä{ýÛ¿/Ï?Ë;eåG#@È© ñ†Ip·Cô­Ñ|믇Ù4XvWb®6¤f–.Ûll&<þwÝ~XÙ -SÙXäÆJÈóꟃcµT îÉ¥–ÁXž%›­wç¹ir@L®õƒ–míú¨«·ë$ÁÖ»ó¼½­˜P±hFˆ¹®W6òFâ·u~«—£Äm½¼ž7c@6öl¦76ÚÖÎJìv(a˜Zf"Lˆq‚–§þÎä;³¼¡Á - xD„il2<ƒ‚±öQ´JüËwébżO_/;óÓiLŽÆ]ß}ùùï¾ùÓ~ÿ-øîùò—_(Ä_¥ñ—žç¿ÎÏy™üZGOË߸¤Õ-?>¶ô´È ¤]ú'^T´ô}gZ™”ººÞCVò»uý«:Ÿ‡r–4ÔÙWòijÄuµØBr•ôë6}ý×ë»o.¿z·Q^ýÌ ò ^B|êoàî\–OÁÝ·”Žà&¹»tÿèž -»çÁ=në&õ~Ÿ7ûô=¬F>ß‘’þ.øé®mCy¹¬q˜¥¦¯ýö$ Ë;`ÈwPñf§Á æOæy4 b%~ˆÄ—b9CJŒ:&I×DAO&ÏÑ›®uå¯~Ø"w•§#TíÝâ…²jûÆv´šôØJIŽo Â=çò<zè³ñX™&?ã´êì§Øgb>QÚ.(ŠóßÝrÆ6õõHÏö½íô)p°YÛ¾Ëm] –ušx!}ŒÎsyÇAÝ&S;‹¾P.; -`ï*k/€¸²Ôwj¶m!œã-àóvæ1"Ïåy8DÇ}£'j¯„ö¯â0‰ _f4Рyxg4rfåÅ~tIñß°TúñF­ŒVÓâ<§hÊTx‹?Oæy4´šãØð‹å´=4ŸXˆW4 #áóŽ t¶8R¼lpê:Ïåy8tžŒ=w¢wi{¦zi'c.Ùį±+¯ø¤nas[Ò”×Þ¥-õ²Ûí5‰ø˜œ61·å›_ÿö7ìÚ’ù[óÝžå¯+³+lªíXÛÚæ ¶mNëÌÒÆùz#_Ò¶Øv.cÓrßÌK½s|½”ëÔzKJÍåV—zs~ßÚÒFÐÚ¶RwΓæ:¸þnÄåž?ÓÕV÷]ͽ«ÀU†:k»³åKO±UÄV=-µù¥¿*ßñ9Ë'3—òíâ繎År”£¿Û–ã-߶ä£qÛrro”ö¹õGvĸg‡jmÉž,íco˜yD¶ñI¯Èø$”=#±•¹FVC•ñ­ŽÜÏn¾Îhz^ìssZ[ú^/¿ÖgdšÅ'vé»´Í/m+hõr;+¢žZýý5b˜ë+Ñ€ÜפÆþ(£ƒÊÜ溶øÖ'®×œee>\µy鯜¶kw¡}?•6¼…ùo.ïã‚¿s,Îe¾Ùi¿Ûï|Ç æ -îÔ\²kŸŒ¸©åù6+të’¾ÖÝacd•¾üÒ'yl²ï‡ñæ:{ãÚ­{Æ<ì'Úô“íú|ìjØ!(¬×1ž\VGëzf~[Ï2ÆÇõNÞæ›O|äïÔÁÉ<¯ºø†Áƒ«ö"´ÎaXˆÁC.ãÂD—á×M´³v+Lá*üË°4®ÁIZäÇå‹.ö¸B¹îùë¹B>oêýžU7ò;?%ßIùó´?„\îD™ùQrÞôÔÑößÞ]²Wo7fb8%4|¿Ís¢ï€Ø mp_ã9>-0^D¿Ÿ©Ýx|Äÿoè9Ç7¥u jT#Æ]êU;Tš™bDè…ÍgÃòY·ÐOåy…XåR¯Ü—ª÷êÜw·)U9×w#«ÇJ®å–´u䛵ü¼=_ŸG£'„ÆŠ´0#-Oùz@c?ÅGqŠÏ+õäbÐC…GHä£È ªøDý©ÛQ'ó<ù„í>oÝ2©lÛ­žÕí–…e9Þ±/ËëgZÒ|·¤ÿÐtÆtéìòûÝðy(û‹˜ó~q¼ÁïS¨Á!{ÈDmÜ-žËó lLj´>B`æ±÷?´ùqôTÛ@Ùz¾Ä¡cÚì¥>fe” \Å{ŸrKëðfìØÆÐòÝÓγmç‰#4Ó¸íìí¦“yö˜Ç­P7x¡¿U½sK†¤[:f«Ëó]ÚRÎ!ìn"l¸Kº­µüîÝ–¿Ù¾ƒ6Œ·ÓVý¶£Fº½¸e|µÔøЬ!È3ÂŽQ{.ÏáÖs(­‘@CïŽ(ê7ï3bÜ|4o¤Ûá/mQ4Ɖã!¶ñv¸tÓòæ¡6¿FÏئÓ7 6h’÷1èÆm¤^Á‘çò<ÚŽcÆÏŽ¶½›ˆ=—þ³ùãÞÌÝAh‹Ï#ô\žGCh¼S~Q+„UcìWì܉JÎðä**ÐËJÌ £Ï«¸O¶oçLû+âÍ“y©7âÍÑ{ïô 1S9æsˆê‚¾£»³ÝûôÆ?‡÷žÀhyöxÓÉ<‡Öã8ó-#½É3B§ŽX~¿>ÞÝAi‹5Oï(Ìóp(½ŸÞ‹¬Ïv|ò‘cÊ{ûm…]<©Í”T47ïñœÌóh(L7bÐOx¦ñÉS»û”_Td £3™û9òdž‡Cç¸ó'ÃC_Bd°ƒÆn/òìcG'ó<oì_~.>úÿL¿ÏÝ^åÙ™ûdž‡CãÇ?Á™þ̃«zÞd¦ßAæ+î•ŸÌóhÈÔê8¨üiѽ ßÿÕT_aùíòÎ¥ààAçøwÑéؽ©bñf’Ð+Ç«¶¯¤¹êá5@ëX(’_¶nãÕa)ðûÉöÎ_™ó©@½çG ØÊúùšÝP´IKšrn+wý ¡z}9²_Ž~ç3\Ë1ó|„;ÍrS×Pí´¡WÝC1ªK7ù¡5×ò#¨m(ûÝiãÊ.>¿kOç÷­*'mø…ìWO“eߤ+¿ß/*Kº¸NçΞœ 9ók쨩–7Û+~Ù-¿è0»¶¿XÒÿþýEóƒOjf†ù-Ô*‹£)Mürá·“ &júܳpù¨jY¤ësCEö‚ SSZ7©åc{¸ˆ Í+­áü®dã·Ã‘*‰l~?Ú\± DN¯·bn½šô"æ7¹ÇÉ.W¾šÂG³«°H}ÚV`ËxS¬­ê„n+T]uMl¨Ø£ùÍŸµë|VÓû®ÓûKZ:w Ÿ.¨ÖŽÒQÊï.E•Þ5ªÊlgTªRWRV‡Põ“ zº¯T)šà̪ï¼Æoᜅ±‰jºTÒ ¢§‹˜n)ÍUO…š*¶¬©ô|3¥AÝÆÚñ‚ì©{/ðWStcDÙòq-æçʪ¯Uìpc6§ ¦Úˆ_..©]ã -å -ôz¡íÊ­â¨^š4ÄÒUšEÆï0äÿgªpUÊÌ“?hªâ÷¶«V Í*^wR/÷•uHêM¡¥íºK‡P1Û÷—ŽJHϤ[S×[I».ëä!ó˜¤*y¬Åº¢®Š™ÂJÿT®ì…ªêšØU(¿¬ÄÍNW‡ªSµ2¯;¸»ZnûÊ/½8Tk~©lµ·–Ûa¾ŸòÄF£˜¡K%MlTíZe°RhâF£/Y\Fke§ü¶‰åcdÒŒûtùû%OsÆ:ÃÏ_y«ùé}䥯ÿAšoðû ]_^Ê9ÏãFU¶0•[Wâ~&ãoÔˆ'Åü EX‡M'4ÔÑ~ëØÒº‚†:Óñ5µìª{I©+ƒêªí…®õa'6ºFÛ®gÜðièÅÈ|ªoÚ„=_ÚÒ8~’Œ_çZ¬ÂEß\Ʀ³mJ¾vE⊓¶)Z2]F\ô…ⲫ05ËVf2 ûJj׳ýÔ»Âr]ÝEFð¥m:jÓl_t‘åVtIÛþ¢/˜t_iZÙ#ýêáõÉ~_Löe°´R{•¾ôÒÚ0–V“¥6_úõesl~tª ÞÖCµ,Û=[ƒœ64ZhA«‹£4¶lK×WJE¶ª´L•. SEVK Å&¿d+Ý’[Å^ -5¦hÒ¿÷=íz¯ÔNt½Wü&-—[f¼-Ù¹ΡµNµ”5–Œ‚Ì÷¹«”ZJ[dÛºi]w™m¯]3¨â[dªZ6·Iù*þ{ÍØ™ó ;Rªò“wGtl¯±ÆDM:Xcó7“©è=8„øË%Ån»äî<Çû'Û]’²Ò×wLd­ -š¿cN6áŠt[ÑU¾NΖ”ؤû:@Øj4§6»CùK†òþeÞ$ª——Šüb–ƒÞÈ_…”–߷·Küjû$ÿüLzŸO -endstream -endobj -656 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 655 0 R -/Resources 4 0 R -/Annots [ 657 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -657 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 461.373057 101.433071 442.623057 ] -/BS << -/W 0 ->> -/Dest (cb160-1) ->> -endobj -658 0 obj -<< -/Filter /FlateDecode -/Length 4656 ->> -stream -xœå]ËŽ%·‘Ý߯¸k¢ù¾C ÃÂx0ã…áEw©[ d@ö¿ïÌä#3#U]ÒÜu¡úÞŒà#<< 2™Yæ®ñó•Á‰ŒJ)Ç@÷§o?Ý´Š^ã_I±¸(êoñýÝhŠÊ›’¿ÿÈUÌÞÆp¾á¿Õå˜üùö—ßÜþ¾ÌcîVå¬ñ÷|d9SŒÚ ª •6NPy¯b´>ª`š˜³¤‚>’¥­* ÊÉK*˜AÖ&A•`†£hÜV•a††$Ì0ÙGAe4ì0VkQC4‘³‚Îxàô é¬Q!e›E]„ÎD+éœS!’ñFÒeBðIÒlñYIça‹7DZÒÁr)Jº[\pZÔÁ›¢“t¶Xx' º[€®$ê`‹ÚHºì”ωHÔeètŠi«³Ú+ŸœÓ’ÎåM'ê0¢ÐI! -: [‚öYÔÁï´•t¶~$x»˜’0Z-n§‘t¶XIÔÁt¾Ä6ÀƒDAŠyI⣢lI"›œ¢DIb›²¢D6{E!G‰œ6ÐY#Ó°ÅS˜ÀØB¹¢¶¸L8 [@âç`‹%'qÛÐ"qÝ’ÎFâ¾%m‚Ä„ 3µÄŒëR ‰ (PÊ8p®wJ\à@ºîQâÖu>X‰ h×Q -8ð®#£%. ð®sÈ)ê` ,qØQ9“œÄÞuFG‰ ¼‹ad%.@¯*à0H\¬(›’–¸€À»6i’¸€À»6b -uQa :‰ ¼k=š ÍÜà]뵕¸€À»–l¸@Qà-Fà]Pˆ—ÃØ‚•¸ó=tÖI\€Â”EX"qéOÓ`+eRxð®Iè>Q—•‰ð¸0náeeBÈxð.&*'qï‚>“Äàx…ÕJ\àÁ»Æa¦–tà]c³–¸Àƒw7x‰ eH\€ùM 芺¬tNIâÞ-Y‰ 0 +À$¢.*ÉH\2Vq¨Ä¼‹é=K\À»Ú;'qAïjBØ#ê`‹KVâÄ5ÐaØ -cÁªÒL!éÀ»Ú ‡%xWë˜%.à]ž6%.‘6I\¢`+qAH¤rŒAâ‚5tùD]P¸öD e‰:ØB‘$.@`­²ËIâ‚ha‹³Vâ‚ha B‰ ¨l}‹Ëظˆë‚-ˆ%.“©”‰$.ˆ>`=¯Jº`UŠ kQ— 3Qâ‚Ia*6`-¥½Ä xÅá’Ä0:CÄ [0T$.ÀT¤’Ed.é lA-q"]茑¸ YØ‚hPâ‚dV™AK\€kI¸ZÒ‘æuf’¸ QP*J\€a‚•&˜RÔÁЄÄè^l{‰ °øÁbËQ[œ'‰ R²¼ÜL¤[¬¶›¼ÞŒd [ñI\uP&K\‡Nk‰ ²IXoZ’¸ã’×›Iâ‚ì4Ö›Xâ ã6»ÀëÍ(qA&Ø⢈:Ø‚ ) 㜄õ&ØIÒØb3I\Càõf’¸aÖ›ˆèÅ-ÄëÍ(n€w}3I:ð."Ó q.°ÞDd+­í5ˆ×GÄg²2ñŠ3‰;Ôë1·Š[Ü‹Ù*Š{äËk$qÓfbe™‚¸k€iJ£Åm þEš‰ÿî2­˜öéÛå`>b:ƒÉÑúû»o¿ýÏoþç¿þü-øîÓí¯¿ÓÚ‡¯¿2ø¤§ùó=d¦_—ïáCÓG\§9Ÿ?ëõ§åµÓ'çkuTÝœ×çýzZÚ§Y6¤]×e¡ç]ïR-ë`yKÿ´´½ä©í ÁŽj͋<ƒÆϺ£¶nÊúúo÷wßÜþðn ¸²RzF¯eùŒþÐÌùšqÙÝÕµ/v³ºtîfc¶0Ó®§_ä©õ§¥Í>¡¾ «œY^í¬6ÔaÐl˜íçOogÙ\ÇÂöµ†ay>¤dxò¤lÄ ?@Íú|^ÌóhÅjþ¡_áª5â›ýê’FUXÙ1§»ÑµƒJ7P £€2¯1ŽQy-Ïá’Žyóÿš{~åÓ·é#a)èxµ›Îx-ÏÃ!ñ$¸ü¹¹æâ ÞPjVH|? zЊUË©£e r½œfÃc¡£žį}TÓ/Û´ƒ¸þñm5"C:œ!îZž‡CÜqÌ(õxíu5ä­—.~ÛSú;ü~º¾œYÇ_-ïE”Ôô/BàÀÇ´*oÁõÕÆÓF®ßŒÌ Ñö8o¦[ Óx6 _ÌóhH´Ç±áábù/}¹³â6¤5ZwfN;Äc|ì „Sô\Ëópè9ŽáÞ’ÇêLù9h\Ϭޭò¬¢ý‚^4Ú¸EiúÜ-gÝ&)^\ů¯›A¶KÛw¹mˆÑ²W@ñšct^Ëópè<Žë6±ØÿÛÿ8†}“]¨KëåD¾b§ðbž‡CäÉ^áª'šW¢ûWq˜Ä…¯ˆ Vëì ffl֜Ùxq]Rü÷‹­Ëe´ºçyMªPá^Ìóhhu'û†_*§í¡ùÂZ|½¿·‰¿èÚAg# ¡·³‹^Ÿ¡óZž‡CçÅØs'z—vhZ/íÄbÌ%›ø5 åÕ>iG$¸-Y•µwmK»N¸Lü†±©´å›ÿøÓÿ{hKáoÞ— ýSƒÏmkÃÒÎ>_°]SZyTÖöM×ùœvík¤OuLgŸ÷4\G¹’+¿×ˆ¢ÛJq.wHSÊæò>Íu?_q,™Ñ±­m÷z¹k¦·u²¸ÞÚÖ⫱݃ÿjûÚµq1É5ÑÒPçÏëÞ'Õúiá&ÌÍ ‹vy>_æøù¢¹]nöŸg_¦ÙWìßï¶åLJïøm9¥ n¶'”6Ù‘Òžzèϧ¡}ìS;;Á6~Ð'h²! eO˜ïe.1Ü}mC¯£õ‡|R~gÜrZW}ofŒ½_êÙ%o­+ÌþI3¾Ÿjü1_ í,Xªi> ã¬Å%S}5æ}íõÚ¥,.³Î ~hKè>ñ£þãb¸Ð³@ÅR¿qµ¸yŲ8o. ,7m.ÌßiNóÝœþC×Y;¤sóïw«ï«²¿ˆ¹ðwÇ·+bŽ-h¼zpôbžA`?mÌ ﬽ÿ¡Ï›ëžêÛ-Ûž¯ñé:mé¥1–e”¬¶ëjï}Î ºW=`_ÖΫm\Ï!Ghä‡A+·]>f-ÏáÑóáÎQÛÖ ã÷L$Ý ²[]é±0¤­å|"Äá–Ȇ»¤›tóïÞ!ƒÓö´A:@?Ú·5Òa…Ó¨è -j©ó¡·X[P`„<r)Ïá6s(-‘@+ï®Q4ÞŠ(ˆñÓíÜr[À­>i‹¢uœ¸>’·¾¹/Ý‚==¢·>º½FÌÕÛ)4ÉûtrS,ÅWpäµ<‡¶ã˜ñGÛ•x~†ÙüqoMï ´Ç‡×z-Ï£!4ŸÄ”_Ô -a@Õ+¯;ãÉET`敘FŸVqŸmŸŒÔüŠxóbž‡CêI¼ùFϳ²ßÙMzñѼ—>ŽrÄÏ_Ä‘È´ö˜ñòa­‹y­Çqæ¯ïËG¡;(í±æ奋y¥'ñéK‘õ«yHó-Gä -‡xÒX•u²üF¥c^Ëóh(4ú$}ä÷,¼òõ%_NhÐàùíüV©èÑ“Þó[ý’7ixIUÃäi’Ö[bÛ—îÜÍêEGË»f(’_'ïÒÝi9òØöî™ÙÓ^3)âcjPróqNvU;ê™ç4õ^Ûp¤«SšsÕ£Aå¾Û|”§«Ê“ܶ£|ºß9z8ö¨Û±%[ŽEêé(ÏpÔ©•pĬµ¡®Qò¦+º…ò6ASÞ(«CRÆò+çï”ã¾Éw~ƒaÒŽLí:Sœmî;9Ë‹új©æw÷k~/¿Ê±tíx1§ÿ©tú7ÃGoõÄÓ{¶u'[/ºøùÆïY'ALÔŸžˆY8Õ­,2í,k•=h£Êë”Î+=í^ù¦áôÕúÚÎïk6~ÿéšÈ•7ÀM Bä f+æÖkef1¿«>)7_…f -³iÂ^ i{=ã©Ø8=ýV¨‡êºØRµÇð»M›ëB6ÓG×™¡ûkZ‡î<>ÝP­[KŸE)¿!»U½ku“¹Á¨Ü¤¾¦lB­ŸÑÓM~¥kÑ„Îlú¡×ø=£“0U QK—k:AôtÓÍ¥ùÖS±¥J=k®žï¦t¨»Ô/Èž†Cá‘ÿøÆ0F´«_—b>sÜúZ§7v#|ºaŠ¡øùæ£Úw®Ð¾Boº¡Ü&Nºã¥Kcª®ÂЬ2~K#ÿ?Q…oRfžòÅPƒ¿iµ_õJxl6ñÒI£<4Ö!ɆbO;¸ËÄØ0;úË$-¤gÒm©›ÇÒÁeƒ<æj“T#¥Ø4Ô51SXõOãÊQ¨«¨®‹}ƒòóBÜíôm¨zÝ* f€»oåvqhÜù<ŠcëqïÍmö¶rÌWáS™Øh-fè§ZI[Ýf»^٠솺¸Óès×ÑÚØ©¼çbþÚ™4ã>Ý~¸•iÎ:où,mp†ß?F^æþ/¤ù¿ÿ{£ûŸëkG§yÜêƶqëB<Îdü7CÒE1ÿ ‘*lÃfZh¿‹Mêi}EC›Ç­øšzv݉½¦4AMÓŽBß}8ˆmƒ®5nðŒ?>­¼Ø™Ïí›>aO—®6ŽOó k«U¸›ƒËÔu®OÉ÷¡H\qÒ>¥CKvȈ‹±P\ænÙÂLäXIs=ÛOcW8®k¸Â¾t]G}š‹®²ÒŠ! B`7^Œ“+Í {£_=¼>»ßg“C,mÔޤϣ´5Œ¥ðd©+—ayÙæWŽÁÖÁÛ=ÔÊrÃØs-ÈéC£‡´¸1:HSÏ6'ðc¥Te‹JëDÑè2]e­ÄXm -s¶ê–Òâ$z)¶˜¢K=íÇ^iNôc¯„MZ.·Îx-Zr-ò#œcoî)[,™Ys7)õ”®Ê¶uÓ²î:ÛÞ‡fP7$ôÈT÷l~“òUü÷š±3åYíDH©êOÙ1©¿¨5™è¬+{M§À!Ä>#R¶K^œçxÿd»KR÷OƚЊœÓÑð_Ñ“Mx†"Ÿ+†¢ê_Ø“³e-6ée¶‡:ÍéÍîPù3Je³lµËÈKEþ«iŽƒÞÄì%j#?ñË›Ùq±}R~þ ½¦Í -endstream -endobj -659 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 658 0 R -/Resources 4 0 R -/Annots [ 660 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -660 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 386.373057 101.433071 367.623057 ] -/BS << -/W 0 ->> -/Dest (cb163-1) ->> -endobj -661 0 obj -<< -/Filter /FlateDecode -/Length 4595 ->> -stream -xœå]ËŽ$·±Ý×WÔÚ€h>‚/ÀЀmX°+`p½¼˜iil-²÷÷ï &_™••ÝÓ#•äiôteÁàáa™É2WŸ/ þKdTJ9º>}ùá¢UôÿJŠÕEQßè¯FSTÞäüõ{¸¨bö6†ëóÿm.çäÏ—¿þêò¯usµ*ç`¿þû;V‘S1Ũ½ ÊPiã•÷*F냠 -F¡‰9K*˜á#YÚ«"Ì œ¼¤‚dmT f8ŠÆíUfØhHRÁ “}TFÃcµu0D9+èŒWN’ÎR¶YÔEèL´’Î9"o$]V!Ÿ$ÁŸµ‘t¶xC¤%l!—¢¤ °Å§El±):Ia‹…w² K°èJ¢¶è ¤ËNùœˆD]†N§˜ö:«½òÉ9-éŒQØt¢# -¢ ³°%hŸElñN[Iç` áGÒ'¼‹) £ÕòèvÚIça‹µ‘DlAçK `lÑ1H€¤˜—$°1*Ê–$°É)J”$°)+ŠÑI4`³Wr”xÀi58 [<‰ œ-„‘+ê`‹Ë$q³° $.p¶Xr8°--8Ð-él$.pà[Ò&H\à@¸0SK\àÀ¸.’¸Àr¥,qç:p§Ä¤‹á%.p`]烕¸Àv¥ qï:2Zâï:‡œ¢¶À€•3ÉI\@à]gt”¸€À»FVâôªƒÄÀŠ²)i‰ ¼k“&‰ ¼k#¦QÆ “¸€À»Ö£ ÒÌ Þµ^[‰ ¼kɉ þÑb4Þ…x9€- X‰ 0ßCgÄ(LY„%xð®É˜þ„1 ¶R&å qïš„îuY™ ã^V&„,qïb¢rxð.è3I\ŽW˜Q­Ä¼kfjIÞ56k‰ Ø‚HQâ0™J™Hâ‚èÖc𪤠V¥ºu :%.ˆ‘¦b#qÖR*aÐK\‚W.I\3 3$qAÌ°CEâLE*YDæ’ÎÀ٠҅Ή ’…-ˆ%.H6a•´Ä˜¡±–„«%i^g&‰ Å¡¢Ä&Xi‚)ElMH\€þáŶ—¸‹,6±Ìu°Åy’¸ %ËËÍ$qAJ°Åj+q°ÉëÍ(qAÖ°ŸÄYÅa²Ä qè´–¸ ›„õ¦%‰ 0.y½™$.ÈNc½‰%ž0n³ ¼ÞŒd‚-!Š¨ƒ-š²0nÁIXo‚$]€-6“Ä9^o&‰ f`½‰ˆ^ÜH¼ÞŒâvx×g0“¤ï"2 àëMD¶ÒÚ^ƒx}D|&+¯8“¸c A½s«¸e Á½˜­¢¸g A¾¼F7 `&V–)ˆ»˜V¡4ZÜ6Ðà_Äc´ã„e÷iÞ¢ -€CÄœ™"72cX¢û¾æÉÜdËþØ¿ÿ~ùõ{L¾×¿ÿçòñWe§¬ý%5!¾Â0°Ýí}4ß,&l1 –½(1Wó0Ë€ë6›… ÿ]÷V¶ÂT6¹± -¼úìÔŠÁ=¥tÂ2+˳ä°õÅyrÀýƒ‘m>šëN’l}qž7·K4,R=BÌu½¢‘÷¿mçOõbµµ¯×`bÆÀ cOfzc£])™;;”8hAxay¿NAËSÿdò ³¼¡Á xD„i\¶<ƒ‚±öÑ´JüÛwiµb٧ùˆé &GØüîû˯ÿôÕÿþåÏ_ï>^¾ùÖÞù…Á_¢å¯eQ¨²øMU^ÓñîrÏóÝÐ_u-ï&ŸÏ7ÊŠ5ýǪ C×í¨:zÚ”mF~|ék[ÌÔ–nGM¯ÝÈ·ÊÛìHk{ºÛ6è©Í;÷ÕO“OßsÚ/ÿv}÷ÕåwïÖ++£`XÈ"Þ 1y.˧`òs@1ç#(2œšÛɯÝÝ!ÖÝ!uÝ]˜L°õOU–§|›º¬®PÍÓ0™ ¼«W*7¼d(ò]¢ -,);ëd98<ÄâÉ<F¬ÔÑøä³AÚOC:¯Y‡h“깕~öÓÝ‘v™Ž^ƒÛió‘œr|ç$ÝCâ¹<‡Ä;ñåçæš“3xG©Ù ñý„êML+rT+§–9Îõrš…zš¿õQK¿nÓ ÄM[‹!£Ÿ épqçò<âîìG -=Þz`ÛCyÛÕ‹ß÷”þ¿ϯh¶ñWÏ{%-ý‹8ñ1mÊ[q}³qâ´™ëw#óHt#Îã3-†i¼7 ŸÌóhHtwö"ÖË'xéç;+îc@Ú¢õÆÌé¦xŒ_a€á.zÎåy8ôÇpoÉcm¦ü4ngVï6y6Ñ~A/m\EQZþÞ,gÛ&)^Üį¯›A¶kÛorÛ£½çw2Ïáó8®ÛÅb¿Øÿ8†ýñvÁo ò{…'ó<"ïìnz¢{õ º‡I\øŠx° a³ÎÞñð•Ð–3;/ΣKŠÿ~´u¹ŒVšî9kR… -ïñçÉ<†Vº³oøså´[h>±ßîï­Fâ:‚n sÄ‘üd›Î.z}çò<:OÆž7¢wi‡¦÷ÒXŒ¹d¿¦©¼Ö'ýq[nKVeíÝÚÒ/§'S)¿Òkl*mùê÷üÃÿLm)üÍû’¡þŒLØÓÛ°¶sÌl×’ÖG•­}ËõN^Óîç„¥¬ÉìÛE¶ ·^ ±ÇçW›ž×úÓÌí[^ç~(þ-Þ=ãNofw~ÓZìõz¯L×Ú&+¹õźÐâ—ZZ§)×n´°{bjñÜZΣÁ‘–&{â”O¯Ó×ûT=jÓê͞ß^püözm³«½ï¹gÒâÅÒßîË Ž_íàׇ÷åèÚ®¥Í¥ÝGv¤tË=Ú²òûÛ.#Q°_#šlÈBÙË(e®Q=|mè£øÙ/×YÂ9­k¾7Ýï×úÒ§¶ö‰«>qµm¡¶m)B; fVè®#¯G*K}- -‘}ôÖ¥,.³Í©~jK>ñ³¾Žº6¯Ú\ýUÒNí.£yöSksÃ[\þ–ò¾«øóÓH?Ùmž»Ñ~»m¿?è;fNßp§—z´[÷É7½¼0ÆbÇþhé{Ý6fÞØ–?˜ö%,»ø³÷ͱhç£Obá øÉŸIðQ\ì}{`GñådÇ­ñÂ2o§úüØúç– ^Í¡/ÃÖ—oÓŸÞ'ö‹ŽµÝ˼³æÍƽۘ2ò;l®¼8Á¯«ŸÚ1:™çU¯¯¼a0éÇ« -¤’÷ +1˜,4¶]¨šF‹SðµtÂÍ -s@xͯ}o*üf³UÒƒÕ\åÇåZ\D¡\ÿñË;¹Á|VB·¬º“ß•=úNÊ_®MîO”YŽ!"ܧŽ¶ƒoÝ8}õös™ð„o[*¼ßç9á; V¡ žÒkzŽOzˆ/b¿Ÿ©Ý|ÄÿoØs>eMåÜÁ¸ÏSOÖµ]PÉ—\æÞ«6çò<ȼsi²þˆKõû_þ¦£ž6X{}¹8á[Ø Øù¤µáüXH!…ŽóX_¹l 5)c é±üžÇ©œJ³©ðpdÛ0ÍgÑœº9z2Ï£>kà`Ü»×¹2«›iÙ*›C—ò&QÿL5Í·5ý‡¡³vJçê﷛ϛ²3nËefÞO¨¬zöÁß“yÃlgŽXlëýcvÞöÔØ8Û÷|c³mÚÒKsÄÌ(Ùl·¶Þû”¬¯z±Ã¾¬gÛ¸eúC4Òඳ7?Oæy84úc>¼ñ¨tï…ùÁ‰s5Is½ÝëJ…)m+çÃÁ|üAà.é&ký½õÈÝö´Azb¶o;j¤‡Mo`œEm|è-V0aǨ=—çáP›Ž9”ÖH w·(šo%Äøåv|¹­ã6i¢í­©í#•Û‡3¤höî#–ÛGï·ˆ9{;l‡&y…îÜÔÌùy.Ï£¡ Þ8$ÉngÞÀú Óùã>[Ð!úu=?)zô¦÷|~]ò&MÇ1u\ÞMr†mÁ·?^æj6Gú¬×„(’Nwéj¶ù¬±[+BûݲKbêV:ß2³X뺰\³«úè\Ó´•ät{©ß2©·–ÚmŽ²ª¬[âeÛ:/rÛo+ê±ûôt{V÷Û'ö}]>éÕm—^þÁí®ÞßÚ°ëÊ.¡œ›gÊÙ©:$e,®~ ¤÷M¾òY}I;2­ëLq¶¹ÞÈYŽ¤ÿ¡§ª§Ôk>¸–-\Ž°Ÿ.júJ§1ü`€^Xb9QZq²íbˆŸ/|¢8 b¢q“üŽ˜…õ£îeñIÙÈ6ª¼M‰Õ¯®Çy^/­o­áü¾eã“ÞH·D®œu¶T,‘3˜½˜[¯•©b>•=)W¯B7…·ª»pHsÚQàÈxWlœž„~/ÔSuCl©ÙcøÏîº0„ÝôÙufêþ–ÖÆ©;„OTë¶ÒgQÊgA·¢šw­î27•»Ô·”½C¨÷“ zºÂ/t+šÐ™]?õŸ¨¹Sõt¹¥DO1]-Í÷žŠ=UYsóü0e@Ý¥îxAö4=¼ùk&¦1¢]û¸óó½¯ušpcw§ ¦Ú‰Ÿ/> -©ýà -íôf¡›Êíâ¤^†4¦æ* Í&ãóùÿ…*|—2ó”†:¤øLÑq5*á±ÙÅk'ÍòÐY‡$Š#íä.cÇìì/“´žI·§î[I'—Mò˜›yLR<ÖbÓQ×ÅLaÍ?+g¡nF º!öÊÏ+ñ°Ó÷¡êu¯,˜ î¾—;Ä¡sçó,Ž½Ç ÛííåN˜o§2±ÑVÌÐO­’!¶ºÏv£²I8 ) ñ Ñç"n£µ³S9¹¡~ƒLšqŸ.ÿ¸”iÎ:où~tp†Ï;Ÿ#/sý?¤ù -¿ÿ¼ÐõÏí€Íe·º³…íܺÏ3;F:)æoËhÂ>l&¡¥‰ö‡Ø¤‘Ö74ôyÜÚ‰¯id׃Ø[JÓÔtí,ôÇ“ØvèZã&ÏøáÓÆ‹=ùÔ¾öréZãøÎ:ÍÚ¬ÂÅÜ\¦¡scJ¾NE⊓Ž)Z²SF\Ì…ârª0ËVf2 çJºëÙ~š»Âq]ÓÅFð¥:Óì\t“•VLI»ùb.˜Ì\i^Ù#ýêáõÉý^Mm°´S{—>ÏÒÞ0–v“¥®\†õeŸs\¹•ÜïðP/ËMcÏõ g ZÐêbÆè$M#[MàçJ©ÉV•¶‰¢ÓdºÉz‰±Ùj¶æ–Òâ$z)ö˜bHÿ1{ÚϽÒèç^ »´\n›ñz´äzäg8ÇÑ:=RöX2 ²0çîR)]“íë¦uÝm¶½NÍ aH‘©Ùü.å«øï5cgɳىRµŸ²;bÒ8’5™è¬+ß2¦Sàâ/Šˆ”¦í’ç9Þ?Ùï’´ý“¹&t‡"çt4ü}q² ÏPäûŠ©¨ö]rr¶¬Å&½ÌÂöР9½Û*_Tö2Ë&Q¿Œ¼Täïsô&þZ“¨ü>?‘VÛ'åçÿÅq“² -endstream -endobj -662 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 661 0 R -/Resources 4 0 R -/Annots [ 663 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -663 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 296.373057 101.433071 277.623057 ] -/BS << -/W 0 ->> -/Dest (cb166-1) ->> -endobj -664 0 obj -<< -/Filter /FlateDecode -/Length 4352 ->> -stream -xœí]K$·‘¾×¯¨³Ñ|Dð,tX`mX°0Ø={µgl-²þûþ‚™|dVTUv«5ªï4zº2‚`ðãAvÛ-¾¾rø/“39—éüôÃéÇ“5‰-þÕ›‡ªþ߉ÏÎR2ìJÌ|þA!™Tاx~>á¿Ýã\üùô¿¿:ým[ǽ)%zÇç¿“rJ–Uʺ ¨˜MJž£¢ŠÎ`ˆ¥h*˜Á‰<]ªÌ ’YSÁ ò>+ª 3%.UføäHSÁ W8)*ga‡óÖª:b‰‚WtŽM„Ó£¦óÎÄ\|Qu :—¼¦ ÁÄDŽ¦+&ÆÈYÓláb¦cØÂŽÈj:ØB!'MaKˆÁª:Øâs -š.ÁïE—a ЕUl±Ñ:MW‚á’‰T]Îæ”/uÞ²á‚ÕtÎ6ƒªÃŠÂ$Ťèj\ ðU³ð.(„õt¶€`5.@¼‡Î ИñHK4.`ð®+Êš[—KÔ¸€Á».cúT]1.Áãʺ…—‹±h\Àà]ª qƒwAŸYãp¼ADõ0x×DjMÞu¾X ¼‹¼5.ŒC qâ›qÐUuÅØ’³Æ Þ-y …Í`U— €ä4.‹î~æm9iíª£Ñ^{Mï/ëØ0êÎö´>šíÜìL×ûîuŸÖŸ»qxéþsHMH‹ìÛ’õu˜~úµ\yKû¾þ¿ó»oNÿõn ¾ºy^Uù)xý9`ZÊ-˜ò~êÞ¯Ðyš`ô~‚{nÐ Û6{»©árNw¶åaS›îMù}ûs™2µ³[Fž~µ‡vöxÅŽI×Êm–]³ç{e|s“ÿd9xªò«›<슼£Lt«ë<X±}>„Ö#Hò)4ÒˆÓ¬¾eëL»i¦÷ˆ{Kr}¼z¥¡Ó#ÉyÈÑó=”«óp(¥›œúbdM<ÀOŸ_ý’+ò - -§¤»Œb³—Ýím«óp(¼“€j|µCÚ#òÔE’«qðÃgWЙ_Á‘Çê<:ïä_ }™ŽF?rD*l¸&‰wÐx°Î£¡ÑßÎ+?ý¤¿Ê~䂇#÷Á:‡Æ7οÀHmZ?oé¯ s䇇£öÁ:‡ÌÛ9å—ÅC/Ý÷þR‘þ -*_q‚y°ÎáòÓœa¾)ýÛDzÁ¾ü÷Aë<:Ã3ËŸ›%Òï]éÔ~‡}µÝ)“©+R2šö‹°µŸ;ª‡WœW¬óp½o~~œó%ì߯ rþ¥¸ €Š¼Uæ6*Õy8TÞÉ5iîùwàÓ¹#äM[ùÕy8$ÞÉ/n®9Áç·Šlø~Bõ·štû[;mµÌy.ëe.x,ÔÓ„ø½Zùí˜tÄÑt¶ æÉ‘ww°Î£!ŽîœG¾á›ƒæ™²Â÷Çã;š}þÕëDI+ÿ"N|L»ö6\ßlœ8mæú‹•ù$Žæ±LÓ½(|°ÎÃ!ñÎYä­ýò^ú|£âeH{´^‰œ4åcòéiï¢çX‡CϱßA¿É›ÝOGã>²¶÷Áô:»lóÆEAQ^~^mg?&-_Ü寯‹  -û­íW¹mÊÑŽþÖï`‡Cçí¼î"ûb3üÛ9ì§;×ɯ8+ún܃u­wÎ ?WN»†æ{ñýùÞf%~Òt#t„Ù.!±½‡ÎcusÏ+Ù»vBÓgéJ.&\r‘¿æ©½6'ý“~2–bêÞ»¥?Nƒ“¢ Îç:–o~ó»ßþ÷4–Êßr.ן='„M}[[GÌÛ–²œde¶1.ÏòµìÈ3µ³eK;ƒƒÅg£ÚÛí •]ôÝÞÕ©Ûl¯6ÅaO/C«¼úùcÃìØïÚ¸Ãú=m}®cX}ÓÆaÁfž¦1¤©žÝ–'±õiY.®¦ÅM)™X‚\qµÚÖ9â(},>¬>þÓe;1Èç¿åŽ¡Ëvì:®ÅOÕW7ìH.\³ÃŽ±lü!sä—5£Ø&wMEK>¥í«£Í-ö†¯}}T?óò\qµb‹y)šïÝòsù ÛÐ×9õ뜄Õ'a[\ÇÖpõqVÜ)Ø9ÅÒ_Ë®ø:ìýQÛ’6[ôãi,qø„gý‡EÖ"æfÌ«¿jÙiÜu Ï~jcnxKËÏÚÞ‡+Ú?2-"]?ïÇÏ7æN8ŽîìÒ Û9Ù㦷ÇZì8pÃnr[lÌ\³oˇë÷]¼é“tá“™3Ûس2ž´ô¯ÙIq”mXV1ÞÊó2çÍþÞážÖk¾ÞCoù$c_·õÉ»[ÞhÜsñ¡ÎÀ&†úágÙû:Û8XçUŸê~ô‡Ç§P‹ÉÌX®jÚS—ñ~Kå-L)Â2 W;,‰ Ü´ëð»Ý¦¾§Uí3»·ÛõˆÇñ”vùã×wjS®ŠÅxͪ;õ9šÂ˜;­~MRvÉ"h³ÞÕIà O}Ÿ¿=N9qÁ˜òkfN®CKX/ê¼éÝG¼Äÿo8s¶â9Ùý¢Úcœ§Ï£ßîkŸúë'×]1x,ªÉ¾$Iüý6QÙ$—áZÂx™ÉeŽE®MÛuZ™â -‰!úP/ÿIw߈°Îƒl!;—ÊÅ@7¸Ô¾Éq»}ñ²ŽáöÅmò‹c²+1Žã°$w ‡ciMx呇k\ñ—Õ„§~F£pNn×áƒß®7á$FEf¹‰,³ËÓÅ:xw‹ÜÁYóÌå}&g·»Žeë<4)W`‡ŒH„…•äÖ¨k®ó–¥ìxìk<ò—×¼­Lûú²–i.Ÿö=¯]óÿ–{U÷¯¹^Í­Ê"÷}ï7QDœrI7r\ÿ~¶§mnÜÛ¿±'échy”‹©üñë h®Þ‚ic6ÎË5ÙçH&ÈÜ€-²Üžȵ©sÕÙî|¥f½\üÇ^j½oÜʤrýÜrùô°–ÿ±Nú'WȈÿë!A½ØVqöíaˆŸOr74)b¢qÒpG,Âõ¥ím‘ëMö6M¦ìK‚ªìúrœnÙ5Ÿi£‘úܪÉ]d[¡Po­Z:V„¨Ý¥XFo[År¿v6a}ŠÝɧºp4HsÙÑà¨xWì‚„|)´SwCì©Ùãä>Æîº8„ÝôÙunšþVÖ§i:o ŸNè6ì¥ÏªTnõmM5ïzÛea2ªt)·’}B¨Ï“"z:)¯lkš0™]?͚ܸsõr¥•SDO'µÜÚ÷™J½TUKóü0e@=äîxEö4&ùƒÓ±¡½ÜŠåªÏµÍnü…ðé„Câç'¥4®°Ü 7 ÃÔng;ð2¤)7Wai6™Ü,'ÿ/TÁ]*ÌS_8ê’Û!ÇÓèDÖfo4ËcgÒ<à(²“»\J³³¿\¶Jy!Ý^º{l#\6ÉSiæ IuòØŠ]G] …5ÿt®œ…¶;”Ÿ7âa'÷¥Ê¶wÝwîíqìÜù<‹SŸq'W}v{{»æ›ð©6Ú‹ú¹u2ÄÞöh7:›„Ã0Â}®â¶Z;;Õü¯/Ç"Ó"îÓé/§æ|`/‡&18¹¹zμÜùŸ(ó ¾ÿz¢óÚU‰K÷¶³…ïܺÏ‘LþÎA>(–¿{Є}ÙLBOí±Ë£,74ô8îýÄ×4ªÛAì­¤ë êºvòðá$öºÞ…É3|Cø´óbOd~êÜŒ€½<†689þ‘K6›Ux˜‡ƒÇö˜êyG[¼ÃC½­0­½Ð“œ±4FjA›‡£“4jkž;¥&ÛtÚE§ Èl“õS³)®Õš[ꈳê¥ÔsŠ!ýËìižg¥;‘çY‰e¥Ýñz¶zÄ3œÓ%{.™Yœkw)’¡É.û¦mß-Úž§aÐ0$ŽÌÔŽj|QòUü÷šµ³ÔÙDh¥ÚW=qy\.Œ@M.êß‹²9FpÉ•ÿ‰òt\òâ:·ÏO.OIÚùÉܦÃP69ùË_º ÏP”ûŠ©©öWÁôjŪCz™”ã¡Asöât¨þé—zXY‰úc’­¢ü¥§ Io–?P‘¬ÓßÞñtq|R¿þrÄ#Y -endstream -endobj -665 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 664 0 R -/Resources 4 0 R -/Annots [ 666 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -666 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 217.623057 101.433071 198.873057 ] -/BS << -/W 0 ->> -/Dest (cb169-1) ->> -endobj -667 0 obj -<< -/Filter /FlateDecode -/Length 4405 ->> -stream -xœí]K·¾Ï¯˜³Ó|_@àC€$ˆ $#‡õXk#XsÈßÏÇî&YÝ]=Ó³ZÉ-9ZŒfºŠ"ùñ«"§‡mÎ_ü—Ȩ”r t¾¼;½?i½Æ¿!ÅìbP¿Æ+ú³Ñ•79$~W.ª˜½áüt‹Kžüéô¯Nÿžç1g«rÖøó/o‹ŠœŠ)FíU†J'¨¼W1ZU0 -MÌYRÁ ÉÒZaåä%Ì k“ J0ÃQ4n­Ê0ÃFC’ -f˜ì£ 2v«µ¨ƒ!šÈYAg¼ -èô é¬Q!e›E]„ÎD+éœS!’ñFÒeBðIÒlñYIça‹7DZÒÁr)Jº[\pZÔÁ›¢“t¶XôNt ¶]IÔÁ´‘tÙ)Ÿ‘¨ËÐéÓZgµW>9§%1Ê›NÔaFaBt¶í³¨ƒ-Þi+él!üI:ð„w1%a¶Ú2»vFÒyØbm$Q[0øØ[t `©ÂKØeK Øä%J Ø”Åè$°Ù+ -9J<à´Î‰œ†-ž‚ÄÎÀÂÌu°Åe’¸ÀYØ‚8[,9‰ Ø–€‰ è–t68ð-i$.p \˜©%.p`\—I\à@¹€R–¸Às¸SâÒÅt8°®óÁJ\à@»ŽR¸Àw-qwCNQ[`‰ ÀŽÊ™ä$. ð®3:J\@à]L#+qFU‡Aâ`EÙ”´ÄÞµI“ÄÞµ.@ÔE…9è$. ð®õh‚ä¹Á»Ök+qw-Ù q€¢À?ZŒÀ» /‡°+qü=tÖI\€Â”EX"qïš ÷'Ìi°•2)‰ “Äàxj%.ðà]ãà©%x×ج%.ðà]Ä ^âÀAÀ¿)£]Q—•Î)I\àÁ» %+q¼°Ò L"ê¢ŒÄ c¥‡J\À»pïYâ‚ÞÕÞ9‰ xWÂQ[\² ®ÓV˜ÓV•¶` -IÞÕ#,éÀ»ZÇ,qAï·)qAˆE`“Ä0P! -¶„D*Ç$.YC‡OÔ…k/qAÔxP–¨ƒ-IâÖ*»œ$.ˆ¶8k%.ˆ¶ d¸ÊÑ·¸<€-€‹¸> Ø‚HQâ0™J™Hâ‚èÖcèUI¬Jt-êt&J\#)¸b#qÖR*aÒK\‚W%\’¸f@gHₘa ¦ŠÄpE*YDæ’ÎÀ٠҅Ή ’…-ˆ%.H6a•´ÄðÐXK¢«%é²ÎL$ -ª„Š`š`¥ ¦u°4!qƧ,¶½ÄXü`±‰eŽ¨ƒ-ΓÄ)Ù²ÜL¤[¬¶›e½%.ȶ ⓸ ë J˜,qh:­%.È&a½iIâÌ˲ÞLd§±ÞÄO˜·Ù…²ÞŒd‚-!Š¨ƒ-š²0oÁIXo‚$]€-6“Ä9„²ÞL ÌÀz½¸%Êz3ŠÛà]ŸÁL’¼‹È4H\€ ¬7ÙJk{ âõñ™¬LeÅ™Ä êõð­â–÷Â[EqÏ@ƒ|ËIÜ4€™XY¦ îÀ­Bi´¸m Á¿ˆÇhÅ ãîߢ -€C„ÏL±42cZ¢û¶æèÉÜF²qì—ŸNß<ÀùžúÏéñ«a§¬þCjB|…i`3†Û#ú6h¾CÑ–þ ÂÀçä)FÄÜ4 ÅiÓ­ ÷Wþ×f–Ãð×Cõ®Vˆ(ªG€‹Àkp Ë×¥ÅwfyAƒ5Gð¶i\¶…9Ä•šf‰ÿæ NVŒ{ íÒ8_ͧb1LÖŸß¼;}ó—Wÿëw¯£7§ï§µ÷ß~mÊ»]¼Çñ]»ñùøòãk&G›áË`2“ö™¦4?Néè:kY:7½~\|^”Moa_š>OöÓt 4k -‹¶˜Þže{éqñžæéJy^O:êåÕÏô°ÎÓÚ£ËëÛžß¼:ýáÍTˆF&PQ LBX‡û²|_~Íj°é5üQúþ‡©ßózœê¸·1aãîœv£K»#š|»Dz;]ênã]±`™­—9þÝ\®Ú{¾{Ú¹·>ìÇ""ÑÆpeÒÑ^ãÎ<‡C£½Î†qÑÛ~1 -¬‡:y*²8"Ûè2Ý0b¥­åüÐëmv¦[2WdÈ%Æœn9ú =·Úw¥ -OÃì[ΚYŸ=Œv®Ø8ÍíÚ…Z×ù븫 ì:j÷å9jýu¥9hÑ»KÕ÷†˜âÓ12ÆLÈáï´FQ«oá;2+6fÈ€³ž’o<»D kÃ}hbÜÌ‘Ùм¶ð ŽÜ—çph»1~r´…E¹ÿ| oÞÚã>ÎsËH‘çyËÐì'ÝV;òFYµMó~4v¡=>ÜÐ}yŽ†P{#¦ü¬Ö UËدڹ•ìáÉYT`¦u˜fŸÖplŸŒTûŒxsgžÃ!õF¼¹½‡ =‹˜i…[¨®èsó2ïâa´›m©Û4ãÞ=üœY9‹™tŽÞ@k½&e %¢[hÝ—çph½g¾œû”M#mØH/÷’üz¼ºÒkîÞPÚ™çp(½ŸÞ‹,Æþò9ñÕ¯9#7PÈâIcUÖÉ–ïm®£p_ž£¡Ð݈A%¾Z íˆ<µ¹OùYE2:¹Ÿ#wæ9:oÄ_ }‘ÁÙ^döÊAâ-4îËs84ÞØ¿üT|ôO¿Íl¯r¯çÞ™çph|áøñ ôôWÑ&Õó"ž~™Ïøª|gž£!“®Ç”_Ý»îýµ<½ŒJzÆæÎ<‡Cå§ÙÃ|Q>úÍxú tºû¿ Ú™çpè¼±gù±ùç(ž~èá{§úTèö å²Hf˜‘ì»x?Õñ±½:=c¿rgžÃ!ôƘŸç| ë÷ Tòïà -(Û[·ïÌs4Tú±æ¯Í=¿qîÙ¾#9åÊÏÒ $îÌs8$î¼'ócqÍNÎï™!ñ¡úÊÝ&Ë»äZÌãÜåÝø‹Xxyo•­³‚ä>ºç~JÏöCÆ8Òáâöå9ânìG¾àýA|¤ôx=î_Ñ,㯖w'Jjú»ÈøX¼/s9Ó§q®_ÍÌ{Øã¼rÚÅ4·¼ðÎ<‡Câ½Èkëå¼ôùzÅu ¸º¿~Ës²x¬œ ´ç'^;ó =aßwÐ/rŸ£ùp4nýaë—³{ Š¦ßŸm–³l“/ -¿F¸ßƒ -¼øÝÜ·£íýÖogžÃ¡óz\·ŠÅ¾Øÿz ûévÁ7ùŒ½Ây‡Èûîol½z%º‡I\øŒxßSØtKÞX -9³ñ"Ÿ]Rü÷ÉÖåheß9ï½wgžÃ¡uß/·?;NÛBóŽµøro6?é Ú@g# a´³‹^ßBç¾úƒ FlC9“5h²! eèïeÎÑÜǤ ®ÖÑÆÍNí©¨õSŸÔ12S»æú¡ì4vn7µ-\A3k瀹šæ±ÛÙ#”±¾}lôuXöG³êK=kKè}â¹þí(«þwÖæ©¿†´¬ÝƒŸãýTÛ\qÇ÷~Eg‰qí°o<ªÛhZ¶ß_»Â˜¾âNõh7“%nZy¡ÏÙ†Óû£¦ou3l,9„—ßöö§¾«Xñ©³ö ˜óøéJ¿”ãÄýRmó¬ýIhSÛÔ8±âæqCÖVç;w¬=ÖWøPP½oåÞ©Ì=ýyµïìjN­ú~´á8íßï%s9Èræ%W1ÅÃÇ)ÚùŠ°Õ(xBYìu,Á¯9š_Ú»ïïË$öe]™´“úªbö™>F´`t9mõ¡ êû†¶öx=Ì=Jò¾Œ™¼I윷¶à¸™äÆúb}*\=[í\þ– ˆÞ8YžÈàÒÙb‘c9ÄP -²K•)lÛ&ï8=íeœ¾C7d6qj^û™¹Ú6€“›­` ÏÀè!rÛº–YàÀÓx^æ´ÜÊ¿âú[êðój(ߟÂp §eÖ!)cËSΔ+c“ÏåФ™:tfèlsÞÈ9<ëâ}K5=þB—±Ëi¨ã³1ØÅ”þý0èï0'ÁÖY8U¯q²õ¢‹ŸNåQ$ˆ‰úT¾!.Âé£ne‘iÌPe¬¢£ÊË”Î×£}”ãðÆÖ×Ö”ü¾f+GH"̵õªÌƒ±bAˆœÁ¬Å¥õZ™I\÷”›®B3Ňj3„½@âi{=ãM±qš ýZ¨Yu]l©ÚcÊñÀ­ëB6Óy×6ü5­l8¯ /'Të–Ò'QZ™¯EÕÞE´PeŽ•›Ô×”m@¨“ ºœá׺M̦g£VŽê…©‰Zº\Ó ¢ËIL7•æÛHÅ–*õ¬¹ö|7¥CÝ¥Öñ‚ìÂ\l,ϯasD»úq..kÁ6Ö:1ÜØ•ðr‚‹¡•øéä£Úw®Ð¾B +·‰“îxéÒ˜jWajVY9è´ü?R…oÒÂ<ÃC Rå°â~Õ+)s³‰çÄ塱I=`(ö´¬»LŒ ³¼¿LÒBúBº-uë±™”u“Ç\Í+$ÕÈc.6 uM\(¬öOãJ.ÔÕTמAùi&îvú6UöZY0 ÛÅ¡qçÇ6⦜<Ýìmå2ÌWáepl´è§ZI[ݼ]¯Œ »a ….î4ú4ˆëlmìTž¦S?öI&yÜËéçÓàæ¬ó6ãCp¦tlºÃ/]mªqåÌçj.xsp™ºÎu—|fEâª$í.Z²,#.x¡¸dænÙÌÌH^Iëúb?ñ¡p¥.vÁˆr麎º›åEWÙÐ -–!°ã¼`2¼Ò<³G0úÙÓëƒÇ}29ÔP¤Ú›ô‰K[Ê´ž,uÃe˜_6ŸãÎå¢NÞÞC­,ÇæžkANŸ=´ ÙÇ(“¦žmJày¥Te³J«£ht™®²Vb¬6…)[í–¡ÅIì¥ØbŠ.ý™÷´ç£Ò:ÑóQ «´¥ÜêñZ´äZä9œcoî)[,™Y๛”zJWeëºi^wõ¶gÖ ê†„™êžÍ¯R>‹ÿž3wÆ<‹)UývGLêgÝÃQ“‰Îºáñ…:…¡òšH‰m—ÜçúþÉz—¤îŸðš0ŠœÓÑ”QÊ&> -endobj -669 0 obj -<< -/Filter /FlateDecode -/Length 4531 ->> -stream -xœí]K·¾Ï¯˜³Ó|_€áC€$ˆ1 $‡ i%ÙVäò÷ó»ùèîšžÞÑJÉÑbµÓU|É_Ù=lsÖøùÚà¿DF¥”c óûÓû“VÑkü+)Eý#~£?MQy“Còçw,pQÅìm çÇþ[]ŽÉOÿêôïes¶*ç`?ÿò†UäTL1j/¨2TÚ8A彊Ñú ¨‚QhbÎ’ -føH–¶ª3('/©`Y›U‚Ž¢q[U†6’T0Ãd•Ñ°ÃX­E ÑDÎ -:ãU@§Ig -)Û,ê"t&ZIçœ -‘Œ7’.«‚O’Ž`‹ÏÚH:[¼!Ò’¶KQÒØâ‚Ó¢¶Ø¤‹°Å¢w² K°èJ¢¶è ¤ËNùœˆD]†N§˜¶:«½òÉ9-éŒQØt¢3 -ƒ¢ ³°%hŸElñN[Iç` áGÒ'¼‹) ³ÕòìvÚIça‹µ‘DlÁàK `lÑ1H€¤˜—$°1*Ê–$°É)J”$°)+ŠÑI4`³Wr”xÀi58 [<‰ œ-„™+ê`‹Ë$q³° $.p¶Xr8°--8Ð-él$.pà[Ò&H\à@¸0SK\àÀ¸.’¸Àr¥,qç:p§Ä¤‹é%.p`]烕¸Àv¥ qï:2Zâï:‡œ¢¶À€•3ÉI\@à]gt”¸€À»˜FV〉ƒÄÀŠ²)i‰ ¼k“&‰ ¼k#\€¨‹ -sÐI\@à]ëÑÉsƒw­×VâïZ²AâE´ €wA!^` Vâø{謓¸…)‹°DâÞ5îO˜Ó`+eRxð®I>Q—•‰èqaÞ¢—• !K\àÁ»pTNâÞ}&‰ Àñ -ÕJ\àÁ»ÆÁSK:ð®±YK\àÁ»ˆ¼Ä€ƒ2$.€SFº¢.+S’¸ÀƒwAKVâxa¥˜DÔE ‰ @ÆJ#•¸ €wá޳ļ«½sð®&„=¢¶¸d%.@\¦­0§¬*mÁ’¼« FXÒwµŽYâ‚Þe·)qAˆ,°Iâ¨[‰ B"•c „¬¡CÈ'ꂵ—¸ j<(KÔÁŠ$qk•]ND [œµD [2H\€ˆ@eƒè[\ÀÀE\lA¤(q˜L¥L$qAôë1ôª¤ V¥ºu :%.ˆ‘\±‘¸k)•0é%.Á+—$.€Ð’¸ fØ‚©"q\‘J‘¹¤3°A¶Äˆt¡3Fâ‚da ¢A‰ ’MXe-q<4Ö’èjIGš×™Iâ‚DAq¨(q¦ Vš`JQ[@`|x±í%.Àâ‹M,sDlqž$.HÉòr3I\l±ÚJ\lòz3J\5lAÄ'qAÖAq˜,qh:­%.È&a½iIâÌK^o&‰ ²ÓXob‰'ÌÛì¯7£Ä™`‹Gˆ"ê` ‚¦,Ì[pÖ›`'I`‹Í$qA×›Iâ„Xo"¢·¯7£¸]ÞõÌ$éÀ»ˆLƒÄ¸Àz‘­´¶× ^ŸÉÊÄ+Î$îhP¯‡o· 4¸Þ*Š{äËk$qÓfbe™‚¸k· -¥Ñâ¶ÿ"£ 'L»OãU"|fŠÜÈŒiAˆîÛš£'s’Mûc¿ütúæ%œïù§ÿœÞ~UvÊêø…²°Æ¼ ÙfŒ7øJ•ò+LÛ–[oÉÃFÄÜ4 ÅyÓ„+†»ÍçöÁ,M†Ål³A0SZF‹z±T0+D[ŸœçÙmEáˆ<¢·±^ÌFÉÈk‰¯Zçøßyûኑ˜B½ "Þ6Ã*N4öh¦g4z èß½8¹§]ãp° ú²àaLÐïNß|ÿ‡?ýñ¯?b´^¼=ýã[Þ¡Ñš’Ö¼[Ã15=à— ›?{¬Ù=f”yýÝ?Ï/¾ëáM® ±bÌËz¸hÎÚErq,+žšª+iB¯‚uÞO×ôvþM“ŒÓ:=›f¦¿ôr©÷nÎ[ë -“¬4-ÌMËóß 7“óµ4o»¼¿ëóœÇóõ¦;†nÏëî(Eq‘~.ÊM ½Kü¨3ÉÌ,[4yî®’vh6·ÀÝT›œæîˆÓßRÊ÷i²©Ô£'[Ž ‡Ñ{ÍçËöû¡3\WEžêÑn9$kØ´òæ6-``zÔô­î­ÃÔÞe¯êj}â{: -¤gX°Ý¥?S‡Ñ”o·¯Ü¦¯Võs”±ñ3TMëRoœ~«m¥M¡O‰ -ÙÖ¿iì³]Û°jYÚfinã0­+3üþEaF×| âTë -3òV":â+)6ÿÀ¦OÍs“šÉ-ÂÂíU.[”ý茅φQ©Ö:uM»4Î7W…¨‹ÀûV‚ýó÷ûËÁ–iæ¿ûÚ0c—¬º’kéì1vR~&_{nn•?Pf¹…G>¸žšˆoþ¼ªc"Á¹>ÛGÌÇù³_êèíêoZ¦›yÖQ/¯~žˆp™ç@ß± -mð”n9Þ%E¸ŸÅq?R;X„Yqäo9,OaQ¯'Õã>#¹[©…J#$K|Ç <ĉcÿjGTò¬ÎÄÄ›fG·áßãɼ›ºª´0ŵ¼)?¢çý3,¯wIô`ž‰H? -—ò=¾Ë\ª1Ö†æpèå6·úKSº†Ê¹a˜Y#Vh`ZÎ䚶Í|»”ßb³Ze4÷¾õVªöÅåß þêÍ×úCÄP1”åóâðìŽe¹Ôu«YzuS0¹DBC†Ûv]ýÇqVâ{·;¬Ô'öðµàed¥MX*0Ø!ûͪÂ=FrÓ­ÀÂ.ÖahÊîô>4Žå¹l4FrüdÁ6V±À%œ˜4ÿúéw!3‹ ^Í„á3Íi^Ïé_uµC:7ÿ¾^}^•ýYÄ>ßVV”H:´Eß±ÉXóÜ »ÙVï!°,í×½ÿªÇI둪#ßFeùÊfë´e”Ƶ‹|ÐÃrô}§d…;Øú°D`ߎ‘¹icìùžÒΣm\Ç »htÛøŽcJåîÙ>å¹;4Ò>ÆUoûÕ(Œ‘ -G²ôñ4ÿj·º2baH[Ëyµã_ ÜäÒÀn=úz®µo§ -Ϙf°o=k}ör²sÃÇii×!Ôú·Þb-I¶ÚcyîµqŸCoŒ«b®Åÿ+­×‹*2+.Ì‚³’o<»FÌІ§¡éÚ:åÚÒ y,ÏÝ¡m?füäh[¯EþùÞ¼µgŒûFž[ÇŠcž7šë ‘KíÈʪmz»ìÇzßfPßãÃÃ=˜çÞê¯Ä”ŸÕ -a@Õ:ö«v^ˆJŽðä"*0óJÌ ³Ï -«¸¶ïRoˆ7æ¹;¤^‰7×£÷rFÏ*f*£p Õ}vÊñŽ0ÚͶõ^Ð:ý?ç¡œÕLºŽ¾€Ö3zMªÜÏ»vð`ž»Cë~œù|ìP6´Fz¸çä×û›¡PÚcÍÃ;JóÜJ¯Ä§OEÖÀüÇçÃW¿æŒ”Q†xÒØc÷úæ¹7†+1¨ÄW+¤Ý#O]ܧü¬"ƒ ètOçȃyîWâÎ/†‡>‡Èà‡½Hx´C—Ìswh¼²ù©øèÿžþ27{•‡=÷±H×x,ÏÝ!ñà3™‹kzðñi‘_¨ÞyÚdý”\‹ƒÇ8wý<þ*^?[eë¬ ¹žòYÑbšÆ«^øXž»C╽Ƚõò^ú|½â6Ü<_Ásæ!ã3ˆ|»ë`ž{CO>vúYžs4ŽÆKßF¸ô͂ų‹Œ¢ùhËY·IŠ…o#<݃ -¼úæÜ%nËCŒvô®ßÁSØtk¾°Ú|óûa¨oç{;Ÿn]ÞÐúã|Ô]ôXïù¨ÑäMNÎk½šä -"7Ç’µ3ÖÎfuDÜò‹ï(’ßqáø` E9ò±—¾önßL‡r˜ù´'>@Ì¢+]=ñ(Ïá­<§©_—/“}"­v‚Ô|ÒV=i©|u>LéˉWy’—“æ“•ÆS–Ú‰J"s}eQ«§Ó·†S¨Zù;§µ6Ô¯äÍP¾?…rÄ©)Ç\ë”±üŒs åxlò™UMÚ‘©CgJg›ó…œåí!ï[ªù…"šÏçóeËÐŽsú÷eÐßøØ&î&Œéð]ÄÉÖ‹.~<ñËH){PÌÂù£ne‘© ›ì¼U^§t^éùcn…ó÷þ§Ö×Öp~_³ñ¡œ¤k"WŽ¥œ*„ÈÌVÌ­×ÊÌb~FRn¾ -Í>© {4¦íöŒWÅÆéAè·B=T×Å–ª=|\Yý˜Û˜ófÕô±ëÌ0ü5­Ãpî N¨Ö­¥¢”í¯EÕÞEx_en0*7©¯)Û€P'Aôp„_ëZ4a0›~5>üx¦ -$jérM'ˆNbº¹4ßF*¶T©g͵ç»)ê.µŽd¥fiä7 sD»úq)æà ÛXë4àÆn„'¸ÚˆO> -©}ç -í+ôF¡Êmâ¤;^º4¦ÚU˜šUÆGÇòÿUø&eæ) 5HññÏýªWÂs³‰—4ÊCc’zÀPìi‡î216ÌŽý…hHHϤÛR·[H‡.ä1Wó˜¤y,Ŧ¡®‰™Âjÿ4®…ºêºØ7(?.ÄÝNߦª×­²`¸ûVn‡Æ£8¶7|–w³·•;`¾ -Šc£µ˜¡Ÿj%]luóv½²AØ )tq§ÑÇ"®³µ±¿Ÿ¨~ì“Lò¸§ŸOÅÍauaùøÃà ¿šbŒ¼Ìù¿Hó=~ÿu¢óõ,äÉ[ÝØÂ6n]ˆGOÆ/2JÅüb£*lÓfZh¿‹Mêi}ECóãÖ|M=»îÄ^SšÆ ¦iG¡ï}8ˆmƒ®5nè¿#|Xõb d>tlºÃž.]mäȧhW«p16—©ë\wÉç¡H\qÒîÒ¡%;dÄÅX(.‡ -s·la&r¬¤u=ÛOãP8®k¸Â¾t]GÝÍŽEWYiÅ!°/Æ‚ÉŒ•æ…=‚Ñ7O¯÷ÙäPgKµ7éã(m ci#‘Ò°]òä<ûû'Û]’º2Ö„áP䜎†_í)›ðE¾®Šª¯ý”³e-6éi lušÛg_ÞíV¶5Ë&Q»Œ¼TäW9:z¿*jùx{Ã;l¯Û'åç ¿† -endstream -endobj -670 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 669 0 R -/Resources 4 0 R -/Annots [ 671 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -671 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 698.673057 101.433071 679.923057 ] -/BS << -/W 0 ->> -/Dest (cb172-1) ->> -endobj -672 0 obj -<< -/Filter /FlateDecode -/Length 2760 ->> -stream -xœµšK¯\· Ç÷çSÌ:€R¤^@áE¶¨Ñ5`4‹¢‹d'(® 8]ôë÷Ï3G3C_¿ÆwDêÁ#Q?‘šá áõ‚ñ_Uµ¶’õr}·½ß(”Dø·×8võk¼Kº0i ‰[®éòÎRBi)–|yÚðß]q­þ´}ÿÍöïs¾ÄÐZŽœ.¿þd*•Pj)”UƒŠXUJ¡”˜²£Êðˆ­y*˜‘ŠF}T˜¡­&O34Æê¨*Ì-,ª3baõT0ƒ[*ŽŠ vp$ru0„T%::N!cÒ³§‹rm±¹º—èéDB.ʉ=] 9çT=–Ԉ=]‚-‰UÉÓÁ•Z<]†-’…\l‰µˆ§+°%bvš£«°ÞU]l¡Lìéš„Ôªª«kÐQ-õQ)…TEÈÓ1‡ßW‡…EÊÅÑEØ’)5W[’Pôt[/ON$)µ:»5Úîöt ¶ÄXÔÕÁ,¾G€˜a •ì!;(—<ÄR‚¶¨b• U«GX[ÐRÄÃ@l)hnÅã€CÙlIš=ÃÅÎuu°Ešz,[PÁcl‰* ´Ux‹Çn•{,ðV‰³Çpa&y,WjVä•šÇsìôX €.¶{ñX  ®¤=°+Z³ÇwE™<(¸+‚–®¶À=€ŽA¸ŠÇw…©x,PpÛ(z,ÀªøaöX_ ±VòX àn¬¤ ÜG€«+{P<(¸Á;¹Áݘ(z,Pp7jÌ à(ü!7w䇰€õX€óº( ÐYˆK<$p—Ž?gOƒVkË ¸ËËçêZà‚wö-f9pÎÍcAwqP‰Ç‚îŸÕcp¢F ÜeÁIíéÀ]Ž<$pqCòXwŒÀcηÀ×uu-P«ÕcAw¥è±§p  -’¸ºàHì±0„8ÔcAwq¼7Ü¥$â± ƒ»¤{\l‘= ®ÛÖÙÓVEÂÓ»ÄXaOî•æ± ƒ»vlz,Èűz,€QpôX«†VJöXA‡ÏÕå€ròXP‚d¹:Ø¢E= °MZõXP"l‘=”[2x,@D#úvÓØwqó…-ˆ=€d¡6U%eäc˜UO—c¨¸vu:. JÑ€£˜= — -›Þc,\òX3 cõXPlÁVñX€£(ÔˆÈÜÓ1lAí±‘.tÌ j„-ˆ=ÔX‘efòX€¹$¦ÚÓ)YžY=TÍÁBEØ&È4AJW[€ XK¶“Ç$?H6‘æ¸:Ø"I=Ô-ݬ j…-‘¢Çø¦å›ÅcA#Ø‚ˆÏcA£,LöXŒCGä± qE¾Õcö¥å›ÕcAB¾‰ÏÙ·M²å›ÅcASØ’¢¸:Ø‚ ©9ûLB¾ :yº [bS-gË7«Ç„È7Ñ»WÕòÍâ^€»©LžÜEdš= €|‘­—ÛÀ› -â3_Y-ã¬î½ g«{e@`/N«âÞàk9’{i3‘YÖìÞàX…’ɽ6 ðñ˜>0ávû´^Qe¸CÁ™Y‹=döPD÷#ç˜ÕäÕn÷c¿þ¼}ûßËÏÿÙÞ~³ß”õ#@Øk+â+lƒØ°Ü Ñ7ãñ%#8Ü-ƒaŸS×-mÅÀàqÉfFḳ—Ç'Kaèë}XÃ6œÚ·aëÈH™á¸éê~¸Xú™Mþ’!œ®u¤E£\ q$é©òïßlpÌÊÛç(.æã|Dò«1]Þ¼Û¾ýË«¿ÿõ»×p›7o·üŽ(¥—/Uo–_ó!ûïzÈzvS7Úü4õœ]o{×.µôUŽúo]^tÏõu=twu˜Ïv¥«…‡ŽnïTÏc¶ô…cÒœ¿¡;þ¦xüíýñ|f¦—ÿ¼¼yµýáÍÙáö¨sw8F‚DÈ}ÄG?­É×øèoášHaŸqÍ1…é<ÝZî–h™Ò±D}9îÜ*Ò¬;–¶.ýõy}0”$ÊbL„|uAæX†VùÈ´?¶cëb¯ûyÍÇ„f„Á¾Ü@t„8…Å6‘¯þøç?ým™HÁތرÞö)Ò2LxcoH¾•Í¿îû‡ÛQ'Ù¤t&Üxv_·É»í3«gþo­/«¯oñn7y´ºùöÞû:>ïýÔ¾xÇx?ØbØ"Ú"ßô§þÓ1öbÏÞGZž!õgxy¿”﷼Ƕ¼ßo OF¾o_€àð bkÓ.O#ØVîKÇûdóå-÷¯ÞZÇ7Id—K–XܾfZ -Gý÷û¢¿Û±¨Í¿íŠÛ­/íb„¶é^ü´Ù­¿:bñÅ&<>Òè Àˆw²+ØRB»¯))Ðñ±Î%õ1õ§±ö©7³h Gì%Û·!Zf~ÛÓSàClßœÔ G)SS´!œêZwv8~TÌB‹0= -inŠ£v{Ø2í1uy -‡éëÔñ²ü½n,Ër>'¼nVî¥O®Ôîk{W}vÏv™,Fµ!M½æXë䈮›#|A½kÅbý²j–õÞ„µ;’Žz­×sD×Í­wô–ÆJ•Q«Î¦­Ïü4eººÔ1ñŽlŸ£CZì«àeôgq¶èkdúM|^71ú ~ÚRqj§É -JÝõV¡,ýq¥é/SŠLí˜*lÍ.³œÁþ¿¡" ©‘gÿÀ:\ÊòþYšƒØÞâó$­ò<¨£Þ °–Yw™..eøì:_\É©oеnj¤Ë”-òÒºy©³˜‡× ±!¬ÏÏ`å*¤n†›â4\ùé$žv¦±Uáì}°Ì‹»§ÑïçÁΧU\ÆŠ³]â {G¿‹Ïwáu?Øô^l®_û Siœvs°E8 ¦xbôi÷Ý:èd_L÷s“y'îuûeÛ¹()6|ÈÂöÄyñ忨ó -ïmzù®'Á·s<Ò El=‰×“̾Á®Ÿ(¶o´»pl›EuÁþsuS÷†qŽÇ¸ðZgsš`ï5y”‡v¦9‡‹8×,Ë̤g„×»YÌ×®Í<°oEé‡aÄ®OºU(¬ƒb:™Gòeé%«:th5. QX;Eq°MËNfšC®ƒŒ©7ûu] -±±–ÂFXQ¦Nç1»vÝeûS,UËZX;V^m'{£¿x{}õº&ç¾L:Ð>¤O«t<˜Ið|©ìÅ|.Ž3G.Vè›wÎÐèK–½'#È™[c†z*¬>ºHëlvTHë Úe§AûA1puÙè±t›òѬOËþÄÕ¥2bŠ)ýe鴮ʘĴ®J~¨kýöoDK2 ´ºs™OG³æˆ%«#Ëkë!ÕYSºìql=ÝOÛËò: É32¥Ù,=Ôü"þ}ÉÞ¹µ¹»‰ðjõ×~;Âu^ã V.eÿ% ÕœÁµ/s‹Öåºä³Û<òxKÒïOÖ‘°AE¨°ý¦Ó7á ŠöqÅÒUÿ½§ß¬‘ûHŸ7¼TØíÐþ£žýîn¿$Åb©¢ý†O,è­öÓƒBì^ñÕÞ§ë“ýõ?¢ÿ -endstream -endobj -673 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 672 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -674 0 obj -<< -/Filter /FlateDecode -/Length 4400 ->> -stream -xœí][¯9n~ï_ÑÏ ŒVê,ü 2ÈÀÈ>y°{ì]ǼyÈßÏGUIbU±/çøÌŒ{ìiw© E‘)©ªÆ-þþàðO!gJ©9ÑùòåôõdMŽZ‰ÍEcÿ„OŽgg)›èj*ñü… !›\£ÏéütÂ?»KYüéô§ßþg[ǽ©5yÏýÄ, -&—œmTX,ë‚ŠÑäìcRXÉ ±V1b&OGV†TKÔXƒ¼/ -«@Œ@Ù…#«B Ÿi,ˆájÌ -ËYÈá¼µ*‚X¢àž‹&AéIãygR©¾ª¼ žË^ã…`R&Æ«&¥‹Æ#È«u/B–般ƃ,JÖx ²„¬Êƒ,¾ä ñ2dñÐNUx²ÀºŠÊƒ,6Y§ñj0±"•WÁ³%—#ÏÛhb Áj<çL„m•Â$¥¬ð“ʃ,˜| |‚,6' àA†qIß³¡êI_‚¡BEC_ª¡œƒ¾FC©f ‚uày§A°%RÒ 8ÈBð\•YB% ‚‡,( aAÅSа m Ö¢aAÜ’­N¼%ë’†€ 1­†ˆJ"  ¦T5,ÀÜìÔ° táîYÃ‚Ô 1y `7PIàn g5, àn¨©ò X࣠® ¸œÍpnä5,À¬ØaÒ°¶b|)VÃîúbIÃîúŒ ò² ¸ë#† Enà®ÖkX@À]O>iXC1À«fÀ]@HÔÓÈ€Õ°ñ<4,@cÆ#-Ñ° w]EøS|he\©IÂÜuÓ§òªqWüZ6.¥ªaAî"P "pðY4,ÆDT¯aAH­ñ€»ÎW«aAî"oˆÀŒC aâ›q¦«òª±µ "p°ä5,@6¶IT^60$§aÀØXä¡$à.Â{Õ° wm AÂܵ„´GåA–P¼†ÈkÀƒÛ*>dÕX¤ÐxÀ]ë0økm®$à.‡M Rf‚/@@ƒ,ØkX -™šsÒ° U R>•— ®£†Ù‚Y*²P& X›jÑ° {ȼװ {È‚”AÃd¦:dßêò²À\ÔõAdŠÉL©D䘰ƒV5^ò¦dÀµÊ+๬aAÎdŠ†XK™§×°o8]Ò°b€çHÂ\! \EÄ"S<2sç ’l é‚眆ÅCdƒ_°ÊLVÃDh¬%¡jG–×™EÂBÉpª¨aÜ+M ¥Êƒ,€ 0?¼ØŽ`ñƒÅ&–9*²„H”ây¹Y4,(²xë5,€mòz3kXP-dAƧaAµÉpš¬a`(¼ïq³5mÑHg_Ož7”×UåKêÜ^‰òÖºTÏㇾ.uÁ™¶SA›~£Ãº×úTYŸ]çÕe¥hù<‰÷Q7ýªBÞ+|Wº‡æþ(d Né×ER¥¼[úuÅœöVù>’µãGmôÁ:¯>ï²ßGmôÁ:¯!ëq«ïÞŸàí~:|W2D¥ò :™O3Îï¿œ~ÿã?ýË?ÿûOhóýçÓþÁZD A­Ë|"‡ÿ’µÄŸø|@üýy¹ŽñÝßÿxr•÷äý0/ÅÜ9 ´áCá®÷½µoZzõåÝüèÖï´~wz\zç²0H·Ð¡÷v} ¯eÑEûÄ°´ýʯø\ÖQ]f¿ôa©×ÚËõÁ®è2®ý¶ú«¼MŽ4e²õ¾[ºôÁýÏòÏÐlÝkÖb¾< iEMòUS}Níeí-­Ý$Ù>ì¡×Ûë Ä4íÛs?ÛIø6¥c;M]éþ–Ù_“Ãα´™ìããÙð kF’ØmÀ.¼kz“zÝÏë3M_x§ê)¸ƒžÜ{owŒ]ØÁ¤}üÍÇXÞ;¿‰b¬EʺèBG> õ–ùc|Ù"’@#}ÜTvã^°gë;Ýÿþñ}‹“aär™L¨-Nòmh¾8oh’ЉØúÜ:/Š­kH5žo”ÀxÑ‘çƒek¢­ÅÒ>Î:ê½.ª—.Ä!o)|ÿ®KØý×ÿãßþ(Ã.OG\ÿÒƒ_w—I¸ÚaMÆV¾Åh×ágK;®™É |~»]ÅOL¸ÈJ»ñó»;µa|_^JפºS+×ÊÖ¥Õo!9Ž¹hê6Û-ïS¸¡©€Öß»>0Ziýû£ÐlÞ¦,ôy'cìáÿ®¬Úý'Ái²ÞÕ>ß… å¥|u¤wêóì[‚=íSñð¸B»·í4V17û¨Œ$˜}RÅ)Bò](.«É/ûøq”7IR¸–ø ¾¯¹ò„»N›·_Bj7.7PC*Ç÷Á´û¦náƒu¾e ÷Kà!ß{퇂¶Þ#’ãÅ{>û¸lË z·‘ïÛíec÷Z¿¥?.ϻǭˇ¥­)N Ûng2L5Lû@¦ÖÈ ¾m‡·,1"Q_­*Úd¼£BtǬó½Y"ÔzË;âŒÅ“ßZÊ@«²~ÒòÙÐóºÔÍ‚Äï•ç~^Ëœ<ï·uÚçgåwü âÖdb©—‹vç¹×»byü€ÆbEü|„G›ý=Ë{¬ÎwbySl~Þë†å…ÅŽÅL~œØ•<²§õßân–‹>«ïòÎ:öq8m¿¹¯fÕ¼ù°nî„Ý7)ýi²ß³˜2q‡FJiwñÞ¶˜Çê|o\¹e1¿úlí¢Ÿf¡C–˶LëslZlyËæŶ Þ +Q°,ju> ,ì#×ÆQ¯´ÕÇ´Ëú&ÊÞB“s϶Ðë|wêobÚo™nõÙûëú¶È»±æø[¶Ô1¾ý¸»‡^³X‘˶ªþ^~°Îwg±wò¿W°HÆÏoµÂkùÖµì``úš¶¼RÁðÑÎ.ÏÝô¿³ðãF¡Ž‡×²Š=ŸÇ;{d¸b¥3ï+|ODhGÜ6Ò‡ª|w6úm™¢j }¥úÀiSö>@‹t÷±}\þåöÚV¹<ÀË=]~·ã‚±ƒ³•á=è{ø(/“TîÐgG’‘MdÙü^P_÷ø/çÊwˆ}ƒ—ÿÛyÞñ_qñšvŽåeÇǶߎÿ†ÞÃ^7‹ÇÏ?ê[y›#¼›zŠ{=íú†nV¿9âݼn»Ïë¼õr}Gº•?ìýMûèõ´vXy'—ÏcGŽ7õSÝN?Ï9&¬pVBŽ?&|°ÎwsLXùî¡«O¿Ò1á¶Ã×;&<¶ûœcÂ+R=|Lx¬ÿíÇ„×5õý¾@ûâ˜ðEÚÇ„¯¦}qLxÛN_ù˜~9EçÅcÂm§»Ã™ŸÖçvsDÅù½ %º"èw·È°;ÜI8î†<®{5~ÃÙ^(´gì—Ž"j&w$óè­q+™ßXLX¯Ò…ãÑ ÎI– ΊwÉ.XAŒG¢ÝM²§.ã·Ç Õ¥I¢KÕ91ý½¬Ïb:o/'töÔ'•Êï ëMuíz;hAU5ö’cBhÌ“Bâg*Äloš0™ƒ/fßä²K7$åj/§.'µÜÚZ3•G©2«Ö®ù)Ê4õP†âÚEì6d~½©ðúÏ-™Äc®mvãÄË !†äöͱtœXac7=I ¢ÝA.vÚˤæÒU×ì4~ÿ»@ETFžöÃÑ0)~—ͼš°oòVI’žê¦Gy–êr9›•úrÅ*åtG顱 U¨LÐsíâ1H ðØ’Ý°ºAfëúX)‰¶ î&9S~Ú§œq¸j´£³ä„¹ÇÑî$§O’œÇŒ;~1Ñw´+l¾/-°ÑžÌ¦_z'“ìíˆv³3Aœ‚&yÂèS#woèÄ/[í?§“i÷rúË©…9¢çEgBJŸ·OÓ¸óÿ¡Ìøü÷‰Îì€-qÜÛ~`ë†,#¿•µ¨O’:ÆÔx:5´Ë´½1'´µfwÞ©¡ÑV¾F’3]c¦´¹6*¨eV[ DÙ)uÚ¦Ó(\€f;m´˜»Li­ÖÕÒF\T-å‘SLê_¤¦£œ•¡Ä(g%Êr»=âl)Œ(JsÎstv–¹dQhIÖTš%C§û¦mß=ÚžÅ0h -’ffjgµx(ù"ü{‰ï¨Ï4j¥ú߶;âÊ|5¹|ho··%%`ñ J3±]òì:·÷O®¿‰Kö„é0‚ÍŽÿ?ºO`Ôû ÑTÿèÕªU‡ô<(ÛCæO ºö¢êvBÞ6‰Æeæ¥"¿—>pÒ[øuºÙÞx"µÝìºÙBiÿZžo -endstream -endobj -675 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 674 0 R -/Resources 4 0 R -/Annots [ 676 0 R 677 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -676 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 563.554475 101.433071 544.804475 ] -/BS << -/W 0 ->> -/Dest (cb175-1) ->> -endobj -677 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 156.557453 101.433071 137.807453 ] -/BS << -/W 0 ->> -/Dest (cb178-1) ->> -endobj -678 0 obj -<< -/Filter /FlateDecode -/Length 3228 ->> -stream -xœÕ›K¹ Çïý)ú¼€µ’Hê{1²A ÙCƒÝöì"ð毟?«Jªf÷ŒHÚSEêÁ’¨)uM8{ü¼ø¯pp¥Ôœø|ùpúxò.‹Ç¿¥ÄîfQ¿Â'Ë9xÎNBMEÎT@Ùå*1§óã ÿnç⧟¾;ýs_'œ£«5Å ç_ß«ŠÉå’³CU¡ò •ˆË9J2T)8å|Ϫñ%¾÷]® °ßó9ƒ6ïͺ_Ðýñï÷a«ÛƹãTmL›qIǧåtÄŽˆþ jÕËßÿñ™¬R⢭ª÷ª›·õ -µP7Aê1êÍkX³ô?uˆè¸!ÈGjáݦ_ì±cÅÀ…ù ÏxfoÄ7:ŽÒË{¾!{.Üò“P¶OZ?;yÞ’'9M×›.¼ÛÊ¿º÷u–Ï;ãzâ ¿Ÿxwð[eL÷ÝöûíÐwÞ´û‡ƒŸËá¹7®É‘¹mÜüIu7<;²Í‹ôÀ3ÂYs|ÊóžWçñ¼nvôù®çÑÁãä8‚ÓL¾‘C¬ƒè Ž$‡hÒg¹Ø³ú¹ÑHûzVÔ<öw7:Ú£ß!5îèQö‹š…ßõ˜gÖùæ<†î²ê¿>[Çhxh·å²/³ô‰ÿ¤^{®Ê;w.{Ž-uÚóðä%ËĪó~b¡lº[ÏQo´Õžéa?Ž!ÞðPŽŸî¡Ï«óÍy¨ÜeÚÿ"2Ýë³õÓ«žT'¯{óÿí©ýùŽÏÝVè-r9R៊ÂϬóÍyìùßWðHåç—zá­|ëVvЙ¾å†K^i0ü¸¿iííú?xøœQð´*Ž<¼•Uõýùã6ÈpÃKGÞWŠºÉrÞyßIŸUå[óQú²LÑô…KÓWÛ)_ ¢ß² e:4ì#ùd‘'Fîúˆ±ˆõç84c‹&õõ>*ŒŽkÖñ[›æˆ¥KÛáÛ±xÁeZï–ËphZS·2m³½z]æâ×͵n¼uùj9ußeã¶øúTµlZ?K[ÛõÒNiG[o¶ ûeÝÌ«~×¾l}OöðvПAÚ3\MåÇSZ¾Ý Ë7ü>¢¾xNìH禞õ¥â‰C›º° v8ߨ¹¼8ù±—ÚÞ¥ôúz…~µ¶¾h9Ýlå?.“þá*;uìõ½'¿ˆA_9ŠOúÞbfŸ)Vávé{[ZÁ.»\Ç’$Îo—µ7®§ëe”ö4Z_Z5ý>‚}+DˉüÚ±!DÍ®ÅúôÞ…M¬ïGÛ]ê¦Hj6C8ä¹ìhpT|RÈOB¹ú©»!ŽÜì ú]sº4„ÝôyèÂ4ý­lÌÓtÞ^Nè–ŽÒGSªo,µ¦ÚèFße4U»TZÉ>!ÜçÉ]N†ð…oM3&³ë§YÓï}WaiŽÄ½\må Ñåd–ÛZ“>S¹—*£jm#?L®N¥¼![Æh“f}zZ#žÚå^œtÚ\û2ùM¼^N1|%~> -endobj -680 0 obj -<< -/Filter /FlateDecode -/Length 4362 ->> -stream -xœí\K$· ¾÷¯è³Ë’H½Ã‡I# b`‚vÛ» ‚Ù›ò÷óQUzTû1»¶ãØ3ƒžé"õ (ò#¥ª’;[ü~îð'³39—ù|yúp²&‹ŸZbsQÙßà“ÂÙYN&¸s8¿%“Jð)žŸNø³»œ‹?¾ýìôÏmwö¦”è]8ÿë­°˜LÊ)Ù ° -XÖ‘Â -Á¤äCTXÑ ±1BbÏGV‚\rÐXƒ½Ï -+C âäèÈ*Ã'Ç b¸’Ârr8o­Êƒ –™¼ÂsÁD(=j<ïLÌÅ•—ÀsÉk<"»à4^11Æ5C–P¬Óx²Çl5daÊIãEÈB‘¬Êƒ,>'Òx ²xh§(¼ Y`]YåA­Óx…L(™YåðlNùÈó6˜‰¬ÆsÎØ&©[Ö°€»>!¨¼dàƒ¤aw}À´È ÜõÁz ¸ëÙG `(øcÕl¸  z:Y° ÞƒçIÃ4f<Ò pׄ?ŧVÆå5,À]—1}*¯— qÅo¡eãb,à.iX€»€Ï¬a0Þ ¢z p×"µÆî:_¬†¸‹¼!hXs0 €†ˆoÆY˜®Ê+Æ–œ5,À]À’×°QØØ $QyÉÀœ†cc‘‡jX»ïEÂܵHÂܵŒ´GåAÊ^Ãä5àÁmŸF²j¬Rh<à®u˜aܵ6 "pW¦†1 Ág  Aì5,ˆ™MI)jX‹)ŸÊ‹×AÂdA€,•Y8±†H¬M¡’5,H²÷$Y2hX€ŒÀ‡ì[]@˜‹º>`È‚LQà ™É…YÂ"ÖcЪƋÞä¸Vy<—4,H‰ B±Ó°k)“áôà¤K@ ðkX -d«hX€Pd²Gf®ñdA’­a2]ðœÓ° {È‚lPÂì3V™ÑjX€µ$T­ñØÊ:3kX9I5,€›`¥ ¤Ty0¡aæGÛAÃ,~°ØÄ2GåA -¬aAÎ^–›YÂœ!‹·^Ãئ¬7“†ÅBd|¤ÉÆÁ³VÂâ2Ö›ž5,€_Êz3kXPÈb½‰%žâ·…¢¬7“†…!K@Š¢ò ’¦¢ø-0 ëM “Æ‹ÅÖ° Ä(ëͬaÒ ¬7‘Ñ«[YÖ›IÝ.î†dÒxÀ]d¦QÃ\`½‰ÌV[Û[oHÈÏtf–gVw , 7 ¶ª[Ø‹h•Ô= ð•5’ºi1±²ÌQÝ5@XÓYuÛÀ‘ñ–ݧy‹*ÂbfN2È·`d÷}Í1ŠÑ•bËþØ¿þvúâ5‚ïùoÿ>½û¬î”µ_± N -˜Ælû‚Ù~‚B°“«ÉNcLº ¥¹¡$·%Ëò¹•Å·qcCƒ¼µ¡™Ž¬©YKïÎI\Ql(ë6”'Ê(½'‡8ÊöîfâlCQµ¡Nƒ] íhú½¼xîÿÄs%U—Ÿóñ‹‚²íô1¼×òª“mô6éþÅù^œïæ|Q~ÒùøåÎ7 æE"Ãœëí+yª$ÉüùÕûÓ_ÿæw¿ýÓ7˜«WïNùÒZWŸº°Ögka¸–#>i¡Á9mðË5ç¯þz~õõ•>äÉŽÞ‡4ëãRVf¹àsÁç5®¥t ½-Ý]F¹*F+ûNº=.Ý:§íÏ­}>.'ô>Êsžr“Â…¯>wò­;n]hÁ­ÿãú¿ÑÃÒ³”­¾YèMâ}-+OUà:J)Wi»QIùÚÞ*CSxoãíÚÆåNýдºÊn×zmliŒoS'?C«¼×jk‘óÐHmyß­Ñ„_%Z¯ëhš ¬#²o0;<&Mõì¶|ÕØeE\Ź^8€Ðm’B³©ÅE›u†¾;¶InÝÈãAÇvf×hã÷Óxë,]VKC›¹[ò’·×äµc̽M¾£ŒA'‹–èJÛÍת{¿[u!m½ž-|²6Âlées«“ÖöW}ë\Ûº‚‹gŒ~ø–¬ýÊ{ŒƒâM=ó^M¦ÚOX¼.¢c7úiº©2¿:js´EŠ;z¤³M¯¡µ×ñÎú½¦7%äÓ¹Ïzývµÿf_a+·úˆó7õ÷zì>ê&™›¿¥ÖæQ¢×ÐìÚ.ò[ÚŽm¯ÇÞWœd^CYØ„-ݦºí^›§U7Ã/nê"t1£mWÞÊZõáž=*F?#Zxû8Ú3"]¡a üG’îbÚUMí¢Gkû9ñ²–½l£™ñztz\·¼×mm# y~˜HȲxú"á±&Ç×äýäHxl»%¿?Eä®Quµ[×"õeÑcå7tY£WJ3Eî!z ½nꜽn>²ÚˆßŽ[†ðó@þ›º Ý"Ä°‰{ȯ¢þëÁ¹!{Åš0ãÉ:þð8òË“™î:òÇ õ¯ ^µ²´j[[Ñ| ª7¾ß–»‰ö_N×ä~\ÓÑí5ýýâ} ò,÷§ãý±}G;V纹\“ã“qüØö Ž÷­H»×ÍÏÇëÜ\¶í†pS/~¯—0Ùchó”uy·Õ|tLÇ€®Ÿ0ln³£²³a)ÛmŒGÕ&öñ䎌ÞjíÜÔ] ;ÝùµÙßšÏþúUÝ¥¾á\ä틺?*/Ëøì|$f#X;ï©>·ÎGmR¯{¨ÆËãܯ¡âåñWk‚-Ùò~cÕqëuQM¿tú~µ FžÇ¦eŸõ÷_ÿù˜÷YŜ׀ʗD›NÁt™„«–hl‘!v.8‚%·ÀTVúív=".’Ònx÷ÕÚœëÛC1^“êNýMëÒêßÒ\ û¡‡àu‹ñn$oçGñ–{{i½niAjiÀý~r2$¯Cj㺫U’¹æúœôGi•­Ü`˞£UÎ^–áù¶m†2iùf_’~§CüPÅ&yv?8Ê.©w8ê7Û€¾I¦èZ‚tL@äË"ï6í:­½~9ùdÁÂOgæ;à÷`O¹¡ôƒ``Ž·0оžCÔná¶Éø²V»x·]lØדý\v<¶<û¯ýßÛ]Ü]»c]ò‹µ´ëÿÛ1=î_ŧ[þÕ}'Ù¾%߶“»ÍÉFV“ OlJ rqÛá-ß*òJdó“ø˜o=Xç§æ[%¦[¾õP´MJºHëÿ‹Bm±¿Òiú¡\MóÖòoÏûmúùNÿÞËM¹ÏÁwvvû¬¨ÜÆÃër‰gû®ÛYÊÃføA;{¬ÎOÄΆØÅßÌc?Õx[çáùlÛ6œö›‚#Ÿ7A¶XÛú1Dz|‹›Ëò|‹Ã<ñ:¾òÔ“ùÍ4­ÎÛÁŸ—ñ{kC“¢ä…ÔœëË•7ÍíÑJ?5{s0§[w`«2—µº2yé8i÷Œ¬–yÀ°ì.Èö²W‚²kû5¯—ïúúÙl¶Ÿ¸¼~—öçdûíXÞ% -Îvcüf}Ó-ÌZÝù†u¸<½8× ðn‘;æv|Í®=‘ux)k›Ÿ I9âŠ2ŸÈÉ“¼z-;ño—U@ÖI¶@û¨m×”Ue®© ü–ÕL[r}ËjÝŽk[.5ÃY·bêÖ^Yè¾o“N«‹hBÔþÖê¶Û´íÕÛ¿±EØÇÚSùáëήžra#B“—c°Î‘<ÏÊYުΖص©sUÙî|¥f=<ìC/µž'fåˆy½|9llºX˨“þþä -ÑݯgÿØJÆ"2ìÉO'9û‰2óØ`¿CâúÕö¶ØõýúF»UåÓ]Idbvý:6õ%=[¾úÐF#õC«&ïä²m…¨¾•ºt¬Q3º#YFo[Ér~V6´^Å.Šl¡tâhç²£ÁQñ.€2Ñh§îÙs“ÇÉy ]uq»è³êÜ4ý­,Öþ!âå„niO}R©rjOkªi×ÛN£I¨Ò©¡•ìÂ}ž’<„| ~n[ÓŒÉìüiÖä샅˜›!q/WZ9…t9©åÖÖBŸ©ÔKåQµ4ÍQ†©SîŠWh—éÆW’'AZâÛ<Ìd¹ŸÓçZ69Ö)®;ˆ—B ÈõÁócé0°Â†fz3‘¦v;9Ûa/ƒ*Iüò ®Ùhòæ¸ü] "tª Oý⸛”œþ0®F'⛼UÒLuXÓ€ã4ÊNêr)u›õ…,B)/ ÛKwm¨“Ê&z*M<©[²ëV×ÉaM?+g¢mB »AÝ”Ÿ6ä!gè®lï,ºÉÜCowcÇΧ™œúŒ;9Ê£ËÛÛl¾/5°ñž,¦Ÿ['ƒìmv£³‰8( ò€Ñ§JnÞÚÑIŽ'l_‡“i÷rúû©†9OÁË}’HNN¦š3/wþÊ|Ï?N|þC{cb‰ãÞv´ð[7ä9’É9†ùA²œk؈Ým&¢ç öÙåQ64kèqÜû ¯yT·Ø[I×ÔuîL C‡ÙwÓõŽ&Í„ÄËN‹=‘ùÔ¹{¹¤68¹ã#‡h4©p1—yðh„äóÔ$®¤èéಟ*âbn—S‡eH¶S r^äçy*Húš.¦4B.iðx„Ù¹éF«£˜Š ¦ùbn˜ÝÜiÙÈ£ýÑîõÉ󾊛µC{§>ÍÔ>0¡vÀÓ©T/ãö²Çª·Jšó õ¶hò=êIÎp‘Zðæb¶Ñ‰šGµµ@˜;åFÛtÚE‡ Ðl£õS“)®ÕšZꈳª¥ÔsŠAýû¬é0ÏJWb˜g%ÊJ»-âõl‰zfsNctv”ì¹dVhq®Ý©> -endobj -682 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 250.954475 101.433071 232.204475 ] -/BS << -/W 0 ->> -/Dest (cb181-1) ->> -endobj -683 0 obj -<< -/Filter /FlateDecode -/Length 4573 ->> -stream -xœí]Mܸ½÷¯ès€Õò£ø{1²A0’Cƒ=¶wŒxsÈßÏ£Z¤JTIÍî™±åÍzÐîVI•ÈÇÇ*ŠõYáïÿ"é!Æ<>ž>Ԝ¿1Åâ`Tÿ€Opg­( N'ÝùcØ0„äLðçÇþkyòÇÓßsú÷2>›!%o´;ÿü>«È!† œ JP)m•sCÆyAåõ€KLIRÁ ÈÐZ`¥è$Ì c¢ Š0ÃRÐv­J0ÃM’ -fèä‚ Ò -vh£”¨ƒ!ŠÈA§ÝàQé^Ò=ø˜Lu:Œ¤³vð´Ó’. Þ{%Á—”–t¶8M¤$l!ƒ¤ó°Åz«Dl11XI`‹Aí$Aa ÐElQ^iI—ìàR$u :C\ëŒrƒ‹Ö*I§õà€M+êУÐH>:[¼rIÔÁg•‘t¶þ$xÂÙ£Ð[MîÝVY-él1&¨ƒ-h|‰Œ‡-*x‰Ѓ†ÌK˜J†$0Ñ)J,`b(+Ñ€In Ÿ‚ÄVi茖ˆÀ*ØâÈKL`5l!ô\Q[l"‰ ¬-H qµ°Å•¸À‚m h‘¸À‚nI%-qß’Ò^â Â…™Jâ ƵѓÄ” (%‰ ,8ׂ;%.° ]t÷ qëZçÄ´k)z‰ ,x×’Vx×Zäu°–¸ì8X­ÄÞµZ‰ ¼‹nd$.@«À¡—¸XLŒJâHâ!@Ô…}ÐJ\@à]ãp ÒÈ Þ5N‰ ¼kÈx‰ ”ü£Do¼ -q²;[@°`¼‡ÎX‰ PØ`à–H\àÀ»:aøú4ØjÐ1y‰ xWG4Ÿ¨Kƒ¨q¡ß¢–í}’¸Àw1PY‰ xô%.ÇQļ«-FjIÞÕ&)‰ x~ƒ“¸p4‰ 0¾ Zº¢. *Å(qï‚–ŒÄ…Á$¢. ’–¸d<(ø¡xð.†÷$qï*g­Ä¼«n¨ƒ-6‰ à×@‡n+ôi8«ƒ2` -IÞU-,éÀ»J…$qïæaSâ²ÀD‰ `à/ØH\à# )/qO -:¸|¢Î8ve‰:ØB$.€c=$›¢ÄÁÀkŒÄÁÀ¸ À#’†÷-†°pã‚-ð%.“ 1I\œG<†Z•tÞ 1€®E]„N‰ B C±–¸±ÔÑé%.ÁÙ]’¸f@§Iâ‚` ºŠÄŠ†hà™K: [àdK\O:­%.ˆ¶À”¸ šˆ(Ó+‰ 0B#–DUK:R9ÎŒDòCv%.@7A¤ ¦u°4!qÚ'ÛNâ?6æˆ:ØbI\£Éáf”¸ FØb”‘¸ØÌñf¸ )ØO₤üÝd‰ @ãÐ)%qAÒñ¦!‰ Ð/s¼%.HV!ÞDˆ'ôÛd}Ž7ƒÄ‰`‹ƒ‹"ê` œ¦$ô[pâM°“¤ó°Å$’¸ yŸãÍ(qÜ Ä›ðèÅ)˜ãÍ N€w]3I:ð.E净13†|‘ Ý‚àÝטcNf7’]æÇ~þñôí ¾çÿsúð›q¦¬üi8cj‚…n`šÛÁûÖ¸|ëÑÐz…4ZïÉ“i6Rƒ§I·l$†¿üï¼þ±°†ÿ0žÞÖÓ'Œâãé³sa‹:cįqnY|c–g4X!8Âhë‘Æ&“Ù ƒ_©h‘øw¯OêdÅe´2ó1^"&ãί?ž¾ýó«¿ýåû£×Nÿø­²¾ûFãÛÙË7=LÇ~:ŽÓ±›¾ÓZ¦¦¼@Òø©eNé(4e‡9}9¯3SšË´4ÙᦲÓ}XÚhòù ³/,ËȺbsµQÏzz»¾¶Ež÷³^»é\Ý×úÝ?ϯ_~ÿz ®Ñã¼t8FS$º‚Ǿ,OÁãKÀáë o‚¾|תWËVûò«´Ë /Ýv•YJ9ï,Mm%≵oÅ@9nqÑ9hól¶Jï“šºržrµo ÚÚ1¶Ë­L)Ç -û¨ëËs8Ôí»¥µòx\ÇȦ63:ÌÃel«ãfÛzšµÒ„ˆœÇf4}Àï8¡ðÞüog”×Vë¼ùqß—¨¨mz{³ÜÒkvÜWÎΡ¹î\ƒÖÚ“¸2NgW‘rÀó}ªk8íËs8œîû—+vÔ¬Í&²´}•78–صõ)­1Òúd›£m‚õ°yíÜoú±¾S¿ms}ï.^Í–_h˜×«tæ9šÌ¿pg<[0R‹’Qšµ(ÛÚVÿ°,gŒ€ -‹%àr|X–ѲÞ*|ÔRVƒę̀[QÈ5_’÷˜ÚÓZ–|'DZKÛrýnÏÆÞUwæ9R÷½Â–ÓZ´en°z?Mnm‹–5ù3µŠñBK·­ÈÐý/lô.í¡Ò÷ÊŽ­˜ÛҢʱsÝËo ÑÝKwæ9¯xŠ_t¦ƒñH3*/t%_é5ͬÔêÊ9úWι‰Ø]lt_ ·áåÞ‰QÕÈeÓŒ½1NgžÃ!wßw<îüÝ>ªŸŒ*v--’Wæ~(-ë©E£4Ô¸«¹ý°šuèýíìsRˆƒ¶üµ{÷ý#r·Çåy‡Â}ßr\+‘Q÷fcV›¦ÚÌiü”Njq!†¿3³Y˜­ùÍÒÂeæý¹Ös\õJà|Úx'í\íV$Cl.²{dîËs8´]ñï¹ÇfTŽãƒ=/G>?YF¨c~`~.Ò_›êÌs4„ºûï§wñÉ­3ÜÏÈ?¿ÌÞ±XææÇéLÔÆ]Cl_žÃ!vß¼y;ß[tS‹Þå|®×ø¬Ï}Ý1ÿ°wýogžÃ!ò뽿~ØÞ²Qm îŽ'v:óqÏì4÷rÄÙ¾&èÍýk%7ï½UëUÏä ¿ Ø zLwC¹ü©ºÝç|æsëS'Ž^¬CmŸÜë¼GØÜdïÛ`:ó±û¾jû|­Ôº_yo„Q¸¤cö܃¼ûÑ„¼võø¡áŽ÷uæ9¿¼Ú> ¹BÆŽ¿ù%Pf㶭«Þxóýÿ DÎþa~}qÔDîÚ}óÎ<‡Cämï$WGä·†é#<¹ë«µw…îªkWoùµÏϬz’›¯á9ÖöÄ;ÖAvæ9jâ¯Ï†/Ù>þ¥{@ç=:FZ­îFʼñÿØ36o¿§Ó™çpH¼âŠ7ž0¢s_‘Ù""³}k`^\V*O~ïj³™Ò¥—åÈtÇ;‰:ó ™éÊ}o­6ŸÙîYëÝÝúÓ;#Ÿ"/kÚÓå{«åk”±ñ^ ~Ω­¾ ƒ÷*æ/>g¹õ4„­mñâúîXÙ™çpèë\ÙöéC®¡9–ŸÙz!Û+ß6PxÇ3Ûy‡Â—™ùM÷^6ùBí¿róY–¯ÙÝ@çÏnwæ9:Ÿà;~Q^yÆž³áç®î•4³Þµg›S}ÓŒÒ\G®}KôÓžÁ¡¹œË{E§#Û£ïj’+8[o˜Q¶8ç¿H~B‡+0äÍjm<õ£òþ.8¯þð§?þ•g|ùÞûËMºË+//à_pë/ǹAÆJÑÅé¾¼°béÖ—.íÆÆÎ ri,5¹oùûòÊ ¸ åŒ7ºrZùŒeM¿Çrb ¹§óåð:çÏD‡‹~Q¾›ÎÍìËpì\¹†ïÚ¦ütòã^EzܯNù8h“7´={ln›tÎû#EeI—¦ÓceëóFÎqàO5Õ´3°Ê›æ¢.Û³ƒ)ý§±Ñ?žt¢!צË.žjGSfñã)ïâJ‚˜ƒtŠ³pú©jY¤KÂ*{›†!µ)­Ôô3Õ­+?+W“ó»’-ï®Cª$²ãþ2— Bäôz-ÎW¯=‰óN¸q°Ó‘¯¦8_l†p.xÚ¹À9ãU±¶Š ÝZ¨Øéf±¡bÎ;§Õªó³°šÎ«N³æ/iM`͹'|8á´¶•>ŠÒ¼ÿf)ªÔ®QUf™Q©J]IY„j; ¢‡“ üF•¢ Yõ¬Õò.fa,@¢š.•t‚èá$¦›Jsµ¥BM笩ÔülÊ ukÅ ²±Ž&iÈ[{³>¢lù¹ûÜ¥­Ud¸1+áà C ­Ä'„Ônæ -å -ô¸Ð²r«8ª/³4?8ù…®Ydy¨üÿ…*\•fæhªÊû¸ÍGóIr߬âe%q¹¯¬CR h -sZV]:„ŠY^_:*!}&ÝšºÖØBʪŒÉC*æe’ªä±늺*ÎVê§r%ªbN7‹]…òãB<ÛéjWØËɼfpwµÜYì+w>rq¨-®ó¦|ÕÞZ.Ã|>Œµâ ýXN2‹ª£Ý|2&œ )Ìâ™FGqé­•Æ7ºO?çN&¸§ŸNã0g¬3 ?¼ÕyYîyéó‘æ>ÿ:Ñùû²©Ùe7ª²…©Üºó‘,ïH;Åy‡ò"¬Ý† 1ÚŸÅ:Îi]ACÇa|Msv5{I©+ƒêªåB7×!› -]£-«·#|hj±:2Om›yÀ¾Úrq8ÍÛá«pÀ/‡qÖÙyH>³"q”“ÎC:´dXFðBqÈN˜fËff@ò“ÔªÏöo -›ÏŘ‘í¬£y˜åEÙx, \`ËxÁ¤ùIÓÂÁ軻דÛ}2Ù—¥•Ú«ô‘Kë…ei%> -endobj -685 0 obj -<< -/Filter /FlateDecode -/Length 4204 ->> -stream -xœí\I¯$· ¾÷¯è³ËZH-@àC€$È b`ŒÆíy6‚7Æ9äï磺´T»ºg±=62ïM©…"©”TUîlñó¹ÃŸLÎä\R¤óåÍéíÉšÄÿj‰ÕMeßÄgg)v%f>¿BH&ö)žŸOø³¹‹?ŸþñÙéßë:îìM)Ñ;>ÿøZXLÊ)YVX,ë‚Âb6)yŽ -+:ƒ!–¢± 'ò´g%ˆA%³Æ‚ä}VXbJ.ìYbøäHcA W8),g!‡óÖª<b‰‚WxŽM„Ò£ÆóÎÄ\|Qy <—¼Æ ÁÄDŽÆ+&ÆÈYãdábÆcÈÂŽÈj<ÈB!'!KˆÁª<Èâs -/Aí…—! ¼+«<Èb£u¯Ã%©¼žÍ)ïyÞ²á‚ÕxΆo•‡#Ťð¿|súâ//þþׯ¾†1_>¾ùâÊcÅeÝwøEHv¯$£þòsž÷ãºòÂTnsÝÊÑkk9/×¼ü¿Üfv4‘âBûv´ÃK9žÊuYìõ—ÓRÖuèÕÔþR†—>ºœ~—!­Ëõ¾âºûMŸù¨þ—ÿ<¿|qúÃ˵cÕ¥ÆÕ±$óFz#AãЫò!¾øS¸`)G.(¦íꌳÛõÿ[SóE1•»mžÞÎÆ\tÙ´¿¡÷öÜÞ•z¿HeÆÈtÚòvn#n_¦~_-å¶ý.ò×vým·­¼œzÎën*G2Íéä”KuI¨ýôÁ:Ÿš£bY|ä©][åj±•ïhø–‹å¶À'´n=^{Q÷Ž8õ/@Ž W–²ß*ý$E¾Càý»iVNÞ»+÷´.SAØoúÝxuoÃO@Ýf|çÝðÊСœì;ÏöžW>Vç“óJ:ÆÏ­×m0´[f -+f(V»‰o~Ýno/èô¾=­e|Ìò8¶;!^f‹îÝÝãúJH,‡±“¥úí´Lp…ä´Õù\-õâþÓßæd‹GÏ.‰€ì}.Þˆ€,#¹–ÅÒóºaÈõ~G_Êv«…+M~¹ãȸ—º•V­%mÔ>\ÓÚí¾^î/Z›W­>¢FäØ“¿i£åÅ?{,_zkšK³ïNušöâÐPÓ\-³­–_ž|b¹¯£Y4ßÚµ@ZO³ÿOõìº|ÕâeÑD\TWcgÙX -ò`Á2ö°Œ‹%ôqÕfµÚwûvb]79ÙÝ·#²†šñûi¼ÕJ—Å;H¢ÙÆrº¼9ߒ׎1¯ôvYdpêäI€ˆÐ‹Òv•¹\ëWÙž–¶^Í3aòàÅÖuLm¶ÌÞ_¦1·:iiÑ÷ð³¥-ec¶Œ~j}?çzÍ›_é5ØíØ·3®Ú}yè Äi¼n雯3‘üñ–ßû<-6hrpk{’MÓ}½~ZÊÅÑfE7ÆÔlXe}í5_Z#ß]]ú­.U¿‹Ý'?é:‹f¹å -’ÖXV&¹æ¿—ëÛK‰¾5&~P´•6¸òø‡]EÜU®’G„üÅò••æF¤½™Ïl"rëo•ÜÉ&jÙ‹]eë,â4췞󄦕:‚¼ÖÌÇÍ/HÍ?B~±oç§É/؆[ò~p~±oûêáGqæ ¯øÕÅC½û­n‹5¿•øq¨Úéfg†OÜ‹;n¢ßàÑÀ—i| —<îD·cCùã TtËh«5´áûà~ãûu9mUÙeyXÇÑnuüóDƒHÎÐGˆûvVÞ«žŽä¨9ÿ4(¿oûÿ(ßõ^¶ºù-¢|µÍeÝ.ó‘^’Ûê…'äf§¬Ë{³Ú¼ÝbÕãQ£ëŒ‡®v¶6~Í~ò;ýU?y$íä>Ô_Øéo£·š‡í¥°iÏ/òÌs¸áÀö¬ y¡>l Ï«?tVð`÷zŠâ#p?vÁDfL9ýœ_¦HÛLoÇ0®Mí)d_p³Ã‘ùȳț¿Ùôc¥²ÐÛõ¡Ž¸IJ»üôåÚ”ëü1Þ’êN}Ž¦0l§Õ?Ò\M$¶G÷û ò€|q!iñ¡'îô““ òF’6®»Z bkª*¾—VÉ’I!ûÀ?V){̲LùǾÉeÒòa_Û# ›2Ë£®ÀŸ¨úH¢Áß®“„U‚n%]û¤¦¢ª¼^°é´jôøålqðÀ*þÞñýƒu>‘ƒÒrüví«9Ä­mÞmφµãóíçj™±9ˆlK†éñ€íq}?êŸTÙ¿óöq€Íè|„¿­üÕ£Úý¯fܪÿËÍå}gÇîûX•OÄ{‡ÔXxï;{Äã&{ XG§dûÏõÑ‘ ÁætnÞ`˜Ñ‹L)œ“Ût¸A¯¯—g£"³¼ë‘ÙåéÑånó»EîØxÿ s{bôì6¼®•W±Mòö¢‡)K’çòo©Î¿¾·dê^2àË8³“p4?½SË4•O«¼¾ -ZVx-c¯ê_2ùºZ\žò}å=—hw癵¿å\²®ä¦•Toÿ`ÕÙÇÀm ;S¾=ÅúŽ‰«ïÚ˜óò!‚s$#Ç}IÎzåý”@®™ÎUe»óšõó o{©å‹V^ò”|®Ÿ{˜n–òo«Ñßœê“ö:¯o_ÛJFÁ[òóIÞ¾'…L4örî…¸\ÚÞ¹¾5ÔhY2e[AÎ.—cÿH"ßõÒsÔçVMÞŠ Û -…ú^Àµc…ˆšÑíÉ2zkÜB–/d–»ØE‘ ºGƒ4— ŽŠwÉ.؉È{¢ºdOM'o¼uÕÅAì¢Ïªs“ù[YŸ&s/'t¶Ôg•*ïM·¦šv½í´0 U:•[ÉnêvRH—“Büܶ¦ ÆìüÉjòöÙ•˜›#Q/WZ9…t9©å–Ö¸[*õRyT-MóC”áê!wÅ+´Ë´Çšä“,Ó±¡]®É²EØmmóä7~G¼œbhG~>qRJóÀ -ä!eO S»œíð—AM¹© -S³ÑäÝù{… -îTAžzᨻ”¼7îF'27;y­¤™;ê¦Gi”ÔåRê>;ëËe«”Ðí¥»ÆVÔIe=•&ž€T5Ùu¯ëd°¦ŸŽ•3Ñ6!ÐÝ swåçyÈÉ}ª²íE7¹;÷v9vì|žÉ©[ÜÉË”]ÞÞîäóx©¶dqýÜ:do{´MÄ!@aŒ>Wr›­ä1írL2-â^N?œj˜ó½l“ÅàäÛsæåÎÿE™øý׉Î_µ—Ñ®qÜÛŽ¾cëŠm&¢§ öÙåQ–›7ô8îý„×4ªÛì­¤ëê:w&òÐáDöÝu½ “fø€xÙh±'2j›°¯·¡ N6üä5Æ&næáà6^!ù<5‰;):B:¸ä§Š¸™ÅíÔa’­Ä‡œ;éªùi6E¾¦›)Û0x4ÂìÜt£ÕQLE‡ùfn˜ÜÜiYÉ£ýÞÓëƒí¾ˆÛ j‡öN}ž©}`B퀧SC½ëÛsBÝ)k“wh¨·¦¹z’3¦ÆH-hu3ûèDÍ£ÚR€çN©ÑV¶@Ñá4Ûh½ÅÔdŠKµ¦–:â¬j)õœbP˜5ͳUºy¶JÜ••v[ÄëÙRè ÏîœÆèì(ÙsɬÐâ\»Si” ¶ï›Ö}·h{ž†AC82S;ªñ®ä{áßûÌkÍN„VªýÔÝ—ÇëÛÔäRð¡~‘Ïæ!$UI”§í’w®s¼²ß%iû'sO0‡¡lròmE]„g0Ê}ÆÔTûî¢^­XuH呂›ïÁï²;T?®U7 -ë&Q¿M²T”oéIz³|(Y§¿ "Û@²í’V[(õçhÉ— -endstream -endobj -686 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 685 0 R -/Resources 4 0 R -/Annots [ 687 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -687 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 244.773057 101.433071 226.023057 ] -/BS << -/W 0 ->> -/Dest (cb184-1) ->> -endobj -688 0 obj -<< -/Filter /FlateDecode -/Length 4639 ->> -stream -xœí]ˮ丑ÝçWäÚ@Ó|?€<7Æ7P°†UY]m nÐö¿ïCI$CTHÉÌ{«KåéºÈJ)‚”Bäá‰`HJª«Äß7 -ÿE«DŒ)x{½ýxùé"Epÿ¦«Iý>Á]•´A8•|t׳À’ÓÁ__.ø¯Û¥Å_.þÕåÿÖuÔU‹”¼Vîú÷ï³ÊbÒ1ª•T†Q9'BÐÎ3*¯.1%N3\°ÚnUfا‚VëȨ"Ì06(³U%˜¡ƒ²œ -f¨ä£Rv(-%«ƒ!ÒZ£r£Ñ=§ÓJø˜tbu:4§3Fø`•Sœ. ｋœÎ—¤ât¶8e­ät°Åš8‡-ÆÉê`‹ŽÁpº[4Z'1º[€®Èê`‹ôRqºd„KÑZV— “1Ä­NK'\4Fr:¥„6 «ÃˆB'ùÀè4lñÒ%V[œ‘šÓØbñÇéÀ΄™Ñªóè6Ò(Nç`‹ÖÁ²:Ø‚Îç@{Ø"ƒç(#Hd^â8@‡ lÒ–#°ÑFŽtL†`8ÐÉ ëSàxÀHV [œõ[,F.«ƒ-&YŽ Œ†-(Àq1°E[ÃqÛZ …㺵2)Ž øÖJå9.0 \˜)9.0`\½å¸À€r¥Äqçp'Ǥ‹á8.0`]ã¼æ¸À€vžãÞ5VIŽ ,x×Ôdu°æ¸ì(ŒŠ†ã Þ5JŽ ,xÃHs\€^À¡ç¸X:FÉqïê(-Ǽ«\« cÐp\`Á»Úá8Ï ÞÕNjŽ ,xW[í9.PøG²Ñxâøp¶€`9.€¿‡NŽ p0¡–p\àÀ»*Áý1cl%TLžãÞUÝÇê’P-ÎŒ[´²PÞ'Ž xŽÊp\àÀ» ÏÈq8^À£jŽ xWxjNÞU:IŽ xqƒã¸p -ÇðoBI@—Õ%!SŒ8ð.hIs\/,d“°º $ÅqÈXHÄ¡xð.Ü{â¸Àƒw¥3†ãÞ•a«ƒ-&jŽ ×@‡aËŒi«Bj0§ïJ…ætà])Câ¸Àƒw³Ûä¸À‡,Бã(kŽ |´"…à9.ðIB‡Õy}ÇqA8P«ƒ-6XŽ X‹dRä¸ hØb´æ¸ hØ‚ãD")Dßìô¶.ìüÀÂDŠ€ÉDLÖr\œÇ| ­Êé¼1€®Y]„NŽ B°®Xq\€¹”ˆô€àE—8.€Ð)ËqAH°C…ã¸"5"sN§` ‚lŽ éB§ÇQÃDƒD1Ëô’ãxhÌ%ÑÔœÎÊ<ÏŒDëE9.À0ÁLLÉê` h‚ãôOžl;Ž 0ùÁdÓV[Œ³Ĩót3r\#lÑRs\læùfภIØ‚ˆã‚$½Èa2Ç qè¤ä¸ ©ˆù¦¶`\æùfä¸ ‰ù&¦x̸MÆçùfภYØ⢰:Ø‚ )1㜄ù&؉ÓyØ¢“å¸ yŸç›‘ㄘo"¢gS1Ï7›.ïºfâtà]D¦žãì`¾‰È–›ÛK¯ ˆÏxeÌ3ÎÈf $¨×Á·²) î…· -lÎ@‚|ó‰MÀLÌ,£g³p«P*ɦ $øñ˜ÝpÂœ}¢)*8øÌòE& ‹è¾Î9Z1³SlÎýý‡Ë¯ßÃù^øÇåÓ¯¦LYùS¦Òñ†Nèn‡è[áòGG+1 íd! |¦N6"¤f¤-.I·l$Ü_þwÝn¬,‡áßM§7åôˆ(€ÒéôàZa - GL_§Ü ±øÁ*oh°ÄäÞÖ£ŒI:³ ‡¸RÚUáß¾»¨‹s´î*ãŠùàTL†­v×w?^~ýßßþéþð`ôîÓå/ÿ!¥¿ùFáÛ…ùÛúå{Ù—fþF`.•Y¾oŒÜÏŸ*7d6Á™Kõq)ÿ¡é´^×™>ùíZuܲmíb¹Žb\ÛÿÐuf{Ýü}÷:Ëö_¯ï¾½üîÝ:ˆ9*tìÚƪ¼mo²j58óe¯F€]×îM9ŠÞéåB noK½÷K9*û´>W6sB^"¨s ‘µ|©Ot¥\­§ˆÍÈõpu¾oú •ú7,ÖOVääXSvC‡`¬s:´éCNë9 ´äÔsLKÔл‡°©Ìª -« ºûæø'÷òûy{#_>+nòõ\k—í|üDFÚ÷[KûT4Ê$šÊa9ƒ†¹¡•þÇ꜉î÷êx/­yã½ÔB7\Öy¼ŠÞÛQ¹l9oAÛ†£ -‚oãü·72Þžç¹Ö$ÒØmÐÖ9㼇 æïÚúr]ç5ˆ,Ǩv¼_#µ+4ÔœÇó>ê[vPIb¼aO=Vçl¨ÔÇq!‡GødYŠ?ÖgAªìõ¡mç2+¾ôÝÌåsÛÑ¡½"xñ2Ò&û,±Åªn¢uZHg}ÆÕ!V뜫ÇQåçî»Ê,å8;,Yf&\l:ˆÒz?ÎÆ}†Í6eèuÚ®-_§\cã;~[Âv¹—mÊɸcÔÕ9êŽ#ÈÒ[Ù»~öë:ômömÕoö½§H/-ˆÈuLFÓ'lÇ…ÏÖÿ@æ4~4Z·"¿ËÏô£«¢¶Aîýr˨9ˆ`);„æ¶sZëH¢6ò8m±¢Í嘹‡Ó±:§Ãéq|¹aGµžOø•]_v8æصmÚbd/ç·±©›…°y4ndÆŸ%c§~›îú>ÎQÍ^\¨ŸHÖ9šÌ¸ðÀŸ­©GI(Ez”ú¶¾×?­3Í€ -‹%2Ç¥(ø´>FÏz›Ù£–cuˆÎŒº7 ¹KÒSGZÏ’™™Ömm[nß=ÿlÔãóêÁ:§Cê\cÇi=Ú¦”›:.“{Û guþ,½¢=ÓÓ}/t¿& -›¢K3#”ûÞ˦oF$µ¥G•#çzj.¿ƒDóø\z°Îéx'Rü¢™Â#W^éJ½2jº¬ÔæÊ9ocÈ+çÜEl.âÝWèíxùu÷†vKòŒ£sœÁ:§Cî{ЧÍߣúÕ¨"×Ò#yƒ`‡Úu;õhä|Awuw 6Y‡ïßbN¢PÆïþÎ`³!×Ç©uFÑÅ‚ÓL“xiêiïE9ØxêÞ“÷YûƒnŠ.Ìö|Æ“˜Ø¯íßÄ‹$ ±:ÖÁ]„C÷¨'ÙgâQ«çÒÁ:§Cäxtï~3õ^{3ÂQG÷´¿Þû1[äm²™ú3#•ÄžJ‹$£ÎO6#u¬ÎézçÞøÈ39Yö‘ð åµ=~Ù{bƒñÀ_Çó>oµì ’ä$“nšœßCåXÓ¡òuÏC>Üo÷ zgVôu |™ñ ¾«s6dº;±æ[Î\xºmhvóÖˆ&蘞D9 ‚y”:rÇ{tF4Xçt(½þ\sïÏàuÏ ÷‘k|ëûGÎ<>/¬s:Ç–õ þ÷;Ymú´«_Êq=ÎÌáïΙIf/¿Yz¸dÞßêyŽ»‘@”O»è¤ÏÕîÍdÉEŽzæÁ:§CÛ˜ñ™;p$£ržìm9úç‹“wJâÀü A¸H/+4XçlõÏßOâ“G3ÜoÈ?ÿž£ƒG¬'1a~_]G¥ÝÝׇꜱÇqäÃÈ;|O… àob<4Ëù¹F\³¾õýuOâÃÑç뜑_ïýõÓŽ–ÃÕâžxgg°Îé7øÎNw/‡Í†Ðg‚Þ?ÿ¬äîý£rûÆE¯{ºâ­#€T’û×£òÁ:gCe¸“‡<+§̆ý€wÿ,£¤ãÊ=~ O¼ç=XçtH¼#ž†3þÍGÅ^<<ñ\æ`Ó¡t0n,-håæ “Í³ãe›§z¹ÌÉÀ[3›'¤<|3‡f§«}7ÆÖî™Nõñõ¨ÛðígoÃûÇçãƒuN‡â;9Î/úí×Ë×Ï wOä3ëœ ñN>óQ~|Å? ½ Ø¡z*÷Àqé[u‡ïùÞÚ¹§çSŽ^=‡Ú¿¹7x(’Üäè/Á Ö9bcÕþýZ®w¿òÞ3^¸”#ö<ƒ¼çÑ„ºvÄ¡ñ‰ß"¬s:~ù8tïWøjOÄ›_e&îÛºßÿßAd‹óúQYëîÝ7¬s:D>ö›DìÓùgvÃòaÞ‰<ŒÕú;ŒL†»êú'Œ÷âÚ÷g6#ɵkx‹g{âÏAÖ9jÒ/ٿþ¥GÀ`ž2‘œãèle°ÎéûKžr5²îk•Ï`>‡ö„šÇoc°Îé:ö;•rAÿöè4›Prs7’çÿ#c‰þñ{:ƒuN‡Ä;1á©xãÆŠÄ™ý¯槓˓ÊKÜ»yŠY/åÒgæÈ'~“h°ÎÙ™[öšJî¾´=ò°÷p÷/?¹úyy¨=Íß{]_§;?¬AÏ9Aµ×xÐaEfM“–{¯CpÓµ=blðûnYm"8ô”syµŸèT$‹WTÌÝ-rbÛ¥.Ê‚×ü×cÈ/àñˆD^fÖÄ«’RÈ+³dÈ|ûŸ¿ÿ¯?ÈL¿êóýœý›»iþeŸé—óü¼ŸGå4’UÍó/ÙXüÖ2ƒœböÜ•s·HS²†å]þܬ³O2h¹¬Ÿ?Ó±–íé8±øòå|Ùoçúù]«0ëWÇw˹‰=Ó1¹W®aÓ•?]ü´ÊšVš“> -¥óR´Wo…É}“®ye£(U¥ëÔÔØêºSsZÀ÷§ZjYÓWæeþòOó‚¿dg)ÿÓÔé?^T²"·f€yýM9‰£.;MürÉë¯ZFlAnƒâ,\6e=–U¥`•Ý@¤A¤¾¤qB.›©<¯ñ1ojW®&×w¥Z^ÇÊRÈL+ÃÌ'f„¨éÕVœ¯^ -µˆó¶Q˜eÏWSœ/6CØhiÙvÀVñ®XI„n+”ätM¬m±Gå5ÏjÓù&¬¦Ó¦S¤ûKYHw oœÖôÒVšWÎ,‡*­«e•bTªRWJÖ±µŸÑí¿‘åÐYõ¤×òúc³0 ÙZ.•rŒèvaË-Gsµ§B-[ÕTZ¾™Ò nbmxF6µÑ" yQn2F¤)›k±Ï=PúZF‚½Þ.p1v#~¹¸À”v+¤+УBCŽ[ÅQ6¼4i~#oÞÂÐ,²¼zSþ¦ -W¥™y¦ e+¤ò -lm¯$Í*^7•ûÊ:–keC+KšK…P1KÛKEɔϤ[K×[II“yHżLR•<ÖbUQWÅ™ÂJûT®¤BYŒÀéšØU(¿¬ÄÍNW‡*À^N滫Çmb_¹ó…ŠCíq•—Ó«öÖãÌámrl¶gèÇr’&Ö²z»v2"l†š¸ÑèË$.£µ²S^"¼l¶AÆyÜÛåo—ÉÍiãt†7*¯K#/uý'Ê|‹Ïÿ^ìõe9²ÙkYÙBWn]‰©'Ëk‰ÇAq^[¼ë°!Bm í7±Š­¬+h¨~\kÂ׶U—ØKIUTU-ºÖ†D¬+tµ2¤eÜðÖµb d^Û7ÍaÏ»¦\NcòBvÅ*ìÐËÁnl:Ó\ò•{¹hséÐZM*b‡»ä„©Y¶23’ž¤6}¶ßÒ®0ù\d‡„y×4mn–ºÈ¦« EºCl=iZÙÃýôðzu¿/&û2²´R{•¾Pi½°,­„ÇKÍ´ë×»Õç˜kÞ)ƒ·µP=–!cÏÔ § ZØÕÅ(‘ÆVm)àèIm‘­NZE¥ Èd‘Õ#†b“_ª•f™®8²­jLѤ£-íh¯ÔFt´Wü¦l>nñx5Z25rΡ]l%k,™§µ«Ô¶’¦È¶ç¶ëso{%—a›!¾E¦²Us›’Oñß3cg®Óe"¸RåoÊŽ¨Øð„£¶*m®)‡;Ñ{pˆÍËjIºäá:Çù“m–¤äOè™ÐÂ#ƒÞvLx"ÝWC½,kŸòÕ’d/é±`ÒCæä&;d„Yò”S’¨î†> -endobj -690 0 obj -<< -/Filter /FlateDecode -/Length 4953 ->> -stream -xœÕ]ÉŽ$9r½ÇWÄy€öá¾BH‚A ¤CC‡ª¨ªÙztÐïëÝIšÓ͗Ȫb*‘áf\ÌÆÅ=õ]áçÿ’ÓSJ9wüzûí¦¦èþ•«‹¢þ¿Ñßµrqò:‡äï¿’ÀÆ)fob¸¿ÝðßpÉ“¿Ýþã·ÿ^çÑw3åŒö÷¿~!•³SL1*/¨2TJ[Aåý£ñAP=ás–T0ÃGgÜVa†ËÉK*˜áŒI‚*Á 뢶[U†&j'©`†Î> -*­`‡6J‰:¢œ³FÐi?8=H:£§²É¢.B§£‘tÖN!:íµ¤ËSÁ'Iç`‹ÏJK:[¼vNI:ØâlŠ’.À¬u°Å¤h%]„-ÞÉ‚.Á +‰:Ø¢‚Ò’.ÛÉç䜨ËЩÓVg”Ÿ|²VI:­'lZQ‡…F -QÐؔϢ¶x«Œ¤³°ÅáGÒ'¼) ½ÕPï¶ÊjIça‹1щ:Ø‚Æ—ÀØ¢b(=h"^’8ÀÄ8¹lœD&ÙÉ%—$0)O.F+Ñ€É~r!G‰¬ÒÐ-U°Å» 1Õ°Å¡çŠ:Øb³“¸ÀØ‚X [Œ³X°­Z$.° [§²–¸À‚oÒAâ Â…™Jâ Ƶ)8‰ ,(PÊXp®wJ\`AºèîQâ Öµ>‰ ,h׺$.°à]ë´’¸Àw­ENQ[`‰ ÀŽ“ÕÉJ\àÀ»V«(q Ъp$.V&“’’¸ÀwMRNâÞ5C€¨‹ú •¸ÀwÇ-H#7x×xe$.pà]ãL¸@™À?JŒÀ» /‡°+qÆ{茕¸…Ma‰Ä¼«3†?¡Oƒ­&r¸ÀƒwuBó‰º<é ý^žtYâÞÅ@e%.ðà]Ðg’¸?aD5x𮶩%xW›¬$.ðà]Ä ^âÀaÒ$.Àø6i芺<©œ’ļ Z2`žT“ˆº8HZâñ¤‡J\À»Þ³Ä¼«¼µð®r{Dl±ÉH\€¸:t[¡O#X”SH:ð®ÒhaIÞU*f‰ x—†M‰ B$IÀÀ Q°‘¸ $7åƒÄ!+èò‰º0áÚK\”%ê`‹‹NâÖS¶9I\ l±ÆH\ lAÈ q"‚)kDßâô¶.âüÀÁDŠ€É¦”“¸ ú€ù¼*é‚™R]‹ºŽÄè& ÅZâÌ¥¦„N/q~¢pIâ˜vÄ [ÐU$.ÀP4%ƒÈ\ÒiØ‚ [âDºÐi-qA2°Ñ ÄÉ$Ì2ƒ’¸#4æ’pµ¤sŠæ™Iâ‚äÂD¡¢Äè&˜i‚)ElMH\€ö¡É¶—¸“L61Íu°Åz'qAJ†¦›Iâ‚”`‹QFâ`“æ›Q€` ">‰ ² -…ɀơSJ€æ›ÆI\€~IóÍ$qA¶ -óMLñ„~›m ùf”¸ ;Ø⢈:Ø‚ ) ýœ„ù&ØIÒØb²“¸ ‡@óÍ$q Ì7Ñ‹K‰æ›Q\.ïú f’tà]D¦Aâ\`¾‰ÈVšÛ+¯ˆÏde¢gW ¨×cl— ¸£U× È—æHâ¢ÌÄÌ2qÕÃ*”Z‰Ë -ü‹xÌm8a^}âKTpˆ3S¤›ÌèÑ}›sôdv'Ù¼>ö×?ßþøƒïýÏÿsûú‡²RV4„’Ú!¾B70Íí}kܾ²c,¦Á²§Sµ1w³4ˆpYf#³0àÑ¿ûöËÊV˜JÆ"7fBfÿ˜\#ðš+÷”Ò¦Á˜%»­Oç959"&È÷öE˶2±z»“$[ŸÎóýmÅ$“f„˜ëze#OßÆïõbZ ԋ鼃> {5Ów6Ú6gejjPGA  Cëu´4ô3“ŸÌò V˜À#" Hc³¡´ˆ¹r«Äÿá2]¬˜×éÛ%31Æà›?üzûã¿üôïÿú§ŸÀ_o¿üR.üøƒ¦Ï4z¿|Æù ¦†c÷XÒ}]§7ÐÓŠ³§4n]É[úšŸéjº–O3{>mmZåùÒõÚ/º½{É;eÅõ=Õü§e=˜½ÁŸÖ÷ØìS?þçýÃO·ø°V™ÍäzÌ*ˆ±x-Ë·`ñ÷€`·¬Ío×Pyª9tF+Û¡L¿»Í®º fV§[—Û›w Gÿø[ê624i·¨’^ö´£ ô›ó¼81c?Bçë’Êwì9zÕÐ ¼>Çž§jš³¡M5ÙvtnÐå廌AÛÉN¹©Àéƒ×ò¼Ý!C¿%ü~Ư}\}h ó§]~?ËßkºS\ ²‘a6ø`évÙv`9Âx+?®qÛìô͆=fët ¯ÃýÈùw˜°‡ž–ñR*KRÇ(¼–çåPx*Rë6†¡åÕúslmÎ*­µô~ µr†kãá8?†×[4µzó2þ¦­nƒ=Œã•™Çzýz\ßCnÑ}cïÛ³YpH‡60ÀÓúÌ1R¯åy9¤žD”Õ[yn±M+ŸxxÅÔr#÷m"*†¢q<.õ»yÌ-c¢Z"¶±ž(ØwÈ]f½’¡w“îë:Máa3Ô; º•Áâ›Öã›NF¥aQ¡¦ígÌžÕ */æy5Tš“HrDÝÀ¡­e†ÑÏx«íò›Y—Ûʳ²ühžr­å¿ñÞNFù¡Šènˆk «ÔRŒb-Õ.Ù¤¡C_Új“JKýôÿüOÿÆã-ϱ˜½õÅÐÓÌi}¤~]9d¾&yŒåñRíoø–rL˜¿SÙ¥ÌGm-òR/»¥w¼u» }”]êŠuD^®’XþÐmqÕþ¼±‹÷/¹;qwÿ²”Z, ½tõ‰>˜×ô€!G8BºÇRsXª ¼.Kg³¥óƒK]vi¨¬¹ ª[Þ–,m®Ñ®m9Å;K+–²Â¡tæP¶Cu¯––t½u½™Y[°òPuÈBÙd›ý2»qöÝr¯Õg¶#¥ØNßÕ‚š<×KéŠoȆÉaR»ÑgÉ¿ø·Þ_±!Ê÷W{CÙj9¡÷¢ŽìC§Ñµ‡¹8øxè-¼ÎgzX¹NÝ/å~„^åk¹qIoyšó\õ¯ç6rÛÒÌP¥íY9Å¿_{yÕà×_Ò¶µ//Ùf—¶g¶úZo8ð爅šæk¯¯¯Í÷®+F)]]òõ Gù€ë¿,¾ªePŸø¸ÆhIËî¥Îw|e6¾bý®Ý3ëkô9ukRûùÊŠõUæ›±Z]µ'†'Ýïa…YÖÏ86E Ö>8rA-Ë3{ÓÚ­ýæF¿õU‚ßi}_s[tìôuÕŽsÜV1uš¥÷~ÑùvŒ^éô_°e»Šd^Š^/æyצáw [}³cvòð”ŸNãr¸®]† s#ìV˜Ã¤2¶*üeaÛDgµ8¸[®±yòQ(×ýñ$·Kå„j{Vä÷aÊm'å?ò\ àÇ ù¼>K'@1/ -G^¼´lvROŠ“¥#÷Ò}zÕR[»rç]^uÊMÑ&cýÿW]2èeÉ!´:ƦÏÌˇu“‘›À@ø'ˆÓCdøi= ŽŒÜnƒG:ÕŸéüìPiñèù¥P‰ÌГ)t–ÏÈïZž™º7„÷Ž8P}Ü.ho¿ÃÐ놛ÕÂc²£é7[¼“ÚB_F‡6Û™›É+#>…Ýàôv.kì— „cw î -f“’‚}Cû9Ù§¨‡ -pKÇ— z®m]Ìór¸Mú·—7PÒ²IDŸA†;¶™T4ã²ÑDéÙÑ{6§x\ÑFwpûôFQZìõóçé}Öï2β阹¸Ay1Ï‹à¬"BŸ8Š¿nçÙ¿ËÛ6ÃG2¢î uu#òbž—C›=dµgd´m .;[t6×PõôQ‰ºñqØ€¬òåwÅ®I};Ûºå;•Ï£Øwœ[‹®q=Åç -gH¼–çåèyï{ÀÙpÙ0æm6åX<Ù⻺-4rTEðGÒŽ¶®¿÷‰Íë÷ºƒDµ]/æy9$GzO!¡-:.2µÎó-ˆ”6*9R77>½ÒÈûìزƒJã]©¯åy5T¦ã¸ðhÛú©þ¯å²~¤Ž‡?Ø ~h­ 6 s—ßÛŽí él¯=­Øç˜}òô!:o&å] \B¿–çå°zUþÞmט¥–óyÀ’æÂÕ¦ƒ(mÇÅø`Ì#°ÙÞ!Iî“æ‹(çik`íÀÓÞ“Œí¨•]6gGÕ.æy9ÔGm5ñÁÆÈÁ›„ó`«tÒø¤Y+-ˆ <–Ðô•­%¿7ÿ'6§V:yÞñàæ36Ô=ÈÜA.c'‚åì\J¾Û[±]Ù(ã´ÇŠŽžjMô2š3œ^Ëór8=Ž/7ìÈW—x²  ±ã›½!ty‹‘½U¿MÃ,ì -›‹G@…þçXßÙ¨«÷÷yŽjöâÂôŽõ‹y^ Mù$.<ÏVŒ4¢dD”f-ÊǶ±Õ‡ã‡š/ŒÂË Cž“ýÕƒ4c<0ì÷´Ö×ýYÈY,É{Ìæø<-.‘ߟ³~~^}1ÏË!õd­qà´meÉM§¡Ö¦ý0C¿K«˜ ´´°«ý=¢°]Ú¡ÒçÞjúá3Á'@¹Œêä²uÆ«sœ‹y^¹'»Ð/»~wŒêoF»—³‡ÒWq¨[ûiD£4´y×°±Yu¸2ú÷˜ÓÅ4iëè|Ü r¯åy5ä Ǥkxð©&¦ùP{ÌGc†f‡òqÙþ ˆ+á…ÝÖg Š7O« å±eˆUYÛ‡aÏ–7.¤?/oFŠ­ç=½™.yØ‹–O“œÀ®vÚ¾8殇—õ¬ÏC¡Hz%ºMwtåHoÛ; e¾Ì«0z9ªM`Ya ó518°¤©§¨Øñövü{9Ú^ŽÊ«ùÓ.G»ËÑèåCÓWQý¤`P›Ç4J}´>Oùiwˆ!oå·o÷àë=lšò·[(oÄÓå­¨*€' ½6ýÜDO0Ä|§·ð%e®M§‹³éø—˜³¼lþ·–jyÿ¼¢WÒÒëç—Ó³‹%ýo¥Ñ½éì&òy^©¼+Zq2õ¢‹ßnô®p'ˆë=ˆI¸|U­,§Û3TUö¹Æ))­ŸÔòµ?hEÇÁæ¯Æ×»¡ü¾f£w¸9UÙò³¹bAˆœAoÅt÷jÒ‹˜Þ·ž&»\…f -‡nÂ^ ãi{=ã©X[Å„~+T¬º.6®Ú£éýœÍu¡ ›éÜuš5Mk"kÎ#áã†jí(}¥ô–çZTõ®QMf™Q¹I}MÙĵvD› üAÕ¢³éY«Ñ»2gaª@r-]®éÑã&¦[Jó­¥bK•zÖ\=ßMéP·©9^=ØÈ‘þ€ë#ÊÖ¯k1=W×ÚZ%†³>nbÜFüvóQHí;W(_¡Ç…–•ÛÄIu¼t)ETó7tÍ*£7 Òÿ3Uø&%æ)_´k¢·…ö«^ õÍ&^;‰ËCc'y@»ØÓ2wéf¹¿tRBz"Ý–ºyl%e.cò˜«yDR<ÖbÝP×ÄDaÕ?+¹PU#P]ûå·•¸Ûé[WõªU4ƒ»oåvqhÜùÆűµ¸¦W¿6{[¹ óUø(›ÅýT+éb£Úh×+cÂnH¡‹;¾qí­Ê[–¯½“I#îãö—[æŒõ†žy -VÓ›Ìyä¥ïÿ‹4?á÷¿nîþ§úêÌy7ª±…iܺó‘ŒþîEº(¦¿ƒQ…­Û0¡qŒö»X§žÖW4´qÜÆ×®gWØkJÝT7-úîC&6 ºF[æ | ^lÌ·¶M°çK[oŽžÞ¢—®V«pÁo—©ël’ï¬H\QÒ>¤Cë ˈ ^(.Y…¹[¶2“É+i®'ûo -Ku± FÐ¥í:ׇY^t••»`I[~Á všWšWöF¿»{}s»/&‡ÚHÚ¨½I߸´ÝIáÉR[.Ãú²9¶<öT;o÷P+˲¾g[Ó»F-Üê‚c”IS϶$ð¼RWe«Jë@Ñè2Ue­ÄXm -K¶ê–rÇIôRl1E—þ…{ÚóViNô¼UÂ&-•[G¼-ÙyçØïNõ”-–L‚,ðÜMêzJ[eۺݺî:ÚÞÙm¸nHè‘©êÙü&å»øï=}gÎ3¬DH©êOYÑ©¿lµÓÑ[þ~˜J!€Cý ˆè[.y:ÏñúÉv•¤®ŸðšÐ“³VEM N6á Š|®`EÕ¿'gËJ¼¥ç ,ušS›Õ¡ò§€ÊÚeY$j—‘¦Šô—¿,½‰þ`ITZ~ÿL= -¶^B)?ÿ1ü‡Ò -endstream -endobj -691 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 690 0 R -/Resources 4 0 R -/Annots [ 692 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -692 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 465.123057 101.433071 446.373057 ] -/BS << -/W 0 ->> -/Dest (cb187-1) ->> -endobj -693 0 obj -<< -/Filter /FlateDecode -/Length 4260 ->> -stream -xœí]K䶾÷¯èsË|_@àC€$È"²À"99ŒÇ;6‚Ùëò÷óQ-’%ªÔR÷ììÊvv0;­*>JäÇÅ›Òg…Ÿ¯4þ‹¤‡Sðt~üpúxRCp -ÿƳ‹Qý¿Áµ¢08|tçY`Ã’3ÁŸŸOø¯»äÉŸOÿÍéßó<úl†”¼ÑîüÓû¬";„‚r‚*A¥´TÎ !ç•×n1%I3\ CKU€”¢“T0ƒŒ‰‚* KAÛ¥*Á 4I*˜¡“ ‚J+Ø¡R¢†("kvƒG£{Igôàc2IÔèt0’ÎÚÁÒNKº4xï]”t[\RZÒ9Øâ4‘’t°…l ’ÎÃë­u°ÅÄ`%]€-­“]„-@Wu°Ey¥%]²ƒK‘HÔ%èT q©3Ê .Z«$Öƒ6­¨ÃˆB'ù è lñÊ%Q[œUFÒYØBø‘tà gCŒÂh5yt[eµ¤s°Å˜@¢¶ ó%0¶¨à% -À2/I`B(’HÀD;P¤(±€‰i ¬D&¹| -X¥¡3Z"«`‹#/1Õ°…0rEl±‰$.°¶ ÄÖÂCVâ ¶% Eâ º%•´Ä|KJ{‰ ,f*‰ ,×FOXP. ”$.°à\ À‚t1܃Ĭk7XЮ¥è%.°à]KZI\@à]k‘SÔÁXâ°ã`u´x×j$. ð.†‘‘¸½:‡^â`e01*‰ ¼k¢"‰ ¼k¦QŒA+qwÃ-H37x×8e$. ð®!ã%.Pð½ð.(ÄÉîlÁJ\€ù:c%.@aƒ["qïê„éOÓ`«AÇä%.pà]Ñ}¢. : Å…q‹V´÷IâÞÅDe%.pà]Ðg”¸?`F58𮶘©%xW›¤$.pà]ø NâÀaÐp$.Àü6h芺4¨£Ä¼ Z2`T“ˆº0HZâñ à‡J\àÁ»˜Þ“ļ«œµxð®"¸=¢¶Øh$.€_†­0¦á¬Ê€)$xWiô°¤ï*’ļ›§M‰ |È%.€¼`#q4¤¼Ä>)èàò‰:?àÚI\”%ê` ’¸ŽõlŠ[¬1[à2H\`HÞ·¸<€-€‹¸> ØOQâ0Ù‘ÄÁy¬ÇЪ’Λ!е¨‹Ðé qA4`*Ö`-5D z‰ @ðCv—$.€Ði’¸ $Ø‚¡"q¦¢!xæ’NÃ8ÙÀÓ…Nk‰ ¢-ð%.ˆ&b•é•Ä˜¡±–DSK:Ry%.ˆä‡ì*J\€a‚•&˜RÔÁЄÄ蟼Øv`ñƒÅ&–9¢¶XGÄhòr3J\#l1ÊH\læõf¸ )ØO₤üÝd‰ @ãÐ)%qAÒëMC`\æõf”¸ Y…õ&–x¸MÖçõf¸ lqpQDlÓ”„q NÂzì$éöÓ§¯0ùžøÏéé7c¤¬üh8cj‚…a`ºÛÁûÖ¸}ëAÐãÄœI*[ïÉ“©©A‹SÐ-‰é/ÿ;/?Ì,‡áoÇêm­>a¿TŒ§ÜËË×Þâ³|Bƒs%˜m=ÒØd2;rð+ÍÿîÝ @¬¸Ä@ë%3ó%ÃdÜù݇Ó×~ó·¿|û0z÷túÇo¼Ào¾Òù¯›þšé/è/Ÿ)Lã<-£Èwù $ž÷ãérÄo,“º2 §§)mÉÃt%]ͧ[ô`Ïó¾éµ›t^¾—Ñ6¡¬znž¯´É˜&.m«å>²6óûäm¦Õ7ÿ<¿{súý»9øFô>L:IaEf7ðº/ËKðú0Åòö -L÷ÀLgÙ÷­‹yZedGØÆ®K|×­ Úz³úžæpø9C¼¶Ádcɯ É¥ŸFXÔÂ÷ ì \ÅäÎê lÇ÷ôè-«œÏ5â:ŸõþѲ‚Hæ:3(G>£ç:"÷å9"¯û”Çå’–«+*q–ùˆŠ†1¬¸‹Ü™çhˆ³×ýʵ9b4„­TÊbУ¦ôJÏ[«¾»ôàkxZ­ÜÓ"<¶ù…»#ä;ó•qÈ£rÚ•UØ.dß0»¿Ê(é¸r•ùÃë¼ñ-Ž›•®#q_žÃ!qÃG< güÂGÅ=³8{d½¥ûò¥;ýÆÒ‚Ä|¯8ïaÞ;uµPfðk[6ºY¾"§ôæ+G«5Cˆ’m2,ª=n}èmóü9®úRÔ-øöÕÛPF1©Û×ã;ó Å´ã¼×ç#&ûòõ=È]EãñÌy‡Æxæ­üø‚Ÿ…¿‹Œ?ìP=¦»¡ÜŠBoklDæºó(´G[·äöÊÓ;Ÿ‹Mú„~Ö¤¶žHîÌs8Ä^÷UMÜîÝ/‚¼a.é˜=÷ ï~4!ï»öø¡Äb‘»¸/ÏáøåýPו³@ÆóK ÌÆu[£ñæçÿ2"]ó󗉣&r[ÏÍwæ9"ÝÎøæµÝù;9aúõÛ½2óÕú'ŒB„»êúÆk~m w -z;>ÕÞwÇ>Èy‡šûŸeÏx¦_{vÜ4ú€ŽGÌkOór4Û 4¢Œ—×­’z>\ØŸ•tköÅ^õ§}¼ôê#`gœÒ±˜ãÞÕÊÎ<‡Cìÿ㔳‘µUÖ*wìÁ¼í+eñƽOwæ9B7¾‹s'ÔÞ`ßÙQØ=”yã×82d$zuû3yŽ†D¿áŠ7^0£s_‘Ù""ó¡C}Þ\v*O~ïb³™Ò¥×åHooçÈy‡ÌçÞšùfÂZrk5³»÷ýÔÃü·ÈËžötù»Öóu•Ñï>.<ÿС¼×tðQÅüeÃc–k߆Vkk¼¸†¾;öCîÌs8ôíÜÙéCî¡9–ŸÙ{!ë;ßVPxÇ7¶wæ9 -÷ù‰Ÿkí:_¨å3GVçêwY~Î~¨ŒÎpÇw·wæ9:à |Ç/Ê+Ÿpä¬ø¹‹g%]Ô»Ž,b1Õ‡n–æ:êt峌Á;¼Äy‡Áë^âØnqŠçÚ¹—f kÏâÙ}/.é6qÕG@;†Yàƒ¥[eێ帷X=K?ÇÕ–7¸ºFé# ýê¾ïîع3ÏáP¸ñ-l¶VéŸPÔçbJîmqõ¬×{¨ŽW×F]ùWŽÊéÑTëMÓü—º]çǨ¥=üÉ̵o’ºŽ¾Õ9;Þ¾“ggžÃ!uß)@cÏ +Ú­^Cqûïtó±.q=͉jòØúzúø{ò´Æ}»O4êŸeî#ªkÌ¿©#¾êV¦b^¡ÎG6cᬶN¦Ú—çh¨Œždºþ)o÷Lp¶C[èµU~ëãëýÓáž›®¬Söõü ïm+æ£×Ð]÷v:y08tŽsùä×ètdV˜m&Ù@ÕòØÃrxà9ÿô°ñ^üEæWŽØxL(…|Jg†É›?üée0ÉÛMÌûËæ.=žøÏp” ¦Pë/×y -ãD2¦·©2+D–É%wO!/5m -»”•ÓÓÓã©œÖ_~Dz¦Ïc9SÙ.Lõeæüù(pÑÏÊwSÝÌž± ÇîÁ•{XtåÇ“OœÕã©ãÊÇA›üZ’³§Áæ¾Iç|ÊmT–té:=6¶>¯ä_æò±¦šÞï¢ò‘ïù¸ßËË_ØÅ”þãØéN:ÑÛ?úË»Ô(Ž¦\4ñó)¿‹ƒ1aU²Sœ…ÓGUË"]VÙ#¸3 ©Oi]9+3shN+w“ó»’-Ÿ‘Jª$²ã)¡—Š!rz½ç»WƒžÄù}&q°Ó•¯¦8_l†°Hv­X™—öM›°/—¶Üª±ùPób.øíà26mSò™‰«œ´MéÐ’aqÁ Å%«05Ëfff@òJjÓgû‰w…Íu± æFäKÛtÔ¦Y^t‘wÁ’À¶ü‚LšWšföFß=¼^Üï“ɾŒ€,­Ô^¥Ï\Zo,K+áÉR;^úùesì9_”ÁÛZ¨–eÙسÕÉiC£¹4»àeÒزM ¯”ŠlVi™(*]@¦Š¬–ŠM~ÊVše¼ã(¶R¨>E“þÈ[Úñ^©èx¯øEÚ\n™ñª·d«ä8œC»;ÕRV_2 -2ÏsW)µ”¶È–uÓ¼î2ÛžÙmP3Ä7ÏTµln‘ò.þ»gì\òt‘)Uù£#:¶—9`¢&¬±ãû9UôBùK" —Üœçzüd%)ñ^ºc kUÐùM«² ÏP¤m+ª¼…UΖ”xK·5€j4§Ñ¡ñU{chr ÕË—ŠùÍš6;½1¿,(-†‹ê¾¾‡Yeüù*´² -endstream -endobj -694 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 693 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -695 0 obj -<< -/Filter /FlateDecode -/Length 3206 ->> -stream -xœí[Ë®·ÝÏWÌÚ€h>Š/Àð"@D@€àE…4–l£rùýœêi’Õ3çêå…¾º˜«á)6Y,VÙ¼áìñó"àW“àZëµÈùòþôáä]Íÿ¶‡Â&þŸšÏÁKu9ôÒòù½©ºÚs¬å|=á×]ÑV¿ž~üæôŸã3á]ï%†|þõ­Š$¹Újõ™ˆ:D>$"ÊÙÕs!¢†Ø;A\%Ê£¨B é-3Ô5¨‘¤†ô(êP#Ö L5BÏ•ˆ‚‡!zOePÄ‹¤Hd!»£&‹Á•Öc§² -Y¨‘ÉRr¥JÈɺ+¥äÆd]r÷É2tÉAÄ3t‘Ô*“è’JòT]b«‰É*t‰°N'²]à]Ê ‹/>0YO.÷&Be2ßj{”EŸ]n)y& Áeøf¢2D&©T"‹Ð¥øÜ© ºää#“%è"øa2ðDNµ5­Q£;ù˜,C—«PtÁä3ˆºøZ ‚œòã€X«“…‘@lÉI“ÆX ¶î¤ÖÄh öì¤ôÊx ùY Œ’‡.Y -c‚ ‹ r© º¤.Œ R„.¨À¸ %è%1.H`[·0.H [ñ=0.Hà[ñ¡0.H \¨é$0njE$P.\©3.HàÜîd\@º÷ʸ uS.‘qAí&i…qAï& žq€wS“T]àÀŒ ÀŽ.…–x7_xa`Vü°0.€¯¸Øšg\ àÝؼ0.ðn¬X¨¬:Ä`b\ àݘ1¶rƒwcö‘q€w£Ä¸ŽâÀ?žfà]PHæétÁ2.ÀzYLŒ И‹HKdðnèXþHLƒ­\h½0.ÈàÝÐ0}TÖ]¨°8‰[XÙ…R:ã‚ ÞÅB•dð.è³1.Ç;¬¨‘qA•šÉÀ»!vϸ ƒw‘7dÆp0.Àú悇ëRYw¾·Æ¸ ƒwAK‘qVa瘄ʪƒ#Æ c瑇2.(à],ïqAïúœã‚Þõ‚´‡Ê Kj‘qòȶ$¦‘¬:ÁLÞõ3Ìdà]ïkg\PÀ»ºl2.(UØ@A‡,82.(M\¯µ0.(ÝC†”ÊŠC93.¨@eQt‘*Œ X»žzc\P#tI12.¨º e`\€ŒÀõ€ì›n  Ü…îº Sd\&s­‹0.¨¹`?«2Y‰®UÐ5•5ÈBe\P«8,ÅqöR®!è€à¦KŒ  dAÔ]*Œ °¹‘™3Y€.H² Ó…,Æ-BdƒŒ ZlØeϸ+4ö’05“‰×}fc\Ф8M L°ÓSRtM0.Àüèf;3.Àæ›Mls¨ º¤,Œ Z‹ºÝlŒ Zƒ.ÑGÆðMÝoVÆÝCd|Œ º/NÓdÆ qȼg\ÐCÃ~3 -ãÄ¥î7ã‚ž<ö›Ø⑸í©è~³2.è]2R*ƒ.Hš:‰[pö›`'&+Ð%va\ÐKÑýfc\€4ûMdôôH é~³Òãðnî`&&ï"3-Œ PÀ~™-ÛÛ{o®Èϸ°éŽ³ÑêÍX[é‘÷bµªôÌÀƒ|uD  &v–­ÐS,«O <øù˜}û‹ïùçÿžÞ}³”õ¬A -šÆlÇŽÙ~‚A°=D^[ |Ü™ÞÁ 4=ê‚\Ë=¸Zlàåd`lKÁA=özılnºC^;jÝ z1#1ðõ„Ív¤wpΫîì΂²Ú]ðÉÕ kЫ«efßËéGLôÛƲ&H¥bYÓ C>à|ÁoÎGt.k_s°ºìg—¨¡ŽŽs©ÖɶSdf™õ5v´ã5V·ðæ²fŒ•ûPµ Ue¸#¬ÛçQßÀƒ-øèCGÖ4¼evt]!>Ô¸5ãC µïá\VÝÙ­êC]ƒ^]-ë0û^ž#÷w‰\MÕõßùñ aÙƒuæÞó€|2Èv3¶¾çàûƒ_Ñõüøå ‚Ï*lFl qðìçà{¾çàûÂà£Aöe)çùxÏ÷ ¼ÏO9÷ûzu‚fÛ¥üßʼn´íu€¾¥¯z¤y~õþôíË¿üí¯ÿøûÇWïNÿüNÃð øˆË{ŒÎûÜðÁÞSÞà£ßÃ^¾àƒú"«œãM®Ïn8¾Kýþ_çW/ŸÐGߪO}T…û.·&óÞüO¬»[ó¡ÃæÝ «ˆý²KRôAŒEßåÆÔFovÀ‡ëÀÞíƒÜýý‹ðÝ­çmpedz1F'FiÆmµ!onßïÛæFÝå{=m/—ýù½YgïcÓ}×u¶ïÆ—~S·²ôÏë¸æ³ù ¬-÷Öf½l,kdþþ·ËÅhߦ£ó—]“²«P}#<%˜~•öÑæró/mC-~zl§$=úÖëílÖz½ë–ÿQ=RôOéá—•·Y'aDtÓk6ÅK,´=BvŒmÚ7ܵßÍÌ=Êâ#l/»­üíÿ­nÚÛ‹{—]ë-*¿ìº¼ÝmïtiF¿Ñö˜—]5å£ö•{Xªºyö­íYŽ7Öï° Ð5Ö[Ä=&ï´c$ÙˆÙìão6Ùê˜9iírYÔ8Ÿ‘Å$÷ºÉ}ûäán6üó«ýhp»3鱜é{¸–C3ÇÊû©ãgTùøÁ$9„Þ{<²ÄreæMêPðFDôõªïLØj¥¡¤.1Ìq딦1ÝûnÚ˜—I‡»mS·‡´¼½ÕÓÙÚÜd7÷6ý†oËTY®5¾ß»ÀÖßëÝ•L¸ÚÏ&ìFîÓ;ÇǦòélïÿÂvÄ—æBÔK¢ç"à_}÷~ÖwŽÍ' cêÂfìp~âÉíjí‡Yk¿mëõŽ¾|½]Å5…½þ‡mÒßcuWÆz°ÝŒóÜâ(,øzÒ›‘B`‘µ¨|Vpÿêg[æ5°Ë)$ÍÀîj¦ìüþu-d)¯1Ñèóy<¦o¬@?q”4nO–ðëè½ ;¬·K›K{©LU¿÷ ®ÅÖ] ®? ‡ä ˜Aoº[p”¡OÐÛÓteSukº`¦ÔÕLçÇÀ 2È1Þ…^)ªwÚFSúX-–ŒR}¢yÔœ"sž¤)úø¦“9åfÖôfÀ lÑdÖë£.'Zoo-Ï™ª³V[öaù¥ÊrõÔ¦á v1É^Õëò&F|_°æ)s®}3~@lŸdFâ‚·mÙcí¼¸ÂçázL¦Ý 7¿üe¡Ø–í¦BhLß«êïUä‰*ól_‚L—Ò»«´:ÑØœðÑH/“u„Y H]u¹B­Óg­½B󤾒î¬=-v@É ^ûPOIj’ÇÓë&¬6ì3¹Ò‚~(+_ðÒ3ÏP…³ÎJ0îžg» .“;¯®sƃ^t™úÎvÏð²-lr«ë·ÑÉ‚£Ÿ«ÝêÌ€K1‚^7xDëd'½¼?¾® c+îåôËi[æbʱëÁ] -zoÓf^áü?Ôy‰Ï¿Orþû8È»­ãÑO¶ˆ“[°]Éô–ûLXoýp†£Ú_ph«nÞ0×ñ _ËzÜ/b5ÃdÐ0¥̈ŽÓucHÆ2ù#àåΊ3‘ù­s³ì[1Á¡›¤WL†V(Øá Ø–,­%ùlšDI«®%R‰æAl£(šûÒì ¦:¤ídš^õ;Iû2“Fh1-™¬eÖ6=°m¦ -Ràd ¶a ¶Ó~Ї(ýÕáõ›ç}W¹ŒPtRûD¯StGÓV,Çâ\sÒY #x—…f[ÉÄ^šIÎ -•ZÈ¡`}Ô m=¶WȶSØ¡Ó±PLºæ6[¬C§²?6̲¸Q+Õ™S,ôkélge1ÛY)uµÝ±âÍl)Í([w®kt~Õœ¹d#X±OOTVÍ4°Ç¾åØ÷XmÏf²)+3õë±üPó«øïkb‡³Zãg; m]­ÃB-¡¦˜¶¿–ô­pˆè…÷*Í—|ñ3_{±Ëö„ép’’¯Aÿî•«p… Z`šËëžéË ðäEú2#lø´½Ù‰f±êVQÿÎ1iÒÛôÏ3ªâå†é±Ë›ÃÊöóÝWó2 -endstream -endobj -696 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 695 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -697 0 obj -<< -/Filter /FlateDecode -/Length 3917 ->> -stream -xœí\K¯\· Þϯ˜u+’H½€À‹mQ£)ÀhEöÔNP\pºèßïGÍ‘Ä3ù/»p‹Ø÷zDêAQäGJGgÂÑãçEÀŸÊÁÕÚJæãéÃáãÁ»’<þõ»Bgÿ€ß’ŽÁsq)´\Óñƒ¨¸ÒR,ùxwÀŸ‹¢®~wøñ›Ã?÷mÂ1ºÖr éøË;a1¹RKñÉ`5°| ƒ•’+%¦l°rp˜bk b¤Â‘¯Ybp«ÉbA Ž±¬ -1ˆK kVƒ±¶X#´T Vð#DïMñÌ ^H.CéÙâÅàrm±™¼^(Ñâ¹\8¤`ñšË9§jñ²¤æƒÅK%foñ S-/CÊäMd‰µÅ+%B;ÍàUÈëª&²øìƒÅkäR«Ì&¯çk©×¼è“K•È[¼\‚m’ɃGa‘r1x²dŸšÉƒ,‰|´xY?8‘¨Ôjxkï&OÁâ%Èca“Y°øÄ Y|ÉÀƒœà’…±Ç-²±’ãÊÕBX›ãRÈ‚Ø’ãÜŠ…äx1X@@²$ÎP€, Ï5y…[X@² ‚…D%2YX@@[†µXX@€[ö-XX@À[ö![X@\ˆé-, .ÕÌ ¦Ô,, `.;-, €.ܽXX@@]J9ZX@€]âš-, à.qð0p—-Md[Xtt*YXÀÀ] -¾XXÀÀ]¸Q´°«ê`‡ÙÂØŠ‹µz ¸«g ¸ B€É+>H0p7&LÁŠÜÀݘ|´°€»‘c¶°†â€?ÞÌ€»€d§kaâ=x‘,,@g."-±° wCCø3|håBmÙ‚Ü Ëgòš 7üZv!çfaAî"P‘… ¸ ø¬ã"j´° w!R[<ànˆÍ[X€»È’…0XX€ø悇隼æ|«Õ‚Ü,E …¯@“W )XX0vy¨…¸‹ðÞ,,ÈÀ]Ÿˆ,,ÈÀ]ÏH{Ld¡-,@^ÜÖði$«ÎG …Åîú€¶xÀ]ïK³° w%lZX‹bµ°:dÁÑ‚\ÙµR²…¹yðò™¼ìPNBd™<ÈÂ…-,@bíµjaA‰…b´° DÈ‚”ÁÂd®dßæö²À\ÌýCdŠÉ\mÌ””±ƒV-^Ž®ÀµÉ«à…baA)슃…ØK¹ -§·°ï$]²°b€ØÂ‚Ò \Å„"W#2s‹ ’l é‚‚…5BdƒÔX±ËÌÞÂDhì%¡j‹Ç^ö™Õ‚ÊÙIªhaÜ;M ¥Éƒ,€ °>²ÙN`óƒÍ&¶9&²Pb j²Ý¬Ô -Y¢À6e¿Y,,h² ã³° ùì$M¶°0ž÷´P±ßŒlaüRö›Õ‚FûMlñ ¿m”e¿Y,,h YR“Y45ÃoIØo,^†,±±…-gÙoV f`¿‰ŒÞ<¨²ß,æqp75 “Åî"3Í €ý&2[koＩ ?³™UvœÕ<1ð€Þ„Øjx`/¢U1Ï <ÀWöHæ¡ÄÄβfóÔaÌàÍcüE>ÆW˜p>}ÒGTæP3k‘I6¸#»Ÿ{ŽUnT;ŸýòÓáÛ7¾ÇŸþuxÿM?)?bIœ0ÕŽ «ýá…`{ˆ¼6+òÈ™é™Ó£.À5_û©E'žŠŒ,èÄbïötìêj§«árÍ­¶’MQOj&Š|wÀf;Ò rJ³îNóêw‘×LîuMz µ´cé÷tø ýQ÷±´ P)kÒé.l/ønh!KYÚZê€à²]¢†ØaØ/5VZÖZ¯Zdkø;¥F=]¥tMXl*º£"·'ËîyÔŧ+â΄yoBšŽägË.HØ4LÈÛ&ä• UÔ¾$§¼¬e§‰Ú„²iB“º&½†ZÚ±ô{úê¸_Äq%S—ÇëÈjí¬9|°ò¦“i½iÝu¾¯Î÷+s¾,ÿÊñúÃœO ¬g¬±·ì¯Î÷Õù¾:ßÏt²§¥œ7ò«ã}u¼_™ã=%åĦÐgyPN£¥³¸Å»¾iÄG9|HÔÂØU>§Íý;Q9Z‰Ûq~°÷¥‚ë'¼7ì{}¤lÊúä6Ÿ]V¦ä¨d»qc3…|¨òƒÒ=jí¯…L!?^ȇ*F!¯¤~óú€ì·¾ðcÇ\ûó*¹FRäÌýøúÃáÛW¿ûÃïÿüVçõûÃ_¿“ÓZüüöëÞÇ÷Þ§è=¿Ù~O(ùüòoÇׯ¡æ±«¨¬?À?©„HU†ÜòòEønûÌÛˆõLKaû?oÿ:$ãz® gòüöL‡Î{Yè¼õ%“8K¼Õ9m4ðùýª;úæ²ÕÙúIò7Þû3!{:mmOK¾^ÎK¾!Oÿ\ž ­v©-ÿZQRJ¯Cª1›5§m´¼ “uÿ°A€7aôO›ÄI¤¯ç™Jáï×ýd’‡r1溟¡±.Oîš¹OŽoÉá×\ºÆÇüNg[û3d“ RÙsÌÍè»[ÖÕ™V´Eܲ†–Ð6Ú{ÕgÛäÝ|) ùë¢I]ßCÿ›LÂá>ý‘¯sŒ†ÜCO¿}ݱ‰&Ê;j$O­’çE¸9`ËÎ7¹|p1àÙì6€ë&½Õ6úýýF¤E)£PŒ~Óû—´F¹±“ó-©hœ¶‰uYí%¤¡¹Jè³_†å”éMñ;Çct-²O¯\¾gk›Ú5íógêO¦å)Ý3´/÷³ ¼\nÎôö²zÀAŠŸOû\#<®2àñ~;MM­Æ­±7”‘ !×ð—Îø(-¦Ü\ƒ7Þ¶—Ú÷§©‰£Õþ™Z”ÛDýRìCÞ>ô’ó -‚SÃƤ§‡¡ßq/Á·‡ øqm¾4/y ¹ZRŒX ó0º•[]ç.rY·Éµ¸‹A{ ºÃñwsq$ÆX0 G¶ù”­É#”3öŸ÷Ø‘ó…liÛßF“>Fï5xßߨ;äÜÈŸãË'xMK÷yÍôˆ²2á¾£Ô^ÓÖ­{M5<nÒ’ìc÷Þç1IÞÙà;òGAî#Ûü¯yŒ\1þ¿@ÞGl‰“\o9ÇöÀ™…:M˜©Þç9QØÑõžñ¹'YíáÛÚóíNÞmô­ÍJq·öZþmìtzüIƒÜ¨^jíV¡¥g&YIw¡A‘JN'È+… Ëg-÷žJddÌ´óáçJ\÷Ó5÷f“í<‡ûä(é–þSO%®ûž§[{™kï[Œs¬~ò0ÆnË¢,9ü8©hÚz/-÷–ÕÞ°Ø“ê{[—é5zÜ9Ö½zn—º¸ï$dô+ãhYtV§9z¾ZÎ1×±žÂóúðæù¡Ÿg‡ik;Ng€g?lwÿJÂD“¤*€ýPÕUÂX¬ò@¹¾x8©¯î©í4º”—~©#ÂE+rOöVxŽïÎÉí<†š"¦OySK;×v ÝL,7ËòÖ¯ÃÉt3©'VÒCü¶,ÝÔÚ™çÁ¡rý¼\¯»È¶DñͶì§ýÒÏþ“Ÿ‡zCž9µóüÎs¸ZʇÜï|‡þÞÏÕ…(/3Ãnå}‹£Ü3¯žx> ]Ùáx£eú㬵½aíå¥+¹p~ýZ¶úû¢¢³ËÃûÛ¾“±­O—仃¼ Ë™y‚ÈBÜ>úÙ‡Wí„lI»]ÔDþá·+øH~rþÓ˜´O£™ÜRÆ%Ž’øÁy`ƒˆ–9\“eöÞ…,oWG[)OQä€kW‡¬ë®WÃɼ"¦k¢WÃ-rä!O7P¦êò"Nѵê‚ZþQ7µœ÷O K—Ô;“*ï1Ž®†v£Ÿ4RBµIM£æ\žëdä¹ìñ…]3sòÕªÉÛ gb†Ä³^õ Òé`ÖÛzKs¥Ê¬UWÓ64¿DY¦Nu*Þ T‚Vä+”x÷dÉ%æZûªì&^O„¾"÷gñ×µÓ -Ÿ†éi"©~'¹úe/‹ZêP\sÐä.½ü=CEšTAžþ!ð4)yf•Ö ⛓¼W’¦ç‰:li pYu•ºB)Ófµ¾BõF}ÝY{jlGU*SôÒ†xR<öä0­n’†~&Vj¢B`¸ENÓ”ïvä%gš®šü,eîiö»Èybç&—¹âA^nšòÎ~•Íâ©6¾$‹é×1È"G?£ÝL—`…E^0z×ÉÃ[':É6ŒËɬˆ{:ü|èa.RŠr„š)È»º:ó -Ç£Î+üþãÀÇïÇ%’s~¢EœØº#ëH&ßìPI–ozÄé6ŠYÁþ"‡ºê¦a 3ŽÇ¨ðšWs¿€}Ô AÃäjbZ:Tä8M7RšI÷OZœ‰Ì§®Í -Øç"ÉÉó8y­hH…‚žŠuñh…ä£ê%©ºB:¸UCt§(ªÛ’l'¦¤dª^äg½$c©‚J#¤H‹Ç+Ìê®­ÏBUA -Lº ;æ m;y ¡Ÿí^Ÿ¼î›Èyx€P'´Oê¦Î‰ užM¥^ÌûâŒ9ÔŸ ç]š}‘ò=šIÎr•Zð® mTQëj¶UHzP´Ý #PL¸ÍÚì± ™òÖl¨¥Ï¸šZ*3§XÔŸµ¦“^•©Ä¤W%_Õ•~GÄ›ÙÍ(is.kv~Õœ¹d5hY·žT^5iЮÇæýØ#ÚÕ4x ’WfêW³tUóYø÷ß1oœYµÆO? u½N‰@Í¡P¤þ Y¾æ aù’ƒÂU—<¹ÍsßæÓ#a9ùä»ÎlîÀh3TWã{ÐìfÍ›Szšn¾—jÞ ýËnúÈ~H4‹E¶ŠòÝV$Io•¯ä(þÆ}A9’c—Óî¥ÿü©0 -endstream -endobj -698 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 697 0 R -/Resources 4 0 R -/Annots [ 699 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -699 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 593.554475 101.433071 574.804475 ] -/BS << -/W 0 ->> -/Dest (cb190-1) ->> -endobj -700 0 obj -<< -/Filter /FlateDecode -/Length 4143 ->> -stream -xœí\K·¾Ï¯˜sÑE²ø $A„8ˆ!>9¬&’`@Î!?_±›î®™}H¶cK^¯vºŠb=¾*rºÛž ?/,þÉlMÎ%E>_ÞޟȤ@ø¯¶Ø\Tö7øMál‰“ ¶ÄÎï„à“I%¸Ï÷'ü³»œ›ßŸ¾ýÝéßÛ>öìL)ÑÙpþñ°Ø›”S¢ ° -Xd½Â -Á¤äBTXÑ,±1BbÇGV‚\rÐXƒË -+C ÏÉú#«@ —,k,ˆaKH -Ëä°ŽHåAböNáÙ`"”5ž³&æâŠÊKàÙä4ž÷&&¶Áj¼bbŒ!k<†,¡Õx²ËL²°ÏIãEÈâ£'•Y\N^ã%Èâ ¢ð2dwe•Y(’ÕxÅ›P2³Ê+àQNùÈsLÈޓƳÖø¦Wyˆ()&…ç K¤PTd žœÆó…ñ£ñ€Á§œ•huÝž¼Õx²8—XåA_C! ¥¨A"È.iàR2\k à²7œ9k(àr1œ’×`À•`8–¤á€' ž³x‚,£†ÞBFäª<Èâ kXàdA ¼‡,Ž½†hËð <à–©X <ð–ÉF <b’†ˆësd < ®T4,ðÀ\ìÔ°ÀtîIÃÔõ!: <`×sŽxà®gK0p×{ôTy¬aÐÑx›½† Üõ–’† ÜE9 `U?ŒÀWŒË™4,`à®ËÄ0p×%¤•— bÐkXÀÀ]°-sw] §aw»¨aÅH­€»€ —«aò=xÎkX€ÁŒCY¢aAîÚ‚ô§Ä4ÐÊØ\¢†¸k3̧òŠ± WâZ66Æ¢aAî"Qy pð™5,ÆdT§aAîZL­ñ€»ÖÒ° wQ7 àƢаùÍX‚몼b¨ä¬aAî–œ†Ȇ2Då%G²Œ ¡Õ° w‘Þ‹†¸KÁ{ "p—eʃ,>; P×€‡°UbŪ!¤ÐxÀ]²°°ÆaAîJÚÔ° &!¸¬a4¨‚†1³))E b!ðPò©¼hp4,HBd©<ȉ5,@amŠ/YÃ‚ä ‹wNÂä J P˜bQ}«ÛÈwQ÷ YP)jX$3¹0kXBÄ~ ZÕxÑ™œ×*/ƒg“†)±A*¶`/e2‚^ü‘rIÈže R, ŠLv¨Ì5ž…,(²5,@¥ žµdYP jX]Æ.3’†ÈÐØKBÕIö™YÂÌÑH©¨aÂ;M ¥Êƒ,€ `Ùl °ùÁfÛ•Y|` rv²ÝÌä Y9 à›²ßL‚,¨ø4,(”ÉÆÁ#Ò° ØŒý¦c —²ßÌOØob‹§ÄmñQö›IÂÂ% DQyESQ☄ý&ÐIãEÈâ -kXPb”ýfÖ°eö›¨èÕ#,ûͤwC2i<à.*Ó¨a.°ßDe«íí Àê3™eÇ™ÕôäVõÈ€€½ÈVI=3 €¯ì‘ÔCˆ‰eŽê©Ò*˜–Ôcþ¢ã&,§OóU„;$äÌœd‘aÁ¨îûžc4óWš-çc?~úâÉ÷üýNoWOÊÚøF LÃÚ®ÀÚïNP¶‡¨kãD¾9£2Ý‘=`ºµ¸Æ=±žZTâå4‘±-:XñØû-»Á\éÓt¨k[ëI¶‰z™V2‘ïOØ,`Gº#‡0Úöéf"qy¬ä~¢ŽE©†v4ý^NßÂÐïç1†6* iM‚õ …óYªÎ.bËT†­-²Ëzv‰âˆvkk˜ZŒ=›h²²6ÿý¤Çy½“ÖgruÙiI÷¡4”á¶dÙ>·¶øt n|h·>4ÓQ55oéÓYÉ+Še݇òäC­÷äGÛ>ÝLœ}(ª>Ô©cÑcª¡M¿—Ï‘û‹D®”êòßùøAAÙvúÞéy5È6z›tÿ9ø>ß'|QþKçã‡'ß,ð´âYÏþ|Ÿƒïsð=1øÔ {ZÉy% ?ÞçÀûÄï)%'6…Æs”oÊY&Ã"n"S7\oÉ™rl»Êçô¹½•³u¸œû}_j½5õ¨‚7ó‹}/9UYŸÜç£Ëä ÎÙ˜Oóº¢ -ùPã¥{”í«ÇÞ߿:A9õŽ*ü-l˜sý.HnÑHrž}~õîôÅË?þùOû õöôÝ—DÂ@P²,ßÉàÿ;¢àðˆø ~óò¹òìÂã$ô¯þq~õòÊœrÛDŸS¦qqCpÁï¿2Õ:<H!-ôÖN¦t­íÛU”¸Lk  ßöb\Nþí\”/ñÏMŠy±ó¢]þê…ýRf]ÿÆõo£‡evi[¥|½Ð›ÔúÚVØÆàuŒ¾’Ö/Ž>2Oh+ŽÓ8;-Ô~~êC«æâVþE›úÚ긼[Û,Kz‚¦y¯iY]›YF¶eÕ~[µš­×´¥u‰B·ûÂK+mÕP÷™2úÐkX”'ëM²TM5_Ë‹+Óe]i\—7kæs¡û²ï2-þ+V‘ñì?ãD/ߧÈ=;Çqºy+Þ’Ã;º&µT½(±¤È&÷nEb‹2v ÷°Æ§=|òÊÉ£ëßÔ-ï×ÚüU]Ö¿ièž.=/òU9_¯mn@mØ@í‚V [Ÿ6nx½®Vjúy;ì!s.16t½I¹·³!“ô±î¦ÎâAgWô#óÉØ×øuÞ7k,)()ko~¢!¤µcŽŠì²RÆä¥]ÓN~ZsÒ͵–¸[ë‚/#ææøÿëZø^» ÍûâåËØàq…‹|A…Rœ+N¥ÄSû<«ÞY+ -Àz1‘“oRÉ*™x_VXn³.ªé—Ö‡.¯|Ñi½óK•ñ——ÿë×s•!&-)µ„ÔÜb¥¿^Mpu:XŠÜQ³›n¬e”jøžâð÷ö˜Î“Ü°ÿÐƼÒ -ø´9×4¹+Ò4o“%ïhnê×Ɖƒ×ǺÇíýh½NÏë7®•&ÛvüvׯÍcç¡%ëõo™tµk³h›ûJ°ì¤#[”ÑÄÆYÎÌÉãúüÒA2äΊ‹z°ëŠyxkþnV;T?ª¾ÅuíÅ}_Of´ï>|ýã–ü÷h.{ÙÍwôIîÆ«õfrü´¡tÅ=åÆ4[·v^îõ„²Óƒþ¸>ÿ?±>G§cø¯a~¾PâI&ºë®Ówç¢yµ¢.›QRwån-ëÚ&§’©•oú6äXæË-÷EnnÝMZ!çJÉ"÷í®®+Ua)>äîìó!‡ ?Eåb¾åõRü Fw“íúw0µSßø6™ÂΣÑc×¾ªo>bo/·*‡Pä‘”‡ÎêßÐäü•«4}ßmeïãŒjuµÝ¥u¼Ëlà Ózê>\‰~6Àw«ÞºœÝ?$š¯ó>þÁHLVN;'4yÞÁÈqœù¬±É^ån&o'[òêÕ†·äM”®ÉûÁ(DZ?­”äöëÿM ¬6ê2†•æöñ·õÝÛ)‰÷ºÛÂ\×Û¯ê`åH‡zÀ” Ò²K\­x¡uG¾¢ÂŒz¯m\ÍqjÛÐ}úJâj¤4OÙÔ¿[įm“í>êáýã5œi¯á›9jéç>¼÷­®âiåyÐÄ*·s“<A!7ÇyÚ¡½<ûpEŽÎ9DZ?­œSh¿þßdΡ1æµCû?+ÜÎ5ÅíuÖk©°ŽÕll‡ mv·n»ÒªŽîú­q¥äžÔ¥k¾ØçõËâ›ë -y·®è¿YŸŒHá,_ÖË“i9Ø<=hÑ·¸6y`G{|,£Ýfp¸‰{\€!å•(>c?-|’§ˆ´-˜„¼{³ØÕÅäûvI’~M˜â"=‰®‰Ó…a²ÙÝzby³º­ÐÒÂþíb¡»ƒ¼ÆoÍê|ë÷ýÕœi„Yÿ†ëõ5„¶†ƒ)ߟb}"ÎÖ§¢I¶õN^›rŽl¼Ø¦œå)¼Lžm3­Ê¶ç+=ëËfÞ÷VëûgHI—Ç——ÓLkû÷Õèï‘ÙÄ–ƒë»"¨’ÜaO¾?É»BX!3DþYˆëGêc±íuA£]NÖË=I»–>Z?ŽâANJ–.´ÕHÿкÉ3\L­‘¯O1-+DôŒöH–Õ“±+YÞ·’_¯b%Ä&3ˆc@žÛŽGÇÉ(l'b8išn7y¬<ŸÛU±‹>«ÎNæom±r"^N˜Öï©÷*UÞòІjÚuÔi~ªtjh-»A¸ÛI!ÉMkâ jC3ŒÙù“ÕäYÙ…˜›#qoWZ;…t9©íÖÑB·Tê­òèZšæ‡(ÃÕ}îŠWh—©ÀNò©)FÈ·[²ÔkÝÖ”'¿qâå„Ãr½QñØ: ¬ Ð\o&úiÜNÎ4üePSnªBh6šŽ Ó2îåôé¦9烓/Ò¢·ò&“¹ò²çÿ¢ÍKüþëÄç¯Û¶KwÔÑÂulÝçL&ï½Ê$Ë{°±‡ÍDt<Áþ Û<Ú†æ =;7á5î4€½µ´AmçÎÄ0t8‘]w]gý¤™pƒxÙi±2j›‘°—Kß'7wÈC×M*\ÌËÁeÏ}ÕÁ<ÌaØ{JVÞ«‹pFy˜1 ÕÞ«w+¤.éi -¸úÖõ [_Xo‹¨‡DýR¾€sòæO/Eo––%ºòÄ·os6G(õçÉÐy -endstream -endobj -701 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 700 0 R -/Resources 4 0 R -/Annots [ 702 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -702 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 527.554475 101.433071 508.804475 ] -/BS << -/W 0 ->> -/Dest (cb193-1) ->> -endobj -703 0 obj -<< -/Filter /FlateDecode -/Length 4059 ->> -stream -xœí\KoǾï¯Øs·«»«_@àC€$ˆ1 $‡ j#Ú¨rùûùj¦_3[Ü]R”D;Aq§ªÕÕÕ_=vfì‘ðó•Å™­É¹¤ÈÇÓûÇ™ÿ–›‹…ý=~S8Zâd‚-1‡ã{!ødR .ÅãÃÿí.ç懿þêð¯m{t¦”èl8þôNXìMÊ)QPX,²^a…`Rr!*¬h –XŠÆ‚!±ãsV‚\rÐXƒË -+C ÏÉúsV.YÖXÖ–%Èa‘ʃ Äì³ÁD(=j; ×€‡c«œi«†BãwÉb‡5p—( "pWܦ†1 Áe  Aì4,ˆ™MI)jX ‡OåEƒë aA" KåAN¬akS|É$Y¼s$Y2hX€ˆÀ‹è[M ÌEͲ RÔ°HfraÖ° …ˆ| ZÕxÑ™œ×*/ƒg“†)±+¶ —2‡^ü‘pIÈže R,8*À™ì™k< YdkX€H£·¶¾ZÒî¬ o{Ö&¬g¬ž/?]ßOó´>w»¾Í’G[Ïæ*Û2Þi>£óY›â†[TŸfÕ/]’Ñ^ä\<Γ~ôVþL·ÛÝig’Ngçq’-žÍyÔ#¥qNçq¢—o•äÎ¥óqÚN/²ÅEƒ—äþ19hàͲÃÕº–ª Œ"›ÜÁ‰],ÊØ-"\0ÍΖ:¬§[ß}§YZ¬×Te㊉Ӿ,}ëºg¬lòªó)Üæ–9µu1/ê6í×Úõ$¬^éSô¼Üž{ºAö\í-®ô¥Í]íZ\3Íeëºê<¶Ž Xí}Ú¸‚ËÚ©ÚSÓÏýØ™³íQ—1TšÛŸ§A“v-~zDwåLwÕß…ŸÝ×} -–›l"=…™_õ1b¼ß5/>goÃü®³e˜4E#2þ¥uÊ DÛu:®c6›Kûê‚açå¹#ÌËWuלõ}¾t¢Æ·OEnv¤×‘^º`ºîe*¦Ê˜Ÿªdú3Nœ`ô&戸öˆÍ’™_Kçoìó¥yÈ[äþ!ì’{•Óç,Šú¼úùìëf)˜ÂËW`7çï7öy5f‹ikúùÿâ§â™å–KD±6ûÏXüÜNz)Mê5“ br”ÜtÙLoëóÚõDö’µ.ÅϽ5íJ:kz¹Íg/M}‰…Ðdí“ôÆ>¯ÄB‡Ø.^²Ð/‡¯¸šbº„²/_ÝNx aå^èj‹ÙHåšùÞÔå•XoÇ×Lé–hàÙØúÜò©Ütr‘Ao*Ÿö|èÓ•P%˜]Úç–Kµ~÷ÃÆ­}|œóbÕíåÐœfU.»ÚÊâÝð  ¦ÄiÿÊJwÿ&\ך,óÝÕ-9³±ÿ‚Íõ5„¶†³­üpˆËã`vy$˜b6ÖÉ;CŽ‘—½)Gy-“gÛ¶Î.ʶÇGz.oZùÐ[Õ—¯<-Ïâ­of™.jû˦¿‡›g›c_^”@ fØ“ò¢ VÈÌ#:¸BbýH},¶=Øh´"àdʾ%âKªGD"AçúÑ…¶éZ7y€‰©5òË#<ëÄ -=£='ËêÉØJ–—dãëU좄Ødq ÈsÛ1àèx•l=MÄpN¤iºAvÜä±r[MW]Ä.ú¬:;mkëÒ´—ˆ§¦õ{êƒJ•W´¡švušŸ„*Z˾!Ü÷I! -ñ+jC36³ó§]“EWbn†Ä½]iíÒé ¶«£…¾S©·Ê£kiš¢ S÷¹+^¡¦¨=ÉÛ“¦3B¾}Ü’%Pë{My²wF<àbøŒüpIiVH!ûœè§q;9Ó°—AM¹© -G³Ñä1;ù…ŠÐ©‚<ËËݤäÖ¡q5&‘³ÙÉ[%ÍôØQ‡5 XN£í¤.›R·ÙY_6“Ò^@··îÛP'•MôTšxR<¶dÛ­®“š~:VÎDjB`ºAÝ”6ä!gèG5PŸ,ÚÉÜCwcÇ·™œúŽ[yî¹ËÛÇl¾O‹cã=YL?·IÙQ÷vc²‰8( ò€Ñ‡…ÜNkG'y—Sû8™æqO‡‹›s>8ù®&z+¯ñ˜#/{üÚ|‹ßøø]{ntõãŽ:Z¸Ž­òìÉä¥OùF²¼ªû±™ˆŽ'Ød›GÛЬ¡ûqç&¼æÑ°·–¶#¨íÜ™†'²ë¦ë¬Ÿ4.O;-ö@æc÷f8ìõÒ·ÅÉ÷ùòÄq“ -órp™Ï—|œ†Ä•4.\vSG\̃ârš° É6bŠAΓtÕ‹ü7oçnÞö8-ƒ‡ qD¦4º…³–Ï¿眵Ϯ¡µj?KuÄæñ¦8j¶É;¿¼<“rŒÀ–÷%ÎS¹äÉ}.×OΫ$­~2Ï„í0ì=%+¯AÕEx£\gLCµW¤êÝ -©Kzš”òЀ9:«-ïÁ[¾aZŠDý2Iª(¯½ôôfy[W"« ÃíwSBY~þ }/n¹ -endstream -endobj -704 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 703 0 R -/Resources 4 0 R -/Annots [ 705 0 R 706 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -705 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb195-1) ->> -endobj -706 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 427.781993 101.433071 409.031993 ] -/BS << -/W 0 ->> -/Dest (cb198-1) ->> -endobj -707 0 obj -<< -/Filter /FlateDecode -/Length 4164 ->> -stream -xœí\K·¾Ï¯˜³ÓE²ø$A„8ˆ!99¬ÆZÁ*€œCþ~¾b7ÝS;3»ZÙ‹$ZìjºŠb±øÕcºÛ ?_[üÉlMÎ%E>ž>>Ȥ@øW[l.*û{ü¦p´ÄÉ[bÇBðɤ\ŠÇ‡þì.ç懿}uøç¶=:SJt6þ ,ö&å”((¬Y¯°B0)¹V´K,EcAŒØñ9+A .9h,ˆÁÎe…•!†çdý9«@ —,k,ˆaKH -Ëä°ŽHåAböNáÙ`"”5ž³&æâŠÊKàÙä4ž÷&&¶Áj¼bbŒ!k<†,¡Õx²ËL²°ÏIãEÈâ£'•Y\N^ã%Èâ ¢ð2due•Y(’ÕxÅ›P2³Ê+àQNùœç(˜½'g­ °M¯òp¢°I1)<Y"…¢ò Kðä4ž‡,Œœ>嬜V'§Û“·/@ç«<È‚Í×ÀEÈB)j€d—4 p).Ž5pÙÎœ5p¹NÉk0àJ0KÒpÀ“ÏY ­j¼èLN€k•—Á³I”ØÀ[ K™ŒC¯aÞH¸¤aÄϲ†©@ àŠLvˆÌ5ž…,²5,@¤ žµdY jX]F–IÃxhä’PµÆc’<3kX9 5,À1A¦ ¤Ty0¡aöG’í a’$›HsTdñ5,ÈÙIº™5,Ȳ8rÀ6%ßL‚,ˆø4,(„ÉÆÁ#Ò° ØŒ|Ó±†8—’of Š'ä›Hñ”s[|”|3iXP²„(*² h*ʹ&!ß:i¼Y\a JŒ’of f ßDD¯–²ä›I-wC2i<à."Ó¨a.o"²Õr{ð†„øLgfÉ8³Z1 @o€oUKì…·JjÍ€¾’#©Eˆ‰Ì2Gµj· -¦%µl@À_Äc|† Kõi.QE˜C‚ÏÌIYp,Ñ}Ï9F3ÿH³¥>öó‡oîà|?þëpÿU­”µ ã÷%±rD\Áv®LŽk1 ñ;ÎùžÑEDHeˆhŠkÉMD„#†³-ÇþÁn†¼"±E(S×Åó´È,6ÊGUÒ§vyqIq²×S)ež×UÆk¯JçåßñüÃ!£›…¼Öøe…œL h›×yo6ÏÛú¼¼}NóÞl ·õyqY£œ ¸¿nØük¿”…&`ÝÍB^kü¥,9Ú2ï°6$׈)%€×,ôÆ>/¾ë󼳎Îe}rŸ—’Õw£sR~Aò[+F^\9"J@%×/€&QŸØå%$%ã$—Šhå‹“Øä›Æ¿}{@²N¿|ÃÕ/­Ýt£aèÖÑñíÇÃ7zó×?÷=¦}{x÷¨ŽËûµÅg>-ÿ»ÐC\éï¿ýûñí› ³!Ê¢"%ˆíl2ùeŒI¼Xi—‡tv‘geÈ»!+—1EH+-¯×qK߈’w4Y/Æb^û„Açû­~xâµv½ŸËä÷“,ZŸƒoÃ:×:ßM}íXƒuË^üîíÖ:³|=L„K@(hYÂï‹}[—gô ÚqA7Ò´„ÌX5dQfÈÛ j -옷aí•~ܴ̀¹e2l’ý àûíæuþÎ »Ñh·´àwkXe ´;¼qû§Okh<':ÎCŽÏ; ºaŠcJ±Æ òEˆuv K/Yæ}^iJE-CµV5Í_#^Çq˜yt×Í£ç¢Ý¸ª5§ÆÆl(cÓSÕç›ßÿñ™ôé`²á½ ‘Mä’5¬Ÿ“Ì/_–.×AxB¿¯ë\œÎ˜ÔÉ÷„E*ò»I«oüá“›[ÌÓ‘—ú¬‹ñšIßÖçsâÈ/<8J—,[<±•2WZ~åºïþh`/¾@>»»×f¾˜f†ipý¢öa,%¾nš·õy=¦YäÆÌÖÄøÌ4ÿÛËNÑGƒhÕfÿ –¶“^*ȽŽ«9yÊè”êíS—¿´º­Ïk+Ä/Y¢øÎ^8KÒV«úáz›_²\õ2ò>Áš“M—¬ùåË\Û /Y2p°Yev F¹bÈ·uymvŒÓvÉŽ?Û&ž[ÂÊrç[.ò(ÙM%¬5ñþR%,Ž#Ak vOžæÄ»µß_Úx]7a$NuŒL›â–è7T¯2m -J£¸8{¡‰wsÑKîÝŠ×t3ëí†ÂÕ¦ôí·ë®×´ÓE;luQÇ CϽ¸QF)¦xšöÑn÷¼áÃå"W–ûE7Øð¼"×ù8­ÐÑôx¹“K|LØ÷¼"×ùØ7¹Ö‚KµÅ©ÐUçµ4Ê´WŠ^‹Ìû?­Eæ8=>ÿYùVÕ]±ûõégfùõ&×­€ÕÊÆ{9z;Ò(èÌŘÆãõ³V¬k…¨iýççåâƒÛ­ñ)Řz³•{êÃX7Ý…qcŸ_;Iè.­È“ÞÊ“ÔªK»”^)ÁÈÍj6ïv“\,Á¬9åËÜÂ@/^ -p¾í·@¬2VÓÛýÖõØN•Œ°>ìÉy±+dæ]]! qýH},¶=Xk´`9™²o è£õãˆè—.´ÕHÿкÉ'L­‘¯\,+DôŒöœ,«'cW²¼"¿^Å.JˆMfÇ€<·ŽŽWÉÖÓD çDš¦dÇM+vÕÅAì¢Ïª³Óö·¶.MÛy‰x:`Z¿§>¨Ty$½ Õ´ë¨Óü$TéÔÐZö á¾O -étPˆ_Sš±™?íš<Ø·s3$îíJk§NµÝ:Zè;•z«<º–¦ù!Ê0uŸ»âÚiÊz’¼íf:#äÛÇ-Y‚æ¾×R”Z·`ÝA<ê]{òÃ!$¥uXA¡™ÞLôÓ¸œiØË ",[U…£ÙhòX”ü] "tª Oý`¹›”ÔÙÆÕ˜DÎf'o•4ÓcGÖ4`9¶“ºlJÝfg}ÙLJ{ÝÞºklCT6ÑSiâ HuðØ’m·ºNkúéX9© é9tS~؇œ¡Õ@}²h's}ÜAŽ;frê;nå9Õ.ow²ùF> -endobj -709 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb201-1) ->> -endobj -710 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 446.531993 101.433071 427.781993 ] -/BS << -/W 0 ->> -/Dest (cb204-1) ->> -endobj -711 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 110.390928 101.433071 91.640928 ] -/BS << -/W 0 ->> -/Dest (cb207-1) ->> -endobj -712 0 obj -<< -/Filter /FlateDecode -/Length 4039 ->> -stream -xœå\K·¾Ï¯˜³Ñ|É"`ø bÄA ÉÁÈa5–l«rùûùª›¯ž®yìj% °«®â£X,~õèîqG‹ŸWÿ19Ã\r¢ãéýáÃÁš-þ--6 ûüæxt–²‰®$ŽÇ÷BÙä}NÇÇþ;»œ›?þùÕá?Û>îèM)É»xüõ­°(˜Ì9Û¨° -XÖ…£ÉÙǤ°’3Xb) bÄLžö¬ 1¨pÔXƒ¼g…Å#PvaÏ*ÃgG b¸³Ârr8o­Êƒ –(x…ç¢IPzÒxÞ™ÄÅ•—ÁsÙk¼LÊä¢ÓxŤ”"k<‚,±X§ñ"d‰ŽÈj<ÈB³ÆK%¤`UdñœƒÆËÅC;Eá1du±Êƒ,6Y§ñJ0±0‘Ê+àYμçyMä¬ÆsÎDØfPy8Qؤ”ž‡,ÉÆ¢ò K Ök¼Y?8CfVN«—Ólp/Bï3©<È‚Í×À'ÈbsÒ 'È.iàs6Tè¼lpƒ†ÜõKÐ<7p×Gë5, à®'Ÿ4,€¡àU£à. $êádÀjXž`0ã–hX»®Àý)ghe—¤aAî:Æö©¼b\†Æ•s -—RÑ° wᨂ†¸ ød €ñÕkX».ÀSk<à®óÅjX»ˆ¢†0ãhXÿfœ…骼blaÖ° wK^ÃxacH¢ò²!9 ÆÆ"Õ° wáÞ‹† ¸kc$à®%„=*²ö ®ÇV9ÓVõ@ -ܵ;¬ñ€»Öæ¢aAîŠÛÔ° e!xÖ°DÁ^ÂÄdJÎIÂT,xùT^2¸Žd Bd©<ÈB™4,@`mJ(¬aAö%x¯aAö!ƒ†ˆLqˆ¾Õô²À\Ôü€ "E €d† ‘†9&äcЪÆKÞp\«<Ïe r&Wì4,@.e‡^ü‘pIÈž# r,8*ÀöˆÌ5žƒ,²5,@¤ žs°‡,ˆ5,`ÏÈ2“Õ°¹$T­ñÈJžÉ0%#¡¢†8&È4”*²&4,ÀþH²5,@òƒdiŽÊƒ,!’†Ì^ÒMÖ°€²xë5,€mJ¾™5,(² âÓ° Ød$LÖ°0žµÇÈ7=iX€s)ù&kXP‚E¾‰O9·%$É7³†… KDˆ¢ò ‚¦¢œ[`òM “ÆKÅÒ° ¤$ù&kX€0ù&"zµ$À’ofµ\ÜȤñ€»ˆL“†¸@¾‰ÈVËí-€7fÄg:“%ãdµb`½¾U-X`/¼UVkà+9’Z4€˜È,9©U¸U0UËø‹xŒv˜°VŸæU‚9døLβȂcAˆî{Î1š… ÍÖúد?¾~€ó=þüßû¯–JYûq0þ@Ik¤gFÊ.Øoà•YÆ£¥š„G$„€Ïé#Bä2„t€ÅZt!áŠán˱p[‘!±ÈìÌ,+£Í¼Hö*$UÖ'÷yqYƒÔtØ–Ræy}Q…¼Õø¦tAþ÷nIpîw y«ñË -9¬ ÈÈz÷^ ½³Ï‹ïú<ï½zgŸ—•ådÀr¼cóo5þT*îön!o5~A!÷û‡×¤kÞöû¯Æål™Á ðuž¯ß¾þîOùóßÀ^½~wøñD# bÃ{Õ¿o¾}徑 ®×ÏÑÕ¿©þmôh-ñÚFo©ö…n–뽶%£ô¿§•ÖÇÏŸ×ñ—þÒ®Ô¿iŒßÇ£I¦i¾è'9OÓX<Ö²ôoc‹Lüí¿Ž¯¿»K³eÖìó„[ÅõÅÔI»BD¨P…8m…êmy«Ì6Çr§9ýP€ûé2OúÙ7ØÜ3¥õ6v«`’>§ª”Tµ‘f5Èy D¢QªBlJ–9VÅ®ríÇIA¢þ¥²¼gYcÛà´èäš)_’ÃŽµ,ÝÖ':÷²Oªlr‡!Yò©(c‹l2~äµÿ0öÉ0Ë:þbXïê\u^ô¡‡að ÏNúwU÷}ßëg[ÇœF<éó·ù–±º Ö6“í-ã•«ºåóõŸÏ%*ßýá®{ßxT?oÝ ->ýð¿ÙÊ>l½þnl¼®‰½ë¬îÏÂóÍþ¯­Snm×¹ž“¡ïÙŽÿøzqa„~rÛÒÀ±"ôŽ˜`×ldÑÀŽÉ…<µÏ³œqõÆKë5¡xÉ[‘ŒØ–vî„Ú¬«jú¥ ±Ë+·áì¼]üÉ_¿ûÇß¾Ÿý‰˜RlðÖ`p¥Oð·nÁÅé¢Ù"õ˳éÖƒ·Ž²TCü½>¦GªÌYkc> iÅ0Û17>ó€yš·ÉÂg4Y±95/4èôn«!šx­]ïçÆ:×·¯÷y;ø.Ö¹ê86½¯kXÚ´ñÏåÒÚÛ ÆŸÉ¤Âf¥,ȼ”¸®ÿ}}~kãò2K-”šñw5V•»iëÚu5¾UÌìÍVåfvÝ443L[}¸â"þú‡ß±‰–bJZŸ×ÇÃ- ½«Ëc TÇP†{Q]u5¨‡®Üž“ˆtª?•ä8Òs5ûðâ˜ßÔ k 4žáR0¹Öäö~‘ig“.^é‚ÃÆÁh»,ŸòR~¾jwöù˜ÔïSøíPâ5ËØ-`—6T‹øév;ÁK÷ÿgÔ씂tx·mÛÆiϘçeä}‚5SH׬¹[j®iA¡w·æ²†À‹ó”oΖL¦”ˆ`ålÂk–LÉ5«d¢Å(7 ù¾._šSN×ìø£mBµ†;ŠQ'„§rt_åô‰k,o§³UÇh©Vë7§l]­dÑÒ³MŠz¹ÍFÞMz½í³¯·TÚ¼®Ô£ù{O³âSºkí¦´ÒsE¶§–_\©ûÚøS½i¤êÊ>vÖìÎöU÷€§šÕ>/[²‰¹Èó­®<¯d³çi%›$Ïçêrر–ç•löcÎ’ìW¬ú<ï»›35íÎV›ÔuÎ×ÛäßÔL'ýµ²#U»i¥šM½uò]mÚ´2ÉzŽ†].å¦ÓXóR汶—³´35R”õŒ¸¶7§×ÖÔbÒrÅ™ÿv¥­)ÇU=ÆÉÖÔdèh‘ãm]³‚¯Kd}ÎW÷ùª,ÅÉò”rT–Ãàƒ<Õ#A¹;}+º½³Ïoðôp g˜‚·zFþÒå¨ít/SŽRÆü]—£.3yy, ñó‘ÊÄ¥Pz˘ïëó[ó7f#b[¯gnk -} yk‡.ôk&ä§í²}Ù¦ÏUªŠçùa‹Ó­žOŽXn¿†Æ[âEr|Üq¸`š‰£å.°<ØPT¨7Ló¾>_Ži"E ˜¶¦¾—Ò®ÏZüŽÃ³ c,7šàÞ9|ÆÂØvÒkå¦ÒÌÓÛ Ïû”n˜ô}¾´‚üÃ5Ë_¼ J?ck;שׂXºf]/_¨ÚNxͲä‘Õj%ýIfºu›èÎ>_še¢«˜ù‰ÜÝ°š¾?Ô‡1s„°1ÊCKOÏvvµßlrCËû'AÛ3VGwöˆØÖ|‹<6™åõ.e–,._2^ÿv…HW+-è‘ …T³Î2UÚJmÓŒ~Ê{æ^3ÄV[@š²º²Ò}¯jLЛFµ¢e‡Ë|õàLöfü+Yk_ClkØmå‡CZÂwË‹XV /ojŒNö¦åÁ¶\Û:·(Û/ô\ÞoÿÐ[ÕWÞ­¼'o@¬ïÃOµý‡eÓß\!#ú_ÊsËë©v!#rˆçäǃ¼žL -gÆßIbýhûXäzÉ°ÑNÀ’lÊyKT[?Žº¢œÞõ£m5Ò?¶nòØ8ÙÖ(,N¯+DôLnO–ÕÑ*Y^ñfêUê¢ÄÔdq HsÛ1àèx“삈qO´Ótƒì©Éãä• ®º4ˆ]ôYunÚþÖÖçi;¯OLΩ*U^,mC5ízÛiaªtjl-û†Pß'…t:(ÄW¶ MØÌΟvM^ÏY‰Ü ‰z»ÒÚ)¤ÓAmWG‹}§roÅ£kiš¢ SܯÐNSí=ËwVLgÁoû0“¥üÚ÷ZnÖ-GwO‡Hý$òã!f¥uXac3½™¦q;™í°—AÍÜT…£Ùhòrƒü¿BEìTAžåƒ£nRr·w\IälvòVI3=uÔ!MŽòh;©ËåÜmvÖ—c«´Ðí­»Æ6ÔIe=—&ž€T-Ùu«ëd°¦ŸŽ•3Ñ6!0Ý ÇnÊò3ö£mŸ,¹ÉÜcwSÇÎÇ™œûŽ;yÛ¬ËÛÇl¾O‹c£s²˜>·IÙÛîíÆdqPä£ ¹ÖŽNò íã8dšÇ=~9,n·è¥4‚“—§çÈËÿ‡6ßá÷ß:~ßÞÖYý¸·-|ÇÖ yödòÄ$ßI–¯ÞhÄ~l&¢§ öÙñh›5t?îý„×4ºÛì­¥ëê:w&ơÉì»éz&ÍÄ+ÄÓ™{ ó±{3özÚ⤤,ïy5©p1/—^½ïUäÔN€P;´wêãLí j<–Ë´½ì>',u¤vx‡†úXa:{¡9ãhŒÐ‚6³NTÝjƒ8OJ¶™´9Š ÙFë#æ&SªÝšZ–³ª¥ÜcŠAýeÖtœw¥+1λ’vmeÜæñz´zgsÎcuv´ì±$+´4÷îT-C£íç¦íÜÍÛ§eÐ$ÈÔŽnq×òYø÷œ³£¾0¥µj?KuÄñx¿Žš\>,_Yf9%`É·Ndâ©\òä>×ë'û*I«ŸÌ3a; …`³“/ŸÓEx£ÜfLCµ/¦Ó»«.éi -¸ø¢°þzÛòíCK©n)õË,©¢|ÙX —å;R²uúƒWTÓ¦„²üüD%¨ -endstream -endobj -713 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 712 0 R -/Resources 4 0 R -/Annots [ 714 0 R 715 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -714 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 657.273057 101.433071 638.523057 ] -/BS << -/W 0 ->> -/Dest (cb208-1) ->> -endobj -715 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 306.131993 101.433071 287.381993 ] -/BS << -/W 0 ->> -/Dest (cb211-1) ->> -endobj -716 0 obj -<< -/Filter /FlateDecode -/Length 3488 ->> -stream -xœÍ[KÇ ¾Ï¯˜s•‹EÖ 0|â „ää°K6‚U9‡üý|¬îzô wg%9ˆ´XÍ4Y¯Y¬du/=~^þ+B®”š“œ/NOÞåèñ¯µ8\4õøÍñL^²‹TS‰ç*àìr!§óã ÿ]]®ÍOûÍé_Ç>t®Ö(žy§*a—KÎ>ª -•'6T1ºœCL†*‘Ã-Öj©#f r«Ê€!µDKB1T0X2ñ­ªFÈ$– -0¨Æl¨ÈïM€x†Ž¢K0z²t\*5TS—¡£,³KY(’¥«.¥‹¥`‰Õ“¥‹ÀIÄ[:`.ÙÒ%`áÄÞÔK(™-]–ëTCW€ÞUL°øäÉÒUv±SW¡ó%—[]ðÑÅÂì-‘‹ðM6uØQX¤” ]–äc5uÀÙKÇÀ"ø±tà‰È¹c·ÝÝì™,]–²˜:`Áâ[ °øœ, -ÀrÊK„œÔ  „ÂNŠ‹B©Nrf‹BNRÍ°'èYDÀX¢$‹ ˜€E°sM°p‹ 8 X\À ,AØâÛ -¼ÅâÝŠ¯dqƒoÅS²¸€A¸€é-.`0.—$0(®T-.`p.ƒ;-.`.¶{¶¸€ÁºS°¸€A»,%Y\Àà]òx—=M°À-.;:¦Âx—Ég‹ ¼‹m,.Àª:øa²¸¾âB)Þâï†âÅâ`ê²Ãd‹ ¼"nÁŠÜàÝ}°¸@À»AB²¸ŽâÀ?ÞÌÀ» h§À‚µ¸ñºÀ`0–X\Á»TþŒ= ¶rTj²¸ ‚w©`ùL]u”aqcßÂÊŽRªDð.[\Á» Ïbq8Þ!¢‹ "x—‘ÚÒw)ToqAï"oˆÀ!°¸ñÍ‘‡ëšºê|-Åâ‚Þ-‹ …/`S—‰,.;<Ôâ‚ÞEx¯$ð®Ì$ð®¤=¦X¸‹ ×@‡mkìi$«Î0…¥ïz -[:ð®÷¹Z\À»6-.HY¡X\€Yp°¸ q5çdqAª:¤|¦.9\G‹ ²‡ ‚²L°H‹ X»ÊµX\°pä,H,.@Fà*!û6Ë`»˜õ 2E‹ Àd®T‹ rL¨Ç`UK—‚+tmê -t”-.ÈYB1Y\€ZÊlz‹ @ðNÓ%‹ :‹ rl‹ Š\ ÈÌ- ’l‹ éBGdqA À‚lÐâ‚ -ªÌä-.@„F- S[:ñZg‹ Š$§©¢ÅØ&¨4Á”¦X@`}´ØŽ øA±‰2ÇÔ G±¸ ” åf±¸ ` >X\ßÔz3[\P=° 㳸 úä4M¶¸4÷T*¨7ƒX\€}©õf±¸ ²G½‰ÏØ·•“Ö›Ùâ‚*À‘¢˜:`AÒT} NB½ v²t XB‹ jJZo‹ f ÞDFo ­7³y\ÞÌdéÀ»ÈL“Ÿ@½‰ÌÖªí=ˆ7fäg¶²hÅYÌꈭæ‘÷"ZeóÌÀƒ|µF2 •eIæ©Â*”äÍcþE>&7œ°>­GT î3KÖ›¬Ø‚ì~Ô³?Ñl;ûå§Ó7¾çŸþ}zÿ›vRÖÎÏ’´°ÈõØë ¾rmù»Wô­f¡ ›²}´¦ý³Ë£÷R¶¶p|/o79ìÓ®oä{[y7Ç} y¿}ö~€Ó~©A G‚8%ü>ìãÝisÀ»cSÝuŸ6Vï×ð|÷ó›×/²t]-­Æƒ?\+nÌaÄ;ˆËÔ7#¦¥ïbØ>G»ŽËœýSøǧuÚÏ¿ÅBËb¤e\=ÄÓEÒ>—Ý(i·FZÍ û%¿#ïvà}£.ҾЮÛqÀMȱÚ9ÞÍ8íyÇ“šMžÃ‘òS8ü¼—¶Ð‡Ðu2±éynòR5ÆVl:¾ÞŸöŸŽ¿8}ÝÆoŽõ~ŸëaŸ—®yÃ9ìO»í÷õŠ»=¯ûÞÌ™æ=ÝlËÑm[–ëûíøu|Ë~݇e÷•õ6cÞ2ï©ß‡u›oÎõþŸ¦_¶}s™÷¬ý»}Ǹq—…ë}1exô{Ýï+n×ÍF—¬š¾ Íùú¾¦ðœ-ƒ¿¶eëÝÖŸú|²­Ój»&·Û M%ð½¹þÏâÓ£æ>Ëþ}üîM ‰<“ =ÇÊÔv®Éšp¢î)No|¹„ÑOíóY É=]ÐJ÷븭ŒîúŠJ÷&¤JŸu3͸$Žoa'ËÑbêŸ^ÿõÏ߯1U—)vJïÔ¿»Í¤üm žœE€¯zBv5ÝF6Û([tìŸÏPŒ¡t‡_Xc>L´ê(}Ž˜{à¹Êò2oÇR®dc#õè;åòþh!Yt½ÝèGó>·v;ÞèónêçfÚuK_š÷Ð7ì3£ÊNµ0Üü’ªÕÓóÎü²>ÿogxYOë(€åÌÇDíè -c ËÑ%&=ѯ»PX–·.®€e¢w† ¾=.ßÐ_¹äpËEÓQæÁk\¶X ßÃÃ×澶kêÈšZéÿpä³ÜuÍöùz\“²À|d»æÝ]ýu¸ïªóc9GE§Ö0#1ëøBfí4ì¿ÝSâê×4ŸJ½oS[}ô\õ!ÏÕ¤-ž=ê±^ÝÕzˆ\}ºã‚/ìó%…óÿ"âëãâgE€©jÖ'O9fx·"ŽfáÔ*œöÚ¯.‡\uoÓz©áF½½×dý ª9wZj®ºÉÃ8‹Xˆ6ùqÆÐk·6ßþ)–ºø0þ35帇Øïá»ë¥üxJí©µ7|*Ž‚¾yNâX×a¿è3êKGÍØt~¢g{¡ôãhµ¿cêõµ}丽€º\ìí?¶Eÿp"Í…û9Y{Ì71öc¼?žô}@1Ä"ó°íŽX…ûW?Ægw]vQdW¯[búýë<àÓ½¹} ±ßö½›>§ßq{R±MlÑ3Ñ­XïÞ;ÚÅúNeq¼_¥tY‡p(kÛ9àìxWLìa¼úeº)Òñ>ƒ¦KS8 ¯¦£eù{Û—å|Nx9aZ¾–>šR}“«Õ­üñªiì-Ç‚ÈX'Ct9ÂW¾-XÌ¡_VMŸ‡oÂÒIF»ÚÛ¢ËÉl·ÇJåѪ̮µ[~B™®ÎeÞ]–Cð¬/‰/{Äsÿzë¡éXkšÓoÂðrBˆ‘ñã)f£uœ\ácw½UÈ˸C\üô—)Í¥› -[³Ëôi¢þ¿QERežö…d¸”¾0¯æ$º7‡øh¤Užëˆe’<Û.梜‡Ï®öBú`´WÒ­‡ÅÒÅd‹<×OIjÇQLÃë†X)¬Ûgpå*ô¦›â8\ùñ ž8ãتÑÉ-îǸSœw>®âí ®ˬ¥¢þukÒ[ô²'û'Ùóá¥ýü|øà -endstream -endobj -717 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 716 0 R -/Resources 4 0 R -/Annots [ 718 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -718 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 642.273057 101.433071 623.523057 ] -/BS << -/W 0 ->> -/Dest (cb214-1) ->> -endobj -719 0 obj -<< -/Filter /FlateDecode -/Length 4874 ->> -stream -xœí]K$Ém¾×¯¨³€MŃñ„=°/$Á ¬ƒáCOí´£GÀÈÿ}ŒŒ3“õèž‘Õ°f=UIƃÁ ¿ £¢¢íÙàç‹ÿ2Ù%ç’"/ŸO_NfIÁà_-±y¨ìŸñ›ÂÙJK°%æpþÌŸ–T‚Kñür»GYüåôÇ_þ²­cÏn)%:ÎýÄ,òKÊ)™ ° -XÆz…Â’’ QaE»`ˆ¥h,ˆ9:²Ä ’ƒÆ‚ä\VXbxJÖYb¸dIcA [BRXÖ@ëŒQyÄy§ðlX"”5ž³KÌÅ•—À³Éi<ȫñÊc Yãd ÅX K°DFãAò9i¼Y|ôFåA—“×x ²8h§(¼ Y`]YåAÕxÅ/¡d"•WÀ39å#Ï™°„ì½ÑxÖ.¶éU< -““Âs%šPTd Þ8ç! áGã'‚O9+ÞêØ»½ñVãÈâ\"•Y0ù¸YLŠÀƒÆ% \J G¸ìÊ”5p¹,”’×`À•°P,IÃo,xÎj@à d 5$ð²`u1H¡ñ€»Æb†5pטT4,ˆÀ]^65,ˆ‰ .kXDÁN˜i))E b1à!äSyqÁsа  KåAJ¤aë¥ø’5,H²xç4,H² dаÁR,¢o5=€,05? È‚HQà ْ ‘†)DäcЪƋnÉ p­ò2x6iX-XŠ­†È¥– §×°¿p¸¤aÄÏ’†©@¸Š†XŠ–ì™k< YdkX€H Š‰ ‡ÉÆÁ3FÂb3òMGÀ/9ßÌoo"ÅSü¶øÈùfÒ° d QTdAÐT¿&!ß:i¼Y\! JŒœof f ßDD¯n dÎ7“º]Ü Ȥñ€»ˆL£†x@¾‰ÈVËí €7$Äg:3sÆ™Õè X[Õ-ìÅj•Ô=ðåIÝ4€˜È,sTw °¬‚iºm`€¿ˆÇè€ ëî“Ü¢Š0‡„53'd[¢û‘sÌbþJ±uì¯:ýú ‹ïùOÿ}zþUÝ)ë?l0Ùv³ýù… =D\ùäŒÈtGö€é^à÷ĺkQ‰—“ #Ê:X¶Ø—-Y]®tÑE¬ÙJ Ùõ"F"È/'$ ÈHwäFÙÙ$ÆÙî$Ï‘¼êôìjjGÓïåôGLôÙÆÔ&@%aYãªwa{ÖTÛ@ žÊTæT[,.më%Øívª1Ó<×r†Ä$kÝ¿5Êá -¥KrµX1Ĥ›P’ %¸-™³ç^ïÄ MòÖ„$ÁO7–ÑåeS1!£›&”QzOqZËèN¥ EÕ„uzv5µ£é÷òÝqÿ.ŽË‘:ÿ;ß( +µ3ÇðYwÈ«N&õ&uÿÝù¾;ß?˜óEþ—ÎÇ7¯p>!°±PÄÖ²¿;ßwçûî|¯t>ÕÉ^r^qÈïŽ÷ÝñþÁï5!'’ÂÅSäʉ÷]Ã*n2KMéìÊâ¢ã å–U¾¥ÎíL”·Ö] roô¼Ôz»Ô -Úô,ò^ã|Te}uo$ëÔQ†?Oâ}Ô‡õú`o®WÙï£z}°Î7—•?«ò)’Ýͧ*ä½Âw¥{ȧŽB&žÓC¿ÞF‘ë+Â>Zé -}Üøû§'xJ=]‡×B Q®Ÿ òqÄŸmœ?|>ýú§ù×ßþÛϘ­ϧÿø1Â@`cë±cÜ'cB6†ðLüú¼¾†ôãž?üt²°3Næý0…7Xà ìËùÌ]n{ùñû›öžZy¥Û^c{íô& —…sú¸Ò¡ûú| ·²•ßÛt»>.ø5‚ž·}C¼µ~§‹öGÛ,ד(cš¦ì¬SË^f»UÖ8å”òWY×qÙë˜Ê*ϘìƒÛ7¡çÝxÆØÍÔeq–éÒf;6¢ìÖeÛûömŽBlÖó¼¶g9¶=ÜÄGšŽíT]týÆ:Ž[r$wM3ÇÂúãã¹pÿñ¶Mð±ÞÞ—åçês—öš„/¥Û|ënêÇôsc<Âœ_ͺo¹.¦©“×ûùM°úžÿÔ1—iÛlvTm«Ï×/·ÇKý¹cZÇÏ,æÆ®óv¶Üýô5í_×ë-=ѧߌC7û¡¼ëÇ5[”8ßãþP³ ?òÞD‹/5?ã¯Øä³Ëg‹&©"ym7åt-áX*ÃxÑ‘ãC8f ¦dCû,ÄRïuUÍx´> ysæï:8¿&%¿ûéßÿð{™”°Y´T.=toK¡ú: W;,q1…cî:\Ã9êcj]œ´Õ,Úû.G~Þ>Ã䚘ȑ–±TÛàÓ¸Éúl“šŽi†RõJ°x͈BÝ”ª))e·XO)Æ;Fô`wcD–žS(.¾Úˆ¨çG´ŸÄ«;Ã'7LAn°íxcL®¯}=wôa8—­aÕ:a'2~ƦÕù4ùœÅ×¾Z;l"£®mãìò¼ÖÈ[Þ~³^ÏÝGc~úñqW߆b°Nw•¶^…»‰5£¯'z^zÌûø+*…ƒï:­vtÅÅb=çWÝY8i¬G`oºØƒu¾f£èoái‘ _÷4óÔ#«­õÌ]ˆ¶ûñ$ ö²-3èÝâÄnžÝ^¶[e·¶N\ž×Xc±·¬qXZšbµ~i2úÌjá<-¥Þ­ÛvxËo[¬Vø ºÉî®%>Xç½Ybòö–%Ö¼æ²æ'ݼ×i]#ÿ¹¬ùË‚—ÚïeKÓÊr›•&úéØ=pÒm-z¬''ò2œ³Y¡ç¾ÚèBþš§5v¯å> -™\{¯ÛS=¥¼Xêo̽àáÁ:ïÄž¦Øü¥‡1„o¿¿ìÞ‡­Þß²–Ê´æEeÚÁ@¬±·¼m£–÷zÙºæËx§îßìP·ÉèÌÛc–7Åî -‚ʼnFžg˜ -Ÿç¾mqÕyw—o"Øk¬àQ د±Ýâ6‘ÙóÑ´tÐ>Í6Ânö7k·ˆ>Gtxi–—¯ËDYY÷«ûŒ«oúÕm²¼«óÞl2›Û(Øí¥ÇSiûz˜‹¸}­z÷íS?åõ`—Wⶃ_ôç§c,8êìdÝgR Š¯ÈºîHŠ÷—é¯×?eÔÑ0ÏØŒ\@ˆÔ¦Ü±¼ë¼;Ë»Ï íîÖ‘í~îgLiSË•Bj{rM³_lg‘2OeD~>ÖÐǸë:ÙfØŽAëëš‘ÚÃkíƒuÞuÝŽînY˜ýÒ/¯°€ÿï1¨ᯰ€ƒ%w‹|R2Š]Œ £ÀØÊÝ+Û¯kòµ8£·³&µ©kV+¢=C‹³”‰îYícuÞÕÞŽ÷8t/’ûšÿï²/ù*kQô!£¾nµWr®}_{ë¦{V)ã=þª4¦<ݳÊÇê¼7«,·cÄpmgCZ敬bM§}¸ºÕìqµ¿;£û˜ôJÛ|9Sõž~ÊÀ¶•¸‹aéJûfYtk+bÏŽOˆç\/3¸imÖywÖv'.ÜGû=3îÖ!²_÷53a•lc‡7÷,q“¤ërj¹ó´¸TÛÚdÀ]W~7ŽË±Ícʺ½ÌǶïåæ¯&Ûµý"âG¾þ B¼s>XçÝYî˜óÓŠ#t_o)¤¬ž_Ý–ÀÀƒå߈Eÿ¯­Ý‰z£Í}Þ½óîG²¤ò†xóÁ:ïÎb‹7¿Ö¢FªâÆÿcÏèíÈHh¿î¨äËñc K¨fvÏ2«óÞ,Z¸ ¦)4·h(é‡L\­0eºtH­:=ìè} ~n×µ£ÛrspÿÍûÿöæøs»b&Ì\|ÅW6‹k† Þ-rÇäŽ÷Ûôïì®CÙ~r&ùniŸÏ–U_Çtís{>ËŸØŽïÀÄöI®8ÍZc|VC;9ìúçý5Û\Mq|“"t$XU^?ûï'KŸ×)gºßr0ó”J4‡Ó⵿v’¼ü'ŽGûý«gsZ¹´,¼Žá0•_N±^-fëõ’&æÅ:¾úiá¶S9óufÙx²}êlU¶=_©Yoíþ2Jµ‹¼ ßíÉ÷º­·|‹‡VþKôÏ'[haýׯ½ÔKwM%g×&ùåħI!ÍïÎÜ!3±½5£-²ã«8v°ò·;w%}XL{;¿¯ÃÖ·.ôÑpýЫñeXdz!_¯ƒZ;Vˆ¨í‘Ì£7‹md¾¸:/¾=Å! -ÂÒ2ˆ³A’egƒ³â]²õFÑhDw“ì¨Ëcù¢Ã¡º8‰Ct©:+¦¿—uILç-âå„nýžú¢RùºÜÞT×®3ƒæ…PePC/9&„Æ<)$þúïøƒéM&sðŬñ¥ƒ+1wC¢Q®ôr -érR˵Ö˜©4JåYµtÍOQ¦©û<¯Ð.â;m‰oâ>b|»%ó×±Æ\›,ìƈ—–:ëW¾¥ÃÄ -ºéI¢ír6Ó^&5å®*¸f§ñ•müÿ -aPyêKäøÚÅù4;aßä­’$=Ô!M–Ò,+ÔeS6+õ…¨A)Ï ;Jm¨Be‚žJAj€Ç–l‡Õ 2CX×ÏÀJI4]t7Éa˜òˆ<å ÃUƒE+Ì=Œv'9ì|‘ä4fÜòšCÞÑ®°ùN¼Ô…öd6ýÜ;™dgÆj7;Ä)@a’'Œ¾Tr÷ÖNüwúÛédÚŠ{9ýùT—9çƒãSÝÑ[¾ZF^öü?(ó~ÿëDçß÷» -ÖuÜ™n`ë†,W2þùA2ÿANn#ˆŽìO²Í³lèÖ0Öqç^Ó¬n&°÷’v ¨\I S‡‚ì†é:ë…f âe§ÅÈ|íÜÌ{}ô}pèÆóí•]*<Èáà1OžŸKòY4‰'.:—tpɉŠxâQtX¦d1Ù e'Cõ,?É©ðÜ—xa?úÉ£¹Ìʦ;­ŽBAìåƒl˜¬ì´läQ„~³{}õ¼7‘c÷¦hÔIcê<êëcÜ>Ž5Ç×#÷Ýy§†F[^øžAÎtZÐæAÚ¨ æY­²Sê´M§}¡pšé´Ñbê2ÅV­«¥Ž8«ZJ#¦˜Ô?KM9+C‰AÎJ<”åvûŠ7¢%?  Í9ÍÑ™YrÄ’Y¡EY{Pi–ôv웶}÷Õö,†AS8#S3«…CÉ7áß[|G½pC+ÕêîˆÍóÖ^,Ôd“w¾þ!&“c†ߥŸ(‹í’W×y륱²'LÇBÞ›dùOjé"¼€Qî3DSýÏméÕŠQ‡ô:\½þX½ÅÖ¿©R÷-ë&ÑxLœ*òŸPòôfþËÉ\¹.…ÚoÞl¡ÔŸÿûmÙ< -endstream -endobj -720 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 719 0 R -/Resources 4 0 R -/Annots [ 721 0 R 722 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -721 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 488.554475 101.433071 469.804475 ] -/BS << -/W 0 ->> -/Dest (cb217-1) ->> -endobj -722 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 427.607453 101.433071 408.857453 ] -/BS << -/W 0 ->> -/Dest (cb218-1) ->> -endobj -723 0 obj -<< -/Filter /FlateDecode -/Length 4751 ->> -stream -xœå]K$Ém¾×¯¨³€MÅû:°-,à ìÇžÚ F¯€•þûþf$++»§%—ìiôT%&ƒüHF>Z_~¾Óø/9½¤”cp×ÛÏ—_.j‰^á_i±9(ìðýU+¯sHþú3l\bö&†ëëÿM‡¼ùëå?~uùÓ¶¾š%ç`´¿þù ±œ]bŠQy•ÁRÚ -,ï—+觘³Ä‚>:ãö¬1\N^bA gLX bXµÝ³2Ä0Q;‰1töQ`i9´QJäAåœ5Oû%@éA⽄”My<ijv Ñi¯%^^B>I<Y|VZâyÈâµsJâAgS”x²Ø`•Èƒ,&E+ñ"d1ÐNx ²Àº’ȃ,*(-ñ²]|NΉ¼ žJ1íyFùÅ'k•ÄÓzñ°M+òàQX¤ž,Aù,ò ‹·ÊH< Y~$pÂÛ˜’à­†¼Û*«%ž‡,ÆD'ò _B ‹ŠA‚xÐB¸$a€‰qqÙ8 L²‹K.I(`R^\ŒV‚“ýâBŽX¥Á3Z« ‹wAB«!‹ƒçŠ<Èb³“°ÀÈ‚X YŒ³X ­ƒµHX`·Ne-aÞ:¥ƒ„€ 1•„ˆkSpX@.L)KX`¹Ø)aèÂÝ£„¨k}0XÀ®u)HX`»Öi%aîZ‹ž"²À€%,:.V'+aîZ­¢„¸ 72`UØa°¶²˜””„¸k’r8உ"/.ðA+aîS"7p×xe$,pÀ]ãL°†²”˜ w!^N VÂÄ{ðŒ•°ƒ-i‰„¸«3ŸàÓ@«E§$,ðÀ]°|"//:Bã‚ßBË‹!KXà»TVÂÜ|& €ñ "ª‘°ÀwµE¤–xÀ]m²’°Àw‘7x `‹F aâÛ¢LWäåEå”$,ðÀ]À’‘°QxQ H"òâCÒŒ…p™¢„@²%eç$,ˆ> ƒV%^0KŠ€k‘—ÀÓQ‚Ý‚P¬%,@-µ$8½„ø…Ò% xÚIX3d«HX€P´$ƒÌ\âiÈ‚$[Âdºài-aA2Ù „É$T™AIX€Zª–xNQ™$,H.,”*JX7A¥ ¤y0!aÖ‡Šm/aŠ›(sDd±ÞIX’¡r3IXd1ÊHXÛ¤z3JXdAÆ'aAVa¡4YÂÀ8xJIXuB½iœ„ðKª7“„Ù*Ô›(ñ¿Í6P½%,Ȳx¤("² iÊ‚ß“Po$^€,&; rTo& f ÞDF/n $ª7£¸]ÜõÈ$ñ€»ÈLƒ„8@½‰ÌVªí€×Gäg23QÅ™Äèõˆ­â–ö"ZEqÏ@|©F7 &*ËÄ]„U0µ· ðù˜ÛaºûÄ·¨Ì!"f¦H'™áÙ}¯9F3{§Ùº?öç?\~ý‚à{ýÃ_._UvÊÚF‚PZ;äWp“±ÜÙ·Æé[ÈâM ’½©1MóKë6‰…€Gÿ®û/Y!* ‹Þ¨„UÿQSâµN ì)£;”Á¨ EÉ!ë›û<9"'È×þE˲ñy‡’$YßÜçƒde:‚ó¡hFŠù½žëóñzeóžÖë¹>.kBñ±Êmç…|ÔøcjÌ›ý÷ó–%µñ’¬'û|°È¶ãO¦%(n‚P‹è– ßÓ.%TLâ7vù@iäÙml6”— Ø ¢„ÊxãütAˆªR¬W?ú!™²FÊaüõÓÏ—_ÿË÷ÿþ¯¿ÿö÷éëåÇPÊ«ß|§ñénë§×ì{åù´=Få}íWy.ÔϯõÓMãÖ¾Fíi½í ‰•¯cj=Úø¼{7§ŸèiÌl[iä*á÷ëz¾ÞÔvq:§¿… vèT«ßüçõÓ÷—ú´5ÀR¤­(YN…6{®Ë·Øì_ÃTs>2Uw öb’u©ü [*¦Þ¶Ì´äÅ ùxuœ6F[®íò€þ™™ÇÜ·™ÚøŸj{¿•·¸Wcö1âösv¯~na2#ß—Õ<ɤ6ŸîœyÑõ¶p(evɹöu²Ï³˜6‡`XVÎLlåšU©íêl¬+³UóÓjÞ¦Õæ ðŒÌdIue¬ôËmw¨¹7Oh²½TyÌèÚ9½ƒÚm;GéÓæslüæUŸ÷ºÑæŽUÚa©cVÉPñwl•çú»»÷e& ¨î^-Úñn²âz«¹Ë)§-oǪ‡·È×qïýxÖ1Ÿ×™ŠÂ¾cOïdŸg³Bû œ²üÝ^ÜTH» ÅÊN\´Ø`å\/JÕÀ{ª’7V?›]é;È÷=»¼ÌK yS•ÓÆñÝ|ä‹òùܵT–+ÒH£üòdŸ§³ÔùåKÝDÓÿC»0÷<Ó³s,r¼ŒLâ]»2AÖç)}G¾y²ÏÓYçƒ|ó®_ïóþzÜ¥žÞþ¿[ø ¹aŽK¹éäÑes]žÎþì ~ÃUŒac]Çý7’./dY]º~ÈîYrÎÑCtÚ¤"Ý÷ÿü»ßþ“®ì²P^éÇÌí»BŽkº…L¹_· ’rmëiWçs³ÖõxG¯mû5 ÇÆbíFîMW³š:f96[¹Öê‘W•c®ÒϯÖ:n Yùý¼ª¬®åÔEߧ¸‚äº2Û±‡WmuÓÚ};² -Ð^˜Yžßø¥MÿeôWŸ×6\§½b:L«ŽTµ,ê)~n^Ñ–ž­çfëzû@sT9Ȇ~Ú,ÝFMêíÇ)çÞÖ&”ó;’æ{r¨q.´v[©5Ó^6zd3(„Æ.ëpk5Ëzž$c³¢^³«ÒöKÕE•©¬[c†,k”`c´ãÚG}­tvN[?bv=÷íöÔì­Éq¨[?Ÿÿê“c¾"û]?l:®sª•ÆýËd_ê¹y6v½r¥õtOôª ˜¼®ŽÍt²õîûU¶pOï‡z‹³ÞvëóÀ.úúøY×s9ZßÁÛ¹†­F ß.çù㊡,#­Ç‘Œ%JÕ9Ú­a-¸£³¼³µy~·ŽYäÓ{ß)z«Ç}½\•éså¿0›ý2ä/<½·Ïbõ\¶~sÇÊvmšŽÃðÙÙ6|“‡¯ÿ_hþßò™Î¯zºk ßàãïòÐéÛª »jk#ÄY¦§–ßņvÜp¯agëÖ×q²÷æ£GúœÇ¤×÷ãС>²›í¦Ú2­M§sCÏ)[n¡§WXèh’¼ãædŸwÝvÿÅ‹ïòZ·$ïábñRB„p+h +–•ë"Ü0‡Eez xšðÇi;£l‡A{sñ$–I¢mÄè—ì±þA,Œ$C)Îz'™»gDIé%åòØ€KæÜÍ 'û<Ñ¡ZG|¼ÙˆÄ;ŽÉ(·Ä¬2r÷íÄcú;Úmùß¹ÏQÞQ]%Ñ (trÊÈ®Rã•où‹-NÈuã¾.£W£dz Á4i±£{.©îB¯/‚/h‚ßc;×çIöš†§åxäiê¥þÚ­…Ï×Çùbó®ònOt¾æ2[ñy;ÊHãì¨ÛH9ÛzÅÙQÍ]‹ý$1þ—›x²ÇŠNÙP¶C®wçî—=ÙçÙlˆe>°¡ù ”fCVz>`¾“™¡Íº#0öùSóCW~²±ÝÕìi>>6¥ ß:îFÖ£½ïƒ~¥š™î¯óv/w©–ïÜ%ÜkuúšâÉ>Ob…ìAe“† ufÕo)ÔA0…;%7·ݦ>G—~¦Ûz6—©§Fº[4s¸M¦ûIJ7ÅêƒòÑcÕ¼§•$¯{î¾ÛßÃ&Ìmÿ”~{âõª§gŒ·Ñ$äFzõ–UåH/•¸KhÏ„ríGêN{‹|¤˜‘^ͨ´i1¨î=•em{ u?©í”xÔö¾ª~çªé{Ú,ç j·ïXæ«{’¥Œ`{R}ü¶ÏÁäÙìg庌åvKùË%”¤èò’,€ü†Þ¢y n¡­¢˜¯ôR–¤¬ÓmétQ¶¾ÞéYÞ=úKoU_Gªè eôvšõ]¥ì ¶ÿ¥,úÏM%¹ZÑ`}u *däc~&¿^¨†vÙ¹q%晈õ«êc9Ý/ì4Ú  —<·DÜTõë¸úCÁtýj|;êï[7z¥‡S­‘-/µX'ˆèôžLg¯]ÉôúÍ´Øzº(>4™A:Þv 8:>$k«ÑM7ÈÆ5y4½®©«. b«N³åomMdËyD¼]0­©¯"•^ú׆jÚ5ªÓ,*wªo-û‚¸¾Névˆß©6´Ãbv>[5zuÒJLÍ\o—[;t»ˆíêh¾¯Tì­Òèš›æ‡(ÃÔiãõ>íÆ®FzŸ0ó„wÓÖ“éâ[_k•˜Ý˜ñvAˆq;òëÅG¡µX¡|3=N´lÜNNjØË RŽ¼~ƒk6½x†þ_¡Âw*!Où¢]7)zyÔ8“ovòVIœ:ê8IÚÅÑ–©KÇØm–ëK'%´'Ðí­»Æ6T¦2F¹‰G ÕÁcKÖÝê:™ ¬é§c%'ª&¦dßMùuCrúîª^õÉ‚fæîû¸ƒ:v¾rrì+®éM`]Þ>.³ùF¼•Àæf2™~j“ ²Q=ÚÉqP䣯…ܼµ£SyH­~N&EÜÛå—æŒõ†öƒÕôbKžyéë£Í÷øý¯‹»þ¾½IiãFu´0[7dÉè5Èé$™^‹ÜˆÝmÑ8ûƒ¬Óhë›5ô8n Ãk7º«ì­¥îª;—ýÐ!#›nºF[¦@¼MZì‰Ì·®ÍØë¡m'‡i,½ƒ«I…~:8LƒgGH¾²!qDMGH×Ö|P² ól#&$Ÿ¤«žäw|),ÍÅXA‡vðܳ|èF+gÁš ¶ü€ì4Ÿ4oä„~·{}óºW‘Có¢vhïÔWNí'FÔx2Ֆð=ì1Ç– Üæ¼CC},Ë|Ïö$g¸ÆH-Üæ€Û(£¦Ñ­6ð|R×h›I[ èpšj´>bl2…Ú­©¥œqµ{N1¨äšö|Uº=_•°kK㶈׳%Û ÏÍ9Ž³S£eÏ%“@ ¼w§ºÑÒ6Ú~n·»EÛ+; 7 #3U£›ßµ|þ½ÇwÖ>ÓN„Ôªý”ÝÆ»¨ŽÖØòç$T -âèÀÑ%¶]òæ>Çû'û]’¶ÂgÂr,ÎZ5ýaY„W0òcªýѹ[Vâ)½MÂöЀ9µÛ*o†/{–e“¨F*éAXJz½¿:*-ßëêïv ¥üüÿeˆ -endstream -endobj -724 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 723 0 R -/Resources 4 0 R -/Annots [ 725 0 R 726 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -725 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 303.873057 101.433071 285.123057 ] -/BS << -/W 0 ->> -/Dest (cb221-1) ->> -endobj -726 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 242.926036 101.433071 224.176036 ] -/BS << -/W 0 ->> -/Dest (cb222-1) ->> -endobj -727 0 obj -<< -/Filter /FlateDecode -/Length 4359 ->> -stream -xœí]K$· ¾÷¯èsËzP/ ð!@d1°HFëÞÁ8€Cþ~>ªôª*vwÍì:éëÁìt‘zPù‘R©esÖøùÂàŸDF¥”c óåÇÓO'­¢×ø¯”X=ö7øþl4EåMÉŸd‚‹*foc8?ŸðÏæq.þ|úë¯NÿX×1g«rÖøóϘENÅ£ö+ƒ¥XÞ«­+…!æ,± †diÏŠƒrò bµI`%ˆá(·geˆa£!‰1LöQ` 9ŒÕZäAMä¬À3^(=H"?“™‰WœIÜ1Ѐ^Ø*nh`/¢U÷ 4À—×Hâ¦ÄÄÊ2q×aL£Åm üE>F;LXvŸæ-ªsˆˆ™)ò 3Ü‚Ý÷5Ç(æ®[öÇ~þþôå;ßó÷ÿ<=ýªì”µƒ¡”&äWp›1ÝÙ·Áð,ÛÛ"${Qaî6æ!–Öm6 ÿ;ï?¬d…¨,,jc%xõ÷Fâµt ì)­–ÁXXŽ’CÖ×¹+rDNÏýƒ‘et4õ;”$Éúâ:ŸHÖIGp>,š‘b¾@¯Çê|z½NýÖë±:Ÿ\Ö€ÅÄiZ÷+ -y¯ð§uªÑ/â¢Ô/%)ï–þÄbº1‡¬öâœÞ"‡`dN -\N¢&Y_Xå -¬•åÜ: ŒË–s¬"Y;Sáß¼=!,U)–7ýqÙ±Ašaýùí§/ÿøæ/úúØÜÛ§Ó·¿Öˆ#_}aøï;ü¦å³GðqùÌûò>Ô2ïëß¼§¡ª¦V¾¶Ùí´^·¶¯Ý賕÷zÝŸy?ÊÑS-cëó¦¯Æïåúü<¶.ûwƒÖÆ:÷Éczm{+ÙÃf iÝ箿iãoç·oN¿}»6ϲB]Ì‹k(ݱècU>Æ¢ CÎù¦!¿Ð0n\Ÿ·icã«ÉÝLå©ý´™ø­ñoú½jʾԡl|Ìݶ à7rÍ`¢¿M˜ßòuˆe£¡Ì™öM>XçÑŒØØÛpüðòê™ü€ßËd5RÛa-Siÿ2BG·:'[ ¡b¶­gI0ì!gÐÇ=á”òÙ^ÝP¤²Y'Ë‹ÑÛöz¬ÎÃÙ+ÝDÝÃhô„Û†Éng"¢¶³I1zýÙf7Ñaëc+›±5Õ¡ÌVqâu;§!wï÷;As“,¾öu7U»‚®S{ØZÕy8k½“ì¾Ô²>Lè¹E´§MÝVï@²û9:ïòf#5Xf>ä{Öy¬ÎÃYç ö?e+kê³Ò·ùx4YY=¦?N;7%26•Ñ¼ùÝ~ÿçi4&.½?z6¾²‰>¢uT¹$K¸”õœÕ|×luyÞÑkYÖÊX¬Á…×û‰Ÿêç\Ë=vJ?~±å.“Þ·UÊ^†üE®0dêe¨Ò‹®)7ÌÊý¶ÝÕ_?ecõ¹Œ¡ê§CÃ-McˆS=½.O,kµªha–Éi~ÇáøŒ[•ÉÕyòœ¦E‡EÇï÷íÇ/€øѾ]ǵè©èê–.]“C±¬ôÁsddãÃfA“†ìÛ.özYì¤ÈYÇ[Ú4-û]ú*e?T]T™š]u[C–n{­­->Uú4¦µýO6+Øñ2ßÍ.›7uë·ã/:Lëþ$ªý…kz®ý¢ŽAÃþyô½.mðJƒmÖÞÏvüO ïš<¥Žæq[ÿŠÛF¥Ì/ýÞÏá­ñ3ú0TmY mÒR®ôiö¾Þl°Ùc¯Ýþ»ZÆë©KÞuCOdvz2ÕÖ§X°µ¡WæÒS‘çݧ}ý«G;M=.6ßÀ8xÖsìÇïÓò×q»ø–«Ï2¸÷ë oWs‡%×]Ǽ"k÷Ç›ºÚáûõœAʬÞÇ -ÊYÇ©÷ý}4Ý…á‹[Ûð »&^³ïæ×~ŽÇqèé—ðñŽ{¹ÙëM½ïãJZbC³!ï¦9|{¼ÔžžÕ±6~Azè¤ÇZûÀ‘öïéõõØrSg™6:³5ÏØÝò™íº -cUÁ•—•üe]B2yçM×±:¯zÁù T¾ËëH%ïË₪À¾_/¦ŠY°y¸ié±LÂÕsP:óËM‡ßNKú 6 Ú‹hâRL´ ½ÊóÄÅ—d(-‘‘’ÏkFÄÇPS.¯i)YeÅîÑÁ:cDÁeLL&¼ØˆÄ]ÄÛÆd5©˜uÆZcÝñʘþgöt^aäÓ.ÄÕzÂ[õy×Bt•ÀGýM"meW©qÈ·|lŠ-NÈëÜ}Üç/¡d>î½é´ØÑ5ƒTwá/ŠÁŒ¾·ÿu°Îƒì OËñ–§éwõ×­-|ì®Lh×uïýÓÑ/…\>}ÃŽºÄ‘³µõY·£š»ûIbü·üš({Ìè¦Ã[6ƒëëIiOçû¦ ¬óh6±ˆ¾aCù66ä„÷Ý&.{´Y½wÌ˪ïcq=Zòêã!’ß:ª3¿s›À´¿n/wY_ÙÉ)5‹òüõ[„‘pï-þÁ:b…ãv7sª1ô]Ý÷v›¿Tµ™ë>yl{ãKø”o4õÄÛÄã2ûvÝW÷Šf ùÞI¶Äd:¶>åw°ÎÃY¢=„‡Ÿ÷½ ƒèxŸyø|ß±:gËþ&ª~>á÷:Ÿz„~)Lèzð„ßÁ:gÅw2ÔÏ'ü®"ñãœðKS^zôÌÔÁ:f¯ùv.ûù„ߦMó™oˆHÚ‘iSgŠ²ÍùJÍrâO½T½QóuI|UÆrqâôPËÿT&ýÇ“á·ÖzqŒå3]Èɶ‡A~>ñkfÈDãPä2ëGÝÛ"ÓÏX6ÚøUÞ–t^éúqÄäýæå£õm4\ß·j|¿éVÈ•oØ/ DÔ fOæÑke*Ù sõ)tQ|h2ƒ8¤¹ìhpT¼K6NOD¿'ê©»A¶Ôä1|wLW]Ä.ú¬:3M+kã4·ˆ—ºu[ê³HåÈZSM»Vwš›„Êê[É>!ÔçI ]Nñ Ýš&LfçO³Æ÷¸,ÄÔ ‰z¹ÜÊ ¤ËI,W[ó}¦b/•FÕÜ4?D¦Îç“®Ó.ÓaåÈ—›N>‚eƒmó0“ù> -endobj -729 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 453.873057 101.433071 435.123057 ] -/BS << -/W 0 ->> -/Dest (cb225-1) ->> -endobj -730 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 392.926036 101.433071 374.176036 ] -/BS << -/W 0 ->> -/Dest (cb226-1) ->> -endobj -731 0 obj -<< -/Filter /FlateDecode -/Length 4799 ->> -stream -xœÝ]KìH±Þׯ¨5Ò˜|?¤Ñ,1ÄHGÜÅ‹î:§A¨i`qÿþý"°v¹»g˜‚ÓêSåˆ|DFF|‘¶³õUáç+ÿ’ÓSJ9w½}ùᢦèþ•‹‹Âþ¿Ñ_µrqò:‡ä¯ßÁÆ)fob¸¾^ðßê’½üÏ/.ÿXÖÑW3åŒö×~!–³SL1*/°2XJ[åý£ñA`=aˆ9K,ˆá£3nËŠÃåä%ÄpÆ$• †uQÛ-+C µ“XCg–VC¥DQÎY#ð´Ÿ”$žÑSHÙd‘ÁÓÑHh%,pÀ]ã1iå„¸kœ ÀP&à£à. ÄËádÀJX€õ‘—'¡qÁo¡åI‡%,ðÀ],TVÂÜ|& €ñVT#aîj‹•ZâwµÉJÂÜEÜà%,€9L€„Xß&­`º"/O*§$aî–Œ„X…'•€$"/N0$-aÀxRˆC%,À],ïY‚ÜUÞZ pW9„="²Ød$,@\ÜVði«“2@ -‰ÜU3,ñ€»JÅ,aAîÒ²)aAˆD0IÂ8! -6„ä¦c° dB>‘&\{ ¢Á²DdqÑIX€ÀzÊ6' ¢,Ö ¢,$,@D0eè[L ÌEÌdA¤(alJÙ9 ¢ÈÇ U‰Ì”"àZä%ðt”° F7a)Ö —šœ^ÂüDá’„<í$,ˆ²ÀU$,ÀR4%ƒÈ\âiÈ‚ [ÂDºài-aA2Ñ „É$d™AIX€¹$T-ñœ¢<3IX\˜(T”°n‚LH)ò `BÂÌ%Û^Â$?H6‘æˆ<Èb½“° %Céf’° %Èb”‘°¶Iùf”° +È‚ˆO‚¬ÂDa²„€qð”’° ë„|Ó8 à—”o& ²UÈ7‘â ~›m |3JXdñQDdAД¿&!ß:I¼YLvä(ßL Ì@¾‰ˆ^ÜH”oFq»¸ë3Iâw™ p|‘­”Û+¯ˆÏdf¢Œ3‰; -Ð뱶Š[ -Ø‹Õ*Š{ -àK9’¸i1‘Y¦ î`YS+qÛ@¹ &Ì»O|‹*À"ÖÌináÝ÷œc³;Åæý±þõòË',¾×¿þëòò‹²SÖ~4Œßº@‰5åZ´í‚ù^M¥=‡ Á¶¡ˆ$„€ï©CBÄ<„ԀźéFBb)Ær›¯ý‹^Š ‰If`¦ŒÌ-úEª 1W6ˆ²¾¹Î$ëБC|†ÑÛy½ž¬ó£ë•÷{V¯'ëüè²z`ºC.«Ü²_QÈ{…ïJgéßuûåž) -ý"ÀBŠg >EaOVú…Þů>]tÕwÙ¿—ÌR£…¨V›týôýå—ßþæw¿ýÓw˜»O/—ÿýk*DƒøH ¾ùJ]¿GŠôèi¦y]?Cýlt¯”KsY8rÏ3:*×z-‹.Êo«Oå]¬4³ìÓ½àû­ò,ãûÑGéÚzb²©¹í.{+{ã(ò…!—¹õSäµøþ—ë§oOé;s}“ŠÕ3Té˜h¬ù"fUWQ‰pûfî,Ô^ož¬Ð!zÁ²VÛ·UppôQUFêü¼m'XŠËnä¦2ô¦ÎPTs$Gˆ{r¨1Rw©ÞДˆ²Ñ®tPˆ$´]Ì‘ÚÊUÎ:ÞÒ¦ž§¾™B)û¥ê¢™ØËrŠ]²P›EÎÖF»®uÔK¥³1-Í{˜ö¦n7§fnMŽCݦõø×®b›ìÕ=¬­ãñjeÆÍ„g×>­Ç¦ÅlTWû´u|­NR‚r¬²„7êrwtgÔFwM礛6?áxÎèŽY³*K×Å'nõ32[ÇüÒÖ=Ò¼ɨõè9ì<—Jh³ÚFéS¯müPgf£³Z¯Ï•«ò<×¹zöFr4Ù OÏ<W¶ÁìgøÁÎÜ3=•~žVøäë§ípfÛ<³[™0â ãiéÃïËüi©mºƒ -™í­~ÆÊS•ggŸ4qæ0Ÿ0Ñ<Í´"fòWl,c}s}‰\úþX‡ÿ“Û“oº -liåu´Áaß0œÏ÷ŽÝ7_ocè|·Å©s~¿²_)cÎC͆ýó°o™Î?õë—C]m0ÛÕ±¶vúÚÇçµÚw›ßsrœÓëû1çhœÖص߸ÚÃó¶¾ÿúS œíHñèödIEè.»Ž6!PÒi* ·âÁö[ë¼+}©±õdh¿ãl6´?¥&¯rBx¿¸]ëuVM¿ÔÖwyéѤF•ˆû÷ßþùà7Mw¶Ý­E»u ±,Òž'a·Ã&•éNŪÃ9\ªÑ¼š—7m6‹ú½ÉÑ®_–×Xj«˜ÈžDÛpŠ6"1ÿAÌ5$C)κŒí‘£½»\²/—̤­‹!Ü1¢“uƈÝÐ1éðf#r-¥qëIÜíØÐÄe•{/;^“iëWKÁ½ÎmiX¥Ž_É£™ŒÏÌؤ:__ûÚWm‡L¤×Õuœ_óxë F^SßÃz-ý5l™ùé›7¸J‚ûê䔑]¥®W¾Å_l-hë„œ÷mó*zz#Ó}ÒU§ÅŽv\Œnc7w¡» èÇÅ;.v²ÎGöD~ -OóÆyšzª‘ÔƒÐæq}#aeQ,ÛïÖú²,Û¬nl4ÔO¡?SÛ°u3ÃÄUù-®,¶õÝ,\­¬·Ê4¢È•…û¥¬gtÐÛXÉ26ZV{Þg<݃Þ÷™îqħ-?í>Sãïâ+IŒuŒfŸ¢^uxè/Ù4Û÷t'Y%CwPŽýå\Gó— Â‘¿” 궴QkešDç¡JÏ´ns¦eãÅú{[Ò¤²Ôf¡±~ºm›Õg\ÚsÙ %ë$ÐÓœ•w:“¿dëqÎ0J¹g&“©ßE{ -´Ö°á¯CS÷Bœ“uÄž†ØVF:m÷êóê»_êý=+>“ þåaû†ò² Ž—ë²%2áQ™ÜéUÆ’©¿3²zW”cd£;™ ,Í°ËtCöØâÎÕy8‹ó‡ö+8këU²YÜ"~|ÙZ˜'wÚ—ÑÆfµå[q{ {«–—öeó¬,¯ûá>ÃìÅ‹~e› ï@ÁsuÎ&Ó1 -6{YEtís/ÚjŸm?°ÜÞ>7v¹ŠZý*÷éôvý´h{•¬ë|#¡Ëû}·z{²ÎûÎìûmøk§¯ô£ÕŽØŒîžçèÊöÓ±å«óh–ã¹®ÝÕ:BˆôáÚ”2úŽÒÎ_SÙì[Y$ßMèeØ.B_CŸ·q!Õ5¼M¿ƒKœé|‰HçàXLŽtÂËÞ“Äô6 =Cªëdô ©¹-ßä+ E&õmZÓž@.S^×ß©þ¦d{»¯<ÜÞÈ{™Í‘覿}­ÆÓýAmÞ¦-ýÕ7mË 3ìmËÞ~{óɳxS3W3+cØLå—PN+ÒåÄ:Ò¤ i{ n¢Gmc¾Ò IIY§ÛÔé¢l}Ý©Yþ¡—ªg+:.ŽŠšfµüeÒ¿¿èì&Òy㾜㩠-9™v1ȯz[Ì dçÆkûwÈD¬_Uo«ó°¤Ý˜qÊë’ÖOª~GУÔóWãÛh¨¾oÕè|§Z![N˜™;ˆ¨ô–L£W“®d: 7M¶^…. -…8t¼ìhpT¼KÖV1¢ßënkòh:;­«. b«N³éoeMdÓyD¼]Э]S_E*ÀÙšjÚ5ªÓ,*wªo%û„¸>Oévˆ_©Ö´Ãdv>›5:Çl&¦fH®—Ë­œ@º]Ärµ5ßg*öRiTÍMóC”aê6uÅ ´;N#ÒáÞÌG”m_—d:&¢ÏµJÌn̆x»`‰qòëÅG¡´X¡|3=N´¬ÝNNjØË ÆÔT×l4:ŠþŸ¡Âw*!Où¢]7):Ém\NÈ7;y©$Nuœ¤íâ(ËÔ¥cì6Ëõ…ÀS(O ÛKw-¨LeŒs@ªƒÇ’¬»Õu2AXÓOÇJNTMt7Ⱦ›òë‚<äôÝU½êÍÌÝ÷v9tì|åäØg\ÓKp]ÞÞ.³ùF¼•…Í­Édú©u2ÈFõÕntƈC0€Â }-äæ­è¨ñöu8™´âÞ.»”eÎXoèmØ`52Ë#/}ý?”ù¿¿¸ëÚ±fó:nTG Ó±uAæ+IžN’éŒòFìnÈÆ1ØdFY߬¡¯ãÆ0¼v£ºÀÞJꎠºs9Ñ2²é¦k´ešñÄÛJ‹=ùèÜŒ{¾´mpèÆÒxM*\ðáà2 žKò•5‰+*:–tpaqÁÅ%ë0Éb’AòNºêI~ǧÂR_ì‚…tiÏe–7Ýhe¬B`Ë/xÃNóNóBAèw»×‡ç½ŠšµC{§¾rjQ;àÉT[.Ãò²¯9¶¼ªÜœwh¨·e™ïÙä ס…[\peÔ4ªÕžwêmÑi[(:\€¦­·›L¡Vkj)#N¢–b)õo\ÓžÏJW¢ç³6e©Ý¶âõhÉöÈssŽctj”ì±dh×îT7JÚFÛöí–}·ÕöʆᆠaD¦jTó›’ï¿÷øŽx&ŸTªý”ÝÆA N<ŒÖØò·]T -âèxîèÛ.ysãý“í.IÛ?á=a:&g­ŠšþJ,Â+ù>ƒ5Õþ‚\-+qHoS€°=4`n{‚bù3 e?²lõËH©"ýUKAo¢Ãä£ÒòŠ®þ>/¶PÊÏÿ{ç=X -endstream -endobj -732 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 731 0 R -/Resources 4 0 R -/Annots [ 733 0 R 734 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -733 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 582.273057 101.433071 563.523057 ] -/BS << -/W 0 ->> -/Dest (cb229-1) ->> -endobj -734 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 521.326036 101.433071 502.576036 ] -/BS << -/W 0 ->> -/Dest (cb230-1) ->> -endobj -735 0 obj -<< -/Filter /FlateDecode -/Length 4812 ->> -stream -xœí]K$¹¾ç¯È³ ëA½€…¼†ë…h؇Áªsºªº ”±Û…êÌ õ (ò©PDÙ³ÁÏ7ÿe²KÎ%E:_~:ý|2K -ÿj‰ÝEe‡ßÎÖPZ‚-1‡óOLðiI%¸Ï/'ü7]Êâ/§¿üêô·}{vK)ÑÙpþå ³È/)§d‚Â*`ëVKJ.D…í‚!–¢± FHäèÈJƒJ bsYaeˆá)Ydˆá’%1l IaY9¬3FåACä³a‰PzÔxÎ.1WT^Ï&§ñ¼_b"¬Æ+KŒ1dG%c5^€,ÁYÈç¤ñ"dñÑ•Y\N^ã%Èâ ¢ð2due•YL4Vã¿„’‰T^Ïä”å¬x«cïöÆ[ ‹s‰TdÁäkà"d1)jZ—4 p)-Ti à²_(SÖPÀå²PJ^ƒWÂB±$ ¼±à9«7%PÔÀ[ÈBð\•Y|! ¼ƒ,( a÷őװÀm Ö¢aÜ’)Vü%c£†€ 1†ˆës$ < ¦T4,ðÀ\ìÔ°ÀtáîIÃÔõ!: <`×SŽxà®'k4, à®÷¨©ò Xàãâmöp×[“4, à.ÜÈiX€Y]`‡QÃØÊâr6p×eCp×%,*/-ðA¯aw]À´•¸ë‚qpב‹ÀPàQ£à. $èádÀjX€õ<ç5,@c‹CX¢aAîÚ‚åOñi Õbs‰à®Í˜>•W› qÅo¡åÅÆX4,À],T^ÂÜ|f €ñ VT§aAîZ•Zãw­+FÂÜEÜ4,€9,€†Xßk`º*¯,¦ä¬aAî–œ†X…“$*/-0$«aÀx1ˆC5,ˆÀ],ïEÂÜ5Á{ "pוY|v ®n«ø4‚ÕÅ8 …Æî‹ÖxÀ]cRÑ° wyÙÔ° &&¸¬a\; b¦¥¤5,ˆÅ€‡OåÅ×AÂd@€,•Y(‘†¬—âKÖ° 9ÈâÓ° 9È‚AÃDK±ˆ¾Õô²À\Ôü€ "E €dK.D¤‘A«/º%'ÀµÊËàÙ¤aAJ´`)¶ —Z2œ^ÃüÂá’†Vçýõ*ú}X¯ÕywYSÑì É~Åš÷ -¿¯S~3£ù¡_xN=c¼&죕ÞYhߨð$°£øšZ&b¹—C*!ò+«¼£ÀfqiG”ñÅqd‚å9%t& ÿöÓ ‹Ô&Åzÿ£_ -ñ+[.œ?ýtúõ}ûçÿþãw°ÀOϧïÿÃzþÍ7ŸÁ­Ÿô„ïiO qûÜèÔ>óFûO ¡ÑVîyßN«güú‰Õ§þ+ä˜ú`~k§·öeæ~ªq_§-íëô~Ð]öã·î7ÿsþôíéwŸö†Q3Åj«¶X›c¼cKUù[úW˜P)·Lh6'§ª©’§¶™ËgxæB_&õ¿Ë”ộ̥µûy/Wk‡å¶O£âM`2呇ÖßÎ3?ã÷²š{¥¥•V¯Ÿ6Úçõ·×å:F7)¾ûµHð<‰T8½iSÖùhFeÝm`Úf­j‘¶Ï8€¡Î ÓÊ:Ö:`ñÈ¥ÁŸ­$kª Ö¬vAÙîl©laÕšØ -6‹gúì!$xÝ3¤LÒzó±=Õ›ÂÖWó€'Q׊±\8àS‡«~XÊ÷VËë|8k¤Û÷/° v}ÕZþÏ0-‰¤X¤œý^_`\³ºZvnOZ꼤N2³uwZ˜ê\[ÞçåºY¥Ô¥2·¾/7ðrgÄ›{.[îYècu>œ…Þäf í³’ÄêæïÏJ·R7YÀ¶ÂÑį3UÞMÖÀüJc6e+k&‹p¯“¯ãÞ+ëɨc×^_W¬p„uÁÐR!ï.N>VçÃYáXp -ñû,4<œR‚‡ý0…ú—É"”l‡•^©©À[R’W¦>2¸f•7Y! /uìM›œ>ïîY‘/éã¹f©NÄŠ|` -!B¼_>Xç£Yª»_^±ÔÝjZ„åNYÇ¿W|wÝ3ƒc•ãiDÝ«®$Ú½­ÖÄÑIŸà¨{C¼ù`gwâÍ«~}ŒûìIKÔ¶Odlxù¿náW,pĆ%-u“×ß3À‡ª|8û»³-øVïßÙX×q¿¡ÄÒ•…-«K×/Å=>0­·.Wé¾ýÏ?üþOBººËÂqe=Û¾‚Ãr»…L±_· –r-xWçs³Öõú@ßÊ£‚ÑV­[#—ÑoÇÒ(ãÊýg­·þ.Çrvk“˜ÿ¼–_Wþ­¼Yi½Þ˜•ªý‡Ôí¥º¿ošëz‹­';FÏZ!ÑWÍ5¾Û—þ¹×r+W5ìÇwû($G$Ú4c¶ª¹½ÍÞl܆wcLpÎâù¼ö6F¿Í\`íçU»ÜžýáØNô|˜ÌÛ©ãð›<±Žó†dý59ÌKq³ì¶Lê(œŽ†€;JÛÕ.-“YÇÉ2®VºÎ¯Ùd¯e¿lºhVý¼Ÿ¯Õ’GvTålm´ë­ŽyÞèbL{ïžu¨Ûm£ÙN“ã¦ný<þoyé*»cZ÷Jf\énòBðÀM¿é(Óﺖn¶–ÅXÝ^'!Lz¢U†ÕGVÄ»?·7uf]õ¹5¿ìæ[~÷eè¨kà„Ĉmüñ•vpÇþº„¶®¯rß²£?lmpÙ5V0ÛNÄÀ*{»¡Wô–TSÚªýÞÏá­ñXÛäÙvÀ½ŒÌE›´–«}Ú£vû²bhëçóÆÚʳ­áÂíÊ£4Í{•ï¦®Š¦«ÿwó/ôTåyšð5ÎBßA¬EÍ70ž ¬ù³k~ËAÙ>ÓÆ3_<>]ZùL㵈±Ú=­4éÏU® ³ëÚJÝ|Vd=º®Â»Á®]ÜǯÇ#‰c5Øì¹ê´é?ží+\Daø­æ# Ú;Óém¼G¼G<é[†o4ÿ ŸÇ|/ôùÃíñöyi˜¸µñå\…ÉšŸ?ÖþM½°ÿºþïëñ6ŽÝ”#—IŽ5ÞÛ¯-æ™sµÈOúz8‡Ž³Ég°·“µë¼é@Ï;¦i¡ËëÊ’CÀ ¦iu‰™·Ú†»ÉÏ: W;,q1…0˜:ü~Ú¸©ýqÐ^&ª ¡j1A»óÕP3”ê¬WÔ«F„¡çRÏ#Qvâx°Î‡1¢ÄÿØ”m|µ©g1n“3´¤b -ò‘}Ç;cú7ÚWúš½«õ®Ümêc~Ò÷NTWIÁ-üÀ¦qº«lëUh±ŸX3Úz¢çÂÇ\“º,üxÓÔiµ£+.–1oîø“¹·©ö`²«6<­¸[žfÚ‰ cŽ§®ÜïwÀ´=zÅŠvwûç»W—cyy·’nÕ»;Mó¹<µ9y‚yËE«;|ÿæû‡Ýîùnšâµ­.Gð>Œë¡ÿ+úŒkVçù1L¸qºwwìÁ:ÄR‡ØüØçuKýëü - |¹ùªÈÝQ9™žMõ ¯!÷aGIAm h 9Ù©Ã[¨³5j?XçƒøBGmˆ|ËêÂeÝ èÈâušF—sßk¸¬{ . ^Ú~/{šV–Û¬4ÑÏá¼ØD­wOØG?¯¹ýž×ý¦Nò×}¨´æ¹µÜg!“Û¾«öcèA32/"4u/Ð~°Î±§!6?6#Þn{¸?LßÃ^ïo‰;çCÊ°ƒ~/y>m¥œ§™ËÖøXæFàñeo_sßøM±ö•Ó«˜äŽFž¥~°Î‡³8º‰`¯±‚G-àÚéÕ[' -dT ³µNû2Ú8¬¶âN \q{q»•×NÖÊ3ˆ¢îW÷W/Þõ«Ûdx ->VçÃÙdº‚Í^¦SÀt%âìs'Nôµ;aÚçÁ.¯œü;øÅxJînì¢çËð×CD{çä_±¹°”Duô¶å=VçÃYÞíxn>ã>?µñU3¦´©e¨!¯<ët-š-rwÞ¦•{Y} gNeÝןãïÖõÝöDx -˜ˆø9Ø,0ï&u·È :>ŽÞì<ÛéaÚ}ªUØ~¿cÊÁPJâ·'\K´øv!‡Øv»ÍÃ!¶»ìoõÕð—Õ¼ÝVu-AÛn»V53n ~1ýö_MÖÚm³gÓsý؈[pÑn¥×þ¶Ûìý6^žÚo·ç„<»[¹e 3ëSùó)Ö7Øú6(áFŽ_yŽ´p&’Ê™ß>’'Û¦ÎVeÛó•šõ%›?÷RÛ{7 ¿Š‹_ò¾”S\l宓þÓÉZXÿõ¼R}Gž©äìÚÅ ¿œø–)d¢qè陉ÛWÓÛ"ÛÏP5Ú8™–2—DRi¶¯ã gšëW€hõC«Æï® Ó -ùúö†µc…ˆšÑÉS©—Ê£jiš¢ S÷¹+^¡]ÄaÄÄ/Î>b|ûº'óù¶>×& »qâå„%†ä—SHJé0°Â„fz’èE»œÍ°—Aåg¹×opÍFã7¬ðÿ+T„Neä©_,u“â·$«Ñ ûf'ï•$é±£i°”FY¡.›R·Y©/äPJyÝ^ºklG*ôTšx R<ödÛ­®“š~:VJ¢iB »AÝ”_vä!gè®Zß;±£æz»ƒ;v¾Hrê3nù•W]ÞÞ®°ùF¼Ô…f2›~n ²3}µ â  0ÈF_*¹ykG§ú,×öu8™¶â^N?žê2ç|p|Ë:zËop”‘—=ÿe¾ÅïÿžèüÇöÊ uw¦£…ëغ#Ë•Œß÷›$óû±» :°?È6²¡YC_ÇxM£ºÀÞJÚŽ ¶s%1  -²ë¦ë¬š 7ˆ—I‹=ùÚ¹ özéÛàÐç—M5©p!‡ƒË> -endobj -737 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 382.623057 101.433071 363.873057 ] -/BS << -/W 0 ->> -/Dest (cb233-1) ->> -endobj -738 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 321.676036 101.433071 302.926036 ] -/BS << -/W 0 ->> -/Dest (cb234-1) ->> -endobj -739 0 obj -<< -/Filter /FlateDecode -/Length 4837 ->> -stream -xœå]K-7ÞŸ_qÖH1~?$” "@Dº‚b1sr„&HŸÏn?ªÝÕ§{æÞ“ÉÜ9]åGµýù«*wG]%¾¾Pø'Z%bLÁÛëíÛËw)‚“ø¯”X]õ×ø ©䣻~›&ˆœþúzÁ?Ó%-þzùóÏ.ÿ\×QW-RòZ¹ë¿>f•5"ĤcT *© £rN„ gT^ ÜbJœ -f¸`µÝªÌ°):N3¬Ö‘QE˜alPf«J0Ce9ÌPÉF¥$ìPZJVC¤µF3:å„Ç {N§•ð1éÄêt*hNgŒðÁ*§8]Þ{9…-.IÅélqÊZÉé`‹51p:[Œ7’ÕÁƒát¶hŒNbt¶]‘ÕÁé¥âtÉ—¢µ¬.A'cˆ[–N¸hŒätJ lV‡…IòÑiØâ¥K¬¶8#5§3°Åâ‹Ó'œ 12«UçÕm¤QœÎÁ­ƒeu°“Ï1€ö°EÏQVȼÄq€Aؤ-G:a£ 蘄 Áp4 “Ö§Àñ€‘ -:­8"0¶8ë9&0 -¶X¬\V[L² [P€ãc`‹¶†ã¶µ@ ÇtkeRð­•Ês\`@¸0Sr\`À¸&zËqåJ‰ãÎ5àNŽ HË=p\`ÀºÆyÍqí=Ǽk¬’Xð®1¨Éê` ÌqØQ Ǽk” Xð.–‘渳*€CÏq°"tŒ’ã ÞÕQZŽ ,xW¸VÖ á¸À‚wµÃ-pž¼«ÔXð®¶Ús\ ðd£ð.(ÄñálÁr\6 1¡–p\àÀ»*Áý1kl%TLžãÞUÓÇê’P#ά[Œ²PÞ'Ž xŽÊp\àÀ» ÏÈq8^À£jŽ xWxjNÞU:IŽ xqƒã¸p -ÇðoBI@—Õ%!SŒ8ð.hIs\/,d“°º $ÅqÈXHÄ¡xð.Ü{â¸Àƒw¥3†ãÞ•a«ƒ-&jŽ ×@‡eˬi«Bj0§ïJ…ætà])Câ¸Àƒw³Ûä¸À‡,Бã(kŽ |´"…à9.ðIB‡ÕykÇqA8P«ƒ-6XŽ X‹dRä¸ hØb´æ¸ hØ‚ãD")Dßlz[6?°°‘"Ç`2“µç‘aT9×"Ð5«‹Ð©ÀqAVÀ+Ž K‰ˆEÏq^äp‰ã˜²„[°T8.€+Q#2çt -¶ È渑.tJq\5lA4ÈqAÔY¦—ÀC#—ÄPs:+sž9.ˆÖ‹*r\€e‚LLÉê` h‚ãÌON¶ÇH~l"Íau°Å8ËqAŒ:§›‘ã‚a‹–šã`3ç›ã‚$a ">Ž ’ô"‡É€Æ¡“’イ"òMm9.ÀºÌùfä¸ ‰|)³n“ñ9ß $ [BV[4%fÝ‚“o‚8‡-:YŽ ’÷9ߌ Ì@¾‰ˆžÝˆ9ß ìvx×%0§ï"2õàù&"[.·— ^Ÿñʘ3ÎÈîHP¯ƒoe· $¸Þ*°{ä›s$vÓf"³ŒžÝ5€[…RIvÛ@‚Ù ',»Ot‹Ê>3†|“ ËÂ"ºï9Ç(fvŠ-ûcÿúÛåçOp¾×¿ýûòò³²SÖ¾„RÚ"¾Â2Ð Óí}+ܾñ‹cÎ$•-„ï©“i©@‹uÓ- ÷—ÿ»n?¬,‡áÙtÔF^äó^@œdíLTZ·HŠ‘èì3‡Éo®shr@„®ýƒâmCEûcÅÙúæ:ŸÉÖ1F á4RhœçÇõdÏ>®´ß³ãz²Îç²Õôå’rÑ2pàä”Y) -h§öÆ*ŸqåNz”1Igo -ŠD$íªð/?\@¬ÕŠeϾ_óß)8Jí®¾½üüw_ýé¿ÿãôáåò—_H0á—_(üt±þ ËOûŒÏºÊ\•Ýêõ7¤^˜t­hÿK'×rÛÊ×þtÎ9žÆµuLÿ­îÇ\xùœÛÎßÖWÝÓ°³×k÷Dê´þ7÷”† M§ßؾE®–ßµóeÝoïS±TòË¿^?|uùÕ‡5K6µ²V6Z{€ÝsU>»ßdSº Ù6”–Lk"Cú²…X~@R¦Ø®§8Ë7SBt&vLSïÿy ™UC¯\íË8±S¯ß -f<ˆ²á,ghÛHæ'L`ÖçD30y²Î£Yþ=T6ØEæD~+¢i²gBªS¹‚>»ž±Ãmmø56È¥ä¨Fù²z0Ê,ÈR!çiõs“Uýç³emfÐYÞ^A›3ƒûh;WçáÐfïs`˜P&×ÐÝššÜØ[gBm¹aæ›#$Ò ÷ÃØÉ!²#Ÿá¾^W®mWt5)2VfºÛ¶Íå§îNßCÃä¡É>ETλROÇOè©Ž¨ç5߯rÑwö8Y;vP8â>ë´Îúx´[~²Îáð`ÿò§ô¼†AÀ*‡ïŠaÚ=ã͈×LyaW-{3wQv®Îáì ÆãffÞ9Ÿs¿½g/ôYÍ;b½{Þ˜òÈ&ê|?«]ýéa|¿—™ÿhF¦<æˆÉlˆBüQ¦p²Îáè~7#EÓÙ¢Ïȼ’r¼¢>׬19/å9f¯ðêÓ(Wžâež{®OõÑɪÏ^ï¶ ¾ÈÂ"+×Õã–úϤnÜç&+ßþþÍÉ:†*{°wמÖ?ÕÜëiñžÏµ\,-3cVgÃ-ó“-’nâ‚&K³‹‰i»›—}žl^z$¶:»“|²Îáñ`ïî{@ÐÞ«P»oJÌ9*ƒH:û«·"&/Ißf {póënón åx7ížô:{~öØÓn̼kÔû¾ÝáK²÷–áBG¥ÝBÏÕy8„Þåf„Îo6>bò³Ý½/=! z¸½\ôèé]o÷ÆĔӮ7}*÷û:ï½±Þê°é¾Îdö{z'ë< -bÁ)ÊßìÅMY·›PPvâ¹ÅŠ+ç|‘ËÞ“•¼1ûYíJï î{v{É*Õy5U;MŸõ c_àïg©N¾ý=›“u ©î ¾ÜAêÊ›þ„vaöV&}Î[ìx‘Ä»veN½Y»ƒÎwÄ›'ë<:âÍÝu½W¯èÏ™åTÞþß¾ƒÀ¦ ʯÚ=9Wåáðwüâ{Ÿb Œõ1î¿ä—­K"#«[×/Éojåü2JÇbÝW¿þíoþH¬+»,9®$o᪫µSÜ×Q‘-\ʺ¼£óܺ\oäµìÈΖ6FV¶”±~ù†ÆšÙYsd¬J?O—¹úSŽv¨ >ïŽSšÇ)¯9ƒ6´Ë¾uÉBóÏœ™Ú|¬"l6·ú3T¬:|0zYŸYŸeÙ?dþÔO‹¬ØŸˆý•G˽¾¬×éÖŸñ¿cÌxß»æŠeg“òĽñŠþÖmüXÓ3v\ó;k¥ñC‹[º¾Ž×.V>:/¦ç†y÷<0æÌh—óÁdœ¶<¿Çíºñb½×¦ï~UŽ1ië¡­á{ã9·4®ïç©»ã‘OØ\ã¦a‘ø€6¦›_tF¢ëM9 Õ©‚‰à~Óù\w(ð×ÇY&ų Jq!ÌëžÅíú/“°ÛaòB¦|ÜÙÔá_¦-‹²Åí‡ìÍ › -±ØH£ë0ÿžM~8 ”źèíw-bŠFæ½h}uD)¢òoûÂ}0iiEŒׯ;^éG´£òüËØáúu=(8Œ³sù¶èT$Ç -uŒ9€äö¢v4ÊUM‡¦¬×-šÌ€›xÕÀA -ùÌ,nÕ–çã)7V¬¾­Ã ÎÒíŒæžjHZ´ô£†™-4Ê?M ;^d)E÷4—¸:/7éH鯦*…=H¨ÚÛoá ±gæ¦ -±r›©üîâËùoªœ*}JçC¯ފ=†tÍgÎEi¬jS§Ê`«ëNÍr´úw½T=m]æXóá{ËQì䢖ÿ®Lú·•¬ðm_¤œŒ,‹ôæfñë%»Nˈ­›+â,¬eo˪¾WÓd7oi. r“õãØÐÉŒ·|Ô®ÝM®ïZµ|b™•­)gv-3BÔôj+Îw/…ªâ|ºx¦^ùnŠóÍfGƒ–– ŽŠ‡be$º­P’î†XÛfʧQö¡óCØM§C§Èô·²:é¼'¼]Э™¥¯¬4ŸiÜšj£«e—bTêR×Jö ±}žÑí¿­i‹Éìz2kùdÈEl/—Z9Ft»°åjk®ÏTè¥â¨šÚÈSÔMìÏÈndÓ3ä?—@Öˆ4íãZœ÷ãú\ËHp£7ÂÛ.Ænį˜Ònp…t zThH»]åÀËæ×E—OXšM–ÏÕËÿ.Táº43Où l‡T>s\NòÚìâõ Q¹ï¬c¹P6Œ²d¸T³t¼T”LùLº½t±•” ‘‡ÔÌË$ÕÉc-Vu]œ)¬OçJ*”Ít7Ä®Cùu%vº¾TìyEàîz»Cì;w¾Rqè3®òA§ÝÞÞ.Á|ÞŠc³³8C?¶N†XËîíFgD8 ) ñ Ñ×"n«µ³S99 ~‹Œó¸·Ëß/ÅÍiãtN ¼QùÜny©ëPæ+|ÿãb¯¿oE.~\ËκsëJL=Yþ+ñ¤8ÿÕ‡&숵%´?Ä*Ž²®¡¡ûq­ _ÛQ]bo%UgPÕµTèƱîÐÕÊ‘qw„·i{ ó©s3öriÚÍ¡“mVá‚Þ.ãЙᒯ¤I\å¢Ã¥Ck5©ˆ Ú(.I‡iX¶23’vÒ‡>ÛoéT˜Ü¹ aD¾4Cg‡›¥M7Y¹ R!°¡´a«h§iecô»—×'Ï{5Ù·¥Ú»ô•Jûei'<^jÊ¥__vŸc®ù¢-Þ1B½-CÖžéAÎX#´°« ŠQ"£Z-àh§¶ÉV6GÑé2Ùd½ÅÐlòµZ–rÇ‘¥ÐcŠ!ý;iGg¥¢£³â7es»ÍãõhÉôÈQ8‡qwr”ì±dddžÖîR;Jš&Ûöm×}7o{%·a‡!~D¦rTs›’ïâ¿÷¬¥Î´Á•j_ewDÅq´2µUÁhSþZ–ŒÞƒClþƒÁF²]òæ:÷÷O¶»$mÿ„ö„éÖTþ»g¼ ¯P¤ciªýM4¾Z’ì-½m˜í¡Asr³;TþðMy‰§lõËSÅüw®LzcþóA*þ —vÚ€Ym¡”¯ÿÜ,V« -endstream -endobj -740 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 739 0 R -/Resources 4 0 R -/Annots [ 741 0 R 742 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -741 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 172.623057 101.433071 153.873057 ] -/BS << -/W 0 ->> -/Dest (cb237-1) ->> -endobj -742 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 111.676036 101.433071 92.926036 ] -/BS << -/W 0 ->> -/Dest (cb238-1) ->> -endobj -743 0 obj -<< -/Filter /FlateDecode -/Length 4534 ->> -stream -xœå\K䶾÷¯ès+|_@àC€$ˆ‘1°HF»íÁ8€Cþ~¾¢D²$U«5³kc‚ì`vZU|‹?KjÙ«ÁÏÿe²SÎ%EºÞ~¸üx1S -ÿj‰ÕEUß®ÖPš‚-1‡ë,ðiJ%¸¯Ïü·¹”ÅŸ/ÿÕå_ë:öê¦R¢³áúÓGV‘ŸRNÉEU 2Ö+ª¦”\ˆŠ*Ú C,ESÁŒÈÑ^•`•4Ì ç²¢Ê0ÃS²~¯*0Ã%Kš -fØ’¢²vXgŒªƒ!†È;EgÃáô¨éœb.®¨ºMNÓy?ÅD6XMW¦cÈšŽ`K(Æjº[‚%2š¶ÏIÓEØâ£7ª¶¸œ¼¦K°ÅÁ;EÑeØteU[L4VÓ?…’‰T]Îä”÷:g²÷FÓY;`Ó«:¬(LRLŠÎÁ–hBQu°%xã4‡-„Mž>嬬VÇ«Ûo5]€-Î%Ru°“¯1€‹°Å¤¨QVÐļ¤q€Ki¢âH#—ýD™²Æ.—‰Rò ¸&Š%i<à…ÎY¼-¢ÆÞÂÂÊUu°ÅÒ¸À;Ø‚x[y <Ø–€ <è–L±xð-5.ð \˜i4.ð`\Ÿ#i\àA¹€RѸÀƒs=¸SãÒÅrOx°®Ñi\àA»žrÔ¸Àƒw=Y£qw½GMU[` ÀŽ“·Ùk\@à]oMÒ¸€À»XFNãÌêF €•Éål4. ð®Ë†4. ð®KØT]š°½ÆÞuCÐvnð® Æi\@à]G.j\ Là£Fà]PHÐÃØ‚Õ¸û=tÎk\€Æ&‡°Dã‚ÞµÛŸ²¦ÁV“Í%j\À»6cúT]™l‚Ç•u /O6Æ¢qAïb£òð.è3k\ŽŸ°£: x×zìÔš¼k]1ð.↠qà0Y`›¬tU]™LÉYã‚Þ-9 ° O&ƒIT]š$«qÈx2ˆC5.ˆà]lïEã‚Þ5Á{ "x×ÂU[|v ®ËVYÓV'ãÀš¼k,fXÓwIEã‚ÞåmS゘Xà²Æ0pBì4.ˆ™¦’RÔ¸ B>U'\ ’ €²Tl¡D °žŠ/Yã‚ä`‹wNã‚ä` B LÅ"úV°pQÏ[)j\&›r!Ò¸ …ˆó¼ªé¢›r]«º M¤D¶b«qÎRSƢ׸?q¸¤q̀ΒÆ©À, °MÙ!2×t¶ ÈÖ¸‘.tÖj\lA4¨qAv§Ìh4.À³$\­éÈð93k\)N*j\€e‚“&˜RÕÁЄÆ˜>l pøÁaÇU[| rv|ÜÌä [œq›|ÞL[ñi\PLœ8LÖ¸41›qÞt¤qÖ%Ÿ7³ÆÅœ7qÄSÖmñ‘Ï›Iã‚B°% DQu°ASQÖ-8 çM°“¦‹°ÅÒ¸ ÄÈçͬq œ7Ñ«)Ìçͤ¦ À»¡€™4x‘iÔ¸8o"²ÕÎöÄâ3]™ùęՌõì­jÊÀ€{±[%5g`@¾|FR“0'ËÕ¬¶U(­QÓü‹xŒvœ0gŸdŠ* {fN<È‚eAˆîû™cówŠÍù±Ÿ¾»üú=6ßëwÿ¾<ýªfÊÚE€PKðŒeà -¦; ú¶¾8`VË`ØKÊr§© £,hpI²±QØîøßuÿae) ýzÝ-0¥õkH5òaéÏk&hÄSäGféY['¸ÇÇ[ù0öÅušœ¸”kÿ`¹Tö+§roë‹ë|&[‡°{á$Ÿ÷ëÉ:ŸÝ¯²ß³~=Yç³ÛÊQhÎceˆ~ƒ¾ªþ¹Ö>¨Né1Ž ¼©4u®Îg4yO¸¿}wAd1ßÕˆW‰L79Ä Ø@ÞýpùõW¿ÿãþú5æêÝÓå›ß`»ËÆ„øEpEøLàùsâ³gçëÀ:–?q™/ÿq}÷•ìÓq‚­ðQvÝ'wc¿‹ÿî]õœïžKŒ^ᜇW,@‰`ZX…³_XåS|½¸CÀñã›|qîaÇè˜×7~§ÖùìŠ~i}è›&Ä"”s¡Nß¾úÛ_þ,¦Á¼_~ý—_X\Ómþâü·É6Þà–Ïi£kÃÒF^·Ñ¯íR·ÍHÓ] Ùà@Ô’f€„0÷²B_vKO®Öì„c ˜ÑM‡G²|4Ÿñ€Ðs2ˆ¤x¾!t²Î›Ãl9ÀÑ CÕóyŽ‰†37°a¿˜â \¶zO‹~ƒ±†É{ýɶ­ùôvW¶nqÆŽêчÑ~ÿë÷vóýGët:ã¢ßW4¶æQx²ÎAá0 Ñb±þ-ØÌ2šüæ/-Þdd¥å7*HØÎbCÀ‚X*âs«C:Zi]—7e>ô‡u · rÝ‚*Ú Ê wÛ„®·ßêYaû¥²ÎG1žp€D×¹-pv.çši:Fâ¹:o‰tŠé½²÷5ﾄ3ÌiVÌà©rÏî3n6È”<Ô¸Ïé¨núíJ‘ܽãIÉm'9øl{+Û?•¿u,‡Á†&g)=Âò¹:oËñU_Š#Ìõ9ñ›66ka5¿›ù#É~y3÷[üß‹Cµ=üAÙ—®)—^gs_ [^ØÆÖ’OÌFN‚]5TÜÃØà\7‡âê'0Ì«gò£ØÏí@òªí¸¶i[ˆSÙ½)»…DÈ.¦Q˜8ÀÎhί„Gl~j…¨xõ".Å°˜ì8~ˆ×“uÞ^ýq,{š^ÀpÛ²ãLe”Ÿ)~s±ì™hMgW/b×Óh=WçÍ¡õA¼ûRd}ì¹e´§MÝVïD¼û;ºîMž´>#µøÂÌÇG ΓuÞ:D°¿:WhúRÍq–É1š~)2žd=?m]V3ž5kÁÙŒ0z¶ïçL§ë#Zï -#–453ÊekªýCÃê|½“/eÙ+üÌ\véâ­›©)üZuÜÇ“YÒöë#û1×á2ó.³¶£¶'íŸsµgç¥ã¾iã -óÚéý›Å6;ìá1Ö~{Î[î¨kß´h­_–Å_ëÜÑzÌ=Ê3Ãß=K}Ûe¤å¸’â*#­ß™jÙNôül ??½o§ŽÁ/öÄ:æ;ÐÊ=;ÌK[óœYr»û¬»ågúÉcömW<ßæè¢Ú¹ŒwÆG‹Žç¾jÙ‹/›:ÛüÆa˱ˆ6ÚõRÇ<-r1¦õús½«ÛqÒpÔì8ô­ÛŽ»ÖØžïf¨ë ò¼^[..vö55—›÷8‰Ýe-Ûø¼ùRpÂ|ÂYôO¢Nó4Ö¢\‡‹-ñžßýF;¿mçç.úü„e¿¿-œt0¿u_nþŽ-Ÿ7ÇAÜƬcLÇúÚÖÖÛhíè£f¸ýÈÜñYÜúl×?ÍmVûì~íT¿-×}¾¨qù¢¿” Ã!,:;ë(mð!0·„‹e} Þߟýw~8ÜËÚ>voÛ¯µ3¸o«s»¬)mŒÌo¼Æ]˜o»§ùoåþ~sÀmù›YtÌnÎd°že¼?1»÷,;òSÜqs(b\·½¯:—Ó÷yGVb'´=BâãÎZiüбÇ\UÿÝãèOà€>–Òð|è«Óò˜CµÑÌøê~úöx¼ý‘ˆÆ‹ —r]Ø—blum‘ð÷Éöùõõyº·7í²íø](}·Ã'Sø»›¿GìJwe/d¢óE¦óÔGùæ?êÏ)@©‹õN°wD)ñ·êcT”Ýd=¥€èd·¢&kS¶ñÅ R³zÇ`r†¦TLAl¿îx¦ÿ™Ë+@.²wë)w%Öwô,‚ºT²‹Ö¸_ðq»u§GOKå`ÇrÉßçwáÁ;Yçä£úJË1­4~ÚnD¬k$¯ÐiÄJôkdõòáÕÌËfE=‰~äê¡u¹9š]²¿ô“‚¢.G°>Œëó~¼ƒÈº<{ á€9œ¬óF9Ì.ö‘ÿ÷(;ϲŻ#–ýüÏ£®;<|¢9¤†ÍÀØæ¹n“3wã ¢Ú¶ûâ”D„¦Å'ë¼< ³K:ŒýòûíæsXûý51¢šÞÙ~³ÉnàÎ4•mIltfù8ntòî7t¡†ë‡V¿VL¦òõ‹µsÇŠ5£Ý‹yôf²‹8ÔÇür»)!6›! ’,;Š­7BöB#ºbGÍ˯Œè®‹CØM—®³bú[Y—Ät otë·ÒgUÊ/jM5ï:Óe^Uº4´’}B¨Ï“"º]á¦5M˜Ì®³Æ¯o˜…¹‰z¹ÒÊ)¢ÛE-·´úL¥^*ª¥y~˜2 îsw¼"»‰‡¿ÓP¬ãÛǵ˜ï5÷¹6YàÆí„· ¶Ú‰Ÿ/!)¥Ãà -ô¤Ð‹v»8›—!M¹¹ -K³ÉøËïüÿL¡K™yêKRü‹q5:áµÙÅk'Iyì¬Cš,¥QV¸Ë¦Ô1+ý…hS)ϤÛKw­¤ÂeBžJ3Iª“ÇZl;꺘)¬ù§s¥šfºâСü¼;C_ªÁô΢p½Ý!Ž;Ÿ¥8õ·ü6’nooW`¾ ouc£­˜¡Ÿ['CìLßíFgB8 ) ñ Ñç*n«µ³¿a±}‹LÛqo—ï/u›s>8NyGoùåZ2ò²×ÿ ÌWøýç…®nos˜÷qg:[¸Î­+±ÜÉøUŒù¤˜_ÍØ„}Ù¡#AûCló(ú>îœàkÕÍ öVÒvµ]+…aøPˆ]‡®³^x&o/ö@æSçflØó¥oƒC7žßҬ….óÐù±%_E“¸â¢cK‡–œ¨ˆ Ù(.E‡eX¶2“);é®gûIN…ç¾Ä…#øÒmV6Ýdu¢B`//dÃde§eebô«—×'Ïûbrl+€¥Ú»ôYJûÀXÚ O—úz×—}Ïñõ~D[¼ÃC½-/ÖžïAÎX#´ Õ…ĨæQm)d§Ôd«NÛFÑé2Ód½ÅÔlŠKµæ–:â¬z)õ˜bH¿—žrVºƒœ•¸+Ëí¶¯GK¾@AÂ9Ñ™Q²Ç’Y‘EY»Ki”ôM¶ï›Ö}·Ýö*†AÃ8"S3ª…]ÉWñßkÖŽú - ­Tû©Ù›Çû°Q“MÞùúJk“c‡¿•0Qé’×9Οì³$-"{ÂtLä½I–_N®›ð Ey¬Mµ—ëÕŠQ‡ô2(é¡AsûžÔ·ÓÖÔdMõËÄGE~µç 7ó;4“Ñß]Ñ¿¯N«Jýù/Axýe -endstream -endobj -744 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 743 0 R -/Resources 4 0 R -/Annots [ 745 0 R 746 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -745 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 327.123057 101.433071 308.373057 ] -/BS << -/W 0 ->> -/Dest (cb241-1) ->> -endobj -746 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 266.176036 101.433071 247.426036 ] -/BS << -/W 0 ->> -/Dest (cb242-1) ->> -endobj -747 0 obj -<< -/Filter /FlateDecode -/Length 4792 ->> -stream -xœå]M$¹q½×¯¨³€Mñ#ø{0`^X†XÁ‡îši Ƭ•þû~d&ÉHfdeV÷¬TZM£§+#øI>¾ˆ`f³õUáë;ÿ"é)Æ<]o?]~¾¨)8…¥ÄꢨÄwpW­(LN'Ýõ§,°a -É™à¯_/øo¸äÅ¿^~ÿ›Ëÿ¬ëè«™RòF»ëŸ¿dÙ)Ä”T *¥­ rn -Á8/¨¼žp‹)I*˜áڪ̠¤‚dLTfX -ÚnU f˜ IRÁ \TZÁm”u0DY#è´›<ÝK:£'“I¢.@§ƒ‘tÖN>vZÒ¥É{#Øâ’Ò’ÎÁ§‰”¤ƒ-dct¶Xo•¨ƒ-&+él1$è"lº¢¨ƒ-Ê+-é’\ŠD¢.A§bˆ[QnrÑZ%é´ž°iEV&ÉAg`‹W.‰:Øâ¬2’Η¤O8bV«É«Û*«%ƒ-Æu°“/1€ñ°E/QVДyIâÂDÉD&Ú‰"E‰LL…`%0ÉMäSxÀ* ÑX[y‰ ¬†-„•+ê`‹M$q5°$.°¶²X°--XÐ-©¤%.°à[RÚK\`A¸0SI\`Á¸6z’¸À‚r¥$qçZp§Ä¤‹å$.°`]뼑¸À‚v-E/qïZÒJâïZ‹š¢¶À€'«£•¸€À»V« qw±ŒŒÄ˜Õ 8ô+“‰QI\@à]I\@à]àD]˜°­ÄÞ5· ynð®qÊH\@à]CÆK\ Là%Fà]Pˆ“ÃØ‚•¸þ:c%.@c“AX"qïê÷'¬i°Õ¤cò8𮎘>Q—&0âºÅ(OÚû$qïÂQY‰ xô%.ÇOð¨FâÞÕžZÒwµIJâÞEÜà$.&@âø·I+@WÔ¥I¥%.pà]Ð’‘¸^xRL"ê i‰ @Æ“B*qï½'‰ )èò‰:?áÚI\”%ê` ’¸õ”lŠ[¬1[2H\€ˆ`JÑ·˜ÀÀE̶ R”¸L6ÅD$qApùFUÒy3źu:$.&¸b-qr©)bÑK\‚Ÿr¸$qÌ€N“Ä!Á,‰ àŠ¦h™K: [dK\€H:­%.ˆ¶ ”¸ šˆ,Ó+‰ à¡‘Kb¨%©œgF‰ "ù)‡Š`™ ÓSŠ:Øš¸ó““m'q’$›HsDl±Ž$.ˆÑät3J\#l1ÊH\læ|3H\lAÄ'qAR~Êa²Ä qè”’¸ éˆ|ÓÄX—9ߌ$«o"ÅÖm²>ç›Aâ‚D°Å!Du°ASÖ-8 ù&ØIÒyØbI\¼Ïùf”¸aòMDôâ–@Ìùf· À».™$x‘©—¸È7ÙJ¹½ñº€øLVÆœqFqÇ@z|«¸e À½ðVAÜ3P ßœ#‰›0™eôâ®Ü*”Z‰Û -ü‹xŒ6œ0ï>ñ-*8øÌòM&, BtßrŽ^Ìî›÷ÇþüÇËo_à|¯üßËÛoÊNYýÒJiB|…e`¦Û!úÖ¸}‹¦1îªü+ÂÀ÷ÔÉF„ÔÔ ÅeÓ- ÷—ÿ]·V–Ãðl:j#/òy/ F6÷&*­’bä&ûÌnòÃuMˆÒµ}в­}¨x¿}¬$[®ó­lµmZS.Zº‘;R^=q‚v@ƒU¾!r'ˆb<ÊØd2ëc)#^W´*üŸ. €ÅŠyo¹]2ó‡hºq×O?]~ûo?üç¿ÿîGŒÓ§·ËþA!Ç·ùþ;Ïô6ÿtqù©y½¶r9zÁ÷m‘a™:?VKyˆÊ7¥^®¶QÛ.ú¥½vy“}ém87ØEÌ–Øí¨uÝ ›mˆû¶9V–×ýpŸˆ‚´úý¯ë§.ÿôi ³Ó˜åt*)€ÌsU>‚Ì_)Ýd -ƒÖ?7á×?Ë cÀÁ(âÏ (i ÈÖÞ¸(ê5ÀîÂZÖê ¶6Ýú§©‹a§ïZoÏÖ²È û|답ɇñÑê{vùYFÒíÉøîNÖy6à!Ÿ¼‡¼6ºq˜Í×o0cB›œ6š¼Òcm—†z·a–õB_"3]òöK™F«®ÛÑÚbu oÓ­ïA¾×tYzØßž¬ót袻¼v mÜ0ËŸ@À°þGÊ3j@ÀÉ‘/ #×°:¾®v§-/šÛ/y/È¨í¼­û•VÛ.jYȧh2š"ÑjÏÕy:ÔÞG: -ãŠ÷5Ë,Óz–³|3+L×¼œ:©ã;í–¾¨#Jœ}ó(Z„ñà!_Em8‡ÌÝt„JîåÍ)Ly8Bå¹:O‡Êû1bå OpdîÄ`…kªì•ñê«é­·?œÑ1&Ýi{Å,a*«‡X¼–‚Ç°´ǾÛmF±´7?Šˆeûø.ÚNÖy6´™ƒ¸pŒökZ\ÑÁR_ó‘™ÐB¶1ðÍWYAØ·SBäçIqé¸% ùjÒl¬ìp·m›%Æäuk¬Jýyæçú-Ù¦ÍrYü˜_¸‚CðG1çÉ:O‡Üƒ˜óËÌ-4G - ÞóÃm1Ü ÿN,ú—F»aõZ›cÞ=¬î3Y’yG¼y²ÎÓ!ö\¼ùQDµ8Tä_ñʨíðHhô;b¹ƒL?&7¹³#dž«ótȼs– „öѶá !ûày«fñ%Ï–6™U•»A^=ðÛ¼±­ò÷›Zí Ž1ð/Þÿ/ŒF«zs²Î³¡ÑÞIÊ8‡ ãp¯·y¼2ÎöÐÖ_á<+úk¡|cà ¯níãOtNÖy:´Ä¡wF~õðí…ÍaÎ ïÞ‰çǺ|/öóR~Ü·dÙ͇ŸOѲKõ²óœŠN"ìqãÉ:O‡°ƒ¸‘ïb†ïäHùëj÷hç‰ÝÞ®R™Í#>sàefPú¥ó6íp5_ «½Ø—¿ôþë*{üç´™’Š&¿u•çê<*ö)?º;} ƒ‡r,cø(Š6¨žŸy¡cŒÊÎãûÕs.¾¢âΧh‡ŒBbϨ™”#vËOÖy6ÒÁþå¯éyô6Ïáß»b„vÏx`êñš-¿Ä¡ç§.wQv®ÎÓ¡ì Æ{¢÷dîycÎ#›¨?Êý¬võ‡çñí^Fþ㙹ï1©Çdâ¤-”)œ¬ót(:xÑ#?pP™]ö¯äƒ¯èo5kBÎËyNØÇ+¼úÒË•§x™ç^—§zéÔ¢Ï^ï6#¾ÈÂ,+׋Ç-õ_YÝx‡›âãïÞœ¬ót¨:Ø»«Oë_–Üëeñ–ÏÕ\,Í3cZgÃ-ã“-–nâ†&âÙÅÀƒ¼ÝÍû>¯k^ZF£c±ÕÙä“už î`ïî@ÐîkÍ{oJŒ9ª€ÈÕ;„ü­ˆÁKò·øÜøÆÛ¸ÛÂ9þ±w ·»,ãn̸kÔú¾íó¥c{où—ðLÔÆ!ô\§CèýXnDèøfS}ÇúhVvߥ^<Ü^.zôônõ²þÞ[j@„y̾Æ{Ö[½6Ü×™ŒÂ½cOïd§CáA,8Dù›½¸!+v -ÊN<·Xqå˜/JÙÀ{²’³ŸÕ®ôù¾g³—­R“WÓb§ ý³yì òýì"5>þžÍÉ:O‡Ôƒør©+oú+Ú…Ù[™ü9o±ã¥GïÚ•9õf­ŒNÿŽxódgC§?ˆ7w×õ6>\½¥?f"œSylxû{Gø{l˜ÂT~­õè1ȹ*O‡¿ã·ßû£c¬qûÅïl]š2²šuí’ýV´O”—Ó&ë~øçý—ÿà¿oæÞ4{ W¿VžrB†Ø¯!#[9—uyWçµ¢u¾ÞÈ—²ý¹ô+¶(¡÷YÚRsÝíËÙ†bGÞ= -½RÏÍ(íoƒ,ut倫e±t±Yv…yÎ y>¦ù:–ë=ª×üƒÝ¡^l–QŽS–¹×~éÒ¯úrùŒ ›Ï8\ú²ËŒ¸ü~_œÛÈ}êÏÛv¼Í€äCæ¶í”Qxé£ûîÙãžˈ—™¹ud”7KV³µ-6脶gdö6×Hcø²ŒÁbK >yŒ‚ÒbŸgõ«ì!×ô6rÙ™/k6¶Èë}‡ûúº[$iTã}ß»Ÿù]Éýû)œ²ØÓ~ËB mÒ\®Ø§çúužæ,®_—1ªuü<ö7¥Œ«<6žWÖ¬Û¬ÔžR†aÇU¿ÇjÄÆ™>óû#Ã<Ê.œY«¤¥¾™õkŒÞ½ïÇ`-Ú7nÆ~ÎaòÏœ×P>¨÷coËÏ°èԢËŸ&Ìú,34¯Qó2ËŠm‰ÝײV .Þ:†ÊÚsK9ß×öˆ W×7Ÿûu Þ†y¯úÊ=áîXù£¹ÿ´{NÓ÷îµcÇY6NŸïß/Õëå¾i¹×æuÝUê6—uCl¼Û¿;n^Ýÿãq¼Ï?÷ì(ç-­±NK?Œß«ãÌäÔä­+¿‚ïó‡\phž¬ó®ãW¾aˆéš½ž¦è¨[>4"ÃBxa¯¸Ë«yv;L~R)b8tø‡!é,›”¾ËqëDþ¸Y,q.Žì¤6{‡E&k{^R=窇ssò¤t ¢É|V¹WÀ”RÈÇ{I‘táÒ/3ïê…#ò ™Ûš_‹ïÕóӶƥ‹ß¨>´¬áÊÃ_ºÿÊ?­gkoÙI6 ·õ·|–|’YüÕèÏZû•Û™=+ÿ–\²ÜÃ÷ãTþ|ñå¨:]Ž+U fmòyæH“¦ì–Bºæãñ¢²¤ëÔé2ØúºS³œÿs+µ ¯òY±ùœÀùÔxv±”ÿ¹LúOÙA/5H/‡8«"Ž¦^tñ×KæÄD=Ò?gáòQµ¶H·Ä¡Ên ¤0¥±$–½Z>öì"sÁüѸz7¹¾«Õòáj¤j![Ž›;„¨éõVœï^MzçƒÐãd—+ßLq¾Ú aoxÙÞ`¯x(ÖV1¡Û -뮋 U{t>8³ ïÂf::ͦ¿–5Mç=áí‚ní(ý*Jóñ˵©:ºF5™eF¥&uµd›jó$ˆnAøªM&³éÙ¬åC,ga¬@¢V.Õr‚èvË-­¹6S¡•Š½jª#ßMéP·± ¼ »± <ä¿ìÀÖˆ²õãZœ“É6×È©:nÌFx»ÀÅÐFüõâ‚PÚu®P®B -k·‰£êxéÒüÓü K³Êò€ùÿ™*\“fæ)45Håc<ûUï$¯Í&^—ûÆ:$€¦Ð˲áÒ!4ÌòñÒQ å3é¶ÒmÄVR6dLR5/“T#µX7Ô5q¦°:>+¹PU#Ð]»å¯+q·Óµ¥êTëÌkw×Úíb߸ó+‡6ã:ŸÉÚìmí2ÌWá­86Åú±vÒÅF5o×;cÂnH¡‹;~-âºZ;•£*–}‘I÷vùÓ¥¸9cÉA³·:1Î#/}ý?”ùßÿ}¡ëïê™–³7ª±…iܺsO–ÿ E<)Π¢ -Û²aBCŒö»XÇ^ÖU44?n ãkêÕU'öZR7ÕMË…®!›]£-wGxF±2›î°çK[oÝØ|jµ -üvp»Îv—|eMâ*í.Z2¬".x£¸d¦nÙÊÌ HÞIúl?ñ©°¹/vÁˆ|i»Žº›åMWY¹ V!°å¼aÒ¼Ó´²G0úÝËëÃó¾˜ìë -ÈÒFíMú•KÛei#B­-ËÖžmAN_=´ ÕÇ(“Æ^m)àx§Te«N«£ht™ª²Öb¨6ù¥Z–rÇQ¥ÐbŠ.ýiÇg¥ ¢ã³â7es»ÕãµhɶÈq8‡~wª—l±ddž×nRê%m•mû¦ußÕÛ^ÙmP7Ä÷ÈTõjnSò]ü÷žµ3×v"¤Rõ«ìŽèØO†£&¬±å{©è=8„òßfÙvÉÃuîïŸlwIêþ ï Ó1‘µ*èü'Úd¾B‘Ž¬©úçÛäjI‰·ôØÛCæÔfw¨üžòl¹lµËSÅü'¹lzcþK"AiùÁký%X·ÚB)_ÿùxõ -endstream -endobj -748 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 747 0 R -/Resources 4 0 R -/Annots [ 749 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -749 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 131.373057 101.433071 112.623057 ] -/BS << -/W 0 ->> -/Dest (cb245-1) ->> -endobj -750 0 obj -<< -/Filter /FlateDecode -/Length 4803 ->> -stream -xœÝ]KäȾç¯È³‘ãýsX`mx`;@c÷°ØCuN— £ÆÀØÿ} Ń’(¥*kÆ“ë.TgŠŒƒÁøH†¤(}UøùJã¿äô”RŽÁ]o?\~¼¨)z…¥Ä⢰¿ÃoôW­\œ¼Î!ùëD°qŠÙ›®oü·ºäÅß.ÿó«Ë_—uôÕL9£ýõo_ˆåìSŒÊ ¬ –ÒV`y?Åh|XAObÎ bøèŒÛ²"Äp9y‰1œ1I`%ˆa]ÔvËÊÃDí$ÄÐÙG¥äÐF)‘A”sÖ<í§¥‰gôR6YäEðt4ÏÚ)D§½–xy -!ø$ñdñYi‰ç!‹×Î)‰YœMQâÈbƒU"²˜­Ä‹Å@;Yà%ÈëJ"²¨ ´ÄËvò99'ò2x*Å´åå'Ÿ¬UOëÉÃ6­ÈÊÂ$…(ð d Êg‘Y¼UFâYÈâð#ñ€ÞÆ”„Õjhu[eµÄóŘèDdÁäK`dQ1H€4.I`bœ\6N“ìä’K -˜”'£•`Àd?¹£„ViðŒ–€À*Èâ]ÀjÈâ°rEd±ÙIX` dA ¬…,ÆY ,ÐÖÁZ$,°€[§²–°ÀoÒA À…˜J ĵ)8 , ¦”%,°À\ ì”°Àt±Ü£„¨k}0XÀ®u)HX`»Öi%aîZ‹š"²À€%,:NV'+aîZ­¢„¸‹ed$,À¬N°Ã ale2)) p×$å$,pÀ]áD^œ°­„¸k<† ynà®ñÊHXà€»Æ™ a eþ(1îB¼@¬„ð÷à+a›  ">“™‰2Î$î(@¯‡o· °Þ*Š{ -àK9’¸i1‘Y¦ îÀ­‚©•¸m €¿ˆÇÜæÝ'¾E`>3EdƲpˆî{Î1ŠÙbóþØßþtùõ œïõO¿¼þªì”µ ã·.PbM¹m»`¾WSiÏ!ÃD°mÈ‘„ð‘:$DÌCH X¬›n$$\1Üm¾ö/z)2$&™5‚™22·è©‚Æ\Ù Êúî:?‘¬CGñB"Doçõz²ÎO®WÞïY½ž¬ó“Ëê鹬rË~E!ï¾+¥×í—{B¦(ô‹ )ž%ø…=Yé'z ÿñ邤è¾ËþÕ¸d–-DµÚ¤ë§.¿þö7¿ûí}‡¹ûôzù߯áS!ÄGJðÍWúëúýF‘>=Í4¯ëg¨Ÿî•ri.‹E Üç™•ë ½–Eå—®}®Ÿ­8ó<µaGŸT¯´K}¾0Ô¶~){[ÖmßKY&›7£Œ¶¬¼cœûþæÿ®Ÿ¾=¥ñÌ5NJv¯ËÁÒuéä¥ -Õ¡]×¥ãt6J²õ׳ÔëÒG 6!{QxµåÉWŸéƒM¾^ÊI´2æ[Ub¨Ú \m´¾â28ìª7ÛÇGmÍmù~ÛN°—}ÖM;ªê£ÈŠ¾ŽäqOŽjl.®ôHÆfhÜ¢l´ß”Ö -m—…Fmå*goiSÏÆÞæµ”ýRuá‡-q£¦ùj²P›EÎÖF»®uÔë˜ÿ6¦åÂe6²®ÛP[`MŽCݦõø—¶7·ÙôÝÇâgù,t`Ñ— ¤ç­ ³íí­ƒ;k@X£Z¿c~š¬ªÚI•ÅEÖ~«“ŽtdÔFG àã WÇÞ9—wl¨Ï¥¯úh¸t` t±µAe麬µ[ýŒl Åc¾6‡ú1ýŒ‡æðh3™Lý.ÚS  § Žt îØÓÉ:ObOCl«£[¿_}÷K½?âEyèYð0;huÆ6õ²RÞÊe‹·ç‘NÙ1Yaj•Ñ¨Ç£•‡"##Ýmhdi†]6÷¼èÉ:OgqþÁÞcg-`íA»—æ1ÙëÖ¤سӾŒ66ÞÖ1Y˜Çíqá­Z^ڗͳ²¼î‡û ó*^ô+ÛdxÏÕy:›LÇ(Øì¥åqù¹mµÏ¢w[o -Ÿ»tK›ô«|¢ÓÛõË@˜îI[•¬ëŠ#¡Ëû}·z{².¢R]íV­dZéG«4±Ý{ÏÑ•-cË;WçÙ,/Çs]»+?BˆôáÚ”r•ŽÒn÷©lö ‚­,’gè½ ËÌ»ý¼ ©®ámúåä±ÊÖG¤vÚמ¬ótÖuÝY¸¿šåïßa«õ¿Æ „¿Ã6–Ü,òEÈ(V17câY:ÇÃw´_|ò^œÑÚYG“âÎÀŽÕ²hO¹Éh—œ»gµçê<ÕGˆkºÉ}$ÆÿEv$ße-‚>xÔ׬v'çZ÷µ¶nwÏ*y¼GOJcÊã=«!®Ï“@º]âWª5í0™ÏfNA›‰©’ëår+'n±\mÍ÷™Š½TUsÓüe˜ºM]ñíÆŽâˆt48[#ʶ¯K2ÅÐçZ%f7fC¼]àb܆üvñQ(íV(ßL-k·““ö2¨15Uai6!EÿÏPá;•§|Ñ®›7®F'´6;y©$Nuœ¤íâ(ËÔ¥cì6Ëõ…ÀS(O ÛKw-¨LeŒs@ªƒÇ’¬»Õu2AXÓOÇJNTMt7Ⱦ›òÛ‚<äô}©zÕ; š™»ïírèØùÆɱϸ¦Cýº¼½]fóx+ŽÍ­Édú©u2ÈFuo7:cÄ!@aŒ¾r[­è òöu,2ÉãÞ.¾7g¬7ô†i°šÎ¨å‘—¾þe¾Åï_.îú‡v(ÚìÇêha:¶.Èܓщæé$™N8oľlÑ8ûƒ¬Ó(ë›5t?n Ãk7ª«ì­¤îª;—ýÐ!#›nºF[¦@¼­´Ø™ÎÍpØó¥mƒC7–ŽÓkRá‚—iðìpÉWÖ$®¨èpéà:Ã*â‚7ŠKÖa’-Ä$ƒätÕ“üŽO…¥¾Ø #èÒžn–7Ýhe¬B`Ë/xÃNóNóBA臗ׇ罊Ú - j‡öN}ãÔ>0¢vÀ“©¶\†åe÷9¶¼þÛïÐPo˲µg{3–Æ-Üâ‚Û(£¦Q­ð¼S×h‹N›£èpšj´Þbl2…Z­©¥Œ8‰ZŠ=¦Ô?sM{>+]‰žÏJØ”¥v›ÇëÑ’íçæÇèÔ(ÙcÉ$Яݩn”´¶íÛ-ûnÞöʆᆠaD¦jTó›’áß#kG<ÑO*Õ~ÊîˆNãÑ@ç%FklùË0*… qt¸wt‰m—¼»ÎñþÉv—¤íŸðž0“³VEMãGá Œ|ŸÁšjÿG®–•8¤÷)@Ø0·=±ü‘‡²Y6‰úe¤T‘þ¦‹¥ 7ÑQôQiùÆvã*,¶PÊÏ?v;I< -endstream -endobj -751 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 750 0 R -/Resources 4 0 R -/Annots [ 752 0 R 753 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -752 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 582.273057 101.433071 563.523057 ] -/BS << -/W 0 ->> -/Dest (cb246-1) ->> -endobj -753 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 521.326036 101.433071 502.576036 ] -/BS << -/W 0 ->> -/Dest (cb247-1) ->> -endobj -754 0 obj -<< -/Filter /FlateDecode -/Length 4851 ->> -stream -xœå][¯ä¸~¯_QÏÆÑ…º‹<,°d°YìÍC°§kú$Xœ 0ÉÃþýý([m³\®Ó=Á Òê*“ºPù‰”e·½üýÎâŸLvʹ¤H×ÛO—Ÿ/fJÁàO-±º¨ìðIáj ¥)Øs¸þÄŸ¦T‚KñúvÁ?›KYüíò‡_]þ²®c¯n*%:®ýÂ,òSÊ)™ ° -XÆz…”’ QaE;aˆ¥h,ˆ9ڳĠ’ƒÆ‚ä\VXbxJÖïYb¸dIcA [BRXÖ@ëŒQyÄy§ðl˜"”5ž³SÌÅ•—À³Éi<律È«ñÊc Yãd ÅX K°DFãAò9i¼Y|ôFåA—“×x ²8h§(¼ Y`]YåAÕxÅO¡d"•WÀ39å=Ï™0…ì½ÑxÖN¶éU< -““Âs%šPTd Þ8ç! á¯ÆNŸrV¼Õ±w{ã­Æ ŹD*²`ò5p²˜5€MŒK¸”&*Ž4pÙO”)k(àr™(%¯Á€+a¢X’†ÞXðœÕ€ÀÈ(jHà-d!x®Êƒ,¾†ÞAаÀ{ÈâÈkXà¶kÑ°ÀnÉ«aÞ’±QÃÀ…˜FÃÄõ9’† S*x`®vjXàºp÷¤aêú†°ë)G ³†Àø +ªÓ° w­ÇJ­ñ€»Ö£aAî"nÀ&‹@ìo“50]•W&SrÖ° wKNì“É@•—&’Õ°`<Ä¡Dà.–÷¢aAîšà½†¸kaʃ,>; ×€·U|ÁêdBãwÅ k<à®1©hX»¼ljX\Ö°Nˆ‚†1ÓTRŠÄbÀCȧòâ„ë aA2 @–ʃ,”HÃÖSñ%kXdñÎiXdAÈ a"‚©XDßjzY`.j~@‘¢†@²)" RˆÈÇ UÝ”àZåeðlÒ° %š°[ KMN¯a~âpIÈž% R,p °MÙ!2×x² ÈÖ°‘.xÖjXdA4¨aAvYf4`…F. Uk<2œgf 2ʼnCE à&È4”*²&4,Àüp²4,@òƒdiŽÊƒ,>†9;N7³†9Cgœ†°MÎ7“†Å@D|'“5,ŒƒgŒ†Åf䛎4,€_r¾™5,(Þ ßDŠ§ømñ‘óͤaA!È¢¨<È‚ ©(~ LB¾ tÒx²¸B”9ßÌ Ì@¾‰ˆ^ÝÈœo&u»¸ -Iãw™F p|‘­–ÛoHˆÏtfæŒ3«;а¶ª[Ø‹Õ*©{àË9’ºi1‘Yæ¨î`YÓuÛÀÑæÝ'¹Ea kfN<È· D÷=çÅübóþØ_ÿtùõ ßëŸþvyýUÝ)k-„Zš_Á \ÁtDßÃ÷Iyª¢A²§ -s·© ±,€pÙfc±°àñŸëþÇJVˆÊ¢62¡ÈÙ?2%^sÇÀžÚ:! FFàx•²>]ç¡È 1A¹öV—uèHö+”¤Èúto$ëÐV6$©!æy½ž¬óÍõ*û=«×“u¾½¬…A4{C²_@±*äƒÂßÖ©F¿QѾ_˦hõ¬cÓó[;½Ý°.³í§Ê×uúØÒºNïÑm=~ë~ó?×Oß_þíÓÚ0j¦X ƒØX¬Í1>°¥sU¾Æ–~ *åÈ„¶fâäT5UòÔ6sùŒ¯Ã\èËFýßdÊðÛlÌ¥µûy-Wk‡å¶/£âMc>(<´~:Ï,üŒÏm6÷JK3­^¿,´Ïó§×å:F7)¾ûµHð<‰T8=´©“u>šQYw LˬU-Òò0Ô™aZ™gKH¯³¿¹4ø[+ š*ˆ5«Ý€ lwk©laÕšØ -6‹gúÖCHðºgH™¤õæ}{ª7…¥¯æ/¢®c¹pÀ§W% ý°”­–'ë|8k¤cŒû,¨]ßµÖ°î3l–DR,RÎ~¯/0®Y]-»mOZêvIÝÈÌÖÝiaSçÞò¾]®›UJ]:!sëûv€—#8#ÞÜsÙºðÈBÏÕùpzÈm-´ÏJ«›<+ÝJÝÆ–Ž6ü:SEàÝƘ_éqÌæ. leÍÆ"ÜsòuÜ{²žŒ:¶ãZë뎎°.š*ä=ÄÉsu>œ>ˆ7!~Ÿ…†‡›” Ça?nBýÛÆ"”l…•¼RS÷¤$O¦>2¸g•·±B^êØ›9}¿Ý«"_ÒÇsÏRˆùÀB„ø(¾l[ù·ü"ï­yãR.KÏ^gJ§Ô쥚ë¼ß“6Ì~Óµg–›v“ÌgE¦á8´Ø´[Ëlëø1Ú‰‹ÑË‘sÃ;-$äJ¢žY—¯š^lÒÆE-q¥.žÏt/úðËBä>–™á™ýqßNô|àÕîÛ1˸æ1×qÈAÖß“ÃŒ±¬ôÁ–á–lk/®Ž†€MJÛÕ[–=Ì*ç2ÞÚæ‚—m~jÙ/‹.™š­ôyC–nO­­}½.t1¦µ÷ ;Tlsžïfck9†'AægºU[‡sá·úZõ¹•ñuæU$Pæ¦ÖqB×Ûú9é&Æ-Û<«“<ô²åÕz©ùñº?F’•|·¹‡óÿ:l¥Î:ÔmÐtÛæÌ žˆçéà˜£Z6 Œ«>í…Ý·6…ÿ¡xmWŒI›[k¥¯ê5íôzϧÛXŒÀËEþ¦«^G]äZÐÖ'ýúžéÒ| ÆŒK\¶ípßÑWÙé+še§jÌQgj±¦Î¯ýŒ‡çðh<ÖŽ>æˆFfc¢MšËÕ>­Ùa®¶Øç–~>/ü—aÇÝö‘žÂní˜oÙYûþ.Ëùzªòé¶FÂÎ’Ð_XÖÆÊØà#Gó¾ ó^ -¹y-¨×aÙ_Y0¤~Ç9ž¨ô¼”§¹¬ƒ\þpÝûµ¢ˆ1,ëuïƧ÷ÑßùˆÒ™LÞÐ!-Qd:êï¹Hó”,ÄâS(TîÈâÚüÆ[û Ÿ]4»Ò|мÎÑ~Í‘žÆ|ýxÕ1¼ ßkþÉ÷Å›?Úµ?—ÚuÃÜe ßãB3ÆÚ×ÀŠ#‡úÚ¯%'åx¤×÷ã䑼є¼s~`VëP‹‘·ùôa‡|ñ—6ùŒäèxàdwû†©èòR™rXbÔÔ¿.aÛ­¥vÇ‹dzž„»–8™Â­l:üãf3°Þ<ŠƒöôÖƒºÉ Û?¿0ÿQÝVÐ ¥:ë„æž%>Ú_ê7ÊîÜÁ “u>Œ%χ÷R¶ñi#RÏ÷“Ãz–Š)È_ׯŒéh¯òkö×îÖ»s³ùEßS]%%7ñCÀÆ鮲¬W¡Å–b-hë·¾w²ß›ày ?2·é´ÚÑ=+©¹KðîÜ9©“u>ÈNm÷´ŒI9ð4óbNÜ 4_}¿Ë¼ˆ¾­Ûoô݉íOÅZÝ~ÇsÙQtiðÎŽñïugä¡.ôã¢üT×lüÜ[ðÁ><.zªÊ±Ø!µOGû´•œG­œòjuDJ#“h;µ–¬§¢UV£Íz˱„œì¦Ã#Ä*ÆwôáÇ:MvæÑ­¥“u>ÈüwÄâÇßæ¿ææ·µ‡{¯Ó4º }9<ßpYðÒò¹­iZYn³ÒD?»ów›sx=îà|‚ŠÏùf…žç½œNò×=ž4çxµÜg!“;B“âr‘u¡©Gpr²Î±§!6ùÃXÓ/Ÿ7¿ÃZïvçmÊ°ƒ¾mO¯)瓶ekl(ãbcÆÙ¢ÛÚþ¾æ>ü»âÌ;§KhtölúÉ:Îââ!‚=cg-àÞià£8d׉L¥Ó¾Œ6v«­¸k*WÜžEÜËË÷e[éu¿ºÏ8{ñ㘊Ÿ(~ÏÕùp6YŽQ°ÙËæTuû¾{Vœ¬Èiõï]Þ9I¹ó‹ð`™Ììev\íÖldÚèçþ“:#8#¦’¨î>xTç\¥f{Ö‡tÛǶÂ|Õ¤)mj‰dÉ;më[â£\%ƒ­ŒØÊé˨8Æ+ë>ÿhD7°–‡ìSÀL„À/9ÉÁfñÌ~7ª‡E˜Ðþ ÿö¬ìÕnžO^g[… 8ñk»,¥$~!Ž\‹ïÄq”m—»õîÚmŽÚ낲DÀ¬æ宥k9ÚrW³ªÙ˜qí‹éw°j¾Öþžë§lÄT4»;Õµ¿å.¶< ±j¿Ýò´»} ¡a7•?_b}¹Š­/Ø2W8~ç5ÒÄÉH*W~¡K6žl›:[•m¯wjÖ÷–þÜK-¯25üv3~³ÍüžSq±”ÿ¹NúO[hbý×£`õµƒ¦’³kƒüvá;¤‰Æy²d&.?Mo‹l?žÖh7 ešÊ¶$òJ³ügØ8ÙœEã¨Z5~™VÈ×bÌ+DÔŒvOæÑ›É.d~ugžür»(!6™A ’,;’ù¡øA {¢Ý ²£&åW=uÕÅAì¢KÕY1ý­¬Kb:ˆ· ºõ[ê›J嶦švé4/„*ZÉ>!ÔçI!Ý. -ñ;Óš&Lfç‹Yã×.ÍÄÜ ‰z¹ÒÊ)¤ÛE-·´úL¥^*ª¥i~ˆ2LÝç®x…vç<¿‹XøB ×æA’ù8`Ÿk“…ݸñvÁC;òÛ%$¥tXaB3=Iô¢ÝNÎfØË òãñó/¸f£ñKkøß*B§2òÔ–ºIñ‹§ÆÕè„}³“×J’ôØQ‡4 XJ£¬P—M©Û¬ÔÒ(¥<ƒn/Ý5¶¢ -• z*M<©k²íV×É aM?+%Ñ4!ÐÝ ‡nÊo+ò3tW ¦w­0÷ÐÛäرóM’SŸqËoëòöv…Í7â­.l´%³éçÖÉ ;ÓW»Ñ™ Á -ƒ<`ô­’›·vtªÇ-?‡“i+îíòçK]朎ïØFoù¥˜2ò²×ÿC™ïñùß ]ßÞÂ4¯ãÎt´p[Wd¹’ñ+”óI2¿R¹»Û¢#ûƒló(š5ôuÜ9×4ª›ì­¤íj;WÃС »nºÎz¡™p@¼m´Ø™¯›±`Ï—¾ Ýx~W“ -r8¸ÌƒçÇ’|M⊋Ž%\r¢".d£¸–!ÙJL6HÙIW=ËOr*<÷%.DÁ—~ðh,³²éF«£E{y!&+;-+y¡ßí^_=ï‹È±yS;´wꛤö1µžNõõ2®/ûšãëíôæ¼CC½-/|Ï÷ g¸Æ-hu!mTP󨶲Sj´U§m¡èpši´Þbj2Å¥ZSKqVµ”zL1¨–šrVºƒœ•¸+Ëí¶¯GK¾@Ašs£3£d%³B‹²v§Ò(émß7­ûn«íU ƒ† qD¦fT »’ï¿÷øÎ\g³¡•jëîˆÍã½…X¨É&ï|ý¯(LŽBü6áDYl—<]çxÿd¿KÒöOdO˜Ž‰¼7Éò*¢‹ðFyÌMµÿpD¯VŒ:¤ç l ˜3»Ý¡úVùºY7‰úeâT‘ÿ ÏAoæw_'cõGúhÙFI«-”ú÷ÿ®Á -endstream -endobj -755 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 754 0 R -/Resources 4 0 R -/Annots [ 756 0 R 757 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -756 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 367.623057 101.433071 348.873057 ] -/BS << -/W 0 ->> -/Dest (cb250-1) ->> -endobj -757 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 306.676036 101.433071 287.926036 ] -/BS << -/W 0 ->> -/Dest (cb251-1) ->> -endobj -758 0 obj -<< -/Filter /FlateDecode -/Length 4867 ->> -stream -xœå]K$¹q¾×¯¨³€Mñ|†dà Ëðû øÐ]3-Áè5°òÁß™|D2£ª²zfµ¥õööte‘äÇ‘Ùl}VøúNãŸHz‰1Oç˧ŸNj Ná¿RbsQÔ?à;¸³V§“îücØ°„äLðç÷þ™.yñ÷ÓüæôßÛ:úl–”¼Ñîü×/YEv 1åU‚Ji+¨œ[B0Î *¯ÜbJ’ -f¸@†öª3(E'©`U„–‚¶{U‚&h’T0C'•V°C¥D QDÖ:íA÷’ÎèÅÇd’¨ Ðé`$µ‹¤–tiñÞ»(鶸¤´¤s°Åi"%é` Ù$‡-Ö[%ê`‹‰ÁJº[ F' º[€®(ê`‹òJKºd—"‘¨KЩâ^g”[\´VI:­lZQ‡…IòAÐØâ•K¢¶8«Œ¤³°…ð%éÀΆ…Õjòê¶ÊjIç`‹1DlÁäK `.û6óXñ¾z¬Jýyåçö-Ù¦Íä2'2?\džàïùë<rï8œ_VþhÈã;ÞG‘BÂîùÕm1Ü!ÿ†#ú·F»aõz›“Ó=¯î­íWûoó`§Cì1ókÕýP‘7~Å+£µÃ=¡yßÃG™†ùÉ-®Àì2Öy6dšÛ>g‰@è:Úvœ!D³ùvmOl÷Áƨ¯‡:ë~µ½÷Þ&«>ÇñÊXU›ÛšãØ/åùZ¬÷ õÍ1uó˜Â\Ã’ªØ§ªÒœJ¬Æ­Žwwðî8øÜßÀmñ1Ú<ù‘½2va‡G¯jfƒáK­÷²ú&²~<ÿ”ïg}—øúýh=úè¿…¤„6ëZ,}ê='t¼k6Tûy­ú—±&Ö}>¾9Ni7Nº®‹0áƒaèÖZÛîÀûõÿøîËsuýxu» OÝ5cöÖøÄ÷g.·™Ç\öbÖ˜?ÿÌy2+¿•kWs—š/¸TUy¬åi-k`¿õ[®(÷U÷¬‚“·©²¯8Õ÷Ƕ.gœ¸¶Ïq^»‚õ¶Æûüz6NZï7Çn·'| /ô{N[Þ/å_«%öbãôùöýR»nÜVïuƒm=ðÜl.ëƒØxßmÿæ8íxþÞøœnÚÓŒõ¶1¾o¾Ò—¥|«-Ç!äÃIu°ÎêíÀì`¡ð C2×í5i‰ÎÊŬl ®e{±,ÐY'áj‡É/*åÞ¦ÿ4%iJRßÙÃ!¡ü‰ØH£ë0ÿ^ ÷$ ”Åzű½ -"ÜzLѪœ}7Ç^Ù8Xçi@¤U>Q‡¨ýÃ(_¼¸&£h I%2SÏ8ýe‘~™÷7;`¨g‡v.Ÿ=Žì(¥ŽÒ»Eî€rðR;欧ƒb¶+MæCÏm<!…|N˜´n‹ãñeuF´k6;+Ü)Ž«^_åèŽGlÓ½Nu6ªóÙœÁüÓ6åMõqLìÙfçeÎTç®ðsL{ûÍböl¹T!Vîa7•?|9óN—sO•‹6ù`ô³§%ûj!ó9{QYÒmêtl}¾R³'ÿS/UO˜WùÐÙ|ààzü<»¨å*“þãI'Zòø—,P9 Z1ÎÍâ÷SÞt~»é|è4›þVÖ6·„—ºµ³ô]”æsœ[Smtê2ËŒJ]êZÉ>!ÔçI]N‚ð;Õš&Lf׳Y˧a®ÂØ€D½\jåÑå$–«­¹>S¡—Š£jj#?LP·±¼ »°oÈ"‚­eÛÇ­8g û\«ÈpcvÂË [ íÄï'„Ònp…r z\hY»]ÕÀËæWd×OXšM–ÏÌÿ®Táº43Où ©C*Ÿ:®F'ymvñv¸ÜwÖ!i4…Q– —¡c–—ŽJ(ŸI·—î#¶‘²!còšy™¤:ylź£®‹3…µñé\É…ªî†Øu(¿oÄÃN×—ªS½3¯Ü]owˆ}çÎw.}Æu>ܵÛÛÛe˜oÂKÙØhgèÇÖÉÕw»ÑÃ@ -CˆŽÏŠß•Íí¶¯{K¶;@ŽÃ9Œ»S£d÷%£ ó¼v—Ò(i›lß7mûn»í™Ý CüðLÕ¨æv%?ÄY;k)!•j_%;¢ã8N5é`-!LEïÁ!”ÿÈC ÈÒ%×¹?ÙgIZþ„÷„éXÈZtþ[o² ïP¤û -ÖTû;prµ¤Ä[zl„ôР9µË•?öS^\*I¢~r¨˜ÿ¶—ÍNoÌ’$(-¿ÕC57)”òõÁˆH -endstream -endobj -759 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 758 0 R -/Resources 4 0 R -/Annots [ 760 0 R 761 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -760 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 157.623057 101.433071 138.873057 ] -/BS << -/W 0 ->> -/Dest (cb254-1) ->> -endobj -761 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 96.676036 101.433071 77.926036 ] -/BS << -/W 0 ->> -/Dest (cb255-1) ->> -endobj -762 0 obj -<< -/Filter /FlateDecode -/Length 4921 ->> -stream -xœå]]ì¸q}ï_ÑÏVæGñ 0öÁ€dYà"y0òÐÓwÇF0`‡üýR"Y¢J-ÍŒ×n¯ï o·ªHªDžª¢Ôl}UøûFã¿HzŠ1O×û—Ÿ.j -Ná_)±:(êïñ -Éé䣻þ˜6L!9üõí‚ÿ†C^üíòŸ¿ºüϺŽ¾š)%o´»þù‡¬";…‚r‚*A¥´TÎM!ç•×.1%I3\ C[U€”¢“T0ƒŒ‰‚* KAÛ­*Á 4I*˜¡“ ‚J+Ø¡R¢†("kv“G§{Igôäc2IÔèt0’ÎÚÉÒNKº4yï]”t[\RZÒ9Øâ4‘’t°…l ’ÎÃë­u°ÅÄ`%]€-½“]„-@Wu°Ey¥%]²“K‘HÔ%èT q«3ÊM.Z«$Ö“6­¨ÃŒÂ ù è lñÊ%Q[œUFÒYØBø“tà gCŒÂl5yv[eµ¤s°Å˜@¢¶`ð%0¶¨à% -À š2/I`B˜(’HÀD;Q¤(±€‰i¢¬D&¹‰| -X¥¡3Z"«`‹#/1Õ°…0sEl±‰$.°¶ €ÄÖÂCVâ ¶% Eâ º%•´Ä|KJ{‰ ,f*‰ ,×FOXP. ”$.°à\ À‚t1݃Ĭk7XЮ¥è%.°à]KZI\@à]kQSÔÁXâ°ãdu´x×j$. ð.¦‘‘¸£:‡^â`e21*‰ ¼k¢"‰ ¼k\€¨ æ •¸€À»Æá$Ï Þ5N‰ ¼kÈx‰ ” ü£Äh¼ -qr8[@°ÀßCg¬Ähl2K$.pà]àþ„9 ¶štL^âÞÕÃ'êÒ¤z\˜·èåI{Ÿ$.pà]8*+qï‚>£Äàø ÕH\àÀ»ÚÂSK:ð®6II\àÀ»ˆœÄ€Ã¤H\ÿ6i芺4©£Ä¼ Z2À O*‚ID]˜$-qÈxRˆC%.ðà]¸÷$qï*g­Ä¼«a¨ƒ-6‰ ×@‡i+Ìi«“2` -IÞU#,éÀ»J…$qïf·)qY`¢Ä0pBl$.𑦂—¸À'B>Qç';‰ ‚‚À²Dl¡@ °ž’MQâ‚``‹5Fâ‚`` B‰ LI#úÓظˆùÁDŠ€É¦˜ˆ$.Î#C¯J:o¦@×¢.B§ƒÄ!ÐW¬%.@.5ELz‰ @ðS—$.€Ði’¸ $Ø‚©"q\Ñ "sI§a ‚l‰ éB§µÄÑÀDƒD‘ez%q<4rItµ¤#•óÌ(qA$?åPQâLdš`JQ[@`|r²í$.@òƒdiŽ¨ƒ-Ö‘Ä1šœnF‰ b„-F‰ €Íœo‰ ’‚-ˆø$.HÊO9L–¸4R$‘o’¸ó2ç›Qâ‚dòM¤x¼MÖç|3H\¶8„(¢¶ hJ¼'!ß;I:[L"‰ ’÷9ߌ Ì@¾‰ˆ^\ˆ9ß ârx×%0“¤ï"2õàù&"[)·W ^ŸÉʘ3Î(®(P¯ƒo— ¸Þ*ˆk -ä›s$qÑf"³Œ^\5€[…R+qÙ@цæÕ'¾D凟C¾È„iAˆî[ÎÑ‹ÙbóúØŸÿxùõ Î÷úÇÿ½¼þª¬”Õ?¡”&àÓÀ$ ·Cô­qùÖ#Ë~9sT6ö} J6!¤n¢).KnÙD8¿üïºý°²f¿>;\˜púÿJ¶–þ š¹íöß~¹€_æµMŸÛFjçór&™[F_~¼üú»ú—þ÷ï‹/¯—?ü J¹¼@±„Ïwg29bÊk ó±Ëº,Íe¾ý¯ë—ïø9MN³Sh×ç̧Ñ_çâ¿ûRú̶ñ ˆ’J—QUL†Ü1q‚v€Ä;«|¦¯—.Æ% ˆÁõ!Ñ7™ô1“quŠ6ýNõäsW´Cm]DƒÐ+Ãð¯ßýÇ¿ýž ƒº¡ÔürîÛo4dÎ,ïa~'Žãüžë_ËZµì}yOì³Ü®[ôÙ6e×í4Ý"wzkoÓÙµMcûíXoëZ3ﮟd fh,ˆ2ä&v*;‡ »u¿ÓyF/Ÿî¡×{Æ1®ž*uÔ:µM§¶Þ¼rÝX–nŒ;õâùn®øË0È÷M]bܼ貼Ù~__[Ñ}àšµÙa0×ÙÈæ¦dŽ¼ëÉ:O‡8ÿÁÞƒ‚³=hEÜÊC¿nÖ¼ö Ù*û¡·±ñ¶Äla·Öu÷yqß6ÇÊòºŸ>§Ÿgñê¼2&ÃXð\§ÃdzÌ‚/ C46 £­ú^ú=3§–ß7¸¤5&[{㼨ǷÎ0Í“Ö:ƒ­›¨•1!¥ýs×z{¶®¢R½àV 6 ý³—kØ›aHðùv:@ÞÉ:φ<û8žk½;ø‘ÌHŸ1¡M)Ë<ÌaîÃ(/>wDdfLÞ~)scÌêºc\˜ëÞ¦[_ƒ|­;èê‘Úi_{²ÎÓ¡ëqt÷ mÜ0Ê_߀aþT‚ðw `ƒäŠÈ›Q 17×”¶¼hÞÙ~ñÉ{qFmgŒ&…Ù¶‹Zí)šŒ¦Ht„Úsužµ#Ä‘‡Ž"¹ÏÄø )ÌËIQ¡“ê0¾+ù’a\}Û}ó^´ýÁ£¾amê™#ºé•<ÞË÷ 1äGKÓ'ë<*LjãÊ[ëIŽÌ¬pM•½¨íê ë#?zûÃcÒ¶WüÈr¦2{ˆÅ«aÉ!x K;qì‡m‘ÑFlÍ.?YcyJà!ÚNÖy6´ÑA\8Fû53®è`Ù¯ùÌHh!Ûø扫¬ ìÛ)!rŒó¤¸TZZeÀµ¯ìp÷m›%Æäuk™—eÝËô—dÛÞú±ø1?W‡px÷äd§CîAÌùÃÌ-4ŸG - ÞóÓm1Ü ÿA,ú×F»aõZ›cÞ=Ìî3Y} Þpgûd§CëAú çW÷ßnl´XsnøðJ<¿¯8Öåk±_—òãº%Ën>}Š–UªÛÎ}*:‰°Ä'ë<ÂâF¾Š=x¾’#>ÛÀWvîØí­*•Ñ<â£1®ÏÅh¥žsbÊÕZìí¯½þºƒÊÿ9mÎ=!v²ÎÓ¡ò`ò³«ÓyêEET -÷ÀÏ<Ó1F -ee‡ñýê>ŸQqgS´CF¡g÷¨9÷ ÷É:φB°~ùKº_#=ÁsøΡÝ3Ø÷xÍ–ïêêòPóc”«ót(;ˆñžè9™GÞ˜óÈ&êòyV«úÃýøv-#ÿñŒÌ<ö˜¾ÇdyñD[ -þ(S8YçéPtðD¢z~à 2ºì^É%^Ñ©Qr^ÎsÂ:^áÕ[/Wîâež{Yîê¦S‹>{½ûŒø" ³¬×o™¼Ì¯V7>à¦ðþgoNÖy:T¬ÝÕ»õ·%÷ºÍ=Þò¹š‹¥yrìQël¸e¼³ÅrÂM¼ÃÐD<»x·»yÞçåïÍKËh ,¶:»’|²Î³¡1¬Ýý Ú}²yïI‰1G¹z†?1xIþ4_ƒŸxW[8Ç¿ïYÃí*˸3®µsß÷ù2°µ·¼×‚‰Ú¸#„ž«ót}ËŸlªÏXÊî³Ô‹‡ÛËEîÞ­ž×ß{ªB ˆ0ﳯñÞ;ë­ž®ëLF>°¦w²ÎÓ¡ð ÜùîgãÃ!+V -ÊNÜ·Xqå˜/JÙÀG²’wf?«Ué½ï˜¾lQÈïJš<›;Ëw—ÏæU°/È׳‹ÔðþçlNÖy:¤Ä—ßR>Z+ûûŠïög&¿Ï[ì¸õHâC«2§ž¬•Ñ?ož¬ólèŒñæî¼ÞƇ«§ôíîÀcÃû?:ÂwØcÃ&o|rG·AÎUy:ü?…øÑ»cߊ»'¤)#«Y×Ù^ -yo-¯­6QÜK¡¬²ä¸’=…;ߟ­\³ì¥°ì³@Ë -»Ïº²5ÍKEè|¼‘GÕ¶­éX/ŸõfÕK³,ïÝP÷y°¾·Uö{`6åoý7yè6–wÏϳìq¢Û’âÝö‡zunÆ|¶d9³îW•¯”Xd®,ŸO몽юÏýf¹zÉo¬‡ë¹b—åžR÷Íüz(o‡Æw¸7Rmç ÞŽ·y?¸¼çð¶b»]l›Ÿ)|dG4{v,}Iu©¦3Kö³µ-ï=í+„¶³m†µ¹FgïãyŽÊÈ›Ço@^EV­_e¯]V¸vi#—yµfmùåºÃc}]UÚÙ.%o•¶¾ðG4?T¹A…|ƒÚ×1”Ð&ÍåŠz®_jN÷ú±²¬ŽŸ;¿èoKW oþ\tzÖQ`MKûãÀlöq‡™M‡±Ž*Ýd»å]õrcéÞ/4sYÙÅÍèËÙF~Ï™™¯Ê±ë;™”Lå¾èh‘Ç¥<ÍeÍmÞ)¥Ø•˜]Ël)×ðÚQÙžÝPêÀ÷Í¡õ -ú÷ËVqÁ¡×œË[uF§#Ûy®ùíÃ"~z»O]Ýïíšÿö·RC“ù7"l¼¸ãò¶Š{Ûå¡Èݨ—a*Ý{_wcÁ¸ž—¿K™JHºcµ ;uœÖ®-nͳ9¶,í˜F†Œd=w³æ¶Ì«ûë­ý -=fÏjž¤%¸+×°ÊŸ.¾lªË6Ñ*/:˜ü;WOSFYH×¼-)‚Òuètél}Ý©Y~}ã§VjùA•÷èÎû³Î¿ÖÁ–ò?•Aÿþ—&_=nÙ<_1èÛâ·KþñÄDÝmˆ³pù¨Z[¤[Pe÷KÞ+%­«Û²P!ï“44®^M®ïjµ¼©%©ZÈ–mç BÔôz+ÎW¯&½ˆóPÄÉ.G¾™â|µÂÞ ñ²½Á^ñP¬­bB·*vº.6TíÑyÃâÖu¾ ›é¼ë4þZÖ6œ„÷ NkGé›(ÍÛÞצjïÕd–•šÔÕ’m@¨“ º_á7ª6M̦g£–7ž…±‰Z¹TË ¢ûE,·´æÚH…V*öª©ö|7¥CÝÆÖñ‚ìÎÂéQ‡Íeëǵ8Gmm¬hwܘð~‹¡øíâ‚PÚu®P®B -k·‰£êxéÒüXÁü S³ÊòÖ«ùÿ™*\“fæ)45Håí“ûQ?Iž›M¼î$.÷uHêM¡—eÝ¥Ch˜åý¥£ÊgÒm¥[­¤¬Ë˜<¤j^&©Fk±n¨kâLaµWr¡ªFàt]ì”ßVân§kS`¯'óšÁݵv»Ø7î|ãâÐF\ç½°›½­]†ù*¼ÇF£8C?Ö“t±QÍÛõ“1a7 ¤ÐÅFߊ¸ÎÖÆNù÷}êÇ>É${¿üéRÜœ±Î$|ðVçŸvà‘—¾þÊ|‡×_èúûº—ðìÇjla·®ÄÜ“åŠ'Åù‡ª°M&4Äh¿‹uìe]ECóãÆ0¾¦^]ub¯%ucPÝ´\èz2±iÐ5Ú²žq„÷¡[ óÙ±é{>´õâp›w¡®Vá€_c×Ùî’¯¬Iå¢Ý¥CK†UÄo‡ì„©[¶23’Ÿ¤u}¶ŸøPØ|.vÀˆ|h»Žº›åMWY¹ -V!°å¼aÒü¤ie`ô‡§×§Ç}1Ù×¥Ú›ôKÛ…ei#> -endobj -764 0 obj -<< -/Filter /FlateDecode -/Length 4865 ->> -stream -xœÝ]K丑¾ç¯È³‘ù¾cìÅÖ а‹=TewÙ0jïÁß)‘ I¡GfõxÒÓ…ê’"ø’Á/R2õUáçÿ"é!Æ<]o?\~¼¨!8…¥Ä즰¿ÇopW­( N'Ýõ‡L°aÉ™à¯ïü·¸åÅß/þÕåÿæuôÕ )y£Ýõï_2‹ìbÊ ¬–ÒV`97„`œX^bJ b¸@†Ö¬1(E'± V„–‚¶kV‚&h’XC'–VC¥DQDÖ<íI÷ÏèÁÇd’È àé`$žµƒ¤–xiðÞ»(ñ²¸¤´ÄsÅi"%ñ Ù$ž‡,Ö[%ò ‹‰ÁJ¼Y f' ¼Y ]QäAå•–xÉ.E"‘—ÀS1Ä5Ï(7¸h­’xZºiEvÉg ‹W.‰<Èâ¬2ÏBÂÄN8bv«É»Û*«%žƒ,Æy‹/!€ñE/AvÐqIÂÂ@É&Ú"E LL…`%0É äSpÀ* žÑXYy ¬†,„+ò ‹M$a5$,°²²X -A[$,°€[RIKX`·¤´—°Àp!¦’°Àqmô$aäB•’„˜kX€.¶{°Àu­óF ص½„¸kI+ ¸k-jŠ<È–°è8X­„ܵZ ¸‹md$,ÀªÐC/ate01* ¸k¢" ¸kL€È ö •°€€»Æa’å„Ü5d¼„P”ø£Do¸ q²;Y°ÀÞƒg¬„hl0pK$,pÀ]`þ„= ´tL^ÂÜÕË'òÒ f\Ø·˜åA{Ÿ$,pÀ]*+aî>£„ÀøÕHXà€»ÚÂRK<à®6IIXà€»ðœ„P‡AðömÐ -ª+òÒ RŒ8à.`ÉHX+<¨$ya€"i ƃ‚*aî¼' )ðàò‰Ç›Q¸ˆ7áÑ‹)˜ãÍ ¦ €».™$pž©—°7ˆ7áÙJ±½ðºÿLfÆqF1c ½¶UL(`/¬Us -à›c$1i1YF/f `VÁÔJL(à/ü1Za˜}â)*u°™1äA&l ‚wßbŽ^ÌnócÿËå×/0¾×¿üÿåíW%SV4”ß’/5pÚ'“°ÞÀ«¡´Gˆ0ál›l€²„ð‘:Yˆº°8%ݲ0Å0·éÚ.ô\dHœeÖpfÊÈhÖ/Bµ²^”õî:_IÖ>Gp°à38xoççõd¯>¯¼ß³óz²ÎW—5ç]VѼ_Qȣ‡ÒÙüﺾ8’°¦ë~=\]†OIس•¾¢Ðk øí§ ,â˜÷W¦›0 ö À÷é‡Ë¯¿ûýþÇ¿Çj}z»üÏo”¢ü’‚»€¿áà˜’õ™èŒo¾àÉ÷ô6Õßúé:×MýÚpznç†ß4ýõ‹öÕx]ÊÆ‘†ÈF7•›Úz‘QO×·é~ê«ÐüD÷lŒùIÄÛ·ÿ{ýôÝÆœ%>gyšœëÓÓ¦dNé&Ó^»¸´3MU¬ÙTÕ©MOÃAì<gJ)ÿ:õ§ÉövõçýáR½›ÚšÆPùus׎ɜé¥\Ø›®|¹˜¯“rÍëÞ<æyÚžÇ]yášÍå54õ3©giÿóØÆï>•ín›aBdlÐülŽBƒ+<Ï,<¢÷Öyt'DÀˆ%b¼ƒM&{Õp•TB”´‚ ª½ŽSÓnµuM^8õ03ª Æ}÷§ÿþCü‰Ü·ßèŒ ·ñoQŸ¬Fvº÷ø;.Âf‡p!UÊù•E‡¹5µ“U2£EF‘JÕbº®rÔû·ù½ U,dÃÍM¸k`ý½ˆ—’¢”Íêû&jØö²­DƹR±„àJÿôþ@‰NÖy%‚;è/ à¾[‰ÈOtZ.âfÇF!ÖL*ù´èx¦Lf‰ڮëô¦8·¹b•:n!f2¾2e“ê|é|í¦¾¦vŠù½Í2Î*ϽJþ›wëM2;ÃƑؘ_¾=¿Uì˜e!eä­2Ù+7Ù.b¶ šoËñ¿Úœµ­1ù™SÊÙÝE§E6¶˜Eà^· „v1?(:Øb'ë|Ä“û)v¦qo§©—êÐ͵e¶»˜–˜4otþæ»`tç4#•«ýW-ä)!ë?­™…Š"SJ/;k1öƳsYšl~q_wçû#é½ýÑt?t—µìG¾?Òä½UwXØ4¤MÕ‹÷öéPõÜå\·Š&çxv÷ÆÉ:϶7ÚÎÞ°9耱6¡ë„µ2M¢s·$ïÆ2(Z°ñÂô{›Ó¤²¹ÍBcý´½bú¬³çš ¯Ž=G ÄèLþL/å>Oå^™Lfº–õ‰LÃM_"4uäÎœ¬ó$úÔÅv~׫±Óïçŵ›Ïû#Ö»Ä%RK]jÚ¦Só6Jy+—-^÷ÀT‰øæ˜Zñ[=îE=äј ól^aJ9e¼¯qçê<ÆÅ]»G ÎjÀÒ‚V›Yæ·µ†I>q£}ém¬¬-1Y˜ÅmþêmÒ¼¸-›ceyÝ÷éÇ]<ëWÖÉô -ž«ól:éÔ> -V}©qN˜ÿÝò¶êß2ïvŒg¤¿+½¤¹NºEœÓèõþ¥#L³¤µÎBÖ•·ÊÒvßÍCÝuæ•êIoÕB¦Åüh%£¡ë¾‡ÈžJªiWóNÖy:ÍÛ÷çÚì.ìHF¤¯˜Ð¦½Æ.·Å*×ÌëB#yæ •aƒfC_×~a®kx›n>y¬ÚÕ=µÓ¶öd§Ó®}ïnOÚ -,sZŸïЀÅþ_bPqÂïЀ•&W|"Š…À½ÀÕ˜¤ÈþÎö‹MÞò3j;KoRÊ•mi-óö FS$:ÒÚsužNk÷=Ä%yrññ–Lé]Ú"Ì÷ú9©#Í\j7i%÷÷òÛXXòp¤•çê<›Vú}q™yk3É5sÃ+XSi¯j•lmêµµ?\Ñ¥OºÑö YÌTv15L1÷aiÃ}XYÛ<ËÙåwLc,ïKîjÛÉ:O§m~áÒÛ¯‘qÕýš¬„¢Þiâ,*ÛrJ¹ôó$¿TÊ Í"à:Wv1ŽÛºÍâcòºµÌë”÷2ýW’m+¿ã™ÿ˜?aƒà|ΓužNs|Î/#~´Ð|\SH°žn‹aàJów|ѵ¶V¯µ¹Œ»—O™NDIþód§ÓØsþæG5ªù¡"nü‚wFmGxºAnh&ó“\Q³#ÍeËOÖy:-<È_þ’ž×HocðþÑ#´{ÆÇî¯Ùrj‰?³«eçê<–øxOôžÌž5æ8²òú£ÜÏ,«¿xßƲÄ?‘™}‹»OF!žûHÏÉ:O§Eo$aæTV—=Á+ñ „+úk­šórœòxW_z¹ò/ãÜëôT/0žšøÙêMp-´0ÒÊýdqKýWV7î`SºÿÝ›“užM«ÒAî®>­™b¯—qÆ[pë3ÞùÓôùÓ½z:Ç'º·|ê×÷ÅÞÐôÌ(©~ÜQ¬j0!D^‚ü™5§¦·FÆ©+Ÿ¯Ç"½õȳUâU?#gº.íĪS9Ï“ëçÈu:£aÖ~=~‰ÉSÏÝicpu «¥üñâËI—ºœv¬|´É_‡põ4d@ éšO׌ʒ®K§ËdëëFÍò%?¶RÓ÷J¨|Ôt>ftüÒ v3•ÿ±,úhÈóŸb<^r4õ¦“ß/ùÌÈ3}’œ‰Ó¥jm‘®í  CZ–´nPÓejç¹—ÆÕÑäú®VËg3’ª…l9pìX ¢¦×kr½ôDÎߣ;Ýù& -\ÑÔˆ½Aâe{ƒ½â!Y[ňnMT¬»N6TåÑùÜÝ6u¾›è|ê4[þZÖ¶œ{ÄÛÝÚ%õ]¤æÓÛkSuvj4Ë„Jêjɶ ÔÖI Ý.ñU›&,fã³UËgàŽÄX‰Z¹TË ¤ÛE,7µæÚJ…V*öª©Î|¥«ºmâZ™£‰òð=¢l½œ“}^ºÖ*2½1+âíC+òûÅ¡´ëX¡\U=N´¬ÝFŽªëK§æœðx…­YiùÑüÿ®Q3ò” MM¥ò)Àý®w’÷f#Ï'‰Ó}C’f@SèeÙtéšÎòùÒQ å3è¶ÒmÆfT6eŒR/ƒT9Y7­kä au~Vr¢ªB »NvM•ßgä.§k[Õ©Ö™×LÝ]k·“}ÃÎwNmÅu>Ò¹ÉÛÚe:_‰·bØhIΪk'lT³v½3Fì‚:¹Ãè{!×ÝÚÐ)MM½ì›L²¸·Ë_/ÅÌëL>“Ì[¿¡€{^úú”ù¿»ÐõõHÜÑŽÕÐÂ4l‘¹%ËßgO’óAp•Ø¶ #b°ßÉ:ö²®jC³ãÆ0¼¦^]u`¯%uCPݸœèú2²iªk´e3ãvˆ·Å,6Gæ£kÓ öxkëàÐ͇)W©pǃÛØy¶›ä+kw¹h7éà’aqÃÅ-ë0uÉfbf…ä´©Ïò_ -›ûb7ÌÈ·¶ó¨›YÞt¥•Q°"p-¿á “榙<‚Ðo¯¯û$²¯; S´7ê;§¶ej<™jË­Ÿß6›cËquóöjmY¶÷lsrúÖè®Ín¸Ž2jìÕ¦ŽwJ•6ë´Š ©Jk-†*“ŸªÕi)#Žâ,…æStê_ùL;¾*m_¿*›Û­¯yK¶9@Ž«sè£S½dó%£@ó¼v£R/i+mÝ7Íû®ÖöʆA]ß=SÕ«¹Uɇð#žç,•ª?%;¢c?D†št°Æ–ïTÑ{`å¯v Yºäî:ûù“u–¤æOxOXŽ¬UAçox”Ex#3XSõÛåjI‰Cºo„ôP‡¹õéÛå+¾J³$‰ÚmÈ¡bþF?›Þ˜¿ˆ((ù4îþJñ,…R~þ GÃ[„ -endstream -endobj -765 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 764 0 R -/Resources 4 0 R -/Annots [ 766 0 R 767 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -766 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 713.673057 101.433071 694.923057 ] -/BS << -/W 0 ->> -/Dest (cb258-1) ->> -endobj -767 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 652.726036 101.433071 633.976036 ] -/BS << -/W 0 ->> -/Dest (cb259-1) ->> -endobj -768 0 obj -<< -/Filter /FlateDecode -/Length 4841 ->> -stream -xœí]K$Ém¾×¯¨³€MŃñ  ؆–áöÁð¡§vF‚1k`åƒÿ¾?Fƃ™Ézu·V-a§ÑS•d< ÆG223Úž ~¾³ø/“]r.)ÒùòÓéç“YR0øWKl.*ûü¦p¶†Òl‰9œb‚OK*Á¥xþv»KYüÛé?~súŸm{vK)ÑÙpþÓf‘_RNÉ…UÀ2Ö+¬–”\ˆ -+ÚC,EcAŒÈÑ‘• •4Ä ç²ÂÊÃS²þÈ*Ã%K bØ’²rXgŒÊƒ †È;…gáô¨ñœ]b.®¨¼žMNãy¿ÄD6XW–cÈ K(Æj¼Y‚%2²ÏIãEÈâ£7*²¸œ¼ÆKÅA;EáeÈëÊ*²˜h¬Æ+~ %©¼žÉ)y΄%dïƳv °M¯ò°¢0I1)<Y¢ EåA–àÓx²~4p"ø”³²Z¯no¼Õx²8—HåAL¾†.B“¢XA ã’†.¥…Š# \ö eÊ -¸\JÉk0àJX(–¤á€7•\ ’!²Td¡D °^Š/YÃ‚ä ‹wNÂä B ,Å"úVÓÈsQó‚,ˆ5,’-¹iXBD>­j¼è–œ×*/ƒg“†)ÑWl5,@.µd,z ð ‡K@ ð,iX -dÁRÑ°®hÉ‘¹Æ³A¶†ˆtÁ³VÃ‚ì ¢A ²ËÈ2£Ñ°¹$T­ñÈpž™5,È5,À2A¦ ¤Ty0¡a懓í a’$›HsTdñ4,ÈÙqº™5,Ȳ8ã4,€mr¾™4,(² âÓ° ˜¸p˜¬a`r¾™4,(YB•Y4eÝ“o4^„,®†%FÎ7³†3o"¢W·2ç›IÝ.î†dÒxÀ]D¦QÃ\ ßDd«åöÀâ3™9ãÌêŽôøVuËÀ{á­’ºg`¾œ#©›™eŽê®Ü*˜Ö¨Ûø‹xŒ˜°î>É-ªsHð™9ñ –!º9Ç,æ¯[÷Çþô‡Óo_à|ÏøßÓ×ßÔ²þc ÔÒ„ø -ËÀLw@ôm1|VÉ Ø3e¹ÓT¦P0Ø6ÙX(¸;þw>~ÙH -AYTÔF9÷ç0Ý´~<µuBŒ|À±œ¢>]ç®È A9/V—uªHö;u¤Éútw’uê(`>‘2#À|\¯Öyw½Ê~ÕëƒuÞ]V±Þж_UÈ{…ßwQÍ~û¸C¿@/Ì©g„ׄ}´Ò; íÇŒž^(„T “Ø…å\¨„ÈOVyGÍâ8ÎŽ(ã‹ã¸Î%t& ÿý§\T“b½û1.…øˆ”-BΟ~:ýö_¾ÿ÷ýý°ÀO_OÿùwÑÙï¾³ø¤¯ëgˆë'ürÃF ­Ì¥]#è!jß{ý<ëÑ—y.VfýÞW/ãÀç{›v[ŸLå{}Á²õzèŒÚè³C«#ä´¡õÕÚ¡Q×ÎqÖúq+o/ˆ1V9^xì¿û¯ó§ïOÿðikT5ǬFU°zLü3|¨Æ[¬ðÏa|¥Ü2>ž€·“Ú•º7’1vo`CÁ#haá€ÑF7.…'²¨àY—«tßÿã?ÿÓ¿ éyÖÞl˜=#YC¼Ï;0ÛaÛ¥†Y°”kYx:˜c7Õõú@oeÝF++¾®Ÿ£žÛöÍ|^j•gÖ>V-M¹*/oeòµöx™òµïcIë8ý½¬ZHÍQª™5ëbkõ ÓzqsáUÍÑÊÚ¹ðÂnŠëõ†Ÿ[›b´æ3­Ûí U-As¦Ù2¹uQÊ”A«çû|M~ºf¹­¦}¶–íDÏI0ßh9¶S5ï›l±Ê~K$Wä0ÍJRÓS³°ª#·‚‘"ßp‹†%JÛUÏ—i¡·¶ÙÀÍ4Ù§Õ^·Xž».Ëê DýºÕ1_]Œi»b¦åê{jßc—ã¦nÃA·hÃ}å•°··ik¾Ï“¯Û„ýQÊq³ßt«_rŒ2kÿŽéŸ×ë=½ÊH+Ý¿¬óæ[ùZ–Öò½ÍŠbìð:ÿem‹—Lm#¶¾z9Ôµe—GåóácÊõ”ÞDnß·oõ-]ëû2=ómþt&ôpWgqÕ-{¶w—ëÖÕ›ìÿuö—ž³_nÝı²Ç±ŠõáˆÇÝ_ ß$F÷hDø÷½\¥—é¿k;{í¾‹é­mjòÔþ¨ù1áSªO ->>wMÖöó[·ôèí^tǯÝ…ž€¬c¹åk¢ÓÇ{RÓâ±(|™YÇq‹_Ûº!#ÏÇ-k<ßú¨végÒtEG‡XåÐ?­mVù¬9ÄÝÆ‚óEM¦Ïÿ2mxDÒÝžm³Ñ´³6–mÜ¡Ç-ÜuM²S³áØå+±F-«Æ7uuˆk‚´evœ¶cì1i·¡:Þ¯MV×ô#lþ/½ÞŸ×çM½â2]÷¦n^ü³ý³ý³ý³ýuÅlþ³¹îî’Àçî+CßSþðJl0|ƒÜ/IÓÿ\ÍéßóŒø¯l}ÒˆõšÏ«þ µ«íÛˆ‡˜ìj¬Ô¯{ØcÉÆ{1fê¤ûÓ³ÜÒç¾ý{z}}\vSüÌÀÖn¨õ#bÞ®ÓýÆ7ú[¢¯·Ôøv›|vùÎÖ÷ƒu^uîw½Ã·Ð’C@H«n{×yOŶ0Û‹ Úu®vXâb -?¸ëpݾ÷cÊ -öô¶»ºÁ®Ú?÷Yæ?ª[êš¡ô ]msðš…h—\ê]DÊn±žRŒ÷nã=VçÃQHq±6eŸ6¢qŒö“xµcghIÅ”XvoŒÉ™¿š›to¹·tµ^¿+âÄ8äÍý^”ºT¢M ¿daœ¾Tš¿ -=?¾ û ý>ÄqŸŸ_”(üHò®ÓjGW–?BÔ–‹·ü`«)þN?XçƒÜ¤+-†tk¥™™]^¿]>,º¯FÃÂͬ;¬í²µFÚÝ«3/Â/;ž«ÀmÛ¼ýíüýÀÝJz–JÞ°åa§iÆuíH[>ä÷ŠÓRJÈÉî:¼eÇÉùn“¦6Ù™{·Ú¬óÑì˜Þ¾aÇ>¯Y†KÓ&¼×i]†œmÔlå²f . ^j¿—-M+Û3ÙÏxœÂí>wö\ïÏ¿´L++ô¼fMƒ.ä¯ÙTR[¹ÏB&×¾ëöDy„ˆE‰ÐÔ½ÐãÁ:Äž¦ØÑߌ@|ûýq÷=lõþO,Ã׊•eÚÁÀÞ^ÇlÛ.íËÖˆAFKŒÛo/[ûc3xmÄóªèÃ]A°4ÑÈó Ság’o[Ücu>œÅå›öŒ89c3rY8Õm¡ÛN>VçÃYÞíxnhwçG‘ÞỤ̾†•~Ù¶_ˈì~øÐÏǸë:ÙfØŽAëµÇróÓ¾öÁ:κnGw·,aÌÀ~ÿéÇ',`·þ÷Tƒð',à`ÉÝ"_”Œb#È(P}Ø}‡‹îÉö«O¾gôvöѤ¶¯uÍjE´gh©o(Ü{«áÁ:ÎjoGˆ{ú[{õà9kQô!£¾nµWr®}_{ë¦{V)ã=~ÝSžîYåcu>šU–Û1â~çmhRZ敬bM§}6ÇÝÁ0g~ïíïÎè>&½ÒöEÎTWOf˶•¸‹aéJûjYtk+bÏŽ1ȹ¾ÓÚ¬óá¬íN\¸öÇ;ÏfwžìÙ™°J¶±Ã›{–¸É -Òu9Õýó]œ§Å¥ÚÞÐ&îºò»q\ŽmÖSÖíe>·}/75Ù®íï?òvpñ^Ìù`g¹wbÎ/+~ŒнÝRHñžonK`àÁòoÄ¢¿´µ;Qo´¹Ï»w«û‘,©¼"Þ|°Î‡³ØÇâÍ·ZÔˆCUÜø^½ íýŽšA^±L?>ú~÷ƒu>šeZs;è¬)]7·h(é‡L\­0eºtH­:=ìèÝm·ëÚãrspÿÙûsü¡“’f.>¦*›Å©+Ãï¹crÇ3Zúyg»;cb{ç¾0'>xÑÁ²Jâ#…®Ý·ç÷:ùŽ­mÏQÖçw/ëàþ\~ñY í~×ï÷×ls5ÅþVrŸ¢?³¿¾mlæó‰e¥»öêxF±}ß¿wSûkïäÔÇÆÄ;£ýþ\«gó>GiYxÃa*>Åz<–­G$šàø ås¤…ol§ræ#¹²ñdûÔÙªl{¾R³ž<ýó(Õ£6|>%ŸM¶žT-.ZùŸë¤ÿt²…Ö}Ѻk*9»~1ÉßNüÌ$)d¢ù¶ö2ÛW3Ú";^þî´ €5-e_҇Ŵ¯ó q~paýêB ×½èD¦òõH£µc…ˆšÑÉòµ7ÕµëÌ y!TÔÐKŽ ¡1O -érRˆß™Þ4a2_Ìœ·s7$åJ/§.'µ\k-Œ™J£TžUK×üeš:¿°vv§($>M^¬ãû×-™_Þsm²°w ^Np1t ;…¤”+Lè¦'‰^´;ÈÙL{™Ô”»ª°4;ãÿW¨ƒÊÈS¿X&ÅGΫ٠¯ÍAÞ*IÒã@Ò4`)ͲB]6¥a³R_ˆ”ò º£ôÐ؆*T&è©tñ¤xlÉvXÝ 3„uý ¬”DÓ…@w“†)Û§œa,Õ`FgÑ -s£ÝIŽ;¿Ir3ÎçôAL?…Íwâ¥:6Ú“Ùôsïd’ù£3Aœ‚&yÂè·Jî«u Ÿmß¿ÎE¦yÜËé§ê朎Ÿ Þò±Æ2ò²çÿC™ïñûß':ÿ¾Ÿ£·úqgZ¸­²ôd|~~̇âwâX6‚èHÀþ$Û<ˆn Ã;'ðšfu3½—´AíàJb˜:d7L×Y/4n/;-Ž@æ­s3özéûàÐç»T¸ÃÁež> -endobj -770 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 483.873057 101.433071 465.123057 ] -/BS << -/W 0 ->> -/Dest (cb262-1) ->> -endobj -771 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 422.926036 101.433071 404.176036 ] -/BS << -/W 0 ->> -/Dest (cb263-1) ->> -endobj -772 0 obj -<< -/Filter /FlateDecode -/Length 3605 ->> -stream -xœå\Ë®·ÝÏWÌÚ€i>ªø/$A„8ˆ!^YHcËFp@Î"¿ŸS=M²º§æ¡+Û¸I4Í°ŠjòðT§û†£ÇëË€ÿ*Wk+™Ž§‡ï -{ü[jl -‹ú5Þ…ÁSqZ®|ü ‚T\iK>>ðß®¨«?¾ýâðÏm›pŒ®µú^T”\©¥x6T *’¡bv¥DΆ*‡KlÍRÁ .éRU`µÊ– -fPŒÕPU˜‘¨„t©j0#–@– -f„ÆÅP;BôÞÔÁO”¢¡ ì2&=[º\®-6SW  %Zº”\.8XºærÎ\-Án>X:†-ˆ¼¥ƒ-”j±t¶¤œ¼©ƒ-±–dé -l‰˜fè*lºª©ƒ->û`éZrÜ*‘©kÐùZê¥.zv\Sò–.ÇÀf2uØQX¤\ ]„-Ùs3u°…“–.ÁÂËÒ'8•ZÝew'Ÿ‚¥cØc!S[°øÄ [|É`9á%‹b)ŽZ$‹bMŽ*U‹bmŽJI ÄÆŽr+$ ‹Á"‚äa S¶˜ ØBع¦¶¤F¤[PÁâ‚”`K¤dqAÛÐbqAÝ’oÁ₾%²Å „ 3½Å Œ›j&‹ (Pj$pnwZ\@ºØîÅâ‚ÖMœ£Å ´›¨f‹ x7Qðx7%´4u°¶¸ìèR¨Éâï¦à‹ÅÞÅ6Š`Up˜-.V\¬Õ[\@àÝX=Y\@àÝXàL]q؃ÉâïFÆ%Xž¼ÙG‹ ¼)f‹ þñf4Þ…°À¬Åð÷ÐÅdq:sa‰Å Þ îÏØÓ`+jË0x7T,Ÿ©k.̸±o1Ë.äÜ,.`ð.U²¸€Á» Ïjq8ÞÁ£F‹ ¼<µ¥ï†Ø¼Å ÞEÜÀ. °¸þÍ蚺æ|«Õâï‚–¢ÅðÂÎW0‰©+@ -€ŒGjqAï½7‹ 2x×sJdð®'„=¦¶¤-.@\¶­±§¬:Á–¼ëVØÒw½/Íâ‚ Þ·iqA."ˆÕâèG‹ r%×JÉäæ¡CÈgê²C™-.(e™:ØB…,.@`íZjÕâ‚aKŠÑâ‚a B‹ ¸}›él\Ìü€` "E‹ Àd®6"‹ -gäc˜UK—£«tmê*t¡X\P -9¸â`qr)W±é-.Á; —,.€Ð²¸ 4Ø‚­bq\‘«‘¹¥ °A¶Åˆt¡ Áâ‚a ¢A‹ j¬È2³·¸¹$¦ÚÒ‘—<³Z\P); -.À6A¦ ¦4u°4aqÖG’m¶¸É’M¤9¦¶$&‹ j’nV‹ j…-ÑG‹ €MÉ7‹ÅÍÃD|4Ÿ„ɀơóÞâ‚*òÍH`_J¾Y-.hÉ#ßDŠgìÛ–²ä›Åâ‚F°…¢˜:Ø‚ ©ûœ„|ìdé2l‰,.h9K¾Y-.@˜|½y$P%ß,æqx—˜ÉÒw™f‹ P@¾‰ÈÖÊí=ˆ— â3[Y%ã¬æ‰õ2|«ydàÁ½ðVÅ<3ð _É‘ÌC˜‰Ì²fóÔnÊàÍcþE}ÒGTp(ð™µÈE6l Bt?rŽY-]©v>ûé‡ÃWoá|?üëðþ‹å¤¬¿„¥6!¾Â6ˆ Ë͈¾.?ÁŽ‹i°ì“*Ë°¥M³ˆp=f³àðäßñòËÆV˜úz7qüöy\l1 ºÉºV$¬Ëi 2õ›üŒË ð¯uR‹ÂG"IO›Ê¿}s4W+Χž£¨Ì‡‡DúK‘o>¾úÓ«¿þù›×Λ÷‡¿ýÆ{ö_ðI§ó'õ}ÕqÝ–QÅ3¯íVåõóýúI»~׶Èb/d£î[Õ•×>C˜u¸]ö}1&ïäuŽ /ëü¯x¿?_/ǵ^Ù]Ó¯aCšsü×?¾yuøÝ›-—8t ¤ ®Èç;˜}¬Éç`ö—€*’ÚP¥Ó¾@r]*~«–JMo_fYò†º¿µŸÞG_®íò@þNÁc߶Ãuø»µ>oí]¶W™}Ž>Êös¿½ÆµåŒï·gx -¤6Ÿô¼äœ½BبÝÁ׃m^ÀëÜDØ;…’¼]UÖà·«³AWS«Æ»Õ<íV[“Â=2Š;$­+ƒÒï'iÓ¢Ö»a٠ݶ·«=qKtýD>Hí´ciÓÇ#ÕßUï.ç&Ä+¨LƒÂvó5JÐw•µyq¨¤›¼7fô{…8^WÂp ŠŠZ­[hع§Ñ.|>Š.Pù~ÛFê¿k·³gÙY»0ßmù~ìˆïv;ª®nö!;® p„ô×#œÄÜFácm^ -oŠ·Vé×å‹ŸaGÞçì£ß‡<ð ×ÒrHÎÒ›({¬Í‹CÙÏZ™²•í=óÞSî½îsc½[ÞXóÈEÈ_íq;ò¶Í¸¶²m3ÆÑéX¼í1ãŒÉ¨TgXò½<áÁ6/ Eñv·GJÔ«¥"ñ‘ؾ[“A‹WÂϵjF«yNgšWßÎz!¯<'°œßCçW½x½Óñ‹¬œeKyõ¸Kûwªm½ÎMQÅTž˜<ØæÅ¡êvÖWm™EZ?³Êçz.ÖÎ+ ±GosÁ-iÇs*'¼ˆwšHg;Ôýî‘*ûïòÒWШb«ÆŽ—ÄókóâÐxçàî@P/_E+oÇä}Žj R¯þh¯8NŸàXpãö^ug³ ›w§'£Í5¿÷ػӘý©Ñûtƒ/ÕÑ›üÈkˆ|¡µyq½Ëí:V¥(ïfägWϾ⫇»–‹¾Û¡Aô‹<ÏÕ¼ˆ)wGÞ¤²‡O±oðÞ'¶ÓQÇþºÉ(Ò3ÎôlóÒP˜îÄ‚»(ÿâ,n—X§ ÊøÑbÕû|ÑÊž“•|bö³9•¾‚@}î9ìU»4ÊnZíLe~ï ûŠ}=W‘ªbE¹q!B¾_>ØæÅ!õN|y©oú?t -smg²ºÆÅŽ·3’xÖ©L¶çó!}F¼ù`›‡Î;ñæÕ}}ŽÕÓH´NPtlxúGøÎØ°—cn|ïgÇš¼8üÝ9üŒ_1&ÆÆ¿^ïb) ƒ˜å.ÂÊ¡ª›bÆÌÞ­rg&/o¡é·¥嵟ª¼ÎQ>¢Ky|-Õ#BjEîø’©yõû?þá/jj q™±žáéTÇŒ7¶TÊçòTÒyJB[ë°LMß -k [ªç©“z²-åSú’úôÞã¥(uóù½ôµ~_ú©}K¬ãÉa®´—ã©rÖoúçuleÏÒ«kà~ Kùñ—»Ãr»ÏÕ…(¸39‰ZJ;Ê“Õ' -}éÂ2Ùáx¥åò`àÇQk}VÐËãrëèùABUXë\ýÃ!4r2ÿôós=~×Ø SütçºÈbñÅ"\¿úÑ…^qÈN ‹âÚ¾fbçׯmtž¸ܯFÚso&÷Û‘ï•ÒrÇÙy`Cˆ–9\Šåê½ «Xž«.­¥óÓ” õTÇIJeŽVi‘‡}Õñ©ÝŠ³¬@_k_nâ…ðt€‹¡ ñÓ‹Q›'WxîÐÓ¤úâê'^¦T~ø9ÃÖì2¹+Tþ?S©0Ïò%Ѐ”ÜÙ=KsÙ›C¼$-σuÈš@eÖUÓJ˜Õóª7ê éŽÚcÆ6R5eJ^Z7OHjÇVê†X(¬ÏÏàJ-ôÝ 7Å< ü´O;ylU€½–ƒ‚;~§8î|Òâ2V<ÈmúÃÞѯÂ|žÇF{±@¿öA¦8úáíæ`J8 )Lñ¤Ñ§EÜwë`§å—äõëÜd–Ç=~<,n.&Ž _r -òÔ™Ž¼Âñߨó -ïèøM¿ÍùìÇ£l·nÄÚ“É3ÊõA±<³Ü…cÛ(a$EûSê¬Ë ÃǨøšfs?‰½× ƒAÃÐj!Ï9Tâ8 CR3Ã7„§Ý,Ž@æs×f:ìs1õ‹Ã0InïV¡ /Å:uiºä£ê%©:]:´UCt§(ªÛ´lc¦R2¦^ì'½IÆRFH1MM7«»î²å*T„ÀItÇô mcaô³·×g¯ûjrî;@¤ƒÚ‡ôIKÇ…‰tž-MK1o‹Ã礣úæ34úJjï¥äÌ­1C Ú4F•´ÎfkÖƒR—míŽbÐd¾ËF¥Û”×f}Z–+®æ,•SLéz¦Y¯Ê˜DÖ«’/êJ¿Ýãh)ˆ5œË¼:?kŽX²²¬[)Íš©Ë.ǦíØÝÛÕeÐ4$ÏÈÔÏf|QóYü÷œ½sn³;‰°jõ×r:ê|0ŽšBI1-ëÅלÁ!$ëªê¸ä“ÛÜ>?¹<%éç'z$,‡£”| òW{lž h÷ª«þ}ìfÍ›—ôipõ «K…œ-¶a9Ž[‰F±Hª(¥%IÐ[åáòâƒy\4žØž†-¯ÿ-·l -endstream -endobj -773 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 772 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -774 0 obj -<< -/Filter /FlateDecode -/Length 4008 ->> -stream -xœí\K¹ ¾÷¯èóÖêA½€ÅH‚ ì!ÈaܶwŒxsÈßÏGuIbU±{º=cä°3ƒžé"YE‘)ÕÃ-~ß8ü)äL)5':ž>¾¬ÉÑâ§I¬ûg|r<:KÙDWS‰ÇÏLÙä}NÇÇþl¥øãá—ïÿ^ŸãŽÞÔš¼‹Çß?2‹‚É%gV˺ °b49û˜VrC¬UcA˜ÉÓž•¡Õ5Ô ï‹Â*P#PvaϪPÃgG j¸³Ârz8o­Êƒ"–(x…ç¢I0zÒxÞ™Tª¯*/ƒç²×x!˜”ÉE§ñªI)Å¢ñºÄjÆ‹Ð%:"«ñ  …’5^‚.!«ò ‹/9h¼ ]<¬S^.ð®¢ò ‹MÖi¼L¬…HåUðlÉeÏó6šXB°Ï9á›Aå!¢0I)+<]’UåA—¬×xº~5p"†\Š­ž£;Øà4^„.ÞgRyГ¯!€OÐÅæ¤A"È0.iàs6T=i àK0T¨h(àK5”sÐ`À×h(Õ¬á@°<ï4 ºDJ]‘«ò K¨¤aAðЄ]< Жà-À-Ùê4,À[².iX¸PÓjX€¸¡$Ò° ráJUÃ‚Ì ÀN @áž5,@ݓװ v•¤aAîrVÃ3UtkXt4Á• awƒ³YÃî"Œ¼†˜U?LÀWŒ/ÅjX@À]_,iX@À]Ÿ‘T^6ˆÁ aw}Ä´Ì ÜõÑz ¸ëÉ' à(øcÕj¸ ‰z9]° ߃烆hÌx”%D஫HJL­Œ+5iX»®`úT^5.ÃâJÜÂÊÆ¥T5,ˆÀ]$ª aAî>‹†ÀxƒŒê5,ˆÀ]©5p×ùj5,ˆÀ]Ô Qøƒq(4,@~3ÎÂuU^5¶–¢aAî–¼†ÈÂÆ ‰ÊËŽä4,‹:TÂÜEz¯$à®!hX€»–Pö¨<èŠ×°u x[%¦Q¬ë¸kfXãw­ÍUÂÜå´©aAÊLðEÃ(hP{ R!SsN¤jÁCɧò’ÁqÔ° [" KåAʤa -kSC-d]‚÷d]P2hX€ŠÀT‡ê[]@¸‹º> è‚JQà ™)•HÂÖc°ªÆKÞ” ¸Vy<—5,È™ R±Ó°k)Sôà —K@ ðiX+tA¨hX€TdŠGe®ñtA‘­a*]ðœÓ° xè‚jPÂâ V™ÉjX€ µ$L­ñÈò:³hXP(.5,@˜`¥ ¤TyÐ0¡aæ‡ÛQÃ,~°ØÄ2GåA—IÂR}×vÊú/ûFä Lc¶}Ål>À X¢®M‚ürAeº!Àt—¸¦-±íZ4âé ÈX–{ì㚎Õ`itÑêÚ.-tÔ“‰ ?°XÀŠtCŽqÊŽî$‘f»“”TÔ9èÙÕ´ŽfßÓkäþ_"—Kuþ9î¿((»²ÎÃg= /ÙÊnÂö¯Á÷|°àKü“û/wŸTXŒXbåÙ¯Á÷|¯Áwgð©Av_Éy! _ï5ðþ`w{ɹ_ïÿôîÍÚM5ø_É•v9€¯ÒgÞÒ<¾û|øþíŸÿú—¿ÿŒõã»O‡ü`­+ø8|ˆ·åù‚¶µž?8öéL#|¡¼È€N8/B.V|?áÓh?þóøîí=øjúЃ»æ.ÚiøDO9wÑMLç¦\ÅdûÞÌC¬‰M Ä7 -xŸøz­¥·ìÊoÜs`r€~áE·üOËÿNçÁ±,kBïÏôf„¬ÐÙ6’O¢ÞnžÆ‚­6$ß=À†ëz´vÙ ~i÷“hû4Ûkr~=†Öïiámúç6úÄŇ…&t?÷kÓÖÚÝb‘fïMã¸îݾçšnmùfŒÄžmÒ¢FZõ0¤Âð£ÐûOgjÖ@Ÿîþx‹›o£Ø·s¶Ð¢[lú]Ñ#x{I;ÇÝBcñœföqE7¾&Yò©*m7Ï=-mŠ8ií×9û݆͎+¯\ì#¼ñìmÓÓ(vœº·ïçs»çó93 -´Ø÷¥{¾ôú©‡»nwÚÚ†aªëÑtø8£†m³ÕUÚjí‹ÞÒg·Ð÷T›M¶l¢®Ë„7c>Gõ™~“î3[y?†³þWm™¶¶TõzXt_ôhíÖEŸ‡9ÇMv[“ï©¡ûW—MgDn©äAŒMÎE·GZÆsZü³ëõIè±`M÷¹FßøòÖ6]~ëÓ´d‰æÇ^ŽïªËÎŽ7Øaè¥Ø|ëw+qÓñ¦éZ…ýîˆÍ{pàyp͆ðƆ·âÓ܇¥ÿKLôÜÓuèþ²èÓÏíú½´ÂÇå˜D>°·gYâk¿O×4qÖ#³Šš¢}ïÖû°ö®^y#r]3ÌYDzžHÂûòú¼—ÉòÄô dù};÷eùèè’ÏÎòû¶_¯¡Â@à,ô“™)^ÈzÃ^yñ™÷—üh‹ªWm¶ãÙwã_=;¶ñ~XGvïo…÷ ’»=*#V™îé¨L÷Gä¨Ì{žè–î5ÊÈaûê~Ž^ ²’Q*#³·íoXÁ\l÷vÖ­ _1ßúˆ±oç>ÄHíîÓoƒû¶Ÿ…-:ôˆÞÔ!suiWõe‹Ê*ltÅבzÕ†e;Î{róèg‹bä´ú¶õ&Òôz¼ouÜ‚H«zž&_Î÷¹¾2?;ñ4õšo\HœÞ¼ò¾‰0/_#äÄû¯Ïø};÷E|±á’ÏŽø}Ûß¼F¸2§OçûµtWî¾jc¿µÃ·ÊñŽQŸŽ°2½ù%vǺ¤.ô2s쨀DÎßo½îghìnõÛDãn¤Ëù)¯bØâ5çÝÇ;,\¶¾´gÙw /¼°ŠÄ¡cÂO›Ï]Åê½±/€ûvºï7}Zl\Õ#–Kz<7öm÷ -ïbls¿ïE%Ð=ë$=ðï³›}«ìêž$³eÓw³š?c±]­ÜŸ°qÞÚûï™™w úNOßmXá#-ý‹Ýì1Ž*„›ñrƒe·{‘E!œw‡Õ9Øk‚‰Ís÷øª ëΆ -öv¿6xXæ?êþ»­»O®Ö—;TÜ"â×ûãnç¯ïàˆÔ]ÜE¯!_®ÙÑ!kl ÙÕ Y(zùjÛõKhai³9yé ± V*.®ÜýéÝr'i{ÄÞcäÇ6°H/â.äå&ÕD®ßǪܳ¼ô¾¿Ãõ§wÒ¨h’߀âÉPÍ|‹½vq³]ÌüÈš÷Ê&r¬1‡#«/¾èã4Y÷Ç‘gÜh»Žv‰©¾oñélZ¦û‘ֱ߱]ú¡˜8#}n´ßcHè#ñ²!ö1ì¦òË!µÇE\{dЦbœçw -ö’\üˆ -jMr}ê\3¶;^8³½‰áËZ^Î`ùyM~Vçüæq°Èi“þe™Ô ƒö µmäâûÁ$?øAzRÈD³ºx‚ÌÄå«m‘ÅJ§.ðûdˆÆ._gEbÿêc Ÿûiü€Ù.Ú-þçŽ"ÎLnOæÑ[ã2¿Œ ˜°¥¡JL]ggƒ$egƒóÄ'É.XAŒ{¢ÝM²§®ã‡×†éÒ$ե霘þ.볘ÎkÄÓ݆-õQ¥ò#н©n]o-¥ê Æ.9&„Æ<)$¾£cG|c{Ó„É|1kü Ù™Xº#Ñ«]N!ªÜÒZ3•‡T™§Önù©ÊtõP†áÚITý™ß®"bËWßçA’¹Xsm‹ð¿#žH1´#·»xöÒqb…Ýõ$1ˆv¹Øé/“Šâq1B³Óø1þ{†Š8¨Œ<í‹£áRü(Ý<šplòÚH’žêfGyÊ -s¹œ‡ÏJ{¹byÝ!=,¶¢ -“ z®]=©k²^7È aÝ>+%Ñv%ÐÝ$ÇáÊ+òÔ3ŽP…³÷Î’îG»“œv>Jr3îø¹È¡ïhWø|'žZb£-™]¿ôN&ÙÛ‘ífg‚8(Lò„ÑÇFîÑ:ЉßõÒ¿Î Ó2îéðÛ¡¥9¢¯|Ÿgpü˜¿¬¼Üñ¿y‹Ï¿tü[¿ïóœÇ½hᶮÈ2“ñKaÊd~IL'Ž°DOö'Ù•)»7Œ<î½Àkš§Û ì]Ò uƒ+‰qÚPýp]ï‚°L¼B¨’:ÆÔx:5´Ã´>9'ù ï´Ðh+ˆØ £È™¡1K ZHÔ2O[¢ì”:mÕiO.@³6ZÌ]§´œÖÍÒF\T+åQSLêoÒÒQÎÊ0b”³’v²ÜnÏx£Z -£ŠÒó’£–, --ɳ•¦dè´}ß´î»gÛ£MEÒ¬Lí<-î$¿ -ÿ¾&vÔ»©5©þÛvG\™Ob#Q“ËÁ‡ör=[R†¿%SÛ%wŸóµÏËžßW‚ÍŽ_“¨«ðF}š!šê¯PÔO«VÒ}¸øH»zï»kïÉj7Í·M¢q˜y©È¯Å \ô~›O¶î…§óÍ.¬¶PÚïÿºy -endstream -endobj -775 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 774 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -776 0 obj -<< -/Filter /FlateDecode -/Length 5464 ->> -stream -xœå][ì¸q~ï_ÑÏV楊`±# ÛÈɃ‘‡9íÁ¬uò÷ó[¢(©Ôb÷ÌwÏΈd‘%²øÕ…Ù³Á¿ï,þ“È)åè|ùùôËÉ ‘ þWJ,JöOø‰|¶†âÀ6‡ÄçŸ%ÁÇ!fv1œßOøÏê±-þ~úßœþ¶¤±g7äœåóß_%‹üSŒ†•¬Œ,c½’Å<Äè8(YÁxÅœµ,°Á‘m³"Ø œXËä\R²Øð­ßfe°á¢%- lØÌQɲ|XgŒšF ‘wJžå! Óƒ–çìRvY͋ȳÑiyÞ!’e«åå!„ÀIË#ðÂÙX-Á ["£åò)jy¼øàš^\Š^Ë‹àÅ¡w²’—À ¤+©yàÅcµ¼ìΉHÍËÈ3)¦mž3ƒ×m·ÿÓ—ðåÛ ç¦¡ ü8sþòóé·?þË¿þîß~B_ÞNúBOÆp2ððsÁßQ¬$üjƒðÃ<¦ûkžýóÿyþòcÛ–3â.mxÙ–Toüµø?AW•˜Ž˜³D?®]å¬Ð¤b,Àe1%æZ{÷n’‡:wìS8™°Zðbðì <¦.ìsC›Žæ©ÕkÔG {dwÙí¿ÿñßÿø‡¦Û¥ké+~Òø÷+~P AM°»ÝÃ’®é4 ÕÛu($ŸÃ5ÂvýûkÓ߯ES¨p1aà)u°÷iÚ¶¿ï&ù6ýMS«×þ®Öse×Æ[nÄ ’Ž’Nå¾³2—ëo¶«tÓÃø¼×©°¯&76Gy³è:qÙ -߈Ë~ L[Ï·ë"ùÈúŒážù†?Z†vÔ†»]‡2â÷¸v+g‰jQ±*•8vH/ÄØÈb]lè­™yqiGœbq{Å;§›E?Kœü—‚Ϧ,o˜ 3戳Nšç(ب ælœØ_\°Ì£€½-ÜJðxõÛ-Ÿ ê5—n‹º`È&VlvŠ¦“Å""J¬Ðó«.Žeµz'ñ£ßGóV3¿b×¢Yx‚ªÒðã i¿»Ñ`ÙÈŸZa© -fŒƒa„‹³É!+uöˆŒ)‚ËïTzž¬Œ¥`¯‘ÔðŽ¸@F”0|xnÙ{‰Ü–Šç•yöV5Q f§[¼–HÃ!¨ß.úY î ÞGmY›Í]82:ižÔe5‘9„äu+!.Á¸ü]ó{ÙH¯µ ±+kR¶ý˯¨p^Ž”MƸÂÀp~ïÍŠ #KÐò£‹½÷†É~;ûÛE?M왇œÄzá# PQÎy@4O(ö*³‘vœÏ•h¤Y̦)1‰U…ÿQ‹£+1‚¯GbäJPr¹b§pð:·Z½®•RäVàÚJ%$ÅÁ+mM"K/+w ÉÛ¸ Óäɳ—H«2UQ†ß7zujgjW܂⬔îô[òÊ䟀À¬øœÚô;<¥ýv'>g°ÚLâ$kže틽H(e‰qÞž3}4O£Ö}†j›õ©ò禓Ý< w"ü¢‘¢yÞ¦M<f!kÔ=­g‚ù- Á -}y+:šÀ³µ¸ÃÁ$H¼¤ý:)”Ý°”:_®q¬–‰+!6Kaö¯%¢3Oâ·ý ½Ñš+7¬ÄÐÖ,®&Öêy_ê¸1a> „rS×ÄSc}óŠïʧìýº(ï¬N`2aðIbøgò$ëEâ;ÜžÀ4O3É™!ë„IÖƒQQÜ-÷â—8úZÅ= U ój~ $7eªÁ6 žoè×}2þ^ØÚ|Ü6/;YM€ÂgÙRYè¾-l}4Ï#l‹0Ù=«Æ þ?ÎÊgªB”á¹tø£-D› _–ÝÚ©á~ŠlSRèõ+Ë>)NLuGëlyˆAú.ýÊ+ëá_„=¾ÿ6:©@¤>¥9 ϲ/{ǬñÈì¤yž)-û¼K̺þ8°¬ï3—Ýkn©ï3•Ö_Æ7ê6wxMÁ_ÍËæ}IX´ÂÂ_W‘}8€&Û:£Z*›woCÉó C„|ðAW«“K¥wOŸ.$g—1ný¬eŒ`PsÊð>úìNšç[ÆN&|–óš{C·OtïïYV|ÿþž-}±sŽ éÆ–Úᡱô:ãÌÎع1ÀVêìZª™Mòm Ÿ½T3¯.Þè[K5Öì.£”<׸#ݲñ–×±•£%’ö}×Ë/;ñÂÃÊ2®ŠrÕnÞËY~r²JvÀzuÌY¶é®ü|ÞþŒì|Ãnÿ¾“æið?ÔU,úomø¬`ÊdïtM¶k·üßì‚å“MèVÓ}$Ï3LQv AÇúv†é]”Îü}Û"è¤yžNð„ÄX};Ë瘔ËFQ´sÄi[WB›ý–-=¯T¯ y±`Ö½«r‡Ï”ñjPêìQÆrq‚SI‰ÏUÆrLù˜Ó­ØSÆê4„*‚kfD¾Û@í¤yši¨@P§›Ûk6š~=@;¶6K•Ê±…ª®Ú4 ˜yûäúZ«¨o*l‚ç‰ZvXöÙÌ$ÛDsòÙjõܲÖ߻ѬÍlÛ<^[’{¥¼Í‰÷8> gH§chC…¾omI&¶u&¨}ÿØÞXÆ\c£ŽÃÝ{7tC'z 6 »3É™Ë$W¥ÁGÍóÀG¹ÞÀ±Ñµx)ÿI[Z궖µ–äyŠìѯ·Ù”á}Õá¢ÝÖò-®Ë6˜}Ì`Œ²—}›Áˆ£t D}4Ï#DQÎÙ é}kÊÎáÅšò' Ä7ÝëÔ½%Ê…H.yØPNî9p&nöí¤y‹ÕP§ï{úGÌ|÷²5:êìõÍ îœÜ‚ÁY®ˆêßqÚIó4—œ{1rËÞÿ…™[ۻܰ`§xWcq}³­ÔJôì9’ÛŸÜ|³†^Ël¶«¶ôÇv¡¬µÂxsuOë€(—aÞ8…¾Ë.ll¸° çX¤RçËh¼éö^ò,‡Žcôp|I®(·ÀÜžð}4Ï3áYÎuy¹/õ&R¸krܦUŸ·P¹L†7éöÕ3Òy'ßáF¿Ù\vÅ­ˆg -rƒF ÎìŒå£Sj}$Ï#QÕ‰õ+ó–ôERS -ƒÙÊv™Þ]ã4OÓùr•Ë™³vr|lpšèæ’ -º¹X•è^WØÖ{ßÑŒ-½Š3Ý›¶ãgŽ¤*ý´‡KñNè'‚È@ý=ÄS³ukKoߦŸþm[ -ñŠØè -/Þôár ÉNoy#ØÓr è‚Õ) €F­Ç;·“5Á:EŠÛÇÔlhÔšâzßµ¦pCge C +÷æuŸ1í¤yàtr&Öúø€Öú<à\0qsYhíAðJק¨;<î#°^â¸1™¨qüÉK9-lÚê[ÊéšÔÛºãCê¡”:¿Ýç7tžTMì}¿ïXi|¶}>;VAÆ·\AoŽø:<Ö+×CfȤ&PÔîp§n'ÍóÀ”\ÀçÈ„]Ÿr¬wÙȽ§9¶Ô÷æØÒß{¬w‡ƒûC­í¸­óeåî¬_üªÇz?¨öx“‡z8NŸlA\Ñ(w`woKé¤yžÉCÙ`vv:óþ­Á‡¸3 ëvë5ê‰î¢Ôù¶,[ñ¢f0Z³ñª½nCùÎɬ-ý!d´VÄC­7îƶõ¾`Í)‘¾ï>FbÇùä…ÊŠÜ(Pç -©-;W8&[ošDÞ&š¦¹9ÙÑÄ•/¤Ô® sbe½í:Û ÿTÖÅf8o%^NhÖ¯SßÕTùÎÖTÕÔ»ÎÔ4ß0•k*O%ë€P'%érR¿3SÕ0ºlÍoFM¾VrML“ Q-—§rJÒ外kã:R±–J3ižz~feuŸjÇ+i¥ÆÔ(ŸðlæˆÜ>C›,×ôÖ±6©‘·I¼œ bh“ü~⨔æ+ O¢×&ú¦ÞšœÌ,/sª¸–׿05§4ùÖƒü÷ -\SyÊ–ªHɪûü47"s³&/;©MuHëKq.Ût—±ÊlÛ_6¥¼€n-]{l‘ÚtY“.7lתCe²­RW“¦þ©XÙ&š‰ 47'så÷EòÌ'ש*‹ècb°¸s­wN;ßÛäXGÜÊÇw*¿µÞFæ§ÄKQl´NÑOS#s²¸“øÔƚę1€Âœ<Ãè{IžfkE'¹€vúsždšÆ½œþz*j¶¸.x+ß’k-/{þ”ù?ÿu¢ó¦—\õ¸3-\ÅÖEr«ÉäË£©3YîÉžë´i5°?'Û4—åIªw®ÁkšÉÍ ìSI[ÔÖÜ6‘ç>l’]]ùÎÅÜ3|#ñ²êÅjÈ|tlf…}}ôÓËÉR¸|öfâ -íëx¹mºæùY%Ÿ›*ñ$Eg•Ž\r !ÚJñØ4˜gÎlŠ@¶Ô®þ© -Ù¹Ñ>4f„<ú9f5ÛV=¥•·hŠÀöíC[1ٶѼàGaúáéõáqYÓ Ô -í5õ½M­/&©ðôT_Ãò±ê_vZL“wî¡Z—o枯FÎ<5fÓ‚­Œ6©i& pÛ(Mi‹F'EQáifJ«5Ɖ§0’MÝRÞ8©½«M1§þµíinG¥v"·£6e¥ÞIãUkÉWˆ[qŽóÛ™¹dµ%“’ZêšJsI?¥mÛ¦eÛ“¶=7¯A3#a¶LÍLÆ›’áß#sGýV”VjúW¢#6ÍŸû‚¢–Kœ/_p7)`ÉG8#¥&\r7ÍíøÉ6J2ÅOÚ–0yo¢…¼í°ðŽŒ|œÑTõ>~)M'ËF}¥û:@ Í0·ý²Wù3þ7‰êcWQ¾½.1¾”䓱ÑX5\TÂ@A~/B(åßÿs0ƒ -endstream -endobj -777 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 776 0 R -/Resources 4 0 R -/Annots [ 778 0 R 779 0 R 780 0 R 781 0 R 782 0 R 783 0 R 784 0 R 785 0 R 786 0 R 787 0 R 788 0 R 789 0 R 790 0 R 791 0 R 792 0 R 793 0 R 794 0 R 795 0 R 796 0 R 797 0 R 798 0 R 799 0 R 800 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -778 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 766.923057 101.433071 748.173057 ] -/BS << -/W 0 ->> -/Dest (cb266-1) ->> -endobj -779 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 747.976036 99.933071 729.226036 ] -/BS << -/W 0 ->> -/Dest (cb266-2) ->> -endobj -780 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 729.029014 99.933071 710.279014 ] -/BS << -/W 0 ->> -/Dest (cb266-3) ->> -endobj -781 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 710.081993 99.933071 691.331993 ] -/BS << -/W 0 ->> -/Dest (cb266-4) ->> -endobj -782 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 691.134971 99.933071 672.384971 ] -/BS << -/W 0 ->> -/Dest (cb266-5) ->> -endobj -783 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 672.187950 99.933071 653.437950 ] -/BS << -/W 0 ->> -/Dest (cb266-6) ->> -endobj -784 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 653.240928 99.933071 634.490928 ] -/BS << -/W 0 ->> -/Dest (cb266-7) ->> -endobj -785 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.293907 99.933071 615.543907 ] -/BS << -/W 0 ->> -/Dest (cb266-8) ->> -endobj -786 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 615.346885 99.933071 596.596885 ] -/BS << -/W 0 ->> -/Dest (cb266-9) ->> -endobj -787 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 596.399864 99.933071 577.649864 ] -/BS << -/W 0 ->> -/Dest (cb266-10) ->> -endobj -788 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 577.452842 99.933071 558.702842 ] -/BS << -/W 0 ->> -/Dest (cb266-11) ->> -endobj -789 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 482.717735 99.933071 463.967735 ] -/BS << -/W 0 ->> -/Dest (cb266-12) ->> -endobj -790 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 463.770714 99.933071 445.020714 ] -/BS << -/W 0 ->> -/Dest (cb266-13) ->> -endobj -791 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 444.823692 99.933071 426.073692 ] -/BS << -/W 0 ->> -/Dest (cb266-14) ->> -endobj -792 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 425.876671 99.933071 407.126671 ] -/BS << -/W 0 ->> -/Dest (cb266-15) ->> -endobj -793 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 406.929649 99.933071 388.179649 ] -/BS << -/W 0 ->> -/Dest (cb266-16) ->> -endobj -794 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 387.982628 99.933071 369.232628 ] -/BS << -/W 0 ->> -/Dest (cb266-17) ->> -endobj -795 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 350.088585 99.933071 331.338585 ] -/BS << -/W 0 ->> -/Dest (cb266-18) ->> -endobj -796 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 331.141563 99.933071 312.391563 ] -/BS << -/W 0 ->> -/Dest (cb266-19) ->> -endobj -797 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 198.512413 99.933071 179.762413 ] -/BS << -/W 0 ->> -/Dest (cb266-20) ->> -endobj -798 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 179.565391 99.933071 160.815391 ] -/BS << -/W 0 ->> -/Dest (cb266-21) ->> -endobj -799 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 160.618370 99.933071 141.868370 ] -/BS << -/W 0 ->> -/Dest (cb266-22) ->> -endobj -800 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 103.777305 99.933071 85.027305 ] -/BS << -/W 0 ->> -/Dest (cb266-23) ->> -endobj -801 0 obj -<< -/Filter /FlateDecode -/Length 3653 ->> -stream -xœÍ\訃 Ýë+´`zXE²@0‹I#d#YYØŠ=ƒà:€'‹ü~[]–x%]ÛƒØöU“õ8Íb’Õ-‡#áçÛ€Š„¥”š“O´d%üY[ì.Võø›õHò¢¡¦¢Ç&à¼äª1§ãÓÿ\\ÎÍŸûæðï}ŸpŒK­)=þòÎTÂK.9“:ª -vTªKÎQ“£JaÁ-Öê©C³D¹VeÀZÔS†ÄXU –øZU#æ ž -0BÕì¨GˆD®@H„££ º$=yº–Tj¬®.CrôtÌKÊ4xºº¤”´x:­<‹òtÀ"\²§KÀ‰ÉÕK,™=]–ëTGW€ÞU\°P¢àé*/Z‹ˆ««ÐQÉåZI-ÌäéBX¾É®; -‹”²£‹À’H««eŠžŽEðãéÀʹg·FÛÝL<KŒY\°`ñ=ˆ X('°ƒã%b΋Ô( Ä‹) ÄRÉ™=ˆUI5{<À ‹Á#&`QIpÁÎuuÀÂU<.à,hàq3°Da l+ð t+TƒÇ ¾ -Éãá&y\À`\.I<.`P.\©z\Àà\wz\À ]l÷ìqƒuYSô¸€A»,%y\Àà]–@x—=]°À=.;. -{\ à]”=.ð.¶Qô¸«ºÀ“Çð•%–Bx7 ¼3B€«Ë ö {\ àݨ¸/rƒw£Rô¸@À»Qbò¸Ž²€ÈÍÀ» õÓ`Áz\€x]d 0Ø‘–x\ àÝPþœ= ¶ZB©Éãåsuu wö-¬¼„”ªÇ -ÞE b ¼ ú,€ãDÔèq‚w#R{:ðnˆ•<.Pð.òõ¸î°$ ¾-ຮ®.TKñ¸@Á» ¥èq¢ðBLâêòG -€ŒBêqAï"¼W x—”Ùã‚Þ%AÚãꀅKô¸y tضΞF²ºPSx:ð.¬°§ïåêqAïZØô¸ eÄâq.È‚£Ç©ÈRsN¤JÐ!åsuiÁµz\ e¹:`‘, ±^*×âqAŽÀÂ1z\#° eð¸ÁR²o·<¸‹[° Sô¸L¶”*âqAÖ„z Võt).%ƒ®]].d r–¡8x\€Zj)Øô€àK—<. è‚x\+°`«x\€P´”ˆÌÜÓ`A’íq2]èBð¸ D`A6èqA‰Uf" ¡QKÂÔžNÈêÌâqA‘´Xªèq¶ *M0¥«ЄÇX+¶Õã?(6Qæ¸:`a J‰Vn J–HÑãø¦Õ›Ùã‚JÀ‚ŒÏã‚Ji±4ÙãÐ8tDÔPPoFñ¸ûÒêÍâqAeB½‰ÏÙ·•“Õ›Ùã‚*À¢HQ\° iªÎ¾'¡Þ;yº,±ŠÇ5%«7‹ÇH3Po"£wŠÕ›Ù=.ïj3y:ð.2Óäq.Po"³õj{ñjF~æ+‹UœÅ=1 P¯"¶ºGîE´Êî™|­Fr •eIî©Â*”Ücÿ"“+N8Ÿ>ÍGT î3K¶›¬Ø‚ì¾×£?Óì|>öËO‡ïÞ øúÏáý7ëIYû p~–d…µÚ‰Ë ºZÖáäˆD R06„ø)} D®d-n‡n¡á¶û‡°‡ Ć9 ™YïLæyQ),'ë‹û|!¬ÃFŒ1 5 @ŠØ=»>Øç‹Ûužw2’ƒõÅ}¾ç6äå"‚ŸPPP­5‚QÍššÞ\ÄǺ|5‹ˆÚzA_E’÷ÅWñÎ*‘Uª©^€X—©nËã/ ê ÕŽ2,-ù›?ÖåëYÅx‘>asÝ6{He¡ÕjÝ$«Ùó¶#âö;l;Æ_†„¸^W:Z1Ÿy-„o/ÃC]¾žeÈö L+Ew^h®bÅV%ÜŽHp»ég˜g7‹=â,Áèñ. ÛM¿Tâ;¦³ƒÛ¼&Ü5/k’Í÷>ÔåëK×¢=²ƒÓâ»Ø§%FÓ.ßOð`fìd+P©Áë¯[Ì4—×íwÃyõ -Ý,’¶™Òqªcö,SQš ^ýþøËdƒ(6¾m7 -6·Ýäe‹ã¦Kç6ÚÚÄóv ÿôÓ3†k s´B$2ÊÏ*ºÚìóÿf/íxíGS*ìºV0óĽk]¹Û;üÝØÍbrgºmÙ».ï—Îœ»k´ :ïûöŒzko¸ºìí÷·•ÕNÍ -åzq»{GÈ íñwÒ êðÝps7åÉåòöysIÖóµnî(ï×û=ßØÔÈ -&²=µŸt]—ç\Éls?{ͼ>à¸í²õùœ²û×ð\ |ËséÍôf¤E=ÑOût©2õ¿¨÷Z[½ˆõ=åzË ÈPì ûy/ì–7ÂÓ–wL^¸î¾ÍûÊŠãÚ‘rW-pŒý„·<s4oâòXÂ÷X—¯ÎÿªÜò¿›ëí®t]"MsöËù 5…Jr×=”ÁŽA¶ßµ–îý¼áÒÍg¬íÊXo›¯`Êý\có©z¤Ær­¯âõ:M‘àrî6^mÖ(R®ñY»ÃŒ/îÇ[}ø´éèö¼ç=°ùú#&×ÙäëʾéqggŽ. y³L·vš¬ª5½µ_“.\xͶSétµKg¬µØÉé´KýX3vïn<"·¼eó¦æ›÷®ÌxºÆqö´s»õ39ÌžÝ3Ã`UmëýÌîÝîØØ9lL&;¿ú¾½ik½´ÇŠ/6_ÜæÉãz½÷)èÜž®q}î}ÍsŒ]>Ýï4OßÓ6¦ž}¶Ï³­S=úü¸=ÉËV«Ú›EC™ ö˜s·Ésý±•¥ÇpQîC7†´WøÖ"B‰Ñ‰>¸ã»st#T˜"žŒ|7sU‡Üu˜õì2ÛVkÄùîÜμúLä› ¼?»·É× -*MK½}n[v^Îøf£“ÓØN»ñuÚil?Õé´ÝÃÕR~<¤õ Ž°¾ÅG¨SC´×üI‹Häí­‘B,¡-]XŽÏô\¿ñ±·Ú¾/Aö -¥½>sþ2Åt±µÿ¸.úD -ä?-6¬ï6Ó*.±] ñÓÁÞmG,2̱ ·ÔÇ’ÐãU“Hå¥^¶DšBÛÇÔ,w9ŒÚîÆúkëfç}B­¯OÝÏ;BôLáZlwOKØÄöýÄÛUêP45ÌŽen;ïŠÓ$Ôk!MÓ ±.·¶±¬}Í!ìÐgÓ…iù[Û˜§å¼%<0-_JŸ\©fm¨fÝH]ƨڥÚZö‘¾NŽètp„ßRZ°˜]?­š½Ûu–æHÒÛÕÖÎn»m4í+•{«2ºÖfùe¸:—nxGvš¿l_xšöqû¸[ÒךÊä7ñJx: ÄÈ•øé Ùi­ƒ+H›ëÍBžÆíâBÃ_†4—f*lÍ&³7cìß3Uh—ó¬‚t—²·ÛÆÕ˜Äöfï4ËSgñ,$¶“¹BÎÝgg{…BN{#ÝÞº[l'L6ÉsmðŒ¤:yìÅ¡{]…5ût®œ…Ô@`º!ÖîÊO;ñÀ©}«*õÉR˜Ü]û¸Cœ:w>ÍâÜW<Ø«Šowòù&<­M.Åæú¥M2Äö4®¹OŸl` …!4ú´ŠÛníìd_¿jÇ&ó"îéðóa s‘5Úñhâ`oÞÏ™W8þm^áï¿rü¡½êuŽã‘:[Äέ;ñÉì{ZåA±}o« û¶™„Q&ÚâPF[mÞÐãxŒ_ËèNƒØ[ËÐ4tí,ÔaÃI»ëÆÀ“eô†ðtaÅžÈ|îÚŒ€}¾ävsö¸Þ^l¨p1ß.ËÐñÉÇiH\YÓÒ¡•8uÄÅ<(.§ ë@¶ƒi9OÒMoøe^ -¶¹¦‹)°K:avºÉÖ»˜š æùbX¨X‰úe¶RѾ©Î–ôû‚]¦àŸ÷Û1P²£”ÝÊúó?ŽŒ;ä -endstream -endobj -802 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 801 0 R -/Resources 4 0 R -/Annots [ 803 0 R 804 0 R 805 0 R 806 0 R 807 0 R 808 0 R 809 0 R 810 0 R 811 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -803 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb266-24) ->> -endobj -804 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb266-25) ->> -endobj -805 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb266-26) ->> -endobj -806 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb266-27) ->> -endobj -807 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb266-28) ->> -endobj -808 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb266-29) ->> -endobj -809 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb266-30) ->> -endobj -810 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb266-31) ->> -endobj -811 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 605.596885 101.433071 586.846885 ] -/BS << -/W 0 ->> -/Dest (cb267-1) ->> -endobj -812 0 obj -<< -/Filter /FlateDecode -/Length 4594 ->> -stream -xœí]K$· ¾÷¯ès+¢D½Ã‡I# b`‘‚fÛÁl€Mùûù¨.=ªŠýOìE²»ž.RŠ"?Rìª2-þ~Aø'3™œKŠ|<}8|z«ò ‹ËÉk¼Y´S^†,°®¬ò ‹–4^ñ&”̬ò -x6§¼ç9LÈÞ[GdlÓ«å¬x«ïöÖ“Æ Å¹Ä*²`ó5p²Ø5€Á% \J†‹c \ö†3g \.†Sò ¸ Ç’4ð–Às¤·%pÔÀdax®Êƒ,¾°†ÞA4аÀ{ÈâØkXචkÑ°ÀnÙÒ°ÀoÙRÔ°Àp!¦Õ°Àq}Ž¬a䔊†˜ëx€.Ü=iXàº>D§aìzÎQÃÜõLVÃîzž*²À€5,:OÙkXÀÀ]O6iXÀÀ]¸‘Ó°»j`‡QÃØŠq9[ ¸ë²e ¸ëB€ÊK>è5,`à® X‚¹».X§aw»¨a Ŭš w!AO VÃÄ{ðœ×°ƒ‡´DÂܥ‚ð§ø4ÐÊP.QÂܥŒíSyÅP‚Æ¿…– ÅX4,À]*¯aAî>³†Àxƒˆê4,À]òˆÔ¸K®X pyCаæ` €†ˆo†,LWåcKÎà.`ÉiX€(ll’¨¼d`H¤aÀØXä¡Dà.Â{Ñ° wmð^ÂܵŒ´GåAŸ†ÈkÀƒÛ*>dÕX¤ÐxÀ]KØaܵ6 "pW¦†1 Áe  Aì4,ˆ™MI)jX‹)ŸÊ‹×AÂdA€,•Y8±†H¬Mñ%kXdñÎiXdAÊ a2SÙ·z<€,0õ|À™¢†@2“ ³†)DœÇ UÉ p­ò2x”4,H‰ B1iX€³”Épz ðFÒ% xĤYà* ™ì™k<‚,H²5,@¦ ‘†ÙAdƒd—qÊŒVÃDhœ%¡jÇVΙYÂÌÑHª¨aÜ'M ¥Êƒ,€ °?rØàðƒÃ&Ž9*²øÀäì丙5,Ȳ8ë4,€mÊy3iXP,dAƧaA±ÑHš¬a`íˆ+äµ Ít$?ÍXút$aS1!«›L(£õ–â°–>ÝLœM(ª&Ô©cÑcª¡M¿§ÏŽû³8®dêòç¸ÿ €ì¬±†ºC^t²Yo³î?;ßgçû?s¾(Òqÿá盞W<)bmÙŸï³ó}v¾Ou²ÇRÎ ùÙñ>;Þÿ™ã=’râPhJñ!øBíTùš>×O¢RZw˱ÐÏ¥äÉÔJ¯æ „s¯u>ª²>ÜçÍe•ï -}ŠL«y]Q…¼Õø¦twíý^ÈW¿[È[ßVÈam™àe˼ÃÚäþ¥œmŽª…ÞÙçÍw}žwèH“õá>o.ka¹Ë†ƒµwlþ­Æo¸ùûjä¯Þà¾õ–?'¢æ\¿¬”{ˆ’|ár|÷áð˯ó»ßþé,üÝóá¯_ZK?„Ÿz¯ þÃOpÖrZ®#>£ ãwÀï€v|ÂoðÃiÓçùÌ“¶õ·´_ýíøîë•~›áãr2H¨ûDÎgõ,ÝW_ЗMÂé3/3Û3­JDÓg»üþö,…´‰ùýÒf¡aßêJ{ûxæ·ß•æÚ2~k¡qY®ý¤™²¬þyÑÜÓò{ÑhÕäfÌ*oÖ±h»Êr¿ËV‡ö½°Ëî¹úüö»L¼øÄ2ÑÕñ8oÆ;ûî°çÙ·~ý®F ß¹ÄÆ/_LœvJÎõ;x É(¦¨õhŸW…Ö%X'wI`½˜ÈÉ·ÊÖ[²åm#n³žUÓ/ɇ.oÎró®óç€öû¯ÿüÇ?L¿PÇ¡Aéò»Á{Z®Ÿ;´^œ¸Dc‹Üg´™ø Ë8OcŽ6vã¿)ízÕ÷yá¹Ü<É)¡ÏŠy®×6kêüÓ…57]¤iÌ2Ö6n#÷·SûÎþuÖ[á<ŽoQÑixþêFodYÙ#Î…K;r£põ\Ã[Súk[ß’ SÈÙè¯ØļWwH…ÐcàKÁ’6æSƒÈõþlíº¶ m®8”‚SqÍ¡‘À‘“õ7Áá¾>?78ty™`)˜–=’» xmô>Oy_^;SWì4FsÊÖÖ>uLîÙ¹H­Â<¹£.rjVîD–÷ëx¸UzÞµÏkäžð"w_n&­ùõ…8‚qÛ¶³ƒ±'Pn˜Ê}~Ì!è¿NG¥+#®'É\¸·² W­ŠöYû¸ ½…§ bOë±ìÆíÔÕoCE?i=O£ÿôëzÀ;$@\ñŽnùid- îÞ1g^¹êlïp‡äd¼žðšg„HåÎ^$NÁÞðŒ;û|jžR¸æ?™õ¨vsÇQ8:‚>btåBåãBm¡íO§¾ðªšAëiš«Íû^Y×t^h¡ôáåL–ï¯ìÀö§­h§¤¶Þov?Ž•ÎU–®vBÛhµž”±2×úÑ$¹Ýôi»‡o[1‰ò@Ó -[^W1ÙóXÅ$Q¸$‡ky]Åd?öÿNÅ$ùíÚ>劉´©s.£ßø¯J²Bkú ;}>\%Iò°ßjŒGª$†‹¤Üzw•äÎ>?÷A¨oœëMÕ’¬'~«*ÉW äaØœYj^{yoV ¼TR¶Ië}ÏìQÉ$yœcßÿ® -„|G/àË·xõ]áà´Ù˦»IÈ`uèÛÒ³<´]Hnô…î8±q9ÐeK¹Ñ?;øNæ¨éþ¾ê…ƒ:²¦ýWVo¦Š–2æÓ”D](l}NRù¹·êƒÔI‡:†©fÊV@ÍwBÝ}}~n¨òÊÓt–<%½æ³Ë&“_EÆ3M2WgÇuÔÛ-k'ìÓÔ®Í1›Ì—ná“õE×2nÍ®Ê,?O·œÄÉÇàbܨ÷Þ"+¶§9Ä®ÿ%^j%Þ×Íl+ñîûßYâM­Ä{A‚e×n[߉Íõ–_Ç  Î !2×û äö«Løìo9ï}}>çžÛÈv)õìò”¦¶šï0ŠÙX„)ï6“Üe‚$\’Í…”Þ÷ÌžƒaŽ‚Ö?<-®} gIʼnuº K¤ZýnŒéå•2¸H¯ÑŠãX_ -!°ð­8©@Ë·›Zÿð~Úï–“¶ØycܧTœ¿¡)7ƒ6ržx_ÎãCéÛnü-˜w ¿1faƒõÊC¯¯Ò%‹uøì|Ðú?×[­c›#O…¹GHº>)Õy>•ü~‘±žôZ¹Vžô]àP^E¢°|Bïìó©•kKô×ô|ËT+ûm f“·mOƒÛÀµùjlûEÏ^LƒOþ«’tèšY½ý7›76õÍò¤o -GÉ.äM 8ÃäéÁánH7›Ü°›¦›ý]‰»§R×êÃòŠ?@”än%ÉSñ—t'U1qÍvןTê}«•³ÁÔzøRƒvMçtvã -sÖŽzðwçvdý·;ŸÏ¹i…ß^©œ\>î+hu¾§eßN£¸?,sOò´ -__ChkØmåÇC¬ox ú–‹dœ¼ðÙxÙ›r”·Jdë¹ßMUÙt¼Ð³¾<ñco%ÿ‚dåKòzºµóÅÒþcÝôB@¨ø+¥óúî3[É÷°%¿šX!3úû ²—¶%•ú ít”›ì7-=vù8jþ‚Gv‰õm5Ò?´nòN¶­‘¯OåŸ'VˆèiO–Õ[C YÞ˜_®by{éÄ1 ÏmÇ€£ãM2y;Þh§éÙq“‡ä}3]uq»è³êhÚþÖÖ¥i;¯OLë·Ô•*o-kC5í:Ûi~ªtjh-û†pß'…$Oaìˆ_Ø64c3;Ú5y÷Ë™˜›!qoWZ;…t:¨í–ÑBß©Ô[åѵ4ÍQ†©ûܯÐNÓ÷bI^ˆ:ùˆõíãš,_©ô½¶y²·#ž1¼#×'oö­ÃÀ -šéÍD?ÛÉÙ{Ô”›ªàš&oÎÏP:U§~ î&%o¿WcñÍN^+i¦ÇŽ:¬i€8¶“º(¥n³³¾([¥½€noÝ5¶¢N*›è©4ñ¤:x¬ÉÔ­®“š~:VÎDÛ„Àtƒº)¿¬ÈCÎÐ]5Ø>Y¤ÉÜCwcÇΗ™œúŽ“¼Ê¨ËÛÇl¾O5°ñ–,¦ŸÛ$ƒ,gÞf>}²‰8( ò€Ñ—JnÞÚÑI^ÏÚ>'Ó"îéðá†9œ¬œ”@¢'y3ßœyÑñßhó5~þ~àãÚ#cç8îlG ×±uEž#™¼Ç5ßI–÷º6bw›‰èx‚ýA¦<Ú†f =Ž;7á5îv{kIA©sgb:œÈ®›®#?i&\!ž6Zì‰ÌÝ›°Ï—¾-NJ#ò¡&.æåà2ž!ù8 ‰+i:B:¸ì¦Ž¸˜Åå4a’­Äƒœ'éªùyÞ -)uÍS!—~ðx„ÙyèF««˜š öóÅ<0Ó×ë'—ßÝ5Ï„í0ì½M$ÿg]„0ÊmÆ4Tû¿èÝŠU—ô˜”òЀ¹Ý¡T_m]‹µHÔ/“åMö^’Þ,/àMö¢R’›‹âª„Rÿþo¶© -endstream -endobj -813 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 812 0 R -/Resources 4 0 R -/Annots [ 814 0 R 815 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -814 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 563.554475 101.433071 544.804475 ] -/BS << -/W 0 ->> -/Dest (cb270-1) ->> -endobj -815 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 231.360432 101.433071 212.610432 ] -/BS << -/W 0 ->> -/Dest (cb273-1) ->> -endobj -816 0 obj -<< -/Filter /FlateDecode -/Length 4418 ->> -stream -xœí\K·¾Ï¯˜³·IVñ>H‚qBr0rX­-Á*€œCþ~¾b7Ý];Ó3ZYr­vgºŠb±êc±ØÝölðó¥ÅŸÄvJ)ÇÀçÇw§÷'3Eoð¯”X]öwøþl ÇÉÛ’?¿Å)fïb8?ðgs9:ýý‹Ó¿ÖuìÙM9gýù—…Å4Å£ñ -+ƒe,),ï§ -+Ø CÌYcA Ùñž!çä5Ä`ç’ÂJƒ8ZÚ³2ÄpѲƂ6û¨°¬Ö£ò ˆa&§ð¬Ÿ”4ž³SHÙe•Á³Ñi<¢)D¶Þj¼<…|Òx Y|6VãyÈâ-³Ñx…)E 2*²¸IãEÈâ ¬ðdu%•YL0Vãeš|NÌ*/ƒgRL{ž3~ò‰Èh`?­j¼à¦×*/g£†1ò„¥ØjX€½Ô”àôà' —4,€àYÖ° fÈWÑ°KÑ”"sg! ‚l é‚g­†ÉADƒ$—°Ë FìÐØKBÕì3“†‰Ã$¡¢†pì4”*²&4,ÀüÈfÛkX€Í6›Øæ¨<ÈBž5,HÉÉv3iXdqÆiXÛ”ýfÔ° È‚ˆOÂlÂ$a²†€qðŒÑ° Û„ý¦c à—²ßLd2Øob‹§øm¦ ûͨaAfÈ⢨<È‚ )+~ LÂ~è¤ñdq™5,È!È~3iX€0ûMDôjJ É~3ªéà®Ï@&ÜEd4,Àö›ˆlµ½½ðúˆøLg&Ùq&5c`½k«š20À^¬VQÍ€¯ì‘Ô¤ÄÄÎ25k€eLkÔ´þ"ã&ÌÙ§1E`kfŠ2È ·`D÷mÏÑ‹Ñ3ÅæüØ/?¾zÀâ{þéߧ·_”LYý±JiF|7pÓí}[ Ÿ6˜E2vKYé4æ.” .I6 -Ëü;￬$… "*jcdïOøj–~<¥u. /ì1àÛ]Ô›ë\9""ÈçöÅê²vývi²Þ\çÅeezÈ9˜ûk…_vò›ÞòZáe¡´¸¥ß£z°Î‹ÏúØïQ =XçÅe•EÚ`•NþÀä_+ü±,{ý„¼Rø……¤Ú/vaPº@ b0l`±å€)‘è ém5^P\X˜ìOÊPvÏa‘ÆNÜðªðï^Ÿ°´/ŖFíÒ’o&‹PÍùóëw§¯þüÍßþòíw°·×oOß¿BÐgæ__iqíÝò™–O0}œ¿;)Ë_ÿãüú›Óï_÷µUÉ“p› írpór®p©HòÍþôÇ¿Ž’,=°ƒ¾ËäÿLãG‘jøn–Ïð™æ²€Ão–2 M4ƒ˜£—¯#C[uôKû¥­…Æyi'ìË•þx#úõ¾·_Úzúªý¾QÆõÐÇQû*×Û:<÷Áiž•CÓÀã4|ÿêm½ÙØCè#¾ƒ´$ŸËì4ÉyÑpêTí‡Þ¾yƒº<ØÙ0ÒRŸ¹ÒðýíÜžýaßN Ùß•3„];el‹ü¥­pIWŽõT9L‹Ìt_±ºÙÏÙä,)v!+m›Eÿ~écÔYµøâ9òæO3Œ¥|]O\Ûùü {ªÖ9zWïÇý¸è¸ŽÕÏeË÷ÖþE½ÑvlÅSÝ"Ó ¿tý™·‹¬µ\­ë»ü£g•1p·éí¸lmciSdš”)}†¹ÎÖ)Š»[é°ú‹(nl½ò}õúªË‹:ô[úE‡M&^ü`iÏÚ‹íe»ioö×nã?a=X­™$†Kè qc­K©ä•.®³ëÜ[}Œ•UõbgÔ¥•ìðƾÂëòÙàt¹~‹Ïy2ží8ì$w¾éx†‚¾ppZ·]yž6PNk¾xP_l–ö~Ê·Eoá_–—Ê6‰%ݶ—׿ýúJm/ùŽd¢VûHïAЬ$ßöõ˨øš’âñIŒäy}8xÜÌeÕ¿Yë›`¥éëÒ³±g+i™;tÇ‘±GñöyK¹RanŽ‰ƒ¦ûCºpM¼MšöWöæ{»Ò¦ÅQ -.hm> ÁQ|ê‚ÏI"¿3̯}b[g†ýg ŽÝ”r"€ˆ—c‡CPw¬Î§†º.¯—3Ę°DhP·—Ãjeì»gúuY©·S¶|šÇ¡\íc4™W·Ná‡É:¯¢k·fWd–߇kNâä«w!lÔ{ÐM=¦§:Ä®þUÓëD6'gïöä<ñÖ8ŽƒDˆS†zN‚eÖ*n¶6›ë-¿´aŸqÞ&ì0%ŸCÑOÉbFèšó«óù8/ðœT`õ8¥ªm£æF`¡ ƒeŠÜ¦“C&hå€Ìš”­RûHïÉOŒÍ‘×êû‡Åõ/·áŒ›b6YÖ:]†e¥jt].Gr .â=ZqÊ-< ÷h¶4eTqZ}ÿf˜ï“ÖµóJ»Ž ²£+šr#h#æ Çbò¹El»ö·`Þ€üJ›™'ŒWŽ(ïÒ%‹u &y­þÃí鳦=¦îÄØÉÍ‘Á¿š8siž×q›ºÝNêé}ºAnEÌrÓϦӒ {f«Çœ+Ê­½V–®@èÁ:’Tý;>¹½î’ZB…º+nToÛî· WZ{hó¦Ø­¿Õ±oé«ò†/YU³˜Ø“(5©Ó¬jL*$ÓÓ£E²OÑn:¼dQ°äjGsôǪ|nöä™/ÚÓèòÉy¹Ù/ë,ß–œ¯–ðÿäü‹$çaÁÃ4Ü›œ_2ö‘õ-Ôœúô‰ûÀqJ+L¸/q¿oç¶Ä}Hôœ¦å¾Äý¾íÿÄ}4Û±ý–÷4ŒUö 5é^}ãåùuè¶:ì ÿi™nJäGÖëöoIäGl²Ë¸‡ùë|ê r[†¡ -lÈ=»õñùëŽ?ÿDþ^Þ[ùÏŒöp"_ÿÖDþóúþø‰ü;t7$òïÒÝÈ¿SwC"ÿ‚3‘¯´ù ùÉʃ+»šã‰üƒu>5ÔuyŸÈølÜM‰|²—Xèóª¹É'ýØMãsMö¿Ôx^ö@’1Ål7ÓtÐ݃q,Ê­þU° °äò±¹³w_ŽÎÉ2±«lä,ÆFT ^—à•r  »2„?åæByÌ’äþ÷k9©ƒu>W–t»$pv®|×)‰ùã€!ñ½îÖÄ÷¾öm‰o¥÷ƒÇa3¼€·êý€ãyÖÜB¬äîÑ -YÄÒ]@á­œ2fÙ$h½ß OŠgl»sº¢©;Ø%yâLž¾Ù·çq€¤ok4v.ÖQã±}ý_ë8 ÉY!C7¿âqÀºÓKÉ[è·Âèá〃u>·ôm&{ SÇåi·ç­êåÖ^´¨œªu=8Vås³'k0— êFxξ[žz‹þ,÷nÈSDz4©TyƒOmªjÛÒJ£A¨Ü¨¾–lÂmžÒãI!~ijÓŒÉlüaÖä=31UCâV.×r -éñ¤–[Zóm¦b+•zÕ\5ßEé¦N©)^¡=–Q^8øˆ¡úuM–s¯6×& vãvÄÇ–Þ‘ŸN>*¥}Ç -ã«éDÚmädº½tjLUUpÍJ“§Èåï ¾QyÊËͤäMýªw"¾ÙÈk%ôÐP‡5 XŽ½ì .c³ÙQ_6¥¼€n+Ý4¶¢*è1Wñ¤x¬É¶Y]# „Uý4¬‰¦ -î:Ù7S~Z‘»œ¾¹ª7­³`s÷­ÝN ;ŸFrl3nåµMÞÖî`ó•øX6Þ’ÅôSí¤“åŽÂj>­³Ø(tr‡Ñ§B®ÞÚÐI^UX¿v'ÓVÜÇÓϧ²Ì9òN2Q»è¸~ÒÔžÿƒ2ßà÷Ÿ'>[_‹0¯ãÎ4´p [Wäq%“w¦ƒdyÇa%6·ˆŽØïd›zY_­¡­ãÎ xͽºéÀ^KÚ† ¶qG¢ï:È®™®³4hÆ_ >n´Ø™›¾`Ï—T'ù7y¡F• -ãpp™:ú’|šÄ•íK:¸ì†Š¸ÅåÐaă;iªùyœ -’¾†‹!ŒKê<îËìØt¥•Q EÓx16,©µÞV^É£}·{}ð¼/"‡êBmÐÞ¨O#µ L¨ ðt*•Ë°¾lk•tPuÞ®¡Ö ¾G-Èé®ÑC ^]Œ6:PS¯¶ðc§\i«NëBÑà4Si­ÅXe -Kµª–2â¤j)¶˜¢S5íÇYiJô㬄]Yi·®x-Z¢ùÑœcé%[,™Zk7*÷’Tiû¾yÝw]mÏÃ0¸ zdjz5¿+yþÝã;sM&B+UJvĦþ"!,Ôl#9*ï†6)`Ëëý"§!]rsËù“}–¤æOÆž0™hå-ߺO`ä댡©úp½Z6ênS€’ê0gvÙ¡òš×’{+I¢ve«(ou& z“¼Œ2«ß»ÊóK$l\¥PÊ϶Á† -endstream -endobj -817 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 816 0 R -/Resources 4 0 R -/Annots [ 818 0 R 819 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -818 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 606.123057 101.433071 587.373057 ] -/BS << -/W 0 ->> -/Dest (cb276-1) ->> -endobj -819 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 254.981993 101.433071 236.231993 ] -/BS << -/W 0 ->> -/Dest (cb279-1) ->> -endobj -820 0 obj -<< -/Filter /FlateDecode -/Length 4393 ->> -stream -xœí\[,Çm~Ÿ_1ÏTfU±n€ ‡I`!"à yü°ZéØVd?äïçcM×¥»93=sÖÒÚ°Žv·›¬ ›ÅúÈ"gÚž ÿ¾²ø•ÙšœKŠ|~ýéôó‰L -„ÿj‹ÕMe‡ŸΖ8™`KÌáü“|2©—âùí„_›Û¹ùÛé¿~súßu{v¦”èl8ÿùGa±7)§DAa°Èz…‚IÉ…¨°¢5xÄR4ĉïY bpÉAcA v.+¬ 1<'ë÷¬1\²¬± †-!),KÃ:"•AˆÙ;…gƒ‰PzÔxΚ˜‹+*/g“ÓxÞ›˜Ø«ñŠ‰1†¬ñ²„BVãÈ,3i<ÈÂ>'!‹žTdq9y— ‹ƒvŠÂËÖ•Ud¡HVãoBÉÌ*¯€G9å=ÏQ0!{OÏZ`›^åaGa‘bRx²D -EåA–àÉi<Yÿ4p"ø”³²[ìnOÞj¼YœK¬ò _C! ¥¨Av\Ò0À¥d¸8Ö@Àeo8sÖPÀåb8%¯Á€+Áp,IÃO• ‰@€,•Y8±†¬Mñ%kXdñÎiXdAÈ a"S,¢oõxY`.êù€! "E €d&f Rˆ8A«/:“àZåeðlÒ° %6pÅVÃœ¥LƦװo$\Ò°b€gYÂT ¶Š†pE&;DæÏBÙ ÒÏZ ²ƒ,ˆ5,È.ã”IÃxhœ%¡jÇ$ç̬aAæh$TÔ°Û'M ¥Êƒ,€ °>rØàðƒÃ&Ž9*²øÀäì丙5,Ȳ8rÀ6å¼™4,(YñiXP( “5,ŒƒG¤aA±çMÇ`_Êy3kXP<Ἁ#ž²o‹rÞL†,!ŠÊƒ,šŠ²oI8o4^„,®°†%F9of f༉ˆ^M d9o&5]Ü Ȥñ€»ˆL£†¸Áy‘­v¶'oHˆÏtf–gV3è ð­jÊ€€½ðVIÍÀWÎHjÒbâd™£š5€[Ó’š6 à/â1ÞaÂ%û4§¨"Ì!Ágæ$Y°-Ñ}?sŒfþJ³K~ìÏ<ýöÎ÷üÇ¿œ>ÿ¦fÊÚ? ã÷å`í;KÚë ¼2u<˜Itk[E„„Ou1RbZã’v1áŒáp˹_صÐY¤¶gê³ñjb,VËG]ÚÇ;½»´,>ž½øñibWT)ï5¾+—ÿÎû‹;B®‡…¼×ø}…—’œŠ/ó¶Ò£Þ}Ýç‰[éÑNï.mAÔáýRîÀ½Æïh{û§O'œˆõÀWãv^wo[Z—ÏŸ~:ýöÛùÝ¿þÇwPçϧÿþ?ó••¿M¦ë…Žþ¿\ó+!´˜®iùûþæK[˜ñ÷K›…]á(?µ~û[in¡Ù1o£qYƉûvu>ÞȆyCã×±^¦¹šŒ8úöˬCQ[òó始´Láýeú:U¹ü­m!¿\þº¼QAœÆäEÄ+j¡ïÑ_Úä¡‚ÞŽ¦þù¢vzýæò qyÂ8?šX3#Tƒ_žÍ‡E…¢î¼<#Ʊ?ìÇ/AX\S¯»qªÚ—å¯cÅ›rÄtMÏRu¹ÒèF•MRð‘ØÅ¢Œ-²¹¸¬S^묙l5}ù‰—¿4=K½NCOÇšs˜tï›iÎÛcÌã~\tÜž5\ÚÖë>þM½åí³Õ­æ™&ýɸm»lõGŸY[»Ö7 ùÅvWÏ°Œ£=—mc,cŠ,B[ïÙ3G÷Ëг¾O†Ž¹±ØîjnêÒÑV—uÜ[i³¯üÂËuëÖt"ü.Ó"wkgíMY$ë¿’å²çÇ>˜÷ä?ªŽÉ Oj˜©œŽÊgM‰à³©†M³3{´ÏSwñaÆIÒÏ ¯é䊓•L¼slÜf½¨¦ßZº¼Ù㜬£êÙþíÛÿü÷ßOžÉ¡y†· :—ûÏÝ\'3*’¶ÜL|“eœ—1G»ñ‚_C~»o|±Þã}¼¦öÝ,üÛòJÕGZÉtìå Ÿ¿¹Ó;ÈQ3SÒz™=JíÂJÞcß¿>ß“@N×!‹‘\×w…”×ÍZ6ýÓZßöP‡¾/=Ku‡wœˆŸÐ'6.{ÝRîôÏ8§ÌQÓý!ÝŒÁfMû+{ “½ÝSª 8뻨ù2…2yÚS7öœ„1òsqë=±ísqMÆØ6±dAŒf -Lòm·ÁîXŸ_캼^럨ƒÝçµÒGLÚük z×»…š@ìâw׋QbîÇ› øº‘Á£wèhÜ×Xck0Ë_zÚµ±v»5©ÎËjx1)ñ|û|cÀ±Óf(ϪÆðð®ú6 K`µ‘}ƒDÕäçå.n&s œ7*>ˆÜÁ‡l¡¤ô¿ë7‚“Ëàb|rvÈ‚ÐûþÇüF€Uºàykb[ŸÛ|ë6Öé+‘®¬,OcLÖWÇÓ·³|@ÂÕ<Ø.%Ï÷¶ó±>g;Þì]ÁöžŠ¹¤b#\…õ›A  ò¡–¢Å¾÷‘Ùc6„¨É;mö—ƒaœÚz]‘a ]º—¸/—sd¬Áó3Zq®–n‰µÞGf÷òQ@Ü$möï§MÐ ¶SwÆÍ°2Ä.´5 ­¦ÜìGG:$·¯=®ˆýø[W"gÿ -×÷‚úØ‚ú§tyù™ ˆ‚”þ/—ç­Ö±=45 ²ŒÄt}¼ôstO´ÊîUæÅö©85¥Z]Þ÷ëÜÇ6o çØö9,ùX`‘àl&­©Û+gv±ÁŸ|ÌÛáÌÈ<ØçKòÙ~ßBNËS Ð‚Š8áFÛmÛôÀÖmåõí») ëï}¦`ÅúG¬ -ûù†Uu‹I#3×2…ݪæ,S®rï-ŠM)€¶™ð–EI\´XΦŽÉÝõÁǺ|4{ -ÎÞ´§ÑŲ¾þ*!F“#´rݤ>ß2‡[té›—ùæv­~2êt¿F‘i*¶qoé ·¶G‹œKÌã5¹íÐ_å»åG©©ïRÞ]ˮ鶮 5ét7ðJt*ñ²KÓØv-__×8yÚ4¯÷M¹­Î¬¿W ’O®û¿öº;îT’‘o‰û…kAë‰?~-h/ï#µ +O{¸´ïÿh-躾ÿúµ 't7Õ‚žÒÝT zRwS-è†îˆž¬)cþªµ LÙX yÇkAûüÚ`7äÅÄÓ–ôZÐÊ`wÆàås[R5;^ :Øçã#\‡ÙÝóý£´–ç™ZÐZÅׂöý©=7û¨íû?Z º"Á׿t-H¾ Sý´íáZÐÁ>g;'Þò-¹üŽµ õ Ö‚ö½«)³?\ º"Ã×_R zB+S-è)­Lµ eöw©ÝÖԗׂöãy-è)]Nµ ¥ÿ/U *Ö›RpÅ¿`-h=é­Ì}‘w¾\àïp-è`Ÿ–»/ÁßBο©Z÷–U½-h=áÆ¢¾[¾-—ÂYâù¶2Ž´yúò]7£»MîXMÓÌþ:g»ùîÎZyE?Éû7$r*I¾YzMu’<–i—$£$´}Kø•‹¹Ì×TÞ²©iI(†K†©&piQÿ’h­‰Ø²€oOèO>Ò.Ñ\ç{Y–í•zÂ|5þœÌŒsÂvz†Ðža·”?Ÿbý–´­oÊ 8këäUZçÈÆËÚÈgOåÖ@þ¶t¶*Ûž¯ô¬/ û¹·’ß ‘¼¦D¾¢^—v¾YÚÿ\ý§“•dÒeS_ÞD•œ]»ä·“+dæQ -ºCârI},ø9·¡½“)Û–Z.GùIÐèréB{éZ7ù^/Skäë7[/+DôŒvO–§G³å\Ùøå.vQ$e܉c@žÛŽGÇ»dëi"†=‘¦éYÊ\­­ë—¥¯9ˆ]ôYuvZþÖÖ¥i9o_O˜Öo©o*UÞüÓ†jÚE°Øh~ªtjh-û‚p_'…ôzRˆ_Qš±˜?­š¼?áBÌ͸·+­Bz=©í–ÑB_©Ô[åѵ4ÍQ†©ûܯÐ^§m’— -N{„|»\“¥òØךòd7nG|=ÁÅðŽüv -IiVPh¦7ý4n'gö2¨)7Uak6š|û\~_ "tª O½°ÜMJÞ 1îÆ$²7;y­¤™;ê°¦Ëi´Ôeåë´Š¾l&¥½€noÝ5¶¢N*›è©4ñ¤:x¬É¶[]' „5ýt¬œ‰Ô„Àtƒº)¿­ÈCÎзj >Y´“¹‡>î ÇŽo39õ·ò:.ow²ùF|­Ž·d1ýÜ&dGÝÛÉ&â  0ÈFß*¹íÖŽNòŠÃv96™æq_O:U7ç|p’ðˆÞÊÛ­æÈËžÿm¾ÅÏÿœøüûö:…‹wÔÑÂul]‘gO&ïBÌÉònÄFìÛf":ž`mmC³†îÇ›ðšGwÀÞZÚŽ ¶sgb:œÈ®›®³~ÒL¸A|Ýh±2_º6Ãa_n}{8IMÈ‹8šT¸™·yðüpÉçiHÜIÓáÒÁe7uÄÍ<(n§ Ël%¦ä…ÏìõuZ«ö¯fGl/ ‚£–ܹóõÒ”c†°¼0qžÒ%÷¹?ÙgIZþdž ËaØ{JVÞ®‹ðF¹Ï˜†jo×»Ré1(é¡sû—£Ô×ÃÖÔcMõÛ$GEy´— 7ËK,Yýs¶’Šõó¶s -¥þû9˜Ó— -endstream -endobj -821 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 820 0 R -/Resources 4 0 R -/Annots [ 822 0 R 823 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -822 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 642.273057 101.433071 623.523057 ] -/BS << -/W 0 ->> -/Dest (cb282-1) ->> -endobj -823 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 257.184971 101.433071 238.434971 ] -/BS << -/W 0 ->> -/Dest (cb285-1) ->> -endobj -824 0 obj -<< -/Filter /FlateDecode -/Length 3985 ->> -stream -xœå\K$· ¾÷¯èsËzP/ ð!@ĈƒX$#‡Ù¶ÇF0`ç¿Ÿ*=XUœž™Ý5$^ÏN©’>QÔGRU½îjñç3‡¿ -9SJ͉®·/?]¬ÉÑâ¿ÖbwÑÔ_ã'Ç«³”Mt5•xý‘!›\£ÏéútÁ_‡KÙüéò×_]þ±¿Ç]½©5y¯?Ç* -&—œmTT*ë‚¢ŠÑäìcRTÉL±VM1“§³*Õ5`÷EQÀ”]8«*`øìHS†«1+*gÃykU€X¢à‹&ÁèIÓygR©¾ªº Ë^Ó…`R&¦«&¥‹¦#`‰Õ:M%:"«é€…BÉš.KHÁª:`ñ%M—ÅÃ:UÑ`wU,6Y§éj0±"UW¡³%—³ÎÛhb Áj:çL„oU‡…EJYÑy`I6VU,1X¯é°þh:ðD ¹e·zÞÝÁ§é"°xŸIÕ _cŸ€Åæ¤Qva^Ò8Àçl¨zÒHÀ—`¨PÑXÀ—j(ç Ñ€¯ÑPªYã`tÞiD,°DJ,„«ê€%TÒ¸ x`A BOAザ%x‹ÆtK¶: ø–¬K.`Z 7”DP.\©j\À¹Ü©qAéb»g X7Ää5. Ý@%i\À»œÕ¸€À»!àNU,p` ÀŽ&¸4. ðnp6k\@à]l#¯qVÕÀ“ÆðãK±x×Kx×g„U— ö`и€À»>b -Zäïúh½ÆÞõä“Æpþ±j6Þ…D=¬Æˆ÷Ðù q:3i‰Æ¼ë*Ÿ²§ÁVÆ•š4.ˆà]W°|ª®—aqeßÂÊÆ¥T5.ˆà]ª qAï‚>‹Æàxƒˆê5.ˆà]©5x×ùj5.ˆà]ä Q㸃qH4.@|3ÎÂuU]5¶–¢qAï‚–¼ÆˆÂÆ0‰ªËŽä4.‹_O0tªJߌͧÍZߣOá™ÍÃù'm¿­˜Kûœ—hôuëkÖñ¶v· ãò¨5Žÿ®ÛxÌ5nmÛçÙÿ]»…ãÜÚŽò“°ßƺýìcÇ:Ú{ãÂO‡9ô~´y¹ÑG4Ù˜'õyßÄ Ÿ» üv¯Å~h×E¬Ëm}nkÓly׆ñhÃØm(ׯíÍÇŽ[úJXÞ¯ulþÓö½¶ç“­¶,÷ÛswñVwÀëIàºí÷+âÂ.vÞ–©W;k‘8:Þ·¯¼çƒ2®_"Âòƒ”‚TÆ[5ÄÒw‚&ã$øí÷B=ÄÒ#~o‹ñìÀ(ÉlåóÊÃÀÕô~֣aøÆõîÞÇ}:0q“ÀÙÓ‡x˜S<ÌiêoÏÌyØ"‹>_ÀØÚøîoEû »þ¾MÛ‘: -GŸ›ÆÇ/^¸¥^ ®–øÜŠ¼pô­˜ £+}3fE/!HÕTܔŸpö™-Š‚¶Ô–ûš ¿ÜÀÇQ÷·èëîùOoÑ…7òó†\@mÚ.:Måήwܲ[„9¸>‰­~;¸ÿÁÍ·èsp×>Ö‰ã­wÜN‡ñòAØ–tDVÏâÕj1üÄÚz5oçˆßï£ïÈ&FV¢gyç,ŠŸ:W~¾s´ÕÏDòaº¢ |ÚïSzÁ}_yÏÇÔt¿D ¡Pîy±}àØÜ=¨Çvû ò¶÷îxïègðÈ{r}ì7xåzÏ˦å•+ŽÜuzY]uHó²¢x™ZcÉî0à=‹übó–`-ì /8Øënùoó/Œ}Ï¿öëûÁç1ŸÒb°·ˆZùý,`à£É£¯?ˆEš·¥oªkŠOSó'Ï,ä>û°šÿÜÏfÅŽ­ñÁ]±<‡ã£kþsß­ÆÒêÛWîëÚaÏfßç6jÕ­M]ïsÔÐAÔÇ£nu‹8Gç³ï:êÉ»6ÏG»kwYçF‘9Œ:5v\mž£œ_Væ:ëðeëY_Óš×Xû8ÚÜD›ØíC›|ÖȯØD_w;ðYN»ýŸðÙ[èõ¶G[o§fçùsY+ñ¬áÓ°\ÎÙØOÀrç~ÞÆr…_Xú…XîÜ÷.·¯b §wÃwÛŒ)é{/úÈ‘ÝÜòÿ8N€{ûÙßaŽ“%âb—ñs:}ô/Ú;mÇX‚Å,û`w'zÇÓᆋÖ:ÎOêý¹‡›¯½+lwÇ^ª­¶Ñ҃YW„¹k¯x²×SZÉ؃ý£˜O¯=ïùeD§¿ÓîÇÓù´·Gëë¥vùÚÁ‡ž;Ý?Ûô®ÝòÑn+‹½’÷sÜEÚSŒXó™'óÊž|ΗV ø:qÌ‚Þ;V¤Feòu­"Ç+Ÿ²ñkmˆE¼¥1k·›¼P«ßéOž¯îð¬_W>‰Èü¢¶GIV3¿‚ô\Ìp8©tãÓõ·Í­šyù)ê©E_B?¶—[ËÒ\ ‡>nQlwÓN%ͼu“{î"’°ãÞ6ÞÃÚÆòÁÅìl1g—ðÕm~ÛNKùÓ%µ×é\{¥Ú¦bœçï\]QTòs\¯ü -_±æ«®Û]Ÿ¹³}Sí§ÙªyÍòûìü.ãöÍ6qÑÛÿÔýGd)dÒÈKÚMl?.–øéÂ_4"EL´’›Ä,ìíì‹ÜÌ•†ìvqðâzl‰rßö+¡â3€í£c6|·ñ `dG£Ð^ÚV„¸3¹³˜goëbþ²V1¡_¥ %¦ÂÕ!ɶ«Ãuã‹b¬ƳЊá–ØÓÀãøåÞiº´„º4Ë?Úú,–óžðvÁ°á(}R¥ü‘ÑÕ°®·S¨:¥q´œ BsÑí¢?³£kÂbN½X5~Ñv–áH4ÛÕÑNÝ.j»Þ[œ+•g«²n­Ãò ÊrõP¦áÙM™¿}*öˆ ãã^Ì9ò\k[„ßø“ðvAˆ¡“øé³Ò:.®°q¸žÑï»üeIQ‰wSak¿¦ÈoT§”™§}p4]Š0×Õ„÷æï$åi²ip”W[a.—óôYi/W¬ÒžIw¶žÛI…É„<ϷϘ¤&yìÅnzÝ3… ûL®”B;@`¸%ŽÓ•Ÿvâ…3έí,9áîqö»Äirç“ç¹âŽßŸxg¿Âç‡ðÖÅìúe ²ÄÞÎh·Â ¤°Ä‹FŸšxìÖÉNü]Øñqm2-âÞ.?\Z˜ó!z~¤˜‚ã¯AÉÌË]ÿ…6_âçïº~5Þ»Ý⸷“-üäÖXF2þÒly¥˜¿D;„sÛ¡'AûKìÊj‡7Ì8î½àkZ·ÛE죥› ê¦V -㲡ûéºÞa™xGx;Xq&2»6+`o—aLÃ~c{ Â…œ.ËÒ…’¯¢K\qÓÒ¡%/nÄ…ì—bÀºí`²CÊA¦é?É¥<–¸i_†¥£fe×CÖf!š òBvLNZwxм½>zÝ;ä4vK'µO铔Ή±tž. í2í/gÌ í™ôؼËB³¯ ö^˜IÎÚ+µ Ý…ôQ!-ë¶Þ ÊAiÈvƒŽ@1é2;d³Ç<0¥~Û0K›qQ­”gN±¤?HKG¹*ÓˆQ®J:µå~GÄ›ÙR˜ P”îœ×ììj9sɢȒ¼{Jiµ Cv›öch{Ó $­ÌÔ®Ûâ©åñ߇ìížÃI„Öjüi§#®¬oª P“ËÁ‡öØ’8„øû£™Š8.yó=÷ÏOΧ$ãüDŽ„å0‚ÍŽÿÂõe…èjü3úmÕªSz›žýÊÏYÁ§CíßhüÛ!ѼÌ\*ò?8é-ümçlþÒé¥ýù7üÞc¯ -endstream -endobj -825 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 824 0 R -/Resources 4 0 R -/Annots [ 826 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -826 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 606.123057 101.433071 587.373057 ] -/BS << -/W 0 ->> -/Dest (cb288-1) ->> -endobj -827 0 obj -<< -/Filter /FlateDecode -/Length 4662 ->> -stream -xœí\K$9n¾ç¯Èó£•DJ¢€ÁXx 0ð ªk§waT˜õÁß•¡GD2ÕÓö °=5Õ•AêAQä'’áìñóMÀ?ÂÁ‰Ô’ùüúéôÓÉ»’<þk-vý=~K:ÏÅ¥P³¤ó'%Pq¥¦Xòùí„—kó·ÓŸ~sú¯}ŸpŽ®ÖC:ÿíGe1¹"¥ød°*X>ÁJÉ•S6X98,±V‹1RáÈ׬1¸J²Xƒcƒ%ƒ¸ºfUˆK`‹1BMÅ`9BôÞäAÏLÑà…ä2”ž-^ .KÕäðB‰ÈåÂ!‹W]Î9‰ÅcÈ’ª/A–˜½Åƒ,LR,^†,”É›<È¥Å+%B;Õà du‰Éƒ,>û`ñ*¹T…ÙäUð¼¹æEŸ\"oñBp ¶I&…MÊÅàEÈ’}ª&²$òÑâdaüX<àD¢"bxkTï&OÁâ%Èca“Y°ùÄ Y|ÉÀƒœâ’…±Ç5²Qȱ°X(¥:.…,ˆ59εX8@>€ƒä!Kâl!ÈÂð\“Y¨²…! XX@Y"“…´eX‹…¸e_ƒ…¼e²…À…˜ÞÂâ’d¶°€¹0¥jas ØiatáîÅÂêRÊÑÂìK¶°€€»ÄÁ[XÀÀ]"ô4ylaÐÑQ²°€»|±°€»p£havÕÁ³…°E¼… ÜâÙÂîÆ‚#Àä$ ¸–`ÜÀݘ|´°€»‘c¶°†â€?ÞŒ€»€d‡kaÎ{ð"YX€Á\DXbaAãÏði • R³… ¸Ûgòª 7üZv!çjaAîâ " pð)ãNÔhaAîÂImñ€»!VoaAî"nHÀ\@`aÎ7; “-,Œƒç½…5òÍÈÀ/5ß *yä›Hñ ¿­”5ß,T†, !ŠÉƒ,šªá·À$ä›@'‹—!K¬laAÍYóM±°aòMDôfI@4ß,f¹¸›*Éâw™f p|‘­•Û{o*ˆÏl¦hÆ)fÅÀzÎV³dà½8­ŠY3ð_Í‘Ì¢ÄDf)Ù¬àX3x³l࿈Çø -.Õ§µD•ag¦]d…[0¢û‘sÌft£Ù¥>ö·¿œ~û‚Ã÷ü—ÿ>}üM«”õµ¤N -˜ÆnÇŠÝþt‚B"®Í ù dAdz `º·¸æ#±U-ñõ´‘–‚ZìÛžŽlP}™qmo½È¶P_—•,ä·’d¤rJ³í˜n%òw’çJÞê\ôœjjÇÒïëéOØèŸÖ1¦6*Çš:âÁã ¾ºè^–:÷:àtÙj—h¡†ö{­ÖÍ^·hÙekþ·Eëz­¯äf²Ë‹mCe¨(ÂíÉš>÷¶øtEÜÙÐ$ïmh¥#jêÖ2¦ z®6$¶ ÉbC‚ÖGrʳí˜n%®6”MÔ¹è9ÕÔŽ¥ßׯžû‹x®†êúßùúƒ²;íŒ5|²ò¦“íô¶èþ«ó}u¾¿3çËú_9_x‡ó­/+^±³ì¯Î÷Õù¾:ß;Ït²÷…œ7ò«ã}u¼¿3Ç{Oȉ¤Ðgý¦œ);Iq‹w-iäv{ˆˆ—ܳÊÏés?ÕÚzL\Ïプ -®•*x7o -È{}¤lÊúî>_\Ö”Åeªš˜/óÆj -ù¨ñCéžÚûk!õ[š§…|ÔøË -9­M*Ll›wZ[Ðê™ÄJbšè³¾ø¾¯O-™Ò¾¿Ó—öºÚ÷ûNðŽvKþVvÌÒ¾ Ô{tŠ~¡qþáÓé·ßýÃ?ýã¿~1øxú÷o½‚ß %qýRÿã3ƒ}ûô‚߈ßÚŸ.4þˆßWüæí¯öÉ¿ûóßÝCï¥rèÔКO:]¹ óeúøz&TÀü¸óï›y¹XHŒYïÔˆÐî6jß}¾‹jy›a㥰ýÍÛßNO—…h[U¸Ð›¤Å ËQò˵*¤ÏÑúÈ~¾1Çq~Ú~㔧mDžcí7hk·´ßoÔ6Ÿ½Yß„çõÌG=ûX//²—E?u¦sµ·iòn|­œ@ÖA©Ë©s\Ö ã…?_“I¿¶Ò[£®Çñ›N›núxév¶Ú\··ËZû8\6§Ýè­MØäîv½éyÚà´¿Þþ0¤»À€6ïêºÑþ]¯©µÙS›×Ð_‡ê°Ñ6»éËýÕmÌ0÷@û¶Ï›ë»~µŸ¥ã¶öm™§ÞyÛ_®‹ìÛ|C··öõã;¥»úËWú«‹ÌåZþŽ¾ýs÷¯.ck»øðЇl{ûaAð×Í~ü6^öÑú­¾\¶ëÍF’!ÛhËî¶öW¶~ô™7=¦ÃZdYß} Ëûþô5ô±C¸7‡|낙˞,˜ö‡ZDA#ìe TI¿ÇOp*ÒnY@Ç  —(ä½}>+Ùbõ¦¬EýÞ»ä«x>$û¬ÕŒË@iÈ«‰j H—ø䟿û·ùãŸðóˆáÔÏío?>K?æÆ‘vs⚯z[Öaâ oã¼Ì9úØ—härÙí½ZçÅD·vwÖÑÚöµ¼.ŸË¾]ò×s´5»Ÿ'|˜c´6|}û«(×u8Ú„ùù8v“3æûó"oœc4þý½Èäõ.•Tª±©·{kEF4®¾±“ú³8½‹°ZýÛªø‘ÑRÕ»IîIpåÔY‹F™2FEõõ‰ûNý\Ÿ_Ú©§¼œ”Tc6:´3㾑w£"={ýS[ª*°0!›Ý ð”AE„gAo"£÷3³ÇvóbôÊ 4ÈÖ[šs¾%Á¦»®£#H Ç=\ùmŒÅÑã=±ÚË­î!KÖ‡º&ä…˜]o¨ú,]³®DJÖì/¬=C÷;rMGsÝiû±ÆLì¤ -Õð9ZÑ[ÚS z‹Òçi¥D%i˜ýòßýzñkxk\l3ò«P}y ©Kž²Í¿ÍÏùmÒ(Ü#’å9‡ÃþÛ?†w`A{RÈÒÅ3Ǭ׃…%þå²ÞàoÄmUï×m• Ä’ˆø"ñ#„ªË¯à‘m»@’ã¨íŒÂË#>†‘¼ EŽÊ–JbÆÔ 'wÄ`ŽVE+ÊÅØ[R%K’Ñ t«¸p¼ëóJUŸ 8LÚ±A{*ÔwRŸÿ ŠSò`÷ŸìósŠ³ÿ±{¹g—²Ûæ¶ý¸?ÄÉ ‚ŽqýñØ’=l ˆ)Öøu@½Ãªr¬÷¬jXLÙÒÀ%­VU—TZšÜ×Ů֤UÐý„÷,*ãTéØŸƒ“çºüÚì)«íß±'¿ÈO¢K`{ÿŸ¨wêS)U}Ó®šEæa_¦ÐÜ-‰·"õ‚ŒEŒQ4nmóžÞÆXÇ­sÎ_¦ˆ\hUj;':.bÕ,w¯ú²…梷ºï¼îó -Í×㼯Ð,^nÉáçZ>¯Ð|=v³É­(×mªõ²Úà´Q0îEáns[ÑòVñõø•Në¿­ëXL¼÷M“õ-Sº»¯úˆÍ~Í­º¤^kÐ à}ŸÚúû:ºOvd_ìpÕ‘êã~áOrvú†ˆ0ä± -MyÑÓëņnØVçb.”’1¾§åôhOí“ÿSÐÒô( ò!P+ÇÇûûüÿ„£i,ã2òî´a„£sëÚg­)lõúav‹ùi»ÐiwÌ´ñ|ïú÷ìÃ]ª¾{!ˆooßÈðßô¨ô\—_:W§:è{›Ót -Í_Ñb8ÐyžÞR*pÀ «÷Cªûʼßôg(q? g¼¾[Eoý‘DZ8y ØS]¾ÈÍ_45Ô‡Ð(åíížß/Aüƒ¢}Ò×°>f¶üÉ\>dq>”DÑèßÊM–(7Ì©G¹J ÍénÓ/fN5ã´‹í!GÒg× aÉ#Éžëóë3¨àõ‘\`lÝé¯_­ie[6Ö÷1³$k€ôã°Èï·§ùJ:ë Fú45ŽiY»þ°ÉƒMîÉÒõ-RWžíói ©¯ñ"9G,°}òõV6­_ÔÆí í7;h‘zLV/„–[m_¬ÇåËó~¨`É[Ô©GlÕo¨[‘rÄçKÜŸ¯cÜ6ß˳-7ìÆï7",òô/†ÇR_ÃÕVþtÊí)îÐÞäáSZ}ñ‚#Ý›zÖ'ÇŇ¾u¡);œoôl/Hûi´ÒAòú}„¾míz±µÿ©mú'¤qìrOÜÚû|#Kì“üvÒZdæ™ý= +qûèÇXFÈ×i¯€2½öÐQ†ß>ÎŒ“fˆ˜új´êÝô¹cö½µ'o/DôÌᚬ«÷.ld}G˜8Ú®ò%å.3ˆs@^ÛÎgLJä@~!¦k¢_¦›äÈ]ž ï”ªË“8D_U–íïmcY¶óñõ„iéH}3©úf¢>T×."´N£E¨:¨©·ÂcŸ ’Þh}EüÆ÷¡›9øË®éû.Dé†Ä£]íí ÒëÉl·–ÆN•ÑJf×Ú5?E™¦N2oÐ^—ªLÑ—.>â©Ü“µ0öÚËb7ñŠøzÂÃWävsýuë4±Â§nz+‘–qYü´—I-ÒU×ì4}:^ÿ½@ETEžö!ð0)}Ãżš“¨oò^I+=ÔaKËl»¨+”2lvÕR9£½‚îh=4¶£.*[è¥vñ¤xìÉaXÝ +„uý ¬\‰¾ é&9 S~Û‘§œi¸jòc²sOcÜIÎ;ßVr;ôu%CÞ1îbóøÚ6>’Õô¥O2ÉÑÓnN¶§`…Iž0úÖÈÝ[:é+ûÇédÖ‰ûzúë©s‘RÔ`;Sзo­‘W8ÿÚ|‡ßÿ<ñùý©Ë9ý@‹8°uG^O2}W£¨o+u,L©ðl*µË¼¿gµ/´»óN ±hñ=AÎtZðîbµÑ…*³ÛÖ ­“r§í&íÅ€ Ð|§K—)oݺZÚŠÅÔR1ŤþuÕtZwe(1­»’¯Úê¸ýÄÑ(­æ\æêül9bI1hyí=¨<[R§]ÏÍû¹ûi{^–ÁS<#S?»¥«–Ÿ…Ÿã;æcNV«þÓª#Aæ ’pPkÕ0R{çµ—œ!¬¯-,¼<Ãõþ>÷ë'·_ϳ΄ípLäKз—Û"¼Q3–¡ú›ÍínÕ›KzŸŒòЄ¹«‡ÒB{}m»{ ‰ÆeÑTQßVMôŠ¾d³ø©ñö5ûË®„Ò~þóŸÕÓ -endstream -endobj -828 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 827 0 R -/Resources 4 0 R -/Annots [ 829 0 R 830 0 R 831 0 R 832 0 R 833 0 R 834 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -829 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 512.554475 101.433071 493.804475 ] -/BS << -/W 0 ->> -/Dest (cb291-1) ->> -endobj -830 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 172.663410 101.433071 153.913410 ] -/BS << -/W 0 ->> -/Dest (cb294-1) ->> -endobj -831 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 153.716389 99.933071 134.966389 ] -/BS << -/W 0 ->> -/Dest (cb294-2) ->> -endobj -832 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 134.769367 99.933071 116.019367 ] -/BS << -/W 0 ->> -/Dest (cb294-3) ->> -endobj -833 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 115.822346 99.933071 97.072346 ] -/BS << -/W 0 ->> -/Dest (cb294-4) ->> -endobj -834 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 96.875324 99.933071 78.125324 ] -/BS << -/W 0 ->> -/Dest (cb294-5) ->> -endobj -835 0 obj -<< -/Filter /FlateDecode -/Length 4156 ->> -stream -xœÍ\Ko$¹ ¾÷¯ès€ÕJ"©°˜C€$È"dArXäàñÚ»<fsÈßÏÇê’JÕEw•g<ˆÇ»‹Ôƒ¢È¤ªªÃÙã盀_…ƒ+¥æÄçû§O'ï²xü›Z¬.&öøŸåX¼JNja6y<_rÙò¢'…È[¼œÀ6ÉäÁ£°I)¼Y’—jò ‹ ãÇâ'„r)†·Fõnò,ž@–3›<È‚Í· &Èâs² ä—, ˆ9;®‘-ˆ…. -ÄRçL Ä*ŽSÍàÅ`yÈ"œ,$ Yžkò U¶°€"dA ˆ Kd²°€€¶ k±°€·ìk°°€€·ìC²°€¸Ó[X@@\*‰-, @.L©ZX@À\vZX@]¸{¶°€€º$)ZX@€]â’,, à.qð0p—=Md[Xtt -YXÀÀ] ->[XÀÀ]¸Q´°»ê`‡ÉÂØŠ‹¥x ¸‹g ¸3B€ÉË>H0p7 -–`Enàn-,`ànä˜,,€¡8à7³à. Dìt²`-,@¼/’…ÌE¤%p7T„?çV.”š,,àn(Ø>“W]Èиá·Ð² )U ¸‹@EpðY,,Æ;DÔhaw!R[<ànˆÕ[X À]ä baÌÁ$ ¾¹àaº&¯:_K±°@€»€¥ha¢°óHbò²ƒ! ÆÎ#µ° wÞ«… ¸ë…È‚ÜõŒ´ÇäA*ÑÂä5àÁm ŸF²ê|RX<à®Øa‹Üõ>W pWæ…)+!  C-,H…]Í9YXª)ŸÉK×baAö  ËäAÎlakW© r„,£…9B¤  #p5 û6ËÈs1ë†,È-,’¹R™-,È’PA«/EW2àÚäðB¶° gvÅÁÂÔR®Àé-,À;M—,,€à¶° WÈW±°¡È•ˆÌÜâÈ‚$ÛÂdºà…`aA‰Ù …%T™É[X€Zª¶xìµÎ,NNSE à&¨4”&²&,,Àþh±- øA±‰2ÇäA¶° ”¨åf±° È}´°¶©õf¶° zÈ‚ŒÏ‚ê“Ó4ÙÂÀ8xÞ[XPCA½ÙÂø¥Ö›Å‚Jõ&J<Ão+%­7³…•!‹ E1yIS5ü˜„zèdñd‰•-,¨)i½Y,,@šz½y$P´ÞÌæqpW*Éâw‘™& pz™­UÛ{¯däg6³hÅYÌèÄVóÈÀ{­²yfà¾Z#™‡•eIæ©Â*˜Á›Çø‹|Œ7˜p9}¨Ì!#f–¬‹¬p FvßkŽ¥=Óìr>öÛ/§oï|Ï¿üçôø»é¤¬ý?qÒÂZÔEbÅv®Ü4£˜HHF)Ä‹„ðsú¨¹.BÀâ|è¦B"#ÜÖsÿÖ"Cb•9 ™™VÆ㼨¶Š’-ë‹û¼’¬ƒŽ`€HËa£’t#Öh`7õz¬Ïëëu˜wQ’%ë‹û¼¾¬À‹ %¯çµ…Üi¼+é¿óöÃŽI‹£Bî5~e!©ÏëµÀEy1“z”tl*\ÍNQ_Øå5öi¶šÐŠjÔèÈF]æyÕø÷ïO°ÃyúË=„~9È|“¡ÛèÏï?ž¾ýË÷ÿë?bÚ÷§Ÿ¾ÓäÝ7ṿÿ<¿ÿþÆÈr}Ȃ赚C‡år†Óü÷ƒ÷w‡D„÷5Õí$Ò3ZF -­î)ó0Ï=^þJ“p–ØË»i’?¼_oǨf°È`dš‹Þ´—#>ËZ¾†‘T„БכF’¼­¸+òk_+DÜÑÆí¦_ ‡õ,H4Aê圵iLiO²c}ÞžcT®ÆîíM“<¸Ì>¾¼È³×“ôàuPð¥«ÿݺo&‰áÕiûùêžk{—È”¿ëAÜ®AÅsf¨.УPZ÷i槮ÕÜ#êÔP¢ÿ°‰J˜Dò×Kž ÃÈhè³³(}#éÀÛÄú°NÁ¯ãèªÏÃà°2Ï%ë¹t¼‰~ÃÙù0_ä]t ¸äÏ×V°q½Õ“Æ~f=9‡º£ì¹Ê±>oÇUô–±Þc)v¶ -“»mÒ4mæ Ü¡mÈ„êá+ŽÆ¾zÚ`(R^!Z•Ê³’\M‚JÇd\Y¼ŠC?R4 [½Þ¢—ôÈu;§<îHŒz)¤ €zNâþE-±0jêmÿiU¼'’fJÐmGgrí–Ï»<Ì8PIÑÜXiì<“{ÈU® 7!!EG„27œSuS¾»‹‡º¼@@Ý%’R!;5¾ª®&‡»ÔûÁïWP¡²óUoè_M7mܽ]1Å’õ‰Þü -¸š|Õ{d·u¬Ï›Q¾>|VB¥l£ñç%.ëA¹îmÔR™lGiÒÐ^Æô °ë™Q›‹Ñ¿!ÿWJÆö£Ad=¤Ú|ÞêHQ¬JÖ껫ía,ie¡#—»ho”¬7Z‹šòó£ð®VtÏxºÍk¬êáXÔÑ£X -²qŒñhj•’:Ô÷&¦g"ŽÕìóv #"ñ,¹çW<£_Oò:‡ôÛ1_vJoôÿÂcz"Ô¸qz<êh?Öåí‹è=S)õkËj’ÑXäÃU^foR^*z³þ¬ÏõdâÝx¬ËÛÙ†¬ïdN|öx¸_ ÚŽ3'uÛj.xï—ãj>Ôåͨ™½>fçõõ3¥ÏÌÆš×TrˆÖXòîÁýí¦¯up¯‹‡<=HP±óúðí v¨ËÛ;¶g{W±ïCf, k=ÁËïTýe{­ŸzmãÐep¨F5’æ™ÒyxÆ…«¦› ù⤃ïÿøç?ýmÐAd_Z}^çÖ5\îGM°«¼ti#­MœOå~¶ã!øNßØ"žÞVIÂÓS†7Mì`Ÿÿ7FH—@†¬7²˜¦Ú¡¥qÚiCeq}é»…÷Üa)_ãÏ3·Þç`õÓ¯¯ÎÄt}==Ú;vFZï" ³VÛÚa™³dÑGh Ûœg³L7ÏŸgÓÖÛ¬z-³Yó㤷Ë–IõdÛ¢GDëI§ý}Æôõ­„ÙŒ£¾èC4=¾óÏ¡>»¦㱤¯áI_.}ÞüÝ »*Äûµ…uz¯f‡þW‰z?‹»ŠÌ=!:$Ë @5Iºe…ÝÂò œÒ¼t°Â©à˜­¯Lrl-]­Rô$p5áM ,±i9–žëòæ쯦[öws¿›æ~œŸÒEñ¥7Zõ-‰"¡ ýv}í6ÙÑO³©í‚çp• -­ÍCêëùØ’éà(ëíÏ]|¸@`9Šâ""6¥ËµÞß <‡§:·iÆ.p9eAþbœj¸üpi§GK“á¦Ù/aKéS‘.ÿ;´¦yœÒÈæùîfƒ¿Ÿ³Žr5¾ÌsòðìD} ÒÖðîz+?ÒôvF˜ÞÐóIël}…ÿœØ‘îM=ë!ҶuaR6¼Ìî9}ñÁ§ÞJƒäõõH}5fÚÚñbnÿiÚô'=ŽTý«ù^Þ[ö^'×䧓¾·Î™ÙŃd%Î}‹CkØi÷ìêuK¸˜Ÿ?Ö>¸úÝå£fgKiÝô¹-ö­MOÔ_&6ˆè™Â–¬«÷.Ìd}÷¿8š¯RPX;qǶˀKÇ]r ?eKôÃt 9r“'è»b]ui!vÑGÕ…aû[Û˜‡í¼E¼?aZº¦>™TM*ÚPM»H·¡j§JkÙ7„û>¤û“AüÆ·¡Ò¡ó‡]Ó÷¶.ÄÒ ‰{»ÚÚ¤û“ÙnMúNåÞª,]kÓü"ÊbêTºâ Ú¤£™šõËLñÔ>®ÉIw íµ/ƒÝÄ ñþ„ÃòÓI²ÑZ¬ðÒLo$Ò0n'¿ØËBÕu.Ÿàš¦o½èï TH§*òLw“Ò7×–«eõÍN^+i¤§Ž:li p^Úê -9w›õŠ7Ú+èöÖ]c+ê ²žkOAªƒÇšºÕu²BXÓOÇʑ蛘n!K7å§y‘Sº«Šï“¥0˜»ôqrêØù4’sßñ ¯!vyû¸ƒÍ7âýØøš¬¦_Ú$ Yð›ùôÉâ"@a!/0ú4‘›·vtÒǧÛÇÅɬˆ{úõ4…¹Hõˆ QзêÇÌ+œÿ‹6ßãÿ¿N|þ¡½Æu‰ãÑw´ˆ[Wä1’éw°”ƒdýN–Fìn3#°¿CYÚJ³†Çcðš—î~öÖ2t ;eÑá@ŽÝtc A3rƒx¥ÅžÈ|éÞ,ûrImq˜†ôÀ&.Æåà²,W'V«ö3Ž„²¼øŒ@­o.Dš¾ËΗ”‚ÞÔCIž¹ Ç%/îsûüd{JÒÎOÆ™°Ž‰|ú­„¶O`Ô}Æ0TûÆB»[õæ’^¦ãxh9¿9š¾–j:d›‰úeÖRQ¿…Ž4é-úå9ÙÛg”Ó1> -endobj -837 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb294-6) ->> -endobj -838 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb294-7) ->> -endobj -839 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb294-8) ->> -endobj -840 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb294-9) ->> -endobj -841 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb294-10) ->> -endobj -842 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb294-11) ->> -endobj -843 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb294-12) ->> -endobj -844 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb294-13) ->> -endobj -845 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb294-14) ->> -endobj -846 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb294-15) ->> -endobj -847 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb294-16) ->> -endobj -848 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb294-17) ->> -endobj -849 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb294-18) ->> -endobj -850 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb294-19) ->> -endobj -851 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 454.020714 101.433071 435.270714 ] -/BS << -/W 0 ->> -/Dest (cb295-1) ->> -endobj -852 0 obj -<< -/Filter /FlateDecode -/Length 3347 ->> -stream -xœí[K$· ¾÷¯è³•%‘z†’ ÄÀ>9ìv<6‚Þëò÷óQU’Xݜٙ]9xvv¦K¤E~¤Têpöøyð§rpµ¶’ù|ùpúxò®$½Æ¡ÐÙßã·¤sð\\ --×tþ *®´K>_OøsSÔÕ¯§¾:ýûØ&œ£k-Çοü(,&Wj)>¬–d°Rr¥Ä” VSlÍbAŒT8ò=«@ n5Y,ˆÁ1VƒU!q tÏj#–À b„–ŠÁ -r„è½Éƒ ž™¢Á Ée(=[¼\®-6“WÀ %Z<"— ‡,^s9çT-C–Ô|°x ²¤Àì-daªÅâeÈB™¼Éƒ,±²x²Dh§¼ -Y`]ÕäAŸ}°x\j•Ùä5ð|-õž}r©y‹‚K°M2yð(,R./B–ìS3y%‘ ãÇâ'•Z oâÝä)X¼Yb,lò ßB€˜!‹/Ù‚x\²0 –â¸E¶@ Vr\¹Z(ks\ -Y0[rœ[±p€|/ ÈC–ÄÙB -…á¹&²Pc (BT°°€²D& hË° p˾ xË>d € 1½…Ä¥šÙÂ䔚…Ì%`§…Ð…» ¨K)G °K\³…Ü%ÞÂɃ,0` €ŽŽB% ¸KÁ ¸ 7Š`Uì0[X[q±VoawcõlawcA0yÅÁÉÂîÆ„)X‘¸“0p7rÌÀPðÇ›Ùp’ìt²`-,@¼/’…èÌE¤%$ành†O­\¨-[X€»¡bùL^s¡@ã†ßBË.äÜ,,HÀ]*²° wŸÕÂ`¼CD$àn Dj‹Ü ±y pyC²°æà ß\ð0]“לoµZX€»€¥ha¢°óHbòŠƒ! ÆÎ#µ° wÞ›…¸ë‘…¸ëiɃ,T£…ÈkÀƒÛ>dÕù¤°xÀ]°Â¸ë}id஄M rB¬@@‡,8ZX+»VJ¶° 7R>“—ÊÉ‚âAH€,“Y¸°…H¬]£V-,(²PŒ”Y2XX€ŒÀµ€ìÛÜ@˜‹¹?`È‚LÑ ™«Ù‚’2öcЪÅËÑÕ¸6y¼P,,(…Bq°°{)Wáôà¤K@ ð[XPd«XX€PäjDfnñdA’ma2]ðB°° FÈ‚lЂ+v™Ù[X€½$TmñØË>³ZXP9;I-,€›`§ ¤4y0aaÖG6ÛÉÂl~°ÙÄ6ÇäAJlaA­Q¶›Õ‚Z!KôÑÂئì7‹…ÍCd|4Ÿ¤ÉÆÁóÞ‚*ö›‘-,€_Ê~³ZXÐÈc¿‰-žá·²ì7‹…!KBŠbò ’¦fø-0 ûM “ÅË%6¶° å,ûÍjaÒ ì7‘Ñ›GUö›Å<.î¦d²xÀ]d¦Ù°ßDfkíí=€7äg6³ÊŽ³š'Л[Í#ìE´*æ™øÊÉ<4€˜ØYÖlž ¬‚¼ylà¿ÈÇø¶Ó'}D•a1³™dƒ[0²û¹çXÕè‘jÛùØ/?¾~‡à{þé?§‡¯úIÙøÛH⤀i¬vlXí'(ÛCäµY‘¯ Wd¦7dLº×|Kì§x9)2¶¥@‡ {=Ò±¬®†C^;j+Ùõ¢f¢È×6 Ø‘ÞSZuçpšÈ«ßE^3¹*êšôjiÇÒïåôú£îci RÖÄé_ðÝ8ÐDÖ²´µÖÑe?»D 1Äp\k,µ,¶^"µÊÖøW¥G=_¥uMî&«æXl*º£"w$ËöyÔÅÓñ`C‹|´!MGÖ4¬e$®6TmªÊ†*jß’S^uçpš¨m(›64©kÒk¨¥K¿—WÏý¿x®¤êòï|ÿ` ìA;sl‡|ÔÉzSºu¾Wçû9_–å|ÿðçÓ«kE,ûÕù^ïÕù^è|¦“½,å|Ä!_ïÕñ~gŽ÷ü”ó~¿ÿ‡·'HÖ/Õà³±c®ýu€¼¥/r¤y~ûáôõwúËŸÿö=öoNÿÆûPñð[äXÿ³÷ ã3¥í9—ò·ÿ8¿ýî‘1äMùCºEOžÍÂöÉïö®üþ¤Û­Ëа óŽÏ›Uľ×g¹ c–w²‘ê!Ôoß„o–ðý“÷IОÛ>bÚxio‡ õO¾ˆ4ûó»½ÎàÙŠè<©×g¨ËmëöúŠ—â>î­Ÿwk>ï 9(úÇäØuÅeé$íÏ)n–nÈ&—g²ç˜›Ñw_ïËfO]NÚûÊÛoç)½û‡½ÞéÝZ£.KU6´÷×Û^6›³dîõóVfÕG·Ëx¯¯å°‘GdùÈfs¢»'uηz]÷vA–îŸÛöͽ ¼¿ñÏAc¥—ÃvÚéàW;í€zÏõ%Á´ç#¤DËòi„LËCc6<_¡âXõ¡‰þü~i}÷š£7Ñ4³¤ý9ãį‡ Š¹ïçe‰“ã‹ä¾ï‹€Úƒ•Å?Ç{Ó³âúˆx;ÚÜÌeöwÙ”|Ú›ÅÒe¯;Ú÷zOê¹Üê¢#ÜÃj?Ñ‚”—t¸±ùÎöÏ7:Û½\Ú0¤°i×Ùå7Ðé@ä‡]·v²Ûàßî¯ú]i4VÞ¿C/U½NÚß6<£ÊÓ/$Œ—Oûè÷¯*¦ªuC—rñ˜Ÿ[‘w¥V–*¦įd«iÁW¼lëTI;Ìí3¦¥Ân…÷ãV/ø›ì÷a3K¡Gö+ðÖõl-O|·›˜2¥CÿI-u6‚MÛæ·Íán)?žrïúÝ/Ÿ« Q.‡Ÿ3#Ó“;7g¹kP=qKº²Ãù‘–ýJýÇYk¿eïåâ\ºØ®à«Â^ÿc_ôˆLìòˆEýF¬ïäGa‘¯'¹Í™y´O…¸?úÙ‡ír -$;¯›š”œßW¥4c³‘öi4“7ÕHFâ(‰lD´Ìáž,³÷.ìd¹U^í¥=h¤„j“šF͹ <×É ÉÖüŽøÆ®‹9ùjÕäFÐF¬ÃxÖk£žAºœÌz{oi®T™µêjÚ†æ—(ËÔ©NÅ´‹J4‹|MFùˆ§ñx$K2×ÚWe7ñŽx9!Äð¹ÇÜ×N +|¦§‰¤úäê—½,j©CUpÍA“ûòwƒŠ4©‚<ý!ð4)¹µJkñÍI>*IÓóD¶4¸¬ºJ]¡”i³Z_¡z£¾€î¬=5v *•)ziC<© Gr˜V7ÉaC?+5Ñ!0Ü"§iÊ×yÉ™¦«ÂØÇ`9(sO³ßEÎ;¯š\抹à6åý*›ÄKl|KÓ¯cEŽ~F»5˜".Á -‹¼`ôÚÉÃ[':É—vÆãr2+â^N?Ÿz˜‹”b“{ -r_[g^áü_Ôù¿ÿ:ñù¯ã‹ãÑO´ˆ[dÉäÛ=õ™dù¶Ï N·QÄÈ -ö9ÔU7 k˜q> -endobj -854 0 obj -<< -/Filter /FlateDecode -/Length 3190 ->> -stream -xœí[M‹$¹½×¯¨óÂh%Eè –=lãƒö`|è)Ïìbj ³>øïûEVê#«^Ï×|Ø2_(¥P(â)¤T…³Çï«€Uƒ«µ•¬çËûÓ‡“w%yül%7›øGü•t^‹K¡åšÎï âJK±äóõ„w·kñëé§oNÿ>>Îѵ–cHç_ßšHÅ•ZŠODÔ òAˆ(%WJL™ˆrpèbkL5RѨ¢5´ÕÄDPCc¬DT¡†h ò(jP#– L5BK…ˆ‚‡!zOePÄ«J$²\†Ñ3“Åàrm±QY,”Èd". )0Ys9çT™L¡Kj>0Y‚.)¨z&ƒ.*µ0Y†.’ÅSt‰µ“èaFdºÀ»*•AŸ}`²&.µªJe 2_K}”EŸ\ª"žÉBp ¾)T†ˆÂ åBdºdŸ•A—$>2™@Å/“'’”ZI´F‹nñ˜,A—‹RtÁà3ˆºø’ ‚œñã€XŠÓ•‘@¬â´je,ksZŠ0ˆ-9Í­0 ‹xè’43&]‘KeÐEš2.]P€qt‰*Œ l«ðƺUßãߪ™q€p¡¦g\ `\©Y(®Ô8WÀŒ ¤‹p/Œ ¬+)GÆÚ­™q€wEƒg\ à]*k.XœÄ-¬ìBÎqAïb¢Æ ¼ ú¬Œ Àñ3jd\À»A0S3x7Äæ$ð.ò†Ä¸îàƘß\ðp]*kηZ$ð.h)2.À,ì|“PYqp¤À¸dì<òPƼ‹é½1.Èà]ŸDdð®W¤=T]¤FÆÈk CØ’˜F²ê|S0x׌0“w½/qAïڴɸ be\²àȸ Wu­”̸ 7R>*Ë÷‰qAñ(‹Ê ‹e\€ÄÚ5i•qA‰ÐEbd\P"tAÊÀ¸kÙ7]@¸ ](tA¦È¸LæjSe\PRÆz Ve²]- k*«…¸ u˜Šã¬¥\EÐ3.Á;K—@ È‚2.( º T`*r5"3g²]d3.@¦ YŒ j„.ÈÔX±ÊÌžqfh¬%aj&SoëÌʸ jv–*2.@˜`¥ ¦¤2èš`\€ñ±Åvb\€Å›XæPt‘¤Œ j¶Ü¬Œ j….ÑGÆðM[oÆÍCd|Œ šÏÎÒdÆ qȼg\ÐBÅz3*ãÄ¥­7+ã‚&ëM,ñHÜ6ɶÞ,Œ šB—„…Ê  ’¦F✄õ&؉É2t‰M´œm½Y ÍÀz=ݨ¶Þ,t»¼›˜‰ÉÀ»ÈL3ãÜ`½‰Ì–­í=ˆ7äg\XmÅYéŽõ&Ì­tËÀƒ{1[ºgàA¾¶F¢›P+˚鮦Uƒ§Ûü‹|L8á¶û´nQe¸CÁœY‹u²!,ÙýXsÌbòL±Ûþد?Ÿ¾}Âä{þù?§wßl;eý×|#Y‚¦1Ú±a´ßŸ`,‘×æ¾®ÈLï`M÷² ×|n»x9-0–¥`‡`{=âX Ö _šC^ÛK/º-èeéÉ_OX,`Ez§4ËŽæVPg½ž=¹.èìôljZ‡Ù÷rú ýa­cZ¤R0­YÐ! p¾à7çÀ#6–¥Í±˜]ö½K”0G DZÆPÛ`¯C´Œ2kÿºØqíïbõÞ\vécá>TÖŠŠ1ܶås/‹«ðàC>úЊ#kêÞ2š 6¯ªÜ‡êâC¥ïá”gÙÑÜ -®>”© tvz65­Ãì{y‰ÜÿKäZªn?çÇ Â²ëŒ>¼çùlì¶Øþ%ø^‚ïw|Ù~Êùñâ ‚oUxéñjˆƒg¿ßKð½ß ²/K9Ÿ È—À{ ¼ßYà}~Êù¸ÞÿÃë4ÛÕ೩S­Ûë{K_lKóüúýéÛþô—?ÿíG¬_¿;ýý;ïCÅ_¸}zñ>â3½ÁÖžj×¥Œë þþÞþý?ίx¦={k>Ú³&¢âTgïË­«®W™â­‰­é<ï­ù”n÷úîÖ\høqèÕ¼Åú؉f;4c¶w·Qjo=Ôï_…ïz'—k][½á)Ü>µí~¶?cÚh7ÒeïÞÊt\¤k½?Óº÷zu¹Îó¹ÑFÞe—¥ÜÓô^ï{Ï4Ý´Ü5܆órÓ&áSžv¼îÃýf×,?õíMs}[Ò“;ënÏ«Ï1_t{·[0Ýžóoìãњ݂›>xÆ_v«åÝ\ù`'„²&ÃGeÝÍ7ë­Ó'üó±ž,¶MnG1ëÙFói×-mú}D‰þ9=ü>eÝCr‰¢›ÉÉ^cn¤îÍs[©½ñ¦ï¦·Õÿfñò·»-Òkw».Ó^Ûx–½ÞÝ~]ÿËLÿî?[e×!Ïú·zd/÷àǵ«Þ÷=È\£rio7ìãõÇ×û–ÚvÖÐc°÷W5…ºlÇî»uŸQäãzdóvoýq«4¿ôUÚÁIÄ{„'¶bïË›[Ùp»‡4‰ÏèAö!4³)näØö2i1U¾¹G†w°ÑÜp7ãæ^í†ýʯóúÞE¶öžv—»ì®_ïêOwCöæH[RïÃÃP~8åí½YØÎNø\]ˆv¸òœ¼iï¬Ïö®®zÑЇ.lÆçgžÜŽ¤~¥öSªÞ®ØKËÛÖåf/ÿaô÷`uu¹óøv¢ÌopýfÂד(T«ÎÉà°û¥uisKÇ.§ –¹Ü•”äü~9' Iý2¦Þ{>õÇìMB2ö;‹ƒ[ÃÄ“9<ÂÖ{ïÂÛ©Ìêd¿ËCä½m€³B]ËÎ -烟„ƒøL _š›pÔ®O°·øÃty‚CõÕtaþ^6–e8?^™õþNôJQ; Ö«êÖÅÌÑ1Y”jM½äãD KmÀW¾W­Ì!_FÍÞ¨ßÀÚIG¹ÖËèr¢åöÚÒ©2JÕùhë–ŸªLW—: O°Ë’¤;f¾Äˆ—~y„mÎcíëâ7ñIJCG$Nx[Î<–N“+|ê®·‚²Ô;à꧿LË™ÝTÍŽÙûHû£Š4Pcží"èp);S0ïf#›>iÅó`eZfÙÅ\¡”ᳫ½°Ø!åtGéa±º˜lÁ±Äö£ê<Èã‡áu6 -ëö\¹‚¾+æ&œ†+_ðÔ3P…³÷ÆrXÜ=z'œw^W¸Œv@dè;ê]|¾ƒ—mbÓ{Ø\¿öF&ý˜ífc 8)LxÒèuƒ{´v²Cïýr›q/§_NÛ4%Åf^ì¼ãšy…óQæüýë¤ç¿ö °Û<ý`‹8¸õ¯3™Ž¯Ÿ ÛiùŽ°YÀ¨ íO8ÔY6uoóxŒ _ë|ÜObï%Ã`Ð0¤+˜¦ 8×Aˤ€—;+ŽDæ·ŽÍœ°o·Ò;‡fÄŽft­p³v·uÊdNÉç¥JÜYÑ9¥Cªqy7k¥¸]lS³ƒšæk#Ãô¦¿®C!ÖÖr³¤v+S¦sš]«îØÖ‹¥R`YoÖŠ5¬¶ƒ>D鯯ß<î»Ê¹G€¡ƒÚz]ÑÑ1CáqT¶Û|¼sŽœí¦ï´Ð¨K–Ø“‘äÌИ©…nV]Ð:Û ¤µQíØ¡Ñ>Q ºæ;6j,]§¼?ÖͲõ¸R+•‘SLô—ÕÒi•aÄ´ŽJ~(kõöodK2 ´ºs™½ó³äÈ%+Áòúô@u–”Ž=¶­Ç¶ûl{^º¡S‘<3S?K%¿Šÿ¾&vè¶2+Õ·Ý‘Pç‘4LÔŠDÙ¾eèkÎൃâEë²]òÅÏ|í¨µ% ‡S_‚}_”«p… }Z°TÕ¿KÊkžvéË ðìÙ>ú l_ÚÞl›Dã¶ØRѾ(–ôVûZCñϼ°m Ûý”ÃÊöû?OâË- -endstream -endobj -855 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 854 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -856 0 obj -<< -/Filter /FlateDecode -/Length 2995 ->> -stream -xœí[Ë®·ÝÏWÌÚ€i’U|A’ Ä€/‚,äFä,òû9ÕÝ|ô̹²$/¼ðÕhîmžb“ÅbÕa‘Ý7\=>_ü¨\­­d½ÞÞ]Þ_¼+ÉãßVãTØÄßà[Ò5x-.…–kº¾3@Š+-Å’¯÷ ~<×ê÷Ë·_\þs¾'\£k-Ç®?o"Wj)>QƒÈ!¢”\)1e"ÊÁaˆ­1ÔHE£>‹ -ÔÐVA ±Q…¢%ȳ¨AX‚2Ô-" -z„è=•A¯*‘ÈBrFÏLƒËµÅFe²P"“‰¸\4¤ÀdÍåœSe2….©ùÀd º¤ ê™ º¨ÔÂdºHOeÐ%Ö"LV K„u‘UèïªT]|öÉš¸Ôª*•5È|-õY}r©Šx& Á%ø¦P" -“” ‘Eè’}jT]’øÈd]&O$)µ’hÝâ%0Y‚.1¥2è‚Ég 3tñ%3 -@9ã%Ʊ§-*#XÅiÕÊX Öæ´a4[rš[a< >@#ñÐ%ifL º("—Ê ‹4e\ º ãèUØVá-Œ t«¾Æ¾U2ãáBMϸ@À¸R³2.P.\©1.p®€;Há^XWRŽŒ ´+Z3ãïŠϸ@Á»"¸“Ê  ˜qØÑI¨Â¸@Á»|a\ à]„Qd\€YuðÃ̸¾âb­žq‚wcõʸ@Á»±`  ²âƒÂ¸@Á»1alåïÆä#ãïF™qÅ<ÍÀ» ÄÓè‚e\€õ²(Œ И‹HK$ðnhXþHLƒ­\¨-3.HàÝP1}TÖ\(°8‰[XÙ…œã‚ÞÅB%Œ xôY€ãVÔȸ wƒ`¥f2ðnˆÍ3.Hà]ä ‰qÜÁ$Œ °¾¹àáºTÖœoµ2.Hà]ÐRd\€UØù -&¡²âàHqÈØyä¡Œ 2xË{c\Á»>‰0.Èà]¯H{¨ ºHŒ ×@†°%1dÕù¦`2ð®˜a&ïz_ã‚ Þµe“qA.Äʸ -:dÁ‘qA®êZ)™qAn2¤|T–ʉqAñ(‹Ê ‹e\€ÄÚ5i•qA‰ÐEbd\P"tAÊÀ¸kÙ7Ý@¸ Ý(tA¦È¸LæjSe\PRÆ~ Ve²]- k*«…¸ uXŠãì¥\EÐ3.Á;K—@ È‚2.( º T`)r5"3g²]d3.@¦ YŒ j„.ÈÔX±ËÌžqVhì%aj&SoûÌʸ jv–*2.@˜`§ ¦¤2èš`\€ù±Ívb\€Í6›ØæPt‘¤Œ j¶Ý¬Œ j….ÑGÆðMÛoÆÍCd|Œ šÏÎÒdÆ qȼg\ÐBÅ~3*ãÄ¥í7+ã‚&ûMlñHÜ6ɶß,Œ šB—„…Ê  ’¦F✄ý&؉É2t‰M´œm¿Y ÍÀ~==¨¶ß,ô¸¼›˜‰ÉÀ»ÈL3ã°ßDfËööÄ› -ò3.¬¶ã¬ôÄÀƒzÖVzdàÁ½X­ -=3ð _Û#ÑC¨‰eÍôÔË*„ÁÓcþE>¦Oœ°Ÿ>­GTîP°fÖbƒl Ev?ö³š¼Pm?ûùÇËWo±ø^üïå‡/¶“²þ1ßH¤ iÌvl˜íwÛCäµyï€+2ÓX@Ó½.È5?‚Û©ÅÞ. ,Æ~0<ö~Âm§X7|é{^{ÑmAoËHø~Áf;Ò8¥Ywt·€S·ÛeÂs$÷ƒž]Më0ûÞ.ßb¢ß¯mLk‚T -–5\"©AÞŠ•ÅoÎ[l.K›s°ºg—¨aŽÎs©¶É^§h™eÖÿ}±ã:ÞÅê+¼¹ì2ÆÂ}¨¬ c¸3lÛç^WOàɇ&|ö¡˨oÝ "–ùPä>ª¨ý§<ëŽîðäC™úÐ@ç gWÓ:̾·×ÈýM"×fÚþ]Ÿ/Ëž¬3ÆðŽä‹A¶Úmµýkð½ßï,ø²ý+×ç‹O¾uˈCœ=û5ø^ƒï5ø>1øh}ZÊùB@¾ÞkàýÎïãSÎçýþŸÞ\ ÙöR ~7uªu{`Oé‹i^ß¼»|õõ_þö×|ƒýã›.ÿüƒ÷¡âðmv,ÿØsêwøÖ½œìÀL:½®î2} 90½ák÷Õ?þëúæëô°§éCë:eÜòý~{’½lÍ>ª¤è.…½‹ÑmÜUì÷(ꦴãÉ@ l¸ÔÝõ—½™“*o÷ß›…ÞNëlª¥_yÌá¡ËmÄ6"³À‡Ë‚¿=š¿íêl×Ǥìݪú3¶µuû€*âÛ£*í0Z:º±æ¾Û'ÄšÆ×]•M¥¸c!ì]ýùÍqþ°½˜å3vØ_S¨ËÙÕq´ñU>|úANºŽÞŸÏE!/šå+š´·Ì¤^# ÒŠ̲˜"Â/£Mþa–ˆ™Šºd‡ŸÆcÆÌ~wÌPw’ï§ò‡#å9k6 †GÝ}:åéßÛ×϶»Għð{h?-Ó—Ï4Æúž¦òý%oÂö ÙçêB´7Ñ®Y±DgW{@!úÔ…ÍØáúÂÛû{ïG­ã•>oOùí Ïþ¾ßR8ê¿ß&ýÝ%À‘Íþö eýÆop½0áûÅ^¿R«ºø‘°Ç¥mièv»1š¨)Éùã²Æ%õ˘úhìþÔo³cqõ½’lÃ{ÇÄ9<Ã6zïÂÛ+lÕÉQÊC$ m€³A]ëÎç¿ñ ˜žA¿t7á¨]Ÿ`<‡éò‡ê«éÂ2ý½n,Ët~¼)ûx'z§¨½8Ó›êÖ~`²(ÕšzÍ1!:æ‰@–<_úÞ´b2‡|™5{ü¸ƒµ;’Žz­×#ÐíBë­¥1SeÔªóÖÖ-?U™®.už`›´Ø;¹KŒxé—g8Û ô¹öuñ›ø"GÓ‰Þr¿çÚir…OÝõVP–v\ýô—‰"÷;L…Ðì˜=¼±Ÿ;U¤ólA‡KÙØYšXløl¤σu”Y h™us…R†Ï®öBDêéŽÚÃb't1Ù‚c?âGÓyÇÃëlÖí3¸r}WÝM8 W¾Ÿà©g¡ -gïå°¸{íN8p3ìiúÐw´»ø|oÛ¦°¹~íL8ú±ÚÍÎp*R˜ð¤Ñû÷hìdo÷ËdlŽ]~ºlË\”›H°—ÃÖÌ+\ÿ‡:_ãûï‹^ÿÞO öu<úÁqpë ^W2{•¸~$l¯wp„ÍF]h¡κ©{ÃXÇc\øZçí~{¯ƒ†!]Á4m¸Àq¸®íN¦eÒÀÛƒG"ókçf.Ø{QúàÐØsì® -ëpP¬S&sI¾.M¢dUç’©ÆåFÖFQ\:lS³“šæk'Ãô¦¿®S!Ö×RXÒ+Ê”é\fצ;¶b©‚XÖÂÚ°†µÓvÒ‡(ýÙáõ«çýP9÷0tPû@ï+:fè <ŽÊVÌçâXsäj…¼ÓB£-YbOF’3Cc¦z*¬>º uÞvTHk§Ú±S§}¡tÌwl´XºNù¸­›eq¥V*#§˜èO«¥Ó:+Èi•üT×Úí+ÞÈ–d$@iuç2GçgÍ‘KV‚åõîê¬){î[Ï}÷Õöº C§"yf¦~Þ–žj~ÿ}NìÐ38V«¶Ó‘Pçû;X¨5‰²ýI–¯9ƒCÔÞª-Z—ã’O¾çsßY{Ât8ñ%Ø×qî´_,Mõ?¼ã·5O‡ôixñE(zb¶¿®ØŽZ·C¢Q,¶U´?¦Kz«½^ü '¨v Tì÷éeûüÏk5f -endstream -endobj -857 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 856 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -858 0 obj -<< -/Filter /FlateDecode -/Length 4328 ->> -stream -xœí\K$· ¾÷¯è³•õ~†’ ÄÀ>9̶=6‚Ùëò÷óQU”XUìžîÝ5rðî`vJ¤J¢(ò#Åî*w¶øyãð_ÎÔÚJŽçËûÓ‡“5%Yüë=6Îþ¿%Å$×rMç÷DÅ”–|Éç—þÛ5e÷—Ó_þ½½Ç½i-{—οýL¬L©¥Ø¤°XÖ…•’)ŧ¬°²3Xbk b¤}<² -Ĉ­&1¢÷UaUˆbqáÈjÃ5Äp-…å,äpÞZ•AlŒÁ+<—L†Ò³ÆóÎäÚ|Sy•×Œ+иâ·Ð²q97 p*hX€»€Ïªa0Þ ¢z p×DjÜu¾Y pyCÒ°æ` ߌ³0]•×ŒmµjX€»€%¯a¢°±H¢òŠ!9 ÆÆ"Õ° wÞ›†¸kSdà®H{Td ÕkX€¼<¸­âÓHVõ@ -ܵ;¬ñ€»Ö–¦aAîRØÔ° "øªa4È‚½†¹FÓJÉäfÁCʧò²A;iXP, ¥ò K,QÃ$Ö¦…V5,(²ï5,(² eаiÙ·z<€,0õ|! 2E €d¦¶5,()ã<­j¼ìM-€k•WÁsEÂR¢A(và,e*œ^ü¡tIÈž‹”Yà* ™ê‘™k<YdkX€L<ç4,¨² Ô° úŠSf¶ Bã, Uk¼héœY5,¨1J5,€›à¤ ¤Ty0¡aö‡ÛIÃ~pØÄ1GåA–¢†µz:nV j…,Þz `›tÞ,4 YñiXÐl6”&kXÏZ š«8oú¨aü’ΛUÂ,Λ8â)~ÛB¦ófÑ° EÈ’¢¨<È‚¤©)~ LÂyè¤ñ2dñ-jXÐr¦ófÕ°iΛÈèÕ’@¥ófQËÀÝÔ€L¸‹Ì4kX€ΛÈlµ³½ð¦‚üLgV:qVµb`½ ±U-X`/¢UQkàKg$µh1q²¬Y­ ¬‚é¬Z6°À_äcñ€ KõI–¨2Ì¡ fÖB‹lp‹ˆì~œ9f·p¥ÛRûí—Ó×O¾ç_þszþªWÊø‡l#‘“¦±Û¾a·ßŸ ‘×fA~¹"3Ý‘`šû\óžØ«x9 2²< ƒ#‹}ÙÒqª«.¦sÈ5×ÞB6A½ˆ•òË ‡œHwä”Fß9$æ9î$Ï•¼ê\ôœjjGÓïåô#6úƒcj RÖèV¤»°=g»màÚÊÒæV;—µt‰d‡n»ÕØiÚk¹Cb“µé_„år…Ò%¹[¬XbÑM¨È -Ü–L§gî‹«qcB“¼5!IGòÃÆ2¦s6²º YaB½÷䔧µŒé$QšPVMhPç¢çTS;š~/_÷ÿ⸔©Ó¿óñBY©¹†÷ºC^u2©7©û/Î÷Åùþ`Ηé_9/p>!°\±PÄÖ²¿8ßçûâ|:Ÿêd¥œWò‹ã}q¼?˜ãÝŸrûß½=A²þümÑÄXû§ô!}¡ŠæùíûÓ×ßÿù¯ùû8?¾}>ýãk]ůÃïUå­õøAKný›ð7â÷yáü&¿´ã¿yýK}ó·ÿ<¿ýþävh|gçÍlâ kBÌýÃ}h(çC%±I¾}ã¾™Òô¿q•*¯’ÔEšT–kâuéЦOãI’ä–qúu^®©?¯¤ßK¼&ÆH´ªµï*ǧ¬÷¥ÍJ;ïí4šãi½§²g?–È´Ÿ/­Úä>÷ë²íu9ÆÌB¼žu§G;®º“²±^óq½öý÷ºí:†µ\VkÈ«èYÊìáM1àXæ0ôCc-cNÝOÇqr B5}â8N_ÛÓÜ3ÈwKŽâ¯É±ê$®ö²±¿xˆ"})&ÛèsSÆî¶Ü„=¯cñžÐøï„]–µïª“±Oní«ÈÄ>Ñ=¸¬íþÓûml@ÛÿuœUö”nê´í×Ý×@óÿ´ÊWáq/SöÄkYчõÐåBg—£>ÆÚóq¬±Ö:÷u¯sϹûQ,oÊë(–¦÷øº “(P¥dÂÓŠBç¾Y O^5ÌcÍõ~+1v´[÷³°:œû±¥ <Æ´ÔµÍÿóŒÎ? Ñ´×(ï}—4 -œy{.ðmîûQ³C,iÙÚkcC†é;÷ù014úôÓ1ñ8Îc˜ðº&Ç'câqìžUä½ån­v±È-¶Øÿ¼â»¥Í8%±Ž±dXxž´—SŸûqë>-íƒÄížØç­Lñù¦Žã^³–NäúÊÔY7M'¯-¬ºfÙxù4Ûý~öþ+8ÚÇ -SŸû¹õ=»©§|ÐÓ=úñdSã:7ë‡í Íë‘^&fp\IyŽÏ¹+MG¿·ä#~ûÉnâ0IªóÃvæµ´êùlZõooÆâX5;ëòý²¯tŠòÑNËÉ+ä½ñ™Åîô—Ž>w¿ Þ\óŸ?¯­~¼^z®ðyÖxÄÞßc¼wÛŠb·J®Ûmö²³]+0ZæâlzÍ÷þèØ7núÝMý0ù0WªÀ“qv1âÝ´oÎíû=?ÝÖŸâ:ù8,²ßZ[¡•ûÜÔ™ŽÃnâѵóÜ8ËÜ_o!·Y¹½Ï|»–—ÍÓ„ŽÑ¯ëW“Ê-Ýå#òu»ËÕ§î†ýɼñ†í©ù&_óZœÛÙV¥=aC#_ùñºö6ë[}_xeöÕ6Þֺװ™¨Ÿ „ؾ?ûÙ¦±O\0Mëy²•û¸ÍÑï±íWd>`õ-›”þ*s|¶ Îëµ\»ÓïÍ]Ü\ù‘\ÿ°‡Oô»æ:^ÑÑ1GÎBÇl+±–ĺ_ó8+òúÏ`÷Ì?ülÅâø¤`¿pîŸV?ÞÅòTî¯@[Œ{½¦“ªpƶž—“Õ¶³©ïkÕí}u¥'¬bWs —ë•ä‰³Ò±©Z‹*{íU9Æî‰:P²V©‹ß땪&^TtäúFÏžB2Þ¬ž”œŒÿ Õ“ã8UOªu×ä°s-W=9ŽýhEÙ߉pŸµú|ÅGn#õMû½ˆ#|GË ng#v›9ì3×(ª®ÈõdÚ›åÌBd±| - ô]vV¬n­ïPÁ¹©¿¸×ßð7ÞÎtv'ñ÷¶•—ÓŸbÃëò¢úÖ#å;!KZ"'Wäø+Ñuâ˜}=ú=P鯕žƒ~5*¬»” -ý=uìG>“[,gíÿNۚݮ[§üÄaÿ¹¡°âÏSãnÐnþ (}ç1”ný³úߥcs=sÞ}=zx’ü”ç‘Ïåöõß+Ÿ÷©µZQã‘ã¾ÏúZÙ¯U¢=Ÿ ùz£ÏaëlOBæ']ï{ä|$šmjùU]º­®ö{#u&ëáé馮Ú^WÉôÀþÊs÷&Z‰3Eß Qϸ×æ5ÿôv}ê ¿ÅžS¢'üp©â•õ†;ºÜ~æAy¼eýø4Äwo¥þ1$½Z¸í£‰­ÐÓXÚaz©ñçÅM$2 ñ‘½C=›àŒX#(>/ê'º—-Ôy½O¸ú|kÙ¾o›p£1>—…<›Þ–õ-k8lå‡SîOºþt¹ÍÕ8O¯ŸÁAÜPT+íLO3V¢ã­s]Ùî|åÎþÒž£×úKöÓcËK~Dcíÿ¡oú{Dçh2ÇãþÎ ÛÉÕsc’_NôΕ¨cœAý2×K;Æ¢„iG»œ\ /wíz†dìz9‰øÒ'^ ÝŸø6z.ZîúÓ`ËÄ -wfw$Óê­q+™Þ[SMX[yˆ’2ËL_hFÙw8o|•ì‚Ät$Z1Ý$ûÈò8zÎy¨.Oâ]ªÎ‰íç¾8bú»ˆ—¦ {ê‹J¥·eðP¬]o-¡Ú &î96$Ž}RHôí¿ñå¡#6sðÅ®Ñ3Ç ±²!Åѯq?…t9©ýÖÑÒØ©2zÕykcÍOQ¦©‡:¯Ð."Ù.ô".á#6ðå–L¹ÕØk[…ÝøñrBˆ‰rÿÆç±wšXa›ž$1î W;íeRKeUÁ5™FOlÒÿ T¤A%äé.“¢§®gkNB¾9È[%Iz¨5 ¸Xf_¡.WÊ°Y©/W­ÒŸ@wôÛP…ʽ4@j€Ç–ì†Õ 2Aëg`¥$ZÓMr¦ü²!O9ÓpU;O–0÷4Æä<°óE’ËØqGÐyǸÂæ™xé-îÉdú•'™doG´›“ â  0ÉF_:™½u ½Œ/§“i÷rúõÔÃœÉ7z& 8z#ŒÌ¼Üù¿èó=~ÿuŠç¿ñ3K÷v …غ!ËHFï«w’é}bLn#ˆ> -ØŸdWgßÄÖ0â¸÷¯ã¼ÝN`çžn ¨\ILS‡‚ì‡éz„fÒ âe§Å‘È|êÞÌ€½4/Ózx¥BC.Í:ya†ä³-ê:C:¸Ñ‹у¢)&lS²˜dr’¡z’?Ê­4—hˆ4‚šaòâ ³rh¦õUˆ.HƒlÈ£““¶<ŠÐí^Ÿ¼ï«È™=€¨ÚõERLj:O§†ÞÌÛæˆ9áL vÞ©¡1V¾F’3]c¦qÓ6*¨uÞ¶vHrÒȴͤ(\€f™6F,,S^ocµôWUKeä“ú«Ôt’»2”˜ä®äC_—#ÞÈ–ÂH€’4ç2WggÏ‘KV…–å݃gÏÀ´ãÜq;7GÛ³XFœ‚ä™™Úy[:ôü(üûßQŸ¼ÑzñO¯Ž¸:_Ú@] >ô÷°Úš30$Ò«´J¬¢\òð=ûÎ9¶ÃÄlqôF]]„0Úë 1¿mW¿­YuI)àêÛOÔç¤\¥bÀª‰F“>¢ôôÕ@Io¥¿{幩¸Tû]Ú”PúÏÿò´°æ -endstream -endobj -859 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 858 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -860 0 obj -<< -/Title (RKE2 CIS v1.6 Benchmark - Self-Assessment Guide - Rancher v2.6) -/Dest [ 6 0 R /XYZ 85.009843 502.818898 0 ] -/Count 142 -/First 861 0 R -/Last 998 0 R -/Parent 1003 0 R ->> -endobj -861 0 obj -<< -/Title (CIS v1.6 Kubernetes Benchmark - Rancher v2.6 RKE2 with Kubernetes v1.21 up to v1.23) -/Dest [ 306 0 R /XYZ 84.259843 784.970079 0 ] -/Count 2 -/First 862 0 R -/Last 863 0 R -/Parent 860 0 R -/Next 864 0 R ->> -endobj -862 0 obj -<< -/Title (Overview) -/Dest [ 306 0 R /XYZ 84.259843 704.822079 0 ] -/Count 0 -/Parent 861 0 R -/Next 863 0 R ->> -endobj -863 0 obj -<< -/Title (Testing controls methodology) -/Dest [ 306 0 R /XYZ 84.259843 169.826079 0 ] -/Count 0 -/Prev 862 0 R -/Parent 861 0 R ->> -endobj -864 0 obj -<< -/Title (Controls) -/Dest [ 313 0 R /XYZ 84.259843 570.381496 0 ] -/Count 0 -/Prev 861 0 R -/Parent 860 0 R -/Next 865 0 R ->> -endobj -865 0 obj -<< -/Title (1.1 Master Node Configuration Files) -/Dest [ 317 0 R /XYZ 41.740157 784.430079 0 ] -/Count 21 -/Prev 864 0 R -/First 866 0 R -/Last 886 0 R -/Parent 860 0 R -/Next 887 0 R ->> -endobj -866 0 obj -<< -/Title (1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 317 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 865 0 R -/Next 867 0 R ->> -endobj -867 0 obj -<< -/Title (1.1.2 Ensure that the API server pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 317 0 R /XYZ 84.259843 352.457453 0 ] -/Count 0 -/Prev 866 0 R -/Parent 865 0 R -/Next 868 0 R ->> -endobj -868 0 obj -<< -/Title (1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 321 0 R /XYZ 84.259843 736.220079 0 ] -/Count 0 -/Prev 867 0 R -/Parent 865 0 R -/Next 869 0 R ->> -endobj -869 0 obj -<< -/Title (1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 321 0 R /XYZ 84.259843 351.131993 0 ] -/Count 0 -/Prev 868 0 R -/Parent 865 0 R -/Next 870 0 R ->> -endobj -870 0 obj -<< -/Title (1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 702.470079 0 ] -/Count 0 -/Prev 869 0 R -/Parent 865 0 R -/Next 871 0 R ->> -endobj -871 0 obj -<< -/Title (1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 325 0 R /XYZ 84.259843 351.329014 0 ] -/Count 0 -/Prev 870 0 R -/Parent 865 0 R -/Next 872 0 R ->> -endobj -872 0 obj -<< -/Title (1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 329 0 R /XYZ 84.259843 736.220079 0 ] -/Count 0 -/Prev 871 0 R -/Parent 865 0 R -/Next 873 0 R ->> -endobj -873 0 obj -<< -/Title (1.1.8 Ensure that the etcd pod specification file ownership is set to root:root \(Automated\)) -/Dest [ 329 0 R /XYZ 84.259843 385.079014 0 ] -/Count 0 -/Prev 872 0 R -/Parent 865 0 R -/Next 874 0 R ->> -endobj -874 0 obj -<< -/Title (1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive \(Manual\)) -/Dest [ 333 0 R /XYZ 84.259843 763.220079 0 ] -/Count 0 -/Prev 873 0 R -/Parent 865 0 R -/Next 875 0 R ->> -endobj -875 0 obj -<< -/Title (1.1.10 Ensure that the Container Network Interface file ownership is set to root:root \(Manual\)) -/Dest [ 333 0 R /XYZ 84.259843 586.473057 0 ] -/Count 0 -/Prev 874 0 R -/Parent 865 0 R -/Next 876 0 R ->> -endobj -876 0 obj -<< -/Title (1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive \(Automated\)) -/Dest [ 333 0 R /XYZ 84.259843 409.726036 0 ] -/Count 0 -/Prev 875 0 R -/Parent 865 0 R -/Next 877 0 R ->> -endobj -877 0 obj -<< -/Title (1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd \(Automated\)) -/Dest [ 338 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 876 0 R -/Parent 865 0 R -/Next 878 0 R ->> -endobj -878 0 obj -<< -/Title (1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 338 0 R /XYZ 84.259843 471.123057 0 ] -/Count 0 -/Prev 877 0 R -/Parent 865 0 R -/Next 879 0 R ->> -endobj -879 0 obj -<< -/Title (1.1.14 Ensure that the admin.conf file ownership is set to root:root \(Automated\)) -/Dest [ 338 0 R /XYZ 84.259843 157.876036 0 ] -/Count 0 -/Prev 878 0 R -/Parent 865 0 R -/Next 880 0 R ->> -endobj -880 0 obj -<< -/Title (1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 342 0 R /XYZ 84.259843 538.576036 0 ] -/Count 0 -/Prev 879 0 R -/Parent 865 0 R -/Next 881 0 R ->> -endobj -881 0 obj -<< -/Title (1.1.16 Ensure that the scheduler.conf file ownership is set to root:root \(Automated\)) -/Dest [ 342 0 R /XYZ 84.259843 221.381993 0 ] -/Count 0 -/Prev 880 0 R -/Parent 865 0 R -/Next 882 0 R ->> -endobj -882 0 obj -<< -/Title (1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 346 0 R /XYZ 84.259843 622.576036 0 ] -/Count 0 -/Prev 881 0 R -/Parent 865 0 R -/Next 883 0 R ->> -endobj -883 0 obj -<< -/Title (1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root \(Automated\)) -/Dest [ 346 0 R /XYZ 84.259843 290.381993 0 ] -/Count 0 -/Prev 882 0 R -/Parent 865 0 R -/Next 884 0 R ->> -endobj -884 0 obj -<< -/Title (1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root \(Automated\)) -/Dest [ 351 0 R /XYZ 84.259843 675.470079 0 ] -/Count 0 -/Prev 883 0 R -/Parent 865 0 R -/Next 885 0 R ->> -endobj -885 0 obj -<< -/Title (1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive \(Manual\)) -/Dest [ 351 0 R /XYZ 84.259843 362.223057 0 ] -/Count 0 -/Prev 884 0 R -/Parent 865 0 R -/Next 886 0 R ->> -endobj -886 0 obj -<< -/Title (1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 \(Manual\)) -/Dest [ 398 0 R /XYZ 84.259843 324.635606 0 ] -/Count 0 -/Prev 885 0 R -/Parent 865 0 R ->> -endobj -887 0 obj -<< -/Title (1.2 API Server) -/Dest [ 486 0 R /XYZ 41.740157 784.430079 0 ] -/Count 35 -/Prev 865 0 R -/First 888 0 R -/Last 922 0 R -/Parent 860 0 R -/Next 923 0 R ->> -endobj -888 0 obj -<< -/Title (1.2.1 Ensure that the --anonymous-auth argument is set to false \(Manual\)) -/Dest [ 486 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 887 0 R -/Next 889 0 R ->> -endobj -889 0 obj -<< -/Title (1.2.2 Ensure that the --basic-auth-file argument is not set \(Automated\)) -/Dest [ 486 0 R /XYZ 84.259843 528.904475 0 ] -/Count 0 -/Prev 888 0 R -/Parent 887 0 R -/Next 890 0 R ->> -endobj -890 0 obj -<< -/Title (1.2.3 Ensure that the --token-auth-file parameter is not set \(Automated\)) -/Dest [ 492 0 R /XYZ 84.259843 427.220079 0 ] -/Count 0 -/Prev 889 0 R -/Parent 887 0 R -/Next 891 0 R ->> -endobj -891 0 obj -<< -/Title (1.2.4 Ensure that the --kubelet-https argument is set to true \(Automated\)) -/Dest [ 497 0 R /XYZ 84.259843 333.470079 0 ] -/Count 0 -/Prev 890 0 R -/Parent 887 0 R -/Next 892 0 R ->> -endobj -892 0 obj -<< -/Title (1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate \(Automated\)) -/Dest [ 502 0 R /XYZ 84.259843 277.220079 0 ] -/Count 0 -/Prev 891 0 R -/Parent 887 0 R -/Next 893 0 R ->> -endobj -893 0 obj -<< -/Title (1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate \(Automated\)) -/Dest [ 507 0 R /XYZ 84.259843 164.720079 0 ] -/Count 0 -/Prev 892 0 R -/Parent 887 0 R -/Next 894 0 R ->> -endobj -894 0 obj -<< -/Title (1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow \(Automated\)) -/Dest [ 514 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 893 0 R -/Parent 887 0 R -/Next 895 0 R ->> -endobj -895 0 obj -<< -/Title (1.2.8 Ensure that the --authorization-mode argument includes Node \(Automated\)) -/Dest [ 519 0 R /XYZ 84.259843 652.220079 0 ] -/Count 0 -/Prev 894 0 R -/Parent 887 0 R -/Next 896 0 R ->> -endobj -896 0 obj -<< -/Title (1.2.9 Ensure that the --authorization-mode argument includes RBAC \(Automated\)) -/Dest [ 524 0 R /XYZ 84.259843 577.220079 0 ] -/Count 0 -/Prev 895 0 R -/Parent 887 0 R -/Next 897 0 R ->> -endobj -897 0 obj -<< -/Title (1.2.10 Ensure that the admission control plugin EventRateLimit is set \(Manual\)) -/Dest [ 529 0 R /XYZ 84.259843 502.220079 0 ] -/Count 0 -/Prev 896 0 R -/Parent 887 0 R -/Next 898 0 R ->> -endobj -898 0 obj -<< -/Title (1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set \(Automated\)) -/Dest [ 529 0 R /XYZ 84.259843 280.473057 0 ] -/Count 0 -/Prev 897 0 R -/Parent 887 0 R -/Next 899 0 R ->> -endobj -899 0 obj -<< -/Title (1.2.12 Ensure that the admission control plugin AlwaysPullImages is set \(Manual\)) -/Dest [ 535 0 R /XYZ 84.259843 202.220079 0 ] -/Count 0 -/Prev 898 0 R -/Parent 887 0 R -/Next 900 0 R ->> -endobj -900 0 obj -<< -/Title (1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used \(Manual\)) -/Dest [ 537 0 R /XYZ 84.259843 736.023057 0 ] -/Count 0 -/Prev 899 0 R -/Parent 887 0 R -/Next 901 0 R ->> -endobj -901 0 obj -<< -/Title (1.2.14 Ensure that the admission control plugin ServiceAccount is set \(Automated\)) -/Dest [ 537 0 R /XYZ 84.259843 499.276036 0 ] -/Count 0 -/Prev 900 0 R -/Parent 887 0 R -/Next 902 0 R ->> -endobj -902 0 obj -<< -/Title (1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set \(Automated\)) -/Dest [ 544 0 R /XYZ 84.259843 370.970079 0 ] -/Count 0 -/Prev 901 0 R -/Parent 887 0 R -/Next 903 0 R ->> -endobj -903 0 obj -<< -/Title (1.2.16 Ensure that the admission control plugin PodSecurityPolicy is set \(Automated\)) -/Dest [ 549 0 R /XYZ 84.259843 295.970079 0 ] -/Count 0 -/Prev 902 0 R -/Parent 887 0 R -/Next 904 0 R ->> -endobj -904 0 obj -<< -/Title (1.2.17 Ensure that the admission control plugin NodeRestriction is set \(Automated\)) -/Dest [ 554 0 R /XYZ 84.259843 183.470079 0 ] -/Count 0 -/Prev 903 0 R -/Parent 887 0 R -/Next 905 0 R ->> -endobj -905 0 obj -<< -/Title (1.2.18 Ensure that the --insecure-bind-address argument is not set \(Automated\)) -/Dest [ 561 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 904 0 R -/Parent 887 0 R -/Next 906 0 R ->> -endobj -906 0 obj -<< -/Title (1.2.19 Ensure that the --insecure-port argument is set to 0 \(Automated\)) -/Dest [ 566 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 905 0 R -/Parent 887 0 R -/Next 907 0 R ->> -endobj -907 0 obj -<< -/Title (1.2.20 Ensure that the --secure-port argument is not set to 0 \(Automated\)) -/Dest [ 571 0 R /XYZ 84.259843 689.720079 0 ] -/Count 0 -/Prev 906 0 R -/Parent 887 0 R -/Next 908 0 R ->> -endobj -908 0 obj -<< -/Title (1.2.21 Ensure that the --profiling argument is set to false \(Automated\)) -/Dest [ 576 0 R /XYZ 84.259843 614.720079 0 ] -/Count 0 -/Prev 907 0 R -/Parent 887 0 R -/Next 909 0 R ->> -endobj -909 0 obj -<< -/Title (1.2.22 Ensure that the --audit-log-path argument is set \(Automated\)) -/Dest [ 581 0 R /XYZ 84.259843 558.470079 0 ] -/Count 0 -/Prev 908 0 R -/Parent 887 0 R -/Next 910 0 R ->> -endobj -910 0 obj -<< -/Title (1.2.23 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate \(Automated\)) -/Dest [ 586 0 R /XYZ 84.259843 464.720079 0 ] -/Count 0 -/Prev 909 0 R -/Parent 887 0 R -/Next 911 0 R ->> -endobj -911 0 obj -<< -/Title (1.2.24 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate \(Automated\)) -/Dest [ 591 0 R /XYZ 84.259843 389.720079 0 ] -/Count 0 -/Prev 910 0 R -/Parent 887 0 R -/Next 912 0 R ->> -endobj -912 0 obj -<< -/Title (1.2.25 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate \(Automated\)) -/Dest [ 596 0 R /XYZ 84.259843 295.970079 0 ] -/Count 0 -/Prev 911 0 R -/Parent 887 0 R -/Next 913 0 R ->> -endobj -913 0 obj -<< -/Title (1.2.26 Ensure that the --request-timeout argument is set as appropriate \(Automated\)) -/Dest [ 601 0 R /XYZ 84.259843 239.720079 0 ] -/Count 0 -/Prev 912 0 R -/Parent 887 0 R -/Next 914 0 R ->> -endobj -914 0 obj -<< -/Title (1.2.27 Ensure that the --service-account-lookup argument is set to true \(Automated\)) -/Dest [ 606 0 R /XYZ 84.259843 164.720079 0 ] -/Count 0 -/Prev 913 0 R -/Parent 887 0 R -/Next 915 0 R ->> -endobj -915 0 obj -<< -/Title (1.2.28 Ensure that the --service-account-key-file argument is set as appropriate \(Automated\)) -/Dest [ 613 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 914 0 R -/Parent 887 0 R -/Next 916 0 R ->> -endobj -916 0 obj -<< -/Title (1.2.29 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate \(Automated\)) -/Dest [ 618 0 R /XYZ 84.259843 670.970079 0 ] -/Count 0 -/Prev 915 0 R -/Parent 887 0 R -/Next 917 0 R ->> -endobj -917 0 obj -<< -/Title (1.2.30 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate \(Automated\)) -/Dest [ 623 0 R /XYZ 84.259843 577.220079 0 ] -/Count 0 -/Prev 916 0 R -/Parent 887 0 R -/Next 918 0 R ->> -endobj -918 0 obj -<< -/Title (1.2.31 Ensure that the --client-ca-file argument is set as appropriate \(Automated\)) -/Dest [ 628 0 R /XYZ 84.259843 464.720079 0 ] -/Count 0 -/Prev 917 0 R -/Parent 887 0 R -/Next 919 0 R ->> -endobj -919 0 obj -<< -/Title (1.2.32 Ensure that the --etcd-cafile argument is set as appropriate \(Automated\)) -/Dest [ 633 0 R /XYZ 84.259843 370.970079 0 ] -/Count 0 -/Prev 918 0 R -/Parent 887 0 R -/Next 920 0 R ->> -endobj -920 0 obj -<< -/Title (1.2.33 Ensure that the --encryption-provider-config argument is set as appropriate \(Automated\)) -/Dest [ 638 0 R /XYZ 84.259843 295.970079 0 ] -/Count 0 -/Prev 919 0 R -/Parent 887 0 R -/Next 921 0 R ->> -endobj -921 0 obj -<< -/Title (1.2.34 Ensure that encryption providers are appropriately configured \(Automated\)) -/Dest [ 643 0 R /XYZ 84.259843 220.970079 0 ] -/Count 0 -/Prev 920 0 R -/Parent 887 0 R -/Next 922 0 R ->> -endobj -922 0 obj -<< -/Title (1.2.35 Ensure that the API Server only makes use of Strong Cryptographic Ciphers \(Manual\)) -/Dest [ 645 0 R /XYZ 84.259843 547.576036 0 ] -/Count 0 -/Prev 921 0 R -/Parent 887 0 R ->> -endobj -923 0 obj -<< -/Title (1.3 Controller Manager) -/Dest [ 653 0 R /XYZ 41.740157 784.430079 0 ] -/Count 7 -/Prev 887 0 R -/First 924 0 R -/Last 930 0 R -/Parent 860 0 R -/Next 931 0 R ->> -endobj -924 0 obj -<< -/Title (1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate \(Automated\)) -/Dest [ 653 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 923 0 R -/Next 925 0 R ->> -endobj -925 0 obj -<< -/Title (1.3.2 Ensure that the --profiling argument is set to false \(Automated\)) -/Dest [ 656 0 R /XYZ 84.259843 633.470079 0 ] -/Count 0 -/Prev 924 0 R -/Parent 923 0 R -/Next 926 0 R ->> -endobj -926 0 obj -<< -/Title (1.3.3 Ensure that the --use-service-account-credentials argument is set to true \(Automated\)) -/Dest [ 659 0 R /XYZ 84.259843 558.470079 0 ] -/Count 0 -/Prev 925 0 R -/Parent 923 0 R -/Next 927 0 R ->> -endobj -927 0 obj -<< -/Title (1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate \(Automated\)) -/Dest [ 662 0 R /XYZ 84.259843 483.470079 0 ] -/Count 0 -/Prev 926 0 R -/Parent 923 0 R -/Next 928 0 R ->> -endobj -928 0 obj -<< -/Title (1.3.5 Ensure that the --root-ca-file argument is set as appropriate \(Automated\)) -/Dest [ 665 0 R /XYZ 84.259843 389.720079 0 ] -/Count 0 -/Prev 927 0 R -/Parent 923 0 R -/Next 929 0 R ->> -endobj -929 0 obj -<< -/Title (1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true \(Automated\)) -/Dest [ 668 0 R /XYZ 84.259843 294.470079 0 ] -/Count 0 -/Prev 928 0 R -/Parent 923 0 R -/Next 930 0 R ->> -endobj -930 0 obj -<< -/Title (1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 \(Automated\)) -/Dest [ 668 0 R /XYZ 84.259843 133.670079 0 ] -/Count 0 -/Prev 929 0 R -/Parent 923 0 R ->> -endobj -931 0 obj -<< -/Title (1.4 Scheduler) -/Dest [ 675 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 923 0 R -/First 932 0 R -/Last 933 0 R -/Parent 860 0 R -/Next 934 0 R ->> -endobj -932 0 obj -<< -/Title (1.4.1 Ensure that the --profiling argument is set to false \(Automated\)) -/Dest [ 675 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 931 0 R -/Next 933 0 R ->> -endobj -933 0 obj -<< -/Title (1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 \(Automated\)) -/Dest [ 675 0 R /XYZ 84.259843 313.654475 0 ] -/Count 0 -/Prev 932 0 R -/Parent 931 0 R ->> -endobj -934 0 obj -<< -/Title (2 Etcd Node Configuration Files) -/Dest [ 681 0 R /XYZ 41.740157 784.430079 0 ] -/Count 7 -/Prev 931 0 R -/First 935 0 R -/Last 941 0 R -/Parent 860 0 R -/Next 942 0 R ->> -endobj -935 0 obj -<< -/Title (2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate \(Automated\)) -/Dest [ 681 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 934 0 R -/Next 936 0 R ->> -endobj -936 0 obj -<< -/Title (2.2 Ensure that the --client-cert-auth argument is set to true \(Automated\)) -/Dest [ 681 0 R /XYZ 84.259843 538.851496 0 ] -/Count 0 -/Prev 935 0 R -/Parent 934 0 R -/Next 937 0 R ->> -endobj -937 0 obj -<< -/Title (2.3 Ensure that the --auto-tls argument is not set to true \(Automated\)) -/Dest [ 681 0 R /XYZ 84.259843 408.051496 0 ] -/Count 0 -/Prev 936 0 R -/Parent 934 0 R -/Next 938 0 R ->> -endobj -938 0 obj -<< -/Title (2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate \(Automated\)) -/Dest [ 686 0 R /XYZ 84.259843 708.470079 0 ] -/Count 0 -/Prev 937 0 R -/Parent 934 0 R -/Next 939 0 R ->> -endobj -939 0 obj -<< -/Title (2.5 Ensure that the --peer-client-cert-auth argument is set to true \(Automated\)) -/Dest [ 686 0 R /XYZ 84.259843 547.670079 0 ] -/Count 0 -/Prev 938 0 R -/Parent 934 0 R -/Next 940 0 R ->> -endobj -940 0 obj -<< -/Title (2.6 Ensure that the --peer-auto-tls argument is not set to true \(Automated\)) -/Dest [ 686 0 R /XYZ 84.259843 416.870079 0 ] -/Count 0 -/Prev 939 0 R -/Parent 934 0 R -/Next 941 0 R ->> -endobj -941 0 obj -<< -/Title (2.7 Ensure that a unique Certificate Authority is used for etcd \(Manual\)) -/Dest [ 691 0 R /XYZ 84.259843 652.220079 0 ] -/Count 0 -/Prev 940 0 R -/Parent 934 0 R ->> -endobj -942 0 obj -<< -/Title (3.1 Authentication and Authorization) -/Dest [ 696 0 R /XYZ 41.740157 784.430079 0 ] -/Count 1 -/Prev 934 0 R -/First 943 0 R -/Last 943 0 R -/Parent 860 0 R -/Next 944 0 R ->> -endobj -943 0 obj -<< -/Title (3.1.1 Client certificate authentication should not be used for users \(Manual\)) -/Dest [ 696 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 942 0 R ->> -endobj -944 0 obj -<< -/Title (3.2 Logging) -/Dest [ 698 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 942 0 R -/First 945 0 R -/Last 946 0 R -/Parent 860 0 R -/Next 947 0 R ->> -endobj -945 0 obj -<< -/Title (3.2.1 Ensure that a minimal audit policy is created \(Automated\)) -/Dest [ 698 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 944 0 R -/Next 946 0 R ->> -endobj -946 0 obj -<< -/Title (3.2.2 Ensure that the audit policy covers key security concerns \(Manual\)) -/Dest [ 698 0 R /XYZ 84.259843 418.457453 0 ] -/Count 0 -/Prev 945 0 R -/Parent 944 0 R ->> -endobj -947 0 obj -<< -/Title (4.1 Worker Node Configuration Files) -/Dest [ 701 0 R /XYZ 41.740157 784.430079 0 ] -/Count 10 -/Prev 944 0 R -/First 948 0 R -/Last 957 0 R -/Parent 860 0 R -/Next 958 0 R ->> -endobj -948 0 obj -<< -/Title (4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 701 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 947 0 R -/Next 949 0 R ->> -endobj -949 0 obj -<< -/Title (4.1.2 Ensure that the kubelet service file ownership is set to root:root \(Automated\)) -/Dest [ 701 0 R /XYZ 84.259843 394.260432 0 ] -/Count 0 -/Prev 948 0 R -/Parent 947 0 R -/Next 950 0 R ->> -endobj -950 0 obj -<< -/Title (4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive \(Manual\)) -/Dest [ 701 0 R /XYZ 84.259843 263.460432 0 ] -/Count 0 -/Prev 949 0 R -/Parent 947 0 R -/Next 951 0 R ->> -endobj -951 0 obj -<< -/Title (4.1.4 Ensure that the proxy kubeconfig file ownership is set to root:root \(Manual\)) -/Dest [ 704 0 R /XYZ 84.259843 584.879014 0 ] -/Count 0 -/Prev 950 0 R -/Parent 947 0 R -/Next 952 0 R ->> -endobj -952 0 obj -<< -/Title (4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive \(Automated\)) -/Dest [ 704 0 R /XYZ 84.259843 214.987950 0 ] -/Count 0 -/Prev 951 0 R -/Parent 947 0 R -/Next 953 0 R ->> -endobj -953 0 obj -<< -/Title (4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root \(Automated\)) -/Dest [ 708 0 R /XYZ 84.259843 603.629014 0 ] -/Count 0 -/Prev 952 0 R -/Parent 947 0 R -/Next 954 0 R ->> -endobj -954 0 obj -<< -/Title (4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive \(Manual\)) -/Dest [ 708 0 R /XYZ 84.259843 252.487950 0 ] -/Count 0 -/Prev 953 0 R -/Parent 947 0 R -/Next 955 0 R ->> -endobj -955 0 obj -<< -/Title (4.1.8 Ensure that the client certificate authorities file ownership is set to root:root \(Automated\)) -/Dest [ 713 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 954 0 R -/Parent 947 0 R -/Next 956 0 R ->> -endobj -956 0 obj -<< -/Title (4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive \(Automated\)) -/Dest [ 713 0 R /XYZ 84.259843 463.229014 0 ] -/Count 0 -/Prev 955 0 R -/Parent 947 0 R -/Next 957 0 R ->> -endobj -957 0 obj -<< -/Title (4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root \(Automated\)) -/Dest [ 717 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 956 0 R -/Parent 947 0 R ->> -endobj -958 0 obj -<< -/Title (4.2 Kubelet) -/Dest [ 720 0 R /XYZ 41.740157 784.430079 0 ] -/Count 13 -/Prev 947 0 R -/First 959 0 R -/Last 971 0 R -/Parent 860 0 R -/Next 972 0 R ->> -endobj -959 0 obj -<< -/Title (4.2.1 Ensure that the anonymous-auth argument is set to false \(Automated\)) -/Dest [ 720 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 958 0 R -/Next 960 0 R ->> -endobj -960 0 obj -<< -/Title (4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow \(Automated\)) -/Dest [ 724 0 R /XYZ 84.259843 520.970079 0 ] -/Count 0 -/Prev 959 0 R -/Parent 958 0 R -/Next 961 0 R ->> -endobj -961 0 obj -<< -/Title (4.2.3 Ensure that the --client-ca-file argument is set as appropriate \(Automated\)) -/Dest [ 728 0 R /XYZ 84.259843 670.970079 0 ] -/Count 0 -/Prev 960 0 R -/Parent 958 0 R -/Next 962 0 R ->> -endobj -962 0 obj -<< -/Title (4.2.4 Ensure that the --read-only-port argument is set to 0 \(Automated\)) -/Dest [ 732 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 961 0 R -/Parent 958 0 R -/Next 963 0 R ->> -endobj -963 0 obj -<< -/Title (4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 \(Automated\)) -/Dest [ 736 0 R /XYZ 84.259843 614.720079 0 ] -/Count 0 -/Prev 962 0 R -/Parent 958 0 R -/Next 964 0 R ->> -endobj -964 0 obj -<< -/Title (4.2.6 Ensure that the --protect-kernel-defaults argument is set to true \(Automated\)) -/Dest [ 740 0 R /XYZ 84.259843 389.720079 0 ] -/Count 0 -/Prev 963 0 R -/Parent 958 0 R -/Next 965 0 R ->> -endobj -965 0 obj -<< -/Title (4.2.7 Ensure that the --make-iptables-util-chains argument is set to true \(Automated\)) -/Dest [ 744 0 R /XYZ 84.259843 544.220079 0 ] -/Count 0 -/Prev 964 0 R -/Parent 958 0 R -/Next 966 0 R ->> -endobj -966 0 obj -<< -/Title (4.2.8 Ensure that the --hostname-override argument is not set \(Manual\)) -/Dest [ 748 0 R /XYZ 84.259843 333.470079 0 ] -/Count 0 -/Prev 965 0 R -/Parent 958 0 R -/Next 967 0 R ->> -endobj -967 0 obj -<< -/Title (4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture \(Manual\)) -/Dest [ 751 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 966 0 R -/Parent 958 0 R -/Next 968 0 R ->> -endobj -968 0 obj -<< -/Title (4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate \(Automated\)) -/Dest [ 755 0 R /XYZ 84.259843 614.720079 0 ] -/Count 0 -/Prev 967 0 R -/Parent 958 0 R -/Next 969 0 R ->> -endobj -969 0 obj -<< -/Title (4.2.11 Ensure that the --rotate-certificates argument is not set to false \(Manual\)) -/Dest [ 759 0 R /XYZ 84.259843 389.720079 0 ] -/Count 0 -/Prev 968 0 R -/Parent 958 0 R -/Next 970 0 R ->> -endobj -970 0 obj -<< -/Title (4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true \(Manual\)) -/Dest [ 763 0 R /XYZ 84.259843 187.970079 0 ] -/Count 0 -/Prev 969 0 R -/Parent 958 0 R -/Next 971 0 R ->> -endobj -971 0 obj -<< -/Title (4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers \(Manual\)) -/Dest [ 769 0 R /XYZ 84.259843 745.970079 0 ] -/Count 0 -/Prev 970 0 R -/Parent 958 0 R ->> -endobj -972 0 obj -<< -/Title (5.1 RBAC and Service Accounts) -/Dest [ 775 0 R /XYZ 41.740157 784.430079 0 ] -/Count 6 -/Prev 958 0 R -/First 973 0 R -/Last 978 0 R -/Parent 860 0 R -/Next 979 0 R ->> -endobj -973 0 obj -<< -/Title (5.1.1 Ensure that the cluster-admin role is only used where required \(Manual\)) -/Dest [ 775 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 972 0 R -/Next 974 0 R ->> -endobj -974 0 obj -<< -/Title (5.1.2 Minimize access to secrets \(Manual\)) -/Dest [ 775 0 R /XYZ 84.259843 523.851496 0 ] -/Count 0 -/Prev 973 0 R -/Parent 972 0 R -/Next 975 0 R ->> -endobj -975 0 obj -<< -/Title (5.1.3 Minimize wildcard use in Roles and ClusterRoles \(Manual\)) -/Dest [ 775 0 R /XYZ 84.259843 423.051496 0 ] -/Count 0 -/Prev 974 0 R -/Parent 972 0 R -/Next 976 0 R ->> -endobj -976 0 obj -<< -/Title (5.1.4 Minimize access to create pods \(Manual\)) -/Dest [ 775 0 R /XYZ 84.259843 307.251496 0 ] -/Count 0 -/Prev 975 0 R -/Parent 972 0 R -/Next 977 0 R ->> -endobj -977 0 obj -<< -/Title (5.1.5 Ensure that default service accounts are not actively used. \(Automated\)) -/Dest [ 775 0 R /XYZ 84.259843 206.451496 0 ] -/Count 0 -/Prev 976 0 R -/Parent 972 0 R -/Next 978 0 R ->> -endobj -978 0 obj -<< -/Title (5.1.6 Ensure that Service Account Tokens are only mounted where necessary \(Manual\)) -/Dest [ 802 0 R /XYZ 84.259843 449.446885 0 ] -/Count 0 -/Prev 977 0 R -/Parent 972 0 R ->> -endobj -979 0 obj -<< -/Title (5.2 Pod Security Policies) -/Dest [ 813 0 R /XYZ 41.740157 784.430079 0 ] -/Count 9 -/Prev 972 0 R -/First 980 0 R -/Last 988 0 R -/Parent 860 0 R -/Next 989 0 R ->> -endobj -980 0 obj -<< -/Title (5.2.1 Minimize the admission of privileged containers \(Automated\)) -/Dest [ 813 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 979 0 R -/Next 981 0 R ->> -endobj -981 0 obj -<< -/Title (5.2.2 Minimize the admission of containers wishing to share the host process ID namespace \(Automated\)) -/Dest [ 813 0 R /XYZ 84.259843 388.457453 0 ] -/Count 0 -/Prev 980 0 R -/Parent 979 0 R -/Next 982 0 R ->> -endobj -982 0 obj -<< -/Title (5.2.3 Minimize the admission of containers wishing to share the host IPC namespace \(Automated\)) -/Dest [ 817 0 R /XYZ 84.259843 763.220079 0 ] -/Count 0 -/Prev 981 0 R -/Parent 979 0 R -/Next 983 0 R ->> -endobj -983 0 obj -<< -/Title (5.2.4 Minimize the admission of containers wishing to share the host network namespace \(Automated\)) -/Dest [ 817 0 R /XYZ 84.259843 412.079014 0 ] -/Count 0 -/Prev 982 0 R -/Parent 979 0 R -/Next 984 0 R ->> -endobj -984 0 obj -<< -/Title (5.2.5 Minimize the admission of containers with allowPrivilegeEscalation \(Automated\)) -/Dest [ 821 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Prev 983 0 R -/Parent 979 0 R -/Next 985 0 R ->> -endobj -985 0 obj -<< -/Title (5.2.6 Minimize the admission of root containers \(Automated\)) -/Dest [ 821 0 R /XYZ 84.259843 429.281993 0 ] -/Count 0 -/Prev 984 0 R -/Parent 979 0 R -/Next 986 0 R ->> -endobj -986 0 obj -<< -/Title (5.2.7 Minimize the admission of containers with the NET_RAW capability \(Manual\)) -/Dest [ 825 0 R /XYZ 84.259843 763.220079 0 ] -/Count 0 -/Prev 985 0 R -/Parent 979 0 R -/Next 987 0 R ->> -endobj -987 0 obj -<< -/Title (5.2.8 Minimize the admission of containers with added capabilities \(Manual\)) -/Dest [ 825 0 R /XYZ 84.259843 431.026036 0 ] -/Count 0 -/Prev 986 0 R -/Parent 979 0 R -/Next 988 0 R ->> -endobj -988 0 obj -<< -/Title (5.2.9 Minimize the admission of containers with capabilities assigned \(Manual\)) -/Dest [ 825 0 R /XYZ 84.259843 315.226036 0 ] -/Count 0 -/Prev 987 0 R -/Parent 979 0 R ->> -endobj -989 0 obj -<< -/Title (5.3 Network Policies and CNI) -/Dest [ 828 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 979 0 R -/First 990 0 R -/Last 991 0 R -/Parent 860 0 R -/Next 992 0 R ->> -endobj -990 0 obj -<< -/Title (5.3.1 Ensure that the CNI in use supports Network Policies \(Automated\)) -/Dest [ 828 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 989 0 R -/Next 991 0 R ->> -endobj -991 0 obj -<< -/Title (5.3.2 Ensure that all Namespaces have Network Policies defined \(Automated\)) -/Dest [ 828 0 R /XYZ 84.259843 318.510432 0 ] -/Count 0 -/Prev 990 0 R -/Parent 989 0 R ->> -endobj -992 0 obj -<< -/Title (5.4 Secrets Management) -/Dest [ 853 0 R /XYZ 41.740157 784.430079 0 ] -/Count 2 -/Prev 989 0 R -/First 993 0 R -/Last 994 0 R -/Parent 860 0 R -/Next 995 0 R ->> -endobj -993 0 obj -<< -/Title (5.4.1 Prefer using secrets as files over secrets as environment variables \(Manual\)) -/Dest [ 853 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 992 0 R -/Next 994 0 R ->> -endobj -994 0 obj -<< -/Title (5.4.2 Consider external secret storage \(Manual\)) -/Dest [ 853 0 R /XYZ 84.259843 568.851496 0 ] -/Count 0 -/Prev 993 0 R -/Parent 992 0 R ->> -endobj -995 0 obj -<< -/Title (5.5 Extensible Admission Control) -/Dest [ 855 0 R /XYZ 41.740157 784.430079 0 ] -/Count 1 -/Prev 992 0 R -/First 996 0 R -/Last 996 0 R -/Parent 860 0 R -/Next 997 0 R ->> -endobj -996 0 obj -<< -/Title (5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller \(Manual\)) -/Dest [ 855 0 R /XYZ 84.259843 684.651496 0 ] -/Count 0 -/Parent 995 0 R ->> -endobj -997 0 obj -<< -/Title (5.6 The v1.5.1 guide skips 5.6 and goes from 5.5 to 5.7. We are including it here merely for explanation.) -/Dest [ 857 0 R /XYZ 41.740157 784.430079 0 ] -/Count 0 -/Prev 995 0 R -/Parent 860 0 R -/Next 998 0 R ->> -endobj -998 0 obj -<< -/Title (5.7 General Policies) -/Dest [ 859 0 R /XYZ 41.740157 784.430079 0 ] -/Count 4 -/Prev 997 0 R -/First 999 0 R -/Last 1002 0 R -/Parent 860 0 R ->> -endobj -999 0 obj -<< -/Title (5.7.1 Create administrative boundaries between resources using namespaces \(Manual\)) -/Dest [ 859 0 R /XYZ 84.259843 720.651496 0 ] -/Count 0 -/Parent 998 0 R -/Next 1000 0 R ->> -endobj -1000 0 obj -<< -/Title (5.7.2 Ensure that the seccomp profile is set to docker/default in your pod definitions \(Manual\)) -/Dest [ 859 0 R /XYZ 84.259843 604.851496 0 ] -/Count 0 -/Prev 999 0 R -/Parent 998 0 R -/Next 1001 0 R ->> -endobj -1001 0 obj -<< -/Title (5.7.3 Apply Security Context to Your Pods and Containers \(Automated\)) -/Dest [ 859 0 R /XYZ 84.259843 324.051496 0 ] -/Count 0 -/Prev 1000 0 R -/Parent 998 0 R -/Next 1002 0 R ->> -endobj -1002 0 obj -<< -/Title (5.7.4 The default namespace should not be used \(Manual\)) -/Dest [ 859 0 R /XYZ 84.259843 193.251496 0 ] -/Count 0 -/Prev 1001 0 R -/Parent 998 0 R ->> -endobj -1003 0 obj -<< -/Count 143 -/First 860 0 R -/Last 860 0 R ->> -endobj -1004 0 obj -<< -/Length1 41480 -/Filter /FlateDecode -/Length 7852 ->> -stream -xœí| xU¶ð¹÷Ô­®^Òéîtg![gOØ¢4‘MŒ -(š@A„Â"F&!‚“ &,*DDÄ€"êˆ""³θ=pЙ€Ì :ŠIõ;U„àŒofÞ÷ÿ{ïûêœô­{oÝåœsÏVõ})`B…yyn®¶k3À¨7ê¦á#òB ­¡vµËnÊ¿e|ò'×´Qûš{Óø‰Ãξûr@CÝ¿tóÐ #ÍC²~ -ð´ƒÚßÜ2>=sö·/.`ÍÔ¾}úœÂ2|E¢õò¿ ö¶é‹xaVô@€;hŒZ\6sμkÝ pëZº¿sfay˜ቑԶͼoIñ/·ßèxˆÖ^2£°(ö/ž‡in.ÝÏ)¡ûSåÔ^@íÄ’9 îß%¹.p?îÝûJ§.ì(£ý'Ó~)sçÞ_vŽLã7ÐxïÜÂ93RÎ_`#ñÏN••–/ðShÿTí~ÙüeƒM¦êm׈ÐdÇïnîóÓ_ï ÎýbÐàýó›«´ël=q8ügLѲ›š -pÍ3ÍQ£©èj‹0ø”~õ=ÇP»æ_]O,‡X -ÇD=”ëm;Ó®rV Íï -´ÅÀõ®QCóÝPõïqéTè×b˜GkÌ“Ú`ÿ ¤ëëºà ¯é÷·êt¢µ~és˜×µŽ¦v(Å@÷vK­pý‡ 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À 0À€ÿm V@º°~¦VH|­Mÿúžö5=7•L¿JPKW/¡¤_ûC&äÀ0n‚Q0ÆC!Ã,(…E~¿>¢\C#†v¸FL‡˜ óý~ÿ™ÿ -ø¿¿‡&YHÈôõ:šxÒ¨¢&ß­“¼oMŽë×÷M¯Ãäm‚ü¦ %Þf¿?’)&7‰¨&LRš¤¤„Oìæ§ýúŽÉŸämê1¼sÕéoü$ªj-ê¦þÃõ{Ú¦M"‰þF4y§—xv<œ0èaÇŒAý ™/óù¿Wñ²¿KÂo3ñoõø¿Vñ’ŠM¿ØñÏõx1 ¿zx¨øJÅ õx¾Û.ãŸ.ãUür~1 Ï©ø‡Lüüìxñy=ž¥gÇã™ÏÒÅ™ËøY:~ªâ'*~œ‰ÿáÆêñC?páûKñ½Cø{ߥáï.ÅÓ§n§—â©›ðäo#ÅI‰¿Qñ×*þJÅ_ªx¢ß9#ÞQñx ¾‰ÇT|s¥S¼…o„â_Wñ*¾¦â«*¾¢âa_V±UÅC*tb˪$Ñ¢bóC¢YÅû§Š‡ðÀ2iÿÏ“Äþ©>?î÷I?OÂ}*¾T{UÜ£b“Š/ª¸»_°ã®IbWîlt‰IØèÂç‰èç/ãŸSq»ŠÏºp›ŠÏlµ‹g2q«Ÿ.ÂÒP[TÜü”MlVñ)>ùD„x²ŸØäODà&n´à×׉õ*ÖaMª«Çdz‹ÇSñ1;>z×ÕëT¬­™*jaí2©ægI¢f*Öø¤Ÿ%á#*®]Ó_¬UqM|˜Ø|x(V¯¶Šj7®¶buTá*’Ôª$\éÄŸª¸b¹S¬Pq¹Rq™Š•*úü?YºTüDÅ¥KñÁ"¬˜àIø€ŠKT¼ßŽ‹m¸È‚ U\pË/ãüË8ï2–©Xªâ\ï‹ÃÙ*Þë&î³T,YŠ3©Q¬â ‹Tœ®â4 aÁe¼Û†SU¼SÅ)*Nžd“/ã$ Þ!îÈÄÛUœH;O†<8ž9Äøp¼Í·Ž·ª˜oÅ[Tw³CŒSñfŽUq Ý£âèQ1:GE‰Q„7©˜W#êq¸Š7ò~âÆË8ìƒ>‡¨xÃõ.qƒ¯Ï ×»0wpÈõùƒqpRq Š× p‹ë.〇àÆœl«Èq`¶¯Á¬ ̼Æ*2U¼ÆŠéV‘„éVìßÏ,ú;°ŸûfbŸÞI¢OöNs‰ÞI˜æÂÔ”$‘:S’09É*’ƒ1ÉŠ‰*&¨ŒqÄgœ ½E{cˆ…˜"ŒÂ(’`”Š‘—±×0Œ F„ŠáEF’ -S1”&…F GE·Š!*ºh€KE'ñꆎ¥\„vƒl¡"HE¶…¢UE‹Í**4LQÑäF¹%º)‘xzQ%¯å¼2‚Š¬™­|„õù¿ð?MÀ Ñ,šá8ákÐO²íÔÒ¾Ï:zø^X ©çuvœUó~Ô·.ÂIYDZQ6²¨à=Áá›ûhÌÍšd ¤qÒ>é6©Y:'€R¹tB*ÊYn·‹íôˆopƒXhfC9Ä/1 [¥á’>ÆØŸÓ.Zì<5° *ˆ7+…J^Áo£ž£âl",¥û'Øfv’¨;ÈVÀiØ€ ›Ùiâë8|+p¯¤ðœÅ‹‰þ£´Ö š¿ Ê%§™TÞ‡úˆzÚkš^Fc?qZÇ‹PI;O€mr³ì6%Ð.šÄ¶³×Y›ü4ÀI¼ çál¥” íFBM@X5´ö&mŽ\Ì–ïVh«óÅRk„/¥Ó4Zû #Ús¿MhßÇm¥ßbÙA< f+±š(ÕîFà Óh)æÓ -¦¥Ä5@)fýT«€Ý°úa=ÔÐJ:¿òñ Í|Rú”x®aðoà‡4(–.h߶uÃÕ_ÍýQ…Þ:Â'dcþ!¾Å½|6á&Âóü<: ·õD©¿Ôøã(|„kÄïä¡ò#„Çä?Ë6Í2U™¾P•…:î2ßiÞ`~ß2Û@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ 4Ð@ÿ?àëVu‚õu[<0(bóJ¾œûáI.1¾s 8Nµe„ÄyâŠxdÇç|Å6mìTìU{Vpá’£½íHoËåIøàäIU¥qŸHSÄi°€Ïç† o™ Á¬ÄÊQ±Ìê85¦)x¤ì»nrÛ‘À"™—Úh×1ã'MŽgû‚1XâSÄ9EvR–“(QÙhu#›ñ6ݾ­Q*Ù<òòéFZ žv-.Blö¥DôŠÄð(§À)„4Ìñ´óñ ÷: 88,œY¢Â(G;ÚÇ4y&Œi -pç˜&÷„;‰ô¿zÝä#§Ú^}ÕéØIÍ%“Cœ7‰ó¬)Êá H´ù2'J·‹ÛMHˆE‘U& ¤©—)¢À"ya¯òÈQËaUÄò^Ë#—G퀑Ω05‰˜ÈÎ99Ù×&'Ä˦윜¬LÉã–M2°jþZûØröHVáÍÏ­ºçäýœšôs¸3B½ÔØظ˜­4gý¨ÅõÃn|çšÌ/~q׳eÑꟀCÿŒTOÜ[!&ùCdZeƒ†P¹!*ôYGƒ­:~]ÔÚ$[¼9*"&$ -ãb#“èȈ³³tÆg/m?븠!1ás‡ãì?'¤ãâ¸lÚ³7†OeSY¼ìq‡fej„3O2ÏQ£1ö­“Áƒ÷Îþ” õâgj‡zå³È±Oààƒ[Ÿ>tèé­ù’æÄdõ/êWwLU¿úÓÔ?²GØ6=£ýïŠ@”qÒ¡_ø†‚“ÔRH\;Sæ´ œ-&:hYë4;Ñ¢h7†šê>dfÅ$ëÿ_bÒ·#aÚ¡æž=ÕFŒ^9Ô:õÉñ{½6Ʀú†³`l -V‚a,‚2X fS¸Œf)”EðÛÙ$žo›ÉJøýlçK‹M÷+Ul5_fÛÀ7b½F'ÏXËb ‡ ¼U½À“ÔŠÏùÀß®î¸gõiaïˆÀÝ—û°Jõ!²¬c¤ÍmĹN𾄨³˜ë\±:Ë ±N«ÂC"bØ£BEDT3D¹¤8:ÐÌv2ÒκêÒñ:. ÌØO<0gœ~p Ý•$:-MëB©”âØclø3O=õŒÚÊú<¾nÝ㪕Kç./{°îYõâ÷_ðcU­Y»’«7”ΟW¶ýÕ=Õ[ÝÞãÞzŸ¬¯ÜF¤þE@Ž¯WÐÓöÝ–:'{vKuaëœk{™"‚ Ãí襑ئkœNÛ72öGÆFr"OÓ§NÊà‘ån…)Åç–ûA½È –Ÿ+¾÷üOÕÔØ*6~Õy1íô=w«GÕß«ï©Gï¾çäÈ‘l £s`[n"ª4)®!)Z¡¿Ï#×I¼Rê¤,‚™M’M#èÔ‘#ÝÒÊØDÂÒ\ž³ów ›:zñ£ù·í7ˆÓj^cÇ™ÆNž×ÏÑ0ÐçÉê0²Nq=íÜí©³¯SÖÆpˆr^+e…GX1´QûÙö#ݼ«§.8ÌHrÆeÇ9e©‹[)¬§¤×ÕýܵPýCƒºU]ÈÖ°»e¦Ò²ö5êõ< a®Ù;N³uÛ;*ÇOdÙ6—m™÷»{ -Ô_ª¿Q«þ2) öÉ¡ÿx‘3âZÒüžCó» û]Ýs3_P†ð‰|Q jÄ!O Ér&{ûmqúr¬#îÕ½ùW¾TÅ)darʲ@g—ESG„Ͳ`’¬hYâuê6×–ÖirR#“4#SP3²GÝœIܬ„òT‘ª à9âZå&ž'nT&ò™|_,VðÕ¢Fyœ?¡œã²=a–#1Â$ÈâMá˜*úȽM9RŽÈ‘³M¶¡è“FŸì3ùlÓ°@*3M‹E™m ®?“kL5¶Mø”ü”i?þÜô¾aú¾kú¿”¾”ÿ†ßŠïä¾SçÁÔy$2\úÓäÁ73©#{©ßtd‘l:ªù⎑ígø¯:®Ñ<–&¥ $¥ŠC{|Ùä}Ð";QBá”$&KàAÉSgv×=d•„ŒN²ÝP»°DDHÎ!nK”MŠÖíøˆfÆ΀!çjÑ5PÃnŸE?= -íõÅèê&@0A.É$yÀÃÜ<ä$HbI<SädS²’löÆä°žÇòx‰X(-‹CVË«Mä ¦XÍA…††…$ i`‚3ÁçÕÜs~>2´â†ï½2zÍý¾ÍÞbо¢£Z}´®îQÞZûµ„UÖO먧ßýý#ù-ªV¬X©iNŦ­d')ð_nÛ­<&6F1s“…ÇÆÆ ³Xcb%ÏÓîÇÃëœR<žDN#5Æb4A|d„½Ÿ)ŸêøðH[{ÛYŠQº\´¸¥;Ç›Î.Õ²kRé¼p‚ÓH8ûcÓÒÓnIÀÑcXX˜îôââ“S²C;#ZJJrröµ‰Y™¡ÒÈòwîyö¥ÅÛøìwêGê¹{¿ZVÑ6ÿ…ÖªMŸ½Í¾žõ¾ØöÆ€œe‹¦ÏˆèóÞþ÷>ÉHÿõˆ¼Õ?™û`lx¿Ww¾y6YÓ… -⻟TA“­”“ÄZÃÌvx>Ln±;½«bFµ$4;׆٠Ã̊5÷ˆd:ýwNµef:õ£?röR;y‹7µÐLê Eç¹Ñ1±ÞŒ¸Œø!)¾h_Œ/ÖçõÅùâó£ócòcó½ùqùñù)e)+£«bªb«¼Uq+ãkSR.¦ÄtMíšÔ5¡ ¦ ¶À[WS[æ-‹[³,v™wY\øUq?{€3!›|súîçw׶£¨ g -áðBÕsüܹãôƒNº*IZnˆ„2_"Y»y•²Zxžg¢ÅÆ…·¸šmk£"=\ñ(0†»‚GDé$ÑsZMxtïR ßK]Ýýëè‹Ñb aCøÏHÑ×”®¤›ûZJ¡”•òROi¤yêûR›æHœ.òG{ ¼­£JvwlžQô•vƯ‘P–ÈÚÿm› Ï.–ö@+L‘ Oq´S8Õ¸lo#ýw˜}æ|s¹ÌLú¯…m'=…½ÖL |ß »¿„N>ž¤õ,ä‹Ò~<~·Âú®®@ÞÕüìp‡ž%ß#q Eïž&ú(·ób>S)§¨½œW‹Ÿ)ñz±^y†»´¨Í­±S1EÒbvŠÏ%X`«Æ•Rµx„"ó&\ojÄçÄ~Ó¦wMËø7é¢ÔK‹ÆZ0ÎbŒ²Ø„ƒ-<éO»ùì‹G[dwû,v¦ãRÇ.žÐññ{E~ñ`=׸é~Jõ9:S2qQÈ¡‘Àd÷wm²2E“}ÅÃ_²ì2‡ƒmòت¢½ØÙá03XQä|§œ®ôÊKЃv{{[à©17÷ì¥Ü#m™º"øB2óËk_Iü8ÑŸh&ÍÐuÁÓS?®(Š' (i#^]þâá–ù k¶·Ì_üÈö––!MK؉Õ.úú3Mmž~RS¾yë¯<ÓQ%ìž9íÁ®Ó."B(¿Jk[ÿ±ÖžíÒÚ}ž_yøõÖóOô–6ÖÔ6à™Ò¾VƒÑ¾¹Å-¶f×Úp³+øVtyF„뎨Ӄ“óNQr¥©R©4WZ*­¶Ê J{ep¥£ÒYájˆ¸á¼úÉ3%Só9¯ÎËßµ³î±]»»È\ê…‹V¿bNüøܱcç¾xëè—Oªo©mêyrCÉÛ¸ÙuÉàh¢Ð ô +[é$­Xeo6·š,²JžËA¹—n[ä/O½£9È}ù![B4™$ÿ@ a8:vTß'ŸkitpeHÿ(Üçr?ܱ—ÄQ<]Ú­”âÚQÚ-Îuæ>ã;SŸñWRŠwÕ’{•§:\‹wIÍWrŸÛ"»IqÇHÕ¨:uUîCðk-º®Î}ºRHÑLrN”%ÊeëOn¼¯µ¯m°y°e°u°Íê/Kä©–Tkïtwº§whjLjlš7-.1e•e•u•mUK{3Ĺl‘­hà ´c0:0{a$FIÑæ”ô´!i÷¤U¦-K«MkH»˜N)ä¼&YZ²ðÃ$K“®·cJuõ´Ç‡yöo¿Ÿòú}Åo._;c§oç†O~Y¼O²;5uÂߨ8{ïÕOîOH8œ=ùÖ1ùIÁ‰uË7ïŠÑ´|9Ê¿ˆÍ¤åíB ÆçÁÉZ•*‹•dLñÙá²kZ®†ÌÀCqÀ.),ì}1àŸµh@ºE.“²-*8ÙbV¡®Sþò˧·VU‰Íê/j:ªÇmÚò[^PÃnмónÒóIº}¹a°/ꊅ­µ°Vw³ìËmG––çÑ~`@£Îfv›Y©çUÍÌB(&»;KHf»53{¡¹ùÆ= _{‹ýŠäÛ; -·l9¼W|ß°«xúEÜ¡q=Ùx¥T2|ïKéô㌠í‚\ÊÜäaáröÈ™À¤½q³t?ùi/½´7`Ú‹/Ð_|…ÞyõxÎPºßŠh¯¼~6’ßË+x%_Å—ñu|W´ÌhÖŸ9za/)’Y¦I^%²Ù $e(yÇFá()OŒ”}Êíp;›Œ“¥|¥ŠÙ,œ%Í%r²° -¬ §’ä•°’Uc5ŃUr=Ô³õ|n6ˆõòñœÜ¤¼ª|¬ø•¦ji/U®ÝÍî~]½ë²TÐ>w}ßÈ)D#Å€Dé IÖS[\xPŒâ´Å9Üc“4ÍÈÔtÑ«eôtîsšƒœÏ»x¯*_/ǺZ­Áé¹ÈÌTs/dR:‘™qU -Ñ#ÐÔǤÝТ±+§PMzZ±»izJ2ûîªü¢+ÇؘšZ2]Ë5¸F¯4šèõP|›Ï™aJh°[R µÈc{]¡WÍ%ês)¤ðŽ*{øáÐ=öõfhL£ö‚ª¿ËˤÔÀOIq-%Ç=5þ{’‰bFWit€Ò^jÑ(ÿ®¥E˺h<ð¢F4ÛûegžÖIã_pXå#›¢8$—}l¨F_€<:ÊÖž7K”Â9Í­A\#LÕ©bºè~˜ˆñÑZÔWr1-‹Š«²1]Br;ížý(6‡§ç…õQz;"=J¯Þfˆ••Äs|òØþWu$S+Ûuq…EÆ&<Ÿè¤¬²ßáÞ{°>Ԕا…ëÌL2Î6G[&ýN¹ó4ä èU×™÷¢ [Z -©ïQ)ãVÐQßÌ÷jâì<{$iÒ±NybTHŠ&Ì.áv±Æ»d«s— ã|¡©yAŠ#4Ü­8Ì%+q‘æØ„±)=8ÓÓÕ <Êû|œ“WÙ’×{Lq­Á½b,]Êý{~è±íjñÿ èiϳè䣛‡=Ï¥ûl(e‚)ÚW†$3±’¡wH«3¥V ÎAayuìÑ/õ¨ gã:ë2¸Y1Ü¥PK`>Ì‚™P À ©0tÀ ™A˜Eµi4 ÃhÌ(§ß|˜…0úRï(˜KãûSm(ÜGè…Ûº×*×[3è:ƒæ,¢²ˆFZþ…]sºw@;-¢½´¯Ì¥Ñ…4çßÛq8Õî¥y·ÃB¡}a©P_m†>£PçÈK«Ì¥²ŒÆL£ugÑ8/Í/¥Ý õ{?\g¼¾J9QDÏq0›zµ]Ëõ/<ÍÕyéOòpÕ¬®9øØ‚ÿ÷t²ÿèDûýúHý[OïW•Öè×󛫺¯Ú[÷ Ö¾Få ŒæIù»*IóI9•JíëŒN¯/•ý d“ÄÑ9œÊ<:Sc`,•ãHš &Âd`!{B~N;hûYhN.ðáyc'€ý¾Âs!\ÿâè_µêªi#ÙìóçÝãú•8ãŸh#‘ã¦ùiœôÐ+cþ¡D’»*þcúØa ]? ü´RÇÝ£>ì*ý«¯,¸÷? þ´NZj;Ûj7õ±©çÕÿ‚ÿ€ÿ…½'õr[ñÛ®¾B‰ý¿­ö¿4îG$ÝÉï=uÿkÿ-‚’ÿ½3õ/õÿAi”&Õnm:ÙsEÿ1῱<#­vÕ%Ñô&{Ñ,l$YÖ8¸•lJ³(vºŸH#“9K&ÛÕlSÒ­RÐœlòÓ9„&š;’¬f¡™ÖG6x+¡•Öºì"a­9™ÖÔ¬4•ü_¢i7ÑY7u^å«FȺýþPD"Z¬DCÕìĵª9Á¥×Ct?JO­ß±v[÷Î/œµ…óçÌ…Úió gAíô¹åT–̘Oå’ù÷AíÌ¥TŸ9Æl¨-)œKcJfL£žÙ…s ¡ö¾ÂR¯V’úéœÂ%P;w¶ÖS:³pÔÎ_8—F.(ž;“ÊmýñSú‰Ý7kfáU¾J‚€¯bºwTHN6â% -â!•|!ÒYܲO¿Ž y‰ä}KÈ^*džì¡Ró¡L/5ž?/ rm`úHÛŠ/§í (ƒ>ØZ«÷„®6+é¬OüçÉì´î¢>N‹7ÚX=îtÕ{¬qUÿ”íùWêü:€ïµX”Û=5•¸´ÑŠÿ„Û ¢ -endstream -endobj -1005 0 obj -<< -/Length1 7456 -/Filter /FlateDecode -/Length 4548 ->> -stream -xœíY{|SUž?çÜ4iB_i´<Ó„¦PúÊmúnš6iÞiK“ÒGÒ–ôݦïj¡-å!«Âˆ2«¢¬òЮðQœEÅññÑÏ(³‹£   ãÌŒã¨ËÌ: -ÍÍþÎMúBÖýwïí¹çžsçü~ç{~¯“"ŒZ€Æƒ4†Ò»n£uBKÎAï -ƒ½¼p¹Qáö·ÐN)±'+«#-a#´×7tº{Èr¡ Ðnéjîd¢å"y±µÉÝ#xå|; %½:‚eÌ0^mykçÀ=Çn…} íq„˜ÆŽîwìžø…0ß}ÐwªÓ}O¶¡x§òt¹;›dò,¡ÅQÐ÷DOwÿ€ïe¤ùž§ß]‹ ÷6|)¬[”ûŸˆaþ=èâ3ÿz†ÖŸþM…ïwƒ9Ã<M"ÈÁ8A w5 -„¾k(˜9ÃÏ4ûºÄ÷\Bi€Ó]0CY†„ЫåÛÆŽBAÐx€TBŸÅ_ãZ¤äÑ¢ŸÐ ¿™ËäЗ _ôúIÿ -„~N¿99ͯL€ÿ>køü@-G2t‡ ¿ƒ: Hñu”GŽ¡ -ôŽ# -”~‹ÖàËHEkPÉG,±¡T²ÅCŸ‰l†‹¤då’ȄΣ5Ss’QÓlä0Ð5!#@ ä-d#Ùð÷ОíSȆ¾ƒBé* ÿy诚סŽ€ïZ¨£ ^}”E–¢Xò4 ö ŠfÎ! ?Έ"¡? E÷òk:Šºøþ+¨ c´ Pl¤Ž¯HJÀ&$áßÇxþjô½ïaü²„@;Æÿ8 Ã ž‡’qŠ~#IE ˜4´†B õ»hÏ[Æïòÿ_ÿ‹/°+­ 0P¢à‰ùZ€Z¡ŽFaÐ#KŒ;ÊÛ¨@NTƒ6¡aŸ!¾?ú UAí÷}ñãûGžåNWä¬;£°Ÿ£ýè$zýMbvã|ÿ O#»ùû`à>Mn0+-SÅt2‡˜ÓÌ»pÎ|+ÈÁ€à¸à£ q"ÈT´î½AGƒ&à~î AW…1ÂRáfáIá5EÑ}¢çDïˆ>  ^ì ~8øÝà+!KCrBjCzB 9ò!ÜŸýŸ»¿m‘‘rŒ˜% "Ø6‘LÁBšŸ¿à -fÉ»pÁŽwøôøÌ´‰¨Ò"2X±ˆ£"$bÙOo[]O¯§¥Î…cÇãwHwéÀ~îz{Œ”ÂÈjÿH‰*=B•F*6:BED)-žÞžº¶^ÏúÓ!í8fÿ¬h qsçsŸÖÒ“‡[ñ8ù…CC"’ˆY†H‘¡È€”x»|×0Ë¢!Å®]+‡Ã¬áä©‚ì,måcUé²3µUûöÁ¨›”“N4ŸzA±L%S±*VÌ’òóå}T~.^´]º}œ¯C£Ôz2Y䱞<Ó(÷7ŠÀ>²±ÿ?x]üšJ¶ÆwAé¬Ñqª´tV-ŽÊJåò˜»e²»…ò˜9-”Vå[‡A¼†y1(>>Ä}]Ùçn *©½ªá¡‚UÎG‹Š˜š*6N%‹“Å -…â¨èhV™ž®R÷ìÞÕÓ³kw~RbÐê ‰A§5HÈÒ7Þ8~âÍ7OT ™ÍCÕ›ÌæMÕÀ7ìðõ lGªØpYxÜ«.2f÷&B¤ÀˆEÁ$Šür¤†¥œD²ôw¡H*–ª^x¨!_¯³0•‚±¦–¡†Ñz\É-?±}OQ¥Üг©¯Ñþä`‰µ€®1Õw‹„‘ÏQB`t‘˜¯èR,¿ÊðOOŸâ§kÖ5°†¬kZ«©ý¾àJ‘C¯756IîrÕiÔVQjMu¨ÔÕ¡AaµÖ,gZws®Õ”m)´âµJ£Iü ³"•ä,š‡BaçT˜ gÅt§#±4×u66V è6*Â×9yÉ¡Cø,·Êl‚œ"‹|ža©8}JìÛä¥b†KÉ|î¤:ÍÞš˜Þ]Ö¶maU¨CSRZZ¬*‰X‡_$;®®XUÔ™gì+ì¨ÎÒÔ—³kð>òP“?Ê+a7`2É4ÄÀ&N&UFKd Á”¦øáWú«*±ÄSزY屶 $aTß3Ïñ¢5¹²Lo/µ‘Gï2-g¬WסqÄÝík‘Xk]…¹¸Š®J -[äšÙQm“ª¤ø÷¥ Çp£øc²Û;@¶Ó‘ëûŠä‚Ö-F -ÐAµxzç¢g$LKÏS -…3¢>›î±Z=êtµI³Ájí×ÕxQ©¾¨Ô^¨µÿ¾Öi³U»,øAGnzËV¤gWˆ#ë´n•Ê] ]q±°P©+Ê|«¨HS\¬Ñx¼|$ ô(†jo:e¤˜–c7Å´ -Qa^3©í%D\Ÿ«mËîÛÝ›è©iî vœ[ç°êíeÅ÷«·=‘Èf´[<Ûyjô™u]å{í‹«Òhª¤–Í[2䌱ÔË€¤¾æ8HÒ„.“N‚ü~ƒ5A!ÞïI0d³]°Ù¨õÞd+y ¨æ!´R*’E²‘2Ãà?sç2×O¶ÝZ_6:º?yÉk¹†×séŒt`™^1|þ€[>…ÿÊ)c‰ðWbŸË_™Þ¢ç&o•Ú Ž{\£×>vÍZª}êv+Í«Uå©d"¹T©^±2W¥Êõ^Ñdek¿£:‘à»IÞ#o¢¥Ô:)ŽÙ×’Á@Npp÷=«è\Úi5 ˜¬ðªx¶Ï}À…ßzÚÕÕ”¯Ó8ì#:m~c§óp3¬Ÿ®e¬eDúé•ÈÂÙ€äá,Þw´ªêhߥKÿôàž££d¢ü)çI;™ðîݳuÛ^ïe*!̂߆“øÓH15_ð?áRñ8Çá%ÜýxtÿÙò­…œ¶h/Ç~_(5ø6|ûà»ïœdÂü3÷|õ¯ú3òos— F!^HjÅãAj–v™L´Ú ¦æó¹/ñ`°ëQ—û`MÍAw÷1þ&O«)+Önj5ë×í±ÃÚ›Ÿvv5d'ÍÀ‘FD -……'ivrO:¸VjÀ¹ÜÛd‚û7¬ Œ@ã0"B$P;) /3KÍ£ ‡~"ÌbŒßå>‡™Vôsoã|¬æÞ„©^ÃZîiî™ÿcÀÏCX%ÿ;wÒi6C—ÿ+ç™~éC§Å˜ s^×üÑYéIð”×JÞIä½ïo“Ü©uõžÎR/ gI¯“Û ²t9I‚÷<9íýY Zàûü°^„–ÃI4n¶3Êøø2³8ž‡µþ Ëu°žb>`6PäEΧ)ªÎÃ--‡%ºQ»c„W·Q]@Ï–óÈDzTÅdÔ\ÄBÐ3ªfdùà¦ÎÏ7nþ…ñŸwîÄösWÈD_õ–¼­>ÄK8Ln€„‘Ôâ‚ÂY»Ï›œ¢ì”¨dx«rO‰Zi{Þû(È8-ïŽ] §Åm<˜t?8ª›9 ï*w!= J™€¸<«Hçš@Át,5 ÊGõ…ójÓñã X1÷á¸ØM9%Áãk¿ÇÉ ‰ªâ'°ƒÓà2î|º¤ÄHbKŒ@y/d»Yí0¢¾ gÕÕbŽX&3¬‚õt¶. ‰õÓÉhb¦bqì) :me~m¡m¾dÄ÷1õ|Ôñ~çpÐ/á ýuf!ï9Ä -Õõóäa]_‰_¿ÿA*H"XGì´—ç'`{2™ 4Ûc\®Åƒi£ŸS–³¬C©t°l¹·nÑ«“½—›SòÛìy®Ô—:¿&5µ†&«È¼†ÈÕ)^³'ºèpÁm¼Þî(2oÔCƒg1øŽ`E¹•+)«oÔú-û6£:¥‰\å>îp¥¤Öä©)¯|ìúaa]‹o‹]þ4d&r醊›¶‹ë®Í-^¿¾8·tQ}ôŽ†â!]J}…2ÛSßБ™æhÒ­›Fëê´Â-³¦k) Õn0 Þ­œë]Ò’ú#¸ü¼ -`wÑL‰¤sCî” J"4Õ”’†35sJÂ`õÅ›õºÍ¥Å÷Ô&•¤¹E¸‡.]ëÌH*iÐo-/+Ò Z Ee1µ€k ™ä3¿y²4kV?¬éw um¹©ýø§À—, -W¹59]Áëwƒ×å(™Æpª7só`Õ,a£oƒoôĸ4ƒ%…V/ª-´Ôù!‚Ý>;$’HêœeCas¥2¯¶¼?EP9¶è/ÏFËu2ag¢ü٬ßèñÁØãýYÕec¹Ó¹¸.‡$©Ú­ ÓË*ª×q€ÝXä?S¡—É“rÛÀ±JóÔNikœddê Vå½Là´F|׸Iô -?R2wd$€”÷h2YL ”D³A·|Ö<Þ&m™œ<>•ÕÜ‚µÐhî·Œ Í·\p:?Ü© sãHlû÷ ÷×™¼àꬼàªw ´ýQü IšÊg£2ÿ COÌŠÆìêµEÿ¤œEí1’ö‚Yx9×q¿ª‰¼$!I›­AÓ;äÏç~Ì¿:<… õÍ‚_^Ì£O5¾ô‰Ï¢:øfÄ`-.5YÆŠSשÒ©J»2µ,5ôÛ±µ¨h¬¼|«¾4ÉUXèJLtj]‰¼â&Á%ÞÉñrÞîÞòtRïYò#ôu¾ßå's1ñMs<µŽÉ;zÖðŸðÏÝ.WL‡Ñx'ÏšŸÔ8y·%in÷~©ìçó«Ëbý°ÿªJs×ã ÷Í´Ú#Üɾ@œHÏÜs2‚ÿ.!P€Î5f3µ‚¢Ì´\6Ï–¡tå8{N&_•š-S%ç˜UXéÌ I°)W¥%Éä²ð0ijµ~•«4!muìb™84l¹Z™h\üc@ yn:ÖSq!Æ~`¿»Z[fµ Î=àq^‹—Úô¥XkyàšÃ}߃W—S+Ç3^—¦ -þœ°Î˜šLMs¨šÉAäÞËkW'ƒêâîýµ k8‘"°€ÿÿ‚œæ -+#YÈX|÷³î íêøoÐ< £€·©,A6•%(ŽÕîØQ{¼fô¾š{‡)ü"g#r. ŸåXü:(<À6ènHp²cXIËDb϶‘º“'[¶ ×=òìY†£ÞŸûî/¼LGVŸ¿Dò‰ÅJî—µWšŽ?Kä“ã`²ŒÓáWvÐ<´sòŠ‡q(· Ïã¾Ç½‹ï1›²Ñ÷(zYL=ƒâu§³¹ú­Øˆ?Áð¿-Ñ >ázñ#ô` -ü«é/À˜æ¦ïòÑÃwïÏÞ”‹ÎÞhºxBóð]w!Z†c‘u£4„úPjA­h¢š¥ðer@OÔ¥> -stream -xœí} xEÖh-]½Ý=wIn¶{/á&„5 ËpÅ°#DDd $@ÂN"$"@p„ˆ²"FŒÀ" h\Ø@GÑ_PÇ ÈøP1$•wªoufþï{ÿ÷Þ÷¾{]Ý]]]uêìUI„BQPHÈß«×;—­x¡W‡Am|ïìž½<Éž6p_÷ã{ç ºÛ›÷Cˆ è}÷==ο»/ˆÐ®æðüÊ »Û¥OknžÞ÷CÇMË/¾úS|B9ïÃýšq÷—ø÷z{¡á»"C -‹'LI¦\Fè®ñð¼bBþÌb¤¢WÀ½yÂÔÙ…¹öÙ/ ´ÆoõÚÄ‚üñžÎ*è¿5<Ïœ¶ïbçýx¿ùÄi%Œ·&äAßp;ðý©EãòOn=”ƒÐ(3B)…Óò(ö¾A–Cû…ÐÀ?=ZAkóí@ï+ÐæÅE3KêÏ 0¾èß_<£ xnþ†¯lGHy ^‘Ñ»ÿð]›¢1¶®? Là“‹Ï?.Οm8vo}5OP—Ë.h«d€÷”i<!õR}uý[êr£§&àÜ.j LCÔ¨ ÈŽBÐbI ܇´‚ìE j:°Õp›>ÓP!‰‚L*¥²Dˆ$Ú†šôSØs<ò#ÿ5,»¸ —+Óð—y××׋§Ò1TØØ’BÿèT Ǧ›ê£"ª£H5Pö Ç×pl€c£àxŽ²†ûypL¦Ð…ßCntÙ‹ö²s¨PÞçYhoÓç¤&|/w€gÐNùí•·‡ëØÙ†g©ðìm´œ•"½±Oõ$êf´9R_ýkãJ gVˆ†±=¨BÚoœG±ihƒRŒëí¨‚ìGF»Ááku(Ú*êYYø=ÑŽ~ï¿ ó<‰âàÙ:Ö ù”ñ¨KE>¸öJ[ШÍéD ˆ@"D ˆ@"D ˆ@"D ˆ@"D ˆ@"D ˆ@"D ˆ@"D ˆ@"D ˆ@"D ødñ=¶K¨)j5pFáþ‰¯ç¹ ÄÆYBâ;‰È5VäG©¨3ê‰ú¢( A÷¢áh š„¼†oáùQ ÔÝÏûÃóÁh¨ñ<MÏë¿ü-ŒyùÖ¯ùÝ -»™I”`ÔÚo¯$Á¾ã+Cw ó¿?<Цõ-·~»â¯D9•–ÙþÝõõ9ä86¼’ÅWÒ Z)“¾ø­‡_´iÝ?g˜7nÑ3»¡ÛžyÙPy÷0¸wP õ=³gbÔJ„}ó*ýã&úÛ'u^l/èÜæöGÈ|œŒ8¢8ˆP6ÇàÃÍQ Ü%!”ÍêšíÄ5Å~ã¹½e"e‚ñ4y¡Œ)PkÔx2Æ(£Òc”nì)1ì6îÄ5ÅNã:Ê(mØŠæÁs›q'®)¶`3zê,F@6cHŽO(”ó¡Î„u” uâ …2u¢†bÍxS5J™R¼!o¦-»Ý‰ec^Ì(%£5fDŒl”(T?ÖÿŽrNk¯µfµœ^kMk8ýùjoöóš@¸èáCo°Ãœz?—zƒš/½ª/ÈÞϥ÷‚ô]NßO«–ÙY§oÇÓ·8=Èé7;³5ôÍ—ãØ›éþ}±l:Ý·×ÁöÅÒ½oØØ^}c™½a£{ÌôuìuNwsºËM_‹¢æt'§;8ÝM_õÒJ}úy¥†nƒÓ¶ú2´9ŽVÀ©b}‰Ó­ÉôEN·pú§›9ý“N7qºqƒ•mätƒ•nIëQëkè:xe]"] §µ5ôy˜üóñô9N×<û[Ãé³å¹ìÙ7è³ó¥ò¥AVžKËCÒjNWv¬âô™¶t%¼¸21TOWÀ«+üôi3]UËûÓ?Âéœ>|xÊC—ÙéÒ ý§K8}’Ó'8]Ìé"N>d 9}È€*„Zq®ÀCP5< -–ÅCàl„hm$/¢¯ÙÛø2Ë"Yh˜4Mê&m—–¶C‹R©PzUB™E>ÖHs¤ãÒ4LP†ˆCÐÊq?ÈÊI9ÎÆ^œMŽ¡ƒHÐß—ã.ì;‚N£Ó8ZV YDÇïáïq;< o‡·~D?bÜe |  ¯BÐaLGåh)ä«`†Ç€î è{4S‚^ÑRvš´b§ÑÛè ô‘ñáÉXd† ´ ; xmA“3_`ÂNË.% ’«¨/ ›ÉUœ„ `ö7GÓcRžôž´žw0¡¨ö€r”hÁNãr â ¹φvçÀ8Õäm²æ¸…yÁèd™CÊÑY¼ ïŠzo“ò”±R*—Ë¥aè’à ú€~äüx=!·G?J2ºLàžÑuÉù¢‹E0‚Œ`#ÁF0‚Œ`#ÁF0‚Œ`#ÁF0‚Œ`#ÁF0‚Œ`#ø?€ã\ûÝ’»Ô}!„Q%ÿžJr² _Ȇ6)­ -–TdrHv«½¶¶*=*+«=²×V§á@2ÉèÕ) ·+ŠJü£Õ\¾ -§ÎŸ_Æ¿ÿ ï;süá"ïrö,ï,zÞÄ¿'WžãC´IÞhµ(ºCFÔj?U•^ÝØ­33*£#I x¢Ü.¢lš€SW-ÿãjþý%üîÙ³ø‹?ðîgÎð;~½á½äKrQµ‹¼„¶JIÐôuú -8E¤¤îIrŽŸ­?@ˆdáÐÚ± m!˜ §ƒ¶GíÂ?Àù*¼€Ï9S߃T³ÓÐÖ¹‹ %A˜¶«2ȼr4 wÀIÄúYÝ?ϲÓ?O¿:¶þKÖŽ]F&’BN­ÂF+Ükl«bÇ-{¢\10’Ë~þJµýR¶''5“vO‡ôLlG?r%Ý6iîÜÉ“çΙŒæø'üc~‡p -NÆ!Rc.\àßð ß|ƒcø> /Ç3q ^Χ‰ù}“€fµ¹é6‰lcK´MScäXŠb6oª«Â3N¿R}ʘ6saîŽÀe<„oÄ#ñt<äZ5Öé;}°ÜçZÿzÞ€tfƒž¥¢‰IÑØ“À˜d1ȺÌfªˆZ#¡WHŒ›!«êqã8+õÚ?í_i2ìuDëÜ6¼ªöTµ#+Ë‘u}t¸Kë÷0Ån½èˆÎRììâðf8”L¢[zZFSŠ¬Ñ,&:::¦jÝ,&eDgÄô@ýX¯è^1¶\”‹Ý833££à¥âÈÌìà—Ü.Y!ÉS×Þ&¾½–”ŽøðÁßó±§>tÇñ 8Ñã¡ìIóï€û´jS}òÁ“¯Â Õ)]‚¶@B­»Â©­Ô_µTÈþ•¾Wã+œ•IkäU©§×…ì^O²ÝC}.Ÿ_KLµ×ž†žªŽÊr„mãü•óµ aþÃ%û¥¬4ìñ¸]R YrŠÃ#D-HMÊh¸hJ3ÝóÌs¼‚ï™ùõ}SOOX½ñ…åžúÃâ‡r÷žñùTœ„‹i0åàŠÏ¾qjf§Éã -']™;ttËTë÷¿y`Á  …£@:€°³;B~°P -Pj "jR¶1L—hج£XY•Ì.0éS]«ªÓ ‘œ?ÕµÌ;,ébƒ4p€IBAä{>oáŠÓ§ùÓu“¥UuKé¶ÚÁüïü2¶ã~ÀÅçÁ -ŒŸ€º…šIJÂŠØ -žұÂU¡l·WÐ˪DÚ áhîCöDû§À»*û¥«øJ0Ÿ"„5ºkÛ…næ0ícº¯îþÖÃÓ°§ñ?óOÊ~žóà™ü'ׯòî½SÙi~ák³…wå{~©}:n׫עÒû¶j#¬´ èKb—ÀJ›£N!ŸUxäŠxóJË -{E³5ñ«‚f-ÖKƒÎXo´¯yPXìyû¥*ƒ&OÈ~2ÌÍ -² Ë^ñVèh%^!U€.VØ×xVãƒJ¬35‹µ²¯ª­=žHU£9w -ëa'YNj†2:6Ê»Y2n¤&zlÔ–ÁÙå#.ð¯qVpsâ‹ùžIpYA!(Eaa»Z¼ÓÓ±é“âfü~¾Šÿ÷‘K üþ‘G~¿`P|È(å\ˆšV²í¶È  9U{-Ø«p¥ ‘Nà<‰?#)Ïã›!A/:Br -èW<B 2Sd™é ªÎd¦‰k’ Q ‡š@%ª'hº$Ñ .¡mª´D&º¦*L"i2…²ÝGqn•¡o]Ï ÏÖ9ÕÐ9%üÏÐÀë5­ÏÑÛéCõB½ •á2µL+Ñëkõ7žÓíQj¼æ3Õ–šßÜWêÅz«}´at¸4”Ý+O¤“¤B6AÎ3— 9øA©”ÍQK´EÒcì1u‘¶ZZÉV¨åÚkêëÚô~‡QÞRi£ñ‡äcå´ú‰ÖÎ4€ ¤R¯ºÍcø<’ŠT>¯n ^}Ûùevº¦ ’Áâov÷‚7]œS‘ùQN(…^·í—_]¢½##…F%šmZLœ¤PO¢Ùã¶+/ €æ¤ŸÑ-ZXB8*)]©ÎJ i¨™½Y»f9Í$œÛ 3IgãEC´I†cBÔÈÑ£Gžþ¯’Ò’Òÿ"}æ.äŸòë&=p']H—ç ¼ó.^U7sì¸ü|>›x›¿µäo§Øé½Ç§­¿Qú> -lÙ‹º„bÍv y¨b×6jߦ­¡«bmÌHnk‡¬®1-áUöwÂ4§íl7&ŽàÜ ¨z£S‹6¼È÷„ °Q“¿šÇŸàðN\:ï«ÉSNÌüKuõ_fž˜2¸Ómx=.À…xýmø‘¾Ùüê7_ó«Ù}A§r–ÁÓhÔ6äF´%ø€G%±6ö¶È£Ñp¦ÉA#çÒ¶çxÁaGƒ‡ÀÀžèp†(·H·ÅÕ?_½X÷^‰‡à;gM*,œô¯œ,m¯½ïÛsŸ}ƒ“òK -øÕ^ä?”ä‹\ è¾:L(-ä’ÈûÉ´D= c¦45b¬ªj”]ÚkvKŽ¥Øªßhm÷J¶ºdBÝj²éÚ=gù·pl ç›0éfä­w¡ƒC·oö¯´ é_i2#„B· ' ø5; ±Fþ÷BöW&t±¡ù>#¯{8äTÔè ~@’Uѧ)LªÅÁ'Dƒš õ0l7w¥]”´¿2’c¹J ,Hž¨ä™ËèJ™Rl~TzLÞªD§tÒEíAú«CÈp6L£æ‘B6M-&€¡•©‹Ùõ%Õ™kLìþ4’VøͺöÔËë¶ zëΑ@]·ÚKd@ÝΦ\f(:d–°‡"*ƒÒ‰YŸot]¢+ƒ×ëŒ÷¡%fô—Pš©¨ -ÉÄD'J4]Ùº®u…P¤3LLS)ÁL—ÛKz{z^6À0GªàêF\T¯{&ˆ6#>jĦÚ4q+>Õ§¹õV¤;éC‘Êý^2\®O$EJ‘¾‚,S—i/’J¥R•°9ª{©ÝÚigœAáí­U‡k9Ö‰x-P'kyÖGÕÅÚ3j x §àÈZ“ ~ -;ÿ?Í·\á[ørvºö ê«i%eמ¡Ák{xÈ~^hhn(AÉ –)+rPLþ Êd¬ö’³ÖóÎj˜uV“\N2æª`”ê¨ËÈä•oýq?y(.ç˜&à‰rži~M¶zI¬ò;ÒAé “)÷J!™¤˜ Ác!)ƒìmć§ñÁu_# ŨÖø«¼FŸ©#'jò˜·)¦mh‰æ´(Xµ·‘L*(¬+ì"-ÜXˆlGn,ŽO -$‰¬}/)à ¸=?Î¿ä¼ ?|ºøÁ‹AÕ¾ýG]]´Ÿ™6~üÔÏ‚ Ï¥ÇÅ’xo‚'–)ÑÑž ×£;a)$o3/‰Ö=Îj÷ÊH²€±GÛ5Åc¢ a3²Àq‡UGP–••Õ4©‚GÀ˘x¡7¾¸˜8oll\\l|¦;ÓÓÓÝÓ3Ô=Ô““Xà.ðä%Ú®»x/iôéóD;)97{Ò¤ÙxË;çÒeƒæ…>à…¯uºo4í>bBá0þ0ÿ±î;ýî‡OïoUö0†gXž½ xöôB¨% -J®‡nJôInü¶ä~;f›CÚ\ãXÕ"Q7ùâ絺o³öO««NUW‰ˆÔ°RÁ~áz J )NÅY Y‰Y¾,¿Ä~¾~þazn¨ÄѾÑþÉñE E‰E¾‰þ"ÿô@‰©Ä\b™ë›ëŸXizÆülb¹o¿<°Ù´Ù¼Ù²5akâVßVÿÖ@‹\ẓ’#ExEÔrz),â””pR˜.IGæ|;qñ#ÃK7ýüW~†Ÿüÿ¯¥K±iîC\¸âóãØ­s°Ä6óªN· ÈézGL ýèÞŸþ™™{¸sÈÀ^iÝ~îr¸y›lä=BV9ì|n×~Sí§úWêà„­†f†6VÅWªÓÓðövÃÉë,;x· ç~m ´=ܳ‘QÉhA(…ŠÕ0IÀ„‰"2’q4 -ó„üJø% 2ôS;—"" ‰˜ç¯{$jˆøI%‘Àdh4Ž¡1R4ó©~µ7êûÐ>ROÖO¾£KUG£KÁ/ÀOâ'ð‚ºx8ÒíÒÀpô膺\øR<-ÔON_Ê€dð¥z¦IÚµL Ü” ë -=SÓaeA(1Cs˜…9¨C¸‘…‹UÍ&ÈÃ_ 1)È"x¨ |Ù¿Ò$ -óv´Ñõ¦ÿºçýµy¥ºfîbvf׃º°3¸áά;ë §† MF³|}7yU¯ŒeDç¯iÌ$›=$†FK^æV]Z¬)ÖœBZЩƒ”ÒÜΚ ±#]Jgirš’¦¦kaþÝ­}i/©ë©õ3‰üpAï‘îaƒåÁÊ`u„v)×\„Šp™N'J•u¢6EŸnšd.2Ï¢³Ô´ûM˜*ª‹Ì»Èúš´›½ªî7÷¿.#ÝÄnø>pÜÏ„ƒàé|ÏæÿäWx6Hêm©›8À‡]{ -$V_ š{Îðd+B-nÖ/JT¤ -ɨ†~ÉD‘:3ê¬hò°QØ Q( ŠõŸi\•FB$‡äöéG‰¥AÜ’fàÎ4M¤é‚M´!p¸ðŸ € Á›ëÞ3¢5qÕ®«{‚” -½C5-õñº‡7æÙðÎîø{µm‹ñ ߉¿Mÿ>ZvAnn ÅÉëÍ'lh½{•ípÜƘ^Ú#*cÖ,±v#d¥_ §RÆÊ¥´ƒóEâÞdEJý Hâà7x5ŠfΞ=sæœ9súí,=ŽuþãñÒýx9.üz˺u[^\»öErzl.ßÅëwåŽÝä€L…¬2@V^XAÇ¢£ø”d=j9¥¯wHë£AD±J ru¹yMqå’ý kL\™XO܈b áh²º2ú.´ò…Vy&4äå{ù|+äF톽$u㟦§½òÜs¯¤·çg|>Xý¸;ùD¾1J¬ê8{˜WÑëÑ «c=;¡®²Æ©KBŠíaê"x%´E˜Âù+ç«íçoðÊÈl y -Z:¤{hSæmÛÂý¶—ç?býxÉŽ ‚w3fϦ{É°Ÿ«7Œ…ûb -Ø7·öàž8ÂzÍtIü|º#@î¿aí¨zŠ­5ãbÖF6¯ŠsÕmAÙÄbëoÐWÕdóDpî’X‰%tODºž[˜æG,¬yL¯}òæÉû.Î+ƒXu‚¿‚ûãfXÅÝø²Yyo' -zèŽl^Ögàh…;ó·–Î+.¤š‹Ý%ÀC'º+ogج®—ñZ´Ñ*ïЉSAŠÆ,–ž6“+¼¡Û7M½t­®íZU^äžO¯­¶_?#HÃ!6_™¯ ZpÑŽ¦I0•ÌtÉ»ºâtþ^¾}ûÑe×?:e¬GµëhFw½bÈ•÷bVi6HµêJŽ·Qä>ê9å]•æ ¶ìHZõ :A“ÍÈb ¹ý=ä.©AÐr>,eþÃ%‘ð€¨[æµlXÏ[7 åèŒ@x÷¦q{‘6Ù¨¢›AäÇøl>Vº½¨ÀK|浪q£wŽ¬ÜT]4÷™Åsçî; -ßQs ß>jÜæZÿžéàèÌŒòMTÞ´²|í¦+7+ÀŸG£QF(Æ"!¬Ÿ‰:á>lßhÅÄ‚z;,›],zÖܵè>-mgžw¾7¬žH“–cpR´‡EñU»»OÛâùÂtïzmú[‡ÈÖº¡Exõòé±I)/¯®ûXvÕm›{Éøâ"‚ -±›” -ÑY´CÚH‚ì7í&…´-O+ÖækÒ<¨Š1x¹”wmìâŸ6Ìk6ô¨£'C.c]ºml\˜Zýæ…éy±0ÍóR/Dh¯ÜEéKû*#h®k`„ÁÙõsµ°š­°FÝ öD“B©qî(MR_Vblgü'’èáÄñ`:®(³j‘û¸,Q}|q–{rxýÖ“ ¬§¶«ØêëÚõü•®çÅv¼C,@À†l¾”´”œ”â”ù)ËR^IQr±!w·¡A'àèx]/ÜÆsiOöÅóU÷XXDøªÐà Åp;±ÇKJ¶Óͧ]ú²n(éc‰5eËÚºOHŸ=S^|®îc)oÓ˜¼bÔ¨³0»[uvǯëìù¯³î£³Ï>Õ¨³@…PÙšTˆŸªµ 9åõQh½ùpÔª­‡míáîrËOÕ^ëäìîíí¤Mc5˜r£·„’œ5oÞ¬Ò¹sKAAzò7ø9þ÷¦s^Z¿þ%q`ÄßçÕ€ïãÛ° ð6AIÊF%"Ît %܈3‡­«ðgtGĘmšDeûù󷆚àuV4ÄcgBI”àI“¼ BôL7êvÊú¦&™v1º!Ҍڄ¯4…ikÈb@\ÞqŸÙV%ÙBò†&Q°‘¶&)CSž9›DCá'd„¼@Ëìë¡0ìWÑYQ¯®Õõ`Ø\âÏ?6h­Î9 gð8¬š¬ zÀô´õ°¶CÑe RíQÂ(œÂ(Tã…†'?G×!9|«EÓêv¹í¯Üê¹c^TË´ÇýêŸêj¥¼ÝÓ (ãN†\e,Œ›‚> …,fb5e&ú™¬¨“ôLŸ/1^5yŒë¨ûTÌZ‡´6xøÆÊùî¸Á°tÎivg YNUŸ¿yíüƒÈl¢¢oþ!¦µa‡@5‰‚Ñš¦éšÉd6Y4KŠ5ÇZb­1¶Öj[­­ÞÖÔÖÜÖ’êÏR»h]ô.¦ÎæΖþZ?½Ÿ©Ÿ¹e–y–eºGÛ£ï1í1ï±­²U±ªVͪ[L,ÝSǤjbÍÒdi-ýâ‡á¥µá¢g~8¦p\ÿüîعŸ_å5EçMù¢dÒä¾ÓºwàJí¸O Ϻœ–Ö!£U[“–´î¥;“’°½cÇÎYií,jâ†?m¯H|ynbÏC¶0-keª®wàêz¤«&h`ö(ëW“Îþ•QDYƒÕȪnd UÕWº -§'^LwÔÝ)>íDmv›#‡äÐwɣቊ@úÙÁSÌÉ„ƒtÀ÷ñ¥¿µ›;õêöíìyþV=âÁêÑ«§ðXlý(^^D–òŒð8H$°ùŒëDô*;Þ%ä´Y½Á§ÙcövÆçÓox´¸ù"su‚ö¹ˆàs˜¹×sÆäux"±:<½Á§‰qן§½uï$Å#ùŶϊKJ®XMR¯­Û`x5Œ qe^ GF'oÝ? 7휻iÿ`í¶0W|Ën¬èÄjn()$óÉ2²Öta<xðs/z‚ÈGu°'Ž§±R2jŽ[ÑT)uÄY4KJS{¡^"¶”§> ?ŽÒÇÙBy%Z‰WÓÕÒ -V.o¡¯á×ióû>½Ñ؃'ðÞ|–”W[CåkëÄìû@ÜL€Ù›ñïoÙèÞè(ö#:ŠýˆŽ¿¹qîW÷#ÄžÎÿ‰­ˆ¯Üº_Ï ,Mïml:ŒÑ§êeºÁ>¨ Ùú Ⱦӣ=Ä+(f—íÐM”æÒ=¦dÜœ´b©rKµ•ÖÜ4·³f  Ü‘taåÎJ'õ6swkoÞ~è­ö1§C¥¡êHmˆi¸yŒµäIcYžœ§ä©ãõ ]j¥6×AÐRµ£žfÉ&ý¥þj¶^dy Í'K¥EòRu¾þ ZAÊ¥Õr¹ºRA®$oJoªB¯ŽKÇÕúGè9'ýM>§žÐ¿FŸ“ï¤oäïÔÏõŸÑrgá ±!4Cjø=>Ϲð žç-üÑÚŸù£¤Iâ;ñ€ºsuñXþ¼°;/¬¨¢@n6œºÃ–€l`ef µŠßâ¹Õ Ù ˜›-hÕ‰!}­ú E;ì6«IS…G/ooZyg8(;øOpŸQÊþ–Ä Xª†¨6ÝpXbó©€”ÇÀe=m8­MœÖ©§#a2¦0ÕƒÝÌ-»•æ¬¹Ü\6vÍ”3•,k–­'ê‰ûÑ~ROÖSž@ -äÇÈcì1y¡e¡õ²«-«­[ÈVºUzÑú¢íÏx/Ý-Uj•ú›¦×-¯[ß'G,G¬‡l’ $ëºcoüO§N^ðpm ÄòÔÞûž=^<òîh°‹:•\½–ùÆŠtîï®BHê\×ТP¬ -A¨pyAˆ—;èz¦ -@²îÝ'ÛdpïGÓ×îyÓïràPöp2T™D -•Ùä~åe¹¢‚’«v ¼£°javS2MUÛ™²Ôî¦~êÓÔçÕWé>”XV¿¤Õª×˜ˆ[ü*JéuÛÉÀZXwDÊ«©+¯G5¤0¼' ‹=Ô+äDqšÅcMq•ÅË–@O_ÒbÁÉ×£GÖv=Ú…1ñþõ;YkIÙèV;baUi0X­œošØPÄN@FÇN™w×”jì t -o$&âòÂÂq”qilðB±Y•²ëqñÙîºw˜qÞ7hÜ;¸w(S+ðP=Œ5†UƒW“,ÈbÂe6Íâìév ‰m¤H7¸rz4e]OÎêl­„6ºôv³ ÚÙ/†·4BŽËqâ*!˜W—ŠS±ñ+h´éÇMŽÅ`¿wûw&œ?çðg*+Å^ÿ7ûךPýJÃ.ŒAµ…B†TZp¬í=!ò4P\Û@¯Ý©J6ÓzMZ‹6:´b{}÷Å äÖ}2‡¼”\Ð{ù×ÆÞ‹X˲Ô_î¾-jhi‰Ú¢¡!¯ÕÛÊ’f÷´.C‰°p•ùdKóžÍ’‡´kÂÇ+b'H(‚;Ö—´¾íÙæ  -­,-AšïˆIˆÕ­ÐCþ>$ݬâ"nV‹þ a®Ò&š!¿U?ù|CCŒ½ 1CÁíTtg(:Æaie·–¡˜§,Q¶$õ ‡´l:»êZ1·¨ØÄfë“¢@ÉS7F+I;ìñþƉýºšrø·ŠÞ ¸3ókº.äô«ên€‚Fˆ¯ËK†qšñ½yq‘îÂש¸WÃ5E~<°áZjrÍP žÞp-£Düº¡b4Í@“Ð4•_£\ó£t”Ø®ÆB ?êmJÐL8f ”¦¡ÖPÛM‡ömáêv4Ð_ïk¦qWçxç~(ÇCKý?5óú¨C`¤ûa¬ÉðÎth-èȇwþ{#fÃÕdxo(*…ã m¾Ñ[ñF¾1#?ô2Êbh3úíüð~Œžo<»µŸ»^fEEÐ~üo<õ_>Ô j&ôUdŒ”´u@nz¯ñ­6×ß -ÿïõ-øué#R_o´4þW€O/Zjœ/>ÿøõ³xb¾þ†Z‹¼ÍuÙQ”ÍAäÚʶ¨=”1Ê*1ÊB¡ì -ˆºPf£PD¹;_v¾ -½Š1tx·+"Ù½ AÖ©ù%ÓQ 衘øÿ¯DK<¥`Æt =L5†gÄ8Ãl臢¥$IsÁcFß<]~²~Jý&~ò7¿+ 5|Y¶>¿¾´~S}×_6©ß>n©}¹~Yý²Ñs»&cü{øE›ú)¿Ý¸~`ÃyÓ-õŸþ¢œws þ2ÿª>Ÿü~KÅ{|!”Ù7Õúå(×i¾Ny˜šßâÉÍ´ÞJù¯?Éó›nò«ü2¿l\]®÷‰úòÕ[¿øU_éM£,‡ 7LÕ )BËáõo¡éºœëw]Ÿc#%/׬ïÚT ‡Â\j(ÿ…¼~ ê -nþK-úO{:t£l -üÓ_¶ý½ ÿoµž÷ïÛ4iýïÛ4[¶ƒ— ‚n¥€—hÞ%¼}'÷t"þ—”{Ðpðt£ ”%V–৤›úa¿Â À¡Bö­ÃÙdx7Ë-cZ¯_Ù®_Ùojáh8G!'Ÿ¤&È-ì0²€gL -šJ†Wa@q&PÐ P -³€‚΀šáE”ÈZzšQ@ xÉ@Ç @ħÁÐ÷=€ðøCaTñ…k'Ìy8Œ=Ð s4„}éó¨¹ _ú`þŒü±hYþŒiÓѲ±3ò'¡eãò§Ï„rbÁ (gϘŠ–M((‚ë 3 -¦ eó§C›‰c¡fJþô|´lj~‘_”à“™–_2-›>EÔMÈŸ†–Í(-K -§O€r¢èÿ7ü¶!Ë©“&äßä»%öݸ#Î æïBñ¨j±‚l9+óç+À×AÎmPp¾ e_x£¾ÿÜþY -endstream -endobj -1007 0 obj -<< -/Length1 17024 -/Filter /FlateDecode -/Length 12118 ->> -stream -xœÝ|yxTŶo {êÝÝé!ÝéÎÔI§é$B&’¦4aÂ<…yTD8Ü0ˆ`B%r1‘ƒxB!r%ÎÈáŠÑë ˆÐ)ުݼë{ïûÞïû^ojïª=T­µjÕZ¿µö#„¬°Pl~þÈÁOMÝûÚKKo°ÿÒ`'¡´õÏÛ§ƒ÷ñúÇPFï]Âï ^+žK²%@¿vÞìŒ.Ð'ÛÆ{ðXŒÞF'Ð -ØNhå:ŒšQ5šõ ¨mÀsPZ -®uƒ@~è"ÌÁEP< -Å{¬ÔŽ!š6H( ahš„VÞ¹Ãu õFÑphO¹sçÎW¿o÷kÈ›²$ -”`Ô)Ö¼Ÿx Šöû*Œ}wŒ;¹Ó}ÍX³» Ûo\[wçΰB!R³_ŒÚO½Ê~Áë¹üG/'w8¬0¶;úö vÛwr89¢ª¼§á|ß>Ú5>ê~Ñ ÿ -&ï6+öIó“žnOš§wK>DVcû­ÄÞQØŽ*…ýj$`bPÕöVd…½E«›µ½ m‡}ˆV7ø¡ŸØÛ‹h%´ È {=J‡½ªõ§ÓîR@¬ËZ]Ò §Ú¢Á¾1Œ2F[WR?£·½•N;Bo®¤¿Þ(eô×cÂëcÄeôÆjázK¼x} ½îZâé¿Iÿ}‹þ’Bÿ‹ÑŸý)^³Ñ+h3ØÌhsÝó¾;Âýè÷W‹Äï+èÕ"ú£ß~)~Ëè7‘ôkF¯Ì¥_1úŸGèå…‹—oÑ…ÓKôKF/2úÅçvñ F?·ÓÏ*觟ØÅOýd£^üÄN?^I?êF› ÑÔ^`ôÃTñCF?PéyFßgôÜ“ñ\ýg=Ëè{´±Ô+62z†ÑÓ+黌žbôFOn7Š Œž`ô8£o3z ú;f£G ´þGÄzFÿñÖñGè?V oñŠoM où„#^z˜ÑC´®¼·ø&£µp¨½E߀¾2úz=PDÿB÷[ékŒîc¾Vú*£{ý›•Ö0úÊžñ•tº'„¾¼Û"¾œHw[èK/&‹/­¤/&Ó¿2º‹Ñ­Þ.VÑÏ›Åáôy3}N¥;}y–ÑíFZµ­³XÅè¶Î´Ư¬ Ï+}të™#ô™ÕÂÖ§¼âÖ t«OØÂèÓŒn†öæ#ô)/-a”÷¦›€ÛM6ºQOËàDY-¡•z铺ÑõŒ–0º®Ø"®c´ØBÿƒÑµŒþÅ’'þe]Ãèê'èª?¯W1úç•t¥‹þ‰Ñ!t9£K}œÑ%Ä%&º¤#ßgÂcúØ1a±•.ö 2ú£‹]¸`„¸°‚.˜Ÿ(.Aç'ÒyŒÎM§sNgÝ¢3ÐŒNg´ˆÑiS]â4F§"³8ÕE§0:™ÑIŒN«'†Ð Etü»t4ÆÙèX=.´ÑÑŒŽbôáÈpñát:’ÑŒgô¡•t£Cmt£ƒq²8˜ÑAGèÀD: À)ȦZÅ'íß×)ög´´úÑ|hå¡}´œè“M̳ˆZéƒuÄçÓ y½Mbž…æÕ­Þ¾±·‰ö®ÃÇ åË5ˆ¾ê«Ã«¡•kЉ¹š[‡}¾"¡£=„ž·hF»'ÒnŒæ€€sŠh×´±ë@šÍhV²MÌb4s í’!vH3àÁh:ܘÎh\N‹ ©4j)NÚY&v>B“;…ŠÉ6š\Gø°Ì±S(íÄÉ­’ðŠIŒ>w>à¥I7±#£‰Œ&0o¢Þ°<ÑÛ—v0Q£q&“Ǩ;6Yt¯¤±É4f uÁÈ.F£ÙF1 ³N# gÔɨzpäÓ0{²–Gí6³hO¦63 …ûBmÔ -Ï[µç–zïƒWÀ˜ Ñ*G2z>«!£³¨¶-èE´Ꜳµ@ÿEt•nÛF®¢±P? XÍ 7$0nB7 §Ò“Ì€ûNBoÛÑv¼ÐÓb+gè²ØD’ ×ƒÀBSѱIÜÆåÇ&ñg¸‚P´T'ÙdpÁ嶿…ÓÈô!<¿¤ãé#ôs\,x„¥ô**'ˆNFsÐ9±I²¡rكʥx™0YÛVpþÈRa2®AW¡Ï©ô&´Ý@Ùc„’áâqG†pn‡¶/ì%3:KoÜ7†û ù4®¬¡mh<™’Ah!Í„Ñ¢âÆÀ†j`K7Ò -è_“Î =Ñ2—µ7@š i” cD‹×P1>t#y%Z,6qøx×/änìÅ{ÈÃdÙF¨…ö¤cé"ZLw -S…Â×bñ/âûR’4F:.GÉ=å™òFyŸüÒ¶WuX—¯[¥Û«ûPí U—ÃvYmÕOÐÿMÿ½ÁÛ£lL4Î7¾’òFÈMSºi„i¡é)Ø^2½dŽ0O5ï6_³äYVZþeuZYKBÅдÐ7C¯Ù°-¶½e§ö'ììWÃRÂ.8 -õNìœá\鼂´¨„Àœ@ü¢C1>-‘Ö¡õŠ@eL%¤š/ôhNÏÉIC)WüÍ©8ÃB=Ô’a÷¡Ñþ‚½±Qlj}„=‹§·Ö1Ъᴆ‚~–Q4¸!¼A€¾@$APÌš¿hNCæè1[<ÚFWí;¿:]ãEÓN$œëQ$ŠA|©Q¹Œ¢p»ìº²P“ÞbŒqEGEF8a¡V‹Ùb4¨:%À#$[¬ùÂ{Nn =zø{ð}Czs: H3B3h†Ý­O¨Š;Ó­ì¡dï¬ë³«g·°oGÞb—fUϾ1{ç,ÆpÇ‘ìN™…•ë#o$›‚w²)ÛÙ*6?ÏK²ïl=Å×õìCa,h‰„Œ(ùr”ÄHî輶³#{K;lŠI,wÙJC7áò·Ë,H”N"ÒK‘¡bblÎå0Q9Õ|ÁÑ£Ù‘îïÑlö_¹ü¼“ÞpíÆ5óµwn\³Xs,9«J2++³K|‚E’ì¶0`+>>!3,Ìa‰Ïì’•™á€†NZÂÂì6I&üiÑ—³–Ì¿P„gœžZÔ8íÝ3-ÂCw|ø…aÿüäê[¤B+š0ó<~NX4qTÑícà¾:w<Ø+à31µÏ ùáÑñjRŠÀŠ9¾}û?uü"7ÌÜ~°z\蔃FùR\ÙщqQ¦Hª{Ú÷tö–ħ;o6m¡O;£"“`&%Õ®GªÁ«vÑǨI†næ† Íþfó5v-‡³ÇgòðÍ€s‹ÆuçÙ–‘ιöÄIr—¬Œô0‡Æœ'.'`‡÷/“ºÙCÏ7hÈì’ºu¾Å;§.Y_Ëú{kG¿cöÐÁóæ <·¤®Ä·$99íðº’:œ;dÎÜÁƒçÍZr(µsÊc¹Å‡ßÄ“ñ®É_ æÎ’“—ô^w¨ŽG™9w¾N‚}|,Ç뾩3±š£!Äa4’;‘ÎIˆ Ó3ºdf‰iÙŒQ^¶A'aQ6¥(‰ÆÉi•Ö8··ÒUa­’(Ëu!;ÉíLŒ4¹…P%ÑêŽTŒæÈÎV{WóÍ þó;qÝÐÄêÁ¥÷Ε×>ú'\ -è‹Å‘“ -a£lùªíñG^ :&c®P \[ž 䆸*i -Å… Ú’m“5Yggee®Nˆ§øÅkš†Œ8<çöÍ“¥é©§^í½6¿ûʾSuË?îÝgß)Òc¦¥­çÓ*fÅæË /ü»ðëùù}°ñ½ ßõ²_Å>§ãÍï¸ÁÝל+øëøÊSN‡ÉaD˜˜i ©×¬‹õMŒÖ `W0*ܤ4h…ÔM B×°_èÉ -+¬š´=b”µJÈ©†ÀSéÍ®]fnÕ$b·Yžx’ÙÅšM—¯][\]¹uk¥d½Âºó5ëöÍ5|â_—pC3ô;úMhëWæýÊX¯ §`ÕC¿þæôßûͳÚmDödY3»<^ë³z—¬Í¬Ç¥Ë¬Gówø¯®àwx^h7 $ÞXÐ_'³QcÊÀ¡BŸ-†çTRaU *ÕIˆ  -7 z]¸Q²j&ú (Í(pqð3 ü´S±(Ê^Ž˜àõf‹– ‹WHbÛñŒ¶1ÛŸƒg°í9xØbY š`©EÕDÀ¡ÀиˆèZ%k_„{¯Â¢ü§Á½æZ´‹À¥IÑÜ™_õûq;|θ“'dkúôP0ìBÄEùííZ–¾bY½ÿÚQ±é·ùpg1¬öA0kzä@¬ú"èK:k¥ËPé¬pUu°»"%7ŠŒ3¹Ü1€èÇ ÆÜ|¥Ž®¥ú>JA)8…¤Ð!EL‘Rä%E—¢¦èsQ.Î%¹4WÈs¥\9WÉÕ媹ú¡h(J†ªCõ“Ð$<‰LR'éw¢x'Ù Hc§¸SÚ)ïTvêvª;õ¯¡×ðkä5úšðšøšôšüšòšî5õ5ýQt%GéQá¨xT:*UŽêŽªGõùDLp(:I˜$N’&É“”I:>ðuÔaž€5[°º¡qÄb¶f¤ƒ¿& ÚÞ£Á_ Îê:lhN×ÅKK7n*-ÝôÓõë?ýÔÒB®e–ÝuÈ ²ƒ½ÏN³3ì}œŠ³p6N­fO°5ì/ì ¼ÿ¯Âëaæn€çɇ™SQ¼ÏF+R)®—Q¥NñH. -ó¢‡)lðk“žÞ“êu[ÄLoWS‚»±}xØœå·FX2¨nÀ­¦¾"·A¯˜Û(ô¯#ŠòŠ’è  ŽH¯$‰yfËËÆJ[…€* 2««.Gœ™vˆ6ûa¤cÇÀÖ‡ky/`qÅ6SëHç…‘3³PvV›ŸÊÌy ÜæJ^Böù—ÁÎÌ¢ü-«Ç½»hæ©)Ÿcý˜¢®M555'qç^Ë+‡®Ü”÷à{iéWÿ1ùØc½yâm]ÌzQ? 7´Ò®–êv+¥˜ÒØÝQ•ž -©ÊþJÇ°PDmá®x³‹ÆÅØt1^ øB³E3Í š\99ÔÓÁý¦3È!EЧz2w®íDSeËöÃõ™ÍœñÎÔÝlÛ¾½tÇæucêg-ûGÁgXÜ@cN=óÏâ;œÎìR±ñ/U»—Ï_¼"1ñplìço¬x‘ûMˆ·„…0‡,kž/©QjÌCT/WŠ˜®×aƒŠ\Š ™ BÌ~?7rš‹»ÂkÖ _NƒˆOs7H#UÍ ‘K¬#þØýý©S§[KÄhÿô¬?ãE¶å’+É%ÃØѨ§/N£#*eK©y“­ÒH*Ñzc•\ã¢.äpa5™c\|Š‚+ùJPZ¬(åfËÙmè^±´Î‘–Ö†¤Ñ¾ÃföÕ¯KOxÊËoyù¡í »5l³ÙÄ®}ßÌ~Ž=›ž¶Wõ¯¤Ñfaœ¨Pç‘cÂ+Õ˜J³ú7”MBeX…¹ÊçBñÆ8YŠÂ¡1^nhüWšvÈûÚ¬AƒT¬9n¢È“'!r§‡á¸ Bé¥ÜMƒ½“Z³à“[.2ÿ/؃m¶²‹k¶n]³nýzñàao»Ä¾-šË~ý÷/Ž—àÍx9ÞÓ:ïð‹/þû«ûöƒDKa6¹-•P¦/\ôJ¨W…<äIEŠ‡bd ´As—)-Ím‹„—1q IØ]J÷¶~ÿ!QZ3ŦQ·ÖˆI<‡¿$2D³¹@Ùú¼NªL*]É•V°º ¯¤: pÙ;¸L:—=2ŽºLî˜T˜´fmÖ¸,ÚT·r@2w ÀÛ†søZ ê8zª$³J¶ÅW&7·¹пëÚ ·ÜPM÷¬Žß8°ÉˆµòÙ }~AëõyçEO«@÷ÝJÂÅl·S»ŸG€0},·Ýá¨RÕUZ×ãJõï1}xL¨BDâ -Ó£ÒtÈeMus zcÍ.6kÞ˜‡ùAãCÛ*^÷]&ÊíÆü´¿âù=?ýðôÚ5[Ù|øë›k×ny™Ý`¿±|rºõâŠOo 3X¯E+)ÚýΞ·…«n丷¬OhW8êì‹«¤Æ× Ì•º -Zšn@iRj'íw§6'p'|ˆ#è\8ÎËÊÒïÇ.³8GÛ5x@ͨ“ ' _)ÈìØWàEx®êØñ\OÀ»³ìŸì}×ó ±áA‰%úìR¥n=ª4IȤ“iº!Mv™S­@†ŸÇnñ€ñ¦gYÌñn%4@‹Û‚Þ9súš\„Gá>ì-VÃÎï¾ûã·o/SØV¶ðã30æo¥Á—4åíè “8rDë•Jáï*ui28ý ƒ6% í“ѾÞy9IoI6µ>JŽú—r/ž_Óú•†Á> -…гuð…{þæC붋è6ÀÃÝz®Ò…­ãÉôÖgyýkZ³µØä+aÌQ4êás‡G 0QuÏ<¹"Ã]”š.eº¸ŒxçÁÉbI£ÿÅŒY<™naØC³+¬ûpôĽÃª™tòäÑwœÞñS¼}ÅÀÁ¸/ä—™unp;Óvš5ŸñæÝ}¿ý5IW*Éß´^•€W1MTj@Žš[èq*©¡¿¹c€½gC#éØØØú1Ï'î E·’È©Ö.ÅÓЫ¤õ -BÕfG¨/“M]2ÊÖx$è”Cíôv1Bàé>M{²S8Çç°S ËÛ«jj„5(Hëi­×JŸ[ñê€ö²@½"JÑAUâE ÿN¸ùXÀ”ð Z³$íFC¸Ñ$‹a¢C‰ã•,’O&£•B]Y..SVèBYD‘ùM b')IŽWºÑÚOê'££ÅB©P­Ì¥Ëé²Ì Ì @ YÎÒ[Q-,|;—Ð,²­õyÿFÒübkpó1p³XËà:}MÍ°‹¦¡.’Yƒ)W²¶»?n$çýEbS0¾Ï-ƒç ¨Ø×Iˆ&SJTL°Žœ§Ê„"„¡³ -6UT%IHQ%”j ,Á64´ç”ÄÓü݆Á˜TŒ&ø:ˆŠªs`'u(N]<îHã•Žº,œC³”®º“ ›J!d uh}®-žQ—€‰¦#ìF»~ØVÈÍ[Ib‚ÿGzëÓ ÆÌ×VØ|_xxL xÊãd„ Å2±i²¥ªm&#@oÎ]‘‚À©t`>ƒñâ”;’x¹«<š‘9ò²\V,åãi4ž‰gIÜ ·°ó”b„ož¸ƒ˜rBlº!œ½•$œ½–l ¬×ñmy_¨®ÒD+í¦*'J7dIéÖL§FT€KÅm11@¶Ð»êtCÉöª’’ªí%þÚÚzãWë¯ä*.Àìö&«cßàp\À–³Õ ¯Ç%x5[ £s«¶D³£¡(ÉçÄ•f†×[ͪB$¦»ZK—a ¬ÈÀDja1¾Ë¹ùBô„Ë¿/{nÇF6¼… »sûû3bJë¹­ÅÅ[võùÅÿlÝñR9ÿ>F …hyŽ¯+8bªJ^*PÑ+4O -öJ„Tëõ‚h ÓIT¥BÕðtÁ’fS]†Ôè€Qä$Yr,Á fN(m>œ/½ sh9oçi±h;äæÃ?=0xÅ´·qõ ,½ÐúþÅÍ›*Ÿ"oÛîf³ð²&´n›¾ü`ë2¦õÚ_Ö­Ùô‡`îÐøz $DïuŸ‘UoLŒ+OÕ»blÝö²³Ò‚_G/ •Þ -KU¢KÕÇDÊ(>2'$Í&çÄ¥&òøÆß|…'²µ£Ýuþ·iXnLWEUF2¨/• 戣=ßêpüÏñxBW: £-ãM,ß=ÿ™9c7ý³/Ì]|vVUÍ’ÍóÞ /ªwíÞ£ø‘i³<δÞhúWròÇ}6¬XôxlxJ}õ»ÿ•¼§À<î‚y”Q”/D -“n௺‰üý–Bi7³ ða1KndI`]o5ñè £- ½žZ,ÝÑç0Q+°®ÒYeÆë]&—%€Euµë>Øl.ŠPXO6ž¥ä’3y“'B8§ññ[ ¦Øe£F?á6ÎääçÕÌ™]ó`ßn¤'Ýæ7ÌŸÞ½GîáãçÑþY_¿Û½[·î½NrJÀÛ´t¯#æ1(òJ€IYjˆ2‰ø Ô‘ISyˆà¤nîR7åÇ6è8&ÎËñÎ ³ð¤ólàu6ð|˜†©)øuŸ¶ 貯xªTUõ*˜'½Ö„ ÐBaE‘‰—2`/2ä©àx*µU Þ>)ÓË(%ø:ËߣÝÎÞK5s+së«p{æÔq%šCc„•gó’TžH쪎%cE½CO€`ü1QIÒyõñ†LÒf -ÝÄîJ7]}–a$‰Ç’B:Z%Ž“ÆÉÕѺBýÃLe–a}\X¦{\ïÅ´d„†òÔC©çdy͇{ËO^:Úxæ(ØÂ>4Îÿ…P+‰.õó5e¹Ôk÷ß@ä¡€$ôüÐæ}!>òjx‘(Ñ#=éóôà—0ë:IÒÌ,t—RPw•ûæ+þ+–û|³ù~ Ý?…² wL]¢ wŠ 8‘vãõ…ú'È2q©^ÏßBš“b#Šâ$Å«ÏŸ>F)ÔO'‹‰™;+ðΚƒ†£>p9äSÖ_ú”=Í–|†[À]%­yþ«dmëJêàºÑß(hŒ¯óý¼*Há¼*yÜ}P ¢€e$ô]¨‡¬ÓÖ^0Vû_*g€,XšÞñ<<÷Sææ€a>Ùâ?Ý:•<Tì*6k^³¯/J df@'󞙩Áãc!IÜk„´´Ãì{Ò3tŒLx¦†.ì>@Oû[`Þõä×Ö±iûí…5Û…Í<_ŽýB—J6°*.ŸI¨•Þ$µè €Â´'¥ýZ(í×’n»Ç’awãsxÏÏ?³BÉVþ›¿üþÜ<Ö A÷åæùdx©µÞÿ#OÎK[øÈcadÒ62­ÅoŠ|dÔS"ü…tƒÐF¶¸3ŒfR -þY²Ýü²\øÈÚWÃŒçúMo :þIÊ{Á7 Àµpþv -›?l -fê< -K ¾ÀŠGÄœ&Õ:å2«»4öhtY‡zk•ÓˆBxˆ*úÄ -’½×æî&ÒoŽR®´ø[‚y*KŽ6穾èÔ˜ÔØTwj\5ªÆÕ¤Z­Öï -«vT;«Ã«#B&`O[+;“CyÍtò4VFfü]‘:XXRßó…¿.Ÿ·m>t¨ûë«ÿöÞíÿŠ×m™xlÜŒ#…¥'{ÆÇ’ŒGM_ôáᎃZ×ì.šôö®#Ç£×-ËêR—0|xúÎéÛÀ)Ì p…R}NƒŽ–™teö*S}ä³áÈjíç4HRD~4p–žH¼k8Ç“mm* k¹Hí…@|ŸzZ±¾xÝ“O®+^ßúU§ggœùîjcQUçº:’òÑgŸ7]øâ3òÄðB;~b?°“£GläŸd`tèñ‚äÃQŠ/ÂPj§.„ò/®Ë,õ†g!’“Ð#øÌ|_‡Ú2ìѬ!®í}y€T-=«™!òÐœGÖ”ÖÕ¥íYü·WHmëR[¹éÍ¿µ–H¶Ö'O»Ì¥vÆëãéuö…+µÈР>‹ŽK¤V ƒôX¡ª“u%ðV¼U0}Ûšã »b§ñ|ÿ<Ÿ]Æ1uuÂdJy9Í#òÔ2:ÌFQ3ŒcDÔÕ-"¬–)¸–šBž¥Ç•V½BtÄ$H!(ð -¼¡!¨²Ú`5%j¯ mRà­•öáR!;¯¿þû~,,)ýºüvÙ‰w¬”^ºÊ¥Ù ÜÐ,ÛI_7ÅÀ&À&:øh‡ ‘°è¬ÁH208Žž f tX’ú ¾wsz,¹r,B;(á¾Tæxm¸D%Œ8E§’IºˆYJ>™A–ÇE0®ŠÖÃ)†K²SI¤ b¢Üv˜#g+hÏŽ—·Vÿiê–Ýdr9îħ‘ÂÄûV}•ŠøŠK_[õLv0+Ò–á"h‡†ñ'êæ.úË“‡¥½òÈÞ=x_ö|Ñ“e·wí™Rt™Ïep8_˜Œ$4Ý×ü˜Ž@œ >LB†hTâqæq‘¿“Ä¢P€úÊÚ[Hmô?-$hAsON)T‘x i…=EGq_œ”=ú#xò‘tïíê¶|ÐbÀ__=¸MU§ò,³^²t@“äà G !â$Dw%DŽÿQB¤å"LH[b¤=]¦× úƒ£IÂÅ(]¤¡1Ä£xÜ‘Ä‹¥Ž²GédÈ!YbŽ”#g(= }•]¾~€a”R¨­c˜AfÓÂlqŽnºþq²\|Bù“îQ}²I '<á.ƨ1jI{‘^bµ¿:FCf‹³Ôd]&¬–‹RW©N[[þóüIí‰êçv¨e·ö¿~`?p)Ék­§nW“­Ûù¬Iƒ$õè_À ²DùA0ÁÔA@>üÛRÕ¡S1?èU­Î’ÍSe -Ì8 ÖH"qœÂcüà[Òÿóô‰¬ÉÓhGžgËãe²$c Eµ«Ý„.êXa”ª‡:¢åG8&²…¸®‰Õ±CM¸Ž-|wÄ ÂäÖïZëðÛ,—ô'N6?Ã9Þc pjEÕ¾¾FGˆ‘«žè qYVÀ{“ñƒÅl 1š!!Æ×wî7-®ó“ã·Îh|?6ô„ûÀ³–e7» ²}bƒ0Ä¿•Îãœ6ò÷úÂTÀÒ…¾(ÊW‚â U֢〼Š\€¤¾*·¨é”‹éÊs1>{íªô¥”Ñ´P™Nç(KèrE½+=óMn]IÖúµ­ÏS÷ø¿¨¨¡^ â";@[óv@3|™á&»^òê"Ìöh½ë¦H_«CµøÝ){mè^ƒN;„…£hU %6ÞS5‰ÞßÍe GütÛìD·`õÛ>KæREÍzþþ1™–¨ÄpÚÜÑÓßo¯Ì-ëÕkãœW~óå—=:}.J7ea,ß?¶æðÞ§µßÿ=“Ú -endstream -endobj -1008 0 obj -<< -/Type /FontDescriptor -/FontName /EMBABY+DejaVuSans -/FontFamily (DejaVu Sans) -/Flags 4 -/FontBBox [ 0 -14 557 560 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 560 -/StemV 80 -/StemH 80 -/FontFile2 1004 0 R ->> -endobj -1009 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /EMBABY+DejaVuSans -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 317 ] 16 [ 360 317 ] 20 [ 636 636 ] 25 [ 636 ] 36 [ 684 686 698 ] 40 [ 631 ] 42 [ 774 ] 44 [ 294 ] 46 [ 655 ] 53 [ 694 634 ] 68 [ 612 ] 70 [ 549 634 615 352 ] 75 [ 633 277 ] 78 [ 579 277 974 633 611 ] 85 [ 411 520 392 633 591 ] ] -/FontDescriptor 1008 0 R ->> -endobj -1010 0 obj -<< -/Length 772 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -32 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0015> <0032> -<0003> <0020> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0048> <0065> -<0051> <006e> -<0046> <0063> -<004b> <0068> -<0050> <006d> -<0044> <0061> -<0055> <0072> -<004e> <006b> -<0010> <002d> -<004f> <006c> -<0049> <0066> -<0024> <0041> -<0056> <0073> -<0057> <0074> -<002a> <0047> -<0058> <0075> -<004c> <0069> -<0047> <0064> -<0052> <006f> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1011 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /EMBABY+DejaVuSans -/Encoding /Identity-H -/DescendantFonts [ 1009 0 R ] -/ToUnicode 1010 0 R ->> -endobj -1012 0 obj -<< -/Type /FontDescriptor -/FontName /JFIGPQ+Poppins -/FontFamily (Poppins) -/Flags 4 -/FontBBox [ -27 -132 753 704 ] -/ItalicAngle 0 -/Ascent 1049 -/Descent -349 -/CapHeight 704 -/StemV 80 -/StemH 80 -/FontFile2 1005 0 R ->> -endobj -1013 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /JFIGPQ+Poppins -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 266 ] 5 [ 291 ] 11 [ 453 453 485 682 197 550 209 475 627 319 574 588 628 627 634 545 630 629 212 ] 32 [ 722 ] 36 [ 673 612 771 706 512 503 777 691 245 ] 46 [ 598 431 860 702 785 578 787 607 586 540 674 675 975 ] 60 [ 583 540 422 ] 64 [ 422 ] 66 [ 732 256 675 675 606 675 619 328 675 639 245 247 514 245 1029 639 639 675 675 372 521 363 639 560 819 478 562 454 ] 95 [ 290 ] ] -/FontDescriptor 1012 0 R ->> -endobj -1014 0 obj -<< -/Length 1402 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -77 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0015> <0032> -<0003> <0020> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0048> <0065> -<0051> <006e> -<0046> <0063> -<004b> <0068> -<0050> <006d> -<0044> <0061> -<0055> <0072> -<004e> <006b> -<0010> <002d> -<004f> <006c> -<0049> <0066> -<0024> <0041> -<0056> <0073> -<0057> <0074> -<002a> <0047> -<0058> <0075> -<004c> <0069> -<0047> <0064> -<0016> <0033> -<0045> <0062> -<005a> <0077> -<0053> <0070> -<0052> <006f> -<0030> <004d> -<0031> <004e> -<004a> <0067> -<0029> <0046> -<0033> <0050> -<0017> <0034> -<001c> <0039> -<0018> <0035> -<0013> <0030> -<001b> <0038> -<005d> <007a> -<002f> <004c> -<003a> <0057> -<001a> <0037> -<005c> <0079> -<005b> <0078> -<0037> <0054> -<0027> <0044> -<0032> <004f> -<000f> <002c> -<001d> <003a> -<002b> <0048> -<0039> <0056> -<000e> <002b> -<003c> <0059> -<000b> <0028> -<000c> <0029> -<004d> <006a> -<0054> <0071> -<0012> <002f> -<005f> <007c> -<000d> <002a> -<0020> <003d> -<0042> <005f> -<0043> <0060> -<0005> <0022> -<003e> <005b> -<0040> <005d> -<0038> <0055> -<003d> <005a> -<0034> <0051> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1015 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /JFIGPQ+Poppins -/Encoding /Identity-H -/DescendantFonts [ 1013 0 R ] -/ToUnicode 1014 0 R ->> -endobj -1016 0 obj -<< -/Type /FontDescriptor -/FontName /AHJHWI+DejaVuSans -/FontFamily (DejaVu Sans) -/Flags 4 -/FontBBox [ -9 -92 733 729 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 729 -/StemV 80 -/StemH 80 -/FontFile2 1006 0 R ->> -endobj -1017 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /AHJHWI+DejaVuSans -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 348 ] 11 [ 457 457 ] 16 [ 415 379 365 695 695 695 695 695 695 695 695 695 695 399 ] 36 [ 773 762 733 830 683 ] 44 [ 372 ] 46 [ 774 637 995 836 ] 51 [ 732 ] 53 [ 770 720 682 ] 57 [ 773 1103 ] 60 [ 724 ] 66 [ 500 ] 68 [ 674 715 592 715 678 435 715 711 342 ] 78 [ 665 342 1041 711 687 715 715 493 595 478 711 651 923 645 651 582 ] 5038 [ 741 ] ] -/FontDescriptor 1016 0 R ->> -endobj -1018 0 obj -<< -/Length 1182 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -61 beginbfchar -<0014> <0031> -<0016> <0033> -<0017> <0034> -<0018> <0035> -<0019> <0036> -<001a> <0037> -<0011> <002e> -<0003> <0020> -<0028> <0045> -<0051> <006e> -<0056> <0073> -<0058> <0075> -<0055> <0072> -<0048> <0065> -<0057> <0074> -<004b> <0068> -<0044> <0061> -<0024> <0041> -<0033> <0050> -<002c> <0049> -<0059> <0076> -<0053> <0070> -<0052> <006f> -<0047> <0064> -<0046> <0063> -<004c> <0069> -<13ae> <00660069> -<004f> <006c> -<0050> <006d> -<000b> <0028> -<000c> <0029> -<0015> <0032> -<005a> <0077> -<001d> <003a> -<004a> <0067> -<001b> <0038> -<001c> <0039> -<0026> <0043> -<0031> <004e> -<004e> <006b> -<0049> <0066> -<0030> <004d> -<0013> <0030> -<005c> <0079> -<0010> <002d> -<002e> <004b> -<0045> <0062> -<005d> <007a> -<0035> <0052> -<0025> <0042> -<002f> <004c> -<0036> <0053> -<005b> <0078> -<0027> <0044> -<0054> <0071> -<0039> <0056> -<0037> <0054> -<0042> <005f> -<003a> <0057> -<0012> <002f> -<003c> <0059> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1019 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /AHJHWI+DejaVuSans -/Encoding /Identity-H -/DescendantFonts [ 1017 0 R ] -/ToUnicode 1018 0 R ->> -endobj -1020 0 obj -<< -/Type /FontDescriptor -/FontName /LJVNMQ+DejaVuSansMono -/FontFamily (DejaVu Sans Mono) -/Flags 4 -/FontBBox [ 0 -14 602 728 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 728 -/StemV 80 -/StemH 80 -/FontFile2 1007 0 R ->> -endobj -1021 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /LJVNMQ+DejaVuSansMono -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 601 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 602 ] 36 [ 601 602 602 602 602 602 602 602 602 ] 46 [ 602 602 602 601 602 602 ] 53 [ 602 602 602 602 602 602 602 602 602 602 602 602 ] 66 [ 601 ] 68 [ 601 601 601 601 601 602 602 602 601 602 602 601 601 602 601 601 602 601 601 601 601 602 602 602 602 602 602 602 602 ] ] -/FontDescriptor 1020 0 R ->> -endobj -1022 0 obj -<< -/Length 1570 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -89 beginbfchar -<0031> <004e> -<0052> <006f> -<0057> <0074> -<0003> <0020> -<0024> <0041> -<0053> <0070> -<004f> <006c> -<004c> <0069> -<0046> <0063> -<0044> <0061> -<0045> <0062> -<0048> <0065> -<0058> <0075> -<0050> <006d> -<0047> <0064> -<0056> <0073> -<0055> <0072> -<0010> <002d> -<0008> <0025> -<0012> <002f> -<0059> <0076> -<0051> <006e> -<004b> <0068> -<004e> <006b> -<0015> <0032> -<004a> <0067> -<0049> <0066> -<0011> <002e> -<005c> <0079> -<000a> <0027> -<0019> <0036> -<0017> <0034> -<001e> <003b> -<0038> <0055> -<001d> <003a> -<002a> <0047> -<0054> <0071> -<001f> <003c> -<0021> <003e> -<001a> <0037> -<0013> <0030> -<0042> <005f> -<0006> <0023> -<0004> <0021> -<0037> <0054> -<005a> <0077> -<0007> <0024> -<0014> <0031> -<0020> <003d> -<005b> <0078> -<000b> <0028> -<000d> <002a> -<000c> <0029> -<0005> <0022> -<002b> <0048> -<0036> <0053> -<0028> <0045> -<0035> <0052> -<002c> <0049> -<0033> <0050> -<003e> <005b> -<005e> <007b> -<0060> <007d> -<0040> <005d> -<0030> <004d> -<0032> <004f> -<0029> <0046> -<002f> <004c> -<003f> <005c> -<005f> <007c> -<0009> <0026> -<0018> <0035> -<001b> <0038> -<0022> <003f> -<0016> <0033> -<005d> <007a> -<000f> <002c> -<0025> <0042> -<0026> <0043> -<004d> <006a> -<001c> <0039> -<003b> <0058> -<0027> <0044> -<0039> <0056> -<003d> <005a> -<002e> <004b> -<003c> <0059> -<000e> <002b> -<003a> <0057> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -1023 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /LJVNMQ+DejaVuSansMono -/Encoding /Identity-H -/DescendantFonts [ 1021 0 R ] -/ToUnicode 1022 0 R ->> -endobj -1024 0 obj -<< -/EMBABY 1011 0 R -/JFIGPQ 1015 0 R -/AHJHWI 1019 0 R -/LJVNMQ 1023 0 R ->> -endobj -xref -0 1025 -0000000000 65535 f -0000000015 00000 n -0000001200 00000 n -0000001339 00000 n -0000037576 00000 n -0000037710 00000 n -0000040416 00000 n -0000040615 00000 n -0000044367 00000 n -0000044773 00000 n -0000044982 00000 n -0000045193 00000 n -0000045328 00000 n -0000045464 00000 n -0000045622 00000 n -0000045781 00000 n -0000046014 00000 n -0000046248 00000 n -0000046463 00000 n -0000046679 00000 n -0000046920 00000 n -0000047162 00000 n -0000047385 00000 n -0000047609 00000 n -0000047841 00000 n -0000048074 00000 n -0000048288 00000 n -0000048503 00000 n -0000048730 00000 n -0000048958 00000 n -0000049167 00000 n -0000049377 00000 n -0000049606 00000 n -0000049836 00000 n -0000050047 00000 n -0000050259 00000 n -0000050478 00000 n -0000050697 00000 n -0000054468 00000 n -0000054933 00000 n -0000055134 00000 n -0000055336 00000 n -0000055551 00000 n -0000055767 00000 n -0000055964 00000 n -0000056162 00000 n -0000056381 00000 n -0000056601 00000 n -0000056802 00000 n -0000057004 00000 n -0000057232 00000 n -0000057461 00000 n -0000057671 00000 n -0000057882 00000 n -0000058097 00000 n -0000058313 00000 n -0000058541 00000 n -0000058770 00000 n -0000058970 00000 n -0000059171 00000 n -0000059308 00000 n -0000059446 00000 n -0000059637 00000 n -0000059829 00000 n -0000060019 00000 n -0000060210 00000 n -0000060401 00000 n -0000060593 00000 n -0000060785 00000 n -0000060978 00000 n -0000061217 00000 n -0000061457 00000 n -0000061672 00000 n -0000061888 00000 n -0000062095 00000 n -0000062302 00000 n -0000065877 00000 n -0000066355 00000 n -0000066551 00000 n -0000066748 00000 n -0000066944 00000 n -0000067141 00000 n -0000067337 00000 n -0000067534 00000 n -0000067734 00000 n -0000067935 00000 n -0000068133 00000 n -0000068332 00000 n -0000068566 00000 n -0000068801 00000 n -0000069000 00000 n -0000069200 00000 n -0000069403 00000 n -0000069607 00000 n -0000069809 00000 n -0000070012 00000 n -0000070212 00000 n -0000070413 00000 n -0000070609 00000 n -0000070806 00000 n -0000070995 00000 n -0000071186 00000 n -0000071378 00000 n -0000071571 00000 n -0000071761 00000 n -0000071952 00000 n -0000072138 00000 n -0000072325 00000 n -0000072537 00000 n -0000072750 00000 n -0000072965 00000 n -0000073181 00000 n -0000073394 00000 n -0000073607 00000 n -0000077325 00000 n -0000077828 00000 n -0000078030 00000 n -0000078233 00000 n -0000078435 00000 n -0000078638 00000 n -0000078849 00000 n -0000079061 00000 n -0000079282 00000 n -0000079504 00000 n -0000079733 00000 n -0000079963 00000 n -0000080164 00000 n -0000080366 00000 n -0000080564 00000 n -0000080763 00000 n -0000080976 00000 n -0000081190 00000 n -0000081389 00000 n -0000081589 00000 n -0000081797 00000 n -0000082006 00000 n -0000082152 00000 n -0000082299 00000 n -0000082513 00000 n -0000082728 00000 n -0000082920 00000 n -0000083113 00000 n -0000083324 00000 n -0000083536 00000 n -0000083755 00000 n -0000083975 00000 n -0000084174 00000 n -0000084374 00000 n -0000084582 00000 n -0000084791 00000 n -0000084987 00000 n -0000085183 00000 n -0000089028 00000 n -0000089499 00000 n -0000089636 00000 n -0000089774 00000 n -0000089966 00000 n -0000090159 00000 n -0000090358 00000 n -0000090558 00000 n -0000090715 00000 n -0000090873 00000 n -0000091086 00000 n -0000091300 00000 n -0000091496 00000 n -0000091693 00000 n -0000091885 00000 n -0000092078 00000 n -0000092301 00000 n -0000092525 00000 n -0000092726 00000 n -0000092928 00000 n -0000093125 00000 n -0000093323 00000 n -0000093517 00000 n -0000093712 00000 n -0000093872 00000 n -0000094033 00000 n -0000094230 00000 n -0000094428 00000 n -0000094563 00000 n -0000094699 00000 n -0000094882 00000 n -0000095066 00000 n -0000095258 00000 n -0000095451 00000 n -0000099307 00000 n -0000099794 00000 n -0000099953 00000 n -0000100113 00000 n -0000100334 00000 n -0000100556 00000 n -0000100759 00000 n -0000100963 00000 n -0000101183 00000 n -0000101404 00000 n -0000101605 00000 n -0000101807 00000 n -0000102038 00000 n -0000102270 00000 n -0000102483 00000 n -0000102697 00000 n -0000102923 00000 n -0000103150 00000 n -0000103368 00000 n -0000103587 00000 n -0000103823 00000 n -0000104060 00000 n -0000104278 00000 n -0000104497 00000 n -0000104632 00000 n -0000104768 00000 n -0000104961 00000 n -0000105155 00000 n -0000105366 00000 n -0000105578 00000 n -0000105781 00000 n -0000105985 00000 n -0000106176 00000 n -0000106368 00000 n -0000106582 00000 n -0000106796 00000 n -0000110880 00000 n -0000111367 00000 n -0000111570 00000 n -0000111774 00000 n -0000111979 00000 n -0000112185 00000 n -0000112375 00000 n -0000112566 00000 n -0000112800 00000 n -0000113035 00000 n -0000113266 00000 n -0000113498 00000 n -0000113699 00000 n -0000113901 00000 n -0000114106 00000 n -0000114312 00000 n -0000114517 00000 n -0000114723 00000 n -0000114876 00000 n -0000115030 00000 n -0000115227 00000 n -0000115425 00000 n -0000115586 00000 n -0000115748 00000 n -0000115930 00000 n -0000116113 00000 n -0000116278 00000 n -0000116444 00000 n -0000116641 00000 n -0000116839 00000 n -0000117041 00000 n -0000117244 00000 n -0000117393 00000 n -0000117543 00000 n -0000117728 00000 n -0000117914 00000 n -0000122104 00000 n -0000122591 00000 n -0000122812 00000 n -0000123034 00000 n -0000123248 00000 n -0000123463 00000 n -0000123681 00000 n -0000123900 00000 n -0000124104 00000 n -0000124309 00000 n -0000124488 00000 n -0000124668 00000 n -0000124867 00000 n -0000125067 00000 n -0000125262 00000 n -0000125458 00000 n -0000125656 00000 n -0000125855 00000 n -0000126007 00000 n -0000126160 00000 n -0000126350 00000 n -0000126541 00000 n -0000126735 00000 n -0000126930 00000 n -0000127076 00000 n -0000127223 00000 n -0000127425 00000 n -0000127628 00000 n -0000127795 00000 n -0000127963 00000 n -0000128119 00000 n -0000128276 00000 n -0000128483 00000 n -0000128691 00000 n -0000128919 00000 n -0000129148 00000 n -0000131463 00000 n -0000131758 00000 n -0000131902 00000 n -0000132047 00000 n -0000132249 00000 n -0000132452 00000 n -0000132666 00000 n -0000132881 00000 n -0000133069 00000 n -0000133258 00000 n -0000133433 00000 n -0000133609 00000 n -0000138442 00000 n -0000138697 00000 n -0000138958 00000 n -0000139226 00000 n -0000139493 00000 n -0000139700 00000 n -0000139906 00000 n -0000143668 00000 n -0000143899 00000 n -0000144097 00000 n -0000144286 00000 n -0000148668 00000 n -0000148899 00000 n -0000149033 00000 n -0000149167 00000 n -0000153295 00000 n -0000153526 00000 n -0000153660 00000 n -0000153795 00000 n -0000157936 00000 n -0000158167 00000 n -0000158302 00000 n -0000158437 00000 n -0000162498 00000 n -0000162729 00000 n -0000162864 00000 n -0000162999 00000 n -0000167078 00000 n -0000167317 00000 n -0000167452 00000 n -0000167587 00000 n -0000167722 00000 n -0000171667 00000 n -0000171898 00000 n -0000172033 00000 n -0000172168 00000 n -0000176112 00000 n -0000176343 00000 n -0000176478 00000 n -0000176613 00000 n -0000180573 00000 n -0000180812 00000 n -0000180947 00000 n -0000181082 00000 n -0000181217 00000 n -0000185470 00000 n -0000185725 00000 n -0000185860 00000 n -0000185995 00000 n -0000186128 00000 n -0000186261 00000 n -0000186394 00000 n -0000191061 00000 n -0000191580 00000 n -0000191713 00000 n -0000191846 00000 n -0000191979 00000 n -0000192112 00000 n -0000192245 00000 n -0000192379 00000 n -0000192513 00000 n -0000192647 00000 n -0000192781 00000 n -0000192915 00000 n -0000193049 00000 n -0000193183 00000 n -0000193317 00000 n -0000193451 00000 n -0000193585 00000 n -0000193719 00000 n -0000193853 00000 n -0000193987 00000 n -0000194121 00000 n -0000194255 00000 n -0000194389 00000 n -0000194523 00000 n -0000194657 00000 n -0000194791 00000 n -0000194925 00000 n -0000195059 00000 n -0000195193 00000 n -0000195327 00000 n -0000195461 00000 n -0000195595 00000 n -0000195729 00000 n -0000195863 00000 n -0000195997 00000 n -0000196131 00000 n -0000196265 00000 n -0000196399 00000 n -0000196532 00000 n -0000196664 00000 n -0000201814 00000 n -0000202213 00000 n -0000202347 00000 n -0000202481 00000 n -0000202615 00000 n -0000202749 00000 n -0000202883 00000 n -0000203017 00000 n -0000203151 00000 n -0000203285 00000 n -0000203419 00000 n -0000203553 00000 n -0000203687 00000 n -0000203821 00000 n -0000203955 00000 n -0000204089 00000 n -0000204223 00000 n -0000204357 00000 n -0000204491 00000 n -0000204625 00000 n -0000204759 00000 n -0000204894 00000 n -0000205029 00000 n -0000205162 00000 n -0000205295 00000 n -0000209925 00000 n -0000210436 00000 n -0000210569 00000 n -0000210702 00000 n -0000210835 00000 n -0000210968 00000 n -0000211101 00000 n -0000211234 00000 n -0000211368 00000 n -0000211502 00000 n -0000211636 00000 n -0000211770 00000 n -0000211904 00000 n -0000212038 00000 n -0000212172 00000 n -0000212306 00000 n -0000212440 00000 n -0000212574 00000 n -0000212708 00000 n -0000212842 00000 n -0000212976 00000 n -0000213110 00000 n -0000213244 00000 n -0000213378 00000 n -0000213512 00000 n -0000213646 00000 n -0000213780 00000 n -0000213914 00000 n -0000214048 00000 n -0000214182 00000 n -0000214316 00000 n -0000214450 00000 n -0000214584 00000 n -0000214718 00000 n -0000214852 00000 n -0000214986 00000 n -0000215120 00000 n -0000215253 00000 n -0000215385 00000 n -0000220167 00000 n -0000220558 00000 n -0000220692 00000 n -0000220826 00000 n -0000220960 00000 n -0000221094 00000 n -0000221228 00000 n -0000221362 00000 n -0000221496 00000 n -0000221630 00000 n -0000221764 00000 n -0000221898 00000 n -0000222032 00000 n -0000222166 00000 n -0000222300 00000 n -0000222434 00000 n -0000222568 00000 n -0000222702 00000 n -0000222836 00000 n -0000222970 00000 n -0000223104 00000 n -0000223238 00000 n -0000223372 00000 n -0000223507 00000 n -0000228216 00000 n -0000228447 00000 n -0000228582 00000 n -0000228717 00000 n -0000233361 00000 n -0000233564 00000 n -0000238118 00000 n -0000238341 00000 n -0000238476 00000 n -0000243155 00000 n -0000243358 00000 n -0000247953 00000 n -0000248176 00000 n -0000248311 00000 n -0000253009 00000 n -0000253212 00000 n -0000257722 00000 n -0000257925 00000 n -0000262760 00000 n -0000262983 00000 n -0000263118 00000 n -0000267531 00000 n -0000267734 00000 n -0000272610 00000 n -0000272833 00000 n -0000272968 00000 n -0000277431 00000 n -0000277634 00000 n -0000282560 00000 n -0000282783 00000 n -0000282918 00000 n -0000287344 00000 n -0000287547 00000 n -0000292458 00000 n -0000292681 00000 n -0000292816 00000 n -0000297278 00000 n -0000297481 00000 n -0000302360 00000 n -0000302583 00000 n -0000302718 00000 n -0000307275 00000 n -0000307478 00000 n -0000311999 00000 n -0000312230 00000 n -0000312365 00000 n -0000312499 00000 n -0000317203 00000 n -0000317406 00000 n -0000321991 00000 n -0000322194 00000 n -0000326696 00000 n -0000326935 00000 n -0000327070 00000 n -0000327205 00000 n -0000327340 00000 n -0000332035 00000 n -0000332238 00000 n -0000336833 00000 n -0000337056 00000 n -0000337191 00000 n -0000341864 00000 n -0000342067 00000 n -0000346591 00000 n -0000346794 00000 n -0000351599 00000 n -0000351822 00000 n -0000351957 00000 n -0000356458 00000 n -0000356661 00000 n -0000361519 00000 n -0000361742 00000 n -0000361878 00000 n -0000366337 00000 n -0000366540 00000 n -0000371391 00000 n -0000371614 00000 n -0000371750 00000 n -0000376243 00000 n -0000376446 00000 n -0000381316 00000 n -0000381539 00000 n -0000381675 00000 n -0000386152 00000 n -0000386355 00000 n -0000391267 00000 n -0000391490 00000 n -0000391626 00000 n -0000396081 00000 n -0000396284 00000 n -0000401201 00000 n -0000401424 00000 n -0000401560 00000 n -0000406030 00000 n -0000406233 00000 n -0000411071 00000 n -0000411294 00000 n -0000411430 00000 n -0000416043 00000 n -0000416246 00000 n -0000420874 00000 n -0000421097 00000 n -0000421233 00000 n -0000425940 00000 n -0000426143 00000 n -0000430578 00000 n -0000430801 00000 n -0000430937 00000 n -0000435610 00000 n -0000435813 00000 n -0000440486 00000 n -0000440709 00000 n -0000440845 00000 n -0000445531 00000 n -0000445734 00000 n -0000450229 00000 n -0000450432 00000 n -0000455269 00000 n -0000455492 00000 n -0000455628 00000 n -0000460030 00000 n -0000460233 00000 n -0000465140 00000 n -0000465363 00000 n -0000465499 00000 n -0000469962 00000 n -0000470165 00000 n -0000475043 00000 n -0000475266 00000 n -0000475402 00000 n -0000479873 00000 n -0000480076 00000 n -0000485029 00000 n -0000485252 00000 n -0000485388 00000 n -0000489849 00000 n -0000490052 00000 n -0000494952 00000 n -0000495175 00000 n -0000495311 00000 n -0000499924 00000 n -0000500127 00000 n -0000504733 00000 n -0000504956 00000 n -0000505092 00000 n -0000509787 00000 n -0000509990 00000 n -0000514554 00000 n -0000514777 00000 n -0000514913 00000 n -0000519586 00000 n -0000519789 00000 n -0000524467 00000 n -0000524690 00000 n -0000524825 00000 n -0000529495 00000 n -0000529698 00000 n -0000534172 00000 n -0000534375 00000 n -0000539254 00000 n -0000539485 00000 n -0000539621 00000 n -0000539757 00000 n -0000544403 00000 n -0000544606 00000 n -0000548146 00000 n -0000548349 00000 n -0000553088 00000 n -0000553311 00000 n -0000553447 00000 n -0000558117 00000 n -0000558340 00000 n -0000558476 00000 n -0000563207 00000 n -0000563430 00000 n -0000563566 00000 n -0000568236 00000 n -0000568459 00000 n -0000568595 00000 n -0000573022 00000 n -0000573245 00000 n -0000573381 00000 n -0000577861 00000 n -0000578064 00000 n -0000582670 00000 n -0000582893 00000 n -0000583029 00000 n -0000585864 00000 n -0000586067 00000 n -0000590542 00000 n -0000590773 00000 n -0000590909 00000 n -0000591045 00000 n -0000594348 00000 n -0000594551 00000 n -0000598988 00000 n -0000599211 00000 n -0000599347 00000 n -0000603995 00000 n -0000604198 00000 n -0000608477 00000 n -0000608700 00000 n -0000608836 00000 n -0000613550 00000 n -0000613753 00000 n -0000618781 00000 n -0000619004 00000 n -0000619140 00000 n -0000623475 00000 n -0000623678 00000 n -0000626959 00000 n -0000627162 00000 n -0000631154 00000 n -0000631377 00000 n -0000631513 00000 n -0000635731 00000 n -0000635954 00000 n -0000636090 00000 n -0000640224 00000 n -0000640455 00000 n -0000640591 00000 n -0000640727 00000 n -0000644966 00000 n -0000645205 00000 n -0000645341 00000 n -0000645477 00000 n -0000645612 00000 n -0000649726 00000 n -0000649957 00000 n -0000650093 00000 n -0000650229 00000 n -0000653792 00000 n -0000654015 00000 n -0000654151 00000 n -0000659100 00000 n -0000659331 00000 n -0000659467 00000 n -0000659603 00000 n -0000664429 00000 n -0000664660 00000 n -0000664796 00000 n -0000664932 00000 n -0000669366 00000 n -0000669597 00000 n -0000669733 00000 n -0000669869 00000 n -0000674743 00000 n -0000674974 00000 n -0000675110 00000 n -0000675246 00000 n -0000680133 00000 n -0000680364 00000 n -0000680500 00000 n -0000680636 00000 n -0000685548 00000 n -0000685779 00000 n -0000685915 00000 n -0000686050 00000 n -0000690659 00000 n -0000690890 00000 n -0000691026 00000 n -0000691162 00000 n -0000696029 00000 n -0000696252 00000 n -0000696388 00000 n -0000701266 00000 n -0000701497 00000 n -0000701633 00000 n -0000701769 00000 n -0000706695 00000 n -0000706926 00000 n -0000707062 00000 n -0000707198 00000 n -0000712140 00000 n -0000712371 00000 n -0000712507 00000 n -0000712641 00000 n -0000717637 00000 n -0000717840 00000 n -0000722780 00000 n -0000723011 00000 n -0000723147 00000 n -0000723283 00000 n -0000728199 00000 n -0000728430 00000 n -0000728566 00000 n -0000728702 00000 n -0000732382 00000 n -0000732585 00000 n -0000736668 00000 n -0000736871 00000 n -0000742410 00000 n -0000742809 00000 n -0000742945 00000 n -0000743079 00000 n -0000743213 00000 n -0000743347 00000 n -0000743481 00000 n -0000743615 00000 n -0000743749 00000 n -0000743883 00000 n -0000744017 00000 n -0000744152 00000 n -0000744287 00000 n -0000744422 00000 n -0000744557 00000 n -0000744692 00000 n -0000744827 00000 n -0000744962 00000 n -0000745097 00000 n -0000745232 00000 n -0000745367 00000 n -0000745502 00000 n -0000745637 00000 n -0000745772 00000 n -0000745906 00000 n -0000749634 00000 n -0000749921 00000 n -0000750056 00000 n -0000750191 00000 n -0000750326 00000 n -0000750461 00000 n -0000750596 00000 n -0000750731 00000 n -0000750866 00000 n -0000751001 00000 n -0000751137 00000 n -0000755806 00000 n -0000756037 00000 n -0000756173 00000 n -0000756309 00000 n -0000760802 00000 n -0000761033 00000 n -0000761169 00000 n -0000761305 00000 n -0000765773 00000 n -0000766004 00000 n -0000766140 00000 n -0000766276 00000 n -0000770336 00000 n -0000770559 00000 n -0000770695 00000 n -0000775432 00000 n -0000775695 00000 n -0000775831 00000 n -0000775967 00000 n -0000776101 00000 n -0000776235 00000 n -0000776368 00000 n -0000776500 00000 n -0000780731 00000 n -0000781066 00000 n -0000781200 00000 n -0000781334 00000 n -0000781468 00000 n -0000781602 00000 n -0000781737 00000 n -0000781872 00000 n -0000782007 00000 n -0000782142 00000 n -0000782277 00000 n -0000782412 00000 n -0000782547 00000 n -0000782682 00000 n -0000782817 00000 n -0000782952 00000 n -0000783088 00000 n -0000786510 00000 n -0000786713 00000 n -0000789978 00000 n -0000790181 00000 n -0000793251 00000 n -0000793454 00000 n -0000797857 00000 n -0000798060 00000 n -0000798256 00000 n -0000798486 00000 n -0000798612 00000 n -0000798758 00000 n -0000798898 00000 n -0000799095 00000 n -0000799329 00000 n -0000799560 00000 n -0000799816 00000 n -0000800055 00000 n -0000800302 00000 n -0000800532 00000 n -0000800774 00000 n -0000800999 00000 n -0000801243 00000 n -0000801471 00000 n -0000801706 00000 n -0000801924 00000 n -0000802155 00000 n -0000802369 00000 n -0000802604 00000 n -0000802822 00000 n -0000803066 00000 n -0000803293 00000 n -0000803525 00000 n -0000803769 00000 n -0000803971 00000 n -0000804147 00000 n -0000804339 00000 n -0000804544 00000 n -0000804750 00000 n -0000804957 00000 n -0000805211 00000 n -0000805441 00000 n -0000805664 00000 n -0000805875 00000 n -0000806086 00000 n -0000806298 00000 n -0000806514 00000 n -0000806728 00000 n -0000806978 00000 n -0000807193 00000 n -0000807412 00000 n -0000807630 00000 n -0000807846 00000 n -0000808058 00000 n -0000808263 00000 n -0000808470 00000 n -0000808675 00000 n -0000808876 00000 n -0000809103 00000 n -0000809333 00000 n -0000809562 00000 n -0000809779 00000 n -0000809996 00000 n -0000810222 00000 n -0000810458 00000 n -0000810702 00000 n -0000810918 00000 n -0000811131 00000 n -0000811359 00000 n -0000811573 00000 n -0000811782 00000 n -0000811965 00000 n -0000812179 00000 n -0000812383 00000 n -0000812608 00000 n -0000812841 00000 n -0000813054 00000 n -0000813276 00000 n -0000813473 00000 n -0000813647 00000 n -0000813837 00000 n -0000814034 00000 n -0000814226 00000 n -0000814437 00000 n -0000814645 00000 n -0000814849 00000 n -0000815084 00000 n -0000815297 00000 n -0000815506 00000 n -0000815698 00000 n -0000815895 00000 n -0000816078 00000 n -0000816250 00000 n -0000816433 00000 n -0000816625 00000 n -0000816822 00000 n -0000817043 00000 n -0000817261 00000 n -0000817495 00000 n -0000817711 00000 n -0000817956 00000 n -0000818184 00000 n -0000818424 00000 n -0000818657 00000 n -0000818907 00000 n -0000819127 00000 n -0000819300 00000 n -0000819493 00000 n -0000819716 00000 n -0000819931 00000 n -0000820136 00000 n -0000820364 00000 n -0000820581 00000 n -0000820800 00000 n -0000821004 00000 n -0000821252 00000 n -0000821496 00000 n -0000821712 00000 n -0000821932 00000 n -0000822138 00000 n -0000822328 00000 n -0000822525 00000 n -0000822700 00000 n -0000822896 00000 n -0000823075 00000 n -0000823286 00000 n -0000823488 00000 n -0000823674 00000 n -0000823859 00000 n -0000824094 00000 n -0000824322 00000 n -0000824554 00000 n -0000824772 00000 n -0000824965 00000 n -0000825178 00000 n -0000825387 00000 n -0000825585 00000 n -0000825774 00000 n -0000825964 00000 n -0000826158 00000 n -0000826341 00000 n -0000826543 00000 n -0000826710 00000 n -0000826903 00000 n -0000827096 00000 n -0000827333 00000 n -0000827501 00000 n -0000827704 00000 n -0000827935 00000 n -0000828140 00000 n -0000828317 00000 n -0000828381 00000 n -0000836324 00000 n -0000840962 00000 n -0000852820 00000 n -0000865030 00000 n -0000865264 00000 n -0000865690 00000 n -0000866516 00000 n -0000866669 00000 n -0000866900 00000 n -0000867468 00000 n -0000868925 00000 n -0000869075 00000 n -0000869310 00000 n -0000869849 00000 n -0000871086 00000 n -0000871239 00000 n -0000871482 00000 n -0000872077 00000 n -0000873702 00000 n -0000873859 00000 n -trailer -<< -/Size 1025 -/Root 3 0 R -/Info 2 0 R ->> -startxref -873951 -%%EOF diff --git a/versioned_docs/version-2.6/security/hardening-guides/rke2-1.6-hardening-2.6/Rancher_RKE2_v2-6_CIS_v1-6_Hardening_Guide.pdf b/versioned_docs/version-2.6/security/hardening-guides/rke2-1.6-hardening-2.6/Rancher_RKE2_v2-6_CIS_v1-6_Hardening_Guide.pdf deleted file mode 100644 index 35251edd3d0..00000000000 --- a/versioned_docs/version-2.6/security/hardening-guides/rke2-1.6-hardening-2.6/Rancher_RKE2_v2-6_CIS_v1-6_Hardening_Guide.pdf +++ /dev/null @@ -1,4149 +0,0 @@ -%PDF-1.7 -%🖤 -1 0 obj -<< -/Type /Pages -/Kids [ 6 0 R 8 0 R 26 0 R 39 0 R 41 0 R 49 0 R 56 0 R 59 0 R 72 0 R 81 0 R 85 0 R 125 0 R 165 0 R 205 0 R 245 0 R ] -/Count 15 ->> -endobj -2 0 obj -<< -/Title (RKE2 Hardening Guide with CIS v1.6 Benchmark) -/Creator (pandoc) -/Producer (WeasyPrint 54.3) ->> -endobj -3 0 obj -<< -/Type /Catalog -/Pages 1 0 R -/Outlines 277 0 R -/Names << -/Dests << -/Names [ (TOC) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (api-server-audit-configuration) [ 59 0 R /XYZ 84.259843 315.158799 0 ] (cb1) [ 41 0 R /XYZ 84.259843 617.450787 0 ] (cb1-1) [ 41 0 R /XYZ 101.433071 615.753766 0 ] (cb1-2) [ 41 0 R /XYZ 99.933071 596.806744 0 ] (cb1-3) [ 41 0 R /XYZ 99.933071 577.859723 0 ] (cb1-4) [ 41 0 R /XYZ 99.933071 558.912701 0 ] (cb1-5) [ 41 0 R /XYZ 99.933071 539.965680 0 ] (cb1-6) [ 41 0 R /XYZ 99.933071 521.018658 0 ] (cb2) [ 49 0 R /XYZ 84.259843 687.872079 0 ] (cb2-1) [ 49 0 R /XYZ 101.433071 686.175057 0 ] (cb2-2) [ 49 0 R /XYZ 99.933071 667.228036 0 ] (cb2-3) [ 49 0 R /XYZ 99.933071 648.281014 0 ] (cb2-4) [ 49 0 R /XYZ 99.933071 629.333993 0 ] (cb3) [ 49 0 R /XYZ 84.259843 389.087993 0 ] (cb3-1) [ 49 0 R /XYZ 101.433071 387.390971 0 ] (cb4) [ 56 0 R /XYZ 84.259843 110.200913 0 ] (cb4-1) [ 56 0 R /XYZ 101.433071 108.503892 0 ] (cb5) [ 59 0 R /XYZ 84.259843 692.870079 0 ] (cb5-1) [ 59 0 R /XYZ 101.433071 691.173057 0 ] (cb5-2) [ 59 0 R /XYZ 99.933071 672.226036 0 ] (cb5-3) [ 59 0 R /XYZ 99.933071 653.279014 0 ] (cb5-4) [ 59 0 R /XYZ 99.933071 634.331993 0 ] (cb5-5) [ 59 0 R /XYZ 99.933071 615.384971 0 ] (cb6) [ 59 0 R /XYZ 84.259843 533.634971 0 ] (cb6-1) [ 59 0 R /XYZ 101.433071 531.937950 0 ] (cb6-2) [ 59 0 R /XYZ 99.933071 512.990928 0 ] (cb6-3) [ 59 0 R /XYZ 99.933071 494.043907 0 ] (cb6-4) [ 59 0 R /XYZ 99.933071 456.149864 0 ] (cb6-5) [ 59 0 R /XYZ 99.933071 437.202842 0 ] (cb6-6) [ 59 0 R /XYZ 99.933071 399.308799 0 ] (cb7) [ 72 0 R /XYZ 84.259843 799.370079 0 ] (cb7-1) [ 72 0 R /XYZ 101.433071 797.673057 0 ] (cb7-2) [ 72 0 R /XYZ 99.933071 778.726036 0 ] (cb7-3) [ 72 0 R /XYZ 99.933071 759.779014 0 ] (cb7-4) [ 72 0 R /XYZ 99.933071 740.831993 0 ] (cb7-5) [ 72 0 R /XYZ 99.933071 721.884971 0 ] (cb7-6) [ 72 0 R /XYZ 99.933071 702.937950 0 ] (cb9) [ 81 0 R /XYZ 84.259843 126.824079 0 ] (cb9-1) [ 81 0 R /XYZ 101.433071 125.127057 0 ] (cb9-10) [ 85 0 R /XYZ 99.933071 666.543907 0 ] (cb9-100) [ 165 0 R /XYZ 99.933071 401.285606 0 ] (cb9-101) [ 165 0 R /XYZ 99.933071 382.338585 0 ] (cb9-102) [ 165 0 R /XYZ 99.933071 363.391563 0 ] (cb9-103) [ 165 0 R /XYZ 99.933071 344.444542 0 ] (cb9-104) [ 165 0 R /XYZ 99.933071 325.497520 0 ] (cb9-105) [ 165 0 R /XYZ 99.933071 306.550499 0 ] (cb9-106) [ 165 0 R /XYZ 99.933071 287.603477 0 ] (cb9-107) [ 165 0 R /XYZ 99.933071 268.656456 0 ] (cb9-108) [ 165 0 R /XYZ 99.933071 249.709434 0 ] (cb9-109) [ 165 0 R /XYZ 99.933071 230.762413 0 ] (cb9-11) [ 85 0 R /XYZ 99.933071 647.596885 0 ] (cb9-110) [ 165 0 R /XYZ 99.933071 211.815391 0 ] (cb9-111) [ 165 0 R /XYZ 99.933071 192.868370 0 ] (cb9-112) [ 165 0 R /XYZ 99.933071 173.921348 0 ] (cb9-113) [ 165 0 R /XYZ 99.933071 154.974327 0 ] (cb9-114) [ 165 0 R /XYZ 99.933071 136.027305 0 ] (cb9-115) [ 165 0 R /XYZ 99.933071 117.080284 0 ] (cb9-116) [ 165 0 R /XYZ 99.933071 98.133262 0 ] (cb9-117) [ 205 0 R /XYZ 99.933071 799.173057 0 ] (cb9-118) [ 205 0 R /XYZ 99.933071 780.226036 0 ] (cb9-119) [ 205 0 R /XYZ 99.933071 761.279014 0 ] (cb9-12) [ 85 0 R /XYZ 99.933071 628.649864 0 ] (cb9-120) [ 205 0 R /XYZ 99.933071 742.331993 0 ] (cb9-121) [ 205 0 R /XYZ 99.933071 723.384971 0 ] (cb9-122) [ 205 0 R /XYZ 99.933071 704.437950 0 ] (cb9-123) [ 205 0 R /XYZ 99.933071 685.490928 0 ] (cb9-124) [ 205 0 R /XYZ 99.933071 666.543907 0 ] (cb9-125) [ 205 0 R /XYZ 99.933071 647.596885 0 ] (cb9-126) [ 205 0 R /XYZ 99.933071 628.649864 0 ] (cb9-127) [ 205 0 R /XYZ 99.933071 609.702842 0 ] (cb9-128) [ 205 0 R /XYZ 99.933071 590.755821 0 ] (cb9-129) [ 205 0 R /XYZ 99.933071 571.808799 0 ] (cb9-13) [ 85 0 R /XYZ 99.933071 609.702842 0 ] (cb9-130) [ 205 0 R /XYZ 99.933071 552.861778 0 ] (cb9-131) [ 205 0 R /XYZ 99.933071 533.914756 0 ] (cb9-132) [ 205 0 R /XYZ 99.933071 514.967735 0 ] (cb9-133) [ 205 0 R /XYZ 99.933071 496.020714 0 ] (cb9-134) [ 205 0 R /XYZ 99.933071 477.073692 0 ] (cb9-135) [ 205 0 R /XYZ 99.933071 458.126671 0 ] (cb9-136) [ 205 0 R /XYZ 99.933071 439.179649 0 ] (cb9-137) [ 205 0 R /XYZ 99.933071 420.232628 0 ] (cb9-138) [ 205 0 R /XYZ 99.933071 401.285606 0 ] (cb9-139) [ 205 0 R /XYZ 99.933071 382.338585 0 ] (cb9-14) [ 85 0 R /XYZ 99.933071 590.755821 0 ] (cb9-140) [ 205 0 R /XYZ 99.933071 363.391563 0 ] (cb9-141) [ 205 0 R /XYZ 99.933071 344.444542 0 ] (cb9-142) [ 205 0 R /XYZ 99.933071 325.497520 0 ] (cb9-143) [ 205 0 R /XYZ 99.933071 306.550499 0 ] (cb9-144) [ 205 0 R /XYZ 99.933071 287.603477 0 ] (cb9-145) [ 205 0 R /XYZ 99.933071 268.656456 0 ] (cb9-146) [ 205 0 R /XYZ 99.933071 249.709434 0 ] (cb9-147) [ 205 0 R /XYZ 99.933071 230.762413 0 ] (cb9-148) [ 205 0 R /XYZ 99.933071 211.815391 0 ] (cb9-149) [ 205 0 R /XYZ 99.933071 192.868370 0 ] (cb9-15) [ 85 0 R /XYZ 99.933071 571.808799 0 ] (cb9-150) [ 205 0 R /XYZ 99.933071 173.921348 0 ] (cb9-151) [ 205 0 R /XYZ 99.933071 154.974327 0 ] (cb9-152) [ 205 0 R /XYZ 99.933071 136.027305 0 ] (cb9-153) [ 205 0 R /XYZ 99.933071 117.080284 0 ] (cb9-154) [ 205 0 R /XYZ 99.933071 98.133262 0 ] (cb9-155) [ 245 0 R /XYZ 99.933071 799.173057 0 ] (cb9-156) [ 245 0 R /XYZ 99.933071 780.226036 0 ] (cb9-157) [ 245 0 R /XYZ 99.933071 761.279014 0 ] (cb9-158) [ 245 0 R /XYZ 99.933071 742.331993 0 ] (cb9-159) [ 245 0 R /XYZ 99.933071 723.384971 0 ] (cb9-16) [ 85 0 R /XYZ 99.933071 552.861778 0 ] (cb9-160) [ 245 0 R /XYZ 99.933071 704.437950 0 ] (cb9-161) [ 245 0 R /XYZ 99.933071 685.490928 0 ] (cb9-162) [ 245 0 R /XYZ 99.933071 666.543907 0 ] (cb9-163) [ 245 0 R /XYZ 99.933071 647.596885 0 ] (cb9-164) [ 245 0 R /XYZ 99.933071 628.649864 0 ] (cb9-165) [ 245 0 R /XYZ 99.933071 609.702842 0 ] (cb9-17) [ 85 0 R /XYZ 99.933071 533.914756 0 ] (cb9-18) [ 85 0 R /XYZ 99.933071 514.967735 0 ] (cb9-19) [ 85 0 R /XYZ 99.933071 496.020714 0 ] (cb9-2) [ 81 0 R /XYZ 99.933071 106.180036 0 ] (cb9-20) [ 85 0 R /XYZ 99.933071 477.073692 0 ] (cb9-21) [ 85 0 R /XYZ 99.933071 458.126671 0 ] (cb9-22) [ 85 0 R /XYZ 99.933071 439.179649 0 ] (cb9-23) [ 85 0 R /XYZ 99.933071 420.232628 0 ] (cb9-24) [ 85 0 R /XYZ 99.933071 401.285606 0 ] (cb9-25) [ 85 0 R /XYZ 99.933071 382.338585 0 ] (cb9-26) [ 85 0 R /XYZ 99.933071 363.391563 0 ] (cb9-27) [ 85 0 R /XYZ 99.933071 344.444542 0 ] (cb9-28) [ 85 0 R /XYZ 99.933071 325.497520 0 ] (cb9-29) [ 85 0 R /XYZ 99.933071 306.550499 0 ] (cb9-3) [ 85 0 R /XYZ 99.933071 799.173057 0 ] (cb9-30) [ 85 0 R /XYZ 99.933071 287.603477 0 ] (cb9-31) [ 85 0 R /XYZ 99.933071 268.656456 0 ] (cb9-32) [ 85 0 R /XYZ 99.933071 249.709434 0 ] (cb9-33) [ 85 0 R /XYZ 99.933071 230.762413 0 ] (cb9-34) [ 85 0 R /XYZ 99.933071 211.815391 0 ] (cb9-35) [ 85 0 R /XYZ 99.933071 192.868370 0 ] (cb9-36) [ 85 0 R /XYZ 99.933071 173.921348 0 ] (cb9-37) [ 85 0 R /XYZ 99.933071 154.974327 0 ] (cb9-38) [ 85 0 R /XYZ 99.933071 136.027305 0 ] (cb9-39) [ 85 0 R /XYZ 99.933071 117.080284 0 ] (cb9-4) [ 85 0 R /XYZ 99.933071 780.226036 0 ] (cb9-40) [ 85 0 R /XYZ 99.933071 98.133262 0 ] (cb9-41) [ 125 0 R /XYZ 99.933071 799.173057 0 ] (cb9-42) [ 125 0 R /XYZ 99.933071 780.226036 0 ] (cb9-43) [ 125 0 R /XYZ 99.933071 761.279014 0 ] (cb9-44) [ 125 0 R /XYZ 99.933071 742.331993 0 ] (cb9-45) [ 125 0 R /XYZ 99.933071 723.384971 0 ] (cb9-46) [ 125 0 R /XYZ 99.933071 704.437950 0 ] (cb9-47) [ 125 0 R /XYZ 99.933071 685.490928 0 ] (cb9-48) [ 125 0 R /XYZ 99.933071 666.543907 0 ] (cb9-49) [ 125 0 R /XYZ 99.933071 647.596885 0 ] (cb9-5) [ 85 0 R /XYZ 99.933071 761.279014 0 ] (cb9-50) [ 125 0 R /XYZ 99.933071 628.649864 0 ] (cb9-51) [ 125 0 R /XYZ 99.933071 609.702842 0 ] (cb9-52) [ 125 0 R /XYZ 99.933071 590.755821 0 ] (cb9-53) [ 125 0 R /XYZ 99.933071 571.808799 0 ] (cb9-54) [ 125 0 R /XYZ 99.933071 552.861778 0 ] (cb9-55) [ 125 0 R /XYZ 99.933071 533.914756 0 ] (cb9-56) [ 125 0 R /XYZ 99.933071 514.967735 0 ] (cb9-57) [ 125 0 R /XYZ 99.933071 496.020714 0 ] (cb9-58) [ 125 0 R /XYZ 99.933071 477.073692 0 ] (cb9-59) [ 125 0 R /XYZ 99.933071 458.126671 0 ] (cb9-6) [ 85 0 R /XYZ 99.933071 742.331993 0 ] (cb9-60) [ 125 0 R /XYZ 99.933071 439.179649 0 ] (cb9-61) [ 125 0 R /XYZ 99.933071 420.232628 0 ] (cb9-62) [ 125 0 R /XYZ 99.933071 401.285606 0 ] (cb9-63) [ 125 0 R /XYZ 99.933071 382.338585 0 ] (cb9-64) [ 125 0 R /XYZ 99.933071 363.391563 0 ] (cb9-65) [ 125 0 R /XYZ 99.933071 344.444542 0 ] (cb9-66) [ 125 0 R /XYZ 99.933071 325.497520 0 ] (cb9-67) [ 125 0 R /XYZ 99.933071 306.550499 0 ] (cb9-68) [ 125 0 R /XYZ 99.933071 287.603477 0 ] (cb9-69) [ 125 0 R /XYZ 99.933071 268.656456 0 ] (cb9-7) [ 85 0 R /XYZ 99.933071 723.384971 0 ] (cb9-70) [ 125 0 R /XYZ 99.933071 249.709434 0 ] (cb9-71) [ 125 0 R /XYZ 99.933071 230.762413 0 ] (cb9-72) [ 125 0 R /XYZ 99.933071 211.815391 0 ] (cb9-73) [ 125 0 R /XYZ 99.933071 192.868370 0 ] (cb9-74) [ 125 0 R /XYZ 99.933071 173.921348 0 ] (cb9-75) [ 125 0 R /XYZ 99.933071 154.974327 0 ] (cb9-76) [ 125 0 R /XYZ 99.933071 136.027305 0 ] (cb9-77) [ 125 0 R /XYZ 99.933071 117.080284 0 ] (cb9-78) [ 125 0 R /XYZ 99.933071 98.133262 0 ] (cb9-79) [ 165 0 R /XYZ 99.933071 799.173057 0 ] (cb9-8) [ 85 0 R /XYZ 99.933071 704.437950 0 ] (cb9-80) [ 165 0 R /XYZ 99.933071 780.226036 0 ] (cb9-81) [ 165 0 R /XYZ 99.933071 761.279014 0 ] (cb9-82) [ 165 0 R /XYZ 99.933071 742.331993 0 ] (cb9-83) [ 165 0 R /XYZ 99.933071 723.384971 0 ] (cb9-84) [ 165 0 R /XYZ 99.933071 704.437950 0 ] (cb9-85) [ 165 0 R /XYZ 99.933071 685.490928 0 ] (cb9-86) [ 165 0 R /XYZ 99.933071 666.543907 0 ] (cb9-87) [ 165 0 R /XYZ 99.933071 647.596885 0 ] (cb9-88) [ 165 0 R /XYZ 99.933071 628.649864 0 ] (cb9-89) [ 165 0 R /XYZ 99.933071 609.702842 0 ] (cb9-9) [ 85 0 R /XYZ 99.933071 685.490928 0 ] (cb9-90) [ 165 0 R /XYZ 99.933071 590.755821 0 ] (cb9-91) [ 165 0 R /XYZ 99.933071 571.808799 0 ] (cb9-92) [ 165 0 R /XYZ 99.933071 552.861778 0 ] (cb9-93) [ 165 0 R /XYZ 99.933071 533.914756 0 ] (cb9-94) [ 165 0 R /XYZ 99.933071 514.967735 0 ] (cb9-95) [ 165 0 R /XYZ 99.933071 496.020714 0 ] (cb9-96) [ 165 0 R /XYZ 99.933071 477.073692 0 ] (cb9-97) [ 165 0 R /XYZ 99.933071 458.126671 0 ] (cb9-98) [ 165 0 R /XYZ 99.933071 439.179649 0 ] (cb9-99) [ 165 0 R /XYZ 99.933071 420.232628 0 ] (conclusion) [ 245 0 R /XYZ 84.259843 575.052842 0 ] (configure-default-service-account) [ 56 0 R /XYZ 84.259843 388.198913 0 ] (contentsbox) [ 8 0 R /XYZ 77.509843 756.850394 0 ] (control-1.1.12) [ 72 0 R /XYZ 84.259843 368.389950 0 ] (control-5.1.5) [ 72 0 R /XYZ 84.259843 164.143950 0 ] (control-5.3.2) [ 81 0 R /XYZ 84.259843 520.622079 0 ] (create-the-etcd-user) [ 49 0 R /XYZ 84.259843 500.585993 0 ] (ensure-etcd-is-configured-properly) [ 41 0 R /XYZ 84.259843 480.020658 0 ] (ensure-protect-kernel-defaults-is-set) [ 39 0 R /XYZ 84.259843 289.233496 0 ] (header_bottom_text) [ 6 0 R /XYZ 85.009843 422.718898 0 ] (host-level-requirements) [ 39 0 R /XYZ 84.259843 369.381496 0 ] (known-issues) [ 72 0 R /XYZ 84.259843 463.537950 0 ] (kubernetes-runtime-requirements) [ 49 0 R /XYZ 84.259843 352.740971 0 ] (networkpolicies) [ 56 0 R /XYZ 84.259843 663.033496 0 ] (overview) [ 26 0 R /XYZ 84.259843 283.881496 0 ] (podsecuritypolicies) [ 49 0 R /XYZ 84.259843 257.592971 0 ] (reference-hardened-rke2-template-configuration) [ 81 0 R /XYZ 84.259843 228.224079 0 ] (set-kernel-parameters) [ 49 0 R /XYZ 84.259843 799.370079 0 ] (setting-up-hosts) [ 41 0 R /XYZ 84.259843 237.872658 0 ] (title-block-header) [ 6 0 R /XYZ 84.259843 502.818898 0 ] ] ->> ->> ->> -endobj -4 0 obj -<< -/ExtGState << -/a1.0 << -/ca 1 ->> -/A1.0 << -/CA 1 ->> ->> -/XObject << ->> -/Pattern << ->> -/Shading << ->> -/Font 298 0 R ->> -endobj -5 0 obj -<< -/Filter /FlateDecode -/Length 2585 ->> -stream -xœ½š[‹£Ç†ïõ+tmØvUWõ ‚/B’B–˜r±k›0Xç"?oR4z÷hȈ™QWõ¡¾>ꢤª™0jHØ›Fu8QX¤\ˆ.–,©QlI&‘é ¶8^LN$+µ’Óûé61eº[b,Nu°‹Ï3l‘’p‚Bçc@,%x‹Î «¯^bmÁK1†ØRðÜ -〉B•À¶$ÏŒ¦°Åqr©¶XsÆ‹° Ì`Ktc,0ÐÖ±[ ¸uiÊX`à­‹fÆpa¦0ˆk5;c¹ØJ±ÀÀ\; ÐÅq/ŒêZÊ‘±À€]óš Ü5Wa,pp× -©¶`3€ŽÁ´cƒ»¦R ÜÅ1ŠŒXÕ€}˜ °WB¬U ÜUœ±ÀÁÝXਮœAc,pp7&<óÜànL Ü3c6J„Fà.’x8[XÆø{è¢1 ³–0$pWÜ9Ó UÐÚ2cAwµbù¨®-˜qrn1ËAsnŒ Ü…£2Ƃ±Œ𨑱 »jðÔLîjlÂXÀ]Ä ‰±Û!(Æø· ‚­Ku-H«•± »ÀRd,€RAª+I ã ˆC 2¸ ÷Þ 2¸+ÉŒ± ƒ»â{¨¶XŒˆk Ã±%gÁjR0¸+Šf:pW¤4Æ‚ îv·ÉXKÄÊX¢àÈX«‡VJf,ÈM CÈGu9 œ Š@€,ªƒ-^œ±uhÖ*cA‰°Åbd,(¶ d`,@Dš"ú¦élÁv¡ùÃDŠŒ Y¨Í± ¤Œ| ³Êt9†Z€kª«Ðia,(Å\±2 — -‡ž±€=\b,€Щ3”[pT àŠBˆÌ™Na ‚lÆDºÐ©2Ô[ 2ÔX‘efa,€‡F.‰©f:—žgVÆ‚ê9ôP‘±Ç™&HIu°˜`,Àúôd;1 ùA²‰4‡ê`‹%g,¨5öt³2Ô -[¢DÆìÍžoÆ‚&°cA“z˜ÌXŒC'ÂXд"ߌÎX€sÙóÍÊXÐLo"Å#ç¶Yîùfa,h[Bªƒ-š9·`òMЉé2l‰Í ZÎ=߬Œ3o"¢§Wµç›…^€»©LLî"2ÍŒ( ßDdËr{xSA|Æ•µgœ•ÞЛà[é•€½ðV…ÞàÛs$zi3‘YÖLo àV¡T¡×þ"ó&\oŸö+ªŒíPà3kéÙp,ÑýÌ9V5{Oµëýد?Ÿ¾~ ç{þù?§Ÿ¾:nÊÆK±ùÍsO¬s¿çÁ&ªr~»ËÜø„ZÏOÛÇä[?·5Þ(!?Œv¤é ǥŦÝh>6Ê«÷“†Ÿr~|óðàóñûW'¿J_ôÕBÞ¡Ž­Ú'¸óˆHõüêíéë?~û·ï_þ+öê§Ó?~×3|lÉ7ø­ Ç‘òãý‚ÅIé-¨ƒ%‡>aUürûÿúVÿS½Ê¯õ¾ùçùÕË÷ƒg3¦Ÿ^ßúìm¼õ g!³|-§N5Ø£yùµNÚlÊ׶Ýü«ÝøÿæjÇ·¯0]ׂ½&GV©^ªî¯oZ¬ï'TéÏTÚÚ¦ -Çx»våÛô6z?]ú|Á@êë-qLC¿"†—uªöåúîÏúÃ÷ÿÔåúÜ¥ºMïÿo ÞòáÙõÈî% Ûé׿8úÁúœ¶s&j¸Ž)×c’ôüž–Ç¥ù»Yëv.=µîaÕõ’}+Üê¿; €€'îÏØ£—ë—b8öô\ <ä~Wó( -ã'Š»ðöVf_HAã3Ùå¤Ø}íyM$²r{Ûfç–Æ[œ¯¼Ú§Ñ¬û"„[q”úþ½L„h™õQÜŸ^e^ÅýÞ¸»•ò4Áu›ÂÕ¡ïuW‡«áGÅj² Ó£P¶á–8ú°G{ž1§./á4}Ÿ:Ý–Ôe[Î /' kÏ¥OTÚo«FWcvá͇Ì6£Ú”¦Qs.ˆÏu"¢Ë‰_ÈèÚ±˜S¿­Zù¯Â:6’ÏzmÔ#¢Ë‰Ö»õ–æJ•Y«®¦mÌü2emu«sâ‰ì˜£›´ô¶3"6ÞÞ‹s_±ÖÀÖ¾‰ÂË ñ©?ˆŸN©Úi±BÒØz»Ð¶~§¸ÊÚ/KŠ8õ6U8šCÖ#¦þ÷ŠŠ4¥<Ç$£YÏzVi ÒÏæßOÒ.Ï“:Îf@½¬ºÛti)sÏîó¥UHýÝY{ÎØt›²M^Ú0¯CjÂã^¬s×MqGؘŸÉÊ](à ·Äinå§;ñ²3Í£šd–uÛîiö»Äy²ói—¹âÚSØiïìwÛóCx9›?÷­_Ç Kez»5Ø&\† -K¼0útˆÇitêË·ë1{9ýr:Ü\´[|‘¯!óÚ#&=ÿu^â÷_'?7R€«2i'[ïÄ»'ëŸßÕO÷Ïó†p›M}Ãþk]uÓØ ÓǸñÚWsY`5uT§v¦5‡›8έÕ¶™I^žÍâ d~ëÚ,‡}-Úx8 c=yV¡°?Šuél¹äóÖ%J½êréÐzÜ¢°wŠâ6`[–Ý™Ù7ä>Èœún¿ïKa}¬­°…½hKçËÍî]Ùñ[„À¶öŽ]÷AÛ=Äè/>^¿yÝo&çqºt¢}JŸvé|°.ÀãR;Šù¾8}Ž{aÞ5C³/ÛÎžÍ gZø]aߣ›´®f· -iÔ‡ìnÐá(&. “!›=–aS¾5Ór.Zìø”ÔœÁïe¯Û5Çg·ùð½ÇãíƸ÷ØGÂr7“¢ýmÜ„'(ÚÇ[WãÛn¼YúHŸ7ï½}¤×pz|¥?·ËY,=Uìß`²ôÖþÁk¥×<êwW'Çëï»ô -endstream -endobj -6 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 5 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -7 0 obj -<< -/Filter /FlateDecode -/Length 3356 ->> -stream -xœíOo$·Åïó)úlÀcþ'râ 1`F1Œ ‡Ý×F X篟bO“õjº¨]¤µÖzú ›EÖãäp5»ú󵥿J°çRÖœÂry<}:™sŽ†¾¶âb{úGúÆ?.†st9Æu±ÙžM.9˜åñd?ûB̨?HÝ°.êý§¯Nÿ7YzXbqô0Ú³-Ö·ÄtN«[­_~ÿå Jg(mΩ~ååøà÷_Oß¼§¶/¿þ÷ôñ«­ë)»ójb´nq.Ÿs¶)†­»”EJ¢/¨×{Ç:UßëA}ï.ßä -Á›X§Ê(aö 'J{aÎQQ{àË õ”Ξë½p-‘}¬^DX¯É dÄ­NõÒQ¨ ëvj‚…n¸ŽÈ™Òó-Ûf ²ô!Å[‡ µÄ¸£*|p@_ÏÖuZD2JóÁ|°ÒnuÒQ¨Â;ðÁ¢ÆuZDΔžï !ºúúµ|QîÍãÝ1Š¨£щèDô€è½«("Š-‘½OJ&¢щè!ª£øüî݉èDt"újÝ’uDY—(®N_EQ? Z -ì¦á&Ö%¢¬#\+PTÇ)Ý»x^D= ß^KèÝÚJCû@Ct(èDjĺš`¡®…åLéù–CÍ>déC†u1dÖs:"*}púˆz@ñUéëQÐaHCD¡ -ìÀ£÷åLéù~¢Ð›ÇºcÅh'&¢щèÑ;_‹ -D±å²÷IéÑDt":}#DuŸ¿Ñ¡;ˆND_m£’¾Š².QŒV_EQ¿EÔ‡ »i¸‰u(èWX•C$,-±¯HD pDJ|< VåÐ`]"Ê:ÂXà(Ôˆu;5Á¨sï1"gJÏ·l7š-|HÒ‡ ëbȬgw@ôÆ7ðtšFŽˆ®FóUÙÖ%¢+­ñæˆB>Ø}pD!Sz¾_‚(÷æq€îE1šÁ‰‰èDt"z@ôÞ×¢ˆ(¶Dö>)=šˆND'¢o„¨Žâó7º#t'¢щèkmtU¨! -º@1X«®¢B¿E4˜ »i¸‰u(èW0«rˆ¢!Äv¼ò u8ሔx{ Š¡º@t€ " -5bÝNM0êÜ{ŒÈ™Òó-Ûf ’ôat1dʹ½ñÁ |pèÃz@4D£ùÀªìëQÐqHsD¡ -ìÀ£÷#r¦ô|¿QîÍãÝ1Šb4ƒщèDô€è¯E¢ØÙû¤ôh":ˆ¾¢:ŠÏßèŽÐˆND'¢¯¶ÑMIG”u‰" vuEý€(•³ÚÐ`]"Ê:•Šrˆ¢‘ò?:ôˆÑðîsû@Ct(èDjÀºš`Ô¹÷ˆ(gJÏ·Òh¶ð!I0¬‹!SÎረôÁ |¦‘¢Ñ¬ªëÀ‡Uú°: iˆ(Táƒø`чpD”3¥çûˆBoèŽQY'&¢щèÑ;_‹ -DEËEï“Ò£‰èDt"úFˆê(>£;Bw":ˆ¾ÚF×E}e]¢è¾Š¢~‹htI}+1èQÐ.W”C$,MA}KwLpÂÿà>´/é?ŽºD4å`" -5`ÝNM0êÞÎ9Sz¾e»ÑláC”>4`XC¦(oé¾ñÁ |=+oéŽiU}X>¬Ò‡†h‚£5ÒQ¨Â;ðÁ¢Ç·tC¦ô|¿QîÍãÝ1Š"‹àÄDt":= zïkQDT´\ô>)=šˆND'¢o„¨Žâó7º#t'¢щè«mtKÔe]¢¸}Eý€hIo*âWO10¬#\¥(‡HX šœWßÒôˆÉñîsû@Ct(èDªÇºš`Ô ¼1œåLéù–CÍ>DéC†u1d²ò–îÜÀ‡>ßÒ\Q}(ŠôatÒQ¨Â;ðÀÈÊ[º!Sz¾_€(ôæq€îE‘Epb":ˆ½óµ¨@T´\ô>)=šˆND'¢o„¨Žâó7º#t'¢щè«mtCÐWQÖ%ŠaÕWQÔoM!rq“øÕS ëWÈÊ!–ˆ¯¾¥;8ህpÚWôG]"Z²r0…ê±n§&XèðÆpŽÈ™Òó-Ûf ‚ô¡ú2YyK÷nàèIyKw*Eõ¡ |(Ò‡†h£5ÒQ¨Â;ðÁ¢Ç·tC¦ô|¿QîÍãÝ1Š"‹àÄDt":= zïkQDT´\ô>)=šˆND'¢o„¨Žâó7º#t'¢щè‹6ºŸN×ÏZŧþôîdÓþ­iIÑC±©„ÅÅíõtŽvy÷xúæû¿ýõÏ?ü¼ÐÅÇÓ?ÿ`ŒMü×òî»Ó·ï¨ê/­œ:œñqñvKаò|_å1§dŸ}=aV^îª<‡ä½[Âõcy†•¯÷µÜ¬ôß®¿gyXù‡û*w)X³Äë/ÎV~y~å!³/Ñ™%]ß =¬<зQ€ë·?@«V®ê×ôü$_OüyŠß*ÿË·ÿøỿså.Cm0ÑÒw6&~w ÕÖDWX?‘¯•®{…q|Ù¯ß?U±hk䇽‘éÚ0jŸ ÷ºöõÚøk¬M§¬ÅrÙ5óœJÖ¶>¥ýþ¼'ër­+¼ßc×çüõq¸ióÓ±$z[ŸÙÛ!‡–ãoÿOÜçýÚÚµ÷õus")ÞÚIuzŠá.×X=GÍóýñ–“¿µ1ï: ÂÐÆÉû½Oq¿¯Ý³•{ªr2è¹´û½ï¯·¸—ÝŸ´·«|®ÿr.ØÆÄžÇ}mù ‰=¨cxëGÜûßÊìùØr¯mu¤Ù=g>³Oulmã¾å£Õýªù“ÓÑí$±õéã^w:Ö¹ÏPmÌ‘ÔßÝA ±´:Ò®l.§úâ?Ѷ÷ ŠÔ6æÕŸ©¹ÔÏ&wîíJûÄísÈsáÇ=º]êŸñLUîjç3­ÞêT¦Ø2´ófJpï¹–ÕaبSE³±bäÖë>ùÛºøîen‡×‡áÖ–_+ѧS,d.52Mûwê«ѹÅÖJØôúÉîdf*®åÜnY²ËàÖí#í?õRû§Ü›sôv¥ Öæ ^ìå¯Ë"½pZiGnê×Rº>"™‚Å[™v“i¥6eÚ-º/”«¸?4½®`[Á®Ñ–Ö×môMIÏf¸öÊIܺØzSïí6ÚR‘}­Pá[`E¤;“=ʵ÷õU×U&b9ûý*õ¦ÄÔÚL"W°,WÈ7~V¶Þ€¢p,Ok+ëúõ{Nbo:¦Î‚ý­¬Ë`çSâåDaý­ú ªÎv͵ì:Ó5Z»[ÉnHè>)R} s¿6­ê@föçÁ5z»‹¥ ¤ÐË­­œ"]Nj¹½¶ØʽTá[×–yn -u_zâmËÑ®Ò4ñ—uêš(§ê@óÚ7î ^NuÞ9ȧ˜•Ò‘ç -ÛÐCÑC½].†Ç «¹´TšM«[÷ú÷uªˆ]­3ÏöÀ†>¤rÁ+RÙì²Lê©Ï:AË­—\Òesîcóe‹QÊ×I·—î*¤ ô¼¶æÕIªOR¶}Ôu¹Na-?}®DÑ´FP8–cÊBævÆŽ* ö,Yî±×Ërêsçʹ;nSèàu Á˜oâe[Ø­\‡~iAXv¦¯v DnM -,ó4ú°ÉÖ>;Ùuí2mŽœ~;mËœóÑ­õ”^{ey¾b—ÿQ™ïèûߧ°|ßÎ[®ë¸3}¶p}n2®dÎùfágeçÖ.vl@t¦}–má²±†¾Ž;óuàÛ Oì­¤í3¨íÏ¢9‡ »>tõ™ø„x¹ÉbßȼÔ^°¯—¾uŽÂx»ôU/°;tYø9ÏKòUÒU-ÊK:=ÜHX)]BÀ•[&šY$驯íh…¯±à¶õÒós—Y¬ºi[/ H°ps°¢â`1è*Ú£4ún¼^ìûÞäÔ¨jŸÚ»ú€jïXUû„§«~»Lò²¯9~© ^ÎP¯Ë{¾or ÞZqcÔ·í" MAÛBѧ ÒLÓz¹µ)í·µ´l=.j–rßS°úf:¢+=‰]I‡²µÞ¶âõÝ’ï ˆÃ9sï —ì{É¢h ïîjà’¾iÇØAÆn«íÝÜÄ;S÷ÅCÉ»æ¿{ØQSµRâXÖ~ah¡6{ç—µnwJJ4‡PGèUs(pÎñì{ž>ø8o´ƒŒDvЋvo²¥ñ6hÂ=±~þ ¨jÿ7ËÁm«Q»ô¼(ç:<ÍOÖýÙïÿJ´îôË\_*Ƶ_7½ùj³´ÇÃÖÿGq~ -endstream -endobj -8 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 7 0 R -/Resources 4 0 R -/Annots [ 9 0 R 10 0 R 11 0 R 12 0 R 13 0 R 14 0 R 15 0 R 16 0 R 17 0 R 18 0 R 19 0 R 20 0 R 21 0 R 22 0 R 23 0 R 24 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -9 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 671.061024 504.679134 638.361024 ] -/BS << -/W 0 ->> -/Dest (overview) ->> -endobj -10 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 489.361263 663.111024 496.429134 648.111024 ] -/BS << -/W 0 ->> -/Dest (overview) ->> -endobj -11 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 626.361024 504.679134 593.661024 ] -/BS << -/W 0 ->> -/Dest (host-level-requirements) ->> -endobj -12 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.881526 618.411024 496.429134 603.411024 ] -/BS << -/W 0 ->> -/Dest (host-level-requirements) ->> -endobj -13 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 581.661024 504.679134 548.961024 ] -/BS << -/W 0 ->> -/Dest (setting-up-hosts) ->> -endobj -14 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.893245 573.711024 496.429134 558.711024 ] -/BS << -/W 0 ->> -/Dest (setting-up-hosts) ->> -endobj -15 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 536.961024 504.679134 504.261024 ] -/BS << -/W 0 ->> -/Dest (kubernetes-runtime-requirements) ->> -endobj -16 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.809749 529.011024 496.429134 514.011024 ] -/BS << -/W 0 ->> -/Dest (kubernetes-runtime-requirements) ->> -endobj -17 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 492.261024 504.679134 459.561024 ] -/BS << -/W 0 ->> -/Dest (api-server-audit-configuration) ->> -endobj -18 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.857357 484.311024 496.429134 469.311024 ] -/BS << -/W 0 ->> -/Dest (api-server-audit-configuration) ->> -endobj -19 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 447.561024 504.679134 414.861024 ] -/BS << -/W 0 ->> -/Dest (known-issues) ->> -endobj -20 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 488.869808 439.611024 496.429134 424.611024 ] -/BS << -/W 0 ->> -/Dest (known-issues) ->> -endobj -21 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 402.861024 504.679134 370.161024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke2-template-configuration) ->> -endobj -22 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 485.053890 394.911024 496.429134 379.911024 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke2-template-configuration) ->> -endobj -23 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 76.423228 358.161024 504.679134 325.461024 ] -/BS << -/W 0 ->> -/Dest (conclusion) ->> -endobj -24 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 485.053890 350.211024 496.429134 335.211024 ] -/BS << -/W 0 ->> -/Dest (conclusion) ->> -endobj -25 0 obj -<< -/Filter /FlateDecode -/Length 4642 ->> -stream -xœí][¯$7~Ÿ_1ÏHÛñ¥|“D -ÒB¤!v'9‰Ð,RÂCþ>ŸÝm»Ü]Ý3sf7D@ŽÎžé*w¹nþ\å™éè³ÂÏ"é)Æüzú꧎÷è³™RòF»óßdÙ)Ä”X ,¥­Àrn -Á8/°¼ž`bJ j¸@†¶¬5(E'± V„–‚¶[V‚&h’XPC'–VÐC¥DQDÖ<í&§{‰gôäc2Iäðt0ÏÚÉÒNK¼4yï]”x]\RZâ9èâ4‘’xÐ…l ÏCë­yÐÅÄ`%^€.ÞI/BdWyÐEy¥%^²“K‘Hä%ðT qË3ÊM.Z«$žÖ“CnZ‘‡… ù ð tñÊ%‘]œUFâYèBø‘xÀ gCŒÂj5yu[eµÄsÐŘ@"º ø]Tð`M—$ 0!L” I `¢(R”PÀÄ4QV‚“ÜD> ¬Òà-UÐÅ‘—ÀjèBX¹"ºØDX]0@Âk¡‹!+aÚ²E ¸%•´„xKJ{ ,j* ,×FOX@.R)IX`¹Ø)aèb¹ ,P×:o$,°€]KÑKX`»–´’°€€»ÖâN‘]À'«£•°€€»V« aw±ŒŒ„ˆê„<ô W&£’°€€»&*’°€€»&` ya´p×8˜ íÜÀ]㔑°€€»†Œ—°‰2”X w!N.  VÂì÷à+a„Me‰„¸«¶?aM­&“—°ÀwuDøD^št€Ç…u /OÚû$aîb£²8à.à3JXŒŸ°£ pW[ìÔ¸«MR8à.ê'aÒaÒ($,Àþ6i…ÔyiR)F p°d$,À.<©$yaB"i Æ“B*aîb{Oxà®rÖJX໊Pöˆ<èb£‘°u xX¶ÂšF±:)¤xÀ]¥a‰ÜU*$ 3JXÉO¹T”°Ë&RäAÀ„„ˆOn¶„h~Ðl¢ÍyÐÅ:’° F“ÛÍ(aAŒÐÅ(#ar3÷›A‚¤  *> ’òS.“%,Œƒ§”„IGô›†$,ÀºÌýf”° Y…~-ž°n“õ¹ß $‚.%ŠÈƒ.(š’°nIè7NÏC“H‚ä}î7£„(3Ðo¢¢bî7ƒx\Üu È$ñ€»¨L½„¸@¿‰ÊVêí€×Ôg23æŽ3Š' -Ðë°·ŠG -Ø‹Ý*ˆg -à›{$ñÐj¢³Œ^<5À¶ -¦Vâ±þ¢£ &̧OüˆÊ#ö̲‘ Ë‚PÝ·ž£³;Ãæó±¾=}ö›ïùÛ^~QNÊê¨[ò¥±ÆºóÉ$Äûà %Ñ”×®Õœ~=yÊ[Ó³žu4"LÕÔ±—£ÔÈÀÎœ·×‘îQG:›PG3ýõÂíaô+è„¢sMGÁØG·ªå²ÙséÍz6#ó”ìïËé+„þûARç„–èR9SÓ:kT7åø†Ô㯱ã,ç™9þ¡˜Ên_ Iœx;æBÚÉ…4æ‚Rk:vóõ\}n©C.è\`ô|ÆÑra™1ÇKÊ¿“ ~Ìy+ºF[ÒF·ê v'½YÏfdž’ý}ùÿšü™¬I<¿?å³ArË=WUåà K¯Šaôë@OªÓ¹N/Û€,L£­àÎdÑlÔ=£@¬‹ýÖ-;ºÄÉ8‰q4 4ÁÜë@çês9³daùÚ[—ßÍ(fÉ¢Ù¨{ÌZïx(ãjvö•ã¤Ý×Îíàr6öÉÂLîГ˧!Å>Y4õûX.äÓ³;ÒÒÞ¢39‡i©Î(ËJ:J"aÌÌ~Ì–ƒ=ûù(-˾µGVí‘E¾Ú¦‡ãÃè×xÞ°GvLă°±¶%õ¯ýõ%áÍ@ßÚ' «ñ’E¾2^ƒJÌ¿!^r¾Þ°Gv€‡<^Ïâa«°†x1ú?úÖ>YX—,ò•ñôPbþ ñ’óõ†=²°|g">¾;¼¼?ì-:“sˆ¾3îóø>س¿¾$üè[{da¾ {˜òþËè×xÞ°Gv€ïLÄÇÃ÷!^ûëKÂϾµO&àûóñôë‹!^r¾Þ°Gv€ï<^ ߇x ¸/×µ{ô­}²0ߟנ‡\_ ñ’óõ†=²°|g"žÂ÷m¯ù›/OÚ/Ÿ†óg¶¤Ñ¢¢ÎWúòÃé³/þø‡ßþé¯g\¼œþöK¥lPŠÞã÷¢”Ëo.X¼ÍüÅ5ôSù*”ÿjü†yŒË¿nçÒ|¾¯Œóã˜J+òÜ2—e•×iSæ~7ÏÙäÑC Þ×¹}i6¾L³=È}¥Óò7Îó<æÂrË,¶†Å—®kË­Ðu©>j6Õ|\rÉ99-ß«}a+«ø¶æÀ ›¿æ5õ|­´æOZr3ŒkÂùC?ÒÚ97Ê|U†c¾Y­¿6†Å®¬åºV¾¹ßÞ²æj.)9Æ8Ást½fM)£7݆8úǯýcvU¼Év]æ¹ÕûY׬C¯.9çy~=ç`ë -R½šäièŽà\y?¦Þ‹Qkü«ùÍbËq”X.ÕyŠßÞmsÐI¾‰r,K ÍÓg}·1aÎ!ök,y>aÚËκdÁ}ÛÖ]Íc¥†=«Ú ®ËÔs½è³òÝ&Ž›øú¯}QãÅó¾ÅøhYåɽ8ÀòºÕ »¥‚5i" µ³yÍeUû¹œ£Øñü­kƒ]cΣ{ëv8¢òùÒæ¼kª¯Z=Uì§>ÏÑÞ[î³=·Ë½uo«¹ŸF}¥ý×f7| ¯ÊØ{ö——CŸùµÏîž#öëÄüë­.¦Qh¿Òå‘ڇɱ9»1ÚÈ1¾1lôÙÄê¦ÿå?£'ùÞZ›žäÐsrçÏÆŒX¥µœ½:Uû2®åܵ6–õ«©ÎwèçÍøy˜ÇͯnZŽ­‡:ÈþârëYG$ätÝߊŸjMÊÖ-¯Qö‡¾W›—½»ÖÙCO•× -h&ÝÆÌ]¼¼»?nûí÷ËŽûçߟ>ûõÒ¥¾(ïVÎN.©(9•§_ó—ØRŠÊ<ý厶CG{¹Ö™)ÎÆ3¬Z¯/³ò[݇òõ»c›¹\Óm68½ÕŠÓ¶™‰G<Žl6µ6^öùî’{±ÛF/·û»õyHíánÛï³ßÚüõ:—×£?e¿9´¿öFEçÚc±c­|õ€†Ç8è‰(.ý)é8Æ`¢[lÖÌŠáͧÉbþ!à¨qúSþñÇþ¡—^¸åD]Ó¼G«vWßç#÷µÜíÏ×Gþ°¾UHNÊ7ñvîèq_á=Š[ð~Ρ»m0/é×&²ŽÓŸ±)cžeý&ßO×gªf¯c=ŠÔ?ÙÐ×CéUøy'ë1>M¾Ž§ŽPײÉsúS¾=ÆÓÁ®Uù -[Ü´¨ Ê+N¿Ó–4å"iù²û›~ÉŠ”•p”³æx·£j¼ò‰±vøŸzAZ+G5qÔkýdçýK\Ú=q¶ãöÙ?ë)Ä^iÿL㹞åÐvíÇmo³#×ÿ<Þ‡øiÞSÈß݉¯î#ÿûßSˆaƒ_O¾§ Ï“4­ã°²§ì©ë3âÀâBK~qŸÒ#ž»j9C,˜@[¿¶‹y7j‚[øUtx×õçgÞcúÏ®ýWûv~FZÏ[k_·>C»+ŸW=!­êä[†üÝ蟿ÀO?’þ’/µ’õîgFœÅ2y-§_f{´:¬S½Ëyß\§ø|ÊÈFÏé×\,¬Ž»Eì_j-ÞñN‰õƒÙyÏ¡ùà¡ýíÞsä˜lî8RþZ8§?ìŸAÌ¡n`Îüññ9·®ÓZmU÷2¡Žê‘”{0¤àÕeœ¬rKŽ–˜ò¾”ù»Jóûûr] ô5ÃÔ†Ïøqü½vàfÃzä ”5ÝóÁƒUQµúºiÔ¦}C¡ø¨…ev m¯ëæ»9üÓÅinË]²ù»ÑËã¶v{,wÜ“­;ŽbŠ·óg[w§ùx‡…Ÿpâã®òN||tö '>>“ú„ÿ§’ëÆ©Ì'œøáäz»|Á78HËÒ¢æß^¾:zÇß.ݘñÒL_>r³c`þ -ßlŸA½‘Bþ2üÞi×z7Û=ÙZ¢b§Z»ØpïâÝ[‘­âŸ|y ƒ.åQ>NÚä§öaåNŸC:ç‡@DeIW—׬ٹ³äOÏ-5§”ÊODÊOÃ(!áËø9ç>œt¢É×Ó£ò¨2UÈÑÔ‹N¾žò£êH £H4w’3qy©š,ÒíD«Ò.'ìKë‘ÖMjyÙ½¬«/«Öäû]½-?BØhêUÎßybˆ;½Þ’³õjÒ 9?î/Nv¹òMÔ€©»@âc»À~ãM²¶ŠÝ–¨Øtl¨ê£óãašë|'6Õ¹ë4 k çñr´vM½ŠÔü±*ªzWi–)•ÕÕ‘- Ôâ$.'øFUÑ„`6>‹Z~TËLŒ5‘¨Kuœ@ºœÄq‹4×"Ú¨ØoMÕó]•žê66Ç ´ ;ùù¥l([_ŽdŸ#Pc?!½„ /ÝN¼œÐ›Ð†|=¹ Œv+”«©Ç‰–Émä¨z¾tjˆÕUXš•–t‘ÿ¡Â5jFžò"o›º -àW}’ˉ‘G'qºo¨C’òWiô† -Ù!´œåþÒQ ã3è¶ÑÍc•¹ŒÑCªêejà1’u˺FÎVýÓ°’UU"±¾‘]Kåë@îzº¶Tj“yÍÒÝ5¹ìv^99´ˆëü䡦o“Ër¾/ec£59§~¬“t²Qm·ë“1bW  ÐÉF¯…\WkC§üú²/2iǽœ¾;•mÎXg^x«óƒôxŤÏ?bÌçøýljÎ_ÔïÁÌû¸Q -LÃÖÌw²üØÕx'9?†µÛ²aDC ö;YÇ>ÖÕlhû¸1 ¯©ß®:°×‘º!¨n\Nt݇ŒlZêm™gÜñ²òb+džMß°çK[Ã4ùáM«ü$¦ .cçÙ¾%Ÿ™H\å¡}K— »\(.Ù„©k6¨™’OÒ\Ÿõ' -›çb¬ŒÈ—¶ó¨o³\t¥+Ø”À–_pÁ¤ù¤iÐGPúÕËëé¸/*ûº2µA{£^9µ–© ðdª-—~¼l{Ž=狺x»‡š,ËÖžmEN_½´ á‚ç(£Æ~Û2ÀñI©Ò†IëFÑà4UiMb¨:ùå¶ê–bq½ZMÑ©ßqO;•æDÇ£â7c³ÜºãµjɶÈñtÝ:ÕG¶Z2 -4ÏïnTê#m¥mç¦qîºÛž™Ôñ½2Uý6·ù*ü{ÍÚÏÉ¥Q醎ýYg>ŸÀkly|½ŠÞC(?4PdÇßs|î±=ݨç|&Œ¬UAçÿ¬ÂŒt›ÁDÕÿI|[R¢I9@8Öé0·=¶*O¢ÆËáN» ¹UÌž·¹èùy¹Aí¼Éá7'Xoÿ Ñ¥€ -endstream -endobj -26 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 25 0 R -/Resources 4 0 R -/Annots [ 27 0 R 28 0 R 29 0 R 30 0 R 31 0 R 32 0 R 33 0 R 34 0 R 35 0 R 36 0 R 37 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -27 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 445.281496 409.271317 430.281496 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://releases.rancher.com/documents/security/2.6/Rancher_RKE2_v2-6_CIS_v1-6_Hardening_Guide.pdf) ->> ->> -endobj -28 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 418.281496 162.214732 403.281496 ] -/BS << -/W 0 ->> -/Dest (overview) ->> -endobj -29 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 403.281496 251.801646 388.281496 ] -/BS << -/W 0 ->> -/Dest (host-level-requirements) ->> -endobj -30 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 388.281496 203.586314 373.281496 ] -/BS << -/W 0 ->> -/Dest (setting-up-hosts) ->> -endobj -31 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 373.281496 309.303355 358.281496 ] -/BS << -/W 0 ->> -/Dest (kubernetes-runtime-requirements) ->> -endobj -32 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 358.281496 284.198131 343.281496 ] -/BS << -/W 0 ->> -/Dest (api-server-audit-configuration) ->> -endobj -33 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 343.281496 187.304576 328.281496 ] -/BS << -/W 0 ->> -/Dest (known-issues) ->> -endobj -34 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 328.281496 407.493297 313.281496 ] -/BS << -/W 0 ->> -/Dest (reference-hardened-rke2-template-configuration) ->> -endobj -35 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 107.173228 313.281496 174.285045 298.281496 ] -/BS << -/W 0 ->> -/Dest (conclusion) ->> -endobj -36 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 293.575760 152.481496 483.355790 137.481496 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke2-1.6-benchmark-2.6/) ->> ->> -endobj -37 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 84.259843 137.481496 285.368241 122.481496 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke2-1.6-benchmark-2.6/) ->> ->> -endobj -38 0 obj -<< -/Filter /FlateDecode -/Length 4975 ->> -stream -xœí]]¯Ém}Ÿ_1϶¶ªÈúÙ$€ãd`A´cÉFpeÀ΃ÿ¾{ºªØÝœ™;wK–pu§Éú`‘¬C²º§Î¿ ø§rpµ¶’ù|ù|úÓÉ»’<þ,-6 û~J:ÏÅ¥ÐrMçÏB âJK±äóË ÿì.uó—Óo~qúã¶O8G×ZŽ!ÿüQXL®ÔR|2X ,È`¥äJ‰)¬–ؚł©pä#«@ n5Y,ˆÁ1VƒU!q td5ˆK`‹1BKÅ`9BôÞäAÏLÑà…ä2”ž-^ .×›É+à…-‘Ë…C -¯¹œsª!Kj>X¼YR`ö²0Õbñ2d¡LÞäA–X Y¼Y"´Ó ^…,ð®jò ‹Ï>X¼F.µÊlòx¾–zäEŸ\ªDÞâ…à|“LvŒ”‹Á‹%ûÔLdIä£Å#ÈÂøkñ€‰J­Æn²»ÉS°x ²ÄXØäAßB€˜!‹/Ù‚ì '¸da@,Åq‹l@¬ä¸rµP Ö渲` ¶ä8·báù^ ‡,‰³… cçš<ÈB-, YÐÀÂ"È™,,  -Ã[,, À-û,, à-û-, .Äô—jf  Wj0—€@Û½XX@@]J9ZX@€]âš-, à.qð0p—=Md[Xtt*YXÀÀ] -¾XXÀÀ]l£ha¬êà‡ÙÂøŠ‹µz ¸«g ¸ B€É+{,,`ànLX‚¹»1ùhaw#Çlaż™ w!ÉN ÖÂÄ{ð"YX€Á\DZbaAðgìi • µe p7T˜Ïä5 -4nì[hÙ…œ›… ¸‹@E$à.à³ZXŒwˆ¨Ñ‚Ü „Hmñ€»!6oaAî"oHÀ\@`aâ› ®kòšó­V p°-,@v¾IL^qp¤`aÀØyä¡dà.Â{³° w}"²° w=#í1y…j´°y xضƞF²ê|RX<à®°°Åîz_š…¸+aÓ‚\„«…Ð! ŽäÊ®•’-,È̓‡”Ïäe‡ëdaAñ $@–Ƀ,\ØÂ$Ö®Q«”Y(F J„,H,,@FàZ@öm–îbÖ Y)ZX$sµ1[XPRF=­Z¼]-€k“WÁ Å‚RØ! PK¹ŠMoaÞIºdaÄ/°…¥Al Š\ÈÌ-^€,H²-,@¦ ^ÔY ZXPcE•™½…ˆÐ¨%¡j‹Ç^êÌjaAåì$U´°Û•&ÒäAÀ„…°ÛÉÂ?(6Qæ˜<ÈB‰-,¨5J¹Y-,¨²D-,€oJ½Y,,h² ã³° ùì$M¶°0ž÷´PQoF¶°ûRêÍjaA#z%ž±oe©7‹…!KBŠbò ’¦fì[`êM “ÅË%6¶° å,õfµ°iêMdôæ‘@•z³˜ÇÀÝÔ€L¸‹Ì4[X€ Ô›Èl­ÚÞxSA~f3«TœÕ<1ð€Þ„Øjx`/¢U1Ï <ÀWj$óÐb¢²¬Ù<5@X3xóØÀ‘ñ®§Oúˆ*à -bf-²È†mÁÈîGÍ1›Ñf×ó±?ÿþôýßóïÿïôéËIYÿêÄY -k&9“‰ öþ|BJädïRÐô—SfQØž.rŽÖK•{ ¶ÞörRtTô‚â·/[zE±ÐÕŒ¼½µ’OQ/z=Šþ:#éÜÓ‘0ÎÖcÆ 5é±ãÔU›:ÙÐÇêÕŒJS¶¾/§ßÀôÚŽ4¸ŒM³X¢Á™C‰‡A'±oiÓþg=Ïû;F2 ”=~Ø8|âÝÖò _È[_ð~OG4Ÿ­¡S>R7¾nø‚¢C§yøÂ:£ØËò…zÃêÖäíèeÉh=fÜP7¾@7|AÑÇêÕŒJS¶¾/ß“ßÈž4Ás™MŽA©‰(ó(úÆ<Š®‹ÙÔÔÛñ…#öF9!ØÚG3ž#oÔ¸ÿã9òf|µ®—-}hA™IiÌÖ»Š»Î"e¹P¿tS²HÛâÖVüRt…u¢ä1êÆÂ-W0Ï‘7ȶw“ñy3þÎH»Âª…œJc¶Þ/ßšßØÖ4ô‡÷'”Z×Û¼Yv]ïŠz#¢pBFýþóéûÿõWÿü¿EŠüþÓé¿ÿÁ£°ÃOĶ+Ò[Ï ?ŸðSåàèú;ý„ßyýLkÛËÚOúçk[¡ ­æŸÖvyGãëç…–Öq…V×öiÎ/·}–öaåóU¾E¶ËJ/ëhœÚú9åÜË»Žùéú“.kZçÌòóËÿ9¿ÿõ Ý6­ÛE}‰«øCD K« -bšâ-*ýI-QxQ1ÿî8¥r3xÙvNB¿ˆ~ㆴ.áÃué}i M™º«V®yUÓ¢ŽËÎTÝ ~ZûfÕ?®ªZUßÍÚÍÂÖvù:nï»´+¬&!5W7ý"KT®Ð×ÐÍ©i£ÿ]½ñ^o·äïóŽñýu¾ÍÚW›‰½†ŽxºÕ2æ[w²ì¾Åº»ò§Û6Y~ç~ÕÖ áîº lJ!ú¾î˜çšÇ¶¿Ì-²÷Ù´Úz‘y•¥ûÉò¹ËRÖ1׶}Ûk[zºÊú/ï‘\¶4@ Ž#×¼±WåùŒxÍÉVõ£,{KŸGùH+yòy|3ñð¦`b^9C`ó­zÞ ¯@oê³^M0.©ñwc‰ïø·ÿúñ?ßM$&š¦§ÕD‹z»{^V_T[e’åwˆ@± )7×äÖpÛË´Ý­—í\væC)ç(¶š’6EÎXo,Ä–õžîòu—éWq/¬Ê/×}?°ë£R˜Þ•Õ÷ÕË=RôÉRØÓ]¾®Â´¸ò Å+¼ýMcq>Õ¸ŸÇò`3ö@œ'p pë ¿‚k¾Ðƒ÷Œ{n¢wj”Ò×3¯`©déà<¤z·ÒÌR¹ bØAC¼¨çhíÌ­dC°>ìK':àg•ë­kÔkº©·5ÐHnjÀ™£ÁÍ÷ Ó~­kýœw ®I%%÷dï¹ó"SSíïä·K¡rÕ0Ùsà®—O÷íÐѺëhŒÑ}‹Uày Qº+y—<ô½Ôõp´¼‚@ƒŠ“äñ®0à¾Þ‰OwùÊ€WÃw£«à%RÙQß$ŸfÖzì伎ƒ„ëÝf¸në¸ì78ë56€ÎÆ(»WÂ>y­±žîò•s1ªCÜûÆêÑhI¯¥Tá+‡öÈXìQÉç"?>0Ö¡üQ•²eDù=JGU²ì«W^Ûj|IKîîvy|Ê.g‚/:¹[¢¥Ý81+¤•ÏýÝS§J¦· ¬Žp*¢,ãÖ‰ y»½âÅŽJérW'i¯“½ Ëœ£¼•ÍŒSš‹ÚäRoJÔ^Nònœ4*¥éS#âe•-=ˆîËu×ÿÐå]=•½žžµïýr˜šå›w³½’ZOÏþ¼Z«ªá4ðÞªãn•×3¶j0…‡rãõcn÷Ý=Éã­;ßÕ¸ý8êx&»=œnNÝÔº×ëe=}ìUsÌB­£ˆîÀny¸ ±;d/ˆ}æ"÷È$èšØM{¿Ü×=}ׇïéÈæ7ÕÄ°»¢§ŽUfÖ¨;çèb†yy7HÀm÷ûÓ]vZþœ¾d|‡·©·ö{u|¯|–vRÎõ‘ Ûî½’<Í™¯C¼ÒOwù&ì‘RR¿Ñí‘=²<‚*û‡{ÖŠ sÃÃH®Â\9¸ bé²çÖاï‚轧«,³ê5öîáŽÄ \Ý¡úyÇF›?¡!«:ê½wt®t÷º'!e'ÆÞÈUº™P;qÁ¾3µÇØ®ƒ½~¬Û #æZw€Ô)Ã8­ ÞŽºz¦9îþh^ûȽ»c‡nÎA÷7^o¡R¯2öüž¡ïÏkÿ?+øð»uŽSöŽàcL-'ûM%?úª«{ÈZü^c¼¬2t $}3¯LP7v‡þzÆ°FwëƪY•7ƒÍê»3ê]ïDÂQUYÙ‚XVäëûFÒµoæ¥Ö®|E-a¤6öÍÍSø¹³ï"l÷æJÞìØõ2zôŽ»™¿ÿ§Þÿû?þÖŽ £äÔN}à ^oß[ú†ëvµê¦®–â¦-·¼v›qÆŒ)÷Ü[¾‹¨Yð &õç~˜ôq]Yš«×1Ì,z­G¬ƒ­<5·¹U´OÈ°I²Ã²žÐ™Éßxº•ö:ý¢¥·²Ú,›uëC¼}Ÿ›¹†¿{ÐðšDvóüJ›òk(I}ß]þ®.Ó^—Où€>„¹»+ÝN”ïQ |&Ø´IÎ-ÒI·|®¢¬jçZH¾õÿ8é~¶ËWNºq7ºú’I·§T!ý~ŽŸt?yònm¢}Õjß!©O„ÒŠsÌA[™Q\`þl> õt—¯ë‰xˆûÀ1úñm}3 sw‘KAO`[ÍŽ&ßëÅZJC9ÇŽS]¾Æklŧû|q!Ç!v"W3ÜÚ>ˆûd;7òäÆef,t'ã£ü¦Çó~Óc¤¤:6ís–§Óë{dò½Í)p? ;@Îîo1æ ÆÝ£?ÖË-és‚š\ Œ qn™º·Þ?'x¶Ë7áßTËzcŠ·„œþÉpÊ–‹<é±›Ã<øW¹”$r·Ž)Äo½ÎϬ-“Ü­‰ZÒFIî‰íÏvù&ì(ïéR¿ÞŽ§4º/cŒç;í®/¼»q0}ëûcá¾WÇ|ï–oÿÎ9³CêEæܨÁž<ߟ<=œüæ‚O¾}rão¼òí­ù×®üÝúÅë’0¢¼¤) ú{ÜcSÞ»Ì ÎY·ÎŽɼ"¦#Ñ«é&9r—'Èk{†êò$ѵê‚2o‹2ç=âå„iiO}1©òò·>T×nôƒFJ¨6¨©·áa'ƒt9Äï|šaÌÁWV‹ãd±vGâÑ®õvér2Û­£¥a©2ZÕÙµuÍOQ¦«SŠ7hu¦È{eÕñÔ?nÉrnØÚWå7ñ@¼œ8É/§TŒÖib…OÝõ4‘Ô¸ƒ\ýô—IE®³ª -[³Óä$òï*Ò  -ò,äflèè«9Éå¤È[%iz¨Ã–—ÙV©K¾uE:Æ®Þh/ ;Zm¨JeŠ^ZO@j€Ç–†× ²@X×ÏÀJMô]ùºî §áÊ/ò”3­šü˜,åîiŒ;Éy`ç‹&—añ o„òŽq•Ïwâe l¼'‹ë×>É$G?¢ÝœL§`…Iž0ú²ûnè$o¹íç&³"îåô‡Óæ"¥(çK™‚¼àPgLáü´ù5~þ÷Äçûu®q<úq`놬#™¼·¾’,¯Çíım1²‚ýIu¶MÝFQá5Ïî~{o‚†ÁÕÄ4u¨Èq¸n ¤4“î/;-ŽDæçÚfìë%õÅa’w1u©ä &J\ÖÉ£’ÏjH\IÓÒÁå¨:âBŠK5a›’mćԓ Õ‹ü¬MA2—ºPi„\Òäñ ³zèN[V¡š &}¡æ 'my ¡ß¼½~¶ÝW‘sßBÐ>¨/š:&Ôx6•–˼½1‡ÎrÑ7ïÔЋÔÞ£‘äÌ­1S Þ\hUÔ:»­ ’ž”;m3i.äõE6F,]¦¼vëjYV\M-•‘SLê´¦“¶ÊPbÒVɇ¶2nx#[¢‘%íÎe®ÎÏ–#—¬-ëރʳ%uÚqnÞÎÝ£íY-ƒ§ yf¦~vK‡–o¿·ìkŸÝ ‚Õjsªê|5yLiùo|ÍÂòfØÂUs<Ýçþ¹Çñt£Ÿ{è™`ÇD¾ù"l^Àhj¨þŸGØÝš7—ôœŒc sÇ£+' Ëáθ,R*Ê@’ôVyqñ7N²ÊáëÝ_ÌË -endstream -endobj -39 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 38 0 R -/Resources 4 0 R -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -40 0 obj -<< -/Filter /FlateDecode -/Length 4845 ->> -stream -xœí\K‹$É‘¾ç¯È³`Bþ0ÐAha¥eviv@¡CujZb©^tØ¿¿ŸEº¹[DZ¾ªŠé9hššª0ó‡…=>7stø÷Çÿ*ù¥ÖV2O_?¸¥$‡ÿÖ›‹•ý ?%½£²$ßrMǯLˆe)-…’¯üow©›¿þð‹ÃÿnûøcXZËÁ§ã?~dÅ¥ÔR\2X ,ç£ÁJi)%¤l°²_p‹­Y,ˆ‘ -ºdˆA­&‹1(„j°*ĈT|¼d5ˆŠ'‹1|KÅ`y9|pÎäAGƒÁóiÉPz¶xÁ/¹¶ÐL^Ï—`ñb\r!Ÿ¼ÅkKÎ9U‹G%5ç-^‚,É9‹Y(Öbñ2d‰9:“YB-ÑâÈ fð*dwU“Y\vÞ⵸¤V‰L^ÏÕR/yÁ¥%ÕÅó~IðÍhòQ0R./@–ìR3y%E,^„,„8‘b©ÕˆÖÀÑ]ô/A– -™<Èã[2dq%[€Z—, ¥,ÔY j\¨RµP Ô¶P)Ñ‚ÐÒB¹ ¢óàoAt%Q¶ zÈBˆ\“Yb# b€,h`aAŒ%P´° m ÞbaAÜ’kÞ‚¼%ç³…€ 1…ˆk& " ®Ô,,ˆÀÜì´° tîÅ‚Ô) "`7RÍDàn$ï,, ànŒèiò Ø ã}p7zW,, à.Â(XX«.ðÃla|e µ: ¸ª# ¸ -–“WÄ`´°€€»!ᬕ¸’ p7PÈÀQà3³à. $ÙédÀZX€õ¼-,À`K@ZbaAîú†åψi ÕâkË$ய0ŸÉk‹/и·Ðòâsn$à.ªhaAî>«…Àø+j°° w}ÄJmñ€»>4gaAî"oHÀÀ¬o‹wp]“××jµ° wKÁ¬‹«@“W8’·°`¼8ä¡dà.–÷faA…¸ëiɃ,± ×€‡°5bÉêâÂâw‡…-p×¹Ò,,ÈÀ]^6-,È… ¡ZXdÁÁ‚\ii¥d rsà!å3yyÁu²° 8 ËäA*daë¥ÅV-,(²Ä,,(² e°°ÁÒ<²o³<€,p³> È‚LÑ ÙR‘…%eÔcЪÅËa©pmò*x¾XXP --XŠ½…¨¥–Š ·°¿pºdaÄÏ“…¥A„Š…XŠ–™[<Yd[X€L<ï-,¨² ´° †Š*3; °B£–„ª-9®3«…•ò©¢…Tš@J“YÀ>\l' Pü ØD™cò KLdaA­ËÍjaA­%¸`a|“ëÍbaAsŸ…Íå…Ód ãà9gaAóõf —\oV Zt¨7QâqÛbæz³XXв$¤(&² ijFÜ“Po,^†,¡‘…-g®7«…H3Po"£7·*×›ÅÜ.î¦d²xÀ]d¦ÙÂ\ ÞDfkÕöÀ› -ò3›Y¹â¬æŽô&¬­æ–öbµ*æžørdn@LT–5›»XVÁôÎÜ6pÀ_äct çÝ'½E•ákf-|“ aAÈîGÍ1›Å+ÍÎûcÿøëá—/X|ýçáË/Ö2ù‡” EFæÂ:qˆ„s=ðĺÑ+ò«M~œxÒx@cà·¶ëJÓ•ž‰­Ó©YZ#[›mõ„[Ér+Šþ -:!ßÜÓÕ«7Ô Ç6Ôôj“'ž€Å¯*«M  ÔÐ\:{ º°QK›FgYû&&e?þ+ÇË?6žGø„¹£íÞvƒü8ñ¤Ø9€¢Ç5?éà\§&ÓÒH[@R·£«W3n¨0Ôôj“'^q¬ÅßO&>" ýh¸R?‹,N"%/G¾ É7uºÇX‚QlÇžE=Ç-æCm·=mæK¨„±tÅl ù|§7 ¹CLŒ,„!̇#ï­ŠU -,#ø(pƒ[Ot¦¬Ïv¹'i\ÿ;^þq)°[ŠܦÎ) -®¡FØ4þÍðA‘â|r5.•ø…¸ã_¿üÍü÷÷ÿõ zúáËáO¿rȬ‘|¢ýˆŸŠë„þ›¾t^é×HTa>—:û¦üë?øýáß ÌÑŸõÏ‚å.2 -¤±¼œö¢|ÿ»ÿíþqŠ‚šÿ<$ßCÃþýÀ±Ž÷h½²xA¿)ZÖ{¶Ç{Œ÷a6‹”DèÛ6;¬6Èl›»6ಔÑè¬nð?ñ<ð“Ï¿Ù>ë´øÛ}9·p™÷ èçS·ãKï÷åü£Ç•qt»uÜϽ»˜?ß’ºåÒ­@Û‘Íp«FÄsÓÙ}ø·÷C+i·ZQ É‘Õålªåá[TV9»qWÝçAùiâêB_¯s¿u7i×¢rm/ªJÛ~©¨>y7)Õïæç¹ÖÈNsoå÷Ð\iùÒfê.tÿ«ÊÿD6‰7¯G?Y=R~Tp¢ì­ýDîOb-n錀w:;ÍlÕGé´6ç]ýe{£è uÛ‹ÏҔ閿>Ÿ×W鉇Wô†ÌÔyâCç®· q6âQ-ɜҖPøl;¬G1k*?³ò®tl)Û‘ä¢þ¹g^^³zìl†]ÍË+4©8ü•$ò;Óv³Ç¼‹Ý/F,¯€Û‹‹¤t¬K*Õ•fÜÑU·©èTùq"(Ú-ÄDø{nóXŸŸÛ4Zªƒ‡ßBƒgݧbåq’ß ¿*Û݇ntrË1úß=8äÔÍqêuÙ{8Û»q™pK°[¶îñŠC‘KK¥D”Ç¡ûülŠ–ÃL Ã<îPO:XÀž%ª»éþ¤rÁ1õ…‰5Ý™¯BXÕÝþö´±oô^€SŽFïs¹•L²ž5TøwG[ßÕ7‘øÂÁ"Ê,ËÄ[§ü¨™_£¸í`õùù8ñÂ\PUü4¶™î–ƒÉvmasîó¢1Înáýöã‹Cw÷H»~cA½Ÿ,³_ú¿Ü«÷!'ÎÉ|å'S.{ßÕ5E~ãÇçbõ÷v÷XUèÚ7ÂÅ™L3/U±—¸F*¾» ²+–U%'Ÿìheµ#§w0ïU„uöÛl‘èû;z‰ß_J9Ä*zŠ»1NšÂ‰“¯~Ý络Ÿ%‘ësʼ´S»’ƒöÌ饧¢*bR45‡lÉ.éØÕßÍÛFe¶¹£·R–ÆÛ—M•?äÖ«6?ùÅú‹G7 -¼ÌI({œ_46óžíò÷òP>‰¸½\®mŒy÷©ª![HûqÍÓͪ¶päøb¿åý¢¥žƒåî1‚s÷OëžuøνÙ"˳¿c'¬ãßrÔÌÏ5o5¹íØ­ã—Ï]Wz¿U¶ÆûÑÇyÛ^9<ç8%a€;ÔúÃ?ÝåÛ:<¿ÔÕÅÝê÷q1Ã!Ó~ÜG^;òpš(=ΧšBÀ½Óß:vîh;6ÜÅiòv\q0d·3ìWq2ê9Æ™N)N;ÚöUg Ü9ïgD¤º÷áHÁÏÎ7\õÊç©Nôpu[e8êžçJ`¾ Æpïüúêʨ®ÐQ øz~…ã‘€{¶Ë7¸’‡¸¾7àøí W Ë~Ü\a.å%5Wúw¸· ýußÇJ[”Ö¼ :}èí^º³½é™ÕçÊÃ"¶LëÛ5A1Ò¸ýAÉ~ÅÔ‡;ïH /ïKŸà­5,ª’|̼£ô@€=ÝåŸà¡¼éâníñΣp!Ê­íǽz„z%y[ðÝt¼è÷+ЮxÀyӢߨù>"ýÓééã©œvfâ7à• <¶Z<Ýå;sòCÜô«?¸ÏoƒÅ´÷ÃW‹+¨r¼²GM}|ýÐ#/–óé´JåÉÊ1Ãuõ“¤É5¿>™äyòKHi}©Íx–ôé>ßÖµøPFä-7q2fµàô† Ÿ"“¬—û»nˆ»…Šq/ÃUL5öˆD½ÀJ9;0¯mÕ0.UÏ/Bº’cJciÓíQND$@äH^RÝ.èØõtzçѬ&jFhÚûX/µM¬§)$ JFúI»k8.¶ìE‚ó1†ŸskLÌiŽ­Ÿn½·¸—o@Í&q¼U‰Õº×–Až¾ÓOÆ\Ü?uÈ -n[]ÞÚéãŠÿë§èÎûf«¬Ò†ùI¨õƒP»]"]_éý"¤˜S¨Õ_aSå¥F™j„à®ä±ùæäéîäøG|Rð†ÉÓí;ÏæäŸúƒ%Ïqy*ÕëPßmr—/ne¼>"Õ_¹I ÉŸ -ÃñËD­ðÛµ>E>ÏøÒO©î—Æð2ýSÚ]+vöûáìïâX¾d÷÷Èw>Àÿòú†·_¿òár]OQ*”aa(+íÈo•WÉ‹ÊÅs®ô\}húWÇŸXá×ëW“è‹Þþìw_|vžî×o¹•\ƒ\Lòë¿}E™Ÿ¬yÌÄþ§cq&°£Þ×ö-cZ\ÿs®Sqn!$¹¿“ŒDV–÷õ­ÜóÄ=³¿$¿®/4ùNæï‡Õ%ö«<xÍ"3ˆs@Òm瀳ã]2?2‰é’èÔt“Häñü½‰¡º<‰Ct­:¯Ì/mCQæ¼E<ÉýNê«Iå¯ÉP¢]”©B‹J¨6¨IZƒÐ°“A: âwN†&sð•Õ‚œÂzÝ‘h´kÒÎ f»>Z–*£U]›h~Š2]=Ö¡xƒvR¹\á"ªqQþÜ’3[@lâ}úM¸ ž‰F$Nòë!£ušXá’¸ž&F5î W7ýeR‘wU!4…ÆoÎóÿÏP‘•‘gýƒOؼ  ¯æ$§ƒ"o•¤éy YðTf[¥.~'^Ð1vuF{ÝÑzhlCU*SôÒD<©[²^7È a¢Ÿ•šèD~…wÓpå× yÊ™F¨&7&Ë^¹{ãNrØùªÉeXÜó§L†¼c\åóB<­ íÉìúU&™d~LÜgL¦ˆS0€Â$O}]É­øóŒòç 2kÅ=þvX—¹Sàº6GÏ_æÒ“?þÚü?ÿs ã÷ò)ˆó:Ü@‹0°uCÖ+DZ>Hæï: -q„"R°?ɾζI¼a¬ã!(¼¦ÙÝM`—–~ ¨\MLS‡Š†ë•fÒ âi§Å‘ȼ×6sÁ>_F¹9Lù#""¿î¯¤Áe¼8—ä£WÜt.éàRPq¡Å¥š°MÉ6b²CêI†êY~Ò¦ˆ<—ºPi_ÆÉ£¹Ìê¡…¶Þ…j‚8ê =0y=iÛÈcýæðz·Ý»ÈY"€©ÚõUSÇ1užMëeÞ^Ž5'ùB‚wjhŒUìÅ‘äÌИ©m.´*jÝzƒ¤'%¡m&•…bÀhNhcÄ"2åÞMÔ²Þq5µTFN1©ÓšNÚ*C‰I[%_´åqeÅÙR PÒî\æݹÙrä’Õ eÝ{Pi¶ŒB»œ›¶sËj{T·AS<3S7»¥‹–o¿·ÄιÏnÁjµÙÕðu~< 5ùC\¿‡íjÎÀâOªj›ãé>·÷=.w7dßCÏs,£+ž¿ln‹ð -F»ÏPCÉWÏínÍ™·ôœŒm s—[Wq|âaÝÜ—…KEþ’uä¤w=(ÎÛ;YõbëÓÿI˜ôÈ -endstream -endobj -41 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 40 0 R -/Resources 4 0 R -/Annots [ 42 0 R 43 0 R 44 0 R 45 0 R 46 0 R 47 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -42 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 615.753766 101.433071 597.003766 ] -/BS << -/W 0 ->> -/Dest (cb1-1) ->> -endobj -43 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 596.806744 99.933071 578.056744 ] -/BS << -/W 0 ->> -/Dest (cb1-2) ->> -endobj -44 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 577.859723 99.933071 559.109723 ] -/BS << -/W 0 ->> -/Dest (cb1-3) ->> -endobj -45 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 558.912701 99.933071 540.162701 ] -/BS << -/W 0 ->> -/Dest (cb1-4) ->> -endobj -46 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 539.965680 99.933071 521.215680 ] -/BS << -/W 0 ->> -/Dest (cb1-5) ->> -endobj -47 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 521.018658 99.933071 502.268658 ] -/BS << -/W 0 ->> -/Dest (cb1-6) ->> -endobj -48 0 obj -<< -/Filter /FlateDecode -/Length 4733 ->> -stream -xœÍ\Y$I ~¯_QÏH›‡ã’ H\ Z1Bˆ‡Ùb„z‘øû|ŽŒÃ™éºfz§5Ý•vÛñ…íÌ,{6øùÂâW&»ä\R¤óå»Ó÷'³¤`ð¯¶Ø\Tö7øŸÂÙJK°%æpþŽ >-©—âùõ„_»KÙüõôÇþ¹ícÏn)%:ÎÿúÀ,òKÊ)™ ° -XÆz…Â’’ QaE»`‰¥h,ˆ9:²Ä ’ƒÆ‚ä\VXbxJÖYb¸dIcA [BRXÖ@ëŒQyÄy§ðlX"”5ž³KÌÅ•—À³Éi<ȫñÊc Yãd ÅX K°DFãAò9i¼Y|ôFåA—“×x ²8h§(¼ Yà]YåAÕxÅ/¡d"•WÀ39å#Ï™°„ì½ÑxÖ.¾éUvŒ“Âs%šPTd Þ8ç! áGã'‚O9+»ÕñîöÆ[ ‹s‰Tdñ5p²˜5ÀZ—4 p)-Ti à²_(SÖPÀå²PJ^ƒWÂB±$ ¼±à9«7%PÔÀ[ÈBع*²øBxYÐ@Ãï!‹#¯aÚ¼Eø%S¬†xKÆF <b <×çHx@.\©hXà¹Ø©aèb»' †9;N7³†9Cgœ†ðMÎ7“†Å@D|“5,ŒƒgŒ†Åf䛎4,À¾ä|3kXP¼A¾‰OÙ·ÅGÎ7“†… K@ˆ¢ò ‚¦¢ì[`òM “Æ‹ÅÒ° ÄÈùfÖ°aòMDôjI s¾™Ôrp7 “Æî"2àù&"[-·7ÞŸéÌÌgV+Ðp¶ª%ìÅi•ÔšørŽ¤ &2ËÕªŽU0­QËø‹xŒ˜°VŸd‰*ÂÎÌœx‘Û‚Ýœc6óWš­õ±ýíôå{¾ç¿ýûôò£Z)ë?Îï)rbÍù>ò”{¯–:ÜÂpt϶Š ?ª‹‘ÊÓ[ÙÅÄaŒ·œÇ»2³ÔáL]m&F²`a-uiŸïô™¤jŠ\'㙃Toœ7LÔTû`ŸÏ®Y9ïT’&ëÓ}>‡¬G—þêÝ Û8›Qþb^ -}r¨ˆxª”ó»ïN_~ý«_þüw ï^Nþ1BOÄ„NP5.ðùC»øo×Ïô²òóiåÕ¿fö}âOþr~÷ëN•µ¢¥,€#( 'aaêüÜ÷Ûµ'RpëœüŸ?‡÷ø|iò¼çvë¿xk8Ç%΋1²Òææl°'aŒU'ÀOwù(û5³-ŽƒNhcñÅñ! äEzehoKú¬«âÆ%»¸ý}ùÕoþðõï¿™Æ 0f¸¬)6ü e âÙ¦}N„Nˆc÷ãï¥(6£ØÕXl6JÈÂ1R3XkWû5¹ª#…æX},#œ‰ÇmEië!´kšRdzÍiº#öyRÓ÷É73¤cv‡ëãlœ46çlsÉ `[ŸýÆ:Œñ£?ºé„ŽínƒpšV‰sµy;/8„ˆDÛ!Ï÷yÛ-I‡¼K¶ß´ê‹ëµ¾mVS²Kq ¾…Š­ª• ' Sd5Uû|Ÿ·U­”7ßD>:xGPßÝ/SÕU•下JH3l@zIûy÷¨dÿ*MÂUÙõpòŠ0!¡­÷(“åyám´1ÉÓ}þ?&¡>몓qi}òºpó@(+`Xû“/,_ûõ/Q»ntº´¿±ýe3¹]IëÛn¸**ß4BòF9ìDeéÜÝÞ‰ïS¦ýB«Ùy-ºá‘iÚè¼á£±Zðȃïþ±>omø)oÀ09Z|ÌðÝ`¥ž„ëç°5ú4j;ešS„$ÀˆþFôic…K7ÊÕEð=8ÀËn·ˆ‡\™(q{[ºê±Þˆ+…c³×\í=—x¬ÏÇ%RÝkÀÌ+ðÜ̸3ÿØߎ€±þ} -3îìcÏ·Æq‘vB>†ëÍ­•Þ¶Iu r\p®ÖZ2ßšz ëóƒ1=$/É þ§¦æ¸hã…û.#Pb»¸gQâØ{ƒW¢õ #ÄÉPê thyÀ>î¶pÛì<‚‰³weÁ‚؈+²6Ù+ÁáÓ}Þ88¤<äݪIMM[JÇA`Mm®ÄݦÆÙkêS?rÌþ¹ƒS¾—Kdmö÷Üc¤¡mÍ#Å¥&ËËL gŠx+UõõžaJÅu‡"Õ©wK!™‡í|c<~¦b;ž÷S®šrF!c_Cئåõ©D¹™î˹Ùtèúeµ£¤ô8ÌtžÇ”)îS”´ÒC•¡ùÔ°½{¤ã4ý(äí¸jnéÏíõGb­#µ¿S{†¿thkÃVU§®Éò~®ß÷2@óÙ^RXK«œT9cöûwFãqjdhòà™Â‹ÛnìEš^ Y¥¹¥±ÀÑTImËT–¢(Ôçs/Âò<~÷†nín¹°^SåÞ°]“n3§çjàç5×ú.ˆ&yµ¢øt—·…mŸÌwc‹#loŒÚ¶+.݃Hâ„*¹èÊ~Ž›UÅž{tïÎÕ*cZÕo”®o80"˜[ÂÎ!+tŠ-È[yTö®9Ž3v±dÃ2Ý `Yoê]­ýl—7.E?ÄÝèìXgcÝäûEžqŽîÆS¤Lœï*Ï™¯¢<{(%‹ó¿—\gt; †S¶3{8Ú®=„f<±q)£²†î¨ã¼‰;ÇŠ±Â‹§$½„Ÿ‰ kNzt¬§»¼­cÅd‡¸·+d½ËɵU†xãæÏÓ]~0åW¸áMÅ|LóSùŽ2úíÇ¿ÉÞ„ÛöWQ ç}çh1F]¸áÑ!Oëª:ÆîøêÒ¾F ´2²üs0yqˆ‹jÝùVàÁ>o]#¥âÈX{Ë}ZnÞkIäî·³mË·ém -Þí&n“EÞ¯Ô!z}¡ÏÙÿ®‰à¶­¤õúCËÕ¸¶Eˆî5 ƒ(í*B’{«0áž%.X+?9¥ëi¬5ŠµºÝ¬]g½:Ûê-vWé5—^Ó×ï}åܼC×ÿ¾FFŸ|­¢R¶°VX‹“‹Íü^‰;îƒðÉ0·ò¬]Mè‘z«mòN'=ÒÇ$1ÏEÐ[[9†¨_©ñd28NJ¬^{Ì!ˆªÑ}©®IÔ¥9‚sÀx 9ás—õNu½sÜžZŽXœyàÊÔ9nå‡PžÐ=¹½žúØ2ì9@+Œë6/µÒROþú]ÿ!«ÃNáCh”>ìæh g]Û¥{—žÈòÈ^7ciŽ9«éá¤ö¤A oêŽöºÛèc—øŠƒïö©²nÚ7›¶]Z@Y›=~Çôë¦ýÙWï~ûSY¦òwÌäoªünü25’6g~ÉîZ¡v_méUžÅÃí´»q‹¬#8h8¿—ù=L9V°üŠÝÕ/žïóÆ•NU›¼[õïã‹Ou‰^„»QqHe)ÎÄtEM&[ˆ¼ÞéœHyxnI†{‰I>·s¿Ü•¹Ôæ¢Ü©i®{sÇ6TþÖÌgƒ¸O™;JYïÉç­$ˆ¤RÈéëÓñŽ’Óä–c¿ÄoæóFé·)úvDß%âÚVêiC]# û·>5R‹s=¼TK«®jI°'ä^Ø©?£Ö í/7mUökQ‹´Z5ª§i®e z‡QÐÞ”+£¢u('F/Ö÷¢Å¾dK[™öÏ›uhs»$Ò7g|%#[z,üÂìØ€Õ—6÷ÉŒš¶iï«ðêB>zÎ\‹¡Çàõù>o\o«ï0¬òn4úðŠÔÖBxhi'Ëÿ^Ŷ“nsž¤+’Ûmék÷àê}Û÷Êøý<(Ð;Õ´æ¡®«Ÿa•nÚ¶÷áÖ‡:¤ÝÔYÙëì. ^æ1'ý ŽH^ë#û#py/¨dŽž”Ýzä÷¹¤îæãÊÆ”úl¦q#[¨¹}SÌp©ôÏIŠŒä›öj7S_ã·”²•oú ¿Ûä^wh;¾r¶»×.¶è‡!ù;y|>;âÊ ¿Æöh–á¾KOB5»ûÁîýô‹Þî~ìUÎû¥ãƒ-šá`‚ïO±¾Jiëëô†ok:þ¾sDêÇ:-g~}3Oã­![•dÏWzÖo)ú~´âß þ.~µšD^´ößWc}‡lç´iùgý’SÉÙõ‹I~=q9›2Ѭ<Ý!3±}4c,²ã„î´ËÉÂûʾ¥‹igµËÏ=ôÕpÿлñËdfVÅþ»N¬Ñ3Ú#™WoÛÈüE=yñí*QêŽNœ’l;œï’ùÉI G¢ÓM²£.¿|Ò?–as‡èRuV˜¿·uI˜óñr´~O}U©üõ }¨®]ÎÒ»|B¨2¨¡·¡a'…t9)Ä/Lš`ÌÁVsýÁX¯9v¥·SH—“Ú®†¥Òh•g×Ò5?E™®îóP¼B»ˆŠpâo{ÄøþqKŽlnk¾ÑÒL0·î$^Nœøȯ§”Öab… Ýõ$Ñ‹q9›é/“ŠÀ°©ŠŸn4~E•¯P•‘§~à—kl@^ÍIxoòVI’ê¦Ki¶겜»+úÂ鯴gЭ‡Æ6T¡2AçbÁ:ðØ’íðºAfëúX)‰¦ é&9 W~ݧœalÕ`ÆdÑ -wcÜIŽ;_%9 ‹[þ΀!ïWø|'^êÁF{2»~î“L²3ã´›“ â  0ÉF_+¹ïÖNü=hýãÜdÚ‰{9ýýT9çƒã|7zË_##&{þÚüÿÿq¢ó×ýëõwf …غ!Ë“Œ É’ù Ô:qlAt$`’mžmC÷†qŽ;'ðšfw3½·´AíàJb˜:d7\×Y/4n/;-Ž@æSm3ìõÒ÷Åñ³ûü¶~— -r9¸Ì“çç‘|C⊛Î#\r¢#.ä ¸–)ÙFLvH9ÉP=ËOÒžç"ŒàK?y4Y9t§ÕUˆ&üt¹¼“•“–<ŠÐ½½>ÙîMäØwS´ꫤŽ…1užNõõ2n/Ç™ãë«!}óN ±¼Ø{~9skÌЂ6ÒG5Ïn­A“R§m&íÅ€ ÐL§S—)¶n]-uÅYÕR1Ťþ]j:H« %i•xhËãöoDK~@Aºsš«3³åˆ%³B‹²÷ Òlé;í87mçî§íY,ƒ¦ qF¦fv ‡–…³wÔ¯aÐZõŸZÕ°y~K j²É;_¿xÖä!Äß–(‹2ÇÓ}n×=ŽÕ^÷3Y~JÆ{“,…°.Â+å>C Õ¿^XïVŒº¤ç ”u&̾4ÃÖïÄ¿VÜ—‰SEþÊXÏAoæoºKÆêï—Mé¤þüe—s -endstream -endobj -49 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 48 0 R -/Resources 4 0 R -/Annots [ 50 0 R 51 0 R 52 0 R 53 0 R 54 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -50 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 686.175057 101.433071 667.425057 ] -/BS << -/W 0 ->> -/Dest (cb2-1) ->> -endobj -51 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 667.228036 99.933071 648.478036 ] -/BS << -/W 0 ->> -/Dest (cb2-2) ->> -endobj -52 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 648.281014 99.933071 629.531014 ] -/BS << -/W 0 ->> -/Dest (cb2-3) ->> -endobj -53 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 629.333993 99.933071 610.583993 ] -/BS << -/W 0 ->> -/Dest (cb2-4) ->> -endobj -54 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 387.390971 101.433071 368.640971 ] -/BS << -/W 0 ->> -/Dest (cb3-1) ->> -endobj -55 0 obj -<< -/Filter /FlateDecode -/Length 5084 ->> -stream -xœí]KäHn¾ç¯Èó£‰ãxlöÚcc° , Ǟܮ]Õz}ðß÷GIÁˆ¤òÑ]³åÃv¡»Rd„‚âã ’RªíÙàç;‹2Ù%ç’"/ŸO_NfIÁàÏ:b8XÙ?ão -gk(-Á–˜Ãù3|ZR .Åóë ÿL‡røëé÷¿9ý÷8ÇžÝRJt6œÿü‰Yä—”S2Aa°Œõ -+„%%¢ÂŠvÁ%–¢± FHäèš• •4Ä ç²ÂÊÃS²þšU †K–4Ä°%$…e ä°Î•A ‘w -φ%BéQã9»Ä\\Qy <›œÆó~‰‰l°¯,1Æ5A–PŒÕx²Kd4d!Ÿ“Æ‹ÅGoTdq9y— ‹ƒvŠÂËÞ•Ud1ÑXWüJ&Ry<“S¾æ9–½7ÏÚ%À7½ÊCDÁH1)<Y¢ EåA–àÓx²~4p"ø”³­Ž£Ûo5^€,Î%RyÆ×ÀEÈbRÔ ´0.iàRZ¨8Ò@Àe¿P¦¬¡€Ëe¡”¼®„…bIxcÁsVo K ¨!·…¹*²øBxY0@Ãï!‹#¯aÚ¼Eø%S¬†xKÆF <b <×çHx@.\©hXà¹Ø©aè"Ü“†¨ëCtxÀ®§5,ðÀ]OÖhX@À]ï1SåA8°†@ÇÅÛì5, à®·&iX@À]„‘Ó°V]à‡QÃøÊâr6p×eCp×%l*/-ˆA¯aw]À%h;7p×ã4, à®#5,€£,À£fÀ]@HÐÓÈ€Õ°û=xÎkX€“-i‰†¸k ¶?%¦V‹Í%jX€»6Ã|*¯,6AãJÜBË‹±hX€»Ø¨¼†¸ øÌãì¨NÂܵ;µÆîZWŒ†¸‹¼!hXwX, °¿-ÖÀuU^YLÉYÂÜ,9 ° /&IT^ZàHVÀñb‡jX»ØÞ‹†¸k‚÷Dà®!¤=*²øì4,@^ÂV‰i$«‹q@ -Ü5ÖxÀ]cRÑ° wyÛÔ° &&¸¬a\; b¦¥¤5,ˆÅ€‡”OåÅÇAÂd@€,•Y(‘†H¬—âKÖ° 9ÈâÓ° 9È‚”AÃdK±È¾Õò²À]Ôú€ 2E €dK.D¤QA«/º%'ÀµÊËàÙ¤aAJ´`+¶ –Z2‚^ÃüÂé’††9;.7³†9Cgœ†ðM®7“†Å@d|N“5,ŒƒgŒ†ÅfÔ›Ž4,@\r½™5,(Þ ÞD‰§Ämñ‘ëͤaA!È¢¨<È‚¤©(q LB½ tÒx²¸B”¹ÞÌ Í@½‰Œ^m d®7“Ú.î†dÒxÀ]d¦QàÞDf«ÕöÀò3™¹âÌjÇÀzöVµe`€½Ø­’Ú30_®‘Ô¦ÄDe™£Ú5À¶ -¦5jÛÀ‘Ñ&lÝ'Ù¢Šp‡„=3'¾È‚° d÷­æèÃüÁ°­?öç?ž¾ÿˆÍ÷üÇÿ9½üfí”Õ P÷¹°"®ÀÜŸOȈ]oùõ‰Õ5‘YÈ:6s…;Ùý6âå$Èά°É.û:Ò‘^lt±œ[x£vÙõ"/EÐ_A'ä›3¹bÝW¨Qž»Ñûå¼Jr¿t±^W’ªçËé÷°ø—á4I•Õ¦´Ñ²0«›`[5•nu‹}fïb²ÕQ÷ãO:_\žðóàI÷€4z€1x UÒL=Àx€ C—±yÀ¾[Iñ€N= Ó7@V7Ñ-ê6º­8Pðº4r¿t±^W’ªçË_ð}PÈ éªGc0‹ vt¡Täª\ÌT¤ªûØÁ4!­m ë'Ó„ÚèÒPà¨mÔÑË:}0 K­öª—çntq=ƒmÄÕ‹…¦t}+ø(¸ qú·ÃÇ+Xdéu(£ìH%èÕX\p®¨ƒØt.Þš4(ŽªÄˆ£4¬ŠÝ:¬õêàþÀ½]½XQhJ×÷寑øÿ$€Ò¶S£¼BþXÔ»54]YPSscͯ™s/œQª”®íƒíqŒå¼]Ö„˜†å‚]xsñQñé9_%✊íDÿñà eévKe&¶¾b …Cᶠž71ž?|>}ÿÓ?ýãßÿë¿cÓü1¨ ý‚¿™›j¨>ð™ÂF øìþ9í¿ã6–ðg^ùÌ[Çâ˜^6~ðÛgh§c˜g¶1õë¹ë¹êunsY¶Ë.Ïe£ñ8zùá?Ï~{ k¥"øÚ×9iÿ+Î×®i—¯:‰›®š\;ßá·çºsÍ~í.m4;m¾¿ܯi×ípí¡ëmý¼Ÿc–¡ê‘¤îÖóßÔ‰çtŠøæ宓0øÃ6÷>°Ÿ—µ‡O6Ÿ­#nÙ—X¸D]b µ?7ëƒ=6žžs/6üúç|ý¡E±Y·,#³Šã0×”lè*\B]|ÓL;äÛàUìAAßÿøÏ¿ûéß~îÑCS´°aV ^öß5bLÓd‹Ü1`ùöl¶>Ù0¯9Gìjm»[߯Ù=bõR–e÷áÕ›°Çn^^½«Fl{žuw­›íuáãµ/)Ö5y"P‚&Ï~éÑÎúä¥ísóú/×Q5DZêhBÕ^—QŽ°£ÕÑz2bÂ}}äYO¡a`EÜ,Ž+Íí6ª6ÙtfmóÃÛqÂÎY–Ex´Ck©^‘Ÿ®Â*euÕ¿ûñÿü­Ü\ÄÆÁfo.‡°ñŠY×ß«»˜5™˜ö·žðC¸‡ôIÝà> -óµ jü(6€_&÷+usÙåÛÍÎöª2³yøT–¶° ¿lüÙ$vÍõEØù°m¼ŽÃ6Œ2É0YÏ•«ë pÜÖL¾ Çž ¡p!¬íÿkp~~Î7äVo€Ê0V•w2ùŒÊ¿žÛñ ÍìJážÛÝÚ…ïz¾ñQ¬³ EÎT‘üyD¢r³qßrÎ7|ÍY* -ڞ׬O]Wë5ˆâ©üðã>G¨êÊ<†CåÝ„,¼êæ]o Em瓺|ðUæ© 2´DŽÚé¦>ã¬OÍÆ­©(ˆ²mQ!H¶0¢¢×¹qXÇï>_[ìG¢-pОŒà2b8µ§üÜz .×Z,TYMVÐÔd÷3nŒIûA,µ‰za'ö»8ùMèë¶vT+znµrIE<ÒP–kî–mÒ½AÓÉñ=¶“Y¤¹éôä~}¨þâKö6›|O Vk=\! ±™µ*¬u÷nAFÌP!}ê;]gNÙ87NHµ fE.ŸÖûPÜ sÖ°ó?;ç½w~ßä4sí»áò˜YíB•ŠÉÜ¡¹¶hEšOû|=º„EÉØ¥D‰f¹UWj(>´"ü¿X~^§›Š¿G -õ¯wžœòÞ†ÜĽc\Ñ’}¬4¢ŸŽñŒ=¯¡ÍŠÓðä,5z³,zzÊ;·LnâÞ1ÂW—Eü˜ÈžC? ‡Qz 8“áo1þæÜ›"º6~ÎÅÏÔ üÌMòá–L•¾‘N“£ñÝÀ”§Šü½ŽGíé)ï]§&î ß7­¿}íÖOk¼í.Š°_ļz#IJA¸vÈ/70ÒìÌq×H «ìU™ ×£&¨Cá8Ý–Éq»ÿY µšT ý¼…î†BKÞѨé¤u¦k·!Ï:”…t+.ô!忺Ÿ¬Êbž»[â·”ƒ›®6Š#b”ýÊÇæBñ" -ßš·ôÅ_qšôõðƒPïD¹ç2N)v´»VGwÏšžÐiÏì~q³È.N›E{º‡úµÞõeštò•:Õ›&2­ä'§“ÍÉ­}h£yzÊ;§•&7qÛ¼áFCüૣݼƯ²Ñ¨ArÓ)©ÌÈLwjxkÏÂwÃöϵ^’Y‘èÚÝË’àûVÝÔnz‹Wz›G|àÚŽnZÇU°ß»”íö¿è`…›`t{ÓlÝ%ñ Ñí› Å¤9ÛÀTíØ>Ø®i™À´nÙþ(äP|Å;4œsqÔÓSÞ Ö‰l⎺K€òüB’Ä“i µþ=º-Hü.)œ x©Ù;Ï;<=ç÷‹ª¼÷ÌñvÏ;¤¼NSÌ#æ‘w-n'qÅ»)–›\ÂÄ­“¼ãŒö4ZO9¿ÝÁš“ íÂŽâ¹ØôX ?;åc2Vq¿e5›ùÑ”³f^ãWI2Œ¹º=sÞyr€Ë½,¹„0W²§·ßÞ²ým~wÐþ=‹Ì¯pI(íÏÙ‹i}ãÇÐèEÏÏù˸ÕU7Õ´CëC“·„›nD?|gYmyÿößnÿmÄç:ÆŽc}Ü~Så‡ý÷ê.;o³õÅGÞÝuRCŸ¦5yìæ(GJñüY¡£RÖ'v9ž§ü~¶¨Ì®(u<»Äºæ7\Ϧ²_Ën zÙ¯Iê¬:ðÏûwéS8›õ½[–®I|5¿yíÝ!wœ´ÂÇô•¡/w¿4„SòK4}>;h $~ï„z‹WÙ \MrCÇ„+†ÄãöÂbNðµG°çgVÙ1ë#Ö¶ìcî>ÇúÃl‚/§¸¾ûÄ®ï¿2ØT¬ãdž#-ì«©œù}+Ù°­w•ÛUIö|0s}­è—6ŠÿÉðËÇøÅ3«IäÁ>þËj¬Ï'[ m;‰ÝÞ -hVrvõ “_OüVHRÈDýi´;d&îM;?\7Ñ.€PìlóHÀ”Ù?¶}xÅ®í£ õjx~¨ÓømdÚ–¹¾¯b[X!bf´×d¾z³ØÌoÖÌ‹ßb%Ä*3?_ÝNHrl?aŸx—l½ÄpM4b¹N^ŸÚǺö±4›ƒØD—ª³Âüuìv+úâå„eýL}U©ü>¿zªª]gÍ ¡J£†:²„šÒ夿3õÔc6¾°š«÷ãa½Ý‘¨+uœBºœÔqûÙB³Tj£rŸZªæ»(ÝÕ}nŠWh«ŽvjâW‹1¾~É‘-PmÍéÉn‚ºx9j‘Øɯ§”Ñ¡c… Õõ$Ñ‹ó6r6Ý_:ñ®*„f¥ñ—"øß *B£2ò¬,5—⌫õE86yT’¤Ç†:¤i€Ÿ ´WTœ;¥æ³R_6e<ƒnÝ46P…Ê=•*ƒT‘l›×52CXÕOÃJI4UþÚv#‡æʯ¹ËZ¨Ó‹V¸{hçíäØ°óU’S³¸å—|5yÛy…ÏWâeÝØh&³ëçºH';Óv»¾˜ vÁ -Üaôu%×hmèÄ/.®{i;îåô§ÓºÍ9'ùÑ[~g¥Ì˜ìù1æ·øû_':ÿT_’´íãü’}}×°u ËŒßpœ$óÛsI µ;Ñ‘€ýN¶¹ ÕÚ>ÎÏT¥‰ÆÓMö:Ò6µ+‰¡ëP]s]g½ÐL¸A¼LZl‰Ì·Ú¦oØÛ¡¯‡eøý4M*~ƒ‡¹ó|ß’Ïâ”8â¡}K—œ˜ˆyRŠK—l“R.ÒTÏò“4…çµÄH#øÐwõmVžºÒÖ«C{y OLV.Zy¡¿:¼¾Ùî»È±FS´7ꫤ¶ cj<ê×Ã8¶=ÇŸù o×P;—±ç[’ÓC£§4HÔܧí‚\”*mX´n .@3•ÖΘªLqŸVÕ²^qVµ”ZNÑ©’šÒ*M‰AZ%^åóÖ¯eK¾%@AºsêWgúÈ–Kf…åìF¥>ÒWÚõÚ4®]wÛ³¸ ê‚Äž™š>-\ü*üûšØÙæLmTýY»6÷× -b£&›¼óëÿarŒÀâ—ý&Ê¢ÍñôœÛ}ëîFí{È•`Ž…¼7Éòÿù¡‹ð -F¹Ï§ªÿˆ>­õ’žS€ÒÖé0g®º:¾½!cmî´ÃÄ¥"ÿž“Þ̯¦NæàKI‡ÖÉúóg‚_ -endstream -endobj -56 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 55 0 R -/Resources 4 0 R -/Annots [ 57 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -57 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 108.503892 101.433071 89.753892 ] -/BS << -/W 0 ->> -/Dest (cb4-1) ->> -endobj -58 0 obj -<< -/Filter /FlateDecode -/Length 5358 ->> -stream -xœÍ]IäƱ¾×¯¨³¥r‰ÜAü@~Þ‚=¥FË€ìƒÿ¾¿H2’Q$«§¥ 4=Œ\Œå‹ˆd2Û\5þ|eðW"£RÊ1ÐõöÓåç‹VÑküWz,.Jó÷ø?ú«Ñ•79$ý‰ .ª˜½áúrÁ_«Ë±ûËåÿ~sùçrŒ¹Z•s°Æ_ÿõÌMäTL1j/4e4iã„&ïUŒÖ¡)…GÌYj>’¥mS”“—šÀY›„¦6Eã¶MlØhHj&û(4 >ŒÕZl#šÈY¡Íx ô µY£BÊ6‹mm&Z©Í9"o¤¶¬B>Im^|ÖFjóàÅ"-µr)Jm¼¸à´Ø^lŠNj‹àÅB:YhKàÖ•Ä6ð¢ƒ6R[vÊçD$¶e´éÓ¶Íj¯|rNKmÆ(Ûtb< -J -Qh³à%hŸÅ6ðâ¶R›/„?Rp»˜’à­–½Ûig¤6^¬$¶(_BÀ‹ŽA‚xb\’0Àƨ([’@À&§(Q’PÀ¦¬(F'Á€Í^QÈQ§ Ú¬‘€Àiðâ)HHà x!x®Ø^\& œ/è asàÅ’“°Àm Ö"aÜ’ÎF¼%m‚„€ 6µ„ˆëR  ¦”%,pÀ\ì”°ÀtáîQÂÔu>X `×Q -8à®#£%, à®s)¶°„@GåLrp×%, à.ÜÈJX­*Øa°¶¢lJZÂîÚ¤IÂîÚˆ ¶Etp×z<‚¹»Ök+aw-Ù a E´˜ w!^NÀ VÂÄ{´Y'a&Si‰„¸k2ŸàÓ@+eRxà®IPŸØ–•‰¸à·²2!d dUi ¤Ú€»Ú@ÃRpWë˜%,À]›„È›$,ƒ -Y°•° $R9Æ aAÈmHùĶ pí%,ˆÈÛÀ E’°‰µÊ.' ¢/ÎZ ¢/H$,@F ²Aö-–àæ"Ö^)JX$S)IX}@=©JmÁª×b[B›‰ÄH -¡ØHX€ZJ%8½„xÅé’„`m†$,ˆ¼ÀU$,@(RÉ"3—Ú xA’-a2]´#aA²àÙ „É&T™AKX€Z¢–ÚHs™$,H§ŠÀMPi)Å6ð˜°úábÛKX€âÅ&ʱ ¼8O¤d¹ÜL¤^¬¶À6¹ÞŒd ^ñIXuPœ&KXG›Öd“PoZ’°~Éõf’° ;z%žà·Ù®7£„™À‹GŠ"¶$MYð[`êM “ÔÀ‹Í$aAëÍ$aÒ Ô›ÈèÅ%Äõf— €»>™¤6à.2Ó a.Po"³•j{ àõù™Ü˜¸âLâŠôzÄVqÉ@{­¢¸f ¾\#‰‹`•e -âªÂ*— 4ðùm0aZ}—¨Ì!"f¦È™á„ì¾Õ½›»ÓmZû×ß._?!ø^ÿöï˧ߔ•²úÇÀø…RXC¶ú^©2Ì– @ZX‡¯ÄlÄÜÙ4ÆyÙÙD0FÀÍ×ö³d<3×éLy6ZÜÅ‚¶\¹}|ÐqÛÅD†sŸà²ÅdJ°O”N¢lÏzsÙŽ7îb¹}|Ð[p»5ëß~¸¿§µãp„³@c¸é‡Ÿ._÷Ç?üî/ÿ9?|ºüð @¸ïQ™û²hƒŸð<ŽæKJø‰ðŒ„¼Ð(t÷åkÇ´2&ÎÿŽs;Úümž—Û>Níu,}½ÞcÛø°ó}gZ¿ñúéμóóðÜe~îÿ„ßæg{šÇ…ÿÛt/¿ýëõßîÈ— W«YŽ,ºÂ’7Ì·3<-ßί¡ÜnîWY.}?­ÄCCßÛªÈFQWÑœâ#I¢›ý÷`Ó("•K—Ž—ÒýÈa1ktŒ!Šï„¬­yÁ£#^å³åÚ‘»CEÊe˹ªTMwðõ®“6Û¥GÞ2s»Pê׿ýóÿ~÷?ßwç jÐyÖFš*6i5G\úŸG˜'‡,me7ÿk¶8+ÇW…Úž=Ç®Ia0€4+ژݹ¼[Íå}•qW|Xùcyn½ôÿ¡W'È;ÆNÇNTïÝçŒ}”BÐÕvMŒ(½b)¡¶æþø˜÷5x”fßèw-~Ç„¨^»Ù3È7f‚ú:>y -ì9ÔÌkžÖ.RMwR¯µNñÖ¢\µF‚,‹6rѤŠMŒêzx̯£.ªwdÒ.ó_fmO]ß~e¾a•L?é6ýtÅcæ6?ÿ Ë>ÞÎ? ~NØp!‹Ç7¼”V õüøíÁhd—)"ë—FæÚ“*5Û¬('¢¢R¨˜Q­©ùܘ÷Vsç—ëüBTóóJuf¾Ž‡ª3…á`¨)—79¥:Ã+F§l„Ѭ:Wjml¹sK«>–ZXcZ>¡wÍ„ç|E)Ž9-'ûVqjÈ—c—v! -#Å”¡bð€V׋öoÕ¨¸³Î!¯ f£Œ¤yE¼¬ÒÂÛ‰èŠÏùrÔDN€zNT{CM,îOçµ -N'd Ëé~½º*­*ø`΄è‰àì0籇cˆÎüBF]’¨¸ô^Ê+.+ýÓ‘Ç:~­—J}ìPÛˊꮑœóÅ oðŒ~v?^¯ü¶ÁŸ>€Âo~%È+€Ú§ ±ï¾a¹È‹¼‰ºÊ)³œ¶ h³}èfŽšÊÂôvôÚ€›áŽ2;H^‡À~©xC¸¾rgdQ»Ôú)L5×ýõ‰Ï¬u/„*€íŽ¢ÊÓb•á­>>2R­óø˜÷­uX•ß¥jÞ´Ö)ª:\°Z¡@ÌF™‰™Ëe»*¥5›©ôÔøÝ\¤ã¨"Ç/(Lyy¸ÕêÃCÞW©ÁØÆî¾R«ÐhB/±Îwü>}zbë²Êeyö Î?9æ½ëüÎoŠ»¶?CÍpúÒŠ~ÆõÇO¿¬8o•Š Zñ{o™¬.û¬—}wáòc‡W]—”ç±T}l€åqÙ¹ä`·ùÞa¹¼Å×÷Ö&È𛳬“?¿6qrÌ{ç:mm‚µï˜˜fqc©9„—‹ØM;­âñlŽF ñy)@Þ/o¸F½ê#Yïvý /ï›z"òÏûQR*{+ö9;7æM^"½iê[v òŸ$#N^¥þ¸²ñ¼“ø=úròSIà°ò±}·Ö ¯-ªÕDq“ Þ/¢)”]˜!ˆœoàði` -p`õZÐeFFºG3d؉ÙJÒ¬ ~«æü·‹„¿ qHš’ph¯|O‘îÉài»€1½‚üLÝ ýFà°´¼®–È!…~æ¶KݬïÕdð‘ÇH=ð§à}Ì’8Â<—(²„Æ»54eDxMD±Ä<±÷"ʹ1ïW¿…>×ÉÉ@’ºYT•9QÿXÂÍôï§ÏXû¦+dÏù4‡9í*R‰Ä¤aËÇ+fp;Ý\i;Ú<FDÉb–F×Ò½ŽßŽ¦qýB4XÞ“ísæ}n.Yþl${`°'Ç|9ë8™È.Ê‘ïÑ•Áåt£ÉÓu‘ŸYÜι@ÂÓQˆ ÚéE0#¼ÃÝÃÊóÍ[Œú:ø pÿ‘¥!á â/ž °€=& -ù@'Åj¼\]xB9,ê§óÊ“c¾§òȃ“E*ü6Nµ˜îJyø£@Jälî7•–Çî1•³¯ñnŒñä:ï«ù_ "„DÔÏod)Å)%z 0‡É™¶ -–íµ¹£¯ölîä³ñ'%äSvâ³e e“ïIæ0ÉÁ)ï2oå,>ÜÒ¨•Œ-_‘XÍ›¬5âPÎùr%ñâêê¼ÿjæÄ;õV“¶Êæ•®8­ª­Œû^ªšV©êMp˜Uš%Þ)¡Ù¶Ïr˜˜B¼&´4ú^ô‚5ð¾o‘šòÇå‰}[;5ä‹15þÈ ÿSc­ßú™A¥çß<þT)jþT^|ót´ÌY—Po}itou½ëÎ×¥RÞÕ'îòß<¡‚IX €ÈP¾‚ƒ.÷wÙ=>æ½ß<™ÆïR5ï·ËÎó7:$ŽÌås6yŠo¥FK?|„úv·?<ä½_·´m•!ú]…¿z#±IÌ_È®ïñØNâî·÷ oZ–ìJß&?òe@Ûiœ—bËÂ[ªÈ_õÚrRÆ4x‘Îñ‹ÐQún‡á©†ï&iÖP9?ÁbCþ -cf]³ëµicvvMGdXF'H¤q&Î]ÅÓùÕ­¯:¿ ´*ÇYþƒ MŽŽ,£ÿôkG»ÓoÐóâµñ¸ {”W}>°ú’¢ÙóZúónï2¶¾zk_›ìíF‘—»ª¡—åúÕë¾­•ÈR_«»§>¾Ú·C»}žâd•ï:úØe¹‘ÙSÞV·Ý‹à»H”b¾Æ€”á><ä}Áß[UvúÛ¾¢Ÿ ²Çb+Ëq4sŠ¿ÍF⽺‡Ír7øÅg?IJjÆýœ6Ñy?JÞ•yǨ¨¬‡‡¼ó~ -dý•Ý}eQè^ÁŽ3¡Žæ#ŸÓ3gi|2Æ´¡s7ÌŸòÞa¾±Ë_ÈïËäÃpnÊ­h3×Æ‚? ´‚œ1,WømÇée0[o «©úÙ¯ÄÚî ßÓ˜2FÚ`FÞ©`rB ´Ð¬SxÚÀªpîá1ï tÞW~ÌAOÿK²Ð‡iô!ø`ÉêìCÓ©•YUÛ˜µ ßYú°…“]g7cvøÚˆ¼Î6Ÿaˆ¼óÜÅ<‡ìu]ÕMLµÂ¥çU*?Wp«çÚdcÆr7sßÏ~J«ß¡ÎnØ\oÈ2θ¡”¥•¬ÑÊÙ–˜yßÙµ{ŸÒç®,ÓF–G™˜°ê°øwä±Ùô¸‚²á[ÂÏýžxÌPëþY?óI~€ÝYÇãŠÇ"så\á˜Çü8@*ß{×F³^Ëõ5vÖtJCÿº?²>GØê¿V)÷ì{ó-õ+ªœIƒm×R÷n†!’̧?üF6[QŽëM¶nÀÛà‡AÐñ¼»bOi>Èn¶ôÔû—gÏ].máJ6¢\ý–Ÿëj[~þ`V•TÝó«àÚɳ¤JÌ Û›Vd¼¥˜tÎŽ?§*²‡‡¼s’ÏkÓ3» ;xÊl´µö’’WWc‚2”øˆ˜Í)E=<ä Œœ»K!ž«ÆÎU)n—Áü¢6‘zØ}ÊóGd©‘bôü–u, -‡¨´,4fù d$ŽyïÂ!7~÷ÕÌçI·d=L×-i1z‘ø–䪶?O43ÓäÂC.Þùð©Z{ñkqOåü±Ýêýä˜/¦|70îCLü§àþèXŽ%^Ýtçxƒïç“©`»ºœbk(&3tÕ´pØå@è•íí)HW³Ú,¾|2LÉGÒ»t-oŽ#Ÿâ&=–Ôí½µ„UBiŸz"YûÝ ð›%÷<ÂâÜÇ÷¾|i¼ŠÎ ê¯}mÔ¹G¼]ø,¤5õE¤òéØuª*]«Í LåFõµgS5= ¤ÛE ~¥ëÔe¶öAk|ÆèDLÕ¨õ˵Ÿ@º]Ä~ól¾i*¶^©ÍUò•nêå[Ê»´Ûðz4ò/Þ|µ¿­zÉUš®uìÆnˆ·‹§æ‰ürñQèí;Vh_Mo$ºaÞFNºÛK§ÆTEL£ñ ü÷¾QyÊ? 5“âïûûU¿ ûf#/…4ÒCC’$`(ö¾ƒ¸x[‰ÛÐ1wÒBÝÖ»IlAD6Ðc®ì1H5ðX’M³ºFf«òiX9ue·ëdßLùeAî|úæª^·›3˜»oóvrhØù2’cÓ¸á#s¿mÞÁæ+ñV­Élú©Þ¤“ùt’j>íf±3Pèä£/…\½µ¡ÿúÏîdRĽ]þ~)aÎ:o9i Îð ðcÆd®ÿAŸ?áÿ\èú]=rtŠã| ä|Û°uA#ÿ¾t’̧Ubs›hi€ýN6©÷õÕZ糟âŠÆÃuöÚÓ45­u$ú.Ãl›éZãÉøâm%Å–È|®nzÀž.]}8þ웫­\áb|\¦ÞæzH¾S⊻öΧ*Ùa .ÆIq9Ü0wÎl²AŽ7i¢gþiT…ã{ CÁ—®·Q³ãÔ•Vžbè‚ØãÄdÆ›æ?Ó¯v¯ÏÖûÌr¨ÀÔíú2RÛƒ1µžLuå2,/[ÌqW¾¨ÎÛ%Ôærƒï¹–ät×è©-.F¨©›;øñ¦Ti‹›Ö@Ñà4]imÆXy -ó°*–òÄI”Rl9E§þ}”´µÒ„èG­„M_ž·F¼–-¹–ùÑœc:Ý{¶\2 ´0ŽnTê=]¥mïMË{×h{ƒ:#¡g¦º󛞯¿×øŽx±Ô«þ)«&õCº¨ùÃnëÊï]Ó)`oJñ‘Ò°Ìñð˜ýuíêF]÷ïu(rNGÿAOfá姶ß0LU»ž<,kñ‘€°¬Óan{^tùJøo^Üi—‘KEþiŽ“ÞÄ¿è%jùühóq±tRþüp¼» -endstream -endobj -59 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 58 0 R -/Resources 4 0 R -/Annots [ 60 0 R 61 0 R 62 0 R 63 0 R 64 0 R 65 0 R 66 0 R 67 0 R 68 0 R 69 0 R 70 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -60 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 691.173057 101.433071 672.423057 ] -/BS << -/W 0 ->> -/Dest (cb5-1) ->> -endobj -61 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 672.226036 99.933071 653.476036 ] -/BS << -/W 0 ->> -/Dest (cb5-2) ->> -endobj -62 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 653.279014 99.933071 634.529014 ] -/BS << -/W 0 ->> -/Dest (cb5-3) ->> -endobj -63 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 634.331993 99.933071 615.581993 ] -/BS << -/W 0 ->> -/Dest (cb5-4) ->> -endobj -64 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 615.384971 99.933071 596.634971 ] -/BS << -/W 0 ->> -/Dest (cb5-5) ->> -endobj -65 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 531.937950 101.433071 513.187950 ] -/BS << -/W 0 ->> -/Dest (cb6-1) ->> -endobj -66 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 512.990928 99.933071 494.240928 ] -/BS << -/W 0 ->> -/Dest (cb6-2) ->> -endobj -67 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 494.043907 99.933071 475.293907 ] -/BS << -/W 0 ->> -/Dest (cb6-3) ->> -endobj -68 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 456.149864 99.933071 437.399864 ] -/BS << -/W 0 ->> -/Dest (cb6-4) ->> -endobj -69 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 437.202842 99.933071 418.452842 ] -/BS << -/W 0 ->> -/Dest (cb6-5) ->> -endobj -70 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 399.308799 99.933071 380.558799 ] -/BS << -/W 0 ->> -/Dest (cb6-6) ->> -endobj -71 0 obj -<< -/Filter /FlateDecode -/Length 4732 ->> -stream -xœÍ\Y$Ém~¯_QÏ66Æ,Ø°-cm¬4€!zè©–`ôXùAß#ã`f±+»gWžÝÙ™î$ã`2ÈGe–»ZüùÆáŸBΔRs¢ëíóåÇ‹59Zü×Fì.û{üÍñê,e]M%^?3!d“kô9]_.øçp)‡¿\þëW—ÿÙÏqWojMÞÅë_?1‹‚É%gV˺ °b49û˜Vr·X«Æ‚1“§{V†TKÔXƒ¼/ -«@Œ@Ù…{V…>;ÒXÃÕ˜–³ÃykU±DÁ+<M‚Ò“ÆóΤR}Uy<—½Æ Á¤L.:WMJ)G%Vë4^„,ÑYY(”¬ñd )X•Y|ÉAãeÈ⡪ð -du•Yl²NãÕ`b-D*¯‚gK.÷4, ஸ-rw}´^ÃîzòIÊþX5îB¢ž@¬†ˆ÷àù a3i‰†¸ë*ŸâÓ@+ãJMDà®+8>•WËиâ·Ð²q)U "p*hX»€Ï¢a0Þ ¢z "p×DjÜu¾Z "pyCÔ°æ` ߌ³0]•W­¥hX»€%¯a¢°±H¢ò²!9 ÆÆ"Õ° wÞ«† ¸kc$à®%¤=*²„â5,@^ÜVñi$«Æz …ÆîZ‡ÖxÀ]ksÕ° w9ljX2|Ñ°dÁ^ÂTÈÔœ“†©Zðò©¼dp5,È„ÈRy…2iX€ÄÚÔP‹†ÙC–ཆÙC¤  #0Õ!ûVËÈsQë‚,È5,’™R‰4,È1¡ƒV5^ò¦dÀµÊ+๬aAÎdŠ†¨¥LÓkX€7œ.iX1Às¤aA®®¢aB‘)™¹ÆsI¶†ÈtÁsNÂâ! ²A Š/¨2“Õ°µ$T­ñÈrY4,(” §ŠÀMPi)UdLhX€óáb;jX€âÅ&Ê•YB$ Jñ\n J,Þz `›\of ª…,Èø4,¨6N“5,Œƒg­†ÕÔ›ž4,€_r½Y4,¨Á¢ÞD‰§øm ‰ëͬaA%È‘¢¨<È‚¤©*~ LB½ tÒx ²øJÔ”¸Þ, Í@½‰Œ^m ®7³Ú.îÆ -dÒxÀ]d¦IÃ\ ÞDf«ÕöÀ3ò3Y¸â,jÇÀz#b«Ú2°À^D«¬ö ,À—k$µi1QY–¤v VÁtVmXà/ò1ºÃ„­û$[T æ3K曬p Bv?kŽ5,¼2lëýõÏ—oŸ|¯þßËó¯Z§lüq0þ@‰ ëÈ.â+ŽpeÚr° -tèÝeß$„€_2‡…Èu 鋽éÆB"#ÜÖëüÅíE†Ä,³C2ÓîŒä¾¨Ž*$]ÖwÏùÙe °y€)Qnì«*äÙàSéÿw½ÿE2Œ}¿Qã üb?†ß®‹—”õS~uB%œ°$Œ -Õ3ÀÃk‘š[Ú þ͇ |½o¿µ‘ç¥ qÈ°DЭ·×Ÿ/ßþæßÿÝ~m?<_þøÎùëo~Æ°ý¤Ûöc©t^ì?Ó~Lôý§ÃÏ?]?üöuy<Ôà89-{yX÷ïO&#º <÷“‘µ¯{èòR>ÈÙ¯ëôOýú㺧É;Ü›÷È“Æ}þó‡½mnu•€‚”&xœ?1§·Mù"sú­hŠ‹¸‚L$#ùÓÍèÓA}NÅãÓuÈ—­Ë‘ãàn“7ن㬠µYuÊl6ŽöJÏ{I©@¼½v¸Œÿ~_+òpQtr¸o›òË9\.» -Í©‡ËÞ‰÷èàk¤ðOÜ9NFÕ–êAyðw‡TƒcDÝï€úVØ=>·Íùå*O€Ý×تm8ó T¶)ØÂÙìn¹?J›ïÀ>÷î5 çàOGÒ„Ç´sb¨VÉ¡žÕd>Eîñ£,õÅ+³Oõ\DåŠ,Ï)³Ç}Ïó,~êfÊÙ’E¯X €‹éÌJß4å—c¤9c6¢×1#¾¢2i*o‡öÝfñÕ4î›JþÐÕºòY(~ãœ_ŒêpbÛšUÕ;{®ZrH”3×ÈûÅÞä+[îgß´H$—¿¼¾v%c+·Œ•µÏ3ÅR´ûÂÿNG´&á0¨YEòy¤¾GºŠ:ÐS2œ°ÕÍø¿û·ý§ÿøÃ:ÆÇè·Q1 Ù)¶ß[Òú¼¡§þ÷ÆÒl¿·y‘¥ÂOžWúï}-þÙÆÈ}xÛ8¾FÒÜ?oýÎS_#Ž3Ù~·Ï#‰ÞËÆüFKƒ..ÂM2…ê -’s”m[Ç„À!"à»KúÛ»§üÿ¸[»n§á¼ú˜†eñ[M5cz÷”¯kL¡¤)îî<îŒ)ÈÃ¥~4®Ï ‹’ãÖgJö¸ÏÑ°œë(‘ûÝ„÷ãÈ$¼áëÖéî¡!Bû%ÁŸÂŒ´^›[–1²á4|îò°¡vk¿û%Oã?õùQ“I7Ö3´Œ¯¡e—¡í3ÖÎ3÷)]ƃ÷{?Ò?–²×—ÿÔ#A²¹$¿×º¹1Šn¼Þ¯KÝ)›n‹'"—rÎÍy7KüÝ¿\¾ýÇÚã:üÓç ?ƒmùY6Aáç+×~׿]~Ïîv×ÂÝÛøny+Š?Ô/BÏA¿ÔÏ/Î[úaºÑú`O4¢ö{fÛ CÜÎÙc®¶zØÑî²™iH?–ÙÂÌ>hÏ—÷0mëé¡ÓQ_â£Îéé|¨q¤æ¼‘ónKØ¥óoœóSZË>-?Çô «Ÿµõ¡É92Ù µD/K«Ç‹¨Ù¼g±NTŠ5­®{únî§ý®Ó[u"›ÉÇ&³ÒxÍÙ‡sý¼wfôãxùælúïËŒfðO"æ>+¾ýÔ}¦¯3±É}Zø9îD÷•BÙ‘çgýøƒÁ¢eTïŸóuS*‚ y÷GwWû~D„ß~=aÉõðþqéäÚhnÐ^1ç%” Ka¨Õk*;î|ÞÅûç|Ý£òærV*Ip^÷ÎN×س¬×“7„Ûõñ¸ÿ«AÝßJUn%zÇò#í2ÿ:öžö}„™ë¥rÔ.V¶¾T(â‡ý.¢†cvQEÑõ•Â,Ç<™rĵŽì)Ð(ãòëYÉÌ£È@FÆølWÆ>²Ë"öp]‘Í^CÕIïA —lë=LìQV`8èð iGù:ËVú”ÕЀ‰!û“€ qN3Èð r8êà˜¹ñºã¼š¿<-ÌjÌuÙÙ†xE?õN?¾ËpÈÒv%þMœñúýPyç³íõrz6#ÏvŸíô$«¬–` _ég33ÐCæ|ìw5˜ïXá­)+Ö¥Ð&*{“œçWcV<ð4ÞÙÑf廟9Ù&“c?´ž”;S¼ø®F›„wýA» m.äèÜÄI?-i¼Lo›u-|<ÔÙw5Šì6Ræ£ðCÕ™R3Ä’Lˆ!²–”nã{§|ånãVŒ²¸ûs¾¡‡Cs?igá22Þó›ǽÞW@gB™U³ S¯wYµáQø©®œ|UÖ{ÕH…M#íF6`q„ 7 ±.HŸð—ë·5z†7æÍ9Ïv¶àeK8À¤’šƒ<Ôc:Þ÷Û=Ý9Ë -[M¦›½kâÌñÇF˜€ü&ûmÝŸœ¯Þ7u}—ÞÇ™?+¼Û^¦–^”­¨ -}¯Up=Ô]9ê®íòýp+‘VÌtêvÐã—_¹šŸ÷ç/Cü áy¿æÎfªg÷iÇÜs4i…·)Ã(‡ò#›eÞo»¬¥Ý -Óa“3u“áý­ú9½‚ñk—Ô^glOÀ¿!(¼{Ê×.qëwF‡ m€>’Oî¸×û‚B¡r0¦0@Ì®Cm |þé翶73ï×}Õ‘åØg²¨F×T8…Ö^{ÊÌ%³³ G°8²¦=ÝáÕÌå½S¾ræÂo*wqwgtb¤§™qöÙsbpXWýŒTDÞ]΢¿Á¼‹ ôdw…ÖL[I‰ºIÇm›,çG\½PzOaQ}4•ì{ -‹r(,~È„ÌË`™{¼³¨š Ñaëø¾Ã~Y*ö=v¥ÝТøàl—³ÚÇöÁXcºåü$þ/gn6­ïûþv@ŽPwàåâäËÓ1O‡œ8âÝçZó±ô«;<¾w…ÚP€ß7öd¨f~“F}êDéÒøB}"¥;¢—ïŸÏ´ãÉBÕ£«Ô¾M[x#-¨ÂÜzŠçEGhtof—¢5SïŽàÇKjos¹öF¯MÅ8Ï_ùqMdès½òdÅ¢t*wMIîúÊÌöE)?ÎQü/HüÆAäWéÚ‘È‹>~ûTóó…ŸEJÃ'Û÷ØF.~\,òË…¿ç‚2Ñê.ž™Øµs-rzívq°¾z (¶ý×ÕÑ ª%iÍc?wð2Uhoàl+DÌLîžÌwÏíéÌßRLèWiŠÓ™_ ˜ ’»\OÉÜ_ÄxO´b»Eö4äqünéT]ZÄ)ºTÇ?Æú,ŽóñvÁ¶áH}Q©ü c©¡]ĉA B¨:©qŒœBóœÒí¢¿±ciÂaN¾85?¢N¯ÍquŒSH·‹:®¯çIå9ª¬©uh~‰²L=”©x…v]ÿÌ_~$|ɯç Éœ‚γ¶EØ¿#Þ.‘¦'.òË%fet\Xaã0=I bÝI.vÙË¢æ2T×4~KŽÿÝ "N*#OûÅÑ4)~Óu]­MØ7'y¯$IOuHÓ€£¼Æ -uñ3½áŽŽµ‹UÆ3èÎÑSc;ªP™ ç:Äcšà±'»iu“Ì6ô3±RíÛ-rœ¦ü²#/9ãtÕhçfÉ ssÝEN;_$9ÏwüÚò”w®+l~o-°Ñ‘̦_Æ&‹ÌÏÑó™› â  °È F_yxëD'þ*¦ñër2-âÞ.¹´0çCô\Ȥàø[8dÆä®Øßâï_èúÝxís‹ãÞN´ð[wdÉø;›ÊÉüNƒ8ÝF= Ø_dWÖØ8¬aÆq~W.h<Ý.`#ÝDP7¹’—ÙOÓõ.ÍÄÄÛA‹3‘ù©g³övÆÍñ{üÂð -òvpY/¬|K⊇®ÎOÜz1rQ\Š ë’l'&¤Üdªžå'y÷"àË°x´Â¬\zÐÚ]ˆ!Hƒ¼ ““›Ö<ŠÐ_ì^?ùÜ»ÈixS'´OꋤÎcê<ÚeÚ_Θ®|1œwih®„ï…™ä,×X©í.¤ -jYÓú€(7¥AÛm:Å„ ~“mÐæŠyÈ”ú´¡–vÇEÕRž9Å¢þEj:ÊS™JŒòTÒÝX^wD¼™-…™EiÎyÝ]#g.YZ’³'•ÖÈ0h÷{Ó~ïm¯â6h ’Vfj×´x7ò‹ðïK|G}2V5þ´®†+단ÉåàCûîK[R†}Q¦"Úïžó¸ïqßÝ}¹ŽÃP6󓯈ðF=gˆ¥Æ7œêÓªUoé} -PÚ: æîŸcn_cמ&mÍy™¹Täo­ œôþ²­lÞ€¿íZ'íÏÿNsTN -endstream -endobj -72 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 71 0 R -/Resources 4 0 R -/Annots [ 73 0 R 74 0 R 75 0 R 76 0 R 77 0 R 78 0 R 79 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -73 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 797.673057 101.433071 778.923057 ] -/BS << -/W 0 ->> -/Dest (cb7-1) ->> -endobj -74 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 778.726036 99.933071 759.976036 ] -/BS << -/W 0 ->> -/Dest (cb7-2) ->> -endobj -75 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 759.779014 99.933071 741.029014 ] -/BS << -/W 0 ->> -/Dest (cb7-3) ->> -endobj -76 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 740.831993 99.933071 722.081993 ] -/BS << -/W 0 ->> -/Dest (cb7-4) ->> -endobj -77 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 721.884971 99.933071 703.134971 ] -/BS << -/W 0 ->> -/Dest (cb7-5) ->> -endobj -78 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 702.937950 99.933071 684.187950 ] -/BS << -/W 0 ->> -/Dest (cb7-6) ->> -endobj -79 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 155.548661 618.187950 248.939042 603.187950 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (https://kubernetes.io/docs/tasks/debug-application-cluster/audit/) ->> ->> -endobj -80 0 obj -<< -/Filter /FlateDecode -/Length 4663 ->> -stream -xœÅ\IÇ ¾¿_ñÎÜ®…µF$œÀˆ€ rž%Á(€œCþ~>V×Âîæ[F#[¤™&ka±ÈdõbÏ¿¶ø/“]r.)Òùòáôñd– þÔ›‹ÊþÿR8[Ci ¶ÄΘàÓ’Jp)žŸNøow)›?þþÕé?Û>öì–R¢³áüË;f‘_RNÉ…UÀ2Ö+¬–”\ˆ -+ÚK,EcAŒÈÑ‘• •4Ä ç²ÂÊÃS²þÈ*Ã%K bØ’²rXgŒÊƒ †È;…gáô¨ñœ]b.®¨¼žMNãy¿ÄD6XW–cÈ K(Æj¼Y‚%2²ÏIãEÈâ£7*²¸œ¼ÆKÅA;EáeÈëÊ*²˜h¬Æ+~ %©¼žÉ)y΄%dïƳv °M¯òàQؤ˜žƒ,Ñ„¢ò KðÆi<Y5p"ø”³â­Ž½Ûo5^€,Î%Ry›¯!€‹Å¤¨Ar½™4,(YR•Y4ÅoI¨7N/BWHÂ#×›Yä¨7‘Ñ«G™ëͤwC2i<à.2Ó¨a.Po"³Õj{à ù™ÎÌ\qfõÄÀzb«zd`€½ˆVI=30_®‘ÔCˆ‰Ê2GõÔaLkÔcüE>FLXOŸäU„9$ÄÌœx‘nAÈîGÍ1›ù+ÍÖó±_~:}óÁ÷üÓO￪'eý¯…ñ{Š\XäeäÕx¤Lf©ãÑ™cÄIOUBø)}XˆT¦°ØÝXH„b„Ûr¿Ø­È˜e¶HfêÊh3/J‹½òQ•õÙ}>‡¬Gµûú|YÏ6ãYh Ž3ç×Nß|ÿ§?þþ/ÿÀ˜¯ßŸþù;>G00ðÅÏ ~¢Š v¥Ñ{ü˯þu~ýÝfìÌv€\¶cópHéŒ{‡!2ºòÐøB2·iøg\Ûßø‡êî«ÈÚ½Žª%Ä67Ï÷fú_×?Ö‘æZ‡Oºµ­µ7ç~7·¿¢7¹ÎîÛC×7dy– V6žv[([ŒÄ’í§ž¯ÏÕ=ê2iµýûÝX7íà¦þÒ^/vµæ^W!\øÉ'Š5ÑÔåh[¦Ÿlö…Ä\R‡B'u’Ðe³Á*—ð±j—aöÙË°ÚÅMý•ƒþ49_ ÓÛöïøÐyÿüÆ~EÌâ{@äGš€äû¡ õì._6jYJCÜ^>cØr|çµ5RÝ¿JÜzŽ3õ¼©Ïó¦'£ƒEŠ·×«‚Nÿ½ä -v7 UÚÎ!ZÀÚ8í•„kj/Ò]—ù]Çê±]Û6ÈŽõwÝ”íüµ]{îÿ¶ñ{!ïeÚÕ¥ƒÎnÖ•àÊ8IBÌ¿ ¿^ŒêHèO&#=P™icU& ÐE’T÷ªé~“ûÛu=Ç@,€Œ<$çóÙ!=k¨·_ -*á5${~Ÿ/ ePJ—w»KG(kðÕÍ·bïþ†+¹nä‡ É8" ˆuÒT‰µ²¦¬GžÛå Ç!.߸½¡üª¬{1Ãz76s;ž3~ïð~&©r3J½ Ì0ÍòS:PWðêf<·ËÞ ¾EÜĽ³-˜×@^­ûn0G~c)ºhÝ5#Ý”7H›¬Ïd¤†o<·Ë>àg+š¸w6å“3-~b ‰ÙœÏRæö—Eûž!µ©e{ÉEsž;Ñ6Å}©z/zŽÓ°Ã¸Îß$çXŽã²¼þ2שEñ:gÏ`úzß·½¹¥óÎøe¦p ¹œ˜÷û4湨:‘º.{¡îÀwjä±À.óÒŽ -d^û¶,—diÚ× ŽF)|™ÙÕÈš¨eN=[v†š…{ÎK¡ ôå§ôb)]¶ÎÀ÷ŠÚÏpKÓÁX¾»"4íúŽE³©?®ÖÂòªå>àiC‹J^îÅáÌþ°­òý\¯ÌÑ7ãJÔêyiÝ•›ºLmuñ²³íãxUþ ¼ÂšCÞ믾¶½àíu_MÁ§µÎ6wcåÙ°þN~gWÖC¢µÏ³;wØ!Ñç¬e¤pf›ÂíáöªV¶ùUmL>“Óë”Ó5>ÂK4Ç´xŸº+G÷Ëû\öz#Õ"®YÃôMŠfs„²yFéÆËHk•Pñèž=tËý"ú5YäcÇòãæc|v¯·»ÏXyåhŽÃg.@ûãxK=ËóóÛ 9S@ø>Î?ÒQ¹Y–¾¯<©¥]¡ß¼!|ØÚ®¤;ÜÆ××"÷ÆöF;¬ïÇ9e®q”åFðÓÄãõ)¿»ÙÎK‰>šröùh úF^jÎ+ï‘>»Ïo“BPŸuU㸴> y¿y5…¼¬í¨_ÖŸ¾šGõö3nÛ(à~M ‡å[~1-ïbì¯îôö†_§‚@Jï5ßIÚ%+BÚ[ÒKz¿^y ¹íg×VÚþD­ÓBn ¯û1Û\VÈe©kí`–Àç¢Ç†[ökëcr÷Ìò±>_Ú,§¼…E ®¯šå»+›’îš'öƦàÝn’‡LÍò\Öäb•Þ=êÛ=²ž¸3é8}‹h/¡¤€ù…uK)[ùNËØ×»MîlcÏ㯓œíîåmª!ùë3üT4R¿°uí}¡§¢ÇàÞÌ» àÊy$+›žö=H•]ñÂmöIÅ[3ïf×sWû-øxŠõ¥A[_70ëøË2çH ç1©œùEÅl<Ù®r[•dÏWzÖïñ|­Ú'z ¿µÏol®ßï­ýǺYP) °êµAýœ†©äìúÅ$?øs*¤‰fq‡ÌÄö«c‘õJ§]2(G÷-_Lûu5~”°.ôÕpÿлñkndÆC#õE¯ub…ˆžÑɼz³ØFæOÒd~l¬^Å! -×ùƒ8$Ùv8;Þ%[o1‰FL7ÉŽº<üðWÿµŒ=qˆ.UgÅö÷¶.‰í¼E¼œ0­ßSŸT*£Õµ»>·Ê'„*ƒz˱!4öI!]N -ñkÓ‡&læà‹]sý1ì^3$íJo§.'µ]-ŒJ£Už]K×üeš:?=zv…âol 1¾ÿº%óaÜØk“…ݸñr -4É$;3¢ÝœL§`…Iž0úTÉÝ[:ñ¿ú¯ÓÉ´ˆ{9ý|ªaÎùà8 ŽÞòÇ^dÆdÏÿC›ïðïß':ßß.^ã¸3-ÜÀÖ YF2þ4X~ÌŸ -ëÄá6‚èHÀþ$Û<Û†n #ŽsY–v4în&°÷–v ¨\I S‡‚ì†éò‹2S3áñ²ÓâHd^º73`¯—¾/Ž+L~/½K… ¹\æÉó3$ŸÅ¸â¦3¤ƒKNtÄ…—bÂ2%ÛˆÉ)'ªgùIn…ç¹Ä…H#øÒOÍ0+‡î´º -Ñ)°—r`²rÒ²‘Gú“ÝëÅûÞDŽÝ˜: }PŸ$u,Œ©ðtª¯—q{9b¿X’†óN ±¼ð=?’œé3µ Í…´QAͳ[kä¤Ôi›I{ pšé´1bê2ÅÖ­«¥®8«ZJ#§˜ÔŸ¥¦ƒÜ•¡Ä w%Úò¸=âlÉ(HsNsuf¶¹dVhQöTš-}§ç¦íÜ=ÚžÅ2h -gfjf·phùIø÷)¾£~ÌAkÕÿÖS ›ç÷8¨É&ï|ýĪɑ_¨ þJV¢,Ž9žÝçö¹Çñt£Ÿ{È™° yo’ååê"> -endobj -82 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 101.433071 125.127057 101.433071 106.377057 ] -/BS << -/W 0 ->> -/Dest (cb9-1) ->> -endobj -83 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 106.180036 99.933071 87.430036 ] -/BS << -/W 0 ->> -/Dest (cb9-2) ->> -endobj -84 0 obj -<< -/Filter /FlateDecode -/Length 4874 ->> -stream -xœÍ]ßä8n~¯¿¢žX(Q¿€Ã> ¹`,²À!ä¡·oæAo€½<äßÏGWI–mº¬î©Ùx=]&E™©O¤$«èjñóá¿Ìdr.)òõõ—˯kR°ø7•X\Lìñ›Â•,'¨Ä®¿Á'“Jp)^ß.øouÙ»üùw—ÿ^ÊÐÕ™R¢£pýûga±7)§dƒÂ*`Yò -+“’ QaE2xÄR4Ô‰oY jpÉAcA v.+¬ 5<'ò[V.k,¨A%$…Ez³VåAËì£`"=jõSTî0fæ$YÐ-Ñ}Ë9æb~§Øm~ìï½üþƒïõ¯ÿsùò»i¦¬þœßs”Ä:Hqæ\™©:†¡ÉLDž4„‚‘%R™•$Àâ}ÒM”ÄPŒá¶\ÛZª EgB03=÷÷E¦@0•º®ï–y–®¾ÞÝé‚7ŒZ á¶èl)ö4Ùiú>‰gè‰ûH$QÊ'ȉ×ò¢ðº Ýo›Ÿm—ÚËMëìõ§_.¿ÿã¿üûÿö#nûӗ˧?X+Ó"œ¿ÿŽäsºýe¾ÿ]]/øÿyýéOû÷Ç@n`K,ËûË-é/ßOÂÿøÓ²}‘3b1<%~‘K¦il{h’1‘Ùä[˜é%".D&ª)Ùã÷ûÇ K£·ýeQÛdKZÙ«Úö JiEËh»©²Új_C¿-’;m…åè ôw½ªŸùûõ—•¿Åe9v(ûrÿüºòÁŸoü&s¯+Ô{ÄUùJ]Ý{í…ÏŽt'@!\³¥”iFÈ–Õ}ÒF°ÕH×F¥€š#¦í}öJü2É ; Pc"ç1!²=Œ–²ìüL€ZV{PNæd³Ì)ŒÔ ÌyZÖvd24Ÿ \DFëSÒÍ^.y>B¨A™ó4{¢†³D-ª{˜ôA²_ÁXˆÀ–›¦‘QT¹÷aö $D\$Ezr±²JòÊåî®Ev•²Öreõôk`Nõ 7Η“Ì—É\¨¬vǘdæó±ï ‰œÆõd'B&à’Þã7Α;çÚåMe@c“£“…?­)Ò-âxÔâƒ2çir‡A!;Œ OéíËê>}Ôÿñ×ûefUëñë”<.ëÙ@z…ú×z¿,Ë·újùÔõm»?U°NëýǦ>dcB($ÍÛ¶¯:ÄzŽ*¬b”õéjE"º•«ÎÑO[ Œ¦¡Tœ¦û¡cÊ>2‚ñ²S¤“Y¹'·îtÏ÷¶ðögñ<ú½Ï²Nƒ„e]sZÜ…ÆdÎÓ…’l>MŸ4Ö,ªë“߆íqå8w¾‹ïŸžuÜ•íœËåÎiêX³cVéÅãíw˜PÌ´Í:+Ϲ°älb°¡/ykX@àh*xPæ4NÄVvXÙùüÌô{Yí'Åy&§Y÷öÔ9ÏpZ²½×{Ò’­ôásvSÍŠôƒÉ›‚kY¾Ñ @àà±C ʜǡ`òˆšÂsjQí§QV‘aŠÖå1JôNµ¹ß»œj#ý.§Ú>kÑ‘7lòøËyÙe­ì²l¼“I±#—“9Ë Ü%„ ¯€ØåÕöfÜ N±Þ<ó»­óØ­ˆäï4éc·r¶L1gMúVEÔpHæÿ‡gLæ<Ž#Û©Bvü«Æ#¨euŸúìy5‘*ƒß:”m}¼fÎ/‡¦Ñjs罘†³¼F ožé”9YƒÅéðéÑòöûñ`Yí"¦ù¹›˜XÅ2SÞ¸†íõ$ç@¼#o¶1sŠ{Ʋý9ò´U™“¼@Iö(Þ”9±å5ŠŒÜmd{Ôèó†™eõú4Ýém¦zðnâ”/ÇFOÓZGæõc>4:£]ÌÎ^s4Åûp8Ë:&r“¡FïGú÷;MßÕËÛ-!Êk2ò"!/¶‰œ§Ùå g[äEѧÂê¢ÚEdŸV=i ‚ïÌ·©w·Çy­Ý9‚¼,™Û4½ÄðØvc2§1^´òVÎô÷·€Éeõ‹…Š×ï—©ÊAšV$6Ëš~UÏÏÛídã¨w,;Já ŠÎÇó®hPB6­"}Ø^~:YåM¥½>œâEŠèK\lßé<$rçõ²3›™Fçλ¨¾_¥8>\lkÙÃDWÆ!'ƒÊA…“‹Gë;ƒ2ç1YÛ‰hgÛƒ²Ûd³û¤FÒ«¥ÉÍ®”ºEàI»Rb”=pÖ¡)dù¹D$»ñ1pPæ˱Û"H(‘SѤ§‹÷§žÚKwÍ(/éË¿;/œG9ºƒäÈgå£.¹ë˜ÌyÜ5Ëyòé¹›–Õ¶ƒÖ[FVâ–ê^/Ê°tñ~áW¯‡÷xºÑ~÷LØ°@/9¥et=|Pæ4.R¬ì ‡#›%Òeõýºx3åë!ªt»\¶õï‘™w¹l¥“U‡‹±k)T Å4×6º†>(sñ²ó9¦øÜ Ëj?‚!ë·KÖÁõÈƉy }«Ï*†¼+…Ãø«Jƒ2ç1zÑ êî Âƒ«nqø›Þ{iiýB’~ƒiã"ÇK‡xa:XΧCœ“9ËdÙ‹,ç}~õªÁæ„U³¯{ºß{Ola?~ÊŽÞnJHŸ®NÞ•—®Žv ÊœÆèÓÑf”08מּŽX}g3ÀîÛõmÁoÓq¼ŸÅ†ÌÃNç6c”ÎÔíÖ wXäÀNSê'•]iu6˜Ø#Þ 1XùdYÓŽ…$çJóÿðÏÿôÿúsó‹Ã»Ï·g(ÜšÄMû¥)¤é¥É¤¤Iî_îå^§oôZnjÒ—{Ùtž§²²ð*Ø{/#G7Ê‚ê£r/º{Å{Æj;]>oLðë%NggÒt~²SðV²wkä]Ñ„4,˹›ž©69MD×Éék)~m¥ä¬^- t“Iú‹{ù_'cýr‘¯ä¥ÜN•·)EX“ß.ò­¬ ¸A²ïm«K–ûW´W ’çuIŒ½,­rêGêÓˆ|¨br†ìú«Wâ¿·+DHFÚ’åéwßÉòÍ ÙøûUlª„Xuq®û²s…³à!YÞ¯š‰aK´Ýíf²U˺ö±4›ƒØT:óײ.uæ|D|½à¶~M}S©r|­ª¶.ò›JóR¥QC-Ù ÂÍN -éõ¢¿³µjÄ°ÔøÕäTÝ1WGâV®Ôr -éõ¢–»×š¥R+•gÑR[~VevuŸ[Ã+´©îÔ$_5ÓõëëÇ%9Šª­‘MÎ~ã6Ä׋lÝß.!)¥ÃŒ6T×뉾«·‘³ýe¦¦\› -]³ÒäLRùÿ¡Qy¦ÄÍ¥ä Åùj¾‰ôÍF^6RO uXkÉ{iCEÝ)5ŸíÛ‹²UÊ è¶Ò­ÅÔ®É:z*U=©K25¯kd°Ú> +{¢­Jàv394W~[g=CëªÁ¶›EêÜ=´zgrlØùÖ“S³8É!ÑMßVoçó•ø: l¼&‹ëçz“™,ûÚªû´›uÄY1€ÂLžaôm"×ÞÚÐI†ëǹ“i#îëåo—i˜s>8‰s£'ù΃>b¢ëÿ¢ÌŸðû_¾þPÙ½ãÎ6´p [ä~$“oÈɃdùÆœJlݦ#:î`&SžË†ê mw®ÃkžÅí ìµ$5¥Æí‰anÃŽìšë:ò]Ë„Ä×U+¶@ækm3Ø·K_N^£”㙫V¸è—yæùyH¾vUâJŠÎC:¸ì:A\ô•â²»a™5[¨)Ùߤ5½èϽ)dú¼¿è¹ô3ça¶¯ºÒ¦§èŠ öýE_1SÓ²ÐGQúÃÝë«í~W9Ö ÔíúÖSÛƒ µžNõÓe\^¶1ÇOoéÖÎ;·P«Ëw}Ï· gîshÁ‹‹ÞG;jžÅîBS®´ÅMë@Ñà4[i­ÆTuŠw±Ú,Ógµ•R‹)fêßú–½UZ#†Þ*qSVê­#^‹–| €BïÎi~:;—l±dVh±—nTžKúJÛÞ›—÷®£íµ{ ž‰sdjg±°)ù!üûH߹ɬf´RõgšÕ <Kš)yç§o´9F`Ë—Å$ÎÝ4Ç»eÏ{lg7ê¼G'˜Ã°÷6‘|g¤®Âå˜ÑUU¿OR“÷v”Gz_(Ó:3ÌÙͬÎô¥aøwŸÜi—IREùŽ@/Ao–¯6J–Ôižiª…Ó'ÓÏÿäGS" -endstream -endobj -85 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 84 0 R -/Resources 4 0 R -/Annots [ 86 0 R 87 0 R 88 0 R 89 0 R 90 0 R 91 0 R 92 0 R 93 0 R 94 0 R 95 0 R 96 0 R 97 0 R 98 0 R 99 0 R 100 0 R 101 0 R 102 0 R 103 0 R 104 0 R 105 0 R 106 0 R 107 0 R 108 0 R 109 0 R 110 0 R 111 0 R 112 0 R 113 0 R 114 0 R 115 0 R 116 0 R 117 0 R 118 0 R 119 0 R 120 0 R 121 0 R 122 0 R 123 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -86 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-3) ->> -endobj -87 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-4) ->> -endobj -88 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-5) ->> -endobj -89 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-6) ->> -endobj -90 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-7) ->> -endobj -91 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-8) ->> -endobj -92 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-9) ->> -endobj -93 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-10) ->> -endobj -94 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-11) ->> -endobj -95 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-12) ->> -endobj -96 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-13) ->> -endobj -97 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-14) ->> -endobj -98 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-15) ->> -endobj -99 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-16) ->> -endobj -100 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-17) ->> -endobj -101 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-18) ->> -endobj -102 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-19) ->> -endobj -103 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-20) ->> -endobj -104 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-21) ->> -endobj -105 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-22) ->> -endobj -106 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-23) ->> -endobj -107 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-24) ->> -endobj -108 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-25) ->> -endobj -109 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-26) ->> -endobj -110 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-27) ->> -endobj -111 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-28) ->> -endobj -112 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-29) ->> -endobj -113 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-30) ->> -endobj -114 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-31) ->> -endobj -115 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-32) ->> -endobj -116 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-33) ->> -endobj -117 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-34) ->> -endobj -118 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-35) ->> -endobj -119 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-36) ->> -endobj -120 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-37) ->> -endobj -121 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-38) ->> -endobj -122 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-39) ->> -endobj -123 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-40) ->> -endobj -124 0 obj -<< -/Filter /FlateDecode -/Length 4375 ->> -stream -xœÕ][ì¶ ~Ÿ_1Ï¢J"© Pä¡h^APAN¶9 ŠM¤ýûýèɲ‡³ã™žº»ØÝ1)Ú´(~"iYŽß_ü*\)5'>¾üpøñà]¯©Åâ`bŸ,Çà9; 59þ Ê.W‰9_øµ:›¿þü³Ã?—2á]­)9þô²˜\.9{1X,È`‰¸œ£$ƒ•‚Ã-Öj± †dŽ|ÉÊPƒk‹58Æb° -Ô Î.YjÄØbAP%¬à¡GˆÞ›<(â™)¼ .¡Ó“Å‹Á¥Rc5y¼£Å#r)s`ñªK)I±x ]¤ú`ñºH`öº0•lñt¡DÞäA—X2Y¼ ]"z§¼]0ºŠÉƒ.>ù`ñ*9©…ÙäUð|Éå’½8)DÞâ…àc“L< -FJÙàEè’¼T“]„|´x]ß8!”K1¼5ªw“§`ñºÄ˜ÙäAßB€˜ ‹ÏÉ‚xS\²0 æì¸F¶@ r\¸X(Kuœ3Y0«8N5[8@>€ƒä¡‹p²€tax®Éƒ.TÙŠР,, ‚.‘ÉÂÚ2F‹…¸e_ƒ…¼e’…À…šÞÂâRIlar1”ª…Ì%`§…Ð…»g ¨K’¢…Ø%.ÉÂîoaw‰ iò  °…@GG¡… ܥ೅ Ü…E `U‡q˜,,ÀXq±oawcñlawcÆ`ò²ƒ’… Ü‚[°fnàn-,`ànä˜,,À@qÀoFÀ]@ˆØátÀZX€ù¼Hàd.",±°@€»¡bú3|håB©ÉÂî†ó™¼êBF~‹^v!¥jaw1Q‘…Ü| €ñ3j´°@€»0S[<ànˆÕ[X À]Ä ba†ƒ ,,Àüæ‚ÇÐ5yÕùZŠ…Ü,E 0 ;_€$&/; ¤`aÀØyÄ¡$à.¦÷jaAîz!²° w=#ì1yÐ…J´°q xp[ç¬:¸ë,lñ€»ÞçjaAîê´iaAÊJˆÅÂ(èG RaWsN¤êÁCÈgò’ñXX=È2yÐ…3[X€ÀÚUªÅ‚¡ ÅhaAŽÐ!ƒ…ˆ\ ˆ¾Íôº`¸˜ùCDŠÉ\©ÌdIÈÇЫ/EW2àÚäðB¶° gv˜Šƒ…È¥\Ó[X€w.YX5À laA®Ð®ba¦"W""s‹  ‚l é‚‚…%BDƒ”Xe&oafhä’èj‹Ç^óÌbaAáä4T´°n‚LHiò  `ÂÂØG“m±°É’M¤9&º°…¥DM7‹…¥@—装›šof ª‡.ˆø,,¨>9 “-,Œƒç½…5䛑-,€_j¾Y,,¨ä‘o"Å3ü¶RÒ|3[XPºB“]4UÃoIÈ7N/A—XÙ‚š’æ›Å„È7Ñ›%¢ùf6ËÀ]©@&‹ÜEdš,,ÀòMD¶Vnï¼’ŸÙÌ¢g1+Ð+˜[Í’öb¶ÊfÍÀ|5G2‹P™eIfÕÓ*˜Á›eüE<Ƙpª>%ª„á1g–¬7YáŒè¾çs3ºÒìTûéÄä{üû¿Ÿ~6UÊÚwÀà'NšX‹ºH¬07àÊM§c:¸‰È“†PðU"×YÉX<ÝTILŘnë±K•¡±êÌLwÆãu‘)˜Š’­ë»ež¥+µëÂý> xCÖÍÈ\ò„¤9öTÀT}§È34Å…4HhE5*vÂ!õz^4þå‡Üè|ùS…¶zc6gtnôÇ?~þËßÿé«?~Ë~øtøæhªs½ýÃùË/ÚŠœþ2Ÿÿ¾œéáô‰œçræÕ3/®dÏü˜‡¶Ÿ–<ÉËãð·³²:g´å›~“^=~øÝá×–¶DöÏüСñ8gb¾aþm"w™ÿ¿auä²ïÝoõµ5ÒùïùÙ÷U <ßb€J¾Ö:Õ=R\ñ7L¶Qf76CÖ€Ä î7Ú§e‡òÊí_R3Ô—vw“Žqšrœ -´B´v«»·É짻‘WE¦èï÷‘Þíß­ìeé­»e¤+åWÌ4=–„ÝXeöcŽ¬%|©>Þm­ò÷‘ßàê[£›Û$”hj¼º4 ?=‹}ÓeöcŽ`§XDþÿ<óÓã5I]·ròôÛ(³ &‹*"÷?­ùl—p˜ØèQ6ÊìÇúH]JäÏçq¤ ÷Ó“MUtÕàX·×$7ÊìÆTâæŸîx2šJ¾=›ch“éVO­:€ ­g{›èbµÄÓ²§õúœo˜p£Ì~L¨‹^ âåûÈ|ŽÊ‰0n;&$§Ûl”Ù)D©‰èoZà¼zªO+3t“ÉÊ<«ðƒò`ª1äX§óOÎýt©³¾ ô-]õåS”[fß&³³ë{j¾êë>w›}]å_U]:x®Ìó¼¥RõEăß'n”Ù¡’×uÔÓ;wwŠ‡z /}à"9n>³šÕÖ¾3µ3’½ÀS¯n}°Qf?F!]"Æð¹2ú?]«!D¡¥g]T:úÈ;œ‰ê-(ß(³ŸaCºþ-åtåQ’éï·†¾é9JX~êlß¼ôº|eç«îùbÈß¼zôÑåêkª†tl(XÅUÝ’u×C>Öʲ‡6Êìg@‰N~Pw Ý9Àbô.è>5eu¹ Ÿh‰}ÚX›xÄ¡×CB©¯ãøõíÆ~CzÚ_@Ý]JOÝz¹r«†Ë8ጶ¡-_®×&ºª%Ö˜ˆ§%FÓo°m2û`E×êÎUŸg€-.÷Ö[?ɹ¨¬ù·c¢õÄÙh—[Ÿ¿ èÕ³ÜuØ< §·ï—õ~1gɺ{7 p]ùˆ V·=¹”¾Ù×Lº,æLKz¶/’vÛª³ÉöÊüF™Ý¸À´eIÈ\øýÏ2µáͳábNûú¼ -Ò??í8—0n¯Ò;ùf“}:u‚¹[È1¬vçоKçNÓIÙéFÈT‚é3EÝ;H»ê«ßþæWøËÜUšˆÄïðSp‹rꖨѬ>×sZb«]£ÝqþûñÜî£È½µ;%Cç¶ùOma’ø¢Ñ°ï)ä…ÀtÅós®•Îù®tùîÂ?Ò´U˜ö0ôi -mkÑ-çœV@ïêžYŇÖåaê¤p¼"9m ýco¥¿Aòº¤>ºšL2œÛÿ8뇃oz:bO;»ú‰¬KÜÖä׃îìËAOÜHVâù£ïçÒß+Ú <>»ºnIâüùcí''i£´»Qyibºã’.kG:~O6ˆLá’¬wØ=“uwäâè|”º*’šÎ Î'ä±í|ÂYð&YÓ“™(—D?\n&ë.­mìk·9ˆ]õ±ëÂ`þÖ6æÁœo_¸,­©¯&U÷dm§j½‹é­ÑhPªvª´–Ý Üíd^ñ ßNX=tþ`5ÝÙîD,m qoW[;ƒôr0ÛÏ&ÝR¹·*³hm=?«2u*½ã ÚÔGgjÖíÞñÔ>.ÉI-Ðl\{7ñ‚ørЕŒä׃d£µÌXᥠ½‘HÃy;¹øy¼ÌÔ\ZWÁ5M÷Óß'¨NUä™>îCJˆÎGóEÔ7;yÙI#=uÔa«ç¹íÐ]Jû˜û+o´WÐí­{-¨C— ô\›z -R<–äÐG]'+„µþéX9}S—›ÉÒ‡òë‚<ë)ÝUÅ÷‹¥0 wéçÉ©cçëHÎÝâA7jìúöóc¾_¦‰×dú¥]d&k ¢ Ÿ~±8+P˜É3Œ¾Näæ­t¡]û8;™5ã¾¾?LÓ\$‰“&„ç±Çˆ)ÿ6¿ÃÏ?|üªmtwšÇ£ïh;¶.ÈãL¦»Ô—dݵ¾»Û ÄÈìÏäPæ¶ÒFCŸÇcðšgq?{k:‚†Î‰2÷á@Ž}èÆ@CÏÈÄ—U/ö@æQÛÌöéÚÍá2¤[$6­p0ÞËÌ£yJ>§Ä‘6§tÝ4%‚8OŠÃá‚uÖl¡¦Èñ"½ëUMAz­á`#ôfÏÓìxêF›îbh‚˜ÆƒñÄÆ‹Ö…>†Òw»×Ãv?«œš(µC{§¾ŽÔ~cJí€gSi:LËÃ>çдKsÞ¹‡ú¹hð=êAÎìshÁ‹ƒqŒÔ2‹ÈxQn´ÅEÛDÑá4ßhýŒ¹é”Îb­[¦;.f/åSÌÔïÇž–Ñ*½e´Jºh«çm3^–¨@2ç<ߟ[öX²´4Jw*Ï-©Ñ.¯ÍËk·Ùö8ÜÏŠ¤92õ³˜\´¼ ÿîñ“̪‚`µjßSU#”ykXLÔ2EšþÛ/)CX7lÏ\†2Ç»eÞ®{\V7ZÝc¼Ìá˜Èç ÿ·ÉVáŒz›1œªýO'[LßÄ0né}`”uf˜óUéwàë\Üé‡YSEý?=¤AoÑ/}0Ë> -endobj -126 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-41) ->> -endobj -127 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-42) ->> -endobj -128 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-43) ->> -endobj -129 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-44) ->> -endobj -130 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-45) ->> -endobj -131 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-46) ->> -endobj -132 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-47) ->> -endobj -133 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-48) ->> -endobj -134 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-49) ->> -endobj -135 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-50) ->> -endobj -136 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-51) ->> -endobj -137 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-52) ->> -endobj -138 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-53) ->> -endobj -139 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-54) ->> -endobj -140 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-55) ->> -endobj -141 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-56) ->> -endobj -142 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-57) ->> -endobj -143 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-58) ->> -endobj -144 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-59) ->> -endobj -145 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-60) ->> -endobj -146 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-61) ->> -endobj -147 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-62) ->> -endobj -148 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-63) ->> -endobj -149 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-64) ->> -endobj -150 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-65) ->> -endobj -151 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-66) ->> -endobj -152 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-67) ->> -endobj -153 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-68) ->> -endobj -154 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-69) ->> -endobj -155 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-70) ->> -endobj -156 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-71) ->> -endobj -157 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-72) ->> -endobj -158 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-73) ->> -endobj -159 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-74) ->> -endobj -160 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-75) ->> -endobj -161 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-76) ->> -endobj -162 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-77) ->> -endobj -163 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-78) ->> -endobj -164 0 obj -<< -/Filter /FlateDecode -/Length 4434 ->> -stream -xœå]ßä¶ ~Ÿ¿bž D‘DêPä!h6EZ= (}Ølï›Iúï÷£Ç’e½£Ý-\ô{3&%™ÅO$%kÝÙâç ‡ÿ2;“sI‘Ï?~>Y“‚Å¿©Äêbb‡ßÎÎr2Á•˜Ãù'!P2©Ÿâùé„ÿ6—}ñ§Ó_~uú纎;{SJô.œù$,&“rJ6(¬–u¤°B0)ùVtXŠÆ‚!±çkV‚\rÐXƒ½Ï -+C âäèšU †OŽ5Äp%$…å,äpÞZ•A,3y…炉èô¨ñ¼31_T^Ï%¯ñˆLLì‚ÓxÅÄCÖx YB±NãȳÕx…)'! E²*²øœHã%ÈâÑ;EáeÈ‚Ñ•Ud±Ñ:WÈ„’™U^Ï支yÞ2‘ÕxΙ€±I*%Ťð%Ãų>“áÌYCŸ‹á”Hƒ_‚áX’†dxÞi@@²Žƒ, ËUy… -kX@² €†DÅ3iX@@[ÆhÑ°€·l‹Ó°€€·l]Ô°€¸ÓjX@@\Ê‘5, @.†RÑ°€€¹ìÔ°€º0÷¤au)D¯av‰sÔ°€€»ÄÎjXÀÀ]"ÔTyXà£!—IÃî’³IÃîÂŒ¼†ЪÁ8Œ`¬Ÿ³Õ°€»>[Ö°€»>a -PyÉÀIÃîú€GÐfnà®ÖkXÀÀ]Ï>jX€b€?Võ€»€ »«aæ{ð`ò=ÿð¯Óç_M™²úã0ø‰£ÖALĨpe¦æŠvf"ò$!¼§Ž‘Ê"¤,ÎI7S1¦Ûrn_ÜZdH,2;83Ó“q_D -ª¢¨Ëúâ:o%+ÕûÂü>Àyƒ•ÂØ0—cðg„ØSþ²“ôe5ÞBNÜG<ˆRT¼ 'Ì>¯åUá¯?œ`Dóí/ùÙvÙ‰¹œÑµÞž?ütúòë?üùÛ?}‡Û~ø|úøk•™ŽF~¿úÛùÃ7§ß~Xw -¢h†Ñ<ú'ËS»ý8V宎|þCL7 îÄÝý'©'毾p(Ïqþü¿—ïÁÍ´|ùôŸ»òaÍãÏ—OŠ:½]ÏüæO?Îí¹¿ëú„±¥”³—,ü'÷g:Xç0…oŽðÝî^ikÕÖ®îUì1¼Ã÷3ŸÖ*¨ª -qýÙ†…_ÊßU<AÏ"I[ùšˆù–êÆêGuˆ„0Ƽ½ß·¿Î^ºU¾ó§jk—ÏêÞZUب¬ZòÃŽŠlû©X`_ ÂVnÁå`ã¨(I">ëßLEÍÚüÚ’ššªø轑³¸ ÁÞÙh'@9ñ–ÇêF^â6ÜñÞ›«±tóÛò&-üi£¦9cBEÉšT«/º·[ƒ†^2Íò]s¯Éž¦h ¶˜ÐgYv‹ß cuŽ3¦›BxízKÛ›hïˆÐŒéÊÈ®kâqOw°ÎqT+Ëë!{Þ±óWzºëæ?VE=®¶ÖºÁ1ìé^ßkÏÓå,ï 88MÞîX•Ã¨5XL7ßv–ƒÞÈÓ]ßæ–§dkZäiYˆ†ñlhãù~¬sœŽ—-®þùÎbΈ§»ãδµ6¯ÃX…¹½Qê#bAôŸŸ^Žà[Æ`ãtj$zý2è{¤“ec±¼záäÝœ€²Ï·T0Vç8*·Âl‘—k^í,l'ÿ‡nOër»Päe>ïOÖ9L÷G+{‘§÷ÖþWRÇÑEƒ©uZ;LdrL.ß²ŠÁ:ÇQ ÉN+f·cw¦Ž×Í~죨ºžüp½–Ì{¡=n—rºtô4n8 -êp©xRäܳQÌõÆß³nzåL^Ju·œ„Á:Ç A¶‹%çv–w^ét¯›.½¼Uø6 &Z_¯Âán@¨ ¸°þ -Ô·åo¬Ë{n.¸¨f ”è5Vç0 -ÊÓ{Ý$‡îÜ­ -$Ü(‡ö¶#oàñÊÚ´¤¤þ– úعÈ1zƒuŽ£,’}‡âïݯ¬~çÛ±¬Î·m -­Š³×sÙv±­9Añ«w²ÌÌr¢Iq‘^ðNÕXã(;Ôç˜î_2‘²óàÚû(4J.Ô‰›9œJ¬s…fÙµè(Ýo½Û5›mªt$õß+ -û´UðΫŒèǵOQ Ö9ŒšŠ•MˆL¯jõÞÔ&pe›Hp4€xm¶¼¸b\LÓa*£^è`㨔dcLñþe¼½7ßÛâ -ã|)Æ-n°ÎqÔ½!îÿ‰ÅÉ1ŠÅc¾šNQóÖ¹tëýáÁ:ÇQi–ý‰r®ÕýwkŸÞƒbqu,×ÎËäͦõ¨OÛÙ®\ms»}lo»ÙvcÏ3s`Õ¦„@rË讋Á:‡<ÓQ .qæû×ü¯Œy@‹CÚøn>…n¹N,tŒ8¢?Ô¤©àf‘=>u‘zFÇÙmNÅžs—FÌwFŽ–ý , Rrbtä·¿ÿÝoþø×¥# ½â?]|./“•¨$®ÒiÒÒ)óçÃ\NbŠ|¡×rSÇ?Ìe…öý\ -ób&s9´÷phÏ•¹Lèî/u/ïÿͲ|új«‚ŸOq:5ÊM'Ú8ùm%ËAoF,8!0Èrâaú›»ÜMäÎ;5§™n¥ä¬Û(Ë “Jú‹¹üÏ“²~:9Xš<£ŒçËyªv"‹ÕmÉO'9O—2¦o?HâüÕ¶¶dùzC{$S¶%);-­q -õ«õi¤~¨Õäœ#ÙÈU¯dü^n¬Q3ºk²<½5n&Ë™ÄÙÐ|›(!V™A\ä¾ìÒàRñ&YÞ\ˆášh»Û-d9£–õíki:±‰ÞwëÔ_ËúÔ©ó9âã ·¥-õI¥ÊI¨µ©Ú»Þ6uB•F µdS7=)¤Ç“BüÂÖ¦áuºÆï´&çÉ]ˆ¹$nåJ-§Oj¹¹µÐ4•Z©¼T-µçQ–¡N¹u¼B›úh¦&9d½³KõëšEU×$—q㯈'Ù]xE~:…¤” VØP‡^O¤®ÝFÎv/ U^¸|ƒiVšœÆ%ÿ_ "4ª ÏôÅqRr¢ÞrµÜDl³‘×ÔÓcCÖzÀqZÊvÝåRjc¶ï/—­R^@·•n=¶¢v]ÖÑS©â H5ðX“]u,Vû§aeO´UÜn!‡6”ŸVäEÎÐL5Øv³èºáZ» 96ì|êÉ©iÜÉñˆMÞÖn7æ+ñqšØxK–¡ŸëM²lè®Ã§Ý¬#.‚ò£O¹ZkC'I5Ô¯‹‘i3îãéÇÓ4Íy -^<ÖHNNûí=&wþ7Ê|ƒßœøüm=^î2{ÛÐÂ7l]‘û™LΆσd9+¾›ÙtDÏì/d——²¡Ž†6{ßá5/Õí쵤kê·'†¥;²oC×;êz&> -endobj -166 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-79) ->> -endobj -167 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-80) ->> -endobj -168 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-81) ->> -endobj -169 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-82) ->> -endobj -170 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-83) ->> -endobj -171 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-84) ->> -endobj -172 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-85) ->> -endobj -173 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-86) ->> -endobj -174 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-87) ->> -endobj -175 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-88) ->> -endobj -176 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-89) ->> -endobj -177 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-90) ->> -endobj -178 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-91) ->> -endobj -179 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-92) ->> -endobj -180 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-93) ->> -endobj -181 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-94) ->> -endobj -182 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-95) ->> -endobj -183 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-96) ->> -endobj -184 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-97) ->> -endobj -185 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-98) ->> -endobj -186 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-99) ->> -endobj -187 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-100) ->> -endobj -188 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-101) ->> -endobj -189 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-102) ->> -endobj -190 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-103) ->> -endobj -191 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-104) ->> -endobj -192 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-105) ->> -endobj -193 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-106) ->> -endobj -194 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-107) ->> -endobj -195 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-108) ->> -endobj -196 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-109) ->> -endobj -197 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-110) ->> -endobj -198 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-111) ->> -endobj -199 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-112) ->> -endobj -200 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-113) ->> -endobj -201 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-114) ->> -endobj -202 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-115) ->> -endobj -203 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-116) ->> -endobj -204 0 obj -<< -/Filter /FlateDecode -/Length 4341 ->> -stream -xœå]Q$¹m~ï_ÑÏN–DJ¢€ÀF$.Á! A‡½ñž`.À9ùûùXÝR©«ØÓêÙ»Œì`¶»H©Š%’I•JÎ?ßü'œH-™Ï/?Ÿ~9yW’Ç¿¥ÅÍÁÂþ¿%ƒçâR¨YÒùg%Pq¥¦Xòùõ„ÿ6‡có×ÓïuúïÛ>á]­9†tþóe1¹"¥ød°*X>ÁJÉ•S6X98Üb­ b¤Â‘÷¬1¸J²Xƒcƒ%ƒ¸Ú³*Ĉ%°Å‚¡¦b°‚‡!zoò ˆg¦hðBrƒž-^ .KÕäðB‰ÈåÂ!‹W]Î9‰ÅcÈ’ª/A–˜½Åƒ,LR,^†,”É›<È¥Å+%btªÁÈë“Y|öÁâUr© -³É«ày)²çEŸ\"oñBp ¶I&%åbð"dÉ>U“Yùhñ²0~,p"Q1¼5ªw“§`ñd‰±°Éƒ,P¾…1C_²𠧸da@,Åql@r,, -D©ŽK! bMŽs-àÅ`yÈ’8[H@²0<×äAªlaEÈ‚A–ÈdamÖbanÙ×`aoÙ‡lap!¦·°€€¸$™-, @.L©ZX@À\vZX@]¸{±°€€º”r´°€»Ä’-, à.qð0p—=Md[Xtt„,,`à._,,`à.Ü(ZX­:Øa¶°¶â¢ˆ·°€»Q<[XÀÀÝXL^qðA²°€»1á¬È ÜÉG ¸9f `(øãÍl¸ Iv:Y° ÞƒÉÂœÌE¤%$àn¨†O­\š-,HÀÝ PŸÉ«.Œ¸á·er®$à.YX€»€O±°ïQ£… ¸‘ÚâwC¬Þ‚ÜEÞ,,€9¸€ÀÂÄ7½\NøŸnMI‡y'ô©óriÉûÞ´¾¹.ï²¾0:”ÒÈ.‘…}ˆÑÝœý]FW†6W¥ñË­QQÄ÷«"SÙ´ÙlÊ oµøÍð¶vÇQ<ÊZDÒèï`Ä3ŠÜptéd+`1€ºöëñ㛀¢3R!ÉTý|(˜ìsE}6ƒñŽïWô6à?Pp3ŒP˜hýQá[ËsºÊ%æüHas}£°¨E;.‹bÿÝ -«wr¾ðo¯ } -*AÃë¼GMö9Ž‚P{D÷úõÙÈ€6Ê!²³§-<î¼m“I-°j+K§» 3êe}jªë*£87Ùç8ÊÒÇf:Ï,ïWV‹SŸ‡AÝzV‹mC¢Òç÷±Œ6ÊîIPþ0ÏÌ®…Ì^†bZT1ᑲçúGÙPŸrúË([öÅñ¶í*TŠN*ê„qô:êsL:×ç0 -Õ*•÷{o/3â­BšÇÍ£ñGCa_¶ -¶«H-£œ#|¨:®ø‘ßMö9Žš"‚‚DÄ…w«©ýÎÅWøV ;ÜT‚³ÄÖv‘µå¹Ÿï¨”Ä…xyÄ:›…Nö9ŽJ“Îh#Ìׯ÷<#F~¨Çer$ú à ›ësõ]ÝX8ÿ?ñ8A°J>³,ëâG¹ðè‰ÁdŸÃ¨”½.ðºÐõÝ—o=¥ïÕé³áq-Ö}ò²d³“Óz4NÛù¡];ç¦ß6î³@Û§÷2ãé2‡¹Šc}2+qYlËåñLö9ŽñD„Š’ÒûgëËJ|‡2H—ÖeO2Γ}Ž£Œ„`RSJwúÊè^Y6Ÿ/½´½ÿÕKõ´[0ï¥MÝxr¶•úµ àäôeâùe²ÏqT­«>’Dž÷»€æ[•7`ܪ¶§S­ý˜"5Ïûtk«ûäßµ‰¾¤8bØ4n&ªþQv;Ùç0jKðñíùÇ^:1°›ßNÑ=™ÍÃgÒU“™u…ãôüêdŸã(GW_ 2òùGSéS”îCå½™¡{¾•C3Jåyßšìsõ%¥f¢¿9ßÊÉéûFA_HKÈ8Hâ£çý“}Ž£}ÒW}£ìYåì|Çû݃¤¯MRÕ·Xc ~>;œìsd¯‹ð—6ŸVÁvÖ$)„Å/— ksÅW½U“úá·RiÙ!3X0kv²l²ÏqTJº:‘9Ì{ÕnøêfxùŽŠÛóƒmÙ\né– ôŒ°¾¥+¸XPp=ñw²ÏqT–tI_ áùGEÛä¡©’ê>ZõÄâƒ'ÈrÖE>bx¦k­É>ÇQ™(У÷óƒv·Ôj¹m¼¯%?~CU!‰Œº.ÍŸM3&ûFUÅëj¸ˆ2ëyUmç.7ó€ßJ ˆÀÎç”uœfÕ0Ùç8jÐ¥þ‚K??¥ü—Èö -GGQ0ˆó 5Ùç8*H -²EêójÞRAú `Ý[áõժʺp@_…Ÿ_=2Ùç8ª]làu7‘o–˜oçÍï,Äûk¦ËË¡‰Š¾œceŒ^~´ -l²ÏaT+ËKæ¤;=­Z¹Îo‡£Bæ,瘢ƒ»,¯é¾ý:Ê\Ÿã ?é:2Mݦ‡»x [ÿ°pü‘‡Ä¿êt’°nŽRC¦ùè6Ùç8ŠM -ñ’ËüãBï-)h«E²eíÒûüö9­ ‘¬c–78f§i'ûGu¢kÌ•yŸ<Æ4­`ôªçe#ÒýH¨$ÿH9s}£œêuµ¼ÿù·sR¶b½q3ë5TrYöEᢻýÿ(E˜ìsœ!']–K~ÿ›pãJª¯M *CHüœ.ïÄBühn²Ïq†<)dBÜy+:zÈ0Ôyã›ÕQ JgлǼLzŽîlXcæ'øMö9ŽE×JéVSc‘žâjJÈLç3´É>‡QβûE(,|çÙÄ¢‰G/'ï2yÑ}OnÏ·¼þíù‡ ¨kÅîvåßöÕ‡Oµæž`=ÚL`݉À¸¥Á¸n‚×/ "šI÷éfò°É«Xd4·Î8‡ÍVªý|½§ŒØåtW`]ÇúTC7ÒQeÿOÿø÷ÿò﫲Çç€!]”¯+­uL’> ZÖ{ê¨êH^??_Ûé 1r¡·v "¾¶-­/m¡¡ø¢š¾¶Ñ½„p€ó…zm“†kåKß‹%]eù²SÁ/§¼læ– ý|^r¤*ºÿšÓ姥žu)ñÄ¡ yX)œïô\öIþ¥·ÒÿAòº›¢N¶/*®íY”õó) ôÕG.Ûœú…¬KF·äדnsËa;N’•xýêû¹ôÑÛ†öÌ*®n[Rrþúµö“Sj_cjw£ýSë¦Ûé"”v¤ö{¹°ADÏöd½{[‹£ëQÜdq=!m×®’ù˜öD?\n%ëî­mì_k×9ˆ]ôqè þÖ6–Ao_N¸,m©¯&U7(m§j£}§Ñ TíÔÔZv…pדAz9Äï|;5²ÍÐùƒÖt›· Qš!qoW[;ƒôr2Û]Ï–º¦Jo%k×ÚF~e5u’>ðm£+µèÞçƒxj_oÉY5Ðt|`µ›¸#¾œteÔŽüzJÅhV¬ð©™ÞH¤á¼,~µ—•Z¤ \³Ñt“,ýÿ©Sy–/»IéFwëÑzõÍN¾¤‘ž;ê°5Z†ç.¥Ûì8^A¼Ñ^A··î#vC†l —ÚÄSêàqKÝê:Y!¬OÇʑ蛸ÜJNÝ”_oÈ«œ©»jòýb9 æžúyWrîØù:’K×xÐ] »¼ý¼ƒÍ7âËØxKVÓ—v‘•}vëÅâ*@a%¯0úº›·vtÒ¥?íëêdVÄ}9ýé´„¹H)jV)è&¼cÆÎÿ‹6¿Ãïøü}ÛõíÇ£ïh;¶ÞÇH¦[¶Ë$Y·poÄî61òû+9ÈÚ65kèq<ƯyíîW`o-CGÐй#1­c8c7Ýh™ôñe3Š=‘ùZݬûrHíæpÒý›T8o‡²òh Éçá”8Ò¦kH—ãÐãIq8\°®’݈©9^¤½ÊÏ£*H¯5 i„ÒÊã5ÌŽ§n´å.†&Hi<OÌa¼h½‘ÇúÝîõÕz¿Šœ›(µC{§¾ŽÔ~cJí€gSi9Ì·‡=æвÙEsÞu„ú¹hð=êIÎêkjÁ7£TY»]¤ñ¢Üh7m¢Ãh¾ÑúK“)_»µaYîXÌQ*=§X©G:Z郘F­ä][=o‹x=[¢ž¥ÑœËzw~mÙsI1hyìÝ©¼¶¤FÛ_›o¯Ý¢íy¸ ^Ékfê×ni×ò]ø÷ß¹ôÙÌ X­ÚÏ2«dÝ'šC¡HËŸ¾ñ’30„u÷òÂ2Ls<ÝçíyýìF›÷¯u8&ò%è1²Ex£>f §jàÈî¦+Ë[znŒiæünVgù+øwÜé‡EKEý£5:'¢{íÌižeª¥ÜLŸ,?ÿw -r -endstream -endobj -205 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 204 0 R -/Resources 4 0 R -/Annots [ 206 0 R 207 0 R 208 0 R 209 0 R 210 0 R 211 0 R 212 0 R 213 0 R 214 0 R 215 0 R 216 0 R 217 0 R 218 0 R 219 0 R 220 0 R 221 0 R 222 0 R 223 0 R 224 0 R 225 0 R 226 0 R 227 0 R 228 0 R 229 0 R 230 0 R 231 0 R 232 0 R 233 0 R 234 0 R 235 0 R 236 0 R 237 0 R 238 0 R 239 0 R 240 0 R 241 0 R 242 0 R 243 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -206 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-117) ->> -endobj -207 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-118) ->> -endobj -208 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-119) ->> -endobj -209 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-120) ->> -endobj -210 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-121) ->> -endobj -211 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-122) ->> -endobj -212 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-123) ->> -endobj -213 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-124) ->> -endobj -214 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-125) ->> -endobj -215 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-126) ->> -endobj -216 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-127) ->> -endobj -217 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 590.755821 99.933071 572.005821 ] -/BS << -/W 0 ->> -/Dest (cb9-128) ->> -endobj -218 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 571.808799 99.933071 553.058799 ] -/BS << -/W 0 ->> -/Dest (cb9-129) ->> -endobj -219 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 552.861778 99.933071 534.111778 ] -/BS << -/W 0 ->> -/Dest (cb9-130) ->> -endobj -220 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 533.914756 99.933071 515.164756 ] -/BS << -/W 0 ->> -/Dest (cb9-131) ->> -endobj -221 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 514.967735 99.933071 496.217735 ] -/BS << -/W 0 ->> -/Dest (cb9-132) ->> -endobj -222 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 496.020714 99.933071 477.270714 ] -/BS << -/W 0 ->> -/Dest (cb9-133) ->> -endobj -223 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 477.073692 99.933071 458.323692 ] -/BS << -/W 0 ->> -/Dest (cb9-134) ->> -endobj -224 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 458.126671 99.933071 439.376671 ] -/BS << -/W 0 ->> -/Dest (cb9-135) ->> -endobj -225 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 439.179649 99.933071 420.429649 ] -/BS << -/W 0 ->> -/Dest (cb9-136) ->> -endobj -226 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 420.232628 99.933071 401.482628 ] -/BS << -/W 0 ->> -/Dest (cb9-137) ->> -endobj -227 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 401.285606 99.933071 382.535606 ] -/BS << -/W 0 ->> -/Dest (cb9-138) ->> -endobj -228 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 382.338585 99.933071 363.588585 ] -/BS << -/W 0 ->> -/Dest (cb9-139) ->> -endobj -229 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 363.391563 99.933071 344.641563 ] -/BS << -/W 0 ->> -/Dest (cb9-140) ->> -endobj -230 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 344.444542 99.933071 325.694542 ] -/BS << -/W 0 ->> -/Dest (cb9-141) ->> -endobj -231 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 325.497520 99.933071 306.747520 ] -/BS << -/W 0 ->> -/Dest (cb9-142) ->> -endobj -232 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 306.550499 99.933071 287.800499 ] -/BS << -/W 0 ->> -/Dest (cb9-143) ->> -endobj -233 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 287.603477 99.933071 268.853477 ] -/BS << -/W 0 ->> -/Dest (cb9-144) ->> -endobj -234 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 268.656456 99.933071 249.906456 ] -/BS << -/W 0 ->> -/Dest (cb9-145) ->> -endobj -235 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 249.709434 99.933071 230.959434 ] -/BS << -/W 0 ->> -/Dest (cb9-146) ->> -endobj -236 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 230.762413 99.933071 212.012413 ] -/BS << -/W 0 ->> -/Dest (cb9-147) ->> -endobj -237 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 211.815391 99.933071 193.065391 ] -/BS << -/W 0 ->> -/Dest (cb9-148) ->> -endobj -238 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 192.868370 99.933071 174.118370 ] -/BS << -/W 0 ->> -/Dest (cb9-149) ->> -endobj -239 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 173.921348 99.933071 155.171348 ] -/BS << -/W 0 ->> -/Dest (cb9-150) ->> -endobj -240 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 154.974327 99.933071 136.224327 ] -/BS << -/W 0 ->> -/Dest (cb9-151) ->> -endobj -241 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 136.027305 99.933071 117.277305 ] -/BS << -/W 0 ->> -/Dest (cb9-152) ->> -endobj -242 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 117.080284 99.933071 98.330284 ] -/BS << -/W 0 ->> -/Dest (cb9-153) ->> -endobj -243 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 98.133262 99.933071 79.383262 ] -/BS << -/W 0 ->> -/Dest (cb9-154) ->> -endobj -244 0 obj -<< -/Filter /FlateDecode -/Length 3716 ->> -stream -xœÍ\ÛŠ$Ç}ï¯ègR‘‘70 Û2²‘´`Œñèµ+af ’üû>QUy©ê˜éžÕZ̽ӑ—ȸœŒÈª&ü|êñ_ïJ©9ÉùòþôÓ‰\Ž„K‹ÝÅÂþŸÏž$»èk*ñü^ œ]®1ät~<á¿ÃåÜüñô·ONÿÞ÷ñçàjMÁÇóÏo•%ìrÉ™¢Áª`‘gƒ£Ë9Äd°’wXb­ bÄ,A®YbH-ÑbA ¡¬1X²çkV…!{±XÃט –'Èá‘Ƀ $ÂÁàù蔞,^ð.•ªÉËàù,³KY|ô¯º”R,O K¬ä-^„,ыŃ,Â%[¼Y81™<ÈJf‹—!K€vªÁ+ÞULd¡DÞâUv±“WÁ£’Ë5/Pt±0“ÅóÞEø&›CãFÜBËΧT-,ˆÀ]lTlaAî>‹…Àx‡5XX»ž±S[<க,,ˆÀ]ä ѸƒóH,,Àþæ<ÁuM^uTK±° wKÁÂìÂŽ -ÄäeGòŒ!µ° w±½W p—"³… ¸K‚´ÇäA.ÁÂä5à!l˜F²ê(),p—<,lñ€»D¹ZX€»ºmZX²B±°:dÁÁ‚TÄÕœ“…©xHùL^r¸Žd!²Ld‘, ±v•k±° ÈÂ!XXdAÊ`a2W=²o³<€,p³>È‚LÑ ™+UÄ‚ê1hÕâ¥àJ\›¼žÏä,[±·°µ”+z ðNÓ% x^,,Ȳ T,,ÀVäJ@fnñ¢Jp‡Œ=³d]dEX²û^sŒfüD³õ|ìçNŸ=`ó=ÿðŸÓ»O–“²öãáü,I ë¨!*Ì ¸rËp‚¢µê!ª¥EBø!}Tˆ\‡°¸º©ØŠ±ÝÖsÿâ÷"Cb•Ù#™YV&󼨖¬ÜæEø¡|@ò¶œBæËÔØËæ$ê »| I1‘æ ­¸ÅN²^’]ã/ÞœFÛôë m¿œäÆn.Pn ó›÷§Ï¾øó·_ýõkLûæÝé¿Ñônþ|þÏó›/ŸYù»Qu OŸêot×Í)jõ×ÝoÏŠ]HB4:KÂÜøÃöÛ¯¿¥n¿/ÛÖc2\dcßßêŒ}^ëÜ”>`yp$±p£s|K¨3Ö~ÿf¤°p»ù¾.äÈÿÿEUŽD í¿iø¯H33>eï -1¯¿C™hU«›Ñ/Æí÷æNÍâG¦´Q­XµÖ¿HìC¹¡ô;û¼­£‚AUù›joŸ6HéØTKÃ$þû­MÚ›¦©¿Ekãá 3 gÓýBë°Ì(Y2r¦[f¸¯Ïë1j¿ è¶÷?qòly·©t3ÁG7EÂŽ^‚éÌHDÝV¾iŠûú¼Sd½Å­†Û@”jõ70J“ª¼§¬ïeÓr:Œ“¦,ûqø׈ТÉzEYü‚½¯Ï«q‹ .¦Eeüb·h&9¨¹›jSwÀ¤ÌÛ÷Ëæ67S§÷/½QX¼î©^ëWJ!Þ0Ë}^YP'ÂÇõ†õ½Àù¡æù5"ISÊ"Z‰"!*UPÊÞ2Ù}}^Éôþ’È–Û‘t0E÷~Ù›®›rÜ›7Žhk}[,Dbkχl²õýÞçÆ“Ë,îsí«Ìô#¹U -Žé5¢:U”âå¦[Ý×çõ¸ªÎS*|Û­{nßc£mþù‡½5>öýÖÿíÁEŽnu9¸Çeon9¸ÔSnÙÃ@å³ _²žêY/ë£=Q–£Ýç _ŸWcx}Ö¢x”··ñäfÆO$t—}òÕ®ŸÄZwóg+TƒwÄnàCRxÌ -ºã\Œ¹LÇÐT¼`k×3]¯÷ 9åpÃ1îìóz#`c,{£é¶ÏSYül”‡VUU½·|˜ô¾% -.Wª©½oŸ(!c“‚ÚÑê}Ü0v…asŒ~P­z­.Ð4M¿œž9àr^Sá¯þôÇßýåïC¯fÚºlÈ×›±\m×µ¦mö4O+ÎPBŠµ­N“d”kh/#é [(|§á³•ÇV×6*É"¶Xù’·qòÖWÇÛ CÃXÇUš¶[Æ{w=oŒ+MS‰ðv=„ò±e[›´Î±ô¡Á[µ3ø2yý¾ô©C6isû¶†ÏŸÓõ'q•¿É¼èËoò|wáa[ÿ¦»uŸšt›&Y6›Þb¼Ö±¶mk[æMCßÝÛØêQjkÞøªÛÅqÈØtÐu×Ú¶6Óº–ÔK6¹0–÷Ïê-õÆG_KCwÄc½ÍVË÷‡w‡¯\­÷Ž5,c¥MÏïVømjºmzY>4ÆŒ›]‚åç|¥›Deö1ºÑ5{QâS§ttE”oþpúì·ŒïÆ úŒT†ºßŸRTÝgažéúzÕ³ÊóOßêfquMñksfó -±†O>ëùj­Í&S¼öø•ÁkxÔð¦ÇÓæ? £[^¦ø™bfù.ÃþÍ—:þbFîð™Žo·d}ÒߟÕW½Â›9îòÆ6&M´0Å”³Ïáf‹iøÞ×Û]×!¤>¡é%?ßÄíÙÎÍ&7’›ë[¾ížäÙîîCêë\Î -©YŸP°\Õ„‹ï&³æÉì—í÷Ãsá= ½™Ä‚ }Xß KW&øé”–§düò¤$¥â|ÐW)ÎI«NëYŸÌ)Xh*÷‹’üù‰žË (?õVÛ;)¤©ê#Jë +ÓÅÖ~¤÷'MÝRKq–çÇi!—Ð.ùñ¤ïˆAyÒ ²·¯ÔÇßÓ®F» õή[rt´}¹Çö5Ķí[7}®C¨5âåɆubƒˆžÉ_“uõäüFÖw0Šãí*uQbj2ƒ8”¹ípt¼IöL1^išnõþVkú×Úmb}VŸÌßÚ"½w/'LËGê£IÕ'¿ÛPM»:'¡j§ÆÖ²Dº Òåd?¥6´À˜?YMŸŸ[‰¥9’ôvµµ3H—“Ùn-vKåÞªŒ®µi~ˆ2\KW¼A»LõK֗ʦ!n_÷ä¤h¶¦2ùM¸"^NQz$òã)f£uXA±¹ÞLäiÜN.4üePsiªBh6š>}¤ÿ¯P;U‘gù⥻”>A8®Æ$›¼WÒLOuÄÒ€—<ÚNêò9wŸõå ít{ë®±uRÙDϵ‰§ ÕÁcOöÝë:Y!¬é§cåL¤&¦äØ]ùqGrƪ‘údÉOîû¸ƒœ:v>ÎäÜ-îõqÐ.owòùF¼,›Éêú¥M2ÈzÑܧO6‡`…A0ú¸[´vtÒWÜÚ×dÖŽ{9ýxZ¶¹À1èáPb¯o7Ì“GâOç/ñù×IÎ_µÇéÖ}Š¥b6©p1/—eðxlÉçiH\iÓ±¥ƒ+aꈋyP\NÖ!ÙNLuÈy’®z•_fS°Î5]Li„^òàÉØfç¡mYÅÔ)0ÏóÀâçIëNCè¯_l÷MäÔ"@©Ú;õq¦ö…)µžMåå2í/ûžÃËÍѼCC},žb{’3Bc¤²»˜}t¢–ÑmkçI¥Ñv“¶¢ÃhÔh}ÄÜdJ[·¦–eÅÅÔRî9Å þ8k:ÎVéJŒ³UÒU[·íx=[âžÅÙóX–=—,-ͽ;UFKn´ë¹e?wÛmÏÓ2d’FfJ£[¼jùAø÷!±cjY­ÚÏrªáËxµøÌ—¿)@%%`ˆèkay~2ûå}ž?÷¸>ÝhçóL0‡fÊ^ÿ:„-Â#õ6cªýå»[%sI/SÀ“OòÛGËëÁø·îôˬ¥¢þ5Ö¤·èKŒ™¼y̳µ”ÝñÉòó?1³Šo -endstream -endobj -245 0 obj -<< -/Type /Page -/Parent 1 0 R -/MediaBox [ 0 0 595.275591 841.889764 ] -/Contents 244 0 R -/Resources 4 0 R -/Annots [ 246 0 R 247 0 R 248 0 R 249 0 R 250 0 R 251 0 R 252 0 R 253 0 R 254 0 R 255 0 R 256 0 R 257 0 R ] -/TrimBox [ 0 0 595.275591 841.889764 ] -/BleedBox [ 0 0 595.275591 841.889764 ] ->> -endobj -246 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 799.173057 99.933071 780.423057 ] -/BS << -/W 0 ->> -/Dest (cb9-155) ->> -endobj -247 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 780.226036 99.933071 761.476036 ] -/BS << -/W 0 ->> -/Dest (cb9-156) ->> -endobj -248 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 761.279014 99.933071 742.529014 ] -/BS << -/W 0 ->> -/Dest (cb9-157) ->> -endobj -249 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 742.331993 99.933071 723.581993 ] -/BS << -/W 0 ->> -/Dest (cb9-158) ->> -endobj -250 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 723.384971 99.933071 704.634971 ] -/BS << -/W 0 ->> -/Dest (cb9-159) ->> -endobj -251 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 704.437950 99.933071 685.687950 ] -/BS << -/W 0 ->> -/Dest (cb9-160) ->> -endobj -252 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 685.490928 99.933071 666.740928 ] -/BS << -/W 0 ->> -/Dest (cb9-161) ->> -endobj -253 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 666.543907 99.933071 647.793907 ] -/BS << -/W 0 ->> -/Dest (cb9-162) ->> -endobj -254 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 647.596885 99.933071 628.846885 ] -/BS << -/W 0 ->> -/Dest (cb9-163) ->> -endobj -255 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 628.649864 99.933071 609.899864 ] -/BS << -/W 0 ->> -/Dest (cb9-164) ->> -endobj -256 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 99.933071 609.702842 99.933071 590.952842 ] -/BS << -/W 0 ->> -/Dest (cb9-165) ->> -endobj -257 0 obj -<< -/Type /Annot -/Subtype /Link -/Rect [ 468.550614 515.652842 489.260575 500.652842 ] -/BS << -/W 0 ->> -/A << -/Type /Action -/S /URI -/URI (file:///doc_tools/%7B%7B%3Cbaseurl%3E%7D%7D/rancher/v2.6/en/security/hardening-guides/rke2-1.6-benchmark-2.6/) ->> ->> -endobj -258 0 obj -<< -/Title (RKE2 Hardening Guide with CIS v1.6 Benchmark) -/Dest [ 6 0 R /XYZ 85.009843 502.818898 0 ] -/Count 18 -/First 259 0 R -/Last 276 0 R -/Parent 277 0 R ->> -endobj -259 0 obj -<< -/Title (Overview) -/Dest [ 26 0 R /XYZ 84.259843 283.881496 0 ] -/Count 0 -/Parent 258 0 R -/Next 260 0 R ->> -endobj -260 0 obj -<< -/Title (Host-level requirements) -/Dest [ 39 0 R /XYZ 84.259843 369.381496 0 ] -/Count 2 -/Prev 259 0 R -/First 261 0 R -/Last 262 0 R -/Parent 258 0 R -/Next 263 0 R ->> -endobj -261 0 obj -<< -/Title (Ensure protect-kernel-defaults is set) -/Dest [ 39 0 R /XYZ 84.259843 289.233496 0 ] -/Count 0 -/Parent 260 0 R -/Next 262 0 R ->> -endobj -262 0 obj -<< -/Title (Ensure etcd is configured properly) -/Dest [ 41 0 R /XYZ 84.259843 480.020658 0 ] -/Count 0 -/Prev 261 0 R -/Parent 260 0 R ->> -endobj -263 0 obj -<< -/Title (Setting up hosts) -/Dest [ 41 0 R /XYZ 84.259843 237.872658 0 ] -/Count 2 -/Prev 260 0 R -/First 264 0 R -/Last 265 0 R -/Parent 258 0 R -/Next 266 0 R ->> -endobj -264 0 obj -<< -/Title (Set kernel parameters) -/Dest [ 49 0 R /XYZ 84.259843 799.370079 0 ] -/Count 0 -/Parent 263 0 R -/Next 265 0 R ->> -endobj -265 0 obj -<< -/Title (Create the etcd user) -/Dest [ 49 0 R /XYZ 84.259843 500.585993 0 ] -/Count 0 -/Prev 264 0 R -/Parent 263 0 R ->> -endobj -266 0 obj -<< -/Title (Kubernetes runtime requirements) -/Dest [ 49 0 R /XYZ 84.259843 352.740971 0 ] -/Count 3 -/Prev 263 0 R -/First 267 0 R -/Last 269 0 R -/Parent 258 0 R -/Next 270 0 R ->> -endobj -267 0 obj -<< -/Title (PodSecurityPolicies) -/Dest [ 49 0 R /XYZ 84.259843 257.592971 0 ] -/Count 0 -/Parent 266 0 R -/Next 268 0 R ->> -endobj -268 0 obj -<< -/Title (NetworkPolicies) -/Dest [ 56 0 R /XYZ 84.259843 663.033496 0 ] -/Count 0 -/Prev 267 0 R -/Parent 266 0 R -/Next 269 0 R ->> -endobj -269 0 obj -<< -/Title (Configure default service account) -/Dest [ 56 0 R /XYZ 84.259843 388.198913 0 ] -/Count 0 -/Prev 268 0 R -/Parent 266 0 R ->> -endobj -270 0 obj -<< -/Title (API Server audit configuration) -/Dest [ 59 0 R /XYZ 84.259843 315.158799 0 ] -/Count 0 -/Prev 266 0 R -/Parent 258 0 R -/Next 271 0 R ->> -endobj -271 0 obj -<< -/Title (Known issues) -/Dest [ 72 0 R /XYZ 84.259843 463.537950 0 ] -/Count 3 -/Prev 270 0 R -/First 272 0 R -/Last 274 0 R -/Parent 258 0 R -/Next 275 0 R ->> -endobj -272 0 obj -<< -/Title (Control 1.1.12) -/Dest [ 72 0 R /XYZ 84.259843 368.389950 0 ] -/Count 0 -/Parent 271 0 R -/Next 273 0 R ->> -endobj -273 0 obj -<< -/Title (Control 5.1.5) -/Dest [ 72 0 R /XYZ 84.259843 164.143950 0 ] -/Count 0 -/Prev 272 0 R -/Parent 271 0 R -/Next 274 0 R ->> -endobj -274 0 obj -<< -/Title (Control 5.3.2) -/Dest [ 81 0 R /XYZ 84.259843 520.622079 0 ] -/Count 0 -/Prev 273 0 R -/Parent 271 0 R ->> -endobj -275 0 obj -<< -/Title (Reference Hardened RKE2 Template Configuration) -/Dest [ 81 0 R /XYZ 84.259843 228.224079 0 ] -/Count 0 -/Prev 271 0 R -/Parent 258 0 R -/Next 276 0 R ->> -endobj -276 0 obj -<< -/Title (Conclusion) -/Dest [ 245 0 R /XYZ 84.259843 575.052842 0 ] -/Count 0 -/Prev 275 0 R -/Parent 258 0 R ->> -endobj -277 0 obj -<< -/Count 19 -/First 258 0 R -/Last 258 0 R ->> -endobj -278 0 obj -<< -/Length1 10956 -/Filter /FlateDecode -/Length 7306 ->> -stream -xœÕz \UUöðÞ{}î¹.÷^î‘×åqyˆÁP‹ ¾ÊPÑQË|% øHÑ0S´|$øÈ” ÍÐŒÌô¢d:Q¾²æ?j“eÓ””5¡9ó·š!Ø|ëœ †ý§ùfæ÷û¾ï÷}Ï~µ×^kíµ×ZûÜC(!Ä3‰$dffß¿nʾ털,د¾ÿžìaú´¤ؾ€íÛ/qÖß^}‚êÁöø©³s‹á3 ë°ÛµSÌs’!©„ú#Œ((ž6{Nÿ3‘'>ß;-·¤˜è0Ò@LÓ]T`ý*xV[ñšžŸ›ç ´âXuþ”éØan’±=ÛQÓgÏ{ló[¶/ž›_ey*ràS–ü}ˆ‡-swü$ ÕwÁßáÇjøÁ ß ¸)à¿]ðW3ü¥n¸à»§îáß ¸^ ת¡¥¾m…? øf |W|•_6å_VC36…+_ôãWZá‹~ð¹€? ø,þh‡O«á²€OlðñR¸t>ð!‚¸.^Ê/.… Cáüïƒøy¿‚ÿð;x_À¹jxïl(OÀÙP8“§¼»ÒÊß †wü¡IÀÛ~+à„€ãÞpLÀ›pÄ - å.Þ Àsø(÷8|h2?|/“½áâ‡&»;à[zü^ ¼& ^À«öçÁ+fØ·×Å÷åÁÞ:ßë‚:¼ŒD¿Ü -{¼$`·€]6¨ðâN31všá…<¨AšjØ!`ûó&¾]Àó&Øö\ ß–Ïmµðça«¶`³€MÕ>|“€j¨ÂAUÕ°ñY3ß ÏšaC+¬_w”¯°níd¾î(¬[&­}ÆÅ×N†µné<-`Íê¾|€Õ}á)dó©{ r•‘WÚa•*°£"ÊQRå.Xi…ž\nåO -Xn…',P&ÀÝñøÒ¥üqK—Â’<(ÍvðR,°HÀcfXh‚˜/`^+”´ÂÜV˜Ó -ÅŠ -x4f ˜iMç3Ç Ó—Â4lÈ'`ª€)rBN+±&à7þü7‰0^À8œy\:d;`,µð±=`ŒFðã£dá£î·ðQî·À}Fâ“‘F ·ð~0<ć·À0* ³2ªaˆ€{Y~o+¤…{F‚[@š€»ï²ñ»íp×`_~— òáƒÝ¾0È -Hpç;¿³¤Xø;¤$yŠ’Ð?’| ñ#Op‡úy‚ô3Bß>zÞ×}ôÐ;â{¹x|ôŠ³ñ^.ˆ³AlŒ‹ÇÞ1.ˆvy´/¸Œ% R@„/„#Ÿá6pæAX+„" ¡yâÁ(Á`A­Ð3±( G ¤øã ÿ@p° ð`C›+òjMËRðͳ“?÷`Bh“?,   ˜"@g9$|(¡8{A Õ²pÖ¨ˆê¡y+Ÿ¦ñÿ?\äÿ5ÿô -¡ÉÄCÎb:AêÈ6ºÌ0{jز’ÌÇž·éYZÉú`ßnrƒœGÈ -rê$BG$ì%ägä&Í&G*µÓT:ti”tP#y¤«‰ J¤sRŽTB“`'Ïwã -ï`|sš„ý c‚#ð $A£4D2“ÏàÔ‘/qÕwžÅ(£½·‡Øi)c¥l öœäçÈVLEøüÝNÏ#uGè“ä"Ù F¶Ó‹È×Yòy²Yºç$V€ôŸD\çpüVR"~‘ˆ`ñ؇Ôã\S´<úð‹ZºAÊpælR+{d».gQ%¶›¾M[ägI 9Áø„®”"¥=Ò0²Ö+È!k÷VuŒ\@!ïj*U±³…R­#ßH9º)ˆû•#œó ÃÕ˜­ï…²yDWB%Rª> !çt#¤~81è–"×ãd2k¥d?9@ú@5Y‹˜4~åü¹Múy^KŸf?s0£Ðéºa[ý«¡Âí—]K#Éo¢É¿’ž¦јÓ$¶Šícû í¶tZÊø'ééÍçòËü²ì–—ȵr­.ÓR]ƒbÐÒ=Ê }}‘mí`\J÷Ñ–õ0y’Ñ@ÒC²´µÜAúµ$ HrD~rþ¼÷9Ft“øEb n·#j$VßБ½& £FË…‘õ¾ÙØ}çÄ–&/’Ä›-Z0v™Aú‚¯Ä&·òdW’5Ü.è±…柡#Új뤒aža­ëÊT:L¶»c{A`+—ˆ•s)Ýò‚u£O}½„{‹X Œ‚, ‡XÚFÖ;²GÖûg?8²Þžý RÇïœØt¡åøq«-µ“š›5: ¿¦ã×h}°ÅŠ´¹ÇIãùxÝbi1_T¨Ã(õÄ°,xY ÏïY4/x9)\ÞsyÐòà=dOu2™ìB&’SÈ€””äþÑ‘².9%%)QrØeLp»Ÿh»¯„>”{ÿKåœlñ… _S{ƃâf]]ÝBº~àìMÃV§ßûÞ‰_ÿö¡]Å!â[Œ€×v\‘ª‘{#‰"ÜQ~2ñ)7‘¹&Ø—¥ÆT±>xË¡ õ †ð° .rÖ|±4ßlnk¶\W2ඣ]¢çØ98'åge]mèP6™N¦²Ã¨NÑH<•vâ°“H'±ZHx¢?«]µcÇ*¼©þ¾çî;uÞwÐYŸS.n|!ÚÅušEƒî{ÙùÂÑ£/ì<Ây¢¢Å_Åw¿™,¾ûö+ñgÔåEt -ݪÆôx‘ZPƒbÅý^wd ©2è«lOÐ*Ã+aV£ÂüÃ81ûóÀà¾zl“‘­Ä6T \£Tm‘IËõÔ„¾Y°†käGÞª¸fUöþ˜K‘áôY:äÅçŸQ4Òøë×oF&]m]¶¤j—¸ñSû×ìtû§«×¬dâsŠw­r§Ýyvó©QK:®ð\…@’âîéó‚y¿¡ÊJ_ÀÃJUÀzëšžº@’`·ôTIlÑä®ÑöÃõ„ƒ¾AaA ÉS¥Ú)É”Y¾%VSpuy7¨…’åW f^[!^‹i9[~O¹øÈÃâ¤øH\'~äü°atF§ÓC‘*UŠ«QŠFÒ×í«$VEžPª¤W œêuL$“JÐ…¦¦[ÒJ8æƒÂòCMµvÞ§¡¾½';ÙžÊþÖv7¿X'2ëÚ¯Ôuò¼y!©n'áA´ -‚ªÛ ÖýŽ*ózeM(#ÁÖþRR@£%'jknkºÅ»¸pÝšššà²†'‡[e©‹[) »¤·Å!f›/¾ª;Å|ºš>¼êŠŠÛV‹ëâõ£¶Y{.Òõ»ÛËÆŽ£[èlZH· ËüÃ#9â}ñ_â÷â}—W|&ƒÞn»RÅ^‘ȹçwêi—ð‡"ÜŒ•„YšÐäX“¬‘˜Gž>ÃþxæL{Ä~±}ËkW¥A¼¸é³ˆHßÃäU¦¢“TËbQ-Ñ,›f©Û'»yÏákù.«Ø+âkïÄÃgjöò;w¬bå2×Ye™ƒUâÓì˜ÂyU/s*ÉŠ:‘Ñ»tVUÓ·$¤z-•tM³VÚ-]›q@äƽÁΨÄôŠ?‹å±Ê–Âû+CY&¿WǦ±l!’­âk•ì9å*sÈ çz9uÜ]ˆåñr/]Š”ÂSäd]‚épKÜ-»unÓÈ‘¦óiº…¼Ø´Vóg䵺µ¦­ð¼ü¼î¼¡{ÞÑý>Ô} ßH_ó?Ë?ÂßøßåÞ“çÉsP4œâO“òv*µAOñC{’*ëJ¶°}XÛöAûêY_•R6JÉ-ýkîd½¢ƒl ¸U’ ÃHŽ*½½Êç £Äe°¢]ð7sC` dM³‚MRˆ¶ØMª‰°zÄ`ÕåØRÕtËÔã­ÙùîPUvé‹ý('œr&ƒNrµ3\ÄE],bäh]´­w†¦Ð–I3Ùt>_šÏú­’Wé6Ë›uahü©¿€_$ vGªZåTMµzMj$<}OéÝç.½5bõc—ÏÐS”´=Ù^)6TUm`þëÓiYõ”öJ~ñÞ>Âh¿^ñä“+UÍ©@ë¿÷` yÜ=ØÇÄÌFªè™ÎÀÂÂBÓ ÆÐ0ÉA‰ãûÆUV©Šlt¡AŠ 5Ât$"(ÐÜGhˆµ\njikiF/ ÉEõ š²¼kíR-³*•Î…ã‡Â9×/î8ðÚ/ÍKh5<":&Ù¿ÓgÄÄDG'÷JJô—†•¼÷È®×î^üÅħâêÌï–•¶Ì}¥±bkéghÀ÷3>æµï HY¶`j~X`ü¥C—þ”Ðïw™«/\Ö£Ïñ½ï6G«ºPŠ|÷ÁÈÎ@\¤½~˜1@o&/È f«³<ìHpC¤Çº&ÀD ‡^1†bψÆÕïBKb¢U[ú¦æ›mh‰ÞUªƒêÿ -BBÂœ á i1îw¨;Ìít‡»#²B²B³Â²œYáYY1Å1+C*B+Â*œá+#ÖÅÔÄ܈ íÚ5¨k@NhNXŽ3'¼8´8¬ØY¾,tYØ2ç²ð·yÖäÖÈd´û(©”¤ðî!‚?;öÙ¾'Š¶4x惩–¼ßÖöþ’©vש˗O¼|ù[ñ…ø&$ôõÞ½Þ|ëÁ©Sè@ -T¢§L­S5ùÆ3y¨3~¤¿;ôÌT®0[=¦MÊ2JÝé™vÕÆßlÑÜ"Òmµ¡ý:ãÐBÀH«—d¬$Yµ%åy–,©Ú×Ðþúüï²Úö‡ØöÛÕ¶WÈööíùyß©ò;“/’Õ·Ì:ï6ËǤ×H#ãT‘H¦bi¬EÍ;¸½[Ÿ¥ÏÑë¹DàDŽÈ¼¤œŸjdû7]|lC|´]q¿îïɦ.‡¯ÌÛ~ó¯:|‹IuZHÌ€Þ>†Åñxe<+`Ó”ôòËY%Fy–UóMʋ̦zyfD 1’êããÑŸO‡S%¬”*ùÓèÉ·Â&]¼ÄéÞÑ}¨ûnÀÒ ©§ê½UçD)FÔ‘G˜ëÛöýlÖö“ ²½m½Ò~³}‹lÿÔ˯.õ/‚LrGË6}_"‡è¦Š'x‚-:bõU9˪øf÷PzfFjŽº­­Å{<¸ùæখDm1Ý~ QYQÅQë¢j0½õYTG”WW[OG÷5þy±ÞÅŽË8¾üÕc sç¯ÝÝ0wáÓ»Òê-Þ •K|ÿ…ºô/lS—žmßùÜ[/¶WH9û§MYrKó?ŒïoӼƬyÍ]šw0ÇñƒýR÷ÿÝÉUÕóZ£ùÚÎ Àë'7ØHƒÉc[ÓCoó 6GFÍøtZm4Ø‘i¥¤T.Ó•)eú2C™±ÔTæSf.ó-³”YKm57­·ŸçbU;ãµä¬dã¾½UÏîÛ÷ì j×oüE|G­ðÙÕÓ§¯~}êä7ÛÄ)Ñ"®¡éIE c§wz%#B+Áã£lÄ•4B…Ù£oÔd…(™6Õ”hûmä…÷T£x0Ëo‡Ÿ*“è_$F„ ï½í¥††GVúõ †ƒ6ëÙcíPS9ÇيЗÄÙbÈÕÎxglg¸3öçp}\¥d/wTöP}œËós¼3&H1ë{DF¬JÕ…Ûâ´pß«NÏv{¼Óîu[Í6ƒM}Ñt÷6ö6 Ò2 22ÄI£X¬!ÖØ˯Ÿ½Ÿ£—lhlXœ3.<*¦ÜPn,7•ûØðH@“ ²Làfð BO‚`)DÓ/.-²¸eqëâjânÄõÀ°qÎ/+5@øe`¥ÊVÚ3©²rÊÆ´¦]?~4éíG ÞÍ]¾&¯{ïæ?½_pPJÛ›ínîµ¥rÛ¡ÈÈcÉÉGÌrùFU-ß¾/TÕòhìþÊ·£–£G6sÅ^&VÚ¨TŒ(côÉ›YÕòÁMøKô²½ûýÂW½6V=6¢n¡ÙÃH½3‰…´T¬Yòæ›wVTðíâ·kÛk*GmÝñ{–³–Þ­ZØý¨ç´ýe'ƒÜÁ?ï°5Úh÷˜pÙ£p§e:T…OõjTsâ­mVä8®n3?´ë^žDÓýê6{Åã¹÷µù'NÑ趻=wÇŽcµ¬ô§š}SoÀ•û»p—I9D&?¹c:m1e\-€É£uBätä-4ØÀ(—ˆN}e¸uÚS_%©ï•Ô×ID{à}“Ôíl¡tÞüšú"é™al&+ee¬œ-cëY-SÔ‰ô ×Î=¡§M¢iÄIN%™$Ó0PJP2I&Ã¥L>Lv+ãÉx:&JYJ) 3`†4O—s”ùd-…R<‰,–W’•´*Ѧ—ËÕ¤šnb[a³´™o’÷ð—äzå¸ò™Ò¡Ü=Y5êzä]oÓ‡éÃo‹‡Z¥œ¶lØ÷S Jh< - %d¢ßº‡óq2×ôÒ8ƒÆŒ6Ž2£Od:E‡'2®(:Up&„FÁ™Ò èÒdžQ1 zÅûÇ(Ú ¯ GÖ[Õwq65óS3£WŒª\éÈzø¨Ý–Ÿ¥û^@çá÷Öé÷vß*½UUÞœù3aH6 gÃy¦Ámx=ÈDz …¬áZ,âe¼‚ma›ùFC#käﳓðáL²däŨÇÂä`à/õäAJÞnt˜Ôó`$‹pÉÅ#äK‰ÑG‘¦TÀ“³’jJ0g²a)¹¥tïÙY¢bbt›Ýfudz,i4#Ñe)cõÙ†qÆ©$æ³™/Íä3噺B}®qš©È<ŸÌ§‹ØRxLZŠ«[&/Ö•éSéËô¥†Æ¥¦ -õ4oÞD6ÑìYØ&=ÇÕsèÅݯڴü›ì¦µ¬öJ{ùËò˺½J­éUóì5xS:Ê=ú·ÌMìmxO:Ãiá§ýh$@äxÏW_^úêKøäÒ_þz u£fª÷O5PÝ6Sý ŸLRÿ—ô¸¥´ÿçÕ:%þØòÖQhfgºõKÝêœô £:ë2±Ór/)"Åd™Kfid:™GœÍN%qX&’LIX›‚N’Ž0óH ÞsI>É%³IoìN -¾/Öî!br’1·p•h­|,óqÌÌóÒð/ÌšrkÖlœiÎ¥þ;PˆÐ*¹8æß›qÖfâ¸ñd>BLEØ\ [¾6"WãȉX -1/F˜)ˆwÂ9q|Ξ«=û%ž±–¤Odöª³– l‘†)çNBûß}T×ï74¤ã#õ[žx1õ뜎 Rû&âãŠ"í•®oy´R}bº5¡կ6¬˜(Z}æ˜(žè£1ÅDI/\9Š´õÅ<eM‘Â!˜g’a˜ ÷a> -%Y(MJÆ!´“ -ŽLØÌû²‰ùÑÜy…¤‡öUѾüèªiгòçªßuŽ•´/K¨Ê{H…dKØûÄŒ4z%qY½ÕüÿöõïÍÙ±®û¨Ž­­_–‡»ÕÏkymgëåÿé«“ÎÃ?Ó¥•'þ3<ÿäÛZçqUÏw®îùÛ±täþg8ÿÉEQ£T]ïEâQSU톚<õx™ˆz̈>ÖvA/lÅcm/HŒ62“Œ£†¡ÕUw‚£G=bƒÚ?“1MÄ]7 “âèÜoè«°¥b ;×°KÝ°KØÕý¦Ãý–‰Ø‡bÒkópïDì÷a2áÞ˺:·6·Y›Û­Èxb!¿Ád%0Ù4jü4jì{tÙHìÚ]œ;7w -Y‘;wv!Y1enî ²bjna æÓóçb¾hî£dÅ´ü"¬O››?‹¬˜ž[ˆ0Óó§`ϬÜÂ\²âÑÜ"§šã^_1;wÞt²¢p–ÚS4-w6Y1w~!BÎ+(œ†ùt7{Ð}]TºþD¾F®›ˆîSµ“-G“‘C¿èRëݯ®6ÞY÷/(€ñ.ø4e’V³¿]õn8nëŸÔ­=÷ç:»“ŸT›<øÖÐXäÒp‹cò¿£Q -endstream -endobj -279 0 obj -<< -/Length1 7012 -/Filter /FlateDecode -/Length 4216 ->> -stream -xœíYyxSG’¯î'K†`À6²Aœ²„øÖ“d|ɲ%ëö%ÙÆ’md|ËÆ'8†` ɲ’’ì&!áH&a¾“d6Î$L®/™I˜r !™0Ù96ÙÍds~Úê'Û„dÿÞo÷=·úuwuWÕ¯««ê=€90˜lU•Ë÷f€Å °w…Íã+]nׄ°­Ãvv¥'K[/Ù@ìØÞ°±'ØWZ{n9@ÌlßÜÞ=ÒvË?¸dn€„Ù­Á…d|>ŽÇbèÀŽX·çK°­îèºöö‰¸KØ>Àµt÷n ¾å:µ ×Û‡}Oõ¯í#åÐÏ«˜<›‚=­*u°¨ûîîë -?ZÅWl˜."ÝçH{¡i^áÇýöÀé~qŒÕ~ý·áó—Ü1îQlÊ€BäÂy…pZ$ÒðyˆåŽ‰+M¿Îˆ=g@‡8-ÄyË2b_ Ö–pr Ä`cEéÀ©I#hE´ØLñ›º^k%¼+V¤De襃+þ‰Q5}VÔLB¾š6ýšh­\yyº±(ÅZx5,P äÃ[N΂ž‘˜ •OË!‡n†UØç Û°¤€’C!Ý8 éß[?zÑÃH× -åœÒèËPN ”“o±=ÛOA9|…ÑÕbÿãÑçßàxŽ›£í 8Ïyt)¤ˆí ‰;ÉⳈw”6 K9më4Ú iÄÉâó˜Èß߆ÏS."‹DŠ´cb‰ÌC:²ëÙEºAývšs8Ì!„8Êcý¬þ1þÿúßu‘{ÁóCcx>ܬÂÂaY€¿D¬%p-Ö -˜=³ñ$¦ MäãÙpCxñl­‡­+RÂaqLiP.¨€j¨Á±  zµû£Ç¾çe~èJ¼|¯€"ðãÝ£°î…£p>#s‰›t»ÉÛä3ª¥!:<㾕þŽ~Ã)¸lÎÂ]ËÝÊ݇÷3Ükx ™+ÉÄ»Fr§äx“œÅûï3ï˜Õ1M1{b^ˆùJ/­•n•Þ)}^zRú‰l¡,OÖ%;({Wö·ØôXglOìŽØý±ÿûûØOþÜßN¿gÅ õ¨¨·mE†Û&Six,#×Ïù ·ä5¼pב -¶‰T"Ͷ9Åql GºÃVòsîA˜Dõº„\^.¥ò ÉrÕ©PgS_¨½)@RŽ%iÁY™÷êš…3•8³>23YoHÐë¨FÏ'%ÈPYv{¨¿¯©³?´áÙY]DqÏ¢éœN="|ØÈ¢Q-ôRíŒ+D%WéUz^ÏËyê;é{çßIb;}ºüÌ á~xm%ÏåT‰ô9F…¿“8ÀOìäwävDáôçlÕôðx¶°U“Rõ:¯M’/ªªÔjÅ"•j‘T­P¨Ya´úp 9„Q×%DEÈ!áózÏ=zñˆ¤ŽJ#þè釸ÖbÌ,&—JIÕ«SSU)R©|AR¯5ôƾ›oêë»éæ>²?Ùf¶Ø’“m³-™.}ñÅ£½ôÒcµ#NçHmýV§sk=òMÅÓõ }›çD=¯ŠO}.@Ç<ðKÐ0âƒwƒq’© QîR™R®Ôÿò–ÅV‹‹«“Œµ¶lm&uÂòÇ®ß[VâòÙú¶´xöWºK˜Ž9á‹t>ý=DD¶ˆL.VL /ªÀØ þÃ$?óX›e#oË+që:]?‰­“y­VGKkòÂ@c™Éè–­§#9^½±>.f~£;ϯëm+t;ò]¥nR`Ô›LÄó'ZG£Ãì-WOøx^Îv:‘(W‘¦ž––z ÛÆCeäO‚ºòÐ!r\¸žÉìÀÌ!¾ƒ~ˆRn˜û -y™˜ñJzp(¹^çéÈ0ôVwîš»>Îkª¬ªªÐW&Ô'é çV¬.ë)²”v×癚**||:Ù‡²1Fú ¢¼wK¾ 1²IÕÏWj“’UsñLò#¿\_G’C¥íÛô!wûP&æ¾ÙÞ'ÝYuÕVOU9½ãÆýº‚±~K·)px˜ôv­»]eãÝMµÎŠõL+ÌŒÈEú&Ëߘµ)õJrQøK€(„Qò½ybˆ^_Áâ@aøo´­n1úuÌ:˜_Þ¹¤) u†\5C!*¡T:%êžBCÈíå F‡i³Û=h²ɼ*kY•§Ôìù¨Ñ_^^p‘Ÿy µ<_kȯ•'6™K‚z}°Ä¼!áti©q­¥líËee¦Š -“Ù&â¦:´#³^c¤¹,ÇnšË&Ä„yÞaôTRys¡¹³ðº›û3B m±Þ5^·ÕS]q£q×Ý|n—+tý¯ž} f“ï6ͨ³;êØÉO2f†),oÆ_r%AéNú4öÌX©”©ùD•†ãÈŸ…k7\꼸¡ztôž{èÓ®ódƒpaŠ™J?Žk(Äì5 -£2~ו“‡ !RÉyr¢x¥¡Ý*\º`kÑšsá.“ÕZzçyw•ù¾C$¨u®ÑûrèxV•Ö¸.ae¡^_8ñ)/ßü5Ûë´ðú:} –²SÇðIVÍp¹|¼´ÀàÀÚž¥=nÇPiéÚ‡‚äåû›Z‹-–í^Ïv‹¹¸¥Ç¸ õgºìC]f‹¹jTU<•<ž'ûZ¿þ¡3gþñg{wŽÒqß}¡Ð~Ÿ¸mïÎ]·Mœeâ*äÌóÑO&ÊÙ±D¿¯”²D¸‘ ÓÞ?»þÃEŸuEiO!ÇYX#R£Ï"§„7¿þÚOÇu -¯âhDë?ÒEKŸ©.»|.š‹ˆmXºÉáØl6ov´Ýpð2¸#<ØÐp0Øûˆ†|Qd6o¯®¾Î¼µÃi­ÙëAÝÛî÷oj‰ÊNÛ#‹(‰ ¿´Í/ì÷ûÉFÔÔF -…Wè¸ð¯DGp‹}H}ÄÏHD™ÙZFMPq!ÂB^>Æ•V -¯bb^Â¥ž'fá~á(÷âôß*шŽ|%<áw:±+ºb?"‹h)Uâ’$ž§ý~á6\q“Ÿ¦Mœ¤ÏN¼K×à^¦…¿£Ÿ!bóßg%©ÓºT%zÿ)É*œæƒÀÁf†ÜÓ9Äð“ùïgØø··öWZF=Þí¢ÑŒZ¢Ö²\Ô/1`†¢bF/—¢µ0c¡Ë‡·îö¼eÛÏíÿ¼{7¹cPø€ŽÔï(-Ú†èüÕ8.{CRrÑéâ™I$“‡†<÷à&ÿÇ•=ç1aÇn§ÓáÚM– -ŸÒqC À¾)A8KÚEF ¢“‡'àÉHgñyš;OÕdÒIÛˆœ¦tò2*Bq¾ÇH‚16K‰+½tm™§ ¿ÓÒ¼=¶%ÆY”gO+Î+CGÖe—åÔæå˜KøUZżuyI^]vóºUEúÔìEóW–—òõkQ|ë#P#³f¦JO.óÿ-yâVÚëd{˜„Zÿi¢¾šOL¤?¾ë¾ ¾ÿ~Ät&¢¢WO‡¹q¤œƒ NÅq2‰ñ½íð‘Ö—ƒûŽ´¼Kœ“3ñ‚;ÿ­œx߀ ;¿˜_ŒC,›‹‚°©Tÿ©ÿ\ëÑ£hpÂwDF~) AmåZK3pS.{–=裮äʤ$Í2ì°w)Å!›c‹…œÐúxÞ«ÕzyÞ§%;¬Æ¬–‰³mÙÅö]ž¢@NvÀXÜ“Ó`wyÐs“¼¦`9;#’+x½Ò]æÜbA†¶Ðb´ò!dŸù´ŒÕFë¯g—ݘÝJÏ ïu²sŠŒŒWqÄW~CyÔkñ¾2Φ<¥e¤¢õzyóÜu…6TVÍkNºacň…©Ò\«Í5oì^«ó¶ZÇj.£unZñ?Ž–Ó´i)¢Ões _­‚l’2±¤=ç{pEx•  YÆ2åL?y¤3)KY”´„œ1­ö‚Êù¨@sÅ6«e[Uŵ%™•ÙeA©'qK×ùs3+7Zwú|ce–a—Í¥©«`¶†p¥ÓKb$Œ.Ï¢¯±Âx•>è·Ùš: sÉACθ4Ÿ;Ë mGøR´åÅb³›™ù”~š°IW@C¶„Ópe÷-qõóK]Mˆp·O‰f0ƒ:áÚ\ÚV§-jô f›*ïëÙéhEüãZÜ™‘è…zDÑù"{rOK^}µÍîóû7ÐL}—}¯¡º¶¾Fx èË5™‘Üž¡ßa ˜Í‘¢éy2®ÓÍöÓýtûd¿~â,ŬŸ†Ï —à×âÌä™3± ¤ˆF<ߢR)X2œ6}jêòiëLÌqXÊ—©é]“Qô"ê¢Gädä²è±ã”ßÿö ¥Ü—_¢Äå¾þ}*›‡ÎM,Á¶8£™“ÙÉtT¦à‰äÂ,SåV´äׯ+³yÅDÊŸ[Ö¥Hî*™†—¿Fxk²¦êÊ´Lca¾ .ï@f$ø>òÜu“ø7 ÙÈÀ4øÕ"úÌâ}hObÜJúñ¸õÅv›c´¢bÔá«È©Ñë¼9Z6§:gÚ·wgY٘ϷÓZ•(- ddJÍ Ñ —Ðe\͉r^é^EmÒZò=ôyqÄg wr«Z{fx v:.]Õ³Æÿˆ¯°…Šn»ýjžµ8³åÒiÒ™iºÒWD,f=b¨•\•ý…õ¦…w¡Üt -ïRµ[²Âß`Þ•ÁÞÝfÄÊ -•É  -ù\£¤l­®Ðd.*ÏÕ -ü}?W¬ÏÉWé³ -œz¢õÌJ+×®ÖeªÔªøùÊýëê@UšnMÊb•>ú¢..e¹Úí¶Ùü{Ñã<¿JYn­"fמ=,g´‡¿E¯®f§œLy]< ‘—”xÊû kMmhj/UOœ]·& M—”o¬C[# 9â×h5‹Ã+yŒ¯én›×Ùíº·±²Ùt¼Ià&vò>¹]ünÁ}_è'·—‹_·QªzöQ‚á°ø]‘=ÌßGŸ)>?}Æ·M8}–À2¸}ŽÙdaôY -ËH -˜¡ú` Ú¡†0Öi![,+À‹=­XW‰T}H³ ÅöötáØFqF lƺûÄñUâJC8cò16eáÚ"Åfh†LœÕ =ØËúÚðy3®Û‚sG°o&¯ÕW´«‘g;ÒwCé(¡*¡à*2Τd:䜷eXû?ÐOÑΤ˸‚.ò…¯ðÓìGWûPŒe6,Á] «Û‹Y¡D¤ÏOG~œ8¾ ¥u‹–áoç̤Åë¿‘iJ -endstream -endobj -280 0 obj -<< -/Length1 15596 -/Filter /FlateDecode -/Length 11081 ->> -stream -xœÝ{y|TEöoU»õíNoéNgíltÂ’°¤‰ì»À@Ø·°¯ -ˆÀ0À0!`Á„ JdÄ # b à„ :ÈHÜ‘aÄè8€ -IñNÝî°øßûãýñ>Ÿ×7uoÕ]NsªÎ9ßS÷†PBˆw‰ëÝ{Ġǧì}†®žîÓ³Woö!«Çv+lö:dxÏG.ÁözlÇõþ›ü¡SÂölçžš>çƒwÒ¡¢=rê¼É á_R[BàØÞ1õ‘ÅqW*T !æ¯aiÓΘ7–͹ˆÇáõ½3&/ZHTÜH—&l[fÌ]6ýÖ¾LˆõBÚ•Îœ6¹0¦ÜáGúíñzÖL.®Êr%6½#¼O¦3'>`VL IŒIþîþ†NïUHü$Ž¬T\ÜE·«óèg“½xûb—™-w -î¤sLž†aa©Ã¸Æìx¾Ë8<·KAà<-ÄrÛÓ±a¹e– Ašeâ|°¾˪Àýt‹ ÑRîáÁ¤_"h`9‰¥…¾hŸ -Þ'êïc¼w‰¸÷ž>ŹT,¢Ÿœ€æïþ°ÿ³Áã˜ÙƒG!ÃkXN`Y,'î9ÀrK#–*,ó‚ç7õ…G: Ayŵ€nÉÀ}ãñÂ=G1S‹ ÷Ô8JDÌQ/Î Vâ#íHÉ"ÝI?2”ŒÂ9:žL$ËÈ -²òöm1WI2I!™¤3éAaÁë“×oöëÛÿ˜‰¿ø½¢*²Œ’öqöýÌׯp¿ÿÁ‚¸7Gǧ´ÿE3ήÆí'C÷‡,‹«½}{h%Þ/G_ò%^úµ‹—RÚZWK=½zÉöšÔO/Àªhái<ß«§qMôº_öá_¿Iûã¦ÎŒ{ÌþXbîcöi¹)‡Øjê>°u¼Ü#šºI9ܯ&uŽõPcï$NÜ;ŒºÝØÛÈvÜ[zÈoúÈ=|4„¬Ä–ÕÔLÒq¯ôLÆ]PÕ¨+Æ=²Q—Œó`œaÆêÍsh^ Mnq¸™?ŸVÂ7Jå9üxLºq}´|£n¬–®_K’¯†ë~éZüç‡Tù?7á‡Tø7‡ï9|—W]ðm94"‹koŸóß–¾é__)”¿.‡+…ð‡/¿ˆ’¿äðE|ÎáòøŒÃ¿ŽÀ¥FÈ—nÂ?#àb9|Êá‡O>vËŸpøØ •Ã‡¸å9|°Ñ,à†÷WÂ{¹Ð€†\8ÏáÝèò»þ¡Ã9ïp8û˜C> ƒ·9¼UgJ|ò§9œZ orxƒÃëNn‘ë9œàpœÃkŽ!½c.8jº¿‘ë8üíÕñòߎÀßVK¯ñɯŽ‡WýÒæp¨jËzȯp¨ÁCÍMxiäðR!(„¿Za¿öqx‘û›á/örø³ª9¼°Ç*¿{¬ðün‡ü|kØí€?=—"ÿi%<—䰋óªvFÈU…°ó»¼3ž±ÃÓ:ìàðvò‡í!P¹­ƒ\Éa[¨Àþ+Ê¡üÉ#r9‡'qn=yž\-m}Ü'o[ýÒOpØŒíÍGàq”¡2ÊzÀ&”v“ 6š¡O”B *­Ä9`‡õŠ9¬+rÈë89àÖrø½#_þýpXÃaõ£°êw+åU~·Vzá·VXa9‡¥á°d±E^bƒ%µ”ø?’[`ñ1i‘ù¥‡9<Äa!‡ó‡Ë Êaþ¼Öòüá0¯5Ìå0'fs˜•3oÂŒ#0Ã4…¦NñÊS9L!vyŠ&s˜Äa"‡ cÌò+Œ/„qoÂXlŒuÁ3àŒ.pÁ(#9ü&*BþM:Œà0œÃ0®„¡†¸`0‡A4EÄaàÐú÷ —ûgC¿œr¿pèÛ+\îË¡¶úBolõ>½Â¡'žè™ ä;äœð@-óûMR~›œï€üZF°ÕÃo•{Ø G-=†-žEö[Á_KWc+Ïb’ó,WKýþB©;‡nÈB·›Ð•C—ÖË!œS;FÊ@6‡¬—œÅ!stJ‹”; € TNqAJ-ݶ·;äö¡Ð^°[.µkë“Ûqh‹w¶õA–+·áКC2‡$øÂòe_/heƒD 6›œÀ!>.EŽ_ q);¼Ø³—C ‡hÔm4‡(•¨ˆäÁ!œƒ)xzC˜;EË·Ë.»SÀe‡P¼/ÔN|ÞÉÁ’;òÁŽ=Ø`èÎfµÈ6غ³†è²ÕÖ€îBPw!:„ îJXÄÜÊ–Ìt”Dç` -Í*I+d -718¤È,(2@S€ØÖÒ¢´Ýÿ??òÿšÿËBÏZrÆ(Õt3§q¦˜­BäѲՒx÷ÕÒ3t=Œõݤ ÷kÉT‡7h6ÖêðÙ)Ï–‘Æ“eð%Y¯’w›}Œµ/ià³ô]O/"µ w{:lÀý -¨ƒKç‘çè‹Hqö¹€¬bxdÃòÛ"{›㶅¿‚Œ€qð|L‹¤Di)\!eŒÀ$2›œ•)SI™2.“&Û -![*M¢Õä -Òœ?a;9ÛaHLÈA6L,(Ïí0öe½b'oÃMÔûfÆi_©7äá•Ò@²ìÂ'“Q3„,€Lì}Y!o l¤·y#”#}C4ƒu#;ØtZ‚ÜÞ@m.€ž$ûˆ‘¯’"zù&êJ²Hnðóž_é‹ÒûèÜŽÒo˜‡²ÇqÛÍŽ°Ø'¤(i9noʉ¸•Ê_(ªÒQ¥<ªì›*©êÓê{hmµÚníœvÎÔÙ4ÂtPgz}9nÿ4O0¯2×›¹å±àögÜÞÂí’åRÈÈgBþemkýõ-Ú6¶ÞvÂÞÃ>ÏþG[ÇtÇ>ÇOÎ<çAçÕP_è—ÝUäºéºéžˆÛëÄȆŽ æM&ë·A±²Ž¬×$P)(D·ŸïÚ˜ž“Ó‘¤^njL£HG†;‘IgÜϺϜ‘šâOÑi͵gÕ0¨œˆÇUƒaˆn@Œ®‰)’¤ÙÏ7~Òؑد!Å4êH46Xõ⹑Î5QŒÙI¤·å:EbIZ´G-!S¯ÛTj3;Bb½1ÑQ‘áž°P§Ãn³†Xt“à‘Š+Î~þ-`·ëù®]›ºŠ}}zc:v¡áŽ7Jbh"–øÌx£ÐDÈ`{g^ŸU5ëÿrÄM~qfÕ¬³vÎ䜶ÁoÐÔ™T»>âB²Ét'Ÿ¼¨äSè3¢TÒÁÛéÎæ7ø”ºŽ¿+ÁY¢’J -ý9ZëVDm.ù\©vO(õ•´ÚÛºÌë* ÝDˬñ^»¤´ÒÚËĬD…ʭ㬨8¯Çjšý¼§k£'½©k£½éòU”çõôú«7®Ú¯¾~ãªÃ™ãÈq8=XrP‘YY™’’Šâv…¡XIIÉ™aaGRRf§¬ìÌ 6ÜxÒæv)*;8ÿ· ?¹dÞùB:ýÔ”Â3Sß<ýîÑB:d׈ß<;ôï\y•õäƒaqáøçèÓÒ # oÃðÕ¡ÍÁîùŒžŽ­)úˆõ›odoä_Ú¥JüÁØãÛ·ÿÝ„I·È‚¥ýè#Ì$³²¶$‡Œô§z³cZ'DÛ¢Àô„-á‰ì-­Ÿè°Ù¶žŽj‡#©èn3Ñ->½“9Vogɵןolj´_åWs„xb$¯¢Ü%wRç™]aéBêÄE픕‘æ1„KLH¢ÉÔãûõˬvÖàAsæ<«¸vQ‡釋××𾉒³M“gÖAsç4§¸¶Ø¿$%¥ãáuŵ4oðì9ƒÍ=¤øPZ‡ÔÅyE‡_¡‘“è®IŸ™…æÌŒRR–ôXw¨–ÊìPÍê p¾BÉz g¿„S_LüzcÚ³ºà„Ç»i5ÿ–(Nœ/h{Êò”5D%àTH¸nŧÒBva{ -s»œžÄ$–ÙÉ™ KŠÖ®-ªªØºµBq^æ]¾øœç~q•žøçEZ߈tÇ!Ý亪 «R³FÂ%§é65¦ß¥›æt»˜š˜åÌìÄè8ƒfRWœ¼ëÅK¼kãWôõÏ.Ó×ŪÉnÖ_j‡ÞßAûÛÛCÌ‘Šjð´ì°<­Sk¹S³è`R˜ç1a“̦ˆÅi8’O°xŒ©+Ô!ÎÔ‹ØN£²¬úP;r²Ï—-;2>©ßN§çðý‹øþ:oÏ¡CÑ¡ÒÅã'¦œáÅtÙ™)'ŽO=C—ñâ3ÈYyHZ&mÁÙ—ìw‘gM´J~V ‹”H´©+ÁÚö¨à`Ä»ã±Ktö*ý`ïI_Å|¥gµ¨TÓWQ“…ô"Ìckq<5¤ŠI”àpÖ *H ÐBfjþ‘­}ïëjŸÓŽx¯½†ìb4††>Õp‰ôJSíÈÏ"·ÓoçKÙÆ, ÷›Nê%Ì âö;s%ýÒ²º¦«G冟çáE·?“¢îÍÄC©î„?™œ^KEx¹·²•Û¥Ä“¨›7>¶,Ò±£ã°_¾†ÇóWÓü勒TšÊR!UJ•S•T5UK5¥ê©æ<’GóXäIyrž’§æiy¦<=Ï<„ ¡CØ}ˆy"™H'²‰úDóN²“îd;a§´SÞ©ìTwj;M;õæ}dÝÇöÁ>iŸ¼OÙ§îÓö™öéûÌGÉQz”…£ÒQù¨rT=ª5Õš{ÿ3Á®`¢4Qž¨LT'jM¢ã_#Ôj<O #XxhsØéX²±O4ÎÐÏeu:$§ó€¢%%7•”lúîúõï¾»v]Í:4»óàl‡Ÿâ§ù;4fÑlšVÅåkøïù£t=ý]E×ãÈÝ@/×GN'I~TH¬B^¯’ -“–¨xÇÅŒCXßd zú5„4_¼CÎôeˆÉF­4—¿H‡ž¦YMoVKKÖö¿ÙP-ìjRMı&úÛhŸ¬Èá‘à‰ò)Šœow<Rá*—H#vQÝëI°C«{ötìzÅ`w×ÞJ0¼@µËß:<9âàIàÞ3³HvV‹OÌÌB}I"( -]Â^lZr„†göÞ²zì› g¼1ùcj]ع¡ººú$íÐ}yÅ•›òx«cú•¿M:¶¸‡X&p.f#¿­Iä7´Â­—˜v‡T(±%q»£+Ë•J÷ mÂB ¸"¼Iv/$ĺL±m_dø|£Ãp85Åäa €ðÑŽ _Aÿ˜yבßa´-;ø7×g¼7cúëSv8°mûö’›×®›¹ìoý>¢òˆM~ãÉ¿“ÔêTf§ò¿¯Ü½|Þ¢­[Ž‹ûøåÏ îËûÃÒÍŸ ©1‘ª£Ä¾ÉUÂ*ÈúJµÚ ^âñR=Øc½BÍõAkºä˜×c -DðEÜ.r?ëÈñYv­¹¾Ý¨ö_Q;ÿìÇ¥';<ùù—<ÿàvœ?Õ|³ÝƯ~ÝÈ¿‹{;½ãþ]U|>D?-VNZ!w‰jlD…[a×ÿ,!,Ý$U„•Û+} ^’’ *Ñ44Ö'Œ½ércÀÖ‘½ÏíF´L£öÅä -Ÿ‡†@âÓÃh‚À -FábÞ¦AÇ^O«žÿÁ·×.ð¦h"uõßÊ/¬Ùºuͺõë僇}Éü"ÿ²pÿñ?? ZB7ÓåtclóÜÃÏ=wø¯yq?ú¦µÈõ`Ã7%"òyÀï ·Šd¥Â›RáDï”üBZ¸¥U[¯»•×fòº£Àk‹MCÅ6šü¶L ÑÊAîïaÒg€™€«ÁyÒ -Ï„ÞéÙÌâ-O­ßò$?½fóïœûaóšòœ_¾Ìoï´zÙòÕ«V,[ÍNV<öXeEé†m#â®:pîÜUããßÜyúòg§ªNÑ)þîw._½F¬6¯B‰zµ"]ü ¡ -©û³=¤Ä²É^‘ TD—'TúB ± &¯%)B Ãe„.-.çÈucZDiÀ˜NÄj8.Æ3âœbD—ŒX³e ª»ø’“©b4þó)•~àø7ý¶²òU7oÀá ñ4¢pÕ¯ý›šøc|/æ‹b‘ç@ü‘’ üåa»È^i—"Ó‰hFnâZ 9‚©©IJæçx†%¤°…_b? ‘ øü¡Ú.þHv){uH U¯!zÀí-!¦jHËPµM'·O Á—z–¯¡«à ~Ž‘y#F楈ŠãýNYB .ËÊ_TÊHk)YÅé›nàF„&榙TZÖô" »uQŠ‡×ø1~âºk7Ý%F¥€N’–Á® „©JaUD“i@Àút#.£Ó5, fá‚Ì­‹Ì²—åWÿ’EFŒ"K/Ò¿¨ -k-ÁM×Æ;Ü„fRÁŽA†Ák5|Ân>áÚ 9ñ¡ÇÞ‹>„a$ØæïA| P8Ÿ,©XŸ,K&èêÓA'>L€AÏ' VÐý°Þ$›4U1ò“œª›íŸ4T+ºÃkß -®þî©iߎN8 ¥ãý‘:±a¶[F$ f&E2©Š¶Žn`Úx2žB¼#0ãa3mþõ:=un~óõ¹çäÄf ^¼ÙŽñâÍÊn”ç!”GCl'T¸ûõ—7<ã -;[uF §´ËŽh—¤ƒ?2â%r ¬B^²°W˜Ê¡224ÝB:*i‘‚µ».­1 #ib x@OÐ5 -¤•-u´k,¿ÄOÐ<3v× þÕ#OÖן,x¡_f›6´œ.¤óie›6g»ù ¼ÍÿÎßñ‹°56,¨±Ö~·RaZO*l -±™TH·tT½ö4'²Ñ$0|@=hz¾ô,‡=)>Ñà%ÞAÞ>}ê6™THGÒžüU^ÍÏí¾EûÒ·n-•SùV¾È“ØïîÐË6 OjãSöÀôºBú«NeSGCV†Å’úú;ƒqÇD9 4`›šfG›–ŠÔ»ºù3 ç -²‰´ò‡)KEÃ2ÈvÒd2ÜK4ñ -,hǦ5?uFÐë[Ýœm ÛϤ¡8F1¤«?>"’„AdEô}ãäŠðZHVhº’é:ªăƒÅÏRÜÿ͈93㥡žy„_æõÔOc&ìú`õÄ“'¾þÀ´6Òí+ ¢t¸Ì¬³ƒúñ³8l§xCl<ݼ[à”·¯¡IDqZû«DÖë -Ê*w4!O èÑp˜]/c%-œp™¸OÜp†µ9s¦ù}±ú±ƒÞlÇÞhÎZ<…Tƒ*úNÝ©ýo6xU’m²ˆŒ.@T€µô;jÄ$þtãoÐœ¦4‡¿º¼µªºZZC‚¼ž2¨Vøã5ŸICp¨új2 uE¬(ô.ãöcW"’+ÓÜqš$œÆ@U“=Z’œ¤e±Þì7l”V`*dËåeÚ -“••T™(L7%Ëí•vj’– ý ÒG £ä¥@¥Íåð¨êA7ƒÊA7ƒÈA5”³ôÛæ#lä5±]hh&ÛÖüLÓFÖø\s-Jó>J³ÈXo -÷[ŒiF½Ð‘tRìFðL½е;þý3ì\S¡ÜDÈ#ñ¹eøœ…ùÛk ˜ -Àtʨ‰1š¯« ˆ†Ê0£[EŸ*ëŠ"¥ê -I ˜`‹BêA•ØåSª] -ùêƦS2ÞßJÖt“‡†ƒG 7%Ñ6¤µ1eÑÈÒ:›¬670é§Td(µ#q$"{F%j;ÂoTòë‡Qlýt³œÜô-„ÞüPø­`Þ`"óüÑû(Cø/Ø>ˆJUæ:ª’¦·¸Œ¿9Anœ"—*F0ÉÃÂÕ6,Ií¬Žb…l¶º„-WõpVzÓ~Ê(:ƒÎTD@>qÞâ.±„úÓ‰Û„k'ä†[ÒÛ7ÛIoßÊ@O¶íu\Kfé5UØ Â]n« 'é–,%Ý™n0ÕmÒhKV…`&ôž:l(Þ^Y\\¹½øÝ››oüØ„‰ñÚFò/ø+¼–A#h?¾œ¯Ælj=-¦«ùjì]xµ%† %íüá´ÂŽéÔz§]טDäôÎâ5e¸H#±¢÷I`ÂDÄ e_—>½c#HÞ¤Œß¾õõi9µùìÖ¢¢-»?ûø¿š÷Q&¾*ÀC1ßšíïŒtÅÈ>I‚|E"nÜ&LÖ›%ÙfR@‰¶‚‘.9:ºt¯%-&àKŽc'†L°&V©Zb¸0=«HÇF'`Šã1V觃iåŽì§Áó‡ùðÀ S_;C«nPåÙæw.lÞTñ8{͵`7ŸI—=;¾yƒÜðé?¶a£›¯þ~Ýš ÁxØÇ.™üÁß-ĬfŸ7Ö«™˜ªûbc½ùºÙ+Q÷K®çÃ+ô%ò¼Tá+wT¶öêæØ(•$EåX;ºÔœ„´Ö(fg—Ų›Æï„Î;&c ¤•AqhLLE]%*N_PˆÃðÓìz<ÿ3£KNH¾"y¸6îЄ²Ýóžœýcü§¦Yçç,z{feõ’Ísßy•†\Y'ïz»K×¢‡¦ÎL ïø—þ™’ò~¿žV,|$."µ®êÍ'£ì©8Ž»pUí·*g’‹ñ*W«ÍF~ÇÍ¢ÂS¥E<å o‡Þõfƒ,ÖßÈÔ^7#lã÷ØÀI$jª¯´Óõ^›×Ñ‘¡GõÞ™ûè³…*BÑž\bMQD"LªES¤ÒBÒ¤¤-g,¶¸e#G=o³œÎé_={Võ½rY7ØÖd™7>¢K×®]"ÆÍ…M3?-¯Knn—î''@¤mȉBúÛP:8âS1² ²4e>‘éA@Ôˆš¥Êi<{Éÿ=ÓMû¶:bò/ðÍ`3éÄs|Àu>à\µÜSCð(Ã,ä’¿'F“n’tèfÝ“ÙŒ6¡j*Ú„¬i*ó³ õK¾ŽG½­f1ë&-ðŒY%©ÁÅ÷¦®wüìý áÑp·ªð¾šðgá&1‰&ÄB¬«‹õ vºXŠê¬acd³ÇœÄÚ‚Oj+·ÖÚ™|æ$K&Ë…L)Wî¢åš:™³,#È:†À(i¤*=¸}p¿-ãÿrøÈÞIÛzä´kG;SnÛ¶­Gî?ß9{©Kw¡—ȇY²ô"Wd©ÕUªUZëèSà‘Ëõq8Í}…^Ā߯k‘ Dñ°{_¯À˜ÚÚ•…g®|}zÚvn[_TTRRT´α~nÜ8|킲ƒfâæ÷>úä|ÃǵðS€zq#?½üIˆÿ©^bÚ¨„ÕP¥4„¾QZR,Ún -SH¿h§=È[}à¥PÕe1í×®©Îù ~Ü-šÃT“L*húìè‹ÇæÍ:1–ßâѸïßû±VÚ¼¾h¯M£¼òfçœWPy94”Z¨Ÿzr× ûwˆñCÿ—Œz %PkvjÑjZJž²*Çuªbb®„ØÌ.±²€-ø4],{¤;…æÜbØÜwÞ"ˆ‰/%Ÿ›:ˆ®á«jym÷ÂKŠ«òÁSËšRá\ÙàÚ íð¾’3h¯}ýÉoT² œ%¡a¦RoX© ÐZ½¶­·MŽ Š¥âtÆ÷Ec­Žýòåú€¡ÚD¦”óËÙêbw£n7vÏÐz23`LñšâMeëW×^ýràîáSªxr}ûmóê¿þº~nyj-Ë9ûþûgÏ~ø¿ÐÄ›¢£j:´ßµ_[9aÍ¥*ÕhîÈ1›IPwQ†îÚûÃMV_ú•:ê,O!&WHÿ“bîívh˜a×F#væoÀ¬KP†{¢Í~hMImmÇ=‹þü«iîÏj*6½òçæbÅÕüܤ©—„Ö`=±? ÁÌ?B«!–zý)r\a5 4SEHèªërà=ÅàÕ$‹f¸¿Oç5ͧóø%[[+MjJ-+ƒ|ö€ñYôa>ìØOfõý12¡z©FkÀ^o} -Žk'œf™˜MR¬$ðR«¾>8eÎ6%/•\J`Ûxa^ÀOÐë/ý‰=MÊŸ—Ý*=ñʟ®m6¢t ÏvÒŸ«y9žGÁì¦yTÌid‚a—IÄÔï8y*˜û™¨¢ô$½,¨ïÆô–\ç²ç~L€RK6rAUDÞa -Ó´0.‡k™¬“œ¥õfÓÙöˆŒÎUÓ"Ð{„ËJ¤®µ†d¹µÚºHÙrŽš­õ‡~êhe´:fI³”Yê2xT^¦,S£‘6ÖÑÏþÉûP~‰OÙP§¸šVÐs|Lso–¿†ç¡ìU(»®Ä¢Æ“0÷  ×XNBŒ0‡(æ&ñ´é­ÀŠ‘ïQ¥i„šÙª±£ß¿ò§—ù‡ô"-ÿíïvœ???Žtç!Ýhg äþäp§Í$©$&JQÝ–Ò8¨‹:aW‰Ã¦ R;ÙG‡Šì•h¿6`¿eÄ€ýŽc ‘ÈÛÇ:nêj$dÆ;–®]/‹ÏD€ˆ?Íß'MJ“Ó”45MK3¥éiæ¼°©”:I©E “ÓÖœîîᆳ¿›OÞ³ÖïK;6³ôñÇK7>þøÆ+?ÞøúÊpñ£÷>þ¸á½vððáŒz—¦ÐXꥄíÖñ‘ÒìWÄÁ ô8Xg­¤'àx ÆÀ>F4¼! ÝÞ …¾;R±Á}˜aé¡C"~ùÕéé•ôûâ^(ÞØ|Jшâ¯ó¯œE¯!ƒàh&ÅJK Ä‚ÝA,u¶Ê¨Çc ¼Ò‘K÷»±ù^ŽîÅô~GŽÉîÀ⢢ ŠŠŠYdJå´SW¾: åÐ!ÖNp€±¸ù±á£i6Æç0š;jxÙÏ7™ÎÆ™°uä CpdT3TXëLÇU]Qˆ†>­ÞX¡Bο%&zšßg'vjwÄ‘8çH#i4Þæð?}Àîw %CéPûP‡sñÑé1»¥œƒ:<8|@ÿDk䎕»$&ÖefN½º#³ÆnYõì¾x”t-ê]“ŸA ìè·ÊfÄAk5ºf61œ”v§õ~¤Ñx­k½ñ¡FðK…û¼žƒM§»ø„ KÎ^:[- †ü ?^Ö\õÛ)[vŸa“Êh÷ >’&üÂê+uzÜ",¾?š¾aõ5É`~Û’à -܆I'jç,üýc‡u|᡽{èaöÂèÙ²[»öL.¼$Ʋ%œ'M vš¿ Æ YbÊdqÀ\ˆ(ÔCˆ’Ï€—Å{*KýH/ÕxÓbôþ+ìd&Öå¬b™AM’dæcL¬Àa€Áä¶ð(íE{å‹‘|ì½UÕ’Ù"/êð÷2cØÔMºX/4ëÈ– yR<"µõR[O µõÜ“ÚÿµÔöÚ¯§¶œ¶%Ž³¢Xj6IæXKcX$DKr´)J4ÇZ’HmÃ’ä6J5QkoÉaYrŽ’£fh]-½´~¦Þæþ–‘Zi”y´e:›Ó¥YòlÓ4ó#l¹ü¨ö[ÓÃæ›ÁDê!Çê±z;ÖNîκË]õ¾úh}6›%ÏÔW°U°LZ%-—«¯ÒÃQo-™0™p͉ª§w¨á7÷¿t`?*p)Ëo®ƒ ·ªØôæíbT¢&‹P“fò‚¿/bUA"©â K”Qð0ÔïÔ=&ŠƒYGÝš<¨Ù|]•¨¤áˆ³`õÓSê›êƒ8årºç¿Žü™ ª¡O/¢u:¥>B—©ŠJ‘Ý­çJô1ÒH]C M,Q`@¤ |­màµüP­å Þ¢mh²4©ù«æZúÏc}Y8ŸCŸ’ÎÄè±%u’*¯5DL33Yârxœˆ÷<6;‡=ÄbóX­!ùöÐb5•ª'Öã¡»-D¬‡95g?k¯P‘`¾u0l+ç¾EÁû%5Ö XÖÙi¼×“ìØ™æÐœ–ˆ$k’­³£Ÿ£Ÿstˆ.VÙ•‰'^f 9×wè35¡Ãcã¶Î{p\>äHûœ 3—å?å¼}B½4¸i+Ì’^àà¢ÍVdº?3Âæ6+>S¤Ýc–ãâ˜kL¤†¾nzÃ]ú²ÏbÒåVa$F—C™‹ÄEtÓm²ï®£r+ÁõìM¼ý­ÓÀH‹i²á·î~Òa,²§ÀKAbášÿçæ”vï¾qö ?û{—þfìüc~SZ÷øÖ ßU,.{¸üû [ÊFmúééMQ›vüT6ò>Ifû³c}Qa›evGØ$9HDM©I|=ê [ãe_tD¤Û&ÞGº‰Œuw³³dÈ’.V¬œ¿.M Í -H”æ ŠtW¦€wdJL¼ÐsãÈ1 æ¹±g÷Ÿ÷ÌÙè÷oœ³ççîuÚ±)*bÓÓ?m,(Ûráûò‡ËW|wa«pY*#þ»M2a½Îø7Q§$[:#ù4X‡{ÎK÷ÔeN¾ÖÄœ<@…dy˜Ì"3ÈL²ïnM¦bþGÒ Ft’µ)xGÉÇ{“EX&ÓÈdDÑíñl?2ïd.nqdØZ‹ŒÖ4> -stream -xœÍ9iXU×µkŸuö¹óĸÌ÷—A@à"z5¨5AAƒFQ™D©!JÕª1|mHâK ±j¬M õ™”DÛ¦Õ~‰Íàó‹¾41ÖÒøš¾|„gúÙ65px뜋ÆtzýÚï}öÙÃÚ{í5ì½ÖÚ÷€úˆà™3§taoÕó"QoäÜ¢Ùs„g¨zŽÚKæßWR´iÁF€(7µ=sK–Î*ÎZóµ§R{Ö}%éþ&ËE#Sæ/[ÓTÙŠå¢Û©½cͦvÏ©yß› {ŠÆì¨m­kZŸ·–à„¾SW¹¡4” ’à`¬kì¬ýøÐÆaÃæÏêk*«£·]¡¹©ŸROÆ. v5µãë›Ú7wå›ô5Y{c˚ʹC ^£ùê8ÓT¹¹ULdl·BseSMlô´$€ðK -Í­-ÚǯÀ´þƒ -¼µ­¦uÁâg²H8‡)Š¬„Uƒ—¼¿lZm)øÄhAyÞÿäà.¥üÅ¡Ÿ½ //JÝ4V«’¡<4OÓ$GŽ‹J“¾ ¾ü”«=å41G+à ,>ˆC¼Hšà<‹?IÍè`‰ïB­B J¢ ˆÁõ¾xŠkgWƒÒ SrÈv@ÓÄ>¬vmüÚ-õðO<âE–öÏÌÿs|àýætL”‡îê림âÀµj¢<¦Ì§¼’ò‰»à=å‰ øÊÏS®¢Ü„±¸‰¾Êý”‹ƒšWçMƒàEÊú2µ¡•J7éIïi0A1”B%ÔBÁ: s|œF܆”d AÖA›ÿð/¥?Ûgwžïk$.¢À Õc|óª‹Ë<¯/÷¦¥þIÓcÕx xÀÔé/.#øò9€>í€è‹úkÀ¡´ÔùÅežWgM`]QD}%eTUZÔMý³‹T˜²è÷Ñ;¯bÀ³¦Þó°õá¸ü‡­5ùi0(l ´ÿPÆÓ§ñ•Óøò-üþ-|éŹü¥Óøâ\<åÇëÓ~ü^5ôá 'ìü…uxÂŽÏÛñ;2ÿ¶‰¿…ß6á±çáÇd<öªøÜÑåü¹Gð¹mâÑ# üèr<$àa¿åÇCN|¶.VÆgÇ/ÆÅþgfòþ¹øÌL<ø´Ÿ\‡Oûñ)ÿõ¸…OÊøD>îÃ>7~CÆ‘q¿Œ½2öÔXyO>ºÏÁuá¾G¬|Ÿ±âÞÚh¾×qÏîÓ|Œ»w•óݧq÷6q××}|W9î -ˆ_÷áξ3 wXp{~MÆnÂ×íÇm[í|[5nµãC}ØU–à¸Å_¥_õc§7SÇæjìq“ŒíL|ƒ ÛÖOæm2®ŸŒ­míÂ767æÍ265–ó¦ÓØ´MllðñÆrl ˆëÖáZj¬= qÖÓœúj¬£¢®ki¥Úh¬±bµŒkd¬’±² +d\U+pð•2>àÀ~\ž—»°Ì…÷C¿_Æe~\*ci‰—Z±Ä€K»ø’>\\lå‹]XlÅûd\´°š/êÃ…Õ¸@Æù2Þ;ÏÎïíÃyvüÊ-œ{ çtáì";ŸÝ…Ev¼Ç3Í8ÃÓmXX`æ…2L3ð3N3`~^(Ïïü©žŠyÛÄ©œs§¸yn.N!⦸1'ÛÍsæcv–ƒg»1ËþL7÷Wc&3ݘ‘î¸xF¦»q2uMŽÆ´TO Á´A!Љ©)žjÃÔA¨•2ÉÌS,˜2È^¥Ö$IÏ'™qÒ Û¸_LžILœ ñnž>*|ó1žDïÆ8ÆF:x¬½žlîŽê÷dcL„ƒÇ„c4Ñ2FÑ(?F:0"ÜÎ#¨PΆN ³ñp;†i -ÓZx˜ ÚîÝ .BçêCg.:rÑNÞÃ~ Cüh³ú¹íZ jõ£Å\Æ-]h¦–¹ M„ÆFƒ…£Ñ`A=õk5F®µ 6È¡†8ÔQ£p¨%®ã’¥A8%ròA0QìCL£é‹Œp°h„dƒ¬zç>–òÿúÿõ£X‹c'Ø>x‰üE0°E°¶à‚$R£¬%edqðA‚£VB‡†g–{ Š`2eZu“h¤zŸhÄ!*wÂa| ·«Bð–ÐäÈ› -°|ß+pž€307™zÏ2Ë„ƒ,°Þ ‘éì2Ó³<¨Â·` Q{ÃO°ÃL‚ "ðËp†àá—ùpˆ¾—ù XÏ”X%J”š8â¬nÁ2Bßà dKð)öø¡T+ΊjÚB3F„•Âá ‰ì8\“´Þv8-ÄU¼…¹âø ‚þ6‘88Ž9|_0ÝZx‡_ -a‹B%QуEPH#½pAsH2³tÍV>B³7(N~ýw„ ®¿3=òWÓÛð6Ó©i+{æ¯%!‚Òjá;ˆ8]Më)½€ˆ+ÄÇÄw¸‡Òrþ‘ä•Žh¼”ÆÒØ<,\£˜Âú"‰‡h½ôf&Ð'ƒym^–&tŒí®ÉWINw‰¤+ôð½À4ŠD. -\Ôp:Ù\dz2 êu¨×Rè©gaµŽë´I&tš— é×G‡Cò2(ÐXùy ½Víyµ:Ñ ö,eT‹Ç -,CÈÐdh5À,‚ˆz1‡å ùšíö °õ§ØiÁ\å ½ŒRê÷²PùUÖÀ¾!¿·RÞ¸B~O|mìi\+…­—{¼/ŠeÄ«J‘&0AÒH>€P­åÓ bŒ&¢ôÖÑáÑa[^xÞíây%/=¹‹1V0Z¦“œlÛ#‰D‹ÝËlŠtmÞVÈú™(ÿT^%òËŸŸ o¥ˆ…ŸŸ%IR¬*n!´`‡Ô@(kµB«®:D+XõÀ½¦Xt‘"À¯R ÒpsøÒp³eù§äd'Äù¼Ñé½q´ÖùÎý7î—+Øc,‡YäOå7š‰í«ò½UUŸ}ð‡?|0vQá¼{üCqíûDh Lw9…PG"$š-V[7ÑM-Ñg§QÝÕâm ×wëZL­Rsx[R¢ÝñV]R¼+4ÆêŠÂØ$ëè9Îu[H8ëõ›Ã7‡­#¿±…NÈÉ|žª)5!!';>Ëï -µ%$ÄÅJN‡+4ÔåR8ˆMH´¹\A¶™|í ?¬{»fõ¹Šum%;¶ž:.ÿNþm˵úMW›6/z–ž>øÆoÒ'Ë¿¼gβâé‹33f½ôW×r²Yfé’5Ë爫Rx¥è_ì k <`”„V±š.PdZë¥s£çTqú3¼^›š„Qù=y§Eù&È2ihɪ”lŽ|°3®“ ,&Ôl” P*ÕÙb(J¬‹Ûck Ýi„¸ÈPD·Iç2LŽA—#94÷&)ÏO¼+ª»>!ëHI,øfÒ²£3b2<ÙÞB›³Íµv³Ýúng·«úY¿®_ÔqÔÙïìwõ‡ö»]åL•š"§ŸÍ›#Iq±$Ó)Þ,e;ÄJšœ)S²<.áÌᯭXؾÅÉWsŽVÿè²ÌôoÔU7üt]åë­ÌvU0—Ö.XñÂÓŽõcÝ]÷ÿìÅs§3çÎËÉ¿•”ð EnLjoŸØa09¶:]õhÚkì´Öëö`[¸Ýe„\)'\ÙšÃwë=ÃGyÀfïÝJ&rsE߽ǫä1ù–ÀpÅs%'—ô8гô›³òRRØTæ`N–7)U¾™;í¥cÇNåå+w¤DÇ4²ïVˆ„)(7s†×›õÚ6óÖŽ¡"¸¿ÍeHŽRªßO‚¾®P3!`õ ªrlÖÚ{Ì1±Íˆ6 eñòéÕ¼zéÝ×VôÈW¯nn^½º,”üqøheËfnÉ2ÊeË¡ÁÁC”ºV]]´/ qF§É±œ=&[=t¢éQ·*¦H§ Œ“Í®—”IÔ Ói -kØúëe¨ÀpÁxÁtÁ|Áy!‚—ß‘_–lN¯z"rQ•£F\T{i“ü‘ü.‹þÙ}LÇ©›l¬ &œF½¶z%¡G¿ADsqÐkÌÊ’ÊšŠÚ†ý¤··3ì´“–"›FùªPÄGÖ$±擯ˆ£ÝCCÂAäã0ÁÛ âÍ€W«#ëÍéjëÓr±Úõ’–î¹’Ž¹¤ÉcTô HÅœ_Ÿ0S·­L.„^Í{‚B… -¶F.c¯Êgä·ªå+’côë’WÍŒrémû2‡Ö6/ÂAËz°]Û£×t¢K³jeTa}ˆ²‰/Ú€YN²xl»ü®°hIÕÇò¹‹/±w°bèóíhú¹‚y aÞN;,¾Ht‡X(Ô†¨Iã4Ö{pODo˜U6K¤;?¼Ð¢Í— --ù¶8ë%ò¸¥ó ¥×µ©^—t(¼MMa¸à׿/ñÛòˆÅB³@®G¤°Aòhkábâýyâ=x·CF L‡Ìl&ÕÛö{õLpA®bbs&ü¢ê–o+4¶U¬8V^ûŽr¦«èL·Ðª…¸€]ªwB½e³Í­s…LF—1Ù­Z˜à ¶Žd|a‰³XðÀZ!.„ú†Í›ÖnÞ<ÖËÂ>aaòG#ŸÈá¾—yYÉò!fa…òYrÑŸÊgY!­Ý'/;hmÅÎåbî¶s{Ìml?öF‘ó«Ö.C±t~’ëw›:ßžClVËþ³WÈôÆOî›0u¬ê¶©;86(éUVÉoÊ¿‘¯Ëÿ^Î>½céÅ÷ nU-]rÀav N ê‚Ô'íñaÅè!—ÎÙ‚&7h¯¤»•*nÍè^X¾7Ám—¯0_ÊÉŽ‚ Ù˜’š:szBcóبXq¢¢!*RÑ>é!…VL„#B“Q0b¢µ:A£Ž‰öéô¢Þ#2ç^G§»ÞÆÈ7‰õ>òÀIÑ:CL„¦G¸Ì¹—#6ço†'¶/E(ÊÉ$/ƒò@Š<,^HÖ&ë’õɆdc²)ÙœjOw¤;S]É‘ÉQÉÑÉ1Éž$o|¢³œÝvs1‚Ê(ù®P&Q yàµÅ/”UUÿ‡¡´ÿ±¯×\jª<¿ªníü}‹øøÒꈅã“7µØŽ„o÷~÷»qq ró ò¦Ć$Ú}èT ɦ…´‘ÉÒÉÈ ¸¹™îá=`c½A“£XqkˆùËV|øfégâh8'l3¨–H8ɶÊÝó–ý~ô¤ü+ÍÊ?™?sèÙ÷qˆM§5ûÕX âOOc›žõ•“˜KGR=£Á}àÿâ4Úmê‚$‰;N8¡ŸùŠî»o¹bÉckèÿÛ%¼7VÚYÿÈCÂöÏû–×^¢5‹‰ÏnZS‚†€O¢.rF_‰Œ:]Pðq„^&Ñõ„qÈÄt ­ôZ_Ü.Ô«EðzAª-“”HÝ…|)(¾FÒ -XŠKÅå’V¹?(·»]ì–Ÿ’—ŸbÉÿ}C¬ø¼ŸÜM® þŽ*^$šŒð£@€(Ñéu¢r2èéò`0è%–.J\«¥K„`#£OOôjQ‚^ƒÖhÐë´ÁŸ\ H3è‚H;¶/9¡/îCÚów]‘ˆþzBÀ¡g1Ãb„ ×Eè# 1Æd– %j“ylŠ´N·ÆЛ5µÆ]°[óuíããð8{BzBÓ§}ÒMœÚ²ìv奫zoîìü¤c×Í7äwï’?&n -9coàŠÏû…9c¯ßÓä2ñ âÛ™Ä-XˆSä:‹Ù“ÑD5“ÙD #‰Ág2è¬`à=¸ßl赚MF½NBÐZDK¦!ÝJz:§º_ëíò¦då`*¬sÚ8q9J$”K:²{…צÙôÛŽ¥).yhð;–ÝypÿµÂ–q&™öwIUòÍL$2á‡êoíJ‡ZÁº@÷®_LÔ&ÑM?Xïªsp3ëD]‚–÷Ðl…Nhƒ¨ƒzh'ŒI°’©ôC¥,ªUÑÌ¢1í°rÔ@%4A*õ΃f?™j3¡‘’‡|ým\ÔV •54g}«i¤Š¨¶–0,ƒ4b ­$,uêHÕüÂÒLßVSExhœ‡æ·Ðº•*ìOñ”¨X iT3åY”iÎÿ6Îsgä2•Ò „¿E]ÝOôfAî—0Üž{vÚÙüeüçêÿ4é¿ -EKÂø¸:R…¿¿«Eýÿåönj©@ŒwfÈZ©/žƒLJŒhʢmEôCš`°€ƒED#ƒ¥Dƒ•”ØÄjzòÇÑÀŠæ,(%:D•Öñ‰u‚ðéI…'T¶UVAƚζFȨk«Y•-åÛÞLûâ˳Õç³kƾ -endstream -endobj -282 0 obj -<< -/Type /FontDescriptor -/FontName /FEVNJS+DejaVuSans -/FontFamily (DejaVu Sans) -/Flags 4 -/FontBBox [ 0 -14 472 560 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 560 -/StemV 80 -/StemH 80 -/FontFile2 278 0 R ->> -endobj -283 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /FEVNJS+DejaVuSans -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 317 ] 17 [ 317 ] 20 [ 636 636 ] 25 [ 636 ] 37 [ 686 698 ] 40 [ 631 ] 42 [ 774 751 294 ] 46 [ 655 ] 53 [ 694 634 ] 68 [ 612 ] 70 [ 549 634 615 ] 74 [ 634 633 277 ] 78 [ 579 ] 80 [ 974 633 611 ] 85 [ 411 520 392 633 591 817 ] ] -/FontDescriptor 282 0 R ->> -endobj -284 0 obj -<< -/Length 758 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -31 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0015> <0032> -<0003> <0020> -<002b> <0048> -<0044> <0061> -<0055> <0072> -<0047> <0064> -<0048> <0065> -<0051> <006e> -<004c> <0069> -<004a> <0067> -<002a> <0047> -<0058> <0075> -<005a> <0077> -<0057> <0074> -<004b> <0068> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0046> <0063> -<0050> <006d> -<004e> <006b> -<0052> <006f> -<0056> <0073> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -285 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /FEVNJS+DejaVuSans -/Encoding /Identity-H -/DescendantFonts [ 283 0 R ] -/ToUnicode 284 0 R ->> -endobj -286 0 obj -<< -/Type /FontDescriptor -/FontName /MIHDNY+Poppins -/FontFamily (Poppins) -/Flags 4 -/FontBBox [ -27 -170 413 547 ] -/ItalicAngle 0 -/Ascent 1049 -/Descent -349 -/CapHeight 547 -/StemV 80 -/StemH 80 -/FontFile2 279 0 R ->> -endobj -287 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /MIHDNY+Poppins -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 266 ] 5 [ 291 ] 10 [ 158 453 453 ] 14 [ 682 197 550 209 475 627 319 574 588 628 627 634 545 630 629 212 ] 36 [ 673 612 771 706 512 503 777 691 245 ] 46 [ 598 431 ] 49 [ 702 785 578 ] 53 [ 607 586 540 ] 57 [ 675 975 ] 60 [ 583 ] 68 [ 675 675 606 675 619 328 675 639 245 247 514 245 1029 639 639 675 675 372 521 363 639 560 819 478 562 454 ] 209 [ 411 ] ] -/FontDescriptor 286 0 R ->> -endobj -288 0 obj -<< -/Length 1276 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -68 beginbfchar -<0035> <0052> -<002e> <004b> -<0028> <0045> -<0015> <0032> -<0003> <0020> -<002b> <0048> -<0044> <0061> -<0055> <0072> -<0047> <0064> -<0048> <0065> -<0051> <006e> -<004c> <0069> -<004a> <0067> -<002a> <0047> -<0058> <0075> -<005a> <0077> -<0057> <0074> -<004b> <0068> -<0026> <0043> -<002c> <0049> -<0036> <0053> -<0059> <0076> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0025> <0042> -<0046> <0063> -<0050> <006d> -<004e> <006b> -<0016> <0033> -<0017> <0034> -<0018> <0035> -<001b> <0038> -<001c> <0039> -<0013> <0030> -<0032> <004f> -<0052> <006f> -<0056> <0073> -<0010> <002d> -<004f> <006c> -<0054> <0071> -<0053> <0070> -<0045> <0062> -<0024> <0041> -<0033> <0050> -<0049> <0066> -<0037> <0054> -<005c> <0079> -<000b> <0028> -<000c> <0029> -<000f> <002c> -<001d> <003a> -<0039> <0056> -<000e> <002b> -<0027> <0044> -<0029> <0046> -<0005> <0022> -<004d> <006a> -<00d1> <2022> -<005b> <0078> -<003c> <0059> -<000a> <0027> -<0012> <002f> -<003a> <0057> -<002f> <004c> -<0031> <004e> -<001a> <0037> -<005d> <007a> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -289 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /MIHDNY+Poppins -/Encoding /Identity-H -/DescendantFonts [ 287 0 R ] -/ToUnicode 288 0 R ->> -endobj -290 0 obj -<< -/Type /FontDescriptor -/FontName /BKSMOQ+DejaVuSansMono -/FontFamily (DejaVu Sans Mono) -/Flags 4 -/FontBBox [ 0 -14 502 728 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 728 -/StemV 80 -/StemH 80 -/FontFile2 280 0 R ->> -endobj -291 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /BKSMOQ+DejaVuSansMono -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 3 [ 602 602 602 602 602 602 ] 10 [ 602 602 602 602 601 ] 16 [ 601 601 601 601 601 601 602 ] 24 [ 601 601 ] 27 [ 602 601 602 602 602 602 602 ] 36 [ 602 602 602 602 602 602 602 602 602 ] 46 [ 602 602 602 601 602 601 ] 53 [ 602 601 602 601 602 602 ] 62 [ 602 ] 64 [ 602 ] 66 [ 602 ] 68 [ 601 601 601 601 601 601 602 602 601 602 601 601 602 601 601 601 602 601 601 601 601 602 601 601 601 ] 94 [ 602 ] 96 [ 602 ] ] -/FontDescriptor 290 0 R ->> -endobj -292 0 obj -<< -/Length 1416 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -78 beginbfchar -<0033> <0050> -<0052> <006f> -<0047> <0064> -<0036> <0053> -<0048> <0065> -<0046> <0063> -<0058> <0075> -<0055> <0072> -<004c> <0069> -<0057> <0074> -<005c> <0079> -<004f> <006c> -<0056> <0073> -<0031> <004e> -<005a> <0077> -<004e> <006b> -<0053> <0070> -<0049> <0066> -<0010> <002d> -<0014> <0031> -<0011> <002e> -<0019> <0036> -<0018> <0035> -<0051> <006e> -<0044> <0061> -<001d> <003a> -<0003> <0020> -<0026> <0043> -<004a> <0067> -<0050> <006d> -<004b> <0068> -<005b> <0078> -<0012> <002f> -<001c> <0039> -<0013> <0030> -<0045> <0062> -<0059> <0076> -<0042> <005f> -<0020> <003d> -<0038> <0055> -<0005> <0022> -<0030> <004d> -<0024> <0041> -<0037> <0054> -<0039> <0056> -<000e> <002b> -<0006> <0023> -<0004> <0021> -<0007> <0024> -<000b> <0028> -<004d> <006a> -<005e> <007b> -<003e> <005b> -<000d> <002a> -<0040> <005d> -<000a> <0027> -<0060> <007d> -<000c> <0029> -<001e> <003b> -<0015> <0032> -<001b> <0038> -<001f> <003c> -<0021> <003e> -<0028> <0045> -<0054> <0071> -<0016> <0033> -<002f> <004c> -<0035> <0052> -<002a> <0047> -<0025> <0042> -<0027> <0044> -<0032> <004f> -<0008> <0025> -<002b> <0048> -<003a> <0057> -<0029> <0046> -<002e> <004b> -<002c> <0049> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -293 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /BKSMOQ+DejaVuSansMono -/Encoding /Identity-H -/DescendantFonts [ 291 0 R ] -/ToUnicode 292 0 R ->> -endobj -294 0 obj -<< -/Type /FontDescriptor -/FontName /CBTLAY+DejaVuSansMono -/FontFamily (DejaVu Sans Mono) -/Flags 5 -/FontBBox [ 0 -207 558 728 ] -/ItalicAngle 0 -/Ascent 928 -/Descent -235 -/CapHeight 728 -/StemV 80 -/StemH 80 -/FontFile2 281 0 R ->> -endobj -295 0 obj -<< -/Type /Font -/Subtype /CIDFontType2 -/BaseFont /CBTLAY+DejaVuSansMono -/CIDSystemInfo << -/Registry (Adobe) -/Ordering (Identity) -/Supplement 0 ->> -/W [ 16 [ 601 ] 36 [ 601 ] 49 [ 601 ] 51 [ 601 ] 54 [ 601 601 ] 68 [ 601 ] 70 [ 601 601 601 601 ] 76 [ 601 ] 78 [ 601 601 601 601 601 601 ] 85 [ 601 601 601 601 601 601 ] 92 [ 601 ] ] -/FontDescriptor 294 0 R ->> -endobj -296 0 obj -<< -/Length 674 ->> -stream -/CIDInit /ProcSet findresource begin -12 dict begin -begincmap -/CIDSystemInfo -<< /Registry (Adobe) -/Ordering (UCS) -/Supplement 0 ->> def -/CMapName /Adobe-Identity-UCS def -/CMapType 2 def -1 begincodespacerange -<0000> -endcodespacerange -25 beginbfchar -<0053> <0070> -<0055> <0072> -<0052> <006f> -<0057> <0074> -<0048> <0065> -<0046> <0063> -<0010> <002d> -<004e> <006b> -<0051> <006e> -<004f> <006c> -<0047> <0064> -<0049> <0066> -<0044> <0061> -<0058> <0075> -<0056> <0073> -<0033> <0050> -<0036> <0053> -<004c> <0069> -<005c> <0079> -<0031> <004e> -<005a> <0077> -<0050> <006d> -<0059> <0076> -<0024> <0041> -<0037> <0054> -endbfchar -endcmap -CMapName currentdict /CMap defineresource pop -end -end -endstream -endobj -297 0 obj -<< -/Type /Font -/Subtype /Type0 -/BaseFont /CBTLAY+DejaVuSansMono -/Encoding /Identity-H -/DescendantFonts [ 295 0 R ] -/ToUnicode 296 0 R ->> -endobj -298 0 obj -<< -/FEVNJS 285 0 R -/MIHDNY 289 0 R -/BKSMOQ 293 0 R -/CBTLAY 297 0 R ->> -endobj -xref -0 299 -0000000000 65535 f -0000000015 00000 n -0000000176 00000 n -0000000297 00000 n -0000011596 00000 n -0000011729 00000 n -0000014387 00000 n -0000014586 00000 n -0000018015 00000 n -0000018337 00000 n -0000018471 00000 n -0000018607 00000 n -0000018757 00000 n -0000018908 00000 n -0000019051 00000 n -0000019195 00000 n -0000019353 00000 n -0000019512 00000 n -0000019669 00000 n -0000019827 00000 n -0000019966 00000 n -0000020106 00000 n -0000020279 00000 n -0000020453 00000 n -0000020590 00000 n -0000020728 00000 n -0000025444 00000 n -0000025734 00000 n -0000025989 00000 n -0000026125 00000 n -0000026276 00000 n -0000026420 00000 n -0000026579 00000 n -0000026737 00000 n -0000026877 00000 n -0000027051 00000 n -0000027189 00000 n -0000027456 00000 n -0000027722 00000 n -0000032771 00000 n -0000032972 00000 n -0000037891 00000 n -0000038146 00000 n -0000038279 00000 n -0000038410 00000 n -0000038541 00000 n -0000038672 00000 n -0000038803 00000 n -0000038934 00000 n -0000043741 00000 n -0000043989 00000 n -0000044122 00000 n -0000044253 00000 n -0000044384 00000 n -0000044515 00000 n -0000044648 00000 n -0000049806 00000 n -0000050026 00000 n -0000050158 00000 n -0000055590 00000 n -0000055880 00000 n -0000056013 00000 n -0000056144 00000 n -0000056275 00000 n -0000056406 00000 n -0000056537 00000 n -0000056670 00000 n -0000056801 00000 n -0000056932 00000 n -0000057063 00000 n -0000057194 00000 n -0000057325 00000 n -0000062131 00000 n -0000062393 00000 n -0000062526 00000 n -0000062657 00000 n -0000062788 00000 n -0000062919 00000 n -0000063050 00000 n -0000063181 00000 n -0000063404 00000 n -0000068141 00000 n -0000068368 00000 n -0000068501 00000 n -0000068631 00000 n -0000073579 00000 n -0000074082 00000 n -0000074213 00000 n -0000074344 00000 n -0000074475 00000 n -0000074606 00000 n -0000074737 00000 n -0000074868 00000 n -0000074999 00000 n -0000075131 00000 n -0000075263 00000 n -0000075395 00000 n -0000075527 00000 n -0000075659 00000 n -0000075791 00000 n -0000075923 00000 n -0000076056 00000 n -0000076189 00000 n -0000076322 00000 n -0000076455 00000 n -0000076588 00000 n -0000076721 00000 n -0000076854 00000 n -0000076987 00000 n -0000077120 00000 n -0000077253 00000 n -0000077386 00000 n -0000077519 00000 n -0000077652 00000 n -0000077785 00000 n -0000077918 00000 n -0000078051 00000 n -0000078184 00000 n -0000078317 00000 n -0000078450 00000 n -0000078583 00000 n -0000078716 00000 n -0000078849 00000 n -0000078981 00000 n -0000079112 00000 n -0000083562 00000 n -0000084081 00000 n -0000084214 00000 n -0000084347 00000 n -0000084480 00000 n -0000084613 00000 n -0000084746 00000 n -0000084879 00000 n -0000085012 00000 n -0000085145 00000 n -0000085278 00000 n -0000085411 00000 n -0000085544 00000 n -0000085677 00000 n -0000085810 00000 n -0000085943 00000 n -0000086076 00000 n -0000086209 00000 n -0000086342 00000 n -0000086475 00000 n -0000086608 00000 n -0000086741 00000 n -0000086874 00000 n -0000087007 00000 n -0000087140 00000 n -0000087273 00000 n -0000087406 00000 n -0000087539 00000 n -0000087672 00000 n -0000087805 00000 n -0000087938 00000 n -0000088071 00000 n -0000088204 00000 n -0000088337 00000 n -0000088470 00000 n -0000088603 00000 n -0000088736 00000 n -0000088869 00000 n -0000089001 00000 n -0000089132 00000 n -0000093641 00000 n -0000094160 00000 n -0000094293 00000 n -0000094426 00000 n -0000094559 00000 n -0000094692 00000 n -0000094825 00000 n -0000094958 00000 n -0000095091 00000 n -0000095224 00000 n -0000095357 00000 n -0000095490 00000 n -0000095623 00000 n -0000095756 00000 n -0000095889 00000 n -0000096022 00000 n -0000096155 00000 n -0000096288 00000 n -0000096421 00000 n -0000096554 00000 n -0000096687 00000 n -0000096820 00000 n -0000096953 00000 n -0000097087 00000 n -0000097221 00000 n -0000097355 00000 n -0000097489 00000 n -0000097623 00000 n -0000097757 00000 n -0000097891 00000 n -0000098025 00000 n -0000098159 00000 n -0000098293 00000 n -0000098427 00000 n -0000098561 00000 n -0000098695 00000 n -0000098829 00000 n -0000098963 00000 n -0000099096 00000 n -0000099228 00000 n -0000103644 00000 n -0000104163 00000 n -0000104297 00000 n -0000104431 00000 n -0000104565 00000 n -0000104699 00000 n -0000104833 00000 n -0000104967 00000 n -0000105101 00000 n -0000105235 00000 n -0000105369 00000 n -0000105503 00000 n -0000105637 00000 n -0000105771 00000 n -0000105905 00000 n -0000106039 00000 n -0000106173 00000 n -0000106307 00000 n -0000106441 00000 n -0000106575 00000 n -0000106709 00000 n -0000106843 00000 n -0000106977 00000 n -0000107111 00000 n -0000107245 00000 n -0000107379 00000 n -0000107513 00000 n -0000107647 00000 n -0000107781 00000 n -0000107915 00000 n -0000108049 00000 n -0000108183 00000 n -0000108317 00000 n -0000108451 00000 n -0000108585 00000 n -0000108719 00000 n -0000108853 00000 n -0000108987 00000 n -0000109120 00000 n -0000109252 00000 n -0000113043 00000 n -0000113354 00000 n -0000113488 00000 n -0000113622 00000 n -0000113756 00000 n -0000113890 00000 n -0000114024 00000 n -0000114158 00000 n -0000114292 00000 n -0000114426 00000 n -0000114560 00000 n -0000114694 00000 n -0000114828 00000 n -0000115096 00000 n -0000115272 00000 n -0000115397 00000 n -0000115580 00000 n -0000115734 00000 n -0000115885 00000 n -0000116061 00000 n -0000116199 00000 n -0000116336 00000 n -0000116527 00000 n -0000116663 00000 n -0000116809 00000 n -0000116959 00000 n -0000117120 00000 n -0000117292 00000 n -0000117423 00000 n -0000117567 00000 n -0000117697 00000 n -0000117874 00000 n -0000118002 00000 n -0000118064 00000 n -0000125460 00000 n -0000129765 00000 n -0000140937 00000 n -0000146172 00000 n -0000146404 00000 n -0000146824 00000 n -0000147635 00000 n -0000147785 00000 n -0000148014 00000 n -0000148559 00000 n -0000149889 00000 n -0000150036 00000 n -0000150277 00000 n -0000150882 00000 n -0000152352 00000 n -0000152506 00000 n -0000152748 00000 n -0000153121 00000 n -0000153848 00000 n -0000154002 00000 n -trailer -<< -/Size 299 -/Root 3 0 R -/Info 2 0 R ->> -startxref -154089 -%%EOF diff --git a/versioned_docs/version-2.7/backups/docker-installs/docker-installs.md b/versioned_docs/version-2.7/backups/docker-installs/docker-installs.md deleted file mode 100644 index cc211cee5e0..00000000000 --- a/versioned_docs/version-2.7/backups/docker-installs/docker-installs.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Backup and Restore for Rancher Installed with Docker ---- - -- [Backups](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) -- [Restores](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) \ No newline at end of file diff --git a/versioned_docs/version-2.7/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/versioned_docs/version-2.7/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md index bc5cc83d471..300e152bb22 100644 --- a/versioned_docs/version-2.7/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md +++ b/versioned_docs/version-2.7/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -109,20 +109,23 @@ This reference contains variables that you can use in `questions.yml` nested und | label | string | true | Define the UI label. | | description | string | false | Specify the description of the variable.| | type | string | false | Default to `string` if not specified (current supported types are string, multiline, boolean, int, enum, password, storageclass, hostname, pvc, and secret).| -| required | bool | false | Define if the variable is required or not (true \| false)| | default | string | false | Specify the default value. Only used if there is no corresponding value in the `values.yml` file. | | group | string | false | Group questions by input value. | -| min_length | int | false | Min character length.| -| max_length | int | false | Max character length.| -| min | int | false | Min integer length. | -| max | int | false | Max integer length. | | options | []string | false | Specify the options when the variable type is `enum`, for example: options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer"| -| valid_chars | string | false | Regular expression for input chars validation. | -| invalid_chars | string | false | Regular expression for invalid input chars validation.| | subquestions | []subquestion | false| Add an array of subquestions.| | show_if | string | false | Show current variable if conditional variable is true. For example `show_if: "serviceType=Nodeport"` | | show\_subquestion_if | string | false | Show subquestions if is true or equal to one of the options. for example `show_subquestion_if: "true"`| + + :::note `subquestions[]` cannot contain `subquestions` or `show_subquestions_if` keys, but all other keys in the above table are supported. diff --git a/versioned_docs/version-2.7/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md b/versioned_docs/version-2.7/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md index b65d7aa9b4b..f176acc177b 100644 --- a/versioned_docs/version-2.7/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md +++ b/versioned_docs/version-2.7/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md @@ -8,7 +8,6 @@ title: Creating an AKS Cluster You can use Rancher to create a cluster hosted in Microsoft Azure Kubernetes Service (AKS). - ## Prerequisites in Microsoft Azure :::caution @@ -65,6 +64,14 @@ az ad sp create-for-rbac \ --role Contributor ``` +### Setting Up the Service Principal with the Azure Command Line Tool + +Create the Resource Group by running this command: + +``` +az group create --location AZURE_LOCATION_NAME --resource-group AZURE_RESOURCE_GROUP_NAME +``` + ### Setting Up the Service Principal from the Azure Portal You can also follow these instructions to set up a service principal and give it role-based access from the Azure Portal. @@ -124,6 +131,17 @@ When provisioning an AKS cluster in the Rancher UI, RBAC is not configurable bec RBAC is required for AKS clusters that are registered or imported into Rancher. +### Setting Up the Role Assignment to Service Principal with the Azure Command Line Tool + +Assign the Rancher AKSv2 role to the service principal with the Azure Command Line Tool: + +``` +az role assignment create \ +--assignee CLIENT_ID \ +--scope "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME" \ +--role "Rancher AKSv2" +``` + ## AKS Cluster Configuration Reference For more information about how to configure AKS clusters from the Rancher UI, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) @@ -146,12 +164,114 @@ Please be aware that when registering an existing AKS cluster, the cluster might For more information about connecting to an AKS private cluster, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster) +## Setting Up the Minimum Permission Role with the Azure Command Line Tool + +1. Create the Minimum Rancher AKSv2 Permission Role by running this command: + + ``` + cat >> rancher-azure.json << EOF + + { + "Name": "Rancher AKSv2", + "IsCustom": true, + "Description": "Everything needed by Rancher AKSv2 operator", + "Actions": [ + "Microsoft.Compute/disks/delete", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.Compute/locations/DiskOperations/read", + "Microsoft.Compute/locations/vmSizes/read", + "Microsoft.Compute/locations/operations/read", + "Microsoft.Compute/proximityPlacementGroups/write", + "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/snapshots/write", + "Microsoft.Compute/virtualMachineScaleSets/manualUpgrade/action", + "Microsoft.Compute/virtualMachineScaleSets/delete", + "Microsoft.Compute/virtualMachineScaleSets/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/ipconfigurations/publicipaddresses/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/write", + "Microsoft.Compute/virtualMachineScaleSets/write", + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/virtualMachines/write", + "Microsoft.ContainerService/managedClusters/read", + "Microsoft.ContainerService/managedClusters/write" + "Microsoft.ContainerService/managedClusters/delete", + "Microsoft.ContainerService/managedClusters/accessProfiles/listCredential/action", + "Microsoft.ContainerService/managedClusters/agentPools/read", + "Microsoft.ContainerService/managedClusters/agentPools/write", + "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action", + "Microsoft.Network/applicationGateways/read", + "Microsoft.Network/applicationGateways/write", + "Microsoft.Network/loadBalancers/write", + "Microsoft.Network/loadBalancers/backendAddressPools/join/action", + "Microsoft.Network/loadBalancers/delete", + "Microsoft.Network/loadBalancers/read", + "Microsoft.Network/networkInterfaces/join/action", + "Microsoft.Network/networkInterfaces/read", + "Microsoft.Network/networkInterfaces/write", + "Microsoft.Network/networkSecurityGroups/read", + "Microsoft.Network/networkSecurityGroups/write", + "Microsoft.Network/publicIPAddresses/delete", + "Microsoft.Network/publicIPAddresses/join/action", + "Microsoft.Network/publicIPAddresses/read", + "Microsoft.Network/publicIPAddresses/write", + "Microsoft.Network/publicIPPrefixes/join/action", + "Microsoft.Network/privatednszones/*", + "Microsoft.Network/routeTables/read", + "Microsoft.Network/routeTables/routes/delete", + "Microsoft.Network/routeTables/routes/read", + "Microsoft.Network/routeTables/routes/write", + "Microsoft.Network/routeTables/write", + "Microsoft.Network/virtualNetworks/read", + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/read", + "Microsoft.Network/virtualNetworks/joinLoadBalancer/action", + "Microsoft.OperationalInsights/workspaces/sharedkeys/read", + "Microsoft.OperationalInsights/workspaces/read", + "Microsoft.OperationsManagement/solutions/write", + "Microsoft.OperationsManagement/solutions/read", + "Microsoft.Resources/subscriptions/resourcegroups/read", + "Microsoft.Resources/subscriptions/resourcegroups/write", + "Microsoft.Storage/operations/read", + "Microsoft.Storage/storageAccounts/listKeys/action", + "Microsoft.Storage/storageAccounts/delete", + "Microsoft.Storage/storageAccounts/read", + "Microsoft.Storage/storageAccounts/write" + ], + "NotActions": [], + "DataActions": [], + "NotDataActions": [], + "AssignableScopes": [ + "/subscriptions/SUBSCRIPTION_ID" + ] + } + EOF + ``` + +2. Apply the Rancher AKSv2 Role: + + ``` + az role definition create --role-definition rancher-azure.json + ``` + +3. Verify if the Rancher AKSv2 Role was created: + + ``` + az role definition list | grep "Rancher AKSv2" + ``` + ## Syncing The AKS provisioner can synchronize the state of an AKS cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) For information on configuring the refresh interval, see [this section.](../../../../pages-for-subheaders/gke-cluster-configuration.md#configuring-the-refresh-interval) + ## Programmatically Creating AKS Clusters -The most common way to programmatically deploy AKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) \ No newline at end of file +The most common way to programmatically deploy AKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster). diff --git a/versioned_docs/version-2.7/pages-for-subheaders/k3s-hardening-guide.md b/versioned_docs/version-2.7/pages-for-subheaders/k3s-hardening-guide.md index e44593ec790..285577fbca5 100644 --- a/versioned_docs/version-2.7/pages-for-subheaders/k3s-hardening-guide.md +++ b/versioned_docs/version-2.7/pages-for-subheaders/k3s-hardening-guide.md @@ -702,7 +702,6 @@ spec: - config: kubelet-arg: - make-iptables-util-chains=true # CIS 4.2.7 - protect-kernel-defaults: true # CIS 4.2.6 ``` @@ -734,6 +733,7 @@ spec: - config: kubelet-arg: - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 ``` diff --git a/versioned_docs/version-2.7/pages-for-subheaders/rancher-hardening-guides.md b/versioned_docs/version-2.7/pages-for-subheaders/rancher-hardening-guides.md index a7ffa3ce2e1..5db6c192945 100644 --- a/versioned_docs/version-2.7/pages-for-subheaders/rancher-hardening-guides.md +++ b/versioned_docs/version-2.7/pages-for-subheaders/rancher-hardening-guides.md @@ -27,25 +27,25 @@ Each self-assessment guide is accompanied by a hardening guide. These guides wer | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | |--------------------|-----------------------|-----------------------|------------------| | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke1-hardening-guide.md) | -| Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | -| Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke1-hardening-guide.md) | ### RKE2 Guides | Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | |------|--------------------|-----------------------|-----------------------|------------------| | Rancher provisioned RKE2 | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke2-hardening-guide.md) | -| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | -| Rancher provisioned RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](rke2-hardening-guide.md) | -| Standalone RKE2 | Kubernetes v1.25 | CIS v1.23 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | +| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke2-hardening-guide.md) | +| Standalone RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | ### K3s Guides | Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | |------|--------------------|-----------------------|-----------------------|------------------| | Rancher provisioned K3s cluster | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](k3s-hardening-guide.md) | -| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | -| Rancher provisioned K3s cluster | Kubernetes v1.25 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](k3s-hardening-guide.md) | | Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment) | [Link](https://docs.k3s.io/security/hardening-guide) | ## Rancher with SELinux diff --git a/versioned_docs/version-2.7/reference-guides/about-the-api/api-tokens.md b/versioned_docs/version-2.7/reference-guides/about-the-api/api-tokens.md index eccd4d962b1..5fe8a0eb5a4 100644 --- a/versioned_docs/version-2.7/reference-guides/about-the-api/api-tokens.md +++ b/versioned_docs/version-2.7/reference-guides/about-the-api/api-tokens.md @@ -3,7 +3,7 @@ title: API Tokens --- - + By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. diff --git a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/backup-configuration.md b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/backup-configuration.md index 2a792778e0a..e7ef2b056bf 100644 --- a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/backup-configuration.md +++ b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/backup-configuration.md @@ -3,7 +3,7 @@ title: Backup Configuration --- - + The Backup Create page lets you configure a schedule, enable encryption and specify the storage location for your backups. diff --git a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/examples.md b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/examples.md index 4336a204d98..bab7533302e 100644 --- a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/examples.md +++ b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + This section contains examples of Backup and Restore custom resources. diff --git a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/restore-configuration.md b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/restore-configuration.md index 5e958fcd674..b162a6ef873 100644 --- a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/restore-configuration.md +++ b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/restore-configuration.md @@ -3,7 +3,7 @@ title: Restore Configuration --- - + The Restore Create page lets you provide details of the backup to restore from diff --git a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/storage-configuration.md b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/storage-configuration.md index 3ad8b94487f..b1b8bad0896 100644 --- a/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/storage-configuration.md +++ b/versioned_docs/version-2.7/reference-guides/backup-restore-configuration/storage-configuration.md @@ -3,7 +3,7 @@ title: Backup Storage Location Configuration --- - + Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible object store. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md index 9fc13c9ce70..dc200ae62b1 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -3,7 +3,7 @@ title: Logging Best Practices --- - + In this guide, we recommend best practices for cluster-level logging and application logging. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md index 9255c76a28b..a6e34b03a50 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -3,7 +3,7 @@ title: Monitoring Best Practices --- - + Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. This is not different when using Kubernetes and Rancher. Fortunately the integrated monitoring and alerting functionality makes this whole process a lot easier. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md index 3012223f423..9d24fb3c5db 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -3,7 +3,7 @@ title: Best Practices for Rancher Managed vSphere Clusters --- - + This guide outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md index 1298ff18b87..0cec0a91864 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -3,7 +3,7 @@ title: Tips for Setting Up Containers --- - + Running well-built containers can greatly impact the overall performance and security of your environment. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md index c19ac243d98..74243559753 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -3,7 +3,7 @@ title: Installing Rancher in a vSphere Environment --- - + This guide outlines a reference architecture for installing Rancher on an RKE Kubernetes cluster in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md index cc8811dc77e..99583f75579 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -3,7 +3,7 @@ title: Rancher Deployment Strategy --- - + There are two recommended deployment strategies for a Rancher instance that manages downstream Kubernetes clusters. Each one has its own pros and cons. Read more about which one would fit best for your use case. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md index 1863ff82503..1ed05696fa2 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -3,7 +3,7 @@ title: Tips for Running Rancher --- - + This guide is geared toward use cases where Rancher is used to manage downstream Kubernetes clusters. The high-availability setup is intended to prevent losing access to downstream clusters if the Rancher server is not available. diff --git a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md index 691ef18f576..e8e919bde9b 100644 --- a/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md +++ b/versioned_docs/version-2.7/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md @@ -3,7 +3,7 @@ title: Tips for Scaling Rancher --- - + This guide aims to introduce the approaches that should be considered to scale Rancher setups, and associated challenges with doing so. As systems grow performance will naturally reduce, but there are steps we can take to minimize the load put on Rancher, as well as optimize Rancher's ability to handle these larger setups. diff --git a/versioned_docs/version-2.7/reference-guides/cli-with-rancher/kubectl-utility.md b/versioned_docs/version-2.7/reference-guides/cli-with-rancher/kubectl-utility.md index 45502753a08..fefa1a66b1b 100644 --- a/versioned_docs/version-2.7/reference-guides/cli-with-rancher/kubectl-utility.md +++ b/versioned_docs/version-2.7/reference-guides/cli-with-rancher/kubectl-utility.md @@ -3,7 +3,7 @@ title: kubectl Utility --- - + ## kubectl diff --git a/versioned_docs/version-2.7/reference-guides/cli-with-rancher/rancher-cli.md b/versioned_docs/version-2.7/reference-guides/cli-with-rancher/rancher-cli.md index 44024172658..0c9a27c1694 100644 --- a/versioned_docs/version-2.7/reference-guides/cli-with-rancher/rancher-cli.md +++ b/versioned_docs/version-2.7/reference-guides/cli-with-rancher/rancher-cli.md @@ -4,7 +4,7 @@ description: Interact with Rancher using command line interface (CLI) tools from --- - + The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md index 6043df8f1ba..4640d142697 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Machine Configuration Reference --- - + For more details about EC2 nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md index 98d299762b0..ba7e806295e 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Machine Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md index 1d62fe9d9d7..634c7f48561 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Machine Configuration --- - + For more details about DigitalOcean, Droplets, refer to the [official documentation](https://docs.digitalocean.com/products/compute/). diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md index ffe36cf7723..bd79dabbed5 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -3,7 +3,7 @@ title: EC2 Node Template Configuration --- - + For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md index e8fa8b382ee..e747dbbb547 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -3,7 +3,7 @@ title: Azure Node Template Configuration --- - + For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md index 0e679ae7d13..87b5fccdcfb 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -3,7 +3,7 @@ title: DigitalOcean Node Template Configuration --- - + Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md index cbecb1be01b..4e44d5ceca7 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -3,7 +3,7 @@ title: Nutanix Node Template Configuration --- - + ## Account Access diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md index 4ff462b1d1c..ace79a39ca7 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -3,7 +3,7 @@ title: VSphere Node Template Configuration --- - + ## Account Access diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md index 136c71a45a2..67cd280e470 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -3,7 +3,7 @@ title: AKS Cluster Configuration Reference --- - + ## Changes in Rancher v2.6 diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md index e830d5b4ffa..4a9bbe4f0ef 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -3,7 +3,7 @@ title: EKS Cluster Configuration Reference --- - + ### Account Access diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md index 60228c3fb97..120be47a641 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -3,7 +3,7 @@ title: Private Clusters --- - + In GKE, [private clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept) are clusters whose nodes are isolated from inbound and outbound traffic by assigning them internal IP addresses only. Private clusters in GKE have the option of exposing the control plane endpoint as a publicly accessible address or as a private address. This is different from other Kubernetes providers, which may refer to clusters with private control plane endpoints as "private clusters" but still allow traffic to and from nodes. You may want to create a cluster with private nodes, with or without a public control plane endpoint, depending on your organization's networking and security requirements. A GKE cluster provisioned from Rancher can use isolated nodes by selecting "Private Cluster" in the Cluster Options (under "Show advanced options"). The control plane endpoint can optionally be made private by selecting "Enable Private Endpoint". diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md index 0d03db56bb1..b0047715921 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -127,7 +127,7 @@ ACE is available on RKE, RKE2, and K3s clusters that are provisioned or register ::: -ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters.md) on RKE2 and K3s clusters. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. +ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) on RKE2 and K3s clusters. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md index 9ccc2cfe9f0..63fa8df3511 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -3,7 +3,7 @@ title: Syncing --- - + Syncing allows Rancher to update cluster values so that they're up to date with the corresponding cluster object hosted in AKS, EKS or GKE. This enables sources other than Rancher to own a hosted cluster’s state. diff --git a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md index a0e092fee87..3c363d053cf 100644 --- a/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md +++ b/versioned_docs/version-2.7/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -3,7 +3,7 @@ title: Rancher Agent Options --- - + Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. diff --git a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/examples.md b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/examples.md index 7c586657eb2..c65722cfa97 100644 --- a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/examples.md +++ b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/examples.md @@ -3,7 +3,7 @@ title: Examples --- - + ### ServiceMonitor diff --git a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/helm-chart-options.md index fabd9fc222f..112c41c9ea0 100644 --- a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/helm-chart-options.md +++ b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -3,7 +3,7 @@ title: Helm Chart Options --- - + ## Configuring Resource Limits and Requests diff --git a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/receivers.md b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/receivers.md index eb3a029b45b..dc459f34378 100644 --- a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/receivers.md +++ b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/receivers.md @@ -3,7 +3,7 @@ title: Receiver Configuration --- - + The [Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret contains the configuration of an Alertmanager instance that sends out notifications based on alerts it receives from Prometheus. diff --git a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/routes.md b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/routes.md index 31424a3815b..3f538d92112 100644 --- a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/routes.md +++ b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/routes.md @@ -3,7 +3,7 @@ title: Route Configuration --- - + The route configuration is the section of the Alertmanager custom resource that controls how the alerts fired by Prometheus are grouped and filtered before they reach the receiver. diff --git a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md index 780d13c2673..30fccb634c8 100644 --- a/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md +++ b/versioned_docs/version-2.7/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -3,7 +3,7 @@ title: ServiceMonitor and PodMonitor Configuration --- - + ServiceMonitors and PodMonitors are both pseudo-CRDs that map the scrape configuration of the Prometheus custom resource. diff --git a/versioned_docs/version-2.7/reference-guides/prometheus-federator/rbac.md b/versioned_docs/version-2.7/reference-guides/prometheus-federator/rbac.md index 47887415d85..8b54ce9559f 100644 --- a/versioned_docs/version-2.7/reference-guides/prometheus-federator/rbac.md +++ b/versioned_docs/version-2.7/reference-guides/prometheus-federator/rbac.md @@ -2,6 +2,10 @@ title: Role-Based Access Control --- + + + + This section describes the expectations for Role-Based Access Control (RBAC) for Prometheus Federator. As described in the section on [namespaces](../../pages-for-subheaders/prometheus-federator.md#namespaces), Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: diff --git a/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/architecture-recommendations.md index d68f5b9320c..1ece990ff8c 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/architecture-recommendations.md +++ b/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -2,6 +2,10 @@ title: Architecture Recommendations --- + + + + If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the node running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) ## Separation of Rancher and User Clusters diff --git a/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md index ddee7a7da62..8abfd0c9f6c 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md +++ b/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -2,6 +2,10 @@ title: Communicating with Downstream User Clusters --- + + + + This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. diff --git a/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/rancher-server-and-components.md index bb1cf170e7a..e9fec332622 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/rancher-server-and-components.md +++ b/versioned_docs/version-2.7/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -2,6 +2,10 @@ title: Rancher Server and Components --- + + + + The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..1d348069a2a --- /dev/null +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3208 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 539 1 0 Sep11 ? 00:00:09 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 45213 45195 3 Sep11 ? 00:45:14 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 47188 1 0 Sep11 ? 00:01:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 02b8fdd94b7628d575ace92f337dcc93202151d07f33858341cfcb0178fea586 -address /run/k3s/containerd/containerd.sock root 47235 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id eb0f03c4bc1125ae34e5de77c7eaeb9c2c1e46bec0bb106b99281df7353f9ded -address /run/k3s/containerd/containerd.sock root 47948 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5eece7110bd3876414f890ed1b7158eb91f60d1f14450a9f8c77a0fc62b73c4a -address /run/k3s/containerd/containerd.sock root 48047 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ef831f4bb8830a0a7b46a1e8be58197be00846ea0caa094014ea7ce3adf008e -address /run/k3s/containerd/containerd.sock root 48220 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id f8d5d3791ad2b1a8534476aa2fbdf8da1e4f0b9fdcdd26b1ae161142458a5336 -address /run/k3s/containerd/containerd.sock root 48878 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 41b93a6581e272b0852257b3d7c44ab4cadcdf465a3579a8b29b493ea75feaf7 -address /run/k3s/containerd/containerd.sock root 49870 1 0 Sep11 ? 00:00:30 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id be329cdccda4049f048c759a2b1915cd72526473ff7768e895bcfc153a66c125 -address /run/k3s/containerd/containerd.sock root 50271 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id cceb5d26eba7d7d3c474a2dc842b72495689f4e40055505ea2598e4ac6849d40 -address /run/k3s/containerd/containerd.sock root 50571 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3a26e691e0d0186165d523bef22b0af71f175c39df82845b32ce7b4e3d652d4a -address /run/k3s/containerd/containerd.sock root 96693 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9ed77e6e58410f1295f730ba4a8f8daf7aff8f4e896e24219a8fa710729c2287 -address /run/k3s/containerd/containerd.sock root 97726 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 1be4fe50e6bb29d4549f8b03102b5dc15b7669fe56b32beae3f701cf6d2e58cc -address /run/k3s/containerd/containerd.sock root 97923 1 1 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7c65c47376f36a8980a5bebad7ffdc1abe1853821cf6c4a1d45f1b6be3452062 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'600' is equal to '600' +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--event-qps' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..e7373288b35 --- /dev/null +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,3215 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 +Not Applicable. + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root +Not Applicable. + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd +Not Applicable. + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/k3s/server/tls + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... +Permissive - Enabling Pod Security Policy can cause applications to unexpectedly fail. + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log +Permissive. + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 +Permissive. + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 +Permissive. + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 +Permissive. + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s +Permissive. + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Not Applicable. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is present OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'RotateKubeletServerCertificate' is present OR 'RotateKubeletServerCertificate' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..5f3c20fb37f --- /dev/null +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3047 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 5 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 4 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:10 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..9fe9ca2045f --- /dev/null +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2863 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is greater or equal to 0 OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..ba80c0a516a --- /dev/null +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3202 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1017917 2489 83 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz root 1017945 2489 75 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/livez +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"akmMjUAq94YvsUftJpiA3b+9SClu0ESPBeckAI7KZBY="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Automated) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r ".spec.runAsUser.rule" || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r ".spec.runAsUser.rule" +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is equal to 'MustRunAsNonRoot' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is equal to 'ALL' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..b7d4a36ea4b --- /dev/null +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2967 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then find /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml -name '*etcd*' | xargs stat -c permissions=%a; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1018768 2419 99 16:17 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 +Not Applicable. + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= +Not Applicable. + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" +Not Applicable. + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= +Not Applicable. + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true +Not Applicable. + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not applicable. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present OR '{.eventRecordQPS}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.podPidsLimit}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md index bef0196f552..2450938a61e 100644 --- a/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md +++ b/versioned_docs/version-2.7/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md @@ -12,7 +12,7 @@ This guide corresponds to the following versions of Rancher, CIS Benchmarks, and | Rancher Version | CIS Benchmark Version | Kubernetes Version | |-----------------|-----------------------|--------------------| -| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 | This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. diff --git a/versioned_docs/version-2.8/cluster-provisioning/rke-clusters/options/options.md b/versioned_docs/version-2.8/cluster-provisioning/rke-clusters/options/options.md new file mode 100644 index 00000000000..da8df215bd2 --- /dev/null +++ b/versioned_docs/version-2.8/cluster-provisioning/rke-clusters/options/options.md @@ -0,0 +1,9 @@ +--- +title: RKE Cluster Configuration +--- + + + + + +This page has moved [here.](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) diff --git a/versioned_docs/version-2.8/contribute-to-rancher.md b/versioned_docs/version-2.8/contribute-to-rancher.md new file mode 100644 index 00000000000..b894ec64e66 --- /dev/null +++ b/versioned_docs/version-2.8/contribute-to-rancher.md @@ -0,0 +1,140 @@ +--- +title: Contributing to Rancher +--- + + + + + +Learn about the repositories used for Rancher and Rancher docs, how to build Rancher repositories, and what information to include when you file an issue. + +For more detailed information on how to contribute to the development of Rancher projects, refer to the [Rancher Developer Wiki](https://github.com/rancher/rancher/wiki). The wiki has resources on many topics, including the following: + +- How to set up the Rancher development environment and run tests +- The typical flow of an issue through the development lifecycle +- Coding guidelines and development best practices +- Debugging and troubleshooting +- Developing the Rancher API + +On the Rancher Users Slack, the channel for developers is **#developer**. + +## Rancher Docs + +If you have suggestions for the documentation on this website, [open](https://github.com/rancher/rancher-docs/issues/new/choose) an issue in the main [Rancher docs](https://github.com/rancher/rancher-docs) repository. This repo contains documentation for Rancher v2.0 and later. + +See the [Rancher docs README](https://github.com/rancher/rancher-docs#readme) for more details on contributing to and building the Rancher v2.x docs repo. + +For documentation describing Rancher v1.6 and earlier, see the [Rancher 1.x docs](https://github.com/rancher/rancher.github.io) repo, which contains source files for https://rancher.com/docs/rancher/v1.6/en/. + +## Rancher Repositories + +All of repositories are located within our main GitHub organization. There are many repositories used for Rancher, but we'll provide descriptions of some of the main ones used in Rancher. + +Repository | URL | Description +-----------|-----|------------- +Rancher | https://github.com/rancher/rancher | This repository is the main source code for Rancher 2.x. +Types | https://github.com/rancher/types | This repository is the repository that has all the API types for Rancher 2.x. +API Framework | https://github.com/rancher/norman | This repository is an API framework for building Rancher style APIs backed by Kubernetes Custom Resources. +User Interface | https://github.com/rancher/dashboard/ | This repository is the source of the Dashboard UI. +(Rancher) Docker Machine | https://github.com/rancher/machine | This repository is the source of the Docker Machine binary used when using Node Drivers. This is a fork of the `docker/machine` repository. +machine-package | https://github.com/rancher/machine-package | This repository is used to build the Rancher Docker Machine binary. +kontainer-engine | https://github.com/rancher/kontainer-engine | This repository is the source of kontainer-engine, the tool to provision hosted Kubernetes clusters. +RKE repository | https://github.com/rancher/rke | This repository is the source of Rancher Kubernetes Engine, the tool to provision Kubernetes clusters on any machine. +CLI | https://github.com/rancher/cli | This repository is the source code for the Rancher CLI used in Rancher 2.x. +(Rancher) Helm repository | https://github.com/rancher/helm | This repository is the source of the packaged Helm binary. This is a fork of the `helm/helm` repository. +Telemetry repository | https://github.com/rancher/telemetry | This repository is the source for the Telemetry binary. +loglevel repository | https://github.com/rancher/loglevel | This repository is the source of the loglevel binary, used to dynamically change log levels. + +To see all libraries/projects used in Rancher, see the [`go.mod` file](https://github.com/rancher/rancher/blob/master/go.mod) in the `rancher/rancher` repository. + +![Rancher diagram](/img/ranchercomponentsdiagram-2.6.svg)
+Rancher components used for provisioning/managing Kubernetes clusters. + +### Building Rancher Repositories + +Every repository should have a Makefile and can be built using the `make` command. The `make` targets are based on the scripts in the `/scripts` directory in the repository, and each target will use [Dapper](https://github.com/rancher/dapper) to run the target in an isolated environment. The `Dockerfile.dapper` will be used for this process, and includes all the necessary build tooling needed. + +The default target is `ci`, and will run `./scripts/validate`, `./scripts/build`, `./scripts/test` and `./scripts/package`. The resulting binaries of the build will be in `./build/bin` and are usually also packaged in a Docker image. + +### Rancher Bugs, Issues or Questions + +If you find any bugs or are having any trouble, please search the [reported issue](https://github.com/rancher/rancher/issues) as someone may have experienced the same issue or we are actively working on a solution. + +If you can't find anything related to your issue, contact us by [filing an issue](https://github.com/rancher/rancher/issues/new). Though we have many repositories related to Rancher, we want the bugs filed in the Rancher repository so we won't miss them! If you want to ask a question or ask fellow users about an use case, we suggest creating a post on the [Rancher Forums](https://forums.rancher.com). + +#### Checklist for Filing Issues + +Please follow this checklist when filing an issue which will helps us investigate and fix the issue. More info means more data we can use to determine what is causing the issue or what might be related to the issue. + +:::note + +For large amounts of data, please use [GitHub Gist](https://gist.github.com/) or similar and link the created resource in the issue. + +::: + +:::note Important: + +Please remove any sensitive data as it will be publicly viewable. + +::: + +- **Resources:** Provide as much as detail as possible on the used resources. As the source of the issue can be many things, including as much of detail as possible helps to determine the root cause. See some examples below: + - **Hosts:** What specifications does the host have, like CPU/memory/disk, what cloud does it happen on, what Amazon Machine Image are you using, what DigitalOcean droplet are you using, what image are you provisioning that we can rebuild or use when we try to reproduce + - **Operating System:** What operating system are you using? Providing specifics helps here like the output of `cat /etc/os-release` for exact OS release and `uname -r` for exact kernel used + - **Docker:** What Docker version are you using, how did you install it? Most of the details of Docker can be found by supplying output of `docker version` and `docker info` + - **Environment:** Are you in a proxy environment, are you using recognized CA/self signed certificates, are you using an external loadbalancer + - **Rancher:** What version of Rancher are you using, this can be found on the bottom left of the UI or be retrieved from the image tag you are running on the host + - **Clusters:** What kind of cluster did you create, how did you create it, what did you specify when you were creating it +- **Steps to reproduce the issue:** Provide as much detail on how you got into the reported situation. This helps the person to reproduce the situation you are in. + - Provide manual steps or automation scripts used to get from a newly created setup to the situation you reported. +- **Logs:** Provide data/logs from the used resources. + - Rancher + - Docker install + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher:|rancher/rancher " | awk '{ print $1 }') + ``` + - Kubernetes install using `kubectl` + + :::note + + Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` if Rancher is installed on a Kubernetes cluster) or are using the embedded kubectl via the UI. + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + -l app=rancher \ + --timestamps=true + ``` + - Docker install using `docker` on each of the nodes in the RKE cluster + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher@|rancher_rancher" | awk '{ print $1 }') + ``` + - Kubernetes Install with RKE Add-On + + :::note + + Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` if the Rancher server is installed on a Kubernetes cluster) or are using the embedded kubectl via the UI. + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + --timestamps=true \ + -f $(kubectl --kubeconfig $KUBECONFIG get pods -n cattle-system -o json | jq -r '.items[] | select(.spec.containers[].name="cattle-server") | .metadata.name') + ``` + - System logging (these might not all exist, depending on operating system) + - `/var/log/messages` + - `/var/log/syslog` + - `/var/log/kern.log` + - Docker daemon logging (these might not all exist, depending on operating system) + - `/var/log/docker.log` +- **Metrics:** If you are experiencing performance issues, please provide as much of data (files or screenshots) of metrics which can help determining what is going on. If you have an issue related to a machine, it helps to supply output of `top`, `free -m`, `df` which shows processes/memory/disk usage. diff --git a/versioned_docs/version-2.8/faq/container-network-interface-providers.md b/versioned_docs/version-2.8/faq/container-network-interface-providers.md new file mode 100644 index 00000000000..490713c7b5a --- /dev/null +++ b/versioned_docs/version-2.8/faq/container-network-interface-providers.md @@ -0,0 +1,208 @@ +--- +title: Container Network Interface (CNI) Providers +description: Learn about Container Network Interface (CNI), the CNI providers Rancher provides, the features they offer, and how to choose a provider for you +--- + + + + + +## What is CNI? + +CNI (Container Network Interface), a [Cloud Native Computing Foundation project](https://cncf.io/), consists of a specification and libraries for writing plugins to configure network interfaces in Linux containers, along with a number of plugins. CNI concerns itself only with network connectivity of containers and removing allocated resources when the container is deleted. + +Kubernetes uses CNI as an interface between network providers and Kubernetes pod networking. + +![CNI Logo](/img/cni-logo.png) + +For more information visit [CNI GitHub project](https://github.com/containernetworking/cni). + +## What Network Models are Used in CNI? + +CNI network providers implement their network fabric using either an encapsulated network model such as Virtual Extensible Lan ([VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)) or an unencapsulated network model such as Border Gateway Protocol ([BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol)). + +### What is an Encapsulated Network? + +This network model provides a logical Layer 2 (L2) network encapsulated over the existing Layer 3 (L3) network topology that spans the Kubernetes cluster nodes. With this model you have an isolated L2 network for containers without needing routing distribution, all at the cost of minimal overhead in terms of processing and increased IP package size, which comes from an IP header generated by overlay encapsulation. Encapsulation information is distributed by UDP ports between Kubernetes workers, interchanging network control plane information about how MAC addresses can be reached. Common encapsulation used in this kind of network model is VXLAN, Internet Protocol Security (IPSec), and IP-in-IP. + +In simple terms, this network model generates a kind of network bridge extended between Kubernetes workers, where pods are connected. + +This network model is used when an extended L2 bridge is preferred. This network model is sensitive to L3 network latencies of the Kubernetes workers. If datacenters are in distinct geolocations, be sure to have low latencies between them to avoid eventual network segmentation. + +CNI network providers using this network model include Flannel, Canal, Weave, and Cilium. By default, Calico is not using this model, but it can be configured to do so. + +![Encapsulated Network](/img/encapsulated-network.png) + +### What is an Unencapsulated Network? + +This network model provides an L3 network to route packets between containers. This model doesn't generate an isolated l2 network, nor generates overhead. These benefits come at the cost of Kubernetes workers having to manage any route distribution that's needed. Instead of using IP headers for encapsulation, this network model uses a network protocol between Kubernetes workers to distribute routing information to reach pods, such as [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol). + +In simple terms, this network model generates a kind of network router extended between Kubernetes workers, which provides information about how to reach pods. + +This network model is used when a routed L3 network is preferred. This mode dynamically updates routes at the OS level for Kubernetes workers. It's less sensitive to latency. + +CNI network providers using this network model include Calico and Cilium. Cilium may be configured with this model although it is not the default mode. + +![Unencapsulated Network](/img/unencapsulated-network.png) + +## What CNI Providers are Provided by Rancher? + +### RKE Kubernetes clusters + +Out-of-the-box, Rancher provides the following CNI network providers for RKE Kubernetes clusters: Canal, Flannel, Calico, and Weave. + +You can choose your CNI network provider when you create new Kubernetes clusters from Rancher. + +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal is a CNI network provider that gives you the best of Flannel and Calico. It allows users to easily deploy Calico and Flannel networking together as a unified networking solution, combining Calico’s network policy enforcement with the rich superset of Calico (unencapsulated) and/or Flannel (encapsulated) network connectivity options. + +In Rancher, Canal is the default CNI network provider combined with Flannel and VXLAN encapsulation. + +Kubernetes workers should open UDP port `8472` (VXLAN) and TCP port `9099` (health checks). If using Wireguard, you should open UDP ports `51820` and `51821`. For more details, refer to [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md). + +![](/img/canal-diagram.png) + +For more information, see the [Canal GitHub Page.](https://github.com/projectcalico/canal) + +#### Flannel + +![Flannel Logo](/img/flannel-logo.png) + +Flannel is a simple and easy way to configure L3 network fabric designed for Kubernetes. Flannel runs a single binary agent named flanneld on each host, which is responsible for allocating a subnet lease to each host out of a larger, preconfigured address space. Flannel uses either the Kubernetes API or etcd directly to store the network configuration, the allocated subnets, and any auxiliary data (such as the host's public IP). Packets are forwarded using one of several backend mechanisms, with the default encapsulation being [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan). + +Encapsulated traffic is unencrypted by default. Flannel provides two solutions for encryption: + +* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec), which makes use of [strongSwan](https://www.strongswan.org/) to establish encrypted IPSec tunnels between Kubernetes workers. It is an experimental backend for encryption. +* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard), which is a more faster-performing alternative to strongSwan. + +Kubernetes workers should open UDP port `8472` (VXLAN). See [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +![Flannel Diagram](/img/flannel-diagram.png) + +For more information, see the [Flannel GitHub Page](https://github.com/flannel-io/flannel). + +#### Weave + +![Weave Logo](/img/weave-logo.png) + +Weave enables networking and network policy in Kubernetes clusters across the cloud. Additionally, it support encrypting traffic between the peers. + +Kubernetes workers should open TCP port `6783` (control port), UDP port `6783` and UDP port `6784` (data ports). See the [port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +For more information, see the following pages: + +- [Weave Net Official Site](https://www.weave.works/) + +### RKE2 Kubernetes clusters + +Out-of-the-box, Rancher provides the following CNI network providers for RKE2 Kubernetes clusters: [Canal](#canal) (see above section), Calico, and Cilium. + +You can choose your CNI network provider when you create new Kubernetes clusters from Rancher. + +#### Calico + +![Calico Logo](/img/calico-logo.png) + +Calico enables networking and network policy in Kubernetes clusters across the cloud. By default, Calico uses a pure, unencapsulated IP network fabric and policy engine to provide networking for your Kubernetes workloads. Workloads are able to communicate over both cloud infrastructure and on-prem using BGP. + +Calico also provides a stateless IP-in-IP or VXLAN encapsulation mode that can be used, if necessary. Calico also offers policy isolation, allowing you to secure and govern your Kubernetes workloads using advanced ingress and egress policies. + +Kubernetes workers should open TCP port `179` if using BGP or UDP port `4789` if using VXLAN encapsulation. In addition, TCP port `5473` is needed when using Typha. See [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +:::note Important: + +In Rancher v2.6.3, Calico probes fail on Windows nodes upon RKE2 installation. Note that this issue is resolved in v2.6.4. + +- To work around this issue, first navigate to `https:///v3/settings/windows-rke2-install-script`. + +- There, change the current setting: `https://raw.githubusercontent.com/rancher/wins/v0.1.3/install.ps1` to this new setting: `https://raw.githubusercontent.com/rancher/rke2/master/windows/rke2-install.ps1`. + +::: + +![Calico Diagram](/img/calico-diagram.svg) + +For more information, see the following pages: + +- [Project Calico Official Site](https://www.projectcalico.org/) +- [Project Calico GitHub Page](https://github.com/projectcalico/calico) + +#### Cilium + +![Cilium Logo](/img/cilium-logo.png) + +Cilium enables networking and network policies (L3, L4, and L7) in Kubernetes. By default, Cilium uses eBPF technologies to route packets inside the node and VXLAN to send packets to other nodes. Unencapsulated techniques can also be configured. + +Cilium recommends kernel versions greater than 5.2 to be able to leverage the full potential of eBPF. Kubernetes workers should open TCP port `8472` for VXLAN and TCP port `4240` for health checks. In addition, ICMP 8/0 must be enabled for health checks. For more information, check [Cilium System Requirements](https://docs.cilium.io/en/latest/operations/system_requirements/#firewall-requirements). + +##### Ingress Routing Across Nodes in Cilium +
+By default, Cilium does not allow pods to contact pods on other nodes. To work around this, enable the ingress controller to route requests across nodes with a `CiliumNetworkPolicy`. + +After selecting the Cilium CNI and enabling Project Network Isolation for your new cluster, configure as follows: + +``` +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: hn-nodes + namespace: default +spec: + endpointSelector: {} + ingress: + - fromEntities: + - remote-node +``` + +## CNI Features by Provider + +The following table summarizes the different features available for each CNI network provider provided by Rancher. + +| Provider | Network Model | Route Distribution | Network Policies | Mesh | External Datastore | Encryption | Ingress/Egress Policies | +| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Canal | Encapsulated (VXLAN) | No | Yes | No | K8s API | Yes | Yes | +| Flannel | Encapsulated (VXLAN) | No | No | No | K8s API | Yes | No | +| Calico | Encapsulated (VXLAN,IPIP) OR Unencapsulated | Yes | Yes | Yes | Etcd and K8s API | Yes | Yes | +| Weave | Encapsulated | Yes | Yes | Yes | No | Yes | Yes | +| Cilium | Encapsulated (VXLAN) | Yes | Yes | Yes | Etcd and K8s API | Yes | Yes | + +- Network Model: Encapsulated or unencapsulated. For more information, see [What Network Models are Used in CNI?](#what-network-models-are-used-in-cni) + +- Route Distribution: An exterior gateway protocol designed to exchange routing and reachability information on the Internet. BGP can assist with pod-to-pod networking between clusters. This feature is a must on unencapsulated CNI network providers, and it is typically done by BGP. If you plan to build clusters split across network segments, route distribution is a feature that's nice-to-have. + +- Network Policies: Kubernetes offers functionality to enforce rules about which services can communicate with each other using network policies. This feature is stable as of Kubernetes v1.7 and is ready to use with certain networking plugins. + +- Mesh: This feature allows service-to-service networking communication between distinct Kubernetes clusters. + +- External Datastore: CNI network providers with this feature need an external datastore for its data. + +- Encryption: This feature allows cyphered and secure network control and data planes. + +- Ingress/Egress Policies: This feature allows you to manage routing control for both Kubernetes and non-Kubernetes communications. + + +## CNI Community Popularity + +The following table summarizes different GitHub metrics to give you an idea of each project's popularity and activity. This data was collected in January 2022. + +| Provider | Project | Stars | Forks | Contributors | +| ---- | ---- | ---- | ---- | ---- | +| Canal | https://github.com/projectcalico/canal | 679 | 100 | 21 | +| Flannel | https://github.com/flannel-io/flannel | 7k | 2.5k | 185 | +| Calico | https://github.com/projectcalico/calico | 3.1k | 741 | 224 | +| Weave | https://github.com/weaveworks/weave/ | 6.2k | 635 | 84 | +| Cilium | https://github.com/cilium/cilium | 10.6k | 1.3k | 352 | + +
+ +## Which CNI Provider Should I Use? + +It depends on your project needs. There are many different providers, which each have various features and options. There isn't one provider that meets everyone's needs. + +Canal is the default CNI network provider. We recommend it for most use cases. It provides encapsulated networking for containers with Flannel, while adding Calico network policies that can provide project/namespace isolation in terms of networking. + +## How can I configure a CNI network provider? + +Please see [Cluster Options](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) on how to configure a network provider for your cluster. For more advanced configuration options, please see how to configure your cluster using a [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) and the options for [Network Plug-ins](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/). diff --git a/versioned_docs/version-2.8/faq/deprecated-features-in-v2.5.md b/versioned_docs/version-2.8/faq/deprecated-features-in-v2.5.md new file mode 100644 index 00000000000..86313d48238 --- /dev/null +++ b/versioned_docs/version-2.8/faq/deprecated-features-in-v2.5.md @@ -0,0 +1,30 @@ +--- +title: Deprecated Features in Rancher +--- + + + + + +### What is Rancher's Deprecation policy? + +We have published our official deprecation policy in the support [terms of service](https://rancher.com/support-maintenance-terms). + +### Where can I find out which features have been deprecated in Rancher? + +Rancher will publish deprecated features as part of the [release notes](https://github.com/rancher/rancher/releases) for Rancher found on GitHub. Please consult the following patch releases for deprecated features: + +| Patch Version | Release Date | +|---------------|---------------| +| [2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) | Aug 31, 2021 | +| [2.6.1](https://github.com/rancher/rancher/releases/tag/v2.6.1) | Oct 11, 2021 | +| [2.6.2](https://github.com/rancher/rancher/releases/tag/v2.6.2) | Oct 19, 2021 | +| [2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) | Dec 21, 2021 | +| [2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | Mar 31, 2022 | +| [2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) | May 12, 2022 | +| [2.6.6](https://github.com/rancher/rancher/releases/tag/v2.6.6) | Jun 30, 2022 | + + +### What can I expect when a feature is marked for deprecation? + +In the release where functionality is marked as "Deprecated", it will still be available and supported allowing upgrades to follow the usual procedure. Once upgraded, users/admins should start planning to move away from the deprecated functionality before upgrading to the release it marked as removed. The recommendation for new deployments is to not use the deprecated feature. \ No newline at end of file diff --git a/versioned_docs/version-2.8/faq/dockershim.md b/versioned_docs/version-2.8/faq/dockershim.md new file mode 100644 index 00000000000..23b982f7b30 --- /dev/null +++ b/versioned_docs/version-2.8/faq/dockershim.md @@ -0,0 +1,49 @@ +--- +title: Dockershim +--- + + + + + +The Dockershim is the CRI compliant layer between the Kubelet and the Docker daemon. As part of the Kubernetes 1.20 release, the [deprecation of the in-tree Dockershim was announced](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/). Removal is currently scheduled for Kubernetes 1.24. For more information on the deprecation and its timelines, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). + +RKE clusters, starting with Kubernetes 1.21, now support the external Dockershim to continue leveraging Docker as the CRI runtime. We now implement the upstream open source community Dockershim announced by [Mirantis and Docker](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) to ensure RKE clusters can continue to leverage Docker. + +To enable the external Dockershim, configure the following option. + +``` +enable_cri_dockerd: true +``` + +For users looking to use another container runtime, Rancher has the edge-focused K3s and datacenter-focused RKE2 Kubernetes distributions that use containerd as the default runtime. Imported RKE2 and K3s Kubernetes clusters can then be upgraded and managed through Rancher even after the removal of in-tree Dockershim in Kubernetes 1.24. + +### FAQ + +
+ +Q. Do I have to upgrade Rancher to get Rancher’s support of the upstream Dockershim? + +The upstream support of Dockershim begins for RKE in Kubernetes 1.21. You will need to be on Rancher 2.6 or above to have support for RKE with Kubernetes 1.21. See our [support matrix](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/) for details. + +
+ +Q. I am currently on RKE with Kubernetes 1.20. Do I need to upgrade to RKE with Kubernetes 1.21 sooner to avoid being out of support for Dockershim? + +A. The version of Dockershim in RKE with Kubernetes 1.20 will continue to work and is not scheduled for removal upstream until Kubernetes 1.24. It will only emit a warning of its future deprecation, which Rancher has mitigated in RKE with Kubernetes 1.21. You can plan your upgrade to Kubernetes 1.21 as you would normally, but should consider enabling the external Dockershim by Kubernetes 1.22. The external Dockershim will need to be enabled before upgrading to Kubernetes 1.24, at which point the existing implementation will be removed. + +For more information on the deprecation and its timeline, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). + +
+ +Q: What are my other options if I don’t want to depend on the Dockershim? + +A: You can use a runtime like containerd with Kubernetes that does not require Dockershim support. RKE2 or K3s are two options for doing this. + +
+ +Q: If I am already using RKE1 and want to switch to RKE2, what are my migration options? + +A: Rancher is exploring the possibility of an in-place upgrade path. Alternatively you can always migrate workloads from one cluster to another using kubectl. + +
diff --git a/versioned_docs/version-2.8/faq/general-faq.md b/versioned_docs/version-2.8/faq/general-faq.md new file mode 100644 index 00000000000..417875a6c6c --- /dev/null +++ b/versioned_docs/version-2.8/faq/general-faq.md @@ -0,0 +1,73 @@ +--- +title: General FAQ +--- + + + + + +This FAQ is a work in progress designed to answers the questions our users most frequently ask about Rancher v2.x. + +See [Technical FAQ](technical-items.md), for frequently asked technical questions. + +
+ +**Does Rancher v2.x support Docker Swarm and Mesos as environment types?** + +When creating an environment in Rancher v2.x, Swarm and Mesos will no longer be standard options you can select. However, both Swarm and Mesos will continue to be available as Catalog applications you can deploy. It was a tough decision to make but, in the end, it came down to adoption. For example, out of more than 15,000 clusters, only about 200 or so are running Swarm. + +
+ +**Is it possible to manage Azure Kubernetes Services with Rancher v2.x?** + +Yes. + +
+ +**Does Rancher support Windows?** + +Rancher supports Windows Server 1809 containers. For details on how to set up a cluster with Windows worker nodes, refer to the section on [configuring custom clusters for Windows.](../pages-for-subheaders/use-windows-clusters.md) + +
+ +**Does Rancher support Istio?** + +Rancher supports [Istio.](../pages-for-subheaders/istio.md) + +Furthermore, Istio is implemented in our micro-PaaS "Rio", which works on Rancher 2.x along with any CNCF compliant Kubernetes cluster. You can read more about it [here](https://rio.io/) + +
+ +**Will Rancher v2.x support Hashicorp's Vault for storing secrets?** + +Secrets management is on our roadmap but we haven't assigned it to a specific release yet. + +
+ +**Does Rancher v2.x support RKT containers as well?** + +At this time, we only support Docker. + +
+ +**Does Rancher v2.x support Calico, Contiv, Contrail, Flannel, Weave net, etc., for embedded and registered Kubernetes?** + +Out-of-the-box, Rancher provides the following CNI network providers for Kubernetes clusters: Canal, Flannel, Calico and Weave. Always refer to the [Rancher Support Matrix](https://rancher.com/support-maintenance-terms/) for details about what is officially supported. + +
+ +**Are you planning on supporting Traefik for existing setups?** + +We don't currently plan on providing embedded Traefik support, but we're still exploring load-balancing approaches. + +
+ +**Can I import OpenShift Kubernetes clusters into v2.x?** + +Our goal is to run any upstream Kubernetes clusters. Therefore, Rancher v2.x should work with OpenShift, but we haven't tested it yet. + +
+ +**Are you going to integrate Longhorn?** + +Yes. Longhorn was integrated into Rancher v2.5+. diff --git a/versioned_docs/version-2.8/faq/install-and-configure-kubectl.md b/versioned_docs/version-2.8/faq/install-and-configure-kubectl.md new file mode 100644 index 00000000000..9bcb56bc3d2 --- /dev/null +++ b/versioned_docs/version-2.8/faq/install-and-configure-kubectl.md @@ -0,0 +1,33 @@ +--- +title: Installing and Configuring kubectl +--- + + + + + +`kubectl` is a CLI utility for running commands against Kubernetes clusters. It's required for many maintenance and administrative tasks in Rancher 2.x. + +### Installation + +See [kubectl Installation](https://kubernetes.io/docs/tasks/tools/install-kubectl/) for installation on your operating system. + +### Configuration + +When you create a Kubernetes cluster with RKE, RKE creates a `kube_config_cluster.yml` in the local directory that contains credentials to connect to your new cluster with tools like `kubectl` or `helm`. + +You can copy this file as `$HOME/.kube/config` or if you are working with multiple Kubernetes clusters, set the `KUBECONFIG` environmental variable to the path of `kube_config_cluster.yml`. + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +Test your connectivity with `kubectl` and see if you can get the list of nodes back. + +``` +kubectl get nodes + NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.10.1 +``` diff --git a/versioned_docs/version-2.8/faq/rancher-is-no-longer-needed.md b/versioned_docs/version-2.8/faq/rancher-is-no-longer-needed.md new file mode 100644 index 00000000000..ce33edd6aec --- /dev/null +++ b/versioned_docs/version-2.8/faq/rancher-is-no-longer-needed.md @@ -0,0 +1,65 @@ +--- +title: Rancher is No Longer Needed +--- + + + + + +This page is intended to answer questions about what happens if you don't want Rancher anymore, if you don't want a cluster to be managed by Rancher anymore, or if the Rancher server is deleted. + + +### If the Rancher server is deleted, what happens to the workloads in my downstream clusters? + +If Rancher is ever deleted or unrecoverable, all workloads in the downstream Kubernetes clusters managed by Rancher will continue to function as normal. + +### If the Rancher server is deleted, how do I access my downstream clusters? + +The capability to access a downstream cluster without Rancher depends on the type of cluster and the way that the cluster was created. To summarize: + +- **Registered clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. +- **Hosted Kubernetes clusters:** If you created the cluster in a cloud-hosted Kubernetes provider such as EKS, GKE, or AKS, you can continue to manage the cluster using your provider's cloud credentials. +- **RKE clusters:** To access an [RKE cluster,](../pages-for-subheaders/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. (The authorized cluster endpoint is enabled by default for RKE clusters.) With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. + +### What if I don't want Rancher anymore? + +:::note + +The previously recommended [System Tools](../reference-guides/system-tools.md) has been deprecated since June 2022. + +::: + +If you [installed Rancher on a Kubernetes cluster,](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) remove Rancher by using the [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) tool. + +Uninstalling Rancher in high-availability (HA) mode will also remove all `helm-operation-*` pods and the following apps: + +- fleet +- fleet-agent +- rancher-operator +- rancher-webhook + +Custom resources (CRDs) and custom namespaces will still need to be manually removed. + +If you installed Rancher with Docker, you can uninstall Rancher by removing the single Docker container that it runs in. + +Imported clusters will not be affected by Rancher being removed. For other types of clusters, refer to the section on [accessing downstream clusters when Rancher is removed.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) + +### What if I don't want my registered cluster managed by Rancher? + +If a registered cluster is deleted from the Rancher UI, the cluster is detached from Rancher, leaving it intact and accessible by the same methods that were used to access it before it was registered in Rancher. + +To detach the cluster, + +1. In the upper left corner, click **☰ > Cluster Management**. +2. Go to the registered cluster that should be detached from Rancher and click **⋮ > Delete**. +3. Click **Delete**. + +**Result:** The registered cluster is detached from Rancher and functions normally outside of Rancher. + +### What if I don't want my RKE cluster or hosted Kubernetes cluster managed by Rancher? + +At this time, there is no functionality to detach these clusters from Rancher. In this context, "detach" is defined as the ability to remove Rancher components from the cluster and manage access to the cluster independently of Rancher. + +The capability to manage these clusters without Rancher is being tracked in this [issue.](https://github.com/rancher/rancher/issues/25234) + +For information about how to access clusters if the Rancher server is deleted, refer to [this section.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) diff --git a/versioned_docs/version-2.8/faq/security.md b/versioned_docs/version-2.8/faq/security.md new file mode 100644 index 00000000000..447fb53d7de --- /dev/null +++ b/versioned_docs/version-2.8/faq/security.md @@ -0,0 +1,18 @@ +--- +title: Security + +--- + + + + + +**Is there a Hardening Guide?** + +The Hardening Guide is now located in the main [Security](../pages-for-subheaders/rancher-security.md) section. + +
+ +**What are the results of Rancher's Kubernetes cluster when it is CIS benchmarked?** + +We have run the CIS Kubernetes benchmark against a hardened Rancher Kubernetes cluster. The results of that assessment can be found in the main [Security](../pages-for-subheaders/rancher-security.md) section. diff --git a/versioned_docs/version-2.8/faq/technical-items.md b/versioned_docs/version-2.8/faq/technical-items.md new file mode 100644 index 00000000000..0aed55b5d0d --- /dev/null +++ b/versioned_docs/version-2.8/faq/technical-items.md @@ -0,0 +1,180 @@ +--- +title: Technical +--- + + + + + +### How can I reset the administrator password? + +Docker Install: +``` +$ docker exec -ti reset-password +New password for default administrator (user-xxxxx): + +``` + +Kubernetes install (Helm): +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher --no-headers | head -1 | awk '{ print $1 }') -c rancher -- reset-password +New password for default administrator (user-xxxxx): + +``` + + + +### I deleted/deactivated the last admin, how can I fix it? +Docker Install: +``` +$ docker exec -ti ensure-default-admin +New default administrator (user-xxxxx) +New password for default administrator (user-xxxxx): + +``` + +Kubernetes install (Helm): +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher | grep '1/1' | head -1 | awk '{ print $1 }') -- ensure-default-admin +New password for default administrator (user-xxxxx): + +``` +### How can I enable debug logging? + +See [Troubleshooting: Logging](../troubleshooting/other-troubleshooting-tips/logging.md) + +### My ClusterIP does not respond to ping + +ClusterIP is a virtual IP, which will not respond to ping. Best way to test if the ClusterIP is configured correctly, is by using `curl` to access the IP and port to see if it responds. + +### Where can I manage Node Templates? + +Node Templates can be accessed by opening your account menu (top right) and selecting `Node Templates`. + +### Why is my Layer-4 Load Balancer in `Pending` state? + +The Layer-4 Load Balancer is created as `type: LoadBalancer`. In Kubernetes, this needs a cloud provider or controller that can satisfy these requests, otherwise these will be in `Pending` state forever. More information can be found on [Cloud Providers](../pages-for-subheaders/set-up-cloud-providers.md) or [Create External Load Balancer](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) + +### Where is the state of Rancher stored? + +- Docker Install: in the embedded etcd of the `rancher/rancher` container, located at `/var/lib/rancher`. +- Kubernetes install: in the etcd of the RKE cluster created to run Rancher. + +### How are the supported Docker versions determined? + +We follow the validated Docker versions for upstream Kubernetes releases. The validated versions can be found under [External Dependencies](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.10.md#external-dependencies) in the Kubernetes release CHANGELOG.md. + +### How can I access nodes created by Rancher? + +SSH keys to access the nodes created by Rancher can be downloaded via the **Nodes** view. Choose the node which you want to access and click on the vertical ⋮ button at the end of the row, and choose **Download Keys** as shown in the picture below. + +![Download Keys](/img/downloadsshkeys.png) + +Unzip the downloaded zip file, and use the file `id_rsa` to connect to you host. Be sure to use the correct username (`rancher` or `docker` for RancherOS, `ubuntu` for Ubuntu, `ec2-user` for Amazon Linux) + +``` +$ ssh -i id_rsa user@ip_of_node +``` + +### How can I automate task X in Rancher? + +The UI consists of static files, and works based on responses of the API. That means every action/task that you can execute in the UI, can be automated via the API. There are 2 ways to do this: + +* Visit `https://your_rancher_ip/v3` and browse the API options. +* Capture the API calls when using the UI (Most commonly used for this is [Chrome Developer Tools](https://developers.google.com/web/tools/chrome-devtools/#network) but you can use anything you like) + +### The IP address of a node changed, how can I recover? + +A node is required to have a static IP configured (or a reserved IP via DHCP). If the IP of a node has changed, you will have to remove it from the cluster and readd it. After it is removed, Rancher will update the cluster to the correct state. If the cluster is no longer in `Provisioning` state, the node is removed from the cluster. + +When the IP address of the node changed, Rancher lost connection to the node, so it will be unable to clean the node properly. See [Cleaning cluster nodes](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) to clean the node. + +When the node is removed from the cluster, and the node is cleaned, you can readd the node to the cluster. + +### How can I add additional arguments/binds/environment variables to Kubernetes components in a Rancher Launched Kubernetes cluster? + +You can add additional arguments/binds/environment variables via the [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) option in Cluster Options. For more information, see the [Extra Args, Extra Binds, and Extra Environment Variables](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/) in the RKE documentation or browse the [Example Cluster.ymls](https://rancher.com/docs/rke/latest/en/example-yamls/). + +### How do I check if my certificate chain is valid? + +Use the `openssl verify` command to validate your certificate chain: + +:::tip + +Configure `SSL_CERT_DIR` and `SSL_CERT_FILE` to a dummy location to make sure the OS-installed certificates are not used when verifying manually. + +::: + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +If you receive the error `unable to get local issuer certificate`, the chain is incomplete. This usually means that there is an intermediate CA certificate that issued your server certificate. If you already have this certificate, you can use it in the verification of the certificate like shown below: + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem -untrusted intermediate.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +If you have successfully verified your certificate chain, you should include needed intermediate CA certificates in the server certificate to complete the certificate chain for any connection made to Rancher (for example, by the Rancher agent). The order of the certificates in the server certificate file should be first the server certificate itself (contents of `rancher.yourdomain.com.pem`), followed by intermediate CA certificate(s) (contents of `intermediate.pem`). + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +If you still get errors during verification, you can retrieve the subject and the issuer of the server certificate using the following command: + +``` +openssl x509 -noout -subject -issuer -in rancher.yourdomain.com.pem +subject= /C=GB/ST=England/O=Alice Ltd/CN=rancher.yourdomain.com +issuer= /C=GB/ST=England/O=Alice Ltd/CN=Alice Intermediate CA +``` + +### How do I check `Common Name` and `Subject Alternative Names` in my server certificate? + +Although technically an entry in `Subject Alternative Names` is required, having the hostname in both `Common Name` and as entry in `Subject Alternative Names` gives you maximum compatibility with older browser/applications. + +Check `Common Name`: + +``` +openssl x509 -noout -subject -in cert.pem +subject= /CN=rancher.my.org +``` + +Check `Subject Alternative Names`: + +``` +openssl x509 -noout -in cert.pem -text | grep DNS + DNS:rancher.my.org +``` + +### Why does it take 5+ minutes for a pod to be rescheduled when a node has failed? + +This is due to a combination of the following default Kubernetes settings: + +* kubelet + * `node-status-update-frequency`: Specifies how often kubelet posts node status to master (default 10s) +* kube-controller-manager + * `node-monitor-period`: The period for syncing NodeStatus in NodeController (default 5s) + * `node-monitor-grace-period`: Amount of time which we allow running Node to be unresponsive before marking it unhealthy (default 40s) + * `pod-eviction-timeout`: The grace period for deleting pods on failed nodes (default 5m0s) + +See [Kubernetes: kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) and [Kubernetes: kube-controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/) for more information on these settings. + +In Kubernetes v1.13, the `TaintBasedEvictions` feature is enabled by default. See [Kubernetes: Taint based Evictions](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions) for more information. + +* kube-apiserver (Kubernetes v1.13 and up) + * `default-not-ready-toleration-seconds`: Indicates the tolerationSeconds of the toleration for notReady:NoExecute that is added by default to every pod that does not already have such a toleration. + * `default-unreachable-toleration-seconds`: Indicates the tolerationSeconds of the toleration for unreachable:NoExecute that is added by default to every pod that does not already have such a toleration. + +### Can I use keyboard shortcuts in the UI? + +Yes, most parts of the UI can be reached using keyboard shortcuts. For an overview of the available shortcuts, press `?` anywhere in the UI. diff --git a/versioned_docs/version-2.8/faq/telemetry.md b/versioned_docs/version-2.8/faq/telemetry.md new file mode 100644 index 00000000000..a510347e556 --- /dev/null +++ b/versioned_docs/version-2.8/faq/telemetry.md @@ -0,0 +1,35 @@ +--- +title: Telemetry +--- + + + + + +### What is Telemetry? + +Telemetry collects aggregate information about the size of Rancher installations, versions of components used, and which features are used. This information is used by Rancher Labs to help make the product better and is not shared with third-parties. + +### What information is collected? + +No specific identifying information like usernames, passwords, or the names or addresses of user resources will ever be collected. + +The primary things collected include: + + - Aggregate counts (smallest, average, largest, total) of nodes per-cluster and their size (e.g. CPU cores & RAM). + - Aggregate counts of logical resources like Clusters, Projects, Namespaces, and Pods. + - Counts of what driver was used to deploy clusters and nodes (e.g. GKE vs EC2 vs Imported vs Custom). + - Versions of Kubernetes components, Operating Systems and Docker that are deployed on nodes. + - Whether some optional components are enabled or not (e.g. which auth providers are used). + - The image name & version of Rancher that is running. + - A unique randomly-generated identifier for this installation. + +### Can I see the information that is being sent? + +If Telemetry is enabled, you can go to `https:///v1-telemetry` in your installation to see the current data. + +If Telemetry is not enabled, the process that collects the data is not running, so there is nothing being collected to look at. + +### How do I turn it on or off? + +After initial setup, an administrator can go to the `Settings` page in the `Global` section of the UI and click Edit to change the `telemetry-opt` setting to either `in` or `out`. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md new file mode 100644 index 00000000000..7b01fb3aac2 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md @@ -0,0 +1,80 @@ +--- +title: Upgrading in an Air-Gapped Environment +--- + + + + + +:::note + +These instructions assume you have already followed the instructions for a Kubernetes upgrade on [this page,](upgrades.md) including the prerequisites, up until step 3. Upgrade Rancher. + +::: + +### Rancher Helm Upgrade Options + +To upgrade with Helm, apply the same options that you used when installing Rancher. Refer to the reference table below to replace each placeholder. Rancher needs to be configured to use the private registry in order to provision any Rancher launched Kubernetes clusters or Rancher tools. + +Based on the choice you made during installation, complete one of the procedures below. + +Placeholder | Description +------------|------------- +`` | The version number of the output tarball. +`` | The DNS name you pointed at your load balancer. +`` | The DNS name for your private registry. +`` | Cert-manager version running on k8s cluster. + + +### Option A: Default Self-signed Certificate + +``` +helm upgrade rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +### Option B: Certificates from Files using Kubernetes Secrets + +```plain +helm upgrade rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +If you are using a Private CA signed cert, add `--set privateCA=true` following `--set ingress.tls.source=secret`: + +```plain +helm upgrade rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +## Verify the Upgrade + +Log into Rancher to confirm that the upgrade succeeded. + +:::tip + +Having network issues following upgrade? + +See [Restoring Cluster Networking](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md). + +::: + +## Known Upgrade Issues + +A list of known issues for each Rancher version can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums.](https://forums.rancher.com/c/announcements/12) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md new file mode 100644 index 00000000000..2300a22016f --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md @@ -0,0 +1,151 @@ +--- +title: Installing Rancher on Azure Kubernetes Service +--- + + + + + +This page covers how to install Rancher on Microsoft's Azure Kubernetes Service (AKS). + +The guide uses command line tools to provision an AKS cluster with an ingress. If you prefer to provision your cluster using the Azure portal, refer to the [official documentation](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal). + +If you already have an AKS Kubernetes cluster, skip to the step about [installing an ingress.](#5-install-an-ingress) Then install the Rancher Helm chart following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) + +## Prerequisites + +:::caution + +Deploying to Microsoft Azure will incur charges. + +::: + +- [Microsoft Azure Account](https://azure.microsoft.com/en-us/free/): A Microsoft Azure Account is required to create resources for deploying Rancher and Kubernetes. +- [Microsoft Azure Subscription](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal): Use this link to follow a tutorial to create a Microsoft Azure subscription if you don't have one yet. +- [Micsoroft Azure Tenant](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant): Use this link and follow instructions to create a Microsoft Azure tenant. +- Your subscription has sufficient quota for at least 2 vCPUs. For details on Rancher server resource requirements, refer to [this section](../../../pages-for-subheaders/installation-requirements.md#rke-and-hosted-kubernetes) +- When installing Rancher with Helm in Azure, use the L7 load balancer to avoid networking issues. For more information, refer to the documentation on [Azure load balancer limitations](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations). + +## 1. Prepare your Workstation + +Install the following command line tools on your workstation: + +- The Azure CLI, **az:** For help, refer to these [installation steps.](https://docs.microsoft.com/en-us/cli/azure/) +- **kubectl:** For help, refer to these [installation steps.](https://kubernetes.io/docs/tasks/tools/#kubectl) +- **helm:** For help, refer to these [installation steps.](https://helm.sh/docs/intro/install/) + +## 2. Create a Resource Group + +After installing the CLI, you will need to log in with your Azure account. + +``` +az login +``` + +Create a [resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) to hold all relevant resources for your cluster. Use a location that applies to your use case. + +``` +az group create --name rancher-rg --location eastus +``` + +## 3. Create the AKS Cluster + +To create an AKS cluster, run the following command. Use a VM size that applies to your use case. Refer to [this article](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes) for available sizes and options. When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +:::note + +If you're updating from an older version of Kubernetes, to Kubernetes v1.22 or above, you also need to [update](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx. + +::: + +``` +az aks create \ + --resource-group rancher-rg \ + --name rancher-server \ + --kubernetes-version \ + --node-count 3 \ + --node-vm-size Standard_D2_v3 +``` + +The cluster will take some time to be deployed. + +## 4. Get Access Credentials + +After the cluster is deployed, get the access credentials. + +``` +az aks get-credentials --resource-group rancher-rg --name rancher-server +``` + +This command merges your cluster's credentials into the existing kubeconfig and allows `kubectl` to interact with the cluster. + +## 5. Install an Ingress + +The cluster needs an Ingress so that Rancher can be accessed from outside the cluster. Installing an Ingress requires allocating a public IP address. Ensure you have sufficient quota, otherwise it will fail to assign the IP address. Limits for public IP addresses are applicable at a regional level per subscription. + +To make sure that you choose the correct Ingress-NGINX Helm chart, first find an `Ingress-NGINX version` that's compatible with your Kubernetes version in the [Kubernetes/ingress-nginx support table](https://github.com/kubernetes/ingress-nginx#supported-versions-table). + +Then, list the Helm charts available to you by running the following command: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +The `helm search` command's output contains an `APP VERSION` column. The versions under this column are equivalent to the `Ingress-NGINX version` you chose earlier. Using the app version, select a chart version that bundles an app compatible with your Kubernetes install. For example, if you have Kubernetes v1.24, you can select the v4.6.0 Helm chart, since Ingress-NGINX v1.7.0 comes bundled with that chart, and v1.7.0 is compatible with Kubernetes v1.24. When in doubt, select the most recent compatible version. + +Now that you know which Helm chart `version` you need, run the following command. It installs an `nginx-ingress-controller` with a Kubernetes load balancer service: + +``` +helm search repo ingress-nginx -l +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ + --set controller.service.externalTrafficPolicy=Local \ + --version 4.6.0 \ + --create-namespace +``` + +## 6. Get Load Balancer IP + +To get the address of the load balancer, run: + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +The result should look similar to the following: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.0.116.18 40.31.180.83 80:31229/TCP,443:31050/TCP + 67s +``` + +Save the `EXTERNAL-IP`. + +## 7. Set up DNS + +External traffic to the Rancher server will need to be directed at the load balancer you created. + +Set up a DNS to point at the `EXTERNAL-IP` that you saved. This DNS will be used as the Rancher server URL. + +There are many valid ways to set up the DNS. For help, refer to the [Azure DNS documentation](https://docs.microsoft.com/en-us/azure/dns/) + +## 8. Install the Rancher Helm Chart + +Next, install the Rancher Helm chart by following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) The Helm instructions are the same for installing Rancher on any Kubernetes distribution. + +Use that DNS name from the previous step as the Rancher server URL when you install Rancher. It can be passed in as a Helm option. For example, if the DNS name is `rancher.my.org`, you could run the Helm installation command with the option `--set hostname=rancher.my.org`. + +When installing Rancher on top of this setup, you will also need to pass the value below into the Rancher Helm install command in order to set the name of the ingress controller to be used with Rancher's ingress resource: + +``` +--set ingress.ingressClassName=nginx +``` + +Refer [here for the Helm install command](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-install-rancher-with-helm-and-your-chosen-certificate-option) for your chosen certificate option. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md new file mode 100644 index 00000000000..6563d4b14eb --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md @@ -0,0 +1,155 @@ +--- +title: Installing Rancher on Amazon EKS +--- + + + + + +This page covers installing Rancher on an Amazon EKS cluster. You can also [install Rancher through the AWS Marketplace](../../quick-start-guides/deploy-rancher-manager/aws-marketplace.md). + +If you already have an EKS Kubernetes cluster, skip to the step about [installing an ingress.](#5-install-an-ingress) Then install the Rancher Helm chart following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) + +## Creating an EKS Cluster for the Rancher Server + +In this section, you'll install an EKS cluster with an ingress by using command line tools. This guide may be useful if you want to use fewer resources while trying out Rancher on EKS. + +:::note Prerequisites: + +- You should already have an AWS account. +- It is recommended to use an IAM user instead of the root AWS account. You will need the IAM user's access key and secret key to configure the AWS command line interface. +- The IAM user needs the minimum IAM policies described in the official [eksctl documentation.](https://eksctl.io/usage/minimum-iam-policies/) + +::: + +### 1. Prepare your Workstation + +Install the following command line tools on your workstation: + +- **The AWS CLI v2:** For help, refer to these [installation steps.](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +- **eksctl:** For help, refer to these [installation steps.](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +- **kubectl:** For help, refer to these [installation steps.](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html) +- **helm:** For help, refer to these [installation steps.](https://helm.sh/docs/intro/install/) + +### 2. Configure the AWS CLI + +To configure the AWS CLI, run the following command: + +``` +aws configure +``` + +Then enter the following values: + +| Value | Description | +|-------|-------------| +| AWS Access Key ID | The access key credential for the IAM user with EKS permissions. | +| AWS Secret Access Key | The secret key credential for the IAM user with EKS permissions. | +| Default region name | An [AWS region](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html#Concepts.RegionsAndAvailabilityZones.Regions) where the cluster nodes will be located. | +| Default output format | Enter `json`. | + +### 3. Create the EKS Cluster + +To create an EKS cluster, run the following command. Use the AWS region that applies to your use case. When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +**Note:** If you're updating from an older version of Kubernetes, to Kubernetes v1.22 or above, you also need to [update](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx. + +``` +eksctl create cluster \ + --name rancher-server \ + --version \ + --region us-west-2 \ + --nodegroup-name ranchernodes \ + --nodes 3 \ + --nodes-min 1 \ + --nodes-max 4 \ + --managed +``` + +The cluster will take some time to be deployed with CloudFormation. + +### 4. Test the Cluster + +To test the cluster, run: + +``` +eksctl get cluster +``` + +The result should look like the following: + +``` +eksctl get cluster +2021-03-18 15:09:35 [ℹ] eksctl version 0.40.0 +2021-03-18 15:09:35 [ℹ] using region us-west-2 +NAME REGION EKSCTL CREATED +rancher-server-cluster us-west-2 True +``` + +### 5. Install an Ingress + +The cluster needs an Ingress so that Rancher can be accessed from outside the cluster. + +To make sure that you choose the correct Ingress-NGINX Helm chart, first find an `Ingress-NGINX version` that's compatible with your Kubernetes version in the [Kubernetes/ingress-nginx support table](https://github.com/kubernetes/ingress-nginx#supported-versions-table). + +Then, list the Helm charts available to you by running the following command: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +The `helm search` command's output contains an `APP VERSION` column. The versions under this column are equivalent to the `Ingress-NGINX version` you chose earlier. Using the app version, select a chart version that bundles an app compatible with your Kubernetes install. For example, if you have Kubernetes v1.23, you can select the v4.6.0 Helm chart, since Ingress-NGINX v1.7.0 comes bundled with that chart, and v1.7.0 is compatible with Kubernetes v1.23. When in doubt, select the most recent compatible version. + +Now that you know which Helm chart `version` you need, run the following command. It installs an `nginx-ingress-controller` with a Kubernetes load balancer service: + +``` +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.6.0 \ + --create-namespace +``` + +### 6. Get Load Balancer IP + +To get the address of the load balancer, run: + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +The result should look similar to the following: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.100.90.18 a904a952c73bf4f668a17c46ac7c56ab-962521486.us-west-2.elb.amazonaws.com 80:31229/TCP,443:31050/TCP + 27m +``` + +Save the `EXTERNAL-IP`. + +### 7. Set up DNS + +External traffic to the Rancher server will need to be directed at the load balancer you created. + +Set up a DNS to point at the external IP that you saved. This DNS will be used as the Rancher server URL. + +There are many valid ways to set up the DNS. For help, refer to the AWS documentation on [routing traffic to an ELB load balancer.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer.html) + +### 8. Install the Rancher Helm Chart + +Next, install the Rancher Helm chart by following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) The Helm instructions are the same for installing Rancher on any Kubernetes distribution. + +Use that DNS name from the previous step as the Rancher server URL when you install Rancher. It can be passed in as a Helm option. For example, if the DNS name is `rancher.my.org`, you could run the Helm installation command with the option `--set hostname=rancher.my.org`. + +When installing Rancher on top of this setup, you will also need to pass the value below into the Rancher Helm install command in order to set the name of the ingress controller to be used with Rancher's ingress resource: + +``` +--set ingress.ingressClassName=nginx +``` + +Refer [here for the Helm install command](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-install-rancher-with-helm-and-your-chosen-certificate-option) for your chosen certificate option. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md new file mode 100644 index 00000000000..4f7f9a2f124 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md @@ -0,0 +1,205 @@ +--- +title: Installing Rancher on a Google Kubernetes Engine Cluster +--- + + + + + +In this section, you'll learn how to install Rancher using Google Kubernetes Engine. + +If you already have a GKE Kubernetes cluster, skip to the step about [installing an ingress.](#7-install-an-ingress) Then install the Rancher Helm chart following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) + +## Prerequisites + +- You will need a Google account. +- You will need a Google Cloud billing account. You can manage your Cloud Billing accounts using the Google Cloud Console. For more information about the Cloud Console, visit [General guide to the console.](https://support.google.com/cloud/answer/3465889?hl=en&ref_topic=3340599) +- You will need a cloud quota for at least one in-use IP address and at least 2 CPUs. For more details about hardware requirements for the Rancher server, refer to [this section.](../../../pages-for-subheaders/installation-requirements.md#rke-and-hosted-kubernetes) + +## 1. Enable the Kubernetes Engine API + +Take the following steps to enable the Kubernetes Engine API: + +1. Visit the [Kubernetes Engine page](https://console.cloud.google.com/projectselector/kubernetes?_ga=2.169595943.767329331.1617810440-856599067.1617343886) in the Google Cloud Console. +1. Create or select a project. +1. Open the project and enable the Kubernetes Engine API for the project. Wait for the API and related services to be enabled. This can take several minutes. +1. Make sure that billing is enabled for your Cloud project. For information on how to enable billing for your project, refer to the [Google Cloud documentation.](https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project) + +## 2. Open the Cloud Shell + +Cloud Shell is a shell environment for managing resources hosted on Google Cloud. Cloud Shell comes preinstalled with the `gcloud` command-line tool and kubectl command-line tool. The `gcloud` tool provides the primary command-line interface for Google Cloud, and `kubectl` provides the primary command-line interface for running commands against Kubernetes clusters. + +The following sections describe how to launch the cloud shell from the Google Cloud Console or from your local workstation. + +### Cloud Shell + +To launch the shell from the [Google Cloud Console,](https://console.cloud.google.com) go to the upper-right corner of the console and click the terminal button. When hovering over the button, it is labeled **Activate Cloud Shell**. + +### Local Shell + +To install `gcloud` and `kubectl`, perform the following steps: + +1. Install the Cloud SDK by following [these steps.](https://cloud.google.com/sdk/docs/install) The Cloud SDK includes the `gcloud` command-line tool. The steps vary based on your OS. +1. After installing Cloud SDK, install the `kubectl` command-line tool by running the following command: + + ``` + gcloud components install kubectl + ``` + In a later step, `kubectl` will be configured to use the new GKE cluster. +1. [Install Helm 3](https://helm.sh/docs/intro/install/) if it is not already installed. +1. Enable Helm experimental [support for OCI images](https://github.com/helm/community/blob/master/hips/hip-0006.md) with the `HELM_EXPERIMENTAL_OCI` variable. Add the following line to `~/.bashrc` (or `~/.bash_profile` in macOS, or wherever your shell stores environment variables): + + ``` + export HELM_EXPERIMENTAL_OCI=1 + ``` +1. Run the following command to load your updated `.bashrc` file: + + ``` + source ~/.bashrc + ``` + If you are running macOS, use this command: + ``` + source ~/.bash_profile + ``` + + + +## 3. Configure the gcloud CLI + + Set up default gcloud settings using one of the following methods: + +- Using gcloud init, if you want to be walked through setting defaults. +- Using gcloud config, to individually set your project ID, zone, and region. + + + + + 1. Run gcloud init and follow the directions: + + ``` + gcloud init + ``` + If you are using SSH on a remote server, use the --console-only flag to prevent the command from launching a browser: + + ``` + gcloud init --console-only + ``` + 2. Follow the instructions to authorize gcloud to use your Google Cloud account and select the new project that you created. + + + + + + + +## 4. Confirm that gcloud is configured correctly + +Run: + +``` +gcloud config list +``` + +The output should resemble the following: + +``` +[compute] +region = us-west1 # Your chosen region +zone = us-west1-b # Your chosen zone +[core] +account = +disable_usage_reporting = True +project = + +Your active configuration is: [default] +``` + +## 5. Create a GKE Cluster + +The following command creates a three-node cluster. + +Replace `cluster-name` with the name of your new cluster. + +When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +To successfully create a GKE cluster with Rancher, your GKE must be in Standard mode. GKE has two modes of operation when creating a Kubernetes cluster, Autopilot and Standard mode. The cluster configuration for Autopilot mode has restrictions on editing the kube-system namespace. However, Rancher needs to create resources in the kube-system namespace during installation. As a result, you will not be able to install Rancher on a GKE cluster created in Autopilot mode. For more information about the difference between GKE Autopilot mode and Standard mode, visit [Compare GKE Autopilot and Standard.](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison) + +**Note:** If you're updating from an older version of Kubernetes, to Kubernetes v1.22 or above, you also need to [update](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx. + +``` +gcloud container clusters create cluster-name --num-nodes=3 --cluster-version= +``` + +## 6. Get Authentication Credentials + +After creating your cluster, you need to get authentication credentials to interact with the cluster: + +``` +gcloud container clusters get-credentials cluster-name +``` + +This command configures `kubectl` to use the cluster you created. + +## 7. Install an Ingress + +The cluster needs an Ingress so that Rancher can be accessed from outside the cluster. + +The following command installs an `nginx-ingress-controller` with a LoadBalancer service: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.0.18 \ + --create-namespace +``` + +## 8. Get the Load Balancer IP + +To get the address of the load balancer, run: + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +The result should look similar to the following: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +ingress-nginx-controller LoadBalancer 10.3.244.156 35.233.206.34 80:31876/TCP,443:32497/TCP 81s +``` + +Save the `EXTERNAL-IP`. + +## 9. Set up DNS + +External traffic to the Rancher server will need to be directed at the load balancer you created. + +Set up a DNS to point at the external IP that you saved. This DNS will be used as the Rancher server URL. + +There are many valid ways to set up the DNS. For help, refer to the Google Cloud documentation about [managing DNS records.](https://cloud.google.com/dns/docs/records) + +## 10. Install the Rancher Helm chart + +Next, install the Rancher Helm chart by following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) The Helm instructions are the same for installing Rancher on any Kubernetes distribution. + +Use the DNS name from the previous step as the Rancher server URL when you install Rancher. It can be passed in as a Helm option. For example, if the DNS name is `rancher.my.org`, you could run the Helm installation command with the option `--set hostname=rancher.my.org`. + +When installing Rancher on top of this setup, you will also need to set the name of the ingress controller to be used with Rancher's ingress resource: + +``` +--set ingress.ingressClassName=nginx +``` + +Refer [here for the Helm install command](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-install-rancher-with-helm-and-your-chosen-certificate-option) for your chosen certificate option. + +In Rancher v2.7.5, if you intend to use the default GKE ingress on your cluster without enabling VPC-native cluster mode, you need to set the following flag: + +``` +--set service.type=NodePort +``` + +This is necessary because of compatibility issues between this setup and ClusterIP, the default type for `cattle-system/rancher`. \ No newline at end of file diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml new file mode 100644 index 00000000000..ae248cc9b7c --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml @@ -0,0 +1,37 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + defaults: + audit: restricted + audit-version: latest + enforce: restricted + enforce-version: latest + warn: restricted + warn-version: latest + exemptions: + namespaces: + - ingress-nginx + - kube-system + - cattle-system + - cattle-epinio-system + - cattle-fleet-system + - longhorn-system + - cattle-neuvector-system + - cattle-monitoring-system + - rancher-alerting-drivers + - cis-operator-system + - cattle-csp-adapter-system + - cattle-externalip-system + - cattle-gatekeeper-system + - istio-system + - cattle-istio-system + - cattle-logging-system + - cattle-windows-gmsa-system + - cattle-sriov-system + - cattle-ui-plugin-system + - tigera-operator + kind: PodSecurityConfiguration + name: PodSecurity + path: "" \ No newline at end of file diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md new file mode 100644 index 00000000000..bb7fed4fe0a --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md @@ -0,0 +1,132 @@ +--- +title: Rollbacks +--- + + + + + +## Additional Steps for Rollbacks with Rancher v2.6.4+ + +Rancher v2.6.4 upgrades the cluster-api module from v0.4.4 to v1.0.2. Version v1.0.2 of the cluster-api, in turn, upgrades the Cluster API's Custom Resource Definitions (CRDs) from `cluster.x-k8s.io/v1alpha4` to `cluster.x-k8s.io/v1beta1`. The CRDs upgrade to v1beta1 causes rollbacks to fail when you attempt to move from Rancher v2.6.4 to any previous version of Rancher v2.6.x. This is because CRDs that use the older apiVersion (v1alpha4) are incompatible with v1beta1. + +To avoid rollback failure, the following Rancher scripts should be run **before** you attempt a restore operation or rollback: + +* `verify.sh`: Checks for any Rancher-related resources in the cluster. +* `cleanup.sh`: Cleans up the cluster. + +See the [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup) for more details and source code. + +:::caution + + There will be downtime while `cleanup.sh` runs, since the script deletes resources created by Rancher. + +::: + +### Rolling back from v2.6.4+ to lower versions of v2.6.x + +1. Follow these [instructions](https://github.com/rancher/rancher-cleanup/blob/main/README.md) to run the scripts. +1. Follow these [instructions](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) to install the rancher-backup Helm chart on the existing cluster and restore the previous state. + 1. Omit Step 3. + 1. When you reach Step 4, install the Rancher v2.6.x version on the local cluster you intend to roll back to. + +## Rolling Back to Rancher v2.5.0+ + +To roll back to Rancher v2.5.0+, use the **Rancher Backups** application and restore Rancher from backup. + +Rancher has to be started with the lower/previous version after a rollback. + +A restore is performed by creating a Restore custom resource. + +:::note Important: + +* Follow the instructions from this page for restoring Rancher on the same cluster where it was backed up from. In order to migrate Rancher to a new cluster, follow the steps to [migrate Rancher.](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +* While restoring Rancher on the same setup, the Rancher deployment is manually scaled down before the restore starts, then the operator will scale it back up once the restore completes. As a result, Rancher and its UI will be unavailable until the restore is complete. While the UI is unavailable, use the original cluster kubeconfig with the restore YAML file: `kubectl create -f restore.yaml`. + +::: + +### Create the Restore Custom Resource + +1. Click **☰ > Cluster Management**. +1. Go to the local cluster and click **Explore**. +1. In the left navigation bar, click **Rancher Backups > Restore**. + :::note + + If the Rancher Backups app is not visible, you will need to install it from the Charts page in **Apps**. Refer [here](../../../pages-for-subheaders/helm-charts-in-rancher.md#charts) for more information. + + ::: + +1. Click **Create**. +1. Create the Restore with the form or with YAML. For help creating the Restore resource using the online form, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) +1. To use the YAML editor, you can click **Create > Create from YAML.** Enter the Restore YAML. The following is an example Restore custom resource: + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + For help configuring the Restore, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) + +1. Click **Create**. + +**Result:** The backup file is created and updated to the target storage location. The resources are restored in this order: + +1. Custom Resource Definitions (CRDs) +2. Cluster-scoped resources +3. Namespaced resources + +To check how the restore is progressing, you can check the logs of the operator. Follow these steps to get the logs: + +```yaml +kubectl get pods -n cattle-resources-system +kubectl logs -n cattle-resources-system -f +``` + +### Roll back to a previous Rancher version + +Rancher can be rolled back using the Helm CLI. To roll back to the previous version: + +```yaml +helm rollback rancher -n cattle-system +``` + +If the previous revision is not the intended target, you can specify a revision to roll back to. To see the deployment history: + +```yaml +helm history rancher -n cattle-system +``` + +When the target revision is determined, perform the rollback. This example will roll back to revision `3`: + +```yaml +helm rollback rancher 3 -n cattle-system +``` + +## Rolling Back to Rancher v2.2-v2.4+ + +To roll back to Rancher before v2.5, follow the procedure detailed here: [Restoring Backups — Kubernetes installs](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Restoring a snapshot of the Rancher server cluster will revert Rancher to the version and state at the time of the snapshot. + +For information on how to roll back Rancher installed with Docker, refer to [this page.](../other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md) + +:::note + +Managed clusters are authoritative for their state. This means restoring the Rancher server will not revert workload deployments or changes made on managed clusters after the snapshot was taken. + +::: + +## Rolling Back to Rancher v2.0-v2.1 + +Rolling back to Rancher v2.0-v2.1 is no longer supported. The instructions for rolling back to these versions are preserved [here](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup/roll-back-to-v2.0-v2.1.md) and are intended to be used only in cases where upgrading to Rancher v2.2+ is not feasible. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md new file mode 100644 index 00000000000..9ebbf27460f --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md @@ -0,0 +1,192 @@ +--- +title: Troubleshooting the Rancher Server Kubernetes Cluster +--- + + + + + +This section describes how to troubleshoot an installation of Rancher on a Kubernetes cluster. + +### Relevant Namespaces + +Most of the troubleshooting will be done on objects in these 3 namespaces. + +- `cattle-system` - `rancher` deployment and pods. +- `ingress-nginx` - Ingress controller pods and services. +- `cert-manager` - `cert-manager` pods. + +### "default backend - 404" + +A number of things can cause the ingress-controller not to forward traffic to your rancher instance. Most of the time its due to a bad ssl configuration. + +Things to check + +- [Is Rancher Running](#check-if-rancher-is-running) +- [Cert CN is "Kubernetes Ingress Controller Fake Certificate"](#cert-cn-is-kubernetes-ingress-controller-fake-certificate) + +### Check if Rancher is Running + +Use `kubectl` to check the `cattle-system` system namespace and see if the Rancher pods are in a Running state. + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +If the state is not `Running`, run a `describe` on the pod and check the Events. + +``` +kubectl -n cattle-system describe pod + +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 11m default-scheduler Successfully assigned rancher-784d94f59b-vgqzh to localhost + Normal SuccessfulMountVolume 11m kubelet, localhost MountVolume.SetUp succeeded for volume "rancher-token-dj4mt" + Normal Pulling 11m kubelet, localhost pulling image "rancher/rancher:v2.0.4" + Normal Pulled 11m kubelet, localhost Successfully pulled image "rancher/rancher:v2.0.4" + Normal Created 11m kubelet, localhost Created container + Normal Started 11m kubelet, localhost Started container +``` + +### Check the Rancher Logs + +Use `kubectl` to list the pods. + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +Use `kubectl` and the pod name to list the logs from the pod. + +``` +kubectl -n cattle-system logs -f rancher-784d94f59b-vgqzh +``` + +### Cert CN is "Kubernetes Ingress Controller Fake Certificate" + +Use your browser to check the certificate details. If it says the Common Name is "Kubernetes Ingress Controller Fake Certificate", something may have gone wrong with reading or issuing your SSL cert. + +:::note + +If you are using LetsEncrypt to issue certs, it can sometimes take a few minutes to issue the cert. + +::: + +### Checking for issues with cert-manager issued certs (Rancher Generated or LetsEncrypt) + +`cert-manager` has 3 parts. + +- `cert-manager` pod in the `cert-manager` namespace. +- `Issuer` object in the `cattle-system` namespace. +- `Certificate` object in the `cattle-system` namespace. + +Work backwards and do a `kubectl describe` on each object and check the events. You can track down what might be missing. + +For example there is a problem with the Issuer: + +``` +kubectl -n cattle-system describe certificate +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning IssuerNotReady 18s (x23 over 19m) cert-manager Issuer rancher not ready +``` + +``` +kubectl -n cattle-system describe issuer +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning ErrInitIssuer 19m (x12 over 19m) cert-manager Error initializing issuer: secret "tls-rancher" not found + Warning ErrGetKeyPair 9m (x16 over 19m) cert-manager Error getting keypair for CA issuer: secret "tls-rancher" not found +``` + +### Checking for Issues with Your Own SSL Certs + +Your certs get applied directly to the Ingress object in the `cattle-system` namespace. + +Check the status of the Ingress object and see if its ready. + +``` +kubectl -n cattle-system describe ingress +``` + +If its ready and the SSL is still not working you may have a malformed cert or secret. + +Check the nginx-ingress-controller logs. Because the nginx-ingress-controller has multiple containers in its pod you will need to specify the name of the container. + +``` +kubectl -n ingress-nginx logs -f nginx-ingress-controller-rfjrq nginx-ingress-controller +... +W0705 23:04:58.240571 7 backend_ssl.go:49] error obtaining PEM from secret cattle-system/tls-rancher-ingress: error retrieving secret cattle-system/tls-rancher-ingress: secret cattle-system/tls-rancher-ingress was not found +``` + +### No matches for kind "Issuer" + +The SSL configuration option you have chosen requires cert-manager to be installed before installing Rancher or else the following error is shown: + +``` +Error: validation failed: unable to recognize "": no matches for kind "Issuer" in version "certmanager.k8s.io/v1alpha1" +``` + +Install cert-manager and try installing Rancher again. + + +### Canal Pods show READY 2/3 + +The most common cause of this issue is port 8472/UDP is not open between the nodes. Check your local firewall, network routing or security groups. + +Once the network issue is resolved, the `canal` pods should timeout and restart to establish their connections. + +### nginx-ingress-controller Pods show RESTARTS + +The most common cause of this issue is the `canal` pods have failed to establish the overlay network. See [canal Pods show READY `2/3`](#canal-pods-show-ready-23) for troubleshooting. + + +### Failed to dial to /var/run/docker.sock: ssh: rejected: administratively prohibited (open failed) + +Some causes of this error include: + +* User specified to connect with does not have permission to access the Docker socket. This can be checked by logging into the host and running the command `docker ps`: + +``` +$ ssh user@server +user@server$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +``` + +See [Manage Docker as a non-root user](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) how to set this up properly. + +* When using RedHat/CentOS as operating system, you cannot use the user `root` to connect to the nodes because of [Bugzilla #1527565](https://bugzilla.redhat.com/show_bug.cgi?id=1527565). You will need to add a separate user and configure it to access the Docker socket. See [Manage Docker as a non-root user](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) how to set this up properly. + +* SSH server version is not version 6.7 or higher. This is needed for socket forwarding to work, which is used to connect to the Docker socket over SSH. This can be checked using `sshd -V` on the host you are connecting to, or using netcat: +``` +$ nc xxx.xxx.xxx.xxx 22 +SSH-2.0-OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.10 +``` + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: no key found + +The key file specified as `ssh_key_path` cannot be accessed. Make sure that you specified the private key file (not the public key, `.pub`), and that the user that is running the `rke` command can access the private key file. + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain + +The key file specified as `ssh_key_path` is not correct for accessing the node. Double-check if you specified the correct `ssh_key_path` for the node and if you specified the correct user to connect with. + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: cannot decode encrypted private keys + +If you want to use encrypted private keys, you should use `ssh-agent` to load your keys with your passphrase. If the `SSH_AUTH_SOCK` environment variable is found in the environment where the `rke` command is run, it will be used automatically to connect to the node. + +### Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? + +The node is not reachable on the configured `address` and `port`. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md new file mode 100644 index 00000000000..4d276be3e23 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md @@ -0,0 +1,68 @@ +--- +title: Upgrade a Hardened Custom/Imported Cluster to Kubernetes v1.25 +--- + +Kubernetes v1.25 changes how clusters describe and implement security policies. From this version forward, [Pod Security Policies (PSPs)](https://kubernetes.io/docs/concepts/security/pod-security-policy/) are no longer available. Kubernetes v1.25 replaces them with new security objects: [Pod Security Standards (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/), and [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/). + +If you have custom or imported hardened clusters, you must take special preparations to ensure that the upgrade from an earlier version of Kubernetes to v1.25 or later goes smoothly. + +:::note + +After you upgrade to v1.25, add the necessary Rancher namespace exemptions. See [Pod Security Admission (PSA) Configuration Templates](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md#exempting-required-rancher-namespaces) for more details. + +::: + +## Upgrading Imported Hardened Clusters to Kubernetes v1.25 or Later + + + + +Perform the following on each node in the cluster: +1. Save [`rancher-psact.yaml`](./rancher-psact.yaml) in `/etc/rancher/rke2`. +1. Edit the RKE2 configuration file: + 1. Update the `profile` field to `cis-1.23`. + 1. Specify the path for the configuration file that you just added: `pod-security-admission-config-file: /etc/rancher/rke2/rancher-psact.yaml`. + + + + +Perform the following on each node in the cluster: + +Follow the official K3s instructions on [Upgrading Hardened Clusters from v1.24.x to v1.25.x](https://docs.k3s.io/known-issues#hardened-125), but use a [custom](./rancher-psact.yaml) Rancher PSA configuration template, instead of the configuration provided on the official K3s site. + + + +After you perform these steps, you can upgrade the cluster's Kubernetes version through the Rancher UI: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config**. +1. In the **Kubernetes Version** dropdown menu, select the version that you would like to use. +1. Click **Save**. + +## Upgrading Custom Hardened Clusters to Kubernetes v1.25 or Later + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config**. +1. Under **Basics > Security**, in the **CIS Profile** dropdown menu, select `cis-1.23`. +1. In the **Pod Security Admission Configuration Template** dropdown menu, select `rancher-restricted`. +1. In the **Kubernetes Version** dropdown menu, select the version that you would like to use. +1. Click **Save**. + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit YAML**. +1. Delete `PodSecurityPolicy` from `kube-apiserver-arg.enable-admission-plugins` +1. Add this line to the `spec` field: `defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted` +1. Update `kubernetesVersion` to your chosen version (v1.25 or later). +1. Click **Save**. + + + diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md new file mode 100644 index 00000000000..7d5f62e329e --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md @@ -0,0 +1,211 @@ +--- +title: Upgrades +--- + + + + + +The following instructions will guide you through upgrading a Rancher server that was installed on a Kubernetes cluster with Helm. These steps also apply to air-gapped installs with Helm. + +For the instructions to upgrade Rancher installed with Docker, refer to [this page.](../other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md) + +To upgrade the components in your Kubernetes cluster, or the definition of the [Kubernetes services](https://rancher.com/docs/rke/latest/en/config-options/services/) or [add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/), refer to the [upgrade documentation for RKE](https://rancher.com/docs/rke/latest/en/upgrades/), the Rancher Kubernetes Engine. + + +## Prerequisites + +### Access to kubeconfig + +Helm should be run from the same location as your kubeconfig file, or the same location where you run your kubectl commands from. + +If you installed Kubernetes with RKE, the config will have been created in the directory you ran `rke up` in. + +The kubeconfig can also be manually targeted for the intended cluster with the `--kubeconfig` tag (see: https://helm.sh/docs/helm/helm/) + +### Review Known Issues + +Review the list of known issues for each Rancher version, which can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums.](https://forums.rancher.com/c/announcements/12) + +Note that upgrades _to_ or _from_ any chart in the [rancher-alpha repository](../resources/choose-a-rancher-version.md#helm-chart-repositories) aren't supported. +### Helm Version + +The upgrade instructions assume you are using Helm 3. + +For migration of installs started with Helm 2, refer to the official [Helm 2 to 3 migration docs.](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/) The [Helm 2 upgrade page here](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/helm2.md)provides a copy of the older upgrade instructions that used Helm 2, and it is intended to be used if upgrading to Helm 3 is not feasible. + +### For air-gapped installs: Populate private registry + +For [air-gapped installs only,](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) collect and populate images for the new Rancher server version. Follow the guide to [populate your private registry](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md) with the images for the Rancher version that you want to upgrade to. + +### For upgrades with cert-manager older than 0.8.0 + +[Let's Encrypt will be blocking cert-manager instances older than 0.8.0 starting November 1st 2019.](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) Upgrade cert-manager to the latest version by following [these instructions.](../resources/upgrade-cert-manager.md) + +## Upgrade Outline + +Follow the steps to upgrade Rancher server: + + +### 1. Back up Your Kubernetes Cluster that is Running Rancher Server + +Use the [backup application](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) to back up Rancher. + +You'll use the backup as a restore point if something goes wrong during upgrade. + +### 2. Update the Helm chart repository + +1. Update your local helm repo cache. + + ``` + helm repo update + ``` + +1. Get the repository name that you used to install Rancher. + + For information about the repos and their differences, see [Helm Chart Repositories](../resources/choose-a-rancher-version.md#helm-chart-repositories). + + - Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + + ``` + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + + :::note + + If you want to switch to a different Helm chart repository, please follow the [steps on how to switch repositories](../resources/choose-a-rancher-version.md#switching-to-a-different-helm-chart-repository). If you switch repositories, make sure to list the repositories again before continuing onto Step 3 to ensure you have the correct one added. + + ::: + +1. Fetch the latest chart to install Rancher from the Helm chart repository. + + This command will pull down the latest charts and save it in the current directory as a `.tgz` file. + + ```plain + helm fetch rancher-/rancher + ``` + You can fetch the chart for the specific version you are upgrading to by adding in the `--version=` tag. For example: + + ```plain + helm fetch rancher-/rancher --version=2.6.8 + ``` + +### 3. Upgrade Rancher + +This section describes how to upgrade normal (Internet-connected) or air-gapped installations of Rancher with Helm. + +:::note Air Gap Instructions: + +If you are installing Rancher in an air-gapped environment, skip the rest of this page and render the Helm template by following the instructions on [this page.](air-gapped-upgrades.md) + +::: + + +Get the values, which were passed with `--set`, from the current Rancher Helm chart that is installed. + +``` +helm get values rancher -n cattle-system + +hostname: rancher.my.org +``` + +:::note + +There will be more values that are listed with this command. This is just an example of one of the values. + +::: + +:::tip + +Your deployment name may vary; for example, if you're deploying Rancher through the AWS Marketplace, the deployment name is 'rancher-stable'. +Thus: +``` +helm get values rancher-stable -n cattle-system + +hostname: rancher.my.org +``` + +::: + +If you are upgrading cert-manager to the latest version from v1.5 or below, follow the [cert-manager upgrade docs](../resources/upgrade-cert-manager.md#option-c-upgrade-cert-manager-from-versions-15-and-below) to learn how to upgrade cert-manager without needing to perform an uninstall or reinstall of Rancher. Otherwise, follow the [steps to upgrade Rancher](#steps-to-upgrade-rancher) below. + +#### Steps to Upgrade Rancher + +Upgrade Rancher to the latest version with all your settings. + +Take all the values from the previous step and append them to the command using `--set key=value`. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` + +:::note + +The above is an example, there may be more values from the previous step that need to be appended. + +::: + +:::tip + +If you deploy Rancher through the AWS Marketplace, the deployment name is 'rancher-stable'. +Thus: +``` +helm upgrade rancher-stable rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` + +::: + +Alternatively, it's possible to export the current values to a file and reference that file during upgrade. For example, to only change the Rancher version: + +1. Export the current values to a file: + ``` + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` +1. Update only the Rancher version: + + For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + + ``` + helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version=2.6.8 + ``` + +### 4. Verify the Upgrade + +Log into Rancher to confirm that the upgrade succeeded. + +:::tip + +Having network issues following upgrade? + +See [Restoring Cluster Networking](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md). + +::: + +## Known Upgrade Issues + +A list of known issues for each Rancher version can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums.](https://forums.rancher.com/c/announcements/12) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/feature-flags.md new file mode 100644 index 00000000000..377d8111523 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -0,0 +1,43 @@ +--- +title: Feature Flags +--- + + + + + +With feature flags, you can try out optional or experimental features, and enable legacy features that are being phased out. + +To learn more about feature values and how to enable them, see [Enabling Experimental Features](../../../pages-for-subheaders/enable-experimental-features.md). + +:::note + +Some feature flags require a restart of the Rancher container. Features that require a restart are marked in the Rancher UI. + +::: + +The following is a list of feature flags available in Rancher. If you've upgraded from a previous Rancher version, you may see additional flags in the Rancher UI, such as `proxy` or `dashboard` (both [discontinued](/versioned_docs/version-2.5/reference-guides/installation-references/feature-flags.md)): + +- `continuous-delivery`: Allows Fleet GitOps to be disabled separately from Fleet. See [Continuous Delivery.](../../../how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md) for more information. +- `fleet`: The Rancher provisioning framework in v2.6 and later requires Fleet. The flag will be automatically enabled when you upgrade, even if you disabled this flag in an earlier version of Rancher. See [Fleet - GitOps at Scale](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) for more information. +- `harvester`: Manages access to the Virtualization Management page, where users can navigate directly to Harvester clusters and access the Harvester UI. See [Harvester Integration](../../../integrations-in-rancher/harvester.md) for more information. +- `istio-virtual-service-ui`: Enables a [visual interface](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md) to create, read, update, and delete Istio virtual services and destination rules, which are Istio traffic management features. +- `legacy`: Enables a set of features from 2.5.x and earlier, that are slowly being phased out in favor of newer implementations. These are a mix of deprecated features as well as features that will eventually be available to newer versions. This flag is disabled by default on new Rancher installations. If you're upgrading from a previous version of Rancher, this flag is enabled. +- `multi-cluster-management`: Allows multi-cluster provisioning and management of Kubernetes clusters. This flag can only be set at install time. It can't be enabled or disabled later. +- `rke1-custom-node-cleanup`: Enables cleanup of deleted RKE1 custom nodes. We recommend that you keep this flag enabled, to prevent removed nodes from attempting to rejoin the cluster. +- `rke2`: Enables provisioning RKE2 clusters. This flag is enabled by default. +- `token-hashing`: Enables token hashing. Once enabled, existing tokens will be hashed and all new tokens will be hashed automatically with the SHA256 algorithm. Once a token is hashed it can't be undone. This flag can't be disabled after its enabled. See [API Tokens](../../../reference-guides/about-the-api/api-tokens.md#token-hashing) for more information. +- `unsupported-storage-drivers`: Enables types for storage providers and provisioners that aren't enabled by default. See [Allow Unsupported Storage Drivers](../../../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) for more information. + +The following table shows the availability and default values for some feature flags in Rancher. Features marked "GA" are generally available: + +| Feature Flag Name | Default Value | Status | Available As Of | +| ----------------------------- | ------------- | ------------ | --------------- | +| `continuous-delivery` | `true` | GA | v2.6.0 | +| `fleet` | `true` | Can no longer be disabled | v2.6.0 | +| `fleet` | `true` | GA | v2.5.0 | +| `harvester` | `true` | Experimental | v2.6.1 | +| `legacy` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | +| `rke1-custom-node-cleanup`| `true` | GA | v2.6.0 | +| `rke2` | `true` | Experimental | v2.6.0 | +| `token-hashing` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md new file mode 100644 index 00000000000..6e0f7263d1c --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md @@ -0,0 +1,314 @@ +--- +title: Rancher Helm Chart Options +keywords: [rancher helm chart, rancher helm options, rancher helm chart options, helm chart rancher, helm options rancher, helm chart options rancher] +--- + + + + + +This page is a configuration reference for the Rancher Helm chart. + +For help choosing a Helm chart version, refer to [this page.](../../../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md) + +For information on enabling experimental features, refer to [this page.](../../../pages-for-subheaders/enable-experimental-features.md) + +## Common Options + +| Option | Default Value | Description | +| ------------------------- | ------------- | ---------------------------------------------------------------------------------- | +| `bootstrapPassword` | " " | `string` - Set the [bootstrap password](#bootstrap-password) for the first admin user. After logging in, the admin will need to reset their password. A randomly generated bootstrap password is used if this value is not set. +| `hostname` | " " | `string` - the Fully Qualified Domain Name for your Rancher Server | +| `ingress.tls.source` | "rancher" | `string` - Where to get the cert for the ingress. - "rancher, letsEncrypt, secret" | +| `letsEncrypt.email` | " " | `string` - Your email address | +| `letsEncrypt.environment` | "production" | `string` - Valid options: "staging, production" | +| `privateCA` | false | `bool` - Set to true if your cert is signed by a private CA | + +
+ +## Advanced Options + +| Option | Default Value | Description | +| ------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `additionalTrustedCAs` | false | `bool` - See [Additional Trusted CAs](#additional-trusted-cas) | +| `addLocal` | "true" | `string` - Have Rancher detect and import the "local" Rancher server cluster. _Note: This option is no longer available in v2.5.0. Consider using the `restrictedAdmin` option to prevent users from modifying the local cluster._ | +| `antiAffinity` | "preferred" | `string` - AntiAffinity rule for Rancher pods - "preferred, required" | +| `auditLog.destination` | "sidecar" | `string` - Stream to sidecar container console or hostPath volume - "sidecar, hostPath" | +| `auditLog.hostPath` | "/var/log/rancher/audit" | `string` - log file destination on host (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.level` | 0 | `int` - set the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) level. 0 is off. [0-3] | +| `auditLog.maxAge` | 1 | `int` - maximum number of days to retain old audit log files (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.maxBackup` | 1 | `int` - maximum number of audit log files to retain (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.maxSize` | 100 | `int` - maximum size in megabytes of the audit log file before it gets rotated (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.image.repository` | "registry.suse.com/bci/bci-micro" | `string` - Location for the image used to collect audit logs. | +| `auditLog.image.tag` | "15.4.14.3" | `string` - Tag for the image used to collect audit logs. | +| `auditLog.image.pullPolicy` | "IfNotPresent" | `string` - Override imagePullPolicy for auditLog images - "Always", "Never", "IfNotPresent". | +| `busyboxImage` | "" | `string` - Image location for busybox image used to collect audit logs. _Note: This option is deprecated use `auditLog.image.repository` to control auditing sidecar image._ | +| `certmanager.version` | "" | `string` - set cert-manager compatibility | +| `debug` | false | `bool` - set debug flag on rancher server | +| `extraEnv` | [] | `list` - set additional environment variables for Rancher | +| `imagePullSecrets` | [] | `list` - list of names of Secret resource containing private registry credentials | +| `ingress.configurationSnippet` | "" | `string` - Add additional Nginx configuration. Can be used for proxy configuration. | +| `ingress.extraAnnotations` | {} | `map` - additional annotations to customize the ingress | +| `ingress.enabled` | true | When set to false, Helm will not install a Rancher ingress. Set the option to false to deploy your own ingress. | +| `letsEncrypt.ingress.class` | "" | `string` - optional ingress class for the cert-manager acmesolver ingress that responds to the Let's Encrypt ACME challenges. Options: traefik, nginx. | | +| `noProxy` | "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local,cattle-system.svc" | `string` - comma separated list of hostnames or ip address not to use the proxy | | +| `proxy` | "" | `string` - HTTP[S] proxy server for Rancher | +| `rancherImage` | "rancher/rancher" | `string` - rancher image source | +| `rancherImagePullPolicy` | "IfNotPresent" | `string` - Override imagePullPolicy for rancher server images - "Always", "Never", "IfNotPresent" | +| `rancherImageTag` | same as chart version | `string` - rancher/rancher image tag | +| `replicas` | 3 | `int` - Number of Rancher server replicas. Setting to -1 will dynamically choose 1, 2, or 3 based on the number of available nodes in the cluster. | +| `resources` | {} | `map` - rancher pod resource requests & limits | +| `restrictedAdmin` | `false` | `bool` - When this option is set to true, the initial Rancher user has restricted access to the local Kubernetes cluster to prevent privilege escalation. For more information, see the section about the [restricted-admin role.](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#restricted-admin) | +| `systemDefaultRegistry` | "" | `string` - private registry to be used for all system container images, e.g., http://registry.example.com/ | +| `tls` | "ingress" | `string` - See [External TLS Termination](#external-tls-termination) for details. - "ingress, external" | +| `useBundledSystemChart` | `false` | `bool` - select to use the system-charts packaged with Rancher server. This option is used for air gapped installations. | +| `global.cattle.psp.enabled` | `true` | `bool` - select 'false' to disable PSPs for Kubernetes v1.25 and above when using Rancher v2.7.2-v2.7.4. When using Rancher v2.7.5 and above, Rancher attempts to detect if a cluster is running a Kubernetes version where PSPs are not supported, and will default it's usage of PSPs to false if it can determine that PSPs are not supported in the cluster. Users can still manually override this by explicitly providing `true` or `false` for this value. Rancher will still use PSPs by default in clusters which support PSPs (such as clusters running Kubernetes v1.24 or lower).| + + +### Bootstrap Password + +When Rancher starts for the first time, a password is randomly generated for the first admin user. When the admin first logs in to Rancher, the UI shows commands that can be used to retrieve the bootstrap password. The admin needs to run those commands and log in with the bootstrap password. Then Rancher gives the admin an opportunity to reset the password. + +If you want to use a specific bootstrap password instead of a randomly generated one, provide the password. + +```plain +--set bootstrapPassword="rancher" +``` + +The password, whether provided or generated, will be stored in a Kubernetes secret. After Rancher is installed, the UI will show instructions for how to retrieve the password using kubectl: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### API Audit Log + +Enabling the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md). + +You can collect this log as you would any container log. Enable [logging](../../../pages-for-subheaders/logging.md) for the `System` Project on the Rancher server cluster. + +```plain +--set auditLog.level=1 +``` + +By default enabling Audit Logging will create a sidecar container in the Rancher pod. This container (`rancher-audit-log`) will stream the log to `stdout`. You can collect this log as you would any container log. When using the sidecar as the audit log destination, the `hostPath`, `maxAge`, `maxBackups`, and `maxSize` options do not apply. It's advised to use your OS or Docker daemon's log rotation features to control disk space use. Enable [logging](../../../pages-for-subheaders/logging.md) for the Rancher server cluster or System Project. + +Set the `auditLog.destination` to `hostPath` to forward logs to volume shared with the host system instead of streaming to a sidecar container. When setting the destination to `hostPath` you may want to adjust the other auditLog parameters for log rotation. + +### Setting Extra Environment Variables + +You can set extra environment variables for Rancher server using `extraEnv`. This list is passed to the Rancher deployment in its YAML format. It is embedded under `env` for the Rancher container. Refer to the Kubernetes documentation for setting container environment variables, `extraEnv` can use any of the keys referenced in [Define Environment Variables for a Container](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container). + +Consider an example that uses the `name` and `value` keys: + +```plain +--set 'extraEnv[0].name=CATTLE_TLS_MIN_VERSION' +--set 'extraEnv[0].value=1.0' +``` + +If passing sensitive data as the value for an environment variable, such as proxy authentication credentials, it is strongly recommended that a secret reference is used. This will prevent sensitive data from being exposed in Helm or the Rancher deployment. + +Consider an example that uses the `name`, `valueFrom.secretKeyRef.name`, and `valueFrom.secretKeyRef.key` keys. See example in [HTTP Proxy](#http-proxy) + +### TLS Settings + +When you install Rancher inside of a Kubernetes cluster, TLS is offloaded at the cluster's ingress controller. The possible TLS settings depend on the used ingress controller. + +See [TLS settings](tls-settings.md) for more information and options. + +### Import `local` Cluster + +By default Rancher server will detect and import the `local` cluster it's running on. User with access to the `local` cluster will essentially have "root" access to all the clusters managed by Rancher server. + +:::caution + +If you turn addLocal off, most Rancher v2.5 features won't work, including the EKS provisioner. + +::: + +If this is a concern in your environment you can set this option to "false" on your initial install. + +This option is only effective on the initial Rancher install. See [Issue 16522](https://github.com/rancher/rancher/issues/16522) for more information. + +```plain +--set addLocal="false" +``` + +### Customizing your Ingress + +To customize or use a different ingress with Rancher server you can set your own Ingress annotations. + +Example on setting a custom certificate issuer: + +```plain +--set ingress.extraAnnotations.'cert-manager\.io/cluster-issuer'=issuer-name +``` + +Example on setting a static proxy header with `ingress.configurationSnippet`. This value is parsed like a template so variables can be used. + +```plain +--set ingress.configurationSnippet='more_set_input_headers X-Forwarded-Host {{ .Values.hostname }};' +``` + +### HTTP Proxy + +Rancher requires internet access for some functionality (Helm charts). Use `proxy` to set your proxy server or use `extraEnv` to set the `HTTPS_PROXY` environment variable to point to your proxy server. + +Add your IP exceptions to the `noProxy` chart value as a comma separated list. Make sure you add the following values: +- Pod cluster IP range (default: `10.42.0.0/16`). +- Service cluster IP range (default: `10.43.0.0/16`). +- Internal cluster domains (default: `.svc,.cluster.local`). +- Any worker cluster `controlplane` nodes. +Rancher supports CIDR notation ranges in this list. + +When not including sensitive data, the `proxy` or `extraEnv` chart options can be used. When using `extraEnv` the `noProxy` Helm option is ignored. Therefore, the `NO_PROXY` environment variable must also be set with `extraEnv`. + +The following is an example of setting proxy using the `extraEnv` chart option: + +```plain +--set proxy="http:///" +``` + +Example of setting proxy using the `extraEnv` chart option: +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].value="http://:/" +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +When including sensitive data, such as proxy authentication credentials, use the `extraEnv` option with `valueFrom.secretRef` to prevent sensitive data from being exposed in Helm or the Rancher deployment. + +The following is an example of using `extraEnv` to configure proxy. This example secret would contain the value `"http://:@:/"` in the secret's `"https-proxy-url"` key: +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].valueFrom.secretKeyRef.name=secret-name +--set extraEnv[1].valueFrom.secretKeyRef.key=https-proxy-url +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +To learn more about how to configure environment variables, refer to [Define Environment Variables for a Container](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container). + +### Additional Trusted CAs + +If you have private registries, catalogs or a proxy that intercepts certificates, you may need to add additional trusted CAs to Rancher. + +```plain +--set additionalTrustedCAs=true +``` + +Once the Rancher deployment is created, copy your CA certs in pem format into a file named `ca-additional.pem` and use `kubectl` to create the `tls-ca-additional` secret in the `cattle-system` namespace. + +```plain +kubectl -n cattle-system create secret generic tls-ca-additional --from-file=ca-additional.pem=./ca-additional.pem +``` + +### Private Registry and Air Gap Installs + +For details on installing Rancher with a private registry, see the [air gap installation docs.](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) + +## External TLS Termination + +We recommend configuring your load balancer as a Layer 4 balancer, forwarding plain 80/tcp and 443/tcp to the Rancher Management cluster nodes. The Ingress Controller on the cluster will redirect http traffic on port 80 to https on port 443. + +You may terminate the SSL/TLS on a L7 load balancer external to the Rancher cluster (ingress). Use the `--set tls=external` option and point your load balancer at port http 80 on all of the Rancher cluster nodes. This will expose the Rancher interface on http port 80. Be aware that clients that are allowed to connect directly to the Rancher cluster will not be encrypted. If you choose to do this we recommend that you restrict direct access at the network level to just your load balancer. + +:::note + +If you are using a Private CA signed certificate, add `--set privateCA=true` and see [Adding TLS Secrets - Using a Private CA Signed Certificate](../../../getting-started/installation-and-upgrade/resources/add-tls-secrets.md) to add the CA cert for Rancher. + +::: + +Your load balancer must support long lived websocket connections and will need to insert proxy headers so Rancher can route links correctly. + +### Configuring Ingress for External TLS when Using NGINX v0.25 + +In NGINX v0.25, the behavior of NGINX has [changed](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0220) regarding forwarding headers and external TLS termination. Therefore, in the scenario that you are using external TLS termination configuration with NGINX v0.25, you must edit the `cluster.yml` to enable the `use-forwarded-headers` option for ingress: + +```yaml +ingress: + provider: nginx + options: + use-forwarded-headers: 'true' +``` + +### Required Headers + +- `Host` +- `X-Forwarded-Proto` +- `X-Forwarded-Port` +- `X-Forwarded-For` + +### Recommended Timeouts + +- Read Timeout: `1800 seconds` +- Write Timeout: `1800 seconds` +- Connect Timeout: `30 seconds` + +### Health Checks + +Rancher will respond `200` to health checks on the `/healthz` endpoint. + +### Example NGINX config + +This NGINX configuration is tested on NGINX 1.14. + +:::caution + +This NGINX configuration is only an example and may not suit your environment. For complete documentation, see [NGINX Load Balancing - HTTP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/). + +::: + +- Replace `IP_NODE1`, `IP_NODE2` and `IP_NODE3` with the IP addresses of the nodes in your cluster. +- Replace both occurrences of `FQDN` to the DNS name for Rancher. +- Replace `/certs/fullchain.pem` and `/certs/privkey.pem` to the location of the server certificate and the server certificate key respectively. + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server IP_NODE_1:80; + server IP_NODE_2:80; + server IP_NODE_3:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # This allows the ability for the execute shell window to remain open for up to 15 minutes. Without this parameter, the default is 1 minute and will automatically close. + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/tls-settings.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/tls-settings.md new file mode 100644 index 00000000000..321cb524dbf --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-references/tls-settings.md @@ -0,0 +1,25 @@ +--- +title: TLS Settings +--- + + + + + +Changing the default TLS settings depends on the chosen installation method. + +## Running Rancher in a highly available Kubernetes cluster + +When you install Rancher inside of a Kubernetes cluster, TLS is offloaded at the cluster's ingress controller. The possible TLS settings depend on the used ingress controller: + +* nginx-ingress-controller (default for RKE1 and RKE2): [Default TLS Version and Ciphers](https://kubernetes.github.io/ingress-nginx/user-guide/tls/#default-tls-version-and-ciphers). +* traefik (default for K3s): [TLS Options](https://doc.traefik.io/traefik/https/tls/#tls-options). + +## Running Rancher in a single Docker container + +The default TLS configuration only accepts TLS 1.2 and secure TLS cipher suites. You can change this by setting the following environment variables: + +| Parameter | Description | Default | Available options | +|-----|-----|-----|-----| +| `CATTLE_TLS_MIN_VERSION` | Minimum TLS version | `1.2` | `1.0`, `1.1`, `1.2`, `1.3` | +| `CATTLE_TLS_CIPHERS` | Allowed TLS cipher suites | `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305`,
`TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305` | See [Golang tls constants](https://golang.org/pkg/crypto/tls/#pkg-constants) | diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/dockershim.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/dockershim.md new file mode 100644 index 00000000000..e215e0cfc2e --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/dockershim.md @@ -0,0 +1,47 @@ +--- +title: Dockershim +--- + +The Dockershim is the CRI compliant layer between the Kubelet and the Docker daemon. As part of the Kubernetes 1.20 release, the [deprecation of the in-tree Dockershim was announced](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/). For more information on the deprecation and its timelines, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). + +RKE clusters now support the external Dockershim to continue leveraging Docker as the CRI runtime. We now implement the upstream open source community external Dockershim announced by [Mirantis and Docker](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) to ensure RKE clusters can continue to leverage Docker. + +RKE2 and K3s clusters use an embedded containerd as a container runtime and are not affected. + +To enable the external Dockershim in versions of RKE before 1.24, configure the following option. + +``` +enable_cri_dockerd: true +``` + +Starting with version 1.24, the above defaults to true. + +For users looking to use another container runtime, Rancher has the edge-focused K3s and datacenter-focused RKE2 Kubernetes distributions that use containerd as the default runtime. Imported RKE2 and K3s Kubernetes clusters can then be upgraded and managed through Rancher going forward. + +### FAQ + +
+ +Q: Do I have to upgrade Rancher to get Rancher’s support of the upstream external Dockershim replacement? + +A: The upstream support of the Dockershim replacement `cri_dockerd` begins for RKE in Kubernetes 1.21. You will need to be on a version of Rancher that supports RKE 1.21. See our support matrix for details. + +
+ +Q: I am currently on RKE with Kubernetes 1.23. What happens when upstream finally removes Dockershim in 1.24? + +A: The version of Dockershim in RKE with Kubernetes will continue to work until 1.23. For information on the timeline, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). After this, starting in 1.24, RKE will default to enabling `cri_dockerd` by default and will continue to do for versions afterwards. + +
+ +Q: What are my other options if I don’t want to depend on the Dockershim or cri_dockerd? + +A: You can use a runtime like containerd with Kubernetes that does not require Dockershim support. RKE2 or K3s are two options for doing this. + +
+ +Q: If I am already using RKE1 and want to switch to RKE2, what are my migration options? + +A: Today, you can stand up a new cluster and migrate workloads to a new RKE2 cluster that uses containerd. Rancher is exploring the possibility of an in-place upgrade path. + +
diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/install-docker.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/install-docker.md new file mode 100644 index 00000000000..89c45c55e79 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/install-docker.md @@ -0,0 +1,27 @@ +--- +title: Installing Docker +--- + + + + + +Docker is required to be installed on nodes where the Rancher server will be installed with Helm on an RKE cluster or with Docker. Docker is not required for RKE2 or K3s clusters. + +There are a couple of options for installing Docker. One option is to refer to the [official Docker documentation](https://docs.docker.com/install/) about how to install Docker on Linux. The steps will vary based on the Linux distribution. + +Another option is to use one of Rancher's Docker installation scripts, which are available for most recent versions of Docker. + +For example, this command could be used to install Docker 20.10 on the main Linux distributions like SUSE Linux Enterprise or Ubuntu: + +``` +curl https://releases.rancher.com/install-docker/20.10.sh | sh +``` + +Rancher has installation scripts for every version of upstream Docker that Kubernetes supports. To find out whether a script is available for installing a certain Docker version, refer to this [GitHub repository,](https://github.com/rancher/install-docker) which contains all of Rancher's Docker installation scripts. + +Note that the following sysctl setting must be applied: + +``` +net.bridge.bridge-nf-call-iptables=1 +``` diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md new file mode 100644 index 00000000000..73c739499a5 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md @@ -0,0 +1,345 @@ +--- +title: Port Requirements +description: Read about port requirements needed in order for Rancher to operate properly, both for Rancher nodes and downstream Kubernetes cluster nodes +--- + + + + + +import PortsIaasNodes from '@site/src/components/PortsIaasNodes' +import PortsCustomNodes from '@site/src/components/PortsCustomNodes' +import PortsImportedHosted from '@site/src/components/PortsImportedHosted' + +To operate properly, Rancher requires a number of ports to be open on Rancher nodes and on downstream Kubernetes cluster nodes. + +## Rancher Nodes + +The following table lists the ports that need to be open to and from nodes that are running the Rancher server. + +The port requirements differ based on the Rancher server architecture. + +Rancher can be installed on any Kubernetes cluster. For Rancher installs on a K3s, RKE, or RKE2 Kubernetes cluster, refer to the tabs below. For other Kubernetes distributions, refer to the distribution's documentation for the port requirements for cluster nodes. + +:::note Notes: + +- Rancher nodes may also require additional outbound access for any external authentication provider which is configured (LDAP for example). +- Kubernetes recommends TCP 30000-32767 for node port services. +- For firewalls, traffic may need to be enabled within the cluster and pod CIDR. +- Rancher nodes may also need outbound access to an external S3 location which is used for storing cluster backups (Minio for example). + +::: + +### Ports for Rancher Server Nodes on K3s + +
+ Click to expand + +The K3s server needs port 6443 to be accessible by the nodes. + +The nodes need to be able to reach other nodes over UDP port 8472 when Flannel VXLAN is used. The node should not listen on any other port. K3s uses reverse tunneling such that the nodes make outbound connections to the server and all kubelet traffic runs through that tunnel. However, if you do not use Flannel and provide your own custom CNI, then port 8472 is not needed by K3s. + +If you wish to utilize the metrics server, you will need to open port 10250 on each node. + +:::note Important: + +The VXLAN port on nodes should not be exposed to the world as it opens up your cluster network to be accessed by anyone. Run your nodes behind a firewall/security group that disables access to port 8472. + +::: + +The following tables break down the port requirements for inbound and outbound traffic: + +
Inbound Rules for Rancher Server Nodes
+ +| Protocol | Port | Source | Description +|-----|-----|----------------|---| +| TCP | 80 | Load balancer/proxy that does external SSL termination | Rancher UI/API when external SSL termination is used | +| TCP | 443 |
  • server nodes
  • agent nodes
  • hosted/registered Kubernetes
  • any source that needs to be able to use the Rancher UI or API
| Rancher agent, Rancher UI/API, kubectl | +| TCP | 6443 | K3s server nodes | Kubernetes API +| UDP | 8472 | K3s server and agent nodes | Required only for Flannel VXLAN. +| TCP | 10250 | K3s server and agent nodes | kubelet + +
Outbound Rules for Rancher Nodes
+ +| Protocol | Port | Destination | Description | +| -------- | ---- | -------------------------------------------------------- | --------------------------------------------- | +| TCP | 22 | Any node IP from a node created using Node Driver | SSH provisioning of nodes using Node Driver | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | Any node IP from a node created using Node driver | Docker daemon TLS port used by Docker Machine | +| TCP | 6443 | Hosted/Imported Kubernetes API | Kubernetes API server | + +
+ +### Ports for Rancher Server Nodes on RKE + +
+ Click to expand + +Typically Rancher is installed on three RKE nodes that all have the etcd, control plane and worker roles. + +The following tables break down the port requirements for traffic between the Rancher nodes: + +
Rules for traffic between Rancher nodes
+ +| Protocol | Port | Description | +|-----|-----|----------------| +| TCP | 443 | Rancher agents | +| TCP | 2379 | etcd client requests | +| TCP | 2380 | etcd peer communication | +| TCP | 6443 | Kubernetes apiserver | +| TCP | 8443 | Nginx Ingress's Validating Webhook | +| UDP | 8472 | Canal/Flannel VXLAN overlay networking | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 10250 | Metrics server communication with all nodes | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | + +The following tables break down the port requirements for inbound and outbound traffic: + +
Inbound Rules for Rancher Nodes
+ +| Protocol | Port | Source | Description | +|-----|-----|----------------|---| +| TCP | 22 | RKE CLI | SSH provisioning of node by RKE | +| TCP | 80 | Load Balancer/Reverse Proxy | HTTP traffic to Rancher UI/API | +| TCP | 443 |
  • Load Balancer/Reverse Proxy
  • IPs of all cluster nodes and other API/UI clients
| HTTPS traffic to Rancher UI/API | +| TCP | 6443 | Kubernetes API clients | HTTPS traffic to Kubernetes API | + +
Outbound Rules for Rancher Nodes
+ +| Protocol | Port | Destination | Description | +|-----|-----|----------------|---| +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 22 | Any node created using a node driver | SSH provisioning of node by node driver | +| TCP | 2376 | Any node created using a node driver | Docker daemon TLS port used by node driver | +| TCP | 6443 | Hosted/Imported Kubernetes API | Kubernetes API server | +| TCP | Provider dependent | Port of the Kubernetes API endpoint in hosted cluster | Kubernetes API | + +
+ +### Ports for Rancher Server Nodes on RKE2 + +
+ Click to expand + +The RKE2 server needs port 6443 and 9345 to be accessible by other nodes in the cluster. + +All nodes need to be able to reach other nodes over UDP port 8472 when Flannel VXLAN is used. + +If you wish to utilize the metrics server, you will need to open port 10250 on each node. + +:::note Important: + +The VXLAN port on nodes should not be exposed to the world as it opens up your cluster network to be accessed by anyone. Run your nodes behind a firewall/security group that disables access to port 8472. + +::: + +
Inbound Rules for RKE2 Server Nodes
+ +| Protocol | Port | Source | Description +|-----|-----|----------------|---| +| TCP | 9345 | RKE2 server and agent nodes | Node registration. Port should be open on all server nodes to all other nodes in the cluster. +| TCP | 6443 | RKE2 agent nodes | Kubernetes API +| UDP | 8472 | RKE2 server and agent nodes | Required only for Flannel VXLAN +| TCP | 10250 | RKE2 server and agent nodes | kubelet +| TCP | 2379 | RKE2 server nodes | etcd client port +| TCP | 2380 | RKE2 server nodes | etcd peer port +| TCP | 30000-32767 | RKE2 server and agent nodes | NodePort port range. Can use TCP or UDP. +| TCP | 5473 | Calico-node pod connecting to typha pod | Required when deploying with Calico +| HTTP | 80 | Load balancer/proxy that does external SSL termination | Rancher UI/API when external SSL termination is used | +| HTTPS | 443 |
  • hosted/registered Kubernetes
  • any source that needs to be able to use the Rancher UI or API
| Rancher agent, Rancher UI/API, kubectl. Not needed if you have a load balancer doing TLS termination. | + +Typically all outbound traffic is allowed. +
+ +### Ports for Rancher Server in Docker + +
+ Click to expand + +The following tables break down the port requirements for Rancher nodes, for inbound and outbound traffic: + +
Inbound Rules for Rancher Node
+ +| Protocol | Port | Source | Description +|-----|-----|----------------|---| +| TCP | 80 | Load balancer/proxy that does external SSL termination | Rancher UI/API when external SSL termination is used +| TCP | 443 |
  • hosted/registered Kubernetes
  • any source that needs to be able to use the Rancher UI or API
| Rancher agent, Rancher UI/API, kubectl + +
Outbound Rules for Rancher Node
+ +| Protocol | Port | Source | Description | +|-----|-----|----------------|---| +| TCP | 22 | Any node IP from a node created using Node Driver | SSH provisioning of nodes using Node Driver | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | Any node IP from a node created using a node driver | Docker daemon TLS port used by Docker Machine | +| TCP | 6443 | Hosted/Imported Kubernetes API | Kubernetes API server | + +
+ +## Downstream Kubernetes Cluster Nodes + +Downstream Kubernetes clusters run your apps and services. This section describes what ports need to be opened on the nodes in downstream clusters so that Rancher can communicate with them. + +The port requirements differ depending on how the downstream cluster was launched. Each of the tabs below list the ports that need to be opened for different [cluster types](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +The following diagram depicts the ports that are opened for each [cluster type](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +
Port Requirements for the Rancher Management Plane
+ +![Basic Port Requirements](/img/port-communications.svg) + +:::tip + +If security isn't a large concern and you're okay with opening a few additional ports, you can use the table in [Commonly Used Ports](#commonly-used-ports) as your port reference instead of the comprehensive tables below. + +::: + +### Ports for Harvester Clusters + +Refer [here](../../../integrations-in-rancher/harvester.md#port-requirements) for more information on Harvester port requirements. + + +### Ports for Rancher Launched Kubernetes Clusters using Node Pools + +
+ Click to expand + +The following table depicts the port requirements for [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) with nodes created in an [Infrastructure Provider](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +:::note + +The required ports are automatically opened by Rancher during creation of clusters in cloud providers like Amazon EC2 or DigitalOcean. + +::: + + + +
+ +### Ports for Rancher Launched Kubernetes Clusters using Custom Nodes + +
+ Click to expand + +The following table depicts the port requirements for [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) with [Custom Nodes](../../../pages-for-subheaders/use-existing-nodes.md). + + + +
+ +### Ports for Hosted Kubernetes Clusters + +
+ Click to expand + +The following table depicts the port requirements for [hosted clusters](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md). + + + +
+ +### Ports for Registered Clusters + +:::note + +Registered clusters were called imported clusters before Rancher v2.5. + +::: + +
+ Click to expand + +The following table depicts the port requirements for [registered clusters](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md). + + + +
+ + +## Other Port Considerations + +### Commonly Used Ports + +These ports are typically opened on your Kubernetes nodes, regardless of what type of cluster it is. + +import CommonPortsTable from '../../../shared-files/_common-ports-table.md'; + + + +---- + +### Local Node Traffic + +Ports marked as `local traffic` (i.e., `9099 TCP`) in the above requirements are used for Kubernetes healthchecks (`livenessProbe` and`readinessProbe`). +These healthchecks are executed on the node itself. In most cloud environments, this local traffic is allowed by default. + +However, this traffic may be blocked when: + +- You have applied strict host firewall policies on the node. +- You are using nodes that have multiple interfaces (multihomed). + +In these cases, you have to explicitly allow this traffic in your host firewall, or in case of public/private cloud hosted machines (i.e. AWS or OpenStack), in your security group configuration. Keep in mind that when using a security group as source or destination in your security group, explicitly opening ports only applies to the private interface of the nodes / instances. + +### Rancher AWS EC2 Security Group + +When using the [AWS EC2 node driver](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) to provision cluster nodes in Rancher, you can choose to let Rancher create a security group called `rancher-nodes`. The following rules are automatically added to this security group. + +| Type | Protocol | Port Range | Source/Destination | Rule Type | +|-----------------|:--------:|:-----------:|------------------------|:---------:| +| SSH | TCP | 22 | 0.0.0.0/0 | Inbound | +| HTTP | TCP | 80 | 0.0.0.0/0 | Inbound | +| Custom TCP Rule | TCP | 443 | 0.0.0.0/0 | Inbound | +| Custom TCP Rule | TCP | 2376 | 0.0.0.0/0 | Inbound | +| Custom TCP Rule | TCP | 2379-2380 | sg-xxx (rancher-nodes) | Inbound | +| Custom UDP Rule | UDP | 4789 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 6443 | 0.0.0.0/0 | Inbound | +| Custom UDP Rule | UDP | 8472 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 10250-10252 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 10256 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 30000-32767 | 0.0.0.0/0 | Inbound | +| Custom UDP Rule | UDP | 30000-32767 | 0.0.0.0/0 | Inbound | +| All traffic | All | All | 0.0.0.0/0 | Outbound | + +### Opening SUSE Linux Ports + +SUSE Linux may have a firewall that blocks all ports by default. To open the ports needed for adding the host to a custom cluster, + + + + +1. SSH into the instance. +1. Start YaST in text mode: + ``` + sudo yast2 + ``` + +1. Navigate to **Security and Users** > **Firewall** > **Zones:public** > **Ports**. To navigate within the interface, follow these [instructions](https://doc.opensuse.org/documentation/leap/reference/html/book-reference/cha-yast-text.html#sec-yast-cli-navigate). +1. To open the required ports, enter them into the **TCP Ports** and **UDP Ports** fields. In this example, ports 9796 and 10250 are also opened for monitoring. The resulting fields should look similar to the following: + ```yaml + TCP Ports + 22, 80, 443, 2376, 2379, 2380, 6443, 9099, 9796, 10250, 10254, 30000-32767 + UDP Ports + 8472, 30000-32767 + ``` + +1. When all required ports are enter, select **Accept**. + + + + +1. SSH into the instance. +1. Edit /`etc/sysconfig/SuSEfirewall2` and open the required ports. In this example, ports 9796 and 10250 are also opened for monitoring: + ``` + FW_SERVICES_EXT_TCP="22 80 443 2376 2379 2380 6443 9099 9796 10250 10254 30000:32767" + FW_SERVICES_EXT_UDP="8472 30000:32767" + FW_ROUTE=yes + ``` +1. Restart the firewall with the new ports: + ``` + SuSEfirewall2 + ``` + + + + +**Result:** The node has the open ports required to be added to a custom cluster. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md new file mode 100644 index 00000000000..3c7c4900443 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md @@ -0,0 +1,149 @@ +--- +title: Docker Install Commands +--- + +The Docker installation is for Rancher users who want to test out Rancher. + +Instead of running on a Kubernetes cluster, you install the Rancher server component on a single node using a `docker run` command. Since there is only one node and a single Docker container, if the node goes down, there is no copy of the etcd data available on other nodes and you will lose all the data of your Rancher server. + +The backup application can be used to migrate the Rancher server from a Docker install to a Kubernetes install using [these steps.](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +| Environment Variable Key | Environment Variable Value | Description | +| -------------------------------- | -------------------------------- | ---- | +| `CATTLE_SYSTEM_DEFAULT_REGISTRY` | `` | Configure Rancher server to always pull from your private registry when provisioning clusters. | +| `CATTLE_SYSTEM_CATALOG` | `bundled` | Configure Rancher server to use the packaged copy of Helm system charts. The [system charts](https://github.com/rancher/system-charts) repository contains all the catalog items required for features such as monitoring, logging, alerting and global DNS. These [Helm charts](https://github.com/rancher/system-charts) are located in GitHub, but since you are in an air gapped environment, using the charts that are bundled within Rancher is much easier than setting up a Git mirror. | + +:::note Do you want to.. + +- Configure custom CA root certificate to access your services? See [Custom CA root certificate](../../resources/custom-ca-root-certificates.md). +- Record all transactions with the Rancher API? See [API Auditing](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log). + +::: + +Choose from the following options: + +### Option A: Default Self-Signed Certificate + +
+ Click to expand + +If you are installing Rancher in a development or testing environment where identity verification isn't a concern, install Rancher using the self-signed certificate that it generates. This installation option omits the hassle of generating a certificate yourself. + +Log into your Linux host, and then run the installation command below. When entering the command, use the table below to replace each placeholder. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | Your private registry URL and port. | +| `` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to install. | + +Privileged access is [required.](./install-rancher-ha.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` + +
+ +### Option B: Bring Your Own Certificate: Self-Signed + +
+ Click to expand + +In development or testing environments where your team will access your Rancher server, create a self-signed certificate for use with your install so that your team can verify they're connecting to your instance of Rancher. + +:::note Prerequisites: + +From a computer with an internet connection, create a self-signed certificate using [OpenSSL](https://www.openssl.org/) or another method of your choice. + +- The certificate files must be in PEM format. +- In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](../rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +::: + +After creating your certificate, log into your Linux host, and then run the installation command below. When entering the command, use the table below to replace each placeholder. Use the `-v` flag and provide the path to your certificates to mount them in your container. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | +| `` | The path to the certificate authority's certificate. | +| `` | Your private registry URL and port. | +| `` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to install. | + +Privileged access is [required.](./install-rancher-ha.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` + +
+ +### Option C: Bring Your Own Certificate: Signed by Recognized CA + +
+ Click to expand + +In development or testing environments where you're exposing an app publicly, use a certificate signed by a recognized CA so that your user base doesn't encounter security warnings. + +:::note Prerequisite: + +The certificate files must be in PEM format. + +::: + +After obtaining your certificate, log into your Linux host, and then run the installation command below. When entering the command, use the table below to replace each placeholder. Because your certificate is signed by a recognized CA, mounting an additional CA certificate file is unnecessary. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | +| `` | Your private registry URL and port. | +| `` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to install. | + +:::note + +Use the `--no-cacerts` as argument to the container to disable the default CA certificate generated by Rancher. + +::: + +Privileged access is [required.](./install-rancher-ha.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged + /rancher/rancher: +``` + +
+ + + +:::note + +If you don't intend to send telemetry data, opt out [telemetry](../../../../faq/telemetry.md) during the initial login. + +::: + diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md new file mode 100644 index 00000000000..f9066582004 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md @@ -0,0 +1,192 @@ +--- +title: '1. Set up Infrastructure and Private Registry' +--- + +In this section, you will provision the underlying infrastructure for your Rancher management server in an air gapped environment. You will also set up the private container image registry that must be available to your Rancher node(s). + +An air gapped environment is an environment where the Rancher server is installed offline or behind a firewall. + +The infrastructure depends on whether you are installing Rancher on a K3s Kubernetes cluster, an RKE Kubernetes cluster, or a single Docker container. For more information on each installation option, refer to [this page.](../../../../pages-for-subheaders/installation-and-upgrade.md) + +Rancher can be installed on any Kubernetes cluster. The RKE and K3s Kubernetes infrastructure tutorials below are still included for convenience. + + + + +We recommend setting up the following infrastructure for a high-availability installation: + +- **Two Linux nodes,** typically virtual machines, in the infrastructure provider of your choice. +- **An external database** to store the cluster data. PostgreSQL, MySQL, and etcd are supported. +- **A load balancer** to direct traffic to the two nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. +- **A private image registry** to distribute container images to your machines. + +### 1. Set up Linux Nodes + +These hosts will be disconnected from the internet, but require being able to connect with your private registry. + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up External Datastore + +The ability to run Kubernetes using a datastore other than etcd sets K3s apart from other Kubernetes distributions. This feature provides flexibility to Kubernetes operators. The available options allow you to select a datastore that best fits your use case. + +For a high-availability K3s installation, you will need to set up one of the following external databases: + +* [PostgreSQL](https://www.postgresql.org/) (certified against versions 10.7 and 11.5) +* [MySQL](https://www.mysql.com/) (certified against version 5.7) +* [etcd](https://etcd.io/) (certified against version 3.3.15) + +When you install Kubernetes, you will pass in details for K3s to connect to the database. + +For an example of one way to set up the database, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md) for setting up a MySQL database on Amazon's RDS service. + +For the complete list of options that are available for configuring a K3s cluster datastore, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/datastore/) + +### 3. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the K3s tool will deploy a Traefik Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the Traefik Ingress controller to listen for traffic destined for the Rancher hostname. The Traefik Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md) + +:::note Important: + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 4. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the load balancer IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) + +### 5. Set up a Private Image Registry + +Rancher supports air gap installs using a private registry. You must have your own private registry or other means of distributing container images to your machines. + +In a later step, when you set up your K3s Kubernetes cluster, you will create a [private registries configuration file](https://rancher.com/docs/k3s/latest/en/installation/private-registry/) with details from this registry. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +To install the Rancher management server on a high-availability RKE cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in an infrastructure provider such as Amazon's EC2, Google Compute Engine, or vSphere. +- **A load balancer** to direct front-end traffic to the three nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. +- **A private image registry** to distribute container images to your machines. + +These nodes must be in the same region/data center. You may place these servers in separate availability zones. + +### Why three nodes? + +In an RKE cluster, Rancher server data is stored on etcd. This etcd database runs on all three nodes. + +The etcd database requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can suffer from [split brain](https://www.quora.com/What-is-split-brain-in-distributed-systems), requiring the cluster to be restored from backup. If one of the three etcd nodes fails, the two remaining nodes can elect a leader because they have the majority of the total number of etcd nodes. + +### 1. Set up Linux Nodes + +These hosts will be disconnected from the internet, but require being able to connect with your private registry. + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE tool will deploy an NGINX Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the NGINX Ingress controller to listen for traffic destined for the Rancher hostname. The NGINX Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 3. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the LB IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) + +### 4. Set up a Private Image Registry + +Rancher supports air gap installs using a secure private registry. You must have your own private registry or other means of distributing container images to your machines. + +In a later step, when you set up your RKE Kubernetes cluster, you will create a [private registries configuration file](https://rancher.com/docs/rke/latest/en/config-options/private-registries/) with details from this registry. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +:::note Notes: + +- The Docker installation is for Rancher users that are wanting to test out Rancher. Since there is only one node and a single Docker container, if the node goes down, you will lose all the data of your Rancher server. + +- The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +::: + +### 1. Set up a Linux Node + +This host will be disconnected from the Internet, but needs to be able to connect to your private registry. + +Make sure that your node fulfills the general installation requirements for [OS, containers, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up a Private Docker Registry + +Rancher supports air gap installs using a private registry on your bastion server. You must have your own private registry or other means of distributing container images to your machines. + +If you need help with creating a private registry, please refer to the [official Docker documentation.](https://docs.docker.com/registry/). + + + + +### [Next: Collect and Publish Images to your Private Registry](publish-images.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md new file mode 100644 index 00000000000..2189e94d095 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md @@ -0,0 +1,390 @@ +--- +title: '3. Install Kubernetes (Skip for Docker Installs)' +--- + + + + + +:::note + +Skip this section if you are installing Rancher on a single node with Docker. + +::: + +This section describes how to install a Kubernetes cluster according to our [best practices for the Rancher server environment.](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#environment-for-kubernetes-installations) This cluster should be dedicated to run only the Rancher server. + +Rancher can be installed on any Kubernetes cluster, including hosted Kubernetes providers. + +The steps to set up an air-gapped Kubernetes cluster on RKE, RKE2, or K3s are shown below. + + + + +In this guide, we are assuming you have created your nodes in your air gapped environment and have a secure Docker private registry on your bastion server. + +### Installation Outline + +1. [Prepare Images Directory](#1-prepare-images-directory) +2. [Create Registry YAML](#2-create-registry-yaml) +3. [Install K3s](#3-install-k3s) +4. [Save and Start Using the kubeconfig File](#4-save-and-start-using-the-kubeconfig-file) + +### 1. Prepare Images Directory +Obtain the images tar file for your architecture from the [releases](https://github.com/k3s-io/k3s/releases) page for the version of K3s you will be running. + +Place the tar file in the `images` directory before starting K3s on each node, for example: + +```sh +sudo mkdir -p /var/lib/rancher/k3s/agent/images/ +sudo cp ./k3s-airgap-images-$ARCH.tar /var/lib/rancher/k3s/agent/images/ +``` + +### 2. Create Registry YAML +Create the registries.yaml file at `/etc/rancher/k3s/registries.yaml`. This will tell K3s the necessary details to connect to your private registry. + +The registries.yaml file should look like this before plugging in the necessary information: + +```yaml +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # this is the registry username + password: xxxxxx # this is the registry password + tls: + cert_file: + key_file: + ca_file: +``` + +Note, at this time only secure registries are supported with K3s (SSL with custom CA). + +For more information on private registries configuration file for K3s, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/private-registry/) + +### 3. Install K3s + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +Obtain the K3s binary from the [releases](https://github.com/k3s-io/k3s/releases) page, matching the same version used to get the airgap images tar. +Also obtain the K3s install script at https://get.k3s.io + +Place the binary in `/usr/local/bin` on each node. +Place the install script anywhere on each node, and name it `install.sh`. + +Install K3s on each server: + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= ./install.sh +``` + +Install K3s on each agent: + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= K3S_URL=https://:6443 K3S_TOKEN= ./install.sh +``` + +Where `` is the IP or valid DNS of the server and `` is the node-token from the server found at `/var/lib/rancher/k3s/server/node-token`. + +:::note + +K3s additionally provides a `--resolv-conf` flag for kubelets, which may help with configuring DNS in air-gapped networks. + +::: + +### 4. Save and Start Using the kubeconfig File + +When you installed K3s on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/k3s/k3s.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/k3s/k3s.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your load balancer, referring to port 6443. (The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443.) Here is an example `k3s.yaml`: + +```yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your K3s cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/cluster-access/) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### Note on Upgrading + +Upgrading an air-gap environment can be accomplished in the following manner: + +1. Download the new air-gap images (tar file) from the [releases](https://github.com/k3s-io/k3s/releases) page for the version of K3s you will be upgrading to. Place the tar in the `/var/lib/rancher/k3s/agent/images/` directory on each node. Delete the old tar file. +2. Copy and replace the old K3s binary in `/usr/local/bin` on each node. Copy over the install script at https://get.k3s.io (as it is possible it has changed since the last release). Run the script again just as you had done in the past with the same environment variables. +3. Restart the K3s service (if not restarted automatically by installer). + + + + +In this guide, we are assuming you have created your nodes in your air-gapped environment and have a secure Docker private registry on your bastion server. + +### Installation Outline + +1. [Create RKE2 configuration](#1-create-rke2-configuration) +2. [Create Registry YAML](#2-create-registry-yaml) +3. [Install RKE2](#3-install-rke2) +4. [Save and Start Using the kubeconfig File](#4-save-and-start-using-the-kubeconfig-file) + +### 1. Create RKE2 configuration +Create the config.yaml file at `/etc/rancher/rke2/config.yaml`. This will contain all the configuration options necessary to create a highly available RKE2 cluster. + +On the first server the minimum config is: + +``` +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +On each other server the config file should contain the same token and tell RKE2 to connect to the existing first server: + +``` +server: https://ip-of-first-server:9345 +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +For more information, refer to the [RKE2 documentation](https://docs.rke2.io/install/ha). + +:::note + +RKE2 additionally provides a `resolv-conf` option for kubelets, which may help with configuring DNS in air-gap networks. + +::: + +### 2. Create Registry YAML +Create the registries.yaml file at `/etc/rancher/rke2/registries.yaml`. This will tell RKE2 the necessary details to connect to your private registry. + +The registries.yaml file should look like this before plugging in the necessary information: + +``` +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # this is the registry username + password: xxxxxx # this is the registry password + tls: + cert_file: + key_file: + ca_file: +``` + +For more information on private registries configuration file for RKE2, refer to the [RKE2 documentation.](https://docs.rke2.io/install/containerd_registry_configuration) + +### 3. Install RKE2 + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) + +Download the install script, rke2, rke2-images, and sha256sum archives from the release and upload them into a directory on each server: + +``` +mkdir /tmp/rke2-artifacts && cd /tmp/rke2-artifacts/ +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2-images.linux-amd64.tar.zst +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2.linux-amd64.tar.gz +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/sha256sum-amd64.txt +curl -sfL https://get.rke2.io --output install.sh +``` + +Next, run install.sh using the directory on each server, as in the example below: + +``` +INSTALL_RKE2_ARTIFACT_PATH=/tmp/rke2-artifacts sh install.sh +``` + +Then enable and start the service on all servers: + +`` +systemctl enable rke2-server.service +systemctl start rke2-server.service +`` + +For more information, refer to the [RKE2 documentation](https://docs.rke2.io/install/airgap). + +### 4. Save and Start Using the kubeconfig File + +When you installed RKE2 on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/rke2/rke2.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl), a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/rke2/rke2.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your load balancer, referring to port 6443. (The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443.) Here is an example `rke2.yaml`: + +``` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your RKE2 cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [RKE2 documentation](https://docs.rke2.io/cluster_access) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### Note on Upgrading + +Upgrading an air-gap environment can be accomplished in the following manner: + +1. Download the new air-gap artifacts and install script from the [releases](https://github.com/rancher/rke2/releases) page for the version of RKE2 you will be upgrading to. +2. Run the script again just as you had done in the past with the same environment variables. +3. Restart the RKE2 service. + + + +We will create a Kubernetes cluster using Rancher Kubernetes Engine (RKE). Before being able to start your Kubernetes cluster, you’ll need to install RKE and create a RKE config file. + +### 1. Install RKE + +Install RKE by following the instructions in the [RKE documentation.](https://rancher.com/docs/rke/latest/en/installation/) + +:::note + +Certified version(s) of RKE based on the Rancher version can be found in the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +::: + +### 2. Create an RKE Config File + +From a system that can access ports 22/TCP and 6443/TCP on the Linux host node(s) that you set up in a previous step, use the sample below to create a new file named `rancher-cluster.yml`. + +This file is an RKE configuration file, which is a configuration for the cluster you're deploying Rancher to. + +Replace values in the code sample below with help of the _RKE Options_ table. Use the IP address or DNS names of the three nodes you created. + +:::tip + +For more details on the options available, see the RKE [Config Options](https://rancher.com/docs/rke/latest/en/config-options/). + +::: + +
RKE Options
+ +| Option | Required | Description | +| ------------------ | -------------------- | --------------------------------------------------------------------------------------- | +| `address` | ✓ | The DNS or IP address for the node within the air gapped network. | +| `user` | ✓ | A user that can run Docker commands. | +| `role` | ✓ | List of Kubernetes roles assigned to the node. | +| `internal_address` | optional1 | The DNS or IP address used for internal cluster traffic. | +| `ssh_key_path` | | Path to the SSH private key used to authenticate to the node (defaults to `~/.ssh/id_rsa`). | + +> 1 Some services like AWS EC2 require setting the `internal_address` if you want to use self-referencing security groups or firewalls. + +```yaml +nodes: + - address: 10.10.3.187 # node air gap network IP + internal_address: 172.31.7.22 # node intra-cluster IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.254 # node air gap network IP + internal_address: 172.31.13.132 # node intra-cluster IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.89 # node air gap network IP + internal_address: 172.31.3.216 # node intra-cluster IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + +private_registries: + - url: # private registry url + user: rancher + password: '*********' + is_default: true +``` + +### 3. Run RKE + +After configuring `rancher-cluster.yml`, bring up your Kubernetes cluster: + +``` +rke up --config ./rancher-cluster.yml +``` + +### 4. Save Your Files + +:::note Important: + +The files mentioned below are needed to maintain, troubleshoot, and upgrade your cluster. + +::: + +Save a copy of the following files in a secure location: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_cluster.yml`: The [Kubeconfig file](https://rancher.com/docs/rke/latest/en/kubeconfig/) for the cluster, this file contains credentials for full access to the cluster. +- `rancher-cluster.rkestate`: The [Kubernetes Cluster State file](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state), this file contains the current state of the cluster including the RKE configuration and the certificates.

_The Kubernetes Cluster State file is only created when using RKE v0.2.0 or higher._ + + + + +:::note + +The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +::: + +### Issues or errors? + +See the [Troubleshooting](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) page. + +### [Next: Install Rancher](install-rancher-ha.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md new file mode 100644 index 00000000000..d09b5eca604 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md @@ -0,0 +1,250 @@ +--- +title: 4. Install Rancher +--- + + + + + +This section is about how to deploy Rancher for your air gapped environment in a high-availability Kubernetes installation. An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +### Privileged Access for Rancher + +When the Rancher server is deployed in the Docker container, a local Kubernetes cluster is installed within the container for Rancher to use. Because many features of Rancher run as deployments, and privileged mode is required to run containers within containers, you will need to install Rancher with the `--privileged` option. + +## Docker Instructions + +If you want to continue the air gapped installation using Docker commands, skip the rest of this page and follow the instructions on [this page.](docker-install-commands.md) + +## Kubernetes Instructions + +Rancher recommends installing Rancher on a Kubernetes cluster. A highly available Kubernetes install is comprised of three nodes running the Rancher server components on a Kubernetes cluster. The persistence layer (etcd) is also replicated on these three nodes, providing redundancy and data duplication in case one of the nodes fails. + +### 1. Add the Helm Chart Repository + +From a system that has access to the internet, fetch the latest Helm chart and copy the resulting manifests to a system that has access to the Rancher server cluster. + +1. If you haven't already, install `helm` locally on a workstation that has internet access. Note: Refer to the [Helm version requirements](../../resources/helm-version-requirements.md) to choose a version of Helm to install Rancher. + +2. Use `helm repo add` command to add the Helm chart repository that contains charts to install Rancher. For more information about the repository choices and which is best for your use case, see [Choosing a Rancher Version](../../resources/choose-a-rancher-version.md). + - Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + +3. Fetch the latest Rancher chart. This will pull down the chart and save it in the current directory as a `.tgz` file. + ```plain + helm fetch rancher-/rancher + ``` + + If you require a specific version of Rancher, you can fetch this with the Helm `--version` parameter like in the following example: + ```plain + helm fetch rancher-stable/rancher --version=v2.4.8 + ``` + +### 2. Choose your SSL Configuration + +Rancher Server is designed to be secure by default and requires SSL/TLS configuration. + +When Rancher is installed on an air gapped Kubernetes cluster, there are two recommended options for the source of the certificate. + +:::note + +If you want terminate SSL/TLS externally, see [TLS termination on an External Load Balancer](../../installation-references/helm-chart-options.md#external-tls-termination). + +::: + +| Configuration | Chart option | Description | Requires cert-manager | +| ------------------------------------------ | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| Rancher Generated Self-Signed Certificates | `ingress.tls.source=rancher` | Use certificates issued by Rancher's generated CA (self signed)
This is the **default** and does not need to be added when rendering the Helm template. | yes | +| Certificates from Files | `ingress.tls.source=secret` | Use your own certificate files by creating Kubernetes Secret(s).
This option must be passed when rendering the Rancher Helm template. | no | + +### Helm Chart Options for Air Gap Installations + +When setting up the Rancher Helm template, there are several options in the Helm chart that are designed specifically for air gap installations. + +| Chart Option | Chart Value | Description | +| ----------------------- | -------------------------------- | ---- | +| `certmanager.version` | `` | Configure proper Rancher TLS issuer depending of running cert-manager version. | +| `systemDefaultRegistry` | `` | Configure Rancher server to always pull from your private registry when provisioning clusters. | +| `useBundledSystemChart` | `true` | Configure Rancher server to use the packaged copy of Helm system charts. The [system charts](https://github.com/rancher/system-charts) repository contains all the catalog items required for features such as monitoring, logging, alerting and global DNS. These [Helm charts](https://github.com/rancher/system-charts) are located in GitHub, but since you are in an air gapped environment, using the charts that are bundled within Rancher is much easier than setting up a Git mirror. | + +### 3. Fetch the Cert-Manager Chart + +Based on the choice your made in [2. Choose your SSL Configuration](#2-choose-your-ssl-configuration), complete one of the procedures below. + +#### Option A: Default Self-Signed Certificate + +By default, Rancher generates a CA and uses cert-manager to issue the certificate for access to the Rancher server interface. + +:::note + +Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.11.0, please see our [upgrade cert-manager documentation](../../resources/upgrade-cert-manager.md). + +::: + +##### 1. Add the cert-manager repo + +From a system connected to the internet, add the cert-manager repo to Helm: + +```plain +helm repo add jetstack https://charts.jetstack.io +helm repo update +``` + +##### 2. Fetch the cert-manager chart + +Fetch the latest cert-manager chart available from the [Helm chart repository](https://artifacthub.io/packages/helm/cert-manager/cert-manager). + +```plain +helm fetch jetstack/cert-manager --version v1.11.0 +``` + +##### 3. Retrieve the Cert-Manager CRDs + +Download the required CRD file for cert-manager: + ```plain + curl -L -o cert-manager-crd.yaml https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + ``` + +### 4. Install Rancher + +Copy the fetched charts to a system that has access to the Rancher server cluster to complete installation. + +##### 1. Install Cert-Manager + +Install cert-manager with the same options you would use to install the chart. Remember to set the `image.repository` option to pull the image from your private registry. + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +
+ Click to expand + +If you are using self-signed certificates, install cert-manager: + +1. Create the namespace for cert-manager. + + ```plain + kubectl create namespace cert-manager + ``` + +2. Create the cert-manager CustomResourceDefinitions (CRDs). + + ```plain + kubectl apply -f cert-manager-crd.yaml + ``` + +3. Install cert-manager. + + ```plain + helm install cert-manager ./cert-manager-v1.11.0.tgz \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller \ + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook \ + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector \ + --set startupapicheck.image.repository=/quay.io/jetstack/cert-manager-ctl + ``` + +
+ +##### 2. Install Rancher +First, refer to [Adding TLS Secrets](../../resources/add-tls-secrets.md) to publish the certificate files so Rancher and the ingress controller can use them. + +Then, create the namespace for Rancher using kubectl: + +```plain +kubectl create namespace cattle-system +``` + +Next, install Rancher, declaring your chosen options. Use the reference table below to replace each placeholder. Rancher needs to be configured to use the private registry in order to provision any Rancher launched Kubernetes clusters or Rancher tools. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +Placeholder | Description +------------|------------- +`` | The version number of the output tarball. +`` | The DNS name you pointed at your load balancer. +`` | The DNS name for your private registry. +`` | Cert-manager version running on k8s cluster. + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +**Optional**: To install a specific Rancher version, set the `rancherImageTag` value, example: `--set rancherImageTag=v2.5.8` + +#### Option B: Certificates From Files using Kubernetes Secrets + +##### 1. Create secrets + +Create Kubernetes secrets from your own certificates for Rancher to use. The common name for the cert will need to match the `hostname` option in the command below, or the ingress controller will fail to provision the site for Rancher. + +##### 2. Install Rancher + +Install Rancher, declaring your chosen options. Use the reference table below to replace each placeholder. Rancher needs to be configured to use the private registry in order to provision any Rancher launched Kubernetes clusters or Rancher tools. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------- | +| `` | The version number of the output tarball. | +| `` | The DNS name you pointed at your load balancer. | +| `` | The DNS name for your private registry. | + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +If you are using a Private CA signed cert, add `--set privateCA=true` following `--set ingress.tls.source=secret`: + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + + +The installation is complete. +:::caution + +If you don't intend to send telemetry data, opt out [telemetry](../../../../faq/telemetry.md) during the initial login. Leaving this active in an air-gapped environment can cause issues if the sockets cannot be opened successfully. + +::: + +## Additional Resources + +These resources could be helpful when installing Rancher: + +- [Rancher Helm chart options](../../installation-references/helm-chart-options.md) +- [Adding TLS secrets](../../resources/add-tls-secrets.md) +- [Troubleshooting Rancher Kubernetes Installations](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md new file mode 100644 index 00000000000..bf242ada285 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md @@ -0,0 +1,309 @@ +--- +title: '2. Collect and Publish Images to your Private Registry' +--- + + + + + +This section describes how to set up your private registry so that when you install Rancher, Rancher will pull all the required images from this registry. + +By default, all images used to [provision Kubernetes clusters](../../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) or launch any tools in Rancher, e.g. monitoring, pipelines, alerts, are pulled from Docker Hub. In an air gapped installation of Rancher, you will need a private registry that is located somewhere accessible by your Rancher server. Then, you will load the registry with all the images. + +Populating the private registry with images is the same process for installing Rancher with Docker and for installing Rancher on a Kubernetes cluster. + +The steps in this section differ depending on whether or not you are planning to use Rancher to provision a downstream cluster with Windows nodes or not. By default, we provide the steps of how to populate your private registry assuming that Rancher will provision downstream Kubernetes clusters with only Linux nodes. But if you plan on provisioning any [downstream Kubernetes clusters using Windows nodes](../../../../pages-for-subheaders/use-windows-clusters.md), there are separate instructions to support the images needed. + +:::note Prerequisites: + +You must have a [private registry](https://docs.docker.com/registry/deploying/#run-an-externally-accessible-registry) available to use. + +If the registry has certs, follow [this K3s documentation](https://rancher.com/docs/k3s/latest/en/installation/private-registry/) about adding a private registry. The certs and registry configuration files need to be mounted into the Rancher container. + +::: + + + + +For Rancher servers that will only provision Linux clusters, these are the steps to populate your private registry. + +1. [Find the required assets for your Rancher version](#1-find-the-required-assets-for-your-rancher-version) +2. [Collect the cert-manager image](#2-collect-the-cert-manager-image) (unless you are bringing your own certificates or terminating TLS on a load balancer) +3. [Save the images to your workstation](#3-save-the-images-to-your-workstation) +4. [Populate the private registry](#4-populate-the-private-registry) + +### Prerequisites + +These steps expect you to use a Linux workstation that has internet access, access to your private registry, and at least 20 GB of disk space. + +If you will use ARM64 hosts, the registry must support manifests. As of April 2020, Amazon Elastic Container Registry does not support manifests. + +### 1. Find the required assets for your Rancher version + +1. Go to our [releases page,](https://github.com/rancher/rancher/releases) find the Rancher v2.x.x release that you want to install, and click **Assets**. Note: Don't use releases marked `rc` or `Pre-release`, as they are not stable for production environments. + +2. From the release's **Assets** section, download the following files, which are required to install Rancher in an air gap environment: + +| Release File | Description | +| ---------------- | -------------- | +| `rancher-images.txt` | This file contains a list of images needed to install Rancher, provision clusters and user Rancher tools. | +| `rancher-save-images.sh` | This script pulls all the images in the `rancher-images.txt` from Docker Hub and saves all of the images as `rancher-images.tar.gz`. | +| `rancher-load-images.sh` | This script loads images from the `rancher-images.tar.gz` file and pushes them to your private registry. | + +### 2. Collect the cert-manager image + +:::note + +Skip this step if you are using your own certificates, or if you are terminating TLS on an external load balancer. + +::: + +In a Kubernetes Install, if you elect to use the Rancher default self-signed TLS certificates, you must add the [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) image to `rancher-images.txt` as well. + +1. Fetch the latest `cert-manager` Helm chart and parse the template for image details: + + :::note + + Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.12.0, please see our [upgrade documentation](../../resources/upgrade-cert-manager.md). + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. Sort and unique the images list to remove any overlap between the sources: + + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + +### 3. Save the images to your workstation + +1. Make `rancher-save-images.sh` an executable: + ``` + chmod +x rancher-save-images.sh + ``` + +1. Run `rancher-save-images.sh` with the `rancher-images.txt` image list to create a tarball of all the required images: + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + **Result:** Docker begins pulling the images used for an air gap install. Be patient. This process takes a few minutes. When the process completes, your current directory will output a tarball named `rancher-images.tar.gz`. Check that the output is in the directory. + +### 4. Populate the private registry + +Next, you will move the images in the `rancher-images.tar.gz` to your private registry using the scripts to load the images. + +Move the images in the `rancher-images.tar.gz` to your private registry using the scripts to load the images. + +The `rancher-images.txt` is expected to be on the workstation in the same directory that you are running the `rancher-load-images.sh` script. The `rancher-images.tar.gz` should also be in the same directory. + +1. Log into your private registry if required: + ```plain + docker login + ``` +1. Make `rancher-load-images.sh` an executable: + ``` + chmod +x rancher-load-images.sh + ``` + +1. Use `rancher-load-images.sh` to extract, tag and push `rancher-images.txt` and `rancher-images.tar.gz` to your private registry: + ```plain + ./rancher-load-images.sh --image-list ./rancher-images.txt --registry + ``` + + + + +For Rancher servers that will provision Linux and Windows clusters, there are distinctive steps to populate your private registry for the Windows images and the Linux images. Since a Windows cluster is a mix of Linux and Windows nodes, the Linux images pushed into the private registry are manifests. + +## Windows Steps + +The Windows images need to be collected and pushed from a Windows server workstation. + +1. Find the required assets for your Rancher version +2. Save the images to your Windows Server workstation +3. Prepare the Docker daemon +4. Populate the private registry + +### Prerequisites + +These steps expect you to use a Windows Server 1809 workstation that has internet access, access to your private registry, and at least 50 GB of disk space. + +The workstation must have Docker 18.02+ in order to support manifests, which are required when provisioning Windows clusters. + +Your registry must support manifests. As of April 2020, Amazon Elastic Container Registry does not support manifests. + + + +### 1. Find the required assets for your Rancher version + +1. Browse to our [releases page](https://github.com/rancher/rancher/releases) and find the Rancher v2.x.x release that you want to install. Don't download releases marked `rc` or `Pre-release`, as they are not stable for production environments. + +2. From the release's "Assets" section, download the following files: + +| Release File | Description | +|----------------------------|------------------| +| `rancher-windows-images.txt` | This file contains a list of Windows images needed to provision Windows clusters. | +| `rancher-save-images.ps1` | This script pulls all the images in the `rancher-windows-images.txt` from Docker Hub and saves all of the images as `rancher-windows-images.tar.gz`. | +| `rancher-load-images.ps1` | This script loads the images from the `rancher-windows-images.tar.gz` file and pushes them to your private registry. | + + + +### 2. Save the images to your Windows Server workstation + +1. Using `powershell`, go to the directory that has the files that were downloaded in the previous step. + +1. Run `rancher-save-images.ps1` to create a tarball of all the required images: + ```plain + ./rancher-save-images.ps1 + ``` + + **Result:** Docker begins pulling the images used for an air gap install. Be patient. This process takes a few minutes. When the process completes, your current directory will output a tarball named `rancher-windows-images.tar.gz`. Check that the output is in the directory. + + + +### 3. Prepare the Docker daemon + +Append your private registry address to the `allow-nondistributable-artifacts` config field in the Docker daemon (`C:\ProgramData\Docker\config\daemon.json`). Since the base image of Windows images are maintained by the `mcr.microsoft.com` registry, this step is required as the layers in the Microsoft registry are missing from Docker Hub and need to be pulled into the private registry. + + ```json + { + ... + "allow-nondistributable-artifacts": [ + ... + "" + ] + ... + } + ``` + + + +### 4. Populate the private registry + +Move the images in the `rancher-windows-images.tar.gz` to your private registry using the scripts to load the images. + +The `rancher-windows-images.txt` is expected to be on the workstation in the same directory that you are running the `rancher-load-images.ps1` script. The `rancher-windows-images.tar.gz` should also be in the same directory. + +1. Using `powershell`, log into your private registry if required: + ```plain + docker login + ``` + +1. Using `powershell`, use `rancher-load-images.ps1` to extract, tag and push the images from `rancher-images.tar.gz` to your private registry: + ```plain + ./rancher-load-images.ps1 --registry + ``` + +## Linux Steps + +The Linux images need to be collected and pushed from a Linux host, but _must be done after_ populating the Windows images into the private registry. These step are different from the Linux only steps as the Linux images that are pushed will actually manifests that support Windows and Linux images. + +1. Find the required assets for your Rancher version +2. Collect all the required images +3. Save the images to your Linux workstation +4. Populate the private registry + +### Prerequisites + +You must populate the private registry with the Windows images before populating the private registry with Linux images. If you have already populated the registry with Linux images, you will need to follow these instructions again as they will publish manifests that support Windows and Linux images. + +These steps expect you to use a Linux workstation that has internet access, access to your private registry, and at least 20 GB of disk space. + +The workstation must have Docker 18.02+ in order to support manifests, which are required when provisioning Windows clusters. + + + +### 1. Find the required assets for your Rancher version + +1. Browse to our [releases page](https://github.com/rancher/rancher/releases) and find the Rancher v2.x.x release that you want to install. Don't download releases marked `rc` or `Pre-release`, as they are not stable for production environments. Click **Assets**. + +2. From the release's **Assets** section, download the following files: + +| Release File | Description | +|----------------------------| -------------------------- | +| `rancher-images.txt` | This file contains a list of images needed to install Rancher, provision clusters and user Rancher tools. | +| `rancher-windows-images.txt` | This file contains a list of images needed to provision Windows clusters. | +| `rancher-save-images.sh` | This script pulls all the images in the `rancher-images.txt` from Docker Hub and saves all of the images as `rancher-images.tar.gz`. | +| `rancher-load-images.sh` | This script loads images from the `rancher-images.tar.gz` file and pushes them to your private registry. | + + + +### 2. Collect all the required images + +**For Kubernetes Installs using Rancher Generated Self-Signed Certificate:** In a Kubernetes Install, if you elect to use the Rancher default self-signed TLS certificates, you must add the [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) image to `rancher-images.txt` as well. You skip this step if you are using you using your own certificates. + + +1. Fetch the latest `cert-manager` Helm chart and parse the template for image details: + + :::note + + Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.12.0, please see our [upgrade documentation](../../resources/upgrade-cert-manager.md). + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. Sort and unique the images list to remove any overlap between the sources: + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + + + +### 3. Save the images to your workstation + +1. Make `rancher-save-images.sh` an executable: + ``` + chmod +x rancher-save-images.sh + ``` + +1. Run `rancher-save-images.sh` with the `rancher-images.txt` image list to create a tarball of all the required images: + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + +**Result:** Docker begins pulling the images used for an air gap install. Be patient. This process takes a few minutes. When the process completes, your current directory will output a tarball named `rancher-images.tar.gz`. Check that the output is in the directory. + + + +### 4. Populate the private registry + +Move the images in the `rancher-images.tar.gz` to your private registry using the `rancher-load-images.sh script` to load the images. + +The image list, `rancher-images.txt` or `rancher-windows-images.txt`, is expected to be on the workstation in the same directory that you are running the `rancher-load-images.sh` script. The `rancher-images.tar.gz` should also be in the same directory. + +1. Log into your private registry if required: + ```plain + docker login + ``` + +1. Make `rancher-load-images.sh` an executable: + ``` + chmod +x rancher-load-images.sh + ``` + +1. Use `rancher-load-images.sh` to extract, tag and push the images from `rancher-images.tar.gz` to your private registry: + +```plain +./rancher-load-images.sh --image-list ./rancher-images.txt \ + --windows-image-list ./rancher-windows-images.txt \ + --registry +``` + + + + +### [Next step for Kubernetes Installs - Launch a Kubernetes Cluster](install-kubernetes.md) + +### [Next step for Docker Installs - Install Rancher](install-rancher-ha.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md new file mode 100644 index 00000000000..a5167fbd77a --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md @@ -0,0 +1,252 @@ +--- +title: '2. Install Kubernetes' +--- + + + + + +Once the infrastructure is ready, you can continue with setting up a Kubernetes cluster to install Rancher in. + +The steps to set up RKE, RKE2, or K3s are shown below. + +For convenience, export the IP address and port of your proxy into an environment variable and set up the HTTP_PROXY variables for your current shell on every node: + +``` +export proxy_host="10.0.0.5:8888" +export HTTP_PROXY=http://${proxy_host} +export HTTPS_PROXY=http://${proxy_host} +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16 +``` + + + + +First configure the HTTP proxy settings on the K3s systemd service, so that K3s's containerd can pull images through the proxy: + +``` +cat <<'EOF' | sudo tee /etc/default/k3s > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +On the first node, create a new cluster: +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --cluster-init +``` + +And then join the other nodes: +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --server https://:6443 +``` + +Where `` is the IP or valid DNS of the server and `` is the node-token from the server found at `/var/lib/rancher/k3s/server/node-token`. + +For more information on installing K3s see the [K3s installation docs](https://docs.k3s.io/installation). + +To have a look at your cluster run: + +``` +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +On every node, run the RKE2 installation script. Ensure that the RKE2 version you are installing is [supported by Rancher](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +``` +curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.xx sh - +``` + +Then you have to configure the HTTP proxy settings on the RKE2 systemd service, so that RKE2's containerd can pull images through the proxy: + +``` +cat <<'EOF' | sudo tee /etc/default/rke2-server > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +Next create the RKE2 configuration file on every node following the [RKE2 High Availability documentation](https://docs.rke2.io/install/ha). + +After that start and enable the `rke2-server` service: + +``` +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` + +For more information on installing RKE2 see the [RKE2 documentation](https://docs.rke2.io). + +To have a look at your cluster run: + +``` +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +alias kubectl=/var/lib/rancher/rke2/bin/kubectl +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +First, you have to install Docker and setup the HTTP proxy on all three Linux nodes. For this perform the following steps on all three nodes. + +Next configure apt to use this proxy when installing packages. If you are not using Ubuntu, you have to adapt this step accordingly: + +``` +cat <<'EOF' | sudo tee /etc/apt/apt.conf.d/proxy.conf > /dev/null +Acquire::http::Proxy "http://${proxy_host}/"; +Acquire::https::Proxy "http://${proxy_host}/"; +EOF +``` + +Now you can install Docker: + +``` +curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh +``` + +Then ensure that your current user is able to access the Docker daemon without sudo: + +``` +sudo usermod -aG docker YOUR_USERNAME +``` + +And configure the Docker daemon to use the proxy to pull images: + +``` +sudo mkdir -p /etc/systemd/system/docker.service.d +cat <<'EOF' | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf > /dev/null +[Service] +Environment="HTTP_PROXY=http://${proxy_host}" +Environment="HTTPS_PROXY=http://${proxy_host}" +Environment="NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16" +EOF +``` + +To apply the configuration, restart the Docker daemon: + +``` +sudo systemctl daemon-reload +sudo systemctl restart docker +``` + +#### Air-gapped proxy + +You can now provision node driver clusters from an air-gapped cluster configured to use a proxy for outbound connections. + +In addition to setting the default rules for a proxy server, you will need to add additional rules, shown below, to provision node driver clusters from a proxied Rancher environment. + +You will configure your filepath according to your setup, e.g., `/etc/apt/apt.conf.d/proxy.conf`: + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` + +### Creating the RKE Cluster + +You need several command line tools on the host where you have SSH access to the Linux nodes to create and interact with the cluster: + +* [RKE CLI binary](https://rancher.com/docs/rke/latest/en/installation/#download-the-rke-binary) + +``` +sudo curl -fsSL -o /usr/local/bin/rke https://github.com/rancher/rke/releases/download/v1.1.4/rke_linux-amd64 +sudo chmod +x /usr/local/bin/rke +``` + +* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + +``` +curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +``` + +Next, create a YAML file that describes the RKE cluster. Ensure that the IP addresses of the nodes and the SSH username are correct. For more information on the cluster YAML, have a look at the [RKE documentation](https://rancher.com/docs/rke/latest/en/example-yamls/). + +```yml +nodes: + - address: 10.0.1.200 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.201 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.202 + user: ubuntu + role: [controlplane,worker,etcd] + +services: + etcd: + backup_config: + interval_hours: 12 + retention: 6 +``` + +After that, you can create the Kubernetes cluster by running: + +``` +rke up --config rancher-cluster.yaml +``` + +RKE creates a state file called `rancher-cluster.rkestate`, this is needed if you want to perform updates, modify your cluster configuration or restore it from a backup. It also creates a `kube_config_cluster.yaml` file, that you can use to connect to the remote Kubernetes cluster locally with tools like kubectl or Helm. Make sure to save all of these files in a secure location, for example by putting them into a version control system. + +To have a look at your cluster run: + +``` +export KUBECONFIG=kube_config_cluster.yaml +kubectl cluster-info +kubectl get pods --all-namespaces +``` + +You can also verify that your external load balancer works, and the DNS entry is set up correctly. If you send a request to either, you should receive HTTP 404 response from the ingress controller: + +``` +$ curl 10.0.1.100 +default backend - 404 +$ curl rancher.example.com +default backend - 404 +``` + +### Save Your Files + +:::note Important: + +The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster. + +::: + +Save a copy of the following files in a secure location: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_cluster.yml`: The [Kubeconfig file](https://rancher.com/docs/rke/latest/en/kubeconfig/) for the cluster, this file contains credentials for full access to the cluster. +- `rancher-cluster.rkestate`: The [Kubernetes Cluster State file](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state), this file contains the current state of the cluster including the RKE configuration and the certificates. + +:::note + +The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +::: + + + + +### Issues or errors? + +See the [Troubleshooting](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) page. + +### [Next: Install Rancher](install-rancher.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md new file mode 100644 index 00000000000..0267e72d0be --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md @@ -0,0 +1,108 @@ +--- +title: 3. Install Rancher +--- + + + + + +Now that you have a running RKE cluster, you can install Rancher in it. For security reasons all traffic to Rancher must be encrypted with TLS. For this tutorial you are going to automatically issue a self-signed certificate through [cert-manager](https://cert-manager.io/). In a real-world use-case you will likely use Let's Encrypt or provide your own certificate. + +### Install the Helm CLI + +Install the [Helm](https://helm.sh/docs/intro/install/) CLI on a host where you have a kubeconfig to access your Kubernetes cluster: + +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod +x get_helm.sh +sudo ./get_helm.sh +``` + +### Install cert-manager + +Add the cert-manager helm repository: + +``` +helm repo add jetstack https://charts.jetstack.io +``` + +Create a namespace for cert-manager: + +``` +kubectl create namespace cert-manager +``` + +Install the CustomResourceDefinitions of cert-manager: + +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml +``` + +And install it with Helm. Note that cert-manager also needs your proxy configured in case it needs to communicate with Let's Encrypt or other external certificate issuers: + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +``` +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --version v1.11.0 \ + --set http_proxy=http://${proxy_host} \ + --set https_proxy=http://${proxy_host} \ + --set no_proxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +Now you should wait until cert-manager is finished starting up: + +``` +kubectl rollout status deployment -n cert-manager cert-manager +kubectl rollout status deployment -n cert-manager cert-manager-webhook +``` + +### Install Rancher + +Next you can install Rancher itself. First add the helm repository: + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +``` + +Create a namespace: + +``` +kubectl create namespace cattle-system +``` + +And install Rancher with Helm. Rancher also needs a proxy configuration so that it can communicate with external application catalogs or retrieve Kubernetes version update metadata: + +``` +helm upgrade --install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.example.com \ + --set proxy=http://${proxy_host} \ + --set noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +After waiting for the deployment to finish: + +``` +kubectl rollout status deployment -n cattle-system rancher +``` + +You can now navigate to `https://rancher.example.com` and start using Rancher. + +:::caution + +If you don't intend to send telemetry data, opt out [telemetry](../../../../faq/telemetry.md) during the initial login. Leaving this active in an air-gapped environment can cause issues if the sockets cannot be opened successfully. + +::: + +### Additional Resources + +These resources could be helpful when installing Rancher: + +- [Rancher Helm chart options](../../installation-references/helm-chart-options.md) +- [Adding TLS secrets](../../resources/add-tls-secrets.md) +- [Troubleshooting Rancher Kubernetes Installations](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md new file mode 100644 index 00000000000..7822fa065cd --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md @@ -0,0 +1,67 @@ +--- +title: '1. Set up Infrastructure' +--- + + + + + +In this section, you will provision the underlying infrastructure for your Rancher management server with internet access through a HTTP proxy. + +To install the Rancher management server on a high-availability RKE cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in an infrastructure provider such as Amazon's EC2, Google Compute Engine, or vSphere. +- **A load balancer** to direct front-end traffic to the three nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +These nodes must be in the same region/data center. You may place these servers in separate availability zones. + +### Why three nodes? + +In an RKE cluster, Rancher server data is stored on etcd. This etcd database runs on all three nodes. + +The etcd database requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can suffer from [split brain](https://www.quora.com/What-is-split-brain-in-distributed-systems), requiring the cluster to be restored from backup. If one of the three etcd nodes fails, the two remaining nodes can elect a leader because they have the majority of the total number of etcd nodes. + +### 1. Set up Linux Nodes + +These hosts will connect to the internet through an HTTP proxy. + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE tool will deploy an NGINX Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the NGINX Ingress controller to listen for traffic destined for the Rancher hostname. The NGINX Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md) + +:::note Important: + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 3. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the LB IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) + + +### [Next: Set up a Kubernetes cluster](install-kubernetes.md) diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md new file mode 100644 index 00000000000..6614dd7a4cd --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md @@ -0,0 +1,92 @@ +--- +title: Certificate Troubleshooting +--- + + + + + +### How Do I Know if My Certificates are in PEM Format? + +You can recognize the PEM format by the following traits: + +- The file begins with the following header: + ``` + -----BEGIN CERTIFICATE----- + ``` +- The header is followed by a long string of characters. +- The file ends with a footer: + -----END CERTIFICATE----- + +PEM Certificate Example: + +``` +----BEGIN CERTIFICATE----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END CERTIFICATE----- +``` + +PEM Certificate Key Example: + +``` +-----BEGIN RSA PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END RSA PRIVATE KEY----- +``` + +If your key looks like the example below, see [Converting a Certificate Key From PKCS8 to PKCS1.](#converting-a-certificate-key-from-pkcs8-to-pkcs1) + +``` +-----BEGIN PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END PRIVATE KEY----- +``` + +### Converting a Certificate Key From PKCS8 to PKCS1 + +If you are using a PKCS8 certificate key file, Rancher will log the following line: + +``` +ListenConfigController cli-config [listener] failed with : failed to read private key: asn1: structure error: tags don't match (2 vs {class:0 tag:16 length:13 isCompound:true}) +``` + +To make this work, you will need to convert the key from PKCS8 to PKCS1 using the command below: + +``` +openssl rsa -in key.pem -out convertedkey.pem +``` + +You can now use `convertedkey.pem` as certificate key file for Rancher. + +### What is the Order of Certificates if I Want to Add My Intermediate(s)? + +The order of adding certificates is as follows: + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +### How Do I Validate My Certificate Chain? + +You can validate the certificate chain by using the `openssl` binary. If the output of the command (see the command example below) ends with `Verify return code: 0 (ok)`, your certificate chain is valid. The `ca.pem` file must be the same as you added to the `rancher/rancher` container. + +When using a certificate signed by a recognized Certificate Authority, you can omit the `-CAfile` parameter. + +Command: + +``` +openssl s_client -CAfile ca.pem -connect rancher.yourdomain.com:443 +... + Verify return code: 0 (ok) +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md new file mode 100644 index 00000000000..1ec111444b9 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md @@ -0,0 +1,95 @@ +--- +title: Rolling Back Rancher Installed with Docker +--- + + + + + +If a Rancher upgrade does not complete successfully, you'll have to roll back to your Rancher setup that you were using before [Docker Upgrade](upgrade-docker-installed-rancher.md). Rolling back restores: + +- Your previous version of Rancher. +- Your data backup created before upgrade. + +## Before You Start + +During rollback to a prior version of Rancher, you'll enter a series of commands, filling placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). Here's an example of a command with a placeholder: + +``` +docker pull rancher/rancher: +``` + +In this command, `` is the version of Rancher you were running before your unsuccessful upgrade. `v2.0.5` for example. + +Cross reference the image and reference table below to learn how to obtain this placeholder data. Write down or copy this information before starting the procedure below. + +Terminal docker ps Command, Displaying Where to Find <PRIOR_RANCHER_VERSION> and <RANCHER_CONTAINER_NAME>![Placeholder Reference](/img/placeholder-ref-2.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | ------------------------------------------------------- | +| `` | `v2.0.5` | The rancher/rancher image you used before upgrade. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.0.5` | The version of Rancher that the backup is for. | +| `` | `9-27-18` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher Server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped using a different command: `docker ps -a`. Use these commands for help anytime during while creating backups. + +## Rolling Back Rancher + +If you have issues upgrading Rancher, roll it back to its latest known healthy state by pulling the last version you used and then restoring the backup you made before upgrade. + +:::danger + +Rolling back to a previous version of Rancher destroys any changes made to Rancher following the upgrade. Unrecoverable data loss may occur. + +::: + +1. Using a remote Terminal connection, log into the node running your Rancher Server. + +1. Pull the version of Rancher that you were running before upgrade. Replace the `` with that version. + + For example, if you were running Rancher v2.0.5 before upgrade, pull v2.0.5. + + ``` + docker pull rancher/rancher: + ``` + +1. Stop the container currently running Rancher Server. Replace `` with the name of your Rancher container. + + ``` + docker stop + ``` + You can obtain the name for your Rancher container by entering `docker ps`. + +1. Move the backup tarball that you created during completion of [Docker Upgrade](upgrade-docker-installed-rancher.md) onto your Rancher Server. Change to the directory that you moved it to. Enter `dir` to confirm that it's there. + + If you followed the naming convention we suggested in [Docker Upgrade](upgrade-docker-installed-rancher.md), it will have a name similar to (`rancher-data-backup--.tar.gz`). + +1. Run the following command to replace the data in the `rancher-data` container with the data in the backup tarball, replacing the placeholder. Don't forget to close the quotes. + + ``` + docker run --volumes-from rancher-data \ + -v $PWD:/backup busybox sh -c "rm /var/lib/rancher/* -rf \ + && tar zxvf /backup/rancher-data-backup--.tar.gz" + ``` + +1. Start a new Rancher Server container with the `` tag placeholder pointing to the data container. + ``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: + ``` + Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + + :::danger + + **_Do not_** stop the rollback after initiating it, even if the rollback process seems longer than expected. Stopping the rollback may result in database issues during future upgrades. + + ::: + +1. Wait a few moments and then open Rancher in a web browser. Confirm that the rollback succeeded and that your data is restored. + +**Result:** Rancher is rolled back to its version and data state before upgrade. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md new file mode 100644 index 00000000000..2a3bdaa29df --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -0,0 +1,397 @@ +--- +title: Upgrading Rancher Installed with Docker +--- + + + + + +The following instructions will guide you through upgrading a Rancher server that was installed with Docker. + +:::caution + +**Docker installs are not supported in production environments.** These instructions are provided for testing and development purposes only. If you have already deployed a Docker install in production and need to upgrade to a new Rancher version, we recommend [migrating to the Helm chart install](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) before upgrading. + +::: + +## Prerequisites + +- **Review the [known upgrade issues](../../install-upgrade-on-a-kubernetes-cluster/upgrades.md#known-upgrade-issues)** section in the Rancher documentation for the most noteworthy issues to consider when upgrading Rancher. A more complete list of known issues for each Rancher version can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums](https://forums.rancher.com/c/announcements/12). Note that upgrades to or from any chart in the [rancher-alpha repository](../../resources/choose-a-rancher-version.md#helm-chart-repositories) aren’t supported. +- **For [air gap installs only,](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md) collect and populate images for the new Rancher server version**. Follow the guide to [populate your private registry](../air-gapped-helm-cli-install/publish-images.md) with the images for the Rancher version that you want to upgrade to. + +## Placeholder Review + +During upgrade, you'll enter a series of commands, filling placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). + +Here's an **example** of a command with a placeholder: + +``` +docker stop +``` + +In this command, `` is the name of your Rancher container. + +## Get Data for Upgrade Commands + +To obtain the data to replace the placeholders, run: + +``` +docker ps +``` + +Write down or copy this information before starting the upgrade. + +Terminal docker ps Command, Displaying Where to Find <RANCHER_CONTAINER_TAG> and <RANCHER_CONTAINER_NAME> + +![Placeholder Reference](/img/placeholder-ref.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.1.3` | The rancher/rancher image you pulled for initial install. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.1.3` | The version of Rancher that you're creating a backup for. | +| `` | `2018-12-19` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped using a different command: `docker ps -a`. Use these commands for help anytime during while creating backups. + +## Upgrade + +During upgrade, you create a copy of the data from your current Rancher container and a backup in case something goes wrong. Then you deploy the new version of Rancher in a new container using your existing data. +### 1. Create a copy of the data from your Rancher server container + +1. Using a remote Terminal connection, log into the node running your Rancher server. + +1. Stop the container currently running Rancher server. Replace `` with the name of your Rancher container. + + ``` + docker stop + ``` + +1. Use the command below, replacing each placeholder, to create a data container from the Rancher container that you just stopped. + + ``` + docker create --volumes-from --name rancher-data rancher/rancher: + ``` + +### 2. Create a backup tarball + +1. From the data container that you just created (rancher-data), create a backup tarball (rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz). + + This tarball will serve as a rollback point if something goes wrong during upgrade. Use the following command, replacing each placeholder. + ``` + docker run --volumes-from rancher-data -v "$PWD:/backup" --rm busybox tar zcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **Step Result:** When you enter this command, a series of commands should run. + +1. Enter the `ls` command to confirm that the backup tarball was created. It will have a name similar to `rancher-data-backup--.tar.gz`. + + ``` + [rancher@ip-10-0-0-50 ~]$ ls + rancher-data-backup-v2.1.3-20181219.tar.gz + ``` + +1. Move your backup tarball to a safe location external from your Rancher server. + +### 3. Pull the New Docker Image + +Pull the image of the Rancher version that you want to upgrade to. + +Placeholder | Description +------------|------------- +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker pull rancher/rancher: +``` + +### 4. Start the New Rancher Server Container + +Start a new Rancher server container using the data from the `rancher-data` container. Remember to pass in all the environment variables that you had used when you started the original container. + +:::danger + +**_Do not_** stop the upgrade after initiating it, even if the upgrade process seems longer than expected. Stopping the upgrade may result in database migration errors during future upgrades. + +::: + +If you used a proxy, see [HTTP Proxy Configuration.](../../../../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) + +If you configured a custom CA root certificate to access your services, see [Custom CA root certificate.](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#custom-ca-certificate) + +If you are recording all transactions with the Rancher API, see [API Auditing](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) + +To see the command to use when starting the new Rancher server container, choose from the following options: + +- Docker Upgrade +- Docker Upgrade for Air Gap Installs + + + + +Select which option you had installed Rancher server + +#### Option A: Default Self-Signed Certificate + +
+ Click to expand + +If you have selected to use the Rancher generated self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container. + +Placeholder | Description +------------|------------- +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +
+ +#### Option B: Bring Your Own Certificate: Self-Signed + +
+Click to expand + +If you have selected to bring your own self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificate that you had originally installed with. + +:::note Reminder of the Cert Prerequisite: + +The certificate files must be in PEM format. In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. + +::: + +Placeholder | Description +------------|------------- + `` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | The path to the certificate authority's certificate. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher: +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +
+ +#### Option C: Bring Your Own Certificate: Signed by Recognized CA + +
+ Click to expand + +If you have selected to use a certificate signed by a recognized CA, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificates that you had originally installed with. Remember to include `--no-cacerts` as an argument to the container to disable the default CA certificate generated by Rancher. + +:::note Reminder of the Cert Prerequisite: + +The certificate files must be in PEM format. In your certificate file, include all intermediate certificates provided by the recognized CA. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](certificate-troubleshooting.md) + +::: + +Placeholder | Description +------------|------------- +`` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher: \ + --no-cacerts +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ +#### Option D: Let's Encrypt Certificate + +
+ Click to expand + +:::caution + +Let's Encrypt provides rate limits for requesting new certificates. Therefore, limit how often you create or destroy the container. For more information, see [Let's Encrypt documentation on rate limits](https://letsencrypt.org/docs/rate-limits/). + +::: + +If you have selected to use [Let's Encrypt](https://letsencrypt.org/) certificates, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to provide the domain that you had used when you originally installed Rancher. + +:::note Reminder of the Cert Prerequisites: + +- Create a record in your DNS that binds your Linux host IP address to the hostname that you want to use for Rancher access (`rancher.mydomain.com` for example). +- Open port `TCP/80` on your Linux host. The Let's Encrypt http-01 challenge can come from any source IP address, so port `TCP/80` must be open to all IP addresses. + +::: + +Placeholder | Description +------------|------------- +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. +`` | The domain address that you had originally started with + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: \ + --acme-domain +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +
+ + + + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +When starting the new Rancher server container, choose from the following options: + +#### Option A: Default Self-Signed Certificate + +
+ Click to expand + +If you have selected to use the Rancher generated self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container. + +Placeholder | Description +------------|------------- +`` | Your private registry URL and port. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to to upgrade to. + +``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ +#### Option B: Bring Your Own Certificate: Self-Signed + +
+ Click to expand + +If you have selected to bring your own self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificate that you had originally installed with. + +:::note Reminder of the Cert Prerequisite: + +The certificate files must be in PEM format. In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](certificate-troubleshooting.md) + +::: + +Placeholder | Description +------------|------------- +`` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | The path to the certificate authority's certificate. +`` | Your private registry URL and port. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ +#### Option C: Bring Your Own Certificate: Signed by Recognized CA + +
+ Click to expand + +If you have selected to use a certificate signed by a recognized CA, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificates that you had originally installed with. + + :::note Reminder of the Cert Prerequisite: + + The certificate files must be in PEM format. In your certificate file, include all intermediate certificates provided by the recognized CA. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](certificate-troubleshooting.md) + + ::: + +Placeholder | Description +------------|------------- +`` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | Your private registry URL and port. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +:::note + +Use the `--no-cacerts` as argument to the container to disable the default CA certificate generated by Rancher. + +::: + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged + /rancher/rancher: +``` +privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ + + + +**Result:** You have upgraded Rancher. Data from your upgraded server is now saved to the `rancher-data` container for use in future upgrades. + +### 5. Verify the Upgrade + +Log into Rancher. Confirm that the upgrade succeeded by checking the version displayed in the bottom-left corner of the browser window. + +:::note Having network issues in your user clusters following upgrade? + +See [Restoring Cluster Networking](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md). + +::: + +### 6. Clean up Your Old Rancher Server Container + +Remove the previous Rancher server container. If you only stop the previous Rancher server container (and don't remove it), the container may restart after the next server reboot. + +## Rolling Back + +If your upgrade does not complete successfully, you can roll back Rancher server and its data back to its last healthy state. For more information, see [Docker Rollback](roll-back-docker-installed-rancher.md). diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/add-tls-secrets.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/add-tls-secrets.md new file mode 100644 index 00000000000..290f180adfd --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/add-tls-secrets.md @@ -0,0 +1,49 @@ +--- +title: Adding TLS Secrets +--- + + + + + +Kubernetes will create all the objects and services for Rancher, but it will not become available until we populate the `tls-rancher-ingress` secret in the `cattle-system` namespace with the certificate and key. + +Combine the server certificate followed by any intermediate certificate(s) needed into a file named `tls.crt`. Copy your certificate key into a file named `tls.key`. + +For example, [acme.sh](https://acme.sh) provides server certificate and CA chains in `fullchain.cer` file. +This `fullchain.cer` should be renamed to `tls.crt` & certificate key file as `tls.key`. + +Use `kubectl` with the `tls` secret type to create the secrets. + +``` +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +:::note + +If you want to replace the certificate, you can delete the `tls-rancher-ingress` secret using `kubectl -n cattle-system delete secret tls-rancher-ingress` and add a new one using the command shown above. If you are using a private CA signed certificate, replacing the certificate is only possible if the new certificate is signed by the same CA as the certificate currently in use. + +::: + +## Using a Private CA Signed Certificate + +If you are using a private CA, Rancher requires a copy of the private CA's root certificate or certificate chain, which the Rancher Agent uses to validate the connection to the server. + +Create a file named `cacerts.pem` that only contains the root CA certificate or certificate chain from your private CA, and use `kubectl` to create the `tls-ca` secret in the `cattle-system` namespace. + +``` +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem=./cacerts.pem +``` + +:::note + +The configured `tls-ca` secret is retrieved when Rancher starts. On a running Rancher installation the updated CA will take effect after new Rancher pods are started. + +::: + +## Updating a Private CA Certificate + +Follow the steps on [this page](update-rancher-certificate.md) to update the SSL certificate of the ingress in a Rancher [high availability Kubernetes installation](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) or to switch from the default self-signed certificate to a custom certificate. \ No newline at end of file diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/bootstrap-password.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/bootstrap-password.md new file mode 100644 index 00000000000..abb0d9a93dd --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/bootstrap-password.md @@ -0,0 +1,31 @@ +--- +title: Bootstrap Password +--- + + + + + +When Rancher starts for the first time, a password is randomly generated for the first admin user. When the admin first logs in to Rancher, the UI shows commands that can be used to retrieve the bootstrap password. The admin needs to run those commands and log in with the bootstrap password. Then Rancher gives the admin an opportunity to reset the password. + +The bootstrap password is randomly generated if it is not set during installation with a variable. For details on how to set the bootstrap password using a variable, see below. + +### Specifying the Bootstrap Password in Helm Installs + +For a Helm install, users can specify the bootstrap password variable by configuring it in the Helm chart values with `.Values.bootstrapPassword`. + +The password will be stored in a Kubernetes secret. After Rancher is installed, the UI will show instructions for how to retrieve the password using kubectl: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### Specifying the Bootstrap Password in Docker Installs + +For a Docker install, you can specify the bootstrap password by passing `-e CATTLE_BOOTSTRAP_PASSWORD=password` to the Docker install command. + +The password will be stored in the Docker container logs. After Rancher is installed, the UI will show instructions for how to retrieve the password using the Docker container ID: + +``` +docker logs container-id 2>&1 | grep "Bootstrap Password:" +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md new file mode 100644 index 00000000000..b2f0e8515e8 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md @@ -0,0 +1,123 @@ +--- +title: Choosing a Rancher Version +--- + + + + + +This section describes how to choose a Rancher version. + +For a high-availability installation of Rancher, which is recommended for production, the Rancher server is installed using a **Helm chart** on a Kubernetes cluster. Refer to the [Helm version requirements](helm-version-requirements.md) to choose a version of Helm to install Rancher. + +For Docker installations of Rancher, which is used for development and testing, you will install Rancher as a **Docker image**. + + + + +When installing, upgrading, or rolling back Rancher Server when it is [installed on a Kubernetes cluster](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md), Rancher server is installed using a Helm chart on a Kubernetes cluster. Therefore, as you prepare to install or upgrade a high availability Rancher configuration, you must add a Helm chart repository that contains the charts for installing Rancher. + +Refer to the [Helm version requirements](helm-version-requirements.md) to choose a version of Helm to install Rancher. + +### Helm Chart Repositories + +Rancher provides several different Helm chart repositories to choose from. We align our latest and stable Helm chart repositories with the Docker tags that are used for a Docker installation. Therefore, the `rancher-latest` repository will contain charts for all the Rancher versions that have been tagged as `rancher/rancher:latest`. When a Rancher version has been promoted to the `rancher/rancher:stable`, it will get added to the `rancher-stable` repository. + +| Type | Command to Add the Repo | Description of the Repo | +| -------------- | ------------ | ----------------- | +| rancher-latest | `helm repo add rancher-latest https://releases.rancher.com/server-charts/latest` | Adds a repository of Helm charts for the latest versions of Rancher. We recommend using this repo for testing out new Rancher builds. | +| rancher-stable | `helm repo add rancher-stable https://releases.rancher.com/server-charts/stable` | Adds a repository of Helm charts for older, stable versions of Rancher. We recommend using this repo for production environments. | +| rancher-alpha | `helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha` | Adds a repository of Helm charts for alpha versions of Rancher for previewing upcoming releases. These releases are discouraged in production environments. Upgrades _to_ or _from_ charts in the rancher-alpha repository to any other chart, regardless or repository, aren't supported. | + +
+Instructions on when to select these repos are available below in [Switching to a Different Helm Chart Repository](#switching-to-a-different-helm-chart-repository). + +:::note + +All charts in the `rancher-stable` repository will correspond with any Rancher version tagged as `stable`. + +::: + +### Helm Chart Versions + +Rancher Helm chart versions match the Rancher version (i.e `appVersion`). Once you've added the repo you can search it to show available versions with the following command:
+    `helm search repo --versions` + +If you have several repos you can specify the repo name, ie. `helm search repo rancher-stable/rancher --versions`
+For more information, see https://helm.sh/docs/helm/helm_search_repo/ + +To fetch a specific version of your chosen repo, define the `--version` parameter like in the following example:
+    `helm fetch rancher-stable/rancher --version=2.4.8` + +### Switching to a Different Helm Chart Repository + +After installing Rancher, if you want to change which Helm chart repository to install Rancher from, you will need to follow these steps. + +:::note + +Because the rancher-alpha repository contains only alpha charts, switching between the rancher-alpha repository and the rancher-stable or rancher-latest repository for upgrades is not supported. + +::: + +- Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + +1. List the current Helm chart repositories. + + ```plain + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + +2. Remove the existing Helm Chart repository that contains your charts to install Rancher, which will either be `rancher-stable` or `rancher-latest` depending on what you had initially added. + + ```plain + helm repo remove rancher- + ``` + +3. Add the Helm chart repository that you want to start installing Rancher from. + + ```plain + helm repo add rancher- https://releases.rancher.com/server-charts/ + ``` + +4. Continue to follow the steps to [upgrade Rancher](../install-upgrade-on-a-kubernetes-cluster/upgrades.md) from the new Helm chart repository. + + + + +When performing [Docker installs](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md), upgrades, or rollbacks, you can use _tags_ to install a specific version of Rancher. + +### Server Tags + +Rancher Server is distributed as a Docker image, which have tags attached to them. You can specify this tag when entering the command to deploy Rancher. Remember that if you use a tag without an explicit version (like `latest` or `stable`), you must explicitly pull a new version of that image tag. Otherwise, any image cached on the host will be used. + +| Tag | Description | +| -------------------------- | ------ | +| `rancher/rancher:latest` | Our latest development release. These builds are validated through our CI automation framework. These releases are not recommended for production environments. | +| `rancher/rancher:stable` | Our newest stable release. This tag is recommended for production. | +| `rancher/rancher:` | You can install specific versions of Rancher by using the tag from a previous release. See what's available at DockerHub. | + +:::note + +- The `master` tag or any tag with `-rc` or another suffix is meant for the Rancher testing team to validate. You should not use these tags, as these builds are not officially supported. +- Want to install an alpha review for preview? Install using one of the alpha tags listed on our [announcements page](https://forums.rancher.com/c/announcements) (e.g., `v2.2.0-alpha1`). Caveat: Alpha releases cannot be upgraded to or from any other release. + +::: + + + diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md new file mode 100644 index 00000000000..18d811c02f1 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md @@ -0,0 +1,28 @@ +--- +title: About Custom CA Root Certificates +--- + + + + + +If you're using Rancher in an internal production environment where you aren't exposing apps publicly, use a certificate from a private certificate authority (CA). + +Services that Rancher needs to access are sometimes configured with a certificate from a custom/internal CA root, also known as self signed certificate. If the presented certificate from the service cannot be validated by Rancher, the following error displays: `x509: certificate signed by unknown authority`. + +To validate the certificate, the CA root certificates need to be added to Rancher. As Rancher is written in Go, we can use the environment variable `SSL_CERT_DIR` to point to the directory where the CA root certificates are located in the container. The CA root certificates directory can be mounted using the Docker volume option (`-v host-source-directory:container-destination-directory`) when starting the Rancher container. + +Examples of services that Rancher can access: + +- Catalogs +- Authentication providers +- Accessing hosting/cloud API when using Node Drivers + +## Installing with the custom CA Certificate + +For details on starting a Rancher container with your private CA certificates mounted, refer to the installation docs: + +- [Docker install Custom CA certificate options](../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#custom-ca-certificate) + +- [Kubernetes install options for Additional Trusted CAs](../installation-references/helm-chart-options.md#additional-trusted-cas) + diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/helm-version-requirements.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/helm-version-requirements.md new file mode 100644 index 00000000000..8d98fe60458 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/helm-version-requirements.md @@ -0,0 +1,16 @@ +--- +title: Helm Version Requirements +--- + + + + + +This section contains the requirements for Helm, which is the tool used to install Rancher on a high-availability Kubernetes cluster. + +> The installation instructions have been updated for Helm 3. For migration of installs started with Helm 2, refer to the official [Helm 2 to 3 Migration Docs.](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/) [This section](/versioned_docs/version-2.0-2.4/pages-for-subheaders/helm2.md) provides a copy of the older high-availability Rancher installation instructions that used Helm 2, and it is intended to be used if upgrading to Helm 3 is not feasible. + +- Helm v3.2.x or higher is required to install or upgrade Rancher v2.5. +- Helm v2.16.0 or higher is required for Kubernetes v1.16. For the default Kubernetes version, refer to the [release notes](https://github.com/rancher/rke/releases) for the version of RKE that you are using. +- Helm v2.15.0 should not be used, because of an issue with converting/comparing numbers. +- Helm v2.12.0 should not be used, because of an issue with `cert-manager`. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/local-system-charts.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/local-system-charts.md new file mode 100644 index 00000000000..e48012d6af4 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/local-system-charts.md @@ -0,0 +1,17 @@ +--- +title: Setting up Local System Charts for Air Gapped Installations +--- + + + + + +The [Charts](https://github.com/rancher/charts) repository contains all the Helm catalog items required for features such as monitoring, logging, alerting and Istio. + +In an air gapped installation of Rancher, you will need to configure Rancher to use a local copy of the system charts. This section describes how to use local system charts using a CLI flag. + +## Using Local System Charts + +A local copy of `system-charts` has been packaged into the `rancher/rancher` container. To be able to use these features in an air gap install, you will need to run the Rancher install command with an extra environment variable, `CATTLE_SYSTEM_CATALOG=bundled`, which tells Rancher to use the local copy of the charts instead of attempting to fetch them from GitHub. + +Example commands for a Rancher installation with a bundled `system-charts` are included in the [air gap installation](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) instructions for Docker and Helm installs. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md new file mode 100644 index 00000000000..513b46ac4d2 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md @@ -0,0 +1,267 @@ +--- +title: Updating the Rancher Certificate +--- + + + + + +## Updating a Private CA Certificate + +Follow these steps to rotate an SSL certificate and private CA used by Rancher [installed on a Kubernetes cluster](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md), or migrate to an SSL certificate signed by a private CA. + +A summary of the steps is as follows: + +1. Create or update the `tls-rancher-ingress` Kubernetes secret object with the new certificate and private key. +1. Create or update the `tls-ca` Kubernetes secret object with the root CA certificate (only required when using a private CA). +1. Update the Rancher installation using the Helm CLI. +1. Reconfigure the Rancher agents to trust the new CA certificate. +1. Select Force Update of Fleet clusters to connect fleet-agent to Rancher. + +The details of these instructions are below. + +### 1. Create/update the certificate secret object + +First, concatenate the server certificate followed by any intermediate certificate(s) to a file named `tls.crt` and provide the corresponding certificate key in a file named `tls.key`. + +Use the following command to create the `tls-rancher-ingress` secret object in the Rancher (local) management cluster: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +Alternatively, to update an existing `tls-rancher-ingress` secret: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. Create/update the CA certificate secret object + +If the new certificate was signed by a private CA, you will need to copy the corresponding root CA certificate into a file named `cacerts.pem` and create or update the `tls-ca` secret in the `cattle-system` namespace. If the certificate was signed by an intermediate CA, then the `cacerts.pem` must contain both the intermediate and root CA certificates (in this order). + +To create the initial `tls-ca` secret: + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem +``` + +To update an existing `tls-ca` secret: + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 3. Reconfigure the Rancher deployment + +If the certificate source remains the same (for example, `secret`), please follow the steps in Step 3a. + +However, if the certificate source is changing (for example, `letsEncrypt` to `secret`), follow the steps in 3b. + +#### 3a. Redeploy the Rancher pods + +This step is required when the certificate source remains the same, but the CA certificate is being updated. + +In this scenario a redeploy of the Rancher pods is needed, this is because the `tls-ca` secret is read by the Rancher pods when starting. + +The command below can be used to redeploy the Rancher pods: +```bash +kubectl rollout restart deploy/rancher -n cattle-system +``` + +When the change is completed, navigate to `https:///v3/settings/cacerts` to verify that the value matches the CA certificate written in the `tls-ca` secret earlier. The CA `cacerts` value may not update until all of the redeployed Rancher pods start. + +#### 3b. Update the Helm values for Rancher + +This step is required if the certificate source is changing. If Rancher was previously configured to use the default self-signed certificate (`ingress.tls.source=rancher`) or Let's Encrypt (`ingress.tls.source=letsEncrypt`), and is now using a certificate signed by a private CA (`ingress.tls.source=secret`). + +The below steps update the Helm values for the Rancher chart, so the Rancher pods and ingress are reconfigured to use the new private CA certificate created in Step 1 & 2. + + 1. Adjust the values that were used during initial installation, store the current values with: + ```bash + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` + 1. Retrieve the version string of the currently deployed Rancher chart to use below: + ```bash + helm ls -n cattle-system + ``` + 1. Update the current Helm values in the `values.yaml` file to contain: + ```yaml + ingress: + tls: + source: secret + privateCA: true + ``` + :::note Important: + As the certificate is signed by a private CA, it is important to ensure [`privateCA: true`](../installation-references/helm-chart-options.md#common-options) is set in the `values.yaml` file. + ::: + 1. Upgrade the Helm application instance using the `values.yaml` file and the current chart version. The version must match to prevent an upgrade of Rancher. + ```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version + ``` + +When the change is completed, navigate to `https:///v3/settings/cacerts` to verify that the value matches the CA certificate written in the `tls-ca` secret earlier. The CA `cacerts` value may not update until all Rancher pods start. + +### 4. Reconfigure Rancher agents to trust the private CA + +This section covers three methods to reconfigure Rancher agents to trust the private CA. This step is required if either of the following is true: + +- Rancher was previously configured to use the Rancher self-signed certificate (`ingress.tls.source=rancher`) or with a Let's Encrypt issued certificate (`ingress.tls.source=letsEncrypt`) +- The certificate was signed by a different private CA + +#### Why is this step required? + +When Rancher is configured with a certificate signed by a private CA, the CA certificate chain is trusted by Rancher agent containers. Agents compare the checksum of the downloaded certificate against the `CATTLE_CA_CHECKSUM` environment variable. This means that, when the private CA certificate used by Rancher has changed, the environment variable `CATTLE_CA_CHECKSUM` must be updated accordingly. + +#### Which method should I choose? + +Method 1 is the easiest, but requires all clusters to be connected to Rancher after the certificates have been rotated. This is usually the case if the process is performed right after updating or redeploying the Rancher deployment (Step 3). + +If the clusters have lost connection to Rancher but [Authorized Cluster Endpoint](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) (ACE) is enabled on all clusters, then go with method 2. + +Method 3 can be used as a fallback if method 1 and 2 are not possible. + +#### Method 1: Force a redeploy of the Rancher agents + +For each downstream cluster run the following command using the Kubeconfig file of the Rancher (local) management cluster. + +```bash +kubectl annotate clusters.management.cattle.io io.cattle.agent.force.deploy=true +``` + +:::note +Locate the cluster ID (c-xxxxx) for the downstream cluster, this can be seen in the browser URL bar when viewing the cluster in the Rancher UI, under Cluster Management. +::: + +This command will cause the agent manifest to be reapplied with the checksum of the new certificate. + +#### Method 2: Manually update the checksum environment variable + +Manually patch the agent Kubernetes objects by updating the `CATTLE_CA_CHECKSUM` environment variable to the value matching the checksum of the new CA certificate. Generate the new checksum value like so: + +```bash +curl -k -s -fL /v3/settings/cacerts | jq -r .value | sha256sum | awk '{print $1}' +``` + +Using a Kubeconfig for each downstream cluster update the environment variable for the two agent deployments. If the [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) is enabled for the cluster, [the kubectl context can be adjusted](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to connect directly to the downstream cluster. + +```bash +kubectl edit -n cattle-system ds/cattle-node-agent +kubectl edit -n cattle-system deployment/cattle-cluster-agent +``` + +#### Method 3: Manually redeploy the Rancher agents + +With this method the Rancher agents are reapplied by running a set of commands on a control plane node of each downstream cluster. + +Repeat the below steps for each downstream cluster: + + 1. Retrieve the agent registration kubectl command: + 1. Locate the cluster ID (c-xxxxx) for the downstream cluster, this can be seen in the URL when viewing the cluster in the Rancher UI under Cluster Management + 1. Add the Rancher server URL and cluster ID to the following URL: `https:///v3/clusterregistrationtokens?clusterId=` + 1. Copy the command from the `insecureCommand` field, this command is used because a private CA is un use + + 2. Run the kubectl command from the previous step using a kubeconfig for the downstream cluster with one of the following methods: + 1. If the [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) is enabled for the cluster, [the context can be adjusted](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to connect directly to the downstream cluster + 1. Alternatively, SSH into the control plane node: + - RKE: Use the [steps in the document here](https://github.com/rancherlabs/support-tools/tree/master/how-to-retrieve-kubeconfig-from-custom-cluster) to generate a kubeconfig + - RKE2/K3s: Use the kubeconfig populated during installation + +### 5. Force Update Fleet clusters to reconnect the fleet-agent to Rancher + +Select 'Force Update' for the clusters within the [Continuous Delivery](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#accessing-fleet-in-the-rancher-ui) view of the Rancher UI to allow the fleet-agent in downstream clusters to successfully connect to Rancher. + +#### Why is this step required? + +Fleet agents in Rancher managed clusters store a kubeconfig that is used to connect to Rancher. The kubeconfig contains a `certificate-authority-data` field containing the CA for the certificate used by Rancher. When changing the CA, this block needs to be updated to allow the fleet-agent to trust the certificate used by Rancher. + +## Updating from a Private CA Certificate to a Public CA Certificate + +Follow these steps to perform the opposite procedure as shown above, to change from a certificate issued by a private CA, to a public or self-signed CA. + +### 1. Create/update the certificate secret object + +First, concatenate the server certificate followed by any intermediate certificate(s) to a file named `tls.crt` and provide the corresponding certificate key in a file named `tls.key`. + +Use the following command to create the `tls-rancher-ingress` secret object in the Rancher (local) management cluster: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +Alternatively, to update an existing `tls-rancher-ingress` secret: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. Delete the CA certificate secret object + +You will delete the `tls-ca` secret in the `cattle-system` namespace as it is no longer needed. You may also optionally save a copy of the `tls-ca` secret if desired. + +To save the existing `tls-ca` secret: + +```bash +kubectl -n cattle-system get secret tls-ca -o yaml > tls-ca.yaml +``` + +To delete the existing `tls-ca` secret: + +```bash +kubectl -n cattle-system delete secret tls-ca +``` + +### 3. Reconfigure the Rancher deployment + +This step is required if the certificate source is changing. In this scenario it's likely only changing because Rancher was previously configured to use the default self-signed certificate (`ingress.tls.source=rancher`). + +The below steps update the Helm values for the Rancher chart, so the Rancher pods and ingress are reconfigured to use the new certificate created in Step 1. + + 1. Adjust the values that were used during initial installation, store the current values with: + ```bash + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` + 1. Also get the version string of the currently deployed Rancher chart: + ```bash + helm ls -n cattle-system + ``` + 1. Update the current Helm values in the `values.yaml` file: + 1. As a private CA is no longer being used, remove the `privateCA: true` field, or set this to `false` + 1. Adjust the `ingress.tls.source` field as necessary. Please [refer to the chart options](../installation-references/helm-chart-options.md#common-options) for more details. Here are some examples: + 1. If using a public CA continue with a value of: `secret` + 1. If using Let's Encrypt update the value to: `letsEncrypt` + 1. Update the Helm values for the Rancher chart using the `values.yaml` file, and the current chart version to prevent an upgrade: + ```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version + ``` + +### 4. Reconfigure Rancher agents for the non-private/common certificate + +As a private CA is no longer being used, the `CATTLE_CA_CHECKSUM` environment variable on the downstream cluster agents should be removed or set to "" (an empty string). + +### 5. Force Update Fleet clusters to reconnect the fleet-agent to Rancher + +Select 'Force Update' for the clusters within the [Continuous Delivery](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#accessing-fleet-in-the-rancher-ui) view of the Rancher UI to allow the fleet-agent in downstream clusters to successfully connect to Rancher. + +#### Why is this step required? + +Fleet agents in Rancher managed clusters store a kubeconfig that is used to connect to Rancher. The kubeconfig contains a `certificate-authority-data` field containing the CA for the certificate used by Rancher. When changing the CA, this block needs to be updated to allow the fleet-agent to trust the certificate used by Rancher. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md new file mode 100644 index 00000000000..9a9ec2b8f83 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md @@ -0,0 +1,285 @@ +--- +title: Upgrading Cert-Manager +--- + + + + + +Rancher uses cert-manager to automatically generate and renew TLS certificates for HA deployments of Rancher. As of Fall 2019, three important changes to cert-manager are set to occur that you need to take action on if you have an HA deployment of Rancher: + +1. [Let's Encrypt will be blocking cert-manager instances older than 0.8.0 starting November 1st 2019.](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) +1. [Cert-manager is deprecating and replacing the certificate.spec.acme.solvers field](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/). This change has no exact deadline. +1. [Cert-manager is deprecating `v1alpha1` API and replacing its API group](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/) + +To address these changes, this guide will do two things: + +1. Document the procedure for upgrading cert-manager +1. Explain the cert-manager API changes and link to cert-manager's official documentation for migrating your data + +:::note Important: + +If you are upgrading cert-manager to the latest version from a version older than 1.5, follow the steps in [Option C](#option-c-upgrade-cert-manager-from-versions-15-and-below) below to do so. Note that you do not need to reinstall Rancher to perform this upgrade. + +::: + +## Upgrade Cert-Manager + +The namespace used in these instructions depends on the namespace cert-manager is currently installed in. If it is in kube-system use that in the instructions below. You can verify by running `kubectl get pods --all-namespaces` and checking which namespace the cert-manager-\* pods are listed in. Do not change the namespace cert-manager is running in or this can cause issues. + +In order to upgrade cert-manager, follow these instructions: + +### Option A: Upgrade cert-manager with Internet Access + +
+ Click to expand + +1. [Back up existing resources](https://cert-manager.io/docs/tutorials/backup/) as a precaution + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note Important: + + If you are upgrading from a version older than 0.11.0, Update the apiVersion on all your backed up resources from `certmanager.k8s.io/v1alpha1` to `cert-manager.io/v1alpha2`. If you use any cert-manager annotations on any of your other resources, you will need to update them to reflect the new API group. For details, refer to the documentation on [additional annotation changes.](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes) + + ::: + +1. [Uninstall existing deployment](https://cert-manager.io/docs/installation/uninstall/kubernetes/#uninstalling-with-helm) + + ```plain + helm uninstall cert-manager + ``` + + Delete the CustomResourceDefinition using the link to the version vX.Y.Z you installed + + ```plain + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + +1. Install the CustomResourceDefinition resources separately + + ```plain + kubectl apply --validate=false -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + + :::note + + If you are running Kubernetes v1.15 or below, you will need to add the `--validate=false` flag to your `kubectl apply` command above. Otherwise, you will receive a validation error relating to the `x-kubernetes-preserve-unknown-fields` field in cert-manager’s CustomResourceDefinition resources. This is a benign error and occurs due to the way kubectl performs resource validation. + + ::: + +1. Create the namespace for cert-manager if needed + + ```plain + kubectl create namespace cert-manager + ``` + +1. Add the Jetstack Helm repository + + ```plain + helm repo add jetstack https://charts.jetstack.io + ``` + +1. Update your local Helm chart repository cache + + ```plain + helm repo update + ``` + +1. Install the new version of cert-manager + + ```plain + helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.11.0 + ``` + +1. [Restore back up resources](https://cert-manager.io/docs/tutorials/backup/#restoring-resources) + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### Option B: Upgrade cert-manager in an Air-Gapped Environment + +
+ Click to expand + +### Prerequisites + +Before you can perform the upgrade, you must prepare your air gapped environment by adding the necessary container images to your private registry and downloading or rendering the required Kubernetes manifest files. + +1. Follow the guide to [Prepare your Private Registry](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md) with the images needed for the upgrade. + +1. From a system connected to the internet, add the cert-manager repo to Helm + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + ``` + +1. Fetch the latest cert-manager chart available from the [Helm chart repository](https://artifacthub.io/packages/helm/cert-manager/cert-manager). + + ```plain + helm fetch jetstack/cert-manager --version v1.11.0 + ``` + +1. Render the cert manager template with the options you would like to use to install the chart. Remember to set the `image.repository` option to pull the image from your private registry. This will create a `cert-manager` directory with the Kubernetes manifest files. + + The Helm 3 command is as follows: + + ```plain + helm template cert-manager ./cert-manager-v0.12.0.tgz --output-dir . \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + + The Helm 2 command is as follows: + + ```plain + helm template ./cert-manager-v0.12.0.tgz --output-dir . \ + --name cert-manager --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + +1. Download the required CRD file for cert-manager (old and new) + + ```plain + curl -L -o cert-manager-crd.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-0.12/deploy/manifests/00-crds.yaml + curl -L -o cert-manager/cert-manager-crd-old.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-X.Y/deploy/manifests/00-crds.yaml + ``` + +### Install cert-manager + +1. Back up existing resources as a precaution + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note Important: + + If you are upgrading from a version older than 0.11.0, Update the apiVersion on all your backed up resources from `certmanager.k8s.io/v1alpha1` to `cert-manager.io/v1alpha2`. If you use any cert-manager annotations on any of your other resources, you will need to update them to reflect the new API group. For details, refer to the documentation on [additional annotation changes.](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes) + + ::: + +1. Delete the existing cert-manager installation + + ```plain + kubectl -n cert-manager \ + delete deployment,sa,clusterrole,clusterrolebinding \ + -l 'app=cert-manager' -l 'chart=cert-manager-v0.5.2' + ``` + + Delete the CustomResourceDefinition using the link to the version vX.Y you installed + + ```plain + kubectl delete -f cert-manager/cert-manager-crd-old.yaml + ``` + +1. Install the CustomResourceDefinition resources separately + + ```plain + kubectl apply -f cert-manager/cert-manager-crd.yaml + ``` + + :::note Important: + + If you are running Kubernetes v1.15 or below, you will need to add the `--validate=false` flag to your `kubectl apply` command above. Otherwise, you will receive a validation error relating to the `x-kubernetes-preserve-unknown-fields` field in cert-manager’s CustomResourceDefinition resources. This is a benign error and occurs due to the way kubectl performs resource validation. + + ::: + +1. Create the namespace for cert-manager + + ```plain + kubectl create namespace cert-manager + ``` + +1. Install cert-manager + + ```plain + kubectl -n cert-manager apply -R -f ./cert-manager + ``` + +1. [Restore back up resources](https://cert-manager.io/docs/tutorials/backup/#restoring-resources) + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### Option C: Upgrade cert-manager from Versions 1.5 and Below + +
+ Click to expand + +Previously, in order to upgrade cert-manager from an older version, an uninstall and reinstall of Rancher was recommended. Using the method below, you may upgrade cert-manager without those additional steps in order to better preserve your production environment: + +1. Install `cmctl`, the cert-manager CLI tool, using [the installation guide](https://cert-manager.io/docs/usage/cmctl/#installation). + +1. Ensure that any cert-manager custom resources that may have been stored in etcd at a deprecated API version get migrated to v1: + + ``` + cmctl upgrade migrate-api-version + ``` + Refer to the [API version migration docs](https://cert-manager.io/docs/usage/cmctl/#migrate-api-version) for more information. Please also see the [docs to upgrade from 1.5 to 1.6](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/) and the [docs to upgrade from 1.6. to 1.7](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/) if needed. + +1. Upgrade cert-manager to v1.7.1 with a normal `helm upgrade`. You may go directly from version 1.5 to 1.7 if desired. + +1. Follow the Helm tutorial to [update the API version of a release manifest](https://helm.sh/docs/topics/kubernetes_apis/#updating-api-versions-of-a-release-manifest). The chart release name is `release_name=rancher` and the release namespace is `release_namespace=cattle-system`. + +1. In the decoded file, search for `cert-manager.io/v1beta1` and **replace it** with `cert-manager.io/v1`. + +1. Upgrade Rancher normally with `helm upgrade`. + +
+ +### Verify the Deployment + +Once you’ve installed cert-manager, you can verify it is deployed correctly by checking the kube-system namespace for running pods: + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +## Cert-Manager API change and data migration + +--- + +Rancher now supports cert-manager versions 1.6.2 and 1.7.1. We recommend v1.7.x because v 1.6.x will reach end-of-life on March 30, 2022. To read more, see the [cert-manager docs](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-install-cert-manager). For instructions on upgrading cert-manager from version 1.5 to 1.6, see the upstream cert-manager documentation [here](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/). For instructions on upgrading cert-manager from version 1.6 to 1.7, see the upstream cert-manager documentation [here](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/). + +--- + +Cert-manager has deprecated the use of the `certificate.spec.acme.solvers` field and will drop support for it completely in an upcoming release. + +Per the cert-manager documentation, a new format for configuring ACME certificate resources was introduced in v0.8. Specifically, the challenge solver configuration field was moved. Both the old format and new are supported as of v0.9, but support for the old format will be dropped in an upcoming release of cert-manager. The cert-manager documentation strongly recommends that after upgrading you update your ACME Issuer and Certificate resources to the new format. + +Details about the change and migration instructions can be found in the [cert-manager v0.7 to v0.8 upgrade instructions](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/). + +The v0.11 release marks the removal of the v1alpha1 API that was used in previous versions of cert-manager, as well as our API group changing to be cert-manager.io instead of certmanager.k8s.io. + +We have also removed support for the old configuration format that was deprecated in the v0.8 release. This means you must transition to using the new solvers style configuration format for your ACME issuers before upgrading to v0.11. For more information, see the [upgrading to v0.8 guide](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/). + +Details about the change and migration instructions can be found in the [cert-manager v0.10 to v0.11 upgrade instructions](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/). + +More info about [cert-manager upgrade information](https://cert-manager.io/docs/installation/upgrading/). + diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md new file mode 100644 index 00000000000..3e89d8584ca --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md @@ -0,0 +1,130 @@ +--- +title: Upgrading and Rolling Back Kubernetes +--- + + + + + +Following an upgrade to the latest version of Rancher, downstream Kubernetes clusters can be upgraded to use the latest supported version of Kubernetes. + +Rancher calls RKE (Rancher Kubernetes Engine) as a library when provisioning and editing RKE clusters. For more information on configuring the upgrade strategy for RKE clusters, refer to the [RKE documentation](https://rancher.com/docs/rke/latest/en/). + + +## Tested Kubernetes Versions + +Before a new version of Rancher is released, it's tested with the latest minor versions of Kubernetes to ensure compatibility. For details on which versions of Kubernetes were tested on each Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/) + +## How Upgrades Work + +RKE v1.1.0 changed the way that clusters are upgraded. + +In this section of the [RKE documentation,](https://rancher.com/docs/rke/latest/en/upgrades/how-upgrades-work) you'll learn what happens when you edit or upgrade your RKE Kubernetes cluster. + + +## Recommended Best Practice for Upgrades + +When upgrading the Kubernetes version of a cluster, we recommend that you: + +1. Take a snapshot. +1. Initiate a Kubernetes upgrade. +1. If the upgrade fails, revert the cluster to the pre-upgrade Kubernetes version. This is achieved by selecting the **Restore etcd and Kubernetes version** option. This will return your cluster to the pre-upgrade kubernetes version before restoring the etcd snapshot. + +The restore operation will work on a cluster that is not in a healthy or active state. + +## Upgrading the Kubernetes Version + +:::note Prerequisites: + +- The options below are available for [Rancher-launched Kubernetes clusters](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) and [Registered K3s Kubernetes clusters](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#additional-features-for-registered-rke2-and-k3s-clusters). +- The following options also apply to imported RKE2 clusters that you have registered. If you import a cluster from an external cloud platform but don't register it, you won't be able to upgrade the Kubernetes version from Rancher. +- Before upgrading Kubernetes, [back up your cluster.](../../pages-for-subheaders/backup-restore-and-disaster-recovery.md) + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to upgrade and click **⋮ > Edit Config**. +1. From the **Kubernetes Version** drop-down, choose the version of Kubernetes that you want to use for the cluster. +1. Click **Save**. + +**Result:** Kubernetes begins upgrading for the cluster. + +## Rolling Back + +A cluster can be restored to a backup in which the previous Kubernetes version was used. For more information, refer to the following sections: + +- [Backing up a cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md#how-snapshots-work) +- [Restoring a cluster from backup](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md#restoring-a-cluster-from-a-snapshot) + +## Configuring the Upgrade Strategy + +As of RKE v1.1.0, additional upgrade options became available to give you more granular control over the upgrade process. These options can be used to maintain availability of your applications during a cluster upgrade if certain [conditions and requirements](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability) are met. + +The upgrade strategy can be configured in the Rancher UI, or by editing the `cluster.yml`. More advanced options are available by editing the `cluster.yml`. + +### Configuring the Maximum Unavailable Worker Nodes in the Rancher UI + +From the Rancher UI, the maximum number of unavailable worker nodes can be configured. During a cluster upgrade, worker nodes will be upgraded in batches of this size. + +By default, the maximum number of unavailable worker is defined as 10 percent of all worker nodes. This number can be configured as a percentage or as an integer. When defined as a percentage, the batch size is rounded down to the nearest node, with a minimum of one node. + +To change the default number or percentage of worker nodes, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to upgrade and click **⋮ > Edit Config**. +1. In the **Upgrade Strategy** tab, enter the **Worker Concurrency** as a fixed number or percentage. To get this number, you can take the number of nodes in your cluster and subtract the max unavailable nodes. +1. Click **Save**. + +**Result:** The cluster is updated to use the new upgrade strategy. + +### Enabling Draining Nodes During Upgrades from the Rancher UI + +By default, RKE [cordons](https://kubernetes.io/docs/concepts/architecture/nodes/#manual-node-administration) each node before upgrading it. [Draining](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/) is disabled during upgrades by default. If draining is enabled in the cluster configuration, RKE will both cordon and drain the node before it is upgraded. + +To enable draining each node during a cluster upgrade, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to enable node draining and click **⋮ > Edit Config**. +1. Click **⋮ > Edit**. +1. In the **Upgrade Strategy** tab, go to the **Drain nodes** field and click **Yes**. Node draining is configured separately for control plane and worker nodes. +1. Configure the options for how pods are deleted. For more information about each option, refer to [this section.](../../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md#aggressive-and-safe-draining-options) +1. Optionally, configure a grace period. The grace period is the timeout given to each pod for cleaning things up, so they will have chance to exit gracefully. Pods might need to finish any outstanding requests, roll back transactions or save state to some external storage. If this value is negative, the default value specified in the pod will be used. +1. Optionally, configure a timeout, which is the amount of time the drain should continue to wait before giving up. +1. Click **Save**. + +**Result:** The cluster is updated to use the new upgrade strategy. + +:::note + +There is a [known issue](https://github.com/rancher/rancher/issues/25478) in which the Rancher UI doesn't show state of etcd and controlplane as drained, even though they are being drained. + +::: + +### Maintaining Availability for Applications During Upgrades + +_Available as of RKE v1.1.0_ + +In [this section of the RKE documentation,](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability/) you'll learn the requirements to prevent downtime for your applications when upgrading the cluster. + +### Configuring the Upgrade Strategy in the cluster.yml + +More advanced upgrade strategy configuration options are available by editing the `cluster.yml`. + +For details, refer to [Configuring the Upgrade Strategy](https://rancher.com/docs/rke/latest/en/upgrades/configuring-strategy) in the RKE documentation. The section also includes an example `cluster.yml` for configuring the upgrade strategy. + +## Troubleshooting + +If a node doesn't come up after an upgrade, the `rke up` command errors out. + +No upgrade will proceed if the number of unavailable nodes exceeds the configured maximum. + +If an upgrade stops, you may need to fix an unavailable node or remove it from the cluster before the upgrade can continue. + +A failed node could be in many different states: + +- Powered off +- Unavailable +- User drains a node while upgrade is in process, so there are no kubelets on the node +- The upgrade itself failed + +If the max unavailable number of nodes is reached during an upgrade, Rancher user clusters will be stuck in updating state and not move forward with upgrading any other control plane nodes. It will continue to evaluate the set of unavailable nodes in case one of the nodes becomes available. If the node cannot be fixed, you must remove the node in order to continue the upgrade. diff --git a/versioned_docs/version-2.8/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md new file mode 100644 index 00000000000..42c0b6348a2 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md @@ -0,0 +1,95 @@ +--- +title: Upgrading Kubernetes without Upgrading Rancher +--- + + + + + +The RKE metadata feature allows you to provision clusters with new versions of Kubernetes as soon as they are released, without upgrading Rancher. This feature is useful for taking advantage of patch versions of Kubernetes, for example, if you want to upgrade to Kubernetes v1.14.7 when your Rancher server originally supported v1.14.6. + +:::note + +The Kubernetes API can change between minor versions. Therefore, we don't support introducing minor Kubernetes versions, such as introducing v1.15 when Rancher currently supports v1.14. You would need to upgrade Rancher to add support for minor Kubernetes versions. + +::: + +Rancher's Kubernetes metadata contains information specific to the Kubernetes version that Rancher uses to provision [RKE clusters](../../pages-for-subheaders/launch-kubernetes-with-rancher.md). Rancher syncs the data periodically and creates custom resource definitions (CRDs) for **system images,** **service options** and **addon templates**. Consequently, when a new Kubernetes version is compatible with the Rancher server version, the Kubernetes metadata makes the new version available to Rancher for provisioning clusters. The metadata gives you an overview of the information that the [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) uses for deploying various Kubernetes versions. + +This table below describes the CRDs that are affected by the periodic data sync. + +:::note + +Only administrators can edit metadata CRDs. It is recommended not to update existing objects unless explicitly advised. + +::: + +| Resource | Description | Rancher API URL | +|----------|-------------|-----------------| +| System Images | List of system images used to deploy Kubernetes through RKE. | `/v3/rkek8ssystemimages` | +| Service Options | Default options passed to Kubernetes components like `kube-api`, `scheduler`, `kubelet`, `kube-proxy`, and `kube-controller-manager` | `/v3/rkek8sserviceoptions` | +| Addon Templates | YAML definitions used to deploy addon components like Canal, Calico, Flannel, Weave, Kube-dns, CoreDNS, `metrics-server`, `nginx-ingress` | `/v3/rkeaddons` | + +Administrators might configure the RKE metadata settings to do the following: + +- Refresh the Kubernetes metadata, if a new patch version of Kubernetes comes out and they want Rancher to provision clusters with the latest version of Kubernetes without having to upgrade Rancher +- Change the metadata URL that Rancher uses to sync the metadata, which is useful for air gap setups if you need to sync Rancher locally instead of with GitHub +- Prevent Rancher from auto-syncing the metadata, which is one way to prevent new and unsupported Kubernetes versions from being available in Rancher + +### Refresh Kubernetes Metadata + +The option to refresh the Kubernetes metadata is available for administrators by default, or for any user who has the **Manage Cluster Drivers** [global role.](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) + +To force Rancher to refresh the Kubernetes metadata, a manual refresh action is available: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **Drivers**. +1. Click **Refresh Kubernetes Metadata**. + +You can configure Rancher to only refresh metadata when desired by setting `refresh-interval-minutes` to `0` (see below) and using this button to perform the metadata refresh manually when desired. + +### Configuring the Metadata Synchronization + +:::caution + +Only administrators can change these settings. + +::: + +The RKE metadata config controls how often Rancher syncs metadata and where it downloads data from. You can configure the metadata from the settings in the Rancher UI, or through the Rancher API at the endpoint `v3/settings/rke-metadata-config`. + +The way that the metadata is configured depends on the Rancher version. + +To edit the metadata config in Rancher, + +1. In the upper left corner, click **☰ > Global Settings**. +1. Go to the **rke-metadata-config** section. Click **⋮ > Edit Setting**. +1. You can optionally fill in the following parameters: + + - `refresh-interval-minutes`: This is the amount of time that Rancher waits to sync the metadata. To disable the periodic refresh, set `refresh-interval-minutes` to 0. + - `url`: This is the HTTP path that Rancher fetches data from. The path must be a direct path to a JSON file. For example, the default URL for Rancher v2.4 is `https://releases.rancher.com/kontainer-driver-metadata/release-v2.4/data.json`. +1. Click **Save**. + +If you don't have an air gap setup, you don't need to specify the URL where Rancher gets the metadata, because the default setting is to pull from [Rancher's metadata Git repository.](https://github.com/rancher/kontainer-driver-metadata/blob/dev-v2.5/data/data.json) + +However, if you have an [air gap setup,](#air-gap-setups) you will need to mirror the Kubernetes metadata repository in a location available to Rancher. Then you need to change the URL to point to the new location of the JSON file. + +### Air Gap Setups + +Rancher relies on a periodic refresh of the `rke-metadata-config` to download new Kubernetes version metadata if it is supported with the current version of the Rancher server. For a table of compatible Kubernetes and Rancher versions, refer to the [service terms section.](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.2.8/) + +If you have an air gap setup, you might not be able to get the automatic periodic refresh of the Kubernetes metadata from Rancher's Git repository. In that case, you should disable the periodic refresh to prevent your logs from showing errors. Optionally, you can configure your metadata settings so that Rancher can sync with a local copy of the RKE metadata. + +To sync Rancher with a local mirror of the RKE metadata, an administrator would configure the `rke-metadata-config` settings to point to the mirror. For details, refer to [Configuring the Metadata Synchronization.](#configuring-the-metadata-synchronization) + +After new Kubernetes versions are loaded into the Rancher setup, additional steps would be required in order to use them for launching clusters. Rancher needs access to updated system images. While the metadata settings can only be changed by administrators, any user can download the Rancher system images and prepare a private container image registry for them. + +To download the system images for the private registry: + +1. Click **☰** in the top left corner. +1. Click **About** at the bottom of the left navigation. +1. Download the OS specific image lists for Linux or Windows. +1. Download `rancher-images.txt`. +1. Prepare the private registry using the same steps during the [air gap install](other-installation-methods/air-gapped-helm-cli-install/publish-images.md), but instead of using the `rancher-images.txt` from the releases page, use the one obtained from the previous steps. + +**Result:** The air gap installation of Rancher can now sync the Kubernetes metadata. If you update your private registry when new versions of Kubernetes are released, you can provision clusters with the new version without having to upgrade Rancher. diff --git a/versioned_docs/version-2.8/getting-started/overview.md b/versioned_docs/version-2.8/getting-started/overview.md new file mode 100644 index 00000000000..b7f35276bc6 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/overview.md @@ -0,0 +1,69 @@ +--- +title: Overview +--- + + + + + +Rancher is a container management platform built for organizations that deploy containers in production. Rancher makes it easy to run Kubernetes everywhere, meet IT requirements, and empower DevOps teams. + +## Run Kubernetes Everywhere + +Kubernetes has become the container orchestration standard. Most cloud and virtualization vendors now offer it as standard infrastructure. Rancher users have the choice of creating Kubernetes clusters with Rancher Kubernetes Engine (RKE) or cloud Kubernetes services, such as GKE, AKS, and EKS. Rancher users can also import and manage their existing Kubernetes clusters created using any Kubernetes distribution or installer. + +## Meet IT Requirements + +Rancher supports centralized authentication, access control, and monitoring for all Kubernetes clusters under its control. For example, you can: + +- Use your Active Directory credentials to access Kubernetes clusters hosted by cloud vendors, such as GKE. +- Setup and enforce access control and security policies across all users, groups, projects, clusters, and clouds. +- View the health and capacity of your Kubernetes clusters from a single-pane-of-glass. + +## Empower DevOps Teams + +Rancher provides an intuitive user interface for DevOps engineers to manage their application workload. The user does not need to have in-depth knowledge of Kubernetes concepts to start using Rancher. Rancher catalog contains a set of useful DevOps tools. Rancher is certified with a wide selection of cloud native ecosystem products, including, for example, security tools, monitoring systems, container registries, and storage and networking drivers. + +The following figure illustrates the role Rancher plays in IT and DevOps organizations. Each team deploys their applications on the public or private clouds they choose. IT administrators gain visibility and enforce policies across all users, clusters, and clouds. + +![Platform](/img/platform.png) + +## Features of the Rancher API Server + +The Rancher API server is built on top of an embedded Kubernetes API server and an etcd database. It implements the following functionalities: + +### Authorization and Role-Based Access Control + +- **User management:** The Rancher API server [manages user identities](../pages-for-subheaders/authentication-config.md) that correspond to external authentication providers like Active Directory or GitHub, in addition to local users. +- **Authorization:** The Rancher API server manages [access control](../pages-for-subheaders/manage-role-based-access-control-rbac.md) and [security](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) policies. + +### Working with Kubernetes + +- **Provisioning Kubernetes clusters:** The Rancher API server can [provision Kubernetes](../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) on existing nodes, or perform [Kubernetes upgrades.](installation-and-upgrade/upgrade-and-roll-back-kubernetes.md) +- **Catalog management:** Rancher provides the ability to use a [catalog of Helm charts](../pages-for-subheaders/helm-charts-in-rancher.md) that make it easy to repeatedly deploy applications. +- **Managing projects:** A project is a group of multiple namespaces and access control policies within a cluster. A project is a Rancher concept, not a Kubernetes concept, which allows you to manage multiple namespaces as a group and perform Kubernetes operations in them. The Rancher UI provides features for [project administration](../pages-for-subheaders/manage-projects.md) and for [managing applications within projects.](../pages-for-subheaders/kubernetes-resources-setup.md) +- **Fleet Continuous Delivery:** Within Rancher, you can leverage [Fleet Continuous Delivery](../pages-for-subheaders/fleet-gitops-at-scale.md) to deploy applications from git repositories, without any manual operation, to targeted downstream Kubernetes clusters. +- **Istio:** Our [integration with Istio](../pages-for-subheaders/istio.md) is designed so that a Rancher operator, such as an administrator or cluster owner, can deliver Istio to developers. Then developers can use Istio to enforce security policies, troubleshoot problems, or manage traffic for green/blue deployments, canary deployments, or A/B testing. + +### Working with Cloud Infrastructure + +- **Tracking nodes:** The Rancher API server tracks identities of all the [nodes](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md) in all clusters. +- **Setting up infrastructure:** When configured to use a cloud provider, Rancher can dynamically provision [new nodes](../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) and [persistent storage](../pages-for-subheaders/create-kubernetes-persistent-storage.md) in the cloud. + +### Cluster Visibility + +- **Logging:** Rancher can integrate with a variety of popular logging services and tools that exist outside of your Kubernetes clusters. +- **Monitoring:** Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with Prometheus, a leading open-source monitoring solution. +- **Alerting:** To keep your clusters and applications healthy and driving your organizational productivity forward, you need to stay informed of events occurring in your clusters and projects, both planned and unplanned. + +## Editing Downstream Clusters with Rancher + +The options and settings available for an existing cluster change based on the method that you used to provision it. For example, only clusters [provisioned by RKE](../pages-for-subheaders/launch-kubernetes-with-rancher.md) have **Cluster Options** available for editing. + +After a cluster is created with Rancher, a cluster administrator can manage cluster membership or manage node pools, among [other options.](../pages-for-subheaders/cluster-configuration.md) + +The following table summarizes the options and settings available for each cluster type: + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md new file mode 100644 index 00000000000..875025ee424 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md @@ -0,0 +1,10 @@ +--- +title: Rancher AWS Marketplace Quick Start +description: Use Amazon EKS to deploy Rancher server. +--- + + + + + +Amazon Elastic Kubernetes Service (EKS) can quickly [deploy Rancher to Amazon Web Services (AWS)](https://documentation.suse.com/trd/kubernetes/single-html/gs_rancher_aws-marketplace/). To learn more, see our [Amazon Marketplace listing](https://aws.amazon.com/marketplace/pp/prodview-go7ent7goo5ae). Watch the [demo](https://youtu.be/9dznJ7Ons0M) for a walkthrough of AWS Marketplace SUSE Rancher setup. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws.md new file mode 100644 index 00000000000..a3fd249d35e --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/aws.md @@ -0,0 +1,99 @@ +--- +title: Rancher AWS Quick Start Guide +description: Read this step by step Rancher AWS guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on AWS in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Amazon AWS will incur charges. + +::: + +- [Amazon AWS Account](https://aws.amazon.com/account/): An Amazon AWS Account is required to create resources for deploying Rancher and Kubernetes. +- [Amazon AWS Access Key](https://docs.aws.amazon.com/general/latest/gr/managing-aws-access-keys.html): Use this link to follow a tutorial to create an Amazon AWS Access Key if you don't have one yet. +- [IAM Policy created](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start): Defines the permissions an account attached with this policy has. +- Install [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Amazon AWS. + +### Example IAM Policy + +The AWS module just creates an EC2 KeyPair, an EC2 SecurityGroup and an EC2 instance. A simple policy would be: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:*", + "Resource": "*" + } + ] +} +``` + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the AWS folder containing the Terraform files by executing `cd quickstart/rancher/aws`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + + - `aws_access_key` - Amazon AWS Access Key + - `aws_secret_key` - Amazon AWS Secret Key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [AWS Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/aws) for more information. +Suggestions include: + + - `aws_region` - Amazon AWS region, choose the closest instead of the default (`us-east-1`) + - `prefix` - Prefix for all created resources + - `instance_type` - EC2 instance size used, minimum is `t3a.medium` but `t3a.large` or `t3a.xlarge` could be used if within budget + - `add_windows_node` - If true, an additional Windows worker node is added to the workload cluster + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/aws`. + +##### Result + +Two Kubernetes clusters are deployed into your AWS account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +## What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/aws` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/azure.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/azure.md new file mode 100644 index 00000000000..c9b968077ab --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/azure.md @@ -0,0 +1,85 @@ +--- +title: Rancher Azure Quick Start Guide +description: Read this step by step Rancher Azure guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on Azure in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Microsoft Azure will incur charges. + +::: + +- [Microsoft Azure Account](https://azure.microsoft.com/en-us/free/): A Microsoft Azure Account is required to create resources for deploying Rancher and Kubernetes. +- [Microsoft Azure Subscription](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal): Use this link to follow a tutorial to create a Microsoft Azure subscription if you don't have one yet. +- [Micsoroft Azure Tenant](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant): Use this link and follow instructions to create a Microsoft Azure tenant. +- [Microsoft Azure Client ID/Secret](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal): Use this link and follow instructions to create a Microsoft Azure client and secret. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Microsoft Azure. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the Azure folder containing the Terraform files by executing `cd quickstart/rancher/azure`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `azure_subscription_id` - Microsoft Azure Subscription ID + - `azure_client_id` - Microsoft Azure Client ID + - `azure_client_secret` - Microsoft Azure Client Secret + - `azure_tenant_id` - Microsoft Azure Tenant ID + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [Azure Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/azure) for more information. Suggestions include: + - `azure_location` - Microsoft Azure region, choose the closest instead of the default (`East US`) + - `prefix` - Prefix for all created resources + - `instance_type` - Compute instance size used, minimum is `Standard_DS2_v2` but `Standard_DS2_v3` or `Standard_DS3_v2` could be used if within budget + - `add_windows_node` - If true, an additional Windows worker node is added to the workload cluster + - `windows_admin_password` - The admin password of the windows worker node + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/azure`. + +#### Result + +Two Kubernetes clusters are deployed into your Azure account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/azure` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md new file mode 100644 index 00000000000..81442f401b9 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md @@ -0,0 +1,78 @@ +--- +title: Rancher DigitalOcean Quick Start Guide +description: Read this step by step Rancher DigitalOcean guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on DigitalOcean in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to DigitalOcean will incur charges. + +::: + +- [DigitalOcean Account](https://www.digitalocean.com): You will require an account on DigitalOcean as this is where the server and cluster will run. +- [DigitalOcean Access Key](https://www.digitalocean.com/community/tutorials/how-to-create-a-digitalocean-space-and-api-key): Use this link to create a DigitalOcean Access Key if you don't have one. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster to DigitalOcean. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the DigitalOcean folder containing the Terraform files by executing `cd quickstart/rancher/do`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `do_token` - DigitalOcean access key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [DO Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/do) for more information. Suggestions include: + - `do_region` - DigitalOcean region, choose the closest instead of the default (`nyc1`) + - `prefix` - Prefix for all created resources + - `droplet_size` - Droplet size used, minimum is `s-2vcpu-4gb` but `s-4vcpu-8gb` could be used if within budget + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/do`. + +#### Result + +Two Kubernetes clusters are deployed into your DigitalOcean account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/do` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md new file mode 100644 index 00000000000..8a8d483492f --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md @@ -0,0 +1,110 @@ +--- +title: Rancher Equinix Metal Quick Start +--- + + + + + +## This tutorial walks you through the following: + +- Provisioning an Equinix Metal Server +- Installation of Rancher 2.x +- Creation of your first cluster +- Deployment of an application, Nginx + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. The Docker install is not recommended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Quick Start Outline + +This Quick Start Guide is divided into different tasks for easier consumption. + +
+ +## Prerequisites + +- An [Equinix Metal account](https://metal.equinix.com/developers/docs/accounts/users/) +- An [Equinix Metal project](https://metal.equinix.com/developers/docs/accounts/projects/) + + +### 1. Provision a Equinix Metal Host + +Begin deploying an Equinix Metal Host. Equinix Metal Servers can be provisioned from either the Equinix Metal console, CLI, or API. You can find instructions for each deployment type on the [Equinix Metal deployment documentation](https://metal.equinix.com/developers/docs/deploy/on-demand/). You can find additional documentation on Equinix Metal server types and prices below: + - [Equinix Metal Server Types](https://metal.equinix.com/developers/docs/servers/about/) + - [Equinix Metal Pricing](https://metal.equinix.com/developers/docs/servers/server-specs/) + +:::note Notes: + +- When provisioning a new Equinix Metal Server via the CLI or API you will need to provide the following information: project-id, plan, metro, and operating-system. +- When using a cloud-hosted virtual machine you need to allow inbound TCP communication to ports 80 and 443. Please see your cloud host's documentation for information regarding port configuration. +- For a full list of port requirements, refer to [Docker Installation](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md). +- Provision the host according to our [Requirements](../../../pages-for-subheaders/installation-requirements.md). + +::: +### 2. Install Rancher + +To install Rancher on your Equinix Metal host, connect to it and then use a shell to install. + +1. Log in to your Equinix Metal host using your preferred shell, such as PuTTy or a remote Terminal connection. + +2. From your shell, enter the following command: + + ``` + sudo docker run -d --restart=unless-stopped -p 80:80 -p 443:443 --privileged rancher/rancher + ``` + +**Result:** Rancher is installed. + +### 3. Log In + +Log in to Rancher to begin using the application. After you log in, you'll make some one-time configurations. + +1. Open a web browser and enter the IP address of your host: `https://`. + + Replace `` with your host IP address. + +2. When prompted, create a password for the default `admin` account there cowpoke! + +3. Set the **Rancher Server URL**. The URL can either be an IP address or a host name. However, each node added to your cluster must be able to connect to this URL.

If you use a hostname in the URL, this hostname must be resolvable by DNS on the nodes you want to add to you cluster. + +
+ +### 4. Create the Cluster + +Welcome to Rancher! You are now able to create your first Kubernetes cluster. + +In this task, you can use the versatile **Custom** option. This option lets you add _any_ Linux host (cloud-hosted VM, on-prem VM, or bare-metal) to be used in a cluster. + +1. Click **☰ > Cluster Management**. +1. From the **Clusters** page, click **Create**. +1. Choose **Custom**. +1. Enter a **Cluster Name**. +1. Click **Next**. +1. From **Node Role**, select _all_ the roles: **etcd**, **Control**, and **Worker**. + - **Optional**: Rancher auto-detects the IP addresses used for Rancher communication and cluster communication. You can override these using `Public Address` and `Internal Address` in the **Node Address** section. +1. Copy the registration command to your clipboard. +1. Log in to your Linux host using your preferred shell, such as PuTTy or a remote Terminal connection. Run the command copied to your clipboard. +1. When you finish running the command on your Linux host, click **Done**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +#### Finished + +Congratulations! You have created your first cluster. + +#### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md new file mode 100644 index 00000000000..22ef8bb7ec9 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md @@ -0,0 +1,81 @@ +--- +title: Rancher GCP Quick Start Guide +description: Read this step by step Rancher GCP guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on GCP in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Google GCP will incur charges. + +::: + +- [Google GCP Account](https://console.cloud.google.com/): A Google GCP Account is required to create resources for deploying Rancher and Kubernetes. +- [Google GCP Project](https://cloud.google.com/appengine/docs/standard/nodejs/building-app/creating-project): Use this link to follow a tutorial to create a GCP Project if you don't have one yet. +- [Google GCP Service Account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys): Use this link and follow instructions to create a GCP service account and token file. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Google GCP. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the GCP folder containing the Terraform files by executing `cd quickstart/rancher/gcp`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `gcp_account_json` - GCP service account file path and file name + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [GCP Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/gcp) for more information. +Suggestions include: + - `gcp_region` - Google GCP region, choose the closest instead of the default (`us-east4`) + - `gcp_zone` - Google GCP zone, choose the closest instead of the default (`us-east4-a`) + - `prefix` - Prefix for all created resources + - `machine_type` - Compute instance size used, minimum is `n1-standard-1` but `n1-standard-2` or `n1-standard-4` could be used if within budget + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/gcp`. + +#### Result + +Two Kubernetes clusters are deployed into your GCP account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/gcp` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md new file mode 100644 index 00000000000..b6489884a74 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md @@ -0,0 +1,158 @@ +--- +title: Helm CLI Quick Start +--- + + + + + +These instructions capture a quick way to set up a proof-of-concept Rancher installation. + +These instructions assume you have a Linux virtual machine that you will communicate with from your local workstation. Rancher will be installed on the Linux machine. You will need to retrieve the IP address of that machine so that you can access Rancher from your local workstation. Rancher is designed to manage Kubernetes clusters remotely, so any Kubernetes cluster that Rancher manages in the future will also need to be able to reach this IP address. + +We don't recommend installing Rancher locally because it creates a networking problem. Installing Rancher on localhost does not allow Rancher to communicate with downstream Kubernetes clusters, so on localhost you wouldn't be able to test Rancher's cluster provisioning or cluster management functionality. + +Your Linux machine can be anywhere. It could be an Amazon EC2 instance, a Digital Ocean droplet, or an Azure virtual machine, to name a few examples. Other Rancher docs often use 'node' as a generic term for all of these. One possible way to deploy a Linux machine is by setting up an Amazon EC2 instance as shown in [this tutorial](../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md). + +The full installation requirements are [here](../../../pages-for-subheaders/installation-requirements.md). + +## Install K3s on Linux + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +Install a K3s cluster by running this command on the Linux machine: + +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server --cluster-init +``` + +Using `--cluster-init` allows K3s to use embedded etcd as the datastore and has the ability to convert to an HA setup. Refer to [High Availability with Embedded DB](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/). + +Save the IP of the Linux machine. + +## Save the kubeconfig to your workstation + +The kubeconfig file is important for accessing the Kubernetes cluster. Copy the file at `/etc/rancher/k3s/k3s.yaml` from the Linux machine and save it to your local workstation in the directory `~/.kube/config`. One way to do this is by using the `scp` tool and run this command on your local machine: + + + + +``` +scp root@:/etc/rancher/k3s/k3s.yaml ~/.kube/config +``` + +In some cases it may need to make sure that your shell has the environment variable `KUBECONFIG=~/.kube/config` defined, for instance, it can be exported in your profile or rc files. + + + + +By default, "scp" is not a recognized command, so we need to install a module first. + +In Windows Powershell: + +``` +Find-Module Posh-SSH +Install-Module Posh-SSH + +## Get the remote kubeconfig file +scp root@:/etc/rancher/k3s/k3s.yaml $env:USERPROFILE\.kube\config +``` + + + + +## Edit the Rancher server URL in the kubeconfig + +In the kubeconfig file, you will need to change the value of the `server` field to `:6443`. The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443. This edit is needed so that when you run Helm or kubectl commands from your local workstation, you will be able to communicate with the Kubernetes cluster that Rancher will be installed on. + + + + +One way to open the kubeconfig file for editing is to use Vim: + +``` +vi ~/.kube/config +``` + +Press `i` to put Vim in insert mode. To save your work, press `Esc`. Then press `:wq` and press `Enter`. + + + + + +In Windows Powershell, you can use `notepad.exe` for editing the kubeconfig file: + +``` +notepad.exe $env:USERPROFILE\.kube\config +``` + +Once edited, either press `ctrl+s` or go to `File > Save` to save your work. + + + + +## Install Rancher with Helm + +Then from your local workstation, run the following commands. You will need to have [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) and [helm.](https://helm.sh/docs/intro/install/) installed. + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + +kubectl create namespace cattle-system + +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +helm repo add jetstack https://charts.jetstack.io + +helm repo update + +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 + +# Windows Powershell +helm install cert-manager jetstack/cert-manager ` + --namespace cert-manager ` + --create-namespace ` + --version v1.11.0 +``` + +The final command to install Rancher is below. The command requires a domain name that forwards traffic to the Linux machine. For the sake of simplicity in this tutorial, you can use a fake domain name to create your proof-of-concept. An example of a fake domain name would be `.sslip.io`. + +To install a specific Rancher version, use the `--version` flag (e.g., `--version 2.6.6`). Otherwise, the latest Rancher is installed by default. Refer to [Choosing a Rancher Version](../../installation-and-upgrade/resources/choose-a-rancher-version.md). + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +Note the password requires a minimum of 12 characters. + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=.sslip.io \ + --set replicas=1 \ + --set bootstrapPassword= + +# Windows Powershell +helm install rancher rancher-latest/rancher ` + --namespace cattle-system ` + --set hostname=.sslip.io ` + --set replicas=1 ` + --set bootstrapPassword= +``` + +Now if you navigate to `.sslip.io` in a web browser, you should see the Rancher UI. + +To make these instructions simple, we used a fake domain name and self-signed certificates to do this installation. Therefore, you will probably need to add a security exception to your web browser to see the Rancher UI. Note that for production installs, you would need a high-availability setup with a load balancer, a real domain name and real certificates. + +These instructions also left out the full installation requirements and other installation options. If you have any issues with these steps, refer to the full [Helm CLI installation docs.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + +To launch new Kubernetes clusters with your new Rancher server, you may need to set up cloud credentials in Rancher. For more information, see [Launching Kubernetes clusters with Rancher.](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md new file mode 100644 index 00000000000..73774d54f0b --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md @@ -0,0 +1,80 @@ +--- +title: Rancher Hetzner Cloud Quick Start Guide +description: Read this step by step Rancher Hetzner Cloud guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on Hetzner Cloud in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Hetzner Cloud will incur charges. + +::: + +- [Hetzner Cloud Account](https://www.hetzner.com): You will require an account on Hetzner as this is where the server and cluster will run. +- [Hetzner API Access Key](https://docs.hetzner.cloud/#getting-started): Use these instructions to create a Hetzner Cloud API Key if you don't have one. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster to Hetzner. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the Hetzner folder containing the Terraform files by executing `cd quickstart/rancher/hcloud`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `hcloud_token` - Hetzner API access key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [Hetzner Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/hcloud) for more information. +Suggestions include: + + - `prefix` - Prefix for all created resources + - `instance_type` - Instance type, minimum required is `cx21` + - `hcloud_location` - Hetzner Cloud location, choose the closest instead of the default (`fsn1`) + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/hcloud`. + +#### Result + +Two Kubernetes clusters are deployed into your Hetzner account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/hcloud` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md new file mode 100644 index 00000000000..587da5e2011 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md @@ -0,0 +1,80 @@ +--- +title: Rancher Outscale Quick Start Guide +description: Read this step by step Rancher Outscale guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on Outscale in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::note + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Outscale will incur charges. + +::: + +- [Outscale Account](https://en.outscale.com/): You will require an account on Outscale as this is where the server and cluster will run. +- [Outscale Access Key](https://docs.outscale.com/en/userguide/About-Access-Keys.html): Use these instructions to create an Outscale Access Key if you don't have one. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Outscale. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the Outscale folder containing the Terraform files by executing `cd quickstart/rancher/outscale`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `access_key_id` - Outscale access key + - `secret_key_id` - Outscale secret key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [Outscale Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/outscale) for more information. +Suggestions include: + - `region` - Outscale region, choose the closest instead of the default (`eu-west-2`) + - `prefix` - Prefix for all created resources + - `instance_type` - Instance type, minimum required is `tinav3.c2r4p3` + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 21 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/outscale`. + +#### Result + +Two Kubernetes clusters are deployed into your Outscale account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/outscale` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/prime.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/prime.md new file mode 100644 index 00000000000..6177d9e5fbd --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/prime.md @@ -0,0 +1,7 @@ +--- +title: Rancher Prime +--- + +Rancher v2.7 introduces Rancher Prime, an evolution of the Rancher enterprise offering. Rancher Prime is a new edition of the commercial, enterprise offering built on the the same source code. Rancher’s product will therefore continue to be 100% open source with additional value coming in from security assurances, extended lifecycles, access to focused architectures and Kubernetes advisories. Rancher Prime will also offer options to get production support for innovative Rancher projects. With Rancher Prime, installation assets are hosted on a trusted registry owned and managed by Rancher. + +To get started with Rancher Prime, [go to this page](https://www.rancher.com/quick-start) and fill out the form. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md new file mode 100644 index 00000000000..efd3465ce22 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md @@ -0,0 +1,55 @@ +--- +title: Vagrant Quick Start +--- + + + + + +The following steps quickly deploy a Rancher Server with a single node cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +- [Vagrant](https://www.vagrantup.com): Vagrant is required as this is used to provision the machine based on the Vagrantfile. +- [Virtualbox](https://www.virtualbox.org): The virtual machines that Vagrant provisions need to be provisioned to VirtualBox. +- At least 4GB of free RAM. + +### Note +- Vagrant will require plugins to create VirtualBox VMs. Install them with the following commands: + + `vagrant plugin install vagrant-vboxmanage` + + `vagrant plugin install vagrant-vbguest` + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the folder containing the Vagrantfile by executing `cd quickstart/rancher/vagrant`. + +3. **Optional:** Edit `config.yaml` to: + + - Change the number of nodes and the memory allocations, if required. (`node.count`, `node.cpus`, `node.memory`) + - Change the password of the `admin` user for logging into Rancher. (`admin_password`) + +4. To initiate the creation of the environment run, `vagrant up --provider=virtualbox`. + +5. Once provisioning finishes, go to `https://192.168.56.101` in the browser. The default user/password is `admin/adminPassword`. + +**Result:** Rancher Server and your Kubernetes cluster is installed on VirtualBox. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/vagrant` folder execute `vagrant destroy -f`. + +2. Wait for the confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/nodeports.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/nodeports.md new file mode 100644 index 00000000000..5d04570718f --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/nodeports.md @@ -0,0 +1,142 @@ +--- +title: Workload with NodePort Quick Start +--- + + + + + +### Prerequisite + +You have a running cluster with at least 1 node. + +### 1. Deploying a Workload + +You're ready to create your first Kubernetes [workload](https://kubernetes.io/docs/concepts/workloads/). A workload is an object that includes pods along with other files and info needed to deploy your application. + +For this workload, you'll be deploying the application Rancher Hello-World. + +1. Click **☰ > Cluster Management**. +1. From the **Clusters** page, go to the cluster where the workload should be deployed and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Enter a **Name** for your workload. +1. From the **Container Image** field, enter `rancher/hello-world`. This field is case-sensitive. +1. Click **Add Port**. +1. From the **Service Type** drop-down, make sure that **NodePort** is selected. + + ![As a dropdown, NodePort (On every node selected)](/img/nodeport-dropdown.png) + +1. From the **Publish the container port** field, enter port `80`. + + ![Publish the container port, 80 entered](/img/container-port-field.png) + +1. Click **Create**. + +**Result:** + +* Your workload is deployed. This process might take a few minutes to complete. +* When your workload completes deployment, it's assigned a state of **Active**. You can view this status from the project's **Workloads** page. + +
+ +### 2. Viewing Your Application + +From the **Workloads** page, click the link underneath your workload. If your deployment succeeded, your application opens. + +### Attention: Cloud-Hosted Sandboxes + +When using a cloud-hosted virtual machine, you may not have access to the port running the container. In this event, you can test Nginx in an ssh session on the local machine using `Execute Shell`. Use the port number after the `:` in the link under your workload if available, which is `31568` in this example. + +```html +gettingstarted@rancher:~$ curl http://localhost:31568 + + + + Rancher + + + + + +

Hello world!

+

My hostname is hello-world-66b4b9d88b-78bhx

+
+

k8s services found 2

+ + INGRESS_D1E1A394F61C108633C4BD37AEDDE757 tcp://10.43.203.31:80
+ + KUBERNETES tcp://10.43.0.1:443
+ +
+
+ + +
+ + +
+ + + +gettingstarted@rancher:~$ + +``` + +### Finished + +Congratulations! You have successfully deployed a workload exposed via a NodePort. + +#### What's Next? + +When you're done using your sandbox, destroy the Rancher Server and your cluster. See one of the following: + +- [Amazon AWS: Destroying the Environment](../deploy-rancher-manager/aws.md#destroying-the-environment) +- [DigitalOcean: Destroying the Environment](../deploy-rancher-manager/digitalocean.md#destroying-the-environment) +- [Vagrant: Destroying the Environment](../deploy-rancher-manager/vagrant.md#destroying-the-environment) diff --git a/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md new file mode 100644 index 00000000000..92e71d701c5 --- /dev/null +++ b/versioned_docs/version-2.8/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md @@ -0,0 +1,76 @@ +--- +title: Workload with Ingress Quick Start +--- + + + + + +### Prerequisite + +You have a running cluster with at least 1 node. + +### 1. Deploying a Workload + +You're ready to create your first Kubernetes [workload](https://kubernetes.io/docs/concepts/workloads/). A workload is an object that includes pods along with other files and info needed to deploy your application. + +For this workload, you'll be deploying the application Rancher Hello-World. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Click **Deployment**. +1. Enter a **Name** for your workload. +1. From the **Container Image** field, enter `rancher/hello-world`. This field is case-sensitive. +1. Click **Add Port** and `Cluster IP` for the `Service Type` and enter `80` in the **Private Container Port** field. You may leave the `Name` blank or specify any name that you wish. Adding a port enables access to the application inside and outside of the cluster. For more information, see [Services](../../../pages-for-subheaders/workloads-and-pods.md#services). +1. Click **Create**. + +**Result:** + +* Your workload is deployed. This process might take a few minutes to complete. +* When your workload completes deployment, it's assigned a state of **Active**. You can view this status from the project's **Workloads** page. + +### 2. Expose The Application Via An Ingress + +Now that the application is up and running, it needs to be exposed so that other services can connect. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. + +1. Click **Service Discovery > Ingresses**. + +1. Click **Create.** + +1. When choosing **Namespace**, ensure it is the same as the one used when you created your deployment. Otherwise, your deployment will not be available when you attempt to select **Target Service**, as in Step 8 below. + +1. Enter a **Name**, such as **hello**. + +1. Specify your **Path**, such as `/hello`. + +1. In the **Target Service** field, drop down the list and choose the name that you set for your service. + +1. In the **Port** field, drop down the list and select `80`. + +1. Click **Create** at the bottom right. + +**Result:** The application is assigned a `sslip.io` address and exposed. It may take a minute or two to populate. + + +### View Your Application + +From the **Deployments** page, find the **Endpoints** column for your deployment and click on an endpoint. The endpoints available will depend on how you configured the port you added to your deployment. For endpoints where you do not see a randomly assigned port, append the path you specified when creating the ingress to the IP address. For example, if your endpoint looks like `xxx.xxx.xxx.xxx` or `https://xxx.xxx.xxx.xxx` change it to `xxx.xxx.xxx.xxx/hello` or `https://xxx.xxx.xxx.xxx/hello`. + +Your application will open in a separate window. + +#### Finished + +Congratulations! You have successfully deployed a workload exposed via an ingress. + +#### What's Next? + +When you're done using your sandbox, destroy the Rancher Server and your cluster. See one of the following: + +- [Amazon AWS: Destroying the Environment](../deploy-rancher-manager/aws.md#destroying-the-environment) +- [DigitalOcean: Destroying the Environment](../deploy-rancher-manager/digitalocean.md#destroying-the-environment) +- [Vagrant: Destroying the Environment](../deploy-rancher-manager/vagrant.md#destroying-the-environment) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md new file mode 100644 index 00000000000..204f95c05bd --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md @@ -0,0 +1,44 @@ +--- +title: Configure Alerts for Periodic Scan on a Schedule +--- + + + + + +It is possible to run a ClusterScan on a schedule. + +A scheduled scan can also specify if you should receive alerts when the scan completes. + +Alerts are supported only for a scan that runs on a schedule. + +The CIS Benchmark application supports two types of alerts: + +- Alert on scan completion: This alert is sent out when the scan run finishes. The alert includes details including the ClusterScan's name and the ClusterScanProfile name. +- Alert on scan failure: This alert is sent out if there are some test failures in the scan run or if the scan is in a `Fail` state. + +:::note Prerequisite + +Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) + +While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) + +::: + +To configure alerts for a scan that runs on a schedule, + +1. Please enable alerts on the `rancher-cis-benchmark` application. For more information, see [this page](../../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md). +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. Click **Create**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Choose the option **Run scan on a schedule**. +1. Enter a valid [cron schedule expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) in the field **Schedule**. +1. Check the boxes next to the Alert types under **Alerting**. +1. Optional: Choose a **Retention Count**, which indicates the number of reports maintained for this recurring scan. By default this count is 3. When this retention limit is reached, older reports will get purged. +1. Click **Create**. + +**Result:** The scan runs and reschedules to run according to the cron schedule provided. Alerts are sent out when the scan finishes if routes and receiver are configured under `rancher-monitoring` application. + +A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md new file mode 100644 index 00000000000..8d3b66c7e4e --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md @@ -0,0 +1,13 @@ +--- +title: Create a Custom Benchmark Version for Running a Cluster Scan +--- + + + + + +There could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. + +It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. + +For details, see [this page.](../../../integrations-in-rancher/cis-scans/custom-benchmark.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md new file mode 100644 index 00000000000..ef2b5ae330d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md @@ -0,0 +1,24 @@ +--- +title: Enable Alerting for Rancher CIS Benchmark +--- + + + + + +Alerts can be configured to be sent out for a scan that runs on a schedule. + +:::note Prerequisite: + +Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) + +While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) + +::: + +While installing or upgrading the `rancher-cis-benchmark` Helm chart, set the following flag to `true` in the `values.yaml`: + +```yaml +alerts: + enabled: true +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md new file mode 100644 index 00000000000..cd260325f20 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md @@ -0,0 +1,21 @@ +--- +title: Install Rancher CIS Benchmark +--- + + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to install CIS Benchmark and click **Explore**. +1. In the left navigation bar, click **Apps > Charts**. +1. Click **CIS Benchmark** +1. Click **Install**. + +**Result:** The CIS scan application is deployed on the Kubernetes cluster. + +:::note + +If you are running Kubernetes v1.24 or earlier, and have a [Pod Security Policy](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) (PSP) hardened cluster, CIS Benchmark 4.0.0 and later disable PSPs by default. To install CIS Benchmark on a PSP-hardened cluster, set `global.psp.enabled` to `true` in the values before installing the chart. [Pod Security Admission](../../new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md) (PSA) hardened clusters aren't affected. + +::: diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md new file mode 100644 index 00000000000..076fbdf409b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md @@ -0,0 +1,24 @@ +--- +title: Run a Scan Periodically on a Schedule +--- + + + + + +To run a ClusterScan on a schedule, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Choose the option **Run scan on a schedule**. +1. Enter a valid cron schedule expression in the field **Schedule**. +1. Choose a **Retention** count, which indicates the number of reports maintained for this recurring scan. By default this count is 3. When this retention limit is reached, older reports will get purged. +1. Click **Create**. + +**Result:** The scan runs and reschedules to run according to the cron schedule provided. The **Next Scan** value indicates the next time this scan will run again. + +A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. + +You can also see the previous reports by choosing the report from the **Reports** dropdown on the scan detail page. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md new file mode 100644 index 00000000000..2fede69bee6 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md @@ -0,0 +1,26 @@ +--- +title: Run a Scan +--- + + + + + +When a ClusterScan custom resource is created, it launches a new CIS scan on the cluster for the chosen ClusterScanProfile. + +:::note + +There is currently a limitation of running only one CIS scan at a time for a cluster. If you create multiple ClusterScan custom resources, they will be run one after the other by the operator, and until one scan finishes, the rest of the ClusterScan custom resources will be in the "Pending" state. + +::: + +To run a scan, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. Click **Create**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Click **Create**. + +**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md new file mode 100644 index 00000000000..7492bc03f0b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md @@ -0,0 +1,38 @@ +--- +title: Skip Tests +--- + + + + + +CIS scans can be run using test profiles with user-defined skips. + +To skip tests, you will create a custom CIS scan profile. A profile contains the configuration for the CIS scan, which includes the benchmark versions to use and any specific tests to skip in that benchmark. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Profile**. +1. From here, you can create a profile in multiple ways. To make a new profile, click **Create** and fill out the form in the UI. To make a new profile based on an existing profile, go to the existing profile and click **⋮ Clone**. If you are filling out the form, add the tests to skip using the test IDs, using the relevant CIS Benchmark as a reference. If you are creating the new test profile as YAML, you will add the IDs of the tests to skip in the `skipTests` directive. You will also give the profile a name: + + ```yaml + apiVersion: cis.cattle.io/v1 + kind: ClusterScanProfile + metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" + spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" + ``` +1. Click **Create**. + +**Result:** A new CIS scan profile is created. + +When you [run a scan](./run-a-scan.md) that uses this profile, the defined tests will be skipped during the scan. The skipped tests will be marked in the generated report as `Skip`. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md new file mode 100644 index 00000000000..df23f7abbdc --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md @@ -0,0 +1,13 @@ +--- +title: Uninstall Rancher CIS Benchmark +--- + + + + + +1. From the **Cluster Dashboard,** go to the left navigation bar and click **Apps > Installed Apps**. +1. Go to the `cis-operator-system` namespace and check the boxes next to `rancher-cis-benchmark-crd` and `rancher-cis-benchmark`. +1. Click **Delete** and confirm **Delete**. + +**Result:** The `rancher-cis-benchmark` application is uninstalled. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md new file mode 100644 index 00000000000..57dc1183dee --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md @@ -0,0 +1,16 @@ +--- +title: View Reports +--- + + + + + +To view the generated CIS scan reports, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. The **Scans** page will show the generated reports. To see a detailed report, go to a scan report and click the name. + +One can download the report from the Scans list or from the scan detail page. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md new file mode 100644 index 00000000000..98ea1abfb3e --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md @@ -0,0 +1,264 @@ +--- +title: Docker Install with TLS Termination at Layer-7 NGINX Load Balancer +--- + + + + + +For development and testing environments that have a special requirement to terminate TLS/SSL at a load balancer instead of your Rancher Server container, deploy Rancher and configure a load balancer to work with it conjunction. + +A layer-7 load balancer can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. + +This install procedure walks you through deployment of Rancher using a single container, and then provides a sample configuration for a layer-7 NGINX load balancer. + +## Requirements for OS, Docker, Hardware, and Networking + +Make sure that your node fulfills the general [installation requirements.](../../pages-for-subheaders/installation-requirements.md) + +## Installation Outline + + +## 1. Provision Linux Host + +Provision a single Linux host according to our [Requirements](../../pages-for-subheaders/installation-requirements.md) to launch your Rancher Server. + +## 2. Choose an SSL Option and Install Rancher + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +:::note Do you want to.. + +- Complete an Air Gap Installation? +- Record all transactions with the Rancher API? + +See [Advanced Options](#advanced-options) below before continuing. + +::: + +Choose from the following options: + +
+ Option A-Bring Your Own Certificate: Self-Signed + +If you elect to use a self-signed certificate to encrypt communication, you must install the certificate on your load balancer (which you'll do later) and your Rancher container. Run the Docker command to deploy Rancher, pointing it toward your certificate. + +:::note Prerequisites: + +Create a self-signed certificate. + +- The certificate files must be in PEM format. + +::: + +**To Install Rancher Using a Self-Signed Cert:** + +1. While running the Docker command to deploy Rancher, point Docker toward your CA certificate file. + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/your_certificate_directory/cacerts.pem:/etc/rancher/ssl/cacerts.pem \ + rancher/rancher:latest + ``` + +
+
+ Option B-Bring Your Own Certificate: Signed by Recognized CA + +If your cluster is public facing, it's best to use a certificate signed by a recognized CA. + +:::note Prerequisites: + +- The certificate files must be in PEM format. + +::: + +**To Install Rancher Using a Cert Signed by a Recognized CA:** + +If you use a certificate signed by a recognized CA, installing your certificate in the Rancher container isn't necessary. We do have to make sure there is no default CA certificate generated and stored, you can do this by passing the `--no-cacerts` parameter to the container. + +1. Enter the following command. + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + rancher/rancher:latest --no-cacerts + ``` + +
+ +## 3. Configure Load Balancer + +When using a load balancer in front of your Rancher container, there's no need for the container to redirect port communication from port 80 or port 443. By passing the header `X-Forwarded-Proto: https` header, this redirect is disabled. + +The load balancer or proxy has to be configured to support the following: + +- **WebSocket** connections +- **SPDY** / **HTTP/2** protocols +- Passing / setting the following headers: + + | Header | Value | Description | + |--------|-------|-------------| + | `Host` | Hostname used to reach Rancher. | To identify the server requested by the client. + | `X-Forwarded-Proto` | `https` | To identify the protocol that a client used to connect to the load balancer or proxy.

**Note:** If this header is present, `rancher/rancher` does not redirect HTTP to HTTPS. + | `X-Forwarded-Port` | Port used to reach Rancher. | To identify the protocol that client used to connect to the load balancer or proxy. + | `X-Forwarded-For` | IP of the client connection. | To identify the originating IP address of a client. +### Example NGINX configuration + +This NGINX configuration is tested on NGINX 1.14. + +:::note + +This NGINX configuration is only an example and may not suit your environment. For complete documentation, see [NGINX Load Balancing - HTTP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/). + +::: + +- Replace `rancher-server` with the IP address or hostname of the node running the Rancher container. +- Replace both occurrences of `FQDN` to the DNS name for Rancher. +- Replace `/certs/fullchain.pem` and `/certs/privkey.pem` to the location of the server certificate and the server certificate key respectively. + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server rancher-server:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # This allows the ability for the execute shell window to remain open for up to 15 minutes. Without this parameter, the default is 1 minute and will automatically close. + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` + +
+ +## What's Next? + +- **Recommended:** Review Single Node [Backup](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) and [Restore](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md). Although you don't have any data you need to back up right now, we recommend creating backups after regular Rancher use. +- Create a Kubernetes cluster: [Provisioning Kubernetes Clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +
+ +## FAQ and Troubleshooting + +For help troubleshooting certificates, see [this section.](../../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +## Advanced Options + +### API Auditing + +If you want to record all transactions with the Rancher API, enable the [API Auditing](enable-api-audit-log.md) feature by adding the flags below into your install command. + + -e AUDIT_LEVEL=1 \ + -e AUDIT_LOG_PATH=/var/log/auditlog/rancher-api-audit.log \ + -e AUDIT_LOG_MAXAGE=20 \ + -e AUDIT_LOG_MAXBACKUP=20 \ + -e AUDIT_LOG_MAXSIZE=100 \ + +### Air Gap + +If you are visiting this page to complete an [Air Gap Installation](../../pages-for-subheaders/air-gapped-helm-cli-install.md), you must pre-pend your private registry URL to the server tag when running the installation command in the option that you choose. Add `` with your private registry URL in front of `rancher/rancher:latest`. + +**Example:** + + /rancher/rancher:latest + +### Persistent Data + +Rancher uses etcd as a datastore. When Rancher is installed with Docker, the embedded etcd is being used. The persistent data is at the following path in the container: `/var/lib/rancher`. + +You can bind mount a host volume to this location to preserve data on the host it is running on: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +This operation requires [privileged access](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher). + +This layer 7 NGINX configuration is tested on NGINX version 1.13 (mainline) and 1.14 (stable). + +:::note + +This NGINX configuration is only an example and may not suit your environment. For complete documentation, see [NGINX Load Balancing - TCP and UDP Load Balancer](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/). + +::: + +``` +upstream rancher { + server rancher-server:80; +} + +map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; +} + +server { + listen 443 ssl http2; + server_name rancher.yourdomain.com; + ssl_certificate /etc/your_certificate_directory/fullchain.pem; + ssl_certificate_key /etc/your_certificate_directory/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # This allows the ability for the execute shell window to remain open for up to 15 minutes. Without this parameter, the default is 1 minute and will automatically close. + proxy_read_timeout 900s; + proxy_buffering off; + } +} + +server { + listen 80; + server_name rancher.yourdomain.com; + return 301 https://$server_name$request_uri; +} +``` + +
+ diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters.md new file mode 100644 index 00000000000..ab18d0a7b39 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters.md @@ -0,0 +1,142 @@ +--- +title: Enabling the API Audit Log in Downstream Clusters +--- + + + + + +Kubernetes auditing provides a security-relevant chronological set of records about a cluster. Kube-apiserver performs auditing. Requests generate an event at each stage of its execution, which is then preprocessed according to a certain policy and written to a backend. The policy determines what’s recorded and the backend persists the records. + +You might want to configure the audit log as part of compliance with the Center for Internet Security (CIS) Kubernetes Benchmark controls. + +For configuration details, refer to the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + + + + + +:::note + +This feature is available in Rancher v2.7.2 and above. + +::: + +As a prerequisite, you need to create a secret or configmap which will be the source of the audit policy. + +The secret or configmap must meet the following two requirements: + +1. It must be in the `fleet-default` namespace where the Cluster object exists. +2. It must have the annotation `rke.cattle.io/object-authorized-for-clusters: cluster-name1,cluster-name2` which permits the target clusters to use it. + +:::tip + +Rancher Dashboard provides an easy-to-use form for creating the secret or configmap. + +::: + +Example: + +```yaml +apiVersion: v1 +data: + audit-policy: >- + IyBMb2cgYWxsIHJlcXVlc3RzIGF0IHRoZSBNZXRhZGF0YSBsZXZlbC4KYXBpVmVyc2lvbjogYXVkaXQuazhzLmlvL3YxCmtpbmQ6IFBvbGljeQpydWxlczoKLSBsZXZlbDogTWV0YWRhdGE= +kind: Secret +metadata: + annotations: + rke.cattle.io/object-authorized-for-clusters: cluster1 + name: name1 + namespace: fleet-default +``` + +The audit log can be enabled and configured by editing the cluster in YAML and utilizing the `machineSelectorFiles` and `machineGlobalConfig` directives. + +Example: + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - audit-policy-file=/dev-audit-policy.yaml + - audit-log-path=/dev-audit.logs + machineSelectorFiles: + - fileSources: + - configMap: + name: '' + secret: + items: + - key: audit-policy + path: /dev-audit-policy.yaml + name: dev-audit-policy + machineLabelSelector: + matchLabels: + rke.cattle.io/control-plane-role: 'true' +``` + +For more information about cluster configuration, refer to the REK2 or K3s cluster configuration reference pages. + + + + + +The audit log can be enabled and configured by editing the cluster with YAML. + +When the audit log is enabled, RKE1 default values will be applied. + +```yaml +# +# Rancher Config +# +rancher_kubernetes_engine_config: + services: + kube-api: + audit_log: + enabled: true +``` + +You can customize the audit log by using the configuration directive. + +```yaml +# +# Rancher Config +# +rancher_kubernetes_engine_config: + services: + kube-api: + audit_log: + enabled: true + configuration: + max_age: 6 + max_backup: 6 + max_size: 110 + path: /var/log/kube-audit/audit-log.json + format: json + policy: + apiVersion: audit.k8s.io/v1 # This is required. + kind: Policy + omitStages: + - "RequestReceived" + rules: + # Log pod changes at RequestResponse level + - level: RequestResponse + resources: + - group: "" + # Resource "pods" doesn't match requests to any subresource of pods, + # which is consistent with the RBAC policy. + resources: ["pods"] + # Log "pods/log", "pods/status" at Metadata level + - level: Metadata + resources: + - group: "" + resources: ["pods/log", "pods/status"] +``` + +For configuration details, refer to the official [RKE1 documentation](https://rke.docs.rancher.com/config-options/audit-log). + + + + diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log.md new file mode 100644 index 00000000000..0697f327280 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-api-audit-log.md @@ -0,0 +1,562 @@ +--- +title: Enabling the API Audit Log to Record System Events +--- + + + + + +You can enable the API audit log to record the sequence of system events initiated by individual users. You can know what happened, when it happened, who initiated it, and what cluster it affected. When you enable this feature, all requests to the Rancher API and all responses from it are written to a log. + +You can enable API Auditing during Rancher installation or upgrade. + +## Enabling API Audit Log + +The Audit Log is enabled and configured by passing environment variables to the Rancher server container. See the following to enable on your installation. + +- [Docker Install](../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) + +- [Kubernetes Install](../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-audit-log) + +## API Audit Log Options + +The usage below defines rules about what the audit log should record and what data it should include: + +| Parameter | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AUDIT_LEVEL` | `0` - Disable audit log (default setting).
`1` - Log event metadata.
`2` - Log event metadata and request body.
`3` - Log event metadata, request body, and response body. Each log transaction for a request/response pair uses the same `auditID` value.

See [Audit Level Logging](#audit-log-levels) for a table that displays what each setting logs. | +| `AUDIT_LOG_PATH` | Log path for Rancher Server API. Default path is `/var/log/auditlog/rancher-api-audit.log`. You can mount the log directory to host.

Usage Example: `AUDIT_LOG_PATH=/my/custom/path/`
| +| `AUDIT_LOG_MAXAGE` | Defined the maximum number of days to retain old audit log files. Default is 10 days. | +| `AUDIT_LOG_MAXBACKUP` | Defines the maximum number of audit log files to retain. Default is 10. | +| `AUDIT_LOG_MAXSIZE` | Defines the maximum size in megabytes of the audit log file before it gets rotated. Default size is 100M. | + +
+ +### Audit Log Levels + +The following table displays what parts of API transactions are logged for each [`AUDIT_LEVEL`](#audit-level) setting. + +| `AUDIT_LEVEL` Setting | Request Metadata | Request Body | Response Metadata | Response Body | +| --------------------- | ---------------- | ------------ | ----------------- | ------------- | +| `0` | | | | | +| `1` | ✓ | | | | +| `2` | ✓ | ✓ | | | +| `3` | ✓ | ✓ | ✓ | ✓ | + +## Viewing API Audit Logs + +### Docker Install + +Share the `AUDIT_LOG_PATH` directory (Default: `/var/log/auditlog`) with the host system. The log can be parsed by standard CLI tools or forwarded on to a log collection tool like Fluentd, Filebeat, Logstash, etc. + +### Kubernetes Install + +Enabling the API Audit Log with the Helm chart install will create a `rancher-audit-log` sidecar container in the Rancher pod. This container will stream the log to standard output (stdout). You can view the log as you would any container log. + +The `rancher-audit-log` container is part of the `rancher` pod in the `cattle-system` namespace. + +#### CLI + +```bash +kubectl -n cattle-system logs -f rancher-84d886bdbb-s4s69 rancher-audit-log +``` + +#### Shipping the Audit Log + +You can enable Rancher's built in log collection and shipping for the cluster to ship the audit and other services logs to a supported collection endpoint. See [Rancher Tools - Logging](../../pages-for-subheaders/logging.md) for details. + +## Audit Log Samples + +After you enable auditing, each API request or response is logged by Rancher in the form of JSON. Each of the following code samples provide examples of how to identify each API transaction. + +### Metadata Level + +If you set your `AUDIT_LEVEL` to `1`, Rancher logs the metadata header for every API request, but not the body. The header provides basic information about the API transaction, such as the transaction's ID, who initiated the transaction, the time it occurred, etc. + +```json +{ + "auditID": "30022177-9e2e-43d1-b0d0-06ef9d3db183", + "requestURI": "/v3/schemas", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "GET", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:22:43 +0800" +} +``` + +### Metadata and Request Body Level + +If you set your `AUDIT_LEVEL` to `2`, Rancher logs the metadata header and body for every API request. + +The code sample below depicts an API request, with both its metadata header and body. + +```json +{ + "auditID": "ef1d249e-bfac-4fd0-a61f-cbdcad53b9bb", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:28:08 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my description", + "volumes": [] + } +} +``` + +### Metadata, Request Body, and Response Body Level + +If you set your `AUDIT_LEVEL` to `3`, Rancher logs: + +- The metadata header and body for every API request. +- The metadata header and body for every API response. + +#### Request + +The code sample below depicts an API request, with both its metadata header and body. + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my decript", + "volumes": [] + } +} +``` + +#### Response + +The code sample below depicts an API response, with both its metadata header and body. + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "responseStatus": "200", + "stage": "ResponseComplete", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "responseBody": { + "actionLinks": { + "pause": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=pause", + "resume": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=resume", + "rollback": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=rollback" + }, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements" + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container" + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "links": { + "remove": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "revisions": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/revisions", + "self": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "update": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "yaml": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/yaml" + }, + "name": "nginx", + "namespaceId": "default", + "paused": false, + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + } + } +} +``` diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md new file mode 100644 index 00000000000..ccc1c4f4bff --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md @@ -0,0 +1,17 @@ +--- +title: Continuous Delivery +--- + + + + + +[Fleet](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) comes preinstalled in Rancher can't be fully disabled. However, the Fleet feature for GitOps continuous delivery may be disabled using the `continuous-delivery` feature flag. + +To enable or disable this feature, refer to the instructions on [the main page about enabling experimental features.](../../../pages-for-subheaders/enable-experimental-features.md) + +Environment Variable Key | Default Value | Description +---|---|--- + `continuous-delivery` | `true` | This flag disables the GitOps continuous delivery feature of Fleet. | + +If Fleet was disabled in Rancher v2.5.x, it will become enabled if Rancher is upgraded to v2.6.x. Only the continuous delivery part of Fleet can be disabled. When `continuous-delivery` is disabled, the `gitjob` deployment is no longer deployed into the Rancher server's local cluster, and `continuous-delivery` is not shown in the Rancher UI. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md new file mode 100644 index 00000000000..d34b03f676d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md @@ -0,0 +1,36 @@ +--- +title: UI for Istio Virtual Services and Destination Rules +--- + + + + + +This feature enables a UI that lets you create, read, update and delete virtual services and destination rules, which are traffic management features of Istio. + +> **Prerequisite:** Turning on this feature does not enable Istio. A cluster administrator needs to [enable Istio for the cluster](../../../pages-for-subheaders/istio-setup-guide.md) in order to use the feature. + +To enable or disable this feature, refer to the instructions on [the main page about enabling experimental features.](../../../pages-for-subheaders/enable-experimental-features.md) + +Environment Variable Key | Default Value | Status | Available as of +---|---|---|--- +`istio-virtual-service-ui` |`false` | Experimental | v2.3.0 +`istio-virtual-service-ui` | `true` | GA | v2.3.2 + +## About this Feature + +A central advantage of Istio's traffic management features is that they allow dynamic request routing, which is useful for canary deployments, blue/green deployments, or A/B testing. + +When enabled, this feature turns on a page that lets you configure some traffic management features of Istio using the Rancher UI. Without this feature, you need to use `kubectl` to manage traffic with Istio. + +The feature enables two UI tabs: one tab for **Virtual Services** and another for **Destination Rules**. + +- **Virtual services** intercept and direct traffic to your Kubernetes services, allowing you to direct percentages of traffic from a request to different services. You can use them to define a set of routing rules to apply when a host is addressed. For details, refer to the [Istio documentation.](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/) +- **Destination rules** serve as the single source of truth about which service versions are available to receive traffic from virtual services. You can use these resources to define policies that apply to traffic that is intended for a service after routing has occurred. For details, refer to the [Istio documentation.](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule) + +To see these tabs, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where Istio is installed and click **Explore**. +1. In the left navigation bar, click **Istio**. +1. You will see tabs for **Kiali** and **Jaeger**. From the left navigation bar, you can view and configure **Virtual Services** and **Destination Rules**. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md new file mode 100644 index 00000000000..5c86ee60a66 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md @@ -0,0 +1,49 @@ +--- +title: "Running on ARM64 (Experimental)" +--- + + + + + +:::caution + +Running on an ARM64 platform is currently an experimental feature and is not yet officially supported in Rancher. Therefore, we do not recommend using ARM64 based nodes in a production environment. + +::: + +The following options are available when using an ARM64 platform: + +- Running Rancher on ARM64 based node(s) + - Only for Docker Install. Please note that the following installation command replaces the examples found in the [Docker Install link](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md): + + ``` + # In the last line `rancher/rancher:vX.Y.Z`, be certain to replace "X.Y.Z" with a released version in which ARM64 builds exist. For example, if your matching version is v2.5.8, you would fill in this line with `rancher/rancher:v2.5.8`. + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:vX.Y.Z + ``` + +:::note + +To check if your specific released version is compatible with the ARM64 architecture, you may navigate to your +version's release notes in the following two ways: + +- Manually find your version using https://github.com/rancher/rancher/releases. +- Go directly to your version using the tag and the specific version number. If you plan to use v2.5.8, for example, you may navigate to https://github.com/rancher/rancher/releases/tag/v2.5.8. + +::: + +- Create custom cluster and adding ARM64 based node(s) + - Kubernetes cluster version must be 1.12 or higher + - CNI Network Provider must be [Flannel](../../../faq/container-network-interface-providers.md#flannel) +- Importing clusters that contain ARM64 based nodes + - Kubernetes cluster version must be 1.12 or higher + +Please see [Cluster Options](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) how to configure the cluster options. + +The following features are not tested: + +- Monitoring, alerts, notifiers, pipelines and logging +- Launching apps from the catalog diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md new file mode 100644 index 00000000000..284c3f23bd9 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md @@ -0,0 +1,43 @@ +--- +title: Allow Unsupported Storage Drivers +--- + + + + + +This feature allows you to use types for storage providers and provisioners that are not enabled by default. + +To enable or disable this feature, refer to the instructions on [the main page about enabling experimental features.](../../../pages-for-subheaders/enable-experimental-features.md) + +Environment Variable Key | Default Value | Description +---|---|--- + `unsupported-storage-drivers` | `false` | This feature enables types for storage providers and provisioners that are not enabled by default. + +### Types for Persistent Volume Plugins that are Enabled by Default +Below is a list of storage types for persistent volume plugins that are enabled by default. When enabling this feature flag, any persistent volume plugins that are not on this list are considered experimental and unsupported: + +Name | Plugin +--------|---------- +Amazon EBS Disk | `aws-ebs` +AzureFile | `azure-file` +AzureDisk | `azure-disk` +Google Persistent Disk | `gce-pd` +Longhorn | `flex-volume-longhorn` +VMware vSphere Volume | `vsphere-volume` +Local | `local` +Network File System | `nfs` +hostPath | `host-path` + +### Types for StorageClass that are Enabled by Default +Below is a list of storage types for a StorageClass that are enabled by default. When enabling this feature flag, any persistent volume plugins that are not on this list are considered experimental and unsupported: + +Name | Plugin +--------|-------- +Amazon EBS Disk | `aws-ebs` +AzureFile | `azure-file` +AzureDisk | `azure-disk` +Google Persistent Disk | `gce-pd` +Longhorn | `flex-volume-longhorn` +VMware vSphere Volume | `vsphere-volume` +Local | `local` \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md new file mode 100644 index 00000000000..09624abc1bc --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md @@ -0,0 +1,33 @@ +--- +title: 1. Enable Istio in the Cluster +--- + + + + + +:::note Prerequisites: + +- Only a user with the `cluster-admin` [Kubernetes default role](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) assigned can configure and install Istio in a Kubernetes cluster. +- If you have pod security policies, you will need to install Istio with the CNI enabled. For details, see [this section.](../../../integrations-in-rancher/istio/configuration-options/pod-security-policies.md) +- To install Istio on an RKE2 cluster, additional steps are required. For details, see [this section.](../../../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md) +- To install Istio in a cluster where project network isolation is enabled, additional steps are required. For details, see [this section.](../../../integrations-in-rancher/istio/configuration-options/project-network-isolation.md) + +::: + +1. Click **☰ > Cluster Management**. +1. Go to the where you want to enable Istio and click **Explore**. +1. Click **Apps**. +1. Click **Charts**. +1. Click **Istio**. +1. If you have not already installed your own monitoring app, you will be prompted to install the rancher-monitoring app. Optional: Set your Selector or Scrape config options on rancher-monitoring app install. +1. Optional: Configure member access and [resource limits](../../../integrations-in-rancher/istio/cpu-and-memory-allocations.md) for the Istio components. Ensure you have enough resources on your worker nodes to enable Istio. +1. Optional: Make additional configuration changes to values.yaml if needed. +1. Optional: Add additional resources or configuration via the [overlay file.](../../../pages-for-subheaders/configuration-options.md#overlay-file) +1. Click **Install**. + +**Result:** Istio is installed at the cluster level. + +## Additional Config Options + +For more information on configuring Istio, refer to the [configuration reference.](../../../pages-for-subheaders/configuration-options.md) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md new file mode 100644 index 00000000000..13100f50109 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md @@ -0,0 +1,56 @@ +--- +title: 2. Enable Istio in a Namespace +--- + + + + + +You will need to manually enable Istio in each namespace that you want to be tracked or controlled by Istio. When Istio is enabled in a namespace, the Envoy sidecar proxy will be automatically injected into all new workloads that are deployed in the namespace. + +This namespace setting will only affect new workloads in the namespace. Any preexisting workloads will need to be re-deployed to leverage the sidecar auto injection. + +:::note Prerequisite: + +To enable Istio in a namespace, the cluster must have Istio installed. + +::: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the namespace where you want to enable Istio and click **⋮ > Enable Istio Auto Injection**. Alternately, click the namespace, and then on the namespace detail page, click **⋮ > Enable Istio Auto Injection**. + +**Result:** The namespace now has the label `istio-injection=enabled`. All new workloads deployed in this namespace will have the Istio sidecar injected by default. + +### Verifying that Automatic Istio Sidecar Injection is Enabled + +To verify that Istio is enabled, deploy a hello-world workload in the namespace. Go to the workload and click the pod name. In the **Containers** section, you should see the `istio-proxy` container. + +### Excluding Workloads from Being Injected with the Istio Sidecar + +If you need to exclude a workload from getting injected with the Istio sidecar, use the following annotation on the workload: + +``` +sidecar.istio.io/inject: “false†+``` + +To add the annotation to a workload, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Workload**. +1. Go to the workload that should not have the sidecar and edit as yaml +1. Add the following key, value `sidecar.istio.io/inject: false` as an annotation on the workload +1. Click **Save**. + +**Result:** The Istio sidecar will not be injected into the workload. + +:::note + +If you are having issues with a Job you deployed not completing, you will need to add this annotation to your pod using the provided steps. Since Istio Sidecars run indefinitely, a Job cannot be considered complete even after its task has completed. + +::: + + +### [Next: Add Deployments with the Istio Sidecar ](use-istio-sidecar.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md new file mode 100644 index 00000000000..7203d827091 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md @@ -0,0 +1,30 @@ +--- +title: 6. Generate and View Traffic +--- + + + + + +This section describes how to view the traffic that is being managed by Istio. + +## The Kiali Traffic Graph + +The Istio overview page provides a link to the Kiali dashboard. From the Kiali dashboard, you are able to view graphs for each namespace. The Kiali graph provides a powerful way to visualize the topology of your Istio service mesh. It shows you which services communicate with each other. + +:::note Prerequisites: + +To enable traffic to show up in the graph, ensure you have prometheus installed in the cluster. Rancher-istio installs Kiali configured by default to work with the rancher-monitoring chart. You can use rancher-monitoring or install your own monitoring solution. Optional: you can change configuration on how data scraping occurs by setting the [Selectors & Scrape Configs](../../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) options. + +::: + +To see the traffic graph, + +1. In the cluster where Istio is installed, click **Istio** in the left navigation bar. +1. Click the **Kiali** link. +1. Click on **Graph** in the side nav. +1. Change the namespace in the **Namespace** dropdown to view the traffic for each namespace. + +If you refresh the URL to the BookInfo app several times, you should be able to see green arrows on the Kiali graph showing traffic to `v1` and `v3` of the `reviews` service. The control panel on the right side of the graph lets you configure details including how many minutes of the most recent traffic should be shown on the graph. + +For additional tools and visualizations, you can go to Grafana, and Prometheus dashboards from the **Monitoring** **Overview** page diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md new file mode 100644 index 00000000000..38548a4921d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md @@ -0,0 +1,150 @@ +--- +title: 4. Set up the Istio Gateway +--- + + + + + +The gateway to each cluster can have its own port or load balancer, which is unrelated to a service mesh. By default, each Rancher-provisioned cluster has one NGINX ingress controller allowing traffic into the cluster. + +You can use the Nginx Ingress controller with or without Istio installed. If this is the only gateway to your cluster, Istio will be able to route traffic from service to service, but Istio will not be able to receive traffic from outside the cluster. + +To allow Istio to receive external traffic, you need to enable Istio's gateway, which works as a north-south proxy for external traffic. When you enable the Istio gateway, the result is that your cluster will have two Ingresses. + +You will also need to set up a Kubernetes gateway for your services. This Kubernetes resource points to Istio's implementation of the ingress gateway to the cluster. + +You can route traffic into the service mesh with a load balancer or use Istio's NodePort gateway. This section describes how to set up the NodePort gateway. + +For more information on the Istio gateway, refer to the [Istio documentation.](https://istio.io/docs/reference/config/networking/v1alpha3/gateway/) + +![In an Istio-enabled cluster, you can have two Ingresses: the default Nginx Ingress, and the default Istio controller.](/img/istio-ingress.svg) + +## Enable an Istio Gateway + +The ingress gateway is a Kubernetes service that will be deployed in your cluster. The Istio Gateway allows for more extensive customization and flexibility. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Istio > Gateways**. +1. Click **Create from Yaml**. +1. Paste your Istio Gateway yaml, or **Read from File**. +1. Click **Create**. + +**Result:** The gateway is deployed, and will now route traffic with applied rules. + +## Example Istio Gateway + +We add the BookInfo app deployments in services when going through the Workloads example. Next we add an Istio Gateway so that the app is accessible from outside your cluster. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Istio > Gateways**. +1. Click **Create from Yaml**. +1. Copy and paste the Gateway yaml provided below. +1. Click **Create**. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: bookinfo-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +--- +``` + +Then to deploy the VirtualService that provides the traffic routing for the Gateway: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Istio > VirtualServices**. +1. Copy and paste the VirtualService yaml provided below. +1. Click **Create**. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: bookinfo +spec: + hosts: + - "*" + gateways: + - bookinfo-gateway + http: + - match: + - uri: + exact: /productpage + - uri: + prefix: /static + - uri: + exact: /login + - uri: + exact: /logout + - uri: + prefix: /api/v1/products + route: + - destination: + host: productpage + port: + number: 9080 +``` + +**Result:** You have configured your gateway resource so that Istio can receive traffic from outside the cluster. + +Confirm that the resource exists by running: +``` +kubectl get gateway -A +``` + +The result should be something like this: +``` +NAME AGE +bookinfo-gateway 64m +``` + +### Access the ProductPage Service from a Web Browser + +To test and see if the BookInfo app deployed correctly, the app can be viewed a web browser using the Istio controller IP and port, combined with the request name specified in your Kubernetes gateway resource: + +`http://:/productpage` + +To get the ingress gateway URL and port, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Scroll down to the `istio-system` namespace. +1. Within `istio-system`, there is a workload named `istio-ingressgateway`. Under the name of this workload, you should see links, such as `80/tcp`. +1. Click one of those links. This should show you the URL of the ingress gateway in your web browser. Append `/productpage` to the URL. + +**Result:** You should see the BookInfo app in the web browser. + +For help inspecting the Istio controller URL and ports, try the commands the [Istio documentation.](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#determining-the-ingress-ip-and-ports) + +## Troubleshooting + +The [official Istio documentation](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#troubleshooting) suggests `kubectl` commands to inspect the correct ingress host and ingress port for external requests. + +### Confirming that the Kubernetes Gateway Matches Istio's Ingress Controller + +You can try the steps in this section to make sure the Kubernetes gateway is configured properly. + +In the gateway resource, the selector refers to Istio's default ingress controller by its label, in which the key of the label is `istio` and the value is `ingressgateway`. To make sure the label is appropriate for the gateway, do the following: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Scroll down to the `istio-system` namespace. +1. Within `istio-system`, there is a workload named `istio-ingressgateway`. Click the name of this workload and go to the **Labels and Annotations** section. You should see that it has the key `istio` and the value `ingressgateway`. This confirms that the selector in the Gateway resource matches Istio's default ingress controller. + +### [Next: Set up Istio's Components for Traffic Management](set-up-traffic-management.md) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md new file mode 100644 index 00000000000..150289745d7 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md @@ -0,0 +1,79 @@ +--- +title: 5. Set up Istio's Components for Traffic Management +--- + + + + + +A central advantage of traffic management in Istio is that it allows dynamic request routing. Some common applications for dynamic request routing include canary deployments and blue/green deployments. The two key resources in Istio traffic management are *virtual services* and *destination rules*. + +- [Virtual services](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/) intercept and direct traffic to your Kubernetes services, allowing you to divide percentages of traffic from a request to different services. You can use them to define a set of routing rules to apply when a host is addressed. +- [Destination rules](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule/) serve as the single source of truth about which service versions are available to receive traffic from virtual services. You can use these resources to define policies that apply to traffic that is intended for a service after routing has occurred. + +This section describes how to add an example virtual service that corresponds to the `reviews` microservice in the sample BookInfo app. The purpose of this service is to divide traffic between two versions of the `reviews` service. + +In this example, we take the traffic to the `reviews` service and intercept it so that 50 percent of it goes to `v1` of the service and 50 percent goes to `v2`. + +After this virtual service is deployed, we will generate traffic and see from the Kiali visualization that traffic is being routed evenly between the two versions of the service. + +To deploy the virtual service and destination rules for the `reviews` service, +1. Click **☰ > Cluster Management**. +1. Go to the cluster where Istio is installed and click **Explore**. +1. In the cluster where Istio is installed, click **Istio > DestinationRules** in the left navigation bar. +1. Click **Create**. +1. Copy and paste the DestinationRule yaml provided below. +1. Click **Create**. +1. Click **Edit as YAML** and use this configuration: + + ```yaml + apiVersion: networking.istio.io/v1alpha3 + kind: DestinationRule + metadata: + name: reviews + spec: + host: reviews + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 + - name: v3 + labels: + version: v3 + ``` +1. Click **Create**. + +Then to deploy the VirtualService that provides the traffic routing that utilizes the DestinationRule: + +1. Click **VirtualService** in the side nav bar. +1. Click **Create from Yaml**. +1. Copy and paste the VirtualService yaml provided below. +1. Click **Create**. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: reviews +spec: + hosts: + - reviews + http: + - route: + - destination: + host: reviews + subset: v1 + weight: 50 + - destination: + host: reviews + subset: v3 + weight: 50 +--- +``` + +**Result:** When you generate traffic to this service (for example, by refreshing the ingress gateway URL), the Kiali traffic graph will reflect that traffic to the `reviews` service is divided evenly between `v1` and `v3`. + +### [Next: Generate and View Traffic](generate-and-view-traffic.md) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md new file mode 100644 index 00000000000..af5b22672c5 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md @@ -0,0 +1,363 @@ +--- +title: 3. Add Deployments and Services with the Istio Sidecar +--- + + + + + +:::note Prerequisite: + +To enable Istio for a workload, the cluster and namespace must have the Istio app installed. + +::: + +Enabling Istio in a namespace only enables automatic sidecar injection for new workloads. To enable the Envoy sidecar for existing workloads, you need to enable it manually for each workload. + +To inject the Istio sidecar on an existing workload in the namespace, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. Click **Workload**. +1. Go to the workload where you want to inject the Istio sidecar and click **⋮ > Redeploy**. When the workload is redeployed, it will have the Envoy sidecar automatically injected. + +Wait a few minutes for the workload to upgrade to have the istio sidecar. Click it and go to the Containers section. You should be able to see `istio-proxy` alongside your original workload. This means the Istio sidecar is enabled for the workload. Istio is doing all the wiring for the sidecar envoy. Now Istio can do all the features automatically if you enable them in the yaml. + +### Add Deployments and Services + +There are a few ways to add new **Deployments** in your namespace: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Click **Deployment**. +1. Fill out the form, or **Edit as Yaml**. +1. Click **Create**. + +To add a **Service** to your namespace: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Service Discovery > Services**. +1. Click **Create**. +1. Select the type of service that you want. +1. Fill out the form, or **Edit as Yaml**. +1. Click **Create** + +You can also create deployments and services using the kubectl **shell** + +1. Run `kubectl create -f .yaml` if your file is stored locally in the cluster +1. Or run `cat<< EOF | kubectl apply -f -`, paste the file contents into the terminal, then run `EOF` to complete the command. + +### Example Deployments and Services + +Next we add the Kubernetes resources for the sample deployments and services for the BookInfo app in Istio's documentation. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the top navigation bar, open the kubectl shell. +1. Run `cat<< EOF | kubectl apply -f -` +1. Copy the below resources into the the shell. +1. Run `EOF` + +This will set up the following sample resources from Istio's example BookInfo app: + +Details service and deployment: + +- A `details` Service +- A ServiceAccount for `bookinfo-details` +- A `details-v1` Deployment + +Ratings service and deployment: + +- A `ratings` Service +- A ServiceAccount for `bookinfo-ratings` +- A `ratings-v1` Deployment + +Reviews service and deployments (three versions): + +- A `reviews` Service +- A ServiceAccount for `bookinfo-reviews` +- A `reviews-v1` Deployment +- A `reviews-v2` Deployment +- A `reviews-v3` Deployment + +Productpage service and deployment: + +This is the main page of the app, which will be visible from a web browser. The other services will be called from this page. + +- A `productpage` service +- A ServiceAccount for `bookinfo-productpage` +- A `productpage-v1` Deployment + +### Resource YAML + +```yaml +# Copyright 2017 Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################################## +# Details service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: details + labels: + app: details + service: details +spec: + ports: + - port: 9080 + name: http + selector: + app: details +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-details +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: details-v1 + labels: + app: details + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: details + version: v1 + template: + metadata: + labels: + app: details + version: v1 + spec: + serviceAccountName: bookinfo-details + containers: + - name: details + image: docker.io/istio/examples-bookinfo-details-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Ratings service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: ratings + labels: + app: ratings + service: ratings +spec: + ports: + - port: 9080 + name: http + selector: + app: ratings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-ratings +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ratings-v1 + labels: + app: ratings + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: ratings + version: v1 + template: + metadata: + labels: + app: ratings + version: v1 + spec: + serviceAccountName: bookinfo-ratings + containers: + - name: ratings + image: docker.io/istio/examples-bookinfo-ratings-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Reviews service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: reviews + labels: + app: reviews + service: reviews +spec: + ports: + - port: 9080 + name: http + selector: + app: reviews +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-reviews +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v1 + labels: + app: reviews + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v1 + template: + metadata: + labels: + app: reviews + version: v1 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v2 + labels: + app: reviews + version: v2 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v2 + template: + metadata: + labels: + app: reviews + version: v2 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v2:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v3 + labels: + app: reviews + version: v3 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v3 + template: + metadata: + labels: + app: reviews + version: v3 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v3:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Productpage services +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: productpage + labels: + app: productpage + service: productpage +spec: + ports: + - port: 9080 + name: http + selector: + app: productpage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-productpage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: productpage-v1 + labels: + app: productpage + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: productpage + version: v1 + template: + metadata: + labels: + app: productpage + version: v1 + spec: + serviceAccountName: bookinfo-productpage + containers: + - name: productpage + image: docker.io/istio/examples-bookinfo-productpage-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +``` + +### [Next: Set up the Istio Gateway](set-up-istio-gateway.md) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md new file mode 100644 index 00000000000..664ee75b85e --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md @@ -0,0 +1,43 @@ +--- +title: Pod Security Policies +--- + + + + + +:::note + +These cluster options are only available for [clusters in which Rancher has launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + +::: + +You can always assign a pod security policy (PSP) to an existing project if you didn't assign one during creation. + +### Prerequisites + +- Create a Pod Security Policy within Rancher. Before you can assign a default PSP to an existing project, you must have a PSP available for assignment. For instruction, see [Creating Pod Security Policies](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md). +- Assign a default Pod Security Policy to the project's cluster. You can't assign a PSP to a project until one is already applied to the cluster. For more information, see [the documentation about adding a pod security policy to a cluster](../../new-user-guides/manage-clusters/add-a-pod-security-policy.md). + +### Applying a Pod Security Policy + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to move a namespace and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the project that you want to add a PSP to. From that project, select **⋮ > Edit Config**. +1. From the **Pod Security Policy** drop-down, select the PSP you want to apply to the project. + Assigning a PSP to a project will: + + - Override the cluster's default PSP. + - Apply the PSP to the project. + - Apply the PSP to any namespaces you add to the project later. + +1. Click **Save**. + +**Result:** The PSP is applied to the project and any namespaces added to the project. + +:::note + +Any workloads that are already running in a cluster or project before a PSP is assigned will not be checked to determine if they comply with the PSP. Workloads would need to be cloned or upgraded to see if they pass the PSP. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md new file mode 100644 index 00000000000..85f57c5b1bd --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md @@ -0,0 +1,95 @@ +--- +title: How Resource Quotas Work in Rancher Projects +--- + + + + + +Resource quotas in Rancher include the same functionality as the [native version of Kubernetes](https://kubernetes.io/docs/concepts/policy/resource-quotas/). However, in Rancher, resource quotas have been extended so that you can apply them to projects. + +In a standard Kubernetes deployment, resource quotas are applied to individual namespaces. However, you cannot apply the quota to your namespaces simultaneously with a single action. Instead, the resource quota must be applied multiple times. + +In the following diagram, a Kubernetes administrator is trying to enforce a resource quota without Rancher. The administrator wants to apply a resource quota that sets the same CPU and memory limit to every namespace in his cluster (`Namespace 1-4`) . However, in the base version of Kubernetes, each namespace requires a unique resource quota. The administrator has to create four different resource quotas that have the same specs configured (`Resource Quota 1-4`) and apply them individually. + +Base Kubernetes: Unique Resource Quotas Being Applied to Each Namespace + +![Native Kubernetes Resource Quota Implementation](/img/kubernetes-resource-quota.svg) + +Resource quotas are a little different in Rancher. In Rancher, you apply a resource quota to the project, and then the quota propagates to each namespace, whereafter Kubernetes enforces your limits using the native version of resource quotas. If you want to change the quota for a specific namespace, you can override it. + +The resource quota includes two limits, which you set while creating or editing a project: + + +- **Project Limits:** + + This set of values configures a total limit for each specified resource shared among all namespaces in the project. + +- **Namespace Default Limits:** + + This set of values configures the default quota limit available for each namespace for each specified resource. + When a namespace is created in the project without overrides, this limit is automatically bound to the namespace and enforced. + + +In the following diagram, a Rancher administrator wants to apply a resource quota that sets the same CPU and memory limit for every namespace in their project (`Namespace 1-4`). However, in Rancher, the administrator can set a resource quota for the project (`Project Resource Quota`) rather than individual namespaces. This quota includes resource limits for both the entire project (`Project Limit`) and individual namespaces (`Namespace Default Limit`). Rancher then propagates the `Namespace Default Limit` quotas to each namespace (`Namespace Resource Quota`) when created. + +Rancher: Resource Quotas Propagating to Each Namespace + +![Rancher Resource Quota Implementation](/img/rancher-resource-quota.png) + +Let's highlight some more nuanced functionality for namespaces created **_within_** the Rancher UI. If a quota is deleted at the project level, it will also be removed from all namespaces contained within that project, despite any overrides that may exist. Further, updating an existing namespace default limit for a quota at the project level will not result in that value being propagated to existing namespaces in the project; the updated value will only be applied to newly created namespaces in that project. To update a namespace default limit for existing namespaces you can delete and subsequently recreate the quota at the project level with the new default value. This will result in the new default value being applied to all existing namespaces in the project. + +Before creating a namespace in a project, Rancher compares the amounts of the project's available resources and requested resources, regardless of whether they come from the default or overridden limits. +If the requested resources exceed the remaining capacity in the project for those resources, Rancher will assign the namespace the remaining capacity for that resource. + +However, this is not the case with namespaces created **_outside_** of Rancher's UI. For namespaces created via `kubectl`, Rancher +will assign a resource quota that has a **zero** amount for any resource that requested more capacity than what remains in the project. + +To create a project with a resource quota via `kubectl`, use the `spec.resourceQuota` and `spec.namespaceDefaultResourceQuota` fields: + +``` +apiVersion: management.cattle.io/v3 +kind: Project +metadata: + generateName: p- + namespace: c-m-abcde +spec: + clusterName: c-m-abcde + displayName: myproject + resourceQuota: + limit: + limitsCpu: 1000m + namespaceDefaultResourceQuota: + limit: + limitsCpu: 50m +``` + +Use `metadata.generateName` to ensure a unique project ID. Set `metadata.namespace` and `spec.clusterName` to the ID for the cluster the project belongs to. + +To create a namespace in an existing project via `kubectl`, use the `field.cattle.io/projectId` annotation. To override the default +requested quota limit, use the `field.cattle.io/resourceQuota` annotation. + +Note that Rancher will only override limits for resources that are defined on the project quota. + +``` +apiVersion: v1 +kind: Namespace +metadata: + annotations: + field.cattle.io/projectId: [your-cluster-ID]:[your-project-ID] + field.cattle.io/resourceQuota: '{"limit":{"limitsCpu":"100m", "configMaps": "50"}}' + name: my-ns +``` +In this example, if the project's quota does not include configMaps in its list of resources, then Rancher will ignore `configMaps` in this override. + +Users are advised to create dedicated `ResourceQuota` objects in namespaces to configure additional custom limits for resources not defined on the project. +Resource quotas are native Kubernetes objects, and Rancher will ignore user-defined quotas in namespaces belonging to a project with a quota, +thus giving users more control. + +The following table explains the key differences between the two quota types. + +| Rancher Resource Quotas | Kubernetes Resource Quotas | +| ---------------------------------------------------------- | -------------------------------------------------------- | +| Applies to projects and namespace. | Applies to namespaces only. | +| Creates resource pool for all namespaces in project. | Applies static resource limits to individual namespaces. | +| Applies resource quotas to namespaces through propagation. | Applies only to the assigned namespace. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md new file mode 100644 index 00000000000..9f6c8ca5655 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md @@ -0,0 +1,38 @@ +--- +title: Overriding the Default Limit for a Namespace +--- + + + + + +Although the **Namespace Default Limit** propagates from the project to each namespace when created, in some cases, you may need to increase (or decrease) the quotas for a specific namespace. In this situation, you can override the default limits by editing the namespace. + +In the diagram below, the Rancher administrator has a resource quota in effect for their project. However, the administrator wants to override the namespace limits for `Namespace 3` so that it has more resources available. Therefore, the administrator [raises the namespace limits](../../../new-user-guides/manage-clusters/projects-and-namespaces.md) for `Namespace 3` so that the namespace can access more resources. + +Namespace Default Limit Override + +![Namespace Default Limit Override](/img/rancher-resource-quota-override.svg) + +How to: [Editing Namespace Resource Quotas](../../../new-user-guides/manage-clusters/projects-and-namespaces.md) + +### Editing Namespace Resource Quotas + +If there is a resource quota configured for a project, you can override the namespace default limit to provide a specific namespace with access to more (or less) project resources. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to edit a namespace resource quota and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the namespace for which you want to edit the resource quota. Click **⋮ > Edit Config**. +1. Edit the resource limits. These limits determine the resources available to the namespace. The limits must be set within the configured project limits. + + For more information about each **Resource Type**, see [the type reference](resource-quota-types.md). + + :::note + + - If a resource quota is not configured for the project, these options will not be available. + - If you enter limits that exceed the configured project limits, Rancher will not let you save your edits. + + ::: + +**Result:** Your override is applied to the namespace's resource quota. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md new file mode 100644 index 00000000000..e789f5bf38b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md @@ -0,0 +1,31 @@ +--- +title: Resource Quota Type Reference +--- + + + + + +When you create a resource quota, you are configuring the pool of resources available to the project. You can set the following resource limits for the following resource types. + +| Resource Type | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU Limit* | The maximum amount of CPU (in [millicores](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)) allocated to the project/namespace.1 | +| CPU Reservation* | The minimum amount of CPU (in millicores) guaranteed to the project/namespace.1 | +| Memory Limit* | The maximum amount of memory (in bytes) allocated to the project/namespace.1 | +| Memory Reservation* | The minimum amount of memory (in bytes) guaranteed to the project/namespace.1 | +| Storage Reservation | The minimum amount of storage (in gigabytes) guaranteed to the project/namespace. | +| Services Load Balancers | The maximum number of load balancers services that can exist in the project/namespace. | +| Services Node Ports | The maximum number of node port services that can exist in the project/namespace. | +| Pods | The maximum number of pods that can exist in the project/namespace in a non-terminal state (i.e., pods with a state of `.status.phase in (Failed, Succeeded)` equal to true). | +| Services | The maximum number of services that can exist in the project/namespace. | +| ConfigMaps | The maximum number of ConfigMaps that can exist in the project/namespace. | +| Persistent Volume Claims | The maximum number of persistent volume claims that can exist in the project/namespace. | +| Replications Controllers | The maximum number of replication controllers that can exist in the project/namespace. | +| Secrets | The maximum number of secrets that can exist in the project/namespace. | + +:::note ***** + +When setting resource quotas, if you set anything related to CPU or Memory (i.e. limits or reservations) on a project / namespace, all containers will require a respective CPU or Memory field set during creation. A container default resource limit can be set at the same time to avoid the need to explicitly set these limits for every workload. See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits) for more details on why this is required. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md new file mode 100644 index 00000000000..dd4e3241319 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md @@ -0,0 +1,44 @@ +--- +title: Setting Container Default Resource Limits +--- + + + + + +When setting resource quotas, if you set anything related to CPU or Memory (i.e. limits or reservations) on a project / namespace, all containers will require a respective CPU or Memory field set during creation. See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits) for more details on why this is required. + +To avoid setting these limits on each and every container during workload creation, a default container resource limit can be specified on the namespace. + +### Editing the Container Default Resource Limit + +Edit the container default resource limit when: + +- You have a CPU or Memory resource quota set on a project, and want to supply the corresponding default values for a container. +- You want to edit the default container resource limit. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to edit the default resource limit and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the project that you want to edit the container default resource limit. From that project, select **⋮ > Edit Config**. +1. Expand **Container Default Resource Limit** and edit the values. + +### Resource Limit Propagation + +When the default container resource limit is set at a project level, the parameter will be propagated to any namespace created in the project after the limit has been set. For any existing namespace in a project, this limit will not be automatically propagated. You will need to manually set the default container resource limit for any existing namespaces in the project in order for it to be used when creating any containers. + +You can set a default container resource limit on a project and launch any catalog applications. + +Once a container default resource limit is configured on a namespace, the default will be pre-populated for any containers created in that namespace. These limits/reservations can always be overridden during workload creation. + +### Container Resource Quota Types + +The following resource limits can be configured: + +| Resource Type | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU Limit | The maximum amount of CPU (in [millicores](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)) allocated to the container.| +| CPU Reservation | The minimum amount of CPU (in millicores) guaranteed to the container. | +| Memory Limit | The maximum amount of memory (in bytes) allocated to the container. | +| Memory Reservation | The minimum amount of memory (in bytes) guaranteed to the container. +| NVIDIA GPU Limit/Reservation | The amount of GPUs allocated to the container. The limit and reservation are always the same for GPUs. | diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md new file mode 100644 index 00000000000..fe129e27bb4 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md @@ -0,0 +1,152 @@ +--- +title: Persistent Grafana Dashboards +--- + + + + + +To allow the Grafana dashboard to persist after the Grafana instance restarts, add the dashboard configuration JSON into a ConfigMap. ConfigMaps also allow the dashboards to be deployed with a GitOps or CD based approach. This allows the dashboard to be put under version control. + +- [Creating a Persistent Grafana Dashboard](#creating-a-persistent-grafana-dashboard) +- [Known Issues](#known-issues) + +## Creating a Persistent Grafana Dashboard + + + + +:::note Prerequisites: + +- The monitoring application needs to be installed. +- To create the persistent dashboard, you must have at least the **Manage Config Maps** Rancher RBAC permissions assigned to you in the project or namespace that contains the Grafana Dashboards. This correlates to the `monitoring-dashboard-edit` or `monitoring-dashboard-admin` Kubernetes native RBAC Roles exposed by the Monitoring chart. +- To see the links to the external monitoring UIs, including Grafana dashboards, you will need at least a [project-member role.](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#users-with-rancher-based-permissions) + +::: + +### 1. Get the JSON model of the dashboard that you want to persist + +To create a persistent dashboard, you will need to get the JSON model of the dashboard you want to persist. You can use a premade dashboard or build your own. + +To use a premade dashboard, go to [https://grafana.com/grafana/dashboards](https://grafana.com/grafana/dashboards), open up its detail page, and click on the **Download JSON** button to get the JSON model for the next step. + +To use your own dashboard: + +1. Click on the link to open Grafana. On the cluster detail page, click **Monitoring**. +1. Log in to Grafana. Note: The default Admin username and password for the Grafana instance is `admin/prom-operator`. Alternative credentials can also be supplied on deploying or upgrading the chart. + + :::note + + Regardless of who has the password, in order to access the Grafana instance, you still need at least the Manage Services or View Monitoring permissions in the project that Rancher Monitoring is deployed into. Alternative credentials can also be supplied on deploying or upgrading the chart. + + ::: + +1. Create a dashboard using Grafana's UI. Once complete, go to the dashboard's settings by clicking on the gear icon in the top navigation menu. In the left navigation menu, click **JSON Model**. +1. Copy the JSON data structure that appears. + +### 2. Create a ConfigMap using the Grafana JSON model + +Create a ConfigMap in the namespace that contains your Grafana Dashboards (e.g. cattle-dashboards by default). + +The ConfigMap should look like this: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards # Change if using a non-default namespace +data: + .json: |- + +``` + +By default, Grafana is configured to watch all ConfigMaps with the `grafana_dashboard` label within the `cattle-dashboards` namespace. + +To specify that you would like Grafana to watch for ConfigMaps across all namespaces, refer to [this section.](#configuring-namespaces-for-the-grafana-dashboard-configmap) + +To create the ConfigMap in the Rancher UI, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. Click **More Resources > Core > ConfigMaps**. +1. Click **Create**. +1. Set up the key-value pairs similar to the example above. When entering the value for `.json`, click **Read from File** to upload the JSON data model as the value. +1. Click **Create**. + +**Result:** After the ConfigMap is created, it should show up on the Grafana UI and be persisted even if the Grafana pod is restarted. + +Dashboards that are persisted using ConfigMaps cannot be deleted or edited from the Grafana UI. + +If you attempt to delete the dashboard in the Grafana UI, you will see the error message "Dashboard cannot be deleted because it was provisioned." To delete the dashboard, you will need to delete the ConfigMap. + +### Configuring Namespaces for the Grafana Dashboard ConfigMap + +To specify that you would like Grafana to watch for ConfigMaps across all namespaces, set this value in the `rancher-monitoring` Helm chart: + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +Note that the RBAC roles exposed by the Monitoring chart to add Grafana Dashboards are still restricted to giving permissions for users to add dashboards in the namespace defined in `grafana.dashboards.namespace`, which defaults to `cattle-dashboards`. + + + + +:::note Prerequisites: + +- The monitoring application needs to be installed. +- You must have the cluster-admin ClusterRole permission. + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to configure the Grafana namespace and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Grafana**. +1. Log in to Grafana. Note: The default Admin username and password for the Grafana instance is `admin/prom-operator`. Alternative credentials can also be supplied on deploying or upgrading the chart. + + :::note + + Regardless of who has the password, cluster administrator permission in Rancher is still required to access the Grafana instance. + + ::: + +1. Go to the dashboard that you want to persist. In the top navigation menu, go to the dashboard settings by clicking the gear icon. +1. In the left navigation menu, click **JSON Model**. +1. Copy the JSON data structure that appears. +1. Create a ConfigMap in the `cattle-dashboards` namespace. The ConfigMap needs to have the label `grafana_dashboard: "1"`. Paste the JSON into the ConfigMap in the format shown in the example below: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards + data: + .json: |- + + ``` + +**Result:** After the ConfigMap is created, it should show up on the Grafana UI and be persisted even if the Grafana pod is restarted. + +Dashboards that are persisted using ConfigMaps cannot be deleted from the Grafana UI. If you attempt to delete the dashboard in the Grafana UI, you will see the error message "Dashboard cannot be deleted because it was provisioned." To delete the dashboard, you will need to delete the ConfigMap. + +To prevent the persistent dashboard from being deleted when Monitoring v2 is uninstalled, add the following annotation to the `cattle-dashboards` namespace: + +``` +helm.sh/resource-policy: "keep" +``` + + + + +## Known Issues + +For users who are using Monitoring V2 v9.4.203 or below, uninstalling the Monitoring chart will delete the `cattle-dashboards` namespace, which will delete all persisted dashboards, unless the namespace is marked with the annotation `helm.sh/resource-policy: "keep"`. + +This annotation will be added by default in the new monitoring chart released by Rancher v2.5.8, but it still needs to be manually applied for users of earlier Rancher versions. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md new file mode 100644 index 00000000000..c3b315f3e5b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md @@ -0,0 +1,43 @@ +--- +title: Customizing Grafana Dashboards +--- + + + + + +In this section, you'll learn how to customize the Grafana dashboard to show metrics that apply to a certain container. + +### Prerequisites + +Before you can customize a Grafana dashboard, the `rancher-monitoring` application must be installed. + +To see the links to the external monitoring UIs, including Grafana dashboards, you will need at least a [project-member role.](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#users-with-rancher-based-permissions) + +### Signing in to Grafana + +1. In the Rancher UI, go to the cluster that has the dashboard you want to customize. +1. In the left navigation menu, click **Monitoring.** +1. Click **Grafana.** The Grafana dashboard should open in a new tab. +1. Go to the log in icon in the lower left corner and click **Sign In.** +1. Log in to Grafana. The default Admin username and password for the Grafana instance is `admin/prom-operator`. (Regardless of who has the password, cluster administrator permission in Rancher is still required access the Grafana instance.) Alternative credentials can also be supplied on deploying or upgrading the chart. + + +### Getting the PromQL Query Powering a Grafana Panel + +For any panel, you can click the title and click **Explore** to get the PromQL queries powering the graphic. + +For this example, we would like to get the CPU usage for the Alertmanager container, so we click **CPU Utilization > Inspect.** + +The **Data** tab shows the underlying data as a time series, with the time in first column and the PromQL query result in the second column. Copy the PromQL query. + +``` +(1 - (avg(irate({__name__=~"node_cpu_seconds_total|windows_cpu_time_total",mode="idle"}[5m])))) * 100 +``` + +You can then modify the query in the Grafana panel or create a new Grafana panel using the query. + +See also: + +- [Grafana docs on editing a panel](https://grafana.com/docs/grafana/latest/panels-visualizations/configure-panel-options/#edit-a-panel) +- [Grafana docs on adding a panel to a dashboard](https://grafana.com/docs/grafana/latest/panels-visualizations/panel-editor-overview) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md new file mode 100644 index 00000000000..020b06924d6 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md @@ -0,0 +1,23 @@ +--- +title: Debugging High Memory Usage +--- + + + + + +Every time series in Prometheus is uniquely identified by its [metric name](https://prometheus.io/docs/practices/naming/#metric-names) and optional key-value pairs called [labels.](https://prometheus.io/docs/practices/naming/#labels) + +The labels allow the ability to filter and aggregate the time series data, but they also multiply the amount of data that Prometheus collects. + +Each time series has a defined set of labels, and Prometheus generates a new time series for all unique combinations of labels. If a metric has two labels attached, two time series are generated for that metric. Changing any label value, including adding or removing a label, will create a new time series. + +Prometheus is optimized to store data that is index-based on series. It is designed for a relatively consistent number of time series and a relatively large number of samples that need to be collected from the exporters over time. + +Inversely, Prometheus is not optimized to accommodate a rapidly changing number of time series. For that reason, large bursts of memory usage can occur when monitoring is installed on clusters where many resources are being created and destroyed, especially on multi-tenant clusters. + +### Reducing Memory Bursts + +To reduce memory consumption, Prometheus can be configured to store fewer time series, by scraping fewer metrics or by attaching fewer labels to the time series. To see which series use the most memory, you can check the TSDB (time series database) status page in the Prometheus UI. + +Distributed Prometheus solutions such as [Thanos](https://thanos.io/) and [Cortex](https://cortexmetrics.io/) use an alternate architecture in which multiple small Prometheus instances are deployed. In the case of Thanos, the metrics from each Prometheus are aggregated into the common Thanos deployment, and then those metrics are exported to a persistent store, such as S3. This more robust architecture avoids burdening any single Prometheus instance with too many time series, while also preserving the ability to query metrics on a global level. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md new file mode 100644 index 00000000000..2e4ab117ca0 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md @@ -0,0 +1,79 @@ +--- +title: Enable Monitoring +--- + + + + + +As an [administrator](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) or [cluster owner](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles), you can configure Rancher to deploy Prometheus to monitor your Kubernetes cluster. + +This page describes how to enable monitoring and alerting within a cluster using the new monitoring application. + +You can enable monitoring with or without SSL. + +## Requirements + +- Allow traffic on port 9796 for each of your nodes. Prometheus scrapes metrics from these ports. + - You may also need to allow traffic on port 10254 for each of your nodes, if [PushProx](../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#pushprox) is disabled (`ingressNginx.enabled` set to `false`), or you've upgraded from a previous Rancher version that had v1 monitoring already installed. +- Make sure that your cluster fulfills the resource requirements. The cluster should have at least 1950Mi memory available, 2700m CPU, and 50Gi storage. See [Configuring Resource Limits and Requests](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#configuring-resource-limits-and-requests) for a breakdown of the resource limits and requests. +- When you install monitoring on an RKE cluster that uses RancherOS or Flatcar Linux nodes, change the etcd node certificate directory to `/opt/rke/etc/kubernetes/ssl`. +- For clusters that have been provisioned with the RKE CLI and that have the address set to a hostname instead of an IP address, set `rkeEtcd.clients.useLocalhost` to `true` when you configure the Values during installation. For example: + +```yaml +rkeEtcd: + clients: + useLocalhost: true +``` + +:::note + +If you want to set up Alertmanager, Grafana or Ingress, it has to be done with the settings on the Helm chart deployment. It's problematic to create Ingress outside the deployment. + +::: + +## Setting Resource Limits and Requests + +The resource requests and limits can be configured when installing `rancher-monitoring`. To configure Prometheus resources from the Rancher UI, click **Apps > Monitoring** in the upper left corner. + +For more information about the default limits, see [this page.](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#configuring-resource-limits-and-requests) + +## Install the Monitoring Application + +### Enable Monitoring for use without SSL + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster Tools** (bottom left corner). +1. Click **Install** by Monitoring. +1. Optional: Customize requests, limits and more for Alerting, Prometheus, and Grafana in the Values step. For help, refer to the [configuration reference.](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md) + +**Result:** The monitoring app is deployed in the `cattle-monitoring-system` namespace. + +### Enable Monitoring for use with SSL + +1. Follow the steps on [this page](../../new-user-guides/kubernetes-resources-setup/secrets.md) to create a secret in order for SSL to be used for alerts. + - The secret should be created in the `cattle-monitoring-system` namespace. If it doesn't exist, create it first. + - Add the `ca`, `cert`, and `key` files to the secret. +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to enable monitoring for use with SSL and click **Explore**. +1. Click **Apps > Charts**. +1. Click **Monitoring**. +1. Click **Install** or **Update**, depending on whether you have already installed Monitoring. +1. Check the box for **Customize Helm options before install** and click **Next**. +1. Click **Alerting**. +1. In the **Additional Secrets** field, add the secrets created earlier. + +**Result:** The monitoring app is deployed in the `cattle-monitoring-system` namespace. + +When [creating a receiver,](../../../reference-guides/monitoring-v2-configuration/receivers.md#creating-receivers-in-the-rancher-ui) SSL-enabled receivers such as email or webhook will have a **SSL** section with fields for **CA File Path**, **Cert File Path**, and **Key File Path**. Fill in these fields with the paths to each of `ca`, `cert`, and `key`. The path will be of the form `/etc/alertmanager/secrets/name-of-file-in-secret`. + +For example, if you created a secret with these key-value pairs: + +```yaml +ca.crt=`base64-content` +cert.pem=`base64-content` +key.pfx=`base64-content` +``` + +Then **Cert File Path** would be set to `/etc/alertmanager/secrets/cert.pem`. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md new file mode 100644 index 00000000000..50a45a16216 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md @@ -0,0 +1,11 @@ +--- +title: Customizing Grafana Dashboards +--- + + + + + +Grafana dashboards are customized the same way whether it's for rancher-monitoring or for Prometheus Federator. + +For instructions, refer to [this page](../customize-grafana-dashboard.md). \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md new file mode 100644 index 00000000000..b4f9fcc166f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md @@ -0,0 +1,90 @@ +--- +title: Enable Prometheus Federator +--- + + + + + +## Requirements + +By default, Prometheus Federator is configured and intended to be deployed alongside [rancher-monitoring](../../../../pages-for-subheaders/monitoring-and-alerting.md), which deploys Prometheus Operator alongside a Cluster Prometheus that each Project Monitoring Stack is configured to federate namespace-scoped metrics from by default. + +For instructions on installing rancher-monitoring, refer to [this page](../enable-monitoring.md). + +The default configuration should already be compatible with your rancher-monitoring stack. However, to optimize the security and usability of Prometheus Federator in your cluster, we recommend making these additional configurations to rancher-monitoring: + +- [Ensure the cattle-monitoring-system namespace is placed into the System Project](#ensure-the-cattle-monitoring-system-namespace-is-placed-into-the-system-project-or-a-similarly-locked-down-project-that-has-access-to-other-projects-in-the-cluster). +- [Configure rancher-monitoring to only watch for resources created by the Helm chart itself](#configure-rancher-monitoring-to-only-watch-for-resources-created-by-the-helm-chart-itself). +- [Increase the CPU / memory limits of the Cluster Prometheus](#increase-the-cpu--memory-limits-of-the-cluster-prometheus). + +### Ensure the cattle-monitoring-system namespace is placed into the System Project (or a similarly locked down Project that has access to other Projects in the cluster) + +![Select Projects-Namespaces](/img/install-in-system-project.png) + +Prometheus Operator's security model expects that the namespace it is deployed into (e.g., `cattle-monitoring-system`) has limited access for anyone except Cluster Admins to avoid privilege escalation via execing into Pods (such as the Jobs executing Helm operations). In addition, deploying Prometheus Federator and all Project Prometheus stacks into the System Project ensures that each Project Prometheus is able to reach out to scrape workloads across all Projects, even if Network Policies are defined via Project Network Isolation. It also provides limited access for Project Owners, Project Members, and other users so that they're unable to access data that they shouldn't have access to (i.e., being allowed to exec into pods, set up the ability to scrape namespaces outside of a given Project, etc.). + +1. Open the `System` project to check your namespaces: + + Click **Cluster > Projects/Namespaces** in the Rancher UI. This will display all of the namespaces in the `System` project: + + ![Select Projects-Namespaces](/img/cattle-monitoring-system.png) + +1. If you have an existing Monitoring V2 installation within the `cattle-monitoring-system` namespace, but that namespace is not in the `System` project, you may move the `cattle-monitoring-system` namespace into the `System` project or into another project of limited access. To do so, you may either: + + - Drag and drop the namespace into the `System` project or + - Select **⋮** to the right of the namespace, click **Move**, then choose `System` from the **Target Project** dropdown + + ![Move to a New Project](/img/move-to-new-project.png) + +### Configure rancher-monitoring to only watch for resources created by the Helm chart itself + +Since each Project Monitoring Stack will watch the other namespaces and collect additional custom workload metrics or dashboards already, it's recommended to configure the following settings on all selectors to ensure that the Cluster Prometheus Stack only monitors resources created by the Helm Chart itself: + +``` +matchLabels: + release: "rancher-monitoring" +``` + +The following selector fields are recommended to have this value: +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +Once this setting is turned on, you can always create ServiceMonitors or PodMonitors that are picked up by the Cluster Prometheus by adding the label `release: "rancher-monitoring"` to them, in which case they will be ignored by Project Monitoring Stacks automatically by default, even if the namespace in which those ServiceMonitors or PodMonitors reside in are not system namespaces. + +:::note + +If you don't want to allow users to be able to create ServiceMonitors and PodMonitors that aggregate into the Cluster Prometheus in Project namespaces, you can additionally set the namespaceSelectors on the chart to only target system namespaces (which must contain `cattle-monitoring-system` and `cattle-dashboards`, where resources are deployed into by default by rancher-monitoring; you will also need to monitor the `default` namespace to get apiserver metrics or create a custom ServiceMonitor to scrape apiserver metrics from the Service residing in the default namespace) to limit your Cluster Prometheus from picking up other Prometheus Operator CRs. In that case, it would be recommended to turn `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true` to allow you to define ServiceMonitors that can monitor non-system namespaces from within a system namespace. + +::: + +### Increase the CPU / memory limits of the Cluster Prometheus + +Depending on a cluster's setup, it's generally recommended to give a large amount of dedicated memory to the Cluster Prometheus to avoid restarts due to out-of-memory errors (OOMKilled) usually caused by churn created in the cluster that causes a large number of high cardinality metrics to be generated and ingested by Prometheus within one block of time. This is one of the reasons why the default Rancher Monitoring stack expects around 4GB of RAM to be able to operate in a normal-sized cluster. However, when introducing Project Monitoring Stacks that are all sending `/federate` requests to the same Cluster Prometheus and are reliant on the Cluster Prometheus being "up" to federate that system data on their namespaces, it's even more important that the Cluster Prometheus has an ample amount of CPU / memory assigned to it to prevent an outage that can cause data gaps across all Project Prometheis in the cluster. + +:::note + +There are no specific recommendations on how much memory the Cluster Prometheus should be configured with since it depends entirely on the user's setup (namely the likelihood of encountering a high churn rate and the scale of metrics that could be generated at that time); it generally varies per setup. + +::: + +## Install the Prometheus Federator Application + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you want to install Prometheus Federator and click **Explore**. +1. Click **Apps -> Charts**. +1. Click the **Prometheus Federator** chart. +1. Click **Install**. +1. On the **Metadata** page, click **Next**. +1. In the **Namespaces** > **Project Release Namespace Project ID** field, the `System Project` is used as the default but can be overridden with another project with similarly [limited access](#ensure-the-cattle-monitoring-system-namespace-is-placed-into-the-system-project-or-a-similarly-locked-down-project-that-has-access-to-other-projects-in-the-cluster). Project IDs can be found with the following command run in the local upstream cluster: + + ```plain + kubectl get projects -A -o custom-columns="NAMESPACE":.metadata.namespace,"ID":.metadata.name,"NAME":.spec.displayName + ``` + +1. Click **Install**. + +**Result:** The Prometheus Federator app is deployed in the `cattle-monitoring-system` namespace. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md new file mode 100644 index 00000000000..8efe48f4ae8 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md @@ -0,0 +1,21 @@ +--- +title: Installing Project Monitors +--- + + + + + +Install **Project Monitors** in each project where you want to enable project monitoring. + +1. Click **☰ > Cluster Management**. + +1. On the **Clusters** page, go to the cluster where you want to enable monitoring and click **Explore**. + +1. Click **Monitoring > Project Monitors** on the left nav bar. Then click **Create** in the upper right. + + ![Project Monitors](/img/project-monitors.png) + +1. Select your project from the drop-down menu, then click **Create** again. + + ![Create Project Monitors](/img/create-project-monitors.png) diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md new file mode 100644 index 00000000000..d816f9b9e68 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md @@ -0,0 +1,17 @@ +--- +title: Setting up Prometheus Federator for a Workload +--- + + + + + +### Display CPU and Memory Metrics for a Workload + +Displaying CPU and memory metrics with Prometheus Federator is done the same way as with rancher-monitoring. For instructions, refer [here](../set-up-monitoring-for-workloads.md#display-cpu-and-memory-metrics-for-a-workload). + +### Setting up Metrics Beyond CPU and Memory + +Setting up metrics beyond CPU and memory with Prometheus Federator is done the same way as with rancher-monitoring. For instructions, refer [here](../set-up-monitoring-for-workloads.md#setting-up-metrics-beyond-cpu-and-memory). + + diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md new file mode 100644 index 00000000000..d5703cf33aa --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md @@ -0,0 +1,17 @@ +--- +title: Uninstall Prometheus Federator +--- + + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Installed Apps**. +1. Go to the `cattle-monitoring-system` namespace and check the boxes for `rancher-monitoring-crd` and `rancher-monitoring`. +1. Click **Delete**. +1. Confirm **Delete**. + +**Result:** `prometheus-federator` is uninstalled. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md new file mode 100644 index 00000000000..34d45c06e5e --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md @@ -0,0 +1,31 @@ +--- +title: Setting up Monitoring for a Workload +--- + + + + + +If you only need CPU and memory time series for the workload, you don't need to deploy a ServiceMonitor or PodMonitor because the monitoring application already collects metrics data on resource usage by default. + +The steps for setting up monitoring for workloads depend on whether you want basic metrics such as CPU and memory for the workload, or whether you want to scrape custom metrics from the workload. + +If you only need CPU and memory time series for the workload, you don't need to deploy a ServiceMonitor or PodMonitor because the monitoring application already collects metrics data on resource usage by default. The resource usage time series data is in Prometheus's local time series database. + +Grafana shows the data in aggregate, but you can see the data for the individual workload by using a PromQL query that extracts the data for that workload. Once you have the PromQL query, you can execute the query individually in the Prometheus UI and see the time series visualized there, or you can use the query to customize a Grafana dashboard to display the workload metrics. For examples of PromQL queries for workload metrics, see [this section.](../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md#workload-metrics) + +To set up custom metrics for your workload, you will need to set up an exporter and create a new ServiceMonitor custom resource to configure Prometheus to scrape metrics from your exporter. + +### Display CPU and Memory Metrics for a Workload + +By default, the monitoring application already scrapes CPU and memory. + +To get some fine-grained detail for a particular workload, you can customize a Grafana dashboard to display the metrics for a particular workload. + +### Setting up Metrics Beyond CPU and Memory + +For custom metrics, you will need to expose the metrics on your application in a format supported by Prometheus. + +Then we recommend that you should create a new ServiceMonitor custom resource. When this resource is created, the Prometheus custom resource will be automatically updated so that its scrape configuration includes the new custom metrics endpoint. Then Prometheus will begin scraping metrics from the endpoint. + +You can also create a PodMonitor to expose the custom metrics endpoint, but ServiceMonitors are more appropriate for the majority of use cases. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md new file mode 100644 index 00000000000..688d54a0506 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md @@ -0,0 +1,23 @@ +--- +title: Uninstall Monitoring +--- + + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Installed Apps**. +1. Go to the `cattle-monitoring-system` namespace and check the boxes for `rancher-monitoring-crd` and `rancher-monitoring`. +1. Click **Delete**. +1. Confirm **Delete**. + +**Result:** `rancher-monitoring` is uninstalled. + +:::note Persistent Grafana Dashboards: + +For users who are using Monitoring V2 v9.4.203 or below, uninstalling the Monitoring chart will delete the cattle-dashboards namespace, which will delete all persisted dashboards, unless the namespace is marked with the annotation `helm.sh/resource-policy: "keep"`. This annotation is added by default in Monitoring V2 v14.5.100+ but can be manually applied on the cattle-dashboards namespace before an uninstall if an older version of the Monitoring chart is currently installed onto your cluster. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md new file mode 100644 index 00000000000..bc213ee0637 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md @@ -0,0 +1,47 @@ +--- +title: Alertmanager Configuration +--- + + + + + +It is usually not necessary to directly edit the Alertmanager custom resource. For most use cases, you will only need to edit the Receivers and Routes to configure notifications. + +When Receivers and Routes are updated, the monitoring application will automatically update the Alertmanager custom resource to be consistent with those changes. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [this section.](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-how-alertmanager-works) + +::: + +## About the Alertmanager Custom Resource + +By default, Rancher Monitoring deploys a single Alertmanager onto a cluster that uses a default Alertmanager Config Secret. + +You may want to edit the Alertmanager custom resource if you would like to take advantage of advanced options that are not exposed in the Rancher UI forms, such as the ability to create a routing tree structure that is more than two levels deep. + +It is also possible to create more than one Alertmanager in a cluster, which may be useful if you want to implement namespace-scoped monitoring. In this case, you should manage the Alertmanager custom resources using the same underlying Alertmanager Config Secret. + +### Deeply Nested Routes + +While the Rancher UI only supports a routing tree that is two levels deep, you can configure more deeply nested routing structures by editing the Alertmanager YAML. + +### Multiple Alertmanager Replicas + +As part of the chart deployment options, you can opt to increase the number of replicas of the Alertmanager deployed onto your cluster. The replicas can all be managed using the same underlying Alertmanager Config Secret. + +This Secret should be updated or modified any time you want to: + +- Add in new notifiers or receivers +- Change the alerts that should be sent to specific notifiers or receivers +- Change the group of alerts that are sent out + +By default, you can either choose to supply an existing Alertmanager Config Secret (i.e. any Secret in the `cattle-monitoring-system` namespace) or allow Rancher Monitoring to deploy a default Alertmanager Config Secret onto your cluster. + +By default, the Alertmanager Config Secret created by Rancher will never be modified or deleted on an upgrade or uninstall of the `rancher-monitoring` chart. This restriction prevents users from losing or overwriting their alerting configuration when executing operations on the chart. + +For more information on what fields can be specified in the Alertmanager Config Secret, please look at the [Prometheus Alertmanager docs.](https://prometheus.io/docs/alerting/latest/alertmanager/) + +The full spec for the Alertmanager configuration file and what it takes in can be found [here.](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md new file mode 100644 index 00000000000..122316d018d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md @@ -0,0 +1,23 @@ +--- +title: Prometheus Configuration +--- + + + + + +It is usually not necessary to directly edit the Prometheus custom resource because the monitoring application automatically updates it based on changes to ServiceMonitors and PodMonitors. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information, see [this section.](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + +## About the Prometheus Custom Resource + +The Prometheus CR defines a desired Prometheus deployment. The Prometheus Operator observes the Prometheus CR. When the CR changes, the Prometheus Operator creates `prometheus-rancher-monitoring-prometheus`, a Prometheus deployment based on the CR configuration. + +The Prometheus CR specifies details such as rules and what Alertmanagers are connected to Prometheus. Rancher builds this CR for you. + +Monitoring V2 only supports one Prometheus per cluster. However, you might want to edit the Prometheus CR if you want to limit monitoring to certain namespaces. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md new file mode 100644 index 00000000000..b4259adf475 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md @@ -0,0 +1,84 @@ +--- +title: Configuring PrometheusRules +--- + + + + + +A PrometheusRule defines a group of Prometheus alerting and/or recording rules. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information, see [this section.](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + +### Creating PrometheusRules in the Rancher UI + +:::note Prerequisite: + +The monitoring application needs to be installed. + +::: + +To create rule groups in the Rancher UI, + +1. Go to the cluster where you want to create rule groups. Click **Monitoring > Advanced** and click **Prometheus Rules**. +1. Click **Create**. +1. Enter a **Group Name**. +1. Configure the rules. In Rancher's UI, we expect a rule group to contain either alert rules or recording rules, but not both. For help filling out the forms, refer to the configuration options below. +1. Click **Create**. + +**Result:** Alerts can be configured to send notifications to the receiver(s). + +### About the PrometheusRule Custom Resource + +When you define a Rule (which is declared within a RuleGroup in a PrometheusRule resource), the [spec of the Rule itself](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule) contains labels that are used by Alertmanager to figure out which Route should receive this Alert. For example, an Alert with the label `team: front-end` will be sent to all Routes that match on that label. + +Prometheus rule files are held in PrometheusRule custom resources. A PrometheusRule allows you to define one or more RuleGroups. Each RuleGroup consists of a set of Rule objects that can each represent either an alerting or a recording rule with the following fields: + +- The name of the new alert or record +- A PromQL expression for the new alert or record +- Labels that should be attached to the alert or record that identify it (e.g. cluster name or severity) +- Annotations that encode any additional important pieces of information that need to be displayed on the notification for an alert (e.g. summary, description, message, runbook URL, etc.). This field is not required for recording rules. + +For more information on what fields can be specified, please look at the [Prometheus Operator spec.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#prometheusrulespec) + +Use the label selector field `ruleSelector` in the Prometheus object to define the rule files that you want to be mounted into Prometheus. + +For examples, refer to the Prometheus documentation on [recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) and [alerting rules.](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) + +## Configuration + +### Rule Group + +| Field | Description | +|-------|----------------| +| Group Name | The name of the group. Must be unique within a rules file. | +| Override Group Interval | Duration in seconds for how often rules in the group are evaluated. | + + +### Alerting Rules + +[Alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) allow you to define alert conditions based on PromQL (Prometheus Query Language) expressions and to send notifications about firing alerts to an external service. + +| Field | Description | +|-------|----------------| +| Alert Name | The name of the alert. Must be a valid label value. | +| Wait To Fire For | Duration in seconds. Alerts are considered firing once they have been returned for this long. Alerts which have not yet fired for long enough are considered pending. | +| PromQL Expression | The PromQL expression to evaluate. Prometheus will evaluate the current value of this PromQL expression on every evaluation cycle and all resultant time series will become pending/firing alerts. For more information, refer to the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) or our [example PromQL expressions.](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md) | +| Labels | Labels to add or overwrite for each alert. | +| Severity | When enabled, labels are attached to the alert or record that identify it by the severity level. | +| Severity Label Value | Critical, warning, or none | +| Annotations | Annotations are a set of informational labels that can be used to store longer additional information, such as alert descriptions or runbook links. A [runbook](https://en.wikipedia.org/wiki/Runbook) is a set of documentation about how to handle alerts. The annotation values can be [templated.](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#templating) | + +### Recording Rules + +[Recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) allow you to precompute frequently needed or computationally expensive PromQL (Prometheus Query Language) expressions and save their result as a new set of time series. + +| Field | Description | +|-------|----------------| +| Time Series Name | The name of the time series to output to. Must be a valid metric name. | +| PromQL Expression | The PromQL expression to evaluate. Prometheus will evaluate the current value of this PromQL expression on every evaluation cycle and the result will be recorded as a new set of time series with the metric name as given by 'record'. For more information about expressions, refer to the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) or our [example PromQL expressions.](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md) | +| Labels | Labels to add or overwrite before storing the result. | diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md new file mode 100644 index 00000000000..90ba4bd4e8c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md @@ -0,0 +1,111 @@ +--- +title: Opening Ports with firewalld +--- + + + + + +> We recommend disabling firewalld. For Kubernetes 1.19.x and higher, firewalld must be turned off. + +Some distributions of Linux [derived from RHEL,](https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux#Rebuilds) including Oracle Linux, may have default firewall rules that block communication with Helm. + +For example, one Oracle Linux image in AWS has REJECT rules that stop Helm from communicating with Tiller: + +``` +Chain INPUT (policy ACCEPT) +target prot opt source destination +ACCEPT all -- anywhere anywhere state RELATED,ESTABLISHED +ACCEPT icmp -- anywhere anywhere +ACCEPT all -- anywhere anywhere +ACCEPT tcp -- anywhere anywhere state NEW tcp dpt:ssh +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain FORWARD (policy ACCEPT) +target prot opt source destination +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain OUTPUT (policy ACCEPT) +target prot opt source destination +``` + +You can check the default firewall rules with this command: + +``` +sudo iptables --list +``` + +This section describes how to use `firewalld` to apply the [firewall port rules](../../pages-for-subheaders/installation-requirements.md#port-requirements) for nodes in a high-availability Rancher server cluster. + +## Prerequisite + +Install v7.x or later ofv`firewalld`: + +``` +yum install firewalld +systemctl start firewalld +systemctl enable firewalld +``` + +## Applying Firewall Port Rules + +In the Rancher high-availability installation instructions, the Rancher server is set up on three nodes that have all three Kubernetes roles: etcd, controlplane, and worker. If your Rancher server nodes have all three roles, run the following commands on each node: + +``` +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` +If your Rancher server nodes have separate roles, use the following commands based on the role of the node: + +``` +# For etcd nodes, run the following commands: +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp + +# For control plane nodes, run the following commands: +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp + +# For worker nodes, run the following commands: +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` + +After the `firewall-cmd` commands have been run on a node, use the following command to enable the firewall rules: + +``` +firewall-cmd --reload +``` + +**Result:** The firewall is updated so that Helm can communicate with the Rancher server nodes. diff --git a/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md new file mode 100644 index 00000000000..e024f1dd779 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md @@ -0,0 +1,43 @@ +--- +title: Tuning etcd for Large Installations +--- + + + + + +When running larger Rancher installations with 15 or more clusters it is recommended to increase the default keyspace for etcd from the default 2GB. The maximum setting is 8GB and the host should have enough RAM to keep the entire dataset in memory. When increasing this value you should also increase the size of the host. The keyspace size can also be adjusted in smaller installations if you anticipate a high rate of change of pods during the garbage collection interval. + +The etcd data set is automatically cleaned up on a five minute interval by Kubernetes. There are situations, e.g. deployment thrashing, where enough events could be written to etcd and deleted before garbage collection occurs and cleans things up causing the keyspace to fill up. If you see `mvcc: database space exceeded` errors, in the etcd logs or Kubernetes API server logs, you should consider increasing the keyspace size. This can be accomplished by setting the [quota-backend-bytes](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) setting on the etcd servers. + +### Example: This snippet of the RKE cluster.yml file increases the keyspace size to 5GB + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + quota-backend-bytes: 5368709120 +``` + +## Scaling etcd disk performance + +You can follow the recommendations from [the etcd docs](https://etcd.io/docs/v3.4.0/tuning/#disk) on how to tune the disk priority on the host. + +Additionally, to reduce IO contention on the disks for etcd, you can use a dedicated device for the data and wal directory. Based on etcd best practices, mirroring RAID configurations are unnecessary because etcd replicates data between the nodes in the cluster. You can use striping RAID configurations to increase available IOPS. + +To implement this solution in an RKE cluster, the `/var/lib/etcd/data` and `/var/lib/etcd/wal` directories will need to have disks mounted and formatted on the underlying host. In the `extra_args` directive of the `etcd` service, you must include the `wal_dir` directory. Without specifying the `wal_dir`, etcd process will try to manipulate the underlying `wal` mount with insufficient permissions. + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + data-dir: '/var/lib/rancher/etcd/data/' + wal-dir: '/var/lib/rancher/etcd/wal/wal_dir' + extra_binds: + - '/var/lib/etcd/data:/var/lib/rancher/etcd/data' + - '/var/lib/etcd/wal:/var/lib/rancher/etcd/wal' +``` diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/add-users-to-projects.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/add-users-to-projects.md new file mode 100644 index 00000000000..d99e7c18120 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/add-users-to-projects.md @@ -0,0 +1,66 @@ +--- +title: Adding Users to Projects +--- + + + + + +If you want to provide a user with access and permissions to _specific_ projects and resources within a cluster, assign the user a project membership. + +You can add members to a project as it is created, or add them to an existing project. + +:::tip + +Want to provide a user with access to _all_ projects within a cluster? See [Adding Cluster Members](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) instead. + +::: + +### Adding Members to a New Project + +You can add members to a project as you create it (recommended if possible). For details on creating a new project, refer to the [cluster administration section.](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) + +### Adding Members to an Existing Project + +Following project creation, you can add users as project members so that they can access its resources. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add members to a project and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the project where you want to add members. Next to the **Create Namespace** button above the project name, click **☰**. Select **Edit Config**. +1. In the **Members** tab, click **Add**. +1. Search for the user or group that you want to add to the project. + + If external authentication is configured: + + - Rancher returns users from your external authentication source as you type. + + - A drop-down allows you to add groups instead of individual users. The dropdown only lists groups that you, the logged in user, are included in. + + :::note + + If you are logged in as a local user, external users do not display in your search results. + + ::: + +1. Assign the user or group **Project** roles. + + [What are Project Roles?](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::note Notes: + + - Users assigned the `Owner` or `Member` role for a project automatically inherit the `namespace creation` role. However, this role is a [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole), meaning its scope extends to all projects in the cluster. Therefore, users explicitly assigned the `Owner` or `Member` role for a project can create or delete namespaces in other projects they're assigned to, even with only the `Read Only` role assigned. + + - By default, the Rancher role of `project-member` inherits from the `Kubernetes-edit` role, and the `project-owner` role inherits from the `Kubernetes-admin` role. As such, both `project-member` and `project-owner` roles will allow for namespace management, including the ability to create and delete namespaces. + + - For `Custom` roles, you can modify the list of individual roles available for assignment. + + - To add roles to the list, [Add a Custom Role](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md). + - To remove roles from the list, [Lock/Unlock Roles](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md). + + ::: + +**Result:** The chosen users are added to the project. + +- To revoke project membership, select the user and click **Delete**. This action deletes membership, not the user. +- To modify a user's roles in the project, delete them from the project, and then re-add them with modified roles. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md new file mode 100644 index 00000000000..8d265e034f8 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md @@ -0,0 +1,46 @@ +--- +title: Cluster Drivers +--- + + + + + +Cluster drivers are used to create clusters in a [hosted Kubernetes provider](../../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md), such as Google GKE. The availability of which cluster driver to display when creating clusters is defined by the cluster driver's status. Only `active` cluster drivers will be displayed as an option for creating clusters. By default, Rancher is packaged with several existing cloud provider cluster drivers, but you can also add custom cluster drivers to Rancher. + +If there are specific cluster drivers that you do not want to show your users, you may deactivate those cluster drivers within Rancher and they will not appear as an option for cluster creation. + +### Managing Cluster Drivers + +:::note Prerequisites: + +To create, edit, or delete cluster drivers, you need _one_ of the following permissions: + +- [Administrator Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md) +- [Custom Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md#custom-global-permissions) with the [Manage Cluster Drivers](../manage-role-based-access-control-rbac/global-permissions.md) role assigned. + +::: + +## Activating/Deactivating Cluster Drivers + +By default, Rancher only activates drivers for the most popular cloud providers, Google GKE, Amazon EKS and Azure AKS. If you want to show or hide any node driver, you can change its status. + +1. In the upper left corner, click **☰ > Cluster Management**. + +2. In the left navigation menu, click **Drivers**. + +3. On the **Cluster Drivers** tab, select the driver that you wish to activate or deactivate and click **⋮ > Activate** or **⋮ > Deactivate**. + +## Adding Custom Cluster Drivers + +If you want to use a cluster driver that Rancher doesn't support out-of-the-box, you can add the provider's driver in order to start using them to create _hosted_ kubernetes clusters. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **Drivers**. +1. On the **Cluster Drivers** tab, click **Add Cluster Driver**. +1. Complete the **Add Cluster Driver** form. Then click **Create**. + + +### Developing your own Cluster Driver + +In order to develop cluster driver to add to Rancher, please refer to our [example](https://github.com/rancher-plugins/kontainer-engine-driver-example). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md new file mode 100644 index 00000000000..44256b18da0 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md @@ -0,0 +1,45 @@ +--- +title: Node Drivers +--- + + + + + +Node drivers are used to provision hosts, which Rancher uses to launch and manage Kubernetes clusters. A node driver is the same as a [Docker Machine driver](https://docs.docker.com/machine/drivers/). The availability of which node driver to display when creating node templates is defined based on the node driver's status. Only `active` node drivers will be displayed as an option for creating node templates. By default, Rancher is packaged with many existing Docker Machine drivers, but you can also create custom node drivers to add to Rancher. + +If there are specific node drivers that you don't want to show to your users, you would need to de-activate these node drivers. + +#### Managing Node Drivers + +:::note Prerequisites: + +To create, edit, or delete drivers, you need _one_ of the following permissions: + +- [Administrator Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md) +- [Custom Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md#custom-global-permissions) with the [Manage Node Drivers](../manage-role-based-access-control-rbac/global-permissions.md) role assigned. + +::: + +## Activating/Deactivating Node Drivers + +By default, Rancher only activates drivers for the most popular cloud providers, Amazon EC2, Azure, DigitalOcean and vSphere. If you want to show or hide any node driver, you can change its status. + +1. In the upper left corner, click **☰ > Cluster Management**. + +2. In the left navigation menu, click **Drivers**. + +2. On the **Node Drivers** tab, select the driver that you wish to activate or deactivate and click **⋮ > Activate** or **⋮ > Deactivate**. + +## Adding Custom Node Drivers + +If you want to use a node driver that Rancher doesn't support out-of-the-box, you can add that provider's driver in order to start using them to create node templates and eventually node pools for your Kubernetes cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **Drivers**. +1. On **Node Drivers** tab, click **Add Node Driver**. +1. Complete the **Add Node Driver** form. Then click **Create**. + +### Developing your own node driver + +Node drivers are implemented with [Docker Machine](https://docs.docker.com/machine/). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md new file mode 100644 index 00000000000..c12a1f03ef0 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md @@ -0,0 +1,68 @@ +--- +title: Access and Sharing +--- + + + + + +If you are an RKE template owner, you can share it with users or groups of users, who can then use the template to create clusters. + +Since RKE templates are specifically shared with users and groups, owners can share different RKE templates with different sets of users. + +When you share a template, each user can have one of two access levels: + +- **Owner:** This user can update, delete, and share the templates that they own. The owner can also share the template with other users. +- **User:** These users can create clusters using the template. They can also upgrade those clusters to new revisions of the same template. When you share a template as **Make Public (read-only),** all users in your Rancher setup have the User access level for the template. + +If you create a template, you automatically become an owner of that template. + +If you want to delegate responsibility for updating the template, you can share ownership of the template. For details on how owners can modify templates, refer to the [documentation about revising templates.](manage-rke1-templates.md) + +There are several ways to share templates: + +- Add users to a new RKE template during template creation +- Add users to an existing RKE template +- Make the RKE template public, sharing it with all users in the Rancher setup +- Share template ownership with users who are trusted to modify the template + +### Sharing Templates with Specific Users or Groups + +To allow users or groups to create clusters using your template, you can give them the basic **User** access level for the template. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the template that you want to share and click the **⋮ > Edit**. +1. In the **Share Template** section, click on **Add Member**. +1. Search in the **Name** field for the user or group you want to share the template with. +1. Choose the **User** access type. +1. Click **Save**. + +**Result:** The user or group can create clusters using the template. + +### Sharing Templates with All Users + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the template that you want to share and click the **⋮ > Edit**. +1. Under **Share Template,** check the box for **Make Public (read-only)**. +1. Click **Save**. + +**Result:** All users in the Rancher setup can create clusters using the template. + +### Sharing Ownership of Templates + +If you are the creator of a template, you might want to delegate responsibility for maintaining and updating a template to another user or group. + +In that case, you can give users the Owner access type, which allows another user to update your template, delete it, or share access to it with other users. + +To give Owner access to a user or group, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the RKE template that you want to share and click the **⋮ > Edit**. +1. Under **Share Template**, click on **Add Member** and search in the **Name** field for the user or group you want to share the template with. +1. In the **Access Type** field, click **Owner**. +1. Click **Save**. + +**Result:** The user or group has the Owner access type, and can modify, share, or delete the template. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md new file mode 100644 index 00000000000..1818e9076ff --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md @@ -0,0 +1,63 @@ +--- +title: Applying Templates +--- + + + + + +You can create a cluster from an RKE template that you created, or from a template that has been [shared with you.](access-or-share-templates.md) + +RKE templates can be applied to new clusters. + +You can [save the configuration of an existing cluster as an RKE template.](#converting-an-existing-cluster-to-use-an-rke-template) Then the cluster's settings can only be changed if the template is updated. + +You can't change a cluster to use a different RKE template. You can only update the cluster to a new revision of the same template. + + +### Creating a Cluster from an RKE Template + +To add a cluster [hosted by an infrastructure provider](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) using an RKE template, use these steps: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create** and choose the infrastructure provider. +1. Provide the cluster name and node template details as usual. +1. To use an RKE template, under the **Cluster Options**, check the box for **Use an existing RKE template and revision**. +1. Choose an RKE template and revision from the dropdown menu. +1. Optional: You can edit any settings that the RKE template owner marked as **Allow User Override** when the template was created. If there are settings that you want to change, but don't have the option to, you will need to contact the template owner to get a new revision of the template. Then you will need to edit the cluster to upgrade it to the new revision. +1. Click **Create** to launch the cluster. + +### Updating a Cluster Created with an RKE Template + +When the template owner creates a template, each setting has a switch in the Rancher UI that indicates if users can override the setting. + +- If the setting allows a user override, you can update these settings in the cluster by [editing the cluster.](../../../../pages-for-subheaders/cluster-configuration.md) +- If the switch is turned off, you cannot change these settings unless the cluster owner creates a template revision that lets you override them. If there are settings that you want to change, but don't have the option to, you will need to contact the template owner to get a new revision of the template. + +If a cluster was created from an RKE template, you can edit the cluster to update the cluster to a new revision of the template. + +An existing cluster's settings can be [saved as an RKE template.](#converting-an-existing-cluster-to-use-an-rke-template) In that situation, you can also edit the cluster to update the cluster to a new revision of the template. + +:::note + +You can't change the cluster to use a different RKE template. You can only update the cluster to a new revision of the same template. + +::: + +### Converting an Existing Cluster to Use an RKE Template + +This section describes how to create an RKE template from an existing cluster. + +RKE templates cannot be applied to existing clusters, except if you save an existing cluster's settings as an RKE template. This exports the cluster's settings as a new RKE template, and also binds the cluster to that template. The result is that the cluster can only be changed if the [template is updated,](manage-rke1-templates.md#updating-a-template) and the cluster is upgraded to [use a newer version of the template.](manage-rke1-templates.md#upgrading-a-cluster-to-use-a-new-template-revision) + +To convert an existing cluster to use an RKE template, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster that will be converted to use an RKE template. Click **⋮ > Save as RKE Template**. +1. Enter a name for the template in the form that appears, and click **Create**. + +**Results:** + +- A new RKE template is created. +- The cluster is converted to use the new template. +- New clusters can be [created from the new template.](apply-templates.md#creating-a-cluster-from-an-rke-template) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md new file mode 100644 index 00000000000..eff8b84ee92 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md @@ -0,0 +1,61 @@ +--- +title: Template Creator Permissions +--- + + + + + +Administrators have the permission to create RKE templates, and only administrators can give that permission to other users. + +For more information on administrator permissions, refer to the [documentation on global permissions](../manage-role-based-access-control-rbac/global-permissions.md). + +## Giving Users Permission to Create Templates + +Templates can only be created by users who have the global permission **Create RKE Templates**. + +Administrators have the global permission to create templates, and only administrators can give that permission to other users. + +For information on allowing users to modify existing templates, refer to [Sharing Templates.](access-or-share-templates.md) + +Administrators can give users permission to create RKE templates in two ways: + +- By editing the permissions of an [individual user](#allowing-a-user-to-create-templates) +- By changing the [default permissions of new users](#allowing-new-users-to-create-templates-by-default) + +### Allowing a User to Create Templates + +An administrator can individually grant the role **Create RKE Templates** to any existing user by following these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Choose the user you want to edit and click **⋮ > Edit Config**. +1. In the **Built-in** section, check the box for **Create new RKE Cluster Templates** role along with any other roles the user should have. You may want to also check the box for **Create RKE Template Revisions**. +1. Click **Save**. + +**Result:** The user has permission to create RKE templates. + +### Allowing New Users to Create Templates by Default + +Alternatively, the administrator can give all new users the default permission to create RKE templates by following the following steps. This will not affect the permissions of existing users. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Go to the role named **Create new RKE Cluster Templates and click **⋮ > Edit Config**. +1. Select the option **Yes: Default role for new users**. +1. Click **Save**. +1. If you would like new users to also be able to create RKE template revisions, enable that role as default as well. + +**Result:** Any new user created in this Rancher installation will be able to create RKE templates. Existing users will not get this permission. + +### Revoking Permission to Create Templates + +Administrators can remove a user's permission to create templates with the following steps. Note: Administrators have full control over all resources regardless of whether fine-grained permissions are selected. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Choose the user you want to edit permissions for and click **⋮ > Edit Config**. +1. In the **Built-in** section, un-check the box for **Create RKE Templates** and **Create RKE Template Revisions,** if applicable. In this section, you can change the user back to a standard user, or give the user a different set of permissions. +1. Click **Save**. + +**Result:** The user cannot create RKE templates. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md new file mode 100644 index 00000000000..1758a8710ac --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md @@ -0,0 +1,47 @@ +--- +title: Template Enforcement +--- + + + + + +This section describes how template administrators can enforce templates in Rancher, restricting the ability of users to create clusters without a template. + +By default, any standard user in Rancher can create clusters. But when RKE template enforcement is turned on, + +- Only an administrator has the ability to create clusters without a template. +- All standard users must use an RKE template to create a new cluster. +- Standard users cannot create a cluster without using a template. + +Users can only create new templates if the administrator [gives them permission.](creator-permissions.md#allowing-a-user-to-create-templates) + +After a cluster is created with an RKE template, the cluster creator cannot edit settings that are defined in the template. The only way to change those settings after the cluster is created is to [upgrade the cluster to a new revision](apply-templates.md#updating-a-cluster-created-with-an-rke-template) of the same template. If cluster creators want to change template-defined settings, they would need to contact the template owner to get a new revision of the template. For details on how template revisions work, refer to the [documentation on revising templates.](manage-rke1-templates.md#updating-a-template) + +## Requiring New Clusters to Use an RKE Template + +You might want to require new clusters to use a template to ensure that any cluster launched by a [standard user](../manage-role-based-access-control-rbac/global-permissions.md) will use the Kubernetes and/or Rancher settings that are vetted by administrators. + +To require new clusters to use an RKE template, administrators can turn on RKE template enforcement with the following steps: + +1. Click **☰ > Global Settings**. +1. Go to the `cluster-template-enforcement` setting. Click **⋮ > Edit Setting**. +1. Set the value to **True** and click **Save**. + + :::note Important: + + When the admin sets the `cluster-template-enforcement` to True, they also need to share the `clusterTemplates` with users so that users can select one of these templates to create the cluster. + + ::: + +**Result:** All clusters provisioned by Rancher must use a template, unless the creator is an administrator. + +## Disabling RKE Template Enforcement + +To allow new clusters to be created without an RKE template, administrators can turn off RKE template enforcement with the following steps: + +1. Click **☰ > Global Settings**. +1. Go to the `cluster-template-enforcement` setting. Click **⋮ > Edit Setting**. +1. Set the value to **False** and click **Save**. + +**Result:** When clusters are provisioned by Rancher, they don't need to use a template. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md new file mode 100644 index 00000000000..4471ffd9300 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md @@ -0,0 +1,74 @@ +--- +title: Example Scenarios +--- + + + + + +These example scenarios describe how an organization could use templates to standardize cluster creation. + +- **Enforcing templates:** Administrators might want to [enforce one or more template settings for everyone](#enforcing-a-template-setting-for-everyone) if they want all new Rancher-provisioned clusters to have those settings. +- **Sharing different templates with different users:** Administrators might give [different templates to basic and advanced users,](#templates-for-basic-and-advanced-users) so that basic users have more restricted options and advanced users have more discretion when creating clusters. +- **Updating template settings:** If an organization's security and DevOps teams decide to embed best practices into the required settings for new clusters, those best practices could change over time. If the best practices change, [a template can be updated to a new revision](#updating-templates-and-clusters-created-with-them) and clusters created from the template can upgrade to the new version of the template. +- **Sharing ownership of a template:** When a template owner no longer wants to maintain a template, or wants to delegate ownership of the template, this scenario describes how [template ownership can be shared.](#allowing-other-users-to-control-and-share-a-template) + + +## Enforcing a Template Setting for Everyone + +Let's say there is an organization in which the administrators decide that all new clusters should be created with Kubernetes version 1.14. + +1. First, an administrator creates a template which specifies the Kubernetes version as 1.14 and marks all other settings as **Allow User Override**. +1. The administrator makes the template public. +1. The administrator turns on template enforcement. + +**Results:** + +- All Rancher users in the organization have access to the template. +- All new clusters created by [standard users](../manage-role-based-access-control-rbac/global-permissions.md) with this template will use Kubernetes 1.14 and they are unable to use a different Kubernetes version. By default, standard users don't have permission to create templates, so this template will be the only template they can use unless more templates are shared with them. +- All standard users must use a cluster template to create a new cluster. They cannot create a cluster without using a template. + +In this way, the administrators enforce the Kubernetes version across the organization, while still allowing end users to configure everything else. + +## Templates for Basic and Advanced Users + +Let's say an organization has both basic and advanced users. Administrators want the basic users to be required to use a template, while the advanced users and administrators create their clusters however they want. + +1. First, an administrator turns on [RKE template enforcement.](enforce-templates.md#requiring-new-clusters-to-use-an-rke-template) This means that every [standard user](../manage-role-based-access-control-rbac/global-permissions.md) in Rancher will need to use an RKE template when they create a cluster. +1. The administrator then creates two templates: + + - One template for basic users, with almost every option specified except for access keys + - One template for advanced users, which has most or all options has **Allow User Override** turned on + +1. The administrator shares the advanced template with only the advanced users. +1. The administrator makes the template for basic users public, so the more restrictive template is an option for everyone who creates a Rancher-provisioned cluster. + +**Result:** All Rancher users, except for administrators, are required to use a template when creating a cluster. Everyone has access to the restrictive template, but only advanced users have permission to use the more permissive template. The basic users are more restricted, while advanced users have more freedom when configuring their Kubernetes clusters. + +## Updating Templates and Clusters Created with Them + +Let's say an organization has a template that requires clusters to use Kubernetes v1.14. However, as time goes on, the administrators change their minds. They decide they want users to be able to upgrade their clusters to use newer versions of Kubernetes. + +In this organization, many clusters were created with a template that requires Kubernetes v1.14. Because the template does not allow that setting to be overridden, the users who created the cluster cannot directly edit that setting. + +The template owner has several options for allowing the cluster creators to upgrade Kubernetes on their clusters: + +- **Specify Kubernetes v1.15 on the template:** The template owner can create a new template revision that specifies Kubernetes v1.15. Then the owner of each cluster that uses that template can upgrade their cluster to a new revision of the template. This template upgrade allows the cluster creator to upgrade Kubernetes to v1.15 on their cluster. +- **Allow any Kubernetes version on the template:** When creating a template revision, the template owner can also mark the the Kubernetes version as **Allow User Override** using the switch near that setting on the Rancher UI. This will allow clusters that upgrade to this template revision to use any version of Kubernetes. +- **Allow the latest minor Kubernetes version on the template:** The template owner can also create a template revision in which the Kubernetes version is defined as **Latest v1.14 (Allows patch version upgrades)**. This means clusters that use that revision will be able to get patch version upgrades, but major version upgrades will not be allowed. + +## Allowing Other Users to Control and Share a Template + +Let's say Alice is a Rancher administrator. She owns an RKE template that reflects her organization's agreed-upon best practices for creating a cluster. + +Bob is an advanced user who can make informed decisions about cluster configuration. Alice trusts Bob to create new revisions of her template as the best practices get updated over time. Therefore, she decides to make Bob an owner of the template. + +To share ownership of the template with Bob, Alice [adds Bob as an owner of her template.](access-or-share-templates.md#sharing-ownership-of-templates) + +The result is that as a template owner, Bob is in charge of version control for that template. Bob can now do all of the following: + +- [Revise the template](manage-rke1-templates.md#updating-a-template) when the best practices change +- [Disable outdated revisions](manage-rke1-templates.md#disabling-a-template-revision) of the template so that no new clusters can be created with it +- [Delete the whole template](manage-rke1-templates.md#deleting-a-template) if the organization wants to go in a different direction +- [Set a certain revision as default](manage-rke1-templates.md#setting-a-template-revision-as-default) when users create a cluster with it. End users of the template will still be able to choose which revision they want to create the cluster with. +- [Share the template](access-or-share-templates.md) with specific users, make the template available to all Rancher users, or share ownership of the template with another user. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md new file mode 100644 index 00000000000..0b2b6f8b8eb --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md @@ -0,0 +1,73 @@ +--- +title: RKE Templates and Infrastructure +--- + + + + + +In Rancher, RKE templates are used to provision Kubernetes and define Rancher settings, while node templates are used to provision nodes. + +Therefore, even if RKE template enforcement is turned on, the end user still has flexibility when picking the underlying hardware when creating a Rancher cluster. The end users of an RKE template can still choose an infrastructure provider and the nodes they want to use. + +If you want to standardize the hardware in your clusters, use RKE templates conjunction with node templates or with a server provisioning tool such as Terraform. + +### Node Templates + +[Node templates](../../../../reference-guides/user-settings/manage-node-templates.md) are responsible for node configuration and node provisioning in Rancher. From your user profile, you can set up node templates to define which templates are used in each of your node pools. With node pools enabled, you can make sure you have the required number of nodes in each node pool, and ensure that all nodes in the pool are the same. + +### Terraform + +Terraform is a server provisioning tool. It uses infrastructure-as-code that lets you create almost every aspect of your infrastructure with Terraform configuration files. It can automate the process of server provisioning in a way that is self-documenting and easy to track in version control. + +This section focuses on how to use Terraform with the [Rancher 2 Terraform provider](https://www.terraform.io/docs/providers/rancher2/), which is a recommended option to standardize the hardware for your Kubernetes clusters. If you use the Rancher Terraform provider to provision hardware, and then use an RKE template to provision a Kubernetes cluster on that hardware, you can quickly create a comprehensive, production-ready cluster. + +Terraform allows you to: + +- Define almost any kind of infrastructure-as-code, including servers, databases, load balancers, monitoring, firewall settings, and SSL certificates +- Leverage catalog apps and multi-cluster apps +- Codify infrastructure across many platforms, including Rancher and major cloud providers +- Commit infrastructure-as-code to version control +- Easily repeat configuration and setup of infrastructure +- Incorporate infrastructure changes into standard development practices +- Prevent configuration drift, in which some servers become configured differently than others + +## How Does Terraform Work? + +Terraform is written in files with the extension `.tf`. It is written in HashiCorp Configuration Language, which is a declarative language that lets you define the infrastructure you want in your cluster, the cloud provider you are using, and your credentials for the provider. Then Terraform makes API calls to the provider in order to efficiently create that infrastructure. + +To create a Rancher-provisioned cluster with Terraform, go to your Terraform configuration file and define the provider as Rancher 2. You can set up your Rancher 2 provider with a Rancher API key. Note: The API key has the same permissions and access level as the user it is associated with. + +Then Terraform calls the Rancher API to provision your infrastructure, and Rancher calls the infrastructure provider. As an example, if you wanted to use Rancher to provision infrastructure on AWS, you would provide both your Rancher API key and your AWS credentials in the Terraform configuration file or in environment variables so that they could be used to provision the infrastructure. + +When you need to make changes to your infrastructure, instead of manually updating the servers, you can make changes in the Terraform configuration files. Then those files can be committed to version control, validated, and reviewed as necessary. Then when you run `terraform apply`, the changes would be deployed. + +## Tips for Working with Terraform + +- There are examples of how to provide most aspects of a cluster in the [documentation for the Rancher 2 provider.](https://www.terraform.io/docs/providers/rancher2/) + +- In the Terraform settings, you can install Docker Machine by using the Docker Machine node driver. + +- You can also modify auth in the Terraform provider. + +- You can reverse engineer how to do define a setting in Terraform by changing the setting in Rancher, then going back and checking your Terraform state file to see how it maps to the current state of your infrastructure. + +- If you want to manage Kubernetes cluster settings, Rancher settings, and hardware settings all in one place, use [Terraform modules](https://github.com/rancher/terraform-modules). You can pass a cluster configuration YAML file or an RKE template configuration file to a Terraform module so that the Terraform module will create it. In that case, you could use your infrastructure-as-code to manage the version control and revision history of both your Kubernetes cluster and its underlying hardware. + +## Tip for Creating CIS Benchmark Compliant Clusters + +This section describes one way that you can make security and compliance-related config files standard in your clusters. + +When you create a [CIS benchmark compliant cluster,](../../../../pages-for-subheaders/rancher-security.md) you have an encryption config file and an audit log config file. + +Your infrastructure provisioning system can write those files to disk. Then in your RKE template, you would specify where those files will be, then add your encryption config file and audit log config file as extra mounts to the `kube-api-server`. + +Then you would make sure that the `kube-api-server` flag in your RKE template uses your CIS-compliant config files. + +In this way, you can create flags that comply with the CIS benchmark. + +## Resources + +- [Terraform documentation](https://www.terraform.io/docs/) +- [Rancher2 Terraform provider documentation](https://www.terraform.io/docs/providers/rancher2/) +- [The RanchCast - Episode 1: Rancher 2 Terraform Provider](https://youtu.be/YNCq-prI8-8): In this demo, Director of Community Jason van Brackel walks through using the Rancher 2 Terraform Provider to provision nodes and create a custom cluster. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md new file mode 100644 index 00000000000..e9eac6fe7f6 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md @@ -0,0 +1,165 @@ +--- +title: Creating and Revising RKE Templates +--- + + + + + +This section describes how to manage RKE templates and revisions. You an create, share, update, and delete templates from the **Cluster Management** view under **RKE1 Configuration > RKE Templates**. + +Template updates are handled through a revision system. When template owners want to change or update a template, they create a new revision of the template. Individual revisions cannot be edited. However, if you want to prevent a revision from being used to create a new cluster, you can disable it. + +Template revisions can be used in two ways: to create a new cluster, or to upgrade a cluster that was created with an earlier version of the template. The template creator can choose a default revision, but when end users create a cluster, they can choose any template and any template revision that is available to them. After the cluster is created from a specific revision, it cannot change to another template, but the cluster can be upgraded to a newer available revision of the same template. + +The template owner has full control over template revisions, and can create new revisions to update the template, delete or disable revisions that should not be used to create clusters, and choose which template revision is the default. + + +### Prerequisites + +You can create RKE templates if you have the **Create RKE Templates** permission, which can be [given by an administrator.](creator-permissions.md) + +You can revise, share, and delete a template if you are an owner of the template. For details on how to become an owner of a template, refer to [the documentation on sharing template ownership.](access-or-share-templates.md#sharing-ownership-of-templates) + +### Creating a Template + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **RKE1 configuration > Node Templates**. +1. Click **Add Template**. +1. Provide a name for the template. An auto-generated name is already provided for the template' first version, which is created along with this template. +1. Optional: Share the template with other users or groups by [adding them as members.](access-or-share-templates.md#sharing-templates-with-specific-users-or-groups) You can also make the template public to share with everyone in the Rancher setup. +1. Then follow the form on screen to save the cluster configuration parameters as part of the template's revision. The revision can be marked as default for this template. + +**Result:** An RKE template with one revision is configured. You can use this RKE template revision later when you [provision a Rancher-launched cluster](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). After a cluster is managed by an RKE template, it cannot be disconnected and the option to uncheck **Use an existing RKE Template and Revision** will be unavailable. + +### Updating a Template + +When you update an RKE template, you are creating a revision of the existing template. Clusters that were created with an older version of the template can be updated to match the new revision. + +You can't edit individual revisions. Since you can't edit individual revisions of a template, in order to prevent a revision from being used, you can [disable it.](#disabling-a-template-revision) + +When new template revisions are created, clusters using an older revision of the template are unaffected. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the template that you want to edit and click the **⋮ > Edit**. +1. Edit the required information and click **Save**. +1. Optional: You can change the default revision of this template and also change who it is shared with. + +**Result:** The template is updated. To apply it to a cluster using an older version of the template, refer to the section on [upgrading a cluster to use a new revision of a template.](#upgrading-a-cluster-to-use-a-new-template-revision) + +### Deleting a Template + +When you no longer use an RKE template for any of your clusters, you can delete it. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **RKE1 configuration > RKE Templates**. +1. Go to the RKE template that you want to delete and click the **⋮ > Delete**. +1. Confirm the deletion. + +**Result:** The template is deleted. + +### Creating a Revision Based on the Default Revision + +You can clone the default template revision and quickly update its settings rather than creating a new revision from scratch. Cloning templates saves you the hassle of re-entering the access keys and other parameters needed for cluster creation. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the RKE template that you want to clone and click the **⋮ > New Revision from Default**. +1. Complete the rest of the form to create a new revision. + +**Result:** The RKE template revision is cloned and configured. + +### Creating a Revision Based on a Cloned Revision + +When creating new RKE template revisions from your user settings, you can clone an existing revision and quickly update its settings rather than creating a new one from scratch. Cloning template revisions saves you the hassle of re-entering the cluster parameters. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the template revision you want to clone. Then select **⋮ > Clone Revision**. +1. Complete the rest of the form. + +**Result:** The RKE template revision is cloned and configured. You can use the RKE template revision later when you provision a cluster. Any existing cluster using this RKE template can be upgraded to this new revision. + +### Disabling a Template Revision + +When you no longer want an RKE template revision to be used for creating new clusters, you can disable it. A disabled revision can be re-enabled. + +You can disable the revision if it is not being used by any cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the template revision you want to disable. Then select **⋮ > Disable**. + +**Result:** The RKE template revision cannot be used to create a new cluster. + +### Re-enabling a Disabled Template Revision + +If you decide that a disabled RKE template revision should be used to create new clusters, you can re-enable it. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the template revision you want to re-enable. Then select **⋮ > Enable**. + +**Result:** The RKE template revision can be used to create a new cluster. + +### Setting a Template Revision as Default + +When end users create a cluster using an RKE template, they can choose which revision to create the cluster with. You can configure which revision is used by default. + +To set an RKE template revision as default, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE templates**. +1. Go to the RKE template revision that should be default and click the **⋮ > Set as Default**. + +**Result:** The RKE template revision will be used as the default option when clusters are created with the template. + +### Deleting a Template Revision + +You can delete all revisions of a template except for the default revision. + +To permanently delete a revision, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE templates**. +1. Go to the RKE template revision that should be deleted and click the **⋮ > Delete**. + +**Result:** The RKE template revision is deleted. + +### Upgrading a Cluster to Use a New Template Revision + +:::note + +This section assumes that you already have a cluster that [has an RKE template applied.](apply-templates.md) + +This section also assumes that you have [updated the template that the cluster is using](#updating-a-template) so that a new template revision is available. + +::: + +To upgrade a cluster to use a new template revision, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you want to upgrade and click **⋮ > Edit Config**. +1. In the **Cluster Options** section, click the dropdown menu for the template revision, then select the new template revision. +1. Click **Save**. + +**Result:** The cluster is upgraded to use the settings defined in the new template revision. + +### Exporting a Running Cluster to a New RKE Template and Revision + +You can save an existing cluster's settings as an RKE template. + +This exports the cluster's settings as a new RKE template, and also binds the cluster to that template. The result is that the cluster can only be changed if the [template is updated,](#updating-a-template) and the cluster is upgraded to [use a newer version of the template](#upgrading-a-cluster-to-use-a-new-template-revision). + +To convert an existing cluster to use an RKE template, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that will be converted to use an RKE template and **⋮ > Save as RKE Template**. +1. Enter a name for the RKE template in the form that appears, and click **Create**. + +**Results:** + +- A new RKE template is created. +- The cluster is converted to use the new template. +- New clusters can be [created from the new template and revision.](apply-templates.md#creating-a-cluster-from-an-rke-template) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md new file mode 100644 index 00000000000..67c6a50edd8 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md @@ -0,0 +1,18 @@ +--- +title: Overriding Template Settings +--- + + + + + +When a user creates an RKE template, each setting in the template has a switch in the Rancher UI that indicates if users can override the setting. This switch marks those settings as **Allow User Override**. + +After a cluster is created with a template, end users can't update any of the settings defined in the template unless the template owner marked them as **Allow User Override**. However, if the template is [updated to a new revision](manage-rke1-templates.md) that changes the settings or allows end users to change them, the cluster can be upgraded to a new revision of the template and the changes in the new revision will be applied to the cluster. + +When any parameter is set as **Allow User Override** on the RKE template, it means that end users have to fill out those fields during cluster creation and they can edit those settings afterward at any time. + +The **Allow User Override** model of the RKE template is useful for situations such as: + +- Administrators know that some settings will need the flexibility to be frequently updated over time +- End users will need to enter their own access keys or secret keys, for example, cloud credentials or credentials for backup snapshots \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md new file mode 100644 index 00000000000..b24c4879071 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md @@ -0,0 +1,223 @@ +--- +title: Configure Active Directory (AD) +--- + + + + + +If your organization uses Microsoft Active Directory as central user repository, you can configure Rancher to communicate with an Active Directory server to authenticate users. This allows Rancher admins to control access to clusters and projects based on users and groups managed externally in the Active Directory, while allowing end-users to authenticate with their AD credentials when logging in to the Rancher UI. + +Rancher uses LDAP to communicate with the Active Directory server. The authentication flow for Active Directory is therefore the same as for the [OpenLDAP authentication](../../../../pages-for-subheaders/configure-openldap.md) integration. + +:::note + +Before you start, please familiarise yourself with the concepts of [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +::: + +## Prerequisites + +You'll need to create or obtain from your AD administrator a new AD user to use as service account for Rancher. This user must have sufficient permissions to perform LDAP searches and read attributes of users and groups under your AD domain. + +Usually a (non-admin) **Domain User** account should be used for this purpose, as by default such user has read-only privileges for most objects in the domain partition. + +Note however, that in some locked-down Active Directory configurations this default behaviour may not apply. In such case you will need to ensure that the service account user has at least **Read** and **List Content** permissions granted either on the Base OU (enclosing users and groups) or globally for the domain. + +:::note Using TLS? + +- If the certificate used by the AD server is self-signed or not from a recognized certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +- Upon an upgrade to v2.6.0, authenticating via Rancher against an active directory using TLS can fail if the certificates on the AD server do not support SAN attributes. This is a check enabled by default in Go v1.15. + + - The error received is "Error creating SSL connection: LDAP Result Code 200 "Network Error": x509: certificate relies on legacy Common Name field, use SANs or temporarily enable Common Name matching with GODEBUG=x509ignoreCN=0". + + - To resolve the error, update or replace the certificates on the AD server with new ones that support the SAN attribute. Alternatively, this error can be ignored by setting `GODEBUG=x509ignoreCN=0` as an environment variable to Rancher server container. + +::: + +## Configuration Steps +### Open Active Directory Configuration + +1. Log into the Rancher UI using the initial local `admin` account. +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **ActiveDirectory**. The **Authentication Provider: ActiveDirectory** form will be displayed. +1. Fill out the form. For help, refer to the details on configuration options below. +1. Click **Enable**. + +### Configure Active Directory Server Settings + +In the section titled `1. Configure an Active Directory server`, complete the fields with the information specific to your Active Directory server. Please refer to the following table for detailed information on the required values for each parameter. + +:::note + +If you are unsure about the correct values to enter in the user/group Search Base field, please refer to [Identify Search Base and Schema using ldapsearch](#annex-identify-search-base-and-schema-using-ldapsearch). + +::: + +**Table 1: AD Server parameters** + +| Parameter | Description | +|:--|:--| +| Hostname | Specify the hostname or IP address of the AD server | +| Port | Specify the port at which the Active Directory server is listening for connections. Unencrypted LDAP normally uses the standard port of 389, while LDAPS uses port 636.| +| TLS | Check this box to enable LDAP over SSL/TLS (commonly known as LDAPS).| +| Server Connection Timeout | The duration in number of seconds that Rancher waits before considering the AD server unreachable. | +| Service Account Username | Enter the username of an AD account with read-only access to your domain partition (see [Prerequisites](#prerequisites)). The username can be entered in NetBIOS format (e.g. "DOMAIN\serviceaccount") or UPN format (e.g. "serviceaccount@domain.com"). | +| Service Account Password | The password for the service account. | +| Default Login Domain | When you configure this field with the NetBIOS name of your AD domain, usernames entered without a domain (e.g. "jdoe") will automatically be converted to a slashed, NetBIOS logon (e.g. "LOGIN_DOMAIN\jdoe") when binding to the AD server. If your users authenticate with the UPN (e.g. "jdoe@acme.com") as username then this field **must** be left empty. | +| User Search Base | The Distinguished Name of the node in your directory tree from which to start searching for user objects. All users must be descendents of this base DN. For example: "ou=people,dc=acme,dc=com".| +| Group Search Base | If your groups live under a different node than the one configured under `User Search Base` you will need to provide the Distinguished Name here. Otherwise leave it empty. For example: "ou=groups,dc=acme,dc=com".| + +--- + +### Configure User/Group Schema + +In the section titled `2. Customize Schema` you must provide Rancher with a correct mapping of user and group attributes corresponding to the schema used in your directory. + +Rancher uses LDAP queries to search for and retrieve information about users and groups within the Active Directory. The attribute mappings configured in this section are used to construct search filters and resolve group membership. It is therefore paramount that the provided settings reflect the reality of your AD domain. + +:::note + +If you are unfamiliar with the schema used in your Active Directory domain, please refer to [Identify Search Base and Schema using ldapsearch](#annex-identify-search-base-and-schema-using-ldapsearch) to determine the correct configuration values. + +::: + +#### User Schema + +The table below details the parameters for the user schema section configuration. + +**Table 2: User schema configuration parameters** + +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for user objects in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Username Attribute | The user attribute whose value is suitable as a display name. | +| Login Attribute | The attribute whose value matches the username part of credentials entered by your users when logging in to Rancher. If your users authenticate with their UPN (e.g. "jdoe@acme.com") as username then this field must normally be set to `userPrincipalName`. Otherwise for the old, NetBIOS-style logon names (e.g. "jdoe") it's usually `sAMAccountName`. | +| User Member Attribute | The attribute containing the groups that a user is a member of. | +| Search Attribute | When a user enters text to add users or groups in the UI, Rancher queries the AD server and attempts to match users by the attributes provided in this setting. Multiple attributes can be specified by separating them with the pipe ("\|") symbol. To match UPN usernames (e.g. jdoe@acme.com) you should usually set the value of this field to `userPrincipalName`. | +| Search Filter | This filter gets applied to the list of users that is searched when Rancher attempts to add users to a site access list or tries to add members to clusters or projects. For example, a user search filter could be (|(memberOf=CN=group1,CN=Users,DC=testad,DC=rancher,DC=io)(memberOf=CN=group2,CN=Users,DC=testad,DC=rancher,DC=io)). Note: If the search filter does not use [valid AD search syntax,](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax) the list of users will be empty. | +| User Enabled Attribute | The attribute containing an integer value representing a bitwise enumeration of user account flags. Rancher uses this to determine if a user account is disabled. You should normally leave this set to the AD standard `userAccountControl`. | +| Disabled Status Bitmask | This is the value of the `User Enabled Attribute` designating a disabled user account. You should normally leave this set to the default value of "2" as specified in the Microsoft Active Directory schema (see [here](https://docs.microsoft.com/en-us/windows/desktop/adschema/a-useraccountcontrol#remarks)). | + +--- + +#### Group Schema + +The table below details the parameters for the group schema configuration. + +**Table 3: Group schema configuration parameters** + +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for group objects in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Name Attribute | The group attribute whose value is suitable for a display name. | +| Group Member User Attribute | The name of the **user attribute** whose format matches the group members in the `Group Member Mapping Attribute`. | +| Group Member Mapping Attribute | The name of the group attribute containing the members of a group. | +| Search Attribute | Attribute used to construct search filters when adding groups to clusters or projects. See description of user schema `Search Attribute`. | +| Search Filter | This filter gets applied to the list of groups that is searched when Rancher attempts to add groups to a site access list or tries to add groups to clusters or projects. For example, a group search filter could be (|(cn=group1)(cn=group2)). Note: If the search filter does not use [valid AD search syntax,](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax) the list of groups will be empty. | +| Group DN Attribute | The name of the group attribute whose format matches the values in the user attribute describing a the user's memberships. See `User Member Attribute`. | +| Nested Group Membership | This settings defines whether Rancher should resolve nested group memberships. Use only if your organization makes use of these nested memberships (i.e., you have groups that contain other groups as members. We advise avoiding nested groups when possible to avoid potential performance issues when there is a large amount of nested memberships). | + +--- + +### Test Authentication + +Once you have completed the configuration, proceed by testing the connection to the AD server **using your AD admin account**. If the test is successful, authentication with the configured Active Directory will be enabled implicitly with the account you test with set as admin. + +:::note + +The AD user pertaining to the credentials entered in this step will be mapped to the local principal account and assigned administrator privileges in Rancher. You should therefore make a conscious decision on which AD account you use to perform this step. + +::: + +1. Enter the **username** and **password** for the AD account that should be mapped to the local principal account. +2. Click **Authenticate with Active Directory** to finalise the setup. + +**Result:** + +- Active Directory authentication has been enabled. +- You have been signed into Rancher as administrator using the provided AD credentials. + +:::note + +You will still be able to login using the locally configured `admin` account and password in case of a disruption of LDAP services. + +::: + +## Annex: Identify Search Base and Schema using ldapsearch + +In order to successfully configure AD authentication it is crucial that you provide the correct configuration pertaining to the hierarchy and schema of your AD server. + +The [`ldapsearch`](https://manpages.ubuntu.com/manpages/kinetic/en/man1/ldapsearch.1.html) tool allows you to query your AD server to learn about the schema used for user and group objects. + +For the purpose of the example commands provided below we will assume: + +- The Active Directory server has a hostname of `ad.acme.com` +- The server is listening for unencrypted connections on port `389` +- The Active Directory domain is `acme` +- You have a valid AD account with the username `jdoe` and password `secret` + +### Identify Search Base + +First we will use `ldapsearch` to identify the Distinguished Name (DN) of the parent node(s) for users and groups: + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "dc=acme,dc=com" -s sub "sAMAccountName=jdoe" +``` + +This command performs an LDAP search with the search base set to the domain root (`-b "dc=acme,dc=com"`) and a filter targeting the user account (`sAMAccountNam=jdoe`), returning the attributes for said user: + +![](/img/ldapsearch-user.png) + +Since in this case the user's DN is `CN=John Doe,CN=Users,DC=acme,DC=com` [5], we should configure the **User Search Base** with the parent node DN `CN=Users,DC=acme,DC=com`. + +Similarly, based on the DN of the group referenced in the **memberOf** attribute [4], the correct value for the **Group Search Base** would be the parent node of that value, i.e., `OU=Groups,DC=acme,DC=com`. + +### Identify User Schema + +The output of the above `ldapsearch` query also allows to determine the correct values to use in the user schema configuration: + +- `Object Class`: **person** [1] +- `Username Attribute`: **name** [2] +- `Login Attribute`: **sAMAccountName** [3] +- `User Member Attribute`: **memberOf** [4] + +:::note + +If the AD users in our organization were to authenticate with their UPN (e.g. jdoe@acme.com) instead of the short logon name, then we would have to set the `Login Attribute` to **userPrincipalName** instead. + +::: + +We'll also set the `Search Attribute` parameter to **sAMAccountName|name**. That way users can be added to clusters/projects in the Rancher UI either by entering their username or full name. + +### Identify Group Schema + +Next, we'll query one of the groups associated with this user, in this case `CN=examplegroup,OU=Groups,DC=acme,DC=com`: + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "ou=groups,dc=acme,dc=com" \ +-s sub "CN=examplegroup" +``` + +This command will inform us on the attributes used for group objects: + +![](/img/ldapsearch-group.png) + +Again, this allows us to determine the correct values to enter in the group schema configuration: + +- `Object Class`: **group** [1] +- `Name Attribute`: **name** [2] +- `Group Member Mapping Attribute`: **member** [3] +- `Search Attribute`: **sAMAccountName** [4] + +Looking at the value of the **member** attribute, we can see that it contains the DN of the referenced user. This corresponds to the **distinguishedName** attribute in our user object. Accordingly will have to set the value of the `Group Member User Attribute` parameter to this attribute. + +In the same way, we can observe that the value in the **memberOf** attribute in the user object corresponds to the **distinguishedName** [5] of the group. We therefore need to set the value for the `Group DN Attribute` parameter to this attribute. + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the Active Directory server, first double-check the credentials entered for the service account as well as the search base configuration. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md new file mode 100644 index 00000000000..d6f2884eab4 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md @@ -0,0 +1,330 @@ +--- +title: Configure Azure AD +--- + + + + + +## Microsoft Graph API + +Microsoft Graph API is now the flow through which you will set up Azure AD. The below sections will assist [new users](#new-user-setup) in configuring Azure AD with a new instance as well as assist existing Azure app owners in [migrating to the new flow](#migrating-from-azure-ad-graph-api-to-microsoft-graph-api). + +The Microsoft Graph API flow in Rancher is constantly evolving. We recommend that you use the latest patched version of 2.7, as it is still in active development and will continue to receive new features and improvements. + +### New User Setup + +If you have an instance of Active Directory (AD) hosted in Azure, you can configure Rancher to allow your users to log in using their AD accounts. Configuration of Azure AD external authentication requires you to make configurations in both Azure and Rancher. + +:::note Notes + +- Azure AD integration only supports Service Provider initiated logins. +- Most of this procedure takes place from the [Microsoft Azure Portal](https://portal.azure.com/). + +::: + +#### Azure Active Directory Configuration Outline + +Configuring Rancher to allow your users to authenticate with their Azure AD accounts involves multiple procedures. Review the outline below before getting started. + +:::tip + +Before you start, open two browser tabs: one for Rancher, and one for the Azure portal. This will help with copying and pasting configuration values from the portal to Rancher. + +::: + + +#### 1. Register Rancher with Azure + +Before enabling Azure AD within Rancher, you must register Rancher with Azure. + +1. Log in to [Microsoft Azure](https://portal.azure.com/) as an administrative user. Configuration in future steps requires administrative access rights. + +1. Use search to open the **App registrations** service. + +1. Click **New registration** and complete the form. + + ![New App Registration](/img/new-app-registration.png) + + 1. Enter a **Name** (something like `Rancher`). + + + 1. From **Supported account types**, select "Accounts in this organizational directory only (AzureADTest only - Single tenant)" This corresponds to the legacy app registration options. + + :::note + + In the updated Azure portal, Redirect URIs are synonymous with Reply URLs. In order to use Azure AD with Rancher, you must whitelist Rancher with Azure (previously done through Reply URLs). Therefore, you must ensure to fill in the Redirect URI with your Rancher server URL, to include the verification path as listed below. + + ::: + + 1. In the [**Redirect URI**](https://docs.microsoft.com/en-us/azure/active-directory/develop/reply-url) section, make sure **Web** is selected from the dropdown and enter the URL of your Rancher Server in the text box next to the dropdown. This Rancher server URL should be appended with the verification path: `/verify-auth-azure`. + + :::tip + + You can find your personalized Azure Redirect URI (reply URL) in Rancher on the Azure AD Authentication page (Global View > Authentication > Web). + + ::: + + 1. Click **Register**. + +:::note + +It can take up to five minutes for this change to take affect, so don't be alarmed if you can't authenticate immediately after Azure AD configuration. + +::: + +#### 2. Create a new client secret + +From the Azure portal, create a client secret. Rancher will use this key to authenticate with Azure AD. + +1. Use search to open **App registrations** services. Then open the entry for Rancher that you created in the last procedure. + + ![Open Rancher Registration](/img/open-rancher-app-reg.png) + +1. From the navigation pane, click **Certificates & secrets**. + +1. Click **New client secret**. + ![Create new client secret](/img/new-client-secret.png) +1. Enter a **Description** (something like `Rancher`). +1. Select the duration from the options under **Expires**. This drop-down menu sets the expiration date for the key. Shorter durations are more secure, but require you to create a new key more frequently. +Note that users won't be able to log in to Rancher if it detects that the application secret has expired. To avoid this problem, rotate the secret in Azure and update it in Rancher before it expires. +1. Click **Add** (you don't need to enter a value—it will automatically populate after you save). + + +1. You'll enter this key into the Rancher UI later as your **Application Secret**. Since you won't be able to access the key value again within the Azure UI, keep this window open for the rest of the setup process. + +#### 3. Set Required Permissions for Rancher + +Next, set API permissions for Rancher within Azure. + +:::caution + +Ensure that you set Application permissions, and *not* Delegated permissions. Otherwise, you won't be able to login to Azure AD. + +::: + +1. From the navigation pane on, select **API permissions**. + +1. Click **Add a permission**. + +1. From the Microsoft Graph API, select the following **Application Permissions**: `Directory.Read.All` + + ![Select API Permissions](/img/api-permissions.png) + +1. Return to **API permissions** in the nav bar. From there, click **Grant admin consent**. Then click **Yes**. The app's permissions should look like the following: + +![Open Required Permissions](/img/select-req-permissions.png) + +:::note + +Rancher doesn't validate the permissions you grant to the app in Azure. You're free to try any permissions you want, as long as they allow Rancher to work with AD users and groups. + +Specifically, Rancher needs permissions that allow the following actions: +- Get a user. +- List all users. +- List groups of which a given user is a member. +- Get a group. +- List all groups. + +Rancher performs these actions either to log in a user or to run a user/group search. Keep in mind that the permissions must be of type `Application`. + +Here are a few examples of permission combinations that satisfy Rancher's needs: +- `Directory.Read.All` +- `User.Read.All` and `GroupMember.Read.All` +- `User.Read.All` and `Group.Read.All` + +::: + +#### 4. Copy Azure Application Data + +![Application ID](/img/app-configuration.png) + +1. Obtain your Rancher **Tenant ID**. + + 1. Use search to open **App registrations**. + + 1. Find the entry you created for Rancher. + + 1. Copy the **Directory ID** and paste it into Rancher as your **Tenant ID**. + +1. Obtain your Rancher **Application (Client) ID**. + + 1. If you aren't already there, use search to open **App registrations**. + + 1. In **Overview**, find the entry you created for Rancher. + + 1. Copy the **Application (Client) ID** and paste it into Rancher as your **Application ID**. + +1. In most cases, your endpoint options will either be [Standard](#global) or [China](#china). For either of these options, you only need to enter the **Tenant ID**, **Application ID**, and **Application Secret**. + +![Standard Endpoint Options](/img/tenant-application-id-secret.png) + +**For Custom Endpoints:** + +:::caution + +Custom Endpoints are not tested or fully supported by Rancher. + +::: + +You'll also need to manually enter the Graph, Token, and Auth Endpoints. + +- From App registrations, click Endpoints: + +![Click Endpoints](/img/endpoints.png) + +- The following endpoints will be your Rancher endpoint values. Make sure to use the v1 version of these endpoints: + - **Microsoft Graph API endpoint** (Graph Endpoint) + - **OAuth 2.0 token endpoint (v1)** (Token Endpoint) + - **OAuth 2.0 authorization endpoint (v1)** (Auth Endpoint) + +#### 5. Configure Azure AD in Rancher + +To complete configuration, enter information about your AD instance in the Rancher UI. + +1. Log into Rancher. + +1. In the top left corner, click **☰ > Users & Authentication**. + +1. In the left navigation menu, click **Auth Provider**. + +1. Click **AzureAD**. + +1. Complete the **Configure Azure AD Account** form using the information you copied while completing [Copy Azure Application Data](#4-copy-azure-application-data). + + :::caution + + The Azure AD account will be granted administrator privileges, since its details will be mapped to the Rancher local principal account. Make sure that this level of privilege is appropriate before you continue. + + ::: + + **For Standard or China Endpoints:** + + The following table maps the values you copied in the Azure portal to the fields in Rancher: + + | Rancher Field | Azure Value | + | ------------------ | ------------------------------------- | + | Tenant ID | Directory ID | + | Application ID | Application ID | + | Application Secret | Key Value | + | Endpoint | https://login.microsoftonline.com/ | + + **For Custom Endpoints:** + + The following table maps your custom config values to Rancher fields: + + | Rancher Field | Azure Value | + | ------------------ | ------------------------------------- | + | Graph Endpoint | Microsoft Graph API Endpoint | + | Token Endpoint | OAuth 2.0 Token Endpoint | + | Auth Endpoint | OAuth 2.0 Authorization Endpoint | + + **Important:** When entering the Graph Endpoint in a custom config, remove the tenant ID from the URL: + + https://graph.microsoft.com/abb5adde-bee8-4821-8b03-e63efdc7701c + +1. Click **Enable**. + +**Result:** Azure Active Directory authentication is configured. + + +### Migrating from Azure AD Graph API to Microsoft Graph API + +Since the [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) is deprecated and slated to retire in June 2023, admins should update their Azure AD App to use the [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api) in Rancher. +This needs to be done well in advance of the endpoint being retired. +If Rancher is still configured to use the Azure AD Graph API when it is retired, users may not be able to log into Rancher using Azure AD. + +#### Updating Endpoints in the Rancher UI + +:::caution + +Admins should create a [Rancher backup](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) before they commit to the endpoint migration described below. + +::: + +1. [Update](#3-set-required-permissions-for-rancher) the permissions of your Azure AD app registration. This is critical. + +1. Log into Rancher. + +1. In the Rancher UI homepage, make note of the banner at the top of screen that advises users to update their Azure AD authentication. Click on the link provided to do so. + + ![Rancher UI Banner](/img/rancher-ui-azure-update.png) + +1. To complete the move to the new Microsoft Graph API, click **Update Endpoint**. + + **Note:** Ensure that your Azure app has a [new set of permissions](#3-set-required-permissions-for-rancher) before starting the update. + + ![Update Endpoint](/img/rancher-button-to-update.png) + +1. When you receive the pop-up warning message, click **Update**. + + ![Azure Update Pop-up](/img/azure-update-popup.png) + +1. Refer to the [tables](#global) below for the full list of endpoint changes that Rancher performs. Admins do not need to do this manually. + +#### Air-Gapped Environments + +In air-gapped environments, admins should ensure that their endpoints are [whitelisted](#3.2) since the Graph Endpoint URL is changing. + +#### Rolling Back the Migration + +If you need to roll back your migration, please note the following: + +1. Admins are encouraged to use the proper restore process if they want to go back. Please see [backup docs](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md), [restore docs](../../../new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md), and [examples](../../../../reference-guides/backup-restore-configuration/examples.md) for reference. + +1. Azure app owners who want to rotate the Application Secret will need to also rotate it in Rancher as Rancher does not automatically update the Application Secret when it is changed in Azure. In Rancher, note that it is stored in a Kubernetes secret called `azureadconfig-applicationsecret` which is in the `cattle-global-data` namespace. + +:::caution + +If you upgrade to Rancher v2.7.0+ with an existing Azure AD setup, and choose to disable the auth provider, you won't be able to restore the previous setup. You also won't be able to set up Azure AD using the old flow. You'll need to re-register with the new auth flow. Since Rancher now uses the Graph API, users need set up the [proper permissions in the Azure portal](#3-set-required-permissions-for-rancher). + +::: + + +#### Global: + +Rancher Field | Deprecated Endpoints +---------------- | ------------------------------------------------------------- +Auth Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/authorize +Endpoint | https://login.microsoftonline.com/ +Graph Endpoint | https://graph.windows.net/ +Token Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/token + +Rancher Field | New Endpoints +---------------- | ------------------------------------------------------------------ +Auth Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/authorize +Endpoint | https://login.microsoftonline.com/ +Graph Endpoint | https://graph.microsoft.com +Token Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/token + +#### China: + +Rancher Field | Deprecated Endpoints +---------------- | ---------------------------------------------------------- +Auth Endpoint | https://login.chinacloudapi.cn/{tenantID}/oauth2/authorize +Endpoint | https://login.chinacloudapi.cn/ +Graph Endpoint | https://graph.chinacloudapi.cn/ +Token Endpoint | https://login.chinacloudapi.cn/{tenantID}/oauth2/token + +Rancher Field | New Endpoints +---------------- | ------------------------------------------------------------------------- +Auth Endpoint | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/authorize +Endpoint | https://login.partner.microsoftonline.cn/ +Graph Endpoint | https://microsoftgraph.chinacloudapi.cn +Token Endpoint | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/token + + +## Deprecated Azure AD Graph API + +>**Important:** +> +>- The [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) is deprecated and will be retired by Microsoft at any time after June 30, 2023, without advance notice. We will update our docs to advise the community when it is retired. Rancher now uses the [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api) as the new flow to set up Azure AD as the external auth provider. +> +> +>- If you're a new user, or wish to migrate, refer to the new flow instructions for Rancher v2.7.0+. +> +> +>- If you don't wish to upgrade to v2.7.0+ after the Azure AD Graph API is retired, you'll need to either: + - Use the built-in Rancher auth or + - Use another third-party auth system and set that up in Rancher. Please see the [authentication docs](../../../../pages-for-subheaders/authentication-config.md) to learn how to configure other open authentication providers. + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md new file mode 100644 index 00000000000..0fe6995d4ae --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md @@ -0,0 +1,64 @@ +--- +title: Configure FreeIPA +--- + + + + + +If your organization uses FreeIPA for user authentication, you can configure Rancher to allow your users to login using their FreeIPA credentials. + +:::note Prerequisites: + +- You must have a [FreeIPA Server](https://www.freeipa.org/) configured. +- Create a service account in FreeIPA with `read-only` access. Rancher uses this account to verify group membership when a user makes a request using an API key. +- Read [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +::: + +1. Sign into Rancher using a local user assigned the `administrator` role (i.e., the _local principal_). +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **FreeIPA**. +1. Complete the **Configure an FreeIPA server** form. + + You may need to log in to your domain controller to find the information requested in the form. + + :::note Using TLS? + + If the certificate is self-signed or not from a recognized certificate authority, make sure you provide the complete chain. That chain is needed to verify the server's certificate. + + ::: + + :::note User Search Base vs. Group Search Base + + Search base allows Rancher to search for users and groups that are in your FreeIPA. These fields are only for search bases and not for search filters. + + * If your users and groups are in the same search base, complete only the User Search Base. + * If your groups are in a different search base, you can optionally complete the Group Search Base. This field is dedicated to searching groups, but is not required. + + ::: + +1. If your FreeIPA deviates from the standard AD schema, complete the **Customize Schema** form to match it. Otherwise, skip this step. + + :::note Search Attribute + + The Search Attribute field defaults with three specific values: `uid|sn|givenName`. After FreeIPA is configured, when a user enters text to add users or groups, Rancher automatically queries the FreeIPA server and attempts to match fields by user id, last name, or first name. Rancher specifically searches for users/groups that begin with the text entered in the search field. + + The default field value `uid|sn|givenName`, but you can configure this field to a subset of these fields. The pipe (`|`) between the fields separates these fields. + + * `uid`: User ID + * `sn`: Last Name + * `givenName`: First Name + + With this search attribute, Rancher creates search filters for users and groups, but you *cannot* add your own search filters in this field. + + ::: + +1. Enter your FreeIPA username and password in **Authenticate with FreeIPA** to confirm that Rancher is configured to use FreeIPA authentication. +1. Click **Enable**. + +**Result:** + +- FreeIPA authentication is configured. +- You are signed into Rancher with your FreeIPA account (i.e., the _external principal_). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md new file mode 100644 index 00000000000..960d968de8d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md @@ -0,0 +1,60 @@ +--- +title: Configure GitHub +--- + + + + + +In environments using GitHub, you can configure Rancher to allow sign on using GitHub credentials. + +:::note Prerequisites: + +Read [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +::: + +1. Sign into Rancher using a local user assigned the `administrator` role (i.e., the _local principal_). +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **GitHub**. +1. Follow the directions displayed to set up a GitHub Application. Rancher redirects you to GitHub to complete registration. + + :::note What's an Authorization Callback URL? + + The Authorization Callback URL is the URL where users go to begin using your application (i.e. the splash screen). + + When you use external authentication, authentication does not actually take place in your application. Instead, authentication takes place externally (in this case, GitHub). After this external authentication completes successfully, the Authorization Callback URL is the location where the user re-enters your application. + + ::: + +1. From GitHub, copy the **Client ID** and **Client Secret**. Paste them into Rancher. + + :::note Where do I find the Client ID and Client Secret? + + From GitHub, select Settings > Developer Settings > OAuth Apps. The Client ID and Client Secret are displayed prominently. + + ::: + +1. Click **Authenticate with GitHub**. + +1. Use the **Site Access** options to configure the scope of user authorization. + + - **Allow any valid Users** + + _Any_ GitHub user can access Rancher. We generally discourage use of this setting! + + - **Allow members of Clusters, Projects, plus Authorized Users and Organizations** + + Any GitHub user or group added as a **Cluster Member** or **Project Member** can log in to Rancher. Additionally, any GitHub user or group you add to the **Authorized Users and Organizations** list may log in to Rancher. + + - **Restrict access to only Authorized Users and Organizations** + + Only GitHub users or groups added to the Authorized Users and Organizations can log in to Rancher. +
+1. Click **Enable**. + +**Result:** + +- GitHub authentication is configured. +- You are signed into Rancher with your GitHub account (i.e., the _external principal_). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md new file mode 100644 index 00000000000..23b5084b594 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md @@ -0,0 +1,115 @@ +--- +title: Configure Google OAuth +--- + + + + + +If your organization uses G Suite for user authentication, you can configure Rancher to allow your users to log in using their G Suite credentials. + +Only admins of the G Suite domain have access to the Admin SDK. Therefore, only G Suite admins can configure Google OAuth for Rancher. + +Within Rancher, only administrators or users with the **Manage Authentication** [global role](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) can configure authentication. + +## Prerequisites + +- You must have a [G Suite admin account](https://admin.google.com) configured. +- G Suite requires a [top private domain FQDN](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains) as an authorized domain. One way to get an FQDN is by creating an A-record in Route53 for your Rancher server. You do not need to update your Rancher Server URL setting with that record, because there could be clusters using that URL. +- You must have the Admin SDK API enabled for your G Suite domain. You can enable it using the steps on [this page.](https://support.google.com/a/answer/60757?hl=en) + +After the Admin SDK API is enabled, your G Suite domain's API screen should look like this: +![Enable Admin APIs](/img/Google-Enable-APIs-Screen.png) + +## Setting up G Suite for OAuth with Rancher + +Before you can set up Google OAuth in Rancher, you need to log in to your G Suite account and do the following: + +1. [Add Rancher as an authorized domain in G Suite](#1-adding-rancher-as-an-authorized-domain) +1. [Generate OAuth2 credentials for the Rancher server](#2-creating-oauth2-credentials-for-the-rancher-server) +1. [Create service account credentials for the Rancher server](#3-creating-service-account-credentials) +1. [Register the service account key as an OAuth Client](#4-register-the-service-account-key-as-an-oauth-client) + +### 1. Adding Rancher as an Authorized Domain + +1. Click [here](https://console.developers.google.com/apis/credentials) to go to credentials page of your Google domain. +1. Select your project and click **OAuth consent screen**. +![OAuth Consent Screen](/img/Google-OAuth-consent-screen-tab.png) +1. Go to **Authorized Domains** and enter the top private domain of your Rancher server URL in the list. The top private domain is the rightmost superdomain. So for example, www.foo.co.uk a top private domain of foo.co.uk. For more information on top-level domains, refer to [this article.](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains) +1. Go to **Scopes for Google APIs** and make sure **email,** **profile** and **openid** are enabled. + +**Result:** Rancher has been added as an authorized domain for the Admin SDK API. + +### 2. Creating OAuth2 Credentials for the Rancher Server + +1. Go to the Google API console, select your project, and go to the [credentials page.](https://console.developers.google.com/apis/credentials) +![Credentials](/img/Google-Credentials-tab.png) +1. On the **Create Credentials** dropdown, select **OAuth client ID**. +1. Click **Web application**. +1. Provide a name. +1. Fill out the **Authorized JavaScript origins** and **Authorized redirect URIs**. Note: The Rancher UI page for setting up Google OAuth (available from the Global view under **Security > Authentication > Google**) provides you the exact links to enter for this step. + - Under **Authorized JavaScript origins,** enter your Rancher server URL. + - Under **Authorized redirect URIs,** enter your Rancher server URL appended with the path `verify-auth`. For example, if your URI is `https://rancherServer`, you will enter `https://rancherServer/verify-auth`. +1. Click on **Create**. +1. After the credential is created, you will see a screen with a list of your credentials. Choose the credential you just created, and in that row on rightmost side, click **Download JSON**. Save the file so that you can provide these credentials to Rancher. + +**Result:** Your OAuth credentials have been successfully created. + +### 3. Creating Service Account Credentials +Since the Google Admin SDK is available only to admins, regular users cannot use it to retrieve profiles of other users or their groups. Regular users cannot even retrieve their own groups. + +Since Rancher provides group-based membership access, we require the users to be able to get their own groups, and look up other users and groups when needed. + +As a workaround to get this capability, G Suite recommends creating a service account and delegating authority of your G Suite domain to that service account. + +This section describes how to: + +- Create a service account +- Create a key for the service account and download the credentials as JSON + +1. Click [here](https://console.developers.google.com/iam-admin/serviceaccounts) and select your project for which you generated OAuth credentials. +1. Click on **Create Service Account**. +1. Enter a name and click **Create**. +![Service account creation Step 1](/img/Google-svc-acc-step1.png) +1. Don't provide any roles on the **Service account permissions** page and click **Continue** +![Service account creation Step 2](/img/Google-svc-acc-step2.png) +1. Click on **Create Key** and select the JSON option. Download the JSON file and save it so that you can provide it as the service account credentials to Rancher. +![Service account creation Step 3](/img/Google-svc-acc-step3-key-creation.png) + +**Result:** Your service account is created. + +### 4. Register the Service Account Key as an OAuth Client + +You will need to grant some permissions to the service account you created in the last step. Rancher requires you to grant only read-only permissions for users and groups. + +Using the Unique ID of the service account key, register it as an Oauth Client using the following steps: + +1. Get the Unique ID of the key you just created. If it's not displayed in the list of keys right next to the one you created, you will have to enable it. To enable it, click **Unique ID** and click **OK**. This will add a **Unique ID** column to the list of service account keys. Save the one listed for the service account you created. NOTE: This is a numeric key, not to be confused with the alphanumeric field **Key ID**. + + ![Service account Unique ID](/img/Google-Select-UniqueID-column.png) +1. Go to the [**Domain-wide Delegation** page.](https://admin.google.com/ac/owl/domainwidedelegation) +1. Add the Unique ID obtained in the previous step in the **Client Name** field. +1. In the **One or More API Scopes** field, add the following scopes: + ``` + openid,profile,email,https://www.googleapis.com/auth/admin.directory.user.readonly,https://www.googleapis.com/auth/admin.directory.group.readonly + ``` +1. Click **Authorize**. + +**Result:** The service account is registered as an OAuth client in your G Suite account. + +## Configuring Google OAuth in Rancher + +1. Sign into Rancher using a local user assigned the [administrator](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) role. This user is also called the local principal. +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Google**. The instructions in the UI cover the steps to set up authentication with Google OAuth. + 1. Admin Email: Provide the email of an administrator account from your GSuite setup. In order to perform user and group lookups, google apis require an administrator's email in conjunction with the service account key. + 1. Domain: Provide the domain on which you have configured GSuite. Provide the exact domain and not any aliases. + 1. Nested Group Membership: Check this box to enable nested group memberships. Rancher admins can disable this at any time after configuring auth. + - **Step One** is about adding Rancher as an authorized domain, which we already covered in [this section.](#1-adding-rancher-as-an-authorized-domain) + - For **Step Two,** provide the OAuth credentials JSON that you downloaded after completing [this section.](#2-creating-oauth2-credentials-for-the-rancher-server) You can upload the file or paste the contents into the **OAuth Credentials** field. + - For **Step Three,** provide the service account credentials JSON that downloaded at the end of [this section.](#3-creating-service-account-credentials) The credentials will only work if you successfully [registered the service account key](#4-register-the-service-account-key-as-an-oauth-client) as an OAuth client in your G Suite account. +1. Click **Authenticate with Google**. +1. Click **Enable**. + +**Result:** Google authentication is successfully configured. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md new file mode 100644 index 00000000000..76787538bd0 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md @@ -0,0 +1,149 @@ +--- +title: Configure Keycloak (OIDC) +description: Create a Keycloak OpenID Connect (OIDC) client and configure Rancher to work with Keycloak. By the end your users will be able to sign into Rancher using their Keycloak logins +--- + + + + + +If your organization uses [Keycloak Identity Provider (IdP)](https://www.keycloak.org) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. Rancher supports integration with Keycloak using the OpenID Connect (OIDC) protocol and the SAML protocol. Both implementations are functionally equivalent when used with Rancher. This page describes the process to configure Rancher to work with Keycloak using the OIDC protocol. + +If you prefer to use Keycloak with the SAML protocol instead, refer to [this page](configure-keycloak-saml.md). + +If you have an existing configuration using the SAML protocol and want to switch to the OIDC protocol, refer to [this section](#migrating-from-saml-to-oidc). + +## Prerequisites + +- On Rancher, Keycloak (SAML) is disabled. +- You must have a [Keycloak IdP Server](https://www.keycloak.org/docs/latest/server_installation/) configured. +- In Keycloak, create a [new OIDC client](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients), with the settings below. See the [Keycloak documentation](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients) for help. + + Setting | Value + ------------|------------ + `Client ID` | <CLIENT_ID> (e.g. `rancher`) + `Name` | <CLIENT_NAME> (e.g. `rancher`) + `Client Protocol` | `openid-connect` + `Access Type` | `confidential` + `Valid Redirect URI` | `https://yourRancherHostURL/verify-auth` + +- In the new OIDC client, create [Mappers](https://www.keycloak.org/docs/latest/server_admin/#_protocol-mappers) to expose the users fields. + - Create a new "Groups Mapper" with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Groups Mapper` + `Mapper Type` | `Group Membership` + `Token Claim Name` | `groups` + `Add to ID token` | `OFF` + `Add to access token` | `OFF` + `Add to user info` | `ON` + + - Create a new "Client Audience" with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Client Audience` + `Mapper Type` | `Audience` + `Included Client Audience` | <CLIENT_NAME> + `Add to access token` | `ON` + + - Create a new "Groups Path" with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Group Path` + `Mapper Type` | `Group Membership` + `Token Claim Name` | `full_group_path` + `Full group path` | `ON` + `Add to user info` | `ON` + +## Configuring Keycloak in Rancher + +1. In the Rancher UI, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. Select **Keycloak (OIDC)**. +1. Complete the **Configure a Keycloak OIDC account** form. For help with filling the form, see the [configuration reference](#configuration-reference). +1. After you complete the **Configure a Keycloak OIDC account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Keycloak IdP to validate your Rancher Keycloak configuration. + + :::note + + You may need to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with Keycloak using the OIDC protocol. Your users can now sign into Rancher using their Keycloak logins. + +## Configuration Reference + +| Field | Description | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Client ID | The `Client ID` of your Keycloak client. | +| Client Secret | The generated `Secret` of your Keycloak client. In the Keycloak console, select **Clients**, select the client you created, select the **Credentials** tab and copy the value of the `Secret` field. | +| Private Key / Certificate | A key/certificate pair to create a secure shell between Rancher and your IdP. Required if HTTPS/SSL is enabled on your Keycloak server. | +| Endpoints | Choose whether to use the generated values for the `Rancher URL`, `Issue`, and `Auth Endpoint` fields or to provide manual overrides if incorrect. | +| Keycloak URL | The URL for your Keycloak server. | +| Keycloak Realm | The name of the realm in which the Keycloak client was created in. | +| Rancher URL | The URL for your Rancher Server. | +| Issuer | The URL of your IdP. | +| Auth Endpoint | The URL where users are redirected to authenticate. | + +## Migrating from SAML to OIDC + +This section describes the process to transition from using Rancher with Keycloak (SAML) to Keycloak (OIDC). + +### Reconfigure Keycloak + +1. Change the existing client to use the OIDC protocol. In the Keycloak console, select **Clients**, select the SAML client to migrate, select the **Settings** tab, change `Client Protocol` from `saml` to `openid-connect`, and click **Save** + +1. Verify the `Valid Redirect URIs` are still valid. + +1. Select the **Mappers** tab and create a new Mapper with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Groups Mapper` + `Mapper Type` | `Group Membership` + `Token Claim Name` | `groups` + `Add to ID token` | `ON` + `Add to access token` | `ON` + `Add to user info` | `ON` + +### Reconfigure Rancher + +Before configuring Rancher to use Keycloak (OIDC), Keycloak (SAML) must be first disabled. + +1. In the Rancher UI, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. Select **Keycloak (SAML)**. +1. Click **Disable**. + +Configure Rancher to use Keycloak (OIDC) by following the steps in [this section](#configuring-keycloak-in-rancher). + +:::note + +After configuration is completed, Rancher user permissions will need to be reapplied as they are not automatically migrated. + +::: + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the Keycloak server, first double-check the configuration options of your OIDC client. You may also inspect the Rancher logs to help pinpoint what's causing issues. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. + +All Keycloak related log entries will be prepended with either `[generic oidc]` or `[keycloak oidc]`. + +### You are not redirected to Keycloak + +When you fill the **Configure a Keycloak OIDC account** form and click on **Enable**, you are not redirected to your IdP. + + * Verify your Keycloak client configuration. + +### The generated `Issuer` and `Auth Endpoint` are incorrect + + * On the **Configure a Keycloak OIDC account** form, change **Endpoints** to `Specify (advanced)` and override the `Issuer` and `Auth Endpoint` values. To find the values, go to the Keycloak console and select **Realm Settings**, select the **General** tab, and click **OpenID Endpoint Configuration**. The JSON output will display values for `issuer` and `authorization_endpoint`. + +### Keycloak Error: "Invalid grant_type" + + * In some cases, this error message may be misleading and is actually caused by setting the `Valid Redirect URI` incorrectly. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md new file mode 100644 index 00000000000..21dfc75e8d8 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md @@ -0,0 +1,194 @@ +--- +title: Configure Keycloak (SAML) +description: Create a Keycloak SAML client and configure Rancher to work with Keycloak. By the end your users will be able to sign into Rancher using their Keycloak logins +--- + + + + + +If your organization uses Keycloak Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +## Prerequisites + +- You must have a [Keycloak IdP Server](https://www.keycloak.org/docs/latest/server_installation/) configured. +- In Keycloak, create a [new SAML client](https://www.keycloak.org/docs/latest/server_admin/#saml-clients), with the settings below. See the [Keycloak documentation](https://www.keycloak.org/docs/latest/server_admin/#saml-clients) for help. + + Setting | Value + ------------|------------ + `Sign Documents` | `ON` 1 + `Sign Assertions` | `ON` 1 + All other `ON/OFF` Settings | `OFF` + `Client ID` | Either `https://yourRancherHostURL/v1-saml/keycloak/saml/metadata` or the value configured in the `Entry ID Field` of the Rancher Keycloak configuration2 + `Client Name` | (e.g. `rancher`) + `Client Protocol` | `SAML` + `Valid Redirect URI` | `https://yourRancherHostURL/v1-saml/keycloak/saml/acs` + + >1: Optionally, you can enable either one or both of these settings. + >2: Rancher SAML metadata won't be generated until a SAML provider is configured and saved. + + ![](/img/keycloak/keycloak-saml-client-configuration.png) + +- In the new SAML client, create Mappers to expose the users fields + - Add all "Builtin Protocol Mappers" + ![](/img/keycloak/keycloak-saml-client-builtin-mappers.png) + - Create a new "Group list" mapper to map the member attribute to a user's groups + ![](/img/keycloak/keycloak-saml-client-group-mapper.png) + +## Getting the IDP Metadata + + + + +To get the IDP metadata, export a `metadata.xml` file from your Keycloak client. +From the **Installation** tab, choose the **SAML Metadata IDPSSODescriptor** format option and download your file. + + + + +1. From the **Configure** section, click the **Realm Settings** tab. +1. Click the **General** tab. +1. From the **Endpoints** field, click **SAML 2.0 Identity Provider Metadata**. + +Verify the IDP metadata contains the following attributes: + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +Some browsers, such as Firefox, may render/process the document such that the contents appear to have been modified, and some attributes appear to be missing. In this situation, use the raw response data that can be found using your browser. + +The following is an example process for Firefox, but will vary slightly for other browsers: + +1. Press **F12** to access the developer console. +1. Click the **Network** tab. +1. From the table, click the row containing `descriptor`. +1. From the details pane, click the **Response** tab. +1. Copy the raw response data. + +The XML obtained contains `EntitiesDescriptor` as the root element. Rancher expects the root element to be `EntityDescriptor` rather than `EntitiesDescriptor`. So before passing this XML to Rancher, follow these steps to adjust it: + +1. Copy all the attributes from `EntitiesDescriptor` to the `EntityDescriptor` that are not present. +1. Remove the `` tag from the beginning. +1. Remove the `` from the end of the xml. + +You are left with something similar as the example below: + +``` + +.... + +``` + + + + +1. From the **Configure** section, click the **Realm Settings** tab. +1. Click the **General** tab. +1. From the **Endpoints** field, click **SAML 2.0 Identity Provider Metadata**. + +Verify the IDP metadata contains the following attributes: + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +Some browsers, such as Firefox, may render/process the document such that the contents appear to have been modified, and some attributes appear to be missing. In this situation, use the raw response data that can be found using your browser. + +The following is an example process for Firefox, but will vary slightly for other browsers: + +1. Press **F12** to access the developer console. +1. Click the **Network** tab. +1. From the table, click the row containing `descriptor`. +1. From the details pane, click the **Response** tab. +1. Copy the raw response data. + + + + +## Configuring Keycloak in Rancher + + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Keycloak SAML**. +1. Complete the **Configure Keycloak Account** form. For help with filling the form, see the [configuration reference](#configuration-reference). +1. After you complete the **Configure a Keycloak Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Keycloak IdP to validate your Rancher Keycloak configuration. + + :::note + + You may have to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with Keycloak. Your users can now sign into Rancher using their Keycloak logins. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## Configuration Reference + +| Field | Description | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Display Name Field | The attribute that contains the display name of users.

Example: `givenName` | +| User Name Field | The attribute that contains the user name/given name.

Example: `email` | +| UID Field | An attribute that is unique to every user.

Example: `email` | +| Groups Field | Make entries for managing group memberships.

Example: `member` | +| Entity ID Field | The ID that needs to be configured as a client ID in the Keycloak client.

Default: `https://yourRancherHostURL/v1-saml/keycloak/saml/metadata` | +| Rancher API Host | The URL for your Rancher Server. | +| Private Key / Certificate | A key/certificate pair to create a secure shell between Rancher and your IdP. | +| IDP-metadata | The `metadata.xml` file that you exported from your IdP server. | + +:::tip + +You can generate a key/certificate pair using an openssl command. For example: + +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.cert + +::: + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the Keycloak server, first double-check the configuration option of your SAML client. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. + +### You are not redirected to Keycloak + +When you click on **Authenticate with Keycloak**, you are not redirected to your IdP. + + * Verify your Keycloak client configuration. + * Make sure `Force Post Binding` set to `OFF`. + + +### Forbidden message displayed after IdP login + +You are correctly redirected to your IdP login page and you are able to enter your credentials, however you get a `Forbidden` message afterwards. + + * Check the Rancher debug log. + * If the log displays `ERROR: either the Response or Assertion must be signed`, make sure either `Sign Documents` or `Sign assertions` is set to `ON` in your Keycloak client. + +### HTTP 502 when trying to access /v1-saml/keycloak/saml/metadata + +This is usually due to the metadata not being created until a SAML provider is configured. +Try configuring and saving keycloak as your SAML provider and then accessing the metadata. + +### Keycloak Error: "We're sorry, failed to process response" + + * Check your Keycloak log. + * If the log displays `failed: org.keycloak.common.VerificationException: Client does not have a public key`, set `Encrypt Assertions` to `OFF` in your Keycloak client. + +### Keycloak Error: "We're sorry, invalid requester" + + * Check your Keycloak log. + * If the log displays `request validation failed: org.keycloak.common.VerificationException: SigAlg was null`, set `Client Signature Required` to `OFF` in your Keycloak client. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md new file mode 100644 index 00000000000..0b572698dc8 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md @@ -0,0 +1,111 @@ +--- +title: Configure Okta (SAML) +--- + + + + + +If your organization uses Okta Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +:::note + +Okta integration only supports Service Provider initiated logins. + +::: +## Prerequisites + +In Okta, create a SAML Application with the settings below. See the [Okta documentation](https://developer.okta.com/standards/SAML/setting_up_a_saml_application_in_okta) for help. + +Setting | Value +------------|------------ +`Single Sign on URL` | `https://yourRancherHostURL/v1-saml/okta/saml/acs` +`Audience URI (SP Entity ID)` | `https://yourRancherHostURL/v1-saml/okta/saml/metadata` + +## Configuring Okta in Rancher + +You can integrate Okta with Rancher, so that authenticated users can access Rancher resources through their group permissions. Okta returns a SAML assertion that authenticates a user, including which groups a user belongs to. + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Okta**. +1. Complete the **Configure Okta Account** form. The examples below describe how you can map Okta attributes from attribute statements to fields within Rancher. + + | Field | Description | + | ------------------------- | ----------------------------------------------------------------------------- | + | Display Name Field | The attribute name from an attribute statement that contains the display name of users. | + | User Name Field | The attribute name from an attribute statement that contains the user name/given name. | + | UID Field | The attribute name from an attribute statement that is unique to every user. | + | Groups Field | The attribute name in a group attribute statement that exposes your groups. | + | Rancher API Host | The URL for your Rancher Server. | + | Private Key / Certificate | A key/certificate pair used for Assertion Encryption. | + | Metadata XML | The `Identity Provider metadata` file that you find in the application `Sign On` section. | + + :::tip + + You can generate a key/certificate pair using an openssl command. For example: + + ``` + openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.crt + ``` + + ::: + + +1. After you complete the **Configure Okta Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Okta IdP to validate your Rancher Okta configuration. + + :::note + + If nothing seems to happen, it's likely because your browser blocked the pop-up. Make sure you disable the pop-up blocker for your rancher domain and whitelist it in any other extensions you might utilize. + + ::: + +**Result:** Rancher is configured to work with Okta. Your users can now sign into Rancher using their Okta logins. + +:::note SAML Provider Caveats: + +If you configure Okta without OpenLDAP, you won't be able to search for or directly lookup users or groups. This brings several caveats: + +- Users and groups aren't validated when you assign permissions to them in Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## Okta with OpenLDAP search + +You can add an OpenLDAP backend to assist with user and group search. Rancher will display additional users and groups from the OpenLDAP service. This allows assigning permissions to groups that the logged-in user is not already a member of. + +### OpenLDAP Prerequisites + +If you use Okta as your IdP, you can [set up an LDAP interface](https://help.okta.com/en-us/Content/Topics/Directory/LDAP-interface-main.htm) for Rancher to use. You can also configure an external OpenLDAP server. + +You must configure Rancher with a LDAP bind account (aka service account) so that you can search and retrieve LDAP entries for users and groups that should have access. Don't use an administrator account or personal account as an LDAP bind account. [Create](https://help.okta.com/en-us/Content/Topics/users-groups-profiles/usgp-add-users.htm) a dedicated account in OpenLDAP, with read-only access to users and groups under the configured searchbase. + +:::warning Security Considerations + +The OpenLDAP service account is used for all searches. Rancher users will see users and groups that the OpenLDAP service account can view, regardless of their individual SAML permissions. + +::: + + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or from an unrecognized certificate authority, Rancher needs the CA certificate (concatenated with any intermediate certificates) in PEM format. Provide this certificate during the configuration so that Rancher can validate the certificate chain. + +### Configure OpenLDAP in Rancher + +[Configure the settings](../configure-openldap/openldap-config-reference.md) for the OpenLDAP server, groups and users. Note that nested group membership isn't available. + +> Before you proceed with the configuration, please familiarise yourself with [external authentication configuration and principal users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +1. Sign into Rancher using a local user assigned the [administrator](https://ranchermanager.docs.rancher.com/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions) role (i.e., the _local principal_). +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Okta** or, if SAML is already configured, **Edit Config** +1. Under **User and Group Search**, check **Configure an OpenLDAP server** + +If you experience issues when you test the connection to the OpenLDAP server, ensure that you entered the credentials for the service account and configured the search base correctly. Inspecting the Rancher logs can help pinpoint the root cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) for more information. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md new file mode 100644 index 00000000000..e45d179881e --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md @@ -0,0 +1,66 @@ +--- +title: Configure PingIdentity (SAML) +--- + + + + + +If your organization uses Ping Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +>**Prerequisites:** +> +>- You must have a [Ping IdP Server](https://www.pingidentity.com/) configured. +>- Following are the Rancher Service Provider URLs needed for configuration: +Metadata URL: `https:///v1-saml/ping/saml/metadata` +Assertion Consumer Service (ACS) URL: `https:///v1-saml/ping/saml/acs` +Note that these URLs will not return valid data until the authentication configuration is saved in Rancher. +>- Export a `metadata.xml` file from your IdP Server. For more information, see the [PingIdentity documentation](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html). + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Ping Identity**. +1. Complete the **Configure a Ping Account** form. Ping IdP lets you specify what data store you want to use. You can either add a database or use an existing ldap server. For example, if you select your Active Directory (AD) server, the examples below describe how you can map AD attributes to fields within Rancher. + + 1. **Display Name Field**: Enter the AD attribute that contains the display name of users (example: `displayName`). + + 1. **User Name Field**: Enter the AD attribute that contains the user name/given name (example: `givenName`). + + 1. **UID Field**: Enter an AD attribute that is unique to every user (example: `sAMAccountName`, `distinguishedName`). + + 1. **Groups Field**: Make entries for managing group memberships (example: `memberOf`). + + 1. **Entity ID Field** (optional): The published, protocol-dependent, unique identifier of your partner. This ID defines your organization as the entity operating the server for SAML 2.0 transactions. This ID may have been obtained out-of-band or via a SAML metadata file. + + 1. **Rancher API Host**: Enter the URL for your Rancher Server. + + 1. **Private Key** and **Certificate**: This is a key-certificate pair to create a secure shell between Rancher and your IdP. + + You can generate one using an openssl command. For example: + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP-metadata**: The `metadata.xml` file that you [exported from your IdP server](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html). + + +1. After you complete the **Configure Ping Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Ping IdP to validate your Rancher PingIdentity configuration. + + :::note + + You may have to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with PingIdentity. Your users can now sign into Rancher using their PingIdentity logins. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md new file mode 100644 index 00000000000..801ed84068b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md @@ -0,0 +1,19 @@ +--- +title: Local Authentication +--- + + + + + +Local authentication is the default until you configure an external authentication provider. Rancher stores user account information, such as usernames and passwords, locally. By default, the `admin` user that logs in to Rancher for the first time is a local user. + +## Adding Local Users + +Regardless of whether you use external authentication, you should create a few local authentication users so that you can continue using Rancher if your external authentication service encounters issues. + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Users**. +1. Click **Create**. +1. Complete the **Add User** form. +1. Click **Create**. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md new file mode 100644 index 00000000000..38fc2a6403c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md @@ -0,0 +1,81 @@ +--- +title: Users and Groups +--- + + + + + +Rancher relies on users and groups to determine who is allowed to log in to Rancher and which resources they can access. When you configure an external authentication provider, users from that provider will be able to log in to your Rancher server. When a user logs in, the authentication provider will supply your Rancher server with a list of groups to which the user belongs. + +Access to clusters, projects, multi-cluster apps, and global DNS providers and entries can be controlled by adding either individual users or groups to these resources. When you add a group to a resource, all users who are members of that group in the authentication provider, will be able to access the resource with the permissions that you've specified for the group. For more information on roles and permissions, see [Role Based Access Control](../../../../pages-for-subheaders/manage-role-based-access-control-rbac.md). + +## Managing Members + +When adding a user or group to a resource, you can search for users or groups by beginning to type their name. The Rancher server will query the authentication provider to find users and groups that match what you've entered. Searching is limited to the authentication provider that you are currently logged in with. For example, if you've enabled GitHub authentication but are logged in using a [local](create-local-users.md) user account, you will not be able to search for GitHub users or groups. + +All users, whether they are local users or from an authentication provider, can be viewed and managed. In the upper left corner, click **☰ > Users & Authentication**. In the left navigation bar, click **Users**. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## User Information + +Rancher maintains information about each user that logs in through an authentication provider. This information includes whether the user is allowed to access your Rancher server and the list of groups that the user belongs to. Rancher keeps this user information so that the CLI, API, and kubectl can accurately reflect the access that the user has based on their group membership in the authentication provider. + +Whenever a user logs in to the UI using an authentication provider, Rancher automatically updates this user information. + +### Automatically Refreshing User Information + +Rancher will periodically refresh the user information even before a user logs in through the UI. You can control how often Rancher performs this refresh. + +Two settings control this behavior: + +- **`auth-user-info-max-age-seconds`** + + This setting controls how old a user's information can be before Rancher refreshes it. If a user makes an API call (either directly or by using the Rancher CLI or kubectl) and the time since the user's last refresh is greater than this setting, then Rancher will trigger a refresh. This setting defaults to `3600` seconds, i.e. 1 hour. + +- **`auth-user-info-resync-cron`** + + This setting controls a recurring schedule for resyncing authentication provider information for all users. Regardless of whether a user has logged in or used the API recently, this will cause the user to be refreshed at the specified interval. This setting defaults to `0 0 * * *`, i.e. once a day at midnight. See the [Cron documentation](https://en.wikipedia.org/wiki/Cron) for more information on valid values for this setting. + +To change these settings, + +1. In the upper left corner, click **☰ > Global Settings**. +1. Go to the setting you want to configure and click **⋮ > Edit Setting**. + +:::note + +Since SAML does not support user lookup, SAML-based authentication providers do not support periodically refreshing user information. User information will only be refreshed when the user logs into the Rancher UI. + +::: +### Manually Refreshing User Information + +If you are not sure the last time Rancher performed an automatic refresh of user information, you can perform a manual refresh of all users. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. On the **Users** page, click on **Refresh Group Memberships**. + +**Results:** Rancher refreshes the user information for all users. Requesting this refresh will update which users can access Rancher as well as all the groups that each user belongs to. + +:::note + +Since SAML does not support user lookup, SAML-based authentication providers do not support the ability to manually refresh user information. User information will only be refreshed when the user logs into the Rancher UI. + +::: + +## Session Length + +The default length (TTL) of each user session is adjustable. The default session length is 16 hours. + +1. In the upper left corner, click **☰ > Global Settings**. +1. Go to **`auth-user-session-ttl-minutes`** and click **⋮ > Edit Setting**. +1. Enter the amount of time in minutes a session length should last and click **Save**. + +**Result:** Users are automatically logged out of Rancher after the set number of minutes. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md new file mode 100644 index 00000000000..925c82951d9 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md @@ -0,0 +1,86 @@ +--- +title: 1. Configuring Microsoft AD FS for Rancher +--- + + + + + +Before you configure Rancher to support Active Directory Federation Service (AD FS), you must add Rancher as a [relying party trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/technical-reference/understanding-key-ad-fs-concepts) in AD FS. + +1. Log into your AD server as an administrative user. + +1. Open the **AD FS Management** console. Select **Add Relying Party Trust..**. from the **Actions** menu and click **Start**. + + ![](/img/adfs/adfs-overview.png) + +1. Select **Enter data about the relying party manually** as the option for obtaining data about the relying party. + + ![](/img/adfs/adfs-add-rpt-2.png) + +1. Enter your desired **Display name** for your Relying Party Trust. For example, `Rancher`. + + ![](/img/adfs/adfs-add-rpt-3.png) + +1. Select **AD FS profile** as the configuration profile for your relying party trust. + + ![](/img/adfs/adfs-add-rpt-4.png) + +1. Leave the **optional token encryption certificate** empty, as Rancher AD FS will not be using one. + + ![](/img/adfs/adfs-add-rpt-5.png) + +1. Select **Enable support for the SAML 2.0 WebSSO protocol** + and enter `https:///v1-saml/adfs/saml/acs` for the service URL. + + ![](/img/adfs/adfs-add-rpt-6.png) + +1. Add `https:///v1-saml/adfs/saml/metadata` as the **Relying party trust identifier**. + + ![](/img/adfs/adfs-add-rpt-7.png) + +1. This tutorial will not cover multi-factor authentication; please refer to the [Microsoft documentation](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/configure-additional-authentication-methods-for-ad-fs) if you would like to configure multi-factor authentication. + + ![](/img/adfs/adfs-add-rpt-8.png) + +1. From **Choose Issuance Authorization RUles**, you may select either of the options available according to use case. However, for the purposes of this guide, select **Permit all users to access this relying party**. + + ![](/img/adfs/adfs-add-rpt-9.png) + +1. After reviewing your settings, select **Next** to add the relying party trust. + + ![](/img/adfs/adfs-add-rpt-10.png) + + +1. Select **Open the Edit Claim Rules..**. and click **Close**. + + ![](/img/adfs/adfs-add-rpt-11.png) + +1. On the **Issuance Transform Rules** tab, click **Add Rule..**.. + + ![](/img/adfs/adfs-edit-cr.png) + +1. Select **Send LDAP Attributes as Claims** as the **Claim rule template**. + + ![](/img/adfs/adfs-add-tcr-1.png) + +1. Set the **Claim rule name** to your desired name (for example, `Rancher Attributes`) and select **Active Directory** as the **Attribute store**. Create the following mapping to reflect the table below: + + | LDAP Attribute | Outgoing Claim Type | + | -------------------------------------------- | ------------------- | + | Given-Name | Given Name | + | User-Principal-Name | UPN | + | Token-Groups - Qualified by Long Domain Name | Group | + | SAM-Account-Name | Name | +
+ + ![](/img/adfs/adfs-add-tcr-2.png) + +1. Download the `federationmetadata.xml` from your AD server at: +``` +https:///federationmetadata/2007-06/federationmetadata.xml +``` + +**Result:** You've added Rancher as a relying trust party. Now you can configure Rancher to leverage AD. + +### [Next: Configuring Rancher for Microsoft AD FS](configure-rancher-for-ms-adfs.md) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md new file mode 100644 index 00000000000..d49f149cd47 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md @@ -0,0 +1,58 @@ +--- +title: 2. Configuring Rancher for Microsoft AD FS +--- + + + + + +After you complete [Configuring Microsoft AD FS for Rancher](configure-ms-adfs-for-rancher.md), enter your Active Directory Federation Service (AD FS) information into Rancher so that AD FS users can authenticate with Rancher. + +:::note Important Notes For Configuring Your ADFS Server: + +- The SAML 2.0 WebSSO Protocol Service URL is: `https:///v1-saml/adfs/saml/acs` +- The Relying Party Trust identifier URL is: `https:///v1-saml/adfs/saml/metadata` +- You must export the `federationmetadata.xml` file from your AD FS server. This can be found at: `https:///federationmetadata/2007-06/federationmetadata.xml` + +::: + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **ADFS**. +1. Complete the **Configure AD FS Account** form. Microsoft AD FS lets you specify an existing Active Directory (AD) server. The [configuration section below](#configuration) describe how you can map AD attributes to fields within Rancher. +1. After you complete the **Configure AD FS Account** form, click **Enable**. + + Rancher redirects you to the AD FS login page. Enter credentials that authenticate with Microsoft AD FS to validate your Rancher AD FS configuration. + + :::note + + You may have to disable your popup blocker to see the AD FS login page. + + ::: + +**Result:** Rancher is configured to work with MS FS. Your users can now sign into Rancher using their MS FS logins. + +## Configuration + +| Field | Description | +|---------------------------|-----------------| +| Display Name Field | The AD attribute that contains the display name of users.

Example: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name` | +| User Name Field | The AD attribute that contains the user name/given name.

Example: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname` | +| UID Field | An AD attribute that is unique to every user.

Example: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn` | +| Groups Field | Make entries for managing group memberships.

Example: `http://schemas.xmlsoap.org/claims/Group` | +| Rancher API Host | The URL for your Rancher Server. | +| Private Key / Certificate | This is a key-certificate pair to create a secure shell between Rancher and your AD FS. Ensure you set the Common Name (CN) to your Rancher Server URL.

[Certificate creation command](#cert-command) | +| Metadata XML | The `federationmetadata.xml` file exported from your AD FS server.

You can find this file at `https:///federationmetadata/2007-06/federationmetadata.xml`. | + + + + +:::tip + +You can generate a certificate using an openssl command. For example: + +``` +openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" +``` + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md new file mode 100644 index 00000000000..bd6f5454ba2 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md @@ -0,0 +1,82 @@ +--- +title: OpenLDAP Configuration Reference +--- + + + + + +For further details on configuring OpenLDAP authentication, refer to the [official documentation.](https://www.openldap.org/doc/) + +> Before you proceed with the configuration, please familiarize yourself with the concepts of [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + + +## Background: OpenLDAP Authentication Flow + +1. When a user attempts to login with LDAP credentials, Rancher creates an initial bind to the LDAP server using a service account with permissions to search the directory and read user/group attributes. +2. Rancher then searches the directory for the user by using a search filter based on the provided username and configured attribute mappings. +3. Once the user has been found, they are authenticated with another LDAP bind request using the user's DN and provided password. +4. Once authentication succeeded, Rancher then resolves the group memberships both from the membership attribute in the user's object and by performing a group search based on the configured user mapping attribute. + +## OpenLDAP Server Configuration + +You will need to enter the address, port, and protocol to connect to your OpenLDAP server. `389` is the standard port for insecure traffic, `636` for TLS traffic. + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or not from a recognized certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +If you are in doubt about the correct values to enter in the user/group Search Base configuration fields, consult your LDAP administrator or refer to the section [Identify Search Base and Schema using ldapsearch](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#annex-identify-search-base-and-schema-using-ldapsearch) in the Active Directory authentication documentation. + +
OpenLDAP Server Parameters
+ +| Parameter | Description | +|:--|:--| +| Hostname | Specify the hostname or IP address of the OpenLDAP server | +| Port | Specify the port at which the OpenLDAP server is listening for connections. Unencrypted LDAP normally uses the standard port of 389, while LDAPS uses port 636.| +| TLS | Check this box to enable LDAP over SSL/TLS (commonly known as LDAPS). You will also need to paste in the CA certificate if the server uses a self-signed/enterprise-signed certificate. | +| Server Connection Timeout | The duration in number of seconds that Rancher waits before considering the server unreachable. | +| Service Account Distinguished Name | Enter the Distinguished Name (DN) of the user that should be used to bind, search and retrieve LDAP entries. | +| Service Account Password | The password for the service account. | +| User Search Base | Enter the Distinguished Name of the node in your directory tree from which to start searching for user objects. All users must be descendents of this base DN. For example: "ou=people,dc=acme,dc=com".| +| Group Search Base | If your groups live under a different node than the one configured under `User Search Base` you will need to provide the Distinguished Name here. Otherwise leave this field empty. For example: "ou=groups,dc=acme,dc=com".| + +## User/Group Schema Configuration + +If your OpenLDAP directory deviates from the standard OpenLDAP schema, you must complete the **Customize Schema** section to match it. + +Note that the attribute mappings configured in this section are used by Rancher to construct search filters and resolve group membership. It is therefore always recommended to verify that the configuration here matches the schema used in your OpenLDAP. + +If you are unfamiliar with the user/group schema used in the OpenLDAP server, consult your LDAP administrator or refer to the section [Identify Search Base and Schema using ldapsearch](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#annex-identify-search-base-and-schema-using-ldapsearch) in the Active Directory authentication documentation. + +### User Schema Configuration + +The table below details the parameters for the user schema configuration. + +
User Schema Configuration Parameters
+ +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for user objects in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Username Attribute | The user attribute whose value is suitable as a display name. | +| Login Attribute | The attribute whose value matches the username part of credentials entered by your users when logging in to Rancher. This is typically `uid`. | +| User Member Attribute | The user attribute containing the Distinguished Name of groups a user is member of. Usually this is one of `memberOf` or `isMemberOf`. | +| Search Attribute | When a user enters text to add users or groups in the UI, Rancher queries the LDAP server and attempts to match users by the attributes provided in this setting. Multiple attributes can be specified by separating them with the pipe ("\|") symbol. | +| User Enabled Attribute | If the schema of your OpenLDAP server supports a user attribute whose value can be evaluated to determine if the account is disabled or locked, enter the name of that attribute. The default OpenLDAP schema does not support this and the field should usually be left empty. | +| Disabled Status Bitmask | This is the value for a disabled/locked user account. The parameter is ignored if `User Enabled Attribute` is empty. | + +### Group Schema Configuration + +The table below details the parameters for the group schema configuration. + +
Group Schema Configuration Parameters
+ +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for group entries in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Name Attribute | The group attribute whose value is suitable for a display name. | +| Group Member User Attribute | The name of the **user attribute** whose format matches the group members in the `Group Member Mapping Attribute`. | +| Group Member Mapping Attribute | The name of the group attribute containing the members of a group. | +| Search Attribute | Attribute used to construct search filters when adding groups to clusters or projects in the UI. See description of user schema `Search Attribute`. | +| Group DN Attribute | The name of the group attribute whose format matches the values in the user's group membership attribute. See `User Member Attribute`. | +| Nested Group Membership | This settings defines whether Rancher should resolve nested group memberships. Use only if your organization makes use of these nested memberships (ie. you have groups that contain other groups as members). This option is disabled if you are using Shibboleth. | \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md new file mode 100644 index 00000000000..a51b1328cd0 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md @@ -0,0 +1,32 @@ +--- +title: Group Permissions with Shibboleth and OpenLDAP +--- + + + + + +Because Shibboleth is a SAML provider, it doesn't support searching for groups. While a Shibboleth integration can validate user credentials, it can't be used to assign permissions to groups in Rancher without additional configuration. + +One solution to this problem is to configure an OpenLDAP identity provider. With an OpenLDAP back end for Shibboleth, you will be able to search for groups in Rancher and assign them to resources such as clusters, projects, or namespaces from the Rancher UI. + +### Terminology + +- **Shibboleth** is a single sign-on log-in system for computer networks and the Internet. It allows people to sign in using just one identity to various systems. It validates user credentials, but does not, on its own, handle group memberships. +- **SAML:** Security Assertion Markup Language, an open standard for exchanging authentication and authorization data between an identity provider and a service provider. +- **OpenLDAP:** a free, open-source implementation of the Lightweight Directory Access Protocol (LDAP). It is used to manage an organization’s computers and users. OpenLDAP is useful for Rancher users because it supports groups. In Rancher, it is possible to assign permissions to groups so that they can access resources such as clusters, projects, or namespaces, as long as the groups already exist in the identity provider. +- **IdP or IDP:** An identity provider. OpenLDAP is an example of an identity provider. + +### Adding OpenLDAP Group Permissions to Rancher Resources + +The diagram below illustrates how members of an OpenLDAP group can access resources in Rancher that the group has permissions for. + +For example, a cluster owner could add an OpenLDAP group to a cluster so that they have permissions view most cluster level resources and create new projects. Then the OpenLDAP group members will have access to the cluster as soon as they log in to Rancher. + +In this scenario, OpenLDAP allows the cluster owner to search for groups when assigning persmissions. Without OpenLDAP, the functionality to search for groups would not be supported. + +When a member of the OpenLDAP group logs in to Rancher, she is redirected to Shibboleth and enters her username and password. + +Shibboleth validates her credentials, and retrieves user attributes from OpenLDAP, including groups. Then Shibboleth sends a SAML assertion to Rancher including the user attributes. Rancher uses the group data so that she can access all of the resources and permissions that her groups have permissions for. + +![Adding OpenLDAP Group Permissions to Rancher Resources](/img/shibboleth-with-openldap-groups.svg) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md new file mode 100644 index 00000000000..b3e84e98c15 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md @@ -0,0 +1,82 @@ +--- +title: Pod Security Policies +--- + + + + + +:::caution +Pod Security Policies are only available in Kubernetes until v1.24. [Pod Security Standards](pod-security-standards.md) are the built-in alternative. +::: + +[Pod Security Policies (PSPs)](https://kubernetes.io/docs/concepts/security/pod-security-policy/) are objects that control security-sensitive aspects of the pod specification (such as root privileges). + +If a pod doesn't meet the conditions specified in the PSP, Kubernetes won't allow it to start, and Rancher will display the following error message: `Pod is forbidden: unable to validate...`. + + +## How PSPs Work + +You can assign PSPs at the cluster or project level. + +PSPs work through inheritance: + +- By default, PSPs assigned to a cluster are inherited by its projects, as well as any namespaces added to those projects. +- **Exception:** Namespaces that are not assigned to projects do not inherit PSPs, regardless of whether the PSP is assigned to a cluster or project. Because these namespaces have no PSPs, workload deployments to these namespaces will fail, which is the default Kubernetes behavior. +- You can override the default PSP by assigning a different PSP directly to the project. + +Any workloads that are already running in a cluster or project before a PSP is assigned will not be checked if it complies with the PSP. Workloads would need to be cloned or upgraded to see if they pass the PSP. + +Read more about Pod Security Policies in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). + +## Default PSPs + +Rancher ships with three default Pod Security Policies (PSPs): the `restricted-noroot`, `restricted` and `unrestricted` policies. + +### Restricted-NoRoot + +This policy is based on the Kubernetes [example restricted policy](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/policy/restricted-psp.yaml). It significantly restricts what types of pods can be deployed to a cluster or project. This policy: + +- Prevents pods from running as a privileged user and prevents escalation of privileges. +- Validates that server-required security mechanisms are in place, such as restricting what volumes can be mounted to only the core volume types and preventing root supplemental groups from being added. + +### Restricted + +This policy is a relaxed version of the `restricted-noroot` policy, with almost all the restrictions in place, except for the fact that it allows running containers as a privileged user. + +### Unrestricted + +This policy is equivalent to running Kubernetes with the PSP controller disabled. It has no restrictions on what pods can be deployed into a cluster or project. + +:::note important + +When disabling PSPs, default PSPs are **not** automatically deleted from your cluster. You must manually delete them if they're no longer needed. + +::: + +## Creating PSPs + +Using Rancher, you can create a Pod Security Policy using our GUI rather than creating a YAML file. + +### Requirements + +Rancher can only assign PSPs for clusters that are [launched using RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + +You must enable PSPs at the cluster level before you can assign them to a project. This can be configured by [editing the cluster](../../../pages-for-subheaders/cluster-configuration.md). + +It is a best practice to set PSP at the cluster level. + +We recommend adding PSPs during cluster and project creation instead of adding it to an existing one. + +### Creating PSPs in the Rancher UI + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation bar, click **Pod Security Policies**. +1. Click **Add Policy**. +1. Name the policy. +1. Complete each section of the form. Refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) for more information on what each policy does. +1. Click **Create**. + +## Configuration + +The Kubernetes documentation on PSPs is [here](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md new file mode 100644 index 00000000000..68bf57e5d79 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md @@ -0,0 +1,196 @@ +--- +title: Custom Branding +--- + + + + + +## Prerequisite + +You will need to have at least cluster member permissions. + +## Brand Configuration + +To configure the brand settings, + +1. Click **☰ > Global settings**. +2. Click **Branding**. + +### Private Label Company Name + +This option replaces "Rancher" with the value you provide in most places. Files that need to have Rancher in the name, such as `rancher-compose.yml`, won't be changed. + +### Support Links + +Use a url address to send new "File an Issue" reports instead of sending users to the Github issues page. Optionally show Rancher community support links. + +### Logo + +Upload light and dark logos to replace the Rancher logo in the top-level navigation header. + +### Primary Color + +You can override the primary color used throughout the UI with a custom color of your choice. + +### Fixed Banners + +Display a custom fixed banner in the header, footer, or both. + +To configure banner settings, + +1. Click **☰ > Global settings**. +2. Click **Banners**. + +## Custom Navigation Links + +In this section, you'll learn how to configure the links in the left navigation bar of the **Cluster Dashboard**. To get to the cluster dashboard, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want custom navigation links and click **Explore**. + +It can be useful to add a link for quick access to services installed on a cluster. For example, you could add a link to the Kiali UI for clusters with Istio installed, or you could add a link to the Grafana UI for clusters with Rancher monitoring installed. + +The custom links don't affect who has access to each service. + +Links can be created at the top level and multiple links can be grouped together. + +### Adding a Custom Navigation Link + +:::note Prerequisite: + +You will need to have at least cluster member or project member permissions. + +::: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you would like to add custom navigation links and click **Explore**. +2. In the top navigation menu, click **🔠(Resource Search)**. +3. Type **Nav** and click **Nav Links**. +4. Click **Create from YAML**. +5. The simplest way to create a navigation link is to add these fields: + + name: linkname + toURL: https://example.com + + For more details on setting up links, including optional fields, see [Link Configuration.](#link-configuration) +6. Click **Create**. + +## Link Configuration + +### `name` + +Display name for the link. Required. + +### `group` + +Name of a group of links that expands when clicked. + +Optional. If not provided, the link appears standalone. + +Groups are displayed separately from standalone links, as shown below: + +![Screenshot of group and standalone link](/img/grouped-vs-standalone-links.png) + +### `iconSrc` + +Icon source in in base64 format. + +Below is an example of the Grafana logo in base64 format: + +``` +data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAgAElEQVR4Aey9d5xkZZn3zb/P+3mffZ9nDcDAzHTuqs49PZEhCBhBJc10nO7pHKbD9PR07p5AWlEEZM2ioph3dXGNqLgCxhVBRIYRmNy5cjqnTlWdqu/7ue5zTk+DiNIsCn4suOacrjrxvq/fle/7PofX+ycDZNT/pIAUaUUmaRBKp8FMQ1L9qL6S4+VYUikIeuBrn+LppiuJNG/BvHYd7HbhbXLBZ/dB8AkwFokCHsAH6Kb8IxTHJIyBHwgDMTBT6h7yWAmb1L3sxyEjfxkYxDDQMa0nWV0vyE2slwZ5JtUO8v5JUhgk0EkRw5TnUg9sgJm03lsezkyTIU0C2VrNJU2WsdtTPVQyAmYU4mGIhDESaULAcSG5RjwJEQ8YsxgEOZoOcSxjvXsyKM8nL24QJ4UmF5TnlL7IWLure/G/3lnn/PVu9SrdaQVApO3/CCAZAYjNSLIVBrD/VMwSnsf4/B38ofWthFu3krhhPZmaLJZuyEY7vJPIV++AwEnImJwJ66qTFcMLSswkGWKkiKqtMIICwB890zL/2lwoHGLYnCIMtMqP3Md5N7mMDRDTBkhKAURAqNBs3TOdspjWERgrmkYuJbQMEPUeKdAEIBrIuSIKEiZ+B0ShADz7W/Tjv8TMLOIjybxcJwNJOUgAkjHQSFoAkedMWn2g7rXKV/9rnfZ3DRCRjgoN0ik2A0nDLgMkbYD3aU7dPcRT7ZfhadmEVldA/IZ1RN6TRahxO8f37CTyxU/B4pySvOlYHNM0sZhQRKgwTRrRWALOZc0lWksoY30n0lPkuMBCPYCcKn/Ic632Iy/ivNvy+4k2EOAamDbJ/rIKFYEhx74AWPIWcpRhv8dyu5mQTEEkmUYTjSsaMBWDiA9mjsF/foWTH76Z33zqEL6jD5IgRkTeR64valfOQQSL6My0Ap/c32qvlNJgq339v8Z5fzcAkcayBKrFkNIBCiDOD/Lj8jG2iZWOwvGHefxQNY+3beHk7grCDS7067JhRwn6dZX4d72L3zVei/6Vz0HYB/EQJINKawRTQXVP+UfYQEjdRphITBlTNIrFqBopIsqcsZnaQYww0iv5yA1XkuJ+eRJREStJvrMFhnO8A5S09ewCDoMkCQUw67KxOMQMCBkpkok4JIIQnYGf3k/s7mlO7N3Fw7VX8J2Ot3Pi/rvBXMLMJKxbx+UR5L4CEJ0IKYUbeV0xAUXDZVSrvZIGeHXPff0DRDGE9PRZPhGec8jhBWcr30uXCWEG4Xf/wW+H38ajXVUcaS7GX+dGvyYX6jeQvL6MZO1lzFx7Mc81XkPyM3eC/xlIz5LJzBIz/bbUtUyKZaksUtPUIS06wzK/LGluM6jwrVg9wkCvECDOe51lE2kL5w2drdU+Ths5bSBbMacsVMtGtKDFug5+5Q00Iw2JFOhhmD0C3/goS6M1HKvfiqfxMo7t3MLv2i7F97nDoJ+BpG45IXLysgYRgIhvJPeRu4QVibZ7LX/+rgDiNLTT58IADjM4rPI8HyXpgYc+yXODF3G0ZyPPtZSxUFeM/9p8MrUVJK4rIX5NMfEdm1jauZ1j7Vfj/exNcPoRSB2HxIICiHI+Hb4U00mYSWkP4RAhHTKiUexggfCEPJDiUOepX/5W3tN5R9m3PpZZJ6bdWbJ+kWPkto51JyaVxbBnpYtcT35XwFDXt8Ee8cJ//wj9X6c40fF2TtZU4qspJV5bidawCU/HxWgfHoTwccuhl4srE0saRnwQwwaIZQKa+BH6B0CcfnvVttIBK8jmFId5zjKEmA+WhJTIkeoYYxY+t5/FtmJ8zeUEWzdypqmM07VFhGpK0W9wYVyXh15dQLxnG/NdF/NE00V4PzgAv/0OLB5RbyWKQD2CML5wV1KMbIlmCSgkapQEkcLCNPJ72mJAsdXl+Vb7cRh+mcnlQvKl0IomUfs2mOT28rwCaiErgiW+hXWOaBSnzRSw4/Mw/wR87zN4xht55vqNzNxQQXj3VoyWzRjX5ZPcWUigrozozU0QeMbSNAnIyA0EcaQRE1N8EPWn0hoSDxSSRnntfl73GkTMAsvXsDnCYZAXMERc2dei2i0HVnWMdpro4etYuv58orUujLatLLZsZKapkqXaYqINRZi7XWQ63ASacwh2lhPtvZwjdVs4M94ETz4M8ajFjI5TLlsJLavwsu0GCA84JpX4uEAAVHBYGHa1H3lVuZaQxXgvAIh86QDFDqu+ECDSIstS3AGWnCdmUnwRjv4Y/XMHON51OSerSwjv2kCqdRta81ZiDZWwqwjq8onWFKIfrIPQs5CIKl/ekJvZDyagFJJbWKFuiQFLePwfJtZq+//PnieNLc64lUuwTYoXMITwZowMgbSu3EIjE8NMiKMdhmd/zlzrRjJ12UQb3IQaKojUbiRUW8VSQynzjQV4mtfjbz6fSNN5hBvXsrjbzXzTZjz1V/Bs0/Xw0A9g7qRy3E3DRzARUJpKni0ZSljpEUkcCEBsQR3BYIEIC2mxw+WBV/dx3v950TP5UshpBxskqURG+cvCjtImwqyyDYZ9pDPiMKfR4hHQY5aJdPIIPPg1jrS/nZndW/E0lRJodBHY5SbYUEq4biOx2goi16+D1iLCO/PwDL0HfvdD5X9JFNwXP+vjyL2UMJDnUs8kRpzkjv4BkNX1/l9wlmiOhHL4RIbaDrA0vs0UwifSMVEkuSWJsyTxRACMIKSi8Nj3WWyphLr16PWFaPVlGDs3ou2swldXpgCy0LoWT+t5RJreqEASaMpRDGLs2E6w+u2c7mkgcM/t8IdHID2PSZAQUaJmSrkAypgXXrClaTIj5kZcRXXiKlH4ygAibXA2Yme3wUqAJDMWWDJgGEmiWgzDFL1hCRcjHkWP+kgaPkgHQUyqIz8l+fHDzLa/i0DdVrTaUmL1LsINBTZIignXVRCpK8W3cx3Jdjehnfl4970bHnsA0rpi/QWxLqUf7SiZ2pd/BBPio0kQQyVO/4LO/hsd8ro2sYQxRPYJSJSZJYwhHSBkWoJUABJRqTLRM0m08KKlPYwgiR99kZOdlRi71pOuKSS90wJIrLqK2M4yZTaEGy0N4ml9M96W89Hqs0lVu0nt2Ii+YztPv8XF8ZZ3EPrkAfj9dyF50gaJhscwEP5U0twGrZlKkEhKwDNkZ7gV27yC7ndUxYtfwkymMZMp5L7KF0pJNj2OmYiR0MMkRWsIoyY8MPsYia/fyZmxGzjdsInAjjKM6nIS1aUKJOH6UgK7LAo2FBPc5WKh9kJiYoJWF7I08G5SP7sfMrryLuZsYCjZJYlFp39EWKhIn4TBVy8gXvyN/2e/fd0DREoY4sqvsEo8lHQSCSV8Z5c8RDJJxAeRUKvmm4FEGDQ/S1+6k2NdG4g1ZWPWFJLZWUZi5wbiOypJXF9GfIcbrT6HYNNaPC0X4G2+AF0BxEVqZ7k6NvRuN96dGzjR+VZOva+LyMNfgOizYrwQJkIkY1qmhcMcSZGccTJmCF0XJ/UVAESuuRytOssY8rWjSGVf12MYsbDKyYh0z4SXyIQWIB60hIU2D7/6FjN37uVo56XMNpQSqXdj1LpIVBdj1JSi1WwgUrtBaQ4xscTc8u7O40z9hYTa3UR2uvD2vpPIA/dBJooXmLOfQwkIx7SS13X6RwIY/wDI2Y77n94TDWKVMCQtP0Q6QRpfJJRsJXyfgWhaoCFaJk5SACKM4ZnlxIemOd2+mWhTnvJB4nUlmDsqMK8vVwDRdxYr0yLSkEekIYfwrhyCjXkqmajVFlvMc0M++k43vh2lnKzfxqmpRrRvfRwWHlPVW7G0D40YyaSBGRVJLV66xSWmZPIV96yyZRQSnIy9JRDkK7m6NIES1CIYMgkykv0W3yvug+giaB7LnDrxK1LfvYeT040cqdvEUm0JdFRhtpbhr84mXF9MuL4crWYjevVG9JoKYhIKbypgsTmH0w3rCLcWkbjBTbD9SrxfuxvSfnyYLDmvpQIXKzSpoFceVJlX/9AgTjP9j2/FsYyRVKQcdWlr4QrHIzQhYUIsbSq7Py2Z8/CC0h4cO8rRw3uZb9mKvstFZJdoixISO8tJ3lCuTKxItUjSYiVNtTqXAotyVBtd6ju91gWNLpI7skheV4BevYHFhks4PnA9S/feQup3D4B+0o7WxIjrYUxdEogWs4j584o+DkCk3kku9YJIlTjikZROUsIGKfEvvJDyg+EB3wkSj3yD2dt7OTnwdk7UbySwq4pE0wYiNYUEavOJNJfiF5OqoVw55HpNOUICEKVBmgo4U7+OsBR2Xu8itvsyFu65GVIegsRUpkN1hiRNRVvY7+3UellZ9FfYBq+oAf/8ya9rE0sAItlZAclLAUTLpIkmJfQoUZoASNLrsV/x9Gg3vqaLSNcWkagtQq8rUY6nxPT9tWXKUQ/WVygJKlJUnFIHMFKSEqvPg+4K0rU5pG7IgdpSkjdUMn/NBmbarmLu1n7MR/4NPE8qkMTQCKSlhtcSnlIoqyTpn++nFz9CACKaUsiWyqI9RD5ICkLyDhF0YmYYQ1+CmJhVi7BwlNPf+zK/PtBGtPsSwlefy+w7z1P+RazrYhYbN3K6upTFunIFDvE35H0jDQVKMKg2qCtV2mWxJksFOHivi2TdxczdNQqJWaKZABKtIxO2KhZsh1we09JuVtWxFaJ/8dd7LXz7ugaINLb4FpZ/YYd5RZ07XJK0SoESaZOY5icj9dcSvZLy7Ie/y+mBBsINW0nXlJKsKVEAEfva11CCTyI0dWUogNRV2FEbkZ6lSoKKjR6pd5FoLMLcVUSqzkXyhgL09+ShS/a9fjvB7qs5PlJP8vufhugzKjEmxp4wsDyiVIorS8PGibP/oowhP4oIdrbOvi2V5VfHtLJyDhLhk5yPbmmN2Ax4j5L61bc58pGDPDTcyFNd78SoLoTq9dBUjL67grn6ck5Ul3GmpgJPQxXB+nLCDW4VwXLMSzG7YrVWW/h2ZqHX5cF1hcTrt+G5bS/EThNP+YjKmwowklEwJWJllZo4Cl4EnHqdF33h18aXr2uAqCa0TQtpaKvCVuobjGVTS3zAuCHJvCCYAZICEM2DcecowV0X2aZDKbo4ohLOFC2xTGJarKA66xjruGIideUqbyI5AUvTyPluxBwzatzqev66TXj73g2fvRWOPQpxj2KaU1qIZ2NhO5ttMbcw+LJ5btsjmYyE44QEHBKIcLZytDU2Q5KOkmqRrELELsBPZOKQDkPKC8YMnPkl+r/fydHhnfy+YRvzjdtJtl+GUVNMosaFVudW7y2ACDRUqPdR4Kh3tKZoEREK8t5WG4mppTfkE63OVu8c7ryMk/vq4fhvVeInYJorhJVIAyvULhUN8p6veXQAr2+ACCrEvLATcAlV8xNUA4WUHo+DRDWTCRnLEADTgxxDeAbz5naSNWXLwBCmf7kkibJw7VYCddvwN2zE21iKv9GlnHkV7arJg8ZKaLmUeP8NxO+cgF9+T4VUpZJ1MR1TCbuzDvXZ6FPaCYvKOzpaQrbCWYoEKOJbpfASZyEVYTERJGRKUFukdRik1mzpaXjk6yzePcLxve9ltmkbkboK0tUlZKrd6LVnGd5h/L90K8IgUZdPpCbb8km6L2W2fycc+ZXKhQTkOaV/RGWIv5XRVd9IxbAChy3cXhu64sWf4u8OIHGkFF23OiYOetQkrQoHxbySMYE6zD5FdLzOju8XK5NJpOHLJqVdJPy5Udnq3sZiPLsL8O7OIdyYrULEyVo3qcbNeOsv4Wjj2zl2yyCZn/4nhE6qkXppCSRImZZpkRPkUmkLqWdaaTeJ8xJNk4mYZMIp0mHNMhkTS6rM3Kpt8oG5CN4j8Mwj+L54JzOHezjW/HbmqrcSq5YQtovUNdkY165Hr7W1ggpGSEDiLycBiFGbpwAigsHTtZ2Z/hvg1z9WWjKsRk39AyAvDr2/xrciXYWBVmiQuCqjtgFiQDRkmSLL5obk1Z98mKW+d2PsLEHCtS8bGDaYRONIIk2rLVcOqwzTXWjJY6ElB//ubKVJQjuyiVaLlN7CfMOlHG27iuMHOkh842Nw9BcQ84IeAc2maBhiMuYiCJEgEm3j2afh6FNw5En4/eOkn3iUzOO/hsd+Bk88BI//CJ78L3juF3Div+G3P8T4z3vwfewQj7VdzTONV6roWrLxIlXGn7khH65bR2ZHNhKJezmgWHmsACReY2kQ365Cljq3MrPnGvjJf6j8SlTU4PM0iDUGRHyjf2iQvyZARHur6Ig4wFGVEFQdk4BYWPS8HCCOokhXP+Eff4XZjiuIVxe9IoCIeZLaKTa85bB7mwqYb81jti1HgUQ0iUR+gtflEb2uGKPhIoJNlzHbdDmefTtI3L4HfnUf/Pwe+OHd8B+3wudGMT7cQ/C23XhursV7Yw3ew9V4DlzP4tS1LIy/h7nRq5nd/w7m9l1FaLgeT88OllquI9xRjdm/G/qbibVdz2LNFczVXsJ8/RYWaivw1rgxGlywuwh25UN1tkoGrmT6l7MvANGr84hU5+JpzGexfSMz3e+Cb34GdC9xGaIrALESMsrekuG/UtHwD4D8DQAiDS9DPsVZd0I6CTFLRJJJNEUAEjnJwlc+yJmWi/4HAOLG3OkiVS0h3wKVPJtvKWCmLQ/Zirnlr8kjurOA5A1ukteXkthZgV5bRbhhM77GSuYb3cwJsHa7WGh2sdTsxtPqwt/qVhRocRNsLiDYlE+gMQ9/Qw6++vV469bhq87FaN5GtHoLwasriF1VBddeDDVXQt1boe5StOrNxHZVqfox784cwrXryezOh0YpmblQDTF+OaBYeawARNuZvwyQhfYK5trfCl+4EyLzmKrU3/ZDRE5lZEShjLBcAZC/Bp+8gnu8/n0Q24GV9heAWMM4ZRyG3TGSaLYdRFI+mP8tpz48ohKEEmmSTrZCti9/K5EfAUequkAlEcUOX2h2M9viZr65GE9TMUvX55KQ7HRtKZnrCkm8NwdzRyHpWpcybxJdm9G6thDr3EqkfSPR1irCzZWEdlcQ3F2Kt86Ft64An02B2gKVxJNEnqIGF776QjUSMlBfonI2oZoygjfIwK9CNfhL8jvplgqSzUWEG7Lw1a5hqfZNeOrWqPCtFcaVUO7LI2m72I48wjtzWGrMYb6tlPnWizE/cRiC4mNJlMQ2g6WfVFhXQs82QETtv8Y/r2+ASOPakRALICKh7LCW9YX6XUq9JcRI0gfHH+H0+9rxNm9RodhXChAJkUrNkoAssEtAUcrC7nI8jRX4d1UQra9QtUzmDS7YUQA78mBnNunq9aTq85h/73pmr8ln/rpCFm9w4a0uIVgrSbhyIg1lBGqKCFbLAC6poC2y/SUr6y8l+r7GXPxt+QQ6ixR5W4tZairEU5uPrzqPeEOxer5kvYtUc5Eqq5mvXcOJujczs+tCAo0FLxsYDpCk7aLXWwBZbMpisa2IxeatpP51BHzPWZM7SD8ICRgUIOz+Wf77tY2Q1z9A7Ia2+kEST45TbneMjHKNZ0gbGsTm4dff4Ll9V+HdVUFkR54aKBWuc7EakkFWMaGaIsK1JWpUnb+uEl99FYHaKjWuRKJGcSlf2VGiRt4ZNdnEa9cRq19DtH6dun+0RiqHy4jWlqhtrKYEi4rUtbWaIixyEa92Ea8pVNtoXT6epizmWtcx15JlUy4Lu3PxNeYTqi8kVluIVmNRTLROg5yTy1xLLvPNuYhzHWhwrYqkzVKitXbmEel2M9ucQ6BlI6cHr4VnfmbNpyUdY/uIFhRWRFVe29hQT/d3ARDBiAMQ9VYKNHZ9kgKIOIoJ0Bbgh59kpu9ygo1lRHfmvyKAWKCywBGqKyNUW6lAEarZpPyC2M5NaIoq0apLFLOG6yVnsA7f7jX4G9cpYMZqyrBIgCH7Jep4BYrqEhscAhKXTWcB4m/MwrN7Hb6mdch+qEEKDHMRMGi1+QpMAhABijyvgMG3y4WnsVCRf5XgkOvI9TKNlerawW4XpxsvVLVbZ/a8Ax7/nnLUpWOk9spRIhZabCH2D4C8+i3gaGoLILYaV2aX3QkSaZTaCylBic3BFw7i6d5MrEkYNl+Vi0id0epIRtcVWyPspCSjbgOxmo3Eajaj79yKXr3ZcpKlTFyNpXCrEvHFlmzm29YiWzlfEncWSb2X7MvWrv1a3p7NaIs5p0LT9S60+jwStdnKbEPqwa4VyoNrC1QBYXqHW5mSco5VmWuFtSUCJyFu+V7GdayG5Fx2b1ZmnL87n5O7zlNaa777Cvjp1yC2ZJm4diBL+mi5ImA5+/nq88grucPrXoM42kPgIPtn292ydSURJ9l09aPvGJHbWwm3lxFvLlYRHLHjpZJ3NRRqLCLYWKK0kfgL4m9odVUqShWv2YReu4loXZUayhtoKsHb7GapJZ+FtlwW2rNYaslV58t50Xo53yJ1rYYSIg0lRO1tZFcRFp191uguqX/KJ1mXT6Y6H3bkg+Q4ri+EGwphhxuzxi7ErC9S1crisCdqyzCry1TwQMbdhxrdqyI5l8aNGLtK8HbmcqrxXPRdBYR6r4AHP2dVDcuQAztXKH1k9Y/s2QLslXDvX+Hcvz+ALKsSVd+gZsuURLoCyPHH8Ey8B6PFhdkiYz0KiO0qItq4Ogo3FRFqKlYUbSwmtqsUvaGUeH05upSIN5QTbiwluLsYX0sRvhYXntYCfK15ioLNecQaXarcXkruHbK+KyDWWIDeKMesoKZcYjbJOJbw7gKCzS517aVWF/PtFkmoWL6X90vWF6mK5UxNMUIpSXDWW88qzx3aXbQqiu0qJlVTqTSIpyub2eY3k2krIrnvbfDtj1rjTaQWzh4DvwwQmfFFjc957YPkdQ0QR3sIJqSplQax2l6VlMiUm6rOT6KKaQj88gGWBq+E1nwyLUVEanPQmtzEmopXRZHdxUR2uxXJdYTijdY21uQm3Owm0FqIt80if2uh+jvcXIjWlE+8KR+tSfbPUmx3IbHd+RYJgGyKNOcRacl5PjXnWfdoKWapvZjZzmJOdbs52eNmpsvNggyFbZH3c5PY5casd5NqKCbRUEx8V4kyM+Udws2rIzFT9etLlgGy0HYedBeT2Xcl+hduBn1WFVeenWLIcRb/AZC/gvKzAOGAYxkgdhRRaq5UqbeNIhFaz/3nF/ANXKoAQoubUHWWYh6HyVezFYYWZtd259lkMXGoNQd/Ww6ejhyWOmWbR6A1TzF+QrLYdYVkJMrU5CYkz/I8soAk4LIoX4Vy/W15rKRQSz7xxmJl4lgMbwFWQCFg9LQXstRhkewLUOU3Aa7zrrK/WlJ+3LVFGA1leLuzWGw/HzpdJHsu4rRUCcwfeXGAqBGVMsJTek0lSP4q/LKam7yGNYg03AtImH0l2X9KM4udq5xASzjZVq8M2LF/MBIcue9u/AOXQ0chtOYS3HkuWnMBEdEmq6Boi4toSwFaS766jlxL9iOt+QTb8vF15CKmh5C3M5tgezZ6cy5mYz7UCxWo84JtBTyfXATank++dhe+9gLObgsIt7owmmQ8SolNRSTFZNttPVegvQBvRwGLXfksdMlWzrfuFWktUM8uzy/7kRbXi9AL26WESMsKai7FU+0i1FaqAOJrPRfackk2l3F85Fr43Q9UTZb0i5hZ0keqS6WPVg7DVUEV6UyJBzsduAI49kzyZ7veCuf/NQD2NweIemm7ASwbyW5FaShFYh8JOWMhbFTY2DH0FKFYnEBaJmawf9OTEI1DPIVKEkpnhDw8dfdBwmPXIqFW/7X/C63h/6J3FBDrKH4F5LbPLUVrq0Br20CsfQORjgrCnaUEu9wEevII9uQQ6c5C78wi1ZYDLXkgJlPTeQR3WxRqXkOweR3B3VmEmnMJiXllP1u0qwShSFcZEdnvLkXvKiXVWUK6vRizrYhUq4u4Ddjobjk/l+DuHIIt2YRa8wi35RNtK0BvdxHvcBPvLCTTX0Zqj7UfFaEhxyvKJbS7AL2t1KZy9LYK9NYNyxTs3MCZvnJO9RcR6F6P2boGGtbArlx8HZt5Yu+74blHIHpazSujy8yWUuErGJCpkJYk9C6DWVIQS9jDcmWAlcxbJgdYCRQVJl4xQbhUTJiq5k7mPZaLvXqfvz1AnBj5WfFgN4wARIAhWmAFQBylImLJljwpE8JpmXTZBoiIKl2mNMkQlepYmazg+GMcOdSJv/cdBGtz0Rr/mVjLG9A684h0Fq+Kwp3FBLtKCXQJEEoJd5YT6Sgn1lFqkwBPAOQm1ulC63Ap5jTaC0i0uZBttD2HyDIJE+cqRhZmDreKlrA0RailUDndgd2FquZLEnzeXXnonUXqurF2l2J+0QYWMCxm1zrc6nfZxtoLcY6LthUSEQ3TkoO3JQtfS5YCUrglj1hrAUabm1R7MYlWtyKjrRi9vRit3Xq3SEcp/p5iTvWXcKbfRbgzF7NlPTQK5aK1lnFqz0Wc/FAXxq+/BimZ4ySKaWhktKQFDBlyvDLEJdEUGXkY91ujEJVQtPIoAgM51NIvUq4SVmR98/cMEOHpleCQ/WU9LA6FTUqb2ClZOcYGhxxtJFLEjJTS2nYLKmUkQzp1meTTnIOf/RvHB65Fb96OUZ2P2VeAv20Nka4CIp3uVZGAwtu9AU9PBd49bvx7cgj3XECs+3z0rgswOrIsBmupING8WRUW6i1b0No2Kk0T7thAcM8lBHovw997Ef6erfh6qvB1leHvKsLfUUi8v1xRYk8ZiT0VJLorMbo2kOisIt6xAb13I9G+DUR6qwjvqSDUU0mop5xgdwXB7jKWWouQ6Ja3vQRfRzGBrnL1e3jPBoL9Vcz3livy9pap4/WOEhKtxdaUq40F0JxHujWHVJul/cI96/H3rmexfy3evvWEOwrR2oqItZURtSncXkKoo4iFnjKe6d3MiffVE/7OR+HMb0Am7ktrpE2ZsClD1ATNnsPMshIMezoima9LhuxafS7yUMAhW+vj8IYwwtH2mK0AACAASURBVKv3+ZtrkBf6FC8KDtU0KxpiBaDiCQMhmR9agUa2KUvaxJUenwf9d6S+cjNL3ZdD6zaodcPQRubb1qsSiWhXEashMXVC3WUEe0oI7ikk2JtFqG8N0T3nE+9eQ7IzC7O1ALO5HHP3ZhK7t6G1bCfcth1/5za8ndsI9V9OsO8tCij+nm0Eujfh79qAv70Uf0cx3uZ8fM35BHbnK5NHImDK92kW/8ClGFGAtJIsX8XyN8I9xaykUHcRDgX2lLDQVcRiVyn+7lJlthldZUpzpFvdpFsKSLflYrZnk+jMQutZR6j3Qnz9F7K0d41633SzG5rE9NpIoHMjiz0bWNhTgWdPsSU0ess51V7BM3vfhudTY/D7H0JK1qAKq8k2ZhI6c8kUkUzGsrClRF40iZhYQmI9KCjZUUpHOFqqxP7y7xkg8m4OwysOFyDI2zsSwop2yOwXzmErtzK6Wf5W/5j2HAEx0FOaWhIMjsLv72Pp5huItGyAzougqRRz31Zm2wsJdYttvzqA6J2FinnS7VkkOnPRugqXr6V1WXZ9ol2kbw6pVrH93Yjd7unaxkzPW5jv2U5kTxnx7gLiXUWWuSQmTGs5sdZKoi2WTyO+jfgClrlUgN6Vi96VTbxrHVrHhWida5ZJNNdKCrW8iXDrm4m0nUus43x1XLz7QoyetaR6sqC3CPaUwZ5i6Cki2eNSzxPtzifcnUeoJ5/Qntw/okhPrvJj2LUBdm0m0bJBaadZ8UkGyjk9UM7inlLlR9FZgtnmxtNejveDTfDYVyEzq2Zc8WCySEZNESTTM1n9qEqwQeYSUHPAyJgFMaDtyQGFPeRPoRVy89WAyd9egyyDwgGGAwirLFqcMGuCaqfe6vlbOdoBiClDVMW3k/GrMsWPcZzQo5/lzGc6ebarglBjIXRuhLYK4n2bWOouI9xTSqy7eFUU73JBe7aiTHuBkrzJ9kqMjg3EOyuJdxUrHyfWlaUcdHHU/T1uPD3lLPRsxttdgdaejdl+gTJhku35JDrcGJ2l6nytu4pY9xYi3VsI7dlCsG8zgf6N+Pduwje4Ed9QFWe6rNzH6Z4SzuwpZqa3jJneEoRRZ/tKWdi7gYW9FcwPVDLXX6a+l9/P7CllsdtNqrecdFcRiY5Com15hNoLCHQVEuotItxfQrC3yNrfU0S0pwi9u0iBOdlZRLqtlExzJWZLFVpHJYEeuUclswMVLPaVEepxk27NgqYLQbY9JSwObufY+xsJPHQvxI+rhT/DagWqJFEzoeYPVh0qnSrLRygUSKeqjrU6X76WiMzfP0DssJ74Fyu0howXsIBhgUN8CecIZyu2qFAsmSSeThNPmWoOWjVBmRGGM0/AL77KsX9t46n9F3O6w0203QWdpdBdTqijhEhvJdGeYtXx0vkvl2KKWYpJdhSTaSuGllKbykm3laN3lePtKWa+z83sgIv5/jy8vVmEe9aS6DxfASO9ex1mSzbptnzMrnxSfS4Sg0XEhyuIjFUxt7eS0/u3cmb87cwcvJ75W1tZ+mA/3rtGmf/INOZ370V/4F7iP7gP44efJ/ngl0j9+IuY//UV0j/5Mvz062qrP/B5fP/xUc588Q6euecmnvzIAf5wRz8nJt/J3P4tnOyt4GRnKTMdxfh6KtH3biWz/2IFVKOjklS79U7yXplWi8SJj3bnEugtZKm3ksU9W/D2bCXSWYnZng9ta6BnHVrdPyEROnPsYkKjb+NY18WEpnfCN2+HM/9F2jyhpmoVq0F8SS1h8b70tQjHsyCRCejsUYoCDvHa/741iAMQx5yytgIQmQjO+k/A8XyAOOCQBtRUOtCad1eNGpRZA71HSXzn4xw9UM+xfVfyXFsZsaHNJPZWqmiS0Vem6qBifWWEbcn4csEhx4v2CfVsINwtGqOcVHupKrVQ5RYSnu0uxttTynxfKbP9xcz3F+DvzVJOvNnxz5jt5xHtKFQOdaC3iuC+LQRGLsY/fQWLN76D2Vuvgf84AN+9HX5yLzz2HXjuUVg6DiEZHSnj1kPWeHZdVqKVCNCfIRn7LueF/OA/BY/+O/z4bmJfmWLu7g5OHtzByeGrmN/7Nvx9VxDu3I7esZVU20YyrZXQWm4JgdZiUh0u5YcsDmSz2F+EV4IEXRXEO0rItOVC6zqMhjcqX0zrkQBBKaHujST7Loaei1jq3U7gK1Okn/0OJGfUNEWJpEnAgGDaWqBKcGCFciWaac/Q6Mx9JoygzIdXw7iyrvk3NrEEIHYo9wU+h0gOAUYsqas1ti2QpNXU/VpcxzBkjiWZORD8yTAZAUZiFk79HP2rt3D64E4W979DRYciPZuI9FQR2VNBuLeEUL+bUH8h4b5CIr1uIr3Fq6JwbxmhgYsI7NtOaGgL4X0VBAeK8PVk42lfy1LzGrQ9Vpg30laI1llEvLcUY28xWl8eS31uToxdzLGb3sviv7ahf3EafvBJ+M234blfw+yzEPFDLAq6zPUlQ4nFiXVyRLIvs+OtklKylqBMYC1zZy2A/1l4+mFS3/8c/k8eYPHWDuZGriW4753oA5eS7tsCPRXQ6YaOPOjMJT5USnS/1F/lYu5dT7L3QrSutfhac5nfXajMQ61rE2ZHBbQXQ1shtBdidhYjkbRj+7ZjfGUcnvsRhE+QNmJqfi9ZWkfm+1KznCnr24CUzOIijrvARozrV//zGgGIo0EsA8oBh4BCZpaV5Q3ipoFuRLEmfLYYJGnGmUv41YqzSgL9/rtEPjPK7ORVeIYuITl2GWLHG50Wad0bCPWV4B9w49+bq8gCyWoBUkJo72Z8e7fgFZ9gpJLAeAWBURf+gWx8XRdi9rthsBIGt2Ds3YZv8BIWxt6B933VLH1sD8YPP0ryv78Gxx4C/x9AZlrXRDNEIBg5G/wXv1XW+FieVtQywxWLCAOthjIQ0ZJoyYwValeTLAgYF8H3DMz9Gn79NTJfO8zcjdfybO9G5nsr0PZvwhypwtxXQbK3HGOPi8SeCzH6ziW+91wi+9bhHShmsbeKQM+laB1vISPRw5YKCyAdORh7soj05ZIa2Yxn5C2EPjEIR36gZoGUyR7mE7AohpcpARd5P5lXQISpRjIZQktbK7682lOXvgYAskIa2supiea0oCLbNEY6QVQLocl0OCr0JwkliWzEVJ4jkzoGj99P7GNDLAxchlcy2AMlxPaVou8pJdlVCe0bSXdUofWWEtjrYnF/LovD2QT3uoj2Fa2aRFJG9pSgDVcR2l+JZ28xnoECtJEiOLABvS9XZdElJzAz9nY8HxtSfgKnnrKWbwuKiSTTg9raQGL/GV2tgmUtj+CYmxZWBCDin0oCWvalrVb7EQmsEttSki5zc6mtRAXl6iLDpdjwGOiPw6lvk3j4Q8x+dg9P3nw1j49cwrHeKlLid7WXkt5bTniomDP7cjk5uJ6FoXxi+ysxujeT7LiEZNtbSXRcjtazmUi/i+DQm4kOvgm616vk5cz+K1n8zCippx8EM6T632OkCMQzhGPidwhARAoY6EkfYdOrVl5XczKvtgH+gvP+tgBRkQp7PT+pwzkbrX0+QJKamhldqVgjjBlcsObXTcv6GsfxPPYFTt3ejm/gbZh9WzD7iwkN5OIbyEbrc5HsLoH2ckWJnlIFioXhXIReKUCMgVJCnYUsteYQ6HRhDFVhjm9BG97AQn8R0ZveQuTO64l/aQR+/jmYewIkiGAzpRIEUhKehFjcJBo3iCY04qZGAs2eBURmApGAhQWI/8mtU8Yhgb9oKkM4lSSc0YgisSUfCRaAU8AxQDTcb0g8803+8O338+Rd7cr8MnsvJdS3nfl+GW5bxtxgEYGhIuJD5eji8HdvJdJzGYHeS/AObMSzz4V3+M1E972BVOMboK8cfeRijo69g2c/PUbm2YfU9EwChlQiSTQSJ67ZdXWyTHVGpuQOqxls/r5LTQQgCiRnnS0l1VZoEGEGNQuGWndcJK3M0O6B6DzEjhH41Sc59slWZgcuJdWzFfZUoHVmsbh3HeEDLqKDOaR686FD7GY3qT3FRAfceIfcLO13E9orf6+OIgOl+PaUEeqtwOgpJdNdCj3lsHczxtTb8PzLTvj5p+H4g5A4LXljtU6JP5ViMawzGwgRNDQFCN1MIsuzyfuKNWGaGSsq5xTwKfPCWbZM5hqWBXGseiWnGVe3XVn45wRJ4qRFi2WiJNJBYkkfkZRHMWWcKBphvJlFwv6nmP/mR/D+6xBnRq9lrutSol2byfRvxezfRKyvAt9gBYv7y5gbKWVuxM3ScD6+/XmE9uUR788j05YNXW6SQ5vxT17Oczdew4l79pF54n5IzEBKSoV04ppOMmXlwtIk1eJ7ii9e5TDW316DOAARBWIXLQqTiGRVklJpFjHD7BIEWedCZicJncH87bd47Jb3MDu+lVTfRugsJ9GWh6f9As4MrcN/MJ/ovrWk+9ZCVxZ05ZDszVcACQ6KJikj8goBstTlVgDR+qvQ+jajD10Od7TC9z8Kz8oquIvq2WViackfz6QzLKUFKhLUzBCKB4kmIyQyoiXOJkOV7y02j+NbSORGrbEh64s4AJGrnHVWpSkdLfyXbOXiGbWIj5irTjLOWqJN1i6MabJMnMwSLzPoy8TYsGgmmU0nmFeGWIq0/wQceRi+fQ/6HYOE915NqGWLKtgUcza4rxDvcBaLYxewNHoBvuF1BPflERkoUwCS+jTJvxgD5SQnt7M0spWjo5exdO8gPHW/NfF2Okg6k0A3TeIpazHSjCwrZ8gqodY7v1r//u0B4jDACwCiwCHS1JnNXCIYRggML/hP4v31jzh272HmhreSEQ3R6YIOF+E9LuYHCzgxms2Z4QuI71sD/edC77nQvwZtMJfwYAnC0JIs1KSj9hatirS9LpjcgD5UxPzQJvx3NsGDn4b5p8CIKZQLvrUURAzQhcdVT6ZVLZKmiZ0vDJ5W852bmRSptEhHK/n5ohpBhIhNZ5ljZSO+jH2ZyC0WISMr28rAfclkv4gfGE0k8IdlASA7ky2HxSGyFLAEl0j5yAI8+RDpL91B+OYmYvsvJj3ottp+4J/IDP4TqcE3EB9cp9o6vHeTigDOdJYQGChH31tKRLL2XbkEhysJvO89nP54Jzz9HYgdVwBWII2nSciO4DlqC5BXCx0yu7u8qwVCJyehQgbqO8dRtjpVDnT8BZHo9okrStXVdV7498t+eOlgub5M/iYmlTT+HERmQJ8Bz+8JPvh5fvPhcZ44cAPcciUMCkByoN+NNlLG3LCL40NZzA+vUwDJDJyLUHLvGqL7cgnvKyLeV0Wytwqjv4j4gMvqtMESooNFimKDLrS9BcQHLBIQRQdLCA+WEdpnkX9ISr3LiN1VAz+5C+Z+CvE5VWQnTq/Id6nkliWRRQEobSDOpvRwRhxhXUlwicwlUjpGUkNPxRFzSyI5MuRLcmEOOf3hbJf7ZYXHptruL/1bhI+Ej6XUPGaQihsqfK4nU8iiQ/KEihclepZIk1RIF7SnIJqBeFqZPoYCV8oSCoFZePS7ZD49RnjyCuJ788kMnAf9b1JgSQ2st9p7oJLA3q3M79uOZ/92fH3l+NrWo7WdR2ZvLsbERmYPvgX/v43CyR9BJqiESzSWsCbhcBpFAdriWcWTTjvL1uFRtSti56yGtnj+zzPnOdIIEuZLRHzWRMpGkIweJRQHX8Jad0KTq0lvSLw9IYVkMjm0tVCNDGlVK4DZW8GQREaU8ycxBzt8n5Sl+WxGERvbuaSEcWUFc7VMmJQTqHU8vJCYA/0EBJ6EmZ/CE1+HH3yI1OeG8XygnpMT7+TkwEai+0tIDuaTHCwkPlRIdL+L0IiL4GghoZF8tOFc4vuzFen7c4kOW8doQyUqY53oySLZl4e+v5zo+EZi4xuIjZYQGxJgZMFQLum9uUT6i1jqK2F+oJyF4U0EDmxn7rZrmPv6nYR++6AFYDH/TA0jFkXTk6TsUv5lIbTcYbYQULkfQ6U6rVUUrVYQplT9smIrESvhCetazvmy1LSMo7BzI85WKjdtyiSd1W1F4NjHiaZQJHMi2ReWrf2gspF7Oc8hW/lbfSQULLNgJERD6giYxPSSY6TfVWjNG4Hjz8ETD+O9o5fZ4Svw7CmFsUoV2ZP2jPesx9hfgXffpfgmrsI79lY8feWkutfAwAUwlEVsbw6nRyvQ75+CpafsnE8aM2pYkb+krpZ6ULP5y4vI/QUrK7MG0lRKP69MPC+/qv1Sf3pzjrycNL6KMZuy4IosmSXrOFg2pyzMIq6gxdEyCCmo1vjLxCIktJgFBFuxpFMmYhsSF3tCA01UtzSmbkkXZxZzZ/bymAf96E+JHvkRoce/jf8XX2PpwXtY+vYdeP79MIEvj3Lqrnrmb78e7/veReCmywkf2kZ0spLoWCmx0SKL4YXpV0H6UCHJvhxM0RLDZUTHNxCcqCI4WkpoqIDI3vUw7ibRn6Wknj6xhfmRbfxh/3ZCH2uFR7+szD1JWjqfRCJBLBZD0zQljZ3vX3wrvWnZ/zJ+XiJVKxnTAYnDoMK4Vu/rloMumeWUQSadIiMqSqSVSB9RVbL5c1slzUTQ2bafLXUdPpNnEZKntLFjD2Kzw+xqQRxL0zjaRiaxFPmZ8SVhdhGO/Abzm59k7uYmTvZvwz9YRmaiBEbzVHLRN1BFYPhyAiOXEhnaBMMlMFkEY9kk+y9Q/ey9/Rq0733YWutEVqtKGKTj0ioy1WzMjmbZAHEALw8u+zZABB4yq7z4U897nxfvmOVvzxEAyMupj7Lzg6QTIaJxHa9usBCPs2RohBMBjKQPM+0V11I9WIogaWbI8BzwDKSPgnEEor+DwOPgeRROPgJP/wh+9Z8gSwR/+cNkPn4Lidsnid/aS+TA9cSm30ls+h1Ep68gOnUZkaltxKY2E5ssJzRWQHgsh/D4WiJj5xMZfxORsf9DZOz/IzL6RqIj2asChwBK2+9SmeDEUBnJkSLi48X4xzewOFbJ4nAxS4NuVQ+VGK2AkQLM0VICt1xF9L4x+O13rYnolJMr89IliMfjpFIpMlK6ncmofadpX3wrjCzhSyHLbBXmdCT4SuZUHa0Uhy3BJR8kmfCMwMoKdr7YViw6iY4JU8jvcm2HrCSb46BLyMAyQZbVuxxonWidZONOvhKRICR5KusJFCdawJRdeXjh4XgKluYxHrqfUx8Z4pnxK5kbLCYxlAWj64n2riXen0+mJ59kr5vwyDYiBy4iPOrC07de8cLMyEV4b3sP/PgOCB612wuloSWJLIsiSTDheRpkBQqsuj4Bk5AVKZRXUzLkxTtm+dtzfAYE4qBpcUjIEgGiQXyoiZ5lXTtZjEXq95NnVHUskT9A6Aj4nwLvf5M59XUyz34W44kPE3zkfSx8e5JTX+jj+CdaOHlXPSduuZaZm65h6dA1BKbeTXT83ehjV5EYfhfsv4K0lB60rrOofS10rCXTfSFmz/kkxLEeW09mYh2pqTUkp8/DOPBm9ANvInbwjcQOnE90NIfYiGt1NOzGGKtSlBgrQRsrwj9RwcJ4BXOj5cwNl+Ef30RkfBOJySrSH7gavvMvajkzlctIWhJbgBGJRBRAlltWBJiYpC/1UWgQ6b1CggtzvZBWMqpjPigT12JKh4dfbCsOdixpoKVSyq8xMk4pqMXcsqSoQUwZuSJd5Vbq/nIxYXKH5L5iItt8L5aHkPxsiWo7BJ0W/8qW5vJ4coK0kyyc+uzPCX39Vv5w8F0cE59utABz34UwKIGUtUqTL+zfyqmxS5gbrWRufwmBqYsIT28jNl1B5K6rMR7+FETFz4OIWPzqGawckWJ4555yXxsBFkAERJJ0PeuHOL+/VBedE08lEScrbUSs9fOiJyzJ/8y34NHPwoN3wPdvg/tvIfXlQ0TvGWPpQwPMvL+ThVvqWRi9FN/wJvwjm/GNbVQMFZjYiH9qE/6pjYQObyZwuArfwUoWD5QyN1nEmYkCTo/msTCUA6NlsK8Y9hXBYBH0F5Lqz1cx8uhArpLy0RE3oVE3ofEiAlPF+KdL8B8sJ3SgVEmaVwKQ1HgF8fEqgpOb8E5swDfhxjPhZmZqA3OTGwlOVOKd3Er0I+3w8/vAd8Raa1ySe7bzqurCEgkr4ma3tmiUaFTc9D/zkU50ACGdqrhcfAMxfSSDbPsJDvfLMSIsbV9PggGJjJVDEfloaRFHq5hKeyjZmZHsQep5f8tKHWFSyh6I2KFcuc3zJPELASIxhhf4RkozpXTSMlow4YOMU0Wlq+XvTPFX0mIO+sHzFNGHP8cfPt7PsQNXkh7Khv1vhulcMjeWEzq4lYWJS5mfuBzPgSuUsIrfUoV5sBD/SCGhuxvgN99S682L5SPPIs8szaIwYfu/0qwWCXjEz3PIEir2j3+mc+Ac9VKiNdIRCBwn9N9f59i9+znz/muIve8KEoc2Yx7YTHzqEiITbyE49jb8I1cSGroMfe8WGK6CoVIYKoEhqTsqJL0vj8S+XLShLEJDa/HvX4t/bC2+qfV4D67Hd2M2gZtzid5UQGK8EEaKyIyWYI6VKDMmNVZOcqxSRTK00SoiYxsJj2wiNLoF/8hWvCNb8I5sVcCMjJagjbpXTfGxMiLj5XgnqlicLMc/6cY37cJ7eAPew1uZP7Ad874BePJ7EJoDQypKlz0Ba3KOdFqZVIq3bPPKMbNeqgfOduJyb2LNdCdOthTnOSTOtQUMcYSFKcR6ERNHysPjCYmAxe0QsZSKOBGbF9864WQtIwCxAjGOLyq3UcyzUhLb9xZQOiFm59klEKFAmohjGCFMQxYp8trl60FSpoDFHs8hyU0zCAkv0T/8kvl/uw3/eBXpobXo0wXoN1eg37gR7/gmTo9eyszEJSyNFhCbyiE5uZ7wcA7+qYuJfX4M89nHl4fjOs8ijynkyBILOKI1rECI1WK2KeYIJTn5JT7npPynSIdnIBmA8Ck8D3+e37x/J8+OFmMeLoDRN8LIG0mOZhMdLyI0uYHQpER7yjAmpPAvm8DeLCKDOWj78zFGXSQnXKQni8hMuUmM5dqUTWIsG2M0i8ToOoyRtcRG1xM56CZ4qJjAwRKC08X4J4vwjxcTGCkjMFKhIkcS3ZDx0+GBzeiD2zGGLiO1/0qSIxersG5srJDVUGSsSDnkgZES/KMFeMfy8EwXEjjsJnmzm+htF6F9dRKe+xnEJaMLft3qbmlXMyVh2hSmaSpSOZuXaOwX/mR1rGPDixx27CdhfWdfutkyfWTPAYcDEHVN5b+IDyN5ColWSeTKIJOMk9AiJPUoqXjMmjBBQrLyezqpQC3Xs3wJxyKRu8i9rXs6v8tWgUcBR7SaDV47IJYwIW5CTGaXkSy8MsAkT7IEyNqQHtLxBTJSjStMLNp39jm0f59Ev+sqZqc3sTRRijblUn6nRK/OjJah7V9LYug8jLE8olOleCY3MHvr1cS+eYe1arDSThaopT3l6eV9HLJmQHkBQBzwy/bPAURlZePyItKwUcK/e4AnPtzOsclyuNUF+/8X7P9/SIydS2wyj/DBUsKHK4gfLiFxuJTgeAH+cReBCTehCTeRiSKi4y70CTfx8UIFltR4AamxfMyRPFJDOST3ZZMaXI+2fz3esRwWpnJZmipQzOmbLiI4XUJ0qhxtagPahPgIm0iObiY1shVz+CIYvkRRemSbKgpcDTjknPC4G+9IMZ4RFwHJ8I5egH86i9BNbqLv38T83deC/7dqzUDhKbF5fRlL4lpdIZ2dUqaVOOcrASKgEd/kpT/SQ1bnOSsvWdEWcTytDLbjBJ/VWfYKthkJKftAPw3h58D3NMz9Fk78itQfHiH++x8TfeKHJJ9+CPOZn6rv1e/+oxA7AfHTYMxb0UUpo3cYXkwhO6sugBVj7awTLuFKu6hSwsiqzN72n+xKY9FEEviRtXZ1Ff4JQnqRdPy0BZC0rFNoB4bk9aNPwX99kOMf2MGxye0EDlWg31RM+MZStEMu0vvXwPCFpCbdyg9ZHC9jZnIz0Q/VkfrG+6wlLdSzW0EOAbGAYzmqpt7BgYsEQ+zkohz4lwBE5a1slS0Pju7H+8h9PHf7e5mfKCA2/Ab0oX8iPHIBwSkXnuly5ibK8E+Xod20geBUGcGpCkWhyQqEIpNliqITJQjFbNLHixGSaJFFbvSJfPTJvGWKT+SzTOOFygQTMywxJtrITVJotFiR/C1AjE0Wroqik26lrbxKjWeRPHgh8Yl/JvIvJfDgYdCetMpaZGJ4ex1ykaSW0HEkvLTyS38EPLpujWER08v5SA4jri2RSPqUByDgEEhJ5wqjiQcTjmtkpP5MCjNTpyH0Gzj2Tfjlx0g/cCOBTzQS/WgN4bt34L/rOjwfeA/zt13F7C3v4Mwtb+fY9GWcOHwFc7e+E9+d16J9tJbUZ3bDfR3w5X2qnJ2nH4KFp60aNwmjSoLWtH3S9JI1mMk4qRK1GVl70IgSNNJEpTRAluU1ksrmk5GAQUAMLClVl9IaaTe1aKdjZknEzp6pRDGoVEZEn4Hf34/vi8OceP+78N6yidShtaT2/2+Sg2/CHF5HeiKPuPieN27Ff+M2jOkNykcJf+cTVhY/EcaMSvGqNf3TYjyCpoSM5YNYsEla95YucwDidMaf2J4jLyDk5DvEZDCO/YLFL45wfHoL4YlczIn1JCfy0cZF/ZUpM0tCsGJmCRhCU2VEJmxgONtlkJQRnRCywLJyG5sQoLjPgmA8n4RNFkjyiE+cBY/Yos+jyTxik/kvCQ5R2S9F/uFcQuM5hCbWEphah3nXBnigD05/H+ILymwRwSFttBwOtxX4yymWE5CIMy+RLdEulo8i/kJSMVw4GkI3pHpXACg9KEwXAP8JMid+jv6rL+K7/zBzn2rm1O3v5PTN21g4UI55axXcVAaHy0kdLCI5VUR8sgB9rIDYmGSkXepv0ebaeN7y97GRPPxjpfhueQ/zd7TgvWeU2P13wi++Bsd/CsGnWewSOQAAIABJREFUrahl4jikjoM5Axlh+6gan+PLZPALMGI6BMOkQgaiMB1eEpDMZ2TuEqvd5L2U1lUmoJPIdELWsrLvKTVoau7rh/j9wUtYGvg/MPZPcFMuifG1aENr0MfzCB3cRHi6HCZySB7exKm7u+Hx74MmGnEJUrIMuEwimCaUljCEFbZQ7SlWkoDT0RxnZdWfgAeoRKG8lLyIkPrI6Lxf3Mfs+68ndnAzTBXDaCEM5sCgC0bKYX8pZl8+KQmPThYRn7C22qSzLVLfy29/mkrQJsrRJiptKle5j9hkKdHpEsIHilRFbuigi9DBAgKH8vEfzrUpm8ChXGLTBS8JgJcChzBScjKb6NhavBPZ+O66nPSDN8HSTyB5gozhU1BQhpI0pmpQm3ntBJ/V2k7D/fFWolkCCgGIbJ19OVKYRq4dk7XSRTWpS8cgeAaO/hfmw/eyeO8w3o8147ntXSxOb2RJqgRGckiM55GeyleCy5i0BIs+nqtsdW0sh/hoLrLlcCncWAIHizEPujGnXDjHByeLmb3xLZw4fCUz0xfhnd6Cfsul8OHr4EsD8K33wZmfQeiYmi5GTC0pwg8SJYIMWJIKWystYbstjnqVOfsIJayRJSJYxMhRIJE2VOZZ1Brbk05jZsScFDEUJn3mlyx8dRrPwU1oI2tI35hN4uA64pNr0cfFhy0iNV4E09nKxD86fSVz98mIxAdAgVnK8wXESfyGGHp2vylgiI8mZDvqf9xdf/TNOVa1pl37lcpYGVnJpi88hv6Nmwh94N1KnaWH1sPgBTAsodkC2F+golWm+Bc2JScKEDImz5IwoZA2ZZEw9DJNFdmAqLSy41PlyvcIT5cSXgGQ8IECwgfzCR3KJXQo26b1hA9m/1mA6NNu/hQZ0wVwcxHBiRw8H3gLPHirFeJOLZJJBkiZcdWxwruKeUX6qQiRdKZjqyvD9I8a1vlC8iOiORz/RLYCFkVSniP952BOxpk/8wsSD3wC/z19LP3L1fimNiFCJzNZCFP5MJUH03lwoAAOFpAczyE+kWMx/VQeyelCktP5pA64SB0oIDFVsPy3hErTh4rIHHYr0Ji3lBG4uRLfjeVEDxRgjGeTHlkLYtIe2IL/pqsI3TuM/zsfRzvykLUwJ34y+EixoMaLCOOr9hHGl3yORN5kdsSkTsaQ3IP1+/OOU+UqkuiU8K+pJsEU40hpaIl0Lf0GfvAvRD70LuZG1pO8OZ/MbRXKWtCGs2HKBYfEesjDe/PFPHvL1YTvPwCen4J5EtIe9ERImaeqH6R9paHl+eTeapIQK8Ln9NOf2p6jXkYShAkdMx5VJM46yVk48QCeL/Vx5raLmBk+j8jUGzBvXoN2+Hy8h9YSuTGflHTYuE0TeTCRp+xFczJP/SYdJIwoYTztQAGxgxZFDhUgFDvgPksHC9EOCBWgH8jDmM4jMZ1DaipLkTm1HqH01FpF5lSWurZ1jnPu87f6QRcvRfFDbvzvu4zQNw/D4qNKikXSCbwJqYsS7rWljlOEqVryLwfICxteTCtHm6hybcl1BL2YR39J8PufZunTQyze9l6CUxswhBluzIcDa2HiPBg9F/avIbN/HYmRXKUlBBzxqWzVVsmDeaQOFZK+sZDMTUVkbnKp743pHLWV4+JTuep4fVLOuwB97P8lMfW/SRw4n8RNBRg3V2Hcsp34TZeh3XQ5wUOXMX/wck697z0sfWEvyUfvBd8vgTNqftyQDNu1eU/5LhLmjS9ZVdcyMExK9O2aNAGSEgiqUSypEIj6kJiXaFJxaSxTSIYzHIHffZngnVeRvHUzocMb8U4XEz6YS+qmfNKHRBMWwPu3MH9oI8++/53Ef/JBMJ6EzDwyGjMptWdyTUGpyDYFEvlHqgYkAGFVD7ywj1b+fQ4pAYc4ZGEy8TAyDkBFtERCmrPEfvkZTt1Ty7ED2YRuPI/UbRcQuvHNzNy4Dv9tJYqRxYESElA4JMBJKHo+QFaCRMAifys6mId2MG8ZGNKpFjhyMCdzSCvKgoksGLdIvhPwvRRA/n/e3gO6ruu689b62swksS1ZYhFI9F4JgBSpFpdxquM4TuI4zngmseOZzKRMkkkmK7YlkUTvAEH0QoIA2FQs23KXYjuO4xa5SLIkq8vsRH29l9+3/vvcB0KyLUfkzGCtg/vKfe/du/f+73b22UcA+GlD2biLLXcQvffvXNOAxJLNtCrQlEYT482tMq2z4T14BJZ2FISMq5tp+hMfK+4QMF7xlwzAc18i8+URLs7+Oc93/ibnD9yB/0CTaX8+tgP+7kb4nze4499tJfP3BaQ+VkHi7hpi+2sJfqyUwN2FBO8uIbS/mMiBcqItZUQPVhBrLScukLRVEGuptOc65t6PHyyEljeTPfgm4i03E2wpZf1gLWsHGlm/R9a8nszBRuJ3KdOnJQT1vNT5qyyf+Esy/zIHF75pzTKcKlHZiwJ8zXOsQUxLE/TcZZfMs/KsiYDiwJIhkgkSS0eJe3OiTq8nHG3TF+HRY8RnPsgLd+3j/D2NRLtqibeVEd9fDa31cLDMUvLP3r2Lp0beR+KJU5A556JGscbhwf2gQKLrsdIUxSeql35t/l1nlWWZhMuZJxLmOMjUKS4x5q/8kMgX+swFibdUQlsJyXuKWLm7hrXWZvwHawgdrLQROVCJRnT/lRG7pwo3aoypYuyVUUXi7ioS98ikv2rodRubz68jcXcdybvcMCG5p+Y1LUT8YA0/bSy338nlEx+DJ78IMZXSLBGO+S17pOktBZqiw4YLYb6ry17JZ47napdeIfU//kRxh4oXFY/oT2BZWVkh9NSXiB95L/RWE7pnB/67dhBuKSPRXUeso45AazXLd5Wwur+SYGsdkY5dxDp2EWmrJXSgytzOtf1lFospPlNMpqPvYBm+gxV2zL2v5/6WcgKtVQTbKgm11xDqrGWltYLltjKbvF1rKcR/sIDAgZ2EDuwkcqCQ5b/dRuxAFfTshZ7biO/fbe5w4uBt+A69h8Sj04TOPALhZ7117EpbuwIAzY3YxKJueqOCO7cAK2OrK6XJ1YQhHlGdX2xjnkeTj6aok0skHr2XlZkPstZ1p5tMPFhF9GA1mdZy+Nib4WAeq+31fL/jLTx2/G8Jq/5PYLVCtE0+3gZABA7FKWGviu3HeZZ75Tp/yN1QOu1KTsRCmbt1FeXqiUozH/88q0f+C6udd5Bsryd9sIrwXXVu7uNgDb7WSnwtVfhaagwwAk3wYJUxUYCJSfhfIfB1JO5qMGF/NUB07mZQRQWAe2Sp6ojsbyB6T8OmY50BUlozdLDcgdPOd6DRb4dbq4m0ViNwJw6WE28pJ9xaZ77rM92/R+Kxh12aUKXqcR/B0DqBVNqyesvqGu/R44olcbZapDFxlzR4SsjccM+i67H9STCSKbLWmCFmWRbOP84LDx/l8dEPsL5/O9x1HRz4t2Rb30yybQfR9lLCHbUEOxpZ72hmtXMPa+1NJgTrrVWst5ThaynCBLqjkvWOatbaq/C1ubHeVk5uBDqr0PB3VNr7662VaKy1CBiVnG2t5XxHLcudVQS7y4h17iTevo1YyxYi+7eSaCsmcHc+vr8vspQ8B13GjL8vx/+3pTx5cB/Pz3+I0LcnYPnrNuchDS3ayL5KX28EyoYT5ZVkc1RZmySuiT4VXmqZg601d+ltNbSOJNLus5EL8J1jrE7+By7cs4fQQSmJeoKysAe2kr3rBuhvYqnnTr7R8hYufL7TpY7Da+4idCGObRvunmZpsqb+XmXVc3zzjtcJDLoZVw9qmXd3aznG68sTARKPf5zHR97PSx23EOtugP0lcFcByZYqp9F0we27CHgj2F5PpK3GzHuypYy0xsEKktLoLbtsDiXS2kS8tda5AXIFNkYN8dYfH5pA0vkbx7Yqou3FRDoKCXYWE2qvInGwieSBZvt8qKOCpdZSIr01JNuLyLTuINmxgzP3lPPs4T8h/O0vuDb7tt5bvrKIdWXWWs82GGwEExLcMACozGI15NJQXgmIKKg158FU0vbCSMdyRVspiKzC9+7n8tQfcv5gFYGOmwn2bCXYcxOhru1EOgWOYosFRKf0gVrSB+pJHqwl0eJoGekoJtSZT7B7B76efNa7K1jvqmG9qwrfxqgg0PmTxuZzquwzq91VrHZX4OsuJdhVSLRjB4n2PJLteaTbdpBsyzeQyE1ztN9FtKXZSkKiLU1mXaQYz/XcwYXjf0zyiaOQesqcVFkE0UmeZUpC5imSZEa9zlQR7pSznaS5l2zcQCWrLbmU6Pm1nkWPkkvw1KcIHvvPLB/cQ/hgLQw1EW8vhJ5yaKm1xESwpZhLfXtIffEeiDwJkRDZtRgRrb7wem35jLFayeb3MiSvQsWmp7ZgShdyBSBrriGARfwuKWGB1vJTXPiHQzw19E4utdVDWxns30HmYKkJtsy+QCGA+Dp2EeioNYGNtpcZgdMtpQgoidYqoq31hFqbCbU1GbiiEvSrGe1lRDrzCXXtINBVTLCjinjLKwHi76/h/P6bCR7YQranEH9bHi9338rlz47DyiWX1dhEkNf1UAwPu10qxcdg0jWyE4OV4wqq4Zssh+q3tCXyN+9nfeqP8Xc2Q+cW6L/BgLHeU4yvu5hgZymR9jKibRUOJC1VZq1NsYh2baXEPYUQ6nRKQdbB11lLoKPGhmggxaDvsdFRahZJVkm8yA29F+oo2wBSsLOMiM7tKCTRnm/AEDiSbYXud9sqTOFJCfram1lv32v8o10ZpUJz5852N/PS7Hu4/PB+ki9/BmIvuSJFT6/YOvtE1lZPJtTAIhs0bFyxzq4qV+DQkECvqdOKiCsARV6C7x8jevQPWWm5Bd+BagLt5QQPFpFRVq+1Cva/mfWPXk94+l3wvWMQugShhLUz1QLnFa30FGpl1mJa3++h9qcwfsOC2Ps2kSJz57Wdt2rVlNXykF6Di9/g/MJf8lLb7SQ6a6GlwPLUsg7S/rIYwXYxS2C5ApB4WymJVoHDnScLIOsRad1FtK3m6sBh1qPMNJ40b7CjglC7rEu95celcQW61FA9F+/eynrrVoLdJbzcUs3K/B/Dy/9sltHSfj+FOP+ql0XfpDCgYj0tNHMzuaYBVfekVObqM2T+aZqzh3+f8wf3EO+phwFljPJItlQ469DqrlfCHegsM42+3l1qR1+XE2RZSCmieIvucZdZy2irrPEuu29peN23rLpAJeXFwWJQMH5AI3/T2GHvyVLLXdH3ihfio7JZOX4Zz9pUSLiZv/WeEqwnoc8drLCM5lJrOc8dqOCp7lu5cOrPiH57DvzPeVbatTbS3jnmlJAgquSQVUCahnbk1ky7uWAu/ltVWbspa7li6mTztLlbvpk/4kd3NxHpamTlYzutRCjTWQWtO4jdvZPVrn28NPZBOPeoldEIDz6to1e1SQ4gWuJqrsBP5/R1uhxZHPsTsy01JofNXWhErT9jEYfg+EUyXz/GhfEPsNzZRORgMdnWEmgpIdVWRqK9nGhHtfnPCgDDHdVEOyqJdZSTaK90o63aiBpra8BGey2x9uqrGx3l5jPHOguIttdZABdrq7PvT7bWWnYj2V1JrLcUf28BL7SW8MLgu+BbRyH2sosHbG4jR4DXd8zRWRv4JBPSd45msWQIdX20tjzrPyTw5RFeGHgnZ/dXE+2uhqF6/N3VXPhoIamWBrIHGu1owtZRTaizEn93OWu9Zaz1lrPWU4m/q5ZgZwOh9mairbegIDlxcB+Jlt0kWptJtjSSbGkg1VJHVu6H5joOVMKBcpsz4ECJc4v3F4GNAns9c7Bh4zs2f0/C6FhHzPhT6/FVvNX1uRHsrMXfXkegrY5Ie4XxOXCwiMsHyjjXfRsvj70P/5cOk9I2bEnFGBmLm0MpBeqebHoAkQCbrFrHlpAp+JxLJKoaLa29rEpTnod/WWB56kOsdr2V1YNVrHeWEuoqIbq/kExbPdGefTyx/y2saf1I+Ix9uRZ+ZjX772UlfxY49LPXuWpHl4qzqzKnW0hxacxQJkI8mySrzmJqjLz8FMmHB3l54Je4IC0lgBwsgBb5+CUeSCoJd3pDgOmsNALHBBIDiKuWzT2+FoAkOneS6Cgi1tpMrHU3UWNoJQJIpqXatEl2oJKLPSU83Xcrvi8M2T2QuAgpr0jz9eFi42wx1NyBTMra0rhIUAyIeKnOF7n8xT7OzPwel9oriXfsINtdQLgtn4utNSx37SHc3kiitYGcQG4+CuzR9gbCNhoJdjTj79iNr/MWfB37CLbfYvctcCRa3RBIHFAcWDIH67DRUmvAEXjcqCZ7sI7sgWYyB1T7tM+GyjcSLfre3UTbmu367Pc7aj2elhPtLCPaWUKoqxxfbyPB3iYHkLt3krpru1ksxSyXWhv5Yfcvc+nBj8JLD7vFd7h17FpLY4pZRPQeO4B4JfGeJZa7agAhSSq86rqvyN3yPQ/fu48LYx/kcvcdrPcXEuzdYUWvvv3NZHpvY7XvLTx39I9IvPAZNx8jFNr+h2tuojYHyg2O/viD61wkL6Z6pkRXrce6IOKEslpxBiGVsvojbrHQi1/iwvE/4+Xe20mq4rc1H1oLoa2YdHspSdPsFUS6qgh31diIdNYQ66wh3lFFsqOCdJsb+ny8Q69fxeisINlZaL+Xat1NsnUPkc46Yp1VpNrqoKXGViKGW4t5oaeWix//b7CkAsSwt1xWMx6yllf3J3qHbLbdBfZJdSuRo61KhMuPE/nnKX44/C6WDu0h1qfrvJHowZsIthbh67mF4ODb8HfvJty5y1K48fZ6Um31ZFtUHqIcvzumW3eRbNM5TQS6drPevZvVnj2s9zQR76izCcF0Ww2p9hq772R7nWUb9X3x9l02Yu1NvHok25rg4G44sAe07ufgXlIttxJvvZVI+62EO/YS7thDuLOJcFed8TPWVU68q5h4Vz6R7mJLgvh76o23sbsLSH30ZtifDy3FxPYXs9JWw/Pdt/PywodJPn7CGuhp8ZTcHO2zmrMcOtqfqollRWwSz4FDHR8jiaRrqCehlnyqgjd+iew3Frk4+X6ebdnGxfYtpjwCrXeQ6GgmPngrz/btZfWLH4Nz33OVy+qpFl8hop3Jchm23G//hON1Lpz0lkjqKjWsVMXlq0PeSjMtLU5Z0VAYwi8T+5djnJn9AKHOJmgthjYBpJBsezGpjmLineVEuioIddcQ6qqzISJLeAUgAcmBqcJe0+uvd8QNIKWk2itwQtREpLPeQCIhsaCtQynREi7M/hbZpxe9+EqqX9ZDSdxrAYgaFsQ2Mi7GOJWOX36O5FcmuDj2XlZ7dxEfLCbRl0eo4yYC7fnE+htI9N9GsGsfa917We/eY3TUHEe6VcBW4FvhuUflcLAcWitMGYmGEkh9r7+nhnRHPrRth/Y8aN9JuqNwE/2dggp0y6XbZUOg0ljrabLfNDAeEBh3kVWCo203sfY9BoxQp65rt513BSClxLsKSXXuJNmVz/KBbfjaipECTHU0uMLJllrMpTuQB4MV+A7czPP7izk/9154TIHzC+ojxLpfcyE5C5GTfEmpBND8IZsLUTtWvysYNqsTVY1XDlnhs6S+Ps6PRu/kTEcJgf47ifW9zVnezkp8PWVcPHwnwS90w+XHbFWj1qRoN6t/JUAkIF6K0wq6nN3R78tPVMQvXzDnM2ZDPrde/fJ3OHf/3Sz33EmqvYp0e7mBQwBJdxSTVEamq9wAEuiuQ0wSUGRVBJBsuwOTgPR6gbH5fFkk+31p0LY608ahrl2mWdNtVXCokXDvHmKf/xisP+Z8YNMUihY1HbgRgf0E/fHaL4lGIrI/rZy+Z3mjQVJfe4DlsT9iuaWRbG859OWRHMonNlxOeKCBgK6vvYlE5257LOENdeWArRKRGgeSlgpXaqJyk7ZSUh2lxEXX7lJCPToWQsdWaL8B2m+0x9mOPBNeaXk7r7uCQE+VgUmA2jzCXVVk25QidQAUvaRYBFRZDSm/QJfGrldYkGRnManOfJJdO8kOlRnoV/aXEm5vJttzJ3TeCm210FHkZeu2kurbwaWeal6eei/+r4zD0nPWTUV0Ew0tHthoLqHn8mpUzRG1OrbcBIR6GKtNlarP7XNKKCWeg0eHeHn4VzjX2kS4u5FYdx3J7nIYKuXiR29ibeJd8Ox9toZGa22UZRQ4za17DTZfl7swd3ruY1dWk+kiNPRlTpSEbK+kIPoMkfkP4ZPpb62GgXrSncpW7STdXUqsq5RQdxUCiLSejuHuCmNytiMfjVhHGeHO8qsaSgCkuhqtvit2oIB0Z4XNxay1NhLurCfZowRAJckj74FnPmlBuXzai+pkbskI3ZX5k69Bop/+lughJ031SM56aFL1q5wb/nNSg78BPU3QkU+m62bCfSWs99ey3ttMuKuZTFsDaNXigTxC7TuMVom+GpJ9TcS7mwm2NrC+v8YeJ7oaSHdVkeosJNN+M5n2G8h0/Dy0/zy03QBtN5Jpu4lMxzZSHZrr2WlaPiZXqLuEaGcR4bYdBFu2ET6wlcjBrTYRmGrdBl159p2a95C7muqrRNcR7Kpktb3ceGZ866rZcLGk/OQl6PxYdz7hnmIDeKBzL+H2O4m13UpaZSDyKto10/3/km7/OeL9hVzuv4VzY79H5stjth+I5E9S5/jgLIr60qXkQtkut5I397oBwnNyVAakoaJEsqrdet7iy5WhO4n3lZEarLbro7MADlWS6Skhcuo/WlcU/d6lgN+aWPxsgEg+7Cx9TDiVCLl8tK039laKCekK6LM57Cn1pqzCPw4RmX4vSwoauxtJ99aQ6Sol011MqrfMABHoqWG9VyCpI9RdgRgn10Aj1lFyVeAwUHVUk+7Za5mWRPsO0t3FRLt3E+y5lWDfLgK9Daz17IaH/hYuPm7VnBJoFUSL2BllM67hT59e9poMWhXrxSfJPjRIeOB3Sbfshu46aC8i0V1CoK+a9b4GfL3NRDsbnYZtKyPdmUei52biPQWEZRm6a1jvrGe5vZHLrY3Ehn+RcP8tbib8wA4C+28k1no9dL8RBncQ6akk0NNowbK/bzf+3lvw9d1CoG8vgf49BHsaCPfUkuiuId1dTqazyEBLWx60bSF04I1Eu24i2b+D1EAB0d4CAp078XUU4tOkZLfc5Arjo1zmWKcUXLl5AbJmod5iAr0CUj0BuWQdil/2uoqL9lLo2k6m5Q2k2n+BdO92/B2lXGrfRWL29+Gro1bek0jEyCTlp7j2RLYNQyJFRk0Kvepbiagstv501NmSVFecouW/wIUfwmf/gtWeUtZbtsOhCtBE4oGtZFpuIDhxJ0vfOkI4eMHm8TW/8rMBol8zJSqAyCf3AiRVYOY60lg1pkCTm573Yhah/OI34B86OT/8W5xr20usrxH6q8n2FMFgGdGeMoK9lfj6am3ocbSnhHRXgQ1ZGRH+qob83r59pHoarYgy3p1Hoq+ZyNAd+AaaOdfTxNLEb9rkkm1Kk4b1jNv5QgBJZLwNQD3Cv96DeLLkbZlH/DzZR4aJDP0WdN9maxfoqUPaP6R4oW+XgTba00CyS8CpIdtZSaq/iPhgAdGBIhM2X5c3M97TZMK+2r3bQO6XsA80ERxsIDBYjb+vlIt9jVw++iHOHP0fNs4e+2vOzf01F4/9FctH/4K1o39CYOYPCYz9Dv6+X8anyb2DSgTUgaohhhoI9exgvXc7Po2ePAI9eYR68wn3FRDtKzZeiYcacVn/rkobic5qIt3VxtP1vnoCvfVEuutJdNaT6qgl1VlNQuf2VeDvyDfQxfpLCHcVsX73zcRbS4iOvxOe/gSsv+S2V1PK3eaO0taFUzsebHg43ky7BFp013DuWYqQukSav5uBHz3I+vHfZFUKoDMP5NJ/JI9Y+82sDNTy4vT7SDz9oMm5Xw2wfwbTr7MzDEa6misWxC5MwNmwMLn3BZKUazOpb9e65pcfZuXev+OZjrey0tVEqq+ajPzjoRLiPUWEe8vw9TliCiCRnjJS3QU24t1lRLsrr2pEumuJ9ewhOdBMrCePcOcW4v01RIf3cXloN88PvoXApz8CF9R0IYlaNkW9mEoaKPKvXzfzE8mo2zf3QAHlS//A2ux/InigzhV03pVHtqeGUE+TaXRpdwEl0V1FuqvSwKGj6p/8A+VEhmoID1YT7K8k0FdBeLCO8OEm/MO3sHJoHxcHbuPs0Nu5OPluAif/mNRn/ob0l/tg+XFYfdEbL8CqxnOw8jQsfx9e/BI8fi98aZDU6f9JaPwD+Pp/g/WOt7PWuY+zHRWc7ytldagS36EKAkNlhAdLLLGQGiol0V3gjRLEK4HE8auacHc9a32NrPU3EOytJdpdQaqrhHRnCQkpvZ5KQkNNLHXXsNRZRWigllRfubl0tLyJTF8Nl+Y/TPrpT0HqsqvgCF22tqZG282pYC3BSIdMLC3es5WCScLErIe1lbLY9N2L8MMp0vO/xtpHt0KPJlQriQ/UstZfy/nOBlKf/4htnaFCSc86/ET+6kUvBtFD11XDlZyYSXHoNYBonkSi4LLSKhE25Jnd0z5350k9usjzY3/AGWVAuivIdBdAXz7JXmmiUoL9VajsQ8dIXzmJ3kKSPYXEesuJyqpczeipMdMeHdxFsDuP9Y43E+0vwD9UxdnhvTw/+3vw9AMQPKPmUWSEDrOGyj5lbYbW7uOnkudnveFlW8IvEvxcK0tDbzd3KttyI3Tnk+gpJ9DfjG/AuTvR3l0ke6vMH870FhHvreJSSx3LvXsJDu4lNCRNXMx69834+rfhGypgbayWlZk7WT/1BwQ/9zGS356H5/4Zln8EIc0uez6HbmTzkIq1Xsra3FPNHc65FPczD8PXjhH+ZA/rp/6Wx9ru4OXBO1mbejuh6bcQOiw3rdw0vcAhPmZ6dprCS/YUb/BL9W3BvgazjP7+BuOfeJru2Umydyex3mLjdXBkn9VGXehsZqWnkWg4+/l8AAAgAElEQVR/FdnefOi5AQYK+VF7E2v3/xU892mIvuB6Msd91g4sF/ua/pYFSav61lkPFxXHrdG3XLJkWP2lBR25Pefg0UGCw2/D39ZoMd/64K2s9zeR6CojPfsr8MRJW1hl2bLXYPN1OQTphzcPez2XSVCmwIYKykT5K2ZOKTerk1n6Lv7PHOTy8C+ZH20A6dlOujfP5gCkGX0DNfgHqgj1a3a7mERvsZngWF8VVzOiskrdtUSG6vF1F+Lr2kp0cAvLvTfzo7HbWfrsRyHwDKgSQK37NVQ5l3VrloNZr1r0NQj0mm8py6LdWZ/7JC9O/h5Lw7eSHa0g1v4GGMoj1ZdPuL/WQKKYINTXbPeZ7Csk1b/d/P611hJLYEQE8v4qVjuLudRZyNJABcvje0k++B/gKx+BZ4+D//uuMZs1lfO692hNkDwMrU60NqNXeCO8RCPy5dWxRn661v6sQfSitwfhN4h8vp31U/+VsyPv5kzf21jpv5XwQJNZg2iLXJSd0J1Htnsn6Z4C45sUnK412F9DpLeOWE+V8TLRt5NY/3aiA3kEB4rxidejt7M2eDuXuvey1NnEuiaQ5V733ES2P8+U6bnOvayc+jN4+XOg5hCJgDU9F0A0TOIsw5rT+HpFHo1iFK2CxeKVVDrrMlsCidrePjrKub63s9631yo/ZMEZqiMy2Ezsof8Boe+49SuvwWRvHkT9ZF13buvMbh/QBag8YNXrqKElkl4DM9PC7hJXVEem89We9NkHSN37QWKDzdBbAN0qENxOoreA4EC5EUxEE1hkVQSSWF/FVYFDgBJAZJUiI40OeAMFxIffyMXuX+D8kXfAD+bNLEuRpiVU2nBFBYQJVUzJiniW8DUI9JpvqT5t7Z9JfPYj/HDoHVwYv43wdA3BwZvIDLwJDm0l3V9gbl9gYB/rg/vwDzUQGSoiPaQg+9+Q7f6/oP/nYXArkd4iVnsaWBv5VaKn/pLUI0Nw6VFYeR4CWqUXcQrSEw8THGUU1fFEbYDMRVYFhGuCLQ6aK6ljNkMsk7BCwbSWxKpTSuwiBB+3lXvh+z/CxdH3szr4a0SGfpFkT70VKtK5Ezq3G0hkSTThGeovxT9QaYou21WMMkWyHAKGf2gn64cKWD0kt62KwKFmwkP7iA7eRrB3N6tdtaz0VLA2WEZwsNDiVK0/We65lewX7rGl3lK4ygpq2wjZhNwwS2LzI86TMWWtxtsJFc+rQthlu9RoxXVb+RHhz/xPwlNvs0VW9NWTGHkLl/puJTX7a/C9Qbdy9jWY/GMA2VjgYggV0QWQNWclcl3APTdFDFjSmgkz7apYfQ6+0odv8teJDtaS7M0j03ezaVIBISyNMqhjGbF+N+KyJgOVXM0xMlhNeLiO2FgjoUNVxEeKyYy9mbNdN/Hi8T+CpW9btaYW7jjbHCezvmSbZ+bW4hvRN/Vr3WxF7b1cHOatRReznGCKMVqWPI9v+le5MHQna6O3EhirIDqyg0D3z8Ooywzp/oKDjQQGmwgYQIpJHfoFGLoO+q9zQBnaYe9dGn03a/ffBd/9NFx+CdS+VJtYumpw85py+UYrJldvrOQFSC07PuGaiGpBqbSvRElDQMlp5Ki2184kSce1BuYyRF6Ey9+Fby5wee5veLHrnSy330aqvxm6S6Bnh41M3w5iA4UWo4QGHf/oLoLuQpL9+YSH8vENF7J6uIjVw2X4hlx5vWJEhu8ge2gfkcEmUxKrY3VGKwZ2wGCxxSO+sV8n/vVJp2yzKfwh3cUVi3gFIKKAZ02sY13MmtWFBRPNk0QduKx+68JDxD7xYWJDt8LoHQRGf8lcPsYa4BMfgOQLr/BMc/x3nlWK69zPu5jDXUAOTnpNFyEYeBOJVz7t+YLq46Qafq+Vik5dfp7ww/2cHXsHofE6ksMFMFwMh+rJ9lcbIeP9+SQGK0gMVJE4VEnsUOVVHquJDCgoryYyWkh2ugRmKjnbu5tL37qXpLZgyAm4lTXrfpRk8NYh6Fb1unZa8jSVYd1r2qbHJlXm2kZtn75lEq77i0qlg0+yNPMrpMbrjPkM7CEzWGpZqdBgCb7BcvyHqu0Y6S+0ib1wa4mluePDNxDr+39gVtm+Yla6dnFp6vdJf/soqD9yKkTCr7okL7ZQutMrstNLEnaR2/iXm3U2fgm+Wrh65aO5r9h8NF4bcVyhpTbPsS0rLr1A5GunWZn7c1b63krmUJNNtikFnBgoIDZURHyohPShMjhcTXKwjsRgnfEgNFJKcLTERvhwqfE0018HfQ3QI6+ikdSAOzc4Uk5wtIjo0M0k+7ZDbyGx/jpWFn8bnj1mnktOHjPaPiOrqUFHDyW7tMAvpcV8Of6+KkRw9xeG2LPw0oNcuv8veHbg7Vwc3kd4uBr63kx48hYuP3GvWZ71RIalQNqslegkmU/FVgSQq/9TQB+3jSmTLqsl+QtFyTz1CEsP/gXnx24lMFxKWqa0Twvsy8n2FZAa2ElS5d6DHkCGy22WWTPNr2sckqtWTWSwnPj4DtKTO2Gshosjv8nFpx4l4CmZjXy1uR6qLXOiZcLn5dklVnpVR7lfAr1ZCiX2JI3am5tLXLJmz0JT2LJ3genbYLQYBhpNAOgrJSH/e6iapUN1rIzsYn2onFh/Hpkel3ZUGjw+mkf40E0wVE5q8DbiJz4M35kDdT5Mhm22WNfvmOUlAxTzKC70wKzrdYJwdTzUd+s7AllYkfepG9YXqhXt9+8n+sm/4/LAW0iN7iNzuIZY906ivfkWZ6WGS1lu3070UB3hQw2EDlcTGiknNFJCeKSE2HAJqcFyS/nTWwc9DeiYGagmcajczgmNFhAbzSc5tMMSOqn+Utan9xH+yt/C0jddBa8tB9e+NRc9d9/F4bpMgcSuVyDRCxvE0Au6mSiZ6Hnr65V85jTPzP8hL/TfYpk6+m9k7VAVz365i0T2slUeBsIZ2yrFyuGzKbJJ37UCRH58VN2RHCNFcU3uBF+C70xzfvY9LA3X45f57Soi2VNKpr/EtGxSbtZguRHrdYFiM4g8gMgnjo/l20gdqiJ0/EPEL71o/aZMw9iEjoRIvvmrAKL3tDegJyy6BblfapBpANF0rZUFrZJkBbc7t9ImPlLfmMU3uQ9Gi2BAmrIe+svRvfkP1bA8Usf6WL2lT02w+opgaBccbiA6VoJvpIz1Q7eROPmf4bE5iD1j20/okgJagOix2dSkFfFpnY4uxpvI3SwTV4MRTQLr67z14i6DKfdFTRdegDMPE/9cK6Hp9xHpaoYele5UkBwsZKW/gKWhYqLDlabUoodL0ZDl0BBPk0OVBohsfy301ZoHkRrMnV9CdKSAzHgR6ZECkoMlluZeGmnkR3O/SeCbYxBTwzq/xZGZ5IoJbA4I4pOWnOeebxw9OrhmDHGS2hE5ex7ijxH6p27OzvwmlwcaifQVsDJUxeOL7ye7/AVXaRyJWB4jJYUopyiTuVaAgPabCKXFMKFWKk8r6Fbh4jeJfKGFy7O/wuWhSlY6dxLuKSQ9VGNDLlZ8qIzkcAXxw1c5hquI9FZZajQxXkBoZIelSnnkHtsJS6tdnYvi9Y81i/cqgJjr5dYnyJNyNNdzJ4TZHECiasK8ip+EbdiiJg+XH/gYvom9ZEeKwbRlFQxUkRysxne4hpXRWoLTjYQOK41dQLa3FA7vIjVcz9qhCi6O30Lw/j+F7y+4HrWpVeu4KBHVzyp2MJAat+ROKJvo/G9dp4RkQ2leLUD0dfZF+iX9oiaLFXsG3A6za98j85VDBMd/l8TQbWQPNRDpL7YYI3K03uK+9HA+qeFikgLGSBkx8XO4iuShGlLG7yrSQxoVHr/L7DzFjAIH46UWQ64N1bI03MgLg7dx4fR/gR/e56p/CZNKh60DpQXmuHUlWrPuGCYllxvuJUcb9R9Tlx7F0IqzHib7T21cnvkNLvXWszLSzGMje0l8tx2C33W7pwVd11VVIgmA1+hiOQaqeEyz7M5HVDcJBZbL8NKX8D30V1yafSsX+vIJDOSTGd5FZqjBxSCHSkkOlxE/fJXDA4jqnGITBQRGCvGP3Q6PTdtKPtMwopRUssBiluHVAHGvb86USPu4lpVeYki3lnAdyjUxZfe59G1emv0g/pE9pA8JIIUwUAoDNSQ1nzFax/p4DdHZBmLjtbZoS6U3HK61DNDZwTrOzv82PPMA+J51lkPdTuKwnAGVUQokuq5XWJBNbta1AiQHsmgsQcKv5MXKxv7rcl8StlOvD0JPw2NHuXzs/ZzpaSR4uIHUzC0kJ6tJH94OwzeRHd5pdDCFN1xDXMmT4ToDilOApcQPbx5lBqjE4E7S45WsjdZxabiB9dE9LA81szKhXmV/CuvfMIumWbqgJnvNgiYIxZIEtPmoaTSl7jUEkisAEX2Up0yoF0D8AiSfg0ufI/bFuzk3/m5e7N/NC4criXzyt+Hl+yB0zpltTR0pJLOJQmPA1f0zAstMW38usVN7QwRA2wpr5ZbM2+MnWP/EBzk/Usn6cB6ZkV2khxotQBcxEyPl1zAqbeIpKndtogDfZDnB+d+Cs582S2baN+eOGu2037cAolvPaU73MAcQ90xkFUu89I+2G7ZOWZeJmZZdgR/cy5nx38F/uJnkIQXaO2CokKyC1uFGgnKtJqoJTVSRnKwl0VdtFQbpkRqWBqo4N/F2/A/fDckX0W5VyZTbi1zrps8m4Ly3gaqzICLw/3oLovtTuiIUD5MIr0My6LZ1SyZtbkEC5hcvZU1SzxF+bJYfHf+PnBt5C6HRvTBZC4e3wMj1cPhmGC4iPVxpFjIx3EB0pJ7oaDWRsTIi40VEx4qIjxWRGCkznqcOlxMfLCIxVsHSeB3nDzcQHN1Damw38ZFaLkrZPTEBseeNF/6E5ntjqOmDwBHJlUIYODyQbEpQiHaK2BJqdapFbNotLfgkvPhpgp89wA8G38HFsRpWj94GT6gJ9osbKT9tuKZK9mu2IMKBJqiypu/E3iiJeNQaNlou2vc80X/q5vyRJtbH8yzjkz7UQHywhuRIHYmRSuKj5Vc19FnNzCoTFh3PJ3Cknuin/gR834JUzK7LLIdTJUbkDYAYup2PoocCiAmjnlga2+u6pzcCApTmGi7YnkwkzhD+UgerU79O4NAuEkPFZIa3wKE80oPSms0EJxrwT1WxcqiA5Fg1GZVZDNSRGG/g0kgTq/d9EF76lKVmtVgzqOF13VjKuhovPdflGJsNIF6ud1PnFfe+g/Xr/S9L6U+qC7oSxk77EtdKJm9vMz1MaxdbpTUElDMknrufyx//c/xj/x6mmmBkC4y+EQ5vM4BkDlUj/uYAEh6rJDhRQnCygOBkvgPJaAmpw5WkDlfbMThcweXxKi6N1jgXeawRFKMNlrL2iT+Giw/bfiYCSMTbFTkmoklvmdVQsC4o5LIyjpfGOq0jyRFJezqGzkLoOXj+YZYf+ihnJ27n/NSd8MQshF6EkLrVO6Wv/Mw1AySkCgDjjITIbylUlZL7c/6BOpp/7xTnjt/B6tQ2UhPSMLVunmS0kfg1AiQ2ICJXG0D8c3uIPHIXxJ81Kyba/WsAousXMe0+vM+oTseCVs1S+5VilHU8R1pBY+QF1j/xF0SPvIPAUJ2lPpPDbyZ1eItpz9jIbkKTuwhMV3OpZ5tpS1kWWc7k9G4uz7yV4MMHIKGgXLu0OsuuBgXy/kU6m7fIXVCuosEL0MVvfUYjx/vXCw6dL1Bo+zaXtXOFBrZE3wiSJOpXax53LYpMNB2npt688HFin/pzVoabLDGSHruJzIjc5zIyh2o3AaSW4EQ5vqkifNP5BKbyCU8UEM8BZLiWzESzxWMrY2WsjpVZZTCHqmC6kMxoAWfHb4UfztoqwFA8ixbwqVWpFzo6gOi6cgDxKj1yPF2JqHWQRyvpAK1n0vp47Wf/wuf5wcRv89z078Czn4DIefArRPBCh/8VABEBjY8miRIqEdxlRmVZzJu5+AOWHnwna7PbiI8VkB1zmiI62kRsrIbYePlVjcRoFYw2GUAyR8pZOnorPLvgUoK2/NW7OF2gN/t/xYJ4cYn3+gZAdK4pU1ebZmWllni/DFllRCKw8n3WTn+A6HgzsdFdJEcryIxvsZEYrSEy1oR/0lkQuRdyKZK9FWRHmgjP7GP9vt+HMw9BTFsKXJkMywm96OcEXxci5lsazfnZnqXLzYM42l8NPLzPbNDFeRf6Xl2Hl7qzZuY5QKrmyWbr0y+Q/eExzp1+PxfEg+N1ZKdrLHhnpJHMcAMRrSuZqCM0WY5/upj12QJ8MwWEpgqJjZeSHqkwDyIyupvg+C5CU8XEJgqRC8pwBRzeCqNbCY5Xc2bud2H9Sbt/TYMabHVRqo4wLSiKeRbEAOLmgXQfup/cJKnOMsKqIkTLrtWX4IXPwsuftyXSBC+5LKzR2GUyr9mC6EeNSTltbZNsTgvq4izRvvQjVj71btaObUXZJgEkeGgXIk70WgAyVgEjDUZUAWR54S2e2xIgq15KuYvTtW0IgtwFuSqvBIi0qLkZ3mdkPdTcOKNcu24ivQTZFcgmrEo2ePp3iY9XkhitMx86PbGF5MQWYmMVhMZ34RNAJmtsXiAzVkFmsJzsaCOBI3cQ+NyfweVvubZDpvlEqNyfLjYXdG4Gh3MfROsc40XfawGIZEyf1y1vFiRl8DSZakLnJThU7xTXHmuWRVsmtfYtVr8zzMr9v41vdg/B8SZSU3tgtJn0oVo313GoiNBkqQeQQtZnigkKCALIaDnJkRpCY7cYQCQX6fE8A05mRHzdbgBJjxWyMv1Wkt9dtNWAiiC1qYEtqLK9F0Q3j2YbINHzV96XQKI72pBV3ZdcrrWXwH8G1IVRnek9vidto4fAtbpYLh0qAhuVxUOvRFkmWV6rcXN9hfXPvIfgwjbSE8UwXk9guN6IGpuoJj5RcVUjOV5pAMmM1pIWQE7/GsmLX7OLyajn0WaAiB4mCDmAvLKuTEG5neEdBBYtEEvZPuhx0iq3kfOTUYDwfSKn30VqopDkWC26jtTkzSSnthKbLCM0VYt/ahcBjaEimKgETZiONeA79g4y39TWYS9dabG0YSFUfKdgfNOw9xw4JNC6vBxANhhu4vD6/+m7xCMNPXb/5SpruCSFZNC0bjpDMqJ9Gl3NV1Z7eYQeI/Hdfl4+8uucGb2T5Pw7SE40kzpUQXa0DH/fNqJTJQRmSvDNluGbrSA0XWE0yoyVkRqrJDTRbO5oZmIHjG/DvS6LXEh2fAeM7SQy2cT5U/8NVtRVft21LY2onsxZCtHFgcRztSxod5edo9VmK2LnC0O54SlPR1vJtFqnrhPj8rUBxE2oOR92g3Oe3Akg8qWNm/41Ap97D+GF7WQnq2G8Gd9IHb6pOqKTlVcFDoFKgpk9XG8WKX2kEt8nf5ek7wemDV1libeQRoTwgKvSGGvRLSJ66lcHWQsTEB1EZ7OEWmvu2h9pP1jLM4m6y08Qu/dXyU7m2TXEx6tJTu4gPrWN6HQhgekKA0hoopFAfxHYPZeQGq9hfUFroxdd5xO7rlyGKpel2lw57SYFxUn9bE7bO0Y6cXbCYRLyuv9JeLwCDvt+A6atu5CFdb+nCnJHpjjp+JrbxXajFiFkmw1d/Mzf8dzku1mefgexid3IYjJWSPzQNuKTRYSnyvBPV9sITVUSmxAAikiNlxGdqCc6UUt2PB/GbyY1XkR8ooz4RIkpICYKyEzV8OKht8D3pyFzxs2Qp5MkE25qM4dho8iGS6pXHY2kSGQhddQ92/2IcJ5M5N7XOe48RVtrpFi6doDEbPGit6xOv+RZkZxmMuD4zhL6zDuJzefB2G4Yu4P18SrWj1QSmS4nPnV1IzVRZQBhvJaUtNMX/iOpxHmb6jKOW5m3R5GfBBBHQyPYhkshCnrpdVcjKosjpyxiwaxpnaUnSdz3yzC5xZIOiYkaElNFxGfyCB/JI3CkGP9Mk9X6hAcqYKoSJnaSmq5m7eR74dIjZj0sRjNGea7Uj1kOd4GbgaHHV0AhhF39Xw4gpsjEOE3yqu5J5bCebtEuUcpxpQmTZp1Map2stxGnlWQkLpF+/iFevPeveGb47QSndsOUXKRtMJVvHkNsqsomTAPTTYSn64hPVpKdKLKhpE1istImCxFoJouIyw3zZILpMpit57JKWj75XyDwLbd5qZKLXqcTiZ0DcS5m87J9KhfxQOKx1cTxCv3c5/SegJGznTG0DEJrT66x1EQWRGJj+5YKGDmYZlxgJLNmV7T6FIGH3k78iCaVfhFGf5n16Qp8CyWEZ0qvASCVZIbrYKLBABL/yp+g7rjatsCIEPfUn2dBHCFkQVSyobUhTric0Hno1sEDSMx2kEgQsSSn61huMeHK0yTueztMv5nURDnxyRoSk6XEZnYSnNuCf26nA8jEPhKH60ECMLGV7JEqC+7xPWrmXfS5wlhdjOcimJZxVkOv/jhzJQhe4G6IvTqQODdSFlKiIffOS9/oorziA4UdcjWdIxYkk9GcTRS1EHVLYiWlZ1n/+gTPTP8WgZk9MFsGozfC7E4k9FIgKgwMTu4jPNVIfLKK7EQBTOTZMauYZLzKKZupAmIzBUSmK4lN1pCcqoIj9eZl+KZ/EZ5ZsK6Y4q+WO0sRS+w8Vjq6mKLRO1dcRd2SznG8lkA4emfScVKag9K0ne5JsZadp+RJ+NosiH4i7NUt2VoRXamG139VP2SLlC79M74H9hGd2goDvwoj78Z/pBL/qXzCs8XXBBAFhEzuMoBkvv6X5hKpKMQAoopbj9l64bUB4gmJBxB9TABxu0gkCXo70Cq2Y/kZDyBvIjVZagCRFRRjg3M34p/bbgAJTtxOdnw3jJfD2BthoYr1+z4EgadsT8L17BXmOsY5Qde163lOs+X0ziuEwFuC6kovrg4gBrLkZbfc1eusLtdYbtcGeFXMqeBVrVWTMdsXPpnN2PqSUEaWRWCKwvkv4//8X+Of20tmpgCmboLJmwwgqnaOjN+K6BGe3G0AscLSyS0wsQ3GC0iMNxKbaDQaxmbzDCCRqTqbcE3N1MNMoZXs8MW/h7Xv2e9e2HStopX9SXFYSY7WyXi1axZPuSSMc8oEeDFaMx1K3/vNauo2teWH9iYRraUYrimLJeK4qlepm9QVjnp5Z110JrgK5z6P73QzkfGtMPjvYfRdBI9UET5ZRHS2hMR0xVUNaRdNFqand6G6oPQ3/952JpIFMYFTJZ4eSOF6LpbsnWnMTVksCaRbn+y5il54ogU4mqXQ5JQ2rzQmCCArT+N78L2EjpQRmakmOlNNcrqE5MwO4nM3EDl6M6HpZqITt8FMswlJcuwGEieauPTgn0LwaaKRDD51c93kG4spulwNPdbviY25c/SaZdqkgZQ8sGXQYrbe0U16msG0gycvGy/pwjedY0pUM2JaA67da3WPrqGF6KfH9jXWKTIO4bDtcaLYTtfn9nF3v2yZvfgZeHKeS/O/juIM5gvIjr4BpnaQnKogOtVEdHIviYm9JCbrSczuID0rEG2HiWKSE7uITzYRnykheiTfaBs8Us3KWBXRiWqYyLe6r8SJ34LlLxKN+FHxT27OSBRw9/cqgJgVcTQSONxw0xHO/njrnTIpuw31IVGJl+PDNRYrOoaI6B7hRXRx1cOKbVGWUlnGAmtzjXCsGCb3kZ64g9hMA7HZSpKzVab9FUO83pE8UmHfEZlrwH/irfDUIeuWKIFywuTRzNMIrqAy559urmLdJFtecJoTTgmC7iORWkcVpbZ4LPUCgX8Z4fuTv8ZLU00Ej1TAfBEcuxlmbyQ5sZ3YaAPp0SY4vMOC+fhiHWcW38aTn/rvEH8SUhHXpdSzwB7ZbNmAkVPVCd61KMMmbaYNZ3Rebuge3TZwcicEGC91vQkLDmy50hmvXkkf3PgSCZRgqHPcy3qmx/Zn37mJxxuAE82SxBKCk1Zpxmzb7PVHOjl/5JdJzVeTmLmR+PxNxOa3E58rJj1db/FndnIvoYVCQovbSR0phekqmKqxuZTkbA2RuQr8i4WsHy/EN1tCbKqSzFgJmdlaLszW4//a39iW1NpuIncbjre6MV19LprwvAJzEV1Bbe7WdZZS+y5oFoCuKI/NeuaaLIgxRBpG6BPh9E+/5S2V1LwnqQvwgzn8RwWQQjIzDTabnJzSsYbUNQAkfrSC8NFSfAsNrJ78JXhyzNYY54TCBEgaT4v/jLEyDXK7LE11BUSbCtxyBLxCeImJAljtuXfR7X+XfpHVJz/J149+mOen9xI6Wgzz2+HYFpjZ7vzqsToYb4DRfDJT+QQWannp1Nt47KE/Bf8/u7XQlkN1gqlr1hANbWwIonxht5+eSz++2qpIGwog7p7sS/RFHr/d/UiQ9D06Ry6Tlh97aJCMXPWfmO0t9xXvEwkyT36OCyc/RPBYI/GFbYSO30R48SaSx7aTnVFscgeM324AWDu5ndhcmQHDgvHpCpIz9YTnalg7kc/ayTz8R4tIzAggFaRmarg4V8rKlz4A4ccM2LonKQ9XO6fr+UlDVBCdHaB067KQOopUr/V3bQCRiJmZ1yo9XaT3i0Z/IVZFbpccQObqYD6P9Gwt8dldJGaqScyWkzxSdtUjfrTMABKY9wDy1Ajax0TXIZJoCByvBZDNbol47ATKASr3Hc790szrZZArkT5D4uzX+N6J/86FiWYS09vJLNxIdnEr6aNlpGeqbGUjM7Wg1YbTVQSPbmflvjqee+B9ZF7+uPWINV7mfkSCugFUWQyXTFBCQWlpCYGbzPQUkWcldK9OezpQXLlx933ufrQOxgmSA5IHEr2pm77qP31BiHD4svsa9d4JvET4iy2sCiAndxJazCO8kGcAYaYURm83gAQWClk/uZXYXCnpmQr0ngDkAFJr1mP9xA6CR4vJHK0iO15OarqSpWNFnP3UO0mc+bR5C8o+ilqiwZWkb26idTNxHUAcPa7oh591+9cOEAt0AsvFCHwAACAASURBVF4a0HNQDSDy9TU7uWTNE/xz1bC4hdTRSuJHaokdKSF2tIjUXBmpo+VXNRJz5Q4gC3WsnnobPHXYNSfecE2c16FGFFcsiLp7iICe9hCFPOHMvZYja+5oK9ekb5JrrtgtcxF8T/PiQ/fgn2qG6ZuIL9xA9MR2YvMVpI7Vw9EaOFoP07eQmakjOPMGIvflc+H0W4g92g8KjqXCcppckm4AcRZDOzA5cAgoEgBPlj1gGBA8cus67R5ymkFKVOd54NHnlbJWulpAM8DbZJpnZtypV/FfQpnAH1hxIFWAHPHBE/dy+fg+Qid3ElwsIbJQQlru50wJjN1mbnZkoYDAyW3E54tMqXCkBFVDJI7WETlWR+B4CYHjRUSPlsJRxSBlpKdKWDtWxEsnbmftu4fdQigVx+aUYY5GdvOeW5h77FVK5EiU+8z/ZoCIE8p7aFLFZ8Q3ThpAVFK2Cl4M4jtWCYs3GCBiR6uIzu0kOpdH4liJA4mA8jpHYq7MCBicr2P91C/C04OWI7/iu3vh0StcLKUynWCIWBuC5Qle7rM5cNhRfXWk7lMh0kG5WW59gf8bo6Rnb4GZm2wSdP3kTgKnikicqAbFXEf2wuxe0keqic38HKkTb8Z/ooLgwx+G4DNYxzNxSmT0LkYa0c055DSj6zbzyuvU5KJLY+r6DPy6fu8ynel00uLcCvUOUBCudkculnHrXTwUXQU0ch+JRuOEoxGi6q6iOjW1Vlp5nNVP/QbLxwsILFYSPVZO5uhOmCmCsVtgcg/RhZ2ET2whPr+T9FGBxAEl7gEkuFBOcLEMeQkGkKkyMlPFBOcKeWmunkuP/A3EvmcWLCfsuv0NgdeDVw97yVkS0cVInruRn3K8RgsizrrOJwnW3Eo7/arlkj2AJNetSZdvvhxOXI+EOjJXRfjYzYTnt5GcLyZ9rPyqRvJYuQeQWnynboOne627R07IjWCeT29CJMmR9XAGJCeTzq/yAGJafJMF0mSe+i0ZktIxsjGf279Cu+I+/0k4/jY4VkDweAHLp3fiv/dm4qdK4KiAcxvZI3tIzZaRnXsTyfl/R+z4Vvwf/yV45iFYWbkCDuOsxF0mxeXvjcE5wc8hVs+lqY3usjLuE3aJuuHceXbzkhH53SmbrwqT0tq8jbkrJSDsZ3+KcPxrXg4EEpYWjWT8ZLNhL610kdS3/orLp+vwHa8nNl8FitNmS2GiCaZ3kVjIJ3piK8mFPNLz+aTnC0nOl5I4Vkd0vp7IQhXRhQrzOCw+mSknO11kivXisUouffJ9cP5+U845gOjWc5jYeJB7IXf0LInTjD9bQfwvAEiALGvE8RkjctpL5lwzkWgfusfvZW2+0gASP1ZOeK6K0PzNhBa2kFhwhBFxXvc4Vk7sSCnh+Wr8p/bC0x1uUcyGL38l4HXC4Pnem+JZ08yirATqp4FEk2WpjCtcTGsWUdYkAr7vwIPvg5O7CJ0sY+X0DgL3XU/slGKtW0hO3E5qtpH4bDGczCMy8/+ROnG9CU30kTY4q/0qvASHSaMuwAuk9VDXJR2UG7lrtDe0TZlK9zyA6PP6jM7JBV5Zl/uXACkolc3JDfUEU4pep1/1nzqze1NNccFO/bb0Q9EAnJth+aFfZvXELgeQI+UwXQGTdTBTRXp+J4njW0kubietGGUhn8R8KbH5GgNIdL6GxLEqMrOVruTkaDnM5BM7utOs0tp9byf5WDdwecPF+tcCxBFKZ2uIaD/97xoBoh/QlHzAupuIEcZMS/OK8VrXHDKA+I5Vw/EbiM9VWpYiOJ9HaH7btQFEBD1SQuRYNcFTzfB0K6TPmxrZUBgeWFyFqjfP8WqAmEBtAoho5lkefY/ejicypHNd9bJpsmqynDxD8rN/Dff/EqHT1ayf2kbw3jcSO7mV+Gw9sfFbSM40EZ0qgvsLWZ/8v0mduhHfXBmB0x+A5z/t9loxRomWHjhela7dDN7cfclF0CdeARAxW7PhueGVWogvSn1LdnNDDpyjyWsLyE8XHUejXN9oS0OnI07m1Jwv+lVWHv5Dlk7tIrpQDrM1lso1kBwpJDOfR1JJjQUHElmU2EIpUVmO+ToDSvJYFdmjFdYBhvlSOLKTxNGdxE+UEzrZSOSrfw7ZM0YH0ws/yYJ4vNywKAYIj9b/+wGiX48Ty4RYT/qNWQbnhJo56JKV+QmR+fYCsZO7SR65gfBMuQVhMd3kwg4SC8XmZsnVev2j1Fyz1Ik6/MdryHzzv9uiJk3+iiAJW5TuzK6yOJbl8LSxCb64bzNfV5ILRj9PCnPCuPlohNbn9GJqDZa/QuJz/5W1kzUkPr6D0OIbCc/dQPZkDSzuIT27l+RcvblWodNvImiZnRpiR29nae79pH74WQhdtM4dmpHOxCOkUgnDhG4jlFaoriDbyZ6oamlgZay9+Sa954ot5ZopFrgyg6wO9loguBJO4Yu7NKeyYQnNxMuGaH/FTMZGNpu1phGvCYrNb+bcVytHcXM1Ya3I04XFniHx/S7OLNYRO14Es41WxY2s6dFtJI9uIXnsJtKLWwwoscWdRI4XE1msMJAkjtWQnquCuTKYK7KJR+YLyS6UkjleRvJEGZfu/RVY/66xIpFME405axzTfSZc4bW9uZmBUj5KUGyY5ddWENdoQUzCiGcS1oI+Z0HEn1AsSkp9YONr8Ogs8ePNJKevJzpTQXxxF4lTNQTnCzyAyM0SQF7vsZjssQoyJ2os+E1/408grd1rnQCr2lMEEjDcDPoVCyJBs4kiESznkohWr9I4r6Cthwu9Zn8qsfA/D49PEPjEW/EvbiN5agepE3nEFgvJnqgnfWS3ASR0Mg/fvVtZO1VOaLEGZitIHLmdcw/8BclnP2kd/qzDeWydYGgdXzyCL6vlrqou04y+A4mu29DjTa/ruQO/nCclTLyOJBbHZAhFde+OJFpfZDeogFrd1JVlzAgkVw8Q0ctau3qxTiBqUmA9gONPHuPsqSZiJ/JhugkmG+FoPtn5LaSOaW7kJjILW8yKRI/nEzpeTOh4KZHFcmQ9MnPlFt9xbAcs7iS7WAiLVbBYAcdvZv30bbD0Vbu7dCpBQp0WBPtEBtWpWkX3qxmoC/4/CRAZCi29EFnECAFTfRuCwSDx8GXX3PnRw8QXmkhMXE9itpTMyWbS9zYRPFpCbKHYAjbFIjKzr+coUDFfSWaxitDJEmJf/U9uua2shGbPdbTyAjkUShp4KterYVRcYieKkhqmXTwUGKmvIOLVdDaQ6PxYABUfJr/631iaqyB5qhTuKyZ8bCeZE5WmBcXs4IlSVk+Vs3KqjpAYfGQrHMnn/HQTa5/9Y/jBGCz/o9sLnCTBDJyNWR5QuUDWSVviN6kYSF0y5CuZPCgF7FkOA0cOIK57pLXv9DS9aVNtkJk8A2vfIqG9XdIqQLxKgIg80jkxZ3k00+JPu6JOKZ3Ui1/k4ifuIHpyB8zsghlNFheY1UgJJPNbPIBsJ7pYSPBEsQ1ZEiVgsnPFsLATjm8nfSKP1PFi0ov1MC93fQvhU03wo4ecxUyGScmdV0ugZBrNwUrvbeZbjqWeuvCsiGmNK2+96tE1WxBrTOilUS2TIqaF427f8PQyxB6Df2kltdBAavx6kmq1eV8z3L+X4NEy4oslJBcLrmqkFmR6nUaJnCok8MhvQ+AJd/+qp/EQq5lm5W9+HCAyyR5IcsccRV9FKD3NvbX5aCDMLMOzR1i7/zfwK7g8WWL3FZ8vILOQj64zdLyW9ZMNrJ1qIHq8DI69CY69gfjCDpYWGrh477sIf+0gXPoyiG4qlMzmaqOyrKvlaTpMUn2e1FZJpR1S3RvaMGdBZEW0j4abuNVOssJTStZUiQVtDbD+T0S+P8xL2l8ktXL1ABHwFJtph9e0RSH4iFoiQG4gS4+z8vl3EjldADPVcHQXmYUiEioxOb7NgMLCNjKLO4ieKDElFzpVQPREAZn5Emc9FreRPbGFxGmXHUwu7IGFBlh8s3khsafnIbPutHJc27ElkTVRvJhbcGhu6Sb+/R8FiMmV/H1ZEYFDWw2oc7hptYtw+eMkv/6nZBY12XM96aN58EADPHALIU0aCiDH869qpBYLYEGFcaXETu1k9TO/CivfdvcvcJg/Yo1trC731QDR5JmbgZX4vwoor3j+E9DiEdw8CpWB+n5A6ls9nJ+/Df9CGdn76okc2UbmxPWkjm8lutBAaGGvAcVcjoU3wOK/IXP852w+QDHM2ifehf/Lf0/iBydh5QnH9HQuThCYZadlOhRnBLxq1Zya1A2LAa73sJ7p7KglGFQ2HHYZvsSj8OQQlz71+zzzwIcgffaaAGLOfjwqBFpdmCJRVQObcQudIfyP/4nwac2GF8HxalKL5cidSp/aTnrxBgwgSvkeLyNwqoTA6R1ET95MZqEA5nfAyTeTPn0Dkfu3E7q3jMTx22H+Fli8kezJKgL/Mux6XqmJtxoWqrIjE7e19IpBJQI5UdBjBxbRUu6F3vnfaUGkJfT9qkpVoy0pMRWvZdVHVcszn2XlsTYCj7wbTpbB9PVkj94I95fDx/cQPVZN4ngpqRMFVzXSx51Pmj1WQvz0Di4++Fay5796RUHYvcv9CFi7HgtkJTWeiyWAKA5xhHMpUWdRRLhXj80AcoDR7UtDxzQ5Jq1+8R9Z+9yHCdx3KzxYR3zuTWRP/QLpk1vMgibmbyG1WEXqxE7Sp64nffrnSJ1+I4lTNxK7txj/6V1cPvGLXLrv9wl/tRWefdAqf4mqe7sAoXuR6GkyUW6jHO1NfN6It5TCzcHFuRpypUg9C+cWWP38B7h8qonAI38AqeevHiD6cbl8aW14rmVVMes8qcXJcnYkC4lv/iXhU+Wkj26HUyU2iRo9XkH6vjxTHizkWWwRO1FJ8HQ5wfvyiNy7jezxnbCoz7yJ5H1vIPjAVgL3VxLVvNP87bBwo8nU0lfarMsMGVUmqLGGytxjqLFXPOEqEF7NSSlKDVeaYkLykzXgtXY1MZ8jnbGsogxHPCIOnQdeIBl8maUXP8sTX/gAlx/aC6eLYe4GMnM/D/fmw8ebiS/UGkASJwq4mmEAOV4Nx4qI33szZ+69lcTL/+BkW0DYAIhm+VUOc6UOX6DQXI3iEEdAN8MqojnC6VUv5bWhbXKkdmARQMICiAJmE9RlePE42X/4ANmPF8Dpf+uAcCKPrCzdXA3MVZJdLCB83434HthC7BPVhE5W2ERj/HQhiVOFRBaKiJ9qJv3J98BjnfDsKbj4bQircYRL2arzoqINSxbqXnVpnjHR01xaN6wgOp0iE3yJ1I8+ZZWwZxabid5fAI99ANJPXj1AFIOpEljzHxpErOOI1uPo+lTgmf3ufoKnakgeeyM8kE/8RD2h4/VkH1CM9iYEEHkAkRNVBE6XE7h/O5F7t5A9ngeL2+D0z5P4+C/g/8QWfA9UEznxKzD/Vpi/CY6Xcf5zH3HN4DLnXZeS/5+99wCvK7vuey+HBexEryQI9jKcrhLLlhRFUYrlOLFjRYmduDyX51jOs2PHzy+2FSdOHMe27Eh2JCeO7TiSZoYVBAESIFhBdICoLGCvYEXH7fee8sv33/scEBwNZ4ag9OaT7MtvcV/ce8655+y9/qvttdfWPimKEmn9TkZCxGrSQC4GvrICFyFI3hUgQc/qgUKNoFZkR/3x3p/5zp6nxFgJEa2nMWkG3g3w+0kPH+Rq62/Q9+bL3Nu7GjQgXyvA+4vl8GYF7H2FzNd2PBNAHGkQOcL/ew3ZXWUMv/4iXN4T2OjGQw8yYZXwJ6n7iOctQEJwSAmGAJEm0bO9Ez0CiNZsG7lu1JBWil2G/t9hatcLUFNBamcpmTdWY8D81XXwlxvwv7aO6d2rGd9bSaz6BSbf2GzylvzqCqguwH9zqZWuezbh7PsY3pGfgP4/hLuHIXnZaESbfBgkmYWKZbbVYMZPs3j3IN5L+vrr3G/+Ja7v/hRX/mIb6Vr11S+aiolzdtJNBFAMoHX00hvTJuImD2pC/aF0mL4vMfn6R0h+dSXUFJHauY2YUnH2VuK/XmCjU1/dgNUgG4jvKie1s8Q45Xy9BH9XLpm9uUxXlzG1dwupNz6K/7W/AV8thTe2cefAL8Jkn82yVgV4E8lThzikJLUDs0qjKWGouZ/3Cg6dG1FKulmPrQ7VQ4V8IR4wyzoVU5ddp/yfADPqGMMWWjgfOMNG2GoBzg0YP0i29xe5Vf0Sk3VFZGqWwe4iUDRj52rYuQHvzZdxRbu24O5eP2ealk9TvRn2V5KQ3d/4y5A6C9nbSqNDm6rIbZVE1eOFzyAnUo8a4j1sTY+GBz2xtUfpfHlbijKNKe6uPsjEYLIbzvxH4oe/l7Fdm4ntrSSzbzXe3rW4O9eTfXMLqV3Pk9i9nWT1RpLVlST3lhDfk0ty7zKy1ctwDuTi1JYQ21/JWPU27u77EMO1f5/Rpp8ideY/wJ3/CeMHIN4N0SsQGzPBEeOiyOSTuZHthZt/SLL9hxmv/zATh17l9u5XuPS1DxNv1sKtertPn32cp/9fnRbyCbIcVJrVztZLYpsgwnAz0T0/QqZ6De6BxWRqVxPbWYH7xhp4vQp2rjf84O7ahLN7A+7uKkPsqsLQ7tW4e1bj7Kkis2cD7s7n8XZugzcrcXa+yoM9Pws3DgfFHMaIE2XKi5NUiXYDYPGt5sBsJFOC0pimM/f+zo8dUYRnBiB62BAkRgJoMY11euxcQdghYg2hNGvX6MjgNQDRhu7tOKd/k9G672GqutyCozoHdufDLgFkA7z5PO7OF3F37kAdM1eAqENjr6/F27ce9pfh7NxItu5HYKIR3CtkmWQKz6h7yRITZVOniYLGasp37qQnfasu0nUFQJXFlDa1/TAJY21w7U94eOQHGa3/AGP7q5jcW0Jq7xq8vZtgz0uw+wXYtwl33xqye8rJ7Com9WYuyTdXkti5koTs75oy4nvLGNtZxv3X13D/jY2M73uFaP3HSDV+H+7Rn8A98StkW36fdMefEe38KhMtX2Hi5K8TO/rPyTZ+GK++Cu9gCfGaNdza9SpXq/8JsbN/CpnLgTnypCd8D58bgOjBh4F7xoQROyjEapK0bvUxtednSO5fS6ZuIdmDhcR2FeMKHPJLJSx3rZ8TJXe9xujuH4aLX7fzX0wHAtEnrXQgSSyFMs2CMIX5Y0E0U3wdKAMB5R1ej4d5jRgN5gTCuQFdaDaZC4ZXDxlCnSHtcRku/1eGD34vt9/YTqx6E44xr4pgT67VIm9uw3vzRZxdz+Ps3mKkhrNnPXOi3RvI7N5GRtJmf4H5reje74NbO8G5Y6I+kvCKqsxokFByCCDmed+hd97tq/BainKmPDzVtVXXCDlmYdZNkre+yoPun+PSgRe5truA2IEiOFQJ+7QKcT18bQN8bRN8XZNfWyxpnuSrG+z3e7fDni2wax3+m3Luy3Dk4O4rxzlQydiu1Yzs38BI/YuMHHmVh0df4cHhrYzUljCxbxmxNyL4e5ea4yd27uBuzQ8ycfr38cds1XSL6Hd70Hf4Xs9rQs1Wg0hzPAKIA7d6mNr706Rq1pOpW0TmUCHx3SV4byg/7dkAkt69g5E9fwuv9z9C9Kzpc5Ut0+/LjEopcBIKRFum26xR8iXcQ0Vg+PnJzxeZYX7zRpe2mmHGHwnBYa4RRnoeuSjupBbxqlTMLbzLf8ndY/+Y4dpXmTrwMun9L+Lu3o6/ewPsLYU95fi7tuLu3kF2z1Yy+zbi7t0wd9oj0+pVUm+sxd+/HKe6kLGd3w1n/hhSDw2zChgCiRk0PUPIwCbi8+SOeU/f6Fq6sCjj4afTpNJZEloKaraukFq5Rfx+Dfd7/jW3Gz/JWN1WsjUVsEtOZoExFYxW3bUZdj0PO1+ANxTnfwG+qkVmz5tiD3x9E7y5DnZXwp4y2FcK1SVk9xeTqCliqjaPsbqlTNTnEDu8AKdxHhyeD/ueg+oiMw5Te/8uifbfgtFmM4v+XqI479oPBiD6TzPzY2/RIA7c7mZq708agGTrlpA5VPw4QHYpJL5hTpTZu5XJ/a8wffJHYVg75Gr9kd3xV0ZVLJMwOLAYkJYL54qCWgUCybsCRAfp+YxtoAtYBycEiMn7MULRLuSxSxxtRXdjxEdVCmIYRvfzoOknuLhnB6N1W3GPvIpb+xLerpfwd76Av3uz6QR37zoy1etIVW80lN0ngKybGwkgNR+0E0v7c0jvX8bIzhdItfwaTN6dCecKJNZJD8KiQce8S9+8K2/Y5w9imhKdXoqEN8GoO8KIM82Yp0rAQmIC4lfg+k7iLf83o9UvMbVzMfHdEZyGlWQO5ZOpKzLmlHyV7M4qk2vkfG0L3td3wNdfwn/dkv5Of3ULib/cQPx/rYc9r8Gul3B2bia+s5LpXcXE9xXg7s+D/dLcq6H6Faj/AWj7DbhVD9mHZng10uqKZ3qpE01HWic91CAS3CbfY7iV6L4fJ1OzCad2BdlDZSR2l+FrScCbmwxPeHvWMRcSH8Xrt/Cw9ruh5/dg+oKZb1AQS+WKNNFqx143Ewh/41dbnjf3+C5MEDGDrPNNjdhAe+hPhbi1LmKmGohmozWfoOWfds2RERfJcXjQSKbvl7hT/z3c2rve2NvZui0QAuTN1/B3voK7ZyvZ6jWkasqJ799gHNTMMwFkI96+l40GcaoXk6pZzET1ZkYbfgLuXLRYN4l0EixKD1e0JeAKo0H0xzO8dLrsN8kVk+ikvpEdPMUUUSbNAiXbl7avJuBBA5z5fxk/9ho39y5j5GABD2vzeLg/l4f78pjYV0S8uoJMzQao3YovU2zvVti7w5AmIOWoOm9ux1M0aOfL8OYOM7ckXyZRXczk3gLGdxUz+vpaxt/4AG79j0L/H8Gdk6AFX1m7F59qQH3zACJPzFaqFytagCThbhPR6h8xAHHrVpE9VE5sT5kJVjwrQLL71pKuX8+DvVtIHFZ29Fdh/ApMJMzWcvIPZT2ofJMBiHwSw7wWIIGcfEcGiFgnRikLIhtINxGeIIIoiaDQWMasGNTKQaVsWDVmBv1hO5z+PA8PfZwHNZuYrN1IbH8Vmeq1ULcdf8/zsOtV/F0fxNnzItmaNaRqy4gfWE+iZiPZ/etxqqvmSBvJ7txKdvc6MvuWkqhZSrxhHXdqvheuttjZKj2AAYPmQSatsDMC4R375T1+6eE7SbxsAleFkM3MvECiPlL8TJVGAuFlR8ok8ZHpxbn9RUZO/xhjJ/8eE43fxcO6l7i/bxOj1WuJ1a3Bb6yE42tIV6/A2bfMkLdvJX51oBkOlMEBTaSthF1L8WqeI1kfYeJohAfHl3P/2HpuH/448bZfhbO74e51u1+3wKx5PUWkg4Vj7/Fh3/Yw48eZb3Rhm4elrBYjiBTRu9dAouYzFiC1BTj1FcT3lpvADLs2wt71+Puq5kTim/SBDUzv20h0/wtwSnuu1MDEPTN3qTuysA3sYIFDKiUYFv35LgqEiPHwTY2lIIw7S2uE6tKGxkIG088Gxyrdu//3SDR8L8NvbuDhntWk6jbi12+GgxvgwEb8vZsMSLzdL+PsfYF0TRWJ2jXEa6sMZfdX4eyvtAB52rZ6vdUee6pI7c0nXrOC9LE13K7+EFzYA9H4jEulaTWl/tkZ1PfQM2/LDo9/aFfraalQnKnsJAnHrs03PyqzypTCCUJd6rZQQWu2lxtAD9z8czj728Ta/hX3Gz7D7eq/yZ3qVxk9sI3pQ+tIHV5Dur6EVF0+yZpckjX5ZA8U49aVwcE1UF2Jt7+CWG0po42rGWnZzlj/p5i69C9JXP0CRLshqRBwIE7VamCVGiSz/BlfhsH0nzFd5BrbyxspqjX892tIHPh+sgc249YWBQApMwAxvocBSGUAkKdrFb1M7Nlorh3bnUviwA4499swNWhuQjETC5AgvPgNAJGn8s5WRMSIEhMKCyWt1UIWHCrXrIJpGVw/Y2qymlTd7Ah+8gZMH+b+wU+QqN1MsuZF0vt34NSsw62pxN+/FqrXQfVGQ17NetyazWRqXiB14HmSB6tIHlqDW7sW74Co6m1bt2atud7bt2vN6j1qN+HUrCVzsIL08RLuHtjCaOMvwd0ztockQJgmyR0eRm+RNqkINrbwLPyh/g75TuVJVWjBqi3lJgWaI5BYxsWzCtrwpz42zGX2c7wNYwNw6yjZ83/B9On/xFjL5xg98c+41/ApHjR8jJGGv8FYwwcZa3iNcVH9Bxk/9D1EG/4xsaM/Q6Ll10gP/BHO1Wq7tcLUdUjIaQ2KgolHdE+iQIOa33+WDtClpJ3N9SQ0fbPxpX7CAuQBXPwTotWfNP4R9atNQCG2rwRqt8C+DbB/PdSshZqqp2/3byaz60VczYMdWkaqeinj1R+Bi182m+RIu92Pxon6Lqr270xmbPqBKS3rEc3Kong3gISdFhhkOkErD2xVjRgJN246wfrwStWYAO8q7tgBJgd/gYn6DWTqCnD2b8WpfhF3/6YAHAJIJdSsgQMVeLUVZGvXk6l5idSBl0geXEvqUBlu7ZoAGCFQHm8Ftnei7G519nrky6TqqsicLOZhXQXjDZ+BK/tgUqvbNGJavz3CpDOM1k+LO73A/Jorj6jrHgHEbjFtfkyTqmLIwC80bWDs6yfDuROda8ZH32mFT3oSErfsjrfRdpg+Qvbqn+Fc+zL+tS/CtS/Atd+Ha79r6eofwd39JgeMBwMwph2SMvam9NshKAJA6Oc0vtKidjZZUcm5v3SuwZveCCkz/l7wXM4wXPg9YjUfNRVdLEAKie8vhroNVoDOAEMgeUravxktmzA+Wl0O1D5H4kAVbudPwugpM5OvqKI2/jGRaPGBOt8YQFlSbuI9AEQjpo40QFJ+ihxxrfO9h8MYWSWhqbNlQ0tAph7AZD0T5/8frh1+nujhlbgHF+PuX4e7bwf+vu341QJJJf6BEqgrwj9YhFe3Gq92A86BHWRrt5M9WIlzsNx+iNxyWgAAIABJREFUXleJ9wRyDqzmSeTWrMZTuLNmrVG1ybqNOKdKGT9cyOjBD+F1fR7GtB2b/CsFGSZJ8ZCoP4JWnRkp9wwcolOtpp2xWiy36QsxvbhnFpMaHAS40TgJINNmpl+bwqiCvNXWnjbpEYWJgEpTl6bJToEzYZfpOqopO4HvTpF1EmZDmWzaMRntM4drvPSjgeYSKFRjK8U0CcZNa1cizg0keszw8ULNFMpbA5jsFTjz6yRrP0hS0bRDFcRri+xc0EFN7lbBgSr82so5ETVKen0Z9m2DmuVwKIfsoVwmG14mPvAbJsUGd1ZIVzyuLcudJI6XfU8r8iNGus0ARPlKIzgMGzLLddQLAocy47Sr6EQP8b7Pc+Poa9xtzCd1eAkcXAZyGvfLnHoeb/8WozG8ukL8Q3kBQKrwajcFtM4C4z0ARBrmSSStRE2ZAWNszwYSBy1Aoo25jNRUMnXkH8GNg5AYMdJEjnMKpSOMkxBoxDx6vmd46XQTCg9MDRPnEEOK9JkJENif0s/ZteTKIrabKkSJMUXMGIBy6xUWNqFJreEIuU9t+F4caC9kri0MKJCmIVKuqOZfdJ7Bls4xzyetoXI/KvIQI8UkCSMs5DPpgnN76dLKhlYxa7PMOdAguj1T8Do5iN/zOdIHXzKpNn5dGfHaAkPGfzqwZk7ACAHFgXVwQDlvyu3Kx9+/FL9+KWM1hYwc/bv4V/8bZIZnKQBFZWM4fswsNdbYvNsrYkSg6SOpW0VfBJAHRnuYkpZCnVFNHkzfhPN/wp2Df4fhA4VkmwvIHsrHqyvGq8vHqy3Fq9mIe2Ajbl0x7sFc/Pp8/EPlJmTJgW1QVwEHpWLLg/dr8A8+mdzaCp5EBiC1miRcS6x6PbH6KtInS4keWcF47UpGa1+CPjltKlOpNAPVhRKDxNEaZpM79W499I7fiwMlOZTNaplY2lYRIpn+0i6PAhzKeQu9dHW4zpVWGwm09YiJss3YR8pk0DWFlpCkyeV5aofXjLaHU4DACxxRa9EZ5jSo1Y3rBFWdUXBC46qJvOngnuzl3wOPPLEHZK4pwqm0DpNmMwMQDy8zZdKO0m0/TKZ+G87+CqgtJXEw35B3sAxqy22gQcGGuVDdWji0CQ5uxtldQWpXHv6hXBIHcxmv28B002fhYYOZ95GNZarSe6rTr37ROAWq9YlPqF1u1aPGvLILi6yJpTmDQLJIa5g9z4fh0k4mjv4YD/dvJ3E4F5oEDqnIdVZLHMpDD+4cKsOpV0ivALe+HP+gkL4NDmyGulI4mA91AUgOVrwjQIxpJvPsbUiShAPrze/L6Y83VBI/WkyscQWx+kVM1FUwffKfw/B+8GQ6iv0yaJcpMYZjp7vfoXve7St1tFIVVSvLmlSKlivmod+ypL4ctyWQjFEVLJZWv2vQXG31NULWH8fxdJ1gIZRSVVRqXNmgIam0p8iohwwyxaQTQs2T9rWKzsNxHLOXeNrXdpcqxyRgKEQvsMjcnLm5Z9KgFiBBxUbFdmcAksVT5GzsKNMnvx+ncaPxQ2Vyi3mTh1aZ3DAOCiASmHMFSAXUr4H6dXj7NpLctcYIaw4Xkj5YxKiiWmd/C8aVOTBNwneY9jX6Fhi++PpdJETESixZw7bOklSmBtYkmylOrm18YsMwXM10288b2z7VsA4aC6FOam0HXs2Lxodw6nNxGueTPTKfbEMe2UOrceu24tdug9qNIMRLatQWQ22eBcrBclxFs55AzsHVPIkETmoEuk1kG9cSP1LK1KE84oeX4xxdQOpIIQ/qP0ri7Bcg8cDIApkk0pMaXLNB57t00DtDRKiQJpBmUB+K9F5kxH3wffDeJNAFzKnTgnXT0jTqcwk0Mbg2qElnYmTMfoCPrqfUEOEqBJ/RFvoj9CNVFE9FGHx5M3ESZvmrZmVmfFOrNnQ74a3q/Dm+LEDSpPwk2cCn0z2ZxVyJh/j3DzF65JM4R6qMteDVFBpwpBpW4R4shEMSls8AEAMwCdtSOLQDr+5F3AProaES6guJ7i8n3vSPyF79Y9zUJZR4orCvFLIBhgSQ1Q5P7IGIzaGRhLOTPDpZp5n13Or41Bjc3k+i++eZOPZxEkd34BxbS7ZhJXHlPx3YbgDi1FXh1OeRPTLPkgBysAq3bpsh/+BajMSQ31BbasBlNIkBSEUAkG9sLTjKA5A83kqrULsODm3AO1ZF8shqJmuKiB/Kh5OL8JpWcL9uPZMdPwsT/Qb1inzLsJEv4D1m2AfqVoB5K5lONGp2piPDQ+wH4jJBT9pEqyHU6u9gSwIdPDv5U1wUAEQ8rbQIs2RZZpmksOeazUMdXwaU9vASqWio/tnx0TjJjDNRGXn8Kd9qGhNREUDSZgY5hK05XvehC+iPbwJALJ95qJBENp0x2knmi5ZQuLHbcGcfIw0fgSMSjJX4e0vJ1JWQqi8kVZ9PtrHImN/+IQuSp20NPx1YhX+gAI6+CMc/aMx7o5GOFuDUrWKiYQuJgZ/FGzthxkQjpa4y5rUZUv1n1wBJAIVkgaP9QdwL4N03kkn9LItapL1nTOdPNOIP/hzRxh1m0so7+SLJk5t5cHIZd48uJN24Gk8MWl+J11CGd7jAkt7rs0PrLdUL1astqUPqywKqwGuowKtfPYe2DBoKoKEEt76KbP0WMod2kG1YT/bYQjInIsRPFDJxXFt3/RkkboIW9gdCdzKrFGhxipg5ULe2v6ykNe8FHKlSda3pUYMfKylDDa3P9b06LaTg+BBJT2hnO/LhIRZ04Y2Eg/Z4oqh+X8eb/3SoOXn2OXYBmO5CNHP8o0NmfWh/8an/12+q+0yKm2z7MaLcI+3fhvR1HrT/LtOHP2yiV1RvsNGmA9vM+ERPFDB9MpdsY4kZd/GG+OBpWuor4FCZvf6hqhleM583LsM9ugSvpYzx4x/GH/oSxIcgOYyTiptKPNFE0G3BpkrihTAlySwBIU3ELJF1Y6YCRMyz4FDI0Umrds8AXPk8tH836YNL4NBiOLKZ5PGtPGjK5cGp5WSOluI3rJk7HV6NP1dqLIYjK/AbC/AaNuIdegHn0Es4DRtxji0ne2IBiWPLiB/fQbrj5+DmPltWx1UMHKKeCg1I6UokBAAJOT/g70c5ajJ1JH5tp4aHGSZ9as76DjlBYAtki8cUSYZJowW3Kit0nJHWf0288RU4pIRJpZVshwPbcQ5vYPJUIZPNeZZ/5jj+NJRDQymmrV+L3yBSmk4x/tHleMeW4p0sJN30Aqn2H4crfwqxQcjGSacgqjVVZjwVvbVmaTj/p4CKQBIxYy7fL6mCW5YFHH8YEtUw/pskWz9K9nAZ1ETMRIx1qDYYcyZ1fA1O4zOAQw9jOqcc//AcSB0RAqR+qwWIwCGt1liKc6SYeH0BiRObud/wYaZO/yKkZGrFSCWSJnPK5mfJeQ0UxGMiV3+IA2ymqhWXdnLNdux3CKPP9TFkqpj6COqTSZL+/UCI3CF2/Q8Ybf00ySNbLUBq1oGobi1OYznRk8VMNRWSlYCdy9gfLofDxXA4DznlHLIpTv7hUrwjefhH8gwPZA9vxm1+jQdHNnO/9R+Zbau1dimbcEwdcw28xlK+t5YxK/iufwqLi8x6EBN5kWUQ6uJUL/7Ir+Fe/hST9ZW4tQVQuxRqta5Ys+Mb4dB6aNxgESsmD5n9ads5do7pVAOQPPzGQryGDTgNmw1gDUAa1uEcXke8roTEkUruH67iYcvfgzt/AdnL4GgpsawMJRfK1LKZy+E8hkWMNIY0jAASgkQdJdH51y8xlqpqJh2bnJlOaxJTxfmGGOn/UUabt5I5oiiTQvqiUuM8e0cKiZ8oJnailOyROQjGkGdkQTSugsY8oz28hnV4jeV44gsB5fBa3MPbcI5v537jSm4c3Ub6whdtpRhV59dgy4IOMtY1stYHF1gUrPKIRLlPQkvtLYxAD/mwlvTQDzPduo2E0tYP7oD656FuG+zfDvuVyr4eDlaCNMiRCvy5tjr3SNmcicYyOFKCf6QE92gJ7pE1uI3r8Ru24TdsIXmwlOThIlKtFUy2v8h474/C2C6zTUJoVdn5ChsMmq1AbM0s63Q+8s6kUWRu6ci/2kDRVECcaSY91X1Ut2gtfBRGD/Kg5UNMnVqFe7QIDhdBQyE05MORlfjHckkfLzOJpe4zjL/GnSOFcCQXjq3AP1qIf3ijGXsatsDh9XBsDenGQmInyhk9uZV77Z+G+18CX0JSmylZB01NCBALEssPkYdcYJrbJlpialqNNeFc+vdEOz7CZGMp2cMb4ejLcOQDcPAFqN0OhzbbUJom/BpD5g4Z/Wnb8Py5tBVwuMoA1D2+EufECpxjZThH1hvJIYBogU5aczany0h2r+P28RdJX/lVu5FmQuvtbUc8Aomteh7+bftPjrK80TCEIa2iCNU3IV/821gRaeJ1iklGmcTsMSQOmxyAi/+esRNVJE8shKO50JgPjSuNv2gY+ViuHaejFbhH5zLu4TkCSBEcXYl/IgfxgNu4EbfhRfz6HdCwEY7bbA+/ZS2pjm3cPL6O6NA/hWSdWQVrXIxZ4XaBwwQFA20SGeE8ca7hch9nopn4lV9lsvtjTDVVkjieS+p4Hl5TKeljVcTq15I6vBZOrYNTRbhHlsOxYjhaCkfL59jq3DnSkUo4vBWOrMM7uRS3aRHuiSLco2vxjmyynzeU4B/Nh5Z8Uq0FPDhRznjH98L1P4eoainZ2sICgt3yTHudaDsHK1FCoEhaKnxpTS2p578GiGbRJ5hmTCaqOEuK9eGfE2/7hNEQcpQ5mmfp+HI4sRKOFbwNz8xx/MV75nqr8E6swj1RgHusEq9RAvwFOLwJGpfhH1sEp8rJNK/l4ckiRjtewb/xKyYbnawmZAMtEqxAFEBCikS5RUZrE9whssP/i8nuTzF2tJh4YwE0lxA/lkPs5BLGGldy9+AqJo8V4Xeuga5csicW4R2f9cAhWN5rO1dghOeFAGncYDuoaRFOkzRJEd7R1SBqLIKThWQOLyJ+dAmp1jU8PPYSMc2NxNpmOkeTXjaLQLPOsRl1GwJFALKhP4FExQCCGXGZpn9FX7LTJ0iaYnFW7I7BvX/DpMyeYyU2BG8kfAHeiZW4J1fhHRMYVsORUjgSCte5AqQIjq2C46vwjhfhHi+1dGw13pEN0LgWrYOnKRfv6ApSR1eRbClnpKmKye6PwvgfgDKOBRAZA8EiwVCLCCSRTDZqioulbuxntOVzJJtfhpMlODUqCFBEtG4BY4efY+T4Ukab8phoKSTenofTtRg6l+E15eKfKMY/XvK2LSdKzfXettV3z0LHK+HIFji6BY5X4jcV4LUuNERTnr22BuJoARxfin8yl8zJKmInX2Sy6XsYPqnatNfMwiY/JSfclkBS/ELLUdVRIoHEmlryOYI5Eb0VfYcDJJlM4rrGw5gRA0plUbE59Uk438h0FG7W4PR+F7GGCN6BFXB0HWgW/dhqOzanivCPr4Nj66zFIRAdfxYeKIETeXCyAP9EFd7JKmPteKfyMON/UsK7HATKk0vh1EqyJzcRP7WFyY4S7rV/GGInbE2xVAIv7ZqJWo23lOG0r1ysWILszS7GOn6V0caPkT2sybw8qF8LTdugZQ20l5LtLCPRUUa8o5hE6yrSzYtwTi0OAFJowXHiG1uBjZOS4m/X6rOA3glIOubtvj8hLbEBjm6CYxvwmkpw2ubjtEfwmxcbyYGcRCPFVhnV7h3bSPbEVuLN6xhp+Qjc/e8Q7wHtXJuJ4SY8m2wYzLiGGkSmlmETASLAyaPZtxne+Y57k1UF+VkvgSOVSpHJZMxkm9GsWpE60YV76TdJd1bhHo9Ag6JLSknaEoxNqVmK4J3YYAEipj1WBCeeYfwNXxSYa4QA0XIHp2UlfstCaM6xADy22gLpRAH+8U2kT1UR75rHWOdquP3FIMviAXhR9LwSjgKIwB9h/C7x7v/O2PHvMrOOnFA4Ls+mghypgJPr4dQGaN0E7evw2yugvQRayuCUkFlkpIPfNJdW54RUErx/760BzbEqK6mOS3qU4LTm4LRF8Noi0LwQjss5FEgqbMqDgHSykmzLSqbaS7nb8Qmc2/8Zkn2QiJuisr7WUwWJd9bcUgasXU6qyKARnValfMdrkFnYMFojnU4jraLWJHuqH5K38O/9IdG+v028qQhOSVrLzKqAxu2g9KSmSpymciPlpe2NVDcAeYbxF0CkgY6X4mvsT5WSPbXaZJmLB/zWCDTJTy4H45NughOr8ZpXku6MMNWZS7Tvx+HO18HrM9u5+dkJ4umMqYov6yHC6HmiLb/JyPHV0LMYWjT5kgtHFkDjAlAlioPFpOvLSB8pIXN8hbH3OVYGRyugqdR2hjpkTlQMp+ZI+u3ja+DYWjhRgd9cjNu6Cqd9MW5HBNrmQdMSOFYIjZq3kaapNOrXb1tAqnMxd04UMNb1d+HG/4DoDSs2BBAzy6p1G9q9V6T3wfJSM2nyV8PECgGi7dlkakmDhCaWieIlJ01KefLSZxlrryB6LA9OFUCr+KECjm6DYy/gN63DbS6DpjI4KbOq2FLTHMdePKPxP1Fh6VQxbnMxmZZSMq35OO0L8NvnQctiOKmMi+ct6bdbF+F2Rkh0LmOi9WOkz/0bmHzd1ghQqkx6ykasDUCifcT6/h13W9cy0TmPZPNyaFkCp8VgEWhZhH8qzyDUbcrHPbUQ9+RCM/egORBOlUFLwZOpOR/eieYEqgCM6tzjZdb8kt3ZnIfTIhVbGHRQBP/UAjiuUKMiGpIgCj4shrbncDqeI9a8gPGmtaS7fgzu7oLkHe1nYBas+07UOO42/UCTR3Yy0ZhYf0V8kLCwtVqB5LGXKqmPt8Dl32SyZzvjrYtJyKIwTFgIHRqbdXBiI37TGiPANEacWgUnCi3jPuv4Gx9WwKvAbS4l07aCdMcy0vKTW/OgdbH9vaObQdSkiOYi/LbnSLevINYmf+Rv41xWTed9wEV8Z8xsnaDxjpC+Cnd3M33lJ5m88Alig99Dov9lol1FjJzIIdO5AbdtB27LDvzW9fgdi6H9OWjRepBiKykkLZ5E7wQefddcNHc6Jd9GpOvkGpC6LavJtpbjti/Ha8+xdujJXDhaaemkgCSALIT2+dC6ALelkHTLh/H6fgZG3gDnnNlsJjl536TFS3M8CveGVpU+lX0upHznvkKN8VZH3Txx5oZZJ+/2fpqp9nyi7QtJN5eQPVlmwdBRbiKhSHM0yyzPh9YlVgCfyoemkrmPvfjmlPwXgbAcmlbjtZSQ6VxKunM5mbYysq1l0Cq+yIWT5ZbEJ61LoXUlbls+XncZE80bmOz6Xrj/e+B2KlZt19QYJ13FAlLjuNNncSaPQ3QPzv0vMXH+3zLe86+Y6vgcsbafJtn8z0m3fJxM1woy3REyHRGy0jDtK6BVKvUJpE55Ij0jQKSZZO/KzhRYTpXjtpTbzmkrwW0rwD+10n5/cjmcXAUGIOqkJQFp0EqMepY0yV74LEx+GZIDZKeUOvFoD0aBRGkpZnvrmXV8Asp37is0q6RBwpc+m5qaInG/iWj3D+C0VxqJnWpbbswbmTlO8xoj0WlbDm3q65XQovcSSvOtQNOYPZOAVPBGjC8AFuC15ZLpyCfZWUC6XQCRv1EZgFPAXGjvwQjmcgMoTq8g1Z7PWPsHmTr3L2F8f1CIW7V9IaI1I1pPnFbRQVMoSTVWr4FzEbyLMNIM9w7BzTfg8m+RufgPiJ17kYnezYx3bCTd+jJOy6s4LS/itm7DbduI37YaX458Rz62g4TYgNRJLeqs3MD0eitIZLIV4bWtxGtbjteabx7EPuhqa8pJKrQKeJIOS+CUACJptBq3tYxsW4mRHnrvyR4WSKQ1TikmLpNPYNY9rIQuResK8ZoWku5YQqy3itjQ98OD/2krhodhLNXaDXK3lOCoORNVC58Jos947gJMSKEdFrLWW9vw+0fM99YjvvFvHTv7+sERb7F+vvG8Wcfp2JCMFgwnPYPU/mBdhCZHwyW11gOTxozhJm8yMdzC5MUv8PDoFtzWZdBZQKYtj3RrLm5HCdmWSpInCvDalwSaejm0CiALLZnx19iHJLBIy4iK8FpFBUbKu225hvnFC9ac17EKEgXgMD7PcmM1CCDpjiIyhgfK8Zqr7LHmd2X5iBdK4FQVXrP4NA+vp5ypzud50PV9pG79PtBrsibcdErp7rM6y3SyOkFBrqQtja9dUc1S0HG0nx3eADgdkGmD1Ammu3+HVOevkez4HIn2Hyba/kmibS8Saysn0bYMvzcHTs+DLmkbRZbmwcn5cCwHjkj654LUrbRBSwlKCXAUVj49n2xPhGRbDpm2cpyWl8k0bSPVtIx0i663AnpLwNiZYvhi/NZSQ15bMV6bfe+3hWZgqOH0d/CZAhIKNhzXdZZC53y8vgWk+0pJnP4E453K/r0KsYytjKCZI8OeWr6kPKQkqewkjjONp6WyBjBxfD+K40Uxc0yGmR9lAAdsOhMrDhfoaKJy9lA89kf4hTHnwvFREFKMHVxKuAmPe1Ib4nF2kCGtPdqDfdqVm5SeJp0cIZoeYcofNxOBYyZbz8wMmN3DmK4hdeHfMNr0UTJtimzm4bcJDCEVojHw2wrx21fhy8poywtoFbSJgvFoXmV8RxMRba7Ab60w45/pLCTVvYrk6aUkexaTOr2ITPdSnI5VZmy9lg34LeusCdW2xJjTbscSAxIDKAnW1gBIBnShQC2E5lJoroDmdbintnGvbjX3jlYy2vMhRs78E+LDvwsZ1TEYeRuAPKFzTeKjQp8qjZX2SKdc/MQUZJWuMQSJdhivgTtfxr/6a2Qv/ATZc/+QB03PM9a0jqlTFSSay0g3y4kuNSFlMTRdq/E6y030IXGqmERTKanWPLI9C/AGFpLpWoXftQE6PoDb+hLptkJSbYtwupbC6VVmYASCuRD6fWO/llrV27nQBid6l+L2rCfd9Sm48QV4cBQS9yFjCiqZ6Q8rQlwyaKGQdiHRbubSKAk8L2FLkqrqvUkPDpg3QIedtbf1qbRG3G4oGWiRsP9DZp7dmu/0gUAi1faWRV76Ssc8FkV4y0rJ8HohSJJBKSEJQbN92SiOPxqUBkrywFHdSNcsozXrPKL7cK/8IomeD5HUOLYXBmNQ8NStAY20T+uKwDKwgi3bUUyqq4Dk6RUGHMmeBaR6FpDpXozTucIAT/NzApPAkO1YOousJnHb5WMoB68YTpdCt3itFDpk2ZSYuT3a1kLXazjN20l2ljPds5oH3duZOP9ZmPg6+L08vj/II/H2De9kXphCG9peN+uS0QaJaS3JUpEBbUg/aXdl1TpqLdNNaOuvq7ao2d1quPUVvKufJzX0fxEd/D7G+j7CWM8Oome2MN27jrG29Yw2lTHZstSEX+nLh4FKaFday0Y4vRm6NuO0bTKZuYnOJWROL8KXGddeNDdqk6ottoMjqdaeCx1F0FmB172WdO9Wov0fxLn+4zD9v8A7axfOyMIxeTvSIirZM2FaUw5ADKjvQ69erfg5OEcdqzRqFQ5QqoYFR3hAwMxvZeKQmdXqOuHfBhCBRjG5YdIooU04u9Uxb0M6X1pR92h8qmEy3CLBXVOlS1eQTDC/qe2nJ0+QvfILTHa/xlR7LunORfid6rP8OZLOlXmWAxJOXTn4nctxO3NxO/Mtda00n9G5HDpWgpjemNgy11YabZPqqCDVWUqqU8DKM0I1073MaJx01xIe0VLrQ3etItOVZzQe/VXQVwI9EdLdEaY7lzLR/Tzu5Z+H+J+/O0AEjJBCgGgPakfFBVzHbHWVcFLGoTECVuMQMofpYY2N6uA8hPQgJI5A9HX86S/jTf4W0Zs/TvTqP2bq/A8x2fdJpk9vJdFdQbZD4FhrHT1FyLoK8bvKcNu3km7fQrxbEmaZ6UTaCwOAzKHVtY26z8NeZzV0VOJ1VZDpLWaqbwWT59YRv/YDMPoFSPWCmEUZJ1qVSNqYIZOoBI9vhIhhKPVByKN6L0YTQwZvw69kYs0s6Q21TQgQtSEYZrch0+ozk1EcACNYjz5jIpu5YJliYUJIYDrPmNCPcmm0Pj/JQ6a4yzSjpsCBWQKh8Uw8hFvVpPp/gWjnh4h2FpLuysE5LYZeOUdw5Btw6Xy/axm+wBEAxABOQk8CrK3IRkhbQodeJpL8xxzr/LeXmCCB07Eap6MCp7MEp6sQpyufbHcusZYc4go/ty0h2b6UlELAinLJMjGgWwYdEdxmmfMR3IEI8Z4iJrv+Jt7Fn353gMzUGnjrxlOqwOFbASTrW6TiZSFpd1UzZZ/FgEemmeeEeymo5pO83jH8dDskD8LUXnjwP+Dar+Ke/Qzprg+Sbi0j0zwfpyVCtjWC07aEbMdqMt2bSfZUkTxdjttVbBxEOiX5C3jqVuAyWiiUgrpeMV5XAU7PCtKDOUwN5jAyWMjEhe8iO/xvYbIBEhNGEIj39cyBR2Aqk5jl66GgCJk5ZPqA58OPH/Mhws42TC/zaRbjP/Ze380qHxSCT79hflzw0x3J5HvkG1ng6MZ0QmCO6TIKZHqPjtaZGipzmPacvPtnuAN/n2RrCamWZZaxThfA6aDPu/JgDmT6uLsUp6sct7PcmNq0r4Y2BWPWQnOQOd6kQMoaE8o1Gl9zKc2Knsq3CfwZ+Zptpfbc1rXQKv8kIJlSum57CX5HIZ78mI4VZsrCO6G5sgiJpgjxtgjZMxGifUsZ73wJZ+gH3h0g0haimbELNEooCbWb6Zg/kwQeBD89s/zd1uNQ7SbPTN0nHEhqP/UEJGKgTUgd1XkyVUDugXcJEi0w8jrc/E9w9V8YU8fpXkCyPUKiPUK6vYRM90bSfevJ9K3F7Q6BIXA8JWlQu1daEsA6iq0W6cjF61xKtlt+0HxSg88xPfgco70FjPZ+F6kLKox9GOIjNgYc+g1GSfhknCzxRIppVZcPlnTObi13iptnuQvSBobpW/u1AAAgAElEQVTBw0iSmFwSX+wqCqW/PhdApHmCc2bODZjecLaAIAqON5GpxwMBOl3Va6TgtUZf1T5mLC79XvYc2cu/Ded/EPrXQvcC6BDlQFe+te3nAIwQTJ7ModMFZE4X48gX7VDe3xpoXWNzAAWQpg3QpOUVa6BZ5rAm/wSMFdAus2uJpfaV0C5zW2NYZq/TVgkdG6FjA7Svx2+rxGkuI3WqhGRTEfGTinZuxu3cQra3nNRgKfGBKsb7XyN16Sdg/D+/O0A0UaS4dzijOns2VR0cDqM6Vqu8tVN2nAfEuE2Cm6S4Tdps7jhhlLhklzHZxAeu3Wdc+1QYZ1bF0hRvFnomJmCyBy7+PAy9QmZgAfFuAaSMbNdG3L5S/L58/O58mCudXgH986FP6l3Om6RMkYm6eJ0RXEXe+nLw+paT6VtJ7HQhk53rifV+DIZ+Dq5+Be6dhrG7kBizRd/MRpEJprwkDzOquWs1jPJ6JArUZ9Zx12KtoPxpaD4ZgIQH6Q91jCgAkzk3+D441jr5ykLW/Iwt56/fMBQIs3AjJNO+xWrzFVTw0wYoRvkId6lpstNv4D78Wcb7N+D0F0LfGji9Gjplii4BZXP3LA8ETC50Pz35PcvIDkQMub0r8but9raCShpB0UUBpgI6ZGYvg57n8PsieL05NkLasxB65sPpoO3VeNoxpW8x9OVCfwl+/zr8/m3Q9xp+/3dB/8dxej8DF34Xd+jLcPO/woMvkb31X0nf/EtTYhfn3nsHSAgSI/Vm/acCcyILHMFFal2p45pPuYNnViuqHcFjHOkWVQT0Xc9YWcbq0vyC4QchJli4p+IYIknqG79AamgLsf4ckt0FuKfL8HuKoEca4OkHJjxHA8Tgc/j98w3QvM5S/I4AIB3zcZXL02s1TLYzD6e7FG9gLe7AJjI9zxNt+YTdj+LqmzB2DNyztqIHWnSl1XYm99HssiugCCTCgl0LHdQoDRjdcrTtWBvynV3uZ5Yror4K+svCR9saq2qkStbY2YoQVo8BIzxvFkCM/2PGSoWwo5CagOlrZO/tYfziP+Nezxrigwtx+iK2n00EqBDkd3QrfC9aMecx8HtW4PblkO1fhNe7BL9nCfSE11WrlKcV+KdzcXsKSfWWEOtbw1R/FRP9m5ns20ayZysp0entJHu2E+/ZQaz3RaZ7X2Gq50Okzv0d0kM/gHv5x+DmL8Dd34SRL8DEV2ByP4xfIHvnEkQvgn/Njp+CTupETRTO4vU5vBUyZMTOIv09Q2FUJTQVQps4XLoqyRdIOyP9BLBAaEolhYffbyZ55WeZPqsiYItwJSVkTyoPTCZWz6q5Ue9y/L7F+P05RlPQuxxkW0ubGCqC7lXQLSCWQPdq6KrE75FEkgRbSeJUFdnOT8LQ5+D+VyDeaje1N7UxrPYI/TOjZV3XhMiduEs2ljUVFNPOFBlPJULF6KpwKVJwVaVoRCqXak2gpI8xV5Ne0JpAlN0YT6asjpMuMUVrfG2HbPfsMAWmFa71M2Qczd/ESKfDQndX7bzWw68Qv/gjTPR+gMmuzcS7V5s5oWz/KpyBHDzzzHm2j06vhNPqrzn2vcasu5xsy3YTmfT7lsPZCJyP4A3KUY4w2RkhdSFC8sJSYhe3Ebv4aaYv/Sumr/4Hpm7+B6K3fpvYtf9C4voXyd7+U3iwEyZk+nZCesguqXVGzF4hpjq+CnVoBaFK+GjORwWFJbFMirYV7D7jQSEP64c9I0AEs3BCUVPy+kFbKcIgcLbpICfG2MQCi25GIAm26TXhTlUWETPYpEBpFqPydfhkGu/OHuKX/qlxmJNKpOyQk7jeDpak/BzJ712FGRxjaklNSzqWQKfs10roEkgEkIKABMjlMBCx1CtToxS371WcMz+Ie+XX4OHXINkO/k18Rs0KxaSXJOlmTd1ps/4o0BxZL0nW1NC1GyGEhctsETNVXFHFS81oPwqKGbAEGkmgC0ldpe8eeR2BSacKHdkkmeQkWWkJV7pNY3AP6IP4Trw7nyd1/h+QHqjC61sJfUUwUIHTU066v4jk2RxSZxeSHViO17fKAuNZwBEAhK4PQOfL+D0FeAPzcM5FSJ2PED0zj9GBFSRvvED85qeJ3/4lUvf/B9mxo/iJPvDOABfsgjd3GNwR+1wh0wcaYCaiqh0lAtktv8vUT9beQmHcyFS/j5tKxyr7E/Zx5Bsmld46yfSOf+suQi9Erf7WyIcOp1UP8jmkKozvEah4fWTUh0BkTpF5oOrrsrJkuz8we1joigZX02M4t/6EyfNVTPVG8HsXwflSAwwx+JxI4JA2kLlm7NWF0KsJSGmRSujURFJpICVzQJqrV8eEtNisrNQMb6p7Gan+IjLnN+Fc+wT+7Z/CHf7PkD4JqSugOSP5zDJJg6Wd5tlCFaruMCnlvgmfy9FPOtrfS7P2tuC2ClUn3/Jv0plmyomZIngJ3xZTNf2tPhUJWWHgylTxiEL6ISRuQ6KD2J3PE7v+A8TOvkqmrwp61tgMhe4VZl5C/ldqYBWxMwXEzq4ieXYx2QEJBYEkz7TSpHMhc43+EhhcA4MbcPrKiPYuY2KwgOjlV0ne+imI1sPEdZiSbyrhK6tDGla1q4JAhfrQc021+2wmTiodJZGeJpqaCjYLst0gDn0raQysHBf/2cqKj8xVlR41vRj25tO2spPtTHCYKhEqDdMGdm94E2pnkwFJeEJQh0gAiZJkmltEGTaAMbeoJxs5TfL6DzN1tojU4AI4vwA0WP3LoF8O91O2koS9hdBTDL35oL/NZ7n2MzMLq+/0uX7nORiYB/0LoHeFOTbVtYBk7zwy/RGcs/Pwzi3COZtHZnAj8b6Pw93fgIf/G6ZaIH0zcOS1eUvW1OD1ZDNJBejBxcxhZwVDoWohUspWfEibqD69FnBpjYpMMn1jB9lEGnWNUEmbCIFCh5q3eQDuZUi2wsguYtf+iPGhX+Th4HczMbiGeO9yXK0HklaQ+dS1yKTe+P0ryAzkkzhTSvxsMalzS3DOLLL9HfbXXNv+JXjdETi3DC5WkT2zgdHTVYye+QjZe78M6Xq7kWKgIhXpVoFvdVnMc4m5jonAKZ9Q2M9mfLPSMZNNkHaiyHQN657ZfrLWibpodldbGaUIn0o8CYA2amgrKz4DQGzKxONMbwAgtRXSrJvRoiO7tkKFEe6ZTTWN3hNIBKYA4eoPYVll/RNmB4yslYKxEbMpZOrazzBxMY/xM2LK56B/SQCOubSLoE+OpsKYMtnKLEgGnoPBSHBtgUHREIFQQFF0ZAX+wDK8C4twLi4ge24h7uCSQJLmQW8xfs8asn1byJ79CM7lz+Ld/XUY/7pNy3HugqJ2GqnZYi3U7eFnihQnbAVDL+0bU0m5X9o2IasJSwFJnR5eR+CwHWjb2ChM9cCD1+H2b5C69hmmL36IyaF1TA+VkDxbQuZMAU7vIrPKLq3VmFps1rsAzhZBr8zHMpz+YgMUZyAXd2AZDEg4LXjUP6Zv1D9PQQOLyZ6O4A5GyAxFSFwoIXrxk2Tu/DokjoP/wFTz1opeJ4UpF6qa0Yp6ahMgU+ZUyHgrxxsmnFENto8MCtRZYehbroFsrVC6BBZN+L2ZdFXp0W8FQGZpCv1+KOGsWpR61IYuD5BDZMwsHSB//y1jLR4xm904U2aDeFIqqHoPRvYyeePT3DuziszZlTCwdI60BAYWghzEnnLoqYRe2d5LrcMop3FwCd5gLt5AkSEGVsLgUhhcjndmCdmhRTiXFsOlfLhQCufLYKDISGLNEDunl5DoXs5EVyljp19kcvAfkrr6y3D/yza12u8B9xw4lyGj4toPQflRiRTE3EfgERNkpU60h7N8Ny17jEImjukXTS5pdZ9209I+h8mzkFQm9p/DzX9n0vhj/a8x1VNMtH8B6XMRGFK4VH2QB2dXwJnnyPZFSPdE8Pol2eXbCSRltu0rAFH/SnMsZ2z/mD5RvzwtDS6BcxHivRHGTkeIDr0AD38dsifBVdjcagWlnaeUca6ijYGbq02BTJhTfaF+MTRr7/JQwDwGFplockQUPAoA8tj3gVkq4ATHRd5G/s9i7/DsJ7UhGnU3eh8au0Jq8LL6y3L/W9+H0k+XD7SIjb9oNyYb01fJRy+TxpeZoH23tVdf7AqJe/uYvvxLJBXbVkfPhfqXW80gp3Qgx4R8zaCfmR8AYyXeQB7uQBHuQImlwQLcMyvxzi7EPbuQZJ9MkGI4VwEXVsOFcjhTjNujXKXlMFiC119CuqeIeHehSWNI9pfinFtL8uJ2Jm/8LcZv/xDT93+S9Oj/B9NfhMROSJ+ATD8412xCaHYEU9hPrcwlkXcrAFcbJPeTnfxjEg9+hanhzzJ57buZurKVyaFyEufzyJ5bhX9mFQzkQ48icuUm58wsk+4shDOFcGkV/pUcspeso5wxgQgFLpbbYEWnzivARPsG50PQTwwoaDEHOpODfz5CvC+HidOvkLn4CzDRaKteah2OapYZ5ah6AFpeoK0lFPtXHyjJUoEGmUWWFMOTESVBrKkEV6onVABv5T39Lf4LNUjY6jOT8mN5O9Aguo2QW5+21U1Ir4tC2M4Cy+z8ovAm1M7GXPi5btpcR4FRXctmz5pGE4jZuO21dJLM5D0y9w8x1vd9JAfXkRkswT2z3A7aGZlHOVYTDIpJl+KdEc0C0oDMseXQK6koiRh5pDXOKtS43IJDgAgoa9o88zvuufkGIJwXcFS1cRVRrSfpzCGl+ZUzBTBURqYnD29A70vgSjFcWW4kt8yK6b4Ik5dzGL2Sz/iVtUxce4mpqx8nevUHiV/5KWKXfxnvzn/Bu/0l3Nt/gnvrT8ne/jMyt//MhDWzw3/M9LXfYPL6zzNx7UcYu/JJRi5t4eHFlYwMqdxAhNGBCNGBiIkOcUFh1FwYkLbcAF3roXuNiQhmOpeQ6FtI9sIS3GuL8a9H4FLE9ouWLGi+qUuRw1zokwZdaISE6degj42Qmq3N1e/qx8GVVgsP6rdX2n7vk+ZaTrwvzwQI0hc/B7e/DmMXIWHNJpM5bgAi01xb1N0K9pdX1oXdnFOTnOH+KWFrLRWBJvSPQ/4OzKgQLGL1t5LhQZkzOkfFq81LR+k1l1bnhNz+1l8L7CZdOryp2a0On/23uQd7YzN3Hn4fPokBXPgAw8Qe/jfunP8h7g++SOJSMdmhYGCv5sKFIvzBfOM0Z8+tIHt+Kd65pXBGJlng0A/KtJBJtQDOzTetf24R/tml+GeXPUbezN9Lsccstt+fWYEvmvl+1nmPfW7Ps781D//8PJyh+WSHFpAdWkxmaBmZ83lkzhaTOVNBZnAdqcGNpAY3kxrcSmpgu6HkYNhuJXlmM/Gz60meW0PyfCmpoVVkhhaTvTCf7IUIzoUI3lDESGr7fIvs8+q5B6VRVoHpg2Wmb7xzi3HOL8I7vwDf9IeOXwyDmlQNyPTXYnO8NJP61j+72PbhwCIQnVEfLsYZWk7qfB6pc6U4Z8vgrLSVwsjSTCXcaf0oYxd/Bab2WG2onXy1D6MsoWCLd+tiaApAQQlpEjnRllcfTarKyVbQSK2lGR6a4WvDYI/+C3krbB99M4OcZ5wHeeyK78Mfk2TSzTy8818YufpZoldfJHrWJpxxYQWcL8Y7U0j2bKFhnNTQMpzzGkgNYuC3nBWTv090TqCcZ+xw2eJzITGxf07MPFeyjDyXPpDAyJ7LI3s23wgII2gGF4EijGd1P4vInFtC4twK4mfzyZ4rsZpUWqx/vqm1Nn7mXzI9/HVIdmH3qlG6keVPhaslLi1ANBckTaKoXbBNYCBfv5WM920OEEmSGzjJg2Qe/g7pW59lerCS6e75NtJyvgD3bBHOuRLSQ/lkLyzHHVpow8NnckB0LgfOv1+ke3nOzB7PrdW5CnfrOnOlZ3j2czJp7YSiWbszKA2x/LE+VUg4O5hDdnAhvo6/WAgX8412muqvxB/5HfyYIlZK9VAoOmXTxGUkBJFNa5/IDxE47D6Nssol+L/Vr29zgMjXUSRMdmsjjH2BzOVPm5VhSUWmhlbinCsykit7vojs0ErcC2IoSevn4OxCOCfGegYmeaZz9du6HzH6XCiYC5ozOJ7x2c/Jj5A2CEAyWIF7Lh/3/DLcoRz8oQX4Z+bBWaslvbPPkT6zgtT5CjJXXiB54zPgVQPnbA6UP24mVJ1k1gaRAh9aGiPc0MZuSRGA4/8HhHybA0TLSpQHFofMXRvWfPgfyVz4W8QHC3AuLsW5UIBzvtSQO7QKLs7DOqvz4Px8OL8ILryPJI32rHRhIcyZnuHZBUwBQNGsc4vh/AqcoWIyFwpxLy6Ciza3yrSXrB80PbiUiaEPkb3972HyEDAUVBEZA2cSJxkjGU+YZd2O6xt/QuB45HAHfu3b+g3ffH3ybQ0Q9VEyyBq3AbQHkKiD4Z8mfqGM9JX5ZC6uMoMmkLhDeXbQNHBy5gUQMef7CpBFAUDm2M4ZGCGongEgQ4H/pPki9eeF+TgXVpG5mGcBoiiY+vpqBK5F8C+tJHbhg8Su/zKMd9mJUi3ZVl6eF8XNREmnEqb2bzqbQbvnWsfbAsR6JIFtFcaDvsVa5NseIJkw3V7uiJYxpgfgweeJXqkgcT1C5tJynKFC3POleOdnA2TeI8n9zEwWMtvTtrOYc2ixBerTtu/bvetZA2AYcFgweJciuJfmmwwDo60FjqsREzb2br8Gd34fJs7a1BpZyCYY5Zg9610nY0ubellcX4ZVOF0QtoH20DlyQtT+NUCerDbVN4pwKK0lE7VFOUwh5YdfYPL6ahK3IqQvLzKawztXDAKIBtVoEJk2YsqnZepv9vHSHIvnRkbzfbPv52muJ60R9Kf69DGah3dpHt61CPGLS5m8uJXM8Ocg3mUHLVwVocltOeOhoDM8H4ZsBQyFtNTOWkH51wB5MihmfxMCRGsjTDxQc5XOBKS/xtjtSkavRshcXYB3cSWcL7STZBpEqf4LS/AvrMC/mIN/acH7RIvs719cgj8n0r0vep/uXX22CG4sw72cg2f8uQVwcQlcXoh/KULsfITJC0VMX/t7ZLQvYKzT1uCSz+hlTI0CZTYrSjV7rtgu5HoLOKQuNOAijXdIsxniW/D+29zEUjzcIe27NpVewsZVbtJeRu+s5+H1COnr8+CSIlrSHsvh0jy4PM/Yw97lVfiXF5sB1aC+L3Rpsb2HS0ufukU5YJdz3p/7vrwQ70oO8cs5pK6thGv5cKUALmqCdjnOhRyiQ2vI3v8XpB9+hezEOUgZO9gWkvAn8bLKywusgEDGWQyEk8WB5ngrOEKQqP0Wv77NAaKYuGZYlbvl4GeVyKe850OM3N3G6M0QIDKllsIFMVMErszDubyK7JU8vCuL8a/Mf59oocl9EqMJqE/bKm/Kv6JrPAvN/dmz1+YzdT1CbHgp7s0KuFJhtfKFUvwrH8e9qYrpbZAew/fs/IWiUWaxljuFrw2LvgEgdgnFjGllEfNIY4RFML7FwAgv/20NEClnVTPMKn6ueirelF0qljnF6L0PMHYrggaRSznW11CI93IE/6qK2q8kbQAiJps7kzzrud5VK4kFjqemq88CDJ37bM+tvo3fWcz0neVMX80jeqGY5MX1cOP7YOSLEDsNWWUdWjNK/qIAYUBiFnBoCfAjDTLznQ1JfqPfEVhZOk6LpRT+tcgJ2fmb336bA8TD99K4qupoikQoy1MiqZ+Je59i4lYOzjXZxEpHj1jtEURU0tdXkL6+yiTmce053he6ugD/2iLcOZLO5apowbPRHJ9ffZu6XcbEjXxGrkcYv1lK+s6/gIk3ID5uI1WBby1RpjWAqqWmajZmnLQc4zGAyDmXnSxnUl58wP86XuCYmVlXzq6WI2vlvQb8W/f69gaIOtBRRpvWWA/bNdYSKpnrTN39DNO3i3CursLY6gLIFRuPd29ESN1YQuLmCsOc7ws4rj2Hb2gB7vUFFiRzaN8vcKjP3KsrSF5ea9Lqo3eex5n6p5B5E3zt92gDUI7WcHguGT9lSNEqJSEaVaLIVYAVtdb8CjPDNQEcHKcvtRRmJjdLK/VjM6b1tw4ez1zV5Ft5a+/h2gKIBI4q0JlU6LvBwDwgevcnid3egHe1CC4vseBQTP5GBO9WhMRwDonbS3FvLAQ58u8LzYfr8/FuiBbOoV0I156Rruse5vj8V3Nxr+yAG/8ERv4Akk1mrYakuoYl5vlmWWzKyZr14mZfu0BDSFPITJL8F+m91gLNFMkzi5ZmAWSWtlFJpYTZy16ZvZKI37rXt78Gyfi2IIJZTKNUaYFmgqm7v8T07VfIXiuxTBTM5gog7q0IyeH5hrybFjT6/BGJYcQ4Ak9Ilpm5MW/WcW85R98F5z1i+Hl4TzxH54fnBNc357/H999wXd3bbGafHwBoCVxdakkmp55p9rla+xHchwVreB2dn2OvoWMM5cD1HNRv2WtFMPxD8FD+Rp9d3aiSs6aMaYqEaoR4abP2/tEiJIFAzrnW1atVnlWoPULzSu1jDsfbahBpkb82sd5ROEj2hLULg8VVEiiph7jRL5Gd+H4SwyW4dyJwOwK3xAg5cHMZ7vB8nLsR0Hez6XYE//ZC/FvLcG8tnyH97d/KgdvPwXDE0h0da8m9GcG58RzpazmkruaQuLqI+JUFZG8vIDs8D2c4gnMngjdsSeeZ+7n5HNycPze6FcG/E8G9Z1u9N/cWXts8rwCRC0ZQlMD1fNBz6BkeO34+7u0cnGHRQtzh5+zz3iiAaysMOPxrEdwruTg38snciTA9vB7YbXaHxZ0GN2N3AJgpMiGtELzMG9lMoc7Qd/Z7e0w4saHv9X7Wef+nvXMNte266vg65+xzzj7Pe+7Nuff03pt3G1objG1ME7W9IpVQCTVSqpZUKkZMiSik2pp+CKgfzAcpVBFbNdoiFB/UQCBSIaCxRaGlEUKj1UaTe85+v9+P9V4/+c+51r773ua2+ZDmZF+yYbDW2WudNedce/zHGHOOMcfQDenqlT21pSNMVpP5e7P/eQ2Pi61BjLpo2fyFenN6twKI1yAYfgGv+wCj8gFBykAGJIe7cLhHUlw2zDVj9pTpzffFVcssxQ3C4gaRoXUiAUf/V7IMaRhMIBOVVgxDhUeb+IfbeEebuEdr+KVlw0xBeQ4gApWYV9pLx1ekZSjM0woUrqDisgG/gB6lNGN68/xlOBRAduHidRYcR1skxRU79jmA6DuBIyht4JUFEgsYLp6Bl09YgBw6hC/t4R+ewa+t06v+KPCNWVraLD3t/PE15NVjedSCA0RBbG0SJfkUQLIZn9cmGn+JsP8rDCoHeLWUoSVZ5wEiUBxtw+EpC5qjbZLChmEgaZ2M6SSZDSikPTIm1bmeV1mGSg7Ka1DKQ3ELSlvEpTxROWe0lJ5zuXRfsYx7tGo1wByDZ22a9tUHUUm0bLSeQJqRwBsW9ggLu/Z6phHNuASOdduOjuqrKAW3GY/uS7/XM/W8oLx+OUAO94y3XCBOjhyiQwmNMwT1c4waHzTlyuYBceX5sXD1a9jowgNEkaAJw0srI9IickxNvkwyfJhh5Rx+PZXyxSU4OgFHJ43Ej8s5OHwLXLzJHgsnoZg31y4zu8pLICqtXKJiDor6O2OyHJRWobgOZdEqSWWZuOIQV+fMOPWhoBCNbShsEpdXCGvWTAqrVx6XiCpLhFUdVwirK5cdo3KeqLBvSIBUW7N+q39qR6Q2pS10vWr7Y8ZeEqgtaOLSGmF5E7+Sx6+uEVbWDMiTw004WjX3xUUHX1TZJ27eRdB+xGQ3VMK7q9FryKvH8qhrACADU/4sVLmRTIOoypVqjkw/ybBy/rsBUjhBXF5CTBEfnic+vIm4cNL8PQNCYQcO9y1wdCxsg5hODJWZJjqXmXSZqSTQrBmQJBULkkQAEXOKStI8q1DYgKNdKJwkKl5HXLiOqHiSuHiCqLRDLO90OY8YWeBQfzXXENiiWnrUuUBS2jb3azy2jysWFCnzZyDIwGPGXs4ZcGZjEcAMQKpr+FVpPgFkjeRIgmDJ9F3j8CoOXv0MtD8Cg88DxauCQ6BZ9M+CA0TqYmQAIoetqvVaM0t5tP4VvN9nVLmZoGElp2XOXSjuoh87ruaIi6dJCgck5S0SmUoymaQBCqfg6ACOzkJhH4oblrkFDgMQSdVV4osOiZaOj7R8rIntBklph6SyTVzbJK6vkEiDZSSwGI0iIOVtO4XTcHTatqN2i3tQ3IGS2pQ2kgknU+6SFphpA2mfWo64sk5S3ki119y92f9Iu5W27DjLW8QpeM1zypqPbRFVdghqqwS1HFFV4F414zL3qJ26w7Tu4LbOQ+dRGP9zmsl/0WFw9f4vPEC0dq4sF0EYm4TEBiDBANxvgv8ZhtVb8RsOSU2MLem9C5LQ6Q8e1WQGrVqGETBKYsydOQYVYFKmFrPpXPMMAefoPMHhJkEhT1DcJSjvE1UPiOpvIWyeIWrv4TVX8VoOQdtS3HKImw5JMwWNeZ6kdEoFmWnSMK90FHA1n0iPpRxx3TECQMxrxqL5lsYqyjSXQCYAHp0nKZwnKe4bQOkdWNBdDSA5Qk329ZyWQ9R0GDUdJu3boPtn4B7aBGxX56+Fv7LgALGhCcqDpGTP0iIGICai97/A/3OGldssQIzUFhMKAFsWIA2HSFRPtUl5awYgM58QIPR/lzGbJvGnSApniUoHTKs7TGu7TGvXM629E7f2Ptz6B/DqP4vfuJ9h6f0My+9lXHkX49oPMandilu/Aa9+lrB+ykr8kky3V0tWc2VzHb+2wbSxjltfNeOUttSYYgkFM/fSsvSOAXRydKMBiLRmXNkgrl7SNJdpkPqy1SBVh1CTeD2n5eA3Hfoth3H73baisetjizIuPA6uOoCFB4h2nYmU7yJK4nQeotpuSjL2JOPq7YQtB2RmVVahvA2VzRlAdM1I9NoqSXUTypv2PgFDjNFO/7eYJzw8QVg4RVw5CXrZgg4AAAtoSURBVO1NktEq3bbDoJ/HG78L/Acg/AxET0P0gu1DUAXvJZh+E8ZfgcET0H2cpPVJks4DJK0fgc710DpH3NwnrO/h17bwqmu41WW8umx+B6/x3TSt5Um6dzOu30W3dJpBZQWv48DQgZEDfS3drpKUdqFyGqpnoLZPXN4lKGziyjko8NeXobpDUjtB1FgjauZIGjk7fr23ngNdh2HNodtycMf3wvg/bKzVnMviqly2wBcWHCByKcndZGM7zTZNpThX2Luv3K1fYVK7IwWImECTZwEkTyIp23TMtVAmT32JpLYC1ZRpxDiipv7OQ/mAuHQ9fuU00/oWk7ZDv+vQ7p+mN76bIHwQ+CPgqxBXLfMoM2ZGZiuEijMewvQ5m5zZ+xuYPgbex8F9CNwHYfIxkvFHiEcfJhrdz7DxfobNCwxb72XY+rHLqf4zMHiUuPfbBIMP4vVvZ9o9ybTtGLNO4Pc1udcSdG0DGinVNohKm4SFHMjUayxBfZu4sUvcXCdurUBTpPELaBlAlui0tnHHD8BEmeJTjb3AAPh+XV98gJgANoFEOkSe3DSpq+KzkmcZ1d5NIKmqH7y6DtUtqK0bE0TAkLQUGUbINI00R00T9h2oHED5RpLqzfiNc0xaJ+h3lul2HarNk/T6v8Zw9DmI/xH4lg2YNHn6bbIVw0RZaQOFsmq/ihIuKwI5Ua5ZBVkeXkEXbTkwlQTzXgTvf9KjzkWqN/J/4L4I4+fBfxaSv4LwEdz+Bbr1M3SqDoOqw7TmmFU8CQOjDVtL0MpBbQ2qcyCob84BRPdYs8ocew5xZ4lBbYtu++1Mp58CL417e1ODfD+MHd/1zDeo1V3l3FNu1kRhpAohVV3j+BvG/Ai7S9CUFM1DbRsa6ySadIrqW0SSrGIgMYWOAkhVK0L7UD5PVLkBt3aGcWufQfeAweAWBoMfptv7RfzpU/iT75AombRSYipUNTH1XAh8peyPbck1PyJRhgl5+udJ4eAaQLZMHdnVOFVDMqtyuq65laK/rzjOIgdUU8/klnoK3MeYdO6jU7uJTjXHpOnga5wyFWe0DNIS+jsDQiNP3Ngmaq0TtZcu3avrPYewk6PfOE2v89P43mchTOv4Lf5K7vdk4IXWIBlAxDeKH1X5MlOlT3HRRv0/z6R+D1FvGdrr1sSo70Azbxggbmsyu2fImFIZQBrSNntQvc7Y5V4jz6jt0Ovs0u/dyWT0q0STJ0imXzf17xJhMiviog2NgUK8lSAwxo083GiKH3oEQWTvU6oi5eFWOq8AVB5bKf7NMbT15ZXuX3Xm9RyVZ1ZMplFEc0cpJBNBYMKYhKgaRP8O7ufwex9l1Lwdt3OCsOsQSYvOAJKedx3Ivm+tkbS2iNt54s6y/V7XDUBWCDp5hs23Mug+SBT+A9A3+zNM+9+TxRb74jUEEC33XgmQbzNt/ARxfwU6eWhuQmPbAqTjkHQEkG2o71oTbAYQzTs2jV0uDeO39ph0rmc6uIA3fgS8v4egOB93Zwq8yKpTni5PlY5izYtk9qkYp6JOxzY1v8K2Vc5R6wjTtE55Visv3RCUKhS7IGeUi4LzsgC9YC7LoA0X97VXQgGwZhVPCRFeAO8JGD5E3LsLBmegl7NMn40xM7kEGoGklSNpbbwyQDrrBJ0NRq07mPQfI4m+ZnZwqp/XuALJsrsvJsqNBpltotEOs4HdU6BfTltEoheZNt9HPJREzENrywKktW5XZnoOSVvmRg5km18pYVs5osZJovZ7oKeUNV8A7zkIW2memjTwVB2RRSdGjxLCWKtq6o9KgGXFgurEtEzcWKLiLaaSZEzsTYi9MYk/MWSrsGpvvW/J7ItIH64G5klGZTRmjGtSH6kP1nyTSvoOeE9C52EY/DgMdu2YBRBjQq5BffWSpmivQnuDpLNB0l0xq1ZaudJ7STqb+J0dxq27mfY/C8l/ktAzuzmUSf1a/iy+BplJ3WzDjSoHmdJUEB/Rb96HPxRzbEB7DVoyr1ZIeg7RwCHpWk0ibWKo6xB3HWNSuK0bGFcvEDQ/AcOnIaxdmj+oqJFKHqkSkeFKa+jZdTUFUSqZRB8vruFTMRSmZecS7Z8XghPXFoFRIRjZW7Fr615okm9SqgagIhmKo8lIdpUSganZOGIYdEyZumy/t9EiMi81B4vK0Pw76PwOcfcCQec0QTN1LMr7LoAIBKIUIFq+pnNJgFjBsYHbyzPs3sN08NeQaGGhk9by07iv3c9CA0RAMOE+EmLSGjqa8lppYQma+NGniZO7jEc7C/kwXmzXIZSvYOLA2B4j18FzHSbuHsPBXXRbvwxodep/LcOKKWecqMYyu0bnGYlZdG73O1hO1n0Z6SGXwDQ7NzPwbD/qFUcB/pXIlAHQ8oTmX5bsttW0CfVV5L4Mk7/EG/wS7vBtuON1omkOvHW8wSphb90KjqY07J4xtYKuJubWl8JwnUnPYTB8D/CMWYyIgxrE3VQ4vAmQN+wbmAEkE+RGoqcOQwZMkj8k4l6C3qaVlNmk1F3DFUCCJZg6uAP5Ndbo925hNLmPIPhd4lCZxwvWrMmePzO89YVE9fFJUJk3mulYcNgkBjo3XbQYtQARSLxvEbt/QeB+FHf6TsajLYajJcbjNYLhBnRWQaZnW9p225il8UDaJQ+DTaaDJUbjnySOv27BqoDQWJrw+Mb/ejDlYmuQbJIoZsgY+DKAjHD5Y+B+kuGe9TDL6dVeIh6fYNhdI/KW8CcO49Eqo94dTFufhsHTELycMoCYQMtOWWNiCGkDgUNHXTiejwWIACESWNJIm6yrpm/qZ5r1JX4egi8RTn7DLFO3uuuM/FWCqQMCgzzm/TUQYCZ5GG9D6xR0zuIODphMfoFQBUfN+5apl6YsOZ7hvy6tLjRAZHWIKYz5MbNabLoY41tgyjT+PAk/TzLcTwGi5cwNgv4NdJtnGfZPMRnfSOz/FISPgvcv4CoaODVPDP8LfQKD3OFyjeuo744PHJY7sjiCOXCkczKZWppIJ5RT56Wck/JdfBvcv8V1H2Ls3s3QO4fnrs8BxCEZrRBNNmC8B63T0LqFYHAn/vS3CIKL1m+TCST9CNfw55oAiPmNMvEp71qWewmPSfBF4vhjBL2zYPwh20T1A/zunXRq9zDs30sw+TgEXwSesxNQ1S9XbXIpjxkOBAyFr6RfqtHjxkfaB5mZOrUaRWDRUvDYACSmSmKqwk7AV7HxLgT/DclTwOOMph/And5KMsgZEzToOfgDB3+cIx5uQ/skNG8j6X+I2PtT/KBpMi3JspL/xzT8JkDemG9ATJFVHJrZF2ZZNPVOyzoPv0wS/Tpe+3oSrWI194kb74DRzzHuChh/AsE/QfCSRYRQEcuZEZo0v+a5BggyVQQO1dhOwaHvdX5cH7UtzWnUqLqiRGpKhdMzWUOSNHtIGA8J/AmhOzXLyoQaR9XsJ/emjxO5H4bRLdBfwu87TAcOk5FDMNIqV564+TYYPAzBk3hB2zgtzWt+EyDH9cu/unYlMTVFNdktLFpsPIYWgQzTyAR5BpJH8btvNev5tG+G3gWIPwHTL0DUtNaTvG3hBCLtb1fohkgJ6dKJuJ4vEijm6dV19Qdzl/ohJpWWM1pEDskyCSUSswwbGLdJpKGFIUE0IIx7JhvlDPj+C2ZbANMPwehGgvEy3sRhqlU+1yHpO8Stm2D0KYifwY2qxtgMtYqctvuDGdwb46kLbmLNAyQtMGEcaRlA9JL/Dfg9wt7boasVmnfA+D7gD4BnLf9rSiFGk3POVDxSphTFVknKyqxKJ+kCXSaxj1t7aGjqg6ZGM0kuR+nLJLyUAiSe1d2QqIgZEtEhSnzj+Y+MM1UhKl8F/zfBvYPE3cL3HfzQIQwdkoEins/B+DHga0ySkpmeyRUj18yxatDXAUP/Dzulf7mPSc8LAAAAAElFTkSuQmCC +``` + +### `annotation` + +A Kubernetes annotation for the NavLink custom resource. + +### `label` + +A Kubernetes label for the NavLink custom resource. + +### `sideLabel` + +Label that appears in the left navigation bar + +### `target` + +Sets the target property of the link's anchor tag (``), which (depending on browsers) determines if it opens in a new window or in an existing tab. + +The default value is `_self`, which opens the link on the current tab. To open the link in a new window or tab, set the target to `_blank`. + +For more information about the target property, see [this page.](https://www.w3schools.com/tags/att_a_target.asp) + +### `toService` + +Has five fields that are constructed to create a URL like the following: `https:///k8s/clusters//k8s/namespace//service/::/proxy/` + +For example, a link to a monitoring service can be set up as follows: + +- name: `rancher-monitoring-grafana` +- namespace: `cattle-monitoring-system` +- path: `proxy/?orgId=1` +- port: `"80"` +- scheme: `http` + +It is required to provide either the `toService` directive or the `toURL` directive. + +### `toUrl` + +Can be any link, even to links outside of the cluster. + +It is required to provide either the `toService` directive or the `toURL` directive. + +## Link Examples + +### Example of Link with `toUrl` + +This example NavLink YAML shows an example of configuring a NavLink to a Grafana dashboard: + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + name: grafana +spec: + group: "Monitoring Dashboards" + toURL: https:///api/v1/namespaces/cattle-monitoring-system/services/http:rancher-monitoring-grafana:80/proxy/?orgId=1 +``` + +Adding the above YAML results in a link to Grafana being created, as shown in the following screenshot: + +![Screenshot of Grafana Link](/img/example-grafana-link.png) + +### Example of Link with `toService` + +This example YAML shows an example of `toService` used for the link target: + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + annotations: + key: annotation + labels: + key: label + name: navlinkname +spec: + description: This is a description field # Optional. + group: "group1" # Optional. If not provided, the links appear standalone. + iconSrc: data:image/jpeg;base64,[icon source string is clipped for brevity] + label: This is a label # Optional. + sideLabel: A side label. # Optional. + target: _blank #Optional. _blank opens the link in a new tab or window. + toService: # toService or #toUrl needs to be provided. + name: rancher-monitoring-grafana + namespace: cattle-monitoring-system + path: proxy/?orgId=1 + port: "80" + scheme: http +``` + +Adding the `toService` parameters above results in a link to Grafana being created, as shown in the following screenshot: + +![Screenshot of Grafana Link](/img/example-service-link.png) + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md new file mode 100644 index 00000000000..eec64c568f2 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md @@ -0,0 +1,61 @@ +--- +title: Configuring a Global Default Private Registry +--- + + + + + +:::note +This page describes how to configure a global default private registry from the Rancher UI, after Rancher is already installed. + +For instructions on how to set up a private registry during Rancher installation, refer to the [air-gapped installation guide](../../../pages-for-subheaders/air-gapped-helm-cli-install.md). + +::: + +A private registry is a private, consistent, and centralized source of truth for the container images in your clusters. You can use a private container image registry to share custom base images within your organization. + +There are two main ways to set up a private registry in Rancher: + +* Set up a global default registry through the **Settings** tab in the global view. +* Set up a private registry in the advanced options under cluster-level settings. + +The global default registry is intended to be used in air-gapped setups, for registries that don't require credentials. The cluster-level private registry is intended to be used in setups where the private registry requires credentials. + +## Set a Private Registry with No Credentials as the Default Registry + +1. Log into Rancher and configure the default administrator password. +1. Select **☰ > Global Settings**. +1. Go to `system-default-registry` and choose **⋮ > Edit Setting**. +1. Enter your registry's hostname and port (e.g. `registry.yourdomain.com:port`). Do not prefix the text with `http://` or `https://`. + +**Result:** Rancher pulls system images from your private registry. + +### Namespaced Private Registry with RKE2 Downstream Clusters + +Most private registries should work, by default, with RKE2 downstream clusters. + +However, you'll need to do some additional steps if you're trying to set a namespaced private registry whose URL is formated like this: `website/subdomain:portnumber`. + +1. Select **☰ > Cluster Management**. +1. Find the RKE2 cluster in the list and click **⋮ >Edit Config**. +1. From the **Cluster config** menu, select **Registries**. +1. In the **Registries** pane, select the **Configure advanced containerd mirroring and registry authentication options** option. +1. In the text fields under **Mirrors**, enter the **Registry Hostname** and **Mirror Endpoints**. +1. Click **Save**. +1. Repeat as necessary for each downstream RKE2 cluster. + +## Configure a Private Registry with Credentials when Creating a Cluster + +There is no global way to set up a private registry with authorization for every Rancher-provisioned cluster. Therefore, if you want a Rancher-provisioned cluster to pull images from a private registry that requires credentials, you'll have to pass the registry credentials through the advanced cluster options every time you create a new cluster. + +Since the private registry cannot be configured after the cluster is created, you'll need to perform these steps during initial cluster setup. + +1. Select **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Choose a cluster type. +1. In the **Cluster Configuration** go to the **Registries** tab and select **Pull images for Rancher from a private registry**. +1. Enter the registry hostname and credentials. +1. Click **Create**. + +**Result:** The new cluster pulls images from the private registry. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md new file mode 100644 index 00000000000..74a2f048391 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md @@ -0,0 +1,240 @@ +--- +title: Cluster and Project Roles +--- + + + + + +Cluster and project roles define user authorization inside a cluster or project. + +To manage these roles, + +1. Click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles** and go to the **Cluster** or **Project/Namespaces** tab. + +### Membership and Role Assignment + +The projects and clusters accessible to non-administrative users is determined by _membership_. Membership is a list of users who have access to a specific cluster or project based on the roles they were assigned in that cluster or project. Each cluster and project includes a tab that a user with the appropriate permissions can use to manage membership. + +When you create a cluster or project, Rancher automatically assigns you as the `Owner` for it. Users assigned the `Owner` role can assign other users roles in the cluster or project. + +:::note + +Non-administrative users cannot access any existing projects/clusters by default. A user with appropriate permissions (typically the owner) must explicitly assign the project and cluster membership. + +::: + +### Cluster Roles + +_Cluster roles_ are roles that you can assign to users, granting them access to a cluster. There are two primary cluster roles: `Owner` and `Member`. + +- **Cluster Owner:** + + These users have full control over the cluster and all resources in it. + +- **Cluster Member:** + + These users can view most cluster level resources and create new projects. + +#### Custom Cluster Roles + +Rancher lets you assign _custom cluster roles_ to a standard user instead of the typical `Owner` or `Member` roles. These roles can be either a built-in custom cluster role or one defined by a Rancher administrator. They are convenient for defining narrow or specialized access for a standard user within a cluster. See the table below for a list of built-in custom cluster roles. + +#### Cluster Role Reference + +The following table lists each built-in custom cluster role available and whether that level of access is included in the default cluster-level permissions, `Cluster Owner` and `Cluster Member`. + +| Built-in Cluster Role | Owner | Member | +| ---------------------------------- | ------------- | --------------------------------- | +| Create Projects | ✓ | ✓ | +| Manage Cluster Backups             | ✓ | | +| Manage Cluster Catalogs | ✓ | | +| Manage Cluster Members | ✓ | | +| Manage Nodes [(see table below)](#Manage-Nodes-Permissions)| ✓ | | +| Manage Storage | ✓ | | +| View All Projects | ✓ | | +| View Cluster Catalogs | ✓ | ✓ | +| View Cluster Members | ✓ | ✓ | +| View Nodes | ✓ | ✓ | + +#### Manage Nodes Permissions + +The following table lists the permissions available for the `Manage Nodes` role in RKE and RKE2. + +| Manage Nodes Permissions | RKE | RKE2 | +|-----------------------------|-------- |--------- | +| SSH Access | ✓ | ✓ | +| Delete Nodes | ✓ | ✓ | +| Scale Clusters Up and Down | ✓ | * | +***In RKE2, you must have permission to edit a cluster to be able to scale clusters up and down.** +
+ +For details on how each cluster role can access Kubernetes resources, you can look them up in the Rancher UI: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Click the **Cluster** tab. +1. Click the name of an individual role. The table shows all of the operations and resources that are permitted by the role. + +:::note + +When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. + +::: + +### Giving a Custom Cluster Role to a Cluster Member + +After an administrator [sets up a custom cluster role,](custom-roles.md) cluster owners and admins can then assign those roles to cluster members. + +To assign a custom role to a new cluster member, you can use the Rancher UI. To modify the permissions of an existing member, you will need to use the Rancher API view. + +To assign the role to a new cluster member, + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to assign a role to a member and click **Explore**. +1. Click **RBAC > Cluster Members**. +1. Click **Add**. +1. In the **Cluster Permissions** section, choose the custom cluster role that should be assigned to the member. +1. Click **Create**. + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to assign a role to a member and click **Explore**. +1. Click **Cluster > Cluster Members**. +1. Click **Add**. +1. In the **Cluster Permissions** section, choose the custom cluster role that should be assigned to the member. +1. Click **Create**. + + + + +**Result:** The member has the assigned role. + +To assign any custom role to an existing cluster member, + +1. Click **☰ > Users & Authentication**. +1. Go to the member you want to give the role to. Click the **⋮ > Edit Config**. +1. If you have added custom roles, they will show in the **Custom** section. Choose the role you want to assign to the member. +1. Click **Save**. + +**Result:** The member has the assigned role. + +### Project Roles + +_Project roles_ are roles that can be used to grant users access to a project. There are three primary project roles: `Owner`, `Member`, and `Read Only`. + +- **Project Owner:** + + These users have full control over the project and all resources in it. + +- **Project Member:** + + These users can manage project-scoped resources like namespaces and workloads, but cannot manage other project members. + +:::note + +By default, the Rancher role of `project-member` inherits from the `Kubernetes-edit` role, and the `project-owner` role inherits from the `Kubernetes-admin` role. As such, both `project-member` and `project-owner` roles will allow for namespace management, including the ability to create and delete namespaces. + +::: + +- **Read Only:** + + These users can view everything in the project but cannot create, update, or delete anything. + +:::danger + +Users assigned the `Owner` or `Member` role for a project automatically inherit the `namespace creation` role. However, this role is a [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole), meaning its scope extends to all projects in the cluster. Therefore, users explicitly assigned the `owner` or `member` role for a project can create namespaces in other projects they're assigned to, even with only the `Read Only` role assigned. + +::: + +#### Custom Project Roles + +Rancher lets you assign _custom project roles_ to a standard user instead of the typical `Owner`, `Member`, or `Read Only` roles. These roles can be either a built-in custom project role or one defined by a Rancher administrator. They are convenient for defining narrow or specialized access for a standard user within a project. See the table below for a list of built-in custom project roles. + +#### Project Role Reference + +The following table lists each built-in custom project role available in Rancher and whether it is also granted by the `Owner`, `Member`, or `Read Only` role. + +| Built-in Project Role | Owner | Member | Read Only | +| ---------------------------------- | ------------- | ----------------------------- | ------------- | +| Manage Project Members | ✓ | | | +| Create Namespaces | ✓ | ✓ | | +| Manage Config Maps | ✓ | ✓ | | +| Manage Ingress | ✓ | ✓ | | +| Manage Project Catalogs | ✓ | | | +| Manage Secrets | ✓ | ✓ | | +| Manage Service Accounts | ✓ | ✓ | | +| Manage Services | ✓ | ✓ | | +| Manage Volumes | ✓ | ✓ | | +| Manage Workloads | ✓ | ✓ | | +| View Secrets | ✓ | ✓ | | +| View Config Maps | ✓ | ✓ | ✓ | +| View Ingress | ✓ | ✓ | ✓ | +| View Project Members | ✓ | ✓ | ✓ | +| View Project Catalogs | ✓ | ✓ | ✓ | +| View Service Accounts | ✓ | ✓ | ✓ | +| View Services | ✓ | ✓ | ✓ | +| View Volumes | ✓ | ✓ | ✓ | +| View Workloads | ✓ | ✓ | ✓ | + +:::note Notes: + +- Each project role listed above, including `Owner`, `Member`, and `Read Only`, is comprised of multiple rules granting access to various resources. You can view the roles and their rules on the Global > Security > Roles page. +- When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. +- The `Manage Project Members` role allows the project owner to manage any members of the project **and** grant them any project scoped role regardless of their access to the project resources. Be cautious when assigning this role out individually. + +::: + +### Defining Custom Roles +As previously mentioned, custom roles can be defined for use at the cluster or project level. The context field defines whether the role will appear on the cluster member page, project member page, or both. + +When defining a custom role, you can grant access to specific resources or specify roles from which the custom role should inherit. A custom role can be made up of a combination of specific grants and inherited roles. All grants are additive. This means that defining a narrower grant for a specific resource **will not** override a broader grant defined in a role that the custom role is inheriting from. + +### Default Cluster and Project Roles + +By default, when a standard user creates a new cluster or project, they are automatically assigned an ownership role: either [cluster owner](#cluster-roles) or [project owner](#project-roles). However, in some organizations, these roles may overextend administrative access. In this use case, you can change the default role to something more restrictive, such as a set of individual roles or a custom role. + +There are two methods for changing default cluster/project roles: + +- **Assign Custom Roles**: Create a [custom role](custom-roles.md) for either your [cluster](#custom-cluster-roles) or [project](#custom-project-roles), and then set the custom role as default. + +- **Assign Individual Roles**: Configure multiple [cluster](#cluster-role-reference)/[project](#project-role-reference) roles as default for assignment to the creating user. + + For example, instead of assigning a role that inherits other roles (such as `cluster owner`), you can choose a mix of individual roles (such as `manage nodes` and `manage storage`). + +:::note + +- Although you can [lock](locked-roles.md) a default role, the system still assigns the role to users who create a cluster/project. +- Only users that create clusters/projects inherit their roles. Users added to the cluster/project membership afterward must be explicitly assigned their roles. + +::: + +### Configuring Default Roles for Cluster and Project Creators + +You can change the cluster or project role(s) that are automatically assigned to the creating user. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Click the **Cluster** or **Project/Namespaces** tab. +1. Find the custom or individual role that you want to use as default. Then edit the role by selecting **⋮ > Edit Config**. +1. In the **Cluster Creator Default** or **Project Creator Default** section, enable the role as the default. +1. Click **Save**. + +**Result:** The default roles are configured based on your changes. Roles assigned to cluster/project creators display a check in the **Cluster/Project Creator Default** column. + +If you want to remove a default role, edit the permission and select **No** from the default roles option. + +### Cluster Membership Revocation Behavior + +When you revoke the cluster membership for a standard user that's explicitly assigned membership to both the cluster _and_ a project within the cluster, that standard user [loses their cluster roles](#cluster-roles) but [retains their project roles](#project-roles). In other words, although you have revoked the user's permissions to access the cluster and its nodes, the standard user can still: + +- Access the projects they hold membership in. +- Exercise any [individual project roles](#project-role-reference) they are assigned. + +If you want to completely revoke a user's access within a cluster, revoke both their cluster and project memberships. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md new file mode 100644 index 00000000000..de61e0f4fb7 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md @@ -0,0 +1,124 @@ +--- +title: Custom Roles +--- + + + + + +Within Rancher, _roles_ determine what actions a user can make within a cluster or project. + +Note that _roles_ are different from _permissions_, which determine what clusters and projects you can access. + +:::danger + +It is possible for a custom role to enable privilege escalation. For details, see [this section.](#privilege-escalation) + +::: + + +## Prerequisites + +To complete the tasks on this page, one of the following permissions are required: + + - [Administrator Global Permissions](global-permissions.md). + - [Custom Global Permissions](global-permissions.md#custom-global-permissions) with the [Manage Roles](global-permissions.md) role assigned. + +## Creating A Custom Role + +While Rancher comes out-of-the-box with a set of default user roles, you can also create default custom roles to provide users with very specific permissions within Rancher. + +The steps to add custom roles differ depending on the version of Rancher. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Select a tab to determine the scope of the role you're adding. The tabs are: + + - **Global:** The role is valid for allowing members to manage global scoped resources. + - **Cluster:** The role is valid for assignment when adding/managing members to clusters. + - **Project/Namespaces:** The role is valid for assignment when adding/managing members to projects or namespaces. + +1. Click **Create Global Role,** **Create Cluster Role** or **Create Project/Namespaces Role,** depending on the scope. +1. Enter a **Name** for the role. +1. Optional: Choose the **Cluster/Project Creator Default** option to assign this role to a user when they create a new cluster or project. Using this feature, you can expand or restrict the default roles for cluster/project creators. + + > Out of the box, the Cluster Creator Default and the Project Creator Default roles are `Cluster Owner` and `Project Owner` respectively. + +1. Use the **Grant Resources** options to assign individual [Kubernetes API endpoints](https://kubernetes.io/docs/reference/) to the role. + + > When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. + + > The Resource text field provides a method to search for pre-defined Kubernetes API resources, or enter a custom resource name for the grant. The pre-defined or `(Custom)` resource must be selected from the dropdown, after entering a resource name into this field. + + You can also choose the individual cURL methods (`Create`, `Delete`, `Get`, etc.) available for use with each endpoint you assign. + +1. Use the **Inherit from** options to assign individual Rancher roles to your custom roles. Note: When a custom role inherits from a parent role, the parent role cannot be deleted until the child role is deleted. + +1. Click **Create**. + +## Creating a Custom Role that Inherits from Another Role + +If you have a group of individuals that need the same level of access in Rancher, it can save time to create a custom role in which all of the rules from another role, such as the administrator role, are copied into a new role. This allows you to only configure the variations between the existing role and the new role. + +The custom role can then be assigned to a user or group so that the role takes effect the first time the user or users sign into Rancher. + +To create a custom role based on an existing role, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Click the **Cluster** or **Project/Namespaces** tab. Click **Create Cluster Role** or **Create Project/Namespaces Role** depending on the scope. Note: Only cluster roles and project/namespace roles can inherit from another role. +1. Enter a name for the role. +1. In the **Inherit From** tab, select the role(s) that the custom role will inherit permissions from. +1. In the **Grant Resources** tab, select the Kubernetes resource operations that will be enabled for users with the custom role. + + > The Resource text field provides a method to search for pre-defined Kubernetes API resources, or enter a custom resource name for the grant. The pre-defined or `(Custom)` resource must be selected from the dropdown, after entering a resource name into this field. +1. Optional: Assign the role as default. +1. Click **Create**. + +## Deleting a Custom Role + +When deleting a custom role, all global role bindings with this custom role are deleted. + +If a user is only assigned one custom role, and the role is deleted, the user would lose access to Rancher. For the user to regain access, an administrator would need to edit the user and apply new global permissions. + +Custom roles can be deleted, but built-in roles cannot be deleted. + +To delete a custom role, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +2. Go to the custom global role that should be deleted and click **⋮ (…) > Delete**. +3. Click **Delete**. + +## Assigning a Custom Role to a Group + +If you have a group of individuals that need the same level of access in Rancher, it can save time to create a custom role. When the role is assigned to a group, the users in the group have the appropriate level of access the first time they sign into Rancher. + +When a user in the group logs in, they get the built-in Standard User global role by default. They will also get the permissions assigned to their groups. + +If a user is removed from the external authentication provider group, they would lose their permissions from the custom role that was assigned to the group. They would continue to have their individual Standard User role. + +:::note Prerequisites: + +You can only assign a global role to a group if: + +* You have set up an [external authentication provider](../../../../pages-for-subheaders/authentication-config.md#external-vs-local-authentication) +* The external authentication provider supports [user groups](../../authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md) +* You have already set up at least one user group with the authentication provider + +::: + +To assign a custom role to a group, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Groups**. +1. Go to the existing group that will be assigned the custom role and click **⋮ > Edit Config**. +1. If you have created roles, they will show in the **Custom** section. Choose any custom role that will be assigned to the group. +1. Optional: In the **Global Permissions** or **Built-in** sections, select any additional permissions that the group should have. +1. Click **Save.**. + +**Result:** The custom role will take effect when the users in the group log into Rancher. + +## Privilege Escalation + +The `Configure Catalogs` custom permission is powerful and should be used with caution. When an admin assigns the `Configure Catalogs` permission to a standard user, it could result in privilege escalation in which the user could give themselves admin access to Rancher provisioned clusters. Anyone with this permission should be considered equivalent to an admin. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md new file mode 100644 index 00000000000..64411298a62 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md @@ -0,0 +1,251 @@ +--- +title: Global Permissions +--- + + + + + +_Permissions_ are individual access rights that you can assign when selecting a custom permission for a user. + +Global Permissions define user authorization outside the scope of any particular cluster. Out-of-the-box, there are four default global permissions: `Administrator`, `Restricted Admin`,`Standard User` and `User-base`. + +- **Administrator:** These users have full control over the entire Rancher system and all clusters within it. + +- **Restricted Admin:** These users have full control over downstream clusters, but cannot alter the local Kubernetes cluster. + +- **Standard User:** These users can create new clusters and use them. Standard users can also assign other users permissions to their clusters. + +- **User-Base:** User-Base users have login-access only. + +You cannot update or delete the built-in Global Permissions. + +## Restricted Admin + +A new `restricted-admin` role was created in Rancher v2.5 in order to prevent privilege escalation from the local Rancher server Kubernetes cluster. This role has full administrator access to all downstream clusters managed by Rancher, but it does not have permission to alter the local Kubernetes cluster. + +The `restricted-admin` can create other `restricted-admin` users with an equal level of access. + +A new setting was added to Rancher to set the initial bootstrapped administrator to have the `restricted-admin` role. This applies to the first user created when the Rancher server is started for the first time. If the environment variable is set, then no global administrator would be created, and it would be impossible to create the global administrator through Rancher. + +To bootstrap Rancher with the `restricted-admin` as the initial user, the Rancher server should be started with the following environment variable: + +``` +CATTLE_RESTRICTED_DEFAULT_ADMIN=true +``` +### List of `restricted-admin` Permissions + +The following table lists the permissions and actions that a `restricted-admin` should have in comparison with the `Administrator` and `Standard User` roles: + +| Category | Action | Global Admin | Standard User | Restricted Admin | Notes for Restricted Admin role | +| -------- | ------ | ------------ | ------------- | ---------------- | ------------------------------- | +| Local Cluster functions | Manage Local Cluster (List, Edit, Import Host) | Yes | No | No | | +| | Create Projects/namespaces | Yes | No | No | | +| | Add cluster/project members | Yes | No | No | | +| | Global DNS | Yes | No | No | | +| | Access to management cluster for CRDs and CRs | Yes | No | Yes | | +| | Save as RKE Template | Yes | No | No | | +| Security | | | | | | +| Enable auth | Configure Authentication | Yes | No | Yes | | +| Roles | Create/Assign GlobalRoles | Yes | No (Can list) | Yes | Auth webhook allows creating globalrole for perms already present | +| | Create/Assign ClusterRoles | Yes | No (Can list) | Yes | Not in local cluster | +| | Create/Assign ProjectRoles | Yes | No (Can list) | Yes | Not in local cluster | +| Users | Add User/Edit/Delete/Deactivate User | Yes | No | Yes | | +| Groups | Assign Global role to groups | Yes | No | Yes | As allowed by the webhook | +| | Refresh Groups | Yes | No | Yes | | +| PSP's | Manage PSP templates | Yes | No (Can list) | Yes | Same privileges as Global Admin for PSPs | +| Tools | | | | | | +| | Manage RKE Templates | Yes | No | Yes | | +| | Manage Global Catalogs | Yes | No | Yes | Cannot edit/delete built-in system catalog. Can manage Helm library | +| | Cluster Drivers | Yes | No | Yes | | +| | Node Drivers | Yes | No | Yes | | +| | GlobalDNS Providers | Yes | Yes (Self) | Yes | | +| | GlobalDNS Entries | Yes | Yes (Self) | Yes | | +| Settings | | | | | | +| | Manage Settings | Yes | No (Can list) | No (Can list) | | +| User | | | | | | +| | Manage API Keys | Yes (Manage all) | Yes (Manage self) | Yes (Manage self) | | +| | Manage Node Templates | Yes | Yes (Manage self) | Yes (Manage self) | Can only manage their own node templates and not those created by other users | +| | Manage Cloud Credentials | Yes | Yes (Manage self) | Yes (Manage self) | Can only manage their own cloud credentials and not those created by other users | +| Downstream Cluster | Create Cluster | Yes | Yes | Yes | | +| | Edit Cluster | Yes | Yes | Yes | | +| | Rotate Certificates | Yes | | Yes | | +| | Snapshot Now | Yes | | Yes | | +| | Restore Snapshot | Yes | | Yes | | +| | Save as RKE Template | Yes | No | Yes | | +| | Run CIS Scan | Yes | Yes | Yes | | +| | Add Members | Yes | Yes | Yes | | +| | Create Projects | Yes | Yes | Yes | | +| Feature Charts since v2.5 | | | | | | +| | Install Fleet | Yes | | Yes | Should not be able to run Fleet in local cluster | +| | Deploy EKS cluster | Yes | Yes | Yes | | +| | Deploy GKE cluster | Yes | Yes | Yes | | +| | Deploy AKS cluster | Yes | Yes | Yes | | + + +### Changing Global Administrators to Restricted Admins + +If Rancher already has a global administrator, they should change all global administrators over to the new `restricted-admin` role. + +This can be done through **Security > Users** and moving any Administrator role over to Restricted Administrator. + +Signed-in users can change themselves over to the `restricted-admin` if they wish, but they should only do that as the last step, otherwise they won't have the permissions to do so. + +## Global Permission Assignment + +Global permissions for local users are assigned differently than users who log in to Rancher using external authentication. + +### Global Permissions for New Local Users + +When you create a new local user, you assign them a global permission as you complete the **Add User** form. + +To see the default permissions for new users, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. The **Roles** page has tabs for roles grouped by scope. Each table lists the roles in that scope. In the **Global** tab, in the **New User Default** column, the permissions given to new users by default are indicated with a checkmark. + +You can [change the default global permissions to meet your needs.](#configuring-default-global-permissions) + +### Global Permissions for Users with External Authentication + +When a user logs into Rancher using an external authentication provider for the first time, they are automatically assigned the **New User Default** global permissions. By default, Rancher assigns the **Standard User** permission for new users. + +To see the default permissions for new users, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. The **Roles** page has tabs for roles grouped by scope. Each table lists the roles in that scope. In the **New User Default** column on each page, the permissions given to new users by default are indicated with a checkmark. + +You can [change the default permissions to meet your needs.](#configuring-default-global-permissions) + +Permissions can be [assigned](#configuring-global-permissions-for-individual-users) to an individual user. + +You can [assign a role to everyone in the group at the same time](#configuring-global-permissions-for-groups) if the external authentication provider supports groups. + +## Custom Global Permissions + +Using custom permissions is convenient for providing users with narrow or specialized access to Rancher. + +When a user from an [external authentication source](../../../../pages-for-subheaders/authentication-config.md) signs into Rancher for the first time, they're automatically assigned a set of global permissions (hereafter, permissions). By default, after a user logs in for the first time, they are created as a user and assigned the default `user` permission. The standard `user` permission allows users to login and create clusters. + +However, in some organizations, these permissions may extend too much access. Rather than assigning users the default global permissions of `Administrator` or `Standard User`, you can assign them a more restrictive set of custom global permissions. + +The default roles, Administrator and Standard User, each come with multiple global permissions built into them. The Administrator role includes all global permissions, while the default user role includes three global permissions: Create Clusters, Use Catalog Templates, and User Base, which is equivalent to the minimum permission to log in to Rancher. In other words, the custom global permissions are modularized so that if you want to change the default user role permissions, you can choose which subset of global permissions are included in the new default user role. + +Administrators can enforce custom global permissions in multiple ways: + +- [Changing the default permissions for new users](#configuring-default-global-permissions) +- [Configuring global permissions for individual users](#configuring-global-permissions-for-individual-users) +- [Configuring global permissions for groups](#configuring-global-permissions-for-groups) + +### Custom Global Permissions Reference + +The following table lists each custom global permission available and whether it is included in the default global permissions, `Administrator`, `Standard User` and `User-Base`. + +| Custom Global Permission | Administrator | Standard User | User-Base | +| ---------------------------------- | ------------- | ------------- |-----------| +| Create Clusters | ✓ | ✓ | | +| Create RKE Templates | ✓ | ✓ | | +| Manage Authentication | ✓ | | | +| Manage Catalogs | ✓ | | | +| Manage Cluster Drivers | ✓ | | | +| Manage Node Drivers | ✓ | | | +| Manage PodSecurityPolicy Templates | ✓ | | | +| Manage Roles | ✓ | | | +| Manage Settings | ✓ | | | +| Manage Users | ✓ | | | +| Use Catalog Templates | ✓ | ✓ | | +| User-Base (Basic log-in access) | ✓ | ✓ | | + +For details on which Kubernetes resources correspond to each global permission, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. If you click the name of an individual role, a table shows all of the operations and resources that are permitted by the role. + +:::note Notes: + +- Each permission listed above is comprised of multiple individual permissions not listed in the Rancher UI. For a full list of these permissions and the rules they are comprised of, access through the API at `/v3/globalRoles`. +- When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. + +::: + +### Configuring Default Global Permissions + +If you want to restrict the default permissions for new users, you can remove the `user` permission as default role and then assign multiple individual permissions as default instead. Conversely, you can also add administrative permissions on top of a set of other standard permissions. + +:::note + +Default roles are only assigned to users added from an external authentication provider. For local users, you must explicitly assign global permissions when adding a user to Rancher. You can customize these global permissions when adding the user. + +::: + +To change the default global permissions that are assigned to external users upon their first log in, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. On the **Roles** page, make sure the **Global** tab is selected. +1. Find the permissions set that you want to add or remove as a default. Then edit the permission by selecting **⋮ > Edit Config**. +1. If you want to add the permission as a default, Select **Yes: Default role for new users** and then click **Save**. If you want to remove a default permission, edit the permission and select **No**. + +**Result:** The default global permissions are configured based on your changes. Permissions assigned to new users display a check in the **New User Default** column. + +### Configuring Global Permissions for Individual Users + +To configure permission for a user, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Go to the user whose access level you want to change and click **⋮ > Edit Config**. +1. In the **Global Permissions** and **Built-in** sections, check the boxes for each permission you want the user to have. If you have created roles from the **Roles** page, they will appear in the **Custom** section and you can choose from them as well. +1. Click **Save**. + +**Result:** The user's global permissions have been updated. + +### Configuring Global Permissions for Groups + +If you have a group of individuals that need the same level of access in Rancher, it can save time to assign permissions to the entire group at once, so that the users in the group have the appropriate level of access the first time they sign into Rancher. + +After you assign a custom global role to a group, the custom global role will be assigned to a user in the group when they log in to Rancher. + +For existing users, the new permissions will take effect when the users log out of Rancher and back in again, or when an administrator [refreshes the group memberships.](#refreshing-group-memberships) + +For new users, the new permissions take effect when the users log in to Rancher for the first time. New users from this group will receive the permissions from the custom global role in addition to the **New User Default** global permissions. By default, the **New User Default** permissions are equivalent to the **Standard User** global role, but the default permissions can be [configured.](#configuring-default-global-permissions) + +If a user is removed from the external authentication provider group, they would lose their permissions from the custom global role that was assigned to the group. They would continue to have any remaining roles that were assigned to them, which would typically include the roles marked as **New User Default**. Rancher will remove the permissions that are associated with the group when the user logs out, or when an administrator [refreshes group memberships,](#refreshing-group-memberships) whichever comes first. + +:::note Prerequisites: + +You can only assign a global role to a group if: + +* You have set up an [external authentication provider](../../../../pages-for-subheaders/authentication-config.md#external-vs-local-authentication) +* The external authentication provider supports [user groups](../authentication-config/manage-users-and-groups.md) +* You have already set up at least one user group with the authentication provider + +::: + +To assign a custom global role to a group, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Groups**. +1. Go to the group you want to assign a custom global role to and click **⋮ > Edit Config**. +1. In the **Global Permissions,** **Custom,** and/or **Built-in** sections, select the permissions that the group should have. +1. Click **Create**. + +**Result:** The custom global role will take effect when the users in the group log into Rancher. + +### Refreshing Group Memberships + +When an administrator updates the global permissions for a group, the changes take effect for individual group members after they log out of Rancher and log in again. + +To make the changes take effect immediately, an administrator or cluster owner can refresh group memberships. + +An administrator might also want to refresh group memberships if a user is removed from a group in the external authentication service. In that case, the refresh makes Rancher aware that the user was removed from the group. + +To refresh group memberships, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Click **Refresh Group Memberships**. + +**Result:** Any changes to the group members' permissions will take effect. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md new file mode 100644 index 00000000000..4c4aec0be54 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md @@ -0,0 +1,42 @@ +--- +title: Locked Roles +--- + + + + + +You can set roles to a status of `locked`. Locking roles prevent them from being assigned to users in the future. + +Locked roles: + +- Cannot be assigned to users that don't already have it assigned. +- Are not listed in the **Member Roles** drop-down when you are adding a user to a cluster or project. +- Do not affect users assigned the role before you lock the role. These users retain access that the role provides. + + **Example:** let's say your organization creates an internal policy that users assigned to a cluster are prohibited from creating new projects. It's your job to enforce this policy. + + To enforce it, before you add new users to the cluster, you should lock the following roles: `Cluster Owner`, `Cluster Member`, and `Create Projects`. Then you could create a new custom role that includes the same permissions as a __Cluster Member__, except the ability to create projects. Then, you use this new custom role when adding users to a cluster. + +Roles can be locked by the following users: + +- Any user assigned the `Administrator` global permission. +- Any user assigned the `Custom Users` permission, along with the `Manage Roles` role. + + +## Locking/Unlocking Roles + +If you want to prevent a role from being assigned to users, you can set it to a status of `locked`. + +You can lock roles in two contexts: + +- When you're [adding a custom role](custom-roles.md). +- When you editing an existing role (see below). + +Cluster roles and project/namespace roles can be locked, but global roles cannot. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Go to the **Cluster** tab or the **Project/Namespaces** tab. +1. From the role that you want to lock (or unlock), select **⋮ > Edit Config**. +1. From the **Locked** option, choose the **Yes** or **No** radio button. Then click **Save**. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md new file mode 100644 index 00000000000..654149cf6e4 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md @@ -0,0 +1,130 @@ +--- +title: Pod Security Standards (PSS) & Pod Security Admission (PSA) +--- + + + + + +[Pod Security Standards (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/) and [Pod Security Admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) define security restrictions for a broad set of workloads. +They became available and were turned on by default in Kubernetes v1.23, and replace [Pod Security Policies (PSP)](https://kubernetes.io/docs/concepts/security/pod-security-policy/) in Kubernetes v1.25 and above. + +PSS define security levels for workloads. PSAs describe requirements for pod security contexts and related fields. PSAs reference PSS levels to define security restrictions. + +## Upgrade to Pod Security Standards (PSS) + +Ensure that you migrate all PSPs to another workload security mechanism. This includes mapping your current PSPs to Pod Security Standards for enforcement with the [PSA controller](https://kubernetes.io/docs/concepts/security/pod-security-admission/). If the PSA controller won't meet all of your organization's needs, we recommend that you use a policy engine, such as [OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper), [Kubewarden](https://www.kubewarden.io/), [Kyverno](https://kyverno.io/), or [NeuVector](https://neuvector.com/). Refer to the documentation of your policy engine of choice for more information on how to migrate from PSPs. + +:::caution +You must add your new policy enforcement mechanisms _before_ you remove the PodSecurityPolicy objects. If you don't, you may create an opportunity for privilege escalation attacks within the cluster. +::: + +### Removing PodSecurityPolicies from Rancher-Maintained Apps & Marketplace Workloads {#remove-psp-rancher-workloads} + +Rancher v2.7.2 offers a new major version of Rancher-maintained Helm charts. v102.x.y allows you to remove PSPs that were installed with previous versions of the chart. This new version replaces non-standard PSPs switches with the standardized `global.cattle.psp.enabled` switch, which is turned off by default. + +You must perform the following steps _while still in Kubernetes v1.24_: +1. Configure the PSA controller to suit your needs. You can use one of Rancher's built-in [PSA Configuration Templates](#psa-config-templates), or create a custom template and apply it to the clusters that you are migrating. + +1. Map your active PSPs to Pod Security Standards: + 1. See which PSPs are still active in your cluster: + :::caution + This strategy may miss workloads that aren't currently running, such as CronJobs, workloads currently scaled to zero, or workloads that haven't rolled out yet. + ::: + + ```shell + kubectl get pods \ + --all-namespaces \ + --output jsonpath='{.items[*].metadata.annotations.kubernetes\.io\/psp}' \ + | tr " " "\n" | sort -u + ``` + + 1. Follow the Kubernetes guide on [Mapping PSPs to Pod Security Standards](https://kubernetes.io/docs/reference/access-authn-authz/psp-to-pod-security-standards/) to apply PSSs to your workloads that were relying on PSPs. See [Migrate from PodSecurityPolicy to the Built-In PodSecurity Admission controller](https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/) for more details. + +1. To remove PSPs from Rancher charts, upgrade the charts to the latest v102.x.y version _before_ you upgrade to Kubernetes v1.25. Make sure that the **Enable PodSecurityPolicies** option is **disabled**. This will remove any PSPs that were installed with previous chart versions. + +:::info important +If you want to upgrade your charts to v102.x.y, but don't plan on upgrading your clusters to Kubernetes v1.25 and moving away from PSPs, make sure that you select the option **Enable PodSecurityPolicies** for each chart that you are upgrading. +::: + +### Cleaning Up Releases After a Kubernetes v1.25 Upgrade + +If you experience problems while removing PSPs from your charts, or have charts that don't contain a built-in mechanism for removing PSPs, your chart upgrades or deletions might fail with an error message such as the following: +```console +Error: UPGRADE FAILED: resource mapping not found for name: "" namespace: "" from "": no matches for kind "PodSecurityPolicy" in version "policy/v1beta1" +ensure CRDs are installed first +``` + +This happens when Helm tries to query the cluster for objects that were stored in a previous release's data blob. To clean up these releases and avoid this error, use the `helm-mapkubeapis` Helm plugin. To learn more about `helm-mapkubeapis`, how it works, and how it can be fine-tuned for your use case, see the [official Helm documentation](https://github.com/helm/helm-mapkubeapis#readme). + +Note that Helm plugin installation is local to the machine that you run the commands from. Therefore, make sure that you run both the installation and cleanup from the same machine. + +#### Install `helm-mapkubeapis` + +1. Open your terminal in the machine you intend to use `helm-mapkubeapis` from and install the plugin: + ```shell + helm plugin install https://github.com/helm/helm-mapkubeapis + ``` + + You will see output similar to the following: + ```console + Downloading and installing helm-mapkubeapis v0.4.1 ... + https://github.com/helm/helm-mapkubeapis/releases/download/v0.4.1/helm-mapkubeapis_0.4.1_darwin_amd64.tar.gz + Installed plugin: mapkubeapis + ``` + + :::info important + Ensure that the `helm-mapkubeapis` plugin is at least v0.4.1, as older versions _do not_ support removal of resources. + ::: + +1. Verify that the plugin was correctly installed: + ```shell + helm mapkubeapis --help + ``` + + You will see output similar to the following: + ```console + Map release deprecated or removed Kubernetes APIs in-place + + Usage: + mapkubeapis [flags] RELEASE + + Flags: + --dry-run simulate a command + -h, --help help for mapkubeapis + --kube-context string name of the kubeconfig context to use + --kubeconfig string path to the kubeconfig file + --mapfile string path to the API mapping file + --namespace string namespace scope of the release + ``` + +#### Cleaning Up Broken Releases + +After you install the `helm-mapkubeapis` plugin, clean up the releases that became broken after the upgrade to Kubernetes v1.25. + +1. Open your preferred terminal and make sure it's connected to the cluster you wish to target by running `kubectl cluster-info`. + +1. List all the releases you have installed in your cluster by running `helm list --all-namespaces`. + +1. Perform a dry run for each release you would like to clean up by running `helm mapkubeapis --dry-run --namespace `. The result of this command will inform you what resources are going to be replaced or removed. + +1. Finally, after reviewing the changes, perform a full run with `helm mapkubeapis --namespace `. + +#### Upgrading Charts to a Version That Supports Kubernetes v1.25 + +You can proceed with your upgrade once any releases that had lingering PSPs are cleaned up. For Rancher-maintained workloads, follow the steps outlined in the [Removing PodSecurityPolicies from Rancher-maintained Apps & Marketplace workloads](#remove-psp-rancher-workloads) section of this document. +For workloads not maintained by Rancher, refer to the vendor documentation. + +:::caution +Do not skip this step. Applications incompatible with Kubernetes v1.25 aren't guaranteed to work after a cleanup. +::: + +## Pod Security Admission Configuration Templates {#psa-config-templates} + +Rancher offers PSA configuration templates. These are pre-defined security configurations that you can apply to a cluster. Rancher admins (or those with the right permissions) can [create, manage, and edit](./psa-config-templates.md) PSA templates. + +### Rancher on PSA-restricted Clusters + +Rancher system namespaces are also affected by the restrictive security policies described by PSA templates. You need to exempt Rancher's system namespaces after you assign the template, or else the cluster won't operate correctly. See [Pod Security Admission (PSA) Configuration Templates](./psa-config-templates.md#exempting-required-rancher-namespaces) for more details. + +For a complete file which has all the exemptions you need to run Rancher, please refer to this [sample Admission Configuration](../../../reference-guides/rancher-security/psa-restricted-exemptions.md). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md new file mode 100644 index 00000000000..dac4d66ee52 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md @@ -0,0 +1,142 @@ +--- +title: Pod Security Admission (PSA) Configuration Templates +--- + + + + + +[Pod Security admission (PSA)](./pod-security-standards.md) configuration templates are a Rancher custom-defined resource (CRD), available in Rancher v2.7.2 and above. The templates provide pre-defined security configurations that you can apply to a cluster: + +- `rancher-privileged`: The most permissive configuration. It doesn't restrict the behavior of any pods. This allows for known privilege escalations. This policy has no exemptions. +- `rancher-restricted`: A heavily restricted configuration that follows current best practices for hardening pods. You must make [namespace-level exemptions](./pod-security-standards.md#rancher-on-psa-restricted-clusters) for Rancher components. + +## Assign a Pod Security Admissions (PSA) Configuration Template + +You can assign a PSA template at the same time that you create a downstream cluster. You can also add a template by configuring an existing cluster. + +### Assign a Template During Cluster Creation + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click the **Create** button. +1. Select a provider. +1. On the **Cluster: Create** page, go to **Basics > Security**. +1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. +1. Click **Create**. + +### Assign a Template to an Existing Cluster + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config** . +1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. +1. Click **Save**. + +### Hardening the Cluster + +If you select the **rancher-restricted** template but don't select a **CIS Profile**, you won't meet required CIS benchmarks. See the [RKE2 hardening guide](../../../pages-for-subheaders/rke2-hardening-guide.md) for more details. + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click the **Create** button. +1. Select a provider. +1. On the **Add Cluster** page, under **Cluster Options**, click **Advanced Options**. +1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. +1. Click **Create**. + +### Assign a Template to an Existing Cluster + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config**. +1. On the **Edit Cluster** page, go to **Cluster Options > Advanced Options**. +1. In the **Pod Security Admission Configuration Template**, select the template you want to assign. +1. Click **Save**. + + + + +## Add or Edit a Pod Security Admissions (PSA) Configuration Template + +If you have administrator privileges, you can customize security restrictions and permissions by creating additional PSA templates, or by editing existing templates. + +:::caution +If you edit an existing PSA template while it is still in use, changes will be applied to all clusters that have been assigned to that template. +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **Advanced** to open the dropdown menu. +1. Select **Pod Security Admissions**. +1. Find the template you want to modify, and click the **⋮**. +1. Select **Edit Config** to edit the template. +1. When you're done editing the configuration, click **Save**. + +### Allow Non-Admin Users to Manage PSA Templates + +If you want to allow other users to manage templates, you can bind that user to a role that grants all verbs (`"*"`) on `management.cattle.io/podsecurityadmissionconfigurationtemplates`. + +:::caution +Any user that is bound to the above permission will be able to change the restriction levels on _all_ managed clusters which use a given PSA template, including ones that they have no permissions on. +::: + +## Exempting Required Rancher Namespaces + +When you run Rancher on a Kubernetes cluster that enforces a restrictive security policy by default, you'll need to [exempt the following namespaces](#exempting-namespaces), otherwise the policy might prevent Rancher system pods from running properly. + +- `calico-apiserver` +- `calico-system` +- `cattle-alerting` +- `cattle-csp-adapter-system` +- `cattle-epinio-system` +- `cattle-externalip-system` +- `cattle-fleet-local-system` +- `cattle-fleet-system` +- `cattle-gatekeeper-system` +- `cattle-global-data` +- `cattle-global-nt` +- `cattle-impersonation-system` +- `cattle-istio` +- `cattle-istio-system` +- `cattle-logging` +- `cattle-logging-system` +- `cattle-monitoring-system` +- `cattle-neuvector-system` +- `cattle-prometheus` +- `cattle-sriov-system` +- `cattle-system` +- `cattle-ui-plugin-system` +- `cattle-windows-gmsa-system` +- `cert-manager` +- `cis-operator-system` +- `fleet-default` +- `ingress-nginx` +- `istio-system` +- `kube-node-lease` +- `kube-public` +- `kube-system` +- `longhorn-system` +- `rancher-alerting-drivers` +- `security-scan` +- `tigera-operator` + +Rancher, some Rancher owned charts, and RKE2 and K3s distributions all use these namespaces. A subset of the listed namespaces are already exempt in the built-in Rancher `rancher-restricted` policy, for use in downstream clusters. For a complete template which has all the exemptions you need to run Rancher, please refer to this [sample Admission Configuration](../../../reference-guides/rancher-security/psa-restricted-exemptions.md). + +## Exempting Namespaces + +If you assign the `rancher-restricted` template to a cluster, by default the restrictions are applied across the entire cluster at the namespace level. To exempt certain namespaces from this highly restricted policy, do the following: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **Advanced** to open the dropdown menu. +1. Select **Pod Security Admissions**. +1. Find the template you want to modify, and click the **⋮**. +1. Select **Edit Config**. +1. Click the **Namespaces** checkbox under **Exemptions** to edit the **Namespaces** field. +1. When you're done exempting namespaces, click **Save**. + +:::note +You need to update the target cluster to make the new template take effect in that cluster. An update can be triggered by editing and saving the cluster without changing values. +::: diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md new file mode 100644 index 00000000000..2f0d0f2c9b7 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md @@ -0,0 +1,78 @@ +--- +title: Backing up Rancher Installed with Docker +--- + + + + + +After completing your Docker installation of Rancher, we recommend creating backups of it on a regular basis. Having a recent backup will let you recover quickly from an unexpected disaster. + +## Before You Start + +During the creation of your backup, you'll enter a series of commands, replacing placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). Here's an example of a command with a placeholder: + +``` +docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher +``` + +In this command, `` is a placeholder for the date that the data container and backup were created. `9-27-18` for example. + +Cross reference the image and reference table below to learn how to obtain this placeholder data. Write down or copy this information before starting the [procedure below](#creating-a-backup). + +Terminal docker ps Command, Displaying Where to Find <RANCHER_CONTAINER_TAG> and <RANCHER_CONTAINER_NAME> + +![Placeholder Reference](/img/placeholder-ref.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | The rancher/rancher image you pulled for initial install. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.0.5` | The version of Rancher that you're creating a backup for. | +| `` | `9-27-18` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher Server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped with `docker ps -a`. Use these commands for help anytime while creating backups. + +## Creating a Backup + +This procedure creates a backup that you can restore if Rancher encounters a disaster scenario. + + +1. Using a remote Terminal connection, log into the node running your Rancher Server. + +1. Stop the container currently running Rancher Server. Replace `` with the name of your Rancher container. + + ``` + docker stop + ``` +1. Use the command below, replacing each placeholder, to create a data container from the Rancher container that you just stopped. + + ``` + docker create --volumes-from --name rancher-data- rancher/rancher: + ``` + +1. From the data container that you just created (rancher-data-<DATE>), create a backup tarball (rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz). Use the following command, replacing each placeholder: + + ``` + docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup:z busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **Step Result:** A stream of commands runs on the screen. + +1. Enter the `ls` command to confirm that the backup tarball was created. It will have a name similar to `rancher-data-backup--.tar.gz`. + +1. Move your backup tarball to a safe location external to your Rancher Server. Then delete the `rancher-data-` and `busybox-backup-` containers from your Rancher Server. + + ``` + docker rm rancher-data- + docker rm busybox-backup- + ``` + +1. Restart Rancher Server. Replace `` with the name of your Rancher container: + + ``` + docker start + ``` + +**Result:** A backup tarball of your Rancher Server data is created. See [Restoring Backups: Docker Installs](restore-docker-installed-rancher.md) if you need to restore backup data. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md new file mode 100644 index 00000000000..a9046a1e64d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md @@ -0,0 +1,311 @@ +--- +title: Backing up a Cluster +--- + + + + + +In the Rancher UI, etcd backup and recovery for [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) can be easily performed. + +Rancher recommends configuring recurrent `etcd` snapshots for all production clusters. Additionally, one-time snapshots can be taken as well. + +Snapshots of the etcd database are taken and saved either [locally onto the etcd nodes](#local-backup-target) or to a [S3 compatible target](#s3-backup-target). The advantages of configuring S3 is that if all etcd nodes are lost, your snapshot is saved remotely and can be used to restore the cluster. + +## How Snapshots Work + +### Snapshot Components + + + + +When Rancher creates a snapshot, it includes three components: + +- The cluster data in etcd +- The Kubernetes version +- The cluster configuration in the form of the `cluster.yml` + +Because the Kubernetes version is now included in the snapshot, it is possible to restore a cluster to a prior Kubernetes version. + + + + +Rancher delegates snapshot creation to the downstream Kubernetes engine. When the Kubernetes engine creates a snapshot, it includes three components: + +- The cluster data in etcd +- The Kubernetes version +- The cluster configuration + +Because the Kubernetes version is included in the snapshot, it is possible to restore a cluster to a prior Kubernetes version while also restoring an etcd snapshot. + + + + +The multiple components of the snapshot allow you to select from the following options if you need to restore a cluster from a snapshot: + +- **Restore just the etcd contents:** This restore is similar to restoring to snapshots in Rancher before v2.4.0. +- **Restore etcd and Kubernetes version:** This option should be used if a Kubernetes upgrade is the reason that your cluster is failing, and you haven't made any cluster configuration changes. +- **Restore etcd, Kubernetes versions and cluster configuration:** This option should be used if you changed both the Kubernetes version and cluster configuration when upgrading. + +It is always recommended to take a new snapshot before performing any configuration changes or upgrades. + + +### Generating the Snapshot from etcd Nodes + + + + +For each etcd node in the cluster, the etcd cluster health is checked. If the node reports that the etcd cluster is healthy, a snapshot is created from it and optionally uploaded to S3. + +The snapshot is stored in `/opt/rke/etcd-snapshots`. If the directory is configured on the nodes as a shared mount, it will be overwritten. On S3, the snapshot will always be from the last node that uploads it, as all etcd nodes upload it and the last will remain. + +In the case when multiple etcd nodes exist, any created snapshot is created after the cluster has been health checked, so it can be considered a valid snapshot of the data in the etcd cluster. + + + + +Snapshots are enabled by default. + +The snapshot directory defaults to `/var/lib/rancher//server/db/snapshots`, where `` is either `rke2` or `k3s`. + +In RKE2, snapshots are stored on each etcd node. If you have multiple etcd or etcd + control-plane nodes, you will have multiple copies of local etcd snapshots. + + + + +### Snapshot Naming Conventions + + + + +The name of the snapshot is auto-generated. The `--name` option can be used to override the name of the snapshot when creating one-time snapshots with the RKE CLI. + +When Rancher creates a snapshot of an RKE cluster, the snapshot name is based on the type (whether the snapshot is manual or recurring) and the target (whether the snapshot is saved locally or uploaded to S3). The naming convention is as follows: + +- `m` stands for manual +- `r` stands for recurring +- `l` stands for local +- `s` stands for S3 + +Some example snapshot names are: + +- c-9dmxz-rl-8b2cx +- c-9dmxz-ml-kr56m +- c-9dmxz-ms-t6bjb +- c-9dmxz-rs-8gxc8 + + + + +The name of the snapshot is auto-generated. The `--name` option can be used to override the base name of the snapshot when creating one-time snapshots with the RKE2 or K3s CLI. + +When Rancher creates a snapshot of an RKE2 or K3s cluster, the snapshot name is based on the type (whether the snapshot is manual or recurring) and the target (whether the snapshot is saved locally or uploaded to S3). The naming convention is as follows: + +`--` + +``: is the base name set by `--name` and can be one of the the following + +- `etcd-snapshot` is prepended on recurring snapshots +- `on-demand` is prepended on manual, on-demand snapshots + +``: Node is the name of the node that the snapshot was taken on. + +`` is a unix-time stamp of the snapshot creation date. + +Some example snapshot names are: + +- `on-demand-my-super-rancher-k8s-node1-1652288934` +- `on-demand-my-super-rancher-k8s-node2-1652288936` +- `etcd-snapshot-my-super-rancher-k8s-node1-1652289945` +- `etcd-snapshot-my-super-rancher-k8s-node2-1652289948` + + + + +### How Restoring from a Snapshot Works + + + + +On restore, the following process is used: + +1. The snapshot is retrieved from S3, if S3 is configured. +2. The snapshot is unzipped (if zipped). +3. One of the etcd nodes in the cluster serves that snapshot file to the other nodes. +4. The other etcd nodes download the snapshot and validate the checksum so that they all use the same snapshot for the restore. +5. The cluster is restored and post-restore actions will be done in the cluster. + + + + +On restore, Rancher delivers a few sets of plans to perform a restoration. A set of phases are used, namely: + +- Started +- Shutdown +- Restore +- RestartCluster +- Finished + +If the etcd snapshot restore fails, the phase will be set to `Failed`. + +1. The etcd snapshot restore request is received, and depending on `restoreRKEConfig`, the cluster configuration/kubernetes version are reconciled. +1. The phase is set to `Started`. +1. The phase is set to `Shutdown`, and the entire cluster is shut down using plans that run the distribution `killall.sh` script. A new init node is elected. If the snapshot being restored is a local snapshot, the node that the snapshot resides on will be selected as the init node. If the snapshot is being restored from S3, the existing init node will be used. +1. The phase is set to `Restore`, and the init node has the snapshot restored onto it. +1. The phase is set to `RestartCluster`, and the cluster is restarted/rejoined to the new init node that has the freshly restored snapshot information. +1. The phase is set to `Finished`, and the cluster is deemed successfully restored. The `cattle-cluster-agent` will reconnect, and the cluster will finish reconciliation. + + + + +## Configuring Recurring Snapshots + + + + +Select how often you want recurring snapshots to be taken as well as how many snapshots to keep. The amount of time is measured in hours. With timestamped snapshots, the user has the ability to do a point-in-time recovery. + +By default, [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) are configured to take recurring snapshots (saved to local disk). To protect against local disk failure, using the [S3 Target](#s3-backup-target) or replicating the path on disk is advised. + +During cluster provisioning or editing the cluster, the configuration for snapshots can be found in the advanced section for **Cluster Options**. Click on **Show advanced options**. + +In the **Advanced Cluster Options** section, there are several options available to configure: + +| Option | Description | Default Value| +| --- | ---| --- | +| etcd Snapshot Backup Target | Select where you want the snapshots to be saved. Options are either local or in S3 | local| +|Recurring etcd Snapshot Enabled| Enable/Disable recurring snapshots | Yes| +| Recurring etcd Snapshot Creation Period | Time in hours between recurring snapshots| 12 hours | +| Recurring etcd Snapshot Retention Count | Number of snapshots to retain| 6 | + + + + +Set the schedule for how you want recurring snapshots to be taken as well as how many snapshots to keep. The schedule is conventional cron format. The retention policy dictates the number of snapshots matching a name to keep per node. + +By default, [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) are configured to take recurring snapshots (saved to local disk) every 5 hours starting at 12 AM. To protect against local disk failure, using the [S3 Target](#s3-backup-target) or replicating the path on disk is advised. + +During cluster provisioning or editing the cluster, the configuration for snapshots can be found under **Cluster Configuration**. Click on **etcd**. + +| Option | Description | Default Value| +| --- | ---| --- | +| Recurring etcd Snapshot Enabled | Enable/Disable recurring snapshots | Yes | +| Recurring etcd Snapshot Creation Period | Cron schedule for recurring snapshot | `0 */5 * * *` | +| Recurring etcd Snapshot Retention Count | Number of snapshots to retain | 5 | + + + + +## One-Time Snapshots + + + + +In addition to recurring snapshots, you may want to take a "one-time" snapshot. For example, before upgrading the Kubernetes version of a cluster it's best to backup the state of the cluster to protect against upgrade failure. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, navigate to the cluster where you want to take a one-time snapshot. +1. Click **⋮ > Take Snapshot**. + + + + +In addition to recurring snapshots, you may want to take a "one-time" snapshot. For example, before upgrading the Kubernetes version of a cluster it's best to backup the state of the cluster to protect against upgrade failure. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, navigate to the cluster where you want to take a one-time snapshot. +1. Navigate to the `Snapshots` tab and click `Snapshot Now` + +### How Taking One-Time Snapshots Works + +On one-time snapshot creation, the Rancher delivers a few sets of plans to perform snapshot creation. A set of phases are used, namely: + +- Started +- RestartCluster +- Finished + +If the etcd snapshot creation fails, the phase will be set to `Failed`. + +1. The etcd snapshot creation request is received. +1. The phase is set to `Started`. All etcd nodes in the cluster receive a plan to create an etcd snapshot, per the cluster configuration. +1. The phase is set to `RestartCluster`, and the plans on every etcd node are reset to the original plan for the etcd nodes. +1. The phase is set to `Finished`. + + + + +**Result:** Based on your [snapshot backup target](#snapshot-backup-targets), a one-time snapshot will be taken and saved in the selected backup target. + +## Snapshot Backup Targets + +Rancher supports two different backup targets: + +- [Local Target](#local-backup-target) +- [S3 Target](#s3-backup-target) + +### Local Backup Target + + + + +By default, the `local` backup target is selected. The benefits of this option is that there is no external configuration. Snapshots are automatically saved locally to the etcd nodes in the [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) in `/opt/rke/etcd-snapshots`. All recurring snapshots are taken at configured intervals. The downside of using the `local` backup target is that if there is a total disaster and _all_ etcd nodes are lost, there is no ability to restore the cluster. + + + + +By default, the `local` backup target is selected. The benefits of this option is that there is no external configuration. Snapshots are automatically saved locally to the etcd nodes in the [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) in `/var/lib/rancher//server/db/snapshots` where `` is either `k3s` or `rke2`. All recurring snapshots are taken per the cron schedule. The downside of using the `local` backup target is that if there is a total disaster and _all_ etcd nodes are lost, there is no ability to restore the cluster. + + + + +### S3 Backup Target + +We recommend that you use the `S3` backup target. It lets you store snapshots externally, on an S3 compatible backend. Since the snapshots aren't stored locally, you can still restore the cluster even if you lose all etcd nodes. + +Although the `S3` target offers advantages over local backup, it does require extra configuration. + +:::caution + +If you use an S3 backup target, make sure that every cluster has its own bucket or folder. Rancher populates snapshot information from any available snapshot listed in the S3 bucket or folder configured for that cluster. + +::: + +| Option | Description | Required| +|---|---|---| +|S3 Bucket Name| Name of S3 bucket to store backups| *| +|S3 Region|S3 region for the backup bucket| | +|S3 Region Endpoint|S3 regions endpoint for the backup bucket|* | +|S3 Access Key|S3 access key with permission to access the backup bucket|*| +|S3 Secret Key|S3 secret key with permission to access the backup bucket|*| +| Custom CA Certificate | A custom certificate used to access private S3 backends || + +### Using a custom CA certificate for S3 + +The backup snapshot can be stored on a custom `S3` backup like [minio](https://min.io/). If the S3 back end uses a self-signed or custom certificate, provide a custom certificate using the `Custom CA Certificate` option to connect to the S3 backend. + +### IAM Support for Storing Snapshots in S3 + +The `S3` backup target supports using IAM authentication to AWS API in addition to using API credentials. An IAM role gives temporary permissions that an application can use when making API calls to S3 storage. To use IAM authentication, the following requirements must be met: + +- The cluster etcd nodes must have an instance role that has read/write access to the designated backup bucket. +- The cluster etcd nodes must have network access to the specified S3 endpoint. +- The Rancher Server worker node(s) must have an instance role that has read/write to the designated backup bucket. +- The Rancher Server worker node(s) must have network access to the specified S3 endpoint. + + To give an application access to S3, refer to the AWS documentation on [Using an IAM Role to Grant Permissions to Applications Running on Amazon EC2 Instances.](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html) + +## Viewing Available Snapshots + +The list of all available snapshots for the cluster is available in the Rancher UI. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to view the snapshots and click its name. +1. Click the **Snapshots** tab to view the list of saved snapshots. These snapshots include a timestamp of when they were created. + +## Safe Timestamps (RKE) + +Snapshot files are timestamped to simplify processing the files using external tools and scripts, but in some S3 compatible backends, these timestamps were unusable. + +The option `safe_timestamp` is added to support compatible file names. When this flag is set to `true`, all special characters in the snapshot filename timestamp are replaced. + +This option is not available directly in the UI, and is only available through the `Edit as Yaml` interface. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md new file mode 100644 index 00000000000..27b02652450 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md @@ -0,0 +1,96 @@ +--- +title: Backing up Rancher +--- + + + + + +In this section, you'll learn how to back up Rancher running on any Kubernetes cluster. To backup Rancher installed with Docker, refer the instructions for [single node backups](back-up-docker-installed-rancher.md) + +The backup-restore operator needs to be installed in the local cluster, and only backs up the Rancher app. The backup and restore operations are performed only in the local Kubernetes cluster. + +Note that the rancher-backup operator version 2.x.x is for Rancher v2.6.x. + +:::caution + +When restoring a backup into a new Rancher setup, the version of the new setup should be the same as the one where the backup is made. The Kubernetes version should also be considered when restoring a backup, since the supported apiVersion in the cluster and in the backup file could be different. + +::: + +### Prerequisites + +The Rancher version must be v2.5.0 and up. + +Refer [here](migrate-rancher-to-new-cluster.md#2-restore-from-backup-using-a-restore-custom-resource) for help on restoring an existing backup file into a v1.22 cluster in Rancher v2.6.3. + +### 1. Install the Rancher Backup operator + +The backup storage location is an operator-level setting, so it needs to be configured when the Rancher Backups application is installed or upgraded. + +Backups are created as .tar.gz files. These files can be pushed to S3 or Minio, or they can be stored in a persistent volume. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. The `local` cluster runs the Rancher server. +1. Click **Apps > Charts**. +1. Click **Rancher Backups**. +1. Click **Install**. +1. Configure the default storage location. For help, refer to the [storage configuration section.](../../../reference-guides/backup-restore-configuration/storage-configuration.md) +1. Click **Install**. + +:::note + +There is a known issue in Fleet that occurs after performing a restoration using the backup-restore-operator: Secrets used for clientSecretName and helmSecretName are not included in Fleet gitrepos. Refer [here](../deploy-apps-across-clusters/fleet.md#troubleshooting) for a workaround. + +::: + +### 2. Perform a Backup + +To perform a backup, a custom resource of type Backup must be created. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. +1. In the left navigation bar, click **Rancher Backups > Backups**. +1. Click **Create**. +1. Create the Backup with the form, or with the YAML editor. +1. For configuring the Backup details using the form, click **Create** and refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/backup-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md#backup) +1. For using the YAML editor, we can click **Create > Create from YAML**. Enter the Backup YAML. This example Backup custom resource would create encrypted recurring backups in S3. The app uses the `credentialSecretNamespace` value to determine where to look for the S3 backup secret: + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Backup + metadata: + name: s3-recurring-backup + spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 + ``` + + :::note + + When creating the Backup resource using YAML editor, the `resourceSetName` must be set to `rancher-resource-set` + + ::: + + For help configuring the Backup, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/backup-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md#backup) + + :::caution + + The `rancher-backup` operator doesn't save the EncryptionConfiguration file. The contents of the EncryptionConfiguration file must be saved when an encrypted backup is created, and the same file must be used when restoring from this backup. + + ::: + +1. Click **Create**. + +**Result:** The backup file is created in the storage location configured in the Backup custom resource. The name of this file is used when performing a restore. + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md new file mode 100644 index 00000000000..3ef56172eb1 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md @@ -0,0 +1,133 @@ +--- +title: Backup Restore Usage Guide +--- + + + + + +The Rancher Backups chart is our solution for disaster recovery and migration. This chart allows you to take backups of your Kubernetes resources and save them to a variety of persistent storage locations. + +This chart is a very simple tool which has its hands in many different areas of the Rancher ecosystem. As a result, edge cases have arisen which lead to undocumented functionality. The purpose of this document is to highlight the proper and defined usage for Rancher Backups, as well as discussing some of these edge cases we’ve run into. + +## Functionality Overview + +### Backup + +The operator collects all the resources captured by the resourceSet in the chart as in-memory unstructured objects. After the resources have been collected, a compressed tar file of the resources are saved as a collection of manifests in JSON and then uploaded to a user-defined object store. This backup can be put on a repeating schedule and can also be encrypted. This encryption option is important since some of the resources are sensitive and the values are stored in plaintext without encryption. + +See the [Backup Configuration documentation](../../../reference-guides/backup-restore-configuration/backup-configuration.md) for more information about the options, including encryption, to configure a backup. + +:::note + +As noted in the [Backing up Rancher documentation](./back-up-rancher.md), you must manually save the encryption configuration file contents since the operator will **not** back it up. + +::: + +### Restore + +There are two main restore scenarios, restoring to a cluster with Rancher running and restoring to a fresh cluster. You can only restore to a cluster with Rancher running if it's the same cluster the backup was taken from and the [`prune` option](../../../reference-guides/backup-restore-configuration/restore-configuration.md#prune-during-restore) is enabled during the restore. A restore has similar inputs as a backup. It requires a backup filename, the encryptionConfigSecret name, and the storage location. + +Resources are restored in this order: + +1. Custom Resource Definitions (CRDs) +2. Cluster-scoped resources +3. Namespaced resources + +See the [Restore Configuration documentation](../../../reference-guides/backup-restore-configuration/restore-configuration.md) for more information about the options to configure a restore. + +### Resource Sets + +The resourceSet determines which resources the backup-restore-operator collects in a backup. It's a set of ResourceSelectors, which define the selection requirements using key/value matching, regular expression matching, or the Kubernetes client labelSelector. + +These are the different fields available for a resourceSelector: + +- apiVersion +- excludeKinds +- excludeResourceNameRegexp +- kinds +- kindsRegexp +- labelSelectors +- namespaceRegexp +- namespaces +- resourceNameRegexp +- resourceNames + +The Rancher Backups chart contains a [default resourceSet](https://github.com/rancher/backup-restore-operator/tree/release/v3.0/charts/rancher-backup/files/default-resourceset-contents), which is a combination of YAML files that are appended to one large resourceSet when the chart is installed. The file order does not matter. The resourceSets may differ between versions. + +:::caution + +If you wish to make edits to the resourceSet please edit it **before** installing the chart. + +::: + +## Proper Usage + +This section outlines the guidelines for the proper usage of the Rancher Backups chart according to its use case. + +### All Cases + +- Rancher Backups must be installed on the local cluster. + - Note: Rancher Backups does not handle any cluster other than the one it is installed on to. It may restore cluster resources that are on the local cluster but will not communicate with the downstream clusters or back them up. +- The Rancher version being restored to must match the Rancher version from backup. +- The Kubernetes version should be be considered since you may be restoring outdated resources (resources deprecated from the version of Kubernetes you are restoring to). + +### Backups + +- Some user generated resources will not be backed up unless they can are captured by the default resourceSet or the resourceSet was altered to capture them. + - We provide a label `resources.cattle.io/backup:true` that when added to a secret in any namespace, will result in it being backed up. +- Backups are non-mutating +- Backups are only of the local cluster + +### Restores + +- A restore refers to restoring a backup to the same cluster it was taken from. This can be with Rancher installed (**prune must be enabled**) or with it not installed (no special instructions). +- One thing to note when restoring is that you may find yourself needing to “wipe†the cluster of any Rancher resources. This can be done by deploying the [Rancher cleanup script](https://github.com/rancher/rancher-cleanup) script as a job to the cluster. This allows you to install Rancher Backups again and restore to a completely fresh cluster. + - Make sure to use kubectl to deploy the scripts. + +### Migrations + +Migration have some more nuance since we are restoring to a different cluster. These are a few things to remember that are commonly missed or forgotten. + +- The Rancher domain must be the same when migrating. This means your previous cluster’s domain name must now point to the new cluster. +- Rancher should **not** be running already on the cluster you are migrating to. This can cause many issues with Rancher backups and certain Rancher services as well. +- Install the **same** version of Rancher from the backup **after** the backup has been restored. +- If you choose to provision the new cluster on a different Kubernetes version know that this can cause a wide variety of unsupported behaviors because the Kubernetes API available may be different from the one you took a backup from. This can lead to deprecated resources being restored which will cause issues. +- You should **not** perform any upgrades during a migration. + +## Edge Cases and Improper Usage + +Below are some examples of some **incorrect** uses or expectations of Rancher Backups. + +### Upgrades + +- Using Rancher backups for upgrading Rancher versions is not a valid use case. The recommended procedure is to take a backup of the current version, then upgrade your Rancher instance using [these instructions](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md), and then taking **another** backup after the upgrade is complete. This way if the upgrade fails you have a backup to restore to, and the second backup will be valid to restore to the upgraded Rancher version. +- Using Rancher backups for upgrading Kubernetes versions is not a valid use case either. Because the Kubernetes API and available resources are tied to the version, upgrading using backup restore can lead to issues with misaligned sets of resources which may be deprecated, unsupported, or updated. How to upgrade your cluster version will depend on how it was provisioned however the same format as above is recommended (backup, upgrade, backup). + +### ResourceSet + +- Because of evolving resources and services from various teams, developers should take note if new resources need to be added to or removed from the default resourceSet. +- Rancher backups only backs up what is captured by the default resourceSets (unless edited). We have added a specific label for user created secrets that will back up a secret of any name and namespace that has said label (see [Proper Usage on Backups](#backups)). + +### Downstream Clusters + +- Rancher Backups **only** backs up Kubernetes resources on the local cluster. This means downstream clusters are **not** touched or backed up other than their presence in resources in the local cluster. The updating and communication of downstream clusters falls upon the rancher-agent and rancher-webhook. + +### Restoring Deleted Resources + +- Some resources have external results produced, such as provisioning a downstream cluster. Deleting a downstream cluster and restoring the cluster resource on the local cluster does **not** cause Rancher to reprovision said cluster. Depending on the resource, restoring may not fully bring back the resource to an available state. +- The corner case of "restoring a deleted cluster" is **not** a supported feature. When it comes to downstream clusters, whether provisioned or imported, deleting them causes a series of cleanup tasks which doesn't allow the user to restore the deleted clusters. Provisioned clusters will have their nodes and Rancher-related provisioning resources destroyed, and imported clusters will likely have their Rancher agents and other resources/services related to registration with a local cluster destroyed. + +:::caution + +Trying to delete and restore a downstream cluster can lead to a variety of issues with Rancher, Rancher Backups, rancher-webhook, Fleet, and more. It is not recommended to do this. + +::: + +### Fleet, Harvester, and Other Services + +Other services, which are backed up by Rancher Backups, often change and evolve. As this happens, their resources and backup needs may change as well. Some resources may not need to be backed up and some may not be backed up at all. It is important for teams to consider this in their development process and assess whether their related resourceSets are correctly capturing the proper set of resources for their services to be restored correctly. + +## Conclusion + +Rancher Backups is a very useful tool, however it is somewhat limited in its scope and intended purposes. In order to avoid possible difficulties, it is important to follow the specific procedures described to ensure the proper operation of the chart. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md new file mode 100644 index 00000000000..777f7e6f34f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md @@ -0,0 +1,188 @@ +--- +title: Migrating Rancher to a New Cluster +--- + + + + + +If you are migrating Rancher to a new Kubernetes cluster, you don't need to install Rancher on the new cluster first. If Rancher is restored to a new cluster with Rancher already installed, it can cause problems. + +### Prerequisites + +These instructions assume you have [created a backup](back-up-rancher.md) and you have already installed a new Kubernetes cluster where Rancher will be deployed. + +:::caution + +It is required to use the same hostname that was set as the server URL in the first cluster. If not done, downstream clusters will show as unavailable in the cluster management page of the UI, and you won't be able to click inside the cluster or on the cluster's Explore button. + +::: + +Rancher version must be v2.5.0 and up + +Rancher can be installed on any Kubernetes cluster, including hosted Kubernetes clusters such as Amazon EKS clusters. For help installing Kubernetes, refer to the documentation of the Kubernetes distribution. One of Rancher's Kubernetes distributions may also be used: + +- [RKE Kubernetes installation docs](https://rancher.com/docs/rke/latest/en/installation/) +- [K3s Kubernetes installation docs](https://rancher.com/docs/k3s/latest/en/installation/) + +### 1. Install the rancher-backup Helm chart +Install the [rancher-backup chart](https://github.com/rancher/backup-restore-operator/tags), using a version in the 2.x.x major version range: + + 1. Add the helm repository: + + ```bash + helm repo add rancher-charts https://charts.rancher.io + helm repo update + ``` + + 1. Select and set `CHART_VERSION` variable with a 2.x.x rancher-backup release version: + ```bash + helm search repo --versions rancher-charts/rancher-backup + CHART_VERSION=<2.x.x> + ``` + + 1. Install the charts: + ```bash + helm install rancher-backup-crd rancher-charts/rancher-backup-crd -n cattle-resources-system --create-namespace --version $CHART_VERSION + helm install rancher-backup rancher-charts/rancher-backup -n cattle-resources-system --version $CHART_VERSION + ``` + + :::note + + The above assumes an environment with outbound connectivity to Docker Hub + + For an **air-gapped environment**, use the Helm value below to pull the `backup-restore-operator` image from your private registry when installing the rancher-backup Helm chart. + + ```bash + --set image.repository $REGISTRY/rancher/backup-restore-operator + ``` + + ::: + +### 2. Restore from backup using a Restore custom resource + +:::note Important: + +Kubernetes v1.22, available as an experimental feature of v2.6.3, does not support restoring from backup files containing CRDs with the apiVersion `apiextensions.k8s.io/v1beta1`. In v1.22, the default `resourceSet` in the rancher-backup app is updated to collect only CRDs that use `apiextensions.k8s.io/v1`. There are currently two ways to work around this issue: + +1. Update the default `resourceSet` to collect the CRDs with the apiVersion v1. +1. Update the default `resourceSet` and the client to use the new APIs internally, with `apiextensions.k8s.io/v1` as the replacement. + + :::note + + When making or restoring backups for v1.22, the Rancher version and the local cluster's Kubernetes version should be the same. The Kubernetes version should be considered when restoring a backup since the supported apiVersion in the cluster and in the backup file could be different. + + ::: + +::: + +1. When using S3 object storage as the backup source for a restore that requires credentials, create a `Secret` object in this cluster to add the S3 credentials. The secret data must have two keys - `accessKey`, and `secretKey`, that contain the S3 credentials. + + The secret can be created in any namespace, this example uses the default namespace. + + ```bash + kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= + ``` + + :::note + + Add your access key and secret key as values for `accessKey` and `secretKey` in the command above. + + ::: + +1. Create a `Restore` object: + + During a migration, `prune` must be set to `false`. See the example below: + + ```yaml + # restore-migration.yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + // highlight-next-line + prune: false + // highlight-next-line + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + :::note Important + + The field `encryptionConfigSecretName` should be used only if your backup was created with encryption enabled. + + If this applies, provide the name of the `Secret` object containing the encryption config file. If you only have the encryption config file, but don't have the secret created in this cluster, use the following steps to create the secret: + + 1. Create an [encryption configuration file](../../../reference-guides/backup-restore-configuration/backup-configuration.md#encryption) + 1. The command below uses a file named `encryption-provider-config.yaml`, with the `--from-file` flag. Run the below once the `EncryptionConfiguration` is saved in a file called `encryption-provider-config.yaml`: + + ```bash + kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system + ``` + + ::: + +1. Apply the manifest, and monitor the Restore status: + 1. Apply the `Restore` object resource: + + ```bash + kubectl apply -f restore-migration.yaml + ``` + + 1. Watch the Restore status: + ```bash + kubectl get restore + ``` + + 1. Watch the restoration logs: + ```bash + kubectl logs -n cattle-resources-system --tail 100 -f -l app.kubernetes.io/instance=rancher-backup + ``` + + 1. Once the Restore resource has the status `Completed`, you can continue the cert-manager and Rancher installation. + +### 3. Install cert-manager + +Follow the steps to [install cert-manager](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-install-cert-manager) in the documentation about installing cert-manager on Kubernetes. + +### 4. Bring up Rancher with Helm + +Use the same version of Helm to install Rancher, that was used on the first cluster. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +```bash +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname= \ + --version x.y.z +``` + +:::note + +If the original Rancher environment is running, you can collect the current values with a kubeconfig for the original environment: + +```bash +helm get values rancher -n cattle-system -o yaml > rancher-values.yaml +``` + +These values can be reused using the `rancher-values.yaml` file. Be sure to switch the kubeconfig to the new Rancher environment. + +```bash +helm install rancher rancher-latest/rancher -n cattle-system -f rancher-values.yaml --version x.y.z +``` + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md new file mode 100644 index 00000000000..a7479ec95cb --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md @@ -0,0 +1,76 @@ +--- +title: Restoring Backups—Docker Installs +--- + + + + + +If you encounter a disaster scenario, you can restore your Rancher Server to your most recent backup. + +## Before You Start + +During restore of your backup, you'll enter a series of commands, filling placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). Here's an example of a command with a placeholder: + +``` +docker run --volumes-from -v $PWD:/backup \ +busybox sh -c "rm /var/lib/rancher/* -rf && \ +tar pzxvf /backup/rancher-data-backup--" +``` + +In this command, `` and `-` are environment variables for your Rancher deployment. + +Cross reference the image and reference table below to learn how to obtain this placeholder data. Write down or copy this information before starting the procedure below. + +Terminal docker ps Command, Displaying Where to Find <RANCHER_CONTAINER_TAG> and <RANCHER_CONTAINER_NAME> + +![Placeholder Reference](/img/placeholder-ref.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | The rancher/rancher image you pulled for initial install. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.0.5` | The version number for your Rancher backup. | +| `` | `9-27-18` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher Server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped using a different command: `docker ps -a`. Use these commands for help anytime during while creating backups. + +## Restoring Backups + +Using a [backup](back-up-docker-installed-rancher.md) that you created earlier, restore Rancher to its last known healthy state. + +1. Using a remote Terminal connection, log into the node running your Rancher Server. + +1. Stop the container currently running Rancher Server. Replace `` with the name of your Rancher container: + + ``` + docker stop + ``` +1. Move the backup tarball that you created during completion of [Creating Backups—Docker Installs](back-up-docker-installed-rancher.md) onto your Rancher Server. Change to the directory that you moved it to. Enter `dir` to confirm that it's there. + + If you followed the naming convention we suggested in [Creating Backups—Docker Installs](back-up-docker-installed-rancher.md), it will have a name similar to `rancher-data-backup--.tar.gz`. + +1. Enter the following command to delete your current state data and replace it with your backup data, replacing the placeholders. Don't forget to close the quotes. + + :::danger + + This command deletes all current state data from your Rancher Server container. Any changes saved after your backup tarball was created will be lost. + + ::: + + ``` + docker run --volumes-from -v $PWD:/backup \ + busybox sh -c "rm /var/lib/rancher/* -rf && \ + tar pzxvf /backup/rancher-data-backup--.tar.gz" + ``` + + **Step Result:** A series of commands should run. + +1. Restart your Rancher Server container, replacing the placeholder. It will restart using your backup data. + + ``` + docker start + ``` + +1. Wait a few moments and then open Rancher in a web browser. Confirm that the restore succeeded and that your data is restored. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md new file mode 100644 index 00000000000..313812b6129 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md @@ -0,0 +1,133 @@ +--- +title: Restoring a Cluster from Backup +--- + + + + + +Etcd backup and recovery for [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) can be easily performed. Snapshots of the etcd database are taken and saved either locally onto the etcd nodes or to a S3 compatible target. The advantages of configuring S3 is that if all etcd nodes are lost, your snapshot is saved remotely and can be used to restore the cluster. + +Rancher recommends enabling the [ability to set up recurring snapshots of etcd](back-up-rancher-launched-kubernetes-clusters.md#configuring-recurring-snapshots), but [one-time snapshots](back-up-rancher-launched-kubernetes-clusters.md#one-time-snapshots) can easily be taken as well. Rancher allows restore from [saved snapshots](#restoring-a-cluster-from-a-snapshot) or if you don't have any snapshots, you can still [restore etcd](#recovering-etcd-without-a-snapshot-rke). + +Clusters can also be restored to a prior Kubernetes version and cluster configuration. + +## Viewing Available Snapshots + +The list of all available snapshots for the cluster is available. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to view the snapshots and click the name of the cluster. +1. Click the **Snapshots** tab. The listed snapshots include a timestamp of when they were created. + +## Restoring a Cluster from a Snapshot + +If your Kubernetes cluster is broken, you can restore the cluster from a snapshot. + +Snapshots are composed of the cluster data in etcd, the Kubernetes version, and the cluster configuration in the `cluster.yml.` These components allow you to select from the following options when restoring a cluster from a snapshot: + +- **Restore just the etcd contents:** This restore is similar to restoring to snapshots in Rancher before v2.4.0. +- **Restore etcd and Kubernetes version:** This option should be used if a Kubernetes upgrade is the reason that your cluster is failing, and you haven't made any cluster configuration changes. +- **Restore etcd, Kubernetes versions and cluster configuration:** This option should be used if you changed both the Kubernetes version and cluster configuration when upgrading. + +When rolling back to a prior Kubernetes version, the [upgrade strategy options](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md#configuring-the-upgrade-strategy) are ignored. Worker nodes are not cordoned or drained before being reverted to the older Kubernetes version, so that an unhealthy cluster can be more quickly restored to a healthy state. + +:::note Prerequisite: + +To restore snapshots from S3, the cluster needs to be configured to [take recurring snapshots on S3.](back-up-rancher-launched-kubernetes-clusters.md#configuring-recurring-snapshots) + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to view the snapshots and click the name of the cluster. +1. Click the **Snapshots** tab to view the list of saved snapshots. +1. Go to the snapshot you want to restore and click **⋮ > Restore**. +1. Select a **Restore Type**. +1. Click **Restore**. + +**Result:** The cluster will go into `updating` state and the process of restoring the `etcd` nodes from the snapshot will start. The cluster is restored when it returns to an `active` state. + +## Restoring a Cluster From a Snapshot When the controlplane/etcd Are Completely Unavailable + +In a disaster recovery scenario, the control plane and etcd nodes managed by Rancher in a downstream cluster may no longer be available or functioning. The cluster can be rebuilt by adding control plane and etcd nodes again, followed by restoring from an available snapshot. + + + + +Follow the procedure described in the [SUSE Knowledgebase](https://www.suse.com/support/kb/doc/?id=000020695). + + + + +If you have a complete cluster failure, you must remove all etcd nodes/machines from your cluster before you can add a "new" etcd node for restore. + +:::note + +Due to a [known issue](https://github.com/rancher/rancher/issues/41080), this procedure requires Rancher v2.7.5 or newer. + +::: + +:::note + +If you are using [local snapshots](./back-up-rancher-launched-kubernetes-clusters.md#local-backup-target), it is **VERY** important that you ensure you back up the corresponding snapshot you want to restore from the `/var/lib/rancher//server/db/snapshots/` folder on the etcd node you are going to be removing. You can copy the snapshot onto your new node in the `/var/lib/rancher//server/db/snapshots/` folder. Furthermore, if using local snapshots and restoring to a new node, restoration cannot be done via the UI as of now. + +::: + +1. Remove all etcd nodes from your cluster. + + 1. In the upper left corner, click **☰ > Cluster Management**. + 1. In the **Clusters** page, go to the cluster where you want to remove nodes. + 1. In the **Machines** tab, click **⋮ > Delete** on each node you want to delete. Initially, you will see the nodes hang in a `deleting` state, but once all etcd nodes are deleting, they will be removed together. This is due to the fact that Rancher sees all etcd nodes deleting and proceeds to "short circuit" the etcd safe-removal logic. + +1. After all etcd nodes are removed, add a new etcd node that you are planning to restore from. + + - For custom clusters, go to the **Registration** tab then copy and run the registration command on your node. If the node has previously been used in a cluster, [clean the node](../manage-clusters/clean-cluster-nodes.md#cleaning-up-nodes) first. + - For node driver clusters, a new node is provisioned automatically. + + At this point, Rancher will indicate that restoration from etcd snapshot is required. + +1. Restore from an etcd snapshot. + + - For S3 snapshots, restore using the UI. + 1. Click the **Snapshots** tab to view the list of saved snapshots. + 1. Go to the snapshot you want to restore and click **⋮ > Restore**. + 1. Select a **Restore Type**. + 1. Click **Restore**. + - For local snapshots, restore using the UI is **not** available. + 1. In the upper right corner, click **⋮ > Edit YAML**. + 1. Define `spec.cluster.rkeConfig.etcdSnapshotRestore.name` as the filename of the snapshot on disk in `/var/lib/rancher//server/db/snapshots/`. + +1. After restoration is successful, you can scale your etcd nodes back up to the desired redundancy. + + + + +## Recovering etcd without a Snapshot (RKE) + +If the group of etcd nodes loses quorum, the Kubernetes cluster will report a failure because no operations, e.g. deploying workloads, can be executed in the Kubernetes cluster. The cluster should have three etcd nodes to prevent a loss of quorum. If you want to recover your set of etcd nodes, follow these instructions: + +1. Keep only one etcd node in the cluster by removing all other etcd nodes. + +2. On the single remaining etcd node, run the following command: + + ```bash + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock assaflavie/runlike etcd + ``` + + This command outputs the running command for etcd, save this command to use later. + +3. Stop the running `etcd` container and rename it to `etcd-old`. + + ```bash + docker stop etcd + docker rename etcd etcd-old + ``` + +4. Take the saved command from Step 2 and revise it: + + - If you originally had more than 1 etcd node, then you need to change `--initial-cluster` to only contain the node that remains. + - Add `--force-new-cluster` to the end of the command. + +5. Run the revised command. + +6. After the single nodes is up and running, Rancher recommends adding additional etcd nodes to your cluster. If you have a [custom cluster](../../../pages-for-subheaders/use-existing-nodes.md) and you want to reuse an old node, you are required to [clean up the nodes](../manage-clusters/clean-cluster-nodes.md) before attempting to add them back into a cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md new file mode 100644 index 00000000000..6e2179a5d5f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md @@ -0,0 +1,101 @@ +--- +title: Restoring Rancher +--- + + + + + +This page outlines how to perform a restore with Rancher. + +:::note Important: + +* Follow the instructions from this page for restoring rancher on the same cluster where it was backed up from. In order to migrate rancher to a new cluster, follow the steps to [migrate rancher.](migrate-rancher-to-new-cluster.md) +* While restoring rancher on the same setup, the operator will scale down the rancher deployment when restore starts, and it will scale back up the deployment once restore completes. So Rancher will be unavailable during the restore. +* If you need to restore Rancher to a previous version after an upgrade, see the [rollback documentation.](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md) + +::: + +## Additional Steps for Rollbacks with Rancher v2.6.4+ + +Rancher v2.6.4 upgrades the cluster-api module from v0.4.4 to v1.0.2. Version v1.0.2 of the cluster-api, in turn, upgrades the Cluster API's Custom Resource Definitions (CRDs) from `cluster.x-k8s.io/v1alpha4` to `cluster.x-k8s.io/v1beta1`. The CRDs upgrade to v1beta1 causes rollbacks to fail when you attempt to move from Rancher v2.6.4 to any previous version of Rancher v2.6.x. This is because CRDs that use the older apiVersion (v1alpha4) are incompatible with v1beta1. + +To avoid rollback failure, the following Rancher scripts should be run **before** you attempt a restore operation or rollback: + +* `verify.sh`: Checks for any Rancher-related resources in the cluster. +* `cleanup.sh`: Cleans up the cluster. + +See the [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup) for more details and source code. + +:::caution + + There will be downtime while `cleanup.sh` runs, since the script deletes resources created by Rancher. + +::: + +### Rolling back from v2.6.4+ to lower versions of v2.6.x + +1. Follow these [instructions](https://github.com/rancher/rancher-cleanup/blob/main/README.md) to run the scripts. +1. Follow these [instructions](../backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) to install the rancher-backup Helm chart on the existing cluster and restore the previous state. + 1. Omit Step 3. + 1. When you reach Step 4, install the Rancher v2.6.x version on the local cluster you intend to roll back to. + +### Create the Restore Custom Resource + +A restore is performed by creating a Restore custom resource. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. The `local` cluster runs the Rancher server. +1. In the left navigation bar, click **Rancher Backups > Restores**. +1. Click **Create**. +1. Create the Restore with the form, or with YAML. For creating the Restore resource using form, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) +1. For using the YAML editor, we can click **Create > Create from YAML**. Enter the Restore YAML. + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + For help configuring the Restore, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) + +1. Click **Create**. + +**Result:** The rancher-operator scales down the rancher deployment during restore, and scales it back up once the restore completes. The resources are restored in this order: + +1. Custom Resource Definitions (CRDs) +2. Cluster-scoped resources +3. Namespaced resources + +### Logs + +To check how the restore is progressing, you can check the logs of the operator. Run this command to follow the logs: + +``` +kubectl logs -n cattle-resources-system -l app.kubernetes.io/name=rancher-backup -f +``` + +### Cleanup + +If you created the restore resource with kubectl, remove the resource to prevent a naming conflict with future restores. + +### Known Issues +In some cases, after restoring the backup, Rancher logs will show errors similar to the following: +``` +2021/10/05 21:30:45 [ERROR] error syncing 'c-89d82/m-4067aa68dd78': handler rke-worker-upgrader: clusters.management.cattle.io "c-89d82" not found, requeuing +``` +This happens because one of the resources that was just restored has finalizers, but the related resources have been deleted so the handler cannot find it. + +To eliminate the errors, we need to find and delete the resource that causes the error. See more information [here](https://github.com/rancher/rancher/issues/35050#issuecomment-937968556) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md new file mode 100644 index 00000000000..8ff8e4cd5be --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md @@ -0,0 +1,71 @@ +--- +title: Continuous Delivery with Fleet +--- + + + + + +Continuous Delivery with Fleet is GitOps at scale. Fleet is designed to manage up to a million clusters. It's also lightweight enough that it works great for a [single cluster](https://fleet.rancher.io/installation#default-install) too, but it really shines when you get to a [large scale.](https://fleet.rancher.io/installation#configuration-for-multi-cluster) By large scale we mean either a lot of clusters, a lot of deployments, or a lot of teams in a single organization. + +Fleet is a separate project from Rancher, and can be installed on any Kubernetes cluster with Helm. + + +## Architecture + +For information about how Fleet works, see [this page.](../../../integrations-in-rancher/fleet-gitops-at-scale/architecture.md) + +## Accessing Fleet in the Rancher UI + +Fleet comes preinstalled in Rancher and is managed by the **Continous Delivery** option in the Rancher UI. For additional information on Continuous Delivery and other Fleet troubleshooting tips, refer [here](https://fleet.rancher.io/troubleshooting). + +Users can leverage continuous delivery to deploy their applications to the Kubernetes clusters in the git repository without any manual operation by following **gitops** practice. + +Follow the steps below to access Continuous Delivery in the Rancher UI: + +1. Click **☰ > Continuous Delivery**. + +1. Select your namespace at the top of the menu, noting the following: + - By default,`fleet-default` is selected which includes all downstream clusters that are registered through Rancher. + - You may switch to `fleet-local`, which only contains the `local` cluster, or you may create your own workspace to which you may assign and move clusters. + - You can then manage clusters by clicking on **Clusters** on the left navigation bar. + +1. Click on **Gitrepos** on the left navigation bar to deploy the gitrepo into your clusters in the current workspace. + +1. Select your [git repository](https://fleet.rancher.io/gitrepo-add) and [target clusters/cluster group](https://fleet.rancher.io/gitrepo-targets). You can also create the cluster group in the UI by clicking on **Cluster Groups** from the left navigation bar. + +1. Once the gitrepo is deployed, you can monitor the application through the Rancher UI. + +## Windows Support + +For details on support for clusters with Windows nodes, see [this page.](../../../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md) + + +## GitHub Repository + +The Fleet Helm charts are available [here.](https://github.com/rancher/fleet/releases/latest) + + +## Using Fleet Behind a Proxy + +For details on using Fleet behind a proxy, see [this page.](../../../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md) + +## Helm Chart Dependencies + +In order for Helm charts with dependencies to deploy successfully, you must run a manual command (as listed below), as it is up to the user to fulfill the dependency list. If you do not do this and proceed to clone your repository and run `helm install`, your installation will fail because the dependencies will be missing. + +The Helm chart in the git repository must include its dependencies in the charts subdirectory. You must either manually run `helm dependencies update $chart` OR run `helm dependencies build $chart` locally, then commit the complete charts directory to your git repository. Note that you will update your commands with the applicable parameters. + +## Troubleshooting + +--- +* **Known Issue:** clientSecretName and helmSecretName secrets for Fleet gitrepos are not included in the backup nor restore created by the [backup-restore-operator](../backup-restore-and-disaster-recovery/back-up-rancher.md#1-install-the-rancher-backup-operator). We will update the community once a permanent solution is in place. + +* **Temporary Workaround:**
+By default, user-defined secrets are not backed up in Fleet. It is necessary to recreate secrets if performing a disaster recovery restore or migration of Rancher into a fresh cluster. To modify resourceSet to include extra resources you want to backup, refer to docs [here](https://github.com/rancher/backup-restore-operator#user-flow). + +--- + +## Documentation + +The Fleet documentation is at [https://fleet.rancher.io/.](https://fleet.rancher.io/) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md new file mode 100644 index 00000000000..64c852c3629 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md @@ -0,0 +1,179 @@ +--- +title: Multi-cluster Apps +--- + + + + + +Typically, most applications are deployed on a single Kubernetes cluster, but there will be times you might want to deploy multiple copies of the same application across different clusters and/or projects. In Rancher, a _multi-cluster application_, is an application deployed using a Helm chart across multiple clusters. With the ability to deploy the same application across multiple clusters, it avoids the repetition of the same action on each cluster, which could introduce user error during application configuration. With multi-cluster applications, you can customize to have the same configuration across all projects/clusters as well as have the ability to change the configuration based on your target project. Since multi-cluster application is considered a single application, it's easy to manage and maintain this application. + +Any Helm charts from a global catalog can be used to deploy and manage multi-cluster applications. + +After creating a multi-cluster application, you can program a global DNS entry to make it easier to access the application. + +## Prerequisites + +### Permissions + +To create a multi-cluster app in Rancher, you must have at least one of the following permissions: + +- A [project-member role](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) in the target cluster(s), which gives you the ability to create, read, update, and delete the workloads +- A [cluster owner role](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles) for the clusters(s) that include the target project(s) + +### Enable Legacy Features + +Because multi-cluster apps were deprecated and replaced with Fleet in Rancher v2.5, you will need to enable multi-cluster apps with a feature flag. + +1. In the upper left corner, click **☰ > Global Settings**. +1. Click **Feature Flags**. +1. Go to the `legacy` feature flag and click **Activate**. + +## Launching a Multi-Cluster App + +1. In the upper left corner, click **☰ > Multi-cluster Apps**. +1. Click **Launch**. +1. Find the application that you want to launch. +1. (Optional) Review the detailed descriptions, which are derived from the Helm chart's `README`. +1. Under **Configuration Options** enter a **Name** for the multi-cluster application. By default, this name is also used to create a Kubernetes namespace in each [target project](#targets) for the multi-cluster application. The namespace is named as `-`. +1. Select a **Template Version**. +1. Complete the [multi-cluster applications specific configuration options](#multi-cluster-app-configuration-options) as well as the [application configuration options](#application-configuration-options). +1. Select the **Members** who can [interact with the multi-cluster application](#members). +1. Add any [custom application configuration answers](#overriding-application-configuration-options-for-specific-projects) that would change the configuration for specific project(s) from the default application configuration answers. +1. Review the files in the **Preview** section. When you're satisfied, click **Launch**. + +**Result**: Your application is deployed to your chosen namespace. You can view the application status from the project's: + +## Multi-cluster App Configuration Options + +Rancher has divided the configuration option for the multi-cluster application into several sections. + +### Targets + +In the **Targets** section, select the projects that you want the application to be deployed in. The list of projects is based on what projects you have access to. For each project that you select, it will be added to the list, which shows the cluster name and project name that were selected. To remove a target project, click on **-**. + +### Upgrades + +In the **Upgrades** section, select the upgrade strategy to use, when you decide to upgrade your application. + +* **Rolling Update (batched):** When selecting this upgrade strategy, the number of applications upgraded at a time is based on the selected **Batch size** and the **Interval** specifies how many seconds to wait before starting the next batch of updates. + +* **Upgrade all apps simultaneously:** When selecting this upgrade strategy, all applications across all projects will be upgraded at the same time. + +### Roles + +In the **Roles** section, you define the role of the multi-cluster application. Typically, when a user [launches catalog applications](../../../pages-for-subheaders/helm-charts-in-rancher.md), that specific user's permissions are used for creation of all workloads/resources that is required by the app. + +For multi-cluster applications, the application is deployed by a _system user_ and is assigned as the creator of all underlying resources. A _system user_ is used instead of the actual user due to the fact that the actual user could be removed from one of the target projects. If the actual user was removed from one of the projects, then that user would no longer be able to manage the application for the other projects. + +Rancher will let you select from two options for Roles, **Project** and **Cluster**. Rancher will allow creation using any of these roles based on the user's permissions. + +- **Project** - This is the equivalent of a [project member](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles). If you select this role, Rancher will check that in all the target projects, the user has minimally the [project member](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) role. While the user might not be explicitly granted the _project member_ role, if the user is an [administrator](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md), a [cluster owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles), or a [project owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles), then the user is considered to have the appropriate level of permissions. + +- **Cluster** - This is the equivalent of a [cluster owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles). If you select this role, Rancher will check that in all the target projects, the user has minimally the [cluster owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) role. While the user might not be explicitly granted the _cluster owner_ role, if the user is an [administrator](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md), then the user is considered to have the appropriate level of permissions. + +When launching the application, Rancher will confirm if you have these permissions in the target projects before launching the application. + +:::note + +There are some applications like _Grafana_ or _Datadog_ that require access to specific cluster-scoped resources. These applications will require the _Cluster_ role. If you find out later that the application requires cluster roles, the multi-cluster application can be upgraded to update the roles. + +::: + +## Application Configuration Options + +For each Helm chart, there are a list of desired answers that must be entered in order to successfully deploy the chart. When entering answers, you must format them using the syntax rules found in [Using Helm: The format and limitations of –set](https://helm.sh/docs/intro/using_helm/#the-format-and-limitations-of---set), as Rancher passes them as `--set` flags to Helm. + +:::note Example + +When entering an answer that includes two values separated by a comma (i.e. `abc, bcd`), it is required to wrap the values with double quotes (i.e., ``"abc, bcd"``). + +::: + +### Using a questions.yml file + +If the Helm chart that you are deploying contains a `questions.yml` file, Rancher's UI will translate this file to display an easy to use UI to collect the answers for the questions. + +### Key Value Pairs for Native Helm Charts + +For native Helm charts (i.e., charts from the **Helm Stable** or **Helm Incubator** catalogs or a custom Helm chart repository, answers are provided as key value pairs in the **Answers** section. These answers are used to override the default values. + +### Members + +By default, multi-cluster applications can only be managed by the user who created it. In the **Members** section, other users can be added so that they can also help manage or view the multi-cluster application. + +1. Find the user that you want to add by typing in the member's name in the **Member** search box. + +2. Select the **Access Type** for that member. There are three access types for a multi-cluster project, but due to how the permissions of a multi-cluster application are launched, please read carefully to understand what these access types mean. + + - **Owner**: This access type can manage any configuration part of the multi-cluster application including the template version, the [multi-cluster applications specific configuration options](#Multi-cluster App Configuration Options), the [application specific configuration options](#application-configuration-options), the members who can interact with the multi-cluster application and the [custom application configuration answers](#overriding-application-configuration-options-for-specific-projects). Since a multi-cluster application is created with a different set of permissions from the user, any _owner_ of the multi-cluster application can manage/remove applications in [target projects](#targets) without explicitly having access to these project(s). Only trusted users should be provided with this access type. + + - **Member**: This access type can only modify the template version, the [application specific configuration options](#application-configuration-options) and the [custom application configuration answers](#overriding-application-configuration-options-for-specific-projects). Since a multi-cluster application is created with a different set of permissions from the user, any _member_ of the multi-cluster application can modify the application without explicitly having access to these project(s). Only trusted users should be provided with this access type. + + - **Read-only**: This access type cannot modify any configuration option for the multi-cluster application. Users can only view these applications. + + :::caution + + Please ensure only trusted users are given _Owner_ or _Member_ access as they will automatically be able to manage applications created for this multi-cluster application in target projects they might not have direct access to. + + ::: + +### Overriding Application Configuration Options for Specific Projects + +The ability to use the same configuration to deploy the same application across multiple clusters/projects is one of the main benefits of multi-cluster applications. There might be a specific project that requires a slightly different configuration option, but you want to manage that application with all the other matching applications. Instead of creating a brand new application, you can override specific [application specific configuration options](#application-configuration-options) for specific projects. + +1. In the **Answer Overrides** section, click **Add Override**. + +2. For each override, you can select the following: + + - **Scope**: Select which target projects you want to override the answer in the configuration option. + + - **Question**: Select which question you want to override. + + - **Answer**: Enter the answer that you want to be used instead. + +## Upgrading Multi-Cluster App Roles and Projects + +- **Changing Roles on an existing Multi-Cluster app** +The creator and any users added with the access-type "owner" to a multi-cluster app, can upgrade its Roles. When adding a new Role, we check if the user has that exact role in all current target projects. These checks allow the same relaxations for global admins, cluster owners and project-owners as described in the installation section for the field `Roles`. + +- **Adding/Removing target projects** +1. The creator and any users added with access-type "owner" to a multi-cluster app, can add or remove its target projects. When adding a new project, we check if the caller of this request has all Roles defined on multi-cluster app, in the new projects they want to add. The roles checks are again relaxed for global admins, cluster-owners and project-owners. +2. We do not do these membership checks when removing target projects. This is because the caller's permissions could have with respect to the target project, or the project could have been deleted and hence the caller wants to remove it from targets list. + + +## Multi-Cluster Application Management + +One of the benefits of using a multi-cluster application as opposed to multiple individual applications of the same type, is the ease of management. Multi-cluster applications can be cloned, upgraded or rolled back. + +:::note Prerequisite: + +The `legacy` feature flag needs to be enabled. + +::: + +1. In the upper left corner, click **☰ > Multi-cluster Apps**. + +2. Choose the multi-cluster application you want to take one of these actions on and click the **⋮**. Select one of the following options: + + * **Clone**: Creates another multi-cluster application with the same configuration. By using this option, you can easily duplicate a multi-cluster application. + * **Upgrade**: Upgrade your multi-cluster application to change some part of the configuration. When performing an upgrade for multi-cluster application, the [upgrade strategy](#upgrades) can be modified if you have the correct [access type](#members). + * **Rollback**: Rollback your application to a specific version. If after an upgrade, there are issues for your multi-cluster application for one or more of your [targets](#targets), Rancher has stored up to 10 versions of the multi-cluster application. Rolling back a multi-cluster application reverts the application for **all** target clusters and projects, not just the targets(s) affected by the upgrade issue. + +## Deleting a Multi-Cluster Application + +:::note Prerequisite: + +The `legacy` feature flag needs to be enabled. + +::: + +1. In the upper left corner, click **☰ > Multi-cluster Apps**. + +2. Choose the multi-cluster application you want to delete and click the **⋮ > Delete**. When deleting the multi-cluster application, all applications and namespaces are deleted in all of the target projects. + + :::note + + The applications in the target projects, that are created for a multi-cluster application, cannot be deleted individually. The applications can only be deleted when the multi-cluster application is deleted. + + ::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md new file mode 100644 index 00000000000..300e152bb22 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -0,0 +1,133 @@ +--- +title: Creating Apps +--- + + + + + +:::tip + +For a complete walkthrough of developing charts, see the [Chart Template Developer's Guide](https://helm.sh/docs/chart_template_guide/) in the official Helm documentation. + +::: + +## Chart Types + +Rancher supports two different types of charts: Helm charts and Rancher charts. + +### Helm Charts + +Native Helm charts include an application along with other software required to run it. When deploying native Helm charts, you' can provide the chart's parameter values in a YAML editor. + +### Rancher Charts + +Rancher charts are native helm charts with two files that enhance user experience: `app-readme.md` and `questions.yaml`. Read more about them in [Additional Files for Rancher Charts.](#additional-files-for-rancher-charts) + +Rancher charts add simplified chart descriptions and configuration forms to make the application deployment easy. Rancher users do not need to read through the entire list of Helm variables to understand how to launch an application. + +## Chart Directory Structure + +You can provide Helm Charts in a standard, HTTP based Helm Repository. For more information see the [Chart Repository Guide](https://helm.sh/docs/topics/chart_repository) in the official Helm documentation. + +Alternatively you can organize your charts in a Git Repository and directly add this to the App Marketplace. + +The following table demonstrates the directory structure for a Git repository. The `charts` directory is the top level directory under the repository base. Adding the repository to Rancher will expose all charts contained within it. The `questions.yaml`, `README.md`, and `requirements.yml` files are specific to Rancher charts, but are optional for chart customization. + +``` +/ + │ + ├── charts/ + │ ├── / # This directory name will be surfaced in the Rancher UI as the chart name + │ │ ├── / # Each directory at this level provides different app versions that will be selectable within the chart in the Rancher UI + │ │ │ ├── Chart.yaml # Required Helm chart information file. + │ │ │ ├── questions.yaml # Form questions displayed within the Rancher UI. Questions display in Configuration Options.* + │ │ │ ├── README.md # Optional: Helm Readme file displayed within Rancher UI. This text displays in Detailed Descriptions. + │ │ │ ├── requirements.yml # Optional: YAML file listing dependencies for the chart. + │ │ │ ├── values.yml # Default configuration values for the chart. + │ │ │ ├── templates/ # Directory containing templates that, when combined with values.yml, generates Kubernetes YAML. +``` + +## Additional Files for Rancher Charts + +Before you create your own custom catalog, you should have a basic understanding about how a Rancher chart differs from a native Helm chart. Rancher charts differ slightly from Helm charts in their directory structures. Rancher charts include two files that Helm charts do not. + +- `app-readme.md` + + A file that provides descriptive text in the chart's UI header. + +- `questions.yml` + + A file that contains questions for a form. These form questions simplify deployment of a chart. Without it, you must configure the deployment using a values YAML config, which is more difficult. The following images display the difference between a Rancher chart (which includes `questions.yml`) and a native Helm chart (which does not). + +
Rancher Chart with questions.yml (top) vs. Helm Chart without (bottom)
+ + ![questions.yml](/img/rancher-app-2.6.png) + ![values.yaml](/img/helm-app-2.6.png) + + +### Chart.yaml annotations + +Rancher supports additional annotations that you can add to the `Chart.yaml` file. These annotations allow you to define application dependencies or configure additional UI defaults: + +| Annotation | Description | Example | +| --------------------------------- | ----------- | ------- | +| catalog.cattle.io/auto-install | If set, will install the specified chart in the specified version before installing this chart | other-chart-name=1.0.0 | +| catalog.cattle.io/display-name | A display name that should be displayed in the App Marketplace instead of the chart name | Display Name of Chart | +| catalog.cattle.io/namespace | A fixed namespace where the chart should be deployed in. If set, this can't be changed by the user | fixed-namespace | +| catalog.cattle.io/release-name | A fixed release name for the Helm installation. If set, this can't be changed by the user | fixed-release-name | +| catalog.cattle.io/requests-cpu | Total amount of CPU that should be unreserverd in the cluster. If less CPU is available, a warning will be shown | 2000m | +| catalog.cattle.io/requests-memory | Total amount of memory that should be unreserverd in the cluster. If less memory is available, a warning will be shown | 2Gi | +| catalog.cattle.io/os | Restricts the OS where this chart can be installed. Possible values: `linux`, `windows`. Default: no restriction | linux | + +### questions.yml + +Inside the `questions.yml`, most of the content will be around the questions to ask the end user, but there are some additional fields that can be set in this file. + +### Min/Max Rancher versions + +For each chart, you can add the minimum and/or maximum Rancher version, which determines whether or not this chart is available to be deployed from Rancher. + +:::note + +Even though Rancher release versions are prefixed with a `v`, there is *no* prefix for the release version when using this option. + +::: + +``` +rancher_min_version: 2.3.0 +rancher_max_version: 2.3.99 +``` + +### Question Variable Reference + +This reference contains variables that you can use in `questions.yml` nested under `questions:`. + +| Variable | Type | Required | Description | +| ------------- | ------------- | --- |------------- | +| variable | string | true | Define the variable name specified in the `values.yml` file, using `foo.bar` for nested objects. | +| label | string | true | Define the UI label. | +| description | string | false | Specify the description of the variable.| +| type | string | false | Default to `string` if not specified (current supported types are string, multiline, boolean, int, enum, password, storageclass, hostname, pvc, and secret).| +| default | string | false | Specify the default value. Only used if there is no corresponding value in the `values.yml` file. | +| group | string | false | Group questions by input value. | +| options | []string | false | Specify the options when the variable type is `enum`, for example: options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer"| +| subquestions | []subquestion | false| Add an array of subquestions.| +| show_if | string | false | Show current variable if conditional variable is true. For example `show_if: "serviceType=Nodeport"` | +| show\_subquestion_if | string | false | Show subquestions if is true or equal to one of the options. for example `show_subquestion_if: "true"`| + + + +:::note + +`subquestions[]` cannot contain `subquestions` or `show_subquestions_if` keys, but all other keys in the above table are supported. + +::: diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md new file mode 100644 index 00000000000..f0157a3f6e0 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md @@ -0,0 +1,179 @@ +--- +title: Setting up Amazon ELB Network Load Balancer +--- + + + + + +This how-to guide describes how to set up a Network Load Balancer (NLB) in Amazon's EC2 service that will direct traffic to multiple instances on EC2. + +These examples show the load balancer being configured to direct traffic to three Rancher server nodes. If Rancher is installed on an RKE Kubernetes cluster, three nodes are required. If Rancher is installed on a K3s Kubernetes cluster, only two nodes are required. + +This tutorial is about one possible way to set up your load balancer, not the only way. Other types of load balancers, such as a Classic Load Balancer or Application Load Balancer, could also direct traffic to the Rancher server nodes. + +Rancher only supports using the Amazon NLB when terminating traffic in `tcp` mode for port 443 rather than `tls` mode. This is due to the fact that the NLB does not inject the correct headers into requests when terminated at the NLB. This means that if you want to use certificates managed by the Amazon Certificate Manager (ACM), you should use an ALB. + + + +## Requirements + +These instructions assume you have already created Linux instances in EC2. The load balancer will direct traffic to these nodes. + +## 1. Create Target Groups + +Begin by creating two target groups for the **TCP** protocol, one with TCP port 443 and one regarding TCP port 80 (providing redirect to TCP port 443). You'll add your Linux nodes to these groups. + +Your first NLB configuration step is to create two target groups. Technically, only port 443 is needed to access Rancher, but it's convenient to add a listener for port 80, because traffic to port 80 will be automatically redirected to port 443. + +Regardless of whether an NGINX Ingress or Traefik Ingress controller is used, the Ingress should redirect traffic from port 80 to port 443. + +1. Log into the [Amazon AWS Console](https://console.aws.amazon.com/ec2/) to get started. Make sure to select the **Region** where your EC2 instances (Linux nodes) are created. +1. Select **Services** and choose **EC2**, find the section **Load Balancing** and open **Target Groups**. +1. Click **Create target group** to create the first target group, regarding TCP port 443. + +:::note + +Health checks are handled differently based on the Ingress. For details, refer to [this section.](#health-check-paths-for-nginx-ingress-and-traefik-ingresses) + +::: + +#### Target Group (TCP port 443) + +Configure the first target group according to the table below. + +| Option | Setting | +|-------------------|-------------------| +| Target Group Name | `rancher-tcp-443` | +| Target type | `instance` | +| Protocol | `TCP` | +| Port | `443` | +| VPC | Choose your VPC | + +Health check settings: + +| Option | Setting | +|---------------------|-----------------| +| Protocol | TCP | +| Port | `override`,`80` | +| Healthy threshold | `3` | +| Unhealthy threshold | `3` | +| Timeout | `6 seconds` | +| Interval | `10 seconds` | + +Click **Create target group** to create the second target group, regarding TCP port 80. + +#### Target Group (TCP port 80) + +Configure the second target group according to the table below. + +| Option | Setting | +|-------------------|------------------| +| Target Group Name | `rancher-tcp-80` | +| Target type | `instance` | +| Protocol | `TCP` | +| Port | `80` | +| VPC | Choose your VPC | + + +Health check settings: + +| Option |Setting | +|---------------------|----------------| +| Protocol | TCP | +| Port | `traffic port` | +| Healthy threshold | `3` | +| Unhealthy threshold | `3` | +| Timeout | `6 seconds` | +| Interval | `10 seconds` | + +## 2. Register Targets + +Next, add your Linux nodes to both target groups. + +Select the target group named **rancher-tcp-443**, click the tab **Targets** and choose **Edit**. + +![](/img/ha/nlb/edit-targetgroup-443.png) + +Select the instances (Linux nodes) you want to add, and click **Add to registered**. + +*** +**Screenshot Add targets to target group TCP port 443**
+ +![](/img/ha/nlb/add-targets-targetgroup-443.png) + +*** +**Screenshot Added targets to target group TCP port 443**
+ +![](/img/ha/nlb/added-targets-targetgroup-443.png) + +When the instances are added, click **Save** on the bottom right of the screen. + +Repeat those steps, replacing **rancher-tcp-443** with **rancher-tcp-80**. The same instances need to be added as targets to this target group. + +## 3. Create Your NLB + +Use Amazon's Wizard to create a Network Load Balancer. As part of this process, you'll add the target groups you created in [1. Create Target Groups](#1-create-target-groups). + +1. From your web browser, navigate to the [Amazon EC2 Console](https://console.aws.amazon.com/ec2/). + +2. From the navigation pane, choose **LOAD BALANCING** > **Load Balancers**. + +3. Click **Create Load Balancer**. + +4. Choose **Network Load Balancer** and click **Create**. Then complete each form. + +- [Step 1: Configure Load Balancer](#step-1-configure-load-balancer) +- [Step 2: Configure Routing](#step-2-configure-routing) +- [Step 3: Register Targets](#step-3-register-targets) +- [Step 4: Review](#step-4-review) + +### Step 1: Configure Load Balancer + +Set the following fields in the form: + +- **Name:** `rancher` +- **Scheme:** `internal` or `internet-facing`. The scheme that you choose for your NLB is dependent on the configuration of your instances and VPC. If your instances do not have public IPs associated with them, or you will only be accessing Rancher internally, you should set your NLB Scheme to `internal` rather than `internet-facing`. +- **Listeners:** The Load Balancer Protocol should be `TCP` and the corresponding Load Balancer Port should be set to `443`. +- **Availability Zones:** Select Your **VPC** and **Availability Zones**. + +### Step 2: Configure Routing + +1. From the **Target Group** drop-down, choose **Existing target group**. +1. From the **Name** drop-down, choose `rancher-tcp-443`. +1. Open **Advanced health check settings**, and configure **Interval** to `10 seconds`. + +### Step 3: Register Targets + +Since you registered your targets earlier, all you have to do is click **Next: Review**. + +### Step 4: Review + +Look over the load balancer details and click **Create** when you're satisfied. + +After AWS creates the NLB, click **Close**. + +## 4. Add listener to NLB for TCP port 80 + +1. Select your newly created NLB and select the **Listeners** tab. + +2. Click **Add listener**. + +3. Use `TCP`:`80` as **Protocol** : **Port** + +4. Click **Add action** and choose **Forward to..**. + +5. From the **Forward to** drop-down, choose `rancher-tcp-80`. + +6. Click **Save** in the top right of the screen. + +## Health Check Paths for NGINX Ingress and Traefik Ingresses + +K3s and RKE Kubernetes clusters handle health checks differently because they use different Ingresses by default. + +For RKE Kubernetes clusters, NGINX Ingress is used by default, whereas for K3s Kubernetes clusters, Traefik is the default Ingress. + +- **Traefik:** The health check path is `/ping`. By default `/ping` is always matched (regardless of Host), and a response from [Traefik itself](https://docs.traefik.io/operations/ping/) is always served. +- **NGINX Ingress:** The default backend of the NGINX Ingress controller has a `/healthz` endpoint. By default `/healthz` is always matched (regardless of Host), and a response from [`ingress-nginx` itself](https://github.com/kubernetes/ingress-nginx/blob/0cbe783f43a9313c9c26136e888324b1ee91a72f/charts/ingress-nginx/values.yaml#L212) is always served. + +To simulate an accurate health check, it is a best practice to use the Host header (Rancher hostname) combined with `/ping` or `/healthz` (for K3s or for RKE clusters, respectively) wherever possible, to get a response from the Rancher Pods, not the Ingress. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md new file mode 100644 index 00000000000..1611ca978e3 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md @@ -0,0 +1,77 @@ +--- +title: Set up Infrastructure for a High Availability K3s Kubernetes Cluster +--- + + + + + +This tutorial is intended to help you provision the underlying infrastructure for a Rancher management server. + +The recommended infrastructure for the Rancher-only Kubernetes cluster differs depending on whether Rancher will be installed on a K3s Kubernetes cluster, an RKE Kubernetes cluster, or a single Docker container. + +For more information about each installation option, refer to [this page.](../../../pages-for-subheaders/installation-and-upgrade.md) + +:::note Important: + +These nodes must be in the same region. You may place these servers in separate availability zones (datacenter). + +::: + +To install the Rancher management server on a high-availability K3s cluster, we recommend setting up the following infrastructure: + +- **Two Linux nodes,** typically virtual machines, in the infrastructure provider of your choice. +- **An external database** to store the cluster data. We recommend MySQL. +- **A load balancer** to direct traffic to the two nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +### 1. Set up Linux Nodes + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up External Datastore + +The ability to run Kubernetes using a datastore other than etcd sets K3s apart from other Kubernetes distributions. This feature provides flexibility to Kubernetes operators. The available options allow you to select a datastore that best fits your use case. + +For a high-availability K3s installation, you will need to set a [MySQL](https://www.mysql.com/) external database. Rancher has been tested on K3s Kubernetes clusters using MySQL version 5.7 as the datastore. + +When you install Kubernetes using the K3s installation script, you will pass in details for K3s to connect to the database. + +For an example of one way to set up the MySQL database, refer to this [tutorial](mysql-database-in-amazon-rds.md) for setting up MySQL on Amazon's RDS service. + +For the complete list of options that are available for configuring a K3s cluster datastore, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/datastore/) + +### 3. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the K3s tool will deploy a Traefik Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the Traefik Ingress controller to listen for traffic destined for the Rancher hostname. The Traefik Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 4. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the load balancer IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md new file mode 100644 index 00000000000..ad266904aed --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md @@ -0,0 +1,68 @@ +--- +title: Set up Infrastructure for a High Availability RKE Kubernetes Cluster +--- + + + + + +This tutorial is intended to help you create a high-availability RKE cluster that can be used to install a Rancher server. + +:::note Important: + +These nodes must be in the same region. You may place these servers in separate availability zones (datacenter). + +::: + +To install the Rancher management server on a high-availability RKE cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in an infrastructure provider such as Amazon's EC2, Google Compute Engine, or vSphere. +- **A load balancer** to direct front-end traffic to the three nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +These nodes must be in the same region/data center. You may place these servers in separate availability zones. + +### Why three nodes? + +In an RKE cluster, Rancher server data is stored on etcd. This etcd database runs on all three nodes. + +The etcd database requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can suffer from [split brain](https://www.quora.com/What-is-split-brain-in-distributed-systems), requiring the cluster to be restored from backup. If one of the three etcd nodes fails, the two remaining nodes can elect a leader because they have the majority of the total number of etcd nodes. + +### 1. Set up Linux Nodes + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on any of the three nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE tool will deploy an NGINX Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the NGINX Ingress controller to listen for traffic destined for the Rancher hostname. The NGINX Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 3. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the LB IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md new file mode 100644 index 00000000000..c00f0061adb --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md @@ -0,0 +1,62 @@ +--- +title: Set up Infrastructure for a High Availability RKE2 Kubernetes Cluster +--- + + + + + +This tutorial is intended to help you provision the underlying infrastructure for a Rancher management server. + +The recommended infrastructure for the Rancher-only Kubernetes cluster differs depending on whether Rancher will be installed on a RKE2 Kubernetes cluster, an RKE Kubernetes cluster, or a single Docker container. + +:::note Important: + +These nodes must be in the same region. You may place these servers in separate availability zones (datacenter). + +::: + +To install the Rancher management server on a high-availability RKE2 cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in the infrastructure provider of your choice. +- **A load balancer** to direct traffic to the two nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +### 1. Set up Linux Nodes + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on all nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE2 tool will deploy an Nginx Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the Nginx Ingress controller to listen for traffic destined for the Rancher hostname. The Nginx Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 4. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the load balancer IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md new file mode 100644 index 00000000000..1d6646dd737 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md @@ -0,0 +1,38 @@ +--- +title: Setting up a MySQL Database in Amazon RDS +--- + + + + + +This tutorial describes how to set up a MySQL database in Amazon's Relational Database Service (RDS). + +This database can later be used as an external datastore for a high-availability K3s Kubernetes cluster. + +1. Log into the [Amazon AWS RDS Console](https://console.aws.amazon.com/rds/) to get started. Make sure to select the **Region** where your EC2 instances (Linux nodes) are created. +1. In the left panel, click **Databases**. +1. Click **Create database**. +1. In the **Engine type** section, click **MySQL**. +1. In the **Version** section, choose **MySQL 5.7.22**. +1. In **Settings** section, under **Credentials Settings,** enter a master password for the **admin** master username. Confirm the password. +1. Expand the **Additional configuration** section. In the **Initial database name** field, enter a name. The name can have only letters, numbers, and underscores. This name will be used to connect to the database. +1. Click **Create database**. + +You'll need to capture the following information about the new database so that the K3s Kubernetes cluster can connect to it. + +To see this information in the Amazon RDS console, click **Databases,** and click the name of the database that you created. + +- **Username:** Use the admin username. +- **Password:** Use the admin password. +- **Hostname:** Use the **Endpoint** as the hostname. The endpoint is available in the **Connectivity & security** section. +- **Port:** The port should be 3306 by default. You can confirm it in the **Connectivity & security** section. +- **Database name:** Confirm the name by going to the **Configuration** tab. The name is listed under **DB name**. + +This information will be used to connect to the database in the following format: + +``` +mysql://username:password@tcp(hostname:3306)/database-name +``` + +For more information on configuring the datastore for K3s, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/datastore/) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md new file mode 100644 index 00000000000..896c3686ad2 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md @@ -0,0 +1,106 @@ +--- +title: Setting up an NGINX Load Balancer +--- + + + + + +NGINX will be configured as Layer 4 load balancer (TCP) that forwards connections to one of your Rancher nodes. + +In this configuration, the load balancer is positioned in front of your nodes. The load balancer can be any host capable of running NGINX. + +:::note + +Do not use one of your Rancher nodes as the load balancer. + +::: + +> These examples show the load balancer being configured to direct traffic to three Rancher server nodes. If Rancher is installed on an RKE Kubernetes cluster, three nodes are required. If Rancher is installed on a K3s Kubernetes cluster, only two nodes are required. + +## Install NGINX + +Start by installing NGINX on the node you want to use as a load balancer. NGINX has packages available for all known operating systems. The versions tested are `1.14` and `1.15`. For help installing NGINX, refer to their [install documentation](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/). + +The `stream` module is required, which is present when using the official NGINX packages. Please refer to your OS documentation on how to install and enable the NGINX `stream` module on your operating system. + +## Create NGINX Configuration + +After installing NGINX, you need to update the NGINX configuration file, `nginx.conf`, with the IP addresses for your nodes. + +1. Copy and paste the code sample below into your favorite text editor. Save it as `nginx.conf`. + +2. From `nginx.conf`, replace both occurrences (port 80 and port 443) of ``, ``, and `` with the IPs of your nodes. + + :::note + + See [NGINX Documentation: TCP and UDP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/) for all configuration options. + + ::: + +
Example NGINX config
+ + ``` + worker_processes 4; + worker_rlimit_nofile 40000; + + events { + worker_connections 8192; + } + + stream { + upstream rancher_servers_http { + least_conn; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + } + server { + listen 80; + proxy_pass rancher_servers_http; + } + + } + + http { + + upstream rancher_servers_https { + least_conn; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + } + server { + listen 443 ssl; + proxy_pass rancher_servers_https; + ssl_certificate /path/to/tls.crt; + ssl_certificate_key /path/to/key.key; + location / { + proxy_pass https://rancher_servers_https; + proxy_set_header Host ; + proxy_ssl_server_name on; + proxy_ssl_name + } + } + } + ``` + + +3. Save `nginx.conf` to your load balancer at the following path: `/etc/nginx/nginx.conf`. + +4. Load the updates to your NGINX configuration by running the following command: + + ``` + # nginx -s reload + ``` + +## Option - Run NGINX as Docker container + +Instead of installing NGINX as a package on the operating system, you can rather run it as a Docker container. Save the edited **Example NGINX config** as `/etc/nginx.conf` and run the following command to launch the NGINX container: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/nginx.conf:/etc/nginx/nginx.conf \ + nginx:1.14 +``` diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md new file mode 100644 index 00000000000..9c7f55b9443 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md @@ -0,0 +1,78 @@ +--- +title: Setting up Nodes in Amazon EC2 +--- + + + + + +In this tutorial, you will learn one way to set up Linux nodes for the Rancher management server. These nodes will fulfill the node requirements for [OS, Docker, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +If the Rancher server will be installed on an RKE Kubernetes cluster, you should provision three instances. + +If the Rancher server will be installed on a K3s Kubernetes cluster, you only need to provision two instances. + +If the Rancher server is installed in a single Docker container, you only need one instance. + +### 1. Optional Preparation + +- **Create IAM role:** To allow Rancher to manipulate AWS resources, such as provisioning new storage or new nodes, you will need to configure Amazon as a cloud provider. There are several things you'll need to do to set up the cloud provider on EC2, but part of this process is setting up an IAM role for the Rancher server nodes. For the full details on setting up the cloud provider, refer to this [page.](../../../pages-for-subheaders/set-up-cloud-providers.md) +- **Create security group:** We also recommend setting up a security group for the Rancher nodes that complies with the [port requirements for Rancher nodes.](../../../pages-for-subheaders/installation-requirements.md#port-requirements) + +### 2. Provision Instances + +1. Log into the [Amazon AWS EC2 Console](https://console.aws.amazon.com/ec2/) to get started. Make sure to take note of the **Region** where your EC2 instances (Linux nodes) are created, because all of the infrastructure for the Rancher management server should be in the same region. +1. In the left panel, click **Instances**. +1. Click **Launch Instance**. +1. In the section called **Step 1: Choose an Amazon Machine Image (AMI),** we will use Ubuntu 18.04 as the Linux OS, using `ami-0d1cd67c26f5fca19 (64-bit x86)`. Go to the Ubuntu AMI and click **Select**. +1. In the **Step 2: Choose an Instance Type** section, select the `t2.medium` type. +1. Click **Next: Configure Instance Details**. +1. In the **Number of instances** field, enter the number of instances. A high-availability K3s cluster requires only two instances, while a high-availability RKE cluster requires three instances. +1. Optional: If you created an IAM role for Rancher to manipulate AWS resources, select the new IAM role in the **IAM role** field. +1. Click **Next: Add Storage,** **Next: Add Tags,** and **Next: Configure Security Group**. +1. In **Step 6: Configure Security Group,** select a security group that complies with the [port requirements](../../../pages-for-subheaders/installation-requirements.md#port-requirements) for Rancher nodes. +1. Click **Review and Launch**. +1. Click **Launch**. +1. Choose a new or existing key pair that you will use to connect to your instance later. If you are using an existing key pair, make sure you already have access to the private key. +1. Click **Launch Instances**. + + +**Result:** You have created Rancher nodes that satisfy the requirements for OS, hardware, and networking. + +:::note + +If the nodes are being used for an RKE Kubernetes cluster, install Docker on each node in the next step. For a K3s Kubernetes cluster, the nodes are now ready to install K3s. + +::: + +### 3. Install Docker and Create User for RKE Kubernetes Cluster Nodes + +1. From the [AWS EC2 console,](https://console.aws.amazon.com/ec2/) click **Instances** in the left panel. +1. Go to the instance that you want to install Docker on. Select the instance and click **Actions > Connect**. +1. Connect to the instance by following the instructions on the screen that appears. Copy the Public DNS of the instance. An example command to SSH into the instance is as follows: +``` +sudo ssh -i [path-to-private-key] ubuntu@[public-DNS-of-instance] +``` +1. Run the following command on the instance to install Docker with one of Rancher's installation scripts: +``` +curl https://releases.rancher.com/install-docker/18.09.sh | sh +``` +1. When you are connected to the instance, run the following command on the instance to create a user: +``` +sudo usermod -aG docker ubuntu +``` +1. Repeat these steps so that Docker is installed on each node that will eventually run the Rancher management server. + +:::tip + +To find out whether a script is available for installing a certain Docker version, refer to this [GitHub repository,](https://github.com/rancher/install-docker) which contains all of Rancher’s Docker installation scripts. + +::: + +**Result:** You have set up Rancher server nodes that fulfill all the node requirements for OS, Docker, hardware and networking. + +### Next Steps for RKE Kubernetes Cluster Nodes + +If you are going to install an RKE cluster on the new nodes, take note of the **IPv4 Public IP** and **Private IP** of each node. This information can be found on the **Description** tab for each node after it is created. The public and private IP will be used to populate the `address` and `internal_address` of each node in the RKE cluster configuration file, `rancher-cluster.yml`. + +RKE will also need access to the private key to connect to each node. Therefore, you might want to take note of the path to your private keys to connect to the nodes, which can also be included in the `rancher-cluster.yml` under the `ssh_key_path` directive for each node. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md new file mode 100644 index 00000000000..a5d69ed221d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md @@ -0,0 +1,29 @@ +--- +title: About High-availability Installations +--- + + + + + +We recommend using Helm, a Kubernetes package manager, to install Rancher on a dedicated Kubernetes cluster. This is called a high-availability Kubernetes installation because increased availability is achieved by running Rancher on multiple nodes. + +In a standard installation, Kubernetes is first installed on three nodes that are hosted in an infrastructure provider such as Amazon's EC2 or Google Compute Engine. + +Then Helm is used to install Rancher on top of the Kubernetes cluster. Helm uses Rancher's Helm chart to install a replica of Rancher on each of the three nodes in the Kubernetes cluster. We recommend using a load balancer to direct traffic to each replica of Rancher in the cluster, in order to increase Rancher's availability. + +The Rancher server data is stored on etcd. This etcd database also runs on all three nodes, and requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can fail, requiring the cluster to be restored from backup. + +For information on how Rancher works, regardless of the installation method, refer to the [architecture section.](../../../pages-for-subheaders/rancher-manager-architecture.md) + +### Recommended Architecture + +- DNS for Rancher should resolve to a layer 4 load balancer +- The Load Balancer should forward port TCP/80 and TCP/443 to all 3 nodes in the Kubernetes cluster. +- The Ingress controller will redirect HTTP to HTTPS and terminate SSL/TLS on port TCP/443. +- The Ingress controller will forward traffic to port TCP/80 on the pod in the Rancher deployment. + +
Kubernetes Rancher install with layer 4 load balancer, depicting SSL termination at ingress controllers
+ +![High-availability Kubernetes Installation of Rancher](/img/ha/rancher2ha.svg) +Kubernetes Rancher install with Layer 4 load balancer (TCP), depicting SSL termination at ingress controllers diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md new file mode 100644 index 00000000000..16bf5703c58 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md @@ -0,0 +1,140 @@ +--- +title: Setting up a High-availability K3s Kubernetes Cluster for Rancher +--- + + + + + +This section describes how to install a Kubernetes cluster according to the [best practices for the Rancher server environment.](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#environment-for-kubernetes-installations) + +For systems without direct internet access, refer to the air gap installation instructions. + +:::tip Single-node Installation Tip: + +In a single-node Kubernetes cluster, the Rancher server does not have high availability, which is important for running Rancher in production. However, installing Rancher on a single-node cluster can be useful if you want to save resources by using a single node in the short term, while preserving a high-availability migration path. + +To set up a single-node K3s cluster, run the Rancher server installation command on just one node instead of two nodes. + +In both single-node setups, Rancher can be installed with Helm on the Kubernetes cluster in the same way that it would be installed on any other cluster. + +::: + +## Prerequisites + +These instructions assume you have set up two nodes, a load balancer, a DNS record, and an external MySQL database as described in [this section.](../infrastructure-setup/ha-k3s-kubernetes-cluster.md) + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://rancher.com/support-maintenance-terms/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +## Installing Kubernetes + +### 1. Install Kubernetes and Set up the K3s Server + +When running the command to start the K3s Kubernetes API server, you will pass in an option to use the external datastore that you set up earlier. + +1. Connect to one of the Linux nodes that you have prepared to run the Rancher server. +1. On the Linux node, run this command to start the K3s server and connect it to the external datastore: + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" + ``` + + Where `` is the connection URI for your datastore. For example, `mysql://username:password@tcp(hostname:3306)/database-name` if you're using MySQL. Valid datastores include etcd, MySQL, PostgreSQL, or SQLite (default). + + :::note + + The datastore endpoint can also be passed in using the environment variable `$K3S_DATASTORE_ENDPOINT`. + + ::: + +1. Get main server node token: + ``` + cat /var/lib/rancher/k3s/server/token + ``` + +1. Run command on your second K3s server node: + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" \ + --token "" + ``` + +### 2. Confirm that K3s is Running + +To confirm that K3s has been set up successfully, run the following command on either of the K3s server nodes: +``` +sudo k3s kubectl get nodes +``` + +Then you should see two nodes with the master role: +``` +ubuntu@ip-172-31-60-194:~$ sudo k3s kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-172-31-60-194 Ready master 44m v1.17.2+k3s1 +ip-172-31-63-88 Ready master 6m8s v1.17.2+k3s1 +``` + +Then test the health of the cluster pods: +``` +sudo k3s kubectl get pods --all-namespaces +``` + +**Result:** You have successfully set up a K3s Kubernetes cluster. + +### 3. Save and Start Using the kubeconfig File + +When you installed K3s on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/k3s/k3s.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/k3s/k3s.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your load balancer, referring to port 6443. (The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443.) Here is an example `k3s.yaml`: + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your K3s cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/cluster-access/) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### 4. Check the Health of Your Cluster Pods + +Now that you have set up the `kubeconfig` file, you can use `kubectl` to access the cluster from your local machine. + +Check that all the required pods and containers are healthy are ready to continue: + +``` +ubuntu@ip-172-31-60-194:~$ sudo kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system metrics-server-6d684c7b5-bw59k 1/1 Running 0 8d +kube-system local-path-provisioner-58fb86bdfd-fmkvd 1/1 Running 0 8d +kube-system coredns-d798c9dd-ljjnf 1/1 Running 0 8d +``` + +**Result:** You have confirmed that you can access the cluster with `kubectl` and the K3s cluster is running successfully. Now the Rancher management server can be installed on the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md new file mode 100644 index 00000000000..1d1c0682ec1 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md @@ -0,0 +1,196 @@ +--- +title: Setting up a High-availability RKE Kubernetes Cluster +--- + + + + + +This section describes how to install a Kubernetes cluster. This cluster should be dedicated to run only the Rancher server. + +:::note + +Rancher can run on any Kubernetes cluster, included hosted Kubernetes solutions such as Amazon EKS. The below instructions represent only one possible way to install Kubernetes. + +::: + +For systems without direct internet access, refer to [Air Gap: Kubernetes install.](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) + +:::tip Single-node Installation Tip: + +In a single-node Kubernetes cluster, the Rancher server does not have high availability, which is important for running Rancher in production. However, installing Rancher on a single-node cluster can be useful if you want to save resources by using a single node in the short term, while preserving a high-availability migration path. + +To set up a single-node RKE cluster, configure only one node in the `cluster.yml` . The single node should have all three roles: `etcd`, `controlplane`, and `worker`. + +In both single-node setups, Rancher can be installed with Helm on the Kubernetes cluster in the same way that it would be installed on any other cluster. + +::: + +## Installing Kubernetes + +### Required CLI Tools + +Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. + +Also install [RKE,](https://rancher.com/docs/rke/latest/en/installation/) the Rancher Kubernetes Engine, a Kubernetes distribution and command-line tool. + +### 1. Create the cluster configuration file + +In this section, you will create a Kubernetes cluster configuration file called `rancher-cluster.yml`. In a later step, when you set up the cluster with an RKE command, it will use this file to install Kubernetes on your nodes. + +Using the sample below as a guide, create the `rancher-cluster.yml` file. Replace the IP addresses in the `nodes` list with the IP address or DNS names of the 3 nodes you created. + +If your node has public and internal addresses, it is recommended to set the `internal_address:` so Kubernetes will use it for intra-cluster communication. Some services like AWS EC2 require setting the `internal_address:` if you want to use self-referencing security groups or firewalls. + +RKE will need to connect to each node over SSH, and it will look for a private key in the default location of `~/.ssh/id_rsa`. If your private key for a certain node is in a different location than the default, you will also need to configure the `ssh_key_path` option for that node. + +When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +```yaml +nodes: + - address: 165.227.114.63 + internal_address: 172.16.22.12 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.116.167 + internal_address: 172.16.32.37 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.127.226 + internal_address: 172.16.42.73 + user: ubuntu + role: [controlplane, worker, etcd] + +services: + etcd: + snapshot: true + creation: 6h + retention: 24h + +# Required for external TLS termination with +# ingress-nginx v0.22+ +ingress: + provider: nginx + options: + use-forwarded-headers: "true" + +kubernetes_version: v1.25.6-rancher4-1 +``` + +
Common RKE Nodes Options
+ +| Option | Required | Description | +| ------------------ | -------- | -------------------------------------------------------------------------------------- | +| `address` | yes | The public DNS or IP address | +| `user` | yes | A user that can run docker commands | +| `role` | yes | List of Kubernetes roles assigned to the node | +| `internal_address` | no | The private DNS or IP address for internal cluster traffic | +| `ssh_key_path` | no | Path to SSH private key used to authenticate to the node (defaults to `~/.ssh/id_rsa`) | + +:::note Advanced Configurations: + +RKE has many configuration options for customizing the install to suit your specific environment. + +Please see the [RKE Documentation](https://rancher.com/docs/rke/latest/en/config-options/) for the full list of options and capabilities. + +For tuning your etcd cluster for larger Rancher installations, see the [etcd settings guide](../../advanced-user-guides/tune-etcd-for-large-installs.md). + +For more information regarding Dockershim support, refer to [this page](../../../getting-started/installation-and-upgrade/installation-requirements/dockershim.md) + +::: + +### 2. Run RKE + +``` +rke up --config ./rancher-cluster.yml +``` + +When finished, it should end with the line: `Finished building Kubernetes cluster successfully`. + +### 3. Test Your Cluster + +This section describes how to set up your workspace so that you can interact with this cluster using the `kubectl` command-line tool. + +Assuming you have installed `kubectl`, you need to place the `kubeconfig` file in a location where `kubectl` can reach it. The `kubeconfig` file contains the credentials necessary to access your cluster with `kubectl`. + +When you ran `rke up`, RKE should have created a `kubeconfig` file named `kube_config_cluster.yml`. This file has the credentials for `kubectl` and `helm`. + +:::note + +If you have used a different file name from `rancher-cluster.yml`, then the kube config file will be named `kube_config_.yml`. + +::: + +Move this file to `$HOME/.kube/config`, or if you are working with multiple Kubernetes clusters, set the `KUBECONFIG` environmental variable to the path of `kube_config_cluster.yml`: + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +Test your connectivity with `kubectl` and see if all your nodes are in `Ready` state: + +``` +kubectl get nodes + +NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.13.5 +``` + +### 4. Check the Health of Your Cluster Pods + +Check that all the required pods and containers are healthy are ready to continue. + +- Pods are in `Running` or `Completed` state. +- `READY` column shows all the containers are running (i.e. `3/3`) for pods with `STATUS` `Running` +- Pods with `STATUS` `Completed` are run-once Jobs. For these pods `READY` should be `0/1`. + +``` +kubectl get pods --all-namespaces + +NAMESPACE NAME READY STATUS RESTARTS AGE +ingress-nginx nginx-ingress-controller-tnsn4 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-tw2ht 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-v874b 1/1 Running 0 30s +kube-system canal-jp4hz 3/3 Running 0 30s +kube-system canal-z2hg8 3/3 Running 0 30s +kube-system canal-z6kpw 3/3 Running 0 30s +kube-system kube-dns-7588d5b5f5-sf4vh 3/3 Running 0 30s +kube-system kube-dns-autoscaler-5db9bbb766-jz2k6 1/1 Running 0 30s +kube-system metrics-server-97bc649d5-4rl2q 1/1 Running 0 30s +kube-system rke-ingress-controller-deploy-job-bhzgm 0/1 Completed 0 30s +kube-system rke-kubedns-addon-deploy-job-gl7t4 0/1 Completed 0 30s +kube-system rke-metrics-addon-deploy-job-7ljkc 0/1 Completed 0 30s +kube-system rke-network-plugin-deploy-job-6pbgj 0/1 Completed 0 30s +``` + +This confirms that you have successfully installed a Kubernetes cluster that the Rancher server will run on. + +### 5. Save Your Files + +:::note Important: + +The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster. + +::: + +Save a copy of the following files in a secure location: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_cluster.yml`: The [Kubeconfig file](https://rancher.com/docs/rke/latest/en/kubeconfig/) for the cluster, this file contains credentials for full access to the cluster. +- `rancher-cluster.rkestate`: The [Kubernetes Cluster State file](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state), this file contains credentials for full access to the cluster.

_The Kubernetes Cluster State file is only created when using RKE v0.2.0 or higher._ + +:::note + +The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +::: + +### Issues or errors? + +See the [Troubleshooting](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) page. + + +### [Next: Install Rancher](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md new file mode 100644 index 00000000000..7a743f731a2 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md @@ -0,0 +1,169 @@ +--- +title: Setting up a High-availability RKE2 Kubernetes Cluster for Rancher +--- + + + + + +This section describes how to install a Kubernetes cluster according to the [best practices for the Rancher server environment.](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#environment-for-kubernetes-installations) + +## Prerequisites + +These instructions assume you have set up three nodes, a load balancer, and a DNS record, as described in [this section.](../infrastructure-setup/ha-rke2-kubernetes-cluster.md) + +Note that in order for RKE2 to work correctly with the load balancer, you need to set up two listeners: one for the supervisor on port 9345, and one for the Kubernetes API on port 6443. + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) To specify the RKE2 version, use the INSTALL_RKE2_VERSION environment variable when running the RKE2 installation script. + +## Installing Kubernetes + +### 1. Install Kubernetes and Set up the RKE2 Server + +RKE2 server runs with embedded etcd so you will not need to set up an external datastore to run in HA mode. + +On the first node, you should set up the configuration file with your own pre-shared secret as the token. The token argument can be set on startup. + +If you do not specify a pre-shared secret, RKE2 will generate one and place it at /var/lib/rancher/rke2/server/node-token. + +To avoid certificate errors with the fixed registration address, you should launch the server with the tls-san parameter set. This option adds an additional hostname or IP as a Subject Alternative Name in the server's TLS cert, and it can be specified as a list if you would like to access via both the IP and the hostname. + +First, you must create the directory where the RKE2 config file is going to be placed: + +``` +mkdir -p /etc/rancher/rke2/ +``` + +Next, create the RKE2 config file at `/etc/rancher/rke2/config.yaml` using the following example: + +``` +token: my-shared-secret +tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +``` +After that, you need to run the install command and enable and start rke2: + +``` +curl -sfL https://get.rke2.io | sh - +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` +1. To join the rest of the nodes, you need to configure each additional node with the same shared token or the one generated automatically. Here is an example of the configuration file: + + token: my-shared-secret + server: https://:9345 + tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +After that, you need to run the installer and enable, then start, rke2: + + curl -sfL https://get.rke2.io | sh - + systemctl enable rke2-server.service + systemctl start rke2-server.service + + +1. Repeat the same command on your third RKE2 server node. + +### 2. Confirm that RKE2 is Running + +Once you've launched the rke2 server process on all server nodes, ensure that the cluster has come up properly with + +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get nodes +You should see your server nodes in the Ready state. +``` + +Then test the health of the cluster pods: +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get pods --all-namespaces +``` + +**Result:** You have successfully set up a RKE2 Kubernetes cluster. + +### 3. Save and Start Using the kubeconfig File + +When you installed RKE2 on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/rke2/rke2.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/rke2/rke2.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your control-plane load balancer, on port 6443. (The RKE2 Kubernetes API Server uses port 6443, while the Rancher server will be served via the NGINX Ingress on ports 80 and 443.) Here is an example `rke2.yaml`: + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your RKE2 cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [RKE2 documentation](https://docs.rke2.io/cluster_access) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### 4. Check the Health of Your Cluster Pods + +Now that you have set up the `kubeconfig` file, you can use `kubectl` to access the cluster from your local machine. + +Check that all the required pods and containers are healthy are ready to continue: + +``` +/var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml get pods -A +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system cloud-controller-manager-rke2-server-1 1/1 Running 0 2m28s +kube-system cloud-controller-manager-rke2-server-2 1/1 Running 0 61s +kube-system cloud-controller-manager-rke2-server-3 1/1 Running 0 49s +kube-system etcd-rke2-server-1 1/1 Running 0 2m13s +kube-system etcd-rke2-server-2 1/1 Running 0 87s +kube-system etcd-rke2-server-3 1/1 Running 0 56s +kube-system helm-install-rke2-canal-hs6sx 0/1 Completed 0 2m17s +kube-system helm-install-rke2-coredns-xmzm8 0/1 Completed 0 2m17s +kube-system helm-install-rke2-ingress-nginx-flwnl 0/1 Completed 0 2m17s +kube-system helm-install-rke2-metrics-server-7sggn 0/1 Completed 0 2m17s +kube-system kube-apiserver-rke2-server-1 1/1 Running 0 116s +kube-system kube-apiserver-rke2-server-2 1/1 Running 0 66s +kube-system kube-apiserver-rke2-server-3 1/1 Running 0 48s +kube-system kube-controller-manager-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-controller-manager-rke2-server-2 1/1 Running 0 57s +kube-system kube-controller-manager-rke2-server-3 1/1 Running 0 42s +kube-system kube-proxy-rke2-server-1 1/1 Running 0 2m25s +kube-system kube-proxy-rke2-server-2 1/1 Running 0 59s +kube-system kube-proxy-rke2-server-3 1/1 Running 0 85s +kube-system kube-scheduler-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-scheduler-rke2-server-2 1/1 Running 0 57s +kube-system kube-scheduler-rke2-server-3 1/1 Running 0 42s +kube-system rke2-canal-b9lvm 2/2 Running 0 91s +kube-system rke2-canal-khwp2 2/2 Running 0 2m5s +kube-system rke2-canal-swfmq 2/2 Running 0 105s +kube-system rke2-coredns-rke2-coredns-547d5499cb-6tvwb 1/1 Running 0 92s +kube-system rke2-coredns-rke2-coredns-547d5499cb-rdttj 1/1 Running 0 2m8s +kube-system rke2-coredns-rke2-coredns-autoscaler-65c9bb465d-85sq5 1/1 Running 0 2m8s +kube-system rke2-ingress-nginx-controller-69qxc 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-7hprp 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-x658h 1/1 Running 0 52s +kube-system rke2-metrics-server-6564db4569-vdfkn 1/1 Running 0 66s +``` + +**Result:** You have confirmed that you can access the cluster with `kubectl` and the RKE2 cluster is running successfully. Now the Rancher management server can be installed on the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md new file mode 100644 index 00000000000..7c5e21424ea --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md @@ -0,0 +1,77 @@ +--- +title: Recommended Cluster Architecture +--- + + + + + +There are three roles that can be assigned to nodes: `etcd`, `controlplane` and `worker`. + +## Separating Worker Nodes from Nodes with Other Roles + +When designing your cluster(s), you have two options: + +* Use dedicated nodes for each role. This ensures resource availability for the components needed for the specified role. It also strictly isolates network traffic between each of the roles according to the [port requirements](../node-requirements-for-rancher-managed-clusters.md#networking-requirements). +* Assign the `etcd` and `controlplane` roles to the same nodes. These nodes must meet the hardware requirements for both roles. + +In either case, the `worker` role should not be used or added to nodes with the `etcd` or `controlplane` role. + +Therefore, each node should have one of the following role configurations: + + * `etcd` + * `controlplane` + * Both `etcd` and `controlplane` + * `worker` + +## Recommended Number of Nodes with Each Role + +The cluster should have: + +- At least three nodes with the role `etcd` to survive losing one node. Increase this count for higher node fault toleration, and spread them across (availability) zones to provide even better fault tolerance. +- At least two nodes with the role `controlplane` for master component high availability. +- At least two nodes with the role `worker` for workload rescheduling upon node failure. + +For more information on what each role is used for, refer to the [section on roles for nodes in Kubernetes.](roles-for-nodes-in-kubernetes.md) + + +### Number of Controlplane Nodes + +Adding more than one node with the `controlplane` role makes every master component highly available. + +### Number of etcd Nodes + +The number of nodes that you can lose at once while maintaining cluster availability is determined by the number of nodes assigned the `etcd` role. For a cluster with n members, the minimum is (n/2)+1. Therefore, we recommend creating an `etcd` node in 3 different availability zones within a region to survive the loss of one availability zone. If you use only two zones, you can only survive the loss of the zone where you don't lose the majority of nodes. + +| Nodes with `etcd` role | Majority | Failure Tolerance | +|--------------|------------|-------------------| +| 1 | 1 | 0 | +| 2 | 2 | 0 | +| 3 | 2 | **1** | +| 4 | 3 | 1 | +| 5 | 3 | **2** | +| 6 | 4 | 2 | +| 7 | 4 | **3** | +| 8 | 5 | 3 | +| 9 | 5 | **4** | + +References: + +* [Official etcd documentation on optimal etcd cluster size](https://etcd.io/docs/v3.4.0/faq/#what-is-failure-tolerance) +* [Official Kubernetes documentation on operating etcd clusters for Kubernetes](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) + +### Number of Worker Nodes + +Adding more than one node with the `worker` role will make sure your workloads can be rescheduled if a node fails. + +### Why Production Requirements are Different for the Rancher Cluster and the Clusters Running Your Applications + +You may have noticed that our [Kubernetes Install](../../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) instructions do not meet our definition of a production-ready cluster, as there are no dedicated nodes for the `worker` role. However, for your Rancher installation, this three node cluster is valid, because: + +* It allows one `etcd` node failure. +* It maintains multiple instances of the master components by having multiple `controlplane` nodes. +* No other workloads than Rancher itself should be created on this cluster. + +## References + +* [Kubernetes: Master Components](https://kubernetes.io/docs/concepts/overview/components/#master-components) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md new file mode 100644 index 00000000000..04eeb4466d4 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md @@ -0,0 +1,54 @@ +--- +title: Roles for Nodes in Kubernetes +--- + + + + + +This section describes the roles for etcd nodes, controlplane nodes, and worker nodes in Kubernetes, and how the roles work together in a cluster. + +This diagram is applicable to Kubernetes clusters [launched with Rancher using RKE.](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + +![Cluster diagram](/img/clusterdiagram.svg)
+Lines show the traffic flow between components. Colors are used purely for visual aid + +## etcd + +Nodes with the `etcd` role run etcd, which is a consistent and highly available key value store used as Kubernetes’ backing store for all cluster data. etcd replicates the data to each node. + +:::note + +Nodes with the `etcd` role are shown as `Unschedulable` in the UI, meaning no pods will be scheduled to these nodes by default. + +::: + +## controlplane + +Nodes with the `controlplane` role run the Kubernetes master components (excluding `etcd`, as it's a separate role). See [Kubernetes: Master Components](https://kubernetes.io/docs/concepts/overview/components/#master-components) for a detailed list of components. + +:::note + +Nodes with the `controlplane` role are shown as `Unschedulable` in the UI, meaning no pods will be scheduled to these nodes by default. + +::: + +### kube-apiserver + +The Kubernetes API server (`kube-apiserver`) scales horizontally. Each node with the role `controlplane` will be added to the NGINX proxy on the nodes with components that need to access the Kubernetes API server. This means that if a node becomes unreachable, the local NGINX proxy on the node will forward the request to another Kubernetes API server in the list. + +### kube-controller-manager + +The Kubernetes controller manager uses leader election using an endpoint in Kubernetes. One instance of the `kube-controller-manager` will create an entry in the Kubernetes endpoints and updates that entry in a configured interval. Other instances will see an active leader and wait for that entry to expire (for example, when a node is unresponsive). + +### kube-scheduler + +The Kubernetes scheduler uses leader election using an endpoint in Kubernetes. One instance of the `kube-scheduler` will create an entry in the Kubernetes endpoints and updates that entry in a configured interval. Other instances will see an active leader and wait for that entry to expire (for example, when a node is unresponsive). + +## worker + +Nodes with the `worker` role run the Kubernetes node components. See [Kubernetes: Node Components](https://kubernetes.io/docs/concepts/overview/components/#node-components) for a detailed list of components. + +## References + +* [Kubernetes: Node Components](https://kubernetes.io/docs/concepts/overview/components/#node-components) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md new file mode 100644 index 00000000000..3f0575a2f76 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md @@ -0,0 +1,133 @@ +--- +title: Node Requirements for Rancher Managed Clusters +--- + + + + + +This page describes the requirements for the Rancher managed Kubernetes clusters where your apps and services will be installed. These downstream clusters should be separate from the three-node cluster running Rancher. + +:::note + +If Rancher is installed on a high-availability Kubernetes cluster, the Rancher server three-node cluster and downstream clusters have different requirements. For Rancher installation requirements, refer to the node requirements in the [installation section.](../../../pages-for-subheaders/installation-requirements.md) + +::: + +## Operating Systems and Container Runtime Requirements + +Rancher should work with any modern Linux distribution and any modern Docker version. Linux is required for the etcd and controlplane nodes of all downstream clusters. Worker nodes may run Linux or [Windows Server.](#windows-nodes) + +For details on which OS and Docker versions were tested with each Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) + +All supported operating systems are 64-bit x86. + +If you plan to use ARM64, see [Running on ARM64 (Experimental).](../../advanced-user-guides/enable-experimental-features/rancher-on-arm64.md) + +For information on how to install Docker, refer to the official [Docker documentation.](https://docs.docker.com/) + +### Oracle Linux and RHEL Derived Linux Nodes + +Some distributions of Linux derived from RHEL, including Oracle Linux, may have default firewall rules that block communication with Helm. We recommend disabling firewalld. For Kubernetes 1.19, firewalld must be turned off. + +:::note + +In RHEL 8.4, two extra services are included on the NetworkManager: `nm-cloud-setup.service` and `nm-cloud-setup.timer`. These services add a routing table that interferes with the CNI plugin's configuration. If these services are enabled, you must disable them using the command below, and then reboot the node to restore connectivity: + + ``` + systemctl disable nm-cloud-setup.service nm-cloud-setup.timer + reboot + ``` + +::: + +### SUSE Linux Nodes + +SUSE Linux may have a firewall that blocks all ports by default. In that situation, follow [these steps](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#opening-suse-linux-ports) to open the ports needed for adding a host to a custom cluster. + +### Flatcar Container Linux Nodes + +When [Launching Kubernetes with Rancher](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) using Flatcar Container Linux nodes, it is required to use the following configuration in the [Cluster Config File](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: canal + options: + canal_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: calico + options: + calico_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +It is also required to enable the Docker service, you can enable the Docker service using the following command: + +``` +systemctl enable docker.service +``` + +The Docker service is enabled automatically when using [Node Drivers](../../../pages-for-subheaders/about-provisioning-drivers.md#node-drivers). + +### Windows Nodes + +Nodes with Windows Server must run Docker Enterprise Edition. + +Windows nodes can be used for worker nodes only. See [Configuring Custom Clusters for Windows](../../../pages-for-subheaders/use-windows-clusters.md) + +## Hardware Requirements + +The hardware requirements for nodes with the `worker` role mostly depend on your workloads. The minimum to run the Kubernetes node components is 1 CPU (core) and 1GB of memory. + +Regarding CPU and memory, it is recommended that the different planes of Kubernetes clusters (etcd, controlplane, and workers) should be hosted on different nodes so that they can scale separately from each other. + +For hardware recommendations for large Kubernetes clusters, refer to the official Kubernetes documentation on [building large clusters.](https://kubernetes.io/docs/setup/best-practices/cluster-large/) + +For hardware recommendations for etcd clusters in production, refer to the official [etcd documentation.](https://etcd.io/docs/v3.4.0/op-guide/hardware/) + +## Networking Requirements + +For a production cluster, we recommend that you restrict traffic by opening only the ports defined in the port requirements below. + +The ports required to be open are different depending on how the user cluster is launched. Each of the sections below list the ports that need to be opened for different [cluster creation options](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +For a breakdown of the port requirements for etcd nodes, controlplane nodes, and worker nodes in a Kubernetes cluster, refer to the [port requirements for the Rancher Kubernetes Engine.](https://rancher.com/docs/rke/latest/en/os/#ports) + +Details on which ports are used in each situation are found under [Downstream Cluster Port Requirements](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#downstream-kubernetes-cluster-nodes). + +:::caution + +You should never register a node with the same hostname or IP address as an existing node. Doing so causes RKE to prevent the node from joining, and provisioning to hang. This can occur for both node driver and custom clusters. If a node must reuse a hostname or IP of an existing node, you must set the `hostname_override` [RKE option](https://rke.docs.rancher.com/config-options/nodes#overriding-the-hostname) before registering the node, so that it can join correctly. + +::: + +## Optional: Security Considerations + +If you want to provision a Kubernetes cluster that is compliant with the CIS (Center for Internet Security) Kubernetes Benchmark, we recommend to following our hardening guide to configure your nodes before installing Kubernetes. + +For more information on the hardening guide and details on which version of the guide corresponds to your Rancher and Kubernetes versions, refer to the [security section.](../../../pages-for-subheaders/rancher-security.md#rancher-hardening-guide) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md new file mode 100644 index 00000000000..02e2604884f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -0,0 +1,286 @@ +--- +title: Registering Existing Clusters +--- + + + + + +The cluster registration feature replaced the feature to import clusters. + +The control that Rancher has to manage a registered cluster depends on the type of cluster. For details, see [Management Capabilities for Registered Clusters.](#management-capabilities-for-registered-clusters) + + +## Prerequisites + +### Kubernetes Node Roles + +Registered RKE Kubernetes clusters must have all three node roles - etcd, controlplane and worker. A cluster with only controlplane components cannot be registered in Rancher. + +For more information on RKE node roles, see the [best practices.](../../../pages-for-subheaders/checklist-for-production-ready-clusters.md#cluster-architecture) + +### Permissions + +To register a cluster in Rancher, you must have `cluster-admin` privileges within that cluster. If you don't, grant these privileges to your user by running: + +```plain +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole cluster-admin \ + --user [USER_ACCOUNT] +``` + +Since, by default, Google Kubernetes Engine (GKE) doesn't grant the `cluster-admin` role, you must run these commands on GKE clusters before you can register them. To learn more about role-based access control for GKE, please see [the official Google documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control). + +### EKS, AKS and GKE Clusters + +To successfully import them into or provision them from Rancher, Elastic Kubernetes Service (EKS), Azure Kubernetes Service (AKS), and Google Kubernetes Engine (GKE) clusters must have at least one managed node group. In addition, GKE Autopilot clusters aren't supported. See [Compare GKE Autopilot and Standard](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison) for more information about the difference between GKE modes. + +## Registering a Cluster + +1. Click **☰ > Cluster Management**. +2. On the **Clusters** page, **Import Existing**. +3. Choose the type of cluster. +4. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +5. If you are importing a generic Kubernetes cluster in Rancher, perform the following steps for setup:
+ a. Click **Agent Environment Variables** under **Cluster Options** to set environment variables for [rancher cluster agent](../launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. If rancher agent requires use of proxy to communicate with Rancher server, `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables can be set using agent environment variables.
+ b. Enable Project Network Isolation to ensure the cluster supports Kubernetes `NetworkPolicy` resources. Users can select the **Project Network Isolation** option under the **Advanced Options** dropdown to do so. +6. Click **Create**. +7. The prerequisite for `cluster-admin` privileges is shown (see **Prerequisites** above), including an example command to fulfil the prerequisite. +8. Copy the `kubectl` command to your clipboard and run it on a node where kubeconfig is configured to point to the cluster you want to import. If you are unsure it is configured correctly, run `kubectl get nodes` to verify before running the command shown in Rancher. +9. If you are using self-signed certificates, you will receive the message `certificate signed by unknown authority`. To work around this validation, copy the command starting with `curl` displayed in Rancher to your clipboard. Then run the command on a node where kubeconfig is configured to point to the cluster you want to import. +10. When you finish running the command(s) on your node, click **Done**. + +**Result:** + +- Your cluster is registered and assigned a state of **Pending**. Rancher is deploying resources to manage your cluster. +- You can access your cluster after its state is updated to **Active**. +- **Active** clusters are assigned two Projects: `Default` (containing the namespace `default`) and `System` (containing the namespaces `cattle-system`, `ingress-nginx`, `kube-public` and `kube-system`, if present). + + +:::note + +You can not re-register a cluster that is currently active in a Rancher setup. + +::: + +### Configuring an Imported EKS, AKS or GKE Cluster with Terraform + +You should define **only** the minimum fields that Rancher requires when importing an EKS, AKS or GKE cluster with Terraform. This is important as Rancher will overwrite what was in the cluster configuration with any config that the user has provided. + +:::caution + +Even a small difference between the current cluster and a user-provided config could have unexpected results. + +::: + +The minimum config fields required by Rancher to import EKS clusters with Terraform using `eks_config_v2` are as follows: + +- cloud_credential_id +- name +- region +- imported (this field should always be set to `true` for imported clusters) + +Example YAML configuration for imported EKS clusters: + +``` +resource "rancher2_cluster" "my-eks-to-import" { + name = "my-eks-to-import" + description = "Terraform EKS Cluster" + eks_config_v2 { + cloud_credential_id = rancher2_cloud_credential.aws.id + name = var.aws_eks_name + region = var.aws_region + imported = true + } +} +``` + +You can find additional examples for other cloud providers in the [Rancher2 Terraform Provider documentation](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster). + +## Management Capabilities for Registered Clusters + +The control that Rancher has to manage a registered cluster depends on the type of cluster. + +- [Features for All Registered Clusters](#features-for-all-registered-clusters) +- [Additional Features for Registered RKE2 and K3s Clusters](#additional-features-for-registered-rke2-and-k3s-clusters) +- [Additional Features for Registered EKS, AKS and GKE Clusters](#additional-features-for-registered-eks-aks-and-gke-clusters) + +### Features for All Registered Clusters + +After registering a cluster, the cluster owner can: + +- [Manage cluster access](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) through role-based access control +- Enable [monitoring, alerts and notifiers](../../../pages-for-subheaders/monitoring-and-alerting.md) +- Enable [logging](../../../pages-for-subheaders/logging.md) +- Enable [Istio](../../../pages-for-subheaders/istio.md) +- Manage projects and workloads + +### Additional Features for Registered RKE2 and K3s Clusters + +[K3s](https://rancher.com/docs/k3s/latest/en/) is a lightweight, fully compliant Kubernetes distribution for edge installations. +[RKE2](https://docs.rke2.io) is Rancher's next-generation Kubernetes distribution for datacenter and cloud installations. + +When an RKE2 or K3s cluster is registered in Rancher, Rancher will recognize it. The Rancher UI will expose the features for [all registered clusters,](#features-for-all-registered-clusters) in addition to the following features for editing and upgrading the cluster: + +- The ability to [upgrade the Kubernetes version](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md) + :::danger + + After a cluster has been imported into Rancher, upgrades should be performed using Rancher. Upgrading an imported cluster outside of Rancher is **not** supported. + + ::: +- The ability to configure the maximum number of nodes that will be upgraded concurrently +- The ability to see a read-only version of the cluster's configuration arguments and environment variables used to launch each node in the cluster + +### Additional Features for Registered EKS, AKS, and GKE Clusters + +Rancher handles registered EKS, AKS, or GKE clusters similarly to clusters created in Rancher. However, Rancher doesn't destroy registered clusters when you delete them through the Rancher UI. + +When you create an EKS, AKS, or GKE cluster in Rancher, then delete it, Rancher destroys the cluster. When you delete a registered cluster through Rancher, the Rancher server _disconnects_ from the cluster. The cluster remains live, although it's no longer in Rancher. You can still access the deregistered cluster in the same way you did before you registered it. + +See [Cluster Management Capabilities by Cluster Type](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) for more information about what features are available for managing registered clusters. + +## Configuring RKE2 and K3s Cluster Upgrades + +:::tip + +It is a Kubernetes best practice to back up the cluster before upgrading. When upgrading a high-availability K3s cluster with an external database, back up the database in whichever way is recommended by the relational database provider. + +::: + +The **concurrency** is the maximum number of nodes that are permitted to be unavailable during an upgrade. If number of unavailable nodes is larger than the **concurrency,** the upgrade will fail. If an upgrade fails, you may need to repair or remove failed nodes before the upgrade can succeed. + +- **Controlplane concurrency:** The maximum number of server nodes to upgrade at a single time; also the maximum unavailable server nodes +- **Worker concurrency:** The maximum number worker nodes to upgrade at the same time; also the maximum unavailable worker nodes + +In the RKE2 and K3s documentation, controlplane nodes are called server nodes. These nodes run the Kubernetes master, which maintains the desired state of the cluster. By default, these controlplane nodes have the capability to have workloads scheduled to them by default. + +Also in the RKE2 and K3s documentation, nodes with the worker role are called agent nodes. Any workloads or pods that are deployed in the cluster can be scheduled to these nodes by default. + +## Debug Logging and Troubleshooting for Registered RKE2 and K3s Clusters + +Nodes are upgraded by the system upgrade controller running in the downstream cluster. Based on the cluster configuration, Rancher deploys two [plans](https://github.com/rancher/system-upgrade-controller#example-upgrade-plan) to upgrade nodes: one for controlplane nodes and one for workers. The system upgrade controller follows the plans and upgrades the nodes. + +To enable debug logging on the system upgrade controller deployment, edit the [configmap](https://github.com/rancher/system-upgrade-controller/blob/50a4c8975543d75f1d76a8290001d87dc298bdb4/manifests/system-upgrade-controller.yaml#L32) to set the debug environment variable to true. Then restart the `system-upgrade-controller` pod. + +Logs created by the `system-upgrade-controller` can be viewed by running this command: + +``` +kubectl logs -n cattle-system system-upgrade-controller +``` + +The current status of the plans can be viewed with this command: + +``` +kubectl get plans -A -o yaml +``` + +If the cluster becomes stuck in upgrading, restart the `system-upgrade-controller`. + +To prevent issues when upgrading, the [Kubernetes upgrade best practices](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/) should be followed. + +## Authorized Cluster Endpoint Support for RKE2 and K3s Clusters + +_Available as of v2.6.3_ + +Authorized Cluster Endpoint (ACE) support has been added for registered RKE2 and K3s clusters. This support includes manual steps you will perform on the downstream cluster to enable the ACE. For additional information on the authorized cluster endpoint, click [here](../manage-clusters/access-clusters/authorized-cluster-endpoint.md). + +:::note Notes: + +- These steps only need to be performed on the control plane nodes of the downstream cluster. You must configure each control plane node individually. + +- The following steps will work on both RKE2 and K3s clusters registered in v2.6.x as well as those registered (or imported) from a previous version of Rancher with an upgrade to v2.6.x. + +- These steps will alter the configuration of the downstream RKE2 and K3s clusters and deploy the `kube-api-authn-webhook`. If a future implementation of the ACE requires an update to the `kube-api-authn-webhook`, then this would also have to be done manually. For more information on this webhook, click [here](../manage-clusters/access-clusters/authorized-cluster-endpoint.md#about-the-kube-api-auth-authentication-webhook). + +::: + +###### **Manual steps to be taken on the control plane of each downstream cluster to enable ACE:** + +1. Create a file at `/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml` with the following contents: + ```yaml + apiVersion: v1 + kind: Config + clusters: + - name: Default + cluster: + insecure-skip-tls-verify: true + server: http://127.0.0.1:6440/v1/authenticate + users: + - name: Default + user: + insecure-skip-tls-verify: true + current-context: webhook + contexts: + - name: webhook + context: + user: Default + cluster: Default + ``` + +2. Add the following to the config file (or create one if it doesn’t exist); note that the default location is `/etc/rancher/{rke2,k3s}/config.yaml`: + ```yaml + kube-apiserver-arg: + - authentication-token-webhook-config-file=/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml + ``` + +3. Run the following commands: + + sudo systemctl stop {rke2,k3s}-server + sudo systemctl start {rke2,k3s}-server + +4. Finally, you **must** go back to the Rancher UI and edit the imported cluster there to complete the ACE enablement. Click on **⋮ > Edit Config**, then click the **Networking** tab under Cluster Configuration. Finally, click the **Enabled** button for **Authorized Endpoint**. Once the ACE is enabled, you then have the option of entering a fully qualified domain name (FQDN) and certificate information. + +:::note + +The FQDN field is optional, and if one is entered, it should point to the downstream cluster. Certificate information is only needed if there is a load balancer in front of the downstream cluster that is using an untrusted certificate. If you have a valid certificate, then nothing needs to be added to the CA Certificates field. + +::: + +## Annotating Registered Clusters + +For all types of registered Kubernetes clusters except for RKE2 and K3s Kubernetes clusters, Rancher doesn't have any information about how the cluster is provisioned or configured. + +Therefore, when Rancher registers a cluster, it assumes that several capabilities are disabled by default. Rancher assumes this in order to avoid exposing UI options to the user even when the capabilities are not enabled in the registered cluster. + +However, if the cluster has a certain capability, such as the ability to use a pod security policy, a user of that cluster might still want to select pod security policies for the cluster in the Rancher UI. In order to do that, the user will need to manually indicate to Rancher that pod security policies are enabled for the cluster. + +By annotating a registered cluster, it is possible to indicate to Rancher that a cluster was given a pod security policy, or another capability, outside of Rancher. + +This example annotation indicates that a pod security policy is enabled: + +```json +"capabilities.cattle.io/pspEnabled": "true" +``` + +The following annotation indicates Ingress capabilities. Note that the values of non-primitive objects need to be JSON encoded, with quotations escaped. + +```json +"capabilities.cattle.io/ingressCapabilities": "[ + { + "customDefaultBackend":true, + "ingressProvider":"asdf" + } +]" +``` + +These capabilities can be annotated for the cluster: + +- `ingressCapabilities` +- `loadBalancerCapabilities` +- `nodePoolScalingSupported` +- `nodePortRange` +- `pspEnabled` +- `taintSupport` + +All the capabilities and their type definitions can be viewed in the Rancher API view, at `[Rancher Server URL]/v3/schemas/capabilities`. + +To annotate a registered cluster, + +1. Click **☰ > Cluster Management**. +2. On the **Clusters** page, go to the custom cluster you want to annotate and click **⋮ > Edit Config**. +3. Expand the **Labels & Annotations** section. +4. Click **Add Annotation**. +5. Add an annotation to the cluster with the format `capabilities/: ` where `value` is the cluster capability that will be overridden by the annotation. In this scenario, Rancher is not aware of any capabilities of the cluster until you add the annotation. +6. Click **Save**. + +**Result:** The annotation does not give the capabilities to the cluster, but it does indicate to Rancher that the cluster has those capabilities. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md new file mode 100644 index 00000000000..5943ee01710 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md @@ -0,0 +1,164 @@ +--- +title: Setting up the Amazon Cloud Provider +weight: 1 +--- + + + + + +When using the `Amazon` cloud provider, you can leverage the following capabilities: + +- **Load Balancers:** Launches an AWS Elastic Load Balancer (ELB) when choosing `Layer-4 Load Balancer` in **Port Mapping** or when launching a `Service` with `type: LoadBalancer`. +- **Persistent Volumes**: Allows you to use AWS Elastic Block Stores (EBS) for persistent volumes. + +See [cloud-provider-aws README](https://kubernetes.github.io/cloud-provider-aws/) for all information regarding the Amazon cloud provider. + +To set up the Amazon cloud provider, + +1. [Create an IAM role and attach to the instances](#1-create-an-iam-role-and-attach-to-the-instances) +2. [Configure the ClusterID](#2-configure-the-clusterid) + +:::note Important: + +Starting with Kubernetes 1.23, you have to deactivate the `CSIMigrationAWS` feature gate in order to use the in-tree AWS cloud provider. You can do this by setting `feature-gates=CSIMigrationAWS=false` as an additional argument for the cluster's Kubelet, Controller Manager, API Server and Scheduler in the advanced cluster configuration. + +::: + +### 1. Create an IAM Role and attach to the instances + +All nodes added to the cluster must be able to interact with EC2 so that they can create and remove resources. You can enable this interaction by using an IAM role attached to the instance. See [Amazon documentation: Creating an IAM Role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#create-iam-role) how to create an IAM role. There are two example policies: + +* The first policy is for the nodes with the `controlplane` role. These nodes have to be able to create/remove EC2 resources. The following IAM policy is an example, please remove any unneeded permissions for your use case. +* The second policy is for the nodes with the `etcd` or `worker` role. These nodes only have to be able to retrieve information from EC2. + +While creating an [Amazon EC2 cluster](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md), you must fill in the **IAM Instance Profile Name** (not ARN) of the created IAM role when creating the **Node Template**. + +While creating a [Custom cluster](../../../../pages-for-subheaders/use-existing-nodes.md), you must manually attach the IAM role to the instance(s). + +IAM Policy for nodes with the `controlplane` role: + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } +] +} +``` + +IAM policy for nodes with the `etcd` or `worker` role: + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } +] +} +``` + +### 2. Configure the ClusterID + +The following resources need to tagged with a `ClusterID`: + +- **Nodes**: All hosts added in Rancher. +- **Subnet**: The subnet used for your cluster. +- **Security Group**: The security group used for your cluster. + +:::note + +Do not tag multiple security groups. Tagging multiple groups generates an error when creating an Elastic Load Balancer (ELB). + +::: + +When you create an [Amazon EC2 Cluster](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md), the `ClusterID` is automatically configured for the created nodes. Other resources still need to be tagged manually. + +Use the following tag: + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `owned` + +`CLUSTERID` can be any string you like, as long as it is equal across all tags set. + +Setting the value of the tag to `owned` tells the cluster that all resources with this tag are owned and managed by this cluster. If you share resources between clusters, you can change the tag to: + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `shared`. + +### Using Amazon Elastic Container Registry (ECR) + +The kubelet component has the ability to automatically obtain ECR credentials, when the IAM profile mentioned in [Create an IAM Role and attach to the instances](#1-create-an-iam-role-and-attach-to-the-instances) is attached to the instance(s). When using a Kubernetes version older than v1.15.0, the Amazon cloud provider needs be configured in the cluster. Starting with Kubernetes version v1.15.0, the kubelet can obtain ECR credentials without having the Amazon cloud provider configured in the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md new file mode 100644 index 00000000000..8720aa0760e --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md @@ -0,0 +1,115 @@ +--- +title: Setting up the Azure Cloud Provider +--- + + + + + +When using the `Azure` cloud provider, you can leverage the following capabilities: + +- **Load Balancers:** Launches an Azure Load Balancer within a specific Network Security Group. + +- **Persistent Volumes:** Supports using Azure Blob disks and Azure Managed Disks with standard and premium storage accounts. + +- **Network Storage:** Support Azure Files via CIFS mounts. + +The following account types are not supported for Azure Subscriptions: + +- Single tenant accounts (i.e. accounts with no subscriptions). +- Multi-subscription accounts. + +## Prerequisites for RKE and RKE2 + +To set up the Azure cloud provider for both RKE and RKE2, the following credentials need to be configured: + +1. [Set up the Azure Tenant ID](#1-set-up-the-azure-tenant-id) +2. [Set up the Azure Client ID and Azure Client Secret](#2-set-up-the-azure-client-id-and-azure-client-secret) +3. [Configure App Registration Permissions](#3-configure-app-registration-permissions) +4. [Set up Azure Network Security Group Name](#4-set-up-azure-network-security-group-name) + +### 1. Set up the Azure Tenant ID + +Visit [Azure portal](https://portal.azure.com), login and go to **Azure Active Directory** and select **Properties**. Your **Directory ID** is your **Tenant ID** (tenantID). + +If you want to use the Azure CLI, you can run the command `az account show` to get the information. + +### 2. Set up the Azure Client ID and Azure Client Secret + +Visit [Azure portal](https://portal.azure.com), login and follow the steps below to create an **App Registration** and the corresponding **Azure Client ID** (aadClientId) and **Azure Client Secret** (aadClientSecret). + +1. Select **Azure Active Directory**. +1. Select **App registrations**. +1. Select **New application registration**. +1. Choose a **Name**, select `Web app / API` as **Application Type** and a **Sign-on URL** which can be anything in this case. +1. Select **Create**. + +In the **App registrations** view, you should see your created App registration. The value shown in the column **APPLICATION ID** is what you need to use as **Azure Client ID**. + +The next step is to generate the **Azure Client Secret**: + +1. Open your created App registration. +1. In the **Settings** view, open **Keys**. +1. Enter a **Key description**, select an expiration time and select **Save**. +1. The generated value shown in the column **Value** is what you need to use as **Azure Client Secret**. This value will only be shown once. + +### 3. Configure App Registration Permissions + +The last thing you will need to do, is assign the appropriate permissions to your App registration. + +1. Go to **More services**, search for **Subscriptions** and open it. +1. Open **Access control (IAM)**. +1. Select **Add**. +1. For **Role**, select `Contributor`. +1. For **Select**, select your created App registration name. +1. Select **Save**. + +### 4. Set up Azure Network Security Group Name + +A custom Azure Network Security Group (securityGroupName) is needed to allow Azure Load Balancers to work. + +If you provision hosts using Rancher Machine Azure driver, you will need to edit them manually to assign them to this Network Security Group. + +You should already assign custom hosts to this Network Security Group during provisioning. + +Only hosts expected to be load balancer back ends need to be in this group. + +## RKE2 Cluster Set-up in Rancher + +1. Choose "Azure" from the Cloud Provider drop-down in the Cluster Configuration section. + +1. * Supply the Cloud Provider Configuration. Note that Rancher will automatically create a new Network Security Group, Resource Group, Availability Set, Subnet, and Virtual Network. If you already have some or all of these created, you will need to specify them before creating the cluster. + * You can click on "Show Advanced" to see more of these automatically generated names and update them if + necessary. Your Cloud Provider Configuration **must** match the fields in the Machine Pools section. If you have multiple pools, they must all use the same Resource Group, Availability Set, Subnet, Virtual Network, and Network Security Group. + * An example is provided below. You will modify it as needed. + +
+ Example Cloud Provider Config + + ```yaml + { + "cloud":"AzurePublicCloud", + "tenantId": "YOUR TENANTID HERE", + "aadClientId": "YOUR AADCLIENTID HERE", + "aadClientSecret": "YOUR AADCLIENTSECRET HERE", + "subscriptionId": "YOUR SUBSCRIPTIONID HERE", + "resourceGroup": "docker-machine", + "location": "westus", + "subnetName": "docker-machine", + "securityGroupName": "rancher-managed-KA4jV9V2", + "securityGroupResourceGroup": "docker-machine", + "vnetName": "docker-machine-vnet", + "vnetResourceGroup": "docker-machine", + "primaryAvailabilitySetName": "docker-machine", + "routeTableResourceGroup": "docker-machine", + "cloudProviderBackoff": false, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true + } + ``` + +
+ +1. Under the **Cluster Configuration > Advanced** section, click **Add** under **Additional Controller Manager Args** and add this flag: `--configure-cloud-routes=false` + +1. Click the **Create** button to submit the form and create the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md new file mode 100644 index 00000000000..aba2b7a587c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md @@ -0,0 +1,23 @@ +--- +title: How to Configure an In-tree vSphere Cloud Provider +--- + + + + + +To set up the in-tree vSphere cloud provider, follow these steps while creating the vSphere cluster in Rancher: + +1. Set **Cloud Provider** option to `Custom` or `Custom (In-Tree)`. +1. Click on **Edit as YAML** +1. Insert the following structure to the pre-populated cluster YAML. This structure must be placed under `rancher_kubernetes_engine_config`. Note that the `name` *must* be set to `vsphere`. + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: vsphere + vsphereCloudProvider: + [Insert provider configuration] + ``` + +Rancher uses RKE (the Rancher Kubernetes Engine) to provision Kubernetes clusters. Refer to the [vSphere configuration reference in the RKE documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere/config-reference/) for details about the properties of the `vsphereCloudProvider` directive. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md new file mode 100644 index 00000000000..2d78b61b57f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md @@ -0,0 +1,68 @@ +--- +title: How to Configure an Out-of-tree vSphere Cloud Provider +--- + + + + + +Kubernetes is moving away from maintaining cloud providers in-tree. vSphere has an out-of-tree cloud provider that can be used by installing the vSphere cloud provider and cloud storage plugins. + +This page covers how to install the Cloud Provider Interface (CPI) and Cloud Storage Interface (CSI) plugins after setting up a cluster. + +## Prerequisites + +The vSphere versions supported: + +* 6.7u3 +* 7.0u1 or higher. + +The Kubernetes version must be 1.19 or higher. + +Using the vSphere out-of-tree cloud provider requires Linux nodes and is not supported on Windows. + +## Installation + +The Cloud Provider Interface (CPI) should be installed first before installing the Cloud Storage Interface (CSI). + +### 1. Create a vSphere cluster + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **VMware vSphere** or **Custom**. +1. On the **Basics** tab in the **Cluster Configuration** section, set the **Cloud Provider** to **vSphere**. +1. In the **Add-On Config** tab, the vSphere Cloud Provider (CPI) and Storage Provider (CSI) options. +1. Finish creating your cluster. + +### 2. Install the CPI plugin + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CPI plugin will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CPI**. +1. Fill out the required vCenter details. +1. vSphere CPI initializes all nodes with ProviderID which is needed by the vSphere CSI driver. Check if all nodes are initialized with the ProviderID before installing CSI driver with the following command: + + ``` + kubectl describe nodes | grep "ProviderID" + ``` + +### 3. Installing the CSI plugin + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CSI plugin will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CSI**. +1. Click **Install**. +1. Fill out the required vCenter details. On the **Features** tab, set **Enable CSI Migration** to **false**. +3. On the **Storage** tab, fill out the details for the StorageClass. This chart creates a StorageClass with the `csi.vsphere.vmware.com` as the provisioner. +1. Click **Install**. + + +## Using the CSI driver for provisioning volumes + +The CSI chart by default creates a storageClass. + +If that option was not selected while launching the chart, create a storageClass with the `csi.vsphere.vmware.com` as the provisioner. + +All volumes provisioned using this StorageClass will get provisioned by the CSI driver. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md new file mode 100644 index 00000000000..ffd1cf6bdb4 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md @@ -0,0 +1,61 @@ +--- +title: Setting up the Google Compute Engine Cloud Provider +--- + + + + + +In this section, you'll learn how to enable the Google Compute Engine (GCE) cloud provider for custom clusters in Rancher. A custom cluster is one in which Rancher installs Kubernetes on existing nodes. + +The official Kubernetes documentation for the GCE cloud provider is [here.](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#gce) + +:::note Prerequisites: + +The service account of `Identity and API` access on GCE needs the `Computer Admin` permission. + +::: + +If you are using Calico, + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the custom cluster and click **⋮ > Edit YAML.* Enter the following configuration: + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + network: + options: + calico_cloud_provider: "gce" + plugin: "calico" + ``` + +If you are using Canal or Flannel, + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the custom cluster and click **⋮ > Edit YAML.* Enter the following configuration: + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + services: + kube_controller: + extra_args: + configure-cloud-routes: true # we need to allow the cloud provider configure the routes for the hosts + ``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md new file mode 100644 index 00000000000..d302213118a --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md @@ -0,0 +1,112 @@ +--- +title: Migrating vSphere In-tree Volumes to Out-of-tree +--- + + + + + +Kubernetes is moving away from maintaining cloud providers in-tree. vSphere has an out-of-tree cloud provider that can be used by installing the vSphere cloud provider and cloud storage plugins. + +This page covers how to migrate from the in-tree vSphere cloud provider to out-of-tree, and manage the existing VMs post migration. + +It follows the steps provided in the official [vSphere migration documentation](https://vsphere-csi-driver.sigs.k8s.io/features/vsphere_csi_migration.html) and provides the steps to be performed in Rancher. + +### Cloud-config Format Limitation + +Existing volumes that were provisioned using the following cloud-config format won't get migrated due to an existing bug in vSphere Cloud Storage Interface (CSI). + +If the cloud-config has this format for datastore and resource pool path, vsphere CSI driver cannot recognize it: + +```yaml +default-datastore: /datastore/ +resourcepool-path: "/host//Resources/" +``` + +Volumes provisioned with the in-tree provider using the following format will get migrated correctly: + +```yaml +default-datastore: +resourcepool-path: "/Resources/" +``` + +Upstream bug: https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/628 + +Rancher issue tracking this bug: https://github.com/rancher/rancher/issues/31105 + +## Prerequisites + +- vSphere CSI Migration requires vSphere 7.0u1. In order to be able to manage existing in-tree vSphere volumes, upgrade vSphere to 7.0u1. +- The Kubernetes version must be 1.19 or higher. + +## Migration + +### 1. Install the CPI plugin + +Before installing CPI, we need to taint all nodes with `node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule`. + +This can be done by running the following commands: + +``` +curl -O https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +``` + +Or: + +``` +wget https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +chmod +x taints.sh +./taints.sh +``` + +Once all nodes are tainted by the running the script, launch the Helm vSphere CPI chart. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CPI chart will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CPI**.. +1. Click **Install**. +1. Fill out the required vCenter details and click **Install**. + +vSphere CPI initializes all nodes with ProviderID, which is needed by the vSphere CSI driver. + +Check if all nodes are initialized with the ProviderID with the following command: + +``` +kubectl describe nodes | grep "ProviderID" +``` + +### 2. Install the CSI driver + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CSI chart will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CSI**.. +1. Click **Install**. +1. Fill out the required vCenter details and click **Install**. +1. Check **Customize Helm options before install** and click **Next**. +1. On the **Features** tab, check **Enable CSI Migration**. +1. Optionally, go to the **Storage** tab and set up a datastore. This chart creates a StorageClass with the `csi.vsphere.vmware.com` as the provisioner. You can provide the URL of the datastore to be used for CSI volume provisioning while creating this StorageClass. The datastore URL can be found in the vSphere client by selecting the datastore and going to the Summary tab. Fill out the details for the StorageClass. +1. Click **Install**. + +### 3. Edit the cluster to enable CSI migration feature flags + +1. While editing the cluster, if the Kubernetes version is less than 1.19, select Kubernetes version 1.19 or higher from the **Kubernetes Version** dropdown. +2. For enabling feature flags, click on "Edit as YAML", and add the following under kube-controller and kubelet: + + ```yaml + extra_args: + feature-gates: "CSIMigration=true,CSIMigrationvSphere=true" + ``` + +### 4. Drain worker nodes + +Worker nodes must be drained during the upgrade before changing the kubelet and kube-controller-manager args. + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you will drain worker nodes and click **⋮ > Edit Config**. +1. In the **Advanced Options** section, set the field **Maximum Worker Nodes Unavailable** to 1. +1. To drain the nodes during upgrade, select **Drain Nodes > Yes**. +1. Set **Force** and **Delete Local Data** to **true**. +1. Click **Save** to upgrade the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md new file mode 100644 index 00000000000..f176acc177b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md @@ -0,0 +1,277 @@ +--- +title: Creating an AKS Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Microsoft Azure Kubernetes Service (AKS). + +## Prerequisites in Microsoft Azure + +:::caution + +Deploying to AKS will incur charges. + +::: + +To interact with Azure APIs, an AKS cluster requires an Azure Active Directory (AD) service principal. The service principal is needed to dynamically create and manage other Azure resources, and it provides credentials for your cluster to communicate with AKS. For more information about the service principal, refer to the [AKS documentation](https://docs.microsoft.com/en-us/azure/aks/kubernetes-service-principal). + +Before creating the service principal, you need to obtain the following information from the [Microsoft Azure Portal](https://portal.azure.com): + +- Subscription ID +- Client ID +- Client secret + +The below sections describe how to set up these prerequisites using either the Azure command line tool or the Azure portal. + +### Setting Up the Service Principal with the Azure Command Line Tool + +You can create the service principal by running this command: + +``` +az ad sp create-for-rbac --skip-assignment +``` + +The result should show information about the new service principal: +``` +{ + "appId": "xxxx--xxx", + "displayName": "", + "name": "http://", + "password": "", + "tenant": "" +} +``` + +You also need to add roles to the service principal so that it has privileges for communication with the AKS API. It also needs access to create and list virtual networks. + +Below is an example command for assigning the Contributor role to a service principal. Contributors can manage anything on AKS but cannot give access to others: + +``` +az role assignment create \ + --assignee $appId \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +You can also create the service principal and give it Contributor privileges by combining the two commands into one. In this command, the scope needs to provide a full path to an Azure resource: + +``` +az ad sp create-for-rbac \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +### Setting Up the Service Principal with the Azure Command Line Tool + +Create the Resource Group by running this command: + +``` +az group create --location AZURE_LOCATION_NAME --resource-group AZURE_RESOURCE_GROUP_NAME +``` + +### Setting Up the Service Principal from the Azure Portal + +You can also follow these instructions to set up a service principal and give it role-based access from the Azure Portal. + +1. Go to the Microsoft Azure Portal [home page](https://portal.azure.com). + +1. Click **Azure Active Directory**. +1. Click **App registrations**. +1. Click **New registration**. +1. Enter a name. This will be the name of your service principal. +1. Optional: Choose which accounts can use the service principal. +1. Click **Register**. +1. You should now see the name of your service principal under **Azure Active Directory > App registrations**. +1. Click the name of your service principal. Take note of the application ID (also called app ID or client ID) so that you can use it when provisioning your AKS cluster. Then click **Certificates & secrets**. +1. Click **New client secret**. +1. Enter a short description, pick an expiration time, and click **Add**. Take note of the client secret so that you can use it when provisioning the AKS cluster. + +**Result:** You have created a service principal and you should be able to see it listed in the **Azure Active Directory** section under **App registrations**. You still need to give the service principal access to AKS. + +To give role-based access to your service principal, + +1. Click **All Services** in the left navigation bar. Then click **Subscriptions**. +1. Click the name of the subscription that you want to associate with your Kubernetes cluster. Take note of the subscription ID so that you can use it when provisioning your AKS cluster. +1. Click **Access Control (IAM)**. +1. In the **Add role assignment** section, click **Add**. +1. In the **Role** field, select a role that will have access to AKS. For example, you can use the **Contributor** role, which has permission to manage everything except for giving access to other users. +1. In the **Assign access to** field, select **Azure AD user, group, or service principal**. +1. In the **Select** field, select the name of your service principal and click **Save**. + +**Result:** Your service principal now has access to AKS. + +## 1. Create the AKS Cloud Credentials + +1. In the Rancher UI, click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Azure**. +1. Fill out the form. For help with filling out the form, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md#cloud-credentials) +1. Click **Create**. + +## 2. Create the AKS Cluster + +Use Rancher to set up and configure your Kubernetes cluster. + +1. Click **☰ > Cluster Management**. +1. In the **Clusters** section, click **Create**. +1. Click **Azure AKS**. +1. Fill out the form. For help with filling out the form, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) +1. Click **Create**. + +**Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +## Role-based Access Control +When provisioning an AKS cluster in the Rancher UI, RBAC is not configurable because it is required to be enabled. + +RBAC is required for AKS clusters that are registered or imported into Rancher. + +### Setting Up the Role Assignment to Service Principal with the Azure Command Line Tool + +Assign the Rancher AKSv2 role to the service principal with the Azure Command Line Tool: + +``` +az role assignment create \ +--assignee CLIENT_ID \ +--scope "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME" \ +--role "Rancher AKSv2" +``` + +## AKS Cluster Configuration Reference + +For more information about how to configure AKS clusters from the Rancher UI, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) + +## Private Clusters + +Typically, AKS worker nodes do not get public IPs, regardless of whether the cluster is private. In a private cluster, the control plane does not have a public endpoint. + +Rancher can connect to a private AKS cluster in one of two ways. + +The first way to ensure that Rancher is running on the same [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) as the AKS nodes. + +The second way is to run a command to register the cluster with Rancher. Once the cluster is provisioned, you can run the displayed command anywhere you can connect to the cluster’s Kubernetes API. This command is displayed in a pop-up when you provision an AKS cluster with a private API endpoint enabled. + +:::note + +Please be aware that when registering an existing AKS cluster, the cluster might take some time, possibly hours, to appear in the `Cluster To register` dropdown list. This outcome will be based on region. + +::: + +For more information about connecting to an AKS private cluster, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster) + +## Setting Up the Minimum Permission Role with the Azure Command Line Tool + +1. Create the Minimum Rancher AKSv2 Permission Role by running this command: + + ``` + cat >> rancher-azure.json << EOF + + { + "Name": "Rancher AKSv2", + "IsCustom": true, + "Description": "Everything needed by Rancher AKSv2 operator", + "Actions": [ + "Microsoft.Compute/disks/delete", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.Compute/locations/DiskOperations/read", + "Microsoft.Compute/locations/vmSizes/read", + "Microsoft.Compute/locations/operations/read", + "Microsoft.Compute/proximityPlacementGroups/write", + "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/snapshots/write", + "Microsoft.Compute/virtualMachineScaleSets/manualUpgrade/action", + "Microsoft.Compute/virtualMachineScaleSets/delete", + "Microsoft.Compute/virtualMachineScaleSets/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/ipconfigurations/publicipaddresses/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/write", + "Microsoft.Compute/virtualMachineScaleSets/write", + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/virtualMachines/write", + "Microsoft.ContainerService/managedClusters/read", + "Microsoft.ContainerService/managedClusters/write" + "Microsoft.ContainerService/managedClusters/delete", + "Microsoft.ContainerService/managedClusters/accessProfiles/listCredential/action", + "Microsoft.ContainerService/managedClusters/agentPools/read", + "Microsoft.ContainerService/managedClusters/agentPools/write", + "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action", + "Microsoft.Network/applicationGateways/read", + "Microsoft.Network/applicationGateways/write", + "Microsoft.Network/loadBalancers/write", + "Microsoft.Network/loadBalancers/backendAddressPools/join/action", + "Microsoft.Network/loadBalancers/delete", + "Microsoft.Network/loadBalancers/read", + "Microsoft.Network/networkInterfaces/join/action", + "Microsoft.Network/networkInterfaces/read", + "Microsoft.Network/networkInterfaces/write", + "Microsoft.Network/networkSecurityGroups/read", + "Microsoft.Network/networkSecurityGroups/write", + "Microsoft.Network/publicIPAddresses/delete", + "Microsoft.Network/publicIPAddresses/join/action", + "Microsoft.Network/publicIPAddresses/read", + "Microsoft.Network/publicIPAddresses/write", + "Microsoft.Network/publicIPPrefixes/join/action", + "Microsoft.Network/privatednszones/*", + "Microsoft.Network/routeTables/read", + "Microsoft.Network/routeTables/routes/delete", + "Microsoft.Network/routeTables/routes/read", + "Microsoft.Network/routeTables/routes/write", + "Microsoft.Network/routeTables/write", + "Microsoft.Network/virtualNetworks/read", + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/read", + "Microsoft.Network/virtualNetworks/joinLoadBalancer/action", + "Microsoft.OperationalInsights/workspaces/sharedkeys/read", + "Microsoft.OperationalInsights/workspaces/read", + "Microsoft.OperationsManagement/solutions/write", + "Microsoft.OperationsManagement/solutions/read", + "Microsoft.Resources/subscriptions/resourcegroups/read", + "Microsoft.Resources/subscriptions/resourcegroups/write", + "Microsoft.Storage/operations/read", + "Microsoft.Storage/storageAccounts/listKeys/action", + "Microsoft.Storage/storageAccounts/delete", + "Microsoft.Storage/storageAccounts/read", + "Microsoft.Storage/storageAccounts/write" + ], + "NotActions": [], + "DataActions": [], + "NotDataActions": [], + "AssignableScopes": [ + "/subscriptions/SUBSCRIPTION_ID" + ] + } + EOF + ``` + +2. Apply the Rancher AKSv2 Role: + + ``` + az role definition create --role-definition rancher-azure.json + ``` + +3. Verify if the Rancher AKSv2 Role was created: + + ``` + az role definition list | grep "Rancher AKSv2" + ``` + +## Syncing + +The AKS provisioner can synchronize the state of an AKS cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) + +For information on configuring the refresh interval, see [this section.](../../../../pages-for-subheaders/gke-cluster-configuration.md#configuring-the-refresh-interval) + + +## Programmatically Creating AKS Clusters + +The most common way to programmatically deploy AKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md new file mode 100644 index 00000000000..82cdde8861f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md @@ -0,0 +1,63 @@ +--- +title: Creating an Aliyun ACK Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Alibaba Cloud Kubernetes (ACK). Rancher has already implemented and packaged the [cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md) for ACK, but by default, this cluster driver is `inactive`. In order to launch ACK clusters, you will need to [enable the ACK cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#activatingdeactivating-cluster-drivers). After enabling the cluster driver, you can start provisioning ACK clusters. + +## Prerequisites Outside of Rancher + +:::caution + +Deploying to ACK will incur charges. + +::: + +1. In Aliyun, activate the following services in their respective consoles. + + - [Container Service](https://cs.console.aliyun.com) + - [Resource Orchestration Service](https://ros.console.aliyun.com) + - [RAM](https://ram.console.aliyun.com) + +2. Make sure that the account you will be using to create the ACK cluster has the appropriate permissions. Referring to the official Alibaba Cloud documentation about [Role authorization](https://www.alibabacloud.com/help/doc-detail/86483.htm) and [Use the Container Service console as a RAM user](https://www.alibabacloud.com/help/doc-detail/86484.htm) for details. + +3. In Alibaba Cloud, create an [access key](https://www.alibabacloud.com/help/doc-detail/53045.html). + +4. In Alibaba Cloud, create an [SSH key pair](https://www.alibabacloud.com/help/doc-detail/51793.html). This key is used to access nodes in the Kubernetes cluster. + +## Prerequisite in Rancher + +You will need to enable the Alibaba ACK cluster driver: + +1. Click **☰ > Cluster Management**. +1. Click **Drivers**. +1. In the **Cluster Drivers** tab, go to the **Alibaba ACK** cluster driver and click **⋮ > Activate**. + +When the cluster driver is finished downloading, you will be able to create Alibaba ACK clusters in Rancher. + +## Create an ACK Cluster + +1. Click **☰ > Cluster Management**. +1. From the **Clusters** page, click **Create**. +1. Choose **Alibaba ACK**. +1. Enter a **Cluster Name**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Configure **Account Access** for the ACK cluster. Choose the geographical region in which to build your cluster, and input the access key that was created as part of the prerequisite steps. +1. Click **Next: Configure Cluster**, then choose cluster type, the version of Kubernetes and the availability zone. +1. If you choose **Kubernetes** as the cluster type, Click **Next: Configure Master Nodes**, then complete the **Master Nodes** form. +1. Click **Next: Configure Worker Nodes**, then complete the **Worker Nodes** form. +1. Review your options to confirm they're correct. Then click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md new file mode 100644 index 00000000000..e6ecf6090ec --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md @@ -0,0 +1,333 @@ +--- +title: Creating an EKS Cluster +--- + + + + + +Amazon EKS provides a managed control plane for your Kubernetes cluster. Amazon EKS runs the Kubernetes control plane instances across multiple Availability Zones to ensure high availability. Rancher provides an intuitive user interface for managing and deploying the Kubernetes clusters you run in Amazon EKS. With this guide, you will use Rancher to quickly and easily launch an Amazon EKS Kubernetes cluster in your AWS account. For more information on Amazon EKS, see this [documentation](https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html). + + +## Prerequisites in Amazon Web Services + +:::caution + +Deploying to Amazon AWS will incur charges. For more information, refer to the [EKS pricing page](https://aws.amazon.com/eks/pricing/). + +::: + +To set up a cluster on EKS, you will need to set up an Amazon VPC (Virtual Private Cloud). You will also need to make sure that the account you will be using to create the EKS cluster has the appropriate [permissions.](#minimum-eks-permissions) For details, refer to the official guide on [Amazon EKS Prerequisites](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-console.html#eks-prereqs). + +### Amazon VPC + +An Amazon VPC is required to launch the EKS cluster. The VPC enables you to launch AWS resources into a virtual network that you've defined. You can set one up yourself and provide it during cluster creation in Rancher. If you do not provide one during creation, Rancher will create one. For more information, refer to the [Tutorial: Creating a VPC with Public and Private Subnets for Your Amazon EKS Cluster](https://docs.aws.amazon.com/eks/latest/userguide/create-public-private-vpc.html). + +### IAM Policies + +Rancher needs access to your AWS account in order to provision and administer your Kubernetes clusters in Amazon EKS. You'll need to create a user for Rancher in your AWS account and define what that user can access. + +1. Create a user with programmatic access by following the steps [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). + +2. Next, create an IAM policy that defines what this user has access to in your AWS account. It's important to only grant this user minimal access within your account. The minimum permissions required for an EKS cluster are listed [here.](#minimum-eks-permissions) Follow the steps [here](https://docs.aws.amazon.com/eks/latest/userguide/EKS_IAM_user_policies.html) to create an IAM policy and attach it to your user. + +3. Finally, follow the steps [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) to create an access key and secret key for this user. + +:::note Important: + +It's important to regularly rotate your access and secret keys. See this [documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#rotating_access_keys_console) for more information. + +::: + +For more detailed information on IAM policies for EKS, refer to the official [documentation on Amazon EKS IAM Policies, Roles, and Permissions](https://docs.aws.amazon.com/eks/latest/userguide/IAM_policies.html). + + +## Create the EKS Cluster + +Use Rancher to set up and configure your Kubernetes cluster. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Choose **Amazon EKS**. +1. Enter a **Cluster Name**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Fill out the rest of the form. For help, refer to the [configuration reference.](#eks-cluster-configuration-reference) +1. Click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## EKS Cluster Configuration Reference + +For the full list of EKS cluster configuration options, see [this page.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) + +## Architecture + +The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two Kubernetes clusters: one created by RKE and another created by EKS. + +
Managing Kubernetes Clusters through Rancher's Authentication Proxy
+ +![Architecture](/img/rancher-architecture-rancher-api-server.svg) + +## AWS Service Events + +To find information on any AWS Service events, please see [this page](https://status.aws.amazon.com/). + +## Security and Compliance + +By default only the IAM user or role that created a cluster has access to it. Attempting to access the cluster with any other user or role without additional configuration will lead to an error. In Rancher, this means using a credential that maps to a user or role that was not used to create the cluster will cause an unauthorized error. For example, an EKSCtl cluster will not register in Rancher unless the credentials used to register the cluster match the role or user used by EKSCtl. Additional users and roles can be authorized to access a cluster by being added to the aws-auth configmap in the kube-system namespace. For a more in-depth explanation and detailed instructions, please see this [documentation](https://aws.amazon.com/premiumsupport/knowledge-center/amazon-eks-cluster-access/). + +For more information on security and compliance with your Amazon EKS Kubernetes cluster, please see this [documentation](https://docs.aws.amazon.com/eks/latest/userguide/shared-responsibilty.html). + +## Tutorial + +This [tutorial](https://aws.amazon.com/blogs/opensource/managing-eks-clusters-rancher/) on the AWS Open Source Blog will walk you through how to set up an EKS cluster with Rancher, deploy a publicly accessible app to test the cluster, and deploy a sample project to track real-time geospatial data using a combination of other open-source software such as Grafana and InfluxDB. + +## Minimum EKS Permissions + +These are the minimum set of permissions necessary to access the full functionality of Rancher's EKS driver. You'll need additional permissions for Rancher to provision the `Service Role` and `VPC` resources. If you create these resources **before** you create the cluster, they'll be available when you configure the cluster. + +Resource | Description +---------|------------ +Service Role | Provides permissions that allow Kubernetes to manage resources on your behalf. Rancher can create the service role with the following [Service Role Permissions](#service-role-permissions). +VPC | Provides isolated network resources utilised by EKS and worker nodes. Rancher can create the VPC resources with the following [VPC Permissions](#vpc-permissions). + + +Resource targeting uses `*` as the ARN of many of the resources created cannot be known before creating the EKS cluster in Rancher. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "EC2Permissions", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:RevokeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:DescribeInstanceTypes", + "ec2:DescribeRegions", + "ec2:DescribeVpcs", + "ec2:DescribeTags", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeRouteTables", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeKeyPairs", + "ec2:DescribeInternetGateways", + "ec2:DescribeImages", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeAccountAttributes", + "ec2:DeleteTags", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteSecurityGroup", + "ec2:DeleteKeyPair", + "ec2:CreateTags", + "ec2:CreateSecurityGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateLaunchTemplate", + "ec2:CreateKeyPair", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress" + ], + "Resource": "*" + }, + { + "Sid": "CloudFormationPermissions", + "Effect": "Allow", + "Action": [ + "cloudformation:ListStacks", + "cloudformation:ListStackResources", + "cloudformation:DescribeStacks", + "cloudformation:DescribeStackResources", + "cloudformation:DescribeStackResource", + "cloudformation:DeleteStack", + "cloudformation:CreateStackSet", + "cloudformation:CreateStack" + ], + "Resource": "*" + }, + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:ListInstanceProfilesForRole", + "iam:ListInstanceProfiles", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "iam:GetInstanceProfile", + "iam:DetachRolePolicy", + "iam:DeleteRole", + "iam:CreateRole", + "iam:AttachRolePolicy" + ], + "Resource": "*" + }, + { + "Sid": "KMSPermissions", + "Effect": "Allow", + "Action": "kms:ListKeys", + "Resource": "*" + }, + { + "Sid": "EKSPermissions", + "Effect": "Allow", + "Action": [ + "eks:UpdateNodegroupVersion", + "eks:UpdateNodegroupConfig", + "eks:UpdateClusterVersion", + "eks:UpdateClusterConfig", + "eks:UntagResource", + "eks:TagResource", + "eks:ListUpdates", + "eks:ListTagsForResource", + "eks:ListNodegroups", + "eks:ListFargateProfiles", + "eks:ListClusters", + "eks:DescribeUpdate", + "eks:DescribeNodegroup", + "eks:DescribeFargateProfile", + "eks:DescribeCluster", + "eks:DeleteNodegroup", + "eks:DeleteFargateProfile", + "eks:DeleteCluster", + "eks:CreateNodegroup", + "eks:CreateFargateProfile", + "eks:CreateCluster" + ], + "Resource": "*" + } + ] +} +``` + +### Service Role Permissions + +These are permissions that are needed during EKS cluster creation, so Rancher can create a service role on the users' behalf. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreateRole", + "iam:CreateServiceLinkedRole", + "iam:DeleteInstanceProfile", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfiles", + "iam:ListInstanceProfilesForRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile" + ], + "Resource": "*" + } + ] +} +``` + +When you create an EKS cluster, Rancher creates a service role with the following trust policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "eks.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} +``` + +This role also has two role policy attachments with the following policies' ARNs: + +``` +arn:aws:iam::aws:policy/AmazonEKSClusterPolicy +arn:aws:iam::aws:policy/AmazonEKSServicePolicy +``` + +### VPC Permissions + +These are permissions that are needed by Rancher to create a Virtual Private Cloud (VPC) and associated resources. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VPCPermissions", + "Effect": "Allow", + "Action": [ + "ec2:ReplaceRoute", + "ec2:ModifyVpcAttribute", + "ec2:ModifySubnetAttribute", + "ec2:DisassociateRouteTable", + "ec2:DetachInternetGateway", + "ec2:DescribeVpcs", + "ec2:DeleteVpc", + "ec2:DeleteTags", + "ec2:DeleteSubnet", + "ec2:DeleteRouteTable", + "ec2:DeleteRoute", + "ec2:DeleteInternetGateway", + "ec2:CreateVpc", + "ec2:CreateSubnet", + "ec2:CreateSecurityGroup", + "ec2:CreateRouteTable", + "ec2:CreateRoute", + "ec2:CreateInternetGateway", + "ec2:AttachInternetGateway", + "ec2:AssociateRouteTable" + ], + "Resource": "*" + } + ] +} +``` + +## Syncing + +The EKS provisioner can synchronize the state of an EKS cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) + +For information on configuring the refresh interval, refer to [this section.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md#configuring-the-refresh-interval) + +## Troubleshooting + +If your changes were overwritten, it could be due to the way the cluster data is synced with EKS. Changes shouldn't be made to the cluster from another source, such as in the EKS console, and in Rancher within a five-minute span. For information on how this works and how to configure the refresh interval, refer to [Syncing.](#syncing) + +If an unauthorized error is returned while attempting to modify or register the cluster and the cluster was not created with the role or user that your credentials belong to, refer to [Security and Compliance.](#security-and-compliance) + +For any issues or troubleshooting details for your Amazon EKS Kubernetes cluster, please see this [documentation](https://docs.aws.amazon.com/eks/latest/userguide/troubleshooting.html). + +## Programmatically Creating EKS Clusters + +The most common way to programmatically deploy EKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md new file mode 100644 index 00000000000..335267c0d66 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md @@ -0,0 +1,105 @@ +--- +title: Creating a GKE Cluster +--- + + + + + +## Prerequisites + +Some setup in Google Kubernetes Engine is required. + +### Service Account Token + +Create a service account using [Google Kubernetes Engine](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts). GKE uses this account to operate your cluster. Creating this account also generates a private key used for authentication. + +The service account requires the following roles: + +- **Compute Viewer:** `roles/compute.viewer` +- **Project Viewer:** `roles/viewer` +- **Kubernetes Engine Admin:** `roles/container.admin` +- **Service Account User:** `roles/iam.serviceAccountUser` + +[Google Documentation: Creating and Enabling Service Accounts](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) + +For help obtaining a private key for your service account, refer to the Google cloud documentation [here.](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) You will need to save the key in JSON format. + +### Google Project ID + +Your cluster will need to be part of a Google Project. + +To create a new project, refer to the Google cloud documentation [here.](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project) + +To get the project ID of an existing project, refer to the Google cloud documentation [here.](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects) + +## Provisioning a GKE Cluster + +:::caution + +Deploying to GKE will incur charges. + +::: + +### 1. Create a Cloud Credential + +1. Click **☰ > Cluster Management**. +1. In the left navigation bar, click **Cloud Credentials**. +1. Click **Create**. +1. Enter a name for your Google cloud credentials. +1. In the **Service Account** text box, paste your service account private key JSON, or upload the JSON file. +1. Click **Create**. + +**Result:** You have created credentials that Rancher will use to provision the new GKE cluster. + +### 2. Create the GKE Cluster +Use Rancher to set up and configure your Kubernetes cluster. To successfully create a GKE cluster with Rancher, your GKE must be in Standard mode. GKE has two modes of operation when creating a Kubernetes cluster, Autopilot and Standard mode. The cluster configuration for Autopilot mode has restrictions on editing the kube-system namespace. However, Rancher needs to create resources in the kube-system namespace during installation. As a result, you will not be able to create a GKE cluster in Autopilot mode. For more information about the difference between GKE Autopilot mode and Standard mode, visit [Compare GKE Autopilot and Standard.](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison) + + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Google GKE**. +1. Enter a **Cluster Name**. +1. Optional: Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Optional: Add Kubernetes [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) or [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to the cluster. +1. Enter your Google project ID and your Google cloud credentials. +1. Fill out the rest of the form. For help, refer to the [GKE cluster configuration reference.](../../../../pages-for-subheaders/gke-cluster-configuration.md) +1. Click **Create**. + +**Result:** You have successfully deployed a GKE cluster. + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Private Clusters + +Private GKE clusters are supported. Note: This advanced setup can require more steps during the cluster provisioning process. For details, see [this section.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md) + +## Configuration Reference + +For details on configuring GKE clusters in Rancher, see [this page.](../../../../pages-for-subheaders/gke-cluster-configuration.md) +## Updating Kubernetes Version + +The Kubernetes version of a cluster can be upgraded to any version available in the region or zone for the GKE cluster. Upgrading the master Kubernetes version does not automatically upgrade worker nodes. Nodes can be upgraded independently. + +:::note + +GKE has removed basic authentication in 1.19+. In order to upgrade a cluster to 1.19+, basic authentication must be disabled in the Google Cloud. Otherwise, an error will appear in Rancher when an upgrade to 1.19+ is attempted. You can follow the [Google documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#disabling_authentication_with_a_static_password). After this, the Kubernetes version can be updated to 1.19+ via Rancher. + +::: + +## Syncing + +The GKE provisioner can synchronize the state of a GKE cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) + +For information on configuring the refresh interval, see [this section.](../../../../pages-for-subheaders/gke-cluster-configuration.md#configuring-the-refresh-interval) + +## Programmatically Creating GKE Clusters + +The most common way to programmatically deploy GKE clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md new file mode 100644 index 00000000000..cbde0642b88 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md @@ -0,0 +1,104 @@ +--- +title: Creating a Huawei CCE Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Huawei Cloud Container Engine (CCE). Rancher has already implemented and packaged the [cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md) for CCE, but by default, this cluster driver is `inactive`. In order to launch CCE clusters, you will need to [enable the CCE cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#activatingdeactivating-cluster-drivers). After enabling the cluster driver, you can start provisioning CCE clusters. + +## Prerequisites in Huawei + +:::caution + +Deploying to CCE will incur charges. + +::: + +1. Find your project ID in Huawei CCE portal. See the CCE documentation on how to [manage your projects](https://doc.hcs.huawei.com/api/cce/cce_02_0341.html#cce_02_0341__section17571944123214). + +2. Create an [Access Key ID and Secret Access Key](https://support.huaweicloud.com/en-us/usermanual-iam/en-us_topic_0079477318.html). + +## Prerequisite in Rancher + +You will need to enable the Huawei CCE cluster driver: + +1. Click **☰ > Cluster Management**. +1. Click **Drivers**. +1. In the **Cluster Drivers** tab, go to the **Huawei CCE** cluster driver and click **⋮ > Activate**. + +When the cluster driver is finished downloading, you will be able to create Huawei CCE clusters in Rancher. + +## Limitations + +Huawei CCE service doesn't support the ability to create clusters with public access through their API. You are required to run Rancher in the same VPC as the CCE clusters that you want to provision. + +## Create the CCE Cluster + +1. From the **Clusters** page, click **Create**. +1. Click **Huawei CCE**. +1. Enter a **Cluster Name**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Enter **Project Id**, Access Key ID as **Access Key** and Secret Access Key **Secret Key**. Then Click **Next: Configure cluster**. Fill in the cluster configuration. For help filling out the form, refer to [Huawei CCE Configuration.](#huawei-cce-configuration) +1. Fill the following node configuration of the cluster. For help filling out the form, refer to [Node Configuration.](#node-configuration) +1. Click **Create** to create the CCE cluster. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Huawei CCE Configuration + +|Settings|Description| +|---|---| +| Cluster Type | Which type or node you want to include into the cluster, `VirtualMachine` or `BareMetal`. | +| Description | The description of the cluster. | +| Master Version | The Kubernetes version. | +| Management Scale Count | The max node count of the cluster. The options are 50, 200 and 1000. The larger of the scale count, the more the cost. | +| High Availability | Enable master node high availability. The cluster with high availability enabled will have more cost. | +| Container Network Mode | The network mode used in the cluster. `overlay_l2` and `vpc-router` is supported in `VirtualMachine` type and `underlay_ipvlan` is supported in `BareMetal` type | +| Container Network CIDR | Network CIDR for the cluster. | +| VPC Name | The VPC name which the cluster is going to deploy into. Rancher will create one if it is blank. | +| Subnet Name | The Subnet name which the cluster is going to deploy into. Rancher will create one if it is blank. | +| External Server | This option is reserved for the future we can enable CCE cluster public access via API. For now, it is always disabled. | +| Cluster Label | The labels for the cluster. | +| Highway Subnet | This option is only supported in `BareMetal` type. It requires you to select a VPC with high network speed for the bare metal machines. | + +:::note + +If you are editing the cluster in the `cluster.yml` instead of the Rancher UI, note that cluster configuration directives must be nested under the `rancher_kubernetes_engine_config` directive in `cluster.yml`. For more information, refer to the section on [the config file structure.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#config-file-structure-in-rancher) + +::: + +## Node Configuration + +|Settings|Description| +|---|---| +| Zone | The available zone at where the node(s) of the cluster is deployed. | +| Billing Mode | The bill mode for the cluster node(s). In `VirtualMachine` type, only `Pay-per-use` is supported. in `BareMetal`, you can choose `Pay-per-use` or `Yearly/Monthly`. | +| Validity Period | This option only shows in `Yearly/Monthly` bill mode. It means how long you want to pay for the cluster node(s). | +| Auto Renew | This option only shows in `Yearly/Monthly` bill mode. It means that the cluster node(s) will renew the `Yearly/Monthly` payment automatically or not. | +| Data Volume Type | Data volume type for the cluster node(s). `SATA`, `SSD` or `SAS` for this option. | +| Data Volume Size | Data volume size for the cluster node(s) | +| Root Volume Type | Root volume type for the cluster node(s). `SATA`, `SSD` or `SAS` for this option. | +| Root Volume Size | Root volume size for the cluster node(s) | +| Node Flavor | The node flavor of the cluster node(s). The flavor list in Rancher UI is fetched from Huawei Cloud. It includes all the supported node flavors. | +| Node Count | The node count of the cluster | +| Node Operating System | The operating system for the cluster node(s). Only `EulerOS 2.2` and `CentOS 7.4` are supported right now. | +| SSH Key Name | The ssh key for the cluster node(s) | +| EIP | The public IP options for the cluster node(s). `Disabled` means that the cluster node(s) are not going to bind a public IP. `Create EIP` means that the cluster node(s) will bind one or many newly created Eips after provisioned and more options will be shown in the UI to set the to-create EIP parameters. And `Select Existed EIP` means that the node(s) will bind to the EIPs you select. | +| EIP Count | This option will only be shown when `Create EIP` is selected. It means how many EIPs you want to create for the node(s). | +| EIP Type | This option will only be shown when `Create EIP` is selected. The options are `5_bgp` and `5_sbgp`. | +| EIP Share Type | This option will only be shown when `Create EIP` is selected. The only option is `PER`. | +| EIP Charge Mode | This option will only be shown when `Create EIP` is selected. The options are pay by `BandWidth` and pay by `Traffic`. | +| EIP Bandwidth Size | This option will only be shown when `Create EIP` is selected. The BandWidth of the EIPs. | +| Authentication Mode | It means enabling `RBAC` or also enabling `Authenticating Proxy`. If you select `Authenticating Proxy`, the certificate which is used for authenticating proxy will be also required. | +| Node Label | The labels for the cluster node(s). Invalid labels can prevent upgrades or can prevent Rancher from starting. For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) | \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md new file mode 100644 index 00000000000..35cd4cc9b1c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md @@ -0,0 +1,103 @@ +--- +title: Creating a Tencent TKE Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Tencent Kubernetes Engine (TKE). Rancher has already implemented and packaged the [cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md) for TKE, but by default, this cluster driver is `inactive`. In order to launch TKE clusters, you will need to [enable the TKE cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#activatingdeactivating-cluster-drivers). After enabling the cluster driver, you can start provisioning TKE clusters. + +## Prerequisites in Tencent + +:::caution + +Deploying to TKE will incur charges. + +::: + +1. Make sure that the account you will be using to create the TKE cluster has the appropriate permissions by referring to the [Cloud Access Management](https://intl.cloud.tencent.com/document/product/598/10600) documentation for details. + +2. Create a [Cloud API Secret ID and Secret Key](https://console.cloud.tencent.com/capi). + +3. Create a [Private Network and Subnet](https://intl.cloud.tencent.com/document/product/215/4927) in the region that you want to deploy your Kubernetes cluster. + +4. Create a [SSH key pair](https://intl.cloud.tencent.com/document/product/213/6092). This key is used to access the nodes in the Kubernetes cluster. + +## Prerequisite in Rancher + +You will need to enable the Tencent TKE cluster driver: + +1. Click **☰ > Cluster Management**. +1. Click **Drivers**. +1. In the **Cluster Drivers** tab, go to the **Tencent TKE** cluster driver and click **⋮ > Activate**. + +When the cluster driver is finished downloading, you will be able to create Tencent TKE clusters in Rancher. + +## Create a TKE Cluster + +1. From the **Clusters** page, click **Create**. + +2. Choose **Tencent TKE**. + +3. Enter a **Cluster Name**. + +4. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. + +5. Configure **Account Access** for the TKE cluster. Complete each drop-down and field using the information obtained in [Prerequisites](#prerequisites-in-tencent). + + | Option | Description | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Region | From the drop-down chooses the geographical region in which to build your cluster. | + | Secret ID | Enter the Secret ID that you obtained from the Tencent Cloud Console. | + | Secret Key | Enter the Secret key that you obtained from Tencent Cloud Console. | + +6. Click `Next: Configure Cluster` to set your TKE cluster configurations. + + | Option | Description | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Kubernetes Version | The TKE only supports Kubernetes version 1.10.5 now. | + | Node Count | Enter the amount of worker node you want to purchase for your Kubernetes cluster, up to 100. | + | VPC | Select the VPC name that you have created in the Tencent Cloud Console. | + | Container Network CIDR | Enter the CIDR range of your Kubernetes cluster, you may check the available range of the CIDR in the VPC service of the Tencent Cloud Console. Default to 172.16.0.0/16. | + + :::note + + If you are editing the cluster in the `cluster.yml` instead of the Rancher UI, note that, cluster configuration directives must be nested under the `rancher_kubernetes_engine_config` directive in `cluster.yml`. For more information, refer to the section on [the config file structure in Rancher v2.3.0+.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#config-file-structure-in-rancher) + + ::: + +7. Click `Next: Select Instance Type` to choose the instance type that will use for your TKE cluster. + + | Option | Description | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Availability Zone | Choose the availability zone of the VPC region. | + | Subnet | Select the Subnet that you have created within the VPC, and add a new one if you don't have it in the chosen availability zone. | + | Instance Type | From the drop-down chooses the VM instance type that you want to use for the TKE cluster, default to S2.MEDIUM4 (CPU 2 Memory 4 GiB). | + +8. Click `Next: Configure Instance` to configure the VM instance that will use for your TKE cluster. + + Option | Description + -------|------------ + Operating System | The name of the operating system, currently supports Centos7.2x86_64 or ubuntu16.04.1 LTSx86_64 + Security Group | Security group ID, default does not bind any security groups. + Root Disk Type | System disk type. System disk type restrictions are detailed in the [CVM instance configuration](https://cloud.tencent.com/document/product/213/11518). + Root Disk Size | System disk size. Linux system adjustment range is 20 - 50G, step size is 1. + Data Disk Type | Data disk type, default value to the SSD cloud drive + Data Disk Size | Data disk size (GB), the step size is 10 + Band Width Type | Type of bandwidth, PayByTraffic or PayByHour + Band Width | Public network bandwidth (Mbps) + Key Pair | Key id, after associating the key can be used to logging to the VM node + +9. Click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md new file mode 100644 index 00000000000..7690a17e458 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md @@ -0,0 +1,45 @@ +--- +title: Configuration for Storage Classes in Azure +--- + + + + + +If you are using Azure VMs for your nodes, you can use [Azure files](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv) as a StorageClass for the cluster. + +In order to have the Azure platform create the required storage resources, follow these steps: + +1. [Configure the Azure cloud provider.](../../kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md) +1. Configure `kubectl` to connect to your cluster. +1. Copy the `ClusterRole` and `ClusterRoleBinding` manifest for the service account: + ```yml + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:azure-cloud-provider + rules: + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:azure-cloud-provider + roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider + subjects: + - kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system + ``` + +1. Create these in your cluster using one of the follow command. + + ``` + # kubectl create -f + ``` diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md new file mode 100644 index 00000000000..65596651f3f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md @@ -0,0 +1,40 @@ +--- +title: Networking Requirements for Host Gateway (L2bridge) +--- + + + + + +This section describes how to configure custom Windows clusters that are using *Host Gateway (L2bridge)* mode. + +### Disabling Private IP Address Checks + +If you are using *Host Gateway (L2bridge)* mode and hosting your nodes on any of the cloud services listed below, you must disable the private IP address checks for both your Linux or Windows hosts on startup. To disable this check for each node, follow the directions provided by each service below. + +Service | Directions to disable private IP address checks +--------|------------------------------------------------ +Amazon EC2 | [Disabling Source/Destination Checks](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_NAT_Instance.html#EIP_Disable_SrcDestCheck) +Google GCE | [Enabling IP Forwarding for Instances](https://cloud.google.com/vpc/docs/using-routes#canipforward) (By default, a VM cannot forward a packet originated by another VM) +Azure VM | [Enable or Disable IP Forwarding](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) + +### Cloud-hosted VM Routes Configuration + +If you are using the [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) backend of Flannel, all containers on the same node belong to a private subnet, and traffic routes from a subnet on one node to a subnet on another node through the host network. + +- When worker nodes are provisioned on AWS, virtualization clusters, or bare metal servers, make sure they belong to the same layer 2 subnet. If the nodes don't belong to the same layer 2 subnet, `host-gw` networking will not work. + +- When worker nodes are provisioned on GCE or Azure, they are not on the same layer 2 subnet. Nodes on GCE and Azure belong to a routable layer 3 network. Follow the instructions below to configure GCE and Azure so that the cloud network knows how to route the host subnets on each node. + +To configure host subnet routing on GCE or Azure, first run the following command to find out the host subnets on each worker node: + +```bash +kubectl get nodes -o custom-columns=nodeName:.metadata.name,nodeIP:status.addresses[0].address,routeDestination:.spec.podCIDR +``` + +Then follow the instructions for each cloud provider to configure routing rules for each node: + +Service | Instructions +--------|------------- +Google GCE | For GCE, add a static route for each node: [Adding a Static Route](https://cloud.google.com/vpc/docs/using-routes#addingroute). +Azure VM | For Azure, create a routing table: [Custom Routes: User-defined](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview#user-defined). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md new file mode 100644 index 00000000000..7999a11defc --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md @@ -0,0 +1,46 @@ +--- +title: Windows and Linux Cluster Feature Parity +--- + + + + + +Windows clusters do not share the same feature support as Linux clusters. + +The following chart describes the feature parity between Windows and Linux on Rancher: + +**Component** | **Linux** | **Windows** +--- | --- | --- +**Distributions** | | +RKE | Supported | Supported +RKE2 | Supported | Supported +K3S | Supported | Not Supported +EKS | Supported | Not Supported +GKE | Supported | Not Supported +AKS | Supported | Not Supported +**Rancher Components** | | +Server | Supported | Not Supported +Agent | Supported | Supported +Fleet | Supported | Supported +EKS Operator | Supported | Not Supported +AKS Operator | Not Supported | Not Supported +GKE Operator | Not Supported | Not Supported +Monitoring/Alerting v2 | Supported | Supported +Logging v2 | Supported | Supported +Istio | Supported | Not Supported +Catalog v2 | Supported | Not Supported +OPA | Supported | Not Supported +Longhorn | Supported | Not Supported +CIS Scans | Supported | Not Supported +Backup/Restore Operator | Supported | Not Supported +**CNI / Add-ons** | | +Flannel | Supported | Supported +Canal | Supported | Not Supported +Calico | Supported | Supported (RKE2 Only) +Cilium | Supported | Not Supported +Multus | Supported | Not Supported +Traefik | Supported | Not Supported +NGINX Ingress | Supported | Not Supported + +For updated information on feature support, you may visit [rancher/windows](https://github.com/rancher/windows) on GitHub. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md new file mode 100644 index 00000000000..66e8c274ee5 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md @@ -0,0 +1,307 @@ +--- +title: RKE1 to RKE2 Windows Migration Guidance +--- + + + + + +**Caution:** The contents of this document are not covered under SLA by Rancher Support. Please proceed with caution. + +This document covers how end users can migrate their Windows workloads from RKE1 to RKE2. + +## RKE1 Windows Scheduling + +RKE1 Windows workload scheduling is based on taints and tolerations. + +Every Linux node in an RKE1 Windows cluster, regardless of the role assigned to it, will have have a default taint that prevents workloads to be scheduled on it unless the workload has a toleration configured. This is a major design feature for RKE1 Windows clusters which were designed to only run Windows workloads. + +- Default RKE1 Linux node `NoSchedule` taint: + +```yml +apiVersion: v1 +kind: Node +spec: + ... + taints: + - effect: NoSchedule + key: cattle.io/os + value: linux +``` +
+ +- RKE1 Linux `NoSchedule` toleration for workloads + +The following toleration would allow an end-user workload to schedule on any Linux node of an RKE1 Windows cluster. These tolerations are used for various core Rancher services and workloads. + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` +
+ +- Aligning with best practices, any end-user workloads being run on Linux nodes would be scheduled on those with the worker role only: + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + weight: 100 + ... +``` + +## RKE2 Windows Scheduling + +Based on feedback and requests for hybrid workload support, RKE2 Windows was designed to support both Linux and Windows workloads by default. RKE2 scheduling relies on node selectors by default. This is a marked change from RKE1 as taints and tolerations were not incorporated into RKE2. Node selectors were a critical part of RKE1 Windows clusters, which makes for an easy migration of your workloads. + +## Example Migrations + +### RKE1 to RKE2 Windows Workload + +- Pre-migration RKE1 Windows deployment: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - linux +``` +
+ +- Migrated RKE2 Windows deployment using `NodeAffinity`: + +```yaml +apiVersion: apps/v1 +kind: Deployment +... +spec: + ... + template: + ... + spec: + ... + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows +``` + +### RKE1 Windows Cluster Linux-Only Deployment + +>**Important:** When leveraging node selectors and node affinity, note the following: +> +>- If both `nodeSelector` and `nodeAffinity` are specified, both must be satisfied for the `Pod` to be scheduled onto a node. +>- If you specify multiple `matchExpressions` associated with a single `nodeSelectorTerms`, then the `Pod` can be scheduled onto a node only if all the `matchExpressions` are satisfied. + +
+ +- Pre-migration RKE1 Windows cluster Linux-only deployment targeting RKE1 Linux worker nodes: + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + +``` +
+ +- Migrated RKE2 hybrid cluster Linux-only deployment targeting RKE2 Linux worker nodes using node selectors: + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + nodeSelector: + kubernetes.io/os: "linux" + node-role.kubernetes.io/worker: "true" + ``` +
+ +- Migrated RKE2 hybrid cluster Linux-only deployment targeting RKE2 Linux worker nodes using node affinity: + + ```yaml + apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + ``` +## RKE1 Windows-Supported Windows Server Versions + +### Long-Term Servicing Channel (LTSC) + +- Windows Server 2019 LTSC ✅ Will reach Mainstream EOL on Jan 9, 2024 and Extended EOL on Jan 9, 2029 + +### Semi-Annual Channel (SAC) + +- Windows Server 20H2 SAC ❌ EOL Reached on Aug 9, 2022 +- Windows Server 2004 SAC ❌ EOL Reached on Dec 14, 2021 +- Windows Server 1909 SAC ❌ EOL Reached on May 11, 2021 +- Windows Server 1903 SAC ❌ EOL Reached on Dec 8, 2020 +- Windows Server 1809 SAC ❌ EOL Reached on Nov 10, 2020 + +## RKE2 Windows-Supported Windows Server Versions + +### Long-Term Servicing Channel in RKE2 + +- Windows Server 2019 LTSC ✅ Will reach Mainstream EOL on Jan 9, 2024 and Extended EOL on Jan 9, 2029 +- Windows Server 2022 LTSC ✅ Will reach Mainstream EOL on Oct 13, 2026 and Extended EOL on Oct 13, 2031 + +>**Note:** SAC is not supported in RKE2. + + +For more information, please see the following references: + +- [Windows Server SAC Lifecycle](https://docs.microsoft.com/en-us/lifecycle/products/windows-server) + +- [Windows Server 2022 LTSC Lifecycle](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2022) + +- [Windows Server 2019 LTSC Lifecycle](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2019) + + +## Kubernetes Version Support + +>**Note:** All versions listed below are SLA Supported per the [Rancher v2.6.7 Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/rancher-v2-6-7/). Any version not listed should be assumed as being EOL and not supported under SLA by SUSE. + +### Rancher 2.5 vs. Rancher 2.6 Support Matrix for Windows Clusters + +**RKE1 vs. RKE2 Windows cluster-supported Kubernetes versions:** + +| Kubernetes Versions | RKE1 | RKE2 | +|-------------- |:----: |:----: | +| 1.18 | ✓ | | +| 1.19 | ✓ | | +| 1.20 | ✓ | | +| 1.21 | ✓ | | +| 1.22 | ✓ | ✓ | +| 1.23 | | ✓ | +| 1.24 | | ✓ | +| 1.25+ | | ✓ | + + +### Rancher 2.5 vs. Rancher 2.6 Supported Kubernetes Versions for Provisioning RKE1 and RKE2 Windows Clusters + +| Rancher Versions | Kubernetes Versions | RKE1 | RKE2 | +|:-----------------------: |:------------------------: |:----: |:----: | +| 2.5 - RKE1 Provisioning | 1.18 1.19 1.20 | ✓ | | +| 2.6 - RKE1 Provisioning | 1.18 1.19 1.20 1.21 1.22 | ✓ | | +| 2.6 - RKE2 Provisioning | 1.22 1.23 1.24 1.25+ | | ✓ | + + +## Guiding Migrations of Workloads to RKE2 Windows + +Referencing the tables in [Rancher 2.5 vs. Rancher 2.6 Support Matrix for Windows Clusters](#rancher-25-vs-rancher-26-support-matrix-for-windows-clusters) and [Rancher 2.5 vs. Rancher 2.6 Supported Kubernetes Versions for Provisioning RKE1 and RKE2 Windows Clusters](#rancher-25-vs-rancher-26-supported-kubernetes-versions-for-provisioning-rke1-and-rke2-windows-clusters), you will find the overlap in Kubernetes versions between RKE1 and RKE2 occurs in 1.22. This will be the base version required to migrate RKE1 Windows workloads when following the Rancher recommended approach. + +### In-Place Upgrade of Rancher 2.5 + +1. Upgrade the Rancher version to v2.6.5+. +1. Upgrade the RKE1 Windows downstream cluster(s) to RKE1 v1.22 using the latest available patch version. +1. Provision a new RKE2 Windows downstream cluster using RKE2 v1.22 using the matching patch version that the RKE1 Windows cluster is at. +1. Begin the migration of the Windows workloads from RKE1 to RKE2 clusters. +1. Perform validation tests to ensure that there has been no functionality loss or change when migrating your application from RKE1 to RKE2. +1. After successful validation tests have occurred, you can opt to upgrade your RKE2 1.22.x cluster to a new minor version such as 1.23 or 1.24. + + +### Migrating Windows Workloads to a New Rancher Environment + +>**Important:** To perform either of the following options requires Rancher v2.6.5 or above. + +**When using matching Kubernetes patch versions for RKE1 and RKE2:** + +1. Provision a new RKE2 Windows downstream cluster using RKE2 v1.22 using the matching patch version that the RKE1 Windows cluster is at. +1. Begin the migration of the Windows workloads from RKE1 to RKE2 clusters. +1. Perform validation tests to ensure that there has been no functionality loss or change when migrating your application from RKE1 to RKE2. +1. After successful validation tests have occurred, you can opt to upgrade your RKE2 1.22.x cluster to a new minor version such as 1.23 or 1.24. + + +**When using a newer Kubernetes patch version for RKE2:** + +1. Provision a new RKE2 Windows downstream cluster using RKE2 v1.23 or v1.24. +1. Begin the migration of the Windows workloads from RKE1 to RKE2 clusters. +1. Perform validation tests to ensure that there has been no functionality loss or change when migrating your application from RKE1 to RKE2. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md new file mode 100644 index 00000000000..097a4d26e34 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md @@ -0,0 +1,39 @@ +--- +title: ConfigMaps +--- + + + + + +While most types of Kubernetes secrets store sensitive information, [ConfigMaps](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) store general configuration information, such as a group of config files. Because ConfigMaps don't store sensitive information, they can be updated automatically, and therefore don't require their containers to be restarted following update (unlike most secret types, which require manual updates and a container restart to take effect). + +ConfigMaps accept key value pairs in common string formats, like config files or JSON blobs. After you upload a config map, any workload can reference it as either an environment variable or a volume mount. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that has the workload that should reference a ConfigMap and click **Explore**. +1. In the left navigation bar, click **More Resources > Core > ConfigMaps**. +1. Click **Create**. +1. Enter a **Name** for the Config Map. + + :::note + + Kubernetes classifies ConfigMaps as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a project or namespace can have duplicate names. Therefore, to prevent conflicts, your ConfigMaps must have a unique name among the other certificates, registries, and secrets within your workspace. + + ::: + +1. Select the **Namespace** you want to add Config Map to. + +1. On the **Data** tab, add a key-value pair to your ConfigMap. Add as many values as you need. You can add multiple key value pairs to the ConfigMap by copying and pasting. Alternatively, use **Read from File** to add the data. Note: If you need to store sensitive data, [use a secret](secrets.md), not a ConfigMap. +1. Click **Create**. + +**Result:** Your ConfigMap is added to the namespace. You can view it in the Rancher UI from the **Resources > Config Maps** view. + +## What's Next? + +Now that you have a ConfigMap added to a namespace, you can add it to a workload that you deploy from the namespace of origin. You can use the ConfigMap to specify information for you application to consume, such as: + +- Application environment variables. +- Specifying parameters for a Volume mounted to the workload. + +For more information on adding ConfigMaps to a workload, see [Deploying Workloads](workloads-and-pods/deploy-workloads.md). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md new file mode 100644 index 00000000000..5415a75ca9c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md @@ -0,0 +1,33 @@ +--- +title: Services +--- + + + + + +Pod configuration is managed by Deployments, StatefulSets and Daemonsets, whereas services direct traffic to pods using selectors. + +For every workload (with at least one port configured) created, a complementing Service Discovery entry is created. This Service Discovery entry enables DNS resolution for the workload's pods using the following naming convention: +`..svc.cluster.local`. + +You can create additional services so that a given namespace resolves with one or more external IP addresses, an external hostname, an alias to another DNS record, other workloads, or a set of pods that match a selector that you create. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a service and click **Explore**. +1. Click **Service Discovery > Services**. +1. Click **Create**. +1. Choose the type of service you want to create. +1. Select a **Namespace** from the drop-down list. +1. Enter a **Name** for the service. This name is used for DNS resolution. +1. Fill out the rest of the form. For help, refer to the upstream Kubernetes documentation about [services.](https://kubernetes.io/docs/concepts/services-networking/service/) +1. Click **Create**. + +**Result:** A new service is created. + +- You can view the record by from the project's **Service Discovery** tab. +- When you visit the new DNS name for the new record that you created (`..svc.cluster.local`), it resolves the chosen namespace. + +## Related Links + +- [Adding entries to Pod /etc/hosts with HostAliases](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md new file mode 100644 index 00000000000..27f41079923 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md @@ -0,0 +1,42 @@ +--- +title: Encrypting HTTP Communication +description: Learn how to add an SSL (Secure Sockets Layer) certificate or TLS (Transport Layer Security) certificate +--- + + + + + +When you create an ingress within Rancher/Kubernetes, you must provide it with a secret that includes a TLS private key and certificate, which are used to encrypt and decrypt communications that come through the ingress. You can make certificates available for ingress use by adding the certificate to the ingress deployment. + +:::note Prerequisite: + +You must have a TLS private key and certificate available to upload. + +::: + +### 1. Create a Secret + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to deploy your ingress and click **More Resources > Core > Secrets**. +1. Click **Create**. +1. Click **TLS Certificate**. +1. Enter a name for the secret. Note: Your secret must have a unique name among the other certificates, registries, and secrets within your project/workspace. +1. In the **Private Key** field, either copy and paste your certificate's private key into the text box (include the header and footer), or click **Read from a file** to browse to the private key on your file system. If possible, we recommend using **Read from a file** to reduce likelihood of error. Note: Private key files end with an extension of `.key`. +1. In the **Certificate** field, either copy and paste your certificate into the text box (include the header and footer), or click **Read from a file** to browse to the certificate on your file system. If possible, we recommend using **Read from a file** to reduce likelihood of error. Note: Certificate files end with an extension of `.crt`. +1. Click **Create**. + +### 2. Add the Secret to an Ingress + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to deploy your ingress and click **Service Discovery > Ingresses**. +1. Click **Create**. +1. Select the **Namespace** of the ingress. +1. Enter a **Name** for the ingress. +1. In the **Certificates** tab, select the secret containing your certificate and private key. +1. Click **Create**. + +## What's Next? + +Now you can add the certificate when launching an ingress within the current project or namespace. For more information, see [Adding Ingress](load-balancer-and-ingress-controller/add-ingresses.md). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md new file mode 100644 index 00000000000..18729cbca41 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md @@ -0,0 +1,43 @@ +--- +title: Background Information on HPAs +--- + + + + + +The [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) (HPA) is a Kubernetes feature that allows you to configure your cluster to automatically scale the services it's running up or down. This section provides explanation on how HPA works with Kubernetes. + +## Why Use Horizontal Pod Autoscaler? + +Using HPA, you can automatically scale the number of pods within a replication controller, deployment, or replica set up or down. HPA automatically scales the number of pods that are running for maximum efficiency. Factors that affect the number of pods include: + +- A minimum and maximum number of pods allowed to run, as defined by the user. +- Observed CPU/memory use, as reported in resource metrics. +- Custom metrics provided by third-party metrics application like Prometheus, Datadog, etc. + +HPA improves your services by: + +- Releasing hardware resources that would otherwise be wasted by an excessive number of pods. +- Increase/decrease performance as needed to accomplish service level agreements. + +## How HPA Works + +![HPA Schema](/img/horizontal-pod-autoscaler.jpg) + +HPA is implemented as a control loop, with a period controlled by the `kube-controller-manager` flags below: + +Flag | Default | Description | +---------|----------|----------| + `--horizontal-pod-autoscaler-sync-period` | `30s` | How often HPA audits resource/custom metrics in a deployment. + `--horizontal-pod-autoscaler-downscale-delay` | `5m0s` | Following completion of a downscale operation, how long HPA must wait before launching another downscale operations. + `--horizontal-pod-autoscaler-upscale-delay` | `3m0s` | Following completion of an upscale operation, how long HPA must wait before launching another upscale operation. + + +For full documentation on HPA, refer to the [Kubernetes Documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). + +## Horizontal Pod Autoscaler API Objects + +HPA is an API resource in the Kubernetes `autoscaling` API group. The current stable version is `autoscaling/v1`, which only includes support for CPU autoscaling. To get additional support for scaling based on memory and custom metrics, use the beta version instead: `autoscaling/v2beta1`. + +For more information about the HPA API object, see the [HPA GitHub Readme](https://github.com/kubernetes/design-proposals-archive/blob/main/autoscaling/horizontal-pod-autoscaler.md#horizontalpodautoscaler-object). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md new file mode 100644 index 00000000000..573119909e8 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md @@ -0,0 +1,208 @@ +--- +title: Managing HPAs with kubectl +--- + + + + + +This section describes HPA management with `kubectl`. This document has instructions for how to: + +- Create an HPA +- Get information on HPAs +- Delete an HPA +- Configure your HPAs to scale with CPU or memory utilization +- Configure your HPAs to scale using custom metrics, if you use a third-party tool such as Prometheus for metrics + + +You can create, view, and delete HPAs from the Rancher UI. You can also configure them to scale based on CPU or memory usage from the Rancher UI. For more information, refer to [Managing HPAs with the Rancher UI](./manage-hpas-with-ui.md). For scaling HPAs based on other metrics than CPU or memory, you still need `kubectl`. + +## Basic kubectl Command for Managing HPAs + +If you have an HPA manifest file, you can create, manage, and delete HPAs using `kubectl`: + +- Creating HPA + + - With manifest: `kubectl create -f ` + + - Without manifest (Just support CPU): `kubectl autoscale deployment hello-world --min=2 --max=5 --cpu-percent=50` + +- Getting HPA info + + - Basic: `kubectl get hpa hello-world` + + - Detailed description: `kubectl describe hpa hello-world` + +- Deleting HPA + + - `kubectl delete hpa hello-world` + +## HPA Manifest Definition Example + +The HPA manifest is the config file used for managing an HPA with `kubectl`. + +The following snippet demonstrates use of different directives in an HPA manifest. See the list below the sample to understand the purpose of each directive. + +```yml +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: hello-world +spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi +``` + + +Directive | Description +---------|----------| + `apiVersion: autoscaling/v2beta1` | The version of the Kubernetes `autoscaling` API group in use. This example manifest uses the beta version, so scaling by CPU and memory is enabled. | + `name: hello-world` | Indicates that HPA is performing autoscaling for the `hello-word` deployment. | + `minReplicas: 1` | Indicates that the minimum number of replicas running can't go below 1. | + `maxReplicas: 10` | Indicates the maximum number of replicas in the deployment can't go above 10. + `targetAverageUtilization: 50` | Indicates the deployment will scale pods up when the average running pod uses more than 50% of its requested CPU. + `targetAverageValue: 100Mi` | Indicates the deployment will scale pods up when the average running pod uses more that 100Mi of memory. +
+ +## Configuring HPA to Scale Using Resource Metrics (CPU and Memory) + +Clusters created in Rancher v2.0.7 and higher have all the requirements needed (metrics-server and Kubernetes cluster configuration) to use Horizontal Pod Autoscaler. + +Run the following commands to check if metrics are available in your installation: + +``` +$ kubectl top nodes +NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% +node-controlplane 196m 9% 1623Mi 42% +node-etcd 80m 4% 1090Mi 28% +node-worker 64m 3% 1146Mi 29% +$ kubectl -n kube-system top pods +NAME CPU(cores) MEMORY(bytes) +canal-pgldr 18m 46Mi +canal-vhkgr 20m 45Mi +canal-x5q5v 17m 37Mi +canal-xknnz 20m 37Mi +kube-dns-7588d5b5f5-298j2 0m 22Mi +kube-dns-autoscaler-5db9bbb766-t24hw 0m 5Mi +metrics-server-97bc649d5-jxrlt 0m 12Mi +$ kubectl -n kube-system logs -l k8s-app=metrics-server +I1002 12:55:32.172841 1 heapster.go:71] /metrics-server --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true +I1002 12:55:32.172994 1 heapster.go:72] Metrics Server version v0.2.1 +I1002 12:55:32.173378 1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default.svc" and version +I1002 12:55:32.173401 1 configs.go:62] Using kubelet port 10250 +I1002 12:55:32.173946 1 heapster.go:128] Starting with Metric Sink +I1002 12:55:32.592703 1 serving.go:308] Generated self-signed cert (apiserver.local.config/certificates/apiserver.crt, apiserver.local.config/certificates/apiserver.key) +I1002 12:55:32.925630 1 heapster.go:101] Starting Heapster API server... +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] listing is available at https:///swaggerapi +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] https:///swaggerui/ is mapped to folder /swagger-ui/ +I1002 12:55:32.928597 1 serve.go:85] Serving securely on 0.0.0.0:443 +``` + + +## Configuring HPA to Scale Using Custom Metrics with Prometheus + +You can configure HPA to autoscale based on custom metrics provided by third-party software. The most common use case for autoscaling using third-party software is based on application-level metrics (i.e., HTTP requests per second). HPA uses the `custom.metrics.k8s.io` API to consume these metrics. This API is enabled by deploying a custom metrics adapter for the metrics collection solution. + +For this example, we are going to use [Prometheus](https://prometheus.io/). We are beginning with the following assumptions: + +- Prometheus is deployed in the cluster. +- Prometheus is configured correctly and collecting proper metrics from pods, nodes, namespaces, etc. +- Prometheus is exposed at the following URL and port: `http://prometheus.mycompany.io:80` + +Prometheus is available for deployment in the Rancher v2.0 catalog. Deploy it from Rancher catalog if it isn't already running in your cluster. + +For HPA to use custom metrics from Prometheus, package [k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) is required in the `kube-system` namespace of your cluster. To install `k8s-prometheus-adapter`, we are using the Helm chart available at [banzai-charts](https://github.com/banzaicloud/banzai-charts). + +1. Initialize Helm in your cluster. + ``` + # kubectl -n kube-system create serviceaccount tiller + kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller + helm init --service-account tiller + ``` + +1. Clone the `banzai-charts` repo from GitHub: + ``` + # git clone https://github.com/banzaicloud/banzai-charts + ``` + +1. Install the `prometheus-adapter` chart, specifying the Prometheus URL and port number. + ``` + # helm install --name prometheus-adapter banzai-charts/prometheus-adapter --set prometheus.url="http://prometheus.mycompany.io",prometheus.port="80" --namespace kube-system + ``` + +1. Check that `prometheus-adapter` is running properly. Check the service pod and logs in the `kube-system` namespace. + + 1. Check that the service pod is `Running`. Enter the following command. + ``` + # kubectl get pods -n kube-system + ``` + From the resulting output, look for a status of `Running`. + ``` + NAME READY STATUS RESTARTS AGE + ... + prometheus-adapter-prometheus-adapter-568674d97f-hbzfx 1/1 Running 0 7h + ... + ``` + 1. Check the service logs to make sure the service is running correctly by entering the command that follows. + ``` + # kubectl logs prometheus-adapter-prometheus-adapter-568674d97f-hbzfx -n kube-system + ``` + Then review the log output to confirm the service is running. + +
+ Prometheus Adaptor Logs + + ... + I0724 10:18:45.696679 1 round_trippers.go:436] GET https://10.43.0.1:443/api/v1/namespaces/default/pods?labelSelector=app%3Dhello-world 200 OK in 2 milliseconds + I0724 10:18:45.696695 1 round_trippers.go:442] Response Headers: + I0724 10:18:45.696699 1 round_trippers.go:445] Date: Tue, 24 Jul 2018 10:18:45 GMT + I0724 10:18:45.696703 1 round_trippers.go:445] Content-Type: application/json + I0724 10:18:45.696706 1 round_trippers.go:445] Content-Length: 2581 + I0724 10:18:45.696766 1 request.go:836] Response Body: {"kind":"PodList","apiVersion":"v1","metadata":{"selfLink":"/api/v1/namespaces/default/pods","resourceVersion":"6237"},"items":[{"metadata":{"name":"hello-world-54764dfbf8-q6l82","generateName":"hello-world-54764dfbf8-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/hello-world-54764dfbf8-q6l82","uid":"484cb929-8f29-11e8-99d2-067cac34e79c","resourceVersion":"4066","creationTimestamp":"2018-07-24T10:06:50Z","labels":{"app":"hello-world","pod-template-hash":"1032089694"},"annotations":{"cni.projectcalico.org/podIP":"10.42.0.7/32"},"ownerReferences":[{"apiVersion":"extensions/v1beta1","kind":"ReplicaSet","name":"hello-world-54764dfbf8","uid":"4849b9b1-8f29-11e8-99d2-067cac34e79c","controller":true,"blockOwnerDeletion":true}]},"spec":{"volumes":[{"name":"default-token-ncvts","secret":{"secretName":"default-token-ncvts","defaultMode":420}}],"containers":[{"name":"hello-world","image":"rancher/hello-world","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"default-token-ncvts","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"34.220.18.140","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}]},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:54Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"}],"hostIP":"34.220.18.140","podIP":"10.42.0.7","startTime":"2018-07-24T10:06:50Z","containerStatuses":[{"name":"hello-world","state":{"running":{"startedAt":"2018-07-24T10:06:54Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"rancher/hello-world:latest","imageID":"docker-pullable://rancher/hello-world@sha256:4b1559cb4b57ca36fa2b313a3c7dde774801aa3a2047930d94e11a45168bc053","containerID":"docker://cce4df5fc0408f03d4adf82c90de222f64c302bf7a04be1c82d584ec31530773"}],"qosClass":"Burstable"}}]} + I0724 10:18:45.699525 1 api.go:74] GET http://prometheus-server.prometheus.34.220.18.140.xip.io/api/v1/query?query=sum%28rate%28container_fs_read_seconds_total%7Bpod_name%3D%22hello-world-54764dfbf8-q6l82%22%2Ccontainer_name%21%3D%22POD%22%2Cnamespace%3D%22default%22%7D%5B5m%5D%29%29+by+%28pod_name%29&time=1532427525.697 200 OK + I0724 10:18:45.699620 1 api.go:93] Response Body: {"status":"success","data":{"resultType":"vector","result":[{"metric":{"pod_name":"hello-world-54764dfbf8-q6l82"},"value":[1532427525.697,"0"]}]}} + I0724 10:18:45.699939 1 wrap.go:42] GET /apis/custom.metrics.k8s.io/v1beta1/namespaces/default/pods/%2A/fs_read?labelSelector=app%3Dhello-world: (12.431262ms) 200 [[kube-controller-manager/v1.10.1 (linux/amd64) kubernetes/d4ab475/system:serviceaccount:kube-system:horizontal-pod-autoscaler] 10.42.0.0:24268] + I0724 10:18:51.727845 1 request.go:836] Request Body: {"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","metadata":{"creationTimestamp":null},"spec":{"nonResourceAttributes":{"path":"/","verb":"get"},"user":"system:anonymous","group":["system:unauthenticated"]},"status":{"allowed":false}} + ... +
+ + + +1. Check that the metrics API is accessible from kubectl. + + - If you are accessing the cluster directly, enter your Server URL in the kubectl config in the following format: `https://:6443`. + ``` + # kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 + ``` + If the API is accessible, you should receive output that's similar to what follows. + +
+ API Response + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
+ + - If you are accessing the cluster through Rancher, enter your Server URL in the kubectl config in the following format: `https:///k8s/clusters/`. Add the suffix `/k8s/clusters/` to API path. + ``` + # kubectl get --raw /k8s/clusters//apis/custom.metrics.k8s.io/v1beta1 + ``` + If the API is accessible, you should receive output that's similar to what follows. + +
+ API Response + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md new file mode 100644 index 00000000000..adc57ae88ee --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md @@ -0,0 +1,55 @@ +--- +title: Managing HPAs with the Rancher UI +--- + + + + + +The Rancher UI supports creating, managing, and deleting HPAs. You can configure CPU or memory usage as the metric that the HPA uses to scale. + +If you want to create HPAs that scale based on other metrics than CPU and memory, refer to [Configuring HPA to Scale Using Custom Metrics with Prometheus](./manage-hpas-with-kubectl.md#configuring-hpa-to-scale-using-custom-metrics-with-prometheus). + +## Creating an HPA + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster you want to create an HPA in and click **Explore**. +1. In the left navigation bar, click **Service Discovery > HorizontalPodAutoscalers**. +1. Click **Create**. +1. Select a **Namespace** for the HPA. +1. Enter a **Name** for the HPA. +1. Select a **Target Reference** as scale target for the HPA. +1. Specify the **Minimum Replicas** and **Maximum Replicas** for the HPA. +1. Configure the metrics for the HPA. You can choose memory or CPU usage as the metric that will cause the HPA to scale the service up or down. In the **Quantity** field, enter the percentage of the workload's memory or CPU usage that will cause the HPA to scale the service. To configure other HPA metrics, including metrics available from Prometheus, you need to [manage HPAs using kubectl](./manage-hpas-with-kubectl.md#configuring-hpa-to-scale-using-custom-metrics-with-prometheus). + +1. Click **Create** to create the HPA. + +:::note Result: + +The HPA is deployed to the chosen namespace. You can view the HPA's status from the project's Resources > HPA view. + +::: + +## Get HPA Metrics and Status + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that has the HPA and click **Explore**. +1. In the left navigation bar, click **Service Discovery > HorizontalPodAutoscalers**. The **HorizontalPodAutoscalers** page shows the number of current replicas. + +For more detailed metrics and status of a specific HPA, click the name of the HPA. This leads to the HPA detail page. + + +## Deleting an HPA + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that has the HPA you want to delete and click **Explore**. +1. In the left navigation bar, click **Service Discovery > HorizontalPodAutoscalers**. +1. Click **Resources > HPA**. +1. Find the HPA which you would like to delete and click **⋮ > Delete**. +1. Click **Delete** to confirm. + +:::note Result: + +The HPA is deleted from the current cluster. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md new file mode 100644 index 00000000000..6607274c4ee --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md @@ -0,0 +1,534 @@ +--- +title: Testing HPAs with kubectl +--- + + + + + +This document describes how to check the status of your HPAs after scaling them up or down with your load testing tool. For information on how to check the status from the Rancher UI (at least version 2.3.x), refer to [Managing HPAs with the Rancher UI](./manage-hpas-with-kubectl.md). + +For HPA to work correctly, service deployments should have resources request definitions for containers. Follow this hello-world example to test if HPA is working correctly. + +1. Configure `kubectl` to connect to your Kubernetes cluster. + +1. Copy the `hello-world` deployment manifest below. + +
+ Hello World Manifest + + ``` + apiVersion: apps/v1beta2 + kind: Deployment + metadata: + labels: + app: hello-world + name: hello-world + namespace: default + spec: + replicas: 1 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + resources: + requests: + cpu: 500m + memory: 64Mi + ports: + - containerPort: 80 + protocol: TCP + restartPolicy: Always + --- + apiVersion: v1 + kind: Service + metadata: + name: hello-world + namespace: default + spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-world + ``` + +
+ +1. Deploy it to your cluster. + + ``` + # kubectl create -f + ``` + +1. Copy one of the HPAs below based on the metric type you're using: + +
+ Hello World HPA: Resource Metrics + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 1000Mi + ``` + +
+
+ Hello World HPA: Custom Metrics + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi + - type: Pods + pods: + metricName: cpu_system + targetAverageValue: 20m + ``` + +
+ +1. View the HPA info and description. Confirm that metric data is shown. + +
+ Resource Metrics + + 1. Enter the following commands. + ``` + # kubectl get hpa + NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE + hello-world Deployment/hello-world 1253376 / 100Mi, 0% / 50% 1 10 1 6m + # kubectl describe hpa + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 20:21:16 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 1253376 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+
+ Custom Metrics + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive the output that follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:36:28 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 3514368 / 100Mi + "cpu_system" on pods: 0 / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+ +1. Generate a load for the service to test that your pods autoscale as intended. You can use any load-testing tool (Hey, Gatling, etc.), but we're using [Hey](https://github.com/rakyll/hey). + +1. Test that pod autoscaling works as intended.

+ **To Test Autoscaling Using Resource Metrics:** + +
+ Upscale to 2 Pods: CPU Usage Up to Target + + Use your load testing tool to scale up to two pods based on CPU Usage. + + 1. View your HPA. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10928128 / 100Mi + resource cpu on pods (as a percentage of request): 56% (280m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. Enter the following command to confirm you've scaled to two pods. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows: + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-k8ph2 1/1 Running 0 1m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ Upscale to 3 pods: CPU Usage Up to Target + + Use your load testing tool to upscale to 3 pods based on CPU usage with `horizontal-pod-autoscaler-upscale-delay` set to 3 minutes. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 9424896 / 100Mi + resource cpu on pods (as a percentage of request): 66% (333m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 4m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + ``` + 2. Enter the following command to confirm three pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-f46kh 0/1 Running 0 1m + hello-world-54764dfbf8-k8ph2 1/1 Running 0 5m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ Downscale to 1 Pod: All Metrics Below Target + + Use your load testing to scale down to 1 pod when all metrics are below target for `horizontal-pod-autoscaler-downscale-delay` (5 minutes by default). + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10070016 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 6m horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 1s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + +
+ + **To Test Autoscaling Using Custom Metrics:** + +
+ Upscale to 2 Pods: CPU Usage Up to Target + + Use your load testing tool to upscale two pods based on CPU usage. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8159232 / 100Mi + "cpu_system" on pods: 7m / 20m + resource cpu on pods (as a percentage of request): 64% (321m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. Enter the following command to confirm two pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ Upscale to 3 Pods: CPU Usage Up to Target + + Use your load testing tool to scale up to three pods when the cpu_system usage limit is up to target. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows: + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3s horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + ``` + 1. Enter the following command to confirm three pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows: + ``` + # kubectl get pods + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ Upscale to 4 Pods: CPU Usage Up to Target + + Use your load testing tool to upscale to four pods based on CPU usage. `horizontal-pod-autoscaler-upscale-delay` is set to three minutes by default. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 4s horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + ``` + 1. Enter the following command to confirm four pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-2p9xb 1/1 Running 0 5m + hello-world-54764dfbf8-5pfdr 1/1 Running 0 2m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ Downscale to 1 Pod: All Metrics Below Target + + Use your load testing tool to scale down to one pod when all metrics below target for `horizontal-pod-autoscaler-downscale-delay`. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive similar output to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8101888 / 100Mi + "cpu_system" on pods: 8m / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 8m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + 1. Enter the following command to confirm a single pods is running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md new file mode 100644 index 00000000000..fb19bea8379 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md @@ -0,0 +1,182 @@ +--- +title: Kubernetes Registry and Container Image Registry +description: Learn about the container image registry and Kubernetes registry, their use cases, and how to use a private registry with the Rancher UI +--- + + + + + +Registries are Kubernetes secrets containing credentials used to authenticate with [private container registries](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). + +The word "registry" can mean two things, depending on whether it is used to refer to a container or Kubernetes registry: + +- A **Container image registry** (formerly "Docker registry") contains container images that you can pull and deploy. The registry is a stateless, scalable server side application that stores and lets you distribute container images. +- The **Kubernetes registry** is an image pull secret that your deployment uses to authenticate with an image registry. + +Deployments use the Kubernetes registry secret to authenticate with a private image registry and then pull a container image hosted on it. + +Currently, deployments pull the private registry credentials automatically only if the workload is created in the Rancher UI and not when it is created via kubectl. + +## Creating a Registry in Namespaces + +:::note Prerequisite: + +You must have an available private registry already deployed. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a registry and click **Explore**. +1. In the left navigation, click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Click **Create**. +1. Click **Registry**. +1. Enter a **Name** for the registry. + + :::note + + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a project or namespace can have duplicate names. Therefore, to prevent conflicts, your registry must have a unique name among all secrets within your workspace. + + ::: + +1. Select a namespace for the registry. +1. Select the website that hosts your private registry. Then enter credentials that authenticate with the registry. For example, if you use DockerHub, provide your DockerHub username and password. +1. Click **Save**. + +**Result:** + +- Your secret is added to the namespace you chose. +- You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. +- Any workload that you create in the Rancher UI will have the credentials to access the registry if the workload is within the registry's scope. + +## Creating a Registry in Projects + +:::note Prerequisites: + +You must have an available private registry already deployed. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +Before v2.6, secrets were required to be in a project scope. Projects are no longer required, and you may use the namespace scope instead. As a result, the Rancher UI was updated to reflect this new functionality. However, you may still create a project-scoped registry if desired. Use the following steps to do so: + +1. In the upper left corner, click **☰ > Global Settings** in the dropdown. +1. Click **Feature Flags**. +1. Go to the `legacy` feature flag and click **Activate**. +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a registry and click **Explore**. +1. In the left navigation, click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Click **Create**. +1. Click **Registry**. +1. In the top navigation bar, filter to see only one project. +1. Enter a **Name** for the registry. + + :::note + + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a project or namespace can have duplicate names. Therefore, to prevent conflicts, your registry must have a unique name among all secrets within your workspace. + + ::: + +1. Select a namespace for the registry. +1. Select the website that hosts your private registry. Then enter credentials that authenticate with the registry. For example, if you use DockerHub, provide your DockerHub username and password. +1. Click **Save**. + +**Result:** + +- Your secret is added to the individual project you chose. +- You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. +- Any workload that you create in the Rancher UI will have the credentials to access the registry if the workload is within the registry's scope. + +:::note + +Project-scoped registries on the local cluster are only visible when a single project is selected. + +::: + +## Using a Private Registry + +You can deploy a workload with an image from a private registry through the Rancher UI, or with `kubectl`. + +### Using the Private Registry with the Rancher UI + +To deploy a workload with an image from your private registry, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to deploy a workload and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Select the type of workload you want to create. +1. Enter a unique name for the workload and choose a namespace. +1. In the **Container Image** field, enter the URL of the path to the image in your private registry. For example, if your private registry is on Quay.io, you could use `quay.io//`. +1. Click **Create**. + +**Result:** Your deployment should launch, authenticate using the private registry credentials you added in the Rancher UI, and pull the container image that you specified. + +### Using the Private Registry with kubectl + +When you create the workload using `kubectl`, you need to configure the pod so that its YAML has the path to the image in the private registry. You also have to create and reference the registry secret because the pod only automatically gets access to the private registry credentials if it is created in the Rancher UI. + +The secret has to be created in the same namespace where the workload gets deployed. + +Below is an example `pod.yml` for a workload that uses an image from a private registry. In this example, the pod uses an image from Quay.io, and the .yml specifies the path to the image. The pod authenticates with the registry using credentials stored in a Kubernetes secret called `testquay`, which is specified in `spec.imagePullSecrets` in the `name` field: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: private-reg +spec: + containers: + - name: private-reg-container + image: quay.io// + imagePullSecrets: + - name: testquay +``` + +In this example, the secret named `testquay` is in the default namespace. + +You can use `kubectl` to create the secret with the private registry credentials. This command creates the secret named `testquay`: + +``` +kubectl create secret docker-registry testquay \ + --docker-server=quay.io \ + --docker-username= \ + --docker-password= +``` + +To see how the secret is stored in Kubernetes, you can use this command: + +``` +kubectl get secret testquay --output="jsonpath={.data.\.dockerconfigjson}" | base64 --decode +``` + +The result looks like this: + +``` +{"auths":{"quay.io":{"username":"","password":"","auth":"c291bXlhbGo6dGVzdGFiYzEyMw=="}}} +``` + +After the workload is deployed, you can check if the image was pulled successfully: + +``` +kubectl get events +``` +The result should look like this: +``` +14s Normal Scheduled Pod Successfully assigned default/private-reg2 to minikube +11s Normal Pulling Pod pulling image "quay.io//" +10s Normal Pulled Pod Successfully pulled image "quay.io//" +``` + +For more information, refer to the Kubernetes documentation on [creating a pod that uses your secret.](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md new file mode 100644 index 00000000000..c102ee9550f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md @@ -0,0 +1,22 @@ +--- +title: Adding Ingresses +description: Ingresses can be added for workloads to provide load balancing, SSL termination and host/path-based routing. Learn how to add Rancher ingress +--- + + + + + +Ingresses can be added for workloads to provide load balancing, SSL termination and host/path based routing. When using ingresses in a project, you can program the ingress hostname to an external DNS by setting up a Global DNS entry. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you want to add an ingress to and click **Explore**. +1. Click **Service Discovery > Ingresses**. +1. Click **Create**. +1. Select an existing **Namespace** from the drop-down list. +1. Enter a **Name** for the ingress. +1. [Create](./ingress-configuration.md) ingress forwarding **Rules**. If any of your ingress rules handle requests for encrypted ports, add a certificate to encrypt/decrypt communications. +1. **Optional:** click **Add Rule** to create additional ingress rules. For example, after you create ingress rules to direct requests for your hostname, you'll likely want to create a default backend to handle 404s. +1. Click **Create** at the bottom right. + +**Result:** Your ingress is added to the project. The ingress begins enforcing your ingress rules. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md new file mode 100644 index 00000000000..11bdcffc4c6 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md @@ -0,0 +1,45 @@ +--- +title: Ingress Configuration +description: Ingress configuration +--- + + + + + +:::note + +For Kubernetes v1.21 and up, the NGINX Ingress controller no longer runs in hostNetwork by default. It instead uses hostPorts for port 80 and port 443, so you can configure the admission webhook to be accessible only through the ClusterIP. This ensures that the webhook is only accessible from within the cluster. + +Because of this change to the controller, the default RKE1 behavior no longer sets `hostNetwork` to `true`. However, you must set `hostNetwork` to `true` for TCP- and UDP-based Services to work. To do so, [edit](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#editing-clusters-with-yaml) the cluster's YAML and follow the steps in the [official RKE1 doccumention](https://rke.docs.rancher.com/config-options/add-ons/ingress-controllers#configuring-network-options). + +::: + +## Specify a hostname to use + +If you use this option, Ingress routes requests for a hostname to the service or workload that you specify. + +1. Specify a path of type `Prefix` and specify a path such as `/`. +1. Add a **Target Service**. +1. **Optional:** If you want to specify a workload or service when a request is sent to a particular hostname path, add a **Path** for the target. For example, if you want requests for `www.mysite.com/contact-us` to be sent to a different service than `www.mysite.com`, enter `/contact-us` in the **Path** field. Typically, the first rule that you create does not include a path. +1. Enter the **Port** number that each target operates on. + +## Certificates + +:::note + +You must have an SSL certificate that Ingress can use to encrypt and decrypt communications. For more information, see [Adding SSL Certificates](../encrypt-http-communication.md). + +::: + +1. To create an Ingress controller, click the **Certificates** tab. +1. Click **Add Certificate**. +1. Select a **Certificate - Secret Name** from the drop-down list. +1. Enter the host using encrypted communication. +1. To add additional hosts that use the certificate, click **Add Hosts**. + +## Labels and Annotations + +Add [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) and/or [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to provide metadata for your Ingress controller. + +For a list of annotations available for use, see the [Nginx Ingress Controller Documentation](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md new file mode 100644 index 00000000000..e0d25c169e6 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md @@ -0,0 +1,72 @@ +--- +title: "Layer 4 and Layer 7 Load Balancing" +description: "Kubernetes supports load balancing in two ways: Layer-4 Load Balancing and Layer-7 Load Balancing. Learn about the support for each way in different deployments" +--- + + + + + +Kubernetes supports load balancing in two ways: Layer-4 Load Balancing and Layer-7 Load Balancing. + +## Layer-4 Load Balancer + +Layer-4 load balancer (or the external load balancer) forwards traffic to Nodeports. Layer-4 load balancer allows you to forward both HTTP and TCP traffic. + +Often, the Layer-4 load balancer is supported by the underlying cloud provider, so when you deploy RKE clusters on bare-metal servers and vSphere clusters, Layer-4 load balancer is not supported. However, a single [globally managed config-map](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) can be used to expose services on NGINX or third-party ingress. + +:::note + +It is possible to deploy a cluster with a non-cloud load balancer, such as [MetalLB.](https://metallb.universe.tf/) However, that use case is more advanced than the Layer-4 load balancer supported by a cloud provider, and it is not configurable in Rancher or RKE. + +::: + +### Support for Layer-4 Load Balancing + +Support for layer-4 load balancer varies based on the underlying cloud provider. + +Cluster Deployment | Layer-4 Load Balancer Support +----------------------------------------------|-------------------------------- +Amazon EKS | Supported by AWS cloud provider +Google GKE | Supported by GCE cloud provider +Azure AKS | Supported by Azure cloud provider +RKE on EC2 | Supported by AWS cloud provider +RKE on DigitalOcean | Limited NGINX or third-party Ingress* +RKE on vSphere | Limited NGINX or third party-Ingress* +RKE on Custom Hosts
(e.g. bare-metal servers) | Limited NGINX or third-party Ingress* +Third-party MetalLB | Limited NGINX or third-party Ingress* + +\* Services can be exposed through a single [globally managed config-map.](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) + +## Layer-7 Load Balancer + +Layer-7 load balancer (or the ingress controller) supports host and path-based load balancing and SSL termination. Layer-7 load balancer only forwards HTTP and HTTPS traffic and therefore they listen on ports 80 and 443 only. Cloud providers such as Amazon and Google support layer-7 load balancer. In addition, RKE clusters deploys the Nginx Ingress Controller. + +### Support for Layer-7 Load Balancing + +Support for layer-7 load balancer varies based on the underlying cloud provider. + +Cluster Deployment | Layer-7 Load Balancer Support +----------------------------------------------|-------------------------------- +Amazon EKS | Supported by AWS cloud provider +Google GKE | Supported by GKE cloud provider +Azure AKS | Not Supported +RKE on EC2 | Nginx Ingress Controller +RKE on DigitalOcean | Nginx Ingress Controller +RKE on vSphere | Nginx Ingress Controller +RKE on Custom Hosts
(e.g. bare-metal servers) | Nginx Ingress Controller + +### Host Names in Layer-7 Load Balancer + +Some cloud-managed layer-7 load balancers (such as the ALB ingress controller on AWS) expose DNS addresses for ingress rules. You need to map (via CNAME) your domain name to the DNS address generated by the layer-7 load balancer. + +Other layer-7 load balancers, such as the Google Load Balancer or Nginx Ingress Controller, directly expose one or more IP addresses. Google Load Balancer provides a single routable IP address. Nginx Ingress Controller exposes the external IP of all nodes that run the Nginx Ingress Controller. You can do either of the following: + +1. Configure your own DNS to map (via A records) your domain name to the IP addresses exposes by the Layer-7 load balancer. +2. Ask Rancher to generate an xip.io host name for your ingress rule. Rancher will take one of your exposed IPs, say `a.b.c.d`, and generate a host name `..a.b.c.d.xip.io`. + +The benefit of using xip.io is that you obtain a working entrypoint URL immediately after you create the ingress rule. Setting up your own domain name, on the other hand, requires you to configure DNS servers and wait for DNS to propagate. + +## Related Links + +- [Create an External Load Balancer](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md new file mode 100644 index 00000000000..89e4a40ba5d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md @@ -0,0 +1,82 @@ +--- +title: Secrets +--- + + + + + +[Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#overview-of-secrets) store sensitive data like passwords, tokens, or keys. They may contain one or more key value pairs. + +:::note + +This page is about secrets in general. For details on setting up a private registry, refer to the section on [registries.](kubernetes-and-docker-registries.md) + +::: + +When configuring a workload, you'll be able to choose which secrets to include. Like config maps, secrets can be referenced by workloads as either an environment variable or a volume mount. + +Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) + +## Creating Secrets in Namespaces + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a secret and click **Explore**. +1. To navigate to secrets, you may click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Click **Create**. +1. Select the type of secret you want to create. +1. Select a **Namespace** for the secret. +1. Enter a **Name** for the secret. + + :::note + + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a namespace can have duplicate names. Therefore, to prevent conflicts, your secret must have a unique name among all secrets within your workspace. + + ::: + +1. From **Data**, click **Add** to add a key-value pair. Add as many values as you need. + + :::tip + + You can add multiple key value pairs to the secret by copying and pasting. + + ::: + + ![](/img/bulk-key-values.gif) + +1. Click **Save**. + +**Result:** Your secret is added to the namespace you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. + +Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) + + +## Creating Secrets in Projects + +Before v2.6, secrets were required to be in a project scope. Projects are no longer required, and you may use the namespace scope instead. As a result, the Rancher UI was updated to reflect this new functionality. However, you may still create project-scoped secrets if desired. Note that you have to first enable the `legacy` feature flag and look at a single project to do so. Use the following steps to set up your project-level secret: + +1. In the upper left corner, click **☰ > Global Settings** in the dropdown. +1. Click **Feature Flags**. +1. Go to the `legacy` feature flag and click **Activate**. +1. In the upper left corner, click **☰ > Cluster Management** in the dropdown. +1. Go to the cluster that you created and click **Explore.** +1. Click **Legacy > Projects**. +1. In the top navigation bar, filter to see only one project. +1. In the left navigation bar, click **Secrets**. +1. Click **Add Secret**. + +**Result:** Your secret is added to the individual project you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. + +Since project-scoped secrets are set at the project level, any changes made at the namespace level will be overwritten. + +:::note + +Project-scoped secrets on the local cluster are only visible when a single project is selected. + +::: + +## What's Next? + +Now that you have a secret added to a namespace, you can add it to a workload that you deploy. + +For more information on adding secret to a workload, see [Deploying Workloads](workloads-and-pods/deploy-workloads.md). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md new file mode 100644 index 00000000000..e5dc6e4b503 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md @@ -0,0 +1,39 @@ +--- +title: Adding a Sidecar +--- + + + + + +A _sidecar_ is a container that extends or enhances the main container in a pod. The main container and the sidecar share a pod, and therefore share the same network space and storage. You can add sidecars to existing workloads by using the **Add a Sidecar** option. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a sidecar and click **Explore**. +1. In the left navigation bar, click **Workload**. + +1. Find the workload that you want to extend. Select **⋮ > + Add Sidecar**. + +1. Enter a **Name** for the sidecar. + +1. In the **General** section, select a sidecar type. This option determines if the sidecar container is deployed before or after the main container is deployed. + + - **Standard Container:** + + The sidecar container is deployed after the main container. + + - **Init Container:** + + The sidecar container is deployed before the main container. + +1. From the **Container Image** field, enter the name of the container image that you want to deploy in support of the main container. During deployment, Rancher pulls this image from [Docker Hub](https://hub.docker.com/explore/). Enter the name exactly as it appears on Docker Hub. + +1. Set the remaining options. You can read about them in [Deploying Workloads](deploy-workloads.md). + +1. Click **Launch**. + +**Result:** The sidecar is deployed according to your parameters. Following its deployment, you can view the sidecar by selecting **⋮ icon (...) > Edit** for the main deployment. + +## Related Links + +- [The Distributed System ToolKit: Patterns for Composite Containers](https://kubernetes.io/blog/2015/06/the-distributed-system-toolkit-patterns/) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md new file mode 100644 index 00000000000..82d495cce05 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md @@ -0,0 +1,61 @@ +--- +title: Deploying Workloads +description: Read this step by step guide for deploying workloads. Deploy a workload to run an application in one or more containers. +--- + + + + + +Deploy a workload to run an application in one or more containers. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to upgrade a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Click **Create**. +1. Choose the type of workload. +1. Select the namespace where the workload will be deployed. +1. Enter a **Name** for the workload. + +1. From the **Container Image** field, enter the name of the Docker image that you want to deploy to the project, optionally prefacing it with the registry host (e.g. `quay.io`, `registry.gitlab.com`, etc.). During deployment, Rancher pulls this image from the specified public or private registry. If no registry host is provided, Rancher will pull the image from [Docker Hub](https://hub.docker.com/explore/). Enter the name exactly as it appears in the registry server, including any required path, and optionally including the desired tag (e.g. `registry.gitlab.com/user/path/image:tag`). If no tag is provided, the `latest` tag will be automatically used. + +1. Either select an existing namespace, or click **Add to a new namespace** and enter a new namespace. + +1. Click **Add Port** to enter a port mapping, which enables access to the application inside and outside of the cluster . For more information, see [Services](../../../../pages-for-subheaders/workloads-and-pods.md#services). + +1. Configure the remaining options: + + - **Environment Variables** + + Use this section to either specify environment variables for your workload to consume on the fly, or to pull them from another source, such as a secret or [ConfigMap](../configmaps.md). + + - **Node Scheduling** + - **Health Check** + - **Volumes** + + Use this section to add storage for your workload. You can manually specify the volume that you want to add, use a persistent volume claim to dynamically create a volume for the workload, or read data for a volume to use from a file such as a [ConfigMap](../configmaps.md). + + When you are deploying a Stateful Set, you should use a Volume Claim Template when using Persistent Volumes. This will ensure that Persistent Volumes are created dynamically when you scale your Stateful Set. + + - **Scaling/Upgrade Policy** + + :::note Amazon Note for Volumes: + + To mount an Amazon EBS volume: + + - In [Amazon AWS](https://aws.amazon.com/), the nodes must be in the same Availability Zone and possess IAM permissions to attach/unattach volumes. + + - The cluster must be using the [AWS cloud provider](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#aws) option. For more information on enabling this option see [Creating an Amazon EC2 Cluster](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) or [Creating a Custom Cluster](../../../../pages-for-subheaders/use-existing-nodes.md). + + ::: + +1. Click **Show Advanced Options** and configure: + + - **Command** + - **Networking** + - **Labels & Annotations** + - **Security and Host Config** + +1. Click **Launch**. + +**Result:** The workload is deployed to the chosen namespace. You can view the workload's status from the project's **Workloads** view. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md new file mode 100644 index 00000000000..e8f87d33055 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md @@ -0,0 +1,18 @@ +--- +title: Rolling Back Workloads +--- + + + + + +Sometimes there is a need to rollback to the previous version of the application, either for debugging purposes or because an upgrade did not go as planned. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to upgrade a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Find the workload that you want to rollback and select **⋮ > Rollback**. + +1. Choose the revision that you want to roll back to. Click **Rollback**. + +**Result:** Your workload reverts to the previous version that you chose. Wait a few minutes for the action to complete. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md new file mode 100644 index 00000000000..05a51a640bf --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md @@ -0,0 +1,25 @@ +--- +title: Upgrading Workloads +--- + + + + + +When a new version of an application image is released on Docker Hub, you can upgrade any workloads running a previous version of the application to the new one. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to upgrade a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. + +1. Find the workload that you want to upgrade and select **⋮ > Edit Config**. + +1. Update the **Container Image** and any options that you want to change. + +1. Review and edit the workload's **Scaling and Upgrade Policy**. + + These options control how the upgrade rolls out to containers that are currently running. For example, for scalable deployments, you can choose whether you want to stop old pods before deploying new ones, or vice versa, as well as the upgrade batch size. + +1. Click **Save**. + +**Result:** The workload begins upgrading its containers, per your specifications. Note that scaling up the deployment or updating the upgrade/scaling policy won't result in the pods recreation. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md new file mode 100644 index 00000000000..bd90885720a --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md @@ -0,0 +1,47 @@ +--- +title: Rancher Agents +--- + + + + + +There are two different agent resources deployed on Rancher managed clusters: + +- [cattle-cluster-agent](#cattle-cluster-agent) +- [cattle-node-agent](#cattle-node-agent) + +For a conceptual overview of how the Rancher server provisions clusters and communicates with them, refer to the [architecture](../../../pages-for-subheaders/rancher-manager-architecture.md). + +### cattle-cluster-agent + +The `cattle-cluster-agent` is used to connect to the Kubernetes API of [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters. The `cattle-cluster-agent` is deployed using a Deployment resource. + +### cattle-node-agent + +The `cattle-node-agent` is used to interact with nodes in a [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) cluster when performing cluster operations. Examples of cluster operations are upgrading Kubernetes version and creating/restoring etcd snapshots. The `cattle-node-agent` is deployed using a DaemonSet resource to make sure it runs on every node. The `cattle-node-agent` is used as fallback option to connect to the Kubernetes API of [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters when `cattle-cluster-agent` is unavailable. + +### Scheduling rules + +The `cattle-cluster-agent` uses either a fixed set of tolerations, or dynamically-added tolerations based on taints applied to the control plane nodes. This structure allows [Taint based Evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions) to work properly for `cattle-cluster-agent`. + +If control plane nodes are present in the cluster, the default tolerations will be replaced with tolerations matching the taints on the control plane nodes. The default set of tolerations are described below. + +| Component | nodeAffinity nodeSelectorTerms | nodeSelector | Tolerations | +| ---------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| `cattle-cluster-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | **Note:** These are the default tolerations, and will be replaced by tolerations matching taints applied to controlplane nodes.

`effect:NoSchedule`
`key:node-role.kubernetes.io/controlplane`
`value:true`

`effect:NoSchedule`
`key:node-role.kubernetes.io/control-plane`
`operator:Exists`

`effect:NoSchedule`
`key:node-role.kubernetes.io/master`
`operator:Exists` | +| `cattle-node-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | `operator:Exists` | + +The `cattle-cluster-agent` Deployment has preferred scheduling rules using `preferredDuringSchedulingIgnoredDuringExecution`, favoring to be scheduled on nodes with the `controlplane` node. When there are no controlplane nodes visible in the cluster (this is usually the case when using [Clusters from Hosted Kubernetes Providers](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)), you can add the label `cattle.io/cluster-agent=true` on a node to prefer scheduling the `cattle-cluster-agent` pod to that node. + +See [Kubernetes: Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) to find more information about scheduling rules. + +The `preferredDuringSchedulingIgnoredDuringExecution` configuration is shown in the table below: + +| Weight | Expression | +| ------ | ------------------------------------------------ | +| 100 | `node-role.kubernetes.io/controlplane:In:"true"` | +| 100 | `node-role.kubernetes.io/control-plane:In:"true"` | +| 100 | `node-role.kubernetes.io/master:In:"true"` | +| 1 | `cattle.io/cluster-agent:In:"true"` | + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md new file mode 100644 index 00000000000..738178fdc41 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md @@ -0,0 +1,35 @@ +--- +title: Behavior Differences Between RKE1 and RKE2 +--- + + + + + +RKE2, also known as RKE Government, is a Kubernetes distribution that focuses on security and compliance for U.S. Federal Government entities. It is considered the next iteration of the Rancher Kubernetes Engine, now known as RKE1. + +RKE1 and RKE2 have several slight behavioral differences to note, and this page will highlight some of these at a high level. + +### Control Plane Components + +RKE1 uses Docker for deploying and managing control plane components, and it also uses Docker as the container runtime for Kubernetes. By contrast, RKE2 launches control plane components as static pods that are managed by the kubelet. RKE2's container runtime is containerd, which allows things such as mirroring a container image registry. RKE1 with Docker does not allow mirroring. + +### Cluster API + +RKE2/K3s provisioning is built on top of the Cluster API (CAPI) upstream framework which often makes RKE2-provisioned clusters behave differently than RKE1-provisioned clusters. + +When you make changes to your cluster configuration in RKE2, this **may** result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. + +The following are some specific example configuration changes that may cause the described behavior: + +- When editing the cluster and enabling `drain before delete`, the existing control plane nodes and worker are deleted and new nodes are created. + +Users who are used to RKE1 provisioning should take note of this new RKE2 behavior which may be unexpected. + +### Terminology + +You will notice that some terms have changed or gone away going from RKE1 to RKE2. For example, in RKE1 provisioning, you use **node templates**; in RKE2 provisioning, you can configure your cluster node pools when creating or editing the cluster. Another example is that the term **node pool** in RKE1 is now known as **machine pool** in RKE2. + + + + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md new file mode 100644 index 00000000000..bef31ab3f23 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md @@ -0,0 +1,103 @@ +--- +title: Creating a DigitalOcean Cluster +--- + + + + + +In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in DigitalOcean. + +First, you will set up your DigitalOcean cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in DigitalOcean. + +Then you will create a DigitalOcean cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + + + + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **DigitalOcean**. +1. Enter your Digital Ocean credentials. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials + +Creating a [node template](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for DigitalOcean will allow Rancher to provision new nodes in DigitalOcean. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Add Template**. +1. Click **DigitalOcean**. +1. Fill out a node template for DigitalOcean. For help filling out the form, refer to [DigitalOcean Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md) + +### 3. Create a cluster with node pools using the node template + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **DigitalOcean**. +1. Enter a **Cluster Name**. +1. Add one or more node pools to your cluster. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) +1. **In the Cluster Configuration** section, choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +### 1. Create your cloud credentials + +If you already have a set of cloud credentials to use, skip this section. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **DigitalOcean**. +1. Enter your Digital Ocean credentials. +1. Click **Create**. + +### 2. Create your cluster + +Use Rancher to create a Kubernetes cluster in DigitalOcean. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Toggle the switch to **RKE2/K3s**. +1. Click **DigitalOcean**. +1. Select a **Cloud Credential**, if more than one exists. Otherwise, it's preselected. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to the [DigitalOcean machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md new file mode 100644 index 00000000000..5bc7d109825 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md @@ -0,0 +1,287 @@ +--- +title: Creating an Amazon EC2 Cluster +description: Learn the prerequisites and steps required in order for you to create an Amazon EC2 cluster using Rancher +--- + + + + + +In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Amazon EC2. + +First, you will set up your EC2 cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in EC2. + +Then you will create an EC2 cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +### Prerequisites + +- **AWS EC2 Access Key and Secret Key** that will be used to create the instances. See [Amazon Documentation: Creating Access Keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) how to create an Access Key and Secret Key. +- **IAM Policy created** to add to the user of the Access Key And Secret Key. See [Amazon Documentation: Creating IAM Policies (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start) how to create an IAM policy. See our three example JSON policies below: + - [Example IAM Policy](#example-iam-policy) + - [Example IAM Policy with PassRole](#example-iam-policy-with-passrole) (needed if you want to use [Kubernetes Cloud Provider](../../../../pages-for-subheaders/set-up-cloud-providers.md) or want to pass an IAM Profile to an instance) + - [Example IAM Policy to allow encrypted EBS volumes](#example-iam-policy-to-allow-encrypted-ebs-volumes) +- **IAM Policy added as Permission** to the user. See [Amazon Documentation: Adding Permissions to a User (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console) how to attach it to an user. + +## Creating an EC2 Cluster + +The steps to create a cluster differ based on your Rancher version. + + + + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials and information from EC2](#2-create-a-node-template-with-your-cloud-credentials-and-information-from-ec2) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Amazon**. +1. Enter a name for the cloud credential. +1. In the **Default Region** field, select the AWS region where your cluster nodes will be located. +1. Enter your AWS EC2 **Access Key** and **Secret Key**. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials and information from EC2 + +Creating a [node template](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for EC2 will allow Rancher to provision new nodes in EC2. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates** +1. Click **Add Template**. +1. Fill out a node template for EC2. For help filling out the form, refer to [EC2 Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md) +1. Click **Create**. + + :::note + + If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. + + ::: + +### 3. Create a cluster with node pools using the node template + +Add one or more node pools to your cluster. For more information about node pools, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Amazon EC2**. +1. Create a node pool for each Kubernetes role. For each node pool, choose a node template that you created. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) +1. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. Refer to [Selecting Cloud Providers](../../../../pages-for-subheaders/set-up-cloud-providers.md) to configure the Kubernetes Cloud Provider. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) + + :::note + + If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. + + ::: + +1. Click **Create**. + + + + +### 1. Create your cloud credentials + +If you already have a set of cloud credentials to use, skip this section. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Amazon**. +1. Enter a name for the cloud credential. +1. In the **Default Region** field, select the AWS region where your cluster nodes will be located. +1. Enter your AWS EC2 **Access Key** and **Secret Key**. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create your cluster + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Toggle the switch to **RKE2/K3s**. +1. Click **Amazon EC2**. +1. Select a **Cloud Credential**, if more than one exists. Otherwise, it's preselected. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to [the EC2 machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +### Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. + +## IAM Policies + +### Example IAM Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` + +### Example IAM Policy with PassRole + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*", + "arn:aws:iam::AWS_ACCOUNT_ID:role/YOUR_ROLE_NAME" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` +### Example IAM Policy to allow encrypted EBS volumes +``` json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:Encrypt", + "kms:DescribeKey", + "kms:CreateGrant", + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:CreateSnapshot" + ], + "Resource": [ + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:snapshot/*", + "arn:aws:kms:REGION:AWS_ACCOUNT_ID:key/KMS_KEY_ID" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots" + ], + "Resource": "*" + } + ] +} +``` diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md new file mode 100644 index 00000000000..758616b057d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md @@ -0,0 +1,145 @@ +--- +title: Creating an Azure Cluster +--- + + + + + +In this section, you'll learn how to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Azure through Rancher. + +First, you will set up your Azure cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in Azure. + +Then you will create an Azure cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +:::caution + +When the Rancher RKE cluster is running in Azure and has an Azure load balancer in front, the outbound flow will fail. The workaround for this problem is as follows: + +- Terminate the SSL/TLS on the internal load balancer +- Use the L7 load balancer + +For more information, refer to the documentation on [Azure load balancer limitations](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations). + +::: + +For more information on configuring the Kubernetes cluster that Rancher will install on the Azure nodes, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) + +For more information on configuring Azure node templates, refer to the [Azure node template configuration reference.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) + +- [Preparation in Azure](#preparation-in-azure) +- [Creating an Azure Cluster](#creating-an-azure-cluster) + +## Preparation in Azure + +Before creating a node template in Rancher using a cloud infrastructure such as Azure, we must configure Rancher to allow the manipulation of resources in an Azure subscription. + +To do this, we will first create a new Azure **service principal (SP)** in Azure **Active Directory (AD)**, which, in Azure, is an application user who has permission to manage Azure resources. + +The following is a template `az cli` script that you have to run for creating an service principal, where you have to enter your SP name, role, and scope: + +``` +az ad sp create-for-rbac \ + --name="" \ + --role="Contributor" \ + --scopes="/subscriptions/" +``` + +The creation of this service principal returns three pieces of identification information, *The application ID, also called the client ID*, and *The client secret*. This information will be used when you create a node template for Azure. + +## Creating an Azure Cluster + + + + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Azure**. +1. Enter your Azure credentials. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials + +Creating a [node template](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for Azure will allow Rancher to provision new nodes in Azure. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Add Template**. +1. Click **Azure**. +1. Fill out a node template for Azure. For help filling out the form, refer to [Azure Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) + +### 3. Create a cluster with node pools using the node template + +Use Rancher to create a Kubernetes cluster in Azure. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Azure**. +1. Enter a **Cluster Name**. +1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) +1. In the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +### 1. Create your cloud credentials + +If you already have a set of cloud credentials to use, skip this section. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Azure**. +1. Enter your Azure credentials. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create your cluster + +Use Rancher to create a Kubernetes cluster in Azure. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Toggle the switch to **RKE2/K3s**. +1. Click **Azure**. +1. Select a **Cloud Credential**, if more than one exists. Otherwise, it's preselected. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to the [Azure machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +### Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md new file mode 100644 index 00000000000..ded99b0679c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md @@ -0,0 +1,96 @@ +--- +title: Provisioning Kubernetes Clusters in Nutanix AOS +--- + + + + + +To use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Nutanix AOS (AHV): + +1. Locate Rancher's built-in Nutanix [node driver and activate it](../../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#activatingdeactivating-node-drivers). + +1. Create a node template, which Rancher will use to provision nodes in Nutanix AOS. + +1. Create a Nutanix AOS cluster in Rancher. When configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +For details on configuring the Nutanix AOS node template, refer to the [Nutanix AOS node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md) + +For details on configuring RKE Kubernetes clusters in Rancher, refer to the [cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) + +- [Preparation in Nutanix AOS](#preparation-in-nutanix-aos) +- [Creating a Nutanix AOS Cluster](#creating-a-nutanix-aos-cluster) + +## Preparation in Nutanix AOS + +The following sections describe the requirements for setting up Nutanix AOS so that Rancher can provision VMs and clusters. + +:::note + +The node templates are documented and tested with Nutanix AOS version 5.20.2 and 6.0.1. + +::: + +### Create Credentials in Nutanix AOS + +Before proceeding to create a cluster, you must ensure that you have a [Nutanix Prism Central user account](https://portal.nutanix.com/page/documents/details?targetId=Nutanix-Security-Guide-v6_0:wc-user-create-wc-t.html) with admin permissions. When you set up a node template, the template will need to use these credentials. + +### Network Permissions + +You must ensure that the hosts running the Rancher server are able to establish the following network connections: + +- To the Nutanix Prism Central API (usually port 9440/TCP). +- To port 22/TCP and 2376/TCP on the created VMs + +See [Node Networking Requirements](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for a detailed list of port requirements applicable for creating nodes on an infrastructure provider. + +### VM-VM Anti-Affinity Policies + +Setting up [VM-VM Anti-Affinity Policies](https://portal.nutanix.com/page/documents/details?targetId=AHV-Admin-Guide-v6_1:ahv-vm-anti-affinity-t.html) is recommended. These rules allow VMs assigned the etcd and control-plane roles to operate on separate AHV hosts when they are assigned to different node pools. This practice ensures that the failure of a single physical machine does not affect the availability of those planes. + +## Creating a Nutanix AOS Cluster + +1. [Create a node template ](#1-create-a-node-template) +2. [Create a cluster with node pools using the node template](#2-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create a node template + +Creating a [node template](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for Nutanix AOS will allow Rancher to provision new nodes in Nutanix AOS. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Create**. +1. Click **Add Template**. +1. Click **Nutanix**. +1. Fill out a node template for Nutanix AOS. For help filling out the form, refer to the Nutanix AOS node template [configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md). +1. Click **Create**. + +### 2. Create a cluster with node pools using the node template + +Use Rancher to create a Kubernetes cluster in Nutanix AOS. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Nutanix**. +1. Enter a **Cluster Name**, then click **Continue**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users who can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used, and whether you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to the nodes, see [this section.](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) +1. Review your options to confirm they're correct. Then click **Create**. + +**Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. + +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md new file mode 100644 index 00000000000..c8a6da86cf2 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md @@ -0,0 +1,164 @@ +--- +title: Creating a vSphere Virtual Machine Template +--- + + + + + +Creating virtual machines in a repeatable and reliable fashion can often be difficult. VMware vSphere offers the ability to build one VM that can then be converted to a template. The template can then be used to create identically configured VMs. Rancher leverages this capability within node pools to create identical RKE1 and RKE2 nodes. + +In order to leverage the template to create new VMs, Rancher has some [specific requirements](#requirements) that the VM must have pre-installed. After you configure the VM with these requirements, you will next need to [prepare the VM](#preparing-your-vm) before [creating the template](#creating-a-template). Finally, once preparation is complete, the VM can be [converted to a template](#converting-to-a-template) and [moved into a content library](#moving-to-a-content-library), ready for Rancher node pool usage. + + +## Requirements + +There is specific tooling required for both Linux and Windows VMs to be usable by the vSphere node driver. The most critical dependency is [cloud-init](https://cloud-init.io/) for Linux and [cloudbase-init](https://cloudbase.it/cloudbase-init/) for Windows. Both of these are used for provisioning the VMs by configuring the hostname and by setting up the SSH access and the default Rancher user. Users can add additional content to these as desired if other configuration is needed. In addition, other requirements are listed below for reference. + +:::note + +If you have any specific firewall rules or configuration, you will need to add this to the VM before creating a template. + +::: + +## Linux Dependencies + +The packages that need to be installed on the template are listed below. These will have slightly different names based on distribution; some distributions ship these by default, for example. + +* curl +* wget +* git +* net-tools +* unzip +* apparmor-parser +* ca-certificates +* cloud-init +* cloud-guest-utils +* cloud-image-utils +* growpart +* cloud-initramfs-growroot +* open-iscsi +* openssh-server +* [open-vm-tools](https://docs.vmware.com/en/VMware-Tools/11.3.0/com.vmware.vsphere.vmwaretools.doc/GUID-8B6EA5B7-453B-48AA-92E5-DB7F061341D1.html) + +## Windows Dependencies + +The list of packages that need to be installed on the template is as follows: + +* Windows Container Feature +* [cloudbase-init](https://cloudbase.it/cloudbase-init/#download) +* [Docker EE](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server#install-docker) - RKE1 Only + +:::note About the configuration for Windows templates varies between RKE1 and RKE2: + +- RKE1 leverages Docker, so any RKE1 templates need to have Docker EE pre-installed as well +- RKE2 does not require Docker EE, and thus it does not need to be installed + +::: + +## Creating a Template + +You may either manually create your VM or you can utilize [other alternatives](#alternatives-to-manual-creation) to create your VM. + +### Manual Creation +1. Manually create your VM by following [these instructions](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AE8AFBF1-75D1-4172-988C-378C35C9FAF2.html) from VMware. Once you have a VM running, you can manually install the dependencies listed above to configure the VM correctly for the vSphere node driver. +2. Customize as needed based on your specific environment and requirements. +3. Proceed with the final preparation before creating your template. + +### Alternatives to Manual Creation + +Other alternative options to create VMs are listed below: + +* [VMware PowerCLI](https://developer.vmware.com/powercli) +* [Packer](https://www.packer.io/) +* [SaltStack](https://saltproject.io/) +* [Ansible](https://www.ansible.com/) + +Packer is a frequently-used alternative. Refer to this [reference](https://github.com/vmware-samples/packer-examples-for-vsphere) for examples of its usage with vSphere. + +## Preparing Your VM + +After creating a VM with all the required dependencies (and any additional required items), you must perform the most critical step next: preparing the VM to be turned into a template. This preparation will reset critical data such as the VM hostname, IPs, etc., to prevent that information from being brought into a new VM. If you fail to perform this step, you could create a VM with the same hostname, IP address, etc. + +Note that these preparatory steps differ between Linux and Windows. + +### Linux Preparation + +The commands below will reset your VM in Linux: + +```bash +# Cleaning logs. +if [ -f /var/log/audit/audit.log ]; then + cat /dev/null > /var/log/audit/audit.log +fi +if [ -f /var/log/wtmp ]; then + cat /dev/null > /var/log/wtmp +fi +if [ -f /var/log/lastlog ]; then + cat /dev/null > /var/log/lastlog +fi + +# Cleaning udev rules. +if [ -f /etc/udev/rules.d/70-persistent-net.rules ]; then + rm /etc/udev/rules.d/70-persistent-net.rules +fi + +# Cleaning the /tmp directories +rm -rf /tmp/* +rm -rf /var/tmp/* + +# Cleaning the SSH host keys +rm -f /etc/ssh/ssh_host_* + +# Cleaning the machine-id +truncate -s 0 /etc/machine-id +rm /var/lib/dbus/machine-id +ln -s /etc/machine-id /var/lib/dbus/machine-id + +# Cleaning the shell history +unset HISTFILE +history -cw +echo > ~/.bash_history +rm -fr /root/.bash_history + +# Truncating hostname, hosts, resolv.conf and setting hostname to localhost +truncate -s 0 /etc/{hostname,hosts,resolv.conf} +hostnamectl set-hostname localhost + +# Clean cloud-init +cloud-init clean -s -l +``` + +### Windows Preparation + +Windows has a utility called [sysprep](https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/sysprep--generalize--a-windows-installation) that is used to generalize an image and reset the same items listed above for Linux. The command is as follows: + +```PowerShell +sysprep.exe /generalize /shutdown /oobe +``` + +## Converting to a Template + +1. Shut down and stop the VM. +2. Right-click on the VM in the inventory list and select **Template**. +3. Click on **Convert to Template**. + +**Result:** Once the process has completed, a template will be available for use. + +For additional information on converting a VM to a template, see the [VMware guide](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-5B3737CC-28DB-4334-BD18-6E12011CDC9F.html). + +## Moving to a Content library + +Rancher has the ability to use templates provided by a content library. Content libraries store and manage content within vSphere, and they also offer the ability to publish and share that content. + +Below are some helpful links on content libraries: + +* [Create a content library](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-2A0F1C13-7336-45CE-B211-610D39A6E1F4.html) +* [Clone the template to the content library](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AC1545F0-F8BA-4CD2-96EB-21B3DFAA1DC1.html) + +## Other Resources + +Here is a list of additional resources that may be useful: + +* [Tutorial for creating a Linux template](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-ubuntu-template) +* [Tutorial for creating a Windows template](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-windows-template) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md new file mode 100644 index 00000000000..d228c27c1e7 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md @@ -0,0 +1,49 @@ +--- +title: Creating Credentials in the vSphere Console +--- + + + + + +This section describes how to create a vSphere username and password. You will need to provide these vSphere credentials to Rancher, which allows Rancher to provision resources in vSphere. + +The following table lists the permissions required for the vSphere user account: + +| Privilege Group | Operations | +|:-------------------------|:-----------------------------------------------------------------------| +| Cns Privileges | Searchable | +| Content library | Read Storage
(Required only if deploying Template from Content Library) | +| Cryptographic operations | Direct Access | +| Datastore | AllocateSpace
Browse
FileManagement (Low level file operations)
UpdateVirtualMachineFiles
UpdateVirtualMachineMetadata | +| Global | Set custom attribute | +| Network | Assign | +| Resource | AssignVMToPool | +| Virtual Machine | Config (All)
GuestOperations (All)
Interact (All)
Inventory (All)
Provisioning (All) | +| vSphere Tagging | Assign or Unassign vSphere Tag
Assign or Unassign vSphere Tag on Object | + +The following steps create a role with the required privileges and then assign it to a new user in the vSphere console: + +1. From the **vSphere** console, go to the **Administration** page. + +2. Go to the **Roles** tab. + +3. Create a new role. Give it a name and select the privileges listed in the permissions table above. + + ![](/img/rancherroles1.png) + +4. Go to the **Users and Groups** tab. + +5. Create a new user. Fill out the form and then click **OK**. Make sure to note the username and password, because you will need it when configuring node templates in Rancher. + + ![](/img/rancheruser.png) + +6. Go to the **Global Permissions** tab. + +7. Create a new Global Permission. Add the user you created earlier and assign it the role you created earlier. Click **OK**. + + ![](/img/globalpermissionuser.png) + + ![](/img/globalpermissionrole.png) + +**Result:** You now have credentials that Rancher can use to manipulate vSphere resources. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md new file mode 100644 index 00000000000..a28520b24dd --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md @@ -0,0 +1,114 @@ +--- +title: Provisioning Kubernetes Clusters in vSphere +--- + + + + + +In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in vSphere. + +First, you will set up your vSphere cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision nodes in vSphere. + +Then you will create a vSphere cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +For details on configuring the vSphere node template, refer to the [vSphere node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) + +For details on configuring RKE Kubernetes clusters in Rancher, refer to the [cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) + + +- [Preparation in vSphere](#preparation-in-vsphere) +- [Creating a vSphere Cluster](#creating-a-vsphere-cluster) + +## Preparation in vSphere + +This section describes the requirements for setting up vSphere so that Rancher can provision VMs and clusters. + +The node templates are documented and tested with the vSphere Web Services API version 6.5. + +### Create Credentials in vSphere + +Before proceeding to create a cluster, you must ensure that you have a vSphere user with sufficient permissions. When you set up a node template, the template will need to use these vSphere credentials. + +Refer to this [how-to guide](create-credentials.md) for instructions on how to create a user in vSphere with the required permissions. These steps result in a username and password that you will need to provide to Rancher, which allows Rancher to provision resources in vSphere. + +### Network Permissions + +It must be ensured that the hosts running the Rancher server are able to establish the following network connections: + +- To the vSphere API on the vCenter server (usually port 443/TCP). +- To the Host API (port 443/TCP) on all ESXi hosts used to instantiate virtual machines for the clusters (*only required when using the ISO creation method*). +- To port 22/TCP and 2376/TCP on the created VMs + +See [Node Networking Requirements](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for a detailed list of port requirements applicable for creating nodes on an infrastructure provider. + +### Valid ESXi License for vSphere API Access + +The free ESXi license does not support API access. The vSphere servers must have a valid or evaluation ESXi license. + +### VM-VM Affinity Rules for Clusters with DRS + +If you have a cluster with DRS enabled, setting up [VM-VM Affinity Rules](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.resmgmt.doc/GUID-7297C302-378F-4AF2-9BD6-6EDB1E0A850A.html) is recommended. These rules allow VMs assigned the etcd and control-plane roles to operate on separate ESXi hosts when they are assigned to different node pools. This practice ensures that the failure of a single physical machine does not affect the availability of those planes. + +## Creating a vSphere Cluster + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **VMware vSphere**. +1. Enter your vSphere credentials. For help, refer to **Account Access** in the [node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials + +Creating a [node template](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for vSphere will allow Rancher to provision new nodes in vSphere. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Create**. +1. Click **Add Template**. +1. Click **vSphere**. +1. Fill out a node template for vSphere. For help filling out the form, refer to the vSphere node template [configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md). +1. Click **Create**. + +### 3. Create a cluster with node pools using the node template + +Use Rancher to create a Kubernetes cluster in vSphere. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **VMware vSphere**. +1. Enter a **Cluster Name** and use your vSphere cloud credentials. Click **Continue**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. If you want to dynamically provision persistent storage or other infrastructure later, you will need to enable the vSphere cloud provider by modifying the cluster YAML file. For details, refer to [in-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). +1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to the nodes, see [this section.](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) +1. Review your options to confirm they're correct. Then click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. +- **Provision Storage:** For an example of how to provision storage in vSphere using Rancher, refer to [this section.](../../../../../pages-for-subheaders/provisioning-storage-examples.md) In order to dynamically provision storage in vSphere, the vSphere provider must be enabled. For details, refer to [in-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md new file mode 100644 index 00000000000..148715a31de --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md @@ -0,0 +1,70 @@ +--- +title: Adding Users to Clusters +--- + + + + + +If you want to provide a user with access and permissions to _all_ projects, nodes, and resources within a cluster, assign the user a cluster membership. + +:::tip + +Want to provide a user with access to a _specific_ project within a cluster? See [Adding Project Members](../../../new-user-guides/add-users-to-projects.md) instead. + +::: + +There are two contexts where you can add cluster members: + +- Adding Members to a New Cluster + + You can add members to a cluster as you create it (recommended if possible). + +- [Adding Members to an Existing Cluster](#editing-cluster-membership) + + You can always add members to a cluster after a cluster is provisioned. + +## Editing Cluster Membership + +Cluster administrators can edit the membership for a cluster, controlling which Rancher users can access the cluster and what features they can use. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to add members to and click **â‹® > Edit Config**. +1. In the **Member Roles** tab, click **Add Member**. +1. Search for the user or group that you want to add to the cluster. + + If external authentication is configured: + + - Rancher returns users from your [external authentication](../../../../pages-for-subheaders/authentication-config.md) source as you type. + + :::note Using AD but can't find your users? + + There may be an issue with your search attribute configuration. See [Configuring Active Directory Authentication: Step 5](../../../new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md). + + ::: + + - A drop-down allows you to add groups instead of individual users. The drop-down only lists groups that you, the logged in user, are part of. + + :::note + + If you are logged in as a local user, external users do not display in your search results. For more information, see [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + + ::: + +1. Assign the user or group **Cluster** roles. + + [What are Cluster Roles?](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::tip + + For Custom Roles, you can modify the list of individual roles available for assignment. + + - To add roles to the list, [Add a Custom Role](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md). + - To remove roles from the list, [Lock/Unlock Roles](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md). + + ::: + +**Result:** The chosen users are added to the cluster. + +- To revoke cluster membership, select the user and click **Delete**. This action deletes membership, not the user. +- To modify a user's roles in the cluster, delete them from the cluster, and then re-add them with modified roles. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md new file mode 100644 index 00000000000..543ed4d10b4 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md @@ -0,0 +1,53 @@ +--- +title: How the Authorized Cluster Endpoint Works +--- + + + + + +This section describes how the kubectl CLI, the kubeconfig file, and the authorized cluster endpoint work together to allow you to access a downstream Kubernetes cluster directly, without authenticating through the Rancher server. It is intended to provide background information and context to the instructions for [how to set up kubectl to directly access a cluster.](use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) + +### About the Kubeconfig File + +The kubeconfig file is used to configure access to Kubernetes when used in conjunction with the kubectl command line tool (or other clients). + +The kubeconfig file and its contents are specific to each cluster. It can be downloaded from the **Clusters** page in Rancher: + +1. Click **☰** in the top left corner. +1. Select **Cluster Management**. +1. Find the cluster whose kubeconfig you want to download, and select **â** at the end of the row. +1. Select **Download KubeConfig** from the submenu. + +You will need a separate kubeconfig file for each cluster that you have access to in Rancher. + +After you download the kubeconfig file, you will be able to use the kubeconfig file and its Kubernetes [contexts](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#kubectl-context-and-configuration) to access your downstream cluster. + +If admins have [kubeconfig token generation turned off](../../../../reference-guides/about-the-api/api-tokens.md#disable-tokens-in-generated-kubeconfigs), the kubeconfig file requires [rancher cli](./authorized-cluster-endpoint.md) to be present in your PATH. + +### Two Authentication Methods for RKE Clusters + +If the cluster is not an [RKE cluster,](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) the kubeconfig file allows you to access the cluster in only one way: it lets you be authenticated with the Rancher server, then Rancher allows you to run kubectl commands on the cluster. + +For RKE clusters, the kubeconfig file allows you to be authenticated in two ways: + +- **Through the Rancher server authentication proxy:** Rancher's authentication proxy validates your identity, then connects you to the downstream cluster that you want to access. +- **Directly with the downstream cluster's API server:** RKE clusters have an authorized cluster endpoint enabled by default. This endpoint allows you to access your downstream Kubernetes cluster with the kubectl CLI and a kubeconfig file, and it is enabled by default for RKE clusters. In this scenario, the downstream cluster's Kubernetes API server authenticates you by calling a webhook (the `kube-api-auth` microservice) that Rancher set up. + +This second method, the capability to connect directly to the cluster's Kubernetes API server, is important because it lets you access your downstream cluster if you can't connect to Rancher. + +To use the authorized cluster endpoint, you will need to configure kubectl to use the extra kubectl context in the kubeconfig file that Rancher generates for you when the RKE cluster is created. This file can be downloaded from the cluster view in the Rancher UI, and the instructions for configuring kubectl are on [this page.](use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) + +These methods of communicating with downstream Kubernetes clusters are also explained in the [architecture page](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md) in the larger context of explaining how Rancher works and how Rancher communicates with downstream clusters. + +### About the kube-api-auth Authentication Webhook + +The `kube-api-auth` microservice is deployed to provide the user authentication functionality for the [authorized cluster endpoint,](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint). When you access the user cluster using `kubectl`, the cluster's Kubernetes API server authenticates you by using the `kube-api-auth` service as a webhook. + +During cluster provisioning, the file `/etc/kubernetes/kube-api-authn-webhook.yaml` is deployed and `kube-apiserver` is configured with `--authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml`. This configures the `kube-apiserver` to query `http://127.0.0.1:6440/v1/authenticate` to determine authentication for bearer tokens. + +The scheduling rules for `kube-api-auth` are listed below: + +| Component | nodeAffinity nodeSelectorTerms | nodeSelector | Tolerations | +| -------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| kube-api-auth | `beta.kubernetes.io/os:NotIn:windows`
`node-role.kubernetes.io/controlplane:In:"true"` | none | `operator:Exists` | diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md new file mode 100644 index 00000000000..6099a7d33d3 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -0,0 +1,110 @@ +--- +title: "Access a Cluster with Kubectl and kubeconfig" +description: "Learn how you can access and manage your Kubernetes clusters using kubectl with kubectl Shell or with kubectl CLI and kubeconfig file. A kubeconfig file is used to configure access to Kubernetes. When you create a cluster with Rancher, it automatically creates a kubeconfig for your cluster." +--- + + + + + +This section describes how to manipulate your downstream Kubernetes cluster with kubectl from the Rancher UI or from your workstation. + +For more information on using kubectl, see [Kubernetes Documentation: Overview of kubectl](https://kubernetes.io/docs/reference/kubectl/overview/). + +### Accessing Clusters with kubectl Shell in the Rancher UI + +You can access and manage your clusters by logging into Rancher and opening the kubectl shell in the UI. No further configuration necessary. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to access with kubectl and click **Explore**. +1. In the top navigation menu, click the **Kubectl Shell** button. Use the window that opens to interact with your Kubernetes cluster. + +### Accessing Clusters with kubectl from Your Workstation + +This section describes how to download your cluster's kubeconfig file, launch kubectl from your workstation, and access your downstream cluster. + +This alternative method of accessing the cluster allows you to authenticate with Rancher and manage your cluster without using the Rancher UI. + +:::note Prerequisites: + +These instructions assume that you have already created a Kubernetes cluster, and that kubectl is installed on your workstation. For help installing kubectl, refer to the official [Kubernetes documentation.](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + +::: + +1. Click **☰** in the top left corner. +1. Select **Cluster Management**. +1. Find the cluster whose kubeconfig you want to download, and select **â** at the end of the row. +1. Select **Download KubeConfig** from the submenu. +1. Save the YAML file on your local computer. Move the file to `~/.kube/config`. Note: The default location that kubectl uses for the kubeconfig file is `~/.kube/config`, but you can use any directory and specify it using the `--kubeconfig` flag, as in this command: + ``` + kubectl --kubeconfig /custom/path/kube.config get pods + ``` +1. From your workstation, launch kubectl. Use it to interact with your kubernetes cluster. + + +### Note on Resources Created Using kubectl + +Rancher will discover and show resources created by `kubectl`. However, these resources might not have all the necessary annotations on discovery. If an operation (for instance, scaling the workload) is done to the resource using the Rancher UI/API, this may trigger recreation of the resources due to the missing annotations. This should only happen the first time an operation is done to the discovered resource. + +## Authenticating Directly with a Downstream Cluster + +This section intended to help you set up an alternative method to access an [RKE cluster.](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +This method is only available for RKE, RKE2, and K3s clusters that have the [authorized cluster endpoint](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled. When Rancher creates the cluster, it generates a kubeconfig file that includes additional kubectl context(s) for accessing your cluster. This additional context allows you to use kubectl to authenticate with the downstream cluster without authenticating through Rancher. For a longer explanation of how the authorized cluster endpoint works, refer to [this page](authorized-cluster-endpoint.md). + +On RKE2 and K3s clusters, you need to [manually enable](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) authorized cluster endpoints. + +We recommend that as a best practice, you should set up this method to access your RKE, RKE2, and K3s clusters, so that just in case you can’t connect to Rancher, you can still access the cluster. + + +:::note Prerequisites: + +The following steps assume that you have created a Kubernetes cluster and followed the steps to [connect to your cluster with kubectl from your workstation.](#accessing-clusters-with-kubectl-from-your-workstation) + +::: + +To find the name of the context(s) in your downloaded kubeconfig file, run: + +``` +kubectl config get-contexts --kubeconfig /custom/path/kube.config +CURRENT NAME CLUSTER AUTHINFO NAMESPACE +* my-cluster my-cluster user-46tmn + my-cluster-controlplane-1 my-cluster-controlplane-1 user-46tmn +``` + +In this example, when you use `kubectl` with the first context, `my-cluster`, you will be authenticated through the Rancher server. + +With the second context, `my-cluster-controlplane-1`, you would authenticate with the authorized cluster endpoint, communicating with an downstream RKE cluster directly. + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +Now that you have the name of the context needed to authenticate directly with the cluster, you can pass the name of the context in as an option when running kubectl commands. The commands will differ depending on whether your cluster has an FQDN defined. Examples are provided in the sections below. + +When `kubectl` works normally, it confirms that you can access your cluster while bypassing Rancher's authentication proxy. + +### Connecting Directly to Clusters with FQDN Defined + +If an FQDN is defined for the cluster, a single context referencing the FQDN will be created. The context will be named `-fqdn`. When you want to use `kubectl` to access this cluster without Rancher, you will need to use this context. + +Assuming the kubeconfig file is located at `~/.kube/config`: + +``` +kubectl --context -fqdn get nodes +``` +Directly referencing the location of the kubeconfig file: +``` +kubectl --kubeconfig /custom/path/kube.config --context -fqdn get pods +``` + +### Connecting Directly to Clusters without FQDN Defined + +If there is no FQDN defined for the cluster, extra contexts will be created referencing the IP address of each node in the control plane. Each context will be named `-`. When you want to use `kubectl` to access this cluster without Rancher, you will need to use this context. + +Assuming the kubeconfig file is located at `~/.kube/config`: +``` +kubectl --context - get nodes +``` +Directly referencing the location of the kubeconfig file: +``` +kubectl --kubeconfig /custom/path/kube.config --context - get pods +``` diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md new file mode 100644 index 00000000000..9b9c30d32cb --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md @@ -0,0 +1,43 @@ +--- +title: Adding a Pod Security Policy +--- + + + + + +:::note Prerequisite: + +The options below are available only for clusters that are [launched using RKE.](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +::: + +When your cluster is running pods with security-sensitive configurations, assign it a [pod security policy](../authentication-permissions-and-global-configuration/create-pod-security-policies.md), which is a set of rules that monitors the conditions and settings in your pods. If a pod doesn't meet the rules specified in your policy, the policy stops it from running. + +You can assign a pod security policy when you provision a cluster. However, if you need to relax or restrict security for your pods later, you can update the policy while editing your cluster. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster to which you want to apply a pod security policy and click **â‹® > Edit Config**. +1. From **Pod Security Policy Support**, select **Enabled**. + + :::note + + This option is only available for clusters [provisioned by RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + + ::: + +4. From the **Default Pod Security Policy** drop-down, select the policy you want to apply to the cluster. + + Rancher ships with [policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#default-psps) of `restricted` and `unrestricted`, although you can [create custom policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#creating-psps) as well. + +5. Click **Save**. + +**Result:** The pod security policy is applied to the cluster and any projects within the cluster. + +:::note + +Workloads already running before assignment of a pod security policy are grandfathered in. Even if they don't meet your pod security policy, workloads running before assignment of the policy continue to run. + +To check if a running workload passes your pod security policy, clone or upgrade it. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md new file mode 100644 index 00000000000..9b4d29eda6f --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md @@ -0,0 +1,29 @@ +--- +title: Assigning Pod Security Policies +--- + + + + + +_Pod Security Policies_ are objects that control security-sensitive aspects of pod specification (like root privileges). + +## Adding a Default Pod Security Policy + +When you create a new cluster with RKE, you can configure it to apply a PSP immediately. As you create the cluster, use the **Cluster Options** to enable a PSP. The PSP assigned to the cluster will be the default PSP for projects within the cluster. + +:::note Prerequisite: + +Create a Pod Security Policy within Rancher. Before you can assign a default PSP to a new cluster, you must have a PSP available for assignment. For instruction, see [Creating Pod Security Policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md). + +::: + +:::note + +For security purposes, we recommend assigning a PSP as you create your clusters. + +::: + +To enable a default Pod Security Policy, set the **Pod Security Policy Support** option to **Enabled**, and then make a selection from the **Default Pod Security Policy** drop-down. + +When the cluster finishes provisioning, the PSP you selected is applied to all projects within the cluster. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md new file mode 100644 index 00000000000..b45beab4572 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -0,0 +1,380 @@ +--- +title: Removing Kubernetes Components from Nodes +description: Learn about cluster cleanup when removing nodes from your Rancher-launched Kubernetes cluster. What is removed, how to do it manually +--- + + + + + +This section describes how to disconnect a node from a Rancher-launched Kubernetes cluster and remove all of the Kubernetes components from the node. This process allows you to use the node for other purposes. + +When you use Rancher to install Kubernetes on new nodes in an infrastructure provider, resources (containers/virtual network interfaces) and configuration items (certificates/configuration files) are created. + +When removing nodes from your Rancher launched Kubernetes cluster (provided that they are in `Active` state), those resources are automatically cleaned, and the only action needed is to restart the node. When a node has become unreachable and the automatic cleanup process cannot be used, we describe the steps that need to be executed before the node can be added to a cluster again. + +## What Gets Removed? + +When cleaning nodes provisioned using Rancher, the following components are deleted based on the type of cluster node you're removing. + +| Removed Component | [Nodes Hosted by Infrastructure Provider][1] | [Custom Nodes][2] | [Hosted Cluster][3] | [Registered Nodes][4] | +|--------------------------------------------------------------------------------|----------------------------------------------|-------------------|---------------------|-----------------------| +| The Rancher deployment namespace (`cattle-system` by default) | ✓ | ✓ | ✓ | ✓ | +| `serviceAccount`, `clusterRoles`, and `clusterRoleBindings` labeled by Rancher | ✓ | ✓ | ✓ | ✓ | +| Labels, Annotations, and Finalizers | ✓ | ✓ | ✓ | ✓ | +| Rancher Deployment | ✓ | ✓ | ✓ | | +| Machines, clusters, projects, and user custom resource definitions (CRDs) | ✓ | ✓ | ✓ | | +| All resources create under the `management.cattle.io` API Group | ✓ | ✓ | ✓ | | +| All CRDs created by Rancher v2.x | ✓ | ✓ | ✓ | | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +## Removing a Node from a Cluster by Rancher UI + +When the node is in `Active` state, removing the node from a cluster will trigger a process to clean up the node. Please restart the node after the automatic cleanup process is done to make sure any non-persistent data is properly removed. + +**To restart a node:** + +``` +# using reboot +$ sudo reboot + +# using shutdown +$ sudo shutdown -r now +``` + +## Removing Rancher Components from a Cluster Manually + +When a node is unreachable and removed from the cluster, the automatic cleaning process can't be triggered because the node is unreachable. Please follow the steps below to manually remove the Rancher components. + +:::danger + +The commands listed below will remove data from the node. Make sure you have created a backup of files you want to keep before executing any of the commands as data will be lost. + +::: + +### Removing Rancher Components from Registered Clusters + +For registered clusters, the process for removing Rancher is a little different. You have the option of simply deleting the cluster in the Rancher UI, or your can run a script that removes Rancher components from the nodes. Both options make the same deletions. + +After the registered cluster is detached from Rancher, the cluster's workloads will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. + + + + +:::danger + +This process will remove data from your cluster. Make sure you have created a backup of files you want to keep before executing the command, as data will be lost. + +::: + +After you initiate the removal of a registered cluster using the Rancher UI (or API), the following events occur. + +1. Rancher creates a `serviceAccount` that it uses to remove the Rancher components from the cluster. This account is assigned the [clusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole) and [clusterRoleBinding](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding) permissions, which are required to remove the Rancher components. + +1. Using the `serviceAccount`, Rancher schedules and runs a [job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) that cleans the Rancher components off of the cluster. This job also references the `serviceAccount` and its roles as dependencies, so the job deletes them before its completion. + +1. Rancher is removed from the cluster. However, the cluster persists, running the native version of Kubernetes. + +**Result:** All components listed for registered clusters in [What Gets Removed?](#what-gets-removed) are deleted. + + + + +Rather than cleaning registered cluster nodes using the Rancher UI, you can run a script instead. + +:::note Prerequisite: + +Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). + +::: + +1. Open a web browser, navigate to [GitHub](https://github.com/rancher/rancher/blob/master/cleanup/user-cluster.sh), and download `user-cluster.sh`. + +1. Make the script executable by running the following command from the same directory as `user-cluster.sh`: + + ``` + chmod +x user-cluster.sh + ``` + +1. **Air Gap Environments Only:** Open `user-cluster.sh` and replace `yaml_url` with the URL in `user-cluster.yml`. + + If you don't have an air gap environment, skip this step. + +1. From the same directory, run the script and provide the `rancher/rancher-agent` image version which should be equal to the version of Rancher used to manage the cluster. (``): + + :::tip + + Add the `-dry-run` flag to preview the script's outcome without making changes. + ``` + ./user-cluster.sh rancher/rancher-agent: + ``` + + ::: + +**Result:** The script runs. All components listed for registered clusters in [What Gets Removed?](#what-gets-removed) are deleted. + + + + +## Cleaning up Nodes + + + + +Before you run the following commands, first remove the node through the Rancher UI. + +To remove a node: + +1. Click **☰** and select **Cluster Management**. +1. In the table of clusters, click the name of the cluster the node belongs to. +1. In the first tab, click the checkbox next to the node's state. +1. Click **Delete**. + +If you remove the entire cluster instead of an individual node, or skip rermoving the node through the Rancher UI, follow these steps: + +1. [Remove](#docker-containers-images-and-volumes) the Docker containers from the node and [unmount](#mounts) any volumes. +1. Reboot the node. +1. [Remove](#directories-and-files) any remaining files. +1. Confirm that network interfaces and IP tables were properly cleaned after the reboot. If not, reboot one more time. + +### Windows Nodes + +To clean up a Windows node, run the script in `c:\etc\rancher`. This script deletes Kubernetes-generated resources and the execution binary. It also drops the firewall rules and network settings: + +``` +pushd c:\etc\rancher +.\cleanup.ps1 +popd +``` + +After you run this script, the node is reset and can be re-added to a Kubernetes cluster. + +### Docker Containers, Images, and Volumes + +:::caution + +Be careful when cleaning up Docker containers. The following command will remove *all* Docker containers, images, and volumes on the node, including non-Rancher related containers: + +::: + +``` +docker rm -f $(docker ps -qa) +docker rmi -f $(docker images -q) +docker volume rm $(docker volume ls -q) +``` + +### Mounts + +Kubernetes components and secrets leave behind the following mounts: + +* `/var/lib/kubelet` +* `/var/lib/rancher` +* Miscellaneous mounts in `/var/lib/kubelet/pods/` + +To unmount all mounts, run: + +``` +for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done +``` + + + + +You need to remove the following components from Rancher-provisioned RKE2 nodes: + +* The rancher-system-agent, which connects to Rancher and installs and manages RKE2. +* RKE2 itself. + +### Removing rancher-system-agent + +To remove the rancher-system-agent, run the [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) script: + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### Removing RKE2 + +To remove the RKE2 installation, run the `rke2-uninstall` script that is already present on the node: + +``` +sudo rke2-uninstall.sh +``` + + + + +You need to remove the following components from Rancher-provisioned K3s nodes: + +* The rancher-system-agent, which connects to Rancher and installs and manages K3s. +* K3s itself. + +### Removing rancher-system-agent + +To remove the rancher-system-agent, run the [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) script: + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### Removing K3s + +To remove the K3s installation, run the `k3s-uninstall` script that is already present on the node: + +``` +sudo k3s-uninstall.sh +``` + + + + +### Directories and Files + +The following directories are used when adding a node to a cluster, and should be removed. You can remove a directory using `rm -rf /directory_name`. + +:::note + +Depending on the role you assigned to the node, some of the directories will or won't be present on the node. + +::: + +| Directories | +|------------------------------| +| `/etc/ceph` | +| `/etc/cni` | +| `/etc/kubernetes` | +| `/etc/rancher` | +| `/opt/cni` | +| `/opt/rke` | +| `/run/secrets/kubernetes.io` | +| `/run/calico` | +| `/run/flannel` | +| `/var/lib/calico` | +| `/var/lib/etcd` | +| `/var/lib/cni` | +| `/var/lib/kubelet` | +| `/var/lib/rancher` | +| `/var/log/containers` | +| `/var/log/kube-audit` | +| `/var/log/pods` | +| `/var/run/calico` | + +**To clean the directories:** + +``` +rm -rf /etc/ceph \ + /etc/cni \ + /etc/kubernetes \ + /etc/rancher \ + /opt/cni \ + /opt/rke \ + /run/secrets/kubernetes.io \ + /run/calico \ + /run/flannel \ + /var/lib/calico \ + /var/lib/etcd \ + /var/lib/cni \ + /var/lib/kubelet \ + /var/lib/rancher\ + /var/log/containers \ + /var/log/kube-audit \ + /var/log/pods \ + /var/run/calico +``` + +### Network Interfaces and Iptables + +The remaining two components that are changed/configured are (virtual) network interfaces and iptables rules. Both are non-persistent to the node, meaning that they will be cleared after a restart of the node. To remove these components, a restart is recommended. + +**To restart a node:** + +``` +# using reboot +$ sudo reboot + +# using shutdown +$ sudo shutdown -r now +``` + +If you want to know more on (virtual) network interfaces or iptables rules, please see the specific subjects below. + +### Network Interfaces + +:::note + +Depending on the network provider configured for the cluster the node was part of, some of the interfaces will or won't be present on the node. + +::: + +| Interfaces | +|--------------------------------------------| +| `flannel.1` | +| `cni0` | +| `tunl0` | +| `caliXXXXXXXXXXX` (random interface names) | +| `vethXXXXXXXX` (random interface names) | + +**To list all interfaces:** + +``` +# Using ip +ip address show + +# Using ifconfig +ifconfig -a +``` + +**To remove an interface:** + +``` +ip link delete interface_name +``` + +### Iptables + +:::note + +Depending on the network provider configured for the cluster the node was part of, some of the chains will or won't be present on the node. + +::: + +Iptables rules are used to route traffic from and to containers. The created rules are not persistent, so restarting the node will restore iptables to its original state. + +| Chains | +|--------------------------------------------------| +| `cali-failsafe-in` | +| `cali-failsafe-out` | +| `cali-fip-dnat` | +| `cali-fip-snat` | +| `cali-from-hep-forward` | +| `cali-from-host-endpoint` | +| `cali-from-wl-dispatch` | +| `cali-fw-caliXXXXXXXXXXX` (random chain names) | +| `cali-nat-outgoing` | +| `cali-pri-kns.NAMESPACE` (chain per namespace) | +| `cali-pro-kns.NAMESPACE` (chain per namespace) | +| `cali-to-hep-forward` | +| `cali-to-host-endpoint` | +| `cali-to-wl-dispatch` | +| `cali-tw-caliXXXXXXXXXXX` (random chain names) | +| `cali-wl-to-host` | +| `KUBE-EXTERNAL-SERVICES` | +| `KUBE-FIREWALL` | +| `KUBE-MARK-DROP` | +| `KUBE-MARK-MASQ` | +| `KUBE-NODEPORTS` | +| `KUBE-SEP-XXXXXXXXXXXXXXXX` (random chain names) | +| `KUBE-SERVICES` | +| `KUBE-SVC-XXXXXXXXXXXXXXXX` (random chain names) | + +**To list all iptables rules:** + +``` +iptables -L -t nat +iptables -L -t mangle +iptables -L +``` diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md new file mode 100644 index 00000000000..aef5e622446 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md @@ -0,0 +1,41 @@ +--- +title: GlusterFS Volumes +--- + + + + + +:::note + +This section only applies to [RKE clusters.](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +::: + +In clusters that store data on GlusterFS volumes, you may experience an issue where pods fail to mount volumes after restarting the `kubelet`. The logging of the `kubelet` will show: `transport endpoint is not connected`. To prevent this from happening, you can configure your cluster to mount the `systemd-run` binary in the `kubelet` container. There are two requirements before you can change the cluster configuration: + +- The node needs to have the `systemd-run` binary installed (this can be checked by using the command `which systemd-run` on each cluster node) +- The `systemd-run` binary needs to be compatible with Debian OS on which the hyperkube image is based (this can be checked using the following command on each cluster node, replacing the image tag with the Kubernetes version you want to use) + +``` +docker run -v /usr/bin/systemd-run:/usr/bin/systemd-run --entrypoint /usr/bin/systemd-run rancher/hyperkube:v1.16.2-rancher1 --version +``` + +:::caution + +Before updating your Kubernetes YAML to mount the `systemd-run` binary, make sure the `systemd` package is installed on your cluster nodes. If this package isn't installed _before_ the bind mounts are created in your Kubernetes YAML, Docker will automatically create the directories and files on each node and will not allow the package install to succeed. + +::: + +``` +services: + kubelet: + extra_binds: + - "/usr/bin/systemd-run:/usr/bin/systemd-run" +``` + +After the cluster has finished provisioning, you can check the `kubelet` container logging to see if the functionality is activated by looking for the following logline: + +``` +Detected OS with systemd +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md new file mode 100644 index 00000000000..f4acbdf85ef --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md @@ -0,0 +1,76 @@ +--- +title: How Persistent Storage Works +--- + + + + + +A persistent volume (PV) is a piece of storage in the Kubernetes cluster, while a persistent volume claim (PVC) is a request for storage. + +There are two ways to use persistent storage in Kubernetes: + +- Use an existing persistent volume +- Dynamically provision new persistent volumes + +To use an existing PV, your application will need to use a PVC that is bound to a PV, and the PV should include the minimum resources that the PVC requires. + +For dynamic storage provisioning, your application will need to use a PVC that is bound to a storage class. The storage class contains the authorization to provision new persistent volumes. + +![Setting Up New and Existing Persistent Storage](/img/rancher-storage.svg) + +For more information, refer to the [official Kubernetes documentation on storage](https://kubernetes.io/docs/concepts/storage/volumes/) + + +## About Persistent Volume Claims + +Persistent volume claims (PVCs) are objects that request storage resources from your cluster. They're similar to a voucher that your deployment can redeem for storage access. A PVC is mounted into a workloads as a volume so that the workload can claim its specified share of the persistent storage. + +To access persistent storage, a pod must have a PVC mounted as a volume. This PVC lets your deployment application store its data in an external location, so that if a pod fails, it can be replaced with a new pod and continue accessing its data stored externally, as though an outage never occurred. + +Each Rancher project contains a list of PVCs that you've created, available from **Resources > Workloads > Volumes**. You can reuse these PVCs when creating deployments in the future. + +### PVCs are Required for Both New and Existing Persistent Storage + +A PVC is required for pods to use any persistent storage, regardless of whether the workload is intended to use storage that already exists, or the workload will need to dynamically provision new storage on demand. + +If you are setting up existing storage for a workload, the workload mounts a PVC, which refers to a PV, which corresponds to existing storage infrastructure. + +If a workload should request new storage, the workload mounts PVC, which refers to a storage class, which has the capability to create a new PV along with its underlying storage infrastructure. + +Rancher lets you create as many PVCs within a project as you'd like. + +You can mount PVCs to a deployment as you create it, or later, after the deployment is running. + +## Setting up Existing Storage with a PVC and PV + +Your pods can store data in [volumes,](https://kubernetes.io/docs/concepts/storage/volumes/) but if the pod fails, that data is lost. To solve this issue, Kubernetes offers persistent volumes (PVs), which are Kubernetes resources that correspond to external storage disks or file systems that your pods can access. If a pod crashes, its replacement pod can access the data in persistent storage without any data loss. + +PVs can represent a physical disk or file system that you host on premise, or a vendor-hosted storage resource, such as Amazon EBS or Azure Disk. + +Creating a persistent volume in Rancher will not create a storage volume. It only creates a Kubernetes resource that maps to an existing volume. Therefore, before you can create a persistent volume as a Kubernetes resource, you must have storage provisioned. + +:::note Important: + +PVs are created at the cluster level, which means that in a multi-tenant cluster, teams with access to separate namespaces could have access to the same PV. + +::: + +### Binding PVs to PVCs + +When pods are set up to use persistent storage, they mount a persistent volume claim (PVC) that is mounted the same way as any other Kubernetes volume. When each PVC is created, the Kubernetes master considers it to be a request for storage and binds it to a PV that matches the minimum resource requirements of the PVC. Not every PVC is guaranteed to be bound to a PV. According to the Kubernetes [documentation,](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + +> Claims will remain unbound indefinitely if a matching volume does not exist. Claims will be bound as matching volumes become available. For example, a cluster provisioned with many 50Gi PVs would not match a PVC requesting 100Gi. The PVC can be bound when a 100Gi PV is added to the cluster. + +In other words, you can create unlimited PVCs, but they will only be bound to PVs if the Kubernetes master can find a sufficient PV that has at least the amount of disk space required by the PVC. + +To dynamically provision new storage, the PVC mounted in the pod would have to correspond to a storage class instead of a persistent volume. + +## Provisioning New Storage with a PVC and Storage Class + +Storage Classes allow you to create PVs dynamically without having to create persistent storage in an infrastructure provider first. + +For example, if a workload is bound to a PVC and the PVC refers to an Amazon EBS Storage Class, the storage class can dynamically create an EBS volume and a corresponding PV. + +The Kubernetes master will then bind the newly created PV to your workload's PVC, allowing your workload to use the persistent storage. + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md new file mode 100644 index 00000000000..5ad0b03cd45 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md @@ -0,0 +1,91 @@ +--- +title: Dynamically Provisioning New Storage in Rancher +--- + + + + + +This section describes how to provision new persistent storage for workloads in Rancher. + +This section assumes that you understand the Kubernetes concepts of storage classes and persistent volume claims. For more information, refer to the section on [how storage works.](about-persistent-storage.md) + +New storage is often provisioned by a cloud provider such as Amazon EBS. However, new storage doesn't have to be in the cloud. + +If you have a pool of block storage, and you don't want to use a cloud provider, Longhorn could help you provide persistent storage to your Kubernetes cluster. For more information, see [this page.](../../../../../integrations-in-rancher/longhorn.md) + +To provision new storage for your workloads, follow these steps: + +1. [Add a storage class and configure it to use your storage.](#1-add-a-storage-class-and-configure-it-to-use-your-storage) +2. [Use the Storage Class for Pods Deployed with a StatefulSet.](#2-use-the-storage-class-for-pods-deployed-with-a-statefulset) + +### Prerequisites + +- To set up persistent storage, the `Manage Volumes` [role](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-role-reference) is required. +- If you are provisioning storage for a cluster hosted in the cloud, the storage and cluster hosts must have the same cloud provider. +- The cloud provider must be enabled. For details on enabling cloud providers, refer to [this page.](../../../../../pages-for-subheaders/set-up-cloud-providers.md) +- Make sure your storage provisioner is available to be enabled. + +The following storage provisioners are enabled by default: + +Name | Plugin +--------|---------- +Amazon EBS Disk | `aws-ebs` +AzureFile | `azure-file` +AzureDisk | `azure-disk` +Google Persistent Disk | `gce-pd` +Longhorn | `flex-volume-longhorn` +VMware vSphere Volume | `vsphere-volume` +Local | `local` +Network File System | `nfs` +hostPath | `host-path` + +To use a storage provisioner that is not on the above list, you will need to use a [feature flag to enable unsupported storage drivers.](../../../../advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) + +### 1. Add a storage class and configure it to use your storage + +These steps describe how to set up a storage class at the cluster level. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to dynamically provision persistent storage volumes and click **Explore**. +1. Click **Storage > Storage Classes**. +1. Click **Create**. +1. Enter a name for your storage class. +1. From the **Provisioner** drop-down, select the service that you want to use to dynamically provision storage volumes. For example, if you have a Amazon EC2 cluster and you want to use cloud storage for it, use the `Amazon EBS Disk` provisioner. +1. In the **Parameters** tab, fill out the information required for the service to dynamically provision storage volumes. Each provisioner requires different information to dynamically provision storage volumes. Consult the service's documentation for help on how to obtain this information. +1. Click **Create**. + +**Result:** The storage class is available to be consumed by a PVC. + +For full information about the storage class parameters, refer to the official [Kubernetes documentation.](https://kubernetes.io/docs/concepts/storage/storage-classes/#parameters). + +### 2. Use the Storage Class for Pods Deployed with a StatefulSet + +StatefulSets manage the deployment and scaling of Pods while maintaining a sticky identity for each Pod. In this StatefulSet, we will configure a VolumeClaimTemplate. Each Pod managed by the StatefulSet will be deployed with a PersistentVolumeClaim based on this VolumeClaimTemplate. The PersistentVolumeClaim will refer to the StorageClass that we created. Therefore, when each Pod managed by the StatefulSet is deployed, it will be bound to dynamically provisioned storage using the StorageClass defined in its PersistentVolumeClaim. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add use the StorageClass for a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Click **Create**. +1. Click **StatefulSet**. +1. In the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Enter a name for the persistent volume. +1. In the **StorageClass* field, select the StorageClass that will dynamically provision storage for pods managed by this StatefulSet. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Launch**. + +**Result:** When each Pod managed by the StatefulSet is deployed, it will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to Pod with a compatible PVC. + +To attach the PVC to an existing workload, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add use the StorageClass for a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Go to the workload that will use storage provisioned with the StorageClass that you cared at click **â‹® > Edit Config**. +1. In the **Volume Claim Templates** section, click **Add Claim Template**. +1. Enter a persistent volume name. +1. In the **StorageClass* field, select the StorageClass that will dynamically provision storage for pods managed by this StatefulSet. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Save**. + +**Result:** The workload will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to the PVC. If not, Rancher will provision new persistent storage. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md new file mode 100644 index 00000000000..cce04e02a24 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md @@ -0,0 +1,66 @@ +--- +title: iSCSI Volumes +--- + + + + + +In [Rancher Launched Kubernetes clusters](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) that store data on iSCSI volumes, you may experience an issue where kubelets fail to automatically connect with iSCSI volumes. This failure is likely due to an incompatibility issue involving the iSCSI initiator tool. You can resolve this issue by installing the iSCSI initiator tool on each of your cluster nodes. + +Rancher Launched Kubernetes clusters storing data on iSCSI volumes leverage the [iSCSI initiator tool](http://www.open-iscsi.com/), which is embedded in the kubelet's `rancher/hyperkube` Docker image. From each kubelet (i.e., the _initiator_), the tool discovers and launches sessions with an iSCSI volume (i.e., the _target_). However, in some instances, the versions of the iSCSI initiator tool installed on the initiator and the target may not match, resulting in a connection failure. + +If you encounter this issue, you can work around it by installing the initiator tool on each node in your cluster. You can install the iSCSI initiator tool by logging into your cluster nodes and entering one of the following commands: + +| Platform | Package Name | Install Command | +| ------------- | ----------------------- | -------------------------------------- | +| Ubuntu/Debian | `open-iscsi` | `sudo apt install open-iscsi` | +| RHEL | `iscsi-initiator-utils` | `yum install iscsi-initiator-utils -y` | + + +After installing the initiator tool on your nodes, edit the YAML for your cluster, editing the kubelet configuration to mount the iSCSI binary and configuration, as shown in the sample below. + +:::note Notes + +- Before updating your Kubernetes YAML to mount the iSCSI binary and configuration, make sure either the `open-iscsi` (deb) or `iscsi-initiator-utils` (yum) package is installed on your cluster nodes. If this package isn't installed _before_ the bind mounts are created in your Kubernetes YAML, Docker will automatically create the directories and files on each node and will not allow the package install to succeed. + +- The example YAML below does not apply to K3s, but only to RKE clusters. Since the K3s kubelet does not run in a container, adding extra binds is not necessary. However, all iSCSI tools must still be installed on your K3s nodes. + +::: + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" +``` + +If you're using RHEL 7.9, you need to mount the file `/usr/lib64/libcrypto.so.10` as shown in the sample below. + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.10:/usr/lib/libcrypto.so.10" +``` + +If you're using RHEL 8.6, you need to mount the file `/usr/lib64/libcrypto.so.1.1` as shown in the sample below. + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.1.1:/usr/lib/libcrypto.so.1.1" +``` + + +:::tip + +To verify iscsiadm is working properly, you can run the command `docker exec kubelet iscsiadm --version` on any node. + +::: diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md new file mode 100644 index 00000000000..60661aea03b --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md @@ -0,0 +1,89 @@ +--- +title: Setting up Existing Storage +--- + + + + + +This section describes how to set up existing persistent storage for workloads in Rancher. + +:::note + +This section assumes that you understand the Kubernetes concepts of persistent volumes and persistent volume claims. For more information, refer to the section on [how storage works.](about-persistent-storage.md) + +::: + +To set up storage, follow these steps: + +1. [Set up persistent storage.](#1-set-up-persistent-storage) +2. [Add a PersistentVolume that refers to the persistent storage.](#2-add-a-persistentvolume-that-refers-to-the-persistent-storage) +3. [Use the Storage Class for Pods Deployed with a StatefulSet.](#3-use-the-storage-class-for-pods-deployed-with-a-statefulset) + +### Prerequisites + +- To create a persistent volume as a Kubernetes resource, you must have the `Manage Volumes` [role.](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-role-reference) +- If you are provisioning storage for a cluster hosted in the cloud, the storage and cluster hosts must have the same cloud provider. + +### 1. Set up persistent storage + +Creating a persistent volume in Rancher will not create a storage volume. It only creates a Kubernetes resource that maps to an existing volume. Therefore, before you can create a persistent volume as a Kubernetes resource, you must have storage provisioned. + +The steps to set up a persistent storage device will differ based on your infrastructure. We provide examples of how to set up storage using [vSphere,](../../provisioning-storage-examples/vsphere-storage.md) [NFS,](../../provisioning-storage-examples/nfs-storage.md) or Amazon's [EBS.](../../provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) + +If you have a pool of block storage, and you don't want to use a cloud provider, Longhorn could help you provide persistent storage to your Kubernetes cluster. For more information, see [this page.](../../../../../integrations-in-rancher/longhorn.md) + +### 2. Add a PersistentVolume that refers to the persistent storage + +These steps describe how to set up a PersistentVolume at the cluster level in Kubernetes. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a persistent volume and click **Explore**. +1. In the left navigation bar, click **Storage > Persistent Volumes**. +1. Click **Create**. +1. Enter a **Name** for the persistent volume. +1. Select the **Volume Plugin** for the disk type or service that you're using. When adding storage to a cluster that's hosted by a cloud provider, use the cloud provider's plug-in for cloud storage. For example, if you have a Amazon EC2 cluster and you want to use cloud storage for it, you must use the `Amazon EBS Disk` volume plugin. +1. Enter the **Capacity** of your volume in gigabytes. +1. Complete the **Plugin Configuration** form. Each plugin type requires information specific to the vendor of disk type. For help regarding each plugin's form and the information that's required, refer to the plug-in's vendor documentation. +1. Optional: In the **Customize** form, configure the [access modes.](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) This options sets how many nodes can access the volume, along with the node read/write permissions. The [Kubernetes Documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) includes a table that lists which access modes are supported by the plugins available. +1. Optional: In the **Customize** form, configure the [mount options.](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options) Each volume plugin allows you to specify additional command line options during the mounting process. Consult each plugin's vendor documentation for the mount options available. +1. Click **Create**. + +**Result:** Your new persistent volume is created. + + +### 3. Use the Storage Class for Pods Deployed with a StatefulSet + +StatefulSets manage the deployment and scaling of Pods while maintaining a sticky identity for each Pod. In this StatefulSet, we will configure a VolumeClaimTemplate. Each Pod managed by the StatefulSet will be deployed with a PersistentVolumeClaim based on this VolumeClaimTemplate. The PersistentVolumeClaim will refer to the PersistentVolume that we created. Therefore, when each Pod managed by the StatefulSet is deployed, it will be bound a PersistentVolume as defined in its PersistentVolumeClaim. + +You can configure storage for the StatefulSet during or after workload creation. + +The following steps describe how to assign existing storage to a new StatefulSet: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to configure storage for the StatefulSet and click **Explore**. +1. In the left navigation bar, click **Workload > StatefulSets**. +1. Click **Create**. +1. Choose the namespace where the workload will be deployed. +1. Enter a name for the StatefulSet. +1. On the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Click **Use an existing Persistent Volume**. +1. In the Persistent Volumes field, select the Persistent Volume that you created. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Launch**. + +**Result:** When the workload is deployed, it will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to the PVC. + +The following steps describe how to assign persistent storage to an existing workload: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to configure storage for the StatefulSet and click **Explore**. +1. In the left navigation bar, click **Workload > StatefulSets**. +1. Go to the workload that you want to add the persistent storage to. Click **â‹® > Edit**. +1. On the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Click **Use an existing Persistent Volume**. +1. In the Persistent Volumes field, select the Persistent Volume that you created. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Launch**. + +**Result:** The workload will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to the PVC. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md new file mode 100644 index 00000000000..67252a62748 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md @@ -0,0 +1,424 @@ +--- +title: Using an External Ceph Driver +--- + + + + + +These instructions are about using the external Ceph driver in an RKE2 cluster. If you are using RKE, additional steps are required. For details, refer to [this section.](#using-the-ceph-driver-with-rke) + + +## Requirements + +Make sure ceph-common and xfsprogs packages are installed on SLE worker nodes. + +## Using the Ceph Driver with RKE + +The resources below are fully compatible with RKE based clusters, but there is a need to do an additional kubelet configuration for RKE. + +On RKE clusters, the kubelet component is running in a Docker container and doesn't have access to the host's kernel modules as rbd and libceph by default. + +To solve this limitation, you can either run `modprobe rbd` on worker nodes, or configure the kubelet containers to automatically mount the `/lib/modules` directory from the host into the container. + +For the kubelet configuration, put the following lines into the `cluster.yml` file prior to RKE cluster provisioning. You can also modify the `cluster.yml` later in the Rancher UI by clicking on **Edit Cluster > Edit as YAML** and restarting the worker nodes. + +```yaml +services: + kubelet: + extra_binds: + - '/lib/modules:/lib/modules:ro' +``` + +For more information about the `extra_binds` directive, refer to [this section.](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds) + +## Installing the ceph-csi driver on an RKE2 cluster + +:::note + +These steps are needed for dynamic RBD provisioning only. + +::: + +For more information about the `ceph-csi-rbd` chart, refer to [this page.](https://github.com/ceph/ceph-csi/blob/devel/charts/ceph-csi-rbd/README.md) + +To get details about your SES cluster, run: + +``` +ceph mon dump +``` + +Read its output: + +``` +dumped monmap epoch 3 +epoch 3 +fsid 79179d9d-98d8-4976-ab2e-58635caa7235 +last_changed 2021-02-11T10:56:42.110184+0000 +created 2021-02-11T10:56:22.913321+0000 +min_mon_release 15 (octopus) +0: [v2:10.85.8.118:3300/0,v1:10.85.8.118:6789/0] mon.a +1: [v2:10.85.8.123:3300/0,v1:10.85.8.123:6789/0] mon.b +2: [v2:10.85.8.124:3300/0,v1:10.85.8.124:6789/0] mon.c +``` + +Later you'll need the fsid and mon addresses values. + +## Install the ceph-csi Driver Using Helm + +Run these commands: + +``` +helm repo add ceph-csi https://ceph.github.io/csi-charts +helm repo update +helm search repo ceph-csi -l +helm inspect values ceph-csi/ceph-csi-rbd > ceph-csi-rbd-values.yaml +``` + +Modify the `ceph-csi-rbd-values.yaml` file and keep there only the required changes: + +```yaml +# ceph-csi-rbd-values.yaml +csiConfig: + - clusterID: "79179d9d-98d8-4976-ab2e-58635caa7235" + monitors: + - "10.85.8.118:6789" + - "10.85.8.123:6789" + - "10.85.8.124:6789" +provisioner: + name: provisioner + replicaCount: 2 +``` + +Make sure the ceph monitors are reachable from the RKE2 cluster, for example, by ping. + +``` +kubectl create namespace ceph-csi-rbd +helm install --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +kubectl rollout status deployment ceph-csi-rbd-provisioner -n ceph-csi-rbd +helm status ceph-csi-rbd -n ceph-csi-rbd +``` + +in case you'd like to modify the configuration directly via Helm, you may adapt the `ceph-csi-rbd-values.yaml` file and call: + +``` +helm upgrade \ + --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +``` + +## Creating RBD Ceph Resources + +``` +# Create a ceph pool: +ceph osd pool create myPool 64 64 + +# Create a block device pool: +rbd pool init myPool + +# Create a block device image: +rbd create -s 2G myPool/image + +# Create a block device user and record the key: +ceph auth get-or-create-key client.myPoolUser mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=myPool" | tr -d '\n' | base64 +QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== + +# Encode the ceph user myPoolUser into a bash64 hash: +echo "myPoolUser" | tr -d '\n' | base64 +bXlQb29sVXNlcg== + +# Create a block device admin user and record the key: +ceph auth get-or-create-key client.myPoolAdmin mds 'allow *' mgr 'allow *' mon 'allow *' osd 'allow * pool=myPool' | tr -d '\n' | base64 +QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== + +# Encode the ceph user myPoolAdmin into a bash64 hash: +echo "myPoolAdmin" | tr -d '\n' | base64 +bXlQb29sQWRtaW4= +``` +## Configure RBD Ceph Access Secrets + +### User Account + +For static RBD provisioning (the image within the ceph pool must exist), run these commands: + +``` +cat > ceph-user-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-user + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sVXNlcg== + userKey: QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== +EOF + +kubectl apply -f ceph-user-secret.yaml +``` + +### Admin Account + +For dynamic RBD provisioning (used for automatic image creation within a given ceph pool), run these commands: + +``` +cat > ceph-admin-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-admin + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sQWRtaW4= + userKey: QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== +EOF + +kubectl apply -f ceph-admin-secret.yaml +``` + +## Create RBD Testing Resources + +### Using RBD in Pods + +```yaml +# pod +cat > ceph-rbd-pod-inline.yaml << EOF +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-inline +spec: + containers: + - name: ceph-rbd-pod-inline + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +EOF + +kubectl apply -f ceph-rbd-pod-inline.yaml +kubectl get pod +kubectl exec pod/ceph-rbd-pod-inline -- df -k | grep rbd +``` + +### Using RBD in Persistent Volumes + +```yaml +# pod-pvc-pv +cat > ceph-rbd-pod-pvc-pv-allinone.yaml << EOF +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ceph-rbd-pv +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ceph-rbd-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-pvc-pv +spec: + containers: + - name: ceph-rbd-pod-pvc-pv + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + persistentVolumeClaim: + claimName: ceph-rbd-pvc +EOF + +kubectl apply -f ceph-rbd-pod-pvc-pv-allinone.yaml +kubectl get pv,pvc,pod +kubectl exec pod/ceph-rbd-pod-pvc-pv -- df -k | grep rbd +``` + +### Using RBD in Storage Classes + +This example is for dynamic provisioning. The ceph-csi driver is needed. + +```yaml +# pod-pvc-sc +cat > ceph-rbd-pod-pvc-sc-allinone.yaml < /root/.bashrc << EOF +export PATH=$PATH:/var/lib/rancher/rke2/bin/ +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +EOF + +cat /var/lib/rancher/rke2/server/node-token +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +``` + +### RKE2 Agent/Worker provisioning + +``` +mkdir -p /etc/rancher/rke2/ + +cat > /etc/rancher/rke2/config.yaml << EOF +server: https://10.100.103.23:9345 +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +EOF + +curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh - +systemctl enable --now rke2-agent.service +``` + +To import the cluster into Rancher, click **☰ > Cluster Management**. Then on the **Clusters** page, click **Import Existing**. Then run the provided kubectl command on the server/master node. + +## Tested Versions + +OS for running RKE2 nodes: JeOS SLE15-SP2 with installed kernel-default-5.3.18-24.49 + +``` +kubectl version +Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.4", GitCommit:"c96aede7b5205121079932896c4ad89bb93260af", GitTreeState:"clean", BuildDate:"2020-06-22T12:00:00Z", GoVersion:"go1.13.11", Compiler:"gc", Platform:"linux/amd64"} +Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7+rke2r1", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-20T01:50:52Z", GoVersion:"go1.15.5b5", Compiler:"gc", Platform:"linux/amd64"} + +helm version +version.BuildInfo{Version:"3.4.1", GitCommit:"c4e74854886b2efe3321e185578e6db9be0a6e29", GitTreeState:"clean", GoVersion:"go1.14.12"} +``` + +Kubernetes version on RKE2 cluster: v1.19.7+rke2r1 + +## Troubleshooting + +In case you are using SUSE's ceph-rook based on SES7, it might be useful to expose the monitors on hostNetwork by editing `rook-1.4.5/ceph/cluster.yaml` and setting `spec.network.hostNetwork=true`. + +Also for operating the ceph-rook cluster, it is useful to deploy a toolbox on the Kubernetes cluster where ceph-rook is provisioned by `kubectl apply -f rook-1.4.5/ceph/toolbox.yaml` Then all the ceph related commands can be executed in the toolbox pod, for example, by running `kubectl exec -it -n rook-ceph rook-ceph-tools-686d8b8bfb-2nvqp -- bash` + +Operating with the ceph - basic commands: + +``` +ceph osd pool stats +ceph osd pool delete myPool myPool --yes-i-really-really-mean-it +rbd list -p myPool +> csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 +> image +``` + +Delete the image: `rbd rm csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 -p myPool` + +CephFS commands in rook toolbox: + +``` +ceph -s +ceph fs ls +ceph fs fail cephfs +ceph fs rm cephfs --yes-i-really-mean-it +ceph osd pool delete cephfs_data cephfs_data --yes-i-really-really-mean-it +ceph osd pool delete cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it +``` + +To prepare a cephfs filesystem, you can run this command on a rook cluster: + +``` +kubectl apply -f rook-1.4.5/ceph/filesystem.yaml +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md new file mode 100644 index 00000000000..c8debb174ad --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md @@ -0,0 +1,579 @@ +--- +title: Cluster Autoscaler with AWS EC2 Auto Scaling Groups +--- + + + + + +This guide will show you how to install and use [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/) on Rancher custom clusters using AWS EC2 Auto Scaling Groups. + +We are going to install a Rancher RKE custom cluster with a fixed number of nodes with the etcd and controlplane roles, and a variable nodes with the worker role, managed by `cluster-autoscaler`. + + +## Prerequisites + +These elements are required to follow this guide: + +* The Rancher server is up and running +* You have an AWS EC2 user with proper permissions to create virtual machines, auto scaling groups, and IAM profiles and roles + +### 1. Create a Custom Cluster + +On Rancher server, we should create a custom k8s cluster. Refer [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/) to check for version compatibility. + +Be sure that cloud_provider name is set to `amazonec2`. Once cluster is created we need to get: + +* clusterID: `c-xxxxx` will be used on EC2 `kubernetes.io/cluster/` instance tag +* clusterName: will be used on EC2 `k8s.io/cluster-autoscaler/` instance tag +* nodeCommand: will be added on EC2 instance user_data to include new nodes on cluster + + ```sh + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum + ``` + +### 2. Configure the Cloud Provider + +On AWS EC2, we should create a few objects to configure our system. We've defined three distinct groups and IAM profiles to configure on AWS. + +1. Autoscaling group: Nodes that will be part of the EC2 Auto Scaling Group (ASG). The ASG will be used by `cluster-autoscaler` to scale up and down. + * IAM profile: Required by k8s nodes where cluster-autoscaler will be running. It is recommended for Kubernetes master nodes. This profile is called `K8sAutoscalerProfile`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:DescribeTags", + "autoscaling:DescribeLaunchConfigurations", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + +2. Master group: Nodes that will be part of the Kubernetes etcd and/or control planes. This will be out of the ASG. + * IAM profile: Required by the Kubernetes cloud_provider integration. Optionally, `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` can be used instead [using-aws-credentials.](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) This profile is called `K8sMasterProfile`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + + * IAM role: `K8sMasterRole: [K8sMasterProfile,K8sAutoscalerProfile]` + * Security group: `K8sMasterSg` More info at [RKE ports (custom nodes tab)](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#downstream-kubernetes-cluster-nodes) + * Tags: + `kubernetes.io/cluster/: owned` + * User data: `K8sMasterUserData` Ubuntu 18.04(ami-0e11cbb34015ff725), installs docker and add etcd+controlplane node to the k8s cluster + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--etcd --controlplane" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +3. Worker group: Nodes that will be part of the k8s worker plane. Worker nodes will be scaled by cluster-autoscaler using the ASG. + * IAM profile: Provides cloud_provider worker integration. + This profile is called `K8sWorkerProfile`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } + ] + } + ``` + + * IAM role: `K8sWorkerRole: [K8sWorkerProfile]` + * Security group: `K8sWorkerSg` More info at [RKE ports (custom nodes tab)](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#downstream-kubernetes-cluster-nodes) + * Tags: + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` + * User data: `K8sWorkerUserData` Ubuntu 18.04(ami-0e11cbb34015ff725), installs docker and add worker node to the k8s cluster + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--worker" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +More info is at [RKE clusters on AWS](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md) and [Cluster Autoscaler on AWS.](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) + +### 3. Deploy Nodes + +Once we've configured AWS, let's create VMs to bootstrap our cluster: + +* master (etcd+controlplane): Depending your needs, deploy three master instances with proper size. More info is at [the recommendations for production-ready clusters.](../../../../pages-for-subheaders/checklist-for-production-ready-clusters.md) + * IAM role: `K8sMasterRole` + * Security group: `K8sMasterSg` + * Tags: + * `kubernetes.io/cluster/: owned` + * User data: `K8sMasterUserData` + +* worker: Define an ASG on EC2 with the following settings: + * Name: `K8sWorkerAsg` + * IAM role: `K8sWorkerRole` + * Security group: `K8sWorkerSg` + * Tags: + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` + * User data: `K8sWorkerUserData` + * Instances: + * minimum: 2 + * desired: 2 + * maximum: 10 + +Once the VMs are deployed, you should have a Rancher custom cluster up and running with three master and two worker nodes. + +### 4. Install Cluster-autoscaler + +At this point, we should have rancher cluster up and running. We are going to install cluster-autoscaler on master nodes and `kube-system` namespace, following cluster-autoscaler recommendation. + +#### Parameters + +This table shows cluster-autoscaler parameters for fine tuning: + +| Parameter | Default | Description | +|---|---|---| +|cluster-name|-|Autoscaled cluster name, if available| +|address|:8085|The address to expose Prometheus metrics| +|kubernetes|-|Kubernetes master location. Leave blank for default| +|kubeconfig|-|Path to kubeconfig file with authorization and master location information| +|cloud-config|-|The path to the cloud provider configuration file. Empty string for no configuration file| +|namespace|"kube-system"|Namespace in which cluster-autoscaler run| +|scale-down-enabled|true|Should CA scale down the cluster| +|scale-down-delay-after-add|"10m"|How long after scale up that scale down evaluation resumes| +|scale-down-delay-after-delete|0|How long after node deletion that scale down evaluation resumes, defaults to scanInterval| +|scale-down-delay-after-failure|"3m"|How long after scale down failure that scale down evaluation resumes| +|scale-down-unneeded-time|"10m"|How long a node should be unneeded before it is eligible for scale down| +|scale-down-unready-time|"20m"|How long an unready node should be unneeded before it is eligible for scale down| +|scale-down-utilization-threshold|0.5|Sum of cpu or memory of all pods running on the node divided by node's corresponding allocatable resource, below which a node can be considered for scale down| +|scale-down-gpu-utilization-threshold|0.5|Sum of gpu requests of all pods running on the node divided by node's allocatable resource, below which a node can be considered for scale down| +|scale-down-non-empty-candidates-count|30|Maximum number of non empty nodes considered in one iteration as candidates for scale down with drain| +|scale-down-candidates-pool-ratio|0.1|A ratio of nodes that are considered as additional non empty candidates for scale down when some candidates from previous iteration are no longer valid| +|scale-down-candidates-pool-min-count|50|Minimum number of nodes that are considered as additional non empty candidates for scale down when some candidates from previous iteration are no longer valid| +|node-deletion-delay-timeout|"2m"|Maximum time CA waits for removing delay-deletion.cluster-autoscaler.kubernetes.io/ annotations before deleting the node| +|scan-interval|"10s"|How often cluster is reevaluated for scale up or down| +|max-nodes-total|0|Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number| +|cores-total|"0:320000"|Minimum and maximum number of cores in cluster, in the format `:.` Cluster autoscaler will not scale the cluster beyond these numbers| +|memory-total|"0:6400000"|Minimum and maximum number of gigabytes of memory in cluster, in the format `:.` Cluster autoscaler will not scale the cluster beyond these numbers| +cloud-provider|-|Cloud provider type| +|max-bulk-soft-taint-count|10|Maximum number of nodes that can be tainted/untainted PreferNoSchedule at the same time. Set to 0 to turn off such tainting| +|max-bulk-soft-taint-time|"3s"|Maximum duration of tainting/untainting nodes as PreferNoSchedule at the same time| +|max-empty-bulk-delete|10|Maximum number of empty nodes that can be deleted at the same time| +|max-graceful-termination-sec|600|Maximum number of seconds CA waits for pod termination when trying to scale down a node| +|max-total-unready-percentage|45|Maximum percentage of unready nodes in the cluster. After this is exceeded, CA halts operations| +|ok-total-unready-count|3|Number of allowed unready nodes, irrespective of max-total-unready-percentage| +|scale-up-from-zero|true|Should CA scale up when there 0 ready nodes| +|max-node-provision-time|"15m"|Maximum time CA waits for node to be provisioned| +|nodes|-|sets min,max size and other configuration data for a node group in a format accepted by cloud provider. Can be used multiple times. Format: `::`| +|node-group-auto-discovery|-|One or more definition(s) of node group auto-discovery. A definition is expressed `:[[=]]`| +|estimator|-|"binpacking"|Type of resource estimator to be used in scale up. Available values: ["binpacking"]| +|expander|"random"|Type of node group expander to be used in scale up. Available values: `["random","most-pods","least-waste","price","priority"]`| +|ignore-daemonsets-utilization|false|Should CA ignore DaemonSet pods when calculating resource utilization for scaling down| +|ignore-mirror-pods-utilization|false|Should CA ignore Mirror pods when calculating resource utilization for scaling down| +|write-status-configmap|true|Should CA write status information to a configmap| +|max-inactivity|"10m"|Maximum time from last recorded autoscaler activity before automatic restart| +|max-failing-time|"15m"|Maximum time from last recorded successful autoscaler run before automatic restart| +|balance-similar-node-groups|false|Detect similar node groups and balance the number of nodes between them| +|node-autoprovisioning-enabled|false|Should CA autoprovision node groups when needed| +|max-autoprovisioned-node-group-count|15|The maximum number of autoprovisioned groups in the cluster| +|unremovable-node-recheck-timeout|"5m"|The timeout before we check again a node that couldn't be removed before| +|expendable-pods-priority-cutoff|-10|Pods with priority below cutoff will be expendable. They can be killed without any consideration during scale down and they don't cause scale up. Pods with null priority (PodPriority disabled) are non expendable| +|regional|false|Cluster is regional| +|new-pod-scale-up-delay|"0s"|Pods less than this old will not be considered for scale-up| +|ignore-taint|-|Specifies a taint to ignore in node templates when considering to scale a node group| +|balancing-ignore-label|-|Specifies a label to ignore in addition to the basic and cloud-provider set of labels when comparing if two node groups are similar| +|aws-use-static-instance-list|false|Should CA fetch instance types in runtime or use a static list. AWS only| +|profiling|false|Is debug/pprof endpoint enabled| + +#### Deployment + +Based on the [cluster-autoscaler-run-on-control-plane.yaml](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml) example, we've created our own `cluster-autoscaler-deployment.yaml` to use preferred [auto-discovery setup](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#auto-discovery-setup), updating tolerations, nodeSelector, image version and command config: + + +```yml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/controlplane + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + containers: + - image: eu.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler: + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + +``` + +Once the manifest file is prepared, deploy it in the Kubernetes cluster (Rancher UI can be used instead): + +```sh +kubectl -n kube-system apply -f cluster-autoscaler-deployment.yaml +``` + +:::note + +Cluster-autoscaler deployment can also be set up using [manual configuration](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#manual-configuration) + +::: + +## Testing + +At this point, we should have a cluster-scaler up and running in our Rancher custom cluster. Cluster-scale should manage `K8sWorkerAsg` ASG to scale up and down between 2 and 10 nodes, when one of the following conditions is true: + +* There are pods that failed to run in the cluster due to insufficient resources. In this case, the cluster is scaled up. +* There are nodes in the cluster that have been underutilized for an extended period of time and their pods can be placed on other existing nodes. In this case, the cluster is scaled down. + +### Generating Load + +We've prepared a `test-deployment.yaml` just to generate load on the Kubernetes cluster and see if cluster-autoscaler is working properly. The test deployment is requesting 1000m CPU and 1024Mi memory by three replicas. Adjust the requested resources and/or replica to be sure you exhaust the Kubernetes cluster resources: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hello-world + name: hello-world +spec: + replicas: 3 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + ports: + - containerPort: 80 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1024Mi + requests: + cpu: 1000m + memory: 1024Mi +``` + +Once the test deployment is prepared, deploy it in the Kubernetes cluster default namespace (Rancher UI can be used instead): + +``` +kubectl -n default apply -f test-deployment.yaml +``` + +### Checking Scale + +Once the Kubernetes resources got exhausted, cluster-autoscaler should scale up worker nodes where pods failed to be scheduled. It should scale up until up until all pods became scheduled. You should see the new nodes on the ASG and on the Kubernetes cluster. Check the logs on the `kube-system` cluster-autoscaler pod. + +Once scale up is checked, let check for scale down. To do it, reduce the replica number on the test deployment until you release enough Kubernetes cluster resources to scale down. You should see nodes disappear on the ASG and on the Kubernetes cluster. Check the logs on the `kube-system` cluster-autoscaler pod. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md new file mode 100644 index 00000000000..0d4b15bc48c --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md @@ -0,0 +1,134 @@ +--- +title: Cluster Templates +--- + + + + + +Cluster templates encompass both Kubernetes configuration and node pool configuration, allowing a single template to contain all the information Rancher needs to provision new nodes in a cloud provider and install Kubernetes on those nodes. + +## Overview + +Cluster templates are provided as Helm charts. To use them, you will need to clone and fork the templates, change them according to your use case, and then install the Helm charts on the Rancher management cluster. When the Helm chart is installed on the Rancher management cluster, a new cluster resource is created, which Rancher uses to provision the new cluster. + +After the cluster is provisioned using the template, no changes to the template will affect the cluster. After the cluster is created from the cluster template, its configuration and infrastructure can change, because no restrictions are enforced by cluster templates. + +### Kubernetes Distribution + +Cluster templates can use any Kubernetes distribution. For now, we provide an example with an RKE2 Kubernetes cluster. We may provide more examples in the future using other Kubernetes distributions. + +### Versioning + +Rancher doesn't manage version control for cluster templates. Version control is handled in the repository containing the template's Helm chart. + +## RKE2 Cluster Template + +See the example repository for an [RKE2 cluster template](https://github.com/rancher/cluster-template-examples). + +## Adding a Cluster Template to Rancher + +In this section, you'll learn how to add the cluster template to the `local` cluster's chart repo list. The result is that Rancher will include the cluster template as an option when users install new Kubernetes clusters. + +:::note Prerequisites: + +- You will need permission to install Helm charts on the `local` Rancher cluster. +- To make the chart viewable in the cluster creation form, both the chart and the index.yaml file must have the annotation, `catalog.cattle.io/type: cluster-template`. + +::: + +1. Go to a cluster template example repository. Rancher's examples are in [this GitHub repository.](https://github.com/rancher/cluster-template-examples) +1. Fork the repository. +1. Optional: Edit the cluster options by editing the `values.yaml` file. For help editing the file, see the cluster template's Helm chart README. +1. Add the chart repository to Rancher. Click **☰ > Cluster Management**. +1. Go to the `local` cluster and click **Explore.** +1. In the left navigation bar, click **Apps > Chart Repositories.** +1. Click **Create.** +1. Enter a name for the cluster template repository. +1. Click **Git Repository containing Helm chart definitions.** +1. In the **Git Repo URL** field, enter the URL for the repository. For example, `https://github.com/rancher/cluster-template-examples.git`. +1. In the **Git Branch** field, enter the branch to use as the source for the template. Rancher's example repository uses `main`. +1. Click **Create.** + +**Result:** The cluster template available from the **Apps** in Rancher's `local` cluster. It can now be used to deploy clusters. + +:::note Restricted Admin access: + +If you are a restricted admin and don’t have access to the `local` cluster, you may still add new RKE2 templates and manage cluster repositories. To navigate to the chart repository, go to the left navigation bar and click **☰ > Cluster Management > Advanced > Repositories**. You will bypass steps 1 - 6 above, then proceed to follow steps 7 - 12 to create the cluster template. + +::: + +## Creating a Cluster from a Cluster Template + +:::note Prerequisites: + +- You will need permission to provision new Kubernetes clusters. +- You will need cloud credentials for provisioning infrastructure using the template. +- In order to show in the form for creating new clusters, the cluster template's Helm chart must have the `catalog.cattle.io/type: cluster-template` annotation. + +::: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create.** +1. Click the name of your cluster template. +1. Finish installing the Helm chart. + +**Result:** After Rancher provisions the new cluster, it is managed in the same way as any other Rancher-launched Kubernetes cluster. You can configure any options through the UI if the cluster template has options for the user to choose from. + +## Updating a Cluster Created from a Cluster Template + +You can update any clusters using a template from the **Apps > Installed Apps** page, given there is a new version of a template being used by those clusters. + +## Deploying Clusters from a Template with Fleet + +:::note Prerequisites: + +- You will need permission to provision new Kubernetes clusters. +- You will need cloud credentials for provisioning infrastructure using the template. +- In order to show in the form for creating new clusters, the cluster template's Helm chart must have the `catalog.cattle.io/type:cluster-template` annotation. +- In order to use a template as part of continuous delivery/GitOps, the cluster template needs to be deployed in the `fleet-local` namespace of the `local` cluster. +- All values must be set in the `values.yaml` of the template. +- Fleet repositories must follow these [guidelines](https://fleet.rancher.io/gitrepo-content). For RKE2 cluster templates, that means a `fleet.yaml` file must be added to the repository. + +::: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create.** +1. Click **Create Cluster from Template.** + +**Result:** After Rancher provisions the new cluster, it is managed by Fleet. + +## Uninstalling Cluster Templates + +1. Click **☰ > Cluster Management**. +1. Go to the `local` cluster and click **Apps > Chart Repositories.** +1. Go to the chart repository for your cluster template and click **â‹® > Delete.** +1. Confirm the deletion. + +**Result:** The cluster template is uninstalled. This action does not affect clusters created with the cluster template. + +An admin with access to the `local` cluster can also remove a cluster deployed via cluster templates through the **Apps > Installed Apps** page. + +## Configuration Options + +Cluster templates are flexible enough that they can be used to configure all of the following options: + +- Node configuration +- Node pools +- Pre-specified cloud credentials +- Enable/configure an authorized cluster endpoint to get kubectl access to the cluster without using Rancher as a proxy +- Install Rancher V2 monitoring +- Kubernetes version +- Assign cluster members +- Infrastructure configuration such as AWS VPC/subnets or vSphere data center +- Cloud provider options +- Pod security options +- Network providers +- Ingress controllers +- Network security configuration +- Network plugins +- Private registry URL and credentials +- Add-ons +- Kubernetes options, including configurations for Kubernetes components such as kube-api, kube-controller, kubelet, and services + +For details on how to configure the template, refer to the cluster template's Helm chart README. \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md new file mode 100644 index 00000000000..2e7f151e5c2 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md @@ -0,0 +1,196 @@ +--- +title: Nodes and Node Pools +--- + + + + + +After you launch a Kubernetes cluster in Rancher, you can manage individual nodes from the cluster's **Node** tab. + +1. Click **☰** in the top left corner. +1. Select **Cluster Management**. +1. Find the cluster whose nodes you want to manage, and click the **Explore** button at the end of the row. +1. Select **Nodes** from the left navigation. + +Depending on the [option used](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) to provision the cluster, there are different node options available. + +:::note + +If you want to manage the _cluster_ and not individual nodes, see [Editing Clusters](../../../pages-for-subheaders/cluster-configuration.md). + +::: + + +## Node Options Available for Each Cluster Creation Option + +The following table lists which node options are available for each type of cluster in Rancher. Click the links in the **Option** column for more detailed information about each feature. + +| Option | [Nodes Hosted by an Infrastructure Provider][1] | [Custom Node][2] | [Hosted Cluster][3] | [Registered EKS Nodes][4] | [All Other Registered Nodes][5] | Description | +| ------------------------------------------------ | ------------------------------------------------ | ---------------- | ------------------- | ------------------- | -------------------| ------------------------------------------------------------------ | +| [Cordon](#cordoning-a-node) | ✓ | ✓ | ✓ | ✓ | ✓ | Marks the node as unschedulable. | +| [Drain](#draining-a-node) | ✓ | ✓ | ✓ | ✓ | ✓ | Marks the node as unschedulable _and_ evicts all pods. | +| [Edit](#managing-and-editing-individual-nodes) | ✓ | ✓ | ✓ | ✓ | ✓ | Enter a custom name, description, label, or taints for a node. | +| [View API](#viewing-a-node-in-the-rancher-api) | ✓ | ✓ | ✓ | ✓ | ✓ | View API data. | +| [Delete](#deleting-a-node) | ✓ | ✓ | | * | * | Deletes defective nodes from the cluster. | +| [Download Keys](#ssh-into-a-node-hosted-by-an-infrastructure-provider) | ✓ | | | | | Download SSH key in order to SSH into the node. | +| [Node Scaling](#scaling-nodes) | ✓ | | | ✓ | | Scale the number of nodes in the node pool up or down. | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +[5]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +\* Delete option accessible via View API + + +### Nodes Hosted by an Infrastructure Provider + +Node pools are available when you provision Rancher-launched Kubernetes clusters on nodes that are [hosted in an infrastructure provider.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) + +Clusters provisioned using [one of the node pool options](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) can be scaled up or down if the node pool is edited. + +A node pool can also automatically maintain the node scale that's set during the initial cluster provisioning if [node auto-replace is enabled.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) This scale determines the number of active nodes that Rancher maintains for the cluster. + +Rancher uses [node templates](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) to replace nodes in the node pool. Each node template uses cloud provider credentials to allow Rancher to set up the node in the infrastructure provider. + +### Nodes Provisioned by Hosted Kubernetes Providers + +Options for managing nodes [hosted by a Kubernetes provider](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md) are somewhat limited in Rancher. Rather than using the Rancher UI to make edits such as scaling the number of nodes up or down, edit the cluster directly. + +### Registered Nodes + +Although you can deploy workloads to a [registered cluster](../../new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md) using Rancher, you cannot manage individual cluster nodes. All management of imported cluster nodes must take place outside of Rancher. + +## Managing and Editing Individual Nodes + +Editing a node lets you: + +* Change its name +* Change its description +* Add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +* Add/Remove [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + +To manage individual nodes, browse to the cluster that you want to manage and then select **Nodes** from the main menu. You can open the options menu for a node by clicking its **â‹®** icon (**..**.). + +## Viewing a Node in the Rancher API + +Select this option to view the node's [API endpoints](../../../pages-for-subheaders/about-the-api.md). + +## Deleting a Node + +Use **Delete** to remove defective nodes from the cloud provider. + +When you the delete a defective node, Rancher can automatically replace it with an identically provisioned node if the node is in a node pool and [node auto-replace is enabled.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) + +:::tip + +If your cluster is hosted by an infrastructure provider, and you want to scale your cluster down instead of deleting a defective node, [scale down](#scaling-nodes) rather than delete. + +::: + +## Scaling Nodes + +For nodes hosted by an infrastructure provider, you can scale the number of nodes in each [node pool](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) by using the scale controls. This option isn't available for other cluster types. + +## SSH into a Node Hosted by an Infrastructure Provider + +For [nodes hosted by an infrastructure provider](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md), you have the option of downloading its SSH key so that you can connect to it remotely from your desktop. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to SSH into a node and click the name of the cluster. +1. On the **Machine Pools** tab, find the node that you want to remote into and click **â‹® > Download SSH Key**. A ZIP file containing files used for SSH will be downloaded. +1. Extract the ZIP file to any location. +1. Open Terminal. Change your location to the extracted ZIP file. +1. Enter the following command: + + ``` + ssh -i id_rsa root@ + ``` + +## Cordoning a Node + +_Cordoning_ a node marks it as unschedulable. This feature is useful for performing short tasks on the node during small maintenance windows, like reboots, upgrades, or decommissions. When you're done, power back on and make the node schedulable again by uncordoning it. + +## Draining a Node + +_Draining_ is the process of first cordoning the node, and then evicting all its pods. This feature is useful for performing node maintenance (like kernel upgrades or hardware maintenance). It prevents new pods from deploying to the node while redistributing existing pods so that users don't experience service interruption. + +- For pods with a replica set, the pod is replaced by a new pod that will be scheduled to a new node. Additionally, if the pod is part of a service, then clients will automatically be redirected to the new pod. + +- For pods with no replica set, you need to bring up a new copy of the pod, and assuming it is not part of a service, redirect clients to it. + +You can drain nodes that are in either a `cordoned` or `active` state. When you drain a node, the node is cordoned, the nodes are evaluated for conditions they must meet to be drained, and then (if it meets the conditions) the node evicts its pods. + +However, you can override the conditions draining when you initiate the drain. You're also given an opportunity to set a grace period and timeout value. + +### Aggressive and Safe Draining Options + +When you configure the upgrade strategy for the cluster, you will be able to enable node draining. If node draining is enabled, you will be able to configure how pods are deleted and rescheduled. + +- **Aggressive Mode** + + In this mode, pods won't get rescheduled to a new node, even if they do not have a controller. Kubernetes expects you to have your own logic that handles the deletion of these pods. + + Kubernetes also expects the implementation to decide what to do with pods using emptyDir. If a pod uses emptyDir to store local data, you might not be able to safely delete it, since the data in the emptyDir will be deleted once the pod is removed from the node. Choosing aggressive mode will delete these pods. + +- **Safe Mode** + + If a node has standalone pods or ephemeral data it will be cordoned but not drained. +### Grace Period + +The timeout given to each pod for cleaning things up, so they will have chance to exit gracefully. For example, when pods might need to finish any outstanding requests, roll back transactions or save state to some external storage. If negative, the default value specified in the pod will be used. + +### Timeout + +The amount of time drain should continue to wait before giving up. + +:::note Kubernetes Known Issue: + +The [timeout setting](https://github.com/kubernetes/kubernetes/pull/64378) was not enforced while draining a node before Kubernetes 1.12. + +::: + +### Drained and Cordoned State + +If there's any error related to user input, the node enters a `cordoned` state because the drain failed. You can either correct the input and attempt to drain the node again, or you can abort by uncordoning the node. + +If the drain continues without error, the node enters a `draining` state. You'll have the option to stop the drain when the node is in this state, which will stop the drain process and change the node's state to `cordoned`. + +Once drain successfully completes, the node will be in a state of `drained`. You can then power off or delete the node. + +**Want to know more about cordon and drain?** See the [Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/). + +## Labeling a Node to be Ignored by Rancher + +Some solutions, such as F5's BIG-IP integration, may require creating a node that is never registered to a cluster. + +Since the node will never finish registering, it will always be shown as unhealthy in the Rancher UI. + +In that case, you may want to label the node to be ignored by Rancher so that Rancher only shows nodes as unhealthy when they are actually failing. + +You can label nodes to be ignored by using a setting in the Rancher UI, or by using `kubectl`. + +:::note + +There is an [open issue](https://github.com/rancher/rancher/issues/24172) in which nodes labeled to be ignored can get stuck in an updating state. + +::: + + +### Labeling Nodes to be Ignored with kubectl + +To add a node that will be ignored by Rancher, use `kubectl` to create a node that has the following label: + +``` +cattle.rancher.io/node-status: ignore +``` + +**Result:** If you add the node to a cluster, Rancher will not attempt to sync with this node. The node can still be part of the cluster and can be listed with `kubectl`. + +If the label is added before the node is added to the cluster, the node will not be shown in the Rancher UI. + +If the label is added after the node is added to a Rancher cluster, the node will not be removed from the UI. + +If you delete the node from the Rancher server using the Rancher UI or API, the node will not be removed from the cluster if the `nodeName` is listed in the Rancher settings in the Rancher API under `v3/settings/ignore-node-name`. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md new file mode 100644 index 00000000000..d74b70822d3 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md @@ -0,0 +1,201 @@ +--- +title: Projects and Kubernetes Namespaces with Rancher +description: Rancher Projects ease the administrative burden of your cluster and support multi-tenancy. Learn to create projects and divide projects into Kubernetes namespaces +--- + + + + + +A namespace is a Kubernetes concept that allows a virtual cluster within a cluster, which is useful for dividing the cluster into separate "virtual clusters" that each have their own access control and resource quotas. + +A project is a group of namespaces, and it is a concept introduced by Rancher. Projects allow you to manage multiple namespaces as a group and perform Kubernetes operations in them. You can use projects to support multi-tenancy, so that a team can access a project within a cluster without having access to other projects in the same cluster. + +:::note + +Projects are de-emphasized on the UI because it is not required to create Kubernetes resources within a project scope. However, resources such as [Secrets](../../new-user-guides/kubernetes-resources-setup/secrets.md#creating-secrets-in-projects) can still be created in a project scope if the legacy feature flag is enabled. + +::: + +This section describes how projects and namespaces work with Rancher. + +## About Namespaces + +A namespace is a concept introduced by Kubernetes. According to the [official Kubernetes documentation on namespaces,](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) + +:::note + +Kubernetes supports multiple virtual clusters backed by the same physical cluster. These virtual clusters are called namespaces. [...] Namespaces are intended for use in environments with many users spread across multiple teams, or projects. For clusters with a few to tens of users, you should not need to create or think about namespaces at all. + +::: + +Namespaces provide the following functionality: + +- **Providing a scope for names:** Names of resources need to be unique within a namespace, but not across namespaces. Namespaces can not be nested inside one another and each Kubernetes resource can only be in one namespace. +- **Resource quotas:** Namespaces provide a way to divide cluster resources between multiple users. + +You can assign resources at the project level so that each namespace in the project can use them. You can also bypass this inheritance by assigning resources explicitly to a namespace. + +You can assign the following resources directly to namespaces: + +- [Workloads](../../../pages-for-subheaders/workloads-and-pods.md) +- [Load Balancers/Ingress](../../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [Service Discovery Records](../../new-user-guides/kubernetes-resources-setup/create-services.md) +- [Persistent Volume Claims](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [Certificates](../../new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMaps](../../new-user-guides/kubernetes-resources-setup/configmaps.md) +- [Registries](../../new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [Secrets](../../new-user-guides/kubernetes-resources-setup/secrets.md) + +To manage permissions in a vanilla Kubernetes cluster, cluster admins configure role-based access policies for each namespace. With Rancher, user permissions are assigned on the project level instead, and permissions are automatically inherited by any namespace owned by the particular project. + +For more information on creating and moving namespaces, see [Namespaces](../../new-user-guides/manage-namespaces.md). + +### Role-based access control issues with namespaces and kubectl + +Because projects are a concept introduced by Rancher, kubectl does not have the capability to restrict the creation of namespaces to a project the creator has access to. + +This means that when standard users with project-scoped permissions create a namespaces with `kubectl`, it may be unusable because `kubectl` doesn't require the new namespace to be scoped within a certain project. + +If your permissions are restricted to the project level, it is better to [create a namespace through Rancher](../../new-user-guides/manage-namespaces.md) to ensure that you will have permission to access the namespace. + +If a standard user is a project owner, the user will be able to create namespaces within that project. The Rancher UI will prevent that user from creating namespaces outside the scope of the projects they have access to. + +## About Projects + +In terms of hierarchy: + +- Clusters contain projects +- Projects contain namespaces + +You can use projects to support multi-tenancy, so that a team can access a project within a cluster without having access to other projects in the same cluster. + +In the base version of Kubernetes, features like role-based access rights or cluster resources are assigned to individual namespaces. A project allows you to save time by giving an individual or a team access to multiple namespaces simultaneously. + +You can use projects to perform actions such as: + +- Assign users to a group of namespaces (i.e., [project membership](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)). +- Assign users specific roles in a project. A role can be owner, member, read-only, or [custom](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md). +- Assign resources to the project. +- Assign Pod Security Policies. + +When you create a cluster, two projects are automatically created within it: + +- [Default Project](#the-clusters-default-project) +- [System Project](#the-system-project) + +### The Cluster's Default Project + +When you provision a cluster with Rancher, it automatically creates a `default` project for the cluster. This is a project you can use to get started with your cluster, but you can always delete it and replace it with projects that have more descriptive names. + +If you don't have a need for more than the default namespace, you also do not need more than the **Default** project in Rancher. + +If you require another level of organization beyond the **Default** project, you can create more projects in Rancher to isolate namespaces, applications and resources. + +### The System Project + +When troubleshooting, you can view the `system` project to check if important namespaces in the Kubernetes system are working properly. This easily accessible project saves you from troubleshooting individual system namespace containers. + +To open it, open the cluster view and click **Cluster > Projects/Namespaces**. This view shows all of the namespaces in the `system` project. + +The `system` project: + +- Is automatically created when you provision a cluster. +- Lists all namespaces that exist in `v3/settings/system-namespaces`, if they exist. +- Allows you to add more namespaces or move its namespaces to other projects. +- Cannot be deleted because it's required for cluster operations. + +:::note + +In RKE clusters where the project network isolation option is enabled, the `system` project overrides the project network isolation option so that it can communicate with other projects, collect logs, and check health. + +::: + +## Project Authorization + +Standard users are only authorized for project access in two situations: + +- An administrator, cluster owner or cluster member explicitly adds the standard user to the project's **Members** tab. +- Standard users can access projects that they create themselves. + +## Pod Security Policies + +Rancher extends Kubernetes to allow the application of [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) at the project level in addition to the cluster level. However, as a best practice, we recommend applying Pod Security Policies at the cluster level. + +## Creating Projects + +This section describes how to create a new project with a name and with optional pod security policy, members, and resource quotas. + +1. [Name a new project.](#1-name-a-new-project) +2. [Optional: Select a pod security policy.](#2-optional-select-a-pod-security-policy) +3. [Recommended: Add project members.](#3-recommended-add-project-members) +4. [Optional: Add resource quotas.](#4-optional-add-resource-quotas) + +### 1. Name a New Project + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to project in and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Click **Create Project**. +1. Enter a **Project Name**. + +### 2. Optional: Select a Pod Security Policy + +This option is only available if you've already created a Pod Security Policy. For instruction, see [Creating Pod Security Policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md). + +Assigning a PSP to a project will: + +- Override the cluster's default PSP. +- Apply the PSP to the project. +- Apply the PSP to any namespaces you add to the project later. + +### 3. Recommended: Add Project Members + +Use the **Members** section to provide other users with project access and roles. + +By default, your user is added as the project `Owner`. + +:::note Notes on Permissions: + +- Users assigned the `Owner` or `Member` role for a project automatically inherit the `namespace creation` role. However, this role is a [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole), meaning its scope extends to all projects in the cluster. Therefore, users explicitly assigned the `Owner` or `Member` role for a project can create namespaces in other projects they're assigned to, even with only the `Read Only` role assigned. + +- By default, the Rancher role of `project-member` inherits from the `Kubernetes-edit` role, and the `project-owner` role inherits from the `Kubernetes-admin` role. As such, both `project-member` and `project-owner` roles will allow for namespace management, including the ability to create and delete namespaces. + +- Choose `Custom` to create a custom role on the fly: [Custom Project Roles](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#custom-project-roles). + +::: + +To add members: + +1. In the **Members** tab, click **Add**. +1. From the **Select Member** field, search for a user or group that you want to assign project access. Note: You can only search for groups if external authentication is enabled. +1. In the **Project Permissions** section, choose a role. For more information, refer to the [documentation on project roles.](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + +### 4. Optional: Add Resource Quotas + +Resource quotas limit the resources that a project (and its namespaces) can consume. For more information, see [Resource Quotas](projects-and-namespaces.md). + +To add a resource quota, + +1. In the **Resource Quotas** tab, click **Add Resource**. +1. Select a **Resource Type**. For more information, see [Resource Quotas.](projects-and-namespaces.md). +1. Enter values for the **Project Limit** and the **Namespace Default Limit**. +1. **Optional:** Specify **Container Default Resource Limit**, which will be applied to every container started in the project. The parameter is recommended if you have CPU or Memory limits set by the Resource Quota. It can be overridden on per an individual namespace or a container level. For more information, see [Container Default Resource Limit](../../../pages-for-subheaders/manage-project-resource-quotas.md) +1. Click **Create**. + +**Result:** Your project is created. You can view it from the cluster's **Projects/Namespaces** view. + +| Field | Description | +| ----------------------- | -------------------------------------------------------------------------------------------------------- | +| Project Limit | The overall resource limit for the project. | +| Namespace Default Limit | The default resource limit available for each namespace. This limit is propagated to each namespace in the project when created. The combined limit of all project namespaces shouldn't exceed the project limit. | + +## Deleting Projects + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster the project is attached to, and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the project you want to delete, and click **â‹®**. +1. Select **Delete**. + +When you delete a project, any namespaces that were formerly associated with the project will remain on the cluster. You can find these namespaces in the Rancher UI, in the **Not in a Project** tab of the **Projects/Namespaces** page. You can reassign these namespaces to a project by [moving](../manage-namespaces.md#moving-namespaces-to-another-project) them. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md new file mode 100644 index 00000000000..9898df21025 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md @@ -0,0 +1,79 @@ +--- +title: NFS Storage +--- + + + + + +Before you can use the NFS storage volume plug-in with Rancher deployments, you need to provision an NFS server. + +:::note + +- If you already have an NFS share, you don't need to provision a new NFS server to use the NFS volume plugin within Rancher. Instead, skip the rest of this procedure and complete [adding storage](../../../../pages-for-subheaders/create-kubernetes-persistent-storage.md). + +- This procedure demonstrates how to set up an NFS server using Ubuntu, although you should be able to use these instructions for other Linux distros (e.g. Debian, RHEL, Arch Linux, etc.). For official instruction on how to create an NFS server using another Linux distro, consult the distro's documentation. + +::: + +:::note Recommended: + +To simplify the process of managing firewall rules, use NFSv4. + +::: + +1. Using a remote Terminal connection, log into the Ubuntu server that you intend to use for NFS storage. + +1. Enter the following command: + + ``` + sudo apt-get install nfs-kernel-server + ``` + +1. Enter the command below, which sets the directory used for storage, along with user access rights. Modify the command if you'd like to keep storage at a different directory. + + ``` + mkdir -p /nfs && chown nobody:nogroup /nfs + ``` + - The `-p /nfs` parameter creates a directory named `nfs` at root. + - The `chown nobody:nogroup /nfs` parameter allows all access to the storage directory. + +1. Create an NFS exports table. This table sets the directory paths on your NFS server that are exposed to the nodes that will use the server for storage. + + 1. Open `/etc/exports` using your text editor of choice. + 1. Add the path of the `/nfs` folder that you created in step 3, along with the IP addresses of your cluster nodes. Add an entry for each IP address in your cluster. Follow each address and its accompanying parameters with a single space that is a delimiter. + + ``` + /nfs (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) + ``` + + :::tip + + You can replace the IP addresses with a subnet. For example: `10.212.50.12/24` + + ::: + + 1. Update the NFS table by entering the following command: + + ``` + exportfs -ra + ``` + +1. Open the ports used by NFS. + + 1. To find out what ports NFS is using, enter the following command: + + ``` + rpcinfo -p | grep nfs + ``` + 2. [Open the ports](https://help.ubuntu.com/lts/serverguide/firewall.html.en) that the previous command outputs. For example, the following command opens port 2049: + + ``` + sudo ufw allow 2049 + ``` + +**Result:** Your NFS server is configured to be used for storage with your Rancher nodes. + +## What's Next? + +Within Rancher, add the NFS server as a storage volume and/or storage class. After adding the server, you can use it for storage for your deployments. diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md new file mode 100644 index 00000000000..059bb688673 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md @@ -0,0 +1,19 @@ +--- +title: Creating Persistent Storage in Amazon's EBS +--- + + + + + +This section describes how to set up Amazon's Elastic Block Store in EC2. + +1. From the EC2 console, go to the **ELASTIC BLOCK STORE** section in the left panel and click **Volumes**. +1. Click **Create Volume**. +1. Optional: Configure the size of the volume or other options. The volume should be created in the same availability zone as the instance it will be attached to. +1. Click **Create Volume**. +1. Click **Close**. + +**Result:** Persistent storage has been created. + +For details on how to set up the newly created storage in Rancher, refer to the section on [setting up existing storage.](../../../new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md new file mode 100644 index 00000000000..b13cee6512d --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md @@ -0,0 +1,76 @@ +--- +title: vSphere Storage +--- + + + + + +To provide stateful workloads with vSphere storage, we recommend creating a vSphereVolume StorageClass. This practice dynamically provisions vSphere storage when workloads request volumes through a PersistentVolumeClaim. + +In order to dynamically provision storage in vSphere, the vSphere provider must be enabled. See the following pages for more: [Out-of-tree vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md) and [in-tree vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md). + + +### Prerequisites + +In order to provision vSphere volumes in a cluster created with the [Rancher Kubernetes Engine (RKE)](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md), the [vSphere cloud provider](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere) must be explicitly enabled in the [cluster options](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md). + +### Creating a StorageClass + +:::tip + +The following steps can also be performed using the `kubectl` command line tool. See [Kubernetes documentation on persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) for details. + +::: + +1. Click **☰ > Cluster Management**. +1. Choose the cluster you want to provide vSphere storage to and click **Exlpore**. +1. In the left navigation bar, select **Storage > StorageClasses**. +1. Click **Create**. +3. Enter a **Name** for the StorageClass. +4. Under **Provisioner**, select **VMWare vSphere Volume**. + + ![](/img/vsphere-storage-class.png) + +5. Optionally, specify additional properties for this storage class under **Parameters**. Refer to the [vSphere storage documentation](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/blob/master/documentation/storageclass.md) for details. +5. Click **Create**. + +### Creating a Workload with a vSphere Volume + +1. In the left navigation bar, click **Workload**. +1. Click **Create**. +1. Click **StatefulSet**. +1. In the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Enter a persistent volume name. +1. In the Storage Class field, select the vSphere StorageClass that you created. +6. Enter the required **Capacity** for the volume. Then click **Define**. +7. Assign a path in the **Mount Point** field. This is the full path where the volume will be mounted in the container file system, e.g. `/persistent`. +8. Click **Create**. + +### Verifying Persistence of the Volume + +1. In the left navigation bar, click **Workload > Pods**. +1. Go to the workload you just created and click **â‹® > Execute Shell**. +2. Note the directory at root where the volume has been mounted to (in this case `/persistent`). +3. Create a file in the volume by executing the command `touch //data.txt`. +4. Close the shell window. +5. Click on the name of the workload to reveal detail information. +7. Click **â‹® > Delete**. +8. Observe that the pod is deleted. Then a new pod is scheduled to replace it so that the workload maintains its configured scale of a single stateful pod. +9. Once the replacement pod is running, click **Execute Shell**. +10. Inspect the contents of the directory where the volume is mounted by entering `ls -l /`. Note that the file you created earlier is still present. + + ![workload-persistent-data](/img/workload-persistent-data.png) + +### Why to Use StatefulSets Instead of Deployments + +You should always use [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) for workloads consuming vSphere storage, as this resource type is designed to address a VMDK block storage caveat. + +Since vSphere volumes are backed by VMDK block storage, they only support an [access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) of `ReadWriteOnce`. This setting restricts the volume so that it can only be mounted to a single pod at a time, unless all pods consuming that volume are co-located on the same node. This behavior makes a deployment resource unusable for scaling beyond a single replica if it consumes vSphere volumes. + +Even using a deployment resource with just a single replica may result in a deadlock situation while updating the deployment. If the updated pod is scheduled to a node different from where the existing pod lives, it will fail to start because the VMDK is still attached to the other node. + +### Related Links + +- [vSphere Storage for Kubernetes](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/tree/master/documentation) +- [Kubernetes Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md new file mode 100644 index 00000000000..7cb1f6bad37 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -0,0 +1,82 @@ +--- +title: Certificate Rotation +--- + + + + + +:::caution + +Rotating Kubernetes certificates may result in your cluster being temporarily unavailable as components are restarted. For production environments, it's recommended to perform this action during a maintenance window. + +::: + +By default, Kubernetes clusters require certificates and Rancher launched Kubernetes clusters automatically generate certificates for the Kubernetes components. Rotating these certificates is important before the certificates expire as well as if a certificate is compromised. After the certificates are rotated, the Kubernetes components are automatically restarted. + +Certificates can be rotated for the following services: + + + + +- etcd +- kubelet (node certificate) +- kubelet (serving certificate, if [enabled](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)) +- kube-apiserver +- kube-proxy +- kube-scheduler +- kube-controller-manager + + + + +- admin +- api-server +- controller-manager +- scheduler +- rke2-controller +- rke2-server +- cloud-controller +- etcd +- auth-proxy +- kubelet +- kube-proxy + + + + +:::note + +For users who didn't rotate their webhook certificates, and they have expired after one year, please see this [page](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md) for help. + +::: + +### Certificate Rotation + +Rancher launched Kubernetes clusters have the ability to rotate the auto-generated certificates through the UI. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to rotate certificates for amd click **â‹® > Rotate Certificates**. +1. Select which certificates that you want to rotate. + + * Rotate all Service certificates (keep the same CA) + * Rotate an individual service and choose one of the services from the drop-down menu + +1. Click **Save**. + +**Results:** The selected certificates will be rotated and the related services will be restarted to start using the new certificate. + +### Additional Notes + + + + +Even though the RKE CLI can use custom certificates for the Kubernetes cluster components, Rancher currently doesn't allow the ability to upload these in Rancher launched Kubernetes clusters. + + + + +In RKE2, both etcd and control plane nodes are treated as the same `server` concept. As such, when rotating certificates of services specific to either of these components will result in certificates being rotated on both. The certificates will only change for the specified service, but you will see nodes for both components go into an updating state. You may also see worker only nodes go into an updating state. This is to restart the workers after a certificate change to ensure they get the latest client certs. + + + diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md new file mode 100644 index 00000000000..5d56f2c60aa --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -0,0 +1,52 @@ +--- +title: Encryption Key Rotation +--- + + + + + +### RKE1 Encryption Key Rotation + +1. Enable encryption key rotation with either of the following two options: + + - Select the `Enabled` radio button in the Rancher UI under **Cluster Options > Advanced Options > Secrets Encryption**: + + ![Enable Encryption Key Rotation](/img/rke1-enable-secrets-encryption.png) + + - OR, apply the following YAML: + + ```yaml + rancher_kubernetes_engine_config: + services: + kube_api: + secrets_encryption_config: + enabled: true + ``` + +2. Rotate keys in the Rancher UI: + + 2.1. Click **☰ > Cluster Management**. + + 2.2. Select **â‹® > Rotate Encryption Keys** on the far right of the screen next to your chosen cluster: + + ![Encryption Key Rotation](/img/rke1-encryption-key.png) + + + +### RKE2 Encryption Key Rotation + +_**New in v2.6.7**_ + +>**Important:** Encryption key rotation is enabled by default and cannot be disabled. + +To rotate keys in the Rancher UI: + +1. Click **☰ > Cluster Management**. + +1. Select **â‹® > Rotate Encryption Keys** on the far right of the screen next to your chosen cluster: + + ![Encryption Key Rotation](/img/rke2-encryption-key.png) + + +>**Note:** For more information on RKE2 secrets encryption config, please see the [RKE2 docs](https://docs.rke2.io/security/secrets_encryption). \ No newline at end of file diff --git a/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-namespaces.md b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-namespaces.md new file mode 100644 index 00000000000..0419be358a6 --- /dev/null +++ b/versioned_docs/version-2.8/how-to-guides/new-user-guides/manage-namespaces.md @@ -0,0 +1,78 @@ +--- +title: Namespaces +--- + + + + + +Within Rancher, you can further divide projects into different [namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/), which are virtual clusters within a project backed by a physical cluster. Should you require another level of organization beyond projects and the `default` namespace, you can use multiple namespaces to isolate applications and resources. + +Although you assign resources at the project level so that each namespace in the project can use them, you can override this inheritance by assigning resources explicitly to a namespace. + +Resources that you can assign directly to namespaces include: + +- [Workloads](../../pages-for-subheaders/workloads-and-pods.md) +- [Load Balancers/Ingress](../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [Service Discovery Records](kubernetes-resources-setup/create-services.md) +- [Persistent Volume Claims](../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [Certificates](kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMaps](kubernetes-resources-setup/configmaps.md) +- [Registries](kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [Secrets](kubernetes-resources-setup/secrets.md) + +To manage permissions in a vanilla Kubernetes cluster, cluster admins configure role-based access policies for each namespace. With Rancher, user permissions are assigned on the project level instead, and permissions are automatically inherited by any namespace owned by the particular project. + +:::note + +If you create a namespace with `kubectl`, it may be unusable because `kubectl` doesn't require your new namespace to be scoped within a project that you have access to. If your permissions are restricted to the project level, it is better to [create a namespace through Rancher](manage-namespaces.md) to ensure that you will have permission to access the namespace. + +::: + +### Creating Namespaces + +Create a new namespace to isolate apps and resources in a project. + +:::tip + +When working with project resources that you can assign to a namespace (i.e., [workloads](kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md), [certificates](kubernetes-resources-setup/encrypt-http-communication.md), [ConfigMaps](kubernetes-resources-setup/configmaps.md), etc.) you can create a namespace on the fly. + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to create a namespace and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the project where you want to add a namespace and click **Create Namespace**. Alternately, go to **Not in a Project** to create a namespace not associated with a project. + +1. **Optional:** If your project has [Resource Quotas](../../pages-for-subheaders/manage-project-resource-quotas.md) in effect, you can override the default resource **Limits** (which places a cap on the resources that the namespace can consume). + +1. Enter a **Name** and then click **Create**. + +**Result:** Your namespace is added to the project. You can begin assigning cluster resources to the namespace. + +### Moving Namespaces to Another Project + +Cluster admins and members may occasionally need to move a namespace to another project, such as when you want a different team to start using the application. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to move a namespace and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the namespace you want to move and click **â‹® > Move**. + +1. Select the namespace(s) that you want to move to a different project. Then click **Move**. You can move multiple namespaces at one. + + :::note Notes: + + - Don't move the namespaces in the `System` project. Moving these namespaces can adversely affect cluster networking. + - You cannot move a namespace into a project that already has a [resource quota](../../pages-for-subheaders/manage-project-resource-quotas.md)configured. + - If you move a namespace from a project that has a quota set to a project with no quota set, the quota is removed from the namespace. + +1. Choose a new project for the new namespace and then click **Move**. Alternatively, you can remove the namespace from all projects by selecting **None**. + +**Result:** Your namespace is moved to a different project (or is unattached from all projects). If any project resources are attached to the namespace, the namespace releases them and then attached resources from the new project. + +### Editing Namespace Resource Quotas + +You can always override the namespace default limit to provide a specific namespace with access to more (or less) project resources. + +For more information, see how to [edit namespace resource quotas](../advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md). \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/configuration-reference.md b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/configuration-reference.md new file mode 100644 index 00000000000..fa9012e6ed8 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/configuration-reference.md @@ -0,0 +1,105 @@ +--- +title: Configuration +--- + +This configuration reference is intended to help you manage the custom resources created by the `rancher-cis-benchmark` application. These resources are used for performing CIS scans on a cluster, skipping tests, setting the test profile that will be used during a scan, and other customization. + +To configure the custom resources, go to the **Cluster Dashboard** To configure the CIS scans, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to configure CIS scans and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark**. + +### Scans + +A scan is created to trigger a CIS scan on the cluster based on the defined profile. A report is created after the scan is completed. + +When configuring a scan, you need to define the name of the scan profile that will be used with the `scanProfileName` directive. + +An example ClusterScan custom resource is below: + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScan +metadata: + name: rke-cis +spec: + scanProfileName: rke-profile-hardened +``` + +### Profiles + +A profile contains the configuration for the CIS scan, which includes the benchmark version to use and any specific tests to skip in that benchmark. + +:::caution + +By default, a few ClusterScanProfiles are installed as part of the `rancher-cis-benchmark` chart. If a user edits these default benchmarks or profiles, the next chart update will reset them back. So it is advisable for users to not edit the default ClusterScanProfiles. + +::: + +Users can clone the ClusterScanProfiles to create custom profiles. + +Skipped tests are listed under the `skipTests` directive. + +When you create a new profile, you will also need to give it a name. + +An example `ClusterScanProfile` is below: + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" +spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" +``` + +### Benchmark Versions + +A benchmark version is the name of benchmark to run using `kube-bench`, as well as the valid configuration parameters for that benchmark. + +A `ClusterScanBenchmark` defines the CIS `BenchmarkVersion` name and test configurations. The `BenchmarkVersion` name is a parameter provided to the `kube-bench` tool. + +By default, a few `BenchmarkVersion` names and test configurations are packaged as part of the CIS scan application. When this feature is enabled, these default BenchmarkVersions will be automatically installed and available for users to create a ClusterScanProfile. + +:::caution + +If the default BenchmarkVersions are edited, the next chart update will reset them back. Therefore we don't recommend editing the default ClusterScanBenchmarks. + +::: + +A ClusterScanBenchmark consists of the fields: + +- `ClusterProvider`: This is the cluster provider name for which this benchmark is applicable. For example: RKE, EKS, GKE, etc. Leave it empty if this benchmark can be run on any cluster type. +- `MinKubernetesVersion`: Specifies the cluster's minimum kubernetes version necessary to run this benchmark. Leave it empty if there is no dependency on a particular Kubernetes version. +- `MaxKubernetesVersion`: Specifies the cluster's maximum Kubernetes version necessary to run this benchmark. Leave it empty if there is no dependency on a particular k8s version. + +An example `ClusterScanBenchmark` is below: + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + creationTimestamp: "2020-08-28T18:18:07Z" + generation: 1 + labels: + app.kubernetes.io/managed-by: Helm + name: cis-1.5 + resourceVersion: "203878" + selfLink: /apis/cis.cattle.io/v1/clusterscanbenchmarks/cis-1.5 + uid: 309e543e-9102-4091-be91-08d7af7fb7a7 +spec: + clusterProvider: "" + minKubernetesVersion: 1.15.0 +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/custom-benchmark.md b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/custom-benchmark.md new file mode 100644 index 00000000000..8ba63bbe8f7 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/custom-benchmark.md @@ -0,0 +1,78 @@ +--- +title: Creating a Custom Benchmark Version for Running a Cluster Scan +--- + +Each Benchmark Version defines a set of test configuration files that define the CIS tests to be run by the kube-bench tool. +The `rancher-cis-benchmark` application installs a few default Benchmark Versions which are listed under CIS Benchmark application menu. + +But there could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. + +It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. + +When a cluster scan is run, you need to select a Profile which points to a specific Benchmark Version. + +Follow all the steps below to add a custom Benchmark Version and run a scan using it. + +### 1. Prepare the Custom Benchmark Version ConfigMap + +To create a custom benchmark version, first you need to create a ConfigMap containing the benchmark version's config files and upload it to your Kubernetes cluster where you want to run the scan. + +To prepare a custom benchmark version ConfigMap, suppose we want to add a custom Benchmark Version named `foo`. + +1. Create a directory named `foo` and inside this directory, place all the config YAML files that the kube-bench tool looks for. For example, here are the config YAML files for a Generic CIS 1.5 Benchmark Version https://github.com/aquasecurity/kube-bench/tree/master/cfg/cis-1.5 +1. Place the complete `config.yaml` file, which includes all the components that should be tested. +1. Add the Benchmark version name to the `target_mapping` section of the `config.yaml`: + + ```yaml + target_mapping: + "foo": + - "master" + - "node" + - "controlplane" + - "etcd" + - "policies" + ``` +1. Upload this directory to your Kubernetes Cluster by creating a ConfigMap: + + ```yaml + kubectl create configmap -n foo --from-file= + ``` + +### 2. Add a Custom Benchmark Version to a Cluster + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark > Benchmark Version**. +1. Click **Create**. +1. Enter the **Name** and a description for your custom benchmark version. +1. Choose the cluster provider that your benchmark version applies to. +1. Choose the ConfigMap you have uploaded from the dropdown. +1. Add the minimum and maximum Kubernetes version limits applicable, if any. +1. Click **Create**. + +### 3. Create a New Profile for the Custom Benchmark Version + +To run a scan using your custom benchmark version, you need to add a new Profile pointing to this benchmark version. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark > Profile**. +1. Click **Create**. +1. Provide a **Name** and description. In this example, we name it `foo-profile`. +1. Choose the Benchmark Version from the dropdown. +1. Click **Create**. + +### 4. Run a Scan Using the Custom Benchmark Version + +Once the Profile pointing to your custom benchmark version `foo` has been created, you can create a new Scan to run the custom test configs in the Benchmark Version. + +To run a scan, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark > Scan**. +1. Click **Create**. +1. Choose the new cluster scan profile. +1. Click **Create**. + +**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md new file mode 100644 index 00000000000..8a88240d963 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md @@ -0,0 +1,48 @@ +--- +title: Roles-based Access Control +--- + +This section describes the permissions required to use the rancher-cis-benchmark App. + +The rancher-cis-benchmark is a cluster-admin only feature by default. + +However, the `rancher-cis-benchmark` chart installs these two default `ClusterRoles`: + +- cis-admin +- cis-view + +In Rancher, only cluster owners and global administrators have `cis-admin` access by default. + +Note: If you were using the `cis-edit` role added in Rancher v2.5 setup, it has now been removed since +Rancher v2.5.2 because it essentially is same as `cis-admin`. If you happen to create any clusterrolebindings +for `cis-edit`, please update them to use `cis-admin` ClusterRole instead. + +## Cluster-Admin Access + +Rancher CIS Scans is a cluster-admin only feature by default. +This means only the Rancher global admins, and the cluster’s cluster-owner can: + +- Install/Uninstall the rancher-cis-benchmark App +- See the navigation links for CIS Benchmark CRDs - ClusterScanBenchmarks, ClusterScanProfiles, ClusterScans +- List the default ClusterScanBenchmarks and ClusterScanProfiles +- Create/Edit/Delete new ClusterScanProfiles +- Create/Edit/Delete a new ClusterScan to run the CIS scan on the cluster +- View and Download the ClusterScanReport created after the ClusterScan is complete + + +## Summary of Default Permissions for Kubernetes Default Roles + +The rancher-cis-benchmark creates three `ClusterRoles` and adds the CIS Benchmark CRD access to the following default K8s `ClusterRoles`: + +| ClusterRole created by chart | Default K8s ClusterRole | Permissions given with Role +| ------------------------------| ---------------------------| ---------------------------| +| `cis-admin` | `admin`| Ability to CRUD clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR +| `cis-view` | `view `| Ability to List(R) clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR + + +By default only cluster-owner role will have ability to manage and use `rancher-cis-benchmark` feature. + +The other Rancher roles (cluster-member, project-owner, project-member) do not have any default permissions to manage and use rancher-cis-benchmark resources. + +But if a cluster-owner wants to delegate access to other users, they can do so by creating ClusterRoleBindings between these users and the above CIS ClusterRoles manually. +There is no automatic role aggregation supported for the `rancher-cis-benchmark` ClusterRoles. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md new file mode 100644 index 00000000000..b11887c76c1 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md @@ -0,0 +1,53 @@ +--- +title: Skipped and Not Applicable Tests +--- + +This section lists the tests that are skipped in the permissive test profile for RKE. + +> All the tests that are skipped and not applicable on this page will be counted as Not Applicable in the v2.5 generated report. The skipped test count will only mention the user-defined skipped tests. This allows user-skipped tests to be distinguished from the tests that are skipped by default in the RKE permissive test profile. + +## CIS Benchmark v1.5 + +### CIS Benchmark v1.5 Skipped Tests + +| Number | Description | Reason for Skipping | +| ---------- | ------------- | --------- | +| 1.1.12 | Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) | A system service account is required for etcd data directory ownership. Refer to Rancher's hardening guide for more details on how to configure this ownership. | +| 1.2.6 | Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | +| 1.2.16 | Ensure that the admission control plugin PodSecurityPolicy is set (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 1.2.33 | Ensure that the --encryption-provider-config argument is set as appropriate (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | +| 1.2.34 | Ensure that encryption providers are appropriately configured (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | +| 4.2.6 | Ensure that the --protect-kernel-defaults argument is set to true (Automated) | System level configurations are required before provisioning the cluster in order for this argument to be set to true. | +| 4.2.10 | Ensure that the--tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | +| 5.1.5 | Ensure that default service accounts are not actively used. (Automated) | Kubernetes provides default service accounts to be used. | +| 5.2.2 | Minimize the admission of containers wishing to share the host process ID namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.2.3 | Minimize the admission of containers wishing to share the host IPC namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.2.4 | Minimize the admission of containers wishing to share the host network namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.2.5 | Minimize the admission of containers with allowPrivilegeEscalation (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.3.2 | Ensure that all Namespaces have Network Policies defined (Automated) | Enabling Network Policies can prevent certain applications from communicating with each other. | +| 5.6.4 | The default namespace should not be used (Automated) | Kubernetes provides a default namespace. | + +### CIS Benchmark v1.5 Not Applicable Tests + +| Number | Description | Reason for being not applicable | +| ---------- | ------------- | --------- | +| 1.1.1 | Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | +| 1.1.2 | Ensure that the API server pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | +| 1.1.3 | Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.1.4 | Ensure that the controller manager pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.1.5 | Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.6 | Ensure that the scheduler pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.7 | Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | +| 1.1.8 | Ensure that the etcd pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | +| 1.1.13 | Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | +| 1.1.14 | Ensure that the admin.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | +| 1.1.15 | Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.16 | Ensure that the scheduler.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.17 | Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.1.18 | Ensure that the controller-manager.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.3.6 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | +| 4.1.1 | Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | +| 4.1.2 | Ensure that the kubelet service file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | +| 4.1.9 | Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | +| 4.1.10 | Ensure that the kubelet configuration file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | +| 4.2.12 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md new file mode 100644 index 00000000000..42937ea2262 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md @@ -0,0 +1,96 @@ +--- +title: Prerequisites +--- + +### 1. Setting Up License Manager and Purchasing Support + +First, complete the [first step](https://docs.aws.amazon.com/license-manager/latest/userguide/getting-started.html) of the license manager one-time setup. +Next, go to the AWS Marketplace. Locate the "Rancher Premium Support Billing Container Starter Pack". Purchase at least one entitlement. + +If you have installed Rancher using the "Rancher Setup" AWS Marketplace offering, skip to [Step 4](#4-create-an-oidc-provider). + +> **Note:** Each entitlement grants access to support for a certain amount of nodes. You can purchase more licenses as necessary later on. + +### 2. Create an EKS Cluster +Follow the [Rancher docs](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md) to create an EKS cluster. When you get to the [final step to install Rancher](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-install-the-rancher-helm-chart), **stop and return to this page**. This cluster will need to meet the following requirements: + +- EKS version 1.22. +- Each node in the cluster has access to the registry containing Rancher and its related images. +- Each node in the cluster has access to the ECR repo storing the CSP Adapter. +- Each node in the cluster has access to the license manager service. +- Each node in the cluster has access to global endpoints for the STS service. + +### 3. Install Rancher + +In addition to the options specified to install Rancher in the [Rancher docs](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-install-the-rancher-helm-chart), you will also need to enable extra metrics. +This can be done through the Helm CLI through the following options: + +```bash +--set extraEnv\[0\].name="CATTLE_PROMETHEUS_METRICS" --set-string extraEnv\[0\].value=true +``` + +You can also use a values.yaml like the below: + +```yaml +extraEnv: + - name: "CATTLE_PROMETHEUS_METRICS" + value: "true" +``` + +You will also need to install Rancher version 2.6.7 or higher. + +### 4. Create an OIDC Provider + +Follow the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) to create an OIDC provider for the cluster specified in the previous section. + +### 5. Create an IAM Role + +An IAM role is required for the CSP adapter to check-in/check-out entitlements. + +First, configure the trust policy as below. Replace `MY_AWS_ACC` with your AWS account number, `MY_AWS_REGION` with your AWS region, and `MY_OIDC_PROVIDER` with the id of your OIDC provider: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${MY_AWS_ACC}:oidc-provider/oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:sub": "system:serviceaccount:cattle-csp-adapter-system:rancher-csp-adapter", + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:aud": "sts.amazonaws.com" + } + } + } + ] +} +``` + +Next, use a policy for the role which has the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "RancherCSPAdapterPermissions", + "Effect": "Allow", + "Action": [ + "license-manager:ListReceivedLicenses", + "license-manager:CheckoutLicense", + "license-manager:ExtendLicenseConsumption", + "license-manager:CheckInLicense", + "license-manager:GetLicense", + "license-manager:GetLicenseUsage" + ], + "Resource": "*" + } + ] +} +``` + +Save the name of the role. You will need it later on when installing the CSP adapter. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md new file mode 100644 index 00000000000..02e6bb86340 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md @@ -0,0 +1,28 @@ +--- +title: Common Issues +--- + +**After installing the adapter, a banner message appears in Rancher that says "AWS Marketplace Adapter: Unable to run the adapter, please check the adapter logs"** + +This error indicates that while the adapter was installed into the cluster, an error has occurred which prevents it from properly checking-in/checking-out licenses. + +This often occurs because the IAM role was not set up properly. Review the [prerequisites](./adapter-requirements.md) and verify that: + +- An OIDC provider has been created/associated with the cluster Rancher is running on. +- The IAM role has been configured to trust this OIDC provider. +- The IAM role has at least the permissions outlined in the policy. + +If all of the above have been configured correctly, reach out to support for assistance. + +**I see a banner message that states, "AWS Marketplace Adapter: You have exceeded your licensed node count. At least x more license(s) are required in AWS to become compliant"** + +This message indicates that you do not have enough entitlements for the amount of nodes Rancher is currently managing. + +Keep in mind the following limitations: + +- Each entitlement is valid for a certain number of nodes. +- Every node currently managed by Rancher counts toward your usage total (with exception of nodes in the cluster rancher is installed on). +- Each entitlement can be used by at most one Rancher instance. For example, if you have two running Rancher instances in your account (each installed on a separate EKS cluster), then you will need at least two entitlements. + +You may also have recently uninstalled/re-installed the adapter. If the adapter loses track of the licenses that it is currently managing, it can take up to an hour to resolve the actual state of the licenses. + diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md new file mode 100644 index 00000000000..3eb473c9902 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -0,0 +1,159 @@ +--- +title: Installing the Adapter +--- + + + + + +> **Important:** If you are attempting to re-install the adapter, you may experience errant out-of-compliance messages for up to an hour. + +### Rancher vs. Adapter Compatibility Matrix + +:::note Important: + +Different versions of the CSP adapter rely on features found in specific versions of Rancher. +In order to deploy and run the adapter successfully, you need to ensure its version corresponds to the necessary version of Rancher. + +::: + +| Rancher Version | Adapter Version | +|-----------------|:---------------:| +| v2.7.0 | v2.0.0 | +| v2.7.1 | v2.0.0 | +| v2.7.2 | v2.0.1 | +| v2.7.3 | v2.0.1 | +| v2.7.4 | v2.0.1 | +| v2.7.5 | v2.0.2 | + + +### 1. Gain Access to the Local Cluster + +> **Note:** Only admin users should have access to the local cluster. Because the CSP adapter must be installed in the local cluster, this installation must be carried out by an admin user. + +First, click on the local cluster and download a kubeconfig token. You can then configure your CLI to use this new token with the following command, replacing `$TOKEN_PATH` with the path on your filesystem to the downloaded token: + +```bash +export KUBECONFIG=$TOKEN_PATH +``` + +### 2. Create the Adapter Namespace + +Create the namespace that the adapter will be installed in. + +```bash +kubectl create ns cattle-csp-adapter-system +``` + +### 3. Create Certificate Secrets + +The adapter requires access to the root CA that Rancher is using to communicate with the Rancher server. You can read more about which certificate options Rancher supports in the [chart options page](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md). + +If your Rancher install uses a certificate signed by a recognized Certificate Authority such as Let's Encrypt, then you can safely skip to [Step 4](#4-install-the-chart). + +However, if your Rancher install uses a custom certificate such as a Rancher-generated certificate or one signed by a private Certificate Authority, you will need to provide the certificate for this authority in PEM-encoded format so that the adapter can communicate with Rancher. + +First, retrieve the certificate that Rancher is using and place in a file named `ca-additional.pem`. If you are using the Rancher-generated certs option, this can be done with the following command: + +```bash +kubectl get secret tls-rancher -n cattle-system -o jsonpath="{.data.tls\.crt}" | base64 -d >> ca-additional.pem +``` + +Then, create a secret which uses this cert: + +```bash +kubectl -n cattle-csp-adapter-system create secret generic tls-ca-additional --from-file=ca-additional.pem +``` + +> **Important:** Do not change the names of the file or of the created secret. Making changes to these values may result in errors when the adapter runs. + +### 4. Install the Chart + +First, add the `rancher/charts` repo using the following command: + +```bash +helm repo add rancher-charts https://charts.rancher.io +``` + +Next, install the CSP adapter. You must specify several values, including the account number, and the name of the role created in the prerequisites. + +Ensure that you use the version of the CSP adapter that matches the version of Rancher that you are running, as defined [above](#rancher-vs-adapter-compatibility-matrix). + +For the below instructions, replace `$MY_ACC_NUM` with your AWS account number and `$MY_ROLE_NAME` with the name of the role created in the prerequisites. In addition, replace `$CSP_ADAPTER_VERSION` with the version that matches your Rancher version in the [version matrix](#rancher-vs-adapter-compatibility-matrix). + +> **Note:** If you use shell variables, do not specify quotation marks. For example, MY_ACC_NUM=123456789012 will work, but MY_ACC_NUM="123456789012" will fail. + +> **Note:** Accounts using the AWS Marketplace listing for the EU and the UK will need to specify an additional `--set image.repository=rancher/rancher-csp-adapter-eu` option. To see if your account needs this option when installing the adapter, refer to the usage instructions of the marketplace listing. + + + + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --version $CSP_ADAPTER_VERSION + ``` + + + Alternatively, you can use a `values.yaml` and specify options like below: + + ```yaml + aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME + ``` + + > **Note:** The account number needs to be specified in a string format, like the above, or the installation will fail. + + You can then install the adapter with the following command: + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION + ``` + + + + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --set additionalTrustedCAs=true --version $CSP_ADAPTER_VERSION + ``` + + Alternatively, you can use a `values.yaml` and specify options the below: + + ```yaml + aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME + additionalTrustedCAs: true + ``` + + > **Note:** The account number needs to be specified in a string format, like the above, or the installation will fail. + + You can then install the adapter with the following command: + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION + ``` + + + + +### 5. Managing Certificate Updates + +If you had to create a secret storing a custom cert in [Step 3](#3-create-certificate-secrets), you will need to update this secret over time as the certificate is rotated. + +First, delete the original secret in the cattle-csp-adapter-system namespace, using the below command: + +```bash +kubectl delete secret tls-ca-additional -n cattle-csp-adapter-system +``` + +Then, follow the original installation steps in [Step 3](#3-create-certificate-secrets) to replace the content of the secret with the updated value. + +Finally, restart the rancher-csp-adapter deployment to ensure that the updated value is made available to the adapter: + +```bash +kubectl rollout restart deploy rancher-csp-adapter -n cattle-csp-adapter-system +``` + +> **Note:** There are methods such as cert-manager's [trust operator](https://cert-manager.io/docs/projects/trust/) which can help reduce the number of manual rotation tasks over time. While these options are not officially supported, they may be useful to users wishing to automate some of these tasks. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md new file mode 100644 index 00000000000..16e0ac3443e --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md @@ -0,0 +1,21 @@ +--- +title: Uninstalling The Adapter +--- + +### 1. Uninstall the adapter chart using helm. + +```bash +helm uninstall rancher-csp-adapter -n cattle-csp-adapter-system +``` + +### 2. Remove the namespace created for the adapter. + +```bash +kubectl delete ns cattle-csp-adapter-system +``` + +### 3. (Optional) remove any outstanding user notifications. + +```bash +kubectl delete RancherUserNotification csp-compliance +``` diff --git a/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/supportconfig.md b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/supportconfig.md new file mode 100644 index 00000000000..9d87830a82e --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/cloud-marketplace/supportconfig.md @@ -0,0 +1,53 @@ +--- +title: Supportconfig bundle +--- + +After installing the CSP adapter, you will have the ability to generate a supportconfig bundle. This bundle is a tar file which can be used to quickly provide information to support. + +These bundles can be created through Rancher or through direct access to the cluster that Rancher is installed on. Note that accessing through Rancher is preferred. + +> **Note:** Only admin users can generate/download supportconfig bundles, regardless of method. + +### Accessing through Rancher + +First, click on the hamburger menu. Then click the `Get Support` button. + +![Get Support](/img/support-help.png) + +In the next page, click on the `Generate Support Config` button. + +> **Note:** If the adapter is not installed, the option to generate the supportconfig bundle will not be present. You must install the CSP adapter to generate a supportconfig bundle. + +![Get Support](/img/generate-support-config.png) + +### Accessing without rancher + +First, generate a kubeconfig for the cluster that Rancher is installed on. + +> **Note:** If Rancher is down, you will not be able to use a kubeconfig token generated by Rancher to access the cluster. + +Configure your shell environment to use this kubeconfig token: + +```bash +export KUBECONFIG=$MY_KUBECONFIG_PATH +``` + +It is recommended to create a temporary working directory while running this command, like below: + +```bash +mkdir temp && cd temp +``` + +Then, retrieve the supportconfig bundle: + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && tar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` + +This will create a `supportconfig_rancher.tar` file in your current directory. + +Users who run these commands on Mac may experience issues due to incompatibilities between gnu-tar and bsd-tar. If support has issues reading a supportconfig that you produce, try using the below command after making gnu-tar accessible as `gtar` on your path: + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && gtar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` diff --git a/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/architecture.md b/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/architecture.md new file mode 100644 index 00000000000..f012a3a9921 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/architecture.md @@ -0,0 +1,8 @@ +--- +title: Architecture +--- + +Fleet can manage deployments from git of raw Kubernetes YAML, Helm charts, or Kustomize or any combination of the three. Regardless of the source, all resources are dynamically turned into Helm charts, and Helm is used as the engine to deploy everything in the cluster. This gives you a high degree of control, consistency, and auditability. Fleet focuses not only on the ability to scale, but to give one a high degree of control and visibility to exactly what is installed on the cluster. + +![Architecture](/img/fleet-architecture.svg) + diff --git a/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md b/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md new file mode 100644 index 00000000000..e6a3f8cf961 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md @@ -0,0 +1,54 @@ +--- +title: Using Fleet Behind a Proxy +--- + +In this section, you'll learn how to enable Fleet in a setup that has a Rancher server with a public IP a Kubernetes cluster that has no public IP, but is configured to use a proxy. + +Rancher does not establish connections with registered downstream clusters. The Rancher agent deployed on the downstream cluster must be able to establish the connection with Rancher. + +To set up Fleet to work behind a proxy, you will need to set the **Agent Environment Variables** for the downstream cluster. These are cluster-level configuration options. + +Through the Rancher UI, you can configure these environment variables for any cluster type, including registered and custom clusters. The variables can be added while editing an existing cluster or while provisioning a new cluster. + +For public downstream clusters, it is sufficient to [set the required environment variables in the Rancher UI.](#setting-environment-variables-in-the-rancher-ui) + +For private nodes or private clusters, the environment variables need to be set on the nodes themselves. Then the environment variables are configured from the Rancher UI, typically when provisioning a custom cluster or when registering the private cluster. For an example of how to set the environment variables on Ubuntu node in a K3s Kubernetes cluster, see [this section.](#setting-environment-variables-on-private-nodes) + +## Required Environment Variables + +When adding Fleet agent environment variables for the proxy, replace with your private proxy IP. + +| Variable Name | Value | +|------------------|--------| +| `HTTP_PROXY` | http://:8888 | +| `HTTPS_PROXY` | http://:8888 +| `NO_PROXY` | 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local | + +## Setting Environment Variables in the Rancher UI + +To add the environment variable to an existing cluster, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add environment variables and click **â‹® > Edit Config**. +1. Click **Advanced Options**. +1. Click **Add Environment Variable**. +1. Enter the [required environment variables](#required-environment-variables) +1. Click **Save**. + +**Result:** The Fleet agent works behind a proxy. + +## Setting Environment Variables on Private Nodes + +For private nodes and private clusters, the proxy environment variables need to be set on the nodes themselves, as well as configured from the Rancher UI. + +This example shows how the environment variables would be set up on an Ubuntu node in a K3s Kubernetes cluster: + +``` +ssh -o ForwardAgent=yes ubuntu@ +ssh +export proxy_private_ip= +export HTTP_PROXY=http://${proxy_private_ip}:8888 +export HTTPS_PROXY=http://${proxy_private_ip}:8888 +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md b/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md new file mode 100644 index 00000000000..aea98b74dbc --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md @@ -0,0 +1,21 @@ +--- +title: Windows Support +--- + +Prior to Rancher v2.5.6, the `agent` did not have native Windows manifests on downstream clusters with Windows nodes. This would result in a failing `agent` pod for the cluster. + +If you are upgrading from an older version of Rancher to v2.5.6+, you can deploy a working `agent` with the following workflow *in the downstream cluster*: + +1. Cordon all Windows nodes. +1. Apply the below toleration to the `agent` workload. +1. Uncordon all Windows nodes. +1. Delete all `agent` pods. New pods should be created with the new toleration. +1. Once the `agent` pods are running, and auto-update is enabled for Fleet, they should be updated to a Windows-compatible `agent` version. + +```yaml +tolerations: +- effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/harvester.md b/versioned_docs/version-2.8/integrations-in-rancher/harvester.md new file mode 100644 index 00000000000..872bf854f8d --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/harvester.md @@ -0,0 +1,48 @@ +--- +title: Harvester Integration +--- + +Introduced in Rancher v2.6.1, [Harvester](https://docs.harvesterhci.io/) is an open-source hyper-converged infrastructure (HCI) software built on Kubernetes. Harvester installs on bare metal servers and provides integrated virtualization and distributed storage capabilities. Although Harvester operates using Kubernetes, it does not require users to know Kubernetes concepts, making it a more user-friendly application. + +### Feature Flag + +The Harvester feature flag is used to manage access to the Virtualization Management (VM) page in Rancher where users can navigate directly to Harvester clusters and access the Harvester UI. The Harvester feature flag is enabled by default. Click [here](../pages-for-subheaders/enable-experimental-features.md) for more information on feature flags in Rancher. + +To navigate to the Harvester cluster, click **☰ > Virtualization Management**. From Harvester Clusters page, click one of the clusters listed to go to the single Harvester cluster view. + +* If the Harvester feature flag is enabled, Harvester clusters will be filtered out from any pages or apps (such as Fleet and the multi-cluster app) that list Kubernetes clusters. + +* If the Harvester feature flag is disabled, and a Harvester cluster is imported, the Harvester cluster will be shown in the Rancher cluster list in the Cluster Management page. Harvester clusters will only be shown on the cluster list when the feature flag is off. + +* With the Harvester integration, Harvester clusters can now be imported into Rancher as a cluster type `Harvester`. + +* Users may import a Harvester cluster only on the Virtualization Management page. Importing a cluster on the Cluster Management page is not supported, and a warning will advise you to return to the VM page to do so. + +### Harvester Node Driver + +The [Harvester node driver](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/) is generally available for RKE and RKE2 options in Rancher. The node driver is available whether or not the Harvester feature flag is enabled. Note that the node driver is off by default. Users may create RKE or RKE2 clusters on Harvester only from the Cluster Management page. + +Harvester allows `.ISO` images to be uploaded and displayed through the Harvester UI, but this is not supported in the Rancher UI. This is because `.ISO` images usually require additional setup that interferes with a clean deployment (without requiring user intervention), and they are not typically used in cloud environments. + +Click [here](../pages-for-subheaders/about-provisioning-drivers.md#node-drivers) for more information on node drivers in Rancher. + +### Port Requirements + +The port requirements for the Harvester cluster can be found [here](https://docs.harvesterhci.io/v1.1/install/requirements#networking). + +In addition, other networking considerations are as follows: + +- Be sure to enable VLAN trunk ports of the physical switch for VM VLAN networks. +- Follow the networking setup guidance [here](https://docs.harvesterhci.io/v1.1/networking/clusternetwork). + +For other port requirements for other guest clusters, such as K3s and RKE1, please see [these docs](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters). + +### Limitations + +--- +**Applicable to Rancher v2.6.1 and v2.6.2 only:** + +- Harvester v0.3.0 doesn’t support air-gapped environment installation. +- Harvester v0.3.0 doesn’t support upgrade from v0.2.0 nor upgrade to v1.0.0. + +--- \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md new file mode 100644 index 00000000000..0c229f288b4 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md @@ -0,0 +1,60 @@ +--- +title: Additional Steps for Installing Istio on RKE2 and K3s Clusters +--- + +When installing or upgrading the Istio Helm chart through **Apps,** + +1. If you are installing the chart, click **Customize Helm options before install** and click **Next**. +1. You will see options for configuring the Istio Helm chart. On the **Components** tab, check the box next to **Enabled CNI**. +1. Add a custom overlay file specifying `cniBinDir` and `cniConfDir`. For more information on these options, refer to the [Istio documentation.](https://istio.io/latest/docs/setup/additional-setup/cni/#helm-chart-parameters) An example is below: + + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /opt/cni/bin + cniConfDir: /etc/cni/net.d +``` + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /var/lib/rancher/k3s/data/current/bin + cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d +``` + + + +**Result:** Now you should be able to utilize Istio as desired, including sidecar injection and monitoring via Kiali. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/pod-security-policies.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/pod-security-policies.md new file mode 100644 index 00000000000..e774b97bf8c --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/pod-security-policies.md @@ -0,0 +1,53 @@ +--- +title: Enable Istio with Pod Security Policies +--- + +If you have restrictive Pod Security Policies enabled, then Istio may not be able to function correctly, because it needs certain permissions in order to install itself and manage pod infrastructure. In this section, we will configure a cluster with PSPs enabled for an Istio install, and also set up the Istio CNI plugin. + +The Istio CNI plugin removes the need for each application pod to have a privileged `NET_ADMIN` container. For further information, see the [Istio CNI Plugin docs](https://istio.io/docs/setup/additional-setup/cni). Please note that the [Istio CNI Plugin is in alpha](https://istio.io/about/feature-stages/). + +:::note Prerequisites: + +- The cluster must be an RKE Kubernetes cluster. +- The cluster must have been created with a default PodSecurityPolicy. + +To enable pod security policy support when creating a Kubernetes cluster in the Rancher UI, go to Advanced Options. In the Pod Security Policy Support section, click Enabled. Then select a default pod security policy. + +::: + +1. [Set the PodSecurityPolicy to unrestricted](#1-set-the-podsecuritypolicy-to-unrestricted) +2. [Enable the CNI](#2-enable-the-cni) +3. [Verify that the CNI is working.](#3-verify-that-the-cni-is-working) + +### 1. Set the PodSecurityPolicy to unrestricted + +An unrestricted PSP allows Istio to be installed. + +Set the PSP to `unrestricted` in the project where is Istio is installed, or the project where you plan to install Istio. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the **Project: System** and select the **â‹® > Edit Config**. +1. Change the Pod Security Policy option to be unrestricted, then click **Save**. + +### 2. Enable the CNI + +When installing or upgrading Istio through **Apps,** + +1. Click **Components**. +2. Check the box next to **Enabled CNI**. +3. Finish installing or upgrading Istio. + +The CNI can also be enabled by editing the `values.yaml`: + +``` +istio_cni.enabled: true +``` + +Istio should install successfully with the CNI enabled in the cluster. + +### 3. Verify that the CNI is working + +Verify that the CNI is working by deploying a [sample application](https://istio.io/latest/docs/examples/bookinfo/) or deploying one of your own applications. + diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/project-network-isolation.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/project-network-isolation.md new file mode 100644 index 00000000000..16fde314183 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/project-network-isolation.md @@ -0,0 +1,21 @@ +--- +title: Additional Steps for Project Network Isolation +--- + +In clusters where: + +- You are using the Canal network plugin with Rancher before v2.5.8, or you are using Rancher v2.5.8+ with an any RKE network plug-in that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin +- The Project Network Isolation option is enabled +- You install the Istio Ingress module + +The Istio Ingress Gateway pod won't be able to redirect ingress traffic to the workloads by default. This is because all the namespaces will be inaccessible from the namespace where Istio is installed. You have two options. + +The first option is to add a new Network Policy in each of the namespaces where you intend to have ingress controlled by Istio. Your policy should include the following lines: + +``` +- podSelector: + matchLabels: + app: istio-ingressgateway +``` + +The second option is to move the `istio-system` namespace to the `system` project, which by default is excluded from the network isolation. \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md new file mode 100644 index 00000000000..10907f4718c --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md @@ -0,0 +1,120 @@ +--- +title: Selectors and Scrape Configs +--- + +The Monitoring app sets `prometheus.prometheusSpec.ignoreNamespaceSelectors=false`, which enables monitoring across all namespaces by default. + +This ensures you can view traffic, metrics and graphs for resources deployed in a namespace with `istio-injection=enabled` label. + +If you would like to limit Prometheus to specific namespaces, set `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`. Once you do this, you will need to add additional configuration to continue to monitor your resources. + + +### Limiting Monitoring to Specific Namespaces by Setting ignoreNamespaceSelectors to True + +To limit monitoring to specific namespaces, you will edit the `ignoreNamespaceSelectors` Helm chart option. You will configure this option when installing or upgrading the Monitoring Helm chart: + +1. When installing or upgrading the Monitoring Helm chart, edit the values.yml and set`prometheus.prometheusSpec.ignoreNamespaceSelectors=true`. +1. Complete the install or upgrade. + +**Result:** Prometheus will be limited to specific namespaces which means one of the following configurations will need to be set up to continue to view data in various dashboards + +### Enabling Prometheus to Detect Resources in Other Namespaces + +There are two different ways to enable Prometheus to detect resources in other namespaces when `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`: + +- **Monitoring specific namespaces:** Add a Service Monitor or Pod Monitor in the namespace with the targets you want to scrape. +- **Monitoring across namespaces:** Add an `additionalScrapeConfig` to your rancher-monitoring instance to scrape all targets in all namespaces. + +### Monitoring Specific Namespaces: Create a Service Monitor or Pod Monitor + +This option allows you to define which specific services or pods you would like monitored in a specific namespace. + +The usability tradeoff is that you have to create the service monitor or pod monitor per namespace since you cannot monitor across namespaces. + +:::note Prerequisite: + +Define a ServiceMonitor or PodMonitor for ``. An example ServiceMonitor is provided below. + +::: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the top navigation bar, open the kubectl shell. +1. If the ServiceMonitor or PodMonitor file is stored locally in your cluster, in `kubectl create -f .yaml`. +1. If the ServiceMonitor or PodMonitor is not stored locally, run `cat<< EOF | kubectl apply -f -`, paste the file contents into the terminal, then run `EOF` to complete the command. +1. Run `kubectl label namespace istio-injection=enabled` to enable the envoy sidecar injection. + +**Result:** `` can be scraped by prometheus. + +
Example Service Monitor for Istio Proxies
+ +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: envoy-stats-monitor + namespace: istio-system + labels: + monitoring: istio-proxies +spec: + selector: + matchExpressions: + - {key: istio-prometheus-ignore, operator: DoesNotExist} + namespaceSelector: + any: true + jobLabel: envoy-stats + endpoints: + - path: /stats/prometheus + targetPort: 15090 + interval: 15s + relabelings: + - sourceLabels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - action: labeldrop + regex: "__meta_kubernetes_pod_label_(.+)" + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: namespace + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: pod_name +``` + +### Monitoring across namespaces: Set ignoreNamespaceSelectors to False + +This enables monitoring across namespaces by giving Prometheus additional scrape configurations. + +The usability tradeoff is that all of Prometheus' `additionalScrapeConfigs` are maintained in a single Secret. This could make upgrading difficult if monitoring is already deployed with additionalScrapeConfigs before installing Istio. + +1. When installing or upgrading the Monitoring Helm chart, edit the values.yml and set the `prometheus.prometheusSpec.additionalScrapeConfigs` array to the **Additional Scrape Config** provided below. +1. Complete the install or upgrade. + +**Result:** All namespaces with the `istio-injection=enabled` label will be scraped by prometheus. + +
Additional Scrape Config
+ +``` yaml +- job_name: 'istio/envoy-stats' + scrape_interval: 15s + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:15090 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name +``` diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/cpu-and-memory-allocations.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/cpu-and-memory-allocations.md new file mode 100644 index 00000000000..37472e328c7 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/cpu-and-memory-allocations.md @@ -0,0 +1,63 @@ +--- +title: CPU and Memory Allocations +--- + +This section describes the minimum recommended computing resources for the Istio components in a cluster. + +The CPU and memory allocations for each component are [configurable.](#configuring-resource-allocations) + +Before enabling Istio, we recommend that you confirm that your Rancher worker nodes have enough CPU and memory to run all of the components of Istio. + +:::tip + +In larger deployments, it is strongly advised that the infrastructure be placed on dedicated nodes in the cluster by adding a node selector for each Istio component. + +::: + +The table below shows a summary of the minimum recommended resource requests and limits for the CPU and memory of each core Istio component. + +In Kubernetes, the resource request indicates that the workload will not deployed on a node unless the node has at least the specified amount of memory and CPU available. If the workload surpasses the limit for CPU or memory, it can be terminated or evicted from the node. For more information on managing resource limits for containers, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) + +| Workload | CPU - Request | Memory - Request | CPU - Limit | Memory - Limit | +|----------------------|---------------|------------|-----------------|-------------------| +| ingress gateway | 100m | 128mi | 2000m | 1024mi | +| egress gateway | 100m | 128mi | 2000m | 1024mi | +| istiod | 500m | 2048mi | No limit | No limit | +| proxy | 10m | 10mi | 2000m | 1024mi | +| **Totals:** | **710m** | **2314Mi** | **6000m** | **3072Mi** | + +## Configuring Resource Allocations + +You can individually configure the resource allocation for each type of Istio component. This section includes the default resource allocations for each component. + +To make it easier to schedule the workloads to a node, a cluster-admin can reduce the CPU and memory resource requests for the component. However, the default CPU and memory allocations are the minimum that we recommend. + +You can find more information about Istio configuration in the [official Istio documentation](https://istio.io/). + +To configure the resources allocated to an Istio component, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Installed Apps**. +1. Go to the `istio-system` namespace. In one of the Istio workloads, such as `rancher-istio`, click **⋮ > Edit/Upgrade**. +1. Click **Upgrade** to edit the base components via changes to the values.yaml or add an [overlay file](../../pages-for-subheaders/configuration-options.md#overlay-file). For more information about editing the overlay file, see [this section.](cpu-and-memory-allocations.md#editing-the-overlay-file) +1. Change the CPU or memory allocations, the nodes where each component will be scheduled to, or the node tolerations. +1. Click **Upgrade**. to rollout changes + +**Result:** The resource allocations for the Istio components are updated. + +### Editing the Overlay File + +The overlay file can contain any of the values in the [Istio Operator spec.](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/#IstioOperatorSpec) The overlay file included with the Istio application is just one example of a potential configuration of the overlay file. + +As long as the file contains `kind: IstioOperator` and the YAML options are valid, the file can be used as an overlay. + +In the example overlay file provided with the Istio application, the following section allows you to change Kubernetes resources: + +``` +# k8s: +# resources: +# requests: +# cpu: 200m +``` diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/disable-istio.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/disable-istio.md new file mode 100644 index 00000000000..052122c4891 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/disable-istio.md @@ -0,0 +1,42 @@ +--- +title: Disabling Istio +--- + +This section describes how to uninstall Istio in a cluster or disable a namespace, or workload. + +## Uninstall Istio in a Cluster + +To uninstall Istio, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps > Installed Apps**. +1. In the `istio-system` namespace, go to `rancher-istio` and click **⋮ > Delete**. +1. After `rancher-istio` is deleted, you can then select all the remaining apps in the `istio-system` namespace and click **Delete**. + +**Result:** The `rancher-istio` app in the cluster gets removed. The Istio sidecar cannot be deployed on any workloads in the cluster. + +:::note + +You can no longer disable and re-enable your Istio installation. If you would like to save your settings for a future install, view and save individual YAMLs to refer back to / reuse for future installations. + +::: + +**Troubleshooting Uninstall:** If you didn't follow the uninstall steps, you may encounter a warning during uninstall: + +`Error: uninstallation completed with 1 error(s): unable to build kubernetes objects for delete: unable to recognize "": no matches for kind "MonitoringDashboard" in version "monitoring.kiali.io/v1alpha1"` + +This could mean a few things. You either selected all the apps in the `istio-system` namespace and deleted them at the same time, or you deleted `rancher-istio` chart dependencies prior to deleting the `rancher-istio` chart. Since the uninstall did not complete properly, you will have resources remaining in the `istio-system` namespace that you will need to manually clean up. Another option to avoid manual clean up is to install `rancher-istio` again, then uninstall it in the correct order. + +## Disable Istio in a Namespace + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the namespace where you want to enable Istio and click **⋮ > Enable Istio Auto Injection**. Alternately, click the namespace, and then on the namespace detail page, click **⋮ > Enable Istio Auto Injection**. + +**Result:** When workloads are deployed in this namespace, they will not have the Istio sidecar. + +## Remove the Istio Sidecar from a Workload + +Disable Istio in the namespace, then redeploy the workloads with in it. They will be deployed without the Istio sidecar. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/istio/rbac-for-istio.md b/versioned_docs/version-2.8/integrations-in-rancher/istio/rbac-for-istio.md new file mode 100644 index 00000000000..e33bdb72540 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/istio/rbac-for-istio.md @@ -0,0 +1,43 @@ +--- +title: Role-based Access Control +--- + +This section describes the permissions required to access Istio features. + +The rancher istio chart installs three `ClusterRoles` + +## Cluster-Admin Access + +By default, only those with the `cluster-admin` `ClusterRole` can: + +- Install istio app in a cluster +- Configure resource allocations for Istio + + +## Admin and Edit access + +By default, only Admin and Edit roles can: + +- Enable and disable Istio sidecar auto-injection for namespaces +- Add the Istio sidecar to workloads +- View the traffic metrics and traffic graph for the cluster +- Configure Istio's resources (such as the gateway, destination rules, or virtual services) + +## Summary of Default Permissions for Kubernetes Default roles + +Istio creates three `ClusterRoles` and adds Istio CRD access to the following default K8s `ClusterRole`: + +ClusterRole create by chart | Default K8s ClusterRole | Rancher Role | + ------------------------------:| ---------------------------:|---------:| + `istio-admin` | admin| Project Owner | + `istio-edit`| edit | Project Member | + `istio-view` | view | Read-only | + +Rancher will continue to use cluster-owner, cluster-member, project-owner, project-member, etc as role names, but will utilize default roles to determine access. For each default K8s `ClusterRole` there are different Istio CRD permissions and K8s actions (Create ( C ), Get ( G ), List ( L ), Watch ( W ), Update ( U ), Patch ( P ), Delete( D ), All ( * )) that can be performed. + + +|CRDs | Admin | Edit | View +|----------------------------| ------| -----| ----- +|
  • `config.istio.io`
    • `adapters`
    • `attributemanifests`
    • `handlers`
    • `httpapispecbindings`
    • `httpapispecs`
    • `instances`
    • `quotaspecbindings`
    • `quotaspecs`
    • `rules`
    • `templates`
| GLW | GLW | GLW +|
  • `networking.istio.io`
    • `destinationrules`
    • `envoyfilters`
    • `gateways`
    • `serviceentries`
    • `sidecars`
    • `virtualservices`
    • `workloadentries`
| * | * | GLW +|
  • `security.istio.io`
    • `authorizationpolicies`
    • `peerauthentications`
    • `requestauthentications`
| * | * | GLW \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md b/versioned_docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md new file mode 100644 index 00000000000..e4020351ab1 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md @@ -0,0 +1,75 @@ +--- +title: Flows and ClusterFlows +--- + +See the [Logging operator documentation](https://kube-logging.github.io/docs/configuration/flow/) for the full details on how to configure `Flows` and `ClusterFlows`. + +See [Rancher Integration with Logging Services: Troubleshooting](../../../pages-for-subheaders/logging.md#The-Logging-Buffer-Overloads-Pods) for how to resolve memory problems with the logging buffer. + +## Flows + +A `Flow` defines which logs to collect and filter and which output to send the logs to. + +The `Flow` is a namespaced resource, which means logs will only be collected from the namespace that the `Flow` is deployed in. + +`Flows` can be configured by filling out forms in the Rancher UI. + +For more details about the `Flow` custom resource, see [FlowSpec.](https://kube-logging.github.io/docs/configuration/crds/v1beta1/flow_types/) + +### Matches + +Match statements are used to select which containers to pull logs from. + +You can specify match statements to select or exclude logs according to Kubernetes labels, container and host names. Match statements are evaluated in the order they are defined and processed only until the first matching select or exclude rule applies. + +Matches can be configured by filling out the `Flow` or `ClusterFlow` forms in the Rancher UI. + +For detailed examples on using the match statement, see the [official documentation on log routing.](https://kube-logging.github.io/docs/configuration/log-routing/) + +### Filters + +You can define one or more filters within a `Flow`. Filters can perform various actions on the logs, for example, add additional data, transform the logs, or parse values from the records. The filters in the `Flow` are applied in the order in the definition. + +For a list of filters supported by the Logging operator, see [the official documentation on Fluentd filters](https://kube-logging.github.io/docs/configuration/plugins/filters/). + +Filters need to be configured in YAML. + +### Outputs + +This `Output` will receive logs from the `Flow`. Because the `Flow` is a namespaced resource, the `Output` must reside in same namespace as the `Flow`. + +`Outputs` can be referenced when filling out the `Flow` or `ClusterFlow` forms in the Rancher UI. + +## ClusterFlows + +Matches, filters and `Outputs` are configured for `ClusterFlows` in the same way that they are configured for `Flows`. The key difference is that the `ClusterFlow` is scoped at the cluster level and can configure log collection across all namespaces. + +`ClusterFlows` can be configured by filling out forms in the Rancher UI. + +After `ClusterFlow` selects logs from all namespaces in the cluster, logs from the cluster will be collected and logged to the selected `ClusterOutput`. + +## YAML Example + +The following example `Flow` transforms the log messages from the default namespace and sends them to an S3 `Output`: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: flow-sample + namespace: default +spec: + filters: + - parser: + remove_key_name_field: true + parse: + type: nginx + - tag_normaliser: + format: ${namespace_name}.${pod_name}.${container_name} + localOutputRefs: + - s3-output + match: + - select: + labels: + app: nginx +``` diff --git a/versioned_docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md b/versioned_docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md new file mode 100644 index 00000000000..aa558385bf0 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md @@ -0,0 +1,295 @@ +--- +title: Outputs and ClusterOutputs +--- + +See the [Logging operator documentation](https://kube-logging.github.io/docs/configuration/flow/) for the full details on how to configure `Flows` and `ClusterFlows`. + +See [Rancher Integration with Logging Services: Troubleshooting](../../../pages-for-subheaders/logging.md#The-Logging-Buffer-Overloads-Pods) for how to resolve memory problems with the logging buffer. + +## Outputs + +The `Output` resource defines where your `Flows` can send the log messages. `Outputs` are the final stage for a logging `Flow`. + +The `Output` is a namespaced resource, which means only a `Flow` within the same namespace can access it. + +You can use secrets in these definitions, but they must also be in the same namespace. + +`Outputs` can be configured by filling out forms in the Rancher UI. + +For the details of `Output` custom resource, see [OutputSpec.](https://kube-logging.github.io/docs/configuration/crds/v1beta1/output_types/). + +The Rancher UI provides forms for configuring the following `Output` types: + +- Amazon ElasticSearch +- Azure Storage +- Cloudwatch +- Datadog +- Elasticsearch +- File +- Fluentd +- GCS +- Kafka +- Kinesis Stream +- LogDNA +- LogZ +- Loki +- New Relic +- Splunk +- SumoLogic +- Syslog + +The Rancher UI provides forms for configuring the `Output` type, target, and access credentials if applicable. + +For example configuration for each logging plugin supported by the logging operator, see the [Logging operator documentation](https://kube-logging.github.io/docs/configuration/plugins/outputs/). + +## ClusterOutputs + +`ClusterOutput` defines an `Output` without namespace restrictions. It is only effective when deployed in the same namespace as the logging operator. + +`ClusterOutputs` can be configured by filling out forms in the Rancher UI. + +For the details of the `ClusterOutput` custom resource, see [ClusterOutput.](https://kube-logging.github.io/docs/configuration/crds/v1beta1/clusteroutput_types/) + +## YAML Examples + +Once logging is installed, you can use these examples to help craft your own logging pipeline. + +- [Cluster Output to ElasticSearch](#cluster-output-to-elasticsearch) +- [Output to Splunk](#output-to-splunk) +- [Output to Syslog](#output-to-syslog) +- [Unsupported Outputs](#unsupported-outputs) + +### Cluster Output to ElasticSearch + +Let's say you wanted to send all logs in your cluster to an `elasticsearch` cluster. First, we create a cluster `Output`. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-es" + namespace: "cattle-logging-system" +spec: + elasticsearch: + host: elasticsearch.example.com + port: 9200 + scheme: http +``` + +We have created this `ClusterOutput`, without elasticsearch configuration, in the same namespace as our operator: `cattle-logging-system.`. Any time we create a `ClusterFlow` or `ClusterOutput`, we have to put it in the `cattle-logging-system` namespace. + +Now that we have configured where we want the logs to go, let's configure all logs to go to that `ClusterOutput`. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: "cattle-logging-system" +spec: + globalOutputRefs: + - "example-es" +``` + +We should now see our configured index with logs in it. + + +### Output to Splunk + +What if we have an application team who only wants logs from a specific namespaces sent to a `splunk` server? For this case, we can use namespaced `Outputs` and `Flows`. + +Before we start, let's set up that team's application: `coolapp`. + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: devteam +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coolapp + namespace: devteam + labels: + app: coolapp +spec: + replicas: 2 + selector: + matchLabels: + app: coolapp + template: + metadata: + labels: + app: coolapp + spec: + containers: + - name: generator + image: paynejacob/loggenerator:latest +``` + +With `coolapp` running, we will follow a similar path as when we created a `ClusterOutput`. However, unlike `ClusterOutputs`, we create our `Output` in our application's namespace. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Output +metadata: + name: "devteam-splunk" + namespace: "devteam" +spec: + splunkHec: + hec_host: splunk.example.com + hec_port: 8088 + protocol: http +``` + +Once again, let's feed our `Output` some logs: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: "devteam-logs" + namespace: "devteam" +spec: + localOutputRefs: + - "devteam-splunk" +``` + + +### Output to Syslog + +Let's say you wanted to send all logs in your cluster to an `syslog` server. First, we create a `ClusterOutput`: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-syslog" + namespace: "cattle-logging-system" +spec: + syslog: + buffer: + timekey: 30s + timekey_use_utc: true + timekey_wait: 10s + flush_interval: 5s + format: + type: json + app_name_field: test + host: syslog.example.com + insecure: true + port: 514 + transport: tcp +``` + +Now that we have configured where we want the logs to go, let's configure all logs to go to that `Output`. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: cattle-logging-system +spec: + globalOutputRefs: + - "example-syslog" +``` + +### Unsupported Outputs + +For the final example, we create an `Output` to write logs to a destination that is not supported out of the box: + +:::note Note on syslog: + +`syslog` is a supported `Output`. However, this example still provides an overview on using unsupported plugins. + +::: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: syslog-config + namespace: cattle-logging-system +type: Opaque +stringData: + fluent-bit.conf: | + [INPUT] + Name forward + Port 24224 + + [OUTPUT] + Name syslog + InstanceName syslog-output + Match * + Addr syslog.example.com + Port 514 + Cluster ranchers + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentbit-syslog-forwarder + namespace: cattle-logging-system + labels: + output: syslog +spec: + selector: + matchLabels: + output: syslog + template: + metadata: + labels: + output: syslog + spec: + containers: + - name: fluentbit + image: paynejacob/fluent-bit-out-syslog:latest + ports: + - containerPort: 24224 + volumeMounts: + - mountPath: "/fluent-bit/etc/" + name: configuration + volumes: + - name: configuration + secret: + secretName: syslog-config +--- +apiVersion: v1 +kind: Service +metadata: + name: syslog-forwarder + namespace: cattle-logging-system +spec: + selector: + output: syslog + ports: + - protocol: TCP + port: 24224 + targetPort: 24224 +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: all-logs + namespace: cattle-logging-system +spec: + globalOutputRefs: + - syslog +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: syslog + namespace: cattle-logging-system +spec: + forward: + servers: + - host: "syslog-forwarder.cattle-logging-system" + require_ack_response: false + ignore_network_errors_at_startup: false +``` + +Let's break down what is happening here. First, we create a deployment of a container that has the additional `syslog` plugin and accepts logs forwarded from another `fluentd`. Next we create an `Output` configured as a forwarder to our deployment. The deployment `fluentd` will then forward all logs to the configured `syslog` destination. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/logging/logging-architecture.md b/versioned_docs/version-2.8/integrations-in-rancher/logging/logging-architecture.md new file mode 100644 index 00000000000..560235f4f7c --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/logging/logging-architecture.md @@ -0,0 +1,28 @@ +--- +title: Architecture +--- + +This section summarizes the architecture of the Rancher logging application. + +For more details about how the Logging operator works, see the [official documentation.](https://kube-logging.github.io/docs/#architecture) + +### How the Logging Operator Works + +The Logging operator automates the deployment and configuration of a Kubernetes logging pipeline. It deploys and configures a Fluent Bit DaemonSet on every node to collect container and application logs from the node file system. + +Fluent Bit queries the Kubernetes API and enriches the logs with metadata about the pods, and transfers both the logs and the metadata to Fluentd. Fluentd receives, filters, and transfers logs to multiple `Outputs`. + +The following custom resources are used to define how logs are filtered and sent to their `Outputs`: + +- A `Flow` is a namespaced custom resource that uses filters and selectors to route log messages to the appropriate `Outputs`. +- A `ClusterFlow` is used to route cluster-level log messages. +- An `Output` is a namespaced resource that defines where the log messages are sent. +- A `ClusterOutput` defines an `Output` that is available from all `Flows` and `ClusterFlows`. + +Each `Flow` must reference an `Output`, and each `ClusterFlow` must reference a `ClusterOutput`. + +The following figure from the [Logging Operator documentation](https://kube-logging.github.io/docs/#architecture) shows the new logging architecture: + +
How the Logging Operator Works with Fluentd and Fluent Bit
+ +![How the Logging Operator Works with Fluentd](/img/banzai-cloud-logging-operator.png) diff --git a/versioned_docs/version-2.8/integrations-in-rancher/logging/logging-helm-chart-options.md b/versioned_docs/version-2.8/integrations-in-rancher/logging/logging-helm-chart-options.md new file mode 100644 index 00000000000..fd6299abc4d --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/logging/logging-helm-chart-options.md @@ -0,0 +1,94 @@ +--- +title: rancher-logging Helm Chart Options +--- + +### Enable/Disable Windows Node Logging + +You can enable or disable Windows node logging by setting `global.cattle.windows.enabled` to either `true` or `false` in the `values.yaml`. + +By default, Windows node logging will be enabled if the Cluster Dashboard UI is used to install the logging application on a Windows cluster. + +In this scenario, setting `global.cattle.windows.enabled` to `false` will disable Windows node logging on the cluster. +When disabled, logs will still be collected from Linux nodes within the Windows cluster. + +:::note + +Currently an [issue](https://github.com/rancher/rancher/issues/32325) exists where Windows nodeAgents are not deleted when performing a `helm upgrade` after disabling Windows logging in a Windows cluster. In this scenario, users may need to manually remove the Windows nodeAgents if they are already installed. + +::: + +### Working with a Custom Docker Root Directory + +If using a custom Docker root directory, you can set `global.dockerRootDirectory` in `values.yaml`. + +This will ensure that the Logging CRs created will use your specified path rather than the default Docker `data-root` location. + +Note that this only affects Linux nodes. + +If there are any Windows nodes in the cluster, the change will not be applicable to those nodes. + +### Adding NodeSelector Settings and Tolerations for Custom Taints + +You can add your own `nodeSelector` settings and add `tolerations` for additional taints by editing the logging Helm chart values. For details, see [this page.](taints-and-tolerations.md) + +### Enabling the Logging Application to Work with SELinux + +:::note Requirements: + +Logging v2 was tested with SELinux on RHEL/CentOS 7 and 8. + +::: + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. + +To use Logging v2 with SELinux, we recommend installing the `rancher-selinux` RPM according to these [instructions](../../pages-for-subheaders/selinux-rpm.md). + +Then, when installing the logging application, configure the chart to be SELinux aware by changing `global.seLinux.enabled` to `true` in the `values.yaml`. + +### Additional Logging Sources + +By default, Rancher collects logs for [control plane components](https://kubernetes.io/docs/concepts/overview/components/#control-plane-components) and [node components](https://kubernetes.io/docs/concepts/overview/components/#node-components) for all cluster types. + +In some cases, Rancher may be able to collect additional logs. + +The following table summarizes the sources where additional logs may be collected for each node types: + +| Logging Source | Linux Nodes (including in Windows cluster) | Windows Nodes | +| --- | --- | ---| +| RKE | ✓ | ✓ | +| RKE2 | ✓ | | +| K3s | ✓ | | +| AKS | ✓ | | +| EKS | ✓ | | +| GKE | ✓ | | + +To enable hosted Kubernetes providers as additional logging sources, enable **Enable enhanced cloud provider logging** option when installing or upgrading the Logging Helm chart. + +When enabled, Rancher collects all additional node and control plane logs the provider has made available, which may vary between providers + +If you're already using a cloud provider's own logging solution such as AWS CloudWatch or Google Cloud operations suite (formerly Stackdriver), it is not necessary to enable this option as the native solution will have unrestricted access to all logs. + +### Systemd Configuration + +In Rancher logging, `SystemdLogPath` must be configured for K3s and RKE2 Kubernetes distributions. + +K3s and RKE2 Kubernetes distributions log to journald, which is the subsystem of systemd that is used for logging. In order to collect these logs, the `systemdLogPath` needs to be defined. While the `run/log/journal` directory is used by default, some Linux distributions do not default to this path. For example, Ubuntu defaults to `var/log/journal`. To determine your `systemdLogPath` configuration, see steps below. + +**Steps for Systemd Configuration:** + +* Run `cat /etc/systemd/journald.conf | grep -E ^\#?Storage | cut -d"=" -f2` on one of your nodes. +* If `persistent` is returned, your `systemdLogPath` should be `/var/log/journal`. +* If `volatile` is returned, your `systemdLogPath` should be `/run/log/journal`. +* If `auto` is returned, check if `/var/log/journal` exists. + * If `/var/log/journal` exists, then use `/var/log/journal`. + * If `/var/log/journal` does not exist, then use `/run/log/journal`. + +:::note Notes: + +If any value not described above is returned, Rancher Logging will not be able to collect control plane logs. To address this issue, you will need to perform the following actions on every control plane node: + +* Set `Storage=volatile` in journald.conf. +* Reboot your machine. +* Set `systemdLogPath` to `/run/log/journal`. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/logging/rbac-for-logging.md b/versioned_docs/version-2.8/integrations-in-rancher/logging/rbac-for-logging.md new file mode 100644 index 00000000000..627e3533c2c --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/logging/rbac-for-logging.md @@ -0,0 +1,23 @@ +--- +title: Role-based Access Control for Logging +--- + +Rancher logging has two roles, `logging-admin` and `logging-view`. + +- `logging-admin` gives users full access to namespaced `Flows` and `Outputs` +- `logging-view` allows users to *view* namespaced `Flows` and `Outputs`, and `ClusterFlows` and `ClusterOutputs` + +:::note Why choose one role over the other? + +Edit access to `ClusterFlow` and `ClusterOutput` resources is powerful. Any user with it has edit access for all logs in the cluster. + +::: + +In Rancher, the cluster administrator role is the only role with full access to all `rancher-logging` resources. Cluster members are not able to edit or read any logging resources. Project owners and members have the following privileges: + +Project Owners | Project Members +--- | --- +able to create namespaced `Flows` and `Outputs` in their projects' namespaces | only able to view the `Flows` and `Outputs` in projects' namespaces +can collect logs from anything in their projects' namespaces | cannot collect any logs in their projects' namespaces + +Both project owners and project members require at least *one* namespace in their project to use logging. If they do not, then they may not see the logging button in the top nav dropdown. \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/logging/taints-and-tolerations.md b/versioned_docs/version-2.8/integrations-in-rancher/logging/taints-and-tolerations.md new file mode 100644 index 00000000000..c5cf0e35578 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/logging/taints-and-tolerations.md @@ -0,0 +1,65 @@ +--- +title: Working with Taints and Tolerations +--- + +"Tainting" a Kubernetes node causes pods to repel running on that node. + +Unless the pods have a `toleration` for that node's taint, they will run on other nodes in the cluster. + +[Taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) can work in conjunction with the `nodeSelector` [field](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) within the `PodSpec`, which enables the *opposite* effect of a taint. + +Using `nodeSelector` gives pods an affinity towards certain nodes. + +Both provide choice for the what node(s) the pod will run on. + +- [Default Implementation in Rancher's Logging Stack](#default-implementation-in-ranchers-logging-stack) +- [Adding NodeSelector Settings and Tolerations for Custom Taints](#adding-nodeselector-settings-and-tolerations-for-custom-taints) + + +### Default Implementation in Rancher's Logging Stack + +By default, Rancher taints all Linux nodes with `cattle.io/os=linux`, and does not taint Windows nodes. +The logging stack pods have `tolerations` for this taint, which enables them to run on Linux nodes. +Moreover, most logging stack pods run on Linux only and have a `nodeSelector` added to ensure they run on Linux nodes. + +This example Pod YAML file shows a nodeSelector being used with a toleration: + +```yaml +apiVersion: v1 +kind: Pod +# metadata... +spec: + # containers... + tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + nodeSelector: + kubernetes.io/os: linux +``` + +In the above example, we ensure that our pod only runs on Linux nodes, and we add a `toleration` for the taint we have on all of our Linux nodes. + +You can do the same with Rancher's existing taints, or with your own custom ones. + +### Adding NodeSelector Settings and Tolerations for Custom Taints + +If you would like to add your own `nodeSelector` settings, or if you would like to add `tolerations` for additional taints, you can pass the following to the chart's values. + +```yaml +tolerations: + # insert tolerations... +nodeSelector: + # insert nodeSelector... +``` + +These values will add both settings to the `fluentd`, `fluentbit`, and `logging-operator` containers. +Essentially, these are global settings for all pods in the logging stack. + +However, if you would like to add tolerations for *only* the `fluentbit` container, you can add the following to the chart's values. + +```yaml +fluentbit_tolerations: + # insert tolerations list for fluentbit containers only... +``` diff --git a/versioned_docs/version-2.8/integrations-in-rancher/longhorn.md b/versioned_docs/version-2.8/integrations-in-rancher/longhorn.md new file mode 100644 index 00000000000..a8870b4e3a8 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/longhorn.md @@ -0,0 +1,70 @@ +--- +title: Longhorn - Cloud native distributed block storage for Kubernetes +--- + +[Longhorn](https://longhorn.io/) is a lightweight, reliable, and easy-to-use distributed block storage system for Kubernetes. + +Longhorn is free, open source software. Originally developed by Rancher Labs, it is now being developed as a sandbox project of the Cloud Native Computing Foundation. It can be installed on any Kubernetes cluster with Helm, with kubectl, or with the Rancher UI. You can learn more about its architecture [here.](https://longhorn.io/docs/latest/concepts/) + +With Longhorn, you can: + +- Use Longhorn volumes as persistent storage for the distributed stateful applications in your Kubernetes cluster +- Partition your block storage into Longhorn volumes so that you can use Kubernetes volumes with or without a cloud provider +- Replicate block storage across multiple nodes and data centers to increase availability +- Store backup data in external storage such as NFS or AWS S3 +- Create cross-cluster disaster recovery volumes so that data from a primary Kubernetes cluster can be quickly recovered from backup in a second Kubernetes cluster +- Schedule recurring snapshots of a volume, and schedule recurring backups to NFS or S3-compatible secondary storage +- Restore volumes from backup +- Upgrade Longhorn without disrupting persistent volumes + +
Longhorn Dashboard
+ +![Longhorn Dashboard](/img/longhorn-screenshot.png) + +### Installing Longhorn with Rancher + +1. Fulfill all [Installation Requirements.](https://longhorn.io/docs/latest/deploy/install/#installation-requirements) +1. Go to the cluster where you want to install Longhorn. +1. Click **Apps**. +1. Click **Charts**. +1. Click **Longhorn**. +1. Optional: To customize the initial settings, click **Longhorn Default Settings** and edit the configuration. For help customizing the settings, refer to the [Longhorn documentation.](https://longhorn.io/docs/latest/references/settings/) +1. Click **Install**. + +**Result:** Longhorn is deployed in the Kubernetes cluster. + +### Accessing Longhorn from the Rancher UI + +1. Go to the cluster where Longhorn is installed. In the left navigation menu, click **Longhorn**. +1. On this page, you can edit Kubernetes resources managed by Longhorn. To view the Longhorn UI, click the **Longhorn** button in the **Overview** section. + +**Result:** You will be taken to the Longhorn UI, where you can manage your Longhorn volumes and their replicas in the Kubernetes cluster, as well as secondary backups of your Longhorn storage that may exist in another Kubernetes cluster or in S3. + +### Uninstalling Longhorn from the Rancher UI + +1. Go to the cluster where Longhorn is installed and click **Apps**. +1. Click **Installed Apps**. +1. Go to the `longhorn-system` namespace and check the boxes next to the `longhorn` and `longhorn-crd` apps. +1. Click **Delete,** and confirm **Delete**. + +**Result:** Longhorn is uninstalled. + +### GitHub Repository + +The Longhorn project is available [here.](https://github.com/longhorn/longhorn) + +### Documentation + +The Longhorn documentation is [here.](https://longhorn.io/docs/) + +### Architecture + +Longhorn creates a dedicated storage controller for each volume and synchronously replicates the volume across multiple replicas stored on multiple nodes. + +The storage controller and replicas are themselves orchestrated using Kubernetes. + +You can learn more about its architecture [here.](https://longhorn.io/docs/latest/concepts/) + +
Longhorn Architecture
+ +![Longhorn Architecture](/img/longhorn-architecture.svg) diff --git a/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md new file mode 100644 index 00000000000..9a464ba3ede --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md @@ -0,0 +1,117 @@ +--- +title: Built-in Dashboards +--- + +## Grafana UI + +[Grafana](https://grafana.com/grafana/) allows you to query, visualize, alert on and understand your metrics no matter where they are stored. Create, explore, and share dashboards with your team and foster a data driven culture. + +To see the default dashboards for time series data visualization, go to the Grafana UI. + +### Customizing Grafana + +To view and customize the PromQL queries powering the Grafana dashboard, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) + +### Persistent Grafana Dashboards + +To create a persistent Grafana dashboard, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) + +### Access to Grafana + +For information about role-based access control for Grafana, see [this section.](rbac-for-monitoring.md#role-based-access-control-for-grafana) + + +## Alertmanager UI + +When `rancher-monitoring` is installed, the Prometheus Alertmanager UI is deployed, allowing you to view your alerts and the current Alertmanager configuration. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [How Alertmanager Works.](how-monitoring-works.md#3-how-alertmanager-works) + +::: + +### Accessing the Alertmanager UI + +The Alertmanager UI lets you see the most recently fired alerts. + +:::note Prerequisite: + +The `rancher-monitoring` application must be installed. + +::: + +To see the Alertmanager UI, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the Alertmanager UI, click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Alertmanager**. + +**Result:** The Alertmanager UI opens in a new tab. For help with configuration, refer to the [official Alertmanager documentation.](https://prometheus.io/docs/alerting/latest/alertmanager/) + +For more information on configuring Alertmanager in Rancher, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +
The Alertmanager UI
+ +![Alertmanager UI](/img/alertmanager-ui.png) + + +### Viewing Default Alerts + +To see alerts that are fired by default, go to the Alertmanager UI and click **Expand all groups**. + + +## Prometheus UI + +By default, the [kube-state-metrics service](https://github.com/kubernetes/kube-state-metrics) provides a wealth of information about CPU and memory utilization to the monitoring application. These metrics cover Kubernetes resources across namespaces. This means that in order to see resource metrics for a service, you don't need to create a new ServiceMonitor for it. Because the data is already in the time series database, you can go to the Prometheus UI and run a PromQL query to get the information. The same query can be used to configure a Grafana dashboard to show a graph of those metrics over time. + +To see the Prometheus UI, install `rancher-monitoring`. Then: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the Prometheus UI and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Prometheus Graph**. + +
Prometheus Graph UI
+ +![Prometheus Graph UI](/img/prometheus-graph-ui.png) + +### Viewing the Prometheus Targets + +To see what services you are monitoring, you will need to see your targets. Targets are set up by ServiceMonitors and PodMonitors as sources to scrape metrics from. You won't need to directly edit targets, but the Prometheus UI can be useful for giving you an overview of all of the sources of metrics that are being scraped. + +To see the Prometheus Targets, install `rancher-monitoring`. Then: + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the Prometheus targets and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Prometheus Targets**. + +
Targets in the Prometheus UI
+ +![Prometheus Targets UI](/img/prometheus-targets-ui.png) + +### Viewing the PrometheusRules + +When you define a Rule (which is declared within a RuleGroup in a PrometheusRule resource), the [spec of the Rule itself](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule) contains labels that are used by Alertmanager to figure out which Route should receive a certain Alert. + +To see the PrometheusRules, install `rancher-monitoring`. Then: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Prometheus Rules**. + +You can also see the rules in the Prometheus UI: + +
Rules in the Prometheus UI
+ +![PrometheusRules UI](/img/prometheus-rules-ui.png) + +For more information on configuring PrometheusRules in Rancher, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) + +## Legacy UI + +For information on the dashboards available in v2.2 to v2.4 of Rancher, before the introduction of the `rancher-monitoring` application, see the [Rancher v2.0—v2.4 docs](../../../versioned_docs/version-2.0-2.4/explanations/integrations-in-rancher/cluster-monitoring/viewing-metrics.md). diff --git a/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md new file mode 100644 index 00000000000..de110cd3d80 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md @@ -0,0 +1,249 @@ +--- +title: How Monitoring Works +--- + +## 1. Architecture Overview + +_**The following sections describe how data flows through the Monitoring V2 application:**_ + +### Prometheus Operator + +Prometheus Operator observes ServiceMonitors, PodMonitors, and PrometheusRules being created. When the Prometheus configuration resources are created, Prometheus Operator calls the Prometheus API to sync the new configuration. As the diagram at the end of this section shows, the Prometheus Operator acts as the intermediary between Prometheus and Kubernetes, calling the Prometheus API to synchronize Prometheus with the monitoring-related resources in Kubernetes. + +### ServiceMonitors and PodMonitors + +ServiceMonitors and PodMonitors declaratively specify targets, such as Services and Pods, that need to be monitored. + +- Targets are scraped on a recurring schedule based on the configured Prometheus scrape interval, and the metrics that are scraped are stored into the Prometheus Time Series Database (TSDB). + +- In order to perform the scrape, ServiceMonitors and PodMonitors are defined with label selectors that determine which Services or Pods should be scraped and endpoints that determine how the scrape should happen on the given target, e.g., scrape/metrics in TCP 10252, proxying through IP addr x.x.x.x. + +- Out of the box, Monitoring V2 comes with certain pre-configured exporters that are deployed based on the type of Kubernetes cluster that it is deployed on. For more information, see [Scraping and Exposing Metrics](#5-scraping-and-exposing-metrics). + +### How PushProx Works + +- Certain internal Kubernetes components are scraped via a proxy deployed as part of Monitoring V2 called **PushProx**. The Kubernetes components that expose metrics to Prometheus through PushProx are the following: +`kube-controller-manager`, `kube-scheduler`, `etcd`, and `kube-proxy`. + +- For each PushProx exporter, we deploy one PushProx client onto all target nodes. For example, a PushProx client is deployed onto all controlplane nodes for kube-controller-manager, all etcd nodes for kube-etcd, and all nodes for kubelet. + +- We deploy exactly one PushProx proxy per exporter. The process for exporting metrics is as follows: + +1. The PushProx Client establishes an outbound connection with the PushProx Proxy. +1. The client then polls the proxy for scrape requests that have come into the proxy. +1. When the proxy receives a scrape request from Prometheus, the client sees it as a result of the poll. +1. The client scrapes the internal component. +1. The internal component responds by pushing metrics back to the proxy. + + +

Process for Exporting Metrics with PushProx:
+ +![Process for Exporting Metrics with PushProx](/img/pushprox-process.svg) + +### PrometheusRules + +PrometheusRules allow users to define rules for what metrics or time series database queries should result in alerts being fired. Rules are evaluated on an interval. + +- **Recording rules** create a new time series based on existing series that have been collected. They are frequently used to precompute complex queries. +- **Alerting rules** run a particular query and fire an alert from Prometheus if the query evaluates to a non-zero value. + +### Alert Routing + +Once Prometheus determines that an alert needs to be fired, alerts are forwarded to **Alertmanager**. + +- Alerts contain labels that come from the PromQL query itself and additional labels and annotations that can be provided as part of specifying the initial PrometheusRule. + +- Before receiving any alerts, Alertmanager will use the **routes** and **receivers** specified in its configuration to form a routing tree on which all incoming alerts are evaluated. Each node of the routing tree can specify additional grouping, labeling, and filtering that needs to happen based on the labels attached to the Prometheus alert. A node on the routing tree (usually a leaf node) can also specify that an alert that reaches it needs to be sent out to a configured Receiver, e.g., Slack, PagerDuty, SMS, etc. Note that Alertmanager will send an alert first to **alertingDriver**, then alertingDriver will send or forward alert to the proper destination. + +- Routes and receivers are also stored in the Kubernetes API via the Alertmanager Secret. When the Secret is updated, Alertmanager is also updated automatically. Note that routing occurs via labels only (not via annotations, etc.). + +
How data flows through the monitoring application:
+ + +## 2. How Prometheus Works + +### Storing Time Series Data + +After collecting metrics from exporters, Prometheus stores the time series in a local on-disk time series database. Prometheus optionally integrates with remote systems, but `rancher-monitoring` uses local storage for the time series database. + +Once stored, users can query this TSDB using PromQL, the query language for Prometheus. + +PromQL queries can be visualized in one of two ways: + +1. By supplying the query in Prometheus's Graph UI, which will show a simple graphical view of the data. +1. By creating a Grafana Dashboard that contains the PromQL query and additional formatting directives that label axes, add units, change colors, use alternative visualizations, etc. + +### Defining Rules for Prometheus + +Rules define queries that Prometheus needs to execute on a regular `evaluationInterval` to perform certain actions, such as firing an alert (alerting rules) or precomputing a query based on others existing in its TSDB (recording rules). These rules are encoded in PrometheusRules custom resources. When PrometheusRule custom resources are created or updated, the Prometheus Operator observes the change and calls the Prometheus API to synchronize the set of rules that Prometheus is currently evaluating on a regular interval. + +A PrometheusRule allows you to define one or more RuleGroups. Each RuleGroup consists of a set of Rule objects that can each represent either an alerting or a recording rule with the following fields: + +- The name of the new alert or record +- A PromQL expression for the new alert or record +- Labels that should be attached to the alert or record that identify it (e.g. cluster name or severity) +- Annotations that encode any additional important pieces of information that need to be displayed on the notification for an alert (e.g. summary, description, message, runbook URL, etc.). This field is not required for recording rules. + +On evaluating a [rule](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#rule), Prometheus will execute the provided PromQL query, add additional provided labels (or annotations - only for alerting rules), and execute the appropriate action for the rule. For example, an Alerting Rule that adds `team: front-end` as a label to the provided PromQL query will append that label to the fired alert, which will allow Alertmanager to forward the alert to the correct Receiver. + +### Alerting and Recording Rules + +Prometheus doesn't maintain the state of whether alerts are active. It fires alerts repetitively at every evaluation interval, relying on Alertmanager to group and filter the alerts into meaningful notifications. + +The `evaluation_interval` constant defines how often Prometheus evaluates its alerting rules against the time series database. Similar to the `scrape_interval`, the `evaluation_interval` also defaults to one minute. + +The rules are contained in a set of rule files. Rule files include both alerting rules and recording rules, but only alerting rules result in alerts being fired after their evaluation. + +For recording rules, Prometheus runs a query, then stores it as a time series. This synthetic time series is useful for storing the results of an expensive or time-consuming query so that it can be queried more quickly in the future. + +Alerting rules are more commonly used. Whenever an alerting rule evaluates to a positive number, Prometheus fires an alert. + +The Rule file adds labels and annotations to alerts before firing them, depending on the use case: + +- Labels indicate information that identifies the alert and could affect the routing of the alert. For example, if when sending an alert about a certain container, the container ID could be used as a label. + +- Annotations denote information that doesn't affect where an alert is routed, for example, a runbook or an error message. + +## 3. How Alertmanager Works + +The Alertmanager handles alerts sent by client applications such as the Prometheus server. It takes care of the following tasks: + +- Deduplicating, grouping, and routing alerts to the correct receiver integration such as email, PagerDuty, or OpsGenie + +- Silencing and inhibition of alerts + +- Tracking alerts that fire over time + +- Sending out the status of whether an alert is currently firing, or if it is resolved + +### Alerts Forwarded by alertingDrivers + +When alertingDrivers are installed, this creates a `Service` that can be used as the receiver's URL for Teams or SMS, based on the alertingDriver's configuration. The URL in the Receiver points to the alertingDrivers; so the Alertmanager sends alert first to alertingDriver, then alertingDriver forwards or sends alert to the proper destination. + +### Routing Alerts to Receivers + +Alertmanager coordinates where alerts are sent. It allows you to group alerts based on labels and fire them based on whether certain labels are matched. One top-level route accepts all alerts. From there, Alertmanager continues routing alerts to receivers based on whether they match the conditions of the next route. + +While the Rancher UI forms only allow editing a routing tree that is two levels deep, you can configure more deeply nested routing structures by editing the Alertmanager Secret. + +### Configuring Multiple Receivers + +By editing the forms in the Rancher UI, you can set up a Receiver resource with all the information Alertmanager needs to send alerts to your notification system. + +By editing custom YAML in the Alertmanager or Receiver configuration, you can also send alerts to multiple notification systems. For more information, see the section on configuring [Receivers.](../../reference-guides/monitoring-v2-configuration/receivers.md#configuring-multiple-receivers) + +## 4. Monitoring V2 Specific Components + +Prometheus Operator introduces a set of [Custom Resource Definitions](https://github.com/prometheus-operator/prometheus-operator#customresourcedefinitions) that allow users to deploy and manage Prometheus and Alertmanager instances by creating and modifying those custom resources on a cluster. + +Prometheus Operator will automatically update your Prometheus configuration based on the live state of the resources and configuration options that are edited in the Rancher UI. + +### Resources Deployed by Default + +By default, a set of resources curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project are deployed onto your cluster as part of installing the Rancher Monitoring Application to set up a basic Monitoring/Alerting stack. + +The resources that get deployed onto your cluster to support this solution can be found in the [`rancher-monitoring`](https://github.com/rancher/charts/tree/main/charts/rancher-monitoring) Helm chart, which closely tracks the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community with certain changes tracked in the [CHANGELOG.md](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/CHANGELOG.md). + +### Default Exporters + +Monitoring V2 deploys three default exporters that provide additional metrics for Prometheus to store: + +1. `node-exporter`: exposes hardware and OS metrics for Linux hosts. For more information on `node-exporter`, refer to the [upstream documentation](https://prometheus.io/docs/guides/node-exporter/). + +1. `windows-exporter`: exposes hardware and OS metrics for Windows hosts (only deployed on Windows clusters). For more information on `windows-exporter`, refer to the [upstream documentation](https://github.com/prometheus-community/windows_exporter). + +1. `kube-state-metrics`: expose additional metrics that track the state of resources contained in the Kubernetes API (e.g., pods, workloads, etc.). For more information on `kube-state-metrics`, refer to the [upstream documentation](https://github.com/kubernetes/kube-state-metrics/tree/master/docs). + +ServiceMonitors and PodMonitors will scrape these exporters, as defined [here](#defining-what-metrics-are-scraped). Prometheus stores these metrics, and you can query the results via either Prometheus's UI or Grafana. + +See the [architecture](#1-architecture-overview) section for more information on recording rules, alerting rules, and Alertmanager. + +### Components Exposed in the Rancher UI + +When the monitoring application is installed, you will be able to edit the following components in the Rancher UI: + +| Component | Type of Component | Purpose and Common Use Cases for Editing | +|--------------|------------------------|---------------------------| +| ServiceMonitor | Custom resource | Sets up Kubernetes Services to scrape custom metrics from. Automatically updates the scrape configuration in the Prometheus custom resource. | +| PodMonitor | Custom resource | Sets up Kubernetes Pods to scrape custom metrics from. Automatically updates the scrape configuration in the Prometheus custom resource. | +| Receiver | Configuration block (part of Alertmanager) | Modifies information on where to send an alert (e.g., Slack, PagerDuty, etc.) and any necessary information to send the alert (e.g., TLS certs, proxy URLs, etc.). Automatically updates the Alertmanager custom resource. | +| Route | Configuration block (part of Alertmanager) | Modifies the routing tree that is used to filter, label, and group alerts based on labels and send them to the appropriate Receiver. Automatically updates the Alertmanager custom resource. | +| PrometheusRule | Custom resource | Defines additional queries that need to trigger alerts or define materialized views of existing series that are within Prometheus's TSDB. Automatically updates the Prometheus custom resource. | + +### PushProx + +PushProx allows Prometheus to scrape metrics across a network boundary, which prevents users from having to expose metrics ports for internal Kubernetes components on each node in a Kubernetes cluster. + +Since the metrics for Kubernetes components are generally exposed on the host network of nodes in the cluster, PushProx deploys a DaemonSet of clients that sit on the hostNetwork of each node and make an outbound connection to a single proxy that is sitting on the Kubernetes API. Prometheus can then be configured to proxy scrape requests through the proxy to each client, which allows it to scrape metrics from the internal Kubernetes components without requiring any inbound node ports to be open. + +Refer to [Scraping Metrics with PushProx](#scraping-metrics-with-pushprox) for more. + +## 5. Scraping and Exposing Metrics + +### Defining what Metrics are Scraped + +ServiceMonitors and PodMonitors define targets that are intended for Prometheus to scrape. The [Prometheus custom resource](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md#prometheus) tells Prometheus which ServiceMonitors or PodMonitors it should use to find out where to scrape metrics from. + +The Prometheus Operator observes the ServiceMonitors and PodMonitors. When it observes that they are created or updated, it calls the Prometheus API to update the scrape configuration in the Prometheus custom resource and keep it in sync with the scrape configuration in the ServiceMonitors or PodMonitors. This scrape configuration tells Prometheus which endpoints to scrape metrics from and how it will label the metrics from those endpoints. + +Prometheus scrapes all of the metrics defined in its scrape configuration at every `scrape_interval`, which is one minute by default. + +The scrape configuration can be viewed as part of the Prometheus custom resource that is exposed in the Rancher UI. + +### How the Prometheus Operator Sets up Metrics Scraping + +The Prometheus Deployment or StatefulSet scrapes metrics, and the configuration of Prometheus is controlled by the Prometheus custom resources. The Prometheus Operator watches for Prometheus and Alertmanager resources, and when they are created, the Prometheus Operator creates a Deployment or StatefulSet for Prometheus or Alertmanager with the user-defined configuration. + +When the Prometheus Operator observes ServiceMonitors, PodMonitors, and PrometheusRules being created, it knows that the scrape configuration needs to be updated in Prometheus. It updates Prometheus by first updating the configuration and rules files in the volumes of Prometheus's Deployment or StatefulSet. Then it calls the Prometheus API to sync the new configuration, resulting in the Prometheus Deployment or StatefulSet to be modified in place. + +### How Kubernetes Component Metrics are Exposed + +Prometheus scrapes metrics from deployments known as [exporters,](https://prometheus.io/docs/instrumenting/exporters/) which export the time series data in a format that Prometheus can ingest. In Prometheus, time series consist of streams of timestamped values belonging to the same metric and the same set of labeled dimensions. + +### Scraping Metrics with PushProx + +Certain internal Kubernetes components are scraped via a proxy deployed as part of Monitoring V2 called PushProx. For detailed information on PushProx, refer [here](#how-pushprox-works) and to the above [architecture](#1-architecture-overview) section. + +### Scraping Metrics + +The following Kubernetes components are directly scraped by Prometheus: + +- kubelet* +- ingress-nginx** +- coreDns/kubeDns +- kube-api-server + +\* You can optionally use `hardenedKubelet.enabled` to use a PushProx, but that is not the default. + +** For RKE and RKE2 clusters, ingress-nginx is deployed by default and treated as an internal Kubernetes component. + + +### Scraping Metrics Based on Kubernetes Distribution + +Metrics are scraped differently based on the Kubernetes distribution. For help with terminology, refer [here](#terminology). For details, see the table below: + +
How Metrics are Exposed to Prometheus
+ +| Kubernetes Component | RKE | RKE2 | KubeADM | K3s | +|-----|-----|-----|-----|-----| +| kube-controller-manager | rkeControllerManager.enabled |rke2ControllerManager.enabled | kubeAdmControllerManager.enabled | k3sServer.enabled | +| kube-scheduler | rkeScheduler.enabled | rke2Scheduler.enabled |kubeAdmScheduler.enabled | k3sServer.enabled | +| etcd | rkeEtcd.enabled | rke2Etcd.enabled | kubeAdmEtcd.enabled | Not available | +| kube-proxy | rkeProxy.enabled | rke2Proxy.enabled | kubeAdmProxy.enabled | k3sServer.enabled | +| kubelet | Collects metrics directly exposed by kubelet | Collects metrics directly exposed by kubelet | Collects metrics directly exposed by kubelet | Collects metrics directly exposed by kubelet | +| ingress-nginx* | Collects metrics directly exposed by kubelet, exposed by rkeIngressNginx.enabled | Collects metrics directly exposed by kubelet, Exposed by rke2IngressNginx.enabled | Not available | Not available | +| coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | +| kube-api-server | Collects metrics directly exposed by kube-api-server |Collects metrics directly exposed by kube-api-server | Collects metrics directly exposed by kube-appi-server | Collects metrics directly exposed by kube-api-server | + +\* For RKE and RKE2 clusters, ingress-nginx is deployed by default and treated as an internal Kubernetes component. + +### Terminology + +- **kube-scheduler:** The internal Kubernetes component that uses information in the pod spec to decide on which node to run a pod. +- **kube-controller-manager:** The internal Kubernetes component that is responsible for node management (detecting if a node fails), pod replication and endpoint creation. +- **etcd:** The internal Kubernetes component that is the distributed key/value store which Kubernetes uses for persistent storage of all cluster information. +- **kube-proxy:** The internal Kubernetes component that watches the API server for pods/services changes in order to maintain the network up to date. +- **kubelet:** The internal Kubernetes component that watches the API server for pods on a node and makes sure they are running. +- **ingress-nginx:** An Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer. +- **coreDns/kubeDns:** The internal Kubernetes component responsible for DNS. +- **kube-api-server:** The main internal Kubernetes component that is responsible for exposing APIs for the other master components. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md new file mode 100644 index 00000000000..e301620d92a --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md @@ -0,0 +1,365 @@ +--- +title: PromQL Expression Reference +--- + +The PromQL expressions in this doc can be used to configure alerts. + +For more information about querying the Prometheus time series database, refer to the official [Prometheus documentation.](https://prometheus.io/docs/prometheus/latest/querying/basics/) + + +## Cluster Metrics + +### Cluster CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance))` | +| Summary | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])))` | + +### Cluster Load Average + +| Catalog | Expression | +| --- | --- | +| Detail |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
| +| Summary |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"})`
| + +### Cluster Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `1 - sum(node_memory_MemAvailable_bytes) by (instance) / sum(node_memory_MemTotal_bytes) by (instance)` | +| Summary | `1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes)` | + +### Cluster Disk Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `(sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance) - sum(node_filesystem_free_bytes{device!="rootfs"}) by (instance)) / sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance)` | +| Summary | `(sum(node_filesystem_size_bytes{device!="rootfs"}) - sum(node_filesystem_free_bytes{device!="rootfs"})) / sum(node_filesystem_size_bytes{device!="rootfs"})` | + +### Cluster Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(node_disk_read_bytes_total[5m])) by (instance)`
written`sum(rate(node_disk_written_bytes_total[5m])) by (instance)`
| +| Summary |
read`sum(rate(node_disk_read_bytes_total[5m]))`
written`sum(rate(node_disk_written_bytes_total[5m]))`
| + +### Cluster Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| Summary |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +### Cluster Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| Summary |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +## Node Metrics + +### Node CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `avg(irate(node_cpu_seconds_total{mode!="idle", instance=~"$instance"}[5m])) by (mode)` | +| Summary | `1 - (avg(irate(node_cpu_seconds_total{mode="idle", instance=~"$instance"}[5m])))` | + +### Node Load Average + +| Catalog | Expression | +| --- | --- | +| Detail |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| +| Summary |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| + +### Node Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"})` | +| Summary | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"}) ` | + +### Node Disk Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"}) by (device)) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device)` | +| Summary | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"})) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"})` | + +### Node Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| +| Summary |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| + +### Node Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| Summary |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +### Node Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| Summary |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +## Etcd Metrics + +### Etcd Has a Leader + +`max(etcd_server_has_leader)` + +### Number of Times the Leader Changes + +`max(etcd_server_leader_changes_seen_total)` + +### Number of Failed Proposals + +`sum(etcd_server_proposals_failed_total)` + +### GRPC Client Traffic + +| Catalog | Expression | +| --- | --- | +| Detail |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m])) by (instance)`
| +| Summary |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m]))`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m]))`
| + +### Peer Traffic + +| Catalog | Expression | +| --- | --- | +| Detail |
in`sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)`
| +| Summary |
in`sum(rate(etcd_network_peer_received_bytes_total[5m]))`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m]))`
| + +### DB Size + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(etcd_debugging_mvcc_db_total_size_in_bytes) by (instance)` | +| Summary | `sum(etcd_debugging_mvcc_db_total_size_in_bytes)` | + +### Active Streams + +| Catalog | Expression | +| --- | --- | +| Detail |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance)`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance)`
| +| Summary |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})`
| + +### Raft Proposals + +| Catalog | Expression | +| --- | --- | +| Detail |
applied`sum(increase(etcd_server_proposals_applied_total[5m])) by (instance)`
committed`sum(increase(etcd_server_proposals_committed_total[5m])) by (instance)`
pending`sum(increase(etcd_server_proposals_pending[5m])) by (instance)`
failed`sum(increase(etcd_server_proposals_failed_total[5m])) by (instance)`
| +| Summary |
applied`sum(increase(etcd_server_proposals_applied_total[5m]))`
committed`sum(increase(etcd_server_proposals_committed_total[5m]))`
pending`sum(increase(etcd_server_proposals_pending[5m]))`
failed`sum(increase(etcd_server_proposals_failed_total[5m]))`
| + +### RPC Rate + +| Catalog | Expression | +| --- | --- | +| Detail |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m])) by (instance)`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m])) by (instance)`
| +| Summary |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m]))`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m]))`
| + +### Disk Operations + +| Catalog | Expression | +| --- | --- | +| Detail |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m])) by (instance)`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m])) by (instance)`
| +| Summary |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m]))`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m]))`
| + +### Disk Sync Duration + +| Catalog | Expression | +| --- | --- | +| Detail |
wal`histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))`
db`histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))`
| +| Summary |
wal`sum(histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)))`
db`sum(histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le)))`
| + +## Kubernetes Components Metrics + +### API Server Request Latency + +| Catalog | Expression | +| --- | --- | +| Detail | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance, verb) /1e+06` | +| Summary | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance) /1e+06` | + +### API Server Request Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(apiserver_request_count[5m])) by (instance, code)` | +| Summary | `sum(rate(apiserver_request_count[5m])) by (instance)` | + +### Scheduling Failed Pods + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(kube_pod_status_scheduled{condition="false"})` | +| Summary | `sum(kube_pod_status_scheduled{condition="false"})` | + +### Controller Manager Queue Depth + +| Catalog | Expression | +| --- | --- | +| Detail |
volumes`sum(volumes_depth) by instance`
deployment`sum(deployment_depth) by instance`
replicaset`sum(replicaset_depth) by instance`
service`sum(service_depth) by instance`
serviceaccount`sum(serviceaccount_depth) by instance`
endpoint`sum(endpoint_depth) by instance`
daemonset`sum(daemonset_depth) by instance`
statefulset`sum(statefulset_depth) by instance`
replicationmanager`sum(replicationmanager_depth) by instance`
| +| Summary |
volumes`sum(volumes_depth)`
deployment`sum(deployment_depth)`
replicaset`sum(replicaset_depth)`
service`sum(service_depth)`
serviceaccount`sum(serviceaccount_depth)`
endpoint`sum(endpoint_depth)`
daemonset`sum(daemonset_depth)`
statefulset`sum(statefulset_depth)`
replicationmanager`sum(replicationmanager_depth)`
| + +### Scheduler E2E Scheduling Latency + +| Catalog | Expression | +| --- | --- | +| Detail | `histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06` | +| Summary | `sum(histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06)` | + +### Scheduler Preemption Attempts + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(scheduler_total_preemption_attempts[5m])) by (instance)` | +| Summary | `sum(rate(scheduler_total_preemption_attempts[5m]))` | + +### Ingress Controller Connections + +| Catalog | Expression | +| --- | --- | +| Detail |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"}) by (instance)`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"}) by (instance)`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"}) by (instance)`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m]))) by (instance)`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m]))) by (instance)`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m]))) by (instance)`
| +| Summary |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"})`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"})`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"})`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m])))`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m])))`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m])))`
| + +### Ingress Controller Request Process Time + +| Catalog | Expression | +| --- | --- | +| Detail | `topk(10, histogram_quantile(0.95,sum by (le, host, path)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | +| Summary | `topk(10, histogram_quantile(0.95,sum by (le, host)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | + +## Rancher Logging Metrics + + +### Fluentd Buffer Queue Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_output_status_buffer_queue_length[5m])) by (instance)` | +| Summary | `sum(rate(fluentd_output_status_buffer_queue_length[5m]))` | + +### Fluentd Input Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)` | +| Summary | `sum(rate(fluentd_input_status_num_records_total[5m]))` | + +### Fluentd Output Errors Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_output_status_num_errors[5m])) by (type)` | +| Summary | `sum(rate(fluentd_output_status_num_errors[5m]))` | + +### Fluentd Output Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_output_status_num_records_total[5m])) by (instance)` | +| Summary | `sum(rate(fluentd_output_status_num_records_total[5m]))` | + +## Workload Metrics + +### Workload CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### Workload Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""}) by (pod_name)` | +| Summary | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""})` | + +### Workload Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### Workload Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### Workload Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +## Pod Metrics + +### Pod CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
| +| Summary |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
| + +### Pod Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""}) by (container_name)` | +| Summary | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""})` | + +### Pod Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| Summary |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| Summary |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
| +| Summary |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +## Container Metrics + +### Container CPU Utilization + +| Catalog | Expression | +| --- | --- | +| cfs throttled seconds | `sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| usage seconds | `sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| system seconds | `sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| user seconds | `sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | + +### Container Memory Utilization + +`sum(container_memory_working_set_bytes{namespace="$namespace",pod_name="$podName",container_name="$containerName"})` + +### Container Disk I/O + +| Catalog | Expression | +| --- | --- | +| read | `sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| write | `sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | diff --git a/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md new file mode 100644 index 00000000000..7993c1040c6 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md @@ -0,0 +1,251 @@ +--- +title: Role-based Access Control +--- +This section describes the expectations for RBAC for Rancher Monitoring. + + +## Cluster Admins + +By default, only those with the cluster-admin `ClusterRole` should be able to: + +- Install the `rancher-monitoring` App onto a cluster and all other relevant configuration performed on the chart deploy + - e.g. whether default dashboards are created, what exporters are deployed onto the cluster to collect metrics, etc. +- Create / modify / delete Prometheus deployments in the cluster via Prometheus CRs +- Create / modify / delete Alertmanager deployments in the cluster via Alertmanager CRs +- Persist new Grafana dashboards or datasources via creating ConfigMaps in the appropriate namespace +- Expose certain Prometheus metrics to the k8s Custom Metrics API for HPA via a Secret in the `cattle-monitoring-system` namespace + +## Users with Kubernetes ClusterRole-based Permissions + +The `rancher-monitoring` chart installs the following three `ClusterRoles`. By default, they aggregate into the corresponding k8s `ClusterRoles`: + +| ClusterRole | Aggregates To Default K8s ClusterRole | +| ------------------------------| ---------------------------| +| `monitoring-admin` | `admin`| +| `monitoring-edit` | `edit` | +| `monitoring-view` | `view ` | + +These `ClusterRoles` provide different levels of access to the Monitoring CRDs based on the actions that can be performed: + +| CRDs (monitoring.coreos.com) | Admin | Edit | View | +| ------------------------------| ---------------------------| ---------------------------| ---------------------------| +|
  • `prometheuses`
  • `alertmanagers`
| Get, List, Watch | Get, List, Watch | Get, List, Watch | +|
  • `servicemonitors`
  • `podmonitors`
  • `prometheusrules`
| * | * | Get, List, Watch | + +On a high level, the following permissions are assigned by default as a result. + +### Users with Kubernetes Admin/Edit Permissions + +Only those with the the cluster-admin, admin or edit `ClusterRole` should be able to: + +- Modify the scrape configuration of Prometheus deployments via ServiceMonitor and PodMonitor CRs +- Modify the alerting / recording rules of a Prometheus deployment via PrometheusRules CRs + +### Users with Kubernetes View Permissions + +Only those with who have some Kubernetes `ClusterRole` should be able to: + +- View the configuration of Prometheuses that are deployed within the cluster +- View the configuration of Alertmanagers that are deployed within the cluster +- View the scrape configuration of Prometheus deployments via ServiceMonitor and PodMonitor CRs +- View the alerting/recording rules of a Prometheus deployment via PrometheusRules CRs + +### Additional Monitoring Roles + +Monitoring also creates additional `Roles` that are not assigned to users by default but are created within the cluster. They can be bound to a namespace by deploying a `RoleBinding` that references it. To define a `RoleBinding` with `kubectl` instead of through Rancher, click [here](#assigning-roles-and-clusterroles-with-kubectl). + +Admins should use these roles to provide more fine-grained access to users: + +| Role | Purpose | +| ------------------------------| ---------------------------| +| monitoring-config-admin | Allow admins to assign roles to users to be able to view / modify Secrets and ConfigMaps within the cattle-monitoring-system namespace. Modifying Secrets / ConfigMaps in this namespace could allow users to alter the cluster's Alertmanager configuration, Prometheus Adapter configuration, additional Grafana datasources, TLS secrets, etc. | +| monitoring-config-edit | Allow admins to assign roles to users to be able to view / modify Secrets and ConfigMaps within the cattle-monitoring-system namespace. Modifying Secrets / ConfigMaps in this namespace could allow users to alter the cluster's Alertmanager configuration, Prometheus Adapter configuration, additional Grafana datasources, TLS secrets, etc. | +| monitoring-config-view | Allow admins to assign roles to users to be able to view Secrets and ConfigMaps within the cattle-monitoring-system namespace. Viewing Secrets / ConfigMaps in this namespace could allow users to observe the cluster's Alertmanager configuration, Prometheus Adapter configuration, additional Grafana datasources, TLS secrets, etc. | +| monitoring-dashboard-admin | Allow admins to assign roles to users to be able to edit / view ConfigMaps within the cattle-dashboards namespace. ConfigMaps in this namespace will correspond to Grafana Dashboards that are persisted onto the cluster. | +| monitoring-dashboard-edit | Allow admins to assign roles to users to be able to edit / view ConfigMaps within the cattle-dashboards namespace. ConfigMaps in this namespace will correspond to Grafana Dashboards that are persisted onto the cluster. | +| monitoring-dashboard-view | Allow admins to assign roles to users to be able to view ConfigMaps within the cattle-dashboards namespace. ConfigMaps in this namespace will correspond to Grafana Dashboards that are persisted onto the cluster. | + + +### Assigning Monitoring Roles through Custom Roles + +Admins may assign custom roles in the Rancher UI for admin, editing, and viewing monitoring. These "roles" are created by default when the monitoring app is installed. Additionally, these roles are also deployed to the corresponding Kubernetes roles: admin, edit, and view `ClusterRoles`. + +:::note Important + +The UI won't offer `monitoring-admin`, `monitoring-edit`, and `monitoring-view` options when users are being added to a cluster. These monitoring roles can only be assigned by manually creating a custom role that inherits from Project Owner and Project Monitoring View roles. + +::: + + 1. Create the custom role: + + 1.1 Click **☰ > Users & Authentication > Roles**. + + 1.2 Select the appropriate tab, e.g., **Cluster** role. Then click **Create Cluster Role**. + + 1.3 In the **Name** field, create a custom role such as `View Monitoring`, `Edit Monitoring`, or `Admin Monitoring`. + + 1.4 Click **Inherit From > Add Resource**, then select the Kubernetes role, as applicable, from the dropdown. + + 1.5 Click **Create**. + + + 2. Assign the custom role to a new user: + + 2.1 Click **☰ > Cluster Management > Cluster Explore > Cluster > Cluster Members > Add**. + + 2.2 Search for your new user name from **Select Member** options displayed. + + 2.3 Assign the new custom role from **Cluster Permissions** to the new user. + + 2.4 Click **Create**. + +**Result:** The new user should now be able to see the monitoring tools. + +### Additional Monitoring ClusterRoles + +Monitoring also creates additional `ClusterRoles` that aren't assigned to users by default but are created within the cluster. They aren't aggregated by default but can be bound to a namespace by deploying a `RoleBinding` or `ClusterRoleBinding` that references it. To define a `RoleBinding` with `kubectl` instead of through Rancher, click [here](#assigning-roles-and-clusterroles-with-kubectl). + +| Role | Purpose | +| ------------------------------| ---------------------------| +| monitoring-ui-view | _Available as of Monitoring v2 14.5.100+_ This ClusterRole allows users to view metrics graphs for the specified cluster in the Rancher UI. This is done by granting read-only access to external Monitoring UIs. Users with this role have permission to list the Prometheus, Alertmanager, and Grafana endpoints and make GET requests to Prometheus, Grafana, and Alertmanager UIs through the Rancher proxy. | + +### Assigning Roles and ClusterRoles with kubectl + +#### Using `kubectl create` + +One method is to use either `kubectl create clusterrolebinding` or `kubectl create rolebinding` to assign a `Role` or `ClusterRole`. This is shown in the following examples: + +- Assign to a specific user: + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx --namespace=my-namespace + ``` + + + +- Assign to all authenticated users: + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated --namespace=my-namespace + ``` + + + + +#### Using YAML Files + +Another method is to define bindings in YAML files that you create. You must first configure the `RoleBinding` or `ClusterRoleBinding` with a YAML file. Then, apply the configuration changes by running the `kubectl apply` command. + +- **Roles**: Below is an example YAML file to help you configure `RoleBindings` in Kubernetes. You'll need to fill in the name below. + +:::note + +Names are case-sensitive. + +::: + +```yaml +# monitoring-config-view-role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: monitoring-config-view + namespace: cattle-monitoring-system +roleRef: + kind: Role + name: monitoring-config-view + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: User + name: u-b4qkhsnliz # this can be found via `kubectl get users -A` + apiGroup: rbac.authorization.k8s.io +``` + +- **kubectl**: Below is an example of a `kubectl` command used to apply the binding you've created in the YAML file. Remember to fill in your YAML filename accordingly. + ```plain + kubectl apply -f monitoring-config-view-role-binding.yaml + ``` + +## Users with Rancher Based Permissions + +The relationship between the default roles deployed by Rancher (i.e. cluster-owner, cluster-member, project-owner, project-member), the default Kubernetes roles, and the roles deployed by the rancher-monitoring chart are detailed in the table below: + +
Default Rancher Permissions and Corresponding Kubernetes ClusterRoles
+ +| Rancher Role | Kubernetes Role | Monitoring ClusterRole / Role | ClusterRoleBinding or RoleBinding? | +| --------- | --------- | --------- | --------- | +| cluster-owner | cluster-admin | N/A | ClusterRoleBinding | +| cluster-member | admin | monitoring-admin | ClusterRoleBinding | +| project-owner | admin | monitoring-admin | RoleBinding within Project namespace | +| project-member | edit | monitoring-edit | RoleBinding within Project namespace | + +In addition to these default Roles, the following additional Rancher project roles can be applied to members of your Cluster to provide additional access to Monitoring. These Rancher Roles will be tied to ClusterRoles deployed by the Monitoring chart: + +
Non-default Rancher Permissions and Corresponding Kubernetes ClusterRoles
+ +| Rancher Role | Kubernetes ClusterRole | Available In Rancher From | Available in Monitoring v2 From | +|--------------------------|-------------------------------|-------|------| +| View Monitoring* | [monitoring-ui-view](#monitoring-ui-view) | 2.4.8+ | 9.4.204+ | + +\* A User bound to the **View Monitoring** Rancher Role only has permissions to access external Monitoring UIs if provided links to those UIs. In order to access the Monitoring Pane to get those links, the User must be a Project Member of at least one Project. + +### Differences in 2.5.x + +Users with the project-member or project-owners roles assigned will not be given access to either Prometheus or Grafana in Rancher 2.5.x since we only create Grafana or Prometheus on a cluster-level. + +In addition, while project owners will still be only able to add ServiceMonitors / PodMonitors that scrape resources within their project's namespace by default, PrometheusRules are not scoped to a single namespace / project. Therefore, any alert rules or recording rules created by project-owners within their project namespace will be applied across the entire cluster, although they will be unable to view / edit / delete any rules that were created outside the project's namespace. + +### Assigning Additional Access + +If cluster-admins would like to provide additional admin/edit access to users outside of the roles offered by the rancher-monitoring chart, the following table identifies the potential impact: + +|CRDs (monitoring.coreos.com) | Can it cause impact outside of a namespace / project? | Impact | +|----------------------------| ------| ----------------------------| +| `prometheuses`| Yes, this resource can scrape metrics from any targets across the entire cluster (unless the Operator itself is otherwise configured). | User will be able to define the configuration of new cluster-level Prometheus deployments that should be created in the cluster. | +| `alertmanagers`| No | User will be able to define the configuration of new cluster-level Alertmanager deployments that should be created in the cluster. Note: if you just want to allow users to configure settings like Routes and Receivers, you should just provide access to the Alertmanager Config Secret instead. | +|
  • `servicemonitors`
  • `podmonitors`
| No, not by default; this is configurable via `ignoreNamespaceSelectors` on the Prometheus CR. | User will be able to set up scrapes by Prometheus on endpoints exposed by Services / Pods within the namespace they are given this permission in. | +| `prometheusrules`| Yes, PrometheusRules are cluster-scoped. | User will be able to define alert or recording rules on Prometheus based on any series collected across the entire cluster. | + +| k8s Resources | Namespace | Can it cause impact outside of a namespace / project? | Impact | +|----------------------------| ------| ------| ----------------------------| +|
  • `secrets`
  • `configmaps`
| `cattle-monitoring-system` | Yes, Configs and Secrets in this namespace can impact the entire monitoring / alerting pipeline. | User will be able to create or edit Secrets / ConfigMaps such as the Alertmanager Config, Prometheus Adapter Config, TLS secrets, additional Grafana datasources, etc. This can have broad impact on all cluster monitoring / alerting. | +|
  • `secrets`
  • `configmaps`
| `cattle-dashboards` | Yes, Configs and Secrets in this namespace can create dashboards that make queries on all metrics collected at a cluster-level. | User will be able to create Secrets / ConfigMaps that persist new Grafana Dashboards only. | + +## Role-based Access Control for Grafana + +Rancher allows any users who are authenticated by Kubernetes and have access the Grafana service deployed by the Rancher Monitoring chart to access Grafana via the Rancher Dashboard UI. By default, all users who are able to access Grafana are given the [Viewer](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#viewer-role) role, which allows them to view any of the default dashboards deployed by Rancher. + +However, users can choose to log in to Grafana as an [Admin](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#admin-role) if necessary. The default Admin username and password for the Grafana instance will be `admin`/`prom-operator`, but alternative credentials can also be supplied on deploying or upgrading the chart. + +To see the Grafana UI, install `rancher-monitoring`. Then: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Grafana**. + +
Cluster Compute Resources Dashboard in Grafana
+ +![Cluster Compute Resources Dashboard in Grafana](/img/cluster-compute-resources-dashboard.png) + +
Default Dashboards in Grafana
+ +![Default Dashboards in Grafana](/img/grafana-default-dashboard.png) \ No newline at end of file diff --git a/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/windows-support.md b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/windows-support.md new file mode 100644 index 00000000000..6fa7e1d84d0 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/monitoring-and-alerting/windows-support.md @@ -0,0 +1,48 @@ +--- +title: Windows Cluster Support for Monitoring V2 +--- + +_Available as of v2.5.8_ + +Starting at Monitoring V2 14.5.100 (used by default in Rancher 2.5.8), Monitoring V2 can now be deployed on a Windows cluster and will scrape metrics from Windows nodes using [prometheus-community/windows_exporter](https://github.com/prometheus-community/windows_exporter) (previously named `wmi_exporter`). + +## Cluster Requirements + +Monitoring V2 for Windows can only scrape metrics from Windows hosts that have a minimum `wins` version of v0.1.0. To be able to fully deploy Monitoring V2 for Windows, all of your hosts must meet this requirement. + +If you provision a fresh RKE1 cluster in Rancher 2.5.8, your cluster should already meet this requirement. + +### Upgrading Existing Clusters to wins v0.1.0 + +If the cluster was provisioned before Rancher 2.5.8 (even if the current Rancher version is 2.5.8), you will not be able to successfully deploy Monitoring V2 for Windows until you upgrade the wins version on each host to at least v0.1.0. + +To facilitate this upgrade, Rancher 2.5.8 has released a brand new Helm chart called `rancher-wins-upgrader`. + +1. Deploy `rancher-wins-upgrader` with the following override: + ```yaml + # Masquerading bootstraps the wins-upgrader installation via + # a previously whitelisted process path since the normal install path, + # c:\etc\rancher\wins\wins-upgrade.exe is not normally whitelisted. + # In this case, we are using the previously whitelisted process + # path used by Monitoring V1. + masquerade: + enabled: true + as: c:\\etc\wmi-exporter\wmi-exporter.exe + ``` + :::note Note for Non-Default Windows Prefix Path: + + - If you set up the RKE cluster with a `cluster.yml` that has a non-default `win_prefix_path`, you will need to update the `masquerade.as` field with your prefix path in place of `c:\\`. + + - For example, if you have `win_prefix_path: 'c:\host\opt\'`, then you will need to set `as: c:\host\opt\etc\wmi-exporter\wmi-exporter.exe`. + + ::: + +2. Once all your hosts have been successfully upgraded, please ensure that you deploy the Helm chart once again with default values to avoid conflicts with the following settings: + ```yaml + masquerade: + enabled: false + ``` + +**Result:** The hosts are ready for Monitoring V2 to be installed. You may choose to uninstall the `rancher-wins-upgrader` chart or keep it in your cluster to facilitate future upgrades. + +For more information on how it can be used, please see the [README.md](https://github.com/rancher/wins/blob/master/charts/rancher-wins-upgrader/README.md) of the chart. diff --git a/versioned_docs/version-2.8/integrations-in-rancher/neuvector.md b/versioned_docs/version-2.8/integrations-in-rancher/neuvector.md new file mode 100644 index 00000000000..bac4c2c0849 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/neuvector.md @@ -0,0 +1,199 @@ +--- +title: NeuVector Integration +--- + +### NeuVector Integration in Rancher + +[NeuVector 5.x](https://open-docs.neuvector.com/) is an open-source container-centric security platform that is integrated with Rancher. NeuVector offers real-time compliance, visibility, and protection for critical applications and data during runtime. NeuVector provides a firewall, container process/file system monitoring, security auditing with CIS benchmarks, and vulnerability scanning. For more information on Rancher security, please see the [security documentation](../pages-for-subheaders/rancher-security.md). + +NeuVector can be enabled through a Helm chart that may be installed either through **Apps** or through the **Cluster Tools** button in the Rancher UI. Once the Helm chart is installed, users can easily [deploy and manage NeuVector clusters within Rancher](https://open-docs.neuvector.com/deploying/rancher#deploy-and-manage-neuvector-through-rancher-apps-marketplace). + +### Installing NeuVector with Rancher + +The Harvester Helm Chart is used to manage access to the NeuVector UI in Rancher where users can navigate directly to deploy and manage their NeuVector clusters. + +**To navigate to and install the NeuVector chart through Apps:** + +1. Click **☰ > Cluster Management**. +1. On the Clusters page, go to the cluster where you want to deploy NeuVector, and click **Explore**. +1. Go to **Apps > Charts**, and install **NeuVector** from the chart repo. +1. Different cluster types require different container runtimes. When configuring Helm chart values, go to the **Container Runtime** section, and select your runtime in accordance with the cluster type. Finally, click **Install** again. + +Some examples are as follows: + + - RKE1: `docker` + - K3s and RKE2: `k3scontainerd` + - AKS: `containerd` for v1.19 and up + - EKS: `docker` for v1.22 and below; `containerd` for v1.23 and up + - GKE: `containerd` (see the [Google docs](https://cloud.google.com/kubernetes-engine/docs/concepts/using-containerd) for more) + + :::note + + Only one container runtime engine may be selected at a time during installation. + + ::: + +**To navigate to and install the NeuVector chart through Cluster Tools:** + +1. Click **☰ > Cluster Management**. +1. On the Clusters page, go to the cluster where you want to deploy NeuVector, and click **Explore**. +1. Click on **Cluster Tools** at the bottom of the left navigation bar. +1. Repeat step 4 above to select your container runtime accordingly, then click **Install** again. + +### Accessing NeuVector from the Rancher UI + +1. Navigate to the cluster explorer of the cluster where NeuVector is installed. In the left navigation bar, click **NeuVector**. +1. Click the external link to go to the NeuVector UI. Once the link is selected, users must accept the `END USER LICENSE AGREEMENT` to access the NeuVector UI. + +### Uninstalling NeuVector from the Rancher UI + +**To uninstall from Apps:** + +1. Click **☰ > Cluster Management**. +1. Under **Apps**, click **Installed Apps**. +1. Under `cattle-neuvector-system`, select both the NeuVector app (and the associated CRD if desired), then click **Delete**. + +**To uninstall from Cluster Tools:** + +1. Click **☰ > Cluster Management**. +1. Click on **Cluster Tools** at the bottom-left of the screen, then click on the trash can icon under the NeuVector chart. Select `Delete the CRD associated with this app` if desired, then click **Delete**. + +### GitHub Repository + +The NeuVector project is available [here](https://github.com/neuvector/neuvector). + +### Documentation + +The NeuVector documentation is [here](https://open-docs.neuvector.com/). + +### Architecture + +The NeuVector security solution contains four types of security containers: Controllers, Enforcers, Managers, and Scanners. A special container called an All-in-One is also provided to combine the Controller, Enforcer, and Manager functions all in one container, primarily for Docker-native deployments. There is also an Updater which, when run, will update the CVE database. + +- **Controller:** Manages the NeuVector Enforcer container; provides REST APIs for the management console. +- **Enforcer:** Enforces security policies. +- **Manager:** Provides a web-UI and CLI console to manage the NeuVector platform. +- **All-in-One:** Includes the Controller, Enforcer, and Manager. +- **Scanner:** Performs the vulnerability and compliance scanning for images, containers, and nodes. +- **Updater:** Updates the CVE database for Neuvector (when run); redeploys scanner pods. + +
**NeuVector Security Containers:**
+ +![NeuVector Security Containers](/img/neuvector-security-containers.png) + +
**NeuVector Architecture:**
+ +![NeuVector Architecture](/img/neuvector-architecture.png) + +To learn more about NeuVector's architecture, please refer [here](https://open-docs.neuvector.com/basics/overview#architecture). + +### CPU and Memory Allocations + +Below are the minimum recommended computing resources for the NeuVector chart installation in a default deployment. Note that the resource limit is not set. + +| Container | CPU - Request | Memory - Request | +|------------|--------|---------| +| Controller | 3 (1GB 1vCPU needed per controller) | * +| Enforcer | On all nodes (500MB .5vCPU) | 1GB +| Manager | 1 (500MB .5vCPU) | * +| Scanner | 3 (100MB .5vCPU) | * + +\* Minimum 1GB of memory total required for Controller, Manager, and Scanner containers combined. + + +### Hardened Cluster Support - Calico and Canal + + + + + - All NeuVector components are deployable if PSP is set to true. + + You will need to set additional configuration for your hardened cluster environment as follows: + + 1. Click **☰ > Cluster Management**. + 1. Go to the cluster that you created and click **Explore**. + 1. In the left navigation bar, click **Apps**. + 1. Install (or upgrade to) NeuVector version `100.0.1+up2.2.2`. + + - Under **Edit Options** > **Other Configuration**, enable **Pod Security Policy** by checking the box (no other config needed): + + ![Enable PSP for RKE1 Hardened Cluster](/img/psp-nv-rke.png) + + 1. Click **Install** at the bottom-right to complete. + + + + + + - NeuVector components Controller and Enforcer are deployable if PSP is set to true. + + + **Applicable to NeuVector chart version 100.0.0+up2.2.0 only:** + + - For Manager, Scanner, and Updater components, additional configuration is required as shown below: + + ``` + kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' + kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' + kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' + ``` +
+ + You will need to set additional configuration for your hardened cluster environment. + + >**Note:** You must update your config in both RKE2 and K3s hardened clusters as shown below. + + 1. Click **☰ > Cluster Management**. + 1. Go to the cluster that you created and click **Explore**. + 1. In the left navigation bar, click **Apps**. + 1. Install (or upgrade to) NeuVector version `100.0.1+up2.2.2`. + + - Under **Edit Options** > **Other Configuration**, enable **Pod Security Policy** by checking the box. Note that you must also enter a value greater than `zero` for `Manager runAsUser ID`, `Scanner runAsUser ID`, and `Updater runAsUser ID`: + + ![Enable PSP for RKE2 and K3s Hardened Clusters](/img/psp-nv-rke2.png) + + 1. Click **Install** at the bottom-right to complete. + + + + + +### SELinux-enabled Cluster Support - Calico and Canal + +To enable SELinux on RKE2 clusters, follow the steps below: + +- NeuVector components Controller and Enforcer are deployable if PSP is set to true. + + +**Applicable to NeuVector chart version 100.0.0+up2.2.0 only:** + +- For Manager, Scanner, and Updater components, additional configuration is required as shown below: + +``` +kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' +``` + +### Cluster Support in an Air-Gapped Environment + +- All NeuVector components are deployable on a cluster in an air-gapped environment without any additional configuration needed. + + +### Support Limitations + +* Only admins and cluster owners are currently supported. + +* Fleet multi-cluster deployment is not supported. + +* NeuVector is not supported on a Windows cluster. + + +### Other Limitations + +* Currently, NeuVector feature chart installation fails when a NeuVector partner chart already exists. To work around this issue, uninstall the NeuVector partner chart and reinstall the NeuVector feature chart. + +* Sometimes when the controllers are not ready, the NeuVector UI is not accessible from the Rancher UI. During this time, controllers will try to restart, and it takes a few minutes for the controllers to be active. + +* Container runtime is not auto-detected for different cluster types when installing the NeuVector chart. To work around this, you can specify the runtime manually. + diff --git a/versioned_docs/version-2.8/integrations-in-rancher/opa-gatekeeper.md b/versioned_docs/version-2.8/integrations-in-rancher/opa-gatekeeper.md new file mode 100644 index 00000000000..26b49d6f891 --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/opa-gatekeeper.md @@ -0,0 +1,111 @@ +--- +title: OPA Gatekeeper +--- + +To ensure consistency and compliance, every organization needs the ability to define and enforce policies in its environment in an automated way. [OPA (Open Policy Agent)](https://www.openpolicyagent.org/) is a policy engine that facilitates policy-based control for cloud native environments. Rancher provides the ability to enable OPA Gatekeeper in Kubernetes clusters, and also installs a couple of built-in policy definitions, which are also called constraint templates. + +OPA provides a high-level declarative language that lets you specify policy as code and ability to extend simple APIs to offload policy decision-making. + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) is a project that provides integration between OPA and Kubernetes. OPA Gatekeeper provides: + +- An extensible, parameterized policy library. +- Native Kubernetes CRDs for instantiating the policy library, also called “constraints." +- Native Kubernetes CRDs for extending the policy library, also called "constraint templates." +- Audit functionality. + +To read more about OPA, please refer to the [official documentation.](https://www.openpolicyagent.org/docs/latest/) + +## How the OPA Gatekeeper Integration Works + +Kubernetes provides the ability to extend API server functionality via admission controller webhooks, which are invoked whenever a resource is created, updated or deleted. Gatekeeper is installed as a validating webhook and enforces policies defined by Kubernetes custom resource definitions. In addition to the admission control usage, Gatekeeper provides the capability to audit existing resources in Kubernetes clusters and mark current violations of enabled policies. + +OPA Gatekeeper is made available via Rancher's Helm system chart, and it is installed in a namespace named `gatekeeper-system.` + +## Enabling OPA Gatekeeper in a Cluster + +:::note + +In Rancher v2.5, the OPA Gatekeeper application was improved. The Rancher v2.4 feature can't be upgraded to the new version in Rancher v2.5. If you installed OPA Gatekeeper in Rancher v2.4, you will need to uninstall OPA Gatekeeper and its CRDs from the old UI, then reinstall it in Rancher v2.5. To uninstall the CRDs run the following command in the kubectl console `kubectl delete crd configs.config.gatekeeper.sh constrainttemplates.templates.gatekeeper.sh`. + +::: + +:::note Prerequisite: + +Only administrators and cluster owners can enable OPA Gatekeeper. + +::: + +The OPA Gatekeeper Helm chart can be installed from **Apps**. + +### Enabling OPA Gatekeeper + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to enable OPA Gatekeeper and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Charts** and click **OPA Gatekeeper**. +1. Click **Install**. + +**Result:** OPA Gatekeeper is deployed in your Kubernetes cluster. + +## Constraint Templates + +[Constraint templates](https://github.com/open-policy-agent/gatekeeper#constraint-templates) are Kubernetes custom resources that define the schema and Rego logic of the OPA policy to be applied by Gatekeeper. For more information on the Rego policy language, refer to the [official documentation.](https://www.openpolicyagent.org/docs/latest/policy-language/) + +When OPA Gatekeeper is enabled, Rancher installs some templates by default. + +To list the constraint templates installed in the cluster, go to the left side menu under OPA Gatekeeper and click on **Templates**. + +Rancher also provides the ability to create your own constraint templates by importing YAML definitions. + +## Creating and Configuring Constraints + +[Constraints](https://github.com/open-policy-agent/gatekeeper#constraints) are Kubernetes custom resources that define the scope of objects to which a specific constraint template applies to. The complete policy is defined by constraint templates and constraints together. + +:::note Prerequisite: + +OPA Gatekeeper must be enabled in the cluster. + +::: + +To list the constraints installed, go to the left side menu under OPA Gatekeeper, and click on **Constraints**. + +New constraints can be created from a constraint template. + +Rancher provides the ability to create a constraint by using a convenient form that lets you input the various constraint fields. + +The **Edit as yaml** option is also available to configure the the constraint's yaml definition. + +### Exempting Rancher's System Namespaces from Constraints + +When a constraint is created, ensure that it does not apply to any Rancher or Kubernetes system namespaces. If the system namespaces are not excluded, then it is possible to see many resources under them marked as violations of the constraint. + +To limit the scope of the constraint only to user namespaces, always specify these namespaces under the **Match** field of the constraint. + +Also, the constraint may interfere with other Rancher functionality and deny system workloads from being deployed. To avoid this, exclude all Rancher-specific namespaces from your constraints. + +## Enforcing Constraints in your Cluster + +When the **Enforcement Action** is **Deny,** the constraint is immediately enabled and will deny any requests that violate the policy defined. By default, the enforcement value is **Deny**. + +When the **Enforcement Action** is **Dryrun,** then any resources that violate the policy are only recorded under the constraint's status field. + +To enforce constraints, create a constraint using the form. In the **Enforcement Action** field, choose **Deny**. + +## Audit and Violations in your Cluster + +OPA Gatekeeper runs a periodic audit to check if any existing resource violates any enforced constraint. The audit-interval (default 300s) can be configured while installing Gatekeeper. + +On the Gatekeeper page, any violations of the defined constraints are listed. + +Also under **Constraints,** the number of violations of the constraint can be found. + +The detail view of each constraint lists information about the resource that violated the constraint. + +## Disabling Gatekeeper + +1. Navigate to the cluster's Dashboard view +1. On the left side menu, expand the cluster menu and click on **OPA Gatekeeper**. +1. Click the **⋮ > Disable**. + +**Result:** Upon disabling OPA Gatekeeper, all constraint templates and constraints will also be deleted. + diff --git a/versioned_docs/version-2.8/integrations-in-rancher/rancher-extensions.md b/versioned_docs/version-2.8/integrations-in-rancher/rancher-extensions.md new file mode 100644 index 00000000000..0a86dad7d2d --- /dev/null +++ b/versioned_docs/version-2.8/integrations-in-rancher/rancher-extensions.md @@ -0,0 +1,84 @@ +--- +title: Rancher Extensions +--- + +New in Rancher v2.7.0, Rancher introduces **extensions**. Extensions allow users, developers, partners, and customers to extend and enhance the Rancher UI. In addition, users can make changes and create enhancements to their UI functionality independent of Rancher releases. Extensions will enable users to build on top of Rancher to better tailor it to their respective environments. Note that users will also have the ability to update to new versions as well as roll back to a previous version. + +Extensions are Helm charts that can only be installed once into a cluster; therefore, these charts have been simplified and separated from the general Helm charts listed under **Apps**. + +Examples of built-in Rancher extensions are Fleet, Explorer, and Harvester. Examples of other extensions that use the Extensions API that can be manually added are Kubewarden and Elemental. + +## Prerequisites + +> You must log in as an admin in order to view and interact with the extensions management page. + +## Installing Extensions + +1. Click **☰ > Extensions** under **Configuration**. + +2. If not already installed in **Apps**, you must enable the extension operator by clicking the **Enable** button. + + - Click **OK** to add the Rancher extension repository if your installation is not air-gapped. Otherwise, uncheck the box to do so and click **OK**. + + ![Rancher extension repository](/img/add-rancher-extension-repo.png) + +3. On the **Extensions** page, click on the **Available** tab to select which extensions you want to install. + +:::info + +In v2.7.0, the built-in extensions will not be displayed under the **Available** tab. Therefore, you will need to manually add the desired repos to install extensions. We will update the community once these extensions have been pulled out to be available for selection. + +::: +
+ +4. If no extensions are showing as available, you may manually add repos as follows: + + 4.1. On the upper right of screen, click on **â‹® > Manage Repositories > Create**. + + 4.2. Add the desired repo name, making sure to also specify the Git Repo URL and the Git Branch. + + 4.3. Click **Create** in the lower right again to complete. + + ![Manage repositories](/img/manage-repos.png) + +5. Under the **Available** tab, click **Install** on the desired extension and version as in the example below. Note that you can easily update your extension as the button to **Update** will appear on the extension if one is available. + + ![Install Kubewarden](/img/install-kubewarden.png) + +6. Click the **Reload** page button that will appear after your extension successfully installs. Note that a logged-in user who has just installed an extension will not see a change to the UI **unless** they reload the page. + + ![Reload button](/img/reload-button.png) + +## Uninstalling Extensions + +There are two ways in which you can uninstall or disable your extensions: + +1. Under the **Installed** tab, click the **Uninstall** button on the extension you wish to remove. + + ![Uninstall extensions](/img/uninstall-extension.png) + +1. On the extensions management page, click **â‹® > Disable Extension Support**. This will disable all installed extensions. + + ![Disable extensions](/img/disable-extension-support.png) + +:::caution + +You must reload the page after disabling extensions or display issues may occur. + +::: + +## Rolling Back Extensions + +Under the **Installed** tab, click the **Rollback** button on the extension you wish to roll back. + +![Roll back extensions](/img/roll-back-extension.png) + +:::caution + +You must reload the page after rolling back extensions or display issues may occur. + +::: + +## Developing Extensions + +To learn how to develop your own extensions, refer to the official [Getting Started](https://rancher.github.io/dashboard/extensions/extensions-getting-started) guide. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/about-provisioning-drivers.md b/versioned_docs/version-2.8/pages-for-subheaders/about-provisioning-drivers.md new file mode 100644 index 00000000000..812197b3b3f --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/about-provisioning-drivers.md @@ -0,0 +1,51 @@ +--- +title: Provisioning Drivers +--- + + + + + +Drivers in Rancher allow you to manage which providers can be used to deploy [hosted Kubernetes clusters](set-up-clusters-from-hosted-kubernetes-providers.md) or [nodes in an infrastructure provider](use-new-nodes-in-an-infra-provider.md) to allow Rancher to deploy and manage Kubernetes. + +### Rancher Drivers + +With Rancher drivers, you can enable/disable existing built-in drivers that are packaged in Rancher. Alternatively, you can add your own driver if Rancher has not yet implemented it. + +There are two types of drivers within Rancher: + +* [Cluster Drivers](#cluster-drivers) +* [Node Drivers](#node-drivers) + +### Cluster Drivers + +Cluster drivers are used to provision [hosted Kubernetes clusters](set-up-clusters-from-hosted-kubernetes-providers.md), such as GKE, EKS, AKS, etc.. The availability of which cluster driver to display when creating a cluster is defined based on the cluster driver's status. Only `active` cluster drivers will be displayed as an option for creating clusters for hosted Kubernetes clusters. By default, Rancher is packaged with several existing cluster drivers, but you can also create custom cluster drivers to add to Rancher. + +By default, Rancher has activated several hosted Kubernetes cloud providers including: + +* [Amazon EKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +* [Google GKE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +* [Azure AKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) + +There are several other hosted Kubernetes cloud providers that are disabled by default, but are packaged in Rancher: + +* [Alibaba ACK](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +* [Huawei CCE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) +* [Tencent](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) + +### Node Drivers + +Node drivers are used to provision hosts, which Rancher uses to launch and manage Kubernetes clusters. A node driver is the same as a [Docker Machine driver](https://docs.docker.com/machine/drivers/). The availability of which node driver to display when creating node templates is defined based on the node driver's status. Only `active` node drivers will be displayed as an option for creating node templates. By default, Rancher is packaged with many existing Docker Machine drivers, but you can also create custom node drivers to add to Rancher. + +If there are specific node drivers that you don't want to show to your users, you would need to de-activate these node drivers. + +Rancher supports several major cloud providers, but by default, these node drivers are active and available for deployment: + +* [Amazon EC2](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) +* [Azure](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md) +* [Digital Ocean](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md) +* [vSphere](vsphere.md) + +There are several other node drivers that are disabled by default, but are packaged in Rancher: + +* [Harvester](../integrations-in-rancher/harvester.md#harvester-node-driver/), available in Rancher v2.6.1 diff --git a/versioned_docs/version-2.8/pages-for-subheaders/about-rke1-templates.md b/versioned_docs/version-2.8/pages-for-subheaders/about-rke1-templates.md new file mode 100644 index 00000000000..601a622a581 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/about-rke1-templates.md @@ -0,0 +1,130 @@ +--- +title: RKE Templates +--- + + + + + +RKE templates are designed to allow DevOps and security teams to standardize and simplify the creation of Kubernetes clusters. + +RKE is the [Rancher Kubernetes Engine,](https://rancher.com/docs/rke/latest/en/) which is the tool that Rancher uses to provision Kubernetes clusters. + +With Kubernetes increasing in popularity, there is a trend toward managing a larger number of smaller clusters. When you want to create many clusters, it’s more important to manage them consistently. Multi-cluster management comes with challenges to enforcing security and add-on configurations that need to be standardized before turning clusters over to end users. + +RKE templates help standardize these configurations. Regardless of whether clusters are created with the Rancher UI, the Rancher API, or an automated process, Rancher will guarantee that every cluster it provisions from an RKE template is uniform and consistent in the way it is produced. + +Admins control which cluster options can be changed by end users. RKE templates can also be shared with specific users and groups, so that admins can create different RKE templates for different sets of users. + +If a cluster was created with an RKE template, you can't change it to a different RKE template. You can only update the cluster to a new revision of the same template. + +You can [save the configuration of an existing cluster as an RKE template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#converting-an-existing-cluster-to-use-an-rke-template) Then the cluster's settings can only be changed if the template is updated. The new template can also be used to launch new clusters. + +The core features of RKE templates allow DevOps and security teams to: + +- Standardize cluster configuration and ensure that Rancher-provisioned clusters are created following best practices +- Prevent less technical users from making uninformed choices when provisioning clusters +- Share different templates with different sets of users and groups +- Delegate ownership of templates to users who are trusted to make changes to them +- Control which users can create templates +- Require users to create clusters from a template + +## Configurable Settings + +RKE templates can be created in the Rancher UI or defined in YAML format. They can define all the same parameters that can be specified when you use Rancher to provision custom nodes or nodes from an infrastructure provider: + +- Cloud provider options +- Pod security options +- Network providers +- Ingress controllers +- Network security configuration +- Network plugins +- Private registry URL and credentials +- Add-ons +- Kubernetes options, including configurations for Kubernetes components such as kube-api, kube-controller, kubelet, and services + +The [add-on section](#add-ons) of an RKE template is especially powerful because it allows a wide range of customization options. + +## Scope of RKE Templates + +RKE templates are supported for Rancher-provisioned clusters. The templates can be used to provision custom clusters or clusters that are launched by an infrastructure provider. + +RKE templates are for defining Kubernetes and Rancher settings. Node templates are responsible for configuring nodes. For tips on how to use RKE templates in conjunction with hardware, refer to [RKE Templates and Hardware](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md). + +RKE templates can be created from scratch to pre-define cluster configuration. They can be applied to launch new clusters, or templates can also be exported from existing running clusters. + +The settings of an existing cluster can be [saved as an RKE template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#converting-an-existing-cluster-to-use-an-rke-template) This creates a new template and binds the cluster settings to the template, so that the cluster can only be upgraded if the [template is updated](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#updating-a-template), and the cluster is upgraded to [use a newer version of the template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#upgrading-a-cluster-to-use-a-new-template-revision) The new template can also be used to create new clusters. + + +## Example Scenarios +When an organization has both basic and advanced Rancher users, administrators might want to give the advanced users more options for cluster creation, while restricting the options for basic users. + +These [example scenarios](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md) describe how an organization could use templates to standardize cluster creation. + +Some of the example scenarios include the following: + +- **Enforcing templates:** Administrators might want to [enforce one or more template settings for everyone](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#enforcing-a-template-setting-for-everyone) if they want all new Rancher-provisioned clusters to have those settings. +- **Sharing different templates with different users:** Administrators might give [different templates to basic and advanced users,](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#templates-for-basic-and-advanced-users) so that basic users can have more restricted options and advanced users can use more discretion when creating clusters. +- **Updating template settings:** If an organization's security and DevOps teams decide to embed best practices into the required settings for new clusters, those best practices could change over time. If the best practices change, [a template can be updated to a new revision](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#updating-templates-and-clusters-created-with-them) and clusters created from the template can [upgrade to the new version](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#upgrading-a-cluster-to-use-a-new-template-revision) of the template. +- **Sharing ownership of a template:** When a template owner no longer wants to maintain a template, or wants to share ownership of the template, this scenario describes how [template ownership can be shared.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#allowing-other-users-to-control-and-share-a-template) + +## Template Management + +When you create an RKE template, it is available in the Rancher UI from the **Cluster Management** view under **RKE Templates**. When you create a template, you become the template owner, which gives you permission to revise and share the template. You can share the RKE templates with specific users or groups, and you can also make it public. + +Administrators can turn on template enforcement to require users to always use RKE templates when creating a cluster. This allows administrators to guarantee that Rancher always provisions clusters with specific settings. + +RKE template updates are handled through a revision system. If you want to change or update a template, you create a new revision of the template. Then a cluster that was created with the older version of the template can be upgraded to the new template revision. + +In an RKE template, settings can be restricted to what the template owner chooses, or they can be open for the end user to select the value. The difference is indicated by the **Allow User Override** toggle over each setting in the Rancher UI when the template is created. + +For the settings that cannot be overridden, the end user will not be able to directly edit them. In order for a user to get different options of these settings, an RKE template owner would need to create a new revision of the RKE template, which would allow the user to upgrade and change that option. + +The documents in this section explain the details of RKE template management: + +- [Getting permission to create templates](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md) +- [Creating and revising templates](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md) +- [Enforcing template settings](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md#requiring-new-clusters-to-use-an-rke-template) +- [Overriding template settings](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md) +- [Sharing templates with cluster creators](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#sharing-templates-with-specific-users-or-groups) +- [Sharing ownership of a template](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#sharing-ownership-of-templates) + +An [example YAML configuration file for a template](../reference-guides/rke1-template-example-yaml.md) is provided for reference. + +## Applying Templates + +You can [create a cluster from a template](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#creating-a-cluster-from-an-rke-template) that you created, or from a template that has been [shared with you.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md) + +If the RKE template owner creates a new revision of the template, you can [upgrade your cluster to that revision.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#updating-a-cluster-created-with-an-rke-template) + +RKE templates can be created from scratch to pre-define cluster configuration. They can be applied to launch new clusters, or templates can also be exported from existing running clusters. + +You can [save the configuration of an existing cluster as an RKE template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#converting-an-existing-cluster-to-use-an-rke-template) Then the cluster's settings can only be changed if the template is updated. + +## Standardizing Hardware + +RKE templates are designed to standardize Kubernetes and Rancher settings. If you want to standardize your infrastructure as well, one option is to use RKE templates [in conjunction with other tools](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md). + +Another option is to use [cluster templates,](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md) which include node pool configuration options, but don't provide configuration enforcement. + +## YAML Customization + +If you define an RKE template as a YAML file, you can modify this [example RKE template YAML](../reference-guides/rke1-template-example-yaml.md). The YAML in the RKE template uses the same customization that Rancher uses when creating an RKE cluster, but since the YAML is located within the context of a Rancher provisioned cluster, you will need to nest the RKE template customization under the `rancher_kubernetes_engine_config` directive in the YAML. + +The RKE documentation also has [annotated](https://rancher.com/docs/rke/latest/en/example-yamls/) `cluster.yml` files that you can use for reference. + +For guidance on available options, refer to the RKE documentation on [cluster configuration.](https://rancher.com/docs/rke/latest/en/config-options/) + +### Add-ons + +The add-on section of the RKE template configuration file works the same way as the [add-on section of a cluster configuration file](https://rancher.com/docs/rke/latest/en/config-options/add-ons/). + +The user-defined add-ons directive allows you to either call out and pull down Kubernetes manifests or put them inline directly. If you include these manifests as part of your RKE template, Rancher will provision those in the cluster. + +Some things you could do with add-ons include: + +- Install applications on the Kubernetes cluster after it starts +- Install plugins on nodes that are deployed with a Kubernetes daemonset +- Automatically set up namespaces, service accounts, or role binding + +The RKE template configuration must be nested within the `rancher_kubernetes_engine_config` directive. To set add-ons, when creating the template, you will click **Edit as YAML**. Then use the `addons` directive to add a manifest, or the `addons_include` directive to set which YAML files are used for the add-ons. For more information on custom add-ons, refer to the [user-defined add-ons documentation.](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/about-the-api.md b/versioned_docs/version-2.8/pages-for-subheaders/about-the-api.md new file mode 100644 index 00000000000..3b39d7c2717 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/about-the-api.md @@ -0,0 +1,84 @@ +--- +title: API +--- + + + + + +## How to use the API + +The API has its own user interface accessible from a web browser. This is an easy way to see resources, perform actions, and see the equivalent cURL or HTTP request & response. To access it: + + + + +1. Click on your user avatar in the upper right corner. +1. Click **Account & API Keys**. +1. Under the **API Keys** section, find the **API Endpoint** field and click the link. The link will look something like `https:///v3`, where `` is the fully qualified domain name of your Rancher deployment. + + + + +Go to the URL endpoint at `https:///v3`, where `` is the fully qualified domain name of your Rancher deployment. + + + + +## Authentication + +API requests must include authentication information. Authentication is done with HTTP basic authentication using [API Keys](../reference-guides/user-settings/api-keys.md). API keys can create new clusters and have access to multiple clusters via `/v3/clusters/`. [Cluster and project roles](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) apply to these keys and restrict what clusters and projects the account can see and what actions they can take. + +By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. For details on how to invalidate them, refer to the [API tokens page](../reference-guides/about-the-api/api-tokens.md). + +## Making requests + +The API is generally RESTful but has several features to make the definition of everything discoverable by a client so that generic clients can be written instead of having to write specific code for every type of resource. For detailed info about the generic API spec, [see here](https://github.com/rancher/api-spec/blob/master/specification.md). + +- Every type has a Schema which describes: + - The URL to get to the collection of this type of resources + - Every field the resource can have, along with their type, basic validation rules, whether they are required or optional, etc. + - Every action that is possible on this type of resource, with their inputs and outputs (also as schemas). + - Every field that filtering is allowed on + - What HTTP verb methods are available for the collection itself, or for individual resources in the collection. + + +- So the theory is that you can load just the list of schemas and know everything about the API. This is in fact how the UI for the API works, it contains no code specific to Rancher itself. The URL to get Schemas is sent in every HTTP response as a `X-Api-Schemas` header. From there you can follow the `collection` link on each schema to know where to list resources, and other `links` inside of the returned resources to get any other information. + +- In practice, you will probably just want to construct URL strings. We highly suggest limiting this to the top-level to list a collection (`/v3/`) or get a specific resource (`/v3//`). Anything deeper than that is subject to change in future releases. + +- Resources have relationships between each other called links. Each resource includes a map of `links` with the name of the link and the URL to retrieve that information. Again you should `GET` the resource and then follow the URL in the `links` map, not construct these strings yourself. + +- Most resources have actions, which do something or change the state of the resource. To use these, send a HTTP `POST` to the URL in the `actions` map for the action you want. Some actions require input or produce output, see the individual documentation for each type or the schemas for specific information. + +- To edit a resource, send a HTTP `PUT` to the `links.update` link on the resource with the fields that you want to change. If the link is missing then you don't have permission to update the resource. Unknown fields and ones that are not editable are ignored. + +- To delete a resource, send a HTTP `DELETE` to the `links.remove` link on the resource. If the link is missing then you don't have permission to update the resource. + +- To create a new resource, HTTP `POST` to the collection URL in the schema (which is `/v3/`). + +## Filtering + +Most collections can be filtered on the server-side by common fields using HTTP query parameters. The `filters` map shows you what fields can be filtered on and what the filtered values were for the request you made. The API UI has controls to setup filtering and show you the appropriate request. For simple "equals" matches it's just `field=value`. Modifiers can be added to the field name, e.g. `field_gt=42` for "field is greater than 42". See the [API spec](https://github.com/rancher/api-spec/blob/master/specification.md#filtering) for full details. + +## Sorting + +Most collections can be sorted on the server-side by common fields using HTTP query parameters. The `sortLinks` map shows you what sorts are available, along with the URL to get the collection sorted by that. It also includes info about what the current response was sorted by, if specified. + +## Pagination + +API responses are paginated with a limit of 100 resources per page by default. This can be changed with the `limit` query parameter, up to a maximum of 1000, e.g. `/v3/pods?limit=1000`. The `pagination` map in collection responses tells you whether or not you have the full result set and has a link to the next page if you do not. + +## Capturing Rancher API Calls + +You can use browser developer tools to capture how the Rancher API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning an RKE cluster: + +1. In the Rancher UI, go to **Cluster Management** and click **Create.** +1. Click one of the cluster types. This example uses Digital Ocean. +1. Fill out the form with a cluster name and node template, but don't click **Create**. +1. You will need to open the developer tools before the cluster creation to see the API call being recorded. To open the tools, right-click on the Rancher UI and click **Inspect.** +1. In the developer tools, click the **Network** tab. +1. On the **Network** tab, make sure **Fetch/XHR** is selected. +1. In the Rancher UI, click **Create**. In the developer tools, you should see a new network request with the name `cluster?_replace=true`. +1. Right-click `cluster?_replace=true` and click **Copy > Copy as cURL.** +1. Paste the result into any text editor. You will be able to see the POST request, including the URL it was sent to, all of the headers, and the full body of the request. This command can be used to create a cluster from the command line. Note: The request should be stored in a safe place because it contains credentials. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/access-clusters.md b/versioned_docs/version-2.8/pages-for-subheaders/access-clusters.md new file mode 100644 index 00000000000..69d18515700 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/access-clusters.md @@ -0,0 +1,65 @@ +--- +title: Cluster Access +--- + + + + + +This section is about what tools can be used to access clusters managed by Rancher. + +For information on how to give users permission to access a cluster, see the section on [adding users to clusters.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) + +For more information on roles-based access control, see [this section.](manage-role-based-access-control-rbac.md) + +For information on how to set up an authentication system, see [this section.](authentication-config.md) + +## Clusters in Rancher UI + +There are several paths to view and manage clusters through the Rancher UI. + +### Clusters Page + +You can access the **Clusters** page from the **☰** menu: + +1. Click **☰**. +1. Select **Cluster Management**. + +You can also access the **Clusters** page by clicking the **Manage** button above the clusters table on the Rancher UI **Home** page. + +On the **Clusters** page, select **â** at the end of each row to view a submenu with the following options: + +* [Kubectl Shell](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) +* Download KubeConfig +* Copy KubeConfig to Clipboard +* Edit Config +* View YAML +* Download YAML + +### Cluster Dashboard + +On the **Clusters** page, select the **Explore** button at the end of each row to view that cluster's **Cluster Dashboard**. You can also view the dashboard by clicking the name of a cluster in the table, then clicking the **Explore** buttton on the **Cluster** page. + +The **Cluster Dashboard** is also accessible from the Rancher UI **Home** page, by clicking on the name of a cluster. + +You can also access the **Cluster Dashboard** from the **☰** in the top navigation bar: + +1. Click **☰**. +1. Select the name of a cluster from the **Explore Cluster** menu option. + +The **Cluster Dashboard** lists information about a specific cluster, such as number of nodes, memory usage, events, and resources. + +## kubectl + +You can use the Kubernetes command-line tool, [kubectl](https://kubernetes.io/docs/reference/kubectl/overview/), to manage your clusters. You have two options for using kubectl: + +- **Rancher kubectl shell:** Interact with your clusters by launching a kubectl shell available in the Rancher UI. This option requires no configuration actions on your part. For more information, see [Accessing Clusters with kubectl Shell](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md). +- **Terminal remote connection:** You can also interact with your clusters by installing [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your local desktop and then copying the cluster's kubeconfig file to your local `~/.kube/config` directory. For more information, see [Accessing Clusters with kubectl and a kubeconfig File](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md). + +## Rancher CLI + +You can control your clusters by downloading Rancher's own command-line interface, [Rancher CLI](cli-with-rancher.md). This CLI tool can interact directly with different clusters and projects or pass them `kubectl` commands. + +## Rancher API + +Finally, you can interact with your clusters over the Rancher API. Before you use the API, you must obtain an [API key](../reference-guides/user-settings/api-keys.md). To view the different resource fields and actions for an API object, open the API UI, which can be accessed by clicking on **View in API** for any Rancher UI object. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/advanced-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/advanced-configuration.md new file mode 100644 index 00000000000..87efa2a0f9e --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/advanced-configuration.md @@ -0,0 +1,19 @@ +--- +title: Advanced Configuration +--- + + + + + +### Alertmanager + +For information on configuring the Alertmanager custom resource, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### Prometheus + +For information on configuring the Prometheus custom resource, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) + +### PrometheusRules + +For information on configuring the Prometheus custom resource, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/advanced-user-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/advanced-user-guides.md new file mode 100644 index 00000000000..b5b3d241037 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/advanced-user-guides.md @@ -0,0 +1,11 @@ +--- +title: Advanced User Guides +--- + + + + + +Advanced user guides are "problem-oriented" docs in which users learn how to answer questions or solve problems. The major difference between these and the new user guides is that these guides are geared toward more experienced or advanced users who have more technical needs from their documentation. These users already have an understanding of Rancher and its functions. They know what they need to accomplish; they just need additional guidance to complete some more complex task they they have encountered while working. + +It should be noted that neither new user guides nor advanced user guides provide detailed explanations or discussions (these kinds of docs belong elsewhere). How-to guides focus on the action of guiding users through repeatable, effective steps to learn new skills, master some task, or overcome some problem. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/air-gapped-helm-cli-install.md b/versioned_docs/version-2.8/pages-for-subheaders/air-gapped-helm-cli-install.md new file mode 100644 index 00000000000..d6fbc09698f --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/air-gapped-helm-cli-install.md @@ -0,0 +1,34 @@ +--- +title: Air-Gapped Helm CLI Install +--- + + + + + +This section is about using the Helm CLI to install the Rancher server in an air gapped environment. An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +The installation steps differ depending on whether Rancher is installed on an RKE Kubernetes cluster, a K3s Kubernetes cluster, or a single Docker container. + +For more information on each installation option, refer to [this page.](installation-and-upgrade.md) + +Throughout the installation instructions, there will be _tabs_ for each installation option. + +:::note Important: + +If you install Rancher following the Docker installation guide, there is no upgrade path to transition your Docker Installation to a Kubernetes Installation. + +::: + +## Installation Outline + +1. [Set up infrastructure and private registry](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) +2. [Collect and publish images to your private registry](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md) +3. [Set up a Kubernetes cluster (Skip this step for Docker installations)](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md) +4. [Install Rancher](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md) + +## Upgrades + +To upgrade Rancher with Helm CLI in an air gap environment, follow [this procedure.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md) + +### [Next: Prepare your Node(s)](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/authentication-config.md b/versioned_docs/version-2.8/pages-for-subheaders/authentication-config.md new file mode 100644 index 00000000000..9bb89f46ee8 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/authentication-config.md @@ -0,0 +1,137 @@ +--- +title: Authentication Config +weight: 10 +--- + + + + + +One of the key features that Rancher adds to Kubernetes is centralized user authentication. This feature allows your users to use one set of credentials to authenticate with any of your Kubernetes clusters. + +This centralized user authentication is accomplished using the Rancher authentication proxy, which is installed along with the rest of Rancher. This proxy authenticates your users and forwards their requests to your Kubernetes clusters using a service account. + +## External vs. Local Authentication + +The Rancher authentication proxy integrates with the following external authentication services. + +| Auth Service | +| ------------------------------------------------------------------------------------------------ | +| [Microsoft Active Directory](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md) | +| [GitHub](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md) | +| [Microsoft Azure AD](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md) | +| [FreeIPA](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md) | +| [OpenLDAP](configure-openldap.md) | +| [Microsoft AD FS](configure-microsoft-ad-federation-service-saml.md) | +| [PingIdentity](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md) | +| [Keycloak (OIDC)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md) | +| [Keycloak (SAML)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md) | +| [Okta](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md) | +| [Google OAuth](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md) | +| [Shibboleth](configure-shibboleth-saml.md) | + +However, Rancher also provides [local authentication](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md). + +In most cases, you should use an external authentication service over local authentication, as external authentication allows user management from a central location. However, you may want a few local authentication users for managing Rancher under rare circumstances, such as if your external authentication provider is unavailable or undergoing maintenance. + +## Users and Groups + +Rancher relies on users and groups to determine who is allowed to log in to Rancher and which resources they can access. When authenticating with an external provider, groups are provided from the external provider based on the user. These users and groups are given specific roles to resources like clusters, projects, multi-cluster apps, and global DNS providers and entries. When you give access to a group, all users who are a member of that group in the authentication provider will be able to access the resource with the permissions that you've specified. For more information on roles and permissions, see [Role Based Access Control](manage-role-based-access-control-rbac.md). + +:::note + +Local authentication does not support creating or managing groups. + +::: + +For more information, see [Users and Groups](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md) + +## Scope of Rancher Authorization + +After you configure Rancher to allow sign on using an external authentication service, you should configure who should be allowed to log in and use Rancher. The following options are available: + +| Access Level | Description | +|----------------------------------------------|-------------| +| Allow any valid Users | _Any_ user in the authorization service can access Rancher. We generally discourage use of this setting! | +| Allow members of Clusters, Projects, plus Authorized Users and Organizations | Any user in the authorization service and any group added as a **Cluster Member** or **Project Member** can log in to Rancher. Additionally, any user in the authentication service or group you add to the **Authorized Users and Organizations** list may log in to Rancher. | +| Restrict access to only Authorized Users and Organizations | Only users in the authentication service or groups added to the Authorized Users and Organizations can log in to Rancher. | + +To set the Rancher access level for users in the authorization service, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. After setting up the configuration details for an auth provider, use the **Site Access** options to configure the scope of user authorization. The table above explains the access level for each option. +1. Optional: If you choose an option other than **Allow any valid Users,** you can add users to the list of authorized users and organizations by searching for them in the text field that appears. +1. Click **Save**. + +**Result:** The Rancher access configuration settings are applied. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## External Authentication Configuration and Principal Users + +Configuration of external authentication requires: + +- A local user assigned the administrator role, called hereafter the _local principal_. +- An external user that can authenticate with your external authentication service, called hereafter the _external principal_. + +Configuration of external authentication affects how principal users are managed within Rancher. Follow the list below to better understand these effects. + +1. Sign into Rancher as the local principal and complete configuration of external authentication. + + ![Sign In](/img/sign-in.png) + +2. Rancher associates the external principal with the local principal. These two users share the local principal's user ID. + + ![Principal ID Sharing](/img/principal-ID.png) + +3. After you complete configuration, Rancher automatically signs out the local principal. + + ![Sign Out Local Principal](/img/sign-out-local.png) + +4. Then, Rancher automatically signs you back in as the external principal. + + ![Sign In External Principal](/img/sign-in-external.png) + +5. Because the external principal and the local principal share an ID, no unique object for the external principal displays on the Users page. + + ![Sign In External Principal](/img/users-page.png) + +6. The external principal and the local principal share the same access rights. + +:::note Reconfiguring a previously set up auth provider + +If you need to reconfigure or disable then re-enable a provider that had been previously set up, ensure that the user who attempts to do so +is logged in to Rancher as an external user, not the local admin. + +::: + +## Disabling An Auth Provider + +When you disable an auth provider, Rancher deletes all resources associated with it, such as: +- Secrets. +- Global role bindings. +- Cluster role template bindings. +- Project role template bindings. +- External users associated with the provider, but who never logged in as local users to Rancher. + +As this operation may lead to a loss of many resources, you may want to add a safeguard on the provider. To ensure that this cleanup process doesn't run when the auth provider is disabled, add a special annotation to the corresponding auth config. + +For example, to add a safeguard to the Azure AD provider, annotate the `azuread` authconfig object: + +`kubectl annotate --overwrite authconfig azuread management.cattle.io/auth-provider-cleanup='user-locked'` + +Rancher won't perform cleanup until you set the annotation to `unlocked`. + +### Running Resource Cleanup Manually + +Rancher might retain resources from a previously disabled auth provider configuration in the local cluster, even after you configure another auth provider. For example, if you used Provider A, then disabled it and started using Provider B, when you upgrade to a new version of Rancher, you can manually trigger cleanup on resources configured by Provider A. + +To manually trigger cleanup for a disabled auth provider, add the `management.cattle.io/auth-provider-cleanup` annotation with the `unlocked` value to its auth config. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/authentication-permissions-and-global-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/authentication-permissions-and-global-configuration.md new file mode 100644 index 00000000000..509c7bdedec --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/authentication-permissions-and-global-configuration.md @@ -0,0 +1,85 @@ +--- +title: Authentication, Permissions and Global Settings +--- + + + + + +After installation, the [system administrator](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) should configure Rancher to configure authentication, authorization, security, default settings, security policies, drivers and global DNS entries. + +## First Log In + +After you log into Rancher for the first time, Rancher will prompt you for a **Rancher Server URL**.You should set the URL to the main entry point to the Rancher Server. When a load balancer sits in front a Rancher Server cluster, the URL should resolve to the load balancer. The system will automatically try to infer the Rancher Server URL from the IP address or host name of the host running the Rancher Server. This is only correct if you are running a single node Rancher Server installation. In most cases, therefore, you need to set the Rancher Server URL to the correct value yourself. + +:::danger + +After you set the Rancher Server URL, we do not support updating it. Set the URL with extreme care. + +::: + +## Authentication + +One of the key features that Rancher adds to Kubernetes is centralized user authentication. This feature allows to set up local users and/or connect to an external authentication provider. By connecting to an external authentication provider, you can leverage that provider's user and groups. + +For more information how authentication works and how to configure each provider, see [Authentication](authentication-config.md). + +## Authorization + +Within Rancher, each person authenticates as a _user_, which is a login that grants you access to Rancher. Once the user logs in to Rancher, their _authorization_, or their access rights within the system, is determined by the user's role. Rancher provides built-in roles to allow you to easily configure a user's permissions to resources, but Rancher also provides the ability to customize the roles for each Kubernetes resource. + +For more information how authorization works and how to customize roles, see [Roles Based Access Control (RBAC)](manage-role-based-access-control-rbac.md). + +## Pod Security Policies + +_Pod Security Policies_ (or PSPs) are objects that control security-sensitive aspects of pod specification, e.g. root privileges. If a pod does not meet the conditions specified in the PSP, Kubernetes will not allow it to start, and Rancher will display an error message. + +For more information how to create and use PSPs, see [Pod Security Policies](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md). + +## Provisioning Drivers + +Drivers in Rancher allow you to manage which providers can be used to provision [hosted Kubernetes clusters](set-up-clusters-from-hosted-kubernetes-providers.md) or [nodes in an infrastructure provider](use-new-nodes-in-an-infra-provider.md) to allow Rancher to deploy and manage Kubernetes. + +For more information, see [Provisioning Drivers](about-provisioning-drivers.md). + +## Adding Kubernetes Versions into Rancher + +With this feature, you can upgrade to the latest version of Kubernetes as soon as it is released, without upgrading Rancher. This feature allows you to easily upgrade Kubernetes patch versions (i.e. `v1.15.X`), but not intended to upgrade Kubernetes minor versions (i.e. `v1.X.0`) as Kubernetes tends to deprecate or add APIs between minor versions. + +The information that Rancher uses to provision [RKE clusters](launch-kubernetes-with-rancher.md) is now located in the Rancher Kubernetes Metadata. For details on metadata configuration and how to change the Kubernetes version used for provisioning RKE clusters, see [Rancher Kubernetes Metadata.](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md) + +Rancher Kubernetes Metadata contains Kubernetes version information which Rancher uses to provision [RKE clusters](launch-kubernetes-with-rancher.md). + +For more information on how metadata works and how to configure metadata config, see [Rancher Kubernetes Metadata](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md). + +## Global Settings + +Options that control certain global-level Rancher settings are available from the top navigation bar. + +Click **☰** in the top left corner, then select **Global Settings**, to view and configure the following settings: + +- **Settings**: Various Rancher defaults, such as the minimum length for a user's password (`password-min-length`). You should be cautious when modifying these settings, as invalid values may break your Rancher installation. +- **Feature Flags**: Rancher features that can be toggled on or off. Some of these flags are for [experimental features](#enabling-experimental-features). +- **Banners**: Elements you can add to fixed locations on the portal. For example, you can use these options to [set a custom banner](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md#fixed-banners) for users when they login to Rancher. +- **Branding**: Rancher UI design elements that you can [customize](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md). You can add a custom logo or favicon, and modify UI colors. +- **Performance**: Performance settings for the Rancher UI, such as incremental resource loading. +- **Home Links**: Links displayed on the Rancher UI **Home** page. You can modify visibility for the default links or add your own links. + +### Enabling Experimental Features + +Rancher includes some features that are experimental and/or disabled by default. Feature flags allow you to enable these features. For more information, refer to the section about [feature flags.](enable-experimental-features.md) + +### Global Configuration + +**Global Configuration** options aren't visible unless you activate the **legacy** [feature flag](enable-experimental-features.md). The **legacy** flag is disabled by default on fresh Rancher installs of v2.6 and later. If you upgrade from an earlier Rancher version, or activate the **legacy** feature flag on Rancher v2.6 and later, **Global Configuration** is available from the top navigation menu: + +1. Click **☰** in the top left corner. +1. Select **Global Configuration** from the **Legacy Apps**. + +The following features are available under **Global Configuration**: + +- **Catalogs** +- **Global DNS Entries** +- **Global DNS Providers** + +As these are legacy features, please see the Rancher v2.0—v2.4 docs on [catalogs](../../versioned_docs/version-2.0-2.4/pages-for-subheaders/helm-charts-in-rancher.md), [global DNS entries](../../versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/globaldns.md#adding-a-global-dns-entry), and [global DNS providers](../../versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/globaldns.md#editing-a-global-dns-provider) for more details. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/aws-cloud-marketplace.md b/versioned_docs/version-2.8/pages-for-subheaders/aws-cloud-marketplace.md new file mode 100644 index 00000000000..c487e1d6f92 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/aws-cloud-marketplace.md @@ -0,0 +1,36 @@ +--- +title: AWS Marketplace Integration +--- + + + + + +## Overview + +Rancher offers an integration with the AWS Marketplace which allows users to purchase a support contract with SUSE. This integration allows you easily adjust your support needs as you start to support more clusters. + +## Limitations + +- You must be running Rancher v2.6.7 or higher +- Rancher must be deployed with additional metrics enabled. +- Rancher must be installed on an EKS cluster. +- You must purchase at least one entitlement to Rancher support through AWS Marketplace. +- You may need additional setup to support proxy/airgap use cases. See the [prerequisites](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md) for more information. + +## How to Use + +1. Complete the [prerequisite steps](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md). +2. [Install the CSP Adapter](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md). + +## FAQ + +**Can I purchase support for more nodes later on?** + +Yes. Simply go to the AWS Marketplace entry that you used to initially purchase support and increase the number of entitlements. + +**Can I use multiple instances of Rancher in the same AWS account?** + +Yes. However, each cluster that Rancher is installed in will need to adhere to the prerequisites. + +In addition, keep in mind that a given entitlement can only be used by one Rancher management server at a time. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/backup-restore-and-disaster-recovery.md b/versioned_docs/version-2.8/pages-for-subheaders/backup-restore-and-disaster-recovery.md new file mode 100644 index 00000000000..19feeb770d0 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/backup-restore-and-disaster-recovery.md @@ -0,0 +1,100 @@ +--- +title: Backups and Disaster Recovery +keywords: [rancher backup restore, rancher backup and restore, backup restore rancher, rancher backup and restore rancher] +--- + + + + + +In this section, you'll learn how to create backups of Rancher, how to restore Rancher from backup, and how to migrate Rancher to a new Kubernetes cluster. + +The `rancher-backup` operator is used to backup and restore Rancher on any Kubernetes cluster. This application is a Helm chart, and it can be deployed through the Rancher **Apps** page, or by using the Helm CLI. The `rancher-backup` Helm chart is [here.](https://github.com/rancher/charts/tree/release-v2.6/charts/rancher-backup) + +The backup-restore operator needs to be installed in the local cluster, and only backs up the Rancher app. The backup and restore operations are performed only in the local Kubernetes cluster. + + +## Backup and Restore for Rancher installed with Docker + +For Rancher installed with Docker, refer to [this page](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) to perform backups and [this page](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) to perform restores. + +## How Backups and Restores Work + +The `rancher-backup` operator introduces three custom resources: Backups, Restores, and ResourceSets. The following cluster-scoped custom resource definitions are added to the cluster: + +- `backups.resources.cattle.io` +- `resourcesets.resources.cattle.io` +- `restores.resources.cattle.io` + +The ResourceSet defines which Kubernetes resources need to be backed up. The ResourceSet is not available to be configured in the Rancher UI because the values required to back up Rancher are predefined. This ResourceSet should not be modified. + +When a Backup custom resource is created, the `rancher-backup` operator calls the `kube-apiserver` to get the resources in the ResourceSet (specifically, the predefined `rancher-resource-set`) that the Backup custom resource refers to. + +The operator then creates the backup file in the .tar.gz format and stores it in the location configured in the Backup resource. + +When a Restore custom resource is created, the operator accesses the backup .tar.gz file specified by the Restore, and restores the application from that file. + +The Backup and Restore custom resources can be created in the Rancher UI, or by using `kubectl apply`. + +:::note + +Refer [here](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md#2-restore-from-backup-using-a-restore-custom-resource) for help on restoring an existing backup file into a v1.22 cluster in Rancher v2.6.3. + +::: + +## Installing the rancher-backup Operator + +The `rancher-backup` operator can be installed from the Rancher UI, or with the Helm CLI. In both cases, the `rancher-backup` Helm chart is installed on the Kubernetes cluster running the Rancher server. It is a cluster-admin only feature and available only for the **local** cluster. (*If you do not see `rancher-backup` in the Rancher UI, you may have selected the wrong cluster.*) + +:::note + +There is a known issue in Fleet that occurs after performing a restoration using the backup-restore-operator: Secrets used for clientSecretName and helmSecretName are not included in Fleet gitrepos. Refer [here](./fleet-gitops-at-scale.md#troubleshooting) for a workaround. + +::: + +### Installing rancher-backup with the Rancher UI + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. +1. In the left navigation bar, **Apps > Charts**. +1. Click **Rancher Backups**. +1. Click **Install**. +1. Optional: Configure the default storage location. For help, refer to the [configuration section.](../reference-guides/backup-restore-configuration/storage-configuration.md) +1. Click **Install**. + +**Result:** The `rancher-backup` operator is installed. + +From the **Cluster Dashboard,** you can see the `rancher-backup` operator listed under **Deployments**. + +To configure the backup app in Rancher, go to the left navigation menu and click **Rancher Backups**. + +### RBAC + +Only the rancher admins and the local cluster’s cluster-owner can: + +* Install the Chart +* See the navigation links for Backup and Restore CRDs +* Perform a backup or restore by creating a Backup CR and Restore CR respectively +* List backups/restores performed so far + +## Backing up Rancher + +A backup is performed by creating a Backup custom resource. For a tutorial, refer to [this page.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) + +## Restoring Rancher + +A restore is performed by creating a Restore custom resource. For a tutorial, refer to [this page.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md) + +## Migrating Rancher to a New Cluster + +A migration is performed by following [these steps.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +## Default Storage Location Configuration + +Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible or Minio object store. + +For information on configuring these options, refer to [this page.](../reference-guides/backup-restore-configuration/storage-configuration.md) + +### Example values.yaml for the rancher-backup Helm Chart + +The example [values.yaml file](../reference-guides/backup-restore-configuration/storage-configuration.md#example-valuesyaml-for-the-rancher-backup-helm-chart) can be used to configure the `rancher-backup` operator when the Helm CLI is used to install it. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/backup-restore-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/backup-restore-configuration.md new file mode 100644 index 00000000000..104584f741d --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/backup-restore-configuration.md @@ -0,0 +1,12 @@ +--- +title: Rancher Backup Configuration Reference +--- + + + + + +- [Backup configuration](../reference-guides/backup-restore-configuration/backup-configuration.md) +- [Restore configuration](../reference-guides/backup-restore-configuration/restore-configuration.md) +- [Storage location configuration](../reference-guides/backup-restore-configuration/storage-configuration.md) +- [Example Backup and Restore Custom Resources](../reference-guides/backup-restore-configuration/examples.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/best-practices.md b/versioned_docs/version-2.8/pages-for-subheaders/best-practices.md new file mode 100644 index 00000000000..7009f6cce70 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/best-practices.md @@ -0,0 +1,21 @@ +--- +title: Best Practices Guide +--- + + + + + +The purpose of this section is to consolidate best practices for Rancher implementations. This also includes recommendations for related technologies, such as Kubernetes, Docker, containers, and more. The objective is to improve the outcome of a Rancher implementation using the operational experience of Rancher and its customers. + +If you have any questions about how these might apply to your use case, please contact your Customer Success Manager or Support. + +Use the navigation bar on the left to find the current best practices for managing and deploying the Rancher Server. + +For more guidance on best practices, you can consult these resources: + +- [Security](rancher-security.md) +- [Rancher Blog](https://www.suse.com/c/rancherblog/) +- [Rancher Forum](https://forums.rancher.com/) +- [Rancher Users Slack](https://slack.rancher.io/) +- [Rancher Labs YouTube Channel - Online Meetups, Demos, Training, and Webinars](https://www.youtube.com/channel/UCh5Xtp82q8wjijP8npkVTBA/featured) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/checklist-for-production-ready-clusters.md b/versioned_docs/version-2.8/pages-for-subheaders/checklist-for-production-ready-clusters.md new file mode 100644 index 00000000000..f5816af3c48 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/checklist-for-production-ready-clusters.md @@ -0,0 +1,53 @@ +--- +title: Checklist for Production-Ready Clusters +--- + + + + + +In this section, we recommend best practices for creating the production-ready Kubernetes clusters that will run your apps and services. + +For a list of requirements for your cluster, including the requirements for OS/Docker, hardware, and networking, refer to the section on [node requirements.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) + +This is a shortlist of best practices that we strongly recommend for all production clusters. + +For a full list of all the best practices that we recommend, refer to the [best practices section.](best-practices.md) + +### Node Requirements + +* Make sure your nodes fulfill all of the [node requirements,](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) including the port requirements. + +### Back up etcd + +* Enable etcd snapshots. Verify that snapshots are being created, and run a disaster recovery scenario to verify the snapshots are valid. etcd is the location where the state of your cluster is stored, and losing etcd data means losing your cluster. Make sure you configure recurring snapshots of etcd for your cluster(s), and make sure the snapshots are stored externally (off the node) as well. + +### Cluster Architecture + +* Nodes should have one of the following role configurations: + * `etcd` + * `controlplane` + * `etcd` and `controlplane` + * `worker` (the `worker` role should not be used or added on nodes with the `etcd` or `controlplane` role) +* Have at least three nodes with the role `etcd` to survive losing one node. Increase this count for higher node fault toleration, and spread them across (availability) zones to provide even better fault tolerance. +* Assign two or more nodes the `controlplane` role for master component high availability. +* Assign two or more nodes the `worker` role for workload rescheduling upon node failure. + +For more information on what each role is used for, refer to the [section on roles for nodes in Kubernetes.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md) + +For more information about the +number of nodes for each Kubernetes role, refer to the section on [recommended architecture.](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) + +### Logging and Monitoring + +* Configure alerts/notifiers for Kubernetes components (System Service). +* Configure logging for cluster analysis and post-mortems. + +### Reliability + +* Perform load tests on your cluster to verify that its hardware can support your workloads. + +### Networking + +* Minimize network latency. Rancher recommends minimizing latency between the etcd nodes. The default setting for `heartbeat-interval` is `500`, and the default setting for `election-timeout` is `5000`. These [settings for etcd tuning](https://coreos.com/etcd/docs/latest/tuning.html) allow etcd to run in most networks (except really high latency networks). +* Cluster nodes should be located within a single region. Most cloud providers provide multiple availability zones within a region, which can be used to create higher availability for your cluster. Using multiple availability zones is fine for nodes with any role. If you are using [Kubernetes Cloud Provider](set-up-cloud-providers.md) resources, consult the documentation for any restrictions (i.e. zone storage restrictions). diff --git a/versioned_docs/version-2.8/pages-for-subheaders/cis-scan-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/cis-scan-guides.md new file mode 100644 index 00000000000..e76d47504e6 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/cis-scan-guides.md @@ -0,0 +1,17 @@ +--- +title: CIS Scan Guides +--- + + + + + +- [Install rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md) +- [Uninstall rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md) +- [Run a Scan](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md) +- [Run a Scan Periodically on a Schedule](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md) +- [Skip Tests](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md) +- [View Reports](../how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md) +- [Enable Alerting for rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) +- [Configure Alerts for Periodic Scan on a Schedule](../how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md) +- [Create a Custom Benchmark Version to Run](../how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/cis-scans.md b/versioned_docs/version-2.8/pages-for-subheaders/cis-scans.md new file mode 100644 index 00000000000..d9c5dbecabb --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/cis-scans.md @@ -0,0 +1,122 @@ +--- +title: CIS Scans +--- + + + + + +Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. The CIS scans can run on any Kubernetes cluster, including hosted Kubernetes providers such as EKS, AKS, and GKE. + +The `rancher-cis-benchmark` app leverages kube-bench, an open-source tool from Aqua Security, to check clusters for CIS Kubernetes Benchmark compliance. Also, to generate a cluster-wide report, the application utilizes Sonobuoy for report aggregation. + + +## About the CIS Benchmark + +The Center for Internet Security is a 501(c\)(3) non-profit organization, formed in October 2000, with a mission to "identify, develop, validate, promote, and sustain best practice solutions for cyber defense and build and lead communities to enable an environment of trust in cyberspace". The organization is headquartered in East Greenbush, New York, with members including large corporations, government agencies, and academic institutions. + +CIS Benchmarks are best practices for the secure configuration of a target system. CIS Benchmarks are developed through the generous volunteer efforts of subject matter experts, technology vendors, public and private community members, and the CIS Benchmark Development team. + +[Sign up](https://learn.cisecurity.org/benchmarks) at the CIS website to view the official Benchmark documents. + +## About the Generated Report + +Each scan generates a report can be viewed in the Rancher UI and can be downloaded in CSV format. + +By default, the CIS Benchmark v1.6 is used. + +The Benchmark version is included in the generated report. + +The Benchmark provides recommendations of two types: Automated and Manual. Recommendations marked as Manual in the Benchmark are not included in the generated report. + +Some tests are designated as "Not Applicable." These tests will not be run on any CIS scan because of the way that Rancher provisions RKE clusters. For information on how test results can be audited, and why some tests are designated to be not applicable, refer to Rancher's [self-assessment guide](./rancher-security.md#the-cis-benchmark-and-self-assessment) for the corresponding Kubernetes version. + +The report contains the following information: + +| Column in Report | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | The ID number of the CIS Benchmark. | +| `description` | The description of the CIS Benchmark test. | +| `remediation` | What needs to be fixed in order to pass the test. | +| `state` | Indicates if the test passed, failed, was skipped, or was not applicable. | +| `node_type` | The node role, which affects which tests are run on the node. Master tests are run on controlplane nodes, etcd tests are run on etcd nodes, and node tests are run on the worker nodes. | +| `audit` | This is the audit check that `kube-bench` runs for this test. | +| `audit_config` | Any configuration applicable to the audit script. | +| `test_info` | Test-related info as reported by `kube-bench`, if any. | +| `commands` | Test-related commands as reported by `kube-bench`, if any. | +| `config_commands` | Test-related configuration data as reported by `kube-bench`, if any. | +| `actual_value` | The test's actual value, present if reported by `kube-bench`. | +| `expected_result` | The test's expected result, present if reported by `kube-bench`. | + +Refer to [the table in the cluster hardening guide](./rancher-security.md) for information on which versions of Kubernetes, the Benchmark, Rancher, and our cluster hardening guide correspond to each other. Also refer to the hardening guide for configuration files of CIS-compliant clusters and information on remediating failed tests. + +## Test Profiles + +The following profiles are available: + +- Generic CIS 1.6 +- Generic CIS 1.20 +- Generic CIS 1.23 +- RKE permissive 1.6 +- RKE hardened 1.6 +- RKE permissive 1.20 +- RKE hardened 1.20 +- RKE permissive 1.23 +- RKE hardened 1.23 +- RKE2 permissive 1.6 +- RKE2 hardened 1.6 +- RKE2 permissive 1.20 +- RKE2 hardened 1.20 +- RKE2 permissive 1.23 +- RKE2 hardened 1.23 +- K3s permissive 1.6 +- K3s hardened 1.6 +- K3s permissive 1.20 +- K3s hardened 1.20 +- K3s permissive 1.23 +- K3s hardened 1.23 +- AKS +- EKS +- GKE + +You also have the ability to customize a profile by saving a set of tests to skip. + +All profiles will have a set of not applicable tests that will be skipped during the CIS scan. These tests are not applicable based on how a RKE cluster manages Kubernetes. + +There are two types of RKE cluster scan profiles: + +- **Permissive:** This profile has a set of tests that have been will be skipped as these tests will fail on a default RKE Kubernetes cluster. Besides the list of skipped tests, the profile will also not run the not applicable tests. +- **Hardened:** This profile will not skip any tests, except for the non-applicable tests. + +The EKS and GKE cluster scan profiles are based on CIS Benchmark versions that are specific to those types of clusters. + +In order to pass the "Hardened" profile, you will need to follow the steps on the [hardening guide](./rancher-security.md#rancher-hardening-guide) and use the `cluster.yml` defined in the hardening guide to provision a hardened cluster. + +The default profile and the supported CIS benchmark version depends on the type of cluster that will be scanned: + +The `rancher-cis-benchmark` supports the CIS 1.6 Benchmark version. + +- For RKE Kubernetes clusters, the RKE Permissive 1.6 profile is the default. +- EKS and GKE have their own CIS Benchmarks published by `kube-bench`. The corresponding test profiles are used by default for those clusters. +- For RKE2 Kubernetes clusters, the RKE2 Permissive 1.6 profile is the default. +- For cluster types other than RKE, RKE2, EKS and GKE, the Generic CIS 1.5 profile will be used by default. + +## About Skipped and Not Applicable Tests + +For a list of skipped and not applicable tests, refer to [this page](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md). + +For now, only user-defined skipped tests are marked as skipped in the generated report. + +Any skipped tests that are defined as being skipped by one of the default profiles are marked as not applicable. + +## Roles-based Access Control + +For information about permissions, refer to [this page](../integrations-in-rancher/cis-scans/rbac-for-cis-scans.md) + +## Configuration + +For more information about configuring the custom resources for the scans, profiles, and benchmark versions, refer to [this page](../integrations-in-rancher/cis-scans/configuration-reference.md) + +## How-to Guides + +Please refer to the [CIS Scan Guides](../pages-for-subheaders/cis-scan-guides.md) to learn how to run CIS scans. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/cli-with-rancher.md b/versioned_docs/version-2.8/pages-for-subheaders/cli-with-rancher.md new file mode 100644 index 00000000000..547d4c50308 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/cli-with-rancher.md @@ -0,0 +1,9 @@ +--- +title: CLI with Rancher +--- + + + + + +Interact with Rancher using command line interface (CLI) tools from your workstation. The following docs will describe the [Rancher CLI](../reference-guides/cli-with-rancher/rancher-cli.md) and [kubectl Utility](../reference-guides/cli-with-rancher/kubectl-utility.md). \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/cloud-marketplace.md b/versioned_docs/version-2.8/pages-for-subheaders/cloud-marketplace.md new file mode 100644 index 00000000000..6f90a2c3ff4 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/cloud-marketplace.md @@ -0,0 +1,11 @@ +--- +title: Cloud Marketplace Integration +--- + + + + + +Rancher offers integration with cloud marketplaces to easily purchase support for installations hosted on certain cloud providers. In addition, this integration also provides the ability to generate a supportconfig bundle which can be provided to rancher support. + +This integration only supports AWS. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/cluster-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/cluster-configuration.md new file mode 100644 index 00000000000..60e02a8cd40 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/cluster-configuration.md @@ -0,0 +1,33 @@ +--- +title: Cluster Configuration +--- + + + + + +After you provision a Kubernetes cluster using Rancher, you can still edit options and settings for the cluster. + +For information on editing cluster membership, go to [this page.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) + +### Cluster Configuration References + +The cluster configuration options depend on the type of Kubernetes cluster: + +- [RKE Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [GKE Cluster Configuration](gke-cluster-configuration.md) +- [AKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) + +### Cluster Management Capabilities by Cluster Type + +The options and settings available for an existing cluster change based on the method that you used to provision it. + +The following table summarizes the options and settings available for each cluster type: + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + diff --git a/versioned_docs/version-2.8/pages-for-subheaders/configuration-options.md b/versioned_docs/version-2.8/pages-for-subheaders/configuration-options.md new file mode 100644 index 00000000000..92a375e948d --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/configuration-options.md @@ -0,0 +1,43 @@ +--- +title: Configuration Options +--- + + + + + +### Egress Support + +By default the Egress gateway is disabled, but can be enabled on install or upgrade through the values.yaml or via the [overlay file](#overlay-file). + +### Enabling Automatic Sidecar Injection + +Automatic sidecar injection is disabled by default. To enable this, set the `sidecarInjectorWebhook.enableNamespacesByDefault=true` in the values.yaml on install or upgrade. This automatically enables Istio sidecar injection into all new namespaces that are deployed. + +### Overlay File + +An Overlay File is designed to support extensive configuration of your Istio installation. It allows you to make changes to any values available in the [IstioOperator API](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/). This will ensure you can customize the default installation to fit any scenario. + +The Overlay File will add configuration on top of the default installation that is provided from the Istio chart installation. This means you do not need to redefine the components that already defined for installation. + +For more information on Overlay Files, refer to the [Istio documentation.](https://istio.io/latest/docs/setup/install/istioctl/#configure-component-settings) + +### Selectors and Scrape Configs + +The Monitoring app sets `prometheus.prometheusSpec.ignoreNamespaceSelectors=false` which enables monitoring across all namespaces by default. This ensures you can view traffic, metrics and graphs for resources deployed in a namespace with `istio-injection=enabled` label. + +If you would like to limit Prometheus to specific namespaces, set `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`. Once you do this, you will need to add additional configuration to continue to monitor your resources. + +For details, refer to [this section.](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) + +### Enable Istio with Pod Security Policies + +Refer to [this section.](../integrations-in-rancher/istio/configuration-options/pod-security-policies.md) + +### Additional Steps for Installing Istio on an RKE2 Cluster + +Refer to [this section.](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md) + +### Additional Steps for Project Network Isolation + +Refer to [this section.](../integrations-in-rancher/istio/configuration-options/project-network-isolation.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md b/versioned_docs/version-2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md new file mode 100644 index 00000000000..8662bf782fb --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md @@ -0,0 +1,40 @@ +--- +title: Configuring Microsoft Active Directory Federation Service (SAML) +--- + + + + + +If your organization uses Microsoft Active Directory Federation Services (AD FS) for user authentication, you can configure Rancher to allow your users to log in using their AD FS credentials. + +## Prerequisites + +You must have Rancher installed. + +- Obtain your Rancher Server URL. During AD FS configuration, substitute this URL for the `` placeholder. +- You must have a global administrator account on your Rancher installation. + +You must have a [Microsoft AD FS Server](https://docs.microsoft.com/en-us/windows-server/identity/active-directory-federation-services) configured. + +- Obtain your AD FS Server IP/DNS name. During AD FS configuration, substitute this IP/DNS name for the `` placeholder. +- You must have access to add [Relying Party Trusts](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) on your AD FS Server. + +## Setup Outline + +Setting up Microsoft AD FS with Rancher Server requires configuring AD FS on your Active Directory server, and configuring Rancher to utilize your AD FS server. The following pages serve as guides for setting up Microsoft AD FS authentication on your Rancher installation. + +- [1. Configuring Microsoft AD FS for Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) +- [2. Configuring Rancher for Microsoft AD FS](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md) + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + + +### [Next: Configuring Microsoft AD FS for Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/configure-openldap.md b/versioned_docs/version-2.8/pages-for-subheaders/configure-openldap.md new file mode 100644 index 00000000000..9eb5fc7db2a --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/configure-openldap.md @@ -0,0 +1,56 @@ +--- +title: Configuring OpenLDAP +--- + + + + + +If your organization uses LDAP for user authentication, you can configure Rancher to communicate with an OpenLDAP server to authenticate users. This allows Rancher admins to control access to clusters and projects based on users and groups managed externally in the organisation's central user repository, while allowing end-users to authenticate with their LDAP credentials when logging in to the Rancher UI. + +## Prerequisites + +Rancher must be configured with a LDAP bind account (aka service account) to search and retrieve LDAP entries pertaining to users and groups that should have access. It is recommended to not use an administrator account or personal account for this purpose and instead create a dedicated account in OpenLDAP with read-only access to users and groups under the configured search base (see below). + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or not from a recognised certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +## Configure OpenLDAP in Rancher + +Configure the settings for the OpenLDAP server, groups and users. For help filling out each field, refer to the [configuration reference.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md) + +> Before you proceed with the configuration, please familiarise yourself with the concepts of [External Authentication Configuration and Principal Users](authentication-config.md#external-authentication-configuration-and-principal-users). + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **OpenLDAP**. Fill out the **Configure an OpenLDAP server** form. +1. Click **Enable**. + +### Test Authentication + +Once you have completed the configuration, proceed by testing the connection to the OpenLDAP server. Authentication with OpenLDAP will be enabled implicitly if the test is successful. + +:::note + +The OpenLDAP user pertaining to the credentials entered in this step will be mapped to the local principal account and assigned administrator privileges in Rancher. You should therefore make a conscious decision on which LDAP account you use to perform this step. + +::: + +1. Enter the **username** and **password** for the OpenLDAP account that should be mapped to the local principal account. +2. Click **Authenticate With OpenLDAP** to test the OpenLDAP connection and finalise the setup. + +**Result:** + +- OpenLDAP authentication is configured. +- The LDAP user pertaining to the entered credentials is mapped to the local principal (administrative) account. + +:::note + +You will still be able to login using the locally configured `admin` account and password in case of a disruption of LDAP services. + +::: + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the OpenLDAP server, first double-check the credentials entered for the service account as well as the search base configuration. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/configure-shibboleth-saml.md b/versioned_docs/version-2.8/pages-for-subheaders/configure-shibboleth-saml.md new file mode 100644 index 00000000000..f8c3480dd08 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/configure-shibboleth-saml.md @@ -0,0 +1,106 @@ +--- +title: Configuring Shibboleth (SAML) +--- + + + + + +If your organization uses Shibboleth Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in to Rancher using their Shibboleth credentials. + +In this configuration, when Rancher users log in, they will be redirected to the Shibboleth IdP to enter their credentials. After authentication, they will be redirected back to the Rancher UI. + +If you also configure OpenLDAP as the back end to Shibboleth, it will return a SAML assertion to Rancher with user attributes that include groups. Then the authenticated user will be able to access resources in Rancher that their groups have permissions for. + +> The instructions in this section assume that you understand how Rancher, Shibboleth, and OpenLDAP work together. For a more detailed explanation of how it works, refer to [this page.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md) + + +## Setting up Shibboleth in Rancher + +### Shibboleth Prerequisites +> +>- You must have a Shibboleth IdP Server configured. +>- Following are the Rancher Service Provider URLs needed for configuration: +Metadata URL: `https:///v1-saml/shibboleth/saml/metadata` +Assertion Consumer Service (ACS) URL: `https:///v1-saml/shibboleth/saml/acs` +>- Export a `metadata.xml` file from your IdP Server. For more information, see the [Shibboleth documentation.](https://wiki.shibboleth.net/confluence/display/SP3/Home) + +### Configure Shibboleth in Rancher + +If your organization uses Shibboleth for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Shibboleth**. +1. Complete the **Configure Shibboleth Account** form. Shibboleth IdP lets you specify what data store you want to use. You can either add a database or use an existing ldap server. For example, if you select your Active Directory (AD) server, the examples below describe how you can map AD attributes to fields within Rancher. + + 1. **Display Name Field**: Enter the AD attribute that contains the display name of users (example: `displayName`). + + 1. **User Name Field**: Enter the AD attribute that contains the user name/given name (example: `givenName`). + + 1. **UID Field**: Enter an AD attribute that is unique to every user (example: `sAMAccountName`, `distinguishedName`). + + 1. **Groups Field**: Make entries for managing group memberships (example: `memberOf`). + + 1. **Rancher API Host**: Enter the URL for your Rancher Server. + + 1. **Private Key** and **Certificate**: This is a key-certificate pair to create a secure shell between Rancher and your IdP. + + You can generate one using an openssl command. For example: + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP-metadata**: The `metadata.xml` file that you exported from your IdP server. + + +1. After you complete the **Configure Shibboleth Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Shibboleth IdP to validate your Rancher Shibboleth configuration. + + :::note + + You may have to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with Shibboleth. Your users can now sign into Rancher using their Shibboleth logins. + +### SAML Provider Caveats + +If you configure Shibboleth without OpenLDAP, the following caveats apply due to the fact that SAML Protocol does not support search or lookup for users or groups. + +- There is no validation on users or groups when assigning permissions to them in Rancher. +- When adding users, the exact user IDs (i.e. UID Field) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +To enable searching for groups when assigning permissions in Rancher, you will need to configure a back end for the SAML provider that supports groups, such as OpenLDAP. + +## Setting up OpenLDAP in Rancher + +If you also configure OpenLDAP as the back end to Shibboleth, it will return a SAML assertion to Rancher with user attributes that include groups. Then authenticated users will be able to access resources in Rancher that their groups have permissions for. + +### OpenLDAP Prerequisites + +Rancher must be configured with a LDAP bind account (aka service account) to search and retrieve LDAP entries pertaining to users and groups that should have access. It is recommended to not use an administrator account or personal account for this purpose and instead create a dedicated account in OpenLDAP with read-only access to users and groups under the configured search base (see below). + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or not from a recognized certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +### Configure OpenLDAP in Rancher + +Configure the settings for the OpenLDAP server, groups and users. For help filling out each field, refer to the [configuration reference.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md) Note that nested group membership is not available for Shibboleth. + +> Before you proceed with the configuration, please familiarise yourself with the concepts of [External Authentication Configuration and Principal Users](authentication-config.md#external-authentication-configuration-and-principal-users). + +1. Log into the Rancher UI using the initial local `admin` account. +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Shibboleth** or, if SAML is already configured, **Edit Config** +1. Under **User and Group Search**, check **Configure an OpenLDAP server** + +## Troubleshooting + +If you are experiencing issues while testing the connection to the OpenLDAP server, first double-check the credentials entered for the service account as well as the search base configuration. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/create-kubernetes-persistent-storage.md b/versioned_docs/version-2.8/pages-for-subheaders/create-kubernetes-persistent-storage.md new file mode 100644 index 00000000000..6bda26af36e --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/create-kubernetes-persistent-storage.md @@ -0,0 +1,78 @@ +--- +title: "Kubernetes Persistent Storage: Volumes and Storage Classes" +description: "Learn about the two ways with which you can create persistent storage in Kubernetes: persistent volumes and storage classes" +--- + + + + + +When deploying an application that needs to retain data, you'll need to create persistent storage. Persistent storage allows you to store application data external from the pod running your application. This storage practice allows you to maintain application data, even if the application's pod fails. + +The documents in this section assume that you understand the Kubernetes concepts of persistent volumes, persistent volume claims, and storage classes. For more information, refer to the section on [how storage works.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md) + +### Prerequisites + +To set up persistent storage, the `Manage Volumes` [role](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-role-reference) is required. + +If you are provisioning storage for a cluster hosted in the cloud, the storage and cluster hosts must have the same cloud provider. + +For provisioning new storage with Rancher, the cloud provider must be enabled. For details on enabling cloud providers, refer to [this page.](../pages-for-subheaders/set-up-cloud-providers.md) + +For attaching existing persistent storage to a cluster, the cloud provider does not need to be enabled. + +### Setting up Existing Storage + +The overall workflow for setting up existing storage is as follows: + +1. Set up your persistent storage. This may be storage in an infrastructure provider, or it could be your own storage. +2. Add a persistent volume (PV) that refers to the persistent storage. +3. Add a persistent volume claim (PVC) that refers to the PV. +4. Mount the PVC as a volume in your workload. + +For details and prerequisites, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md) + +### Dynamically Provisioning New Storage in Rancher + +The overall workflow for provisioning new storage is as follows: + +1. Add a StorageClass and configure it to use your storage provider. The StorageClass could refer to storage in an infrastructure provider, or it could refer to your own storage. +2. Add a persistent volume claim (PVC) that refers to the storage class. +3. Mount the PVC as a volume for your workload. + +For details and prerequisites, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md) + +### Longhorn Storage + +[Longhorn](https://longhorn.io/) is a lightweight, reliable and easy-to-use distributed block storage system for Kubernetes. + +Longhorn is free, open source software. Originally developed by Rancher Labs, it is now being developed as a sandbox project of the Cloud Native Computing Foundation. It can be installed on any Kubernetes cluster with Helm, with kubectl, or with the Rancher UI. + +If you have a pool of block storage, Longhorn can help you provide persistent storage to your Kubernetes cluster without relying on cloud providers. For more information about Longhorn features, refer to the [documentation.](https://longhorn.io/docs/latest/what-is-longhorn/) + +Rancher v2.5 simplified the process of installing Longhorn on a Rancher-managed cluster. For more information, see [this page.](../integrations-in-rancher/longhorn.md) + +### Provisioning Storage Examples + +We provide examples of how to provision storage with [NFS,](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) [vSphere,](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) and [Amazon's EBS.](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) + +### GlusterFS Volumes + +In clusters that store data on GlusterFS volumes, you may experience an issue where pods fail to mount volumes after restarting the `kubelet`. For details on preventing this from happening, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md) + +### iSCSI Volumes + +In [Rancher Launched Kubernetes clusters](../pages-for-subheaders/launch-kubernetes-with-rancher.md) that store data on iSCSI volumes, you may experience an issue where kubelets fail to automatically connect with iSCSI volumes. For details on resolving this issue, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md) + +### hostPath Volumes +Before you create a hostPath volume, you need to set up an [extra_bind](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds/) in your cluster configuration. This will mount the path as a volume in your kubelets, which can then be used for hostPath volumes in your workloads. + +### Migrating vSphere Cloud Provider from In-tree to Out-of-tree + +Kubernetes is moving away from maintaining cloud providers in-tree. vSphere has an out-of-tree cloud provider that can be used by installing the vSphere cloud provider and cloud storage plugins. + +For instructions on how to migrate from the in-tree vSphere cloud provider to out-of-tree, and manage the existing VMs post migration, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md) + +### Related Links + +- [Kubernetes Documentation: Storage](https://kubernetes.io/docs/concepts/storage/) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/custom-resource-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/custom-resource-configuration.md new file mode 100644 index 00000000000..37c2f337d78 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/custom-resource-configuration.md @@ -0,0 +1,12 @@ +--- +title: Custom Resource Configuration +--- + + + + + +The following Custom Resource Definitions are used to configure logging: + +- [Flow and ClusterFlow](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md) +- [Output and ClusterOutput](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/deploy-apps-across-clusters.md b/versioned_docs/version-2.8/pages-for-subheaders/deploy-apps-across-clusters.md new file mode 100644 index 00000000000..8089f7a914d --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/deploy-apps-across-clusters.md @@ -0,0 +1,18 @@ +--- +title: Deploying Applications across Clusters +--- + + + + +### Fleet + +Rancher v2.5 introduced Fleet, a new way to deploy applications across clusters. + +Continuous Delivery with Fleet is GitOps at scale. For more information, refer to the [Fleet section.](../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) + +### Multi-cluster Apps + +In Rancher before v2.5, the multi-cluster apps feature was used to deploy applications across clusters. The multi-cluster apps feature is deprecated, but still available in Rancher v2.5. + +Refer to the documentation [here.](../how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/deploy-rancher-manager.md b/versioned_docs/version-2.8/pages-for-subheaders/deploy-rancher-manager.md new file mode 100644 index 00000000000..74e282f0832 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/deploy-rancher-manager.md @@ -0,0 +1,23 @@ +--- +title: Deploying Rancher Server +--- + + + + + +Use one of the following guides to deploy and provision Rancher and a Kubernetes cluster in the provider of your choice. + +- [AWS](../getting-started/quick-start-guides/deploy-rancher-manager/aws.md) (uses Terraform) +- [AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md) (uses Amazon EKS) +- [Azure](../getting-started/quick-start-guides/deploy-rancher-manager/azure.md) (uses Terraform) +- [DigitalOcean](../getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md) (uses Terraform) +- [GCP](../getting-started/quick-start-guides/deploy-rancher-manager/gcp.md) (uses Terraform) +- [Hetzner Cloud](../getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md) (uses Terraform) +- [Vagrant](../getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md) +- [Equinix Metal](../getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md) +- [Outscale](../getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md) (uses Terraform) + +If you prefer, the following guide will take you through the same process in individual steps. Use this if you want to run Rancher in a different provider, on prem, or if you would just like to see how easy it is. + +- [Manual Install](../getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/deploy-rancher-workloads.md b/versioned_docs/version-2.8/pages-for-subheaders/deploy-rancher-workloads.md new file mode 100644 index 00000000000..b2898cd513b --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/deploy-rancher-workloads.md @@ -0,0 +1,12 @@ +--- +title: Deploying Workloads +--- + + + + + +These guides walk you through the deployment of an application, including how to expose the application for use outside of the cluster. + +- [Workload with Ingress](../getting-started/quick-start-guides/deploy-workloads/workload-ingress.md) +- [Workload with NodePort](../getting-started/quick-start-guides/deploy-workloads/nodeports.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/downstream-cluster-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/downstream-cluster-configuration.md new file mode 100644 index 00000000000..b9fbad0b966 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/downstream-cluster-configuration.md @@ -0,0 +1,9 @@ +--- +title: Downstream Cluster Configuration +--- + + + + + +The following docs will discuss [node template configuration](./node-template-configuration.md) and [machine configuration](./machine-configuration.md). \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/enable-experimental-features.md b/versioned_docs/version-2.8/pages-for-subheaders/enable-experimental-features.md new file mode 100644 index 00000000000..0e5ad863608 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/enable-experimental-features.md @@ -0,0 +1,126 @@ +--- +title: Enabling Experimental Features +--- + + + + + +Rancher includes some features that are experimental and disabled by default. You might want to enable these features, for example, if you decide that the benefits of using an [unsupported storage type](../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) outweighs the risk of using an untested feature. Feature flags were introduced to allow you to try these features that are not enabled by default. + +The features can be enabled in three ways: + +- [Enable features when starting Rancher.](#enabling-features-when-starting-rancher) When installing Rancher with a CLI, you can use a feature flag to enable a feature by default. +- [Enable features from the Rancher UI](#enabling-features-with-the-rancher-ui) by going to the **Settings** page. +- [Enable features with the Rancher API](#enabling-features-with-the-rancher-api) after installing Rancher. + +Each feature has two values: + +- A default value, which can be configured with a flag or environment variable from the command line +- A set value, which can be configured with the Rancher API or UI + +If no value has been set, Rancher uses the default value. + +Because the API sets the actual value and the command line sets the default value, that means that if you enable or disable a feature with the API or UI, it will override any value set with the command line. + +For example, if you install Rancher, then set a feature flag to true with the Rancher API, then upgrade Rancher with a command that sets the feature flag to false, the default value will still be false, but the feature will still be enabled because it was set with the Rancher API. If you then deleted the set value (true) with the Rancher API, setting it to NULL, the default value (false) would take effect. See the [feature flags page](../getting-started/installation-and-upgrade/installation-references/feature-flags.md) for more information. + +## Enabling Features when Starting Rancher + +When you install Rancher, enable the feature you want with a feature flag. The command is different depending on whether you are installing Rancher on a single node or if you are doing a Kubernetes Installation of Rancher. + +### Enabling Features for Kubernetes Installs + +:::note + +Values set from the Rancher API will override the value passed in through the command line. + +::: + +When installing Rancher with a Helm chart, use the `--set` option. In the below example, two features are enabled by passing the feature flag names in a comma separated list: + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +:::note + +If you are installing an alpha version, Helm requires adding the `--devel` option to the command. + +::: + +### Enabling Features for Air Gap Installs + +To perform an [air gap installation of Rancher](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md), add a Helm chart repository and download a Helm chart, then install Rancher with Helm. + +When you install the Helm chart, you should pass in feature flag names in a comma separated list, as in the following example: + +``` +helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +### Enabling Features for Docker Installs + +When installing Rancher with Docker, use the `--features` option. In the below example, two features are enabled by passing the feature flag names in a comma separated list: + +``` +docker run -d -p 80:80 -p 443:443 \ + --restart=unless-stopped \ + rancher/rancher:rancher-latest \ + --features==true,=true +``` + + +## Enabling Features with the Rancher UI + +1. In the upper left corner, click **☰ > Global Settings**. +1. Click **Feature Flags**. +1. To enable a feature, go to the disabled feature you want to enable and click **â‹® > Activate**. + +**Result:** The feature is enabled. + +### Disabling Features with the Rancher UI + +1. In the upper left corner, click **☰ > Global Settings**. +1. Click **Feature Flags**. You will see a list of experimental features. +1. To disable a feature, go to the enabled feature you want to disable and click **â‹® > Deactivate**. + +**Result:** The feature is disabled. + +## Enabling Features with the Rancher API + +1. Go to `/v3/features`. +1. In the `data` section, you will see an array containing all of the features that can be turned on with feature flags. The name of the feature is in the `id` field. Click the name of the feature you want to enable. +1. In the upper left corner of the screen, under **Operations,** click **Edit**. +1. In the **Value** drop-down menu, click **True**. +1. Click **Show Request**. +1. Click **Send Request**. +1. Click **Close**. + +**Result:** The feature is enabled. + +### Disabling Features with the Rancher API + +1. Go to `/v3/features`. +1. In the `data` section, you will see an array containing all of the features that can be turned on with feature flags. The name of the feature is in the `id` field. Click the name of the feature you want to enable. +1. In the upper left corner of the screen, under **Operations,** click **Edit**. +1. In the **Value** drop-down menu, click **False**. +1. Click **Show Request**. +1. Click **Send Request**. +1. Click **Close**. + +**Result:** The feature is disabled. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/fleet-gitops-at-scale.md b/versioned_docs/version-2.8/pages-for-subheaders/fleet-gitops-at-scale.md new file mode 100644 index 00000000000..54e38cc48eb --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/fleet-gitops-at-scale.md @@ -0,0 +1,68 @@ +--- +title: Continuous Delivery with Fleet +--- + + + + + +Continuous Delivery with Fleet is GitOps at scale. Fleet is designed to manage up to a million clusters. It’s also lightweight enough that it works great for a [single cluster](https://fleet.rancher.io/installation#default-install) too, but it really shines when you get to a [large scale](https://fleet.rancher.io/installation#configuration-for-multi-cluster). By large scale we mean either a lot of clusters, a lot of deployments, or a lot of teams in a single organization. + +Fleet is a separate project from Rancher, and can be installed on any Kubernetes cluster with Helm. + + +## Architecture + +For information about how Fleet works, see [this page](../integrations-in-rancher/fleet-gitops-at-scale/architecture.md). + +## Accessing Fleet in the Rancher UI + +Fleet comes preinstalled in Rancher and is managed by the **Continuous Delivery** option in the Rancher UI. For additional information on Continuous Delivery and other Fleet troubleshooting tips, refer [here](https://fleet.rancher.io/troubleshooting). + +Users can leverage continuous delivery to deploy their applications to the Kubernetes clusters in the git repository without any manual operation by following **gitops** practice. + +Follow the steps below to access Continuous Delivery in the Rancher UI: + +1. Click **☰ > Continous Delivery**. + +1. Select your namespace at the top of the menu, noting the following: + + - By default, **fleet-default** is selected which includes all downstream clusters that are registered through Rancher. + + - You may switch to **fleet-local**, which only contains the **local** cluster, or you may create your own workspace to which you may assign and move clusters. + + - You can then manage clusters by clicking on **Clusters** on the left navigation bar. + +1. Click on **Gitrepos** on the left navigation bar to deploy the gitrepo into your clusters in the current workspace. + +1. Select your [git repository](https://fleet.rancher.io/gitrepo-add) and [target clusters/cluster group](https://fleet.rancher.io/gitrepo-targets). You can also create the cluster group in the UI by clicking on **Cluster Groups** from the left navigation bar. + +1. Once the gitrepo is deployed, you can monitor the application through the Rancher UI. + +## Windows Support + +For details on support for clusters with Windows nodes, see [this page](../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md). + +## GitHub Repository + +The Fleet Helm charts are available [here](https://github.com/rancher/fleet/releases). + +## Using Fleet Behind a Proxy + +For details on using Fleet behind a proxy, see [this page](../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md). + +## Helm Chart Dependencies + +In order for Helm charts with dependencies to deploy successfully, you must run a manual command (as listed below), as it is up to the user to fulfill the dependency list. If you do not do this and proceed to clone your repository and run `helm install`, your installation will fail because the dependencies will be missing. + +The Helm chart in the git repository must include its dependencies in the charts subdirectory. You must either manually run `helm dependencies update $chart` OR run `helm dependencies build $chart` locally, then commit the complete charts directory to your git repository. Note that you will update your commands with the applicable parameters + +## Troubleshooting + +- **Known Issue**: clientSecretName and helmSecretName secrets for Fleet gitrepos are not included in the backup nor restore created by the [backup-restore-operator](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md#1-install-the-rancher-backup-operator). We will update the community once a permanent solution is in place. + +- **Temporary Workaround**: By default, user-defined secrets are not backed up in Fleet. It is necessary to recreate secrets if performing a disaster recovery restore or migration of Rancher into a fresh cluster. To modify resourceSet to include extra resources you want to backup, refer to docs [here](https://github.com/rancher/backup-restore-operator#user-flow). + +## Documentation + +The Fleet documentation is at https://fleet.rancher.io/. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/gke-cluster-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/gke-cluster-configuration.md new file mode 100644 index 00000000000..de9a1638a9c --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/gke-cluster-configuration.md @@ -0,0 +1,325 @@ +--- +title: GKE Cluster Configuration Reference +--- + + + + + +## Changes in Rancher v2.6 + +- Support for additional configuration options: + - Project network isolation + - Network tags + +## Cluster Location + +| Value | Description | +|--------|--------------| +| Location Type | Zonal or Regional. With GKE, you can create a cluster tailored to the availability requirements of your workload and your budget. By default, a cluster's nodes run in a single compute zone. When multiple zones are selected, the cluster's nodes will span multiple compute zones, while the controlplane is located in a single zone. Regional clusters increase the availability of the controlplane as well. For help choosing the type of cluster availability, refer to [these docs.](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#choosing_a_regional_or_zonal_control_plane) | +| Zone | Each region in Compute engine contains a number of zones. For more information about available regions and zones, refer to [these docs.](https://cloud.google.com/compute/docs/regions-zones#available) | +| Additional Zones | For zonal clusters, you can select additional zones to create a [multi-zone cluster.](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#multi-zonal_clusters) | +| Region | For [regional clusters,](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#regional_clusters) you can select a region. For more information about available regions and zones, refer to [this section](https://cloud.google.com/compute/docs/regions-zones#available). The first part of each zone name is the name of the region. | + +## Cluster Options + +### Kubernetes Version + +_Mutable: yes_ + +For more information on GKE Kubernetes versions, refer to [these docs.](https://cloud.google.com/kubernetes-engine/versioning) + +### Container Address Range + +_Mutable: no_ + +The IP address range for pods in the cluster. Must be a valid CIDR range, e.g. 10.42.0.0/16. If not specified, a random range is automatically chosen from 10.0.0.0/8 and will exclude ranges already allocated to VMs, other clusters, or routes. Automatically chosen ranges may conflict with reserved IP addresses, dynamic routes, or routes within VPCs peering with the cluster. + +### Network + +_Mutable: no_ + +The Compute Engine Network that the cluster connects to. Routes and firewalls will be created using this network. If using [Shared VPCs](https://cloud.google.com/vpc/docs/shared-vpc), the VPC networks that are shared to your project will appear here. will be available to select in this field. For more information, refer to [this page](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets). + +### Node Subnet / Subnet + +_Mutable: no_ + +The Compute Engine subnetwork that the cluster connects to. This subnetwork must belong to the network specified in the **Network** field. Select an existing subnetwork, or select "Auto Create Subnetwork" to have one automatically created. If not using an existing network, **Subnetwork Name** is required to generate one. If using [Shared VPCs](https://cloud.google.com/vpc/docs/shared-vpc), the VPC subnets that are shared to your project will appear here. If using a Shared VPC network, you cannot select "Auto Create Subnetwork". For more information, refer to [this page.](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets) + +### Subnetwork Name + +_Mutable: no_ + +Automatically create a subnetwork with the provided name. Required if "Auto Create Subnetwork" is selected for **Node Subnet** or **Subnet**. For more information on subnetworks, refer to [this page.](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets) + +### Ip Aliases + +_Mutable: no_ + +Enable [alias IPs](https://cloud.google.com/vpc/docs/alias-ip). This enables VPC-native traffic routing. Required if using [Shared VPCs](https://cloud.google.com/vpc/docs/shared-vpc). + +### Network Policy + +_Mutable: yes_ + +Enable network policy enforcement on the cluster. A network policy defines the level of communication that can occur between pods and services in the cluster. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/how-to/network-policy) + +### Project Network Isolation + +_Mutable: yes_ + +choose whether to enable or disable inter-project communication. Note that enabling Project Network Isolation will automatically enable Network Policy and Network Policy Config, but not vice versa. + +### Node Ipv4 CIDR Block + +_Mutable: no_ + +The IP address range of the instance IPs in this cluster. Can be set if "Auto Create Subnetwork" is selected for **Node Subnet** or **Subnet**. Must be a valid CIDR range, e.g. 10.96.0.0/14. For more information on how to determine the IP address range, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing) + +### Cluster Secondary Range Name + +_Mutable: no_ + +The name of an existing secondary range for Pod IP addresses. If selected, **Cluster Pod Address Range** will automatically be populated. Required if using a Shared VPC network. + +### Cluster Pod Address Range + +_Mutable: no_ + +The IP address range assigned to pods in the cluster. Must be a valid CIDR range, e.g. 10.96.0.0/11. If not provided, will be created automatically. Must be provided if using a Shared VPC network. For more information on how to determine the IP address range for your pods, refer to [this section.](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_pods) + +### Services Secondary Range Name + +_Mutable: no_ + +The name of an existing secondary range for service IP addresses. If selected, **Service Address Range** will be automatically populated. Required if using a Shared VPC network. + +### Service Address Range + +_Mutable: no_ + +The address range assigned to the services in the cluster. Must be a valid CIDR range, e.g. 10.94.0.0/18. If not provided, will be created automatically. Must be provided if using a Shared VPC network. For more information on how to determine the IP address range for your services, refer to [this section.](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_svcs) + +### Private Cluster + +_Mutable: no_ + +:::caution + +Private clusters require additional planning and configuration outside of Rancher. Refer to the [private cluster guide](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md). + +::: + +Assign nodes only internal IP addresses. Private cluster nodes cannot access the public internet unless additional networking steps are taken in GCP. + +### Enable Private Endpoint + +:::caution + +Private clusters require additional planning and configuration outside of Rancher. Refer to the [private cluster guide](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md). + +::: + +_Mutable: no_ + +Locks down external access to the control plane endpoint. Only available if **Private Cluster** is also selected. If selected, and if Rancher does not have direct access to the Virtual Private Cloud network the cluster is running in, Rancher will provide a registration command to run on the cluster to enable Rancher to connect to it. + +### Master IPV4 CIDR Block + +_Mutable: no_ + +The IP range for the control plane VPC. + +### Master Authorized Network + +_Mutable: yes_ + +Enable control plane authorized networks to block untrusted non-GCP source IPs from accessing the Kubernetes master through HTTPS. If selected, additional authorized networks may be added. If the cluster is created with a public endpoint, this option is useful for locking down access to the public endpoint to only certain networks, such as the network where your Rancher service is running. If the cluster only has a private endpoint, this setting is required. + +## Additional Options + +### Cluster Addons + +Additional Kubernetes cluster components. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.locations.clusters#Cluster.AddonsConfig) + +#### Horizontal Pod Autoscaling + +_Mutable: yes_ + +The Horizontal Pod Autoscaler changes the shape of your Kubernetes workload by automatically increasing or decreasing the number of Pods in response to the workload's CPU or memory consumption, or in response to custom metrics reported from within Kubernetes or external metrics from sources outside of your cluster. For more information, see [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/horizontalpodautoscaler) + +#### HTTP (L7) Load Balancing + +_Mutable: yes_ + +HTTP (L7) Load Balancing distributes HTTP and HTTPS traffic to backends hosted on GKE. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/tutorials/http-balancer) + +#### Network Policy Config (master only) + +_Mutable: yes_ + +Configuration for NetworkPolicy. This only tracks whether the addon is enabled or not on the master, it does not track whether network policy is enabled for the nodes. + +### Cluster Features (Alpha Features) + +_Mutable: no_ + +Turns on all Kubernetes alpha API groups and features for the cluster. When enabled, the cluster cannot be upgraded and will be deleted automatically after 30 days. Alpha clusters are not recommended for production use as they are not covered by the GKE SLA. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/alpha-clusters) + +### Logging Service + +_Mutable: yes_ + +The logging service the cluster uses to write logs. Use either [Cloud Logging](https://cloud.google.com/logging) or no logging service in which case no logs are exported from the cluster. + +### Monitoring Service + +_Mutable: yes_ + +The monitoring service the cluster uses to write metrics. Use either [Cloud Monitoring](https://cloud.google.com/monitoring) or monitoring service in which case no metrics are exported from the cluster. + + +### Maintenance Window + +_Mutable: yes_ + +Set the start time for a 4 hour maintenance window. The time is specified in the UTC time zone using the HH:MM format. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/maintenance-windows-and-exclusions) + +## Node Pools + +In this section, enter details describing the configuration of each node in the node pool. + +### Kubernetes Version + +_Mutable: yes_ + +The Kubernetes version for each node in the node pool. For more information on GKE Kubernetes versions, refer to [these docs.](https://cloud.google.com/kubernetes-engine/versioning) + +### Image Type + +_Mutable: yes_ + +The node operating system image. For more information for the node image options that GKE offers for each OS, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/node-images#available_node_images) + +:::note + +The default option is "Container-Optimized OS with Docker". The read-only filesystem on GCP's Container-Optimized OS is not compatible with the [legacy logging](/versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-logging.md) implementation in Rancher. If you need to use the legacy logging feature, select "Ubuntu with Docker" or "Ubuntu with Containerd". The [current logging feature](logging.md) is compatible with the Container-Optimized OS image. + +::: + +:::note + +If selecting "Windows Long Term Service Channel" or "Windows Semi-Annual Channel" for the node pool image type, you must also add at least one Container-Optimized OS or Ubuntu node pool. + +::: + +### Machine Type + +_Mutable: no_ + +The virtualized hardware resources available to node instances. For more information on Google Cloud machine types, refer to [this page.](https://cloud.google.com/compute/docs/machine-types#machine_types) + +### Root Disk Type + +_Mutable: no_ + +Standard persistent disks are backed by standard hard disk drives (HDD), while SSD persistent disks are backed by solid state drives (SSD). For more information, refer to [this section.](https://cloud.google.com/compute/docs/disks) + +### Local SSD Disks + +_Mutable: no_ + +Configure each node's local SSD disk storage in GB. Local SSDs are physically attached to the server that hosts your VM instance. Local SSDs have higher throughput and lower latency than standard persistent disks or SSD persistent disks. The data that you store on a local SSD persists only until the instance is stopped or deleted. For more information, see [this section.](https://cloud.google.com/compute/docs/disks#localssds) + +### Preemptible nodes (beta) + +_Mutable: no_ + +Preemptible nodes, also called preemptible VMs, are Compute Engine VM instances that last a maximum of 24 hours in general, and provide no availability guarantees. For more information, see [this page.](https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms) + +### Taints + +_Mutable: no_ + +When you apply a taint to a node, only Pods that tolerate the taint are allowed to run on the node. In a GKE cluster, you can apply a taint to a node pool, which applies the taint to all nodes in the pool. + +### Node Labels + +_Mutable: no_ + +You can apply labels to the node pool, which applies the labels to all nodes in the pool. + +Invalid labels can prevent upgrades or can prevent Rancher from starting. For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +### Network Tags + +_Mutable: no_ + +You can add network tags to the node pool to make firewall rules and routes between subnets. Tags will apply to all nodes in the pool. + +For details on tag syntax and requirements, see the [Kubernetes documentation](https://cloud.google.com/vpc/docs/add-remove-network-tags). + +## Group Details + +In this section, enter details describing the node pool. + +### Name + +_Mutable: no_ + +Enter a name for the node pool. + +### Initial Node Count + +_Mutable: yes_ + +Integer for the starting number of nodes in the node pool. + +### Max Pod Per Node + +_Mutable: no_ + +GKE has a hard limit of 110 Pods per node. For more information on the Kubernetes limits, see [this section.](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#dimension_limits) + +### Autoscaling + +_Mutable: yes_ + +Node pool autoscaling dynamically creates or deletes nodes based on the demands of your workload. For more information, see [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-autoscaler) + +### Auto Repair + +_Mutable: yes_ + +GKE's node auto-repair feature helps you keep the nodes in your cluster in a healthy, running state. When enabled, GKE makes periodic checks on the health state of each node in your cluster. If a node fails consecutive health checks over an extended time period, GKE initiates a repair process for that node. For more information, see the section on [auto-repairing nodes.](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-repair) + +### Auto Upgrade + +_Mutable: yes_ + +When enabled, the auto-upgrade feature keeps the nodes in your cluster up-to-date with the cluster control plane (master) version when your control plane is [updated on your behalf.](https://cloud.google.com/kubernetes-engine/upgrades#automatic_cp_upgrades) For more information about auto-upgrading nodes, see [this page.](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades) + +### Access Scopes + +_Mutable: no_ + +Access scopes are the legacy method of specifying permissions for your nodes. + +- **Allow default access:** The default access for new clusters is the [Compute Engine default service account.](https://cloud.google.com/compute/docs/access/service-accounts?hl=en_US#default_service_account) +- **Allow full access to all Cloud APIs:** Generally, you can just set the cloud-platform access scope to allow full access to all Cloud APIs, then grant the service account only relevant IAM roles. The combination of access scopes granted to the virtual machine instance and the IAM roles granted to the service account determines the amount of access the service account has for that instance. +- **Set access for each API:** Alternatively, you can choose to set specific scopes that permit access to the particular API methods that the service will call. + +For more information, see the [section about enabling service accounts for a VM.](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) + + +### Configuring the Refresh Interval + +The refresh interval can be configured through the setting "gke-refresh", which is an integer representing seconds. + +The default value is 300 seconds. + +The syncing interval can be changed by running `kubectl edit setting gke-refresh`. + +The shorter the refresh window, the less likely any race conditions will occur, but it does increase the likelihood of encountering request limits that may be in place for GCP APIs. + diff --git a/versioned_docs/version-2.8/pages-for-subheaders/helm-charts-in-rancher.md b/versioned_docs/version-2.8/pages-for-subheaders/helm-charts-in-rancher.md new file mode 100644 index 00000000000..f0d7ba63fca --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/helm-charts-in-rancher.md @@ -0,0 +1,156 @@ +--- +title: Helm Charts in Rancher +--- + + + + + +In this section, you'll learn how to manage Helm chart repositories and applications in Rancher. Helm chart repositories are managed using **Apps**. It uses a catalog-like system to import bundles of charts from repositories and then uses those charts to either deploy custom Helm applications or Rancher's tools such as Monitoring or Istio. Rancher tools come as pre-loaded repositories which deploy as standalone Helm charts. Any additional repositories are only added to the current cluster. + +### Versioning Scheme + +The Rancher feature charts versioning scheme is centered around the major version of the charts and the `+up` annotation for upstream charts, where applicable. + +**Major Version:** The major version of the charts is tied to Rancher minor versions. When you upgrade to a new Rancher minor version, you should ensure that all of your **Apps** charts are also upgraded to the correct release line for the chart. + +**Feature Charts:** + +| **Name** | **Supported Minimum Version** | **Supported Maximum Version** | +| ---------------- | ------------ | ------------ | +| external-ip-webhook | 100.0.0+up1.0.0 | 100.0.1+up1.0.1 | +| harvester-cloud-provider | 100.0.2+up0.1.12 | 100.0.2+up0.1.12 | +| harvester-csi-driver | 100.0.2+up0.1.11 | 100.0.2+up0.1.11 | +| neuvector | 100.0.0+up2.2.0 | 100.0.0+up2.2.0 | +| rancher-alerting-drivers | 100.0.0 | 100.0.2 | +| rancher-backup | 2.0.1 | 2.1.2 | +| rancher-cis-benchmark | 2.0.1 | 2.0.4 | +| rancher-gatekeeper | 100.0.0+up3.6.0 | 100.1.0+up3.7.1 | +| rancher-istio | 100.0.0+up1.10.4 | 100.3.0+up1.13.3 | +| rancher-logging | 100.0.0+up3.12.0 | 100.1.2+up3.17.4 | +| rancher-longhorn | 100.0.0+up1.1.2 | 100.1.2+up1.2.4 | +| rancher-monitoring | 100.0.0+up16.6.0 | 100.1.2+up19.0.3 | +| rancher-sriov (experimental) | 100.0.0+up0.1.0 | 100.0.3+up0.1.0 | +| rancher-vsphere-cpi | 100.3.0+up1.2.1 | 100.3.0+up1.2.1 | +| rancher-vsphere-csi | 100.3.0+up2.5.1-rancher1 | 100.3.0+up2.5.1-rancher1 | +| rancher-wins-upgrader | 0.0.100 | 100.0.1+up0.0.1 | + +
+**Charts based on upstream:** For charts that are based on upstreams, the +up annotation should inform you of what upstream version the Rancher chart is tracking. Check the upstream version compatibility with Rancher during upgrades also. + +- As an example, `100.x.x+up16.6.0` for Monitoring tracks upstream kube-prometheus-stack `16.6.0` with some Rancher patches added to it. + +- On upgrades, ensure that you are not downgrading the version of the chart that you are using. For example, if you are using a version of Monitoring > `16.6.0` in Rancher 2.5, you should not upgrade to `100.x.x+up16.6.0`. Instead, you should upgrade to the appropriate version in the next release. + +### Prerelease Versions + +Prereleases adhere to [the specification](https://semver.org/#spec-item-9) defined by [Semantic Versioning 2.0.0](https://semver.org/). For example, a Helm chart with a version of `0.1.3-dev.12ab4f` is considered a prerelease. Prerelease versions are not displayed by default and must be configured to do so. + +To display prerelease versions: + +1. Click on your user avatar in the upper right corner. +1. Click **Preferences**. +1. Under **Helm Charts**, select **Include Prerelease Versions**. + +### Charts + +From the top-left menu select _"Apps"_ and you will be taken to the Charts page. + +The charts page contains all Rancher, Partner, and Custom Charts. + +* Rancher tools such as Logging or Monitoring are included under the Rancher label +* Partner charts reside under the Partners label +* Custom charts will show up under the name of the repository + +All three types are deployed and managed in the same way. + +:::note + +Apps managed by the Cluster Manager (the global view in the legacy Rancher UI) should continue to be managed only by the Cluster Manager, and apps managed with Apps in the new UI must be managed only by Apps. + +::: + +### Repositories + +From the left sidebar select _"Repositories"_. + +These items represent helm repositories, and can be either traditional helm endpoints which have an index.yaml, or git repositories which will be cloned and can point to a specific branch. In order to use custom charts, simply add your repository here and they will become available in the Charts tab under the name of the repository. + +To add a private CA for Helm Chart repositories: + +- **HTTP-based chart repositories**: You must add a base64 encoded copy of the CA certificate in DER format to the spec.caBundle field of the chart repo, such as `openssl x509 -outform der -in ca.pem | base64 -w0`. Click **Edit YAML** for the chart repo and set, as in the following example:
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + + +- **Git-based chart repositories**: You must add a base64 encoded copy of the CA certificate in DER format to the spec.caBundle field of the chart repo, such as `openssl x509 -outform der -in ca.pem | base64 -w0`. Click **Edit YAML** for the chart repo and set, as in the following example:
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + +:::note Helm chart repositories with authentication + +The Repo.Spec contains a `disableSameOriginCheck` value that allows users to bypass the same origin checks, sending the repository Authentication information as a Basic Auth Header with all API calls. This is not recommended but can be used as a temporary solution in cases of non-standard Helm chart repositories such as those that have redirects to a different origin URL. + +To use this feature for an existing Helm chart repository, click ⋮ > Edit YAML. On the `spec` portion of the YAML file, add `disableSameOriginCheck` and set it to `true`. + +```yaml +[...] +spec: + disableSameOriginCheck: true +[...] +``` + +::: + +### Helm Compatibility + +Only Helm 3 compatible charts are supported. + + +### Deployment and Upgrades + +From the _"Charts"_ tab select a Chart to install. Rancher and Partner charts may have extra configurations available through custom pages or questions.yaml files, but all chart installations can modify the values.yaml and other basic settings. Once you click install, a Helm operation job is deployed, and the console for the job is displayed. + +To view all recent changes, go to the _"Recent Operations"_ tab. From there you can view the call that was made, conditions, events, and logs. + +After installing a chart, you can find it in the _"Installed Apps"_ tab. In this section you can upgrade or delete the installation, and see further details. When choosing to upgrade, the form and values presented will be the same as installation. + +Most Rancher tools have additional pages located in the toolbar below the _"Apps"_ section to help manage and use the features. These pages include links to dashboards, forms to easily add Custom Resources, and additional information. + +:::caution + +If you are upgrading your chart using _"Customize Helm options before upgrade"_ , please be aware that using the _"--force"_ option may result in errors if your chart has immutable fields. This is because some objects in Kubernetes cannot be changed once they are created. To ensure you do not get this error you can: + + * use the default upgrade option ( i.e do not use _"--force"_ option ) + * uninstall the existing chart and install the upgraded chart + * delete the resources with immutable fields from the cluster before performing the _"--force"_ upgrade + +::: + +#### Legacy Apps + +The upgrade button has been removed for legacy apps from the **Apps > Installed Apps** page. + +If you have a legacy app installed and want to upgrade it: + +- The legacy [feature flag](enable-experimental-features.md) must be turned on (if it's not turned on automatically because of having a legacy app before upgrading) +- You can upgrade the app from cluster explorer, from the left nav section **Legacy > Project > Apps** +- For multi-cluster apps, you can go to **≡ > Multi-cluster Apps** and upgrade the app from there + +### Limitations + +[Dashboard apps or Rancher feature charts](helm-charts-in-rancher.md) **cannot** be installed using the Rancher CLI. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/horizontal-pod-autoscaler.md b/versioned_docs/version-2.8/pages-for-subheaders/horizontal-pod-autoscaler.md new file mode 100644 index 00000000000..d18ce147d06 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/horizontal-pod-autoscaler.md @@ -0,0 +1,30 @@ +--- +title: The Horizontal Pod Autoscaler +description: Learn about the horizontal pod autoscaler (HPA). How to manage HPAs and how to test them with a service deployment +--- + + + + + +The [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) (HPA) is a Kubernetes feature that allows you to configure your cluster to automatically scale the services it's running up or down. + +Rancher provides some additional features to help manage HPAs, depending on the version of Rancher. + +You can create, manage, and delete HPAs using the Rancher UI. It only supports HPA in the `autoscaling/v2beta2` API. + +## Managing HPAs + +The way that you manage HPAs is different based on your version of the Kubernetes API: + +- **For Kubernetes API version autoscaling/V2beta1:** This version of the Kubernetes API lets you autoscale your pods based on the CPU and memory utilization of your application. +- **For Kubernetes API Version autoscaling/V2beta2:** This version of the Kubernetes API lets you autoscale your pods based on CPU and memory utilization, in addition to custom metrics. + +You can create, manage, and delete HPAs using the Rancher UI. From the Rancher UI you can configure the HPA to scale based on CPU and memory utilization. For more information, refer to [Managing HPAs with the Rancher UI](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md). To scale the HPA based on custom metrics, you still need to use `kubectl`. For more information, refer to [Configuring HPA to Scale Using Custom Metrics with Prometheus](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md#configuring-hpa-to-scale-using-custom-metrics-with-prometheus). + +Clusters created in Rancher v2.0.7 and higher automatically have all the requirements needed (metrics-server and Kubernetes cluster configuration) to use HPA. +## Testing HPAs with a Service Deployment + +You can see your HPA's current number of replicas by going to your project and clicking **Resources > HPA**. For more information, refer to [Get HPA Metrics and Status](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md). + +You can also use `kubectl` to get the status of HPAs that you test with your load testing tool. For more information, refer to [Testing HPAs with kubectl](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md). diff --git a/versioned_docs/version-2.8/pages-for-subheaders/infrastructure-setup.md b/versioned_docs/version-2.8/pages-for-subheaders/infrastructure-setup.md new file mode 100644 index 00000000000..fabdc72e975 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/infrastructure-setup.md @@ -0,0 +1,12 @@ +--- +title: Don't have infrastructure for your Kubernetes cluster? Try one of these tutorials. +--- + + + + + +To set up infrastructure for a high-availability K3s Kubernetes cluster with an external DB, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md) + + +To set up infrastructure for a high-availability RKE Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/install-cluster-autoscaler.md b/versioned_docs/version-2.8/pages-for-subheaders/install-cluster-autoscaler.md new file mode 100644 index 00000000000..8b749aae5ee --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/install-cluster-autoscaler.md @@ -0,0 +1,28 @@ +--- +title: Cluster Autoscaler +--- + + + + + +In this section, you'll learn how to install and use the [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/) on Rancher custom clusters using AWS EC2 Auto Scaling Groups. + +The cluster autoscaler is a tool that automatically adjusts the size of the Kubernetes cluster when one of the following conditions is true: + +* There are pods that failed to run in the cluster due to insufficient resources. +* There are nodes in the cluster that have been underutilized for an extended period of time and their pods can be placed on other existing nodes. + +To prevent your pod from being evicted, set a `priorityClassName: system-cluster-critical` property on your pod spec. + +Cluster Autoscaler is designed to run on Kubernetes master nodes. It can run in the `kube-system` namespace. Cluster Autoscaler doesn't scale down nodes with non-mirrored `kube-system` pods running on them. + +It's possible to run a customized deployment of Cluster Autoscaler on worker nodes, but extra care needs to be taken to ensure that Cluster Autoscaler remains up and running. + +## Cloud Providers + +Cluster Autoscaler provides support to distinct cloud providers. For more information, go to [cluster-autoscaler supported cloud providers.](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#deployment) + +### Setting up Cluster Autoscaler on Amazon Cloud Provider + +For details on running the cluster autoscaler on Amazon cloud provider, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md b/versioned_docs/version-2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md new file mode 100644 index 00000000000..4eda613d25f --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md @@ -0,0 +1,356 @@ +--- +title: Install/Upgrade Rancher on a Kubernetes Cluster +description: Learn how to install Rancher in development and production environments. Read about single node and high availability installation +--- + + + + + +In this section, you'll learn how to deploy Rancher on a Kubernetes cluster using the Helm CLI. + + +## Prerequisites + +- [Kubernetes Cluster](#kubernetes-cluster) +- [Ingress Controller](#ingress-controller) +- [CLI Tools](#cli-tools) + +### Kubernetes Cluster + +Set up the Rancher server's local Kubernetes cluster. + +Rancher can be installed on any Kubernetes cluster. This cluster can use upstream Kubernetes, or it can use one of Rancher's Kubernetes distributions, or it can be a managed Kubernetes cluster from a provider such as Amazon EKS. + +For help setting up a Kubernetes cluster, we provide these tutorials: + +- **RKE:** For the tutorial to install an RKE Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md) For help setting up the infrastructure for a high-availability RKE cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md) +- **K3s:** For the tutorial to install a K3s Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md) For help setting up the infrastructure for a high-availability K3s cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md) +- **RKE2:** For the tutorial to install an RKE2 Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md) For help setting up the infrastructure for a high-availability RKE2 cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md) +- **Amazon EKS:** For details on how to install Rancher on Amazon EKS, including how to install an Ingress controller so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md) +- **AKS:** For details on how to install Rancher with Azure Kubernetes Service, including how to install an Ingress controller so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md) +- **GKE:** For details on how to install Rancher with Google Kubernetes Engine, including how to install an Ingress controller so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md) GKE has two modes of operation when creating a Kubernetes cluster, Autopilot and Standard mode. The cluster configuration for Autopilot mode has restrictions on editing the kube-system namespace. However, Rancher needs to create resources in the kube-system namespace during installation. As a result, you will not be able to install Rancher on a GKE cluster created in Autopilot mode. + + +### Ingress Controller + +The Rancher UI and API are exposed through an Ingress. This means the Kubernetes cluster that you install Rancher in must contain an Ingress controller. + +For RKE, RKE2, and K3s installations, you don't have to install the Ingress controller manually because one is installed by default. + +For distributions that do not include an Ingress Controller by default, like a hosted Kubernetes cluster such as EKS, GKE, or AKS, you have to deploy an Ingress controller first. Note that the Rancher Helm chart does not set an `ingressClassName` on the ingress by default. Because of this, you have to configure the Ingress controller to also watch ingresses without an `ingressClassName`. + +Examples are included in the **Amazon EKS**, **AKS**, and **GKE** tutorials above. + +### CLI Tools + +The following CLI tools are required for setting up the Kubernetes cluster. Please make sure these tools are installed and available in your `$PATH`. + +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) - Kubernetes command-line tool. +- [helm](https://docs.helm.sh/using_helm/#installing-helm) - Package management for Kubernetes. Refer to the [Helm version requirements](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md) to choose a version of Helm to install Rancher. Refer to the [instructions provided by the Helm project](https://helm.sh/docs/intro/install/) for your specific platform. + +## Install the Rancher Helm Chart + +Rancher is installed using the [Helm](https://helm.sh/) package manager for Kubernetes. Helm charts provide templating syntax for Kubernetes YAML manifest documents. With Helm, we can create configurable deployments instead of just using static files. + +For systems without direct internet access, see [Air Gap: Kubernetes install](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md). + +To choose a Rancher version to install, refer to [Choosing a Rancher Version.](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md) + +To choose a version of Helm to install Rancher with, refer to the [Helm version requirements](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md) + +:::note + +The installation instructions assume you are using Helm 3. + +::: + +To set up Rancher, + +1. [Add the Helm chart repository](#1-add-the-helm-chart-repository) +2. [Create a namespace for Rancher](#2-create-a-namespace-for-rancher) +3. [Choose your SSL configuration](#3-choose-your-ssl-configuration) +4. [Install cert-manager](#4-install-cert-manager) (unless you are bringing your own certificates, or TLS will be terminated on a load balancer) +5. [Install Rancher with Helm and your chosen certificate option](#5-install-rancher-with-helm-and-your-chosen-certificate-option) +6. [Verify that the Rancher server is successfully deployed](#6-verify-that-the-rancher-server-is-successfully-deployed) +7. [Save your options](#7-save-your-options) + +### 1. Add the Helm Chart Repository + +Use `helm repo add` command to add the Helm chart repository that contains charts to install Rancher. For more information about the repository choices and which is best for your use case, see [Choosing a Rancher Version](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md). + +- Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + +### 2. Create a Namespace for Rancher + +We'll need to define a Kubernetes namespace where the resources created by the Chart should be installed. This should always be `cattle-system`: + +``` +kubectl create namespace cattle-system +``` + +### 3. Choose your SSL Configuration + +The Rancher management server is designed to be secure by default and requires SSL/TLS configuration. + +:::note + +If you want terminate SSL/TLS externally, see [TLS termination on an External Load Balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination). + +::: + +There are three recommended options for the source of the certificate used for TLS termination at the Rancher server: + +- **Rancher-generated TLS certificate:** In this case, you will need to install `cert-manager` into the cluster. Rancher utilizes `cert-manager` to issue and maintain its certificates. Rancher will generate a CA certificate of its own, and sign a cert using that CA. `cert-manager` is then responsible for managing that certificate. +- **Let's Encrypt:** The Let's Encrypt option also uses `cert-manager`. However, in this case, cert-manager is combined with a special Issuer for Let's Encrypt that performs all actions (including request and validation) necessary for getting a Let's Encrypt issued cert. This configuration uses HTTP validation (`HTTP-01`), so the load balancer must have a public DNS record and be accessible from the internet. +- **Bring your own certificate:** This option allows you to bring your own public- or private-CA signed certificate. Rancher will use that certificate to secure websocket and HTTPS traffic. In this case, you must upload this certificate (and associated key) as PEM-encoded files with the name `tls.crt` and `tls.key`. If you are using a private CA, you must also upload that certificate. This is due to the fact that this private CA may not be trusted by your nodes. Rancher will take that CA certificate, and generate a checksum from it, which the various Rancher components will use to validate their connection to Rancher. + + +| Configuration | Helm Chart Option | Requires cert-manager | +| ------------------------------ | ----------------------- | ------------------------------------- | +| Rancher Generated Certificates (Default) | `ingress.tls.source=rancher` | [yes](#4-install-cert-manager) | +| Let’s Encrypt | `ingress.tls.source=letsEncrypt` | [yes](#4-install-cert-manager) | +| Certificates from Files | `ingress.tls.source=secret` | no | + +### 4. Install cert-manager + +> You should skip this step if you are bringing your own certificate files (option `ingress.tls.source=secret`), or if you use [TLS termination on an external load balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination). + +This step is only required to use certificates issued by Rancher's generated CA (`ingress.tls.source=rancher`) or to request Let's Encrypt issued certificates (`ingress.tls.source=letsEncrypt`). + +
+ Click to Expand + +:::note Important: + +Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.11.0, please see our [upgrade documentation](../getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md). + +::: + +These instructions are adapted from the [official cert-manager documentation](https://cert-manager.io/docs/installation/kubernetes/#installing-with-helm). + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +``` +# If you have installed the CRDs manually instead of with the `--set installCRDs=true` option added to your Helm install command, you should upgrade your CRD resources before upgrading the Helm chart: +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +# Add the Jetstack Helm repository +helm repo add jetstack https://charts.jetstack.io + +# Update your local Helm chart repository cache +helm repo update + +# Install the cert-manager Helm chart +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 +``` + +Once you’ve installed cert-manager, you can verify it is deployed correctly by checking the cert-manager namespace for running pods: + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +
+ +### 5. Install Rancher with Helm and Your Chosen Certificate Option + +The exact command to install Rancher differs depending on the certificate configuration. + +However, irrespective of the certificate configuration, the name of the Rancher installation in the `cattle-system` namespace should always be `rancher`. + +:::tip Testing and Development: + +This final command to install Rancher requires a domain name that forwards traffic to Rancher. If you are using the Helm CLI to set up a proof-of-concept, you can use a fake domain name when passing the `hostname` option. An example of a fake domain name would be `.sslip.io`, which would expose Rancher on an IP where it is running. Production installs would require a real domain name. + +::: + + + + +The default is for Rancher to generate a CA and uses `cert-manager` to issue the certificate for access to the Rancher server interface. + +Because `rancher` is the default option for `ingress.tls.source`, we are not specifying `ingress.tls.source` when running the `helm install` command. + +- Set the `hostname` to the DNS name you pointed at your load balancer. +- Set the `bootstrapPassword` to something unique for the `admin` user. +- To install a specific Rancher version, use the `--version` flag, example: `--version 2.7.0` +- For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin +``` + +If you are installing an alpha version, Helm requires adding the `--devel` option to the install command: + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +Wait for Rancher to be rolled out: + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + + +This option uses `cert-manager` to automatically request and renew [Let's Encrypt](https://letsencrypt.org/) certificates. This is a free service that provides you with a valid certificate as Let's Encrypt is a trusted CA. + +:::note + +You need to have port 80 open as the HTTP-01 challenge can only be done on port 80. + +::: + +In the following command, + +- `hostname` is set to the public DNS record, +- Set the `bootstrapPassword` to something unique for the `admin` user. +- `ingress.tls.source` is set to `letsEncrypt` +- `letsEncrypt.email` is set to the email address used for communication about your certificate (for example, expiry notices) +- Set `letsEncrypt.ingress.class` to whatever your ingress controller is, e.g., `traefik`, `nginx`, `haproxy`, etc. +- For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=letsEncrypt \ + --set letsEncrypt.email=me@example.org \ + --set letsEncrypt.ingress.class=nginx +``` + +If you are installing an alpha version, Helm requires adding the `--devel` option to the install command: + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +Wait for Rancher to be rolled out: + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + +In this option, Kubernetes secrets are created from your own certificates for Rancher to use. + +When you run this command, the `hostname` option must match the `Common Name` or a `Subject Alternative Names` entry in the server certificate or the Ingress controller will fail to configure correctly. + +Although an entry in the `Subject Alternative Names` is technically required, having a matching `Common Name` maximizes compatibility with older browsers and applications. + +:::note + +If you want to check if your certificates are correct, see [How do I check Common Name and Subject Alternative Names in my server certificate?](../faq/technical-items.md#how-do-i-check-common-name-and-subject-alternative-names-in-my-server-certificate) + +::: + +- Set the `hostname`. +- Set the `bootstrapPassword` to something unique for the `admin` user. +- Set `ingress.tls.source` to `secret`. +- For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret +``` +If you are installing an alpha version, Helm requires adding the `--devel` option to the install command: + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +If you are using a Private CA signed certificate , add `--set privateCA=true` to the command: + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret \ + --set privateCA=true +``` + +Now that Rancher is deployed, see [Adding TLS Secrets](../getting-started/installation-and-upgrade/resources/add-tls-secrets.md) to publish the certificate files so Rancher and the Ingress controller can use them. + + + + +The Rancher chart configuration has many options for customizing the installation to suit your specific environment. Here are some common advanced scenarios. + +- [HTTP Proxy](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#http-proxy) +- [Private Container Image Registry](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#private-registry-and-air-gap-installs) +- [TLS Termination on an External Load Balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +See the [Chart Options](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md) for the full list of options. + + +### 6. Verify that the Rancher Server is Successfully Deployed + +After adding the secrets, check if Rancher was rolled out successfully: + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + +If you see the following error: `error: deployment "rancher" exceeded its progress deadline`, you can check the status of the deployment by running the following command: + +``` +kubectl -n cattle-system get deploy rancher +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +rancher 3 3 3 3 3m +``` + +It should show the same count for `DESIRED` and `AVAILABLE`. + +### 7. Save Your Options + +Make sure you save the `--set` options you used. You will need to use the same options when you upgrade Rancher to new versions with Helm. + +### Finishing Up + +That's it. You should have a functional Rancher server. + +In a web browser, go to the DNS name that forwards traffic to your load balancer. Then you should be greeted by the colorful login page. + +Doesn't work? Take a look at the [Troubleshooting](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) Page diff --git a/versioned_docs/version-2.8/pages-for-subheaders/installation-and-upgrade.md b/versioned_docs/version-2.8/pages-for-subheaders/installation-and-upgrade.md new file mode 100644 index 00000000000..3077b14edc2 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/installation-and-upgrade.md @@ -0,0 +1,98 @@ +--- +title: Installing/Upgrading Rancher +description: Learn how to install Rancher in development and production environments. Read about single node and high availability installation +--- + + + + + +This section provides an overview of the architecture options of installing Rancher, describing advantages of each option. + +## Terminology + +In this section, + +- **The Rancher server** manages and provisions Kubernetes clusters. You can interact with downstream Kubernetes clusters through the Rancher server's user interface. The Rancher management server can be installed on any Kubernetes cluster, including hosted clusters, such as Amazon EKS clusters. +- **RKE (Rancher Kubernetes Engine)** is a certified Kubernetes distribution and CLI/library which creates and manages a Kubernetes cluster. +- **K3s (Lightweight Kubernetes)** is also a fully compliant Kubernetes distribution. It is newer than RKE, easier to use, and more lightweight, with a binary size of less than 100 MB. +- **RKE2** is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. + +Note the `restrictedAdmin` Helm chart option available for **the Rancher Server**. When this option is set to true, the initial Rancher user has restricted access to the local Kubernetes cluster to prevent privilege escalation. For more information, see the section about the [restricted-admin role.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#restricted-admin) + +## Overview of Installation Options + +Rancher can be installed on these main architectures: + +### High-availability Kubernetes Install with the Helm CLI + +We recommend using Helm, a Kubernetes package manager, to install Rancher on multiple nodes on a dedicated Kubernetes cluster. For RKE clusters, three nodes are required to achieve a high-availability cluster. For K3s clusters, only two nodes are required. + +### Rancher on EKS Install with the AWS Marketplace + +Rancher can be installed on to Amazon Elastic Kubernetes Service (EKS) [through the AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md). The EKS cluster deployed is production-ready and follows AWS best practices. + +### Single-node Kubernetes Install + +Rancher can be installed on a single-node Kubernetes cluster. In this case, the Rancher server doesn't have high availability, which is important for running Rancher in production. + +However, this option is useful if you want to save resources by using a single node in the short term, while preserving a high-availability migration path. In the future, you can add nodes to the cluster to get a high-availability Rancher server. + +### Docker Install + +For test and demonstration purposes, Rancher can be installed with Docker on a single node. A local Kubernetes cluster is installed in the single Docker container, and Rancher is installed on the local cluster. + +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +### Other Options + +There are also separate instructions for installing Rancher in an air gap environment or behind an HTTP proxy: + +| Level of Internet Access | Kubernetes Installation - Strongly Recommended | Docker Installation | +| ---------------------------------- | ------------------------------ | ---------- | +| With direct access to the Internet | [Docs](install-upgrade-on-a-kubernetes-cluster.md) | [Docs](rancher-on-a-single-node-with-docker.md) | +| Behind an HTTP proxy | [Docs](rancher-behind-an-http-proxy.md) | These [docs,](rancher-on-a-single-node-with-docker.md) plus this [configuration](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) | +| In an air gap environment | [Docs](air-gapped-helm-cli-install.md) | [Docs](air-gapped-helm-cli-install.md) | + +We recommend installing Rancher on a Kubernetes cluster, because in a multi-node cluster, the Rancher management server becomes highly available. This high-availability configuration helps maintain consistent access to the downstream Kubernetes clusters that Rancher will manage. + +For that reason, we recommend that for a production-grade architecture, you should set up a high-availability Kubernetes cluster, then install Rancher on it. After Rancher is installed, you can use Rancher to deploy and manage Kubernetes clusters. + +For testing or demonstration purposes, you can install Rancher in single Docker container. In this Docker install, you can use Rancher to set up Kubernetes clusters out-of-the-box. The Docker install allows you to explore the Rancher server functionality, but it is intended to be used for development and testing purposes only. + +Our [instructions for installing Rancher on Kubernetes](install-upgrade-on-a-kubernetes-cluster.md) describe how to first use K3s or RKE to create and manage a Kubernetes cluster, then install Rancher onto that cluster. + +When the nodes in your Kubernetes cluster are running and fulfill the [node requirements,](installation-requirements.md) you will use Helm to deploy Rancher onto Kubernetes. Helm uses Rancher's Helm chart to install a replica of Rancher on each node in the Kubernetes cluster. We recommend using a load balancer to direct traffic to each replica of Rancher in the cluster. + +For a longer discussion of Rancher architecture, refer to the [architecture overview,](rancher-manager-architecture.md) [recommendations for production-grade architecture,](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) or our [best practices guide.](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md) + +## Prerequisites + +Before installing Rancher, make sure that your nodes fulfill all of the [installation requirements.](installation-requirements.md) + +## Architecture Tip + +For the best performance and greater security, we recommend a separate, dedicated Kubernetes cluster for the Rancher management server. Running user workloads on this cluster is not advised. After deploying Rancher, you can [create or import clusters](kubernetes-clusters-in-rancher-setup.md) for running your workloads. + +For more architecture recommendations, refer to [this page.](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) + +### More Options for Installations on a Kubernetes Cluster + +Refer to the [Helm chart options](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md) for details on installing Rancher on a Kubernetes cluster with other configurations, including: + +- With [API auditing to record all transactions](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-audit-log) +- With [TLS termination on a load balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) +- With a [custom Ingress](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#customizing-your-ingress) + +In the Rancher installation instructions, we recommend using K3s or RKE to set up a Kubernetes cluster before installing Rancher on the cluster. Both K3s and RKE have many configuration options for customizing the Kubernetes cluster to suit your specific environment. For the full list of their capabilities, refer to their documentation: + +- [RKE configuration options](https://rancher.com/docs/rke/latest/en/config-options/) +- [K3s configuration options](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +### More Options for Installations with Docker + +Refer to the [docs about options for Docker installs](rancher-on-a-single-node-with-docker.md) for details about other configurations including: + +- With [API auditing to record all transactions](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) +- With an [external load balancer](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md) +- With a [persistent data store](../reference-guides/single-node-rancher-in-docker/advanced-options.md#persistent-data) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/installation-references.md b/versioned_docs/version-2.8/pages-for-subheaders/installation-references.md new file mode 100644 index 00000000000..6108728b04f --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/installation-references.md @@ -0,0 +1,9 @@ +--- +title: Installation References +--- + + + + + +Please see the following reference guides for other installation resources: [Rancher Helm chart options](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md), [TLS settings](../getting-started/installation-and-upgrade/installation-references/tls-settings.md), and [feature flags](../getting-started/installation-and-upgrade/installation-references/feature-flags.md). \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/installation-requirements.md b/versioned_docs/version-2.8/pages-for-subheaders/installation-requirements.md new file mode 100644 index 00000000000..b7214336b13 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/installation-requirements.md @@ -0,0 +1,159 @@ +--- +title: Installation Requirements +description: Learn the node requirements for each node running Rancher server when you’re configuring Rancher to run either in a Docker or Kubernetes setup +--- + + + + + +This page describes the software, hardware, and networking requirements for the nodes where the Rancher server will be installed. The Rancher server can be installed on a single node or a high-availability Kubernetes cluster. + +:::note Important: + +If you install Rancher on a Kubernetes cluster, requirements are different from the [node requirements for downstream user clusters,](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) which will run your apps and services. + +::: + +The Rancher UI works best in Firefox or Chromium based browsers (Chrome, Edge, Opera, Brave, etc). + +See our page on [best practices](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md) for a list of recommendations for running a Rancher server in production. + +## Kubernetes Compatibility with Rancher + +Rancher needs to be installed on a supported Kubernetes version. Consult the [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions) to ensure that your intended version of Kubernetes is supported. + +## Operating Systems and Container Runtime Requirements + +All supported operating systems are 64-bit x86. Rancher should work with any modern Linux distribution. + +The [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions) lists which OS and Docker versions were tested for each Rancher version. + +Docker is required for nodes that will run RKE clusters. It is not required for RKE2 or K3s clusters. + +The `ntp` (Network Time Protocol) package should be installed. This prevents errors with certificate validation that can occur when the time is not synchronized between the client and server. + +Some distributions of Linux may have default firewall rules that block communication within the Kubernetes cluster. Since Kubernetes v1.19, firewalld must be turned off, because it conflicts with the Kubernetes networking plugins. + +If you don't feel comfortable doing so, you might check suggestions in the [respective issue](https://github.com/rancher/rancher/issues/28840). Some users were successful [creating a separate firewalld zone with a policy of ACCEPT for the Pod CIDR](https://github.com/rancher/rancher/issues/28840#issuecomment-787404822). + +If you plan to run Rancher on ARM64, see [Running on ARM64 (Experimental).](../how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md) + +### RKE Specific Requirements + +For the container runtime, RKE should work with any modern Docker version. + +For more information see [Installing Docker,](../getting-started/installation-and-upgrade/installation-requirements/install-docker.md) + +### K3s Specific Requirements + +For the container runtime, K3s bundles its own containerd by default. Alternatively, you can configure K3s to use an already installed Docker runtime. For more information on using K3s with Docker see the [K3s documentation.](https://docs.k3s.io/advanced#using-docker-as-the-container-runtime) + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions). To specify the K3s version, use the INSTALL_K3S_VERSION environment variable when running the K3s installation script. + +If you are installing Rancher on a K3s cluster with **Raspbian Buster**, follow [these steps](https://rancher.com/docs/k3s/latest/en/advanced/#enabling-legacy-iptables-on-raspbian-buster) to switch to legacy iptables. + +If you are installing Rancher on a K3s cluster with Alpine Linux, follow [these steps](https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-alpine-linux-setup) for additional setup. + +### RKE2 Specific Requirements + +For the container runtime, RKE2 bundles its own containerd. Docker is not required for RKE2 installs. + +For details on which OS versions were tested with RKE2, refer to the [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions). + +## Hardware Requirements + +The following sections describe the CPU, memory, and disk requirements for the nodes where the Rancher server is installed. + +## CPU and Memory + +Hardware requirements scale based on the size of your Rancher deployment. Provision each individual node according to the requirements. The requirements are different depending on if you are installing Rancher in a single container with Docker, or if you are installing Rancher on a Kubernetes cluster. + +### RKE and Hosted Kubernetes + +These CPU and memory requirements apply to each host in the Kubernetes cluster where the Rancher server is installed. + +These requirements apply to RKE Kubernetes clusters, as well as to hosted Kubernetes clusters such as EKS. + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | +| --------------- | ---------- | ------------ | -------| ------- | +| Small | Up to 150 | Up to 1500 | 2 | 8 GB | +| Medium | Up to 300 | Up to 3000 | 4 | 16 GB | +| Large | Up to 500 | Up to 5000 | 8 | 32 GB | +| X-Large | Up to 1000 | Up to 10,000 | 16 | 64 GB | +| XX-Large | Up to 2000 | Up to 20,000 | 32 | 128 GB | + +Every use case and environment is different. Please [contact Rancher](https://rancher.com/contact/) to review yours. + +### K3s Kubernetes + +These CPU and memory requirements apply to each host in a [K3s Kubernetes cluster where the Rancher server is installed.](install-upgrade-on-a-kubernetes-cluster.md) + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | Database Size | +| --------------- | ---------- | ------------ | -------| ---------| ------------------------- | +| Small | Up to 150 | Up to 1500 | 2 | 8 GB | 2 cores, 4 GB + 1000 IOPS | +| Medium | Up to 300 | Up to 3000 | 4 | 16 GB | 2 cores, 4 GB + 1000 IOPS | +| Large | Up to 500 | Up to 5000 | 8 | 32 GB | 2 cores, 4 GB + 1000 IOPS | +| X-Large | Up to 1000 | Up to 10,000 | 16 | 64 GB | 2 cores, 4 GB + 1000 IOPS | +| XX-Large | Up to 2000 | Up to 20,000 | 32 | 128 GB | 2 cores, 4 GB + 1000 IOPS | + +Every use case and environment is different. Please [contact Rancher](https://rancher.com/contact/) to review yours. + + +### RKE2 Kubernetes + +These CPU and memory requirements apply to each instance with RKE2 installed. Minimum recommendations are outlined here. + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | +| --------------- | -------- | --------- | ----- | ---- | +| Small | Up to 5 | Up to 50 | 2 | 5 GB | +| Medium | Up to 15 | Up to 200 | 3 | 9 GB | + +### Docker + +These CPU and memory requirements apply to a host with a [single-node](rancher-on-a-single-node-with-docker.md) installation of Rancher. + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | +| --------------- | -------- | --------- | ----- | ---- | +| Small | Up to 5 | Up to 50 | 1 | 4 GB | +| Medium | Up to 15 | Up to 200 | 2 | 8 GB | + +## Ingress + +Each node in the Kubernetes cluster that Rancher is installed on should run an Ingress. + +The Ingress should be deployed as DaemonSet to ensure your load balancer can successfully route traffic to all nodes. + +For RKE, RKE2 and K3s installations, you don't have to install the Ingress manually because it is installed by default. + +For hosted Kubernetes clusters (EKS, GKE, AKS), you will need to set up the ingress. + +- **Amazon EKS:** For details on how to install Rancher on Amazon EKS, including how to install an ingress so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md) +- **AKS:** For details on how to install Rancher with Azure Kubernetes Service, including how to install an ingress so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md) +- **GKE:** For details on how to install Rancher with Google Kubernetes Engine, including how to install an ingress so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md) + +## Disks + +Rancher performance depends on etcd in the cluster performance. To ensure optimal speed, we recommend always using SSD disks to back your Rancher management Kubernetes cluster. On cloud providers, you will also want to use the minimum size that allows the maximum IOPS. In larger clusters, consider using dedicated storage devices for etcd data and wal directories. + +## Networking Requirements + +This section describes the networking requirements for the node(s) where the Rancher server is installed. + +:::caution + +If a server containing Rancher has the `X-Frame-Options=DENY` header, some pages in the new Rancher UI will not be able to render after upgrading from the legacy UI. This is because some legacy pages are embedded as iFrames in the new UI. + +::: + +### Node IP Addresses + +Each node used should have a static IP configured, regardless of whether you are installing Rancher on a single node or on an HA cluster. In case of DHCP, each node should have a DHCP reservation to make sure the node gets the same IP allocated. + +### Port Requirements + +To operate properly, Rancher requires a number of ports to be open on Rancher nodes and on downstream Kubernetes cluster nodes. [Port Requirements](../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md) lists all the necessary ports for Rancher and Downstream Clusters for the different cluster types. + +## Dockershim Support + +For more information on Dockershim support, refer to [this page](../getting-started/installation-and-upgrade/installation-requirements/dockershim.md). diff --git a/versioned_docs/version-2.8/pages-for-subheaders/istio-setup-guide.md b/versioned_docs/version-2.8/pages-for-subheaders/istio-setup-guide.md new file mode 100644 index 00000000000..24475f7ffea --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/istio-setup-guide.md @@ -0,0 +1,34 @@ +--- +title: Setup Guide +--- + + + + + +This section describes how to enable Istio and start using it in your projects. + +If you use Istio for traffic management, you will need to allow external traffic to the cluster. In that case, you will need to follow all of the steps below. + +## Prerequisites + +This guide assumes you have already [installed Rancher,](installation-and-upgrade.md) and you have already [provisioned a separate Kubernetes cluster](kubernetes-clusters-in-rancher-setup.md) on which you will install Istio. + +The nodes in your cluster must meet the [CPU and memory requirements.](../integrations-in-rancher/istio/cpu-and-memory-allocations.md) + +The workloads and services that you want to be controlled by Istio must meet [Istio's requirements.](https://istio.io/docs/setup/additional-setup/requirements/) + +## Install + +:::tip Quick Setup Tip: + +If you don't need external traffic to reach Istio, and you just want to set up Istio for monitoring and tracing traffic within the cluster, skip the steps for [setting up the Istio gateway](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) and [setting up Istio's components for traffic management.](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md) + +::: + +1. [Enable Istio in the cluster.](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md) +1. [Enable Istio in all the namespaces where you want to use it.](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md) +1. [Add deployments and services that have the Istio sidecar injected.](../how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md) +1. [Set up the Istio gateway. ](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) +1. [Set up Istio's components for traffic management.](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md) +1. [Generate traffic and see Istio in action.](../how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/istio.md b/versioned_docs/version-2.8/pages-for-subheaders/istio.md new file mode 100644 index 00000000000..93f1e9b7cc5 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/istio.md @@ -0,0 +1,141 @@ +--- +title: Istio +--- + + + + + +[Istio](https://istio.io/) is an open-source tool that makes it easier for DevOps teams to observe, secure, control, and troubleshoot the traffic within a complex network of microservices. + +As a network of microservices changes and grows, the interactions between them can become increasingly difficult to manage and understand. In such a situation, it is useful to have a service mesh as a separate infrastructure layer. Istio's service mesh lets you manipulate traffic between microservices without changing the microservices directly. + +Our integration of Istio is designed so that a Rancher operator, such as an administrator or cluster owner, can deliver Istio to a team of developers. Then developers can use Istio to enforce security policies, troubleshoot problems, or manage traffic for green/blue deployments, canary deployments, or A/B testing. + +This core service mesh provides features that include but are not limited to the following: + +- **Traffic Management** such as ingress and egress routing, circuit breaking, mirroring. +- **Security** with resources to authenticate and authorize traffic and users, mTLS included. +- **Observability** of logs, metrics, and distributed traffic flows. + +After [setting up istio](istio-setup-guide.md) you can leverage Istio's control plane functionality through the Rancher UI, `kubectl`, or `istioctl`. + +Istio needs to be set up by a `cluster-admin` before it can be used in a project. + + +## What's New in Rancher v2.5 + +The overall architecture of Istio has been simplified. A single component, Istiod, has been created by combining Pilot, Citadel, Galley and the sidecar injector. Node Agent functionality has also been merged into istio-agent. + +Addons that were previously installed by Istio (cert-manager, Grafana, Jaeger, Kiali, Prometheus, Zipkin) will now need to be installed separately. Istio will support installation of integrations that are from the Istio Project and will maintain compatibility with those that are not. + +A Prometheus integration will still be available through an installation of [Rancher Monitoring](monitoring-and-alerting.md), or by installing your own Prometheus operator. Rancher's Istio chart will also install Kiali by default to ensure you can get a full picture of your microservices out of the box. + +Istio has migrated away from Helm as a way to install Istio and now provides installation through the istioctl binary or Istio Operator. To ensure the easiest interaction with Istio, Rancher's Istio will maintain a Helm chart that utilizes the istioctl binary to manage your Istio installation. + +This Helm chart will be available via the Apps and Marketplace in the UI. A user that has access to the Rancher Chart's catalog will need to set up Istio before it can be used in the project. + +## Tools Bundled with Istio + +Our [Istio](https://istio.io/) installer wraps the istioctl binary commands in a handy Helm chart, including an overlay file option to allow complex customization. + +It also includes the following: + +### Kiali + +Kiali is a comprehensive visualization aid used for graphing traffic flow throughout the service mesh. It allows you to see how they are connected, including the traffic rates and latencies between them. + +You can check the health of the service mesh, or drill down to see the incoming and outgoing requests to a single component. + +### Jaeger + +Our Istio installer includes a quick-start, all-in-one installation of [Jaeger,](https://www.jaegertracing.io/) a tool used for tracing distributed systems. + +Note that this is not a production-qualified deployment of Jaeger. This deployment uses an in-memory storage component, while a persistent storage component is recommended for production. For more information on which deployment strategy you may need, refer to the [Jaeger documentation.](https://www.jaegertracing.io/docs/latest/operator/#production-strategy) + +## Prerequisites + +Before enabling Istio, we recommend that you confirm that your Rancher worker nodes have enough [CPU and memory](../integrations-in-rancher/istio/cpu-and-memory-allocations.md) to run all of the components of Istio. + +If you are installing Istio on RKE2 cluster, some additional steps are required. For details, see [this section.](#additional-steps-for-installing-istio-on-an-rke2-cluster) + +## Setup Guide + +Refer to the [setup guide](istio-setup-guide.md) for instructions on how to set up Istio and use it in a project. + +## Remove Istio + +To remove Istio components from a cluster, namespace, or workload, refer to the section on [uninstalling Istio.](../integrations-in-rancher/istio/disable-istio.md) + +## Accessing Visualizations + +> By default, only cluster-admins have access to Kiali. For instructions on how to allow admin, edit or views roles to access them, see [this section.](../integrations-in-rancher/istio/rbac-for-istio.md) + +After Istio is set up in a cluster, Grafana, Prometheus, and Kiali are available in the Rancher UI. + +To access the Grafana and Prometheus visualizations, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Grafana** or any of the other dashboards. + +To access the Kiali visualization, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see Kiali and click **Explore**. +1. In the left navigation bar, click **Istio**. +1. Click **Kiali**. From here you can access the **Traffic Graph** tab or the **Traffic Metrics** tab to see network visualizations and metrics. + +By default, all namespace will picked up by prometheus and make data available for Kiali graphs. Refer to [selector/scrape config setup](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) if you would like to use a different configuration for prometheus data scraping. + +Your access to the visualizations depend on your role. Grafana and Prometheus are only available for `cluster-admin` roles. The Kiali UI is available only to `cluster-admin` by default, but `cluster-admin` can allow other roles to access them by editing the Istio values.yaml. + +## Architecture + +Istio installs a service mesh that uses [Envoy](https://www.envoyproxy.io) sidecar proxies to intercept traffic to each workload. These sidecars intercept and manage service-to-service communication, allowing fine-grained observation and control over traffic within the cluster. + +Only workloads that have the Istio sidecar injected can be tracked and controlled by Istio. + +When a namespace has Istio enabled, new workloads deployed in the namespace will automatically have the Istio sidecar. You need to manually enable Istio in preexisting workloads. + +For more information on the Istio sidecar, refer to the [Istio sidecare-injection docs](https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/) and for more information on Istio's architecture, refer to the [Istio Architecture docs](https://istio.io/latest/docs/ops/deployment/architecture/) + +### Multiple Ingresses + +By default, each Rancher-provisioned cluster has one NGINX ingress controller allowing traffic into the cluster. Istio also installs an ingress gateway by default into the `istio-system` namespace. The result is that your cluster will have two ingresses in your cluster. + +![In an Istio-enabled cluster, you can have two ingresses: the default Nginx ingress, and the default Istio controller.](/img/istio-ingress.svg) + + Additional Istio Ingress gateways can be enabled via the [overlay file](configuration-options.md#overlay-file). + +### Egress Support + +By default the Egress gateway is disabled, but can be enabled on install or upgrade through the values.yaml or via the [overlay file](configuration-options.md#overlay-file). + +## Additional Steps for Installing Istio on an RKE2 Cluster + +To install Istio on an RKE2 cluster, follow the steps in [this section.](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md) + +## Upgrading Istio in an Air-Gapped Environment + +The Istio pod security policy is now enabled by default. A new value, `installer.releaseMirror.enabled`, has been added to the rancher-istio chart to enable and disable the server that supports air-gapped upgrades. Note that `installer.releaseMirror.enabled` is set to `false` by default. You can set this value as needed when you install or upgrade. Follow the steps below: + +1. Provision an air-gapped Rancher instance and an air-gapped custom cluster in the Rancher UI. +2. Install Monitoring in the cluster: **Cluster Explorer -> Apps & Marketplace -> Charts -> Monitoring**. +3. Pull all required images for Istio into the private registry you will use in the air-gapped environment. +4. Install Istio in the cluster: **Cluster Explorer -> Apps & Marketplace -> Charts -> Istio**. + +:::note + +You can enable [Jaeger](https://www.jaegertracing.io/) and [Kiali](https://kiali.io/) on a fresh Istio install. To ensure that Jaeger and Kiali work, set `installer.releaseMirror.enabled` to `true` in `values.yaml` during installation. + +::: + +5. Upgrade the Istio installation. + +:::caution + +If you haven't already, set `installer.releaseMirror.enabled=true` to upgrade Istio. + +::: diff --git a/versioned_docs/version-2.8/pages-for-subheaders/k3s-hardening-guide.md b/versioned_docs/version-2.8/pages-for-subheaders/k3s-hardening-guide.md new file mode 100644 index 00000000000..285577fbca5 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/k3s-hardening-guide.md @@ -0,0 +1,744 @@ +--- +title: K3s Hardening Guide +--- + + + + + +This document provides prescriptive guidance for how to harden a K3s cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for K3s clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | + +:::note +In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter isn't required anymore, and was removed by CIS. +::: + +For more details on how to evaluate a hardened K3s cluster against the official CIS benchmark, refer to the K3s self-assessment guides for specific Kubernetes and CIS benchmark versions. + +K3s passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. K3s does not modify the host operating system. Any host-level modifications need to be done manually. +2. Certain CIS policy controls for `NetworkPolicies` and `PodSecurityStandards` (`PodSecurityPolicies` on v1.24 and older) restrict cluster functionality. + You must opt into having K3s configure these policies. Add the appropriate options to your command-line flags or configuration file (enable admission plugins), and manually apply the appropriate policies. + See further for more details. + +The first section (1.1) of the CIS Benchmark primarily focuses on pod manifest permissions and ownership. Since everything in the distribution is packaged in a single binary, this section does not apply to the core components of K3s. + +## Host-level Requirements + +### Ensure `protect-kernel-defaults` is set + + + + +The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. + + + + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + protect-kernel-defaults: true +``` + + + + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +This configuration needs to be done before setting the kubelet flag, otherwise K3s will fail to start. + +## Kubernetes Runtime Requirements + +The CIS Benchmark runtime requirements center around pod security (via PSP or PSA), network policies and API Server auditing logs. + +By default, K3s does not include any pod security or network policies. However, K3s ships with a controller that enforces any network policies you create. By default, K3s enables both the `PodSecurity` and `NodeRestriction` admission controllers, among others. + +### Pod Security + + + + +K3s v1.25 and newer support [Pod Security admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) for controlling pod security. + +You can specify the PSA configuration by setting the `defaultPodSecurityAdmissionConfigurationTemplateName` field in the cluster configuration in Rancher: + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted +``` + +The `rancher-restricted` template is provided by Rancher to enforce the highly-restrictive Kubernetes upstream [`Restricted`](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile with best practices for pod hardening. + + + + +K3s v1.24 and older support [Pod Security Policy (PSP)](https://v1-24.docs.kubernetes.io/docs/concepts/security/pod-security-policy/) for controlling pod security. + +You can enable PSPs by passing the following flags in the cluster configuration in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount +``` + +This maintains the `NodeRestriction` plugin and enables the `PodSecurityPolicy`. + +Once you enable PSPs, you can apply a policy to satisfy the necessary controls described in section 5.2 of the CIS Benchmark. + +:::note +These are manual checks in the CIS Benchmark. The CIS scan flags the results as `warning`, because manual inspection is necessary by the cluster operator. +::: + +Here is an example of a compliant PSP: + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false # CIS - 5.2.1 + allowPrivilegeEscalation: false # CIS - 5.2.5 + requiredDropCapabilities: # CIS - 5.2.7/8/9 + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false # CIS - 5.2.4 + hostIPC: false # CIS - 5.2.3 + hostPID: false # CIS - 5.2.2 + runAsUser: + rule: 'MustRunAsNonRoot' # CIS - 5.2.6 + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +``` + +For the example PSP to be effective, we need to create a `ClusterRole` and a `ClusterRoleBinding`. We also need to include a "system unrestricted policy" for system-level pods that require additional privileges, and an additional policy that allows the necessary sysctls for full functionality of ServiceLB. + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: system-unrestricted-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + hostIPC: true + hostNetwork: true + hostPID: true + hostPorts: + - max: 65535 + min: 0 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: svclb-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + allowedUnsafeSysctls: + - net.ipv4.ip_forward + - net.ipv6.conf.all.forwarding + fsGroup: + rule: RunAsAny + hostPorts: + - max: 65535 + min: 0 + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - restricted-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:system-unrestricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svclb-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - svclb-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-local-path-provisioner-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-coredns-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-cis-operator-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-psp +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-unrestricted-node-psp-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-unrestricted-svc-acct-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svclb-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svclb-psp +subjects: +- kind: ServiceAccount + name: svclb +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-local-path-provisioner-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-local-path-provisioner-psp +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-coredns-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-coredns-psp +subjects: +- kind: ServiceAccount + name: coredns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-cis-operator-psp-rolebinding + namespace: cis-operator-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-cis-operator-psp +subjects: +- kind: ServiceAccount + name: cis-operator-serviceaccount +``` + +The policies presented above can be placed in a file named `policy.yaml` in the `/var/lib/rancher/k3s/server/manifests` directory. Both the policy file and the its directory hierarchy must be created before starting K3s. A restrictive access permission is recommended to avoid leaking potential sensitive information. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +:::note +The critical Kubernetes additions such as CNI, DNS, and Ingress are run as pods in the `kube-system` namespace. Therefore, this namespace has a less restrictive policy, so that these components can run properly. +::: + + + + +### Network Policies + +CIS requires that all namespaces apply a network policy that reasonably limits traffic into namespaces and pods. + +:::note +This is a manual check in the CIS Benchmark. The CIS scan flags the result as a `warning`, because manual inspection is necessary by the cluster operator. +::: + +The network policies can be placed in the `policy.yaml` file in `/var/lib/rancher/k3s/server/manifests` directory. If the directory was not created as part of the PSP (as described above), it must be created first. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +Here is an example of a compliant network policy: + +```yaml +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-system +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-system +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: default +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: default +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-public +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-public +``` + +The active restrictions block DNS unless purposely allowed. Below is a network policy that allows DNS-related traffic: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-network-dns-policy + namespace: +spec: + ingress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Ingress +``` + +The metrics-server and Traefik ingress controller are blocked by default if network policies are not created to allow access. + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-metrics-server + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: metrics-server + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-svclbtraefik-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + svccontroller.k3s.cattle.io/svcname: traefik + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-traefik-v121-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ingress: + - {} + policyTypes: + - Ingress +``` + +:::note +You must manage network policies as normal for any additional namespaces you create. +::: + +### API Server audit configuration + +CIS requirements 1.2.22 to 1.2.25 are related to configuring audit logs for the API Server. K3s does not create by default the log directory and audit policy, as auditing requirements are specific to each user's policies and environment. + +If you need a log directory, it must be created before you start K3s. We recommend a restrictive access permission to avoid leaking sensitive information. + +```bash +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/logs +``` + +The following is a starter audit policy to log request metadata. This policy should be written to a file named `audit.yaml` in the `/var/lib/rancher/k3s/server` directory. Detailed information about policy configuration for the API server can be found in the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + +```yaml +--- +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata +``` + +Further configurations are also needed to pass CIS checks. These are not configured by default in K3s, because they vary based on your environment and needs: + +- Ensure that the `--audit-log-path` argument is set. +- Ensure that the `--audit-log-maxage` argument is set to 30 or as appropriate. +- Ensure that the `--audit-log-maxbackup` argument is set to 10 or as appropriate. +- Ensure that the `--audit-log-maxsize` argument is set to 100 or as appropriate. + +Combined, to enable and configure audit logs, add the following lines to the K3s cluster configuration file in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 +``` + +### Controller Manager Requirements + +CIS requirement 1.3.1 checks for garbage collection settings in the Controller Manager. Garbage collection is important to ensure sufficient resource availability and avoid degraded performance and availability. Based on your system resources and tests, choose an appropriate threshold value to activate garbage collection. + +This can be remediated by setting the following configuration in the K3s cluster file in Rancher. The value below is only an example. The appropriate threshold value is specific to each user's environment. + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 +``` + +### Configure `default` Service Account + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. + +For CIS requirement 5.1.5 the `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +This can be remediated by updating the `automountServiceAccountToken` field to `false` for the `default` service account in each namespace. + +For `default` service accounts in the built-in namespaces (`kube-system`, `kube-public`, `kube-node-lease`, and `default)`, K3s does not automatically do this. + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```shell +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Run the script every time a new service account is added to your cluster. + +## Reference Hardened K3s Template Configuration + +The following reference template configuration is used in Rancher to create a hardened K3s custom cluster based on each CIS control in this guide. This reference does not include other required **cluster configuration** directives, which vary based on your environment. + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,ServiceAccount # CIS 1.2.15, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount # CIS 1.2.15, 5.2, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 +``` + + + + +## Conclusion + +If you have followed this guide, your K3s custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our K3s self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-cluster-setup.md b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-cluster-setup.md new file mode 100644 index 00000000000..9291d55bee4 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-cluster-setup.md @@ -0,0 +1,11 @@ +--- +title: "Don't have a Kubernetes cluster? Try one of these tutorials." +--- + + + + + +This section contains information on how to install a Kubernetes cluster that the Rancher server can be installed on. + +Rancher can run on any Kubernetes cluster. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md new file mode 100644 index 00000000000..899d4bb5937 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md @@ -0,0 +1,80 @@ +--- +title: Setting up Kubernetes Clusters in Rancher +description: Provisioning Kubernetes Clusters +--- + + + + + +Rancher simplifies the creation of clusters by allowing you to create them through the Rancher UI rather than more complex alternatives. Rancher provides multiple options for launching a cluster. Use the option that best fits your use case. + +This section assumes a basic familiarity with Docker and Kubernetes. For a brief explanation of how Kubernetes components work together, refer to the [concepts](../reference-guides/kubernetes-concepts.md) page. + +For a conceptual overview of how the Rancher server provisions clusters and what tools it uses to provision them, refer to the [architecture](rancher-manager-architecture.md) page. + + + +### Cluster Management Capabilities by Cluster Type + +The following table summarizes the options and settings available for each cluster type: + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## Setting up Clusters in a Hosted Kubernetes Provider + +In this scenario, Rancher does not provision Kubernetes because it is installed by providers such as Google Kubernetes Engine (GKE), Amazon Elastic Container Service for Kubernetes, or Azure Kubernetes Service. + +If you use a Kubernetes provider such as Google GKE, Rancher integrates with its cloud APIs, allowing you to create and manage role-based access control for the hosted cluster from the Rancher UI. + +For more information, refer to the section on [hosted Kubernetes clusters.](set-up-clusters-from-hosted-kubernetes-providers.md) + +## Launching Kubernetes with Rancher + +Rancher uses the [Rancher Kubernetes Engine (RKE)](https://rancher.com/docs/rke/latest/en/) as a library when provisioning Kubernetes on your own nodes. RKE is Rancher’s own lightweight Kubernetes installer. + +In RKE clusters, Rancher manages the deployment of Kubernetes. These clusters can be deployed on any bare metal server, cloud provider, or virtualization platform. + +These nodes can be dynamically provisioned through Rancher's UI, which calls [Docker Machine](https://docs.docker.com/machine/) to launch nodes on various cloud providers. + +If you already have a node that you want to add to an RKE cluster, you can add it to the cluster by running a Rancher agent container on it. + +For more information, refer to the section on [RKE clusters.](../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +### Launching Kubernetes and Provisioning Nodes in an Infrastructure Provider + +Rancher can dynamically provision nodes in infrastructure providers such as Amazon EC2, DigitalOcean, Azure, or vSphere, then install Kubernetes on them. + +Using Rancher, you can create pools of nodes based on a [node template](use-new-nodes-in-an-infra-provider.md#node-templates). This template defines the parameters used to launch nodes in your cloud providers. + +One benefit of using nodes hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically replace it, thus maintaining the expected cluster configuration. + +The cloud providers available for creating a node template are decided based on the [node drivers](use-new-nodes-in-an-infra-provider.md#node-drivers) active in the Rancher UI. + +For more information, refer to the section on [nodes hosted by an infrastructure provider](use-new-nodes-in-an-infra-provider.md) + +### Launching Kubernetes on Existing Custom Nodes + +When setting up this type of cluster, Rancher installs Kubernetes on existing [custom nodes,](use-existing-nodes.md) which creates a custom cluster. + +You can bring any nodes you want to Rancher and use them to create a cluster. + +These nodes include on-prem bare metal servers, cloud-hosted virtual machines, or on-prem virtual machines. + +## Registering Existing Clusters + +The cluster registration feature replaces the feature to import clusters. + +Registering EKS clusters now provides additional benefits. For the most part, registered EKS clusters and EKS clusters created in Rancher are treated the same way in the Rancher UI, except for deletion. + +When you delete an EKS cluster that was created in Rancher, the cluster is destroyed. When you delete an EKS cluster that was registered in Rancher, it is disconnected from the Rancher server, but it still exists and you can still access it in the same way you did before it was registered in Rancher. + +For more information, see [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md) + +## Programmatically Creating Clusters + +The most common way to programmatically deploy Kubernetes clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) + +EKS, GKE, AKS clusters and RKE clusters can be created or imported with Terraform. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-components.md b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-components.md new file mode 100644 index 00000000000..f048b5ba19b --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-components.md @@ -0,0 +1,21 @@ +--- +title: Kubernetes Components +--- + + + + + +The commands and steps listed in this section apply to the core Kubernetes components on [Rancher Launched Kubernetes](../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters. + +This section includes troubleshooting tips in the following categories: + +- [Troubleshooting etcd Nodes](../troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md) +- [Troubleshooting Controlplane Nodes](../troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md) +- [Troubleshooting nginx-proxy Nodes](../troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md) +- [Troubleshooting Worker Nodes and Generic Components](../troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md) + +## Kubernetes Component Diagram + +![Cluster diagram](/img/clusterdiagram.svg)
+Lines show the traffic flow between components. Colors are used purely for visual aid \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-resources-setup.md b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-resources-setup.md new file mode 100644 index 00000000000..865f5ae5c46 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/kubernetes-resources-setup.md @@ -0,0 +1,67 @@ +--- +title: Kubernetes Resources +--- + + + + + +You can view and manipulate all of the custom resources and CRDs in a Kubernetes cluster from the Rancher UI. + +## Workloads + +Deploy applications to your cluster nodes using [workloads](workloads-and-pods.md), which are objects that contain pods that run your apps, along with metadata that set rules for the deployment's behavior. Workloads can be deployed within the scope of the entire clusters or within a namespace. + +When deploying a workload, you can deploy from any image. There are a variety of [workload types](workloads-and-pods.md#workload-types) to choose from which determine how your application should run. + +Following a workload deployment, you can continue working with it. You can: + +- [Upgrade](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md) the workload to a newer version of the application it's running. +- [Roll back](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md) a workload to a previous version, if an issue occurs during upgrade. +- [Add a sidecar](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md), which is a workload that supports a primary workload. + +## Load Balancing and Ingress + +### Load Balancers + +After you launch an application, it's only available within the cluster. It can't be reached externally. + +If you want your applications to be externally accessible, you must add a load balancer to your cluster. Load balancers create a gateway for external connections to access your cluster, provided that the user knows the load balancer's IP address and the application's port number. + +Rancher supports two types of load balancers: + +- [Layer-4 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-4-load-balancer) +- [Layer-7 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-7-load-balancer) + +For more information, see [load balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md). + +#### Ingress + +Load Balancers can only handle one IP address per service, which means if you run multiple services in your cluster, you must have a load balancer for each service. Running multiples load balancers can be expensive. You can get around this issue by using an ingress. + +Ingress is a set of rules that act as a load balancer. Ingress works in conjunction with one or more ingress controllers to dynamically route service requests. When the ingress receives a request, the ingress controller(s) in your cluster program the load balancer to direct the request to the correct service based on service subdomains or path rules that you've configured. + +For more information, see [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md). + +When using ingresses in a project, you can program the ingress hostname to an external DNS by setting up a Global DNS entry. + +## Service Discovery + +After you expose your cluster to external requests using a load balancer and/or ingress, it's only available by IP address. To create a resolveable hostname, you must create a service record, which is a record that maps an IP address, external hostname, DNS record alias, workload(s), or labelled pods to a specific hostname. + +For more information, see [Service Discovery](../how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md). + +## Applications + +Besides launching individual components of an application, you can use the Rancher catalog to start launching applications, which are Helm charts. + +## Kubernetes Resources + +Within the context of a Rancher project or namespace, _resources_ are files and data that support operation of your pods. Within Rancher, certificates, registries, and secrets are all considered resources. However, Kubernetes classifies resources as different types of [secrets](https://kubernetes.io/docs/concepts/configuration/secret/). Therefore, within a single project or namespace, individual resources must have unique names to avoid conflicts. Although resources are primarily used to carry sensitive information, they have other uses as well. + +Resources include: + +- [Certificates](../how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md): Files used to encrypt/decrypt data entering or leaving the cluster. +- [ConfigMaps](../how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md): Files that store general configuration information, such as a group of config files. +- [Secrets](../how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md): Files that store sensitive data like passwords, tokens, or keys. +- [Registries](../how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md): Files that carry credentials used to authenticate with private registries. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/launch-kubernetes-with-rancher.md b/versioned_docs/version-2.8/pages-for-subheaders/launch-kubernetes-with-rancher.md new file mode 100644 index 00000000000..5b7f4363bab --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/launch-kubernetes-with-rancher.md @@ -0,0 +1,84 @@ +--- +title: Launching Kubernetes with Rancher +--- + + + + + +You can have Rancher launch a Kubernetes cluster using any nodes you want. When Rancher deploys Kubernetes onto these nodes, you can choose between [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) or [RKE2](https://docs.rke2.io) distributions. Rancher can launch Kubernetes on any computers, including: + +- Bare-metal servers +- On-premise virtual machines +- Virtual machines hosted by an infrastructure provider + +Rancher can install Kubernetes on existing nodes, or it can dynamically provision nodes in an infrastructure provider and install Kubernetes on them. + +Rancher can also create pools of nodes. One benefit of installing Kubernetes on node pools hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically create another node to join the cluster to ensure that the count of the node pool is as expected. + +## RKE + +### Requirements + +If you use RKE to set up a cluster, your nodes must meet the [requirements](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) for nodes in downstream user clusters. + +### Launching Kubernetes on New Nodes in an Infrastructure Provider + +Using Rancher, you can create pools of nodes based on a [node template](use-new-nodes-in-an-infra-provider.md#node-templates). This node template defines the parameters you want to use to launch nodes in your cloud providers. + +One benefit of installing Kubernetes on node pools hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically create another node to join the cluster to ensure that the count of the node pool is as expected. + +For more information, refer to the section on [launching Kubernetes on new nodes.](use-new-nodes-in-an-infra-provider.md) + +### Launching Kubernetes on Existing Custom Nodes + +In this scenario, you want to install Kubernetes on bare-metal servers, on-prem virtual machines, or virtual machines that already exist in a cloud provider. With this option, you will run a Rancher agent Docker container on the machine. + +If you want to reuse a node from a previous custom cluster, [clean the node](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) before using it in a cluster again. If you reuse a node that hasn't been cleaned, cluster provisioning may fail. + +For more information, refer to the section on [custom nodes.](use-existing-nodes.md) + +### Programmatically Creating RKE Clusters + +The most common way to programmatically deploy RKE clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) + +## RKE2 + +Rancher v2.6 introduced provisioning for [RKE2](https://docs.rke2.io/) clusters directly from the Rancher UI. RKE2, also known as RKE Government, is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. In Rancher v.2.6.4 and earlier, RKE2 provisioning was in tech preview. + +Note that in Rancher v2.6.5, RKE2 provisioning became GA. + +### Requirements + +If you use RKE2 to set up a cluster, your nodes must meet the [requirements](https://docs.rke2.io/install/requirements) for nodes in downstream user clusters. + +### Launching Kubernetes on New Nodes in an Infrastructure Provider + +RKE2 provisioning is built on top of a new provisioning framework that leverages the upstream [Cluster API](https://github.com/kubernetes-sigs/cluster-api) project. With this new provisioning framework, you can: + +- Provision RKE2 clusters onto any provider for which Rancher has a node driver +- Fully configure RKE2 clusters within Rancher +- Choose CNI options Calico, Cilium, and Multus in addition to Canal + +RKE2 provisioning also includes installing RKE2 on clusters with Windows nodes. + +Windows features for RKE2 include: + +- Windows supports the vSphere node driver +- Calico CNI for Windows RKE2 custom clusters +- Project Network Isolation (PNI) for Calico +- Windows Containers with RKE2 powered by containerd +- Provisioning of Windows RKE2 clusters through Terraform +- Provisioning of Windows RKE2 custom clusters directly from the Rancher UI + +Windows Support for RKE2 Custom Clusters requires choosing Calico as the CNI. + +### Launching Kubernetes on Existing Custom Nodes + +RKE2 provisioning also allows you to install custom clusters on pre-provisioned VMs or bare-metal nodes. + +If you want to reuse a node from a previous custom cluster, clean the node before using it in a cluster again. If you reuse a node that hasn't been cleaned, cluster provisioning may fail. + +### Programmatically Creating RKE2 Clusters + +The most common way to programmatically deploy RKE2 clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster_v2) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/load-balancer-and-ingress-controller.md b/versioned_docs/version-2.8/pages-for-subheaders/load-balancer-and-ingress-controller.md new file mode 100644 index 00000000000..41bdf40a323 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/load-balancer-and-ingress-controller.md @@ -0,0 +1,65 @@ +--- +title: Set Up Load Balancer and Ingress Controller within Rancher +description: Learn how you can set up load balancers and ingress controllers to redirect service requests within Rancher, and learn about the limitations of load balancers +--- + + + + + +Within Rancher, you can set up load balancers and ingress controllers to redirect service requests. + +## Load Balancers + +After you launch an application, the app is only available within the cluster. It can't be reached from outside the cluster. + +If you want your applications to be externally accessible, you must add a load balancer or ingress to your cluster. Load balancers create a gateway for external connections to access your cluster, provided that the user knows the load balancer's IP address and the application's port number. + +Rancher supports two types of load balancers: + +- [Layer-4 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-4-load-balancer) +- [Layer-7 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-7-load-balancer) + +For more information, see [load balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md). + +### Load Balancer Limitations + +Load Balancers have a couple of limitations you should be aware of: + +- Load Balancers can only handle one IP address per service, which means if you run multiple services in your cluster, you must have a load balancer for each service. Running multiples load balancers can be expensive. + +- If you want to use a load balancer with a Hosted Kubernetes cluster (i.e., clusters hosted in GKE, EKS, or AKS), the load balancer must be running within that cloud provider's infrastructure. Please review the compatibility tables regarding support for load balancers based on how you've provisioned your clusters: + +- [Support for Layer-4 Load Balancing](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#support-for-layer-4-load-balancing) + +- [Support for Layer-7 Load Balancing](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#support-for-layer-7-load-balancing) + +## Ingress + +As mentioned in the limitations above, the disadvantages of using a load balancer are: + +- Load Balancers can only handle one IP address per service. +- If you run multiple services in your cluster, you must have a load balancer for each service. +- It can be expensive to have a load balancer for every service. + +In contrast, when an ingress is used as the entrypoint into a cluster, the ingress can route traffic to multiple services with greater flexibility. It can map multiple HTTP requests to services without individual IP addresses for each service. + +Therefore, it is useful to have an ingress if you want multiple services to be exposed with the same IP address, the same Layer 7 protocol, or the same privileged node-ports: 80 and 443. + +Ingress works in conjunction with one or more ingress controllers to dynamically route service requests. When the ingress receives a request, the ingress controller(s) in your cluster direct the request to the correct service based on service subdomains or path rules that you've configured. + +Each Kubernetes Ingress resource corresponds roughly to a file in `/etc/nginx/sites-available/` containing a `server{}` configuration block, where requests for specific files and folders are configured. + +Your ingress, which creates a port of entry to your cluster similar to a load balancer, can reside within your cluster or externally. Ingress and ingress controllers residing in RKE-launched clusters are powered by [Nginx](https://www.nginx.com/). + +Ingress can provide other functionality as well, such as SSL termination, name-based virtual hosting, and more. + +:::note Using Rancher in a High Availability Configuration? + +Refrain from adding an Ingress to the `local` cluster. The Nginx Ingress Controller that Rancher uses acts as a global entry point for _all_ clusters managed by Rancher, including the `local` cluster. Therefore, when users try to access an application, your Rancher connection may drop due to the Nginx configuration being reloaded. We recommend working around this issue by deploying applications only in clusters that you launch using Rancher. + +::: + +- For more information on how to set up ingress in Rancher, see [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md). +- For complete information about ingress and ingress controllers, see the [Kubernetes Ingress Documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) +- When using ingresses in a project, you can program the ingress hostname to an external DNS by setting up a Global DNS entry. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/logging.md b/versioned_docs/version-2.8/pages-for-subheaders/logging.md new file mode 100644 index 00000000000..427422627f3 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/logging.md @@ -0,0 +1,121 @@ +--- +title: Rancher Integration with Logging Services +description: Rancher integrates with popular logging services. Learn the requirements and benefits of integrating with logging services, and enable logging on your cluster. +--- + + + + + +The [Logging operator](https://kube-logging.github.io/docs/) now powers Rancher's logging solution in place of the former, in-house solution. + +## Enabling Logging + +You can enable the logging for a Rancher managed cluster by going to the Apps page and installing the logging app. + +1. Go to the cluster where you want to install logging and click **Apps**. +1. Click the **Logging** app. +1. Scroll to the bottom of the Helm chart README and click **Install**. + +**Result:** The logging app is deployed in the `cattle-logging-system` namespace. + +## Uninstall Logging + +1. Go to the cluster where you want to install logging and click **Apps**. +1. Click **Installed Apps**. +1. Go to the `cattle-logging-system` namespace and check the boxes for `rancher-logging` and `rancher-logging-crd`. +1. Click **Delete**. +1. Confirm **Delete**. + +**Result** `rancher-logging` is uninstalled. + +## Architecture + +For more information about how the logging application works, see [this section.](../integrations-in-rancher/logging/logging-architecture.md) + + + +## Role-based Access Control + +Rancher logging has two roles, `logging-admin` and `logging-view`. For more information on how and when to use these roles, see [this page.](../integrations-in-rancher/logging/rbac-for-logging.md) + +## Configuring Logging Custom Resources + +To manage `Flows,` `ClusterFlows`, `Outputs`, and `ClusterOutputs`, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to configure logging custom resources and click **Explore**. +1. In the left navigation bar, click **Logging**. + +### Flows and ClusterFlows + +For help with configuring `Flows` and `ClusterFlows`, see [this page.](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md) + +### Outputs and ClusterOutputs + +For help with configuring `Outputs` and `ClusterOutputs`, see [this page.](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md) + +## Configuring the Logging Helm Chart + +For a list of options that can be configured when the logging application is installed or upgraded, see [this page.](../integrations-in-rancher/logging/logging-helm-chart-options.md) + +### Windows Support + +You can [enable logging](../integrations-in-rancher/logging/logging-helm-chart-options.md#enabledisable-windows-node-logging) from Windows nodes. + + +### Working with a Custom Docker Root Directory + +For details on using a custom Docker root directory, see [this section.](../integrations-in-rancher/logging/logging-helm-chart-options.md#working-with-a-custom-docker-root-directory) + + +### Working with Taints and Tolerations + +For information on how to use taints and tolerations with the logging application, see [this page.](../integrations-in-rancher/logging/taints-and-tolerations.md) + + +### Logging V2 with SELinux + +For information on enabling the logging application for SELinux-enabled nodes, see [this section.](../integrations-in-rancher/logging/logging-helm-chart-options.md#enabling-the-logging-application-to-work-with-selinux) + +### Additional Logging Sources + +By default, Rancher collects logs for control plane components and node components for all cluster types. In some cases additional logs can be collected. For details, see [this section.](../integrations-in-rancher/logging/logging-helm-chart-options.md#additional-logging-sources) + + +## Troubleshooting + +### The Logging Buffer Overloads Pods + +Depending on your configuration, the default buffer size may be too large and cause pod failures. One way to reduce the load is to lower the logger's flush interval. This prevents logs from overfilling the buffer. You can also add more flush threads to handle moments when many logs are attempting to fill the buffer at once. + +For a more complete description of how to configure the logging buffer to suit your organization's needs, see the official Logging operator documentation on [buffers](https://kube-logging.github.io/docs/configuration/plugins/outputs/buffer/) and on [Fluentd configuration](https://kube-logging.github.io/docs/logging-infrastructure/fluentd/). + +### The `cattle-logging` Namespace Being Recreated + +If your cluster previously deployed logging from the global view in the legacy Rancher UI, you may encounter an issue where its `cattle-logging` namespace is continually being recreated. + +The solution is to delete all `clusterloggings.management.cattle.io` and `projectloggings.management.cattle.io` custom resources from the cluster specific namespace in the management cluster. +The existence of these custom resources causes Rancher to create the `cattle-logging` namespace in the downstream cluster if it does not exist. + +The cluster namespace matches the cluster ID, so we need to find the cluster ID for each cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to get the ID of and click **Explore**. +2. Copy the `` portion from one of the URLs below. The `` portion is the cluster namespace name. + +```bash +# Cluster Management UI +https:///c// + +# Cluster Dashboard +https:///dashboard/c// +``` + +Now that we have the `` namespace, we can delete the CRs that cause `cattle-logging` to be continually recreated. +*Warning:* ensure that logging, the version installed from the global view in the legacy Rancher UI, is not currently in use. + +```bash +kubectl delete crd clusterloggings.management.cattle.io -n +kubectl delete crd projectloggings.management.cattle.io -n +``` diff --git a/versioned_docs/version-2.8/pages-for-subheaders/machine-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/machine-configuration.md new file mode 100644 index 00000000000..e1b9bb72f0a --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/machine-configuration.md @@ -0,0 +1,9 @@ +--- +title: Machine Configuration +--- + + + + + +Machine configuration is the arrangement of resources assigned to a virtual machine. Please see the docs for [Amazon EC2](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md), [DigitalOcean](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md), and [Azure](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) to learn more. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/manage-clusters.md b/versioned_docs/version-2.8/pages-for-subheaders/manage-clusters.md new file mode 100644 index 00000000000..e6f69885338 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/manage-clusters.md @@ -0,0 +1,36 @@ +--- +title: Cluster Administration +--- + + + + + +After you provision a cluster in Rancher, you can begin using powerful Kubernetes features to deploy and scale your containerized applications in development, testing, or production environments. + +:::note + +This section assumes a basic familiarity with Docker and Kubernetes. For a brief explanation of how Kubernetes components work together, refer to the [concepts](../reference-guides/kubernetes-concepts.md) page. + +::: + +## Managing Clusters in Rancher + +After clusters have been [provisioned into Rancher](kubernetes-clusters-in-rancher-setup.md), [cluster owners](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles) will need to manage these clusters. There are many different options of how to manage your cluster. + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## Configuring Tools + +Rancher contains a variety of tools that aren't included in Kubernetes to assist in your DevOps operations. Rancher can integrate with external services to help your clusters run more efficiently. Tools are divided into the following categories: + +- Alerts +- Notifiers +- Logging +- Monitoring +- Istio Service Mesh +- OPA Gatekeeper + +Tools can be installed through **Apps.** diff --git a/versioned_docs/version-2.8/pages-for-subheaders/manage-project-resource-quotas.md b/versioned_docs/version-2.8/pages-for-subheaders/manage-project-resource-quotas.md new file mode 100644 index 00000000000..fa6f10ec84c --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/manage-project-resource-quotas.md @@ -0,0 +1,50 @@ +--- +title: Project Resource Quotas +--- + + + + + +In situations where several teams share a cluster, one team may overconsume the resources available: CPU, memory, storage, services, Kubernetes objects like pods or secrets, and so on. To prevent this overconsumption, you can apply a _resource quota_, which is a Rancher feature that limits the resources available to a project or namespace. + +This page is a how-to guide for creating resource quotas in existing projects. + +Resource quotas can also be set when a new project is created. For details, refer to the section on [creating new projects.](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md#creating-projects) + +Resource quotas in Rancher include the same functionality as the [native version of Kubernetes](https://kubernetes.io/docs/concepts/policy/resource-quotas/). In Rancher, resource quotas have been extended so that you can apply them to projects. For details on how resource quotas work with projects in Rancher, refer to [this page.](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md) + +### Applying Resource Quotas to Existing Projects + +Edit resource quotas when: + +- You want to limit the resources that a project and its namespaces can use. +- You want to scale the resources available to a project up or down when a research quota is already in effect. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to apply a resource quota and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Make sure that the **Projects/Namespaces** page is in **Group by Project** view mode. + ![Screenshot highlighting the "Group by Project" icon, above the list of projects. It resembles a folder.](/img/edit-project-config-for-resource-quotas-group-by-project.png) + +1. Find the project that you want to add a resource quota to, and select the **⋮** that's on the same row as the project's name. + ![Screenshot highlighting triple dots icon at the end of the same row as the project name.](/img/edit-project-config-for-resource-quotas-dots.png) + +1. Select **Edit Config**. + +1. Expand **Resource Quotas** and click **Add Resource**. Alternatively, you can edit existing quotas. + +1. Select a Resource Type. For more information on types, see the [quota type reference.](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md) + +1. Enter values for the **Project Limit** and the **Namespace Default Limit**. + + | Field | Description | + | ----------------------- | -------------------------------------------------------------------------------------------------------- | + | Project Limit | The overall resource limit for the project. | + | Namespace Default Limit | The default resource limit available for each namespace. This limit is propagated to each namespace in the project. The combined limit of all project namespaces shouldn't exceed the project limit. | + +1. **Optional:** Add more quotas. + +1. Click **Create**. + +**Result:** The resource quota is applied to your project and namespaces. When you add more namespaces in the future, Rancher validates that the project can accommodate the namespace. If the project can't allocate the resources, you may still create namespaces, but they will be given a resource quota of 0. Subsequently, Rancher will not allow you to create any resources restricted by this quota. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/manage-projects.md b/versioned_docs/version-2.8/pages-for-subheaders/manage-projects.md new file mode 100644 index 00000000000..be308c7e342 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/manage-projects.md @@ -0,0 +1,41 @@ +--- +title: Project Administration +--- + + + + + +_Projects_ are objects introduced in Rancher that help organize namespaces in your Kubernetes cluster. You can use projects to create multi-tenant clusters, which allows a group of users to share the same underlying resources without interacting with each other's applications. + +In terms of hierarchy: + +- Clusters contain projects +- Projects contain namespaces + +Within Rancher, projects allow you to manage multiple namespaces as a single entity. In native Kubernetes, which does not include projects, features like role-based access rights or cluster resources are assigned to individual namespaces. In clusters where multiple namespaces require the same set of access rights, assigning these rights to each individual namespace can become tedious. Even though all namespaces require the same rights, there's no way to apply those rights to all of your namespaces in a single action. You'd have to repetitively assign these rights to each namespace! + +Rancher projects resolve this issue by allowing you to apply resources and access rights at the project level. Each namespace in the project then inherits these resources and policies, so you only have to assign them to the project once, rather than assigning them to each individual namespace. + +You can use projects to perform actions like: + +- [Assign users access to a group of namespaces](../how-to-guides/new-user-guides/add-users-to-projects.md) +- Assign users [specific roles in a project](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles). A role can be owner, member, read-only, or [custom](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md) +- [Set resource quotas](manage-project-resource-quotas.md) +- [Manage namespaces](../how-to-guides/new-user-guides/manage-namespaces.md) +- [Configure tools](../reference-guides/rancher-project-tools.md) +- [Configure pod security policies](../how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md) + +### Authorization + +Non-administrative users are only authorized for project access after an [administrator](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md), [cluster owner or member](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles), or [project owner](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) adds them to the project's **Members** tab. + +Whoever creates the project automatically becomes a [project owner](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles). + +## Switching between Projects + +To switch between projects, use the drop-down available in the navigation bar. Alternatively, you can switch between projects directly in the navigation bar. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to switch projects and click **Explore**. +1. In the top navigation bar, select the project that you want to open. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/manage-role-based-access-control-rbac.md b/versioned_docs/version-2.8/pages-for-subheaders/manage-role-based-access-control-rbac.md new file mode 100644 index 00000000000..c6c87eff5fc --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/manage-role-based-access-control-rbac.md @@ -0,0 +1,29 @@ +--- +title: Role-Based Access Control (RBAC) +--- + + + + + +Within Rancher, each person authenticates as a _user_, which is a login that grants you access to Rancher. As mentioned in [Authentication](authentication-config.md), users can either be local or external. + +After you configure external authentication, the users that display on the **Users** page changes. + +- If you are logged in as a local user, only local users display. + +- If you are logged in as an external user, both external and local users display. + +## Users and Roles + +Once the user logs in to Rancher, their _authorization_, or their access rights within the system, is determined by _global permissions_, and _cluster and project roles_. + +- [Global Permissions](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md): + + Define user authorization outside the scope of any particular cluster. + +- [Cluster and Project Roles](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md): + + Define user authorization inside the specific cluster or project where they are assigned the role. + +Both global permissions and cluster and project roles are implemented on top of [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/). Therefore, enforcement of permissions and roles is performed by Kubernetes. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/monitoring-alerting-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-alerting-guides.md new file mode 100644 index 00000000000..97e3e801b26 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-alerting-guides.md @@ -0,0 +1,14 @@ +--- +title: Monitoring Guides +--- + + + + + +- [Enable monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [Uninstall monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring workloads](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [Customizing Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [Persistent Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [Debugging high memory usage](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/monitoring-and-alerting.md b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-and-alerting.md new file mode 100644 index 00000000000..e76dba53edb --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-and-alerting.md @@ -0,0 +1,99 @@ +--- +title: Monitoring and Alerting +description: Prometheus lets you view metrics from your different Rancher and Kubernetes objects. Learn about the scope of monitoring and how to enable cluster monitoring +--- + + + + + +The `rancher-monitoring` application can quickly deploy leading open-source monitoring and alerting solutions onto your cluster. + +Introduced in Rancher v2.5, the application is powered by [Prometheus](https://prometheus.io/), [Grafana](https://grafana.com/grafana/), [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/), the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator), and the [Prometheus adapter.](https://github.com/DirectXMan12/k8s-prometheus-adapter) + +For information on V1 monitoring and alerting, available in Rancher v2.2 up to v2.4, please see the Rancher v2.0—v2.4 docs on [cluster monitoring](../../versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-monitoring.md), [alerting](../../versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-alerts.md), [notifiers](../../versioned_docs/version-2.0-2.4/explanations/integrations-in-rancher/notifiers.md) and other [tools](../../versioned_docs/version-2.0-2.4/pages-for-subheaders/project-tools.md). + +Using the `rancher-monitoring` application, you can quickly deploy leading open-source monitoring and alerting solutions onto your cluster. + +### Features + +Prometheus lets you view metrics from your Rancher and Kubernetes objects. Using timestamps, Prometheus lets you query and view these metrics in easy-to-read graphs and visuals, either through the Rancher UI or Grafana, which is an analytics viewing platform deployed along with Prometheus. + +By viewing data that Prometheus scrapes from your cluster control plane, nodes, and deployments, you can stay on top of everything happening in your cluster. You can then use these analytics to better run your organization: stop system emergencies before they start, develop maintenance strategies, or restore crashed servers. + +The monitoring application: + +- Monitors the state and processes of your cluster nodes, Kubernetes components, and software deployments. +- Defines alerts based on metrics collected via Prometheus. +- Creates custom Grafana dashboards. +- Configures alert-based notifications via email, Slack, PagerDuty, etc. using Prometheus Alertmanager. +- Defines precomputed, frequently needed or computationally expensive expressions as new time series based on metrics collected via Prometheus. +- Exposes collected metrics from Prometheus to the Kubernetes Custom Metrics API via Prometheus Adapter for use in HPA. + +See [How Monitoring Works](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) for an explanation of how the monitoring components work together. + +## Default Components and Deployments + +### Built-in Dashboards + +By default, the monitoring application deploys Grafana dashboards (curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project) onto a cluster. + +It also deploys an Alertmanager UI and a Prometheus UI. For more information about these tools, see [Built-in Dashboards.](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md) +### Default Metrics Exporters + +By default, Rancher Monitoring deploys exporters (such as [node-exporter](https://github.com/prometheus/node_exporter) and [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics)). + +These default exporters automatically scrape metrics for CPU and memory from all components of your Kubernetes cluster, including your workloads. + +### Default Alerts + +The monitoring application deploys some alerts by default. To see the default alerts, go to the [Alertmanager UI](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md#alertmanager-ui) and click **Expand all groups.** + +### Components Exposed in the Rancher UI + +For a list of monitoring components exposed in the Rancher UI, along with common use cases for editing them, see [this section.](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#components-exposed-in-the-rancher-ui) + +## Role-based Access Control + +For information on configuring access to monitoring, see [this page.](../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md) + +## Guides + +- [Enable monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [Uninstall monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring workloads](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [Customizing Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [Persistent Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [Debugging high memory usage](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) + +## Configuration + +### Configuring Monitoring Resources in Rancher + +The configuration reference assumes familiarity with how monitoring components work together. For more information, see [How Monitoring Works.](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +- [ServiceMonitor and PodMonitor](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [Receiver](../reference-guides/monitoring-v2-configuration/receivers.md) +- [Route](../reference-guides/monitoring-v2-configuration/routes.md) +- [PrometheusRule](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) +- [Prometheus](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) +- [Alertmanager](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### Configuring Helm Chart Options + +For more information on `rancher-monitoring` chart options, including options to set resource limits and requests, see [Helm Chart Options](../reference-guides/monitoring-v2-configuration/helm-chart-options.md). + +## Windows Cluster Support + +When deployed onto an RKE1 Windows cluster, Monitoring V2 will now automatically deploy a [windows-exporter](https://github.com/prometheus-community/windows_exporter) DaemonSet and set up a ServiceMonitor to collect metrics from each of the deployed Pods. This will populate Prometheus with `windows_` metrics that are akin to the `node_` metrics exported by [node_exporter](https://github.com/prometheus/node_exporter) for Linux hosts. + +To be able to fully deploy Monitoring V2 for Windows, all of your Windows hosts must have a minimum [wins](https://github.com/rancher/wins) version of v0.1.0. + +For more details on how to upgrade wins on existing Windows hosts, see [Windows cluster support for Monitoring V2.](../integrations-in-rancher/monitoring-and-alerting/windows-support.md). + + +## Known Issues + +There is a [known issue](https://github.com/rancher/rancher/issues/28787#issuecomment-693611821) that K3s clusters require more than the allotted default memory. If you enable monitoring on a K3s cluster, set `prometheus.prometheusSpec.resources.memory.limit` to 2500 Mi and `prometheus.prometheusSpec.resources.memory.request` to 1750 Mi. + +See [Debugging High Memory Usage](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) for advice and recommendations. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration-guides.md new file mode 100644 index 00000000000..bd0de341f46 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration-guides.md @@ -0,0 +1,55 @@ +--- +title: Configuration +--- + + + + + +This page captures some of the most important options for configuring Monitoring V2 in the Rancher UI. + +For information on configuring custom scrape targets and rules for Prometheus, please refer to the upstream documentation for the [Prometheus Operator.](https://github.com/prometheus-operator/prometheus-operator) Some of the most important custom resources are explained in the Prometheus Operator [design documentation.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md) The Prometheus Operator documentation can help also you set up RBAC, Thanos, or custom configuration. + +## Setting Resource Limits and Requests + +The resource requests and limits for the monitoring application can be configured when installing `rancher-monitoring`. For more information about the default limits, see [this page.](../reference-guides/monitoring-v2-configuration/helm-chart-options.md#configuring-resource-limits-and-requests) + +:::tip + +On an idle cluster, Monitoring may have high CPU usage. To improve performance, turn off the Prometheus adapter. + +::: + +## Prometheus Configuration + +It is usually not necessary to directly edit the Prometheus custom resource. + +Instead, to configure Prometheus to scrape custom metrics, you will only need to create a new ServiceMonitor or PodMonitor to configure Prometheus to scrape additional metrics. + + +### ServiceMonitor and PodMonitor Configuration + +For details, see [this page.](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) + +### Advanced Prometheus Configuration + +For more information about directly editing the Prometheus custom resource, which may be helpful in advanced use cases, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) + +## Alertmanager Configuration + +The Alertmanager custom resource usually doesn't need to be edited directly. For most common use cases, you can manage alerts by updating Routes and Receivers. + +Routes and receivers are part of the configuration of the alertmanager custom resource. In the Rancher UI, Routes and Receivers are not true custom resources, but pseudo-custom resources that the Prometheus Operator uses to synchronize your configuration with the Alertmanager custom resource. When routes and receivers are updated, the monitoring application will automatically update Alertmanager to reflect those changes. + +For some advanced use cases, you may want to configure alertmanager directly. For more information, refer to [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### Receivers + +Receivers are used to set up notifications. For details on how to configure receivers, see [this page.](../reference-guides/monitoring-v2-configuration/receivers.md) +### Routes + +Routes filter notifications before they reach receivers. Each route needs to refer to a receiver that has already been configured. For details on how to configure routes, see [this page.](../reference-guides/monitoring-v2-configuration/routes.md) + +### Advanced + +For more information about directly editing the Alertmanager custom resource, which may be helpful in advanced use cases, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration.md new file mode 100644 index 00000000000..79f97d9513d --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/monitoring-v2-configuration.md @@ -0,0 +1,15 @@ +--- +title: Monitoring V2 Configuration +--- + + + + + +The following sections will explain important options essential to configuring Monitoring V2 in Rancher: + +- [Receiver Configuration](../reference-guides/monitoring-v2-configuration/receivers.md) +- [Route Configuration](../reference-guides/monitoring-v2-configuration/routes.md) +- [ServiceMonitor and PodMonitor Configuration](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [Helm Chart Options](../reference-guides/monitoring-v2-configuration/helm-chart-options.md) +- [Examples](../reference-guides/monitoring-v2-configuration/examples.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/new-user-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/new-user-guides.md new file mode 100644 index 00000000000..e2e68ff0089 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/new-user-guides.md @@ -0,0 +1,13 @@ +--- +title: New User Guides +--- + + + + + +New user guides, also known as **tutorials**, describe practical steps for users to follow in order to complete some concrete action. These docs are known as "learning-oriented" docs in which users learn by "doing". + +The new user guides are designed to guide beginners, or the everyday users of Rancher, through a series of steps to learn how to do something. The goal is that the user will be able to learn how to complete tasks by using easy-to-follow, meaningful, and repeatable directions. These guides will assist users to do work to then get the promised results immediately. + +The average Rancher user has a level of technical skill that is above the level of "beginner"; however, the new user guides are designed to help new, or beginner, users as well as the seasoned Rancher customer equally. This is accomplished by using a combination of high-level and technical language to introduce topics and guide the user through general tasks that are essential for every Rancher user to know. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/node-template-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/node-template-configuration.md new file mode 100644 index 00000000000..e6c22d5e852 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/node-template-configuration.md @@ -0,0 +1,9 @@ +--- +title: Node Template Configuration +--- + + + + + +To learn about node template config, refer to [EC2 Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md), [DigitalOcean Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md), [Azure Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md), [vSphere Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md), and [Nutanix Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md). diff --git a/versioned_docs/version-2.8/pages-for-subheaders/nutanix.md b/versioned_docs/version-2.8/pages-for-subheaders/nutanix.md new file mode 100644 index 00000000000..b626cf2bd12 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/nutanix.md @@ -0,0 +1,21 @@ +--- +title: Creating a Nutanix AOS Cluster +description: Use Rancher to create a Nutanix AOS (AHV) cluster. It may consist of groups of VMs with distinct properties which allow for fine-grained control over the sizing of nodes. +--- + + + + + +[Nutanix Acropolis Operating System](https://www.nutanix.com/products/acropolis) (Nutanix AOS) is an operating system for the Nutanix hyper-converged infrastructure platform. AOS comes with a built-in hypervisor called [Acropolis Hypervisor](https://www.nutanix.com/products/ahv), or AHV. By using Rancher with Nutanix AOS (AHV), you can bring cloud operations on-premises. + +Rancher can provision nodes in AOS (AHV) and install Kubernetes on them. When creating a Kubernetes cluster in AOS, Rancher first provisions the specified number of virtual machines by communicating with the Prism Central API. Then it installs Kubernetes on top of the VMs. + +A Nutanix cluster may consist of multiple groups of VMs with distinct properties, such as the amount of memory or the number of vCPUs. This grouping allows for fine-grained control over the sizing of nodes for each Kubernetes role. + +- [Creating a Nutanix Cluster](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md#creating-a-nutanix-aos-cluster) +- [Provisioning Storage](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md) + +## Creating a Nutanix Cluster + +In [this section,](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Nutanix AOS. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/other-installation-methods.md b/versioned_docs/version-2.8/pages-for-subheaders/other-installation-methods.md new file mode 100644 index 00000000000..7cd497a8d48 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/other-installation-methods.md @@ -0,0 +1,23 @@ +--- +title: Other Installation Methods +--- + + + + + +### Air Gapped Installations + +Follow [these steps](air-gapped-helm-cli-install.md) to install the Rancher server in an air gapped environment. + +An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +### Docker Installations + +The [single-node Docker installation](rancher-on-a-single-node-with-docker.md) is for Rancher users that are wanting to test out Rancher. Instead of running on a Kubernetes cluster using Helm, you install the Rancher server component on a single node using a `docker run` command. + +The Docker installation is for development and testing environments only. + +Since there is only one node and a single Docker container, if the node goes down, there is no copy of the etcd data available on other nodes and you will lose all the data of your Rancher server. + +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/prometheus-federator-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/prometheus-federator-guides.md new file mode 100644 index 00000000000..2d1c0ae8224 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/prometheus-federator-guides.md @@ -0,0 +1,12 @@ +--- +title: Prometheus Federator Guides +--- + + + + + +- [Enable Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md) +- [Uninstall Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md) +- [Customize Grafana Dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md) +- [Set Up Workloads](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/prometheus-federator.md b/versioned_docs/version-2.8/pages-for-subheaders/prometheus-federator.md new file mode 100644 index 00000000000..efef5f5abae --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/prometheus-federator.md @@ -0,0 +1,108 @@ +--- +title: Prometheus Federator +--- + + + + + +Prometheus Federator, also referred to as Project Monitoring v2, deploys a Helm Project Operator (based on the [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator)), an operator that manages deploying Helm charts each containing a Project Monitoring Stack, where each stack contains: + +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed resources + +:::note Important: + +Prometheus Federator is designed to be deployed alongside an existing Prometheus Operator deployment in a cluster that has already installed the Prometheus Operator CRDs. + +::: + +## How does the operator work? + +1. On deploying this chart, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `monitoring.cattle.io/v1alpha1` (also known as "Project Monitors" in the Rancher UI) in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy a Project Prometheus stack on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--monitoring`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack deployed; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). For more information, see the section on [configuring RBAC](../reference-guides/prometheus-federator/rbac.md). + +### What is a Project? + +In Prometheus Federator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`. By default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given Rancher Project. + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart's resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: + +- View the chart's definition located at [`rancher/prometheus-federator` under `charts/rancher-project-monitoring`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) (where the chart version will be tied to the version of this operator). +- Look for the ConfigMap named `monitoring.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `prometheus-federator` binary). + +### Namespaces + +As a Project Operator based on [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator), Prometheus Federator has three different classifications of namespaces that the operator looks out for: + +1. **Operator / System Namespace**: The namespace that the operator is deployed into (e.g., `cattle-monitoring-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** + +2. **Project Registration Namespace (`cattle-project-`)**: The set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace. For details, refer to the [RBAC page](../reference-guides/prometheus-federator/rbac.md). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace.** + + :::note Notes: + + - Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided, which is set to `field.cattle.io/projectId` by default. This indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g., this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted), or it is no longer watching that project because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects. These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project. + + - If `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace. + + ::: + +3. **Project Release Namespace (`cattle-project--monitoring`):** The set of namespaces that the operator deploys Project Monitoring Stacks within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Project Monitoring Stack based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Prometheus Federator.** + + :::note Notes: + + - Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue`, which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified, whenever a ProjectHelmChart is specified in a Project Registration Namespace. + + - Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above). + + - If `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace. + + ::: + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Prometheus Federator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: + +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): This custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR. This CR is automatically updated on changes to the ProjectHelmChart (e.g., modifying the values.yaml) or changes to the underlying Project definition (e.g., adding or removing namespaces from a project). + +:::note Important: + +If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation. However, this is generally only accessible by a **Cluster Admin.** + +::: + +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): This custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). + +:::note + +HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place. To view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources modifications were attempted on. + +::: + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### Advanced Helm Project Operator Configuration + +For more information on advanced configurations, refer to [this page](https://github.com/rancher/prometheus-federator/blob/main/charts/prometheus-federator/0.0.1/README.md#advanced-helm-project-operator-configuration). + + + +### Prometheus Federator on the Local Cluster + +Prometheus Federator is a resource intensive application. Installing it to the local cluster is possible, but **not recommended**. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/provisioning-storage-examples.md b/versioned_docs/version-2.8/pages-for-subheaders/provisioning-storage-examples.md new file mode 100644 index 00000000000..44fd9593fba --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/provisioning-storage-examples.md @@ -0,0 +1,15 @@ +--- +title: Provisioning Storage Examples +--- + + + + + +Rancher supports persistent storage with a variety of volume plugins. However, before you use any of these plugins to bind persistent storage to your workloads, you have to configure the storage itself, whether its a cloud-based solution from a service-provider or an on-prem solution that you manage yourself. + +For your convenience, Rancher offers documentation on how to configure some of the popular storage methods: + +- [NFS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) +- [vSphere](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) +- [EBS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/quick-start-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/quick-start-guides.md new file mode 100644 index 00000000000..8a7f8028dfb --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/quick-start-guides.md @@ -0,0 +1,20 @@ +--- +title: Rancher Deployment Quick Start Guides +--- + + + + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](installation-and-upgrade.md). + +::: + +Use this section of the docs to jump start your deployment and testing of Rancher 2.x. It contains instructions for a simple Rancher setup and some common use cases. We plan on adding more content to this section in the future. + +We have Quick Start Guides for: + +- [Deploying Rancher Server](deploy-rancher-manager.md): Get started running Rancher using the method most convenient for you. + +- [Deploying Workloads](deploy-rancher-workloads.md): Deploy a simple [workload](https://kubernetes.io/docs/concepts/workloads/) and expose it, letting you access it from outside the cluster. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-behind-an-http-proxy.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-behind-an-http-proxy.md new file mode 100644 index 00000000000..fd8a41b8e08 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-behind-an-http-proxy.md @@ -0,0 +1,17 @@ +--- +title: Installing Rancher behind an HTTP Proxy +--- + + + + + +In a lot of enterprise environments, servers or VMs running on premise do not have direct Internet access, but must connect to external services through a HTTP(S) proxy for security reasons. This tutorial shows step by step how to set up a highly available Rancher installation in such an environment. + +Alternatively, it is also possible to set up Rancher completely air-gapped without any Internet access. This process is described in detail in the [Rancher docs](air-gapped-helm-cli-install.md). + +## Installation Outline + +1. [Set up infrastructure](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md) +2. [Set up a Kubernetes cluster](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md) +3. [Install Rancher](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-hardening-guides.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-hardening-guides.md new file mode 100644 index 00000000000..5db6c192945 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-hardening-guides.md @@ -0,0 +1,56 @@ +--- +title: Self-Assessment and Hardening Guides for Rancher +--- + + + + + +Rancher provides specific security hardening guides for each supported Rancher version's Kubernetes distributions. + +## Rancher Kubernetes Distributions + +Rancher uses the following Kubernetes distributions: + +- [**RKE**](https://rancher.com/docs/rke/latest/en/), Rancher Kubernetes Engine, is a CNCF-certified Kubernetes distribution that runs entirely within Docker containers. +- [**RKE2**](https://docs.rke2.io/) is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. +- [**K3s**](https://docs.k3s.io/) is a fully conformant, lightweight Kubernetes distribution. It is easy to install, with half the memory requirement of upstream Kubernetes, all in a binary of less than 100 MB. + +To harden a Kubernetes cluster that's running a distribution other than those listed, refer to your Kubernetes provider docs. + +## Hardening Guides and Benchmark Versions + +Each self-assessment guide is accompanied by a hardening guide. These guides were tested alongside the listed Rancher releases. Each self-assessment guides was tested on a specific Kubernetes version and CIS benchmark version. If a CIS benchmark has not been validated for your Kubernetes version, you can use the existing guides until a guide for your version is added. + +### RKE Guides + +| Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|--------------------|-----------------------|-----------------------|------------------| +| Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke1-hardening-guide.md) | + +### RKE2 Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned RKE2 | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke2-hardening-guide.md) | +| Standalone RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | + +### K3s Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned K3s cluster | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](k3s-hardening-guide.md) | +| Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment) | [Link](https://docs.k3s.io/security/hardening-guide) | + +## Rancher with SELinux + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a kernel module that adds extra access controls and security tools to Linux. Historically used by government agencies, SELinux is now industry-standard. SELinux is enabled by default on RHEL and CentOS. + +To use Rancher with SELinux, we recommend [installing](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) the `rancher-selinux` RPM. + diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-managed-clusters.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-managed-clusters.md new file mode 100644 index 00000000000..2cdb03fd909 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-managed-clusters.md @@ -0,0 +1,23 @@ +--- +title: Best Practices for Rancher Managed Clusters +--- + + + + + +### Logging + +Refer to [this guide](../reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md) for our recommendations for cluster-level logging and application logging. + +### Monitoring + +Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. Refer to this [guide](../reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md) for our recommendations. + +### Tips for Setting Up Containers + +Running well-built containers can greatly impact the overall performance and security of your environment. Refer to this [guide](../reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md) for tips. + +### Best Practices for Rancher Managed vSphere Clusters + +This [guide](../reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md) outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-manager-architecture.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-manager-architecture.md new file mode 100644 index 00000000000..d7e76f28573 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-manager-architecture.md @@ -0,0 +1,21 @@ +--- +title: Architecture +--- + + + + + +This section focuses on the [Rancher server and its components](../reference-guides/rancher-manager-architecture/rancher-server-and-components.md) and how [Rancher communicates with downstream Kubernetes clusters](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md). + +For information on the different ways that Rancher can be installed, refer to the [overview of installation options.](installation-and-upgrade.md#overview-of-installation-options) + +For a list of main features of the Rancher API server, refer to the [overview section.](../getting-started/overview.md#features-of-the-rancher-api-server) + +For guidance about setting up the underlying infrastructure for the Rancher server, refer to the [architecture recommendations.](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) + +:::note + +This section assumes a basic familiarity with Docker and Kubernetes. For a brief explanation of how Kubernetes components work together, refer to the [concepts](../reference-guides/kubernetes-concepts.md) page. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker.md new file mode 100644 index 00000000000..da5b39209cf --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-on-a-single-node-with-docker.md @@ -0,0 +1,211 @@ +--- +title: Installing Rancher on a Single Node Using Docker +description: For development and testing environments only, use a Docker install. Install Docker on a single Linux host, and deploy Rancher with a single Docker container. +--- + + + + + +Rancher can be installed by running a single Docker container. + +In this installation scenario, you'll install Docker on a single Linux host, and then deploy Rancher on your host using a single Docker container. + +:::note Want to use an external load balancer? + +See [Docker Install with an External Load Balancer](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md) instead. + +::: + +A Docker installation of Rancher is recommended only for development and testing purposes. The ability to migrate Rancher to a high-availability cluster depends on the Rancher version: + +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +## Privileged Access for Rancher + +When the Rancher server is deployed in the Docker container, a local Kubernetes cluster is installed within the container for Rancher to use. Because many features of Rancher run as deployments, and privileged mode is required to run containers within containers, you will need to install Rancher with the `--privileged` option. + +## Requirements for OS, Docker, Hardware, and Networking + +Make sure that your node fulfills the general [installation requirements.](installation-requirements.md) + +## 1. Provision Linux Host + +Provision a single Linux host according to our [Requirements](installation-requirements.md) to launch your Rancher server. + +## 2. Choose an SSL Option and Install Rancher + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +:::tip Do you want to.. + +- Use a proxy? See [HTTP Proxy Configuration](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) +- Configure custom CA root certificate to access your services? See [Custom CA root certificate](../reference-guides/single-node-rancher-in-docker/advanced-options.md#custom-ca-certificate/) +- Complete an Air Gap Installation? See [Air Gap: Docker Install](air-gapped-helm-cli-install.md) +- Record all transactions with the Rancher API? See [API Auditing](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) + +::: + +Choose from the following options: + +- [Option A: Default Rancher-generated Self-signed Certificate](#option-a-default-rancher-generated-self-signed-certificate) +- [Option B: Bring Your Own Certificate, Self-signed](#option-b-bring-your-own-certificate-self-signed) +- [Option C: Bring Your Own Certificate, Signed by a Recognized CA](#option-c-bring-your-own-certificate-signed-by-a-recognized-ca) +- [Option D: Let's Encrypt Certificate](#option-d-lets-encrypt-certificate) +- [Option E: Localhost tunneling, no Certificate](#option-e-localhost-tunneling-no-certificate) + +### Option A: Default Rancher-generated Self-signed Certificate + +If you are installing Rancher in a development or testing environment where identity verification isn't a concern, install Rancher using the self-signed certificate that it generates. This installation option omits the hassle of generating a certificate yourself. + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest +``` + +### Option B: Bring Your Own Certificate, Self-signed +In development or testing environments where your team will access your Rancher server, create a self-signed certificate for use with your install so that your team can verify they're connecting to your instance of Rancher. + +:::note Prerequisites: + +Create a self-signed certificate using [OpenSSL](https://www.openssl.org/) or another method of your choice. + +- The certificate files must be in PEM format. +- In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +::: + +After creating your certificate, run the Docker command below to install Rancher. Use the `-v` flag and provide the path to your certificates to mount them in your container. + +| Placeholder | Description | +| ------------------- | --------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | +| `` | The path to the certificate authority's certificate. | + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher:latest +``` + +### Option C: Bring Your Own Certificate, Signed by a Recognized CA + +In production environments where you're exposing an app publicly, you would use a certificate signed by a recognized CA so that your user base doesn't encounter security warnings. + +The Docker install is not recommended for production. These instructions are provided for testing and development purposes only. + +:::note Prerequisites: + +- The certificate files must be in PEM format. +- In your certificate file, include all intermediate certificates provided by the recognized CA. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +::: + +After obtaining your certificate, run the Docker command below. + +- Use the `-v` flag and provide the path to your certificates to mount them in your container. Because your certificate is signed by a recognized CA, mounting an additional CA certificate file is unnecessary. +- Use the `--no-cacerts` as argument to the container to disable the default CA certificate generated by Rancher. + +| Placeholder | Description | +| ------------------- | ----------------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +### Option D: Let's Encrypt Certificate + +:::caution + +Let's Encrypt provides rate limits for requesting new certificates. Therefore, limit how often you create or destroy the container. For more information, see [Let's Encrypt documentation on rate limits](https://letsencrypt.org/docs/rate-limits/). + +::: + +For production environments, you also have the option of using [Let's Encrypt](https://letsencrypt.org/) certificates. Let's Encrypt uses an http-01 challenge to verify that you have control over your domain. You can confirm that you control the domain by pointing the hostname that you want to use for Rancher access (for example, `rancher.mydomain.com`) to the IP of the machine it is running on. You can bind the hostname to the IP address by creating an A record in DNS. + +The Docker install is not recommended for production. These instructions are provided for testing and development purposes only. + +:::note Prerequisites: + +- Let's Encrypt is an Internet service. Therefore, this option cannot be used in an internal/air gapped network. +- Create a record in your DNS that binds your Linux host IP address to the hostname that you want to use for Rancher access (`rancher.mydomain.com` for example). +- Open port `TCP/80` on your Linux host. The Let's Encrypt http-01 challenge can come from any source IP address, so port `TCP/80` must be open to all IP addresses. + +::: + +After you fulfill the prerequisites, you can install Rancher using a Let's Encrypt certificate by running the following command. + +| Placeholder | Description | +| ----------------- | ------------------- | +| `` | Your domain address | + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --acme-domain +``` + +### Option E: Localhost tunneling, no Certificate + +If you are installing Rancher in a development or testing environment where you have a localhost tunneling solution running, such as [ngrok](https://ngrok.com/), avoid generating a certificate. This installation option doesn't require a certificate. + +- You will use `--no-cacerts` in the argument to disable the default CA certificate generated by Rancher. + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +## Advanced Options + +When installing Rancher on a single node with Docker, there are several advanced options that can be enabled: + +- Custom CA Certificate +- API Audit Log +- TLS Settings +- Air Gap +- Persistent Data +- Running `rancher/rancher` and `rancher/rancher-agent` on the Same Node + +Refer to [this page](../reference-guides/single-node-rancher-in-docker/advanced-options.md) for details. + +## Troubleshooting + +Refer to [this page](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) for frequently asked questions and troubleshooting tips. + +## What's Next? + +- **Recommended:** Review Single Node [Backup](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) and [Restore](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md). Although you don't have any data you need to back up right now, we recommend creating backups after regular Rancher use. +- Create a Kubernetes cluster: [Provisioning Kubernetes Clusters](kubernetes-clusters-in-rancher-setup.md). diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-security.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-security.md new file mode 100644 index 00000000000..8207aac036f --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-security.md @@ -0,0 +1,93 @@ +--- +title: Security +--- + + + + + + + + + + + +
+

Security policy

+

Rancher Labs supports responsible disclosure, and endeavours to resolve all issues in a reasonable time frame.

+		 +

Reporting process

+

Please submit possible security issues by emailing security-rancher@suse.com .

+		 +

Announcements

+

Subscribe to the Rancher announcements forum for release updates.

+
+ +Security is at the heart of all Rancher features. From integrating with all the popular authentication tools and services, to an enterprise grade [RBAC capability](manage-role-based-access-control-rbac.md), Rancher makes your Kubernetes clusters even more secure. + +On this page, we provide security related documentation along with resources to help you secure your Rancher installation and your downstream Kubernetes clusters. + +### NeuVector Integration with Rancher + +NeuVector is an open-source, container-focused security application that is now integrated into Rancher. NeuVector provides production security, DevOps vulnerability protection, and a container firewall, et al. Please see the [Rancher docs](../integrations-in-rancher/neuvector.md) and the [NeuVector docs](https://open-docs.neuvector.com/) for more information. + +### Running a CIS Security Scan on a Kubernetes Cluster + +Rancher leverages [kube-bench](https://github.com/aquasecurity/kube-bench) to run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the [CIS](https://www.cisecurity.org/cis-benchmarks/) (Center for Internet Security) Kubernetes Benchmark. + +The CIS Kubernetes Benchmark is a reference document that can be used to establish a secure configuration baseline for Kubernetes. + +The Center for Internet Security (CIS) is a 501(c\)(3) non-profit organization, formed in October 2000, with a mission to "identify, develop, validate, promote, and sustain best practice solutions for cyber defense and build and lead communities to enable an environment of trust in cyberspace". + +CIS Benchmarks are best practices for the secure configuration of a target system. CIS Benchmarks are developed through the generous volunteer efforts of subject matter experts, technology vendors, public and private community members, and the CIS Benchmark Development team. + +The Benchmark provides recommendations of two types: Automated and Manual. We run tests related to only Automated recommendations. + +When Rancher runs a CIS security scan on a cluster, it generates a report showing the results of each test, including a summary with the number of passed, skipped and failed tests. The report also includes remediation steps for any failed tests. + +For details, refer to the section on [security scans](cis-scan-guides.md). + +### SELinux RPM + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. + +We provide two RPMs (Red Hat packages) that enable Rancher products to function properly on SELinux-enforcing hosts: `rancher-selinux` and `rke2-selinux`. For details, see [this page](selinux-rpm.md). + +### Rancher Hardening Guide + +The Rancher Hardening Guide is based on controls and best practices found in the CIS Kubernetes Benchmark from the Center for Internet Security. + +The hardening guides provide prescriptive guidance for hardening a production installation of Rancher. See Rancher's guides for [Self Assessment of the CIS Kubernetes Benchmark](#the-cis-benchmark-and-self-assessment) for the full list of security controls. + +> The hardening guides describe how to secure the nodes in your cluster, and it is recommended to follow a hardening guide before installing Kubernetes. + +Each version of the hardening guide is intended to be used with specific versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher. + +### The CIS Benchmark and Self-Assessment + +The benchmark self-assessment is a companion to the Rancher security hardening guide. While the hardening guide shows you how to harden the cluster, the benchmark guide is meant to help you evaluate the level of security of the hardened cluster. + +Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. This guide will walk through the various controls and provide updated example commands to audit compliance in Rancher created clusters. The original benchmark documents can be downloaded from the [CIS website](https://www.cisecurity.org/benchmark/kubernetes/). + +Each version of Rancher's self-assessment guide corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark. + +### Third-party Penetration Test Reports + +Rancher periodically hires third parties to perform security audits and penetration tests of the Rancher 2.x software stack. The environments under test follow the Rancher provided hardening guides at the time of the testing. Previous penetration test reports are available below. + +Results: + +- [Cure53 Pen Test - July 2019](https://releases.rancher.com/documents/security/pen-tests/2019/RAN-01-cure53-report.final.pdf) +- [Untamed Theory Pen Test - March 2019](https://releases.rancher.com/documents/security/pen-tests/2019/UntamedTheory-Rancher_SecurityAssessment-20190712_v5.pdf) + +### Rancher Security Advisories and CVEs + +Rancher is committed to informing the community of security issues in our products. For the list of CVEs (Common Vulnerabilities and Exposures) for issues we have resolved, refer to [this page.](../reference-guides/rancher-security/security-advisories-and-cves.md) + +### Kubernetes Security Best Practices + +For recommendations on securing your Kubernetes cluster, refer to the [Kubernetes Security Best Practices](../reference-guides/rancher-security/kubernetes-security-best-practices.md) guide. + +### Rancher Webhook Hardening + +The Rancher webhook deploys on both the upstream Rancher cluster and all provisioned clusters. For recommendations on hardening the Rancher webhook, see the [Hardening the Rancher Webhook](../reference-guides/rancher-security/rancher-webhook-hardening.md) guide. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-server-configuration.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-server-configuration.md new file mode 100644 index 00000000000..5e18f69e740 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-server-configuration.md @@ -0,0 +1,16 @@ +--- +title: Rancher Server Configuration +--- + + + + + +- [RKE1 Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [AKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) +- [GKE Cluster Configuration](../pages-for-subheaders/gke-cluster-configuration.md) +- [Use Existing Nodes](../pages-for-subheaders/use-existing-nodes.md) +- [Sync Clusters](../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rancher-server.md b/versioned_docs/version-2.8/pages-for-subheaders/rancher-server.md new file mode 100644 index 00000000000..45c3917cd58 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rancher-server.md @@ -0,0 +1,21 @@ +--- +title: Best Practices for the Rancher Server +--- + + + + + +This guide contains our recommendations for running the Rancher server, and is intended to be used in situations in which Rancher manages downstream Kubernetes clusters. + +### Recommended Architecture and Infrastructure + +Refer to this [guide](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md) for our general advice for setting up the Rancher server on a high-availability Kubernetes cluster. + +### Deployment Strategies + +This [guide](../reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md) is designed to help you choose whether a regional deployment strategy or a hub-and-spoke deployment strategy is better for a Rancher server that manages downstream Kubernetes clusters. + +### Installing Rancher in a vSphere Environment + +This [guide](../reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md) outlines a reference architecture for installing Rancher in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/resources.md b/versioned_docs/version-2.8/pages-for-subheaders/resources.md new file mode 100644 index 00000000000..52e61353441 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/resources.md @@ -0,0 +1,29 @@ +--- +title: Resources +--- + + + + + +### Docker Installations + +The [single-node Docker installation](rancher-on-a-single-node-with-docker.md) is for Rancher users that are wanting to test out Rancher. Instead of running on a Kubernetes cluster using Helm, you install the Rancher server component on a single node using a `docker run` command. + +Since there is only one node and a single Docker container, if the node goes down, there is no copy of the etcd data available on other nodes and you will lose all the data of your Rancher server. + +### Air-Gapped Installations + +Follow [these steps](air-gapped-helm-cli-install.md) to install the Rancher server in an air gapped environment. + +An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +### Advanced Options + +When installing Rancher, there are several advanced options that can be enabled during installation. Within each install guide, these options are presented. Learn more about these options: + +- [Custom CA Certificate](../getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md) +- [API Audit Log](../how-to-guides/advanced-user-guides/enable-api-audit-log.md) +- [TLS Settings](../getting-started/installation-and-upgrade/installation-references/tls-settings.md) +- [etcd configuration](../how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md) +- [Local System Charts for Air Gap Installations](../getting-started/installation-and-upgrade/resources/local-system-charts.md) | v2.3.0 | diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rke1-hardening-guide.md b/versioned_docs/version-2.8/pages-for-subheaders/rke1-hardening-guide.md new file mode 100644 index 00000000000..4d6f97e18c7 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rke1-hardening-guide.md @@ -0,0 +1,509 @@ +--- +title: RKE Hardening Guide +--- + + + + + +This document provides prescriptive guidance for how to harden an RKE cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | + +:::note +- In Benchmark v1.24 and later, check id `4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated)` might fail, as `/etc/kubernetes/ssl/kube-ca.pem` is set to 644 by default. +- In Benchmark v1.7, the `--protect-kernel-defaults` (`4.2.6`) parameter isn't required anymore, and was removed by CIS. +::: + +For more details on how to evaluate a hardened RKE cluster against the official CIS benchmark, refer to the RKE self-assessment guides for specific Kubernetes and CIS benchmark versions. + +## Host-level requirements + +### Configure Kernel Runtime Parameters + +The following `sysctl` configuration is recommended for all nodes types in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.overcommit_memory=1 +vm.panic_on_oom=0 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Configure `etcd` user and group + +A user account and group for the **etcd** service is required to be set up before installing RKE. + +#### Create `etcd` user and group + +To create the **etcd** user and group run the following console commands. +The commands below use `52034` for **uid** and **gid** for example purposes. +Any valid unused **uid** or **gid** could also be used in lieu of `52034`. + +```bash +groupadd --gid 52034 etcd +useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin +``` + +When deploying RKE through its cluster configuration `config.yml` file, update the `uid` and `gid` of the `etcd` user: + +```yaml +services: + etcd: + gid: 52034 + uid: 52034 +``` + +## Kubernetes runtime requirements + +### Configure `default` Service Account + +#### Set `automountServiceAccountToken` to `false` for `default` service accounts + +Kubernetes provides a default service account which is used by cluster workloads where no specific service account is assigned to the pod. +Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. +The default service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. +Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### Configure Network Policy + +#### Ensure that all Namespaces have Network Policies defined + +Running different applications on the same Kubernetes cluster creates a risk of one compromised application attacking a neighboring application. Network segmentation is important to ensure that containers can communicate only with those they are supposed to. A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. + +Network Policies are namespace scoped. When a network policy is introduced to a given namespace, all traffic not allowed by the policy is denied. However, if there are no network policies in a namespace all traffic will be allowed into and out of the pods in that namespace. To enforce network policies, a container network interface (CNI) plugin must be enabled. This guide uses [Canal](https://github.com/projectcalico/canal) to provide the policy enforcement. Additional information about CNI providers can be found [here](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/). + +Once a CNI provider is enabled on a cluster a default network policy can be applied. For reference purposes a **permissive** example is provided below. If you want to allow all traffic to all pods in a namespace (even if policies are added that cause some pods to be treated as “isolatedâ€), you can create a policy that explicitly allows all traffic in that namespace. Save the following configuration as `default-allow-all.yaml`. Additional [documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) about network policies can be found on the Kubernetes site. + +:::caution +This network policy is just an example and is not recommended for production use. +::: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +``` + +Create a bash script file called `apply_networkPolicy_to_all_ns.sh`. Be sure to `chmod +x apply_networkPolicy_to_all_ns.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl apply -f default-allow-all.yaml -n ${namespace} +done +``` + +Execute this script to apply the `default-allow-all.yaml` configuration with the **permissive** `NetworkPolicy` to all namespaces. + +## Known Limitations + +- Rancher **exec shell** and **view logs** for pods are **not** functional in a hardened setup when only a public IP is provided when registering custom nodes. This functionality requires a private IP to be provided when registering the custom nodes. +- When setting `default_pod_security_policy_template_id:` to `restricted` or `restricted-noroot`, based on the pod security policies (PSP) [provided](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) by Rancher, Rancher creates `RoleBindings` and `ClusterRoleBindings` on the `default` service accounts. The CIS check 5.1.5 requires that the `default` service accounts have no roles or cluster roles bound to it apart from the defaults. In addition, the `default` service accounts should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +## Reference Hardened RKE `cluster.yml` Configuration + +The reference `cluster.yml` is used by the RKE CLI that provides the configuration needed to achieve a hardened installation of RKE. RKE [documentation](https://rancher.com/docs/rke/latest/en/installation/) provides additional details about the configuration items. This reference `cluster.yml` does not include the required `nodes` directive which will vary depending on your environment. Documentation for node configuration in RKE can be found [here](https://rancher.com/docs/rke/latest/en/config-options/nodes/). + +The example `cluster.yml` configuration file contains an Admission Configuration policy in the `services.kube-api.admission_configuration` field. This [sample](../reference-guides/rancher-security/psa-restricted-exemptions.md) policy contains the namespace exemptions necessary for an imported RKE cluster to run properly in Rancher, similar to Rancher's pre-defined [`rancher-restricted`](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) policy. + +If you prefer to use RKE's default `restricted` policy, then leave the `services.kube-api.admission_configuration` field empty and set `services.pod_security_configuration` to `restricted`. See [the RKE docs](https://rke.docs.rancher.com/config-options/services/pod-security-admission) for more information. + + + + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + # Leave `pod_security_configuration` out if you are setting a + # custom policy in `admission_configuration`. Otherwise set + # it to `restricted` to use RKE's pre-defined restricted policy, + # and remove everything inside `admission_configuration` field. + # + # pod_security_configuration: restricted + # + admission_configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [ calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator ] + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + generate_serving_certificate: true +addons: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + generate_serving_certificate: true +addons: | + # Upstream Kubernetes restricted PSP policy + # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: restricted-noroot + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: psp:restricted-noroot + rules: + - apiGroups: + - extensions + resourceNames: + - restricted-noroot + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: psp:restricted-noroot + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-noroot + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +## Reference Hardened RKE Cluster Template Configuration + +The reference RKE cluster template provides the minimum required configuration to achieve a hardened installation of Kubernetes. RKE templates are used to provision Kubernetes and define Rancher settings. Follow the Rancher [documentation](installation-and-upgrade.md) for additional information about installing RKE and its template details. + + + + +```yaml +# +# Cluster Config +# +default_pod_security_admission_configuration_template_name: rancher-restricted +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: false + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +```yaml +# +# Cluster Config +# +default_pod_security_policy_template_id: restricted-noroot +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +## Conclusion + +If you have followed this guide, your RKE custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/rke2-hardening-guide.md b/versioned_docs/version-2.8/pages-for-subheaders/rke2-hardening-guide.md new file mode 100644 index 00000000000..378050c8e45 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/rke2-hardening-guide.md @@ -0,0 +1,274 @@ +--- +title: RKE2 Hardening Guide +--- + + + + + +This document provides prescriptive guidance for how to harden an RKE2 cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE2 clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | + +:::note +- In Benchmark v1.24 and later, some check ids might fail due to new file permission requirements (600 instead of 644). Impacted check ids: `1.1.1`, `1.1.3`, `1.1.5`, `1.1.7`, `1.1.13`, `1.1.15`, `1.1.17`, `4.1.3`, `4.1.5` and `4.1.9`. + - In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter is not required anymore, and was removed by CIS. +::: + +For more details on how to evaluate a hardened RKE2 cluster against the official CIS benchmark, refer to the RKE2 self-assessment guides for specific Kubernetes and CIS benchmark versions. + +RKE2 passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. RKE2 will not modify the host operating system. Therefore, you, the operator, must make a few host-level modifications. +2. Certain CIS controls for Network Policies and Pod Security Standards (or Pod Security Policies (PSP) on RKE2 versions prior to v1.25) will restrict the functionality of the cluster. You must opt into having RKE2 configure these for you. To help ensure these requirements are met, RKE2 can be started with the profile flag set to `cis-1.23` for v1.25 and newer or `cis-1.6` for v1.24 and older. + +## Host-level requirements + +There are two areas of host-level requirements: kernel parameters and etcd process/directory configuration. These are outlined in this section. + +### Ensure `protect-kernel-defaults` is set + + + + +The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. + + + + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + protect-kernel-defaults: true +``` + + + + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Ensure etcd is configured properly + +The CIS Benchmark requires that the etcd data directory be owned by the `etcd` user and group. This implicitly requires the etcd process run as the host-level `etcd` user. To achieve this, RKE2 takes several steps when started with a valid `cis-1.xx` profile: + +1. Check that the `etcd` user and group exists on the host. If they don't, exit with an error. +2. Create etcd's data directory with `etcd` as the user and group owner. +3. Ensure the etcd process is ran as the `etcd` user and group by setting the etcd static pod's `SecurityContext` appropriately. + +To meet the above requirements, you must: + +#### Create the etcd user + +On some Linux distributions, the `useradd` command will not create a group. The `-U` flag is included below to account for that. This flag tells `useradd` to create a group with the same name as the user. + +```bash +sudo useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U +``` + +## Kubernetes runtime requirements + +The runtime requirements to pass the CIS Benchmark are centered around pod security, network policies and kernel parameters. Most of this is automatically handled by RKE2 when using a valid `cis-1.xx` profile, but some additional operator intervention is required. These are outlined in this section. + +### PodSecurity + +RKE2 always runs with some amount of pod security. + + + + +On v1.25 and newer, [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) are used for pod security. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.7-hardened` available in Rancher. + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 +``` + +When both the `defaultPodSecurityAdmissionConfigurationTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures the Pod Security Admission Controller with the PSA configuration template `rancher-restricted`, to enforce restricted mode in all namespaces, except the ones in the template's exemption list. + These namespaces are exempted to allow system pods to run without restrictions, which is required for proper operation of the cluster. + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + + + + + +On Kubernetes v1.24 and older, the `PodSecurityPolicy` admission controller is always enabled. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. + +:::note +In the following example the profile is set to `cis-1.6` which is the value defined in the upstream RKE2, but the cluster is actually configured to pass the CIS v1.23 hardened profile +::: + +```yaml +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 +``` + + +When both the `defaultPodSecurityPolicyTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures runtime pod security policies that allow the cluster to pass associated controls. + + + + +:::note +The Kubernetes control plane components and critical additions such as CNI, DNS, and Ingress are ran as pods in the `kube-system` namespace. Therefore, this namespace will have a policy that is less restrictive so that these components can run properly. +::: + +### NetworkPolicies + +When ran with a valid `cis-1.xx` profile, RKE2 will put `NetworkPolicies` in place that passes the CIS Benchmark for Kubernetes' built-in namespaces. These namespaces are: `kube-system`, `kube-public`, `kube-node-lease`, and `default`. + +The `NetworkPolicy` used will only allow pods within the same namespace to talk to each other. The notable exception to this is that it allows DNS requests to be resolved. + +:::note +Operators must manage network policies as normal for additional namespaces that are created. +::: + +### Configure `default` service account + +**Set `automountServiceAccountToken` to `false` for `default` service accounts** + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. The `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE2 install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +For namespaces created by the cluster operator, the following script and configuration file can be used to configure the `default` service account. + +The configuration bellow must be saved to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `sudo chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + echo -n "Patching namespace $namespace - " + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### API Server audit configuration + +CIS requirements 1.2.19 to 1.2.22 are related to configuring audit logs for the API Server. When RKE2 is started with the `profile` flag set, it will automatically configure hardened `--audit-log-` parameters in the API Server to pass those CIS checks. + +RKE2's default audit policy is configured to not log requests in the API Server. This is done to allow cluster operators flexibility to customize an audit policy that suits their auditing requirements and needs, as these are specific to each users' environment and policies. + +A default audit policy is created by RKE2 when started with the `profile` flag set. The policy is defined in `/etc/rancher/rke2/audit-policy.yaml`. + +```yaml +apiVersion: audit.k8s.io/v1 +kind: Policy +metadata: + creationTimestamp: null +rules: +- level: None +``` + +## Reference Hardened RKE2 Template Configuration + +The reference template configuration is used in Rancher to create a hardened RKE2 custom cluster. This reference does not include other required **cluster configuration** directives which will vary depending on your environment. + + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 + protect-kernel-defaults: true +``` + + + + +## Conclusion + +If you have followed this guide, your RKE2 custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE2 self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/selinux-rpm.md b/versioned_docs/version-2.8/pages-for-subheaders/selinux-rpm.md new file mode 100644 index 00000000000..c72c72ff56a --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/selinux-rpm.md @@ -0,0 +1,20 @@ +--- +title: SELinux RPM +--- + + + + + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. + +Developed by Red Hat, it is an implementation of mandatory access controls (MAC) on Linux. Mandatory access controls allow an administrator of a system to define how applications and users can access different resources such as files, devices, networks and inter-process communication. SELinux also enhances security by making an OS restrictive by default. + +After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. To check whether SELinux is enabled and enforcing on your system, use `getenforce`: + +``` +# getenforce +Enforcing +``` + +We provide two RPMs (Red Hat packages) that enable Rancher products to function properly on SELinux-enforcing hosts: [`rancher-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) and [`rke2-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md). \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/set-up-cloud-providers.md b/versioned_docs/version-2.8/pages-for-subheaders/set-up-cloud-providers.md new file mode 100644 index 00000000000..9a02515dba9 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/set-up-cloud-providers.md @@ -0,0 +1,47 @@ +--- +title: Setting up Cloud Providers +--- + + + + + +A _cloud provider_ is a module in Kubernetes that provides an interface for managing nodes, load balancers, and networking routes. + +When a cloud provider is set up in Rancher, the Rancher server can automatically provision new nodes, load balancers or persistent storage devices when launching Kubernetes definitions, if the cloud provider you're using supports such automation. + +Your cluster will not provision correctly if you configure a cloud provider cluster of nodes that do not meet the prerequisites. + +By default, the **Cloud Provider** option is set to `None`. + +The following cloud providers can be enabled: + +* Amazon +* Azure +* GCE (Google Compute Engine) +* vSphere + +### Setting up the Amazon Cloud Provider + +For details on enabling the Amazon cloud provider, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md) + +### Setting up the Azure Cloud Provider + +For details on enabling the Azure cloud provider, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md) + +### Setting up the GCE Cloud Provider + +For details on enabling the Google Compute Engine cloud provider, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md) + +### Setting up the vSphere Cloud Provider + +For details on enabling the vSphere cloud provider, refer to [in-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). + +### Setting up a Custom Cloud Provider + +The `Custom` cloud provider is available if you want to configure any Kubernetes cloud provider. + +For the custom cloud provider option, you can refer to the [RKE docs](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to edit the yaml file for your specific cloud provider. There are specific cloud providers that have more detailed configuration: + +* [vSphere](https://rke.docs.rancher.com/config-options/cloud-providers/vsphere) +* [OpenStack](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/openstack/) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md b/versioned_docs/version-2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md new file mode 100644 index 00000000000..fd4f4ed5bda --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md @@ -0,0 +1,33 @@ +--- +title: Setting up Clusters from Hosted Kubernetes Providers +--- + + + + + +In this scenario, Rancher does not provision Kubernetes because it is installed by providers such as Google Kubernetes Engine (GKE), Amazon Elastic Container Service for Kubernetes, or Azure Kubernetes Service. + +If you use a Kubernetes provider such as Google GKE, Rancher integrates with its cloud APIs, allowing you to create and manage role-based access control for the hosted cluster from the Rancher UI. + +In this use case, Rancher sends a request to a hosted provider using the provider's API. The provider then provisions and hosts the cluster for you. When the cluster finishes building, you can manage it from the Rancher UI along with clusters you've provisioned that are hosted on-prem or in an infrastructure provider. + +Rancher supports the following Kubernetes providers: + +- [Google GKE (Google Kubernetes Engine)](https://cloud.google.com/kubernetes-engine/) +- [Amazon EKS (Amazon Elastic Container Service for Kubernetes)](https://aws.amazon.com/eks/) +- [Microsoft AKS (Azure Kubernetes Service)](https://azure.microsoft.com/en-us/services/kubernetes-service/) +- [Alibaba ACK (Alibaba Cloud Container Service for Kubernetes)](https://www.alibabacloud.com/product/kubernetes) +- [Tencent TKE (Tencent Kubernetes Engine)](https://intl.cloud.tencent.com/product/tke) +- [Huawei CCE (Huawei Cloud Container Engine)](https://www.huaweicloud.com/en-us/product/cce.html) + +## Hosted Kubernetes Provider Authentication + +When using Rancher to create a cluster hosted by a provider, you are prompted for authentication information. This information is required to access the provider's API. For more information on how to obtain this information, see the following procedures: + +- [Creating a GKE Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +- [Creating an EKS Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +- [Creating an AKS Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) +- [Creating an ACK Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +- [Creating a TKE Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) +- [Creating a CCE Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/single-node-rancher-in-docker.md b/versioned_docs/version-2.8/pages-for-subheaders/single-node-rancher-in-docker.md new file mode 100644 index 00000000000..91072d2b3b4 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/single-node-rancher-in-docker.md @@ -0,0 +1,9 @@ +--- +title: Single Node Rancher in Docker +--- + + + + + +The following docs will discuss [HTTP proxy configuration](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) and [advanced options](../reference-guides/single-node-rancher-in-docker/advanced-options.md) for Docker installs. \ No newline at end of file diff --git a/versioned_docs/version-2.8/pages-for-subheaders/use-existing-nodes.md b/versioned_docs/version-2.8/pages-for-subheaders/use-existing-nodes.md new file mode 100644 index 00000000000..2aeb05bc488 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/use-existing-nodes.md @@ -0,0 +1,141 @@ +--- +title: Launching Kubernetes on Existing Custom Nodes +description: To create a cluster with custom nodes, you’ll need to access servers in your cluster and provision them according to Rancher requirements +--- + + + + + +When you create a custom cluster, Rancher uses RKE (the Rancher Kubernetes Engine) to create a Kubernetes cluster in on-prem bare-metal servers, on-prem virtual machines, or in any node hosted by an infrastructure provider. + +To use this option you'll need access to servers you intend to use in your Kubernetes cluster. Provision each server according to the [requirements](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md), which includes some hardware specifications and Docker. After you install Docker on each server, you willl also run the command provided in the Rancher UI on each server to turn each one into a Kubernetes node. + +This section describes how to set up a custom cluster. + +## Creating a Cluster with Custom Nodes + +:::note Want to use Windows hosts as Kubernetes workers? + +See [Configuring Custom Clusters for Windows](use-windows-clusters.md) before you start. + +::: + +### 1. Provision a Linux Host + +Begin creation of a custom cluster by provisioning a Linux host. Your host can be: + +- A cloud-host virtual machine (VM) +- An on-prem VM +- A bare-metal server + +If you want to reuse a node from a previous custom cluster, [clean the node](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) before using it in a cluster again. If you reuse a node that hasn't been cleaned, cluster provisioning may fail. + +Provision the host according to the [installation requirements](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) and the [checklist for production-ready clusters.](checklist-for-production-ready-clusters.md) + +If you're using Amazon EC2 as your host and want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) when provisioning the host. + +### 2. Create the Custom Cluster + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Custom**. +1. Enter a **Cluster Name**. +1. Use **Cluster Configuration** section to choose the version of Kubernetes, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. + + :::note Using Windows nodes as Kubernetes workers? + + - See [Enable the Windows Support Option](use-windows-clusters.md). + - The only Network Provider available for clusters with Windows support is Flannel. + + ::: + + :::note Dual-stack on Amazon EC2: + + If you're using Amazon EC2 as your host and want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) when configuring RKE. + + ::: + +6. Click **Next**. + +4. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. + +7. From **Node Role**, choose the roles that you want filled by a cluster node. You must provision at least one node for each role: `etcd`, `worker`, and `control plane`. All three roles are required for a custom cluster to finish provisioning. For more information on roles, see [this section.](../reference-guides/kubernetes-concepts.md#roles-for-nodes-in-kubernetes-clusters) + +:::note + +- Using Windows nodes as Kubernetes workers? See [this section](use-windows-clusters.md). +- Bare-Metal Server Reminder: If you plan on dedicating bare-metal servers to each role, you must provision a bare-metal server for each role (i.e. provision multiple bare-metal servers). + +::: + +8. **Optional**: Click **[Show advanced options](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md)** to specify IP address(es) to use when registering the node, override the hostname of the node, or to add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) or [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) to the node. + +9. Copy the command displayed on screen to your clipboard. + +10. Log in to your Linux host using your preferred shell, such as PuTTy or a remote Terminal connection. Run the command copied to your clipboard. + +:::note + +Repeat steps 7-10 if you want to dedicate specific hosts to specific node roles. Repeat the steps as many times as needed. + +::: + +11. When you finish running the command(s) on your Linux host(s), click **Done**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +### 3. Amazon Only: Tag Resources + +If you have configured your cluster to use Amazon as **Cloud Provider**, tag your AWS resources with a cluster ID. + +[Amazon Documentation: Tagging Your Amazon EC2 Resources](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html) + +:::note + +You can use Amazon EC2 instances without configuring a cloud provider in Kubernetes. You only have to configure the cloud provider if you want to use specific Kubernetes cloud provider functionality. For more information, see [Kubernetes Cloud Providers](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/) + +::: + +The following resources need to be tagged with a `ClusterID`: + +- **Nodes**: All hosts added in Rancher. +- **Subnet**: The subnet used for your cluster +- **Security Group**: The security group used for your cluster. + +:::note + +Do not tag multiple security groups. Tagging multiple groups generates an error when creating Elastic Load Balancer. + +::: + +The tag that should be used is: + +``` +Key=kubernetes.io/cluster/, Value=owned +``` + +`` can be any string you choose. However, the same string must be used on every resource you tag. Setting the tag value to `owned` informs the cluster that all resources tagged with the `` are owned and managed by this cluster. + +If you share resources between clusters, you can change the tag to: + +``` +Key=kubernetes.io/cluster/CLUSTERID, Value=shared +``` + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md b/versioned_docs/version-2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md new file mode 100644 index 00000000000..d875e2325c2 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md @@ -0,0 +1,156 @@ +--- +title: Launching Kubernetes on New Nodes in an Infrastructure Provider +--- + + + + + +When you create an RKE or RKE2 cluster using a node template in Rancher, each resulting node pool is shown in a new **Machine Pools** tab. You can see the machine pools by doing the following: + +1. Click **☰ > Cluster Management**. +1. Click the name of the RKE or RKE2 cluster. + +## RKE Clusters + +Using Rancher, you can create pools of nodes based on a [node template](use-new-nodes-in-an-infra-provider.md#node-templates). This node template defines the parameters you want to use to launch nodes in your infrastructure providers or cloud providers. + +One benefit of installing Kubernetes on node pools hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically create another node to join the cluster to ensure that the count of the node pool is as expected. + +The available cloud providers to create a node template are decided based on active [node drivers](use-new-nodes-in-an-infra-provider.md#node-drivers). + +### Node Templates + +A node template is the saved configuration for the parameters to use when provisioning nodes in a specific cloud provider. These nodes can be launched from the UI. Rancher uses [Docker Machine](https://docs.docker.com/machine/) to provision these nodes. The available cloud providers to create node templates are based on the active node drivers in Rancher. + +After you create a node template in Rancher, it's saved so that you can use this template again to create node pools. Node templates are bound to your login. After you add a template, you can remove them from your user profile. + +#### Node Labels + +You can add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) on each node template, so that any nodes created from the node template will automatically have these labels on them. + +Invalid labels can prevent upgrades or can prevent Rancher from starting. For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +#### Node Taints + +You can add [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) on each node template, so that any nodes created from the node template will automatically have these taints on them. + +Since taints can be added at a node template and node pool, if there is no conflict with the same key and effect of the taints, all taints will be added to the nodes. If there are taints with the same key and different effect, the taints from the node pool will override the taints from the node template. + +#### Administrator Control of Node Templates + +Administrators can control all node templates. Admins can now maintain all the node templates within Rancher. When a node template owner is no longer using Rancher, the node templates created by them can be managed by administrators so the cluster can continue to be updated and maintained. + +To access all node templates, an administrator will need to do the following: + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. + +**Result:** All node templates are listed. The templates can be edited or cloned by clicking the **â‹®**. + +### Node Pools + +Using Rancher, you can create pools of nodes based on a [node template](#node-templates). + +A node template defines the configuration of a node, like what operating system to use, number of CPUs, and amount of memory. + +The benefit of using a node pool is that if a node is destroyed or deleted, you can increase the number of live nodes to compensate for the node that was lost. The node pool helps you ensure that the count of the node pool is as expected. + +Each node pool must have one or more nodes roles assigned. + +Each node role (i.e. etcd, controlplane, and worker) should be assigned to a distinct node pool. Although it is possible to assign multiple node roles to a node pool, this should not be done for production clusters. + +The recommended setup is to have: + +- a node pool with the etcd node role and a count of three +- a node pool with the controlplane node role and a count of at least two +- a node pool with the worker node role and a count of at least two + +**RKE1 downstream cluster nodes in an air-gapped environment:** + +By default, Rancher tries to run the Docker Install script when provisioning RKE1 downstream cluster nodes, such as in vSphere. However, the Rancher Docker installation script would fail in air-gapped environments. To work around this issue, you may choose to skip installing Docker when creating a Node Template where Docker is pre-installed onto a VM image. You can accomplish this by selecting **None** in the dropdown list for `Docker Install URL` under **Engine Options** in the Rancher UI. + +
**Engine Options Dropdown:**
+ +![Engine Options Dropdown](/img/node-template-engine-options-rke1.png) + +#### Node Pool Taints + +If you haven't defined [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) on your node template, you can add taints for each node pool. The benefit of adding taints to a node pool is that you can change the node template without having to first ensure that the taint exists in the new template. + +For each taint, they will automatically be added to any created node in the node pool. Therefore, if you add taints to a node pool that have existing nodes, the taints won't apply to existing nodes in the node pool, but any new node added into the node pool will get the taint. + +When there are taints on the node pool and node template, if there is no conflict with the same key and effect of the taints, all taints will be added to the nodes. If there are taints with the same key and different effect, the taints from the node pool will override the taints from the node template. + +#### About Node Auto-replace + +If a node is in a node pool, Rancher can automatically replace unreachable nodes. Rancher will use the existing node template for the given node pool to recreate the node if it becomes inactive for a specified number of minutes. + +:::caution + +Self-healing node pools are designed to help you replace worker nodes for stateless applications. It is not recommended to enable node auto-replace on a node pool of master nodes or nodes with persistent volumes attached, because VMs are treated ephemerally. When a node in a node pool loses connectivity with the cluster, its persistent volumes are destroyed, resulting in data loss for stateful applications. + +::: + +Node auto-replace works on top of the Kubernetes node controller. The node controller periodically checks the status of all the nodes (configurable via the `--node-monitor-period` flag of the `kube-controller`). When a node is unreachable, the node controller will taint that node. When this occurs, Rancher will begin its deletion countdown. You can configure the amount of time Rancher waits to delete the node. If the taint is not removed before the deletion countdown ends, Rancher will proceed to delete the node object. Rancher will then provision a node in accordance with the set quantity of the node pool. + +#### Enabling Node Auto-replace + +When you create the node pool, you can specify the amount of time in minutes that Rancher will wait to replace an unresponsive node. + +1. In the form for creating or editing a cluster, go to the **Node Pools** section. +1. Go to the node pool where you want to enable node auto-replace. In the **Recreate Unreachable After** field, enter the number of minutes that Rancher should wait for a node to respond before replacing the node. +1. Fill out the rest of the form for creating or editing the cluster. + +**Result:** Node auto-replace is enabled for the node pool. + +#### Disabling Node Auto-replace + +You can disable node auto-replace from the Rancher UI with the following steps: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to disable node auto-replace and click **⋮ > Edit Config**. +1. In the **Node Pools** section, go to the node pool where you want to enable node auto-replace. In the **Recreate Unreachable After** field, enter 0. +1. Click **Save**. + +**Result:** Node auto-replace is disabled for the node pool. + +### Cloud Credentials + +Node templates can use cloud credentials to store credentials for launching nodes in your cloud provider, which has some benefits: + +- Credentials are stored as a Kubernetes secret, which is not only more secure, but it also allows you to edit a node template without having to enter your credentials every time. + +- After the cloud credential is created, it can be re-used to create additional node templates. + +- Multiple node templates can share the same cloud credential to create node pools. If your key is compromised or expired, the cloud credential can be updated in a single place, which allows all node templates that are using it to be updated at once. + +After cloud credentials are created, the user can start [managing the cloud credentials that they created](../reference-guides/user-settings/manage-cloud-credentials.md). + +### Node Drivers + +If you don't find the node driver that you want to use, you can see if it is available in Rancher's built-in [node drivers and activate it](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#activatingdeactivating-node-drivers), or you can [add your own custom node driver](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#adding-custom-node-drivers). + +## RKE2 Clusters + +Rancher v2.6 introduces provisioning for [RKE2](https://docs.rke2.io/) clusters directly from the Rancher UI. RKE2, also known as RKE Government, is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. + +:::note + +For RKE2 cluster templates, please refer to [this page](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md#rke2-cluster-template) for additional information. + +::: + +### Node Roles + +The RKE2 CLI exposes two roles, `server` and `agent`, which represent the Kubernetes node-roles `etcd` + `controlplane` and `worker` respectively. With RKE2 integration in Rancher v2.6, RKE2 node pools can represent more fine-grained role assignments such that `etcd` and `controlplane` roles can be represented. + +The same functionality of using `etcd`, `controlplane` and `worker` nodes is possible in the RKE2 CLI by using flags and node tainting to control where workloads and the Kubernetes master were scheduled. The reason those roles were not implemented as first-class roles in the RKE2 CLI is that RKE2 is conceptualized as a set of raw building blocks that are best leveraged through an orchestration system such as Rancher. + +The implementation of the three node roles in Rancher means that Rancher managed RKE2 clusters are able to easily leverage all of the same architectural best practices that are recommended for RKE clusters. + +In our [recommended cluster architecture](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md), we outline how many nodes of each role clusters should have: + +- At least three nodes with the role etcd to survive losing one node +- At least two nodes with the role controlplane for master component high availability +- At least two nodes with the role worker for workload rescheduling upon node failure diff --git a/versioned_docs/version-2.8/pages-for-subheaders/use-windows-clusters.md b/versioned_docs/version-2.8/pages-for-subheaders/use-windows-clusters.md new file mode 100644 index 00000000000..36fe47c422f --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/use-windows-clusters.md @@ -0,0 +1,281 @@ +--- +title: Launching Kubernetes on Windows Clusters +--- + + + + + +When provisioning a [custom cluster](use-existing-nodes.md) using Rancher, Rancher uses RKE (the Rancher Kubernetes Engine) to install Kubernetes on your existing nodes. + +In a Windows cluster provisioned with Rancher, the cluster must contain both Linux and Windows nodes. The Kubernetes controlplane can only run on Linux nodes, and the Windows nodes can only have the worker role. Windows nodes can only be used for deploying workloads. + +Some other requirements for Windows clusters include: + +- You can only add Windows nodes to a cluster if Windows support is enabled when the cluster is created. Windows support cannot be enabled for existing clusters. +- Kubernetes 1.15+ is required. +- The Flannel network provider must be used. +- Windows nodes must have 50 GB of disk space. + +For the full list of requirements, see [this section.](#requirements-for-windows-clusters) + +For a summary of Kubernetes features supported in Windows, see the Kubernetes documentation on [supported functionality and limitations for using Kubernetes with Windows](https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#supported-functionality-and-limitations) or the [guide for scheduling Windows containers in Kubernetes](https://kubernetes.io/docs/setup/production-environment/windows/user-guide-windows-containers/). + +### RKE2 Windows + +The RKE2 provisioning feature also includes installing RKE2 on Windows clusters. Windows features for RKE2 include: + +- Windows Containers with RKE2 powered by containerd +- Added provisioning of Windows RKE2 custom clusters directly from the Rancher UI +- Calico CNI for Windows RKE2 custom clusters +- SAC releases of Windows Server (2004 and 20H2) are included in the technical preview + +Windows Support for RKE2 Custom Clusters requires choosing Calico as the CNI. + +:::note + +Rancher will allow Windows workload pods to deploy on both Windows and Linux worker nodes by default. When creating mixed clusters in RKE2, you must edit the `nodeSelector` in the chart to direct the pods to be placed onto a compatible Windows node. Refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for more information on how to use `nodeSelector` to assign pods to nodes. + +::: + +- HostProcess containers in Windows RKE2 are supported in Kubernetes v1.24.1 and up. See [the upstream documentation](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/) for more information. + +## Requirements for Windows Clusters + +The general node requirements for networking, operating systems, and Docker are the same as the node requirements for a [Rancher installation](installation-requirements.md). + +### OS and Docker Requirements + +Our support for Windows Server and Windows containers match the Microsoft official lifecycle for LTSC (Long-Term Servicing Channel) and SAC (Semi-Annual Channel). + +For the support lifecycle dates for Windows Server, see the [Microsoft Documentation.](https://docs.microsoft.com/en-us/windows-server/get-started/windows-server-release-info) + +### Kubernetes Version + +Kubernetes v1.15+ is required. + +If you are using Kubernetes v1.21 with Windows Server 20H2 Standard Core, the patch "2019-08 Servicing Stack Update for Windows Server" must be installed on the node. + +### Node Requirements + +The hosts in the cluster need to have at least: + +- 2 core CPUs +- 5 GB memory +- 50 GB disk space + +Rancher will not provision the node if the node does not meet these requirements. + +### Networking Requirements + +Before provisioning a new cluster, be sure that you have already installed Rancher on a device that accepts inbound network traffic. This is required in order for the cluster nodes to communicate with Rancher. If you have not already installed Rancher, please refer to the [installation documentation](installation-and-upgrade.md) before proceeding with this guide. + +Rancher only supports Windows using Flannel as the network provider. + +There are two network options: [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) and [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan). The default option is **VXLAN (Overlay)** mode. + +For **Host Gateway (L2bridge)** networking, it's best to use the same Layer 2 network for all nodes. Otherwise, you need to configure the route rules for them. For details, refer to the [documentation on configuring cloud-hosted VM routes.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#cloud-hosted-vm-routes-configuration) You will also need to [disable private IP address checks](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#disabling-private-ip-address-checks) if you are using Amazon EC2, Google GCE, or Azure VM. + +For **VXLAN (Overlay)** networking, the [KB4489899](https://support.microsoft.com/en-us/help/4489899) hotfix must be installed. Most cloud-hosted VMs already have this hotfix. + +If you are configuring DHCP options sets for an AWS virtual private cloud, note that in the `domain-name` option field, only one domain name can be specified. According to the DHCP options [documentation:](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_DHCP_Options.html) + +:::note + +Some Linux operating systems accept multiple domain names separated by spaces. However, other Linux operating systems and Windows treat the value as a single domain, which results in unexpected behavior. If your DHCP options set is associated with a VPC that has instances with multiple operating systems, specify only one domain name. + +::: + +### Rancher on vSphere with ESXi 6.7u2 and above + +If you are using Rancher on VMware vSphere with ESXi 6.7u2 or later with Red Hat Enterprise Linux 8.3, CentOS 8.3, or SUSE Enterprise Linux 15 SP2 or later, it is necessary to disable the `vmxnet3` virtual network adapter hardware offloading feature. Failure to do so will result in all network connections between pods on different cluster nodes to fail with timeout errors. All connections from Windows pods to critical services running on Linux nodes, such as CoreDNS, will fail as well. It is also possible that external connections may fail. This issue is the result of Linux distributions enabling the hardware offloading feature in `vmxnet3` and a bug in the `vmxnet3` hardware offloading feature that results in the discarding of packets for guest overlay traffic. To address this issue, it is necessary disable the `vmxnet3` hardware offloading feature. This setting does not survive reboot, so it is necessary to disable on every boot. The recommended course of action is to create a systemd unit file at `/etc/systemd/system/disable_hw_offloading.service`, which disables the `vmxnet3` hardware offloading feature on boot. A sample systemd unit file which disables the `vmxnet3` hardware offloading feature is as follows. Note that `` must be customized to the host `vmxnet3` network interface, e.g., `ens192`: + +``` +[Unit] +Description=Disable vmxnet3 hardware offloading feature + +[Service] +Type=oneshot +ExecStart=ethtool -K tx-udp_tnl-segmentation off +ExecStart=ethtool -K tx-udp_tnl-csum-segmentation off +StandardOutput=journal + +[Install] +WantedBy=multi-user.target +``` +Then set the appropriate permissions on the systemd unit file: +``` +chmod 0644 /etc/systemd/system/disable_hw_offloading.service +``` +Finally, enable the systemd service: +``` +systemctl enable disable_hw_offloading.service +``` + +### Architecture Requirements + +The Kubernetes cluster management nodes (`etcd` and `controlplane`) must be run on Linux nodes. + +The `worker` nodes, which is where your workloads will be deployed on, will typically be Windows nodes, but there must be at least one `worker` node that is run on Linux in order to run the Rancher cluster agent, DNS, metrics server, and Ingress related containers. + +#### Recommended Architecture + +We recommend the minimum three-node architecture listed in the table below, but you can always add additional Linux and Windows workers to scale up your cluster for redundancy: + +| Node | Operating System | Kubernetes Cluster Role(s) | Purpose | +| ------ | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| Node 1 | Linux (Ubuntu Server 18.04 recommended) | Control plane, etcd, worker | Manage the Kubernetes cluster | +| Node 2 | Linux (Ubuntu Server 18.04 recommended) | Worker | Support the Rancher Cluster agent, Metrics server, DNS, and Ingress for the cluster | +| Node 3 | Windows (Windows Server core version 1809 or above) | Worker | Run your Windows containers | + +### Container Requirements + +Windows requires that containers must be built on the same Windows Server version that they are being deployed on. Therefore, containers must be built on Windows Server core version 1809 or above. If you have existing containers built for an earlier Windows Server core version, they must be re-built on Windows Server core version 1809 or above. + +### Cloud Provider Specific Requirements + +If you set a Kubernetes cloud provider in your cluster, some additional steps are required. You might want to set a cloud provider if you want to want to leverage a cloud provider's capabilities, for example, to automatically provision storage, load balancers, or other infrastructure for your cluster. Refer to [this page](../pages-for-subheaders/set-up-cloud-providers.md) for details on how to configure a cloud provider cluster of nodes that meet the prerequisites. + +If you are using the GCE (Google Compute Engine) cloud provider, you must do the following: + +- Enable the GCE cloud provider in the `cluster.yml` by following [these steps.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md) +- When provisioning the cluster in Rancher, choose **Custom cloud provider** as the cloud provider in the Rancher UI. + +## Tutorial: How to Create a Cluster with Windows Support + +This tutorial describes how to create a Rancher-provisioned cluster with the three nodes in the [recommended architecture.](#recommended-architecture) + +When you provision a cluster with Rancher on existing nodes, you will add nodes to the cluster by installing the [Rancher agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) on each one. When you create or edit your cluster from the Rancher UI, you will see a **Customize Node Run Command** that you can run on each server to add it to your cluster. + +To set up a cluster with support for Windows nodes and containers, you will need to complete the tasks below. + + +### 1. Provision Hosts + +To begin provisioning a cluster on existing nodes with Windows support, prepare your hosts. + +Your hosts can be: + +- Cloud-hosted VMs +- VMs from virtualization clusters +- Bare-metal servers + +You will provision three nodes: + +- One Linux node, which manages the Kubernetes control plane and stores your `etcd` +- A second Linux node, which will be another worker node +- The Windows node, which will run your Windows containers as a worker node + +| Node | Operating System | +| ------ | ------------------------------------------------------------ | +| Node 1 | Linux (Ubuntu Server 18.04 recommended) | +| Node 2 | Linux (Ubuntu Server 18.04 recommended) | +| Node 3 | Windows (Windows Server core version 1809 or above required) | + +If your nodes are hosted by a **Cloud Provider** and you want automation support such as loadbalancers or persistent storage devices, your nodes have additional configuration requirements. For details, see [Selecting Cloud Providers.](../pages-for-subheaders/set-up-cloud-providers.md) + +### 2. Create the Cluster on Existing Nodes + +The instructions for creating a Windows cluster on existing nodes are very similar to the general [instructions for creating a custom cluster](use-existing-nodes.md) with some Windows-specific requirements. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Custom**. +1. Enter a name for your cluster in the **Cluster Name** field. +1. In the **Kubernetes Version** dropdown menu, select v1.19 or above. +1. In the **Network Provider** field, select **Flannel**. +1. In the **Windows Support** section, click **Enabled**. +1. Optional: After you enable Windows support, you will be able to choose the Flannel backend. There are two network options: [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) and [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan). The default option is **VXLAN (Overlay)** mode. +1. Click **Next**. + +:::note Important: + +For Host Gateway (L2bridge) networking, it's best to use the same Layer 2 network for all nodes. Otherwise, you need to configure the route rules for them. For details, refer to the [documentation on configuring cloud-hosted VM routes.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#cloud-hosted-vm-routes-configuration) You will also need to [disable private IP address checks](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#disabling-private-ip-address-checks) if you are using Amazon EC2, Google GCE, or Azure VM. + +::: + +### 3. Add Nodes to the Cluster + +This section describes how to register your Linux and Worker nodes to your cluster. You will run a command on each node, which will install the Rancher agent and allow Rancher to manage each node. + +#### Add Linux Master Node + +In this section, we fill out a form on the Rancher UI to get a custom command to install the Rancher agent on the Linux master node. Then we will copy the command and run it on our Linux master node to register the node in the cluster. + +The first node in your cluster should be a Linux host has both the **Control Plane** and **etcd** roles. At a minimum, both of these roles must be enabled for this node, and this node must be added to your cluster before you can add Windows hosts. + +1. In the **Node Operating System** section, click **Linux**. +1. In the **Node Role** section, choose at least **etcd** and **Control Plane**. We recommend selecting all three. +1. Optional: If you click **Show advanced options,** you can customize the settings for the [Rancher agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) and [node labels.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +1. Copy the command displayed on the screen to your clipboard. +1. SSH into your Linux host and run the command that you copied to your clipboard. +1. When you are finished provisioning your Linux node(s), select **Done**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +It may take a few minutes for the node to be registered in your cluster. + +#### Add Linux Worker Node + +In this section, we run a command to register the Linux worker node to the cluster. + +After the initial provisioning of your cluster, your cluster only has a single Linux host. Next, we add another Linux `worker` host, which will be used to support _Rancher cluster agent_, _Metrics server_, _DNS_ and _Ingress_ for your cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **⋮ > Edit Config**. +1. Scroll down to **Node Operating System**. Choose **Linux**. +1. In the **Customize Node Run Command** section, go to the **Node Options** and select the **Worker** role. +1. Copy the command displayed on screen to your clipboard. +1. Log in to your Linux host using a remote Terminal connection. Run the command copied to your clipboard. +1. From **Rancher**, click **Save**. + +**Result:** The **Worker** role is installed on your Linux host, and the node registers with Rancher. It may take a few minutes for the node to be registered in your cluster. + +:::note + +Taints on Linux Worker Nodes + +For each Linux worker node added into the cluster, the following taints will be added to Linux worker node. By adding this taint to the Linux worker node, any workloads added to the Windows cluster will be automatically scheduled to the Windows worker node. If you want to schedule workloads specifically onto the Linux worker node, you will need to add tolerations to those workloads. + +| Taint Key | Taint Value | Taint Effect | +| -------------- | ----------- | ------------ | +| `cattle.io/os` | `linux` | `NoSchedule` | + +::: + +#### Add a Windows Worker Node + +In this section, we run a command to register the Windows worker node to the cluster. + +You can add Windows hosts to the cluster by editing the cluster and choosing the **Windows** option. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **⋮ > Edit Config**. +1. Scroll down to **Node Operating System**. Choose **Windows**. Note: You will see that the **worker** role is the only available role. +1. Copy the command displayed on screen to your clipboard. +1. Log in to your Windows host using your preferred tool, such as [Microsoft Remote Desktop](https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-clients). Run the command copied to your clipboard in the **Command Prompt (CMD)**. +1. From Rancher, click **Save**. +1. Optional: Repeat these instructions if you want to add more Windows nodes to your cluster. + +**Result:** The **Worker** role is installed on your Windows host, and the node registers with Rancher. It may take a few minutes for the node to be registered in your cluster. You now have a Windows Kubernetes cluster. + +### Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through the Rancher server. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. + +## Configuration for Storage Classes in Azure + +If you are using Azure VMs for your nodes, you can use [Azure files](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv) as a StorageClass for the cluster. For details, refer to [this section.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md) diff --git a/versioned_docs/version-2.8/pages-for-subheaders/user-settings.md b/versioned_docs/version-2.8/pages-for-subheaders/user-settings.md new file mode 100644 index 00000000000..a9ed1c72d92 --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/user-settings.md @@ -0,0 +1,19 @@ +--- +title: User Settings +--- + + + + + +Within Rancher, each user has a number of settings associated with their login: personal preferences, API keys, etc. You can configure these settings by choosing from the **User Settings** menu. You can open this menu by clicking your avatar, located within the main menu. + +![User Settings Menu](/img/user-settings.png) + +The available user settings are: + +- [API & Keys](../reference-guides/user-settings/api-keys.md): If you want to interact with Rancher programmatically, you need an API key. Follow the directions in this section to obtain a key. +- [Cloud Credentials](../reference-guides/user-settings/manage-cloud-credentials.md): Manage cloud credentials [used by node templates](use-new-nodes-in-an-infra-provider.md#node-templates) to [provision nodes for clusters](../pages-for-subheaders/launch-kubernetes-with-rancher.md). +- [Node Templates](../reference-guides/user-settings/manage-node-templates.md): Manage templates [used by Rancher to provision nodes for clusters](../pages-for-subheaders/launch-kubernetes-with-rancher.md). +- [Preferences](../reference-guides/user-settings/user-preferences.md): Sets superficial preferences for the Rancher UI. +- Log Out: Ends your user session. diff --git a/versioned_docs/version-2.8/pages-for-subheaders/vsphere.md b/versioned_docs/version-2.8/pages-for-subheaders/vsphere.md new file mode 100644 index 00000000000..634a037c1cd --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/vsphere.md @@ -0,0 +1,60 @@ +--- +title: Creating a vSphere Cluster +description: Use Rancher to create a vSphere cluster. It may consist of groups of VMs with distinct properties which allow for fine-grained control over the sizing of nodes. +--- + + + + + +import YouTube from '@site/src/components/YouTube' + +By using Rancher with vSphere, you can bring cloud operations on-premises. + +Rancher can provision nodes in vSphere and install Kubernetes on them. When creating a Kubernetes cluster in vSphere, Rancher first provisions the specified number of virtual machines by communicating with the vCenter API. Then it installs Kubernetes on top of them. + +A vSphere cluster may consist of multiple groups of VMs with distinct properties, such as the amount of memory or the number of vCPUs. This grouping allows for fine-grained control over the sizing of nodes for each Kubernetes role. + +## vSphere Enhancements in Rancher v2.3 + +The vSphere node templates have been updated, allowing you to bring cloud operations on-premises with the following enhancements: + +### Self-healing Node Pools + +One of the biggest advantages of provisioning vSphere nodes with Rancher is that it allows you to take advantage of Rancher's self-healing node pools, also called the [node auto-replace feature,](use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) in your on-premises clusters. Self-healing node pools are designed to help you replace worker nodes for stateless applications. When Rancher provisions nodes from a node template, Rancher can automatically replace unreachable nodes. + +:::caution + +It is not recommended to enable node auto-replace on a node pool of master nodes or nodes with persistent volumes attached, because VMs are treated ephemerally. When a node in a node pool loses connectivity with the cluster, its persistent volumes are destroyed, resulting in data loss for stateful applications. + +::: + +### Dynamically Populated Options for Instances and Scheduling + +Node templates for vSphere have been updated so that when you create a node template with your vSphere credentials, the template is automatically populated with the same options for provisioning VMs that you have access to in the vSphere console. + +For the fields to be populated, your setup needs to fulfill the [prerequisites.](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md#preparation-in-vsphere) + +### More Supported Operating Systems + +You can provision VMs with any operating system that supports `cloud-init`. Only YAML format is supported for the [cloud config.](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) + +### Video Walkthrough of v2.3.3 Node Template Features + +In this YouTube video, we demonstrate how to set up a node template with the new features designed to help you bring cloud operations to on-premises clusters. + + + +## Creating a vSphere Cluster + +In [this section,](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in vSphere. + +## Provisioning Storage + +For an example of how to provision storage in vSphere using Rancher, refer to [this section.](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) In order to dynamically provision storage in vSphere, the vSphere provider must be enabled. Refer to [in-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). + +## Enabling the vSphere Cloud Provider + +When a cloud provider is set up in Rancher, the Rancher server can automatically provision new infrastructure for the cluster, including new nodes or persistent storage devices. + +For details on enabling the vSphere cloud provider, refer to [in-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). diff --git a/versioned_docs/version-2.8/pages-for-subheaders/workloads-and-pods.md b/versioned_docs/version-2.8/pages-for-subheaders/workloads-and-pods.md new file mode 100644 index 00000000000..5cfe84668af --- /dev/null +++ b/versioned_docs/version-2.8/pages-for-subheaders/workloads-and-pods.md @@ -0,0 +1,82 @@ +--- +title: "Kubernetes Workloads and Pods" +description: "Learn about the two constructs with which you can build any complex containerized application in Kubernetes: Kubernetes workloads and pods" +--- + + + + + +You can build any complex containerized application in Kubernetes using two basic constructs: pods and workloads. Once you build an application, you can expose it for access either within the same cluster or on the Internet using a third construct: services. + +### Pods + +[_Pods_](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) are one or more containers that share network namespaces and storage volumes. Most pods have only one container. Therefore when we discuss _pods_, the term is often synonymous with _containers_. You scale pods the same way you scale containers—by having multiple instances of the same pod that implement a service. Usually pods get scaled and managed by the workload. + +### Workloads + +_Workloads_ are objects that set deployment rules for pods. Based on these rules, Kubernetes performs the deployment and updates the workload with the current state of the application. +Workloads let you define the rules for application scheduling, scaling, and upgrade. + +#### Workload Types + +Kubernetes divides workloads into different types. The most popular types supported by Kubernetes are: + +- [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) + + _Deployments_ are best used for stateless applications (i.e., when you don't have to maintain the workload's state). Pods managed by deployment workloads are treated as independent and disposable. If a pod encounters disruption, Kubernetes removes it and then recreates it. An example application would be an Nginx web server. + +- [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + + _StatefulSets_, in contrast to deployments, are best used when your application needs to maintain its identity and store data. An application would be something like Zookeeper—an application that requires a database for storage. + +- [DaemonSets](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) + + _Daemonsets_ ensures that every node in the cluster runs a copy of pod. For use cases where you're collecting logs or monitoring node performance, this daemon-like workload works best. + +- [Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) + + _Jobs_ launch one or more pods and ensure that a specified number of them successfully terminate. Jobs are best used to run a finite task to completion as opposed to managing an ongoing desired application state. + +- [CronJobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) + + _CronJobs_ are similar to jobs. CronJobs, however, runs to completion on a cron-based schedule. + +### Services + +In many use cases, a workload has to be either: + +- Accessed by other workloads in the cluster. +- Exposed to the outside world. + +You can achieve these goals by creating a _Service_. Services are mapped to the underlying workload's pods using a [selector/label approach (view the code samples)](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#service-and-replicationcontroller). Rancher UI simplifies this mapping process by automatically creating a service along with the workload, using the service port and type that you select. + +#### Service Types + +There are several types of services available in Rancher. The descriptions below are sourced from the [Kubernetes Documentation](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types). + +- **ClusterIP** + + >Exposes the service on a cluster-internal IP. Choosing this value makes the service only reachable from within the cluster. This is the default `ServiceType`. + +- **NodePort** + + >Exposes the service on each Node’s IP at a static port (the `NodePort`). A `ClusterIP` service, to which the `NodePort` service will route, is automatically created. You’ll be able to contact the `NodePort` service, from outside the cluster, by requesting `:`. + +- **LoadBalancer** + + >Exposes the service externally using a cloud provider’s load balancer. `NodePort` and `ClusterIP` services, to which the external load balancer will route, are automatically created. + +## Workload Options + +This section of the documentation contains instructions for deploying workloads and using workload options. + +- [Deploy Workloads](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md) +- [Upgrade Workloads](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md) +- [Rollback Workloads](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md) + +## Related Links + +### External Links + +- [Services](https://kubernetes.io/docs/concepts/services-networking/service/) diff --git a/versioned_docs/version-2.8/rancher-manager.md b/versioned_docs/version-2.8/rancher-manager.md new file mode 100644 index 00000000000..452b4f33ae5 --- /dev/null +++ b/versioned_docs/version-2.8/rancher-manager.md @@ -0,0 +1,20 @@ +--- +slug: / +title: "What is Rancher?" +sidebar_label: What is Rancher? +description: "Rancher adds significant value on top of Kubernetes: managing hundreds of clusters from one interface, centralizing RBAC, enabling monitoring and alerting. Read more." +--- + + + + + +Rancher is a Kubernetes management tool to deploy and run clusters anywhere and on any provider. + +Rancher can provision Kubernetes from a hosted provider, provision compute nodes and then install Kubernetes onto them, or import existing Kubernetes clusters running anywhere. + +Rancher adds significant value on top of Kubernetes, first by centralizing authentication and role-based access control (RBAC) for all of the clusters, giving global admins the ability to control cluster access from one location. + +It then enables detailed monitoring and alerting for clusters and their resources, ships logs to external providers, and integrates directly with Helm via the Application Catalog. If you have an external CI/CD system, you can plug it into Rancher, but if you don't, Rancher even includes Fleet to help you automatically deploy and upgrade workloads. + +Rancher is a _complete_ container management platform for Kubernetes, giving you the tools to successfully run Kubernetes anywhere. diff --git a/versioned_docs/version-2.8/reference-guides/about-the-api/api-tokens.md b/versioned_docs/version-2.8/reference-guides/about-the-api/api-tokens.md new file mode 100644 index 00000000000..5fe8a0eb5a4 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/about-the-api/api-tokens.md @@ -0,0 +1,90 @@ +--- +title: API Tokens +--- + + + + + +By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. + +You can deactivate API tokens by deleting them or by deactivating the user account. + +### Deleting tokens +To delete a token, + +1. Go to the list of all tokens in the Rancher API view at `https:///v3/tokens`. + +1. Access the token you want to delete by its ID. For example, `https:///v3/tokens/kubectl-shell-user-vqkqt` + +1. Click **Delete**. + +Here is the complete list of tokens that are generated with `ttl=0`: + +| Token | Description | +| ----------------- | -------------------------------------------------------------------------------------- | +| `kubeconfig-*` | Kubeconfig token | +| `kubectl-shell-*` | Access to `kubectl` shell in the browser | +| `agent-*` | Token for agent deployment | +| `compose-token-*` | Token for compose | +| `helm-token-*` | Token for Helm chart deployment | +| `telemetry-*` | Telemetry token | +| `drain-node-*` | Token for drain (we use `kubectl` for drain because there is no native Kubernetes API) | + + +### Setting TTL on Kubeconfig Tokens + +Admins can set a global time-to-live (TTL) on Kubeconfig tokens. Changing the default kubeconfig TTL can be done by navigating to global settings and setting [`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) to the desired duration in minutes. The default value of [`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) is 0, which means tokens never expire. + +:::note + +This setting is used by all kubeconfig tokens except those created by the CLI to [generate kubeconfig tokens](#disable-tokens-in-generated-kubeconfigs). + +::: + +### Disable Tokens in Generated Kubeconfigs + +1. Set the `kubeconfig-generate-token` setting to `false`. This setting instructs Rancher to no longer automatically generate a token when a user clicks on download a kubeconfig file. Once this setting is deactivated, a generated kubeconfig will reference the [Rancher CLI](../cli-with-rancher/kubectl-utility.md#authentication-with-kubectl-and-kubeconfig-tokens-with-ttl) to retrieve a short-lived token for the cluster. When this kubeconfig is used in a client, such as `kubectl`, the Rancher CLI needs to be installed to complete the log in request. + +2. Set the `kubeconfig-token-ttl-minutes` setting to the desired duration in minutes. By default, `kubeconfig-token-ttl-minutes` is 960 (16 hours). + +### Token Hashing + +Users can enable token hashing, where tokens will undergo a one-way hash using the SHA256 algorithm. This is a non-reversible process, once enabled, this feature cannot be disabled. It is advisable to take backups prior to enabling and/or evaluating in a test environment first. + +To enable token hashing, refer to [this section](../../pages-for-subheaders/enable-experimental-features.md). + +This feature will affect all tokens which include, but are not limited to, the following: + +- Kubeconfig tokens +- Bearer tokens API keys/calls +- Tokens used by internal operations + +### Token Settings +These global settings affect Rancher token behavior. + +| Setting | Description | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes) | TTL in minutes on a user auth session token. | +| [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes) | Default TTL applied to all kubeconfig tokens except those [generated by Rancher CLI](#disable-tokens-in-generated-kubeconfigs). **Introduced in version 2.6.6.** | +| [`kubeconfig-token-ttl-minutes`](#kubeconfig-token-ttl-minutes) | TTL used for tokens generated via the CLI. **Deprecated since version 2.6.6, and will be removed in 2.8.0.** This setting will be removed, and `kubeconfig-default-token-TTL-minutes` will be used for all kubeconfig tokens. | +| [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) | Max TTL for all tokens except those controlled by [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes). | +| [`kubeconfig-generate-token`](#kubeconfig-generate-token) | If true, automatically generate tokens when a user downloads a kubeconfig. | + +#### auth-user-session-ttl-minutes +Time to live (TTL) duration in minutes used to determine when a user auth session token expires. When expired, the user will be required to log in and obtain a new token. This setting is not affected by [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes). Session tokens are created when a user logs into Rancher. + +#### kubeconfig-default-token-TTL-minutes +Time to live (TTL) duration in minutes used to determine when a kubeconfig token expires. When the token is expired, the API will reject the token. This setting can not be larger than [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes). This setting applies to a token generated in a requested kubeconfig file. Except those [generated by Rancher CLI](#disable-tokens-in-generated-kubeconfigs). +**Introduced in version 2.6.6**. + +#### kubeconfig-token-ttl-minutes +Time to live (TTL) duration in minutes used to determine when a kubeconfig token that was generated by the CLI expires. Tokens are generated by the CLI when [`kubeconfig-generate-token`](#kubeconfig-generate-token) is false. When the token is expired, the API will reject the token. This setting can not be larger than [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes). +**Deprecated since version 2.6.6, and will be removed in 2.8.0: This setting will be replaced with the value of [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes).** + +#### auth-token-max-ttl-minutes +Maximum Time to Live (TTL) in minutes allowed for auth tokens. If a user attempts to create a token with a TTL greater than `auth-token-max-ttl-minutes`, Rancher will set the token TTL to the value of `auth-token-max-ttl-minutes`. Auth tokens are tokens created for authenticating API requests. +**Changed in version 2.6.6: Applies to all kubeconfig tokens and api tokens.** + +#### kubeconfig-generate-token +When true, kubeconfigs requested through the UI will contain a valid token. When false, the kubeconfig will contain a command that uses the Rancher CLI to prompt the user to log in. [The CLI then will retrieve and cache a token for the user](../cli-with-rancher/kubectl-utility.md#authentication-with-kubectl-and-kubeconfig-tokens-with-ttl). diff --git a/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/backup-configuration.md b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/backup-configuration.md new file mode 100644 index 00000000000..2bb79d074b7 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/backup-configuration.md @@ -0,0 +1,196 @@ +--- +title: Backup Configuration +--- + + + + + +The **Backup: Create** page lets you configure a schedule, enable encryption and specify the storage location for your backups. + + +You must first [install](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) the `rancher-backup` operator. After you do so, you can access the **Backup: Create** page: + +## Schedule + +Select the first option to perform a one-time backup, or select the second option to schedule recurring backups. Selecting **Recurring Backups** lets you configure following two fields: + +- **Schedule**: This field accepts + - Standard [cron expressions](https://en.wikipedia.org/wiki/Cron), such as `"0 * * * *"` + - Descriptors, such as `"@midnight"` or `"@every 1h30m"` +- **Retention Count**: This value specifies how many backup files must be retained. If files exceed the given retentionCount, the oldest files will be deleted. The default value is 10. + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `schedule` | Provide the cron string for scheduling recurring backups. | +| `retentionCount` | Provide the number of backup files to be retained. | + +## Encryption + +The rancher-backup gathers resources by making calls to the kube-apiserver. Objects returned by apiserver are decrypted, so even if [encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) is enabled, even the encrypted objects gathered by the backup will be in plaintext. + +To avoid storing them in plaintext, you can use the same `EncryptionConfiguration` file that was used for at rest encryption, to encrypt certain resources in your backup. + +:::note Important: + +When encrypting objects in the backup you must save the `EncryptionConfiguration` file for future use, because it won’t be saved by the rancher-backup operator. + +For example, when [migrating Rancher to a new cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) the file is used to re-create the secret in the new cluster. + +::: + +The operator consumes the `EncryptionConfiguration` as a Kubernetes Secret in the `cattle-resources-system` namespace under the key named `encryption-provider-config.yaml` in the secret data. + +For the `EncryptionConfiguration`, you can use the [sample file provided in the Kubernetes documentation.](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#understanding-the-encryption-at-rest-configuration) + +To ensure the correct key is used in the secret, the encryption configuration file must be named `encryption-provider-config.yaml`. The below command uses the `--from-file` flag to create the secret with the correct key name. + +Save the `EncryptionConfiguration` in a file called `encryption-provider-config.yaml` and run this command: + +```bash +kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system +``` + +This will ensure that the secret contains a key named `encryption-provider-config.yaml`, and the operator will use this key to get the encryption configuration. + +The `Encryption Config Secret` dropdown will filter out and list only those Secrets that have this exact key. + +![](/img/backup_restore/backup/encryption.png) + +In the example command above, the name `encryptionconfig` can be changed to anything. + + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | Provide the name of the Secret from `cattle-resources-system` namespace, that contains the encryption config file. | + +## Storage Location + +If the StorageLocation is specified in the Backup, the operator will retrieve the backup location from that particular S3 bucket. If not specified, the operator will try to find this file in the default operator-level S3 store, and in the operator-level PVC store. The default storage location is configured during the deployment of the `rancher-backup` operator. + +Selecting the first option stores this backup in the storage location configured while installing the rancher-backup chart. The second option lets you configure a different S3 compatible storage provider for storing the backup. + +### S3 + +:::caution + +If you use an S3 backup target, make sure that every cluster has its own bucket or folder. Rancher populates snapshot information from any available snapshot listed in the S3 bucket or folder configured for that cluster. + +::: + +The S3 storage location contains the following configuration fields: + +1. **Credential Secret** (optional): If you need an AWS access key or secret key to access an S3 bucket, [create a secret](../../how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md) using your credentials, with keys and directives named `accessKey` and `secretKey`. The secret can be in any namespace. An example secret is [here](#example-credentialsecret). This directive is unnecessary if the nodes running your operator are in EC2 and assigned [IAM permissions to access S3](#iam-permissions-for-ec2-nodes-to-access-s3). The **Credential Secret** dropdown lists secrets in all namespaces. +1. **Bucket Name**: The name of the S3 bucket where backup files will be stored. +1. **Region** (optional): The AWS [region](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) where the S3 bucket is located. This field isn't needed for configuring MinIO. +1. **Folder** (optional): The name of the folder in the S3 bucket where backup files will be stored. Nested folders (e.g., `rancher/cluster1`) are not supported. If this field is left empty, the default behavior is to store the backup files in the root folder of the S3 bucket. +1. **Endpoint**: The [endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) that is used to access S3 in the region of your bucket. +1. **Endpoint CA** (optional): This should be the Base64 encoded CA cert. For an example, refer to the [example S3 compatible configuration.](#example-s3-storage-configuration) +1. **Skip TLS Verifications** (optional): Set to true if you are not using TLS. + + +#### YAML Directive Fields + +| YAML Directive Name | Description | Required | +| ---------------- | ---------------- | ------------ | +| `credentialSecretName` | If you need an AWS access key or secret key to access an S3 bucket, [create a secret](../../how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md) using your credentials, with keys and directives named `accessKey` and `secretKey`. The secret can be in any namespace. An example secret is [here](#example-credentialsecret). This directive is unnecessary if the nodes running your operator are in EC2 and assigned [IAM permissions to access S3](#iam-permissions-for-ec2-nodes-to-access-s3). | | +| `credentialSecretNamespace` | The namespace of the secret containing the credentials to access S3. This directive is unnecessary if the nodes running your operator are in EC2 and set up with IAM permissions that allow them to access S3, as described in [this section.](#iam-permissions-for-ec2-nodes-to-access-s3) | | +| `bucketName` | The name of the S3 bucket where backup files will be stored. | ✓ | +| `folder` | The name of the folder in the S3 bucket where backup files will be stored. Nested folders (e.g., `rancher/cluster1`) are not supported. If this field is left empty, the default behavior is to store the backup files in the root folder of the S3 bucket. | | +| `region` | The AWS [region](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) where the S3 bucket is located. | ✓ | +| `endpoint` | The [endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) that is used to access S3 in the region of your bucket. | ✓ | +| `endpointCA` | This should be the Base64 encoded CA cert. For an example, refer to the [example S3 compatible configuration.](#example-s3-storage-configuration) | | +| `insecureTLSSkipVerify` | Set to true if you are not using TLS. | | + +### Example S3 Storage Configuration + +```yaml +s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### Example MinIO Configuration + +```yaml +s3: + credentialSecretName: minio-creds + bucketName: rancherbackups + endpoint: minio.35.202.130.254.xip.io + endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURHakNDQWdLZ0F3SUJBZ0lKQUtpWFZpNEpBb0J5TUEwR0NTcUdTSWIzRFFFQkN3VUFNQkl4RURBT0JnTlYKQkFNTUIzUmxjM1F0WTJFd0hoY05NakF3T0RNd01UZ3lOVFE1V2hjTk1qQXhNREk1TVRneU5UUTVXakFTTVJBdwpEZ1lEVlFRRERBZDBaWE4wTFdOaE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBCjA4dnV3Q2Y0SEhtR2Q2azVNTmozRW5NOG00T2RpS3czSGszd1NlOUlXQkwyVzY5WDZxenBhN2I2M3U2L05mMnkKSnZWNDVqeXplRFB6bFJycjlpbEpWaVZ1NFNqWlFjdG9jWmFCaVNsL0xDbEFDdkFaUlYvKzN0TFVTZSs1ZDY0QQpWcUhDQlZObU5xM3E3aVY0TE1aSVpRc3N6K0FxaU1Sd0pOMVVKQTZ6V0tUc2Yzc3ByQ0J2dWxJWmZsVXVETVAyCnRCTCt6cXZEc0pDdWlhNEEvU2JNT29tVmM2WnNtTGkwMjdub3dGRld3MnRpSkM5d0xMRE14NnJoVHQ4a3VvVHYKQXJpUjB4WktiRU45L1Uzb011eUVKbHZyck9YS2ZuUDUwbk8ycGNaQnZCb3pUTStYZnRvQ1d5UnhKUmI5cFNTRApKQjlmUEFtLzNZcFpMMGRKY2sxR1h3SURBUUFCbzNNd2NUQWRCZ05WSFE0RUZnUVU5NHU4WXlMdmE2MTJnT1pyCm44QnlFQ2NucVFjd1FnWURWUjBqQkRzd09ZQVU5NHU4WXlMdmE2MTJnT1pybjhCeUVDY25xUWVoRnFRVU1CSXgKRURBT0JnTlZCQU1NQjNSbGMzUXRZMkdDQ1FDb2wxWXVDUUtBY2pBTUJnTlZIUk1FQlRBREFRSC9NQTBHQ1NxRwpTSWIzRFFFQkN3VUFBNElCQVFER1JRZ1RtdzdVNXRQRHA5Q2psOXlLRW9Vd2pYWWM2UlAwdm1GSHpubXJ3dUVLCjFrTkVJNzhBTUw1MEpuS29CY0ljVDNEeGQ3TGdIbTNCRE5mVVh2anArNnZqaXhJYXR2UWhsSFNVaWIyZjJsSTkKVEMxNzVyNCtROFkzelc1RlFXSDdLK08vY3pJTGh5ei93aHRDUlFkQ29lS1dXZkFiby8wd0VSejZzNkhkVFJzNwpHcWlGNWZtWGp6S0lOcTBjMHRyZ0xtalNKd1hwSnU0ZnNGOEcyZUh4b2pOKzdJQ1FuSkg5cGRIRVpUQUtOL2ppCnIvem04RlZtd1kvdTBndEZneWVQY1ZWbXBqRm03Y0ZOSkc4Y2ZYd0QzcEFwVjhVOGNocTZGeFBHTkVvWFZnclMKY1VRMklaU0RJd1FFY3FvSzFKSGdCUWw2RXBaUVpWMW1DRklrdFBwSQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t +``` +### Example credentialSecret + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +:::note + +To avoid encoding issues, the `credentialSecret` can be created with the below command, updating the values for `accessKey` and `secretKey`. + +```bash +kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= +``` + +::: + +### IAM Permissions for EC2 Nodes to Access S3 + +There are two ways to set up the `rancher-backup` operator to use S3 as the backup storage location. + +One way is to configure the `credentialSecretName` in the Backup custom resource, which refers to AWS credentials that have access to S3. + +If the cluster nodes are in Amazon EC2, the S3 access can also be set up by assigning IAM permissions to the EC2 nodes so that they can access S3. + +To allow a node to access S3, follow the instructions in the [AWS documentation](https://aws.amazon.com/premiumsupport/knowledge-center/ec2-instance-access-s3-bucket/) to create an IAM role for EC2. When you add a custom policy to the role, add the following permissions, and replace the `Resource` with your bucket name: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups/*" + ] + } + ] +} +``` + +After the role is created, and you have attached the corresponding instance profile to your EC2 instance(s), the `credentialSecretName` directive can be left empty in the Backup custom resource. + +## Examples + +For example Backup custom resources, refer to [this page.](examples.md#backup) diff --git a/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/examples.md b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/examples.md new file mode 100644 index 00000000000..0afd8a359fb --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/examples.md @@ -0,0 +1,284 @@ +--- +title: Examples +--- + + + + + +This section contains examples of Backup and Restore custom resources. + +The default backup storage location is configured when the `rancher-backup` operator is installed or upgraded. + +Encrypted backups can only be restored if the Restore custom resource uses the same encryption configuration secret that was used to create the backup. + +## Backup + +This section contains example Backup custom resources. + +>**Note:** Refer to the [backup config reference page](./backup-configuration.md) for more information on configuring the options below. + +### Backup in the Default Location with Encryption + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-encrypted-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### Recurring Backup in the Default Location + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-recurring-backup +spec: + resourceSetName: rancher-resource-set + schedule: "@every 1h" + retentionCount: 10 +``` + +### Encrypted Recurring Backup in the Default Location + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-enc-recurring-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 3 +``` + +### Encrypted Backup in Minio + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: minio-backup +spec: + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### Backup in S3 Using AWS Credential Secret + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### Recurring Backup in S3 Using AWS Credential Secret + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-recurring-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 +``` + +### Backup from EC2 Nodes with IAM Permission to Access S3 + +This example shows that the AWS credential secret does not have to be provided to create a backup if the nodes running `rancher-backup` have [these permissions for access to S3.](backup-configuration.md#iam-permissions-for-ec2-nodes-to-access-s3) + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-iam-backup +spec: + storageLocation: + s3: + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +## Restore + +This section contains example Restore custom resources. + +>**Note:** Refer to the [restore config reference page](./restore-configuration.md) for more information on configuring the options below. + +### Restore Using the Default Backup File Location + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-default +spec: + backupFilename: default-location-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-29-54-07-00.tar.gz +# encryptionConfigSecretName: test-encryptionconfig +``` + +### Restore for Rancher Migration +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-migration +spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + prune: false + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### Restore from Encrypted Backup + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-encrypted +spec: + backupFilename: default-test-s3-def-backup-c583d8f2-6daf-4648-8ead-ed826c591471-2020-08-24T20-47-05Z.tar.gz + encryptionConfigSecretName: encryptionconfig +``` + +### Restore an Encrypted Backup from Minio + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-minio +spec: + backupFilename: default-minio-backup-demo-aa5c04b7-4dba-4c48-9ac4-ab7916812eaa-2020-08-30T13-18-17-07-00.tar.gz + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + encryptionConfigSecretName: test-encryptionconfig +``` + +### Restore from Backup Using an AWS Credential Secret to Access S3 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: test-s3-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-49-34-07-00.tar.gz.enc + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +### Restore from EC2 Nodes with IAM Permissions to Access S3 + +This example shows that the AWS credential secret does not have to be provided to restore from backup if the nodes running `rancher-backup` have [these permissions for access to S3.](backup-configuration.md#iam-permissions-for-ec2-nodes-to-access-s3) + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: default-test-s3-recurring-backup-84bf8dd8-0ef3-4240-8ad1-fc7ec308e216-2020-08-24T10#52#44-07#00.tar.gz + storageLocation: + s3: + bucketName: rajashree-backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +## Example EncryptionConfiguration + +The snippet below demonstrates two different types of secrets and their relevance with respect to Backup and Restore of custom resources. + +The first example is that of a secret that is used to encrypt the backup files. The backup operator, in this case, will not be able to read the secrets encryption file. It only uses the contents of the secret. + +The second example is that of a Kubernetes secrets encryption config file that is used to encrypt secrets when stored in etcd. **When backing up the etcd datastore, be sure to also back up the EncryptionConfiguration.** Failure to do so will result in an inability to use the restored data if secrets encryption was in use at the time the data was backed up. + + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: + - secrets + providers: + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - secretbox: + keys: + - name: key1 + secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY= +``` + + + diff --git a/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/restore-configuration.md b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/restore-configuration.md new file mode 100644 index 00000000000..b162a6ef873 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/restore-configuration.md @@ -0,0 +1,85 @@ +--- +title: Restore Configuration +--- + + + + + +The Restore Create page lets you provide details of the backup to restore from + +![](/img/backup_restore/restore/restore.png) + + +## Backup Source +Provide details of the backup file and its storage location, which the operator will then use to perform the restore. Select from the following options to provide these details + + + + +### An existing backup config + +Selecting this option will populate the **Target Backup** dropdown with the Backups available in this cluster. Select the Backup from the dropdown, and that will fill out the **Backup Filename** field for you, and will also pass the backup source information from the selected Backup to the operator. + +![](/img/backup_restore/restore/existing.png) + +If the Backup custom resource does not exist in the cluster, you need to get the exact filename and provide the backup source details with the default storage target or an S3-compatible object store. + + +### The default storage target + +Select this option if you are restoring from a backup file that exists in the default storage location configured at the operator-level. The operator-level configuration is the storage location that was configured when the `rancher-backup` operator was installed or upgraded. Provide the exact filename in the **Backup Filename** field. + +![](/img/backup_restore/restore/default.png) + +### An S3-compatible object store + +Select this option if no default storage location is configured at the operator-level, OR if the backup file exists in a different S3 bucket than the one configured as the default storage location. Provide the exact filename in the **Backup Filename** field. Refer to [this section](#getting-the-backup-filename-from-s3) for exact steps on getting the backup filename from s3. Fill in all the details for the S3 compatible object store. Its fields are exactly same as ones for the `backup.StorageLocation` configuration in the [Backup custom resource.](backup-configuration.md#storage-location) + +![](/img/backup_restore/restore/s3store.png) + +## Encryption + +If the backup was created with encryption enabled, its file will have `.enc` suffix. Choosing such a Backup, or providing a backup filename with `.enc` suffix will display another dropdown named **Encryption Config Secret**. + +![](/img/backup_restore/restore/encryption.png) + +The Secret selected from this dropdown must have the same contents as the one used for the Backup custom resource while performing the backup. If the encryption configuration doesn't match, the restore will fail + +The `Encryption Config Secret` dropdown will filter out and list only those Secrets that have this exact key + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | Provide the name of the Secret from `cattle-resources-system` namespace, that contains the encryption config file. | + +:::note Important: + +This field should only be set if the backup was created with encryption enabled. Providing the incorrect encryption config will cause the restore to fail. + +::: + +## Prune During Restore + +* **Prune**: In order to fully restore Rancher from a backup, and to go back to the exact state it was at when the backup was performed, we need to delete any additional resources that were created by Rancher after the backup was taken. The operator does so if the **Prune** flag is enabled. Prune is enabled by default and it is recommended to keep it enabled. +* **Delete Timeout**: This is the amount of time the operator will wait while deleting a resource before editing the resource to remove finalizers and attempt deletion again. + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `prune` | Delete the resources managed by Rancher that are not present in the backup (Recommended). | +| `deleteTimeoutSeconds` | Amount of time the operator will wait while deleting a resource before editing the resource to remove finalizers and attempt deletion again. | + +## Getting the Backup Filename from S3 + +This is the name of the backup file that the `rancher-backup` operator will use to perform the restore. + +To obtain this file name from S3, go to your S3 bucket (and folder if it was specified while performing backup). + +Copy the filename and store it in your Restore custom resource. So assuming the name of your backup file is `backupfile`, + +- If your bucket name is `s3bucket` and no folder was specified, then the `backupFilename` to use will be `backupfile`. +- If your bucket name is `s3bucket` and the base folder is`s3folder`, the `backupFilename` to use is only `backupfile` . +- If there is a subfolder inside `s3Folder` called `s3sub`, and that has your backup file, then the `backupFilename` to use is `s3sub/backupfile`. + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `backupFilename` | This is the name of the backup file that the `rancher-backup` operator will use to perform the restore. | diff --git a/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/storage-configuration.md b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/storage-configuration.md new file mode 100644 index 00000000000..b1b8bad0896 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/backup-restore-configuration/storage-configuration.md @@ -0,0 +1,60 @@ +--- +title: Backup Storage Location Configuration +--- + + + + + +Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible object store. + +Only one storage location can be configured at the operator level. + + +## Storage Location Configuration + +### No Default Storage Location + +You can choose to not have any operator-level storage location configured. If you select this option, you must configure an S3-compatible object store as the storage location for each individual backup. + +### S3-compatible Object Store + +| Parameter | Description | +| -------------- | -------------- | +| Credential Secret | Choose the credentials for S3 from your secrets in Rancher. [Example](examples.md#example-credential-secret-for-storing-backups-in-s3). | +| Bucket Name | Enter the name of the [S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html) where the backups will be stored. Default: `rancherbackups`. | +| Region | The [AWS region](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) where the S3 bucket is located. | +| Folder | The [folder in the S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html) where the backups will be stored. If this field is left empty, the default behavior is to store the backup files in the root folder of the S3 bucket. | +| Endpoint | The [S3 endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) For example, `s3.us-west-2.amazonaws.com`. | +| Endpoint CA | The CA cert used to for the S3 endpoint. Default: base64 encoded CA cert | +| insecureTLSSkipVerify | Set to true if you are not using TLS. | + +### Existing StorageClass + +Installing the `rancher-backup` chart by selecting the StorageClass option will create a Persistent Volume Claim (PVC), and Kubernetes will in turn dynamically provision a Persistent Volume (PV) where all the backups will be saved by default. + +For information about creating storage classes refer to [this section.](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md) + +:::note Important: + +It is highly recommended to use a StorageClass with a reclaim policy of "Retain". Otherwise if the PVC created by the `rancher-backup` chart gets deleted (either during app upgrade, or accidentally), the PV will get deleted too, which means all backups saved in it will get deleted. +If no such StorageClass is available, after the PV is provisioned, make sure to edit its reclaim policy and set it to "Retain" before storing backups in it. + +::: + +### Existing Persistent Volume + +Select an existing Persistent Volume (PV) that will be used to store your backups. For information about creating PersistentVolumes in Rancher, refer to [this section.](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md#2-add-a-persistentvolume-that-refers-to-the-persistent-storage) + +:::note Important: + +It is highly recommended to use a Persistent Volume with a reclaim policy of "Retain". Otherwise if the PVC created by the `rancher-backup` chart gets deleted (either during app upgrade, or accidentally), the PV will get deleted too, which means all backups saved in it will get deleted. + +::: + +## Example values.yaml for the rancher-backup Helm Chart + +The documented `values.yaml` file that can be used to configure `rancher-backup` operator when the Helm CLI is used can be found in the [backup-restore-operator repository.](https://github.com/rancher/backup-restore-operator/blob/master/charts/rancher-backup/values.yaml) + +For more information about `values.yaml` files and configuring Helm charts during installation, refer to the [Helm documentation.](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing) + diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md new file mode 100644 index 00000000000..dc200ae62b1 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -0,0 +1,92 @@ +--- +title: Logging Best Practices +--- + + + + + +In this guide, we recommend best practices for cluster-level logging and application logging. + +- [Cluster-level Logging](#cluster-level-logging) +- [Application Logging](#application-logging) +- [General Best Practices](#general-best-practices) + +Before Rancher v2.5, logging in Rancher has historically been a pretty static integration. There were a fixed list of aggregators to choose from (ElasticSearch, Splunk, Kafka, Fluentd and Syslog), and only two configuration points to choose (Cluster-level and Project-level). + +Rancher provides a flexible experience for log aggregation. With the logging feature, administrators and users alike can deploy logging that meets fine-grained collection criteria while offering a wider array of destinations and configuration options. + +"Under the hood", Rancher logging uses the [Logging operator](https://github.com/kube-logging/logging-operator). We provide manageability of this operator (and its resources), and tie that experience in with managing your Rancher clusters. + +## Cluster-level Logging + +### Cluster-wide Scraping + +For some users, it is desirable to scrape logs from every container running in the cluster. This usually coincides with your security team's request (or requirement) to collect all logs from all points of execution. + +In this scenario, it is recommended to create at least two _ClusterOutput_ objects - one for your security team (if you have that requirement), and one for yourselves, the cluster administrators. When creating these objects take care to choose an output endpoint that can handle the significant log traffic coming from the entire cluster. Also make sure to choose an appropriate index to receive all these logs. + +Once you have created these _ClusterOutput_ objects, create a _ClusterFlow_ to collect all the logs. Do not define any _Include_ or _Exclude_ rules on this flow. This will ensure that all logs from across the cluster are collected. If you have two _ClusterOutputs_, make sure to send logs to both of them. + +### Kubernetes Components + +_ClusterFlows_ have the ability to collect logs from all containers on all hosts in the Kubernetes cluster. This works well in cases where those containers are part of a Kubernetes pod; however, RKE containers exist outside of the scope of Kubernetes. + +Currently the logs from RKE containers are collected, but are not able to easily be filtered. This is because those logs do not contain information as to the source container (e.g. `etcd` or `kube-apiserver`). + +A future release of Rancher will include the source container name which will enable filtering of these component logs. Once that change is made, you will be able to customize a _ClusterFlow_ to retrieve **only** the Kubernetes component logs, and direct them to an appropriate output. + +## Application Logging + +Best practice not only in Kubernetes but in all container-based applications is to direct application logs to `stdout`/`stderr`. The container runtime will then trap these logs and do **something** with them - typically writing them to a file. Depending on the container runtime (and its configuration), these logs can end up in any number of locations. + +In the case of writing the logs to a file, Kubernetes helps by creating a `/var/log/containers` directory on each host. This directory symlinks the log files to their actual destination (which can differ based on configuration or container runtime). + +Rancher logging will read all log entries in `/var/log/containers`, ensuring that all log entries from all containers (assuming a default configuration) will have the opportunity to be collected and processed. + +### Specific Log Files + +Log collection only retrieves `stdout`/`stderr` logs from pods in Kubernetes. But what if we want to collect logs from other files that are generated by applications? Here, a log streaming sidecar (or two) may come in handy. + +The goal of setting up a streaming sidecar is to take log files that are written to disk, and have their contents streamed to `stdout`. This way, the Logging Operator can pick up those logs and send them to your desired output. + +To set this up, edit your workload resource (e.g. Deployment) and add the following sidecar definition: + +```yaml +... +containers: +- args: + - -F + - /path/to/your/log/file.log + command: + - tail + image: busybox + name: stream-log-file-[name] + volumeMounts: + - mountPath: /path/to/your/log + name: mounted-log +... +``` + +This will add a container to your workload definition that will now stream the contents of (in this example) `/path/to/your/log/file.log` to `stdout`. + +This log stream is then automatically collected according to any _Flows_ or _ClusterFlows_ you have setup. You may also wish to consider creating a _Flow_ specifically for this log file by targeting the name of the container. See example: + +```yaml +... +spec: + match: + - select: + container_names: + - stream-log-file-name +... +``` + + +## General Best Practices + +- Where possible, output structured log entries (e.g. `syslog`, JSON). This makes handling of the log entry easier as there are already parsers written for these formats. +- Try to provide the name of the application that is creating the log entry, in the entry itself. This can make troubleshooting easier as Kubernetes objects do not always carry the name of the application as the object name. For instance, a pod ID may be something like `myapp-098kjhsdf098sdf98` which does not provide much information about the application running inside the container. +- Except in the case of collecting all logs cluster-wide, try to scope your _Flow_ and _ClusterFlow_ objects tightly. This makes it easier to troubleshoot when problems arise, and also helps ensure unrelated log entries do not show up in your aggregator. An example of tight scoping would be to constrain a _Flow_ to a single _Deployment_ in a namespace, or perhaps even a single container within a _Pod_. +- Keep the log verbosity down except when troubleshooting. High log verbosity poses a number of issues, chief among them being **noise**: significant events can be drowned out in a sea of `DEBUG` messages. This is somewhat mitigated with automated alerting and scripting, but highly verbose logging still places an inordinate amount of stress on the logging infrastructure. +- Where possible, try to provide a transaction or request ID with the log entry. This can make tracing application activity across multiple log sources easier, especially when dealing with distributed applications. diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md new file mode 100644 index 00000000000..a6e34b03a50 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -0,0 +1,115 @@ +--- +title: Monitoring Best Practices +--- + + + + + +Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. This is not different when using Kubernetes and Rancher. Fortunately the integrated monitoring and alerting functionality makes this whole process a lot easier. + +The [Rancher monitoring documentation](../../../pages-for-subheaders/monitoring-and-alerting.md) describes how you can set up a complete Prometheus and Grafana stack. Out of the box this will scrape monitoring data from all system and Kubernetes components in your cluster and provide sensible dashboards and alerts for them to get started. But for a reliable setup, you also need to monitor your own workloads and adapt Prometheus and Grafana to your own specific use cases and cluster sizes. This document aims to give you best practices for this. + +## What to Monitor + +Kubernetes itself, as well as applications running inside of it, form a distributed system where different components interact with each other. For the whole system and each individual component, you have to ensure performance, availability, reliability and scalability. A good resource with more details and information is Google's free [Site Reliability Engineering Book](https://sre.google/sre-book/table-of-contents/), especially the chapter about [Monitoring distributed systems](https://sre.google/sre-book/monitoring-distributed-systems/). + +## Configuring Prometheus Resource Usage + +When installing the integrated monitoring stack, Rancher allows to configure several settings that are dependent on the size of your cluster and the workloads running in it. This chapter covers these in more detail. + +### Storage and Data Retention + +The amount of storage needed for Prometheus directly correlates to the amount of time series and labels that you store and the data retention you have configured. It is important to note that Prometheus is not meant to be used as a long-term metrics storage. Data retention time is usually only a couple of days and not weeks or months. The reason for this is that Prometheus does not perform any aggregation on its stored metrics. This is great because aggregation can dilute data, but it also means that the needed storage grows linearly over time without retention. + +One way to calculate the necessary storage is to look at the average size of a storage chunk in Prometheus with this query + +``` +rate(prometheus_tsdb_compaction_chunk_size_bytes_sum[1h]) / rate(prometheus_tsdb_compaction_chunk_samples_sum[1h]) +``` + +Next, find out your data ingestion rate per second: + +``` +rate(prometheus_tsdb_head_samples_appended_total[1h]) +``` + +and then multiply this with the retention time, adding a few percentage points as buffer: + +``` +average chunk size in bytes * ingestion rate per second * retention time in seconds * 1.1 = necessary storage in bytes +``` + +You can find more information about how to calculate the necessary storage in this [blog post](https://www.robustperception.io/how-much-disk-space-do-prometheus-blocks-use). + +You can read more about the Prometheus storage concept in the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/storage). + +### CPU and Memory Requests and Limits + +In larger Kubernetes clusters Prometheus can consume quite a bit of memory. The amount of memory Prometheus needs directly correlates to the amount of time series and amount of labels it stores and the scrape interval in which these are filled. + +You can find more information about how to calculate the necessary memory in this [blog post](https://www.robustperception.io/how-much-ram-does-prometheus-2-x-need-for-cardinality-and-ingestion). + +The amount of necessary CPUs correlate with the amount of queries you are performing. + +### Federation and Long-term Storage + +Prometheus is not meant to store metrics for a long amount of time, but should only be used for short term storage. + +In order to store some, or all metrics for a long time, you can leverage Prometheus' [remote read/write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations) capabilities to connect it to storage systems like [Thanos](https://thanos.io/), [InfluxDB](https://www.influxdata.com/), [M3DB](https://www.m3db.io/), or others. You can find an example setup in this [blog post](https://rancher.com/blog/2020/prometheus-metric-federation). + +## Scraping Custom Workloads + +While the integrated Rancher Monitoring already scrapes system metrics from a cluster's nodes and system components, the custom workloads that you deploy on Kubernetes should also be scraped for data. For that you can configure Prometheus to do an HTTP request to an endpoint of your applications in a certain interval. These endpoints should then return their metrics in a Prometheus format. + +In general, you want to scrape data from all the workloads running in your cluster so that you can use them for alerts or debugging issues. Often, you recognize that you need some data only when you actually need the metrics during an incident. It is good, if it is already scraped and stored. Since Prometheus is only meant to be a short-term metrics storage, scraping and keeping lots of data is usually not that expensive. If you are using a long-term storage solution with Prometheus, you can then still decide which data you are actually persisting and keeping there. + +### About Prometheus Exporters + +A lot of 3rd party workloads like databases, queues or web-servers either already support exposing metrics in a Prometheus format, or there are so called exporters available that translate between the tool's metrics and the format that Prometheus understands. Usually you can add these exporters as additional sidecar containers to the workload's Pods. A lot of helm charts already include options to deploy the correct exporter. Additionally you can find a curated list of exports by SysDig on [promcat.io](https://promcat.io/) and on [ExporterHub](https://exporterhub.io/). + +### Prometheus support in Programming Languages and Frameworks + +To get your own custom application metrics into Prometheus, you have to collect and expose these metrics directly from your application's code. Fortunately, there are already libraries and integrations available to help with this for most popular programming languages and frameworks. One example for this is the Prometheus support in the [Spring Framework](https://docs.spring.io/spring-metrics/docs/current/public/prometheus). + +### ServiceMonitors and PodMonitors + +Once all your workloads expose metrics in a Prometheus format, you have to configure Prometheus to scrape it. Under the hood Rancher is using the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator). This makes it easy to add additional scraping targets with ServiceMonitors and PodMonitors. A lot of helm charts already include an option to create these monitors directly. You can also find more information in the Rancher documentation. + +### Prometheus Push Gateway + +There are some workloads that are traditionally hard to scrape by Prometheus. Examples for these are short lived workloads like Jobs and CronJobs, or applications that do not allow sharing data between individual handled incoming requests, like PHP applications. + +To still get metrics for these use cases, you can set up [prometheus-pushgateways](https://github.com/prometheus/pushgateway). The CronJob or PHP application would push metric updates to the pushgateway. The pushgateway aggregates and exposes them through an HTTP endpoint, which then can be scraped by Prometheus. + +### Prometheus Blackbox Monitor + +Sometimes it is useful to monitor workloads from the outside. For this, you can use the [Prometheus blackbox-exporter](https://github.com/prometheus/blackbox_exporter) which allows probing any kind of endpoint over HTTP, HTTPS, DNS, TCP and ICMP. + +## Monitoring in a (Micro)Service Architecture + +If you have a (micro)service architecture where multiple individual workloads within your cluster are communicating with each other, it is really important to have detailed metrics and traces about this traffic to understand how all these workloads are communicating with each other and where a problem or bottleneck may be. + +Of course you can monitor all this internal traffic in all your workloads and expose these metrics to Prometheus. But this can quickly become quite work intensive. Service Meshes like Istio, which can be installed with [a click](../../../pages-for-subheaders/istio.md) in Rancher, can do this automatically and provide rich telemetry about the traffic between all services. + +## Real User Monitoring + +Monitoring the availability and performance of all your internal workloads is vitally important to run stable, reliable and fast applications. But these metrics only show you parts of the picture. To get a complete view it is also necessary to know how your end users are actually perceiving it. For this you can look into various [Real user monitoring solutions](https://en.wikipedia.org/wiki/Real_user_monitoring). + +## Security Monitoring + +In addition to monitoring workloads to detect performance, availability or scalability problems, the cluster and the workloads running into it should also be monitored for potential security problems. A good starting point is to frequently run and alert on [CIS Scans](../../../pages-for-subheaders/cis-scan-guides.md) which check if the cluster is configured according to security best practices. + +For the workloads, you can have a look at Kubernetes and Container security solutions like [NeuVector](https://www.suse.com/products/neuvector/), [Falco](https://falco.org/), [Aqua Kubernetes Security](https://www.aquasec.com/solutions/kubernetes-container-security/), [SysDig](https://sysdig.com/). + +## Setting up Alerts + +Getting all the metrics into a monitoring systems and visualizing them in dashboards is great, but you also want to be pro-actively alerted if something goes wrong. + +The integrated Rancher monitoring already configures a sensible set of alerts that make sense in any Kubernetes cluster. You should extend these to cover your specific workloads and use cases. + +When setting up alerts, configure them for all the workloads that are critical to the availability of your applications. But also make sure that they are not too noisy. Ideally every alert you are receiving should be because of a problem that needs your attention and needs to be fixed. If you have alerts that are firing all the time but are not that critical, there is a danger that you start ignoring your alerts all together and then miss the real important ones. Less may be more here. Start to focus on the real important metrics first, for example alert if your application is offline. Fix all the problems that start to pop up and then start to create more detailed alerts. + +If an alert starts firing, but there is nothing you can do about it at the moment, it's also fine to silence the alert for a certain amount of time, so that you can look at it later. + +You can find more information on how to set up alerts and notification channels in the [Rancher Documentation](../../../pages-for-subheaders/monitoring-and-alerting.md). diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md new file mode 100644 index 00000000000..9d24fb3c5db --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -0,0 +1,62 @@ +--- +title: Best Practices for Rancher Managed vSphere Clusters +--- + + + + + +This guide outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. + +- [1. VM Considerations](#1-vm-considerations) +- [2. Network Considerations](#2-network-considerations) +- [3. Storage Considerations](#3-storage-considerations) +- [4. Backups and Disaster Recovery](#4-backups-and-disaster-recovery) + +
Solution Overview
+ +![Solution Overview](/img/solution_overview.drawio.svg) + +## 1. VM Considerations + +### Leverage VM Templates to Construct the Environment + +To facilitate consistency across the deployed Virtual Machines across the environment, consider the use of "Golden Images" in the form of VM templates. Packer can be used to accomplish this, adding greater customisation options. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Downstream Cluster Nodes Across ESXi Hosts + +Doing so will ensure node VM's are spread across multiple ESXi hosts - preventing a single point of failure at the host level. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Downstream Cluster Nodes Across Datastores + +Doing so will ensure node VM's are spread across multiple datastores - preventing a single point of failure at the datastore level. + +### Configure VM's as Appropriate for Kubernetes + +It’s important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double-checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node. + +## 2. Network Considerations + +### Leverage Low Latency, High Bandwidth Connectivity Between ETCD Nodes + +Deploy etcd members within a single data center where possible to avoid latency overheads and reduce the likelihood of network partitioning. For most setups, 1Gb connections will suffice. For large clusters, 10Gb connections can reduce the time taken to restore from backup. + +### Consistent IP Addressing for VM's + +Each node used should have a static IP configured. In the case of DHCP, each node should have a DHCP reservation to make sure the node gets the same IP allocated. + +## 3. Storage Considerations + +### Leverage SSD Drives for ETCD Nodes + +ETCD is very sensitive to write latency. Therefore, leverage SSD disks where possible. + +## 4. Backups and Disaster Recovery + +### Perform Regular Downstream Cluster Backups + +Kubernetes uses etcd to store all its data - from configuration, state and metadata. Backing this up is crucial in the event of disaster recovery. + +### Back up Downstream Node VMs + +Incorporate the Rancher downstream node VM's within a standard VM backup policy. \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md new file mode 100644 index 00000000000..0cec0a91864 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -0,0 +1,56 @@ +--- +title: Tips for Setting Up Containers +--- + + + + + +Running well-built containers can greatly impact the overall performance and security of your environment. + +Below are a few tips for setting up your containers. + +For a more detailed discussion of security for containers, you can also refer to Rancher's [Guide to Container Security.](https://rancher.com/complete-guide-container-security) + +### Use a Common Container OS + +When possible, you should try to standardize on a common container base OS. + +Smaller distributions such as Alpine and BusyBox reduce container image size and generally have a smaller attack/vulnerability surface. + +Popular distributions such as Ubuntu, Fedora, and CentOS are more field-tested and offer more functionality. + +### Start with a FROM scratch container +If your microservice is a standalone static binary, you should use a FROM scratch container. + +The FROM scratch container is an [official Docker image](https://hub.docker.com/_/scratch) that is empty so that you can use it to design minimal images. + +This will have the smallest attack surface and smallest image size. + +### Run Container Processes as Unprivileged +When possible, use a non-privileged user when running processes within your container. While container runtimes provide isolation, vulnerabilities and attacks are still possible. Inadvertent or accidental host mounts can also be impacted if the container is running as root. For details on configuring a security context for a pod or container, refer to the [Kubernetes docs](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + +### Define Resource Limits +Apply CPU and memory limits to your pods. This can help manage the resources on your worker nodes and avoid a malfunctioning microservice from impacting other microservices. + +In standard Kubernetes, you can set resource limits on the namespace level. In Rancher, you can set resource limits on the project level and they will propagate to all the namespaces within the project. For details, refer to the Rancher docs. + +When setting resource quotas, if you set anything related to CPU or Memory (i.e. limits or reservations) on a project or namespace, all containers will require a respective CPU or Memory field set during creation. To avoid setting these limits on each and every container during workload creation, a default container resource limit can be specified on the namespace. + +The Kubernetes docs have more information on how resource limits can be set at the [container level](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) and the namespace level. + +### Define Resource Requirements +You should apply CPU and memory requirements to your pods. This is crucial for informing the scheduler which type of compute node your pod needs to be placed on, and ensuring it does not over-provision that node. In Kubernetes, you can set a resource requirement by defining `resources.requests` in the resource requests field in a pod's container spec. For details, refer to the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container). + +:::note + +If you set a resource limit for the namespace that the pod is deployed in, and the container doesn't have a specific resource request, the pod will not be allowed to start. To avoid setting these fields on each and every container during workload creation, a default container resource limit can be specified on the namespace. + +::: + +It is recommended to define resource requirements on the container level because otherwise, the scheduler makes assumptions that will likely not be helpful to your application when the cluster experiences load. + +### Liveness and Readiness Probes +Set up liveness and readiness probes for your container. Unless your container completely crashes, Kubernetes will not know it's unhealthy unless you create an endpoint or mechanism that can report container status. Alternatively, make sure your container halts and crashes if unhealthy. + +The Kubernetes docs show how to [configure liveness and readiness probes for containers.](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md new file mode 100644 index 00000000000..74243559753 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -0,0 +1,88 @@ +--- +title: Installing Rancher in a vSphere Environment +--- + + + + + +This guide outlines a reference architecture for installing Rancher on an RKE Kubernetes cluster in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. + + +
Solution Overview
+ +![Solution Overview](/img/rancher-on-prem-vsphere.svg) + +## 1. Load Balancer Considerations + +A load balancer is required to direct traffic to the Rancher workloads residing on the RKE nodes. + +### Leverage Fault Tolerance and High Availability + +Leverage the use of an external (hardware or software) load balancer that has inherit high-availability functionality (F5, NSX-T, Keepalived, etc). + +### Back Up Load Balancer Configuration + +In the event of a Disaster Recovery activity, availability of the Load balancer configuration will expedite the recovery process. + +### Configure Health Checks + +Configure the Load balancer to automatically mark nodes as unavailable if a health check is failed. For example, NGINX can facilitate this with: + +`max_fails=3 fail_timeout=5s` + +### Leverage an External Load Balancer + +Avoid implementing a software load balancer within the management cluster. + +### Secure Access to Rancher + +Configure appropriate Firewall / ACL rules to only expose access to Rancher + +## 2. VM Considerations + +### Size the VM's According to Rancher Documentation + +See [Installation Requirements](../../../pages-for-subheaders/installation-requirements.md). + +### Leverage VM Templates to Construct the Environment + +To facilitate the consistency of Virtual Machines deployed across the environment, consider the use of "Golden Images" in the form of VM templates. Packer can be used to accomplish this, adding greater customization options. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Rancher Cluster Nodes Across ESXi Hosts + +Doing so will ensure node VM's are spread across multiple ESXi hosts - preventing a single point of failure at the host level. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Rancher Cluster Nodes Across Datastores + +Doing so will ensure node VM's are spread across multiple datastores - preventing a single point of failure at the datastore level. + +### Configure VM's as Appropriate for Kubernetes + +It’s important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double-checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node. + +## 3. Network Considerations + +### Leverage Low Latency, High Bandwidth Connectivity Between ETCD Nodes + +Deploy etcd members within a single data center where possible to avoid latency overheads and reduce the likelihood of network partitioning. For most setups, 1Gb connections will suffice. For large clusters, 10Gb connections can reduce the time taken to restore from backup. + +### Consistent IP Addressing for VM's + +Each node used should have a static IP configured. In the case of DHCP, each node should have a DHCP reservation to make sure the node gets the same IP allocated. + +## 4. Storage Considerations + +### Leverage SSD Drives for ETCD Nodes + +ETCD is very sensitive to write latency. Therefore, leverage SSD disks where possible. + +## 5. Backups and Disaster Recovery + +### Perform Regular Management Cluster Backups + +Rancher stores its data in the ETCD datastore of the Kubernetes cluster it resides on. Like with any Kubernetes cluster, perform frequent, tested backups of this cluster. + +### Back up Rancher Cluster Node VMs + +Incorporate the Rancher management node VM's within a standard VM backup policy. diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md new file mode 100644 index 00000000000..99583f75579 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -0,0 +1,40 @@ +--- +title: Rancher Deployment Strategy +--- + + + + + +There are two recommended deployment strategies for a Rancher instance that manages downstream Kubernetes clusters. Each one has its own pros and cons. Read more about which one would fit best for your use case. + +## Hub & Spoke Strategy +--- + +In this deployment scenario, there is a single Rancher instance managing Kubernetes clusters across the globe. The Rancher instance would be run on a high-availability Kubernetes cluster, and there would be impact due to latencies. + +### Pros + +* Single control plane interface to view/see all regions and environments. +* Kubernetes does not require Rancher to operate and can tolerate losing connectivity to the Rancher instance. + +### Cons + +* Subject to network latencies. +* If Rancher goes down, global provisioning of new services is unavailable until it is restored. However, each Kubernetes cluster can continue to be managed individually. + +## Regional Strategy +--- +In the regional deployment model a Rancher instance is deployed in close proximity to the downstream Kubernetes clusters. + +### Pros + +* Rancher functionality in regions stay operational if a Rancher instance in another region goes down. +* Network latency between Rancher and downstream clusters is greatly reduced, improving the performance of functionality in Rancher. +* Upgrades of Rancher can be done independently per region. + +### Cons + +* Overhead of managing multiple Rancher installations. +* Visibility into Kubernetes clusters in different regions requires multiple interfaces/panes of glass. +* Deploying multi-cluster apps in Rancher requires repeating the process for each Rancher server. diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md new file mode 100644 index 00000000000..1ed05696fa2 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -0,0 +1,40 @@ +--- +title: Tips for Running Rancher +--- + + + + + +This guide is geared toward use cases where Rancher is used to manage downstream Kubernetes clusters. The high-availability setup is intended to prevent losing access to downstream clusters if the Rancher server is not available. + +A high-availability Kubernetes installation, defined as an installation of Rancher on a Kubernetes cluster with at least three nodes, should be used in any production installation of Rancher, as well as any installation deemed "important." Multiple Rancher instances running on multiple nodes ensure high availability that cannot be accomplished with a single node environment. + +If you are installing Rancher in a vSphere environment, refer to the best practices documented [here.](on-premises-rancher-in-vsphere.md) + +When you set up your high-availability Rancher installation, consider the following: + +### Run Rancher on a Separate Cluster +Don't run other workloads or microservices in the Kubernetes cluster that Rancher is installed on. + +### Make sure nodes are configured correctly for Kubernetes +It's important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node, checking that all correct ports are opened, and deploying with ssd backed etcd. More details can be found in the [kubernetes docs](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin) and [etcd's performance op guide](https://etcd.io/docs/v3.4/op-guide/performance/). + +### When using RKE: Back up the Statefile +RKE keeps record of the cluster state in a file called `cluster.rkestate`. This file is important for the recovery of a cluster and/or the continued maintenance of the cluster through RKE. Because this file contains certificate material, we strongly recommend encrypting this file before backing up. After each run of `rke up` you should backup the state file. + +### Run All Nodes in the Cluster in the Same Datacenter +For best performance, run all three of your nodes in the same geographic datacenter. If you are running nodes in the cloud, such as AWS, run each node in a separate Availability Zone. For example, launch node 1 in us-west-2a, node 2 in us-west-2b, and node 3 in us-west-2c. + +### Development and Production Environments Should be Similar +It's strongly recommended to have a "staging" or "pre-production" environment of the Kubernetes cluster that Rancher runs on. This environment should mirror your production environment as closely as possible in terms of software and hardware configuration. + +### Monitor Your Clusters to Plan Capacity +The Rancher server's Kubernetes cluster should run within the [system and hardware requirements](../../../pages-for-subheaders/installation-requirements.md) as closely as possible. The more you deviate from the system and hardware requirements, the more risk you take. + +However, metrics-driven capacity planning analysis should be the ultimate guidance for scaling Rancher, because the published requirements take into account a variety of workload types. + +Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with Prometheus, a leading open-source monitoring solution, and Grafana, which lets you visualize the metrics from Prometheus. + +After you [enable monitoring](../../../pages-for-subheaders/monitoring-and-alerting.md) in the cluster, you can set up alerts to let you know if your cluster is approaching its capacity. You can also use the Prometheus and Grafana monitoring framework to establish a baseline for key metrics as you scale. + diff --git a/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md new file mode 100644 index 00000000000..e8e919bde9b --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md @@ -0,0 +1,65 @@ +--- +title: Tips for Scaling Rancher +--- + + + + + +This guide aims to introduce the approaches that should be considered to scale Rancher setups, and associated challenges with doing so. As systems grow performance will naturally reduce, but there are steps we can take to minimize the load put on Rancher, as well as optimize Rancher's ability to handle these larger setups. + +## General Tips on Optimizing Rancher's Performance +* It is advisable to keep Rancher up to date with patch releases. Performance improvements and bug fixes are made throughout the life of a minor release. You can review the release notes to help inform your own decisions on whether an upgrade is necessary but we recommend keeping yourself up to date in most cases. + +* Performance will be negatively impacted by increased latency between Rancher's infrastructure and a downstream cluster's infrastructure (eg. geographic distance). If a user or organization requires clusters/nodes all over the world or spread across many regions, it is best to use multiple Rancher installations. + +* Please always try to scale up gradually, monitoring and observing any change in behavior while doing do. It is usually easier to resolve performance problems as soon as they surface, and before other problems confuse symptoms. + +## Minimizing Load on the local cluster +The largest bottleneck when scaling Rancher is resource growth in the local Kubernetes cluster. The local cluster contains information for all downstream clusters. Many operations that apply to downstream clusters will create new objects in the local cluster and require computation from handlers running in the local cluster. + +### Managing Your Object Counts +ETCD eventually encounters limitations to the number of a single Kubernetes resource type it can store. These exact numbers are not well documented. From internal observations we usually see performance issues once a single resource type's object count exceeds 60k, and often that type is Rolebindings. + +Rolebindings are created in the local cluster as a side effect of many operations. + +Considerations when attempting reduce rolebindings in the local cluster: +* Only add users to clusters and projects when necessary +* Remove clusters and projects when they are no longer needed +* Only use custom roles if necessary +* Use as few rules as possible in custom roles +* Consider whether adding a role to a user is redundant +* Consider that using less, but more powerful, clusters may be more efficient +* Experiment to see if creating new projects or creating new clusters manifests in fewer rolebindings for your specific use case. + +### Using New Apps Over Legacy Apps +There are two app kubernetes resources that Rancher uses: apps.projects.cattle.io and apps.cattle.cattle.io. The legacy apps, apps.projects.cattle.io, were introduced first in the Cluster Manager and are now outdated. The new apps, apps.catalog.cattle.io, are found in the Cluster Explorer for their respective cluster. The new apps are preferrable because they live in the downstream cluster while the legacy apps live in the local cluster. + +We recommend removing apps that appear in the Cluster Manager, replacing them with apps in the Cluster Explorer for their target cluster if necessary and creating any future apps in the cluster's Cluster Explorer only. + +### Using the Authorized Cluster Endpoint (ACE) +There is an _Authorized Cluster Endpoint_ option for Rancher provisioned RKE1, RKE2, and K3s clusters. When enabled this adds a context to kubeconfigs generated for the cluster that uses a direct endpoint to the cluster and bypasses Rancher. However, it is not enough to only enable this option. The user of the Kubeconfig needs to use `kubectl use-context ` in order to start using it. + +Without using ACE, all kubeconfig requests first route through Rancher. + +### Experimental: Option to Reduce Event Handler Executions +The bulk of Rancher's logic occurs on event handlers. These event handlers run on an object whenever the object is updated, and when Rancher is started. Additionally, they run every 15 hours when caches are synced. In scaled setups these scheduled runs come with huge performance costs because every handler is being run on every applicable object. However, this scheduled execution of handlers can be disabled using the CATTLE_SYNC_ONLY_CHANGED_OBJECTS environment variable. If resource allocation spikes are seen on an interval of about 15 hours it is possible this setting can help. + +The value for the environment variable can be a comma separated list of the following options. The values refer to types of controllers (the structures that contain and run handlers) and their handlers. Adding the controller types to the variable will disable that set of controllers from running their handlers as part of cache resyncing. + +* `mgmt` refers to management controllers which only run on one Rancher node. +* `user` refers to user controllers which run for every cluster. Some of these are ran on the same node as management controllers, while other run in the downstream cluster. This will option targets the former. +* `scaled` refers to scaled controllers which run on every Rancher node. This is not recommended to be set due to the critical functionality the scaled handlers are responsible for. + +In short, if you notice CPU usage peaks every 15 hours, add the CATTLE_SYNC_ONLY_CHANGED_OBJECTS environment variable to your rancher deployment with the value `mgmt,user`. + +## Optimizations Outside of Rancher +A large component of performance is the local cluster and how it was configured. This cluster can introduce a bottleneck before Rancher software ever runs. When Rancher nodes experience high resource usage, you can use the command "top" to identify whether it is Rancher or a Kubernetes component that is consuming the resource in excess. + +### Keeping Kubernetes Versions Up to Date +Similar to Rancher versions, it is advisable to keep your kubernetes cluster up to date. This will ensure that your cluster contains any available performance enhancements or bug fixes. + +### Optimizing ETCD +The two main bottlenecks to [ETCD performance](https://etcd.io/docs/v3.4/op-guide/performance/) are disk speed and network speed. Optimization to either should improve performance. For information regarding ETCD performance see [Slow etcd performance (performance testing and optimization)](https://www.suse.com/support/kb/doc/?id=000020100) and [Tuning etcd for Large Installations](https://docs.ranchermanager.rancher.io/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs). Information on disks can also be found [in our docs](https://docs.Ranchermanager.Rancher.io/v2.5/pages-for-subheaders/installation-requirements#disks). + +Theoretically, the more nodes in an ETCD cluster the slower it will be due to replication requirements [source](https://etcd.io/docs/v3.3/faq). This may be counter-intuitive to common scaling approaches. It can also be inferred that ETCD performance will be inversely affected by distance between nodes as that will slow down network communication. diff --git a/versioned_docs/version-2.8/reference-guides/cli-with-rancher/kubectl-utility.md b/versioned_docs/version-2.8/reference-guides/cli-with-rancher/kubectl-utility.md new file mode 100644 index 00000000000..fefa1a66b1b --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cli-with-rancher/kubectl-utility.md @@ -0,0 +1,36 @@ +--- +title: kubectl Utility +--- + + + + + +## kubectl + +Interact with Rancher using kubectl. + +### kubectl Utility + +Install the `kubectl` utility. See [install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). + +Configure kubectl by visiting your cluster in the Rancher Web UI, clicking on `Kubeconfig`, copying contents, and putting them into your `~/.kube/config` file. + +Run `kubectl cluster-info` or `kubectl get pods` successfully. + +### Authentication with kubectl and kubeconfig Tokens with TTL + +_Requirements_ + +If admins have [kubeconfig token generation turned off](../about-the-api/api-tokens.md#disable-tokens-in-generated-kubeconfigs), the kubeconfig file requires the [Rancher CLI](./rancher-cli.md) to be present in your PATH when you run `kubectl`. Otherwise, you’ll see an error like: +`Unable to connect to the server: getting credentials: exec: exec: "rancher": executable file not found in $PATH`. + +This feature enables kubectl to authenticate with the Rancher server and get a new kubeconfig token when required. The following auth providers are currently supported: + +1. Local +2. Active Directory (LDAP only) +3. FreeIPA +4. OpenLDAP +5. SAML providers: Ping, Okta, ADFS, Keycloak, Shibboleth + +When you first run kubectl, for example, `kubectl get pods`, it will ask you to pick an auth provider and log in with the Rancher server. The kubeconfig token is cached in the path where you run kubectl under `./.cache/token`. This token is valid until [it expires](../about-the-api/api-tokens.md#disable-tokens-in-generated-kubeconfigs), or [gets deleted from the Rancher server](../about-the-api/api-tokens.md#deleting-tokens). Upon expiration, the next `kubectl get pods` will ask you to log in with the Rancher server again. \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/cli-with-rancher/rancher-cli.md b/versioned_docs/version-2.8/reference-guides/cli-with-rancher/rancher-cli.md new file mode 100644 index 00000000000..0c9a27c1694 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cli-with-rancher/rancher-cli.md @@ -0,0 +1,91 @@ +--- +title: Rancher CLI +description: Interact with Rancher using command line interface (CLI) tools from your workstation. +--- + + + + + +The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. + +### Download Rancher CLI + +The binary can be downloaded directly from the UI. + +1. In the upper left corner, click **☰**. +1. At the bottom of the navigation sidebar menu, click **About**. +1. Under the **CLI Downloads section**, there are links to download the binaries for Windows, Mac, and Linux. You can also check the [releases page for our CLI](https://github.com/rancher/cli/releases) for direct downloads of the binary. + +### Requirements + +After you download the Rancher CLI, you need to make a few configurations. Rancher CLI requires: + +- Your Rancher Server URL, which is used to connect to Rancher Server. +- An API Bearer Token, which is used to authenticate with Rancher. For more information about obtaining a Bearer Token, see [Creating an API Key](../user-settings/api-keys.md). + +### CLI Authentication + +Before you can use Rancher CLI to control your Rancher Server, you must authenticate using an API Bearer Token. Log in using the following command (replace `` and `` with your information): + +```bash +$ ./rancher login https:// --token +``` + +If Rancher Server uses a self-signed certificate, Rancher CLI prompts you to continue with the connection. + +### Project Selection + +Before you can perform any commands, you must select a Rancher project to perform those commands against. To select a [project](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) to work on, use the command `./rancher context switch`. When you enter this command, a list of available projects displays. Enter a number to choose your project. + +**Example: `./rancher context switch` Output** +``` +User:rancher-cli-directory user$ ./rancher context switch +NUMBER CLUSTER NAME PROJECT ID PROJECT NAME +1 cluster-2 c-7q96s:p-h4tmb project-2 +2 cluster-2 c-7q96s:project-j6z6d Default +3 cluster-1 c-lchzv:p-xbpdt project-1 +4 cluster-1 c-lchzv:project-s2mch Default +Select a Project: +``` + +After you enter a number, the console displays a message that you've changed projects. + +``` +INFO[0005] Setting new context to project project-1 +INFO[0005] Saving config to /Users/markbishop/.ranchcli2.json +``` + +Ensure you can run `rancher kubectl get pods` successfully. + +### Commands + +The following commands are available for use in Rancher CLI. + +| Command | Result | +|---|---| +| `apps, [app]` | Performs operations on catalog applications (i.e., individual [Helm charts](https://docs.helm.sh/developing_charts/)) or Rancher charts. | +| `catalog` | Performs operations on [catalogs](../../pages-for-subheaders/helm-charts-in-rancher.md). | +| `clusters, [cluster]` | Performs operations on your [clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). | +| `context` | Switches between Rancher [projects](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md). For an example, see [Project Selection](#project-selection). | +| `inspect [OPTIONS] [RESOURCEID RESOURCENAME]` | Displays details about [Kubernetes resources](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#resource-types) or Rancher resources (i.e.: [projects](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) and [workloads](../../pages-for-subheaders/workloads-and-pods.md)). Specify resources by name or ID. | +| `kubectl` |Runs [kubectl commands](https://kubernetes.io/docs/reference/kubectl/overview/#operations). | +| `login, [l]` | Logs into a Rancher Server. For an example, see [CLI Authentication](#cli-authentication). | +| `namespaces, [namespace]` |Performs operations on namespaces. | +| `nodes, [node]` |Performs operations on nodes. | +| `projects, [project]` | Performs operations on [projects](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md). | +| `ps` | Displays [workloads](../../pages-for-subheaders/workloads-and-pods.md) in a project. | +| `settings, [setting]` | Shows the current settings for your Rancher Server. | +| `ssh` | Connects to one of your cluster nodes using the SSH protocol. | +| `help, [h]` | Shows a list of commands or help for one command. | + + +### Rancher CLI Help + +Once logged into Rancher Server using the CLI, enter `./rancher --help` for a list of commands. + +All commands accept the `--help` flag, which documents each command's usage. + +### Limitations + +The Rancher CLI **cannot** be used to install [dashboard apps or Rancher feature charts](../../pages-for-subheaders/helm-charts-in-rancher.md). diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md new file mode 100644 index 00000000000..4640d142697 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -0,0 +1,82 @@ +--- +title: EC2 Machine Configuration Reference +--- + + + + + +For more details about EC2 nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). + +### Region + +The geographical [region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html) in which to build your cluster. + +### Zone + +The [zone](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones), an isolated location within a region to build your cluster + +### Instance Type + +The [instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html), which determines the hardware characteristics, used to provision your cluster. + +### Root Disk Size + +Configure the size (in GB) for your [root device](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/RootDeviceStorage.html). + +### VPC/Subnet + +The [VPC](https://docs.aws.amazon.com/vpc/latest/userguide/configure-your-vpc.html) or specific [subnet](https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), an IP range in your VPC, to add your resources to. + +### IAM Instance Profile Name + +The name of the [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) used to pass an IAM role to an EC2 instance. + +## Advanced Options + +### AMI ID + +The [Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) used for the nodes in your cluster. + +### SSH Username for AMI + +The username for connecting to your launched instances. Refer to [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connection-prereqs.html) for the default usernames to selected AMIs. For AMIs not listed, check with the AMI provider. + +### Security Group + +Choose the default security group or configure a security group. + +Please refer to [Amazon EC2 security group when using Node Driver](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-security-group) to see what rules are created in the `rancher-nodes` Security Group. + +### EBS Root Volume Type + +The [EBS volume type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) to use for the root device. + +### Encrypt EBS Volume + +Enable [Amazon EBS Encryption](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html). + +### Request Spot Instance + +Enable option to [request spot instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html) and specify the maximum instance price per hour you're willing to pay. + +### Use only private address + +Enable option on use only [private addresses](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html). + +### EBS-Optimized Instance + +Use an [EBS-optimized instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html). + +### Allow access to EC2 metadata + +Enable access to [EC2 metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html). + +### Use tokens for metadata + +Use [Instance Metadata Service Version 2 (IMDSv2)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html), a token-based method to access metadata. + +### Add Tag + +Add metadata using [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html) to categorize resources. + diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md new file mode 100644 index 00000000000..ba7e806295e --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -0,0 +1,124 @@ +--- +title: Azure Machine Configuration +--- + + + + + +For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) + +### Environment + +Microsoft provides multiple [clouds](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest) for compliance with regional laws, which are available for your use: + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +### Location + +Configure the cluster and node [location](https://docs.microsoft.com/en-us/azure/virtual-machines/regions). + +### Resource Group + +A resource group is a container that holds related resources for an Azure solution. The resource group can include all the resources for the solution, or only those resources that you want to manage as a group. You decide how you want to allocate resources to resource groups based on what makes the most sense for your organization. Generally, add resources that share the same lifecycle to the same resource group so you can easily deploy, update, and delete them as a group. + +Use an existing resource group or enter a resource group name and one will be created for you. + +For information on managing resource groups, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) + +### Availability Set (unmanaged) + +Name or ID of an existing [availability set](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview) to add the VM to. + +### Image + +The name of the operating system image provided as an ARM resource identifier. Requires using managed disk. + +### VM Size + +Choose a size for each VM in the node pool. For details about each VM size, see [this page.](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) + +## Advanced Options + +### Fault Domain Count + +Fault domains define the group of virtual machines that share a common power source and network switch. If the availability set has already been created, the fault domain count will be ignored. + +For more information on fault domains, see [refer here](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work). + +### Update Domain Count + +Update domains indicate groups of virtual machines and underlying physical hardware that can be rebooted at the same time. If the availability set has already been created, the update domain count will be ignored. + +For more information on update domains, see [refer here](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work). + +### Purchase Plan + +Some VM images in the Azure Marketplace require a plan. If applicable, select a purchase plan, formatted as `publisher:product:plan`, to use with your chosen image. + +### Subnet + +The name of the subnet when creating a new VNet or referencing an existing one. + +Default: `docker-machine` + +### Subnet Prefix + +The subnet IP address prefix to use when creating a new VNet in CIDR format. + +Default: `192.168.0.0/16` + +### Virtual Network + +The [virtual network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview) to use or create if one does not exist. Formatted as `[resourcegroup:]name`. + +### Public IP Options + +#### No Public IP + +Do not allocate a public IP address. + +#### Static Public IP + +Allocate a static public IP address. + +### Use Private IP + +Use a static private IP address. + +### Private IP Address + +Configure a static private IP address to use. + +### Network Security Group + +The [network security group](https://docs.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview) to use. All nodes using this template will use the supplied network security group. If no network security group is provided, a new one will be created for each node. + +### DNS Label + +A unique DNS name label for the public IP address. + +### Storage Type + +The [storage account](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview) type to use with your VMs. Options include Standard LRS, Standard ZRS, Standard GRS, Standard RAGRS, and Premium LRS. + +### Use Managed Disks + +[Azure managed disks](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview) are block-level storage volumes that are managed by Azure and used with Azure Virtual Machines. Managed disks are designed for 99.999% availability. Managed disks achieve this by providing you with three replicas of your data, allowing for high durability. + +### Managed Disk Size + +The size in GB for the disk for each node. + +### SSH Username + +The username used to create an SSH connection to your nodes. + +### Open Port + +Opens inbound traffic on specified ports. When using an existing Network Security Group, Open Ports are ignored. + +Default: `2379/tcp, 2380/tcp, 6443/tcp, 9796/tcp, 10250/tcp, 10251/tcp, 10252/tcp, 10256/tcp` and `8472/udp, 4789/udp` \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md new file mode 100644 index 00000000000..634c7f48561 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -0,0 +1,37 @@ +--- +title: DigitalOcean Machine Configuration +--- + + + + + +For more details about DigitalOcean, Droplets, refer to the [official documentation](https://docs.digitalocean.com/products/compute/). + +### Region + +Configure the [region](https://docs.digitalocean.com/glossary/region/) where Droplets are created. + +### Size + +Configure the [size](https://docs.digitalocean.com/products/droplets/resources/choose-plan/) of Droplets. + +### OS Image + +Configure the operating system [image](https://docs.digitalocean.com/products/images/) Droplets are created from. + +### Monitoring + +Enable the DigitalOcean agent for additional [monitoring](https://docs.digitalocean.com/products/monitoring/). + +### IPv6 + +Enable IPv6 for Droplets. + +### Private Networking + +Enable private networking for Droplets. + +### Droplet Tags + +Apply a tag (label) to a Droplet. Tags may only contain letters, numbers, colons, dashes, and underscores. For example, `my_server`. diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md new file mode 100644 index 00000000000..bd79dabbed5 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -0,0 +1,51 @@ +--- +title: EC2 Node Template Configuration +--- + + + + + +For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). + +### Region + +In the **Region** field, select the same region that you used when creating your cloud credentials. + +### Cloud Credentials + +Your AWS account access information, stored in a [cloud credential.](../../../user-settings/manage-cloud-credentials.md) + +See [Amazon Documentation: Creating Access Keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) how to create an Access Key and Secret Key. + +See [Amazon Documentation: Creating IAM Policies (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start) how to create an IAM policy. + +See [Amazon Documentation: Adding Permissions to a User (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console) how to attach an IAM + +See our three example JSON policies: + +- [Example IAM Policy](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy) +- [Example IAM Policy with PassRole](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy-with-passrole) (needed if you want to use [Kubernetes Cloud Provider](../../../../pages-for-subheaders/set-up-cloud-providers.md) or want to pass an IAM Profile to an instance) +- [Example IAM Policy to allow encrypted EBS volumes](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy-to-allow-encrypted-ebs-volumes) policy to an user. + +### Authenticate & Configure Nodes + +Choose an availability zone and network settings for your cluster. + +### Security Group + +Choose the default security group or configure a security group. + +Please refer to [Amazon EC2 security group when using Node Driver](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-security-group) to see what rules are created in the `rancher-nodes` Security Group. + +If you provide your own security group for an EC2 instance, please note that Rancher will not modify it. As such, you will be responsible for ensuring that your security group is set to allow the [necessary ports for Rancher to provision the instance](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#ports-for-rancher-server-nodes-on-rke). For more information on controlling inbound and outbound traffic to EC2 instances with security groups, refer [here](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#WorkingWithSecurityGroups). + +### Instance Options + +Configure the instances that will be created. Make sure you configure the correct **SSH User** for the configured AMI. It is possible that a selected region does not support the default instance type. In this scenario you must select an instance type that does exist, otherwise an error will occur stating the requested configuration is not supported. + +If you need to pass an **IAM Instance Profile Name** (not ARN), for example, when you want to use a [Kubernetes Cloud Provider](../../../../pages-for-subheaders/set-up-cloud-providers.md), you will need an additional permission in your policy. See [Example IAM policy with PassRole](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy-with-passrole) for an example policy. + +### Engine Options + +In the **Engine Options** section of the node template, you can configure the container daemon. You may want to specify the container version or a container image registry mirror. diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md new file mode 100644 index 00000000000..e747dbbb547 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -0,0 +1,37 @@ +--- +title: Azure Node Template Configuration +--- + + + + + +For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) + +Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. + +- **Placement** sets the geographical region where your cluster is hosted and other location metadata. +- **Network** configures the networking used in your cluster. +- **Instance** customizes your VM configuration. + + :::note + + If using a VNet in a different Resource Group than the VMs, the VNet name should be prefixed with the Resource Group name. For example, `:`. + + ::: + +If you use Docker, the [Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) configuration options include: + +- **Labels:** For information on labels, refer to the [Docker object label documentation.](https://docs.docker.com/config/labels-custom-metadata/). +- **Docker Engine Install URL:** Determines what Docker version will be installed on the instance. + + :::note + + If you're provisioning Red Hat Enterprise Linux (RHEL) or CentOS nodes, leave the **Docker Install URL** field as the default value, or select **none**. This will bypass a check for Docker installation, as Docker is already installed on these node types. + + If you set **Docker Install URL** to a value other than the default or **none**, you might see an error message such as the following: `Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories.` + + ::: + +- **Registry mirrors:** Docker Registry mirror to be used by the Docker daemon. +- **Other advanced options:** Refer to the [Docker daemon option reference](https://docs.docker.com/engine/reference/commandline/dockerd/). diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md new file mode 100644 index 00000000000..87b5fccdcfb --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -0,0 +1,22 @@ +--- +title: DigitalOcean Node Template Configuration +--- + + + + + +Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. + +### Droplet Options + +The **Droplet Options** provision your cluster's geographical region and specifications. + +### Docker Daemon + +If you use Docker, the [Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) configuration options include: + +- **Labels:** For information on labels, refer to the [Docker object label documentation.](https://docs.docker.com/config/labels-custom-metadata/) +- **Docker Engine Install URL:** Determines what Docker version will be installed on the instance. +- **Registry mirrors:** Docker Registry mirror to be used by the Docker daemon +- **Other advanced options:** Refer to the [Docker daemon option reference](https://docs.docker.com/engine/reference/commandline/dockerd/). diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md new file mode 100644 index 00000000000..4e44d5ceca7 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -0,0 +1,70 @@ +--- +title: Nutanix Node Template Configuration +--- + + + + + +## Account Access + +| Parameter | Required | Description | Default +|:-----------------------------|:--------:|:-----------------------------------------------------------------|:----- +| Management Endpoint | ✓ | Hostname/IP address of Prism Central | +| Username | ✓ | Username of the Prism Central user | +| Password | ✓ | Password of the Prism Central user | +| Allow insecure communication | | Set to true to allow insecure SSL communication to Prism Central | False + +## Scheduling + +Choose what Nutanix cluster the virtual machine will be scheduled to. + +| Parameter | Required | Description +|:----------|:--------:|:---------------------------------------------------------------------------- +| Cluster | ✓ | Name of the Nutanix cluster where the VM should be deployed (case sensitive) + +## Instance Options + +In the **Instance Options** section, configure the number of vCPUs, memory, and disk size for the VMs created by this template. + +| Parameter | Required | Description | Default +|:---------------------|:--------:|:--------------------------------------------------------------------------------------------|:------- +| CPUs | | Number of vCPUs allocated to the VM (cores) | 2 +| Memory | | Amount of RAM allocated to the VM (MB) | 2 GB +| Template Image | ✓ | Name of the Disk Image template to clone as the VM's primary disk (must support cloud-init) | +| VM Disk Size | | New size of the VM's primary disk (in GiB) | +| Additional Disk Size | | Size of an additional disk to add to the VM (in GiB) | +| Storage Container | | Storage container _UUID_ in which to provision an additional disk | +| Cloud Config YAML | | Cloud-init to provide to the VM (will be patched with Rancher root user) | +| Network | ✓ | Name(s) of the network(s) to attach to the VM | +| VM Categories | | Name(s) of any categories to be applied to the VM | + +The VM may use any modern Linux operating system that is configured with support for [cloud-init](https://cloudinit.readthedocs.io/en/latest/) using the [Config Drive v2 datasource](https://cloudinit.readthedocs.io/en/latest/reference/datasources/configdrive.html). + +## Networks + +The node template allows a VM to be provisioned with multiple networks. In the **Network** field, you can click **Add** to add any networks available to you in AOS. + +## VM Categories + +A category is a grouping of entities into a key value pair. Typically, VMs are assigned to a category based on some criteria. Policies can then be tied to those entities that are assigned (grouped by) a specific category value. + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) allows you to initialize your nodes by applying configuration on the first boot. This may involve things such as creating users or authorizing SSH keys. + +To make use of cloud-init initialization, paste a cloud config using valid YAML syntax into the **Cloud Config YAML** field. Refer to the [cloud-init documentation](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) for a commented set of examples of supported cloud config directives. + +Note that cloud-init based network configuration is not recommended and only supported via user data `runcmd` rather than by NoCloud or other network configuration datasources. + +Nutanix IP Address Management (IPAM) or another DHCP service is recommended. + +## Engine Options + +In the **Engine Options** section of the node template, you can configure the container daemon. You may want to specify the container version or a container image registry mirror. + +:::note +If you're provisioning Red Hat Enterprise Linux (RHEL) or CentOS nodes, leave the **Docker Install URL** field as the default value, or select **none**. This will bypass a check for Docker installation, as Docker is already installed on these node types. + +If you set **Docker Install URL** to a value other than the default or **none**, you might see an error message such as the following: `Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories.` +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md new file mode 100644 index 00000000000..135c3edf7cc --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -0,0 +1,99 @@ +--- +title: VSphere Node Template Configuration +--- + + + + + +## Account Access + +| Parameter | Required | Description | +|:----------------------|:--------:|:-----| +| Cloud Credentials | * | Your vSphere account access information, stored in a [cloud credential.](../../../user-settings/manage-cloud-credentials.md) | + +Your cloud credential has these fields: + +| Credential Field | Description | +|-----------------|--------------| +| vCenter or ESXi Server | Enter the vCenter or ESXi hostname/IP. ESXi is the virtualization platform where you create and run virtual machines and virtual appliances. vCenter Server is the service through which you manage multiple hosts connected in a network and pool host resources. | +| Port | Optional: configure configure the port of the vCenter or ESXi server. | +| Username and password | Enter your vSphere login username and password. | + +## Scheduling + +Choose what hypervisor the virtual machine will be scheduled to. + +The fields in the **Scheduling** section should auto-populate with the data center and other scheduling options that are available to you in vSphere. + +| Field | Required | Explanation | +|---------|---------------|-----------| +| Data Center | * | Choose the name/path of the data center where the VM will be scheduled. | +| Resource Pool | | Name of the resource pool to schedule the VMs in. Resource pools can be used to partition available CPU and memory resources of a standalone host or cluster, and they can also be nested. Leave blank for standalone ESXi. If not specified, the default resource pool is used. | +| Data Store | * | If you have a data store cluster, you can toggle the **Data Store** field. This lets you select a data store cluster where your VM will be scheduled to. If the field is not toggled, you can select an individual disk. | +| Folder | | Name of a folder in the datacenter to create the VMs in. Must already exist. The VM folders in this dropdown menu directly correspond to your VM folders in vSphere. The folder name should be prefaced with `vm/` in your vSphere config file. | +| Host | | The IP of the host system to schedule VMs in. Leave this field blank for a standalone ESXi or for a cluster with DRS (Distributed Resource Scheduler). If specified, the host system's pool will be used and the **Resource Pool** parameter will be ignored. | + +## Instance Options + +In the **Instance Options** section, configure the number of vCPUs, memory, and disk size for the VMs created by this template. + +| Parameter | Required | Description | +|:----------------|:--------:|:-----------| +| CPUs | * | Number of vCPUS to assign to VMs. | +| Memory | * | Amount of memory to assign to VMs. | +| Disk | * | Size of the disk (in MB) to attach to the VMs. | +| Creation method | * | The method for setting up an operating system on the node. The operating system can be installed from an ISO or from a VM template. Depending on the creation method, you will also have to specify a VM template, content library, existing VM, or ISO. For more information on creation methods, refer to [About VM Creation Methods.](#about-vm-creation-methods) | +| Cloud Init | | URL of a `cloud-config.yml` file or URL to provision VMs with. This file allows further customization of the operating system, such as network configuration, DNS servers, or system daemons. The operating system must support `cloud-init`. | +| Networks | | Name(s) of the network to attach the VM to. | +| Configuration Parameters used for guestinfo | | Additional configuration parameters for the VMs. These correspond to the [Advanced Settings](https://kb.vmware.com/s/article/1016098) in the vSphere console. Example use cases include providing RancherOS [guestinfo](https://rancher.com/docs/os/v1.x/en/installation/cloud/vmware-esxi/#vmware-guestinfo) parameters or enabling disk UUIDs for the VMs (`disk.EnableUUID=TRUE`). | + + +### About VM Creation Methods + +In the **Creation method** field, configure the method used to provision VMs in vSphere. Available options include creating VMs that boot from a RancherOS ISO or creating VMs by cloning from an existing virtual machine or [VM template](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-F7BF0E6B-7C4F-4E46-8BBF-76229AEA7220.html). + +The existing VM or template may use any modern Linux operating system that is configured with support for [cloud-init](https://cloudinit.readthedocs.io/en/latest/) using the [NoCloud datasource](https://canonical-cloud-init.readthedocs-hosted.com/en/latest/reference/datasources/nocloud.html). + +Choose the way that the VM will be created: + +- **Deploy from template: Data Center:** Choose a VM template that exists in the data center that you selected. +- **Deploy from template: Content Library:** First, select the [Content Library](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-254B2CE8-20A8-43F0-90E8-3F6776C2C896.html) that contains your template, then select the template from the populated list **Library templates**. +- **Clone an existing virtual machine:** In the **Virtual machine** field, choose an existing VM that the new VM will be cloned from. +- **Install from boot2docker ISO:** Ensure that the **OS ISO URL** field contains the URL of a VMware ISO release for RancherOS (`rancheros-vmware.iso`). Note that this URL must be accessible from the nodes running your Rancher server installation. + +## Networks + +The node template now allows a VM to be provisioned with multiple networks. In the **Networks** field, you can now click **Add Network** to add any networks available to you in vSphere. + +## Node Tags and Custom Attributes + +Tags allow you to attach metadata to objects in the vSphere inventory to make it easier to sort and search for these objects. + +For tags, all your vSphere tags will show up as options to select from in your node template. + +In the custom attributes, Rancher will let you select all the custom attributes you have already set up in vSphere. The custom attributes are keys and you can enter values for each one. + +:::note + +Custom attributes are a legacy feature that will eventually be removed from vSphere. + +::: + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) allows you to initialize your nodes by applying configuration on the first boot. This may involve things such as creating users, authorizing SSH keys or setting up the network. + +To make use of cloud-init initialization, create a cloud config file using valid YAML syntax and paste the file content in the the **Cloud Init** field. Refer to the [cloud-init documentation.](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) for a commented set of examples of supported cloud config directives. + +Note that cloud-init is not supported when using the ISO creation method. + +## Engine Options + +In the **Engine Options** section of the node template, you can configure the container daemon. You may want to specify the container version or a container image registry mirror. + +:::note +If you're provisioning Red Hat Enterprise Linux (RHEL) or CentOS nodes, leave the **Docker Install URL** field as the default value, or select **none**. This will bypass a check for Docker installation, as Docker is already installed on these node types. + +If you set **Docker Install URL** to a value other than the default or **none**, you might see an error message such as the following: `Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories.` +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md new file mode 100644 index 00000000000..67cd280e470 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -0,0 +1,228 @@ +--- +title: AKS Cluster Configuration Reference +--- + + + + + +## Changes in Rancher v2.6 + +- Support for adding more than one node pool +- Support for private clusters +- Enabled autoscaling node pools +- The AKS permissions are now configured in cloud credentials + +## Role-based Access Control + +When provisioning an AKS cluster in the Rancher UI, RBAC cannot be disabled. If role-based access control is disabled for the cluster in AKS, the cluster cannot be registered or imported into Rancher. + +Rancher can configure member roles for AKS clusters in the same way as any other cluster. For more information, see the section on [role-based access control.](../../../pages-for-subheaders/manage-role-based-access-control-rbac.md) + +## Cloud Credentials + +:::note + +The configuration information in this section assumes you have already set up a service principal for Rancher. For step-by-step instructions for how to set up the service principal, see [this section.](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md#prerequisites-in-microsoft-azure) + +::: + +### Subscription ID + +To get the subscription ID, click **All Services** in the left navigation bar. Then click **Subscriptions**. Go to the name of the subscription that you want to associate with your Kubernetes cluster and copy the **Subscription ID**. + +### Client ID + +To get the client ID, go to the Azure Portal, then click **Azure Active Directory**, then click **App registrations,** then click the name of the service principal. The client ID is listed on the app registration detail page as **Application (client) ID**. + +### Client Secret + +You can't retrieve the client secret value after it is created, so if you don't already have a client secret value, you will need to create a new client secret. + +To get a new client secret, go to the Azure Portal, then click **Azure Active Directory**, then click **App registrations,** then click the name of the service principal. + +Then click **Certificates & secrets** and click **New client secret**. Click **Add**. Then copy the **Value** of the new client secret. + +### Environment + +Microsoft provides multiple [clouds](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest) for compliance with regional laws, which are available for your use: + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +## Account Access + +In this section you will need to select an existing Azure cloud credential or create a new one. + +For help configuring your Azure cloud credential, see [this section.](#cloud-credentials) + +## Cluster Location + +Configure the cluster and node location. For more information on availability zones for AKS, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/availability-zones) + +The high availability locations include multiple availability zones. + +## Cluster Options + +### Kubernetes Version + +The available Kubernetes versions are dynamically fetched from the Azure API. + +### Cluster Resource Group + +A resource group is a container that holds related resources for an Azure solution. The resource group can include all the resources for the solution, or only those resources that you want to manage as a group. You decide how you want to allocate resources to resource groups based on what makes the most sense for your organization. Generally, add resources that share the same lifecycle to the same resource group so you can easily deploy, update, and delete them as a group. + +Use an existing resource group or enter a resource group name and one will be created for you. + +Using a resource group containing an existing AKS cluster will create a new resource group. Azure AKS only allows one AKS cluster per resource group. + +For information on managing resource groups, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) + +### Linux Admin Username + +The username used to create an SSH connection to the Linux nodes. + +The default username for AKS nodes is `azureuser`. + +### SSH Public Key + +The key used to create an SSH connection to the Linux nodes. + +### Tags + +Cluster tags can be useful if your organization uses tags as a way to organize resources across multiple Azure services. These tags don't apply to resources within the cluster. + +## Networking Options + +### LoadBalancer SKU + +Azure load balancers support both standard and basic SKUs (stock keeping units). + +For a comparison of standard and basic load balancers, see the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/load-balancer/skus#skus) Microsoft recommends the Standard load balancer. + +The Standard load balancer is required if you have selected one or more availability zones, or if you have more than one node pool. + +### Network Policy + +All pods in an AKS cluster can send and receive traffic without limitations, by default. To improve security, you can define rules that control the flow of traffic. The Network Policy feature in Kubernetes lets you define rules for ingress and egress traffic between pods in a cluster. + +Azure provides two ways to implement network policy. You choose a network policy option when you create an AKS cluster. The policy option can't be changed after the cluster is created: + +- Azure's own implementation, called Azure Network Policies. The Azure network policy requires the Azure CNI. +- Calico Network Policies, an open-source network and network security solution founded by [Tigera](https://www.tigera.io/). + +You can also choose to have no network policy. + +For more information about the differences between Azure and Calico network policies and their capabilities, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/use-network-policies#differences-between-azure-and-calico-policies-and-their-capabilities) + +### DNS Prefix +Enter a unique DNS prefix for your cluster's Kubernetes API server FQDN. + +### Network Plugin +There are two network plugins: kubenet and Azure CNI. + +The [kubenet](https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet) Kubernetes plugin is the default configuration for AKS cluster creation. When kubenet is used, each node in the cluster receives a routable IP address. The pods use NAT to communicate with other resources outside the AKS cluster. This approach reduces the number of IP addresses you need to reserve in your network space for pods to use. + +With the Azure CNI (advanced) networking plugin, pods get full virtual network connectivity and can be directly reached via their private IP address from connected networks. This plugin requires more IP address space. + +For more information on the differences between kubenet and Azure CNI, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/concepts-network#compare-network-models) + +### HTTP Application Routing + +When enabled, the HTTP application routing add-on makes it easier to access applications deployed to the AKS cluster. It deploys two components: a [Kubernetes Ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress/) and an [External-DNS](https://github.com/kubernetes-incubator/external-dns) controller. + +For more information, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/http-application-routing) + +### Set Authorized IP Ranges + +You can secure access to the Kubernetes API server using [authorized IP address ranges.](https://docs.microsoft.com/en-us/azure/aks/api-server-authorized-ip-ranges#overview-of-api-server-authorized-ip-ranges) + +The Kubernetes API server exposes the Kubernetes API. This component provides the interaction for management tools, such as kubectl. AKS provides a single-tenant cluster control plane with a dedicated API server. By default, the API server is assigned a public IP address, and you should control access to it using Kubernetes-based or Azure-based RBAC. + +To secure access to the otherwise publicly accessible AKS control plane and API server, you can enable and use authorized IP ranges. These authorized IP ranges only allow defined IP address ranges to communicate with the API server. + +However, even if you use authorized IP address ranges, you should still use Kubernetes RBAC or Azure RBAC to authorize users and the actions they request. + +### Container Monitoring + +Container monitoring gives you performance visibility by collecting memory and processor metrics from controllers, nodes, and containers that are available in Kubernetes through the Metrics API. Container logs are also collected. After you enable monitoring, metrics and logs are automatically collected for you through a containerized version of the Log Analytics agent for Linux. Metrics are written to the metrics store and log data is written to the logs store associated with your [Log Analytics](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview) workspace. + +### Log Analytics Workspace Resource Group + +The [resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups) containing the Log Analytics Workspace. You must create at least one workspace to use Azure Monitor Logs. + +### Log Analytics Workspace Name + +Data collected by Azure Monitor Logs is stored in one or more [Log Analytics workspaces.](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/design-logs-deployment) The workspace defines the geographic location of the data, access rights defining which users can access data, and configuration settings such as the pricing tier and data retention. + +You must create at least one workspace to use Azure Monitor Logs. A single workspace may be suffxicient for all of your monitoring data, or may choose to create multiple workspaces depending on your requirements. For example, you might have one workspace for your production data and another for testing. + +For more information about Azure Monitor Logs, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-platform-logs) + +### Support Private Kubernetes Service + +Typically, AKS worker nodes do not get public IPs, regardless of whether the cluster is private. In a private cluster, the control plane does not have a public endpoint. + +Rancher can connect to a private AKS cluster in one of two ways. + +The first way to ensure that Rancher is running on the same [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) as the AKS nodes. + +The second way is to run a command to register the cluster with Rancher. Once the cluster is provisioned, you can run the displayed command anywhere you can connect to the cluster’s Kubernetes API. This command is displayed in a pop-up when you provision an AKS cluster with a private API endpoint enabled. + +:::note + +Please be aware that when registering an existing AKS cluster, the cluster might take some time, possibly hours, to appear in the `Cluster To register` dropdown list. This outcome will be based on region. + +::: + +For more information about connecting to an AKS private cluster, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster) + +## Node Pools + +### Mode + +The Azure interface allows users to specify whether a Primary Node Pool relies on either `system` (normally used for control planes) or `user` (what is most typically needed for Rancher). + +For Primary Node Pools, you can specify Mode, OS, Count and Size. + +System node pools always require running nodes, so they cannot be scaled below one node. At least one system node pool is required. + +For subsequent node pools, the Rancher UI forces the default of User. User node pools allow you to scale to zero nodes. User node pools don't run any part of the Kubernetes controlplane. + +AKS doesn't expose the nodes that run the Kubernetes controlplane components. + +### Availability Zones + +[Availability zones](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview) are unique physical locations within a region. Each zone is made up of one or more data centers equipped with independent power, cooling, and networking. + +Not all regions have support for availability zones. For a list of Azure regions with availability zones, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/availability-zones/az-region#azure-regions-with-availability-zones) + +### VM Size + +Choose a size for each VM in the node pool. For details about each VM size, see [this page.](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) + +### OS Disk Type + +The nodes in the node pool can have either managed or ephemeral disks. + +[Ephemeral OS disks](https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks) are created on the local virtual machine storage and not saved to the remote Azure Storage. Ephemeral OS disks work well for stateless workloads, where applications are tolerant of individual VM failures, but are more affected by VM deployment time or reimaging the individual VM instances. With Ephemeral OS disk, you get lower read/write latency to the OS disk and faster VM reimage. + +[Azure managed disks](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview) are block-level storage volumes that are managed by Azure and used with Azure Virtual Machines. Managed disks are designed for 99.999% availability. Managed disks achieve this by providing you with three replicas of your data, allowing for high durability. + +### OS Disk Size + +The size in GB for the disk for each node. + +### Node Count +The number of nodes in the node pool. The maximum number of nodes may be limited by your [Azure subscription.](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits) + +### Max Pods Per Node +The maximum number of pods per node defaults to 110 with a maximum of 250. + +### Enable Auto Scaling + +When auto scaling is enabled, you will need to enter a minimum and maximum node count. + +When Auto Scaling is enabled, you can't manually scale the node pool. The scale is controlled by the AKS autoscaler. diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md new file mode 100644 index 00000000000..ba6b826b4f1 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -0,0 +1,155 @@ +--- +title: EKS Cluster Configuration Reference +--- + + + + + +### Account Access + +Complete each drop-down and field using the information obtained for your IAM policy. + +| Setting | Description | +| ---------- | -------------------------------------------------------------------------------------------------------------------- | +| Region | From the drop-down choose the geographical region in which to build your cluster. | +| Cloud Credentials | Select the cloud credentials that you created for your IAM policy. For more information on creating cloud credentials in Rancher, refer to [this page.](../../user-settings/manage-cloud-credentials.md) | + +### Service Role + +Choose a [service role](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html). + +Service Role | Description +-------------|--------------------------- +Standard: Rancher generated service role | If you choose this role, Rancher automatically adds a service role for use with the cluster. +Custom: Choose from your existing service roles | If you choose this role, Rancher lets you choose from service roles that you're already created within AWS. For more information on creating a custom service role in AWS, see the [Amazon documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html#create-service-linked-role). + +### Secrets Encryption + +Optional: To encrypt secrets, select or enter a key created in [AWS Key Management Service (KMS)](https://docs.aws.amazon.com/kms/latest/developerguide/overview.html) + +### API Server Endpoint Access + +Configuring Public/Private API access is an advanced use case. For details, refer to the EKS cluster endpoint access control [documentation.](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) + +### Private-only API Endpoints + +If you enable private and disable public API endpoint access when creating a cluster, then there is an extra step you must take in order for Rancher to connect to the cluster successfully. In this case, a pop-up will be displayed with a command that you will run on the cluster to register it with Rancher. Once the cluster is provisioned, you can run the displayed command anywhere you can connect to the cluster's Kubernetes API. + +There are two ways to avoid this extra manual step: +- You can create the cluster with both private and public API endpoint access on cluster creation. You can disable public access after the cluster is created and in an active state and Rancher will continue to communicate with the EKS cluster. +- You can ensure that Rancher shares a subnet with the EKS cluster. Then security groups can be used to enable Rancher to communicate with the cluster's API endpoint. In this case, the command to register the cluster is not needed, and Rancher will be able to communicate with your cluster. For more information on configuring security groups, refer to the [security groups documentation](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html). + +### Public Access Endpoints + +Optionally limit access to the public endpoint via explicit CIDR blocks. + +If you limit access to specific CIDR blocks, then it is recommended that you also enable the private access to avoid losing network communication to the cluster. + +One of the following is required to enable private access: +- Rancher's IP must be part of an allowed CIDR block +- Private access should be enabled, and Rancher must share a subnet with the cluster and have network access to the cluster, which can be configured with a security group + +For more information about public and private access to the cluster endpoint, refer to the [Amazon EKS documentation.](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) + +### Subnet + +| Option | Description | +| ------- | ------------ | +| Standard: Rancher generated VPC and Subnet | While provisioning your cluster, Rancher generates a new VPC with 3 public subnets. | +| Custom: Choose from your existing VPC and Subnets | While provisioning your cluster, Rancher configures your Control Plane and nodes to use a VPC and Subnet that you've already [created in AWS](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html). | + + For more information, refer to the AWS documentation for [Cluster VPC Considerations](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). Follow one of the sets of instructions below based on your selection from the previous step. + +- [What Is Amazon VPC?](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) +- [VPCs and Subnets](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) + +### Security Group + +Amazon Documentation: + +- [Cluster Security Group Considerations](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) +- [Security Groups for Your VPC](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) +- [Create a Security Group](https://docs.aws.amazon.com/vpc/latest/userguide/getting-started-ipv4.html#getting-started-create-security-group) + +### Logging + +Configure control plane logs to send to Amazon CloudWatch. You are charged the standard CloudWatch Logs data ingestion and storage costs for any logs sent to CloudWatch Logs from your clusters. + +Each log type corresponds to a component of the Kubernetes control plane. To learn more about these components, see [Kubernetes Components](https://kubernetes.io/docs/concepts/overview/components/) in the Kubernetes documentation. + +For more information on EKS control plane logging, refer to the official [documentation.](https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) + +### Managed Node Groups + +Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS Kubernetes clusters. + +For more information about how node groups work and how they are configured, refer to the [EKS documentation.](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) + +#### User-provided Launch Templates + +You can provide your own launch template ID and version to configure the EC2 instances in a node group. If you provide the launch template, none of the template settings will be configurable from Rancher. You must set all of the required options listed below in your launch template. + +Also, if you provide the launch template, you can only update the template version, not the template ID. To use a new template ID, create a new managed node group. + +| Option | Description | Required/Optional | +| ------ | ----------- | ----------------- | +| Instance Type | Choose the [hardware specs](https://aws.amazon.com/ec2/instance-types/) for the instance you're provisioning. | Required | +| Image ID | Specify a custom AMI for the nodes. Custom AMIs used with EKS must be [configured properly](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/) | Optional | +| Node Volume Size | The launch template must specify an EBS volume with the desired size | Required | +| SSH Key | A key to be added to the instances to provide SSH access to the nodes | Optional | +| User Data | Cloud init script in [MIME multi-part format](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data) | Optional | +| Instance Resource Tags | Tag each EC2 instance and its volumes in the node group | Optional | + +#### Rancher-managed Launch Templates + +If you do not specify a launch template, then you will be able to configure the above options in the Rancher UI and all of them can be updated after creation. In order to take advantage of all of these options, Rancher will create and manage a launch template for you. Each cluster in Rancher will have one Rancher-managed launch template and each managed node group that does not have a specified launch template will have one version of the managed launch template. The name of this launch template will have the prefix "rancher-managed-lt-" followed by the display name of the cluster. In addition, the Rancher-managed launch template will be tagged with the key "rancher-managed-template" and value "do-not-modify-or-delete" to help identify it as Rancher-managed. It is important that this launch template and its versions not be modified, deleted, or used with any other clusters or managed node groups. Doing so could result in your node groups being "degraded" and needing to be destroyed and recreated. + +#### Custom AMIs + +If you specify a custom AMI, whether in a launch template or in Rancher, then the image must be [configured properly](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/) and you must provide user data to [bootstrap the node](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami). This is considered an advanced use case and understanding the requirements is imperative. + +If you specify a launch template that does not contain a custom AMI, then Amazon will use the [EKS-optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) for the Kubernetes version and selected region. You can also select a [GPU enabled instance](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html#gpu-ami) for workloads that would benefit from it. + +:::note + +The GPU enabled instance setting in Rancher is ignored if a custom AMI is provided, either in the dropdown or in a launch template. + +::: + +#### Spot instances + +Spot instances are now [supported by EKS](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types-spot). If a launch template is specified, Amazon recommends that the template not provide an instance type. Instead, Amazon recommends providing multiple instance types. If the "Request Spot Instances" checkbox is enabled for a node group, then you will have the opportunity to provide multiple instance types. + +:::note + +Any selection you made in the instance type dropdown will be ignored in this situation and you must specify at least one instance type to the "Spot Instance Types" section. Furthermore, a launch template used with EKS cannot request spot instances. Requesting spot instances must be part of the EKS configuration. + +::: + +#### Node Group Settings + +The following settings are also configurable. All of these except for the "Node Group Name" are editable after the node group is created. + +| Option | Description | +| ------- | ------------ | +| Node Group Name | The name of the node group. | +| Desired ASG Size | The desired number of instances. | +| Maximum ASG Size | The maximum number of instances. This setting won't take effect until the [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) is installed. | +| Minimum ASG Size | The minimum number of instances. This setting won't take effect until the [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) is installed. | +| Labels | Kubernetes labels applied to the nodes in the managed node group. | +| Tags | These are tags for the managed node group and do not propagate to any of the associated resources. | + + +### Configuring the Refresh Interval + +The `eks-refresh-cron` setting is deprecated. It has been migrated to the `eks-refresh` setting, which is an integer representing seconds. + +The default value is 300 seconds. + +The syncing interval can be changed by running `kubectl edit setting eks-refresh`. + +If the `eks-refresh-cron` setting was previously set, the migration will happen automatically. + +The shorter the refresh window, the less likely any race conditions will occur, but it does increase the likelihood of encountering request limits that may be in place for AWS APIs. + diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md new file mode 100644 index 00000000000..120be47a641 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -0,0 +1,54 @@ +--- +title: Private Clusters +--- + + + + + +In GKE, [private clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept) are clusters whose nodes are isolated from inbound and outbound traffic by assigning them internal IP addresses only. Private clusters in GKE have the option of exposing the control plane endpoint as a publicly accessible address or as a private address. This is different from other Kubernetes providers, which may refer to clusters with private control plane endpoints as "private clusters" but still allow traffic to and from nodes. You may want to create a cluster with private nodes, with or without a public control plane endpoint, depending on your organization's networking and security requirements. A GKE cluster provisioned from Rancher can use isolated nodes by selecting "Private Cluster" in the Cluster Options (under "Show advanced options"). The control plane endpoint can optionally be made private by selecting "Enable Private Endpoint". + +### Private Nodes + +Because the nodes in a private cluster only have internal IP addresses, they will not be able to install the cluster agent and Rancher will not be able to fully manage the cluster. This can be overcome in a few ways. + +#### Cloud NAT + +:::caution + +Cloud NAT will [incur charges](https://cloud.google.com/nat/pricing). + +::: + +If restricting outgoing internet access is not a concern for your organization, use Google's [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) service to allow nodes in the private network to access the internet, enabling them to download the required images from Dockerhub and contact the Rancher management server. This is the simplest solution. + +#### Private registry + +:::caution + +This scenario is not officially supported, but is described for cases in which using the Cloud NAT service is not sufficient. + +::: + +If restricting both incoming and outgoing traffic to nodes is a requirement, follow the air-gapped installation instructions to set up a private container image [registry](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md) on the VPC where the cluster is going to be, allowing the cluster nodes to access and download the images they need to run the cluster agent. If the control plane endpoint is also private, Rancher will need [direct access](#direct-access) to it. + +### Private Control Plane Endpoint + +If the cluster has a public endpoint exposed, Rancher will be able to reach the cluster, and no additional steps need to be taken. However, if the cluster has no public endpoint, then considerations must be made to ensure Rancher can access the cluster. + +#### Cloud NAT + +:::caution + +Cloud NAT will [incur charges](https://cloud.google.com/nat/pricing). + +::: + +As above, if restricting outgoing internet access to the nodes is not a concern, then Google's [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) service can be used to allow the nodes to access the internet. While the cluster is provisioning, Rancher will provide a registration command to run on the cluster. Download the [kubeconfig](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) for the new cluster and run the provided kubectl command on the cluster. Gaining access +to the cluster in order to run this command can be done by creating a temporary node or using an existing node in the VPC, or by logging on to or creating an SSH tunnel through one of the cluster nodes. + +#### Direct access + +If the Rancher server is run on the same VPC as the cluster's control plane, it will have direct access to the control plane's private endpoint. The cluster nodes will need to have access to a [private registry](#private-registry) to download images as described above. + +You can also use services from Google such as [Cloud VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/overview) or [Cloud Interconnect VLAN](https://cloud.google.com/network-connectivity/docs/interconnect) to facilitate connectivity between your organization's network and your Google VPC. diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md new file mode 100644 index 00000000000..ca377342025 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md @@ -0,0 +1,434 @@ +--- +title: K3s Cluster Configuration Reference +--- + + + + + +This section covers the configuration options that are available in Rancher for a new or existing K3s Kubernetes cluster. + +## Overview + +You can configure the Kubernetes options one of two ways: + +- [Rancher UI](#configuration-options-in-the-rancher-ui): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. +- [Cluster Config File](#cluster-config-file): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create a K3s config file. Using a config file allows you to set any of the [options](https://rancher.com/docs/k3s/latest/en/installation/install-options/) available in an K3s installation. + +## Editing Clusters in the Rancher UI + +The Rancher UI provides two ways to edit a cluster: +1. With a form. +1. With YAML. + +### Editing Clusters with a Form + +The form covers the most frequently needed options for clusters. + +To edit your cluster, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit Config**. + +### Editing Clusters in YAML + +For a complete reference of configurable options for K3s clusters in YAML, see the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +To edit your cluster with YAML: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit as YAML**. +1. Edit the RKE options under the `rkeConfig` directive. + +## Configuration Options in the Rancher UI + +### Machine Pool Configuration + +This subsection covers generic machine pool configurations. For specific infrastructure provider configurations, refer to the following: + +- [Azure](../downstream-cluster-configuration/machine-configuration/azure.md) +- [DigitalOcean](../downstream-cluster-configuration/machine-configuration/digitalocean.md) +- [EC2](../downstream-cluster-configuration/machine-configuration/amazon-ec2.md) + +##### Pool Name + +The name of the machine pool. + +##### Machine Count + +The number of machines in the pool. + +##### Roles + +Option to assign etcd, control plane, and worker roles to nodes. + +#### Advanced + +##### Auto Replace + +The amount of time nodes can be unreachable before they are automatically deleted and replaced. + +##### Drain Before Delete + +Enables draining nodes by evicting all pods before the node is deleted. + +##### Kubernetes Node Labels + +Add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to nodes to help with organization and object selection. + +For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +##### Taints + +Add [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) to nodes, to prevent pods from being scheduled to or executed on the nodes, unless the pods have matching tolerations. + +### Cluster Configuration +#### Basics +##### Kubernetes Version + +The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). + +For more detail, see [Upgrading Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md). + +##### Pod Security Admission Configuration Template + +The default [pod security admission configuration template](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for the cluster. + +##### Encrypt Secrets + +Option to enable or disable secrets encryption. When enabled, secrets will be encrypted using a AES-CBC key. If disabled, any previously secrets will not be readable until encryption is enabled again. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/advanced/#secrets-encryption-config-experimental) for details. + +##### Project Network Isolation + +If your network provider allows project network isolation, you can choose whether to enable or disable inter-project communication. + +##### SELinux + +Option to enable or disable [SELinux](https://rancher.com/docs/k3s/latest/en/advanced/#selinux-support) support. + +##### CoreDNS + +By default, [CoreDNS](https://coredns.io/) is installed as the default DNS provider. If CoreDNS is not installed, an alternate DNS provider must be installed yourself. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/networking/#coredns) for details.. + +##### Klipper Service LB + +Option to enable or disable the [Klipper](https://github.com/rancher/klipper-lb) service load balancer. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer) for details. + +##### Traefik Ingress + +Option to enable or disable the [Traefik](https://traefik.io/) HTTP reverse proxy and load balancer. For more details and configuration options, see the [K3s documentation](https://rancher.com/docs/k3s/latest/en/networking/#traefik-ingress-controller). + +##### Local Storage + +Option to enable or disable [local storage](https://rancher.com/docs/k3s/latest/en/storage/) on the node(s). + +##### Metrics Server + +Option to enable or disable the [metrics server](https://github.com/kubernetes-incubator/metrics-server). If enabled, ensure port 10250 is opened for inbound TCP traffic. + +#### Add-On Config + +Additional Kubernetes manifests, managed as a [Add-on](https://kubernetes.io/docs/concepts/cluster-administration/addons/), to apply to the cluster on startup. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/helm/#automatically-deploying-manifests-and-helm-charts) for details. + +#### Agent Environment Vars + +Option to set environment variables for [K3s agents](https://rancher.com/docs/k3s/latest/en/architecture/). The environment variables can be set using key value pairs. Refer to the [K3 documentation](https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/) for more details. + +#### etcd + +##### Automatic Snapshots + +Option to enable or disable recurring etcd snapshots. If enabled, users have the option to configure the frequency of snapshots. For details, refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/backup-restore/#creating-snapshots). + +##### Metrics + +Option to choose whether to expose etcd metrics to the public or only within the cluster. + +#### Networking + +##### Cluster CIDR + +IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16). + +##### Service CIDR + +IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16). + +##### Cluster DNS + +IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10). + +##### Cluster Domain + +Select the domain for the cluster. The default is `cluster.local`. + +##### NodePort Service Port Range + +Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). The default is `30000-32767`. + +##### Truncate Hostnames + +Option to truncate hostnames to 15 characters or less. You can only set this field during the initial creation of the cluster. You can't enable or disable the 15 character limit after cluster creation. + +This setting only affects machine-provisioned clusters. Since custom clusters set hostnames during their own node creation process, which occurs outside of Rancher, this field doesn't restrict custom cluster hostname length. + +Truncating hostnames in a cluster improves compatibility with Windows-based systems. Although Kubernetes allows hostnames up to 63 characters in length, systems that use NetBIOS restrict hostnames to 15 characters or less. + +##### TLS Alternate Names + +Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert. + +##### Authorized Cluster Endpoint + +Authorized Cluster Endpoint can be used to directly access the Kubernetes API server, without requiring communication through Rancher. + +For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +#### Registries + +Select the image repository to pull Rancher images from. For more details and configuration options, see the [K3s documentation](https://rancher.com/docs/k3s/latest/en/installation/private-registry/). + +#### Upgrade Strategy + +##### Control Plane Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Worker Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Drain Nodes (Control Plane) + +Option to remove all pods from the node prior to upgrading. + +##### Drain Nodes (Worker Nodes) + +Option to remove all pods from the node prior to upgrading. + +#### Advanced + +Option to set kubelet options for different nodes. For available options, refer to the [Kubernetes documentation](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/). + +## Cluster Config File Reference + +Editing clusters in YAML allows you to set configurations that are already listed in [Configuration Options in the Rancher UI](#configuration-options-in-the-rancher-ui), as well as set Rancher-specific parameters. + +
+ + Example Cluster Config File Snippet + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +spec: + cloudCredentialSecretName: cattle-global-data:cc-fllv6 + clusterAgentDeploymentCustomization: {} + fleetAgentDeploymentCustomization: {} + kubernetesVersion: v1.26.7+k3s1 + localClusterAuthEndpoint: {} + rkeConfig: + chartValues: {} + etcd: + snapshotRetention: 5 + snapshotScheduleCron: 0 */5 * * * + machineGlobalConfig: + disable-apiserver: false + disable-cloud-controller: false + disable-controller-manager: false + disable-etcd: false + disable-kube-proxy: false + disable-network-policy: false + disable-scheduler: false + etcd-expose-metrics: false + kube-apiserver-arg: + - audit-policy-file=/etc/rancher/k3s/user-audit-policy.yaml + - audit-log-path=/etc/rancher/k3s/user-audit.logs + profile: null + secrets-encryption: false + machinePools: + - controlPlaneRole: true + etcdRole: true + machineConfigRef: + kind: Amazonec2Config + name: nc-test-pool1-pwl5h + name: pool1 + quantity: 1 + unhealthyNodeTimeout: 0s + workerRole: true + machineSelectorConfig: + - config: + docker: false + protect-kernel-defaults: false + selinux: false + machineSelectorFiles: + - fileSources: + - configMap: + name: '' + secret: + name: audit-policy + items: + - key: audit-policy + path: /etc/rancher/k3s/user-audit-policy.yaml + machineLabelSelector: + matchLabels: + rke.cattle.io/control-plane-role: 'true' + registries: {} + upgradeStrategy: + controlPlaneConcurrency: '1' + controlPlaneDrainOptions: + deleteEmptyDirData: true + disableEviction: false + enabled: false + force: false + gracePeriod: -1 + ignoreDaemonSets: true + ignoreErrors: false + postDrainHooks: null + preDrainHooks: null + skipWaitForDeleteTimeoutSeconds: 0 + timeout: 120 + workerConcurrency: '1' + workerDrainOptions: + deleteEmptyDirData: true + disableEviction: false + enabled: false + force: false + gracePeriod: -1 + ignoreDaemonSets: true + ignoreErrors: false + postDrainHooks: null + preDrainHooks: null + skipWaitForDeleteTimeoutSeconds: 0 + timeout: 120 +``` +
+ +### chartValues + +Specify the values for the system charts installed by K3s. + +Example: + +```yaml +chartValues: + chart-name: + key: value +``` +### machineGlobalConfig + +Specify K3s configurations. Any configuration change made here will apply to every node. The configuration options available in the [standalone version of k3s](https://docs.k3s.io/cli/server) can be applied here. + +Example: + +```yaml +machineGlobalConfig: + etcd-arg: + - key1=value1 + - key2=value2 +``` + +### machineSelectorConfig + +`machineSelectorConfig` is the same as [`machineGlobalConfig`](#machineglobalconfig) except that a [label](#kubernetes-node-labels) selector can be specified with the configuration. The configuration will only be applied to nodes that match the provided label selector. + +Multiple `config` entries are allowed, each specifying their own `machineLabelSelector`. A user can specify `matchExpressions`, `matchLabels`, both, or neither. Omitting the `machineLabelSelector` section of this field has the same effect as putting the config in the `machineGlobalConfig` section. + +Example: + +```yaml +machineSelectorConfig + - config: + config-key: config-value + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` +### machineSelectorFiles + +:::note + +This feature is available in Rancher v2.7.2 and later. + +::: + +Deliver files to nodes, so that the files can be in place before initiating K3s server or agent processes. +The content of the file is retrieved from either a secret or a configmap. The target nodes are filtered by the `machineLabelSelector`. + +Example : + +```yaml +machineSelectorFiles: + - fileSources: + - secret: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-secret-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 + - fileSources: + - configMap: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-configmap-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` + +The secret or configmap must meet the following requirements: + +1. It must be in the `fleet-default` namespace where the Cluster object exists. +2. It must have the annotation `rke.cattle.io/object-authorized-for-clusters: cluster-name1,cluster-name2`, which permits the target clusters to use it. + +:::tip + +Rancher Dashboard provides an easy-to-use form for creating the secret or configmap. + +::: + +Example: + +```yaml +apiVersion: v1 +data: + audit-policy: >- + IyBMb2cgYWxsIHJlcXVlc3RzIGF0IHRoZSBNZXRhZGF0YSBsZXZlbC4KYXBpVmVyc2lvbjogYXVkaXQuazhzLmlvL3YxCmtpbmQ6IFBvbGljeQpydWxlczoKLSBsZXZlbDogTWV0YWRhdGE= +kind: Secret +metadata: + annotations: + rke.cattle.io/object-authorized-for-clusters: cluster1 + name: name1 + namespace: fleet-default +``` diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md new file mode 100644 index 00000000000..6d260b3dd68 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -0,0 +1,365 @@ +--- +title: RKE Cluster Configuration Reference +--- + + + + + +When Rancher installs Kubernetes, it uses [RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) or [RKE2](https://docs.rke2.io/) as the Kubernetes distribution. + +This section covers the configuration options that are available in Rancher for a new or existing RKE Kubernetes cluster. + + +## Overview + +You can configure the Kubernetes options one of two ways: + +- [Rancher UI](#configuration-options-in-the-rancher-ui): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. +- [Cluster Config File](#rke-cluster-config-file-reference): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the options available in an RKE installation, except for system_images configuration, by specifying them in YAML. + +The RKE cluster config options are nested under the `rancher_kubernetes_engine_config` directive. For more information, see the section about the [cluster config file.](#rke-cluster-config-file-reference) + +In [clusters launched by RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md), you can edit any of the remaining options that follow. + +For an example of RKE config file syntax, see the [RKE documentation](https://rancher.com/docs/rke/latest/en/example-yamls/). + +The forms in the Rancher UI don't include all advanced options for configuring RKE. For the complete reference of configurable options for RKE Kubernetes clusters in YAML, see the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/) + +## Editing Clusters with a Form in the Rancher UI + +To edit your cluster, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit Config**. + + +## Editing Clusters with YAML + +Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the options available in an RKE installation, except for system_images configuration, by specifying them in YAML. + +RKE clusters (also called RKE1 clusters) are edited differently than RKE2 and K3s clusters. + +To edit an RKE config file directly from the Rancher UI, + +1. Click **☰ > Cluster Management**. +1. Go to the RKE cluster you want to configure. Click and click **⋮ > Edit Config**. This take you to the RKE configuration form. Note: Because cluster provisioning changed in Rancher 2.6, the **⋮ > Edit as YAML** can be used for configuring RKE2 clusters, but it can't be used for editing RKE1 configuration. +1. In the configuration form, scroll down and click **Edit as YAML**. +1. Edit the RKE options under the `rancher_kubernetes_engine_config` directive. + +## Configuration Options in the Rancher UI + +:::tip + +Some advanced configuration options are not exposed in the Rancher UI forms, but they can be enabled by editing the RKE cluster configuration file in YAML. For the complete reference of configurable options for RKE Kubernetes clusters in YAML, see the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/) + +::: + +### Kubernetes Version + +The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). + +For more detail, see [Upgrading Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md). + +### Network Provider + +The [Network Provider](https://kubernetes.io/docs/concepts/cluster-administration/networking/) that the cluster uses. For more details on the different networking providers, please view our [Networking FAQ](../../../faq/container-network-interface-providers.md). + +:::caution + +After you launch the cluster, you cannot change your network provider. Therefore, choose which network provider you want to use carefully, as Kubernetes doesn't allow switching between network providers. Once a cluster is created with a network provider, changing network providers would require you tear down the entire cluster and all its applications. + +::: + +Out of the box, Rancher is compatible with the following network providers: + +- [Canal](https://github.com/projectcalico/canal) +- [Flannel](https://github.com/coreos/flannel#flannel) +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Weave](https://github.com/weaveworks/weave) + +:::note Notes on Weave: + +When Weave is selected as network provider, Rancher will automatically enable encryption by generating a random password. If you want to specify the password manually, please see how to configure your cluster using a [Config File](#rke-cluster-config-file-reference) and the [Weave Network Plug-in Options](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/#weave-network-plug-in-options). + +::: + +### Project Network Isolation + +If your network provider allows project network isolation, you can choose whether to enable or disable inter-project communication. + +Project network isolation is available if you are using any RKE network plugin that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin. + +### Kubernetes Cloud Providers + +You can configure a [Kubernetes cloud provider](../../../pages-for-subheaders/set-up-cloud-providers.md). If you want to use dynamically provisioned [volumes and storage](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) in Kubernetes, typically you must select the specific cloud provider in order to use it. For example, if you want to use Amazon EBS, you would need to select the `aws` cloud provider. + +:::note + +If the cloud provider you want to use is not listed as an option, you will need to use the [config file option](#rke-cluster-config-file-reference) to configure the cloud provider. Please reference the [RKE cloud provider documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to configure the cloud provider. + +::: + +### Private Registries + +The cluster-level private registry configuration is only used for provisioning clusters. + +There are two main ways to set up private registries in Rancher: by setting up the [global default registry](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md) through the **Settings** tab in the global view, and by setting up a private registry in the advanced options in the cluster-level settings. The global default registry is intended to be used for air-gapped setups, for registries that do not require credentials. The cluster-level private registry is intended to be used in all setups in which the private registry requires credentials. + +If your private registry requires credentials, you need to pass the credentials to Rancher by editing the cluster options for each cluster that needs to pull images from the registry. + +The private registry configuration option tells Rancher where to pull the [system images](https://rancher.com/docs/rke/latest/en/config-options/system-images/) or [addon images](https://rancher.com/docs/rke/latest/en/config-options/add-ons/) that will be used in your cluster. + +- **System images** are components needed to maintain the Kubernetes cluster. +- **Add-ons** are used to deploy several cluster components, including network plug-ins, the ingress controller, the DNS provider, or the metrics server. + +For more information on setting up a private registry for components applied during the provisioning of the cluster, see the [RKE documentation on private registries](https://rancher.com/docs/rke/latest/en/config-options/private-registries/). + +Rancher v2.6 introduced the ability to configure [ECR registries for RKE clusters](https://rancher.com/docs/rke/latest/en/config-options/private-registries/#amazon-elastic-container-registry-ecr-private-registry-setup). + +### Authorized Cluster Endpoint + +Authorized Cluster Endpoint (ACE) can be used to directly access the Kubernetes API server, without requiring communication through Rancher. + +:::note + +ACE is available on RKE, RKE2, and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. + +::: + +ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) on RKE2 and K3s clusters. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. + +For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +### Node Pools + +For information on using the Rancher UI to set up node pools in an RKE cluster, refer to [this page.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) + +### NGINX Ingress + +If you want to publish your applications in a high-availability configuration, and you're hosting your nodes with a cloud-provider that doesn't have a native load-balancing feature, enable this option to use NGINX Ingress within the cluster. + +### Metrics Server Monitoring + +Option to enable or disable [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/). + +Each cloud provider capable of launching a cluster using RKE can collect metrics and monitor for your cluster nodes. Enable this option to view your node metrics from your cloud provider's portal. + +### Pod Security Policy Support + +Enables [pod security policies](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) for the cluster. After enabling this option, choose a policy using the **Default Pod Security Policy** drop-down. + +You must have an existing Pod Security Policy configured before you can use this option. + +### Docker Version on Nodes + +Configures whether nodes are allowed to run versions of Docker that Rancher doesn't officially support. + +If you choose to require a supported Docker version, Rancher will stop pods from running on nodes that don't have a supported Docker version installed. + +For details on which Docker versions were tested with each Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) + +### Docker Root Directory + +If the nodes you are adding to the cluster have Docker configured with a non-default Docker Root Directory (default is `/var/lib/docker`), specify the correct Docker Root Directory in this option. + +### Default Pod Security Policy + +If you enable **Pod Security Policy Support**, use this drop-down to choose the pod security policy that's applied to the cluster. + +### Node Port Range + +Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). Default is `30000-32767`. + +### Recurring etcd Snapshots + +Option to enable or disable [recurring etcd snapshots](https://rancher.com/docs/rke/latest/en/etcd-snapshots/#etcd-recurring-snapshots). + +### Agent Environment Variables + +Option to set environment variables for [rancher agents](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. If rancher agent requires use of proxy to communicate with Rancher server, `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables can be set using agent environment variables. + +### Updating ingress-nginx + +Clusters that were created before Kubernetes 1.16 will have an `ingress-nginx` `updateStrategy` of `OnDelete`. Clusters that were created with Kubernetes 1.16 or newer will have `RollingUpdate`. + +If the `updateStrategy` of `ingress-nginx` is `OnDelete`, you will need to delete these pods to get the correct version for your deployment. + +### Cluster Agent Configuration and Fleet Agent Configuration + +You can configure the scheduling fields and resource limits for the Cluster Agent and the cluster's Fleet Agent. You can use these fields to customize tolerations, affinity rules, and resource requirements. Additional tolerations are appended to a list of default tolerations and control plane node taints. If you define custom affinity rules, they override the global default affinity setting. Defining resource requirements sets requests or limits where there previously were none. + +:::note + +With this option, it's possible to override or remove rules that are required for the functioning of the cluster. We strongly recommend against removing or overriding these and any other affinity rules, as this may cause unwanted side effects: + +- `affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution` for `cattle-cluster-agent` +- `cluster-agent-default-affinity` for `cattle-cluster-agent` +- `fleet-agent-default-affinity` for `fleet-agent` + +::: + +If you downgrade Rancher to v2.7.4 or below, your changes will be lost and the agents will re-deploy without your customizations. The Fleet agent will fallback to using its built-in default values when it re-deploys. If the Fleet version doesn't change during the downgrade, the re-deploy won't be immediate. + + +## RKE Cluster Config File Reference + +Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the [options available](https://rancher.com/docs/rke/latest/en/config-options/) in an RKE installation, except for `system_images` configuration. The `system_images` option is not supported when creating a cluster with the Rancher UI or API. + +For the complete reference for configurable options for RKE Kubernetes clusters in YAML, see the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/) + +### Config File Structure in Rancher + +RKE (Rancher Kubernetes Engine) is the tool that Rancher uses to provision Kubernetes clusters. Rancher's cluster config files used to have the same structure as [RKE config files,](https://rancher.com/docs/rke/latest/en/example-yamls/) but the structure changed so that in Rancher, RKE cluster config items are separated from non-RKE config items. Therefore, configuration for your cluster needs to be nested under the `rancher_kubernetes_engine_config` directive in the cluster config file. Cluster config files created with earlier versions of Rancher will need to be updated for this format. An example cluster config file is included below. + +
+ Example Cluster Config File + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker +enable_cluster_alerting: false +enable_cluster_monitoring: false +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # Currently only nginx ingress provider is supported. +# # To disable ingress controller, set `provider: none` +# # To enable ingress on specific nodes, use the node_selector, eg: +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` +
+ +### Default DNS provider + +The table below indicates what DNS provider is deployed by default. See [RKE documentation on DNS provider](https://rancher.com/docs/rke/latest/en/config-options/add-ons/dns/) for more information how to configure a different DNS provider. CoreDNS can only be used on Kubernetes v1.12.0 and higher. + +| Rancher version | Kubernetes version | Default DNS provider | +|-------------|--------------------|----------------------| +| v2.2.5 and higher | v1.14.0 and higher | CoreDNS | +| v2.2.5 and higher | v1.13.x and lower | kube-dns | +| v2.2.4 and lower | any | kube-dns | + +## Rancher Specific Parameters in YAML + +Besides the RKE config file options, there are also Rancher specific settings that can be configured in the Config File (YAML): + +### docker_root_dir + +See [Docker Root Directory](#docker-root-directory). + +### enable_cluster_monitoring + +Option to enable or disable [Cluster Monitoring](../../../pages-for-subheaders/monitoring-and-alerting.md). + +### enable_network_policy + +Option to enable or disable Project Network Isolation. + +Project network isolation is available if you are using any RKE network plugin that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin. + +### local_cluster_auth_endpoint + +See [Authorized Cluster Endpoint](#authorized-cluster-endpoint). + +Example: + +```yaml +local_cluster_auth_endpoint: + enabled: true + fqdn: "FQDN" + ca_certs: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +### Custom Network Plug-in + +You can add a custom network plug-in by using the [user-defined add-on functionality](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/) of RKE. You define any add-on that you want deployed after the Kubernetes cluster is deployed. + +There are two ways that you can specify an add-on: + +- [In-line Add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#in-line-add-ons) +- [Referencing YAML Files for Add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#referencing-yaml-files-for-add-ons) + +For an example of how to configure a custom network plug-in by editing the `cluster.yml`, refer to the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/custom-network-plugin-example) \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md new file mode 100644 index 00000000000..116b9379947 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md @@ -0,0 +1,464 @@ +--- +title: RKE2 Cluster Configuration Reference +--- + + + + + +This section covers the configuration options that are available in Rancher for a new or existing RKE2 Kubernetes cluster. + +## Overview + +You can configure the Kubernetes options in one of the two following ways: + +- [Rancher UI](#configuration-options-in-the-rancher-ui): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. +- [Cluster Config File](#cluster-config-file-reference): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE2 config file. Using a config file allows you to set many additional [options](https://docs.rke2.io/install/configuration) available for an RKE2 installation. + +## Editing Clusters in the Rancher UI + +The Rancher UI provides two ways to edit a cluster: +1. With a form. +1. With YAML. + +### Editing Clusters with a Form + +The form covers the most frequently needed options for clusters. + +To edit your cluster, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit Config**. + +### Editing Clusters in YAML + +For a complete reference of configurable options for K3s clusters in YAML, see the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +To edit your cluster in YAML: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit as YAML**. +1. Edit the RKE options under the `rkeConfig` directive. + +## Configuration Options in the Rancher UI + +### Machine Pool Configuration + +This subsection covers generic machine pool configurations. For specific infrastructure provider configurations, refer to the following: + +- [Azure](../downstream-cluster-configuration/machine-configuration/azure.md) +- [DigitalOcean](../downstream-cluster-configuration/machine-configuration/digitalocean.md) +- [EC2](../downstream-cluster-configuration/machine-configuration/amazon-ec2.md) + +##### Pool Name + +The name of the machine pool. + +##### Machine Count + +The number of machines in the pool. + +##### Roles + +Option to assign etcd, control plane, and worker roles to nodes. + +#### Advanced + +##### Auto Replace + +The amount of time nodes can be unreachable before they are automatically deleted and replaced. + +##### Drain Before Delete + +Enables draining nodes by evicting all pods before the node is deleted. + +##### Kubernetes Node Labels + +Add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to nodes to help with organization and object selection. + +For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +##### Taints + +Add [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) to nodes, to prevent pods from being scheduled to or executed on the nodes, unless the pods have matching tolerations. + +### Cluster Configuration +#### Basics +##### Kubernetes Version + +The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). + +For more detail, see [Upgrading Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md). + +##### Container Network Provider + +The [Network Provider](https://kubernetes.io/docs/concepts/cluster-administration/networking/) that the cluster uses. + +:::caution + +After you launch the cluster, you cannot change your network provider. Therefore, choose which network provider you want to use carefully, as Kubernetes doesn't allow switching between network providers. Once a cluster is created with a network provider, changing network providers would require you to tear down the entire cluster and all its applications. + +::: + +Out of the box, Rancher is compatible with the following network providers: + +- [Canal](https://github.com/projectcalico/canal) +- [Cilium](https://cilium.io/)* +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Multus](https://github.com/k8snetworkplumbingwg/multus-cni) + +\* When using [project network isolation](#project-network-isolation) in the [Cilium CNI](../../../faq/container-network-interface-providers.md#cilium), it is possible to enable cross-node ingress routing. Click the [CNI provider docs](../../../faq/container-network-interface-providers.md#ingress-routing-across-nodes-in-cilium) to learn more. + +For more details on the different networking providers and how to configure them, please view our [RKE2 documentation](https://docs.rke2.io/install/network_options). + +###### Dual-stack Networking + +[Dual-stack](https://docs.rke2.io/install/network_options#dual-stack-configuration) networking is supported for all CNI providers. To configure RKE2 in dual-stack mode, set valid IPv4/IPv6 CIDRs for your [Cluster CIDR](#cluster-cidr) and/or [Service CIDR](#service-cidr). + +###### Additional Configuration {#dual-stack-additional-config} + +When using `cilium` or `multus,cilium` as your container network interface provider, ensure the **Enable IPv6 Support** option is also enabled. + +##### Cloud Provider + +You can configure a [Kubernetes cloud provider](../../../pages-for-subheaders/set-up-cloud-providers.md). If you want to use dynamically provisioned [volumes and storage](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) in Kubernetes, typically you must select the specific cloud provider in order to use it. For example, if you want to use Amazon EBS, you would need to select the `aws` cloud provider. + +:::note + +If the cloud provider you want to use is not listed as an option, you will need to use the [config file option](#cluster-config-file-reference) to configure the cloud provider. Please reference [this documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to configure the cloud provider. + +::: + +##### Default Pod Security Policy + +The default [pod security policy](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) for the cluster. Please refer to the [RKE2 documentation](https://docs.rke2.io/security/pod_security_policies) on the specifications of each available policy. + +##### Pod Security Admission Configuration Template + +The default [pod security admission configuration template](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for the cluster. + +##### Worker CIS Profile + +Select a [CIS benchmark](../../../pages-for-subheaders/cis-scan-guides.md) to validate the system configuration against. + +##### Project Network Isolation + +If your network provider allows project network isolation, you can choose whether to enable or disable inter-project communication. + +Project network isolation is available if you are using any RKE2 network plugin that supports the enforcement of Kubernetes network policies, such as Canal. + +##### CoreDNS + +By default, [CoreDNS](https://coredns.io/) is installed as the default DNS provider. If CoreDNS is not installed, an alternate DNS provider must be installed yourself. Refer to the [RKE2 documentation](https://docs.rke2.io/networking#coredns) for additional CoreDNS configurations. + +##### NGINX Ingress + +If you want to publish your applications in a high-availability configuration, and you're hosting your nodes with a cloud-provider that doesn't have a native load-balancing feature, enable this option to use NGINX Ingress within the cluster. Refer to the [RKE2 documentation](https://docs.rke2.io/networking#nginx-ingress-controller) for additional configuration options. + +Refer to the [RKE2 documentation](https://docs.rke2.io/networking#nginx-ingress-controller) for additional configuration options. + +##### Metrics Server + +Option to enable or disable [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/). + +Each cloud provider capable of launching a cluster using RKE2 can collect metrics and monitor for your cluster nodes. Enable this option to view your node metrics from your cloud provider's portal. + +#### Add-On Config + +Additional Kubernetes manifests, managed as an [Add-on](https://kubernetes.io/docs/concepts/cluster-administration/addons/), to apply to the cluster on startup. Refer to the [RKE2 documentation](https://docs.rke2.io/helm#automatically-deploying-manifests-and-helm-charts) for details. + +#### Agent Environment Vars + +Option to set environment variables for [Rancher agents](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. Refer to the [RKE2 documentation](https://docs.rke2.io/reference/linux_agent_config) for more details. + +#### etcd + +##### Automatic Snapshots + +Option to enable or disable recurring etcd snapshots. If enabled, users have the option to configure the frequency of snapshots. For details, refer to the [RKE2 documentation](https://docs.rke2.io/backup_restore#creating-snapshots). Note that with RKE2, snapshots are stored on each etcd node. This varies from RKE1 which only stores one snapshot per cluster. + +##### Metrics + +Option to choose whether to expose etcd metrics to the public or only within the cluster. + +#### Networking + +##### Cluster CIDR + +IPv4 and/or IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16). + +###### Dual-stack Networking + +To configure [dual-stack](https://docs.rke2.io/install/network_options#dual-stack-configuration) mode, enter a valid IPv4/IPv6 CIDR. For example `10.42.0.0/16,2001:cafe:42:0::/56`. + +[Additional configuration](#dual-stack-additional-config) is required when using `cilium` or `multus,cilium` as your [container network](#container-network-provider) interface provider. + +##### Service CIDR + +IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16). + +###### Dual-stack Networking + +To configure [dual-stack](https://docs.rke2.io/install/network_options#dual-stack-configuration) mode, enter a valid IPv4/IPv6 CIDR. For example `10.42.0.0/16,2001:cafe:42:0::/56`. + +[Additional configuration](#dual-stack-additional-config) is required when using `cilium ` or `multus,cilium` as your [container network](#container-network-provider) interface provider. + +##### Cluster DNS + +IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10). + +##### Cluster Domain + +Select the domain for the cluster. The default is `cluster.local`. + +##### NodePort Service Port Range + +Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). The default is `30000-32767`. + +##### Truncate Hostnames + +Option to truncate hostnames to 15 characters or less. You can only set this field during the initial creation of the cluster. You can't enable or disable the 15 character limit after cluster creation. + +This setting only affects machine-provisioned clusters. Since custom clusters set hostnames during their own node creation process, which occurs outside of Rancher, this field doesn't restrict custom cluster hostname length. + +Truncating hostnames in a cluster improves compatibility with Windows-based systems. Although Kubernetes allows hostnames up to 63 characters in length, systems that use NetBIOS restrict hostnames to 15 characters or less. + +##### TLS Alternate Names + +Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert. + +##### Authorized Cluster Endpoint + +Authorized Cluster Endpoint can be used to directly access the Kubernetes API server, without requiring communication through Rancher. + +This is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self signed certificates. + +For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +#### Registries + +Select the image repository to pull Rancher images from. For more details and configuration options, see the [RKE2 documentation](https://docs.rke2.io/install/containerd_registry_configuration). + +#### Upgrade Strategy + +##### Control Plane Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Worker Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Drain Nodes (Control Plane) + +Option to remove all pods from the node prior to upgrading. + +##### Drain Nodes (Worker Nodes) + +Option to remove all pods from the node prior to upgrading. + +#### Advanced + +Option to set kubelet options for different nodes. For available options, refer to the [Kubernetes documentation](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/). + +## Cluster Config File Reference + +Editing clusters in YAML allows you to set the [options available](https://docs.rke2.io/install/configuration) in an RKE2 installation, including those already listed in [Configuration Options in the Rancher UI](#configuration-options-in-the-rancher-ui), as well as set Rancher-specific parameters. + +
+ + Example Cluster Config File Snippet + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +spec: + cloudCredentialSecretName: cattle-global-data:cc-s879v + kubernetesVersion: v1.25.12+rke2r1 + localClusterAuthEndpoint: {} + rkeConfig: + chartValues: + rke2-calico: {} + etcd: + snapshotRetention: 5 + snapshotScheduleCron: 0 */5 * * * + machineGlobalConfig: + cni: calico + disable-kube-proxy: false + etcd-expose-metrics: false + profile: null + kube-apiserver-arg: + - audit-policy-file=/etc/rancher/rke2/user-audit-policy.yaml + - audit-log-path=/etc/rancher/rke2/user-audit.logs + machinePools: + - controlPlaneRole: true + etcdRole: true + machineConfigRef: + kind: Amazonec2Config + name: nc-test-pool1-pwl5h + name: pool1 + quantity: 1 + unhealthyNodeTimeout: 0s + workerRole: true + machineSelectorConfig: + - config: + protect-kernel-defaults: false + machineSelectorFiles: + - fileSources: + - configMap: + name: '' + secret: + name: audit-policy + items: + - key: audit-policy + path: /etc/rancher/rke2/user-audit-policy.yaml + machineLabelSelector: + matchLabels: + rke.cattle.io/control-plane-role: 'true' + registries: {} + upgradeStrategy: + controlPlaneConcurrency: "1" + controlPlaneDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 + workerConcurrency: "1" + workerDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 +``` +
+ +### chartValues + +Specify the values for the system charts installed by RKE2. + +Example: + +```yaml +chartValues: + chart-name: + key: value +``` +### machineGlobalConfig + +Specify RKE2 configurations. Any configuration change made here will apply to every node. The configuration options available in the [standalone version of RKE2](https://docs.rke2.io/reference/server_config) can be applied here. + +Example: + +```yaml +machineGlobalConfig: + etcd-arg: + - key1=value1 + - key2=value2 +``` + +### machineSelectorConfig + +`machineSelectorConfig` is the same as [`machineGlobalConfig`](#machineglobalconfig) except that a [label](#kubernetes-node-labels) selector can be specified with the configuration. The configuration will only be applied to nodes that match the provided label selector. + +Multiple `config` entries are allowed, each specifying their own `machineLabelSelector`. A user can specify `matchExpressions`, `matchLabels`, both, or neither. Omitting the `machineLabelSelector` section of this field has the same effect as putting the config in the `machineGlobalConfig` section. + +Example: + +```yaml +machineSelectorConfig + - config: + config-key: config-value + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` +### machineSelectorFiles + +:::note + +This feature is available in Rancher v2.7.2 and later. + +::: + +Deliver files to nodes, so that the files can be in place before initiating RKE2 server or agent processes. +The content of the file is retrieved from either a secret or a configmap. The target nodes are filtered by the `machineLabelSelector`. + +Example : + +```yaml +machineSelectorFiles: + - fileSources: + - secret: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-secret-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 + - fileSources: + - configMap: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-configmap-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` + +The secret or configmap must meet the following requirements: + +1. It must be in the `fleet-default` namespace where the Cluster object exists. +2. It must have the annotation `rke.cattle.io/object-authorized-for-clusters: cluster-name1,cluster-name2`, which permits the target clusters to use it. + +:::tip + +Rancher Dashboard provides an easy-to-use form for creating the secret or configmap. + +::: + +Example: + +```yaml +apiVersion: v1 +data: + audit-policy: >- + IyBMb2cgYWxsIHJlcXVlc3RzIGF0IHRoZSBNZXRhZGF0YSBsZXZlbC4KYXBpVmVyc2lvbjogYXVkaXQuazhzLmlvL3YxCmtpbmQ6IFBvbGljeQpydWxlczoKLSBsZXZlbDogTWV0YWRhdGE= +kind: Secret +metadata: + annotations: + rke.cattle.io/object-authorized-for-clusters: cluster1 + name: name1 + namespace: fleet-default +``` diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md new file mode 100644 index 00000000000..63fa8df3511 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -0,0 +1,45 @@ +--- +title: Syncing +--- + + + + + +Syncing allows Rancher to update cluster values so that they're up to date with the corresponding cluster object hosted in AKS, EKS or GKE. This enables sources other than Rancher to own a hosted cluster’s state. + +:::warning +You may accidentally overwrite the state from one source if you simultaneously process an update from another source. This may also occur if you process an update from one source within 5 minutes of finishing an update from another source. +::: + +### How it works + +There are two fields on the Rancher Cluster object that must be understood to understand how syncing works: + +1. The config object for the cluster, located on the Spec of the Cluster: + + * For AKS, the field is called AKSConfig + * For EKS, the field is called EKSConfig + * For GKE, the field is called GKEConfig + +2. The UpstreamSpec object + + * For AKS, this is located on the AKSStatus field on the Status of the Cluster. + * For EKS, this is located on the EKSStatus field on the Status of the Cluster. + * For GKE, this is located on the GKEStatus field on the Status of the Cluster. + +The struct types that define these objects can be found in their corresponding operator projects: + + * [aks-operator](https://github.com/rancher/aks-operator/blob/master/pkg/apis/aks.cattle.io/v1/types.go) + * [eks-operator](https://github.com/rancher/eks-operator/blob/master/pkg/apis/eks.cattle.io/v1/types.go) + * [gke-operator](https://github.com/rancher/gke-operator/blob/master/pkg/apis/gke.cattle.io/v1/types.go) + +All fields are nillable, except for the following: the cluster name, the location (region or zone), Imported, and the cloud credential reference. + +The AKSConfig, EKSConfig or GKEConfig represents the desired state. Nil values are ignored. Fields that are non-nil in the config object can be thought of as managed. When a cluster is created in Rancher, all fields are non-nil and therefore managed. When a pre-existing cluster is registered in Rancher all nillable fields are set to nil and aren't managed. Those fields become managed once their value has been changed by Rancher. + +UpstreamSpec represents the cluster as it is in the hosted Kubernetes provider. It's refreshed every 5 minutes. After the UpstreamSpec is refreshed, Rancher checks if the cluster has an update in progress. If it's currently updating, nothing further is done. If it is not currently updating, any managed fields on AKSConfig, EKSConfig or GKEConfig are overwritten with their corresponding value from the recently updated UpstreamSpec. + +The effective desired state can be thought of as the UpstreamSpec, plus all non-nil fields in the AKSConfig, EKSConfig or GKEConfig. This is what is displayed in the UI. + +If Rancher and another source attempt to update a cluster at the same time, or within 5 minutes of an update finishing, any managed fields are likely to get caught in a race condition. To use EKS as an example, a cluster may have PrivateAccess as a managed field. If PrivateAccess is false and then enabled in EKS console at 11:01, and tags are updated from Rancher before 11:05, then the value is likely to be overwritten. This can also occur if tags are updated while the cluster is still processing the update. The issue described in this example shouldn't occur if the cluster is registered and the PrivateAccess fields are nil. diff --git a/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md new file mode 100644 index 00000000000..3c363d053cf --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -0,0 +1,57 @@ +--- +title: Rancher Agent Options +--- + + + + + +Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. + +For an overview of how Rancher communicates with downstream clusters using node agents, refer to the [architecture section.](../../../rancher-manager-architecture/communicating-with-downstream-user-clusters.md#3-node-agents) + +## General options + +| Parameter | Environment variable | Description | +| ---------- | -------------------- | ----------- | +| `--server` | `CATTLE_SERVER` | The configured Rancher `server-url` setting which the agent connects to | +| `--token` | `CATTLE_TOKEN` | Token that is needed to register the node in Rancher | +| `--ca-checksum` | `CATTLE_CA_CHECKSUM` | The SHA256 checksum of the configured Rancher `cacerts` setting to validate | +| `--node-name` | `CATTLE_NODE_NAME` | Override the hostname that is used to register the node (defaults to `hostname -s`) | +| `--label` | `CATTLE_NODE_LABEL` | Add node labels to the node. For multiple labels, pass additional `--label` options. (`--label key=value`) | +| `--taints` | `CATTLE_NODE_TAINTS` | Add node taints to the node. For multiple taints, pass additional `--taints` options. (`--taints key=value:effect`) | + +## Role options + +| Parameter | Environment variable | Description | +| ---------- | -------------------- | ----------- | +| `--all-roles` | `ALL=true` | Apply all roles (`etcd`,`controlplane`,`worker`) to the node | +| `--etcd` | `ETCD=true` | Apply the role `etcd` to the node | +| `--controlplane` | `CONTROL=true` | Apply the role `controlplane` to the node | +| `--worker` | `WORKER=true` | Apply the role `worker` to the node | + +## IP address options + +| Parameter | Environment variable | Description | +| ---------- | -------------------- | ----------- | +| `--address` | `CATTLE_ADDRESS` | The IP address the node will be registered with (defaults to the IP used to reach `8.8.8.8`) | +| `--internal-address` | `CATTLE_INTERNAL_ADDRESS` | The IP address used for inter-host communication on a private network | + +### Dynamic IP address options + +For automation purposes, you can't have a specific IP address in a command as it has to be generic to be used for every node. For this, we have dynamic IP address options. They are used as a value to the existing IP address options. This is supported for `--address` and `--internal-address`. + +| Value | Example | Description | +| ---------- | -------------------- | ----------- | +| Interface name | `--address eth0` | The first configured IP address will be retrieved from the given interface | +| `ipify` | `--address ipify` | Value retrieved from `https://api.ipify.org` will be used | +| `awslocal` | `--address awslocal` | Value retrieved from `http://169.254.169.254/latest/meta-data/local-ipv4` will be used | +| `awspublic` | `--address awspublic` | Value retrieved from `http://169.254.169.254/latest/meta-data/public-ipv4` will be used | +| `doprivate` | `--address doprivate` | Value retrieved from `http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address` will be used | +| `dopublic` | `--address dopublic` | Value retrieved from `http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address` will be used | +| `azprivate` | `--address azprivate` | Value retrieved from `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text` will be used | +| `azpublic` | `--address azpublic` | Value retrieved from `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-08-01&format=text` will be used | +| `gceinternal` | `--address gceinternal` | Value retrieved from `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip` will be used | +| `gceexternal` | `--address gceexternal` | Value retrieved from `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip` will be used | +| `packetlocal` | `--address packetlocal` | Value retrieved from `https://metadata.packet.net/2009-04-04/meta-data/local-ipv4` will be used | +| `packetpublic` | `--address packetlocal` | Value retrieved from `https://metadata.packet.net/2009-04-04/meta-data/public-ipv4` will be used | diff --git a/versioned_docs/version-2.8/reference-guides/kubernetes-concepts.md b/versioned_docs/version-2.8/reference-guides/kubernetes-concepts.md new file mode 100644 index 00000000000..631b7cfd118 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/kubernetes-concepts.md @@ -0,0 +1,64 @@ +--- +title: Kubernetes Concepts +--- + +This page explains concepts related to Kubernetes that are important for understanding how Rancher works. The descriptions below provide a simplified overview of Kubernetes components. For more details, refer to the [official documentation on Kubernetes components.](https://kubernetes.io/docs/concepts/overview/components/) + +## About Docker + +Docker is the container packaging and runtime standard. Developers build container images from Dockerfiles and distribute container images from Docker registries. [Docker Hub](https://hub.docker.com) is the most popular public registry. Many organizations also set up private Docker registries. Docker is primarily used to manage containers on individual nodes. + +:::note + +Although Rancher 1.6 supported Docker Swarm clustering technology, it is no longer supported in Rancher 2.x due to the success of Kubernetes. + +::: + +## About Kubernetes + +Kubernetes is the container cluster management standard. YAML files specify containers and other resources that form an application. Kubernetes performs functions such as scheduling, scaling, service discovery, health check, secret management, and configuration management. + +## What is a Kubernetes Cluster? + +A cluster is a group of computers that work together as a single system. + +A _Kubernetes Cluster_ is a cluster that uses the [Kubernetes container-orchestration system](https://kubernetes.io/) to deploy, maintain, and scale Docker containers, allowing your organization to automate application operations. + +## Roles for Nodes in Kubernetes Clusters + +Each computing resource in a Kubernetes cluster is called a _node_. Nodes can be either bare-metal servers or virtual machines. Kubernetes classifies nodes into three types: _etcd_ nodes, _control plane_ nodes, and _worker_ nodes. + +A Kubernetes cluster consists of at least one etcd, controlplane, and worker node. + +### etcd Nodes + +Rancher uses etcd as a data store in both single node and high-availability installations. In Kubernetes, etcd is also a role for nodes that store the cluster state. + +The state of a Kubernetes cluster is maintained in [etcd.](https://kubernetes.io/docs/concepts/overview/components/#etcd) The etcd nodes run the etcd database. + +The etcd database component is a distributed key-value store used as Kubernetes storage for all cluster data, such as cluster coordination and state management. It is recommended to run etcd on multiple nodes so that there's always a backup available for failover. + +Although you can run etcd on just one node, etcd requires a majority of nodes, a quorum, to agree on updates to the cluster state. The cluster should always contain enough healthy etcd nodes to form a quorum. For a cluster with n members, a quorum is (n/2)+1. For any odd-sized cluster, adding one node will always increase the number of nodes necessary for a quorum. + +Three etcd nodes is generally sufficient for smaller clusters and five etcd nodes for large clusters. + +### Controlplane Nodes + +Controlplane nodes run the Kubernetes API server, scheduler, and controller manager. These nodes take care of routine tasks to ensure that your cluster maintains your configuration. Because all cluster data is stored on your etcd nodes, control plane nodes are stateless. You can run control plane on a single node, although three or more nodes are recommended for redundancy. Additionally, a single node can share the control plane and etcd roles. + +### Worker Nodes + +Each [worker node](https://kubernetes.io/docs/concepts/architecture/nodes/) runs the following: + +- **Kubelets:** An agent that monitors the state of the node, ensuring your containers are healthy. +- **Workloads:** The containers and pods that hold your apps, as well as other types of deployments. + +Worker nodes also run storage and networking drivers, and ingress controllers when required. You create as many worker nodes as necessary to run your [workloads](../pages-for-subheaders/workloads-and-pods.md). + +## About Helm + +For high-availability installations of Rancher, Helm is the tool used to install Rancher on a Kubernetes cluster. + +Helm is the package management tool of choice for Kubernetes. Helm charts provide templating syntax for Kubernetes YAML manifest documents. With Helm we can create configurable deployments instead of just using static files. For more information about creating your own catalog of deployments, check out the docs at [https://helm.sh/](https://helm.sh). + +For more information on service accounts and cluster role binding, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) diff --git a/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/examples.md b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/examples.md new file mode 100644 index 00000000000..c65722cfa97 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/examples.md @@ -0,0 +1,27 @@ +--- +title: Examples +--- + + + + + +### ServiceMonitor + +An example ServiceMonitor custom resource can be found [here.](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + +### PodMonitor + +An example PodMonitor can be found [here.](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/example-app-pod-monitor.yaml) An example Prometheus resource that refers to it can be found [here.](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/prometheus-pod-monitor.yaml) + +### PrometheusRule + +For users who are familiar with Prometheus, a PrometheusRule contains the alerting and recording rules that you would normally place in a [Prometheus rule file](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/). + +For a more fine-grained application of PrometheusRules within your cluster, the ruleSelector field on a Prometheus resource allows you to select which PrometheusRules should be loaded onto Prometheus based on the labels attached to the PrometheusRules resources. + +An example PrometheusRule is on [this page.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/alerting.md) + +### Alertmanager Config + +For an example configuration, refer to [this section](./receivers.md#example-alertmanager-configs). diff --git a/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/helm-chart-options.md new file mode 100644 index 00000000000..112c41c9ea0 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -0,0 +1,72 @@ +--- +title: Helm Chart Options +--- + + + + + +## Configuring Resource Limits and Requests + +The resource requests and limits can be configured when installing `rancher-monitoring`. + +The default values are in the [values.yaml](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/values.yaml) in the `rancher-monitoring` Helm chart. + +The default values in the table below are the minimum required resource limits and requests. + +| Resource Name | Memory Limit | CPU Limit | Memory Request | CPU Request | +| ------------- | ------------ | ----------- | ---------------- | ------------------ | +| alertmanager | 500Mi | 1000m | 100Mi | 100m | +| grafana | 200Mi | 200m | 100Mi | 100m | +| kube-state-metrics subchart | 200Mi | 100m | 130Mi | 100m | +| prometheus-node-exporter subchart | 50Mi | 200m | 30Mi | 100m | +| prometheusOperator | 500Mi | 200m | 100Mi | 100m | +| prometheus | 2500Mi | 1000m | 1750Mi | 750m | +| **Total** | **3950Mi** | **2700m** | **2210Mi** | **1250m** | + +At least 50Gi storage is recommended. + + +## Trusted CA for Notifiers + +If you need to add a trusted CA to your notifier, follow these steps: + +1. Create the `cattle-monitoring-system` namespace. +1. Add your trusted CA secret to the `cattle-monitoring-system` namespace. +1. Deploy or upgrade the `rancher-monitoring` Helm chart. In the chart options, reference the secret in **Alerting > Additional Secrets**. + +**Result:** The default Alertmanager custom resource will have access to your trusted CA. + + +## Additional Scrape Configurations + +If the scrape configuration you want cannot be specified via a ServiceMonitor or PodMonitor at the moment, you can provide an `additionalScrapeConfigSecret` on deploying or upgrading `rancher-monitoring`. + +A [scrape_config section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) specifies a set of targets and parameters describing how to scrape them. In the general case, one scrape configuration specifies a single job. + +An example of where this might be used is with Istio. For more information, see [this section.](../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) + + +## Configuring Applications Packaged within Monitoring v2 + +We deploy kube-state-metrics and node-exporter with monitoring v2. Node exporter are deployed as DaemonSets. In the monitoring v2 helm chart, in the values.yaml, each of the things are deployed as sub charts. + +We also deploy grafana which is not managed by prometheus. + +If you look at what the helm chart is doing like in kube-state-metrics, there are plenty more values that you can set that aren’t exposed in the top level chart. + +But in the top level chart you can add values that override values that exist in the sub chart. + +### Increase the Replicas of Alertmanager + +As part of the chart deployment options, you can opt to increase the number of replicas of the Alertmanager deployed onto your cluster. The replicas can all be managed using the same underlying Alertmanager Config Secret. For more information on the Alertmanager Config Secret, refer to [this section.](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md#multiple-alertmanager-replicas) + +### Configuring the Namespace for a Persistent Grafana Dashboard + +To specify that you would like Grafana to watch for ConfigMaps across all namespaces, set this value in the `rancher-monitoring` Helm chart: + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +Note that the RBAC roles exposed by the Monitoring chart to add Grafana Dashboards are still restricted to giving permissions for users to add dashboards in the namespace defined in `grafana.dashboards.namespace`, which defaults to `cattle-dashboards`. \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/receivers.md b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/receivers.md new file mode 100644 index 00000000000..dc459f34378 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/receivers.md @@ -0,0 +1,303 @@ +--- +title: Receiver Configuration +--- + + + + + +The [Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret contains the configuration of an Alertmanager instance that sends out notifications based on alerts it receives from Prometheus. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [this section.](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-how-alertmanager-works) + +::: + + +## Creating Receivers in the Rancher UI + +:::note Prerequisites: + +- The monitoring application needs to be installed. +- If you configured monitoring with an existing Alertmanager Secret, it must have a format that is supported by Rancher's UI. Otherwise you will only be able to make changes based on modifying the Alertmanager Secret directly. Note: We are continuing to make enhancements to what kinds of Alertmanager Configurations we can support using the Routes and Receivers UI, so please [file an issue](https://github.com/rancher/rancher/issues/new) if you have a request for a feature enhancement. + +::: + +To create notification receivers in the Rancher UI, + + + + +1. Go to the cluster where you want to create receivers. Click **Monitoring -> Alerting -> AlertManagerConfigs**. +1. Click **Create**. +1. Click **Add Receiver**. +1. Enter a **Name** for the receiver. +1. Configure one or more providers for the receiver. For help filling out the forms, refer to the configuration options below. +1. Click **Create**. + + + + +1. Go to the cluster where you want to create receivers. Click **Monitoring** and click **Receiver**. +2. Enter a name for the receiver. +3. Configure one or more providers for the receiver. For help filling out the forms, refer to the configuration options below. +4. Click **Create**. + + + + +**Result:** Alerts can be configured to send notifications to the receiver(s). + +## Receiver Configuration + +The notification integrations are configured with the `receiver`, which is explained in the [Prometheus documentation.](https://prometheus.io/docs/alerting/latest/configuration/#receiver) + +### Native vs. Non-native Receivers + +By default, AlertManager provides native integration with some receivers, which are listed in [this section.](https://prometheus.io/docs/alerting/latest/configuration/#receiver) All natively supported receivers are configurable through the Rancher UI. + +For notification mechanisms not natively supported by AlertManager, integration is achieved using the [webhook receiver.](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config) A list of third-party drivers providing such integrations can be found [here.](https://prometheus.io/docs/operating/integrations/#alertmanager-webhook-receiver) Access to these drivers, and their associated integrations, is provided through the Alerting Drivers app. Once enabled, configuring non-native receivers can also be done through the Rancher UI. + +Currently the Rancher Alerting Drivers app provides access to the following integrations: +- Microsoft Teams, based on the [prom2teams](https://github.com/idealista/prom2teams) driver +- SMS, based on the [Sachet](https://github.com/messagebird/sachet) driver + +The following types of receivers can be configured in the Rancher UI: + +- Slack +- Email +- PagerDuty +- Opsgenie +- Webhook +- Custom +- Teams +- SMS + +The custom receiver option can be used to configure any receiver in YAML that cannot be configured by filling out the other forms in the Rancher UI. + +## Slack + +| Field | Type | Description | +|------|--------------|------| +| URL | String | Enter your Slack webhook URL. For instructions to create a Slack webhook, see the [Slack documentation.](https://get.slack.help/hc/en-us/articles/115005265063-Incoming-WebHooks-for-Slack) | +| Default Channel | String | Enter the name of the channel that you want to send alert notifications in the following format: `#`. | +| Proxy URL | String | Proxy for the webhook notifications. | +| Enable Send Resolved Alerts | Bool | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +## Email + +| Field | Type | Description | +|------|--------------|------| +| Default Recipient Address | String | The email address that will receive notifications. | +| Enable Send Resolved Alerts | Bool | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +SMTP options: + +| Field | Type | Description | +|------|--------------|------| +| Sender | String | Enter an email address available on your SMTP mail server that you want to send the notification from. | +| Host | String | Enter the IP address or hostname for your SMTP server. Example: `smtp.email.com`. | +| Use TLS | Bool | Use TLS for encryption. | +| Username | String | Enter a username to authenticate with the SMTP server. | +| Password | String | Enter a password to authenticate with the SMTP server. | + +## PagerDuty + +| Field | Type | Description | +|------|------|-------| +| Integration Type | String | `Events API v2` or `Prometheus`. | +| Default Integration Key | String | For instructions to get an integration key, see the [PagerDuty documentation.](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/) | +| Proxy URL | String | Proxy for the PagerDuty notifications. | +| Enable Send Resolved Alerts | Bool | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +## Opsgenie + +| Field | Description | +|------|-------------| +| API Key | For instructions to get an API key, refer to the [Opsgenie documentation.](https://docs.opsgenie.com/docs/api-key-management) | +| Proxy URL | Proxy for the Opsgenie notifications. | +| Enable Send Resolved Alerts | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +Opsgenie Responders: + +| Field | Type | Description | +|-------|------|--------| +| Type | String | Schedule, Team, User, or Escalation. For more information on alert responders, refer to the [Opsgenie documentation.](https://docs.opsgenie.com/docs/alert-recipients-and-teams) | +| Send To | String | Id, Name, or Username of the Opsgenie recipient. | + +## Webhook + +| Field | Description | +|-------|--------------| +| URL | Webhook URL for the app of your choice. | +| Proxy URL | Proxy for the webhook notification. | +| Enable Send Resolved Alerts | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + + + +## Custom + +The YAML provided here will be directly appended to your receiver within the Alertmanager Config Secret. + +## Teams + +### Enabling the Teams Receiver for Rancher Managed Clusters + +The Teams receiver is not a native receiver and must be enabled before it can be used. You can enable the Teams receiver for a Rancher managed cluster by going to the Apps page and installing the rancher-alerting-drivers app with the Teams option selected. + +1. In the Rancher UI, go to the cluster where you want to install rancher-alerting-drivers and click **Apps**. +1. Click the **Alerting Drivers** app. +1. Click the **Helm Deploy Options** tab. +1. Select the **Teams** option and click **Install**. +1. Take note of the namespace used as it will be required in a later step. + +### Configure the Teams Receiver + +The Teams receiver can be configured by updating its ConfigMap. For example, the following is a minimal Teams receiver configuration. + +```yaml +[Microsoft Teams] +teams-instance-1: https://your-teams-webhook-url +``` + +When configuration is complete, add the receiver using the steps in [this section](#creating-receivers-in-the-rancher-ui). + +Use the example below as the URL where: + +- `ns-1` is replaced with the namespace where the `rancher-alerting-drivers` app is installed + +```yaml +url: http://rancher-alerting-drivers-prom2teams.ns-1.svc:8089/v2/teams-instance-1 +``` + + + +## SMS + +### Enabling the SMS Receiver for Rancher Managed Clusters + +The SMS receiver is not a native receiver and must be enabled before it can be used. You can enable the SMS receiver for a Rancher managed cluster by going to the Apps page and installing the rancher-alerting-drivers app with the SMS option selected. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to install `rancher-alerting-drivers` and click **Explore**. +1. In the left navigation bar, click +1. Click the **Alerting Drivers** app. +1. Click the **Helm Deploy Options** tab +1. Select the **SMS** option and click **Install**. +1. Take note of the namespace used as it will be required in a later step. + +### Configure the SMS Receiver + +The SMS receiver can be configured by updating its ConfigMap. For example, the following is a minimal SMS receiver configuration. + +```yaml +providers: + telegram: + token: 'your-token-from-telegram' + +receivers: +- name: 'telegram-receiver-1' + provider: 'telegram' + to: + - '123456789' +``` + +When configuration is complete, add the receiver using the steps in [this section](#creating-receivers-in-the-rancher-ui). + +Use the example below as the name and URL, where: + +- the name assigned to the receiver, e.g. `telegram-receiver-1`, must match the name in the `receivers.name` field in the ConfigMap, e.g. `telegram-receiver-1` +- `ns-1` in the URL is replaced with the namespace where the `rancher-alerting-drivers` app is installed + +```yaml +name: telegram-receiver-1 +url http://rancher-alerting-drivers-sachet.ns-1.svc:9876/alert +``` + + + + +## Configuring Multiple Receivers + +By editing the forms in the Rancher UI, you can set up a Receiver resource with all the information Alertmanager needs to send alerts to your notification system. + +It is also possible to send alerts to multiple notification systems. One way is to configure the Receiver using custom YAML, in which case you can add the configuration for multiple notification systems, as long as you are sure that both systems should receive the same messages. + +You can also set up multiple receivers by using the `continue` option for a route, so that the alerts sent to a receiver continue being evaluated in the next level of the routing tree, which could contain another receiver. + + +## Example Alertmanager Configs + +### Slack +To set up notifications via Slack, the following Alertmanager Config YAML can be placed into the `alertmanager.yaml` key of the Alertmanager Config Secret, where the `api_url` should be updated to use your Webhook URL from Slack: + +```yaml +route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + receiver: 'slack-notifications' +receivers: +- name: 'slack-notifications' + slack_configs: + - send_resolved: true + text: '{{ template "slack.rancher.text" . }}' + api_url: +templates: +- /etc/alertmanager/config/*.tmpl +``` + +### PagerDuty +To set up notifications via PagerDuty, use the example below from the [PagerDuty documentation](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/) as a guideline. This example sets up a route that captures alerts for a database service and sends them to a receiver linked to a service that will directly notify the DBAs in PagerDuty, while all other alerts will be directed to a default receiver with a different PagerDuty integration key. + +The following Alertmanager Config YAML can be placed into the `alertmanager.yaml` key of the Alertmanager Config Secret. The `service_key` should be updated to use your PagerDuty integration key and can be found as per the "Integrating with Global Event Routing" section of the PagerDuty documentation. For the full list of configuration options, refer to the [Prometheus documentation](https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config). + +```yaml +route: + group_by: [cluster] + receiver: 'pagerduty-notifications' + group_interval: 5m + routes: + - match: + service: database + receiver: 'database-notifcations' + +receivers: +- name: 'pagerduty-notifications' + pagerduty_configs: + - service_key: 'primary-integration-key' + +- name: 'database-notifcations' + pagerduty_configs: + - service_key: 'database-integration-key' +``` + +## Example Route Config for CIS Scan Alerts + +While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. + +For example, the following example route configuration could be used with a Slack receiver named `test-cis`: + +```yaml +spec: + receiver: test-cis + group_by: +# - string + group_wait: 30s + group_interval: 30s + repeat_interval: 30s + match: + job: rancher-cis-scan +# key: string + match_re: + {} +# key: string +``` + +For more information on enabling alerting for `rancher-cis-benchmark`, see [this section.](../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) + +## Trusted CA for Notifiers + +If you need to add a trusted CA to your notifier, follow the steps in [this section.](helm-chart-options.md#trusted-ca-for-notifiers) diff --git a/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/routes.md b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/routes.md new file mode 100644 index 00000000000..f79d1e190c6 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/routes.md @@ -0,0 +1,68 @@ +--- +title: Route Configuration +--- + + + + + +The route configuration is the section of the Alertmanager custom resource that controls how the alerts fired by Prometheus are grouped and filtered before they reach the receiver. + +When a Route is changed, the Prometheus Operator regenerates the Alertmanager custom resource to reflect the changes. + +For more information about configuring routes, refer to the [official Alertmanager documentation.](https://www.prometheus.io/docs/alerting/latest/configuration/#route) + +:::note + +This section assumes familiarity with how monitoring components work together. For more information, see [this section.](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + + + +## Route Restrictions + +Alertmanager proxies alerts for Prometheus based on its receivers and a routing tree that filters alerts to certain receivers based on labels. + +Alerting drivers proxy alerts for Alertmanager to non-native receivers, such as Microsoft Teams and SMS. + +In the Rancher UI for configuring routes and receivers, you can configure routing trees with one root and then a depth of one more level, for a tree with a depth of two. But if you use a `continue` route when configuring Alertmanager directly, you can make the tree deeper. + +Each receiver is for one or more notification providers. So if you know that every alert for Slack should also go to PagerDuty, you can configure both in the same receiver. + +## Route Configuration + +### Note on Labels and Annotations + +Labels should be used for identifying information that can affect the routing of notifications. Identifying information about the alert could consist of a container name, or the name of the team that should be notified. + +Annotations should be used for information that does not affect who receives the alert, such as a runbook url or error message. + + +### Receiver +The route needs to refer to a [receiver](./receivers.md) that has already been configured. + +### Grouping + +| Field | Default | Description | +|-------|--------------|---------| +| Group By | N/a | List of labels to group by. All labels must be unique. Special label "..." (aggregate by all possible labels), if provided, must be the only element in the list. Accepts a list of strings. See the [upstream documentation](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#route) for details. | +| Group Wait | 30s | How long to wait to buffer alerts of the same group before sending initially. | +| Group Interval | 5m | How long to wait before sending an alert that has been added to a group of alerts for which an initial notification has already been sent. | +| Repeat Interval | 4h | How long to wait before re-sending a given alert that has already been sent. | + +### Matching + +The **Match** field refers to a set of equality matchers used to identify which alerts to send to a given Route based on labels defined on that alert. When you add key-value pairs to the Rancher UI, they correspond to the YAML in this format: + +```yaml +match: + [ : , ... ] +``` + +The **Match Regex** field refers to a set of regex-matchers used to identify which alerts to send to a given Route based on labels defined on that alert. When you add key-value pairs in the Rancher UI, they correspond to the YAML in this format: + +```yaml +match_re: + [ : , ... ] +``` diff --git a/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md new file mode 100644 index 00000000000..30fccb634c8 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -0,0 +1,37 @@ +--- +title: ServiceMonitor and PodMonitor Configuration +--- + + + + + +ServiceMonitors and PodMonitors are both pseudo-CRDs that map the scrape configuration of the Prometheus custom resource. + +These configuration objects declaratively specify the endpoints that Prometheus will scrape metrics from. + +ServiceMonitors are more commonly used than PodMonitors, and we recommend them for most use cases. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [this section.](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + +### ServiceMonitors + +This pseudo-CRD maps to a section of the Prometheus custom resource configuration. It declaratively specifies how groups of Kubernetes services should be monitored. + +When a ServiceMonitor is created, the Prometheus Operator updates the Prometheus scrape configuration to include the ServiceMonitor configuration. Then Prometheus begins scraping metrics from the endpoint defined in the ServiceMonitor. + +Any Services in your cluster that match the labels located within the ServiceMonitor `selector` field will be monitored based on the `endpoints` specified on the ServiceMonitor. For more information on what fields can be specified, please look at the [spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) provided by Prometheus Operator. + +For more information about how ServiceMonitors work, refer to the [Prometheus Operator documentation.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/running-exporters.md) + +### PodMonitors + +This pseudo-CRD maps to a section of the Prometheus custom resource configuration. It declaratively specifies how group of pods should be monitored. + +When a PodMonitor is created, the Prometheus Operator updates the Prometheus scrape configuration to include the PodMonitor configuration. Then Prometheus begins scraping metrics from the endpoint defined in the PodMonitor. + +Any Pods in your cluster that match the labels located within the PodMonitor `selector` field will be monitored based on the `podMetricsEndpoints` specified on the PodMonitor. For more information on what fields can be specified, please look at the [spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#podmonitorspec) provided by Prometheus Operator. diff --git a/versioned_docs/version-2.8/reference-guides/prometheus-federator/rbac.md b/versioned_docs/version-2.8/reference-guides/prometheus-federator/rbac.md new file mode 100644 index 00000000000..8b54ce9559f --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/prometheus-federator/rbac.md @@ -0,0 +1,35 @@ +--- +title: Role-Based Access Control +--- + + + + + +This section describes the expectations for Role-Based Access Control (RBAC) for Prometheus Federator. + +As described in the section on [namespaces](../../pages-for-subheaders/prometheus-federator.md#namespaces), Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: + +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under: + +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view` + +By default, these roleRefs will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +:::note + +For Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +::: + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: + +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, `rancher-project-monitoring`, the underlying chart deployed by Prometheus Federator, creates three default Roles per Project Release Namespace that provide `admin`, `edit`, and `view` users to permissions to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack to provide least privilege. However, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or create Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/rancher-cluster-tools.md b/versioned_docs/version-2.8/reference-guides/rancher-cluster-tools.md new file mode 100644 index 00000000000..41305db554b --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-cluster-tools.md @@ -0,0 +1,48 @@ +--- +title: Cluster Tools for Logging, Monitoring, and Visibility +--- + +Rancher contains a variety of tools that aren't included in Kubernetes to assist in your DevOps operations. Rancher can integrate with external services to help your clusters run more efficiently. Tools are divided into following categories: + + +## Logging + +Logging is helpful because it allows you to: + +- Capture and analyze the state of your cluster +- Look for trends in your environment +- Save your logs to a safe location outside of your cluster +- Stay informed of events like a container crashing, a pod eviction, or a node dying +- More easily debugg and troubleshoot problems + +Rancher can integrate with Elasticsearch, splunk, kafka, syslog, and fluentd. + +For more information, refer to the logging documentation [here.](../pages-for-subheaders/logging.md) +## Monitoring and Alerts + +Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with [Prometheus](https://prometheus.io/), a leading open-source monitoring solution. + +After monitoring is enabled, you can set up alerts and notifiers that provide the mechanism to receive them. + +Notifiers are services that inform you of alert events. You can configure notifiers to send alert notifications to staff best suited to take corrective action. Notifications can be sent with Slack, email, PagerDuty, WeChat, and webhooks. + +Alerts are rules that trigger those notifications. Before you can receive alerts, you must configure one or more notifier in Rancher. The scope for alerts can be set at either the cluster or project level. + +For more information, refer to the monitoring documentation [here.](../pages-for-subheaders/monitoring-and-alerting.md) + +## Istio + +[Istio](https://istio.io/) is an open-source tool that makes it easier for DevOps teams to observe, control, troubleshoot, and secure the traffic within a complex network of microservices. + +Rancher's integration with Istio was improved in Rancher v2.5. + +For more information, refer to the Istio documentation [here.](../pages-for-subheaders/istio.md) +## OPA Gatekeeper + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) is an open-source project that provides integration between OPA and Kubernetes to provide policy control via admission controller webhooks. For details on how to enable Gatekeeper in Rancher, refer to the [OPA Gatekeeper section.](../integrations-in-rancher/opa-gatekeeper.md) + +## CIS Scans + +Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. + +For more information, refer to the CIS scan documentation [here.](../pages-for-subheaders/cis-scan-guides.md) \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/architecture-recommendations.md new file mode 100644 index 00000000000..1ece990ff8c --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -0,0 +1,110 @@ +--- +title: Architecture Recommendations +--- + + + + + +If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the node running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) + +## Separation of Rancher and User Clusters + +A user cluster is a downstream Kubernetes cluster that runs your apps and services. + +If you have a Docker installation of Rancher, the node running the Rancher server should be separate from your downstream clusters. + +If Rancher is intended to manage downstream Kubernetes clusters, the Kubernetes cluster that the Rancher server runs on should also be separate from the downstream user clusters. + +![Separation of Rancher Server from User Clusters](/img/rancher-architecture-separation-of-rancher-server.svg) + +## Why HA is Better for Rancher in Production + +We recommend installing the Rancher server on a high-availability Kubernetes cluster, primarily because it protects the Rancher server data. In a high-availability installation, a load balancer serves as the single point of contact for clients, distributing network traffic across multiple servers in the cluster and helping to prevent any one server from becoming a point of failure. + +We don't recommend installing Rancher in a single Docker container, because if the node goes down, there is no copy of the cluster data available on other nodes and you could lose the data on your Rancher server. + +### K3s Kubernetes Cluster Installations + +One option for the underlying Kubernetes cluster is to use K3s Kubernetes. K3s is Rancher's CNCF certified Kubernetes distribution. It is easy to install and uses half the memory of Kubernetes, all in a binary of less than 100 MB. Another advantage of K3s is that it allows an external datastore to hold the cluster data, allowing the K3s server nodes to be treated as ephemeral. + +
Architecture of a K3s Kubernetes Cluster Running the Rancher Management Server
+ +![Architecture of a K3s Kubernetes Cluster Running the Rancher Management Server](/img/k3s-server-storage.svg) + +### RKE Kubernetes Cluster Installations + +In an RKE installation, the cluster data is replicated on each of three etcd nodes in the cluster, providing redundancy and data duplication in case one of the nodes fails. + +
Architecture of an RKE Kubernetes Cluster Running the Rancher Management Server
+ +![Architecture of an RKE Kubernetes cluster running the Rancher management server](/img/rke-server-storage.svg) + +## Recommended Load Balancer Configuration for Kubernetes Installations + +We recommend the following configurations for the load balancer and Ingress controllers: + +* The DNS for Rancher should resolve to a Layer 4 load balancer (TCP). +* The Load Balancer should forward port TCP/80 and TCP/443 to all 3 nodes in the Kubernetes cluster. +* The Ingress controller will redirect HTTP to HTTPS and terminate SSL/TLS on port TCP/443. +* The Ingress controller will forward traffic to port TCP/80 on the pod in the Rancher deployment. + +
Rancher installed on a Kubernetes cluster with layer 4 load balancer, depicting SSL termination at Ingress controllers
+ +![Rancher HA](/img/ha/rancher2ha.svg) + +## Environment for Kubernetes Installations + +It is strongly recommended to install Rancher on a Kubernetes cluster on hosted infrastructure such as Amazon's EC2 or Google Compute Engine. + +For the best performance and greater security, we recommend a dedicated Kubernetes cluster for the Rancher management server. Running user workloads on this cluster is not advised. After deploying Rancher, you can [create or import clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) for running your workloads. + +## Recommended Node Roles for Kubernetes Installations + +The below recommendations apply when Rancher is installed on a K3s Kubernetes cluster or an RKE Kubernetes cluster. + +### K3s Cluster Roles + +In K3s clusters, there are two types of nodes: server nodes and agent nodes. Both servers and agents can have workloads scheduled on them. Server nodes run the Kubernetes master. + +For the cluster running the Rancher management server, we recommend using two server nodes. Agent nodes are not required. + +### RKE Cluster Roles + +If Rancher is installed on an RKE Kubernetes cluster, the cluster should have three nodes, and each node should have all three Kubernetes roles: etcd, controlplane, and worker. + +### Contrasting RKE Cluster Architecture for Rancher Server and for Downstream Kubernetes Clusters + +Our recommendation for RKE node roles on the Rancher server cluster contrasts with our recommendations for the downstream user clusters that run your apps and services. + +Rancher uses RKE as a library when provisioning downstream Kubernetes clusters. Note: The capability to provision downstream K3s clusters will be added in a future version of Rancher. + +For downstream Kubernetes clusters, we recommend that each node in a user cluster should have a single role for stability and scalability. + +![Kubernetes Roles for Nodes in Rancher Server Cluster vs. User Clusters](/img/rancher-architecture-node-roles.svg) + +RKE only requires at least one node with each role and does not require nodes to be restricted to one role. However, for the clusters that run your apps, we recommend separate roles for each node so that workloads on worker nodes don't interfere with the Kubernetes master or cluster data as your services scale. + +We recommend that downstream user clusters should have at least: + +- **Three nodes with only the etcd role** to maintain a quorum if one node is lost, making the state of your cluster highly available +- **Two nodes with only the controlplane role** to make the master component highly available +- **One or more nodes with only the worker role** to run the Kubernetes node components, as well as the workloads for your apps and services + +With that said, it is safe to use all three roles on three nodes when setting up the Rancher server because: + +* It allows one `etcd` node failure. +* It maintains multiple instances of the master components by having multiple `controlplane` nodes. +* No other workloads than Rancher itself should be created on this cluster. + +Because no additional workloads will be deployed on the Rancher server cluster, in most cases it is not necessary to use the same architecture that we recommend for the scalability and reliability of downstream clusters. + +For more best practices for downstream clusters, refer to the [production checklist](../../pages-for-subheaders/checklist-for-production-ready-clusters.md) or our [best practices guide.](../../pages-for-subheaders/best-practices.md) + +## Architecture for an Authorized Cluster Endpoint (ACE) + +If you are using an [authorized cluster endpoint (ACE),](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) we recommend creating an FQDN pointing to a load balancer which balances traffic across your nodes with the `controlplane` role. + +If you are using private CA signed certificates on the load balancer, you have to supply the CA certificate, which will be included in the generated kubeconfig file to validate the certificate chain. See the documentation on [kubeconfig files](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) and [API keys](../user-settings/api-keys.md#creating-an-api-key) for more information. + +ACE support is available for registered RKE2 and K3s clusters. To view the manual steps to perform on the downstream cluster to enable the ACE, click [here](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters). diff --git a/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md new file mode 100644 index 00000000000..586b55b0db8 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -0,0 +1,218 @@ +--- +title: Communicating with Downstream User Clusters +--- + + + + + +This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. + +The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. + +
Communicating with Downstream Clusters
+ +![Rancher Components](/img/rancher-architecture-cluster-controller.svg) + +The following descriptions correspond to the numbers in the diagram above: + +1. [The Authentication Proxy](#1-the-authentication-proxy) +2. [Cluster Controllers and Cluster Agents](#2-cluster-controllers-and-cluster-agents) +3. [Node Agents](#3-node-agents) +4. [Authorized Cluster Endpoint](#4-authorized-cluster-endpoint) + +### 1. The Authentication Proxy + +In this diagram, a user named Bob wants to see all pods running on a downstream user cluster called User Cluster 1. From within Rancher, he can run a `kubectl` command to see +the pods. Bob is authenticated through Rancher's authentication proxy. + +The authentication proxy forwards all Kubernetes API calls to downstream clusters. It integrates with authentication services like local authentication, Active Directory, and GitHub. On every Kubernetes API call, the authentication proxy authenticates the caller and sets the proper Kubernetes impersonation headers before forwarding the call to Kubernetes masters. + +Rancher communicates with Kubernetes clusters using a [service account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/). Each user account in Rancher correlates with an equivalent service account in the downstream cluster. Rancher uses the service account to [impersonate](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation) the user, which provides all the permissions the user is intended to have. + +By default, Rancher generates a [kubeconfig file](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) that contains credentials for proxying through the Rancher server to connect to the Kubernetes API server on a downstream user cluster. The kubeconfig file (`kube_config_rancher-cluster.yml`) contains full access to the cluster. + +### 2. Cluster Controllers and Cluster Agents + +Each downstream user cluster has a cluster agent, which opens a tunnel to the corresponding cluster controller within the Rancher server. + +There is one cluster controller and one cluster agent for each downstream cluster. Each cluster controller: + +- Watches for resource changes in the downstream cluster +- Brings the current state of the downstream cluster to the desired state +- Configures access control policies to clusters and projects +- Provisions clusters by calling the required Docker machine drivers and Kubernetes engines, such as RKE and GKE + +By default, to enable Rancher to communicate with a downstream cluster, the cluster controller connects to the cluster agent. If the cluster agent is not available, the cluster controller can connect to a [node agent](#3-node-agents) instead. + +The cluster agent, also called `cattle-cluster-agent`, is a component that runs in a downstream user cluster. It performs the following tasks: + +- Connects to the Kubernetes API of Rancher-launched Kubernetes clusters +- Manages workloads, pod creation and deployment within each cluster +- Applies the roles and bindings defined in each cluster's global policies +- Communicates between the cluster and Rancher server (through a tunnel to the cluster controller) about events, stats, node info, and health + +### 3. Node Agents + +If the cluster agent (also called `cattle-cluster-agent`) is not available, one of the node agents creates a tunnel to the cluster controller to communicate with Rancher. + +The `cattle-node-agent` is deployed using a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) resource to make sure it runs on every node in a Rancher-launched Kubernetes cluster. It is used to interact with the nodes when performing cluster operations. Examples of cluster operations include upgrading the Kubernetes version and creating or restoring etcd snapshots. + +### 4. Authorized Cluster Endpoint + +An authorized cluster endpoint (ACE) allows users to connect to the Kubernetes API server of a downstream cluster without having to route their requests through the Rancher authentication proxy. + +> ACE is available on RKE, RKE2, and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. + + +There are two main reasons why a user might need the authorized cluster endpoint: + +- To access a downstream user cluster while Rancher is down +- To reduce latency in situations where the Rancher server and downstream cluster are separated by a long distance + +The `kube-api-auth` microservice is deployed to provide the user authentication functionality for the authorized cluster endpoint. When you access the user cluster using `kubectl`, the cluster's Kubernetes API server authenticates you by using the `kube-api-auth` service as a webhook. + +Like the authorized cluster endpoint, the `kube-api-auth` authentication service is also only available for Rancher-launched Kubernetes clusters. + +> **Example scenario:** Let's say that the Rancher server is located in the United States, and User Cluster 1 is located in Australia. A user, Alice, also lives in Australia. Alice can manipulate resources in User Cluster 1 by using the Rancher UI, but her requests will have to be sent from Australia to the Rancher server in the United States, then be proxied back to Australia, where the downstream user cluster is. The geographical distance may cause significant latency, which Alice can reduce by using the authorized cluster endpoint. + +With this endpoint enabled for the downstream cluster, Rancher generates an extra Kubernetes context in the kubeconfig file in order to connect directly to the cluster. This file has the credentials for `kubectl` and `helm`. + +You will need to use a context defined in this kubeconfig file to access the cluster if Rancher goes down. Therefore, we recommend exporting the kubeconfig file so that if Rancher goes down, you can still use the credentials in the file to access your cluster. For more information, refer to the section on accessing your cluster with [kubectl and the kubeconfig file.](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) + +## Impersonation + +Users technically exist only on the upstream cluster. Rancher creates [RoleBindings and ClusterRoleBindings](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding) that refer to Rancher users, even though there is [no actual User resource](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes) on the downstream cluster. + +When users interact with a downstream cluster through the authentication proxy, there needs to be some entity downstream to serve as the actor for those requests. Rancher creates service accounts to be that entity. Each service account is only granted one permission, which is to **impersonate** the user they belong to. If there was only one service account that could impersonate any user, then it would be possible for a malicious user to corrupt that account and escalate their privileges by impersonating another user. This issue was the basis for a [CVE](https://github.com/rancher/rancher/security/advisories/GHSA-pvxj-25m6-7vqr). + +### Impersonation Troubleshooting + +On the downstream cluster, five resources handle impersonation: + +- namespace: `cattle-impersonation-system` +- service account: `cattle-impersonation-system/cattle-impersonation-` +- account token secret: `cattle-impersonation-system/cattle-impersonation--token-` +- cluster role: `cattle-impersonation-` +- cluster role binding: `cattle-impersonation-` + +In this example of a typical impersonation cluster role, the system is configured to use `github` as the auth provider: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: "2021-10-06T18:20:13Z" + labels: + authz.cluster.cattle.io/impersonator: "true" + cattle.io/creator: norman + name: cattle-impersonation-user-abcde + resourceVersion: "3528" + uid: a7478731-72a0-4343-b09f-c3bf12552d77 +rules: +# allowed to impersonate user user-abcde +- apiGroups: + - "" + resourceNames: + - user-abcde + resources: + - users + verbs: + - impersonate +# allowed to impersonate listed groups +- apiGroups: + - "" + resourceNames: + - github_team://123 # group from GitHub auth provider + - system:authenticated # automatic group from Kubernetes + - system:cattle:authenticated # automatic group from Rancher + resources: + - groups + verbs: + - impersonate +# allowed to impersonate principal ID github_user://098 +- apiGroups: + - authentication.k8s.io + resourceNames: + - github_user://098 # principal ID from GitHub auth provider + resources: + - userextras/principalid + verbs: + - impersonate +# allowed to impersonate username example +- apiGroups: + - authentication.k8s.io + resourceNames: + - example # username from GitHub auth provider + resources: + - userextras/username + verbs: + - impersonate +``` + +When you troubleshoot impersonation issues, check whether these resources exist for the user, and whether the rules in the cluster role look similar to the above. For example: + +```bash +kubectl --namespace cattle-impersonation-system get serviceaccount cattle-impersonation- +kubectl --namespace cattle-impersonation-system get secret cattle-impersonation--token- +kubectl get clusterrole cattle-impersonation- --output yaml +kubectl get clusterrolebinding cattle-impersonation- +``` + +If you see an error related to "impersonation" in the UI, pay close attention to the *end* of the error message, which should indicate the real reason that the request failed. + +## Important Files + +The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_rancher-cluster.yml`: The Kubeconfig file for the cluster, this file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. +- `rancher-cluster.rkestate`: The Kubernetes cluster state file. This file contains credentials for full access to the cluster. Note: This state file is only created when using RKE v0.2.0 or higher. + +> **Note:** The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +For more information on connecting to a cluster without the Rancher authentication proxy and other configuration options, refer to the [kubeconfig file](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) documentation. + +## Tools for Provisioning Kubernetes Clusters + +The tools that Rancher uses to provision downstream user clusters depends on the type of cluster that is being provisioned. + +### Rancher Launched Kubernetes for Nodes Hosted in an Infrastructure Provider + +Rancher can dynamically provision nodes in a provider such as Amazon EC2, DigitalOcean, Azure, or vSphere, then install Kubernetes on them. + +Rancher provisions this type of cluster using [RKE](https://github.com/rancher/rke) and [docker-machine.](https://github.com/rancher/machine) + +### Rancher Launched Kubernetes for Custom Nodes + +When setting up this type of cluster, Rancher installs Kubernetes on existing nodes, which creates a custom cluster. + +Rancher provisions this type of cluster using [RKE.](https://github.com/rancher/rke) + +### Hosted Kubernetes Providers + +When setting up this type of cluster, Kubernetes is installed by providers such as Google Kubernetes Engine, Amazon Elastic Container Service for Kubernetes, or Azure Kubernetes Service. + +Rancher provisions this type of cluster using [kontainer-engine.](https://github.com/rancher/kontainer-engine) + +### Imported Kubernetes Clusters + +In this type of cluster, Rancher connects to a Kubernetes cluster that has already been set up. Therefore, Rancher does not provision Kubernetes, but only sets up the Rancher agents to communicate with the cluster. + +## Rancher Server Components and Source Code + +This diagram shows each component that the Rancher server is composed of: + +![Rancher Components](/img/rancher-architecture-rancher-components.svg) + +The GitHub repositories for Rancher can be found at the following links: + +- [Main Rancher server repository](https://github.com/rancher/rancher) +- [Rancher UI](https://github.com/rancher/ui) +- [Rancher API UI](https://github.com/rancher/api-ui) +- [Norman,](https://github.com/rancher/norman) Rancher's API framework +- [Types](https://github.com/rancher/types) +- [Rancher CLI](https://github.com/rancher/cli) +- [Catalog applications](https://github.com/rancher/helm) + +This is a partial list of the most important Rancher repositories. For more details about Rancher source code, refer to the section on [contributing to Rancher.](../../contribute-to-rancher.md#rancher-repositories) To see all libraries and projects used in Rancher, see the [`go.mod` file](https://github.com/rancher/rancher/blob/master/go.mod) in the `rancher/rancher` repository. \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/rancher-server-and-components.md new file mode 100644 index 00000000000..e9fec332622 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -0,0 +1,39 @@ +--- +title: Rancher Server and Components +--- + + + + + +The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. + +The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). + +For the best performance and security, we recommend a dedicated Kubernetes cluster for the Rancher management server. Running user workloads on this cluster is not advised. After deploying Rancher, you can [create or import clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) for running your workloads. + +The diagram below shows how users can manipulate both [Rancher-launched Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters and [hosted Kubernetes](../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md) clusters through Rancher's authentication proxy: + +
Managing Kubernetes Clusters through Rancher's Authentication Proxy
+ +![Architecture](/img/rancher-architecture-rancher-api-server.svg) + +You can install Rancher on a single node, or on a high-availability Kubernetes cluster. + +A high-availability Kubernetes installation is recommended for production. + +A Docker installation of Rancher is recommended only for development and testing purposes. The ability to migrate Rancher to a high-availability cluster depends on the Rancher version. + +To see deployed resources, run the following command: + +```bash +kubectl get all -n +``` +If you have administrator privileges, you can also see these resources listed in the Rancher UI: + +1. Click **☰** and select a cluster. +1. Select **More Resources** from the side navigation menu to view deployed resources by type. +1. Select **Cluster** > **Projects/Namespaces** from the side navigation menu, then select a namespace, to view deployed resources by namespace. +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md). + +The Rancher server, regardless of the installation method, should always run on nodes that are separate from the downstream user clusters that it manages. If Rancher is installed on a high-availability Kubernetes cluster, it should run on a separate cluster from the cluster(s) it manages. diff --git a/versioned_docs/version-2.8/reference-guides/rancher-project-tools.md b/versioned_docs/version-2.8/reference-guides/rancher-project-tools.md new file mode 100644 index 00000000000..567cc5cf408 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-project-tools.md @@ -0,0 +1,32 @@ +--- +title: Project Tools for Logging, Monitoring, and Visibility +--- + +Rancher contains a variety of tools that aren't included in Kubernetes to assist in your DevOps operations. Rancher can integrate with external services to help your clusters run more efficiently. + + +## Notifiers and Alerts + +Notifiers and alerts are two features that work together to inform you of events in the Rancher system. Before they can be enabled, the monitoring application must be installed. + +Notifiers are services that inform you of alert events. You can configure notifiers to send alert notifications to staff best suited to take corrective action. Notifications can be sent with Slack, email, PagerDuty, WeChat, and webhooks. + +Alerts are rules that trigger those notifications. Before you can receive alerts, you must configure one or more notifier in Rancher. The scope for alerts can be set at either the cluster or project level. + +## Logging + +Logging is helpful because it allows you to: + +- Capture and analyze the state of your cluster +- Look for trends in your environment +- Save your logs to a safe location outside of your cluster +- Stay informed of events like a container crashing, a pod eviction, or a node dying +- More easily debugg and troubleshoot problems + +Rancher can integrate with Elasticsearch, splunk, kafka, syslog, and fluentd. + +For details, refer to the [logging section.](../pages-for-subheaders/logging.md) + +## Monitoring + +Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with [Prometheus](https://prometheus.io/), a leading open-source monitoring solution. For details, refer to the [monitoring section.](../pages-for-subheaders/monitoring-and-alerting.md) diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..d0ddba7d1e7 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3152 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 518 1 0 Feb26 ? 00:03:06 /usr/bin/containerd root 794 1 0 Feb26 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 94093 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7927fe0998264b8fa49789e8e2793a21ea8f22001cb580de46e1fcc01b1d1455 -address /run/k3s/containerd/containerd.sock root 94107 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ccec5bdaf56b4aad4018e8fefabfa103122905ecf4f908dbddaf9fbb985bc5c -address /run/k3s/containerd/containerd.sock root 94188 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d0f840cefd5b5ba804b2795420963e2c43a0c8b7b6b2c796e859c57ba0d38b3f -address /run/k3s/containerd/containerd.sock root 95349 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3849ee071d3f5a8e8a63ee1c4d590f88e0d21bc5fc25e20bbecb1e104b2835d0 -address /run/k3s/containerd/containerd.sock root 95421 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id a608e22cc1520272c91a2aca6a029e662a5d13defc7cf67de6a59594b859c550 -address /run/k3s/containerd/containerd.sock root 96569 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id f46f18b427a026156a41a54daad5c55a7d04bee1d112ae32e9c3d07c8cfc5a4e -address /run/k3s/containerd/containerd.sock root 97820 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id aa5731bfdbd6ff9c87043ccd4e30adc1c384eb8653b97169629598852931375d -address /run/k3s/containerd/containerd.sock root 97868 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 99f90b82253554453d0d1e57a23e770f000cf8972ca751387fd5682e9e1607f2 -address /run/k3s/containerd/containerd.sock root 100124 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 632f8fcf8174c12048354376e1f22e87ac95baa26a46e77f4cb8c1549f00fc4f -address /run/k3s/containerd/containerd.sock root 101049 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 05cc13a10b18601034393b12dcb0b140799cef2f3c7f712de671054cb65a4cff -address /run/k3s/containerd/containerd.sock root 105484 105441 2 21:41 ? 00:00:10 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 107618 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d7baeb7e8d463b1afd831ca4367d4c552c8e87afc29f7cae7cb5bf987c2e4b1f -address /run/k3s/containerd/containerd.sock root 107751 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 8fd7e8824474bb98e13d39dca0e130444b7b6fb917b7dc5e662c5a5f669fe584 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'644' is equal to '644' OR '640' is present OR '600' is present OR '444' is present OR '440' is present OR '400' is present OR '000' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..1d348069a2a --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3208 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 539 1 0 Sep11 ? 00:00:09 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 45213 45195 3 Sep11 ? 00:45:14 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 47188 1 0 Sep11 ? 00:01:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 02b8fdd94b7628d575ace92f337dcc93202151d07f33858341cfcb0178fea586 -address /run/k3s/containerd/containerd.sock root 47235 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id eb0f03c4bc1125ae34e5de77c7eaeb9c2c1e46bec0bb106b99281df7353f9ded -address /run/k3s/containerd/containerd.sock root 47948 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5eece7110bd3876414f890ed1b7158eb91f60d1f14450a9f8c77a0fc62b73c4a -address /run/k3s/containerd/containerd.sock root 48047 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ef831f4bb8830a0a7b46a1e8be58197be00846ea0caa094014ea7ce3adf008e -address /run/k3s/containerd/containerd.sock root 48220 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id f8d5d3791ad2b1a8534476aa2fbdf8da1e4f0b9fdcdd26b1ae161142458a5336 -address /run/k3s/containerd/containerd.sock root 48878 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 41b93a6581e272b0852257b3d7c44ab4cadcdf465a3579a8b29b493ea75feaf7 -address /run/k3s/containerd/containerd.sock root 49870 1 0 Sep11 ? 00:00:30 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id be329cdccda4049f048c759a2b1915cd72526473ff7768e895bcfc153a66c125 -address /run/k3s/containerd/containerd.sock root 50271 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id cceb5d26eba7d7d3c474a2dc842b72495689f4e40055505ea2598e4ac6849d40 -address /run/k3s/containerd/containerd.sock root 50571 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3a26e691e0d0186165d523bef22b0af71f175c39df82845b32ce7b4e3d652d4a -address /run/k3s/containerd/containerd.sock root 96693 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9ed77e6e58410f1295f730ba4a8f8daf7aff8f4e896e24219a8fa710729c2287 -address /run/k3s/containerd/containerd.sock root 97726 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 1be4fe50e6bb29d4549f8b03102b5dc15b7669fe56b32beae3f701cf6d2e58cc -address /run/k3s/containerd/containerd.sock root 97923 1 1 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7c65c47376f36a8980a5bebad7ffdc1abe1853821cf6c4a1d45f1b6be3452062 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'600' is equal to '600' +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--event-qps' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..e7373288b35 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,3215 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 +Not Applicable. + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root +Not Applicable. + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd +Not Applicable. + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/k3s/server/tls + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... +Permissive - Enabling Pod Security Policy can cause applications to unexpectedly fail. + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log +Permissive. + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 +Permissive. + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 +Permissive. + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 +Permissive. + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s +Permissive. + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Not Applicable. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is present OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'RotateKubeletServerCertificate' is present OR 'RotateKubeletServerCertificate' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..9c6d1369a90 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3087 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/!(*key).pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 5 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 2 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): replicationcontrollers is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicationcontrollers" in API group "" in the namespace "default" Error from server (Forbidden): services is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "services" in API group "" in the namespace "default" Error from server (Forbidden): daemonsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "daemonsets" in API group "apps" in the namespace "default" Error from server (Forbidden): deployments.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "deployments" in API group "apps" in the namespace "default" Error from server (Forbidden): replicasets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicasets" in API group "apps" in the namespace "default" Error from server (Forbidden): statefulsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "statefulsets" in API group "apps" in the namespace "default" Error from server (Forbidden): horizontalpodautoscalers.autoscaling is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "horizontalpodautoscalers" in API group "autoscaling" in the namespace "default" Error from server (Forbidden): cronjobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "cronjobs" in API group "batch" in the namespace "default" Error from server (Forbidden): jobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "jobs" in API group "batch" in the namespace "default" true +``` diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..5f3c20fb37f --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3047 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 5 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 4 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:10 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..9fe9ca2045f --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2863 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is greater or equal to 0 OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..3099612cacb --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3200 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"zF6yWmUNDvLZW5QnHG580wVBYOwq9S+VXJ3G4JjDaEk="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..ba80c0a516a --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3202 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1017917 2489 83 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz root 1017945 2489 75 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/livez +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"akmMjUAq94YvsUftJpiA3b+9SClu0ESPBeckAI7KZBY="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Automated) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r ".spec.runAsUser.rule" || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r ".spec.runAsUser.rule" +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is equal to 'MustRunAsNonRoot' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is equal to 'ALL' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..b7d4a36ea4b --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2967 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then find /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml -name '*etcd*' | xargs stat -c permissions=%a; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1018768 2419 99 16:17 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 +Not Applicable. + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= +Not Applicable. + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" +Not Applicable. + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= +Not Applicable. + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true +Not Applicable. + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not applicable. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present OR '{.eventRecordQPS}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.podPidsLimit}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/kubernetes-security-best-practices.md b/versioned_docs/version-2.8/reference-guides/rancher-security/kubernetes-security-best-practices.md new file mode 100644 index 00000000000..ace8bd95fab --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/kubernetes-security-best-practices.md @@ -0,0 +1,15 @@ +--- +title: Kubernetes Security Best Practices +--- + + + + + +### Restricting cloud metadata API access + +Cloud providers such as AWS, Azure, DigitalOcean or GCP often expose metadata services locally to instances. By default, this endpoint is accessible by pods running on a cloud instance, including pods in hosted Kubernetes providers such as EKS, AKS, DigitalOcean Kubernetes or GKE, and can contain cloud credentials for that node, provisioning data such as kubelet credentials, or other sensitive data. To mitigate this risk when running on a cloud platform, follow the [Kubernetes security recommendations](https://kubernetes.io/docs/tasks/administer-cluster/securing-a-cluster/#restricting-cloud-metadata-api-access): limit permissions given to instance credentials, use network policies to restrict pod access to the metadata API, and avoid using provisioning data to deliver secrets. + +It is advised to consult your cloud provider's security best practices for further recommendations and specific details on how to restrict access to cloud instance metadata API. + +Further references: MITRE ATT&CK knowledge base on - [Unsecured Credentials: Cloud Instance Metadata API](https://attack.mitre.org/techniques/T1552/005/). diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/psa-restricted-exemptions.md b/versioned_docs/version-2.8/reference-guides/rancher-security/psa-restricted-exemptions.md new file mode 100644 index 00000000000..0f35551cf31 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/psa-restricted-exemptions.md @@ -0,0 +1,66 @@ +--- +title: Sample PodSecurityConfiguration +--- + + + + + +The following PodSecurityConfiguration contains the required Rancher namespace exemptions for a `rancher-restricted` cluster to run properly. + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-elemental-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-resources-system, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator] +``` diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/rancher-webhook-hardening.md b/versioned_docs/version-2.8/reference-guides/rancher-security/rancher-webhook-hardening.md new file mode 100644 index 00000000000..1f771dfdfe5 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/rancher-webhook-hardening.md @@ -0,0 +1,133 @@ +--- +title: Hardening the Rancher Webhook +--- + +Rancher Webhook is an important component within Rancher, playing a role in enforcing security requirements for Rancher and its workloads. To decrease its attack surface, access to it should be limited to the only valid caller it has: the Kubernetes API server. This can be done by using network policies and authentication independently or in conjunction with each other to harden the webhook against attacks. + +## Block External Traffic Using Network Policies + +The webhook is only expected to accept requests from the Kubernetes API server. By default, however, the webhook can accept traffic from any source. If you are using a CNI that supports Network Policies, you can create a policy that blocks traffic that doesn't originate from the API server. + +The built-in NetworkPolicy resource in Kubernetes can't block or admit traffic from the cluster hosts, and the `kube-apiserver` process is always running on the host network. Therefore, you must use the advanced network policy resources from the CNI in use. Examples for Calico and Cilium follow. Consult the documentation for your CNI for more details. + +### Calico + +Use the NetworkPolicy resource in the `crd.projectcalico.org/v1` API group. Use the selector `app == 'rancher-webhook'` to create a rule for the webhook, and set the CIDR of the control plane hosts as the ingress source: + +```yaml +apiVersion: crd.projectcalico.org/v1 +kind: NetworkPolicy +metadata: + name: allow-k8s + namespace: cattle-system +spec: + selector: app == 'rancher-webhook' + types: + - Ingress + ingress: + - action: Allow + protocol: TCP + source: + nets: + - 192.168.42.0/24 # CIDR of the control plane host. May list more than 1 if the hosts are in different subnets. + destination: + selector: + app == 'rancher-webhook' +``` + +### Cilium + +Use the CiliumNetworkPolicy resource in the `cilium.io/v2` API group. Add the `host` and `remote-node` keys to the `fromEntities` ingress rule. This blocks in-cluster and external traffic while allowing traffic from the hosts. + +```yaml +apiVersion: "cilium.io/v2" +kind: CiliumNetworkPolicy +metadata: + name: allow-k8s + namespace: cattle-system +spec: + endpointSelector: + matchLabels: + app: rancher-webhook + ingress: + - fromEntities: + - host + - remote-node +``` + +## Require the Kubernetes API Server to Authenticate to the Webhook + +The webhook should only accept requests from the Kubernetes API server. By default, the webhook doesn't require clients to authenticate to it. It will accept any request. You can configure the webhook to require credentials so that only the API server can access it. More information can be found in the [Kubernetes documentation](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#authenticate-apiservers). + +1. Configure the API server to present a client certificate to the webhook, pointing to an AdmissionConfiguration file to configure the ValidatingAdmissionWebhook and MutatingAdmissionWebhook plugins: + + ```yaml + # /etc/rancher/admission/admission.yaml + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: ValidatingAdmissionWebhook + configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: WebhookAdmissionConfiguration + kubeConfigFile: "/etc/rancher/admission/kubeconfig" + - name: MutatingAdmissionWebhook + configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: WebhookAdmissionConfiguration + kubeConfigFile: "/etc/rancher/admission/kubeconfig" + ``` + + This is also the same config file where other admission plugins are configured, such as PodSecurity. If your distro or your setup uses additional admission plugins, configure those as well. For example, add [RKE2's PodSecurity configuration](https://docs.rke2.io/security/pod_security_standards) to this file. + +2. Create the kubeconfig file that the admission plugins refer to. Rancher Webhook only supports client certificate authentication, so generate a TLS key pair, and set the kubeconfig to use either `client-certificate` and `client-key` or `client-certificate-data` and `client-key-data`. For example: + + ```yaml + # /etc/rancher/admission/kubeconfig + apiVersion: v1 + kind: Config + users: + - name: 'rancher-webhook.cattle-system.svc' + user: + client-certificate: /path/to/client/cert + client-key: /path/to/client/key + ``` + +3. Start the kube-apiserver binary with the flag `--admission-control-config-file` pointing to your AdmissionConfiguration file. The way to do this varies by distro, and it isn't supported universally, such as in hosted Kubernetes providers. Consult the documentation for your Kubernetes distribution. + + For RKE2, `rke2-server` can be started with a config file like so: + + ```yaml + # /etc/rancher/rke2/config.yaml + kube-apiserver-arg: + - admission-control-config-file=/etc/rancher/admission/admission.yaml + kube-apiserver-extra-mount: + - /etc/rancher/admission:/etc/rancher/admission:ro + ``` + + :::danger + Some distros set this flag by default. If your distro provisions its own AdmissionConfiguration, you must include it in your custom admission control config file. For example, RKE2 installs an AdmissionConfiguration file at `/etc/rancher/rke2/rke2-pss.yaml`, which configures the PodSecurity admission plugin. Setting `admission-control-config-file` in config.yaml will override this essential security setting. To include both plugins, consult [the Default Pod Security Standards documentation](https://docs.rke2.io/security/pod_security_standards) and copy the appropriate plugin configuration to your admission.yaml. + ::: + +4. If you're using Rancher to provision your cluster using existing nodes, create these files on the node before you provision them. + + If you're using Rancher to provision your cluster on new nodes, allow the provisioning to complete, then use the provided SSH key and IP address to connect to the nodes, and place the RKE2 config file in the `/etc/rancher/rke2/config.yaml.d/` directory. + +5. After the cluster is configured with these credentials, configure the Rancher cluster agent to enable authentication in the webhook. Create a file containing these chart values: + + ```yaml + # values.yaml + auth: + clientCA: + allowedCNs: + - + - + ``` + +6. Create a configmap in the `cattle-system` namespace on the provisioned cluster with these values: + + ``` + kubectl --namespace cattle-system create configmap --from-file=rancher-webhook=values.yaml + ``` + + The webhook will restart with these values. diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/security-advisories-and-cves.md b/versioned_docs/version-2.8/reference-guides/rancher-security/security-advisories-and-cves.md new file mode 100644 index 00000000000..8f3649e7a85 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/security-advisories-and-cves.md @@ -0,0 +1,45 @@ +--- +title: Security Advisories and CVEs +--- + + + + + +Rancher is committed to informing the community of security issues in our products. Rancher will publish security advisories and CVEs (Common Vulnerabilities and Exposures) for issues we have resolved. New security advisories are also published in Rancher's GitHub [security page](https://github.com/rancher/rancher/security/advisories). + +| ID | Description | Date | Resolution | +|----|-------------|------|------------| +| [CVE-2023-22648](https://github.com/rancher/rancher/security/advisories/GHSA-vf6j-6739-78m8) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, in which permission changes in Azure AD are not reflected to users until they logout and log back into the Rancher UI. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2022-43760](https://github.com/rancher/rancher/security/advisories/GHSA-46v3-ggjg-qq3x) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, where multiple Cross-Site Scripting (XSS) vulnerabilities can be exploited via the Rancher UI. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2020-10676](https://github.com/rancher/rancher/security/advisories/GHSA-8vhc-hwhc-cpj4) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, in which users with update privileges on a namespace, can move that namespace into a project they don't have access to. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22647](https://github.com/rancher/rancher/security/advisories/GHSA-p976-h52c-26p6) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, where Standard users or above are able to elevate their permissions to Administrator in the local cluster. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22651](https://github.com/rancher/rancher/security/advisories/GHSA-6m9f-pj6w-w87g) | The Rancher admissions webhook may become misconfigured due to a failure in the webhook's update logic. The admissions webhook enforces validation rules and security checks before resources are admitted into the Kubernetes cluster. When the webhook is operating in a degraded state, it no longer validates any resources, which can result in severe privilege escalations and data corruption. | 24 April 2023 | Rancher [v2.7.3](https://github.com/rancher/rancher/releases/tag/v2.7.3) | +| [CVE-2022-43758](https://github.com/rancher/rancher/security/advisories/GHSA-34p5-jp77-fcrc) | An issue was discovered in Rancher from versions 2.5.0 up to and including 2.5.16, 2.6.0 up to and including 2.6.9 and 2.7.0, where a command injection vulnerability is present in the Rancher Git package. This package uses the underlying Git binary available in the Rancher container image to execute Git operations. Specially crafted commands, when not properly disambiguated, can cause confusion when executed through Git, resulting in command injection in the underlying Rancher host. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43757](https://github.com/rancher/rancher/security/advisories/GHSA-cq4p-vp5q-4522) | This issue affects Rancher versions from 2.5.0 up to and including 2.5.16, from 2.6.0 up to and including 2.6.9 and 2.7.0. It was discovered that the security advisory [CVE-2021-36782](https://github.com/advisories/GHSA-g7j7-h4q8-8w2f), previously released by Rancher, missed addressing some sensitive fields, secret tokens, encryption keys, and SSH keys that were still being stored in plaintext directly on Kubernetes objects like `Clusters`. The exposed credentials are visible in Rancher to authenticated `Cluster Owners`, `Cluster Members`, `Project Owners` and `Project Members` of that cluster. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43755](https://github.com/rancher/rancher/security/advisories/GHSA-8c69-r38j-rpfj) | An issue was discovered in Rancher versions up to and including 2.6.9 and 2.7.0, where the `cattle-token` secret, used by the `cattle-cluster-agent`, is predictable. Even after the token is regenerated, it will have the same value. This can pose a serious problem if the token is compromised and needs to be recreated for security purposes. The `cattle-token` is used by Rancher's `cattle-cluster-agent` to connect to the Kubernetes API of Rancher provisioned downstream clusters. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1) and [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) | +| [CVE-2022-21953](https://github.com/rancher/rancher/security/advisories/GHSA-g25r-gvq3-wrq7) | An issue was discovered in Rancher versions up to and including 2.5.16, 2.6.9 and 2.7.0, where an authorization logic flaw allows an authenticated user on any downstream cluster to (1) open a shell pod in the Rancher `local` cluster and (2) have limited kubectl access to it. The expected behavior is that a user does not have such access in the Rancher `local` cluster unless explicitly granted. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [GHSA-c45c-39f6-6gw9](https://github.com/rancher/rancher/security/advisories/GHSA-c45c-39f6-6gw9) | This issue affects Rancher versions from 2.5.0 up to and including 2.5.16, from 2.6.0 up to and including 2.6.9 and 2.7.0. It only affects Rancher setups that have an external authentication provider configured or had one configured in the past. It was discovered that when an external authentication provider is configured in Rancher and then disabled, the Rancher generated tokens associated with users who had access granted through the now disabled auth provider are not revoked. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-31247](https://github.com/rancher/rancher/security/advisories/GHSA-6x34-89p7-95wg) | An issue was discovered in Rancher versions up to and including 2.5.15 and 2.6.6 where a flaw with authorization logic allows privilege escalation in downstream clusters through cluster role template binding (CRTB) and project role template binding (PRTB). The vulnerability can be exploited by any user who has permissions to create/edit CRTB or PRTB (such as `cluster-owner`, `manage cluster members`, `project-owner`, and `manage project members`) to gain owner permission in another project in the same cluster or in another project on a different downstream cluster. | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36783](https://github.com/rancher/rancher/security/advisories/GHSA-8w87-58w6-hfv8) | It was discovered that in Rancher versions up to and including 2.5.12 and 2.6.3, there is a failure to properly sanitize credentials in cluster template answers. This failure can lead to plaintext storage and exposure of credentials, passwords, and API tokens. The exposed credentials are visible in Rancher to authenticated `Cluster Owners`, `Cluster Members`, `Project Owners`, and `Project Members` on the endpoints `/v1/management.cattle.io.clusters`, `/v3/clusters`, and `/k8s/clusters/local/apis/management.cattle.io/v3/clusters`. | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36782](https://github.com/rancher/rancher/security/advisories/GHSA-g7j7-h4q8-8w2f) | An issue was discovered in Rancher versions up to and including 2.5.15 and 2.6.6 where sensitive fields like passwords, API keys, and Rancher's service account token (used to provision clusters) were stored in plaintext directly on Kubernetes objects like `Clusters` (e.g., `cluster.management.cattle.io`). Anyone with read access to those objects in the Kubernetes API could retrieve the plaintext version of those sensitive data. The issue was partially found and reported by Florian Struck (from [Continum AG](https://www.continum.net/)) and [Marco Stuurman](https://github.com/fe-ax) (from [Shock Media B.V.](https://www.shockmedia.nl/)). | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2022-21951](https://github.com/rancher/rancher/security/advisories/GHSA-vrph-m5jj-c46c) | This vulnerability only affects customers using [Weave](../../faq/container-network-interface-providers.md#weave) Container Network Interface (CNI) when configured through [RKE templates](../../pages-for-subheaders/about-rke1-templates.md). A vulnerability was discovered in Rancher versions 2.5.0 up to and including 2.5.13, and 2.6.0 up to and including 2.6.4, where a user interface (UI) issue with RKE templates does not include a value for the Weave password when Weave is chosen as the CNI. If a cluster is created based on the mentioned template, and Weave is configured as the CNI, no password will be created for [network encryption](https://www.weave.works/docs/net/latest/tasks/manage/security-untrusted-networks/) in Weave; therefore, network traffic in the cluster will be sent unencrypted. | 24 May 2022 | [Rancher v2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) and [Rancher v2.5.14](https://github.com/rancher/rancher/releases/tag/v2.5.14) | +| [CVE-2021-36784](https://github.com/rancher/rancher/security/advisories/GHSA-jwvr-vv7p-gpwq) | A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3 which allows users who have create or update permissions on [Global Roles](../../pages-for-subheaders/manage-role-based-access-control-rbac.md) to escalate their permissions, or those of another user, to admin-level permissions. Global Roles grant users Rancher-wide permissions, such as the ability to create clusters. In the identified versions of Rancher, when users are given permission to edit or create Global Roles, they are not restricted to only granting permissions which they already posses. This vulnerability affects customers who utilize non-admin users that are able to create or edit Global Roles. The most common use case for this scenario is the `restricted-admin` role. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-4200](https://github.com/rancher/rancher/security/advisories/GHSA-hx8w-ghh8-r4xf) | This vulnerability only affects customers using the `restricted-admin` role in Rancher. A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3 where the `global-data` role in `cattle-global-data` namespace grants write access to the Catalogs. Since each user with any level of catalog access was bound to the `global-data` role, this grants write access to templates (`CatalogTemplates`) and template versions (`CatalogTemplateVersions`) for any user with any level of catalog access. New users created in Rancher are by default assigned to the `user` role (standard user), which is not designed to grant write catalog access. This vulnerability effectively elevates the privilege of any user to write access for the catalog template and catalog template version resources. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [GHSA-wm2r-rp98-8pmh](https://github.com/rancher/rancher/security/advisories/GHSA-wm2r-rp98-8pmh) | This vulnerability only affects customers using [Fleet](../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) for continuous delivery with authenticated Git and/or Helm repositories. An issue was discovered in `go-getter` library in versions prior to [`v1.5.11`](https://github.com/hashicorp/go-getter/releases/tag/v1.5.11) that exposes SSH private keys in base64 format due to a failure in redacting such information from error messages. The vulnerable version of this library is used in Rancher through Fleet in versions of Fleet prior to [`v0.3.9`](https://github.com/rancher/fleet/releases/tag/v0.3.9). This issue affects Rancher versions 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3. The issue was found and reported by Dagan Henderson from Raft Engineering. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-36778](https://github.com/rancher/rancher/security/advisories/GHSA-4fc7-hc63-7fjg) | A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.11 and from 2.6.0 up to and including 2.6.2, where an insufficient check of the same-origin policy when downloading Helm charts from a configured private repository can lead to exposure of the repository credentials to a third-party provider. This issue only happens when the user configures access credentials to a private repository in Rancher inside `Apps & Marketplace > Repositories`. The issue was found and reported by Martin Andreas Ullrich. | 14 Apr 2022 | [Rancher v2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) and [Rancher v2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) | +| [GHSA-hwm2-4ph6-w6m5](https://github.com/rancher/rancher/security/advisories/GHSA-hwm2-4ph6-w6m5) | A vulnerability was discovered in versions of Rancher starting 2.0 up to and including 2.6.3. The `restricted` pod security policy (PSP) provided in Rancher deviated from the upstream `restricted` policy provided in Kubernetes on account of which Rancher's PSP had `runAsUser` set to `runAsAny`, while upstream had `runAsUser` set to `MustRunAsNonRoot`. This allowed containers to run as any user, including a privileged user (`root`), even when Rancher's `restricted` policy was enforced on a project or at the cluster level. | 31 Mar 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | +| [CVE-2021-36775](https://github.com/rancher/rancher/security/advisories/GHSA-28g7-896h-695v) | A vulnerability was discovered in Rancher versions up to and including 2.4.17, 2.5.11 and 2.6.2. After removing a `Project Role` associated with a group from the project, the bindings that granted access to cluster-scoped resources for those subjects were not deleted. This was due to an incomplete authorization logic check. A user who was a member of the affected group with authenticated access to Rancher could exploit this vulnerability to access resources they shouldn't have had access to. The exposure level would depend on the original permission level granted to the affected project role. This vulnerability only affected customers using group based authentication in Rancher. | 31 Mar 2022 | [Rancher v2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3), [Rancher v2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) and [Rancher v2.4.18](https://github.com/rancher/rancher/releases/tag/v2.4.18) | +| [CVE-2021-36776](https://github.com/rancher/rancher/security/advisories/GHSA-gvh9-xgrq-r8hw) | A vulnerability was discovered in Rancher versions starting 2.5.0 up to and including 2.5.9, that allowed an authenticated user to impersonate any user on a cluster through an API proxy, without requiring knowledge of the impersonated user's credentials. This was due to the API proxy not dropping the impersonation header before sending the request to the Kubernetes API. A malicious user with authenticated access to Rancher could use this to impersonate another user with administrator access in Rancher, thereby gaining administrator level access to the cluster. | 31 Mar 2022 | [Rancher v2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) and [Rancher v2.5.10](https://github.com/rancher/rancher/releases/tag/v2.5.10) | +| [CVE-2021-25318](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25318) | A vulnerability was discovered in Rancher versions 2.0 through the aforementioned fixed versions, where users were granted access to resources regardless of the resource's API group. For example, Rancher should have allowed users access to `apps.catalog.cattle.io`, but instead incorrectly gave access to `apps.*`. Resources affected in the **Downstream clusters** and **Rancher management cluster** can be found [here](https://github.com/rancher/rancher/security/advisories/GHSA-f9xf-jq4j-vqw4). There is not a direct mitigation besides upgrading to the patched Rancher versions. | 14 Jul 2021 | [Rancher v2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) and [Rancher v2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-31999](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-31999) | A vulnerability was discovered in Rancher 2.0.0 through the aforementioned patched versions, where a malicious Rancher user could craft an API request directed at the proxy for the Kubernetes API of a managed cluster to gain access to information they do not have access to. This is done by passing the "Impersonate-User" or "Impersonate-Group" header in the Connection header, which is then correctly removed by the proxy. At this point, instead of impersonating the user and their permissions, the request will act as if it was from the Rancher management server and incorrectly return the information. The vulnerability is limited to valid Rancher users with some level of permissions on the cluster. There is not a direct mitigation besides upgrading to the patched Rancher versions. | 14 Jul 2021 | [Rancher v2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) and [Rancher v2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25320](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25320) | A vulnerability was discovered in Rancher 2.2.0 through the aforementioned patched versions, where cloud credentials weren't being properly validated through the Rancher API. Specifically through a proxy designed to communicate with cloud providers. Any Rancher user that was logged-in and aware of a cloud-credential ID that was valid for a given cloud provider, could call that cloud provider's API through the proxy API, and the cloud-credential would be attached. The exploit is limited to valid Rancher users. There is not a direct mitigation outside of upgrading to the patched Rancher versions. | 14 Jul 2021 | [Rancher v2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) and [Rancher v2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25313](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25313) | A security vulnerability was discovered on all Rancher 2 versions. When accessing the Rancher API with a browser, the URL was not properly escaped, making it vulnerable to an XSS attack. Specially crafted URLs to these API endpoints could include JavaScript which would be embedded in the page and execute in a browser. There is no direct mitigation. Avoid clicking on untrusted links to your Rancher server. | 2 Mar 2021 | [Rancher v2.5.6](https://github.com/rancher/rancher/releases/tag/v2.5.6), [Rancher v2.4.14](https://github.com/rancher/rancher/releases/tag/v2.4.14), and [Rancher v2.3.11](https://github.com/rancher/rancher/releases/tag/v2.3.11) | +| [CVE-2019-14435](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14435) | This vulnerability allows authenticated users to potentially extract otherwise private data out of IPs reachable from system service containers used by Rancher. This can include but not only limited to services such as cloud provider metadata services. Although Rancher allow users to configure whitelisted domains for system service access, this flaw can still be exploited by a carefully crafted HTTP request. The issue was found and reported by Matt Belisle and Alex Stevenson at Workiva. | 5 Aug 2019 | [Rancher v2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) and [Rancher v2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-14436](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14436) | The vulnerability allows a member of a project that has access to edit role bindings to be able to assign themselves or others a cluster level role granting them administrator access to that cluster. The issue was found and reported by Michal Lipinski at Nokia. | 5 Aug 2019 | [Rancher v2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) and [Rancher v2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-13209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-13209) | The vulnerability is known as a [Cross-Site Websocket Hijacking attack](https://www.christian-schneider.net/CrossSiteWebSocketHijacking.html). This attack allows an exploiter to gain access to clusters managed by Rancher with the roles/permissions of a victim. It requires that a victim to be logged into a Rancher server and then access a third-party site hosted by the exploiter. Once that is accomplished, the exploiter is able to execute commands against the Kubernetes API with the permissions and identity of the victim. Reported by Matt Belisle and Alex Stevenson from Workiva. | 15 Jul 2019 | [Rancher v2.2.5](https://github.com/rancher/rancher/releases/tag/v2.2.5), [Rancher v2.1.11](https://github.com/rancher/rancher/releases/tag/v2.1.11) and [Rancher v2.0.16](https://github.com/rancher/rancher/releases/tag/v2.0.16) | +| [CVE-2019-12303](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12303) | Project owners can inject extra fluentd logging configurations that makes it possible to read files or execute arbitrary commands inside the fluentd container. Reported by Tyler Welton from Untamed Theory. | 5 Jun 2019 | [Rancher v2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4), [Rancher v2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) and [Rancher v2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-12274](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12274) | Nodes using the built-in node drivers using a file path option allows the machine to read arbitrary files including sensitive ones from inside the Rancher server container. | 5 Jun 2019 | [Rancher v2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4), [Rancher v2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) and [Rancher v2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-11202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11202) | The default admin, that is shipped with Rancher, will be re-created upon restart of Rancher despite being explicitly deleted. | 16 Apr 2019 | [Rancher v2.2.2](https://github.com/rancher/rancher/releases/tag/v2.2.2), [Rancher v2.1.9](https://github.com/rancher/rancher/releases/tag/v2.1.9) and [Rancher v2.0.14](https://github.com/rancher/rancher/releases/tag/v2.0.14) | +| [CVE-2019-6287](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6287) | Project members continue to get access to namespaces from projects that they were removed from if they were added to more than one project. | 29 Jan 2019 | [Rancher v2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) and [Rancher v2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) | +| [CVE-2018-20321](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-20321) | Any project member with access to the `default` namespace can mount the `netes-default` service account in a pod and then use that pod to execute administrative privileged commands against the Kubernetes cluster. | 29 Jan 2019 | [Rancher v2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) and [Rancher v2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) - Rolling back from these versions or greater have specific [instructions](../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md). | diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md b/versioned_docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md new file mode 100644 index 00000000000..1d9d46c75d4 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md @@ -0,0 +1,70 @@ +--- +title: About rancher-selinux +--- + + + + + +To allow Rancher to work with SELinux, some functionality has to be manually enabled for the SELinux nodes. To help with that, Rancher provides a SELinux RPM. + +The `rancher-selinux` RPM only contains policies for the [rancher-logging application.](https://github.com/rancher/charts/tree/dev-v2.5/charts/rancher-logging) + +The `rancher-selinux` GitHub repository is [here.](https://github.com/rancher/rancher-selinux) + +## Installing the rancher-selinux RPM + +:::note Requirement: + +The rancher-selinux RPM was tested with CentOS 7 and 8. + +::: + +### 1. Set up the yum repo + +Set up the yum repo to install `rancher-selinux` directly on all hosts in the cluster. + +In order to use the RPM repository, on a CentOS 7 or RHEL 7 system, run the following bash snippet: + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/7/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` + +In order to use the RPM repository, on a CentOS 8 or RHEL 8 system, run the following bash snippet: + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/8/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` +### 2. Installing the RPM + +Install the RPM: + +``` +yum -y install rancher-selinux +``` + +## Configuring the Logging Application to Work with SELinux + +:::note Requirement: + +Logging v2 was tested with SELinux on RHEL/CentOS 7 and 8. + +::: + +Applications do not automatically work once the `rancher-selinux` RPM is installed on the host. They need to be configured to run in an allowed SELinux container domain provided by the RPM. + +To configure the `rancher-logging` chart to be SELinux aware, change `global.seLinux.enabled` to true in the `values.yaml` when installing the chart. \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md b/versioned_docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md new file mode 100644 index 00000000000..c5edf5e0017 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md @@ -0,0 +1,13 @@ +--- +title: About rke2-selinux +--- + + + + + +`rke2-selinux` provides policies for RKE2. It is installed automatically when the RKE2 installer script detects that it is running on an RPM-based distro. + +The `rke2-selinux` GitHub repository is [here.](https://github.com/rancher/rke2-selinux) + +For more information about installing RKE2 on SELinux-enabled hosts, see the [RKE2 documentation.](https://docs.rke2.io/install/methods#rpm) \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/rancher-webhook.md b/versioned_docs/version-2.8/reference-guides/rancher-webhook.md new file mode 100644 index 00000000000..6f993753254 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rancher-webhook.md @@ -0,0 +1,108 @@ +--- +title: Rancher Webhook +--- + + + + + +Rancher-Webhook is an essential component of Rancher that works in conjunction with Kubernetes to enhance security and enable critical features for Rancher-managed clusters. + +It integrates with Kubernetes' extensible admission controllers, as described in the [Kubernetes documentation](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), which allows Rancher-Webhook to inspect specific requests sent to the Kubernetes API server, and add custom, Rancher-specific validation and mutations to the requests that are specific to Rancher. Rancher-Webhook manages the resources to be validated using the `rancher.cattle.io` `ValidatingWebhookConfiguration` and the `rancher.cattle.io` `MutatingWebhookConfiguration`, and will override any manual edits. +Rancher deploys Rancher-Webhook as a separate deployment and service in both local and downstream clusters. Rancher manages Rancher-Webhook using Helm. It's important to note that Rancher may override modifications made by users to the Helm release. + +Each Rancher version is designed to be compatible with a single version of the webhook. The compatible versions are provided below for convenience. + +**Note:** Rancher manages deployment and upgrade of the webhook. Under most circumstances, no user intervention should be needed to ensure that the webhook version is compatible with the version of Rancher that you are running. + +| Rancher Version | Webhook Version | +|-----------------|:---------------:| +| v2.7.0 | v0.3.0 | +| v2.7.1 | v0.3.0 | +| v2.7.2 | v0.3.2 | +| v2.7.3 | v0.3.3 | +| v2.7.4 | v0.3.4 | +| v2.7.5 | v0.3.5 | +| v2.7.6 | v0.3.5 | + + +## Why Do We Need It? + +Rancher-Webhook is crucial for Rancher to protect clusters against malicious attacks and enable various features. +Rancher relies on the Rancher-Webhook as an integral part of its functionality. Without the webhook, Rancher would not be a complete product. +It provides essential protection for Rancher-managed clusters, preventing security vulnerabilities and ensuring the consistency and stability of the cluster. + +## What Resources Does the Webhook Validate? + +An in-progress list of the resources that the webhook validates can be found in the [webhook's repo](https://github.com/rancher/webhook/blob/release/v0.4/docs.md). These docs are organized by group/version and resource (top-level header is group/version, next level header is resource). Checks specific to one version can be found by viewing the `docs.md` file associated with a particular tag (note that webhook versions prior to `v0.3.6` won't have this file). + +## Bypassing the Webhook + +Sometimes, it may be necessary to bypass Rancher's webhook validation to perform emergency restore operations, or fix other critical issues. The bypass operation is exhaustive, meaning that no webhook validations or mutations will apply when this is used. It is not possible to bypass some mutations or validations and have others still apply - they are either all bypassed, or all active. + +:::danger + +Rancher's webhook provides critical security protections. Bypassing the webhook should only be done by administrators in specific scenarios, after all other options have been exhausted. In addition, permission to bypass the webhook should be carefully controlled, and never given to users who are not admins. + +::: + +To bypass the webhook, impersonate both the `rancher-webhook-sudo` service account and the `system:masters` group (both are required): + +```bash +kubectl create -f example.yaml --as=system:serviceaccount:cattle-system:rancher-webhook-sudo --as-group=system:masters +``` + +## Common Issues + +### EKS Cluster with Calico CNI + +Users running an EKS cluster with Calico CNI may run into errors when the Kubernetes API server attempts to contact the Rancher-Webhook. +One workaround for this issue [documented by Calico](https://docs.tigera.io/calico/latest/getting-started/kubernetes/managed-public-cloud/eks#install-eks-with-calico-networking) involves setting `hostNetwork=true` for the webhook deployment. Users can change this using the Helm commands below on the affected clusters. + +``` bash +helm repo add rancher-charts https://charts.rancher.io +helm upgrade --reuse-values rancher-webhook rancher-charts/rancher-webhook -n cattle-system --set global.hostNetwork=true +``` +**Note:** This temporary workaround may violate an environment's security policy. This workaround also requires that port 9443 is unused on the host network. + +**Note:** Helm, by default, uses a type that some webhook versions validate (secrets) to store information. In these cases, it's recommended to first directly update the deployment with the hostNetwork=true value using kubectl, and then perform the helm commands listed above to avoid drift between the helm configuration and the actual state in the cluster. + +### Private GKE Cluster + +When using a private GKE cluster, errors may occur that prevent the Kubernetes API server from communicating with the webhook. The following error message may appear: + +``` +Internal error occurred: failed calling webhook "rancher.cattle.io.namespaces.create-non-kubesystem": failed to call webhook: Post "https://rancher-webhook.cattle-system.svc:443/v1/webhook/validation/namespaces?timeout=10s": context deadline exceeded +``` +This issue occurs because firewall rules restrict communication between the API server and the private cluster. To resolve this communication problem, users must add firewall rules to allow the GKE control plane to communicate with the Rancher-Webhook on port 9443. Please refer to the [GKE documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) for detailed information and steps on updating the firewall rules. + +### Application Fails to Deploy Due to rancher-webhook Blocking Access + +The webhook provides extra validations on [namespaces](https://github.com/rancher/webhook/blob/release/v0.4/docs.md#psa-label-validation). One of these validations ensures that users can only update PSA relevant labels if they have the proper permissions (`updatepsa` for `projects` in `management.cattle.io`). This can result in specific operators, such as Tigera or Trident, failing when they attempt to deploy namespaces with PSA labels. There are several ways to resolve this issue: + +- Configure the application to create a namespace with no PSA labels. If users wish to apply a PSA to these namespaces, they can add them to a project with the desired PSA after configuration. See the [docs on PSS and PSA resources](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards) for instructions on how. + - This is the preferred option, though not all applications can be configured in this fashion. +- Manually grant the operator permissions to manage PSAs for namespaces. + - This option will introduce security risks, since the operator will now be able to set the PSA for the namespaces it has access to. This could allow the operator to deploy a privileged pod, or effect cluster takeover through other means. +- A user account with the proper permissions can pre-create the namespace with the appropriate configuration. + - This option depends on the ability of the application to handle existing resources. + +## Issues on Specific Versions + +**Note:** The following is an incomplete list of high-severity issues affecting specific Rancher/webhook versions. In most cases, these issues can be resolved by upgrading to a more recent Rancher version. + +### Incompatible Webhook Version on Rollback + +**Note:** This affects rolling back to Rancher v2.7.5 or earlier. + +If you roll back to Rancher v2.7.5 or earlier, you may see webhook versions that are too recent to be compatible with downstream clusters running pre-v2.7.5 version of Rancher. This may cause various incompatibility issues. For example, project members may be unable to create namespaces. In addition, when you roll back to versions before the webhook was installed in downstream clusters, the webhook may remain installed, which can result in similar incompatibility issues. + +To help alleviate these issues, you can run the [adjust-downstream-webhook](https://github.com/rancherlabs/support-tools/tree/master/adjust-downstream-webhook) shell script after roll back. This script selects and installs the proper webhook version (or removes the webhook entirely) for the corresponding Rancher version. + +### Project Members Can't Create Namespaces + +**Note:** This affects Rancher versions `v2.7.2 - v2.7.4` + +Project users who aren't owners may not be able to create namespaces in projects. This issue is caused by Rancher automatically upgrading the webhook to a version compatible with a more recent version of Rancher than the one currently installed. + +To help alleviate these issues, you can run the [adjust-downstream-webhook](https://github.com/rancherlabs/support-tools/tree/master/adjust-downstream-webhook) shell script after roll back. This script selects and installs the proper webhook version (or removes the webhook entirely) for the corresponding Rancher version. diff --git a/versioned_docs/version-2.8/reference-guides/rke1-template-example-yaml.md b/versioned_docs/version-2.8/reference-guides/rke1-template-example-yaml.md new file mode 100644 index 00000000000..d14e3863ff9 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/rke1-template-example-yaml.md @@ -0,0 +1,111 @@ +--- +title: RKE1 Example YAML +--- + +Below is an example RKE template configuration file for reference. + +The YAML in the RKE template uses the same customization that is used when you create an RKE cluster. However, since the YAML is within the context of a Rancher provisioned RKE cluster, the customization from the RKE docs needs to be nested under the `rancher_kubernetes_engine` directive. + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker + +enable_cluster_alerting: false +# This setting is not enforced. Clusters +# created with this sample template +# would have alerting turned off by default, +# but end users could still turn alerting +# on or off. + +enable_cluster_monitoring: true +# This setting is not enforced. Clusters +# created with this sample template +# would have monitoring turned on +# by default, but end users could still +# turn monitoring on or off. + +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # Currently only nginx ingress provider is supported. +# # To disable ingress controller, set `provider: none` +# # To enable ingress on specific nodes, use the node_selector, eg: +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/single-node-rancher-in-docker/advanced-options.md b/versioned_docs/version-2.8/reference-guides/single-node-rancher-in-docker/advanced-options.md new file mode 100644 index 00000000000..d020910622b --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/single-node-rancher-in-docker/advanced-options.md @@ -0,0 +1,105 @@ +--- +title: Advanced Options for Docker Installs +--- + +### Custom CA Certificate + +If you want to configure Rancher to use a CA root certificate to be used when validating services, you would start the Rancher container sharing the directory that contains the CA root certificate. + +Use the command example to start a Rancher container with your private CA certificates mounted. + +- The volume flag (`-v`) should specify the host directory containing the CA root certificates. +- The environment variable flag (`-e`) in combination with `SSL_CERT_DIR` and directory declares an environment variable that specifies the mounted CA root certificates directory location inside the container. +- Passing environment variables to the Rancher container can be done using `-e KEY=VALUE` or `--env KEY=VALUE`. +- Mounting a host directory inside the container can be done using `-v host-source-directory:container-destination-directory` or `--volume host-source-directory:container-destination-directory`. + +The example below is based on having the CA root certificates in the `/host/certs` directory on the host and mounting this directory on `/container/certs` inside the Rancher container. + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /host/certs:/container/certs \ + -e SSL_CERT_DIR="/container/certs" \ + --privileged \ + rancher/rancher:latest +``` + +### API Audit Log + +The API Audit Log records all the user and system transactions made through Rancher server. + +The API Audit Log writes to `/var/log/auditlog` inside the rancher container by default. Share that directory as a volume and set your `AUDIT_LEVEL` to enable the log. + +See [API Audit Log](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) for more information and options. + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /var/log/rancher/auditlog:/var/log/auditlog \ + -e AUDIT_LEVEL=1 \ + --privileged \ + rancher/rancher:latest +``` + +### TLS settings + +To set a different TLS configuration, you can use the `CATTLE_TLS_MIN_VERSION` and `CATTLE_TLS_CIPHERS` environment variables. For example, to configure TLS 1.0 as minimum accepted TLS version: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_TLS_MIN_VERSION="1.0" \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +See [TLS settings](../../getting-started/installation-and-upgrade/installation-references/tls-settings.md) for more information and options. + +### Air Gap + +If you are visiting this page to complete an air gap installation, you must prepend your private registry URL to the server tag when running the installation command in the option that you choose. Add `` with your private registry URL in front of `rancher/rancher:latest`. + +**Example:** + + /rancher/rancher:latest + +### Persistent Data + +Rancher uses etcd as a datastore. When Rancher is installed with Docker, the embedded etcd is being used. The persistent data is at the following path in the container: `/var/lib/rancher`. + +You can bind mount a host volume to this location to preserve data on the host it is running on: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +### Running `rancher/rancher` and `rancher/rancher-agent` on the Same Node + +In the situation where you want to use a single node to run Rancher and to be able to add the same node to a cluster, you have to adjust the host ports mapped for the `rancher/rancher` container. + +If a node is added to a cluster, it deploys the nginx ingress controller which will use port 80 and 443. This will conflict with the default ports we advise to expose for the `rancher/rancher` container. + +Please note that this setup is not recommended for production use, but can be convenient for development/demo purposes. + +To change the host ports mapping, replace the following part `-p 80:80 -p 443:443` with `-p 8080:80 -p 8443:443`: + +``` +docker run -d --restart=unless-stopped \ + -p 8080:80 -p 8443:443 \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) diff --git a/versioned_docs/version-2.8/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md b/versioned_docs/version-2.8/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md new file mode 100644 index 00000000000..22210ccd8ff --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md @@ -0,0 +1,61 @@ +--- +title: HTTP Proxy Configuration +--- + +If you operate Rancher behind a proxy and you want to access services through the proxy (such as retrieving catalogs), you must provide Rancher information about your proxy. As Rancher is written in Go, it uses the common proxy environment variables as shown below. + +Make sure `NO_PROXY` contains the network addresses, network address ranges and domains that should be excluded from using the proxy. + +| Environment variable | Purpose | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| HTTP_PROXY | Proxy address to use when initiating HTTP connection(s) | +| HTTPS_PROXY | Proxy address to use when initiating HTTPS connection(s) | +| NO_PROXY | Network address(es), network address range(s) and domains to exclude from using the proxy when initiating connection(s) | + +:::note Important: + +NO_PROXY must be in uppercase to use network range (CIDR) notation. + +::: + +## Docker Installation + +Passing environment variables to the Rancher container can be done using `-e KEY=VALUE` or `--env KEY=VALUE`. Required values for `NO_PROXY` in a [Docker Installation](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md) are: + +- `localhost` +- `127.0.0.1` +- `0.0.0.0` +- `10.0.0.0/8` +- `cattle-system.svc` +- `.svc` +- `.cluster.local` + +The example below is based on a proxy server accessible at `http://192.168.0.1:3128`, and excluding usage the proxy when accessing network range `192.168.10.0/24` and every hostname under the domain `example.com`. + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e HTTP_PROXY="http://192.168.10.1:3128" \ + -e HTTPS_PROXY="http://192.168.10.1:3128" \ + -e NO_PROXY="localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,cattle-system.svc,192.168.10.0/24,.svc,.cluster.local,example.com" \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +### Air-gapped proxy configuration + +You can now provision node driver clusters from an air-gapped cluster configured to use a proxy for outbound connections. + +In addition to setting the default rules for a proxy server as shown above, you will need to add additional rules, shown below, to provision node driver clusters from a proxied Rancher environment. + +You will configure your filepath according to your setup, e.g., `/etc/apt/apt.conf.d/proxy.conf`: + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/reference-guides/system-tools.md b/versioned_docs/version-2.8/reference-guides/system-tools.md new file mode 100644 index 00000000000..d9480976757 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/system-tools.md @@ -0,0 +1,31 @@ +--- +title: System Tools +--- + +:::note + +System Tools has been deprecated since June 2022. + +::: + +## Logs + +Please use [logs-collector](https://github.com/rancherlabs/support-tools/tree/master/collection/rancher/v2.x/logs-collector) to collect logs from your cluster. + +## Stats + +If you want to replicate the stats command, you can run the following command on your cluster nodes: + +:::note + +This command below requires the package `sysstat` on the cluster node. + +::: + +``` +/usr/bin/sar -u -r -F 1 1 +``` + +## Remove + +Please use the [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) tool. diff --git a/versioned_docs/version-2.8/reference-guides/user-settings/api-keys.md b/versioned_docs/version-2.8/reference-guides/user-settings/api-keys.md new file mode 100644 index 00000000000..ade1aee3d9b --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/user-settings/api-keys.md @@ -0,0 +1,59 @@ +--- +title: API Keys +--- + +## API Keys and User Authentication + +If you want to access your Rancher clusters, projects, or other objects using external applications, you can do so using the Rancher API. However, before your application can access the API, you must provide the app with a key used to authenticate with Rancher. You can obtain a key using the Rancher UI. + +An API key is also required for using Rancher CLI. + +API Keys are composed of four components: + +- **Endpoint:** This is the IP address and path that other applications use to send requests to the Rancher API. +- **Access Key:** The token's username. +- **Secret Key:** The token's password. For applications that prompt you for two different strings for API authentication, you usually enter the two keys together. +- **Bearer Token:** The token username and password concatenated together. Use this string for applications that prompt you for one authentication string. + +:::note + +Users may opt to enable [token hashing](../about-the-api/api-tokens.md). + +::: + +## Creating an API Key + +1. Select **User Avatar > Account & API Keys** from upper right corner. + +2. Click **Create API Key**. + +3. **Optional:** Enter a description for the API key and select an expiration period or a scope. We recommend setting an expiration date. + + The API key won't be valid after expiration. Shorter expiration periods are more secure. + + Expiration period will be bound by `v3/settings/auth-token-max-ttl-minutes`. If it exceeds the max-ttl, API key will be created with max-ttl as the expiration period. + + A scope will limit the API key so that it will only work against the Kubernetes API of the specified cluster. If the cluster is configured with an Authorized Cluster Endpoint, you will be able to use a scoped token directly against the cluster's API without proxying through the Rancher server. See [Authorized Cluster Endpoints](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) for more information. + +4. Click **Create**. + + **Step Result:** Your API Key is created. Your API **Endpoint**, **Access Key**, **Secret Key**, and **Bearer Token** are displayed. + + Use the **Bearer Token** to authenticate with Rancher CLI. + +5. Copy the information displayed to a secure location. This information is only displayed once, so if you lose your key, you'll have to make a new one. + +## What's Next? + +- Enter your API key information into the application that will send requests to the Rancher API. +- Learn more about the Rancher endpoints and parameters by selecting **View in API** for an object in the Rancher UI. +- API keys are used for API calls and [Rancher CLI](../../pages-for-subheaders/cli-with-rancher.md). + +## Deleting API Keys + +If you need to revoke an API key, delete it. You should delete API keys: + +- That may have been compromised. +- That have expired. + +To delete an API, select the stale key and click **Delete**. diff --git a/versioned_docs/version-2.8/reference-guides/user-settings/manage-cloud-credentials.md b/versioned_docs/version-2.8/reference-guides/user-settings/manage-cloud-credentials.md new file mode 100644 index 00000000000..e387346c02c --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/user-settings/manage-cloud-credentials.md @@ -0,0 +1,51 @@ +--- +title: Managing Cloud Credentials +--- + +When you create a cluster [hosted by an infrastructure provider](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md), [node templates](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) are used to provision the cluster nodes. These templates use Docker Machine configuration options to define an operating system image and settings/parameters for the node. + +Node templates can use cloud credentials to access the credential information required to provision nodes in the infrastructure providers. The same cloud credential can be used by multiple node templates. By using a cloud credential, you do not have to re-enter access keys for the same cloud provider. Cloud credentials are stored as Kubernetes secrets. + +Cloud credentials are only used by node templates if there are fields marked as `password`. The default `active` node drivers have their account access fields marked as `password`, but there may be some `inactive` node drivers, which are not using them yet. These node drivers will not use cloud credentials. + +You can create cloud credentials in two contexts: + +- [During creation of a node template](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for a cluster. +- In the **User Settings** + +All cloud credentials are bound to the user profile of who created it. They **cannot** be shared across users. + +## Creating a Cloud Credential from User Settings + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click a cloud credential type. The values of this dropdown is based on the `active` [node drivers](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md) in Rancher. +1. Enter a name for the cloud credential. +1. Based on the selected cloud credential type, enter the required values to authenticate with the infrastructure provider. +1. Click **Create**. + +**Result:** The cloud credential is created and can immediately be used to [create node templates](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates). + +## Updating a Cloud Credential + +When access credentials are changed or compromised, updating a cloud credential allows you to rotate those credentials while keeping the same node template. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Choose the cloud credential you want to edit and click the **⋮ > Edit Config**. +1. Update the credential information and click **Save**. + +**Result:** The cloud credential is updated with the new access credentials. All existing node templates using this cloud credential will automatically use the updated information whenever [new nodes are added](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +## Deleting a Cloud Credential + +In order to delete cloud credentials, there must not be any node template associated with it. If you are unable to delete the cloud credential, [delete any node templates](manage-node-templates.md#deleting-a-node-template) that are still associated to that cloud credential. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. You can either individually delete a cloud credential or bulk delete. + + - To individually delete one, choose the cloud credential you want to edit and click the **⋮ > Delete**. + - To bulk delete cloud credentials, select one or more cloud credentials from the list. Click **Delete**. +1. Confirm that you want to delete these cloud credentials. diff --git a/versioned_docs/version-2.8/reference-guides/user-settings/manage-node-templates.md b/versioned_docs/version-2.8/reference-guides/user-settings/manage-node-templates.md new file mode 100644 index 00000000000..22c29e0ae54 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/user-settings/manage-node-templates.md @@ -0,0 +1,54 @@ +--- +title: Managing Node Templates +--- + +When you provision a cluster [hosted by an infrastructure provider](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md), [node templates](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) are used to provision the cluster nodes. These templates use Docker Machine configuration options to define an operating system image and settings/parameters for the node. You can create node templates in two contexts: + +- While [provisioning a node pool cluster](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). +- At any time, from your [user settings](../../pages-for-subheaders/user-settings.md). + +When you create a node template, it is bound to your user profile. Node templates cannot be shared among users. You can delete stale node templates that you no longer user from your user settings. + +## Creating a Node Template + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Add Template**. +1. Select one of the cloud providers available. Then follow the instructions on screen to configure the template. + +**Result:** The template is configured. You can use the template later when you [provision a node pool cluster](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +## Updating a Node Template + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Choose the node template that you want to edit and click the **⋮ > Edit**. + + :::note + + The default `active` [node drivers](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md) and any node driver, that has fields marked as `password`, are required to use [cloud credentials](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#cloud-credentials). + + ::: + +1. Edit the required information and click **Save**. + +**Result:** The node template is updated. All node pools using this node template will automatically use the updated information when new nodes are added. + +## Cloning Node Templates + +When creating new node templates from your user settings, you can clone an existing template and quickly update its settings rather than creating a new one from scratch. Cloning templates saves you the hassle of re-entering access keys for the cloud provider. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Find the template you want to clone. Then select **⋮ > Clone**. +1. Complete the rest of the form. + +**Result:** The template is cloned and configured. You can use the template later when you [provision a node pool cluster](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +## Deleting a Node Template + +When you no longer use a node template, you can delete it from your user settings. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Select one or more template from the list. Then click **Delete**. Confirm the delete when prompted. diff --git a/versioned_docs/version-2.8/reference-guides/user-settings/user-preferences.md b/versioned_docs/version-2.8/reference-guides/user-settings/user-preferences.md new file mode 100644 index 00000000000..3a36abd6e18 --- /dev/null +++ b/versioned_docs/version-2.8/reference-guides/user-settings/user-preferences.md @@ -0,0 +1,62 @@ +--- +title: User Preferences +--- + +You can set preferences to personalize your Rancher experience. To change preference settings: + +1. Click on your user avatar in the upper right corner. +1. Click **Preferences**. + +## Language + +Choose your language for the Rancher UI. Options include: + +- English +- 简体中文 (Simplified Chinese) + +## Theme + +Choose your background color for the Rancher UI. If you choose **Auto**, the background color changes from light to dark at 6 PM, and then changes back at 6 AM. + +## Login Landing Page + +Choose the page displayed after logging in. Options include: + +- The home page. +- The area you last visited. +- A specific cluster of your choosing. + +## Display Settings + +Choose how certain information is displayed: + +- Date format +- Time format +- Table rows per page +- Number of clusters to show in side menu + +## Confirmation Setting + +_Available as of v2.7.2_ + +Choose whether to ask for confirmation when scaling down node pools. + +## Advanced Features + +- Enable "View in API". +- Show system Namespaces managed by Rancher (not intended for editing or deletion). +- Enable Dark/Light Theme keyboard shortcut toggle (shift+T). +- Hide All Type Descriptions. +- Enable Extension developer features. + +![](/img/user-preferences-2-7.png) + +## YAML Editor Key Mapping + +- Normal human (Sublime) +- Emacs +- Vim + +## Helm Charts + +Choose whether to display released Helm charts only or to include prerelease versions as well. A version is considered to be a prerelease if it adheres to [the specification](https://semver.org/#spec-item-9) defined by [Semantic Versioning 2.0.0](https://semver.org/). For example, a Helm chart with a version of `0.1.3-dev.12ab4f` requires `Include Prerelease Versions` to be selected in order to be displayed. diff --git a/versioned_docs/version-2.8/security/security-scan/security-scan.md b/versioned_docs/version-2.8/security/security-scan/security-scan.md new file mode 100644 index 00000000000..061d0af8edd --- /dev/null +++ b/versioned_docs/version-2.8/security/security-scan/security-scan.md @@ -0,0 +1,9 @@ +--- +title: Security Scans +--- + + + + + +The documentation about CIS security scans has moved [here.](../../pages-for-subheaders/cis-scan-guides.md) diff --git a/versioned_docs/version-2.8/shared-files/_cluster-capabilities-table.md b/versioned_docs/version-2.8/shared-files/_cluster-capabilities-table.md new file mode 100644 index 00000000000..b74973f5a67 --- /dev/null +++ b/versioned_docs/version-2.8/shared-files/_cluster-capabilities-table.md @@ -0,0 +1,23 @@ +| Action | Rancher Launched Kubernetes Clusters | EKS, GKE and AKS Clusters1 | Other Hosted Kubernetes Clusters | Non-EKS or GKE Registered Clusters | +| --- | --- | ---| ---|----| +| [Using kubectl and a kubeconfig file to Access a Cluster](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) | ✓ | ✓ | ✓ | ✓ | +| [Managing Cluster Members](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) | ✓ | ✓ | ✓ | ✓ | +| [Editing and Upgrading Clusters](../pages-for-subheaders/cluster-configuration.md) | ✓ | ✓ | ✓ | ✓2 | +| [Managing Nodes](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md) | ✓ | ✓ | ✓ | ✓3 | +| [Managing Persistent Volumes and Storage Classes](../pages-for-subheaders/create-kubernetes-persistent-storage.md) | ✓ | ✓ | ✓ | ✓ | +| [Managing Projects, Namespaces and Workloads](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) | ✓ | ✓ | ✓ | ✓ | +| [Using App Catalogs](../pages-for-subheaders/helm-charts-in-rancher.md) | ✓ | ✓ | ✓ | ✓ | +| Configuring Tools ([Alerts, Notifiers, Monitoring](../pages-for-subheaders/monitoring-and-alerting.md), [Logging](../pages-for-subheaders/logging.md), [Istio](../pages-for-subheaders/istio.md)) | ✓ | ✓ | ✓ | ✓ | +| [Running Security Scans](../pages-for-subheaders/cis-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | +| [Ability to rotate certificates](../how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md) | ✓ | ✓ | | | +| Ability to [backup](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md) and [restore](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Rancher-launched clusters | ✓ | ✓ | | ✓4 | +| [Cleaning Kubernetes components when clusters are no longer reachable from Rancher](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) | ✓ | | | | +| [Configuring Pod Security Policies](../how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md) | ✓ | ✓ | || + +1. Registered EKS, GKE and AKS clusters have the same options available as EKS, GKE and AKS clusters created from the Rancher UI. The difference is that when a registered cluster is deleted from the Rancher UI, it is not destroyed. + +2. Cluster configuration options can't be edited for registered clusters, except for [K3s and RKE2 clusters.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md) + +3. For registered cluster nodes, the Rancher UI exposes the ability to cordon, drain, and edit the node. + +4. For registered clusters using etcd as a control plane, snapshots must be taken manually outside of the Rancher UI to use for backup and recovery. diff --git a/versioned_docs/version-2.8/shared-files/_common-ports-table.md b/versioned_docs/version-2.8/shared-files/_common-ports-table.md new file mode 100644 index 00000000000..1835beba031 --- /dev/null +++ b/versioned_docs/version-2.8/shared-files/_common-ports-table.md @@ -0,0 +1,19 @@ +| Protocol | Port | Description | +|:--------: |:----------------: |---------------------------------------------------------------------------------- | +| TCP | 22 | Node driver SSH provisioning | +| TCP | 179 | Calico BGP Port | +| TCP | 2376 | Node driver Docker daemon TLS port | +| TCP | 2379 | etcd client requests | +| TCP | 2380 | etcd peer communication | +| UDP | 8472 | Canal/Flannel VXLAN overlay networking | +| UDP | 4789 | Flannel VXLAN overlay networking on Windows cluster | +| TCP | 8443 | Rancher webhook | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 9100 | Default port required by Monitoring to scrape metrics from Linux node-exporters | +| TCP | 9443 | Rancher webhook | +| TCP | 9796 | Default port required by Monitoring to scrape metrics from Windows node-exporters | +| TCP | 6783 | Weave Port | +| UDP | 6783-6784 | Weave UDP Ports | +| TCP | 10250 | Metrics server communication with all nodes API | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | +| TCP/UDP | 30000-32767 | NodePort port range | diff --git a/versioned_docs/version-2.8/troubleshooting/general-troubleshooting.md b/versioned_docs/version-2.8/troubleshooting/general-troubleshooting.md new file mode 100644 index 00000000000..77ab7e24806 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/general-troubleshooting.md @@ -0,0 +1,48 @@ +--- +title: General Troubleshooting +--- + + + + + +This section contains information to help you troubleshoot issues when using Rancher. + +- [Kubernetes components](../pages-for-subheaders/kubernetes-components.md) + + If you need help troubleshooting core Kubernetes cluster components like: + * `etcd` + * `kube-apiserver` + * `kube-controller-manager` + * `kube-scheduler` + * `kubelet` + * `kube-proxy` + * `nginx-proxy` + +- [Kubernetes resources](other-troubleshooting-tips/kubernetes-resources.md) + + Options for troubleshooting Kubernetes resources like Nodes, Ingress Controller and Rancher Agents are described in this section. + +- [Networking](other-troubleshooting-tips/networking.md) + + Steps to troubleshoot networking issues can be found here. + +- [DNS](other-troubleshooting-tips/dns.md) + + When you experience name resolution issues in your cluster. + +- [Troubleshooting Rancher installed on Kubernetes](other-troubleshooting-tips/rancher-ha.md) + + If you experience issues with your [Rancher server installed on Kubernetes](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + +- [Logging](other-troubleshooting-tips/logging.md) + + Read more about what log levels can be configured and how to configure a log level. + +- [User ID Tracking in Audit Logs](other-troubleshooting-tips/user-id-tracking-in-audit-logs.md) + + Read more about how a Rancher Admin can trace an event from the Rancher audit logs and into the Kubernetes audit logs using the external Identity Provider username. + +- [Expired Webhook Certificates](other-troubleshooting-tips/expired-webhook-certificate-rotation.md) + + Read more about how to rotate a Rancher webhook certificate secret after it expires on an annual basis. diff --git a/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md new file mode 100644 index 00000000000..84877bc8fab --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md @@ -0,0 +1,55 @@ +--- +title: Troubleshooting Controlplane Nodes +--- + + + + + +This section applies to nodes with the `controlplane` role. + +## Check if the Controlplane Containers are Running + +There are three specific containers launched on nodes with the `controlplane` role: + +* `kube-apiserver` +* `kube-controller-manager` +* `kube-scheduler` + +The containers should have status **Up**. The duration shown after **Up** is the time the container has been running. + +``` +docker ps -a -f=name='kube-apiserver|kube-controller-manager|kube-scheduler' +``` + +Example output: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +26c7159abbcc rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-apiserver +f3d287ca4549 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-scheduler +bdf3898b8063 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-controller-manager +``` + +## Controlplane Container Logging + +:::note + +If you added multiple nodes with the `controlplane` role, both `kube-controller-manager` and `kube-scheduler` use a leader election process to determine the leader. Only the current leader will log the performed actions. See [Kubernetes leader election](../other-troubleshooting-tips/kubernetes-resources.md#kubernetes-leader-election) how to retrieve the current leader. + +::: + +The logging of the containers can contain information on what the problem could be. + +``` +docker logs kube-apiserver +docker logs kube-controller-manager +docker logs kube-scheduler +``` + +## RKE2 Server Logging + +If Rancher provisions an RKE2 cluster that can't communicate with Rancher, you can run this command on a server node in the downstream cluster to get the RKE2 server logs: + +``` +journalctl -u rke2-server -f +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md new file mode 100644 index 00000000000..9785d8e4f68 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md @@ -0,0 +1,369 @@ +--- +title: Troubleshooting etcd Nodes +--- + + + + + +This section contains commands and tips for troubleshooting nodes with the `etcd` role. + + +## Checking if the etcd Container is Running + +The container for etcd should have status **Up**. The duration shown after **Up** is the time the container has been running. + +``` +docker ps -a -f=name=etcd$ +``` + +Example output: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +605a124503b9 rancher/coreos-etcd:v3.2.18 "/usr/local/bin/et..." 2 hours ago Up 2 hours etcd +``` + +## etcd Container Logging + +The logging of the container can contain information on what the problem could be. + +``` +docker logs etcd +``` +| Log | Explanation | +|-----|------------------| +| `health check for peer xxx could not connect: dial tcp IP:2380: getsockopt: connection refused` | A connection to the address shown on port 2380 cannot be established. Check if the etcd container is running on the host with the address shown. | +| `xxx is starting a new election at term x` | The etcd cluster has lost its quorum and is trying to establish a new leader. This can happen when the majority of the nodes running etcd go down/unreachable. | +| `connection error: desc = "transport: Error while dialing dial tcp 0.0.0.0:2379: i/o timeout"; Reconnecting to {0.0.0.0:2379 0 }` | The host firewall is preventing network communication. | +| `rafthttp: request cluster ID mismatch` | The node with the etcd instance logging `rafthttp: request cluster ID mismatch` is trying to join a cluster that has already been formed with another peer. The node should be removed from the cluster, and re-added. | +| `rafthttp: failed to find member` | The cluster state (`/var/lib/etcd`) contains wrong information to join the cluster. The node should be removed from the cluster, the state directory should be cleaned and the node should be re-added. + +## etcd Cluster and Connectivity Checks + +The address where etcd is listening depends on the address configuration of the host etcd is running on. If an internal address is configured for the host etcd is running on, the endpoint for `etcdctl` needs to be specified explicitly. If any of the commands respond with `Error: context deadline exceeded`, the etcd instance is unhealthy (either quorum is lost or the instance is not correctly joined in the cluster) + +### Check etcd Members on all Nodes + +Output should contain all the nodes with the `etcd` role and the output should be identical on all nodes. + +Command: +``` +docker exec etcd etcdctl member list +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list" +``` + +Example output: +``` +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +``` + +### Check Endpoint Status + +The values for `RAFT TERM` should be equal and `RAFT INDEX` should be not be too far apart from each other. + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table +``` + +Example output: +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | 333ef673fc4add56 | 3.2.18 | 24 MB | false | 72 | 66887 | +| https://IP:2379 | 5feed52d940ce4cf | 3.2.18 | 24 MB | true | 72 | 66887 | +| https://IP:2379 | db6b3bdb559a848d | 3.2.18 | 25 MB | false | 72 | 66887 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### Check Endpoint Health + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint health +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl endpoint health --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") +``` + +Example output: +``` +https://IP:2379 is healthy: successfully committed proposal: took = 2.113189ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.649963ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.451201ms +``` + +### Check Connectivity on Port TCP/2379 + +Command: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health" + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +Example output: +``` +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +``` + +### Check Connectivity on Port TCP/2380 + +Command: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +Example output: +``` +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +``` + +## etcd Alarms + +etcd will trigger alarms, for instance when it runs out of space. + +Command: +``` +docker exec etcd etcdctl alarm list +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +Example output when NOSPACE alarm is triggered: +``` +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +``` + +## etcd Space Errors + +Related error messages are `etcdserver: mvcc: database space exceeded` or `applying raft message exceeded backend quota`. Alarm `NOSPACE` will be triggered. + +Resolutions: + +- [Compact the Keyspace](#compact-the-keyspace) +- [Defrag All etcd Members](#defrag-all-etcd-members) +- [Check Endpoint Status](#check-endpoint-status) +- [Disarm Alarm](#disarm-alarm) + +### Compact the Keyspace + +Command: +``` +rev=$(docker exec etcd etcdctl endpoint status --write-out json | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*') +docker exec etcd etcdctl compact "$rev" +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +rev=$(docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT endpoint status --write-out json | egrep -o '\"revision\":[0-9]*' | egrep -o '[0-9]*'") +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT compact \"$rev\"" +``` + +Example output: +``` +compacted revision xxx +``` + +### Defrag All etcd Members + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl defrag +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl defrag --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','")" +``` + +Example output: +``` +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +``` + +### Check Endpoint Status + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table" +``` + +Example output: +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | e973e4419737125 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | 4a509c997b26c206 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | b217e736575e9dd3 | 3.2.18 | 553 kB | true | 32 | 2449410 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### Disarm Alarm + +After verifying that the DB size went down after compaction and defragmenting, the alarm needs to be disarmed for etcd to allow writes again. + +Command: +``` +docker exec etcd etcdctl alarm list +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm disarm" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +Example output: +``` +docker exec etcd etcdctl alarm list +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +## Configure Log Level + +:::note + +You can no longer dynamically change the log level in etcd v3.5 or later. + +::: + +### etcd v3.5 And Later + +To configure the log level for etcd, edit the cluster YAML: + +``` +services: + etcd: + extra_args: + log-level: "debug" +``` + +### etcd v3.4 And Earlier + +In earlier etcd versions, you can use the API to dynamically change the log level. Configure debug logging using the commands below: + +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +To reset the log level back to the default (`INFO`), you can use the following command. + +Command: +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +## etcd Content + +If you want to investigate the contents of your etcd, you can either watch streaming events or you can query etcd directly, see below for examples. + +### Watch Streaming Events + +Command: +``` +docker exec etcd etcdctl watch --prefix /registry +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT watch --prefix /registry +``` + +If you only want to see the affected keys (and not the binary data), you can append `| grep -a ^/registry` to the command to filter for keys only. + +### Query etcd Directly + +Command: +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT get /registry --prefix=true --keys-only +``` + +You can process the data to get a summary of count per key, using the command below: + +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only | grep -v ^$ | awk -F'/' '{ if ($3 ~ /cattle.io/) {h[$3"/"$4]++} else { h[$3]++ }} END { for(k in h) print h[k], k }' | sort -nr +``` + +## Replacing Unhealthy etcd Nodes + +When a node in your etcd cluster becomes unhealthy, the recommended approach is to fix or remove the failed or unhealthy node before adding a new etcd node to the cluster. diff --git a/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md new file mode 100644 index 00000000000..6c381ee5bb0 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md @@ -0,0 +1,72 @@ +--- +title: Troubleshooting nginx-proxy +--- + + + + + +The `nginx-proxy` container is deployed on every node that does not have the `controlplane` role. It provides access to all the nodes with the `controlplane` role by dynamically generating the NGINX configuration based on available nodes with the `controlplane` role. + +## Check if the Container is Running + +The container is called `nginx-proxy` and should have status `Up`. The duration shown after `Up` is the time the container has been running. + +``` +docker ps -a -f=name=nginx-proxy +``` + +Example output: + +``` +docker ps -a -f=name=nginx-proxy +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +c3e933687c0e rancher/rke-tools:v0.1.15 "nginx-proxy CP_HO..." 3 hours ago Up 3 hours nginx-proxy +``` + +## Check Generated NGINX Configuration + +The generated configuration should include the IP addresses of the nodes with the `controlplane` role. The configuration can be checked using the following command: + +``` +docker exec nginx-proxy cat /etc/nginx/nginx.conf +``` + +Example output: +``` +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + + server ip_of_controlplane_node1:6443; + + server ip_of_controlplane_node2:6443; + + } + + server { + listen 6443; + proxy_pass kube_apiserver; + proxy_timeout 30; + proxy_connect_timeout 2s; + + } + +} +``` + +## nginx-proxy Container Logging + +The logging of the containers can contain information on what the problem could be. + +``` +docker logs nginx-proxy +``` \ No newline at end of file diff --git a/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md new file mode 100644 index 00000000000..5a5de6c0156 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md @@ -0,0 +1,38 @@ +--- +title: Troubleshooting Worker Nodes and Generic Components +--- + + + + + +This section applies to every node as it includes components that run on nodes with any role. + +## Check if the Containers are Running + +There are two specific containers launched on nodes with the `worker` role: + +* kubelet +* kube-proxy + +The containers should have status `Up`. The duration shown after `Up` is the time the container has been running. + +``` +docker ps -a -f=name='kubelet|kube-proxy' +``` + +Example output: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +158d0dcc33a5 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-proxy +a30717ecfb55 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kubelet +``` + +## Container Logging + +The logging of the containers can contain information on what the problem could be. + +``` +docker logs kubelet +docker logs kube-proxy +``` diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/dns.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/dns.md new file mode 100644 index 00000000000..af1108b6f80 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/dns.md @@ -0,0 +1,224 @@ +--- +title: DNS +--- + + + + + +The commands/steps listed on this page can be used to check name resolution issues in your cluster. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` for Rancher HA) or are using the embedded kubectl via the UI. + +Before running the DNS checks, check the [default DNS provider](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#default-dns-provider) for your cluster and make sure that [the overlay network is functioning correctly](networking.md#check-if-overlay-network-is-functioning-correctly) as this can also be the reason why DNS resolution (partly) fails. + +### Check if DNS pods are running + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns +``` + +Example output when using CoreDNS: +``` +NAME READY STATUS RESTARTS AGE +coredns-799dffd9c4-6jhlz 1/1 Running 0 76m +``` + +Example output when using kube-dns: +``` +NAME READY STATUS RESTARTS AGE +kube-dns-5fd74c7488-h6f7n 3/3 Running 0 4m13s +``` + +### Check if the DNS service is present with the correct cluster-ip + +``` +kubectl -n kube-system get svc -l k8s-app=kube-dns +``` + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/kube-dns ClusterIP 10.43.0.10 53/UDP,53/TCP 4m13s +``` + +### Check if domain names are resolving + +Check if internal cluster names are resolving (in this example, `kubernetes.default`), the IP shown after `Server:` should be the same as the `CLUSTER-IP` from the `kube-dns` service. + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup kubernetes.default +``` + +Example output: +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: kubernetes.default +Address 1: 10.43.0.1 kubernetes.default.svc.cluster.local +pod "busybox" deleted +``` + +Check if external names are resolving (in this example, `www.google.com`) + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup www.google.com +``` + +Example output: +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: www.google.com +Address 1: 2a00:1450:4009:80b::2004 lhr35s04-in-x04.1e100.net +Address 2: 216.58.211.100 ams15s32-in-f4.1e100.net +pod "busybox" deleted +``` + +If you want to check resolving of domain names on all of the hosts, execute the following steps: + +1. Save the following file as `ds-dnstest.yml` + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: dnstest + spec: + selector: + matchLabels: + name: dnstest + template: + metadata: + labels: + name: dnstest + spec: + tolerations: + - operator: Exists + containers: + - image: busybox:1.28 + imagePullPolicy: Always + name: alpine + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + ``` + +2. Launch it using `kubectl create -f ds-dnstest.yml` +3. Wait until `kubectl rollout status ds/dnstest -w` returns: `daemon set "dnstest" successfully rolled out`. +4. Configure the environment variable `DOMAIN` to a fully qualified domain name (FQDN) that the host should be able to resolve (`www.google.com` is used as an example) and run the following command to let each container on every host resolve the configured domain name (it's a single line command). + + ``` + export DOMAIN=www.google.com; echo "=> Start DNS resolve test"; kubectl get pods -l name=dnstest --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do kubectl exec $pod -- /bin/sh -c "nslookup $DOMAIN > /dev/null 2>&1"; RC=$?; if [ $RC -ne 0 ]; then echo $host cannot resolve $DOMAIN; fi; done; echo "=> End DNS resolve test" + ``` + +5. When this command has finished running, the output indicating everything is correct is: + + ``` + => Start DNS resolve test + => End DNS resolve test + ``` + +If you see error in the output, that means that the mentioned host(s) is/are not able to resolve the given FQDN. + +Example error output of a situation where host with IP 209.97.182.150 had the UDP ports blocked. + +``` +=> Start DNS resolve test +command terminated with exit code 1 +209.97.182.150 cannot resolve www.google.com +=> End DNS resolve test +``` + +Cleanup the alpine DaemonSet by running `kubectl delete ds/dnstest`. + +### CoreDNS specific + +#### Check CoreDNS logging + +``` +kubectl -n kube-system logs -l k8s-app=kube-dns +``` + +#### Check configuration + +CoreDNS configuration is stored in the configmap `coredns` in the `kube-system` namespace. + +``` +kubectl -n kube-system get configmap coredns -o go-template={{.data.Corefile}} +``` + +#### Check upstream nameservers in resolv.conf + +By default, the configured nameservers on the host (in `/etc/resolv.conf`) will be used as upstream nameservers for CoreDNS. You can check this file on the host or run the following Pod with `dnsPolicy` set to `Default`, which will inherit the `/etc/resolv.conf` from the host it is running on. + +``` +kubectl run -i --restart=Never --rm test-${RANDOM} --image=ubuntu --overrides='{"kind":"Pod", "apiVersion":"v1", "spec": {"dnsPolicy":"Default"}}' -- sh -c 'cat /etc/resolv.conf' +``` + +#### Enable query logging + +Enabling query logging can be done by enabling the [log plugin](https://coredns.io/plugins/log/) in the Corefile configuration in the configmap `coredns`. You can do so by using `kubectl -n kube-system edit configmap coredns` or use the command below to replace the configuration in place: + +``` +kubectl get configmap -n kube-system coredns -o json | sed -e 's_loadbalance_log\\n loadbalance_g' | kubectl apply -f - +``` + +All queries will now be logged and can be checked using the command in [Check CoreDNS logging](#check-coredns-logging). + +### kube-dns specific + +#### Check upstream nameservers in kubedns container + +By default, the configured nameservers on the host (in `/etc/resolv.conf`) will be used as upstream nameservers for kube-dns. Sometimes the host will run a local caching DNS nameserver, which means the address in `/etc/resolv.conf` will point to an address in the loopback range (`127.0.0.0/8`) which will be unreachable by the container. In case of Ubuntu 18.04, this is done by `systemd-resolved`. We detect if `systemd-resolved` is running, and will automatically use the `/etc/resolv.conf` file with the correct upstream nameservers (which is located at `/run/systemd/resolve/resolv.conf`). + +Use the following command to check the upstream nameservers used by the kubedns container: + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do echo "Pod ${pod} on host ${host}"; kubectl -n kube-system exec $pod -c kubedns cat /etc/resolv.conf; done +``` + +Example output: +``` +Pod kube-dns-667c7cb9dd-z4dsf on host x.x.x.x +nameserver 1.1.1.1 +nameserver 8.8.4.4 +``` + +If the output shows an address in the loopback range (`127.0.0.0/8`), you can correct this in two ways: + +* Make sure the correct nameservers are listed in `/etc/resolv.conf` on your nodes in the cluster, please consult your operating system documentation on how to do this. Make sure you execute this before provisioning a cluster, or reboot the nodes after making the modification. +* Configure the `kubelet` to use a different file for resolving names, by using `extra_args` as shown below (where `/run/resolvconf/resolv.conf` is the file with the correct nameservers): + +``` +services: + kubelet: + extra_args: + resolv-conf: "/run/resolvconf/resolv.conf" +``` + +:::note + +As the `kubelet` is running inside a container, the path for files located in `/etc` and `/usr` are in `/host/etc` and `/host/usr` inside the `kubelet` container. + +::: + +See [Editing Cluster as YAML](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#editing-clusters-with-yaml) how to apply this change. When the provisioning of the cluster has finished, you have to remove the kube-dns pod to activate the new setting in the pod: + +``` +kubectl delete pods -n kube-system -l k8s-app=kube-dns +pod "kube-dns-5fd74c7488-6pwsf" deleted +``` + +Try to resolve name again using [Check if domain names are resolving](#check-if-domain-names-are-resolving). + +If you want to check the kube-dns configuration in your cluster (for example, to check if there are different upstream nameservers configured), you can run the following command to list the kube-dns configuration: + +``` +kubectl -n kube-system get configmap kube-dns -o go-template='{{range $key, $value := .data}}{{ $key }}{{":"}}{{ $value }}{{"\n"}}{{end}}' +``` + +Example output: +``` +upstreamNameservers:["1.1.1.1"] +``` diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md new file mode 100644 index 00000000000..106479c0bb7 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md @@ -0,0 +1,32 @@ +--- +title: Rotation of Expired Webhook Certificates +--- + + + + + +For Rancher versions that have `rancher-webhook` installed, certain versions created certificates that will expire after one year. It will be necessary for you to rotate your webhook certificate if the certificate did not renew. + +In Rancher v2.6.3 and up, rancher-webhook deployments will automatically renew their TLS certificate when it is within 30 or fewer days of its expiration date. If you are using v2.6.2 or below, there are two methods to work around this issue: + +##### 1. Users with cluster access, run the following commands: +``` +kubectl delete secret -n cattle-system cattle-webhook-tls +kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io --ignore-not-found=true rancher.cattle.io +kubectl delete pod -n cattle-system -l app=rancher-webhook +``` + +##### 2. Users with no cluster access via `kubectl`: + +1. Delete the `cattle-webhook-tls` secret in the `cattle-system` namespace in the local cluster. + +2. Delete the `rancher.cattle.io` mutating webhook + +3. Delete the `rancher-webhook` pod in the `cattle-system` namespace in the local cluster. + +:::note + +The webhook certificate expiration issue is not specific to `cattle-webhook-tls` as listed in the examples. You will fill in your expired certificate secret accordingly. + +::: \ No newline at end of file diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md new file mode 100644 index 00000000000..f1e6b8f8594 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md @@ -0,0 +1,259 @@ +--- +title: Kubernetes Resources +--- + + + + + +The commands/steps listed on this page can be used to check the most important Kubernetes resources and apply to [Rancher Launched Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` for Rancher HA) or are using the embedded kubectl via the UI. + + +## Nodes + +### Get nodes + +Run the command below and check the following: + +- All nodes in your cluster should be listed, make sure there is not one missing. +- All nodes should have the **Ready** status (if not in **Ready** state, check the `kubelet` container logs on that node using `docker logs kubelet`) +- Check if all nodes report the correct version. +- Check if OS/Kernel/Docker values are shown as expected (possibly you can relate issues due to upgraded OS/Kernel/Docker) + + +``` +kubectl get nodes -o wide +``` + +Example output: + +``` +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +controlplane-0 Ready controlplane 31m v1.13.5 138.68.188.91 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +etcd-0 Ready etcd 31m v1.13.5 138.68.180.33 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +worker-0 Ready worker 30m v1.13.5 139.59.179.88 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +``` + +### Get node conditions + +Run the command below to list nodes with [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' +``` + +Run the command below to list nodes with [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) that are active that could prevent normal operation. + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{if ne .type "Ready"}}{{if eq .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{else}}{{if ne .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{": "}}{{.status}}{{"\n"}}{{end}}{{end}}{{end}}{{end}}' +``` + +Example output: + +``` +worker-0: DiskPressure:True +``` + +## Kubernetes leader election + +### Kubernetes Controller Manager leader + +The leader is determined by a leader election process. After the leader has been determined, the leader (`holderIdentity`) is saved in the `kube-controller-manager` endpoint (in this example, `controlplane-0`). + +``` +kubectl -n kube-system get endpoints kube-controller-manager -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +### Kubernetes Scheduler leader + +The leader is determined by a leader election process. After the leader has been determined, the leader (`holderIdentity`) is saved in the `kube-scheduler` endpoint (in this example, `controlplane-0`). + +``` +kubectl -n kube-system get endpoints kube-scheduler -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +## Ingress Controller + +The default Ingress Controller is NGINX and is deployed as a DaemonSet in the `ingress-nginx` namespace. The pods are only scheduled to nodes with the `worker` role. + +Check if the pods are running on all nodes: + +``` +kubectl -n ingress-nginx get pods -o wide +``` + +Example output: + +``` +kubectl -n ingress-nginx get pods -o wide +NAME READY STATUS RESTARTS AGE IP NODE +default-http-backend-797c5bc547-kwwlq 1/1 Running 0 17m x.x.x.x worker-1 +nginx-ingress-controller-4qd64 1/1 Running 0 14m x.x.x.x worker-1 +nginx-ingress-controller-8wxhm 1/1 Running 0 13m x.x.x.x worker-0 +``` + +If a pod is unable to run (Status is not **Running**, Ready status is not showing `1/1` or you see a high count of Restarts), check the pod details, logs and namespace events. + +### Pod details + +``` +kubectl -n ingress-nginx describe pods -l app=ingress-nginx +``` + +### Pod container logs + +The below command can show the logs of all the pods labeled "app=ingress-nginx", but it will display only 10 lines of log because of the restrictions of the `kubectl logs` command. Refer to `--tail` of `kubectl logs -h` for more information. + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +If the full log is needed, specify the pod name in the trailing command: + +``` +kubectl -n ingress-nginx logs +``` + +### Namespace events + +``` +kubectl -n ingress-nginx get events +``` + +### Debug logging + +To enable debug logging: + +``` +kubectl -n ingress-nginx patch ds nginx-ingress-controller --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--v=5"}]' +``` + +### Check configuration + +Retrieve generated configuration in each pod: + +``` +kubectl -n ingress-nginx get pods -l app=ingress-nginx --no-headers -o custom-columns=.NAME:.metadata.name | while read pod; do kubectl -n ingress-nginx exec $pod -- cat /etc/nginx/nginx.conf; done +``` + +## Rancher agents + +Communication to the cluster (Kubernetes API via `cattle-cluster-agent`) and communication to the nodes (cluster provisioning via `cattle-node-agent`) is done through Rancher agents. + +#### cattle-node-agent + +Check if the cattle-node-agent pods are present on each node, have status **Running** and don't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +Check logging of a specific cattle-node-agent pod or all cattle-node-agent pods: + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +Check if the cattle-cluster-agent pod is present in the cluster, has status **Running** and doesn't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +Check logging of cattle-cluster-agent pod: + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` + +## Jobs and Pods + +### Check that pods or jobs have status **Running**/**Completed** + +To check, run the command: + +``` +kubectl get pods --all-namespaces +``` + +If a pod is not in **Running** state, you can dig into the root cause by running: + +### Describe pod + +``` +kubectl describe pod POD_NAME -n NAMESPACE +``` + +### Pod container logs + +``` +kubectl logs POD_NAME -n NAMESPACE +``` + +If a job is not in **Completed** state, you can dig into the root cause by running: + +### Describe job + +``` +kubectl describe job JOB_NAME -n NAMESPACE +``` + +### Logs from the containers of pods of the job + +``` +kubectl logs -l job-name=JOB_NAME -n NAMESPACE +``` + +### Evicted pods + +Pods can be evicted based on [eviction signals](https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/#eviction-policy). + +Retrieve a list of evicted pods (podname and namespace): + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' +``` + +To delete all evicted pods: + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace delete pod $epod; done +``` + +Retrieve a list of evicted pods, scheduled node and the reason: + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace get pod $epod -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName,MSG:.status.message; done +``` + +### Job does not complete + +If you have enabled Istio, and you are having issues with a Job you deployed not completing, you will need to add an annotation to your pod using [these steps.](../../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md) + +Since Istio Sidecars run indefinitely, a Job cannot be considered complete even after its task has completed. This is a temporary workaround and will disable Istio for any traffic to/from the annotated Pod. Keep in mind this may not allow you to continue to use a Job for integration testing, as the Job will not have access to the service mesh. \ No newline at end of file diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/logging.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/logging.md new file mode 100644 index 00000000000..f4ba45226bc --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/logging.md @@ -0,0 +1,96 @@ +--- +title: Logging +--- + + + + + +## Log Levels + +The following log levels are used in Rancher: + +| Name | Description | +|---------|-------------| +| `info` | Logs informational messages. This is the default log level. | +| `debug` | Logs more detailed messages that can be used to debug. | +| `trace` | Logs very detailed messages on internal functions. This is very verbose and can contain sensitive information. | + +### How to Configure a Log Level + +#### Kubernetes Install + +* Configure debug log level + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set debug; done +OK +OK +OK +$ kubectl -n cattle-system logs -l app=rancher -c rancher +``` + +* Configure info log level + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set info; done +OK +OK +OK +``` + +#### Docker Install + +* Configure debug log level + +``` +$ docker exec -ti loglevel --set debug +OK +$ docker logs -f +``` + +* Configure info log level + +``` +$ docker exec -ti loglevel --set info +OK +``` + +## Rancher Machine Debug Logs +If you need to troubleshoot the creation of objects in your infrastructure provider of choice, `rancher-machine` +debug logs might be helpful to you. + +It's possible to enable debug logs for `rancher-machine` by setting environment variables when launching Rancher. + +The `CATTLE_WHITELIST_ENVVARS` environment variable allows users to whitelist specific environment variables to be +passed down to `rancher-machine` during provisioning. + +The `MACHINE_DEBUG` variable enables debug logs in `rancher-machine`. + +Thus, by setting `MACHINE_DEBUG=true` and adding `MACHINE_DEBUG` to the default list of variables in +`CATTLE_WHITELIST_ENVVARS` (e.g. `CATTLE_WHITELIST_ENVVARS=HTTP_PROXY,HTTPS_PROXY,NO_PROXY,MACHINE_DEBUG`) it is +possible to enable debug logs in `rancher-machine` when provisioning RKE1, RKE2 and k3s clusters. + +:::caution + +Just like the `trace` log level above, `rancher-machine` debug logs can contain sensitive information. + +::: + + +## Cattle-cluster-agent Debug Logs + +The `cattle-cluster-agent` log levels can be set when you initialize downstream clusters. + +When you create a cluster under **Cluster Configuration > Agent Environment Vars** you can set variables to define the log level. +- Trace-level logging: Set `CATTLE_TRACE` or `RANCHER_TRACE` to `true` + +- Debug-level logging: Set `CATTLE_DEBUG` or `RANCHER_DEBUG` to `true` + +:::caution + +The `cattle-cluster-agent` debug logs may contain sensitive information. + +::: diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/networking.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/networking.md new file mode 100644 index 00000000000..bea850ffe13 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/networking.md @@ -0,0 +1,125 @@ +--- +title: Networking +--- + + + + + +The commands/steps listed on this page can be used to check networking related issues in your cluster. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` for Rancher HA) or are using the embedded kubectl via the UI. + +### Double check if all the required ports are opened in your (host) firewall + +Double check if all the [required ports](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) are opened in your (host) firewall. The overlay network uses UDP in comparison to all other required ports which are TCP. +### Check if overlay network is functioning correctly + +The pod can be scheduled to any of the hosts you used for your cluster, but that means that the NGINX ingress controller needs to be able to route the request from `NODE_1` to `NODE_2`. This happens over the overlay network. If the overlay network is not functioning, you will experience intermittent TCP/HTTP connection failures due to the NGINX ingress controller not being able to route to the pod. + +To test the overlay network, you can launch the following `DaemonSet` definition. This will run a `swiss-army-knife` container on every host (image was developed by Rancher engineers and can be found here: https://github.com/rancherlabs/swiss-army-knife), which we will use to run a `ping` test between containers on all hosts. + +:::note + +This container [does not support ARM nodes](https://github.com/leodotcloud/swiss-army-knife/issues/18), such as a Raspberry Pi. This will be seen in the pod logs as `exec user process caused: exec format error`. + +::: + +1. Save the following file as `overlaytest.yml` + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: overlaytest + spec: + selector: + matchLabels: + name: overlaytest + template: + metadata: + labels: + name: overlaytest + spec: + tolerations: + - operator: Exists + containers: + - image: rancherlabs/swiss-army-knife + imagePullPolicy: Always + name: overlaytest + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + + ``` + +2. Launch it using `kubectl create -f overlaytest.yml` +3. Wait until `kubectl rollout status ds/overlaytest -w` returns: `daemon set "overlaytest" successfully rolled out`. +4. Run the following script, from the same location. It will have each `overlaytest` container on every host ping each other: + ``` + #!/bin/bash + echo "=> Start network overlay test" + kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.metadata.name}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read spod shost + do kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.status.podIP}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read tip thost + do kubectl --request-timeout='10s' exec $spod -c overlaytest -- /bin/sh -c "ping -c2 $tip > /dev/null 2>&1" + RC=$? + if [ $RC -ne 0 ] + then echo FAIL: $spod on $shost cannot reach pod IP $tip on $thost + else echo $shost can reach $thost + fi + done + done + echo "=> End network overlay test" + ``` + +5. When this command has finished running, it will output the state of each route: + + ``` + => Start network overlay test + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.7.3 on wk2 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.0.5 on cp1 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.2.12 on wk1 + command terminated with exit code 1 + FAIL: overlaytest-v4qkl on cp1 cannot reach pod IP 10.42.7.3 on wk2 + cp1 can reach cp1 + cp1 can reach wk1 + command terminated with exit code 1 + FAIL: overlaytest-xpxwp on wk1 cannot reach pod IP 10.42.7.3 on wk2 + wk1 can reach cp1 + wk1 can reach wk1 + => End network overlay test + ``` + If you see error in the output, there is some issue with the route between the pods on the two hosts. In the above output the node `wk2` has no connectivity over the overlay network. This could be because the [required ports](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for overlay networking are not opened for `wk2`. +6. You can now clean up the DaemonSet by running `kubectl delete ds/overlaytest`. + + +### Check if MTU is correctly configured on hosts and on peering/tunnel appliances/devices + +When the MTU is incorrectly configured (either on hosts running Rancher, nodes in created/imported clusters or on appliances/devices in between), error messages will be logged in Rancher and in the agents, similar to: + +* `websocket: bad handshake` +* `Failed to connect to proxy` +* `read tcp: i/o timeout` + +See [Google Cloud VPN: MTU Considerations](https://cloud.google.com/vpn/docs/concepts/mtu-considerations#gateway_mtu_vs_system_mtu) for an example how to configure MTU correctly when using Google Cloud VPN between Rancher and cluster nodes. + +### Resolved issues + +#### Overlay network broken when using Canal/Flannel due to missing node annotations + +| | | +|------------|------------| +| GitHub issue | [#13644](https://github.com/rancher/rancher/issues/13644) | +| Resolved in | v2.1.2 | + +To check if your cluster is affected, the following command will list nodes that are broken (this command requires `jq` to be installed): + +``` +kubectl get nodes -o json | jq '.items[].metadata | select(.annotations["flannel.alpha.coreos.com/public-ip"] == null or .annotations["flannel.alpha.coreos.com/kube-subnet-manager"] == null or .annotations["flannel.alpha.coreos.com/backend-type"] == null or .annotations["flannel.alpha.coreos.com/backend-data"] == null) | .name' +``` + +If there is no output, the cluster is not affected. diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/rancher-ha.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/rancher-ha.md new file mode 100644 index 00000000000..8917f80da4d --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/rancher-ha.md @@ -0,0 +1,83 @@ +--- +title: Rancher HA +--- + + + + + +The commands/steps listed on this page can be used to check your Rancher Kubernetes Installation. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml`). + +### Check Rancher pods + +Rancher pods are deployed as a Deployment in the `cattle-system` namespace. + +Check if the pods are running on all nodes: + +``` +kubectl -n cattle-system get pods -l app=rancher -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +rancher-7dbd7875f7-n6t5t 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qbj5k 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qw7wb 1/1 Running 0 8m x.x.x.x x.x.x.x +``` + +If a pod is unable to run (Status is not **Running**, Ready status is not showing `1/1` or you see a high count of Restarts), check the pod details, logs and namespace events. + +#### Pod details + +``` +kubectl -n cattle-system describe pods -l app=rancher +``` + +#### Pod container logs + +``` +kubectl -n cattle-system logs -l app=rancher +``` + +#### Namespace events + +``` +kubectl -n cattle-system get events +``` + +### Check ingress + +Ingress should have the correct `HOSTS` (showing the configured FQDN) and `ADDRESS` (host address(es) it will be routed to). + +``` +kubectl -n cattle-system get ingress +``` + +Example output: + +``` +NAME HOSTS ADDRESS PORTS AGE +rancher rancher.yourdomain.com x.x.x.x,x.x.x.x,x.x.x.x 80, 443 2m +``` + +### Check ingress controller logs + +When accessing your configured Rancher FQDN does not show you the UI, check the ingress controller logging to see what happens when you try to access Rancher: + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +### Leader election + +The leader is determined by a leader election process. After the leader has been determined, the leader (`holderIdentity`) is saved in the `cattle-controllers` ConfigMap (in this example, `rancher-7dbd7875f7-qbj5k`). + +``` +kubectl -n kube-system get configmap cattle-controllers -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"rancher-7dbd7875f7-qbj5k","leaseDurationSeconds":45,"acquireTime":"2019-04-04T11:53:12Z","renewTime":"2019-04-04T12:24:08Z","leaderTransitions":0} +``` + diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/registered-clusters.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/registered-clusters.md new file mode 100644 index 00000000000..cce0e089621 --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/registered-clusters.md @@ -0,0 +1,71 @@ +--- +title: Registered Clusters +--- + + + + + +The commands/steps listed on this page can be used to check clusters that you are registering or that are registered in Rancher. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kubeconfig_from_imported_cluster.yml`) + +### Rancher agents + +Communication to the cluster (Kubernetes API via cattle-cluster-agent) and communication to the nodes is done through Rancher agents. + +If the cattle-cluster-agent cannot connect to the configured `server-url`, the cluster will remain in **Pending** state, showing `Waiting for full cluster configuration`. + +#### cattle-node-agent + +:::note + +cattle-node-agents are only present in clusters created in Rancher with RKE. + +::: + +Check if the cattle-node-agent pods are present on each node, have status **Running** and don't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +Check logging of a specific cattle-node-agent pod or all cattle-node-agent pods: + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +Check if the cattle-cluster-agent pod is present in the cluster, has status **Running** and doesn't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +Check logging of cattle-cluster-agent pod: + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` diff --git a/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md new file mode 100644 index 00000000000..6a25ae1565e --- /dev/null +++ b/versioned_docs/version-2.8/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md @@ -0,0 +1,26 @@ +--- +title: User ID Tracking in Audit Logs +--- + + + + + +The following audit logs are used in Rancher to track events occuring on the local and downstream clusters: + +* [Kubernetes Audit Logs](https://rancher.com/docs/rke/latest/en/config-options/audit-log/) +* [Rancher API Audit Logs](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) + +Audit logs in Rancher v2.6 have been enhanced to include the external Identity Provider name (common name of the user in the external Auth provider) in both the Rancher and downstream Kubernetes audit logs. + +Before v2.6, a Rancher Admin could not trace an event from the Rancher audit logs and into the Kubernetes audit logs without knowing the mapping of the external Identity Provider username to the userId (`u-xXXX`) used in Rancher. +To know this mapping, the cluster admins needed to have access to Rancher API, UI, and the local management cluster. + +Now with this feature, a downstream cluster admin should be able to look at the Kubernetes audit logs and know which specific external Identity Provider (IDP) user performed an action without needing to view anything in Rancher. +If the audit logs are shipped off of the cluster, a user of the logging system should be able to identify the user in the external Identity Provider system. +A Rancher Admin should now be able to view Rancher audit logs and follow through to the Kubernetes audit log by using the external Identity Provider username. + +### Feature Description + +- When Kubernetes Audit logs are enabled on the downstream cluster, in each event that is logged, the external Identity Provider's username is now logged for each request, at the "metadata" level. +- When Rancher API Audit logs are enabled on the Rancher installation, the external Identity Provider's username is also logged now at the `auditLog.level=1` for each request that hits the Rancher API server, including the login requests. diff --git a/versioned_docs/version-latest/cluster-provisioning/rke-clusters/options/options.md b/versioned_docs/version-latest/cluster-provisioning/rke-clusters/options/options.md new file mode 100644 index 00000000000..da8df215bd2 --- /dev/null +++ b/versioned_docs/version-latest/cluster-provisioning/rke-clusters/options/options.md @@ -0,0 +1,9 @@ +--- +title: RKE Cluster Configuration +--- + + + + + +This page has moved [here.](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) diff --git a/versioned_docs/version-latest/contribute-to-rancher.md b/versioned_docs/version-latest/contribute-to-rancher.md new file mode 100644 index 00000000000..b894ec64e66 --- /dev/null +++ b/versioned_docs/version-latest/contribute-to-rancher.md @@ -0,0 +1,140 @@ +--- +title: Contributing to Rancher +--- + + + + + +Learn about the repositories used for Rancher and Rancher docs, how to build Rancher repositories, and what information to include when you file an issue. + +For more detailed information on how to contribute to the development of Rancher projects, refer to the [Rancher Developer Wiki](https://github.com/rancher/rancher/wiki). The wiki has resources on many topics, including the following: + +- How to set up the Rancher development environment and run tests +- The typical flow of an issue through the development lifecycle +- Coding guidelines and development best practices +- Debugging and troubleshooting +- Developing the Rancher API + +On the Rancher Users Slack, the channel for developers is **#developer**. + +## Rancher Docs + +If you have suggestions for the documentation on this website, [open](https://github.com/rancher/rancher-docs/issues/new/choose) an issue in the main [Rancher docs](https://github.com/rancher/rancher-docs) repository. This repo contains documentation for Rancher v2.0 and later. + +See the [Rancher docs README](https://github.com/rancher/rancher-docs#readme) for more details on contributing to and building the Rancher v2.x docs repo. + +For documentation describing Rancher v1.6 and earlier, see the [Rancher 1.x docs](https://github.com/rancher/rancher.github.io) repo, which contains source files for https://rancher.com/docs/rancher/v1.6/en/. + +## Rancher Repositories + +All of repositories are located within our main GitHub organization. There are many repositories used for Rancher, but we'll provide descriptions of some of the main ones used in Rancher. + +Repository | URL | Description +-----------|-----|------------- +Rancher | https://github.com/rancher/rancher | This repository is the main source code for Rancher 2.x. +Types | https://github.com/rancher/types | This repository is the repository that has all the API types for Rancher 2.x. +API Framework | https://github.com/rancher/norman | This repository is an API framework for building Rancher style APIs backed by Kubernetes Custom Resources. +User Interface | https://github.com/rancher/dashboard/ | This repository is the source of the Dashboard UI. +(Rancher) Docker Machine | https://github.com/rancher/machine | This repository is the source of the Docker Machine binary used when using Node Drivers. This is a fork of the `docker/machine` repository. +machine-package | https://github.com/rancher/machine-package | This repository is used to build the Rancher Docker Machine binary. +kontainer-engine | https://github.com/rancher/kontainer-engine | This repository is the source of kontainer-engine, the tool to provision hosted Kubernetes clusters. +RKE repository | https://github.com/rancher/rke | This repository is the source of Rancher Kubernetes Engine, the tool to provision Kubernetes clusters on any machine. +CLI | https://github.com/rancher/cli | This repository is the source code for the Rancher CLI used in Rancher 2.x. +(Rancher) Helm repository | https://github.com/rancher/helm | This repository is the source of the packaged Helm binary. This is a fork of the `helm/helm` repository. +Telemetry repository | https://github.com/rancher/telemetry | This repository is the source for the Telemetry binary. +loglevel repository | https://github.com/rancher/loglevel | This repository is the source of the loglevel binary, used to dynamically change log levels. + +To see all libraries/projects used in Rancher, see the [`go.mod` file](https://github.com/rancher/rancher/blob/master/go.mod) in the `rancher/rancher` repository. + +![Rancher diagram](/img/ranchercomponentsdiagram-2.6.svg)
+Rancher components used for provisioning/managing Kubernetes clusters. + +### Building Rancher Repositories + +Every repository should have a Makefile and can be built using the `make` command. The `make` targets are based on the scripts in the `/scripts` directory in the repository, and each target will use [Dapper](https://github.com/rancher/dapper) to run the target in an isolated environment. The `Dockerfile.dapper` will be used for this process, and includes all the necessary build tooling needed. + +The default target is `ci`, and will run `./scripts/validate`, `./scripts/build`, `./scripts/test` and `./scripts/package`. The resulting binaries of the build will be in `./build/bin` and are usually also packaged in a Docker image. + +### Rancher Bugs, Issues or Questions + +If you find any bugs or are having any trouble, please search the [reported issue](https://github.com/rancher/rancher/issues) as someone may have experienced the same issue or we are actively working on a solution. + +If you can't find anything related to your issue, contact us by [filing an issue](https://github.com/rancher/rancher/issues/new). Though we have many repositories related to Rancher, we want the bugs filed in the Rancher repository so we won't miss them! If you want to ask a question or ask fellow users about an use case, we suggest creating a post on the [Rancher Forums](https://forums.rancher.com). + +#### Checklist for Filing Issues + +Please follow this checklist when filing an issue which will helps us investigate and fix the issue. More info means more data we can use to determine what is causing the issue or what might be related to the issue. + +:::note + +For large amounts of data, please use [GitHub Gist](https://gist.github.com/) or similar and link the created resource in the issue. + +::: + +:::note Important: + +Please remove any sensitive data as it will be publicly viewable. + +::: + +- **Resources:** Provide as much as detail as possible on the used resources. As the source of the issue can be many things, including as much of detail as possible helps to determine the root cause. See some examples below: + - **Hosts:** What specifications does the host have, like CPU/memory/disk, what cloud does it happen on, what Amazon Machine Image are you using, what DigitalOcean droplet are you using, what image are you provisioning that we can rebuild or use when we try to reproduce + - **Operating System:** What operating system are you using? Providing specifics helps here like the output of `cat /etc/os-release` for exact OS release and `uname -r` for exact kernel used + - **Docker:** What Docker version are you using, how did you install it? Most of the details of Docker can be found by supplying output of `docker version` and `docker info` + - **Environment:** Are you in a proxy environment, are you using recognized CA/self signed certificates, are you using an external loadbalancer + - **Rancher:** What version of Rancher are you using, this can be found on the bottom left of the UI or be retrieved from the image tag you are running on the host + - **Clusters:** What kind of cluster did you create, how did you create it, what did you specify when you were creating it +- **Steps to reproduce the issue:** Provide as much detail on how you got into the reported situation. This helps the person to reproduce the situation you are in. + - Provide manual steps or automation scripts used to get from a newly created setup to the situation you reported. +- **Logs:** Provide data/logs from the used resources. + - Rancher + - Docker install + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher:|rancher/rancher " | awk '{ print $1 }') + ``` + - Kubernetes install using `kubectl` + + :::note + + Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` if Rancher is installed on a Kubernetes cluster) or are using the embedded kubectl via the UI. + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + -l app=rancher \ + --timestamps=true + ``` + - Docker install using `docker` on each of the nodes in the RKE cluster + + ``` + docker logs \ + --timestamps \ + $(docker ps | grep -E "rancher/rancher@|rancher_rancher" | awk '{ print $1 }') + ``` + - Kubernetes Install with RKE Add-On + + :::note + + Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` if the Rancher server is installed on a Kubernetes cluster) or are using the embedded kubectl via the UI. + + ::: + + ``` + kubectl -n cattle-system \ + logs \ + --timestamps=true \ + -f $(kubectl --kubeconfig $KUBECONFIG get pods -n cattle-system -o json | jq -r '.items[] | select(.spec.containers[].name="cattle-server") | .metadata.name') + ``` + - System logging (these might not all exist, depending on operating system) + - `/var/log/messages` + - `/var/log/syslog` + - `/var/log/kern.log` + - Docker daemon logging (these might not all exist, depending on operating system) + - `/var/log/docker.log` +- **Metrics:** If you are experiencing performance issues, please provide as much of data (files or screenshots) of metrics which can help determining what is going on. If you have an issue related to a machine, it helps to supply output of `top`, `free -m`, `df` which shows processes/memory/disk usage. diff --git a/versioned_docs/version-latest/faq/container-network-interface-providers.md b/versioned_docs/version-latest/faq/container-network-interface-providers.md new file mode 100644 index 00000000000..490713c7b5a --- /dev/null +++ b/versioned_docs/version-latest/faq/container-network-interface-providers.md @@ -0,0 +1,208 @@ +--- +title: Container Network Interface (CNI) Providers +description: Learn about Container Network Interface (CNI), the CNI providers Rancher provides, the features they offer, and how to choose a provider for you +--- + + + + + +## What is CNI? + +CNI (Container Network Interface), a [Cloud Native Computing Foundation project](https://cncf.io/), consists of a specification and libraries for writing plugins to configure network interfaces in Linux containers, along with a number of plugins. CNI concerns itself only with network connectivity of containers and removing allocated resources when the container is deleted. + +Kubernetes uses CNI as an interface between network providers and Kubernetes pod networking. + +![CNI Logo](/img/cni-logo.png) + +For more information visit [CNI GitHub project](https://github.com/containernetworking/cni). + +## What Network Models are Used in CNI? + +CNI network providers implement their network fabric using either an encapsulated network model such as Virtual Extensible Lan ([VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan)) or an unencapsulated network model such as Border Gateway Protocol ([BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol)). + +### What is an Encapsulated Network? + +This network model provides a logical Layer 2 (L2) network encapsulated over the existing Layer 3 (L3) network topology that spans the Kubernetes cluster nodes. With this model you have an isolated L2 network for containers without needing routing distribution, all at the cost of minimal overhead in terms of processing and increased IP package size, which comes from an IP header generated by overlay encapsulation. Encapsulation information is distributed by UDP ports between Kubernetes workers, interchanging network control plane information about how MAC addresses can be reached. Common encapsulation used in this kind of network model is VXLAN, Internet Protocol Security (IPSec), and IP-in-IP. + +In simple terms, this network model generates a kind of network bridge extended between Kubernetes workers, where pods are connected. + +This network model is used when an extended L2 bridge is preferred. This network model is sensitive to L3 network latencies of the Kubernetes workers. If datacenters are in distinct geolocations, be sure to have low latencies between them to avoid eventual network segmentation. + +CNI network providers using this network model include Flannel, Canal, Weave, and Cilium. By default, Calico is not using this model, but it can be configured to do so. + +![Encapsulated Network](/img/encapsulated-network.png) + +### What is an Unencapsulated Network? + +This network model provides an L3 network to route packets between containers. This model doesn't generate an isolated l2 network, nor generates overhead. These benefits come at the cost of Kubernetes workers having to manage any route distribution that's needed. Instead of using IP headers for encapsulation, this network model uses a network protocol between Kubernetes workers to distribute routing information to reach pods, such as [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol). + +In simple terms, this network model generates a kind of network router extended between Kubernetes workers, which provides information about how to reach pods. + +This network model is used when a routed L3 network is preferred. This mode dynamically updates routes at the OS level for Kubernetes workers. It's less sensitive to latency. + +CNI network providers using this network model include Calico and Cilium. Cilium may be configured with this model although it is not the default mode. + +![Unencapsulated Network](/img/unencapsulated-network.png) + +## What CNI Providers are Provided by Rancher? + +### RKE Kubernetes clusters + +Out-of-the-box, Rancher provides the following CNI network providers for RKE Kubernetes clusters: Canal, Flannel, Calico, and Weave. + +You can choose your CNI network provider when you create new Kubernetes clusters from Rancher. + +#### Canal + +![Canal Logo](/img/canal-logo.png) + +Canal is a CNI network provider that gives you the best of Flannel and Calico. It allows users to easily deploy Calico and Flannel networking together as a unified networking solution, combining Calico’s network policy enforcement with the rich superset of Calico (unencapsulated) and/or Flannel (encapsulated) network connectivity options. + +In Rancher, Canal is the default CNI network provider combined with Flannel and VXLAN encapsulation. + +Kubernetes workers should open UDP port `8472` (VXLAN) and TCP port `9099` (health checks). If using Wireguard, you should open UDP ports `51820` and `51821`. For more details, refer to [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md). + +![](/img/canal-diagram.png) + +For more information, see the [Canal GitHub Page.](https://github.com/projectcalico/canal) + +#### Flannel + +![Flannel Logo](/img/flannel-logo.png) + +Flannel is a simple and easy way to configure L3 network fabric designed for Kubernetes. Flannel runs a single binary agent named flanneld on each host, which is responsible for allocating a subnet lease to each host out of a larger, preconfigured address space. Flannel uses either the Kubernetes API or etcd directly to store the network configuration, the allocated subnets, and any auxiliary data (such as the host's public IP). Packets are forwarded using one of several backend mechanisms, with the default encapsulation being [VXLAN](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#vxlan). + +Encapsulated traffic is unencrypted by default. Flannel provides two solutions for encryption: + +* [IPSec](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#ipsec), which makes use of [strongSwan](https://www.strongswan.org/) to establish encrypted IPSec tunnels between Kubernetes workers. It is an experimental backend for encryption. +* [WireGuard](https://github.com/flannel-io/flannel/blob/master/Documentation/backends.md#wireguard), which is a more faster-performing alternative to strongSwan. + +Kubernetes workers should open UDP port `8472` (VXLAN). See [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +![Flannel Diagram](/img/flannel-diagram.png) + +For more information, see the [Flannel GitHub Page](https://github.com/flannel-io/flannel). + +#### Weave + +![Weave Logo](/img/weave-logo.png) + +Weave enables networking and network policy in Kubernetes clusters across the cloud. Additionally, it support encrypting traffic between the peers. + +Kubernetes workers should open TCP port `6783` (control port), UDP port `6783` and UDP port `6784` (data ports). See the [port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +For more information, see the following pages: + +- [Weave Net Official Site](https://www.weave.works/) + +### RKE2 Kubernetes clusters + +Out-of-the-box, Rancher provides the following CNI network providers for RKE2 Kubernetes clusters: [Canal](#canal) (see above section), Calico, and Cilium. + +You can choose your CNI network provider when you create new Kubernetes clusters from Rancher. + +#### Calico + +![Calico Logo](/img/calico-logo.png) + +Calico enables networking and network policy in Kubernetes clusters across the cloud. By default, Calico uses a pure, unencapsulated IP network fabric and policy engine to provide networking for your Kubernetes workloads. Workloads are able to communicate over both cloud infrastructure and on-prem using BGP. + +Calico also provides a stateless IP-in-IP or VXLAN encapsulation mode that can be used, if necessary. Calico also offers policy isolation, allowing you to secure and govern your Kubernetes workloads using advanced ingress and egress policies. + +Kubernetes workers should open TCP port `179` if using BGP or UDP port `4789` if using VXLAN encapsulation. In addition, TCP port `5473` is needed when using Typha. See [the port requirements for user clusters](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for more details. + +:::note Important: + +In Rancher v2.6.3, Calico probes fail on Windows nodes upon RKE2 installation. Note that this issue is resolved in v2.6.4. + +- To work around this issue, first navigate to `https:///v3/settings/windows-rke2-install-script`. + +- There, change the current setting: `https://raw.githubusercontent.com/rancher/wins/v0.1.3/install.ps1` to this new setting: `https://raw.githubusercontent.com/rancher/rke2/master/windows/rke2-install.ps1`. + +::: + +![Calico Diagram](/img/calico-diagram.svg) + +For more information, see the following pages: + +- [Project Calico Official Site](https://www.projectcalico.org/) +- [Project Calico GitHub Page](https://github.com/projectcalico/calico) + +#### Cilium + +![Cilium Logo](/img/cilium-logo.png) + +Cilium enables networking and network policies (L3, L4, and L7) in Kubernetes. By default, Cilium uses eBPF technologies to route packets inside the node and VXLAN to send packets to other nodes. Unencapsulated techniques can also be configured. + +Cilium recommends kernel versions greater than 5.2 to be able to leverage the full potential of eBPF. Kubernetes workers should open TCP port `8472` for VXLAN and TCP port `4240` for health checks. In addition, ICMP 8/0 must be enabled for health checks. For more information, check [Cilium System Requirements](https://docs.cilium.io/en/latest/operations/system_requirements/#firewall-requirements). + +##### Ingress Routing Across Nodes in Cilium +
+By default, Cilium does not allow pods to contact pods on other nodes. To work around this, enable the ingress controller to route requests across nodes with a `CiliumNetworkPolicy`. + +After selecting the Cilium CNI and enabling Project Network Isolation for your new cluster, configure as follows: + +``` +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: hn-nodes + namespace: default +spec: + endpointSelector: {} + ingress: + - fromEntities: + - remote-node +``` + +## CNI Features by Provider + +The following table summarizes the different features available for each CNI network provider provided by Rancher. + +| Provider | Network Model | Route Distribution | Network Policies | Mesh | External Datastore | Encryption | Ingress/Egress Policies | +| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| Canal | Encapsulated (VXLAN) | No | Yes | No | K8s API | Yes | Yes | +| Flannel | Encapsulated (VXLAN) | No | No | No | K8s API | Yes | No | +| Calico | Encapsulated (VXLAN,IPIP) OR Unencapsulated | Yes | Yes | Yes | Etcd and K8s API | Yes | Yes | +| Weave | Encapsulated | Yes | Yes | Yes | No | Yes | Yes | +| Cilium | Encapsulated (VXLAN) | Yes | Yes | Yes | Etcd and K8s API | Yes | Yes | + +- Network Model: Encapsulated or unencapsulated. For more information, see [What Network Models are Used in CNI?](#what-network-models-are-used-in-cni) + +- Route Distribution: An exterior gateway protocol designed to exchange routing and reachability information on the Internet. BGP can assist with pod-to-pod networking between clusters. This feature is a must on unencapsulated CNI network providers, and it is typically done by BGP. If you plan to build clusters split across network segments, route distribution is a feature that's nice-to-have. + +- Network Policies: Kubernetes offers functionality to enforce rules about which services can communicate with each other using network policies. This feature is stable as of Kubernetes v1.7 and is ready to use with certain networking plugins. + +- Mesh: This feature allows service-to-service networking communication between distinct Kubernetes clusters. + +- External Datastore: CNI network providers with this feature need an external datastore for its data. + +- Encryption: This feature allows cyphered and secure network control and data planes. + +- Ingress/Egress Policies: This feature allows you to manage routing control for both Kubernetes and non-Kubernetes communications. + + +## CNI Community Popularity + +The following table summarizes different GitHub metrics to give you an idea of each project's popularity and activity. This data was collected in January 2022. + +| Provider | Project | Stars | Forks | Contributors | +| ---- | ---- | ---- | ---- | ---- | +| Canal | https://github.com/projectcalico/canal | 679 | 100 | 21 | +| Flannel | https://github.com/flannel-io/flannel | 7k | 2.5k | 185 | +| Calico | https://github.com/projectcalico/calico | 3.1k | 741 | 224 | +| Weave | https://github.com/weaveworks/weave/ | 6.2k | 635 | 84 | +| Cilium | https://github.com/cilium/cilium | 10.6k | 1.3k | 352 | + +
+ +## Which CNI Provider Should I Use? + +It depends on your project needs. There are many different providers, which each have various features and options. There isn't one provider that meets everyone's needs. + +Canal is the default CNI network provider. We recommend it for most use cases. It provides encapsulated networking for containers with Flannel, while adding Calico network policies that can provide project/namespace isolation in terms of networking. + +## How can I configure a CNI network provider? + +Please see [Cluster Options](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) on how to configure a network provider for your cluster. For more advanced configuration options, please see how to configure your cluster using a [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) and the options for [Network Plug-ins](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/). diff --git a/versioned_docs/version-latest/faq/deprecated-features-in-v2.5.md b/versioned_docs/version-latest/faq/deprecated-features-in-v2.5.md new file mode 100644 index 00000000000..86313d48238 --- /dev/null +++ b/versioned_docs/version-latest/faq/deprecated-features-in-v2.5.md @@ -0,0 +1,30 @@ +--- +title: Deprecated Features in Rancher +--- + + + + + +### What is Rancher's Deprecation policy? + +We have published our official deprecation policy in the support [terms of service](https://rancher.com/support-maintenance-terms). + +### Where can I find out which features have been deprecated in Rancher? + +Rancher will publish deprecated features as part of the [release notes](https://github.com/rancher/rancher/releases) for Rancher found on GitHub. Please consult the following patch releases for deprecated features: + +| Patch Version | Release Date | +|---------------|---------------| +| [2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) | Aug 31, 2021 | +| [2.6.1](https://github.com/rancher/rancher/releases/tag/v2.6.1) | Oct 11, 2021 | +| [2.6.2](https://github.com/rancher/rancher/releases/tag/v2.6.2) | Oct 19, 2021 | +| [2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) | Dec 21, 2021 | +| [2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | Mar 31, 2022 | +| [2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) | May 12, 2022 | +| [2.6.6](https://github.com/rancher/rancher/releases/tag/v2.6.6) | Jun 30, 2022 | + + +### What can I expect when a feature is marked for deprecation? + +In the release where functionality is marked as "Deprecated", it will still be available and supported allowing upgrades to follow the usual procedure. Once upgraded, users/admins should start planning to move away from the deprecated functionality before upgrading to the release it marked as removed. The recommendation for new deployments is to not use the deprecated feature. \ No newline at end of file diff --git a/versioned_docs/version-latest/faq/dockershim.md b/versioned_docs/version-latest/faq/dockershim.md new file mode 100644 index 00000000000..23b982f7b30 --- /dev/null +++ b/versioned_docs/version-latest/faq/dockershim.md @@ -0,0 +1,49 @@ +--- +title: Dockershim +--- + + + + + +The Dockershim is the CRI compliant layer between the Kubelet and the Docker daemon. As part of the Kubernetes 1.20 release, the [deprecation of the in-tree Dockershim was announced](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/). Removal is currently scheduled for Kubernetes 1.24. For more information on the deprecation and its timelines, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). + +RKE clusters, starting with Kubernetes 1.21, now support the external Dockershim to continue leveraging Docker as the CRI runtime. We now implement the upstream open source community Dockershim announced by [Mirantis and Docker](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) to ensure RKE clusters can continue to leverage Docker. + +To enable the external Dockershim, configure the following option. + +``` +enable_cri_dockerd: true +``` + +For users looking to use another container runtime, Rancher has the edge-focused K3s and datacenter-focused RKE2 Kubernetes distributions that use containerd as the default runtime. Imported RKE2 and K3s Kubernetes clusters can then be upgraded and managed through Rancher even after the removal of in-tree Dockershim in Kubernetes 1.24. + +### FAQ + +
+ +Q. Do I have to upgrade Rancher to get Rancher’s support of the upstream Dockershim? + +The upstream support of Dockershim begins for RKE in Kubernetes 1.21. You will need to be on Rancher 2.6 or above to have support for RKE with Kubernetes 1.21. See our [support matrix](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/) for details. + +
+ +Q. I am currently on RKE with Kubernetes 1.20. Do I need to upgrade to RKE with Kubernetes 1.21 sooner to avoid being out of support for Dockershim? + +A. The version of Dockershim in RKE with Kubernetes 1.20 will continue to work and is not scheduled for removal upstream until Kubernetes 1.24. It will only emit a warning of its future deprecation, which Rancher has mitigated in RKE with Kubernetes 1.21. You can plan your upgrade to Kubernetes 1.21 as you would normally, but should consider enabling the external Dockershim by Kubernetes 1.22. The external Dockershim will need to be enabled before upgrading to Kubernetes 1.24, at which point the existing implementation will be removed. + +For more information on the deprecation and its timeline, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). + +
+ +Q: What are my other options if I don’t want to depend on the Dockershim? + +A: You can use a runtime like containerd with Kubernetes that does not require Dockershim support. RKE2 or K3s are two options for doing this. + +
+ +Q: If I am already using RKE1 and want to switch to RKE2, what are my migration options? + +A: Rancher is exploring the possibility of an in-place upgrade path. Alternatively you can always migrate workloads from one cluster to another using kubectl. + +
diff --git a/versioned_docs/version-latest/faq/general-faq.md b/versioned_docs/version-latest/faq/general-faq.md new file mode 100644 index 00000000000..417875a6c6c --- /dev/null +++ b/versioned_docs/version-latest/faq/general-faq.md @@ -0,0 +1,73 @@ +--- +title: General FAQ +--- + + + + + +This FAQ is a work in progress designed to answers the questions our users most frequently ask about Rancher v2.x. + +See [Technical FAQ](technical-items.md), for frequently asked technical questions. + +
+ +**Does Rancher v2.x support Docker Swarm and Mesos as environment types?** + +When creating an environment in Rancher v2.x, Swarm and Mesos will no longer be standard options you can select. However, both Swarm and Mesos will continue to be available as Catalog applications you can deploy. It was a tough decision to make but, in the end, it came down to adoption. For example, out of more than 15,000 clusters, only about 200 or so are running Swarm. + +
+ +**Is it possible to manage Azure Kubernetes Services with Rancher v2.x?** + +Yes. + +
+ +**Does Rancher support Windows?** + +Rancher supports Windows Server 1809 containers. For details on how to set up a cluster with Windows worker nodes, refer to the section on [configuring custom clusters for Windows.](../pages-for-subheaders/use-windows-clusters.md) + +
+ +**Does Rancher support Istio?** + +Rancher supports [Istio.](../pages-for-subheaders/istio.md) + +Furthermore, Istio is implemented in our micro-PaaS "Rio", which works on Rancher 2.x along with any CNCF compliant Kubernetes cluster. You can read more about it [here](https://rio.io/) + +
+ +**Will Rancher v2.x support Hashicorp's Vault for storing secrets?** + +Secrets management is on our roadmap but we haven't assigned it to a specific release yet. + +
+ +**Does Rancher v2.x support RKT containers as well?** + +At this time, we only support Docker. + +
+ +**Does Rancher v2.x support Calico, Contiv, Contrail, Flannel, Weave net, etc., for embedded and registered Kubernetes?** + +Out-of-the-box, Rancher provides the following CNI network providers for Kubernetes clusters: Canal, Flannel, Calico and Weave. Always refer to the [Rancher Support Matrix](https://rancher.com/support-maintenance-terms/) for details about what is officially supported. + +
+ +**Are you planning on supporting Traefik for existing setups?** + +We don't currently plan on providing embedded Traefik support, but we're still exploring load-balancing approaches. + +
+ +**Can I import OpenShift Kubernetes clusters into v2.x?** + +Our goal is to run any upstream Kubernetes clusters. Therefore, Rancher v2.x should work with OpenShift, but we haven't tested it yet. + +
+ +**Are you going to integrate Longhorn?** + +Yes. Longhorn was integrated into Rancher v2.5+. diff --git a/versioned_docs/version-latest/faq/install-and-configure-kubectl.md b/versioned_docs/version-latest/faq/install-and-configure-kubectl.md new file mode 100644 index 00000000000..9bcb56bc3d2 --- /dev/null +++ b/versioned_docs/version-latest/faq/install-and-configure-kubectl.md @@ -0,0 +1,33 @@ +--- +title: Installing and Configuring kubectl +--- + + + + + +`kubectl` is a CLI utility for running commands against Kubernetes clusters. It's required for many maintenance and administrative tasks in Rancher 2.x. + +### Installation + +See [kubectl Installation](https://kubernetes.io/docs/tasks/tools/install-kubectl/) for installation on your operating system. + +### Configuration + +When you create a Kubernetes cluster with RKE, RKE creates a `kube_config_cluster.yml` in the local directory that contains credentials to connect to your new cluster with tools like `kubectl` or `helm`. + +You can copy this file as `$HOME/.kube/config` or if you are working with multiple Kubernetes clusters, set the `KUBECONFIG` environmental variable to the path of `kube_config_cluster.yml`. + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +Test your connectivity with `kubectl` and see if you can get the list of nodes back. + +``` +kubectl get nodes + NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.10.1 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.10.1 +``` diff --git a/versioned_docs/version-latest/faq/rancher-is-no-longer-needed.md b/versioned_docs/version-latest/faq/rancher-is-no-longer-needed.md new file mode 100644 index 00000000000..ce33edd6aec --- /dev/null +++ b/versioned_docs/version-latest/faq/rancher-is-no-longer-needed.md @@ -0,0 +1,65 @@ +--- +title: Rancher is No Longer Needed +--- + + + + + +This page is intended to answer questions about what happens if you don't want Rancher anymore, if you don't want a cluster to be managed by Rancher anymore, or if the Rancher server is deleted. + + +### If the Rancher server is deleted, what happens to the workloads in my downstream clusters? + +If Rancher is ever deleted or unrecoverable, all workloads in the downstream Kubernetes clusters managed by Rancher will continue to function as normal. + +### If the Rancher server is deleted, how do I access my downstream clusters? + +The capability to access a downstream cluster without Rancher depends on the type of cluster and the way that the cluster was created. To summarize: + +- **Registered clusters:** The cluster will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. +- **Hosted Kubernetes clusters:** If you created the cluster in a cloud-hosted Kubernetes provider such as EKS, GKE, or AKS, you can continue to manage the cluster using your provider's cloud credentials. +- **RKE clusters:** To access an [RKE cluster,](../pages-for-subheaders/launch-kubernetes-with-rancher.md) the cluster must have the [authorized cluster endpoint](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled, and you must have already downloaded the cluster's kubeconfig file from the Rancher UI. (The authorized cluster endpoint is enabled by default for RKE clusters.) With this endpoint, you can access your cluster with kubectl directly instead of communicating through the Rancher server's [authentication proxy.](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#1-the-authentication-proxy) For instructions on how to configure kubectl to use the authorized cluster endpoint, refer to the section about directly accessing clusters with [kubectl and the kubeconfig file.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) These clusters will use a snapshot of the authentication as it was configured when Rancher was removed. + +### What if I don't want Rancher anymore? + +:::note + +The previously recommended [System Tools](../reference-guides/system-tools.md) has been deprecated since June 2022. + +::: + +If you [installed Rancher on a Kubernetes cluster,](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) remove Rancher by using the [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) tool. + +Uninstalling Rancher in high-availability (HA) mode will also remove all `helm-operation-*` pods and the following apps: + +- fleet +- fleet-agent +- rancher-operator +- rancher-webhook + +Custom resources (CRDs) and custom namespaces will still need to be manually removed. + +If you installed Rancher with Docker, you can uninstall Rancher by removing the single Docker container that it runs in. + +Imported clusters will not be affected by Rancher being removed. For other types of clusters, refer to the section on [accessing downstream clusters when Rancher is removed.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) + +### What if I don't want my registered cluster managed by Rancher? + +If a registered cluster is deleted from the Rancher UI, the cluster is detached from Rancher, leaving it intact and accessible by the same methods that were used to access it before it was registered in Rancher. + +To detach the cluster, + +1. In the upper left corner, click **☰ > Cluster Management**. +2. Go to the registered cluster that should be detached from Rancher and click **⋮ > Delete**. +3. Click **Delete**. + +**Result:** The registered cluster is detached from Rancher and functions normally outside of Rancher. + +### What if I don't want my RKE cluster or hosted Kubernetes cluster managed by Rancher? + +At this time, there is no functionality to detach these clusters from Rancher. In this context, "detach" is defined as the ability to remove Rancher components from the cluster and manage access to the cluster independently of Rancher. + +The capability to manage these clusters without Rancher is being tracked in this [issue.](https://github.com/rancher/rancher/issues/25234) + +For information about how to access clusters if the Rancher server is deleted, refer to [this section.](#if-the-rancher-server-is-deleted-how-do-i-access-my-downstream-clusters) diff --git a/versioned_docs/version-latest/faq/security.md b/versioned_docs/version-latest/faq/security.md new file mode 100644 index 00000000000..447fb53d7de --- /dev/null +++ b/versioned_docs/version-latest/faq/security.md @@ -0,0 +1,18 @@ +--- +title: Security + +--- + + + + + +**Is there a Hardening Guide?** + +The Hardening Guide is now located in the main [Security](../pages-for-subheaders/rancher-security.md) section. + +
+ +**What are the results of Rancher's Kubernetes cluster when it is CIS benchmarked?** + +We have run the CIS Kubernetes benchmark against a hardened Rancher Kubernetes cluster. The results of that assessment can be found in the main [Security](../pages-for-subheaders/rancher-security.md) section. diff --git a/versioned_docs/version-latest/faq/technical-items.md b/versioned_docs/version-latest/faq/technical-items.md new file mode 100644 index 00000000000..0aed55b5d0d --- /dev/null +++ b/versioned_docs/version-latest/faq/technical-items.md @@ -0,0 +1,180 @@ +--- +title: Technical +--- + + + + + +### How can I reset the administrator password? + +Docker Install: +``` +$ docker exec -ti reset-password +New password for default administrator (user-xxxxx): + +``` + +Kubernetes install (Helm): +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher --no-headers | head -1 | awk '{ print $1 }') -c rancher -- reset-password +New password for default administrator (user-xxxxx): + +``` + + + +### I deleted/deactivated the last admin, how can I fix it? +Docker Install: +``` +$ docker exec -ti ensure-default-admin +New default administrator (user-xxxxx) +New password for default administrator (user-xxxxx): + +``` + +Kubernetes install (Helm): +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl --kubeconfig $KUBECONFIG -n cattle-system exec $(kubectl --kubeconfig $KUBECONFIG -n cattle-system get pods -l app=rancher | grep '1/1' | head -1 | awk '{ print $1 }') -- ensure-default-admin +New password for default administrator (user-xxxxx): + +``` +### How can I enable debug logging? + +See [Troubleshooting: Logging](../troubleshooting/other-troubleshooting-tips/logging.md) + +### My ClusterIP does not respond to ping + +ClusterIP is a virtual IP, which will not respond to ping. Best way to test if the ClusterIP is configured correctly, is by using `curl` to access the IP and port to see if it responds. + +### Where can I manage Node Templates? + +Node Templates can be accessed by opening your account menu (top right) and selecting `Node Templates`. + +### Why is my Layer-4 Load Balancer in `Pending` state? + +The Layer-4 Load Balancer is created as `type: LoadBalancer`. In Kubernetes, this needs a cloud provider or controller that can satisfy these requests, otherwise these will be in `Pending` state forever. More information can be found on [Cloud Providers](../pages-for-subheaders/set-up-cloud-providers.md) or [Create External Load Balancer](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) + +### Where is the state of Rancher stored? + +- Docker Install: in the embedded etcd of the `rancher/rancher` container, located at `/var/lib/rancher`. +- Kubernetes install: in the etcd of the RKE cluster created to run Rancher. + +### How are the supported Docker versions determined? + +We follow the validated Docker versions for upstream Kubernetes releases. The validated versions can be found under [External Dependencies](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.10.md#external-dependencies) in the Kubernetes release CHANGELOG.md. + +### How can I access nodes created by Rancher? + +SSH keys to access the nodes created by Rancher can be downloaded via the **Nodes** view. Choose the node which you want to access and click on the vertical ⋮ button at the end of the row, and choose **Download Keys** as shown in the picture below. + +![Download Keys](/img/downloadsshkeys.png) + +Unzip the downloaded zip file, and use the file `id_rsa` to connect to you host. Be sure to use the correct username (`rancher` or `docker` for RancherOS, `ubuntu` for Ubuntu, `ec2-user` for Amazon Linux) + +``` +$ ssh -i id_rsa user@ip_of_node +``` + +### How can I automate task X in Rancher? + +The UI consists of static files, and works based on responses of the API. That means every action/task that you can execute in the UI, can be automated via the API. There are 2 ways to do this: + +* Visit `https://your_rancher_ip/v3` and browse the API options. +* Capture the API calls when using the UI (Most commonly used for this is [Chrome Developer Tools](https://developers.google.com/web/tools/chrome-devtools/#network) but you can use anything you like) + +### The IP address of a node changed, how can I recover? + +A node is required to have a static IP configured (or a reserved IP via DHCP). If the IP of a node has changed, you will have to remove it from the cluster and readd it. After it is removed, Rancher will update the cluster to the correct state. If the cluster is no longer in `Provisioning` state, the node is removed from the cluster. + +When the IP address of the node changed, Rancher lost connection to the node, so it will be unable to clean the node properly. See [Cleaning cluster nodes](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) to clean the node. + +When the node is removed from the cluster, and the node is cleaned, you can readd the node to the cluster. + +### How can I add additional arguments/binds/environment variables to Kubernetes components in a Rancher Launched Kubernetes cluster? + +You can add additional arguments/binds/environment variables via the [Config File](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) option in Cluster Options. For more information, see the [Extra Args, Extra Binds, and Extra Environment Variables](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/) in the RKE documentation or browse the [Example Cluster.ymls](https://rancher.com/docs/rke/latest/en/example-yamls/). + +### How do I check if my certificate chain is valid? + +Use the `openssl verify` command to validate your certificate chain: + +:::tip + +Configure `SSL_CERT_DIR` and `SSL_CERT_FILE` to a dummy location to make sure the OS-installed certificates are not used when verifying manually. + +::: + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +If you receive the error `unable to get local issuer certificate`, the chain is incomplete. This usually means that there is an intermediate CA certificate that issued your server certificate. If you already have this certificate, you can use it in the verification of the certificate like shown below: + +``` +SSL_CERT_DIR=/dummy SSL_CERT_FILE=/dummy openssl verify -CAfile ca.pem -untrusted intermediate.pem rancher.yourdomain.com.pem +rancher.yourdomain.com.pem: OK +``` + +If you have successfully verified your certificate chain, you should include needed intermediate CA certificates in the server certificate to complete the certificate chain for any connection made to Rancher (for example, by the Rancher agent). The order of the certificates in the server certificate file should be first the server certificate itself (contents of `rancher.yourdomain.com.pem`), followed by intermediate CA certificate(s) (contents of `intermediate.pem`). + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +If you still get errors during verification, you can retrieve the subject and the issuer of the server certificate using the following command: + +``` +openssl x509 -noout -subject -issuer -in rancher.yourdomain.com.pem +subject= /C=GB/ST=England/O=Alice Ltd/CN=rancher.yourdomain.com +issuer= /C=GB/ST=England/O=Alice Ltd/CN=Alice Intermediate CA +``` + +### How do I check `Common Name` and `Subject Alternative Names` in my server certificate? + +Although technically an entry in `Subject Alternative Names` is required, having the hostname in both `Common Name` and as entry in `Subject Alternative Names` gives you maximum compatibility with older browser/applications. + +Check `Common Name`: + +``` +openssl x509 -noout -subject -in cert.pem +subject= /CN=rancher.my.org +``` + +Check `Subject Alternative Names`: + +``` +openssl x509 -noout -in cert.pem -text | grep DNS + DNS:rancher.my.org +``` + +### Why does it take 5+ minutes for a pod to be rescheduled when a node has failed? + +This is due to a combination of the following default Kubernetes settings: + +* kubelet + * `node-status-update-frequency`: Specifies how often kubelet posts node status to master (default 10s) +* kube-controller-manager + * `node-monitor-period`: The period for syncing NodeStatus in NodeController (default 5s) + * `node-monitor-grace-period`: Amount of time which we allow running Node to be unresponsive before marking it unhealthy (default 40s) + * `pod-eviction-timeout`: The grace period for deleting pods on failed nodes (default 5m0s) + +See [Kubernetes: kubelet](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/) and [Kubernetes: kube-controller-manager](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-controller-manager/) for more information on these settings. + +In Kubernetes v1.13, the `TaintBasedEvictions` feature is enabled by default. See [Kubernetes: Taint based Evictions](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions) for more information. + +* kube-apiserver (Kubernetes v1.13 and up) + * `default-not-ready-toleration-seconds`: Indicates the tolerationSeconds of the toleration for notReady:NoExecute that is added by default to every pod that does not already have such a toleration. + * `default-unreachable-toleration-seconds`: Indicates the tolerationSeconds of the toleration for unreachable:NoExecute that is added by default to every pod that does not already have such a toleration. + +### Can I use keyboard shortcuts in the UI? + +Yes, most parts of the UI can be reached using keyboard shortcuts. For an overview of the available shortcuts, press `?` anywhere in the UI. diff --git a/versioned_docs/version-latest/faq/telemetry.md b/versioned_docs/version-latest/faq/telemetry.md new file mode 100644 index 00000000000..a510347e556 --- /dev/null +++ b/versioned_docs/version-latest/faq/telemetry.md @@ -0,0 +1,35 @@ +--- +title: Telemetry +--- + + + + + +### What is Telemetry? + +Telemetry collects aggregate information about the size of Rancher installations, versions of components used, and which features are used. This information is used by Rancher Labs to help make the product better and is not shared with third-parties. + +### What information is collected? + +No specific identifying information like usernames, passwords, or the names or addresses of user resources will ever be collected. + +The primary things collected include: + + - Aggregate counts (smallest, average, largest, total) of nodes per-cluster and their size (e.g. CPU cores & RAM). + - Aggregate counts of logical resources like Clusters, Projects, Namespaces, and Pods. + - Counts of what driver was used to deploy clusters and nodes (e.g. GKE vs EC2 vs Imported vs Custom). + - Versions of Kubernetes components, Operating Systems and Docker that are deployed on nodes. + - Whether some optional components are enabled or not (e.g. which auth providers are used). + - The image name & version of Rancher that is running. + - A unique randomly-generated identifier for this installation. + +### Can I see the information that is being sent? + +If Telemetry is enabled, you can go to `https:///v1-telemetry` in your installation to see the current data. + +If Telemetry is not enabled, the process that collects the data is not running, so there is nothing being collected to look at. + +### How do I turn it on or off? + +After initial setup, an administrator can go to the `Settings` page in the `Global` section of the UI and click Edit to change the `telemetry-opt` setting to either `in` or `out`. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md new file mode 100644 index 00000000000..7b01fb3aac2 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades.md @@ -0,0 +1,80 @@ +--- +title: Upgrading in an Air-Gapped Environment +--- + + + + + +:::note + +These instructions assume you have already followed the instructions for a Kubernetes upgrade on [this page,](upgrades.md) including the prerequisites, up until step 3. Upgrade Rancher. + +::: + +### Rancher Helm Upgrade Options + +To upgrade with Helm, apply the same options that you used when installing Rancher. Refer to the reference table below to replace each placeholder. Rancher needs to be configured to use the private registry in order to provision any Rancher launched Kubernetes clusters or Rancher tools. + +Based on the choice you made during installation, complete one of the procedures below. + +Placeholder | Description +------------|------------- +`` | The version number of the output tarball. +`` | The DNS name you pointed at your load balancer. +`` | The DNS name for your private registry. +`` | Cert-manager version running on k8s cluster. + + +### Option A: Default Self-signed Certificate + +``` +helm upgrade rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +### Option B: Certificates from Files using Kubernetes Secrets + +```plain +helm upgrade rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +If you are using a Private CA signed cert, add `--set privateCA=true` following `--set ingress.tls.source=secret`: + +```plain +helm upgrade rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +## Verify the Upgrade + +Log into Rancher to confirm that the upgrade succeeded. + +:::tip + +Having network issues following upgrade? + +See [Restoring Cluster Networking](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md). + +::: + +## Known Upgrade Issues + +A list of known issues for each Rancher version can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums.](https://forums.rancher.com/c/announcements/12) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md new file mode 100644 index 00000000000..2300a22016f --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md @@ -0,0 +1,151 @@ +--- +title: Installing Rancher on Azure Kubernetes Service +--- + + + + + +This page covers how to install Rancher on Microsoft's Azure Kubernetes Service (AKS). + +The guide uses command line tools to provision an AKS cluster with an ingress. If you prefer to provision your cluster using the Azure portal, refer to the [official documentation](https://docs.microsoft.com/en-us/azure/aks/kubernetes-walkthrough-portal). + +If you already have an AKS Kubernetes cluster, skip to the step about [installing an ingress.](#5-install-an-ingress) Then install the Rancher Helm chart following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) + +## Prerequisites + +:::caution + +Deploying to Microsoft Azure will incur charges. + +::: + +- [Microsoft Azure Account](https://azure.microsoft.com/en-us/free/): A Microsoft Azure Account is required to create resources for deploying Rancher and Kubernetes. +- [Microsoft Azure Subscription](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal): Use this link to follow a tutorial to create a Microsoft Azure subscription if you don't have one yet. +- [Micsoroft Azure Tenant](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant): Use this link and follow instructions to create a Microsoft Azure tenant. +- Your subscription has sufficient quota for at least 2 vCPUs. For details on Rancher server resource requirements, refer to [this section](../../../pages-for-subheaders/installation-requirements.md#rke-and-hosted-kubernetes) +- When installing Rancher with Helm in Azure, use the L7 load balancer to avoid networking issues. For more information, refer to the documentation on [Azure load balancer limitations](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations). + +## 1. Prepare your Workstation + +Install the following command line tools on your workstation: + +- The Azure CLI, **az:** For help, refer to these [installation steps.](https://docs.microsoft.com/en-us/cli/azure/) +- **kubectl:** For help, refer to these [installation steps.](https://kubernetes.io/docs/tasks/tools/#kubectl) +- **helm:** For help, refer to these [installation steps.](https://helm.sh/docs/intro/install/) + +## 2. Create a Resource Group + +After installing the CLI, you will need to log in with your Azure account. + +``` +az login +``` + +Create a [resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) to hold all relevant resources for your cluster. Use a location that applies to your use case. + +``` +az group create --name rancher-rg --location eastus +``` + +## 3. Create the AKS Cluster + +To create an AKS cluster, run the following command. Use a VM size that applies to your use case. Refer to [this article](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes) for available sizes and options. When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +:::note + +If you're updating from an older version of Kubernetes, to Kubernetes v1.22 or above, you also need to [update](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx. + +::: + +``` +az aks create \ + --resource-group rancher-rg \ + --name rancher-server \ + --kubernetes-version \ + --node-count 3 \ + --node-vm-size Standard_D2_v3 +``` + +The cluster will take some time to be deployed. + +## 4. Get Access Credentials + +After the cluster is deployed, get the access credentials. + +``` +az aks get-credentials --resource-group rancher-rg --name rancher-server +``` + +This command merges your cluster's credentials into the existing kubeconfig and allows `kubectl` to interact with the cluster. + +## 5. Install an Ingress + +The cluster needs an Ingress so that Rancher can be accessed from outside the cluster. Installing an Ingress requires allocating a public IP address. Ensure you have sufficient quota, otherwise it will fail to assign the IP address. Limits for public IP addresses are applicable at a regional level per subscription. + +To make sure that you choose the correct Ingress-NGINX Helm chart, first find an `Ingress-NGINX version` that's compatible with your Kubernetes version in the [Kubernetes/ingress-nginx support table](https://github.com/kubernetes/ingress-nginx#supported-versions-table). + +Then, list the Helm charts available to you by running the following command: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +The `helm search` command's output contains an `APP VERSION` column. The versions under this column are equivalent to the `Ingress-NGINX version` you chose earlier. Using the app version, select a chart version that bundles an app compatible with your Kubernetes install. For example, if you have Kubernetes v1.24, you can select the v4.6.0 Helm chart, since Ingress-NGINX v1.7.0 comes bundled with that chart, and v1.7.0 is compatible with Kubernetes v1.24. When in doubt, select the most recent compatible version. + +Now that you know which Helm chart `version` you need, run the following command. It installs an `nginx-ingress-controller` with a Kubernetes load balancer service: + +``` +helm search repo ingress-nginx -l +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-health-probe-request-path"=/healthz \ + --set controller.service.externalTrafficPolicy=Local \ + --version 4.6.0 \ + --create-namespace +``` + +## 6. Get Load Balancer IP + +To get the address of the load balancer, run: + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +The result should look similar to the following: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.0.116.18 40.31.180.83 80:31229/TCP,443:31050/TCP + 67s +``` + +Save the `EXTERNAL-IP`. + +## 7. Set up DNS + +External traffic to the Rancher server will need to be directed at the load balancer you created. + +Set up a DNS to point at the `EXTERNAL-IP` that you saved. This DNS will be used as the Rancher server URL. + +There are many valid ways to set up the DNS. For help, refer to the [Azure DNS documentation](https://docs.microsoft.com/en-us/azure/dns/) + +## 8. Install the Rancher Helm Chart + +Next, install the Rancher Helm chart by following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) The Helm instructions are the same for installing Rancher on any Kubernetes distribution. + +Use that DNS name from the previous step as the Rancher server URL when you install Rancher. It can be passed in as a Helm option. For example, if the DNS name is `rancher.my.org`, you could run the Helm installation command with the option `--set hostname=rancher.my.org`. + +When installing Rancher on top of this setup, you will also need to pass the value below into the Rancher Helm install command in order to set the name of the ingress controller to be used with Rancher's ingress resource: + +``` +--set ingress.ingressClassName=nginx +``` + +Refer [here for the Helm install command](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-install-rancher-with-helm-and-your-chosen-certificate-option) for your chosen certificate option. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md new file mode 100644 index 00000000000..6563d4b14eb --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md @@ -0,0 +1,155 @@ +--- +title: Installing Rancher on Amazon EKS +--- + + + + + +This page covers installing Rancher on an Amazon EKS cluster. You can also [install Rancher through the AWS Marketplace](../../quick-start-guides/deploy-rancher-manager/aws-marketplace.md). + +If you already have an EKS Kubernetes cluster, skip to the step about [installing an ingress.](#5-install-an-ingress) Then install the Rancher Helm chart following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) + +## Creating an EKS Cluster for the Rancher Server + +In this section, you'll install an EKS cluster with an ingress by using command line tools. This guide may be useful if you want to use fewer resources while trying out Rancher on EKS. + +:::note Prerequisites: + +- You should already have an AWS account. +- It is recommended to use an IAM user instead of the root AWS account. You will need the IAM user's access key and secret key to configure the AWS command line interface. +- The IAM user needs the minimum IAM policies described in the official [eksctl documentation.](https://eksctl.io/usage/minimum-iam-policies/) + +::: + +### 1. Prepare your Workstation + +Install the following command line tools on your workstation: + +- **The AWS CLI v2:** For help, refer to these [installation steps.](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) +- **eksctl:** For help, refer to these [installation steps.](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +- **kubectl:** For help, refer to these [installation steps.](https://docs.aws.amazon.com/eks/latest/userguide/install-kubectl.html) +- **helm:** For help, refer to these [installation steps.](https://helm.sh/docs/intro/install/) + +### 2. Configure the AWS CLI + +To configure the AWS CLI, run the following command: + +``` +aws configure +``` + +Then enter the following values: + +| Value | Description | +|-------|-------------| +| AWS Access Key ID | The access key credential for the IAM user with EKS permissions. | +| AWS Secret Access Key | The secret key credential for the IAM user with EKS permissions. | +| Default region name | An [AWS region](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html#Concepts.RegionsAndAvailabilityZones.Regions) where the cluster nodes will be located. | +| Default output format | Enter `json`. | + +### 3. Create the EKS Cluster + +To create an EKS cluster, run the following command. Use the AWS region that applies to your use case. When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +**Note:** If you're updating from an older version of Kubernetes, to Kubernetes v1.22 or above, you also need to [update](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx. + +``` +eksctl create cluster \ + --name rancher-server \ + --version \ + --region us-west-2 \ + --nodegroup-name ranchernodes \ + --nodes 3 \ + --nodes-min 1 \ + --nodes-max 4 \ + --managed +``` + +The cluster will take some time to be deployed with CloudFormation. + +### 4. Test the Cluster + +To test the cluster, run: + +``` +eksctl get cluster +``` + +The result should look like the following: + +``` +eksctl get cluster +2021-03-18 15:09:35 [ℹ] eksctl version 0.40.0 +2021-03-18 15:09:35 [ℹ] using region us-west-2 +NAME REGION EKSCTL CREATED +rancher-server-cluster us-west-2 True +``` + +### 5. Install an Ingress + +The cluster needs an Ingress so that Rancher can be accessed from outside the cluster. + +To make sure that you choose the correct Ingress-NGINX Helm chart, first find an `Ingress-NGINX version` that's compatible with your Kubernetes version in the [Kubernetes/ingress-nginx support table](https://github.com/kubernetes/ingress-nginx#supported-versions-table). + +Then, list the Helm charts available to you by running the following command: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm search repo ingress-nginx -l +``` + +The `helm search` command's output contains an `APP VERSION` column. The versions under this column are equivalent to the `Ingress-NGINX version` you chose earlier. Using the app version, select a chart version that bundles an app compatible with your Kubernetes install. For example, if you have Kubernetes v1.23, you can select the v4.6.0 Helm chart, since Ingress-NGINX v1.7.0 comes bundled with that chart, and v1.7.0 is compatible with Kubernetes v1.23. When in doubt, select the most recent compatible version. + +Now that you know which Helm chart `version` you need, run the following command. It installs an `nginx-ingress-controller` with a Kubernetes load balancer service: + +``` +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.6.0 \ + --create-namespace +``` + +### 6. Get Load Balancer IP + +To get the address of the load balancer, run: + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +The result should look similar to the following: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) + AGE +ingress-nginx-controller LoadBalancer 10.100.90.18 a904a952c73bf4f668a17c46ac7c56ab-962521486.us-west-2.elb.amazonaws.com 80:31229/TCP,443:31050/TCP + 27m +``` + +Save the `EXTERNAL-IP`. + +### 7. Set up DNS + +External traffic to the Rancher server will need to be directed at the load balancer you created. + +Set up a DNS to point at the external IP that you saved. This DNS will be used as the Rancher server URL. + +There are many valid ways to set up the DNS. For help, refer to the AWS documentation on [routing traffic to an ELB load balancer.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer.html) + +### 8. Install the Rancher Helm Chart + +Next, install the Rancher Helm chart by following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) The Helm instructions are the same for installing Rancher on any Kubernetes distribution. + +Use that DNS name from the previous step as the Rancher server URL when you install Rancher. It can be passed in as a Helm option. For example, if the DNS name is `rancher.my.org`, you could run the Helm installation command with the option `--set hostname=rancher.my.org`. + +When installing Rancher on top of this setup, you will also need to pass the value below into the Rancher Helm install command in order to set the name of the ingress controller to be used with Rancher's ingress resource: + +``` +--set ingress.ingressClassName=nginx +``` + +Refer [here for the Helm install command](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-install-rancher-with-helm-and-your-chosen-certificate-option) for your chosen certificate option. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md new file mode 100644 index 00000000000..64dd8fc9df2 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md @@ -0,0 +1,205 @@ +--- +title: Installing Rancher on a Google Kubernetes Engine Cluster +--- + + + + + +In this section, you'll learn how to install Rancher using Google Kubernetes Engine. + +If you already have a GKE Kubernetes cluster, skip to the step about [installing an ingress.](#7-install-an-ingress) Then install the Rancher Helm chart following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) + +## Prerequisites + +- You will need a Google account. +- You will need a Google Cloud billing account. You can manage your Cloud Billing accounts using the Google Cloud Console. For more information about the Cloud Console, visit [General guide to the console.](https://support.google.com/cloud/answer/3465889?hl=en&ref_topic=3340599) +- You will need a cloud quota for at least one in-use IP address and at least 2 CPUs. For more details about hardware requirements for the Rancher server, refer to [this section.](../../../pages-for-subheaders/installation-requirements.md#rke-and-hosted-kubernetes) + +## 1. Enable the Kubernetes Engine API + +Take the following steps to enable the Kubernetes Engine API: + +1. Visit the [Kubernetes Engine page](https://console.cloud.google.com/projectselector/kubernetes?_ga=2.169595943.767329331.1617810440-856599067.1617343886) in the Google Cloud Console. +1. Create or select a project. +1. Open the project and enable the Kubernetes Engine API for the project. Wait for the API and related services to be enabled. This can take several minutes. +1. Make sure that billing is enabled for your Cloud project. For information on how to enable billing for your project, refer to the [Google Cloud documentation.](https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project) + +## 2. Open the Cloud Shell + +Cloud Shell is a shell environment for managing resources hosted on Google Cloud. Cloud Shell comes preinstalled with the `gcloud` command-line tool and kubectl command-line tool. The `gcloud` tool provides the primary command-line interface for Google Cloud, and `kubectl` provides the primary command-line interface for running commands against Kubernetes clusters. + +The following sections describe how to launch the cloud shell from the Google Cloud Console or from your local workstation. + +### Cloud Shell + +To launch the shell from the [Google Cloud Console,](https://console.cloud.google.com) go to the upper-right corner of the console and click the terminal button. When hovering over the button, it is labeled **Activate Cloud Shell**. + +### Local Shell + +To install `gcloud` and `kubectl`, perform the following steps: + +1. Install the Cloud SDK by following [these steps.](https://cloud.google.com/sdk/docs/install) The Cloud SDK includes the `gcloud` command-line tool. The steps vary based on your OS. +1. After installing Cloud SDK, install the `kubectl` command-line tool by running the following command: + + ``` + gcloud components install kubectl + ``` + In a later step, `kubectl` will be configured to use the new GKE cluster. +1. [Install Helm 3](https://helm.sh/docs/intro/install/) if it is not already installed. +1. Enable Helm experimental [support for OCI images](https://github.com/helm/community/blob/master/hips/hip-0006.md) with the `HELM_EXPERIMENTAL_OCI` variable. Add the following line to `~/.bashrc` (or `~/.bash_profile` in macOS, or wherever your shell stores environment variables): + + ``` + export HELM_EXPERIMENTAL_OCI=1 + ``` +1. Run the following command to load your updated `.bashrc` file: + + ``` + source ~/.bashrc + ``` + If you are running macOS, use this command: + ``` + source ~/.bash_profile + ``` + + + +## 3. Configure the gcloud CLI + + Set up default gcloud settings using one of the following methods: + +- Using gcloud init, if you want to be walked through setting defaults. +- Using gcloud config, to individually set your project ID, zone, and region. + + + + + 1. Run gcloud init and follow the directions: + + ``` + gcloud init + ``` + If you are using SSH on a remote server, use the --console-only flag to prevent the command from launching a browser: + + ``` + gcloud init --console-only + ``` + 2. Follow the instructions to authorize gcloud to use your Google Cloud account and select the new project that you created. + + + + + + + +## 4. Confirm that gcloud is configured correctly + +Run: + +``` +gcloud config list +``` + +The output should resemble the following: + +``` +[compute] +region = us-west1 # Your chosen region +zone = us-west1-b # Your chosen zone +[core] +account = +disable_usage_reporting = True +project = + +Your active configuration is: [default] +``` + +## 5. Create a GKE Cluster + +The following command creates a three-node cluster. + +Replace `cluster-name` with the name of your new cluster. + +When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +To successfully create a GKE cluster with Rancher, your GKE must be in Standard mode. GKE has two modes of operation when creating a Kubernetes cluster, Autopilot and Standard mode. The cluster configuration for Autopilot mode has restrictions on editing the kube-system namespace. However, Rancher needs to create resources in the kube-system namespace during installation. As a result, you will not be able to install Rancher on a GKE cluster created in Autopilot mode. For more information about the difference between GKE Autopilot mode and Standard mode, visit [Compare GKE Autopilot and Standard.](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison) + +**Note:** If you're updating from an older version of Kubernetes, to Kubernetes v1.22 or above, you also need to [update](https://kubernetes.github.io/ingress-nginx/user-guide/k8s-122-migration/) ingress-nginx. + +``` +gcloud container clusters create cluster-name --num-nodes=3 --cluster-version= +``` + +## 6. Get Authentication Credentials + +After creating your cluster, you need to get authentication credentials to interact with the cluster: + +``` +gcloud container clusters get-credentials cluster-name +``` + +This command configures `kubectl` to use the cluster you created. + +## 7. Install an Ingress + +The cluster needs an Ingress so that Rancher can be accessed from outside the cluster. + +The following command installs an `nginx-ingress-controller` with a LoadBalancer service: + +``` +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm upgrade --install \ + ingress-nginx ingress-nginx/ingress-nginx \ + --namespace ingress-nginx \ + --set controller.service.type=LoadBalancer \ + --version 4.0.18 \ + --create-namespace +``` + +## 8. Get the Load Balancer IP + +To get the address of the load balancer, run: + +``` +kubectl get service ingress-nginx-controller --namespace=ingress-nginx +``` + +The result should look similar to the following: + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +ingress-nginx-controller LoadBalancer 10.3.244.156 35.233.206.34 80:31876/TCP,443:32497/TCP 81s +``` + +Save the `EXTERNAL-IP`. + +## 9. Set up DNS + +External traffic to the Rancher server will need to be directed at the load balancer you created. + +Set up a DNS to point at the external IP that you saved. This DNS will be used as the Rancher server URL. + +There are many valid ways to set up the DNS. For help, refer to the Google Cloud documentation about [managing DNS records.](https://cloud.google.com/dns/docs/records) + +## 10. Install the Rancher Helm chart + +Next, install the Rancher Helm chart by following the instructions on [this page.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#install-the-rancher-helm-chart) The Helm instructions are the same for installing Rancher on any Kubernetes distribution. + +Use the DNS name from the previous step as the Rancher server URL when you install Rancher. It can be passed in as a Helm option. For example, if the DNS name is `rancher.my.org`, you could run the Helm installation command with the option `--set hostname=rancher.my.org`. + +When installing Rancher on top of this setup, you will also need to set the name of the ingress controller to be used with Rancher's ingress resource: + +``` +--set ingress.ingressClassName=nginx +``` + +Refer [here for the Helm install command](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#5-install-rancher-with-helm-and-your-chosen-certificate-option) for your chosen certificate option. + +In Rancher v2.7.5, if you intend to use the default GKE ingress on your cluster without enabling VPC-native cluster mode, you need to set the following flag: + +``` +--set service.type=NodePort +``` + +This is necessary because of compatibility issues between this setup and ClusterIP, the default type for `cattle-system/rancher`. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml new file mode 100644 index 00000000000..ae248cc9b7c --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-psact.yaml @@ -0,0 +1,37 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +- configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + defaults: + audit: restricted + audit-version: latest + enforce: restricted + enforce-version: latest + warn: restricted + warn-version: latest + exemptions: + namespaces: + - ingress-nginx + - kube-system + - cattle-system + - cattle-epinio-system + - cattle-fleet-system + - longhorn-system + - cattle-neuvector-system + - cattle-monitoring-system + - rancher-alerting-drivers + - cis-operator-system + - cattle-csp-adapter-system + - cattle-externalip-system + - cattle-gatekeeper-system + - istio-system + - cattle-istio-system + - cattle-logging-system + - cattle-windows-gmsa-system + - cattle-sriov-system + - cattle-ui-plugin-system + - tigera-operator + kind: PodSecurityConfiguration + name: PodSecurity + path: "" \ No newline at end of file diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md new file mode 100644 index 00000000000..bb7fed4fe0a --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md @@ -0,0 +1,132 @@ +--- +title: Rollbacks +--- + + + + + +## Additional Steps for Rollbacks with Rancher v2.6.4+ + +Rancher v2.6.4 upgrades the cluster-api module from v0.4.4 to v1.0.2. Version v1.0.2 of the cluster-api, in turn, upgrades the Cluster API's Custom Resource Definitions (CRDs) from `cluster.x-k8s.io/v1alpha4` to `cluster.x-k8s.io/v1beta1`. The CRDs upgrade to v1beta1 causes rollbacks to fail when you attempt to move from Rancher v2.6.4 to any previous version of Rancher v2.6.x. This is because CRDs that use the older apiVersion (v1alpha4) are incompatible with v1beta1. + +To avoid rollback failure, the following Rancher scripts should be run **before** you attempt a restore operation or rollback: + +* `verify.sh`: Checks for any Rancher-related resources in the cluster. +* `cleanup.sh`: Cleans up the cluster. + +See the [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup) for more details and source code. + +:::caution + + There will be downtime while `cleanup.sh` runs, since the script deletes resources created by Rancher. + +::: + +### Rolling back from v2.6.4+ to lower versions of v2.6.x + +1. Follow these [instructions](https://github.com/rancher/rancher-cleanup/blob/main/README.md) to run the scripts. +1. Follow these [instructions](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) to install the rancher-backup Helm chart on the existing cluster and restore the previous state. + 1. Omit Step 3. + 1. When you reach Step 4, install the Rancher v2.6.x version on the local cluster you intend to roll back to. + +## Rolling Back to Rancher v2.5.0+ + +To roll back to Rancher v2.5.0+, use the **Rancher Backups** application and restore Rancher from backup. + +Rancher has to be started with the lower/previous version after a rollback. + +A restore is performed by creating a Restore custom resource. + +:::note Important: + +* Follow the instructions from this page for restoring Rancher on the same cluster where it was backed up from. In order to migrate Rancher to a new cluster, follow the steps to [migrate Rancher.](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +* While restoring Rancher on the same setup, the Rancher deployment is manually scaled down before the restore starts, then the operator will scale it back up once the restore completes. As a result, Rancher and its UI will be unavailable until the restore is complete. While the UI is unavailable, use the original cluster kubeconfig with the restore YAML file: `kubectl create -f restore.yaml`. + +::: + +### Create the Restore Custom Resource + +1. Click **☰ > Cluster Management**. +1. Go to the local cluster and click **Explore**. +1. In the left navigation bar, click **Rancher Backups > Restore**. + :::note + + If the Rancher Backups app is not visible, you will need to install it from the Charts page in **Apps**. Refer [here](../../../pages-for-subheaders/helm-charts-in-rancher.md#charts) for more information. + + ::: + +1. Click **Create**. +1. Create the Restore with the form or with YAML. For help creating the Restore resource using the online form, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) +1. To use the YAML editor, you can click **Create > Create from YAML.** Enter the Restore YAML. The following is an example Restore custom resource: + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + For help configuring the Restore, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) + +1. Click **Create**. + +**Result:** The backup file is created and updated to the target storage location. The resources are restored in this order: + +1. Custom Resource Definitions (CRDs) +2. Cluster-scoped resources +3. Namespaced resources + +To check how the restore is progressing, you can check the logs of the operator. Follow these steps to get the logs: + +```yaml +kubectl get pods -n cattle-resources-system +kubectl logs -n cattle-resources-system -f +``` + +### Roll back to a previous Rancher version + +Rancher can be rolled back using the Helm CLI. To roll back to the previous version: + +```yaml +helm rollback rancher -n cattle-system +``` + +If the previous revision is not the intended target, you can specify a revision to roll back to. To see the deployment history: + +```yaml +helm history rancher -n cattle-system +``` + +When the target revision is determined, perform the rollback. This example will roll back to revision `3`: + +```yaml +helm rollback rancher 3 -n cattle-system +``` + +## Rolling Back to Rancher v2.2-v2.4+ + +To roll back to Rancher before v2.5, follow the procedure detailed here: [Restoring Backups — Kubernetes installs](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Restoring a snapshot of the Rancher server cluster will revert Rancher to the version and state at the time of the snapshot. + +For information on how to roll back Rancher installed with Docker, refer to [this page.](../other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md) + +:::note + +Managed clusters are authoritative for their state. This means restoring the Rancher server will not revert workload deployments or changes made on managed clusters after the snapshot was taken. + +::: + +## Rolling Back to Rancher v2.0-v2.1 + +Rolling back to Rancher v2.0-v2.1 is no longer supported. The instructions for rolling back to these versions are preserved [here](/versioned_docs/version-2.0-2.4/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup/roll-back-to-v2.0-v2.1.md) and are intended to be used only in cases where upgrading to Rancher v2.2+ is not feasible. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md new file mode 100644 index 00000000000..9ebbf27460f --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md @@ -0,0 +1,192 @@ +--- +title: Troubleshooting the Rancher Server Kubernetes Cluster +--- + + + + + +This section describes how to troubleshoot an installation of Rancher on a Kubernetes cluster. + +### Relevant Namespaces + +Most of the troubleshooting will be done on objects in these 3 namespaces. + +- `cattle-system` - `rancher` deployment and pods. +- `ingress-nginx` - Ingress controller pods and services. +- `cert-manager` - `cert-manager` pods. + +### "default backend - 404" + +A number of things can cause the ingress-controller not to forward traffic to your rancher instance. Most of the time its due to a bad ssl configuration. + +Things to check + +- [Is Rancher Running](#check-if-rancher-is-running) +- [Cert CN is "Kubernetes Ingress Controller Fake Certificate"](#cert-cn-is-kubernetes-ingress-controller-fake-certificate) + +### Check if Rancher is Running + +Use `kubectl` to check the `cattle-system` system namespace and see if the Rancher pods are in a Running state. + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +If the state is not `Running`, run a `describe` on the pod and check the Events. + +``` +kubectl -n cattle-system describe pod + +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Scheduled 11m default-scheduler Successfully assigned rancher-784d94f59b-vgqzh to localhost + Normal SuccessfulMountVolume 11m kubelet, localhost MountVolume.SetUp succeeded for volume "rancher-token-dj4mt" + Normal Pulling 11m kubelet, localhost pulling image "rancher/rancher:v2.0.4" + Normal Pulled 11m kubelet, localhost Successfully pulled image "rancher/rancher:v2.0.4" + Normal Created 11m kubelet, localhost Created container + Normal Started 11m kubelet, localhost Started container +``` + +### Check the Rancher Logs + +Use `kubectl` to list the pods. + +``` +kubectl -n cattle-system get pods + +NAME READY STATUS RESTARTS AGE +pod/rancher-784d94f59b-vgqzh 1/1 Running 0 10m +``` + +Use `kubectl` and the pod name to list the logs from the pod. + +``` +kubectl -n cattle-system logs -f rancher-784d94f59b-vgqzh +``` + +### Cert CN is "Kubernetes Ingress Controller Fake Certificate" + +Use your browser to check the certificate details. If it says the Common Name is "Kubernetes Ingress Controller Fake Certificate", something may have gone wrong with reading or issuing your SSL cert. + +:::note + +If you are using LetsEncrypt to issue certs, it can sometimes take a few minutes to issue the cert. + +::: + +### Checking for issues with cert-manager issued certs (Rancher Generated or LetsEncrypt) + +`cert-manager` has 3 parts. + +- `cert-manager` pod in the `cert-manager` namespace. +- `Issuer` object in the `cattle-system` namespace. +- `Certificate` object in the `cattle-system` namespace. + +Work backwards and do a `kubectl describe` on each object and check the events. You can track down what might be missing. + +For example there is a problem with the Issuer: + +``` +kubectl -n cattle-system describe certificate +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning IssuerNotReady 18s (x23 over 19m) cert-manager Issuer rancher not ready +``` + +``` +kubectl -n cattle-system describe issuer +... +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Warning ErrInitIssuer 19m (x12 over 19m) cert-manager Error initializing issuer: secret "tls-rancher" not found + Warning ErrGetKeyPair 9m (x16 over 19m) cert-manager Error getting keypair for CA issuer: secret "tls-rancher" not found +``` + +### Checking for Issues with Your Own SSL Certs + +Your certs get applied directly to the Ingress object in the `cattle-system` namespace. + +Check the status of the Ingress object and see if its ready. + +``` +kubectl -n cattle-system describe ingress +``` + +If its ready and the SSL is still not working you may have a malformed cert or secret. + +Check the nginx-ingress-controller logs. Because the nginx-ingress-controller has multiple containers in its pod you will need to specify the name of the container. + +``` +kubectl -n ingress-nginx logs -f nginx-ingress-controller-rfjrq nginx-ingress-controller +... +W0705 23:04:58.240571 7 backend_ssl.go:49] error obtaining PEM from secret cattle-system/tls-rancher-ingress: error retrieving secret cattle-system/tls-rancher-ingress: secret cattle-system/tls-rancher-ingress was not found +``` + +### No matches for kind "Issuer" + +The SSL configuration option you have chosen requires cert-manager to be installed before installing Rancher or else the following error is shown: + +``` +Error: validation failed: unable to recognize "": no matches for kind "Issuer" in version "certmanager.k8s.io/v1alpha1" +``` + +Install cert-manager and try installing Rancher again. + + +### Canal Pods show READY 2/3 + +The most common cause of this issue is port 8472/UDP is not open between the nodes. Check your local firewall, network routing or security groups. + +Once the network issue is resolved, the `canal` pods should timeout and restart to establish their connections. + +### nginx-ingress-controller Pods show RESTARTS + +The most common cause of this issue is the `canal` pods have failed to establish the overlay network. See [canal Pods show READY `2/3`](#canal-pods-show-ready-23) for troubleshooting. + + +### Failed to dial to /var/run/docker.sock: ssh: rejected: administratively prohibited (open failed) + +Some causes of this error include: + +* User specified to connect with does not have permission to access the Docker socket. This can be checked by logging into the host and running the command `docker ps`: + +``` +$ ssh user@server +user@server$ docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +``` + +See [Manage Docker as a non-root user](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) how to set this up properly. + +* When using RedHat/CentOS as operating system, you cannot use the user `root` to connect to the nodes because of [Bugzilla #1527565](https://bugzilla.redhat.com/show_bug.cgi?id=1527565). You will need to add a separate user and configure it to access the Docker socket. See [Manage Docker as a non-root user](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) how to set this up properly. + +* SSH server version is not version 6.7 or higher. This is needed for socket forwarding to work, which is used to connect to the Docker socket over SSH. This can be checked using `sshd -V` on the host you are connecting to, or using netcat: +``` +$ nc xxx.xxx.xxx.xxx 22 +SSH-2.0-OpenSSH_6.6.1p1 Ubuntu-2ubuntu2.10 +``` + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: no key found + +The key file specified as `ssh_key_path` cannot be accessed. Make sure that you specified the private key file (not the public key, `.pub`), and that the user that is running the `rke` command can access the private key file. + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain + +The key file specified as `ssh_key_path` is not correct for accessing the node. Double-check if you specified the correct `ssh_key_path` for the node and if you specified the correct user to connect with. + +### Failed to dial ssh using address [xxx.xxx.xxx.xxx:xx]: Error configuring SSH: ssh: cannot decode encrypted private keys + +If you want to use encrypted private keys, you should use `ssh-agent` to load your keys with your passphrase. If the `SSH_AUTH_SOCK` environment variable is found in the environment where the `rke` command is run, it will be used automatically to connect to the node. + +### Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running? + +The node is not reachable on the configured `address` and `port`. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md new file mode 100644 index 00000000000..4d276be3e23 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25.md @@ -0,0 +1,68 @@ +--- +title: Upgrade a Hardened Custom/Imported Cluster to Kubernetes v1.25 +--- + +Kubernetes v1.25 changes how clusters describe and implement security policies. From this version forward, [Pod Security Policies (PSPs)](https://kubernetes.io/docs/concepts/security/pod-security-policy/) are no longer available. Kubernetes v1.25 replaces them with new security objects: [Pod Security Standards (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/), and [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/). + +If you have custom or imported hardened clusters, you must take special preparations to ensure that the upgrade from an earlier version of Kubernetes to v1.25 or later goes smoothly. + +:::note + +After you upgrade to v1.25, add the necessary Rancher namespace exemptions. See [Pod Security Admission (PSA) Configuration Templates](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md#exempting-required-rancher-namespaces) for more details. + +::: + +## Upgrading Imported Hardened Clusters to Kubernetes v1.25 or Later + + + + +Perform the following on each node in the cluster: +1. Save [`rancher-psact.yaml`](./rancher-psact.yaml) in `/etc/rancher/rke2`. +1. Edit the RKE2 configuration file: + 1. Update the `profile` field to `cis-1.23`. + 1. Specify the path for the configuration file that you just added: `pod-security-admission-config-file: /etc/rancher/rke2/rancher-psact.yaml`. + + + + +Perform the following on each node in the cluster: + +Follow the official K3s instructions on [Upgrading Hardened Clusters from v1.24.x to v1.25.x](https://docs.k3s.io/known-issues#hardened-125), but use a [custom](./rancher-psact.yaml) Rancher PSA configuration template, instead of the configuration provided on the official K3s site. + + + +After you perform these steps, you can upgrade the cluster's Kubernetes version through the Rancher UI: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config**. +1. In the **Kubernetes Version** dropdown menu, select the version that you would like to use. +1. Click **Save**. + +## Upgrading Custom Hardened Clusters to Kubernetes v1.25 or Later + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config**. +1. Under **Basics > Security**, in the **CIS Profile** dropdown menu, select `cis-1.23`. +1. In the **Pod Security Admission Configuration Template** dropdown menu, select `rancher-restricted`. +1. In the **Kubernetes Version** dropdown menu, select the version that you would like to use. +1. Click **Save**. + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit YAML**. +1. Delete `PodSecurityPolicy` from `kube-apiserver-arg.enable-admission-plugins` +1. Add this line to the `spec` field: `defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted` +1. Update `kubernetesVersion` to your chosen version (v1.25 or later). +1. Click **Save**. + + + diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md new file mode 100644 index 00000000000..fb4f1cf42bb --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md @@ -0,0 +1,207 @@ +--- +title: Upgrades +--- + + + + + +The following instructions will guide you through upgrading a Rancher server that was installed on a Kubernetes cluster with Helm. These steps also apply to air-gapped installs with Helm. + +For the instructions to upgrade Rancher installed with Docker, refer to [this page.](../other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md) + +To upgrade the components in your Kubernetes cluster, or the definition of the [Kubernetes services](https://rancher.com/docs/rke/latest/en/config-options/services/) or [add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/), refer to the [upgrade documentation for RKE](https://rancher.com/docs/rke/latest/en/upgrades/), the Rancher Kubernetes Engine. + + +## Prerequisites + +### Access to kubeconfig + +Helm should be run from the same location as your kubeconfig file, or the same location where you run your kubectl commands from. + +If you installed Kubernetes with RKE, the config will have been created in the directory you ran `rke up` in. + +The kubeconfig can also be manually targeted for the intended cluster with the `--kubeconfig` tag (see: https://helm.sh/docs/helm/helm/) + +### Review Known Issues + +Review the list of known issues for each Rancher version, which can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums.](https://forums.rancher.com/c/announcements/12) + +Note that upgrades _to_ or _from_ any chart in the [rancher-alpha repository](../resources/choose-a-rancher-version.md#helm-chart-repositories) aren't supported. +### Helm Version + +The upgrade instructions assume you are using Helm 3. + +For migration of installs started with Helm 2, refer to the official [Helm 2 to 3 migration docs.](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/) The [Helm 2 upgrade page here](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/helm2.md)provides a copy of the older upgrade instructions that used Helm 2, and it is intended to be used if upgrading to Helm 3 is not feasible. + +### For air-gapped installs: Populate private registry + +For [air-gapped installs only,](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) collect and populate images for the new Rancher server version. Follow the guide to [populate your private registry](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md) with the images for the Rancher version that you want to upgrade to. + +### For upgrades with cert-manager older than 0.8.0 + +[Let's Encrypt will be blocking cert-manager instances older than 0.8.0 starting November 1st 2019.](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) Upgrade cert-manager to the latest version by following [these instructions.](../resources/upgrade-cert-manager.md) + +## Upgrade Outline + +Follow the steps to upgrade Rancher server: + + +### 1. Back up Your Kubernetes Cluster that is Running Rancher Server + +Use the [backup application](../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) to back up Rancher. + +You'll use the backup as a restore point if something goes wrong during upgrade. + +### 2. Update the Helm chart repository + +1. Update your local helm repo cache. + + ``` + helm repo update + ``` + +1. Get the repository name that you used to install Rancher. + + For information about the repos and their differences, see [Helm Chart Repositories](../resources/choose-a-rancher-version.md#helm-chart-repositories). + + - Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + + ``` + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + + :::note + + If you want to switch to a different Helm chart repository, please follow the [steps on how to switch repositories](../resources/choose-a-rancher-version.md#switching-to-a-different-helm-chart-repository). If you switch repositories, make sure to list the repositories again before continuing onto Step 3 to ensure you have the correct one added. + + ::: + +1. Fetch the latest chart to install Rancher from the Helm chart repository. + + This command will pull down the latest charts and save it in the current directory as a `.tgz` file. + + ```plain + helm fetch rancher-/rancher + ``` + You can fetch the chart for the specific version you are upgrading to by adding in the `--version=` tag. For example: + + ```plain + helm fetch rancher-/rancher --version=2.6.8 + ``` + +### 3. Upgrade Rancher + +This section describes how to upgrade normal (Internet-connected) or air-gapped installations of Rancher with Helm. + +:::note Air Gap Instructions: + +If you are installing Rancher in an air-gapped environment, skip the rest of this page and render the Helm template by following the instructions on [this page.](air-gapped-upgrades.md) + +::: + + +Get the values, which were passed with `--set`, from the current Rancher Helm chart that is installed. + +``` +helm get values rancher -n cattle-system + +hostname: rancher.my.org +``` + +:::note + +There will be more values that are listed with this command. This is just an example of one of the values. + +::: + +:::tip +Your deployment name may vary; for example, if you're deploying Rancher through the AWS Marketplace, the deployment name is 'rancher-stable'. +Thus: +``` +helm get values rancher-stable -n cattle-system + +hostname: rancher.my.org +``` +::: + +If you are upgrading cert-manager to the latest version from v1.5 or below, follow the [cert-manager upgrade docs](../resources/upgrade-cert-manager.md#option-c-upgrade-cert-manager-from-versions-15-and-below) to learn how to upgrade cert-manager without needing to perform an uninstall or reinstall of Rancher. Otherwise, follow the [steps to upgrade Rancher](#steps-to-upgrade-rancher) below. + +#### Steps to Upgrade Rancher + +Upgrade Rancher to the latest version with all your settings. + +Take all the values from the previous step and append them to the command using `--set key=value`. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` + +:::note + +The above is an example, there may be more values from the previous step that need to be appended. + +::: + +:::tip +If you deploy Rancher through the AWS Marketplace, the deployment name is 'rancher-stable'. +Thus: +``` +helm upgrade rancher-stable rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org +``` +::: + +Alternatively, it's possible to export the current values to a file and reference that file during upgrade. For example, to only change the Rancher version: + +1. Export the current values to a file: + ``` + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` +1. Update only the Rancher version: + + For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + + ``` + helm upgrade rancher rancher-/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version=2.6.8 + ``` + +### 4. Verify the Upgrade + +Log into Rancher to confirm that the upgrade succeeded. + +:::tip + +Having network issues following upgrade? + +See [Restoring Cluster Networking](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md). + +::: + +## Known Upgrade Issues + +A list of known issues for each Rancher version can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums.](https://forums.rancher.com/c/announcements/12) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/feature-flags.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/feature-flags.md new file mode 100644 index 00000000000..377d8111523 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/feature-flags.md @@ -0,0 +1,43 @@ +--- +title: Feature Flags +--- + + + + + +With feature flags, you can try out optional or experimental features, and enable legacy features that are being phased out. + +To learn more about feature values and how to enable them, see [Enabling Experimental Features](../../../pages-for-subheaders/enable-experimental-features.md). + +:::note + +Some feature flags require a restart of the Rancher container. Features that require a restart are marked in the Rancher UI. + +::: + +The following is a list of feature flags available in Rancher. If you've upgraded from a previous Rancher version, you may see additional flags in the Rancher UI, such as `proxy` or `dashboard` (both [discontinued](/versioned_docs/version-2.5/reference-guides/installation-references/feature-flags.md)): + +- `continuous-delivery`: Allows Fleet GitOps to be disabled separately from Fleet. See [Continuous Delivery.](../../../how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md) for more information. +- `fleet`: The Rancher provisioning framework in v2.6 and later requires Fleet. The flag will be automatically enabled when you upgrade, even if you disabled this flag in an earlier version of Rancher. See [Fleet - GitOps at Scale](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) for more information. +- `harvester`: Manages access to the Virtualization Management page, where users can navigate directly to Harvester clusters and access the Harvester UI. See [Harvester Integration](../../../integrations-in-rancher/harvester.md) for more information. +- `istio-virtual-service-ui`: Enables a [visual interface](../../../how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md) to create, read, update, and delete Istio virtual services and destination rules, which are Istio traffic management features. +- `legacy`: Enables a set of features from 2.5.x and earlier, that are slowly being phased out in favor of newer implementations. These are a mix of deprecated features as well as features that will eventually be available to newer versions. This flag is disabled by default on new Rancher installations. If you're upgrading from a previous version of Rancher, this flag is enabled. +- `multi-cluster-management`: Allows multi-cluster provisioning and management of Kubernetes clusters. This flag can only be set at install time. It can't be enabled or disabled later. +- `rke1-custom-node-cleanup`: Enables cleanup of deleted RKE1 custom nodes. We recommend that you keep this flag enabled, to prevent removed nodes from attempting to rejoin the cluster. +- `rke2`: Enables provisioning RKE2 clusters. This flag is enabled by default. +- `token-hashing`: Enables token hashing. Once enabled, existing tokens will be hashed and all new tokens will be hashed automatically with the SHA256 algorithm. Once a token is hashed it can't be undone. This flag can't be disabled after its enabled. See [API Tokens](../../../reference-guides/about-the-api/api-tokens.md#token-hashing) for more information. +- `unsupported-storage-drivers`: Enables types for storage providers and provisioners that aren't enabled by default. See [Allow Unsupported Storage Drivers](../../../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) for more information. + +The following table shows the availability and default values for some feature flags in Rancher. Features marked "GA" are generally available: + +| Feature Flag Name | Default Value | Status | Available As Of | +| ----------------------------- | ------------- | ------------ | --------------- | +| `continuous-delivery` | `true` | GA | v2.6.0 | +| `fleet` | `true` | Can no longer be disabled | v2.6.0 | +| `fleet` | `true` | GA | v2.5.0 | +| `harvester` | `true` | Experimental | v2.6.1 | +| `legacy` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | +| `rke1-custom-node-cleanup`| `true` | GA | v2.6.0 | +| `rke2` | `true` | Experimental | v2.6.0 | +| `token-hashing` | `false` for new installs, `true` for upgrades | GA | v2.6.0 | diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md new file mode 100644 index 00000000000..bf6ea01ca23 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/helm-chart-options.md @@ -0,0 +1,314 @@ +--- +title: Rancher Helm Chart Options +keywords: [rancher helm chart, rancher helm options, rancher helm chart options, helm chart rancher, helm options rancher, helm chart options rancher] +--- + + + + + +This page is a configuration reference for the Rancher Helm chart. + +For help choosing a Helm chart version, refer to [this page.](../../../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md) + +For information on enabling experimental features, refer to [this page.](../../../pages-for-subheaders/enable-experimental-features.md) + +## Common Options + +| Option | Default Value | Description | +| ------------------------- | ------------- | ---------------------------------------------------------------------------------- | +| `bootstrapPassword` | " " | `string` - Set the [bootstrap password](#bootstrap-password) for the first admin user. After logging in, the admin will need to reset their password. A randomly generated bootstrap password is used if this value is not set. +| `hostname` | " " | `string` - the Fully Qualified Domain Name for your Rancher Server | +| `ingress.tls.source` | "rancher" | `string` - Where to get the cert for the ingress. - "rancher, letsEncrypt, secret" | +| `letsEncrypt.email` | " " | `string` - Your email address | +| `letsEncrypt.environment` | "production" | `string` - Valid options: "staging, production" | +| `privateCA` | false | `bool` - Set to true if your cert is signed by a private CA | + +
+ +## Advanced Options + +| Option | Default Value | Description | +| ------------------------------ | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `additionalTrustedCAs` | false | `bool` - See [Additional Trusted CAs](#additional-trusted-cas) | +| `addLocal` | "true" | `string` - Have Rancher detect and import the "local" Rancher server cluster. _Note: This option is no longer available in v2.5.0. Consider using the `restrictedAdmin` option to prevent users from modifying the local cluster._ | +| `antiAffinity` | "preferred" | `string` - AntiAffinity rule for Rancher pods - "preferred, required" | +| `auditLog.destination` | "sidecar" | `string` - Stream to sidecar container console or hostPath volume - "sidecar, hostPath" | +| `auditLog.hostPath` | "/var/log/rancher/audit" | `string` - log file destination on host (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.level` | 0 | `int` - set the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) level. 0 is off. [0-3] | +| `auditLog.maxAge` | 1 | `int` - maximum number of days to retain old audit log files (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.maxBackup` | 1 | `int` - maximum number of audit log files to retain (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.maxSize` | 100 | `int` - maximum size in megabytes of the audit log file before it gets rotated (only applies when `auditLog.destination` is set to `hostPath`) | +| `auditLog.image.repository` | "registry.suse.com/bci/bci-micro" | `string` - Location for the image used to collect audit logs. | +| `auditLog.image.tag` | "15.4.14.3" | `string` - Tag for the image used to collect audit logs. | +| `auditLog.image.pullPolicy` | "IfNotPresent" | `string` - Override imagePullPolicy for auditLog images - "Always", "Never", "IfNotPresent". | +| `busyboxImage` | "" | `string` - Image location for busybox image used to collect audit logs. _Note: This option is deprecated use `auditLog.image.repository` to control auditing sidecar image._ | +| `certmanager.version` | "" | `string` - set cert-manager compatibility | +| `debug` | false | `bool` - set debug flag on rancher server | +| `extraEnv` | [] | `list` - set additional environment variables for Rancher | +| `imagePullSecrets` | [] | `list` - list of names of Secret resource containing private registry credentials | +| `ingress.configurationSnippet` | "" | `string` - Add additional Nginx configuration. Can be used for proxy configuration. | +| `ingress.extraAnnotations` | {} | `map` - additional annotations to customize the ingress | +| `ingress.enabled` | true | When set to false, Helm will not install a Rancher ingress. Set the option to false to deploy your own ingress. | +| `letsEncrypt.ingress.class` | "" | `string` - optional ingress class for the cert-manager acmesolver ingress that responds to the Let's Encrypt ACME challenges. Options: traefik, nginx. | | +| `noProxy` | "127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local,cattle-system.svc" | `string` - comma separated list of hostnames or ip address not to use the proxy | | +| `proxy` | "" | `string` - HTTP[S] proxy server for Rancher | +| `rancherImage` | "rancher/rancher" | `string` - rancher image source | +| `rancherImagePullPolicy` | "IfNotPresent" | `string` - Override imagePullPolicy for rancher server images - "Always", "Never", "IfNotPresent" | +| `rancherImageTag` | same as chart version | `string` - rancher/rancher image tag | +| `replicas` | 3 | `int` - Number of Rancher server replicas. Setting to -1 will dynamically choose 1, 2, or 3 based on the number of available nodes in the cluster. | +| `resources` | {} | `map` - rancher pod resource requests & limits | +| `restrictedAdmin` | `false` | `bool` - When this option is set to true, the initial Rancher user has restricted access to the local Kubernetes cluster to prevent privilege escalation. For more information, see the section about the [restricted-admin role.](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#restricted-admin) | +| `systemDefaultRegistry` | "" | `string` - private registry to be used for all system container images, e.g., http://registry.example.com/ | +| `tls` | "ingress" | `string` - See [External TLS Termination](#external-tls-termination) for details. - "ingress, external" | +| `useBundledSystemChart` | `false` | `bool` - select to use the system-charts packaged with Rancher server. This option is used for air gapped installations. | +| `global.cattle.psp.enabled` | `true` | `bool` - select 'false' to disable PSPs for Kubernetes v1.25 and above when using Rancher v2.7.2-v2.7.4. When using Rancher v2.7.5 and above, Rancher attempts to detect if a cluster is running a Kubernetes version where PSPs are not supported, and will default it's usage of PSPs to false if it can determine that PSPs are not supported in the cluster. Users can still manually override this by explicitly providing `true` or `false` for this value. Rancher will still use PSPs by default in clusters which support PSPs (such as clusters running Kubernetes v1.24 or lower). | + + +### Bootstrap Password + +When Rancher starts for the first time, a password is randomly generated for the first admin user. When the admin first logs in to Rancher, the UI shows commands that can be used to retrieve the bootstrap password. The admin needs to run those commands and log in with the bootstrap password. Then Rancher gives the admin an opportunity to reset the password. + +If you want to use a specific bootstrap password instead of a randomly generated one, provide the password. + +```plain +--set bootstrapPassword="rancher" +``` + +The password, whether provided or generated, will be stored in a Kubernetes secret. After Rancher is installed, the UI will show instructions for how to retrieve the password using kubectl: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### API Audit Log + +Enabling the [API Audit Log](../../../how-to-guides/advanced-user-guides/enable-api-audit-log.md). + +You can collect this log as you would any container log. Enable [logging](../../../pages-for-subheaders/logging.md) for the `System` Project on the Rancher server cluster. + +```plain +--set auditLog.level=1 +``` + +By default enabling Audit Logging will create a sidecar container in the Rancher pod. This container (`rancher-audit-log`) will stream the log to `stdout`. You can collect this log as you would any container log. When using the sidecar as the audit log destination, the `hostPath`, `maxAge`, `maxBackups`, and `maxSize` options do not apply. It's advised to use your OS or Docker daemon's log rotation features to control disk space use. Enable [logging](../../../pages-for-subheaders/logging.md) for the Rancher server cluster or System Project. + +Set the `auditLog.destination` to `hostPath` to forward logs to volume shared with the host system instead of streaming to a sidecar container. When setting the destination to `hostPath` you may want to adjust the other auditLog parameters for log rotation. + +### Setting Extra Environment Variables + +You can set extra environment variables for Rancher server using `extraEnv`. This list is passed to the Rancher deployment in its YAML format. It is embedded under `env` for the Rancher container. Refer to the Kubernetes documentation for setting container environment variables, `extraEnv` can use any of the keys referenced in [Define Environment Variables for a Container](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container). + +Consider an example that uses the `name` and `value` keys: + +```plain +--set 'extraEnv[0].name=CATTLE_TLS_MIN_VERSION' +--set 'extraEnv[0].value=1.0' +``` + +If passing sensitive data as the value for an environment variable, such as proxy authentication credentials, it is strongly recommended that a secret reference is used. This will prevent sensitive data from being exposed in Helm or the Rancher deployment. + +Consider an example that uses the `name`, `valueFrom.secretKeyRef.name`, and `valueFrom.secretKeyRef.key` keys. See example in [HTTP Proxy](#http-proxy) + +### TLS Settings + +When you install Rancher inside of a Kubernetes cluster, TLS is offloaded at the cluster's ingress controller. The possible TLS settings depend on the used ingress controller. + +See [TLS settings](tls-settings.md) for more information and options. + +### Import `local` Cluster + +By default Rancher server will detect and import the `local` cluster it's running on. User with access to the `local` cluster will essentially have "root" access to all the clusters managed by Rancher server. + +:::caution + +If you turn addLocal off, most Rancher v2.5 features won't work, including the EKS provisioner. + +::: + +If this is a concern in your environment you can set this option to "false" on your initial install. + +This option is only effective on the initial Rancher install. See [Issue 16522](https://github.com/rancher/rancher/issues/16522) for more information. + +```plain +--set addLocal="false" +``` + +### Customizing your Ingress + +To customize or use a different ingress with Rancher server you can set your own Ingress annotations. + +Example on setting a custom certificate issuer: + +```plain +--set ingress.extraAnnotations.'cert-manager\.io/cluster-issuer'=issuer-name +``` + +Example on setting a static proxy header with `ingress.configurationSnippet`. This value is parsed like a template so variables can be used. + +```plain +--set ingress.configurationSnippet='more_set_input_headers X-Forwarded-Host {{ .Values.hostname }};' +``` + +### HTTP Proxy + +Rancher requires internet access for some functionality (Helm charts). Use `proxy` to set your proxy server or use `extraEnv` to set the `HTTPS_PROXY` environment variable to point to your proxy server. + +Add your IP exceptions to the `noProxy` chart value as a comma separated list. Make sure you add the following values: +- Pod cluster IP range (default: `10.42.0.0/16`). +- Service cluster IP range (default: `10.43.0.0/16`). +- Internal cluster domains (default: `.svc,.cluster.local`). +- Any worker cluster `controlplane` nodes. +Rancher supports CIDR notation ranges in this list. + +When not including sensitive data, the `proxy` or `extraEnv` chart options can be used. When using `extraEnv` the `noProxy` Helm option is ignored. Therefore, the `NO_PROXY` environment variable must also be set with `extraEnv`. + +The following is an example of setting proxy using the `extraEnv` chart option: + +```plain +--set proxy="http:///" +``` + +Example of setting proxy using the `extraEnv` chart option: +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].value="http://:/" +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +When including sensitive data, such as proxy authentication credentials, use the `extraEnv` option with `valueFrom.secretRef` to prevent sensitive data from being exposed in Helm or the Rancher deployment. + +The following is an example of using `extraEnv` to configure proxy. This example secret would contain the value `"http://:@:/"` in the secret's `"https-proxy-url"` key: +```plain +--set extraEnv[1].name=HTTPS_PROXY +--set extraEnv[1].valueFrom.secretKeyRef.name=secret-name +--set extraEnv[1].valueFrom.secretKeyRef.key=https-proxy-url +--set extraEnv[2].name=NO_PROXY +--set extraEnv[2].value="127.0.0.0/8\,10.0.0.0/8\,172.16.0.0/12\,192.168.0.0/16\,.svc\,.cluster.local" +``` + +To learn more about how to configure environment variables, refer to [Define Environment Variables for a Container](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container). + +### Additional Trusted CAs + +If you have private registries, catalogs or a proxy that intercepts certificates, you may need to add additional trusted CAs to Rancher. + +```plain +--set additionalTrustedCAs=true +``` + +Once the Rancher deployment is created, copy your CA certs in pem format into a file named `ca-additional.pem` and use `kubectl` to create the `tls-ca-additional` secret in the `cattle-system` namespace. + +```plain +kubectl -n cattle-system create secret generic tls-ca-additional --from-file=ca-additional.pem=./ca-additional.pem +``` + +### Private Registry and Air Gap Installs + +For details on installing Rancher with a private registry, see the [air gap installation docs.](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) + +## External TLS Termination + +We recommend configuring your load balancer as a Layer 4 balancer, forwarding plain 80/tcp and 443/tcp to the Rancher Management cluster nodes. The Ingress Controller on the cluster will redirect http traffic on port 80 to https on port 443. + +You may terminate the SSL/TLS on a L7 load balancer external to the Rancher cluster (ingress). Use the `--set tls=external` option and point your load balancer at port http 80 on all of the Rancher cluster nodes. This will expose the Rancher interface on http port 80. Be aware that clients that are allowed to connect directly to the Rancher cluster will not be encrypted. If you choose to do this we recommend that you restrict direct access at the network level to just your load balancer. + +:::note + +If you are using a Private CA signed certificate, add `--set privateCA=true` and see [Adding TLS Secrets - Using a Private CA Signed Certificate](../../../getting-started/installation-and-upgrade/resources/add-tls-secrets.md) to add the CA cert for Rancher. + +::: + +Your load balancer must support long lived websocket connections and will need to insert proxy headers so Rancher can route links correctly. + +### Configuring Ingress for External TLS when Using NGINX v0.25 + +In NGINX v0.25, the behavior of NGINX has [changed](https://github.com/kubernetes/ingress-nginx/blob/master/Changelog.md#0220) regarding forwarding headers and external TLS termination. Therefore, in the scenario that you are using external TLS termination configuration with NGINX v0.25, you must edit the `cluster.yml` to enable the `use-forwarded-headers` option for ingress: + +```yaml +ingress: + provider: nginx + options: + use-forwarded-headers: 'true' +``` + +### Required Headers + +- `Host` +- `X-Forwarded-Proto` +- `X-Forwarded-Port` +- `X-Forwarded-For` + +### Recommended Timeouts + +- Read Timeout: `1800 seconds` +- Write Timeout: `1800 seconds` +- Connect Timeout: `30 seconds` + +### Health Checks + +Rancher will respond `200` to health checks on the `/healthz` endpoint. + +### Example NGINX config + +This NGINX configuration is tested on NGINX 1.14. + +:::caution + +This NGINX configuration is only an example and may not suit your environment. For complete documentation, see [NGINX Load Balancing - HTTP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/). + +::: + +- Replace `IP_NODE1`, `IP_NODE2` and `IP_NODE3` with the IP addresses of the nodes in your cluster. +- Replace both occurrences of `FQDN` to the DNS name for Rancher. +- Replace `/certs/fullchain.pem` and `/certs/privkey.pem` to the location of the server certificate and the server certificate key respectively. + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server IP_NODE_1:80; + server IP_NODE_2:80; + server IP_NODE_3:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # This allows the ability for the execute shell window to remain open for up to 15 minutes. Without this parameter, the default is 1 minute and will automatically close. + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/tls-settings.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/tls-settings.md new file mode 100644 index 00000000000..321cb524dbf --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-references/tls-settings.md @@ -0,0 +1,25 @@ +--- +title: TLS Settings +--- + + + + + +Changing the default TLS settings depends on the chosen installation method. + +## Running Rancher in a highly available Kubernetes cluster + +When you install Rancher inside of a Kubernetes cluster, TLS is offloaded at the cluster's ingress controller. The possible TLS settings depend on the used ingress controller: + +* nginx-ingress-controller (default for RKE1 and RKE2): [Default TLS Version and Ciphers](https://kubernetes.github.io/ingress-nginx/user-guide/tls/#default-tls-version-and-ciphers). +* traefik (default for K3s): [TLS Options](https://doc.traefik.io/traefik/https/tls/#tls-options). + +## Running Rancher in a single Docker container + +The default TLS configuration only accepts TLS 1.2 and secure TLS cipher suites. You can change this by setting the following environment variables: + +| Parameter | Description | Default | Available options | +|-----|-----|-----|-----| +| `CATTLE_TLS_MIN_VERSION` | Minimum TLS version | `1.2` | `1.0`, `1.1`, `1.2`, `1.3` | +| `CATTLE_TLS_CIPHERS` | Allowed TLS cipher suites | `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305`,
`TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256`,
`TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384`,
`TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305` | See [Golang tls constants](https://golang.org/pkg/crypto/tls/#pkg-constants) | diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/dockershim.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/dockershim.md new file mode 100644 index 00000000000..e215e0cfc2e --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/dockershim.md @@ -0,0 +1,47 @@ +--- +title: Dockershim +--- + +The Dockershim is the CRI compliant layer between the Kubelet and the Docker daemon. As part of the Kubernetes 1.20 release, the [deprecation of the in-tree Dockershim was announced](https://kubernetes.io/blog/2020/12/02/dont-panic-kubernetes-and-docker/). For more information on the deprecation and its timelines, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). + +RKE clusters now support the external Dockershim to continue leveraging Docker as the CRI runtime. We now implement the upstream open source community external Dockershim announced by [Mirantis and Docker](https://www.mirantis.com/blog/mirantis-to-take-over-support-of-kubernetes-dockershim-2/) to ensure RKE clusters can continue to leverage Docker. + +RKE2 and K3s clusters use an embedded containerd as a container runtime and are not affected. + +To enable the external Dockershim in versions of RKE before 1.24, configure the following option. + +``` +enable_cri_dockerd: true +``` + +Starting with version 1.24, the above defaults to true. + +For users looking to use another container runtime, Rancher has the edge-focused K3s and datacenter-focused RKE2 Kubernetes distributions that use containerd as the default runtime. Imported RKE2 and K3s Kubernetes clusters can then be upgraded and managed through Rancher going forward. + +### FAQ + +
+ +Q: Do I have to upgrade Rancher to get Rancher’s support of the upstream external Dockershim replacement? + +A: The upstream support of the Dockershim replacement `cri_dockerd` begins for RKE in Kubernetes 1.21. You will need to be on a version of Rancher that supports RKE 1.21. See our support matrix for details. + +
+ +Q: I am currently on RKE with Kubernetes 1.23. What happens when upstream finally removes Dockershim in 1.24? + +A: The version of Dockershim in RKE with Kubernetes will continue to work until 1.23. For information on the timeline, see the [Kubernetes Dockershim Deprecation FAQ](https://kubernetes.io/blog/2020/12/02/dockershim-faq/#when-will-dockershim-be-removed). After this, starting in 1.24, RKE will default to enabling `cri_dockerd` by default and will continue to do for versions afterwards. + +
+ +Q: What are my other options if I don’t want to depend on the Dockershim or cri_dockerd? + +A: You can use a runtime like containerd with Kubernetes that does not require Dockershim support. RKE2 or K3s are two options for doing this. + +
+ +Q: If I am already using RKE1 and want to switch to RKE2, what are my migration options? + +A: Today, you can stand up a new cluster and migrate workloads to a new RKE2 cluster that uses containerd. Rancher is exploring the possibility of an in-place upgrade path. + +
diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/install-docker.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/install-docker.md new file mode 100644 index 00000000000..89c45c55e79 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/install-docker.md @@ -0,0 +1,27 @@ +--- +title: Installing Docker +--- + + + + + +Docker is required to be installed on nodes where the Rancher server will be installed with Helm on an RKE cluster or with Docker. Docker is not required for RKE2 or K3s clusters. + +There are a couple of options for installing Docker. One option is to refer to the [official Docker documentation](https://docs.docker.com/install/) about how to install Docker on Linux. The steps will vary based on the Linux distribution. + +Another option is to use one of Rancher's Docker installation scripts, which are available for most recent versions of Docker. + +For example, this command could be used to install Docker 20.10 on the main Linux distributions like SUSE Linux Enterprise or Ubuntu: + +``` +curl https://releases.rancher.com/install-docker/20.10.sh | sh +``` + +Rancher has installation scripts for every version of upstream Docker that Kubernetes supports. To find out whether a script is available for installing a certain Docker version, refer to this [GitHub repository,](https://github.com/rancher/install-docker) which contains all of Rancher's Docker installation scripts. + +Note that the following sysctl setting must be applied: + +``` +net.bridge.bridge-nf-call-iptables=1 +``` diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md new file mode 100644 index 00000000000..73c739499a5 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/installation-requirements/port-requirements.md @@ -0,0 +1,345 @@ +--- +title: Port Requirements +description: Read about port requirements needed in order for Rancher to operate properly, both for Rancher nodes and downstream Kubernetes cluster nodes +--- + + + + + +import PortsIaasNodes from '@site/src/components/PortsIaasNodes' +import PortsCustomNodes from '@site/src/components/PortsCustomNodes' +import PortsImportedHosted from '@site/src/components/PortsImportedHosted' + +To operate properly, Rancher requires a number of ports to be open on Rancher nodes and on downstream Kubernetes cluster nodes. + +## Rancher Nodes + +The following table lists the ports that need to be open to and from nodes that are running the Rancher server. + +The port requirements differ based on the Rancher server architecture. + +Rancher can be installed on any Kubernetes cluster. For Rancher installs on a K3s, RKE, or RKE2 Kubernetes cluster, refer to the tabs below. For other Kubernetes distributions, refer to the distribution's documentation for the port requirements for cluster nodes. + +:::note Notes: + +- Rancher nodes may also require additional outbound access for any external authentication provider which is configured (LDAP for example). +- Kubernetes recommends TCP 30000-32767 for node port services. +- For firewalls, traffic may need to be enabled within the cluster and pod CIDR. +- Rancher nodes may also need outbound access to an external S3 location which is used for storing cluster backups (Minio for example). + +::: + +### Ports for Rancher Server Nodes on K3s + +
+ Click to expand + +The K3s server needs port 6443 to be accessible by the nodes. + +The nodes need to be able to reach other nodes over UDP port 8472 when Flannel VXLAN is used. The node should not listen on any other port. K3s uses reverse tunneling such that the nodes make outbound connections to the server and all kubelet traffic runs through that tunnel. However, if you do not use Flannel and provide your own custom CNI, then port 8472 is not needed by K3s. + +If you wish to utilize the metrics server, you will need to open port 10250 on each node. + +:::note Important: + +The VXLAN port on nodes should not be exposed to the world as it opens up your cluster network to be accessed by anyone. Run your nodes behind a firewall/security group that disables access to port 8472. + +::: + +The following tables break down the port requirements for inbound and outbound traffic: + +
Inbound Rules for Rancher Server Nodes
+ +| Protocol | Port | Source | Description +|-----|-----|----------------|---| +| TCP | 80 | Load balancer/proxy that does external SSL termination | Rancher UI/API when external SSL termination is used | +| TCP | 443 |
  • server nodes
  • agent nodes
  • hosted/registered Kubernetes
  • any source that needs to be able to use the Rancher UI or API
| Rancher agent, Rancher UI/API, kubectl | +| TCP | 6443 | K3s server nodes | Kubernetes API +| UDP | 8472 | K3s server and agent nodes | Required only for Flannel VXLAN. +| TCP | 10250 | K3s server and agent nodes | kubelet + +
Outbound Rules for Rancher Nodes
+ +| Protocol | Port | Destination | Description | +| -------- | ---- | -------------------------------------------------------- | --------------------------------------------- | +| TCP | 22 | Any node IP from a node created using Node Driver | SSH provisioning of nodes using Node Driver | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | Any node IP from a node created using Node driver | Docker daemon TLS port used by Docker Machine | +| TCP | 6443 | Hosted/Imported Kubernetes API | Kubernetes API server | + +
+ +### Ports for Rancher Server Nodes on RKE + +
+ Click to expand + +Typically Rancher is installed on three RKE nodes that all have the etcd, control plane and worker roles. + +The following tables break down the port requirements for traffic between the Rancher nodes: + +
Rules for traffic between Rancher nodes
+ +| Protocol | Port | Description | +|-----|-----|----------------| +| TCP | 443 | Rancher agents | +| TCP | 2379 | etcd client requests | +| TCP | 2380 | etcd peer communication | +| TCP | 6443 | Kubernetes apiserver | +| TCP | 8443 | Nginx Ingress's Validating Webhook | +| UDP | 8472 | Canal/Flannel VXLAN overlay networking | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 10250 | Metrics server communication with all nodes | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | + +The following tables break down the port requirements for inbound and outbound traffic: + +
Inbound Rules for Rancher Nodes
+ +| Protocol | Port | Source | Description | +|-----|-----|----------------|---| +| TCP | 22 | RKE CLI | SSH provisioning of node by RKE | +| TCP | 80 | Load Balancer/Reverse Proxy | HTTP traffic to Rancher UI/API | +| TCP | 443 |
  • Load Balancer/Reverse Proxy
  • IPs of all cluster nodes and other API/UI clients
| HTTPS traffic to Rancher UI/API | +| TCP | 6443 | Kubernetes API clients | HTTPS traffic to Kubernetes API | + +
Outbound Rules for Rancher Nodes
+ +| Protocol | Port | Destination | Description | +|-----|-----|----------------|---| +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 22 | Any node created using a node driver | SSH provisioning of node by node driver | +| TCP | 2376 | Any node created using a node driver | Docker daemon TLS port used by node driver | +| TCP | 6443 | Hosted/Imported Kubernetes API | Kubernetes API server | +| TCP | Provider dependent | Port of the Kubernetes API endpoint in hosted cluster | Kubernetes API | + +
+ +### Ports for Rancher Server Nodes on RKE2 + +
+ Click to expand + +The RKE2 server needs port 6443 and 9345 to be accessible by other nodes in the cluster. + +All nodes need to be able to reach other nodes over UDP port 8472 when Flannel VXLAN is used. + +If you wish to utilize the metrics server, you will need to open port 10250 on each node. + +:::note Important: + +The VXLAN port on nodes should not be exposed to the world as it opens up your cluster network to be accessed by anyone. Run your nodes behind a firewall/security group that disables access to port 8472. + +::: + +
Inbound Rules for RKE2 Server Nodes
+ +| Protocol | Port | Source | Description +|-----|-----|----------------|---| +| TCP | 9345 | RKE2 server and agent nodes | Node registration. Port should be open on all server nodes to all other nodes in the cluster. +| TCP | 6443 | RKE2 agent nodes | Kubernetes API +| UDP | 8472 | RKE2 server and agent nodes | Required only for Flannel VXLAN +| TCP | 10250 | RKE2 server and agent nodes | kubelet +| TCP | 2379 | RKE2 server nodes | etcd client port +| TCP | 2380 | RKE2 server nodes | etcd peer port +| TCP | 30000-32767 | RKE2 server and agent nodes | NodePort port range. Can use TCP or UDP. +| TCP | 5473 | Calico-node pod connecting to typha pod | Required when deploying with Calico +| HTTP | 80 | Load balancer/proxy that does external SSL termination | Rancher UI/API when external SSL termination is used | +| HTTPS | 443 |
  • hosted/registered Kubernetes
  • any source that needs to be able to use the Rancher UI or API
| Rancher agent, Rancher UI/API, kubectl. Not needed if you have a load balancer doing TLS termination. | + +Typically all outbound traffic is allowed. +
+ +### Ports for Rancher Server in Docker + +
+ Click to expand + +The following tables break down the port requirements for Rancher nodes, for inbound and outbound traffic: + +
Inbound Rules for Rancher Node
+ +| Protocol | Port | Source | Description +|-----|-----|----------------|---| +| TCP | 80 | Load balancer/proxy that does external SSL termination | Rancher UI/API when external SSL termination is used +| TCP | 443 |
  • hosted/registered Kubernetes
  • any source that needs to be able to use the Rancher UI or API
| Rancher agent, Rancher UI/API, kubectl + +
Outbound Rules for Rancher Node
+ +| Protocol | Port | Source | Description | +|-----|-----|----------------|---| +| TCP | 22 | Any node IP from a node created using Node Driver | SSH provisioning of nodes using Node Driver | +| TCP | 443 | git.rancher.io | Rancher catalog | +| TCP | 2376 | Any node IP from a node created using a node driver | Docker daemon TLS port used by Docker Machine | +| TCP | 6443 | Hosted/Imported Kubernetes API | Kubernetes API server | + +
+ +## Downstream Kubernetes Cluster Nodes + +Downstream Kubernetes clusters run your apps and services. This section describes what ports need to be opened on the nodes in downstream clusters so that Rancher can communicate with them. + +The port requirements differ depending on how the downstream cluster was launched. Each of the tabs below list the ports that need to be opened for different [cluster types](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +The following diagram depicts the ports that are opened for each [cluster type](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +
Port Requirements for the Rancher Management Plane
+ +![Basic Port Requirements](/img/port-communications.svg) + +:::tip + +If security isn't a large concern and you're okay with opening a few additional ports, you can use the table in [Commonly Used Ports](#commonly-used-ports) as your port reference instead of the comprehensive tables below. + +::: + +### Ports for Harvester Clusters + +Refer [here](../../../integrations-in-rancher/harvester.md#port-requirements) for more information on Harvester port requirements. + + +### Ports for Rancher Launched Kubernetes Clusters using Node Pools + +
+ Click to expand + +The following table depicts the port requirements for [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) with nodes created in an [Infrastructure Provider](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +:::note + +The required ports are automatically opened by Rancher during creation of clusters in cloud providers like Amazon EC2 or DigitalOcean. + +::: + + + +
+ +### Ports for Rancher Launched Kubernetes Clusters using Custom Nodes + +
+ Click to expand + +The following table depicts the port requirements for [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) with [Custom Nodes](../../../pages-for-subheaders/use-existing-nodes.md). + + + +
+ +### Ports for Hosted Kubernetes Clusters + +
+ Click to expand + +The following table depicts the port requirements for [hosted clusters](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md). + + + +
+ +### Ports for Registered Clusters + +:::note + +Registered clusters were called imported clusters before Rancher v2.5. + +::: + +
+ Click to expand + +The following table depicts the port requirements for [registered clusters](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md). + + + +
+ + +## Other Port Considerations + +### Commonly Used Ports + +These ports are typically opened on your Kubernetes nodes, regardless of what type of cluster it is. + +import CommonPortsTable from '../../../shared-files/_common-ports-table.md'; + + + +---- + +### Local Node Traffic + +Ports marked as `local traffic` (i.e., `9099 TCP`) in the above requirements are used for Kubernetes healthchecks (`livenessProbe` and`readinessProbe`). +These healthchecks are executed on the node itself. In most cloud environments, this local traffic is allowed by default. + +However, this traffic may be blocked when: + +- You have applied strict host firewall policies on the node. +- You are using nodes that have multiple interfaces (multihomed). + +In these cases, you have to explicitly allow this traffic in your host firewall, or in case of public/private cloud hosted machines (i.e. AWS or OpenStack), in your security group configuration. Keep in mind that when using a security group as source or destination in your security group, explicitly opening ports only applies to the private interface of the nodes / instances. + +### Rancher AWS EC2 Security Group + +When using the [AWS EC2 node driver](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) to provision cluster nodes in Rancher, you can choose to let Rancher create a security group called `rancher-nodes`. The following rules are automatically added to this security group. + +| Type | Protocol | Port Range | Source/Destination | Rule Type | +|-----------------|:--------:|:-----------:|------------------------|:---------:| +| SSH | TCP | 22 | 0.0.0.0/0 | Inbound | +| HTTP | TCP | 80 | 0.0.0.0/0 | Inbound | +| Custom TCP Rule | TCP | 443 | 0.0.0.0/0 | Inbound | +| Custom TCP Rule | TCP | 2376 | 0.0.0.0/0 | Inbound | +| Custom TCP Rule | TCP | 2379-2380 | sg-xxx (rancher-nodes) | Inbound | +| Custom UDP Rule | UDP | 4789 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 6443 | 0.0.0.0/0 | Inbound | +| Custom UDP Rule | UDP | 8472 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 10250-10252 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 10256 | sg-xxx (rancher-nodes) | Inbound | +| Custom TCP Rule | TCP | 30000-32767 | 0.0.0.0/0 | Inbound | +| Custom UDP Rule | UDP | 30000-32767 | 0.0.0.0/0 | Inbound | +| All traffic | All | All | 0.0.0.0/0 | Outbound | + +### Opening SUSE Linux Ports + +SUSE Linux may have a firewall that blocks all ports by default. To open the ports needed for adding the host to a custom cluster, + + + + +1. SSH into the instance. +1. Start YaST in text mode: + ``` + sudo yast2 + ``` + +1. Navigate to **Security and Users** > **Firewall** > **Zones:public** > **Ports**. To navigate within the interface, follow these [instructions](https://doc.opensuse.org/documentation/leap/reference/html/book-reference/cha-yast-text.html#sec-yast-cli-navigate). +1. To open the required ports, enter them into the **TCP Ports** and **UDP Ports** fields. In this example, ports 9796 and 10250 are also opened for monitoring. The resulting fields should look similar to the following: + ```yaml + TCP Ports + 22, 80, 443, 2376, 2379, 2380, 6443, 9099, 9796, 10250, 10254, 30000-32767 + UDP Ports + 8472, 30000-32767 + ``` + +1. When all required ports are enter, select **Accept**. + + + + +1. SSH into the instance. +1. Edit /`etc/sysconfig/SuSEfirewall2` and open the required ports. In this example, ports 9796 and 10250 are also opened for monitoring: + ``` + FW_SERVICES_EXT_TCP="22 80 443 2376 2379 2380 6443 9099 9796 10250 10254 30000:32767" + FW_SERVICES_EXT_UDP="8472 30000:32767" + FW_ROUTE=yes + ``` +1. Restart the firewall with the new ports: + ``` + SuSEfirewall2 + ``` + + + + +**Result:** The node has the open ports required to be added to a custom cluster. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md new file mode 100644 index 00000000000..3c7c4900443 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands.md @@ -0,0 +1,149 @@ +--- +title: Docker Install Commands +--- + +The Docker installation is for Rancher users who want to test out Rancher. + +Instead of running on a Kubernetes cluster, you install the Rancher server component on a single node using a `docker run` command. Since there is only one node and a single Docker container, if the node goes down, there is no copy of the etcd data available on other nodes and you will lose all the data of your Rancher server. + +The backup application can be used to migrate the Rancher server from a Docker install to a Kubernetes install using [these steps.](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +| Environment Variable Key | Environment Variable Value | Description | +| -------------------------------- | -------------------------------- | ---- | +| `CATTLE_SYSTEM_DEFAULT_REGISTRY` | `` | Configure Rancher server to always pull from your private registry when provisioning clusters. | +| `CATTLE_SYSTEM_CATALOG` | `bundled` | Configure Rancher server to use the packaged copy of Helm system charts. The [system charts](https://github.com/rancher/system-charts) repository contains all the catalog items required for features such as monitoring, logging, alerting and global DNS. These [Helm charts](https://github.com/rancher/system-charts) are located in GitHub, but since you are in an air gapped environment, using the charts that are bundled within Rancher is much easier than setting up a Git mirror. | + +:::note Do you want to.. + +- Configure custom CA root certificate to access your services? See [Custom CA root certificate](../../resources/custom-ca-root-certificates.md). +- Record all transactions with the Rancher API? See [API Auditing](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log). + +::: + +Choose from the following options: + +### Option A: Default Self-Signed Certificate + +
+ Click to expand + +If you are installing Rancher in a development or testing environment where identity verification isn't a concern, install Rancher using the self-signed certificate that it generates. This installation option omits the hassle of generating a certificate yourself. + +Log into your Linux host, and then run the installation command below. When entering the command, use the table below to replace each placeholder. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | Your private registry URL and port. | +| `` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to install. | + +Privileged access is [required.](./install-rancher-ha.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` + +
+ +### Option B: Bring Your Own Certificate: Self-Signed + +
+ Click to expand + +In development or testing environments where your team will access your Rancher server, create a self-signed certificate for use with your install so that your team can verify they're connecting to your instance of Rancher. + +:::note Prerequisites: + +From a computer with an internet connection, create a self-signed certificate using [OpenSSL](https://www.openssl.org/) or another method of your choice. + +- The certificate files must be in PEM format. +- In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](../rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +::: + +After creating your certificate, log into your Linux host, and then run the installation command below. When entering the command, use the table below to replace each placeholder. Use the `-v` flag and provide the path to your certificates to mount them in your container. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | +| `` | The path to the certificate authority's certificate. | +| `` | Your private registry URL and port. | +| `` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to install. | + +Privileged access is [required.](./install-rancher-ha.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` + +
+ +### Option C: Bring Your Own Certificate: Signed by Recognized CA + +
+ Click to expand + +In development or testing environments where you're exposing an app publicly, use a certificate signed by a recognized CA so that your user base doesn't encounter security warnings. + +:::note Prerequisite: + +The certificate files must be in PEM format. + +::: + +After obtaining your certificate, log into your Linux host, and then run the installation command below. When entering the command, use the table below to replace each placeholder. Because your certificate is signed by a recognized CA, mounting an additional CA certificate file is unnecessary. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | +| `` | Your private registry URL and port. | +| `` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to install. | + +:::note + +Use the `--no-cacerts` as argument to the container to disable the default CA certificate generated by Rancher. + +::: + +Privileged access is [required.](./install-rancher-ha.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged + /rancher/rancher: +``` + +
+ + + +:::note + +If you don't intend to send telemetry data, opt out [telemetry](../../../../faq/telemetry.md) during the initial login. + +::: + diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md new file mode 100644 index 00000000000..f9066582004 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md @@ -0,0 +1,192 @@ +--- +title: '1. Set up Infrastructure and Private Registry' +--- + +In this section, you will provision the underlying infrastructure for your Rancher management server in an air gapped environment. You will also set up the private container image registry that must be available to your Rancher node(s). + +An air gapped environment is an environment where the Rancher server is installed offline or behind a firewall. + +The infrastructure depends on whether you are installing Rancher on a K3s Kubernetes cluster, an RKE Kubernetes cluster, or a single Docker container. For more information on each installation option, refer to [this page.](../../../../pages-for-subheaders/installation-and-upgrade.md) + +Rancher can be installed on any Kubernetes cluster. The RKE and K3s Kubernetes infrastructure tutorials below are still included for convenience. + + + + +We recommend setting up the following infrastructure for a high-availability installation: + +- **Two Linux nodes,** typically virtual machines, in the infrastructure provider of your choice. +- **An external database** to store the cluster data. PostgreSQL, MySQL, and etcd are supported. +- **A load balancer** to direct traffic to the two nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. +- **A private image registry** to distribute container images to your machines. + +### 1. Set up Linux Nodes + +These hosts will be disconnected from the internet, but require being able to connect with your private registry. + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up External Datastore + +The ability to run Kubernetes using a datastore other than etcd sets K3s apart from other Kubernetes distributions. This feature provides flexibility to Kubernetes operators. The available options allow you to select a datastore that best fits your use case. + +For a high-availability K3s installation, you will need to set up one of the following external databases: + +* [PostgreSQL](https://www.postgresql.org/) (certified against versions 10.7 and 11.5) +* [MySQL](https://www.mysql.com/) (certified against version 5.7) +* [etcd](https://etcd.io/) (certified against version 3.3.15) + +When you install Kubernetes, you will pass in details for K3s to connect to the database. + +For an example of one way to set up the database, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md) for setting up a MySQL database on Amazon's RDS service. + +For the complete list of options that are available for configuring a K3s cluster datastore, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/datastore/) + +### 3. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the K3s tool will deploy a Traefik Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the Traefik Ingress controller to listen for traffic destined for the Rancher hostname. The Traefik Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md) + +:::note Important: + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 4. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the load balancer IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) + +### 5. Set up a Private Image Registry + +Rancher supports air gap installs using a private registry. You must have your own private registry or other means of distributing container images to your machines. + +In a later step, when you set up your K3s Kubernetes cluster, you will create a [private registries configuration file](https://rancher.com/docs/k3s/latest/en/installation/private-registry/) with details from this registry. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +To install the Rancher management server on a high-availability RKE cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in an infrastructure provider such as Amazon's EC2, Google Compute Engine, or vSphere. +- **A load balancer** to direct front-end traffic to the three nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. +- **A private image registry** to distribute container images to your machines. + +These nodes must be in the same region/data center. You may place these servers in separate availability zones. + +### Why three nodes? + +In an RKE cluster, Rancher server data is stored on etcd. This etcd database runs on all three nodes. + +The etcd database requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can suffer from [split brain](https://www.quora.com/What-is-split-brain-in-distributed-systems), requiring the cluster to be restored from backup. If one of the three etcd nodes fails, the two remaining nodes can elect a leader because they have the majority of the total number of etcd nodes. + +### 1. Set up Linux Nodes + +These hosts will be disconnected from the internet, but require being able to connect with your private registry. + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE tool will deploy an NGINX Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the NGINX Ingress controller to listen for traffic destined for the Rancher hostname. The NGINX Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 3. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the LB IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) + +### 4. Set up a Private Image Registry + +Rancher supports air gap installs using a secure private registry. You must have your own private registry or other means of distributing container images to your machines. + +In a later step, when you set up your RKE Kubernetes cluster, you will create a [private registries configuration file](https://rancher.com/docs/rke/latest/en/config-options/private-registries/) with details from this registry. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + + + + +:::note Notes: + +- The Docker installation is for Rancher users that are wanting to test out Rancher. Since there is only one node and a single Docker container, if the node goes down, you will lose all the data of your Rancher server. + +- The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +::: + +### 1. Set up a Linux Node + +This host will be disconnected from the Internet, but needs to be able to connect to your private registry. + +Make sure that your node fulfills the general installation requirements for [OS, containers, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up a Private Docker Registry + +Rancher supports air gap installs using a private registry on your bastion server. You must have your own private registry or other means of distributing container images to your machines. + +If you need help with creating a private registry, please refer to the [official Docker documentation.](https://docs.docker.com/registry/). + + + + +### [Next: Collect and Publish Images to your Private Registry](publish-images.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md new file mode 100644 index 00000000000..2189e94d095 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md @@ -0,0 +1,390 @@ +--- +title: '3. Install Kubernetes (Skip for Docker Installs)' +--- + + + + + +:::note + +Skip this section if you are installing Rancher on a single node with Docker. + +::: + +This section describes how to install a Kubernetes cluster according to our [best practices for the Rancher server environment.](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#environment-for-kubernetes-installations) This cluster should be dedicated to run only the Rancher server. + +Rancher can be installed on any Kubernetes cluster, including hosted Kubernetes providers. + +The steps to set up an air-gapped Kubernetes cluster on RKE, RKE2, or K3s are shown below. + + + + +In this guide, we are assuming you have created your nodes in your air gapped environment and have a secure Docker private registry on your bastion server. + +### Installation Outline + +1. [Prepare Images Directory](#1-prepare-images-directory) +2. [Create Registry YAML](#2-create-registry-yaml) +3. [Install K3s](#3-install-k3s) +4. [Save and Start Using the kubeconfig File](#4-save-and-start-using-the-kubeconfig-file) + +### 1. Prepare Images Directory +Obtain the images tar file for your architecture from the [releases](https://github.com/k3s-io/k3s/releases) page for the version of K3s you will be running. + +Place the tar file in the `images` directory before starting K3s on each node, for example: + +```sh +sudo mkdir -p /var/lib/rancher/k3s/agent/images/ +sudo cp ./k3s-airgap-images-$ARCH.tar /var/lib/rancher/k3s/agent/images/ +``` + +### 2. Create Registry YAML +Create the registries.yaml file at `/etc/rancher/k3s/registries.yaml`. This will tell K3s the necessary details to connect to your private registry. + +The registries.yaml file should look like this before plugging in the necessary information: + +```yaml +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # this is the registry username + password: xxxxxx # this is the registry password + tls: + cert_file: + key_file: + ca_file: +``` + +Note, at this time only secure registries are supported with K3s (SSL with custom CA). + +For more information on private registries configuration file for K3s, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/private-registry/) + +### 3. Install K3s + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +Obtain the K3s binary from the [releases](https://github.com/k3s-io/k3s/releases) page, matching the same version used to get the airgap images tar. +Also obtain the K3s install script at https://get.k3s.io + +Place the binary in `/usr/local/bin` on each node. +Place the install script anywhere on each node, and name it `install.sh`. + +Install K3s on each server: + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= ./install.sh +``` + +Install K3s on each agent: + +``` +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION= K3S_URL=https://:6443 K3S_TOKEN= ./install.sh +``` + +Where `` is the IP or valid DNS of the server and `` is the node-token from the server found at `/var/lib/rancher/k3s/server/node-token`. + +:::note + +K3s additionally provides a `--resolv-conf` flag for kubelets, which may help with configuring DNS in air-gapped networks. + +::: + +### 4. Save and Start Using the kubeconfig File + +When you installed K3s on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/k3s/k3s.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/k3s/k3s.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your load balancer, referring to port 6443. (The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443.) Here is an example `k3s.yaml`: + +```yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your K3s cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/cluster-access/) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### Note on Upgrading + +Upgrading an air-gap environment can be accomplished in the following manner: + +1. Download the new air-gap images (tar file) from the [releases](https://github.com/k3s-io/k3s/releases) page for the version of K3s you will be upgrading to. Place the tar in the `/var/lib/rancher/k3s/agent/images/` directory on each node. Delete the old tar file. +2. Copy and replace the old K3s binary in `/usr/local/bin` on each node. Copy over the install script at https://get.k3s.io (as it is possible it has changed since the last release). Run the script again just as you had done in the past with the same environment variables. +3. Restart the K3s service (if not restarted automatically by installer). + + + + +In this guide, we are assuming you have created your nodes in your air-gapped environment and have a secure Docker private registry on your bastion server. + +### Installation Outline + +1. [Create RKE2 configuration](#1-create-rke2-configuration) +2. [Create Registry YAML](#2-create-registry-yaml) +3. [Install RKE2](#3-install-rke2) +4. [Save and Start Using the kubeconfig File](#4-save-and-start-using-the-kubeconfig-file) + +### 1. Create RKE2 configuration +Create the config.yaml file at `/etc/rancher/rke2/config.yaml`. This will contain all the configuration options necessary to create a highly available RKE2 cluster. + +On the first server the minimum config is: + +``` +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +On each other server the config file should contain the same token and tell RKE2 to connect to the existing first server: + +``` +server: https://ip-of-first-server:9345 +token: my-shared-secret +tls-san: + - loadbalancer-dns-domain.com +``` + +For more information, refer to the [RKE2 documentation](https://docs.rke2.io/install/ha). + +:::note + +RKE2 additionally provides a `resolv-conf` option for kubelets, which may help with configuring DNS in air-gap networks. + +::: + +### 2. Create Registry YAML +Create the registries.yaml file at `/etc/rancher/rke2/registries.yaml`. This will tell RKE2 the necessary details to connect to your private registry. + +The registries.yaml file should look like this before plugging in the necessary information: + +``` +--- +mirrors: + customreg: + endpoint: + - "https://ip-to-server:5000" +configs: + customreg: + auth: + username: xxxxxx # this is the registry username + password: xxxxxx # this is the registry password + tls: + cert_file: + key_file: + ca_file: +``` + +For more information on private registries configuration file for RKE2, refer to the [RKE2 documentation.](https://docs.rke2.io/install/containerd_registry_configuration) + +### 3. Install RKE2 + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) + +Download the install script, rke2, rke2-images, and sha256sum archives from the release and upload them into a directory on each server: + +``` +mkdir /tmp/rke2-artifacts && cd /tmp/rke2-artifacts/ +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2-images.linux-amd64.tar.zst +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/rke2.linux-amd64.tar.gz +wget https://github.com/rancher/rke2/releases/download/v1.21.5%2Brke2r2/sha256sum-amd64.txt +curl -sfL https://get.rke2.io --output install.sh +``` + +Next, run install.sh using the directory on each server, as in the example below: + +``` +INSTALL_RKE2_ARTIFACT_PATH=/tmp/rke2-artifacts sh install.sh +``` + +Then enable and start the service on all servers: + +`` +systemctl enable rke2-server.service +systemctl start rke2-server.service +`` + +For more information, refer to the [RKE2 documentation](https://docs.rke2.io/install/airgap). + +### 4. Save and Start Using the kubeconfig File + +When you installed RKE2 on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/rke2/rke2.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl), a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/rke2/rke2.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your load balancer, referring to port 6443. (The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443.) Here is an example `rke2.yaml`: + +``` +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your RKE2 cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [RKE2 documentation](https://docs.rke2.io/cluster_access) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### Note on Upgrading + +Upgrading an air-gap environment can be accomplished in the following manner: + +1. Download the new air-gap artifacts and install script from the [releases](https://github.com/rancher/rke2/releases) page for the version of RKE2 you will be upgrading to. +2. Run the script again just as you had done in the past with the same environment variables. +3. Restart the RKE2 service. + + + +We will create a Kubernetes cluster using Rancher Kubernetes Engine (RKE). Before being able to start your Kubernetes cluster, you’ll need to install RKE and create a RKE config file. + +### 1. Install RKE + +Install RKE by following the instructions in the [RKE documentation.](https://rancher.com/docs/rke/latest/en/installation/) + +:::note + +Certified version(s) of RKE based on the Rancher version can be found in the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +::: + +### 2. Create an RKE Config File + +From a system that can access ports 22/TCP and 6443/TCP on the Linux host node(s) that you set up in a previous step, use the sample below to create a new file named `rancher-cluster.yml`. + +This file is an RKE configuration file, which is a configuration for the cluster you're deploying Rancher to. + +Replace values in the code sample below with help of the _RKE Options_ table. Use the IP address or DNS names of the three nodes you created. + +:::tip + +For more details on the options available, see the RKE [Config Options](https://rancher.com/docs/rke/latest/en/config-options/). + +::: + +
RKE Options
+ +| Option | Required | Description | +| ------------------ | -------------------- | --------------------------------------------------------------------------------------- | +| `address` | ✓ | The DNS or IP address for the node within the air gapped network. | +| `user` | ✓ | A user that can run Docker commands. | +| `role` | ✓ | List of Kubernetes roles assigned to the node. | +| `internal_address` | optional1 | The DNS or IP address used for internal cluster traffic. | +| `ssh_key_path` | | Path to the SSH private key used to authenticate to the node (defaults to `~/.ssh/id_rsa`). | + +> 1 Some services like AWS EC2 require setting the `internal_address` if you want to use self-referencing security groups or firewalls. + +```yaml +nodes: + - address: 10.10.3.187 # node air gap network IP + internal_address: 172.31.7.22 # node intra-cluster IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.254 # node air gap network IP + internal_address: 172.31.13.132 # node intra-cluster IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + - address: 10.10.3.89 # node air gap network IP + internal_address: 172.31.3.216 # node intra-cluster IP + user: rancher + role: ['controlplane', 'etcd', 'worker'] + ssh_key_path: /home/user/.ssh/id_rsa + +private_registries: + - url: # private registry url + user: rancher + password: '*********' + is_default: true +``` + +### 3. Run RKE + +After configuring `rancher-cluster.yml`, bring up your Kubernetes cluster: + +``` +rke up --config ./rancher-cluster.yml +``` + +### 4. Save Your Files + +:::note Important: + +The files mentioned below are needed to maintain, troubleshoot, and upgrade your cluster. + +::: + +Save a copy of the following files in a secure location: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_cluster.yml`: The [Kubeconfig file](https://rancher.com/docs/rke/latest/en/kubeconfig/) for the cluster, this file contains credentials for full access to the cluster. +- `rancher-cluster.rkestate`: The [Kubernetes Cluster State file](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state), this file contains the current state of the cluster including the RKE configuration and the certificates.

_The Kubernetes Cluster State file is only created when using RKE v0.2.0 or higher._ + + + + +:::note + +The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +::: + +### Issues or errors? + +See the [Troubleshooting](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) page. + +### [Next: Install Rancher](install-rancher-ha.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md new file mode 100644 index 00000000000..d09b5eca604 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md @@ -0,0 +1,250 @@ +--- +title: 4. Install Rancher +--- + + + + + +This section is about how to deploy Rancher for your air gapped environment in a high-availability Kubernetes installation. An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +### Privileged Access for Rancher + +When the Rancher server is deployed in the Docker container, a local Kubernetes cluster is installed within the container for Rancher to use. Because many features of Rancher run as deployments, and privileged mode is required to run containers within containers, you will need to install Rancher with the `--privileged` option. + +## Docker Instructions + +If you want to continue the air gapped installation using Docker commands, skip the rest of this page and follow the instructions on [this page.](docker-install-commands.md) + +## Kubernetes Instructions + +Rancher recommends installing Rancher on a Kubernetes cluster. A highly available Kubernetes install is comprised of three nodes running the Rancher server components on a Kubernetes cluster. The persistence layer (etcd) is also replicated on these three nodes, providing redundancy and data duplication in case one of the nodes fails. + +### 1. Add the Helm Chart Repository + +From a system that has access to the internet, fetch the latest Helm chart and copy the resulting manifests to a system that has access to the Rancher server cluster. + +1. If you haven't already, install `helm` locally on a workstation that has internet access. Note: Refer to the [Helm version requirements](../../resources/helm-version-requirements.md) to choose a version of Helm to install Rancher. + +2. Use `helm repo add` command to add the Helm chart repository that contains charts to install Rancher. For more information about the repository choices and which is best for your use case, see [Choosing a Rancher Version](../../resources/choose-a-rancher-version.md). + - Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` + - Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` + - Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + +3. Fetch the latest Rancher chart. This will pull down the chart and save it in the current directory as a `.tgz` file. + ```plain + helm fetch rancher-/rancher + ``` + + If you require a specific version of Rancher, you can fetch this with the Helm `--version` parameter like in the following example: + ```plain + helm fetch rancher-stable/rancher --version=v2.4.8 + ``` + +### 2. Choose your SSL Configuration + +Rancher Server is designed to be secure by default and requires SSL/TLS configuration. + +When Rancher is installed on an air gapped Kubernetes cluster, there are two recommended options for the source of the certificate. + +:::note + +If you want terminate SSL/TLS externally, see [TLS termination on an External Load Balancer](../../installation-references/helm-chart-options.md#external-tls-termination). + +::: + +| Configuration | Chart option | Description | Requires cert-manager | +| ------------------------------------------ | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| Rancher Generated Self-Signed Certificates | `ingress.tls.source=rancher` | Use certificates issued by Rancher's generated CA (self signed)
This is the **default** and does not need to be added when rendering the Helm template. | yes | +| Certificates from Files | `ingress.tls.source=secret` | Use your own certificate files by creating Kubernetes Secret(s).
This option must be passed when rendering the Rancher Helm template. | no | + +### Helm Chart Options for Air Gap Installations + +When setting up the Rancher Helm template, there are several options in the Helm chart that are designed specifically for air gap installations. + +| Chart Option | Chart Value | Description | +| ----------------------- | -------------------------------- | ---- | +| `certmanager.version` | `` | Configure proper Rancher TLS issuer depending of running cert-manager version. | +| `systemDefaultRegistry` | `` | Configure Rancher server to always pull from your private registry when provisioning clusters. | +| `useBundledSystemChart` | `true` | Configure Rancher server to use the packaged copy of Helm system charts. The [system charts](https://github.com/rancher/system-charts) repository contains all the catalog items required for features such as monitoring, logging, alerting and global DNS. These [Helm charts](https://github.com/rancher/system-charts) are located in GitHub, but since you are in an air gapped environment, using the charts that are bundled within Rancher is much easier than setting up a Git mirror. | + +### 3. Fetch the Cert-Manager Chart + +Based on the choice your made in [2. Choose your SSL Configuration](#2-choose-your-ssl-configuration), complete one of the procedures below. + +#### Option A: Default Self-Signed Certificate + +By default, Rancher generates a CA and uses cert-manager to issue the certificate for access to the Rancher server interface. + +:::note + +Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.11.0, please see our [upgrade cert-manager documentation](../../resources/upgrade-cert-manager.md). + +::: + +##### 1. Add the cert-manager repo + +From a system connected to the internet, add the cert-manager repo to Helm: + +```plain +helm repo add jetstack https://charts.jetstack.io +helm repo update +``` + +##### 2. Fetch the cert-manager chart + +Fetch the latest cert-manager chart available from the [Helm chart repository](https://artifacthub.io/packages/helm/cert-manager/cert-manager). + +```plain +helm fetch jetstack/cert-manager --version v1.11.0 +``` + +##### 3. Retrieve the Cert-Manager CRDs + +Download the required CRD file for cert-manager: + ```plain + curl -L -o cert-manager-crd.yaml https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + ``` + +### 4. Install Rancher + +Copy the fetched charts to a system that has access to the Rancher server cluster to complete installation. + +##### 1. Install Cert-Manager + +Install cert-manager with the same options you would use to install the chart. Remember to set the `image.repository` option to pull the image from your private registry. + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +
+ Click to expand + +If you are using self-signed certificates, install cert-manager: + +1. Create the namespace for cert-manager. + + ```plain + kubectl create namespace cert-manager + ``` + +2. Create the cert-manager CustomResourceDefinitions (CRDs). + + ```plain + kubectl apply -f cert-manager-crd.yaml + ``` + +3. Install cert-manager. + + ```plain + helm install cert-manager ./cert-manager-v1.11.0.tgz \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller \ + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook \ + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector \ + --set startupapicheck.image.repository=/quay.io/jetstack/cert-manager-ctl + ``` + +
+ +##### 2. Install Rancher +First, refer to [Adding TLS Secrets](../../resources/add-tls-secrets.md) to publish the certificate files so Rancher and the ingress controller can use them. + +Then, create the namespace for Rancher using kubectl: + +```plain +kubectl create namespace cattle-system +``` + +Next, install Rancher, declaring your chosen options. Use the reference table below to replace each placeholder. Rancher needs to be configured to use the private registry in order to provision any Rancher launched Kubernetes clusters or Rancher tools. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +Placeholder | Description +------------|------------- +`` | The version number of the output tarball. +`` | The DNS name you pointed at your load balancer. +`` | The DNS name for your private registry. +`` | Cert-manager version running on k8s cluster. + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set certmanager.version= \ + --set rancherImage=/rancher/rancher \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +**Optional**: To install a specific Rancher version, set the `rancherImageTag` value, example: `--set rancherImageTag=v2.5.8` + +#### Option B: Certificates From Files using Kubernetes Secrets + +##### 1. Create secrets + +Create Kubernetes secrets from your own certificates for Rancher to use. The common name for the cert will need to match the `hostname` option in the command below, or the ingress controller will fail to provision the site for Rancher. + +##### 2. Install Rancher + +Install Rancher, declaring your chosen options. Use the reference table below to replace each placeholder. Rancher needs to be configured to use the private registry in order to provision any Rancher launched Kubernetes clusters or Rancher tools. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +| Placeholder | Description | +| -------------------------------- | ----------------------------------------------- | +| `` | The version number of the output tarball. | +| `` | The DNS name you pointed at your load balancer. | +| `` | The DNS name for your private registry. | + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + +If you are using a Private CA signed cert, add `--set privateCA=true` following `--set ingress.tls.source=secret`: + +```plain + helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set privateCA=true \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts +``` + + +The installation is complete. +:::caution + +If you don't intend to send telemetry data, opt out [telemetry](../../../../faq/telemetry.md) during the initial login. Leaving this active in an air-gapped environment can cause issues if the sockets cannot be opened successfully. + +::: + +## Additional Resources + +These resources could be helpful when installing Rancher: + +- [Rancher Helm chart options](../../installation-references/helm-chart-options.md) +- [Adding TLS secrets](../../resources/add-tls-secrets.md) +- [Troubleshooting Rancher Kubernetes Installations](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md new file mode 100644 index 00000000000..bf242ada285 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md @@ -0,0 +1,309 @@ +--- +title: '2. Collect and Publish Images to your Private Registry' +--- + + + + + +This section describes how to set up your private registry so that when you install Rancher, Rancher will pull all the required images from this registry. + +By default, all images used to [provision Kubernetes clusters](../../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) or launch any tools in Rancher, e.g. monitoring, pipelines, alerts, are pulled from Docker Hub. In an air gapped installation of Rancher, you will need a private registry that is located somewhere accessible by your Rancher server. Then, you will load the registry with all the images. + +Populating the private registry with images is the same process for installing Rancher with Docker and for installing Rancher on a Kubernetes cluster. + +The steps in this section differ depending on whether or not you are planning to use Rancher to provision a downstream cluster with Windows nodes or not. By default, we provide the steps of how to populate your private registry assuming that Rancher will provision downstream Kubernetes clusters with only Linux nodes. But if you plan on provisioning any [downstream Kubernetes clusters using Windows nodes](../../../../pages-for-subheaders/use-windows-clusters.md), there are separate instructions to support the images needed. + +:::note Prerequisites: + +You must have a [private registry](https://docs.docker.com/registry/deploying/#run-an-externally-accessible-registry) available to use. + +If the registry has certs, follow [this K3s documentation](https://rancher.com/docs/k3s/latest/en/installation/private-registry/) about adding a private registry. The certs and registry configuration files need to be mounted into the Rancher container. + +::: + + + + +For Rancher servers that will only provision Linux clusters, these are the steps to populate your private registry. + +1. [Find the required assets for your Rancher version](#1-find-the-required-assets-for-your-rancher-version) +2. [Collect the cert-manager image](#2-collect-the-cert-manager-image) (unless you are bringing your own certificates or terminating TLS on a load balancer) +3. [Save the images to your workstation](#3-save-the-images-to-your-workstation) +4. [Populate the private registry](#4-populate-the-private-registry) + +### Prerequisites + +These steps expect you to use a Linux workstation that has internet access, access to your private registry, and at least 20 GB of disk space. + +If you will use ARM64 hosts, the registry must support manifests. As of April 2020, Amazon Elastic Container Registry does not support manifests. + +### 1. Find the required assets for your Rancher version + +1. Go to our [releases page,](https://github.com/rancher/rancher/releases) find the Rancher v2.x.x release that you want to install, and click **Assets**. Note: Don't use releases marked `rc` or `Pre-release`, as they are not stable for production environments. + +2. From the release's **Assets** section, download the following files, which are required to install Rancher in an air gap environment: + +| Release File | Description | +| ---------------- | -------------- | +| `rancher-images.txt` | This file contains a list of images needed to install Rancher, provision clusters and user Rancher tools. | +| `rancher-save-images.sh` | This script pulls all the images in the `rancher-images.txt` from Docker Hub and saves all of the images as `rancher-images.tar.gz`. | +| `rancher-load-images.sh` | This script loads images from the `rancher-images.tar.gz` file and pushes them to your private registry. | + +### 2. Collect the cert-manager image + +:::note + +Skip this step if you are using your own certificates, or if you are terminating TLS on an external load balancer. + +::: + +In a Kubernetes Install, if you elect to use the Rancher default self-signed TLS certificates, you must add the [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) image to `rancher-images.txt` as well. + +1. Fetch the latest `cert-manager` Helm chart and parse the template for image details: + + :::note + + Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.12.0, please see our [upgrade documentation](../../resources/upgrade-cert-manager.md). + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. Sort and unique the images list to remove any overlap between the sources: + + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + +### 3. Save the images to your workstation + +1. Make `rancher-save-images.sh` an executable: + ``` + chmod +x rancher-save-images.sh + ``` + +1. Run `rancher-save-images.sh` with the `rancher-images.txt` image list to create a tarball of all the required images: + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + **Result:** Docker begins pulling the images used for an air gap install. Be patient. This process takes a few minutes. When the process completes, your current directory will output a tarball named `rancher-images.tar.gz`. Check that the output is in the directory. + +### 4. Populate the private registry + +Next, you will move the images in the `rancher-images.tar.gz` to your private registry using the scripts to load the images. + +Move the images in the `rancher-images.tar.gz` to your private registry using the scripts to load the images. + +The `rancher-images.txt` is expected to be on the workstation in the same directory that you are running the `rancher-load-images.sh` script. The `rancher-images.tar.gz` should also be in the same directory. + +1. Log into your private registry if required: + ```plain + docker login + ``` +1. Make `rancher-load-images.sh` an executable: + ``` + chmod +x rancher-load-images.sh + ``` + +1. Use `rancher-load-images.sh` to extract, tag and push `rancher-images.txt` and `rancher-images.tar.gz` to your private registry: + ```plain + ./rancher-load-images.sh --image-list ./rancher-images.txt --registry + ``` + + + + +For Rancher servers that will provision Linux and Windows clusters, there are distinctive steps to populate your private registry for the Windows images and the Linux images. Since a Windows cluster is a mix of Linux and Windows nodes, the Linux images pushed into the private registry are manifests. + +## Windows Steps + +The Windows images need to be collected and pushed from a Windows server workstation. + +1. Find the required assets for your Rancher version +2. Save the images to your Windows Server workstation +3. Prepare the Docker daemon +4. Populate the private registry + +### Prerequisites + +These steps expect you to use a Windows Server 1809 workstation that has internet access, access to your private registry, and at least 50 GB of disk space. + +The workstation must have Docker 18.02+ in order to support manifests, which are required when provisioning Windows clusters. + +Your registry must support manifests. As of April 2020, Amazon Elastic Container Registry does not support manifests. + + + +### 1. Find the required assets for your Rancher version + +1. Browse to our [releases page](https://github.com/rancher/rancher/releases) and find the Rancher v2.x.x release that you want to install. Don't download releases marked `rc` or `Pre-release`, as they are not stable for production environments. + +2. From the release's "Assets" section, download the following files: + +| Release File | Description | +|----------------------------|------------------| +| `rancher-windows-images.txt` | This file contains a list of Windows images needed to provision Windows clusters. | +| `rancher-save-images.ps1` | This script pulls all the images in the `rancher-windows-images.txt` from Docker Hub and saves all of the images as `rancher-windows-images.tar.gz`. | +| `rancher-load-images.ps1` | This script loads the images from the `rancher-windows-images.tar.gz` file and pushes them to your private registry. | + + + +### 2. Save the images to your Windows Server workstation + +1. Using `powershell`, go to the directory that has the files that were downloaded in the previous step. + +1. Run `rancher-save-images.ps1` to create a tarball of all the required images: + ```plain + ./rancher-save-images.ps1 + ``` + + **Result:** Docker begins pulling the images used for an air gap install. Be patient. This process takes a few minutes. When the process completes, your current directory will output a tarball named `rancher-windows-images.tar.gz`. Check that the output is in the directory. + + + +### 3. Prepare the Docker daemon + +Append your private registry address to the `allow-nondistributable-artifacts` config field in the Docker daemon (`C:\ProgramData\Docker\config\daemon.json`). Since the base image of Windows images are maintained by the `mcr.microsoft.com` registry, this step is required as the layers in the Microsoft registry are missing from Docker Hub and need to be pulled into the private registry. + + ```json + { + ... + "allow-nondistributable-artifacts": [ + ... + "" + ] + ... + } + ``` + + + +### 4. Populate the private registry + +Move the images in the `rancher-windows-images.tar.gz` to your private registry using the scripts to load the images. + +The `rancher-windows-images.txt` is expected to be on the workstation in the same directory that you are running the `rancher-load-images.ps1` script. The `rancher-windows-images.tar.gz` should also be in the same directory. + +1. Using `powershell`, log into your private registry if required: + ```plain + docker login + ``` + +1. Using `powershell`, use `rancher-load-images.ps1` to extract, tag and push the images from `rancher-images.tar.gz` to your private registry: + ```plain + ./rancher-load-images.ps1 --registry + ``` + +## Linux Steps + +The Linux images need to be collected and pushed from a Linux host, but _must be done after_ populating the Windows images into the private registry. These step are different from the Linux only steps as the Linux images that are pushed will actually manifests that support Windows and Linux images. + +1. Find the required assets for your Rancher version +2. Collect all the required images +3. Save the images to your Linux workstation +4. Populate the private registry + +### Prerequisites + +You must populate the private registry with the Windows images before populating the private registry with Linux images. If you have already populated the registry with Linux images, you will need to follow these instructions again as they will publish manifests that support Windows and Linux images. + +These steps expect you to use a Linux workstation that has internet access, access to your private registry, and at least 20 GB of disk space. + +The workstation must have Docker 18.02+ in order to support manifests, which are required when provisioning Windows clusters. + + + +### 1. Find the required assets for your Rancher version + +1. Browse to our [releases page](https://github.com/rancher/rancher/releases) and find the Rancher v2.x.x release that you want to install. Don't download releases marked `rc` or `Pre-release`, as they are not stable for production environments. Click **Assets**. + +2. From the release's **Assets** section, download the following files: + +| Release File | Description | +|----------------------------| -------------------------- | +| `rancher-images.txt` | This file contains a list of images needed to install Rancher, provision clusters and user Rancher tools. | +| `rancher-windows-images.txt` | This file contains a list of images needed to provision Windows clusters. | +| `rancher-save-images.sh` | This script pulls all the images in the `rancher-images.txt` from Docker Hub and saves all of the images as `rancher-images.tar.gz`. | +| `rancher-load-images.sh` | This script loads images from the `rancher-images.tar.gz` file and pushes them to your private registry. | + + + +### 2. Collect all the required images + +**For Kubernetes Installs using Rancher Generated Self-Signed Certificate:** In a Kubernetes Install, if you elect to use the Rancher default self-signed TLS certificates, you must add the [`cert-manager`](https://artifacthub.io/packages/helm/cert-manager/cert-manager) image to `rancher-images.txt` as well. You skip this step if you are using you using your own certificates. + + +1. Fetch the latest `cert-manager` Helm chart and parse the template for image details: + + :::note + + Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.12.0, please see our [upgrade documentation](../../resources/upgrade-cert-manager.md). + + ::: + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm fetch jetstack/cert-manager --version v1.11.0 + helm template ./cert-manager-.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g >> ./rancher-images.txt + ``` + +2. Sort and unique the images list to remove any overlap between the sources: + ```plain + sort -u rancher-images.txt -o rancher-images.txt + ``` + + + +### 3. Save the images to your workstation + +1. Make `rancher-save-images.sh` an executable: + ``` + chmod +x rancher-save-images.sh + ``` + +1. Run `rancher-save-images.sh` with the `rancher-images.txt` image list to create a tarball of all the required images: + ```plain + ./rancher-save-images.sh --image-list ./rancher-images.txt + ``` + +**Result:** Docker begins pulling the images used for an air gap install. Be patient. This process takes a few minutes. When the process completes, your current directory will output a tarball named `rancher-images.tar.gz`. Check that the output is in the directory. + + + +### 4. Populate the private registry + +Move the images in the `rancher-images.tar.gz` to your private registry using the `rancher-load-images.sh script` to load the images. + +The image list, `rancher-images.txt` or `rancher-windows-images.txt`, is expected to be on the workstation in the same directory that you are running the `rancher-load-images.sh` script. The `rancher-images.tar.gz` should also be in the same directory. + +1. Log into your private registry if required: + ```plain + docker login + ``` + +1. Make `rancher-load-images.sh` an executable: + ``` + chmod +x rancher-load-images.sh + ``` + +1. Use `rancher-load-images.sh` to extract, tag and push the images from `rancher-images.tar.gz` to your private registry: + +```plain +./rancher-load-images.sh --image-list ./rancher-images.txt \ + --windows-image-list ./rancher-windows-images.txt \ + --registry +``` + + + + +### [Next step for Kubernetes Installs - Launch a Kubernetes Cluster](install-kubernetes.md) + +### [Next step for Docker Installs - Install Rancher](install-rancher-ha.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md new file mode 100644 index 00000000000..a5167fbd77a --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md @@ -0,0 +1,252 @@ +--- +title: '2. Install Kubernetes' +--- + + + + + +Once the infrastructure is ready, you can continue with setting up a Kubernetes cluster to install Rancher in. + +The steps to set up RKE, RKE2, or K3s are shown below. + +For convenience, export the IP address and port of your proxy into an environment variable and set up the HTTP_PROXY variables for your current shell on every node: + +``` +export proxy_host="10.0.0.5:8888" +export HTTP_PROXY=http://${proxy_host} +export HTTPS_PROXY=http://${proxy_host} +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16 +``` + + + + +First configure the HTTP proxy settings on the K3s systemd service, so that K3s's containerd can pull images through the proxy: + +``` +cat <<'EOF' | sudo tee /etc/default/k3s > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +On the first node, create a new cluster: +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --cluster-init +``` + +And then join the other nodes: +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= K3S_TOKEN= sh -s - server --server https://:6443 +``` + +Where `` is the IP or valid DNS of the server and `` is the node-token from the server found at `/var/lib/rancher/k3s/server/node-token`. + +For more information on installing K3s see the [K3s installation docs](https://docs.k3s.io/installation). + +To have a look at your cluster run: + +``` +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +On every node, run the RKE2 installation script. Ensure that the RKE2 version you are installing is [supported by Rancher](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +``` +curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=v1.xx sh - +``` + +Then you have to configure the HTTP proxy settings on the RKE2 systemd service, so that RKE2's containerd can pull images through the proxy: + +``` +cat <<'EOF' | sudo tee /etc/default/rke2-server > /dev/null +HTTP_PROXY=http://${proxy_host} +HTTPS_PROXY=http://${proxy_host}" +NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +EOF +``` + +Next create the RKE2 configuration file on every node following the [RKE2 High Availability documentation](https://docs.rke2.io/install/ha). + +After that start and enable the `rke2-server` service: + +``` +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` + +For more information on installing RKE2 see the [RKE2 documentation](https://docs.rke2.io). + +To have a look at your cluster run: + +``` +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +alias kubectl=/var/lib/rancher/rke2/bin/kubectl +kubectl cluster-info +kubectl get pods --all-namespaces +``` + + + + +First, you have to install Docker and setup the HTTP proxy on all three Linux nodes. For this perform the following steps on all three nodes. + +Next configure apt to use this proxy when installing packages. If you are not using Ubuntu, you have to adapt this step accordingly: + +``` +cat <<'EOF' | sudo tee /etc/apt/apt.conf.d/proxy.conf > /dev/null +Acquire::http::Proxy "http://${proxy_host}/"; +Acquire::https::Proxy "http://${proxy_host}/"; +EOF +``` + +Now you can install Docker: + +``` +curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh +``` + +Then ensure that your current user is able to access the Docker daemon without sudo: + +``` +sudo usermod -aG docker YOUR_USERNAME +``` + +And configure the Docker daemon to use the proxy to pull images: + +``` +sudo mkdir -p /etc/systemd/system/docker.service.d +cat <<'EOF' | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf > /dev/null +[Service] +Environment="HTTP_PROXY=http://${proxy_host}" +Environment="HTTPS_PROXY=http://${proxy_host}" +Environment="NO_PROXY=127.0.0.0/8,10.0.0.0/8,cattle-system.svc,172.16.0.0/12,192.168.0.0/16" +EOF +``` + +To apply the configuration, restart the Docker daemon: + +``` +sudo systemctl daemon-reload +sudo systemctl restart docker +``` + +#### Air-gapped proxy + +You can now provision node driver clusters from an air-gapped cluster configured to use a proxy for outbound connections. + +In addition to setting the default rules for a proxy server, you will need to add additional rules, shown below, to provision node driver clusters from a proxied Rancher environment. + +You will configure your filepath according to your setup, e.g., `/etc/apt/apt.conf.d/proxy.conf`: + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` + +### Creating the RKE Cluster + +You need several command line tools on the host where you have SSH access to the Linux nodes to create and interact with the cluster: + +* [RKE CLI binary](https://rancher.com/docs/rke/latest/en/installation/#download-the-rke-binary) + +``` +sudo curl -fsSL -o /usr/local/bin/rke https://github.com/rancher/rke/releases/download/v1.1.4/rke_linux-amd64 +sudo chmod +x /usr/local/bin/rke +``` + +* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + +``` +curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" +chmod +x ./kubectl +sudo mv ./kubectl /usr/local/bin/kubectl +``` + +Next, create a YAML file that describes the RKE cluster. Ensure that the IP addresses of the nodes and the SSH username are correct. For more information on the cluster YAML, have a look at the [RKE documentation](https://rancher.com/docs/rke/latest/en/example-yamls/). + +```yml +nodes: + - address: 10.0.1.200 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.201 + user: ubuntu + role: [controlplane,worker,etcd] + - address: 10.0.1.202 + user: ubuntu + role: [controlplane,worker,etcd] + +services: + etcd: + backup_config: + interval_hours: 12 + retention: 6 +``` + +After that, you can create the Kubernetes cluster by running: + +``` +rke up --config rancher-cluster.yaml +``` + +RKE creates a state file called `rancher-cluster.rkestate`, this is needed if you want to perform updates, modify your cluster configuration or restore it from a backup. It also creates a `kube_config_cluster.yaml` file, that you can use to connect to the remote Kubernetes cluster locally with tools like kubectl or Helm. Make sure to save all of these files in a secure location, for example by putting them into a version control system. + +To have a look at your cluster run: + +``` +export KUBECONFIG=kube_config_cluster.yaml +kubectl cluster-info +kubectl get pods --all-namespaces +``` + +You can also verify that your external load balancer works, and the DNS entry is set up correctly. If you send a request to either, you should receive HTTP 404 response from the ingress controller: + +``` +$ curl 10.0.1.100 +default backend - 404 +$ curl rancher.example.com +default backend - 404 +``` + +### Save Your Files + +:::note Important: + +The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster. + +::: + +Save a copy of the following files in a secure location: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_cluster.yml`: The [Kubeconfig file](https://rancher.com/docs/rke/latest/en/kubeconfig/) for the cluster, this file contains credentials for full access to the cluster. +- `rancher-cluster.rkestate`: The [Kubernetes Cluster State file](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state), this file contains the current state of the cluster including the RKE configuration and the certificates. + +:::note + +The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +::: + + + + +### Issues or errors? + +See the [Troubleshooting](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) page. + +### [Next: Install Rancher](install-rancher.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md new file mode 100644 index 00000000000..0267e72d0be --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md @@ -0,0 +1,108 @@ +--- +title: 3. Install Rancher +--- + + + + + +Now that you have a running RKE cluster, you can install Rancher in it. For security reasons all traffic to Rancher must be encrypted with TLS. For this tutorial you are going to automatically issue a self-signed certificate through [cert-manager](https://cert-manager.io/). In a real-world use-case you will likely use Let's Encrypt or provide your own certificate. + +### Install the Helm CLI + +Install the [Helm](https://helm.sh/docs/intro/install/) CLI on a host where you have a kubeconfig to access your Kubernetes cluster: + +``` +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod +x get_helm.sh +sudo ./get_helm.sh +``` + +### Install cert-manager + +Add the cert-manager helm repository: + +``` +helm repo add jetstack https://charts.jetstack.io +``` + +Create a namespace for cert-manager: + +``` +kubectl create namespace cert-manager +``` + +Install the CustomResourceDefinitions of cert-manager: + +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml +``` + +And install it with Helm. Note that cert-manager also needs your proxy configured in case it needs to communicate with Let's Encrypt or other external certificate issuers: + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +``` +helm upgrade --install cert-manager jetstack/cert-manager \ + --namespace cert-manager --version v1.11.0 \ + --set http_proxy=http://${proxy_host} \ + --set https_proxy=http://${proxy_host} \ + --set no_proxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +Now you should wait until cert-manager is finished starting up: + +``` +kubectl rollout status deployment -n cert-manager cert-manager +kubectl rollout status deployment -n cert-manager cert-manager-webhook +``` + +### Install Rancher + +Next you can install Rancher itself. First add the helm repository: + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +``` + +Create a namespace: + +``` +kubectl create namespace cattle-system +``` + +And install Rancher with Helm. Rancher also needs a proxy configuration so that it can communicate with external application catalogs or retrieve Kubernetes version update metadata: + +``` +helm upgrade --install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.example.com \ + --set proxy=http://${proxy_host} \ + --set noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local +``` + +After waiting for the deployment to finish: + +``` +kubectl rollout status deployment -n cattle-system rancher +``` + +You can now navigate to `https://rancher.example.com` and start using Rancher. + +:::caution + +If you don't intend to send telemetry data, opt out [telemetry](../../../../faq/telemetry.md) during the initial login. Leaving this active in an air-gapped environment can cause issues if the sockets cannot be opened successfully. + +::: + +### Additional Resources + +These resources could be helpful when installing Rancher: + +- [Rancher Helm chart options](../../installation-references/helm-chart-options.md) +- [Adding TLS secrets](../../resources/add-tls-secrets.md) +- [Troubleshooting Rancher Kubernetes Installations](../../install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md new file mode 100644 index 00000000000..7822fa065cd --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md @@ -0,0 +1,67 @@ +--- +title: '1. Set up Infrastructure' +--- + + + + + +In this section, you will provision the underlying infrastructure for your Rancher management server with internet access through a HTTP proxy. + +To install the Rancher management server on a high-availability RKE cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in an infrastructure provider such as Amazon's EC2, Google Compute Engine, or vSphere. +- **A load balancer** to direct front-end traffic to the three nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +These nodes must be in the same region/data center. You may place these servers in separate availability zones. + +### Why three nodes? + +In an RKE cluster, Rancher server data is stored on etcd. This etcd database runs on all three nodes. + +The etcd database requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can suffer from [split brain](https://www.quora.com/What-is-split-brain-in-distributed-systems), requiring the cluster to be restored from backup. If one of the three etcd nodes fails, the two remaining nodes can elect a leader because they have the majority of the total number of etcd nodes. + +### 1. Set up Linux Nodes + +These hosts will connect to the internet through an HTTP proxy. + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](../../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE tool will deploy an NGINX Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the NGINX Ingress controller to listen for traffic destined for the Rancher hostname. The NGINX Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](../../../../how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md) + +:::note Important: + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 3. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the LB IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) + + +### [Next: Set up a Kubernetes cluster](install-kubernetes.md) diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md new file mode 100644 index 00000000000..6614dd7a4cd --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md @@ -0,0 +1,92 @@ +--- +title: Certificate Troubleshooting +--- + + + + + +### How Do I Know if My Certificates are in PEM Format? + +You can recognize the PEM format by the following traits: + +- The file begins with the following header: + ``` + -----BEGIN CERTIFICATE----- + ``` +- The header is followed by a long string of characters. +- The file ends with a footer: + -----END CERTIFICATE----- + +PEM Certificate Example: + +``` +----BEGIN CERTIFICATE----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END CERTIFICATE----- +``` + +PEM Certificate Key Example: + +``` +-----BEGIN RSA PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END RSA PRIVATE KEY----- +``` + +If your key looks like the example below, see [Converting a Certificate Key From PKCS8 to PKCS1.](#converting-a-certificate-key-from-pkcs8-to-pkcs1) + +``` +-----BEGIN PRIVATE KEY----- +MIIGVDCCBDygAwIBAgIJAMiIrEm29kRLMA0GCSqGSIb3DQEBCwUAMHkxCzAJBgNV +... more lines +VWQqljhfacYPgp8KJUJENQ9h5hZ2nSCrI+W00Jcw4QcEdCI8HL5wmg== +-----END PRIVATE KEY----- +``` + +### Converting a Certificate Key From PKCS8 to PKCS1 + +If you are using a PKCS8 certificate key file, Rancher will log the following line: + +``` +ListenConfigController cli-config [listener] failed with : failed to read private key: asn1: structure error: tags don't match (2 vs {class:0 tag:16 length:13 isCompound:true}) +``` + +To make this work, you will need to convert the key from PKCS8 to PKCS1 using the command below: + +``` +openssl rsa -in key.pem -out convertedkey.pem +``` + +You can now use `convertedkey.pem` as certificate key file for Rancher. + +### What is the Order of Certificates if I Want to Add My Intermediate(s)? + +The order of adding certificates is as follows: + +``` +-----BEGIN CERTIFICATE----- +%YOUR_CERTIFICATE% +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +%YOUR_INTERMEDIATE_CERTIFICATE% +-----END CERTIFICATE----- +``` + +### How Do I Validate My Certificate Chain? + +You can validate the certificate chain by using the `openssl` binary. If the output of the command (see the command example below) ends with `Verify return code: 0 (ok)`, your certificate chain is valid. The `ca.pem` file must be the same as you added to the `rancher/rancher` container. + +When using a certificate signed by a recognized Certificate Authority, you can omit the `-CAfile` parameter. + +Command: + +``` +openssl s_client -CAfile ca.pem -connect rancher.yourdomain.com:443 +... + Verify return code: 0 (ok) +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md new file mode 100644 index 00000000000..1ec111444b9 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher.md @@ -0,0 +1,95 @@ +--- +title: Rolling Back Rancher Installed with Docker +--- + + + + + +If a Rancher upgrade does not complete successfully, you'll have to roll back to your Rancher setup that you were using before [Docker Upgrade](upgrade-docker-installed-rancher.md). Rolling back restores: + +- Your previous version of Rancher. +- Your data backup created before upgrade. + +## Before You Start + +During rollback to a prior version of Rancher, you'll enter a series of commands, filling placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). Here's an example of a command with a placeholder: + +``` +docker pull rancher/rancher: +``` + +In this command, `` is the version of Rancher you were running before your unsuccessful upgrade. `v2.0.5` for example. + +Cross reference the image and reference table below to learn how to obtain this placeholder data. Write down or copy this information before starting the procedure below. + +Terminal docker ps Command, Displaying Where to Find <PRIOR_RANCHER_VERSION> and <RANCHER_CONTAINER_NAME>![Placeholder Reference](/img/placeholder-ref-2.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | ------------------------------------------------------- | +| `` | `v2.0.5` | The rancher/rancher image you used before upgrade. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.0.5` | The version of Rancher that the backup is for. | +| `` | `9-27-18` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher Server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped using a different command: `docker ps -a`. Use these commands for help anytime during while creating backups. + +## Rolling Back Rancher + +If you have issues upgrading Rancher, roll it back to its latest known healthy state by pulling the last version you used and then restoring the backup you made before upgrade. + +:::danger + +Rolling back to a previous version of Rancher destroys any changes made to Rancher following the upgrade. Unrecoverable data loss may occur. + +::: + +1. Using a remote Terminal connection, log into the node running your Rancher Server. + +1. Pull the version of Rancher that you were running before upgrade. Replace the `` with that version. + + For example, if you were running Rancher v2.0.5 before upgrade, pull v2.0.5. + + ``` + docker pull rancher/rancher: + ``` + +1. Stop the container currently running Rancher Server. Replace `` with the name of your Rancher container. + + ``` + docker stop + ``` + You can obtain the name for your Rancher container by entering `docker ps`. + +1. Move the backup tarball that you created during completion of [Docker Upgrade](upgrade-docker-installed-rancher.md) onto your Rancher Server. Change to the directory that you moved it to. Enter `dir` to confirm that it's there. + + If you followed the naming convention we suggested in [Docker Upgrade](upgrade-docker-installed-rancher.md), it will have a name similar to (`rancher-data-backup--.tar.gz`). + +1. Run the following command to replace the data in the `rancher-data` container with the data in the backup tarball, replacing the placeholder. Don't forget to close the quotes. + + ``` + docker run --volumes-from rancher-data \ + -v $PWD:/backup busybox sh -c "rm /var/lib/rancher/* -rf \ + && tar zxvf /backup/rancher-data-backup--.tar.gz" + ``` + +1. Start a new Rancher Server container with the `` tag placeholder pointing to the data container. + ``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: + ``` + Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + + :::danger + + **_Do not_** stop the rollback after initiating it, even if the rollback process seems longer than expected. Stopping the rollback may result in database issues during future upgrades. + + ::: + +1. Wait a few moments and then open Rancher in a web browser. Confirm that the rollback succeeded and that your data is restored. + +**Result:** Rancher is rolled back to its version and data state before upgrade. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md new file mode 100644 index 00000000000..2a3bdaa29df --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher.md @@ -0,0 +1,397 @@ +--- +title: Upgrading Rancher Installed with Docker +--- + + + + + +The following instructions will guide you through upgrading a Rancher server that was installed with Docker. + +:::caution + +**Docker installs are not supported in production environments.** These instructions are provided for testing and development purposes only. If you have already deployed a Docker install in production and need to upgrade to a new Rancher version, we recommend [migrating to the Helm chart install](../../../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) before upgrading. + +::: + +## Prerequisites + +- **Review the [known upgrade issues](../../install-upgrade-on-a-kubernetes-cluster/upgrades.md#known-upgrade-issues)** section in the Rancher documentation for the most noteworthy issues to consider when upgrading Rancher. A more complete list of known issues for each Rancher version can be found in the release notes on [GitHub](https://github.com/rancher/rancher/releases) and on the [Rancher forums](https://forums.rancher.com/c/announcements/12). Note that upgrades to or from any chart in the [rancher-alpha repository](../../resources/choose-a-rancher-version.md#helm-chart-repositories) aren’t supported. +- **For [air gap installs only,](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md) collect and populate images for the new Rancher server version**. Follow the guide to [populate your private registry](../air-gapped-helm-cli-install/publish-images.md) with the images for the Rancher version that you want to upgrade to. + +## Placeholder Review + +During upgrade, you'll enter a series of commands, filling placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). + +Here's an **example** of a command with a placeholder: + +``` +docker stop +``` + +In this command, `` is the name of your Rancher container. + +## Get Data for Upgrade Commands + +To obtain the data to replace the placeholders, run: + +``` +docker ps +``` + +Write down or copy this information before starting the upgrade. + +Terminal docker ps Command, Displaying Where to Find <RANCHER_CONTAINER_TAG> and <RANCHER_CONTAINER_NAME> + +![Placeholder Reference](/img/placeholder-ref.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.1.3` | The rancher/rancher image you pulled for initial install. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.1.3` | The version of Rancher that you're creating a backup for. | +| `` | `2018-12-19` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped using a different command: `docker ps -a`. Use these commands for help anytime during while creating backups. + +## Upgrade + +During upgrade, you create a copy of the data from your current Rancher container and a backup in case something goes wrong. Then you deploy the new version of Rancher in a new container using your existing data. +### 1. Create a copy of the data from your Rancher server container + +1. Using a remote Terminal connection, log into the node running your Rancher server. + +1. Stop the container currently running Rancher server. Replace `` with the name of your Rancher container. + + ``` + docker stop + ``` + +1. Use the command below, replacing each placeholder, to create a data container from the Rancher container that you just stopped. + + ``` + docker create --volumes-from --name rancher-data rancher/rancher: + ``` + +### 2. Create a backup tarball + +1. From the data container that you just created (rancher-data), create a backup tarball (rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz). + + This tarball will serve as a rollback point if something goes wrong during upgrade. Use the following command, replacing each placeholder. + ``` + docker run --volumes-from rancher-data -v "$PWD:/backup" --rm busybox tar zcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **Step Result:** When you enter this command, a series of commands should run. + +1. Enter the `ls` command to confirm that the backup tarball was created. It will have a name similar to `rancher-data-backup--.tar.gz`. + + ``` + [rancher@ip-10-0-0-50 ~]$ ls + rancher-data-backup-v2.1.3-20181219.tar.gz + ``` + +1. Move your backup tarball to a safe location external from your Rancher server. + +### 3. Pull the New Docker Image + +Pull the image of the Rancher version that you want to upgrade to. + +Placeholder | Description +------------|------------- +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker pull rancher/rancher: +``` + +### 4. Start the New Rancher Server Container + +Start a new Rancher server container using the data from the `rancher-data` container. Remember to pass in all the environment variables that you had used when you started the original container. + +:::danger + +**_Do not_** stop the upgrade after initiating it, even if the upgrade process seems longer than expected. Stopping the upgrade may result in database migration errors during future upgrades. + +::: + +If you used a proxy, see [HTTP Proxy Configuration.](../../../../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) + +If you configured a custom CA root certificate to access your services, see [Custom CA root certificate.](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#custom-ca-certificate) + +If you are recording all transactions with the Rancher API, see [API Auditing](../../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) + +To see the command to use when starting the new Rancher server container, choose from the following options: + +- Docker Upgrade +- Docker Upgrade for Air Gap Installs + + + + +Select which option you had installed Rancher server + +#### Option A: Default Self-Signed Certificate + +
+ Click to expand + +If you have selected to use the Rancher generated self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container. + +Placeholder | Description +------------|------------- +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +
+ +#### Option B: Bring Your Own Certificate: Self-Signed + +
+Click to expand + +If you have selected to bring your own self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificate that you had originally installed with. + +:::note Reminder of the Cert Prerequisite: + +The certificate files must be in PEM format. In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. + +::: + +Placeholder | Description +------------|------------- + `` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | The path to the certificate authority's certificate. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher: +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +
+ +#### Option C: Bring Your Own Certificate: Signed by Recognized CA + +
+ Click to expand + +If you have selected to use a certificate signed by a recognized CA, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificates that you had originally installed with. Remember to include `--no-cacerts` as an argument to the container to disable the default CA certificate generated by Rancher. + +:::note Reminder of the Cert Prerequisite: + +The certificate files must be in PEM format. In your certificate file, include all intermediate certificates provided by the recognized CA. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](certificate-troubleshooting.md) + +::: + +Placeholder | Description +------------|------------- +`` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher: \ + --no-cacerts +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ +#### Option D: Let's Encrypt Certificate + +
+ Click to expand + +:::caution + +Let's Encrypt provides rate limits for requesting new certificates. Therefore, limit how often you create or destroy the container. For more information, see [Let's Encrypt documentation on rate limits](https://letsencrypt.org/docs/rate-limits/). + +::: + +If you have selected to use [Let's Encrypt](https://letsencrypt.org/) certificates, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to provide the domain that you had used when you originally installed Rancher. + +:::note Reminder of the Cert Prerequisites: + +- Create a record in your DNS that binds your Linux host IP address to the hostname that you want to use for Rancher access (`rancher.mydomain.com` for example). +- Open port `TCP/80` on your Linux host. The Let's Encrypt http-01 challenge can come from any source IP address, so port `TCP/80` must be open to all IP addresses. + +::: + +Placeholder | Description +------------|------------- +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. +`` | The domain address that you had originally started with + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher: \ + --acme-domain +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +
+ + + + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +When starting the new Rancher server container, choose from the following options: + +#### Option A: Default Self-Signed Certificate + +
+ Click to expand + +If you have selected to use the Rancher generated self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container. + +Placeholder | Description +------------|------------- +`` | Your private registry URL and port. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to to upgrade to. + +``` + docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` + +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ +#### Option B: Bring Your Own Certificate: Self-Signed + +
+ Click to expand + +If you have selected to bring your own self-signed certificate, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificate that you had originally installed with. + +:::note Reminder of the Cert Prerequisite: + +The certificate files must be in PEM format. In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](certificate-troubleshooting.md) + +::: + +Placeholder | Description +------------|------------- +`` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | The path to the certificate authority's certificate. +`` | Your private registry URL and port. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged \ + /rancher/rancher: +``` +Privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ +#### Option C: Bring Your Own Certificate: Signed by Recognized CA + +
+ Click to expand + +If you have selected to use a certificate signed by a recognized CA, you add the `--volumes-from rancher-data` to the command that you had started your original Rancher server container and need to have access to the same certificates that you had originally installed with. + + :::note Reminder of the Cert Prerequisite: + + The certificate files must be in PEM format. In your certificate file, include all intermediate certificates provided by the recognized CA. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](certificate-troubleshooting.md) + + ::: + +Placeholder | Description +------------|------------- +`` | The path to the directory containing your certificate files. +`` | The path to your full certificate chain. +`` | The path to the private key for your certificate. +`` | Your private registry URL and port. +`` | The release tag of the [Rancher version](../../installation-references/helm-chart-options.md) that you want to upgrade to. + +:::note + +Use the `--no-cacerts` as argument to the container to disable the default CA certificate generated by Rancher. + +::: + +``` +docker run -d --volumes-from rancher-data \ + --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --no-cacerts \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -e CATTLE_SYSTEM_DEFAULT_REGISTRY= \ # Set a default private registry to be used in Rancher + -e CATTLE_SYSTEM_CATALOG=bundled \ # Use the packaged Rancher system charts + --privileged + /rancher/rancher: +``` +privileged access is [required.](../../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) +
+ + + + +**Result:** You have upgraded Rancher. Data from your upgraded server is now saved to the `rancher-data` container for use in future upgrades. + +### 5. Verify the Upgrade + +Log into Rancher. Confirm that the upgrade succeeded by checking the version displayed in the bottom-left corner of the browser window. + +:::note Having network issues in your user clusters following upgrade? + +See [Restoring Cluster Networking](/versioned_docs/version-2.0-2.4/getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades/namespace-migration.md). + +::: + +### 6. Clean up Your Old Rancher Server Container + +Remove the previous Rancher server container. If you only stop the previous Rancher server container (and don't remove it), the container may restart after the next server reboot. + +## Rolling Back + +If your upgrade does not complete successfully, you can roll back Rancher server and its data back to its last healthy state. For more information, see [Docker Rollback](roll-back-docker-installed-rancher.md). diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/add-tls-secrets.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/add-tls-secrets.md new file mode 100644 index 00000000000..290f180adfd --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/add-tls-secrets.md @@ -0,0 +1,49 @@ +--- +title: Adding TLS Secrets +--- + + + + + +Kubernetes will create all the objects and services for Rancher, but it will not become available until we populate the `tls-rancher-ingress` secret in the `cattle-system` namespace with the certificate and key. + +Combine the server certificate followed by any intermediate certificate(s) needed into a file named `tls.crt`. Copy your certificate key into a file named `tls.key`. + +For example, [acme.sh](https://acme.sh) provides server certificate and CA chains in `fullchain.cer` file. +This `fullchain.cer` should be renamed to `tls.crt` & certificate key file as `tls.key`. + +Use `kubectl` with the `tls` secret type to create the secrets. + +``` +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +:::note + +If you want to replace the certificate, you can delete the `tls-rancher-ingress` secret using `kubectl -n cattle-system delete secret tls-rancher-ingress` and add a new one using the command shown above. If you are using a private CA signed certificate, replacing the certificate is only possible if the new certificate is signed by the same CA as the certificate currently in use. + +::: + +## Using a Private CA Signed Certificate + +If you are using a private CA, Rancher requires a copy of the private CA's root certificate or certificate chain, which the Rancher Agent uses to validate the connection to the server. + +Create a file named `cacerts.pem` that only contains the root CA certificate or certificate chain from your private CA, and use `kubectl` to create the `tls-ca` secret in the `cattle-system` namespace. + +``` +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem=./cacerts.pem +``` + +:::note + +The configured `tls-ca` secret is retrieved when Rancher starts. On a running Rancher installation the updated CA will take effect after new Rancher pods are started. + +::: + +## Updating a Private CA Certificate + +Follow the steps on [this page](update-rancher-certificate.md) to update the SSL certificate of the ingress in a Rancher [high availability Kubernetes installation](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) or to switch from the default self-signed certificate to a custom certificate. \ No newline at end of file diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/bootstrap-password.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/bootstrap-password.md new file mode 100644 index 00000000000..abb0d9a93dd --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/bootstrap-password.md @@ -0,0 +1,31 @@ +--- +title: Bootstrap Password +--- + + + + + +When Rancher starts for the first time, a password is randomly generated for the first admin user. When the admin first logs in to Rancher, the UI shows commands that can be used to retrieve the bootstrap password. The admin needs to run those commands and log in with the bootstrap password. Then Rancher gives the admin an opportunity to reset the password. + +The bootstrap password is randomly generated if it is not set during installation with a variable. For details on how to set the bootstrap password using a variable, see below. + +### Specifying the Bootstrap Password in Helm Installs + +For a Helm install, users can specify the bootstrap password variable by configuring it in the Helm chart values with `.Values.bootstrapPassword`. + +The password will be stored in a Kubernetes secret. After Rancher is installed, the UI will show instructions for how to retrieve the password using kubectl: + +``` +kubectl get secret --namespace cattle-system bootstrap-secret -o go-template='{{ .data.bootstrapPassword|base64decode}}{{ "\n" }}' +``` + +### Specifying the Bootstrap Password in Docker Installs + +For a Docker install, you can specify the bootstrap password by passing `-e CATTLE_BOOTSTRAP_PASSWORD=password` to the Docker install command. + +The password will be stored in the Docker container logs. After Rancher is installed, the UI will show instructions for how to retrieve the password using the Docker container ID: + +``` +docker logs container-id 2>&1 | grep "Bootstrap Password:" +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md new file mode 100644 index 00000000000..b2f0e8515e8 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md @@ -0,0 +1,123 @@ +--- +title: Choosing a Rancher Version +--- + + + + + +This section describes how to choose a Rancher version. + +For a high-availability installation of Rancher, which is recommended for production, the Rancher server is installed using a **Helm chart** on a Kubernetes cluster. Refer to the [Helm version requirements](helm-version-requirements.md) to choose a version of Helm to install Rancher. + +For Docker installations of Rancher, which is used for development and testing, you will install Rancher as a **Docker image**. + + + + +When installing, upgrading, or rolling back Rancher Server when it is [installed on a Kubernetes cluster](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md), Rancher server is installed using a Helm chart on a Kubernetes cluster. Therefore, as you prepare to install or upgrade a high availability Rancher configuration, you must add a Helm chart repository that contains the charts for installing Rancher. + +Refer to the [Helm version requirements](helm-version-requirements.md) to choose a version of Helm to install Rancher. + +### Helm Chart Repositories + +Rancher provides several different Helm chart repositories to choose from. We align our latest and stable Helm chart repositories with the Docker tags that are used for a Docker installation. Therefore, the `rancher-latest` repository will contain charts for all the Rancher versions that have been tagged as `rancher/rancher:latest`. When a Rancher version has been promoted to the `rancher/rancher:stable`, it will get added to the `rancher-stable` repository. + +| Type | Command to Add the Repo | Description of the Repo | +| -------------- | ------------ | ----------------- | +| rancher-latest | `helm repo add rancher-latest https://releases.rancher.com/server-charts/latest` | Adds a repository of Helm charts for the latest versions of Rancher. We recommend using this repo for testing out new Rancher builds. | +| rancher-stable | `helm repo add rancher-stable https://releases.rancher.com/server-charts/stable` | Adds a repository of Helm charts for older, stable versions of Rancher. We recommend using this repo for production environments. | +| rancher-alpha | `helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha` | Adds a repository of Helm charts for alpha versions of Rancher for previewing upcoming releases. These releases are discouraged in production environments. Upgrades _to_ or _from_ charts in the rancher-alpha repository to any other chart, regardless or repository, aren't supported. | + +
+Instructions on when to select these repos are available below in [Switching to a Different Helm Chart Repository](#switching-to-a-different-helm-chart-repository). + +:::note + +All charts in the `rancher-stable` repository will correspond with any Rancher version tagged as `stable`. + +::: + +### Helm Chart Versions + +Rancher Helm chart versions match the Rancher version (i.e `appVersion`). Once you've added the repo you can search it to show available versions with the following command:
+    `helm search repo --versions` + +If you have several repos you can specify the repo name, ie. `helm search repo rancher-stable/rancher --versions`
+For more information, see https://helm.sh/docs/helm/helm_search_repo/ + +To fetch a specific version of your chosen repo, define the `--version` parameter like in the following example:
+    `helm fetch rancher-stable/rancher --version=2.4.8` + +### Switching to a Different Helm Chart Repository + +After installing Rancher, if you want to change which Helm chart repository to install Rancher from, you will need to follow these steps. + +:::note + +Because the rancher-alpha repository contains only alpha charts, switching between the rancher-alpha repository and the rancher-stable or rancher-latest repository for upgrades is not supported. + +::: + +- Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + +1. List the current Helm chart repositories. + + ```plain + helm repo list + + NAME URL + stable https://charts.helm.sh/stable + rancher- https://releases.rancher.com/server-charts/ + ``` + +2. Remove the existing Helm Chart repository that contains your charts to install Rancher, which will either be `rancher-stable` or `rancher-latest` depending on what you had initially added. + + ```plain + helm repo remove rancher- + ``` + +3. Add the Helm chart repository that you want to start installing Rancher from. + + ```plain + helm repo add rancher- https://releases.rancher.com/server-charts/ + ``` + +4. Continue to follow the steps to [upgrade Rancher](../install-upgrade-on-a-kubernetes-cluster/upgrades.md) from the new Helm chart repository. + + + + +When performing [Docker installs](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md), upgrades, or rollbacks, you can use _tags_ to install a specific version of Rancher. + +### Server Tags + +Rancher Server is distributed as a Docker image, which have tags attached to them. You can specify this tag when entering the command to deploy Rancher. Remember that if you use a tag without an explicit version (like `latest` or `stable`), you must explicitly pull a new version of that image tag. Otherwise, any image cached on the host will be used. + +| Tag | Description | +| -------------------------- | ------ | +| `rancher/rancher:latest` | Our latest development release. These builds are validated through our CI automation framework. These releases are not recommended for production environments. | +| `rancher/rancher:stable` | Our newest stable release. This tag is recommended for production. | +| `rancher/rancher:` | You can install specific versions of Rancher by using the tag from a previous release. See what's available at DockerHub. | + +:::note + +- The `master` tag or any tag with `-rc` or another suffix is meant for the Rancher testing team to validate. You should not use these tags, as these builds are not officially supported. +- Want to install an alpha review for preview? Install using one of the alpha tags listed on our [announcements page](https://forums.rancher.com/c/announcements) (e.g., `v2.2.0-alpha1`). Caveat: Alpha releases cannot be upgraded to or from any other release. + +::: + + + diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md new file mode 100644 index 00000000000..18d811c02f1 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md @@ -0,0 +1,28 @@ +--- +title: About Custom CA Root Certificates +--- + + + + + +If you're using Rancher in an internal production environment where you aren't exposing apps publicly, use a certificate from a private certificate authority (CA). + +Services that Rancher needs to access are sometimes configured with a certificate from a custom/internal CA root, also known as self signed certificate. If the presented certificate from the service cannot be validated by Rancher, the following error displays: `x509: certificate signed by unknown authority`. + +To validate the certificate, the CA root certificates need to be added to Rancher. As Rancher is written in Go, we can use the environment variable `SSL_CERT_DIR` to point to the directory where the CA root certificates are located in the container. The CA root certificates directory can be mounted using the Docker volume option (`-v host-source-directory:container-destination-directory`) when starting the Rancher container. + +Examples of services that Rancher can access: + +- Catalogs +- Authentication providers +- Accessing hosting/cloud API when using Node Drivers + +## Installing with the custom CA Certificate + +For details on starting a Rancher container with your private CA certificates mounted, refer to the installation docs: + +- [Docker install Custom CA certificate options](../../../reference-guides/single-node-rancher-in-docker/advanced-options.md#custom-ca-certificate) + +- [Kubernetes install options for Additional Trusted CAs](../installation-references/helm-chart-options.md#additional-trusted-cas) + diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/helm-version-requirements.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/helm-version-requirements.md new file mode 100644 index 00000000000..8d98fe60458 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/helm-version-requirements.md @@ -0,0 +1,16 @@ +--- +title: Helm Version Requirements +--- + + + + + +This section contains the requirements for Helm, which is the tool used to install Rancher on a high-availability Kubernetes cluster. + +> The installation instructions have been updated for Helm 3. For migration of installs started with Helm 2, refer to the official [Helm 2 to 3 Migration Docs.](https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/) [This section](/versioned_docs/version-2.0-2.4/pages-for-subheaders/helm2.md) provides a copy of the older high-availability Rancher installation instructions that used Helm 2, and it is intended to be used if upgrading to Helm 3 is not feasible. + +- Helm v3.2.x or higher is required to install or upgrade Rancher v2.5. +- Helm v2.16.0 or higher is required for Kubernetes v1.16. For the default Kubernetes version, refer to the [release notes](https://github.com/rancher/rke/releases) for the version of RKE that you are using. +- Helm v2.15.0 should not be used, because of an issue with converting/comparing numbers. +- Helm v2.12.0 should not be used, because of an issue with `cert-manager`. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/local-system-charts.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/local-system-charts.md new file mode 100644 index 00000000000..e48012d6af4 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/local-system-charts.md @@ -0,0 +1,17 @@ +--- +title: Setting up Local System Charts for Air Gapped Installations +--- + + + + + +The [Charts](https://github.com/rancher/charts) repository contains all the Helm catalog items required for features such as monitoring, logging, alerting and Istio. + +In an air gapped installation of Rancher, you will need to configure Rancher to use a local copy of the system charts. This section describes how to use local system charts using a CLI flag. + +## Using Local System Charts + +A local copy of `system-charts` has been packaged into the `rancher/rancher` container. To be able to use these features in an air gap install, you will need to run the Rancher install command with an extra environment variable, `CATTLE_SYSTEM_CATALOG=bundled`, which tells Rancher to use the local copy of the charts instead of attempting to fetch them from GitHub. + +Example commands for a Rancher installation with a bundled `system-charts` are included in the [air gap installation](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) instructions for Docker and Helm installs. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md new file mode 100644 index 00000000000..0933d7eab43 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/update-rancher-certificate.md @@ -0,0 +1,267 @@ +--- +title: Updating the Rancher Certificate +--- + + + + + +## Updating a Private CA Certificate + +Follow these steps to rotate an SSL certificate and private CA used by Rancher [installed on a Kubernetes cluster](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md), or migrate to an SSL certificate signed by a private CA. + +A summary of the steps is as follows: + +1. Create or update the `tls-rancher-ingress` Kubernetes secret object with the new certificate and private key. +1. Create or update the `tls-ca` Kubernetes secret object with the root CA certificate (only required when using a private CA). +1. Update the Rancher installation using the Helm CLI. +1. Reconfigure the Rancher agents to trust the new CA certificate. +1. Select Force Update of Fleet clusters to connect fleet-agent to Rancher. + +The details of these instructions are below. + +### 1. Create/update the certificate secret object + +First, concatenate the server certificate followed by any intermediate certificate(s) to a file named `tls.crt` and provide the corresponding certificate key in a file named `tls.key`. + +Use the following command to create the `tls-rancher-ingress` secret object in the Rancher (local) management cluster: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +Alternatively, to update an existing `tls-rancher-ingress` secret: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. Create/update the CA certificate secret object + +If the new certificate was signed by a private CA, you will need to copy the corresponding root CA certificate into a file named `cacerts.pem` and create or update the `tls-ca` secret in the `cattle-system` namespace. If the certificate was signed by an intermediate CA, then the `cacerts.pem` must contain both the intermediate and root CA certificates (in this order). + +To create the initial `tls-ca` secret: + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem +``` + +To update an existing `tls-ca` secret: + +```bash +kubectl -n cattle-system create secret generic tls-ca \ + --from-file=cacerts.pem \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 3. Reconfigure the Rancher deployment + +If the certificate source remains the same (for example, `secret`), please follow the steps in Step 3a. + +However, if the certificate source is changing (for example, `letsEncrypt` to `secret`), follow the steps in 3b. + +#### 3a. Redeploy the Rancher pods + +This step is required when the certificate source remains the same, but the CA certificate is being updated. + +In this scenario a redeploy of the Rancher pods is needed, this is because the `tls-ca` secret is read by the Rancher pods when starting. + +The command below can be used to redeploy the Rancher pods: +```bash +kubectl rollout restart deploy/rancher -n cattle-system +``` + +When the change is completed, navigate to `https:///v3/settings/cacerts` to verify that the value matches the CA certificate written in the `tls-ca` secret earlier. The CA `cacerts` value may not update until all of the redeployed Rancher pods start. + +#### 3b. Update the Helm values for Rancher + +This step is required if the certificate source is changing. If Rancher was previously configured to use the default self-signed certificate (`ingress.tls.source=rancher`) or Let's Encrypt (`ingress.tls.source=letsEncrypt`), and is now using a certificate signed by a private CA (`ingress.tls.source=secret`). + +The below steps update the Helm values for the Rancher chart, so the Rancher pods and ingress are reconfigured to use the new private CA certificate created in Step 1 & 2. + + 1. Adjust the values that were used during initial installation, store the current values with: + ```bash + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` + 1. Retrieve the version string of the currently deployed Rancher chart to use below: + ```bash + helm ls -n cattle-system + ``` + 1. Update the current Helm values in the `values.yaml` file to contain: + ```yaml + ingress: + tls: + source: secret + privateCA: true + ``` + :::note Important: + As the certificate is signed by a private CA, it is important to ensure [`privateCA: true`](../installation-references/helm-chart-options.md#common-options) is set in the `values.yaml` file. + ::: + 1. Upgrade the Helm application instance using the `values.yaml` file and the current chart version. The version must match to prevent an upgrade of Rancher. + ```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version + ``` + +When the change is completed, navigate to `https:///v3/settings/cacerts` to verify that the value matches the CA certificate written in the `tls-ca` secret earlier. The CA `cacerts` value may not update until all Rancher pods start. + +### 4. Reconfigure Rancher agents to trust the private CA + +This section covers three methods to reconfigure Rancher agents to trust the private CA. This step is required if either of the following is true: + +- Rancher was previously configured to use the Rancher self-signed certificate (`ingress.tls.source=rancher`) or with a Let's Encrypt issued certificate (`ingress.tls.source=letsEncrypt`) +- The certificate was signed by a different private CA + +#### Why is this step required? + +When Rancher is configured with a certificate signed by a private CA, the CA certificate chain is trusted by Rancher agent containers. Agents compare the checksum of the downloaded certificate against the `CATTLE_CA_CHECKSUM` environment variable. This means that, when the private CA certificate used by Rancher has changed, the environvment variable `CATTLE_CA_CHECKSUM` must be updated accordingly. + +#### Which method should I choose? + +Method 1 is the easiest, but requires all clusters to be connected to Rancher after the certificates have been rotated. This is usually the case if the process is performed right after updating or redeploying the Rancher deployment (Step 3). + +If the clusters have lost connection to Rancher but [Authorized Cluster Endpoint](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) (ACE) is enabled on all clusters, then go with method 2. + +Method 3 can be used as a fallback if method 1 and 2 are not possible. + +#### Method 1: Force a redeploy of the Rancher agents + +For each downstream cluster run the following command using the Kubeconfig file of the Rancher (local) management cluster. + +```bash +kubectl annotate clusters.management.cattle.io io.cattle.agent.force.deploy=true +``` + +:::note +Locate the cluster ID (c-xxxxx) for the downstream cluster, this can be seen in the browser URL bar when viewing the cluster in the Rancher UI, under Cluster Management. +::: + +This command will cause the agent manifest to be reapplied with the checksum of the new certificate. + +#### Method 2: Manually update the checksum environment variable + +Manually patch the agent Kubernetes objects by updating the `CATTLE_CA_CHECKSUM` environment variable to the value matching the checksum of the new CA certificate. Generate the new checksum value like so: + +```bash +curl -k -s -fL /v3/settings/cacerts | jq -r .value | sha256sum | awk '{print $1}' +``` + +Using a Kubeconfig for each downstream cluster update the environment variable for the two agent deployments. If the [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) is enabled for the cluster, [the kubectl context can be adjusted](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to connect directly to the downstream cluster. + +```bash +kubectl edit -n cattle-system ds/cattle-node-agent +kubectl edit -n cattle-system deployment/cattle-cluster-agent +``` + +#### Method 3: Manually redeploy the Rancher agents + +With this method the Rancher agents are reapplied by running a set of commands on a control plane node of each downstream cluster. + +Repeat the below steps for each downstream cluster: + + 1. Retrieve the agent registration kubectl command: + 1. Locate the cluster ID (c-xxxxx) for the downstream cluster, this can be seen in the URL when viewing the cluster in the Rancher UI under Cluster Management + 1. Add the Rancher server URL and cluster ID to the following URL: `https:///v3/clusterregistrationtokens?clusterId=` + 1. Copy the command from the `insecureCommand` field, this command is used because a private CA is un use + + 2. Run the kubectl command from the previous step using a kubeconfig for the downstream cluster with one of the following methods: + 1. If the [ACE](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md) is enabled for the cluster, [the context can be adjusted](../../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to connect directly to the downstream cluster + 1. Alternatively, SSH into the control plane node: + - RKE: Use the [steps in the document here](https://github.com/rancherlabs/support-tools/tree/master/how-to-retrieve-kubeconfig-from-custom-cluster) to generate a kubeconfig + - RKE2/K3s: Use the kubeconfig populated during installation + +### 5. Force Update Fleet clusters to reconnect the fleet-agent to Rancher + +Select 'Force Update' for the clusters within the [Continuous Delivery](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#accessing-fleet-in-the-rancher-ui) view of the Rancher UI to allow the fleet-agent in downstream clusters to successfully connect to Rancher. + +#### Why is this step required? + +Fleet agents in Rancher managed clusters store a kubeconfig that is used to connect to Rancher. The kubeconfig contains a `certificate-authority-data` field containing the CA for the certificate used by Rancher. When changing the CA, this block needs to be updated to allow the fleet-agent to trust the certificate used by Rancher. + +## Updating from a Private CA Certificate to a Public CA Certificate + +Follow these steps to perform the opposite procedure as shown above, to change from a certificate issued by a private CA, to a public or self-signed CA. + +### 1. Create/update the certificate secret object + +First, concatenate the server certificate followed by any intermediate certificate(s) to a file named `tls.crt` and provide the corresponding certificate key in a file named `tls.key`. + +Use the following command to create the `tls-rancher-ingress` secret object in the Rancher (local) management cluster: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key +``` + +Alternatively, to update an existing `tls-rancher-ingress` secret: + +```bash +kubectl -n cattle-system create secret tls tls-rancher-ingress \ + --cert=tls.crt \ + --key=tls.key \ + --dry-run --save-config -o yaml | kubectl apply -f - +``` + +### 2. Delete the CA certificate secret object + +You will delete the `tls-ca` secret in the `cattle-system` namespace as it is no longer needed. You may also optionally save a copy of the `tls-ca` secret if desired. + +To save the existing `tls-ca` secret: + +```bash +kubectl -n cattle-system get secret tls-ca -o yaml > tls-ca.yaml +``` + +To delete the existing `tls-ca` secret: + +```bash +kubectl -n cattle-system delete secret tls-ca +``` + +### 3. Reconfigure the Rancher deployment + +This step is required if the certificate source is changing. In this scenario it's likely only changing because Rancher was previously configured to use the default self-signed certificate (`ingress.tls.source=rancher`). + +The below steps update the Helm values for the Rancher chart, so the Rancher pods and ingress are reconfigured to use the new certificate created in Step 1. + + 1. Adjust the values that were used during initial installation, store the current values with: + ```bash + helm get values rancher -n cattle-system -o yaml > values.yaml + ``` + 1. Also get the version string of the currently deployed Rancher chart: + ```bash + helm ls -n cattle-system + ``` + 1. Update the current Helm values in the `values.yaml` file: + 1. As a private CA is no longer being used, remove the `privateCA: true` field, or set this to `false` + 1. Adjust the `ingress.tls.source` field as necessary. Please [refer to the chart options](../installation-references/helm-chart-options.md#common-options) for more details. Here are some examples: + 1. If using a public CA continue with a value of: `secret` + 1. If using Let's Encrypt update the value to: `letsEncrypt` + 1. Update the Helm values for the Rancher chart using the `values.yaml` file, and the current chart version to prevent an upgrade: + ```bash + helm upgrade rancher rancher-stable/rancher \ + --namespace cattle-system \ + -f values.yaml \ + --version + ``` + +### 4. Reconfigure Rancher agents for the non-private/common certificate + +As a private CA is no longer being used, the `CATTLE_CA_CHECKSUM` environment variable on the downstream cluster agents should be removed or set to "" (an empty string). + +### 5. Force Update Fleet clusters to reconnect the fleet-agent to Rancher + +Select 'Force Update' for the clusters within the [Continuous Delivery](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md#accessing-fleet-in-the-rancher-ui) view of the Rancher UI to allow the fleet-agent in downstream clusters to successfully connect to Rancher. + +#### Why is this step required? + +Fleet agents in Rancher managed clusters store a kubeconfig that is used to connect to Rancher. The kubeconfig contains a `certificate-authority-data` field containing the CA for the certificate used by Rancher. When changing the CA, this block needs to be updated to allow the fleet-agent to trust the certificate used by Rancher. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md new file mode 100644 index 00000000000..9a9ec2b8f83 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md @@ -0,0 +1,285 @@ +--- +title: Upgrading Cert-Manager +--- + + + + + +Rancher uses cert-manager to automatically generate and renew TLS certificates for HA deployments of Rancher. As of Fall 2019, three important changes to cert-manager are set to occur that you need to take action on if you have an HA deployment of Rancher: + +1. [Let's Encrypt will be blocking cert-manager instances older than 0.8.0 starting November 1st 2019.](https://community.letsencrypt.org/t/blocking-old-cert-manager-versions/98753) +1. [Cert-manager is deprecating and replacing the certificate.spec.acme.solvers field](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/). This change has no exact deadline. +1. [Cert-manager is deprecating `v1alpha1` API and replacing its API group](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/) + +To address these changes, this guide will do two things: + +1. Document the procedure for upgrading cert-manager +1. Explain the cert-manager API changes and link to cert-manager's official documentation for migrating your data + +:::note Important: + +If you are upgrading cert-manager to the latest version from a version older than 1.5, follow the steps in [Option C](#option-c-upgrade-cert-manager-from-versions-15-and-below) below to do so. Note that you do not need to reinstall Rancher to perform this upgrade. + +::: + +## Upgrade Cert-Manager + +The namespace used in these instructions depends on the namespace cert-manager is currently installed in. If it is in kube-system use that in the instructions below. You can verify by running `kubectl get pods --all-namespaces` and checking which namespace the cert-manager-\* pods are listed in. Do not change the namespace cert-manager is running in or this can cause issues. + +In order to upgrade cert-manager, follow these instructions: + +### Option A: Upgrade cert-manager with Internet Access + +
+ Click to expand + +1. [Back up existing resources](https://cert-manager.io/docs/tutorials/backup/) as a precaution + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note Important: + + If you are upgrading from a version older than 0.11.0, Update the apiVersion on all your backed up resources from `certmanager.k8s.io/v1alpha1` to `cert-manager.io/v1alpha2`. If you use any cert-manager annotations on any of your other resources, you will need to update them to reflect the new API group. For details, refer to the documentation on [additional annotation changes.](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes) + + ::: + +1. [Uninstall existing deployment](https://cert-manager.io/docs/installation/uninstall/kubernetes/#uninstalling-with-helm) + + ```plain + helm uninstall cert-manager + ``` + + Delete the CustomResourceDefinition using the link to the version vX.Y.Z you installed + + ```plain + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + +1. Install the CustomResourceDefinition resources separately + + ```plain + kubectl apply --validate=false -f https://github.com/cert-manager/cert-manager/releases/download/vX.Y.Z/cert-manager.crds.yaml + + ``` + + :::note + + If you are running Kubernetes v1.15 or below, you will need to add the `--validate=false` flag to your `kubectl apply` command above. Otherwise, you will receive a validation error relating to the `x-kubernetes-preserve-unknown-fields` field in cert-manager’s CustomResourceDefinition resources. This is a benign error and occurs due to the way kubectl performs resource validation. + + ::: + +1. Create the namespace for cert-manager if needed + + ```plain + kubectl create namespace cert-manager + ``` + +1. Add the Jetstack Helm repository + + ```plain + helm repo add jetstack https://charts.jetstack.io + ``` + +1. Update your local Helm chart repository cache + + ```plain + helm repo update + ``` + +1. Install the new version of cert-manager + + ```plain + helm install \ + cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --version v1.11.0 + ``` + +1. [Restore back up resources](https://cert-manager.io/docs/tutorials/backup/#restoring-resources) + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### Option B: Upgrade cert-manager in an Air-Gapped Environment + +
+ Click to expand + +### Prerequisites + +Before you can perform the upgrade, you must prepare your air gapped environment by adding the necessary container images to your private registry and downloading or rendering the required Kubernetes manifest files. + +1. Follow the guide to [Prepare your Private Registry](../other-installation-methods/air-gapped-helm-cli-install/publish-images.md) with the images needed for the upgrade. + +1. From a system connected to the internet, add the cert-manager repo to Helm + + ```plain + helm repo add jetstack https://charts.jetstack.io + helm repo update + ``` + +1. Fetch the latest cert-manager chart available from the [Helm chart repository](https://artifacthub.io/packages/helm/cert-manager/cert-manager). + + ```plain + helm fetch jetstack/cert-manager --version v1.11.0 + ``` + +1. Render the cert manager template with the options you would like to use to install the chart. Remember to set the `image.repository` option to pull the image from your private registry. This will create a `cert-manager` directory with the Kubernetes manifest files. + + The Helm 3 command is as follows: + + ```plain + helm template cert-manager ./cert-manager-v0.12.0.tgz --output-dir . \ + --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + + The Helm 2 command is as follows: + + ```plain + helm template ./cert-manager-v0.12.0.tgz --output-dir . \ + --name cert-manager --namespace cert-manager \ + --set image.repository=/quay.io/jetstack/cert-manager-controller + --set webhook.image.repository=/quay.io/jetstack/cert-manager-webhook + --set cainjector.image.repository=/quay.io/jetstack/cert-manager-cainjector + ``` + +1. Download the required CRD file for cert-manager (old and new) + + ```plain + curl -L -o cert-manager-crd.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-0.12/deploy/manifests/00-crds.yaml + curl -L -o cert-manager/cert-manager-crd-old.yaml https://raw.githubusercontent.com/cert-manager/cert-manager/release-X.Y/deploy/manifests/00-crds.yaml + ``` + +### Install cert-manager + +1. Back up existing resources as a precaution + + ```plain + kubectl get -o yaml --all-namespaces \ + issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml + ``` + + :::note Important: + + If you are upgrading from a version older than 0.11.0, Update the apiVersion on all your backed up resources from `certmanager.k8s.io/v1alpha1` to `cert-manager.io/v1alpha2`. If you use any cert-manager annotations on any of your other resources, you will need to update them to reflect the new API group. For details, refer to the documentation on [additional annotation changes.](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/#additional-annotation-changes) + + ::: + +1. Delete the existing cert-manager installation + + ```plain + kubectl -n cert-manager \ + delete deployment,sa,clusterrole,clusterrolebinding \ + -l 'app=cert-manager' -l 'chart=cert-manager-v0.5.2' + ``` + + Delete the CustomResourceDefinition using the link to the version vX.Y you installed + + ```plain + kubectl delete -f cert-manager/cert-manager-crd-old.yaml + ``` + +1. Install the CustomResourceDefinition resources separately + + ```plain + kubectl apply -f cert-manager/cert-manager-crd.yaml + ``` + + :::note Important: + + If you are running Kubernetes v1.15 or below, you will need to add the `--validate=false` flag to your `kubectl apply` command above. Otherwise, you will receive a validation error relating to the `x-kubernetes-preserve-unknown-fields` field in cert-manager’s CustomResourceDefinition resources. This is a benign error and occurs due to the way kubectl performs resource validation. + + ::: + +1. Create the namespace for cert-manager + + ```plain + kubectl create namespace cert-manager + ``` + +1. Install cert-manager + + ```plain + kubectl -n cert-manager apply -R -f ./cert-manager + ``` + +1. [Restore back up resources](https://cert-manager.io/docs/tutorials/backup/#restoring-resources) + + ```plain + kubectl apply -f cert-manager-backup.yaml + ``` + +
+ +### Option C: Upgrade cert-manager from Versions 1.5 and Below + +
+ Click to expand + +Previously, in order to upgrade cert-manager from an older version, an uninstall and reinstall of Rancher was recommended. Using the method below, you may upgrade cert-manager without those additional steps in order to better preserve your production environment: + +1. Install `cmctl`, the cert-manager CLI tool, using [the installation guide](https://cert-manager.io/docs/usage/cmctl/#installation). + +1. Ensure that any cert-manager custom resources that may have been stored in etcd at a deprecated API version get migrated to v1: + + ``` + cmctl upgrade migrate-api-version + ``` + Refer to the [API version migration docs](https://cert-manager.io/docs/usage/cmctl/#migrate-api-version) for more information. Please also see the [docs to upgrade from 1.5 to 1.6](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/) and the [docs to upgrade from 1.6. to 1.7](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/) if needed. + +1. Upgrade cert-manager to v1.7.1 with a normal `helm upgrade`. You may go directly from version 1.5 to 1.7 if desired. + +1. Follow the Helm tutorial to [update the API version of a release manifest](https://helm.sh/docs/topics/kubernetes_apis/#updating-api-versions-of-a-release-manifest). The chart release name is `release_name=rancher` and the release namespace is `release_namespace=cattle-system`. + +1. In the decoded file, search for `cert-manager.io/v1beta1` and **replace it** with `cert-manager.io/v1`. + +1. Upgrade Rancher normally with `helm upgrade`. + +
+ +### Verify the Deployment + +Once you’ve installed cert-manager, you can verify it is deployed correctly by checking the kube-system namespace for running pods: + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +## Cert-Manager API change and data migration + +--- + +Rancher now supports cert-manager versions 1.6.2 and 1.7.1. We recommend v1.7.x because v 1.6.x will reach end-of-life on March 30, 2022. To read more, see the [cert-manager docs](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-install-cert-manager). For instructions on upgrading cert-manager from version 1.5 to 1.6, see the upstream cert-manager documentation [here](https://cert-manager.io/docs/installation/upgrading/upgrading-1.5-1.6/). For instructions on upgrading cert-manager from version 1.6 to 1.7, see the upstream cert-manager documentation [here](https://cert-manager.io/docs/installation/upgrading/upgrading-1.6-1.7/). + +--- + +Cert-manager has deprecated the use of the `certificate.spec.acme.solvers` field and will drop support for it completely in an upcoming release. + +Per the cert-manager documentation, a new format for configuring ACME certificate resources was introduced in v0.8. Specifically, the challenge solver configuration field was moved. Both the old format and new are supported as of v0.9, but support for the old format will be dropped in an upcoming release of cert-manager. The cert-manager documentation strongly recommends that after upgrading you update your ACME Issuer and Certificate resources to the new format. + +Details about the change and migration instructions can be found in the [cert-manager v0.7 to v0.8 upgrade instructions](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/). + +The v0.11 release marks the removal of the v1alpha1 API that was used in previous versions of cert-manager, as well as our API group changing to be cert-manager.io instead of certmanager.k8s.io. + +We have also removed support for the old configuration format that was deprecated in the v0.8 release. This means you must transition to using the new solvers style configuration format for your ACME issuers before upgrading to v0.11. For more information, see the [upgrading to v0.8 guide](https://cert-manager.io/docs/installation/upgrading/upgrading-0.7-0.8/). + +Details about the change and migration instructions can be found in the [cert-manager v0.10 to v0.11 upgrade instructions](https://cert-manager.io/docs/installation/upgrading/upgrading-0.10-0.11/). + +More info about [cert-manager upgrade information](https://cert-manager.io/docs/installation/upgrading/). + diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md new file mode 100644 index 00000000000..e54524ca7ad --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md @@ -0,0 +1,130 @@ +--- +title: Upgrading and Rolling Back Kubernetes +--- + + + + + +Following an upgrade to the latest version of Rancher, downstream Kubernetes clusters can be upgraded to use the latest supported version of Kubernetes. + +Rancher calls RKE (Rancher Kubernetes Engine) as a library when provisioning and editing RKE clusters. For more information on configuring the upgrade strategy for RKE clusters, refer to the [RKE documentation](https://rancher.com/docs/rke/latest/en/). + + +## Tested Kubernetes Versions + +Before a new version of Rancher is released, it's tested with the latest minor versions of Kubernetes to ensure compatibility. For details on which versions of Kubernetes were tested on each Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.6.0/) + +## How Upgrades Work + +RKE v1.1.0 changed the way that clusters are upgraded. + +In this section of the [RKE documentation,](https://rancher.com/docs/rke/latest/en/upgrades/how-upgrades-work) you'll learn what happens when you edit or upgrade your RKE Kubernetes cluster. + + +## Recommended Best Practice for Upgrades + +When upgrading the Kubernetes version of a cluster, we recommend that you: + +1. Take a snapshot. +1. Initiate a Kubernetes upgrade. +1. If the upgrade fails, revert the cluster to the pre-upgrade Kubernetes version. This is achieved by selecting the **Restore etcd and Kubernetes version** option. This will return your cluster to the pre-upgrade kubernetes version before restoring the etcd snapshot. + +The restore operation will work on a cluster that is not in a healthy or active state. + +## Upgrading the Kubernetes Version + +:::note Prerequisites: + +- The options below are available for [Rancher-launched Kubernetes clusters](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) and [Registered K3s Kubernetes clusters](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#additional-features-for-registered-rke2-and-k3s-clusters). +- The following options also apply to imported RKE2 clusters that you have registered. If you import a cluster from an external cloud platform but don't register it, you won't be able to upgrade the Kubernetes version from Rancher. +- Before upgrading Kubernetes, [back up your cluster.](../../pages-for-subheaders/backup-restore-and-disaster-recovery.md) + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to upgrade and click **⋮ > Edit Config**. +1. From the **Kubernetes Version** drop-down, choose the version of Kubernetes that you want to use for the cluster. +1. Click **Save**. + +**Result:** Kubernetes begins upgrading for the cluster. + +## Rolling Back + +A cluster can be restored to a backup in which the previous Kubernetes version was used. For more information, refer to the following sections: + +- [Backing up a cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md#how-snapshots-work) +- [Restoring a cluster from backup](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md#restoring-a-cluster-from-a-snapshot) + +## Configuring the Upgrade Strategy + +As of RKE v1.1.0, additional upgrade options became available to give you more granular control over the upgrade process. These options can be used to maintain availability of your applications during a cluster upgrade if certain [conditions and requirements](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability) are met. + +The upgrade strategy can be configured in the Rancher UI, or by editing the `cluster.yml`. More advanced options are available by editing the `cluster.yml`. + +### Configuring the Maximum Unavailable Worker Nodes in the Rancher UI + +From the Rancher UI, the maximum number of unavailable worker nodes can be configured. During a cluster upgrade, worker nodes will be upgraded in batches of this size. + +By default, the maximum number of unavailable worker is defined as 10 percent of all worker nodes. This number can be configured as a percentage or as an integer. When defined as a percentage, the batch size is rounded down to the nearest node, with a minimum of one node. + +To change the default number or percentage of worker nodes, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to upgrade and click **⋮ > Edit Config**. +1. In the **Upgrade Strategy** tab, enter the **Worker Concurrency** as a fixed number or percentage. To get this number, you can take the number of nodes in your cluster and subtract the max unavailable nodes. +1. Click **Save**. + +**Result:** The cluster is updated to use the new upgrade strategy. + +### Enabling Draining Nodes During Upgrades from the Rancher UI + +By default, RKE [cordons](https://kubernetes.io/docs/concepts/architecture/nodes/#manual-node-administration) each node before upgrading it. [Draining](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/) is disabled during upgrades by default. If draining is enabled in the cluster configuration, RKE will both cordon and drain the node before it is upgraded. + +To enable draining each node during a cluster upgrade, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to enable node draining and click **⋮ > Edit Config**. +1. Click **⋮ > Edit**. +1. In the **Upgrade Strategy** tab, go to the **Drain nodes** field and click **Yes**. Node draining is configured separately for control plane and worker nodes. +1. Configure the options for how pods are deleted. For more information about each option, refer to [this section.](../../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md#aggressive-and-safe-draining-options) +1. Optionally, configure a grace period. The grace period is the timeout given to each pod for cleaning things up, so they will have chance to exit gracefully. Pods might need to finish any outstanding requests, roll back transactions or save state to some external storage. If this value is negative, the default value specified in the pod will be used. +1. Optionally, configure a timeout, which is the amount of time the drain should continue to wait before giving up. +1. Click **Save**. + +**Result:** The cluster is updated to use the new upgrade strategy. + +:::note + +There is a [known issue](https://github.com/rancher/rancher/issues/25478) in which the Rancher UI doesn't show the state of etcd and controlplane as drained, even though they are being drained. + +::: + +### Maintaining Availability for Applications During Upgrades + +_Available as of RKE v1.1.0_ + +In [this section of the RKE documentation,](https://rancher.com/docs/rke/latest/en/upgrades/maintaining-availability/) you'll learn the requirements to prevent downtime for your applications when upgrading the cluster. + +### Configuring the Upgrade Strategy in the cluster.yml + +More advanced upgrade strategy configuration options are available by editing the `cluster.yml`. + +For details, refer to [Configuring the Upgrade Strategy](https://rancher.com/docs/rke/latest/en/upgrades/configuring-strategy) in the RKE documentation. The section also includes an example `cluster.yml` for configuring the upgrade strategy. + +## Troubleshooting + +If a node doesn't come up after an upgrade, the `rke up` command errors out. + +No upgrade will proceed if the number of unavailable nodes exceeds the configured maximum. + +If an upgrade stops, you may need to fix an unavailable node or remove it from the cluster before the upgrade can continue. + +A failed node could be in many different states: + +- Powered off +- Unavailable +- User drains a node while upgrade is in process, so there are no kubelets on the node +- The upgrade itself failed + +If the max unavailable number of nodes is reached during an upgrade, Rancher user clusters will be stuck in updating state and not move forward with upgrading any other control plane nodes. It will continue to evaluate the set of unavailable nodes in case one of the nodes becomes available. If the node cannot be fixed, you must remove the node in order to continue the upgrade. diff --git a/versioned_docs/version-latest/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md b/versioned_docs/version-latest/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md new file mode 100644 index 00000000000..42c0b6348a2 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md @@ -0,0 +1,95 @@ +--- +title: Upgrading Kubernetes without Upgrading Rancher +--- + + + + + +The RKE metadata feature allows you to provision clusters with new versions of Kubernetes as soon as they are released, without upgrading Rancher. This feature is useful for taking advantage of patch versions of Kubernetes, for example, if you want to upgrade to Kubernetes v1.14.7 when your Rancher server originally supported v1.14.6. + +:::note + +The Kubernetes API can change between minor versions. Therefore, we don't support introducing minor Kubernetes versions, such as introducing v1.15 when Rancher currently supports v1.14. You would need to upgrade Rancher to add support for minor Kubernetes versions. + +::: + +Rancher's Kubernetes metadata contains information specific to the Kubernetes version that Rancher uses to provision [RKE clusters](../../pages-for-subheaders/launch-kubernetes-with-rancher.md). Rancher syncs the data periodically and creates custom resource definitions (CRDs) for **system images,** **service options** and **addon templates**. Consequently, when a new Kubernetes version is compatible with the Rancher server version, the Kubernetes metadata makes the new version available to Rancher for provisioning clusters. The metadata gives you an overview of the information that the [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) uses for deploying various Kubernetes versions. + +This table below describes the CRDs that are affected by the periodic data sync. + +:::note + +Only administrators can edit metadata CRDs. It is recommended not to update existing objects unless explicitly advised. + +::: + +| Resource | Description | Rancher API URL | +|----------|-------------|-----------------| +| System Images | List of system images used to deploy Kubernetes through RKE. | `/v3/rkek8ssystemimages` | +| Service Options | Default options passed to Kubernetes components like `kube-api`, `scheduler`, `kubelet`, `kube-proxy`, and `kube-controller-manager` | `/v3/rkek8sserviceoptions` | +| Addon Templates | YAML definitions used to deploy addon components like Canal, Calico, Flannel, Weave, Kube-dns, CoreDNS, `metrics-server`, `nginx-ingress` | `/v3/rkeaddons` | + +Administrators might configure the RKE metadata settings to do the following: + +- Refresh the Kubernetes metadata, if a new patch version of Kubernetes comes out and they want Rancher to provision clusters with the latest version of Kubernetes without having to upgrade Rancher +- Change the metadata URL that Rancher uses to sync the metadata, which is useful for air gap setups if you need to sync Rancher locally instead of with GitHub +- Prevent Rancher from auto-syncing the metadata, which is one way to prevent new and unsupported Kubernetes versions from being available in Rancher + +### Refresh Kubernetes Metadata + +The option to refresh the Kubernetes metadata is available for administrators by default, or for any user who has the **Manage Cluster Drivers** [global role.](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) + +To force Rancher to refresh the Kubernetes metadata, a manual refresh action is available: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **Drivers**. +1. Click **Refresh Kubernetes Metadata**. + +You can configure Rancher to only refresh metadata when desired by setting `refresh-interval-minutes` to `0` (see below) and using this button to perform the metadata refresh manually when desired. + +### Configuring the Metadata Synchronization + +:::caution + +Only administrators can change these settings. + +::: + +The RKE metadata config controls how often Rancher syncs metadata and where it downloads data from. You can configure the metadata from the settings in the Rancher UI, or through the Rancher API at the endpoint `v3/settings/rke-metadata-config`. + +The way that the metadata is configured depends on the Rancher version. + +To edit the metadata config in Rancher, + +1. In the upper left corner, click **☰ > Global Settings**. +1. Go to the **rke-metadata-config** section. Click **⋮ > Edit Setting**. +1. You can optionally fill in the following parameters: + + - `refresh-interval-minutes`: This is the amount of time that Rancher waits to sync the metadata. To disable the periodic refresh, set `refresh-interval-minutes` to 0. + - `url`: This is the HTTP path that Rancher fetches data from. The path must be a direct path to a JSON file. For example, the default URL for Rancher v2.4 is `https://releases.rancher.com/kontainer-driver-metadata/release-v2.4/data.json`. +1. Click **Save**. + +If you don't have an air gap setup, you don't need to specify the URL where Rancher gets the metadata, because the default setting is to pull from [Rancher's metadata Git repository.](https://github.com/rancher/kontainer-driver-metadata/blob/dev-v2.5/data/data.json) + +However, if you have an [air gap setup,](#air-gap-setups) you will need to mirror the Kubernetes metadata repository in a location available to Rancher. Then you need to change the URL to point to the new location of the JSON file. + +### Air Gap Setups + +Rancher relies on a periodic refresh of the `rke-metadata-config` to download new Kubernetes version metadata if it is supported with the current version of the Rancher server. For a table of compatible Kubernetes and Rancher versions, refer to the [service terms section.](https://rancher.com/support-maintenance-terms/all-supported-versions/rancher-v2.2.8/) + +If you have an air gap setup, you might not be able to get the automatic periodic refresh of the Kubernetes metadata from Rancher's Git repository. In that case, you should disable the periodic refresh to prevent your logs from showing errors. Optionally, you can configure your metadata settings so that Rancher can sync with a local copy of the RKE metadata. + +To sync Rancher with a local mirror of the RKE metadata, an administrator would configure the `rke-metadata-config` settings to point to the mirror. For details, refer to [Configuring the Metadata Synchronization.](#configuring-the-metadata-synchronization) + +After new Kubernetes versions are loaded into the Rancher setup, additional steps would be required in order to use them for launching clusters. Rancher needs access to updated system images. While the metadata settings can only be changed by administrators, any user can download the Rancher system images and prepare a private container image registry for them. + +To download the system images for the private registry: + +1. Click **☰** in the top left corner. +1. Click **About** at the bottom of the left navigation. +1. Download the OS specific image lists for Linux or Windows. +1. Download `rancher-images.txt`. +1. Prepare the private registry using the same steps during the [air gap install](other-installation-methods/air-gapped-helm-cli-install/publish-images.md), but instead of using the `rancher-images.txt` from the releases page, use the one obtained from the previous steps. + +**Result:** The air gap installation of Rancher can now sync the Kubernetes metadata. If you update your private registry when new versions of Kubernetes are released, you can provision clusters with the new version without having to upgrade Rancher. diff --git a/versioned_docs/version-latest/getting-started/overview.md b/versioned_docs/version-latest/getting-started/overview.md new file mode 100644 index 00000000000..b7f35276bc6 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/overview.md @@ -0,0 +1,69 @@ +--- +title: Overview +--- + + + + + +Rancher is a container management platform built for organizations that deploy containers in production. Rancher makes it easy to run Kubernetes everywhere, meet IT requirements, and empower DevOps teams. + +## Run Kubernetes Everywhere + +Kubernetes has become the container orchestration standard. Most cloud and virtualization vendors now offer it as standard infrastructure. Rancher users have the choice of creating Kubernetes clusters with Rancher Kubernetes Engine (RKE) or cloud Kubernetes services, such as GKE, AKS, and EKS. Rancher users can also import and manage their existing Kubernetes clusters created using any Kubernetes distribution or installer. + +## Meet IT Requirements + +Rancher supports centralized authentication, access control, and monitoring for all Kubernetes clusters under its control. For example, you can: + +- Use your Active Directory credentials to access Kubernetes clusters hosted by cloud vendors, such as GKE. +- Setup and enforce access control and security policies across all users, groups, projects, clusters, and clouds. +- View the health and capacity of your Kubernetes clusters from a single-pane-of-glass. + +## Empower DevOps Teams + +Rancher provides an intuitive user interface for DevOps engineers to manage their application workload. The user does not need to have in-depth knowledge of Kubernetes concepts to start using Rancher. Rancher catalog contains a set of useful DevOps tools. Rancher is certified with a wide selection of cloud native ecosystem products, including, for example, security tools, monitoring systems, container registries, and storage and networking drivers. + +The following figure illustrates the role Rancher plays in IT and DevOps organizations. Each team deploys their applications on the public or private clouds they choose. IT administrators gain visibility and enforce policies across all users, clusters, and clouds. + +![Platform](/img/platform.png) + +## Features of the Rancher API Server + +The Rancher API server is built on top of an embedded Kubernetes API server and an etcd database. It implements the following functionalities: + +### Authorization and Role-Based Access Control + +- **User management:** The Rancher API server [manages user identities](../pages-for-subheaders/authentication-config.md) that correspond to external authentication providers like Active Directory or GitHub, in addition to local users. +- **Authorization:** The Rancher API server manages [access control](../pages-for-subheaders/manage-role-based-access-control-rbac.md) and [security](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) policies. + +### Working with Kubernetes + +- **Provisioning Kubernetes clusters:** The Rancher API server can [provision Kubernetes](../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) on existing nodes, or perform [Kubernetes upgrades.](installation-and-upgrade/upgrade-and-roll-back-kubernetes.md) +- **Catalog management:** Rancher provides the ability to use a [catalog of Helm charts](../pages-for-subheaders/helm-charts-in-rancher.md) that make it easy to repeatedly deploy applications. +- **Managing projects:** A project is a group of multiple namespaces and access control policies within a cluster. A project is a Rancher concept, not a Kubernetes concept, which allows you to manage multiple namespaces as a group and perform Kubernetes operations in them. The Rancher UI provides features for [project administration](../pages-for-subheaders/manage-projects.md) and for [managing applications within projects.](../pages-for-subheaders/kubernetes-resources-setup.md) +- **Fleet Continuous Delivery:** Within Rancher, you can leverage [Fleet Continuous Delivery](../pages-for-subheaders/fleet-gitops-at-scale.md) to deploy applications from git repositories, without any manual operation, to targeted downstream Kubernetes clusters. +- **Istio:** Our [integration with Istio](../pages-for-subheaders/istio.md) is designed so that a Rancher operator, such as an administrator or cluster owner, can deliver Istio to developers. Then developers can use Istio to enforce security policies, troubleshoot problems, or manage traffic for green/blue deployments, canary deployments, or A/B testing. + +### Working with Cloud Infrastructure + +- **Tracking nodes:** The Rancher API server tracks identities of all the [nodes](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md) in all clusters. +- **Setting up infrastructure:** When configured to use a cloud provider, Rancher can dynamically provision [new nodes](../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) and [persistent storage](../pages-for-subheaders/create-kubernetes-persistent-storage.md) in the cloud. + +### Cluster Visibility + +- **Logging:** Rancher can integrate with a variety of popular logging services and tools that exist outside of your Kubernetes clusters. +- **Monitoring:** Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with Prometheus, a leading open-source monitoring solution. +- **Alerting:** To keep your clusters and applications healthy and driving your organizational productivity forward, you need to stay informed of events occurring in your clusters and projects, both planned and unplanned. + +## Editing Downstream Clusters with Rancher + +The options and settings available for an existing cluster change based on the method that you used to provision it. For example, only clusters [provisioned by RKE](../pages-for-subheaders/launch-kubernetes-with-rancher.md) have **Cluster Options** available for editing. + +After a cluster is created with Rancher, a cluster administrator can manage cluster membership or manage node pools, among [other options.](../pages-for-subheaders/cluster-configuration.md) + +The following table summarizes the options and settings available for each cluster type: + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md new file mode 100644 index 00000000000..875025ee424 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md @@ -0,0 +1,10 @@ +--- +title: Rancher AWS Marketplace Quick Start +description: Use Amazon EKS to deploy Rancher server. +--- + + + + + +Amazon Elastic Kubernetes Service (EKS) can quickly [deploy Rancher to Amazon Web Services (AWS)](https://documentation.suse.com/trd/kubernetes/single-html/gs_rancher_aws-marketplace/). To learn more, see our [Amazon Marketplace listing](https://aws.amazon.com/marketplace/pp/prodview-go7ent7goo5ae). Watch the [demo](https://youtu.be/9dznJ7Ons0M) for a walkthrough of AWS Marketplace SUSE Rancher setup. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws.md new file mode 100644 index 00000000000..a3fd249d35e --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/aws.md @@ -0,0 +1,99 @@ +--- +title: Rancher AWS Quick Start Guide +description: Read this step by step Rancher AWS guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on AWS in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Amazon AWS will incur charges. + +::: + +- [Amazon AWS Account](https://aws.amazon.com/account/): An Amazon AWS Account is required to create resources for deploying Rancher and Kubernetes. +- [Amazon AWS Access Key](https://docs.aws.amazon.com/general/latest/gr/managing-aws-access-keys.html): Use this link to follow a tutorial to create an Amazon AWS Access Key if you don't have one yet. +- [IAM Policy created](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start): Defines the permissions an account attached with this policy has. +- Install [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Amazon AWS. + +### Example IAM Policy + +The AWS module just creates an EC2 KeyPair, an EC2 SecurityGroup and an EC2 instance. A simple policy would be: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "ec2:*", + "Resource": "*" + } + ] +} +``` + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the AWS folder containing the Terraform files by executing `cd quickstart/rancher/aws`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + + - `aws_access_key` - Amazon AWS Access Key + - `aws_secret_key` - Amazon AWS Secret Key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [AWS Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/aws) for more information. +Suggestions include: + + - `aws_region` - Amazon AWS region, choose the closest instead of the default (`us-east-1`) + - `prefix` - Prefix for all created resources + - `instance_type` - EC2 instance size used, minimum is `t3a.medium` but `t3a.large` or `t3a.xlarge` could be used if within budget + - `add_windows_node` - If true, an additional Windows worker node is added to the workload cluster + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/aws`. + +##### Result + +Two Kubernetes clusters are deployed into your AWS account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +## What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/aws` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/azure.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/azure.md new file mode 100644 index 00000000000..c9b968077ab --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/azure.md @@ -0,0 +1,85 @@ +--- +title: Rancher Azure Quick Start Guide +description: Read this step by step Rancher Azure guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on Azure in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Microsoft Azure will incur charges. + +::: + +- [Microsoft Azure Account](https://azure.microsoft.com/en-us/free/): A Microsoft Azure Account is required to create resources for deploying Rancher and Kubernetes. +- [Microsoft Azure Subscription](https://docs.microsoft.com/en-us/azure/cost-management-billing/manage/create-subscription#create-a-subscription-in-the-azure-portal): Use this link to follow a tutorial to create a Microsoft Azure subscription if you don't have one yet. +- [Micsoroft Azure Tenant](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant): Use this link and follow instructions to create a Microsoft Azure tenant. +- [Microsoft Azure Client ID/Secret](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal): Use this link and follow instructions to create a Microsoft Azure client and secret. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Microsoft Azure. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the Azure folder containing the Terraform files by executing `cd quickstart/rancher/azure`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `azure_subscription_id` - Microsoft Azure Subscription ID + - `azure_client_id` - Microsoft Azure Client ID + - `azure_client_secret` - Microsoft Azure Client Secret + - `azure_tenant_id` - Microsoft Azure Tenant ID + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [Azure Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/azure) for more information. Suggestions include: + - `azure_location` - Microsoft Azure region, choose the closest instead of the default (`East US`) + - `prefix` - Prefix for all created resources + - `instance_type` - Compute instance size used, minimum is `Standard_DS2_v2` but `Standard_DS2_v3` or `Standard_DS3_v2` could be used if within budget + - `add_windows_node` - If true, an additional Windows worker node is added to the workload cluster + - `windows_admin_password` - The admin password of the windows worker node + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/azure`. + +#### Result + +Two Kubernetes clusters are deployed into your Azure account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/azure` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md new file mode 100644 index 00000000000..81442f401b9 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md @@ -0,0 +1,78 @@ +--- +title: Rancher DigitalOcean Quick Start Guide +description: Read this step by step Rancher DigitalOcean guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on DigitalOcean in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to DigitalOcean will incur charges. + +::: + +- [DigitalOcean Account](https://www.digitalocean.com): You will require an account on DigitalOcean as this is where the server and cluster will run. +- [DigitalOcean Access Key](https://www.digitalocean.com/community/tutorials/how-to-create-a-digitalocean-space-and-api-key): Use this link to create a DigitalOcean Access Key if you don't have one. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster to DigitalOcean. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the DigitalOcean folder containing the Terraform files by executing `cd quickstart/rancher/do`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `do_token` - DigitalOcean access key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [DO Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/do) for more information. Suggestions include: + - `do_region` - DigitalOcean region, choose the closest instead of the default (`nyc1`) + - `prefix` - Prefix for all created resources + - `droplet_size` - Droplet size used, minimum is `s-2vcpu-4gb` but `s-4vcpu-8gb` could be used if within budget + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/do`. + +#### Result + +Two Kubernetes clusters are deployed into your DigitalOcean account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/do` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md new file mode 100644 index 00000000000..ab53c046c5e --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md @@ -0,0 +1,110 @@ +--- +title: Rancher Equinix Metal Quick Start +--- + + + + + +## This tutorial walks you through the following: + +- Provisioning an Equinix Metal Server +- Installation of Rancher 2.x +- Creation of your first cluster +- Deployment of an application, Nginx + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. The Docker install is not recommended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Quick Start Outline + +This Quick Start Guide is divided into different tasks for easier consumption. + +
+ +## Prerequisites + +- An [Equinix Metal account](https://metal.equinix.com/developers/docs/accounts/users/) +- An [Equinix Metal project](https://metal.equinix.com/developers/docs/accounts/projects/) + + +### 1. Provision a Equinix Metal Host + +Begin deploying an Equinix Metal Host. Equinix Metal Servers can be provisioned from either the Equinix Metal console, API, or CLI. You can find instructions for each deployment type on the [Equinix Metal deployment documentation](https://metal.equinix.com/developers/docs/deploy/on-demand/). You can find additional documentation on Equinix Metal server types and prices below: + - [Equinix Metal Server Types](https://metal.equinix.com/developers/docs/servers/about/) + - [Equinix Metal Pricing](https://metal.equinix.com/developers/docs/servers/server-specs/) + +:::note Notes: + +- When provisioning a new Equinix Metal Server via the CLI or API you will need to be able to provide the following information: project-id, plan, metro, and the operating-system +- When using a cloud-hosted virtual machine you need to allow inbound TCP communication to ports 80 and 443. Please see your cloud-host's documentation for information regarding port configuration. +- For a full list of port requirements, refer to [Docker Installation](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md). +- Provision the host according to our [Requirements](../../../pages-for-subheaders/installation-requirements.md). + +::: +### 2. Install Rancher + +To install Rancher on your Equinix Metal host, connect to it and then use a shell to install. + +1. Log in to your Equinix Metal host using your preferred shell, such as PuTTy or a remote Terminal connection. + +2. From your shell, enter the following command: + + ``` + sudo docker run -d --restart=unless-stopped -p 80:80 -p 443:443 --privileged rancher/rancher + ``` + +**Result:** Rancher is installed. + +### 3. Log In + +Log in to Rancher to begin using the application. After you log in, you'll make some one-time configurations. + +1. Open a web browser and enter the IP address of your host: `https://`. + + Replace `` with your host IP address. + +2. When prompted, create a password for the default `admin` account there cowpoke! + +3. Set the **Rancher Server URL**. The URL can either be an IP address or a host name. However, each node added to your cluster must be able to connect to this URL.

If you use a hostname in the URL, this hostname must be resolvable by DNS on the nodes you want to add to you cluster. + +
+ +### 4. Create the Cluster + +Welcome to Rancher! You are now able to create your first Kubernetes cluster. + +In this task, you can use the versatile **Custom** option. This option lets you add _any_ Linux host (cloud-hosted VM, on-prem VM, or bare-metal) to be used in a cluster. + +1. Click **☰ > Cluster Management**. +1. From the **Clusters** page, click **Create**. +1. Choose **Custom**. +1. Enter a **Cluster Name**. +1. Click **Next**. +1. From **Node Role**, select _all_ the roles: **etcd**, **Control**, and **Worker**. + - **Optional**: Rancher auto-detects the IP addresses used for Rancher communication and cluster communication. You can override these using `Public Address` and `Internal Address` in the **Node Address** section. +1. Copy the registration command to your clipboard. +1. Log in to your Linux host using your preferred shell, such as PuTTy or a remote Terminal connection. Run the command copied to your clipboard. +1. When you finish running the command on your Linux host, click **Done**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +#### Finished + +Congratulations! You have created your first cluster. + +#### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md new file mode 100644 index 00000000000..22ef8bb7ec9 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/gcp.md @@ -0,0 +1,81 @@ +--- +title: Rancher GCP Quick Start Guide +description: Read this step by step Rancher GCP guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on GCP in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Google GCP will incur charges. + +::: + +- [Google GCP Account](https://console.cloud.google.com/): A Google GCP Account is required to create resources for deploying Rancher and Kubernetes. +- [Google GCP Project](https://cloud.google.com/appengine/docs/standard/nodejs/building-app/creating-project): Use this link to follow a tutorial to create a GCP Project if you don't have one yet. +- [Google GCP Service Account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys): Use this link and follow instructions to create a GCP service account and token file. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Google GCP. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the GCP folder containing the Terraform files by executing `cd quickstart/rancher/gcp`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `gcp_account_json` - GCP service account file path and file name + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [GCP Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/gcp) for more information. +Suggestions include: + - `gcp_region` - Google GCP region, choose the closest instead of the default (`us-east4`) + - `gcp_zone` - Google GCP zone, choose the closest instead of the default (`us-east4-a`) + - `prefix` - Prefix for all created resources + - `machine_type` - Compute instance size used, minimum is `n1-standard-1` but `n1-standard-2` or `n1-standard-4` could be used if within budget + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 16 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/gcp`. + +#### Result + +Two Kubernetes clusters are deployed into your GCP account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/gcp` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md new file mode 100644 index 00000000000..b6489884a74 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md @@ -0,0 +1,158 @@ +--- +title: Helm CLI Quick Start +--- + + + + + +These instructions capture a quick way to set up a proof-of-concept Rancher installation. + +These instructions assume you have a Linux virtual machine that you will communicate with from your local workstation. Rancher will be installed on the Linux machine. You will need to retrieve the IP address of that machine so that you can access Rancher from your local workstation. Rancher is designed to manage Kubernetes clusters remotely, so any Kubernetes cluster that Rancher manages in the future will also need to be able to reach this IP address. + +We don't recommend installing Rancher locally because it creates a networking problem. Installing Rancher on localhost does not allow Rancher to communicate with downstream Kubernetes clusters, so on localhost you wouldn't be able to test Rancher's cluster provisioning or cluster management functionality. + +Your Linux machine can be anywhere. It could be an Amazon EC2 instance, a Digital Ocean droplet, or an Azure virtual machine, to name a few examples. Other Rancher docs often use 'node' as a generic term for all of these. One possible way to deploy a Linux machine is by setting up an Amazon EC2 instance as shown in [this tutorial](../../../how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md). + +The full installation requirements are [here](../../../pages-for-subheaders/installation-requirements.md). + +## Install K3s on Linux + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +Install a K3s cluster by running this command on the Linux machine: + +``` +curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server --cluster-init +``` + +Using `--cluster-init` allows K3s to use embedded etcd as the datastore and has the ability to convert to an HA setup. Refer to [High Availability with Embedded DB](https://rancher.com/docs/k3s/latest/en/installation/ha-embedded/). + +Save the IP of the Linux machine. + +## Save the kubeconfig to your workstation + +The kubeconfig file is important for accessing the Kubernetes cluster. Copy the file at `/etc/rancher/k3s/k3s.yaml` from the Linux machine and save it to your local workstation in the directory `~/.kube/config`. One way to do this is by using the `scp` tool and run this command on your local machine: + + + + +``` +scp root@:/etc/rancher/k3s/k3s.yaml ~/.kube/config +``` + +In some cases it may need to make sure that your shell has the environment variable `KUBECONFIG=~/.kube/config` defined, for instance, it can be exported in your profile or rc files. + + + + +By default, "scp" is not a recognized command, so we need to install a module first. + +In Windows Powershell: + +``` +Find-Module Posh-SSH +Install-Module Posh-SSH + +## Get the remote kubeconfig file +scp root@:/etc/rancher/k3s/k3s.yaml $env:USERPROFILE\.kube\config +``` + + + + +## Edit the Rancher server URL in the kubeconfig + +In the kubeconfig file, you will need to change the value of the `server` field to `:6443`. The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443. This edit is needed so that when you run Helm or kubectl commands from your local workstation, you will be able to communicate with the Kubernetes cluster that Rancher will be installed on. + + + + +One way to open the kubeconfig file for editing is to use Vim: + +``` +vi ~/.kube/config +``` + +Press `i` to put Vim in insert mode. To save your work, press `Esc`. Then press `:wq` and press `Enter`. + + + + + +In Windows Powershell, you can use `notepad.exe` for editing the kubeconfig file: + +``` +notepad.exe $env:USERPROFILE\.kube\config +``` + +Once edited, either press `ctrl+s` or go to `File > Save` to save your work. + + + + +## Install Rancher with Helm + +Then from your local workstation, run the following commands. You will need to have [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) and [helm.](https://helm.sh/docs/intro/install/) installed. + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +``` +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + +kubectl create namespace cattle-system + +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +helm repo add jetstack https://charts.jetstack.io + +helm repo update + +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 + +# Windows Powershell +helm install cert-manager jetstack/cert-manager ` + --namespace cert-manager ` + --create-namespace ` + --version v1.11.0 +``` + +The final command to install Rancher is below. The command requires a domain name that forwards traffic to the Linux machine. For the sake of simplicity in this tutorial, you can use a fake domain name to create your proof-of-concept. An example of a fake domain name would be `.sslip.io`. + +To install a specific Rancher version, use the `--version` flag (e.g., `--version 2.6.6`). Otherwise, the latest Rancher is installed by default. Refer to [Choosing a Rancher Version](../../installation-and-upgrade/resources/choose-a-rancher-version.md). + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +Note the password requires a minimum of 12 characters. + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=.sslip.io \ + --set replicas=1 \ + --set bootstrapPassword= + +# Windows Powershell +helm install rancher rancher-latest/rancher ` + --namespace cattle-system ` + --set hostname=.sslip.io ` + --set replicas=1 ` + --set bootstrapPassword= +``` + +Now if you navigate to `.sslip.io` in a web browser, you should see the Rancher UI. + +To make these instructions simple, we used a fake domain name and self-signed certificates to do this installation. Therefore, you will probably need to add a security exception to your web browser to see the Rancher UI. Note that for production installs, you would need a high-availability setup with a load balancer, a real domain name and real certificates. + +These instructions also left out the full installation requirements and other installation options. If you have any issues with these steps, refer to the full [Helm CLI installation docs.](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + +To launch new Kubernetes clusters with your new Rancher server, you may need to set up cloud credentials in Rancher. For more information, see [Launching Kubernetes clusters with Rancher.](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md new file mode 100644 index 00000000000..73774d54f0b --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md @@ -0,0 +1,80 @@ +--- +title: Rancher Hetzner Cloud Quick Start Guide +description: Read this step by step Rancher Hetzner Cloud guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on Hetzner Cloud in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Hetzner Cloud will incur charges. + +::: + +- [Hetzner Cloud Account](https://www.hetzner.com): You will require an account on Hetzner as this is where the server and cluster will run. +- [Hetzner API Access Key](https://docs.hetzner.cloud/#getting-started): Use these instructions to create a Hetzner Cloud API Key if you don't have one. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster to Hetzner. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the Hetzner folder containing the Terraform files by executing `cd quickstart/rancher/hcloud`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `hcloud_token` - Hetzner API access key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [Hetzner Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/hcloud) for more information. +Suggestions include: + + - `prefix` - Prefix for all created resources + - `instance_type` - Instance type, minimum required is `cx21` + - `hcloud_location` - Hetzner Cloud location, choose the closest instead of the default (`fsn1`) + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 15 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/hcloud`. + +#### Result + +Two Kubernetes clusters are deployed into your Hetzner account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/hcloud` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md new file mode 100644 index 00000000000..587da5e2011 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md @@ -0,0 +1,80 @@ +--- +title: Rancher Outscale Quick Start Guide +description: Read this step by step Rancher Outscale guide to quickly deploy a Rancher server with a single-node downstream Kubernetes cluster attached. +--- + + + + + +The following steps will quickly deploy a Rancher server on Outscale in a single-node K3s Kubernetes cluster, with a single-node downstream Kubernetes cluster attached. + +:::note + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +:::caution + +Deploying to Outscale will incur charges. + +::: + +- [Outscale Account](https://en.outscale.com/): You will require an account on Outscale as this is where the server and cluster will run. +- [Outscale Access Key](https://docs.outscale.com/en/userguide/About-Access-Keys.html): Use these instructions to create an Outscale Access Key if you don't have one. +- [Terraform](https://www.terraform.io/downloads.html): Used to provision the server and cluster in Outscale. + + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the Outscale folder containing the Terraform files by executing `cd quickstart/rancher/outscale`. + +3. Rename the `terraform.tfvars.example` file to `terraform.tfvars`. + +4. Edit `terraform.tfvars` and customize the following variables: + - `access_key_id` - Outscale access key + - `secret_key_id` - Outscale secret key + - `rancher_server_admin_password` - Admin password for created Rancher server (minimum 12 characters) + +5. **Optional:** Modify optional variables within `terraform.tfvars`. +See the [Quickstart Readme](https://github.com/rancher/quickstart) and the [Outscale Quickstart Readme](https://github.com/rancher/quickstart/tree/master/rancher/outscale) for more information. +Suggestions include: + - `region` - Outscale region, choose the closest instead of the default (`eu-west-2`) + - `prefix` - Prefix for all created resources + - `instance_type` - Instance type, minimum required is `tinav3.c2r4p3` + +6. Run `terraform init`. + +7. To initiate the creation of the environment, run `terraform apply --auto-approve`. Then wait for output similar to the following: + + ``` + Apply complete! Resources: 21 added, 0 changed, 0 destroyed. + + Outputs: + + rancher_node_ip = xx.xx.xx.xx + rancher_server_url = https://rancher.xx.xx.xx.xx.sslip.io + workload_node_ip = yy.yy.yy.yy + ``` + +8. Paste the `rancher_server_url` from the output above into the browser. Log in when prompted (default username is `admin`, use the password set in `rancher_server_admin_password`). +9. ssh to the Rancher Server using the `id_rsa` key generated in `quickstart/rancher/outscale`. + +#### Result + +Two Kubernetes clusters are deployed into your Outscale account, one running Rancher Server and the other ready for experimentation deployments. Please note that while this setup is a great way to explore Rancher functionality, a production setup should follow our high availability setup guidelines. SSH keys for the VMs are auto-generated and stored in the module directory. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/outscale` folder, execute `terraform destroy --auto-approve`. + +2. Wait for confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/prime.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/prime.md new file mode 100644 index 00000000000..6177d9e5fbd --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/prime.md @@ -0,0 +1,7 @@ +--- +title: Rancher Prime +--- + +Rancher v2.7 introduces Rancher Prime, an evolution of the Rancher enterprise offering. Rancher Prime is a new edition of the commercial, enterprise offering built on the the same source code. Rancher’s product will therefore continue to be 100% open source with additional value coming in from security assurances, extended lifecycles, access to focused architectures and Kubernetes advisories. Rancher Prime will also offer options to get production support for innovative Rancher projects. With Rancher Prime, installation assets are hosted on a trusted registry owned and managed by Rancher. + +To get started with Rancher Prime, [go to this page](https://www.rancher.com/quick-start) and fill out the form. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md new file mode 100644 index 00000000000..efd3465ce22 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md @@ -0,0 +1,55 @@ +--- +title: Vagrant Quick Start +--- + + + + + +The following steps quickly deploy a Rancher Server with a single node cluster attached. + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](../../../pages-for-subheaders/installation-and-upgrade.md). + +::: + +## Prerequisites + +- [Vagrant](https://www.vagrantup.com): Vagrant is required as this is used to provision the machine based on the Vagrantfile. +- [Virtualbox](https://www.virtualbox.org): The virtual machines that Vagrant provisions need to be provisioned to VirtualBox. +- At least 4GB of free RAM. + +### Note +- Vagrant will require plugins to create VirtualBox VMs. Install them with the following commands: + + `vagrant plugin install vagrant-vboxmanage` + + `vagrant plugin install vagrant-vbguest` + +## Getting Started + +1. Clone [Rancher Quickstart](https://github.com/rancher/quickstart) to a folder using `git clone https://github.com/rancher/quickstart`. + +2. Go into the folder containing the Vagrantfile by executing `cd quickstart/rancher/vagrant`. + +3. **Optional:** Edit `config.yaml` to: + + - Change the number of nodes and the memory allocations, if required. (`node.count`, `node.cpus`, `node.memory`) + - Change the password of the `admin` user for logging into Rancher. (`admin_password`) + +4. To initiate the creation of the environment run, `vagrant up --provider=virtualbox`. + +5. Once provisioning finishes, go to `https://192.168.56.101` in the browser. The default user/password is `admin/adminPassword`. + +**Result:** Rancher Server and your Kubernetes cluster is installed on VirtualBox. + +### What's Next? + +Use Rancher to create a deployment. For more information, see [Creating Deployments](../../../pages-for-subheaders/deploy-rancher-workloads.md). + +## Destroying the Environment + +1. From the `quickstart/rancher/vagrant` folder execute `vagrant destroy -f`. + +2. Wait for the confirmation that all resources have been destroyed. diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-workloads/nodeports.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-workloads/nodeports.md new file mode 100644 index 00000000000..5d04570718f --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-workloads/nodeports.md @@ -0,0 +1,142 @@ +--- +title: Workload with NodePort Quick Start +--- + + + + + +### Prerequisite + +You have a running cluster with at least 1 node. + +### 1. Deploying a Workload + +You're ready to create your first Kubernetes [workload](https://kubernetes.io/docs/concepts/workloads/). A workload is an object that includes pods along with other files and info needed to deploy your application. + +For this workload, you'll be deploying the application Rancher Hello-World. + +1. Click **☰ > Cluster Management**. +1. From the **Clusters** page, go to the cluster where the workload should be deployed and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Enter a **Name** for your workload. +1. From the **Container Image** field, enter `rancher/hello-world`. This field is case-sensitive. +1. Click **Add Port**. +1. From the **Service Type** drop-down, make sure that **NodePort** is selected. + + ![As a dropdown, NodePort (On every node selected)](/img/nodeport-dropdown.png) + +1. From the **Publish the container port** field, enter port `80`. + + ![Publish the container port, 80 entered](/img/container-port-field.png) + +1. Click **Create**. + +**Result:** + +* Your workload is deployed. This process might take a few minutes to complete. +* When your workload completes deployment, it's assigned a state of **Active**. You can view this status from the project's **Workloads** page. + +
+ +### 2. Viewing Your Application + +From the **Workloads** page, click the link underneath your workload. If your deployment succeeded, your application opens. + +### Attention: Cloud-Hosted Sandboxes + +When using a cloud-hosted virtual machine, you may not have access to the port running the container. In this event, you can test Nginx in an ssh session on the local machine using `Execute Shell`. Use the port number after the `:` in the link under your workload if available, which is `31568` in this example. + +```html +gettingstarted@rancher:~$ curl http://localhost:31568 + + + + Rancher + + + + + +

Hello world!

+

My hostname is hello-world-66b4b9d88b-78bhx

+
+

k8s services found 2

+ + INGRESS_D1E1A394F61C108633C4BD37AEDDE757 tcp://10.43.203.31:80
+ + KUBERNETES tcp://10.43.0.1:443
+ +
+
+ + +
+ + +
+ + + +gettingstarted@rancher:~$ + +``` + +### Finished + +Congratulations! You have successfully deployed a workload exposed via a NodePort. + +#### What's Next? + +When you're done using your sandbox, destroy the Rancher Server and your cluster. See one of the following: + +- [Amazon AWS: Destroying the Environment](../deploy-rancher-manager/aws.md#destroying-the-environment) +- [DigitalOcean: Destroying the Environment](../deploy-rancher-manager/digitalocean.md#destroying-the-environment) +- [Vagrant: Destroying the Environment](../deploy-rancher-manager/vagrant.md#destroying-the-environment) diff --git a/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md new file mode 100644 index 00000000000..92e71d701c5 --- /dev/null +++ b/versioned_docs/version-latest/getting-started/quick-start-guides/deploy-workloads/workload-ingress.md @@ -0,0 +1,76 @@ +--- +title: Workload with Ingress Quick Start +--- + + + + + +### Prerequisite + +You have a running cluster with at least 1 node. + +### 1. Deploying a Workload + +You're ready to create your first Kubernetes [workload](https://kubernetes.io/docs/concepts/workloads/). A workload is an object that includes pods along with other files and info needed to deploy your application. + +For this workload, you'll be deploying the application Rancher Hello-World. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Click **Deployment**. +1. Enter a **Name** for your workload. +1. From the **Container Image** field, enter `rancher/hello-world`. This field is case-sensitive. +1. Click **Add Port** and `Cluster IP` for the `Service Type` and enter `80` in the **Private Container Port** field. You may leave the `Name` blank or specify any name that you wish. Adding a port enables access to the application inside and outside of the cluster. For more information, see [Services](../../../pages-for-subheaders/workloads-and-pods.md#services). +1. Click **Create**. + +**Result:** + +* Your workload is deployed. This process might take a few minutes to complete. +* When your workload completes deployment, it's assigned a state of **Active**. You can view this status from the project's **Workloads** page. + +### 2. Expose The Application Via An Ingress + +Now that the application is up and running, it needs to be exposed so that other services can connect. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. + +1. Click **Service Discovery > Ingresses**. + +1. Click **Create.** + +1. When choosing **Namespace**, ensure it is the same as the one used when you created your deployment. Otherwise, your deployment will not be available when you attempt to select **Target Service**, as in Step 8 below. + +1. Enter a **Name**, such as **hello**. + +1. Specify your **Path**, such as `/hello`. + +1. In the **Target Service** field, drop down the list and choose the name that you set for your service. + +1. In the **Port** field, drop down the list and select `80`. + +1. Click **Create** at the bottom right. + +**Result:** The application is assigned a `sslip.io` address and exposed. It may take a minute or two to populate. + + +### View Your Application + +From the **Deployments** page, find the **Endpoints** column for your deployment and click on an endpoint. The endpoints available will depend on how you configured the port you added to your deployment. For endpoints where you do not see a randomly assigned port, append the path you specified when creating the ingress to the IP address. For example, if your endpoint looks like `xxx.xxx.xxx.xxx` or `https://xxx.xxx.xxx.xxx` change it to `xxx.xxx.xxx.xxx/hello` or `https://xxx.xxx.xxx.xxx/hello`. + +Your application will open in a separate window. + +#### Finished + +Congratulations! You have successfully deployed a workload exposed via an ingress. + +#### What's Next? + +When you're done using your sandbox, destroy the Rancher Server and your cluster. See one of the following: + +- [Amazon AWS: Destroying the Environment](../deploy-rancher-manager/aws.md#destroying-the-environment) +- [DigitalOcean: Destroying the Environment](../deploy-rancher-manager/digitalocean.md#destroying-the-environment) +- [Vagrant: Destroying the Environment](../deploy-rancher-manager/vagrant.md#destroying-the-environment) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md new file mode 100644 index 00000000000..204f95c05bd --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md @@ -0,0 +1,44 @@ +--- +title: Configure Alerts for Periodic Scan on a Schedule +--- + + + + + +It is possible to run a ClusterScan on a schedule. + +A scheduled scan can also specify if you should receive alerts when the scan completes. + +Alerts are supported only for a scan that runs on a schedule. + +The CIS Benchmark application supports two types of alerts: + +- Alert on scan completion: This alert is sent out when the scan run finishes. The alert includes details including the ClusterScan's name and the ClusterScanProfile name. +- Alert on scan failure: This alert is sent out if there are some test failures in the scan run or if the scan is in a `Fail` state. + +:::note Prerequisite + +Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) + +While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) + +::: + +To configure alerts for a scan that runs on a schedule, + +1. Please enable alerts on the `rancher-cis-benchmark` application. For more information, see [this page](../../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md). +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. Click **Create**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Choose the option **Run scan on a schedule**. +1. Enter a valid [cron schedule expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) in the field **Schedule**. +1. Check the boxes next to the Alert types under **Alerting**. +1. Optional: Choose a **Retention Count**, which indicates the number of reports maintained for this recurring scan. By default this count is 3. When this retention limit is reached, older reports will get purged. +1. Click **Create**. + +**Result:** The scan runs and reschedules to run according to the cron schedule provided. Alerts are sent out when the scan finishes if routes and receiver are configured under `rancher-monitoring` application. + +A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md new file mode 100644 index 00000000000..8d3b66c7e4e --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md @@ -0,0 +1,13 @@ +--- +title: Create a Custom Benchmark Version for Running a Cluster Scan +--- + + + + + +There could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. + +It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. + +For details, see [this page.](../../../integrations-in-rancher/cis-scans/custom-benchmark.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md new file mode 100644 index 00000000000..ef2b5ae330d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md @@ -0,0 +1,24 @@ +--- +title: Enable Alerting for Rancher CIS Benchmark +--- + + + + + +Alerts can be configured to be sent out for a scan that runs on a schedule. + +:::note Prerequisite: + +Before enabling alerts for `rancher-cis-benchmark`, make sure to install the `rancher-monitoring` application and configure the Receivers and Routes. For more information, see [this section.](../../../reference-guides/monitoring-v2-configuration/receivers.md) + +While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. An example route configuration is [here.](../../../reference-guides/monitoring-v2-configuration/receivers.md#example-route-config-for-cis-scan-alerts) + +::: + +While installing or upgrading the `rancher-cis-benchmark` Helm chart, set the following flag to `true` in the `values.yaml`: + +```yaml +alerts: + enabled: true +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md new file mode 100644 index 00000000000..cd260325f20 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md @@ -0,0 +1,21 @@ +--- +title: Install Rancher CIS Benchmark +--- + + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to install CIS Benchmark and click **Explore**. +1. In the left navigation bar, click **Apps > Charts**. +1. Click **CIS Benchmark** +1. Click **Install**. + +**Result:** The CIS scan application is deployed on the Kubernetes cluster. + +:::note + +If you are running Kubernetes v1.24 or earlier, and have a [Pod Security Policy](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) (PSP) hardened cluster, CIS Benchmark 4.0.0 and later disable PSPs by default. To install CIS Benchmark on a PSP-hardened cluster, set `global.psp.enabled` to `true` in the values before installing the chart. [Pod Security Admission](../../new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md) (PSA) hardened clusters aren't affected. + +::: diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md new file mode 100644 index 00000000000..076fbdf409b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md @@ -0,0 +1,24 @@ +--- +title: Run a Scan Periodically on a Schedule +--- + + + + + +To run a ClusterScan on a schedule, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Choose the option **Run scan on a schedule**. +1. Enter a valid cron schedule expression in the field **Schedule**. +1. Choose a **Retention** count, which indicates the number of reports maintained for this recurring scan. By default this count is 3. When this retention limit is reached, older reports will get purged. +1. Click **Create**. + +**Result:** The scan runs and reschedules to run according to the cron schedule provided. The **Next Scan** value indicates the next time this scan will run again. + +A report is generated with the scan results every time the scan runs. To see the latest results, click the name of the scan that appears. + +You can also see the previous reports by choosing the report from the **Reports** dropdown on the scan detail page. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md new file mode 100644 index 00000000000..2fede69bee6 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md @@ -0,0 +1,26 @@ +--- +title: Run a Scan +--- + + + + + +When a ClusterScan custom resource is created, it launches a new CIS scan on the cluster for the chosen ClusterScanProfile. + +:::note + +There is currently a limitation of running only one CIS scan at a time for a cluster. If you create multiple ClusterScan custom resources, they will be run one after the other by the operator, and until one scan finishes, the rest of the ClusterScan custom resources will be in the "Pending" state. + +::: + +To run a scan, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. Click **Create**. +1. Choose a cluster scan profile. The profile determines which CIS Benchmark version will be used and which tests will be performed. If you choose the Default profile, then the CIS Operator will choose a profile applicable to the type of Kubernetes cluster it is installed on. +1. Click **Create**. + +**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md new file mode 100644 index 00000000000..7492bc03f0b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md @@ -0,0 +1,38 @@ +--- +title: Skip Tests +--- + + + + + +CIS scans can be run using test profiles with user-defined skips. + +To skip tests, you will create a custom CIS scan profile. A profile contains the configuration for the CIS scan, which includes the benchmark versions to use and any specific tests to skip in that benchmark. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Profile**. +1. From here, you can create a profile in multiple ways. To make a new profile, click **Create** and fill out the form in the UI. To make a new profile based on an existing profile, go to the existing profile and click **⋮ Clone**. If you are filling out the form, add the tests to skip using the test IDs, using the relevant CIS Benchmark as a reference. If you are creating the new test profile as YAML, you will add the IDs of the tests to skip in the `skipTests` directive. You will also give the profile a name: + + ```yaml + apiVersion: cis.cattle.io/v1 + kind: ClusterScanProfile + metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" + spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" + ``` +1. Click **Create**. + +**Result:** A new CIS scan profile is created. + +When you [run a scan](./run-a-scan.md) that uses this profile, the defined tests will be skipped during the scan. The skipped tests will be marked in the generated report as `Skip`. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md new file mode 100644 index 00000000000..df23f7abbdc --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md @@ -0,0 +1,13 @@ +--- +title: Uninstall Rancher CIS Benchmark +--- + + + + + +1. From the **Cluster Dashboard,** go to the left navigation bar and click **Apps > Installed Apps**. +1. Go to the `cis-operator-system` namespace and check the boxes next to `rancher-cis-benchmark-crd` and `rancher-cis-benchmark`. +1. Click **Delete** and confirm **Delete**. + +**Result:** The `rancher-cis-benchmark` application is uninstalled. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md new file mode 100644 index 00000000000..57dc1183dee --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md @@ -0,0 +1,16 @@ +--- +title: View Reports +--- + + + + + +To view the generated CIS scan reports, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to run a CIS scan and click **Explore**. +1. Click **CIS Benchmark > Scan**. +1. The **Scans** page will show the generated reports. To see a detailed report, go to a scan report and click the name. + +One can download the report from the Scans list or from the scan detail page. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md new file mode 100644 index 00000000000..98ea1abfb3e --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md @@ -0,0 +1,264 @@ +--- +title: Docker Install with TLS Termination at Layer-7 NGINX Load Balancer +--- + + + + + +For development and testing environments that have a special requirement to terminate TLS/SSL at a load balancer instead of your Rancher Server container, deploy Rancher and configure a load balancer to work with it conjunction. + +A layer-7 load balancer can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. + +This install procedure walks you through deployment of Rancher using a single container, and then provides a sample configuration for a layer-7 NGINX load balancer. + +## Requirements for OS, Docker, Hardware, and Networking + +Make sure that your node fulfills the general [installation requirements.](../../pages-for-subheaders/installation-requirements.md) + +## Installation Outline + + +## 1. Provision Linux Host + +Provision a single Linux host according to our [Requirements](../../pages-for-subheaders/installation-requirements.md) to launch your Rancher Server. + +## 2. Choose an SSL Option and Install Rancher + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +:::note Do you want to.. + +- Complete an Air Gap Installation? +- Record all transactions with the Rancher API? + +See [Advanced Options](#advanced-options) below before continuing. + +::: + +Choose from the following options: + +
+ Option A-Bring Your Own Certificate: Self-Signed + +If you elect to use a self-signed certificate to encrypt communication, you must install the certificate on your load balancer (which you'll do later) and your Rancher container. Run the Docker command to deploy Rancher, pointing it toward your certificate. + +:::note Prerequisites: + +Create a self-signed certificate. + +- The certificate files must be in PEM format. + +::: + +**To Install Rancher Using a Self-Signed Cert:** + +1. While running the Docker command to deploy Rancher, point Docker toward your CA certificate file. + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/your_certificate_directory/cacerts.pem:/etc/rancher/ssl/cacerts.pem \ + rancher/rancher:latest + ``` + +
+
+ Option B-Bring Your Own Certificate: Signed by Recognized CA + +If your cluster is public facing, it's best to use a certificate signed by a recognized CA. + +:::note Prerequisites: + +- The certificate files must be in PEM format. + +::: + +**To Install Rancher Using a Cert Signed by a Recognized CA:** + +If you use a certificate signed by a recognized CA, installing your certificate in the Rancher container isn't necessary. We do have to make sure there is no default CA certificate generated and stored, you can do this by passing the `--no-cacerts` parameter to the container. + +1. Enter the following command. + + ``` + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + rancher/rancher:latest --no-cacerts + ``` + +
+ +## 3. Configure Load Balancer + +When using a load balancer in front of your Rancher container, there's no need for the container to redirect port communication from port 80 or port 443. By passing the header `X-Forwarded-Proto: https` header, this redirect is disabled. + +The load balancer or proxy has to be configured to support the following: + +- **WebSocket** connections +- **SPDY** / **HTTP/2** protocols +- Passing / setting the following headers: + + | Header | Value | Description | + |--------|-------|-------------| + | `Host` | Hostname used to reach Rancher. | To identify the server requested by the client. + | `X-Forwarded-Proto` | `https` | To identify the protocol that a client used to connect to the load balancer or proxy.

**Note:** If this header is present, `rancher/rancher` does not redirect HTTP to HTTPS. + | `X-Forwarded-Port` | Port used to reach Rancher. | To identify the protocol that client used to connect to the load balancer or proxy. + | `X-Forwarded-For` | IP of the client connection. | To identify the originating IP address of a client. +### Example NGINX configuration + +This NGINX configuration is tested on NGINX 1.14. + +:::note + +This NGINX configuration is only an example and may not suit your environment. For complete documentation, see [NGINX Load Balancing - HTTP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/). + +::: + +- Replace `rancher-server` with the IP address or hostname of the node running the Rancher container. +- Replace both occurrences of `FQDN` to the DNS name for Rancher. +- Replace `/certs/fullchain.pem` and `/certs/privkey.pem` to the location of the server certificate and the server certificate key respectively. + +``` +worker_processes 4; +worker_rlimit_nofile 40000; + +events { + worker_connections 8192; +} + +http { + upstream rancher { + server rancher-server:80; + } + + map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; + } + + server { + listen 443 ssl http2; + server_name FQDN; + ssl_certificate /certs/fullchain.pem; + ssl_certificate_key /certs/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # This allows the ability for the execute shell window to remain open for up to 15 minutes. Without this parameter, the default is 1 minute and will automatically close. + proxy_read_timeout 900s; + proxy_buffering off; + } + } + + server { + listen 80; + server_name FQDN; + return 301 https://$server_name$request_uri; + } +} +``` + +
+ +## What's Next? + +- **Recommended:** Review Single Node [Backup](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) and [Restore](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md). Although you don't have any data you need to back up right now, we recommend creating backups after regular Rancher use. +- Create a Kubernetes cluster: [Provisioning Kubernetes Clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +
+ +## FAQ and Troubleshooting + +For help troubleshooting certificates, see [this section.](../../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +## Advanced Options + +### API Auditing + +If you want to record all transactions with the Rancher API, enable the [API Auditing](enable-api-audit-log.md) feature by adding the flags below into your install command. + + -e AUDIT_LEVEL=1 \ + -e AUDIT_LOG_PATH=/var/log/auditlog/rancher-api-audit.log \ + -e AUDIT_LOG_MAXAGE=20 \ + -e AUDIT_LOG_MAXBACKUP=20 \ + -e AUDIT_LOG_MAXSIZE=100 \ + +### Air Gap + +If you are visiting this page to complete an [Air Gap Installation](../../pages-for-subheaders/air-gapped-helm-cli-install.md), you must pre-pend your private registry URL to the server tag when running the installation command in the option that you choose. Add `` with your private registry URL in front of `rancher/rancher:latest`. + +**Example:** + + /rancher/rancher:latest + +### Persistent Data + +Rancher uses etcd as a datastore. When Rancher is installed with Docker, the embedded etcd is being used. The persistent data is at the following path in the container: `/var/lib/rancher`. + +You can bind mount a host volume to this location to preserve data on the host it is running on: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +This operation requires [privileged access](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher). + +This layer 7 NGINX configuration is tested on NGINX version 1.13 (mainline) and 1.14 (stable). + +:::note + +This NGINX configuration is only an example and may not suit your environment. For complete documentation, see [NGINX Load Balancing - TCP and UDP Load Balancer](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/). + +::: + +``` +upstream rancher { + server rancher-server:80; +} + +map $http_upgrade $connection_upgrade { + default Upgrade; + '' close; +} + +server { + listen 443 ssl http2; + server_name rancher.yourdomain.com; + ssl_certificate /etc/your_certificate_directory/fullchain.pem; + ssl_certificate_key /etc/your_certificate_directory/privkey.pem; + + location / { + proxy_set_header Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://rancher; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + # This allows the ability for the execute shell window to remain open for up to 15 minutes. Without this parameter, the default is 1 minute and will automatically close. + proxy_read_timeout 900s; + proxy_buffering off; + } +} + +server { + listen 80; + server_name rancher.yourdomain.com; + return 301 https://$server_name$request_uri; +} +``` + +
+ diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters.md new file mode 100644 index 00000000000..ab18d0a7b39 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters.md @@ -0,0 +1,142 @@ +--- +title: Enabling the API Audit Log in Downstream Clusters +--- + + + + + +Kubernetes auditing provides a security-relevant chronological set of records about a cluster. Kube-apiserver performs auditing. Requests generate an event at each stage of its execution, which is then preprocessed according to a certain policy and written to a backend. The policy determines what’s recorded and the backend persists the records. + +You might want to configure the audit log as part of compliance with the Center for Internet Security (CIS) Kubernetes Benchmark controls. + +For configuration details, refer to the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + + + + + +:::note + +This feature is available in Rancher v2.7.2 and above. + +::: + +As a prerequisite, you need to create a secret or configmap which will be the source of the audit policy. + +The secret or configmap must meet the following two requirements: + +1. It must be in the `fleet-default` namespace where the Cluster object exists. +2. It must have the annotation `rke.cattle.io/object-authorized-for-clusters: cluster-name1,cluster-name2` which permits the target clusters to use it. + +:::tip + +Rancher Dashboard provides an easy-to-use form for creating the secret or configmap. + +::: + +Example: + +```yaml +apiVersion: v1 +data: + audit-policy: >- + IyBMb2cgYWxsIHJlcXVlc3RzIGF0IHRoZSBNZXRhZGF0YSBsZXZlbC4KYXBpVmVyc2lvbjogYXVkaXQuazhzLmlvL3YxCmtpbmQ6IFBvbGljeQpydWxlczoKLSBsZXZlbDogTWV0YWRhdGE= +kind: Secret +metadata: + annotations: + rke.cattle.io/object-authorized-for-clusters: cluster1 + name: name1 + namespace: fleet-default +``` + +The audit log can be enabled and configured by editing the cluster in YAML and utilizing the `machineSelectorFiles` and `machineGlobalConfig` directives. + +Example: + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - audit-policy-file=/dev-audit-policy.yaml + - audit-log-path=/dev-audit.logs + machineSelectorFiles: + - fileSources: + - configMap: + name: '' + secret: + items: + - key: audit-policy + path: /dev-audit-policy.yaml + name: dev-audit-policy + machineLabelSelector: + matchLabels: + rke.cattle.io/control-plane-role: 'true' +``` + +For more information about cluster configuration, refer to the REK2 or K3s cluster configuration reference pages. + + + + + +The audit log can be enabled and configured by editing the cluster with YAML. + +When the audit log is enabled, RKE1 default values will be applied. + +```yaml +# +# Rancher Config +# +rancher_kubernetes_engine_config: + services: + kube-api: + audit_log: + enabled: true +``` + +You can customize the audit log by using the configuration directive. + +```yaml +# +# Rancher Config +# +rancher_kubernetes_engine_config: + services: + kube-api: + audit_log: + enabled: true + configuration: + max_age: 6 + max_backup: 6 + max_size: 110 + path: /var/log/kube-audit/audit-log.json + format: json + policy: + apiVersion: audit.k8s.io/v1 # This is required. + kind: Policy + omitStages: + - "RequestReceived" + rules: + # Log pod changes at RequestResponse level + - level: RequestResponse + resources: + - group: "" + # Resource "pods" doesn't match requests to any subresource of pods, + # which is consistent with the RBAC policy. + resources: ["pods"] + # Log "pods/log", "pods/status" at Metadata level + - level: Metadata + resources: + - group: "" + resources: ["pods/log", "pods/status"] +``` + +For configuration details, refer to the official [RKE1 documentation](https://rke.docs.rancher.com/config-options/audit-log). + + + + diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log.md new file mode 100644 index 00000000000..0697f327280 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-api-audit-log.md @@ -0,0 +1,562 @@ +--- +title: Enabling the API Audit Log to Record System Events +--- + + + + + +You can enable the API audit log to record the sequence of system events initiated by individual users. You can know what happened, when it happened, who initiated it, and what cluster it affected. When you enable this feature, all requests to the Rancher API and all responses from it are written to a log. + +You can enable API Auditing during Rancher installation or upgrade. + +## Enabling API Audit Log + +The Audit Log is enabled and configured by passing environment variables to the Rancher server container. See the following to enable on your installation. + +- [Docker Install](../../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) + +- [Kubernetes Install](../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-audit-log) + +## API Audit Log Options + +The usage below defines rules about what the audit log should record and what data it should include: + +| Parameter | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `AUDIT_LEVEL` | `0` - Disable audit log (default setting).
`1` - Log event metadata.
`2` - Log event metadata and request body.
`3` - Log event metadata, request body, and response body. Each log transaction for a request/response pair uses the same `auditID` value.

See [Audit Level Logging](#audit-log-levels) for a table that displays what each setting logs. | +| `AUDIT_LOG_PATH` | Log path for Rancher Server API. Default path is `/var/log/auditlog/rancher-api-audit.log`. You can mount the log directory to host.

Usage Example: `AUDIT_LOG_PATH=/my/custom/path/`
| +| `AUDIT_LOG_MAXAGE` | Defined the maximum number of days to retain old audit log files. Default is 10 days. | +| `AUDIT_LOG_MAXBACKUP` | Defines the maximum number of audit log files to retain. Default is 10. | +| `AUDIT_LOG_MAXSIZE` | Defines the maximum size in megabytes of the audit log file before it gets rotated. Default size is 100M. | + +
+ +### Audit Log Levels + +The following table displays what parts of API transactions are logged for each [`AUDIT_LEVEL`](#audit-level) setting. + +| `AUDIT_LEVEL` Setting | Request Metadata | Request Body | Response Metadata | Response Body | +| --------------------- | ---------------- | ------------ | ----------------- | ------------- | +| `0` | | | | | +| `1` | ✓ | | | | +| `2` | ✓ | ✓ | | | +| `3` | ✓ | ✓ | ✓ | ✓ | + +## Viewing API Audit Logs + +### Docker Install + +Share the `AUDIT_LOG_PATH` directory (Default: `/var/log/auditlog`) with the host system. The log can be parsed by standard CLI tools or forwarded on to a log collection tool like Fluentd, Filebeat, Logstash, etc. + +### Kubernetes Install + +Enabling the API Audit Log with the Helm chart install will create a `rancher-audit-log` sidecar container in the Rancher pod. This container will stream the log to standard output (stdout). You can view the log as you would any container log. + +The `rancher-audit-log` container is part of the `rancher` pod in the `cattle-system` namespace. + +#### CLI + +```bash +kubectl -n cattle-system logs -f rancher-84d886bdbb-s4s69 rancher-audit-log +``` + +#### Shipping the Audit Log + +You can enable Rancher's built in log collection and shipping for the cluster to ship the audit and other services logs to a supported collection endpoint. See [Rancher Tools - Logging](../../pages-for-subheaders/logging.md) for details. + +## Audit Log Samples + +After you enable auditing, each API request or response is logged by Rancher in the form of JSON. Each of the following code samples provide examples of how to identify each API transaction. + +### Metadata Level + +If you set your `AUDIT_LEVEL` to `1`, Rancher logs the metadata header for every API request, but not the body. The header provides basic information about the API transaction, such as the transaction's ID, who initiated the transaction, the time it occurred, etc. + +```json +{ + "auditID": "30022177-9e2e-43d1-b0d0-06ef9d3db183", + "requestURI": "/v3/schemas", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "GET", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:22:43 +0800" +} +``` + +### Metadata and Request Body Level + +If you set your `AUDIT_LEVEL` to `2`, Rancher logs the metadata header and body for every API request. + +The code sample below depicts an API request, with both its metadata header and body. + +```json +{ + "auditID": "ef1d249e-bfac-4fd0-a61f-cbdcad53b9bb", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:28:08 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my description", + "volumes": [] + } +} +``` + +### Metadata, Request Body, and Response Body Level + +If you set your `AUDIT_LEVEL` to `3`, Rancher logs: + +- The metadata header and body for every API request. +- The metadata header and body for every API response. + +#### Request + +The code sample below depicts an API request, with both its metadata header and body. + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "requestURI": "/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "sourceIPs": ["::1"], + "user": { + "name": "user-f4tt2", + "group": ["system:authenticated"] + }, + "verb": "PUT", + "stage": "RequestReceived", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "requestBody": { + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "paused": false, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements", + "requests": {}, + "limits": {} + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container", + "environmentFrom": [], + "capAdd": [], + "capDrop": [], + "livenessProbe": null, + "volumeMounts": [] + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "name": "nginx", + "namespaceId": "default", + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport", + "type": "publicEndpoint" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "scheduling": { + "node": {} + }, + "description": "my decript", + "volumes": [] + } +} +``` + +#### Response + +The code sample below depicts an API response, with both its metadata header and body. + +```json +{ + "auditID": "a886fd9f-5d6b-4ae3-9a10-5bff8f3d68af", + "responseStatus": "200", + "stage": "ResponseComplete", + "stageTimestamp": "2018-07-20 10:33:06 +0800", + "responseBody": { + "actionLinks": { + "pause": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=pause", + "resume": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=resume", + "rollback": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx?action=rollback" + }, + "annotations": {}, + "baseType": "workload", + "containers": [ + { + "allowPrivilegeEscalation": false, + "image": "nginx", + "imagePullPolicy": "Always", + "initContainer": false, + "name": "nginx", + "ports": [ + { + "containerPort": 80, + "dnsName": "nginx-nodeport", + "kind": "NodePort", + "name": "80tcp01", + "protocol": "TCP", + "sourcePort": 0, + "type": "/v3/project/schemas/containerPort" + } + ], + "privileged": false, + "readOnly": false, + "resources": { + "type": "/v3/project/schemas/resourceRequirements" + }, + "restartCount": 0, + "runAsNonRoot": false, + "stdin": true, + "stdinOnce": false, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "tty": true, + "type": "/v3/project/schemas/container" + } + ], + "created": "2018-07-18T07:34:16Z", + "createdTS": 1531899256000, + "creatorId": null, + "deploymentConfig": { + "maxSurge": 1, + "maxUnavailable": 0, + "minReadySeconds": 0, + "progressDeadlineSeconds": 600, + "revisionHistoryLimit": 10, + "strategy": "RollingUpdate" + }, + "deploymentStatus": { + "availableReplicas": 1, + "conditions": [ + { + "lastTransitionTime": "2018-07-18T07:34:38Z", + "lastTransitionTimeTS": 1531899278000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "Deployment has minimum availability.", + "reason": "MinimumReplicasAvailable", + "status": "True", + "type": "Available" + }, + { + "lastTransitionTime": "2018-07-18T07:34:16Z", + "lastTransitionTimeTS": 1531899256000, + "lastUpdateTime": "2018-07-18T07:34:38Z", + "lastUpdateTimeTS": 1531899278000, + "message": "ReplicaSet \"nginx-64d85666f9\" has successfully progressed.", + "reason": "NewReplicaSetAvailable", + "status": "True", + "type": "Progressing" + } + ], + "observedGeneration": 2, + "readyReplicas": 1, + "replicas": 1, + "type": "/v3/project/schemas/deploymentStatus", + "unavailableReplicas": 0, + "updatedReplicas": 1 + }, + "dnsPolicy": "ClusterFirst", + "hostIPC": false, + "hostNetwork": false, + "hostPID": false, + "id": "deployment:default:nginx", + "labels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "links": { + "remove": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "revisions": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/revisions", + "self": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "update": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx", + "yaml": "https://localhost:8443/v3/project/c-bcz5t:p-fdr4s/workloads/deployment:default:nginx/yaml" + }, + "name": "nginx", + "namespaceId": "default", + "paused": false, + "projectId": "c-bcz5t:p-fdr4s", + "publicEndpoints": [ + { + "addresses": ["10.64.3.58"], + "allNodes": true, + "ingressId": null, + "nodeId": null, + "podId": null, + "port": 30917, + "protocol": "TCP", + "serviceId": "default:nginx-nodeport" + } + ], + "restartPolicy": "Always", + "scale": 1, + "schedulerName": "default-scheduler", + "selector": { + "matchLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + }, + "type": "/v3/project/schemas/labelSelector" + }, + "state": "active", + "terminationGracePeriodSeconds": 30, + "transitioning": "no", + "transitioningMessage": "", + "type": "deployment", + "uuid": "f998037d-8a5c-11e8-a4cf-0245a7ebb0fd", + "workloadAnnotations": { + "deployment.kubernetes.io/revision": "1", + "field.cattle.io/creatorId": "user-f4tt2" + }, + "workloadLabels": { + "workload.user.cattle.io/workloadselector": "deployment-default-nginx" + } + } +} +``` diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md new file mode 100644 index 00000000000..ccc1c4f4bff --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery.md @@ -0,0 +1,17 @@ +--- +title: Continuous Delivery +--- + + + + + +[Fleet](../../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) comes preinstalled in Rancher can't be fully disabled. However, the Fleet feature for GitOps continuous delivery may be disabled using the `continuous-delivery` feature flag. + +To enable or disable this feature, refer to the instructions on [the main page about enabling experimental features.](../../../pages-for-subheaders/enable-experimental-features.md) + +Environment Variable Key | Default Value | Description +---|---|--- + `continuous-delivery` | `true` | This flag disables the GitOps continuous delivery feature of Fleet. | + +If Fleet was disabled in Rancher v2.5.x, it will become enabled if Rancher is upgraded to v2.6.x. Only the continuous delivery part of Fleet can be disabled. When `continuous-delivery` is disabled, the `gitjob` deployment is no longer deployed into the Rancher server's local cluster, and `continuous-delivery` is not shown in the Rancher UI. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md new file mode 100644 index 00000000000..d34b03f676d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features.md @@ -0,0 +1,36 @@ +--- +title: UI for Istio Virtual Services and Destination Rules +--- + + + + + +This feature enables a UI that lets you create, read, update and delete virtual services and destination rules, which are traffic management features of Istio. + +> **Prerequisite:** Turning on this feature does not enable Istio. A cluster administrator needs to [enable Istio for the cluster](../../../pages-for-subheaders/istio-setup-guide.md) in order to use the feature. + +To enable or disable this feature, refer to the instructions on [the main page about enabling experimental features.](../../../pages-for-subheaders/enable-experimental-features.md) + +Environment Variable Key | Default Value | Status | Available as of +---|---|---|--- +`istio-virtual-service-ui` |`false` | Experimental | v2.3.0 +`istio-virtual-service-ui` | `true` | GA | v2.3.2 + +## About this Feature + +A central advantage of Istio's traffic management features is that they allow dynamic request routing, which is useful for canary deployments, blue/green deployments, or A/B testing. + +When enabled, this feature turns on a page that lets you configure some traffic management features of Istio using the Rancher UI. Without this feature, you need to use `kubectl` to manage traffic with Istio. + +The feature enables two UI tabs: one tab for **Virtual Services** and another for **Destination Rules**. + +- **Virtual services** intercept and direct traffic to your Kubernetes services, allowing you to direct percentages of traffic from a request to different services. You can use them to define a set of routing rules to apply when a host is addressed. For details, refer to the [Istio documentation.](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/) +- **Destination rules** serve as the single source of truth about which service versions are available to receive traffic from virtual services. You can use these resources to define policies that apply to traffic that is intended for a service after routing has occurred. For details, refer to the [Istio documentation.](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule) + +To see these tabs, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where Istio is installed and click **Explore**. +1. In the left navigation bar, click **Istio**. +1. You will see tabs for **Kiali** and **Jaeger**. From the left navigation bar, you can view and configure **Virtual Services** and **Destination Rules**. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md new file mode 100644 index 00000000000..5c86ee60a66 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md @@ -0,0 +1,49 @@ +--- +title: "Running on ARM64 (Experimental)" +--- + + + + + +:::caution + +Running on an ARM64 platform is currently an experimental feature and is not yet officially supported in Rancher. Therefore, we do not recommend using ARM64 based nodes in a production environment. + +::: + +The following options are available when using an ARM64 platform: + +- Running Rancher on ARM64 based node(s) + - Only for Docker Install. Please note that the following installation command replaces the examples found in the [Docker Install link](../../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md): + + ``` + # In the last line `rancher/rancher:vX.Y.Z`, be certain to replace "X.Y.Z" with a released version in which ARM64 builds exist. For example, if your matching version is v2.5.8, you would fill in this line with `rancher/rancher:v2.5.8`. + docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:vX.Y.Z + ``` + +:::note + +To check if your specific released version is compatible with the ARM64 architecture, you may navigate to your +version's release notes in the following two ways: + +- Manually find your version using https://github.com/rancher/rancher/releases. +- Go directly to your version using the tag and the specific version number. If you plan to use v2.5.8, for example, you may navigate to https://github.com/rancher/rancher/releases/tag/v2.5.8. + +::: + +- Create custom cluster and adding ARM64 based node(s) + - Kubernetes cluster version must be 1.12 or higher + - CNI Network Provider must be [Flannel](../../../faq/container-network-interface-providers.md#flannel) +- Importing clusters that contain ARM64 based nodes + - Kubernetes cluster version must be 1.12 or higher + +Please see [Cluster Options](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) how to configure the cluster options. + +The following features are not tested: + +- Monitoring, alerts, notifiers, pipelines and logging +- Launching apps from the catalog diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md new file mode 100644 index 00000000000..284c3f23bd9 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md @@ -0,0 +1,43 @@ +--- +title: Allow Unsupported Storage Drivers +--- + + + + + +This feature allows you to use types for storage providers and provisioners that are not enabled by default. + +To enable or disable this feature, refer to the instructions on [the main page about enabling experimental features.](../../../pages-for-subheaders/enable-experimental-features.md) + +Environment Variable Key | Default Value | Description +---|---|--- + `unsupported-storage-drivers` | `false` | This feature enables types for storage providers and provisioners that are not enabled by default. + +### Types for Persistent Volume Plugins that are Enabled by Default +Below is a list of storage types for persistent volume plugins that are enabled by default. When enabling this feature flag, any persistent volume plugins that are not on this list are considered experimental and unsupported: + +Name | Plugin +--------|---------- +Amazon EBS Disk | `aws-ebs` +AzureFile | `azure-file` +AzureDisk | `azure-disk` +Google Persistent Disk | `gce-pd` +Longhorn | `flex-volume-longhorn` +VMware vSphere Volume | `vsphere-volume` +Local | `local` +Network File System | `nfs` +hostPath | `host-path` + +### Types for StorageClass that are Enabled by Default +Below is a list of storage types for a StorageClass that are enabled by default. When enabling this feature flag, any persistent volume plugins that are not on this list are considered experimental and unsupported: + +Name | Plugin +--------|-------- +Amazon EBS Disk | `aws-ebs` +AzureFile | `azure-file` +AzureDisk | `azure-disk` +Google Persistent Disk | `gce-pd` +Longhorn | `flex-volume-longhorn` +VMware vSphere Volume | `vsphere-volume` +Local | `local` \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md new file mode 100644 index 00000000000..09624abc1bc --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md @@ -0,0 +1,33 @@ +--- +title: 1. Enable Istio in the Cluster +--- + + + + + +:::note Prerequisites: + +- Only a user with the `cluster-admin` [Kubernetes default role](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) assigned can configure and install Istio in a Kubernetes cluster. +- If you have pod security policies, you will need to install Istio with the CNI enabled. For details, see [this section.](../../../integrations-in-rancher/istio/configuration-options/pod-security-policies.md) +- To install Istio on an RKE2 cluster, additional steps are required. For details, see [this section.](../../../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md) +- To install Istio in a cluster where project network isolation is enabled, additional steps are required. For details, see [this section.](../../../integrations-in-rancher/istio/configuration-options/project-network-isolation.md) + +::: + +1. Click **☰ > Cluster Management**. +1. Go to the where you want to enable Istio and click **Explore**. +1. Click **Apps**. +1. Click **Charts**. +1. Click **Istio**. +1. If you have not already installed your own monitoring app, you will be prompted to install the rancher-monitoring app. Optional: Set your Selector or Scrape config options on rancher-monitoring app install. +1. Optional: Configure member access and [resource limits](../../../integrations-in-rancher/istio/cpu-and-memory-allocations.md) for the Istio components. Ensure you have enough resources on your worker nodes to enable Istio. +1. Optional: Make additional configuration changes to values.yaml if needed. +1. Optional: Add additional resources or configuration via the [overlay file.](../../../pages-for-subheaders/configuration-options.md#overlay-file) +1. Click **Install**. + +**Result:** Istio is installed at the cluster level. + +## Additional Config Options + +For more information on configuring Istio, refer to the [configuration reference.](../../../pages-for-subheaders/configuration-options.md) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md new file mode 100644 index 00000000000..13100f50109 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md @@ -0,0 +1,56 @@ +--- +title: 2. Enable Istio in a Namespace +--- + + + + + +You will need to manually enable Istio in each namespace that you want to be tracked or controlled by Istio. When Istio is enabled in a namespace, the Envoy sidecar proxy will be automatically injected into all new workloads that are deployed in the namespace. + +This namespace setting will only affect new workloads in the namespace. Any preexisting workloads will need to be re-deployed to leverage the sidecar auto injection. + +:::note Prerequisite: + +To enable Istio in a namespace, the cluster must have Istio installed. + +::: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the namespace where you want to enable Istio and click **⋮ > Enable Istio Auto Injection**. Alternately, click the namespace, and then on the namespace detail page, click **⋮ > Enable Istio Auto Injection**. + +**Result:** The namespace now has the label `istio-injection=enabled`. All new workloads deployed in this namespace will have the Istio sidecar injected by default. + +### Verifying that Automatic Istio Sidecar Injection is Enabled + +To verify that Istio is enabled, deploy a hello-world workload in the namespace. Go to the workload and click the pod name. In the **Containers** section, you should see the `istio-proxy` container. + +### Excluding Workloads from Being Injected with the Istio Sidecar + +If you need to exclude a workload from getting injected with the Istio sidecar, use the following annotation on the workload: + +``` +sidecar.istio.io/inject: “false†+``` + +To add the annotation to a workload, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Workload**. +1. Go to the workload that should not have the sidecar and edit as yaml +1. Add the following key, value `sidecar.istio.io/inject: false` as an annotation on the workload +1. Click **Save**. + +**Result:** The Istio sidecar will not be injected into the workload. + +:::note + +If you are having issues with a Job you deployed not completing, you will need to add this annotation to your pod using the provided steps. Since Istio Sidecars run indefinitely, a Job cannot be considered complete even after its task has completed. + +::: + + +### [Next: Add Deployments with the Istio Sidecar ](use-istio-sidecar.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md new file mode 100644 index 00000000000..7203d827091 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md @@ -0,0 +1,30 @@ +--- +title: 6. Generate and View Traffic +--- + + + + + +This section describes how to view the traffic that is being managed by Istio. + +## The Kiali Traffic Graph + +The Istio overview page provides a link to the Kiali dashboard. From the Kiali dashboard, you are able to view graphs for each namespace. The Kiali graph provides a powerful way to visualize the topology of your Istio service mesh. It shows you which services communicate with each other. + +:::note Prerequisites: + +To enable traffic to show up in the graph, ensure you have prometheus installed in the cluster. Rancher-istio installs Kiali configured by default to work with the rancher-monitoring chart. You can use rancher-monitoring or install your own monitoring solution. Optional: you can change configuration on how data scraping occurs by setting the [Selectors & Scrape Configs](../../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) options. + +::: + +To see the traffic graph, + +1. In the cluster where Istio is installed, click **Istio** in the left navigation bar. +1. Click the **Kiali** link. +1. Click on **Graph** in the side nav. +1. Change the namespace in the **Namespace** dropdown to view the traffic for each namespace. + +If you refresh the URL to the BookInfo app several times, you should be able to see green arrows on the Kiali graph showing traffic to `v1` and `v3` of the `reviews` service. The control panel on the right side of the graph lets you configure details including how many minutes of the most recent traffic should be shown on the graph. + +For additional tools and visualizations, you can go to Grafana, and Prometheus dashboards from the **Monitoring** **Overview** page diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md new file mode 100644 index 00000000000..38548a4921d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md @@ -0,0 +1,150 @@ +--- +title: 4. Set up the Istio Gateway +--- + + + + + +The gateway to each cluster can have its own port or load balancer, which is unrelated to a service mesh. By default, each Rancher-provisioned cluster has one NGINX ingress controller allowing traffic into the cluster. + +You can use the Nginx Ingress controller with or without Istio installed. If this is the only gateway to your cluster, Istio will be able to route traffic from service to service, but Istio will not be able to receive traffic from outside the cluster. + +To allow Istio to receive external traffic, you need to enable Istio's gateway, which works as a north-south proxy for external traffic. When you enable the Istio gateway, the result is that your cluster will have two Ingresses. + +You will also need to set up a Kubernetes gateway for your services. This Kubernetes resource points to Istio's implementation of the ingress gateway to the cluster. + +You can route traffic into the service mesh with a load balancer or use Istio's NodePort gateway. This section describes how to set up the NodePort gateway. + +For more information on the Istio gateway, refer to the [Istio documentation.](https://istio.io/docs/reference/config/networking/v1alpha3/gateway/) + +![In an Istio-enabled cluster, you can have two Ingresses: the default Nginx Ingress, and the default Istio controller.](/img/istio-ingress.svg) + +## Enable an Istio Gateway + +The ingress gateway is a Kubernetes service that will be deployed in your cluster. The Istio Gateway allows for more extensive customization and flexibility. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Istio > Gateways**. +1. Click **Create from Yaml**. +1. Paste your Istio Gateway yaml, or **Read from File**. +1. Click **Create**. + +**Result:** The gateway is deployed, and will now route traffic with applied rules. + +## Example Istio Gateway + +We add the BookInfo app deployments in services when going through the Workloads example. Next we add an Istio Gateway so that the app is accessible from outside your cluster. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Istio > Gateways**. +1. Click **Create from Yaml**. +1. Copy and paste the Gateway yaml provided below. +1. Click **Create**. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: bookinfo-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +--- +``` + +Then to deploy the VirtualService that provides the traffic routing for the Gateway: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Istio > VirtualServices**. +1. Copy and paste the VirtualService yaml provided below. +1. Click **Create**. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: bookinfo +spec: + hosts: + - "*" + gateways: + - bookinfo-gateway + http: + - match: + - uri: + exact: /productpage + - uri: + prefix: /static + - uri: + exact: /login + - uri: + exact: /logout + - uri: + prefix: /api/v1/products + route: + - destination: + host: productpage + port: + number: 9080 +``` + +**Result:** You have configured your gateway resource so that Istio can receive traffic from outside the cluster. + +Confirm that the resource exists by running: +``` +kubectl get gateway -A +``` + +The result should be something like this: +``` +NAME AGE +bookinfo-gateway 64m +``` + +### Access the ProductPage Service from a Web Browser + +To test and see if the BookInfo app deployed correctly, the app can be viewed a web browser using the Istio controller IP and port, combined with the request name specified in your Kubernetes gateway resource: + +`http://:/productpage` + +To get the ingress gateway URL and port, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Scroll down to the `istio-system` namespace. +1. Within `istio-system`, there is a workload named `istio-ingressgateway`. Under the name of this workload, you should see links, such as `80/tcp`. +1. Click one of those links. This should show you the URL of the ingress gateway in your web browser. Append `/productpage` to the URL. + +**Result:** You should see the BookInfo app in the web browser. + +For help inspecting the Istio controller URL and ports, try the commands the [Istio documentation.](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#determining-the-ingress-ip-and-ports) + +## Troubleshooting + +The [official Istio documentation](https://istio.io/docs/tasks/traffic-management/ingress/ingress-control/#troubleshooting) suggests `kubectl` commands to inspect the correct ingress host and ingress port for external requests. + +### Confirming that the Kubernetes Gateway Matches Istio's Ingress Controller + +You can try the steps in this section to make sure the Kubernetes gateway is configured properly. + +In the gateway resource, the selector refers to Istio's default ingress controller by its label, in which the key of the label is `istio` and the value is `ingressgateway`. To make sure the label is appropriate for the gateway, do the following: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Scroll down to the `istio-system` namespace. +1. Within `istio-system`, there is a workload named `istio-ingressgateway`. Click the name of this workload and go to the **Labels and Annotations** section. You should see that it has the key `istio` and the value `ingressgateway`. This confirms that the selector in the Gateway resource matches Istio's default ingress controller. + +### [Next: Set up Istio's Components for Traffic Management](set-up-traffic-management.md) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md new file mode 100644 index 00000000000..150289745d7 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md @@ -0,0 +1,79 @@ +--- +title: 5. Set up Istio's Components for Traffic Management +--- + + + + + +A central advantage of traffic management in Istio is that it allows dynamic request routing. Some common applications for dynamic request routing include canary deployments and blue/green deployments. The two key resources in Istio traffic management are *virtual services* and *destination rules*. + +- [Virtual services](https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service/) intercept and direct traffic to your Kubernetes services, allowing you to divide percentages of traffic from a request to different services. You can use them to define a set of routing rules to apply when a host is addressed. +- [Destination rules](https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule/) serve as the single source of truth about which service versions are available to receive traffic from virtual services. You can use these resources to define policies that apply to traffic that is intended for a service after routing has occurred. + +This section describes how to add an example virtual service that corresponds to the `reviews` microservice in the sample BookInfo app. The purpose of this service is to divide traffic between two versions of the `reviews` service. + +In this example, we take the traffic to the `reviews` service and intercept it so that 50 percent of it goes to `v1` of the service and 50 percent goes to `v2`. + +After this virtual service is deployed, we will generate traffic and see from the Kiali visualization that traffic is being routed evenly between the two versions of the service. + +To deploy the virtual service and destination rules for the `reviews` service, +1. Click **☰ > Cluster Management**. +1. Go to the cluster where Istio is installed and click **Explore**. +1. In the cluster where Istio is installed, click **Istio > DestinationRules** in the left navigation bar. +1. Click **Create**. +1. Copy and paste the DestinationRule yaml provided below. +1. Click **Create**. +1. Click **Edit as YAML** and use this configuration: + + ```yaml + apiVersion: networking.istio.io/v1alpha3 + kind: DestinationRule + metadata: + name: reviews + spec: + host: reviews + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 + - name: v3 + labels: + version: v3 + ``` +1. Click **Create**. + +Then to deploy the VirtualService that provides the traffic routing that utilizes the DestinationRule: + +1. Click **VirtualService** in the side nav bar. +1. Click **Create from Yaml**. +1. Copy and paste the VirtualService yaml provided below. +1. Click **Create**. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: reviews +spec: + hosts: + - reviews + http: + - route: + - destination: + host: reviews + subset: v1 + weight: 50 + - destination: + host: reviews + subset: v3 + weight: 50 +--- +``` + +**Result:** When you generate traffic to this service (for example, by refreshing the ingress gateway URL), the Kiali traffic graph will reflect that traffic to the `reviews` service is divided evenly between `v1` and `v3`. + +### [Next: Generate and View Traffic](generate-and-view-traffic.md) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md new file mode 100644 index 00000000000..af5b22672c5 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md @@ -0,0 +1,363 @@ +--- +title: 3. Add Deployments and Services with the Istio Sidecar +--- + + + + + +:::note Prerequisite: + +To enable Istio for a workload, the cluster and namespace must have the Istio app installed. + +::: + +Enabling Istio in a namespace only enables automatic sidecar injection for new workloads. To enable the Envoy sidecar for existing workloads, you need to enable it manually for each workload. + +To inject the Istio sidecar on an existing workload in the namespace, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. Click **Workload**. +1. Go to the workload where you want to inject the Istio sidecar and click **⋮ > Redeploy**. When the workload is redeployed, it will have the Envoy sidecar automatically injected. + +Wait a few minutes for the workload to upgrade to have the istio sidecar. Click it and go to the Containers section. You should be able to see `istio-proxy` alongside your original workload. This means the Istio sidecar is enabled for the workload. Istio is doing all the wiring for the sidecar envoy. Now Istio can do all the features automatically if you enable them in the yaml. + +### Add Deployments and Services + +There are a few ways to add new **Deployments** in your namespace: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Click **Deployment**. +1. Fill out the form, or **Edit as Yaml**. +1. Click **Create**. + +To add a **Service** to your namespace: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Service Discovery > Services**. +1. Click **Create**. +1. Select the type of service that you want. +1. Fill out the form, or **Edit as Yaml**. +1. Click **Create** + +You can also create deployments and services using the kubectl **shell** + +1. Run `kubectl create -f .yaml` if your file is stored locally in the cluster +1. Or run `cat<< EOF | kubectl apply -f -`, paste the file contents into the terminal, then run `EOF` to complete the command. + +### Example Deployments and Services + +Next we add the Kubernetes resources for the sample deployments and services for the BookInfo app in Istio's documentation. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the top navigation bar, open the kubectl shell. +1. Run `cat<< EOF | kubectl apply -f -` +1. Copy the below resources into the the shell. +1. Run `EOF` + +This will set up the following sample resources from Istio's example BookInfo app: + +Details service and deployment: + +- A `details` Service +- A ServiceAccount for `bookinfo-details` +- A `details-v1` Deployment + +Ratings service and deployment: + +- A `ratings` Service +- A ServiceAccount for `bookinfo-ratings` +- A `ratings-v1` Deployment + +Reviews service and deployments (three versions): + +- A `reviews` Service +- A ServiceAccount for `bookinfo-reviews` +- A `reviews-v1` Deployment +- A `reviews-v2` Deployment +- A `reviews-v3` Deployment + +Productpage service and deployment: + +This is the main page of the app, which will be visible from a web browser. The other services will be called from this page. + +- A `productpage` service +- A ServiceAccount for `bookinfo-productpage` +- A `productpage-v1` Deployment + +### Resource YAML + +```yaml +# Copyright 2017 Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################################################## +# Details service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: details + labels: + app: details + service: details +spec: + ports: + - port: 9080 + name: http + selector: + app: details +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-details +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: details-v1 + labels: + app: details + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: details + version: v1 + template: + metadata: + labels: + app: details + version: v1 + spec: + serviceAccountName: bookinfo-details + containers: + - name: details + image: docker.io/istio/examples-bookinfo-details-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Ratings service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: ratings + labels: + app: ratings + service: ratings +spec: + ports: + - port: 9080 + name: http + selector: + app: ratings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-ratings +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ratings-v1 + labels: + app: ratings + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: ratings + version: v1 + template: + metadata: + labels: + app: ratings + version: v1 + spec: + serviceAccountName: bookinfo-ratings + containers: + - name: ratings + image: docker.io/istio/examples-bookinfo-ratings-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Reviews service +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: reviews + labels: + app: reviews + service: reviews +spec: + ports: + - port: 9080 + name: http + selector: + app: reviews +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-reviews +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v1 + labels: + app: reviews + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v1 + template: + metadata: + labels: + app: reviews + version: v1 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v2 + labels: + app: reviews + version: v2 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v2 + template: + metadata: + labels: + app: reviews + version: v2 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v2:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reviews-v3 + labels: + app: reviews + version: v3 +spec: + replicas: 1 + selector: + matchLabels: + app: reviews + version: v3 + template: + metadata: + labels: + app: reviews + version: v3 + spec: + serviceAccountName: bookinfo-reviews + containers: + - name: reviews + image: docker.io/istio/examples-bookinfo-reviews-v3:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +################################################################################################## +# Productpage services +################################################################################################## +apiVersion: v1 +kind: Service +metadata: + name: productpage + labels: + app: productpage + service: productpage +spec: + ports: + - port: 9080 + name: http + selector: + app: productpage +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bookinfo-productpage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: productpage-v1 + labels: + app: productpage + version: v1 +spec: + replicas: 1 + selector: + matchLabels: + app: productpage + version: v1 + template: + metadata: + labels: + app: productpage + version: v1 + spec: + serviceAccountName: bookinfo-productpage + containers: + - name: productpage + image: docker.io/istio/examples-bookinfo-productpage-v1:1.15.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9080 +--- +``` + +### [Next: Set up the Istio Gateway](set-up-istio-gateway.md) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md new file mode 100644 index 00000000000..664ee75b85e --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md @@ -0,0 +1,43 @@ +--- +title: Pod Security Policies +--- + + + + + +:::note + +These cluster options are only available for [clusters in which Rancher has launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + +::: + +You can always assign a pod security policy (PSP) to an existing project if you didn't assign one during creation. + +### Prerequisites + +- Create a Pod Security Policy within Rancher. Before you can assign a default PSP to an existing project, you must have a PSP available for assignment. For instruction, see [Creating Pod Security Policies](../../new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md). +- Assign a default Pod Security Policy to the project's cluster. You can't assign a PSP to a project until one is already applied to the cluster. For more information, see [the documentation about adding a pod security policy to a cluster](../../new-user-guides/manage-clusters/add-a-pod-security-policy.md). + +### Applying a Pod Security Policy + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to move a namespace and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the project that you want to add a PSP to. From that project, select **⋮ > Edit Config**. +1. From the **Pod Security Policy** drop-down, select the PSP you want to apply to the project. + Assigning a PSP to a project will: + + - Override the cluster's default PSP. + - Apply the PSP to the project. + - Apply the PSP to any namespaces you add to the project later. + +1. Click **Save**. + +**Result:** The PSP is applied to the project and any namespaces added to the project. + +:::note + +Any workloads that are already running in a cluster or project before a PSP is assigned will not be checked to determine if they comply with the PSP. Workloads would need to be cloned or upgraded to see if they pass the PSP. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md new file mode 100644 index 00000000000..be7d1b5a57a --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md @@ -0,0 +1,74 @@ +--- +title: How Resource Quotas Work in Rancher Projects +--- + + + + + +Resource quotas in Rancher include the same functionality as the [native version of Kubernetes](https://kubernetes.io/docs/concepts/policy/resource-quotas/). However, in Rancher, resource quotas have been extended so that you can apply them to projects. + +In a standard Kubernetes deployment, resource quotas are applied to individual namespaces. However, you cannot apply the quota to your namespaces simultaneously with a single action. Instead, the resource quota must be applied multiple times. + +In the following diagram, a Kubernetes administrator is trying to enforce a resource quota without Rancher. The administrator wants to apply a resource quota that sets the same CPU and memory limit to every namespace in his cluster (`Namespace 1-4`) . However, in the base version of Kubernetes, each namespace requires a unique resource quota. The administrator has to create four different resource quotas that have the same specs configured (`Resource Quota 1-4`) and apply them individually. + +Base Kubernetes: Unique Resource Quotas Being Applied to Each Namespace + +![Native Kubernetes Resource Quota Implementation](/img/kubernetes-resource-quota.svg) + +Resource quotas are a little different in Rancher. In Rancher, you apply a resource quota to the project, and then the quota propagates to each namespace, whereafter Kubernetes enforces your limits using the native version of resource quotas. If you want to change the quota for a specific namespace, you can override it. + +The resource quota includes two limits, which you set while creating or editing a project: + + +- **Project Limits:** + + This set of values configures a total limit for each specified resource shared among all namespaces in the project. + +- **Namespace Default Limits:** + + This set of values configures the default quota limit available for each namespace for each specified resource. + When a namespace is created in the project without overrides, this limit is automatically bound to the namespace and enforced. + + +In the following diagram, a Rancher administrator wants to apply a resource quota that sets the same CPU and memory limit for every namespace in their project (`Namespace 1-4`). However, in Rancher, the administrator can set a resource quota for the project (`Project Resource Quota`) rather than individual namespaces. This quota includes resource limits for both the entire project (`Project Limit`) and individual namespaces (`Namespace Default Limit`). Rancher then propagates the `Namespace Default Limit` quotas to each namespace (`Namespace Resource Quota`) when created. + +Rancher: Resource Quotas Propagating to Each Namespace + +![Rancher Resource Quota Implementation](/img/rancher-resource-quota.png) + +Let's highlight some more nuanced functionality for namespaces created **_within_** the Rancher UI. If a quota is deleted at the project level, it will also be removed from all namespaces contained within that project, despite any overrides that may exist. Further, updating an existing namespace default limit for a quota at the project level will not result in that value being propagated to existing namespaces in the project; the updated value will only be applied to newly created namespaces in that project. To update a namespace default limit for existing namespaces you can delete and subsequently recreate the quota at the project level with the new default value. This will result in the new default value being applied to all existing namespaces in the project. + +Before creating a namespace in a project, Rancher compares the amounts of the project's available resources and requested resources, regardless of whether they come from the default or overridden limits. +If the requested resources exceed the remaining capacity in the project for those resources, Rancher will assign the namespace the remaining capacity for that resource. + +However, this is not the case with namespaces created **_outside_** of Rancher's UI. For namespaces created via `kubectl`, Rancher +will assign a resource quota that has a **zero** amount for any resource that requested more capacity than what remains in the project. + +To create a namespace in an existing project via `kubectl`, use the `field.cattle.io/projectId` annotation. To override the default +requested quota limit, use the `field.cattle.io/resourceQuota` annotation. + +Note that Rancher will only override limits for resources that are defined on the project quota. + +``` +apiVersion: v1 +kind: Namespace +metadata: + annotations: + field.cattle.io/projectId: [your-cluster-ID]:[your-project-ID] + field.cattle.io/resourceQuota: '{"limit":{"limitsCpu":"100m", "configMaps": "50"}}' + name: my-ns +``` +In this example, if the project's quota does not include configMaps in its list of resources, then Rancher will ignore `configMaps` in this override. + +Users are advised to create dedicated `ResourceQuota` objects in namespaces to configure additional custom limits for resources not defined on the project. +Resource quotas are native Kubernetes objects, and Rancher will ignore user-defined quotas in namespaces belonging to a project with a quota, +thus giving users more control. + +The following table explains the key differences between the two quota types. + +| Rancher Resource Quotas | Kubernetes Resource Quotas | +| ---------------------------------------------------------- | -------------------------------------------------------- | +| Applies to projects and namespace. | Applies to namespaces only. | +| Creates resource pool for all namespaces in project. | Applies static resource limits to individual namespaces. | +| Applies resource quotas to namespaces through propagation. | Applies only to the assigned namespace. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md new file mode 100644 index 00000000000..9f6c8ca5655 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md @@ -0,0 +1,38 @@ +--- +title: Overriding the Default Limit for a Namespace +--- + + + + + +Although the **Namespace Default Limit** propagates from the project to each namespace when created, in some cases, you may need to increase (or decrease) the quotas for a specific namespace. In this situation, you can override the default limits by editing the namespace. + +In the diagram below, the Rancher administrator has a resource quota in effect for their project. However, the administrator wants to override the namespace limits for `Namespace 3` so that it has more resources available. Therefore, the administrator [raises the namespace limits](../../../new-user-guides/manage-clusters/projects-and-namespaces.md) for `Namespace 3` so that the namespace can access more resources. + +Namespace Default Limit Override + +![Namespace Default Limit Override](/img/rancher-resource-quota-override.svg) + +How to: [Editing Namespace Resource Quotas](../../../new-user-guides/manage-clusters/projects-and-namespaces.md) + +### Editing Namespace Resource Quotas + +If there is a resource quota configured for a project, you can override the namespace default limit to provide a specific namespace with access to more (or less) project resources. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to edit a namespace resource quota and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the namespace for which you want to edit the resource quota. Click **⋮ > Edit Config**. +1. Edit the resource limits. These limits determine the resources available to the namespace. The limits must be set within the configured project limits. + + For more information about each **Resource Type**, see [the type reference](resource-quota-types.md). + + :::note + + - If a resource quota is not configured for the project, these options will not be available. + - If you enter limits that exceed the configured project limits, Rancher will not let you save your edits. + + ::: + +**Result:** Your override is applied to the namespace's resource quota. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md new file mode 100644 index 00000000000..e789f5bf38b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md @@ -0,0 +1,31 @@ +--- +title: Resource Quota Type Reference +--- + + + + + +When you create a resource quota, you are configuring the pool of resources available to the project. You can set the following resource limits for the following resource types. + +| Resource Type | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU Limit* | The maximum amount of CPU (in [millicores](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)) allocated to the project/namespace.1 | +| CPU Reservation* | The minimum amount of CPU (in millicores) guaranteed to the project/namespace.1 | +| Memory Limit* | The maximum amount of memory (in bytes) allocated to the project/namespace.1 | +| Memory Reservation* | The minimum amount of memory (in bytes) guaranteed to the project/namespace.1 | +| Storage Reservation | The minimum amount of storage (in gigabytes) guaranteed to the project/namespace. | +| Services Load Balancers | The maximum number of load balancers services that can exist in the project/namespace. | +| Services Node Ports | The maximum number of node port services that can exist in the project/namespace. | +| Pods | The maximum number of pods that can exist in the project/namespace in a non-terminal state (i.e., pods with a state of `.status.phase in (Failed, Succeeded)` equal to true). | +| Services | The maximum number of services that can exist in the project/namespace. | +| ConfigMaps | The maximum number of ConfigMaps that can exist in the project/namespace. | +| Persistent Volume Claims | The maximum number of persistent volume claims that can exist in the project/namespace. | +| Replications Controllers | The maximum number of replication controllers that can exist in the project/namespace. | +| Secrets | The maximum number of secrets that can exist in the project/namespace. | + +:::note ***** + +When setting resource quotas, if you set anything related to CPU or Memory (i.e. limits or reservations) on a project / namespace, all containers will require a respective CPU or Memory field set during creation. A container default resource limit can be set at the same time to avoid the need to explicitly set these limits for every workload. See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits) for more details on why this is required. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md new file mode 100644 index 00000000000..dd4e3241319 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits.md @@ -0,0 +1,44 @@ +--- +title: Setting Container Default Resource Limits +--- + + + + + +When setting resource quotas, if you set anything related to CPU or Memory (i.e. limits or reservations) on a project / namespace, all containers will require a respective CPU or Memory field set during creation. See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/resource-quotas/#requests-vs-limits) for more details on why this is required. + +To avoid setting these limits on each and every container during workload creation, a default container resource limit can be specified on the namespace. + +### Editing the Container Default Resource Limit + +Edit the container default resource limit when: + +- You have a CPU or Memory resource quota set on a project, and want to supply the corresponding default values for a container. +- You want to edit the default container resource limit. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to edit the default resource limit and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the project that you want to edit the container default resource limit. From that project, select **⋮ > Edit Config**. +1. Expand **Container Default Resource Limit** and edit the values. + +### Resource Limit Propagation + +When the default container resource limit is set at a project level, the parameter will be propagated to any namespace created in the project after the limit has been set. For any existing namespace in a project, this limit will not be automatically propagated. You will need to manually set the default container resource limit for any existing namespaces in the project in order for it to be used when creating any containers. + +You can set a default container resource limit on a project and launch any catalog applications. + +Once a container default resource limit is configured on a namespace, the default will be pre-populated for any containers created in that namespace. These limits/reservations can always be overridden during workload creation. + +### Container Resource Quota Types + +The following resource limits can be configured: + +| Resource Type | Description | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU Limit | The maximum amount of CPU (in [millicores](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu)) allocated to the container.| +| CPU Reservation | The minimum amount of CPU (in millicores) guaranteed to the container. | +| Memory Limit | The maximum amount of memory (in bytes) allocated to the container. | +| Memory Reservation | The minimum amount of memory (in bytes) guaranteed to the container. +| NVIDIA GPU Limit/Reservation | The amount of GPUs allocated to the container. The limit and reservation are always the same for GPUs. | diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md new file mode 100644 index 00000000000..fe129e27bb4 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md @@ -0,0 +1,152 @@ +--- +title: Persistent Grafana Dashboards +--- + + + + + +To allow the Grafana dashboard to persist after the Grafana instance restarts, add the dashboard configuration JSON into a ConfigMap. ConfigMaps also allow the dashboards to be deployed with a GitOps or CD based approach. This allows the dashboard to be put under version control. + +- [Creating a Persistent Grafana Dashboard](#creating-a-persistent-grafana-dashboard) +- [Known Issues](#known-issues) + +## Creating a Persistent Grafana Dashboard + + + + +:::note Prerequisites: + +- The monitoring application needs to be installed. +- To create the persistent dashboard, you must have at least the **Manage Config Maps** Rancher RBAC permissions assigned to you in the project or namespace that contains the Grafana Dashboards. This correlates to the `monitoring-dashboard-edit` or `monitoring-dashboard-admin` Kubernetes native RBAC Roles exposed by the Monitoring chart. +- To see the links to the external monitoring UIs, including Grafana dashboards, you will need at least a [project-member role.](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#users-with-rancher-based-permissions) + +::: + +### 1. Get the JSON model of the dashboard that you want to persist + +To create a persistent dashboard, you will need to get the JSON model of the dashboard you want to persist. You can use a premade dashboard or build your own. + +To use a premade dashboard, go to [https://grafana.com/grafana/dashboards](https://grafana.com/grafana/dashboards), open up its detail page, and click on the **Download JSON** button to get the JSON model for the next step. + +To use your own dashboard: + +1. Click on the link to open Grafana. On the cluster detail page, click **Monitoring**. +1. Log in to Grafana. Note: The default Admin username and password for the Grafana instance is `admin/prom-operator`. Alternative credentials can also be supplied on deploying or upgrading the chart. + + :::note + + Regardless of who has the password, in order to access the Grafana instance, you still need at least the Manage Services or View Monitoring permissions in the project that Rancher Monitoring is deployed into. Alternative credentials can also be supplied on deploying or upgrading the chart. + + ::: + +1. Create a dashboard using Grafana's UI. Once complete, go to the dashboard's settings by clicking on the gear icon in the top navigation menu. In the left navigation menu, click **JSON Model**. +1. Copy the JSON data structure that appears. + +### 2. Create a ConfigMap using the Grafana JSON model + +Create a ConfigMap in the namespace that contains your Grafana Dashboards (e.g. cattle-dashboards by default). + +The ConfigMap should look like this: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards # Change if using a non-default namespace +data: + .json: |- + +``` + +By default, Grafana is configured to watch all ConfigMaps with the `grafana_dashboard` label within the `cattle-dashboards` namespace. + +To specify that you would like Grafana to watch for ConfigMaps across all namespaces, refer to [this section.](#configuring-namespaces-for-the-grafana-dashboard-configmap) + +To create the ConfigMap in the Rancher UI, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. Click **More Resources > Core > ConfigMaps**. +1. Click **Create**. +1. Set up the key-value pairs similar to the example above. When entering the value for `.json`, click **Read from File** to upload the JSON data model as the value. +1. Click **Create**. + +**Result:** After the ConfigMap is created, it should show up on the Grafana UI and be persisted even if the Grafana pod is restarted. + +Dashboards that are persisted using ConfigMaps cannot be deleted or edited from the Grafana UI. + +If you attempt to delete the dashboard in the Grafana UI, you will see the error message "Dashboard cannot be deleted because it was provisioned." To delete the dashboard, you will need to delete the ConfigMap. + +### Configuring Namespaces for the Grafana Dashboard ConfigMap + +To specify that you would like Grafana to watch for ConfigMaps across all namespaces, set this value in the `rancher-monitoring` Helm chart: + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +Note that the RBAC roles exposed by the Monitoring chart to add Grafana Dashboards are still restricted to giving permissions for users to add dashboards in the namespace defined in `grafana.dashboards.namespace`, which defaults to `cattle-dashboards`. + + + + +:::note Prerequisites: + +- The monitoring application needs to be installed. +- You must have the cluster-admin ClusterRole permission. + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to configure the Grafana namespace and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Grafana**. +1. Log in to Grafana. Note: The default Admin username and password for the Grafana instance is `admin/prom-operator`. Alternative credentials can also be supplied on deploying or upgrading the chart. + + :::note + + Regardless of who has the password, cluster administrator permission in Rancher is still required to access the Grafana instance. + + ::: + +1. Go to the dashboard that you want to persist. In the top navigation menu, go to the dashboard settings by clicking the gear icon. +1. In the left navigation menu, click **JSON Model**. +1. Copy the JSON data structure that appears. +1. Create a ConfigMap in the `cattle-dashboards` namespace. The ConfigMap needs to have the label `grafana_dashboard: "1"`. Paste the JSON into the ConfigMap in the format shown in the example below: + + ```yaml + apiVersion: v1 + kind: ConfigMap + metadata: + labels: + grafana_dashboard: "1" + name: + namespace: cattle-dashboards + data: + .json: |- + + ``` + +**Result:** After the ConfigMap is created, it should show up on the Grafana UI and be persisted even if the Grafana pod is restarted. + +Dashboards that are persisted using ConfigMaps cannot be deleted from the Grafana UI. If you attempt to delete the dashboard in the Grafana UI, you will see the error message "Dashboard cannot be deleted because it was provisioned." To delete the dashboard, you will need to delete the ConfigMap. + +To prevent the persistent dashboard from being deleted when Monitoring v2 is uninstalled, add the following annotation to the `cattle-dashboards` namespace: + +``` +helm.sh/resource-policy: "keep" +``` + + + + +## Known Issues + +For users who are using Monitoring V2 v9.4.203 or below, uninstalling the Monitoring chart will delete the `cattle-dashboards` namespace, which will delete all persisted dashboards, unless the namespace is marked with the annotation `helm.sh/resource-policy: "keep"`. + +This annotation will be added by default in the new monitoring chart released by Rancher v2.5.8, but it still needs to be manually applied for users of earlier Rancher versions. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md new file mode 100644 index 00000000000..c3b315f3e5b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md @@ -0,0 +1,43 @@ +--- +title: Customizing Grafana Dashboards +--- + + + + + +In this section, you'll learn how to customize the Grafana dashboard to show metrics that apply to a certain container. + +### Prerequisites + +Before you can customize a Grafana dashboard, the `rancher-monitoring` application must be installed. + +To see the links to the external monitoring UIs, including Grafana dashboards, you will need at least a [project-member role.](../../../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md#users-with-rancher-based-permissions) + +### Signing in to Grafana + +1. In the Rancher UI, go to the cluster that has the dashboard you want to customize. +1. In the left navigation menu, click **Monitoring.** +1. Click **Grafana.** The Grafana dashboard should open in a new tab. +1. Go to the log in icon in the lower left corner and click **Sign In.** +1. Log in to Grafana. The default Admin username and password for the Grafana instance is `admin/prom-operator`. (Regardless of who has the password, cluster administrator permission in Rancher is still required access the Grafana instance.) Alternative credentials can also be supplied on deploying or upgrading the chart. + + +### Getting the PromQL Query Powering a Grafana Panel + +For any panel, you can click the title and click **Explore** to get the PromQL queries powering the graphic. + +For this example, we would like to get the CPU usage for the Alertmanager container, so we click **CPU Utilization > Inspect.** + +The **Data** tab shows the underlying data as a time series, with the time in first column and the PromQL query result in the second column. Copy the PromQL query. + +``` +(1 - (avg(irate({__name__=~"node_cpu_seconds_total|windows_cpu_time_total",mode="idle"}[5m])))) * 100 +``` + +You can then modify the query in the Grafana panel or create a new Grafana panel using the query. + +See also: + +- [Grafana docs on editing a panel](https://grafana.com/docs/grafana/latest/panels-visualizations/configure-panel-options/#edit-a-panel) +- [Grafana docs on adding a panel to a dashboard](https://grafana.com/docs/grafana/latest/panels-visualizations/panel-editor-overview) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md new file mode 100644 index 00000000000..020b06924d6 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md @@ -0,0 +1,23 @@ +--- +title: Debugging High Memory Usage +--- + + + + + +Every time series in Prometheus is uniquely identified by its [metric name](https://prometheus.io/docs/practices/naming/#metric-names) and optional key-value pairs called [labels.](https://prometheus.io/docs/practices/naming/#labels) + +The labels allow the ability to filter and aggregate the time series data, but they also multiply the amount of data that Prometheus collects. + +Each time series has a defined set of labels, and Prometheus generates a new time series for all unique combinations of labels. If a metric has two labels attached, two time series are generated for that metric. Changing any label value, including adding or removing a label, will create a new time series. + +Prometheus is optimized to store data that is index-based on series. It is designed for a relatively consistent number of time series and a relatively large number of samples that need to be collected from the exporters over time. + +Inversely, Prometheus is not optimized to accommodate a rapidly changing number of time series. For that reason, large bursts of memory usage can occur when monitoring is installed on clusters where many resources are being created and destroyed, especially on multi-tenant clusters. + +### Reducing Memory Bursts + +To reduce memory consumption, Prometheus can be configured to store fewer time series, by scraping fewer metrics or by attaching fewer labels to the time series. To see which series use the most memory, you can check the TSDB (time series database) status page in the Prometheus UI. + +Distributed Prometheus solutions such as [Thanos](https://thanos.io/) and [Cortex](https://cortexmetrics.io/) use an alternate architecture in which multiple small Prometheus instances are deployed. In the case of Thanos, the metrics from each Prometheus are aggregated into the common Thanos deployment, and then those metrics are exported to a persistent store, such as S3. This more robust architecture avoids burdening any single Prometheus instance with too many time series, while also preserving the ability to query metrics on a global level. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md new file mode 100644 index 00000000000..2e4ab117ca0 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md @@ -0,0 +1,79 @@ +--- +title: Enable Monitoring +--- + + + + + +As an [administrator](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) or [cluster owner](../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles), you can configure Rancher to deploy Prometheus to monitor your Kubernetes cluster. + +This page describes how to enable monitoring and alerting within a cluster using the new monitoring application. + +You can enable monitoring with or without SSL. + +## Requirements + +- Allow traffic on port 9796 for each of your nodes. Prometheus scrapes metrics from these ports. + - You may also need to allow traffic on port 10254 for each of your nodes, if [PushProx](../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#pushprox) is disabled (`ingressNginx.enabled` set to `false`), or you've upgraded from a previous Rancher version that had v1 monitoring already installed. +- Make sure that your cluster fulfills the resource requirements. The cluster should have at least 1950Mi memory available, 2700m CPU, and 50Gi storage. See [Configuring Resource Limits and Requests](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#configuring-resource-limits-and-requests) for a breakdown of the resource limits and requests. +- When you install monitoring on an RKE cluster that uses RancherOS or Flatcar Linux nodes, change the etcd node certificate directory to `/opt/rke/etc/kubernetes/ssl`. +- For clusters that have been provisioned with the RKE CLI and that have the address set to a hostname instead of an IP address, set `rkeEtcd.clients.useLocalhost` to `true` when you configure the Values during installation. For example: + +```yaml +rkeEtcd: + clients: + useLocalhost: true +``` + +:::note + +If you want to set up Alertmanager, Grafana or Ingress, it has to be done with the settings on the Helm chart deployment. It's problematic to create Ingress outside the deployment. + +::: + +## Setting Resource Limits and Requests + +The resource requests and limits can be configured when installing `rancher-monitoring`. To configure Prometheus resources from the Rancher UI, click **Apps > Monitoring** in the upper left corner. + +For more information about the default limits, see [this page.](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md#configuring-resource-limits-and-requests) + +## Install the Monitoring Application + +### Enable Monitoring for use without SSL + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster Tools** (bottom left corner). +1. Click **Install** by Monitoring. +1. Optional: Customize requests, limits and more for Alerting, Prometheus, and Grafana in the Values step. For help, refer to the [configuration reference.](../../../reference-guides/monitoring-v2-configuration/helm-chart-options.md) + +**Result:** The monitoring app is deployed in the `cattle-monitoring-system` namespace. + +### Enable Monitoring for use with SSL + +1. Follow the steps on [this page](../../new-user-guides/kubernetes-resources-setup/secrets.md) to create a secret in order for SSL to be used for alerts. + - The secret should be created in the `cattle-monitoring-system` namespace. If it doesn't exist, create it first. + - Add the `ca`, `cert`, and `key` files to the secret. +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to enable monitoring for use with SSL and click **Explore**. +1. Click **Apps > Charts**. +1. Click **Monitoring**. +1. Click **Install** or **Update**, depending on whether you have already installed Monitoring. +1. Check the box for **Customize Helm options before install** and click **Next**. +1. Click **Alerting**. +1. In the **Additional Secrets** field, add the secrets created earlier. + +**Result:** The monitoring app is deployed in the `cattle-monitoring-system` namespace. + +When [creating a receiver,](../../../reference-guides/monitoring-v2-configuration/receivers.md#creating-receivers-in-the-rancher-ui) SSL-enabled receivers such as email or webhook will have a **SSL** section with fields for **CA File Path**, **Cert File Path**, and **Key File Path**. Fill in these fields with the paths to each of `ca`, `cert`, and `key`. The path will be of the form `/etc/alertmanager/secrets/name-of-file-in-secret`. + +For example, if you created a secret with these key-value pairs: + +```yaml +ca.crt=`base64-content` +cert.pem=`base64-content` +key.pfx=`base64-content` +``` + +Then **Cert File Path** would be set to `/etc/alertmanager/secrets/cert.pem`. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md new file mode 100644 index 00000000000..50a45a16216 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md @@ -0,0 +1,11 @@ +--- +title: Customizing Grafana Dashboards +--- + + + + + +Grafana dashboards are customized the same way whether it's for rancher-monitoring or for Prometheus Federator. + +For instructions, refer to [this page](../customize-grafana-dashboard.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md new file mode 100644 index 00000000000..b4f9fcc166f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md @@ -0,0 +1,90 @@ +--- +title: Enable Prometheus Federator +--- + + + + + +## Requirements + +By default, Prometheus Federator is configured and intended to be deployed alongside [rancher-monitoring](../../../../pages-for-subheaders/monitoring-and-alerting.md), which deploys Prometheus Operator alongside a Cluster Prometheus that each Project Monitoring Stack is configured to federate namespace-scoped metrics from by default. + +For instructions on installing rancher-monitoring, refer to [this page](../enable-monitoring.md). + +The default configuration should already be compatible with your rancher-monitoring stack. However, to optimize the security and usability of Prometheus Federator in your cluster, we recommend making these additional configurations to rancher-monitoring: + +- [Ensure the cattle-monitoring-system namespace is placed into the System Project](#ensure-the-cattle-monitoring-system-namespace-is-placed-into-the-system-project-or-a-similarly-locked-down-project-that-has-access-to-other-projects-in-the-cluster). +- [Configure rancher-monitoring to only watch for resources created by the Helm chart itself](#configure-rancher-monitoring-to-only-watch-for-resources-created-by-the-helm-chart-itself). +- [Increase the CPU / memory limits of the Cluster Prometheus](#increase-the-cpu--memory-limits-of-the-cluster-prometheus). + +### Ensure the cattle-monitoring-system namespace is placed into the System Project (or a similarly locked down Project that has access to other Projects in the cluster) + +![Select Projects-Namespaces](/img/install-in-system-project.png) + +Prometheus Operator's security model expects that the namespace it is deployed into (e.g., `cattle-monitoring-system`) has limited access for anyone except Cluster Admins to avoid privilege escalation via execing into Pods (such as the Jobs executing Helm operations). In addition, deploying Prometheus Federator and all Project Prometheus stacks into the System Project ensures that each Project Prometheus is able to reach out to scrape workloads across all Projects, even if Network Policies are defined via Project Network Isolation. It also provides limited access for Project Owners, Project Members, and other users so that they're unable to access data that they shouldn't have access to (i.e., being allowed to exec into pods, set up the ability to scrape namespaces outside of a given Project, etc.). + +1. Open the `System` project to check your namespaces: + + Click **Cluster > Projects/Namespaces** in the Rancher UI. This will display all of the namespaces in the `System` project: + + ![Select Projects-Namespaces](/img/cattle-monitoring-system.png) + +1. If you have an existing Monitoring V2 installation within the `cattle-monitoring-system` namespace, but that namespace is not in the `System` project, you may move the `cattle-monitoring-system` namespace into the `System` project or into another project of limited access. To do so, you may either: + + - Drag and drop the namespace into the `System` project or + - Select **⋮** to the right of the namespace, click **Move**, then choose `System` from the **Target Project** dropdown + + ![Move to a New Project](/img/move-to-new-project.png) + +### Configure rancher-monitoring to only watch for resources created by the Helm chart itself + +Since each Project Monitoring Stack will watch the other namespaces and collect additional custom workload metrics or dashboards already, it's recommended to configure the following settings on all selectors to ensure that the Cluster Prometheus Stack only monitors resources created by the Helm Chart itself: + +``` +matchLabels: + release: "rancher-monitoring" +``` + +The following selector fields are recommended to have this value: +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +Once this setting is turned on, you can always create ServiceMonitors or PodMonitors that are picked up by the Cluster Prometheus by adding the label `release: "rancher-monitoring"` to them, in which case they will be ignored by Project Monitoring Stacks automatically by default, even if the namespace in which those ServiceMonitors or PodMonitors reside in are not system namespaces. + +:::note + +If you don't want to allow users to be able to create ServiceMonitors and PodMonitors that aggregate into the Cluster Prometheus in Project namespaces, you can additionally set the namespaceSelectors on the chart to only target system namespaces (which must contain `cattle-monitoring-system` and `cattle-dashboards`, where resources are deployed into by default by rancher-monitoring; you will also need to monitor the `default` namespace to get apiserver metrics or create a custom ServiceMonitor to scrape apiserver metrics from the Service residing in the default namespace) to limit your Cluster Prometheus from picking up other Prometheus Operator CRs. In that case, it would be recommended to turn `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true` to allow you to define ServiceMonitors that can monitor non-system namespaces from within a system namespace. + +::: + +### Increase the CPU / memory limits of the Cluster Prometheus + +Depending on a cluster's setup, it's generally recommended to give a large amount of dedicated memory to the Cluster Prometheus to avoid restarts due to out-of-memory errors (OOMKilled) usually caused by churn created in the cluster that causes a large number of high cardinality metrics to be generated and ingested by Prometheus within one block of time. This is one of the reasons why the default Rancher Monitoring stack expects around 4GB of RAM to be able to operate in a normal-sized cluster. However, when introducing Project Monitoring Stacks that are all sending `/federate` requests to the same Cluster Prometheus and are reliant on the Cluster Prometheus being "up" to federate that system data on their namespaces, it's even more important that the Cluster Prometheus has an ample amount of CPU / memory assigned to it to prevent an outage that can cause data gaps across all Project Prometheis in the cluster. + +:::note + +There are no specific recommendations on how much memory the Cluster Prometheus should be configured with since it depends entirely on the user's setup (namely the likelihood of encountering a high churn rate and the scale of metrics that could be generated at that time); it generally varies per setup. + +::: + +## Install the Prometheus Federator Application + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you want to install Prometheus Federator and click **Explore**. +1. Click **Apps -> Charts**. +1. Click the **Prometheus Federator** chart. +1. Click **Install**. +1. On the **Metadata** page, click **Next**. +1. In the **Namespaces** > **Project Release Namespace Project ID** field, the `System Project` is used as the default but can be overridden with another project with similarly [limited access](#ensure-the-cattle-monitoring-system-namespace-is-placed-into-the-system-project-or-a-similarly-locked-down-project-that-has-access-to-other-projects-in-the-cluster). Project IDs can be found with the following command run in the local upstream cluster: + + ```plain + kubectl get projects -A -o custom-columns="NAMESPACE":.metadata.namespace,"ID":.metadata.name,"NAME":.spec.displayName + ``` + +1. Click **Install**. + +**Result:** The Prometheus Federator app is deployed in the `cattle-monitoring-system` namespace. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md new file mode 100644 index 00000000000..8efe48f4ae8 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors.md @@ -0,0 +1,21 @@ +--- +title: Installing Project Monitors +--- + + + + + +Install **Project Monitors** in each project where you want to enable project monitoring. + +1. Click **☰ > Cluster Management**. + +1. On the **Clusters** page, go to the cluster where you want to enable monitoring and click **Explore**. + +1. Click **Monitoring > Project Monitors** on the left nav bar. Then click **Create** in the upper right. + + ![Project Monitors](/img/project-monitors.png) + +1. Select your project from the drop-down menu, then click **Create** again. + + ![Create Project Monitors](/img/create-project-monitors.png) diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md new file mode 100644 index 00000000000..d816f9b9e68 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md @@ -0,0 +1,17 @@ +--- +title: Setting up Prometheus Federator for a Workload +--- + + + + + +### Display CPU and Memory Metrics for a Workload + +Displaying CPU and memory metrics with Prometheus Federator is done the same way as with rancher-monitoring. For instructions, refer [here](../set-up-monitoring-for-workloads.md#display-cpu-and-memory-metrics-for-a-workload). + +### Setting up Metrics Beyond CPU and Memory + +Setting up metrics beyond CPU and memory with Prometheus Federator is done the same way as with rancher-monitoring. For instructions, refer [here](../set-up-monitoring-for-workloads.md#setting-up-metrics-beyond-cpu-and-memory). + + diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md new file mode 100644 index 00000000000..d5703cf33aa --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md @@ -0,0 +1,17 @@ +--- +title: Uninstall Prometheus Federator +--- + + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Installed Apps**. +1. Go to the `cattle-monitoring-system` namespace and check the boxes for `rancher-monitoring-crd` and `rancher-monitoring`. +1. Click **Delete**. +1. Confirm **Delete**. + +**Result:** `prometheus-federator` is uninstalled. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md new file mode 100644 index 00000000000..72206789fce --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md @@ -0,0 +1,31 @@ +--- +title: Setting up Monitoring for a Workload +--- + + + + + +If you only need CPU and memory time series for the workload, you don't need to deploy a ServiceMonitor or PodMonitor because the monitoring application already collects metrics data on resource usage by default. + +The steps for setting up monitoring for workloads depend on whether you want basic metrics such as CPU and memory for the workload, or whether you want to scrape custom metrics from the workload. + +If you only need CPU and memory time series for the workload, you don't need to deploy a ServiceMonitor or PodMonitor because the monitoring application already collects metrics data on resource usage by default. The resource usage time series data is in Prometheus's local time series database. + +Grafana shows the data in aggregate, but you can see the data for the individual workload by using a PromQL query that extracts the data for that workload. Once you have the PromQL query, you can execute the query individually in the Prometheus UI and see the time series visualized there, or you can use the query to customize a Grafana dashboard to display the workload metrics. For examples of PromQL queries for workload metrics, see [this section.](../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md#workload-metrics). + +To set up custom metrics for your workload, you will need to set up an exporter and create a new ServiceMonitor custom resource to configure Prometheus to scrape metrics from your exporter. + +### Display CPU and Memory Metrics for a Workload + +By default, the monitoring application already scrapes CPU and memory. + +To get some fine-grained detail for a particular workload, you can customize a Grafana dashboard to display the metrics for a particular workload. + +### Setting up Metrics Beyond CPU and Memory + +For custom metrics, you will need to expose the metrics on your application in a format supported by Prometheus. + +Then we recommend that you should create a new ServiceMonitor custom resource. When this resource is created, the Prometheus custom resource will be automatically updated so that its scrape configuration includes the new custom metrics endpoint. Then Prometheus will begin scraping metrics from the endpoint. + +You can also create a PodMonitor to expose the custom metrics endpoint, but ServiceMonitors are more appropriate for the majority of use cases. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md new file mode 100644 index 00000000000..688d54a0506 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md @@ -0,0 +1,23 @@ +--- +title: Uninstall Monitoring +--- + + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Installed Apps**. +1. Go to the `cattle-monitoring-system` namespace and check the boxes for `rancher-monitoring-crd` and `rancher-monitoring`. +1. Click **Delete**. +1. Confirm **Delete**. + +**Result:** `rancher-monitoring` is uninstalled. + +:::note Persistent Grafana Dashboards: + +For users who are using Monitoring V2 v9.4.203 or below, uninstalling the Monitoring chart will delete the cattle-dashboards namespace, which will delete all persisted dashboards, unless the namespace is marked with the annotation `helm.sh/resource-policy: "keep"`. This annotation is added by default in Monitoring V2 v14.5.100+ but can be manually applied on the cattle-dashboards namespace before an uninstall if an older version of the Monitoring chart is currently installed onto your cluster. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md new file mode 100644 index 00000000000..bc213ee0637 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md @@ -0,0 +1,47 @@ +--- +title: Alertmanager Configuration +--- + + + + + +It is usually not necessary to directly edit the Alertmanager custom resource. For most use cases, you will only need to edit the Receivers and Routes to configure notifications. + +When Receivers and Routes are updated, the monitoring application will automatically update the Alertmanager custom resource to be consistent with those changes. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [this section.](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-how-alertmanager-works) + +::: + +## About the Alertmanager Custom Resource + +By default, Rancher Monitoring deploys a single Alertmanager onto a cluster that uses a default Alertmanager Config Secret. + +You may want to edit the Alertmanager custom resource if you would like to take advantage of advanced options that are not exposed in the Rancher UI forms, such as the ability to create a routing tree structure that is more than two levels deep. + +It is also possible to create more than one Alertmanager in a cluster, which may be useful if you want to implement namespace-scoped monitoring. In this case, you should manage the Alertmanager custom resources using the same underlying Alertmanager Config Secret. + +### Deeply Nested Routes + +While the Rancher UI only supports a routing tree that is two levels deep, you can configure more deeply nested routing structures by editing the Alertmanager YAML. + +### Multiple Alertmanager Replicas + +As part of the chart deployment options, you can opt to increase the number of replicas of the Alertmanager deployed onto your cluster. The replicas can all be managed using the same underlying Alertmanager Config Secret. + +This Secret should be updated or modified any time you want to: + +- Add in new notifiers or receivers +- Change the alerts that should be sent to specific notifiers or receivers +- Change the group of alerts that are sent out + +By default, you can either choose to supply an existing Alertmanager Config Secret (i.e. any Secret in the `cattle-monitoring-system` namespace) or allow Rancher Monitoring to deploy a default Alertmanager Config Secret onto your cluster. + +By default, the Alertmanager Config Secret created by Rancher will never be modified or deleted on an upgrade or uninstall of the `rancher-monitoring` chart. This restriction prevents users from losing or overwriting their alerting configuration when executing operations on the chart. + +For more information on what fields can be specified in the Alertmanager Config Secret, please look at the [Prometheus Alertmanager docs.](https://prometheus.io/docs/alerting/latest/alertmanager/) + +The full spec for the Alertmanager configuration file and what it takes in can be found [here.](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md new file mode 100644 index 00000000000..122316d018d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md @@ -0,0 +1,23 @@ +--- +title: Prometheus Configuration +--- + + + + + +It is usually not necessary to directly edit the Prometheus custom resource because the monitoring application automatically updates it based on changes to ServiceMonitors and PodMonitors. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information, see [this section.](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + +## About the Prometheus Custom Resource + +The Prometheus CR defines a desired Prometheus deployment. The Prometheus Operator observes the Prometheus CR. When the CR changes, the Prometheus Operator creates `prometheus-rancher-monitoring-prometheus`, a Prometheus deployment based on the CR configuration. + +The Prometheus CR specifies details such as rules and what Alertmanagers are connected to Prometheus. Rancher builds this CR for you. + +Monitoring V2 only supports one Prometheus per cluster. However, you might want to edit the Prometheus CR if you want to limit monitoring to certain namespaces. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md new file mode 100644 index 00000000000..b4259adf475 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md @@ -0,0 +1,84 @@ +--- +title: Configuring PrometheusRules +--- + + + + + +A PrometheusRule defines a group of Prometheus alerting and/or recording rules. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information, see [this section.](../../../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + +### Creating PrometheusRules in the Rancher UI + +:::note Prerequisite: + +The monitoring application needs to be installed. + +::: + +To create rule groups in the Rancher UI, + +1. Go to the cluster where you want to create rule groups. Click **Monitoring > Advanced** and click **Prometheus Rules**. +1. Click **Create**. +1. Enter a **Group Name**. +1. Configure the rules. In Rancher's UI, we expect a rule group to contain either alert rules or recording rules, but not both. For help filling out the forms, refer to the configuration options below. +1. Click **Create**. + +**Result:** Alerts can be configured to send notifications to the receiver(s). + +### About the PrometheusRule Custom Resource + +When you define a Rule (which is declared within a RuleGroup in a PrometheusRule resource), the [spec of the Rule itself](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule) contains labels that are used by Alertmanager to figure out which Route should receive this Alert. For example, an Alert with the label `team: front-end` will be sent to all Routes that match on that label. + +Prometheus rule files are held in PrometheusRule custom resources. A PrometheusRule allows you to define one or more RuleGroups. Each RuleGroup consists of a set of Rule objects that can each represent either an alerting or a recording rule with the following fields: + +- The name of the new alert or record +- A PromQL expression for the new alert or record +- Labels that should be attached to the alert or record that identify it (e.g. cluster name or severity) +- Annotations that encode any additional important pieces of information that need to be displayed on the notification for an alert (e.g. summary, description, message, runbook URL, etc.). This field is not required for recording rules. + +For more information on what fields can be specified, please look at the [Prometheus Operator spec.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#prometheusrulespec) + +Use the label selector field `ruleSelector` in the Prometheus object to define the rule files that you want to be mounted into Prometheus. + +For examples, refer to the Prometheus documentation on [recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) and [alerting rules.](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) + +## Configuration + +### Rule Group + +| Field | Description | +|-------|----------------| +| Group Name | The name of the group. Must be unique within a rules file. | +| Override Group Interval | Duration in seconds for how often rules in the group are evaluated. | + + +### Alerting Rules + +[Alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) allow you to define alert conditions based on PromQL (Prometheus Query Language) expressions and to send notifications about firing alerts to an external service. + +| Field | Description | +|-------|----------------| +| Alert Name | The name of the alert. Must be a valid label value. | +| Wait To Fire For | Duration in seconds. Alerts are considered firing once they have been returned for this long. Alerts which have not yet fired for long enough are considered pending. | +| PromQL Expression | The PromQL expression to evaluate. Prometheus will evaluate the current value of this PromQL expression on every evaluation cycle and all resultant time series will become pending/firing alerts. For more information, refer to the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) or our [example PromQL expressions.](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md) | +| Labels | Labels to add or overwrite for each alert. | +| Severity | When enabled, labels are attached to the alert or record that identify it by the severity level. | +| Severity Label Value | Critical, warning, or none | +| Annotations | Annotations are a set of informational labels that can be used to store longer additional information, such as alert descriptions or runbook links. A [runbook](https://en.wikipedia.org/wiki/Runbook) is a set of documentation about how to handle alerts. The annotation values can be [templated.](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#templating) | + +### Recording Rules + +[Recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) allow you to precompute frequently needed or computationally expensive PromQL (Prometheus Query Language) expressions and save their result as a new set of time series. + +| Field | Description | +|-------|----------------| +| Time Series Name | The name of the time series to output to. Must be a valid metric name. | +| PromQL Expression | The PromQL expression to evaluate. Prometheus will evaluate the current value of this PromQL expression on every evaluation cycle and the result will be recorded as a new set of time series with the metric name as given by 'record'. For more information about expressions, refer to the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/querying/basics/) or our [example PromQL expressions.](../../../../integrations-in-rancher/monitoring-and-alerting/promql-expressions.md) | +| Labels | Labels to add or overwrite before storing the result. | diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md new file mode 100644 index 00000000000..90ba4bd4e8c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/open-ports-with-firewalld.md @@ -0,0 +1,111 @@ +--- +title: Opening Ports with firewalld +--- + + + + + +> We recommend disabling firewalld. For Kubernetes 1.19.x and higher, firewalld must be turned off. + +Some distributions of Linux [derived from RHEL,](https://en.wikipedia.org/wiki/Red_Hat_Enterprise_Linux#Rebuilds) including Oracle Linux, may have default firewall rules that block communication with Helm. + +For example, one Oracle Linux image in AWS has REJECT rules that stop Helm from communicating with Tiller: + +``` +Chain INPUT (policy ACCEPT) +target prot opt source destination +ACCEPT all -- anywhere anywhere state RELATED,ESTABLISHED +ACCEPT icmp -- anywhere anywhere +ACCEPT all -- anywhere anywhere +ACCEPT tcp -- anywhere anywhere state NEW tcp dpt:ssh +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain FORWARD (policy ACCEPT) +target prot opt source destination +REJECT all -- anywhere anywhere reject-with icmp-host-prohibited + +Chain OUTPUT (policy ACCEPT) +target prot opt source destination +``` + +You can check the default firewall rules with this command: + +``` +sudo iptables --list +``` + +This section describes how to use `firewalld` to apply the [firewall port rules](../../pages-for-subheaders/installation-requirements.md#port-requirements) for nodes in a high-availability Rancher server cluster. + +## Prerequisite + +Install v7.x or later ofv`firewalld`: + +``` +yum install firewalld +systemctl start firewalld +systemctl enable firewalld +``` + +## Applying Firewall Port Rules + +In the Rancher high-availability installation instructions, the Rancher server is set up on three nodes that have all three Kubernetes roles: etcd, controlplane, and worker. If your Rancher server nodes have all three roles, run the following commands on each node: + +``` +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` +If your Rancher server nodes have separate roles, use the following commands based on the role of the node: + +``` +# For etcd nodes, run the following commands: +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=2379/tcp +firewall-cmd --permanent --add-port=2380/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp + +# For control plane nodes, run the following commands: +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=6443/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp + +# For worker nodes, run the following commands: +firewall-cmd --permanent --add-port=22/tcp +firewall-cmd --permanent --add-port=80/tcp +firewall-cmd --permanent --add-port=443/tcp +firewall-cmd --permanent --add-port=2376/tcp +firewall-cmd --permanent --add-port=8472/udp +firewall-cmd --permanent --add-port=9099/tcp +firewall-cmd --permanent --add-port=10250/tcp +firewall-cmd --permanent --add-port=10254/tcp +firewall-cmd --permanent --add-port=30000-32767/tcp +firewall-cmd --permanent --add-port=30000-32767/udp +``` + +After the `firewall-cmd` commands have been run on a node, use the following command to enable the firewall rules: + +``` +firewall-cmd --reload +``` + +**Result:** The firewall is updated so that Helm can communicate with the Rancher server nodes. diff --git a/versioned_docs/version-latest/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md new file mode 100644 index 00000000000..e024f1dd779 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md @@ -0,0 +1,43 @@ +--- +title: Tuning etcd for Large Installations +--- + + + + + +When running larger Rancher installations with 15 or more clusters it is recommended to increase the default keyspace for etcd from the default 2GB. The maximum setting is 8GB and the host should have enough RAM to keep the entire dataset in memory. When increasing this value you should also increase the size of the host. The keyspace size can also be adjusted in smaller installations if you anticipate a high rate of change of pods during the garbage collection interval. + +The etcd data set is automatically cleaned up on a five minute interval by Kubernetes. There are situations, e.g. deployment thrashing, where enough events could be written to etcd and deleted before garbage collection occurs and cleans things up causing the keyspace to fill up. If you see `mvcc: database space exceeded` errors, in the etcd logs or Kubernetes API server logs, you should consider increasing the keyspace size. This can be accomplished by setting the [quota-backend-bytes](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) setting on the etcd servers. + +### Example: This snippet of the RKE cluster.yml file increases the keyspace size to 5GB + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + quota-backend-bytes: 5368709120 +``` + +## Scaling etcd disk performance + +You can follow the recommendations from [the etcd docs](https://etcd.io/docs/v3.4.0/tuning/#disk) on how to tune the disk priority on the host. + +Additionally, to reduce IO contention on the disks for etcd, you can use a dedicated device for the data and wal directory. Based on etcd best practices, mirroring RAID configurations are unnecessary because etcd replicates data between the nodes in the cluster. You can use striping RAID configurations to increase available IOPS. + +To implement this solution in an RKE cluster, the `/var/lib/etcd/data` and `/var/lib/etcd/wal` directories will need to have disks mounted and formatted on the underlying host. In the `extra_args` directive of the `etcd` service, you must include the `wal_dir` directory. Without specifying the `wal_dir`, etcd process will try to manipulate the underlying `wal` mount with insufficient permissions. + +```yaml +# RKE cluster.yml +--- +services: + etcd: + extra_args: + data-dir: '/var/lib/rancher/etcd/data/' + wal-dir: '/var/lib/rancher/etcd/wal/wal_dir' + extra_binds: + - '/var/lib/etcd/data:/var/lib/rancher/etcd/data' + - '/var/lib/etcd/wal:/var/lib/rancher/etcd/wal' +``` diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/add-users-to-projects.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/add-users-to-projects.md new file mode 100644 index 00000000000..d99e7c18120 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/add-users-to-projects.md @@ -0,0 +1,66 @@ +--- +title: Adding Users to Projects +--- + + + + + +If you want to provide a user with access and permissions to _specific_ projects and resources within a cluster, assign the user a project membership. + +You can add members to a project as it is created, or add them to an existing project. + +:::tip + +Want to provide a user with access to _all_ projects within a cluster? See [Adding Cluster Members](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) instead. + +::: + +### Adding Members to a New Project + +You can add members to a project as you create it (recommended if possible). For details on creating a new project, refer to the [cluster administration section.](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) + +### Adding Members to an Existing Project + +Following project creation, you can add users as project members so that they can access its resources. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add members to a project and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the project where you want to add members. Next to the **Create Namespace** button above the project name, click **☰**. Select **Edit Config**. +1. In the **Members** tab, click **Add**. +1. Search for the user or group that you want to add to the project. + + If external authentication is configured: + + - Rancher returns users from your external authentication source as you type. + + - A drop-down allows you to add groups instead of individual users. The dropdown only lists groups that you, the logged in user, are included in. + + :::note + + If you are logged in as a local user, external users do not display in your search results. + + ::: + +1. Assign the user or group **Project** roles. + + [What are Project Roles?](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::note Notes: + + - Users assigned the `Owner` or `Member` role for a project automatically inherit the `namespace creation` role. However, this role is a [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole), meaning its scope extends to all projects in the cluster. Therefore, users explicitly assigned the `Owner` or `Member` role for a project can create or delete namespaces in other projects they're assigned to, even with only the `Read Only` role assigned. + + - By default, the Rancher role of `project-member` inherits from the `Kubernetes-edit` role, and the `project-owner` role inherits from the `Kubernetes-admin` role. As such, both `project-member` and `project-owner` roles will allow for namespace management, including the ability to create and delete namespaces. + + - For `Custom` roles, you can modify the list of individual roles available for assignment. + + - To add roles to the list, [Add a Custom Role](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md). + - To remove roles from the list, [Lock/Unlock Roles](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md). + + ::: + +**Result:** The chosen users are added to the project. + +- To revoke project membership, select the user and click **Delete**. This action deletes membership, not the user. +- To modify a user's roles in the project, delete them from the project, and then re-add them with modified roles. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md new file mode 100644 index 00000000000..8d265e034f8 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md @@ -0,0 +1,46 @@ +--- +title: Cluster Drivers +--- + + + + + +Cluster drivers are used to create clusters in a [hosted Kubernetes provider](../../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md), such as Google GKE. The availability of which cluster driver to display when creating clusters is defined by the cluster driver's status. Only `active` cluster drivers will be displayed as an option for creating clusters. By default, Rancher is packaged with several existing cloud provider cluster drivers, but you can also add custom cluster drivers to Rancher. + +If there are specific cluster drivers that you do not want to show your users, you may deactivate those cluster drivers within Rancher and they will not appear as an option for cluster creation. + +### Managing Cluster Drivers + +:::note Prerequisites: + +To create, edit, or delete cluster drivers, you need _one_ of the following permissions: + +- [Administrator Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md) +- [Custom Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md#custom-global-permissions) with the [Manage Cluster Drivers](../manage-role-based-access-control-rbac/global-permissions.md) role assigned. + +::: + +## Activating/Deactivating Cluster Drivers + +By default, Rancher only activates drivers for the most popular cloud providers, Google GKE, Amazon EKS and Azure AKS. If you want to show or hide any node driver, you can change its status. + +1. In the upper left corner, click **☰ > Cluster Management**. + +2. In the left navigation menu, click **Drivers**. + +3. On the **Cluster Drivers** tab, select the driver that you wish to activate or deactivate and click **⋮ > Activate** or **⋮ > Deactivate**. + +## Adding Custom Cluster Drivers + +If you want to use a cluster driver that Rancher doesn't support out-of-the-box, you can add the provider's driver in order to start using them to create _hosted_ kubernetes clusters. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **Drivers**. +1. On the **Cluster Drivers** tab, click **Add Cluster Driver**. +1. Complete the **Add Cluster Driver** form. Then click **Create**. + + +### Developing your own Cluster Driver + +In order to develop cluster driver to add to Rancher, please refer to our [example](https://github.com/rancher-plugins/kontainer-engine-driver-example). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md new file mode 100644 index 00000000000..44256b18da0 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md @@ -0,0 +1,45 @@ +--- +title: Node Drivers +--- + + + + + +Node drivers are used to provision hosts, which Rancher uses to launch and manage Kubernetes clusters. A node driver is the same as a [Docker Machine driver](https://docs.docker.com/machine/drivers/). The availability of which node driver to display when creating node templates is defined based on the node driver's status. Only `active` node drivers will be displayed as an option for creating node templates. By default, Rancher is packaged with many existing Docker Machine drivers, but you can also create custom node drivers to add to Rancher. + +If there are specific node drivers that you don't want to show to your users, you would need to de-activate these node drivers. + +#### Managing Node Drivers + +:::note Prerequisites: + +To create, edit, or delete drivers, you need _one_ of the following permissions: + +- [Administrator Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md) +- [Custom Global Permissions](../manage-role-based-access-control-rbac/global-permissions.md#custom-global-permissions) with the [Manage Node Drivers](../manage-role-based-access-control-rbac/global-permissions.md) role assigned. + +::: + +## Activating/Deactivating Node Drivers + +By default, Rancher only activates drivers for the most popular cloud providers, Amazon EC2, Azure, DigitalOcean and vSphere. If you want to show or hide any node driver, you can change its status. + +1. In the upper left corner, click **☰ > Cluster Management**. + +2. In the left navigation menu, click **Drivers**. + +2. On the **Node Drivers** tab, select the driver that you wish to activate or deactivate and click **⋮ > Activate** or **⋮ > Deactivate**. + +## Adding Custom Node Drivers + +If you want to use a node driver that Rancher doesn't support out-of-the-box, you can add that provider's driver in order to start using them to create node templates and eventually node pools for your Kubernetes cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **Drivers**. +1. On **Node Drivers** tab, click **Add Node Driver**. +1. Complete the **Add Node Driver** form. Then click **Create**. + +### Developing your own node driver + +Node drivers are implemented with [Docker Machine](https://docs.docker.com/machine/). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md new file mode 100644 index 00000000000..c12a1f03ef0 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md @@ -0,0 +1,68 @@ +--- +title: Access and Sharing +--- + + + + + +If you are an RKE template owner, you can share it with users or groups of users, who can then use the template to create clusters. + +Since RKE templates are specifically shared with users and groups, owners can share different RKE templates with different sets of users. + +When you share a template, each user can have one of two access levels: + +- **Owner:** This user can update, delete, and share the templates that they own. The owner can also share the template with other users. +- **User:** These users can create clusters using the template. They can also upgrade those clusters to new revisions of the same template. When you share a template as **Make Public (read-only),** all users in your Rancher setup have the User access level for the template. + +If you create a template, you automatically become an owner of that template. + +If you want to delegate responsibility for updating the template, you can share ownership of the template. For details on how owners can modify templates, refer to the [documentation about revising templates.](manage-rke1-templates.md) + +There are several ways to share templates: + +- Add users to a new RKE template during template creation +- Add users to an existing RKE template +- Make the RKE template public, sharing it with all users in the Rancher setup +- Share template ownership with users who are trusted to modify the template + +### Sharing Templates with Specific Users or Groups + +To allow users or groups to create clusters using your template, you can give them the basic **User** access level for the template. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the template that you want to share and click the **⋮ > Edit**. +1. In the **Share Template** section, click on **Add Member**. +1. Search in the **Name** field for the user or group you want to share the template with. +1. Choose the **User** access type. +1. Click **Save**. + +**Result:** The user or group can create clusters using the template. + +### Sharing Templates with All Users + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the template that you want to share and click the **⋮ > Edit**. +1. Under **Share Template,** check the box for **Make Public (read-only)**. +1. Click **Save**. + +**Result:** All users in the Rancher setup can create clusters using the template. + +### Sharing Ownership of Templates + +If you are the creator of a template, you might want to delegate responsibility for maintaining and updating a template to another user or group. + +In that case, you can give users the Owner access type, which allows another user to update your template, delete it, or share access to it with other users. + +To give Owner access to a user or group, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the RKE template that you want to share and click the **⋮ > Edit**. +1. Under **Share Template**, click on **Add Member** and search in the **Name** field for the user or group you want to share the template with. +1. In the **Access Type** field, click **Owner**. +1. Click **Save**. + +**Result:** The user or group has the Owner access type, and can modify, share, or delete the template. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md new file mode 100644 index 00000000000..1818e9076ff --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md @@ -0,0 +1,63 @@ +--- +title: Applying Templates +--- + + + + + +You can create a cluster from an RKE template that you created, or from a template that has been [shared with you.](access-or-share-templates.md) + +RKE templates can be applied to new clusters. + +You can [save the configuration of an existing cluster as an RKE template.](#converting-an-existing-cluster-to-use-an-rke-template) Then the cluster's settings can only be changed if the template is updated. + +You can't change a cluster to use a different RKE template. You can only update the cluster to a new revision of the same template. + + +### Creating a Cluster from an RKE Template + +To add a cluster [hosted by an infrastructure provider](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) using an RKE template, use these steps: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create** and choose the infrastructure provider. +1. Provide the cluster name and node template details as usual. +1. To use an RKE template, under the **Cluster Options**, check the box for **Use an existing RKE template and revision**. +1. Choose an RKE template and revision from the dropdown menu. +1. Optional: You can edit any settings that the RKE template owner marked as **Allow User Override** when the template was created. If there are settings that you want to change, but don't have the option to, you will need to contact the template owner to get a new revision of the template. Then you will need to edit the cluster to upgrade it to the new revision. +1. Click **Create** to launch the cluster. + +### Updating a Cluster Created with an RKE Template + +When the template owner creates a template, each setting has a switch in the Rancher UI that indicates if users can override the setting. + +- If the setting allows a user override, you can update these settings in the cluster by [editing the cluster.](../../../../pages-for-subheaders/cluster-configuration.md) +- If the switch is turned off, you cannot change these settings unless the cluster owner creates a template revision that lets you override them. If there are settings that you want to change, but don't have the option to, you will need to contact the template owner to get a new revision of the template. + +If a cluster was created from an RKE template, you can edit the cluster to update the cluster to a new revision of the template. + +An existing cluster's settings can be [saved as an RKE template.](#converting-an-existing-cluster-to-use-an-rke-template) In that situation, you can also edit the cluster to update the cluster to a new revision of the template. + +:::note + +You can't change the cluster to use a different RKE template. You can only update the cluster to a new revision of the same template. + +::: + +### Converting an Existing Cluster to Use an RKE Template + +This section describes how to create an RKE template from an existing cluster. + +RKE templates cannot be applied to existing clusters, except if you save an existing cluster's settings as an RKE template. This exports the cluster's settings as a new RKE template, and also binds the cluster to that template. The result is that the cluster can only be changed if the [template is updated,](manage-rke1-templates.md#updating-a-template) and the cluster is upgraded to [use a newer version of the template.](manage-rke1-templates.md#upgrading-a-cluster-to-use-a-new-template-revision) + +To convert an existing cluster to use an RKE template, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster that will be converted to use an RKE template. Click **⋮ > Save as RKE Template**. +1. Enter a name for the template in the form that appears, and click **Create**. + +**Results:** + +- A new RKE template is created. +- The cluster is converted to use the new template. +- New clusters can be [created from the new template.](apply-templates.md#creating-a-cluster-from-an-rke-template) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md new file mode 100644 index 00000000000..eff8b84ee92 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md @@ -0,0 +1,61 @@ +--- +title: Template Creator Permissions +--- + + + + + +Administrators have the permission to create RKE templates, and only administrators can give that permission to other users. + +For more information on administrator permissions, refer to the [documentation on global permissions](../manage-role-based-access-control-rbac/global-permissions.md). + +## Giving Users Permission to Create Templates + +Templates can only be created by users who have the global permission **Create RKE Templates**. + +Administrators have the global permission to create templates, and only administrators can give that permission to other users. + +For information on allowing users to modify existing templates, refer to [Sharing Templates.](access-or-share-templates.md) + +Administrators can give users permission to create RKE templates in two ways: + +- By editing the permissions of an [individual user](#allowing-a-user-to-create-templates) +- By changing the [default permissions of new users](#allowing-new-users-to-create-templates-by-default) + +### Allowing a User to Create Templates + +An administrator can individually grant the role **Create RKE Templates** to any existing user by following these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Choose the user you want to edit and click **⋮ > Edit Config**. +1. In the **Built-in** section, check the box for **Create new RKE Cluster Templates** role along with any other roles the user should have. You may want to also check the box for **Create RKE Template Revisions**. +1. Click **Save**. + +**Result:** The user has permission to create RKE templates. + +### Allowing New Users to Create Templates by Default + +Alternatively, the administrator can give all new users the default permission to create RKE templates by following the following steps. This will not affect the permissions of existing users. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Go to the role named **Create new RKE Cluster Templates and click **⋮ > Edit Config**. +1. Select the option **Yes: Default role for new users**. +1. Click **Save**. +1. If you would like new users to also be able to create RKE template revisions, enable that role as default as well. + +**Result:** Any new user created in this Rancher installation will be able to create RKE templates. Existing users will not get this permission. + +### Revoking Permission to Create Templates + +Administrators can remove a user's permission to create templates with the following steps. Note: Administrators have full control over all resources regardless of whether fine-grained permissions are selected. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Choose the user you want to edit permissions for and click **⋮ > Edit Config**. +1. In the **Built-in** section, un-check the box for **Create RKE Templates** and **Create RKE Template Revisions,** if applicable. In this section, you can change the user back to a standard user, or give the user a different set of permissions. +1. Click **Save**. + +**Result:** The user cannot create RKE templates. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md new file mode 100644 index 00000000000..1758a8710ac --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md @@ -0,0 +1,47 @@ +--- +title: Template Enforcement +--- + + + + + +This section describes how template administrators can enforce templates in Rancher, restricting the ability of users to create clusters without a template. + +By default, any standard user in Rancher can create clusters. But when RKE template enforcement is turned on, + +- Only an administrator has the ability to create clusters without a template. +- All standard users must use an RKE template to create a new cluster. +- Standard users cannot create a cluster without using a template. + +Users can only create new templates if the administrator [gives them permission.](creator-permissions.md#allowing-a-user-to-create-templates) + +After a cluster is created with an RKE template, the cluster creator cannot edit settings that are defined in the template. The only way to change those settings after the cluster is created is to [upgrade the cluster to a new revision](apply-templates.md#updating-a-cluster-created-with-an-rke-template) of the same template. If cluster creators want to change template-defined settings, they would need to contact the template owner to get a new revision of the template. For details on how template revisions work, refer to the [documentation on revising templates.](manage-rke1-templates.md#updating-a-template) + +## Requiring New Clusters to Use an RKE Template + +You might want to require new clusters to use a template to ensure that any cluster launched by a [standard user](../manage-role-based-access-control-rbac/global-permissions.md) will use the Kubernetes and/or Rancher settings that are vetted by administrators. + +To require new clusters to use an RKE template, administrators can turn on RKE template enforcement with the following steps: + +1. Click **☰ > Global Settings**. +1. Go to the `cluster-template-enforcement` setting. Click **⋮ > Edit Setting**. +1. Set the value to **True** and click **Save**. + + :::note Important: + + When the admin sets the `cluster-template-enforcement` to True, they also need to share the `clusterTemplates` with users so that users can select one of these templates to create the cluster. + + ::: + +**Result:** All clusters provisioned by Rancher must use a template, unless the creator is an administrator. + +## Disabling RKE Template Enforcement + +To allow new clusters to be created without an RKE template, administrators can turn off RKE template enforcement with the following steps: + +1. Click **☰ > Global Settings**. +1. Go to the `cluster-template-enforcement` setting. Click **⋮ > Edit Setting**. +1. Set the value to **False** and click **Save**. + +**Result:** When clusters are provisioned by Rancher, they don't need to use a template. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md new file mode 100644 index 00000000000..4471ffd9300 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md @@ -0,0 +1,74 @@ +--- +title: Example Scenarios +--- + + + + + +These example scenarios describe how an organization could use templates to standardize cluster creation. + +- **Enforcing templates:** Administrators might want to [enforce one or more template settings for everyone](#enforcing-a-template-setting-for-everyone) if they want all new Rancher-provisioned clusters to have those settings. +- **Sharing different templates with different users:** Administrators might give [different templates to basic and advanced users,](#templates-for-basic-and-advanced-users) so that basic users have more restricted options and advanced users have more discretion when creating clusters. +- **Updating template settings:** If an organization's security and DevOps teams decide to embed best practices into the required settings for new clusters, those best practices could change over time. If the best practices change, [a template can be updated to a new revision](#updating-templates-and-clusters-created-with-them) and clusters created from the template can upgrade to the new version of the template. +- **Sharing ownership of a template:** When a template owner no longer wants to maintain a template, or wants to delegate ownership of the template, this scenario describes how [template ownership can be shared.](#allowing-other-users-to-control-and-share-a-template) + + +## Enforcing a Template Setting for Everyone + +Let's say there is an organization in which the administrators decide that all new clusters should be created with Kubernetes version 1.14. + +1. First, an administrator creates a template which specifies the Kubernetes version as 1.14 and marks all other settings as **Allow User Override**. +1. The administrator makes the template public. +1. The administrator turns on template enforcement. + +**Results:** + +- All Rancher users in the organization have access to the template. +- All new clusters created by [standard users](../manage-role-based-access-control-rbac/global-permissions.md) with this template will use Kubernetes 1.14 and they are unable to use a different Kubernetes version. By default, standard users don't have permission to create templates, so this template will be the only template they can use unless more templates are shared with them. +- All standard users must use a cluster template to create a new cluster. They cannot create a cluster without using a template. + +In this way, the administrators enforce the Kubernetes version across the organization, while still allowing end users to configure everything else. + +## Templates for Basic and Advanced Users + +Let's say an organization has both basic and advanced users. Administrators want the basic users to be required to use a template, while the advanced users and administrators create their clusters however they want. + +1. First, an administrator turns on [RKE template enforcement.](enforce-templates.md#requiring-new-clusters-to-use-an-rke-template) This means that every [standard user](../manage-role-based-access-control-rbac/global-permissions.md) in Rancher will need to use an RKE template when they create a cluster. +1. The administrator then creates two templates: + + - One template for basic users, with almost every option specified except for access keys + - One template for advanced users, which has most or all options has **Allow User Override** turned on + +1. The administrator shares the advanced template with only the advanced users. +1. The administrator makes the template for basic users public, so the more restrictive template is an option for everyone who creates a Rancher-provisioned cluster. + +**Result:** All Rancher users, except for administrators, are required to use a template when creating a cluster. Everyone has access to the restrictive template, but only advanced users have permission to use the more permissive template. The basic users are more restricted, while advanced users have more freedom when configuring their Kubernetes clusters. + +## Updating Templates and Clusters Created with Them + +Let's say an organization has a template that requires clusters to use Kubernetes v1.14. However, as time goes on, the administrators change their minds. They decide they want users to be able to upgrade their clusters to use newer versions of Kubernetes. + +In this organization, many clusters were created with a template that requires Kubernetes v1.14. Because the template does not allow that setting to be overridden, the users who created the cluster cannot directly edit that setting. + +The template owner has several options for allowing the cluster creators to upgrade Kubernetes on their clusters: + +- **Specify Kubernetes v1.15 on the template:** The template owner can create a new template revision that specifies Kubernetes v1.15. Then the owner of each cluster that uses that template can upgrade their cluster to a new revision of the template. This template upgrade allows the cluster creator to upgrade Kubernetes to v1.15 on their cluster. +- **Allow any Kubernetes version on the template:** When creating a template revision, the template owner can also mark the the Kubernetes version as **Allow User Override** using the switch near that setting on the Rancher UI. This will allow clusters that upgrade to this template revision to use any version of Kubernetes. +- **Allow the latest minor Kubernetes version on the template:** The template owner can also create a template revision in which the Kubernetes version is defined as **Latest v1.14 (Allows patch version upgrades)**. This means clusters that use that revision will be able to get patch version upgrades, but major version upgrades will not be allowed. + +## Allowing Other Users to Control and Share a Template + +Let's say Alice is a Rancher administrator. She owns an RKE template that reflects her organization's agreed-upon best practices for creating a cluster. + +Bob is an advanced user who can make informed decisions about cluster configuration. Alice trusts Bob to create new revisions of her template as the best practices get updated over time. Therefore, she decides to make Bob an owner of the template. + +To share ownership of the template with Bob, Alice [adds Bob as an owner of her template.](access-or-share-templates.md#sharing-ownership-of-templates) + +The result is that as a template owner, Bob is in charge of version control for that template. Bob can now do all of the following: + +- [Revise the template](manage-rke1-templates.md#updating-a-template) when the best practices change +- [Disable outdated revisions](manage-rke1-templates.md#disabling-a-template-revision) of the template so that no new clusters can be created with it +- [Delete the whole template](manage-rke1-templates.md#deleting-a-template) if the organization wants to go in a different direction +- [Set a certain revision as default](manage-rke1-templates.md#setting-a-template-revision-as-default) when users create a cluster with it. End users of the template will still be able to choose which revision they want to create the cluster with. +- [Share the template](access-or-share-templates.md) with specific users, make the template available to all Rancher users, or share ownership of the template with another user. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md new file mode 100644 index 00000000000..0b2b6f8b8eb --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md @@ -0,0 +1,73 @@ +--- +title: RKE Templates and Infrastructure +--- + + + + + +In Rancher, RKE templates are used to provision Kubernetes and define Rancher settings, while node templates are used to provision nodes. + +Therefore, even if RKE template enforcement is turned on, the end user still has flexibility when picking the underlying hardware when creating a Rancher cluster. The end users of an RKE template can still choose an infrastructure provider and the nodes they want to use. + +If you want to standardize the hardware in your clusters, use RKE templates conjunction with node templates or with a server provisioning tool such as Terraform. + +### Node Templates + +[Node templates](../../../../reference-guides/user-settings/manage-node-templates.md) are responsible for node configuration and node provisioning in Rancher. From your user profile, you can set up node templates to define which templates are used in each of your node pools. With node pools enabled, you can make sure you have the required number of nodes in each node pool, and ensure that all nodes in the pool are the same. + +### Terraform + +Terraform is a server provisioning tool. It uses infrastructure-as-code that lets you create almost every aspect of your infrastructure with Terraform configuration files. It can automate the process of server provisioning in a way that is self-documenting and easy to track in version control. + +This section focuses on how to use Terraform with the [Rancher 2 Terraform provider](https://www.terraform.io/docs/providers/rancher2/), which is a recommended option to standardize the hardware for your Kubernetes clusters. If you use the Rancher Terraform provider to provision hardware, and then use an RKE template to provision a Kubernetes cluster on that hardware, you can quickly create a comprehensive, production-ready cluster. + +Terraform allows you to: + +- Define almost any kind of infrastructure-as-code, including servers, databases, load balancers, monitoring, firewall settings, and SSL certificates +- Leverage catalog apps and multi-cluster apps +- Codify infrastructure across many platforms, including Rancher and major cloud providers +- Commit infrastructure-as-code to version control +- Easily repeat configuration and setup of infrastructure +- Incorporate infrastructure changes into standard development practices +- Prevent configuration drift, in which some servers become configured differently than others + +## How Does Terraform Work? + +Terraform is written in files with the extension `.tf`. It is written in HashiCorp Configuration Language, which is a declarative language that lets you define the infrastructure you want in your cluster, the cloud provider you are using, and your credentials for the provider. Then Terraform makes API calls to the provider in order to efficiently create that infrastructure. + +To create a Rancher-provisioned cluster with Terraform, go to your Terraform configuration file and define the provider as Rancher 2. You can set up your Rancher 2 provider with a Rancher API key. Note: The API key has the same permissions and access level as the user it is associated with. + +Then Terraform calls the Rancher API to provision your infrastructure, and Rancher calls the infrastructure provider. As an example, if you wanted to use Rancher to provision infrastructure on AWS, you would provide both your Rancher API key and your AWS credentials in the Terraform configuration file or in environment variables so that they could be used to provision the infrastructure. + +When you need to make changes to your infrastructure, instead of manually updating the servers, you can make changes in the Terraform configuration files. Then those files can be committed to version control, validated, and reviewed as necessary. Then when you run `terraform apply`, the changes would be deployed. + +## Tips for Working with Terraform + +- There are examples of how to provide most aspects of a cluster in the [documentation for the Rancher 2 provider.](https://www.terraform.io/docs/providers/rancher2/) + +- In the Terraform settings, you can install Docker Machine by using the Docker Machine node driver. + +- You can also modify auth in the Terraform provider. + +- You can reverse engineer how to do define a setting in Terraform by changing the setting in Rancher, then going back and checking your Terraform state file to see how it maps to the current state of your infrastructure. + +- If you want to manage Kubernetes cluster settings, Rancher settings, and hardware settings all in one place, use [Terraform modules](https://github.com/rancher/terraform-modules). You can pass a cluster configuration YAML file or an RKE template configuration file to a Terraform module so that the Terraform module will create it. In that case, you could use your infrastructure-as-code to manage the version control and revision history of both your Kubernetes cluster and its underlying hardware. + +## Tip for Creating CIS Benchmark Compliant Clusters + +This section describes one way that you can make security and compliance-related config files standard in your clusters. + +When you create a [CIS benchmark compliant cluster,](../../../../pages-for-subheaders/rancher-security.md) you have an encryption config file and an audit log config file. + +Your infrastructure provisioning system can write those files to disk. Then in your RKE template, you would specify where those files will be, then add your encryption config file and audit log config file as extra mounts to the `kube-api-server`. + +Then you would make sure that the `kube-api-server` flag in your RKE template uses your CIS-compliant config files. + +In this way, you can create flags that comply with the CIS benchmark. + +## Resources + +- [Terraform documentation](https://www.terraform.io/docs/) +- [Rancher2 Terraform provider documentation](https://www.terraform.io/docs/providers/rancher2/) +- [The RanchCast - Episode 1: Rancher 2 Terraform Provider](https://youtu.be/YNCq-prI8-8): In this demo, Director of Community Jason van Brackel walks through using the Rancher 2 Terraform Provider to provision nodes and create a custom cluster. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md new file mode 100644 index 00000000000..e9eac6fe7f6 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md @@ -0,0 +1,165 @@ +--- +title: Creating and Revising RKE Templates +--- + + + + + +This section describes how to manage RKE templates and revisions. You an create, share, update, and delete templates from the **Cluster Management** view under **RKE1 Configuration > RKE Templates**. + +Template updates are handled through a revision system. When template owners want to change or update a template, they create a new revision of the template. Individual revisions cannot be edited. However, if you want to prevent a revision from being used to create a new cluster, you can disable it. + +Template revisions can be used in two ways: to create a new cluster, or to upgrade a cluster that was created with an earlier version of the template. The template creator can choose a default revision, but when end users create a cluster, they can choose any template and any template revision that is available to them. After the cluster is created from a specific revision, it cannot change to another template, but the cluster can be upgraded to a newer available revision of the same template. + +The template owner has full control over template revisions, and can create new revisions to update the template, delete or disable revisions that should not be used to create clusters, and choose which template revision is the default. + + +### Prerequisites + +You can create RKE templates if you have the **Create RKE Templates** permission, which can be [given by an administrator.](creator-permissions.md) + +You can revise, share, and delete a template if you are an owner of the template. For details on how to become an owner of a template, refer to [the documentation on sharing template ownership.](access-or-share-templates.md#sharing-ownership-of-templates) + +### Creating a Template + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **RKE1 configuration > Node Templates**. +1. Click **Add Template**. +1. Provide a name for the template. An auto-generated name is already provided for the template' first version, which is created along with this template. +1. Optional: Share the template with other users or groups by [adding them as members.](access-or-share-templates.md#sharing-templates-with-specific-users-or-groups) You can also make the template public to share with everyone in the Rancher setup. +1. Then follow the form on screen to save the cluster configuration parameters as part of the template's revision. The revision can be marked as default for this template. + +**Result:** An RKE template with one revision is configured. You can use this RKE template revision later when you [provision a Rancher-launched cluster](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). After a cluster is managed by an RKE template, it cannot be disconnected and the option to uncheck **Use an existing RKE Template and Revision** will be unavailable. + +### Updating a Template + +When you update an RKE template, you are creating a revision of the existing template. Clusters that were created with an older version of the template can be updated to match the new revision. + +You can't edit individual revisions. Since you can't edit individual revisions of a template, in order to prevent a revision from being used, you can [disable it.](#disabling-a-template-revision) + +When new template revisions are created, clusters using an older revision of the template are unaffected. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the template that you want to edit and click the **⋮ > Edit**. +1. Edit the required information and click **Save**. +1. Optional: You can change the default revision of this template and also change who it is shared with. + +**Result:** The template is updated. To apply it to a cluster using an older version of the template, refer to the section on [upgrading a cluster to use a new revision of a template.](#upgrading-a-cluster-to-use-a-new-template-revision) + +### Deleting a Template + +When you no longer use an RKE template for any of your clusters, you can delete it. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **RKE1 configuration > RKE Templates**. +1. Go to the RKE template that you want to delete and click the **⋮ > Delete**. +1. Confirm the deletion. + +**Result:** The template is deleted. + +### Creating a Revision Based on the Default Revision + +You can clone the default template revision and quickly update its settings rather than creating a new revision from scratch. Cloning templates saves you the hassle of re-entering the access keys and other parameters needed for cluster creation. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the RKE template that you want to clone and click the **⋮ > New Revision from Default**. +1. Complete the rest of the form to create a new revision. + +**Result:** The RKE template revision is cloned and configured. + +### Creating a Revision Based on a Cloned Revision + +When creating new RKE template revisions from your user settings, you can clone an existing revision and quickly update its settings rather than creating a new one from scratch. Cloning template revisions saves you the hassle of re-entering the cluster parameters. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the template revision you want to clone. Then select **⋮ > Clone Revision**. +1. Complete the rest of the form. + +**Result:** The RKE template revision is cloned and configured. You can use the RKE template revision later when you provision a cluster. Any existing cluster using this RKE template can be upgraded to this new revision. + +### Disabling a Template Revision + +When you no longer want an RKE template revision to be used for creating new clusters, you can disable it. A disabled revision can be re-enabled. + +You can disable the revision if it is not being used by any cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE Templates**. +1. Go to the template revision you want to disable. Then select **⋮ > Disable**. + +**Result:** The RKE template revision cannot be used to create a new cluster. + +### Re-enabling a Disabled Template Revision + +If you decide that a disabled RKE template revision should be used to create new clusters, you can re-enable it. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Under **RKE1 configuration**, click **RKE Templates**. +1. Go to the template revision you want to re-enable. Then select **⋮ > Enable**. + +**Result:** The RKE template revision can be used to create a new cluster. + +### Setting a Template Revision as Default + +When end users create a cluster using an RKE template, they can choose which revision to create the cluster with. You can configure which revision is used by default. + +To set an RKE template revision as default, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE templates**. +1. Go to the RKE template revision that should be default and click the **⋮ > Set as Default**. + +**Result:** The RKE template revision will be used as the default option when clusters are created with the template. + +### Deleting a Template Revision + +You can delete all revisions of a template except for the default revision. + +To permanently delete a revision, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation menu, click **RKE1 Configuration > RKE templates**. +1. Go to the RKE template revision that should be deleted and click the **⋮ > Delete**. + +**Result:** The RKE template revision is deleted. + +### Upgrading a Cluster to Use a New Template Revision + +:::note + +This section assumes that you already have a cluster that [has an RKE template applied.](apply-templates.md) + +This section also assumes that you have [updated the template that the cluster is using](#updating-a-template) so that a new template revision is available. + +::: + +To upgrade a cluster to use a new template revision, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you want to upgrade and click **⋮ > Edit Config**. +1. In the **Cluster Options** section, click the dropdown menu for the template revision, then select the new template revision. +1. Click **Save**. + +**Result:** The cluster is upgraded to use the settings defined in the new template revision. + +### Exporting a Running Cluster to a New RKE Template and Revision + +You can save an existing cluster's settings as an RKE template. + +This exports the cluster's settings as a new RKE template, and also binds the cluster to that template. The result is that the cluster can only be changed if the [template is updated,](#updating-a-template) and the cluster is upgraded to [use a newer version of the template](#upgrading-a-cluster-to-use-a-new-template-revision). + +To convert an existing cluster to use an RKE template, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that will be converted to use an RKE template and **⋮ > Save as RKE Template**. +1. Enter a name for the RKE template in the form that appears, and click **Create**. + +**Results:** + +- A new RKE template is created. +- The cluster is converted to use the new template. +- New clusters can be [created from the new template and revision.](apply-templates.md#creating-a-cluster-from-an-rke-template) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md new file mode 100644 index 00000000000..67c6a50edd8 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md @@ -0,0 +1,18 @@ +--- +title: Overriding Template Settings +--- + + + + + +When a user creates an RKE template, each setting in the template has a switch in the Rancher UI that indicates if users can override the setting. This switch marks those settings as **Allow User Override**. + +After a cluster is created with a template, end users can't update any of the settings defined in the template unless the template owner marked them as **Allow User Override**. However, if the template is [updated to a new revision](manage-rke1-templates.md) that changes the settings or allows end users to change them, the cluster can be upgraded to a new revision of the template and the changes in the new revision will be applied to the cluster. + +When any parameter is set as **Allow User Override** on the RKE template, it means that end users have to fill out those fields during cluster creation and they can edit those settings afterward at any time. + +The **Allow User Override** model of the RKE template is useful for situations such as: + +- Administrators know that some settings will need the flexibility to be frequently updated over time +- End users will need to enter their own access keys or secret keys, for example, cloud credentials or credentials for backup snapshots \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md new file mode 100644 index 00000000000..b24c4879071 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md @@ -0,0 +1,223 @@ +--- +title: Configure Active Directory (AD) +--- + + + + + +If your organization uses Microsoft Active Directory as central user repository, you can configure Rancher to communicate with an Active Directory server to authenticate users. This allows Rancher admins to control access to clusters and projects based on users and groups managed externally in the Active Directory, while allowing end-users to authenticate with their AD credentials when logging in to the Rancher UI. + +Rancher uses LDAP to communicate with the Active Directory server. The authentication flow for Active Directory is therefore the same as for the [OpenLDAP authentication](../../../../pages-for-subheaders/configure-openldap.md) integration. + +:::note + +Before you start, please familiarise yourself with the concepts of [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +::: + +## Prerequisites + +You'll need to create or obtain from your AD administrator a new AD user to use as service account for Rancher. This user must have sufficient permissions to perform LDAP searches and read attributes of users and groups under your AD domain. + +Usually a (non-admin) **Domain User** account should be used for this purpose, as by default such user has read-only privileges for most objects in the domain partition. + +Note however, that in some locked-down Active Directory configurations this default behaviour may not apply. In such case you will need to ensure that the service account user has at least **Read** and **List Content** permissions granted either on the Base OU (enclosing users and groups) or globally for the domain. + +:::note Using TLS? + +- If the certificate used by the AD server is self-signed or not from a recognized certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +- Upon an upgrade to v2.6.0, authenticating via Rancher against an active directory using TLS can fail if the certificates on the AD server do not support SAN attributes. This is a check enabled by default in Go v1.15. + + - The error received is "Error creating SSL connection: LDAP Result Code 200 "Network Error": x509: certificate relies on legacy Common Name field, use SANs or temporarily enable Common Name matching with GODEBUG=x509ignoreCN=0". + + - To resolve the error, update or replace the certificates on the AD server with new ones that support the SAN attribute. Alternatively, this error can be ignored by setting `GODEBUG=x509ignoreCN=0` as an environment variable to Rancher server container. + +::: + +## Configuration Steps +### Open Active Directory Configuration + +1. Log into the Rancher UI using the initial local `admin` account. +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **ActiveDirectory**. The **Authentication Provider: ActiveDirectory** form will be displayed. +1. Fill out the form. For help, refer to the details on configuration options below. +1. Click **Enable**. + +### Configure Active Directory Server Settings + +In the section titled `1. Configure an Active Directory server`, complete the fields with the information specific to your Active Directory server. Please refer to the following table for detailed information on the required values for each parameter. + +:::note + +If you are unsure about the correct values to enter in the user/group Search Base field, please refer to [Identify Search Base and Schema using ldapsearch](#annex-identify-search-base-and-schema-using-ldapsearch). + +::: + +**Table 1: AD Server parameters** + +| Parameter | Description | +|:--|:--| +| Hostname | Specify the hostname or IP address of the AD server | +| Port | Specify the port at which the Active Directory server is listening for connections. Unencrypted LDAP normally uses the standard port of 389, while LDAPS uses port 636.| +| TLS | Check this box to enable LDAP over SSL/TLS (commonly known as LDAPS).| +| Server Connection Timeout | The duration in number of seconds that Rancher waits before considering the AD server unreachable. | +| Service Account Username | Enter the username of an AD account with read-only access to your domain partition (see [Prerequisites](#prerequisites)). The username can be entered in NetBIOS format (e.g. "DOMAIN\serviceaccount") or UPN format (e.g. "serviceaccount@domain.com"). | +| Service Account Password | The password for the service account. | +| Default Login Domain | When you configure this field with the NetBIOS name of your AD domain, usernames entered without a domain (e.g. "jdoe") will automatically be converted to a slashed, NetBIOS logon (e.g. "LOGIN_DOMAIN\jdoe") when binding to the AD server. If your users authenticate with the UPN (e.g. "jdoe@acme.com") as username then this field **must** be left empty. | +| User Search Base | The Distinguished Name of the node in your directory tree from which to start searching for user objects. All users must be descendents of this base DN. For example: "ou=people,dc=acme,dc=com".| +| Group Search Base | If your groups live under a different node than the one configured under `User Search Base` you will need to provide the Distinguished Name here. Otherwise leave it empty. For example: "ou=groups,dc=acme,dc=com".| + +--- + +### Configure User/Group Schema + +In the section titled `2. Customize Schema` you must provide Rancher with a correct mapping of user and group attributes corresponding to the schema used in your directory. + +Rancher uses LDAP queries to search for and retrieve information about users and groups within the Active Directory. The attribute mappings configured in this section are used to construct search filters and resolve group membership. It is therefore paramount that the provided settings reflect the reality of your AD domain. + +:::note + +If you are unfamiliar with the schema used in your Active Directory domain, please refer to [Identify Search Base and Schema using ldapsearch](#annex-identify-search-base-and-schema-using-ldapsearch) to determine the correct configuration values. + +::: + +#### User Schema + +The table below details the parameters for the user schema section configuration. + +**Table 2: User schema configuration parameters** + +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for user objects in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Username Attribute | The user attribute whose value is suitable as a display name. | +| Login Attribute | The attribute whose value matches the username part of credentials entered by your users when logging in to Rancher. If your users authenticate with their UPN (e.g. "jdoe@acme.com") as username then this field must normally be set to `userPrincipalName`. Otherwise for the old, NetBIOS-style logon names (e.g. "jdoe") it's usually `sAMAccountName`. | +| User Member Attribute | The attribute containing the groups that a user is a member of. | +| Search Attribute | When a user enters text to add users or groups in the UI, Rancher queries the AD server and attempts to match users by the attributes provided in this setting. Multiple attributes can be specified by separating them with the pipe ("\|") symbol. To match UPN usernames (e.g. jdoe@acme.com) you should usually set the value of this field to `userPrincipalName`. | +| Search Filter | This filter gets applied to the list of users that is searched when Rancher attempts to add users to a site access list or tries to add members to clusters or projects. For example, a user search filter could be (|(memberOf=CN=group1,CN=Users,DC=testad,DC=rancher,DC=io)(memberOf=CN=group2,CN=Users,DC=testad,DC=rancher,DC=io)). Note: If the search filter does not use [valid AD search syntax,](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax) the list of users will be empty. | +| User Enabled Attribute | The attribute containing an integer value representing a bitwise enumeration of user account flags. Rancher uses this to determine if a user account is disabled. You should normally leave this set to the AD standard `userAccountControl`. | +| Disabled Status Bitmask | This is the value of the `User Enabled Attribute` designating a disabled user account. You should normally leave this set to the default value of "2" as specified in the Microsoft Active Directory schema (see [here](https://docs.microsoft.com/en-us/windows/desktop/adschema/a-useraccountcontrol#remarks)). | + +--- + +#### Group Schema + +The table below details the parameters for the group schema configuration. + +**Table 3: Group schema configuration parameters** + +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for group objects in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Name Attribute | The group attribute whose value is suitable for a display name. | +| Group Member User Attribute | The name of the **user attribute** whose format matches the group members in the `Group Member Mapping Attribute`. | +| Group Member Mapping Attribute | The name of the group attribute containing the members of a group. | +| Search Attribute | Attribute used to construct search filters when adding groups to clusters or projects. See description of user schema `Search Attribute`. | +| Search Filter | This filter gets applied to the list of groups that is searched when Rancher attempts to add groups to a site access list or tries to add groups to clusters or projects. For example, a group search filter could be (|(cn=group1)(cn=group2)). Note: If the search filter does not use [valid AD search syntax,](https://docs.microsoft.com/en-us/windows/win32/adsi/search-filter-syntax) the list of groups will be empty. | +| Group DN Attribute | The name of the group attribute whose format matches the values in the user attribute describing a the user's memberships. See `User Member Attribute`. | +| Nested Group Membership | This settings defines whether Rancher should resolve nested group memberships. Use only if your organization makes use of these nested memberships (i.e., you have groups that contain other groups as members. We advise avoiding nested groups when possible to avoid potential performance issues when there is a large amount of nested memberships). | + +--- + +### Test Authentication + +Once you have completed the configuration, proceed by testing the connection to the AD server **using your AD admin account**. If the test is successful, authentication with the configured Active Directory will be enabled implicitly with the account you test with set as admin. + +:::note + +The AD user pertaining to the credentials entered in this step will be mapped to the local principal account and assigned administrator privileges in Rancher. You should therefore make a conscious decision on which AD account you use to perform this step. + +::: + +1. Enter the **username** and **password** for the AD account that should be mapped to the local principal account. +2. Click **Authenticate with Active Directory** to finalise the setup. + +**Result:** + +- Active Directory authentication has been enabled. +- You have been signed into Rancher as administrator using the provided AD credentials. + +:::note + +You will still be able to login using the locally configured `admin` account and password in case of a disruption of LDAP services. + +::: + +## Annex: Identify Search Base and Schema using ldapsearch + +In order to successfully configure AD authentication it is crucial that you provide the correct configuration pertaining to the hierarchy and schema of your AD server. + +The [`ldapsearch`](https://manpages.ubuntu.com/manpages/kinetic/en/man1/ldapsearch.1.html) tool allows you to query your AD server to learn about the schema used for user and group objects. + +For the purpose of the example commands provided below we will assume: + +- The Active Directory server has a hostname of `ad.acme.com` +- The server is listening for unencrypted connections on port `389` +- The Active Directory domain is `acme` +- You have a valid AD account with the username `jdoe` and password `secret` + +### Identify Search Base + +First we will use `ldapsearch` to identify the Distinguished Name (DN) of the parent node(s) for users and groups: + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "dc=acme,dc=com" -s sub "sAMAccountName=jdoe" +``` + +This command performs an LDAP search with the search base set to the domain root (`-b "dc=acme,dc=com"`) and a filter targeting the user account (`sAMAccountNam=jdoe`), returning the attributes for said user: + +![](/img/ldapsearch-user.png) + +Since in this case the user's DN is `CN=John Doe,CN=Users,DC=acme,DC=com` [5], we should configure the **User Search Base** with the parent node DN `CN=Users,DC=acme,DC=com`. + +Similarly, based on the DN of the group referenced in the **memberOf** attribute [4], the correct value for the **Group Search Base** would be the parent node of that value, i.e., `OU=Groups,DC=acme,DC=com`. + +### Identify User Schema + +The output of the above `ldapsearch` query also allows to determine the correct values to use in the user schema configuration: + +- `Object Class`: **person** [1] +- `Username Attribute`: **name** [2] +- `Login Attribute`: **sAMAccountName** [3] +- `User Member Attribute`: **memberOf** [4] + +:::note + +If the AD users in our organization were to authenticate with their UPN (e.g. jdoe@acme.com) instead of the short logon name, then we would have to set the `Login Attribute` to **userPrincipalName** instead. + +::: + +We'll also set the `Search Attribute` parameter to **sAMAccountName|name**. That way users can be added to clusters/projects in the Rancher UI either by entering their username or full name. + +### Identify Group Schema + +Next, we'll query one of the groups associated with this user, in this case `CN=examplegroup,OU=Groups,DC=acme,DC=com`: + +``` +$ ldapsearch -x -D "acme\jdoe" -w "secret" -p 389 \ +-h ad.acme.com -b "ou=groups,dc=acme,dc=com" \ +-s sub "CN=examplegroup" +``` + +This command will inform us on the attributes used for group objects: + +![](/img/ldapsearch-group.png) + +Again, this allows us to determine the correct values to enter in the group schema configuration: + +- `Object Class`: **group** [1] +- `Name Attribute`: **name** [2] +- `Group Member Mapping Attribute`: **member** [3] +- `Search Attribute`: **sAMAccountName** [4] + +Looking at the value of the **member** attribute, we can see that it contains the DN of the referenced user. This corresponds to the **distinguishedName** attribute in our user object. Accordingly will have to set the value of the `Group Member User Attribute` parameter to this attribute. + +In the same way, we can observe that the value in the **memberOf** attribute in the user object corresponds to the **distinguishedName** [5] of the group. We therefore need to set the value for the `Group DN Attribute` parameter to this attribute. + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the Active Directory server, first double-check the credentials entered for the service account as well as the search base configuration. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md new file mode 100644 index 00000000000..d6f2884eab4 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md @@ -0,0 +1,330 @@ +--- +title: Configure Azure AD +--- + + + + + +## Microsoft Graph API + +Microsoft Graph API is now the flow through which you will set up Azure AD. The below sections will assist [new users](#new-user-setup) in configuring Azure AD with a new instance as well as assist existing Azure app owners in [migrating to the new flow](#migrating-from-azure-ad-graph-api-to-microsoft-graph-api). + +The Microsoft Graph API flow in Rancher is constantly evolving. We recommend that you use the latest patched version of 2.7, as it is still in active development and will continue to receive new features and improvements. + +### New User Setup + +If you have an instance of Active Directory (AD) hosted in Azure, you can configure Rancher to allow your users to log in using their AD accounts. Configuration of Azure AD external authentication requires you to make configurations in both Azure and Rancher. + +:::note Notes + +- Azure AD integration only supports Service Provider initiated logins. +- Most of this procedure takes place from the [Microsoft Azure Portal](https://portal.azure.com/). + +::: + +#### Azure Active Directory Configuration Outline + +Configuring Rancher to allow your users to authenticate with their Azure AD accounts involves multiple procedures. Review the outline below before getting started. + +:::tip + +Before you start, open two browser tabs: one for Rancher, and one for the Azure portal. This will help with copying and pasting configuration values from the portal to Rancher. + +::: + + +#### 1. Register Rancher with Azure + +Before enabling Azure AD within Rancher, you must register Rancher with Azure. + +1. Log in to [Microsoft Azure](https://portal.azure.com/) as an administrative user. Configuration in future steps requires administrative access rights. + +1. Use search to open the **App registrations** service. + +1. Click **New registration** and complete the form. + + ![New App Registration](/img/new-app-registration.png) + + 1. Enter a **Name** (something like `Rancher`). + + + 1. From **Supported account types**, select "Accounts in this organizational directory only (AzureADTest only - Single tenant)" This corresponds to the legacy app registration options. + + :::note + + In the updated Azure portal, Redirect URIs are synonymous with Reply URLs. In order to use Azure AD with Rancher, you must whitelist Rancher with Azure (previously done through Reply URLs). Therefore, you must ensure to fill in the Redirect URI with your Rancher server URL, to include the verification path as listed below. + + ::: + + 1. In the [**Redirect URI**](https://docs.microsoft.com/en-us/azure/active-directory/develop/reply-url) section, make sure **Web** is selected from the dropdown and enter the URL of your Rancher Server in the text box next to the dropdown. This Rancher server URL should be appended with the verification path: `/verify-auth-azure`. + + :::tip + + You can find your personalized Azure Redirect URI (reply URL) in Rancher on the Azure AD Authentication page (Global View > Authentication > Web). + + ::: + + 1. Click **Register**. + +:::note + +It can take up to five minutes for this change to take affect, so don't be alarmed if you can't authenticate immediately after Azure AD configuration. + +::: + +#### 2. Create a new client secret + +From the Azure portal, create a client secret. Rancher will use this key to authenticate with Azure AD. + +1. Use search to open **App registrations** services. Then open the entry for Rancher that you created in the last procedure. + + ![Open Rancher Registration](/img/open-rancher-app-reg.png) + +1. From the navigation pane, click **Certificates & secrets**. + +1. Click **New client secret**. + ![Create new client secret](/img/new-client-secret.png) +1. Enter a **Description** (something like `Rancher`). +1. Select the duration from the options under **Expires**. This drop-down menu sets the expiration date for the key. Shorter durations are more secure, but require you to create a new key more frequently. +Note that users won't be able to log in to Rancher if it detects that the application secret has expired. To avoid this problem, rotate the secret in Azure and update it in Rancher before it expires. +1. Click **Add** (you don't need to enter a value—it will automatically populate after you save). + + +1. You'll enter this key into the Rancher UI later as your **Application Secret**. Since you won't be able to access the key value again within the Azure UI, keep this window open for the rest of the setup process. + +#### 3. Set Required Permissions for Rancher + +Next, set API permissions for Rancher within Azure. + +:::caution + +Ensure that you set Application permissions, and *not* Delegated permissions. Otherwise, you won't be able to login to Azure AD. + +::: + +1. From the navigation pane on, select **API permissions**. + +1. Click **Add a permission**. + +1. From the Microsoft Graph API, select the following **Application Permissions**: `Directory.Read.All` + + ![Select API Permissions](/img/api-permissions.png) + +1. Return to **API permissions** in the nav bar. From there, click **Grant admin consent**. Then click **Yes**. The app's permissions should look like the following: + +![Open Required Permissions](/img/select-req-permissions.png) + +:::note + +Rancher doesn't validate the permissions you grant to the app in Azure. You're free to try any permissions you want, as long as they allow Rancher to work with AD users and groups. + +Specifically, Rancher needs permissions that allow the following actions: +- Get a user. +- List all users. +- List groups of which a given user is a member. +- Get a group. +- List all groups. + +Rancher performs these actions either to log in a user or to run a user/group search. Keep in mind that the permissions must be of type `Application`. + +Here are a few examples of permission combinations that satisfy Rancher's needs: +- `Directory.Read.All` +- `User.Read.All` and `GroupMember.Read.All` +- `User.Read.All` and `Group.Read.All` + +::: + +#### 4. Copy Azure Application Data + +![Application ID](/img/app-configuration.png) + +1. Obtain your Rancher **Tenant ID**. + + 1. Use search to open **App registrations**. + + 1. Find the entry you created for Rancher. + + 1. Copy the **Directory ID** and paste it into Rancher as your **Tenant ID**. + +1. Obtain your Rancher **Application (Client) ID**. + + 1. If you aren't already there, use search to open **App registrations**. + + 1. In **Overview**, find the entry you created for Rancher. + + 1. Copy the **Application (Client) ID** and paste it into Rancher as your **Application ID**. + +1. In most cases, your endpoint options will either be [Standard](#global) or [China](#china). For either of these options, you only need to enter the **Tenant ID**, **Application ID**, and **Application Secret**. + +![Standard Endpoint Options](/img/tenant-application-id-secret.png) + +**For Custom Endpoints:** + +:::caution + +Custom Endpoints are not tested or fully supported by Rancher. + +::: + +You'll also need to manually enter the Graph, Token, and Auth Endpoints. + +- From App registrations, click Endpoints: + +![Click Endpoints](/img/endpoints.png) + +- The following endpoints will be your Rancher endpoint values. Make sure to use the v1 version of these endpoints: + - **Microsoft Graph API endpoint** (Graph Endpoint) + - **OAuth 2.0 token endpoint (v1)** (Token Endpoint) + - **OAuth 2.0 authorization endpoint (v1)** (Auth Endpoint) + +#### 5. Configure Azure AD in Rancher + +To complete configuration, enter information about your AD instance in the Rancher UI. + +1. Log into Rancher. + +1. In the top left corner, click **☰ > Users & Authentication**. + +1. In the left navigation menu, click **Auth Provider**. + +1. Click **AzureAD**. + +1. Complete the **Configure Azure AD Account** form using the information you copied while completing [Copy Azure Application Data](#4-copy-azure-application-data). + + :::caution + + The Azure AD account will be granted administrator privileges, since its details will be mapped to the Rancher local principal account. Make sure that this level of privilege is appropriate before you continue. + + ::: + + **For Standard or China Endpoints:** + + The following table maps the values you copied in the Azure portal to the fields in Rancher: + + | Rancher Field | Azure Value | + | ------------------ | ------------------------------------- | + | Tenant ID | Directory ID | + | Application ID | Application ID | + | Application Secret | Key Value | + | Endpoint | https://login.microsoftonline.com/ | + + **For Custom Endpoints:** + + The following table maps your custom config values to Rancher fields: + + | Rancher Field | Azure Value | + | ------------------ | ------------------------------------- | + | Graph Endpoint | Microsoft Graph API Endpoint | + | Token Endpoint | OAuth 2.0 Token Endpoint | + | Auth Endpoint | OAuth 2.0 Authorization Endpoint | + + **Important:** When entering the Graph Endpoint in a custom config, remove the tenant ID from the URL: + + https://graph.microsoft.com/abb5adde-bee8-4821-8b03-e63efdc7701c + +1. Click **Enable**. + +**Result:** Azure Active Directory authentication is configured. + + +### Migrating from Azure AD Graph API to Microsoft Graph API + +Since the [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) is deprecated and slated to retire in June 2023, admins should update their Azure AD App to use the [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api) in Rancher. +This needs to be done well in advance of the endpoint being retired. +If Rancher is still configured to use the Azure AD Graph API when it is retired, users may not be able to log into Rancher using Azure AD. + +#### Updating Endpoints in the Rancher UI + +:::caution + +Admins should create a [Rancher backup](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) before they commit to the endpoint migration described below. + +::: + +1. [Update](#3-set-required-permissions-for-rancher) the permissions of your Azure AD app registration. This is critical. + +1. Log into Rancher. + +1. In the Rancher UI homepage, make note of the banner at the top of screen that advises users to update their Azure AD authentication. Click on the link provided to do so. + + ![Rancher UI Banner](/img/rancher-ui-azure-update.png) + +1. To complete the move to the new Microsoft Graph API, click **Update Endpoint**. + + **Note:** Ensure that your Azure app has a [new set of permissions](#3-set-required-permissions-for-rancher) before starting the update. + + ![Update Endpoint](/img/rancher-button-to-update.png) + +1. When you receive the pop-up warning message, click **Update**. + + ![Azure Update Pop-up](/img/azure-update-popup.png) + +1. Refer to the [tables](#global) below for the full list of endpoint changes that Rancher performs. Admins do not need to do this manually. + +#### Air-Gapped Environments + +In air-gapped environments, admins should ensure that their endpoints are [whitelisted](#3.2) since the Graph Endpoint URL is changing. + +#### Rolling Back the Migration + +If you need to roll back your migration, please note the following: + +1. Admins are encouraged to use the proper restore process if they want to go back. Please see [backup docs](../../../new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md), [restore docs](../../../new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md), and [examples](../../../../reference-guides/backup-restore-configuration/examples.md) for reference. + +1. Azure app owners who want to rotate the Application Secret will need to also rotate it in Rancher as Rancher does not automatically update the Application Secret when it is changed in Azure. In Rancher, note that it is stored in a Kubernetes secret called `azureadconfig-applicationsecret` which is in the `cattle-global-data` namespace. + +:::caution + +If you upgrade to Rancher v2.7.0+ with an existing Azure AD setup, and choose to disable the auth provider, you won't be able to restore the previous setup. You also won't be able to set up Azure AD using the old flow. You'll need to re-register with the new auth flow. Since Rancher now uses the Graph API, users need set up the [proper permissions in the Azure portal](#3-set-required-permissions-for-rancher). + +::: + + +#### Global: + +Rancher Field | Deprecated Endpoints +---------------- | ------------------------------------------------------------- +Auth Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/authorize +Endpoint | https://login.microsoftonline.com/ +Graph Endpoint | https://graph.windows.net/ +Token Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/token + +Rancher Field | New Endpoints +---------------- | ------------------------------------------------------------------ +Auth Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/authorize +Endpoint | https://login.microsoftonline.com/ +Graph Endpoint | https://graph.microsoft.com +Token Endpoint | https://login.microsoftonline.com/{tenantID}/oauth2/v2.0/token + +#### China: + +Rancher Field | Deprecated Endpoints +---------------- | ---------------------------------------------------------- +Auth Endpoint | https://login.chinacloudapi.cn/{tenantID}/oauth2/authorize +Endpoint | https://login.chinacloudapi.cn/ +Graph Endpoint | https://graph.chinacloudapi.cn/ +Token Endpoint | https://login.chinacloudapi.cn/{tenantID}/oauth2/token + +Rancher Field | New Endpoints +---------------- | ------------------------------------------------------------------------- +Auth Endpoint | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/authorize +Endpoint | https://login.partner.microsoftonline.cn/ +Graph Endpoint | https://microsoftgraph.chinacloudapi.cn +Token Endpoint | https://login.partner.microsoftonline.cn/{tenantID}/oauth2/v2.0/token + + +## Deprecated Azure AD Graph API + +>**Important:** +> +>- The [Azure AD Graph API](https://docs.microsoft.com/en-us/graph/migrate-azure-ad-graph-overview) is deprecated and will be retired by Microsoft at any time after June 30, 2023, without advance notice. We will update our docs to advise the community when it is retired. Rancher now uses the [Microsoft Graph API](https://docs.microsoft.com/en-us/graph/use-the-api) as the new flow to set up Azure AD as the external auth provider. +> +> +>- If you're a new user, or wish to migrate, refer to the new flow instructions for Rancher v2.7.0+. +> +> +>- If you don't wish to upgrade to v2.7.0+ after the Azure AD Graph API is retired, you'll need to either: + - Use the built-in Rancher auth or + - Use another third-party auth system and set that up in Rancher. Please see the [authentication docs](../../../../pages-for-subheaders/authentication-config.md) to learn how to configure other open authentication providers. + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md new file mode 100644 index 00000000000..0fe6995d4ae --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md @@ -0,0 +1,64 @@ +--- +title: Configure FreeIPA +--- + + + + + +If your organization uses FreeIPA for user authentication, you can configure Rancher to allow your users to login using their FreeIPA credentials. + +:::note Prerequisites: + +- You must have a [FreeIPA Server](https://www.freeipa.org/) configured. +- Create a service account in FreeIPA with `read-only` access. Rancher uses this account to verify group membership when a user makes a request using an API key. +- Read [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +::: + +1. Sign into Rancher using a local user assigned the `administrator` role (i.e., the _local principal_). +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **FreeIPA**. +1. Complete the **Configure an FreeIPA server** form. + + You may need to log in to your domain controller to find the information requested in the form. + + :::note Using TLS? + + If the certificate is self-signed or not from a recognized certificate authority, make sure you provide the complete chain. That chain is needed to verify the server's certificate. + + ::: + + :::note User Search Base vs. Group Search Base + + Search base allows Rancher to search for users and groups that are in your FreeIPA. These fields are only for search bases and not for search filters. + + * If your users and groups are in the same search base, complete only the User Search Base. + * If your groups are in a different search base, you can optionally complete the Group Search Base. This field is dedicated to searching groups, but is not required. + + ::: + +1. If your FreeIPA deviates from the standard AD schema, complete the **Customize Schema** form to match it. Otherwise, skip this step. + + :::note Search Attribute + + The Search Attribute field defaults with three specific values: `uid|sn|givenName`. After FreeIPA is configured, when a user enters text to add users or groups, Rancher automatically queries the FreeIPA server and attempts to match fields by user id, last name, or first name. Rancher specifically searches for users/groups that begin with the text entered in the search field. + + The default field value `uid|sn|givenName`, but you can configure this field to a subset of these fields. The pipe (`|`) between the fields separates these fields. + + * `uid`: User ID + * `sn`: Last Name + * `givenName`: First Name + + With this search attribute, Rancher creates search filters for users and groups, but you *cannot* add your own search filters in this field. + + ::: + +1. Enter your FreeIPA username and password in **Authenticate with FreeIPA** to confirm that Rancher is configured to use FreeIPA authentication. +1. Click **Enable**. + +**Result:** + +- FreeIPA authentication is configured. +- You are signed into Rancher with your FreeIPA account (i.e., the _external principal_). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md new file mode 100644 index 00000000000..960d968de8d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md @@ -0,0 +1,60 @@ +--- +title: Configure GitHub +--- + + + + + +In environments using GitHub, you can configure Rancher to allow sign on using GitHub credentials. + +:::note Prerequisites: + +Read [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +::: + +1. Sign into Rancher using a local user assigned the `administrator` role (i.e., the _local principal_). +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **GitHub**. +1. Follow the directions displayed to set up a GitHub Application. Rancher redirects you to GitHub to complete registration. + + :::note What's an Authorization Callback URL? + + The Authorization Callback URL is the URL where users go to begin using your application (i.e. the splash screen). + + When you use external authentication, authentication does not actually take place in your application. Instead, authentication takes place externally (in this case, GitHub). After this external authentication completes successfully, the Authorization Callback URL is the location where the user re-enters your application. + + ::: + +1. From GitHub, copy the **Client ID** and **Client Secret**. Paste them into Rancher. + + :::note Where do I find the Client ID and Client Secret? + + From GitHub, select Settings > Developer Settings > OAuth Apps. The Client ID and Client Secret are displayed prominently. + + ::: + +1. Click **Authenticate with GitHub**. + +1. Use the **Site Access** options to configure the scope of user authorization. + + - **Allow any valid Users** + + _Any_ GitHub user can access Rancher. We generally discourage use of this setting! + + - **Allow members of Clusters, Projects, plus Authorized Users and Organizations** + + Any GitHub user or group added as a **Cluster Member** or **Project Member** can log in to Rancher. Additionally, any GitHub user or group you add to the **Authorized Users and Organizations** list may log in to Rancher. + + - **Restrict access to only Authorized Users and Organizations** + + Only GitHub users or groups added to the Authorized Users and Organizations can log in to Rancher. +
+1. Click **Enable**. + +**Result:** + +- GitHub authentication is configured. +- You are signed into Rancher with your GitHub account (i.e., the _external principal_). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md new file mode 100644 index 00000000000..23b5084b594 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md @@ -0,0 +1,115 @@ +--- +title: Configure Google OAuth +--- + + + + + +If your organization uses G Suite for user authentication, you can configure Rancher to allow your users to log in using their G Suite credentials. + +Only admins of the G Suite domain have access to the Admin SDK. Therefore, only G Suite admins can configure Google OAuth for Rancher. + +Within Rancher, only administrators or users with the **Manage Authentication** [global role](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) can configure authentication. + +## Prerequisites + +- You must have a [G Suite admin account](https://admin.google.com) configured. +- G Suite requires a [top private domain FQDN](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains) as an authorized domain. One way to get an FQDN is by creating an A-record in Route53 for your Rancher server. You do not need to update your Rancher Server URL setting with that record, because there could be clusters using that URL. +- You must have the Admin SDK API enabled for your G Suite domain. You can enable it using the steps on [this page.](https://support.google.com/a/answer/60757?hl=en) + +After the Admin SDK API is enabled, your G Suite domain's API screen should look like this: +![Enable Admin APIs](/img/Google-Enable-APIs-Screen.png) + +## Setting up G Suite for OAuth with Rancher + +Before you can set up Google OAuth in Rancher, you need to log in to your G Suite account and do the following: + +1. [Add Rancher as an authorized domain in G Suite](#1-adding-rancher-as-an-authorized-domain) +1. [Generate OAuth2 credentials for the Rancher server](#2-creating-oauth2-credentials-for-the-rancher-server) +1. [Create service account credentials for the Rancher server](#3-creating-service-account-credentials) +1. [Register the service account key as an OAuth Client](#4-register-the-service-account-key-as-an-oauth-client) + +### 1. Adding Rancher as an Authorized Domain + +1. Click [here](https://console.developers.google.com/apis/credentials) to go to credentials page of your Google domain. +1. Select your project and click **OAuth consent screen**. +![OAuth Consent Screen](/img/Google-OAuth-consent-screen-tab.png) +1. Go to **Authorized Domains** and enter the top private domain of your Rancher server URL in the list. The top private domain is the rightmost superdomain. So for example, www.foo.co.uk a top private domain of foo.co.uk. For more information on top-level domains, refer to [this article.](https://github.com/google/guava/wiki/InternetDomainNameExplained#public-suffixes-and-private-domains) +1. Go to **Scopes for Google APIs** and make sure **email,** **profile** and **openid** are enabled. + +**Result:** Rancher has been added as an authorized domain for the Admin SDK API. + +### 2. Creating OAuth2 Credentials for the Rancher Server + +1. Go to the Google API console, select your project, and go to the [credentials page.](https://console.developers.google.com/apis/credentials) +![Credentials](/img/Google-Credentials-tab.png) +1. On the **Create Credentials** dropdown, select **OAuth client ID**. +1. Click **Web application**. +1. Provide a name. +1. Fill out the **Authorized JavaScript origins** and **Authorized redirect URIs**. Note: The Rancher UI page for setting up Google OAuth (available from the Global view under **Security > Authentication > Google**) provides you the exact links to enter for this step. + - Under **Authorized JavaScript origins,** enter your Rancher server URL. + - Under **Authorized redirect URIs,** enter your Rancher server URL appended with the path `verify-auth`. For example, if your URI is `https://rancherServer`, you will enter `https://rancherServer/verify-auth`. +1. Click on **Create**. +1. After the credential is created, you will see a screen with a list of your credentials. Choose the credential you just created, and in that row on rightmost side, click **Download JSON**. Save the file so that you can provide these credentials to Rancher. + +**Result:** Your OAuth credentials have been successfully created. + +### 3. Creating Service Account Credentials +Since the Google Admin SDK is available only to admins, regular users cannot use it to retrieve profiles of other users or their groups. Regular users cannot even retrieve their own groups. + +Since Rancher provides group-based membership access, we require the users to be able to get their own groups, and look up other users and groups when needed. + +As a workaround to get this capability, G Suite recommends creating a service account and delegating authority of your G Suite domain to that service account. + +This section describes how to: + +- Create a service account +- Create a key for the service account and download the credentials as JSON + +1. Click [here](https://console.developers.google.com/iam-admin/serviceaccounts) and select your project for which you generated OAuth credentials. +1. Click on **Create Service Account**. +1. Enter a name and click **Create**. +![Service account creation Step 1](/img/Google-svc-acc-step1.png) +1. Don't provide any roles on the **Service account permissions** page and click **Continue** +![Service account creation Step 2](/img/Google-svc-acc-step2.png) +1. Click on **Create Key** and select the JSON option. Download the JSON file and save it so that you can provide it as the service account credentials to Rancher. +![Service account creation Step 3](/img/Google-svc-acc-step3-key-creation.png) + +**Result:** Your service account is created. + +### 4. Register the Service Account Key as an OAuth Client + +You will need to grant some permissions to the service account you created in the last step. Rancher requires you to grant only read-only permissions for users and groups. + +Using the Unique ID of the service account key, register it as an Oauth Client using the following steps: + +1. Get the Unique ID of the key you just created. If it's not displayed in the list of keys right next to the one you created, you will have to enable it. To enable it, click **Unique ID** and click **OK**. This will add a **Unique ID** column to the list of service account keys. Save the one listed for the service account you created. NOTE: This is a numeric key, not to be confused with the alphanumeric field **Key ID**. + + ![Service account Unique ID](/img/Google-Select-UniqueID-column.png) +1. Go to the [**Domain-wide Delegation** page.](https://admin.google.com/ac/owl/domainwidedelegation) +1. Add the Unique ID obtained in the previous step in the **Client Name** field. +1. In the **One or More API Scopes** field, add the following scopes: + ``` + openid,profile,email,https://www.googleapis.com/auth/admin.directory.user.readonly,https://www.googleapis.com/auth/admin.directory.group.readonly + ``` +1. Click **Authorize**. + +**Result:** The service account is registered as an OAuth client in your G Suite account. + +## Configuring Google OAuth in Rancher + +1. Sign into Rancher using a local user assigned the [administrator](../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) role. This user is also called the local principal. +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Google**. The instructions in the UI cover the steps to set up authentication with Google OAuth. + 1. Admin Email: Provide the email of an administrator account from your GSuite setup. In order to perform user and group lookups, google apis require an administrator's email in conjunction with the service account key. + 1. Domain: Provide the domain on which you have configured GSuite. Provide the exact domain and not any aliases. + 1. Nested Group Membership: Check this box to enable nested group memberships. Rancher admins can disable this at any time after configuring auth. + - **Step One** is about adding Rancher as an authorized domain, which we already covered in [this section.](#1-adding-rancher-as-an-authorized-domain) + - For **Step Two,** provide the OAuth credentials JSON that you downloaded after completing [this section.](#2-creating-oauth2-credentials-for-the-rancher-server) You can upload the file or paste the contents into the **OAuth Credentials** field. + - For **Step Three,** provide the service account credentials JSON that downloaded at the end of [this section.](#3-creating-service-account-credentials) The credentials will only work if you successfully [registered the service account key](#4-register-the-service-account-key-as-an-oauth-client) as an OAuth client in your G Suite account. +1. Click **Authenticate with Google**. +1. Click **Enable**. + +**Result:** Google authentication is successfully configured. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md new file mode 100644 index 00000000000..76787538bd0 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md @@ -0,0 +1,149 @@ +--- +title: Configure Keycloak (OIDC) +description: Create a Keycloak OpenID Connect (OIDC) client and configure Rancher to work with Keycloak. By the end your users will be able to sign into Rancher using their Keycloak logins +--- + + + + + +If your organization uses [Keycloak Identity Provider (IdP)](https://www.keycloak.org) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. Rancher supports integration with Keycloak using the OpenID Connect (OIDC) protocol and the SAML protocol. Both implementations are functionally equivalent when used with Rancher. This page describes the process to configure Rancher to work with Keycloak using the OIDC protocol. + +If you prefer to use Keycloak with the SAML protocol instead, refer to [this page](configure-keycloak-saml.md). + +If you have an existing configuration using the SAML protocol and want to switch to the OIDC protocol, refer to [this section](#migrating-from-saml-to-oidc). + +## Prerequisites + +- On Rancher, Keycloak (SAML) is disabled. +- You must have a [Keycloak IdP Server](https://www.keycloak.org/docs/latest/server_installation/) configured. +- In Keycloak, create a [new OIDC client](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients), with the settings below. See the [Keycloak documentation](https://www.keycloak.org/docs/latest/server_admin/#oidc-clients) for help. + + Setting | Value + ------------|------------ + `Client ID` | <CLIENT_ID> (e.g. `rancher`) + `Name` | <CLIENT_NAME> (e.g. `rancher`) + `Client Protocol` | `openid-connect` + `Access Type` | `confidential` + `Valid Redirect URI` | `https://yourRancherHostURL/verify-auth` + +- In the new OIDC client, create [Mappers](https://www.keycloak.org/docs/latest/server_admin/#_protocol-mappers) to expose the users fields. + - Create a new "Groups Mapper" with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Groups Mapper` + `Mapper Type` | `Group Membership` + `Token Claim Name` | `groups` + `Add to ID token` | `OFF` + `Add to access token` | `OFF` + `Add to user info` | `ON` + + - Create a new "Client Audience" with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Client Audience` + `Mapper Type` | `Audience` + `Included Client Audience` | <CLIENT_NAME> + `Add to access token` | `ON` + + - Create a new "Groups Path" with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Group Path` + `Mapper Type` | `Group Membership` + `Token Claim Name` | `full_group_path` + `Full group path` | `ON` + `Add to user info` | `ON` + +## Configuring Keycloak in Rancher + +1. In the Rancher UI, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. Select **Keycloak (OIDC)**. +1. Complete the **Configure a Keycloak OIDC account** form. For help with filling the form, see the [configuration reference](#configuration-reference). +1. After you complete the **Configure a Keycloak OIDC account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Keycloak IdP to validate your Rancher Keycloak configuration. + + :::note + + You may need to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with Keycloak using the OIDC protocol. Your users can now sign into Rancher using their Keycloak logins. + +## Configuration Reference + +| Field | Description | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Client ID | The `Client ID` of your Keycloak client. | +| Client Secret | The generated `Secret` of your Keycloak client. In the Keycloak console, select **Clients**, select the client you created, select the **Credentials** tab and copy the value of the `Secret` field. | +| Private Key / Certificate | A key/certificate pair to create a secure shell between Rancher and your IdP. Required if HTTPS/SSL is enabled on your Keycloak server. | +| Endpoints | Choose whether to use the generated values for the `Rancher URL`, `Issue`, and `Auth Endpoint` fields or to provide manual overrides if incorrect. | +| Keycloak URL | The URL for your Keycloak server. | +| Keycloak Realm | The name of the realm in which the Keycloak client was created in. | +| Rancher URL | The URL for your Rancher Server. | +| Issuer | The URL of your IdP. | +| Auth Endpoint | The URL where users are redirected to authenticate. | + +## Migrating from SAML to OIDC + +This section describes the process to transition from using Rancher with Keycloak (SAML) to Keycloak (OIDC). + +### Reconfigure Keycloak + +1. Change the existing client to use the OIDC protocol. In the Keycloak console, select **Clients**, select the SAML client to migrate, select the **Settings** tab, change `Client Protocol` from `saml` to `openid-connect`, and click **Save** + +1. Verify the `Valid Redirect URIs` are still valid. + +1. Select the **Mappers** tab and create a new Mapper with the settings below. + + Setting | Value + ------------|------------ + `Name` | `Groups Mapper` + `Mapper Type` | `Group Membership` + `Token Claim Name` | `groups` + `Add to ID token` | `ON` + `Add to access token` | `ON` + `Add to user info` | `ON` + +### Reconfigure Rancher + +Before configuring Rancher to use Keycloak (OIDC), Keycloak (SAML) must be first disabled. + +1. In the Rancher UI, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. Select **Keycloak (SAML)**. +1. Click **Disable**. + +Configure Rancher to use Keycloak (OIDC) by following the steps in [this section](#configuring-keycloak-in-rancher). + +:::note + +After configuration is completed, Rancher user permissions will need to be reapplied as they are not automatically migrated. + +::: + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the Keycloak server, first double-check the configuration options of your OIDC client. You may also inspect the Rancher logs to help pinpoint what's causing issues. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. + +All Keycloak related log entries will be prepended with either `[generic oidc]` or `[keycloak oidc]`. + +### You are not redirected to Keycloak + +When you fill the **Configure a Keycloak OIDC account** form and click on **Enable**, you are not redirected to your IdP. + + * Verify your Keycloak client configuration. + +### The generated `Issuer` and `Auth Endpoint` are incorrect + + * On the **Configure a Keycloak OIDC account** form, change **Endpoints** to `Specify (advanced)` and override the `Issuer` and `Auth Endpoint` values. To find the values, go to the Keycloak console and select **Realm Settings**, select the **General** tab, and click **OpenID Endpoint Configuration**. The JSON output will display values for `issuer` and `authorization_endpoint`. + +### Keycloak Error: "Invalid grant_type" + + * In some cases, this error message may be misleading and is actually caused by setting the `Valid Redirect URI` incorrectly. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md new file mode 100644 index 00000000000..21dfc75e8d8 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md @@ -0,0 +1,194 @@ +--- +title: Configure Keycloak (SAML) +description: Create a Keycloak SAML client and configure Rancher to work with Keycloak. By the end your users will be able to sign into Rancher using their Keycloak logins +--- + + + + + +If your organization uses Keycloak Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +## Prerequisites + +- You must have a [Keycloak IdP Server](https://www.keycloak.org/docs/latest/server_installation/) configured. +- In Keycloak, create a [new SAML client](https://www.keycloak.org/docs/latest/server_admin/#saml-clients), with the settings below. See the [Keycloak documentation](https://www.keycloak.org/docs/latest/server_admin/#saml-clients) for help. + + Setting | Value + ------------|------------ + `Sign Documents` | `ON` 1 + `Sign Assertions` | `ON` 1 + All other `ON/OFF` Settings | `OFF` + `Client ID` | Either `https://yourRancherHostURL/v1-saml/keycloak/saml/metadata` or the value configured in the `Entry ID Field` of the Rancher Keycloak configuration2 + `Client Name` | (e.g. `rancher`) + `Client Protocol` | `SAML` + `Valid Redirect URI` | `https://yourRancherHostURL/v1-saml/keycloak/saml/acs` + + >1: Optionally, you can enable either one or both of these settings. + >2: Rancher SAML metadata won't be generated until a SAML provider is configured and saved. + + ![](/img/keycloak/keycloak-saml-client-configuration.png) + +- In the new SAML client, create Mappers to expose the users fields + - Add all "Builtin Protocol Mappers" + ![](/img/keycloak/keycloak-saml-client-builtin-mappers.png) + - Create a new "Group list" mapper to map the member attribute to a user's groups + ![](/img/keycloak/keycloak-saml-client-group-mapper.png) + +## Getting the IDP Metadata + + + + +To get the IDP metadata, export a `metadata.xml` file from your Keycloak client. +From the **Installation** tab, choose the **SAML Metadata IDPSSODescriptor** format option and download your file. + + + + +1. From the **Configure** section, click the **Realm Settings** tab. +1. Click the **General** tab. +1. From the **Endpoints** field, click **SAML 2.0 Identity Provider Metadata**. + +Verify the IDP metadata contains the following attributes: + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +Some browsers, such as Firefox, may render/process the document such that the contents appear to have been modified, and some attributes appear to be missing. In this situation, use the raw response data that can be found using your browser. + +The following is an example process for Firefox, but will vary slightly for other browsers: + +1. Press **F12** to access the developer console. +1. Click the **Network** tab. +1. From the table, click the row containing `descriptor`. +1. From the details pane, click the **Response** tab. +1. Copy the raw response data. + +The XML obtained contains `EntitiesDescriptor` as the root element. Rancher expects the root element to be `EntityDescriptor` rather than `EntitiesDescriptor`. So before passing this XML to Rancher, follow these steps to adjust it: + +1. Copy all the attributes from `EntitiesDescriptor` to the `EntityDescriptor` that are not present. +1. Remove the `` tag from the beginning. +1. Remove the `` from the end of the xml. + +You are left with something similar as the example below: + +``` + +.... + +``` + + + + +1. From the **Configure** section, click the **Realm Settings** tab. +1. Click the **General** tab. +1. From the **Endpoints** field, click **SAML 2.0 Identity Provider Metadata**. + +Verify the IDP metadata contains the following attributes: + +``` +xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" +xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" +xmlns:ds="http://www.w3.org/2000/09/xmldsig#" +``` + +Some browsers, such as Firefox, may render/process the document such that the contents appear to have been modified, and some attributes appear to be missing. In this situation, use the raw response data that can be found using your browser. + +The following is an example process for Firefox, but will vary slightly for other browsers: + +1. Press **F12** to access the developer console. +1. Click the **Network** tab. +1. From the table, click the row containing `descriptor`. +1. From the details pane, click the **Response** tab. +1. Copy the raw response data. + + + + +## Configuring Keycloak in Rancher + + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Keycloak SAML**. +1. Complete the **Configure Keycloak Account** form. For help with filling the form, see the [configuration reference](#configuration-reference). +1. After you complete the **Configure a Keycloak Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Keycloak IdP to validate your Rancher Keycloak configuration. + + :::note + + You may have to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with Keycloak. Your users can now sign into Rancher using their Keycloak logins. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## Configuration Reference + +| Field | Description | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Display Name Field | The attribute that contains the display name of users.

Example: `givenName` | +| User Name Field | The attribute that contains the user name/given name.

Example: `email` | +| UID Field | An attribute that is unique to every user.

Example: `email` | +| Groups Field | Make entries for managing group memberships.

Example: `member` | +| Entity ID Field | The ID that needs to be configured as a client ID in the Keycloak client.

Default: `https://yourRancherHostURL/v1-saml/keycloak/saml/metadata` | +| Rancher API Host | The URL for your Rancher Server. | +| Private Key / Certificate | A key/certificate pair to create a secure shell between Rancher and your IdP. | +| IDP-metadata | The `metadata.xml` file that you exported from your IdP server. | + +:::tip + +You can generate a key/certificate pair using an openssl command. For example: + +openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.cert + +::: + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the Keycloak server, first double-check the configuration option of your SAML client. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. + +### You are not redirected to Keycloak + +When you click on **Authenticate with Keycloak**, you are not redirected to your IdP. + + * Verify your Keycloak client configuration. + * Make sure `Force Post Binding` set to `OFF`. + + +### Forbidden message displayed after IdP login + +You are correctly redirected to your IdP login page and you are able to enter your credentials, however you get a `Forbidden` message afterwards. + + * Check the Rancher debug log. + * If the log displays `ERROR: either the Response or Assertion must be signed`, make sure either `Sign Documents` or `Sign assertions` is set to `ON` in your Keycloak client. + +### HTTP 502 when trying to access /v1-saml/keycloak/saml/metadata + +This is usually due to the metadata not being created until a SAML provider is configured. +Try configuring and saving keycloak as your SAML provider and then accessing the metadata. + +### Keycloak Error: "We're sorry, failed to process response" + + * Check your Keycloak log. + * If the log displays `failed: org.keycloak.common.VerificationException: Client does not have a public key`, set `Encrypt Assertions` to `OFF` in your Keycloak client. + +### Keycloak Error: "We're sorry, invalid requester" + + * Check your Keycloak log. + * If the log displays `request validation failed: org.keycloak.common.VerificationException: SigAlg was null`, set `Client Signature Required` to `OFF` in your Keycloak client. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md new file mode 100644 index 00000000000..cf84a9998a6 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md @@ -0,0 +1,111 @@ +--- +title: Configure Okta (SAML) +--- + + + + + +If your organization uses Okta Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +:::note + +Okta integration only supports Service Provider initiated logins. + +::: +## Prerequisites + +In Okta, create a SAML Application with the settings below. See the [Okta documentation](https://developer.okta.com/standards/SAML/setting_up_a_saml_application_in_okta) for help. + +Setting | Value +------------|------------ +`Single Sign on URL` | `https://yourRancherHostURL/v1-saml/okta/saml/acs` +`Audience URI (SP Entity ID)` | `https://yourRancherHostURL/v1-saml/okta/saml/metadata` + +## Configuring Okta in Rancher + +You can integrate Okta with Rancher, so that authenticated users can access Rancher resources through their group permissions. Okta returns a SAML assertion that authenticates a user, including which groups a user belongs to. + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Okta**. +1. Complete the **Configure Okta Account** form. The examples below describe how you can map Okta attributes from attribute statements to fields within Rancher. + + | Field | Description | + | ------------------------- | ----------------------------------------------------------------------------- | + | Display Name Field | The attribute name from an attribute statement that contains the display name of users. | + | User Name Field | The attribute name from an attribute statement that contains the user name/given name. | + | UID Field | The attribute name from an attribute statement that is unique to every user. | + | Groups Field | The attribute name in a group attribute statement that exposes your groups. | + | Rancher API Host | The URL for your Rancher Server. | + | Private Key / Certificate | A key/certificate pair used for Assertion Encryption. | + | Metadata XML | The `Identity Provider metadata` file that you find in the application `Sign On` section. | + + :::tip + + You can generate a key/certificate pair using an openssl command. For example: + + ``` + openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout myservice.key -out myservice.crt + ``` + + ::: + + +1. After you complete the **Configure Okta Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Okta IdP to validate your Rancher Okta configuration. + + :::note + + If nothing seems to happen, it's likely because your browser blocked the pop-up. Make sure you disable the pop-up blocker for your rancher domain and whitelist it in any other extensions you might utilize. + + ::: + +**Result:** Rancher is configured to work with Okta. Your users can now sign into Rancher using their Okta logins. + +:::note SAML Provider Caveats: + +If you configure Okta without OpenLDAP, you won't be able to search for or directly lookup users or groups. This brings several caveats: + +- Users and groups aren't validated when you assign permissions to them in Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## Okta with OpenLDAP search + +You can add an OpenLDAP backend to assist with user and group search. Rancher will display additional users and groups from the OpenLDAP service. This allows assigning permissions to groups that the logged-in user is not already a member of. + +### OpenLDAP Prerequisites + +If you use Okta as your IdP, you can [set up an LDAP interface](https://help.okta.com/en-us/Content/Topics/Directory/LDAP-interface-main.htm) for Rancher to use. You can also configure an external OpenLDAP server. + +You must configure Rancher with a LDAP bind account (aka service account) so that you can search and retrieve LDAP entries for users and groups that should have access. Don't use an administrator account or personal account as an LDAP bind account. [Create](https://help.okta.com/en-us/Content/Topics/users-groups-profiles/usgp-add-users.htm) a dedicated account in OpenLDAP, with read-only access to users and groups under the configured searchbase. + +:::warning Security Considerations + +The OpenLDAP service account is used for all searches. Rancher users will see users and groups that the OpenLDAP service account can view, regardless of their individual SAML permissions. + +::: + + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or from an unrecognized certificate authority, Rancher needs the CA certificate (concatenated with any intermediate certificates) in PEM format. Provide this certificate during the configuration so that Rancher can validate the certificate chain. + +### Configure OpenLDAP in Rancher + +[Configure the settings](../configure-openldap/openldap-config-reference.md) for the OpenLDAP server, groups and users. Note that nested group membership isn't available. + +> Before you proceed with the configuration, please familiarise yourself with [external authentication configuration and principal users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + +1. Sign into Rancher using a local user assigned the [administrator](https://ranchermanager.docs.rancher.com/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions) role (i.e., the _local principal_). +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Okta** or, if SAML is already configured, **Edit Config** +1. Under **User and Group Search**, check **Configure an OpenLDAP server** + +If you experience issues when you test the connection to the OpenLDAP server, ensure that you entered the credentials for the service account and configured the search base correctly. Inspecting the Rancher logs can help pinpoint the root cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../../../../faq/technical-items.md#how-can-i-enable-debug-logging) for more information. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md new file mode 100644 index 00000000000..e45d179881e --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md @@ -0,0 +1,66 @@ +--- +title: Configure PingIdentity (SAML) +--- + + + + + +If your organization uses Ping Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +>**Prerequisites:** +> +>- You must have a [Ping IdP Server](https://www.pingidentity.com/) configured. +>- Following are the Rancher Service Provider URLs needed for configuration: +Metadata URL: `https:///v1-saml/ping/saml/metadata` +Assertion Consumer Service (ACS) URL: `https:///v1-saml/ping/saml/acs` +Note that these URLs will not return valid data until the authentication configuration is saved in Rancher. +>- Export a `metadata.xml` file from your IdP Server. For more information, see the [PingIdentity documentation](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html). + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Ping Identity**. +1. Complete the **Configure a Ping Account** form. Ping IdP lets you specify what data store you want to use. You can either add a database or use an existing ldap server. For example, if you select your Active Directory (AD) server, the examples below describe how you can map AD attributes to fields within Rancher. + + 1. **Display Name Field**: Enter the AD attribute that contains the display name of users (example: `displayName`). + + 1. **User Name Field**: Enter the AD attribute that contains the user name/given name (example: `givenName`). + + 1. **UID Field**: Enter an AD attribute that is unique to every user (example: `sAMAccountName`, `distinguishedName`). + + 1. **Groups Field**: Make entries for managing group memberships (example: `memberOf`). + + 1. **Entity ID Field** (optional): The published, protocol-dependent, unique identifier of your partner. This ID defines your organization as the entity operating the server for SAML 2.0 transactions. This ID may have been obtained out-of-band or via a SAML metadata file. + + 1. **Rancher API Host**: Enter the URL for your Rancher Server. + + 1. **Private Key** and **Certificate**: This is a key-certificate pair to create a secure shell between Rancher and your IdP. + + You can generate one using an openssl command. For example: + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP-metadata**: The `metadata.xml` file that you [exported from your IdP server](https://documentation.pingidentity.com/pingfederate/pf83/index.shtml#concept_exportingMetadata.html). + + +1. After you complete the **Configure Ping Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Ping IdP to validate your Rancher PingIdentity configuration. + + :::note + + You may have to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with PingIdentity. Your users can now sign into Rancher using their PingIdentity logins. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md new file mode 100644 index 00000000000..801ed84068b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md @@ -0,0 +1,19 @@ +--- +title: Local Authentication +--- + + + + + +Local authentication is the default until you configure an external authentication provider. Rancher stores user account information, such as usernames and passwords, locally. By default, the `admin` user that logs in to Rancher for the first time is a local user. + +## Adding Local Users + +Regardless of whether you use external authentication, you should create a few local authentication users so that you can continue using Rancher if your external authentication service encounters issues. + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Users**. +1. Click **Create**. +1. Complete the **Add User** form. +1. Click **Create**. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md new file mode 100644 index 00000000000..38fc2a6403c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md @@ -0,0 +1,81 @@ +--- +title: Users and Groups +--- + + + + + +Rancher relies on users and groups to determine who is allowed to log in to Rancher and which resources they can access. When you configure an external authentication provider, users from that provider will be able to log in to your Rancher server. When a user logs in, the authentication provider will supply your Rancher server with a list of groups to which the user belongs. + +Access to clusters, projects, multi-cluster apps, and global DNS providers and entries can be controlled by adding either individual users or groups to these resources. When you add a group to a resource, all users who are members of that group in the authentication provider, will be able to access the resource with the permissions that you've specified for the group. For more information on roles and permissions, see [Role Based Access Control](../../../../pages-for-subheaders/manage-role-based-access-control-rbac.md). + +## Managing Members + +When adding a user or group to a resource, you can search for users or groups by beginning to type their name. The Rancher server will query the authentication provider to find users and groups that match what you've entered. Searching is limited to the authentication provider that you are currently logged in with. For example, if you've enabled GitHub authentication but are logged in using a [local](create-local-users.md) user account, you will not be able to search for GitHub users or groups. + +All users, whether they are local users or from an authentication provider, can be viewed and managed. In the upper left corner, click **☰ > Users & Authentication**. In the left navigation bar, click **Users**. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## User Information + +Rancher maintains information about each user that logs in through an authentication provider. This information includes whether the user is allowed to access your Rancher server and the list of groups that the user belongs to. Rancher keeps this user information so that the CLI, API, and kubectl can accurately reflect the access that the user has based on their group membership in the authentication provider. + +Whenever a user logs in to the UI using an authentication provider, Rancher automatically updates this user information. + +### Automatically Refreshing User Information + +Rancher will periodically refresh the user information even before a user logs in through the UI. You can control how often Rancher performs this refresh. + +Two settings control this behavior: + +- **`auth-user-info-max-age-seconds`** + + This setting controls how old a user's information can be before Rancher refreshes it. If a user makes an API call (either directly or by using the Rancher CLI or kubectl) and the time since the user's last refresh is greater than this setting, then Rancher will trigger a refresh. This setting defaults to `3600` seconds, i.e. 1 hour. + +- **`auth-user-info-resync-cron`** + + This setting controls a recurring schedule for resyncing authentication provider information for all users. Regardless of whether a user has logged in or used the API recently, this will cause the user to be refreshed at the specified interval. This setting defaults to `0 0 * * *`, i.e. once a day at midnight. See the [Cron documentation](https://en.wikipedia.org/wiki/Cron) for more information on valid values for this setting. + +To change these settings, + +1. In the upper left corner, click **☰ > Global Settings**. +1. Go to the setting you want to configure and click **⋮ > Edit Setting**. + +:::note + +Since SAML does not support user lookup, SAML-based authentication providers do not support periodically refreshing user information. User information will only be refreshed when the user logs into the Rancher UI. + +::: +### Manually Refreshing User Information + +If you are not sure the last time Rancher performed an automatic refresh of user information, you can perform a manual refresh of all users. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. On the **Users** page, click on **Refresh Group Memberships**. + +**Results:** Rancher refreshes the user information for all users. Requesting this refresh will update which users can access Rancher as well as all the groups that each user belongs to. + +:::note + +Since SAML does not support user lookup, SAML-based authentication providers do not support the ability to manually refresh user information. User information will only be refreshed when the user logs into the Rancher UI. + +::: + +## Session Length + +The default length (TTL) of each user session is adjustable. The default session length is 16 hours. + +1. In the upper left corner, click **☰ > Global Settings**. +1. Go to **`auth-user-session-ttl-minutes`** and click **⋮ > Edit Setting**. +1. Enter the amount of time in minutes a session length should last and click **Save**. + +**Result:** Users are automatically logged out of Rancher after the set number of minutes. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md new file mode 100644 index 00000000000..88fe107990a --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md @@ -0,0 +1,86 @@ +--- +title: 1. Configuring Microsoft AD FS for Rancher +--- + + + + + +Before configuring Rancher to support AD FS users, you must add Rancher as a [relying party trust](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/technical-reference/understanding-key-ad-fs-concepts) in AD FS. + +1. Log into your AD server as an administrative user. + +1. Open the **AD FS Management** console. Select **Add Relying Party Trust..**. from the **Actions** menu and click **Start**. + + ![](/img/adfs/adfs-overview.png) + +1. Select **Enter data about the relying party manually** as the option for obtaining data about the relying party. + + ![](/img/adfs/adfs-add-rpt-2.png) + +1. Enter your desired **Display name** for your Relying Party Trust. For example, `Rancher`. + + ![](/img/adfs/adfs-add-rpt-3.png) + +1. Select **AD FS profile** as the configuration profile for your relying party trust. + + ![](/img/adfs/adfs-add-rpt-4.png) + +1. Leave the **optional token encryption certificate** empty, as Rancher AD FS will not be using one. + + ![](/img/adfs/adfs-add-rpt-5.png) + +1. Select **Enable support for the SAML 2.0 WebSSO protocol** + and enter `https:///v1-saml/adfs/saml/acs` for the service URL. + + ![](/img/adfs/adfs-add-rpt-6.png) + +1. Add `https:///v1-saml/adfs/saml/metadata` as the **Relying party trust identifier**. + + ![](/img/adfs/adfs-add-rpt-7.png) + +1. This tutorial will not cover multi-factor authentication; please refer to the [Microsoft documentation](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/configure-additional-authentication-methods-for-ad-fs) if you would like to configure multi-factor authentication. + + ![](/img/adfs/adfs-add-rpt-8.png) + +1. From **Choose Issuance Authorization RUles**, you may select either of the options available according to use case. However, for the purposes of this guide, select **Permit all users to access this relying party**. + + ![](/img/adfs/adfs-add-rpt-9.png) + +1. After reviewing your settings, select **Next** to add the relying party trust. + + ![](/img/adfs/adfs-add-rpt-10.png) + + +1. Select **Open the Edit Claim Rules..**. and click **Close**. + + ![](/img/adfs/adfs-add-rpt-11.png) + +1. On the **Issuance Transform Rules** tab, click **Add Rule..**.. + + ![](/img/adfs/adfs-edit-cr.png) + +1. Select **Send LDAP Attributes as Claims** as the **Claim rule template**. + + ![](/img/adfs/adfs-add-tcr-1.png) + +1. Set the **Claim rule name** to your desired name (for example, `Rancher Attributes`) and select **Active Directory** as the **Attribute store**. Create the following mapping to reflect the table below: + + | LDAP Attribute | Outgoing Claim Type | + | -------------------------------------------- | ------------------- | + | Given-Name | Given Name | + | User-Principal-Name | UPN | + | Token-Groups - Qualified by Long Domain Name | Group | + | SAM-Account-Name | Name | +
+ + ![](/img/adfs/adfs-add-tcr-2.png) + +1. Download the `federationmetadata.xml` from your AD server at: +``` +https:///federationmetadata/2007-06/federationmetadata.xml +``` + +**Result:** You've added Rancher as a relying trust party. Now you can configure Rancher to leverage AD. + +### [Next: Configuring Rancher for Microsoft AD FS](configure-rancher-for-ms-adfs.md) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md new file mode 100644 index 00000000000..d49f149cd47 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md @@ -0,0 +1,58 @@ +--- +title: 2. Configuring Rancher for Microsoft AD FS +--- + + + + + +After you complete [Configuring Microsoft AD FS for Rancher](configure-ms-adfs-for-rancher.md), enter your Active Directory Federation Service (AD FS) information into Rancher so that AD FS users can authenticate with Rancher. + +:::note Important Notes For Configuring Your ADFS Server: + +- The SAML 2.0 WebSSO Protocol Service URL is: `https:///v1-saml/adfs/saml/acs` +- The Relying Party Trust identifier URL is: `https:///v1-saml/adfs/saml/metadata` +- You must export the `federationmetadata.xml` file from your AD FS server. This can be found at: `https:///federationmetadata/2007-06/federationmetadata.xml` + +::: + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **ADFS**. +1. Complete the **Configure AD FS Account** form. Microsoft AD FS lets you specify an existing Active Directory (AD) server. The [configuration section below](#configuration) describe how you can map AD attributes to fields within Rancher. +1. After you complete the **Configure AD FS Account** form, click **Enable**. + + Rancher redirects you to the AD FS login page. Enter credentials that authenticate with Microsoft AD FS to validate your Rancher AD FS configuration. + + :::note + + You may have to disable your popup blocker to see the AD FS login page. + + ::: + +**Result:** Rancher is configured to work with MS FS. Your users can now sign into Rancher using their MS FS logins. + +## Configuration + +| Field | Description | +|---------------------------|-----------------| +| Display Name Field | The AD attribute that contains the display name of users.

Example: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name` | +| User Name Field | The AD attribute that contains the user name/given name.

Example: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname` | +| UID Field | An AD attribute that is unique to every user.

Example: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn` | +| Groups Field | Make entries for managing group memberships.

Example: `http://schemas.xmlsoap.org/claims/Group` | +| Rancher API Host | The URL for your Rancher Server. | +| Private Key / Certificate | This is a key-certificate pair to create a secure shell between Rancher and your AD FS. Ensure you set the Common Name (CN) to your Rancher Server URL.

[Certificate creation command](#cert-command) | +| Metadata XML | The `federationmetadata.xml` file exported from your AD FS server.

You can find this file at `https:///federationmetadata/2007-06/federationmetadata.xml`. | + + + + +:::tip + +You can generate a certificate using an openssl command. For example: + +``` +openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" +``` + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md new file mode 100644 index 00000000000..bd6f5454ba2 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md @@ -0,0 +1,82 @@ +--- +title: OpenLDAP Configuration Reference +--- + + + + + +For further details on configuring OpenLDAP authentication, refer to the [official documentation.](https://www.openldap.org/doc/) + +> Before you proceed with the configuration, please familiarize yourself with the concepts of [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + + +## Background: OpenLDAP Authentication Flow + +1. When a user attempts to login with LDAP credentials, Rancher creates an initial bind to the LDAP server using a service account with permissions to search the directory and read user/group attributes. +2. Rancher then searches the directory for the user by using a search filter based on the provided username and configured attribute mappings. +3. Once the user has been found, they are authenticated with another LDAP bind request using the user's DN and provided password. +4. Once authentication succeeded, Rancher then resolves the group memberships both from the membership attribute in the user's object and by performing a group search based on the configured user mapping attribute. + +## OpenLDAP Server Configuration + +You will need to enter the address, port, and protocol to connect to your OpenLDAP server. `389` is the standard port for insecure traffic, `636` for TLS traffic. + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or not from a recognized certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +If you are in doubt about the correct values to enter in the user/group Search Base configuration fields, consult your LDAP administrator or refer to the section [Identify Search Base and Schema using ldapsearch](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#annex-identify-search-base-and-schema-using-ldapsearch) in the Active Directory authentication documentation. + +
OpenLDAP Server Parameters
+ +| Parameter | Description | +|:--|:--| +| Hostname | Specify the hostname or IP address of the OpenLDAP server | +| Port | Specify the port at which the OpenLDAP server is listening for connections. Unencrypted LDAP normally uses the standard port of 389, while LDAPS uses port 636.| +| TLS | Check this box to enable LDAP over SSL/TLS (commonly known as LDAPS). You will also need to paste in the CA certificate if the server uses a self-signed/enterprise-signed certificate. | +| Server Connection Timeout | The duration in number of seconds that Rancher waits before considering the server unreachable. | +| Service Account Distinguished Name | Enter the Distinguished Name (DN) of the user that should be used to bind, search and retrieve LDAP entries. | +| Service Account Password | The password for the service account. | +| User Search Base | Enter the Distinguished Name of the node in your directory tree from which to start searching for user objects. All users must be descendents of this base DN. For example: "ou=people,dc=acme,dc=com".| +| Group Search Base | If your groups live under a different node than the one configured under `User Search Base` you will need to provide the Distinguished Name here. Otherwise leave this field empty. For example: "ou=groups,dc=acme,dc=com".| + +## User/Group Schema Configuration + +If your OpenLDAP directory deviates from the standard OpenLDAP schema, you must complete the **Customize Schema** section to match it. + +Note that the attribute mappings configured in this section are used by Rancher to construct search filters and resolve group membership. It is therefore always recommended to verify that the configuration here matches the schema used in your OpenLDAP. + +If you are unfamiliar with the user/group schema used in the OpenLDAP server, consult your LDAP administrator or refer to the section [Identify Search Base and Schema using ldapsearch](../../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md#annex-identify-search-base-and-schema-using-ldapsearch) in the Active Directory authentication documentation. + +### User Schema Configuration + +The table below details the parameters for the user schema configuration. + +
User Schema Configuration Parameters
+ +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for user objects in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Username Attribute | The user attribute whose value is suitable as a display name. | +| Login Attribute | The attribute whose value matches the username part of credentials entered by your users when logging in to Rancher. This is typically `uid`. | +| User Member Attribute | The user attribute containing the Distinguished Name of groups a user is member of. Usually this is one of `memberOf` or `isMemberOf`. | +| Search Attribute | When a user enters text to add users or groups in the UI, Rancher queries the LDAP server and attempts to match users by the attributes provided in this setting. Multiple attributes can be specified by separating them with the pipe ("\|") symbol. | +| User Enabled Attribute | If the schema of your OpenLDAP server supports a user attribute whose value can be evaluated to determine if the account is disabled or locked, enter the name of that attribute. The default OpenLDAP schema does not support this and the field should usually be left empty. | +| Disabled Status Bitmask | This is the value for a disabled/locked user account. The parameter is ignored if `User Enabled Attribute` is empty. | + +### Group Schema Configuration + +The table below details the parameters for the group schema configuration. + +
Group Schema Configuration Parameters
+ +| Parameter | Description | +|:--|:--| +| Object Class | The name of the object class used for group entries in your domain. If defined, only specify the name of the object class - *don't* include it in an LDAP wrapper such as &(objectClass=xxxx) | +| Name Attribute | The group attribute whose value is suitable for a display name. | +| Group Member User Attribute | The name of the **user attribute** whose format matches the group members in the `Group Member Mapping Attribute`. | +| Group Member Mapping Attribute | The name of the group attribute containing the members of a group. | +| Search Attribute | Attribute used to construct search filters when adding groups to clusters or projects in the UI. See description of user schema `Search Attribute`. | +| Group DN Attribute | The name of the group attribute whose format matches the values in the user's group membership attribute. See `User Member Attribute`. | +| Nested Group Membership | This settings defines whether Rancher should resolve nested group memberships. Use only if your organization makes use of these nested memberships (ie. you have groups that contain other groups as members). This option is disabled if you are using Shibboleth. | \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md new file mode 100644 index 00000000000..a51b1328cd0 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md @@ -0,0 +1,32 @@ +--- +title: Group Permissions with Shibboleth and OpenLDAP +--- + + + + + +Because Shibboleth is a SAML provider, it doesn't support searching for groups. While a Shibboleth integration can validate user credentials, it can't be used to assign permissions to groups in Rancher without additional configuration. + +One solution to this problem is to configure an OpenLDAP identity provider. With an OpenLDAP back end for Shibboleth, you will be able to search for groups in Rancher and assign them to resources such as clusters, projects, or namespaces from the Rancher UI. + +### Terminology + +- **Shibboleth** is a single sign-on log-in system for computer networks and the Internet. It allows people to sign in using just one identity to various systems. It validates user credentials, but does not, on its own, handle group memberships. +- **SAML:** Security Assertion Markup Language, an open standard for exchanging authentication and authorization data between an identity provider and a service provider. +- **OpenLDAP:** a free, open-source implementation of the Lightweight Directory Access Protocol (LDAP). It is used to manage an organization’s computers and users. OpenLDAP is useful for Rancher users because it supports groups. In Rancher, it is possible to assign permissions to groups so that they can access resources such as clusters, projects, or namespaces, as long as the groups already exist in the identity provider. +- **IdP or IDP:** An identity provider. OpenLDAP is an example of an identity provider. + +### Adding OpenLDAP Group Permissions to Rancher Resources + +The diagram below illustrates how members of an OpenLDAP group can access resources in Rancher that the group has permissions for. + +For example, a cluster owner could add an OpenLDAP group to a cluster so that they have permissions view most cluster level resources and create new projects. Then the OpenLDAP group members will have access to the cluster as soon as they log in to Rancher. + +In this scenario, OpenLDAP allows the cluster owner to search for groups when assigning persmissions. Without OpenLDAP, the functionality to search for groups would not be supported. + +When a member of the OpenLDAP group logs in to Rancher, she is redirected to Shibboleth and enters her username and password. + +Shibboleth validates her credentials, and retrieves user attributes from OpenLDAP, including groups. Then Shibboleth sends a SAML assertion to Rancher including the user attributes. Rancher uses the group data so that she can access all of the resources and permissions that her groups have permissions for. + +![Adding OpenLDAP Group Permissions to Rancher Resources](/img/shibboleth-with-openldap-groups.svg) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md new file mode 100644 index 00000000000..b3e84e98c15 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md @@ -0,0 +1,82 @@ +--- +title: Pod Security Policies +--- + + + + + +:::caution +Pod Security Policies are only available in Kubernetes until v1.24. [Pod Security Standards](pod-security-standards.md) are the built-in alternative. +::: + +[Pod Security Policies (PSPs)](https://kubernetes.io/docs/concepts/security/pod-security-policy/) are objects that control security-sensitive aspects of the pod specification (such as root privileges). + +If a pod doesn't meet the conditions specified in the PSP, Kubernetes won't allow it to start, and Rancher will display the following error message: `Pod is forbidden: unable to validate...`. + + +## How PSPs Work + +You can assign PSPs at the cluster or project level. + +PSPs work through inheritance: + +- By default, PSPs assigned to a cluster are inherited by its projects, as well as any namespaces added to those projects. +- **Exception:** Namespaces that are not assigned to projects do not inherit PSPs, regardless of whether the PSP is assigned to a cluster or project. Because these namespaces have no PSPs, workload deployments to these namespaces will fail, which is the default Kubernetes behavior. +- You can override the default PSP by assigning a different PSP directly to the project. + +Any workloads that are already running in a cluster or project before a PSP is assigned will not be checked if it complies with the PSP. Workloads would need to be cloned or upgraded to see if they pass the PSP. + +Read more about Pod Security Policies in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). + +## Default PSPs + +Rancher ships with three default Pod Security Policies (PSPs): the `restricted-noroot`, `restricted` and `unrestricted` policies. + +### Restricted-NoRoot + +This policy is based on the Kubernetes [example restricted policy](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/policy/restricted-psp.yaml). It significantly restricts what types of pods can be deployed to a cluster or project. This policy: + +- Prevents pods from running as a privileged user and prevents escalation of privileges. +- Validates that server-required security mechanisms are in place, such as restricting what volumes can be mounted to only the core volume types and preventing root supplemental groups from being added. + +### Restricted + +This policy is a relaxed version of the `restricted-noroot` policy, with almost all the restrictions in place, except for the fact that it allows running containers as a privileged user. + +### Unrestricted + +This policy is equivalent to running Kubernetes with the PSP controller disabled. It has no restrictions on what pods can be deployed into a cluster or project. + +:::note important + +When disabling PSPs, default PSPs are **not** automatically deleted from your cluster. You must manually delete them if they're no longer needed. + +::: + +## Creating PSPs + +Using Rancher, you can create a Pod Security Policy using our GUI rather than creating a YAML file. + +### Requirements + +Rancher can only assign PSPs for clusters that are [launched using RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + +You must enable PSPs at the cluster level before you can assign them to a project. This can be configured by [editing the cluster](../../../pages-for-subheaders/cluster-configuration.md). + +It is a best practice to set PSP at the cluster level. + +We recommend adding PSPs during cluster and project creation instead of adding it to an existing one. + +### Creating PSPs in the Rancher UI + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the left navigation bar, click **Pod Security Policies**. +1. Click **Add Policy**. +1. Name the policy. +1. Complete each section of the form. Refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) for more information on what each policy does. +1. Click **Create**. + +## Configuration + +The Kubernetes documentation on PSPs is [here](https://kubernetes.io/docs/concepts/policy/pod-security-policy/). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md new file mode 100644 index 00000000000..68bf57e5d79 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md @@ -0,0 +1,196 @@ +--- +title: Custom Branding +--- + + + + + +## Prerequisite + +You will need to have at least cluster member permissions. + +## Brand Configuration + +To configure the brand settings, + +1. Click **☰ > Global settings**. +2. Click **Branding**. + +### Private Label Company Name + +This option replaces "Rancher" with the value you provide in most places. Files that need to have Rancher in the name, such as `rancher-compose.yml`, won't be changed. + +### Support Links + +Use a url address to send new "File an Issue" reports instead of sending users to the Github issues page. Optionally show Rancher community support links. + +### Logo + +Upload light and dark logos to replace the Rancher logo in the top-level navigation header. + +### Primary Color + +You can override the primary color used throughout the UI with a custom color of your choice. + +### Fixed Banners + +Display a custom fixed banner in the header, footer, or both. + +To configure banner settings, + +1. Click **☰ > Global settings**. +2. Click **Banners**. + +## Custom Navigation Links + +In this section, you'll learn how to configure the links in the left navigation bar of the **Cluster Dashboard**. To get to the cluster dashboard, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want custom navigation links and click **Explore**. + +It can be useful to add a link for quick access to services installed on a cluster. For example, you could add a link to the Kiali UI for clusters with Istio installed, or you could add a link to the Grafana UI for clusters with Rancher monitoring installed. + +The custom links don't affect who has access to each service. + +Links can be created at the top level and multiple links can be grouped together. + +### Adding a Custom Navigation Link + +:::note Prerequisite: + +You will need to have at least cluster member or project member permissions. + +::: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you would like to add custom navigation links and click **Explore**. +2. In the top navigation menu, click **🔠(Resource Search)**. +3. Type **Nav** and click **Nav Links**. +4. Click **Create from YAML**. +5. The simplest way to create a navigation link is to add these fields: + + name: linkname + toURL: https://example.com + + For more details on setting up links, including optional fields, see [Link Configuration.](#link-configuration) +6. Click **Create**. + +## Link Configuration + +### `name` + +Display name for the link. Required. + +### `group` + +Name of a group of links that expands when clicked. + +Optional. If not provided, the link appears standalone. + +Groups are displayed separately from standalone links, as shown below: + +![Screenshot of group and standalone link](/img/grouped-vs-standalone-links.png) + +### `iconSrc` + +Icon source in in base64 format. + +Below is an example of the Grafana logo in base64 format: + +``` +data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADICAYAAACtWK6eAAAgAElEQVR4Aey9d5xkZZn3zb/P+3mffZ9nDcDAzHTuqs49PZEhCBhBJc10nO7pHKbD9PR07p5AWlEEZM2ioph3dXGNqLgCxhVBRIYRmNy5cjqnTlWdqu/7ue5zTk+DiNIsCn4suOacrjrxvq/fle/7PofX+ycDZNT/pIAUaUUmaRBKp8FMQ1L9qL6S4+VYUikIeuBrn+LppiuJNG/BvHYd7HbhbXLBZ/dB8AkwFokCHsAH6Kb8IxTHJIyBHwgDMTBT6h7yWAmb1L3sxyEjfxkYxDDQMa0nWV0vyE2slwZ5JtUO8v5JUhgk0EkRw5TnUg9sgJm03lsezkyTIU0C2VrNJU2WsdtTPVQyAmYU4mGIhDESaULAcSG5RjwJEQ8YsxgEOZoOcSxjvXsyKM8nL24QJ4UmF5TnlL7IWLure/G/3lnn/PVu9SrdaQVApO3/CCAZAYjNSLIVBrD/VMwSnsf4/B38ofWthFu3krhhPZmaLJZuyEY7vJPIV++AwEnImJwJ66qTFcMLSswkGWKkiKqtMIICwB890zL/2lwoHGLYnCIMtMqP3Md5N7mMDRDTBkhKAURAqNBs3TOdspjWERgrmkYuJbQMEPUeKdAEIBrIuSIKEiZ+B0ShADz7W/Tjv8TMLOIjybxcJwNJOUgAkjHQSFoAkedMWn2g7rXKV/9rnfZ3DRCRjgoN0ik2A0nDLgMkbYD3aU7dPcRT7ZfhadmEVldA/IZ1RN6TRahxO8f37CTyxU/B4pySvOlYHNM0sZhQRKgwTRrRWALOZc0lWksoY30n0lPkuMBCPYCcKn/Ic632Iy/ivNvy+4k2EOAamDbJ/rIKFYEhx74AWPIWcpRhv8dyu5mQTEEkmUYTjSsaMBWDiA9mjsF/foWTH76Z33zqEL6jD5IgRkTeR64valfOQQSL6My0Ap/c32qvlNJgq339v8Z5fzcAkcayBKrFkNIBCiDOD/Lj8jG2iZWOwvGHefxQNY+3beHk7grCDS7067JhRwn6dZX4d72L3zVei/6Vz0HYB/EQJINKawRTQXVP+UfYQEjdRphITBlTNIrFqBopIsqcsZnaQYww0iv5yA1XkuJ+eRJREStJvrMFhnO8A5S09ewCDoMkCQUw67KxOMQMCBkpkok4JIIQnYGf3k/s7mlO7N3Fw7VX8J2Ot3Pi/rvBXMLMJKxbx+UR5L4CEJ0IKYUbeV0xAUXDZVSrvZIGeHXPff0DRDGE9PRZPhGec8jhBWcr30uXCWEG4Xf/wW+H38ajXVUcaS7GX+dGvyYX6jeQvL6MZO1lzFx7Mc81XkPyM3eC/xlIz5LJzBIz/bbUtUyKZaksUtPUIS06wzK/LGluM6jwrVg9wkCvECDOe51lE2kL5w2drdU+Ths5bSBbMacsVMtGtKDFug5+5Q00Iw2JFOhhmD0C3/goS6M1HKvfiqfxMo7t3MLv2i7F97nDoJ+BpG45IXLysgYRgIhvJPeRu4QVibZ7LX/+rgDiNLTT58IADjM4rPI8HyXpgYc+yXODF3G0ZyPPtZSxUFeM/9p8MrUVJK4rIX5NMfEdm1jauZ1j7Vfj/exNcPoRSB2HxIICiHI+Hb4U00mYSWkP4RAhHTKiUexggfCEPJDiUOepX/5W3tN5R9m3PpZZJ6bdWbJ+kWPkto51JyaVxbBnpYtcT35XwFDXt8Ee8cJ//wj9X6c40fF2TtZU4qspJV5bidawCU/HxWgfHoTwccuhl4srE0saRnwQwwaIZQKa+BH6B0CcfnvVttIBK8jmFId5zjKEmA+WhJTIkeoYYxY+t5/FtmJ8zeUEWzdypqmM07VFhGpK0W9wYVyXh15dQLxnG/NdF/NE00V4PzgAv/0OLB5RbyWKQD2CML5wV1KMbIlmCSgkapQEkcLCNPJ72mJAsdXl+Vb7cRh+mcnlQvKl0IomUfs2mOT28rwCaiErgiW+hXWOaBSnzRSw4/Mw/wR87zN4xht55vqNzNxQQXj3VoyWzRjX5ZPcWUigrozozU0QeMbSNAnIyA0EcaQRE1N8EPWn0hoSDxSSRnntfl73GkTMAsvXsDnCYZAXMERc2dei2i0HVnWMdpro4etYuv58orUujLatLLZsZKapkqXaYqINRZi7XWQ63ASacwh2lhPtvZwjdVs4M94ETz4M8ajFjI5TLlsJLavwsu0GCA84JpX4uEAAVHBYGHa1H3lVuZaQxXgvAIh86QDFDqu+ECDSIstS3AGWnCdmUnwRjv4Y/XMHON51OSerSwjv2kCqdRta81ZiDZWwqwjq8onWFKIfrIPQs5CIKl/ekJvZDyagFJJbWKFuiQFLePwfJtZq+//PnieNLc64lUuwTYoXMITwZowMgbSu3EIjE8NMiKMdhmd/zlzrRjJ12UQb3IQaKojUbiRUW8VSQynzjQV4mtfjbz6fSNN5hBvXsrjbzXzTZjz1V/Bs0/Xw0A9g7qRy3E3DRzARUJpKni0ZSljpEUkcCEBsQR3BYIEIC2mxw+WBV/dx3v950TP5UshpBxskqURG+cvCjtImwqyyDYZ9pDPiMKfR4hHQY5aJdPIIPPg1jrS/nZndW/E0lRJodBHY5SbYUEq4biOx2goi16+D1iLCO/PwDL0HfvdD5X9JFNwXP+vjyL2UMJDnUs8kRpzkjv4BkNX1/l9wlmiOhHL4RIbaDrA0vs0UwifSMVEkuSWJsyTxRACMIKSi8Nj3WWyphLr16PWFaPVlGDs3ou2swldXpgCy0LoWT+t5RJreqEASaMpRDGLs2E6w+u2c7mkgcM/t8IdHID2PSZAQUaJmSrkAypgXXrClaTIj5kZcRXXiKlH4ygAibXA2Yme3wUqAJDMWWDJgGEmiWgzDFL1hCRcjHkWP+kgaPkgHQUyqIz8l+fHDzLa/i0DdVrTaUmL1LsINBTZIignXVRCpK8W3cx3Jdjehnfl4970bHnsA0rpi/QWxLqUf7SiZ2pd/BBPio0kQQyVO/4LO/hsd8ro2sYQxRPYJSJSZJYwhHSBkWoJUABJRqTLRM0m08KKlPYwgiR99kZOdlRi71pOuKSS90wJIrLqK2M4yZTaEGy0N4ml9M96W89Hqs0lVu0nt2Ii+YztPv8XF8ZZ3EPrkAfj9dyF50gaJhscwEP5U0twGrZlKkEhKwDNkZ7gV27yC7ndUxYtfwkymMZMp5L7KF0pJNj2OmYiR0MMkRWsIoyY8MPsYia/fyZmxGzjdsInAjjKM6nIS1aUKJOH6UgK7LAo2FBPc5WKh9kJiYoJWF7I08G5SP7sfMrryLuZsYCjZJYlFp39EWKhIn4TBVy8gXvyN/2e/fd0DREoY4sqvsEo8lHQSCSV8Z5c8RDJJxAeRUKvmm4FEGDQ/S1+6k2NdG4g1ZWPWFJLZWUZi5wbiOypJXF9GfIcbrT6HYNNaPC0X4G2+AF0BxEVqZ7k6NvRuN96dGzjR+VZOva+LyMNfgOizYrwQJkIkY1qmhcMcSZGccTJmCF0XJ/UVAESuuRytOssY8rWjSGVf12MYsbDKyYh0z4SXyIQWIB60hIU2D7/6FjN37uVo56XMNpQSqXdj1LpIVBdj1JSi1WwgUrtBaQ4xscTc8u7O40z9hYTa3UR2uvD2vpPIA/dBJooXmLOfQwkIx7SS13X6RwIY/wDI2Y77n94TDWKVMCQtP0Q6QRpfJJRsJXyfgWhaoCFaJk5SACKM4ZnlxIemOd2+mWhTnvJB4nUlmDsqMK8vVwDRdxYr0yLSkEekIYfwrhyCjXkqmajVFlvMc0M++k43vh2lnKzfxqmpRrRvfRwWHlPVW7G0D40YyaSBGRVJLV66xSWmZPIV96yyZRQSnIy9JRDkK7m6NIES1CIYMgkykv0W3yvug+giaB7LnDrxK1LfvYeT040cqdvEUm0JdFRhtpbhr84mXF9MuL4crWYjevVG9JoKYhIKbypgsTmH0w3rCLcWkbjBTbD9SrxfuxvSfnyYLDmvpQIXKzSpoFceVJlX/9AgTjP9j2/FsYyRVKQcdWlr4QrHIzQhYUIsbSq7Py2Z8/CC0h4cO8rRw3uZb9mKvstFZJdoixISO8tJ3lCuTKxItUjSYiVNtTqXAotyVBtd6ju91gWNLpI7skheV4BevYHFhks4PnA9S/feQup3D4B+0o7WxIjrYUxdEogWs4j584o+DkCk3kku9YJIlTjikZROUsIGKfEvvJDyg+EB3wkSj3yD2dt7OTnwdk7UbySwq4pE0wYiNYUEavOJNJfiF5OqoVw55HpNOUICEKVBmgo4U7+OsBR2Xu8itvsyFu65GVIegsRUpkN1hiRNRVvY7+3UellZ9FfYBq+oAf/8ya9rE0sAItlZAclLAUTLpIkmJfQoUZoASNLrsV/x9Gg3vqaLSNcWkagtQq8rUY6nxPT9tWXKUQ/WVygJKlJUnFIHMFKSEqvPg+4K0rU5pG7IgdpSkjdUMn/NBmbarmLu1n7MR/4NPE8qkMTQCKSlhtcSnlIoqyTpn++nFz9CACKaUsiWyqI9RD5ICkLyDhF0YmYYQ1+CmJhVi7BwlNPf+zK/PtBGtPsSwlefy+w7z1P+RazrYhYbN3K6upTFunIFDvE35H0jDQVKMKg2qCtV2mWxJksFOHivi2TdxczdNQqJWaKZABKtIxO2KhZsh1we09JuVtWxFaJ/8dd7LXz7ugaINLb4FpZ/YYd5RZ07XJK0SoESaZOY5icj9dcSvZLy7Ie/y+mBBsINW0nXlJKsKVEAEfva11CCTyI0dWUogNRV2FEbkZ6lSoKKjR6pd5FoLMLcVUSqzkXyhgL09+ShS/a9fjvB7qs5PlJP8vufhugzKjEmxp4wsDyiVIorS8PGibP/oowhP4oIdrbOvi2V5VfHtLJyDhLhk5yPbmmN2Ax4j5L61bc58pGDPDTcyFNd78SoLoTq9dBUjL67grn6ck5Ul3GmpgJPQxXB+nLCDW4VwXLMSzG7YrVWW/h2ZqHX5cF1hcTrt+G5bS/EThNP+YjKmwowklEwJWJllZo4Cl4EnHqdF33h18aXr2uAqCa0TQtpaKvCVuobjGVTS3zAuCHJvCCYAZICEM2DcecowV0X2aZDKbo4ohLOFC2xTGJarKA66xjruGIideUqbyI5AUvTyPluxBwzatzqev66TXj73g2fvRWOPQpxj2KaU1qIZ2NhO5ttMbcw+LJ5btsjmYyE44QEHBKIcLZytDU2Q5KOkmqRrELELsBPZOKQDkPKC8YMnPkl+r/fydHhnfy+YRvzjdtJtl+GUVNMosaFVudW7y2ACDRUqPdR4Kh3tKZoEREK8t5WG4mppTfkE63OVu8c7ryMk/vq4fhvVeInYJorhJVIAyvULhUN8p6veXQAr2+ACCrEvLATcAlV8xNUA4WUHo+DRDWTCRnLEADTgxxDeAbz5naSNWXLwBCmf7kkibJw7VYCddvwN2zE21iKv9GlnHkV7arJg8ZKaLmUeP8NxO+cgF9+T4VUpZJ1MR1TCbuzDvXZ6FPaCYvKOzpaQrbCWYoEKOJbpfASZyEVYTERJGRKUFukdRik1mzpaXjk6yzePcLxve9ltmkbkboK0tUlZKrd6LVnGd5h/L90K8IgUZdPpCbb8km6L2W2fycc+ZXKhQTkOaV/RGWIv5XRVd9IxbAChy3cXhu64sWf4u8OIHGkFF23OiYOetQkrQoHxbySMYE6zD5FdLzOju8XK5NJpOHLJqVdJPy5Udnq3sZiPLsL8O7OIdyYrULEyVo3qcbNeOsv4Wjj2zl2yyCZn/4nhE6qkXppCSRImZZpkRPkUmkLqWdaaTeJ8xJNk4mYZMIp0mHNMhkTS6rM3Kpt8oG5CN4j8Mwj+L54JzOHezjW/HbmqrcSq5YQtovUNdkY165Hr7W1ggpGSEDiLycBiFGbpwAigsHTtZ2Z/hvg1z9WWjKsRk39AyAvDr2/xrciXYWBVmiQuCqjtgFiQDRkmSLL5obk1Z98mKW+d2PsLEHCtS8bGDaYRONIIk2rLVcOqwzTXWjJY6ElB//ubKVJQjuyiVaLlN7CfMOlHG27iuMHOkh842Nw9BcQ84IeAc2maBhiMuYiCJEgEm3j2afh6FNw5En4/eOkn3iUzOO/hsd+Bk88BI//CJ78L3juF3Div+G3P8T4z3vwfewQj7VdzTONV6roWrLxIlXGn7khH65bR2ZHNhKJezmgWHmsACReY2kQ365Cljq3MrPnGvjJf6j8SlTU4PM0iDUGRHyjf2iQvyZARHur6Ig4wFGVEFQdk4BYWPS8HCCOokhXP+Eff4XZjiuIVxe9IoCIeZLaKTa85bB7mwqYb81jti1HgUQ0iUR+gtflEb2uGKPhIoJNlzHbdDmefTtI3L4HfnUf/Pwe+OHd8B+3wudGMT7cQ/C23XhursV7Yw3ew9V4DlzP4tS1LIy/h7nRq5nd/w7m9l1FaLgeT88OllquI9xRjdm/G/qbibVdz2LNFczVXsJ8/RYWaivw1rgxGlywuwh25UN1tkoGrmT6l7MvANGr84hU5+JpzGexfSMz3e+Cb34GdC9xGaIrALESMsrekuG/UtHwD4D8DQAiDS9DPsVZd0I6CTFLRJJJNEUAEjnJwlc+yJmWi/4HAOLG3OkiVS0h3wKVPJtvKWCmLQ/Zirnlr8kjurOA5A1ukteXkthZgV5bRbhhM77GSuYb3cwJsHa7WGh2sdTsxtPqwt/qVhRocRNsLiDYlE+gMQ9/Qw6++vV469bhq87FaN5GtHoLwasriF1VBddeDDVXQt1boe5StOrNxHZVqfox784cwrXryezOh0YpmblQDTF+OaBYeawARNuZvwyQhfYK5trfCl+4EyLzmKrU3/ZDRE5lZEShjLBcAZC/Bp+8gnu8/n0Q24GV9heAWMM4ZRyG3TGSaLYdRFI+mP8tpz48ohKEEmmSTrZCti9/K5EfAUequkAlEcUOX2h2M9viZr65GE9TMUvX55KQ7HRtKZnrCkm8NwdzRyHpWpcybxJdm9G6thDr3EqkfSPR1irCzZWEdlcQ3F2Kt86Ft64An02B2gKVxJNEnqIGF776QjUSMlBfonI2oZoygjfIwK9CNfhL8jvplgqSzUWEG7Lw1a5hqfZNeOrWqPCtFcaVUO7LI2m72I48wjtzWGrMYb6tlPnWizE/cRiC4mNJlMQ2g6WfVFhXQs82QETtv8Y/r2+ASOPakRALICKh7LCW9YX6XUq9JcRI0gfHH+H0+9rxNm9RodhXChAJkUrNkoAssEtAUcrC7nI8jRX4d1UQra9QtUzmDS7YUQA78mBnNunq9aTq85h/73pmr8ln/rpCFm9w4a0uIVgrSbhyIg1lBGqKCFbLAC6poC2y/SUr6y8l+r7GXPxt+QQ6ixR5W4tZairEU5uPrzqPeEOxer5kvYtUc5Eqq5mvXcOJujczs+tCAo0FLxsYDpCk7aLXWwBZbMpisa2IxeatpP51BHzPWZM7SD8ICRgUIOz+Wf77tY2Q1z9A7Ia2+kEST45TbneMjHKNZ0gbGsTm4dff4Ll9V+HdVUFkR54aKBWuc7EakkFWMaGaIsK1JWpUnb+uEl99FYHaKjWuRKJGcSlf2VGiRt4ZNdnEa9cRq19DtH6dun+0RiqHy4jWlqhtrKYEi4rUtbWaIixyEa92Ea8pVNtoXT6epizmWtcx15JlUy4Lu3PxNeYTqi8kVluIVmNRTLROg5yTy1xLLvPNuYhzHWhwrYqkzVKitXbmEel2M9ucQ6BlI6cHr4VnfmbNpyUdY/uIFhRWRFVe29hQT/d3ARDBiAMQ9VYKNHZ9kgKIOIoJ0Bbgh59kpu9ygo1lRHfmvyKAWKCywBGqKyNUW6lAEarZpPyC2M5NaIoq0apLFLOG6yVnsA7f7jX4G9cpYMZqyrBIgCH7Jep4BYrqEhscAhKXTWcB4m/MwrN7Hb6mdch+qEEKDHMRMGi1+QpMAhABijyvgMG3y4WnsVCRf5XgkOvI9TKNlerawW4XpxsvVLVbZ/a8Ax7/nnLUpWOk9spRIhZabCH2D4C8+i3gaGoLILYaV2aX3QkSaZTaCylBic3BFw7i6d5MrEkYNl+Vi0id0epIRtcVWyPspCSjbgOxmo3Eajaj79yKXr3ZcpKlTFyNpXCrEvHFlmzm29YiWzlfEncWSb2X7MvWrv1a3p7NaIs5p0LT9S60+jwStdnKbEPqwa4VyoNrC1QBYXqHW5mSco5VmWuFtSUCJyFu+V7GdayG5Fx2b1ZmnL87n5O7zlNaa777Cvjp1yC2ZJm4diBL+mi5ImA5+/nq88grucPrXoM42kPgIPtn292ydSURJ9l09aPvGJHbWwm3lxFvLlYRHLHjpZJ3NRRqLCLYWKK0kfgL4m9odVUqShWv2YReu4loXZUayhtoKsHb7GapJZ+FtlwW2rNYaslV58t50Xo53yJ1rYYSIg0lRO1tZFcRFp191uguqX/KJ1mXT6Y6H3bkg+Q4ri+EGwphhxuzxi7ErC9S1crisCdqyzCry1TwQMbdhxrdqyI5l8aNGLtK8HbmcqrxXPRdBYR6r4AHP2dVDcuQAztXKH1k9Y/s2QLslXDvX+Hcvz+ALKsSVd+gZsuURLoCyPHH8Ey8B6PFhdkiYz0KiO0qItq4Ogo3FRFqKlYUbSwmtqsUvaGUeH05upSIN5QTbiwluLsYX0sRvhYXntYCfK15ioLNecQaXarcXkruHbK+KyDWWIDeKMesoKZcYjbJOJbw7gKCzS517aVWF/PtFkmoWL6X90vWF6mK5UxNMUIpSXDWW88qzx3aXbQqiu0qJlVTqTSIpyub2eY3k2krIrnvbfDtj1rjTaQWzh4DvwwQmfFFjc957YPkdQ0QR3sIJqSplQax2l6VlMiUm6rOT6KKaQj88gGWBq+E1nwyLUVEanPQmtzEmopXRZHdxUR2uxXJdYTijdY21uQm3Owm0FqIt80if2uh+jvcXIjWlE+8KR+tSfbPUmx3IbHd+RYJgGyKNOcRacl5PjXnWfdoKWapvZjZzmJOdbs52eNmpsvNggyFbZH3c5PY5casd5NqKCbRUEx8V4kyM+Udws2rIzFT9etLlgGy0HYedBeT2Xcl+hduBn1WFVeenWLIcRb/AZC/gvKzAOGAYxkgdhRRaq5UqbeNIhFaz/3nF/ANXKoAQoubUHWWYh6HyVezFYYWZtd259lkMXGoNQd/Ww6ejhyWOmWbR6A1TzF+QrLYdYVkJMrU5CYkz/I8soAk4LIoX4Vy/W15rKRQSz7xxmJl4lgMbwFWQCFg9LQXstRhkewLUOU3Aa7zrrK/WlJ+3LVFGA1leLuzWGw/HzpdJHsu4rRUCcwfeXGAqBGVMsJTek0lSP4q/LKam7yGNYg03AtImH0l2X9KM4udq5xASzjZVq8M2LF/MBIcue9u/AOXQ0chtOYS3HkuWnMBEdEmq6Boi4toSwFaS766jlxL9iOt+QTb8vF15CKmh5C3M5tgezZ6cy5mYz7UCxWo84JtBTyfXATank++dhe+9gLObgsIt7owmmQ8SolNRSTFZNttPVegvQBvRwGLXfksdMlWzrfuFWktUM8uzy/7kRbXi9AL26WESMsKai7FU+0i1FaqAOJrPRfackk2l3F85Fr43Q9UTZb0i5hZ0keqS6WPVg7DVUEV6UyJBzsduAI49kzyZ7veCuf/NQD2NweIemm7ASwbyW5FaShFYh8JOWMhbFTY2DH0FKFYnEBaJmawf9OTEI1DPIVKEkpnhDw8dfdBwmPXIqFW/7X/C63h/6J3FBDrKH4F5LbPLUVrq0Br20CsfQORjgrCnaUEu9wEevII9uQQ6c5C78wi1ZYDLXkgJlPTeQR3WxRqXkOweR3B3VmEmnMJiXllP1u0qwShSFcZEdnvLkXvKiXVWUK6vRizrYhUq4u4Ddjobjk/l+DuHIIt2YRa8wi35RNtK0BvdxHvcBPvLCTTX0Zqj7UfFaEhxyvKJbS7AL2t1KZy9LYK9NYNyxTs3MCZvnJO9RcR6F6P2boGGtbArlx8HZt5Yu+74blHIHpazSujy8yWUuErGJCpkJYk9C6DWVIQS9jDcmWAlcxbJgdYCRQVJl4xQbhUTJiq5k7mPZaLvXqfvz1AnBj5WfFgN4wARIAhWmAFQBylImLJljwpE8JpmXTZBoiIKl2mNMkQlepYmazg+GMcOdSJv/cdBGtz0Rr/mVjLG9A684h0Fq+Kwp3FBLtKCXQJEEoJd5YT6Sgn1lFqkwBPAOQm1ulC63Ap5jTaC0i0uZBttD2HyDIJE+cqRhZmDreKlrA0RailUDndgd2FquZLEnzeXXnonUXqurF2l2J+0QYWMCxm1zrc6nfZxtoLcY6LthUSEQ3TkoO3JQtfS5YCUrglj1hrAUabm1R7MYlWtyKjrRi9vRit3Xq3SEcp/p5iTvWXcKbfRbgzF7NlPTQK5aK1lnFqz0Wc/FAXxq+/BimZ4ySKaWhktKQFDBlyvDLEJdEUGXkY91ujEJVQtPIoAgM51NIvUq4SVmR98/cMEOHpleCQ/WU9LA6FTUqb2ClZOcYGhxxtJFLEjJTS2nYLKmUkQzp1meTTnIOf/RvHB65Fb96OUZ2P2VeAv20Nka4CIp3uVZGAwtu9AU9PBd49bvx7cgj3XECs+3z0rgswOrIsBmupING8WRUW6i1b0No2Kk0T7thAcM8lBHovw997Ef6erfh6qvB1leHvKsLfUUi8v1xRYk8ZiT0VJLorMbo2kOisIt6xAb13I9G+DUR6qwjvqSDUU0mop5xgdwXB7jKWWouQ6Ja3vQRfRzGBrnL1e3jPBoL9Vcz3livy9pap4/WOEhKtxdaUq40F0JxHujWHVJul/cI96/H3rmexfy3evvWEOwrR2oqItZURtSncXkKoo4iFnjKe6d3MiffVE/7OR+HMb0Am7ktrpE2ZsClD1ATNnsPMshIMezoima9LhuxafS7yUMAhW+vj8IYwwtH2mK0AACAASURBVKv3+ZtrkBf6FC8KDtU0KxpiBaDiCQMhmR9agUa2KUvaxJUenwf9d6S+cjNL3ZdD6zaodcPQRubb1qsSiWhXEashMXVC3WUEe0oI7ikk2JtFqG8N0T3nE+9eQ7IzC7O1ALO5HHP3ZhK7t6G1bCfcth1/5za8ndsI9V9OsO8tCij+nm0Eujfh79qAv70Uf0cx3uZ8fM35BHbnK5NHImDK92kW/8ClGFGAtJIsX8XyN8I9xaykUHcRDgX2lLDQVcRiVyn+7lJlthldZUpzpFvdpFsKSLflYrZnk+jMQutZR6j3Qnz9F7K0d41633SzG5rE9NpIoHMjiz0bWNhTgWdPsSU0ess51V7BM3vfhudTY/D7H0JK1qAKq8k2ZhI6c8kUkUzGsrClRF40iZhYQmI9KCjZUUpHOFqqxP7y7xkg8m4OwysOFyDI2zsSwop2yOwXzmErtzK6Wf5W/5j2HAEx0FOaWhIMjsLv72Pp5huItGyAzougqRRz31Zm2wsJdYttvzqA6J2FinnS7VkkOnPRugqXr6V1WXZ9ol2kbw6pVrH93Yjd7unaxkzPW5jv2U5kTxnx7gLiXUWWuSQmTGs5sdZKoi2WTyO+jfgClrlUgN6Vi96VTbxrHVrHhWida5ZJNNdKCrW8iXDrm4m0nUus43x1XLz7QoyetaR6sqC3CPaUwZ5i6Cki2eNSzxPtzifcnUeoJ5/Qntw/okhPrvJj2LUBdm0m0bJBaadZ8UkGyjk9UM7inlLlR9FZgtnmxtNejveDTfDYVyEzq2Zc8WCySEZNESTTM1n9qEqwQeYSUHPAyJgFMaDtyQGFPeRPoRVy89WAyd9egyyDwgGGAwirLFqcMGuCaqfe6vlbOdoBiClDVMW3k/GrMsWPcZzQo5/lzGc6ebarglBjIXRuhLYK4n2bWOouI9xTSqy7eFUU73JBe7aiTHuBkrzJ9kqMjg3EOyuJdxUrHyfWlaUcdHHU/T1uPD3lLPRsxttdgdaejdl+gTJhku35JDrcGJ2l6nytu4pY9xYi3VsI7dlCsG8zgf6N+Pduwje4Ed9QFWe6rNzH6Z4SzuwpZqa3jJneEoRRZ/tKWdi7gYW9FcwPVDLXX6a+l9/P7CllsdtNqrecdFcRiY5Com15hNoLCHQVEuotItxfQrC3yNrfU0S0pwi9u0iBOdlZRLqtlExzJWZLFVpHJYEeuUclswMVLPaVEepxk27NgqYLQbY9JSwObufY+xsJPHQvxI+rhT/DagWqJFEzoeYPVh0qnSrLRygUSKeqjrU6X76WiMzfP0DssJ74Fyu0howXsIBhgUN8CecIZyu2qFAsmSSeThNPmWoOWjVBmRGGM0/AL77KsX9t46n9F3O6w0203QWdpdBdTqijhEhvJdGeYtXx0vkvl2KKWYpJdhSTaSuGllKbykm3laN3lePtKWa+z83sgIv5/jy8vVmEe9aS6DxfASO9ex1mSzbptnzMrnxSfS4Sg0XEhyuIjFUxt7eS0/u3cmb87cwcvJ75W1tZ+mA/3rtGmf/INOZ370V/4F7iP7gP44efJ/ngl0j9+IuY//UV0j/5Mvz062qrP/B5fP/xUc588Q6euecmnvzIAf5wRz8nJt/J3P4tnOyt4GRnKTMdxfh6KtH3biWz/2IFVKOjklS79U7yXplWi8SJj3bnEugtZKm3ksU9W/D2bCXSWYnZng9ta6BnHVrdPyEROnPsYkKjb+NY18WEpnfCN2+HM/9F2jyhpmoVq0F8SS1h8b70tQjHsyCRCejsUYoCDvHa/741iAMQx5yytgIQmQjO+k/A8XyAOOCQBtRUOtCad1eNGpRZA71HSXzn4xw9UM+xfVfyXFsZsaHNJPZWqmiS0Vem6qBifWWEbcn4csEhx4v2CfVsINwtGqOcVHupKrVQ5RYSnu0uxttTynxfKbP9xcz3F+DvzVJOvNnxz5jt5xHtKFQOdaC3iuC+LQRGLsY/fQWLN76D2Vuvgf84AN+9HX5yLzz2HXjuUVg6DiEZHSnj1kPWeHZdVqKVCNCfIRn7LueF/OA/BY/+O/z4bmJfmWLu7g5OHtzByeGrmN/7Nvx9VxDu3I7esZVU20YyrZXQWm4JgdZiUh0u5YcsDmSz2F+EV4IEXRXEO0rItOVC6zqMhjcqX0zrkQBBKaHujST7Loaei1jq3U7gK1Okn/0OJGfUNEWJpEnAgGDaWqBKcGCFciWaac/Q6Mx9JoygzIdXw7iyrvk3NrEEIHYo9wU+h0gOAUYsqas1ti2QpNXU/VpcxzBkjiWZORD8yTAZAUZiFk79HP2rt3D64E4W979DRYciPZuI9FQR2VNBuLeEUL+bUH8h4b5CIr1uIr3Fq6JwbxmhgYsI7NtOaGgL4X0VBAeK8PVk42lfy1LzGrQ9Vpg30laI1llEvLcUY28xWl8eS31uToxdzLGb3sviv7ahf3EafvBJ+M234blfw+yzEPFDLAq6zPUlQ4nFiXVyRLIvs+OtklKylqBMYC1zZy2A/1l4+mFS3/8c/k8eYPHWDuZGriW4753oA5eS7tsCPRXQ6YaOPOjMJT5USnS/1F/lYu5dT7L3QrSutfhac5nfXajMQ61rE2ZHBbQXQ1shtBdidhYjkbRj+7ZjfGUcnvsRhE+QNmJqfi9ZWkfm+1KznCnr24CUzOIijrvARozrV//zGgGIo0EsA8oBh4BCZpaV5Q3ipoFuRLEmfLYYJGnGmUv41YqzSgL9/rtEPjPK7ORVeIYuITl2GWLHG50Wad0bCPWV4B9w49+bq8gCyWoBUkJo72Z8e7fgFZ9gpJLAeAWBURf+gWx8XRdi9rthsBIGt2Ds3YZv8BIWxt6B933VLH1sD8YPP0ryv78Gxx4C/x9AZlrXRDNEIBg5G/wXv1XW+FieVtQywxWLCAOthjIQ0ZJoyYwValeTLAgYF8H3DMz9Gn79NTJfO8zcjdfybO9G5nsr0PZvwhypwtxXQbK3HGOPi8SeCzH6ziW+91wi+9bhHShmsbeKQM+laB1vISPRw5YKCyAdORh7soj05ZIa2Yxn5C2EPjEIR36gZoGUyR7mE7AohpcpARd5P5lXQISpRjIZQktbK7682lOXvgYAskIa2supiea0oCLbNEY6QVQLocl0OCr0JwkliWzEVJ4jkzoGj99P7GNDLAxchlcy2AMlxPaVou8pJdlVCe0bSXdUofWWEtjrYnF/LovD2QT3uoj2Fa2aRFJG9pSgDVcR2l+JZ28xnoECtJEiOLABvS9XZdElJzAz9nY8HxtSfgKnnrKWbwuKiSTTg9raQGL/GV2tgmUtj+CYmxZWBCDin0oCWvalrVb7EQmsEttSki5zc6mtRAXl6iLDpdjwGOiPw6lvk3j4Q8x+dg9P3nw1j49cwrHeKlLid7WXkt5bTniomDP7cjk5uJ6FoXxi+ysxujeT7LiEZNtbSXRcjtazmUi/i+DQm4kOvgm616vk5cz+K1n8zCippx8EM6T632OkCMQzhGPidwhARAoY6EkfYdOrVl5XczKvtgH+gvP+tgBRkQp7PT+pwzkbrX0+QJKamhldqVgjjBlcsObXTcv6GsfxPPYFTt3ejm/gbZh9WzD7iwkN5OIbyEbrc5HsLoH2ckWJnlIFioXhXIReKUCMgVJCnYUsteYQ6HRhDFVhjm9BG97AQn8R0ZveQuTO64l/aQR+/jmYewIkiGAzpRIEUhKehFjcJBo3iCY04qZGAs2eBURmApGAhQWI/8mtU8Yhgb9oKkM4lSSc0YgisSUfCRaAU8AxQDTcb0g8803+8O338+Rd7cr8MnsvJdS3nfl+GW5bxtxgEYGhIuJD5eji8HdvJdJzGYHeS/AObMSzz4V3+M1E972BVOMboK8cfeRijo69g2c/PUbm2YfU9EwChlQiSTQSJ67ZdXWyTHVGpuQOqxls/r5LTQQgCiRnnS0l1VZoEGEGNQuGWndcJK3M0O6B6DzEjhH41Sc59slWZgcuJdWzFfZUoHVmsbh3HeEDLqKDOaR686FD7GY3qT3FRAfceIfcLO13E9orf6+OIgOl+PaUEeqtwOgpJdNdCj3lsHczxtTb8PzLTvj5p+H4g5A4LXljtU6JP5ViMawzGwgRNDQFCN1MIsuzyfuKNWGaGSsq5xTwKfPCWbZM5hqWBXGseiWnGVe3XVn45wRJ4qRFi2WiJNJBYkkfkZRHMWWcKBphvJlFwv6nmP/mR/D+6xBnRq9lrutSol2byfRvxezfRKyvAt9gBYv7y5gbKWVuxM3ScD6+/XmE9uUR788j05YNXW6SQ5vxT17Oczdew4l79pF54n5IzEBKSoV04ppOMmXlwtIk1eJ7ii9e5TDW316DOAARBWIXLQqTiGRVklJpFjHD7BIEWedCZicJncH87bd47Jb3MDu+lVTfRugsJ9GWh6f9As4MrcN/MJ/ovrWk+9ZCVxZ05ZDszVcACQ6KJikj8goBstTlVgDR+qvQ+jajD10Od7TC9z8Kz8oquIvq2WViackfz6QzLKUFKhLUzBCKB4kmIyQyoiXOJkOV7y02j+NbSORGrbEh64s4AJGrnHVWpSkdLfyXbOXiGbWIj5irTjLOWqJN1i6MabJMnMwSLzPoy8TYsGgmmU0nmFeGWIq0/wQceRi+fQ/6HYOE915NqGWLKtgUcza4rxDvcBaLYxewNHoBvuF1BPflERkoUwCS+jTJvxgD5SQnt7M0spWjo5exdO8gPHW/NfF2Okg6k0A3TeIpazHSjCwrZ8gqodY7v1r//u0B4jDACwCiwCHS1JnNXCIYRggML/hP4v31jzh272HmhreSEQ3R6YIOF+E9LuYHCzgxms2Z4QuI71sD/edC77nQvwZtMJfwYAnC0JIs1KSj9hatirS9LpjcgD5UxPzQJvx3NsGDn4b5p8CIKZQLvrUURAzQhcdVT6ZVLZKmiZ0vDJ5W852bmRSptEhHK/n5ohpBhIhNZ5ljZSO+jH2ZyC0WISMr28rAfclkv4gfGE0k8IdlASA7ky2HxSGyFLAEl0j5yAI8+RDpL91B+OYmYvsvJj3ottp+4J/IDP4TqcE3EB9cp9o6vHeTigDOdJYQGChH31tKRLL2XbkEhysJvO89nP54Jzz9HYgdVwBWII2nSciO4DlqC5BXCx0yu7u8qwVCJyehQgbqO8dRtjpVDnT8BZHo9okrStXVdV7498t+eOlgub5M/iYmlTT+HERmQJ8Bz+8JPvh5fvPhcZ44cAPcciUMCkByoN+NNlLG3LCL40NZzA+vUwDJDJyLUHLvGqL7cgnvKyLeV0Wytwqjv4j4gMvqtMESooNFimKDLrS9BcQHLBIQRQdLCA+WEdpnkX9ISr3LiN1VAz+5C+Z+CvE5VWQnTq/Id6nkliWRRQEobSDOpvRwRhxhXUlwicwlUjpGUkNPxRFzSyI5MuRLcmEOOf3hbJf7ZYXHptruL/1bhI+Ej6XUPGaQihsqfK4nU8iiQ/KEihclepZIk1RIF7SnIJqBeFqZPoYCV8oSCoFZePS7ZD49RnjyCuJ788kMnAf9b1JgSQ2st9p7oJLA3q3M79uOZ/92fH3l+NrWo7WdR2ZvLsbERmYPvgX/v43CyR9BJqiESzSWsCbhcBpFAdriWcWTTjvL1uFRtSti56yGtnj+zzPnOdIIEuZLRHzWRMpGkIweJRQHX8Jad0KTq0lvSLw9IYVkMjm0tVCNDGlVK4DZW8GQREaU8ycxBzt8n5Sl+WxGERvbuaSEcWUFc7VMmJQTqHU8vJCYA/0EBJ6EmZ/CE1+HH3yI1OeG8XygnpMT7+TkwEai+0tIDuaTHCwkPlRIdL+L0IiL4GghoZF8tOFc4vuzFen7c4kOW8doQyUqY53oySLZl4e+v5zo+EZi4xuIjZYQGxJgZMFQLum9uUT6i1jqK2F+oJyF4U0EDmxn7rZrmPv6nYR++6AFYDH/TA0jFkXTk6TsUv5lIbTcYbYQULkfQ6U6rVUUrVYQplT9smIrESvhCetazvmy1LSMo7BzI85WKjdtyiSd1W1F4NjHiaZQJHMi2ReWrf2gspF7Oc8hW/lbfSQULLNgJERD6giYxPSSY6TfVWjNG4Hjz8ETD+O9o5fZ4Svw7CmFsUoV2ZP2jPesx9hfgXffpfgmrsI79lY8feWkutfAwAUwlEVsbw6nRyvQ75+CpafsnE8aM2pYkb+krpZ6ULP5y4vI/QUrK7MG0lRKP69MPC+/qv1Sf3pzjrycNL6KMZuy4IosmSXrOFg2pyzMIq6gxdEyCCmo1vjLxCIktJgFBFuxpFMmYhsSF3tCA01UtzSmbkkXZxZzZ/bymAf96E+JHvkRoce/jf8XX2PpwXtY+vYdeP79MIEvj3Lqrnrmb78e7/veReCmywkf2kZ0spLoWCmx0SKL4YXpV0H6UCHJvhxM0RLDZUTHNxCcqCI4WkpoqIDI3vUw7ibRn6Wknj6xhfmRbfxh/3ZCH2uFR7+szD1JWjqfRCJBLBZD0zQljZ3vX3wrvWnZ/zJ+XiJVKxnTAYnDoMK4Vu/rloMumeWUQSadIiMqSqSVSB9RVbL5c1slzUTQ2bafLXUdPpNnEZKntLFjD2Kzw+xqQRxL0zjaRiaxFPmZ8SVhdhGO/Abzm59k7uYmTvZvwz9YRmaiBEbzVHLRN1BFYPhyAiOXEhnaBMMlMFkEY9kk+y9Q/ey9/Rq0733YWutEVqtKGKTj0ioy1WzMjmbZAHEALw8u+zZABB4yq7z4U897nxfvmOVvzxEAyMupj7Lzg6QTIaJxHa9usBCPs2RohBMBjKQPM+0V11I9WIogaWbI8BzwDKSPgnEEor+DwOPgeRROPgJP/wh+9Z8gSwR/+cNkPn4Lidsnid/aS+TA9cSm30ls+h1Ep68gOnUZkaltxKY2E5ssJzRWQHgsh/D4WiJj5xMZfxORsf9DZOz/IzL6RqIj2asChwBK2+9SmeDEUBnJkSLi48X4xzewOFbJ4nAxS4NuVQ+VGK2AkQLM0VICt1xF9L4x+O13rYnolJMr89IliMfjpFIpMlK6ncmofadpX3wrjCzhSyHLbBXmdCT4SuZUHa0Uhy3BJR8kmfCMwMoKdr7YViw6iY4JU8jvcm2HrCSb46BLyMAyQZbVuxxonWidZONOvhKRICR5KusJFCdawJRdeXjh4XgKluYxHrqfUx8Z4pnxK5kbLCYxlAWj64n2riXen0+mJ59kr5vwyDYiBy4iPOrC07de8cLMyEV4b3sP/PgOCB612wuloSWJLIsiSTDheRpkBQqsuj4Bk5AVKZRXUzLkxTtm+dtzfAYE4qBpcUjIEgGiQXyoiZ5lXTtZjEXq95NnVHUskT9A6Aj4nwLvf5M59XUyz34W44kPE3zkfSx8e5JTX+jj+CdaOHlXPSduuZaZm65h6dA1BKbeTXT83ehjV5EYfhfsv4K0lB60rrOofS10rCXTfSFmz/kkxLEeW09mYh2pqTUkp8/DOPBm9ANvInbwjcQOnE90NIfYiGt1NOzGGKtSlBgrQRsrwj9RwcJ4BXOj5cwNl+Ef30RkfBOJySrSH7gavvMvajkzlctIWhJbgBGJRBRAlltWBJiYpC/1UWgQ6b1CggtzvZBWMqpjPigT12JKh4dfbCsOdixpoKVSyq8xMk4pqMXcsqSoQUwZuSJd5Vbq/nIxYXKH5L5iItt8L5aHkPxsiWo7BJ0W/8qW5vJ4coK0kyyc+uzPCX39Vv5w8F0cE59utABz34UwKIGUtUqTL+zfyqmxS5gbrWRufwmBqYsIT28jNl1B5K6rMR7+FETFz4OIWPzqGawckWJ4555yXxsBFkAERJJ0PeuHOL+/VBedE08lEScrbUSs9fOiJyzJ/8y34NHPwoN3wPdvg/tvIfXlQ0TvGWPpQwPMvL+ThVvqWRi9FN/wJvwjm/GNbVQMFZjYiH9qE/6pjYQObyZwuArfwUoWD5QyN1nEmYkCTo/msTCUA6NlsK8Y9hXBYBH0F5Lqz1cx8uhArpLy0RE3oVE3ofEiAlPF+KdL8B8sJ3SgVEmaVwKQ1HgF8fEqgpOb8E5swDfhxjPhZmZqA3OTGwlOVOKd3Er0I+3w8/vAd8Raa1ySe7bzqurCEgkr4ma3tmiUaFTc9D/zkU50ACGdqrhcfAMxfSSDbPsJDvfLMSIsbV9PggGJjJVDEfloaRFHq5hKeyjZmZHsQep5f8tKHWFSyh6I2KFcuc3zJPELASIxhhf4RkozpXTSMlow4YOMU0Wlq+XvTPFX0mIO+sHzFNGHP8cfPt7PsQNXkh7Khv1vhulcMjeWEzq4lYWJS5mfuBzPgSuUsIrfUoV5sBD/SCGhuxvgN99S682L5SPPIs8szaIwYfu/0qwWCXjEz3PIEir2j3+mc+Ac9VKiNdIRCBwn9N9f59i9+znz/muIve8KEoc2Yx7YTHzqEiITbyE49jb8I1cSGroMfe8WGK6CoVIYKoEhqTsqJL0vj8S+XLShLEJDa/HvX4t/bC2+qfV4D67Hd2M2gZtzid5UQGK8EEaKyIyWYI6VKDMmNVZOcqxSRTK00SoiYxsJj2wiNLoF/8hWvCNb8I5sVcCMjJagjbpXTfGxMiLj5XgnqlicLMc/6cY37cJ7eAPew1uZP7Ad874BePJ7EJoDQypKlz0Ba3KOdFqZVIq3bPPKMbNeqgfOduJyb2LNdCdOthTnOSTOtQUMcYSFKcR6ERNHysPjCYmAxe0QsZSKOBGbF9864WQtIwCxAjGOLyq3UcyzUhLb9xZQOiFm59klEKFAmohjGCFMQxYp8trl60FSpoDFHs8hyU0zCAkv0T/8kvl/uw3/eBXpobXo0wXoN1eg37gR7/gmTo9eyszEJSyNFhCbyiE5uZ7wcA7+qYuJfX4M89nHl4fjOs8ijynkyBILOKI1rECI1WK2KeYIJTn5JT7npPynSIdnIBmA8Ck8D3+e37x/J8+OFmMeLoDRN8LIG0mOZhMdLyI0uYHQpER7yjAmpPAvm8DeLCKDOWj78zFGXSQnXKQni8hMuUmM5dqUTWIsG2M0i8ToOoyRtcRG1xM56CZ4qJjAwRKC08X4J4vwjxcTGCkjMFKhIkcS3ZDx0+GBzeiD2zGGLiO1/0qSIxersG5srJDVUGSsSDnkgZES/KMFeMfy8EwXEjjsJnmzm+htF6F9dRKe+xnEJaMLft3qbmlXMyVh2hSmaSpSOZuXaOwX/mR1rGPDixx27CdhfWdfutkyfWTPAYcDEHVN5b+IDyN5ColWSeTKIJOMk9AiJPUoqXjMmjBBQrLyezqpQC3Xs3wJxyKRu8i9rXs6v8tWgUcBR7SaDV47IJYwIW5CTGaXkSy8MsAkT7IEyNqQHtLxBTJSjStMLNp39jm0f59Ev+sqZqc3sTRRijblUn6nRK/OjJah7V9LYug8jLE8olOleCY3MHvr1cS+eYe1arDSThaopT3l6eV9HLJmQHkBQBzwy/bPAURlZePyItKwUcK/e4AnPtzOsclyuNUF+/8X7P9/SIydS2wyj/DBUsKHK4gfLiFxuJTgeAH+cReBCTehCTeRiSKi4y70CTfx8UIFltR4AamxfMyRPFJDOST3ZZMaXI+2fz3esRwWpnJZmipQzOmbLiI4XUJ0qhxtagPahPgIm0iObiY1shVz+CIYvkRRemSbKgpcDTjknPC4G+9IMZ4RFwHJ8I5egH86i9BNbqLv38T83deC/7dqzUDhKbF5fRlL4lpdIZ2dUqaVOOcrASKgEd/kpT/SQ1bnOSsvWdEWcTytDLbjBJ/VWfYKthkJKftAPw3h58D3NMz9Fk78itQfHiH++x8TfeKHJJ9+CPOZn6rv1e/+oxA7AfHTYMxb0UUpo3cYXkwhO6sugBVj7awTLuFKu6hSwsiqzN72n+xKY9FEEviRtXZ1Ff4JQnqRdPy0BZC0rFNoB4bk9aNPwX99kOMf2MGxye0EDlWg31RM+MZStEMu0vvXwPCFpCbdyg9ZHC9jZnIz0Q/VkfrG+6wlLdSzW0EOAbGAYzmqpt7BgYsEQ+zkohz4lwBE5a1slS0Pju7H+8h9PHf7e5mfKCA2/Ab0oX8iPHIBwSkXnuly5ibK8E+Xod20geBUGcGpCkWhyQqEIpNliqITJQjFbNLHixGSaJFFbvSJfPTJvGWKT+SzTOOFygQTMywxJtrITVJotFiR/C1AjE0Wroqik26lrbxKjWeRPHgh8Yl/JvIvJfDgYdCetMpaZGJ4ex1ykaSW0HEkvLTyS38EPLpujWER08v5SA4jri2RSPqUByDgEEhJ5wqjiQcTjmtkpP5MCjNTpyH0Gzj2Tfjlx0g/cCOBTzQS/WgN4bt34L/rOjwfeA/zt13F7C3v4Mwtb+fY9GWcOHwFc7e+E9+d16J9tJbUZ3bDfR3w5X2qnJ2nH4KFp60aNwmjSoLWtH3S9JI1mMk4qRK1GVl70IgSNNJEpTRAluU1ksrmk5GAQUAMLClVl9IaaTe1aKdjZknEzp6pRDGoVEZEn4Hf34/vi8OceP+78N6yidShtaT2/2+Sg2/CHF5HeiKPuPieN27Ff+M2jOkNykcJf+cTVhY/EcaMSvGqNf3TYjyCpoSM5YNYsEla95YucwDidMaf2J4jLyDk5DvEZDCO/YLFL45wfHoL4YlczIn1JCfy0cZF/ZUpM0tCsGJmCRhCU2VEJmxgONtlkJQRnRCywLJyG5sQoLjPgmA8n4RNFkjyiE+cBY/Yos+jyTxik/kvCQ5R2S9F/uFcQuM5hCbWEphah3nXBnigD05/H+ILymwRwSFttBwOtxX4yymWE5CIMy+RLdEulo8i/kJSMVw4GkI3pHpXACg9KEwXAP8JMid+jv6rL+K7/zBzn2rm1O3v5PTN21g4UI55axXcVAaHy0kdLCI5VUR8sgB9rIDYmGSkXepv0ebaeN7y97GRPPxjpfhueQ/zd7TgvWeU2P13wi++Bsd/CsGnWewSOQAAIABJREFUrahl4jikjoM5Axlh+6gan+PLZPALMGI6BMOkQgaiMB1eEpDMZ2TuEqvd5L2U1lUmoJPIdELWsrLvKTVoau7rh/j9wUtYGvg/MPZPcFMuifG1aENr0MfzCB3cRHi6HCZySB7exKm7u+Hx74MmGnEJUrIMuEwimCaUljCEFbZQ7SlWkoDT0RxnZdWfgAeoRKG8lLyIkPrI6Lxf3Mfs+68ndnAzTBXDaCEM5sCgC0bKYX8pZl8+KQmPThYRn7C22qSzLVLfy29/mkrQJsrRJiptKle5j9hkKdHpEsIHilRFbuigi9DBAgKH8vEfzrUpm8ChXGLTBS8JgJcChzBScjKb6NhavBPZ+O66nPSDN8HSTyB5gozhU1BQhpI0pmpQm3ntBJ/V2k7D/fFWolkCCgGIbJ19OVKYRq4dk7XSRTWpS8cgeAaO/hfmw/eyeO8w3o8147ntXSxOb2RJqgRGckiM55GeyleCy5i0BIs+nqtsdW0sh/hoLrLlcCncWAIHizEPujGnXDjHByeLmb3xLZw4fCUz0xfhnd6Cfsul8OHr4EsD8K33wZmfQeiYmi5GTC0pwg8SJYIMWJIKWystYbstjnqVOfsIJayRJSJYxMhRIJE2VOZZ1Brbk05jZsScFDEUJn3mlyx8dRrPwU1oI2tI35hN4uA64pNr0cfFhy0iNV4E09nKxD86fSVz98mIxAdAgVnK8wXESfyGGHp2vylgiI8mZDvqf9xdf/TNOVa1pl37lcpYGVnJpi88hv6Nmwh94N1KnaWH1sPgBTAsodkC2F+golWm+Bc2JScKEDImz5IwoZA2ZZEw9DJNFdmAqLSy41PlyvcIT5cSXgGQ8IECwgfzCR3KJXQo26b1hA9m/1mA6NNu/hQZ0wVwcxHBiRw8H3gLPHirFeJOLZJJBkiZcdWxwruKeUX6qQiRdKZjqyvD9I8a1vlC8iOiORz/RLYCFkVSniP952BOxpk/8wsSD3wC/z19LP3L1fimNiFCJzNZCFP5MJUH03lwoAAOFpAczyE+kWMx/VQeyelCktP5pA64SB0oIDFVsPy3hErTh4rIHHYr0Ji3lBG4uRLfjeVEDxRgjGeTHlkLYtIe2IL/pqsI3TuM/zsfRzvykLUwJ34y+EixoMaLCOOr9hHGl3yORN5kdsSkTsaQ3IP1+/OOU+UqkuiU8K+pJsEU40hpaIl0Lf0GfvAvRD70LuZG1pO8OZ/MbRXKWtCGs2HKBYfEesjDe/PFPHvL1YTvPwCen4J5EtIe9ERImaeqH6R9paHl+eTeapIQK8Ln9NOf2p6jXkYShAkdMx5VJM46yVk48QCeL/Vx5raLmBk+j8jUGzBvXoN2+Hy8h9YSuTGflHTYuE0TeTCRp+xFczJP/SYdJIwoYTztQAGxgxZFDhUgFDvgPksHC9EOCBWgH8jDmM4jMZ1DaipLkTm1HqH01FpF5lSWurZ1jnPu87f6QRcvRfFDbvzvu4zQNw/D4qNKikXSCbwJqYsS7rWljlOEqVryLwfICxteTCtHm6hybcl1BL2YR39J8PufZunTQyze9l6CUxswhBluzIcDa2HiPBg9F/avIbN/HYmRXKUlBBzxqWzVVsmDeaQOFZK+sZDMTUVkbnKp743pHLWV4+JTuep4fVLOuwB97P8lMfW/SRw4n8RNBRg3V2Hcsp34TZeh3XQ5wUOXMX/wck697z0sfWEvyUfvBd8vgTNqftyQDNu1eU/5LhLmjS9ZVdcyMExK9O2aNAGSEgiqUSypEIj6kJiXaFJxaSxTSIYzHIHffZngnVeRvHUzocMb8U4XEz6YS+qmfNKHRBMWwPu3MH9oI8++/53Ef/JBMJ6EzDwyGjMptWdyTUGpyDYFEvlHqgYkAGFVD7ywj1b+fQ4pAYc4ZGEy8TAyDkBFtERCmrPEfvkZTt1Ty7ED2YRuPI/UbRcQuvHNzNy4Dv9tJYqRxYESElA4JMBJKHo+QFaCRMAifys6mId2MG8ZGNKpFjhyMCdzSCvKgoksGLdIvhPwvRRA/n/e3gO6ruu689b62swksS1ZYhFI9F4JgBSpFpdxquM4TuI4zngmseOZzKRMkkkmK7YlkUTvAEH0QoIA2FQs23KXYjuO4xa5SLIkq8vsRH29l9+3/vvcB0KyLUfkzGCtg/vKfe/du/f+73b22UcA+GlD2biLLXcQvffvXNOAxJLNtCrQlEYT482tMq2z4T14BJZ2FISMq5tp+hMfK+4QMF7xlwzAc18i8+URLs7+Oc93/ibnD9yB/0CTaX8+tgP+7kb4nze4499tJfP3BaQ+VkHi7hpi+2sJfqyUwN2FBO8uIbS/mMiBcqItZUQPVhBrLScukLRVEGuptOc65t6PHyyEljeTPfgm4i03E2wpZf1gLWsHGlm/R9a8nszBRuJ3KdOnJQT1vNT5qyyf+Esy/zIHF75pzTKcKlHZiwJ8zXOsQUxLE/TcZZfMs/KsiYDiwJIhkgkSS0eJe3OiTq8nHG3TF+HRY8RnPsgLd+3j/D2NRLtqibeVEd9fDa31cLDMUvLP3r2Lp0beR+KJU5A556JGscbhwf2gQKLrsdIUxSeql35t/l1nlWWZhMuZJxLmOMjUKS4x5q/8kMgX+swFibdUQlsJyXuKWLm7hrXWZvwHawgdrLQROVCJRnT/lRG7pwo3aoypYuyVUUXi7ioS98ikv2rodRubz68jcXcdybvcMCG5p+Y1LUT8YA0/bSy338nlEx+DJ78IMZXSLBGO+S17pOktBZqiw4YLYb6ry17JZ47napdeIfU//kRxh4oXFY/oT2BZWVkh9NSXiB95L/RWE7pnB/67dhBuKSPRXUeso45AazXLd5Wwur+SYGsdkY5dxDp2EWmrJXSgytzOtf1lFospPlNMpqPvYBm+gxV2zL2v5/6WcgKtVQTbKgm11xDqrGWltYLltjKbvF1rKcR/sIDAgZ2EDuwkcqCQ5b/dRuxAFfTshZ7biO/fbe5w4uBt+A69h8Sj04TOPALhZ7117EpbuwIAzY3YxKJueqOCO7cAK2OrK6XJ1YQhHlGdX2xjnkeTj6aok0skHr2XlZkPstZ1p5tMPFhF9GA1mdZy+Nib4WAeq+31fL/jLTx2/G8Jq/5PYLVCtE0+3gZABA7FKWGviu3HeZZ75Tp/yN1QOu1KTsRCmbt1FeXqiUozH/88q0f+C6udd5Bsryd9sIrwXXVu7uNgDb7WSnwtVfhaagwwAk3wYJUxUYCJSfhfIfB1JO5qMGF/NUB07mZQRQWAe2Sp6ojsbyB6T8OmY50BUlozdLDcgdPOd6DRb4dbq4m0ViNwJw6WE28pJ9xaZ77rM92/R+Kxh12aUKXqcR/B0DqBVNqyesvqGu/R44olcbZapDFxlzR4SsjccM+i67H9STCSKbLWmCFmWRbOP84LDx/l8dEPsL5/O9x1HRz4t2Rb30yybQfR9lLCHbUEOxpZ72hmtXMPa+1NJgTrrVWst5ThaynCBLqjkvWOatbaq/C1ubHeVk5uBDqr0PB3VNr7662VaKy1CBiVnG2t5XxHLcudVQS7y4h17iTevo1YyxYi+7eSaCsmcHc+vr8vspQ8B13GjL8vx/+3pTx5cB/Pz3+I0LcnYPnrNuchDS3ayL5KX28EyoYT5ZVkc1RZmySuiT4VXmqZg601d+ltNbSOJNLus5EL8J1jrE7+By7cs4fQQSmJeoKysAe2kr3rBuhvYqnnTr7R8hYufL7TpY7Da+4idCGObRvunmZpsqb+XmXVc3zzjtcJDLoZVw9qmXd3aznG68sTARKPf5zHR97PSx23EOtugP0lcFcByZYqp9F0we27CHgj2F5PpK3GzHuypYy0xsEKktLoLbtsDiXS2kS8tda5AXIFNkYN8dYfH5pA0vkbx7Yqou3FRDoKCXYWE2qvInGwieSBZvt8qKOCpdZSIr01JNuLyLTuINmxgzP3lPPs4T8h/O0vuDb7tt5bvrKIdWXWWs82GGwEExLcMACozGI15NJQXgmIKKg158FU0vbCSMdyRVspiKzC9+7n8tQfcv5gFYGOmwn2bCXYcxOhru1EOgWOYosFRKf0gVrSB+pJHqwl0eJoGekoJtSZT7B7B76efNa7K1jvqmG9qwrfxqgg0PmTxuZzquwzq91VrHZX4OsuJdhVSLRjB4n2PJLteaTbdpBsyzeQyE1ztN9FtKXZSkKiLU1mXaQYz/XcwYXjf0zyiaOQesqcVFkE0UmeZUpC5imSZEa9zlQR7pSznaS5l2zcQCWrLbmU6Pm1nkWPkkvw1KcIHvvPLB/cQ/hgLQw1EW8vhJ5yaKm1xESwpZhLfXtIffEeiDwJkRDZtRgRrb7wem35jLFayeb3MiSvQsWmp7ZgShdyBSBrriGARfwuKWGB1vJTXPiHQzw19E4utdVDWxns30HmYKkJtsy+QCGA+Dp2EeioNYGNtpcZgdMtpQgoidYqoq31hFqbCbU1GbiiEvSrGe1lRDrzCXXtINBVTLCjinjLKwHi76/h/P6bCR7YQranEH9bHi9338rlz47DyiWX1dhEkNf1UAwPu10qxcdg0jWyE4OV4wqq4Zssh+q3tCXyN+9nfeqP8Xc2Q+cW6L/BgLHeU4yvu5hgZymR9jKibRUOJC1VZq1NsYh2baXEPYUQ6nRKQdbB11lLoKPGhmggxaDvsdFRahZJVkm8yA29F+oo2wBSsLOMiM7tKCTRnm/AEDiSbYXud9sqTOFJCfram1lv32v8o10ZpUJz5852N/PS7Hu4/PB+ki9/BmIvuSJFT6/YOvtE1lZPJtTAIhs0bFyxzq4qV+DQkECvqdOKiCsARV6C7x8jevQPWWm5Bd+BagLt5QQPFpFRVq+1Cva/mfWPXk94+l3wvWMQugShhLUz1QLnFa30FGpl1mJa3++h9qcwfsOC2Ps2kSJz57Wdt2rVlNXykF6Di9/g/MJf8lLb7SQ6a6GlwPLUsg7S/rIYwXYxS2C5ApB4WymJVoHDnScLIOsRad1FtK3m6sBh1qPMNJ40b7CjglC7rEu95celcQW61FA9F+/eynrrVoLdJbzcUs3K/B/Dy/9sltHSfj+FOP+ql0XfpDCgYj0tNHMzuaYBVfekVObqM2T+aZqzh3+f8wf3EO+phwFljPJItlQ469DqrlfCHegsM42+3l1qR1+XE2RZSCmieIvucZdZy2irrPEuu29peN23rLpAJeXFwWJQMH5AI3/T2GHvyVLLXdH3ihfio7JZOX4Zz9pUSLiZv/WeEqwnoc8drLCM5lJrOc8dqOCp7lu5cOrPiH57DvzPeVbatTbS3jnmlJAgquSQVUCahnbk1ky7uWAu/ltVWbspa7li6mTztLlbvpk/4kd3NxHpamTlYzutRCjTWQWtO4jdvZPVrn28NPZBOPeoldEIDz6to1e1SQ4gWuJqrsBP5/R1uhxZHPsTsy01JofNXWhErT9jEYfg+EUyXz/GhfEPsNzZRORgMdnWEmgpIdVWRqK9nGhHtfnPCgDDHdVEOyqJdZSTaK90o63aiBpra8BGey2x9uqrGx3l5jPHOguIttdZABdrq7PvT7bWWnYj2V1JrLcUf28BL7SW8MLgu+BbRyH2sosHbG4jR4DXd8zRWRv4JBPSd45msWQIdX20tjzrPyTw5RFeGHgnZ/dXE+2uhqF6/N3VXPhoIamWBrIHGu1owtZRTaizEn93OWu9Zaz1lrPWU4m/q5ZgZwOh9mairbegIDlxcB+Jlt0kWptJtjSSbGkg1VJHVu6H5joOVMKBcpsz4ECJc4v3F4GNAns9c7Bh4zs2f0/C6FhHzPhT6/FVvNX1uRHsrMXfXkegrY5Ie4XxOXCwiMsHyjjXfRsvj70P/5cOk9I2bEnFGBmLm0MpBeqebHoAkQCbrFrHlpAp+JxLJKoaLa29rEpTnod/WWB56kOsdr2V1YNVrHeWEuoqIbq/kExbPdGefTyx/y2saf1I+Ix9uRZ+ZjX772UlfxY49LPXuWpHl4qzqzKnW0hxacxQJkI8mySrzmJqjLz8FMmHB3l54Je4IC0lgBwsgBb5+CUeSCoJd3pDgOmsNALHBBIDiKuWzT2+FoAkOneS6Cgi1tpMrHU3UWNoJQJIpqXatEl2oJKLPSU83Xcrvi8M2T2QuAgpr0jz9eFi42wx1NyBTMra0rhIUAyIeKnOF7n8xT7OzPwel9oriXfsINtdQLgtn4utNSx37SHc3kiitYGcQG4+CuzR9gbCNhoJdjTj79iNr/MWfB37CLbfYvctcCRa3RBIHFAcWDIH67DRUmvAEXjcqCZ7sI7sgWYyB1T7tM+GyjcSLfre3UTbmu367Pc7aj2elhPtLCPaWUKoqxxfbyPB3iYHkLt3krpru1ksxSyXWhv5Yfcvc+nBj8JLD7vFd7h17FpLY4pZRPQeO4B4JfGeJZa7agAhSSq86rqvyN3yPQ/fu48LYx/kcvcdrPcXEuzdYUWvvv3NZHpvY7XvLTx39I9IvPAZNx8jFNr+h2tuojYHyg2O/viD61wkL6Z6pkRXrce6IOKEslpxBiGVsvojbrHQi1/iwvE/4+Xe20mq4rc1H1oLoa2YdHspSdPsFUS6qgh31diIdNYQ66wh3lFFsqOCdJsb+ny8Q69fxeisINlZaL+Xat1NsnUPkc46Yp1VpNrqoKXGViKGW4t5oaeWix//b7CkAsSwt1xWMx6yllf3J3qHbLbdBfZJdSuRo61KhMuPE/nnKX44/C6WDu0h1qfrvJHowZsIthbh67mF4ODb8HfvJty5y1K48fZ6Um31ZFtUHqIcvzumW3eRbNM5TQS6drPevZvVnj2s9zQR76izCcF0Ww2p9hq772R7nWUb9X3x9l02Yu1NvHok25rg4G44sAe07ufgXlIttxJvvZVI+62EO/YS7thDuLOJcFed8TPWVU68q5h4Vz6R7mJLgvh76o23sbsLSH30ZtifDy3FxPYXs9JWw/Pdt/PywodJPn7CGuhp8ZTcHO2zmrMcOtqfqollRWwSz4FDHR8jiaRrqCehlnyqgjd+iew3Frk4+X6ebdnGxfYtpjwCrXeQ6GgmPngrz/btZfWLH4Nz33OVy+qpFl8hop3Jchm23G//hON1Lpz0lkjqKjWsVMXlq0PeSjMtLU5Z0VAYwi8T+5djnJn9AKHOJmgthjYBpJBsezGpjmLineVEuioIddcQ6qqzISJLeAUgAcmBqcJe0+uvd8QNIKWk2itwQtREpLPeQCIhsaCtQynREi7M/hbZpxe9+EqqX9ZDSdxrAYgaFsQ2Mi7GOJWOX36O5FcmuDj2XlZ7dxEfLCbRl0eo4yYC7fnE+htI9N9GsGsfa917We/eY3TUHEe6VcBW4FvhuUflcLAcWitMGYmGEkh9r7+nhnRHPrRth/Y8aN9JuqNwE/2dggp0y6XbZUOg0ljrabLfNDAeEBh3kVWCo203sfY9BoxQp65rt513BSClxLsKSXXuJNmVz/KBbfjaipECTHU0uMLJllrMpTuQB4MV+A7czPP7izk/9154TIHzC+ojxLpfcyE5C5GTfEmpBND8IZsLUTtWvysYNqsTVY1XDlnhs6S+Ps6PRu/kTEcJgf47ifW9zVnezkp8PWVcPHwnwS90w+XHbFWj1qRoN6t/JUAkIF6K0wq6nN3R78tPVMQvXzDnM2ZDPrde/fJ3OHf/3Sz33EmqvYp0e7mBQwBJdxSTVEamq9wAEuiuQ0wSUGRVBJBsuwOTgPR6gbH5fFkk+31p0LY608ahrl2mWdNtVXCokXDvHmKf/xisP+Z8YNMUihY1HbgRgf0E/fHaL4lGIrI/rZy+Z3mjQVJfe4DlsT9iuaWRbG859OWRHMonNlxOeKCBgK6vvYlE5257LOENdeWArRKRGgeSlgpXaqJyk7ZSUh2lxEXX7lJCPToWQsdWaL8B2m+0x9mOPBNeaXk7r7uCQE+VgUmA2jzCXVVk25QidQAUvaRYBFRZDSm/QJfGrldYkGRnManOfJJdO8kOlRnoV/aXEm5vJttzJ3TeCm210FHkZeu2kurbwaWeal6eei/+r4zD0nPWTUV0Ew0tHthoLqHn8mpUzRG1OrbcBIR6GKtNlarP7XNKKCWeg0eHeHn4VzjX2kS4u5FYdx3J7nIYKuXiR29ibeJd8Ox9toZGa22UZRQ4za17DTZfl7swd3ruY1dWk+kiNPRlTpSEbK+kIPoMkfkP4ZPpb62GgXrSncpW7STdXUqsq5RQdxUCiLSejuHuCmNytiMfjVhHGeHO8qsaSgCkuhqtvit2oIB0Z4XNxay1NhLurCfZowRAJckj74FnPmlBuXzai+pkbskI3ZX5k69Bop/+lughJ031SM56aFL1q5wb/nNSg78BPU3QkU+m62bCfSWs99ey3ttMuKuZTFsDaNXigTxC7TuMVom+GpJ9TcS7mwm2NrC+v8YeJ7oaSHdVkeosJNN+M5n2G8h0/Dy0/zy03QBtN5Jpu4lMxzZSHZrr2WlaPiZXqLuEaGcR4bYdBFu2ET6wlcjBrTYRmGrdBl159p2a95C7muqrRNcR7Kpktb3ceGZ866rZcLGk/OQl6PxYdz7hnmIDeKBzL+H2O4m13UpaZSDyKto10/3/km7/OeL9hVzuv4VzY79H5stjth+I5E9S5/jgLIr60qXkQtkut5I397oBwnNyVAakoaJEsqrdet7iy5WhO4n3lZEarLbro7MADlWS6Skhcuo/WlcU/d6lgN+aWPxsgEg+7Cx9TDiVCLl8tK039laKCekK6LM57Cn1pqzCPw4RmX4vSwoauxtJ99aQ6Sol011MqrfMABHoqWG9VyCpI9RdgRgn10Aj1lFyVeAwUHVUk+7Za5mWRPsO0t3FRLt3E+y5lWDfLgK9Daz17IaH/hYuPm7VnBJoFUSL2BllM67hT59e9poMWhXrxSfJPjRIeOB3Sbfshu46aC8i0V1CoK+a9b4GfL3NRDsbnYZtKyPdmUei52biPQWEZRm6a1jvrGe5vZHLrY3Ehn+RcP8tbib8wA4C+28k1no9dL8RBncQ6akk0NNowbK/bzf+3lvw9d1CoG8vgf49BHsaCPfUkuiuId1dTqazyEBLWx60bSF04I1Eu24i2b+D1EAB0d4CAp078XUU4tOkZLfc5Arjo1zmWKcUXLl5AbJmod5iAr0CUj0BuWQdil/2uoqL9lLo2k6m5Q2k2n+BdO92/B2lXGrfRWL29+Gro1bek0jEyCTlp7j2RLYNQyJFRk0Kvepbiagstv501NmSVFecouW/wIUfwmf/gtWeUtZbtsOhCtBE4oGtZFpuIDhxJ0vfOkI4eMHm8TW/8rMBol8zJSqAyCf3AiRVYOY60lg1pkCTm573Yhah/OI34B86OT/8W5xr20usrxH6q8n2FMFgGdGeMoK9lfj6am3ocbSnhHRXgQ1ZGRH+qob83r59pHoarYgy3p1Hoq+ZyNAd+AaaOdfTxNLEb9rkkm1Kk4b1jNv5QgBJZLwNQD3Cv96DeLLkbZlH/DzZR4aJDP0WdN9maxfoqUPaP6R4oW+XgTba00CyS8CpIdtZSaq/iPhgAdGBIhM2X5c3M97TZMK+2r3bQO6XsA80ERxsIDBYjb+vlIt9jVw++iHOHP0fNs4e+2vOzf01F4/9FctH/4K1o39CYOYPCYz9Dv6+X8anyb2DSgTUgaohhhoI9exgvXc7Po2ePAI9eYR68wn3FRDtKzZeiYcacVn/rkobic5qIt3VxtP1vnoCvfVEuutJdNaT6qgl1VlNQuf2VeDvyDfQxfpLCHcVsX73zcRbS4iOvxOe/gSsv+S2V1PK3eaO0taFUzsebHg43ky7BFp013DuWYqQukSav5uBHz3I+vHfZFUKoDMP5NJ/JI9Y+82sDNTy4vT7SDz9oMm5Xw2wfwbTr7MzDEa6misWxC5MwNmwMLn3BZKUazOpb9e65pcfZuXev+OZjrey0tVEqq+ajPzjoRLiPUWEe8vw9TliCiCRnjJS3QU24t1lRLsrr2pEumuJ9ewhOdBMrCePcOcW4v01RIf3cXloN88PvoXApz8CF9R0IYlaNkW9mEoaKPKvXzfzE8mo2zf3QAHlS//A2ux/InigzhV03pVHtqeGUE+TaXRpdwEl0V1FuqvSwKGj6p/8A+VEhmoID1YT7K8k0FdBeLCO8OEm/MO3sHJoHxcHbuPs0Nu5OPluAif/mNRn/ob0l/tg+XFYfdEbL8CqxnOw8jQsfx9e/BI8fi98aZDU6f9JaPwD+Pp/g/WOt7PWuY+zHRWc7ytldagS36EKAkNlhAdLLLGQGiol0V3gjRLEK4HE8auacHc9a32NrPU3EOytJdpdQaqrhHRnCQkpvZ5KQkNNLHXXsNRZRWigllRfubl0tLyJTF8Nl+Y/TPrpT0HqsqvgCF22tqZG282pYC3BSIdMLC3es5WCScLErIe1lbLY9N2L8MMp0vO/xtpHt0KPJlQriQ/UstZfy/nOBlKf/4htnaFCSc86/ET+6kUvBtFD11XDlZyYSXHoNYBonkSi4LLSKhE25Jnd0z5350k9usjzY3/AGWVAuivIdBdAXz7JXmmiUoL9VajsQ8dIXzmJ3kKSPYXEesuJyqpczeipMdMeHdxFsDuP9Y43E+0vwD9UxdnhvTw/+3vw9AMQPKPmUWSEDrOGyj5lbYbW7uOnkudnveFlW8IvEvxcK0tDbzd3KttyI3Tnk+gpJ9DfjG/AuTvR3l0ke6vMH870FhHvreJSSx3LvXsJDu4lNCRNXMx69834+rfhGypgbayWlZk7WT/1BwQ/9zGS356H5/4Zln8EIc0uez6HbmTzkIq1Xsra3FPNHc65FPczD8PXjhH+ZA/rp/6Wx9ru4OXBO1mbejuh6bcQOiw3rdw0vcAhPmZ6dprCS/YUb/BL9W3BvgazjP7+BuOfeJru2Umydyex3mLjdXBkn9VGXehsZqWnkWg4+/l8AAAgAElEQVR/FdnefOi5AQYK+VF7E2v3/xU892mIvuB6Msd91g4sF/ua/pYFSav61lkPFxXHrdG3XLJkWP2lBR25Pefg0UGCw2/D39ZoMd/64K2s9zeR6CojPfsr8MRJW1hl2bLXYPN1OQTphzcPez2XSVCmwIYKykT5K2ZOKTerk1n6Lv7PHOTy8C+ZH20A6dlOujfP5gCkGX0DNfgHqgj1a3a7mERvsZngWF8VVzOiskrdtUSG6vF1F+Lr2kp0cAvLvTfzo7HbWfrsRyHwDKgSQK37NVQ5l3VrloNZr1r0NQj0mm8py6LdWZ/7JC9O/h5Lw7eSHa0g1v4GGMoj1ZdPuL/WQKKYINTXbPeZ7Csk1b/d/P611hJLYEQE8v4qVjuLudRZyNJABcvje0k++B/gKx+BZ4+D//uuMZs1lfO692hNkDwMrU60NqNXeCO8RCPy5dWxRn661v6sQfSitwfhN4h8vp31U/+VsyPv5kzf21jpv5XwQJNZg2iLXJSd0J1Htnsn6Z4C45sUnK412F9DpLeOWE+V8TLRt5NY/3aiA3kEB4rxidejt7M2eDuXuvey1NnEuiaQ5V733ES2P8+U6bnOvayc+jN4+XOg5hCJgDU9F0A0TOIsw5rT+HpFHo1iFK2CxeKVVDrrMlsCidrePjrKub63s9631yo/ZMEZqiMy2Ezsof8Boe+49SuvwWRvHkT9ZF13buvMbh/QBag8YNXrqKElkl4DM9PC7hJXVEem89We9NkHSN37QWKDzdBbAN0qENxOoreA4EC5EUxEE1hkVQSSWF/FVYFDgBJAZJUiI40OeAMFxIffyMXuX+D8kXfAD+bNLEuRpiVU2nBFBYQJVUzJiniW8DUI9JpvqT5t7Z9JfPYj/HDoHVwYv43wdA3BwZvIDLwJDm0l3V9gbl9gYB/rg/vwDzUQGSoiPaQg+9+Q7f6/oP/nYXArkd4iVnsaWBv5VaKn/pLUI0Nw6VFYeR4CWqUXcQrSEw8THGUU1fFEbYDMRVYFhGuCLQ6aK6ljNkMsk7BCwbSWxKpTSuwiBB+3lXvh+z/CxdH3szr4a0SGfpFkT70VKtK5Ezq3G0hkSTThGeovxT9QaYou21WMMkWyHAKGf2gn64cKWD0kt62KwKFmwkP7iA7eRrB3N6tdtaz0VLA2WEZwsNDiVK0/We65lewX7rGl3lK4ygpq2wjZhNwwS2LzI86TMWWtxtsJFc+rQthlu9RoxXVb+RHhz/xPwlNvs0VW9NWTGHkLl/puJTX7a/C9Qbdy9jWY/GMA2VjgYggV0QWQNWclcl3APTdFDFjSmgkz7apYfQ6+0odv8teJDtaS7M0j03ezaVIBISyNMqhjGbF+N+KyJgOVXM0xMlhNeLiO2FgjoUNVxEeKyYy9mbNdN/Hi8T+CpW9btaYW7jjbHCezvmSbZ+bW4hvRN/Vr3WxF7b1cHOatRReznGCKMVqWPI9v+le5MHQna6O3EhirIDqyg0D3z8Ooywzp/oKDjQQGmwgYQIpJHfoFGLoO+q9zQBnaYe9dGn03a/ffBd/9NFx+CdS+VJtYumpw85py+UYrJldvrOQFSC07PuGaiGpBqbSvRElDQMlp5Ki2184kSce1BuYyRF6Ey9+Fby5wee5veLHrnSy330aqvxm6S6Bnh41M3w5iA4UWo4QGHf/oLoLuQpL9+YSH8vENF7J6uIjVw2X4hlx5vWJEhu8ge2gfkcEmUxKrY3VGKwZ2wGCxxSO+sV8n/vVJp2yzKfwh3cUVi3gFIKKAZ02sY13MmtWFBRPNk0QduKx+68JDxD7xYWJDt8LoHQRGf8lcPsYa4BMfgOQLr/BMc/x3nlWK69zPu5jDXUAOTnpNFyEYeBOJVz7t+YLq46Qafq+Vik5dfp7ww/2cHXsHofE6ksMFMFwMh+rJ9lcbIeP9+SQGK0gMVJE4VEnsUOVVHquJDCgoryYyWkh2ugRmKjnbu5tL37qXpLZgyAm4lTXrfpRk8NYh6Fb1unZa8jSVYd1r2qbHJlXm2kZtn75lEq77i0qlg0+yNPMrpMbrjPkM7CEzWGpZqdBgCb7BcvyHqu0Y6S+0ib1wa4mluePDNxDr+39gVtm+Yla6dnFp6vdJf/soqD9yKkTCr7okL7ZQutMrstNLEnaR2/iXm3U2fgm+Wrh65aO5r9h8NF4bcVyhpTbPsS0rLr1A5GunWZn7c1b63krmUJNNtikFnBgoIDZURHyohPShMjhcTXKwjsRgnfEgNFJKcLTERvhwqfE0018HfQ3QI6+ikdSAOzc4Uk5wtIjo0M0k+7ZDbyGx/jpWFn8bnj1mnktOHjPaPiOrqUFHDyW7tMAvpcV8Of6+KkRw9xeG2LPw0oNcuv8veHbg7Vwc3kd4uBr63kx48hYuP3GvWZ71RIalQNqslegkmU/FVgSQq/9TQB+3jSmTLqsl+QtFyTz1CEsP/gXnx24lMFxKWqa0Twvsy8n2FZAa2ElS5d6DHkCGy22WWTPNr2sckqtWTWSwnPj4DtKTO2Gshosjv8nFpx4l4CmZjXy1uR6qLXOiZcLn5dklVnpVR7lfAr1ZCiX2JI3am5tLXLJmz0JT2LJ3genbYLQYBhpNAOgrJSH/e6iapUN1rIzsYn2onFh/Hpkel3ZUGjw+mkf40E0wVE5q8DbiJz4M35kDdT5Mhm22WNfvmOUlAxTzKC70wKzrdYJwdTzUd+s7AllYkfepG9YXqhXt9+8n+sm/4/LAW0iN7iNzuIZY906ivfkWZ6WGS1lu3070UB3hQw2EDlcTGiknNFJCeKSE2HAJqcFyS/nTWwc9DeiYGagmcajczgmNFhAbzSc5tMMSOqn+Utan9xH+yt/C0jddBa8tB9e+NRc9d9/F4bpMgcSuVyDRCxvE0Au6mSiZ6Hnr65V85jTPzP8hL/TfYpk6+m9k7VAVz365i0T2slUeBsIZ2yrFyuGzKbJJ37UCRH58VN2RHCNFcU3uBF+C70xzfvY9LA3X45f57Soi2VNKpr/EtGxSbtZguRHrdYFiM4g8gMgnjo/l20gdqiJ0/EPEL71o/aZMw9iEjoRIvvmrAKL3tDegJyy6BblfapBpANF0rZUFrZJkBbc7t9ImPlLfmMU3uQ9Gi2BAmrIe+svRvfkP1bA8Usf6WL2lT02w+opgaBccbiA6VoJvpIz1Q7eROPmf4bE5iD1j20/okgJagOix2dSkFfFpnY4uxpvI3SwTV4MRTQLr67z14i6DKfdFTRdegDMPE/9cK6Hp9xHpaoYele5UkBwsZKW/gKWhYqLDlabUoodL0ZDl0BBPk0OVBohsfy301ZoHkRrMnV9CdKSAzHgR6ZECkoMlluZeGmnkR3O/SeCbYxBTwzq/xZGZ5IoJbA4I4pOWnOeebxw9OrhmDHGS2hE5ex7ijxH6p27OzvwmlwcaifQVsDJUxeOL7ye7/AVXaRyJWB4jJYUopyiTuVaAgPabCKXFMKFWKk8r6Fbh4jeJfKGFy7O/wuWhSlY6dxLuKSQ9VGNDLlZ8qIzkcAXxw1c5hquI9FZZajQxXkBoZIelSnnkHtsJS6tdnYvi9Y81i/cqgJjr5dYnyJNyNNdzJ4TZHECiasK8ip+EbdiiJg+XH/gYvom9ZEeKwbRlFQxUkRysxne4hpXRWoLTjYQOK41dQLa3FA7vIjVcz9qhCi6O30Lw/j+F7y+4HrWpVeu4KBHVzyp2MJAat+ROKJvo/G9dp4RkQ2leLUD0dfZF+iX9oiaLFXsG3A6za98j85VDBMd/l8TQbWQPNRDpL7YYI3K03uK+9HA+qeFikgLGSBkx8XO4iuShGlLG7yrSQxoVHr/L7DzFjAIH46UWQ64N1bI03MgLg7dx4fR/gR/e56p/CZNKh60DpQXmuHUlWrPuGCYllxvuJUcb9R9Tlx7F0IqzHib7T21cnvkNLvXWszLSzGMje0l8tx2C33W7pwVd11VVIgmA1+hiOQaqeEyz7M5HVDcJBZbL8NKX8D30V1yafSsX+vIJDOSTGd5FZqjBxSCHSkkOlxE/fJXDA4jqnGITBQRGCvGP3Q6PTdtKPtMwopRUssBiluHVAHGvb86USPu4lpVeYki3lnAdyjUxZfe59G1emv0g/pE9pA8JIIUwUAoDNSQ1nzFax/p4DdHZBmLjtbZoS6U3HK61DNDZwTrOzv82PPMA+J51lkPdTuKwnAGVUQokuq5XWJBNbta1AiQHsmgsQcKv5MXKxv7rcl8StlOvD0JPw2NHuXzs/ZzpaSR4uIHUzC0kJ6tJH94OwzeRHd5pdDCFN1xDXMmT4ToDilOApcQPbx5lBqjE4E7S45WsjdZxabiB9dE9LA81szKhXmV/CuvfMIumWbqgJnvNgiYIxZIEtPmoaTSl7jUEkisAEX2Up0yoF0D8AiSfg0ufI/bFuzk3/m5e7N/NC4criXzyt+Hl+yB0zpltTR0pJLOJQmPA1f0zAstMW38usVN7QwRA2wpr5ZbM2+MnWP/EBzk/Usn6cB6ZkV2khxotQBcxEyPl1zAqbeIpKndtogDfZDnB+d+Cs582S2baN+eOGu2037cAolvPaU73MAcQ90xkFUu89I+2G7ZOWZeJmZZdgR/cy5nx38F/uJnkIQXaO2CokKyC1uFGgnKtJqoJTVSRnKwl0VdtFQbpkRqWBqo4N/F2/A/fDckX0W5VyZTbi1zrps8m4Ly3gaqzICLw/3oLovtTuiIUD5MIr0My6LZ1SyZtbkEC5hcvZU1SzxF+bJYfHf+PnBt5C6HRvTBZC4e3wMj1cPhmGC4iPVxpFjIx3EB0pJ7oaDWRsTIi40VEx4qIjxWRGCkznqcOlxMfLCIxVsHSeB3nDzcQHN1Damw38ZFaLkrZPTEBseeNF/6E5ntjqOmDwBHJlUIYODyQbEpQiHaK2BJqdapFbNotLfgkvPhpgp89wA8G38HFsRpWj94GT6gJ9osbKT9tuKZK9mu2IMKBJqiypu/E3iiJeNQaNlou2vc80X/q5vyRJtbH8yzjkz7UQHywhuRIHYmRSuKj5Vc19FnNzCoTFh3PJ3Cknuin/gR834JUzK7LLIdTJUbkDYAYup2PoocCiAmjnlga2+u6pzcCApTmGi7YnkwkzhD+UgerU79O4NAuEkPFZIa3wKE80oPSms0EJxrwT1WxcqiA5Fg1GZVZDNSRGG/g0kgTq/d9EF76lKVmtVgzqOF13VjKuhovPdflGJsNIF6ud1PnFfe+g/Xr/S9L6U+qC7oSxk77EtdKJm9vMz1MaxdbpTUElDMknrufyx//c/xj/x6mmmBkC4y+EQ5vM4BkDlUj/uYAEh6rJDhRQnCygOBkvgPJaAmpw5WkDlfbMThcweXxKi6N1jgXeawRFKMNlrL2iT+Giw/bfiYCSMTbFTkmoklvmdVQsC4o5LIyjpfGOq0jyRFJezqGzkLoOXj+YZYf+ihnJ27n/NSd8MQshF6EkLrVO6Wv/Mw1AySkCgDjjITIbylUlZL7c/6BOpp/7xTnjt/B6tQ2UhPSMLVunmS0kfg1AiQ2ICJXG0D8c3uIPHIXxJ81Kyba/WsAousXMe0+vM+oTseCVs1S+5VilHU8R1pBY+QF1j/xF0SPvIPAUJ2lPpPDbyZ1eItpz9jIbkKTuwhMV3OpZ5tpS1kWWc7k9G4uz7yV4MMHIKGgXLu0OsuuBgXy/kU6m7fIXVCuosEL0MVvfUYjx/vXCw6dL1Bo+zaXtXOFBrZE3wiSJOpXax53LYpMNB2npt688HFin/pzVoabLDGSHruJzIjc5zIyh2o3AaSW4EQ5vqkifNP5BKbyCU8UEM8BZLiWzESzxWMrY2WsjpVZZTCHqmC6kMxoAWfHb4UfztoqwFA8ixbwqVWpFzo6gOi6cgDxKj1yPF2JqHWQRyvpAK1n0vp47Wf/wuf5wcRv89z078Czn4DIefArRPBCh/8VABEBjY8miRIqEdxlRmVZzJu5+AOWHnwna7PbiI8VkB1zmiI62kRsrIbYePlVjcRoFYw2GUAyR8pZOnorPLvgUoK2/NW7OF2gN/t/xYJ4cYn3+gZAdK4pU1ebZmWllni/DFllRCKw8n3WTn+A6HgzsdFdJEcryIxvsZEYrSEy1oR/0lkQuRdyKZK9FWRHmgjP7GP9vt+HMw9BTFsKXJkMywm96OcEXxci5lsazfnZnqXLzYM42l8NPLzPbNDFeRf6Xl2Hl7qzZuY5QKrmyWbr0y+Q/eExzp1+PxfEg+N1ZKdrLHhnpJHMcAMRrSuZqCM0WY5/upj12QJ8MwWEpgqJjZeSHqkwDyIyupvg+C5CU8XEJgqRC8pwBRzeCqNbCY5Xc2bud2H9Sbt/TYMabHVRqo4wLSiKeRbEAOLmgXQfup/cJKnOMsKqIkTLrtWX4IXPwsuftyXSBC+5LKzR2GUyr9mC6EeNSTltbZNsTgvq4izRvvQjVj71btaObUXZJgEkeGgXIk70WgAyVgEjDUZUAWR54S2e2xIgq15KuYvTtW0IgtwFuSqvBIi0qLkZ3mdkPdTcOKNcu24ivQTZFcgmrEo2ePp3iY9XkhitMx86PbGF5MQWYmMVhMZ34RNAJmtsXiAzVkFmsJzsaCOBI3cQ+NyfweVvubZDpvlEqNyfLjYXdG4Gh3MfROsc40XfawGIZEyf1y1vFiRl8DSZakLnJThU7xTXHmuWRVsmtfYtVr8zzMr9v41vdg/B8SZSU3tgtJn0oVo313GoiNBkqQeQQtZnigkKCALIaDnJkRpCY7cYQCQX6fE8A05mRHzdbgBJjxWyMv1Wkt9dtNWAiiC1qYEtqLK9F0Q3j2YbINHzV96XQKI72pBV3ZdcrrWXwH8G1IVRnek9vidto4fAtbpYLh0qAhuVxUOvRFkmWV6rcXN9hfXPvIfgwjbSE8UwXk9guN6IGpuoJj5RcVUjOV5pAMmM1pIWQE7/GsmLX7OLyajn0WaAiB4mCDmAvLKuTEG5neEdBBYtEEvZPuhx0iq3kfOTUYDwfSKn30VqopDkWC26jtTkzSSnthKbLCM0VYt/ahcBjaEimKgETZiONeA79g4y39TWYS9dabG0YSFUfKdgfNOw9xw4JNC6vBxANhhu4vD6/+m7xCMNPXb/5SpruCSFZNC0bjpDMqJ9Gl3NV1Z7eYQeI/Hdfl4+8uucGb2T5Pw7SE40kzpUQXa0DH/fNqJTJQRmSvDNluGbrSA0XWE0yoyVkRqrJDTRbO5oZmIHjG/DvS6LXEh2fAeM7SQy2cT5U/8NVtRVft21LY2onsxZCtHFgcRztSxod5edo9VmK2LnC0O54SlPR1vJtFqnrhPj8rUBxE2oOR92g3Oe3Akg8qWNm/41Ap97D+GF7WQnq2G8Gd9IHb6pOqKTlVcFDoFKgpk9XG8WKX2kEt8nf5ek7wemDV1libeQRoTwgKvSGGvRLSJ66lcHWQsTEB1EZ7OEWmvu2h9pP1jLM4m6y08Qu/dXyU7m2TXEx6tJTu4gPrWN6HQhgekKA0hoopFAfxHYPZeQGq9hfUFroxdd5xO7rlyGKpel2lw57SYFxUn9bE7bO0Y6cXbCYRLyuv9JeLwCDvt+A6atu5CFdb+nCnJHpjjp+JrbxXajFiFkmw1d/Mzf8dzku1mefgexid3IYjJWSPzQNuKTRYSnyvBPV9sITVUSmxAAikiNlxGdqCc6UUt2PB/GbyY1XkR8ooz4RIkpICYKyEzV8OKht8D3pyFzxs2Qp5MkE25qM4dho8iGS6pXHY2kSGQhddQ92/2IcJ5M5N7XOe48RVtrpFi6doDEbPGit6xOv+RZkZxmMuD4zhL6zDuJzefB2G4Yu4P18SrWj1QSmS4nPnV1IzVRZQBhvJaUtNMX/iOpxHmb6jKOW5m3R5GfBBBHQyPYhkshCnrpdVcjKosjpyxiwaxpnaUnSdz3yzC5xZIOiYkaElNFxGfyCB/JI3CkGP9Mk9X6hAcqYKoSJnaSmq5m7eR74dIjZj0sRjNGea7Uj1kOd4GbgaHHV0AhhF39Xw4gpsjEOE3yqu5J5bCebtEuUcpxpQmTZp1Map2stxGnlWQkLpF+/iFevPeveGb47QSndsOUXKRtMJVvHkNsqsomTAPTTYSn64hPVpKdKLKhpE1istImCxFoJouIyw3zZILpMpit57JKWj75XyDwLbd5qZKLXqcTiZ0DcS5m87J9KhfxQOKx1cTxCv3c5/SegJGznTG0DEJrT66x1EQWRGJj+5YKGDmYZlxgJLNmV7T6FIGH3k78iCaVfhFGf5n16Qp8CyWEZ0qvASCVZIbrYKLBABL/yp+g7rjatsCIEPfUn2dBHCFkQVSyobUhTric0Hno1sEDSMx2kEgQsSSn61huMeHK0yTueztMv5nURDnxyRoSk6XEZnYSnNuCf26nA8jEPhKH60ECMLGV7JEqC+7xPWrmXfS5wlhdjOcimJZxVkOv/jhzJQhe4G6IvTqQODdSFlKiIffOS9/oorziA4UdcjWdIxYkk9GcTRS1EHVLYiWlZ1n/+gTPTP8WgZk9MFsGozfC7E4k9FIgKgwMTu4jPNVIfLKK7EQBTOTZMauYZLzKKZupAmIzBUSmK4lN1pCcqoIj9eZl+KZ/EZ5ZsK6Y4q+WO0sRS+w8Vjq6mKLRO1dcRd2SznG8lkA4emfScVKag9K0ne5JsZadp+RJ+NosiH4i7NUt2VoRXamG139VP2SLlC79M74H9hGd2goDvwoj78Z/pBL/qXzCs8XXBBAFhEzuMoBkvv6X5hKpKMQAoopbj9l64bUB4gmJBxB9TABxu0gkCXo70Cq2Y/kZDyBvIjVZagCRFRRjg3M34p/bbgAJTtxOdnw3jJfD2BthoYr1+z4EgadsT8L17BXmOsY5Qde163lOs+X0ziuEwFuC6kovrg4gBrLkZbfc1eusLtdYbtcGeFXMqeBVrVWTMdsXPpnN2PqSUEaWRWCKwvkv4//8X+Of20tmpgCmboLJmwwgqnaOjN+K6BGe3G0AscLSyS0wsQ3GC0iMNxKbaDQaxmbzDCCRqTqbcE3N1MNMoZXs8MW/h7Xv2e9e2HStopX9SXFYSY7WyXi1axZPuSSMc8oEeDFaMx1K3/vNauo2teWH9iYRraUYrimLJeK4qlepm9QVjnp5Z110JrgK5z6P73QzkfGtMPjvYfRdBI9UET5ZRHS2hMR0xVUNaRdNFqand6G6oPQ3/952JpIFMYFTJZ4eSOF6LpbsnWnMTVksCaRbn+y5il54ogU4mqXQ5JQ2rzQmCCArT+N78L2EjpQRmakmOlNNcrqE5MwO4nM3EDl6M6HpZqITt8FMswlJcuwGEieauPTgn0LwaaKRDD51c93kG4spulwNPdbviY25c/SaZdqkgZQ8sGXQYrbe0U16msG0gycvGy/pwjedY0pUM2JaA67da3WPrqGF6KfH9jXWKTIO4bDtcaLYTtfn9nF3v2yZvfgZeHKeS/O/juIM5gvIjr4BpnaQnKogOtVEdHIviYm9JCbrSczuID0rEG2HiWKSE7uITzYRnykheiTfaBs8Us3KWBXRiWqYyLe6r8SJ34LlLxKN+FHxT27OSBRw9/cqgJgVcTQSONxw0xHO/njrnTIpuw31IVGJl+PDNRYrOoaI6B7hRXRx1cOKbVGWUlnGAmtzjXCsGCb3kZ64g9hMA7HZSpKzVab9FUO83pE8UmHfEZlrwH/irfDUIeuWKIFywuTRzNMIrqAy559urmLdJFtecJoTTgmC7iORWkcVpbZ4LPUCgX8Z4fuTv8ZLU00Ej1TAfBEcuxlmbyQ5sZ3YaAPp0SY4vMOC+fhiHWcW38aTn/rvEH8SUhHXpdSzwB7ZbNmAkVPVCd61KMMmbaYNZ3Rebuge3TZwcicEGC91vQkLDmy50hmvXkkf3PgSCZRgqHPcy3qmx/Zn37mJxxuAE82SxBKCk1Zpxmzb7PVHOjl/5JdJzVeTmLmR+PxNxOa3E58rJj1db/FndnIvoYVCQovbSR0phekqmKqxuZTkbA2RuQr8i4WsHy/EN1tCbKqSzFgJmdlaLszW4//a39iW1NpuIncbjre6MV19LprwvAJzEV1Bbe7WdZZS+y5oFoCuKI/NeuaaLIgxRBpG6BPh9E+/5S2V1LwnqQvwgzn8RwWQQjIzDTabnJzSsYbUNQAkfrSC8NFSfAsNrJ78JXhyzNYY54TCBEgaT4v/jLEyDXK7LE11BUSbCtxyBLxCeImJAljtuXfR7X+XfpHVJz/J149+mOen9xI6Wgzz2+HYFpjZ7vzqsToYb4DRfDJT+QQWannp1Nt47KE/Bf8/u7XQlkN1gqlr1hANbWwIonxht5+eSz++2qpIGwog7p7sS/RFHr/d/UiQ9D06Ry6Tlh97aJCMXPWfmO0t9xXvEwkyT36OCyc/RPBYI/GFbYSO30R48SaSx7aTnVFscgeM324AWDu5ndhcmQHDgvHpCpIz9YTnalg7kc/ayTz8R4tIzAggFaRmarg4V8rKlz4A4ccM2LonKQ9XO6fr+UlDVBCdHaB067KQOopUr/V3bQCRiJmZ1yo9XaT3i0Z/IVZFbpccQObqYD6P9Gwt8dldJGaqScyWkzxSdtUjfrTMABKY9wDy1Ajax0TXIZJoCByvBZDNbol47ATKASr3Hc790szrZZArkT5D4uzX+N6J/86FiWYS09vJLNxIdnEr6aNlpGeqbGUjM7Wg1YbTVQSPbmflvjqee+B9ZF7+uPWINV7mfkSCugFUWQyXTFBCQWlpCYGbzPQUkWcldK9OezpQXLlx933ufrQOxgmSA5IHEr2pm77qP31BiHD4svsa9d4JvET4iy2sCiAndxJazCO8kGcAYaYURm83gAQWClk/uZXYXCnpmQr0ngDkAFJr1mP9xA6CR4vJHK0iO15OarqSpWNFnP3UO0mc+bR5C8o+ilqiwZWkb26idTNxHUAcPa7oh591+9cOEAt0AsvFCHwAACAASURBVF4a0HNQDSDy9TU7uWTNE/xz1bC4hdTRSuJHaokdKSF2tIjUXBmpo+VXNRJz5Q4gC3WsnnobPHXYNSfecE2c16FGFFcsiLp7iICe9hCFPOHMvZYja+5oK9ekb5JrrtgtcxF8T/PiQ/fgn2qG6ZuIL9xA9MR2YvMVpI7Vw9EaOFoP07eQmakjOPMGIvflc+H0W4g92g8KjqXCcppckm4AcRZDOzA5cAgoEgBPlj1gGBA8cus67R5ymkFKVOd54NHnlbJWulpAM8DbZJpnZtypV/FfQpnAH1hxIFWAHPHBE/dy+fg+Qid3ElwsIbJQQlru50wJjN1mbnZkoYDAyW3E54tMqXCkBFVDJI7WETlWR+B4CYHjRUSPlsJRxSBlpKdKWDtWxEsnbmftu4fdQigVx+aUYY5GdvOeW5h77FVK5EiU+8z/ZoCIE8p7aFLFZ8Q3ThpAVFK2Cl4M4jtWCYs3GCBiR6uIzu0kOpdH4liJA4mA8jpHYq7MCBicr2P91C/C04OWI7/iu3vh0StcLKUynWCIWBuC5Qle7rM5cNhRfXWk7lMh0kG5WW59gf8bo6Rnb4GZm2wSdP3kTgKnikicqAbFXEf2wuxe0keqic38HKkTb8Z/ooLgwx+G4DNYxzNxSmT0LkYa0c055DSj6zbzyuvU5KJLY+r6DPy6fu8ynel00uLcCvUOUBCudkculnHrXTwUXQU0ch+JRuOEoxGi6q6iOjW1Vlp5nNVP/QbLxwsILFYSPVZO5uhOmCmCsVtgcg/RhZ2ET2whPr+T9FGBxAEl7gEkuFBOcLEMeQkGkKkyMlPFBOcKeWmunkuP/A3EvmcWLCfsuv0NgdeDVw97yVkS0cVInruRn3K8RgsizrrOJwnW3Eo7/arlkj2AJNetSZdvvhxOXI+EOjJXRfjYzYTnt5GcLyZ9rPyqRvJYuQeQWnynboOne627R07IjWCeT29CJMmR9XAGJCeTzq/yAGJafJMF0mSe+i0ZktIxsjGf279Cu+I+/0k4/jY4VkDweAHLp3fiv/dm4qdK4KiAcxvZI3tIzZaRnXsTyfl/R+z4Vvwf/yV45iFYWbkCDuOsxF0mxeXvjcE5wc8hVs+lqY3usjLuE3aJuuHceXbzkhH53SmbrwqT0tq8jbkrJSDsZ3+KcPxrXg4EEpYWjWT8ZLNhL610kdS3/orLp+vwHa8nNl8FitNmS2GiCaZ3kVjIJ3piK8mFPNLz+aTnC0nOl5I4Vkd0vp7IQhXRhQrzOCw+mSknO11kivXisUouffJ9cP5+U845gOjWc5jYeJB7IXf0LInTjD9bQfwvAEiALGvE8RkjctpL5lwzkWgfusfvZW2+0gASP1ZOeK6K0PzNhBa2kFhwhBFxXvc4Vk7sSCnh+Wr8p/bC0x1uUcyGL38l4HXC4Pnem+JZ08yirATqp4FEk2WpjCtcTGsWUdYkAr7vwIPvg5O7CJ0sY+X0DgL3XU/slGKtW0hO3E5qtpH4bDGczCMy8/+ROnG9CU30kTY4q/0qvASHSaMuwAuk9VDXJR2UG7lrtDe0TZlK9zyA6PP6jM7JBV5Zl/uXACkolc3JDfUEU4pep1/1nzqze1NNccFO/bb0Q9EAnJth+aFfZvXELgeQI+UwXQGTdTBTRXp+J4njW0kubietGGUhn8R8KbH5GgNIdL6GxLEqMrOVruTkaDnM5BM7utOs0tp9byf5WDdwecPF+tcCxBFKZ2uIaD/97xoBoh/QlHzAupuIEcZMS/OK8VrXHDKA+I5Vw/EbiM9VWpYiOJ9HaH7btQFEBD1SQuRYNcFTzfB0K6TPmxrZUBgeWFyFqjfP8WqAmEBtAoho5lkefY/ejicypHNd9bJpsmqynDxD8rN/Dff/EqHT1ayf2kbw3jcSO7mV+Gw9sfFbSM40EZ0qgvsLWZ/8v0mduhHfXBmB0x+A5z/t9loxRomWHjhela7dDN7cfclF0CdeARAxW7PhueGVWogvSn1LdnNDDpyjyWsLyE8XHUejXN9oS0OnI07m1Jwv+lVWHv5Dlk7tIrpQDrM1lso1kBwpJDOfR1JJjQUHElmU2EIpUVmO+ToDSvJYFdmjFdYBhvlSOLKTxNGdxE+UEzrZSOSrfw7ZM0YH0ws/yYJ4vNywKAYIj9b/+wGiX48Ty4RYT/qNWQbnhJo56JKV+QmR+fYCsZO7SR65gfBMuQVhMd3kwg4SC8XmZsnVev2j1Fyz1Ik6/MdryHzzv9uiJk3+iiAJW5TuzK6yOJbl8LSxCb64bzNfV5ILRj9PCnPCuPlohNbn9GJqDZa/QuJz/5W1kzUkPr6D0OIbCc/dQPZkDSzuIT27l+RcvblWodNvImiZnRpiR29nae79pH74WQhdtM4dmpHOxCOkUgnDhG4jlFaoriDbyZ6oamlgZay9+Sa954ot5ZopFrgyg6wO9loguBJO4Yu7NKeyYQnNxMuGaH/FTMZGNpu1phGvCYrNb+bcVytHcXM1Ya3I04XFniHx/S7OLNYRO14Es41WxY2s6dFtJI9uIXnsJtKLWwwoscWdRI4XE1msMJAkjtWQnquCuTKYK7KJR+YLyS6UkjleRvJEGZfu/RVY/66xIpFME405axzTfSZc4bW9uZmBUj5KUGyY5ddWENdoQUzCiGcS1oI+Z0HEn1AsSkp9YONr8Ogs8ePNJKevJzpTQXxxF4lTNQTnCzyAyM0SQF7vsZjssQoyJ2os+E1/408grd1rnQCr2lMEEjDcDPoVCyJBs4kiESznkohWr9I4r6Cthwu9Zn8qsfA/D49PEPjEW/EvbiN5agepE3nEFgvJnqgnfWS3ASR0Mg/fvVtZO1VOaLEGZitIHLmdcw/8BclnP2kd/qzDeWydYGgdXzyCL6vlrqou04y+A4mu29DjTa/ruQO/nCclTLyOJBbHZAhFde+OJFpfZDeogFrd1JVlzAgkVw8Q0ctau3qxTiBqUmA9gONPHuPsqSZiJ/JhugkmG+FoPtn5LaSOaW7kJjILW8yKRI/nEzpeTOh4KZHFcmQ9MnPlFt9xbAcs7iS7WAiLVbBYAcdvZv30bbD0Vbu7dCpBQp0WBPtEBtWpWkX3qxmoC/4/CRAZCi29EFnECAFTfRuCwSDx8GXX3PnRw8QXmkhMXE9itpTMyWbS9zYRPFpCbKHYAjbFIjKzr+coUDFfSWaxitDJEmJf/U9uua2shGbPdbTyAjkUShp4KterYVRcYieKkhqmXTwUGKmvIOLVdDaQ6PxYABUfJr/631iaqyB5qhTuKyZ8bCeZE5WmBcXs4IlSVk+Vs3KqjpAYfGQrHMnn/HQTa5/9Y/jBGCz/o9sLnCTBDJyNWR5QuUDWSVviN6kYSF0y5CuZPCgF7FkOA0cOIK57pLXv9DS9aVNtkJk8A2vfIqG9XdIqQLxKgIg80jkxZ3k00+JPu6JOKZ3Ui1/k4ifuIHpyB8zsghlNFheY1UgJJPNbPIBsJ7pYSPBEsQ1ZEiVgsnPFsLATjm8nfSKP1PFi0ov1MC93fQvhU03wo4ecxUyGScmdV0ugZBrNwUrvbeZbjqWeuvCsiGmNK2+96tE1WxBrTOilUS2TIqaF427f8PQyxB6Df2kltdBAavx6kmq1eV8z3L+X4NEy4oslJBcLrmqkFmR6nUaJnCok8MhvQ+AJd/+qp/EQq5lm5W9+HCAyyR5IcsccRV9FKD3NvbX5aCDMLMOzR1i7/zfwK7g8WWL3FZ8vILOQj64zdLyW9ZMNrJ1qIHq8DI69CY69gfjCDpYWGrh477sIf+0gXPoyiG4qlMzmaqOyrKvlaTpMUn2e1FZJpR1S3RvaMGdBZEW0j4abuNVOssJTStZUiQVtDbD+T0S+P8xL2l8ktXL1ABHwFJtph9e0RSH4iFoiQG4gS4+z8vl3EjldADPVcHQXmYUiEioxOb7NgMLCNjKLO4ieKDElFzpVQPREAZn5Emc9FreRPbGFxGmXHUwu7IGFBlh8s3khsafnIbPutHJc27ElkTVRvJhbcGhu6Sb+/R8FiMmV/H1ZEYFDWw2oc7hptYtw+eMkv/6nZBY12XM96aN58EADPHALIU0aCiDH869qpBYLYEGFcaXETu1k9TO/CivfdvcvcJg/Yo1trC731QDR5JmbgZX4vwoor3j+E9DiEdw8CpWB+n5A6ls9nJ+/Df9CGdn76okc2UbmxPWkjm8lutBAaGGvAcVcjoU3wOK/IXP852w+QDHM2ifehf/Lf0/iBydh5QnH9HQuThCYZadlOhRnBLxq1Zya1A2LAa73sJ7p7KglGFQ2HHYZvsSj8OQQlz71+zzzwIcgffaaAGLOfjwqBFpdmCJRVQObcQudIfyP/4nwac2GF8HxalKL5cidSp/aTnrxBgwgSvkeLyNwqoTA6R1ET95MZqEA5nfAyTeTPn0Dkfu3E7q3jMTx22H+Fli8kezJKgL/Mux6XqmJtxoWqrIjE7e19IpBJQI5UdBjBxbRUu6F3vnfaUGkJfT9qkpVoy0pMRWvZdVHVcszn2XlsTYCj7wbTpbB9PVkj94I95fDx/cQPVZN4ngpqRMFVzXSx51Pmj1WQvz0Di4++Fay5796RUHYvcv9CFi7HgtkJTWeiyWAKA5xhHMpUWdRRLhXj80AcoDR7UtDxzQ5Jq1+8R9Z+9yHCdx3KzxYR3zuTWRP/QLpk1vMgibmbyG1WEXqxE7Sp64nffrnSJ1+I4lTNxK7txj/6V1cPvGLXLrv9wl/tRWefdAqf4mqe7sAoXuR6GkyUW6jHO1NfN6It5TCzcHFuRpypUg9C+cWWP38B7h8qonAI38AqeevHiD6cbl8aW14rmVVMes8qcXJcnYkC4lv/iXhU+Wkj26HUyU2iRo9XkH6vjxTHizkWWwRO1FJ8HQ5wfvyiNy7jezxnbCoz7yJ5H1vIPjAVgL3VxLVvNP87bBwo8nU0lfarMsMGVUmqLGGytxjqLFXPOEqEF7NSSlKDVeaYkLykzXgtXY1MZ8jnbGsogxHPCIOnQdeIBl8maUXP8sTX/gAlx/aC6eLYe4GMnM/D/fmw8ebiS/UGkASJwq4mmEAOV4Nx4qI33szZ+69lcTL/+BkW0DYAIhm+VUOc6UOX6DQXI3iEEdAN8MqojnC6VUv5bWhbXKkdmARQMICiAJmE9RlePE42X/4ANmPF8Dpf+uAcCKPrCzdXA3MVZJdLCB83434HthC7BPVhE5W2ERj/HQhiVOFRBaKiJ9qJv3J98BjnfDsKbj4bQircYRL2arzoqINSxbqXnVpnjHR01xaN6wgOp0iE3yJ1I8+ZZWwZxabid5fAI99ANJPXj1AFIOpEljzHxpErOOI1uPo+lTgmf3ufoKnakgeeyM8kE/8RD2h4/VkH1CM9iYEEHkAkRNVBE6XE7h/O5F7t5A9ngeL2+D0z5P4+C/g/8QWfA9UEznxKzD/Vpi/CY6Xcf5zH3HN4DLnXZeS/5+99wCvK7vuey+HBexEryQI9jKcrhLLlhRFUYrlOLFjRYmduDyX51jOs2PHzy+2FSdOHMe27Eh2JCeO7TiSZoYVBAESIFhBdICoLGCvYEXH7fee8sv33/scEBwNZ4ag9OaT7MtvcV/ce8655+y9/qvttdfWPimKEmn9TkZCxGrSQC4GvrICFyFI3hUgQc/qgUKNoFZkR/3x3p/5zp6nxFgJEa2nMWkG3g3w+0kPH+Rq62/Q9+bL3Nu7GjQgXyvA+4vl8GYF7H2FzNd2PBNAHGkQOcL/ew3ZXWUMv/4iXN4T2OjGQw8yYZXwJ6n7iOctQEJwSAmGAJEm0bO9Ez0CiNZsG7lu1JBWil2G/t9hatcLUFNBamcpmTdWY8D81XXwlxvwv7aO6d2rGd9bSaz6BSbf2GzylvzqCqguwH9zqZWuezbh7PsY3pGfgP4/hLuHIXnZaESbfBgkmYWKZbbVYMZPs3j3IN5L+vrr3G/+Ja7v/hRX/mIb6Vr11S+aiolzdtJNBFAMoHX00hvTJuImD2pC/aF0mL4vMfn6R0h+dSXUFJHauY2YUnH2VuK/XmCjU1/dgNUgG4jvKie1s8Q45Xy9BH9XLpm9uUxXlzG1dwupNz6K/7W/AV8thTe2cefAL8Jkn82yVgV4E8lThzikJLUDs0qjKWGouZ/3Cg6dG1FKulmPrQ7VQ4V8IR4wyzoVU5ddp/yfADPqGMMWWjgfOMNG2GoBzg0YP0i29xe5Vf0Sk3VFZGqWwe4iUDRj52rYuQHvzZdxRbu24O5eP2ealk9TvRn2V5KQ3d/4y5A6C9nbSqNDm6rIbZVE1eOFzyAnUo8a4j1sTY+GBz2xtUfpfHlbijKNKe6uPsjEYLIbzvxH4oe/l7Fdm4ntrSSzbzXe3rW4O9eTfXMLqV3Pk9i9nWT1RpLVlST3lhDfk0ty7zKy1ctwDuTi1JYQ21/JWPU27u77EMO1f5/Rpp8ideY/wJ3/CeMHIN4N0SsQGzPBEeOiyOSTuZHthZt/SLL9hxmv/zATh17l9u5XuPS1DxNv1sKtertPn32cp/9fnRbyCbIcVJrVztZLYpsgwnAz0T0/QqZ6De6BxWRqVxPbWYH7xhp4vQp2rjf84O7ahLN7A+7uKkPsqsLQ7tW4e1bj7Kkis2cD7s7n8XZugzcrcXa+yoM9Pws3DgfFHMaIE2XKi5NUiXYDYPGt5sBsJFOC0pimM/f+zo8dUYRnBiB62BAkRgJoMY11euxcQdghYg2hNGvX6MjgNQDRhu7tOKd/k9G672GqutyCozoHdufDLgFkA7z5PO7OF3F37kAdM1eAqENjr6/F27ce9pfh7NxItu5HYKIR3CtkmWQKz6h7yRITZVOniYLGasp37qQnfasu0nUFQJXFlDa1/TAJY21w7U94eOQHGa3/AGP7q5jcW0Jq7xq8vZtgz0uw+wXYtwl33xqye8rJ7Com9WYuyTdXkti5koTs75oy4nvLGNtZxv3X13D/jY2M73uFaP3HSDV+H+7Rn8A98StkW36fdMefEe38KhMtX2Hi5K8TO/rPyTZ+GK++Cu9gCfGaNdza9SpXq/8JsbN/CpnLgTnypCd8D58bgOjBh4F7xoQROyjEapK0bvUxtednSO5fS6ZuIdmDhcR2FeMKHPJLJSx3rZ8TJXe9xujuH4aLX7fzX0wHAtEnrXQgSSyFMs2CMIX5Y0E0U3wdKAMB5R1ej4d5jRgN5gTCuQFdaDaZC4ZXDxlCnSHtcRku/1eGD34vt9/YTqx6E44xr4pgT67VIm9uw3vzRZxdz+Ps3mKkhrNnPXOi3RvI7N5GRtJmf4H5reje74NbO8G5Y6I+kvCKqsxokFByCCDmed+hd97tq/BainKmPDzVtVXXCDlmYdZNkre+yoPun+PSgRe5truA2IEiOFQJ+7QKcT18bQN8bRN8XZNfWyxpnuSrG+z3e7fDni2wax3+m3Luy3Dk4O4rxzlQydiu1Yzs38BI/YuMHHmVh0df4cHhrYzUljCxbxmxNyL4e5ea4yd27uBuzQ8ycfr38cds1XSL6Hd70Hf4Xs9rQs1Wg0hzPAKIA7d6mNr706Rq1pOpW0TmUCHx3SV4byg/7dkAkt69g5E9fwuv9z9C9Kzpc5Ut0+/LjEopcBIKRFum26xR8iXcQ0Vg+PnJzxeZYX7zRpe2mmHGHwnBYa4RRnoeuSjupBbxqlTMLbzLf8ndY/+Y4dpXmTrwMun9L+Lu3o6/ewPsLYU95fi7tuLu3kF2z1Yy+zbi7t0wd9oj0+pVUm+sxd+/HKe6kLGd3w1n/hhSDw2zChgCiRk0PUPIwCbi8+SOeU/f6Fq6sCjj4afTpNJZEloKaraukFq5Rfx+Dfd7/jW3Gz/JWN1WsjUVsEtOZoExFYxW3bUZdj0PO1+ANxTnfwG+qkVmz5tiD3x9E7y5DnZXwp4y2FcK1SVk9xeTqCliqjaPsbqlTNTnEDu8AKdxHhyeD/ueg+oiMw5Te/8uifbfgtFmM4v+XqI479oPBiD6TzPzY2/RIA7c7mZq708agGTrlpA5VPw4QHYpJL5hTpTZu5XJ/a8wffJHYVg75Gr9kd3xV0ZVLJMwOLAYkJYL54qCWgUCybsCRAfp+YxtoAtYBycEiMn7MULRLuSxSxxtRXdjxEdVCmIYRvfzoOknuLhnB6N1W3GPvIpb+xLerpfwd76Av3uz6QR37zoy1etIVW80lN0ngKybGwkgNR+0E0v7c0jvX8bIzhdItfwaTN6dCecKJNZJD8KiQce8S9+8K2/Y5w9imhKdXoqEN8GoO8KIM82Yp0rAQmIC4lfg+k7iLf83o9UvMbVzMfHdEZyGlWQO5ZOpKzLmlHyV7M4qk2vkfG0L3td3wNdfwn/dkv5Of3ULib/cQPx/rYc9r8Gul3B2bia+s5LpXcXE9xXg7s+D/dLcq6H6Faj/AWj7DbhVD9mHZng10uqKZ3qpE01HWic91CAS3CbfY7iV6L4fJ1OzCad2BdlDZSR2l+FrScCbmwxPeHvWMRcSH8Xrt/Cw9ruh5/dg+oKZb1AQS+WKNNFqx143Ewh/41dbnjf3+C5MEDGDrPNNjdhAe+hPhbi1LmKmGohmozWfoOWfds2RERfJcXjQSKbvl7hT/z3c2rve2NvZui0QAuTN1/B3voK7ZyvZ6jWkasqJ799gHNTMMwFkI96+l40GcaoXk6pZzET1ZkYbfgLuXLRYN4l0EixKD1e0JeAKo0H0xzO8dLrsN8kVk+ikvpEdPMUUUSbNAiXbl7avJuBBA5z5fxk/9ho39y5j5GABD2vzeLg/l4f78pjYV0S8uoJMzQao3YovU2zvVti7w5AmIOWoOm9ux1M0aOfL8OYOM7ckXyZRXczk3gLGdxUz+vpaxt/4AG79j0L/H8Gdk6AFX1m7F59qQH3zACJPzFaqFytagCThbhPR6h8xAHHrVpE9VE5sT5kJVjwrQLL71pKuX8+DvVtIHFZ29Fdh/ApMJMzWcvIPZT2ofJMBiHwSw7wWIIGcfEcGiFgnRikLIhtINxGeIIIoiaDQWMasGNTKQaVsWDVmBv1hO5z+PA8PfZwHNZuYrN1IbH8Vmeq1ULcdf8/zsOtV/F0fxNnzItmaNaRqy4gfWE+iZiPZ/etxqqvmSBvJ7txKdvc6MvuWkqhZSrxhHXdqvheuttjZKj2AAYPmQSatsDMC4R375T1+6eE7SbxsAleFkM3MvECiPlL8TJVGAuFlR8ok8ZHpxbn9RUZO/xhjJ/8eE43fxcO6l7i/bxOj1WuJ1a3Bb6yE42tIV6/A2bfMkLdvJX51oBkOlMEBTaSthF1L8WqeI1kfYeJohAfHl3P/2HpuH/448bZfhbO74e51u1+3wKx5PUWkg4Vj7/Fh3/Yw48eZb3Rhm4elrBYjiBTRu9dAouYzFiC1BTj1FcT3lpvADLs2wt71+Puq5kTim/SBDUzv20h0/wtwSnuu1MDEPTN3qTuysA3sYIFDKiUYFv35LgqEiPHwTY2lIIw7S2uE6tKGxkIG088Gxyrdu//3SDR8L8NvbuDhntWk6jbi12+GgxvgwEb8vZsMSLzdL+PsfYF0TRWJ2jXEa6sMZfdX4eyvtAB52rZ6vdUee6pI7c0nXrOC9LE13K7+EFzYA9H4jEulaTWl/tkZ1PfQM2/LDo9/aFfraalQnKnsJAnHrs03PyqzypTCCUJd6rZQQWu2lxtAD9z8czj728Ta/hX3Gz7D7eq/yZ3qVxk9sI3pQ+tIHV5Dur6EVF0+yZpckjX5ZA8U49aVwcE1UF2Jt7+CWG0po42rGWnZzlj/p5i69C9JXP0CRLshqRBwIE7VamCVGiSz/BlfhsH0nzFd5BrbyxspqjX892tIHPh+sgc249YWBQApMwAxvocBSGUAkKdrFb1M7Nlorh3bnUviwA4499swNWhuQjETC5AgvPgNAJGn8s5WRMSIEhMKCyWt1UIWHCrXrIJpGVw/Y2qymlTd7Ah+8gZMH+b+wU+QqN1MsuZF0vt34NSsw62pxN+/FqrXQfVGQ17NetyazWRqXiB14HmSB6tIHlqDW7sW74Co6m1bt2atud7bt2vN6j1qN+HUrCVzsIL08RLuHtjCaOMvwd0ztockQJgmyR0eRm+RNqkINrbwLPyh/g75TuVJVWjBqi3lJgWaI5BYxsWzCtrwpz42zGX2c7wNYwNw6yjZ83/B9On/xFjL5xg98c+41/ApHjR8jJGGv8FYwwcZa3iNcVH9Bxk/9D1EG/4xsaM/Q6Ll10gP/BHO1Wq7tcLUdUjIaQ2KgolHdE+iQIOa33+WDtClpJ3N9SQ0fbPxpX7CAuQBXPwTotWfNP4R9atNQCG2rwRqt8C+DbB/PdSshZqqp2/3byaz60VczYMdWkaqeinj1R+Bi182m+RIu92Pxon6Lqr270xmbPqBKS3rEc3Kong3gISdFhhkOkErD2xVjRgJN246wfrwStWYAO8q7tgBJgd/gYn6DWTqCnD2b8WpfhF3/6YAHAJIJdSsgQMVeLUVZGvXk6l5idSBl0geXEvqUBlu7ZoAGCFQHm8Ftnei7G519nrky6TqqsicLOZhXQXjDZ+BK/tgUqvbNGJavz3CpDOM1k+LO73A/Jorj6jrHgHEbjFtfkyTqmLIwC80bWDs6yfDuROda8ZH32mFT3oSErfsjrfRdpg+Qvbqn+Fc+zL+tS/CtS/Atd+Ha79r6eofwd39JgeMBwMwph2SMvam9NshKAJA6Oc0vtKidjZZUcm5v3SuwZveCCkz/l7wXM4wXPg9YjUfNRVdLEAKie8vhroNVoDOAEMgeUravxktmzA+Wl0O1D5H4kAVbudPwugpM5OvqKI2/jGRaPGBOt8YQFlSbuI9AEQjpo40QFJ+ihxxrfO9h8MYWSWhqbNlQ0tAph7AZD0T5/8frh1+nujhlbgHF+PuX4e7bwf+vu341QJJJf6BEqgrwj9YhFe3Gq92A86BHWRrt5M9WIlzsNx+iNxyWgAAIABJREFUXleJ9wRyDqzmSeTWrMZTuLNmrVG1ybqNOKdKGT9cyOjBD+F1fR7GtB2b/CsFGSZJ8ZCoP4JWnRkp9wwcolOtpp2xWiy36QsxvbhnFpMaHAS40TgJINNmpl+bwqiCvNXWnjbpEYWJgEpTl6bJToEzYZfpOqopO4HvTpF1EmZDmWzaMRntM4drvPSjgeYSKFRjK8U0CcZNa1cizg0keszw8ULNFMpbA5jsFTjz6yRrP0hS0bRDFcRri+xc0EFN7lbBgSr82so5ETVKen0Z9m2DmuVwKIfsoVwmG14mPvAbJsUGd1ZIVzyuLcudJI6XfU8r8iNGus0ARPlKIzgMGzLLddQLAocy47Sr6EQP8b7Pc+Poa9xtzCd1eAkcXAZyGvfLnHoeb/8WozG8ukL8Q3kBQKrwajcFtM4C4z0ARBrmSSStRE2ZAWNszwYSBy1Aoo25jNRUMnXkH8GNg5AYMdJEjnMKpSOMkxBoxDx6vmd46XQTCg9MDRPnEEOK9JkJENif0s/ZteTKIrabKkSJMUXMGIBy6xUWNqFJreEIuU9t+F4caC9kri0MKJCmIVKuqOZfdJ7Bls4xzyetoXI/KvIQI8UkCSMs5DPpgnN76dLKhlYxa7PMOdAguj1T8Do5iN/zOdIHXzKpNn5dGfHaAkPGfzqwZk7ACAHFgXVwQDlvyu3Kx9+/FL9+KWM1hYwc/bv4V/8bZIZnKQBFZWM4fswsNdbYvNsrYkSg6SOpW0VfBJAHRnuYkpZCnVFNHkzfhPN/wp2Df4fhA4VkmwvIHsrHqyvGq8vHqy3Fq9mIe2Ajbl0x7sFc/Pp8/EPlJmTJgW1QVwEHpWLLg/dr8A8+mdzaCp5EBiC1miRcS6x6PbH6KtInS4keWcF47UpGa1+CPjltKlOpNAPVhRKDxNEaZpM79W499I7fiwMlOZTNaplY2lYRIpn+0i6PAhzKeQu9dHW4zpVWGwm09YiJss3YR8pk0DWFlpCkyeV5aofXjLaHU4DACxxRa9EZ5jSo1Y3rBFWdUXBC46qJvOngnuzl3wOPPLEHZK4pwqm0DpNmMwMQDy8zZdKO0m0/TKZ+G87+CqgtJXEw35B3sAxqy22gQcGGuVDdWji0CQ5uxtldQWpXHv6hXBIHcxmv28B002fhYYOZ95GNZarSe6rTr37ROAWq9YlPqF1u1aPGvLILi6yJpTmDQLJIa5g9z4fh0k4mjv4YD/dvJ3E4F5oEDqnIdVZLHMpDD+4cKsOpV0ivALe+HP+gkL4NDmyGulI4mA91AUgOVrwjQIxpJvPsbUiShAPrze/L6Y83VBI/WkyscQWx+kVM1FUwffKfw/B+8GQ6iv0yaJcpMYZjp7vfoXve7St1tFIVVSvLmlSKlivmod+ypL4ctyWQjFEVLJZWv2vQXG31NULWH8fxdJ1gIZRSVVRqXNmgIam0p8iohwwyxaQTQs2T9rWKzsNxHLOXeNrXdpcqxyRgKEQvsMjcnLm5Z9KgFiBBxUbFdmcAksVT5GzsKNMnvx+ncaPxQ2Vyi3mTh1aZ3DAOCiASmHMFSAXUr4H6dXj7NpLctcYIaw4Xkj5YxKiiWmd/C8aVOTBNwneY9jX6Fhi++PpdJETESixZw7bOklSmBtYkmylOrm18YsMwXM10288b2z7VsA4aC6FOam0HXs2Lxodw6nNxGueTPTKfbEMe2UOrceu24tdug9qNIMRLatQWQ22eBcrBclxFs55AzsHVPIkETmoEuk1kG9cSP1LK1KE84oeX4xxdQOpIIQ/qP0ri7Bcg8cDIApkk0pMaXLNB57t00DtDRKiQJpBmUB+K9F5kxH3wffDeJNAFzKnTgnXT0jTqcwk0Mbg2qElnYmTMfoCPrqfUEOEqBJ/RFvoj9CNVFE9FGHx5M3ESZvmrZmVmfFOrNnQ74a3q/Dm+LEDSpPwk2cCn0z2ZxVyJh/j3DzF65JM4R6qMteDVFBpwpBpW4R4shEMSls8AEAMwCdtSOLQDr+5F3AProaES6guJ7i8n3vSPyF79Y9zUJZR4orCvFLIBhgSQ1Q5P7IGIzaGRhLOTPDpZp5n13Or41Bjc3k+i++eZOPZxEkd34BxbS7ZhJXHlPx3YbgDi1FXh1OeRPTLPkgBysAq3bpsh/+BajMSQ31BbasBlNIkBSEUAkG9sLTjKA5A83kqrULsODm3AO1ZF8shqJmuKiB/Kh5OL8JpWcL9uPZMdPwsT/Qb1inzLsJEv4D1m2AfqVoB5K5lONGp2piPDQ+wH4jJBT9pEqyHU6u9gSwIdPDv5U1wUAEQ8rbQIs2RZZpmksOeazUMdXwaU9vASqWio/tnx0TjJjDNRGXn8Kd9qGhNREUDSZgY5hK05XvehC+iPbwJALJ95qJBENp0x2knmi5ZQuLHbcGcfIw0fgSMSjJX4e0vJ1JWQqi8kVZ9PtrHImN/+IQuSp20NPx1YhX+gAI6+CMc/aMx7o5GOFuDUrWKiYQuJgZ/FGzthxkQjpa4y5rUZUv1n1wBJAIVkgaP9QdwL4N03kkn9LItapL1nTOdPNOIP/hzRxh1m0so7+SLJk5t5cHIZd48uJN24Gk8MWl+J11CGd7jAkt7rs0PrLdUL1astqUPqywKqwGuowKtfPYe2DBoKoKEEt76KbP0WMod2kG1YT/bYQjInIsRPFDJxXFt3/RkkboIW9gdCdzKrFGhxipg5ULe2v6ykNe8FHKlSda3pUYMfKylDDa3P9b06LaTg+BBJT2hnO/LhIRZ04Y2Eg/Z4oqh+X8eb/3SoOXn2OXYBmO5CNHP8o0NmfWh/8an/12+q+0yKm2z7MaLcI+3fhvR1HrT/LtOHP2yiV1RvsNGmA9vM+ERPFDB9MpdsY4kZd/GG+OBpWuor4FCZvf6hqhleM583LsM9ugSvpYzx4x/GH/oSxIcgOYyTiptKPNFE0G3BpkrihTAlySwBIU3ELJF1Y6YCRMyz4FDI0Umrds8AXPk8tH836YNL4NBiOLKZ5PGtPGjK5cGp5WSOluI3rJk7HV6NP1dqLIYjK/AbC/AaNuIdegHn0Es4DRtxji0ne2IBiWPLiB/fQbrj5+DmPltWx1UMHKKeCg1I6UokBAAJOT/g70c5ajJ1JH5tp4aHGSZ9as76DjlBYAtki8cUSYZJowW3Kit0nJHWf0288RU4pIRJpZVshwPbcQ5vYPJUIZPNeZZ/5jj+NJRDQymmrV+L3yBSmk4x/tHleMeW4p0sJN30Aqn2H4crfwqxQcjGSacgqjVVZjwVvbVmaTj/p4CKQBIxYy7fL6mCW5YFHH8YEtUw/pskWz9K9nAZ1ETMRIx1qDYYcyZ1fA1O4zOAQw9jOqcc//AcSB0RAqR+qwWIwCGt1liKc6SYeH0BiRObud/wYaZO/yKkZGrFSCWSJnPK5mfJeQ0UxGMiV3+IA2ymqhWXdnLNdux3CKPP9TFkqpj6COqTSZL+/UCI3CF2/Q8Ybf00ySNbLUBq1oGobi1OYznRk8VMNRWSlYCdy9gfLofDxXA4DznlHLIpTv7hUrwjefhH8gwPZA9vxm1+jQdHNnO/9R+Zbau1dimbcEwdcw28xlK+t5YxK/iufwqLi8x6EBN5kWUQ6uJUL/7Ir+Fe/hST9ZW4tQVQuxRqta5Ys+Mb4dB6aNxgESsmD5n9ads5do7pVAOQPPzGQryGDTgNmw1gDUAa1uEcXke8roTEkUruH67iYcvfgzt/AdnL4GgpsawMJRfK1LKZy+E8hkWMNIY0jAASgkQdJdH51y8xlqpqJh2bnJlOaxJTxfmGGOn/UUabt5I5oiiTQvqiUuM8e0cKiZ8oJnailOyROQjGkGdkQTSugsY8oz28hnV4jeV44gsB5fBa3MPbcI5v537jSm4c3Ub6whdtpRhV59dgy4IOMtY1stYHF1gUrPKIRLlPQkvtLYxAD/mwlvTQDzPduo2E0tYP7oD656FuG+zfDvuVyr4eDlaCNMiRCvy5tjr3SNmcicYyOFKCf6QE92gJ7pE1uI3r8Ru24TdsIXmwlOThIlKtFUy2v8h474/C2C6zTUJoVdn5ChsMmq1AbM0s63Q+8s6kUWRu6ci/2kDRVECcaSY91X1Ut2gtfBRGD/Kg5UNMnVqFe7QIDhdBQyE05MORlfjHckkfLzOJpe4zjL/GnSOFcCQXjq3AP1qIf3ijGXsatsDh9XBsDenGQmInyhk9uZV77Z+G+18CX0JSmylZB01NCBALEssPkYdcYJrbJlpialqNNeFc+vdEOz7CZGMp2cMb4ejLcOQDcPAFqN0OhzbbUJom/BpD5g4Z/Wnb8Py5tBVwuMoA1D2+EufECpxjZThH1hvJIYBogU5aczany0h2r+P28RdJX/lVu5FmQuvtbUc8Aomteh7+bftPjrK80TCEIa2iCNU3IV/821gRaeJ1iklGmcTsMSQOmxyAi/+esRNVJE8shKO50JgPjSuNv2gY+ViuHaejFbhH5zLu4TkCSBEcXYl/IgfxgNu4EbfhRfz6HdCwEY7bbA+/ZS2pjm3cPL6O6NA/hWSdWQVrXIxZ4XaBwwQFA20SGeE8ca7hch9nopn4lV9lsvtjTDVVkjieS+p4Hl5TKeljVcTq15I6vBZOrYNTRbhHlsOxYjhaCkfL59jq3DnSkUo4vBWOrMM7uRS3aRHuiSLco2vxjmyynzeU4B/Nh5Z8Uq0FPDhRznjH98L1P4eoainZ2sICgt3yTHudaDsHK1FCoEhaKnxpTS2p578GiGbRJ5hmTCaqOEuK9eGfE2/7hNEQcpQ5mmfp+HI4sRKOFbwNz8xx/MV75nqr8E6swj1RgHusEq9RAvwFOLwJGpfhH1sEp8rJNK/l4ckiRjtewb/xKyYbnawmZAMtEqxAFEBCikS5RUZrE9whssP/i8nuTzF2tJh4YwE0lxA/lkPs5BLGGldy9+AqJo8V4Xeuga5csicW4R2f9cAhWN5rO1dghOeFAGncYDuoaRFOkzRJEd7R1SBqLIKThWQOLyJ+dAmp1jU8PPYSMc2NxNpmOkeTXjaLQLPOsRl1GwJFALKhP4FExQCCGXGZpn9FX7LTJ0iaYnFW7I7BvX/DpMyeYyU2BG8kfAHeiZW4J1fhHRMYVsORUjgSCte5AqQIjq2C46vwjhfhHi+1dGw13pEN0LgWrYOnKRfv6ApSR1eRbClnpKmKye6PwvgfgDKOBRAZA8EiwVCLCCSRTDZqioulbuxntOVzJJtfhpMlODUqCFBEtG4BY4efY+T4Ukab8phoKSTenofTtRg6l+E15eKfKMY/XvK2LSdKzfXettV3z0LHK+HIFji6BY5X4jcV4LUuNERTnr22BuJoARxfin8yl8zJKmInX2Sy6XsYPqnatNfMwiY/JSfclkBS/ELLUdVRIoHEmlryOYI5Eb0VfYcDJJlM4rrGw5gRA0plUbE59Uk438h0FG7W4PR+F7GGCN6BFXB0HWgW/dhqOzanivCPr4Nj66zFIRAdfxYeKIETeXCyAP9EFd7JKmPteKfyMON/UsK7HATKk0vh1EqyJzcRP7WFyY4S7rV/GGInbE2xVAIv7ZqJWo23lOG0r1ysWILszS7GOn6V0caPkT2sybw8qF8LTdugZQ20l5LtLCPRUUa8o5hE6yrSzYtwTi0OAFJowXHiG1uBjZOS4m/X6rOA3glIOubtvj8hLbEBjm6CYxvwmkpw2ubjtEfwmxcbyYGcRCPFVhnV7h3bSPbEVuLN6xhp+Qjc/e8Q7wHtXJuJ4SY8m2wYzLiGGkSmlmETASLAyaPZtxne+Y57k1UF+VkvgSOVSpHJZMxkm9GsWpE60YV76TdJd1bhHo9Ag6JLSknaEoxNqVmK4J3YYAEipj1WBCeeYfwNXxSYa4QA0XIHp2UlfstCaM6xADy22gLpRAH+8U2kT1UR75rHWOdquP3FIMviAXhR9LwSjgKIwB9h/C7x7v/O2PHvMrOOnFA4Ls+mghypgJPr4dQGaN0E7evw2yugvQRayuCUkFlkpIPfNJdW54RUErx/760BzbEqK6mOS3qU4LTm4LRF8Noi0LwQjss5FEgqbMqDgHSykmzLSqbaS7nb8Qmc2/8Zkn2QiJuisr7WUwWJd9bcUgasXU6qyKARnValfMdrkFnYMFojnU4jraLWJHuqH5K38O/9IdG+v028qQhOSVrLzKqAxu2g9KSmSpymciPlpe2NVDcAeYbxF0CkgY6X4mvsT5WSPbXaZJmLB/zWCDTJTy4H45NughOr8ZpXku6MMNWZS7Tvx+HO18HrM9u5+dkJ4umMqYov6yHC6HmiLb/JyPHV0LMYWjT5kgtHFkDjAlAlioPFpOvLSB8pIXN8hbH3OVYGRyugqdR2hjpkTlQMp+ZI+u3ja+DYWjhRgd9cjNu6Cqd9MW5HBNrmQdMSOFYIjZq3kaapNOrXb1tAqnMxd04UMNb1d+HG/4DoDSs2BBAzy6p1G9q9V6T3wfJSM2nyV8PECgGi7dlkakmDhCaWieIlJ01KefLSZxlrryB6LA9OFUCr+KECjm6DYy/gN63DbS6DpjI4KbOq2FLTHMdePKPxP1Fh6VQxbnMxmZZSMq35OO0L8NvnQctiOKmMi+ct6bdbF+F2Rkh0LmOi9WOkz/0bmHzd1ghQqkx6ykasDUCifcT6/h13W9cy0TmPZPNyaFkCp8VgEWhZhH8qzyDUbcrHPbUQ9+RCM/egORBOlUFLwZOpOR/eieYEqgCM6tzjZdb8kt3ZnIfTIhVbGHRQBP/UAjiuUKMiGpIgCj4shrbncDqeI9a8gPGmtaS7fgzu7oLkHe1nYBas+07UOO42/UCTR3Yy0ZhYf0V8kLCwtVqB5LGXKqmPt8Dl32SyZzvjrYtJyKIwTFgIHRqbdXBiI37TGiPANEacWgUnCi3jPuv4Gx9WwKvAbS4l07aCdMcy0vKTW/OgdbH9vaObQdSkiOYi/LbnSLevINYmf+Rv41xWTed9wEV8Z8xsnaDxjpC+Cnd3M33lJ5m88Alig99Dov9lol1FjJzIIdO5AbdtB27LDvzW9fgdi6H9OWjRepBiKykkLZ5E7wQefddcNHc6Jd9GpOvkGpC6LavJtpbjti/Ha8+xdujJXDhaaemkgCSALIT2+dC6ALelkHTLh/H6fgZG3gDnnNlsJjl536TFS3M8CveGVpU+lX0upHznvkKN8VZH3Txx5oZZJ+/2fpqp9nyi7QtJN5eQPVlmwdBRbiKhSHM0yyzPh9YlVgCfyoemkrmPvfjmlPwXgbAcmlbjtZSQ6VxKunM5mbYysq1l0Cq+yIWT5ZbEJ61LoXUlbls+XncZE80bmOz6Xrj/e+B2KlZt19QYJ13FAlLjuNNncSaPQ3QPzv0vMXH+3zLe86+Y6vgcsbafJtn8z0m3fJxM1woy3REyHRGy0jDtK6BVKvUJpE55Ij0jQKSZZO/KzhRYTpXjtpTbzmkrwW0rwD+10n5/cjmcXAUGIOqkJQFp0EqMepY0yV74LEx+GZIDZKeUOvFoD0aBRGkpZnvrmXV8Asp37is0q6RBwpc+m5qaInG/iWj3D+C0VxqJnWpbbswbmTlO8xoj0WlbDm3q65XQovcSSvOtQNOYPZOAVPBGjC8AFuC15ZLpyCfZWUC6XQCRv1EZgFPAXGjvwQjmcgMoTq8g1Z7PWPsHmTr3L2F8f1CIW7V9IaI1I1pPnFbRQVMoSTVWr4FzEbyLMNIM9w7BzTfg8m+RufgPiJ17kYnezYx3bCTd+jJOy6s4LS/itm7DbduI37YaX458Rz62g4TYgNRJLeqs3MD0eitIZLIV4bWtxGtbjteabx7EPuhqa8pJKrQKeJIOS+CUACJptBq3tYxsW4mRHnrvyR4WSKQ1TikmLpNPYNY9rIQuResK8ZoWku5YQqy3itjQ98OD/2krhodhLNXaDXK3lOCoORNVC58Jos947gJMSKEdFrLWW9vw+0fM99YjvvFvHTv7+sERb7F+vvG8Wcfp2JCMFgwnPYPU/mBdhCZHwyW11gOTxozhJm8yMdzC5MUv8PDoFtzWZdBZQKYtj3RrLm5HCdmWSpInCvDalwSaejm0CiALLZnx19iHJLBIy4iK8FpFBUbKu225hvnFC9ac17EKEgXgMD7PcmM1CCDpjiIyhgfK8Zqr7LHmd2X5iBdK4FQVXrP4NA+vp5ypzud50PV9pG79PtBrsibcdErp7rM6y3SyOkFBrqQtja9dUc1S0HG0nx3eADgdkGmD1Ammu3+HVOevkez4HIn2Hyba/kmibS8Saysn0bYMvzcHTs+DLmkbRZbmwcn5cCwHjkj654LUrbRBSwlKCXAUVj49n2xPhGRbDpm2cpyWl8k0bSPVtIx0i663AnpLwNiZYvhi/NZSQ15bMV6bfe+3hWZgqOH0d/CZAhIKNhzXdZZC53y8vgWk+0pJnP4E453K/r0KsYytjKCZI8OeWr6kPKQkqewkjjONp6WyBjBxfD+K40Uxc0yGmR9lAAdsOhMrDhfoaKJy9lA89kf4hTHnwvFREFKMHVxKuAmPe1Ib4nF2kCGtPdqDfdqVm5SeJp0cIZoeYcofNxOBYyZbz8wMmN3DmK4hdeHfMNr0UTJtimzm4bcJDCEVojHw2wrx21fhy8poywtoFbSJgvFoXmV8RxMRba7Ab60w45/pLCTVvYrk6aUkexaTOr2ITPdSnI5VZmy9lg34LeusCdW2xJjTbscSAxIDKAnW1gBIBnShQC2E5lJoroDmdbintnGvbjX3jlYy2vMhRs78E+LDvwsZ1TEYeRuAPKFzTeKjQp8qjZX2SKdc/MQUZJWuMQSJdhivgTtfxr/6a2Qv/ATZc/+QB03PM9a0jqlTFSSay0g3y4kuNSFlMTRdq/E6y030IXGqmERTKanWPLI9C/AGFpLpWoXftQE6PoDb+hLptkJSbYtwupbC6VVmYASCuRD6fWO/llrV27nQBid6l+L2rCfd9Sm48QV4cBQS9yFjCiqZ6Q8rQlwyaKGQdiHRbubSKAk8L2FLkqrqvUkPDpg3QIedtbf1qbRG3G4oGWiRsP9DZp7dmu/0gUAi1faWRV76Ssc8FkV4y0rJ8HohSJJBKSEJQbN92SiOPxqUBkrywFHdSNcsozXrPKL7cK/8IomeD5HUOLYXBmNQ8NStAY20T+uKwDKwgi3bUUyqq4Dk6RUGHMmeBaR6FpDpXozTucIAT/NzApPAkO1YOousJnHb5WMoB68YTpdCt3itFDpk2ZSYuT3a1kLXazjN20l2ljPds5oH3duZOP9ZmPg6+L08vj/II/H2De9kXphCG9peN+uS0QaJaS3JUpEBbUg/aXdl1TpqLdNNaOuvq7ao2d1quPUVvKufJzX0fxEd/D7G+j7CWM8Oome2MN27jrG29Yw2lTHZstSEX+nLh4FKaFday0Y4vRm6NuO0bTKZuYnOJWROL8KXGddeNDdqk6ottoMjqdaeCx1F0FmB172WdO9Wov0fxLn+4zD9v8A7axfOyMIxeTvSIirZM2FaUw5ADKjvQ69erfg5OEcdqzRqFQ5QqoYFR3hAwMxvZeKQmdXqOuHfBhCBRjG5YdIooU04u9Uxb0M6X1pR92h8qmEy3CLBXVOlS1eQTDC/qe2nJ0+QvfILTHa/xlR7LunORfid6rP8OZLOlXmWAxJOXTn4nctxO3NxO/Mtda00n9G5HDpWgpjemNgy11YabZPqqCDVWUqqU8DKM0I1073MaJx01xIe0VLrQ3etItOVZzQe/VXQVwI9EdLdEaY7lzLR/Tzu5Z+H+J+/O0AEjJBCgGgPakfFBVzHbHWVcFLGoTECVuMQMofpYY2N6uA8hPQgJI5A9HX86S/jTf4W0Zs/TvTqP2bq/A8x2fdJpk9vJdFdQbZD4FhrHT1FyLoK8bvKcNu3km7fQrxbEmaZ6UTaCwOAzKHVtY26z8NeZzV0VOJ1VZDpLWaqbwWT59YRv/YDMPoFSPWCmEUZJ1qVSNqYIZOoBI9vhIhhKPVByKN6L0YTQwZvw69kYs0s6Q21TQgQtSEYZrch0+ozk1EcACNYjz5jIpu5YJliYUJIYDrPmNCPcmm0Pj/JQ6a4yzSjpsCBWQKh8Uw8hFvVpPp/gWjnh4h2FpLuysE5LYZeOUdw5Btw6Xy/axm+wBEAxABOQk8CrK3IRkhbQodeJpL8xxzr/LeXmCCB07Eap6MCp7MEp6sQpyufbHcusZYc4go/ty0h2b6UlELAinLJMjGgWwYdEdxmmfMR3IEI8Z4iJrv+Jt7Fn353gMzUGnjrxlOqwOFbASTrW6TiZSFpd1UzZZ/FgEemmeeEeymo5pO83jH8dDskD8LUXnjwP+Dar+Ke/Qzprg+Sbi0j0zwfpyVCtjWC07aEbMdqMt2bSfZUkTxdjttVbBxEOiX5C3jqVuAyWiiUgrpeMV5XAU7PCtKDOUwN5jAyWMjEhe8iO/xvYbIBEhNGEIj39cyBR2Aqk5jl66GgCJk5ZPqA58OPH/Mhws42TC/zaRbjP/Ze380qHxSCT79hflzw0x3J5HvkG1ng6MZ0QmCO6TIKZHqPjtaZGipzmPacvPtnuAN/n2RrCamWZZaxThfA6aDPu/JgDmT6uLsUp6sct7PcmNq0r4Y2BWPWQnOQOd6kQMoaE8o1Gl9zKc2Knsq3CfwZ+Zptpfbc1rXQKv8kIJlSum57CX5HIZ78mI4VZsrCO6G5sgiJpgjxtgjZMxGifUsZ73wJZ+gH3h0g0haimbELNEooCbWb6Zg/kwQeBD89s/zd1uNQ7SbPTN0nHEhqP/UEJGKgTUgd1XkyVUDugXcJEi0w8jrc/E9w9V8YU8fpXkCyPUKiPUK6vYRM90bSfevJ9K3F7Q6BIXA8JWlQu1daEsA6iq0W6cjF61xKtlt+0HxSg88xPfgco70FjPZ+F6kLKox9GOIjNgYc+g1GSfhknCzxRIppVZcPlnTObi13iptnuQvSBobpW/u1AAAgAElEQVTBw0iSmFwSX+wqCqW/PhdApHmCc2bODZjecLaAIAqON5GpxwMBOl3Va6TgtUZf1T5mLC79XvYc2cu/Ded/EPrXQvcC6BDlQFe+te3nAIwQTJ7ModMFZE4X48gX7VDe3xpoXWNzAAWQpg3QpOUVa6BZ5rAm/wSMFdAus2uJpfaV0C5zW2NYZq/TVgkdG6FjA7Svx2+rxGkuI3WqhGRTEfGTinZuxu3cQra3nNRgKfGBKsb7XyN16Sdg/D+/O0A0UaS4dzijOns2VR0cDqM6Vqu8tVN2nAfEuE2Cm6S4Tdps7jhhlLhklzHZxAeu3Wdc+1QYZ1bF0hRvFnomJmCyBy7+PAy9QmZgAfFuAaSMbNdG3L5S/L58/O58mCudXgH986FP6l3Om6RMkYm6eJ0RXEXe+nLw+paT6VtJ7HQhk53rifV+DIZ+Dq5+Be6dhrG7kBizRd/MRpEJprwkDzOquWs1jPJ6JArUZ9Zx12KtoPxpaD4ZgIQH6Q91jCgAkzk3+D441jr5ykLW/Iwt56/fMBQIs3AjJNO+xWrzFVTw0wYoRvkId6lpstNv4D78Wcb7N+D0F0LfGji9Gjplii4BZXP3LA8ETC50Pz35PcvIDkQMub0r8but9raCShpB0UUBpgI6ZGYvg57n8PsieL05NkLasxB65sPpoO3VeNoxpW8x9OVCfwl+/zr8/m3Q9xp+/3dB/8dxej8DF34Xd+jLcPO/woMvkb31X0nf/EtTYhfn3nsHSAgSI/Vm/acCcyILHMFFal2p45pPuYNnViuqHcFjHOkWVQT0Xc9YWcbq0vyC4QchJli4p+IYIknqG79AamgLsf4ckt0FuKfL8HuKoEca4OkHJjxHA8Tgc/j98w3QvM5S/I4AIB3zcZXL02s1TLYzD6e7FG9gLe7AJjI9zxNt+YTdj+LqmzB2DNyztqIHWnSl1XYm99HssiugCCTCgl0LHdQoDRjdcrTtWBvynV3uZ5Yror4K+svCR9saq2qkStbY2YoQVo8BIzxvFkCM/2PGSoWwo5CagOlrZO/tYfziP+Nezxrigwtx+iK2n00EqBDkd3QrfC9aMecx8HtW4PblkO1fhNe7BL9nCfSE11WrlKcV+KdzcXsKSfWWEOtbw1R/FRP9m5ns20ayZysp0entJHu2E+/ZQaz3RaZ7X2Gq50Okzv0d0kM/gHv5x+DmL8Dd34SRL8DEV2ByP4xfIHvnEkQvgn/Njp+CTupETRTO4vU5vBUyZMTOIv09Q2FUJTQVQps4XLoqyRdIOyP9BLBAaEolhYffbyZ55WeZPqsiYItwJSVkTyoPTCZWz6q5Ue9y/L7F+P05RlPQuxxkW0ubGCqC7lXQLSCWQPdq6KrE75FEkgRbSeJUFdnOT8LQ5+D+VyDeaje1N7UxrPYI/TOjZV3XhMiduEs2ljUVFNPOFBlPJULF6KpwKVJwVaVoRCqXak2gpI8xV5Ne0JpAlN0YT6asjpMuMUVrfG2HbPfsMAWmFa71M2Qczd/ESKfDQndX7bzWw68Qv/gjTPR+gMmuzcS7V5s5oWz/KpyBHDzzzHm2j06vhNPqrzn2vcasu5xsy3YTmfT7lsPZCJyP4A3KUY4w2RkhdSFC8sJSYhe3Ebv4aaYv/Sumr/4Hpm7+B6K3fpvYtf9C4voXyd7+U3iwEyZk+nZCesguqXVGzF4hpjq+CnVoBaFK+GjORwWFJbFMirYV7D7jQSEP64c9I0AEs3BCUVPy+kFbKcIgcLbpICfG2MQCi25GIAm26TXhTlUWETPYpEBpFqPydfhkGu/OHuKX/qlxmJNKpOyQk7jeDpak/BzJ712FGRxjaklNSzqWQKfs10roEkgEkIKABMjlMBCx1CtToxS371WcMz+Ie+XX4OHXINkO/k18Rs0KxaSXJOlmTd1ps/4o0BxZL0nW1NC1GyGEhctsETNVXFHFS81oPwqKGbAEGkmgC0ldpe8eeR2BSacKHdkkmeQkWWkJV7pNY3AP6IP4Trw7nyd1/h+QHqjC61sJfUUwUIHTU066v4jk2RxSZxeSHViO17fKAuNZwBEAhK4PQOfL+D0FeAPzcM5FSJ2PED0zj9GBFSRvvED85qeJ3/4lUvf/B9mxo/iJPvDOABfsgjd3GNwR+1wh0wcaYCaiqh0lAtktv8vUT9beQmHcyFS/j5tKxyr7E/Zx5Bsmld46yfSOf+suQi9Erf7WyIcOp1UP8jmkKozvEah4fWTUh0BkTpF5oOrrsrJkuz8we1joigZX02M4t/6EyfNVTPVG8HsXwflSAwwx+JxI4JA2kLlm7NWF0KsJSGmRSujURFJpICVzQJqrV8eEtNisrNQMb6p7Gan+IjLnN+Fc+wT+7Z/CHf7PkD4JqSugOSP5zDJJg6Wd5tlCFaruMCnlvgmfy9FPOtrfS7P2tuC2ClUn3/Jv0plmyomZIngJ3xZTNf2tPhUJWWHgylTxiEL6ISRuQ6KD2J3PE7v+A8TOvkqmrwp61tgMhe4VZl5C/ldqYBWxMwXEzq4ieXYx2QEJBYEkz7TSpHMhc43+EhhcA4MbcPrKiPYuY2KwgOjlV0ne+imI1sPEdZiSbyrhK6tDGla1q4JAhfrQc021+2wmTiodJZGeJpqaCjYLst0gDn0raQysHBf/2cqKj8xVlR41vRj25tO2spPtTHCYKhEqDdMGdm94E2pnkwFJeEJQh0gAiZJkmltEGTaAMbeoJxs5TfL6DzN1tojU4AI4vwA0WP3LoF8O91O2koS9hdBTDL35oL/NZ7n2MzMLq+/0uX7nORiYB/0LoHeFOTbVtYBk7zwy/RGcs/Pwzi3COZtHZnAj8b6Pw93fgIf/G6ZaIH0zcOS1eUvW1OD1ZDNJBejBxcxhZwVDoWohUspWfEibqD69FnBpjYpMMn1jB9lEGnWNUEmbCIFCh5q3eQDuZUi2wsguYtf+iPGhX+Th4HczMbiGeO9yXK0HklaQ+dS1yKTe+P0ryAzkkzhTSvxsMalzS3DOLLL9HfbXXNv+JXjdETi3DC5WkT2zgdHTVYye+QjZe78M6Xq7kWKgIhXpVoFvdVnMc4m5jonAKZ9Q2M9mfLPSMZNNkHaiyHQN657ZfrLWibpodldbGaUIn0o8CYA2amgrKz4DQGzKxONMbwAgtRXSrJvRoiO7tkKFEe6ZTTWN3hNIBKYA4eoPYVll/RNmB4yslYKxEbMpZOrazzBxMY/xM2LK56B/SQCOubSLoE+OpsKYMtnKLEgGnoPBSHBtgUHREIFQQFF0ZAX+wDK8C4twLi4ge24h7uCSQJLmQW8xfs8asn1byJ79CM7lz+Ld/XUY/7pNy3HugqJ2GqnZYi3U7eFnihQnbAVDL+0bU0m5X9o2IasJSwFJnR5eR+CwHWjb2ChM9cCD1+H2b5C69hmmL36IyaF1TA+VkDxbQuZMAU7vIrPKLq3VmFps1rsAzhZBr8zHMpz+YgMUZyAXd2AZDEg4LXjUP6Zv1D9PQQOLyZ6O4A5GyAxFSFwoIXrxk2Tu/DokjoP/wFTz1opeJ4UpF6qa0Yp6ahMgU+ZUyHgrxxsmnFENto8MCtRZYehbroFsrVC6BBZN+L2ZdFXp0W8FQGZpCv1+KOGsWpR61IYuD5BDZMwsHSB//y1jLR4xm904U2aDeFIqqHoPRvYyeePT3DuziszZlTCwdI60BAYWghzEnnLoqYRe2d5LrcMop3FwCd5gLt5AkSEGVsLgUhhcjndmCdmhRTiXFsOlfLhQCufLYKDISGLNEDunl5DoXs5EVyljp19kcvAfkrr6y3D/yza12u8B9xw4lyGj4toPQflRiRTE3EfgERNkpU60h7N8Ny17jEImjukXTS5pdZ9209I+h8mzkFQm9p/DzX9n0vhj/a8x1VNMtH8B6XMRGFK4VH2QB2dXwJnnyPZFSPdE8Pol2eXbCSRltu0rAFH/SnMsZ2z/mD5RvzwtDS6BcxHivRHGTkeIDr0AD38dsifBVdjcagWlnaeUca6ijYGbq02BTJhTfaF+MTRr7/JQwDwGFplockQUPAoA8tj3gVkq4ATHRd5G/s9i7/DsJ7UhGnU3eh8au0Jq8LL6y3L/W9+H0k+XD7SIjb9oNyYb01fJRy+TxpeZoH23tVdf7AqJe/uYvvxLJBXbVkfPhfqXW80gp3Qgx4R8zaCfmR8AYyXeQB7uQBHuQImlwQLcMyvxzi7EPbuQZJ9MkGI4VwEXVsOFcjhTjNujXKXlMFiC119CuqeIeHehSWNI9pfinFtL8uJ2Jm/8LcZv/xDT93+S9Oj/B9NfhMROSJ+ATD8412xCaHYEU9hPrcwlkXcrAFcbJPeTnfxjEg9+hanhzzJ57buZurKVyaFyEufzyJ5bhX9mFQzkQ48icuUm58wsk+4shDOFcGkV/pUcspeso5wxgQgFLpbbYEWnzivARPsG50PQTwwoaDEHOpODfz5CvC+HidOvkLn4CzDRaKteah2OapYZ5ah6AFpeoK0lFPtXHyjJUoEGmUWWFMOTESVBrKkEV6onVABv5T39Lf4LNUjY6jOT8mN5O9Aguo2QW5+21U1Ir4tC2M4Cy+z8ovAm1M7GXPi5btpcR4FRXctmz5pGE4jZuO21dJLM5D0y9w8x1vd9JAfXkRkswT2z3A7aGZlHOVYTDIpJl+KdEc0C0oDMseXQK6koiRh5pDXOKtS43IJDgAgoa9o88zvuufkGIJwXcFS1cRVRrSfpzCGl+ZUzBTBURqYnD29A70vgSjFcWW4kt8yK6b4Ik5dzGL2Sz/iVtUxce4mpqx8nevUHiV/5KWKXfxnvzn/Bu/0l3Nt/gnvrT8ne/jMyt//MhDWzw3/M9LXfYPL6zzNx7UcYu/JJRi5t4eHFlYwMqdxAhNGBCNGBiIkOcUFh1FwYkLbcAF3roXuNiQhmOpeQ6FtI9sIS3GuL8a9H4FLE9ouWLGi+qUuRw1zokwZdaISE6degj42Qmq3N1e/qx8GVVgsP6rdX2n7vk+ZaTrwvzwQI0hc/B7e/DmMXIWHNJpM5bgAi01xb1N0K9pdX1oXdnFOTnOH+KWFrLRWBJvSPQ/4OzKgQLGL1t5LhQZkzOkfFq81LR+k1l1bnhNz+1l8L7CZdOryp2a0On/23uQd7YzN3Hn4fPokBXPgAw8Qe/jfunP8h7g++SOJSMdmhYGCv5sKFIvzBfOM0Z8+tIHt+Kd65pXBGJlng0A/KtJBJtQDOzTetf24R/tml+GeXPUbezN9Lsccstt+fWYEvmvl+1nmPfW7Ps781D//8PJyh+WSHFpAdWkxmaBmZ83lkzhaTOVNBZnAdqcGNpAY3kxrcSmpgu6HkYNhuJXlmM/Gz60meW0PyfCmpoVVkhhaTvTCf7IUIzoUI3lDESGr7fIvs8+q5B6VRVoHpg2Wmb7xzi3HOL8I7vwDf9IeOXwyDmlQNyPTXYnO8NJP61j+72PbhwCIQnVEfLsYZWk7qfB6pc6U4Z8vgrLSVwsjSTCXcaf0oYxd/Bab2WG2onXy1D6MsoWCLd+tiaApAQQlpEjnRllcfTarKyVbQSK2lGR6a4WvDYI/+C3krbB99M4OcZ5wHeeyK78Mfk2TSzTy8818YufpZoldfJHrWJpxxYQWcL8Y7U0j2bKFhnNTQMpzzGkgNYuC3nBWTv090TqCcZ+xw2eJzITGxf07MPFeyjDyXPpDAyJ7LI3s23wgII2gGF4EijGd1P4vInFtC4twK4mfzyZ4rsZpUWqx/vqm1Nn7mXzI9/HVIdmH3qlG6keVPhaslLi1ANBckTaKoXbBNYCBfv5WM920OEEmSGzjJg2Qe/g7pW59lerCS6e75NtJyvgD3bBHOuRLSQ/lkLyzHHVpow8NnckB0LgfOv1+ke3nOzB7PrdW5CnfrOnOlZ3j2czJp7YSiWbszKA2x/LE+VUg4O5hDdnAhvo6/WAgX8412muqvxB/5HfyYIlZK9VAoOmXTxGUkBJFNa5/IDxE47D6Nssol+L/Vr29zgMjXUSRMdmsjjH2BzOVPm5VhSUWmhlbinCsykit7vojs0ErcC2IoSevn4OxCOCfGegYmeaZz9du6HzH6XCiYC5ozOJ7x2c/Jj5A2CEAyWIF7Lh/3/DLcoRz8oQX4Z+bBWaslvbPPkT6zgtT5CjJXXiB54zPgVQPnbA6UP24mVJ1k1gaRAh9aGiPc0MZuSRGA4/8HhHybA0TLSpQHFofMXRvWfPgfyVz4W8QHC3AuLsW5UIBzvtSQO7QKLs7DOqvz4Px8OL8ILryPJI32rHRhIcyZnuHZBUwBQNGsc4vh/AqcoWIyFwpxLy6Ciza3yrSXrB80PbiUiaEPkb3972HyEDAUVBEZA2cSJxkjGU+YZd2O6xt/QuB45HAHfu3b+g3ffH3ybQ0Q9VEyyBq3AbQHkKiD4Z8mfqGM9JX5ZC6uMoMmkLhDeXbQNHBy5gUQMef7CpBFAUDm2M4ZGCGongEgQ4H/pPki9eeF+TgXVpG5mGcBoiiY+vpqBK5F8C+tJHbhg8Su/zKMd9mJUi3ZVl6eF8XNREmnEqb2bzqbQbvnWsfbAsR6JIFtFcaDvsVa5NseIJkw3V7uiJYxpgfgweeJXqkgcT1C5tJynKFC3POleOdnA2TeI8n9zEwWMtvTtrOYc2ixBerTtu/bvetZA2AYcFgweJciuJfmmwwDo60FjqsREzb2br8Gd34fJs7a1BpZyCYY5Zg9610nY0ubellcX4ZVOF0QtoH20DlyQtT+NUCerDbVN4pwKK0lE7VFOUwh5YdfYPL6ahK3IqQvLzKawztXDAKIBtVoEJk2YsqnZepv9vHSHIvnRkbzfbPv52muJ60R9Kf69DGah3dpHt61CPGLS5m8uJXM8Ocg3mUHLVwVocltOeOhoDM8H4ZsBQyFtNTOWkH51wB5MihmfxMCRGsjTDxQc5XOBKS/xtjtSkavRshcXYB3cSWcL7STZBpEqf4LS/AvrMC/mIN/acH7RIvs719cgj8n0r0vep/uXX22CG4sw72cg2f8uQVwcQlcXoh/KULsfITJC0VMX/t7ZLQvYKzT1uCSz+hlTI0CZTYrSjV7rtgu5HoLOKQuNOAijXdIsxniW/D+29zEUjzcIe27NpVewsZVbtJeRu+s5+H1COnr8+CSIlrSHsvh0jy4PM/Yw97lVfiXF5sB1aC+L3Rpsb2HS0ufukU5YJdz3p/7vrwQ70oO8cs5pK6thGv5cKUALmqCdjnOhRyiQ2vI3v8XpB9+hezEOUgZO9gWkvAn8bLKywusgEDGWQyEk8WB5ngrOEKQqP0Wv77NAaKYuGZYlbvl4GeVyKe850OM3N3G6M0QIDKllsIFMVMErszDubyK7JU8vCuL8a/Mf59oocl9EqMJqE/bKm/Kv6JrPAvN/dmz1+YzdT1CbHgp7s0KuFJhtfKFUvwrH8e9qYrpbZAew/fs/IWiUWaxljuFrw2LvgEgdgnFjGllEfNIY4RFML7FwAgv/20NEClnVTPMKn6ueirelF0qljnF6L0PMHYrggaRSznW11CI93IE/6qK2q8kbQAiJps7kzzrud5VK4kFjqemq88CDJ37bM+tvo3fWcz0neVMX80jeqGY5MX1cOP7YOSLEDsNWWUdWjNK/qIAYUBiFnBoCfAjDTLznQ1JfqPfEVhZOk6LpRT+tcgJ2fmb336bA8TD99K4qupoikQoy1MiqZ+Je59i4lYOzjXZxEpHj1jtEURU0tdXkL6+yiTmce053he6ugD/2iLcOZLO5apowbPRHJ9ffZu6XcbEjXxGrkcYv1lK+s6/gIk3ID5uI1WBby1RpjWAqqWmajZmnLQc4zGAyDmXnSxnUl58wP86XuCYmVlXzq6WI2vlvQb8W/f69gaIOtBRRpvWWA/bNdYSKpnrTN39DNO3i3CursLY6gLIFRuPd29ESN1YQuLmCsOc7ws4rj2Hb2gB7vUFFiRzaN8vcKjP3KsrSF5ea9Lqo3eex5n6p5B5E3zt92gDUI7WcHguGT9lSNEqJSEaVaLIVYAVtdb8CjPDNQEcHKcvtRRmJjdLK/VjM6b1tw4ez1zV5Ft5a+/h2gKIBI4q0JlU6LvBwDwgevcnid3egHe1CC4vseBQTP5GBO9WhMRwDonbS3FvLAQ58u8LzYfr8/FuiBbOoV0I156Rruse5vj8V3Nxr+yAG/8ERv4Akk1mrYakuoYl5vlmWWzKyZr14mZfu0BDSFPITJL8F+m91gLNFMkzi5ZmAWSWtlFJpYTZy16ZvZKI37rXt78Gyfi2IIJZTKNUaYFmgqm7v8T07VfIXiuxTBTM5gog7q0IyeH5hrybFjT6/BGJYcQ4Ak9Ilpm5MW/WcW85R98F5z1i+Hl4TzxH54fnBNc357/H999wXd3bbGafHwBoCVxdakkmp55p9rla+xHchwVreB2dn2OvoWMM5cD1HNRv2WtFMPxD8FD+Rp9d3aiSs6aMaYqEaoR4abP2/tEiJIFAzrnW1atVnlWoPULzSu1jDsfbahBpkb82sd5ROEj2hLULg8VVEiiph7jRL5Gd+H4SwyW4dyJwOwK3xAg5cHMZ7vB8nLsR0Hez6XYE//ZC/FvLcG8tnyH97d/KgdvPwXDE0h0da8m9GcG58RzpazmkruaQuLqI+JUFZG8vIDs8D2c4gnMngjdsSeeZ+7n5HNycPze6FcG/E8G9Z1u9N/cWXts8rwCRC0ZQlMD1fNBz6BkeO34+7u0cnGHRQtzh5+zz3iiAaysMOPxrEdwruTg38snciTA9vB7YbXaHxZ0GN2N3AJgpMiGtELzMG9lMoc7Qd/Z7e0w4saHv9X7Wef+nvXMNte266vg65+xzzj7Pe+7Nuff03pt3G1objG1ME7W9IpVQCTVSqpZUKkZMiSik2pp+CKgfzAcpVBFbNdoiFB/UQCBSIaCxRaGlEUKj1UaTe85+v9+P9V4/+c+51r773ua2+ZDmZF+yYbDW2WudNedce/zHGHOOMcfQDenqlT21pSNMVpP5e7P/eQ2Pi61BjLpo2fyFenN6twKI1yAYfgGv+wCj8gFBykAGJIe7cLhHUlw2zDVj9pTpzffFVcssxQ3C4gaRoXUiAUf/V7IMaRhMIBOVVgxDhUeb+IfbeEebuEdr+KVlw0xBeQ4gApWYV9pLx1ekZSjM0woUrqDisgG/gB6lNGN68/xlOBRAduHidRYcR1skxRU79jmA6DuBIyht4JUFEgsYLp6Bl09YgBw6hC/t4R+ewa+t06v+KPCNWVraLD3t/PE15NVjedSCA0RBbG0SJfkUQLIZn9cmGn+JsP8rDCoHeLWUoSVZ5wEiUBxtw+EpC5qjbZLChmEgaZ2M6SSZDSikPTIm1bmeV1mGSg7Ka1DKQ3ELSlvEpTxROWe0lJ5zuXRfsYx7tGo1wByDZ22a9tUHUUm0bLSeQJqRwBsW9ggLu/Z6phHNuASOdduOjuqrKAW3GY/uS7/XM/W8oLx+OUAO94y3XCBOjhyiQwmNMwT1c4waHzTlyuYBceX5sXD1a9jowgNEkaAJw0srI9IickxNvkwyfJhh5Rx+PZXyxSU4OgFHJ43Ej8s5OHwLXLzJHgsnoZg31y4zu8pLICqtXKJiDor6O2OyHJRWobgOZdEqSWWZuOIQV+fMOPWhoBCNbShsEpdXCGvWTAqrVx6XiCpLhFUdVwirK5cdo3KeqLBvSIBUW7N+q39qR6Q2pS10vWr7Y8ZeEqgtaOLSGmF5E7+Sx6+uEVbWDMiTw004WjX3xUUHX1TZJ27eRdB+xGQ3VMK7q9FryKvH8qhrACADU/4sVLmRTIOoypVqjkw/ybBy/rsBUjhBXF5CTBEfnic+vIm4cNL8PQNCYQcO9y1wdCxsg5hODJWZJjqXmXSZqSTQrBmQJBULkkQAEXOKStI8q1DYgKNdKJwkKl5HXLiOqHiSuHiCqLRDLO90OY8YWeBQfzXXENiiWnrUuUBS2jb3azy2jysWFCnzZyDIwGPGXs4ZcGZjEcAMQKpr+FVpPgFkjeRIgmDJ9F3j8CoOXv0MtD8Cg88DxauCQ6BZ9M+CA0TqYmQAIoetqvVaM0t5tP4VvN9nVLmZoGElp2XOXSjuoh87ruaIi6dJCgck5S0SmUoymaQBCqfg6ACOzkJhH4oblrkFDgMQSdVV4osOiZaOj7R8rIntBklph6SyTVzbJK6vkEiDZSSwGI0iIOVtO4XTcHTatqN2i3tQ3IGS2pQ2kgknU+6SFphpA2mfWo64sk5S3ki119y92f9Iu5W27DjLW8QpeM1zypqPbRFVdghqqwS1HFFV4F414zL3qJ26w7Tu4LbOQ+dRGP9zmsl/0WFw9f4vPEC0dq4sF0EYm4TEBiDBANxvgv8ZhtVb8RsOSU2MLem9C5LQ6Q8e1WQGrVqGETBKYsydOQYVYFKmFrPpXPMMAefoPMHhJkEhT1DcJSjvE1UPiOpvIWyeIWrv4TVX8VoOQdtS3HKImw5JMwWNeZ6kdEoFmWnSMK90FHA1n0iPpRxx3TECQMxrxqL5lsYqyjSXQCYAHp0nKZwnKe4bQOkdWNBdDSA5Qk329ZyWQ9R0GDUdJu3boPtn4B7aBGxX56+Fv7LgALGhCcqDpGTP0iIGICai97/A/3OGldssQIzUFhMKAFsWIA2HSFRPtUl5awYgM58QIPR/lzGbJvGnSApniUoHTKs7TGu7TGvXM629E7f2Ptz6B/DqP4vfuJ9h6f0My+9lXHkX49oPMandilu/Aa9+lrB+ykr8kky3V0tWc2VzHb+2wbSxjltfNeOUttSYYgkFM/fSsvSOAXRydKMBiLRmXNkgrl7SNJdpkPqy1SBVh1CTeD2n5eA3Hfoth3H73baisetjizIuPA6uOoCFB4h2nYmU7yJK4nQeotpuSjL2JOPq7YQtB2RmVVahvA2VzRlAdM1I9NoqSXUTypv2PgFDjNFO/7eYJzw8QVg4RVw5CXrZgg4AAAtoSURBVO1NktEq3bbDoJ/HG78L/Acg/AxET0P0gu1DUAXvJZh+E8ZfgcET0H2cpPVJks4DJK0fgc710DpH3NwnrO/h17bwqmu41WW8umx+B6/x3TSt5Um6dzOu30W3dJpBZQWv48DQgZEDfS3drpKUdqFyGqpnoLZPXN4lKGziyjko8NeXobpDUjtB1FgjauZIGjk7fr23ngNdh2HNodtycMf3wvg/bKzVnMviqly2wBcWHCByKcndZGM7zTZNpThX2Luv3K1fYVK7IwWImECTZwEkTyIp23TMtVAmT32JpLYC1ZRpxDiipv7OQ/mAuHQ9fuU00/oWk7ZDv+vQ7p+mN76bIHwQ+CPgqxBXLfMoM2ZGZiuEijMewvQ5m5zZ+xuYPgbex8F9CNwHYfIxkvFHiEcfJhrdz7DxfobNCwxb72XY+rHLqf4zMHiUuPfbBIMP4vVvZ9o9ybTtGLNO4Pc1udcSdG0DGinVNohKm4SFHMjUayxBfZu4sUvcXCdurUBTpPELaBlAlui0tnHHD8BEmeJTjb3AAPh+XV98gJgANoFEOkSe3DSpq+KzkmcZ1d5NIKmqH7y6DtUtqK0bE0TAkLQUGUbINI00R00T9h2oHED5RpLqzfiNc0xaJ+h3lul2HarNk/T6v8Zw9DmI/xH4lg2YNHn6bbIVw0RZaQOFsmq/ihIuKwI5Ua5ZBVkeXkEXbTkwlQTzXgTvf9KjzkWqN/J/4L4I4+fBfxaSv4LwEdz+Bbr1M3SqDoOqw7TmmFU8CQOjDVtL0MpBbQ2qcyCob84BRPdYs8ocew5xZ4lBbYtu++1Mp58CL417e1ODfD+MHd/1zDeo1V3l3FNu1kRhpAohVV3j+BvG/Ai7S9CUFM1DbRsa6ySadIrqW0SSrGIgMYWOAkhVK0L7UD5PVLkBt3aGcWufQfeAweAWBoMfptv7RfzpU/iT75AombRSYipUNTH1XAh8peyPbck1PyJRhgl5+udJ4eAaQLZMHdnVOFVDMqtyuq65laK/rzjOIgdUU8/klnoK3MeYdO6jU7uJTjXHpOnga5wyFWe0DNIS+jsDQiNP3Ngmaq0TtZcu3avrPYewk6PfOE2v89P43mchTOv4Lf5K7vdk4IXWIBlAxDeKH1X5MlOlT3HRRv0/z6R+D1FvGdrr1sSo70Azbxggbmsyu2fImFIZQBrSNntQvc7Y5V4jz6jt0Ovs0u/dyWT0q0STJ0imXzf17xJhMiviog2NgUK8lSAwxo083GiKH3oEQWTvU6oi5eFWOq8AVB5bKf7NMbT15ZXuX3Xm9RyVZ1ZMplFEc0cpJBNBYMKYhKgaRP8O7ufwex9l1Lwdt3OCsOsQSYvOAJKedx3Ivm+tkbS2iNt54s6y/V7XDUBWCDp5hs23Mug+SBT+A9A3+zNM+9+TxRb74jUEEC33XgmQbzNt/ARxfwU6eWhuQmPbAqTjkHQEkG2o71oTbAYQzTs2jV0uDeO39ph0rmc6uIA3fgS8v4egOB93Zwq8yKpTni5PlY5izYtk9qkYp6JOxzY1v8K2Vc5R6wjTtE55Visv3RCUKhS7IGeUi4LzsgC9YC7LoA0X97VXQgGwZhVPCRFeAO8JGD5E3LsLBmegl7NMn40xM7kEGoGklSNpbbwyQDrrBJ0NRq07mPQfI4m+ZnZwqp/XuALJsrsvJsqNBpltotEOs4HdU6BfTltEoheZNt9HPJREzENrywKktW5XZnoOSVvmRg5km18pYVs5osZJovZ7oKeUNV8A7zkIW2memjTwVB2RRSdGjxLCWKtq6o9KgGXFgurEtEzcWKLiLaaSZEzsTYi9MYk/MWSrsGpvvW/J7ItIH64G5klGZTRmjGtSH6kP1nyTSvoOeE9C52EY/DgMdu2YBRBjQq5BffWSpmivQnuDpLNB0l0xq1ZaudJ7STqb+J0dxq27mfY/C8l/ktAzuzmUSf1a/iy+BplJ3WzDjSoHmdJUEB/Rb96HPxRzbEB7DVoyr1ZIeg7RwCHpWk0ibWKo6xB3HWNSuK0bGFcvEDQ/AcOnIaxdmj+oqJFKHqkSkeFKa+jZdTUFUSqZRB8vruFTMRSmZecS7Z8XghPXFoFRIRjZW7Fr615okm9SqgagIhmKo8lIdpUSganZOGIYdEyZumy/t9EiMi81B4vK0Pw76PwOcfcCQec0QTN1LMr7LoAIBKIUIFq+pnNJgFjBsYHbyzPs3sN08NeQaGGhk9by07iv3c9CA0RAMOE+EmLSGjqa8lppYQma+NGniZO7jEc7C/kwXmzXIZSvYOLA2B4j18FzHSbuHsPBXXRbvwxodep/LcOKKWecqMYyu0bnGYlZdG73O1hO1n0Z6SGXwDQ7NzPwbD/qFUcB/pXIlAHQ8oTmX5bsttW0CfVV5L4Mk7/EG/wS7vBtuON1omkOvHW8wSphb90KjqY07J4xtYKuJubWl8JwnUnPYTB8D/CMWYyIgxrE3VQ4vAmQN+wbmAEkE+RGoqcOQwZMkj8k4l6C3qaVlNmk1F3DFUCCJZg6uAP5Ndbo925hNLmPIPhd4lCZxwvWrMmePzO89YVE9fFJUJk3mulYcNgkBjo3XbQYtQARSLxvEbt/QeB+FHf6TsajLYajJcbjNYLhBnRWQaZnW9p225il8UDaJQ+DTaaDJUbjnySOv27BqoDQWJrw+Mb/ejDlYmuQbJIoZsgY+DKAjHD5Y+B+kuGe9TDL6dVeIh6fYNhdI/KW8CcO49Eqo94dTFufhsHTELycMoCYQMtOWWNiCGkDgUNHXTiejwWIACESWNJIm6yrpm/qZ5r1JX4egi8RTn7DLFO3uuuM/FWCqQMCgzzm/TUQYCZ5GG9D6xR0zuIODphMfoFQBUfN+5apl6YsOZ7hvy6tLjRAZHWIKYz5MbNabLoY41tgyjT+PAk/TzLcTwGi5cwNgv4NdJtnGfZPMRnfSOz/FISPgvcv4CoaODVPDP8LfQKD3OFyjeuo744PHJY7sjiCOXCkczKZWppIJ5RT56Wck/JdfBvcv8V1H2Ls3s3QO4fnrs8BxCEZrRBNNmC8B63T0LqFYHAn/vS3CIKL1m+TCST9CNfw55oAiPmNMvEp71qWewmPSfBF4vhjBL2zYPwh20T1A/zunXRq9zDs30sw+TgEXwSesxNQ1S9XbXIpjxkOBAyFr6RfqtHjxkfaB5mZOrUaRWDRUvDYACSmSmKqwk7AV7HxLgT/DclTwOOMph/And5KMsgZEzToOfgDB3+cIx5uQ/skNG8j6X+I2PtT/KBpMi3JspL/xzT8JkDemG9ATJFVHJrZF2ZZNPVOyzoPv0wS/Tpe+3oSrWI194kb74DRzzHuChh/AsE/QfCSRYRQEcuZEZo0v+a5BggyVQQO1dhOwaHvdX5cH7UtzWnUqLqiRGpKhdMzWUOSNHtIGA8J/AmhOzXLyoQaR9XsJ/emjxO5H4bRLdBfwu87TAcOk5FDMNIqV564+TYYPAzBk3hB2zgtzWt+EyDH9cu/unYlMTVFNdktLFpsPIYWgQzTyAR5BpJH8btvNev5tG+G3gWIPwHTL0DUtNaTvG3hBCLtb1fohkgJ6dKJuJ4vEijm6dV19Qdzl/ohJpWWM1pEDskyCSUSswwbGLdJpKGFIUE0IIx7JhvlDPj+C2ZbANMPwehGgvEy3sRhqlU+1yHpO8Stm2D0KYifwY2qxtgMtYqctvuDGdwb46kLbmLNAyQtMGEcaRlA9JL/Dfg9wt7boasVmnfA+D7gD4BnLf9rSiFGk3POVDxSphTFVknKyqxKJ+kCXSaxj1t7aGjqg6ZGM0kuR+nLJLyUAiSe1d2QqIgZEtEhSnzj+Y+MM1UhKl8F/zfBvYPE3cL3HfzQIQwdkoEins/B+DHga0ySkpmeyRUj18yxatDXAUP/Dzulf7mPSc8LAAAAAElFTkSuQmCC +``` + +### `annotation` + +A Kubernetes annotation for the NavLink custom resource. + +### `label` + +A Kubernetes label for the NavLink custom resource. + +### `sideLabel` + +Label that appears in the left navigation bar + +### `target` + +Sets the target property of the link's anchor tag (``), which (depending on browsers) determines if it opens in a new window or in an existing tab. + +The default value is `_self`, which opens the link on the current tab. To open the link in a new window or tab, set the target to `_blank`. + +For more information about the target property, see [this page.](https://www.w3schools.com/tags/att_a_target.asp) + +### `toService` + +Has five fields that are constructed to create a URL like the following: `https:///k8s/clusters//k8s/namespace//service/::/proxy/` + +For example, a link to a monitoring service can be set up as follows: + +- name: `rancher-monitoring-grafana` +- namespace: `cattle-monitoring-system` +- path: `proxy/?orgId=1` +- port: `"80"` +- scheme: `http` + +It is required to provide either the `toService` directive or the `toURL` directive. + +### `toUrl` + +Can be any link, even to links outside of the cluster. + +It is required to provide either the `toService` directive or the `toURL` directive. + +## Link Examples + +### Example of Link with `toUrl` + +This example NavLink YAML shows an example of configuring a NavLink to a Grafana dashboard: + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + name: grafana +spec: + group: "Monitoring Dashboards" + toURL: https:///api/v1/namespaces/cattle-monitoring-system/services/http:rancher-monitoring-grafana:80/proxy/?orgId=1 +``` + +Adding the above YAML results in a link to Grafana being created, as shown in the following screenshot: + +![Screenshot of Grafana Link](/img/example-grafana-link.png) + +### Example of Link with `toService` + +This example YAML shows an example of `toService` used for the link target: + +```yaml +apiVersion: ui.cattle.io/v1 +kind: NavLink +metadata: + annotations: + key: annotation + labels: + key: label + name: navlinkname +spec: + description: This is a description field # Optional. + group: "group1" # Optional. If not provided, the links appear standalone. + iconSrc: data:image/jpeg;base64,[icon source string is clipped for brevity] + label: This is a label # Optional. + sideLabel: A side label. # Optional. + target: _blank #Optional. _blank opens the link in a new tab or window. + toService: # toService or #toUrl needs to be provided. + name: rancher-monitoring-grafana + namespace: cattle-monitoring-system + path: proxy/?orgId=1 + port: "80" + scheme: http +``` + +Adding the `toService` parameters above results in a link to Grafana being created, as shown in the following screenshot: + +![Screenshot of Grafana Link](/img/example-service-link.png) + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md new file mode 100644 index 00000000000..eec64c568f2 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md @@ -0,0 +1,61 @@ +--- +title: Configuring a Global Default Private Registry +--- + + + + + +:::note +This page describes how to configure a global default private registry from the Rancher UI, after Rancher is already installed. + +For instructions on how to set up a private registry during Rancher installation, refer to the [air-gapped installation guide](../../../pages-for-subheaders/air-gapped-helm-cli-install.md). + +::: + +A private registry is a private, consistent, and centralized source of truth for the container images in your clusters. You can use a private container image registry to share custom base images within your organization. + +There are two main ways to set up a private registry in Rancher: + +* Set up a global default registry through the **Settings** tab in the global view. +* Set up a private registry in the advanced options under cluster-level settings. + +The global default registry is intended to be used in air-gapped setups, for registries that don't require credentials. The cluster-level private registry is intended to be used in setups where the private registry requires credentials. + +## Set a Private Registry with No Credentials as the Default Registry + +1. Log into Rancher and configure the default administrator password. +1. Select **☰ > Global Settings**. +1. Go to `system-default-registry` and choose **⋮ > Edit Setting**. +1. Enter your registry's hostname and port (e.g. `registry.yourdomain.com:port`). Do not prefix the text with `http://` or `https://`. + +**Result:** Rancher pulls system images from your private registry. + +### Namespaced Private Registry with RKE2 Downstream Clusters + +Most private registries should work, by default, with RKE2 downstream clusters. + +However, you'll need to do some additional steps if you're trying to set a namespaced private registry whose URL is formated like this: `website/subdomain:portnumber`. + +1. Select **☰ > Cluster Management**. +1. Find the RKE2 cluster in the list and click **⋮ >Edit Config**. +1. From the **Cluster config** menu, select **Registries**. +1. In the **Registries** pane, select the **Configure advanced containerd mirroring and registry authentication options** option. +1. In the text fields under **Mirrors**, enter the **Registry Hostname** and **Mirror Endpoints**. +1. Click **Save**. +1. Repeat as necessary for each downstream RKE2 cluster. + +## Configure a Private Registry with Credentials when Creating a Cluster + +There is no global way to set up a private registry with authorization for every Rancher-provisioned cluster. Therefore, if you want a Rancher-provisioned cluster to pull images from a private registry that requires credentials, you'll have to pass the registry credentials through the advanced cluster options every time you create a new cluster. + +Since the private registry cannot be configured after the cluster is created, you'll need to perform these steps during initial cluster setup. + +1. Select **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Choose a cluster type. +1. In the **Cluster Configuration** go to the **Registries** tab and select **Pull images for Rancher from a private registry**. +1. Enter the registry hostname and credentials. +1. Click **Create**. + +**Result:** The new cluster pulls images from the private registry. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md new file mode 100644 index 00000000000..74a2f048391 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md @@ -0,0 +1,240 @@ +--- +title: Cluster and Project Roles +--- + + + + + +Cluster and project roles define user authorization inside a cluster or project. + +To manage these roles, + +1. Click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles** and go to the **Cluster** or **Project/Namespaces** tab. + +### Membership and Role Assignment + +The projects and clusters accessible to non-administrative users is determined by _membership_. Membership is a list of users who have access to a specific cluster or project based on the roles they were assigned in that cluster or project. Each cluster and project includes a tab that a user with the appropriate permissions can use to manage membership. + +When you create a cluster or project, Rancher automatically assigns you as the `Owner` for it. Users assigned the `Owner` role can assign other users roles in the cluster or project. + +:::note + +Non-administrative users cannot access any existing projects/clusters by default. A user with appropriate permissions (typically the owner) must explicitly assign the project and cluster membership. + +::: + +### Cluster Roles + +_Cluster roles_ are roles that you can assign to users, granting them access to a cluster. There are two primary cluster roles: `Owner` and `Member`. + +- **Cluster Owner:** + + These users have full control over the cluster and all resources in it. + +- **Cluster Member:** + + These users can view most cluster level resources and create new projects. + +#### Custom Cluster Roles + +Rancher lets you assign _custom cluster roles_ to a standard user instead of the typical `Owner` or `Member` roles. These roles can be either a built-in custom cluster role or one defined by a Rancher administrator. They are convenient for defining narrow or specialized access for a standard user within a cluster. See the table below for a list of built-in custom cluster roles. + +#### Cluster Role Reference + +The following table lists each built-in custom cluster role available and whether that level of access is included in the default cluster-level permissions, `Cluster Owner` and `Cluster Member`. + +| Built-in Cluster Role | Owner | Member | +| ---------------------------------- | ------------- | --------------------------------- | +| Create Projects | ✓ | ✓ | +| Manage Cluster Backups             | ✓ | | +| Manage Cluster Catalogs | ✓ | | +| Manage Cluster Members | ✓ | | +| Manage Nodes [(see table below)](#Manage-Nodes-Permissions)| ✓ | | +| Manage Storage | ✓ | | +| View All Projects | ✓ | | +| View Cluster Catalogs | ✓ | ✓ | +| View Cluster Members | ✓ | ✓ | +| View Nodes | ✓ | ✓ | + +#### Manage Nodes Permissions + +The following table lists the permissions available for the `Manage Nodes` role in RKE and RKE2. + +| Manage Nodes Permissions | RKE | RKE2 | +|-----------------------------|-------- |--------- | +| SSH Access | ✓ | ✓ | +| Delete Nodes | ✓ | ✓ | +| Scale Clusters Up and Down | ✓ | * | +***In RKE2, you must have permission to edit a cluster to be able to scale clusters up and down.** +
+ +For details on how each cluster role can access Kubernetes resources, you can look them up in the Rancher UI: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Click the **Cluster** tab. +1. Click the name of an individual role. The table shows all of the operations and resources that are permitted by the role. + +:::note + +When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. + +::: + +### Giving a Custom Cluster Role to a Cluster Member + +After an administrator [sets up a custom cluster role,](custom-roles.md) cluster owners and admins can then assign those roles to cluster members. + +To assign a custom role to a new cluster member, you can use the Rancher UI. To modify the permissions of an existing member, you will need to use the Rancher API view. + +To assign the role to a new cluster member, + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to assign a role to a member and click **Explore**. +1. Click **RBAC > Cluster Members**. +1. Click **Add**. +1. In the **Cluster Permissions** section, choose the custom cluster role that should be assigned to the member. +1. Click **Create**. + + + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to assign a role to a member and click **Explore**. +1. Click **Cluster > Cluster Members**. +1. Click **Add**. +1. In the **Cluster Permissions** section, choose the custom cluster role that should be assigned to the member. +1. Click **Create**. + + + + +**Result:** The member has the assigned role. + +To assign any custom role to an existing cluster member, + +1. Click **☰ > Users & Authentication**. +1. Go to the member you want to give the role to. Click the **⋮ > Edit Config**. +1. If you have added custom roles, they will show in the **Custom** section. Choose the role you want to assign to the member. +1. Click **Save**. + +**Result:** The member has the assigned role. + +### Project Roles + +_Project roles_ are roles that can be used to grant users access to a project. There are three primary project roles: `Owner`, `Member`, and `Read Only`. + +- **Project Owner:** + + These users have full control over the project and all resources in it. + +- **Project Member:** + + These users can manage project-scoped resources like namespaces and workloads, but cannot manage other project members. + +:::note + +By default, the Rancher role of `project-member` inherits from the `Kubernetes-edit` role, and the `project-owner` role inherits from the `Kubernetes-admin` role. As such, both `project-member` and `project-owner` roles will allow for namespace management, including the ability to create and delete namespaces. + +::: + +- **Read Only:** + + These users can view everything in the project but cannot create, update, or delete anything. + +:::danger + +Users assigned the `Owner` or `Member` role for a project automatically inherit the `namespace creation` role. However, this role is a [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole), meaning its scope extends to all projects in the cluster. Therefore, users explicitly assigned the `owner` or `member` role for a project can create namespaces in other projects they're assigned to, even with only the `Read Only` role assigned. + +::: + +#### Custom Project Roles + +Rancher lets you assign _custom project roles_ to a standard user instead of the typical `Owner`, `Member`, or `Read Only` roles. These roles can be either a built-in custom project role or one defined by a Rancher administrator. They are convenient for defining narrow or specialized access for a standard user within a project. See the table below for a list of built-in custom project roles. + +#### Project Role Reference + +The following table lists each built-in custom project role available in Rancher and whether it is also granted by the `Owner`, `Member`, or `Read Only` role. + +| Built-in Project Role | Owner | Member | Read Only | +| ---------------------------------- | ------------- | ----------------------------- | ------------- | +| Manage Project Members | ✓ | | | +| Create Namespaces | ✓ | ✓ | | +| Manage Config Maps | ✓ | ✓ | | +| Manage Ingress | ✓ | ✓ | | +| Manage Project Catalogs | ✓ | | | +| Manage Secrets | ✓ | ✓ | | +| Manage Service Accounts | ✓ | ✓ | | +| Manage Services | ✓ | ✓ | | +| Manage Volumes | ✓ | ✓ | | +| Manage Workloads | ✓ | ✓ | | +| View Secrets | ✓ | ✓ | | +| View Config Maps | ✓ | ✓ | ✓ | +| View Ingress | ✓ | ✓ | ✓ | +| View Project Members | ✓ | ✓ | ✓ | +| View Project Catalogs | ✓ | ✓ | ✓ | +| View Service Accounts | ✓ | ✓ | ✓ | +| View Services | ✓ | ✓ | ✓ | +| View Volumes | ✓ | ✓ | ✓ | +| View Workloads | ✓ | ✓ | ✓ | + +:::note Notes: + +- Each project role listed above, including `Owner`, `Member`, and `Read Only`, is comprised of multiple rules granting access to various resources. You can view the roles and their rules on the Global > Security > Roles page. +- When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. +- The `Manage Project Members` role allows the project owner to manage any members of the project **and** grant them any project scoped role regardless of their access to the project resources. Be cautious when assigning this role out individually. + +::: + +### Defining Custom Roles +As previously mentioned, custom roles can be defined for use at the cluster or project level. The context field defines whether the role will appear on the cluster member page, project member page, or both. + +When defining a custom role, you can grant access to specific resources or specify roles from which the custom role should inherit. A custom role can be made up of a combination of specific grants and inherited roles. All grants are additive. This means that defining a narrower grant for a specific resource **will not** override a broader grant defined in a role that the custom role is inheriting from. + +### Default Cluster and Project Roles + +By default, when a standard user creates a new cluster or project, they are automatically assigned an ownership role: either [cluster owner](#cluster-roles) or [project owner](#project-roles). However, in some organizations, these roles may overextend administrative access. In this use case, you can change the default role to something more restrictive, such as a set of individual roles or a custom role. + +There are two methods for changing default cluster/project roles: + +- **Assign Custom Roles**: Create a [custom role](custom-roles.md) for either your [cluster](#custom-cluster-roles) or [project](#custom-project-roles), and then set the custom role as default. + +- **Assign Individual Roles**: Configure multiple [cluster](#cluster-role-reference)/[project](#project-role-reference) roles as default for assignment to the creating user. + + For example, instead of assigning a role that inherits other roles (such as `cluster owner`), you can choose a mix of individual roles (such as `manage nodes` and `manage storage`). + +:::note + +- Although you can [lock](locked-roles.md) a default role, the system still assigns the role to users who create a cluster/project. +- Only users that create clusters/projects inherit their roles. Users added to the cluster/project membership afterward must be explicitly assigned their roles. + +::: + +### Configuring Default Roles for Cluster and Project Creators + +You can change the cluster or project role(s) that are automatically assigned to the creating user. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Click the **Cluster** or **Project/Namespaces** tab. +1. Find the custom or individual role that you want to use as default. Then edit the role by selecting **⋮ > Edit Config**. +1. In the **Cluster Creator Default** or **Project Creator Default** section, enable the role as the default. +1. Click **Save**. + +**Result:** The default roles are configured based on your changes. Roles assigned to cluster/project creators display a check in the **Cluster/Project Creator Default** column. + +If you want to remove a default role, edit the permission and select **No** from the default roles option. + +### Cluster Membership Revocation Behavior + +When you revoke the cluster membership for a standard user that's explicitly assigned membership to both the cluster _and_ a project within the cluster, that standard user [loses their cluster roles](#cluster-roles) but [retains their project roles](#project-roles). In other words, although you have revoked the user's permissions to access the cluster and its nodes, the standard user can still: + +- Access the projects they hold membership in. +- Exercise any [individual project roles](#project-role-reference) they are assigned. + +If you want to completely revoke a user's access within a cluster, revoke both their cluster and project memberships. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md new file mode 100644 index 00000000000..de61e0f4fb7 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md @@ -0,0 +1,124 @@ +--- +title: Custom Roles +--- + + + + + +Within Rancher, _roles_ determine what actions a user can make within a cluster or project. + +Note that _roles_ are different from _permissions_, which determine what clusters and projects you can access. + +:::danger + +It is possible for a custom role to enable privilege escalation. For details, see [this section.](#privilege-escalation) + +::: + + +## Prerequisites + +To complete the tasks on this page, one of the following permissions are required: + + - [Administrator Global Permissions](global-permissions.md). + - [Custom Global Permissions](global-permissions.md#custom-global-permissions) with the [Manage Roles](global-permissions.md) role assigned. + +## Creating A Custom Role + +While Rancher comes out-of-the-box with a set of default user roles, you can also create default custom roles to provide users with very specific permissions within Rancher. + +The steps to add custom roles differ depending on the version of Rancher. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Select a tab to determine the scope of the role you're adding. The tabs are: + + - **Global:** The role is valid for allowing members to manage global scoped resources. + - **Cluster:** The role is valid for assignment when adding/managing members to clusters. + - **Project/Namespaces:** The role is valid for assignment when adding/managing members to projects or namespaces. + +1. Click **Create Global Role,** **Create Cluster Role** or **Create Project/Namespaces Role,** depending on the scope. +1. Enter a **Name** for the role. +1. Optional: Choose the **Cluster/Project Creator Default** option to assign this role to a user when they create a new cluster or project. Using this feature, you can expand or restrict the default roles for cluster/project creators. + + > Out of the box, the Cluster Creator Default and the Project Creator Default roles are `Cluster Owner` and `Project Owner` respectively. + +1. Use the **Grant Resources** options to assign individual [Kubernetes API endpoints](https://kubernetes.io/docs/reference/) to the role. + + > When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. + + > The Resource text field provides a method to search for pre-defined Kubernetes API resources, or enter a custom resource name for the grant. The pre-defined or `(Custom)` resource must be selected from the dropdown, after entering a resource name into this field. + + You can also choose the individual cURL methods (`Create`, `Delete`, `Get`, etc.) available for use with each endpoint you assign. + +1. Use the **Inherit from** options to assign individual Rancher roles to your custom roles. Note: When a custom role inherits from a parent role, the parent role cannot be deleted until the child role is deleted. + +1. Click **Create**. + +## Creating a Custom Role that Inherits from Another Role + +If you have a group of individuals that need the same level of access in Rancher, it can save time to create a custom role in which all of the rules from another role, such as the administrator role, are copied into a new role. This allows you to only configure the variations between the existing role and the new role. + +The custom role can then be assigned to a user or group so that the role takes effect the first time the user or users sign into Rancher. + +To create a custom role based on an existing role, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Click the **Cluster** or **Project/Namespaces** tab. Click **Create Cluster Role** or **Create Project/Namespaces Role** depending on the scope. Note: Only cluster roles and project/namespace roles can inherit from another role. +1. Enter a name for the role. +1. In the **Inherit From** tab, select the role(s) that the custom role will inherit permissions from. +1. In the **Grant Resources** tab, select the Kubernetes resource operations that will be enabled for users with the custom role. + + > The Resource text field provides a method to search for pre-defined Kubernetes API resources, or enter a custom resource name for the grant. The pre-defined or `(Custom)` resource must be selected from the dropdown, after entering a resource name into this field. +1. Optional: Assign the role as default. +1. Click **Create**. + +## Deleting a Custom Role + +When deleting a custom role, all global role bindings with this custom role are deleted. + +If a user is only assigned one custom role, and the role is deleted, the user would lose access to Rancher. For the user to regain access, an administrator would need to edit the user and apply new global permissions. + +Custom roles can be deleted, but built-in roles cannot be deleted. + +To delete a custom role, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +2. Go to the custom global role that should be deleted and click **⋮ (…) > Delete**. +3. Click **Delete**. + +## Assigning a Custom Role to a Group + +If you have a group of individuals that need the same level of access in Rancher, it can save time to create a custom role. When the role is assigned to a group, the users in the group have the appropriate level of access the first time they sign into Rancher. + +When a user in the group logs in, they get the built-in Standard User global role by default. They will also get the permissions assigned to their groups. + +If a user is removed from the external authentication provider group, they would lose their permissions from the custom role that was assigned to the group. They would continue to have their individual Standard User role. + +:::note Prerequisites: + +You can only assign a global role to a group if: + +* You have set up an [external authentication provider](../../../../pages-for-subheaders/authentication-config.md#external-vs-local-authentication) +* The external authentication provider supports [user groups](../../authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md) +* You have already set up at least one user group with the authentication provider + +::: + +To assign a custom role to a group, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Groups**. +1. Go to the existing group that will be assigned the custom role and click **⋮ > Edit Config**. +1. If you have created roles, they will show in the **Custom** section. Choose any custom role that will be assigned to the group. +1. Optional: In the **Global Permissions** or **Built-in** sections, select any additional permissions that the group should have. +1. Click **Save.**. + +**Result:** The custom role will take effect when the users in the group log into Rancher. + +## Privilege Escalation + +The `Configure Catalogs` custom permission is powerful and should be used with caution. When an admin assigns the `Configure Catalogs` permission to a standard user, it could result in privilege escalation in which the user could give themselves admin access to Rancher provisioned clusters. Anyone with this permission should be considered equivalent to an admin. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md new file mode 100644 index 00000000000..64411298a62 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md @@ -0,0 +1,251 @@ +--- +title: Global Permissions +--- + + + + + +_Permissions_ are individual access rights that you can assign when selecting a custom permission for a user. + +Global Permissions define user authorization outside the scope of any particular cluster. Out-of-the-box, there are four default global permissions: `Administrator`, `Restricted Admin`,`Standard User` and `User-base`. + +- **Administrator:** These users have full control over the entire Rancher system and all clusters within it. + +- **Restricted Admin:** These users have full control over downstream clusters, but cannot alter the local Kubernetes cluster. + +- **Standard User:** These users can create new clusters and use them. Standard users can also assign other users permissions to their clusters. + +- **User-Base:** User-Base users have login-access only. + +You cannot update or delete the built-in Global Permissions. + +## Restricted Admin + +A new `restricted-admin` role was created in Rancher v2.5 in order to prevent privilege escalation from the local Rancher server Kubernetes cluster. This role has full administrator access to all downstream clusters managed by Rancher, but it does not have permission to alter the local Kubernetes cluster. + +The `restricted-admin` can create other `restricted-admin` users with an equal level of access. + +A new setting was added to Rancher to set the initial bootstrapped administrator to have the `restricted-admin` role. This applies to the first user created when the Rancher server is started for the first time. If the environment variable is set, then no global administrator would be created, and it would be impossible to create the global administrator through Rancher. + +To bootstrap Rancher with the `restricted-admin` as the initial user, the Rancher server should be started with the following environment variable: + +``` +CATTLE_RESTRICTED_DEFAULT_ADMIN=true +``` +### List of `restricted-admin` Permissions + +The following table lists the permissions and actions that a `restricted-admin` should have in comparison with the `Administrator` and `Standard User` roles: + +| Category | Action | Global Admin | Standard User | Restricted Admin | Notes for Restricted Admin role | +| -------- | ------ | ------------ | ------------- | ---------------- | ------------------------------- | +| Local Cluster functions | Manage Local Cluster (List, Edit, Import Host) | Yes | No | No | | +| | Create Projects/namespaces | Yes | No | No | | +| | Add cluster/project members | Yes | No | No | | +| | Global DNS | Yes | No | No | | +| | Access to management cluster for CRDs and CRs | Yes | No | Yes | | +| | Save as RKE Template | Yes | No | No | | +| Security | | | | | | +| Enable auth | Configure Authentication | Yes | No | Yes | | +| Roles | Create/Assign GlobalRoles | Yes | No (Can list) | Yes | Auth webhook allows creating globalrole for perms already present | +| | Create/Assign ClusterRoles | Yes | No (Can list) | Yes | Not in local cluster | +| | Create/Assign ProjectRoles | Yes | No (Can list) | Yes | Not in local cluster | +| Users | Add User/Edit/Delete/Deactivate User | Yes | No | Yes | | +| Groups | Assign Global role to groups | Yes | No | Yes | As allowed by the webhook | +| | Refresh Groups | Yes | No | Yes | | +| PSP's | Manage PSP templates | Yes | No (Can list) | Yes | Same privileges as Global Admin for PSPs | +| Tools | | | | | | +| | Manage RKE Templates | Yes | No | Yes | | +| | Manage Global Catalogs | Yes | No | Yes | Cannot edit/delete built-in system catalog. Can manage Helm library | +| | Cluster Drivers | Yes | No | Yes | | +| | Node Drivers | Yes | No | Yes | | +| | GlobalDNS Providers | Yes | Yes (Self) | Yes | | +| | GlobalDNS Entries | Yes | Yes (Self) | Yes | | +| Settings | | | | | | +| | Manage Settings | Yes | No (Can list) | No (Can list) | | +| User | | | | | | +| | Manage API Keys | Yes (Manage all) | Yes (Manage self) | Yes (Manage self) | | +| | Manage Node Templates | Yes | Yes (Manage self) | Yes (Manage self) | Can only manage their own node templates and not those created by other users | +| | Manage Cloud Credentials | Yes | Yes (Manage self) | Yes (Manage self) | Can only manage their own cloud credentials and not those created by other users | +| Downstream Cluster | Create Cluster | Yes | Yes | Yes | | +| | Edit Cluster | Yes | Yes | Yes | | +| | Rotate Certificates | Yes | | Yes | | +| | Snapshot Now | Yes | | Yes | | +| | Restore Snapshot | Yes | | Yes | | +| | Save as RKE Template | Yes | No | Yes | | +| | Run CIS Scan | Yes | Yes | Yes | | +| | Add Members | Yes | Yes | Yes | | +| | Create Projects | Yes | Yes | Yes | | +| Feature Charts since v2.5 | | | | | | +| | Install Fleet | Yes | | Yes | Should not be able to run Fleet in local cluster | +| | Deploy EKS cluster | Yes | Yes | Yes | | +| | Deploy GKE cluster | Yes | Yes | Yes | | +| | Deploy AKS cluster | Yes | Yes | Yes | | + + +### Changing Global Administrators to Restricted Admins + +If Rancher already has a global administrator, they should change all global administrators over to the new `restricted-admin` role. + +This can be done through **Security > Users** and moving any Administrator role over to Restricted Administrator. + +Signed-in users can change themselves over to the `restricted-admin` if they wish, but they should only do that as the last step, otherwise they won't have the permissions to do so. + +## Global Permission Assignment + +Global permissions for local users are assigned differently than users who log in to Rancher using external authentication. + +### Global Permissions for New Local Users + +When you create a new local user, you assign them a global permission as you complete the **Add User** form. + +To see the default permissions for new users, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. The **Roles** page has tabs for roles grouped by scope. Each table lists the roles in that scope. In the **Global** tab, in the **New User Default** column, the permissions given to new users by default are indicated with a checkmark. + +You can [change the default global permissions to meet your needs.](#configuring-default-global-permissions) + +### Global Permissions for Users with External Authentication + +When a user logs into Rancher using an external authentication provider for the first time, they are automatically assigned the **New User Default** global permissions. By default, Rancher assigns the **Standard User** permission for new users. + +To see the default permissions for new users, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. The **Roles** page has tabs for roles grouped by scope. Each table lists the roles in that scope. In the **New User Default** column on each page, the permissions given to new users by default are indicated with a checkmark. + +You can [change the default permissions to meet your needs.](#configuring-default-global-permissions) + +Permissions can be [assigned](#configuring-global-permissions-for-individual-users) to an individual user. + +You can [assign a role to everyone in the group at the same time](#configuring-global-permissions-for-groups) if the external authentication provider supports groups. + +## Custom Global Permissions + +Using custom permissions is convenient for providing users with narrow or specialized access to Rancher. + +When a user from an [external authentication source](../../../../pages-for-subheaders/authentication-config.md) signs into Rancher for the first time, they're automatically assigned a set of global permissions (hereafter, permissions). By default, after a user logs in for the first time, they are created as a user and assigned the default `user` permission. The standard `user` permission allows users to login and create clusters. + +However, in some organizations, these permissions may extend too much access. Rather than assigning users the default global permissions of `Administrator` or `Standard User`, you can assign them a more restrictive set of custom global permissions. + +The default roles, Administrator and Standard User, each come with multiple global permissions built into them. The Administrator role includes all global permissions, while the default user role includes three global permissions: Create Clusters, Use Catalog Templates, and User Base, which is equivalent to the minimum permission to log in to Rancher. In other words, the custom global permissions are modularized so that if you want to change the default user role permissions, you can choose which subset of global permissions are included in the new default user role. + +Administrators can enforce custom global permissions in multiple ways: + +- [Changing the default permissions for new users](#configuring-default-global-permissions) +- [Configuring global permissions for individual users](#configuring-global-permissions-for-individual-users) +- [Configuring global permissions for groups](#configuring-global-permissions-for-groups) + +### Custom Global Permissions Reference + +The following table lists each custom global permission available and whether it is included in the default global permissions, `Administrator`, `Standard User` and `User-Base`. + +| Custom Global Permission | Administrator | Standard User | User-Base | +| ---------------------------------- | ------------- | ------------- |-----------| +| Create Clusters | ✓ | ✓ | | +| Create RKE Templates | ✓ | ✓ | | +| Manage Authentication | ✓ | | | +| Manage Catalogs | ✓ | | | +| Manage Cluster Drivers | ✓ | | | +| Manage Node Drivers | ✓ | | | +| Manage PodSecurityPolicy Templates | ✓ | | | +| Manage Roles | ✓ | | | +| Manage Settings | ✓ | | | +| Manage Users | ✓ | | | +| Use Catalog Templates | ✓ | ✓ | | +| User-Base (Basic log-in access) | ✓ | ✓ | | + +For details on which Kubernetes resources correspond to each global permission, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. If you click the name of an individual role, a table shows all of the operations and resources that are permitted by the role. + +:::note Notes: + +- Each permission listed above is comprised of multiple individual permissions not listed in the Rancher UI. For a full list of these permissions and the rules they are comprised of, access through the API at `/v3/globalRoles`. +- When viewing the resources associated with default roles created by Rancher, if there are multiple Kubernetes API resources on one line item, the resource will have `(Custom)` appended to it. These are not custom resources but just an indication that there are multiple Kubernetes API resources as one resource. + +::: + +### Configuring Default Global Permissions + +If you want to restrict the default permissions for new users, you can remove the `user` permission as default role and then assign multiple individual permissions as default instead. Conversely, you can also add administrative permissions on top of a set of other standard permissions. + +:::note + +Default roles are only assigned to users added from an external authentication provider. For local users, you must explicitly assign global permissions when adding a user to Rancher. You can customize these global permissions when adding the user. + +::: + +To change the default global permissions that are assigned to external users upon their first log in, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. On the **Roles** page, make sure the **Global** tab is selected. +1. Find the permissions set that you want to add or remove as a default. Then edit the permission by selecting **⋮ > Edit Config**. +1. If you want to add the permission as a default, Select **Yes: Default role for new users** and then click **Save**. If you want to remove a default permission, edit the permission and select **No**. + +**Result:** The default global permissions are configured based on your changes. Permissions assigned to new users display a check in the **New User Default** column. + +### Configuring Global Permissions for Individual Users + +To configure permission for a user, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Go to the user whose access level you want to change and click **⋮ > Edit Config**. +1. In the **Global Permissions** and **Built-in** sections, check the boxes for each permission you want the user to have. If you have created roles from the **Roles** page, they will appear in the **Custom** section and you can choose from them as well. +1. Click **Save**. + +**Result:** The user's global permissions have been updated. + +### Configuring Global Permissions for Groups + +If you have a group of individuals that need the same level of access in Rancher, it can save time to assign permissions to the entire group at once, so that the users in the group have the appropriate level of access the first time they sign into Rancher. + +After you assign a custom global role to a group, the custom global role will be assigned to a user in the group when they log in to Rancher. + +For existing users, the new permissions will take effect when the users log out of Rancher and back in again, or when an administrator [refreshes the group memberships.](#refreshing-group-memberships) + +For new users, the new permissions take effect when the users log in to Rancher for the first time. New users from this group will receive the permissions from the custom global role in addition to the **New User Default** global permissions. By default, the **New User Default** permissions are equivalent to the **Standard User** global role, but the default permissions can be [configured.](#configuring-default-global-permissions) + +If a user is removed from the external authentication provider group, they would lose their permissions from the custom global role that was assigned to the group. They would continue to have any remaining roles that were assigned to them, which would typically include the roles marked as **New User Default**. Rancher will remove the permissions that are associated with the group when the user logs out, or when an administrator [refreshes group memberships,](#refreshing-group-memberships) whichever comes first. + +:::note Prerequisites: + +You can only assign a global role to a group if: + +* You have set up an [external authentication provider](../../../../pages-for-subheaders/authentication-config.md#external-vs-local-authentication) +* The external authentication provider supports [user groups](../authentication-config/manage-users-and-groups.md) +* You have already set up at least one user group with the authentication provider + +::: + +To assign a custom global role to a group, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Groups**. +1. Go to the group you want to assign a custom global role to and click **⋮ > Edit Config**. +1. In the **Global Permissions,** **Custom,** and/or **Built-in** sections, select the permissions that the group should have. +1. Click **Create**. + +**Result:** The custom global role will take effect when the users in the group log into Rancher. + +### Refreshing Group Memberships + +When an administrator updates the global permissions for a group, the changes take effect for individual group members after they log out of Rancher and log in again. + +To make the changes take effect immediately, an administrator or cluster owner can refresh group memberships. + +An administrator might also want to refresh group memberships if a user is removed from a group in the external authentication service. In that case, the refresh makes Rancher aware that the user was removed from the group. + +To refresh group memberships, + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Users**. +1. Click **Refresh Group Memberships**. + +**Result:** Any changes to the group members' permissions will take effect. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md new file mode 100644 index 00000000000..4c4aec0be54 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md @@ -0,0 +1,42 @@ +--- +title: Locked Roles +--- + + + + + +You can set roles to a status of `locked`. Locking roles prevent them from being assigned to users in the future. + +Locked roles: + +- Cannot be assigned to users that don't already have it assigned. +- Are not listed in the **Member Roles** drop-down when you are adding a user to a cluster or project. +- Do not affect users assigned the role before you lock the role. These users retain access that the role provides. + + **Example:** let's say your organization creates an internal policy that users assigned to a cluster are prohibited from creating new projects. It's your job to enforce this policy. + + To enforce it, before you add new users to the cluster, you should lock the following roles: `Cluster Owner`, `Cluster Member`, and `Create Projects`. Then you could create a new custom role that includes the same permissions as a __Cluster Member__, except the ability to create projects. Then, you use this new custom role when adding users to a cluster. + +Roles can be locked by the following users: + +- Any user assigned the `Administrator` global permission. +- Any user assigned the `Custom Users` permission, along with the `Manage Roles` role. + + +## Locking/Unlocking Roles + +If you want to prevent a role from being assigned to users, you can set it to a status of `locked`. + +You can lock roles in two contexts: + +- When you're [adding a custom role](custom-roles.md). +- When you editing an existing role (see below). + +Cluster roles and project/namespace roles can be locked, but global roles cannot. + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Roles**. +1. Go to the **Cluster** tab or the **Project/Namespaces** tab. +1. From the role that you want to lock (or unlock), select **⋮ > Edit Config**. +1. From the **Locked** option, choose the **Yes** or **No** radio button. Then click **Save**. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md new file mode 100644 index 00000000000..654149cf6e4 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards.md @@ -0,0 +1,130 @@ +--- +title: Pod Security Standards (PSS) & Pod Security Admission (PSA) +--- + + + + + +[Pod Security Standards (PSS)](https://kubernetes.io/docs/concepts/security/pod-security-standards/) and [Pod Security Admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) define security restrictions for a broad set of workloads. +They became available and were turned on by default in Kubernetes v1.23, and replace [Pod Security Policies (PSP)](https://kubernetes.io/docs/concepts/security/pod-security-policy/) in Kubernetes v1.25 and above. + +PSS define security levels for workloads. PSAs describe requirements for pod security contexts and related fields. PSAs reference PSS levels to define security restrictions. + +## Upgrade to Pod Security Standards (PSS) + +Ensure that you migrate all PSPs to another workload security mechanism. This includes mapping your current PSPs to Pod Security Standards for enforcement with the [PSA controller](https://kubernetes.io/docs/concepts/security/pod-security-admission/). If the PSA controller won't meet all of your organization's needs, we recommend that you use a policy engine, such as [OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper), [Kubewarden](https://www.kubewarden.io/), [Kyverno](https://kyverno.io/), or [NeuVector](https://neuvector.com/). Refer to the documentation of your policy engine of choice for more information on how to migrate from PSPs. + +:::caution +You must add your new policy enforcement mechanisms _before_ you remove the PodSecurityPolicy objects. If you don't, you may create an opportunity for privilege escalation attacks within the cluster. +::: + +### Removing PodSecurityPolicies from Rancher-Maintained Apps & Marketplace Workloads {#remove-psp-rancher-workloads} + +Rancher v2.7.2 offers a new major version of Rancher-maintained Helm charts. v102.x.y allows you to remove PSPs that were installed with previous versions of the chart. This new version replaces non-standard PSPs switches with the standardized `global.cattle.psp.enabled` switch, which is turned off by default. + +You must perform the following steps _while still in Kubernetes v1.24_: +1. Configure the PSA controller to suit your needs. You can use one of Rancher's built-in [PSA Configuration Templates](#psa-config-templates), or create a custom template and apply it to the clusters that you are migrating. + +1. Map your active PSPs to Pod Security Standards: + 1. See which PSPs are still active in your cluster: + :::caution + This strategy may miss workloads that aren't currently running, such as CronJobs, workloads currently scaled to zero, or workloads that haven't rolled out yet. + ::: + + ```shell + kubectl get pods \ + --all-namespaces \ + --output jsonpath='{.items[*].metadata.annotations.kubernetes\.io\/psp}' \ + | tr " " "\n" | sort -u + ``` + + 1. Follow the Kubernetes guide on [Mapping PSPs to Pod Security Standards](https://kubernetes.io/docs/reference/access-authn-authz/psp-to-pod-security-standards/) to apply PSSs to your workloads that were relying on PSPs. See [Migrate from PodSecurityPolicy to the Built-In PodSecurity Admission controller](https://kubernetes.io/docs/tasks/configure-pod-container/migrate-from-psp/) for more details. + +1. To remove PSPs from Rancher charts, upgrade the charts to the latest v102.x.y version _before_ you upgrade to Kubernetes v1.25. Make sure that the **Enable PodSecurityPolicies** option is **disabled**. This will remove any PSPs that were installed with previous chart versions. + +:::info important +If you want to upgrade your charts to v102.x.y, but don't plan on upgrading your clusters to Kubernetes v1.25 and moving away from PSPs, make sure that you select the option **Enable PodSecurityPolicies** for each chart that you are upgrading. +::: + +### Cleaning Up Releases After a Kubernetes v1.25 Upgrade + +If you experience problems while removing PSPs from your charts, or have charts that don't contain a built-in mechanism for removing PSPs, your chart upgrades or deletions might fail with an error message such as the following: +```console +Error: UPGRADE FAILED: resource mapping not found for name: "" namespace: "" from "": no matches for kind "PodSecurityPolicy" in version "policy/v1beta1" +ensure CRDs are installed first +``` + +This happens when Helm tries to query the cluster for objects that were stored in a previous release's data blob. To clean up these releases and avoid this error, use the `helm-mapkubeapis` Helm plugin. To learn more about `helm-mapkubeapis`, how it works, and how it can be fine-tuned for your use case, see the [official Helm documentation](https://github.com/helm/helm-mapkubeapis#readme). + +Note that Helm plugin installation is local to the machine that you run the commands from. Therefore, make sure that you run both the installation and cleanup from the same machine. + +#### Install `helm-mapkubeapis` + +1. Open your terminal in the machine you intend to use `helm-mapkubeapis` from and install the plugin: + ```shell + helm plugin install https://github.com/helm/helm-mapkubeapis + ``` + + You will see output similar to the following: + ```console + Downloading and installing helm-mapkubeapis v0.4.1 ... + https://github.com/helm/helm-mapkubeapis/releases/download/v0.4.1/helm-mapkubeapis_0.4.1_darwin_amd64.tar.gz + Installed plugin: mapkubeapis + ``` + + :::info important + Ensure that the `helm-mapkubeapis` plugin is at least v0.4.1, as older versions _do not_ support removal of resources. + ::: + +1. Verify that the plugin was correctly installed: + ```shell + helm mapkubeapis --help + ``` + + You will see output similar to the following: + ```console + Map release deprecated or removed Kubernetes APIs in-place + + Usage: + mapkubeapis [flags] RELEASE + + Flags: + --dry-run simulate a command + -h, --help help for mapkubeapis + --kube-context string name of the kubeconfig context to use + --kubeconfig string path to the kubeconfig file + --mapfile string path to the API mapping file + --namespace string namespace scope of the release + ``` + +#### Cleaning Up Broken Releases + +After you install the `helm-mapkubeapis` plugin, clean up the releases that became broken after the upgrade to Kubernetes v1.25. + +1. Open your preferred terminal and make sure it's connected to the cluster you wish to target by running `kubectl cluster-info`. + +1. List all the releases you have installed in your cluster by running `helm list --all-namespaces`. + +1. Perform a dry run for each release you would like to clean up by running `helm mapkubeapis --dry-run --namespace `. The result of this command will inform you what resources are going to be replaced or removed. + +1. Finally, after reviewing the changes, perform a full run with `helm mapkubeapis --namespace `. + +#### Upgrading Charts to a Version That Supports Kubernetes v1.25 + +You can proceed with your upgrade once any releases that had lingering PSPs are cleaned up. For Rancher-maintained workloads, follow the steps outlined in the [Removing PodSecurityPolicies from Rancher-maintained Apps & Marketplace workloads](#remove-psp-rancher-workloads) section of this document. +For workloads not maintained by Rancher, refer to the vendor documentation. + +:::caution +Do not skip this step. Applications incompatible with Kubernetes v1.25 aren't guaranteed to work after a cleanup. +::: + +## Pod Security Admission Configuration Templates {#psa-config-templates} + +Rancher offers PSA configuration templates. These are pre-defined security configurations that you can apply to a cluster. Rancher admins (or those with the right permissions) can [create, manage, and edit](./psa-config-templates.md) PSA templates. + +### Rancher on PSA-restricted Clusters + +Rancher system namespaces are also affected by the restrictive security policies described by PSA templates. You need to exempt Rancher's system namespaces after you assign the template, or else the cluster won't operate correctly. See [Pod Security Admission (PSA) Configuration Templates](./psa-config-templates.md#exempting-required-rancher-namespaces) for more details. + +For a complete file which has all the exemptions you need to run Rancher, please refer to this [sample Admission Configuration](../../../reference-guides/rancher-security/psa-restricted-exemptions.md). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md new file mode 100644 index 00000000000..dac4d66ee52 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md @@ -0,0 +1,142 @@ +--- +title: Pod Security Admission (PSA) Configuration Templates +--- + + + + + +[Pod Security admission (PSA)](./pod-security-standards.md) configuration templates are a Rancher custom-defined resource (CRD), available in Rancher v2.7.2 and above. The templates provide pre-defined security configurations that you can apply to a cluster: + +- `rancher-privileged`: The most permissive configuration. It doesn't restrict the behavior of any pods. This allows for known privilege escalations. This policy has no exemptions. +- `rancher-restricted`: A heavily restricted configuration that follows current best practices for hardening pods. You must make [namespace-level exemptions](./pod-security-standards.md#rancher-on-psa-restricted-clusters) for Rancher components. + +## Assign a Pod Security Admissions (PSA) Configuration Template + +You can assign a PSA template at the same time that you create a downstream cluster. You can also add a template by configuring an existing cluster. + +### Assign a Template During Cluster Creation + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click the **Create** button. +1. Select a provider. +1. On the **Cluster: Create** page, go to **Basics > Security**. +1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. +1. Click **Create**. + +### Assign a Template to an Existing Cluster + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config** . +1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. +1. Click **Save**. + +### Hardening the Cluster + +If you select the **rancher-restricted** template but don't select a **CIS Profile**, you won't meet required CIS benchmarks. See the [RKE2 hardening guide](../../../pages-for-subheaders/rke2-hardening-guide.md) for more details. + + + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click the **Create** button. +1. Select a provider. +1. On the **Add Cluster** page, under **Cluster Options**, click **Advanced Options**. +1. In the **Pod Security Admission Configuration Template** dropdown menu, select the template you want to assign. +1. Click **Create**. + +### Assign a Template to an Existing Cluster + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Find the cluster you want to update in the **Clusters** table, and click the **⋮**. +1. Select **Edit Config**. +1. On the **Edit Cluster** page, go to **Cluster Options > Advanced Options**. +1. In the **Pod Security Admission Configuration Template**, select the template you want to assign. +1. Click **Save**. + + + + +## Add or Edit a Pod Security Admissions (PSA) Configuration Template + +If you have administrator privileges, you can customize security restrictions and permissions by creating additional PSA templates, or by editing existing templates. + +:::caution +If you edit an existing PSA template while it is still in use, changes will be applied to all clusters that have been assigned to that template. +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **Advanced** to open the dropdown menu. +1. Select **Pod Security Admissions**. +1. Find the template you want to modify, and click the **⋮**. +1. Select **Edit Config** to edit the template. +1. When you're done editing the configuration, click **Save**. + +### Allow Non-Admin Users to Manage PSA Templates + +If you want to allow other users to manage templates, you can bind that user to a role that grants all verbs (`"*"`) on `management.cattle.io/podsecurityadmissionconfigurationtemplates`. + +:::caution +Any user that is bound to the above permission will be able to change the restriction levels on _all_ managed clusters which use a given PSA template, including ones that they have no permissions on. +::: + +## Exempting Required Rancher Namespaces + +When you run Rancher on a Kubernetes cluster that enforces a restrictive security policy by default, you'll need to [exempt the following namespaces](#exempting-namespaces), otherwise the policy might prevent Rancher system pods from running properly. + +- `calico-apiserver` +- `calico-system` +- `cattle-alerting` +- `cattle-csp-adapter-system` +- `cattle-epinio-system` +- `cattle-externalip-system` +- `cattle-fleet-local-system` +- `cattle-fleet-system` +- `cattle-gatekeeper-system` +- `cattle-global-data` +- `cattle-global-nt` +- `cattle-impersonation-system` +- `cattle-istio` +- `cattle-istio-system` +- `cattle-logging` +- `cattle-logging-system` +- `cattle-monitoring-system` +- `cattle-neuvector-system` +- `cattle-prometheus` +- `cattle-sriov-system` +- `cattle-system` +- `cattle-ui-plugin-system` +- `cattle-windows-gmsa-system` +- `cert-manager` +- `cis-operator-system` +- `fleet-default` +- `ingress-nginx` +- `istio-system` +- `kube-node-lease` +- `kube-public` +- `kube-system` +- `longhorn-system` +- `rancher-alerting-drivers` +- `security-scan` +- `tigera-operator` + +Rancher, some Rancher owned charts, and RKE2 and K3s distributions all use these namespaces. A subset of the listed namespaces are already exempt in the built-in Rancher `rancher-restricted` policy, for use in downstream clusters. For a complete template which has all the exemptions you need to run Rancher, please refer to this [sample Admission Configuration](../../../reference-guides/rancher-security/psa-restricted-exemptions.md). + +## Exempting Namespaces + +If you assign the `rancher-restricted` template to a cluster, by default the restrictions are applied across the entire cluster at the namespace level. To exempt certain namespaces from this highly restricted policy, do the following: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Click **Advanced** to open the dropdown menu. +1. Select **Pod Security Admissions**. +1. Find the template you want to modify, and click the **⋮**. +1. Select **Edit Config**. +1. Click the **Namespaces** checkbox under **Exemptions** to edit the **Namespaces** field. +1. When you're done exempting namespaces, click **Save**. + +:::note +You need to update the target cluster to make the new template take effect in that cluster. An update can be triggered by editing and saving the cluster without changing values. +::: diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md new file mode 100644 index 00000000000..2f0d0f2c9b7 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md @@ -0,0 +1,78 @@ +--- +title: Backing up Rancher Installed with Docker +--- + + + + + +After completing your Docker installation of Rancher, we recommend creating backups of it on a regular basis. Having a recent backup will let you recover quickly from an unexpected disaster. + +## Before You Start + +During the creation of your backup, you'll enter a series of commands, replacing placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). Here's an example of a command with a placeholder: + +``` +docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher +``` + +In this command, `` is a placeholder for the date that the data container and backup were created. `9-27-18` for example. + +Cross reference the image and reference table below to learn how to obtain this placeholder data. Write down or copy this information before starting the [procedure below](#creating-a-backup). + +Terminal docker ps Command, Displaying Where to Find <RANCHER_CONTAINER_TAG> and <RANCHER_CONTAINER_NAME> + +![Placeholder Reference](/img/placeholder-ref.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | The rancher/rancher image you pulled for initial install. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.0.5` | The version of Rancher that you're creating a backup for. | +| `` | `9-27-18` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher Server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped with `docker ps -a`. Use these commands for help anytime while creating backups. + +## Creating a Backup + +This procedure creates a backup that you can restore if Rancher encounters a disaster scenario. + + +1. Using a remote Terminal connection, log into the node running your Rancher Server. + +1. Stop the container currently running Rancher Server. Replace `` with the name of your Rancher container. + + ``` + docker stop + ``` +1. Use the command below, replacing each placeholder, to create a data container from the Rancher container that you just stopped. + + ``` + docker create --volumes-from --name rancher-data- rancher/rancher: + ``` + +1. From the data container that you just created (rancher-data-<DATE>), create a backup tarball (rancher-data-backup-<RANCHER_VERSION>-<DATE>.tar.gz). Use the following command, replacing each placeholder: + + ``` + docker run --name busybox-backup- --volumes-from rancher-data- -v $PWD:/backup:z busybox tar pzcvf /backup/rancher-data-backup--.tar.gz /var/lib/rancher + ``` + + **Step Result:** A stream of commands runs on the screen. + +1. Enter the `ls` command to confirm that the backup tarball was created. It will have a name similar to `rancher-data-backup--.tar.gz`. + +1. Move your backup tarball to a safe location external to your Rancher Server. Then delete the `rancher-data-` and `busybox-backup-` containers from your Rancher Server. + + ``` + docker rm rancher-data- + docker rm busybox-backup- + ``` + +1. Restart Rancher Server. Replace `` with the name of your Rancher container: + + ``` + docker start + ``` + +**Result:** A backup tarball of your Rancher Server data is created. See [Restoring Backups: Docker Installs](restore-docker-installed-rancher.md) if you need to restore backup data. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md new file mode 100644 index 00000000000..e7a4999a53f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md @@ -0,0 +1,303 @@ +--- +title: Backing up a Cluster +--- + + + + + +In the Rancher UI, etcd backup and recovery for [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) can be easily performed. + +Rancher recommends configuring recurrent `etcd` snapshots for all production clusters. Additionally, one-time snapshots can be taken as well. + +Snapshots of the etcd database are taken and saved either [locally onto the etcd nodes](#local-backup-target) or to a [S3 compatible target](#s3-backup-target). The advantages of configuring S3 is that if all etcd nodes are lost, your snapshot is saved remotely and can be used to restore the cluster. + +## How Snapshots Work + +### Snapshot Components + + + + +When Rancher creates a snapshot, it includes three components: + +- The cluster data in etcd +- The Kubernetes version +- The cluster configuration in the form of the `cluster.yml` + +Because the Kubernetes version is now included in the snapshot, it is possible to restore a cluster to a prior Kubernetes version. + + + + +Rancher delegates snapshot creation to the downstream Kubernetes engine. When the Kubernetes engine creates a snapshot, it includes three components: + +- The cluster data in etcd +- The Kubernetes version +- The cluster configuration + +Because the Kubernetes version is included in the snapshot, it is possible to restore a cluster to a prior Kubernetes version while also restoring an etcd snapshot. + + + + +The multiple components of the snapshot allow you to select from the following options if you need to restore a cluster from a snapshot: + +- **Restore just the etcd contents:** This restore is similar to restoring to snapshots in Rancher before v2.4.0. +- **Restore etcd and Kubernetes version:** This option should be used if a Kubernetes upgrade is the reason that your cluster is failing, and you haven't made any cluster configuration changes. +- **Restore etcd, Kubernetes versions and cluster configuration:** This option should be used if you changed both the Kubernetes version and cluster configuration when upgrading. + +It is always recommended to take a new snapshot before performing any configuration changes or upgrades. + + +### Generating the Snapshot from etcd Nodes + + + + +For each etcd node in the cluster, the etcd cluster health is checked. If the node reports that the etcd cluster is healthy, a snapshot is created from it and optionally uploaded to S3. + +The snapshot is stored in `/opt/rke/etcd-snapshots`. If the directory is configured on the nodes as a shared mount, it will be overwritten. On S3, the snapshot will always be from the last node that uploads it, as all etcd nodes upload it and the last will remain. + +In the case when multiple etcd nodes exist, any created snapshot is created after the cluster has been health checked, so it can be considered a valid snapshot of the data in the etcd cluster. + + + + +Snapshots are enabled by default. + +The snapshot directory defaults to `/var/lib/rancher//server/db/snapshots`, where `` is either `rke2` or `k3s`. + +In RKE2, snapshots are stored on each etcd node. If you have multiple etcd or etcd + control-plane nodes, you will have multiple copies of local etcd snapshots. + + + + +### Snapshot Naming Conventions + + + + +The name of the snapshot is auto-generated. The `--name` option can be used to override the name of the snapshot when creating one-time snapshots with the RKE CLI. + +When Rancher creates a snapshot of an RKE cluster, the snapshot name is based on the type (whether the snapshot is manual or recurring) and the target (whether the snapshot is saved locally or uploaded to S3). The naming convention is as follows: + +- `m` stands for manual +- `r` stands for recurring +- `l` stands for local +- `s` stands for S3 + +Some example snapshot names are: + +- c-9dmxz-rl-8b2cx +- c-9dmxz-ml-kr56m +- c-9dmxz-ms-t6bjb +- c-9dmxz-rs-8gxc8 + + + + +The name of the snapshot is auto-generated. The `--name` option can be used to override the base name of the snapshot when creating one-time snapshots with the RKE2 or K3s CLI. + +When Rancher creates a snapshot of an RKE2 or K3s cluster, the snapshot name is based on the type (whether the snapshot is manual or recurring) and the target (whether the snapshot is saved locally or uploaded to S3). The naming convention is as follows: + +`--` + +``: is the base name set by `--name` and can be one of the the following + +- `etcd-snapshot` is prepended on recurring snapshots +- `on-demand` is prepended on manual, on-demand snapshots + +``: Node is the name of the node that the snapshot was taken on. + +`` is a unix-time stamp of the snapshot creation date. + +Some example snapshot names are: + +- `on-demand-my-super-rancher-k8s-node1-1652288934` +- `on-demand-my-super-rancher-k8s-node2-1652288936` +- `etcd-snapshot-my-super-rancher-k8s-node1-1652289945` +- `etcd-snapshot-my-super-rancher-k8s-node2-1652289948` + + + + +### How Restoring from a Snapshot Works + + + + +On restore, the following process is used: + +1. The snapshot is retrieved from S3, if S3 is configured. +2. The snapshot is unzipped (if zipped). +3. One of the etcd nodes in the cluster serves that snapshot file to the other nodes. +4. The other etcd nodes download the snapshot and validate the checksum so that they all use the same snapshot for the restore. +5. The cluster is restored and post-restore actions will be done in the cluster. + + + + +On restore, Rancher delivers a few sets of plans to perform a restoration. A set of phases are used, namely: + +- Started +- Shutdown +- Restore +- RestartCluster +- Finished + +If the etcd snapshot restore fails, the phase will be set to `Failed`. + +1. The etcd snapshot restore request is received, and depending on `restoreRKEConfig`, the cluster configuration/kubernetes version are reconciled. +1. The phase is set to `Started`. +1. The phase is set to `Shutdown`, and the entire cluster is shut down using plans that run the distribution `killall.sh` script. A new init node is elected. If the snapshot being restored is a local snapshot, the node that the snapshot resides on will be selected as the init node. If the snapshot is being restored from S3, the existing init node will be used. +1. The phase is set to `Restore`, and the init node has the snapshot restored onto it. +1. The phase is set to `RestartCluster`, and the cluster is restarted/rejoined to the new init node that has the freshly restored snapshot information. +1. The phase is set to `Finished`, and the cluster is deemed successfully restored. The `cattle-cluster-agent` will reconnect, and the cluster will finish reconciliation. + + + + +## Configuring Recurring Snapshots + + + + +Select how often you want recurring snapshots to be taken as well as how many snapshots to keep. The amount of time is measured in hours. With timestamped snapshots, the user has the ability to do a point-in-time recovery. + +By default, [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) are configured to take recurring snapshots (saved to local disk). To protect against local disk failure, using the [S3 Target](#s3-backup-target) or replicating the path on disk is advised. + +During cluster provisioning or editing the cluster, the configuration for snapshots can be found in the advanced section for **Cluster Options**. Click on **Show advanced options**. + +In the **Advanced Cluster Options** section, there are several options available to configure: + +| Option | Description | Default Value| +| --- | ---| --- | +| etcd Snapshot Backup Target | Select where you want the snapshots to be saved. Options are either local or in S3 | local| +|Recurring etcd Snapshot Enabled| Enable/Disable recurring snapshots | Yes| +| Recurring etcd Snapshot Creation Period | Time in hours between recurring snapshots| 12 hours | +| Recurring etcd Snapshot Retention Count | Number of snapshots to retain| 6 | + + + + +Set the schedule for how you want recurring snapshots to be taken as well as how many snapshots to keep. The schedule is conventional cron format. The retention policy dictates the number of snapshots matching a name to keep per node. + +By default, [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) are configured to take recurring snapshots (saved to local disk) every 5 hours starting at 12 AM. To protect against local disk failure, using the [S3 Target](#s3-backup-target) or replicating the path on disk is advised. + +During cluster provisioning or editing the cluster, the configuration for snapshots can be found under **Cluster Configuration**. Click on **etcd**. + +| Option | Description | Default Value| +| --- | ---| --- | +| Recurring etcd Snapshot Enabled | Enable/Disable recurring snapshots | Yes | +| Recurring etcd Snapshot Creation Period | Cron schedule for recurring snapshot | `0 */5 * * *` | +| Recurring etcd Snapshot Retention Count | Number of snapshots to retain | 5 | + + + + +## One-Time Snapshots + + + + +In addition to recurring snapshots, you may want to take a "one-time" snapshot. For example, before upgrading the Kubernetes version of a cluster it's best to backup the state of the cluster to protect against upgrade failure. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, navigate to the cluster where you want to take a one-time snapshot. +1. Click **⋮ > Take Snapshot**. + + + + +In addition to recurring snapshots, you may want to take a "one-time" snapshot. For example, before upgrading the Kubernetes version of a cluster it's best to backup the state of the cluster to protect against upgrade failure. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, navigate to the cluster where you want to take a one-time snapshot. +1. Navigate to the `Snapshots` tab and click `Snapshot Now` + +### How Taking One-Time Snapshots Works + +On one-time snapshot creation, the Rancher delivers a few sets of plans to perform snapshot creation. A set of phases are used, namely: + +- Started +- RestartCluster +- Finished + +If the etcd snapshot creation fails, the phase will be set to `Failed`. + +1. The etcd snapshot creation request is received. +1. The phase is set to `Started`. All etcd nodes in the cluster receive a plan to create an etcd snapshot, per the cluster configuration. +1. The phase is set to `RestartCluster`, and the plans on every etcd node are reset to the original plan for the etcd nodes. +1. The phase is set to `Finished`. + + + + +**Result:** Based on your [snapshot backup target](#snapshot-backup-targets), a one-time snapshot will be taken and saved in the selected backup target. + +## Snapshot Backup Targets + +Rancher supports two different backup targets: + +- [Local Target](#local-backup-target) +- [S3 Target](#s3-backup-target) + +### Local Backup Target + + + + +By default, the `local` backup target is selected. The benefits of this option is that there is no external configuration. Snapshots are automatically saved locally to the etcd nodes in the [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) in `/opt/rke/etcd-snapshots`. All recurring snapshots are taken at configured intervals. The downside of using the `local` backup target is that if there is a total disaster and _all_ etcd nodes are lost, there is no ability to restore the cluster. + + + + +By default, the `local` backup target is selected. The benefits of this option is that there is no external configuration. Snapshots are automatically saved locally to the etcd nodes in the [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) in `/var/lib/rancher//server/db/snapshots` where `` is either `k3s` or `rke2`. All recurring snapshots are taken per the cron schedule. The downside of using the `local` backup target is that if there is a total disaster and _all_ etcd nodes are lost, there is no ability to restore the cluster. + + + + +### S3 Backup Target + +The `S3` backup target allows users to configure a S3 compatible backend to store the snapshots. The primary benefit of this option is that if the cluster loses all the etcd nodes, the cluster can still be restored as the snapshots are stored externally. Rancher recommends external targets like `S3` backup, however its configuration requirements do require additional effort that should be considered. Additionally, it is recommended to ensure that every cluster has a unique bucket and/or folder, as Rancher will populate snapshot information for any available snapshot that is listed in the S3 bucket/folder that is configured for the cluster. + +| Option | Description | Required| +|---|---|---| +|S3 Bucket Name| S3 bucket name where backups will be stored| *| +|S3 Region|S3 region for the backup bucket| | +|S3 Region Endpoint|S3 regions endpoint for the backup bucket|* | +|S3 Access Key|S3 access key with permission to access the backup bucket|*| +|S3 Secret Key|S3 secret key with permission to access the backup bucket|*| +| Custom CA Certificate | A custom certificate used to access private S3 backends || + +### Using a custom CA certificate for S3 + +The backup snapshot can be stored on a custom `S3` backup like [minio](https://min.io/). If the S3 back end uses a self-signed or custom certificate, provide a custom certificate using the `Custom CA Certificate` option to connect to the S3 backend. + +### IAM Support for Storing Snapshots in S3 + +The `S3` backup target supports using IAM authentication to AWS API in addition to using API credentials. An IAM role gives temporary permissions that an application can use when making API calls to S3 storage. To use IAM authentication, the following requirements must be met: + +- The cluster etcd nodes must have an instance role that has read/write access to the designated backup bucket. +- The cluster etcd nodes must have network access to the specified S3 endpoint. +- The Rancher Server worker node(s) must have an instance role that has read/write to the designated backup bucket. +- The Rancher Server worker node(s) must have network access to the specified S3 endpoint. + + To give an application access to S3, refer to the AWS documentation on [Using an IAM Role to Grant Permissions to Applications Running on Amazon EC2 Instances.](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html) + +## Viewing Available Snapshots + +The list of all available snapshots for the cluster is available in the Rancher UI. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to view the snapshots and click its name. +1. Click the **Snapshots** tab to view the list of saved snapshots. These snapshots include a timestamp of when they were created. + +## Safe Timestamps (RKE) + +Snapshot files are timestamped to simplify processing the files using external tools and scripts, but in some S3 compatible backends, these timestamps were unusable. + +The option `safe_timestamp` is added to support compatible file names. When this flag is set to `true`, all special characters in the snapshot filename timestamp are replaced. + +This option is not available directly in the UI, and is only available through the `Edit as Yaml` interface. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md new file mode 100644 index 00000000000..27b02652450 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md @@ -0,0 +1,96 @@ +--- +title: Backing up Rancher +--- + + + + + +In this section, you'll learn how to back up Rancher running on any Kubernetes cluster. To backup Rancher installed with Docker, refer the instructions for [single node backups](back-up-docker-installed-rancher.md) + +The backup-restore operator needs to be installed in the local cluster, and only backs up the Rancher app. The backup and restore operations are performed only in the local Kubernetes cluster. + +Note that the rancher-backup operator version 2.x.x is for Rancher v2.6.x. + +:::caution + +When restoring a backup into a new Rancher setup, the version of the new setup should be the same as the one where the backup is made. The Kubernetes version should also be considered when restoring a backup, since the supported apiVersion in the cluster and in the backup file could be different. + +::: + +### Prerequisites + +The Rancher version must be v2.5.0 and up. + +Refer [here](migrate-rancher-to-new-cluster.md#2-restore-from-backup-using-a-restore-custom-resource) for help on restoring an existing backup file into a v1.22 cluster in Rancher v2.6.3. + +### 1. Install the Rancher Backup operator + +The backup storage location is an operator-level setting, so it needs to be configured when the Rancher Backups application is installed or upgraded. + +Backups are created as .tar.gz files. These files can be pushed to S3 or Minio, or they can be stored in a persistent volume. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. The `local` cluster runs the Rancher server. +1. Click **Apps > Charts**. +1. Click **Rancher Backups**. +1. Click **Install**. +1. Configure the default storage location. For help, refer to the [storage configuration section.](../../../reference-guides/backup-restore-configuration/storage-configuration.md) +1. Click **Install**. + +:::note + +There is a known issue in Fleet that occurs after performing a restoration using the backup-restore-operator: Secrets used for clientSecretName and helmSecretName are not included in Fleet gitrepos. Refer [here](../deploy-apps-across-clusters/fleet.md#troubleshooting) for a workaround. + +::: + +### 2. Perform a Backup + +To perform a backup, a custom resource of type Backup must be created. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. +1. In the left navigation bar, click **Rancher Backups > Backups**. +1. Click **Create**. +1. Create the Backup with the form, or with the YAML editor. +1. For configuring the Backup details using the form, click **Create** and refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/backup-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md#backup) +1. For using the YAML editor, we can click **Create > Create from YAML**. Enter the Backup YAML. This example Backup custom resource would create encrypted recurring backups in S3. The app uses the `credentialSecretNamespace` value to determine where to look for the S3 backup secret: + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Backup + metadata: + name: s3-recurring-backup + spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 + ``` + + :::note + + When creating the Backup resource using YAML editor, the `resourceSetName` must be set to `rancher-resource-set` + + ::: + + For help configuring the Backup, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/backup-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md#backup) + + :::caution + + The `rancher-backup` operator doesn't save the EncryptionConfiguration file. The contents of the EncryptionConfiguration file must be saved when an encrypted backup is created, and the same file must be used when restoring from this backup. + + ::: + +1. Click **Create**. + +**Result:** The backup file is created in the storage location configured in the Backup custom resource. The name of this file is used when performing a restore. + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md new file mode 100644 index 00000000000..3ef56172eb1 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide.md @@ -0,0 +1,133 @@ +--- +title: Backup Restore Usage Guide +--- + + + + + +The Rancher Backups chart is our solution for disaster recovery and migration. This chart allows you to take backups of your Kubernetes resources and save them to a variety of persistent storage locations. + +This chart is a very simple tool which has its hands in many different areas of the Rancher ecosystem. As a result, edge cases have arisen which lead to undocumented functionality. The purpose of this document is to highlight the proper and defined usage for Rancher Backups, as well as discussing some of these edge cases we’ve run into. + +## Functionality Overview + +### Backup + +The operator collects all the resources captured by the resourceSet in the chart as in-memory unstructured objects. After the resources have been collected, a compressed tar file of the resources are saved as a collection of manifests in JSON and then uploaded to a user-defined object store. This backup can be put on a repeating schedule and can also be encrypted. This encryption option is important since some of the resources are sensitive and the values are stored in plaintext without encryption. + +See the [Backup Configuration documentation](../../../reference-guides/backup-restore-configuration/backup-configuration.md) for more information about the options, including encryption, to configure a backup. + +:::note + +As noted in the [Backing up Rancher documentation](./back-up-rancher.md), you must manually save the encryption configuration file contents since the operator will **not** back it up. + +::: + +### Restore + +There are two main restore scenarios, restoring to a cluster with Rancher running and restoring to a fresh cluster. You can only restore to a cluster with Rancher running if it's the same cluster the backup was taken from and the [`prune` option](../../../reference-guides/backup-restore-configuration/restore-configuration.md#prune-during-restore) is enabled during the restore. A restore has similar inputs as a backup. It requires a backup filename, the encryptionConfigSecret name, and the storage location. + +Resources are restored in this order: + +1. Custom Resource Definitions (CRDs) +2. Cluster-scoped resources +3. Namespaced resources + +See the [Restore Configuration documentation](../../../reference-guides/backup-restore-configuration/restore-configuration.md) for more information about the options to configure a restore. + +### Resource Sets + +The resourceSet determines which resources the backup-restore-operator collects in a backup. It's a set of ResourceSelectors, which define the selection requirements using key/value matching, regular expression matching, or the Kubernetes client labelSelector. + +These are the different fields available for a resourceSelector: + +- apiVersion +- excludeKinds +- excludeResourceNameRegexp +- kinds +- kindsRegexp +- labelSelectors +- namespaceRegexp +- namespaces +- resourceNameRegexp +- resourceNames + +The Rancher Backups chart contains a [default resourceSet](https://github.com/rancher/backup-restore-operator/tree/release/v3.0/charts/rancher-backup/files/default-resourceset-contents), which is a combination of YAML files that are appended to one large resourceSet when the chart is installed. The file order does not matter. The resourceSets may differ between versions. + +:::caution + +If you wish to make edits to the resourceSet please edit it **before** installing the chart. + +::: + +## Proper Usage + +This section outlines the guidelines for the proper usage of the Rancher Backups chart according to its use case. + +### All Cases + +- Rancher Backups must be installed on the local cluster. + - Note: Rancher Backups does not handle any cluster other than the one it is installed on to. It may restore cluster resources that are on the local cluster but will not communicate with the downstream clusters or back them up. +- The Rancher version being restored to must match the Rancher version from backup. +- The Kubernetes version should be be considered since you may be restoring outdated resources (resources deprecated from the version of Kubernetes you are restoring to). + +### Backups + +- Some user generated resources will not be backed up unless they can are captured by the default resourceSet or the resourceSet was altered to capture them. + - We provide a label `resources.cattle.io/backup:true` that when added to a secret in any namespace, will result in it being backed up. +- Backups are non-mutating +- Backups are only of the local cluster + +### Restores + +- A restore refers to restoring a backup to the same cluster it was taken from. This can be with Rancher installed (**prune must be enabled**) or with it not installed (no special instructions). +- One thing to note when restoring is that you may find yourself needing to “wipe†the cluster of any Rancher resources. This can be done by deploying the [Rancher cleanup script](https://github.com/rancher/rancher-cleanup) script as a job to the cluster. This allows you to install Rancher Backups again and restore to a completely fresh cluster. + - Make sure to use kubectl to deploy the scripts. + +### Migrations + +Migration have some more nuance since we are restoring to a different cluster. These are a few things to remember that are commonly missed or forgotten. + +- The Rancher domain must be the same when migrating. This means your previous cluster’s domain name must now point to the new cluster. +- Rancher should **not** be running already on the cluster you are migrating to. This can cause many issues with Rancher backups and certain Rancher services as well. +- Install the **same** version of Rancher from the backup **after** the backup has been restored. +- If you choose to provision the new cluster on a different Kubernetes version know that this can cause a wide variety of unsupported behaviors because the Kubernetes API available may be different from the one you took a backup from. This can lead to deprecated resources being restored which will cause issues. +- You should **not** perform any upgrades during a migration. + +## Edge Cases and Improper Usage + +Below are some examples of some **incorrect** uses or expectations of Rancher Backups. + +### Upgrades + +- Using Rancher backups for upgrading Rancher versions is not a valid use case. The recommended procedure is to take a backup of the current version, then upgrade your Rancher instance using [these instructions](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md), and then taking **another** backup after the upgrade is complete. This way if the upgrade fails you have a backup to restore to, and the second backup will be valid to restore to the upgraded Rancher version. +- Using Rancher backups for upgrading Kubernetes versions is not a valid use case either. Because the Kubernetes API and available resources are tied to the version, upgrading using backup restore can lead to issues with misaligned sets of resources which may be deprecated, unsupported, or updated. How to upgrade your cluster version will depend on how it was provisioned however the same format as above is recommended (backup, upgrade, backup). + +### ResourceSet + +- Because of evolving resources and services from various teams, developers should take note if new resources need to be added to or removed from the default resourceSet. +- Rancher backups only backs up what is captured by the default resourceSets (unless edited). We have added a specific label for user created secrets that will back up a secret of any name and namespace that has said label (see [Proper Usage on Backups](#backups)). + +### Downstream Clusters + +- Rancher Backups **only** backs up Kubernetes resources on the local cluster. This means downstream clusters are **not** touched or backed up other than their presence in resources in the local cluster. The updating and communication of downstream clusters falls upon the rancher-agent and rancher-webhook. + +### Restoring Deleted Resources + +- Some resources have external results produced, such as provisioning a downstream cluster. Deleting a downstream cluster and restoring the cluster resource on the local cluster does **not** cause Rancher to reprovision said cluster. Depending on the resource, restoring may not fully bring back the resource to an available state. +- The corner case of "restoring a deleted cluster" is **not** a supported feature. When it comes to downstream clusters, whether provisioned or imported, deleting them causes a series of cleanup tasks which doesn't allow the user to restore the deleted clusters. Provisioned clusters will have their nodes and Rancher-related provisioning resources destroyed, and imported clusters will likely have their Rancher agents and other resources/services related to registration with a local cluster destroyed. + +:::caution + +Trying to delete and restore a downstream cluster can lead to a variety of issues with Rancher, Rancher Backups, rancher-webhook, Fleet, and more. It is not recommended to do this. + +::: + +### Fleet, Harvester, and Other Services + +Other services, which are backed up by Rancher Backups, often change and evolve. As this happens, their resources and backup needs may change as well. Some resources may not need to be backed up and some may not be backed up at all. It is important for teams to consider this in their development process and assess whether their related resourceSets are correctly capturing the proper set of resources for their services to be restored correctly. + +## Conclusion + +Rancher Backups is a very useful tool, however it is somewhat limited in its scope and intended purposes. In order to avoid possible difficulties, it is important to follow the specific procedures described to ensure the proper operation of the chart. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md new file mode 100644 index 00000000000..777f7e6f34f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md @@ -0,0 +1,188 @@ +--- +title: Migrating Rancher to a New Cluster +--- + + + + + +If you are migrating Rancher to a new Kubernetes cluster, you don't need to install Rancher on the new cluster first. If Rancher is restored to a new cluster with Rancher already installed, it can cause problems. + +### Prerequisites + +These instructions assume you have [created a backup](back-up-rancher.md) and you have already installed a new Kubernetes cluster where Rancher will be deployed. + +:::caution + +It is required to use the same hostname that was set as the server URL in the first cluster. If not done, downstream clusters will show as unavailable in the cluster management page of the UI, and you won't be able to click inside the cluster or on the cluster's Explore button. + +::: + +Rancher version must be v2.5.0 and up + +Rancher can be installed on any Kubernetes cluster, including hosted Kubernetes clusters such as Amazon EKS clusters. For help installing Kubernetes, refer to the documentation of the Kubernetes distribution. One of Rancher's Kubernetes distributions may also be used: + +- [RKE Kubernetes installation docs](https://rancher.com/docs/rke/latest/en/installation/) +- [K3s Kubernetes installation docs](https://rancher.com/docs/k3s/latest/en/installation/) + +### 1. Install the rancher-backup Helm chart +Install the [rancher-backup chart](https://github.com/rancher/backup-restore-operator/tags), using a version in the 2.x.x major version range: + + 1. Add the helm repository: + + ```bash + helm repo add rancher-charts https://charts.rancher.io + helm repo update + ``` + + 1. Select and set `CHART_VERSION` variable with a 2.x.x rancher-backup release version: + ```bash + helm search repo --versions rancher-charts/rancher-backup + CHART_VERSION=<2.x.x> + ``` + + 1. Install the charts: + ```bash + helm install rancher-backup-crd rancher-charts/rancher-backup-crd -n cattle-resources-system --create-namespace --version $CHART_VERSION + helm install rancher-backup rancher-charts/rancher-backup -n cattle-resources-system --version $CHART_VERSION + ``` + + :::note + + The above assumes an environment with outbound connectivity to Docker Hub + + For an **air-gapped environment**, use the Helm value below to pull the `backup-restore-operator` image from your private registry when installing the rancher-backup Helm chart. + + ```bash + --set image.repository $REGISTRY/rancher/backup-restore-operator + ``` + + ::: + +### 2. Restore from backup using a Restore custom resource + +:::note Important: + +Kubernetes v1.22, available as an experimental feature of v2.6.3, does not support restoring from backup files containing CRDs with the apiVersion `apiextensions.k8s.io/v1beta1`. In v1.22, the default `resourceSet` in the rancher-backup app is updated to collect only CRDs that use `apiextensions.k8s.io/v1`. There are currently two ways to work around this issue: + +1. Update the default `resourceSet` to collect the CRDs with the apiVersion v1. +1. Update the default `resourceSet` and the client to use the new APIs internally, with `apiextensions.k8s.io/v1` as the replacement. + + :::note + + When making or restoring backups for v1.22, the Rancher version and the local cluster's Kubernetes version should be the same. The Kubernetes version should be considered when restoring a backup since the supported apiVersion in the cluster and in the backup file could be different. + + ::: + +::: + +1. When using S3 object storage as the backup source for a restore that requires credentials, create a `Secret` object in this cluster to add the S3 credentials. The secret data must have two keys - `accessKey`, and `secretKey`, that contain the S3 credentials. + + The secret can be created in any namespace, this example uses the default namespace. + + ```bash + kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= + ``` + + :::note + + Add your access key and secret key as values for `accessKey` and `secretKey` in the command above. + + ::: + +1. Create a `Restore` object: + + During a migration, `prune` must be set to `false`. See the example below: + + ```yaml + # restore-migration.yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + // highlight-next-line + prune: false + // highlight-next-line + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + :::note Important + + The field `encryptionConfigSecretName` should be used only if your backup was created with encryption enabled. + + If this applies, provide the name of the `Secret` object containing the encryption config file. If you only have the encryption config file, but don't have the secret created in this cluster, use the following steps to create the secret: + + 1. Create an [encryption configuration file](../../../reference-guides/backup-restore-configuration/backup-configuration.md#encryption) + 1. The command below uses a file named `encryption-provider-config.yaml`, with the `--from-file` flag. Run the below once the `EncryptionConfiguration` is saved in a file called `encryption-provider-config.yaml`: + + ```bash + kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system + ``` + + ::: + +1. Apply the manifest, and monitor the Restore status: + 1. Apply the `Restore` object resource: + + ```bash + kubectl apply -f restore-migration.yaml + ``` + + 1. Watch the Restore status: + ```bash + kubectl get restore + ``` + + 1. Watch the restoration logs: + ```bash + kubectl logs -n cattle-resources-system --tail 100 -f -l app.kubernetes.io/instance=rancher-backup + ``` + + 1. Once the Restore resource has the status `Completed`, you can continue the cert-manager and Rancher installation. + +### 3. Install cert-manager + +Follow the steps to [install cert-manager](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md#4-install-cert-manager) in the documentation about installing cert-manager on Kubernetes. + +### 4. Bring up Rancher with Helm + +Use the same version of Helm to install Rancher, that was used on the first cluster. + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +```bash +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname= \ + --version x.y.z +``` + +:::note + +If the original Rancher environment is running, you can collect the current values with a kubeconfig for the original environment: + +```bash +helm get values rancher -n cattle-system -o yaml > rancher-values.yaml +``` + +These values can be reused using the `rancher-values.yaml` file. Be sure to switch the kubeconfig to the new Rancher environment. + +```bash +helm install rancher rancher-latest/rancher -n cattle-system -f rancher-values.yaml --version x.y.z +``` + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md new file mode 100644 index 00000000000..a7479ec95cb --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md @@ -0,0 +1,76 @@ +--- +title: Restoring Backups—Docker Installs +--- + + + + + +If you encounter a disaster scenario, you can restore your Rancher Server to your most recent backup. + +## Before You Start + +During restore of your backup, you'll enter a series of commands, filling placeholders with data from your environment. These placeholders are denoted with angled brackets and all capital letters (``). Here's an example of a command with a placeholder: + +``` +docker run --volumes-from -v $PWD:/backup \ +busybox sh -c "rm /var/lib/rancher/* -rf && \ +tar pzxvf /backup/rancher-data-backup--" +``` + +In this command, `` and `-` are environment variables for your Rancher deployment. + +Cross reference the image and reference table below to learn how to obtain this placeholder data. Write down or copy this information before starting the procedure below. + +Terminal docker ps Command, Displaying Where to Find <RANCHER_CONTAINER_TAG> and <RANCHER_CONTAINER_NAME> + +![Placeholder Reference](/img/placeholder-ref.png) + +| Placeholder | Example | Description | +| -------------------------- | -------------------------- | --------------------------------------------------------- | +| `` | `v2.0.5` | The rancher/rancher image you pulled for initial install. | +| `` | `festive_mestorf` | The name of your Rancher container. | +| `` | `v2.0.5` | The version number for your Rancher backup. | +| `` | `9-27-18` | The date that the data container or backup was created. | +
+ +You can obtain `` and `` by logging into your Rancher Server by remote connection and entering the command to view the containers that are running: `docker ps`. You can also view containers that are stopped using a different command: `docker ps -a`. Use these commands for help anytime during while creating backups. + +## Restoring Backups + +Using a [backup](back-up-docker-installed-rancher.md) that you created earlier, restore Rancher to its last known healthy state. + +1. Using a remote Terminal connection, log into the node running your Rancher Server. + +1. Stop the container currently running Rancher Server. Replace `` with the name of your Rancher container: + + ``` + docker stop + ``` +1. Move the backup tarball that you created during completion of [Creating Backups—Docker Installs](back-up-docker-installed-rancher.md) onto your Rancher Server. Change to the directory that you moved it to. Enter `dir` to confirm that it's there. + + If you followed the naming convention we suggested in [Creating Backups—Docker Installs](back-up-docker-installed-rancher.md), it will have a name similar to `rancher-data-backup--.tar.gz`. + +1. Enter the following command to delete your current state data and replace it with your backup data, replacing the placeholders. Don't forget to close the quotes. + + :::danger + + This command deletes all current state data from your Rancher Server container. Any changes saved after your backup tarball was created will be lost. + + ::: + + ``` + docker run --volumes-from -v $PWD:/backup \ + busybox sh -c "rm /var/lib/rancher/* -rf && \ + tar pzxvf /backup/rancher-data-backup--.tar.gz" + ``` + + **Step Result:** A series of commands should run. + +1. Restart your Rancher Server container, replacing the placeholder. It will restart using your backup data. + + ``` + docker start + ``` + +1. Wait a few moments and then open Rancher in a web browser. Confirm that the restore succeeded and that your data is restored. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md new file mode 100644 index 00000000000..313812b6129 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md @@ -0,0 +1,133 @@ +--- +title: Restoring a Cluster from Backup +--- + + + + + +Etcd backup and recovery for [Rancher launched Kubernetes clusters](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) can be easily performed. Snapshots of the etcd database are taken and saved either locally onto the etcd nodes or to a S3 compatible target. The advantages of configuring S3 is that if all etcd nodes are lost, your snapshot is saved remotely and can be used to restore the cluster. + +Rancher recommends enabling the [ability to set up recurring snapshots of etcd](back-up-rancher-launched-kubernetes-clusters.md#configuring-recurring-snapshots), but [one-time snapshots](back-up-rancher-launched-kubernetes-clusters.md#one-time-snapshots) can easily be taken as well. Rancher allows restore from [saved snapshots](#restoring-a-cluster-from-a-snapshot) or if you don't have any snapshots, you can still [restore etcd](#recovering-etcd-without-a-snapshot-rke). + +Clusters can also be restored to a prior Kubernetes version and cluster configuration. + +## Viewing Available Snapshots + +The list of all available snapshots for the cluster is available. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to view the snapshots and click the name of the cluster. +1. Click the **Snapshots** tab. The listed snapshots include a timestamp of when they were created. + +## Restoring a Cluster from a Snapshot + +If your Kubernetes cluster is broken, you can restore the cluster from a snapshot. + +Snapshots are composed of the cluster data in etcd, the Kubernetes version, and the cluster configuration in the `cluster.yml.` These components allow you to select from the following options when restoring a cluster from a snapshot: + +- **Restore just the etcd contents:** This restore is similar to restoring to snapshots in Rancher before v2.4.0. +- **Restore etcd and Kubernetes version:** This option should be used if a Kubernetes upgrade is the reason that your cluster is failing, and you haven't made any cluster configuration changes. +- **Restore etcd, Kubernetes versions and cluster configuration:** This option should be used if you changed both the Kubernetes version and cluster configuration when upgrading. + +When rolling back to a prior Kubernetes version, the [upgrade strategy options](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md#configuring-the-upgrade-strategy) are ignored. Worker nodes are not cordoned or drained before being reverted to the older Kubernetes version, so that an unhealthy cluster can be more quickly restored to a healthy state. + +:::note Prerequisite: + +To restore snapshots from S3, the cluster needs to be configured to [take recurring snapshots on S3.](back-up-rancher-launched-kubernetes-clusters.md#configuring-recurring-snapshots) + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to view the snapshots and click the name of the cluster. +1. Click the **Snapshots** tab to view the list of saved snapshots. +1. Go to the snapshot you want to restore and click **⋮ > Restore**. +1. Select a **Restore Type**. +1. Click **Restore**. + +**Result:** The cluster will go into `updating` state and the process of restoring the `etcd` nodes from the snapshot will start. The cluster is restored when it returns to an `active` state. + +## Restoring a Cluster From a Snapshot When the controlplane/etcd Are Completely Unavailable + +In a disaster recovery scenario, the control plane and etcd nodes managed by Rancher in a downstream cluster may no longer be available or functioning. The cluster can be rebuilt by adding control plane and etcd nodes again, followed by restoring from an available snapshot. + + + + +Follow the procedure described in the [SUSE Knowledgebase](https://www.suse.com/support/kb/doc/?id=000020695). + + + + +If you have a complete cluster failure, you must remove all etcd nodes/machines from your cluster before you can add a "new" etcd node for restore. + +:::note + +Due to a [known issue](https://github.com/rancher/rancher/issues/41080), this procedure requires Rancher v2.7.5 or newer. + +::: + +:::note + +If you are using [local snapshots](./back-up-rancher-launched-kubernetes-clusters.md#local-backup-target), it is **VERY** important that you ensure you back up the corresponding snapshot you want to restore from the `/var/lib/rancher//server/db/snapshots/` folder on the etcd node you are going to be removing. You can copy the snapshot onto your new node in the `/var/lib/rancher//server/db/snapshots/` folder. Furthermore, if using local snapshots and restoring to a new node, restoration cannot be done via the UI as of now. + +::: + +1. Remove all etcd nodes from your cluster. + + 1. In the upper left corner, click **☰ > Cluster Management**. + 1. In the **Clusters** page, go to the cluster where you want to remove nodes. + 1. In the **Machines** tab, click **⋮ > Delete** on each node you want to delete. Initially, you will see the nodes hang in a `deleting` state, but once all etcd nodes are deleting, they will be removed together. This is due to the fact that Rancher sees all etcd nodes deleting and proceeds to "short circuit" the etcd safe-removal logic. + +1. After all etcd nodes are removed, add a new etcd node that you are planning to restore from. + + - For custom clusters, go to the **Registration** tab then copy and run the registration command on your node. If the node has previously been used in a cluster, [clean the node](../manage-clusters/clean-cluster-nodes.md#cleaning-up-nodes) first. + - For node driver clusters, a new node is provisioned automatically. + + At this point, Rancher will indicate that restoration from etcd snapshot is required. + +1. Restore from an etcd snapshot. + + - For S3 snapshots, restore using the UI. + 1. Click the **Snapshots** tab to view the list of saved snapshots. + 1. Go to the snapshot you want to restore and click **⋮ > Restore**. + 1. Select a **Restore Type**. + 1. Click **Restore**. + - For local snapshots, restore using the UI is **not** available. + 1. In the upper right corner, click **⋮ > Edit YAML**. + 1. Define `spec.cluster.rkeConfig.etcdSnapshotRestore.name` as the filename of the snapshot on disk in `/var/lib/rancher//server/db/snapshots/`. + +1. After restoration is successful, you can scale your etcd nodes back up to the desired redundancy. + + + + +## Recovering etcd without a Snapshot (RKE) + +If the group of etcd nodes loses quorum, the Kubernetes cluster will report a failure because no operations, e.g. deploying workloads, can be executed in the Kubernetes cluster. The cluster should have three etcd nodes to prevent a loss of quorum. If you want to recover your set of etcd nodes, follow these instructions: + +1. Keep only one etcd node in the cluster by removing all other etcd nodes. + +2. On the single remaining etcd node, run the following command: + + ```bash + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock assaflavie/runlike etcd + ``` + + This command outputs the running command for etcd, save this command to use later. + +3. Stop the running `etcd` container and rename it to `etcd-old`. + + ```bash + docker stop etcd + docker rename etcd etcd-old + ``` + +4. Take the saved command from Step 2 and revise it: + + - If you originally had more than 1 etcd node, then you need to change `--initial-cluster` to only contain the node that remains. + - Add `--force-new-cluster` to the end of the command. + +5. Run the revised command. + +6. After the single nodes is up and running, Rancher recommends adding additional etcd nodes to your cluster. If you have a [custom cluster](../../../pages-for-subheaders/use-existing-nodes.md) and you want to reuse an old node, you are required to [clean up the nodes](../manage-clusters/clean-cluster-nodes.md) before attempting to add them back into a cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md new file mode 100644 index 00000000000..d78bde42e1b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md @@ -0,0 +1,101 @@ +--- +title: Restoring Rancher +--- + + + + + +This page outlines how to perform a restore with Rancher. + +:::note Important: + +* Follow the instructions from this page for restoring rancher on the same cluster where it was backed up from. In order to migrate rancher to a new cluster, follow the steps to [migrate rancher.](migrate-rancher-to-new-cluster.md) +* While restoring rancher on the same setup, the operator will scale down the rancher deployment when restore starts, and it will scale back up the deployment once restore completes. So Rancher will be unavailable during the restore. +* If you need to restore Rancher to a previous version after an upgrade, see the [rollback documentation.](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md) + +::: + +## Additional Steps for Rollbacks with Rancher v2.6.4+ + +Rancher v2.6.4 upgrades the cluster-api module from v0.4.4 to v1.0.2. Version v1.0.2 of the cluster-api, in turn, upgrades the Cluster API's Custom Resource Definitions (CRDs) from `cluster.x-k8s.io/v1alpha4` to `cluster.x-k8s.io/v1beta1`. The CRDs upgrade to v1beta1 causes rollbacks to fail when you attempt to move from Rancher v2.6.4 to any previous version of Rancher v2.6.x. This is because CRDs that use the older apiVersion (v1alpha4) are incompatible with v1beta1. + +To avoid rollback failure, the following Rancher scripts should be run **before** you attempt a restore operation or rollback: + +* `verify.sh`: Checks for any Rancher-related resources in the cluster. +* `cleanup.sh`: Cleans up the cluster. + +See the [rancher/rancher-cleanup repo](https://github.com/rancher/rancher-cleanup) for more details and source code. + +:::caution + + There will be downtime while `cleanup.sh` runs, since the script deletes resources created by Rancher. + +::: + +### Rolling back from v2.6.4+ to lower versions of v2.6.x + +1. Follow these [instructions](https://github.com/rancher/rancher-cleanup/blob/main/README.md) to run the scripts. +1. Follow these [instructions](./migrate-rancher-to-new-cluster.md) to install the rancher-backup Helm chart on the existing cluster and restore the previous state. + 1. Omit Step 3. + 1. When you reach Step 4, install the Rancher v2.6.x version on the local cluster you intend to roll back to. + +### Create the Restore Custom Resource + +A restore is performed by creating a Restore custom resource. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. The `local` cluster runs the Rancher server. +1. In the left navigation bar, click **Rancher Backups > Restores**. +1. Click **Create**. +1. Create the Restore with the form, or with YAML. For creating the Restore resource using form, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) +1. For using the YAML editor, we can click **Create > Create from YAML**. Enter the Restore YAML. + + ```yaml + apiVersion: resources.cattle.io/v1 + kind: Restore + metadata: + name: restore-migration + spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + encryptionConfigSecretName: encryptionconfig + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + ``` + + For help configuring the Restore, refer to the [configuration reference](../../../reference-guides/backup-restore-configuration/restore-configuration.md) and to the [examples.](../../../reference-guides/backup-restore-configuration/examples.md) + +1. Click **Create**. + +**Result:** The rancher-operator scales down the rancher deployment during restore, and scales it back up once the restore completes. The resources are restored in this order: + +1. Custom Resource Definitions (CRDs) +2. Cluster-scoped resources +3. Namespaced resources + +### Logs + +To check how the restore is progressing, you can check the logs of the operator. Run this command to follow the logs: + +``` +kubectl logs -n cattle-resources-system -l app.kubernetes.io/name=rancher-backup -f +``` + +### Cleanup + +If you created the restore resource with kubectl, remove the resource to prevent a naming conflict with future restores. + +### Known Issues +In some cases, after restoring the backup, Rancher logs will show errors similar to the following: +``` +2021/10/05 21:30:45 [ERROR] error syncing 'c-89d82/m-4067aa68dd78': handler rke-worker-upgrader: clusters.management.cattle.io "c-89d82" not found, requeuing +``` +This happens because one of the resources that was just restored has finalizers, but the related resources have been deleted so the handler cannot find it. + +To eliminate the errors, we need to find and delete the resource that causes the error. See more information [here](https://github.com/rancher/rancher/issues/35050#issuecomment-937968556) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md new file mode 100644 index 00000000000..8ff8e4cd5be --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md @@ -0,0 +1,71 @@ +--- +title: Continuous Delivery with Fleet +--- + + + + + +Continuous Delivery with Fleet is GitOps at scale. Fleet is designed to manage up to a million clusters. It's also lightweight enough that it works great for a [single cluster](https://fleet.rancher.io/installation#default-install) too, but it really shines when you get to a [large scale.](https://fleet.rancher.io/installation#configuration-for-multi-cluster) By large scale we mean either a lot of clusters, a lot of deployments, or a lot of teams in a single organization. + +Fleet is a separate project from Rancher, and can be installed on any Kubernetes cluster with Helm. + + +## Architecture + +For information about how Fleet works, see [this page.](../../../integrations-in-rancher/fleet-gitops-at-scale/architecture.md) + +## Accessing Fleet in the Rancher UI + +Fleet comes preinstalled in Rancher and is managed by the **Continous Delivery** option in the Rancher UI. For additional information on Continuous Delivery and other Fleet troubleshooting tips, refer [here](https://fleet.rancher.io/troubleshooting). + +Users can leverage continuous delivery to deploy their applications to the Kubernetes clusters in the git repository without any manual operation by following **gitops** practice. + +Follow the steps below to access Continuous Delivery in the Rancher UI: + +1. Click **☰ > Continuous Delivery**. + +1. Select your namespace at the top of the menu, noting the following: + - By default,`fleet-default` is selected which includes all downstream clusters that are registered through Rancher. + - You may switch to `fleet-local`, which only contains the `local` cluster, or you may create your own workspace to which you may assign and move clusters. + - You can then manage clusters by clicking on **Clusters** on the left navigation bar. + +1. Click on **Gitrepos** on the left navigation bar to deploy the gitrepo into your clusters in the current workspace. + +1. Select your [git repository](https://fleet.rancher.io/gitrepo-add) and [target clusters/cluster group](https://fleet.rancher.io/gitrepo-targets). You can also create the cluster group in the UI by clicking on **Cluster Groups** from the left navigation bar. + +1. Once the gitrepo is deployed, you can monitor the application through the Rancher UI. + +## Windows Support + +For details on support for clusters with Windows nodes, see [this page.](../../../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md) + + +## GitHub Repository + +The Fleet Helm charts are available [here.](https://github.com/rancher/fleet/releases/latest) + + +## Using Fleet Behind a Proxy + +For details on using Fleet behind a proxy, see [this page.](../../../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md) + +## Helm Chart Dependencies + +In order for Helm charts with dependencies to deploy successfully, you must run a manual command (as listed below), as it is up to the user to fulfill the dependency list. If you do not do this and proceed to clone your repository and run `helm install`, your installation will fail because the dependencies will be missing. + +The Helm chart in the git repository must include its dependencies in the charts subdirectory. You must either manually run `helm dependencies update $chart` OR run `helm dependencies build $chart` locally, then commit the complete charts directory to your git repository. Note that you will update your commands with the applicable parameters. + +## Troubleshooting + +--- +* **Known Issue:** clientSecretName and helmSecretName secrets for Fleet gitrepos are not included in the backup nor restore created by the [backup-restore-operator](../backup-restore-and-disaster-recovery/back-up-rancher.md#1-install-the-rancher-backup-operator). We will update the community once a permanent solution is in place. + +* **Temporary Workaround:**
+By default, user-defined secrets are not backed up in Fleet. It is necessary to recreate secrets if performing a disaster recovery restore or migration of Rancher into a fresh cluster. To modify resourceSet to include extra resources you want to backup, refer to docs [here](https://github.com/rancher/backup-restore-operator#user-flow). + +--- + +## Documentation + +The Fleet documentation is at [https://fleet.rancher.io/.](https://fleet.rancher.io/) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md new file mode 100644 index 00000000000..64c852c3629 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md @@ -0,0 +1,179 @@ +--- +title: Multi-cluster Apps +--- + + + + + +Typically, most applications are deployed on a single Kubernetes cluster, but there will be times you might want to deploy multiple copies of the same application across different clusters and/or projects. In Rancher, a _multi-cluster application_, is an application deployed using a Helm chart across multiple clusters. With the ability to deploy the same application across multiple clusters, it avoids the repetition of the same action on each cluster, which could introduce user error during application configuration. With multi-cluster applications, you can customize to have the same configuration across all projects/clusters as well as have the ability to change the configuration based on your target project. Since multi-cluster application is considered a single application, it's easy to manage and maintain this application. + +Any Helm charts from a global catalog can be used to deploy and manage multi-cluster applications. + +After creating a multi-cluster application, you can program a global DNS entry to make it easier to access the application. + +## Prerequisites + +### Permissions + +To create a multi-cluster app in Rancher, you must have at least one of the following permissions: + +- A [project-member role](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) in the target cluster(s), which gives you the ability to create, read, update, and delete the workloads +- A [cluster owner role](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles) for the clusters(s) that include the target project(s) + +### Enable Legacy Features + +Because multi-cluster apps were deprecated and replaced with Fleet in Rancher v2.5, you will need to enable multi-cluster apps with a feature flag. + +1. In the upper left corner, click **☰ > Global Settings**. +1. Click **Feature Flags**. +1. Go to the `legacy` feature flag and click **Activate**. + +## Launching a Multi-Cluster App + +1. In the upper left corner, click **☰ > Multi-cluster Apps**. +1. Click **Launch**. +1. Find the application that you want to launch. +1. (Optional) Review the detailed descriptions, which are derived from the Helm chart's `README`. +1. Under **Configuration Options** enter a **Name** for the multi-cluster application. By default, this name is also used to create a Kubernetes namespace in each [target project](#targets) for the multi-cluster application. The namespace is named as `-`. +1. Select a **Template Version**. +1. Complete the [multi-cluster applications specific configuration options](#multi-cluster-app-configuration-options) as well as the [application configuration options](#application-configuration-options). +1. Select the **Members** who can [interact with the multi-cluster application](#members). +1. Add any [custom application configuration answers](#overriding-application-configuration-options-for-specific-projects) that would change the configuration for specific project(s) from the default application configuration answers. +1. Review the files in the **Preview** section. When you're satisfied, click **Launch**. + +**Result**: Your application is deployed to your chosen namespace. You can view the application status from the project's: + +## Multi-cluster App Configuration Options + +Rancher has divided the configuration option for the multi-cluster application into several sections. + +### Targets + +In the **Targets** section, select the projects that you want the application to be deployed in. The list of projects is based on what projects you have access to. For each project that you select, it will be added to the list, which shows the cluster name and project name that were selected. To remove a target project, click on **-**. + +### Upgrades + +In the **Upgrades** section, select the upgrade strategy to use, when you decide to upgrade your application. + +* **Rolling Update (batched):** When selecting this upgrade strategy, the number of applications upgraded at a time is based on the selected **Batch size** and the **Interval** specifies how many seconds to wait before starting the next batch of updates. + +* **Upgrade all apps simultaneously:** When selecting this upgrade strategy, all applications across all projects will be upgraded at the same time. + +### Roles + +In the **Roles** section, you define the role of the multi-cluster application. Typically, when a user [launches catalog applications](../../../pages-for-subheaders/helm-charts-in-rancher.md), that specific user's permissions are used for creation of all workloads/resources that is required by the app. + +For multi-cluster applications, the application is deployed by a _system user_ and is assigned as the creator of all underlying resources. A _system user_ is used instead of the actual user due to the fact that the actual user could be removed from one of the target projects. If the actual user was removed from one of the projects, then that user would no longer be able to manage the application for the other projects. + +Rancher will let you select from two options for Roles, **Project** and **Cluster**. Rancher will allow creation using any of these roles based on the user's permissions. + +- **Project** - This is the equivalent of a [project member](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles). If you select this role, Rancher will check that in all the target projects, the user has minimally the [project member](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) role. While the user might not be explicitly granted the _project member_ role, if the user is an [administrator](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md), a [cluster owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles), or a [project owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles), then the user is considered to have the appropriate level of permissions. + +- **Cluster** - This is the equivalent of a [cluster owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles). If you select this role, Rancher will check that in all the target projects, the user has minimally the [cluster owner](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) role. While the user might not be explicitly granted the _cluster owner_ role, if the user is an [administrator](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md), then the user is considered to have the appropriate level of permissions. + +When launching the application, Rancher will confirm if you have these permissions in the target projects before launching the application. + +:::note + +There are some applications like _Grafana_ or _Datadog_ that require access to specific cluster-scoped resources. These applications will require the _Cluster_ role. If you find out later that the application requires cluster roles, the multi-cluster application can be upgraded to update the roles. + +::: + +## Application Configuration Options + +For each Helm chart, there are a list of desired answers that must be entered in order to successfully deploy the chart. When entering answers, you must format them using the syntax rules found in [Using Helm: The format and limitations of –set](https://helm.sh/docs/intro/using_helm/#the-format-and-limitations-of---set), as Rancher passes them as `--set` flags to Helm. + +:::note Example + +When entering an answer that includes two values separated by a comma (i.e. `abc, bcd`), it is required to wrap the values with double quotes (i.e., ``"abc, bcd"``). + +::: + +### Using a questions.yml file + +If the Helm chart that you are deploying contains a `questions.yml` file, Rancher's UI will translate this file to display an easy to use UI to collect the answers for the questions. + +### Key Value Pairs for Native Helm Charts + +For native Helm charts (i.e., charts from the **Helm Stable** or **Helm Incubator** catalogs or a custom Helm chart repository, answers are provided as key value pairs in the **Answers** section. These answers are used to override the default values. + +### Members + +By default, multi-cluster applications can only be managed by the user who created it. In the **Members** section, other users can be added so that they can also help manage or view the multi-cluster application. + +1. Find the user that you want to add by typing in the member's name in the **Member** search box. + +2. Select the **Access Type** for that member. There are three access types for a multi-cluster project, but due to how the permissions of a multi-cluster application are launched, please read carefully to understand what these access types mean. + + - **Owner**: This access type can manage any configuration part of the multi-cluster application including the template version, the [multi-cluster applications specific configuration options](#Multi-cluster App Configuration Options), the [application specific configuration options](#application-configuration-options), the members who can interact with the multi-cluster application and the [custom application configuration answers](#overriding-application-configuration-options-for-specific-projects). Since a multi-cluster application is created with a different set of permissions from the user, any _owner_ of the multi-cluster application can manage/remove applications in [target projects](#targets) without explicitly having access to these project(s). Only trusted users should be provided with this access type. + + - **Member**: This access type can only modify the template version, the [application specific configuration options](#application-configuration-options) and the [custom application configuration answers](#overriding-application-configuration-options-for-specific-projects). Since a multi-cluster application is created with a different set of permissions from the user, any _member_ of the multi-cluster application can modify the application without explicitly having access to these project(s). Only trusted users should be provided with this access type. + + - **Read-only**: This access type cannot modify any configuration option for the multi-cluster application. Users can only view these applications. + + :::caution + + Please ensure only trusted users are given _Owner_ or _Member_ access as they will automatically be able to manage applications created for this multi-cluster application in target projects they might not have direct access to. + + ::: + +### Overriding Application Configuration Options for Specific Projects + +The ability to use the same configuration to deploy the same application across multiple clusters/projects is one of the main benefits of multi-cluster applications. There might be a specific project that requires a slightly different configuration option, but you want to manage that application with all the other matching applications. Instead of creating a brand new application, you can override specific [application specific configuration options](#application-configuration-options) for specific projects. + +1. In the **Answer Overrides** section, click **Add Override**. + +2. For each override, you can select the following: + + - **Scope**: Select which target projects you want to override the answer in the configuration option. + + - **Question**: Select which question you want to override. + + - **Answer**: Enter the answer that you want to be used instead. + +## Upgrading Multi-Cluster App Roles and Projects + +- **Changing Roles on an existing Multi-Cluster app** +The creator and any users added with the access-type "owner" to a multi-cluster app, can upgrade its Roles. When adding a new Role, we check if the user has that exact role in all current target projects. These checks allow the same relaxations for global admins, cluster owners and project-owners as described in the installation section for the field `Roles`. + +- **Adding/Removing target projects** +1. The creator and any users added with access-type "owner" to a multi-cluster app, can add or remove its target projects. When adding a new project, we check if the caller of this request has all Roles defined on multi-cluster app, in the new projects they want to add. The roles checks are again relaxed for global admins, cluster-owners and project-owners. +2. We do not do these membership checks when removing target projects. This is because the caller's permissions could have with respect to the target project, or the project could have been deleted and hence the caller wants to remove it from targets list. + + +## Multi-Cluster Application Management + +One of the benefits of using a multi-cluster application as opposed to multiple individual applications of the same type, is the ease of management. Multi-cluster applications can be cloned, upgraded or rolled back. + +:::note Prerequisite: + +The `legacy` feature flag needs to be enabled. + +::: + +1. In the upper left corner, click **☰ > Multi-cluster Apps**. + +2. Choose the multi-cluster application you want to take one of these actions on and click the **⋮**. Select one of the following options: + + * **Clone**: Creates another multi-cluster application with the same configuration. By using this option, you can easily duplicate a multi-cluster application. + * **Upgrade**: Upgrade your multi-cluster application to change some part of the configuration. When performing an upgrade for multi-cluster application, the [upgrade strategy](#upgrades) can be modified if you have the correct [access type](#members). + * **Rollback**: Rollback your application to a specific version. If after an upgrade, there are issues for your multi-cluster application for one or more of your [targets](#targets), Rancher has stored up to 10 versions of the multi-cluster application. Rolling back a multi-cluster application reverts the application for **all** target clusters and projects, not just the targets(s) affected by the upgrade issue. + +## Deleting a Multi-Cluster Application + +:::note Prerequisite: + +The `legacy` feature flag needs to be enabled. + +::: + +1. In the upper left corner, click **☰ > Multi-cluster Apps**. + +2. Choose the multi-cluster application you want to delete and click the **⋮ > Delete**. When deleting the multi-cluster application, all applications and namespaces are deleted in all of the target projects. + + :::note + + The applications in the target projects, that are created for a multi-cluster application, cannot be deleted individually. The applications can only be deleted when the multi-cluster application is deleted. + + ::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md new file mode 100644 index 00000000000..300e152bb22 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps.md @@ -0,0 +1,133 @@ +--- +title: Creating Apps +--- + + + + + +:::tip + +For a complete walkthrough of developing charts, see the [Chart Template Developer's Guide](https://helm.sh/docs/chart_template_guide/) in the official Helm documentation. + +::: + +## Chart Types + +Rancher supports two different types of charts: Helm charts and Rancher charts. + +### Helm Charts + +Native Helm charts include an application along with other software required to run it. When deploying native Helm charts, you' can provide the chart's parameter values in a YAML editor. + +### Rancher Charts + +Rancher charts are native helm charts with two files that enhance user experience: `app-readme.md` and `questions.yaml`. Read more about them in [Additional Files for Rancher Charts.](#additional-files-for-rancher-charts) + +Rancher charts add simplified chart descriptions and configuration forms to make the application deployment easy. Rancher users do not need to read through the entire list of Helm variables to understand how to launch an application. + +## Chart Directory Structure + +You can provide Helm Charts in a standard, HTTP based Helm Repository. For more information see the [Chart Repository Guide](https://helm.sh/docs/topics/chart_repository) in the official Helm documentation. + +Alternatively you can organize your charts in a Git Repository and directly add this to the App Marketplace. + +The following table demonstrates the directory structure for a Git repository. The `charts` directory is the top level directory under the repository base. Adding the repository to Rancher will expose all charts contained within it. The `questions.yaml`, `README.md`, and `requirements.yml` files are specific to Rancher charts, but are optional for chart customization. + +``` +/ + │ + ├── charts/ + │ ├── / # This directory name will be surfaced in the Rancher UI as the chart name + │ │ ├── / # Each directory at this level provides different app versions that will be selectable within the chart in the Rancher UI + │ │ │ ├── Chart.yaml # Required Helm chart information file. + │ │ │ ├── questions.yaml # Form questions displayed within the Rancher UI. Questions display in Configuration Options.* + │ │ │ ├── README.md # Optional: Helm Readme file displayed within Rancher UI. This text displays in Detailed Descriptions. + │ │ │ ├── requirements.yml # Optional: YAML file listing dependencies for the chart. + │ │ │ ├── values.yml # Default configuration values for the chart. + │ │ │ ├── templates/ # Directory containing templates that, when combined with values.yml, generates Kubernetes YAML. +``` + +## Additional Files for Rancher Charts + +Before you create your own custom catalog, you should have a basic understanding about how a Rancher chart differs from a native Helm chart. Rancher charts differ slightly from Helm charts in their directory structures. Rancher charts include two files that Helm charts do not. + +- `app-readme.md` + + A file that provides descriptive text in the chart's UI header. + +- `questions.yml` + + A file that contains questions for a form. These form questions simplify deployment of a chart. Without it, you must configure the deployment using a values YAML config, which is more difficult. The following images display the difference between a Rancher chart (which includes `questions.yml`) and a native Helm chart (which does not). + +
Rancher Chart with questions.yml (top) vs. Helm Chart without (bottom)
+ + ![questions.yml](/img/rancher-app-2.6.png) + ![values.yaml](/img/helm-app-2.6.png) + + +### Chart.yaml annotations + +Rancher supports additional annotations that you can add to the `Chart.yaml` file. These annotations allow you to define application dependencies or configure additional UI defaults: + +| Annotation | Description | Example | +| --------------------------------- | ----------- | ------- | +| catalog.cattle.io/auto-install | If set, will install the specified chart in the specified version before installing this chart | other-chart-name=1.0.0 | +| catalog.cattle.io/display-name | A display name that should be displayed in the App Marketplace instead of the chart name | Display Name of Chart | +| catalog.cattle.io/namespace | A fixed namespace where the chart should be deployed in. If set, this can't be changed by the user | fixed-namespace | +| catalog.cattle.io/release-name | A fixed release name for the Helm installation. If set, this can't be changed by the user | fixed-release-name | +| catalog.cattle.io/requests-cpu | Total amount of CPU that should be unreserverd in the cluster. If less CPU is available, a warning will be shown | 2000m | +| catalog.cattle.io/requests-memory | Total amount of memory that should be unreserverd in the cluster. If less memory is available, a warning will be shown | 2Gi | +| catalog.cattle.io/os | Restricts the OS where this chart can be installed. Possible values: `linux`, `windows`. Default: no restriction | linux | + +### questions.yml + +Inside the `questions.yml`, most of the content will be around the questions to ask the end user, but there are some additional fields that can be set in this file. + +### Min/Max Rancher versions + +For each chart, you can add the minimum and/or maximum Rancher version, which determines whether or not this chart is available to be deployed from Rancher. + +:::note + +Even though Rancher release versions are prefixed with a `v`, there is *no* prefix for the release version when using this option. + +::: + +``` +rancher_min_version: 2.3.0 +rancher_max_version: 2.3.99 +``` + +### Question Variable Reference + +This reference contains variables that you can use in `questions.yml` nested under `questions:`. + +| Variable | Type | Required | Description | +| ------------- | ------------- | --- |------------- | +| variable | string | true | Define the variable name specified in the `values.yml` file, using `foo.bar` for nested objects. | +| label | string | true | Define the UI label. | +| description | string | false | Specify the description of the variable.| +| type | string | false | Default to `string` if not specified (current supported types are string, multiline, boolean, int, enum, password, storageclass, hostname, pvc, and secret).| +| default | string | false | Specify the default value. Only used if there is no corresponding value in the `values.yml` file. | +| group | string | false | Group questions by input value. | +| options | []string | false | Specify the options when the variable type is `enum`, for example: options:
- "ClusterIP"
- "NodePort"
- "LoadBalancer"| +| subquestions | []subquestion | false| Add an array of subquestions.| +| show_if | string | false | Show current variable if conditional variable is true. For example `show_if: "serviceType=Nodeport"` | +| show\_subquestion_if | string | false | Show subquestions if is true or equal to one of the options. for example `show_subquestion_if: "true"`| + + + +:::note + +`subquestions[]` cannot contain `subquestions` or `show_subquestions_if` keys, but all other keys in the above table are supported. + +::: diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md new file mode 100644 index 00000000000..f0157a3f6e0 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer.md @@ -0,0 +1,179 @@ +--- +title: Setting up Amazon ELB Network Load Balancer +--- + + + + + +This how-to guide describes how to set up a Network Load Balancer (NLB) in Amazon's EC2 service that will direct traffic to multiple instances on EC2. + +These examples show the load balancer being configured to direct traffic to three Rancher server nodes. If Rancher is installed on an RKE Kubernetes cluster, three nodes are required. If Rancher is installed on a K3s Kubernetes cluster, only two nodes are required. + +This tutorial is about one possible way to set up your load balancer, not the only way. Other types of load balancers, such as a Classic Load Balancer or Application Load Balancer, could also direct traffic to the Rancher server nodes. + +Rancher only supports using the Amazon NLB when terminating traffic in `tcp` mode for port 443 rather than `tls` mode. This is due to the fact that the NLB does not inject the correct headers into requests when terminated at the NLB. This means that if you want to use certificates managed by the Amazon Certificate Manager (ACM), you should use an ALB. + + + +## Requirements + +These instructions assume you have already created Linux instances in EC2. The load balancer will direct traffic to these nodes. + +## 1. Create Target Groups + +Begin by creating two target groups for the **TCP** protocol, one with TCP port 443 and one regarding TCP port 80 (providing redirect to TCP port 443). You'll add your Linux nodes to these groups. + +Your first NLB configuration step is to create two target groups. Technically, only port 443 is needed to access Rancher, but it's convenient to add a listener for port 80, because traffic to port 80 will be automatically redirected to port 443. + +Regardless of whether an NGINX Ingress or Traefik Ingress controller is used, the Ingress should redirect traffic from port 80 to port 443. + +1. Log into the [Amazon AWS Console](https://console.aws.amazon.com/ec2/) to get started. Make sure to select the **Region** where your EC2 instances (Linux nodes) are created. +1. Select **Services** and choose **EC2**, find the section **Load Balancing** and open **Target Groups**. +1. Click **Create target group** to create the first target group, regarding TCP port 443. + +:::note + +Health checks are handled differently based on the Ingress. For details, refer to [this section.](#health-check-paths-for-nginx-ingress-and-traefik-ingresses) + +::: + +#### Target Group (TCP port 443) + +Configure the first target group according to the table below. + +| Option | Setting | +|-------------------|-------------------| +| Target Group Name | `rancher-tcp-443` | +| Target type | `instance` | +| Protocol | `TCP` | +| Port | `443` | +| VPC | Choose your VPC | + +Health check settings: + +| Option | Setting | +|---------------------|-----------------| +| Protocol | TCP | +| Port | `override`,`80` | +| Healthy threshold | `3` | +| Unhealthy threshold | `3` | +| Timeout | `6 seconds` | +| Interval | `10 seconds` | + +Click **Create target group** to create the second target group, regarding TCP port 80. + +#### Target Group (TCP port 80) + +Configure the second target group according to the table below. + +| Option | Setting | +|-------------------|------------------| +| Target Group Name | `rancher-tcp-80` | +| Target type | `instance` | +| Protocol | `TCP` | +| Port | `80` | +| VPC | Choose your VPC | + + +Health check settings: + +| Option |Setting | +|---------------------|----------------| +| Protocol | TCP | +| Port | `traffic port` | +| Healthy threshold | `3` | +| Unhealthy threshold | `3` | +| Timeout | `6 seconds` | +| Interval | `10 seconds` | + +## 2. Register Targets + +Next, add your Linux nodes to both target groups. + +Select the target group named **rancher-tcp-443**, click the tab **Targets** and choose **Edit**. + +![](/img/ha/nlb/edit-targetgroup-443.png) + +Select the instances (Linux nodes) you want to add, and click **Add to registered**. + +*** +**Screenshot Add targets to target group TCP port 443**
+ +![](/img/ha/nlb/add-targets-targetgroup-443.png) + +*** +**Screenshot Added targets to target group TCP port 443**
+ +![](/img/ha/nlb/added-targets-targetgroup-443.png) + +When the instances are added, click **Save** on the bottom right of the screen. + +Repeat those steps, replacing **rancher-tcp-443** with **rancher-tcp-80**. The same instances need to be added as targets to this target group. + +## 3. Create Your NLB + +Use Amazon's Wizard to create a Network Load Balancer. As part of this process, you'll add the target groups you created in [1. Create Target Groups](#1-create-target-groups). + +1. From your web browser, navigate to the [Amazon EC2 Console](https://console.aws.amazon.com/ec2/). + +2. From the navigation pane, choose **LOAD BALANCING** > **Load Balancers**. + +3. Click **Create Load Balancer**. + +4. Choose **Network Load Balancer** and click **Create**. Then complete each form. + +- [Step 1: Configure Load Balancer](#step-1-configure-load-balancer) +- [Step 2: Configure Routing](#step-2-configure-routing) +- [Step 3: Register Targets](#step-3-register-targets) +- [Step 4: Review](#step-4-review) + +### Step 1: Configure Load Balancer + +Set the following fields in the form: + +- **Name:** `rancher` +- **Scheme:** `internal` or `internet-facing`. The scheme that you choose for your NLB is dependent on the configuration of your instances and VPC. If your instances do not have public IPs associated with them, or you will only be accessing Rancher internally, you should set your NLB Scheme to `internal` rather than `internet-facing`. +- **Listeners:** The Load Balancer Protocol should be `TCP` and the corresponding Load Balancer Port should be set to `443`. +- **Availability Zones:** Select Your **VPC** and **Availability Zones**. + +### Step 2: Configure Routing + +1. From the **Target Group** drop-down, choose **Existing target group**. +1. From the **Name** drop-down, choose `rancher-tcp-443`. +1. Open **Advanced health check settings**, and configure **Interval** to `10 seconds`. + +### Step 3: Register Targets + +Since you registered your targets earlier, all you have to do is click **Next: Review**. + +### Step 4: Review + +Look over the load balancer details and click **Create** when you're satisfied. + +After AWS creates the NLB, click **Close**. + +## 4. Add listener to NLB for TCP port 80 + +1. Select your newly created NLB and select the **Listeners** tab. + +2. Click **Add listener**. + +3. Use `TCP`:`80` as **Protocol** : **Port** + +4. Click **Add action** and choose **Forward to..**. + +5. From the **Forward to** drop-down, choose `rancher-tcp-80`. + +6. Click **Save** in the top right of the screen. + +## Health Check Paths for NGINX Ingress and Traefik Ingresses + +K3s and RKE Kubernetes clusters handle health checks differently because they use different Ingresses by default. + +For RKE Kubernetes clusters, NGINX Ingress is used by default, whereas for K3s Kubernetes clusters, Traefik is the default Ingress. + +- **Traefik:** The health check path is `/ping`. By default `/ping` is always matched (regardless of Host), and a response from [Traefik itself](https://docs.traefik.io/operations/ping/) is always served. +- **NGINX Ingress:** The default backend of the NGINX Ingress controller has a `/healthz` endpoint. By default `/healthz` is always matched (regardless of Host), and a response from [`ingress-nginx` itself](https://github.com/kubernetes/ingress-nginx/blob/0cbe783f43a9313c9c26136e888324b1ee91a72f/charts/ingress-nginx/values.yaml#L212) is always served. + +To simulate an accurate health check, it is a best practice to use the Host header (Rancher hostname) combined with `/ping` or `/healthz` (for K3s or for RKE clusters, respectively) wherever possible, to get a response from the Rancher Pods, not the Ingress. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md new file mode 100644 index 00000000000..1611ca978e3 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md @@ -0,0 +1,77 @@ +--- +title: Set up Infrastructure for a High Availability K3s Kubernetes Cluster +--- + + + + + +This tutorial is intended to help you provision the underlying infrastructure for a Rancher management server. + +The recommended infrastructure for the Rancher-only Kubernetes cluster differs depending on whether Rancher will be installed on a K3s Kubernetes cluster, an RKE Kubernetes cluster, or a single Docker container. + +For more information about each installation option, refer to [this page.](../../../pages-for-subheaders/installation-and-upgrade.md) + +:::note Important: + +These nodes must be in the same region. You may place these servers in separate availability zones (datacenter). + +::: + +To install the Rancher management server on a high-availability K3s cluster, we recommend setting up the following infrastructure: + +- **Two Linux nodes,** typically virtual machines, in the infrastructure provider of your choice. +- **An external database** to store the cluster data. We recommend MySQL. +- **A load balancer** to direct traffic to the two nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +### 1. Set up Linux Nodes + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up External Datastore + +The ability to run Kubernetes using a datastore other than etcd sets K3s apart from other Kubernetes distributions. This feature provides flexibility to Kubernetes operators. The available options allow you to select a datastore that best fits your use case. + +For a high-availability K3s installation, you will need to set a [MySQL](https://www.mysql.com/) external database. Rancher has been tested on K3s Kubernetes clusters using MySQL version 5.7 as the datastore. + +When you install Kubernetes using the K3s installation script, you will pass in details for K3s to connect to the database. + +For an example of one way to set up the MySQL database, refer to this [tutorial](mysql-database-in-amazon-rds.md) for setting up MySQL on Amazon's RDS service. + +For the complete list of options that are available for configuring a K3s cluster datastore, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/datastore/) + +### 3. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on both nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the K3s tool will deploy a Traefik Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the Traefik Ingress controller to listen for traffic destined for the Rancher hostname. The Traefik Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 4. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the load balancer IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md new file mode 100644 index 00000000000..ad266904aed --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md @@ -0,0 +1,68 @@ +--- +title: Set up Infrastructure for a High Availability RKE Kubernetes Cluster +--- + + + + + +This tutorial is intended to help you create a high-availability RKE cluster that can be used to install a Rancher server. + +:::note Important: + +These nodes must be in the same region. You may place these servers in separate availability zones (datacenter). + +::: + +To install the Rancher management server on a high-availability RKE cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in an infrastructure provider such as Amazon's EC2, Google Compute Engine, or vSphere. +- **A load balancer** to direct front-end traffic to the three nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +These nodes must be in the same region/data center. You may place these servers in separate availability zones. + +### Why three nodes? + +In an RKE cluster, Rancher server data is stored on etcd. This etcd database runs on all three nodes. + +The etcd database requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can suffer from [split brain](https://www.quora.com/What-is-split-brain-in-distributed-systems), requiring the cluster to be restored from backup. If one of the three etcd nodes fails, the two remaining nodes can elect a leader because they have the majority of the total number of etcd nodes. + +### 1. Set up Linux Nodes + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on any of the three nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE tool will deploy an NGINX Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the NGINX Ingress controller to listen for traffic destined for the Rancher hostname. The NGINX Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 3. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the LB IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md new file mode 100644 index 00000000000..c00f0061adb --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md @@ -0,0 +1,62 @@ +--- +title: Set up Infrastructure for a High Availability RKE2 Kubernetes Cluster +--- + + + + + +This tutorial is intended to help you provision the underlying infrastructure for a Rancher management server. + +The recommended infrastructure for the Rancher-only Kubernetes cluster differs depending on whether Rancher will be installed on a RKE2 Kubernetes cluster, an RKE Kubernetes cluster, or a single Docker container. + +:::note Important: + +These nodes must be in the same region. You may place these servers in separate availability zones (datacenter). + +::: + +To install the Rancher management server on a high-availability RKE2 cluster, we recommend setting up the following infrastructure: + +- **Three Linux nodes,** typically virtual machines, in the infrastructure provider of your choice. +- **A load balancer** to direct traffic to the two nodes. +- **A DNS record** to map a URL to the load balancer. This will become the Rancher server URL, and downstream Kubernetes clusters will need to reach it. + +### 1. Set up Linux Nodes + +Make sure that your nodes fulfill the general installation requirements for [OS, container runtime, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +For an example of one way to set up Linux nodes, refer to this [tutorial](nodes-in-amazon-ec2.md) for setting up nodes as instances in Amazon EC2. + +### 2. Set up the Load Balancer + +You will also need to set up a load balancer to direct traffic to the Rancher replica on all nodes. That will prevent an outage of any single node from taking down communications to the Rancher management server. + +When Kubernetes gets set up in a later step, the RKE2 tool will deploy an Nginx Ingress controller. This controller will listen on ports 80 and 443 of the worker nodes, answering traffic destined for specific hostnames. + +When Rancher is installed (also in a later step), the Rancher system creates an Ingress resource. That Ingress tells the Nginx Ingress controller to listen for traffic destined for the Rancher hostname. The Nginx Ingress controller, when receiving traffic destined for the Rancher hostname, will forward that traffic to the running Rancher pods in the cluster. + +For your implementation, consider if you want or need to use a Layer-4 or Layer-7 load balancer: + +- **A layer-4 load balancer** is the simpler of the two choices, in which you are forwarding TCP traffic to your nodes. We recommend configuring your load balancer as a Layer 4 balancer, forwarding traffic to ports TCP/80 and TCP/443 to the Rancher management cluster nodes. The Ingress controller on the cluster will redirect HTTP traffic to HTTPS and terminate SSL/TLS on port TCP/443. The Ingress controller will forward traffic to port TCP/80 to the Ingress pod in the Rancher deployment. +- **A layer-7 load balancer** is a bit more complicated but can offer features that you may want. For instance, a layer-7 load balancer is capable of handling TLS termination at the load balancer, as opposed to Rancher doing TLS termination itself. This can be beneficial if you want to centralize your TLS termination in your infrastructure. Layer-7 load balancing also offers the capability for your load balancer to make decisions based on HTTP attributes such as cookies, etc. that a layer-4 load balancer is not able to concern itself with. If you decide to terminate the SSL/TLS traffic on a layer-7 load balancer, you will need to use the `--set tls=external` option when installing Rancher in a later step. For more information, refer to the [Rancher Helm chart options.](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +For an example showing how to set up an NGINX load balancer, refer to [this page.](nginx-load-balancer.md) + +For a how-to guide for setting up an Amazon ELB Network Load Balancer, refer to [this page.](amazon-elb-load-balancer.md) + +:::caution + +Do not use this load balancer (i.e, the `local` cluster Ingress) to load balance applications other than Rancher following installation. Sharing this Ingress with other applications may result in websocket errors to Rancher following Ingress configuration reloads for other apps. We recommend dedicating the `local` cluster to Rancher and no other applications. + +::: + +### 4. Set up the DNS Record + +Once you have set up your load balancer, you will need to create a DNS record to send traffic to this load balancer. + +Depending on your environment, this may be an A record pointing to the load balancer IP, or it may be a CNAME pointing to the load balancer hostname. In either case, make sure this record is the hostname that you intend Rancher to respond on. + +You will need to specify this hostname in a later step when you install Rancher, and it is not possible to change it later. Make sure that your decision is a final one. + +For a how-to guide for setting up a DNS record to route domain traffic to an Amazon ELB load balancer, refer to the [official AWS documentation.](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-elb-load-balancer) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md new file mode 100644 index 00000000000..1d6646dd737 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds.md @@ -0,0 +1,38 @@ +--- +title: Setting up a MySQL Database in Amazon RDS +--- + + + + + +This tutorial describes how to set up a MySQL database in Amazon's Relational Database Service (RDS). + +This database can later be used as an external datastore for a high-availability K3s Kubernetes cluster. + +1. Log into the [Amazon AWS RDS Console](https://console.aws.amazon.com/rds/) to get started. Make sure to select the **Region** where your EC2 instances (Linux nodes) are created. +1. In the left panel, click **Databases**. +1. Click **Create database**. +1. In the **Engine type** section, click **MySQL**. +1. In the **Version** section, choose **MySQL 5.7.22**. +1. In **Settings** section, under **Credentials Settings,** enter a master password for the **admin** master username. Confirm the password. +1. Expand the **Additional configuration** section. In the **Initial database name** field, enter a name. The name can have only letters, numbers, and underscores. This name will be used to connect to the database. +1. Click **Create database**. + +You'll need to capture the following information about the new database so that the K3s Kubernetes cluster can connect to it. + +To see this information in the Amazon RDS console, click **Databases,** and click the name of the database that you created. + +- **Username:** Use the admin username. +- **Password:** Use the admin password. +- **Hostname:** Use the **Endpoint** as the hostname. The endpoint is available in the **Connectivity & security** section. +- **Port:** The port should be 3306 by default. You can confirm it in the **Connectivity & security** section. +- **Database name:** Confirm the name by going to the **Configuration** tab. The name is listed under **DB name**. + +This information will be used to connect to the database in the following format: + +``` +mysql://username:password@tcp(hostname:3306)/database-name +``` + +For more information on configuring the datastore for K3s, refer to the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/datastore/) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md new file mode 100644 index 00000000000..896c3686ad2 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer.md @@ -0,0 +1,106 @@ +--- +title: Setting up an NGINX Load Balancer +--- + + + + + +NGINX will be configured as Layer 4 load balancer (TCP) that forwards connections to one of your Rancher nodes. + +In this configuration, the load balancer is positioned in front of your nodes. The load balancer can be any host capable of running NGINX. + +:::note + +Do not use one of your Rancher nodes as the load balancer. + +::: + +> These examples show the load balancer being configured to direct traffic to three Rancher server nodes. If Rancher is installed on an RKE Kubernetes cluster, three nodes are required. If Rancher is installed on a K3s Kubernetes cluster, only two nodes are required. + +## Install NGINX + +Start by installing NGINX on the node you want to use as a load balancer. NGINX has packages available for all known operating systems. The versions tested are `1.14` and `1.15`. For help installing NGINX, refer to their [install documentation](https://www.nginx.com/resources/wiki/start/topics/tutorials/install/). + +The `stream` module is required, which is present when using the official NGINX packages. Please refer to your OS documentation on how to install and enable the NGINX `stream` module on your operating system. + +## Create NGINX Configuration + +After installing NGINX, you need to update the NGINX configuration file, `nginx.conf`, with the IP addresses for your nodes. + +1. Copy and paste the code sample below into your favorite text editor. Save it as `nginx.conf`. + +2. From `nginx.conf`, replace both occurrences (port 80 and port 443) of ``, ``, and `` with the IPs of your nodes. + + :::note + + See [NGINX Documentation: TCP and UDP Load Balancing](https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/) for all configuration options. + + ::: + +
Example NGINX config
+ + ``` + worker_processes 4; + worker_rlimit_nofile 40000; + + events { + worker_connections 8192; + } + + stream { + upstream rancher_servers_http { + least_conn; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + server :80 max_fails=3 fail_timeout=5s; + } + server { + listen 80; + proxy_pass rancher_servers_http; + } + + } + + http { + + upstream rancher_servers_https { + least_conn; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + server :443 max_fails=3 fail_timeout=5s; + } + server { + listen 443 ssl; + proxy_pass rancher_servers_https; + ssl_certificate /path/to/tls.crt; + ssl_certificate_key /path/to/key.key; + location / { + proxy_pass https://rancher_servers_https; + proxy_set_header Host ; + proxy_ssl_server_name on; + proxy_ssl_name + } + } + } + ``` + + +3. Save `nginx.conf` to your load balancer at the following path: `/etc/nginx/nginx.conf`. + +4. Load the updates to your NGINX configuration by running the following command: + + ``` + # nginx -s reload + ``` + +## Option - Run NGINX as Docker container + +Instead of installing NGINX as a package on the operating system, you can rather run it as a Docker container. Save the edited **Example NGINX config** as `/etc/nginx.conf` and run the following command to launch the NGINX container: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /etc/nginx.conf:/etc/nginx/nginx.conf \ + nginx:1.14 +``` diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md new file mode 100644 index 00000000000..9c7f55b9443 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2.md @@ -0,0 +1,78 @@ +--- +title: Setting up Nodes in Amazon EC2 +--- + + + + + +In this tutorial, you will learn one way to set up Linux nodes for the Rancher management server. These nodes will fulfill the node requirements for [OS, Docker, hardware, and networking.](../../../pages-for-subheaders/installation-requirements.md) + +If the Rancher server will be installed on an RKE Kubernetes cluster, you should provision three instances. + +If the Rancher server will be installed on a K3s Kubernetes cluster, you only need to provision two instances. + +If the Rancher server is installed in a single Docker container, you only need one instance. + +### 1. Optional Preparation + +- **Create IAM role:** To allow Rancher to manipulate AWS resources, such as provisioning new storage or new nodes, you will need to configure Amazon as a cloud provider. There are several things you'll need to do to set up the cloud provider on EC2, but part of this process is setting up an IAM role for the Rancher server nodes. For the full details on setting up the cloud provider, refer to this [page.](../../../pages-for-subheaders/set-up-cloud-providers.md) +- **Create security group:** We also recommend setting up a security group for the Rancher nodes that complies with the [port requirements for Rancher nodes.](../../../pages-for-subheaders/installation-requirements.md#port-requirements) + +### 2. Provision Instances + +1. Log into the [Amazon AWS EC2 Console](https://console.aws.amazon.com/ec2/) to get started. Make sure to take note of the **Region** where your EC2 instances (Linux nodes) are created, because all of the infrastructure for the Rancher management server should be in the same region. +1. In the left panel, click **Instances**. +1. Click **Launch Instance**. +1. In the section called **Step 1: Choose an Amazon Machine Image (AMI),** we will use Ubuntu 18.04 as the Linux OS, using `ami-0d1cd67c26f5fca19 (64-bit x86)`. Go to the Ubuntu AMI and click **Select**. +1. In the **Step 2: Choose an Instance Type** section, select the `t2.medium` type. +1. Click **Next: Configure Instance Details**. +1. In the **Number of instances** field, enter the number of instances. A high-availability K3s cluster requires only two instances, while a high-availability RKE cluster requires three instances. +1. Optional: If you created an IAM role for Rancher to manipulate AWS resources, select the new IAM role in the **IAM role** field. +1. Click **Next: Add Storage,** **Next: Add Tags,** and **Next: Configure Security Group**. +1. In **Step 6: Configure Security Group,** select a security group that complies with the [port requirements](../../../pages-for-subheaders/installation-requirements.md#port-requirements) for Rancher nodes. +1. Click **Review and Launch**. +1. Click **Launch**. +1. Choose a new or existing key pair that you will use to connect to your instance later. If you are using an existing key pair, make sure you already have access to the private key. +1. Click **Launch Instances**. + + +**Result:** You have created Rancher nodes that satisfy the requirements for OS, hardware, and networking. + +:::note + +If the nodes are being used for an RKE Kubernetes cluster, install Docker on each node in the next step. For a K3s Kubernetes cluster, the nodes are now ready to install K3s. + +::: + +### 3. Install Docker and Create User for RKE Kubernetes Cluster Nodes + +1. From the [AWS EC2 console,](https://console.aws.amazon.com/ec2/) click **Instances** in the left panel. +1. Go to the instance that you want to install Docker on. Select the instance and click **Actions > Connect**. +1. Connect to the instance by following the instructions on the screen that appears. Copy the Public DNS of the instance. An example command to SSH into the instance is as follows: +``` +sudo ssh -i [path-to-private-key] ubuntu@[public-DNS-of-instance] +``` +1. Run the following command on the instance to install Docker with one of Rancher's installation scripts: +``` +curl https://releases.rancher.com/install-docker/18.09.sh | sh +``` +1. When you are connected to the instance, run the following command on the instance to create a user: +``` +sudo usermod -aG docker ubuntu +``` +1. Repeat these steps so that Docker is installed on each node that will eventually run the Rancher management server. + +:::tip + +To find out whether a script is available for installing a certain Docker version, refer to this [GitHub repository,](https://github.com/rancher/install-docker) which contains all of Rancher’s Docker installation scripts. + +::: + +**Result:** You have set up Rancher server nodes that fulfill all the node requirements for OS, Docker, hardware and networking. + +### Next Steps for RKE Kubernetes Cluster Nodes + +If you are going to install an RKE cluster on the new nodes, take note of the **IPv4 Public IP** and **Private IP** of each node. This information can be found on the **Description** tab for each node after it is created. The public and private IP will be used to populate the `address` and `internal_address` of each node in the RKE cluster configuration file, `rancher-cluster.yml`. + +RKE will also need access to the private key to connect to each node. Therefore, you might want to take note of the path to your private keys to connect to the nodes, which can also be included in the `rancher-cluster.yml` under the `ssh_key_path` directive for each node. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md new file mode 100644 index 00000000000..a5d69ed221d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs.md @@ -0,0 +1,29 @@ +--- +title: About High-availability Installations +--- + + + + + +We recommend using Helm, a Kubernetes package manager, to install Rancher on a dedicated Kubernetes cluster. This is called a high-availability Kubernetes installation because increased availability is achieved by running Rancher on multiple nodes. + +In a standard installation, Kubernetes is first installed on three nodes that are hosted in an infrastructure provider such as Amazon's EC2 or Google Compute Engine. + +Then Helm is used to install Rancher on top of the Kubernetes cluster. Helm uses Rancher's Helm chart to install a replica of Rancher on each of the three nodes in the Kubernetes cluster. We recommend using a load balancer to direct traffic to each replica of Rancher in the cluster, in order to increase Rancher's availability. + +The Rancher server data is stored on etcd. This etcd database also runs on all three nodes, and requires an odd number of nodes so that it can always elect a leader with a majority of the etcd cluster. If the etcd database cannot elect a leader, etcd can fail, requiring the cluster to be restored from backup. + +For information on how Rancher works, regardless of the installation method, refer to the [architecture section.](../../../pages-for-subheaders/rancher-manager-architecture.md) + +### Recommended Architecture + +- DNS for Rancher should resolve to a layer 4 load balancer +- The Load Balancer should forward port TCP/80 and TCP/443 to all 3 nodes in the Kubernetes cluster. +- The Ingress controller will redirect HTTP to HTTPS and terminate SSL/TLS on port TCP/443. +- The Ingress controller will forward traffic to port TCP/80 on the pod in the Rancher deployment. + +
Kubernetes Rancher install with layer 4 load balancer, depicting SSL termination at ingress controllers
+ +![High-availability Kubernetes Installation of Rancher](/img/ha/rancher2ha.svg) +Kubernetes Rancher install with Layer 4 load balancer (TCP), depicting SSL termination at ingress controllers diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md new file mode 100644 index 00000000000..16bf5703c58 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md @@ -0,0 +1,140 @@ +--- +title: Setting up a High-availability K3s Kubernetes Cluster for Rancher +--- + + + + + +This section describes how to install a Kubernetes cluster according to the [best practices for the Rancher server environment.](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#environment-for-kubernetes-installations) + +For systems without direct internet access, refer to the air gap installation instructions. + +:::tip Single-node Installation Tip: + +In a single-node Kubernetes cluster, the Rancher server does not have high availability, which is important for running Rancher in production. However, installing Rancher on a single-node cluster can be useful if you want to save resources by using a single node in the short term, while preserving a high-availability migration path. + +To set up a single-node K3s cluster, run the Rancher server installation command on just one node instead of two nodes. + +In both single-node setups, Rancher can be installed with Helm on the Kubernetes cluster in the same way that it would be installed on any other cluster. + +::: + +## Prerequisites + +These instructions assume you have set up two nodes, a load balancer, a DNS record, and an external MySQL database as described in [this section.](../infrastructure-setup/ha-k3s-kubernetes-cluster.md) + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher Support Matrix](https://rancher.com/support-maintenance-terms/). + +To specify the K3s (Kubernetes) version, use the INSTALL_K3S_VERSION (e.g., `INSTALL_K3S_VERSION="v1.24.10+k3s1"`) environment variable when running the K3s installation script. + +## Installing Kubernetes + +### 1. Install Kubernetes and Set up the K3s Server + +When running the command to start the K3s Kubernetes API server, you will pass in an option to use the external datastore that you set up earlier. + +1. Connect to one of the Linux nodes that you have prepared to run the Rancher server. +1. On the Linux node, run this command to start the K3s server and connect it to the external datastore: + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" + ``` + + Where `` is the connection URI for your datastore. For example, `mysql://username:password@tcp(hostname:3306)/database-name` if you're using MySQL. Valid datastores include etcd, MySQL, PostgreSQL, or SQLite (default). + + :::note + + The datastore endpoint can also be passed in using the environment variable `$K3S_DATASTORE_ENDPOINT`. + + ::: + +1. Get main server node token: + ``` + cat /var/lib/rancher/k3s/server/token + ``` + +1. Run command on your second K3s server node: + ``` + curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION= sh -s - server \ + --datastore-endpoint="" \ + --token "" + ``` + +### 2. Confirm that K3s is Running + +To confirm that K3s has been set up successfully, run the following command on either of the K3s server nodes: +``` +sudo k3s kubectl get nodes +``` + +Then you should see two nodes with the master role: +``` +ubuntu@ip-172-31-60-194:~$ sudo k3s kubectl get nodes +NAME STATUS ROLES AGE VERSION +ip-172-31-60-194 Ready master 44m v1.17.2+k3s1 +ip-172-31-63-88 Ready master 6m8s v1.17.2+k3s1 +``` + +Then test the health of the cluster pods: +``` +sudo k3s kubectl get pods --all-namespaces +``` + +**Result:** You have successfully set up a K3s Kubernetes cluster. + +### 3. Save and Start Using the kubeconfig File + +When you installed K3s on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/k3s/k3s.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/k3s/k3s.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your load balancer, referring to port 6443. (The Kubernetes API server will be reached at port 6443, while the Rancher server will be reached at ports 80 and 443.) Here is an example `k3s.yaml`: + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your K3s cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/k3s.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/cluster-access/) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### 4. Check the Health of Your Cluster Pods + +Now that you have set up the `kubeconfig` file, you can use `kubectl` to access the cluster from your local machine. + +Check that all the required pods and containers are healthy are ready to continue: + +``` +ubuntu@ip-172-31-60-194:~$ sudo kubectl get pods --all-namespaces +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system metrics-server-6d684c7b5-bw59k 1/1 Running 0 8d +kube-system local-path-provisioner-58fb86bdfd-fmkvd 1/1 Running 0 8d +kube-system coredns-d798c9dd-ljjnf 1/1 Running 0 8d +``` + +**Result:** You have confirmed that you can access the cluster with `kubectl` and the K3s cluster is running successfully. Now the Rancher management server can be installed on the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md new file mode 100644 index 00000000000..1d1c0682ec1 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md @@ -0,0 +1,196 @@ +--- +title: Setting up a High-availability RKE Kubernetes Cluster +--- + + + + + +This section describes how to install a Kubernetes cluster. This cluster should be dedicated to run only the Rancher server. + +:::note + +Rancher can run on any Kubernetes cluster, included hosted Kubernetes solutions such as Amazon EKS. The below instructions represent only one possible way to install Kubernetes. + +::: + +For systems without direct internet access, refer to [Air Gap: Kubernetes install.](../../../pages-for-subheaders/air-gapped-helm-cli-install.md) + +:::tip Single-node Installation Tip: + +In a single-node Kubernetes cluster, the Rancher server does not have high availability, which is important for running Rancher in production. However, installing Rancher on a single-node cluster can be useful if you want to save resources by using a single node in the short term, while preserving a high-availability migration path. + +To set up a single-node RKE cluster, configure only one node in the `cluster.yml` . The single node should have all three roles: `etcd`, `controlplane`, and `worker`. + +In both single-node setups, Rancher can be installed with Helm on the Kubernetes cluster in the same way that it would be installed on any other cluster. + +::: + +## Installing Kubernetes + +### Required CLI Tools + +Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. + +Also install [RKE,](https://rancher.com/docs/rke/latest/en/installation/) the Rancher Kubernetes Engine, a Kubernetes distribution and command-line tool. + +### 1. Create the cluster configuration file + +In this section, you will create a Kubernetes cluster configuration file called `rancher-cluster.yml`. In a later step, when you set up the cluster with an RKE command, it will use this file to install Kubernetes on your nodes. + +Using the sample below as a guide, create the `rancher-cluster.yml` file. Replace the IP addresses in the `nodes` list with the IP address or DNS names of the 3 nodes you created. + +If your node has public and internal addresses, it is recommended to set the `internal_address:` so Kubernetes will use it for intra-cluster communication. Some services like AWS EC2 require setting the `internal_address:` if you want to use self-referencing security groups or firewalls. + +RKE will need to connect to each node over SSH, and it will look for a private key in the default location of `~/.ssh/id_rsa`. If your private key for a certain node is in a different location than the default, you will also need to configure the `ssh_key_path` option for that node. + +When choosing a Kubernetes version, be sure to first consult the [support matrix](https://rancher.com/support-matrix/) to find the highest version of Kubernetes that has been validated for your Rancher version. + +```yaml +nodes: + - address: 165.227.114.63 + internal_address: 172.16.22.12 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.116.167 + internal_address: 172.16.32.37 + user: ubuntu + role: [controlplane, worker, etcd] + - address: 165.227.127.226 + internal_address: 172.16.42.73 + user: ubuntu + role: [controlplane, worker, etcd] + +services: + etcd: + snapshot: true + creation: 6h + retention: 24h + +# Required for external TLS termination with +# ingress-nginx v0.22+ +ingress: + provider: nginx + options: + use-forwarded-headers: "true" + +kubernetes_version: v1.25.6-rancher4-1 +``` + +
Common RKE Nodes Options
+ +| Option | Required | Description | +| ------------------ | -------- | -------------------------------------------------------------------------------------- | +| `address` | yes | The public DNS or IP address | +| `user` | yes | A user that can run docker commands | +| `role` | yes | List of Kubernetes roles assigned to the node | +| `internal_address` | no | The private DNS or IP address for internal cluster traffic | +| `ssh_key_path` | no | Path to SSH private key used to authenticate to the node (defaults to `~/.ssh/id_rsa`) | + +:::note Advanced Configurations: + +RKE has many configuration options for customizing the install to suit your specific environment. + +Please see the [RKE Documentation](https://rancher.com/docs/rke/latest/en/config-options/) for the full list of options and capabilities. + +For tuning your etcd cluster for larger Rancher installations, see the [etcd settings guide](../../advanced-user-guides/tune-etcd-for-large-installs.md). + +For more information regarding Dockershim support, refer to [this page](../../../getting-started/installation-and-upgrade/installation-requirements/dockershim.md) + +::: + +### 2. Run RKE + +``` +rke up --config ./rancher-cluster.yml +``` + +When finished, it should end with the line: `Finished building Kubernetes cluster successfully`. + +### 3. Test Your Cluster + +This section describes how to set up your workspace so that you can interact with this cluster using the `kubectl` command-line tool. + +Assuming you have installed `kubectl`, you need to place the `kubeconfig` file in a location where `kubectl` can reach it. The `kubeconfig` file contains the credentials necessary to access your cluster with `kubectl`. + +When you ran `rke up`, RKE should have created a `kubeconfig` file named `kube_config_cluster.yml`. This file has the credentials for `kubectl` and `helm`. + +:::note + +If you have used a different file name from `rancher-cluster.yml`, then the kube config file will be named `kube_config_.yml`. + +::: + +Move this file to `$HOME/.kube/config`, or if you are working with multiple Kubernetes clusters, set the `KUBECONFIG` environmental variable to the path of `kube_config_cluster.yml`: + +``` +export KUBECONFIG=$(pwd)/kube_config_cluster.yml +``` + +Test your connectivity with `kubectl` and see if all your nodes are in `Ready` state: + +``` +kubectl get nodes + +NAME STATUS ROLES AGE VERSION +165.227.114.63 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.116.167 Ready controlplane,etcd,worker 11m v1.13.5 +165.227.127.226 Ready controlplane,etcd,worker 11m v1.13.5 +``` + +### 4. Check the Health of Your Cluster Pods + +Check that all the required pods and containers are healthy are ready to continue. + +- Pods are in `Running` or `Completed` state. +- `READY` column shows all the containers are running (i.e. `3/3`) for pods with `STATUS` `Running` +- Pods with `STATUS` `Completed` are run-once Jobs. For these pods `READY` should be `0/1`. + +``` +kubectl get pods --all-namespaces + +NAMESPACE NAME READY STATUS RESTARTS AGE +ingress-nginx nginx-ingress-controller-tnsn4 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-tw2ht 1/1 Running 0 30s +ingress-nginx nginx-ingress-controller-v874b 1/1 Running 0 30s +kube-system canal-jp4hz 3/3 Running 0 30s +kube-system canal-z2hg8 3/3 Running 0 30s +kube-system canal-z6kpw 3/3 Running 0 30s +kube-system kube-dns-7588d5b5f5-sf4vh 3/3 Running 0 30s +kube-system kube-dns-autoscaler-5db9bbb766-jz2k6 1/1 Running 0 30s +kube-system metrics-server-97bc649d5-4rl2q 1/1 Running 0 30s +kube-system rke-ingress-controller-deploy-job-bhzgm 0/1 Completed 0 30s +kube-system rke-kubedns-addon-deploy-job-gl7t4 0/1 Completed 0 30s +kube-system rke-metrics-addon-deploy-job-7ljkc 0/1 Completed 0 30s +kube-system rke-network-plugin-deploy-job-6pbgj 0/1 Completed 0 30s +``` + +This confirms that you have successfully installed a Kubernetes cluster that the Rancher server will run on. + +### 5. Save Your Files + +:::note Important: + +The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster. + +::: + +Save a copy of the following files in a secure location: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_cluster.yml`: The [Kubeconfig file](https://rancher.com/docs/rke/latest/en/kubeconfig/) for the cluster, this file contains credentials for full access to the cluster. +- `rancher-cluster.rkestate`: The [Kubernetes Cluster State file](https://rancher.com/docs/rke/latest/en/installation/#kubernetes-cluster-state), this file contains credentials for full access to the cluster.

_The Kubernetes Cluster State file is only created when using RKE v0.2.0 or higher._ + +:::note + +The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +::: + +### Issues or errors? + +See the [Troubleshooting](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) page. + + +### [Next: Install Rancher](../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md new file mode 100644 index 00000000000..7a743f731a2 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md @@ -0,0 +1,169 @@ +--- +title: Setting up a High-availability RKE2 Kubernetes Cluster for Rancher +--- + + + + + +This section describes how to install a Kubernetes cluster according to the [best practices for the Rancher server environment.](../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#environment-for-kubernetes-installations) + +## Prerequisites + +These instructions assume you have set up three nodes, a load balancer, and a DNS record, as described in [this section.](../infrastructure-setup/ha-rke2-kubernetes-cluster.md) + +Note that in order for RKE2 to work correctly with the load balancer, you need to set up two listeners: one for the supervisor on port 9345, and one for the Kubernetes API on port 6443. + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) To specify the RKE2 version, use the INSTALL_RKE2_VERSION environment variable when running the RKE2 installation script. + +## Installing Kubernetes + +### 1. Install Kubernetes and Set up the RKE2 Server + +RKE2 server runs with embedded etcd so you will not need to set up an external datastore to run in HA mode. + +On the first node, you should set up the configuration file with your own pre-shared secret as the token. The token argument can be set on startup. + +If you do not specify a pre-shared secret, RKE2 will generate one and place it at /var/lib/rancher/rke2/server/node-token. + +To avoid certificate errors with the fixed registration address, you should launch the server with the tls-san parameter set. This option adds an additional hostname or IP as a Subject Alternative Name in the server's TLS cert, and it can be specified as a list if you would like to access via both the IP and the hostname. + +First, you must create the directory where the RKE2 config file is going to be placed: + +``` +mkdir -p /etc/rancher/rke2/ +``` + +Next, create the RKE2 config file at `/etc/rancher/rke2/config.yaml` using the following example: + +``` +token: my-shared-secret +tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +``` +After that, you need to run the install command and enable and start rke2: + +``` +curl -sfL https://get.rke2.io | sh - +systemctl enable rke2-server.service +systemctl start rke2-server.service +``` +1. To join the rest of the nodes, you need to configure each additional node with the same shared token or the one generated automatically. Here is an example of the configuration file: + + token: my-shared-secret + server: https://:9345 + tls-san: + - my-kubernetes-domain.com + - another-kubernetes-domain.com +After that, you need to run the installer and enable, then start, rke2: + + curl -sfL https://get.rke2.io | sh - + systemctl enable rke2-server.service + systemctl start rke2-server.service + + +1. Repeat the same command on your third RKE2 server node. + +### 2. Confirm that RKE2 is Running + +Once you've launched the rke2 server process on all server nodes, ensure that the cluster has come up properly with + +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get nodes +You should see your server nodes in the Ready state. +``` + +Then test the health of the cluster pods: +``` +/var/lib/rancher/rke2/bin/kubectl \ + --kubeconfig /etc/rancher/rke2/rke2.yaml get pods --all-namespaces +``` + +**Result:** You have successfully set up a RKE2 Kubernetes cluster. + +### 3. Save and Start Using the kubeconfig File + +When you installed RKE2 on each Rancher server node, a `kubeconfig` file was created on the node at `/etc/rancher/rke2/rke2.yaml`. This file contains credentials for full access to the cluster, and you should save this file in a secure location. + +To use this `kubeconfig` file, + +1. Install [kubectl,](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) a Kubernetes command-line tool. +2. Copy the file at `/etc/rancher/rke2/rke2.yaml` and save it to the directory `~/.kube/config` on your local machine. +3. In the kubeconfig file, the `server` directive is defined as localhost. Configure the server as the DNS of your control-plane load balancer, on port 6443. (The RKE2 Kubernetes API Server uses port 6443, while the Rancher server will be served via the NGINX Ingress on ports 80 and 443.) Here is an example `rke2.yaml`: + +```yml +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: [CERTIFICATE-DATA] + server: [LOAD-BALANCER-DNS]:6443 # Edit this line + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + password: [PASSWORD] + username: admin +``` + +**Result:** You can now use `kubectl` to manage your RKE2 cluster. If you have more than one kubeconfig file, you can specify which one you want to use by passing in the path to the file when using `kubectl`: + +``` +kubectl --kubeconfig ~/.kube/config/rke2.yaml get pods --all-namespaces +``` + +For more information about the `kubeconfig` file, refer to the [RKE2 documentation](https://docs.rke2.io/cluster_access) or the [official Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/) about organizing cluster access using `kubeconfig` files. + +### 4. Check the Health of Your Cluster Pods + +Now that you have set up the `kubeconfig` file, you can use `kubectl` to access the cluster from your local machine. + +Check that all the required pods and containers are healthy are ready to continue: + +``` +/var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml get pods -A +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system cloud-controller-manager-rke2-server-1 1/1 Running 0 2m28s +kube-system cloud-controller-manager-rke2-server-2 1/1 Running 0 61s +kube-system cloud-controller-manager-rke2-server-3 1/1 Running 0 49s +kube-system etcd-rke2-server-1 1/1 Running 0 2m13s +kube-system etcd-rke2-server-2 1/1 Running 0 87s +kube-system etcd-rke2-server-3 1/1 Running 0 56s +kube-system helm-install-rke2-canal-hs6sx 0/1 Completed 0 2m17s +kube-system helm-install-rke2-coredns-xmzm8 0/1 Completed 0 2m17s +kube-system helm-install-rke2-ingress-nginx-flwnl 0/1 Completed 0 2m17s +kube-system helm-install-rke2-metrics-server-7sggn 0/1 Completed 0 2m17s +kube-system kube-apiserver-rke2-server-1 1/1 Running 0 116s +kube-system kube-apiserver-rke2-server-2 1/1 Running 0 66s +kube-system kube-apiserver-rke2-server-3 1/1 Running 0 48s +kube-system kube-controller-manager-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-controller-manager-rke2-server-2 1/1 Running 0 57s +kube-system kube-controller-manager-rke2-server-3 1/1 Running 0 42s +kube-system kube-proxy-rke2-server-1 1/1 Running 0 2m25s +kube-system kube-proxy-rke2-server-2 1/1 Running 0 59s +kube-system kube-proxy-rke2-server-3 1/1 Running 0 85s +kube-system kube-scheduler-rke2-server-1 1/1 Running 0 2m30s +kube-system kube-scheduler-rke2-server-2 1/1 Running 0 57s +kube-system kube-scheduler-rke2-server-3 1/1 Running 0 42s +kube-system rke2-canal-b9lvm 2/2 Running 0 91s +kube-system rke2-canal-khwp2 2/2 Running 0 2m5s +kube-system rke2-canal-swfmq 2/2 Running 0 105s +kube-system rke2-coredns-rke2-coredns-547d5499cb-6tvwb 1/1 Running 0 92s +kube-system rke2-coredns-rke2-coredns-547d5499cb-rdttj 1/1 Running 0 2m8s +kube-system rke2-coredns-rke2-coredns-autoscaler-65c9bb465d-85sq5 1/1 Running 0 2m8s +kube-system rke2-ingress-nginx-controller-69qxc 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-7hprp 1/1 Running 0 52s +kube-system rke2-ingress-nginx-controller-x658h 1/1 Running 0 52s +kube-system rke2-metrics-server-6564db4569-vdfkn 1/1 Running 0 66s +``` + +**Result:** You have confirmed that you can access the cluster with `kubectl` and the RKE2 cluster is running successfully. Now the Rancher management server can be installed on the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md new file mode 100644 index 00000000000..7c5e21424ea --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md @@ -0,0 +1,77 @@ +--- +title: Recommended Cluster Architecture +--- + + + + + +There are three roles that can be assigned to nodes: `etcd`, `controlplane` and `worker`. + +## Separating Worker Nodes from Nodes with Other Roles + +When designing your cluster(s), you have two options: + +* Use dedicated nodes for each role. This ensures resource availability for the components needed for the specified role. It also strictly isolates network traffic between each of the roles according to the [port requirements](../node-requirements-for-rancher-managed-clusters.md#networking-requirements). +* Assign the `etcd` and `controlplane` roles to the same nodes. These nodes must meet the hardware requirements for both roles. + +In either case, the `worker` role should not be used or added to nodes with the `etcd` or `controlplane` role. + +Therefore, each node should have one of the following role configurations: + + * `etcd` + * `controlplane` + * Both `etcd` and `controlplane` + * `worker` + +## Recommended Number of Nodes with Each Role + +The cluster should have: + +- At least three nodes with the role `etcd` to survive losing one node. Increase this count for higher node fault toleration, and spread them across (availability) zones to provide even better fault tolerance. +- At least two nodes with the role `controlplane` for master component high availability. +- At least two nodes with the role `worker` for workload rescheduling upon node failure. + +For more information on what each role is used for, refer to the [section on roles for nodes in Kubernetes.](roles-for-nodes-in-kubernetes.md) + + +### Number of Controlplane Nodes + +Adding more than one node with the `controlplane` role makes every master component highly available. + +### Number of etcd Nodes + +The number of nodes that you can lose at once while maintaining cluster availability is determined by the number of nodes assigned the `etcd` role. For a cluster with n members, the minimum is (n/2)+1. Therefore, we recommend creating an `etcd` node in 3 different availability zones within a region to survive the loss of one availability zone. If you use only two zones, you can only survive the loss of the zone where you don't lose the majority of nodes. + +| Nodes with `etcd` role | Majority | Failure Tolerance | +|--------------|------------|-------------------| +| 1 | 1 | 0 | +| 2 | 2 | 0 | +| 3 | 2 | **1** | +| 4 | 3 | 1 | +| 5 | 3 | **2** | +| 6 | 4 | 2 | +| 7 | 4 | **3** | +| 8 | 5 | 3 | +| 9 | 5 | **4** | + +References: + +* [Official etcd documentation on optimal etcd cluster size](https://etcd.io/docs/v3.4.0/faq/#what-is-failure-tolerance) +* [Official Kubernetes documentation on operating etcd clusters for Kubernetes](https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/) + +### Number of Worker Nodes + +Adding more than one node with the `worker` role will make sure your workloads can be rescheduled if a node fails. + +### Why Production Requirements are Different for the Rancher Cluster and the Clusters Running Your Applications + +You may have noticed that our [Kubernetes Install](../../../../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) instructions do not meet our definition of a production-ready cluster, as there are no dedicated nodes for the `worker` role. However, for your Rancher installation, this three node cluster is valid, because: + +* It allows one `etcd` node failure. +* It maintains multiple instances of the master components by having multiple `controlplane` nodes. +* No other workloads than Rancher itself should be created on this cluster. + +## References + +* [Kubernetes: Master Components](https://kubernetes.io/docs/concepts/overview/components/#master-components) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md new file mode 100644 index 00000000000..04eeb4466d4 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md @@ -0,0 +1,54 @@ +--- +title: Roles for Nodes in Kubernetes +--- + + + + + +This section describes the roles for etcd nodes, controlplane nodes, and worker nodes in Kubernetes, and how the roles work together in a cluster. + +This diagram is applicable to Kubernetes clusters [launched with Rancher using RKE.](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + +![Cluster diagram](/img/clusterdiagram.svg)
+Lines show the traffic flow between components. Colors are used purely for visual aid + +## etcd + +Nodes with the `etcd` role run etcd, which is a consistent and highly available key value store used as Kubernetes’ backing store for all cluster data. etcd replicates the data to each node. + +:::note + +Nodes with the `etcd` role are shown as `Unschedulable` in the UI, meaning no pods will be scheduled to these nodes by default. + +::: + +## controlplane + +Nodes with the `controlplane` role run the Kubernetes master components (excluding `etcd`, as it's a separate role). See [Kubernetes: Master Components](https://kubernetes.io/docs/concepts/overview/components/#master-components) for a detailed list of components. + +:::note + +Nodes with the `controlplane` role are shown as `Unschedulable` in the UI, meaning no pods will be scheduled to these nodes by default. + +::: + +### kube-apiserver + +The Kubernetes API server (`kube-apiserver`) scales horizontally. Each node with the role `controlplane` will be added to the NGINX proxy on the nodes with components that need to access the Kubernetes API server. This means that if a node becomes unreachable, the local NGINX proxy on the node will forward the request to another Kubernetes API server in the list. + +### kube-controller-manager + +The Kubernetes controller manager uses leader election using an endpoint in Kubernetes. One instance of the `kube-controller-manager` will create an entry in the Kubernetes endpoints and updates that entry in a configured interval. Other instances will see an active leader and wait for that entry to expire (for example, when a node is unresponsive). + +### kube-scheduler + +The Kubernetes scheduler uses leader election using an endpoint in Kubernetes. One instance of the `kube-scheduler` will create an entry in the Kubernetes endpoints and updates that entry in a configured interval. Other instances will see an active leader and wait for that entry to expire (for example, when a node is unresponsive). + +## worker + +Nodes with the `worker` role run the Kubernetes node components. See [Kubernetes: Node Components](https://kubernetes.io/docs/concepts/overview/components/#node-components) for a detailed list of components. + +## References + +* [Kubernetes: Node Components](https://kubernetes.io/docs/concepts/overview/components/#node-components) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md new file mode 100644 index 00000000000..3f0575a2f76 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md @@ -0,0 +1,133 @@ +--- +title: Node Requirements for Rancher Managed Clusters +--- + + + + + +This page describes the requirements for the Rancher managed Kubernetes clusters where your apps and services will be installed. These downstream clusters should be separate from the three-node cluster running Rancher. + +:::note + +If Rancher is installed on a high-availability Kubernetes cluster, the Rancher server three-node cluster and downstream clusters have different requirements. For Rancher installation requirements, refer to the node requirements in the [installation section.](../../../pages-for-subheaders/installation-requirements.md) + +::: + +## Operating Systems and Container Runtime Requirements + +Rancher should work with any modern Linux distribution and any modern Docker version. Linux is required for the etcd and controlplane nodes of all downstream clusters. Worker nodes may run Linux or [Windows Server.](#windows-nodes) + +For details on which OS and Docker versions were tested with each Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) + +All supported operating systems are 64-bit x86. + +If you plan to use ARM64, see [Running on ARM64 (Experimental).](../../advanced-user-guides/enable-experimental-features/rancher-on-arm64.md) + +For information on how to install Docker, refer to the official [Docker documentation.](https://docs.docker.com/) + +### Oracle Linux and RHEL Derived Linux Nodes + +Some distributions of Linux derived from RHEL, including Oracle Linux, may have default firewall rules that block communication with Helm. We recommend disabling firewalld. For Kubernetes 1.19, firewalld must be turned off. + +:::note + +In RHEL 8.4, two extra services are included on the NetworkManager: `nm-cloud-setup.service` and `nm-cloud-setup.timer`. These services add a routing table that interferes with the CNI plugin's configuration. If these services are enabled, you must disable them using the command below, and then reboot the node to restore connectivity: + + ``` + systemctl disable nm-cloud-setup.service nm-cloud-setup.timer + reboot + ``` + +::: + +### SUSE Linux Nodes + +SUSE Linux may have a firewall that blocks all ports by default. In that situation, follow [these steps](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#opening-suse-linux-ports) to open the ports needed for adding a host to a custom cluster. + +### Flatcar Container Linux Nodes + +When [Launching Kubernetes with Rancher](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) using Flatcar Container Linux nodes, it is required to use the following configuration in the [Cluster Config File](../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: canal + options: + canal_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +```yaml +rancher_kubernetes_engine_config: + network: + plugin: calico + options: + calico_flex_volume_plugin_dir: /opt/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + flannel_backend_type: vxlan + + services: + kube-controller: + extra_args: + flex-volume-plugin-dir: /opt/kubernetes/kubelet-plugins/volume/exec/ +``` + + + + +It is also required to enable the Docker service, you can enable the Docker service using the following command: + +``` +systemctl enable docker.service +``` + +The Docker service is enabled automatically when using [Node Drivers](../../../pages-for-subheaders/about-provisioning-drivers.md#node-drivers). + +### Windows Nodes + +Nodes with Windows Server must run Docker Enterprise Edition. + +Windows nodes can be used for worker nodes only. See [Configuring Custom Clusters for Windows](../../../pages-for-subheaders/use-windows-clusters.md) + +## Hardware Requirements + +The hardware requirements for nodes with the `worker` role mostly depend on your workloads. The minimum to run the Kubernetes node components is 1 CPU (core) and 1GB of memory. + +Regarding CPU and memory, it is recommended that the different planes of Kubernetes clusters (etcd, controlplane, and workers) should be hosted on different nodes so that they can scale separately from each other. + +For hardware recommendations for large Kubernetes clusters, refer to the official Kubernetes documentation on [building large clusters.](https://kubernetes.io/docs/setup/best-practices/cluster-large/) + +For hardware recommendations for etcd clusters in production, refer to the official [etcd documentation.](https://etcd.io/docs/v3.4.0/op-guide/hardware/) + +## Networking Requirements + +For a production cluster, we recommend that you restrict traffic by opening only the ports defined in the port requirements below. + +The ports required to be open are different depending on how the user cluster is launched. Each of the sections below list the ports that need to be opened for different [cluster creation options](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). + +For a breakdown of the port requirements for etcd nodes, controlplane nodes, and worker nodes in a Kubernetes cluster, refer to the [port requirements for the Rancher Kubernetes Engine.](https://rancher.com/docs/rke/latest/en/os/#ports) + +Details on which ports are used in each situation are found under [Downstream Cluster Port Requirements](../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#downstream-kubernetes-cluster-nodes). + +:::caution + +You should never register a node with the same hostname or IP address as an existing node. Doing so causes RKE to prevent the node from joining, and provisioning to hang. This can occur for both node driver and custom clusters. If a node must reuse a hostname or IP of an existing node, you must set the `hostname_override` [RKE option](https://rke.docs.rancher.com/config-options/nodes#overriding-the-hostname) before registering the node, so that it can join correctly. + +::: + +## Optional: Security Considerations + +If you want to provision a Kubernetes cluster that is compliant with the CIS (Center for Internet Security) Kubernetes Benchmark, we recommend to following our hardening guide to configure your nodes before installing Kubernetes. + +For more information on the hardening guide and details on which version of the guide corresponds to your Rancher and Kubernetes versions, refer to the [security section.](../../../pages-for-subheaders/rancher-security.md#rancher-hardening-guide) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md new file mode 100644 index 00000000000..02e2604884f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md @@ -0,0 +1,286 @@ +--- +title: Registering Existing Clusters +--- + + + + + +The cluster registration feature replaced the feature to import clusters. + +The control that Rancher has to manage a registered cluster depends on the type of cluster. For details, see [Management Capabilities for Registered Clusters.](#management-capabilities-for-registered-clusters) + + +## Prerequisites + +### Kubernetes Node Roles + +Registered RKE Kubernetes clusters must have all three node roles - etcd, controlplane and worker. A cluster with only controlplane components cannot be registered in Rancher. + +For more information on RKE node roles, see the [best practices.](../../../pages-for-subheaders/checklist-for-production-ready-clusters.md#cluster-architecture) + +### Permissions + +To register a cluster in Rancher, you must have `cluster-admin` privileges within that cluster. If you don't, grant these privileges to your user by running: + +```plain +kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole cluster-admin \ + --user [USER_ACCOUNT] +``` + +Since, by default, Google Kubernetes Engine (GKE) doesn't grant the `cluster-admin` role, you must run these commands on GKE clusters before you can register them. To learn more about role-based access control for GKE, please see [the official Google documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control). + +### EKS, AKS and GKE Clusters + +To successfully import them into or provision them from Rancher, Elastic Kubernetes Service (EKS), Azure Kubernetes Service (AKS), and Google Kubernetes Engine (GKE) clusters must have at least one managed node group. In addition, GKE Autopilot clusters aren't supported. See [Compare GKE Autopilot and Standard](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison) for more information about the difference between GKE modes. + +## Registering a Cluster + +1. Click **☰ > Cluster Management**. +2. On the **Clusters** page, **Import Existing**. +3. Choose the type of cluster. +4. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +5. If you are importing a generic Kubernetes cluster in Rancher, perform the following steps for setup:
+ a. Click **Agent Environment Variables** under **Cluster Options** to set environment variables for [rancher cluster agent](../launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. If rancher agent requires use of proxy to communicate with Rancher server, `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables can be set using agent environment variables.
+ b. Enable Project Network Isolation to ensure the cluster supports Kubernetes `NetworkPolicy` resources. Users can select the **Project Network Isolation** option under the **Advanced Options** dropdown to do so. +6. Click **Create**. +7. The prerequisite for `cluster-admin` privileges is shown (see **Prerequisites** above), including an example command to fulfil the prerequisite. +8. Copy the `kubectl` command to your clipboard and run it on a node where kubeconfig is configured to point to the cluster you want to import. If you are unsure it is configured correctly, run `kubectl get nodes` to verify before running the command shown in Rancher. +9. If you are using self-signed certificates, you will receive the message `certificate signed by unknown authority`. To work around this validation, copy the command starting with `curl` displayed in Rancher to your clipboard. Then run the command on a node where kubeconfig is configured to point to the cluster you want to import. +10. When you finish running the command(s) on your node, click **Done**. + +**Result:** + +- Your cluster is registered and assigned a state of **Pending**. Rancher is deploying resources to manage your cluster. +- You can access your cluster after its state is updated to **Active**. +- **Active** clusters are assigned two Projects: `Default` (containing the namespace `default`) and `System` (containing the namespaces `cattle-system`, `ingress-nginx`, `kube-public` and `kube-system`, if present). + + +:::note + +You can not re-register a cluster that is currently active in a Rancher setup. + +::: + +### Configuring an Imported EKS, AKS or GKE Cluster with Terraform + +You should define **only** the minimum fields that Rancher requires when importing an EKS, AKS or GKE cluster with Terraform. This is important as Rancher will overwrite what was in the cluster configuration with any config that the user has provided. + +:::caution + +Even a small difference between the current cluster and a user-provided config could have unexpected results. + +::: + +The minimum config fields required by Rancher to import EKS clusters with Terraform using `eks_config_v2` are as follows: + +- cloud_credential_id +- name +- region +- imported (this field should always be set to `true` for imported clusters) + +Example YAML configuration for imported EKS clusters: + +``` +resource "rancher2_cluster" "my-eks-to-import" { + name = "my-eks-to-import" + description = "Terraform EKS Cluster" + eks_config_v2 { + cloud_credential_id = rancher2_cloud_credential.aws.id + name = var.aws_eks_name + region = var.aws_region + imported = true + } +} +``` + +You can find additional examples for other cloud providers in the [Rancher2 Terraform Provider documentation](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster). + +## Management Capabilities for Registered Clusters + +The control that Rancher has to manage a registered cluster depends on the type of cluster. + +- [Features for All Registered Clusters](#features-for-all-registered-clusters) +- [Additional Features for Registered RKE2 and K3s Clusters](#additional-features-for-registered-rke2-and-k3s-clusters) +- [Additional Features for Registered EKS, AKS and GKE Clusters](#additional-features-for-registered-eks-aks-and-gke-clusters) + +### Features for All Registered Clusters + +After registering a cluster, the cluster owner can: + +- [Manage cluster access](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) through role-based access control +- Enable [monitoring, alerts and notifiers](../../../pages-for-subheaders/monitoring-and-alerting.md) +- Enable [logging](../../../pages-for-subheaders/logging.md) +- Enable [Istio](../../../pages-for-subheaders/istio.md) +- Manage projects and workloads + +### Additional Features for Registered RKE2 and K3s Clusters + +[K3s](https://rancher.com/docs/k3s/latest/en/) is a lightweight, fully compliant Kubernetes distribution for edge installations. +[RKE2](https://docs.rke2.io) is Rancher's next-generation Kubernetes distribution for datacenter and cloud installations. + +When an RKE2 or K3s cluster is registered in Rancher, Rancher will recognize it. The Rancher UI will expose the features for [all registered clusters,](#features-for-all-registered-clusters) in addition to the following features for editing and upgrading the cluster: + +- The ability to [upgrade the Kubernetes version](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md) + :::danger + + After a cluster has been imported into Rancher, upgrades should be performed using Rancher. Upgrading an imported cluster outside of Rancher is **not** supported. + + ::: +- The ability to configure the maximum number of nodes that will be upgraded concurrently +- The ability to see a read-only version of the cluster's configuration arguments and environment variables used to launch each node in the cluster + +### Additional Features for Registered EKS, AKS, and GKE Clusters + +Rancher handles registered EKS, AKS, or GKE clusters similarly to clusters created in Rancher. However, Rancher doesn't destroy registered clusters when you delete them through the Rancher UI. + +When you create an EKS, AKS, or GKE cluster in Rancher, then delete it, Rancher destroys the cluster. When you delete a registered cluster through Rancher, the Rancher server _disconnects_ from the cluster. The cluster remains live, although it's no longer in Rancher. You can still access the deregistered cluster in the same way you did before you registered it. + +See [Cluster Management Capabilities by Cluster Type](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) for more information about what features are available for managing registered clusters. + +## Configuring RKE2 and K3s Cluster Upgrades + +:::tip + +It is a Kubernetes best practice to back up the cluster before upgrading. When upgrading a high-availability K3s cluster with an external database, back up the database in whichever way is recommended by the relational database provider. + +::: + +The **concurrency** is the maximum number of nodes that are permitted to be unavailable during an upgrade. If number of unavailable nodes is larger than the **concurrency,** the upgrade will fail. If an upgrade fails, you may need to repair or remove failed nodes before the upgrade can succeed. + +- **Controlplane concurrency:** The maximum number of server nodes to upgrade at a single time; also the maximum unavailable server nodes +- **Worker concurrency:** The maximum number worker nodes to upgrade at the same time; also the maximum unavailable worker nodes + +In the RKE2 and K3s documentation, controlplane nodes are called server nodes. These nodes run the Kubernetes master, which maintains the desired state of the cluster. By default, these controlplane nodes have the capability to have workloads scheduled to them by default. + +Also in the RKE2 and K3s documentation, nodes with the worker role are called agent nodes. Any workloads or pods that are deployed in the cluster can be scheduled to these nodes by default. + +## Debug Logging and Troubleshooting for Registered RKE2 and K3s Clusters + +Nodes are upgraded by the system upgrade controller running in the downstream cluster. Based on the cluster configuration, Rancher deploys two [plans](https://github.com/rancher/system-upgrade-controller#example-upgrade-plan) to upgrade nodes: one for controlplane nodes and one for workers. The system upgrade controller follows the plans and upgrades the nodes. + +To enable debug logging on the system upgrade controller deployment, edit the [configmap](https://github.com/rancher/system-upgrade-controller/blob/50a4c8975543d75f1d76a8290001d87dc298bdb4/manifests/system-upgrade-controller.yaml#L32) to set the debug environment variable to true. Then restart the `system-upgrade-controller` pod. + +Logs created by the `system-upgrade-controller` can be viewed by running this command: + +``` +kubectl logs -n cattle-system system-upgrade-controller +``` + +The current status of the plans can be viewed with this command: + +``` +kubectl get plans -A -o yaml +``` + +If the cluster becomes stuck in upgrading, restart the `system-upgrade-controller`. + +To prevent issues when upgrading, the [Kubernetes upgrade best practices](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/) should be followed. + +## Authorized Cluster Endpoint Support for RKE2 and K3s Clusters + +_Available as of v2.6.3_ + +Authorized Cluster Endpoint (ACE) support has been added for registered RKE2 and K3s clusters. This support includes manual steps you will perform on the downstream cluster to enable the ACE. For additional information on the authorized cluster endpoint, click [here](../manage-clusters/access-clusters/authorized-cluster-endpoint.md). + +:::note Notes: + +- These steps only need to be performed on the control plane nodes of the downstream cluster. You must configure each control plane node individually. + +- The following steps will work on both RKE2 and K3s clusters registered in v2.6.x as well as those registered (or imported) from a previous version of Rancher with an upgrade to v2.6.x. + +- These steps will alter the configuration of the downstream RKE2 and K3s clusters and deploy the `kube-api-authn-webhook`. If a future implementation of the ACE requires an update to the `kube-api-authn-webhook`, then this would also have to be done manually. For more information on this webhook, click [here](../manage-clusters/access-clusters/authorized-cluster-endpoint.md#about-the-kube-api-auth-authentication-webhook). + +::: + +###### **Manual steps to be taken on the control plane of each downstream cluster to enable ACE:** + +1. Create a file at `/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml` with the following contents: + ```yaml + apiVersion: v1 + kind: Config + clusters: + - name: Default + cluster: + insecure-skip-tls-verify: true + server: http://127.0.0.1:6440/v1/authenticate + users: + - name: Default + user: + insecure-skip-tls-verify: true + current-context: webhook + contexts: + - name: webhook + context: + user: Default + cluster: Default + ``` + +2. Add the following to the config file (or create one if it doesn’t exist); note that the default location is `/etc/rancher/{rke2,k3s}/config.yaml`: + ```yaml + kube-apiserver-arg: + - authentication-token-webhook-config-file=/var/lib/rancher/{rke2,k3s}/kube-api-authn-webhook.yaml + ``` + +3. Run the following commands: + + sudo systemctl stop {rke2,k3s}-server + sudo systemctl start {rke2,k3s}-server + +4. Finally, you **must** go back to the Rancher UI and edit the imported cluster there to complete the ACE enablement. Click on **⋮ > Edit Config**, then click the **Networking** tab under Cluster Configuration. Finally, click the **Enabled** button for **Authorized Endpoint**. Once the ACE is enabled, you then have the option of entering a fully qualified domain name (FQDN) and certificate information. + +:::note + +The FQDN field is optional, and if one is entered, it should point to the downstream cluster. Certificate information is only needed if there is a load balancer in front of the downstream cluster that is using an untrusted certificate. If you have a valid certificate, then nothing needs to be added to the CA Certificates field. + +::: + +## Annotating Registered Clusters + +For all types of registered Kubernetes clusters except for RKE2 and K3s Kubernetes clusters, Rancher doesn't have any information about how the cluster is provisioned or configured. + +Therefore, when Rancher registers a cluster, it assumes that several capabilities are disabled by default. Rancher assumes this in order to avoid exposing UI options to the user even when the capabilities are not enabled in the registered cluster. + +However, if the cluster has a certain capability, such as the ability to use a pod security policy, a user of that cluster might still want to select pod security policies for the cluster in the Rancher UI. In order to do that, the user will need to manually indicate to Rancher that pod security policies are enabled for the cluster. + +By annotating a registered cluster, it is possible to indicate to Rancher that a cluster was given a pod security policy, or another capability, outside of Rancher. + +This example annotation indicates that a pod security policy is enabled: + +```json +"capabilities.cattle.io/pspEnabled": "true" +``` + +The following annotation indicates Ingress capabilities. Note that the values of non-primitive objects need to be JSON encoded, with quotations escaped. + +```json +"capabilities.cattle.io/ingressCapabilities": "[ + { + "customDefaultBackend":true, + "ingressProvider":"asdf" + } +]" +``` + +These capabilities can be annotated for the cluster: + +- `ingressCapabilities` +- `loadBalancerCapabilities` +- `nodePoolScalingSupported` +- `nodePortRange` +- `pspEnabled` +- `taintSupport` + +All the capabilities and their type definitions can be viewed in the Rancher API view, at `[Rancher Server URL]/v3/schemas/capabilities`. + +To annotate a registered cluster, + +1. Click **☰ > Cluster Management**. +2. On the **Clusters** page, go to the custom cluster you want to annotate and click **⋮ > Edit Config**. +3. Expand the **Labels & Annotations** section. +4. Click **Add Annotation**. +5. Add an annotation to the cluster with the format `capabilities/: ` where `value` is the cluster capability that will be overridden by the annotation. In this scenario, Rancher is not aware of any capabilities of the cluster until you add the annotation. +6. Click **Save**. + +**Result:** The annotation does not give the capabilities to the cluster, but it does indicate to Rancher that the cluster has those capabilities. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md new file mode 100644 index 00000000000..5943ee01710 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md @@ -0,0 +1,164 @@ +--- +title: Setting up the Amazon Cloud Provider +weight: 1 +--- + + + + + +When using the `Amazon` cloud provider, you can leverage the following capabilities: + +- **Load Balancers:** Launches an AWS Elastic Load Balancer (ELB) when choosing `Layer-4 Load Balancer` in **Port Mapping** or when launching a `Service` with `type: LoadBalancer`. +- **Persistent Volumes**: Allows you to use AWS Elastic Block Stores (EBS) for persistent volumes. + +See [cloud-provider-aws README](https://kubernetes.github.io/cloud-provider-aws/) for all information regarding the Amazon cloud provider. + +To set up the Amazon cloud provider, + +1. [Create an IAM role and attach to the instances](#1-create-an-iam-role-and-attach-to-the-instances) +2. [Configure the ClusterID](#2-configure-the-clusterid) + +:::note Important: + +Starting with Kubernetes 1.23, you have to deactivate the `CSIMigrationAWS` feature gate in order to use the in-tree AWS cloud provider. You can do this by setting `feature-gates=CSIMigrationAWS=false` as an additional argument for the cluster's Kubelet, Controller Manager, API Server and Scheduler in the advanced cluster configuration. + +::: + +### 1. Create an IAM Role and attach to the instances + +All nodes added to the cluster must be able to interact with EC2 so that they can create and remove resources. You can enable this interaction by using an IAM role attached to the instance. See [Amazon documentation: Creating an IAM Role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#create-iam-role) how to create an IAM role. There are two example policies: + +* The first policy is for the nodes with the `controlplane` role. These nodes have to be able to create/remove EC2 resources. The following IAM policy is an example, please remove any unneeded permissions for your use case. +* The second policy is for the nodes with the `etcd` or `worker` role. These nodes only have to be able to retrieve information from EC2. + +While creating an [Amazon EC2 cluster](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md), you must fill in the **IAM Instance Profile Name** (not ARN) of the created IAM role when creating the **Node Template**. + +While creating a [Custom cluster](../../../../pages-for-subheaders/use-existing-nodes.md), you must manually attach the IAM role to the instance(s). + +IAM Policy for nodes with the `controlplane` role: + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } +] +} +``` + +IAM policy for nodes with the `etcd` or `worker` role: + +```json +{ +"Version": "2012-10-17", +"Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } +] +} +``` + +### 2. Configure the ClusterID + +The following resources need to tagged with a `ClusterID`: + +- **Nodes**: All hosts added in Rancher. +- **Subnet**: The subnet used for your cluster. +- **Security Group**: The security group used for your cluster. + +:::note + +Do not tag multiple security groups. Tagging multiple groups generates an error when creating an Elastic Load Balancer (ELB). + +::: + +When you create an [Amazon EC2 Cluster](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md), the `ClusterID` is automatically configured for the created nodes. Other resources still need to be tagged manually. + +Use the following tag: + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `owned` + +`CLUSTERID` can be any string you like, as long as it is equal across all tags set. + +Setting the value of the tag to `owned` tells the cluster that all resources with this tag are owned and managed by this cluster. If you share resources between clusters, you can change the tag to: + +**Key** = `kubernetes.io/cluster/CLUSTERID` **Value** = `shared`. + +### Using Amazon Elastic Container Registry (ECR) + +The kubelet component has the ability to automatically obtain ECR credentials, when the IAM profile mentioned in [Create an IAM Role and attach to the instances](#1-create-an-iam-role-and-attach-to-the-instances) is attached to the instance(s). When using a Kubernetes version older than v1.15.0, the Amazon cloud provider needs be configured in the cluster. Starting with Kubernetes version v1.15.0, the kubelet can obtain ECR credentials without having the Amazon cloud provider configured in the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md new file mode 100644 index 00000000000..8720aa0760e --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md @@ -0,0 +1,115 @@ +--- +title: Setting up the Azure Cloud Provider +--- + + + + + +When using the `Azure` cloud provider, you can leverage the following capabilities: + +- **Load Balancers:** Launches an Azure Load Balancer within a specific Network Security Group. + +- **Persistent Volumes:** Supports using Azure Blob disks and Azure Managed Disks with standard and premium storage accounts. + +- **Network Storage:** Support Azure Files via CIFS mounts. + +The following account types are not supported for Azure Subscriptions: + +- Single tenant accounts (i.e. accounts with no subscriptions). +- Multi-subscription accounts. + +## Prerequisites for RKE and RKE2 + +To set up the Azure cloud provider for both RKE and RKE2, the following credentials need to be configured: + +1. [Set up the Azure Tenant ID](#1-set-up-the-azure-tenant-id) +2. [Set up the Azure Client ID and Azure Client Secret](#2-set-up-the-azure-client-id-and-azure-client-secret) +3. [Configure App Registration Permissions](#3-configure-app-registration-permissions) +4. [Set up Azure Network Security Group Name](#4-set-up-azure-network-security-group-name) + +### 1. Set up the Azure Tenant ID + +Visit [Azure portal](https://portal.azure.com), login and go to **Azure Active Directory** and select **Properties**. Your **Directory ID** is your **Tenant ID** (tenantID). + +If you want to use the Azure CLI, you can run the command `az account show` to get the information. + +### 2. Set up the Azure Client ID and Azure Client Secret + +Visit [Azure portal](https://portal.azure.com), login and follow the steps below to create an **App Registration** and the corresponding **Azure Client ID** (aadClientId) and **Azure Client Secret** (aadClientSecret). + +1. Select **Azure Active Directory**. +1. Select **App registrations**. +1. Select **New application registration**. +1. Choose a **Name**, select `Web app / API` as **Application Type** and a **Sign-on URL** which can be anything in this case. +1. Select **Create**. + +In the **App registrations** view, you should see your created App registration. The value shown in the column **APPLICATION ID** is what you need to use as **Azure Client ID**. + +The next step is to generate the **Azure Client Secret**: + +1. Open your created App registration. +1. In the **Settings** view, open **Keys**. +1. Enter a **Key description**, select an expiration time and select **Save**. +1. The generated value shown in the column **Value** is what you need to use as **Azure Client Secret**. This value will only be shown once. + +### 3. Configure App Registration Permissions + +The last thing you will need to do, is assign the appropriate permissions to your App registration. + +1. Go to **More services**, search for **Subscriptions** and open it. +1. Open **Access control (IAM)**. +1. Select **Add**. +1. For **Role**, select `Contributor`. +1. For **Select**, select your created App registration name. +1. Select **Save**. + +### 4. Set up Azure Network Security Group Name + +A custom Azure Network Security Group (securityGroupName) is needed to allow Azure Load Balancers to work. + +If you provision hosts using Rancher Machine Azure driver, you will need to edit them manually to assign them to this Network Security Group. + +You should already assign custom hosts to this Network Security Group during provisioning. + +Only hosts expected to be load balancer back ends need to be in this group. + +## RKE2 Cluster Set-up in Rancher + +1. Choose "Azure" from the Cloud Provider drop-down in the Cluster Configuration section. + +1. * Supply the Cloud Provider Configuration. Note that Rancher will automatically create a new Network Security Group, Resource Group, Availability Set, Subnet, and Virtual Network. If you already have some or all of these created, you will need to specify them before creating the cluster. + * You can click on "Show Advanced" to see more of these automatically generated names and update them if + necessary. Your Cloud Provider Configuration **must** match the fields in the Machine Pools section. If you have multiple pools, they must all use the same Resource Group, Availability Set, Subnet, Virtual Network, and Network Security Group. + * An example is provided below. You will modify it as needed. + +
+ Example Cloud Provider Config + + ```yaml + { + "cloud":"AzurePublicCloud", + "tenantId": "YOUR TENANTID HERE", + "aadClientId": "YOUR AADCLIENTID HERE", + "aadClientSecret": "YOUR AADCLIENTSECRET HERE", + "subscriptionId": "YOUR SUBSCRIPTIONID HERE", + "resourceGroup": "docker-machine", + "location": "westus", + "subnetName": "docker-machine", + "securityGroupName": "rancher-managed-KA4jV9V2", + "securityGroupResourceGroup": "docker-machine", + "vnetName": "docker-machine-vnet", + "vnetResourceGroup": "docker-machine", + "primaryAvailabilitySetName": "docker-machine", + "routeTableResourceGroup": "docker-machine", + "cloudProviderBackoff": false, + "useManagedIdentityExtension": false, + "useInstanceMetadata": true + } + ``` + +
+ +1. Under the **Cluster Configuration > Advanced** section, click **Add** under **Additional Controller Manager Args** and add this flag: `--configure-cloud-routes=false` + +1. Click the **Create** button to submit the form and create the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md new file mode 100644 index 00000000000..aba2b7a587c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md @@ -0,0 +1,23 @@ +--- +title: How to Configure an In-tree vSphere Cloud Provider +--- + + + + + +To set up the in-tree vSphere cloud provider, follow these steps while creating the vSphere cluster in Rancher: + +1. Set **Cloud Provider** option to `Custom` or `Custom (In-Tree)`. +1. Click on **Edit as YAML** +1. Insert the following structure to the pre-populated cluster YAML. This structure must be placed under `rancher_kubernetes_engine_config`. Note that the `name` *must* be set to `vsphere`. + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: vsphere + vsphereCloudProvider: + [Insert provider configuration] + ``` + +Rancher uses RKE (the Rancher Kubernetes Engine) to provision Kubernetes clusters. Refer to the [vSphere configuration reference in the RKE documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere/config-reference/) for details about the properties of the `vsphereCloudProvider` directive. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md new file mode 100644 index 00000000000..2d78b61b57f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md @@ -0,0 +1,68 @@ +--- +title: How to Configure an Out-of-tree vSphere Cloud Provider +--- + + + + + +Kubernetes is moving away from maintaining cloud providers in-tree. vSphere has an out-of-tree cloud provider that can be used by installing the vSphere cloud provider and cloud storage plugins. + +This page covers how to install the Cloud Provider Interface (CPI) and Cloud Storage Interface (CSI) plugins after setting up a cluster. + +## Prerequisites + +The vSphere versions supported: + +* 6.7u3 +* 7.0u1 or higher. + +The Kubernetes version must be 1.19 or higher. + +Using the vSphere out-of-tree cloud provider requires Linux nodes and is not supported on Windows. + +## Installation + +The Cloud Provider Interface (CPI) should be installed first before installing the Cloud Storage Interface (CSI). + +### 1. Create a vSphere cluster + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **VMware vSphere** or **Custom**. +1. On the **Basics** tab in the **Cluster Configuration** section, set the **Cloud Provider** to **vSphere**. +1. In the **Add-On Config** tab, the vSphere Cloud Provider (CPI) and Storage Provider (CSI) options. +1. Finish creating your cluster. + +### 2. Install the CPI plugin + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CPI plugin will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CPI**. +1. Fill out the required vCenter details. +1. vSphere CPI initializes all nodes with ProviderID which is needed by the vSphere CSI driver. Check if all nodes are initialized with the ProviderID before installing CSI driver with the following command: + + ``` + kubectl describe nodes | grep "ProviderID" + ``` + +### 3. Installing the CSI plugin + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CSI plugin will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CSI**. +1. Click **Install**. +1. Fill out the required vCenter details. On the **Features** tab, set **Enable CSI Migration** to **false**. +3. On the **Storage** tab, fill out the details for the StorageClass. This chart creates a StorageClass with the `csi.vsphere.vmware.com` as the provisioner. +1. Click **Install**. + + +## Using the CSI driver for provisioning volumes + +The CSI chart by default creates a storageClass. + +If that option was not selected while launching the chart, create a storageClass with the `csi.vsphere.vmware.com` as the provisioner. + +All volumes provisioned using this StorageClass will get provisioned by the CSI driver. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md new file mode 100644 index 00000000000..ffd1cf6bdb4 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md @@ -0,0 +1,61 @@ +--- +title: Setting up the Google Compute Engine Cloud Provider +--- + + + + + +In this section, you'll learn how to enable the Google Compute Engine (GCE) cloud provider for custom clusters in Rancher. A custom cluster is one in which Rancher installs Kubernetes on existing nodes. + +The official Kubernetes documentation for the GCE cloud provider is [here.](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#gce) + +:::note Prerequisites: + +The service account of `Identity and API` access on GCE needs the `Computer Admin` permission. + +::: + +If you are using Calico, + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the custom cluster and click **⋮ > Edit YAML.* Enter the following configuration: + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + network: + options: + calico_cloud_provider: "gce" + plugin: "calico" + ``` + +If you are using Canal or Flannel, + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the custom cluster and click **⋮ > Edit YAML.* Enter the following configuration: + + ```yaml + rancher_kubernetes_engine_config: + cloud_provider: + name: gce + customCloudProvider: |- + [Global] + project-id= + network-name= + subnetwork-name= + node-instance-prefix= + node-tags= + services: + kube_controller: + extra_args: + configure-cloud-routes: true # we need to allow the cloud provider configure the routes for the hosts + ``` \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md new file mode 100644 index 00000000000..d302213118a --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree.md @@ -0,0 +1,112 @@ +--- +title: Migrating vSphere In-tree Volumes to Out-of-tree +--- + + + + + +Kubernetes is moving away from maintaining cloud providers in-tree. vSphere has an out-of-tree cloud provider that can be used by installing the vSphere cloud provider and cloud storage plugins. + +This page covers how to migrate from the in-tree vSphere cloud provider to out-of-tree, and manage the existing VMs post migration. + +It follows the steps provided in the official [vSphere migration documentation](https://vsphere-csi-driver.sigs.k8s.io/features/vsphere_csi_migration.html) and provides the steps to be performed in Rancher. + +### Cloud-config Format Limitation + +Existing volumes that were provisioned using the following cloud-config format won't get migrated due to an existing bug in vSphere Cloud Storage Interface (CSI). + +If the cloud-config has this format for datastore and resource pool path, vsphere CSI driver cannot recognize it: + +```yaml +default-datastore: /datastore/ +resourcepool-path: "/host//Resources/" +``` + +Volumes provisioned with the in-tree provider using the following format will get migrated correctly: + +```yaml +default-datastore: +resourcepool-path: "/Resources/" +``` + +Upstream bug: https://github.com/kubernetes-sigs/vsphere-csi-driver/issues/628 + +Rancher issue tracking this bug: https://github.com/rancher/rancher/issues/31105 + +## Prerequisites + +- vSphere CSI Migration requires vSphere 7.0u1. In order to be able to manage existing in-tree vSphere volumes, upgrade vSphere to 7.0u1. +- The Kubernetes version must be 1.19 or higher. + +## Migration + +### 1. Install the CPI plugin + +Before installing CPI, we need to taint all nodes with `node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule`. + +This can be done by running the following commands: + +``` +curl -O https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +``` + +Or: + +``` +wget https://raw.githubusercontent.com/rancher/helm3-charts/56b622f519728378abeddfe95074f1b87ab73b1e/charts/vsphere-cpi/taints.sh +chmod +x taints.sh +./taints.sh +``` + +Once all nodes are tainted by the running the script, launch the Helm vSphere CPI chart. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CPI chart will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CPI**.. +1. Click **Install**. +1. Fill out the required vCenter details and click **Install**. + +vSphere CPI initializes all nodes with ProviderID, which is needed by the vSphere CSI driver. + +Check if all nodes are initialized with the ProviderID with the following command: + +``` +kubectl describe nodes | grep "ProviderID" +``` + +### 2. Install the CSI driver + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where the vSphere CSI chart will be installed and click **Explore**. +1. Click **Apps > Charts**. +1. Click **vSphere CSI**.. +1. Click **Install**. +1. Fill out the required vCenter details and click **Install**. +1. Check **Customize Helm options before install** and click **Next**. +1. On the **Features** tab, check **Enable CSI Migration**. +1. Optionally, go to the **Storage** tab and set up a datastore. This chart creates a StorageClass with the `csi.vsphere.vmware.com` as the provisioner. You can provide the URL of the datastore to be used for CSI volume provisioning while creating this StorageClass. The datastore URL can be found in the vSphere client by selecting the datastore and going to the Summary tab. Fill out the details for the StorageClass. +1. Click **Install**. + +### 3. Edit the cluster to enable CSI migration feature flags + +1. While editing the cluster, if the Kubernetes version is less than 1.19, select Kubernetes version 1.19 or higher from the **Kubernetes Version** dropdown. +2. For enabling feature flags, click on "Edit as YAML", and add the following under kube-controller and kubelet: + + ```yaml + extra_args: + feature-gates: "CSIMigration=true,CSIMigrationvSphere=true" + ``` + +### 4. Drain worker nodes + +Worker nodes must be drained during the upgrade before changing the kubelet and kube-controller-manager args. + + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you will drain worker nodes and click **⋮ > Edit Config**. +1. In the **Advanced Options** section, set the field **Maximum Worker Nodes Unavailable** to 1. +1. To drain the nodes during upgrade, select **Drain Nodes > Yes**. +1. Set **Force** and **Delete Local Data** to **true**. +1. Click **Save** to upgrade the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md new file mode 100644 index 00000000000..f176acc177b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md @@ -0,0 +1,277 @@ +--- +title: Creating an AKS Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Microsoft Azure Kubernetes Service (AKS). + +## Prerequisites in Microsoft Azure + +:::caution + +Deploying to AKS will incur charges. + +::: + +To interact with Azure APIs, an AKS cluster requires an Azure Active Directory (AD) service principal. The service principal is needed to dynamically create and manage other Azure resources, and it provides credentials for your cluster to communicate with AKS. For more information about the service principal, refer to the [AKS documentation](https://docs.microsoft.com/en-us/azure/aks/kubernetes-service-principal). + +Before creating the service principal, you need to obtain the following information from the [Microsoft Azure Portal](https://portal.azure.com): + +- Subscription ID +- Client ID +- Client secret + +The below sections describe how to set up these prerequisites using either the Azure command line tool or the Azure portal. + +### Setting Up the Service Principal with the Azure Command Line Tool + +You can create the service principal by running this command: + +``` +az ad sp create-for-rbac --skip-assignment +``` + +The result should show information about the new service principal: +``` +{ + "appId": "xxxx--xxx", + "displayName": "", + "name": "http://", + "password": "", + "tenant": "" +} +``` + +You also need to add roles to the service principal so that it has privileges for communication with the AKS API. It also needs access to create and list virtual networks. + +Below is an example command for assigning the Contributor role to a service principal. Contributors can manage anything on AKS but cannot give access to others: + +``` +az role assignment create \ + --assignee $appId \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +You can also create the service principal and give it Contributor privileges by combining the two commands into one. In this command, the scope needs to provide a full path to an Azure resource: + +``` +az ad sp create-for-rbac \ + --scope /subscriptions/$/resourceGroups/$ \ + --role Contributor +``` + +### Setting Up the Service Principal with the Azure Command Line Tool + +Create the Resource Group by running this command: + +``` +az group create --location AZURE_LOCATION_NAME --resource-group AZURE_RESOURCE_GROUP_NAME +``` + +### Setting Up the Service Principal from the Azure Portal + +You can also follow these instructions to set up a service principal and give it role-based access from the Azure Portal. + +1. Go to the Microsoft Azure Portal [home page](https://portal.azure.com). + +1. Click **Azure Active Directory**. +1. Click **App registrations**. +1. Click **New registration**. +1. Enter a name. This will be the name of your service principal. +1. Optional: Choose which accounts can use the service principal. +1. Click **Register**. +1. You should now see the name of your service principal under **Azure Active Directory > App registrations**. +1. Click the name of your service principal. Take note of the application ID (also called app ID or client ID) so that you can use it when provisioning your AKS cluster. Then click **Certificates & secrets**. +1. Click **New client secret**. +1. Enter a short description, pick an expiration time, and click **Add**. Take note of the client secret so that you can use it when provisioning the AKS cluster. + +**Result:** You have created a service principal and you should be able to see it listed in the **Azure Active Directory** section under **App registrations**. You still need to give the service principal access to AKS. + +To give role-based access to your service principal, + +1. Click **All Services** in the left navigation bar. Then click **Subscriptions**. +1. Click the name of the subscription that you want to associate with your Kubernetes cluster. Take note of the subscription ID so that you can use it when provisioning your AKS cluster. +1. Click **Access Control (IAM)**. +1. In the **Add role assignment** section, click **Add**. +1. In the **Role** field, select a role that will have access to AKS. For example, you can use the **Contributor** role, which has permission to manage everything except for giving access to other users. +1. In the **Assign access to** field, select **Azure AD user, group, or service principal**. +1. In the **Select** field, select the name of your service principal and click **Save**. + +**Result:** Your service principal now has access to AKS. + +## 1. Create the AKS Cloud Credentials + +1. In the Rancher UI, click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Azure**. +1. Fill out the form. For help with filling out the form, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md#cloud-credentials) +1. Click **Create**. + +## 2. Create the AKS Cluster + +Use Rancher to set up and configure your Kubernetes cluster. + +1. Click **☰ > Cluster Management**. +1. In the **Clusters** section, click **Create**. +1. Click **Azure AKS**. +1. Fill out the form. For help with filling out the form, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) +1. Click **Create**. + +**Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +## Role-based Access Control +When provisioning an AKS cluster in the Rancher UI, RBAC is not configurable because it is required to be enabled. + +RBAC is required for AKS clusters that are registered or imported into Rancher. + +### Setting Up the Role Assignment to Service Principal with the Azure Command Line Tool + +Assign the Rancher AKSv2 role to the service principal with the Azure Command Line Tool: + +``` +az role assignment create \ +--assignee CLIENT_ID \ +--scope "/subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME" \ +--role "Rancher AKSv2" +``` + +## AKS Cluster Configuration Reference + +For more information about how to configure AKS clusters from the Rancher UI, see the [configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) + +## Private Clusters + +Typically, AKS worker nodes do not get public IPs, regardless of whether the cluster is private. In a private cluster, the control plane does not have a public endpoint. + +Rancher can connect to a private AKS cluster in one of two ways. + +The first way to ensure that Rancher is running on the same [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) as the AKS nodes. + +The second way is to run a command to register the cluster with Rancher. Once the cluster is provisioned, you can run the displayed command anywhere you can connect to the cluster’s Kubernetes API. This command is displayed in a pop-up when you provision an AKS cluster with a private API endpoint enabled. + +:::note + +Please be aware that when registering an existing AKS cluster, the cluster might take some time, possibly hours, to appear in the `Cluster To register` dropdown list. This outcome will be based on region. + +::: + +For more information about connecting to an AKS private cluster, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster) + +## Setting Up the Minimum Permission Role with the Azure Command Line Tool + +1. Create the Minimum Rancher AKSv2 Permission Role by running this command: + + ``` + cat >> rancher-azure.json << EOF + + { + "Name": "Rancher AKSv2", + "IsCustom": true, + "Description": "Everything needed by Rancher AKSv2 operator", + "Actions": [ + "Microsoft.Compute/disks/delete", + "Microsoft.Compute/disks/read", + "Microsoft.Compute/disks/write", + "Microsoft.Compute/diskEncryptionSets/read", + "Microsoft.Compute/locations/DiskOperations/read", + "Microsoft.Compute/locations/vmSizes/read", + "Microsoft.Compute/locations/operations/read", + "Microsoft.Compute/proximityPlacementGroups/write", + "Microsoft.Compute/snapshots/delete", + "Microsoft.Compute/snapshots/read", + "Microsoft.Compute/snapshots/write", + "Microsoft.Compute/virtualMachineScaleSets/manualUpgrade/action", + "Microsoft.Compute/virtualMachineScaleSets/delete", + "Microsoft.Compute/virtualMachineScaleSets/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/networkInterfaces/ipconfigurations/publicipaddresses/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualmachines/instanceView/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/read", + "Microsoft.Compute/virtualMachineScaleSets/virtualMachines/write", + "Microsoft.Compute/virtualMachineScaleSets/write", + "Microsoft.Compute/virtualMachines/read", + "Microsoft.Compute/virtualMachines/write", + "Microsoft.ContainerService/managedClusters/read", + "Microsoft.ContainerService/managedClusters/write" + "Microsoft.ContainerService/managedClusters/delete", + "Microsoft.ContainerService/managedClusters/accessProfiles/listCredential/action", + "Microsoft.ContainerService/managedClusters/agentPools/read", + "Microsoft.ContainerService/managedClusters/agentPools/write", + "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action", + "Microsoft.Network/applicationGateways/read", + "Microsoft.Network/applicationGateways/write", + "Microsoft.Network/loadBalancers/write", + "Microsoft.Network/loadBalancers/backendAddressPools/join/action", + "Microsoft.Network/loadBalancers/delete", + "Microsoft.Network/loadBalancers/read", + "Microsoft.Network/networkInterfaces/join/action", + "Microsoft.Network/networkInterfaces/read", + "Microsoft.Network/networkInterfaces/write", + "Microsoft.Network/networkSecurityGroups/read", + "Microsoft.Network/networkSecurityGroups/write", + "Microsoft.Network/publicIPAddresses/delete", + "Microsoft.Network/publicIPAddresses/join/action", + "Microsoft.Network/publicIPAddresses/read", + "Microsoft.Network/publicIPAddresses/write", + "Microsoft.Network/publicIPPrefixes/join/action", + "Microsoft.Network/privatednszones/*", + "Microsoft.Network/routeTables/read", + "Microsoft.Network/routeTables/routes/delete", + "Microsoft.Network/routeTables/routes/read", + "Microsoft.Network/routeTables/routes/write", + "Microsoft.Network/routeTables/write", + "Microsoft.Network/virtualNetworks/read", + "Microsoft.Network/virtualNetworks/subnets/join/action", + "Microsoft.Network/virtualNetworks/subnets/read", + "Microsoft.Network/virtualNetworks/joinLoadBalancer/action", + "Microsoft.OperationalInsights/workspaces/sharedkeys/read", + "Microsoft.OperationalInsights/workspaces/read", + "Microsoft.OperationsManagement/solutions/write", + "Microsoft.OperationsManagement/solutions/read", + "Microsoft.Resources/subscriptions/resourcegroups/read", + "Microsoft.Resources/subscriptions/resourcegroups/write", + "Microsoft.Storage/operations/read", + "Microsoft.Storage/storageAccounts/listKeys/action", + "Microsoft.Storage/storageAccounts/delete", + "Microsoft.Storage/storageAccounts/read", + "Microsoft.Storage/storageAccounts/write" + ], + "NotActions": [], + "DataActions": [], + "NotDataActions": [], + "AssignableScopes": [ + "/subscriptions/SUBSCRIPTION_ID" + ] + } + EOF + ``` + +2. Apply the Rancher AKSv2 Role: + + ``` + az role definition create --role-definition rancher-azure.json + ``` + +3. Verify if the Rancher AKSv2 Role was created: + + ``` + az role definition list | grep "Rancher AKSv2" + ``` + +## Syncing + +The AKS provisioner can synchronize the state of an AKS cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) + +For information on configuring the refresh interval, see [this section.](../../../../pages-for-subheaders/gke-cluster-configuration.md#configuring-the-refresh-interval) + + +## Programmatically Creating AKS Clusters + +The most common way to programmatically deploy AKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md new file mode 100644 index 00000000000..82cdde8861f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md @@ -0,0 +1,63 @@ +--- +title: Creating an Aliyun ACK Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Alibaba Cloud Kubernetes (ACK). Rancher has already implemented and packaged the [cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md) for ACK, but by default, this cluster driver is `inactive`. In order to launch ACK clusters, you will need to [enable the ACK cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#activatingdeactivating-cluster-drivers). After enabling the cluster driver, you can start provisioning ACK clusters. + +## Prerequisites Outside of Rancher + +:::caution + +Deploying to ACK will incur charges. + +::: + +1. In Aliyun, activate the following services in their respective consoles. + + - [Container Service](https://cs.console.aliyun.com) + - [Resource Orchestration Service](https://ros.console.aliyun.com) + - [RAM](https://ram.console.aliyun.com) + +2. Make sure that the account you will be using to create the ACK cluster has the appropriate permissions. Referring to the official Alibaba Cloud documentation about [Role authorization](https://www.alibabacloud.com/help/doc-detail/86483.htm) and [Use the Container Service console as a RAM user](https://www.alibabacloud.com/help/doc-detail/86484.htm) for details. + +3. In Alibaba Cloud, create an [access key](https://www.alibabacloud.com/help/doc-detail/53045.html). + +4. In Alibaba Cloud, create an [SSH key pair](https://www.alibabacloud.com/help/doc-detail/51793.html). This key is used to access nodes in the Kubernetes cluster. + +## Prerequisite in Rancher + +You will need to enable the Alibaba ACK cluster driver: + +1. Click **☰ > Cluster Management**. +1. Click **Drivers**. +1. In the **Cluster Drivers** tab, go to the **Alibaba ACK** cluster driver and click **⋮ > Activate**. + +When the cluster driver is finished downloading, you will be able to create Alibaba ACK clusters in Rancher. + +## Create an ACK Cluster + +1. Click **☰ > Cluster Management**. +1. From the **Clusters** page, click **Create**. +1. Choose **Alibaba ACK**. +1. Enter a **Cluster Name**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Configure **Account Access** for the ACK cluster. Choose the geographical region in which to build your cluster, and input the access key that was created as part of the prerequisite steps. +1. Click **Next: Configure Cluster**, then choose cluster type, the version of Kubernetes and the availability zone. +1. If you choose **Kubernetes** as the cluster type, Click **Next: Configure Master Nodes**, then complete the **Master Nodes** form. +1. Click **Next: Configure Worker Nodes**, then complete the **Worker Nodes** form. +1. Review your options to confirm they're correct. Then click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md new file mode 100644 index 00000000000..e6ecf6090ec --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md @@ -0,0 +1,333 @@ +--- +title: Creating an EKS Cluster +--- + + + + + +Amazon EKS provides a managed control plane for your Kubernetes cluster. Amazon EKS runs the Kubernetes control plane instances across multiple Availability Zones to ensure high availability. Rancher provides an intuitive user interface for managing and deploying the Kubernetes clusters you run in Amazon EKS. With this guide, you will use Rancher to quickly and easily launch an Amazon EKS Kubernetes cluster in your AWS account. For more information on Amazon EKS, see this [documentation](https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html). + + +## Prerequisites in Amazon Web Services + +:::caution + +Deploying to Amazon AWS will incur charges. For more information, refer to the [EKS pricing page](https://aws.amazon.com/eks/pricing/). + +::: + +To set up a cluster on EKS, you will need to set up an Amazon VPC (Virtual Private Cloud). You will also need to make sure that the account you will be using to create the EKS cluster has the appropriate [permissions.](#minimum-eks-permissions) For details, refer to the official guide on [Amazon EKS Prerequisites](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-console.html#eks-prereqs). + +### Amazon VPC + +An Amazon VPC is required to launch the EKS cluster. The VPC enables you to launch AWS resources into a virtual network that you've defined. You can set one up yourself and provide it during cluster creation in Rancher. If you do not provide one during creation, Rancher will create one. For more information, refer to the [Tutorial: Creating a VPC with Public and Private Subnets for Your Amazon EKS Cluster](https://docs.aws.amazon.com/eks/latest/userguide/create-public-private-vpc.html). + +### IAM Policies + +Rancher needs access to your AWS account in order to provision and administer your Kubernetes clusters in Amazon EKS. You'll need to create a user for Rancher in your AWS account and define what that user can access. + +1. Create a user with programmatic access by following the steps [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). + +2. Next, create an IAM policy that defines what this user has access to in your AWS account. It's important to only grant this user minimal access within your account. The minimum permissions required for an EKS cluster are listed [here.](#minimum-eks-permissions) Follow the steps [here](https://docs.aws.amazon.com/eks/latest/userguide/EKS_IAM_user_policies.html) to create an IAM policy and attach it to your user. + +3. Finally, follow the steps [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) to create an access key and secret key for this user. + +:::note Important: + +It's important to regularly rotate your access and secret keys. See this [documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#rotating_access_keys_console) for more information. + +::: + +For more detailed information on IAM policies for EKS, refer to the official [documentation on Amazon EKS IAM Policies, Roles, and Permissions](https://docs.aws.amazon.com/eks/latest/userguide/IAM_policies.html). + + +## Create the EKS Cluster + +Use Rancher to set up and configure your Kubernetes cluster. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Choose **Amazon EKS**. +1. Enter a **Cluster Name**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Fill out the rest of the form. For help, refer to the [configuration reference.](#eks-cluster-configuration-reference) +1. Click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## EKS Cluster Configuration Reference + +For the full list of EKS cluster configuration options, see [this page.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) + +## Architecture + +The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two Kubernetes clusters: one created by RKE and another created by EKS. + +
Managing Kubernetes Clusters through Rancher's Authentication Proxy
+ +![Architecture](/img/rancher-architecture-rancher-api-server.svg) + +## AWS Service Events + +To find information on any AWS Service events, please see [this page](https://status.aws.amazon.com/). + +## Security and Compliance + +By default only the IAM user or role that created a cluster has access to it. Attempting to access the cluster with any other user or role without additional configuration will lead to an error. In Rancher, this means using a credential that maps to a user or role that was not used to create the cluster will cause an unauthorized error. For example, an EKSCtl cluster will not register in Rancher unless the credentials used to register the cluster match the role or user used by EKSCtl. Additional users and roles can be authorized to access a cluster by being added to the aws-auth configmap in the kube-system namespace. For a more in-depth explanation and detailed instructions, please see this [documentation](https://aws.amazon.com/premiumsupport/knowledge-center/amazon-eks-cluster-access/). + +For more information on security and compliance with your Amazon EKS Kubernetes cluster, please see this [documentation](https://docs.aws.amazon.com/eks/latest/userguide/shared-responsibilty.html). + +## Tutorial + +This [tutorial](https://aws.amazon.com/blogs/opensource/managing-eks-clusters-rancher/) on the AWS Open Source Blog will walk you through how to set up an EKS cluster with Rancher, deploy a publicly accessible app to test the cluster, and deploy a sample project to track real-time geospatial data using a combination of other open-source software such as Grafana and InfluxDB. + +## Minimum EKS Permissions + +These are the minimum set of permissions necessary to access the full functionality of Rancher's EKS driver. You'll need additional permissions for Rancher to provision the `Service Role` and `VPC` resources. If you create these resources **before** you create the cluster, they'll be available when you configure the cluster. + +Resource | Description +---------|------------ +Service Role | Provides permissions that allow Kubernetes to manage resources on your behalf. Rancher can create the service role with the following [Service Role Permissions](#service-role-permissions). +VPC | Provides isolated network resources utilised by EKS and worker nodes. Rancher can create the VPC resources with the following [VPC Permissions](#vpc-permissions). + + +Resource targeting uses `*` as the ARN of many of the resources created cannot be known before creating the EKS cluster in Rancher. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "EC2Permissions", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:RevokeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:DescribeInstanceTypes", + "ec2:DescribeRegions", + "ec2:DescribeVpcs", + "ec2:DescribeTags", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeRouteTables", + "ec2:DescribeLaunchTemplateVersions", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeKeyPairs", + "ec2:DescribeInternetGateways", + "ec2:DescribeImages", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeAccountAttributes", + "ec2:DeleteTags", + "ec2:DeleteLaunchTemplate", + "ec2:DeleteSecurityGroup", + "ec2:DeleteKeyPair", + "ec2:CreateTags", + "ec2:CreateSecurityGroup", + "ec2:CreateLaunchTemplateVersion", + "ec2:CreateLaunchTemplate", + "ec2:CreateKeyPair", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress" + ], + "Resource": "*" + }, + { + "Sid": "CloudFormationPermissions", + "Effect": "Allow", + "Action": [ + "cloudformation:ListStacks", + "cloudformation:ListStackResources", + "cloudformation:DescribeStacks", + "cloudformation:DescribeStackResources", + "cloudformation:DescribeStackResource", + "cloudformation:DeleteStack", + "cloudformation:CreateStackSet", + "cloudformation:CreateStack" + ], + "Resource": "*" + }, + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:ListInstanceProfilesForRole", + "iam:ListInstanceProfiles", + "iam:ListAttachedRolePolicies", + "iam:GetRole", + "iam:GetInstanceProfile", + "iam:DetachRolePolicy", + "iam:DeleteRole", + "iam:CreateRole", + "iam:AttachRolePolicy" + ], + "Resource": "*" + }, + { + "Sid": "KMSPermissions", + "Effect": "Allow", + "Action": "kms:ListKeys", + "Resource": "*" + }, + { + "Sid": "EKSPermissions", + "Effect": "Allow", + "Action": [ + "eks:UpdateNodegroupVersion", + "eks:UpdateNodegroupConfig", + "eks:UpdateClusterVersion", + "eks:UpdateClusterConfig", + "eks:UntagResource", + "eks:TagResource", + "eks:ListUpdates", + "eks:ListTagsForResource", + "eks:ListNodegroups", + "eks:ListFargateProfiles", + "eks:ListClusters", + "eks:DescribeUpdate", + "eks:DescribeNodegroup", + "eks:DescribeFargateProfile", + "eks:DescribeCluster", + "eks:DeleteNodegroup", + "eks:DeleteFargateProfile", + "eks:DeleteCluster", + "eks:CreateNodegroup", + "eks:CreateFargateProfile", + "eks:CreateCluster" + ], + "Resource": "*" + } + ] +} +``` + +### Service Role Permissions + +These are permissions that are needed during EKS cluster creation, so Rancher can create a service role on the users' behalf. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "IAMPermissions", + "Effect": "Allow", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:AttachRolePolicy", + "iam:CreateInstanceProfile", + "iam:CreateRole", + "iam:CreateServiceLinkedRole", + "iam:DeleteInstanceProfile", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:GetInstanceProfile", + "iam:GetRole", + "iam:ListAttachedRolePolicies", + "iam:ListInstanceProfiles", + "iam:ListInstanceProfilesForRole", + "iam:ListRoles", + "iam:ListRoleTags", + "iam:PassRole", + "iam:RemoveRoleFromInstanceProfile" + ], + "Resource": "*" + } + ] +} +``` + +When you create an EKS cluster, Rancher creates a service role with the following trust policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Principal": { + "Service": "eks.amazonaws.com" + }, + "Effect": "Allow", + "Sid": "" + } + ] +} +``` + +This role also has two role policy attachments with the following policies' ARNs: + +``` +arn:aws:iam::aws:policy/AmazonEKSClusterPolicy +arn:aws:iam::aws:policy/AmazonEKSServicePolicy +``` + +### VPC Permissions + +These are permissions that are needed by Rancher to create a Virtual Private Cloud (VPC) and associated resources. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VPCPermissions", + "Effect": "Allow", + "Action": [ + "ec2:ReplaceRoute", + "ec2:ModifyVpcAttribute", + "ec2:ModifySubnetAttribute", + "ec2:DisassociateRouteTable", + "ec2:DetachInternetGateway", + "ec2:DescribeVpcs", + "ec2:DeleteVpc", + "ec2:DeleteTags", + "ec2:DeleteSubnet", + "ec2:DeleteRouteTable", + "ec2:DeleteRoute", + "ec2:DeleteInternetGateway", + "ec2:CreateVpc", + "ec2:CreateSubnet", + "ec2:CreateSecurityGroup", + "ec2:CreateRouteTable", + "ec2:CreateRoute", + "ec2:CreateInternetGateway", + "ec2:AttachInternetGateway", + "ec2:AssociateRouteTable" + ], + "Resource": "*" + } + ] +} +``` + +## Syncing + +The EKS provisioner can synchronize the state of an EKS cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) + +For information on configuring the refresh interval, refer to [this section.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md#configuring-the-refresh-interval) + +## Troubleshooting + +If your changes were overwritten, it could be due to the way the cluster data is synced with EKS. Changes shouldn't be made to the cluster from another source, such as in the EKS console, and in Rancher within a five-minute span. For information on how this works and how to configure the refresh interval, refer to [Syncing.](#syncing) + +If an unauthorized error is returned while attempting to modify or register the cluster and the cluster was not created with the role or user that your credentials belong to, refer to [Security and Compliance.](#security-and-compliance) + +For any issues or troubleshooting details for your Amazon EKS Kubernetes cluster, please see this [documentation](https://docs.aws.amazon.com/eks/latest/userguide/troubleshooting.html). + +## Programmatically Creating EKS Clusters + +The most common way to programmatically deploy EKS clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md new file mode 100644 index 00000000000..d091b83a6ac --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md @@ -0,0 +1,105 @@ +--- +title: Creating a GKE Cluster +--- + + + + + +## Prerequisites + +Some setup in Google Kubernetes Engine is required. + +### Service Account Token + +Create a service account using [Google Kubernetes Engine](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts). GKE uses this account to operate your cluster. Creating this account also generates a private key used for authentication. + +The service account requires the following roles: + +- **Compute Viewer:** `roles/compute.viewer` +- **Project Viewer:** `roles/viewer` +- **Kubernetes Engine Admin:** `roles/container.admin` +- **Service Account User:** `roles/iam.serviceAccountUser` + +[Google Documentation: Creating and Enabling Service Accounts](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) + +For help obtaining a private key for your service account, refer to the Google cloud documentation [here.](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) You will need to save the key in JSON format. + +### Google Project ID + +Your cluster will need to be part of a Google Project. + +To create a new project, refer to the Google cloud documentation [here.](https://cloud.google.com/resource-manager/docs/creating-managing-projects#creating_a_project) + +To get the project ID of an existing project, refer to the Google cloud documentation [here.](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects) + +## Provisioning a GKE Cluster + +:::caution + +Deploying to GKE will incur charges. + +::: + +### 1. Create a Cloud Credential + +1. Click **☰ > Cluster Management**. +1. In the left navigation bar, click **Cloud Credentials**. +1. Click **Create**. +1. Enter a name for your Google cloud credentials. +1. In the **Service Account** text box, paste your service account private key JSON, or upload the JSON file. +1. Click **Create**. + +**Result:** You have created credentials that Rancher will use to provision the new GKE cluster. + +### 2. Create the GKE Cluster +Use Rancher to set up and configure your Kubernetes cluster. To successfully create a GKE cluster with Rancher, your GKE must be in Standard mode. GKE has two modes of operation when creating a Kubernetes cluster, Autopilot and Standard mode. The cluster configuration for Autopilot mode has restrictions on editing the kube-system namespace. However, Rancher needs to create resources in the kube-system namespace during installation. As a result, you will not be able to create a GKE cluster in Autopilot mode. For more information about the difference between GKE Autopilot mode and Standard mode, visit [Compare GKE Autopilot and Standard.](https://cloud.google.com/kubernetes-engine/docs/resources/autopilot-standard-feature-comparison) + + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Google GKE**. +1. Enter a **Cluster Name**. +1. Optional: Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Optional: Add Kubernetes [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) or [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to the cluster. +1. Enter your Google project ID and your Google cloud credentials. +1. Fill out the rest of the form. For help, refer to the [GKE cluster configuration reference.](../../../../pages-for-subheaders/gke-cluster-configuration.md) +1. Click **Create**. + +**Result:** You have successfully deployed a GKE cluster. + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Private Clusters + +Private GKE clusters are supported. Note: This advanced setup can require more steps during the cluster provisioning process. For details, see [this section.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md) + +## Configuration Reference + +For details on configuring GKE clusters in Rancher, see [this page.](../../../../pages-for-subheaders/gke-cluster-configuration.md) +## Updating Kubernetes Version + +The Kubernetes version of a cluster can be upgraded to any version available in the region or zone fo the GKE cluster. Upgrading the master Kubernetes version does not automatically upgrade worker nodes. Nodes can be upgraded independently. + +:::note + +GKE has removed basic authentication in 1.19+. In order to upgrade a cluster to 1.19+, basic authentication must be disabled in the Google Cloud. Otherwise, an error will appear in Rancher when an upgrade to 1.19+ is attempted. You can follow the [Google documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#disabling_authentication_with_a_static_password). After this, the Kubernetes version can be updated to 1.19+ via Rancher. + +::: + +## Syncing + +The GKE provisioner can synchronize the state of a GKE cluster between Rancher and the provider. For an in-depth technical explanation of how this works, see [Syncing.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) + +For information on configuring the refresh interval, see [this section.](../../../../pages-for-subheaders/gke-cluster-configuration.md#configuring-the-refresh-interval) + +## Programmatically Creating GKE Clusters + +The most common way to programmatically deploy GKE clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md new file mode 100644 index 00000000000..cbde0642b88 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md @@ -0,0 +1,104 @@ +--- +title: Creating a Huawei CCE Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Huawei Cloud Container Engine (CCE). Rancher has already implemented and packaged the [cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md) for CCE, but by default, this cluster driver is `inactive`. In order to launch CCE clusters, you will need to [enable the CCE cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#activatingdeactivating-cluster-drivers). After enabling the cluster driver, you can start provisioning CCE clusters. + +## Prerequisites in Huawei + +:::caution + +Deploying to CCE will incur charges. + +::: + +1. Find your project ID in Huawei CCE portal. See the CCE documentation on how to [manage your projects](https://doc.hcs.huawei.com/api/cce/cce_02_0341.html#cce_02_0341__section17571944123214). + +2. Create an [Access Key ID and Secret Access Key](https://support.huaweicloud.com/en-us/usermanual-iam/en-us_topic_0079477318.html). + +## Prerequisite in Rancher + +You will need to enable the Huawei CCE cluster driver: + +1. Click **☰ > Cluster Management**. +1. Click **Drivers**. +1. In the **Cluster Drivers** tab, go to the **Huawei CCE** cluster driver and click **⋮ > Activate**. + +When the cluster driver is finished downloading, you will be able to create Huawei CCE clusters in Rancher. + +## Limitations + +Huawei CCE service doesn't support the ability to create clusters with public access through their API. You are required to run Rancher in the same VPC as the CCE clusters that you want to provision. + +## Create the CCE Cluster + +1. From the **Clusters** page, click **Create**. +1. Click **Huawei CCE**. +1. Enter a **Cluster Name**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Enter **Project Id**, Access Key ID as **Access Key** and Secret Access Key **Secret Key**. Then Click **Next: Configure cluster**. Fill in the cluster configuration. For help filling out the form, refer to [Huawei CCE Configuration.](#huawei-cce-configuration) +1. Fill the following node configuration of the cluster. For help filling out the form, refer to [Node Configuration.](#node-configuration) +1. Click **Create** to create the CCE cluster. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Huawei CCE Configuration + +|Settings|Description| +|---|---| +| Cluster Type | Which type or node you want to include into the cluster, `VirtualMachine` or `BareMetal`. | +| Description | The description of the cluster. | +| Master Version | The Kubernetes version. | +| Management Scale Count | The max node count of the cluster. The options are 50, 200 and 1000. The larger of the scale count, the more the cost. | +| High Availability | Enable master node high availability. The cluster with high availability enabled will have more cost. | +| Container Network Mode | The network mode used in the cluster. `overlay_l2` and `vpc-router` is supported in `VirtualMachine` type and `underlay_ipvlan` is supported in `BareMetal` type | +| Container Network CIDR | Network CIDR for the cluster. | +| VPC Name | The VPC name which the cluster is going to deploy into. Rancher will create one if it is blank. | +| Subnet Name | The Subnet name which the cluster is going to deploy into. Rancher will create one if it is blank. | +| External Server | This option is reserved for the future we can enable CCE cluster public access via API. For now, it is always disabled. | +| Cluster Label | The labels for the cluster. | +| Highway Subnet | This option is only supported in `BareMetal` type. It requires you to select a VPC with high network speed for the bare metal machines. | + +:::note + +If you are editing the cluster in the `cluster.yml` instead of the Rancher UI, note that cluster configuration directives must be nested under the `rancher_kubernetes_engine_config` directive in `cluster.yml`. For more information, refer to the section on [the config file structure.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#config-file-structure-in-rancher) + +::: + +## Node Configuration + +|Settings|Description| +|---|---| +| Zone | The available zone at where the node(s) of the cluster is deployed. | +| Billing Mode | The bill mode for the cluster node(s). In `VirtualMachine` type, only `Pay-per-use` is supported. in `BareMetal`, you can choose `Pay-per-use` or `Yearly/Monthly`. | +| Validity Period | This option only shows in `Yearly/Monthly` bill mode. It means how long you want to pay for the cluster node(s). | +| Auto Renew | This option only shows in `Yearly/Monthly` bill mode. It means that the cluster node(s) will renew the `Yearly/Monthly` payment automatically or not. | +| Data Volume Type | Data volume type for the cluster node(s). `SATA`, `SSD` or `SAS` for this option. | +| Data Volume Size | Data volume size for the cluster node(s) | +| Root Volume Type | Root volume type for the cluster node(s). `SATA`, `SSD` or `SAS` for this option. | +| Root Volume Size | Root volume size for the cluster node(s) | +| Node Flavor | The node flavor of the cluster node(s). The flavor list in Rancher UI is fetched from Huawei Cloud. It includes all the supported node flavors. | +| Node Count | The node count of the cluster | +| Node Operating System | The operating system for the cluster node(s). Only `EulerOS 2.2` and `CentOS 7.4` are supported right now. | +| SSH Key Name | The ssh key for the cluster node(s) | +| EIP | The public IP options for the cluster node(s). `Disabled` means that the cluster node(s) are not going to bind a public IP. `Create EIP` means that the cluster node(s) will bind one or many newly created Eips after provisioned and more options will be shown in the UI to set the to-create EIP parameters. And `Select Existed EIP` means that the node(s) will bind to the EIPs you select. | +| EIP Count | This option will only be shown when `Create EIP` is selected. It means how many EIPs you want to create for the node(s). | +| EIP Type | This option will only be shown when `Create EIP` is selected. The options are `5_bgp` and `5_sbgp`. | +| EIP Share Type | This option will only be shown when `Create EIP` is selected. The only option is `PER`. | +| EIP Charge Mode | This option will only be shown when `Create EIP` is selected. The options are pay by `BandWidth` and pay by `Traffic`. | +| EIP Bandwidth Size | This option will only be shown when `Create EIP` is selected. The BandWidth of the EIPs. | +| Authentication Mode | It means enabling `RBAC` or also enabling `Authenticating Proxy`. If you select `Authenticating Proxy`, the certificate which is used for authenticating proxy will be also required. | +| Node Label | The labels for the cluster node(s). Invalid labels can prevent upgrades or can prevent Rancher from starting. For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) | \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md new file mode 100644 index 00000000000..35cd4cc9b1c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md @@ -0,0 +1,103 @@ +--- +title: Creating a Tencent TKE Cluster +--- + + + + + +You can use Rancher to create a cluster hosted in Tencent Kubernetes Engine (TKE). Rancher has already implemented and packaged the [cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md) for TKE, but by default, this cluster driver is `inactive`. In order to launch TKE clusters, you will need to [enable the TKE cluster driver](../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers.md#activatingdeactivating-cluster-drivers). After enabling the cluster driver, you can start provisioning TKE clusters. + +## Prerequisites in Tencent + +:::caution + +Deploying to TKE will incur charges. + +::: + +1. Make sure that the account you will be using to create the TKE cluster has the appropriate permissions by referring to the [Cloud Access Management](https://intl.cloud.tencent.com/document/product/598/10600) documentation for details. + +2. Create a [Cloud API Secret ID and Secret Key](https://console.cloud.tencent.com/capi). + +3. Create a [Private Network and Subnet](https://intl.cloud.tencent.com/document/product/215/4927) in the region that you want to deploy your Kubernetes cluster. + +4. Create a [SSH key pair](https://intl.cloud.tencent.com/document/product/213/6092). This key is used to access the nodes in the Kubernetes cluster. + +## Prerequisite in Rancher + +You will need to enable the Tencent TKE cluster driver: + +1. Click **☰ > Cluster Management**. +1. Click **Drivers**. +1. In the **Cluster Drivers** tab, go to the **Tencent TKE** cluster driver and click **⋮ > Activate**. + +When the cluster driver is finished downloading, you will be able to create Tencent TKE clusters in Rancher. + +## Create a TKE Cluster + +1. From the **Clusters** page, click **Create**. + +2. Choose **Tencent TKE**. + +3. Enter a **Cluster Name**. + +4. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. + +5. Configure **Account Access** for the TKE cluster. Complete each drop-down and field using the information obtained in [Prerequisites](#prerequisites-in-tencent). + + | Option | Description | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Region | From the drop-down chooses the geographical region in which to build your cluster. | + | Secret ID | Enter the Secret ID that you obtained from the Tencent Cloud Console. | + | Secret Key | Enter the Secret key that you obtained from Tencent Cloud Console. | + +6. Click `Next: Configure Cluster` to set your TKE cluster configurations. + + | Option | Description | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Kubernetes Version | The TKE only supports Kubernetes version 1.10.5 now. | + | Node Count | Enter the amount of worker node you want to purchase for your Kubernetes cluster, up to 100. | + | VPC | Select the VPC name that you have created in the Tencent Cloud Console. | + | Container Network CIDR | Enter the CIDR range of your Kubernetes cluster, you may check the available range of the CIDR in the VPC service of the Tencent Cloud Console. Default to 172.16.0.0/16. | + + :::note + + If you are editing the cluster in the `cluster.yml` instead of the Rancher UI, note that, cluster configuration directives must be nested under the `rancher_kubernetes_engine_config` directive in `cluster.yml`. For more information, refer to the section on [the config file structure in Rancher v2.3.0+.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#config-file-structure-in-rancher) + + ::: + +7. Click `Next: Select Instance Type` to choose the instance type that will use for your TKE cluster. + + | Option | Description | + | ---------- | -------------------------------------------------------------------------------------------------------------------- | + | Availability Zone | Choose the availability zone of the VPC region. | + | Subnet | Select the Subnet that you have created within the VPC, and add a new one if you don't have it in the chosen availability zone. | + | Instance Type | From the drop-down chooses the VM instance type that you want to use for the TKE cluster, default to S2.MEDIUM4 (CPU 2 Memory 4 GiB). | + +8. Click `Next: Configure Instance` to configure the VM instance that will use for your TKE cluster. + + Option | Description + -------|------------ + Operating System | The name of the operating system, currently supports Centos7.2x86_64 or ubuntu16.04.1 LTSx86_64 + Security Group | Security group ID, default does not bind any security groups. + Root Disk Type | System disk type. System disk type restrictions are detailed in the [CVM instance configuration](https://cloud.tencent.com/document/product/213/11518). + Root Disk Size | System disk size. Linux system adjustment range is 20 - 50G, step size is 1. + Data Disk Type | Data disk type, default value to the SSD cloud drive + Data Disk Size | Data disk size (GB), the step size is 10 + Band Width Type | Type of bandwidth, PayByTraffic or PayByHour + Band Width | Public network bandwidth (Mbps) + Key Pair | Key id, after associating the key can be used to logging to the VM node + +9. Click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md new file mode 100644 index 00000000000..7690a17e458 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md @@ -0,0 +1,45 @@ +--- +title: Configuration for Storage Classes in Azure +--- + + + + + +If you are using Azure VMs for your nodes, you can use [Azure files](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv) as a StorageClass for the cluster. + +In order to have the Azure platform create the required storage resources, follow these steps: + +1. [Configure the Azure cloud provider.](../../kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md) +1. Configure `kubectl` to connect to your cluster. +1. Copy the `ClusterRole` and `ClusterRoleBinding` manifest for the service account: + ```yml + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:azure-cloud-provider + rules: + - apiGroups: [''] + resources: ['secrets'] + verbs: ['get','create'] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: system:azure-cloud-provider + roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:azure-cloud-provider + subjects: + - kind: ServiceAccount + name: persistent-volume-binder + namespace: kube-system + ``` + +1. Create these in your cluster using one of the follow command. + + ``` + # kubectl create -f + ``` diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md new file mode 100644 index 00000000000..65596651f3f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md @@ -0,0 +1,40 @@ +--- +title: Networking Requirements for Host Gateway (L2bridge) +--- + + + + + +This section describes how to configure custom Windows clusters that are using *Host Gateway (L2bridge)* mode. + +### Disabling Private IP Address Checks + +If you are using *Host Gateway (L2bridge)* mode and hosting your nodes on any of the cloud services listed below, you must disable the private IP address checks for both your Linux or Windows hosts on startup. To disable this check for each node, follow the directions provided by each service below. + +Service | Directions to disable private IP address checks +--------|------------------------------------------------ +Amazon EC2 | [Disabling Source/Destination Checks](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_NAT_Instance.html#EIP_Disable_SrcDestCheck) +Google GCE | [Enabling IP Forwarding for Instances](https://cloud.google.com/vpc/docs/using-routes#canipforward) (By default, a VM cannot forward a packet originated by another VM) +Azure VM | [Enable or Disable IP Forwarding](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-network-interface#enable-or-disable-ip-forwarding) + +### Cloud-hosted VM Routes Configuration + +If you are using the [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) backend of Flannel, all containers on the same node belong to a private subnet, and traffic routes from a subnet on one node to a subnet on another node through the host network. + +- When worker nodes are provisioned on AWS, virtualization clusters, or bare metal servers, make sure they belong to the same layer 2 subnet. If the nodes don't belong to the same layer 2 subnet, `host-gw` networking will not work. + +- When worker nodes are provisioned on GCE or Azure, they are not on the same layer 2 subnet. Nodes on GCE and Azure belong to a routable layer 3 network. Follow the instructions below to configure GCE and Azure so that the cloud network knows how to route the host subnets on each node. + +To configure host subnet routing on GCE or Azure, first run the following command to find out the host subnets on each worker node: + +```bash +kubectl get nodes -o custom-columns=nodeName:.metadata.name,nodeIP:status.addresses[0].address,routeDestination:.spec.podCIDR +``` + +Then follow the instructions for each cloud provider to configure routing rules for each node: + +Service | Instructions +--------|------------- +Google GCE | For GCE, add a static route for each node: [Adding a Static Route](https://cloud.google.com/vpc/docs/using-routes#addingroute). +Azure VM | For Azure, create a routing table: [Custom Routes: User-defined](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview#user-defined). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md new file mode 100644 index 00000000000..7999a11defc --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity.md @@ -0,0 +1,46 @@ +--- +title: Windows and Linux Cluster Feature Parity +--- + + + + + +Windows clusters do not share the same feature support as Linux clusters. + +The following chart describes the feature parity between Windows and Linux on Rancher: + +**Component** | **Linux** | **Windows** +--- | --- | --- +**Distributions** | | +RKE | Supported | Supported +RKE2 | Supported | Supported +K3S | Supported | Not Supported +EKS | Supported | Not Supported +GKE | Supported | Not Supported +AKS | Supported | Not Supported +**Rancher Components** | | +Server | Supported | Not Supported +Agent | Supported | Supported +Fleet | Supported | Supported +EKS Operator | Supported | Not Supported +AKS Operator | Not Supported | Not Supported +GKE Operator | Not Supported | Not Supported +Monitoring/Alerting v2 | Supported | Supported +Logging v2 | Supported | Supported +Istio | Supported | Not Supported +Catalog v2 | Supported | Not Supported +OPA | Supported | Not Supported +Longhorn | Supported | Not Supported +CIS Scans | Supported | Not Supported +Backup/Restore Operator | Supported | Not Supported +**CNI / Add-ons** | | +Flannel | Supported | Supported +Canal | Supported | Not Supported +Calico | Supported | Supported (RKE2 Only) +Cilium | Supported | Not Supported +Multus | Supported | Not Supported +Traefik | Supported | Not Supported +NGINX Ingress | Supported | Not Supported + +For updated information on feature support, you may visit [rancher/windows](https://github.com/rancher/windows) on GitHub. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md new file mode 100644 index 00000000000..66e8c274ee5 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance.md @@ -0,0 +1,307 @@ +--- +title: RKE1 to RKE2 Windows Migration Guidance +--- + + + + + +**Caution:** The contents of this document are not covered under SLA by Rancher Support. Please proceed with caution. + +This document covers how end users can migrate their Windows workloads from RKE1 to RKE2. + +## RKE1 Windows Scheduling + +RKE1 Windows workload scheduling is based on taints and tolerations. + +Every Linux node in an RKE1 Windows cluster, regardless of the role assigned to it, will have have a default taint that prevents workloads to be scheduled on it unless the workload has a toleration configured. This is a major design feature for RKE1 Windows clusters which were designed to only run Windows workloads. + +- Default RKE1 Linux node `NoSchedule` taint: + +```yml +apiVersion: v1 +kind: Node +spec: + ... + taints: + - effect: NoSchedule + key: cattle.io/os + value: linux +``` +
+ +- RKE1 Linux `NoSchedule` toleration for workloads + +The following toleration would allow an end-user workload to schedule on any Linux node of an RKE1 Windows cluster. These tolerations are used for various core Rancher services and workloads. + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` +
+ +- Aligning with best practices, any end-user workloads being run on Linux nodes would be scheduled on those with the worker role only: + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + weight: 100 + ... +``` + +## RKE2 Windows Scheduling + +Based on feedback and requests for hybrid workload support, RKE2 Windows was designed to support both Linux and Windows workloads by default. RKE2 scheduling relies on node selectors by default. This is a marked change from RKE1 as taints and tolerations were not incorporated into RKE2. Node selectors were a critical part of RKE1 Windows clusters, which makes for an easy migration of your workloads. + +## Example Migrations + +### RKE1 to RKE2 Windows Workload + +- Pre-migration RKE1 Windows deployment: + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: NotIn + values: + - linux +``` +
+ +- Migrated RKE2 Windows deployment using `NodeAffinity`: + +```yaml +apiVersion: apps/v1 +kind: Deployment +... +spec: + ... + template: + ... + spec: + ... + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows +``` + +### RKE1 Windows Cluster Linux-Only Deployment + +>**Important:** When leveraging node selectors and node affinity, note the following: +> +>- If both `nodeSelector` and `nodeAffinity` are specified, both must be satisfied for the `Pod` to be scheduled onto a node. +>- If you specify multiple `matchExpressions` associated with a single `nodeSelectorTerms`, then the `Pod` can be scheduled onto a node only if all the `matchExpressions` are satisfied. + +
+ +- Pre-migration RKE1 Windows cluster Linux-only deployment targeting RKE1 Linux worker nodes: + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + tolerations: + - effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + +``` +
+ +- Migrated RKE2 hybrid cluster Linux-only deployment targeting RKE2 Linux worker nodes using node selectors: + +```yml +apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + nodeSelector: + kubernetes.io/os: "linux" + node-role.kubernetes.io/worker: "true" + ``` +
+ +- Migrated RKE2 hybrid cluster Linux-only deployment targeting RKE2 Linux worker nodes using node affinity: + + ```yaml + apiVersion: apps/v1 +kind: Deployment +spec: + ... + template: + ... + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/worker + operator: In + values: + - "true" + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + ``` +## RKE1 Windows-Supported Windows Server Versions + +### Long-Term Servicing Channel (LTSC) + +- Windows Server 2019 LTSC ✅ Will reach Mainstream EOL on Jan 9, 2024 and Extended EOL on Jan 9, 2029 + +### Semi-Annual Channel (SAC) + +- Windows Server 20H2 SAC ❌ EOL Reached on Aug 9, 2022 +- Windows Server 2004 SAC ❌ EOL Reached on Dec 14, 2021 +- Windows Server 1909 SAC ❌ EOL Reached on May 11, 2021 +- Windows Server 1903 SAC ❌ EOL Reached on Dec 8, 2020 +- Windows Server 1809 SAC ❌ EOL Reached on Nov 10, 2020 + +## RKE2 Windows-Supported Windows Server Versions + +### Long-Term Servicing Channel in RKE2 + +- Windows Server 2019 LTSC ✅ Will reach Mainstream EOL on Jan 9, 2024 and Extended EOL on Jan 9, 2029 +- Windows Server 2022 LTSC ✅ Will reach Mainstream EOL on Oct 13, 2026 and Extended EOL on Oct 13, 2031 + +>**Note:** SAC is not supported in RKE2. + + +For more information, please see the following references: + +- [Windows Server SAC Lifecycle](https://docs.microsoft.com/en-us/lifecycle/products/windows-server) + +- [Windows Server 2022 LTSC Lifecycle](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2022) + +- [Windows Server 2019 LTSC Lifecycle](https://docs.microsoft.com/en-us/lifecycle/products/windows-server-2019) + + +## Kubernetes Version Support + +>**Note:** All versions listed below are SLA Supported per the [Rancher v2.6.7 Support Matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions/rancher-v2-6-7/). Any version not listed should be assumed as being EOL and not supported under SLA by SUSE. + +### Rancher 2.5 vs. Rancher 2.6 Support Matrix for Windows Clusters + +**RKE1 vs. RKE2 Windows cluster-supported Kubernetes versions:** + +| Kubernetes Versions | RKE1 | RKE2 | +|-------------- |:----: |:----: | +| 1.18 | ✓ | | +| 1.19 | ✓ | | +| 1.20 | ✓ | | +| 1.21 | ✓ | | +| 1.22 | ✓ | ✓ | +| 1.23 | | ✓ | +| 1.24 | | ✓ | +| 1.25+ | | ✓ | + + +### Rancher 2.5 vs. Rancher 2.6 Supported Kubernetes Versions for Provisioning RKE1 and RKE2 Windows Clusters + +| Rancher Versions | Kubernetes Versions | RKE1 | RKE2 | +|:-----------------------: |:------------------------: |:----: |:----: | +| 2.5 - RKE1 Provisioning | 1.18 1.19 1.20 | ✓ | | +| 2.6 - RKE1 Provisioning | 1.18 1.19 1.20 1.21 1.22 | ✓ | | +| 2.6 - RKE2 Provisioning | 1.22 1.23 1.24 1.25+ | | ✓ | + + +## Guiding Migrations of Workloads to RKE2 Windows + +Referencing the tables in [Rancher 2.5 vs. Rancher 2.6 Support Matrix for Windows Clusters](#rancher-25-vs-rancher-26-support-matrix-for-windows-clusters) and [Rancher 2.5 vs. Rancher 2.6 Supported Kubernetes Versions for Provisioning RKE1 and RKE2 Windows Clusters](#rancher-25-vs-rancher-26-supported-kubernetes-versions-for-provisioning-rke1-and-rke2-windows-clusters), you will find the overlap in Kubernetes versions between RKE1 and RKE2 occurs in 1.22. This will be the base version required to migrate RKE1 Windows workloads when following the Rancher recommended approach. + +### In-Place Upgrade of Rancher 2.5 + +1. Upgrade the Rancher version to v2.6.5+. +1. Upgrade the RKE1 Windows downstream cluster(s) to RKE1 v1.22 using the latest available patch version. +1. Provision a new RKE2 Windows downstream cluster using RKE2 v1.22 using the matching patch version that the RKE1 Windows cluster is at. +1. Begin the migration of the Windows workloads from RKE1 to RKE2 clusters. +1. Perform validation tests to ensure that there has been no functionality loss or change when migrating your application from RKE1 to RKE2. +1. After successful validation tests have occurred, you can opt to upgrade your RKE2 1.22.x cluster to a new minor version such as 1.23 or 1.24. + + +### Migrating Windows Workloads to a New Rancher Environment + +>**Important:** To perform either of the following options requires Rancher v2.6.5 or above. + +**When using matching Kubernetes patch versions for RKE1 and RKE2:** + +1. Provision a new RKE2 Windows downstream cluster using RKE2 v1.22 using the matching patch version that the RKE1 Windows cluster is at. +1. Begin the migration of the Windows workloads from RKE1 to RKE2 clusters. +1. Perform validation tests to ensure that there has been no functionality loss or change when migrating your application from RKE1 to RKE2. +1. After successful validation tests have occurred, you can opt to upgrade your RKE2 1.22.x cluster to a new minor version such as 1.23 or 1.24. + + +**When using a newer Kubernetes patch version for RKE2:** + +1. Provision a new RKE2 Windows downstream cluster using RKE2 v1.23 or v1.24. +1. Begin the migration of the Windows workloads from RKE1 to RKE2 clusters. +1. Perform validation tests to ensure that there has been no functionality loss or change when migrating your application from RKE1 to RKE2. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md new file mode 100644 index 00000000000..097a4d26e34 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md @@ -0,0 +1,39 @@ +--- +title: ConfigMaps +--- + + + + + +While most types of Kubernetes secrets store sensitive information, [ConfigMaps](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/) store general configuration information, such as a group of config files. Because ConfigMaps don't store sensitive information, they can be updated automatically, and therefore don't require their containers to be restarted following update (unlike most secret types, which require manual updates and a container restart to take effect). + +ConfigMaps accept key value pairs in common string formats, like config files or JSON blobs. After you upload a config map, any workload can reference it as either an environment variable or a volume mount. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that has the workload that should reference a ConfigMap and click **Explore**. +1. In the left navigation bar, click **More Resources > Core > ConfigMaps**. +1. Click **Create**. +1. Enter a **Name** for the Config Map. + + :::note + + Kubernetes classifies ConfigMaps as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a project or namespace can have duplicate names. Therefore, to prevent conflicts, your ConfigMaps must have a unique name among the other certificates, registries, and secrets within your workspace. + + ::: + +1. Select the **Namespace** you want to add Config Map to. + +1. On the **Data** tab, add a key-value pair to your ConfigMap. Add as many values as you need. You can add multiple key value pairs to the ConfigMap by copying and pasting. Alternatively, use **Read from File** to add the data. Note: If you need to store sensitive data, [use a secret](secrets.md), not a ConfigMap. +1. Click **Create**. + +**Result:** Your ConfigMap is added to the namespace. You can view it in the Rancher UI from the **Resources > Config Maps** view. + +## What's Next? + +Now that you have a ConfigMap added to a namespace, you can add it to a workload that you deploy from the namespace of origin. You can use the ConfigMap to specify information for you application to consume, such as: + +- Application environment variables. +- Specifying parameters for a Volume mounted to the workload. + +For more information on adding ConfigMaps to a workload, see [Deploying Workloads](workloads-and-pods/deploy-workloads.md). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md new file mode 100644 index 00000000000..5415a75ca9c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md @@ -0,0 +1,33 @@ +--- +title: Services +--- + + + + + +Pod configuration is managed by Deployments, StatefulSets and Daemonsets, whereas services direct traffic to pods using selectors. + +For every workload (with at least one port configured) created, a complementing Service Discovery entry is created. This Service Discovery entry enables DNS resolution for the workload's pods using the following naming convention: +`..svc.cluster.local`. + +You can create additional services so that a given namespace resolves with one or more external IP addresses, an external hostname, an alias to another DNS record, other workloads, or a set of pods that match a selector that you create. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a service and click **Explore**. +1. Click **Service Discovery > Services**. +1. Click **Create**. +1. Choose the type of service you want to create. +1. Select a **Namespace** from the drop-down list. +1. Enter a **Name** for the service. This name is used for DNS resolution. +1. Fill out the rest of the form. For help, refer to the upstream Kubernetes documentation about [services.](https://kubernetes.io/docs/concepts/services-networking/service/) +1. Click **Create**. + +**Result:** A new service is created. + +- You can view the record by from the project's **Service Discovery** tab. +- When you visit the new DNS name for the new record that you created (`..svc.cluster.local`), it resolves the chosen namespace. + +## Related Links + +- [Adding entries to Pod /etc/hosts with HostAliases](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md new file mode 100644 index 00000000000..27f41079923 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md @@ -0,0 +1,42 @@ +--- +title: Encrypting HTTP Communication +description: Learn how to add an SSL (Secure Sockets Layer) certificate or TLS (Transport Layer Security) certificate +--- + + + + + +When you create an ingress within Rancher/Kubernetes, you must provide it with a secret that includes a TLS private key and certificate, which are used to encrypt and decrypt communications that come through the ingress. You can make certificates available for ingress use by adding the certificate to the ingress deployment. + +:::note Prerequisite: + +You must have a TLS private key and certificate available to upload. + +::: + +### 1. Create a Secret + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to deploy your ingress and click **More Resources > Core > Secrets**. +1. Click **Create**. +1. Click **TLS Certificate**. +1. Enter a name for the secret. Note: Your secret must have a unique name among the other certificates, registries, and secrets within your project/workspace. +1. In the **Private Key** field, either copy and paste your certificate's private key into the text box (include the header and footer), or click **Read from a file** to browse to the private key on your file system. If possible, we recommend using **Read from a file** to reduce likelihood of error. Note: Private key files end with an extension of `.key`. +1. In the **Certificate** field, either copy and paste your certificate into the text box (include the header and footer), or click **Read from a file** to browse to the certificate on your file system. If possible, we recommend using **Read from a file** to reduce likelihood of error. Note: Certificate files end with an extension of `.crt`. +1. Click **Create**. + +### 2. Add the Secret to an Ingress + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to deploy your ingress and click **Service Discovery > Ingresses**. +1. Click **Create**. +1. Select the **Namespace** of the ingress. +1. Enter a **Name** for the ingress. +1. In the **Certificates** tab, select the secret containing your certificate and private key. +1. Click **Create**. + +## What's Next? + +Now you can add the certificate when launching an ingress within the current project or namespace. For more information, see [Adding Ingress](load-balancer-and-ingress-controller/add-ingresses.md). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md new file mode 100644 index 00000000000..18729cbca41 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas.md @@ -0,0 +1,43 @@ +--- +title: Background Information on HPAs +--- + + + + + +The [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) (HPA) is a Kubernetes feature that allows you to configure your cluster to automatically scale the services it's running up or down. This section provides explanation on how HPA works with Kubernetes. + +## Why Use Horizontal Pod Autoscaler? + +Using HPA, you can automatically scale the number of pods within a replication controller, deployment, or replica set up or down. HPA automatically scales the number of pods that are running for maximum efficiency. Factors that affect the number of pods include: + +- A minimum and maximum number of pods allowed to run, as defined by the user. +- Observed CPU/memory use, as reported in resource metrics. +- Custom metrics provided by third-party metrics application like Prometheus, Datadog, etc. + +HPA improves your services by: + +- Releasing hardware resources that would otherwise be wasted by an excessive number of pods. +- Increase/decrease performance as needed to accomplish service level agreements. + +## How HPA Works + +![HPA Schema](/img/horizontal-pod-autoscaler.jpg) + +HPA is implemented as a control loop, with a period controlled by the `kube-controller-manager` flags below: + +Flag | Default | Description | +---------|----------|----------| + `--horizontal-pod-autoscaler-sync-period` | `30s` | How often HPA audits resource/custom metrics in a deployment. + `--horizontal-pod-autoscaler-downscale-delay` | `5m0s` | Following completion of a downscale operation, how long HPA must wait before launching another downscale operations. + `--horizontal-pod-autoscaler-upscale-delay` | `3m0s` | Following completion of an upscale operation, how long HPA must wait before launching another upscale operation. + + +For full documentation on HPA, refer to the [Kubernetes Documentation](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/). + +## Horizontal Pod Autoscaler API Objects + +HPA is an API resource in the Kubernetes `autoscaling` API group. The current stable version is `autoscaling/v1`, which only includes support for CPU autoscaling. To get additional support for scaling based on memory and custom metrics, use the beta version instead: `autoscaling/v2beta1`. + +For more information about the HPA API object, see the [HPA GitHub Readme](https://github.com/kubernetes/design-proposals-archive/blob/main/autoscaling/horizontal-pod-autoscaler.md#horizontalpodautoscaler-object). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md new file mode 100644 index 00000000000..573119909e8 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md @@ -0,0 +1,208 @@ +--- +title: Managing HPAs with kubectl +--- + + + + + +This section describes HPA management with `kubectl`. This document has instructions for how to: + +- Create an HPA +- Get information on HPAs +- Delete an HPA +- Configure your HPAs to scale with CPU or memory utilization +- Configure your HPAs to scale using custom metrics, if you use a third-party tool such as Prometheus for metrics + + +You can create, view, and delete HPAs from the Rancher UI. You can also configure them to scale based on CPU or memory usage from the Rancher UI. For more information, refer to [Managing HPAs with the Rancher UI](./manage-hpas-with-ui.md). For scaling HPAs based on other metrics than CPU or memory, you still need `kubectl`. + +## Basic kubectl Command for Managing HPAs + +If you have an HPA manifest file, you can create, manage, and delete HPAs using `kubectl`: + +- Creating HPA + + - With manifest: `kubectl create -f ` + + - Without manifest (Just support CPU): `kubectl autoscale deployment hello-world --min=2 --max=5 --cpu-percent=50` + +- Getting HPA info + + - Basic: `kubectl get hpa hello-world` + + - Detailed description: `kubectl describe hpa hello-world` + +- Deleting HPA + + - `kubectl delete hpa hello-world` + +## HPA Manifest Definition Example + +The HPA manifest is the config file used for managing an HPA with `kubectl`. + +The following snippet demonstrates use of different directives in an HPA manifest. See the list below the sample to understand the purpose of each directive. + +```yml +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: hello-world +spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi +``` + + +Directive | Description +---------|----------| + `apiVersion: autoscaling/v2beta1` | The version of the Kubernetes `autoscaling` API group in use. This example manifest uses the beta version, so scaling by CPU and memory is enabled. | + `name: hello-world` | Indicates that HPA is performing autoscaling for the `hello-word` deployment. | + `minReplicas: 1` | Indicates that the minimum number of replicas running can't go below 1. | + `maxReplicas: 10` | Indicates the maximum number of replicas in the deployment can't go above 10. + `targetAverageUtilization: 50` | Indicates the deployment will scale pods up when the average running pod uses more than 50% of its requested CPU. + `targetAverageValue: 100Mi` | Indicates the deployment will scale pods up when the average running pod uses more that 100Mi of memory. +
+ +## Configuring HPA to Scale Using Resource Metrics (CPU and Memory) + +Clusters created in Rancher v2.0.7 and higher have all the requirements needed (metrics-server and Kubernetes cluster configuration) to use Horizontal Pod Autoscaler. + +Run the following commands to check if metrics are available in your installation: + +``` +$ kubectl top nodes +NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% +node-controlplane 196m 9% 1623Mi 42% +node-etcd 80m 4% 1090Mi 28% +node-worker 64m 3% 1146Mi 29% +$ kubectl -n kube-system top pods +NAME CPU(cores) MEMORY(bytes) +canal-pgldr 18m 46Mi +canal-vhkgr 20m 45Mi +canal-x5q5v 17m 37Mi +canal-xknnz 20m 37Mi +kube-dns-7588d5b5f5-298j2 0m 22Mi +kube-dns-autoscaler-5db9bbb766-t24hw 0m 5Mi +metrics-server-97bc649d5-jxrlt 0m 12Mi +$ kubectl -n kube-system logs -l k8s-app=metrics-server +I1002 12:55:32.172841 1 heapster.go:71] /metrics-server --source=kubernetes.summary_api:https://kubernetes.default.svc?kubeletHttps=true&kubeletPort=10250&useServiceAccount=true&insecure=true +I1002 12:55:32.172994 1 heapster.go:72] Metrics Server version v0.2.1 +I1002 12:55:32.173378 1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default.svc" and version +I1002 12:55:32.173401 1 configs.go:62] Using kubelet port 10250 +I1002 12:55:32.173946 1 heapster.go:128] Starting with Metric Sink +I1002 12:55:32.592703 1 serving.go:308] Generated self-signed cert (apiserver.local.config/certificates/apiserver.crt, apiserver.local.config/certificates/apiserver.key) +I1002 12:55:32.925630 1 heapster.go:101] Starting Heapster API server... +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] listing is available at https:///swaggerapi +[restful] 2018/10/02 12:55:32 log.go:33: [restful/swagger] https:///swaggerui/ is mapped to folder /swagger-ui/ +I1002 12:55:32.928597 1 serve.go:85] Serving securely on 0.0.0.0:443 +``` + + +## Configuring HPA to Scale Using Custom Metrics with Prometheus + +You can configure HPA to autoscale based on custom metrics provided by third-party software. The most common use case for autoscaling using third-party software is based on application-level metrics (i.e., HTTP requests per second). HPA uses the `custom.metrics.k8s.io` API to consume these metrics. This API is enabled by deploying a custom metrics adapter for the metrics collection solution. + +For this example, we are going to use [Prometheus](https://prometheus.io/). We are beginning with the following assumptions: + +- Prometheus is deployed in the cluster. +- Prometheus is configured correctly and collecting proper metrics from pods, nodes, namespaces, etc. +- Prometheus is exposed at the following URL and port: `http://prometheus.mycompany.io:80` + +Prometheus is available for deployment in the Rancher v2.0 catalog. Deploy it from Rancher catalog if it isn't already running in your cluster. + +For HPA to use custom metrics from Prometheus, package [k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) is required in the `kube-system` namespace of your cluster. To install `k8s-prometheus-adapter`, we are using the Helm chart available at [banzai-charts](https://github.com/banzaicloud/banzai-charts). + +1. Initialize Helm in your cluster. + ``` + # kubectl -n kube-system create serviceaccount tiller + kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller + helm init --service-account tiller + ``` + +1. Clone the `banzai-charts` repo from GitHub: + ``` + # git clone https://github.com/banzaicloud/banzai-charts + ``` + +1. Install the `prometheus-adapter` chart, specifying the Prometheus URL and port number. + ``` + # helm install --name prometheus-adapter banzai-charts/prometheus-adapter --set prometheus.url="http://prometheus.mycompany.io",prometheus.port="80" --namespace kube-system + ``` + +1. Check that `prometheus-adapter` is running properly. Check the service pod and logs in the `kube-system` namespace. + + 1. Check that the service pod is `Running`. Enter the following command. + ``` + # kubectl get pods -n kube-system + ``` + From the resulting output, look for a status of `Running`. + ``` + NAME READY STATUS RESTARTS AGE + ... + prometheus-adapter-prometheus-adapter-568674d97f-hbzfx 1/1 Running 0 7h + ... + ``` + 1. Check the service logs to make sure the service is running correctly by entering the command that follows. + ``` + # kubectl logs prometheus-adapter-prometheus-adapter-568674d97f-hbzfx -n kube-system + ``` + Then review the log output to confirm the service is running. + +
+ Prometheus Adaptor Logs + + ... + I0724 10:18:45.696679 1 round_trippers.go:436] GET https://10.43.0.1:443/api/v1/namespaces/default/pods?labelSelector=app%3Dhello-world 200 OK in 2 milliseconds + I0724 10:18:45.696695 1 round_trippers.go:442] Response Headers: + I0724 10:18:45.696699 1 round_trippers.go:445] Date: Tue, 24 Jul 2018 10:18:45 GMT + I0724 10:18:45.696703 1 round_trippers.go:445] Content-Type: application/json + I0724 10:18:45.696706 1 round_trippers.go:445] Content-Length: 2581 + I0724 10:18:45.696766 1 request.go:836] Response Body: {"kind":"PodList","apiVersion":"v1","metadata":{"selfLink":"/api/v1/namespaces/default/pods","resourceVersion":"6237"},"items":[{"metadata":{"name":"hello-world-54764dfbf8-q6l82","generateName":"hello-world-54764dfbf8-","namespace":"default","selfLink":"/api/v1/namespaces/default/pods/hello-world-54764dfbf8-q6l82","uid":"484cb929-8f29-11e8-99d2-067cac34e79c","resourceVersion":"4066","creationTimestamp":"2018-07-24T10:06:50Z","labels":{"app":"hello-world","pod-template-hash":"1032089694"},"annotations":{"cni.projectcalico.org/podIP":"10.42.0.7/32"},"ownerReferences":[{"apiVersion":"extensions/v1beta1","kind":"ReplicaSet","name":"hello-world-54764dfbf8","uid":"4849b9b1-8f29-11e8-99d2-067cac34e79c","controller":true,"blockOwnerDeletion":true}]},"spec":{"volumes":[{"name":"default-token-ncvts","secret":{"secretName":"default-token-ncvts","defaultMode":420}}],"containers":[{"name":"hello-world","image":"rancher/hello-world","ports":[{"containerPort":80,"protocol":"TCP"}],"resources":{"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"default-token-ncvts","readOnly":true,"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}],"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File","imagePullPolicy":"Always"}],"restartPolicy":"Always","terminationGracePeriodSeconds":30,"dnsPolicy":"ClusterFirst","serviceAccountName":"default","serviceAccount":"default","nodeName":"34.220.18.140","securityContext":{},"schedulerName":"default-scheduler","tolerations":[{"key":"node.kubernetes.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":300},{"key":"node.kubernetes.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":300}]},"status":{"phase":"Running","conditions":[{"type":"Initialized","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"},{"type":"Ready","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:54Z"},{"type":"PodScheduled","status":"True","lastProbeTime":null,"lastTransitionTime":"2018-07-24T10:06:50Z"}],"hostIP":"34.220.18.140","podIP":"10.42.0.7","startTime":"2018-07-24T10:06:50Z","containerStatuses":[{"name":"hello-world","state":{"running":{"startedAt":"2018-07-24T10:06:54Z"}},"lastState":{},"ready":true,"restartCount":0,"image":"rancher/hello-world:latest","imageID":"docker-pullable://rancher/hello-world@sha256:4b1559cb4b57ca36fa2b313a3c7dde774801aa3a2047930d94e11a45168bc053","containerID":"docker://cce4df5fc0408f03d4adf82c90de222f64c302bf7a04be1c82d584ec31530773"}],"qosClass":"Burstable"}}]} + I0724 10:18:45.699525 1 api.go:74] GET http://prometheus-server.prometheus.34.220.18.140.xip.io/api/v1/query?query=sum%28rate%28container_fs_read_seconds_total%7Bpod_name%3D%22hello-world-54764dfbf8-q6l82%22%2Ccontainer_name%21%3D%22POD%22%2Cnamespace%3D%22default%22%7D%5B5m%5D%29%29+by+%28pod_name%29&time=1532427525.697 200 OK + I0724 10:18:45.699620 1 api.go:93] Response Body: {"status":"success","data":{"resultType":"vector","result":[{"metric":{"pod_name":"hello-world-54764dfbf8-q6l82"},"value":[1532427525.697,"0"]}]}} + I0724 10:18:45.699939 1 wrap.go:42] GET /apis/custom.metrics.k8s.io/v1beta1/namespaces/default/pods/%2A/fs_read?labelSelector=app%3Dhello-world: (12.431262ms) 200 [[kube-controller-manager/v1.10.1 (linux/amd64) kubernetes/d4ab475/system:serviceaccount:kube-system:horizontal-pod-autoscaler] 10.42.0.0:24268] + I0724 10:18:51.727845 1 request.go:836] Request Body: {"kind":"SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","metadata":{"creationTimestamp":null},"spec":{"nonResourceAttributes":{"path":"/","verb":"get"},"user":"system:anonymous","group":["system:unauthenticated"]},"status":{"allowed":false}} + ... +
+ + + +1. Check that the metrics API is accessible from kubectl. + + - If you are accessing the cluster directly, enter your Server URL in the kubectl config in the following format: `https://:6443`. + ``` + # kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 + ``` + If the API is accessible, you should receive output that's similar to what follows. + +
+ API Response + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
+ + - If you are accessing the cluster through Rancher, enter your Server URL in the kubectl config in the following format: `https:///k8s/clusters/`. Add the suffix `/k8s/clusters/` to API path. + ``` + # kubectl get --raw /k8s/clusters//apis/custom.metrics.k8s.io/v1beta1 + ``` + If the API is accessible, you should receive output that's similar to what follows. + +
+ API Response + + {"kind":"APIResourceList","apiVersion":"v1","groupVersion":"custom.metrics.k8s.io/v1beta1","resources":[{"name":"pods/fs_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_rss","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_period","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_read","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_user","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/last_seen","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/tasks_state","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_quota","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/start_time_seconds","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_write","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_cache","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_cfs_throttled_periods","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_working_set_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_udp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes_free","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_inodes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_time_weighted","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failures","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_swap","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_cpu_shares","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_swap_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_io_current","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_failcnt","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_writes_merged","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/network_tcp_usage","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/memory_max_usage_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/spec_memory_reservation_limit_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_load_average_10s","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/cpu_system","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_reads_bytes","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]},{"name":"pods/fs_sector_reads","singularName":"","namespaced":true,"kind":"MetricValueList","verbs":["get"]}]} +
diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md new file mode 100644 index 00000000000..adc57ae88ee --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md @@ -0,0 +1,55 @@ +--- +title: Managing HPAs with the Rancher UI +--- + + + + + +The Rancher UI supports creating, managing, and deleting HPAs. You can configure CPU or memory usage as the metric that the HPA uses to scale. + +If you want to create HPAs that scale based on other metrics than CPU and memory, refer to [Configuring HPA to Scale Using Custom Metrics with Prometheus](./manage-hpas-with-kubectl.md#configuring-hpa-to-scale-using-custom-metrics-with-prometheus). + +## Creating an HPA + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster you want to create an HPA in and click **Explore**. +1. In the left navigation bar, click **Service Discovery > HorizontalPodAutoscalers**. +1. Click **Create**. +1. Select a **Namespace** for the HPA. +1. Enter a **Name** for the HPA. +1. Select a **Target Reference** as scale target for the HPA. +1. Specify the **Minimum Replicas** and **Maximum Replicas** for the HPA. +1. Configure the metrics for the HPA. You can choose memory or CPU usage as the metric that will cause the HPA to scale the service up or down. In the **Quantity** field, enter the percentage of the workload's memory or CPU usage that will cause the HPA to scale the service. To configure other HPA metrics, including metrics available from Prometheus, you need to [manage HPAs using kubectl](./manage-hpas-with-kubectl.md#configuring-hpa-to-scale-using-custom-metrics-with-prometheus). + +1. Click **Create** to create the HPA. + +:::note Result: + +The HPA is deployed to the chosen namespace. You can view the HPA's status from the project's Resources > HPA view. + +::: + +## Get HPA Metrics and Status + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that has the HPA and click **Explore**. +1. In the left navigation bar, click **Service Discovery > HorizontalPodAutoscalers**. The **HorizontalPodAutoscalers** page shows the number of current replicas. + +For more detailed metrics and status of a specific HPA, click the name of the HPA. This leads to the HPA detail page. + + +## Deleting an HPA + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that has the HPA you want to delete and click **Explore**. +1. In the left navigation bar, click **Service Discovery > HorizontalPodAutoscalers**. +1. Click **Resources > HPA**. +1. Find the HPA which you would like to delete and click **⋮ > Delete**. +1. Click **Delete** to confirm. + +:::note Result: + +The HPA is deleted from the current cluster. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md new file mode 100644 index 00000000000..6607274c4ee --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md @@ -0,0 +1,534 @@ +--- +title: Testing HPAs with kubectl +--- + + + + + +This document describes how to check the status of your HPAs after scaling them up or down with your load testing tool. For information on how to check the status from the Rancher UI (at least version 2.3.x), refer to [Managing HPAs with the Rancher UI](./manage-hpas-with-kubectl.md). + +For HPA to work correctly, service deployments should have resources request definitions for containers. Follow this hello-world example to test if HPA is working correctly. + +1. Configure `kubectl` to connect to your Kubernetes cluster. + +1. Copy the `hello-world` deployment manifest below. + +
+ Hello World Manifest + + ``` + apiVersion: apps/v1beta2 + kind: Deployment + metadata: + labels: + app: hello-world + name: hello-world + namespace: default + spec: + replicas: 1 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + resources: + requests: + cpu: 500m + memory: 64Mi + ports: + - containerPort: 80 + protocol: TCP + restartPolicy: Always + --- + apiVersion: v1 + kind: Service + metadata: + name: hello-world + namespace: default + spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: hello-world + ``` + +
+ +1. Deploy it to your cluster. + + ``` + # kubectl create -f + ``` + +1. Copy one of the HPAs below based on the metric type you're using: + +
+ Hello World HPA: Resource Metrics + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 1000Mi + ``` + +
+
+ Hello World HPA: Custom Metrics + + ``` + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + metadata: + name: hello-world + namespace: default + spec: + scaleTargetRef: + apiVersion: extensions/v1beta1 + kind: Deployment + name: hello-world + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: 50 + - type: Resource + resource: + name: memory + targetAverageValue: 100Mi + - type: Pods + pods: + metricName: cpu_system + targetAverageValue: 20m + ``` + +
+ +1. View the HPA info and description. Confirm that metric data is shown. + +
+ Resource Metrics + + 1. Enter the following commands. + ``` + # kubectl get hpa + NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE + hello-world Deployment/hello-world 1253376 / 100Mi, 0% / 50% 1 10 1 6m + # kubectl describe hpa + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 20:21:16 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 1253376 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+
+ Custom Metrics + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive the output that follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:36:28 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 3514368 / 100Mi + "cpu_system" on pods: 0 / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True ReadyForNewScale the last scale time was sufficiently old as to warrant a new scale + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + ``` + +
+ +1. Generate a load for the service to test that your pods autoscale as intended. You can use any load-testing tool (Hey, Gatling, etc.), but we're using [Hey](https://github.com/rakyll/hey). + +1. Test that pod autoscaling works as intended.

+ **To Test Autoscaling Using Resource Metrics:** + +
+ Upscale to 2 Pods: CPU Usage Up to Target + + Use your load testing tool to scale up to two pods based on CPU Usage. + + 1. View your HPA. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10928128 / 100Mi + resource cpu on pods (as a percentage of request): 56% (280m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. Enter the following command to confirm you've scaled to two pods. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows: + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-k8ph2 1/1 Running 0 1m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ Upscale to 3 pods: CPU Usage Up to Target + + Use your load testing tool to upscale to 3 pods based on CPU usage with `horizontal-pod-autoscaler-upscale-delay` set to 3 minutes. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 9424896 / 100Mi + resource cpu on pods (as a percentage of request): 66% (333m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 4m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + ``` + 2. Enter the following command to confirm three pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-f46kh 0/1 Running 0 1m + hello-world-54764dfbf8-k8ph2 1/1 Running 0 5m + hello-world-54764dfbf8-q6l4v 1/1 Running 0 3h + ``` + +
+
+ Downscale to 1 Pod: All Metrics Below Target + + Use your load testing to scale down to 1 pod when all metrics are below target for `horizontal-pod-autoscaler-downscale-delay` (5 minutes by default). + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Mon, 23 Jul 2018 22:22:04 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 10070016 / 100Mi + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 6m horizontal-pod-autoscaler New size: 3; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 1s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + +
+ + **To Test Autoscaling Using Custom Metrics:** + +
+ Upscale to 2 Pods: CPU Usage Up to Target + + Use your load testing tool to upscale two pods based on CPU usage. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8159232 / 100Mi + "cpu_system" on pods: 7m / 20m + resource cpu on pods (as a percentage of request): 64% (321m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 2 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 16s horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + ``` + 1. Enter the following command to confirm two pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ Upscale to 3 Pods: CPU Usage Up to Target + + Use your load testing tool to scale up to three pods when the cpu_system usage limit is up to target. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows: + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3s horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + ``` + 1. Enter the following command to confirm three pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows: + ``` + # kubectl get pods + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-5pfdr 1/1 Running 0 3m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ Upscale to 4 Pods: CPU Usage Up to Target + + Use your load testing tool to upscale to four pods based on CPU usage. `horizontal-pod-autoscaler-upscale-delay` is set to three minutes by default. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive output similar to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8374272 / 100Mi + "cpu_system" on pods: 27m / 20m + resource cpu on pods (as a percentage of request): 71% (357m) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 3 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from cpu resource utilization (percentage of request) + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 3m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 4s horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + ``` + 1. Enter the following command to confirm four pods are running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-2p9xb 1/1 Running 0 5m + hello-world-54764dfbf8-5pfdr 1/1 Running 0 2m + hello-world-54764dfbf8-m2hrl 1/1 Running 0 1s + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
+
+ Downscale to 1 Pod: All Metrics Below Target + + Use your load testing tool to scale down to one pod when all metrics below target for `horizontal-pod-autoscaler-downscale-delay`. + + 1. Enter the following command. + ``` + # kubectl describe hpa + ``` + You should receive similar output to what follows. + ``` + Name: hello-world + Namespace: default + Labels: + Annotations: + CreationTimestamp: Tue, 24 Jul 2018 18:01:11 +0200 + Reference: Deployment/hello-world + Metrics: ( current / target ) + resource memory on pods: 8101888 / 100Mi + "cpu_system" on pods: 8m / 20m + resource cpu on pods (as a percentage of request): 0% (0) / 50% + Min replicas: 1 + Max replicas: 10 + Conditions: + Type Status Reason Message + ---- ------ ------ ------- + AbleToScale True SucceededRescale the HPA controller was able to update the target scale to 1 + ScalingActive True ValidMetricFound the HPA was able to successfully calculate a replica count from memory resource + ScalingLimited False DesiredWithinRange the desired count is within the acceptable range + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal SuccessfulRescale 10m horizontal-pod-autoscaler New size: 2; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 8m horizontal-pod-autoscaler New size: 3; reason: pods metric cpu_system above target + Normal SuccessfulRescale 5m horizontal-pod-autoscaler New size: 4; reason: cpu resource utilization (percentage of request) above target + Normal SuccessfulRescale 13s horizontal-pod-autoscaler New size: 1; reason: All metrics below target + ``` + 1. Enter the following command to confirm a single pods is running. + ``` + # kubectl get pods + ``` + You should receive output similar to what follows. + ``` + NAME READY STATUS RESTARTS AGE + hello-world-54764dfbf8-q6l82 1/1 Running 0 6h + ``` + +
diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md new file mode 100644 index 00000000000..fb19bea8379 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md @@ -0,0 +1,182 @@ +--- +title: Kubernetes Registry and Container Image Registry +description: Learn about the container image registry and Kubernetes registry, their use cases, and how to use a private registry with the Rancher UI +--- + + + + + +Registries are Kubernetes secrets containing credentials used to authenticate with [private container registries](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). + +The word "registry" can mean two things, depending on whether it is used to refer to a container or Kubernetes registry: + +- A **Container image registry** (formerly "Docker registry") contains container images that you can pull and deploy. The registry is a stateless, scalable server side application that stores and lets you distribute container images. +- The **Kubernetes registry** is an image pull secret that your deployment uses to authenticate with an image registry. + +Deployments use the Kubernetes registry secret to authenticate with a private image registry and then pull a container image hosted on it. + +Currently, deployments pull the private registry credentials automatically only if the workload is created in the Rancher UI and not when it is created via kubectl. + +## Creating a Registry in Namespaces + +:::note Prerequisite: + +You must have an available private registry already deployed. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a registry and click **Explore**. +1. In the left navigation, click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Click **Create**. +1. Click **Registry**. +1. Enter a **Name** for the registry. + + :::note + + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a project or namespace can have duplicate names. Therefore, to prevent conflicts, your registry must have a unique name among all secrets within your workspace. + + ::: + +1. Select a namespace for the registry. +1. Select the website that hosts your private registry. Then enter credentials that authenticate with the registry. For example, if you use DockerHub, provide your DockerHub username and password. +1. Click **Save**. + +**Result:** + +- Your secret is added to the namespace you chose. +- You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. +- Any workload that you create in the Rancher UI will have the credentials to access the registry if the workload is within the registry's scope. + +## Creating a Registry in Projects + +:::note Prerequisites: + +You must have an available private registry already deployed. + +If you need to create a private registry, refer to the documentation pages for your respective runtime: + +* [Containerd](https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration). + * [Nerdctl commands and managed registry services](https://github.com/containerd/nerdctl/blob/main/docs/registry.md). +* [Docker](https://docs.docker.com/registry/deploying/). + +::: + +Before v2.6, secrets were required to be in a project scope. Projects are no longer required, and you may use the namespace scope instead. As a result, the Rancher UI was updated to reflect this new functionality. However, you may still create a project-scoped registry if desired. Use the following steps to do so: + +1. In the upper left corner, click **☰ > Global Settings** in the dropdown. +1. Click **Feature Flags**. +1. Go to the `legacy` feature flag and click **Activate**. +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a registry and click **Explore**. +1. In the left navigation, click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Click **Create**. +1. Click **Registry**. +1. In the top navigation bar, filter to see only one project. +1. Enter a **Name** for the registry. + + :::note + + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a project or namespace can have duplicate names. Therefore, to prevent conflicts, your registry must have a unique name among all secrets within your workspace. + + ::: + +1. Select a namespace for the registry. +1. Select the website that hosts your private registry. Then enter credentials that authenticate with the registry. For example, if you use DockerHub, provide your DockerHub username and password. +1. Click **Save**. + +**Result:** + +- Your secret is added to the individual project you chose. +- You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. +- Any workload that you create in the Rancher UI will have the credentials to access the registry if the workload is within the registry's scope. + +:::note + +Project-scoped registries on the local cluster are only visible when a single project is selected. + +::: + +## Using a Private Registry + +You can deploy a workload with an image from a private registry through the Rancher UI, or with `kubectl`. + +### Using the Private Registry with the Rancher UI + +To deploy a workload with an image from your private registry, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to deploy a workload and click **Explore**. +1. Click **Workload**. +1. Click **Create**. +1. Select the type of workload you want to create. +1. Enter a unique name for the workload and choose a namespace. +1. In the **Container Image** field, enter the URL of the path to the image in your private registry. For example, if your private registry is on Quay.io, you could use `quay.io//`. +1. Click **Create**. + +**Result:** Your deployment should launch, authenticate using the private registry credentials you added in the Rancher UI, and pull the container image that you specified. + +### Using the Private Registry with kubectl + +When you create the workload using `kubectl`, you need to configure the pod so that its YAML has the path to the image in the private registry. You also have to create and reference the registry secret because the pod only automatically gets access to the private registry credentials if it is created in the Rancher UI. + +The secret has to be created in the same namespace where the workload gets deployed. + +Below is an example `pod.yml` for a workload that uses an image from a private registry. In this example, the pod uses an image from Quay.io, and the .yml specifies the path to the image. The pod authenticates with the registry using credentials stored in a Kubernetes secret called `testquay`, which is specified in `spec.imagePullSecrets` in the `name` field: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: private-reg +spec: + containers: + - name: private-reg-container + image: quay.io// + imagePullSecrets: + - name: testquay +``` + +In this example, the secret named `testquay` is in the default namespace. + +You can use `kubectl` to create the secret with the private registry credentials. This command creates the secret named `testquay`: + +``` +kubectl create secret docker-registry testquay \ + --docker-server=quay.io \ + --docker-username= \ + --docker-password= +``` + +To see how the secret is stored in Kubernetes, you can use this command: + +``` +kubectl get secret testquay --output="jsonpath={.data.\.dockerconfigjson}" | base64 --decode +``` + +The result looks like this: + +``` +{"auths":{"quay.io":{"username":"","password":"","auth":"c291bXlhbGo6dGVzdGFiYzEyMw=="}}} +``` + +After the workload is deployed, you can check if the image was pulled successfully: + +``` +kubectl get events +``` +The result should look like this: +``` +14s Normal Scheduled Pod Successfully assigned default/private-reg2 to minikube +11s Normal Pulling Pod pulling image "quay.io//" +10s Normal Pulled Pod Successfully pulled image "quay.io//" +``` + +For more information, refer to the Kubernetes documentation on [creating a pod that uses your secret.](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md new file mode 100644 index 00000000000..c102ee9550f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md @@ -0,0 +1,22 @@ +--- +title: Adding Ingresses +description: Ingresses can be added for workloads to provide load balancing, SSL termination and host/path-based routing. Learn how to add Rancher ingress +--- + + + + + +Ingresses can be added for workloads to provide load balancing, SSL termination and host/path based routing. When using ingresses in a project, you can program the ingress hostname to an external DNS by setting up a Global DNS entry. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you want to add an ingress to and click **Explore**. +1. Click **Service Discovery > Ingresses**. +1. Click **Create**. +1. Select an existing **Namespace** from the drop-down list. +1. Enter a **Name** for the ingress. +1. [Create](./ingress-configuration.md) ingress forwarding **Rules**. If any of your ingress rules handle requests for encrypted ports, add a certificate to encrypt/decrypt communications. +1. **Optional:** click **Add Rule** to create additional ingress rules. For example, after you create ingress rules to direct requests for your hostname, you'll likely want to create a default backend to handle 404s. +1. Click **Create** at the bottom right. + +**Result:** Your ingress is added to the project. The ingress begins enforcing your ingress rules. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md new file mode 100644 index 00000000000..47ed1d03fb1 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration.md @@ -0,0 +1,46 @@ +--- +title: Ingress Configuration +description: Ingress configuration +--- + + + + + +:::note + +For Kubernetes v1.21 and up, the NGINX Ingress controller no longer runs in hostNetwork by default. It instead uses hostPorts for port 80 and port 443, so you can configure the admission webhook to be accessible only through the ClusterIP. This ensures that the webhook is only accessible from within the cluster. + +Because of this change to the controller, the default RKE1 behavior no longer sets `hostNetwork` to `true`. However, you must set `hostNetwork` to `true` for TCP- and UDP-based Services to work. To do so, [edit](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#editing-clusters-with-yaml) the cluster's YAML and follow the steps in the [official RKE1 doccumention](https://rke.docs.rancher.com/config-options/add-ons/ingress-controllers#configuring-network-options). + + +::: + +## Specify a hostname to use + +If you use this option, Ingress routes requests for a hostname to the service or workload that you specify. + +1. Specify a path of type `Prefix` and specify a path such as `/`. +1. Add a **Target Service**. +1. **Optional:** If you want to specify a workload or service when a request is sent to a particular hostname path, add a **Path** for the target. For example, if you want requests for `www.mysite.com/contact-us` to be sent to a different service than `www.mysite.com`, enter `/contact-us` in the **Path** field. Typically, the first rule that you create does not include a path. +1. Enter the **Port** number that each target operates on. + +## Certificates + +:::note + +You must have an SSL certificate that Ingress can use to encrypt and decrypt communications. For more information, see [Adding SSL Certificates](../encrypt-http-communication.md). + +::: + +1. To create an Ingress controller, click the **Certificates** tab. +1. Click **Add Certificate**. +1. Select a **Certificate - Secret Name** from the drop-down list. +1. Enter the host using encrypted communication. +1. To add additional hosts that use the certificate, click **Add Hosts**. + +## Labels and Annotations + +Add [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) and/or [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to provide metadata for your Ingress controller. + +For a list of annotations available for use, see the [Nginx Ingress Controller Documentation](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md new file mode 100644 index 00000000000..e0d25c169e6 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md @@ -0,0 +1,72 @@ +--- +title: "Layer 4 and Layer 7 Load Balancing" +description: "Kubernetes supports load balancing in two ways: Layer-4 Load Balancing and Layer-7 Load Balancing. Learn about the support for each way in different deployments" +--- + + + + + +Kubernetes supports load balancing in two ways: Layer-4 Load Balancing and Layer-7 Load Balancing. + +## Layer-4 Load Balancer + +Layer-4 load balancer (or the external load balancer) forwards traffic to Nodeports. Layer-4 load balancer allows you to forward both HTTP and TCP traffic. + +Often, the Layer-4 load balancer is supported by the underlying cloud provider, so when you deploy RKE clusters on bare-metal servers and vSphere clusters, Layer-4 load balancer is not supported. However, a single [globally managed config-map](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) can be used to expose services on NGINX or third-party ingress. + +:::note + +It is possible to deploy a cluster with a non-cloud load balancer, such as [MetalLB.](https://metallb.universe.tf/) However, that use case is more advanced than the Layer-4 load balancer supported by a cloud provider, and it is not configurable in Rancher or RKE. + +::: + +### Support for Layer-4 Load Balancing + +Support for layer-4 load balancer varies based on the underlying cloud provider. + +Cluster Deployment | Layer-4 Load Balancer Support +----------------------------------------------|-------------------------------- +Amazon EKS | Supported by AWS cloud provider +Google GKE | Supported by GCE cloud provider +Azure AKS | Supported by Azure cloud provider +RKE on EC2 | Supported by AWS cloud provider +RKE on DigitalOcean | Limited NGINX or third-party Ingress* +RKE on vSphere | Limited NGINX or third party-Ingress* +RKE on Custom Hosts
(e.g. bare-metal servers) | Limited NGINX or third-party Ingress* +Third-party MetalLB | Limited NGINX or third-party Ingress* + +\* Services can be exposed through a single [globally managed config-map.](https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/) + +## Layer-7 Load Balancer + +Layer-7 load balancer (or the ingress controller) supports host and path-based load balancing and SSL termination. Layer-7 load balancer only forwards HTTP and HTTPS traffic and therefore they listen on ports 80 and 443 only. Cloud providers such as Amazon and Google support layer-7 load balancer. In addition, RKE clusters deploys the Nginx Ingress Controller. + +### Support for Layer-7 Load Balancing + +Support for layer-7 load balancer varies based on the underlying cloud provider. + +Cluster Deployment | Layer-7 Load Balancer Support +----------------------------------------------|-------------------------------- +Amazon EKS | Supported by AWS cloud provider +Google GKE | Supported by GKE cloud provider +Azure AKS | Not Supported +RKE on EC2 | Nginx Ingress Controller +RKE on DigitalOcean | Nginx Ingress Controller +RKE on vSphere | Nginx Ingress Controller +RKE on Custom Hosts
(e.g. bare-metal servers) | Nginx Ingress Controller + +### Host Names in Layer-7 Load Balancer + +Some cloud-managed layer-7 load balancers (such as the ALB ingress controller on AWS) expose DNS addresses for ingress rules. You need to map (via CNAME) your domain name to the DNS address generated by the layer-7 load balancer. + +Other layer-7 load balancers, such as the Google Load Balancer or Nginx Ingress Controller, directly expose one or more IP addresses. Google Load Balancer provides a single routable IP address. Nginx Ingress Controller exposes the external IP of all nodes that run the Nginx Ingress Controller. You can do either of the following: + +1. Configure your own DNS to map (via A records) your domain name to the IP addresses exposes by the Layer-7 load balancer. +2. Ask Rancher to generate an xip.io host name for your ingress rule. Rancher will take one of your exposed IPs, say `a.b.c.d`, and generate a host name `..a.b.c.d.xip.io`. + +The benefit of using xip.io is that you obtain a working entrypoint URL immediately after you create the ingress rule. Setting up your own domain name, on the other hand, requires you to configure DNS servers and wait for DNS to propagate. + +## Related Links + +- [Create an External Load Balancer](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md new file mode 100644 index 00000000000..89e4a40ba5d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md @@ -0,0 +1,82 @@ +--- +title: Secrets +--- + + + + + +[Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#overview-of-secrets) store sensitive data like passwords, tokens, or keys. They may contain one or more key value pairs. + +:::note + +This page is about secrets in general. For details on setting up a private registry, refer to the section on [registries.](kubernetes-and-docker-registries.md) + +::: + +When configuring a workload, you'll be able to choose which secrets to include. Like config maps, secrets can be referenced by workloads as either an environment variable or a volume mount. + +Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) + +## Creating Secrets in Namespaces + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a secret and click **Explore**. +1. To navigate to secrets, you may click either **Storage > Secrets** or **More Resources > Core > Secrets**. +1. Click **Create**. +1. Select the type of secret you want to create. +1. Select a **Namespace** for the secret. +1. Enter a **Name** for the secret. + + :::note + + Kubernetes classifies secrets, certificates, and registries all as [secrets](https://kubernetes.io/docs/concepts/configuration/secret/), and no two secrets in a namespace can have duplicate names. Therefore, to prevent conflicts, your secret must have a unique name among all secrets within your workspace. + + ::: + +1. From **Data**, click **Add** to add a key-value pair. Add as many values as you need. + + :::tip + + You can add multiple key value pairs to the secret by copying and pasting. + + ::: + + ![](/img/bulk-key-values.gif) + +1. Click **Save**. + +**Result:** Your secret is added to the namespace you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. + +Mounted secrets will be updated automatically unless they are mounted as subpath volumes. For details on how updated secrets are propagated, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) + + +## Creating Secrets in Projects + +Before v2.6, secrets were required to be in a project scope. Projects are no longer required, and you may use the namespace scope instead. As a result, the Rancher UI was updated to reflect this new functionality. However, you may still create project-scoped secrets if desired. Note that you have to first enable the `legacy` feature flag and look at a single project to do so. Use the following steps to set up your project-level secret: + +1. In the upper left corner, click **☰ > Global Settings** in the dropdown. +1. Click **Feature Flags**. +1. Go to the `legacy` feature flag and click **Activate**. +1. In the upper left corner, click **☰ > Cluster Management** in the dropdown. +1. Go to the cluster that you created and click **Explore.** +1. Click **Legacy > Projects**. +1. In the top navigation bar, filter to see only one project. +1. In the left navigation bar, click **Secrets**. +1. Click **Add Secret**. + +**Result:** Your secret is added to the individual project you chose. You can view the secret in the Rancher UI by clicking either **Storage > Secrets** or **More Resources > Core > Secrets**. + +Since project-scoped secrets are set at the project level, any changes made at the namespace level will be overwritten. + +:::note + +Project-scoped secrets on the local cluster are only visible when a single project is selected. + +::: + +## What's Next? + +Now that you have a secret added to a namespace, you can add it to a workload that you deploy. + +For more information on adding secret to a workload, see [Deploying Workloads](workloads-and-pods/deploy-workloads.md). diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md new file mode 100644 index 00000000000..e5dc6e4b503 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md @@ -0,0 +1,39 @@ +--- +title: Adding a Sidecar +--- + + + + + +A _sidecar_ is a container that extends or enhances the main container in a pod. The main container and the sidecar share a pod, and therefore share the same network space and storage. You can add sidecars to existing workloads by using the **Add a Sidecar** option. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a sidecar and click **Explore**. +1. In the left navigation bar, click **Workload**. + +1. Find the workload that you want to extend. Select **⋮ > + Add Sidecar**. + +1. Enter a **Name** for the sidecar. + +1. In the **General** section, select a sidecar type. This option determines if the sidecar container is deployed before or after the main container is deployed. + + - **Standard Container:** + + The sidecar container is deployed after the main container. + + - **Init Container:** + + The sidecar container is deployed before the main container. + +1. From the **Container Image** field, enter the name of the container image that you want to deploy in support of the main container. During deployment, Rancher pulls this image from [Docker Hub](https://hub.docker.com/explore/). Enter the name exactly as it appears on Docker Hub. + +1. Set the remaining options. You can read about them in [Deploying Workloads](deploy-workloads.md). + +1. Click **Launch**. + +**Result:** The sidecar is deployed according to your parameters. Following its deployment, you can view the sidecar by selecting **⋮ icon (...) > Edit** for the main deployment. + +## Related Links + +- [The Distributed System ToolKit: Patterns for Composite Containers](https://kubernetes.io/blog/2015/06/the-distributed-system-toolkit-patterns/) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md new file mode 100644 index 00000000000..82d495cce05 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md @@ -0,0 +1,61 @@ +--- +title: Deploying Workloads +description: Read this step by step guide for deploying workloads. Deploy a workload to run an application in one or more containers. +--- + + + + + +Deploy a workload to run an application in one or more containers. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to upgrade a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Click **Create**. +1. Choose the type of workload. +1. Select the namespace where the workload will be deployed. +1. Enter a **Name** for the workload. + +1. From the **Container Image** field, enter the name of the Docker image that you want to deploy to the project, optionally prefacing it with the registry host (e.g. `quay.io`, `registry.gitlab.com`, etc.). During deployment, Rancher pulls this image from the specified public or private registry. If no registry host is provided, Rancher will pull the image from [Docker Hub](https://hub.docker.com/explore/). Enter the name exactly as it appears in the registry server, including any required path, and optionally including the desired tag (e.g. `registry.gitlab.com/user/path/image:tag`). If no tag is provided, the `latest` tag will be automatically used. + +1. Either select an existing namespace, or click **Add to a new namespace** and enter a new namespace. + +1. Click **Add Port** to enter a port mapping, which enables access to the application inside and outside of the cluster . For more information, see [Services](../../../../pages-for-subheaders/workloads-and-pods.md#services). + +1. Configure the remaining options: + + - **Environment Variables** + + Use this section to either specify environment variables for your workload to consume on the fly, or to pull them from another source, such as a secret or [ConfigMap](../configmaps.md). + + - **Node Scheduling** + - **Health Check** + - **Volumes** + + Use this section to add storage for your workload. You can manually specify the volume that you want to add, use a persistent volume claim to dynamically create a volume for the workload, or read data for a volume to use from a file such as a [ConfigMap](../configmaps.md). + + When you are deploying a Stateful Set, you should use a Volume Claim Template when using Persistent Volumes. This will ensure that Persistent Volumes are created dynamically when you scale your Stateful Set. + + - **Scaling/Upgrade Policy** + + :::note Amazon Note for Volumes: + + To mount an Amazon EBS volume: + + - In [Amazon AWS](https://aws.amazon.com/), the nodes must be in the same Availability Zone and possess IAM permissions to attach/unattach volumes. + + - The cluster must be using the [AWS cloud provider](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#aws) option. For more information on enabling this option see [Creating an Amazon EC2 Cluster](../../launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) or [Creating a Custom Cluster](../../../../pages-for-subheaders/use-existing-nodes.md). + + ::: + +1. Click **Show Advanced Options** and configure: + + - **Command** + - **Networking** + - **Labels & Annotations** + - **Security and Host Config** + +1. Click **Launch**. + +**Result:** The workload is deployed to the chosen namespace. You can view the workload's status from the project's **Workloads** view. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md new file mode 100644 index 00000000000..e8f87d33055 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md @@ -0,0 +1,18 @@ +--- +title: Rolling Back Workloads +--- + + + + + +Sometimes there is a need to rollback to the previous version of the application, either for debugging purposes or because an upgrade did not go as planned. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to upgrade a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Find the workload that you want to rollback and select **⋮ > Rollback**. + +1. Choose the revision that you want to roll back to. Click **Rollback**. + +**Result:** Your workload reverts to the previous version that you chose. Wait a few minutes for the action to complete. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md new file mode 100644 index 00000000000..05a51a640bf --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md @@ -0,0 +1,25 @@ +--- +title: Upgrading Workloads +--- + + + + + +When a new version of an application image is released on Docker Hub, you can upgrade any workloads running a previous version of the application to the new one. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster where you want to upgrade a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. + +1. Find the workload that you want to upgrade and select **⋮ > Edit Config**. + +1. Update the **Container Image** and any options that you want to change. + +1. Review and edit the workload's **Scaling and Upgrade Policy**. + + These options control how the upgrade rolls out to containers that are currently running. For example, for scalable deployments, you can choose whether you want to stop old pods before deploying new ones, or vice versa, as well as the upgrade batch size. + +1. Click **Save**. + +**Result:** The workload begins upgrading its containers, per your specifications. Note that scaling up the deployment or updating the upgrade/scaling policy won't result in the pods recreation. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md new file mode 100644 index 00000000000..bd90885720a --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md @@ -0,0 +1,47 @@ +--- +title: Rancher Agents +--- + + + + + +There are two different agent resources deployed on Rancher managed clusters: + +- [cattle-cluster-agent](#cattle-cluster-agent) +- [cattle-node-agent](#cattle-node-agent) + +For a conceptual overview of how the Rancher server provisions clusters and communicates with them, refer to the [architecture](../../../pages-for-subheaders/rancher-manager-architecture.md). + +### cattle-cluster-agent + +The `cattle-cluster-agent` is used to connect to the Kubernetes API of [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters. The `cattle-cluster-agent` is deployed using a Deployment resource. + +### cattle-node-agent + +The `cattle-node-agent` is used to interact with nodes in a [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) cluster when performing cluster operations. Examples of cluster operations are upgrading Kubernetes version and creating/restoring etcd snapshots. The `cattle-node-agent` is deployed using a DaemonSet resource to make sure it runs on every node. The `cattle-node-agent` is used as fallback option to connect to the Kubernetes API of [Rancher Launched Kubernetes](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters when `cattle-cluster-agent` is unavailable. + +### Scheduling rules + +The `cattle-cluster-agent` uses either a fixed set of tolerations, or dynamically-added tolerations based on taints applied to the control plane nodes. This structure allows [Taint based Evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/#taint-based-evictions) to work properly for `cattle-cluster-agent`. + +If control plane nodes are present in the cluster, the default tolerations will be replaced with tolerations matching the taints on the control plane nodes. The default set of tolerations are described below. + +| Component | nodeAffinity nodeSelectorTerms | nodeSelector | Tolerations | +| ---------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| `cattle-cluster-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | **Note:** These are the default tolerations, and will be replaced by tolerations matching taints applied to controlplane nodes.

`effect:NoSchedule`
`key:node-role.kubernetes.io/controlplane`
`value:true`

`effect:NoSchedule`
`key:node-role.kubernetes.io/control-plane`
`operator:Exists`

`effect:NoSchedule`
`key:node-role.kubernetes.io/master`
`operator:Exists` | +| `cattle-node-agent` | `beta.kubernetes.io/os:NotIn:windows` | none | `operator:Exists` | + +The `cattle-cluster-agent` Deployment has preferred scheduling rules using `preferredDuringSchedulingIgnoredDuringExecution`, favoring to be scheduled on nodes with the `controlplane` node. When there are no controlplane nodes visible in the cluster (this is usually the case when using [Clusters from Hosted Kubernetes Providers](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md)), you can add the label `cattle.io/cluster-agent=true` on a node to prefer scheduling the `cattle-cluster-agent` pod to that node. + +See [Kubernetes: Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) to find more information about scheduling rules. + +The `preferredDuringSchedulingIgnoredDuringExecution` configuration is shown in the table below: + +| Weight | Expression | +| ------ | ------------------------------------------------ | +| 100 | `node-role.kubernetes.io/controlplane:In:"true"` | +| 100 | `node-role.kubernetes.io/control-plane:In:"true"` | +| 100 | `node-role.kubernetes.io/master:In:"true"` | +| 1 | `cattle.io/cluster-agent:In:"true"` | + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md new file mode 100644 index 00000000000..738178fdc41 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences.md @@ -0,0 +1,35 @@ +--- +title: Behavior Differences Between RKE1 and RKE2 +--- + + + + + +RKE2, also known as RKE Government, is a Kubernetes distribution that focuses on security and compliance for U.S. Federal Government entities. It is considered the next iteration of the Rancher Kubernetes Engine, now known as RKE1. + +RKE1 and RKE2 have several slight behavioral differences to note, and this page will highlight some of these at a high level. + +### Control Plane Components + +RKE1 uses Docker for deploying and managing control plane components, and it also uses Docker as the container runtime for Kubernetes. By contrast, RKE2 launches control plane components as static pods that are managed by the kubelet. RKE2's container runtime is containerd, which allows things such as mirroring a container image registry. RKE1 with Docker does not allow mirroring. + +### Cluster API + +RKE2/K3s provisioning is built on top of the Cluster API (CAPI) upstream framework which often makes RKE2-provisioned clusters behave differently than RKE1-provisioned clusters. + +When you make changes to your cluster configuration in RKE2, this **may** result in nodes reprovisioning. This is controlled by CAPI controllers and not by Rancher itself. Note that for etcd nodes, the same behavior does not apply. + +The following are some specific example configuration changes that may cause the described behavior: + +- When editing the cluster and enabling `drain before delete`, the existing control plane nodes and worker are deleted and new nodes are created. + +Users who are used to RKE1 provisioning should take note of this new RKE2 behavior which may be unexpected. + +### Terminology + +You will notice that some terms have changed or gone away going from RKE1 to RKE2. For example, in RKE1 provisioning, you use **node templates**; in RKE2 provisioning, you can configure your cluster node pools when creating or editing the cluster. Another example is that the term **node pool** in RKE1 is now known as **machine pool** in RKE2. + + + + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md new file mode 100644 index 00000000000..bef31ab3f23 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md @@ -0,0 +1,103 @@ +--- +title: Creating a DigitalOcean Cluster +--- + + + + + +In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in DigitalOcean. + +First, you will set up your DigitalOcean cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in DigitalOcean. + +Then you will create a DigitalOcean cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + + + + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **DigitalOcean**. +1. Enter your Digital Ocean credentials. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials + +Creating a [node template](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for DigitalOcean will allow Rancher to provision new nodes in DigitalOcean. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Add Template**. +1. Click **DigitalOcean**. +1. Fill out a node template for DigitalOcean. For help filling out the form, refer to [DigitalOcean Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md) + +### 3. Create a cluster with node pools using the node template + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **DigitalOcean**. +1. Enter a **Cluster Name**. +1. Add one or more node pools to your cluster. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) +1. **In the Cluster Configuration** section, choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +### 1. Create your cloud credentials + +If you already have a set of cloud credentials to use, skip this section. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **DigitalOcean**. +1. Enter your Digital Ocean credentials. +1. Click **Create**. + +### 2. Create your cluster + +Use Rancher to create a Kubernetes cluster in DigitalOcean. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Toggle the switch to **RKE2/K3s**. +1. Click **DigitalOcean**. +1. Select a **Cloud Credential**, if more than one exists. Otherwise, it's preselected. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to the [DigitalOcean machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md new file mode 100644 index 00000000000..5bc7d109825 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md @@ -0,0 +1,287 @@ +--- +title: Creating an Amazon EC2 Cluster +description: Learn the prerequisites and steps required in order for you to create an Amazon EC2 cluster using Rancher +--- + + + + + +In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Amazon EC2. + +First, you will set up your EC2 cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in EC2. + +Then you will create an EC2 cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +### Prerequisites + +- **AWS EC2 Access Key and Secret Key** that will be used to create the instances. See [Amazon Documentation: Creating Access Keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) how to create an Access Key and Secret Key. +- **IAM Policy created** to add to the user of the Access Key And Secret Key. See [Amazon Documentation: Creating IAM Policies (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start) how to create an IAM policy. See our three example JSON policies below: + - [Example IAM Policy](#example-iam-policy) + - [Example IAM Policy with PassRole](#example-iam-policy-with-passrole) (needed if you want to use [Kubernetes Cloud Provider](../../../../pages-for-subheaders/set-up-cloud-providers.md) or want to pass an IAM Profile to an instance) + - [Example IAM Policy to allow encrypted EBS volumes](#example-iam-policy-to-allow-encrypted-ebs-volumes) +- **IAM Policy added as Permission** to the user. See [Amazon Documentation: Adding Permissions to a User (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console) how to attach it to an user. + +## Creating an EC2 Cluster + +The steps to create a cluster differ based on your Rancher version. + + + + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials and information from EC2](#2-create-a-node-template-with-your-cloud-credentials-and-information-from-ec2) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Amazon**. +1. Enter a name for the cloud credential. +1. In the **Default Region** field, select the AWS region where your cluster nodes will be located. +1. Enter your AWS EC2 **Access Key** and **Secret Key**. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials and information from EC2 + +Creating a [node template](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for EC2 will allow Rancher to provision new nodes in EC2. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates** +1. Click **Add Template**. +1. Fill out a node template for EC2. For help filling out the form, refer to [EC2 Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md) +1. Click **Create**. + + :::note + + If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. + + ::: + +### 3. Create a cluster with node pools using the node template + +Add one or more node pools to your cluster. For more information about node pools, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Amazon EC2**. +1. Create a node pool for each Kubernetes role. For each node pool, choose a node template that you created. For more information about node pools, including best practices for assigning Kubernetes roles to them, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) +1. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. Refer to [Selecting Cloud Providers](../../../../pages-for-subheaders/set-up-cloud-providers.md) to configure the Kubernetes Cloud Provider. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) + + :::note + + If you want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) that must be taken into consideration. + + ::: + +1. Click **Create**. + + + + +### 1. Create your cloud credentials + +If you already have a set of cloud credentials to use, skip this section. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Amazon**. +1. Enter a name for the cloud credential. +1. In the **Default Region** field, select the AWS region where your cluster nodes will be located. +1. Enter your AWS EC2 **Access Key** and **Secret Key**. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create your cluster + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Toggle the switch to **RKE2/K3s**. +1. Click **Amazon EC2**. +1. Select a **Cloud Credential**, if more than one exists. Otherwise, it's preselected. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to [the EC2 machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +### Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. + +## IAM Policies + +### Example IAM Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` + +### Example IAM Policy with PassRole + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:Describe*", + "ec2:ImportKeyPair", + "ec2:CreateKeyPair", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteKeyPair", + "ec2:ModifyInstanceMetadataOptions" + ], + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "iam:PassRole", + "ec2:RunInstances" + ], + "Resource": [ + "arn:aws:ec2:REGION::image/ami-*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:placement-group/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:subnet/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:key-pair/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:network-interface/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:security-group/*", + "arn:aws:iam::AWS_ACCOUNT_ID:role/YOUR_ROLE_NAME" + ] + }, + { + "Sid": "VisualEditor2", + "Effect": "Allow", + "Action": [ + "ec2:RebootInstances", + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances" + ], + "Resource": "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*" + } + ] +} +``` +### Example IAM Policy to allow encrypted EBS volumes +``` json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:Encrypt", + "kms:DescribeKey", + "kms:CreateGrant", + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteTags", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:CreateSnapshot" + ], + "Resource": [ + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:volume/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:instance/*", + "arn:aws:ec2:REGION:AWS_ACCOUNT_ID:snapshot/*", + "arn:aws:kms:REGION:AWS_ACCOUNT_ID:key/KMS_KEY_ID" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeTags", + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots" + ], + "Resource": "*" + } + ] +} +``` diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md new file mode 100644 index 00000000000..758616b057d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md @@ -0,0 +1,145 @@ +--- +title: Creating an Azure Cluster +--- + + + + + +In this section, you'll learn how to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Azure through Rancher. + +First, you will set up your Azure cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision new nodes in Azure. + +Then you will create an Azure cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +:::caution + +When the Rancher RKE cluster is running in Azure and has an Azure load balancer in front, the outbound flow will fail. The workaround for this problem is as follows: + +- Terminate the SSL/TLS on the internal load balancer +- Use the L7 load balancer + +For more information, refer to the documentation on [Azure load balancer limitations](https://docs.microsoft.com/en-us/azure/load-balancer/components#limitations). + +::: + +For more information on configuring the Kubernetes cluster that Rancher will install on the Azure nodes, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) + +For more information on configuring Azure node templates, refer to the [Azure node template configuration reference.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) + +- [Preparation in Azure](#preparation-in-azure) +- [Creating an Azure Cluster](#creating-an-azure-cluster) + +## Preparation in Azure + +Before creating a node template in Rancher using a cloud infrastructure such as Azure, we must configure Rancher to allow the manipulation of resources in an Azure subscription. + +To do this, we will first create a new Azure **service principal (SP)** in Azure **Active Directory (AD)**, which, in Azure, is an application user who has permission to manage Azure resources. + +The following is a template `az cli` script that you have to run for creating an service principal, where you have to enter your SP name, role, and scope: + +``` +az ad sp create-for-rbac \ + --name="" \ + --role="Contributor" \ + --scopes="/subscriptions/" +``` + +The creation of this service principal returns three pieces of identification information, *The application ID, also called the client ID*, and *The client secret*. This information will be used when you create a node template for Azure. + +## Creating an Azure Cluster + + + + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Azure**. +1. Enter your Azure credentials. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials + +Creating a [node template](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for Azure will allow Rancher to provision new nodes in Azure. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Add Template**. +1. Click **Azure**. +1. Fill out a node template for Azure. For help filling out the form, refer to [Azure Node Template Configuration.](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md) + +### 3. Create a cluster with node pools using the node template + +Use Rancher to create a Kubernetes cluster in Azure. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Azure**. +1. Enter a **Cluster Name**. +1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices, see [this section.](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) +1. In the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +### 1. Create your cloud credentials + +If you already have a set of cloud credentials to use, skip this section. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **Azure**. +1. Enter your Azure credentials. +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create your cluster + +Use Rancher to create a Kubernetes cluster in Azure. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Toggle the switch to **RKE2/K3s**. +1. Click **Azure**. +1. Select a **Cloud Credential**, if more than one exists. Otherwise, it's preselected. +1. Enter a **Cluster Name**. +1. Create a machine pool for each Kubernetes role. Refer to the [best practices](../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-roles) for recommendations on role assignments and counts. + 1. For each machine pool, define the machine configuration. Refer to the [Azure machine configuration reference](../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) for information on configuration options. +1. Use the **Cluster Configuration** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. For help configuring the cluster, refer to the [RKE2 cluster configuration reference.](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Click **Create**. + + + + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +### Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md new file mode 100644 index 00000000000..ded99b0679c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md @@ -0,0 +1,96 @@ +--- +title: Provisioning Kubernetes Clusters in Nutanix AOS +--- + + + + + +To use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Nutanix AOS (AHV): + +1. Locate Rancher's built-in Nutanix [node driver and activate it](../../../authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#activatingdeactivating-node-drivers). + +1. Create a node template, which Rancher will use to provision nodes in Nutanix AOS. + +1. Create a Nutanix AOS cluster in Rancher. When configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +For details on configuring the Nutanix AOS node template, refer to the [Nutanix AOS node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md) + +For details on configuring RKE Kubernetes clusters in Rancher, refer to the [cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) + +- [Preparation in Nutanix AOS](#preparation-in-nutanix-aos) +- [Creating a Nutanix AOS Cluster](#creating-a-nutanix-aos-cluster) + +## Preparation in Nutanix AOS + +The following sections describe the requirements for setting up Nutanix AOS so that Rancher can provision VMs and clusters. + +:::note + +The node templates are documented and tested with Nutanix AOS version 5.20.2 and 6.0.1. + +::: + +### Create Credentials in Nutanix AOS + +Before proceeding to create a cluster, you must ensure that you have a [Nutanix Prism Central user account](https://portal.nutanix.com/page/documents/details?targetId=Nutanix-Security-Guide-v6_0:wc-user-create-wc-t.html) with admin permissions. When you set up a node template, the template will need to use these credentials. + +### Network Permissions + +You must ensure that the hosts running the Rancher server are able to establish the following network connections: + +- To the Nutanix Prism Central API (usually port 9440/TCP). +- To port 22/TCP and 2376/TCP on the created VMs + +See [Node Networking Requirements](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for a detailed list of port requirements applicable for creating nodes on an infrastructure provider. + +### VM-VM Anti-Affinity Policies + +Setting up [VM-VM Anti-Affinity Policies](https://portal.nutanix.com/page/documents/details?targetId=AHV-Admin-Guide-v6_1:ahv-vm-anti-affinity-t.html) is recommended. These rules allow VMs assigned the etcd and control-plane roles to operate on separate AHV hosts when they are assigned to different node pools. This practice ensures that the failure of a single physical machine does not affect the availability of those planes. + +## Creating a Nutanix AOS Cluster + +1. [Create a node template ](#1-create-a-node-template) +2. [Create a cluster with node pools using the node template](#2-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create a node template + +Creating a [node template](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for Nutanix AOS will allow Rancher to provision new nodes in Nutanix AOS. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Create**. +1. Click **Add Template**. +1. Click **Nutanix**. +1. Fill out a node template for Nutanix AOS. For help filling out the form, refer to the Nutanix AOS node template [configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md). +1. Click **Create**. + +### 2. Create a cluster with node pools using the node template + +Use Rancher to create a Kubernetes cluster in Nutanix AOS. + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Nutanix**. +1. Enter a **Cluster Name**, then click **Continue**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users who can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used, and whether you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to the nodes, see [this section.](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) +1. Review your options to confirm they're correct. Then click **Create**. + +**Result:** Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. + +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md new file mode 100644 index 00000000000..c8a6da86cf2 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template.md @@ -0,0 +1,164 @@ +--- +title: Creating a vSphere Virtual Machine Template +--- + + + + + +Creating virtual machines in a repeatable and reliable fashion can often be difficult. VMware vSphere offers the ability to build one VM that can then be converted to a template. The template can then be used to create identically configured VMs. Rancher leverages this capability within node pools to create identical RKE1 and RKE2 nodes. + +In order to leverage the template to create new VMs, Rancher has some [specific requirements](#requirements) that the VM must have pre-installed. After you configure the VM with these requirements, you will next need to [prepare the VM](#preparing-your-vm) before [creating the template](#creating-a-template). Finally, once preparation is complete, the VM can be [converted to a template](#converting-to-a-template) and [moved into a content library](#moving-to-a-content-library), ready for Rancher node pool usage. + + +## Requirements + +There is specific tooling required for both Linux and Windows VMs to be usable by the vSphere node driver. The most critical dependency is [cloud-init](https://cloud-init.io/) for Linux and [cloudbase-init](https://cloudbase.it/cloudbase-init/) for Windows. Both of these are used for provisioning the VMs by configuring the hostname and by setting up the SSH access and the default Rancher user. Users can add additional content to these as desired if other configuration is needed. In addition, other requirements are listed below for reference. + +:::note + +If you have any specific firewall rules or configuration, you will need to add this to the VM before creating a template. + +::: + +## Linux Dependencies + +The packages that need to be installed on the template are listed below. These will have slightly different names based on distribution; some distributions ship these by default, for example. + +* curl +* wget +* git +* net-tools +* unzip +* apparmor-parser +* ca-certificates +* cloud-init +* cloud-guest-utils +* cloud-image-utils +* growpart +* cloud-initramfs-growroot +* open-iscsi +* openssh-server +* [open-vm-tools](https://docs.vmware.com/en/VMware-Tools/11.3.0/com.vmware.vsphere.vmwaretools.doc/GUID-8B6EA5B7-453B-48AA-92E5-DB7F061341D1.html) + +## Windows Dependencies + +The list of packages that need to be installed on the template is as follows: + +* Windows Container Feature +* [cloudbase-init](https://cloudbase.it/cloudbase-init/#download) +* [Docker EE](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server#install-docker) - RKE1 Only + +:::note About the configuration for Windows templates varies between RKE1 and RKE2: + +- RKE1 leverages Docker, so any RKE1 templates need to have Docker EE pre-installed as well +- RKE2 does not require Docker EE, and thus it does not need to be installed + +::: + +## Creating a Template + +You may either manually create your VM or you can utilize [other alternatives](#alternatives-to-manual-creation) to create your VM. + +### Manual Creation +1. Manually create your VM by following [these instructions](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AE8AFBF1-75D1-4172-988C-378C35C9FAF2.html) from VMware. Once you have a VM running, you can manually install the dependencies listed above to configure the VM correctly for the vSphere node driver. +2. Customize as needed based on your specific environment and requirements. +3. Proceed with the final preparation before creating your template. + +### Alternatives to Manual Creation + +Other alternative options to create VMs are listed below: + +* [VMware PowerCLI](https://developer.vmware.com/powercli) +* [Packer](https://www.packer.io/) +* [SaltStack](https://saltproject.io/) +* [Ansible](https://www.ansible.com/) + +Packer is a frequently-used alternative. Refer to this [reference](https://github.com/vmware-samples/packer-examples-for-vsphere) for examples of its usage with vSphere. + +## Preparing Your VM + +After creating a VM with all the required dependencies (and any additional required items), you must perform the most critical step next: preparing the VM to be turned into a template. This preparation will reset critical data such as the VM hostname, IPs, etc., to prevent that information from being brought into a new VM. If you fail to perform this step, you could create a VM with the same hostname, IP address, etc. + +Note that these preparatory steps differ between Linux and Windows. + +### Linux Preparation + +The commands below will reset your VM in Linux: + +```bash +# Cleaning logs. +if [ -f /var/log/audit/audit.log ]; then + cat /dev/null > /var/log/audit/audit.log +fi +if [ -f /var/log/wtmp ]; then + cat /dev/null > /var/log/wtmp +fi +if [ -f /var/log/lastlog ]; then + cat /dev/null > /var/log/lastlog +fi + +# Cleaning udev rules. +if [ -f /etc/udev/rules.d/70-persistent-net.rules ]; then + rm /etc/udev/rules.d/70-persistent-net.rules +fi + +# Cleaning the /tmp directories +rm -rf /tmp/* +rm -rf /var/tmp/* + +# Cleaning the SSH host keys +rm -f /etc/ssh/ssh_host_* + +# Cleaning the machine-id +truncate -s 0 /etc/machine-id +rm /var/lib/dbus/machine-id +ln -s /etc/machine-id /var/lib/dbus/machine-id + +# Cleaning the shell history +unset HISTFILE +history -cw +echo > ~/.bash_history +rm -fr /root/.bash_history + +# Truncating hostname, hosts, resolv.conf and setting hostname to localhost +truncate -s 0 /etc/{hostname,hosts,resolv.conf} +hostnamectl set-hostname localhost + +# Clean cloud-init +cloud-init clean -s -l +``` + +### Windows Preparation + +Windows has a utility called [sysprep](https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/sysprep--generalize--a-windows-installation) that is used to generalize an image and reset the same items listed above for Linux. The command is as follows: + +```PowerShell +sysprep.exe /generalize /shutdown /oobe +``` + +## Converting to a Template + +1. Shut down and stop the VM. +2. Right-click on the VM in the inventory list and select **Template**. +3. Click on **Convert to Template**. + +**Result:** Once the process has completed, a template will be available for use. + +For additional information on converting a VM to a template, see the [VMware guide](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-5B3737CC-28DB-4334-BD18-6E12011CDC9F.html). + +## Moving to a Content library + +Rancher has the ability to use templates provided by a content library. Content libraries store and manage content within vSphere, and they also offer the ability to publish and share that content. + +Below are some helpful links on content libraries: + +* [Create a content library](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-2A0F1C13-7336-45CE-B211-610D39A6E1F4.html) +* [Clone the template to the content library](https://docs.vmware.com/en/VMware-vSphere/7.0/com.vmware.vsphere.vm_admin.doc/GUID-AC1545F0-F8BA-4CD2-96EB-21B3DFAA1DC1.html) + +## Other Resources + +Here is a list of additional resources that may be useful: + +* [Tutorial for creating a Linux template](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-ubuntu-template) +* [Tutorial for creating a Windows template](https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/manage/hybrid/server/best-practices/vmware-windows-template) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md new file mode 100644 index 00000000000..d228c27c1e7 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials.md @@ -0,0 +1,49 @@ +--- +title: Creating Credentials in the vSphere Console +--- + + + + + +This section describes how to create a vSphere username and password. You will need to provide these vSphere credentials to Rancher, which allows Rancher to provision resources in vSphere. + +The following table lists the permissions required for the vSphere user account: + +| Privilege Group | Operations | +|:-------------------------|:-----------------------------------------------------------------------| +| Cns Privileges | Searchable | +| Content library | Read Storage
(Required only if deploying Template from Content Library) | +| Cryptographic operations | Direct Access | +| Datastore | AllocateSpace
Browse
FileManagement (Low level file operations)
UpdateVirtualMachineFiles
UpdateVirtualMachineMetadata | +| Global | Set custom attribute | +| Network | Assign | +| Resource | AssignVMToPool | +| Virtual Machine | Config (All)
GuestOperations (All)
Interact (All)
Inventory (All)
Provisioning (All) | +| vSphere Tagging | Assign or Unassign vSphere Tag
Assign or Unassign vSphere Tag on Object | + +The following steps create a role with the required privileges and then assign it to a new user in the vSphere console: + +1. From the **vSphere** console, go to the **Administration** page. + +2. Go to the **Roles** tab. + +3. Create a new role. Give it a name and select the privileges listed in the permissions table above. + + ![](/img/rancherroles1.png) + +4. Go to the **Users and Groups** tab. + +5. Create a new user. Fill out the form and then click **OK**. Make sure to note the username and password, because you will need it when configuring node templates in Rancher. + + ![](/img/rancheruser.png) + +6. Go to the **Global Permissions** tab. + +7. Create a new Global Permission. Add the user you created earlier and assign it the role you created earlier. Click **OK**. + + ![](/img/globalpermissionuser.png) + + ![](/img/globalpermissionrole.png) + +**Result:** You now have credentials that Rancher can use to manipulate vSphere resources. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md new file mode 100644 index 00000000000..f4e454c45a9 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md @@ -0,0 +1,116 @@ +--- +title: Provisioning Kubernetes Clusters in vSphere +--- + + + + + +In this section, you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in vSphere. + +First, you will set up your vSphere cloud credentials in Rancher. Then you will use your cloud credentials to create a node template, which Rancher will use to provision nodes in vSphere. + +Then you will create a vSphere cluster in Rancher, and when configuring the new cluster, you will define node pools for it. Each node pool will have a Kubernetes role of etcd, controlplane, or worker. Rancher will install RKE Kubernetes on the new nodes, and it will set up each node with the Kubernetes role defined by the node pool. + +For details on configuring the vSphere node template, refer to the [vSphere node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) + +For details on configuring RKE Kubernetes clusters in Rancher, refer to the [cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#rke-cluster-config-file-reference) + + +- [Preparation in vSphere](#preparation-in-vsphere) +- [Creating a vSphere Cluster](#creating-a-vsphere-cluster) + +## Preparation in vSphere + +This section describes the requirements for setting up vSphere so that Rancher can provision VMs and clusters. + +The node templates are documented and tested with the vSphere Web Services API version 6.5. + +### Create Credentials in vSphere + +Before proceeding to create a cluster, you must ensure that you have a vSphere user with sufficient permissions. When you set up a node template, the template will need to use these vSphere credentials. + +Refer to this [how-to guide](create-credentials.md) for instructions on how to create a user in vSphere with the required permissions. These steps result in a username and password that you will need to provide to Rancher, which allows Rancher to provision resources in vSphere. + +### Network Permissions + +It must be ensured that the hosts running the Rancher server are able to establish the following network connections: + +- To the vSphere API on the vCenter server (usually port 443/TCP). +- To the Host API (port 443/TCP) on all ESXi hosts used to instantiate virtual machines for the clusters (*only required when using the ISO creation method*). +- To port 22/TCP and 2376/TCP on the created VMs + +See [Node Networking Requirements](../../../kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for a detailed list of port requirements applicable for creating nodes on an infrastructure provider. + +### Valid ESXi License for vSphere API Access + +The free ESXi license does not support API access. The vSphere servers must have a valid or evaluation ESXi license. + +### VM-VM Affinity Rules for Clusters with DRS + +If you have a cluster with DRS enabled, setting up [VM-VM Affinity Rules](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.resmgmt.doc/GUID-7297C302-378F-4AF2-9BD6-6EDB1E0A850A.html) is recommended. These rules allow VMs assigned the etcd and control-plane roles to operate on separate ESXi hosts when they are assigned to different node pools. This practice ensures that the failure of a single physical machine does not affect the availability of those planes. + +## Creating a vSphere Cluster + +The a vSphere cluster is created in Rancher depends on the Rancher version. + +1. [Create your cloud credentials](#1-create-your-cloud-credentials) +2. [Create a node template with your cloud credentials](#2-create-a-node-template-with-your-cloud-credentials) +3. [Create a cluster with node pools using the node template](#3-create-a-cluster-with-node-pools-using-the-node-template) + +### 1. Create your cloud credentials + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click **VMware vSphere**. +1. Enter your vSphere credentials. For help, refer to **Account Access** in the [node template configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md) +1. Click **Create**. + +**Result:** You have created the cloud credentials that will be used to provision nodes in your cluster. You can reuse these credentials for other node templates, or in other clusters. + +### 2. Create a node template with your cloud credentials + +Creating a [node template](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for vSphere will allow Rancher to provision new nodes in vSphere. Node templates can be reused for other clusters. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Create**. +1. Click **Add Template**. +1. Click **vSphere**. +1. Fill out a node template for vSphere. For help filling out the form, refer to the vSphere node template [configuration reference.](../../../../../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md). +1. Click **Create**. + +### 3. Create a cluster with node pools using the node template + +Use Rancher to create a Kubernetes cluster in vSphere. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **VMware vSphere**. +1. Enter a **Cluster Name** and use your vSphere cloud credentials. Click **Continue**. +1. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. +1. Use **Cluster Options** to choose the version of Kubernetes that will be installed, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. For help configuring the cluster, refer to the [RKE cluster configuration reference.](../../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +1. If you want to dynamically provision persistent storage or other infrastructure later, you will need to enable the vSphere cloud provider by modifying the cluster YAML file. For details, refer to [in-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). +1. Add one or more node pools to your cluster. Each node pool uses a node template to provision new nodes. For more information about node pools, including best practices for assigning Kubernetes roles to the nodes, see [this section.](../../../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) +1. Review your options to confirm they're correct. Then click **Create**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../../../../new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. +- **Provision Storage:** For an example of how to provision storage in vSphere using Rancher, refer to [this section.](../../../../../pages-for-subheaders/provisioning-storage-examples.md) In order to dynamically provision storage in vSphere, the vSphere provider must be enabled. For details, refer to [in-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere cloud provider docs](../../../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md new file mode 100644 index 00000000000..148715a31de --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md @@ -0,0 +1,70 @@ +--- +title: Adding Users to Clusters +--- + + + + + +If you want to provide a user with access and permissions to _all_ projects, nodes, and resources within a cluster, assign the user a cluster membership. + +:::tip + +Want to provide a user with access to a _specific_ project within a cluster? See [Adding Project Members](../../../new-user-guides/add-users-to-projects.md) instead. + +::: + +There are two contexts where you can add cluster members: + +- Adding Members to a New Cluster + + You can add members to a cluster as you create it (recommended if possible). + +- [Adding Members to an Existing Cluster](#editing-cluster-membership) + + You can always add members to a cluster after a cluster is provisioned. + +## Editing Cluster Membership + +Cluster administrators can edit the membership for a cluster, controlling which Rancher users can access the cluster and what features they can use. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to add members to and click **â‹® > Edit Config**. +1. In the **Member Roles** tab, click **Add Member**. +1. Search for the user or group that you want to add to the cluster. + + If external authentication is configured: + + - Rancher returns users from your [external authentication](../../../../pages-for-subheaders/authentication-config.md) source as you type. + + :::note Using AD but can't find your users? + + There may be an issue with your search attribute configuration. See [Configuring Active Directory Authentication: Step 5](../../../new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md). + + ::: + + - A drop-down allows you to add groups instead of individual users. The drop-down only lists groups that you, the logged in user, are part of. + + :::note + + If you are logged in as a local user, external users do not display in your search results. For more information, see [External Authentication Configuration and Principal Users](../../../../pages-for-subheaders/authentication-config.md#external-authentication-configuration-and-principal-users). + + ::: + +1. Assign the user or group **Cluster** roles. + + [What are Cluster Roles?](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + + :::tip + + For Custom Roles, you can modify the list of individual roles available for assignment. + + - To add roles to the list, [Add a Custom Role](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md). + - To remove roles from the list, [Lock/Unlock Roles](../../../new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles.md). + + ::: + +**Result:** The chosen users are added to the cluster. + +- To revoke cluster membership, select the user and click **Delete**. This action deletes membership, not the user. +- To modify a user's roles in the cluster, delete them from the cluster, and then re-add them with modified roles. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md new file mode 100644 index 00000000000..543ed4d10b4 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint.md @@ -0,0 +1,53 @@ +--- +title: How the Authorized Cluster Endpoint Works +--- + + + + + +This section describes how the kubectl CLI, the kubeconfig file, and the authorized cluster endpoint work together to allow you to access a downstream Kubernetes cluster directly, without authenticating through the Rancher server. It is intended to provide background information and context to the instructions for [how to set up kubectl to directly access a cluster.](use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) + +### About the Kubeconfig File + +The kubeconfig file is used to configure access to Kubernetes when used in conjunction with the kubectl command line tool (or other clients). + +The kubeconfig file and its contents are specific to each cluster. It can be downloaded from the **Clusters** page in Rancher: + +1. Click **☰** in the top left corner. +1. Select **Cluster Management**. +1. Find the cluster whose kubeconfig you want to download, and select **â** at the end of the row. +1. Select **Download KubeConfig** from the submenu. + +You will need a separate kubeconfig file for each cluster that you have access to in Rancher. + +After you download the kubeconfig file, you will be able to use the kubeconfig file and its Kubernetes [contexts](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#kubectl-context-and-configuration) to access your downstream cluster. + +If admins have [kubeconfig token generation turned off](../../../../reference-guides/about-the-api/api-tokens.md#disable-tokens-in-generated-kubeconfigs), the kubeconfig file requires [rancher cli](./authorized-cluster-endpoint.md) to be present in your PATH. + +### Two Authentication Methods for RKE Clusters + +If the cluster is not an [RKE cluster,](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) the kubeconfig file allows you to access the cluster in only one way: it lets you be authenticated with the Rancher server, then Rancher allows you to run kubectl commands on the cluster. + +For RKE clusters, the kubeconfig file allows you to be authenticated in two ways: + +- **Through the Rancher server authentication proxy:** Rancher's authentication proxy validates your identity, then connects you to the downstream cluster that you want to access. +- **Directly with the downstream cluster's API server:** RKE clusters have an authorized cluster endpoint enabled by default. This endpoint allows you to access your downstream Kubernetes cluster with the kubectl CLI and a kubeconfig file, and it is enabled by default for RKE clusters. In this scenario, the downstream cluster's Kubernetes API server authenticates you by calling a webhook (the `kube-api-auth` microservice) that Rancher set up. + +This second method, the capability to connect directly to the cluster's Kubernetes API server, is important because it lets you access your downstream cluster if you can't connect to Rancher. + +To use the authorized cluster endpoint, you will need to configure kubectl to use the extra kubectl context in the kubeconfig file that Rancher generates for you when the RKE cluster is created. This file can be downloaded from the cluster view in the Rancher UI, and the instructions for configuring kubectl are on [this page.](use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) + +These methods of communicating with downstream Kubernetes clusters are also explained in the [architecture page](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md) in the larger context of explaining how Rancher works and how Rancher communicates with downstream clusters. + +### About the kube-api-auth Authentication Webhook + +The `kube-api-auth` microservice is deployed to provide the user authentication functionality for the [authorized cluster endpoint,](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint). When you access the user cluster using `kubectl`, the cluster's Kubernetes API server authenticates you by using the `kube-api-auth` service as a webhook. + +During cluster provisioning, the file `/etc/kubernetes/kube-api-authn-webhook.yaml` is deployed and `kube-apiserver` is configured with `--authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml`. This configures the `kube-apiserver` to query `http://127.0.0.1:6440/v1/authenticate` to determine authentication for bearer tokens. + +The scheduling rules for `kube-api-auth` are listed below: + +| Component | nodeAffinity nodeSelectorTerms | nodeSelector | Tolerations | +| -------------------- | ------------------------------------------ | ------------ | ------------------------------------------------------------------------------ | +| kube-api-auth | `beta.kubernetes.io/os:NotIn:windows`
`node-role.kubernetes.io/controlplane:In:"true"` | none | `operator:Exists` | diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md new file mode 100644 index 00000000000..6099a7d33d3 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md @@ -0,0 +1,110 @@ +--- +title: "Access a Cluster with Kubectl and kubeconfig" +description: "Learn how you can access and manage your Kubernetes clusters using kubectl with kubectl Shell or with kubectl CLI and kubeconfig file. A kubeconfig file is used to configure access to Kubernetes. When you create a cluster with Rancher, it automatically creates a kubeconfig for your cluster." +--- + + + + + +This section describes how to manipulate your downstream Kubernetes cluster with kubectl from the Rancher UI or from your workstation. + +For more information on using kubectl, see [Kubernetes Documentation: Overview of kubectl](https://kubernetes.io/docs/reference/kubectl/overview/). + +### Accessing Clusters with kubectl Shell in the Rancher UI + +You can access and manage your clusters by logging into Rancher and opening the kubectl shell in the UI. No further configuration necessary. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to access with kubectl and click **Explore**. +1. In the top navigation menu, click the **Kubectl Shell** button. Use the window that opens to interact with your Kubernetes cluster. + +### Accessing Clusters with kubectl from Your Workstation + +This section describes how to download your cluster's kubeconfig file, launch kubectl from your workstation, and access your downstream cluster. + +This alternative method of accessing the cluster allows you to authenticate with Rancher and manage your cluster without using the Rancher UI. + +:::note Prerequisites: + +These instructions assume that you have already created a Kubernetes cluster, and that kubectl is installed on your workstation. For help installing kubectl, refer to the official [Kubernetes documentation.](https://kubernetes.io/docs/tasks/tools/install-kubectl/) + +::: + +1. Click **☰** in the top left corner. +1. Select **Cluster Management**. +1. Find the cluster whose kubeconfig you want to download, and select **â** at the end of the row. +1. Select **Download KubeConfig** from the submenu. +1. Save the YAML file on your local computer. Move the file to `~/.kube/config`. Note: The default location that kubectl uses for the kubeconfig file is `~/.kube/config`, but you can use any directory and specify it using the `--kubeconfig` flag, as in this command: + ``` + kubectl --kubeconfig /custom/path/kube.config get pods + ``` +1. From your workstation, launch kubectl. Use it to interact with your kubernetes cluster. + + +### Note on Resources Created Using kubectl + +Rancher will discover and show resources created by `kubectl`. However, these resources might not have all the necessary annotations on discovery. If an operation (for instance, scaling the workload) is done to the resource using the Rancher UI/API, this may trigger recreation of the resources due to the missing annotations. This should only happen the first time an operation is done to the discovered resource. + +## Authenticating Directly with a Downstream Cluster + +This section intended to help you set up an alternative method to access an [RKE cluster.](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +This method is only available for RKE, RKE2, and K3s clusters that have the [authorized cluster endpoint](../../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) enabled. When Rancher creates the cluster, it generates a kubeconfig file that includes additional kubectl context(s) for accessing your cluster. This additional context allows you to use kubectl to authenticate with the downstream cluster without authenticating through Rancher. For a longer explanation of how the authorized cluster endpoint works, refer to [this page](authorized-cluster-endpoint.md). + +On RKE2 and K3s clusters, you need to [manually enable](../../kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) authorized cluster endpoints. + +We recommend that as a best practice, you should set up this method to access your RKE, RKE2, and K3s clusters, so that just in case you can’t connect to Rancher, you can still access the cluster. + + +:::note Prerequisites: + +The following steps assume that you have created a Kubernetes cluster and followed the steps to [connect to your cluster with kubectl from your workstation.](#accessing-clusters-with-kubectl-from-your-workstation) + +::: + +To find the name of the context(s) in your downloaded kubeconfig file, run: + +``` +kubectl config get-contexts --kubeconfig /custom/path/kube.config +CURRENT NAME CLUSTER AUTHINFO NAMESPACE +* my-cluster my-cluster user-46tmn + my-cluster-controlplane-1 my-cluster-controlplane-1 user-46tmn +``` + +In this example, when you use `kubectl` with the first context, `my-cluster`, you will be authenticated through the Rancher server. + +With the second context, `my-cluster-controlplane-1`, you would authenticate with the authorized cluster endpoint, communicating with an downstream RKE cluster directly. + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../../../reference-guides/rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +Now that you have the name of the context needed to authenticate directly with the cluster, you can pass the name of the context in as an option when running kubectl commands. The commands will differ depending on whether your cluster has an FQDN defined. Examples are provided in the sections below. + +When `kubectl` works normally, it confirms that you can access your cluster while bypassing Rancher's authentication proxy. + +### Connecting Directly to Clusters with FQDN Defined + +If an FQDN is defined for the cluster, a single context referencing the FQDN will be created. The context will be named `-fqdn`. When you want to use `kubectl` to access this cluster without Rancher, you will need to use this context. + +Assuming the kubeconfig file is located at `~/.kube/config`: + +``` +kubectl --context -fqdn get nodes +``` +Directly referencing the location of the kubeconfig file: +``` +kubectl --kubeconfig /custom/path/kube.config --context -fqdn get pods +``` + +### Connecting Directly to Clusters without FQDN Defined + +If there is no FQDN defined for the cluster, extra contexts will be created referencing the IP address of each node in the control plane. Each context will be named `-`. When you want to use `kubectl` to access this cluster without Rancher, you will need to use this context. + +Assuming the kubeconfig file is located at `~/.kube/config`: +``` +kubectl --context - get nodes +``` +Directly referencing the location of the kubeconfig file: +``` +kubectl --kubeconfig /custom/path/kube.config --context - get pods +``` diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md new file mode 100644 index 00000000000..9b9c30d32cb --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md @@ -0,0 +1,43 @@ +--- +title: Adding a Pod Security Policy +--- + + + + + +:::note Prerequisite: + +The options below are available only for clusters that are [launched using RKE.](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +::: + +When your cluster is running pods with security-sensitive configurations, assign it a [pod security policy](../authentication-permissions-and-global-configuration/create-pod-security-policies.md), which is a set of rules that monitors the conditions and settings in your pods. If a pod doesn't meet the rules specified in your policy, the policy stops it from running. + +You can assign a pod security policy when you provision a cluster. However, if you need to relax or restrict security for your pods later, you can update the policy while editing your cluster. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster to which you want to apply a pod security policy and click **â‹® > Edit Config**. +1. From **Pod Security Policy Support**, select **Enabled**. + + :::note + + This option is only available for clusters [provisioned by RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md). + + ::: + +4. From the **Default Pod Security Policy** drop-down, select the policy you want to apply to the cluster. + + Rancher ships with [policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#default-psps) of `restricted` and `unrestricted`, although you can [create custom policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md#creating-psps) as well. + +5. Click **Save**. + +**Result:** The pod security policy is applied to the cluster and any projects within the cluster. + +:::note + +Workloads already running before assignment of a pod security policy are grandfathered in. Even if they don't meet your pod security policy, workloads running before assignment of the policy continue to run. + +To check if a running workload passes your pod security policy, clone or upgrade it. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md new file mode 100644 index 00000000000..9b4d29eda6f --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies.md @@ -0,0 +1,29 @@ +--- +title: Assigning Pod Security Policies +--- + + + + + +_Pod Security Policies_ are objects that control security-sensitive aspects of pod specification (like root privileges). + +## Adding a Default Pod Security Policy + +When you create a new cluster with RKE, you can configure it to apply a PSP immediately. As you create the cluster, use the **Cluster Options** to enable a PSP. The PSP assigned to the cluster will be the default PSP for projects within the cluster. + +:::note Prerequisite: + +Create a Pod Security Policy within Rancher. Before you can assign a default PSP to a new cluster, you must have a PSP available for assignment. For instruction, see [Creating Pod Security Policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md). + +::: + +:::note + +For security purposes, we recommend assigning a PSP as you create your clusters. + +::: + +To enable a default Pod Security Policy, set the **Pod Security Policy Support** option to **Enabled**, and then make a selection from the **Default Pod Security Policy** drop-down. + +When the cluster finishes provisioning, the PSP you selected is applied to all projects within the cluster. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md new file mode 100644 index 00000000000..b45beab4572 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md @@ -0,0 +1,380 @@ +--- +title: Removing Kubernetes Components from Nodes +description: Learn about cluster cleanup when removing nodes from your Rancher-launched Kubernetes cluster. What is removed, how to do it manually +--- + + + + + +This section describes how to disconnect a node from a Rancher-launched Kubernetes cluster and remove all of the Kubernetes components from the node. This process allows you to use the node for other purposes. + +When you use Rancher to install Kubernetes on new nodes in an infrastructure provider, resources (containers/virtual network interfaces) and configuration items (certificates/configuration files) are created. + +When removing nodes from your Rancher launched Kubernetes cluster (provided that they are in `Active` state), those resources are automatically cleaned, and the only action needed is to restart the node. When a node has become unreachable and the automatic cleanup process cannot be used, we describe the steps that need to be executed before the node can be added to a cluster again. + +## What Gets Removed? + +When cleaning nodes provisioned using Rancher, the following components are deleted based on the type of cluster node you're removing. + +| Removed Component | [Nodes Hosted by Infrastructure Provider][1] | [Custom Nodes][2] | [Hosted Cluster][3] | [Registered Nodes][4] | +|--------------------------------------------------------------------------------|----------------------------------------------|-------------------|---------------------|-----------------------| +| The Rancher deployment namespace (`cattle-system` by default) | ✓ | ✓ | ✓ | ✓ | +| `serviceAccount`, `clusterRoles`, and `clusterRoleBindings` labeled by Rancher | ✓ | ✓ | ✓ | ✓ | +| Labels, Annotations, and Finalizers | ✓ | ✓ | ✓ | ✓ | +| Rancher Deployment | ✓ | ✓ | ✓ | | +| Machines, clusters, projects, and user custom resource definitions (CRDs) | ✓ | ✓ | ✓ | | +| All resources create under the `management.cattle.io` API Group | ✓ | ✓ | ✓ | | +| All CRDs created by Rancher v2.x | ✓ | ✓ | ✓ | | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +## Removing a Node from a Cluster by Rancher UI + +When the node is in `Active` state, removing the node from a cluster will trigger a process to clean up the node. Please restart the node after the automatic cleanup process is done to make sure any non-persistent data is properly removed. + +**To restart a node:** + +``` +# using reboot +$ sudo reboot + +# using shutdown +$ sudo shutdown -r now +``` + +## Removing Rancher Components from a Cluster Manually + +When a node is unreachable and removed from the cluster, the automatic cleaning process can't be triggered because the node is unreachable. Please follow the steps below to manually remove the Rancher components. + +:::danger + +The commands listed below will remove data from the node. Make sure you have created a backup of files you want to keep before executing any of the commands as data will be lost. + +::: + +### Removing Rancher Components from Registered Clusters + +For registered clusters, the process for removing Rancher is a little different. You have the option of simply deleting the cluster in the Rancher UI, or your can run a script that removes Rancher components from the nodes. Both options make the same deletions. + +After the registered cluster is detached from Rancher, the cluster's workloads will be unaffected and you can access the cluster using the same methods that you did before the cluster was registered into Rancher. + + + + +:::danger + +This process will remove data from your cluster. Make sure you have created a backup of files you want to keep before executing the command, as data will be lost. + +::: + +After you initiate the removal of a registered cluster using the Rancher UI (or API), the following events occur. + +1. Rancher creates a `serviceAccount` that it uses to remove the Rancher components from the cluster. This account is assigned the [clusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole) and [clusterRoleBinding](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding) permissions, which are required to remove the Rancher components. + +1. Using the `serviceAccount`, Rancher schedules and runs a [job](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) that cleans the Rancher components off of the cluster. This job also references the `serviceAccount` and its roles as dependencies, so the job deletes them before its completion. + +1. Rancher is removed from the cluster. However, the cluster persists, running the native version of Kubernetes. + +**Result:** All components listed for registered clusters in [What Gets Removed?](#what-gets-removed) are deleted. + + + + +Rather than cleaning registered cluster nodes using the Rancher UI, you can run a script instead. + +:::note Prerequisite: + +Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). + +::: + +1. Open a web browser, navigate to [GitHub](https://github.com/rancher/rancher/blob/master/cleanup/user-cluster.sh), and download `user-cluster.sh`. + +1. Make the script executable by running the following command from the same directory as `user-cluster.sh`: + + ``` + chmod +x user-cluster.sh + ``` + +1. **Air Gap Environments Only:** Open `user-cluster.sh` and replace `yaml_url` with the URL in `user-cluster.yml`. + + If you don't have an air gap environment, skip this step. + +1. From the same directory, run the script and provide the `rancher/rancher-agent` image version which should be equal to the version of Rancher used to manage the cluster. (``): + + :::tip + + Add the `-dry-run` flag to preview the script's outcome without making changes. + ``` + ./user-cluster.sh rancher/rancher-agent: + ``` + + ::: + +**Result:** The script runs. All components listed for registered clusters in [What Gets Removed?](#what-gets-removed) are deleted. + + + + +## Cleaning up Nodes + + + + +Before you run the following commands, first remove the node through the Rancher UI. + +To remove a node: + +1. Click **☰** and select **Cluster Management**. +1. In the table of clusters, click the name of the cluster the node belongs to. +1. In the first tab, click the checkbox next to the node's state. +1. Click **Delete**. + +If you remove the entire cluster instead of an individual node, or skip rermoving the node through the Rancher UI, follow these steps: + +1. [Remove](#docker-containers-images-and-volumes) the Docker containers from the node and [unmount](#mounts) any volumes. +1. Reboot the node. +1. [Remove](#directories-and-files) any remaining files. +1. Confirm that network interfaces and IP tables were properly cleaned after the reboot. If not, reboot one more time. + +### Windows Nodes + +To clean up a Windows node, run the script in `c:\etc\rancher`. This script deletes Kubernetes-generated resources and the execution binary. It also drops the firewall rules and network settings: + +``` +pushd c:\etc\rancher +.\cleanup.ps1 +popd +``` + +After you run this script, the node is reset and can be re-added to a Kubernetes cluster. + +### Docker Containers, Images, and Volumes + +:::caution + +Be careful when cleaning up Docker containers. The following command will remove *all* Docker containers, images, and volumes on the node, including non-Rancher related containers: + +::: + +``` +docker rm -f $(docker ps -qa) +docker rmi -f $(docker images -q) +docker volume rm $(docker volume ls -q) +``` + +### Mounts + +Kubernetes components and secrets leave behind the following mounts: + +* `/var/lib/kubelet` +* `/var/lib/rancher` +* Miscellaneous mounts in `/var/lib/kubelet/pods/` + +To unmount all mounts, run: + +``` +for mount in $(mount | grep tmpfs | grep '/var/lib/kubelet' | awk '{ print $3 }') /var/lib/kubelet /var/lib/rancher; do umount $mount; done +``` + + + + +You need to remove the following components from Rancher-provisioned RKE2 nodes: + +* The rancher-system-agent, which connects to Rancher and installs and manages RKE2. +* RKE2 itself. + +### Removing rancher-system-agent + +To remove the rancher-system-agent, run the [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) script: + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### Removing RKE2 + +To remove the RKE2 installation, run the `rke2-uninstall` script that is already present on the node: + +``` +sudo rke2-uninstall.sh +``` + + + + +You need to remove the following components from Rancher-provisioned K3s nodes: + +* The rancher-system-agent, which connects to Rancher and installs and manages K3s. +* K3s itself. + +### Removing rancher-system-agent + +To remove the rancher-system-agent, run the [system-agent-uninstall.sh](https://github.com/rancher/system-agent/blob/main/system-agent-uninstall.sh) script: + +``` +curl https://raw.githubusercontent.com/rancher/system-agent/main/system-agent-uninstall.sh | sudo sh +``` + +### Removing K3s + +To remove the K3s installation, run the `k3s-uninstall` script that is already present on the node: + +``` +sudo k3s-uninstall.sh +``` + + + + +### Directories and Files + +The following directories are used when adding a node to a cluster, and should be removed. You can remove a directory using `rm -rf /directory_name`. + +:::note + +Depending on the role you assigned to the node, some of the directories will or won't be present on the node. + +::: + +| Directories | +|------------------------------| +| `/etc/ceph` | +| `/etc/cni` | +| `/etc/kubernetes` | +| `/etc/rancher` | +| `/opt/cni` | +| `/opt/rke` | +| `/run/secrets/kubernetes.io` | +| `/run/calico` | +| `/run/flannel` | +| `/var/lib/calico` | +| `/var/lib/etcd` | +| `/var/lib/cni` | +| `/var/lib/kubelet` | +| `/var/lib/rancher` | +| `/var/log/containers` | +| `/var/log/kube-audit` | +| `/var/log/pods` | +| `/var/run/calico` | + +**To clean the directories:** + +``` +rm -rf /etc/ceph \ + /etc/cni \ + /etc/kubernetes \ + /etc/rancher \ + /opt/cni \ + /opt/rke \ + /run/secrets/kubernetes.io \ + /run/calico \ + /run/flannel \ + /var/lib/calico \ + /var/lib/etcd \ + /var/lib/cni \ + /var/lib/kubelet \ + /var/lib/rancher\ + /var/log/containers \ + /var/log/kube-audit \ + /var/log/pods \ + /var/run/calico +``` + +### Network Interfaces and Iptables + +The remaining two components that are changed/configured are (virtual) network interfaces and iptables rules. Both are non-persistent to the node, meaning that they will be cleared after a restart of the node. To remove these components, a restart is recommended. + +**To restart a node:** + +``` +# using reboot +$ sudo reboot + +# using shutdown +$ sudo shutdown -r now +``` + +If you want to know more on (virtual) network interfaces or iptables rules, please see the specific subjects below. + +### Network Interfaces + +:::note + +Depending on the network provider configured for the cluster the node was part of, some of the interfaces will or won't be present on the node. + +::: + +| Interfaces | +|--------------------------------------------| +| `flannel.1` | +| `cni0` | +| `tunl0` | +| `caliXXXXXXXXXXX` (random interface names) | +| `vethXXXXXXXX` (random interface names) | + +**To list all interfaces:** + +``` +# Using ip +ip address show + +# Using ifconfig +ifconfig -a +``` + +**To remove an interface:** + +``` +ip link delete interface_name +``` + +### Iptables + +:::note + +Depending on the network provider configured for the cluster the node was part of, some of the chains will or won't be present on the node. + +::: + +Iptables rules are used to route traffic from and to containers. The created rules are not persistent, so restarting the node will restore iptables to its original state. + +| Chains | +|--------------------------------------------------| +| `cali-failsafe-in` | +| `cali-failsafe-out` | +| `cali-fip-dnat` | +| `cali-fip-snat` | +| `cali-from-hep-forward` | +| `cali-from-host-endpoint` | +| `cali-from-wl-dispatch` | +| `cali-fw-caliXXXXXXXXXXX` (random chain names) | +| `cali-nat-outgoing` | +| `cali-pri-kns.NAMESPACE` (chain per namespace) | +| `cali-pro-kns.NAMESPACE` (chain per namespace) | +| `cali-to-hep-forward` | +| `cali-to-host-endpoint` | +| `cali-to-wl-dispatch` | +| `cali-tw-caliXXXXXXXXXXX` (random chain names) | +| `cali-wl-to-host` | +| `KUBE-EXTERNAL-SERVICES` | +| `KUBE-FIREWALL` | +| `KUBE-MARK-DROP` | +| `KUBE-MARK-MASQ` | +| `KUBE-NODEPORTS` | +| `KUBE-SEP-XXXXXXXXXXXXXXXX` (random chain names) | +| `KUBE-SERVICES` | +| `KUBE-SVC-XXXXXXXXXXXXXXXX` (random chain names) | + +**To list all iptables rules:** + +``` +iptables -L -t nat +iptables -L -t mangle +iptables -L +``` diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md new file mode 100644 index 00000000000..aef5e622446 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md @@ -0,0 +1,41 @@ +--- +title: GlusterFS Volumes +--- + + + + + +:::note + +This section only applies to [RKE clusters.](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +::: + +In clusters that store data on GlusterFS volumes, you may experience an issue where pods fail to mount volumes after restarting the `kubelet`. The logging of the `kubelet` will show: `transport endpoint is not connected`. To prevent this from happening, you can configure your cluster to mount the `systemd-run` binary in the `kubelet` container. There are two requirements before you can change the cluster configuration: + +- The node needs to have the `systemd-run` binary installed (this can be checked by using the command `which systemd-run` on each cluster node) +- The `systemd-run` binary needs to be compatible with Debian OS on which the hyperkube image is based (this can be checked using the following command on each cluster node, replacing the image tag with the Kubernetes version you want to use) + +``` +docker run -v /usr/bin/systemd-run:/usr/bin/systemd-run --entrypoint /usr/bin/systemd-run rancher/hyperkube:v1.16.2-rancher1 --version +``` + +:::caution + +Before updating your Kubernetes YAML to mount the `systemd-run` binary, make sure the `systemd` package is installed on your cluster nodes. If this package isn't installed _before_ the bind mounts are created in your Kubernetes YAML, Docker will automatically create the directories and files on each node and will not allow the package install to succeed. + +::: + +``` +services: + kubelet: + extra_binds: + - "/usr/bin/systemd-run:/usr/bin/systemd-run" +``` + +After the cluster has finished provisioning, you can check the `kubelet` container logging to see if the functionality is activated by looking for the following logline: + +``` +Detected OS with systemd +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md new file mode 100644 index 00000000000..f4acbdf85ef --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md @@ -0,0 +1,76 @@ +--- +title: How Persistent Storage Works +--- + + + + + +A persistent volume (PV) is a piece of storage in the Kubernetes cluster, while a persistent volume claim (PVC) is a request for storage. + +There are two ways to use persistent storage in Kubernetes: + +- Use an existing persistent volume +- Dynamically provision new persistent volumes + +To use an existing PV, your application will need to use a PVC that is bound to a PV, and the PV should include the minimum resources that the PVC requires. + +For dynamic storage provisioning, your application will need to use a PVC that is bound to a storage class. The storage class contains the authorization to provision new persistent volumes. + +![Setting Up New and Existing Persistent Storage](/img/rancher-storage.svg) + +For more information, refer to the [official Kubernetes documentation on storage](https://kubernetes.io/docs/concepts/storage/volumes/) + + +## About Persistent Volume Claims + +Persistent volume claims (PVCs) are objects that request storage resources from your cluster. They're similar to a voucher that your deployment can redeem for storage access. A PVC is mounted into a workloads as a volume so that the workload can claim its specified share of the persistent storage. + +To access persistent storage, a pod must have a PVC mounted as a volume. This PVC lets your deployment application store its data in an external location, so that if a pod fails, it can be replaced with a new pod and continue accessing its data stored externally, as though an outage never occurred. + +Each Rancher project contains a list of PVCs that you've created, available from **Resources > Workloads > Volumes**. You can reuse these PVCs when creating deployments in the future. + +### PVCs are Required for Both New and Existing Persistent Storage + +A PVC is required for pods to use any persistent storage, regardless of whether the workload is intended to use storage that already exists, or the workload will need to dynamically provision new storage on demand. + +If you are setting up existing storage for a workload, the workload mounts a PVC, which refers to a PV, which corresponds to existing storage infrastructure. + +If a workload should request new storage, the workload mounts PVC, which refers to a storage class, which has the capability to create a new PV along with its underlying storage infrastructure. + +Rancher lets you create as many PVCs within a project as you'd like. + +You can mount PVCs to a deployment as you create it, or later, after the deployment is running. + +## Setting up Existing Storage with a PVC and PV + +Your pods can store data in [volumes,](https://kubernetes.io/docs/concepts/storage/volumes/) but if the pod fails, that data is lost. To solve this issue, Kubernetes offers persistent volumes (PVs), which are Kubernetes resources that correspond to external storage disks or file systems that your pods can access. If a pod crashes, its replacement pod can access the data in persistent storage without any data loss. + +PVs can represent a physical disk or file system that you host on premise, or a vendor-hosted storage resource, such as Amazon EBS or Azure Disk. + +Creating a persistent volume in Rancher will not create a storage volume. It only creates a Kubernetes resource that maps to an existing volume. Therefore, before you can create a persistent volume as a Kubernetes resource, you must have storage provisioned. + +:::note Important: + +PVs are created at the cluster level, which means that in a multi-tenant cluster, teams with access to separate namespaces could have access to the same PV. + +::: + +### Binding PVs to PVCs + +When pods are set up to use persistent storage, they mount a persistent volume claim (PVC) that is mounted the same way as any other Kubernetes volume. When each PVC is created, the Kubernetes master considers it to be a request for storage and binds it to a PV that matches the minimum resource requirements of the PVC. Not every PVC is guaranteed to be bound to a PV. According to the Kubernetes [documentation,](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + +> Claims will remain unbound indefinitely if a matching volume does not exist. Claims will be bound as matching volumes become available. For example, a cluster provisioned with many 50Gi PVs would not match a PVC requesting 100Gi. The PVC can be bound when a 100Gi PV is added to the cluster. + +In other words, you can create unlimited PVCs, but they will only be bound to PVs if the Kubernetes master can find a sufficient PV that has at least the amount of disk space required by the PVC. + +To dynamically provision new storage, the PVC mounted in the pod would have to correspond to a storage class instead of a persistent volume. + +## Provisioning New Storage with a PVC and Storage Class + +Storage Classes allow you to create PVs dynamically without having to create persistent storage in an infrastructure provider first. + +For example, if a workload is bound to a PVC and the PVC refers to an Amazon EBS Storage Class, the storage class can dynamically create an EBS volume and a corresponding PV. + +The Kubernetes master will then bind the newly created PV to your workload's PVC, allowing your workload to use the persistent storage. + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md new file mode 100644 index 00000000000..5ad0b03cd45 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md @@ -0,0 +1,91 @@ +--- +title: Dynamically Provisioning New Storage in Rancher +--- + + + + + +This section describes how to provision new persistent storage for workloads in Rancher. + +This section assumes that you understand the Kubernetes concepts of storage classes and persistent volume claims. For more information, refer to the section on [how storage works.](about-persistent-storage.md) + +New storage is often provisioned by a cloud provider such as Amazon EBS. However, new storage doesn't have to be in the cloud. + +If you have a pool of block storage, and you don't want to use a cloud provider, Longhorn could help you provide persistent storage to your Kubernetes cluster. For more information, see [this page.](../../../../../integrations-in-rancher/longhorn.md) + +To provision new storage for your workloads, follow these steps: + +1. [Add a storage class and configure it to use your storage.](#1-add-a-storage-class-and-configure-it-to-use-your-storage) +2. [Use the Storage Class for Pods Deployed with a StatefulSet.](#2-use-the-storage-class-for-pods-deployed-with-a-statefulset) + +### Prerequisites + +- To set up persistent storage, the `Manage Volumes` [role](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-role-reference) is required. +- If you are provisioning storage for a cluster hosted in the cloud, the storage and cluster hosts must have the same cloud provider. +- The cloud provider must be enabled. For details on enabling cloud providers, refer to [this page.](../../../../../pages-for-subheaders/set-up-cloud-providers.md) +- Make sure your storage provisioner is available to be enabled. + +The following storage provisioners are enabled by default: + +Name | Plugin +--------|---------- +Amazon EBS Disk | `aws-ebs` +AzureFile | `azure-file` +AzureDisk | `azure-disk` +Google Persistent Disk | `gce-pd` +Longhorn | `flex-volume-longhorn` +VMware vSphere Volume | `vsphere-volume` +Local | `local` +Network File System | `nfs` +hostPath | `host-path` + +To use a storage provisioner that is not on the above list, you will need to use a [feature flag to enable unsupported storage drivers.](../../../../advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) + +### 1. Add a storage class and configure it to use your storage + +These steps describe how to set up a storage class at the cluster level. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to dynamically provision persistent storage volumes and click **Explore**. +1. Click **Storage > Storage Classes**. +1. Click **Create**. +1. Enter a name for your storage class. +1. From the **Provisioner** drop-down, select the service that you want to use to dynamically provision storage volumes. For example, if you have a Amazon EC2 cluster and you want to use cloud storage for it, use the `Amazon EBS Disk` provisioner. +1. In the **Parameters** tab, fill out the information required for the service to dynamically provision storage volumes. Each provisioner requires different information to dynamically provision storage volumes. Consult the service's documentation for help on how to obtain this information. +1. Click **Create**. + +**Result:** The storage class is available to be consumed by a PVC. + +For full information about the storage class parameters, refer to the official [Kubernetes documentation.](https://kubernetes.io/docs/concepts/storage/storage-classes/#parameters). + +### 2. Use the Storage Class for Pods Deployed with a StatefulSet + +StatefulSets manage the deployment and scaling of Pods while maintaining a sticky identity for each Pod. In this StatefulSet, we will configure a VolumeClaimTemplate. Each Pod managed by the StatefulSet will be deployed with a PersistentVolumeClaim based on this VolumeClaimTemplate. The PersistentVolumeClaim will refer to the StorageClass that we created. Therefore, when each Pod managed by the StatefulSet is deployed, it will be bound to dynamically provisioned storage using the StorageClass defined in its PersistentVolumeClaim. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add use the StorageClass for a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Click **Create**. +1. Click **StatefulSet**. +1. In the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Enter a name for the persistent volume. +1. In the **StorageClass* field, select the StorageClass that will dynamically provision storage for pods managed by this StatefulSet. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Launch**. + +**Result:** When each Pod managed by the StatefulSet is deployed, it will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to Pod with a compatible PVC. + +To attach the PVC to an existing workload, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add use the StorageClass for a workload and click **Explore**. +1. In the left navigation bar, click **Workload**. +1. Go to the workload that will use storage provisioned with the StorageClass that you cared at click **â‹® > Edit Config**. +1. In the **Volume Claim Templates** section, click **Add Claim Template**. +1. Enter a persistent volume name. +1. In the **StorageClass* field, select the StorageClass that will dynamically provision storage for pods managed by this StatefulSet. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Save**. + +**Result:** The workload will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to the PVC. If not, Rancher will provision new persistent storage. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md new file mode 100644 index 00000000000..cce04e02a24 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md @@ -0,0 +1,66 @@ +--- +title: iSCSI Volumes +--- + + + + + +In [Rancher Launched Kubernetes clusters](../../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) that store data on iSCSI volumes, you may experience an issue where kubelets fail to automatically connect with iSCSI volumes. This failure is likely due to an incompatibility issue involving the iSCSI initiator tool. You can resolve this issue by installing the iSCSI initiator tool on each of your cluster nodes. + +Rancher Launched Kubernetes clusters storing data on iSCSI volumes leverage the [iSCSI initiator tool](http://www.open-iscsi.com/), which is embedded in the kubelet's `rancher/hyperkube` Docker image. From each kubelet (i.e., the _initiator_), the tool discovers and launches sessions with an iSCSI volume (i.e., the _target_). However, in some instances, the versions of the iSCSI initiator tool installed on the initiator and the target may not match, resulting in a connection failure. + +If you encounter this issue, you can work around it by installing the initiator tool on each node in your cluster. You can install the iSCSI initiator tool by logging into your cluster nodes and entering one of the following commands: + +| Platform | Package Name | Install Command | +| ------------- | ----------------------- | -------------------------------------- | +| Ubuntu/Debian | `open-iscsi` | `sudo apt install open-iscsi` | +| RHEL | `iscsi-initiator-utils` | `yum install iscsi-initiator-utils -y` | + + +After installing the initiator tool on your nodes, edit the YAML for your cluster, editing the kubelet configuration to mount the iSCSI binary and configuration, as shown in the sample below. + +:::note Notes + +- Before updating your Kubernetes YAML to mount the iSCSI binary and configuration, make sure either the `open-iscsi` (deb) or `iscsi-initiator-utils` (yum) package is installed on your cluster nodes. If this package isn't installed _before_ the bind mounts are created in your Kubernetes YAML, Docker will automatically create the directories and files on each node and will not allow the package install to succeed. + +- The example YAML below does not apply to K3s, but only to RKE clusters. Since the K3s kubelet does not run in a container, adding extra binds is not necessary. However, all iSCSI tools must still be installed on your K3s nodes. + +::: + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" +``` + +If you're using RHEL 7.9, you need to mount the file `/usr/lib64/libcrypto.so.10` as shown in the sample below. + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.10:/usr/lib/libcrypto.so.10" +``` + +If you're using RHEL 8.6, you need to mount the file `/usr/lib64/libcrypto.so.1.1` as shown in the sample below. + +``` +services: + kubelet: + extra_binds: + - "/etc/iscsi:/etc/iscsi" + - "/sbin/iscsiadm:/sbin/iscsiadm" + - "/usr/lib64/libcrypto.so.1.1:/usr/lib/libcrypto.so.1.1" +``` + + +:::tip + +To verify iscsiadm is working properly, you can run the command `docker exec kubelet iscsiadm --version` on any node. + +::: diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md new file mode 100644 index 00000000000..60661aea03b --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md @@ -0,0 +1,89 @@ +--- +title: Setting up Existing Storage +--- + + + + + +This section describes how to set up existing persistent storage for workloads in Rancher. + +:::note + +This section assumes that you understand the Kubernetes concepts of persistent volumes and persistent volume claims. For more information, refer to the section on [how storage works.](about-persistent-storage.md) + +::: + +To set up storage, follow these steps: + +1. [Set up persistent storage.](#1-set-up-persistent-storage) +2. [Add a PersistentVolume that refers to the persistent storage.](#2-add-a-persistentvolume-that-refers-to-the-persistent-storage) +3. [Use the Storage Class for Pods Deployed with a StatefulSet.](#3-use-the-storage-class-for-pods-deployed-with-a-statefulset) + +### Prerequisites + +- To create a persistent volume as a Kubernetes resource, you must have the `Manage Volumes` [role.](../../../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-role-reference) +- If you are provisioning storage for a cluster hosted in the cloud, the storage and cluster hosts must have the same cloud provider. + +### 1. Set up persistent storage + +Creating a persistent volume in Rancher will not create a storage volume. It only creates a Kubernetes resource that maps to an existing volume. Therefore, before you can create a persistent volume as a Kubernetes resource, you must have storage provisioned. + +The steps to set up a persistent storage device will differ based on your infrastructure. We provide examples of how to set up storage using [vSphere,](../../provisioning-storage-examples/vsphere-storage.md) [NFS,](../../provisioning-storage-examples/nfs-storage.md) or Amazon's [EBS.](../../provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) + +If you have a pool of block storage, and you don't want to use a cloud provider, Longhorn could help you provide persistent storage to your Kubernetes cluster. For more information, see [this page.](../../../../../integrations-in-rancher/longhorn.md) + +### 2. Add a PersistentVolume that refers to the persistent storage + +These steps describe how to set up a PersistentVolume at the cluster level in Kubernetes. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add a persistent volume and click **Explore**. +1. In the left navigation bar, click **Storage > Persistent Volumes**. +1. Click **Create**. +1. Enter a **Name** for the persistent volume. +1. Select the **Volume Plugin** for the disk type or service that you're using. When adding storage to a cluster that's hosted by a cloud provider, use the cloud provider's plug-in for cloud storage. For example, if you have a Amazon EC2 cluster and you want to use cloud storage for it, you must use the `Amazon EBS Disk` volume plugin. +1. Enter the **Capacity** of your volume in gigabytes. +1. Complete the **Plugin Configuration** form. Each plugin type requires information specific to the vendor of disk type. For help regarding each plugin's form and the information that's required, refer to the plug-in's vendor documentation. +1. Optional: In the **Customize** form, configure the [access modes.](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) This options sets how many nodes can access the volume, along with the node read/write permissions. The [Kubernetes Documentation](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) includes a table that lists which access modes are supported by the plugins available. +1. Optional: In the **Customize** form, configure the [mount options.](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#mount-options) Each volume plugin allows you to specify additional command line options during the mounting process. Consult each plugin's vendor documentation for the mount options available. +1. Click **Create**. + +**Result:** Your new persistent volume is created. + + +### 3. Use the Storage Class for Pods Deployed with a StatefulSet + +StatefulSets manage the deployment and scaling of Pods while maintaining a sticky identity for each Pod. In this StatefulSet, we will configure a VolumeClaimTemplate. Each Pod managed by the StatefulSet will be deployed with a PersistentVolumeClaim based on this VolumeClaimTemplate. The PersistentVolumeClaim will refer to the PersistentVolume that we created. Therefore, when each Pod managed by the StatefulSet is deployed, it will be bound a PersistentVolume as defined in its PersistentVolumeClaim. + +You can configure storage for the StatefulSet during or after workload creation. + +The following steps describe how to assign existing storage to a new StatefulSet: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to configure storage for the StatefulSet and click **Explore**. +1. In the left navigation bar, click **Workload > StatefulSets**. +1. Click **Create**. +1. Choose the namespace where the workload will be deployed. +1. Enter a name for the StatefulSet. +1. On the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Click **Use an existing Persistent Volume**. +1. In the Persistent Volumes field, select the Persistent Volume that you created. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Launch**. + +**Result:** When the workload is deployed, it will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to the PVC. + +The following steps describe how to assign persistent storage to an existing workload: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to configure storage for the StatefulSet and click **Explore**. +1. In the left navigation bar, click **Workload > StatefulSets**. +1. Go to the workload that you want to add the persistent storage to. Click **â‹® > Edit**. +1. On the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Click **Use an existing Persistent Volume**. +1. In the Persistent Volumes field, select the Persistent Volume that you created. +1. In the **Mount Point** field, enter the path that the workload will use to access the volume. +1. Click **Launch**. + +**Result:** The workload will make a request for the specified amount of disk space to the Kubernetes master. If a PV with the specified resources is available when the workload is deployed, the Kubernetes master will bind the PV to the PVC. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md new file mode 100644 index 00000000000..67252a62748 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver.md @@ -0,0 +1,424 @@ +--- +title: Using an External Ceph Driver +--- + + + + + +These instructions are about using the external Ceph driver in an RKE2 cluster. If you are using RKE, additional steps are required. For details, refer to [this section.](#using-the-ceph-driver-with-rke) + + +## Requirements + +Make sure ceph-common and xfsprogs packages are installed on SLE worker nodes. + +## Using the Ceph Driver with RKE + +The resources below are fully compatible with RKE based clusters, but there is a need to do an additional kubelet configuration for RKE. + +On RKE clusters, the kubelet component is running in a Docker container and doesn't have access to the host's kernel modules as rbd and libceph by default. + +To solve this limitation, you can either run `modprobe rbd` on worker nodes, or configure the kubelet containers to automatically mount the `/lib/modules` directory from the host into the container. + +For the kubelet configuration, put the following lines into the `cluster.yml` file prior to RKE cluster provisioning. You can also modify the `cluster.yml` later in the Rancher UI by clicking on **Edit Cluster > Edit as YAML** and restarting the worker nodes. + +```yaml +services: + kubelet: + extra_binds: + - '/lib/modules:/lib/modules:ro' +``` + +For more information about the `extra_binds` directive, refer to [this section.](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds) + +## Installing the ceph-csi driver on an RKE2 cluster + +:::note + +These steps are needed for dynamic RBD provisioning only. + +::: + +For more information about the `ceph-csi-rbd` chart, refer to [this page.](https://github.com/ceph/ceph-csi/blob/devel/charts/ceph-csi-rbd/README.md) + +To get details about your SES cluster, run: + +``` +ceph mon dump +``` + +Read its output: + +``` +dumped monmap epoch 3 +epoch 3 +fsid 79179d9d-98d8-4976-ab2e-58635caa7235 +last_changed 2021-02-11T10:56:42.110184+0000 +created 2021-02-11T10:56:22.913321+0000 +min_mon_release 15 (octopus) +0: [v2:10.85.8.118:3300/0,v1:10.85.8.118:6789/0] mon.a +1: [v2:10.85.8.123:3300/0,v1:10.85.8.123:6789/0] mon.b +2: [v2:10.85.8.124:3300/0,v1:10.85.8.124:6789/0] mon.c +``` + +Later you'll need the fsid and mon addresses values. + +## Install the ceph-csi Driver Using Helm + +Run these commands: + +``` +helm repo add ceph-csi https://ceph.github.io/csi-charts +helm repo update +helm search repo ceph-csi -l +helm inspect values ceph-csi/ceph-csi-rbd > ceph-csi-rbd-values.yaml +``` + +Modify the `ceph-csi-rbd-values.yaml` file and keep there only the required changes: + +```yaml +# ceph-csi-rbd-values.yaml +csiConfig: + - clusterID: "79179d9d-98d8-4976-ab2e-58635caa7235" + monitors: + - "10.85.8.118:6789" + - "10.85.8.123:6789" + - "10.85.8.124:6789" +provisioner: + name: provisioner + replicaCount: 2 +``` + +Make sure the ceph monitors are reachable from the RKE2 cluster, for example, by ping. + +``` +kubectl create namespace ceph-csi-rbd +helm install --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +kubectl rollout status deployment ceph-csi-rbd-provisioner -n ceph-csi-rbd +helm status ceph-csi-rbd -n ceph-csi-rbd +``` + +in case you'd like to modify the configuration directly via Helm, you may adapt the `ceph-csi-rbd-values.yaml` file and call: + +``` +helm upgrade \ + --namespace ceph-csi-rbd ceph-csi-rbd ceph-csi/ceph-csi-rbd --values ceph-csi-rbd-values.yaml +``` + +## Creating RBD Ceph Resources + +``` +# Create a ceph pool: +ceph osd pool create myPool 64 64 + +# Create a block device pool: +rbd pool init myPool + +# Create a block device image: +rbd create -s 2G myPool/image + +# Create a block device user and record the key: +ceph auth get-or-create-key client.myPoolUser mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=myPool" | tr -d '\n' | base64 +QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== + +# Encode the ceph user myPoolUser into a bash64 hash: +echo "myPoolUser" | tr -d '\n' | base64 +bXlQb29sVXNlcg== + +# Create a block device admin user and record the key: +ceph auth get-or-create-key client.myPoolAdmin mds 'allow *' mgr 'allow *' mon 'allow *' osd 'allow * pool=myPool' | tr -d '\n' | base64 +QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== + +# Encode the ceph user myPoolAdmin into a bash64 hash: +echo "myPoolAdmin" | tr -d '\n' | base64 +bXlQb29sQWRtaW4= +``` +## Configure RBD Ceph Access Secrets + +### User Account + +For static RBD provisioning (the image within the ceph pool must exist), run these commands: + +``` +cat > ceph-user-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-user + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sVXNlcg== + userKey: QVFDZ0R5VmdyRk9KREJBQTJ5b2s5R1E2NUdSWExRQndhVVBwWXc9PQ== +EOF + +kubectl apply -f ceph-user-secret.yaml +``` + +### Admin Account + +For dynamic RBD provisioning (used for automatic image creation within a given ceph pool), run these commands: + +``` +cat > ceph-admin-secret.yaml << EOF +apiVersion: v1 +kind: Secret +metadata: + name: ceph-admin + namespace: default +type: kubernetes.io/rbd +data: + userID: bXlQb29sQWRtaW4= + userKey: QVFCK0hDVmdXSjQ1T0JBQXBrc0VtcVhlZFpjc0JwaStIcmU5M3c9PQ== +EOF + +kubectl apply -f ceph-admin-secret.yaml +``` + +## Create RBD Testing Resources + +### Using RBD in Pods + +```yaml +# pod +cat > ceph-rbd-pod-inline.yaml << EOF +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-inline +spec: + containers: + - name: ceph-rbd-pod-inline + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +EOF + +kubectl apply -f ceph-rbd-pod-inline.yaml +kubectl get pod +kubectl exec pod/ceph-rbd-pod-inline -- df -k | grep rbd +``` + +### Using RBD in Persistent Volumes + +```yaml +# pod-pvc-pv +cat > ceph-rbd-pod-pvc-pv-allinone.yaml << EOF +apiVersion: v1 +kind: PersistentVolume +metadata: + name: ceph-rbd-pv +spec: + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + rbd: + monitors: + - 10.85.8.118:6789 + - 10.85.8.123:6789 + - 10.85.8.124:6789 + pool: myPool + image: image + user: myPoolUser + secretRef: + name: ceph-user + fsType: ext4 + readOnly: false +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: ceph-rbd-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: ceph-rbd-pod-pvc-pv +spec: + containers: + - name: ceph-rbd-pod-pvc-pv + image: busybox + command: ["sleep", "infinity"] + volumeMounts: + - mountPath: /mnt/ceph_rbd + name: volume + volumes: + - name: volume + persistentVolumeClaim: + claimName: ceph-rbd-pvc +EOF + +kubectl apply -f ceph-rbd-pod-pvc-pv-allinone.yaml +kubectl get pv,pvc,pod +kubectl exec pod/ceph-rbd-pod-pvc-pv -- df -k | grep rbd +``` + +### Using RBD in Storage Classes + +This example is for dynamic provisioning. The ceph-csi driver is needed. + +```yaml +# pod-pvc-sc +cat > ceph-rbd-pod-pvc-sc-allinone.yaml < /root/.bashrc << EOF +export PATH=$PATH:/var/lib/rancher/rke2/bin/ +export KUBECONFIG=/etc/rancher/rke2/rke2.yaml +EOF + +cat /var/lib/rancher/rke2/server/node-token +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +``` + +### RKE2 Agent/Worker provisioning + +``` +mkdir -p /etc/rancher/rke2/ + +cat > /etc/rancher/rke2/config.yaml << EOF +server: https://10.100.103.23:9345 +token: K10ca0c38d4ff90d8b80319ab34092e315a8b732622e6adf97bc9eb0536REDACTED::server:ec0308000b8a6b595da000efREDACTED +EOF + +curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh - +systemctl enable --now rke2-agent.service +``` + +To import the cluster into Rancher, click **☰ > Cluster Management**. Then on the **Clusters** page, click **Import Existing**. Then run the provided kubectl command on the server/master node. + +## Tested Versions + +OS for running RKE2 nodes: JeOS SLE15-SP2 with installed kernel-default-5.3.18-24.49 + +``` +kubectl version +Client Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.4", GitCommit:"c96aede7b5205121079932896c4ad89bb93260af", GitTreeState:"clean", BuildDate:"2020-06-22T12:00:00Z", GoVersion:"go1.13.11", Compiler:"gc", Platform:"linux/amd64"} +Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.7+rke2r1", GitCommit:"1dd5338295409edcfff11505e7bb246f0d325d15", GitTreeState:"clean", BuildDate:"2021-01-20T01:50:52Z", GoVersion:"go1.15.5b5", Compiler:"gc", Platform:"linux/amd64"} + +helm version +version.BuildInfo{Version:"3.4.1", GitCommit:"c4e74854886b2efe3321e185578e6db9be0a6e29", GitTreeState:"clean", GoVersion:"go1.14.12"} +``` + +Kubernetes version on RKE2 cluster: v1.19.7+rke2r1 + +## Troubleshooting + +In case you are using SUSE's ceph-rook based on SES7, it might be useful to expose the monitors on hostNetwork by editing `rook-1.4.5/ceph/cluster.yaml` and setting `spec.network.hostNetwork=true`. + +Also for operating the ceph-rook cluster, it is useful to deploy a toolbox on the Kubernetes cluster where ceph-rook is provisioned by `kubectl apply -f rook-1.4.5/ceph/toolbox.yaml` Then all the ceph related commands can be executed in the toolbox pod, for example, by running `kubectl exec -it -n rook-ceph rook-ceph-tools-686d8b8bfb-2nvqp -- bash` + +Operating with the ceph - basic commands: + +``` +ceph osd pool stats +ceph osd pool delete myPool myPool --yes-i-really-really-mean-it +rbd list -p myPool +> csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 +> image +``` + +Delete the image: `rbd rm csi-vol-f5d3766c-7296-11eb-b32a-c2b045952d38 -p myPool` + +CephFS commands in rook toolbox: + +``` +ceph -s +ceph fs ls +ceph fs fail cephfs +ceph fs rm cephfs --yes-i-really-mean-it +ceph osd pool delete cephfs_data cephfs_data --yes-i-really-really-mean-it +ceph osd pool delete cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it +``` + +To prepare a cephfs filesystem, you can run this command on a rook cluster: + +``` +kubectl apply -f rook-1.4.5/ceph/filesystem.yaml +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md new file mode 100644 index 00000000000..c8debb174ad --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md @@ -0,0 +1,579 @@ +--- +title: Cluster Autoscaler with AWS EC2 Auto Scaling Groups +--- + + + + + +This guide will show you how to install and use [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/) on Rancher custom clusters using AWS EC2 Auto Scaling Groups. + +We are going to install a Rancher RKE custom cluster with a fixed number of nodes with the etcd and controlplane roles, and a variable nodes with the worker role, managed by `cluster-autoscaler`. + + +## Prerequisites + +These elements are required to follow this guide: + +* The Rancher server is up and running +* You have an AWS EC2 user with proper permissions to create virtual machines, auto scaling groups, and IAM profiles and roles + +### 1. Create a Custom Cluster + +On Rancher server, we should create a custom k8s cluster. Refer [here](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/) to check for version compatibility. + +Be sure that cloud_provider name is set to `amazonec2`. Once cluster is created we need to get: + +* clusterID: `c-xxxxx` will be used on EC2 `kubernetes.io/cluster/` instance tag +* clusterName: will be used on EC2 `k8s.io/cluster-autoscaler/` instance tag +* nodeCommand: will be added on EC2 instance user_data to include new nodes on cluster + + ```sh + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum + ``` + +### 2. Configure the Cloud Provider + +On AWS EC2, we should create a few objects to configure our system. We've defined three distinct groups and IAM profiles to configure on AWS. + +1. Autoscaling group: Nodes that will be part of the EC2 Auto Scaling Group (ASG). The ASG will be used by `cluster-autoscaler` to scale up and down. + * IAM profile: Required by k8s nodes where cluster-autoscaler will be running. It is recommended for Kubernetes master nodes. This profile is called `K8sAutoscalerProfile`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:DescribeTags", + "autoscaling:DescribeLaunchConfigurations", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + +2. Master group: Nodes that will be part of the Kubernetes etcd and/or control planes. This will be out of the ASG. + * IAM profile: Required by the Kubernetes cloud_provider integration. Optionally, `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` can be used instead [using-aws-credentials.](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#using-aws-credentials) This profile is called `K8sMasterProfile`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ] + } + ] + } + ``` + + * IAM role: `K8sMasterRole: [K8sMasterProfile,K8sAutoscalerProfile]` + * Security group: `K8sMasterSg` More info at [RKE ports (custom nodes tab)](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#downstream-kubernetes-cluster-nodes) + * Tags: + `kubernetes.io/cluster/: owned` + * User data: `K8sMasterUserData` Ubuntu 18.04(ami-0e11cbb34015ff725), installs docker and add etcd+controlplane node to the k8s cluster + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--etcd --controlplane" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +3. Worker group: Nodes that will be part of the k8s worker plane. Worker nodes will be scaled by cluster-autoscaler using the ASG. + * IAM profile: Provides cloud_provider worker integration. + This profile is called `K8sWorkerProfile`. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*" + } + ] + } + ``` + + * IAM role: `K8sWorkerRole: [K8sWorkerProfile]` + * Security group: `K8sWorkerSg` More info at [RKE ports (custom nodes tab)](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#downstream-kubernetes-cluster-nodes) + * Tags: + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` + * User data: `K8sWorkerUserData` Ubuntu 18.04(ami-0e11cbb34015ff725), installs docker and add worker node to the k8s cluster + + ```sh + #!/bin/bash -x + + cat < /etc/sysctl.d/90-kubelet.conf + vm.overcommit_memory = 1 + vm.panic_on_oom = 0 + kernel.panic = 10 + kernel.panic_on_oops = 1 + kernel.keys.root_maxkeys = 1000000 + kernel.keys.root_maxbytes = 25000000 + EOF + sysctl -p /etc/sysctl.d/90-kubelet.conf + + curl -sL https://releases.rancher.com/install-docker/19.03.sh | sh + sudo usermod -aG docker ubuntu + + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + PRIVATE_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/local-ipv4) + PUBLIC_IP=$(curl -H "X-aws-ec2-metadata-token: ${TOKEN}" -s http://169.254.169.254/latest/meta-data/public-ipv4) + K8S_ROLES="--worker" + + sudo docker run -d --privileged --restart=unless-stopped --net=host -v /etc/kubernetes:/etc/kubernetes -v /var/run:/var/run rancher/rancher-agent: --server https:// --token --ca-checksum --address ${PUBLIC_IP} --internal-address ${PRIVATE_IP} ${K8S_ROLES} + ``` + +More info is at [RKE clusters on AWS](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md) and [Cluster Autoscaler on AWS.](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md) + +### 3. Deploy Nodes + +Once we've configured AWS, let's create VMs to bootstrap our cluster: + +* master (etcd+controlplane): Depending your needs, deploy three master instances with proper size. More info is at [the recommendations for production-ready clusters.](../../../../pages-for-subheaders/checklist-for-production-ready-clusters.md) + * IAM role: `K8sMasterRole` + * Security group: `K8sMasterSg` + * Tags: + * `kubernetes.io/cluster/: owned` + * User data: `K8sMasterUserData` + +* worker: Define an ASG on EC2 with the following settings: + * Name: `K8sWorkerAsg` + * IAM role: `K8sWorkerRole` + * Security group: `K8sWorkerSg` + * Tags: + * `kubernetes.io/cluster/: owned` + * `k8s.io/cluster-autoscaler/: true` + * `k8s.io/cluster-autoscaler/enabled: true` + * User data: `K8sWorkerUserData` + * Instances: + * minimum: 2 + * desired: 2 + * maximum: 10 + +Once the VMs are deployed, you should have a Rancher custom cluster up and running with three master and two worker nodes. + +### 4. Install Cluster-autoscaler + +At this point, we should have rancher cluster up and running. We are going to install cluster-autoscaler on master nodes and `kube-system` namespace, following cluster-autoscaler recommendation. + +#### Parameters + +This table shows cluster-autoscaler parameters for fine tuning: + +| Parameter | Default | Description | +|---|---|---| +|cluster-name|-|Autoscaled cluster name, if available| +|address|:8085|The address to expose Prometheus metrics| +|kubernetes|-|Kubernetes master location. Leave blank for default| +|kubeconfig|-|Path to kubeconfig file with authorization and master location information| +|cloud-config|-|The path to the cloud provider configuration file. Empty string for no configuration file| +|namespace|"kube-system"|Namespace in which cluster-autoscaler run| +|scale-down-enabled|true|Should CA scale down the cluster| +|scale-down-delay-after-add|"10m"|How long after scale up that scale down evaluation resumes| +|scale-down-delay-after-delete|0|How long after node deletion that scale down evaluation resumes, defaults to scanInterval| +|scale-down-delay-after-failure|"3m"|How long after scale down failure that scale down evaluation resumes| +|scale-down-unneeded-time|"10m"|How long a node should be unneeded before it is eligible for scale down| +|scale-down-unready-time|"20m"|How long an unready node should be unneeded before it is eligible for scale down| +|scale-down-utilization-threshold|0.5|Sum of cpu or memory of all pods running on the node divided by node's corresponding allocatable resource, below which a node can be considered for scale down| +|scale-down-gpu-utilization-threshold|0.5|Sum of gpu requests of all pods running on the node divided by node's allocatable resource, below which a node can be considered for scale down| +|scale-down-non-empty-candidates-count|30|Maximum number of non empty nodes considered in one iteration as candidates for scale down with drain| +|scale-down-candidates-pool-ratio|0.1|A ratio of nodes that are considered as additional non empty candidates for scale down when some candidates from previous iteration are no longer valid| +|scale-down-candidates-pool-min-count|50|Minimum number of nodes that are considered as additional non empty candidates for scale down when some candidates from previous iteration are no longer valid| +|node-deletion-delay-timeout|"2m"|Maximum time CA waits for removing delay-deletion.cluster-autoscaler.kubernetes.io/ annotations before deleting the node| +|scan-interval|"10s"|How often cluster is reevaluated for scale up or down| +|max-nodes-total|0|Maximum number of nodes in all node groups. Cluster autoscaler will not grow the cluster beyond this number| +|cores-total|"0:320000"|Minimum and maximum number of cores in cluster, in the format `:.` Cluster autoscaler will not scale the cluster beyond these numbers| +|memory-total|"0:6400000"|Minimum and maximum number of gigabytes of memory in cluster, in the format `:.` Cluster autoscaler will not scale the cluster beyond these numbers| +cloud-provider|-|Cloud provider type| +|max-bulk-soft-taint-count|10|Maximum number of nodes that can be tainted/untainted PreferNoSchedule at the same time. Set to 0 to turn off such tainting| +|max-bulk-soft-taint-time|"3s"|Maximum duration of tainting/untainting nodes as PreferNoSchedule at the same time| +|max-empty-bulk-delete|10|Maximum number of empty nodes that can be deleted at the same time| +|max-graceful-termination-sec|600|Maximum number of seconds CA waits for pod termination when trying to scale down a node| +|max-total-unready-percentage|45|Maximum percentage of unready nodes in the cluster. After this is exceeded, CA halts operations| +|ok-total-unready-count|3|Number of allowed unready nodes, irrespective of max-total-unready-percentage| +|scale-up-from-zero|true|Should CA scale up when there 0 ready nodes| +|max-node-provision-time|"15m"|Maximum time CA waits for node to be provisioned| +|nodes|-|sets min,max size and other configuration data for a node group in a format accepted by cloud provider. Can be used multiple times. Format: `::`| +|node-group-auto-discovery|-|One or more definition(s) of node group auto-discovery. A definition is expressed `:[[=]]`| +|estimator|-|"binpacking"|Type of resource estimator to be used in scale up. Available values: ["binpacking"]| +|expander|"random"|Type of node group expander to be used in scale up. Available values: `["random","most-pods","least-waste","price","priority"]`| +|ignore-daemonsets-utilization|false|Should CA ignore DaemonSet pods when calculating resource utilization for scaling down| +|ignore-mirror-pods-utilization|false|Should CA ignore Mirror pods when calculating resource utilization for scaling down| +|write-status-configmap|true|Should CA write status information to a configmap| +|max-inactivity|"10m"|Maximum time from last recorded autoscaler activity before automatic restart| +|max-failing-time|"15m"|Maximum time from last recorded successful autoscaler run before automatic restart| +|balance-similar-node-groups|false|Detect similar node groups and balance the number of nodes between them| +|node-autoprovisioning-enabled|false|Should CA autoprovision node groups when needed| +|max-autoprovisioned-node-group-count|15|The maximum number of autoprovisioned groups in the cluster| +|unremovable-node-recheck-timeout|"5m"|The timeout before we check again a node that couldn't be removed before| +|expendable-pods-priority-cutoff|-10|Pods with priority below cutoff will be expendable. They can be killed without any consideration during scale down and they don't cause scale up. Pods with null priority (PodPriority disabled) are non expendable| +|regional|false|Cluster is regional| +|new-pod-scale-up-delay|"0s"|Pods less than this old will not be considered for scale-up| +|ignore-taint|-|Specifies a taint to ignore in node templates when considering to scale a node group| +|balancing-ignore-label|-|Specifies a label to ignore in addition to the basic and cloud-provider set of labels when comparing if two node groups are similar| +|aws-use-static-instance-list|false|Should CA fetch instance types in runtime or use a static list. AWS only| +|profiling|false|Is debug/pprof endpoint enabled| + +#### Deployment + +Based on the [cluster-autoscaler-run-on-control-plane.yaml](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml) example, we've created our own `cluster-autoscaler-deployment.yaml` to use preferred [auto-discovery setup](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#auto-discovery-setup), updating tolerations, nodeSelector, image version and command config: + + +```yml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + operator: "Equal" + value: "true" + key: node-role.kubernetes.io/controlplane + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + containers: + - image: eu.gcr.io/k8s-artifacts-prod/autoscaling/cluster-autoscaler: + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=4 + - --stderrthreshold=info + - --cloud-provider=aws + - --skip-nodes-with-local-storage=false + - --expander=least-waste + - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/ + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + +``` + +Once the manifest file is prepared, deploy it in the Kubernetes cluster (Rancher UI can be used instead): + +```sh +kubectl -n kube-system apply -f cluster-autoscaler-deployment.yaml +``` + +:::note + +Cluster-autoscaler deployment can also be set up using [manual configuration](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws#manual-configuration) + +::: + +## Testing + +At this point, we should have a cluster-scaler up and running in our Rancher custom cluster. Cluster-scale should manage `K8sWorkerAsg` ASG to scale up and down between 2 and 10 nodes, when one of the following conditions is true: + +* There are pods that failed to run in the cluster due to insufficient resources. In this case, the cluster is scaled up. +* There are nodes in the cluster that have been underutilized for an extended period of time and their pods can be placed on other existing nodes. In this case, the cluster is scaled down. + +### Generating Load + +We've prepared a `test-deployment.yaml` just to generate load on the Kubernetes cluster and see if cluster-autoscaler is working properly. The test deployment is requesting 1000m CPU and 1024Mi memory by three replicas. Adjust the requested resources and/or replica to be sure you exhaust the Kubernetes cluster resources: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hello-world + name: hello-world +spec: + replicas: 3 + selector: + matchLabels: + app: hello-world + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: hello-world + spec: + containers: + - image: rancher/hello-world + imagePullPolicy: Always + name: hello-world + ports: + - containerPort: 80 + protocol: TCP + resources: + limits: + cpu: 1000m + memory: 1024Mi + requests: + cpu: 1000m + memory: 1024Mi +``` + +Once the test deployment is prepared, deploy it in the Kubernetes cluster default namespace (Rancher UI can be used instead): + +``` +kubectl -n default apply -f test-deployment.yaml +``` + +### Checking Scale + +Once the Kubernetes resources got exhausted, cluster-autoscaler should scale up worker nodes where pods failed to be scheduled. It should scale up until up until all pods became scheduled. You should see the new nodes on the ASG and on the Kubernetes cluster. Check the logs on the `kube-system` cluster-autoscaler pod. + +Once scale up is checked, let check for scale down. To do it, reduce the replica number on the test deployment until you release enough Kubernetes cluster resources to scale down. You should see nodes disappear on the ASG and on the Kubernetes cluster. Check the logs on the `kube-system` cluster-autoscaler pod. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md new file mode 100644 index 00000000000..0d4b15bc48c --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md @@ -0,0 +1,134 @@ +--- +title: Cluster Templates +--- + + + + + +Cluster templates encompass both Kubernetes configuration and node pool configuration, allowing a single template to contain all the information Rancher needs to provision new nodes in a cloud provider and install Kubernetes on those nodes. + +## Overview + +Cluster templates are provided as Helm charts. To use them, you will need to clone and fork the templates, change them according to your use case, and then install the Helm charts on the Rancher management cluster. When the Helm chart is installed on the Rancher management cluster, a new cluster resource is created, which Rancher uses to provision the new cluster. + +After the cluster is provisioned using the template, no changes to the template will affect the cluster. After the cluster is created from the cluster template, its configuration and infrastructure can change, because no restrictions are enforced by cluster templates. + +### Kubernetes Distribution + +Cluster templates can use any Kubernetes distribution. For now, we provide an example with an RKE2 Kubernetes cluster. We may provide more examples in the future using other Kubernetes distributions. + +### Versioning + +Rancher doesn't manage version control for cluster templates. Version control is handled in the repository containing the template's Helm chart. + +## RKE2 Cluster Template + +See the example repository for an [RKE2 cluster template](https://github.com/rancher/cluster-template-examples). + +## Adding a Cluster Template to Rancher + +In this section, you'll learn how to add the cluster template to the `local` cluster's chart repo list. The result is that Rancher will include the cluster template as an option when users install new Kubernetes clusters. + +:::note Prerequisites: + +- You will need permission to install Helm charts on the `local` Rancher cluster. +- To make the chart viewable in the cluster creation form, both the chart and the index.yaml file must have the annotation, `catalog.cattle.io/type: cluster-template`. + +::: + +1. Go to a cluster template example repository. Rancher's examples are in [this GitHub repository.](https://github.com/rancher/cluster-template-examples) +1. Fork the repository. +1. Optional: Edit the cluster options by editing the `values.yaml` file. For help editing the file, see the cluster template's Helm chart README. +1. Add the chart repository to Rancher. Click **☰ > Cluster Management**. +1. Go to the `local` cluster and click **Explore.** +1. In the left navigation bar, click **Apps > Chart Repositories.** +1. Click **Create.** +1. Enter a name for the cluster template repository. +1. Click **Git Repository containing Helm chart definitions.** +1. In the **Git Repo URL** field, enter the URL for the repository. For example, `https://github.com/rancher/cluster-template-examples.git`. +1. In the **Git Branch** field, enter the branch to use as the source for the template. Rancher's example repository uses `main`. +1. Click **Create.** + +**Result:** The cluster template available from the **Apps** in Rancher's `local` cluster. It can now be used to deploy clusters. + +:::note Restricted Admin access: + +If you are a restricted admin and don’t have access to the `local` cluster, you may still add new RKE2 templates and manage cluster repositories. To navigate to the chart repository, go to the left navigation bar and click **☰ > Cluster Management > Advanced > Repositories**. You will bypass steps 1 - 6 above, then proceed to follow steps 7 - 12 to create the cluster template. + +::: + +## Creating a Cluster from a Cluster Template + +:::note Prerequisites: + +- You will need permission to provision new Kubernetes clusters. +- You will need cloud credentials for provisioning infrastructure using the template. +- In order to show in the form for creating new clusters, the cluster template's Helm chart must have the `catalog.cattle.io/type: cluster-template` annotation. + +::: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create.** +1. Click the name of your cluster template. +1. Finish installing the Helm chart. + +**Result:** After Rancher provisions the new cluster, it is managed in the same way as any other Rancher-launched Kubernetes cluster. You can configure any options through the UI if the cluster template has options for the user to choose from. + +## Updating a Cluster Created from a Cluster Template + +You can update any clusters using a template from the **Apps > Installed Apps** page, given there is a new version of a template being used by those clusters. + +## Deploying Clusters from a Template with Fleet + +:::note Prerequisites: + +- You will need permission to provision new Kubernetes clusters. +- You will need cloud credentials for provisioning infrastructure using the template. +- In order to show in the form for creating new clusters, the cluster template's Helm chart must have the `catalog.cattle.io/type:cluster-template` annotation. +- In order to use a template as part of continuous delivery/GitOps, the cluster template needs to be deployed in the `fleet-local` namespace of the `local` cluster. +- All values must be set in the `values.yaml` of the template. +- Fleet repositories must follow these [guidelines](https://fleet.rancher.io/gitrepo-content). For RKE2 cluster templates, that means a `fleet.yaml` file must be added to the repository. + +::: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create.** +1. Click **Create Cluster from Template.** + +**Result:** After Rancher provisions the new cluster, it is managed by Fleet. + +## Uninstalling Cluster Templates + +1. Click **☰ > Cluster Management**. +1. Go to the `local` cluster and click **Apps > Chart Repositories.** +1. Go to the chart repository for your cluster template and click **â‹® > Delete.** +1. Confirm the deletion. + +**Result:** The cluster template is uninstalled. This action does not affect clusters created with the cluster template. + +An admin with access to the `local` cluster can also remove a cluster deployed via cluster templates through the **Apps > Installed Apps** page. + +## Configuration Options + +Cluster templates are flexible enough that they can be used to configure all of the following options: + +- Node configuration +- Node pools +- Pre-specified cloud credentials +- Enable/configure an authorized cluster endpoint to get kubectl access to the cluster without using Rancher as a proxy +- Install Rancher V2 monitoring +- Kubernetes version +- Assign cluster members +- Infrastructure configuration such as AWS VPC/subnets or vSphere data center +- Cloud provider options +- Pod security options +- Network providers +- Ingress controllers +- Network security configuration +- Network plugins +- Private registry URL and credentials +- Add-ons +- Kubernetes options, including configurations for Kubernetes components such as kube-api, kube-controller, kubelet, and services + +For details on how to configure the template, refer to the cluster template's Helm chart README. \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md new file mode 100644 index 00000000000..2e7f151e5c2 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md @@ -0,0 +1,196 @@ +--- +title: Nodes and Node Pools +--- + + + + + +After you launch a Kubernetes cluster in Rancher, you can manage individual nodes from the cluster's **Node** tab. + +1. Click **☰** in the top left corner. +1. Select **Cluster Management**. +1. Find the cluster whose nodes you want to manage, and click the **Explore** button at the end of the row. +1. Select **Nodes** from the left navigation. + +Depending on the [option used](../../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) to provision the cluster, there are different node options available. + +:::note + +If you want to manage the _cluster_ and not individual nodes, see [Editing Clusters](../../../pages-for-subheaders/cluster-configuration.md). + +::: + + +## Node Options Available for Each Cluster Creation Option + +The following table lists which node options are available for each type of cluster in Rancher. Click the links in the **Option** column for more detailed information about each feature. + +| Option | [Nodes Hosted by an Infrastructure Provider][1] | [Custom Node][2] | [Hosted Cluster][3] | [Registered EKS Nodes][4] | [All Other Registered Nodes][5] | Description | +| ------------------------------------------------ | ------------------------------------------------ | ---------------- | ------------------- | ------------------- | -------------------| ------------------------------------------------------------------ | +| [Cordon](#cordoning-a-node) | ✓ | ✓ | ✓ | ✓ | ✓ | Marks the node as unschedulable. | +| [Drain](#draining-a-node) | ✓ | ✓ | ✓ | ✓ | ✓ | Marks the node as unschedulable _and_ evicts all pods. | +| [Edit](#managing-and-editing-individual-nodes) | ✓ | ✓ | ✓ | ✓ | ✓ | Enter a custom name, description, label, or taints for a node. | +| [View API](#viewing-a-node-in-the-rancher-api) | ✓ | ✓ | ✓ | ✓ | ✓ | View API data. | +| [Delete](#deleting-a-node) | ✓ | ✓ | | * | * | Deletes defective nodes from the cluster. | +| [Download Keys](#ssh-into-a-node-hosted-by-an-infrastructure-provider) | ✓ | | | | | Download SSH key in order to SSH into the node. | +| [Node Scaling](#scaling-nodes) | ✓ | | | ✓ | | Scale the number of nodes in the node pool up or down. | + +[1]: ../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md +[2]: ../../../pages-for-subheaders/use-existing-nodes.md +[3]: ../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md +[4]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md +[5]: ../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md + +\* Delete option accessible via View API + + +### Nodes Hosted by an Infrastructure Provider + +Node pools are available when you provision Rancher-launched Kubernetes clusters on nodes that are [hosted in an infrastructure provider.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) + +Clusters provisioned using [one of the node pool options](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) can be scaled up or down if the node pool is edited. + +A node pool can also automatically maintain the node scale that's set during the initial cluster provisioning if [node auto-replace is enabled.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) This scale determines the number of active nodes that Rancher maintains for the cluster. + +Rancher uses [node templates](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) to replace nodes in the node pool. Each node template uses cloud provider credentials to allow Rancher to set up the node in the infrastructure provider. + +### Nodes Provisioned by Hosted Kubernetes Providers + +Options for managing nodes [hosted by a Kubernetes provider](../../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md) are somewhat limited in Rancher. Rather than using the Rancher UI to make edits such as scaling the number of nodes up or down, edit the cluster directly. + +### Registered Nodes + +Although you can deploy workloads to a [registered cluster](../../new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md) using Rancher, you cannot manage individual cluster nodes. All management of imported cluster nodes must take place outside of Rancher. + +## Managing and Editing Individual Nodes + +Editing a node lets you: + +* Change its name +* Change its description +* Add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +* Add/Remove [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + +To manage individual nodes, browse to the cluster that you want to manage and then select **Nodes** from the main menu. You can open the options menu for a node by clicking its **â‹®** icon (**..**.). + +## Viewing a Node in the Rancher API + +Select this option to view the node's [API endpoints](../../../pages-for-subheaders/about-the-api.md). + +## Deleting a Node + +Use **Delete** to remove defective nodes from the cloud provider. + +When you the delete a defective node, Rancher can automatically replace it with an identically provisioned node if the node is in a node pool and [node auto-replace is enabled.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) + +:::tip + +If your cluster is hosted by an infrastructure provider, and you want to scale your cluster down instead of deleting a defective node, [scale down](#scaling-nodes) rather than delete. + +::: + +## Scaling Nodes + +For nodes hosted by an infrastructure provider, you can scale the number of nodes in each [node pool](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-pools) by using the scale controls. This option isn't available for other cluster types. + +## SSH into a Node Hosted by an Infrastructure Provider + +For [nodes hosted by an infrastructure provider](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md), you have the option of downloading its SSH key so that you can connect to it remotely from your desktop. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to SSH into a node and click the name of the cluster. +1. On the **Machine Pools** tab, find the node that you want to remote into and click **â‹® > Download SSH Key**. A ZIP file containing files used for SSH will be downloaded. +1. Extract the ZIP file to any location. +1. Open Terminal. Change your location to the extracted ZIP file. +1. Enter the following command: + + ``` + ssh -i id_rsa root@ + ``` + +## Cordoning a Node + +_Cordoning_ a node marks it as unschedulable. This feature is useful for performing short tasks on the node during small maintenance windows, like reboots, upgrades, or decommissions. When you're done, power back on and make the node schedulable again by uncordoning it. + +## Draining a Node + +_Draining_ is the process of first cordoning the node, and then evicting all its pods. This feature is useful for performing node maintenance (like kernel upgrades or hardware maintenance). It prevents new pods from deploying to the node while redistributing existing pods so that users don't experience service interruption. + +- For pods with a replica set, the pod is replaced by a new pod that will be scheduled to a new node. Additionally, if the pod is part of a service, then clients will automatically be redirected to the new pod. + +- For pods with no replica set, you need to bring up a new copy of the pod, and assuming it is not part of a service, redirect clients to it. + +You can drain nodes that are in either a `cordoned` or `active` state. When you drain a node, the node is cordoned, the nodes are evaluated for conditions they must meet to be drained, and then (if it meets the conditions) the node evicts its pods. + +However, you can override the conditions draining when you initiate the drain. You're also given an opportunity to set a grace period and timeout value. + +### Aggressive and Safe Draining Options + +When you configure the upgrade strategy for the cluster, you will be able to enable node draining. If node draining is enabled, you will be able to configure how pods are deleted and rescheduled. + +- **Aggressive Mode** + + In this mode, pods won't get rescheduled to a new node, even if they do not have a controller. Kubernetes expects you to have your own logic that handles the deletion of these pods. + + Kubernetes also expects the implementation to decide what to do with pods using emptyDir. If a pod uses emptyDir to store local data, you might not be able to safely delete it, since the data in the emptyDir will be deleted once the pod is removed from the node. Choosing aggressive mode will delete these pods. + +- **Safe Mode** + + If a node has standalone pods or ephemeral data it will be cordoned but not drained. +### Grace Period + +The timeout given to each pod for cleaning things up, so they will have chance to exit gracefully. For example, when pods might need to finish any outstanding requests, roll back transactions or save state to some external storage. If negative, the default value specified in the pod will be used. + +### Timeout + +The amount of time drain should continue to wait before giving up. + +:::note Kubernetes Known Issue: + +The [timeout setting](https://github.com/kubernetes/kubernetes/pull/64378) was not enforced while draining a node before Kubernetes 1.12. + +::: + +### Drained and Cordoned State + +If there's any error related to user input, the node enters a `cordoned` state because the drain failed. You can either correct the input and attempt to drain the node again, or you can abort by uncordoning the node. + +If the drain continues without error, the node enters a `draining` state. You'll have the option to stop the drain when the node is in this state, which will stop the drain process and change the node's state to `cordoned`. + +Once drain successfully completes, the node will be in a state of `drained`. You can then power off or delete the node. + +**Want to know more about cordon and drain?** See the [Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/). + +## Labeling a Node to be Ignored by Rancher + +Some solutions, such as F5's BIG-IP integration, may require creating a node that is never registered to a cluster. + +Since the node will never finish registering, it will always be shown as unhealthy in the Rancher UI. + +In that case, you may want to label the node to be ignored by Rancher so that Rancher only shows nodes as unhealthy when they are actually failing. + +You can label nodes to be ignored by using a setting in the Rancher UI, or by using `kubectl`. + +:::note + +There is an [open issue](https://github.com/rancher/rancher/issues/24172) in which nodes labeled to be ignored can get stuck in an updating state. + +::: + + +### Labeling Nodes to be Ignored with kubectl + +To add a node that will be ignored by Rancher, use `kubectl` to create a node that has the following label: + +``` +cattle.rancher.io/node-status: ignore +``` + +**Result:** If you add the node to a cluster, Rancher will not attempt to sync with this node. The node can still be part of the cluster and can be listed with `kubectl`. + +If the label is added before the node is added to the cluster, the node will not be shown in the Rancher UI. + +If the label is added after the node is added to a Rancher cluster, the node will not be removed from the UI. + +If you delete the node from the Rancher server using the Rancher UI or API, the node will not be removed from the cluster if the `nodeName` is listed in the Rancher settings in the Rancher API under `v3/settings/ignore-node-name`. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md new file mode 100644 index 00000000000..d74b70822d3 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md @@ -0,0 +1,201 @@ +--- +title: Projects and Kubernetes Namespaces with Rancher +description: Rancher Projects ease the administrative burden of your cluster and support multi-tenancy. Learn to create projects and divide projects into Kubernetes namespaces +--- + + + + + +A namespace is a Kubernetes concept that allows a virtual cluster within a cluster, which is useful for dividing the cluster into separate "virtual clusters" that each have their own access control and resource quotas. + +A project is a group of namespaces, and it is a concept introduced by Rancher. Projects allow you to manage multiple namespaces as a group and perform Kubernetes operations in them. You can use projects to support multi-tenancy, so that a team can access a project within a cluster without having access to other projects in the same cluster. + +:::note + +Projects are de-emphasized on the UI because it is not required to create Kubernetes resources within a project scope. However, resources such as [Secrets](../../new-user-guides/kubernetes-resources-setup/secrets.md#creating-secrets-in-projects) can still be created in a project scope if the legacy feature flag is enabled. + +::: + +This section describes how projects and namespaces work with Rancher. + +## About Namespaces + +A namespace is a concept introduced by Kubernetes. According to the [official Kubernetes documentation on namespaces,](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) + +:::note + +Kubernetes supports multiple virtual clusters backed by the same physical cluster. These virtual clusters are called namespaces. [...] Namespaces are intended for use in environments with many users spread across multiple teams, or projects. For clusters with a few to tens of users, you should not need to create or think about namespaces at all. + +::: + +Namespaces provide the following functionality: + +- **Providing a scope for names:** Names of resources need to be unique within a namespace, but not across namespaces. Namespaces can not be nested inside one another and each Kubernetes resource can only be in one namespace. +- **Resource quotas:** Namespaces provide a way to divide cluster resources between multiple users. + +You can assign resources at the project level so that each namespace in the project can use them. You can also bypass this inheritance by assigning resources explicitly to a namespace. + +You can assign the following resources directly to namespaces: + +- [Workloads](../../../pages-for-subheaders/workloads-and-pods.md) +- [Load Balancers/Ingress](../../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [Service Discovery Records](../../new-user-guides/kubernetes-resources-setup/create-services.md) +- [Persistent Volume Claims](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [Certificates](../../new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMaps](../../new-user-guides/kubernetes-resources-setup/configmaps.md) +- [Registries](../../new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [Secrets](../../new-user-guides/kubernetes-resources-setup/secrets.md) + +To manage permissions in a vanilla Kubernetes cluster, cluster admins configure role-based access policies for each namespace. With Rancher, user permissions are assigned on the project level instead, and permissions are automatically inherited by any namespace owned by the particular project. + +For more information on creating and moving namespaces, see [Namespaces](../../new-user-guides/manage-namespaces.md). + +### Role-based access control issues with namespaces and kubectl + +Because projects are a concept introduced by Rancher, kubectl does not have the capability to restrict the creation of namespaces to a project the creator has access to. + +This means that when standard users with project-scoped permissions create a namespaces with `kubectl`, it may be unusable because `kubectl` doesn't require the new namespace to be scoped within a certain project. + +If your permissions are restricted to the project level, it is better to [create a namespace through Rancher](../../new-user-guides/manage-namespaces.md) to ensure that you will have permission to access the namespace. + +If a standard user is a project owner, the user will be able to create namespaces within that project. The Rancher UI will prevent that user from creating namespaces outside the scope of the projects they have access to. + +## About Projects + +In terms of hierarchy: + +- Clusters contain projects +- Projects contain namespaces + +You can use projects to support multi-tenancy, so that a team can access a project within a cluster without having access to other projects in the same cluster. + +In the base version of Kubernetes, features like role-based access rights or cluster resources are assigned to individual namespaces. A project allows you to save time by giving an individual or a team access to multiple namespaces simultaneously. + +You can use projects to perform actions such as: + +- Assign users to a group of namespaces (i.e., [project membership](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md)). +- Assign users specific roles in a project. A role can be owner, member, read-only, or [custom](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md). +- Assign resources to the project. +- Assign Pod Security Policies. + +When you create a cluster, two projects are automatically created within it: + +- [Default Project](#the-clusters-default-project) +- [System Project](#the-system-project) + +### The Cluster's Default Project + +When you provision a cluster with Rancher, it automatically creates a `default` project for the cluster. This is a project you can use to get started with your cluster, but you can always delete it and replace it with projects that have more descriptive names. + +If you don't have a need for more than the default namespace, you also do not need more than the **Default** project in Rancher. + +If you require another level of organization beyond the **Default** project, you can create more projects in Rancher to isolate namespaces, applications and resources. + +### The System Project + +When troubleshooting, you can view the `system` project to check if important namespaces in the Kubernetes system are working properly. This easily accessible project saves you from troubleshooting individual system namespace containers. + +To open it, open the cluster view and click **Cluster > Projects/Namespaces**. This view shows all of the namespaces in the `system` project. + +The `system` project: + +- Is automatically created when you provision a cluster. +- Lists all namespaces that exist in `v3/settings/system-namespaces`, if they exist. +- Allows you to add more namespaces or move its namespaces to other projects. +- Cannot be deleted because it's required for cluster operations. + +:::note + +In RKE clusters where the project network isolation option is enabled, the `system` project overrides the project network isolation option so that it can communicate with other projects, collect logs, and check health. + +::: + +## Project Authorization + +Standard users are only authorized for project access in two situations: + +- An administrator, cluster owner or cluster member explicitly adds the standard user to the project's **Members** tab. +- Standard users can access projects that they create themselves. + +## Pod Security Policies + +Rancher extends Kubernetes to allow the application of [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) at the project level in addition to the cluster level. However, as a best practice, we recommend applying Pod Security Policies at the cluster level. + +## Creating Projects + +This section describes how to create a new project with a name and with optional pod security policy, members, and resource quotas. + +1. [Name a new project.](#1-name-a-new-project) +2. [Optional: Select a pod security policy.](#2-optional-select-a-pod-security-policy) +3. [Recommended: Add project members.](#3-recommended-add-project-members) +4. [Optional: Add resource quotas.](#4-optional-add-resource-quotas) + +### 1. Name a New Project + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to project in and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Click **Create Project**. +1. Enter a **Project Name**. + +### 2. Optional: Select a Pod Security Policy + +This option is only available if you've already created a Pod Security Policy. For instruction, see [Creating Pod Security Policies](../authentication-permissions-and-global-configuration/create-pod-security-policies.md). + +Assigning a PSP to a project will: + +- Override the cluster's default PSP. +- Apply the PSP to the project. +- Apply the PSP to any namespaces you add to the project later. + +### 3. Recommended: Add Project Members + +Use the **Members** section to provide other users with project access and roles. + +By default, your user is added as the project `Owner`. + +:::note Notes on Permissions: + +- Users assigned the `Owner` or `Member` role for a project automatically inherit the `namespace creation` role. However, this role is a [Kubernetes ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole), meaning its scope extends to all projects in the cluster. Therefore, users explicitly assigned the `Owner` or `Member` role for a project can create namespaces in other projects they're assigned to, even with only the `Read Only` role assigned. + +- By default, the Rancher role of `project-member` inherits from the `Kubernetes-edit` role, and the `project-owner` role inherits from the `Kubernetes-admin` role. As such, both `project-member` and `project-owner` roles will allow for namespace management, including the ability to create and delete namespaces. + +- Choose `Custom` to create a custom role on the fly: [Custom Project Roles](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#custom-project-roles). + +::: + +To add members: + +1. In the **Members** tab, click **Add**. +1. From the **Select Member** field, search for a user or group that you want to assign project access. Note: You can only search for groups if external authentication is enabled. +1. In the **Project Permissions** section, choose a role. For more information, refer to the [documentation on project roles.](../authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) + +### 4. Optional: Add Resource Quotas + +Resource quotas limit the resources that a project (and its namespaces) can consume. For more information, see [Resource Quotas](projects-and-namespaces.md). + +To add a resource quota, + +1. In the **Resource Quotas** tab, click **Add Resource**. +1. Select a **Resource Type**. For more information, see [Resource Quotas.](projects-and-namespaces.md). +1. Enter values for the **Project Limit** and the **Namespace Default Limit**. +1. **Optional:** Specify **Container Default Resource Limit**, which will be applied to every container started in the project. The parameter is recommended if you have CPU or Memory limits set by the Resource Quota. It can be overridden on per an individual namespace or a container level. For more information, see [Container Default Resource Limit](../../../pages-for-subheaders/manage-project-resource-quotas.md) +1. Click **Create**. + +**Result:** Your project is created. You can view it from the cluster's **Projects/Namespaces** view. + +| Field | Description | +| ----------------------- | -------------------------------------------------------------------------------------------------------- | +| Project Limit | The overall resource limit for the project. | +| Namespace Default Limit | The default resource limit available for each namespace. This limit is propagated to each namespace in the project when created. The combined limit of all project namespaces shouldn't exceed the project limit. | + +## Deleting Projects + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster the project is attached to, and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the project you want to delete, and click **â‹®**. +1. Select **Delete**. + +When you delete a project, any namespaces that were formerly associated with the project will remain on the cluster. You can find these namespaces in the Rancher UI, in the **Not in a Project** tab of the **Projects/Namespaces** page. You can reassign these namespaces to a project by [moving](../manage-namespaces.md#moving-namespaces-to-another-project) them. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md new file mode 100644 index 00000000000..9898df21025 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md @@ -0,0 +1,79 @@ +--- +title: NFS Storage +--- + + + + + +Before you can use the NFS storage volume plug-in with Rancher deployments, you need to provision an NFS server. + +:::note + +- If you already have an NFS share, you don't need to provision a new NFS server to use the NFS volume plugin within Rancher. Instead, skip the rest of this procedure and complete [adding storage](../../../../pages-for-subheaders/create-kubernetes-persistent-storage.md). + +- This procedure demonstrates how to set up an NFS server using Ubuntu, although you should be able to use these instructions for other Linux distros (e.g. Debian, RHEL, Arch Linux, etc.). For official instruction on how to create an NFS server using another Linux distro, consult the distro's documentation. + +::: + +:::note Recommended: + +To simplify the process of managing firewall rules, use NFSv4. + +::: + +1. Using a remote Terminal connection, log into the Ubuntu server that you intend to use for NFS storage. + +1. Enter the following command: + + ``` + sudo apt-get install nfs-kernel-server + ``` + +1. Enter the command below, which sets the directory used for storage, along with user access rights. Modify the command if you'd like to keep storage at a different directory. + + ``` + mkdir -p /nfs && chown nobody:nogroup /nfs + ``` + - The `-p /nfs` parameter creates a directory named `nfs` at root. + - The `chown nobody:nogroup /nfs` parameter allows all access to the storage directory. + +1. Create an NFS exports table. This table sets the directory paths on your NFS server that are exposed to the nodes that will use the server for storage. + + 1. Open `/etc/exports` using your text editor of choice. + 1. Add the path of the `/nfs` folder that you created in step 3, along with the IP addresses of your cluster nodes. Add an entry for each IP address in your cluster. Follow each address and its accompanying parameters with a single space that is a delimiter. + + ``` + /nfs (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) (rw,sync,no_subtree_check) + ``` + + :::tip + + You can replace the IP addresses with a subnet. For example: `10.212.50.12/24` + + ::: + + 1. Update the NFS table by entering the following command: + + ``` + exportfs -ra + ``` + +1. Open the ports used by NFS. + + 1. To find out what ports NFS is using, enter the following command: + + ``` + rpcinfo -p | grep nfs + ``` + 2. [Open the ports](https://help.ubuntu.com/lts/serverguide/firewall.html.en) that the previous command outputs. For example, the following command opens port 2049: + + ``` + sudo ufw allow 2049 + ``` + +**Result:** Your NFS server is configured to be used for storage with your Rancher nodes. + +## What's Next? + +Within Rancher, add the NFS server as a storage volume and/or storage class. After adding the server, you can use it for storage for your deployments. diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md new file mode 100644 index 00000000000..059bb688673 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md @@ -0,0 +1,19 @@ +--- +title: Creating Persistent Storage in Amazon's EBS +--- + + + + + +This section describes how to set up Amazon's Elastic Block Store in EC2. + +1. From the EC2 console, go to the **ELASTIC BLOCK STORE** section in the left panel and click **Volumes**. +1. Click **Create Volume**. +1. Optional: Configure the size of the volume or other options. The volume should be created in the same availability zone as the instance it will be attached to. +1. Click **Create Volume**. +1. Click **Close**. + +**Result:** Persistent storage has been created. + +For details on how to set up the newly created storage in Rancher, refer to the section on [setting up existing storage.](../../../new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md new file mode 100644 index 00000000000..b13cee6512d --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md @@ -0,0 +1,76 @@ +--- +title: vSphere Storage +--- + + + + + +To provide stateful workloads with vSphere storage, we recommend creating a vSphereVolume StorageClass. This practice dynamically provisions vSphere storage when workloads request volumes through a PersistentVolumeClaim. + +In order to dynamically provision storage in vSphere, the vSphere provider must be enabled. See the following pages for more: [Out-of-tree vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md) and [in-tree vSphere](../../../new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md). + + +### Prerequisites + +In order to provision vSphere volumes in a cluster created with the [Rancher Kubernetes Engine (RKE)](../../../../pages-for-subheaders/launch-kubernetes-with-rancher.md), the [vSphere cloud provider](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/vsphere) must be explicitly enabled in the [cluster options](../../../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md). + +### Creating a StorageClass + +:::tip + +The following steps can also be performed using the `kubectl` command line tool. See [Kubernetes documentation on persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) for details. + +::: + +1. Click **☰ > Cluster Management**. +1. Choose the cluster you want to provide vSphere storage to and click **Exlpore**. +1. In the left navigation bar, select **Storage > StorageClasses**. +1. Click **Create**. +3. Enter a **Name** for the StorageClass. +4. Under **Provisioner**, select **VMWare vSphere Volume**. + + ![](/img/vsphere-storage-class.png) + +5. Optionally, specify additional properties for this storage class under **Parameters**. Refer to the [vSphere storage documentation](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/blob/master/documentation/storageclass.md) for details. +5. Click **Create**. + +### Creating a Workload with a vSphere Volume + +1. In the left navigation bar, click **Workload**. +1. Click **Create**. +1. Click **StatefulSet**. +1. In the **Volume Claim Templates** tab, click **Add Claim Template**. +1. Enter a persistent volume name. +1. In the Storage Class field, select the vSphere StorageClass that you created. +6. Enter the required **Capacity** for the volume. Then click **Define**. +7. Assign a path in the **Mount Point** field. This is the full path where the volume will be mounted in the container file system, e.g. `/persistent`. +8. Click **Create**. + +### Verifying Persistence of the Volume + +1. In the left navigation bar, click **Workload > Pods**. +1. Go to the workload you just created and click **â‹® > Execute Shell**. +2. Note the directory at root where the volume has been mounted to (in this case `/persistent`). +3. Create a file in the volume by executing the command `touch //data.txt`. +4. Close the shell window. +5. Click on the name of the workload to reveal detail information. +7. Click **â‹® > Delete**. +8. Observe that the pod is deleted. Then a new pod is scheduled to replace it so that the workload maintains its configured scale of a single stateful pod. +9. Once the replacement pod is running, click **Execute Shell**. +10. Inspect the contents of the directory where the volume is mounted by entering `ls -l /`. Note that the file you created earlier is still present. + + ![workload-persistent-data](/img/workload-persistent-data.png) + +### Why to Use StatefulSets Instead of Deployments + +You should always use [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) for workloads consuming vSphere storage, as this resource type is designed to address a VMDK block storage caveat. + +Since vSphere volumes are backed by VMDK block storage, they only support an [access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims) of `ReadWriteOnce`. This setting restricts the volume so that it can only be mounted to a single pod at a time, unless all pods consuming that volume are co-located on the same node. This behavior makes a deployment resource unusable for scaling beyond a single replica if it consumes vSphere volumes. + +Even using a deployment resource with just a single replica may result in a deadlock situation while updating the deployment. If the updated pod is scheduled to a node different from where the existing pod lives, it will fail to start because the VMDK is still attached to the other node. + +### Related Links + +- [vSphere Storage for Kubernetes](https://github.com/vmware-archive/vsphere-storage-for-kubernetes/tree/master/documentation) +- [Kubernetes Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md new file mode 100644 index 00000000000..7cb1f6bad37 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md @@ -0,0 +1,82 @@ +--- +title: Certificate Rotation +--- + + + + + +:::caution + +Rotating Kubernetes certificates may result in your cluster being temporarily unavailable as components are restarted. For production environments, it's recommended to perform this action during a maintenance window. + +::: + +By default, Kubernetes clusters require certificates and Rancher launched Kubernetes clusters automatically generate certificates for the Kubernetes components. Rotating these certificates is important before the certificates expire as well as if a certificate is compromised. After the certificates are rotated, the Kubernetes components are automatically restarted. + +Certificates can be rotated for the following services: + + + + +- etcd +- kubelet (node certificate) +- kubelet (serving certificate, if [enabled](https://rancher.com/docs/rke/latest/en/config-options/services/#kubelet-options)) +- kube-apiserver +- kube-proxy +- kube-scheduler +- kube-controller-manager + + + + +- admin +- api-server +- controller-manager +- scheduler +- rke2-controller +- rke2-server +- cloud-controller +- etcd +- auth-proxy +- kubelet +- kube-proxy + + + + +:::note + +For users who didn't rotate their webhook certificates, and they have expired after one year, please see this [page](../../../troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md) for help. + +::: + +### Certificate Rotation + +Rancher launched Kubernetes clusters have the ability to rotate the auto-generated certificates through the UI. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to rotate certificates for amd click **â‹® > Rotate Certificates**. +1. Select which certificates that you want to rotate. + + * Rotate all Service certificates (keep the same CA) + * Rotate an individual service and choose one of the services from the drop-down menu + +1. Click **Save**. + +**Results:** The selected certificates will be rotated and the related services will be restarted to start using the new certificate. + +### Additional Notes + + + + +Even though the RKE CLI can use custom certificates for the Kubernetes cluster components, Rancher currently doesn't allow the ability to upload these in Rancher launched Kubernetes clusters. + + + + +In RKE2, both etcd and control plane nodes are treated as the same `server` concept. As such, when rotating certificates of services specific to either of these components will result in certificates being rotated on both. The certificates will only change for the specified service, but you will see nodes for both components go into an updating state. You may also see worker only nodes go into an updating state. This is to restart the workers after a certificate change to ensure they get the latest client certs. + + + diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md new file mode 100644 index 00000000000..5d56f2c60aa --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key.md @@ -0,0 +1,52 @@ +--- +title: Encryption Key Rotation +--- + + + + + +### RKE1 Encryption Key Rotation + +1. Enable encryption key rotation with either of the following two options: + + - Select the `Enabled` radio button in the Rancher UI under **Cluster Options > Advanced Options > Secrets Encryption**: + + ![Enable Encryption Key Rotation](/img/rke1-enable-secrets-encryption.png) + + - OR, apply the following YAML: + + ```yaml + rancher_kubernetes_engine_config: + services: + kube_api: + secrets_encryption_config: + enabled: true + ``` + +2. Rotate keys in the Rancher UI: + + 2.1. Click **☰ > Cluster Management**. + + 2.2. Select **â‹® > Rotate Encryption Keys** on the far right of the screen next to your chosen cluster: + + ![Encryption Key Rotation](/img/rke1-encryption-key.png) + + + +### RKE2 Encryption Key Rotation + +_**New in v2.6.7**_ + +>**Important:** Encryption key rotation is enabled by default and cannot be disabled. + +To rotate keys in the Rancher UI: + +1. Click **☰ > Cluster Management**. + +1. Select **â‹® > Rotate Encryption Keys** on the far right of the screen next to your chosen cluster: + + ![Encryption Key Rotation](/img/rke2-encryption-key.png) + + +>**Note:** For more information on RKE2 secrets encryption config, please see the [RKE2 docs](https://docs.rke2.io/security/secrets_encryption). \ No newline at end of file diff --git a/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-namespaces.md b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-namespaces.md new file mode 100644 index 00000000000..0419be358a6 --- /dev/null +++ b/versioned_docs/version-latest/how-to-guides/new-user-guides/manage-namespaces.md @@ -0,0 +1,78 @@ +--- +title: Namespaces +--- + + + + + +Within Rancher, you can further divide projects into different [namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/), which are virtual clusters within a project backed by a physical cluster. Should you require another level of organization beyond projects and the `default` namespace, you can use multiple namespaces to isolate applications and resources. + +Although you assign resources at the project level so that each namespace in the project can use them, you can override this inheritance by assigning resources explicitly to a namespace. + +Resources that you can assign directly to namespaces include: + +- [Workloads](../../pages-for-subheaders/workloads-and-pods.md) +- [Load Balancers/Ingress](../../pages-for-subheaders/load-balancer-and-ingress-controller.md) +- [Service Discovery Records](kubernetes-resources-setup/create-services.md) +- [Persistent Volume Claims](../../pages-for-subheaders/create-kubernetes-persistent-storage.md) +- [Certificates](kubernetes-resources-setup/encrypt-http-communication.md) +- [ConfigMaps](kubernetes-resources-setup/configmaps.md) +- [Registries](kubernetes-resources-setup/kubernetes-and-docker-registries.md) +- [Secrets](kubernetes-resources-setup/secrets.md) + +To manage permissions in a vanilla Kubernetes cluster, cluster admins configure role-based access policies for each namespace. With Rancher, user permissions are assigned on the project level instead, and permissions are automatically inherited by any namespace owned by the particular project. + +:::note + +If you create a namespace with `kubectl`, it may be unusable because `kubectl` doesn't require your new namespace to be scoped within a project that you have access to. If your permissions are restricted to the project level, it is better to [create a namespace through Rancher](manage-namespaces.md) to ensure that you will have permission to access the namespace. + +::: + +### Creating Namespaces + +Create a new namespace to isolate apps and resources in a project. + +:::tip + +When working with project resources that you can assign to a namespace (i.e., [workloads](kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md), [certificates](kubernetes-resources-setup/encrypt-http-communication.md), [ConfigMaps](kubernetes-resources-setup/configmaps.md), etc.) you can create a namespace on the fly. + +::: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to create a namespace and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the project where you want to add a namespace and click **Create Namespace**. Alternately, go to **Not in a Project** to create a namespace not associated with a project. + +1. **Optional:** If your project has [Resource Quotas](../../pages-for-subheaders/manage-project-resource-quotas.md) in effect, you can override the default resource **Limits** (which places a cap on the resources that the namespace can consume). + +1. Enter a **Name** and then click **Create**. + +**Result:** Your namespace is added to the project. You can begin assigning cluster resources to the namespace. + +### Moving Namespaces to Another Project + +Cluster admins and members may occasionally need to move a namespace to another project, such as when you want a different team to start using the application. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to move a namespace and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the namespace you want to move and click **â‹® > Move**. + +1. Select the namespace(s) that you want to move to a different project. Then click **Move**. You can move multiple namespaces at one. + + :::note Notes: + + - Don't move the namespaces in the `System` project. Moving these namespaces can adversely affect cluster networking. + - You cannot move a namespace into a project that already has a [resource quota](../../pages-for-subheaders/manage-project-resource-quotas.md)configured. + - If you move a namespace from a project that has a quota set to a project with no quota set, the quota is removed from the namespace. + +1. Choose a new project for the new namespace and then click **Move**. Alternatively, you can remove the namespace from all projects by selecting **None**. + +**Result:** Your namespace is moved to a different project (or is unattached from all projects). If any project resources are attached to the namespace, the namespace releases them and then attached resources from the new project. + +### Editing Namespace Resource Quotas + +You can always override the namespace default limit to provide a specific namespace with access to more (or less) project resources. + +For more information, see how to [edit namespace resource quotas](../advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/cis-scans/configuration-reference.md b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/configuration-reference.md new file mode 100644 index 00000000000..fa9012e6ed8 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/configuration-reference.md @@ -0,0 +1,105 @@ +--- +title: Configuration +--- + +This configuration reference is intended to help you manage the custom resources created by the `rancher-cis-benchmark` application. These resources are used for performing CIS scans on a cluster, skipping tests, setting the test profile that will be used during a scan, and other customization. + +To configure the custom resources, go to the **Cluster Dashboard** To configure the CIS scans, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to configure CIS scans and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark**. + +### Scans + +A scan is created to trigger a CIS scan on the cluster based on the defined profile. A report is created after the scan is completed. + +When configuring a scan, you need to define the name of the scan profile that will be used with the `scanProfileName` directive. + +An example ClusterScan custom resource is below: + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScan +metadata: + name: rke-cis +spec: + scanProfileName: rke-profile-hardened +``` + +### Profiles + +A profile contains the configuration for the CIS scan, which includes the benchmark version to use and any specific tests to skip in that benchmark. + +:::caution + +By default, a few ClusterScanProfiles are installed as part of the `rancher-cis-benchmark` chart. If a user edits these default benchmarks or profiles, the next chart update will reset them back. So it is advisable for users to not edit the default ClusterScanProfiles. + +::: + +Users can clone the ClusterScanProfiles to create custom profiles. + +Skipped tests are listed under the `skipTests` directive. + +When you create a new profile, you will also need to give it a name. + +An example `ClusterScanProfile` is below: + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanProfile +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + labels: + app.kubernetes.io/managed-by: Helm + name: "" +spec: + benchmarkVersion: cis-1.5 + skipTests: + - "1.1.20" + - "1.1.21" +``` + +### Benchmark Versions + +A benchmark version is the name of benchmark to run using `kube-bench`, as well as the valid configuration parameters for that benchmark. + +A `ClusterScanBenchmark` defines the CIS `BenchmarkVersion` name and test configurations. The `BenchmarkVersion` name is a parameter provided to the `kube-bench` tool. + +By default, a few `BenchmarkVersion` names and test configurations are packaged as part of the CIS scan application. When this feature is enabled, these default BenchmarkVersions will be automatically installed and available for users to create a ClusterScanProfile. + +:::caution + +If the default BenchmarkVersions are edited, the next chart update will reset them back. Therefore we don't recommend editing the default ClusterScanBenchmarks. + +::: + +A ClusterScanBenchmark consists of the fields: + +- `ClusterProvider`: This is the cluster provider name for which this benchmark is applicable. For example: RKE, EKS, GKE, etc. Leave it empty if this benchmark can be run on any cluster type. +- `MinKubernetesVersion`: Specifies the cluster's minimum kubernetes version necessary to run this benchmark. Leave it empty if there is no dependency on a particular Kubernetes version. +- `MaxKubernetesVersion`: Specifies the cluster's maximum Kubernetes version necessary to run this benchmark. Leave it empty if there is no dependency on a particular k8s version. + +An example `ClusterScanBenchmark` is below: + +```yaml +apiVersion: cis.cattle.io/v1 +kind: ClusterScanBenchmark +metadata: + annotations: + meta.helm.sh/release-name: clusterscan-operator + meta.helm.sh/release-namespace: cis-operator-system + creationTimestamp: "2020-08-28T18:18:07Z" + generation: 1 + labels: + app.kubernetes.io/managed-by: Helm + name: cis-1.5 + resourceVersion: "203878" + selfLink: /apis/cis.cattle.io/v1/clusterscanbenchmarks/cis-1.5 + uid: 309e543e-9102-4091-be91-08d7af7fb7a7 +spec: + clusterProvider: "" + minKubernetesVersion: 1.15.0 +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/cis-scans/custom-benchmark.md b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/custom-benchmark.md new file mode 100644 index 00000000000..8ba63bbe8f7 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/custom-benchmark.md @@ -0,0 +1,78 @@ +--- +title: Creating a Custom Benchmark Version for Running a Cluster Scan +--- + +Each Benchmark Version defines a set of test configuration files that define the CIS tests to be run by the kube-bench tool. +The `rancher-cis-benchmark` application installs a few default Benchmark Versions which are listed under CIS Benchmark application menu. + +But there could be some Kubernetes cluster setups that require custom configurations of the Benchmark tests. For example, the path to the Kubernetes config files or certs might be different than the standard location where the upstream CIS Benchmarks look for them. + +It is now possible to create a custom Benchmark Version for running a cluster scan using the `rancher-cis-benchmark` application. + +When a cluster scan is run, you need to select a Profile which points to a specific Benchmark Version. + +Follow all the steps below to add a custom Benchmark Version and run a scan using it. + +### 1. Prepare the Custom Benchmark Version ConfigMap + +To create a custom benchmark version, first you need to create a ConfigMap containing the benchmark version's config files and upload it to your Kubernetes cluster where you want to run the scan. + +To prepare a custom benchmark version ConfigMap, suppose we want to add a custom Benchmark Version named `foo`. + +1. Create a directory named `foo` and inside this directory, place all the config YAML files that the kube-bench tool looks for. For example, here are the config YAML files for a Generic CIS 1.5 Benchmark Version https://github.com/aquasecurity/kube-bench/tree/master/cfg/cis-1.5 +1. Place the complete `config.yaml` file, which includes all the components that should be tested. +1. Add the Benchmark version name to the `target_mapping` section of the `config.yaml`: + + ```yaml + target_mapping: + "foo": + - "master" + - "node" + - "controlplane" + - "etcd" + - "policies" + ``` +1. Upload this directory to your Kubernetes Cluster by creating a ConfigMap: + + ```yaml + kubectl create configmap -n foo --from-file= + ``` + +### 2. Add a Custom Benchmark Version to a Cluster + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark > Benchmark Version**. +1. Click **Create**. +1. Enter the **Name** and a description for your custom benchmark version. +1. Choose the cluster provider that your benchmark version applies to. +1. Choose the ConfigMap you have uploaded from the dropdown. +1. Add the minimum and maximum Kubernetes version limits applicable, if any. +1. Click **Create**. + +### 3. Create a New Profile for the Custom Benchmark Version + +To run a scan using your custom benchmark version, you need to add a new Profile pointing to this benchmark version. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark > Profile**. +1. Click **Create**. +1. Provide a **Name** and description. In this example, we name it `foo-profile`. +1. Choose the Benchmark Version from the dropdown. +1. Click **Create**. + +### 4. Run a Scan Using the Custom Benchmark Version + +Once the Profile pointing to your custom benchmark version `foo` has been created, you can create a new Scan to run the custom test configs in the Benchmark Version. + +To run a scan, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to add a custom benchmark and click **Explore**. +1. In the left navigation bar, click **CIS Benchmark > Scan**. +1. Click **Create**. +1. Choose the new cluster scan profile. +1. Click **Create**. + +**Result:** A report is generated with the scan results. To see the results, click the name of the scan that appears. diff --git a/versioned_docs/version-latest/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md new file mode 100644 index 00000000000..8a88240d963 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/rbac-for-cis-scans.md @@ -0,0 +1,48 @@ +--- +title: Roles-based Access Control +--- + +This section describes the permissions required to use the rancher-cis-benchmark App. + +The rancher-cis-benchmark is a cluster-admin only feature by default. + +However, the `rancher-cis-benchmark` chart installs these two default `ClusterRoles`: + +- cis-admin +- cis-view + +In Rancher, only cluster owners and global administrators have `cis-admin` access by default. + +Note: If you were using the `cis-edit` role added in Rancher v2.5 setup, it has now been removed since +Rancher v2.5.2 because it essentially is same as `cis-admin`. If you happen to create any clusterrolebindings +for `cis-edit`, please update them to use `cis-admin` ClusterRole instead. + +## Cluster-Admin Access + +Rancher CIS Scans is a cluster-admin only feature by default. +This means only the Rancher global admins, and the cluster’s cluster-owner can: + +- Install/Uninstall the rancher-cis-benchmark App +- See the navigation links for CIS Benchmark CRDs - ClusterScanBenchmarks, ClusterScanProfiles, ClusterScans +- List the default ClusterScanBenchmarks and ClusterScanProfiles +- Create/Edit/Delete new ClusterScanProfiles +- Create/Edit/Delete a new ClusterScan to run the CIS scan on the cluster +- View and Download the ClusterScanReport created after the ClusterScan is complete + + +## Summary of Default Permissions for Kubernetes Default Roles + +The rancher-cis-benchmark creates three `ClusterRoles` and adds the CIS Benchmark CRD access to the following default K8s `ClusterRoles`: + +| ClusterRole created by chart | Default K8s ClusterRole | Permissions given with Role +| ------------------------------| ---------------------------| ---------------------------| +| `cis-admin` | `admin`| Ability to CRUD clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR +| `cis-view` | `view `| Ability to List(R) clusterscanbenchmarks, clusterscanprofiles, clusterscans, clusterscanreports CR + + +By default only cluster-owner role will have ability to manage and use `rancher-cis-benchmark` feature. + +The other Rancher roles (cluster-member, project-owner, project-member) do not have any default permissions to manage and use rancher-cis-benchmark resources. + +But if a cluster-owner wants to delegate access to other users, they can do so by creating ClusterRoleBindings between these users and the above CIS ClusterRoles manually. +There is no automatic role aggregation supported for the `rancher-cis-benchmark` ClusterRoles. diff --git a/versioned_docs/version-latest/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md new file mode 100644 index 00000000000..b11887c76c1 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests.md @@ -0,0 +1,53 @@ +--- +title: Skipped and Not Applicable Tests +--- + +This section lists the tests that are skipped in the permissive test profile for RKE. + +> All the tests that are skipped and not applicable on this page will be counted as Not Applicable in the v2.5 generated report. The skipped test count will only mention the user-defined skipped tests. This allows user-skipped tests to be distinguished from the tests that are skipped by default in the RKE permissive test profile. + +## CIS Benchmark v1.5 + +### CIS Benchmark v1.5 Skipped Tests + +| Number | Description | Reason for Skipping | +| ---------- | ------------- | --------- | +| 1.1.12 | Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) | A system service account is required for etcd data directory ownership. Refer to Rancher's hardening guide for more details on how to configure this ownership. | +| 1.2.6 | Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | +| 1.2.16 | Ensure that the admission control plugin PodSecurityPolicy is set (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 1.2.33 | Ensure that the --encryption-provider-config argument is set as appropriate (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | +| 1.2.34 | Ensure that encryption providers are appropriately configured (Manual) | Enabling encryption changes how data can be recovered as data is encrypted. | +| 4.2.6 | Ensure that the --protect-kernel-defaults argument is set to true (Automated) | System level configurations are required before provisioning the cluster in order for this argument to be set to true. | +| 4.2.10 | Ensure that the--tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) | When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. | +| 5.1.5 | Ensure that default service accounts are not actively used. (Automated) | Kubernetes provides default service accounts to be used. | +| 5.2.2 | Minimize the admission of containers wishing to share the host process ID namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.2.3 | Minimize the admission of containers wishing to share the host IPC namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.2.4 | Minimize the admission of containers wishing to share the host network namespace (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.2.5 | Minimize the admission of containers with allowPrivilegeEscalation (Automated) | Enabling Pod Security Policy can cause applications to unexpectedly fail. | +| 5.3.2 | Ensure that all Namespaces have Network Policies defined (Automated) | Enabling Network Policies can prevent certain applications from communicating with each other. | +| 5.6.4 | The default namespace should not be used (Automated) | Kubernetes provides a default namespace. | + +### CIS Benchmark v1.5 Not Applicable Tests + +| Number | Description | Reason for being not applicable | +| ---------- | ------------- | --------- | +| 1.1.1 | Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | +| 1.1.2 | Ensure that the API server pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. All configuration is passed in as arguments at container run time. | +| 1.1.3 | Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.1.4 | Ensure that the controller manager pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.1.5 | Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.6 | Ensure that the scheduler pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.7 | Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | +| 1.1.8 | Ensure that the etcd pod specification file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for etcd. All configuration is passed in as arguments at container run time. | +| 1.1.13 | Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | +| 1.1.14 | Ensure that the admin.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. | +| 1.1.15 | Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.16 | Ensure that the scheduler.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for scheduler. All configuration is passed in as arguments at container run time. | +| 1.1.17 | Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.1.18 | Ensure that the controller-manager.conf file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn't require or maintain a configuration file for controller-manager. All configuration is passed in as arguments at container run time. | +| 1.3.6 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | +| 4.1.1 | Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | +| 4.1.2 | Ensure that the kubelet service file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. All configuration is passed in as arguments at container run time. | +| 4.1.9 | Ensure that the kubelet configuration file has permissions set to 644 or more restrictive (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | +| 4.1.10 | Ensure that the kubelet configuration file ownership is set to root:root (Automated) | Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. All configuration is passed in as arguments at container run time. | +| 4.2.12 | Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) | Clusters provisioned by RKE handles certificate rotation directly through RKE. | \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md new file mode 100644 index 00000000000..42937ea2262 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md @@ -0,0 +1,96 @@ +--- +title: Prerequisites +--- + +### 1. Setting Up License Manager and Purchasing Support + +First, complete the [first step](https://docs.aws.amazon.com/license-manager/latest/userguide/getting-started.html) of the license manager one-time setup. +Next, go to the AWS Marketplace. Locate the "Rancher Premium Support Billing Container Starter Pack". Purchase at least one entitlement. + +If you have installed Rancher using the "Rancher Setup" AWS Marketplace offering, skip to [Step 4](#4-create-an-oidc-provider). + +> **Note:** Each entitlement grants access to support for a certain amount of nodes. You can purchase more licenses as necessary later on. + +### 2. Create an EKS Cluster +Follow the [Rancher docs](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md) to create an EKS cluster. When you get to the [final step to install Rancher](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-install-the-rancher-helm-chart), **stop and return to this page**. This cluster will need to meet the following requirements: + +- EKS version 1.22. +- Each node in the cluster has access to the registry containing Rancher and its related images. +- Each node in the cluster has access to the ECR repo storing the CSP Adapter. +- Each node in the cluster has access to the license manager service. +- Each node in the cluster has access to global endpoints for the STS service. + +### 3. Install Rancher + +In addition to the options specified to install Rancher in the [Rancher docs](../../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md#8-install-the-rancher-helm-chart), you will also need to enable extra metrics. +This can be done through the Helm CLI through the following options: + +```bash +--set extraEnv\[0\].name="CATTLE_PROMETHEUS_METRICS" --set-string extraEnv\[0\].value=true +``` + +You can also use a values.yaml like the below: + +```yaml +extraEnv: + - name: "CATTLE_PROMETHEUS_METRICS" + value: "true" +``` + +You will also need to install Rancher version 2.6.7 or higher. + +### 4. Create an OIDC Provider + +Follow the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) to create an OIDC provider for the cluster specified in the previous section. + +### 5. Create an IAM Role + +An IAM role is required for the CSP adapter to check-in/check-out entitlements. + +First, configure the trust policy as below. Replace `MY_AWS_ACC` with your AWS account number, `MY_AWS_REGION` with your AWS region, and `MY_OIDC_PROVIDER` with the id of your OIDC provider: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${MY_AWS_ACC}:oidc-provider/oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:sub": "system:serviceaccount:cattle-csp-adapter-system:rancher-csp-adapter", + "oidc.eks.${MY_AWS_REGION}.amazonaws.com/id/${MY_OIDC_PROVIDER}:aud": "sts.amazonaws.com" + } + } + } + ] +} +``` + +Next, use a policy for the role which has the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "RancherCSPAdapterPermissions", + "Effect": "Allow", + "Action": [ + "license-manager:ListReceivedLicenses", + "license-manager:CheckoutLicense", + "license-manager:ExtendLicenseConsumption", + "license-manager:CheckInLicense", + "license-manager:GetLicense", + "license-manager:GetLicenseUsage" + ], + "Resource": "*" + } + ] +} +``` + +Save the name of the role. You will need it later on when installing the CSP adapter. diff --git a/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md new file mode 100644 index 00000000000..02e6bb86340 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues.md @@ -0,0 +1,28 @@ +--- +title: Common Issues +--- + +**After installing the adapter, a banner message appears in Rancher that says "AWS Marketplace Adapter: Unable to run the adapter, please check the adapter logs"** + +This error indicates that while the adapter was installed into the cluster, an error has occurred which prevents it from properly checking-in/checking-out licenses. + +This often occurs because the IAM role was not set up properly. Review the [prerequisites](./adapter-requirements.md) and verify that: + +- An OIDC provider has been created/associated with the cluster Rancher is running on. +- The IAM role has been configured to trust this OIDC provider. +- The IAM role has at least the permissions outlined in the policy. + +If all of the above have been configured correctly, reach out to support for assistance. + +**I see a banner message that states, "AWS Marketplace Adapter: You have exceeded your licensed node count. At least x more license(s) are required in AWS to become compliant"** + +This message indicates that you do not have enough entitlements for the amount of nodes Rancher is currently managing. + +Keep in mind the following limitations: + +- Each entitlement is valid for a certain number of nodes. +- Every node currently managed by Rancher counts toward your usage total (with exception of nodes in the cluster rancher is installed on). +- Each entitlement can be used by at most one Rancher instance. For example, if you have two running Rancher instances in your account (each installed on a separate EKS cluster), then you will need at least two entitlements. + +You may also have recently uninstalled/re-installed the adapter. If the adapter loses track of the licenses that it is currently managing, it can take up to an hour to resolve the actual state of the licenses. + diff --git a/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md new file mode 100644 index 00000000000..420e05b2d81 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md @@ -0,0 +1,159 @@ +--- +title: Installing the Adapter +--- + + + + + +> **Important:** If you are attempting to re-install the adapter, you may experience errant out-of-compliance messages for up to an hour. + +### Rancher vs. Adapter Compatibility Matrix + +:::note Important: + +Different versions of the CSP adapter rely on features found in specific versions of Rancher. +In order to deploy and run the adapter successfully, you need to ensure its version corresponds to the necessary version of Rancher. + +::: + +| Rancher Version | Adapter Version | +|-----------------|:---------------:| +| v2.7.0 | v2.0.0 | +| v2.7.1 | v2.0.0 | +| v2.7.2 | v2.0.1 | +| v2.7.3 | v2.0.1 | +| v2.7.4 | v2.0.1 | +| v2.7.5 | v2.0.2 | + + +### 1. Gain Access to the Local Cluster + +> **Note:** Only admin users should have access to the local cluster. Because the CSP adapter must be installed in the local cluster, this installation must be carried out by an admin user. + +First, click on the local cluster and download a kubeconfig token. You can then configure your CLI to use this new token with the following command, replacing `$TOKEN_PATH` with the path on your filesystem to the downloaded token: + +```bash +export KUBECONFIG=$TOKEN_PATH +``` + +### 2. Create the Adapter Namespace + +Create the namespace that the adapter will be installed in. + +```bash +kubectl create ns cattle-csp-adapter-system +``` + +### 3. Create Certificate Secrets + +The adapter requires access to the root CA that Rancher is using to communicate with the Rancher server. You can read more about which certificate options Rancher supports in the [chart options page](../../../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md). + +If your Rancher install uses a certificate signed by a recognized Certificate Authority such as Let's Encrypt, then you can safely skip to [Step 4](#4-install-the-chart). + +However, if your Rancher install uses a custom certificate such as a Rancher-generated certificate or one signed by a private Certificate Authority, you will need to provide the certificate for this authority in PEM-encoded format so that the adapter can communicate with Rancher. + +First, retrieve the certificate that Rancher is using and place in a file named `ca-additional.pem`. If you are using the Rancher-generated certs option, this can be done with the following command: + +```bash +kubectl get secret tls-rancher -n cattle-system -o jsonpath="{.data.tls\.crt}" | base64 -d >> ca-additional.pem +``` + +Then, create a secret which uses this cert: + +```bash +kubectl -n cattle-csp-adapter-system create secret generic tls-ca-additional --from-file=ca-additional.pem +``` + +> **Important:** Do not change the names of the file or of the created secret. Making changes to these values may result in errors when the adapter runs. + +### 4. Install the Chart + +First, add the `rancher/charts` repo using the following command: + +```bash +helm repo add rancher-charts https://charts.rancher.io +``` + +Next, install the CSP adapter. You must specify several values, including the account number, and the name of the role created in the prerequisites. + +Ensure that you use the version of the CSP adapter that matches the version of Rancher that you are running, as defined [above](#rancher-vs-adapter-compatibility-matrix). + +For the below instructions, replace `$MY_ACC_NUM` with your AWS account number and `$MY_ROLE_NAME` with the name of the role created in the prerequisites. In addition, replace `$CSP_ADAPTER_VERSION` with the version that matches your Rancher version in the [version matrix](#rancher-vs-adapter-compatibility-matrix). + +> **Note:** If you use shell variables, do not specify quotation marks. For example, MY_ACC_NUM=123456789012 will work, but MY_ACC_NUM="123456789012" will fail. + +> **Note:** Accounts using the AWS Marketplace listing for the EU and the UK will need to specify an additional `--set image.repository=rancher/rancher-csp-adapter-eu` option. To see if your account needs this option when installing the adapter, refer to the usage instructions of the marketplace listing. + + + + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --version $CSP_ADAPTER_VERSION + ``` + + + Alternatively, you can use a `values.yaml` and specify options like below: + + ```yaml + aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME + ``` + + > **Note:** The account number needs to be specified in a string format, like the above, or the installation will fail. + + You can then install the adapter with the following command: + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION + ``` + + + + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter --namespace cattle-csp-adapter-system --set aws.enabled=true --set aws.roleName=$MY_ROLE_NAME --set-string aws.accountNumber=$MY_ACC_NUM --set additionalTrustedCAs=true --version $CSP_ADAPTER_VERSION + ``` + + Alternatively, you can use a `values.yaml` and specify options the below: + + ```yaml + aws: + enabled: true + accountNumber: "$MY_ACC_NUM" + roleName: $MY_ROLE_NAME + additionalTrustedCAs: true + ``` + + > **Note:** The account number needs to be specified in a string format, like the above, or the installation will fail. + + You can then install the adapter with the following command: + + ```bash + helm install rancher-csp-adapter rancher-charts/rancher-csp-adapter -f values.yaml --version $CSP_ADAPTER_VERSION + ``` + + + + +### 5. Managing Certificate Updates + +If you had to create a secret storing a custom cert in [Step 3](#3-create-certificate-secrets), you will need to update this secret over time as the certificate is rotated. + +First, delete the original secret in the cattle-csp-adapter-system namespace, using the below command: + +```bash +kubectl delete secret tls-ca-additional -n cattle-csp-adapter-system +``` + +Then, follow the original installation steps in [Step 3](#3-create-certificate-secrets) to replace the content of the secret with the updated value. + +Finally, restart the rancher-csp-adapter deployment to ensure that the updated value is made available to the adapter: + +```bash +kubectl rollout restart deploy rancher-csp-adapter -n cattle-csp-adapter-system +``` + +> **Note:** There are methods such as cert-manager's [trust operator](https://cert-manager.io/docs/projects/trust/) which can help reduce the number of manual rotation tasks over time. While these options are not officially supported, they may be useful to users wishing to automate some of these tasks. \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md new file mode 100644 index 00000000000..16e0ac3443e --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter.md @@ -0,0 +1,21 @@ +--- +title: Uninstalling The Adapter +--- + +### 1. Uninstall the adapter chart using helm. + +```bash +helm uninstall rancher-csp-adapter -n cattle-csp-adapter-system +``` + +### 2. Remove the namespace created for the adapter. + +```bash +kubectl delete ns cattle-csp-adapter-system +``` + +### 3. (Optional) remove any outstanding user notifications. + +```bash +kubectl delete RancherUserNotification csp-compliance +``` diff --git a/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/supportconfig.md b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/supportconfig.md new file mode 100644 index 00000000000..9d87830a82e --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/cloud-marketplace/supportconfig.md @@ -0,0 +1,53 @@ +--- +title: Supportconfig bundle +--- + +After installing the CSP adapter, you will have the ability to generate a supportconfig bundle. This bundle is a tar file which can be used to quickly provide information to support. + +These bundles can be created through Rancher or through direct access to the cluster that Rancher is installed on. Note that accessing through Rancher is preferred. + +> **Note:** Only admin users can generate/download supportconfig bundles, regardless of method. + +### Accessing through Rancher + +First, click on the hamburger menu. Then click the `Get Support` button. + +![Get Support](/img/support-help.png) + +In the next page, click on the `Generate Support Config` button. + +> **Note:** If the adapter is not installed, the option to generate the supportconfig bundle will not be present. You must install the CSP adapter to generate a supportconfig bundle. + +![Get Support](/img/generate-support-config.png) + +### Accessing without rancher + +First, generate a kubeconfig for the cluster that Rancher is installed on. + +> **Note:** If Rancher is down, you will not be able to use a kubeconfig token generated by Rancher to access the cluster. + +Configure your shell environment to use this kubeconfig token: + +```bash +export KUBECONFIG=$MY_KUBECONFIG_PATH +``` + +It is recommended to create a temporary working directory while running this command, like below: + +```bash +mkdir temp && cd temp +``` + +Then, retrieve the supportconfig bundle: + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && tar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` + +This will create a `supportconfig_rancher.tar` file in your current directory. + +Users who run these commands on Mac may experience issues due to incompatibilities between gnu-tar and bsd-tar. If support has issues reading a supportconfig that you produce, try using the below command after making gnu-tar accessible as `gtar` on your path: + +```bash +mkdir rancher && kubectl get configmap csp-config -n cattle-csp-adapter-system -o=jsonpath='{.data.data}' >> rancher/config.json && gtar -c -f supportconfig_rancher.tar rancher && rm -rf rancher +``` diff --git a/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/architecture.md b/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/architecture.md new file mode 100644 index 00000000000..f012a3a9921 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/architecture.md @@ -0,0 +1,8 @@ +--- +title: Architecture +--- + +Fleet can manage deployments from git of raw Kubernetes YAML, Helm charts, or Kustomize or any combination of the three. Regardless of the source, all resources are dynamically turned into Helm charts, and Helm is used as the engine to deploy everything in the cluster. This gives you a high degree of control, consistency, and auditability. Fleet focuses not only on the ability to scale, but to give one a high degree of control and visibility to exactly what is installed on the cluster. + +![Architecture](/img/fleet-architecture.svg) + diff --git a/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md b/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md new file mode 100644 index 00000000000..e6a3f8cf961 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md @@ -0,0 +1,54 @@ +--- +title: Using Fleet Behind a Proxy +--- + +In this section, you'll learn how to enable Fleet in a setup that has a Rancher server with a public IP a Kubernetes cluster that has no public IP, but is configured to use a proxy. + +Rancher does not establish connections with registered downstream clusters. The Rancher agent deployed on the downstream cluster must be able to establish the connection with Rancher. + +To set up Fleet to work behind a proxy, you will need to set the **Agent Environment Variables** for the downstream cluster. These are cluster-level configuration options. + +Through the Rancher UI, you can configure these environment variables for any cluster type, including registered and custom clusters. The variables can be added while editing an existing cluster or while provisioning a new cluster. + +For public downstream clusters, it is sufficient to [set the required environment variables in the Rancher UI.](#setting-environment-variables-in-the-rancher-ui) + +For private nodes or private clusters, the environment variables need to be set on the nodes themselves. Then the environment variables are configured from the Rancher UI, typically when provisioning a custom cluster or when registering the private cluster. For an example of how to set the environment variables on Ubuntu node in a K3s Kubernetes cluster, see [this section.](#setting-environment-variables-on-private-nodes) + +## Required Environment Variables + +When adding Fleet agent environment variables for the proxy, replace with your private proxy IP. + +| Variable Name | Value | +|------------------|--------| +| `HTTP_PROXY` | http://:8888 | +| `HTTPS_PROXY` | http://:8888 +| `NO_PROXY` | 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local | + +## Setting Environment Variables in the Rancher UI + +To add the environment variable to an existing cluster, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster where you want to add environment variables and click **â‹® > Edit Config**. +1. Click **Advanced Options**. +1. Click **Add Environment Variable**. +1. Enter the [required environment variables](#required-environment-variables) +1. Click **Save**. + +**Result:** The Fleet agent works behind a proxy. + +## Setting Environment Variables on Private Nodes + +For private nodes and private clusters, the proxy environment variables need to be set on the nodes themselves, as well as configured from the Rancher UI. + +This example shows how the environment variables would be set up on an Ubuntu node in a K3s Kubernetes cluster: + +``` +ssh -o ForwardAgent=yes ubuntu@ +ssh +export proxy_private_ip= +export HTTP_PROXY=http://${proxy_private_ip}:8888 +export HTTPS_PROXY=http://${proxy_private_ip}:8888 +export NO_PROXY=127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md b/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md new file mode 100644 index 00000000000..aea98b74dbc --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/fleet-gitops-at-scale/windows-support.md @@ -0,0 +1,21 @@ +--- +title: Windows Support +--- + +Prior to Rancher v2.5.6, the `agent` did not have native Windows manifests on downstream clusters with Windows nodes. This would result in a failing `agent` pod for the cluster. + +If you are upgrading from an older version of Rancher to v2.5.6+, you can deploy a working `agent` with the following workflow *in the downstream cluster*: + +1. Cordon all Windows nodes. +1. Apply the below toleration to the `agent` workload. +1. Uncordon all Windows nodes. +1. Delete all `agent` pods. New pods should be created with the new toleration. +1. Once the `agent` pods are running, and auto-update is enabled for Fleet, they should be updated to a Windows-compatible `agent` version. + +```yaml +tolerations: +- effect: NoSchedule + key: cattle.io/os + operator: Equal + value: linux +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/harvester.md b/versioned_docs/version-latest/integrations-in-rancher/harvester.md new file mode 100644 index 00000000000..872bf854f8d --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/harvester.md @@ -0,0 +1,48 @@ +--- +title: Harvester Integration +--- + +Introduced in Rancher v2.6.1, [Harvester](https://docs.harvesterhci.io/) is an open-source hyper-converged infrastructure (HCI) software built on Kubernetes. Harvester installs on bare metal servers and provides integrated virtualization and distributed storage capabilities. Although Harvester operates using Kubernetes, it does not require users to know Kubernetes concepts, making it a more user-friendly application. + +### Feature Flag + +The Harvester feature flag is used to manage access to the Virtualization Management (VM) page in Rancher where users can navigate directly to Harvester clusters and access the Harvester UI. The Harvester feature flag is enabled by default. Click [here](../pages-for-subheaders/enable-experimental-features.md) for more information on feature flags in Rancher. + +To navigate to the Harvester cluster, click **☰ > Virtualization Management**. From Harvester Clusters page, click one of the clusters listed to go to the single Harvester cluster view. + +* If the Harvester feature flag is enabled, Harvester clusters will be filtered out from any pages or apps (such as Fleet and the multi-cluster app) that list Kubernetes clusters. + +* If the Harvester feature flag is disabled, and a Harvester cluster is imported, the Harvester cluster will be shown in the Rancher cluster list in the Cluster Management page. Harvester clusters will only be shown on the cluster list when the feature flag is off. + +* With the Harvester integration, Harvester clusters can now be imported into Rancher as a cluster type `Harvester`. + +* Users may import a Harvester cluster only on the Virtualization Management page. Importing a cluster on the Cluster Management page is not supported, and a warning will advise you to return to the VM page to do so. + +### Harvester Node Driver + +The [Harvester node driver](https://docs.harvesterhci.io/v1.1/rancher/node/node-driver/) is generally available for RKE and RKE2 options in Rancher. The node driver is available whether or not the Harvester feature flag is enabled. Note that the node driver is off by default. Users may create RKE or RKE2 clusters on Harvester only from the Cluster Management page. + +Harvester allows `.ISO` images to be uploaded and displayed through the Harvester UI, but this is not supported in the Rancher UI. This is because `.ISO` images usually require additional setup that interferes with a clean deployment (without requiring user intervention), and they are not typically used in cloud environments. + +Click [here](../pages-for-subheaders/about-provisioning-drivers.md#node-drivers) for more information on node drivers in Rancher. + +### Port Requirements + +The port requirements for the Harvester cluster can be found [here](https://docs.harvesterhci.io/v1.1/install/requirements#networking). + +In addition, other networking considerations are as follows: + +- Be sure to enable VLAN trunk ports of the physical switch for VM VLAN networks. +- Follow the networking setup guidance [here](https://docs.harvesterhci.io/v1.1/networking/clusternetwork). + +For other port requirements for other guest clusters, such as K3s and RKE1, please see [these docs](https://docs.harvesterhci.io/v1.1/install/requirements/#guest-clusters). + +### Limitations + +--- +**Applicable to Rancher v2.6.1 and v2.6.2 only:** + +- Harvester v0.3.0 doesn’t support air-gapped environment installation. +- Harvester v0.3.0 doesn’t support upgrade from v0.2.0 nor upgrade to v1.0.0. + +--- \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md new file mode 100644 index 00000000000..0c229f288b4 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md @@ -0,0 +1,60 @@ +--- +title: Additional Steps for Installing Istio on RKE2 and K3s Clusters +--- + +When installing or upgrading the Istio Helm chart through **Apps,** + +1. If you are installing the chart, click **Customize Helm options before install** and click **Next**. +1. You will see options for configuring the Istio Helm chart. On the **Components** tab, check the box next to **Enabled CNI**. +1. Add a custom overlay file specifying `cniBinDir` and `cniConfDir`. For more information on these options, refer to the [Istio documentation.](https://istio.io/latest/docs/setup/additional-setup/cni/#helm-chart-parameters) An example is below: + + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /opt/cni/bin + cniConfDir: /etc/cni/net.d +``` + + + +```yaml +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +spec: + components: + cni: + enabled: true + k8s: + overlays: + - apiVersion: "apps/v1" + kind: "DaemonSet" + name: "istio-cni-node" + patches: + - path: spec.template.spec.containers.[name:install-cni].securityContext.privileged + value: true + values: + cni: + cniBinDir: /var/lib/rancher/k3s/data/current/bin + cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d +``` + + + +**Result:** Now you should be able to utilize Istio as desired, including sidecar injection and monitoring via Kiali. diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/pod-security-policies.md b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/pod-security-policies.md new file mode 100644 index 00000000000..e774b97bf8c --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/pod-security-policies.md @@ -0,0 +1,53 @@ +--- +title: Enable Istio with Pod Security Policies +--- + +If you have restrictive Pod Security Policies enabled, then Istio may not be able to function correctly, because it needs certain permissions in order to install itself and manage pod infrastructure. In this section, we will configure a cluster with PSPs enabled for an Istio install, and also set up the Istio CNI plugin. + +The Istio CNI plugin removes the need for each application pod to have a privileged `NET_ADMIN` container. For further information, see the [Istio CNI Plugin docs](https://istio.io/docs/setup/additional-setup/cni). Please note that the [Istio CNI Plugin is in alpha](https://istio.io/about/feature-stages/). + +:::note Prerequisites: + +- The cluster must be an RKE Kubernetes cluster. +- The cluster must have been created with a default PodSecurityPolicy. + +To enable pod security policy support when creating a Kubernetes cluster in the Rancher UI, go to Advanced Options. In the Pod Security Policy Support section, click Enabled. Then select a default pod security policy. + +::: + +1. [Set the PodSecurityPolicy to unrestricted](#1-set-the-podsecuritypolicy-to-unrestricted) +2. [Enable the CNI](#2-enable-the-cni) +3. [Verify that the CNI is working.](#3-verify-that-the-cni-is-working) + +### 1. Set the PodSecurityPolicy to unrestricted + +An unrestricted PSP allows Istio to be installed. + +Set the PSP to `unrestricted` in the project where is Istio is installed, or the project where you plan to install Istio. + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Find the **Project: System** and select the **â‹® > Edit Config**. +1. Change the Pod Security Policy option to be unrestricted, then click **Save**. + +### 2. Enable the CNI + +When installing or upgrading Istio through **Apps,** + +1. Click **Components**. +2. Check the box next to **Enabled CNI**. +3. Finish installing or upgrading Istio. + +The CNI can also be enabled by editing the `values.yaml`: + +``` +istio_cni.enabled: true +``` + +Istio should install successfully with the CNI enabled in the cluster. + +### 3. Verify that the CNI is working + +Verify that the CNI is working by deploying a [sample application](https://istio.io/latest/docs/examples/bookinfo/) or deploying one of your own applications. + diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/project-network-isolation.md b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/project-network-isolation.md new file mode 100644 index 00000000000..16fde314183 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/project-network-isolation.md @@ -0,0 +1,21 @@ +--- +title: Additional Steps for Project Network Isolation +--- + +In clusters where: + +- You are using the Canal network plugin with Rancher before v2.5.8, or you are using Rancher v2.5.8+ with an any RKE network plug-in that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin +- The Project Network Isolation option is enabled +- You install the Istio Ingress module + +The Istio Ingress Gateway pod won't be able to redirect ingress traffic to the workloads by default. This is because all the namespaces will be inaccessible from the namespace where Istio is installed. You have two options. + +The first option is to add a new Network Policy in each of the namespaces where you intend to have ingress controlled by Istio. Your policy should include the following lines: + +``` +- podSelector: + matchLabels: + app: istio-ingressgateway +``` + +The second option is to move the `istio-system` namespace to the `system` project, which by default is excluded from the network isolation. \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md new file mode 100644 index 00000000000..10907f4718c --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md @@ -0,0 +1,120 @@ +--- +title: Selectors and Scrape Configs +--- + +The Monitoring app sets `prometheus.prometheusSpec.ignoreNamespaceSelectors=false`, which enables monitoring across all namespaces by default. + +This ensures you can view traffic, metrics and graphs for resources deployed in a namespace with `istio-injection=enabled` label. + +If you would like to limit Prometheus to specific namespaces, set `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`. Once you do this, you will need to add additional configuration to continue to monitor your resources. + + +### Limiting Monitoring to Specific Namespaces by Setting ignoreNamespaceSelectors to True + +To limit monitoring to specific namespaces, you will edit the `ignoreNamespaceSelectors` Helm chart option. You will configure this option when installing or upgrading the Monitoring Helm chart: + +1. When installing or upgrading the Monitoring Helm chart, edit the values.yml and set`prometheus.prometheusSpec.ignoreNamespaceSelectors=true`. +1. Complete the install or upgrade. + +**Result:** Prometheus will be limited to specific namespaces which means one of the following configurations will need to be set up to continue to view data in various dashboards + +### Enabling Prometheus to Detect Resources in Other Namespaces + +There are two different ways to enable Prometheus to detect resources in other namespaces when `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`: + +- **Monitoring specific namespaces:** Add a Service Monitor or Pod Monitor in the namespace with the targets you want to scrape. +- **Monitoring across namespaces:** Add an `additionalScrapeConfig` to your rancher-monitoring instance to scrape all targets in all namespaces. + +### Monitoring Specific Namespaces: Create a Service Monitor or Pod Monitor + +This option allows you to define which specific services or pods you would like monitored in a specific namespace. + +The usability tradeoff is that you have to create the service monitor or pod monitor per namespace since you cannot monitor across namespaces. + +:::note Prerequisite: + +Define a ServiceMonitor or PodMonitor for ``. An example ServiceMonitor is provided below. + +::: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the top navigation bar, open the kubectl shell. +1. If the ServiceMonitor or PodMonitor file is stored locally in your cluster, in `kubectl create -f .yaml`. +1. If the ServiceMonitor or PodMonitor is not stored locally, run `cat<< EOF | kubectl apply -f -`, paste the file contents into the terminal, then run `EOF` to complete the command. +1. Run `kubectl label namespace istio-injection=enabled` to enable the envoy sidecar injection. + +**Result:** `` can be scraped by prometheus. + +
Example Service Monitor for Istio Proxies
+ +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: envoy-stats-monitor + namespace: istio-system + labels: + monitoring: istio-proxies +spec: + selector: + matchExpressions: + - {key: istio-prometheus-ignore, operator: DoesNotExist} + namespaceSelector: + any: true + jobLabel: envoy-stats + endpoints: + - path: /stats/prometheus + targetPort: 15090 + interval: 15s + relabelings: + - sourceLabels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - action: labeldrop + regex: "__meta_kubernetes_pod_label_(.+)" + - sourceLabels: [__meta_kubernetes_namespace] + action: replace + targetLabel: namespace + - sourceLabels: [__meta_kubernetes_pod_name] + action: replace + targetLabel: pod_name +``` + +### Monitoring across namespaces: Set ignoreNamespaceSelectors to False + +This enables monitoring across namespaces by giving Prometheus additional scrape configurations. + +The usability tradeoff is that all of Prometheus' `additionalScrapeConfigs` are maintained in a single Secret. This could make upgrading difficult if monitoring is already deployed with additionalScrapeConfigs before installing Istio. + +1. When installing or upgrading the Monitoring Helm chart, edit the values.yml and set the `prometheus.prometheusSpec.additionalScrapeConfigs` array to the **Additional Scrape Config** provided below. +1. Complete the install or upgrade. + +**Result:** All namespaces with the `istio-injection=enabled` label will be scraped by prometheus. + +
Additional Scrape Config
+ +``` yaml +- job_name: 'istio/envoy-stats' + scrape_interval: 15s + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:15090 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name +``` diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/cpu-and-memory-allocations.md b/versioned_docs/version-latest/integrations-in-rancher/istio/cpu-and-memory-allocations.md new file mode 100644 index 00000000000..37472e328c7 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/cpu-and-memory-allocations.md @@ -0,0 +1,63 @@ +--- +title: CPU and Memory Allocations +--- + +This section describes the minimum recommended computing resources for the Istio components in a cluster. + +The CPU and memory allocations for each component are [configurable.](#configuring-resource-allocations) + +Before enabling Istio, we recommend that you confirm that your Rancher worker nodes have enough CPU and memory to run all of the components of Istio. + +:::tip + +In larger deployments, it is strongly advised that the infrastructure be placed on dedicated nodes in the cluster by adding a node selector for each Istio component. + +::: + +The table below shows a summary of the minimum recommended resource requests and limits for the CPU and memory of each core Istio component. + +In Kubernetes, the resource request indicates that the workload will not deployed on a node unless the node has at least the specified amount of memory and CPU available. If the workload surpasses the limit for CPU or memory, it can be terminated or evicted from the node. For more information on managing resource limits for containers, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) + +| Workload | CPU - Request | Memory - Request | CPU - Limit | Memory - Limit | +|----------------------|---------------|------------|-----------------|-------------------| +| ingress gateway | 100m | 128mi | 2000m | 1024mi | +| egress gateway | 100m | 128mi | 2000m | 1024mi | +| istiod | 500m | 2048mi | No limit | No limit | +| proxy | 10m | 10mi | 2000m | 1024mi | +| **Totals:** | **710m** | **2314Mi** | **6000m** | **3072Mi** | + +## Configuring Resource Allocations + +You can individually configure the resource allocation for each type of Istio component. This section includes the default resource allocations for each component. + +To make it easier to schedule the workloads to a node, a cluster-admin can reduce the CPU and memory resource requests for the component. However, the default CPU and memory allocations are the minimum that we recommend. + +You can find more information about Istio configuration in the [official Istio documentation](https://istio.io/). + +To configure the resources allocated to an Istio component, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Installed Apps**. +1. Go to the `istio-system` namespace. In one of the Istio workloads, such as `rancher-istio`, click **⋮ > Edit/Upgrade**. +1. Click **Upgrade** to edit the base components via changes to the values.yaml or add an [overlay file](../../pages-for-subheaders/configuration-options.md#overlay-file). For more information about editing the overlay file, see [this section.](cpu-and-memory-allocations.md#editing-the-overlay-file) +1. Change the CPU or memory allocations, the nodes where each component will be scheduled to, or the node tolerations. +1. Click **Upgrade**. to rollout changes + +**Result:** The resource allocations for the Istio components are updated. + +### Editing the Overlay File + +The overlay file can contain any of the values in the [Istio Operator spec.](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/#IstioOperatorSpec) The overlay file included with the Istio application is just one example of a potential configuration of the overlay file. + +As long as the file contains `kind: IstioOperator` and the YAML options are valid, the file can be used as an overlay. + +In the example overlay file provided with the Istio application, the following section allows you to change Kubernetes resources: + +``` +# k8s: +# resources: +# requests: +# cpu: 200m +``` diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/disable-istio.md b/versioned_docs/version-latest/integrations-in-rancher/istio/disable-istio.md new file mode 100644 index 00000000000..052122c4891 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/disable-istio.md @@ -0,0 +1,42 @@ +--- +title: Disabling Istio +--- + +This section describes how to uninstall Istio in a cluster or disable a namespace, or workload. + +## Uninstall Istio in a Cluster + +To uninstall Istio, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. In the left navigation bar, click **Apps > Installed Apps**. +1. In the `istio-system` namespace, go to `rancher-istio` and click **⋮ > Delete**. +1. After `rancher-istio` is deleted, you can then select all the remaining apps in the `istio-system` namespace and click **Delete**. + +**Result:** The `rancher-istio` app in the cluster gets removed. The Istio sidecar cannot be deployed on any workloads in the cluster. + +:::note + +You can no longer disable and re-enable your Istio installation. If you would like to save your settings for a future install, view and save individual YAMLs to refer back to / reuse for future installations. + +::: + +**Troubleshooting Uninstall:** If you didn't follow the uninstall steps, you may encounter a warning during uninstall: + +`Error: uninstallation completed with 1 error(s): unable to build kubernetes objects for delete: unable to recognize "": no matches for kind "MonitoringDashboard" in version "monitoring.kiali.io/v1alpha1"` + +This could mean a few things. You either selected all the apps in the `istio-system` namespace and deleted them at the same time, or you deleted `rancher-istio` chart dependencies prior to deleting the `rancher-istio` chart. Since the uninstall did not complete properly, you will have resources remaining in the `istio-system` namespace that you will need to manually clean up. Another option to avoid manual clean up is to install `rancher-istio` again, then uninstall it in the correct order. + +## Disable Istio in a Namespace + +1. Click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Go to the namespace where you want to enable Istio and click **⋮ > Enable Istio Auto Injection**. Alternately, click the namespace, and then on the namespace detail page, click **⋮ > Enable Istio Auto Injection**. + +**Result:** When workloads are deployed in this namespace, they will not have the Istio sidecar. + +## Remove the Istio Sidecar from a Workload + +Disable Istio in the namespace, then redeploy the workloads with in it. They will be deployed without the Istio sidecar. diff --git a/versioned_docs/version-latest/integrations-in-rancher/istio/rbac-for-istio.md b/versioned_docs/version-latest/integrations-in-rancher/istio/rbac-for-istio.md new file mode 100644 index 00000000000..e33bdb72540 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/istio/rbac-for-istio.md @@ -0,0 +1,43 @@ +--- +title: Role-based Access Control +--- + +This section describes the permissions required to access Istio features. + +The rancher istio chart installs three `ClusterRoles` + +## Cluster-Admin Access + +By default, only those with the `cluster-admin` `ClusterRole` can: + +- Install istio app in a cluster +- Configure resource allocations for Istio + + +## Admin and Edit access + +By default, only Admin and Edit roles can: + +- Enable and disable Istio sidecar auto-injection for namespaces +- Add the Istio sidecar to workloads +- View the traffic metrics and traffic graph for the cluster +- Configure Istio's resources (such as the gateway, destination rules, or virtual services) + +## Summary of Default Permissions for Kubernetes Default roles + +Istio creates three `ClusterRoles` and adds Istio CRD access to the following default K8s `ClusterRole`: + +ClusterRole create by chart | Default K8s ClusterRole | Rancher Role | + ------------------------------:| ---------------------------:|---------:| + `istio-admin` | admin| Project Owner | + `istio-edit`| edit | Project Member | + `istio-view` | view | Read-only | + +Rancher will continue to use cluster-owner, cluster-member, project-owner, project-member, etc as role names, but will utilize default roles to determine access. For each default K8s `ClusterRole` there are different Istio CRD permissions and K8s actions (Create ( C ), Get ( G ), List ( L ), Watch ( W ), Update ( U ), Patch ( P ), Delete( D ), All ( * )) that can be performed. + + +|CRDs | Admin | Edit | View +|----------------------------| ------| -----| ----- +|
  • `config.istio.io`
    • `adapters`
    • `attributemanifests`
    • `handlers`
    • `httpapispecbindings`
    • `httpapispecs`
    • `instances`
    • `quotaspecbindings`
    • `quotaspecs`
    • `rules`
    • `templates`
| GLW | GLW | GLW +|
  • `networking.istio.io`
    • `destinationrules`
    • `envoyfilters`
    • `gateways`
    • `serviceentries`
    • `sidecars`
    • `virtualservices`
    • `workloadentries`
| * | * | GLW +|
  • `security.istio.io`
    • `authorizationpolicies`
    • `peerauthentications`
    • `requestauthentications`
| * | * | GLW \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md b/versioned_docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md new file mode 100644 index 00000000000..e4020351ab1 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md @@ -0,0 +1,75 @@ +--- +title: Flows and ClusterFlows +--- + +See the [Logging operator documentation](https://kube-logging.github.io/docs/configuration/flow/) for the full details on how to configure `Flows` and `ClusterFlows`. + +See [Rancher Integration with Logging Services: Troubleshooting](../../../pages-for-subheaders/logging.md#The-Logging-Buffer-Overloads-Pods) for how to resolve memory problems with the logging buffer. + +## Flows + +A `Flow` defines which logs to collect and filter and which output to send the logs to. + +The `Flow` is a namespaced resource, which means logs will only be collected from the namespace that the `Flow` is deployed in. + +`Flows` can be configured by filling out forms in the Rancher UI. + +For more details about the `Flow` custom resource, see [FlowSpec.](https://kube-logging.github.io/docs/configuration/crds/v1beta1/flow_types/) + +### Matches + +Match statements are used to select which containers to pull logs from. + +You can specify match statements to select or exclude logs according to Kubernetes labels, container and host names. Match statements are evaluated in the order they are defined and processed only until the first matching select or exclude rule applies. + +Matches can be configured by filling out the `Flow` or `ClusterFlow` forms in the Rancher UI. + +For detailed examples on using the match statement, see the [official documentation on log routing.](https://kube-logging.github.io/docs/configuration/log-routing/) + +### Filters + +You can define one or more filters within a `Flow`. Filters can perform various actions on the logs, for example, add additional data, transform the logs, or parse values from the records. The filters in the `Flow` are applied in the order in the definition. + +For a list of filters supported by the Logging operator, see [the official documentation on Fluentd filters](https://kube-logging.github.io/docs/configuration/plugins/filters/). + +Filters need to be configured in YAML. + +### Outputs + +This `Output` will receive logs from the `Flow`. Because the `Flow` is a namespaced resource, the `Output` must reside in same namespace as the `Flow`. + +`Outputs` can be referenced when filling out the `Flow` or `ClusterFlow` forms in the Rancher UI. + +## ClusterFlows + +Matches, filters and `Outputs` are configured for `ClusterFlows` in the same way that they are configured for `Flows`. The key difference is that the `ClusterFlow` is scoped at the cluster level and can configure log collection across all namespaces. + +`ClusterFlows` can be configured by filling out forms in the Rancher UI. + +After `ClusterFlow` selects logs from all namespaces in the cluster, logs from the cluster will be collected and logged to the selected `ClusterOutput`. + +## YAML Example + +The following example `Flow` transforms the log messages from the default namespace and sends them to an S3 `Output`: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: flow-sample + namespace: default +spec: + filters: + - parser: + remove_key_name_field: true + parse: + type: nginx + - tag_normaliser: + format: ${namespace_name}.${pod_name}.${container_name} + localOutputRefs: + - s3-output + match: + - select: + labels: + app: nginx +``` diff --git a/versioned_docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md b/versioned_docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md new file mode 100644 index 00000000000..aa558385bf0 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md @@ -0,0 +1,295 @@ +--- +title: Outputs and ClusterOutputs +--- + +See the [Logging operator documentation](https://kube-logging.github.io/docs/configuration/flow/) for the full details on how to configure `Flows` and `ClusterFlows`. + +See [Rancher Integration with Logging Services: Troubleshooting](../../../pages-for-subheaders/logging.md#The-Logging-Buffer-Overloads-Pods) for how to resolve memory problems with the logging buffer. + +## Outputs + +The `Output` resource defines where your `Flows` can send the log messages. `Outputs` are the final stage for a logging `Flow`. + +The `Output` is a namespaced resource, which means only a `Flow` within the same namespace can access it. + +You can use secrets in these definitions, but they must also be in the same namespace. + +`Outputs` can be configured by filling out forms in the Rancher UI. + +For the details of `Output` custom resource, see [OutputSpec.](https://kube-logging.github.io/docs/configuration/crds/v1beta1/output_types/). + +The Rancher UI provides forms for configuring the following `Output` types: + +- Amazon ElasticSearch +- Azure Storage +- Cloudwatch +- Datadog +- Elasticsearch +- File +- Fluentd +- GCS +- Kafka +- Kinesis Stream +- LogDNA +- LogZ +- Loki +- New Relic +- Splunk +- SumoLogic +- Syslog + +The Rancher UI provides forms for configuring the `Output` type, target, and access credentials if applicable. + +For example configuration for each logging plugin supported by the logging operator, see the [Logging operator documentation](https://kube-logging.github.io/docs/configuration/plugins/outputs/). + +## ClusterOutputs + +`ClusterOutput` defines an `Output` without namespace restrictions. It is only effective when deployed in the same namespace as the logging operator. + +`ClusterOutputs` can be configured by filling out forms in the Rancher UI. + +For the details of the `ClusterOutput` custom resource, see [ClusterOutput.](https://kube-logging.github.io/docs/configuration/crds/v1beta1/clusteroutput_types/) + +## YAML Examples + +Once logging is installed, you can use these examples to help craft your own logging pipeline. + +- [Cluster Output to ElasticSearch](#cluster-output-to-elasticsearch) +- [Output to Splunk](#output-to-splunk) +- [Output to Syslog](#output-to-syslog) +- [Unsupported Outputs](#unsupported-outputs) + +### Cluster Output to ElasticSearch + +Let's say you wanted to send all logs in your cluster to an `elasticsearch` cluster. First, we create a cluster `Output`. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-es" + namespace: "cattle-logging-system" +spec: + elasticsearch: + host: elasticsearch.example.com + port: 9200 + scheme: http +``` + +We have created this `ClusterOutput`, without elasticsearch configuration, in the same namespace as our operator: `cattle-logging-system.`. Any time we create a `ClusterFlow` or `ClusterOutput`, we have to put it in the `cattle-logging-system` namespace. + +Now that we have configured where we want the logs to go, let's configure all logs to go to that `ClusterOutput`. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: "cattle-logging-system" +spec: + globalOutputRefs: + - "example-es" +``` + +We should now see our configured index with logs in it. + + +### Output to Splunk + +What if we have an application team who only wants logs from a specific namespaces sent to a `splunk` server? For this case, we can use namespaced `Outputs` and `Flows`. + +Before we start, let's set up that team's application: `coolapp`. + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: devteam +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coolapp + namespace: devteam + labels: + app: coolapp +spec: + replicas: 2 + selector: + matchLabels: + app: coolapp + template: + metadata: + labels: + app: coolapp + spec: + containers: + - name: generator + image: paynejacob/loggenerator:latest +``` + +With `coolapp` running, we will follow a similar path as when we created a `ClusterOutput`. However, unlike `ClusterOutputs`, we create our `Output` in our application's namespace. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Output +metadata: + name: "devteam-splunk" + namespace: "devteam" +spec: + splunkHec: + hec_host: splunk.example.com + hec_port: 8088 + protocol: http +``` + +Once again, let's feed our `Output` some logs: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: "devteam-logs" + namespace: "devteam" +spec: + localOutputRefs: + - "devteam-splunk" +``` + + +### Output to Syslog + +Let's say you wanted to send all logs in your cluster to an `syslog` server. First, we create a `ClusterOutput`: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: "example-syslog" + namespace: "cattle-logging-system" +spec: + syslog: + buffer: + timekey: 30s + timekey_use_utc: true + timekey_wait: 10s + flush_interval: 5s + format: + type: json + app_name_field: test + host: syslog.example.com + insecure: true + port: 514 + transport: tcp +``` + +Now that we have configured where we want the logs to go, let's configure all logs to go to that `Output`. + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: "all-logs" + namespace: cattle-logging-system +spec: + globalOutputRefs: + - "example-syslog" +``` + +### Unsupported Outputs + +For the final example, we create an `Output` to write logs to a destination that is not supported out of the box: + +:::note Note on syslog: + +`syslog` is a supported `Output`. However, this example still provides an overview on using unsupported plugins. + +::: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: syslog-config + namespace: cattle-logging-system +type: Opaque +stringData: + fluent-bit.conf: | + [INPUT] + Name forward + Port 24224 + + [OUTPUT] + Name syslog + InstanceName syslog-output + Match * + Addr syslog.example.com + Port 514 + Cluster ranchers + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentbit-syslog-forwarder + namespace: cattle-logging-system + labels: + output: syslog +spec: + selector: + matchLabels: + output: syslog + template: + metadata: + labels: + output: syslog + spec: + containers: + - name: fluentbit + image: paynejacob/fluent-bit-out-syslog:latest + ports: + - containerPort: 24224 + volumeMounts: + - mountPath: "/fluent-bit/etc/" + name: configuration + volumes: + - name: configuration + secret: + secretName: syslog-config +--- +apiVersion: v1 +kind: Service +metadata: + name: syslog-forwarder + namespace: cattle-logging-system +spec: + selector: + output: syslog + ports: + - protocol: TCP + port: 24224 + targetPort: 24224 +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterFlow +metadata: + name: all-logs + namespace: cattle-logging-system +spec: + globalOutputRefs: + - syslog +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: syslog + namespace: cattle-logging-system +spec: + forward: + servers: + - host: "syslog-forwarder.cattle-logging-system" + require_ack_response: false + ignore_network_errors_at_startup: false +``` + +Let's break down what is happening here. First, we create a deployment of a container that has the additional `syslog` plugin and accepts logs forwarded from another `fluentd`. Next we create an `Output` configured as a forwarder to our deployment. The deployment `fluentd` will then forward all logs to the configured `syslog` destination. diff --git a/versioned_docs/version-latest/integrations-in-rancher/logging/logging-architecture.md b/versioned_docs/version-latest/integrations-in-rancher/logging/logging-architecture.md new file mode 100644 index 00000000000..560235f4f7c --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/logging/logging-architecture.md @@ -0,0 +1,28 @@ +--- +title: Architecture +--- + +This section summarizes the architecture of the Rancher logging application. + +For more details about how the Logging operator works, see the [official documentation.](https://kube-logging.github.io/docs/#architecture) + +### How the Logging Operator Works + +The Logging operator automates the deployment and configuration of a Kubernetes logging pipeline. It deploys and configures a Fluent Bit DaemonSet on every node to collect container and application logs from the node file system. + +Fluent Bit queries the Kubernetes API and enriches the logs with metadata about the pods, and transfers both the logs and the metadata to Fluentd. Fluentd receives, filters, and transfers logs to multiple `Outputs`. + +The following custom resources are used to define how logs are filtered and sent to their `Outputs`: + +- A `Flow` is a namespaced custom resource that uses filters and selectors to route log messages to the appropriate `Outputs`. +- A `ClusterFlow` is used to route cluster-level log messages. +- An `Output` is a namespaced resource that defines where the log messages are sent. +- A `ClusterOutput` defines an `Output` that is available from all `Flows` and `ClusterFlows`. + +Each `Flow` must reference an `Output`, and each `ClusterFlow` must reference a `ClusterOutput`. + +The following figure from the [Logging Operator documentation](https://kube-logging.github.io/docs/#architecture) shows the new logging architecture: + +
How the Logging Operator Works with Fluentd and Fluent Bit
+ +![How the Logging Operator Works with Fluentd](/img/banzai-cloud-logging-operator.png) diff --git a/versioned_docs/version-latest/integrations-in-rancher/logging/logging-helm-chart-options.md b/versioned_docs/version-latest/integrations-in-rancher/logging/logging-helm-chart-options.md new file mode 100644 index 00000000000..fd6299abc4d --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/logging/logging-helm-chart-options.md @@ -0,0 +1,94 @@ +--- +title: rancher-logging Helm Chart Options +--- + +### Enable/Disable Windows Node Logging + +You can enable or disable Windows node logging by setting `global.cattle.windows.enabled` to either `true` or `false` in the `values.yaml`. + +By default, Windows node logging will be enabled if the Cluster Dashboard UI is used to install the logging application on a Windows cluster. + +In this scenario, setting `global.cattle.windows.enabled` to `false` will disable Windows node logging on the cluster. +When disabled, logs will still be collected from Linux nodes within the Windows cluster. + +:::note + +Currently an [issue](https://github.com/rancher/rancher/issues/32325) exists where Windows nodeAgents are not deleted when performing a `helm upgrade` after disabling Windows logging in a Windows cluster. In this scenario, users may need to manually remove the Windows nodeAgents if they are already installed. + +::: + +### Working with a Custom Docker Root Directory + +If using a custom Docker root directory, you can set `global.dockerRootDirectory` in `values.yaml`. + +This will ensure that the Logging CRs created will use your specified path rather than the default Docker `data-root` location. + +Note that this only affects Linux nodes. + +If there are any Windows nodes in the cluster, the change will not be applicable to those nodes. + +### Adding NodeSelector Settings and Tolerations for Custom Taints + +You can add your own `nodeSelector` settings and add `tolerations` for additional taints by editing the logging Helm chart values. For details, see [this page.](taints-and-tolerations.md) + +### Enabling the Logging Application to Work with SELinux + +:::note Requirements: + +Logging v2 was tested with SELinux on RHEL/CentOS 7 and 8. + +::: + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. + +To use Logging v2 with SELinux, we recommend installing the `rancher-selinux` RPM according to these [instructions](../../pages-for-subheaders/selinux-rpm.md). + +Then, when installing the logging application, configure the chart to be SELinux aware by changing `global.seLinux.enabled` to `true` in the `values.yaml`. + +### Additional Logging Sources + +By default, Rancher collects logs for [control plane components](https://kubernetes.io/docs/concepts/overview/components/#control-plane-components) and [node components](https://kubernetes.io/docs/concepts/overview/components/#node-components) for all cluster types. + +In some cases, Rancher may be able to collect additional logs. + +The following table summarizes the sources where additional logs may be collected for each node types: + +| Logging Source | Linux Nodes (including in Windows cluster) | Windows Nodes | +| --- | --- | ---| +| RKE | ✓ | ✓ | +| RKE2 | ✓ | | +| K3s | ✓ | | +| AKS | ✓ | | +| EKS | ✓ | | +| GKE | ✓ | | + +To enable hosted Kubernetes providers as additional logging sources, enable **Enable enhanced cloud provider logging** option when installing or upgrading the Logging Helm chart. + +When enabled, Rancher collects all additional node and control plane logs the provider has made available, which may vary between providers + +If you're already using a cloud provider's own logging solution such as AWS CloudWatch or Google Cloud operations suite (formerly Stackdriver), it is not necessary to enable this option as the native solution will have unrestricted access to all logs. + +### Systemd Configuration + +In Rancher logging, `SystemdLogPath` must be configured for K3s and RKE2 Kubernetes distributions. + +K3s and RKE2 Kubernetes distributions log to journald, which is the subsystem of systemd that is used for logging. In order to collect these logs, the `systemdLogPath` needs to be defined. While the `run/log/journal` directory is used by default, some Linux distributions do not default to this path. For example, Ubuntu defaults to `var/log/journal`. To determine your `systemdLogPath` configuration, see steps below. + +**Steps for Systemd Configuration:** + +* Run `cat /etc/systemd/journald.conf | grep -E ^\#?Storage | cut -d"=" -f2` on one of your nodes. +* If `persistent` is returned, your `systemdLogPath` should be `/var/log/journal`. +* If `volatile` is returned, your `systemdLogPath` should be `/run/log/journal`. +* If `auto` is returned, check if `/var/log/journal` exists. + * If `/var/log/journal` exists, then use `/var/log/journal`. + * If `/var/log/journal` does not exist, then use `/run/log/journal`. + +:::note Notes: + +If any value not described above is returned, Rancher Logging will not be able to collect control plane logs. To address this issue, you will need to perform the following actions on every control plane node: + +* Set `Storage=volatile` in journald.conf. +* Reboot your machine. +* Set `systemdLogPath` to `/run/log/journal`. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/logging/rbac-for-logging.md b/versioned_docs/version-latest/integrations-in-rancher/logging/rbac-for-logging.md new file mode 100644 index 00000000000..627e3533c2c --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/logging/rbac-for-logging.md @@ -0,0 +1,23 @@ +--- +title: Role-based Access Control for Logging +--- + +Rancher logging has two roles, `logging-admin` and `logging-view`. + +- `logging-admin` gives users full access to namespaced `Flows` and `Outputs` +- `logging-view` allows users to *view* namespaced `Flows` and `Outputs`, and `ClusterFlows` and `ClusterOutputs` + +:::note Why choose one role over the other? + +Edit access to `ClusterFlow` and `ClusterOutput` resources is powerful. Any user with it has edit access for all logs in the cluster. + +::: + +In Rancher, the cluster administrator role is the only role with full access to all `rancher-logging` resources. Cluster members are not able to edit or read any logging resources. Project owners and members have the following privileges: + +Project Owners | Project Members +--- | --- +able to create namespaced `Flows` and `Outputs` in their projects' namespaces | only able to view the `Flows` and `Outputs` in projects' namespaces +can collect logs from anything in their projects' namespaces | cannot collect any logs in their projects' namespaces + +Both project owners and project members require at least *one* namespace in their project to use logging. If they do not, then they may not see the logging button in the top nav dropdown. \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/logging/taints-and-tolerations.md b/versioned_docs/version-latest/integrations-in-rancher/logging/taints-and-tolerations.md new file mode 100644 index 00000000000..c5cf0e35578 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/logging/taints-and-tolerations.md @@ -0,0 +1,65 @@ +--- +title: Working with Taints and Tolerations +--- + +"Tainting" a Kubernetes node causes pods to repel running on that node. + +Unless the pods have a `toleration` for that node's taint, they will run on other nodes in the cluster. + +[Taints and tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) can work in conjunction with the `nodeSelector` [field](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) within the `PodSpec`, which enables the *opposite* effect of a taint. + +Using `nodeSelector` gives pods an affinity towards certain nodes. + +Both provide choice for the what node(s) the pod will run on. + +- [Default Implementation in Rancher's Logging Stack](#default-implementation-in-ranchers-logging-stack) +- [Adding NodeSelector Settings and Tolerations for Custom Taints](#adding-nodeselector-settings-and-tolerations-for-custom-taints) + + +### Default Implementation in Rancher's Logging Stack + +By default, Rancher taints all Linux nodes with `cattle.io/os=linux`, and does not taint Windows nodes. +The logging stack pods have `tolerations` for this taint, which enables them to run on Linux nodes. +Moreover, most logging stack pods run on Linux only and have a `nodeSelector` added to ensure they run on Linux nodes. + +This example Pod YAML file shows a nodeSelector being used with a toleration: + +```yaml +apiVersion: v1 +kind: Pod +# metadata... +spec: + # containers... + tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + nodeSelector: + kubernetes.io/os: linux +``` + +In the above example, we ensure that our pod only runs on Linux nodes, and we add a `toleration` for the taint we have on all of our Linux nodes. + +You can do the same with Rancher's existing taints, or with your own custom ones. + +### Adding NodeSelector Settings and Tolerations for Custom Taints + +If you would like to add your own `nodeSelector` settings, or if you would like to add `tolerations` for additional taints, you can pass the following to the chart's values. + +```yaml +tolerations: + # insert tolerations... +nodeSelector: + # insert nodeSelector... +``` + +These values will add both settings to the `fluentd`, `fluentbit`, and `logging-operator` containers. +Essentially, these are global settings for all pods in the logging stack. + +However, if you would like to add tolerations for *only* the `fluentbit` container, you can add the following to the chart's values. + +```yaml +fluentbit_tolerations: + # insert tolerations list for fluentbit containers only... +``` diff --git a/versioned_docs/version-latest/integrations-in-rancher/longhorn.md b/versioned_docs/version-latest/integrations-in-rancher/longhorn.md new file mode 100644 index 00000000000..a8870b4e3a8 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/longhorn.md @@ -0,0 +1,70 @@ +--- +title: Longhorn - Cloud native distributed block storage for Kubernetes +--- + +[Longhorn](https://longhorn.io/) is a lightweight, reliable, and easy-to-use distributed block storage system for Kubernetes. + +Longhorn is free, open source software. Originally developed by Rancher Labs, it is now being developed as a sandbox project of the Cloud Native Computing Foundation. It can be installed on any Kubernetes cluster with Helm, with kubectl, or with the Rancher UI. You can learn more about its architecture [here.](https://longhorn.io/docs/latest/concepts/) + +With Longhorn, you can: + +- Use Longhorn volumes as persistent storage for the distributed stateful applications in your Kubernetes cluster +- Partition your block storage into Longhorn volumes so that you can use Kubernetes volumes with or without a cloud provider +- Replicate block storage across multiple nodes and data centers to increase availability +- Store backup data in external storage such as NFS or AWS S3 +- Create cross-cluster disaster recovery volumes so that data from a primary Kubernetes cluster can be quickly recovered from backup in a second Kubernetes cluster +- Schedule recurring snapshots of a volume, and schedule recurring backups to NFS or S3-compatible secondary storage +- Restore volumes from backup +- Upgrade Longhorn without disrupting persistent volumes + +
Longhorn Dashboard
+ +![Longhorn Dashboard](/img/longhorn-screenshot.png) + +### Installing Longhorn with Rancher + +1. Fulfill all [Installation Requirements.](https://longhorn.io/docs/latest/deploy/install/#installation-requirements) +1. Go to the cluster where you want to install Longhorn. +1. Click **Apps**. +1. Click **Charts**. +1. Click **Longhorn**. +1. Optional: To customize the initial settings, click **Longhorn Default Settings** and edit the configuration. For help customizing the settings, refer to the [Longhorn documentation.](https://longhorn.io/docs/latest/references/settings/) +1. Click **Install**. + +**Result:** Longhorn is deployed in the Kubernetes cluster. + +### Accessing Longhorn from the Rancher UI + +1. Go to the cluster where Longhorn is installed. In the left navigation menu, click **Longhorn**. +1. On this page, you can edit Kubernetes resources managed by Longhorn. To view the Longhorn UI, click the **Longhorn** button in the **Overview** section. + +**Result:** You will be taken to the Longhorn UI, where you can manage your Longhorn volumes and their replicas in the Kubernetes cluster, as well as secondary backups of your Longhorn storage that may exist in another Kubernetes cluster or in S3. + +### Uninstalling Longhorn from the Rancher UI + +1. Go to the cluster where Longhorn is installed and click **Apps**. +1. Click **Installed Apps**. +1. Go to the `longhorn-system` namespace and check the boxes next to the `longhorn` and `longhorn-crd` apps. +1. Click **Delete,** and confirm **Delete**. + +**Result:** Longhorn is uninstalled. + +### GitHub Repository + +The Longhorn project is available [here.](https://github.com/longhorn/longhorn) + +### Documentation + +The Longhorn documentation is [here.](https://longhorn.io/docs/) + +### Architecture + +Longhorn creates a dedicated storage controller for each volume and synchronously replicates the volume across multiple replicas stored on multiple nodes. + +The storage controller and replicas are themselves orchestrated using Kubernetes. + +You can learn more about its architecture [here.](https://longhorn.io/docs/latest/concepts/) + +
Longhorn Architecture
+ +![Longhorn Architecture](/img/longhorn-architecture.svg) diff --git a/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md new file mode 100644 index 00000000000..6186565900b --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md @@ -0,0 +1,117 @@ +--- +title: Built-in Dashboards +--- + +## Grafana UI + +[Grafana](https://grafana.com/grafana/) allows you to query, visualize, alert on and understand your metrics no matter where they are stored. Create, explore, and share dashboards with your team and foster a data driven culture. + +To see the default dashboards for time series data visualization, go to the Grafana UI. + +### Customizing Grafana + +To view and customize the PromQL queries powering the Grafana dashboard, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) + +### Persistent Grafana Dashboards + +To create a persistent Grafana dashboard, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) + +### Access to Grafana + +For information about role-based access control for Grafana, see [this section.](rbac-for-monitoring.md#role-based-access-control-for-grafana) + + +## Alertmanager UI + +When `rancher-monitoring` is installed, the Prometheus Alertmanager UI is deployed, allowing you to view your alerts and the current Alertmanager configuration. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [How Alertmanager Works.](how-monitoring-works.md#3-how-alertmanager-works) + +::: + +### Accessing the Alertmanager UI + +The Alertmanager UI lets you see the most recently fired alerts. + +:::note Prerequisite: + +The `rancher-monitoring` application must be installed. + +::: + +To see the Alertmanager UI, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the Alertmanager UI, click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Alertmanager**. + +**Result:** The Alertmanager UI opens in a new tab. For help with configuration, refer to the [official Alertmanager documentation.](https://prometheus.io/docs/alerting/latest/alertmanager/) + +For more information on configuring Alertmanager in Rancher, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +
The Alertmanager UI
+ +![Alertmanager UI](/img/alertmanager-ui.png) + + +### Viewing Default Alerts + +To see alerts that are fired by default, go to the Alertmanager UI and click **Expand all groups**. + + +## Prometheus UI + +By default, the [kube-state-metrics service](https://github.com/kubernetes/kube-state-metrics) provides a wealth of information about CPU and memory utilization to the monitoring application. These metrics cover Kubernetes resources across namespaces. This means that in order to see resource metrics for a service, you don't need to create a new ServiceMonitor for it. Because the data is already in the time series database, you can go to the Prometheus UI and run a PromQL query to get the information. The same query can be used to configure a Grafana dashboard to show a graph of those metrics over time. + +To see the Prometheus UI, install `rancher-monitoring`. Then: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the Prometheus UI and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Prometheus Graph**. + +
Prometheus Graph UI
+ +![Prometheus Graph UI](/img/prometheus-graph-ui.png) + +### Viewing the Prometheus Targets + +To see what services you are monitoring, you will need to see your targets. Targets are set up by ServiceMonitors and PodMonitors as sources to scrape metrics from. You won't need to directly edit targets, but the Prometheus UI can be useful for giving you an overview of all of the sources of metrics that are being scraped. + +To see the Prometheus Targets, install `rancher-monitoring`. Then: + + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the Prometheus targets and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Prometheus Targets**. + +
Targets in the Prometheus UI
+ +![Prometheus Targets UI](/img/prometheus-targets-ui.png) + +### Viewing the PrometheusRules + +When you define a Rule (which is declared within a RuleGroup in a PrometheusRule resource), the [spec of the Rule itself](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#rule) contains labels that are used by Alertmanager to figure out which Route should receive a certain Alert. + +To see the PrometheusRules, install `rancher-monitoring`. Then: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Prometheus Rules**. + +You can also see the rules in the Prometheus UI: + +
Rules in the Prometheus UI
+ +![PrometheusRules UI](/img/prometheus-rules-ui.png) + +For more information on configuring PrometheusRules in Rancher, see [this page.](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) + +## Legacy UI + +For information on the dashboards available in v2.2 to v2.4 of Rancher, before the introduction of the `rancher-monitoring` application, see the [Rancher v2.0—v2.4 docs](../../../version-2.0-2.4/explanations/integrations-in-rancher/cluster-monitoring/viewing-metrics.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md new file mode 100644 index 00000000000..de110cd3d80 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md @@ -0,0 +1,249 @@ +--- +title: How Monitoring Works +--- + +## 1. Architecture Overview + +_**The following sections describe how data flows through the Monitoring V2 application:**_ + +### Prometheus Operator + +Prometheus Operator observes ServiceMonitors, PodMonitors, and PrometheusRules being created. When the Prometheus configuration resources are created, Prometheus Operator calls the Prometheus API to sync the new configuration. As the diagram at the end of this section shows, the Prometheus Operator acts as the intermediary between Prometheus and Kubernetes, calling the Prometheus API to synchronize Prometheus with the monitoring-related resources in Kubernetes. + +### ServiceMonitors and PodMonitors + +ServiceMonitors and PodMonitors declaratively specify targets, such as Services and Pods, that need to be monitored. + +- Targets are scraped on a recurring schedule based on the configured Prometheus scrape interval, and the metrics that are scraped are stored into the Prometheus Time Series Database (TSDB). + +- In order to perform the scrape, ServiceMonitors and PodMonitors are defined with label selectors that determine which Services or Pods should be scraped and endpoints that determine how the scrape should happen on the given target, e.g., scrape/metrics in TCP 10252, proxying through IP addr x.x.x.x. + +- Out of the box, Monitoring V2 comes with certain pre-configured exporters that are deployed based on the type of Kubernetes cluster that it is deployed on. For more information, see [Scraping and Exposing Metrics](#5-scraping-and-exposing-metrics). + +### How PushProx Works + +- Certain internal Kubernetes components are scraped via a proxy deployed as part of Monitoring V2 called **PushProx**. The Kubernetes components that expose metrics to Prometheus through PushProx are the following: +`kube-controller-manager`, `kube-scheduler`, `etcd`, and `kube-proxy`. + +- For each PushProx exporter, we deploy one PushProx client onto all target nodes. For example, a PushProx client is deployed onto all controlplane nodes for kube-controller-manager, all etcd nodes for kube-etcd, and all nodes for kubelet. + +- We deploy exactly one PushProx proxy per exporter. The process for exporting metrics is as follows: + +1. The PushProx Client establishes an outbound connection with the PushProx Proxy. +1. The client then polls the proxy for scrape requests that have come into the proxy. +1. When the proxy receives a scrape request from Prometheus, the client sees it as a result of the poll. +1. The client scrapes the internal component. +1. The internal component responds by pushing metrics back to the proxy. + + +

Process for Exporting Metrics with PushProx:
+ +![Process for Exporting Metrics with PushProx](/img/pushprox-process.svg) + +### PrometheusRules + +PrometheusRules allow users to define rules for what metrics or time series database queries should result in alerts being fired. Rules are evaluated on an interval. + +- **Recording rules** create a new time series based on existing series that have been collected. They are frequently used to precompute complex queries. +- **Alerting rules** run a particular query and fire an alert from Prometheus if the query evaluates to a non-zero value. + +### Alert Routing + +Once Prometheus determines that an alert needs to be fired, alerts are forwarded to **Alertmanager**. + +- Alerts contain labels that come from the PromQL query itself and additional labels and annotations that can be provided as part of specifying the initial PrometheusRule. + +- Before receiving any alerts, Alertmanager will use the **routes** and **receivers** specified in its configuration to form a routing tree on which all incoming alerts are evaluated. Each node of the routing tree can specify additional grouping, labeling, and filtering that needs to happen based on the labels attached to the Prometheus alert. A node on the routing tree (usually a leaf node) can also specify that an alert that reaches it needs to be sent out to a configured Receiver, e.g., Slack, PagerDuty, SMS, etc. Note that Alertmanager will send an alert first to **alertingDriver**, then alertingDriver will send or forward alert to the proper destination. + +- Routes and receivers are also stored in the Kubernetes API via the Alertmanager Secret. When the Secret is updated, Alertmanager is also updated automatically. Note that routing occurs via labels only (not via annotations, etc.). + +
How data flows through the monitoring application:
+ + +## 2. How Prometheus Works + +### Storing Time Series Data + +After collecting metrics from exporters, Prometheus stores the time series in a local on-disk time series database. Prometheus optionally integrates with remote systems, but `rancher-monitoring` uses local storage for the time series database. + +Once stored, users can query this TSDB using PromQL, the query language for Prometheus. + +PromQL queries can be visualized in one of two ways: + +1. By supplying the query in Prometheus's Graph UI, which will show a simple graphical view of the data. +1. By creating a Grafana Dashboard that contains the PromQL query and additional formatting directives that label axes, add units, change colors, use alternative visualizations, etc. + +### Defining Rules for Prometheus + +Rules define queries that Prometheus needs to execute on a regular `evaluationInterval` to perform certain actions, such as firing an alert (alerting rules) or precomputing a query based on others existing in its TSDB (recording rules). These rules are encoded in PrometheusRules custom resources. When PrometheusRule custom resources are created or updated, the Prometheus Operator observes the change and calls the Prometheus API to synchronize the set of rules that Prometheus is currently evaluating on a regular interval. + +A PrometheusRule allows you to define one or more RuleGroups. Each RuleGroup consists of a set of Rule objects that can each represent either an alerting or a recording rule with the following fields: + +- The name of the new alert or record +- A PromQL expression for the new alert or record +- Labels that should be attached to the alert or record that identify it (e.g. cluster name or severity) +- Annotations that encode any additional important pieces of information that need to be displayed on the notification for an alert (e.g. summary, description, message, runbook URL, etc.). This field is not required for recording rules. + +On evaluating a [rule](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#rule), Prometheus will execute the provided PromQL query, add additional provided labels (or annotations - only for alerting rules), and execute the appropriate action for the rule. For example, an Alerting Rule that adds `team: front-end` as a label to the provided PromQL query will append that label to the fired alert, which will allow Alertmanager to forward the alert to the correct Receiver. + +### Alerting and Recording Rules + +Prometheus doesn't maintain the state of whether alerts are active. It fires alerts repetitively at every evaluation interval, relying on Alertmanager to group and filter the alerts into meaningful notifications. + +The `evaluation_interval` constant defines how often Prometheus evaluates its alerting rules against the time series database. Similar to the `scrape_interval`, the `evaluation_interval` also defaults to one minute. + +The rules are contained in a set of rule files. Rule files include both alerting rules and recording rules, but only alerting rules result in alerts being fired after their evaluation. + +For recording rules, Prometheus runs a query, then stores it as a time series. This synthetic time series is useful for storing the results of an expensive or time-consuming query so that it can be queried more quickly in the future. + +Alerting rules are more commonly used. Whenever an alerting rule evaluates to a positive number, Prometheus fires an alert. + +The Rule file adds labels and annotations to alerts before firing them, depending on the use case: + +- Labels indicate information that identifies the alert and could affect the routing of the alert. For example, if when sending an alert about a certain container, the container ID could be used as a label. + +- Annotations denote information that doesn't affect where an alert is routed, for example, a runbook or an error message. + +## 3. How Alertmanager Works + +The Alertmanager handles alerts sent by client applications such as the Prometheus server. It takes care of the following tasks: + +- Deduplicating, grouping, and routing alerts to the correct receiver integration such as email, PagerDuty, or OpsGenie + +- Silencing and inhibition of alerts + +- Tracking alerts that fire over time + +- Sending out the status of whether an alert is currently firing, or if it is resolved + +### Alerts Forwarded by alertingDrivers + +When alertingDrivers are installed, this creates a `Service` that can be used as the receiver's URL for Teams or SMS, based on the alertingDriver's configuration. The URL in the Receiver points to the alertingDrivers; so the Alertmanager sends alert first to alertingDriver, then alertingDriver forwards or sends alert to the proper destination. + +### Routing Alerts to Receivers + +Alertmanager coordinates where alerts are sent. It allows you to group alerts based on labels and fire them based on whether certain labels are matched. One top-level route accepts all alerts. From there, Alertmanager continues routing alerts to receivers based on whether they match the conditions of the next route. + +While the Rancher UI forms only allow editing a routing tree that is two levels deep, you can configure more deeply nested routing structures by editing the Alertmanager Secret. + +### Configuring Multiple Receivers + +By editing the forms in the Rancher UI, you can set up a Receiver resource with all the information Alertmanager needs to send alerts to your notification system. + +By editing custom YAML in the Alertmanager or Receiver configuration, you can also send alerts to multiple notification systems. For more information, see the section on configuring [Receivers.](../../reference-guides/monitoring-v2-configuration/receivers.md#configuring-multiple-receivers) + +## 4. Monitoring V2 Specific Components + +Prometheus Operator introduces a set of [Custom Resource Definitions](https://github.com/prometheus-operator/prometheus-operator#customresourcedefinitions) that allow users to deploy and manage Prometheus and Alertmanager instances by creating and modifying those custom resources on a cluster. + +Prometheus Operator will automatically update your Prometheus configuration based on the live state of the resources and configuration options that are edited in the Rancher UI. + +### Resources Deployed by Default + +By default, a set of resources curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project are deployed onto your cluster as part of installing the Rancher Monitoring Application to set up a basic Monitoring/Alerting stack. + +The resources that get deployed onto your cluster to support this solution can be found in the [`rancher-monitoring`](https://github.com/rancher/charts/tree/main/charts/rancher-monitoring) Helm chart, which closely tracks the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community with certain changes tracked in the [CHANGELOG.md](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/CHANGELOG.md). + +### Default Exporters + +Monitoring V2 deploys three default exporters that provide additional metrics for Prometheus to store: + +1. `node-exporter`: exposes hardware and OS metrics for Linux hosts. For more information on `node-exporter`, refer to the [upstream documentation](https://prometheus.io/docs/guides/node-exporter/). + +1. `windows-exporter`: exposes hardware and OS metrics for Windows hosts (only deployed on Windows clusters). For more information on `windows-exporter`, refer to the [upstream documentation](https://github.com/prometheus-community/windows_exporter). + +1. `kube-state-metrics`: expose additional metrics that track the state of resources contained in the Kubernetes API (e.g., pods, workloads, etc.). For more information on `kube-state-metrics`, refer to the [upstream documentation](https://github.com/kubernetes/kube-state-metrics/tree/master/docs). + +ServiceMonitors and PodMonitors will scrape these exporters, as defined [here](#defining-what-metrics-are-scraped). Prometheus stores these metrics, and you can query the results via either Prometheus's UI or Grafana. + +See the [architecture](#1-architecture-overview) section for more information on recording rules, alerting rules, and Alertmanager. + +### Components Exposed in the Rancher UI + +When the monitoring application is installed, you will be able to edit the following components in the Rancher UI: + +| Component | Type of Component | Purpose and Common Use Cases for Editing | +|--------------|------------------------|---------------------------| +| ServiceMonitor | Custom resource | Sets up Kubernetes Services to scrape custom metrics from. Automatically updates the scrape configuration in the Prometheus custom resource. | +| PodMonitor | Custom resource | Sets up Kubernetes Pods to scrape custom metrics from. Automatically updates the scrape configuration in the Prometheus custom resource. | +| Receiver | Configuration block (part of Alertmanager) | Modifies information on where to send an alert (e.g., Slack, PagerDuty, etc.) and any necessary information to send the alert (e.g., TLS certs, proxy URLs, etc.). Automatically updates the Alertmanager custom resource. | +| Route | Configuration block (part of Alertmanager) | Modifies the routing tree that is used to filter, label, and group alerts based on labels and send them to the appropriate Receiver. Automatically updates the Alertmanager custom resource. | +| PrometheusRule | Custom resource | Defines additional queries that need to trigger alerts or define materialized views of existing series that are within Prometheus's TSDB. Automatically updates the Prometheus custom resource. | + +### PushProx + +PushProx allows Prometheus to scrape metrics across a network boundary, which prevents users from having to expose metrics ports for internal Kubernetes components on each node in a Kubernetes cluster. + +Since the metrics for Kubernetes components are generally exposed on the host network of nodes in the cluster, PushProx deploys a DaemonSet of clients that sit on the hostNetwork of each node and make an outbound connection to a single proxy that is sitting on the Kubernetes API. Prometheus can then be configured to proxy scrape requests through the proxy to each client, which allows it to scrape metrics from the internal Kubernetes components without requiring any inbound node ports to be open. + +Refer to [Scraping Metrics with PushProx](#scraping-metrics-with-pushprox) for more. + +## 5. Scraping and Exposing Metrics + +### Defining what Metrics are Scraped + +ServiceMonitors and PodMonitors define targets that are intended for Prometheus to scrape. The [Prometheus custom resource](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md#prometheus) tells Prometheus which ServiceMonitors or PodMonitors it should use to find out where to scrape metrics from. + +The Prometheus Operator observes the ServiceMonitors and PodMonitors. When it observes that they are created or updated, it calls the Prometheus API to update the scrape configuration in the Prometheus custom resource and keep it in sync with the scrape configuration in the ServiceMonitors or PodMonitors. This scrape configuration tells Prometheus which endpoints to scrape metrics from and how it will label the metrics from those endpoints. + +Prometheus scrapes all of the metrics defined in its scrape configuration at every `scrape_interval`, which is one minute by default. + +The scrape configuration can be viewed as part of the Prometheus custom resource that is exposed in the Rancher UI. + +### How the Prometheus Operator Sets up Metrics Scraping + +The Prometheus Deployment or StatefulSet scrapes metrics, and the configuration of Prometheus is controlled by the Prometheus custom resources. The Prometheus Operator watches for Prometheus and Alertmanager resources, and when they are created, the Prometheus Operator creates a Deployment or StatefulSet for Prometheus or Alertmanager with the user-defined configuration. + +When the Prometheus Operator observes ServiceMonitors, PodMonitors, and PrometheusRules being created, it knows that the scrape configuration needs to be updated in Prometheus. It updates Prometheus by first updating the configuration and rules files in the volumes of Prometheus's Deployment or StatefulSet. Then it calls the Prometheus API to sync the new configuration, resulting in the Prometheus Deployment or StatefulSet to be modified in place. + +### How Kubernetes Component Metrics are Exposed + +Prometheus scrapes metrics from deployments known as [exporters,](https://prometheus.io/docs/instrumenting/exporters/) which export the time series data in a format that Prometheus can ingest. In Prometheus, time series consist of streams of timestamped values belonging to the same metric and the same set of labeled dimensions. + +### Scraping Metrics with PushProx + +Certain internal Kubernetes components are scraped via a proxy deployed as part of Monitoring V2 called PushProx. For detailed information on PushProx, refer [here](#how-pushprox-works) and to the above [architecture](#1-architecture-overview) section. + +### Scraping Metrics + +The following Kubernetes components are directly scraped by Prometheus: + +- kubelet* +- ingress-nginx** +- coreDns/kubeDns +- kube-api-server + +\* You can optionally use `hardenedKubelet.enabled` to use a PushProx, but that is not the default. + +** For RKE and RKE2 clusters, ingress-nginx is deployed by default and treated as an internal Kubernetes component. + + +### Scraping Metrics Based on Kubernetes Distribution + +Metrics are scraped differently based on the Kubernetes distribution. For help with terminology, refer [here](#terminology). For details, see the table below: + +
How Metrics are Exposed to Prometheus
+ +| Kubernetes Component | RKE | RKE2 | KubeADM | K3s | +|-----|-----|-----|-----|-----| +| kube-controller-manager | rkeControllerManager.enabled |rke2ControllerManager.enabled | kubeAdmControllerManager.enabled | k3sServer.enabled | +| kube-scheduler | rkeScheduler.enabled | rke2Scheduler.enabled |kubeAdmScheduler.enabled | k3sServer.enabled | +| etcd | rkeEtcd.enabled | rke2Etcd.enabled | kubeAdmEtcd.enabled | Not available | +| kube-proxy | rkeProxy.enabled | rke2Proxy.enabled | kubeAdmProxy.enabled | k3sServer.enabled | +| kubelet | Collects metrics directly exposed by kubelet | Collects metrics directly exposed by kubelet | Collects metrics directly exposed by kubelet | Collects metrics directly exposed by kubelet | +| ingress-nginx* | Collects metrics directly exposed by kubelet, exposed by rkeIngressNginx.enabled | Collects metrics directly exposed by kubelet, Exposed by rke2IngressNginx.enabled | Not available | Not available | +| coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | Collects metrics directly exposed by coreDns/kubeDns | +| kube-api-server | Collects metrics directly exposed by kube-api-server |Collects metrics directly exposed by kube-api-server | Collects metrics directly exposed by kube-appi-server | Collects metrics directly exposed by kube-api-server | + +\* For RKE and RKE2 clusters, ingress-nginx is deployed by default and treated as an internal Kubernetes component. + +### Terminology + +- **kube-scheduler:** The internal Kubernetes component that uses information in the pod spec to decide on which node to run a pod. +- **kube-controller-manager:** The internal Kubernetes component that is responsible for node management (detecting if a node fails), pod replication and endpoint creation. +- **etcd:** The internal Kubernetes component that is the distributed key/value store which Kubernetes uses for persistent storage of all cluster information. +- **kube-proxy:** The internal Kubernetes component that watches the API server for pods/services changes in order to maintain the network up to date. +- **kubelet:** The internal Kubernetes component that watches the API server for pods on a node and makes sure they are running. +- **ingress-nginx:** An Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer. +- **coreDns/kubeDns:** The internal Kubernetes component responsible for DNS. +- **kube-api-server:** The main internal Kubernetes component that is responsible for exposing APIs for the other master components. diff --git a/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md new file mode 100644 index 00000000000..e301620d92a --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md @@ -0,0 +1,365 @@ +--- +title: PromQL Expression Reference +--- + +The PromQL expressions in this doc can be used to configure alerts. + +For more information about querying the Prometheus time series database, refer to the official [Prometheus documentation.](https://prometheus.io/docs/prometheus/latest/querying/basics/) + + +## Cluster Metrics + +### Cluster CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance))` | +| Summary | `1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])))` | + +### Cluster Load Average + +| Catalog | Expression | +| --- | --- | +| Detail |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)`
| +| Summary |
load1`sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load5`sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"})`
load15`sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"})`
| + +### Cluster Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `1 - sum(node_memory_MemAvailable_bytes) by (instance) / sum(node_memory_MemTotal_bytes) by (instance)` | +| Summary | `1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes)` | + +### Cluster Disk Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `(sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance) - sum(node_filesystem_free_bytes{device!="rootfs"}) by (instance)) / sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance)` | +| Summary | `(sum(node_filesystem_size_bytes{device!="rootfs"}) - sum(node_filesystem_free_bytes{device!="rootfs"})) / sum(node_filesystem_size_bytes{device!="rootfs"})` | + +### Cluster Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(node_disk_read_bytes_total[5m])) by (instance)`
written`sum(rate(node_disk_written_bytes_total[5m])) by (instance)`
| +| Summary |
read`sum(rate(node_disk_read_bytes_total[5m]))`
written`sum(rate(node_disk_written_bytes_total[5m]))`
| + +### Cluster Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| Summary |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +### Cluster Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m])) by (instance)
| +| Summary |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*"}[5m]))
| + +## Node Metrics + +### Node CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `avg(irate(node_cpu_seconds_total{mode!="idle", instance=~"$instance"}[5m])) by (mode)` | +| Summary | `1 - (avg(irate(node_cpu_seconds_total{mode="idle", instance=~"$instance"}[5m])))` | + +### Node Load Average + +| Catalog | Expression | +| --- | --- | +| Detail |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| +| Summary |
load1`sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load5`sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
load15`sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})`
| + +### Node Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"})` | +| Summary | `1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"}) ` | + +### Node Disk Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"}) by (device)) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device)` | +| Summary | `(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"})) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"})` | + +### Node Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| +| Summary |
read`sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))`
written`sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))`
| + +### Node Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| Summary |
receive-droppedsum(rate(node_network_receive_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-errssum(rate(node_network_receive_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
receive-packetssum(rate(node_network_receive_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-droppedsum(rate(node_network_transmit_drop_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-errssum(rate(node_network_transmit_errs_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmit-packetssum(rate(node_network_transmit_packets_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +### Node Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m])) by (device)
| +| Summary |
receivesum(rate(node_network_receive_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
transmitsum(rate(node_network_transmit_bytes_total{device!~"lo | veth.* | docker.* | flannel.* | cali.* | cbr.*",instance=~"$instance"}[5m]))
| + +## Etcd Metrics + +### Etcd Has a Leader + +`max(etcd_server_has_leader)` + +### Number of Times the Leader Changes + +`max(etcd_server_leader_changes_seen_total)` + +### Number of Failed Proposals + +`sum(etcd_server_proposals_failed_total)` + +### GRPC Client Traffic + +| Catalog | Expression | +| --- | --- | +| Detail |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m])) by (instance)`
| +| Summary |
in`sum(rate(etcd_network_client_grpc_received_bytes_total[5m]))`
out`sum(rate(etcd_network_client_grpc_sent_bytes_total[5m]))`
| + +### Peer Traffic + +| Catalog | Expression | +| --- | --- | +| Detail |
in`sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)`
| +| Summary |
in`sum(rate(etcd_network_peer_received_bytes_total[5m]))`
out`sum(rate(etcd_network_peer_sent_bytes_total[5m]))`
| + +### DB Size + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(etcd_debugging_mvcc_db_total_size_in_bytes) by (instance)` | +| Summary | `sum(etcd_debugging_mvcc_db_total_size_in_bytes)` | + +### Active Streams + +| Catalog | Expression | +| --- | --- | +| Detail |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance)`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance)`
| +| Summary |
lease-watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})`
watch`sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})`
| + +### Raft Proposals + +| Catalog | Expression | +| --- | --- | +| Detail |
applied`sum(increase(etcd_server_proposals_applied_total[5m])) by (instance)`
committed`sum(increase(etcd_server_proposals_committed_total[5m])) by (instance)`
pending`sum(increase(etcd_server_proposals_pending[5m])) by (instance)`
failed`sum(increase(etcd_server_proposals_failed_total[5m])) by (instance)`
| +| Summary |
applied`sum(increase(etcd_server_proposals_applied_total[5m]))`
committed`sum(increase(etcd_server_proposals_committed_total[5m]))`
pending`sum(increase(etcd_server_proposals_pending[5m]))`
failed`sum(increase(etcd_server_proposals_failed_total[5m]))`
| + +### RPC Rate + +| Catalog | Expression | +| --- | --- | +| Detail |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m])) by (instance)`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m])) by (instance)`
| +| Summary |
total`sum(rate(grpc_server_started_total{grpc_type="unary"}[5m]))`
fail`sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m]))`
| + +### Disk Operations + +| Catalog | Expression | +| --- | --- | +| Detail |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m])) by (instance)`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m])) by (instance)`
| +| Summary |
commit-called-by-backend`sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m]))`
fsync-called-by-wal`sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m]))`
| + +### Disk Sync Duration + +| Catalog | Expression | +| --- | --- | +| Detail |
wal`histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))`
db`histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))`
| +| Summary |
wal`sum(histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)))`
db`sum(histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le)))`
| + +## Kubernetes Components Metrics + +### API Server Request Latency + +| Catalog | Expression | +| --- | --- | +| Detail | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance, verb) /1e+06` | +| Summary | `avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance) /1e+06` | + +### API Server Request Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(apiserver_request_count[5m])) by (instance, code)` | +| Summary | `sum(rate(apiserver_request_count[5m])) by (instance)` | + +### Scheduling Failed Pods + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(kube_pod_status_scheduled{condition="false"})` | +| Summary | `sum(kube_pod_status_scheduled{condition="false"})` | + +### Controller Manager Queue Depth + +| Catalog | Expression | +| --- | --- | +| Detail |
volumes`sum(volumes_depth) by instance`
deployment`sum(deployment_depth) by instance`
replicaset`sum(replicaset_depth) by instance`
service`sum(service_depth) by instance`
serviceaccount`sum(serviceaccount_depth) by instance`
endpoint`sum(endpoint_depth) by instance`
daemonset`sum(daemonset_depth) by instance`
statefulset`sum(statefulset_depth) by instance`
replicationmanager`sum(replicationmanager_depth) by instance`
| +| Summary |
volumes`sum(volumes_depth)`
deployment`sum(deployment_depth)`
replicaset`sum(replicaset_depth)`
service`sum(service_depth)`
serviceaccount`sum(serviceaccount_depth)`
endpoint`sum(endpoint_depth)`
daemonset`sum(daemonset_depth)`
statefulset`sum(statefulset_depth)`
replicationmanager`sum(replicationmanager_depth)`
| + +### Scheduler E2E Scheduling Latency + +| Catalog | Expression | +| --- | --- | +| Detail | `histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06` | +| Summary | `sum(histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06)` | + +### Scheduler Preemption Attempts + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(scheduler_total_preemption_attempts[5m])) by (instance)` | +| Summary | `sum(rate(scheduler_total_preemption_attempts[5m]))` | + +### Ingress Controller Connections + +| Catalog | Expression | +| --- | --- | +| Detail |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"}) by (instance)`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"}) by (instance)`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"}) by (instance)`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m]))) by (instance)`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m]))) by (instance)`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m]))) by (instance)`
| +| Summary |
reading`sum(nginx_ingress_controller_nginx_process_connections{state="reading"})`
waiting`sum(nginx_ingress_controller_nginx_process_connections{state="waiting"})`
writing`sum(nginx_ingress_controller_nginx_process_connections{state="writing"})`
accepted`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m])))`
active`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m])))`
handled`sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m])))`
| + +### Ingress Controller Request Process Time + +| Catalog | Expression | +| --- | --- | +| Detail | `topk(10, histogram_quantile(0.95,sum by (le, host, path)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | +| Summary | `topk(10, histogram_quantile(0.95,sum by (le, host)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))` | + +## Rancher Logging Metrics + + +### Fluentd Buffer Queue Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_output_status_buffer_queue_length[5m])) by (instance)` | +| Summary | `sum(rate(fluentd_output_status_buffer_queue_length[5m]))` | + +### Fluentd Input Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)` | +| Summary | `sum(rate(fluentd_input_status_num_records_total[5m]))` | + +### Fluentd Output Errors Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_output_status_num_errors[5m])) by (type)` | +| Summary | `sum(rate(fluentd_output_status_num_errors[5m]))` | + +### Fluentd Output Rate + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(rate(fluentd_output_status_num_records_total[5m])) by (instance)` | +| Summary | `sum(rate(fluentd_output_status_num_records_total[5m]))` | + +## Workload Metrics + +### Workload CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### Workload Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""}) by (pod_name)` | +| Summary | `sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""})` | + +### Workload Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### Workload Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +### Workload Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)`
| +| Summary |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))`
| + +## Pod Metrics + +### Pod CPU Utilization + +| Catalog | Expression | +| --- | --- | +| Detail |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)`
| +| Summary |
cfs throttled seconds`sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
usage seconds`sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
system seconds`sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
user seconds`sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))`
| + +### Pod Memory Utilization + +| Catalog | Expression | +| --- | --- | +| Detail | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""}) by (container_name)` | +| Summary | `sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""})` | + +### Pod Network Packets + +| Catalog | Expression | +| --- | --- | +| Detail |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| Summary |
receive-packets`sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-dropped`sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
receive-errors`sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-packets`sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-dropped`sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit-errors`sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod Network I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| +| Summary |
receive`sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
transmit`sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +### Pod Disk I/O + +| Catalog | Expression | +| --- | --- | +| Detail |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)`
| +| Summary |
read`sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
write`sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))`
| + +## Container Metrics + +### Container CPU Utilization + +| Catalog | Expression | +| --- | --- | +| cfs throttled seconds | `sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| usage seconds | `sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| system seconds | `sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| user seconds | `sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | + +### Container Memory Utilization + +`sum(container_memory_working_set_bytes{namespace="$namespace",pod_name="$podName",container_name="$containerName"})` + +### Container Disk I/O + +| Catalog | Expression | +| --- | --- | +| read | `sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | +| write | `sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))` | diff --git a/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md new file mode 100644 index 00000000000..7993c1040c6 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md @@ -0,0 +1,251 @@ +--- +title: Role-based Access Control +--- +This section describes the expectations for RBAC for Rancher Monitoring. + + +## Cluster Admins + +By default, only those with the cluster-admin `ClusterRole` should be able to: + +- Install the `rancher-monitoring` App onto a cluster and all other relevant configuration performed on the chart deploy + - e.g. whether default dashboards are created, what exporters are deployed onto the cluster to collect metrics, etc. +- Create / modify / delete Prometheus deployments in the cluster via Prometheus CRs +- Create / modify / delete Alertmanager deployments in the cluster via Alertmanager CRs +- Persist new Grafana dashboards or datasources via creating ConfigMaps in the appropriate namespace +- Expose certain Prometheus metrics to the k8s Custom Metrics API for HPA via a Secret in the `cattle-monitoring-system` namespace + +## Users with Kubernetes ClusterRole-based Permissions + +The `rancher-monitoring` chart installs the following three `ClusterRoles`. By default, they aggregate into the corresponding k8s `ClusterRoles`: + +| ClusterRole | Aggregates To Default K8s ClusterRole | +| ------------------------------| ---------------------------| +| `monitoring-admin` | `admin`| +| `monitoring-edit` | `edit` | +| `monitoring-view` | `view ` | + +These `ClusterRoles` provide different levels of access to the Monitoring CRDs based on the actions that can be performed: + +| CRDs (monitoring.coreos.com) | Admin | Edit | View | +| ------------------------------| ---------------------------| ---------------------------| ---------------------------| +|
  • `prometheuses`
  • `alertmanagers`
| Get, List, Watch | Get, List, Watch | Get, List, Watch | +|
  • `servicemonitors`
  • `podmonitors`
  • `prometheusrules`
| * | * | Get, List, Watch | + +On a high level, the following permissions are assigned by default as a result. + +### Users with Kubernetes Admin/Edit Permissions + +Only those with the the cluster-admin, admin or edit `ClusterRole` should be able to: + +- Modify the scrape configuration of Prometheus deployments via ServiceMonitor and PodMonitor CRs +- Modify the alerting / recording rules of a Prometheus deployment via PrometheusRules CRs + +### Users with Kubernetes View Permissions + +Only those with who have some Kubernetes `ClusterRole` should be able to: + +- View the configuration of Prometheuses that are deployed within the cluster +- View the configuration of Alertmanagers that are deployed within the cluster +- View the scrape configuration of Prometheus deployments via ServiceMonitor and PodMonitor CRs +- View the alerting/recording rules of a Prometheus deployment via PrometheusRules CRs + +### Additional Monitoring Roles + +Monitoring also creates additional `Roles` that are not assigned to users by default but are created within the cluster. They can be bound to a namespace by deploying a `RoleBinding` that references it. To define a `RoleBinding` with `kubectl` instead of through Rancher, click [here](#assigning-roles-and-clusterroles-with-kubectl). + +Admins should use these roles to provide more fine-grained access to users: + +| Role | Purpose | +| ------------------------------| ---------------------------| +| monitoring-config-admin | Allow admins to assign roles to users to be able to view / modify Secrets and ConfigMaps within the cattle-monitoring-system namespace. Modifying Secrets / ConfigMaps in this namespace could allow users to alter the cluster's Alertmanager configuration, Prometheus Adapter configuration, additional Grafana datasources, TLS secrets, etc. | +| monitoring-config-edit | Allow admins to assign roles to users to be able to view / modify Secrets and ConfigMaps within the cattle-monitoring-system namespace. Modifying Secrets / ConfigMaps in this namespace could allow users to alter the cluster's Alertmanager configuration, Prometheus Adapter configuration, additional Grafana datasources, TLS secrets, etc. | +| monitoring-config-view | Allow admins to assign roles to users to be able to view Secrets and ConfigMaps within the cattle-monitoring-system namespace. Viewing Secrets / ConfigMaps in this namespace could allow users to observe the cluster's Alertmanager configuration, Prometheus Adapter configuration, additional Grafana datasources, TLS secrets, etc. | +| monitoring-dashboard-admin | Allow admins to assign roles to users to be able to edit / view ConfigMaps within the cattle-dashboards namespace. ConfigMaps in this namespace will correspond to Grafana Dashboards that are persisted onto the cluster. | +| monitoring-dashboard-edit | Allow admins to assign roles to users to be able to edit / view ConfigMaps within the cattle-dashboards namespace. ConfigMaps in this namespace will correspond to Grafana Dashboards that are persisted onto the cluster. | +| monitoring-dashboard-view | Allow admins to assign roles to users to be able to view ConfigMaps within the cattle-dashboards namespace. ConfigMaps in this namespace will correspond to Grafana Dashboards that are persisted onto the cluster. | + + +### Assigning Monitoring Roles through Custom Roles + +Admins may assign custom roles in the Rancher UI for admin, editing, and viewing monitoring. These "roles" are created by default when the monitoring app is installed. Additionally, these roles are also deployed to the corresponding Kubernetes roles: admin, edit, and view `ClusterRoles`. + +:::note Important + +The UI won't offer `monitoring-admin`, `monitoring-edit`, and `monitoring-view` options when users are being added to a cluster. These monitoring roles can only be assigned by manually creating a custom role that inherits from Project Owner and Project Monitoring View roles. + +::: + + 1. Create the custom role: + + 1.1 Click **☰ > Users & Authentication > Roles**. + + 1.2 Select the appropriate tab, e.g., **Cluster** role. Then click **Create Cluster Role**. + + 1.3 In the **Name** field, create a custom role such as `View Monitoring`, `Edit Monitoring`, or `Admin Monitoring`. + + 1.4 Click **Inherit From > Add Resource**, then select the Kubernetes role, as applicable, from the dropdown. + + 1.5 Click **Create**. + + + 2. Assign the custom role to a new user: + + 2.1 Click **☰ > Cluster Management > Cluster Explore > Cluster > Cluster Members > Add**. + + 2.2 Search for your new user name from **Select Member** options displayed. + + 2.3 Assign the new custom role from **Cluster Permissions** to the new user. + + 2.4 Click **Create**. + +**Result:** The new user should now be able to see the monitoring tools. + +### Additional Monitoring ClusterRoles + +Monitoring also creates additional `ClusterRoles` that aren't assigned to users by default but are created within the cluster. They aren't aggregated by default but can be bound to a namespace by deploying a `RoleBinding` or `ClusterRoleBinding` that references it. To define a `RoleBinding` with `kubectl` instead of through Rancher, click [here](#assigning-roles-and-clusterroles-with-kubectl). + +| Role | Purpose | +| ------------------------------| ---------------------------| +| monitoring-ui-view | _Available as of Monitoring v2 14.5.100+_ This ClusterRole allows users to view metrics graphs for the specified cluster in the Rancher UI. This is done by granting read-only access to external Monitoring UIs. Users with this role have permission to list the Prometheus, Alertmanager, and Grafana endpoints and make GET requests to Prometheus, Grafana, and Alertmanager UIs through the Rancher proxy. | + +### Assigning Roles and ClusterRoles with kubectl + +#### Using `kubectl create` + +One method is to use either `kubectl create clusterrolebinding` or `kubectl create rolebinding` to assign a `Role` or `ClusterRole`. This is shown in the following examples: + +- Assign to a specific user: + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --user=u-l4npx --namespace=my-namespace + ``` + + + +- Assign to all authenticated users: + + + + ```plain + kubectl create clusterrolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated + ``` + + + + + ```plain + kubectl create rolebinding my-binding --clusterrole=monitoring-ui-view --group=system:authenticated --namespace=my-namespace + ``` + + + + +#### Using YAML Files + +Another method is to define bindings in YAML files that you create. You must first configure the `RoleBinding` or `ClusterRoleBinding` with a YAML file. Then, apply the configuration changes by running the `kubectl apply` command. + +- **Roles**: Below is an example YAML file to help you configure `RoleBindings` in Kubernetes. You'll need to fill in the name below. + +:::note + +Names are case-sensitive. + +::: + +```yaml +# monitoring-config-view-role-binding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: monitoring-config-view + namespace: cattle-monitoring-system +roleRef: + kind: Role + name: monitoring-config-view + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: User + name: u-b4qkhsnliz # this can be found via `kubectl get users -A` + apiGroup: rbac.authorization.k8s.io +``` + +- **kubectl**: Below is an example of a `kubectl` command used to apply the binding you've created in the YAML file. Remember to fill in your YAML filename accordingly. + ```plain + kubectl apply -f monitoring-config-view-role-binding.yaml + ``` + +## Users with Rancher Based Permissions + +The relationship between the default roles deployed by Rancher (i.e. cluster-owner, cluster-member, project-owner, project-member), the default Kubernetes roles, and the roles deployed by the rancher-monitoring chart are detailed in the table below: + +
Default Rancher Permissions and Corresponding Kubernetes ClusterRoles
+ +| Rancher Role | Kubernetes Role | Monitoring ClusterRole / Role | ClusterRoleBinding or RoleBinding? | +| --------- | --------- | --------- | --------- | +| cluster-owner | cluster-admin | N/A | ClusterRoleBinding | +| cluster-member | admin | monitoring-admin | ClusterRoleBinding | +| project-owner | admin | monitoring-admin | RoleBinding within Project namespace | +| project-member | edit | monitoring-edit | RoleBinding within Project namespace | + +In addition to these default Roles, the following additional Rancher project roles can be applied to members of your Cluster to provide additional access to Monitoring. These Rancher Roles will be tied to ClusterRoles deployed by the Monitoring chart: + +
Non-default Rancher Permissions and Corresponding Kubernetes ClusterRoles
+ +| Rancher Role | Kubernetes ClusterRole | Available In Rancher From | Available in Monitoring v2 From | +|--------------------------|-------------------------------|-------|------| +| View Monitoring* | [monitoring-ui-view](#monitoring-ui-view) | 2.4.8+ | 9.4.204+ | + +\* A User bound to the **View Monitoring** Rancher Role only has permissions to access external Monitoring UIs if provided links to those UIs. In order to access the Monitoring Pane to get those links, the User must be a Project Member of at least one Project. + +### Differences in 2.5.x + +Users with the project-member or project-owners roles assigned will not be given access to either Prometheus or Grafana in Rancher 2.5.x since we only create Grafana or Prometheus on a cluster-level. + +In addition, while project owners will still be only able to add ServiceMonitors / PodMonitors that scrape resources within their project's namespace by default, PrometheusRules are not scoped to a single namespace / project. Therefore, any alert rules or recording rules created by project-owners within their project namespace will be applied across the entire cluster, although they will be unable to view / edit / delete any rules that were created outside the project's namespace. + +### Assigning Additional Access + +If cluster-admins would like to provide additional admin/edit access to users outside of the roles offered by the rancher-monitoring chart, the following table identifies the potential impact: + +|CRDs (monitoring.coreos.com) | Can it cause impact outside of a namespace / project? | Impact | +|----------------------------| ------| ----------------------------| +| `prometheuses`| Yes, this resource can scrape metrics from any targets across the entire cluster (unless the Operator itself is otherwise configured). | User will be able to define the configuration of new cluster-level Prometheus deployments that should be created in the cluster. | +| `alertmanagers`| No | User will be able to define the configuration of new cluster-level Alertmanager deployments that should be created in the cluster. Note: if you just want to allow users to configure settings like Routes and Receivers, you should just provide access to the Alertmanager Config Secret instead. | +|
  • `servicemonitors`
  • `podmonitors`
| No, not by default; this is configurable via `ignoreNamespaceSelectors` on the Prometheus CR. | User will be able to set up scrapes by Prometheus on endpoints exposed by Services / Pods within the namespace they are given this permission in. | +| `prometheusrules`| Yes, PrometheusRules are cluster-scoped. | User will be able to define alert or recording rules on Prometheus based on any series collected across the entire cluster. | + +| k8s Resources | Namespace | Can it cause impact outside of a namespace / project? | Impact | +|----------------------------| ------| ------| ----------------------------| +|
  • `secrets`
  • `configmaps`
| `cattle-monitoring-system` | Yes, Configs and Secrets in this namespace can impact the entire monitoring / alerting pipeline. | User will be able to create or edit Secrets / ConfigMaps such as the Alertmanager Config, Prometheus Adapter Config, TLS secrets, additional Grafana datasources, etc. This can have broad impact on all cluster monitoring / alerting. | +|
  • `secrets`
  • `configmaps`
| `cattle-dashboards` | Yes, Configs and Secrets in this namespace can create dashboards that make queries on all metrics collected at a cluster-level. | User will be able to create Secrets / ConfigMaps that persist new Grafana Dashboards only. | + +## Role-based Access Control for Grafana + +Rancher allows any users who are authenticated by Kubernetes and have access the Grafana service deployed by the Rancher Monitoring chart to access Grafana via the Rancher Dashboard UI. By default, all users who are able to access Grafana are given the [Viewer](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#viewer-role) role, which allows them to view any of the default dashboards deployed by Rancher. + +However, users can choose to log in to Grafana as an [Admin](https://grafana.com/docs/grafana/latest/permissions/organization_roles/#admin-role) if necessary. The default Admin username and password for the Grafana instance will be `admin`/`prom-operator`, but alternative credentials can also be supplied on deploying or upgrading the chart. + +To see the Grafana UI, install `rancher-monitoring`. Then: + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Grafana**. + +
Cluster Compute Resources Dashboard in Grafana
+ +![Cluster Compute Resources Dashboard in Grafana](/img/cluster-compute-resources-dashboard.png) + +
Default Dashboards in Grafana
+ +![Default Dashboards in Grafana](/img/grafana-default-dashboard.png) \ No newline at end of file diff --git a/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/windows-support.md b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/windows-support.md new file mode 100644 index 00000000000..6fa7e1d84d0 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/monitoring-and-alerting/windows-support.md @@ -0,0 +1,48 @@ +--- +title: Windows Cluster Support for Monitoring V2 +--- + +_Available as of v2.5.8_ + +Starting at Monitoring V2 14.5.100 (used by default in Rancher 2.5.8), Monitoring V2 can now be deployed on a Windows cluster and will scrape metrics from Windows nodes using [prometheus-community/windows_exporter](https://github.com/prometheus-community/windows_exporter) (previously named `wmi_exporter`). + +## Cluster Requirements + +Monitoring V2 for Windows can only scrape metrics from Windows hosts that have a minimum `wins` version of v0.1.0. To be able to fully deploy Monitoring V2 for Windows, all of your hosts must meet this requirement. + +If you provision a fresh RKE1 cluster in Rancher 2.5.8, your cluster should already meet this requirement. + +### Upgrading Existing Clusters to wins v0.1.0 + +If the cluster was provisioned before Rancher 2.5.8 (even if the current Rancher version is 2.5.8), you will not be able to successfully deploy Monitoring V2 for Windows until you upgrade the wins version on each host to at least v0.1.0. + +To facilitate this upgrade, Rancher 2.5.8 has released a brand new Helm chart called `rancher-wins-upgrader`. + +1. Deploy `rancher-wins-upgrader` with the following override: + ```yaml + # Masquerading bootstraps the wins-upgrader installation via + # a previously whitelisted process path since the normal install path, + # c:\etc\rancher\wins\wins-upgrade.exe is not normally whitelisted. + # In this case, we are using the previously whitelisted process + # path used by Monitoring V1. + masquerade: + enabled: true + as: c:\\etc\wmi-exporter\wmi-exporter.exe + ``` + :::note Note for Non-Default Windows Prefix Path: + + - If you set up the RKE cluster with a `cluster.yml` that has a non-default `win_prefix_path`, you will need to update the `masquerade.as` field with your prefix path in place of `c:\\`. + + - For example, if you have `win_prefix_path: 'c:\host\opt\'`, then you will need to set `as: c:\host\opt\etc\wmi-exporter\wmi-exporter.exe`. + + ::: + +2. Once all your hosts have been successfully upgraded, please ensure that you deploy the Helm chart once again with default values to avoid conflicts with the following settings: + ```yaml + masquerade: + enabled: false + ``` + +**Result:** The hosts are ready for Monitoring V2 to be installed. You may choose to uninstall the `rancher-wins-upgrader` chart or keep it in your cluster to facilitate future upgrades. + +For more information on how it can be used, please see the [README.md](https://github.com/rancher/wins/blob/master/charts/rancher-wins-upgrader/README.md) of the chart. diff --git a/versioned_docs/version-latest/integrations-in-rancher/neuvector.md b/versioned_docs/version-latest/integrations-in-rancher/neuvector.md new file mode 100644 index 00000000000..bac4c2c0849 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/neuvector.md @@ -0,0 +1,199 @@ +--- +title: NeuVector Integration +--- + +### NeuVector Integration in Rancher + +[NeuVector 5.x](https://open-docs.neuvector.com/) is an open-source container-centric security platform that is integrated with Rancher. NeuVector offers real-time compliance, visibility, and protection for critical applications and data during runtime. NeuVector provides a firewall, container process/file system monitoring, security auditing with CIS benchmarks, and vulnerability scanning. For more information on Rancher security, please see the [security documentation](../pages-for-subheaders/rancher-security.md). + +NeuVector can be enabled through a Helm chart that may be installed either through **Apps** or through the **Cluster Tools** button in the Rancher UI. Once the Helm chart is installed, users can easily [deploy and manage NeuVector clusters within Rancher](https://open-docs.neuvector.com/deploying/rancher#deploy-and-manage-neuvector-through-rancher-apps-marketplace). + +### Installing NeuVector with Rancher + +The Harvester Helm Chart is used to manage access to the NeuVector UI in Rancher where users can navigate directly to deploy and manage their NeuVector clusters. + +**To navigate to and install the NeuVector chart through Apps:** + +1. Click **☰ > Cluster Management**. +1. On the Clusters page, go to the cluster where you want to deploy NeuVector, and click **Explore**. +1. Go to **Apps > Charts**, and install **NeuVector** from the chart repo. +1. Different cluster types require different container runtimes. When configuring Helm chart values, go to the **Container Runtime** section, and select your runtime in accordance with the cluster type. Finally, click **Install** again. + +Some examples are as follows: + + - RKE1: `docker` + - K3s and RKE2: `k3scontainerd` + - AKS: `containerd` for v1.19 and up + - EKS: `docker` for v1.22 and below; `containerd` for v1.23 and up + - GKE: `containerd` (see the [Google docs](https://cloud.google.com/kubernetes-engine/docs/concepts/using-containerd) for more) + + :::note + + Only one container runtime engine may be selected at a time during installation. + + ::: + +**To navigate to and install the NeuVector chart through Cluster Tools:** + +1. Click **☰ > Cluster Management**. +1. On the Clusters page, go to the cluster where you want to deploy NeuVector, and click **Explore**. +1. Click on **Cluster Tools** at the bottom of the left navigation bar. +1. Repeat step 4 above to select your container runtime accordingly, then click **Install** again. + +### Accessing NeuVector from the Rancher UI + +1. Navigate to the cluster explorer of the cluster where NeuVector is installed. In the left navigation bar, click **NeuVector**. +1. Click the external link to go to the NeuVector UI. Once the link is selected, users must accept the `END USER LICENSE AGREEMENT` to access the NeuVector UI. + +### Uninstalling NeuVector from the Rancher UI + +**To uninstall from Apps:** + +1. Click **☰ > Cluster Management**. +1. Under **Apps**, click **Installed Apps**. +1. Under `cattle-neuvector-system`, select both the NeuVector app (and the associated CRD if desired), then click **Delete**. + +**To uninstall from Cluster Tools:** + +1. Click **☰ > Cluster Management**. +1. Click on **Cluster Tools** at the bottom-left of the screen, then click on the trash can icon under the NeuVector chart. Select `Delete the CRD associated with this app` if desired, then click **Delete**. + +### GitHub Repository + +The NeuVector project is available [here](https://github.com/neuvector/neuvector). + +### Documentation + +The NeuVector documentation is [here](https://open-docs.neuvector.com/). + +### Architecture + +The NeuVector security solution contains four types of security containers: Controllers, Enforcers, Managers, and Scanners. A special container called an All-in-One is also provided to combine the Controller, Enforcer, and Manager functions all in one container, primarily for Docker-native deployments. There is also an Updater which, when run, will update the CVE database. + +- **Controller:** Manages the NeuVector Enforcer container; provides REST APIs for the management console. +- **Enforcer:** Enforces security policies. +- **Manager:** Provides a web-UI and CLI console to manage the NeuVector platform. +- **All-in-One:** Includes the Controller, Enforcer, and Manager. +- **Scanner:** Performs the vulnerability and compliance scanning for images, containers, and nodes. +- **Updater:** Updates the CVE database for Neuvector (when run); redeploys scanner pods. + +
**NeuVector Security Containers:**
+ +![NeuVector Security Containers](/img/neuvector-security-containers.png) + +
**NeuVector Architecture:**
+ +![NeuVector Architecture](/img/neuvector-architecture.png) + +To learn more about NeuVector's architecture, please refer [here](https://open-docs.neuvector.com/basics/overview#architecture). + +### CPU and Memory Allocations + +Below are the minimum recommended computing resources for the NeuVector chart installation in a default deployment. Note that the resource limit is not set. + +| Container | CPU - Request | Memory - Request | +|------------|--------|---------| +| Controller | 3 (1GB 1vCPU needed per controller) | * +| Enforcer | On all nodes (500MB .5vCPU) | 1GB +| Manager | 1 (500MB .5vCPU) | * +| Scanner | 3 (100MB .5vCPU) | * + +\* Minimum 1GB of memory total required for Controller, Manager, and Scanner containers combined. + + +### Hardened Cluster Support - Calico and Canal + + + + + - All NeuVector components are deployable if PSP is set to true. + + You will need to set additional configuration for your hardened cluster environment as follows: + + 1. Click **☰ > Cluster Management**. + 1. Go to the cluster that you created and click **Explore**. + 1. In the left navigation bar, click **Apps**. + 1. Install (or upgrade to) NeuVector version `100.0.1+up2.2.2`. + + - Under **Edit Options** > **Other Configuration**, enable **Pod Security Policy** by checking the box (no other config needed): + + ![Enable PSP for RKE1 Hardened Cluster](/img/psp-nv-rke.png) + + 1. Click **Install** at the bottom-right to complete. + + + + + + - NeuVector components Controller and Enforcer are deployable if PSP is set to true. + + + **Applicable to NeuVector chart version 100.0.0+up2.2.0 only:** + + - For Manager, Scanner, and Updater components, additional configuration is required as shown below: + + ``` + kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' + kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' + kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' + ``` +
+ + You will need to set additional configuration for your hardened cluster environment. + + >**Note:** You must update your config in both RKE2 and K3s hardened clusters as shown below. + + 1. Click **☰ > Cluster Management**. + 1. Go to the cluster that you created and click **Explore**. + 1. In the left navigation bar, click **Apps**. + 1. Install (or upgrade to) NeuVector version `100.0.1+up2.2.2`. + + - Under **Edit Options** > **Other Configuration**, enable **Pod Security Policy** by checking the box. Note that you must also enter a value greater than `zero` for `Manager runAsUser ID`, `Scanner runAsUser ID`, and `Updater runAsUser ID`: + + ![Enable PSP for RKE2 and K3s Hardened Clusters](/img/psp-nv-rke2.png) + + 1. Click **Install** at the bottom-right to complete. + + + + + +### SELinux-enabled Cluster Support - Calico and Canal + +To enable SELinux on RKE2 clusters, follow the steps below: + +- NeuVector components Controller and Enforcer are deployable if PSP is set to true. + + +**Applicable to NeuVector chart version 100.0.0+up2.2.0 only:** + +- For Manager, Scanner, and Updater components, additional configuration is required as shown below: + +``` +kubectl patch deploy neuvector-manager-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch deploy neuvector-scanner-pod -n cattle-neuvector-system --patch '{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}' +kubectl patch cronjob neuvector-updater-pod -n cattle-neuvector-system --patch '{"spec":{"jobTemplate":{"spec":{"template":{"spec":{"securityContext":{"runAsUser": 5400}}}}}}}' +``` + +### Cluster Support in an Air-Gapped Environment + +- All NeuVector components are deployable on a cluster in an air-gapped environment without any additional configuration needed. + + +### Support Limitations + +* Only admins and cluster owners are currently supported. + +* Fleet multi-cluster deployment is not supported. + +* NeuVector is not supported on a Windows cluster. + + +### Other Limitations + +* Currently, NeuVector feature chart installation fails when a NeuVector partner chart already exists. To work around this issue, uninstall the NeuVector partner chart and reinstall the NeuVector feature chart. + +* Sometimes when the controllers are not ready, the NeuVector UI is not accessible from the Rancher UI. During this time, controllers will try to restart, and it takes a few minutes for the controllers to be active. + +* Container runtime is not auto-detected for different cluster types when installing the NeuVector chart. To work around this, you can specify the runtime manually. + diff --git a/versioned_docs/version-latest/integrations-in-rancher/opa-gatekeeper.md b/versioned_docs/version-latest/integrations-in-rancher/opa-gatekeeper.md new file mode 100644 index 00000000000..26b49d6f891 --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/opa-gatekeeper.md @@ -0,0 +1,111 @@ +--- +title: OPA Gatekeeper +--- + +To ensure consistency and compliance, every organization needs the ability to define and enforce policies in its environment in an automated way. [OPA (Open Policy Agent)](https://www.openpolicyagent.org/) is a policy engine that facilitates policy-based control for cloud native environments. Rancher provides the ability to enable OPA Gatekeeper in Kubernetes clusters, and also installs a couple of built-in policy definitions, which are also called constraint templates. + +OPA provides a high-level declarative language that lets you specify policy as code and ability to extend simple APIs to offload policy decision-making. + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) is a project that provides integration between OPA and Kubernetes. OPA Gatekeeper provides: + +- An extensible, parameterized policy library. +- Native Kubernetes CRDs for instantiating the policy library, also called “constraints." +- Native Kubernetes CRDs for extending the policy library, also called "constraint templates." +- Audit functionality. + +To read more about OPA, please refer to the [official documentation.](https://www.openpolicyagent.org/docs/latest/) + +## How the OPA Gatekeeper Integration Works + +Kubernetes provides the ability to extend API server functionality via admission controller webhooks, which are invoked whenever a resource is created, updated or deleted. Gatekeeper is installed as a validating webhook and enforces policies defined by Kubernetes custom resource definitions. In addition to the admission control usage, Gatekeeper provides the capability to audit existing resources in Kubernetes clusters and mark current violations of enabled policies. + +OPA Gatekeeper is made available via Rancher's Helm system chart, and it is installed in a namespace named `gatekeeper-system.` + +## Enabling OPA Gatekeeper in a Cluster + +:::note + +In Rancher v2.5, the OPA Gatekeeper application was improved. The Rancher v2.4 feature can't be upgraded to the new version in Rancher v2.5. If you installed OPA Gatekeeper in Rancher v2.4, you will need to uninstall OPA Gatekeeper and its CRDs from the old UI, then reinstall it in Rancher v2.5. To uninstall the CRDs run the following command in the kubectl console `kubectl delete crd configs.config.gatekeeper.sh constrainttemplates.templates.gatekeeper.sh`. + +::: + +:::note Prerequisite: + +Only administrators and cluster owners can enable OPA Gatekeeper. + +::: + +The OPA Gatekeeper Helm chart can be installed from **Apps**. + +### Enabling OPA Gatekeeper + +1. In the upper left corner, click **☰ > Cluster Management**. +1. In the **Clusters** page, go to the cluster where you want to enable OPA Gatekeeper and click **Explore**. +1. In the left navigation bar, click **Apps**. +1. Click **Charts** and click **OPA Gatekeeper**. +1. Click **Install**. + +**Result:** OPA Gatekeeper is deployed in your Kubernetes cluster. + +## Constraint Templates + +[Constraint templates](https://github.com/open-policy-agent/gatekeeper#constraint-templates) are Kubernetes custom resources that define the schema and Rego logic of the OPA policy to be applied by Gatekeeper. For more information on the Rego policy language, refer to the [official documentation.](https://www.openpolicyagent.org/docs/latest/policy-language/) + +When OPA Gatekeeper is enabled, Rancher installs some templates by default. + +To list the constraint templates installed in the cluster, go to the left side menu under OPA Gatekeeper and click on **Templates**. + +Rancher also provides the ability to create your own constraint templates by importing YAML definitions. + +## Creating and Configuring Constraints + +[Constraints](https://github.com/open-policy-agent/gatekeeper#constraints) are Kubernetes custom resources that define the scope of objects to which a specific constraint template applies to. The complete policy is defined by constraint templates and constraints together. + +:::note Prerequisite: + +OPA Gatekeeper must be enabled in the cluster. + +::: + +To list the constraints installed, go to the left side menu under OPA Gatekeeper, and click on **Constraints**. + +New constraints can be created from a constraint template. + +Rancher provides the ability to create a constraint by using a convenient form that lets you input the various constraint fields. + +The **Edit as yaml** option is also available to configure the the constraint's yaml definition. + +### Exempting Rancher's System Namespaces from Constraints + +When a constraint is created, ensure that it does not apply to any Rancher or Kubernetes system namespaces. If the system namespaces are not excluded, then it is possible to see many resources under them marked as violations of the constraint. + +To limit the scope of the constraint only to user namespaces, always specify these namespaces under the **Match** field of the constraint. + +Also, the constraint may interfere with other Rancher functionality and deny system workloads from being deployed. To avoid this, exclude all Rancher-specific namespaces from your constraints. + +## Enforcing Constraints in your Cluster + +When the **Enforcement Action** is **Deny,** the constraint is immediately enabled and will deny any requests that violate the policy defined. By default, the enforcement value is **Deny**. + +When the **Enforcement Action** is **Dryrun,** then any resources that violate the policy are only recorded under the constraint's status field. + +To enforce constraints, create a constraint using the form. In the **Enforcement Action** field, choose **Deny**. + +## Audit and Violations in your Cluster + +OPA Gatekeeper runs a periodic audit to check if any existing resource violates any enforced constraint. The audit-interval (default 300s) can be configured while installing Gatekeeper. + +On the Gatekeeper page, any violations of the defined constraints are listed. + +Also under **Constraints,** the number of violations of the constraint can be found. + +The detail view of each constraint lists information about the resource that violated the constraint. + +## Disabling Gatekeeper + +1. Navigate to the cluster's Dashboard view +1. On the left side menu, expand the cluster menu and click on **OPA Gatekeeper**. +1. Click the **⋮ > Disable**. + +**Result:** Upon disabling OPA Gatekeeper, all constraint templates and constraints will also be deleted. + diff --git a/versioned_docs/version-latest/integrations-in-rancher/rancher-extensions.md b/versioned_docs/version-latest/integrations-in-rancher/rancher-extensions.md new file mode 100644 index 00000000000..0a86dad7d2d --- /dev/null +++ b/versioned_docs/version-latest/integrations-in-rancher/rancher-extensions.md @@ -0,0 +1,84 @@ +--- +title: Rancher Extensions +--- + +New in Rancher v2.7.0, Rancher introduces **extensions**. Extensions allow users, developers, partners, and customers to extend and enhance the Rancher UI. In addition, users can make changes and create enhancements to their UI functionality independent of Rancher releases. Extensions will enable users to build on top of Rancher to better tailor it to their respective environments. Note that users will also have the ability to update to new versions as well as roll back to a previous version. + +Extensions are Helm charts that can only be installed once into a cluster; therefore, these charts have been simplified and separated from the general Helm charts listed under **Apps**. + +Examples of built-in Rancher extensions are Fleet, Explorer, and Harvester. Examples of other extensions that use the Extensions API that can be manually added are Kubewarden and Elemental. + +## Prerequisites + +> You must log in as an admin in order to view and interact with the extensions management page. + +## Installing Extensions + +1. Click **☰ > Extensions** under **Configuration**. + +2. If not already installed in **Apps**, you must enable the extension operator by clicking the **Enable** button. + + - Click **OK** to add the Rancher extension repository if your installation is not air-gapped. Otherwise, uncheck the box to do so and click **OK**. + + ![Rancher extension repository](/img/add-rancher-extension-repo.png) + +3. On the **Extensions** page, click on the **Available** tab to select which extensions you want to install. + +:::info + +In v2.7.0, the built-in extensions will not be displayed under the **Available** tab. Therefore, you will need to manually add the desired repos to install extensions. We will update the community once these extensions have been pulled out to be available for selection. + +::: +
+ +4. If no extensions are showing as available, you may manually add repos as follows: + + 4.1. On the upper right of screen, click on **â‹® > Manage Repositories > Create**. + + 4.2. Add the desired repo name, making sure to also specify the Git Repo URL and the Git Branch. + + 4.3. Click **Create** in the lower right again to complete. + + ![Manage repositories](/img/manage-repos.png) + +5. Under the **Available** tab, click **Install** on the desired extension and version as in the example below. Note that you can easily update your extension as the button to **Update** will appear on the extension if one is available. + + ![Install Kubewarden](/img/install-kubewarden.png) + +6. Click the **Reload** page button that will appear after your extension successfully installs. Note that a logged-in user who has just installed an extension will not see a change to the UI **unless** they reload the page. + + ![Reload button](/img/reload-button.png) + +## Uninstalling Extensions + +There are two ways in which you can uninstall or disable your extensions: + +1. Under the **Installed** tab, click the **Uninstall** button on the extension you wish to remove. + + ![Uninstall extensions](/img/uninstall-extension.png) + +1. On the extensions management page, click **â‹® > Disable Extension Support**. This will disable all installed extensions. + + ![Disable extensions](/img/disable-extension-support.png) + +:::caution + +You must reload the page after disabling extensions or display issues may occur. + +::: + +## Rolling Back Extensions + +Under the **Installed** tab, click the **Rollback** button on the extension you wish to roll back. + +![Roll back extensions](/img/roll-back-extension.png) + +:::caution + +You must reload the page after rolling back extensions or display issues may occur. + +::: + +## Developing Extensions + +To learn how to develop your own extensions, refer to the official [Getting Started](https://rancher.github.io/dashboard/extensions/extensions-getting-started) guide. diff --git a/versioned_docs/version-latest/pages-for-subheaders/about-provisioning-drivers.md b/versioned_docs/version-latest/pages-for-subheaders/about-provisioning-drivers.md new file mode 100644 index 00000000000..812197b3b3f --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/about-provisioning-drivers.md @@ -0,0 +1,51 @@ +--- +title: Provisioning Drivers +--- + + + + + +Drivers in Rancher allow you to manage which providers can be used to deploy [hosted Kubernetes clusters](set-up-clusters-from-hosted-kubernetes-providers.md) or [nodes in an infrastructure provider](use-new-nodes-in-an-infra-provider.md) to allow Rancher to deploy and manage Kubernetes. + +### Rancher Drivers + +With Rancher drivers, you can enable/disable existing built-in drivers that are packaged in Rancher. Alternatively, you can add your own driver if Rancher has not yet implemented it. + +There are two types of drivers within Rancher: + +* [Cluster Drivers](#cluster-drivers) +* [Node Drivers](#node-drivers) + +### Cluster Drivers + +Cluster drivers are used to provision [hosted Kubernetes clusters](set-up-clusters-from-hosted-kubernetes-providers.md), such as GKE, EKS, AKS, etc.. The availability of which cluster driver to display when creating a cluster is defined based on the cluster driver's status. Only `active` cluster drivers will be displayed as an option for creating clusters for hosted Kubernetes clusters. By default, Rancher is packaged with several existing cluster drivers, but you can also create custom cluster drivers to add to Rancher. + +By default, Rancher has activated several hosted Kubernetes cloud providers including: + +* [Amazon EKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +* [Google GKE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +* [Azure AKS](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) + +There are several other hosted Kubernetes cloud providers that are disabled by default, but are packaged in Rancher: + +* [Alibaba ACK](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +* [Huawei CCE](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) +* [Tencent](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) + +### Node Drivers + +Node drivers are used to provision hosts, which Rancher uses to launch and manage Kubernetes clusters. A node driver is the same as a [Docker Machine driver](https://docs.docker.com/machine/drivers/). The availability of which node driver to display when creating node templates is defined based on the node driver's status. Only `active` node drivers will be displayed as an option for creating node templates. By default, Rancher is packaged with many existing Docker Machine drivers, but you can also create custom node drivers to add to Rancher. + +If there are specific node drivers that you don't want to show to your users, you would need to de-activate these node drivers. + +Rancher supports several major cloud providers, but by default, these node drivers are active and available for deployment: + +* [Amazon EC2](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md) +* [Azure](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster.md) +* [Digital Ocean](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster.md) +* [vSphere](vsphere.md) + +There are several other node drivers that are disabled by default, but are packaged in Rancher: + +* [Harvester](../integrations-in-rancher/harvester.md#harvester-node-driver/), available in Rancher v2.6.1 diff --git a/versioned_docs/version-latest/pages-for-subheaders/about-rke1-templates.md b/versioned_docs/version-latest/pages-for-subheaders/about-rke1-templates.md new file mode 100644 index 00000000000..601a622a581 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/about-rke1-templates.md @@ -0,0 +1,130 @@ +--- +title: RKE Templates +--- + + + + + +RKE templates are designed to allow DevOps and security teams to standardize and simplify the creation of Kubernetes clusters. + +RKE is the [Rancher Kubernetes Engine,](https://rancher.com/docs/rke/latest/en/) which is the tool that Rancher uses to provision Kubernetes clusters. + +With Kubernetes increasing in popularity, there is a trend toward managing a larger number of smaller clusters. When you want to create many clusters, it’s more important to manage them consistently. Multi-cluster management comes with challenges to enforcing security and add-on configurations that need to be standardized before turning clusters over to end users. + +RKE templates help standardize these configurations. Regardless of whether clusters are created with the Rancher UI, the Rancher API, or an automated process, Rancher will guarantee that every cluster it provisions from an RKE template is uniform and consistent in the way it is produced. + +Admins control which cluster options can be changed by end users. RKE templates can also be shared with specific users and groups, so that admins can create different RKE templates for different sets of users. + +If a cluster was created with an RKE template, you can't change it to a different RKE template. You can only update the cluster to a new revision of the same template. + +You can [save the configuration of an existing cluster as an RKE template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#converting-an-existing-cluster-to-use-an-rke-template) Then the cluster's settings can only be changed if the template is updated. The new template can also be used to launch new clusters. + +The core features of RKE templates allow DevOps and security teams to: + +- Standardize cluster configuration and ensure that Rancher-provisioned clusters are created following best practices +- Prevent less technical users from making uninformed choices when provisioning clusters +- Share different templates with different sets of users and groups +- Delegate ownership of templates to users who are trusted to make changes to them +- Control which users can create templates +- Require users to create clusters from a template + +## Configurable Settings + +RKE templates can be created in the Rancher UI or defined in YAML format. They can define all the same parameters that can be specified when you use Rancher to provision custom nodes or nodes from an infrastructure provider: + +- Cloud provider options +- Pod security options +- Network providers +- Ingress controllers +- Network security configuration +- Network plugins +- Private registry URL and credentials +- Add-ons +- Kubernetes options, including configurations for Kubernetes components such as kube-api, kube-controller, kubelet, and services + +The [add-on section](#add-ons) of an RKE template is especially powerful because it allows a wide range of customization options. + +## Scope of RKE Templates + +RKE templates are supported for Rancher-provisioned clusters. The templates can be used to provision custom clusters or clusters that are launched by an infrastructure provider. + +RKE templates are for defining Kubernetes and Rancher settings. Node templates are responsible for configuring nodes. For tips on how to use RKE templates in conjunction with hardware, refer to [RKE Templates and Hardware](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md). + +RKE templates can be created from scratch to pre-define cluster configuration. They can be applied to launch new clusters, or templates can also be exported from existing running clusters. + +The settings of an existing cluster can be [saved as an RKE template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#converting-an-existing-cluster-to-use-an-rke-template) This creates a new template and binds the cluster settings to the template, so that the cluster can only be upgraded if the [template is updated](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#updating-a-template), and the cluster is upgraded to [use a newer version of the template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#upgrading-a-cluster-to-use-a-new-template-revision) The new template can also be used to create new clusters. + + +## Example Scenarios +When an organization has both basic and advanced Rancher users, administrators might want to give the advanced users more options for cluster creation, while restricting the options for basic users. + +These [example scenarios](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md) describe how an organization could use templates to standardize cluster creation. + +Some of the example scenarios include the following: + +- **Enforcing templates:** Administrators might want to [enforce one or more template settings for everyone](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#enforcing-a-template-setting-for-everyone) if they want all new Rancher-provisioned clusters to have those settings. +- **Sharing different templates with different users:** Administrators might give [different templates to basic and advanced users,](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#templates-for-basic-and-advanced-users) so that basic users can have more restricted options and advanced users can use more discretion when creating clusters. +- **Updating template settings:** If an organization's security and DevOps teams decide to embed best practices into the required settings for new clusters, those best practices could change over time. If the best practices change, [a template can be updated to a new revision](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#updating-templates-and-clusters-created-with-them) and clusters created from the template can [upgrade to the new version](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md#upgrading-a-cluster-to-use-a-new-template-revision) of the template. +- **Sharing ownership of a template:** When a template owner no longer wants to maintain a template, or wants to share ownership of the template, this scenario describes how [template ownership can be shared.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases.md#allowing-other-users-to-control-and-share-a-template) + +## Template Management + +When you create an RKE template, it is available in the Rancher UI from the **Cluster Management** view under **RKE Templates**. When you create a template, you become the template owner, which gives you permission to revise and share the template. You can share the RKE templates with specific users or groups, and you can also make it public. + +Administrators can turn on template enforcement to require users to always use RKE templates when creating a cluster. This allows administrators to guarantee that Rancher always provisions clusters with specific settings. + +RKE template updates are handled through a revision system. If you want to change or update a template, you create a new revision of the template. Then a cluster that was created with the older version of the template can be upgraded to the new template revision. + +In an RKE template, settings can be restricted to what the template owner chooses, or they can be open for the end user to select the value. The difference is indicated by the **Allow User Override** toggle over each setting in the Rancher UI when the template is created. + +For the settings that cannot be overridden, the end user will not be able to directly edit them. In order for a user to get different options of these settings, an RKE template owner would need to create a new revision of the RKE template, which would allow the user to upgrade and change that option. + +The documents in this section explain the details of RKE template management: + +- [Getting permission to create templates](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions.md) +- [Creating and revising templates](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates.md) +- [Enforcing template settings](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates.md#requiring-new-clusters-to-use-an-rke-template) +- [Overriding template settings](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings.md) +- [Sharing templates with cluster creators](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#sharing-templates-with-specific-users-or-groups) +- [Sharing ownership of a template](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md#sharing-ownership-of-templates) + +An [example YAML configuration file for a template](../reference-guides/rke1-template-example-yaml.md) is provided for reference. + +## Applying Templates + +You can [create a cluster from a template](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#creating-a-cluster-from-an-rke-template) that you created, or from a template that has been [shared with you.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates.md) + +If the RKE template owner creates a new revision of the template, you can [upgrade your cluster to that revision.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#updating-a-cluster-created-with-an-rke-template) + +RKE templates can be created from scratch to pre-define cluster configuration. They can be applied to launch new clusters, or templates can also be exported from existing running clusters. + +You can [save the configuration of an existing cluster as an RKE template.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates.md#converting-an-existing-cluster-to-use-an-rke-template) Then the cluster's settings can only be changed if the template is updated. + +## Standardizing Hardware + +RKE templates are designed to standardize Kubernetes and Rancher settings. If you want to standardize your infrastructure as well, one option is to use RKE templates [in conjunction with other tools](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure.md). + +Another option is to use [cluster templates,](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md) which include node pool configuration options, but don't provide configuration enforcement. + +## YAML Customization + +If you define an RKE template as a YAML file, you can modify this [example RKE template YAML](../reference-guides/rke1-template-example-yaml.md). The YAML in the RKE template uses the same customization that Rancher uses when creating an RKE cluster, but since the YAML is located within the context of a Rancher provisioned cluster, you will need to nest the RKE template customization under the `rancher_kubernetes_engine_config` directive in the YAML. + +The RKE documentation also has [annotated](https://rancher.com/docs/rke/latest/en/example-yamls/) `cluster.yml` files that you can use for reference. + +For guidance on available options, refer to the RKE documentation on [cluster configuration.](https://rancher.com/docs/rke/latest/en/config-options/) + +### Add-ons + +The add-on section of the RKE template configuration file works the same way as the [add-on section of a cluster configuration file](https://rancher.com/docs/rke/latest/en/config-options/add-ons/). + +The user-defined add-ons directive allows you to either call out and pull down Kubernetes manifests or put them inline directly. If you include these manifests as part of your RKE template, Rancher will provision those in the cluster. + +Some things you could do with add-ons include: + +- Install applications on the Kubernetes cluster after it starts +- Install plugins on nodes that are deployed with a Kubernetes daemonset +- Automatically set up namespaces, service accounts, or role binding + +The RKE template configuration must be nested within the `rancher_kubernetes_engine_config` directive. To set add-ons, when creating the template, you will click **Edit as YAML**. Then use the `addons` directive to add a manifest, or the `addons_include` directive to set which YAML files are used for the add-ons. For more information on custom add-ons, refer to the [user-defined add-ons documentation.](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/) diff --git a/versioned_docs/version-latest/pages-for-subheaders/about-the-api.md b/versioned_docs/version-latest/pages-for-subheaders/about-the-api.md new file mode 100644 index 00000000000..3b39d7c2717 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/about-the-api.md @@ -0,0 +1,84 @@ +--- +title: API +--- + + + + + +## How to use the API + +The API has its own user interface accessible from a web browser. This is an easy way to see resources, perform actions, and see the equivalent cURL or HTTP request & response. To access it: + + + + +1. Click on your user avatar in the upper right corner. +1. Click **Account & API Keys**. +1. Under the **API Keys** section, find the **API Endpoint** field and click the link. The link will look something like `https:///v3`, where `` is the fully qualified domain name of your Rancher deployment. + + + + +Go to the URL endpoint at `https:///v3`, where `` is the fully qualified domain name of your Rancher deployment. + + + + +## Authentication + +API requests must include authentication information. Authentication is done with HTTP basic authentication using [API Keys](../reference-guides/user-settings/api-keys.md). API keys can create new clusters and have access to multiple clusters via `/v3/clusters/`. [Cluster and project roles](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md) apply to these keys and restrict what clusters and projects the account can see and what actions they can take. + +By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. For details on how to invalidate them, refer to the [API tokens page](../reference-guides/about-the-api/api-tokens.md). + +## Making requests + +The API is generally RESTful but has several features to make the definition of everything discoverable by a client so that generic clients can be written instead of having to write specific code for every type of resource. For detailed info about the generic API spec, [see here](https://github.com/rancher/api-spec/blob/master/specification.md). + +- Every type has a Schema which describes: + - The URL to get to the collection of this type of resources + - Every field the resource can have, along with their type, basic validation rules, whether they are required or optional, etc. + - Every action that is possible on this type of resource, with their inputs and outputs (also as schemas). + - Every field that filtering is allowed on + - What HTTP verb methods are available for the collection itself, or for individual resources in the collection. + + +- So the theory is that you can load just the list of schemas and know everything about the API. This is in fact how the UI for the API works, it contains no code specific to Rancher itself. The URL to get Schemas is sent in every HTTP response as a `X-Api-Schemas` header. From there you can follow the `collection` link on each schema to know where to list resources, and other `links` inside of the returned resources to get any other information. + +- In practice, you will probably just want to construct URL strings. We highly suggest limiting this to the top-level to list a collection (`/v3/`) or get a specific resource (`/v3//`). Anything deeper than that is subject to change in future releases. + +- Resources have relationships between each other called links. Each resource includes a map of `links` with the name of the link and the URL to retrieve that information. Again you should `GET` the resource and then follow the URL in the `links` map, not construct these strings yourself. + +- Most resources have actions, which do something or change the state of the resource. To use these, send a HTTP `POST` to the URL in the `actions` map for the action you want. Some actions require input or produce output, see the individual documentation for each type or the schemas for specific information. + +- To edit a resource, send a HTTP `PUT` to the `links.update` link on the resource with the fields that you want to change. If the link is missing then you don't have permission to update the resource. Unknown fields and ones that are not editable are ignored. + +- To delete a resource, send a HTTP `DELETE` to the `links.remove` link on the resource. If the link is missing then you don't have permission to update the resource. + +- To create a new resource, HTTP `POST` to the collection URL in the schema (which is `/v3/`). + +## Filtering + +Most collections can be filtered on the server-side by common fields using HTTP query parameters. The `filters` map shows you what fields can be filtered on and what the filtered values were for the request you made. The API UI has controls to setup filtering and show you the appropriate request. For simple "equals" matches it's just `field=value`. Modifiers can be added to the field name, e.g. `field_gt=42` for "field is greater than 42". See the [API spec](https://github.com/rancher/api-spec/blob/master/specification.md#filtering) for full details. + +## Sorting + +Most collections can be sorted on the server-side by common fields using HTTP query parameters. The `sortLinks` map shows you what sorts are available, along with the URL to get the collection sorted by that. It also includes info about what the current response was sorted by, if specified. + +## Pagination + +API responses are paginated with a limit of 100 resources per page by default. This can be changed with the `limit` query parameter, up to a maximum of 1000, e.g. `/v3/pods?limit=1000`. The `pagination` map in collection responses tells you whether or not you have the full result set and has a link to the next page if you do not. + +## Capturing Rancher API Calls + +You can use browser developer tools to capture how the Rancher API is called. For example, you could follow these steps to use the Chrome developer tools to get the API call for provisioning an RKE cluster: + +1. In the Rancher UI, go to **Cluster Management** and click **Create.** +1. Click one of the cluster types. This example uses Digital Ocean. +1. Fill out the form with a cluster name and node template, but don't click **Create**. +1. You will need to open the developer tools before the cluster creation to see the API call being recorded. To open the tools, right-click on the Rancher UI and click **Inspect.** +1. In the developer tools, click the **Network** tab. +1. On the **Network** tab, make sure **Fetch/XHR** is selected. +1. In the Rancher UI, click **Create**. In the developer tools, you should see a new network request with the name `cluster?_replace=true`. +1. Right-click `cluster?_replace=true` and click **Copy > Copy as cURL.** +1. Paste the result into any text editor. You will be able to see the POST request, including the URL it was sent to, all of the headers, and the full body of the request. This command can be used to create a cluster from the command line. Note: The request should be stored in a safe place because it contains credentials. diff --git a/versioned_docs/version-latest/pages-for-subheaders/access-clusters.md b/versioned_docs/version-latest/pages-for-subheaders/access-clusters.md new file mode 100644 index 00000000000..b31664726f9 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/access-clusters.md @@ -0,0 +1,65 @@ +--- +title: Cluster Access +--- + + + + + +This section is about what tools can be used to access clusters managed by Rancher. + +For information on how to give users permission to access a cluster, see the section on [adding users to clusters.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) + +For more information on roles-based access control, see [this section.](manage-role-based-access-control-rbac.md) + +For information on how to set up an authentication system, see [this section.](authentication-config.md) + +On the **Clusters** page, select **â** at the end of each row to view a submenu with the following options: + +* [Kubectl Shell](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) +* Download KubeConfig +* Copy KubeConfig to Clipboard +* Edit Config +* View YAML +* Download YAML + +### Cluster Dashboard + +On the **Clusters** page, select the **Explore** button at the end of each row to view that cluster's **Cluster Dashboard**. You can also view the dashboard by clicking the name of a cluster in the table, then clicking the **Explore** buttton on the **Cluster** page. + +The **Cluster Dashboard** is also accessible from the Rancher UI **Home** page, by clicking on the name of a cluster. + +You can also access the **Cluster Dashboard** from the **☰** in the top navigation bar: + +1. Click **☰**. +1. Select the name of a cluster from the **Explore Cluster** menu option. + +The **Cluster Dashboard** lists information about a specific cluster, such as number of nodes, memory usage, events, and resources. + +## kubectl + +## Clusters in Rancher UI + +There are several paths to view and manage clusters through the Rancher UI. + +### Clusters Page + +You can access the **Clusters** page from the **☰** menu: + +1. Click **☰**. +1. Select **Cluster Management**. + +You can also access the **Clusters** page by clicking the **Manage** button above the clusters table on the Rancher UI **Home** page. + +You can use the Kubernetes command-line tool, [kubectl](https://kubernetes.io/docs/reference/kubectl/overview/), to manage your clusters. You have two options for using kubectl: + +- **Rancher kubectl shell:** Interact with your clusters by launching a kubectl shell available in the Rancher UI. This option requires no configuration actions on your part. For more information, see [Accessing Clusters with kubectl Shell](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md). +- **Terminal remote connection:** You can also interact with your clusters by installing [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) on your local desktop and then copying the cluster's kubeconfig file to your local `~/.kube/config` directory. For more information, see [Accessing Clusters with kubectl and a kubeconfig File](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md). + +## Rancher CLI + +You can control your clusters by downloading Rancher's own command-line interface, [Rancher CLI](cli-with-rancher.md). This CLI tool can interact directly with different clusters and projects or pass them `kubectl` commands. + +## Rancher API + +Finally, you can interact with your clusters over the Rancher API. Before you use the API, you must obtain an [API key](../reference-guides/user-settings/api-keys.md). To view the different resource fields and actions for an API object, open the API UI, which can be accessed by clicking on **View in API** for any Rancher UI object. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/advanced-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/advanced-configuration.md new file mode 100644 index 00000000000..87efa2a0f9e --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/advanced-configuration.md @@ -0,0 +1,19 @@ +--- +title: Advanced Configuration +--- + + + + + +### Alertmanager + +For information on configuring the Alertmanager custom resource, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### Prometheus + +For information on configuring the Prometheus custom resource, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) + +### PrometheusRules + +For information on configuring the Prometheus custom resource, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/advanced-user-guides.md b/versioned_docs/version-latest/pages-for-subheaders/advanced-user-guides.md new file mode 100644 index 00000000000..b5b3d241037 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/advanced-user-guides.md @@ -0,0 +1,11 @@ +--- +title: Advanced User Guides +--- + + + + + +Advanced user guides are "problem-oriented" docs in which users learn how to answer questions or solve problems. The major difference between these and the new user guides is that these guides are geared toward more experienced or advanced users who have more technical needs from their documentation. These users already have an understanding of Rancher and its functions. They know what they need to accomplish; they just need additional guidance to complete some more complex task they they have encountered while working. + +It should be noted that neither new user guides nor advanced user guides provide detailed explanations or discussions (these kinds of docs belong elsewhere). How-to guides focus on the action of guiding users through repeatable, effective steps to learn new skills, master some task, or overcome some problem. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/air-gapped-helm-cli-install.md b/versioned_docs/version-latest/pages-for-subheaders/air-gapped-helm-cli-install.md new file mode 100644 index 00000000000..d6fbc09698f --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/air-gapped-helm-cli-install.md @@ -0,0 +1,34 @@ +--- +title: Air-Gapped Helm CLI Install +--- + + + + + +This section is about using the Helm CLI to install the Rancher server in an air gapped environment. An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +The installation steps differ depending on whether Rancher is installed on an RKE Kubernetes cluster, a K3s Kubernetes cluster, or a single Docker container. + +For more information on each installation option, refer to [this page.](installation-and-upgrade.md) + +Throughout the installation instructions, there will be _tabs_ for each installation option. + +:::note Important: + +If you install Rancher following the Docker installation guide, there is no upgrade path to transition your Docker Installation to a Kubernetes Installation. + +::: + +## Installation Outline + +1. [Set up infrastructure and private registry](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) +2. [Collect and publish images to your private registry](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images.md) +3. [Set up a Kubernetes cluster (Skip this step for Docker installations)](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes.md) +4. [Install Rancher](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md) + +## Upgrades + +To upgrade Rancher with Helm CLI in an air gap environment, follow [this procedure.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades.md) + +### [Next: Prepare your Node(s)](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/authentication-config.md b/versioned_docs/version-latest/pages-for-subheaders/authentication-config.md new file mode 100644 index 00000000000..9bb89f46ee8 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/authentication-config.md @@ -0,0 +1,137 @@ +--- +title: Authentication Config +weight: 10 +--- + + + + + +One of the key features that Rancher adds to Kubernetes is centralized user authentication. This feature allows your users to use one set of credentials to authenticate with any of your Kubernetes clusters. + +This centralized user authentication is accomplished using the Rancher authentication proxy, which is installed along with the rest of Rancher. This proxy authenticates your users and forwards their requests to your Kubernetes clusters using a service account. + +## External vs. Local Authentication + +The Rancher authentication proxy integrates with the following external authentication services. + +| Auth Service | +| ------------------------------------------------------------------------------------------------ | +| [Microsoft Active Directory](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory.md) | +| [GitHub](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github.md) | +| [Microsoft Azure AD](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad.md) | +| [FreeIPA](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa.md) | +| [OpenLDAP](configure-openldap.md) | +| [Microsoft AD FS](configure-microsoft-ad-federation-service-saml.md) | +| [PingIdentity](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity.md) | +| [Keycloak (OIDC)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc.md) | +| [Keycloak (SAML)](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml.md) | +| [Okta](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml.md) | +| [Google OAuth](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth.md) | +| [Shibboleth](configure-shibboleth-saml.md) | + +However, Rancher also provides [local authentication](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users.md). + +In most cases, you should use an external authentication service over local authentication, as external authentication allows user management from a central location. However, you may want a few local authentication users for managing Rancher under rare circumstances, such as if your external authentication provider is unavailable or undergoing maintenance. + +## Users and Groups + +Rancher relies on users and groups to determine who is allowed to log in to Rancher and which resources they can access. When authenticating with an external provider, groups are provided from the external provider based on the user. These users and groups are given specific roles to resources like clusters, projects, multi-cluster apps, and global DNS providers and entries. When you give access to a group, all users who are a member of that group in the authentication provider will be able to access the resource with the permissions that you've specified. For more information on roles and permissions, see [Role Based Access Control](manage-role-based-access-control-rbac.md). + +:::note + +Local authentication does not support creating or managing groups. + +::: + +For more information, see [Users and Groups](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups.md) + +## Scope of Rancher Authorization + +After you configure Rancher to allow sign on using an external authentication service, you should configure who should be allowed to log in and use Rancher. The following options are available: + +| Access Level | Description | +|----------------------------------------------|-------------| +| Allow any valid Users | _Any_ user in the authorization service can access Rancher. We generally discourage use of this setting! | +| Allow members of Clusters, Projects, plus Authorized Users and Organizations | Any user in the authorization service and any group added as a **Cluster Member** or **Project Member** can log in to Rancher. Additionally, any user in the authentication service or group you add to the **Authorized Users and Organizations** list may log in to Rancher. | +| Restrict access to only Authorized Users and Organizations | Only users in the authentication service or groups added to the Authorized Users and Organizations can log in to Rancher. | + +To set the Rancher access level for users in the authorization service, follow these steps: + +1. In the upper left corner, click **☰ > Users & Authentication**. +1. In the left navigation bar, click **Auth Provider**. +1. After setting up the configuration details for an auth provider, use the **Site Access** options to configure the scope of user authorization. The table above explains the access level for each option. +1. Optional: If you choose an option other than **Allow any valid Users,** you can add users to the list of authorized users and organizations by searching for them in the text field that appears. +1. Click **Save**. + +**Result:** The Rancher access configuration settings are applied. + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + +## External Authentication Configuration and Principal Users + +Configuration of external authentication requires: + +- A local user assigned the administrator role, called hereafter the _local principal_. +- An external user that can authenticate with your external authentication service, called hereafter the _external principal_. + +Configuration of external authentication affects how principal users are managed within Rancher. Follow the list below to better understand these effects. + +1. Sign into Rancher as the local principal and complete configuration of external authentication. + + ![Sign In](/img/sign-in.png) + +2. Rancher associates the external principal with the local principal. These two users share the local principal's user ID. + + ![Principal ID Sharing](/img/principal-ID.png) + +3. After you complete configuration, Rancher automatically signs out the local principal. + + ![Sign Out Local Principal](/img/sign-out-local.png) + +4. Then, Rancher automatically signs you back in as the external principal. + + ![Sign In External Principal](/img/sign-in-external.png) + +5. Because the external principal and the local principal share an ID, no unique object for the external principal displays on the Users page. + + ![Sign In External Principal](/img/users-page.png) + +6. The external principal and the local principal share the same access rights. + +:::note Reconfiguring a previously set up auth provider + +If you need to reconfigure or disable then re-enable a provider that had been previously set up, ensure that the user who attempts to do so +is logged in to Rancher as an external user, not the local admin. + +::: + +## Disabling An Auth Provider + +When you disable an auth provider, Rancher deletes all resources associated with it, such as: +- Secrets. +- Global role bindings. +- Cluster role template bindings. +- Project role template bindings. +- External users associated with the provider, but who never logged in as local users to Rancher. + +As this operation may lead to a loss of many resources, you may want to add a safeguard on the provider. To ensure that this cleanup process doesn't run when the auth provider is disabled, add a special annotation to the corresponding auth config. + +For example, to add a safeguard to the Azure AD provider, annotate the `azuread` authconfig object: + +`kubectl annotate --overwrite authconfig azuread management.cattle.io/auth-provider-cleanup='user-locked'` + +Rancher won't perform cleanup until you set the annotation to `unlocked`. + +### Running Resource Cleanup Manually + +Rancher might retain resources from a previously disabled auth provider configuration in the local cluster, even after you configure another auth provider. For example, if you used Provider A, then disabled it and started using Provider B, when you upgrade to a new version of Rancher, you can manually trigger cleanup on resources configured by Provider A. + +To manually trigger cleanup for a disabled auth provider, add the `management.cattle.io/auth-provider-cleanup` annotation with the `unlocked` value to its auth config. diff --git a/versioned_docs/version-latest/pages-for-subheaders/authentication-permissions-and-global-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/authentication-permissions-and-global-configuration.md new file mode 100644 index 00000000000..ae1a7e2fdd4 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/authentication-permissions-and-global-configuration.md @@ -0,0 +1,85 @@ +--- +title: Authentication, Permissions and Global Settings +--- + + + + + +After installation, the [system administrator](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md) should configure Rancher to configure authentication, authorization, security, default settings, security policies, drivers and global DNS entries. + +## First Log In + +After you log into Rancher for the first time, Rancher will prompt you for a **Rancher Server URL**.You should set the URL to the main entry point to the Rancher Server. When a load balancer sits in front a Rancher Server cluster, the URL should resolve to the load balancer. The system will automatically try to infer the Rancher Server URL from the IP address or host name of the host running the Rancher Server. This is only correct if you are running a single node Rancher Server installation. In most cases, therefore, you need to set the Rancher Server URL to the correct value yourself. + +:::danger + +After you set the Rancher Server URL, we do not support updating it. Set the URL with extreme care. + +::: + +## Authentication + +One of the key features that Rancher adds to Kubernetes is centralized user authentication. This feature allows to set up local users and/or connect to an external authentication provider. By connecting to an external authentication provider, you can leverage that provider's user and groups. + +For more information how authentication works and how to configure each provider, see [Authentication](authentication-config.md). + +## Authorization + +Within Rancher, each person authenticates as a _user_, which is a login that grants you access to Rancher. Once the user logs in to Rancher, their _authorization_, or their access rights within the system, is determined by the user's role. Rancher provides built-in roles to allow you to easily configure a user's permissions to resources, but Rancher also provides the ability to customize the roles for each Kubernetes resource. + +For more information how authorization works and how to customize roles, see [Roles Based Access Control (RBAC)](manage-role-based-access-control-rbac.md). + +## Pod Security Policies + +_Pod Security Policies_ (or PSPs) are objects that control security-sensitive aspects of pod specification, e.g. root privileges. If a pod does not meet the conditions specified in the PSP, Kubernetes will not allow it to start, and Rancher will display an error message. + +For more information how to create and use PSPs, see [Pod Security Policies](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md). + +## Provisioning Drivers + +Drivers in Rancher allow you to manage which providers can be used to provision [hosted Kubernetes clusters](set-up-clusters-from-hosted-kubernetes-providers.md) or [nodes in an infrastructure provider](use-new-nodes-in-an-infra-provider.md) to allow Rancher to deploy and manage Kubernetes. + +For more information, see [Provisioning Drivers](about-provisioning-drivers.md). + +## Adding Kubernetes Versions into Rancher + +With this feature, you can upgrade to the latest version of Kubernetes as soon as it is released, without upgrading Rancher. This feature allows you to easily upgrade Kubernetes patch versions (i.e. `v1.15.X`), but not intended to upgrade Kubernetes minor versions (i.e. `v1.X.0`) as Kubernetes tends to deprecate or add APIs between minor versions. + +The information that Rancher uses to provision [RKE clusters](launch-kubernetes-with-rancher.md) is now located in the Rancher Kubernetes Metadata. For details on metadata configuration and how to change the Kubernetes version used for provisioning RKE clusters, see [Rancher Kubernetes Metadata.](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md) + +Rancher Kubernetes Metadata contains Kubernetes version information which Rancher uses to provision [RKE clusters](launch-kubernetes-with-rancher.md). + +For more information on how metadata works and how to configure metadata config, see [Rancher Kubernetes Metadata](../getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher.md). + +## Global Settings + +Options that control certain global-level Rancher settings are available from the top navigation bar. + +Click **☰** in the top left corner, then select **Global Settings**, to view and configure the following settings: + +- **Settings**: Various Rancher defaults, such as the minimum length for a user's password (`password-min-length`). You should be cautious when modifying these settings, as invalid values may break your Rancher installation. +- **Feature Flags**: Rancher features that can be toggled on or off. Some of these flags are for [experimental features](#enabling-experimental-features). +- **Banners**: Elements you can add to fixed locations on the portal. For example, you can use these options to [set a custom banner](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md#fixed-banners) for users when they login to Rancher. +- **Branding**: Rancher UI design elements that you can [customize](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding.md). You can add a custom logo or favicon, and modify UI colors. +- **Performance**: Performance settings for the Rancher UI, such as incremental resource loading. +- **Home Links**: Links displayed on the Rancher UI **Home** page. You can modify visibility for the default links or add your own links. + +### Enabling Experimental Features + +Rancher includes some features that are experimental and/or disabled by default. Feature flags allow you to enable these features. For more information, refer to the section about [feature flags.](enable-experimental-features.md) + +### Global Configuration + +**Global Configuration** options aren't visible unless you activate the **legacy** [feature flag](enable-experimental-features.md). The **legacy** flag is disabled by default on fresh Rancher installs of v2.6 and later. If you upgrade from an earlier Rancher version, or activate the **legacy** feature flag on Rancher v2.6 and later, **Global Configuration** is available from the top navigation menu: + +1. Click **☰** in the top left corner. +1. Select **Global Configuration** from the **Legacy Apps**. + +The following features are available under **Global Configuration**: + +- **Catalogs** +- **Global DNS Entries** +- **Global DNS Providers** + +As these are legacy features, please see the Rancher v2.0—v2.4 docs on [catalogs](../../version-2.0-2.4/pages-for-subheaders/helm-charts-in-rancher.md), [global DNS entries](../../version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/globaldns.md#adding-a-global-dns-entry), and [global DNS providers](../../version-2.0-2.4/how-to-guides/new-user-guides/helm-charts-in-rancher/globaldns.md#editing-a-global-dns-provider) for more details. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/aws-cloud-marketplace.md b/versioned_docs/version-latest/pages-for-subheaders/aws-cloud-marketplace.md new file mode 100644 index 00000000000..c487e1d6f92 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/aws-cloud-marketplace.md @@ -0,0 +1,36 @@ +--- +title: AWS Marketplace Integration +--- + + + + + +## Overview + +Rancher offers an integration with the AWS Marketplace which allows users to purchase a support contract with SUSE. This integration allows you easily adjust your support needs as you start to support more clusters. + +## Limitations + +- You must be running Rancher v2.6.7 or higher +- Rancher must be deployed with additional metrics enabled. +- Rancher must be installed on an EKS cluster. +- You must purchase at least one entitlement to Rancher support through AWS Marketplace. +- You may need additional setup to support proxy/airgap use cases. See the [prerequisites](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md) for more information. + +## How to Use + +1. Complete the [prerequisite steps](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements.md). +2. [Install the CSP Adapter](../integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter.md). + +## FAQ + +**Can I purchase support for more nodes later on?** + +Yes. Simply go to the AWS Marketplace entry that you used to initially purchase support and increase the number of entitlements. + +**Can I use multiple instances of Rancher in the same AWS account?** + +Yes. However, each cluster that Rancher is installed in will need to adhere to the prerequisites. + +In addition, keep in mind that a given entitlement can only be used by one Rancher management server at a time. diff --git a/versioned_docs/version-latest/pages-for-subheaders/backup-restore-and-disaster-recovery.md b/versioned_docs/version-latest/pages-for-subheaders/backup-restore-and-disaster-recovery.md new file mode 100644 index 00000000000..19feeb770d0 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/backup-restore-and-disaster-recovery.md @@ -0,0 +1,100 @@ +--- +title: Backups and Disaster Recovery +keywords: [rancher backup restore, rancher backup and restore, backup restore rancher, rancher backup and restore rancher] +--- + + + + + +In this section, you'll learn how to create backups of Rancher, how to restore Rancher from backup, and how to migrate Rancher to a new Kubernetes cluster. + +The `rancher-backup` operator is used to backup and restore Rancher on any Kubernetes cluster. This application is a Helm chart, and it can be deployed through the Rancher **Apps** page, or by using the Helm CLI. The `rancher-backup` Helm chart is [here.](https://github.com/rancher/charts/tree/release-v2.6/charts/rancher-backup) + +The backup-restore operator needs to be installed in the local cluster, and only backs up the Rancher app. The backup and restore operations are performed only in the local Kubernetes cluster. + + +## Backup and Restore for Rancher installed with Docker + +For Rancher installed with Docker, refer to [this page](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) to perform backups and [this page](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md) to perform restores. + +## How Backups and Restores Work + +The `rancher-backup` operator introduces three custom resources: Backups, Restores, and ResourceSets. The following cluster-scoped custom resource definitions are added to the cluster: + +- `backups.resources.cattle.io` +- `resourcesets.resources.cattle.io` +- `restores.resources.cattle.io` + +The ResourceSet defines which Kubernetes resources need to be backed up. The ResourceSet is not available to be configured in the Rancher UI because the values required to back up Rancher are predefined. This ResourceSet should not be modified. + +When a Backup custom resource is created, the `rancher-backup` operator calls the `kube-apiserver` to get the resources in the ResourceSet (specifically, the predefined `rancher-resource-set`) that the Backup custom resource refers to. + +The operator then creates the backup file in the .tar.gz format and stores it in the location configured in the Backup resource. + +When a Restore custom resource is created, the operator accesses the backup .tar.gz file specified by the Restore, and restores the application from that file. + +The Backup and Restore custom resources can be created in the Rancher UI, or by using `kubectl apply`. + +:::note + +Refer [here](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md#2-restore-from-backup-using-a-restore-custom-resource) for help on restoring an existing backup file into a v1.22 cluster in Rancher v2.6.3. + +::: + +## Installing the rancher-backup Operator + +The `rancher-backup` operator can be installed from the Rancher UI, or with the Helm CLI. In both cases, the `rancher-backup` Helm chart is installed on the Kubernetes cluster running the Rancher server. It is a cluster-admin only feature and available only for the **local** cluster. (*If you do not see `rancher-backup` in the Rancher UI, you may have selected the wrong cluster.*) + +:::note + +There is a known issue in Fleet that occurs after performing a restoration using the backup-restore-operator: Secrets used for clientSecretName and helmSecretName are not included in Fleet gitrepos. Refer [here](./fleet-gitops-at-scale.md#troubleshooting) for a workaround. + +::: + +### Installing rancher-backup with the Rancher UI + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the `local` cluster and click **Explore**. +1. In the left navigation bar, **Apps > Charts**. +1. Click **Rancher Backups**. +1. Click **Install**. +1. Optional: Configure the default storage location. For help, refer to the [configuration section.](../reference-guides/backup-restore-configuration/storage-configuration.md) +1. Click **Install**. + +**Result:** The `rancher-backup` operator is installed. + +From the **Cluster Dashboard,** you can see the `rancher-backup` operator listed under **Deployments**. + +To configure the backup app in Rancher, go to the left navigation menu and click **Rancher Backups**. + +### RBAC + +Only the rancher admins and the local cluster’s cluster-owner can: + +* Install the Chart +* See the navigation links for Backup and Restore CRDs +* Perform a backup or restore by creating a Backup CR and Restore CR respectively +* List backups/restores performed so far + +## Backing up Rancher + +A backup is performed by creating a Backup custom resource. For a tutorial, refer to [this page.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md) + +## Restoring Rancher + +A restore is performed by creating a Restore custom resource. For a tutorial, refer to [this page.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher.md) + +## Migrating Rancher to a New Cluster + +A migration is performed by following [these steps.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +## Default Storage Location Configuration + +Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible or Minio object store. + +For information on configuring these options, refer to [this page.](../reference-guides/backup-restore-configuration/storage-configuration.md) + +### Example values.yaml for the rancher-backup Helm Chart + +The example [values.yaml file](../reference-guides/backup-restore-configuration/storage-configuration.md#example-valuesyaml-for-the-rancher-backup-helm-chart) can be used to configure the `rancher-backup` operator when the Helm CLI is used to install it. diff --git a/versioned_docs/version-latest/pages-for-subheaders/backup-restore-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/backup-restore-configuration.md new file mode 100644 index 00000000000..104584f741d --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/backup-restore-configuration.md @@ -0,0 +1,12 @@ +--- +title: Rancher Backup Configuration Reference +--- + + + + + +- [Backup configuration](../reference-guides/backup-restore-configuration/backup-configuration.md) +- [Restore configuration](../reference-guides/backup-restore-configuration/restore-configuration.md) +- [Storage location configuration](../reference-guides/backup-restore-configuration/storage-configuration.md) +- [Example Backup and Restore Custom Resources](../reference-guides/backup-restore-configuration/examples.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/best-practices.md b/versioned_docs/version-latest/pages-for-subheaders/best-practices.md new file mode 100644 index 00000000000..7009f6cce70 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/best-practices.md @@ -0,0 +1,21 @@ +--- +title: Best Practices Guide +--- + + + + + +The purpose of this section is to consolidate best practices for Rancher implementations. This also includes recommendations for related technologies, such as Kubernetes, Docker, containers, and more. The objective is to improve the outcome of a Rancher implementation using the operational experience of Rancher and its customers. + +If you have any questions about how these might apply to your use case, please contact your Customer Success Manager or Support. + +Use the navigation bar on the left to find the current best practices for managing and deploying the Rancher Server. + +For more guidance on best practices, you can consult these resources: + +- [Security](rancher-security.md) +- [Rancher Blog](https://www.suse.com/c/rancherblog/) +- [Rancher Forum](https://forums.rancher.com/) +- [Rancher Users Slack](https://slack.rancher.io/) +- [Rancher Labs YouTube Channel - Online Meetups, Demos, Training, and Webinars](https://www.youtube.com/channel/UCh5Xtp82q8wjijP8npkVTBA/featured) diff --git a/versioned_docs/version-latest/pages-for-subheaders/checklist-for-production-ready-clusters.md b/versioned_docs/version-latest/pages-for-subheaders/checklist-for-production-ready-clusters.md new file mode 100644 index 00000000000..f5816af3c48 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/checklist-for-production-ready-clusters.md @@ -0,0 +1,53 @@ +--- +title: Checklist for Production-Ready Clusters +--- + + + + + +In this section, we recommend best practices for creating the production-ready Kubernetes clusters that will run your apps and services. + +For a list of requirements for your cluster, including the requirements for OS/Docker, hardware, and networking, refer to the section on [node requirements.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) + +This is a shortlist of best practices that we strongly recommend for all production clusters. + +For a full list of all the best practices that we recommend, refer to the [best practices section.](best-practices.md) + +### Node Requirements + +* Make sure your nodes fulfill all of the [node requirements,](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) including the port requirements. + +### Back up etcd + +* Enable etcd snapshots. Verify that snapshots are being created, and run a disaster recovery scenario to verify the snapshots are valid. etcd is the location where the state of your cluster is stored, and losing etcd data means losing your cluster. Make sure you configure recurring snapshots of etcd for your cluster(s), and make sure the snapshots are stored externally (off the node) as well. + +### Cluster Architecture + +* Nodes should have one of the following role configurations: + * `etcd` + * `controlplane` + * `etcd` and `controlplane` + * `worker` (the `worker` role should not be used or added on nodes with the `etcd` or `controlplane` role) +* Have at least three nodes with the role `etcd` to survive losing one node. Increase this count for higher node fault toleration, and spread them across (availability) zones to provide even better fault tolerance. +* Assign two or more nodes the `controlplane` role for master component high availability. +* Assign two or more nodes the `worker` role for workload rescheduling upon node failure. + +For more information on what each role is used for, refer to the [section on roles for nodes in Kubernetes.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes.md) + +For more information about the +number of nodes for each Kubernetes role, refer to the section on [recommended architecture.](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) + +### Logging and Monitoring + +* Configure alerts/notifiers for Kubernetes components (System Service). +* Configure logging for cluster analysis and post-mortems. + +### Reliability + +* Perform load tests on your cluster to verify that its hardware can support your workloads. + +### Networking + +* Minimize network latency. Rancher recommends minimizing latency between the etcd nodes. The default setting for `heartbeat-interval` is `500`, and the default setting for `election-timeout` is `5000`. These [settings for etcd tuning](https://coreos.com/etcd/docs/latest/tuning.html) allow etcd to run in most networks (except really high latency networks). +* Cluster nodes should be located within a single region. Most cloud providers provide multiple availability zones within a region, which can be used to create higher availability for your cluster. Using multiple availability zones is fine for nodes with any role. If you are using [Kubernetes Cloud Provider](set-up-cloud-providers.md) resources, consult the documentation for any restrictions (i.e. zone storage restrictions). diff --git a/versioned_docs/version-latest/pages-for-subheaders/cis-scan-guides.md b/versioned_docs/version-latest/pages-for-subheaders/cis-scan-guides.md new file mode 100644 index 00000000000..e76d47504e6 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/cis-scan-guides.md @@ -0,0 +1,17 @@ +--- +title: CIS Scan Guides +--- + + + + + +- [Install rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark.md) +- [Uninstall rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark.md) +- [Run a Scan](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan.md) +- [Run a Scan Periodically on a Schedule](../how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule.md) +- [Skip Tests](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md) +- [View Reports](../how-to-guides/advanced-user-guides/cis-scan-guides/view-reports.md) +- [Enable Alerting for rancher-cis-benchmark](../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) +- [Configure Alerts for Periodic Scan on a Schedule](../how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule.md) +- [Create a Custom Benchmark Version to Run](../how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/cis-scans.md b/versioned_docs/version-latest/pages-for-subheaders/cis-scans.md new file mode 100644 index 00000000000..d9c5dbecabb --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/cis-scans.md @@ -0,0 +1,122 @@ +--- +title: CIS Scans +--- + + + + + +Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. The CIS scans can run on any Kubernetes cluster, including hosted Kubernetes providers such as EKS, AKS, and GKE. + +The `rancher-cis-benchmark` app leverages kube-bench, an open-source tool from Aqua Security, to check clusters for CIS Kubernetes Benchmark compliance. Also, to generate a cluster-wide report, the application utilizes Sonobuoy for report aggregation. + + +## About the CIS Benchmark + +The Center for Internet Security is a 501(c\)(3) non-profit organization, formed in October 2000, with a mission to "identify, develop, validate, promote, and sustain best practice solutions for cyber defense and build and lead communities to enable an environment of trust in cyberspace". The organization is headquartered in East Greenbush, New York, with members including large corporations, government agencies, and academic institutions. + +CIS Benchmarks are best practices for the secure configuration of a target system. CIS Benchmarks are developed through the generous volunteer efforts of subject matter experts, technology vendors, public and private community members, and the CIS Benchmark Development team. + +[Sign up](https://learn.cisecurity.org/benchmarks) at the CIS website to view the official Benchmark documents. + +## About the Generated Report + +Each scan generates a report can be viewed in the Rancher UI and can be downloaded in CSV format. + +By default, the CIS Benchmark v1.6 is used. + +The Benchmark version is included in the generated report. + +The Benchmark provides recommendations of two types: Automated and Manual. Recommendations marked as Manual in the Benchmark are not included in the generated report. + +Some tests are designated as "Not Applicable." These tests will not be run on any CIS scan because of the way that Rancher provisions RKE clusters. For information on how test results can be audited, and why some tests are designated to be not applicable, refer to Rancher's [self-assessment guide](./rancher-security.md#the-cis-benchmark-and-self-assessment) for the corresponding Kubernetes version. + +The report contains the following information: + +| Column in Report | Description | +|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `id` | The ID number of the CIS Benchmark. | +| `description` | The description of the CIS Benchmark test. | +| `remediation` | What needs to be fixed in order to pass the test. | +| `state` | Indicates if the test passed, failed, was skipped, or was not applicable. | +| `node_type` | The node role, which affects which tests are run on the node. Master tests are run on controlplane nodes, etcd tests are run on etcd nodes, and node tests are run on the worker nodes. | +| `audit` | This is the audit check that `kube-bench` runs for this test. | +| `audit_config` | Any configuration applicable to the audit script. | +| `test_info` | Test-related info as reported by `kube-bench`, if any. | +| `commands` | Test-related commands as reported by `kube-bench`, if any. | +| `config_commands` | Test-related configuration data as reported by `kube-bench`, if any. | +| `actual_value` | The test's actual value, present if reported by `kube-bench`. | +| `expected_result` | The test's expected result, present if reported by `kube-bench`. | + +Refer to [the table in the cluster hardening guide](./rancher-security.md) for information on which versions of Kubernetes, the Benchmark, Rancher, and our cluster hardening guide correspond to each other. Also refer to the hardening guide for configuration files of CIS-compliant clusters and information on remediating failed tests. + +## Test Profiles + +The following profiles are available: + +- Generic CIS 1.6 +- Generic CIS 1.20 +- Generic CIS 1.23 +- RKE permissive 1.6 +- RKE hardened 1.6 +- RKE permissive 1.20 +- RKE hardened 1.20 +- RKE permissive 1.23 +- RKE hardened 1.23 +- RKE2 permissive 1.6 +- RKE2 hardened 1.6 +- RKE2 permissive 1.20 +- RKE2 hardened 1.20 +- RKE2 permissive 1.23 +- RKE2 hardened 1.23 +- K3s permissive 1.6 +- K3s hardened 1.6 +- K3s permissive 1.20 +- K3s hardened 1.20 +- K3s permissive 1.23 +- K3s hardened 1.23 +- AKS +- EKS +- GKE + +You also have the ability to customize a profile by saving a set of tests to skip. + +All profiles will have a set of not applicable tests that will be skipped during the CIS scan. These tests are not applicable based on how a RKE cluster manages Kubernetes. + +There are two types of RKE cluster scan profiles: + +- **Permissive:** This profile has a set of tests that have been will be skipped as these tests will fail on a default RKE Kubernetes cluster. Besides the list of skipped tests, the profile will also not run the not applicable tests. +- **Hardened:** This profile will not skip any tests, except for the non-applicable tests. + +The EKS and GKE cluster scan profiles are based on CIS Benchmark versions that are specific to those types of clusters. + +In order to pass the "Hardened" profile, you will need to follow the steps on the [hardening guide](./rancher-security.md#rancher-hardening-guide) and use the `cluster.yml` defined in the hardening guide to provision a hardened cluster. + +The default profile and the supported CIS benchmark version depends on the type of cluster that will be scanned: + +The `rancher-cis-benchmark` supports the CIS 1.6 Benchmark version. + +- For RKE Kubernetes clusters, the RKE Permissive 1.6 profile is the default. +- EKS and GKE have their own CIS Benchmarks published by `kube-bench`. The corresponding test profiles are used by default for those clusters. +- For RKE2 Kubernetes clusters, the RKE2 Permissive 1.6 profile is the default. +- For cluster types other than RKE, RKE2, EKS and GKE, the Generic CIS 1.5 profile will be used by default. + +## About Skipped and Not Applicable Tests + +For a list of skipped and not applicable tests, refer to [this page](../how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests.md). + +For now, only user-defined skipped tests are marked as skipped in the generated report. + +Any skipped tests that are defined as being skipped by one of the default profiles are marked as not applicable. + +## Roles-based Access Control + +For information about permissions, refer to [this page](../integrations-in-rancher/cis-scans/rbac-for-cis-scans.md) + +## Configuration + +For more information about configuring the custom resources for the scans, profiles, and benchmark versions, refer to [this page](../integrations-in-rancher/cis-scans/configuration-reference.md) + +## How-to Guides + +Please refer to the [CIS Scan Guides](../pages-for-subheaders/cis-scan-guides.md) to learn how to run CIS scans. diff --git a/versioned_docs/version-latest/pages-for-subheaders/cli-with-rancher.md b/versioned_docs/version-latest/pages-for-subheaders/cli-with-rancher.md new file mode 100644 index 00000000000..547d4c50308 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/cli-with-rancher.md @@ -0,0 +1,9 @@ +--- +title: CLI with Rancher +--- + + + + + +Interact with Rancher using command line interface (CLI) tools from your workstation. The following docs will describe the [Rancher CLI](../reference-guides/cli-with-rancher/rancher-cli.md) and [kubectl Utility](../reference-guides/cli-with-rancher/kubectl-utility.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/cloud-marketplace.md b/versioned_docs/version-latest/pages-for-subheaders/cloud-marketplace.md new file mode 100644 index 00000000000..6f90a2c3ff4 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/cloud-marketplace.md @@ -0,0 +1,11 @@ +--- +title: Cloud Marketplace Integration +--- + + + + + +Rancher offers integration with cloud marketplaces to easily purchase support for installations hosted on certain cloud providers. In addition, this integration also provides the ability to generate a supportconfig bundle which can be provided to rancher support. + +This integration only supports AWS. diff --git a/versioned_docs/version-latest/pages-for-subheaders/cluster-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/cluster-configuration.md new file mode 100644 index 00000000000..60e02a8cd40 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/cluster-configuration.md @@ -0,0 +1,33 @@ +--- +title: Cluster Configuration +--- + + + + + +After you provision a Kubernetes cluster using Rancher, you can still edit options and settings for the cluster. + +For information on editing cluster membership, go to [this page.](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) + +### Cluster Configuration References + +The cluster configuration options depend on the type of Kubernetes cluster: + +- [RKE Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [GKE Cluster Configuration](gke-cluster-configuration.md) +- [AKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) + +### Cluster Management Capabilities by Cluster Type + +The options and settings available for an existing cluster change based on the method that you used to provision it. + +The following table summarizes the options and settings available for each cluster type: + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + diff --git a/versioned_docs/version-latest/pages-for-subheaders/configuration-options.md b/versioned_docs/version-latest/pages-for-subheaders/configuration-options.md new file mode 100644 index 00000000000..92a375e948d --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/configuration-options.md @@ -0,0 +1,43 @@ +--- +title: Configuration Options +--- + + + + + +### Egress Support + +By default the Egress gateway is disabled, but can be enabled on install or upgrade through the values.yaml or via the [overlay file](#overlay-file). + +### Enabling Automatic Sidecar Injection + +Automatic sidecar injection is disabled by default. To enable this, set the `sidecarInjectorWebhook.enableNamespacesByDefault=true` in the values.yaml on install or upgrade. This automatically enables Istio sidecar injection into all new namespaces that are deployed. + +### Overlay File + +An Overlay File is designed to support extensive configuration of your Istio installation. It allows you to make changes to any values available in the [IstioOperator API](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/). This will ensure you can customize the default installation to fit any scenario. + +The Overlay File will add configuration on top of the default installation that is provided from the Istio chart installation. This means you do not need to redefine the components that already defined for installation. + +For more information on Overlay Files, refer to the [Istio documentation.](https://istio.io/latest/docs/setup/install/istioctl/#configure-component-settings) + +### Selectors and Scrape Configs + +The Monitoring app sets `prometheus.prometheusSpec.ignoreNamespaceSelectors=false` which enables monitoring across all namespaces by default. This ensures you can view traffic, metrics and graphs for resources deployed in a namespace with `istio-injection=enabled` label. + +If you would like to limit Prometheus to specific namespaces, set `prometheus.prometheusSpec.ignoreNamespaceSelectors=true`. Once you do this, you will need to add additional configuration to continue to monitor your resources. + +For details, refer to [this section.](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) + +### Enable Istio with Pod Security Policies + +Refer to [this section.](../integrations-in-rancher/istio/configuration-options/pod-security-policies.md) + +### Additional Steps for Installing Istio on an RKE2 Cluster + +Refer to [this section.](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md) + +### Additional Steps for Project Network Isolation + +Refer to [this section.](../integrations-in-rancher/istio/configuration-options/project-network-isolation.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md b/versioned_docs/version-latest/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md new file mode 100644 index 00000000000..8662bf782fb --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/configure-microsoft-ad-federation-service-saml.md @@ -0,0 +1,40 @@ +--- +title: Configuring Microsoft Active Directory Federation Service (SAML) +--- + + + + + +If your organization uses Microsoft Active Directory Federation Services (AD FS) for user authentication, you can configure Rancher to allow your users to log in using their AD FS credentials. + +## Prerequisites + +You must have Rancher installed. + +- Obtain your Rancher Server URL. During AD FS configuration, substitute this URL for the `` placeholder. +- You must have a global administrator account on your Rancher installation. + +You must have a [Microsoft AD FS Server](https://docs.microsoft.com/en-us/windows-server/identity/active-directory-federation-services) configured. + +- Obtain your AD FS Server IP/DNS name. During AD FS configuration, substitute this IP/DNS name for the `` placeholder. +- You must have access to add [Relying Party Trusts](https://docs.microsoft.com/en-us/windows-server/identity/ad-fs/operations/create-a-relying-party-trust) on your AD FS Server. + +## Setup Outline + +Setting up Microsoft AD FS with Rancher Server requires configuring AD FS on your Active Directory server, and configuring Rancher to utilize your AD FS server. The following pages serve as guides for setting up Microsoft AD FS authentication on your Rancher installation. + +- [1. Configuring Microsoft AD FS for Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) +- [2. Configuring Rancher for Microsoft AD FS](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs.md) + +:::note SAML Provider Caveats: + +- SAML Protocol does not support search or lookup for users or groups. Therefore, there is no validation on users or groups when adding them to Rancher. +- When adding users, the exact user IDs (i.e. `UID Field`) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +::: + + +### [Next: Configuring Microsoft AD FS for Rancher](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/configure-openldap.md b/versioned_docs/version-latest/pages-for-subheaders/configure-openldap.md new file mode 100644 index 00000000000..9eb5fc7db2a --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/configure-openldap.md @@ -0,0 +1,56 @@ +--- +title: Configuring OpenLDAP +--- + + + + + +If your organization uses LDAP for user authentication, you can configure Rancher to communicate with an OpenLDAP server to authenticate users. This allows Rancher admins to control access to clusters and projects based on users and groups managed externally in the organisation's central user repository, while allowing end-users to authenticate with their LDAP credentials when logging in to the Rancher UI. + +## Prerequisites + +Rancher must be configured with a LDAP bind account (aka service account) to search and retrieve LDAP entries pertaining to users and groups that should have access. It is recommended to not use an administrator account or personal account for this purpose and instead create a dedicated account in OpenLDAP with read-only access to users and groups under the configured search base (see below). + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or not from a recognised certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +## Configure OpenLDAP in Rancher + +Configure the settings for the OpenLDAP server, groups and users. For help filling out each field, refer to the [configuration reference.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md) + +> Before you proceed with the configuration, please familiarise yourself with the concepts of [External Authentication Configuration and Principal Users](authentication-config.md#external-authentication-configuration-and-principal-users). + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **OpenLDAP**. Fill out the **Configure an OpenLDAP server** form. +1. Click **Enable**. + +### Test Authentication + +Once you have completed the configuration, proceed by testing the connection to the OpenLDAP server. Authentication with OpenLDAP will be enabled implicitly if the test is successful. + +:::note + +The OpenLDAP user pertaining to the credentials entered in this step will be mapped to the local principal account and assigned administrator privileges in Rancher. You should therefore make a conscious decision on which LDAP account you use to perform this step. + +::: + +1. Enter the **username** and **password** for the OpenLDAP account that should be mapped to the local principal account. +2. Click **Authenticate With OpenLDAP** to test the OpenLDAP connection and finalise the setup. + +**Result:** + +- OpenLDAP authentication is configured. +- The LDAP user pertaining to the entered credentials is mapped to the local principal (administrative) account. + +:::note + +You will still be able to login using the locally configured `admin` account and password in case of a disruption of LDAP services. + +::: + +## Annex: Troubleshooting + +If you are experiencing issues while testing the connection to the OpenLDAP server, first double-check the credentials entered for the service account as well as the search base configuration. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. diff --git a/versioned_docs/version-latest/pages-for-subheaders/configure-shibboleth-saml.md b/versioned_docs/version-latest/pages-for-subheaders/configure-shibboleth-saml.md new file mode 100644 index 00000000000..ea45e5311b6 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/configure-shibboleth-saml.md @@ -0,0 +1,106 @@ +--- +title: Configuring Shibboleth (SAML) +--- + + + + + +If your organization uses Shibboleth Identity Provider (IdP) for user authentication, you can configure Rancher to allow your users to log in to Rancher using their Shibboleth credentials. + +In this configuration, when Rancher users log in, they will be redirected to the Shibboleth IdP to enter their credentials. After authentication, they will be redirected back to the Rancher UI. + +If you also configure OpenLDAP as the back end to Shibboleth, it will return a SAML assertion to Rancher with user attributes that include groups. Then the authenticated user will be able to access resources in Rancher that their groups have permissions for. + +> The instructions in this section assume that you understand how Rancher, Shibboleth, and OpenLDAP work together. For a more detailed explanation of how it works, refer to [this page.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions.md) + + +## Setting up Shibboleth in Rancher + +### Shibboleth Prerequisites +> +>- You must have a Shibboleth IdP Server configured. +>- Following are the Rancher Service Provider URLs needed for configuration: +Metadata URL: `https:///v1-saml/shibboleth/saml/metadata` +Assertion Consumer Service (ACS) URL: `https:///v1-saml/shibboleth/saml/acs` +>- Export a `metadata.xml` file from your IdP Server. For more information, see the [Shibboleth documentation.](https://wiki.shibboleth.net/confluence/display/SP3/Home) + +### Configure Shibboleth in Rancher + +If your organization uses Shibboleth for user authentication, you can configure Rancher to allow your users to log in using their IdP credentials. + +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Shibboleth**. +1. Complete the **Configure Shibboleth Account** form. Shibboleth IdP lets you specify what data store you want to use. You can either add a database or use an existing ldap server. For example, if you select your Active Directory (AD) server, the examples below describe how you can map AD attributes to fields within Rancher. + + 1. **Display Name Field**: Enter the AD attribute that contains the display name of users (example: `displayName`). + + 1. **User Name Field**: Enter the AD attribute that contains the user name/given name (example: `givenName`). + + 1. **UID Field**: Enter an AD attribute that is unique to every user (example: `sAMAccountName`, `distinguishedName`). + + 1. **Groups Field**: Make entries for managing group memberships (example: `memberOf`). + + 1. **Rancher API Host**: Enter the URL for your Rancher Server. + + 1. **Private Key** and **Certificate**: This is a key-certificate pair to create a secure shell between Rancher and your IdP. + + You can generate one using an openssl command. For example: + + ``` + openssl req -x509 -newkey rsa:2048 -keyout myservice.key -out myservice.cert -days 365 -nodes -subj "/CN=myservice.example.com" + ``` + 1. **IDP-metadata**: The `metadata.xml` file that you exported from your IdP server. + + +1. After you complete the **Configure Shibboleth Account** form, click **Enable**. + + Rancher redirects you to the IdP login page. Enter credentials that authenticate with Shibboleth IdP to validate your Rancher Shibboleth configuration. + + :::note + + You may have to disable your popup blocker to see the IdP login page. + + ::: + +**Result:** Rancher is configured to work with Shibboleth. Your users can now sign into Rancher using their Shibboleth logins. + +### SAML Provider Caveats + +If you configure Shibboleth without OpenLDAP, the following caveats apply due to the fact that SAML Protocol does not support search or lookup for users or groups. + +- There is no validation on users or groups when assigning permissions to them in Rancher. +- When adding users, the exact user IDs (i.e. UID Field) must be entered correctly. As you type the user ID, there will be no search for other user IDs that may match. +- When adding groups, you must select the group from the drop-down that is next to the text box. Rancher assumes that any input from the text box is a user. +- The group drop-down shows only the groups that you are a member of. You will not be able to add groups that you are not a member of. + +To enable searching for groups when assigning permissions in Rancher, you will need to configure a back end for the SAML provider that supports groups, such as OpenLDAP. + +## Setting up OpenLDAP in Rancher + +If you also configure OpenLDAP as the back end to Shibboleth, it will return a SAML assertion to Rancher with user attributes that include groups. Then authenticated users will be able to access resources in Rancher that their groups have permissions for. + +### OpenLDAP Prerequisites + +Rancher must be configured with a LDAP bind account (aka service account) to search and retrieve LDAP entries pertaining to users and groups that should have access. It is recommended to not use an administrator account or personal account for this purpose and instead create a dedicated account in OpenLDAP with read-only access to users and groups under the configured search base (see below). + +> **Using TLS?** +> +> If the certificate used by the OpenLDAP server is self-signed or not from a recognized certificate authority, make sure have at hand the CA certificate (concatenated with any intermediate certificates) in PEM format. You will have to paste in this certificate during the configuration so that Rancher is able to validate the certificate chain. + +### Configure OpenLDAP in Rancher + +Configure the settings for the OpenLDAP server, groups and users. For help filling out each field, refer to the [configuration reference.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference.md) Note that nested group membership is not available for Shibboleth. + +> Before you proceed with the configuration, please familiarise yourself with the concepts of [External Authentication Configuration and Principal Users](authentication-config.md#external-authentication-configuration-and-principal-users). + +1. Log into the Rancher UI using the initial local `admin` account. +1. In the top left corner, click **☰ > Users & Authentication**. +1. In the left navigation menu, click **Auth Provider**. +1. Click **Shibboleth** or, if SAML is already configured, **Edit Config** +1. Under **User and Group Search**, check **Configure an OpenLDAP server** + +## Troubleshooting + +If you are experiencing issues while testing the connection to the OpenLDAP server, first double-check the credentials entered for the service account as well as the search base configuration. You may also inspect the Rancher logs to help pinpointing the problem cause. Debug logs may contain more detailed information about the error. Please refer to [How can I enable debug logging](../faq/technical-items.md#how-can-i-enable-debug-logging) in this documentation. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/create-kubernetes-persistent-storage.md b/versioned_docs/version-latest/pages-for-subheaders/create-kubernetes-persistent-storage.md new file mode 100644 index 00000000000..6bda26af36e --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/create-kubernetes-persistent-storage.md @@ -0,0 +1,78 @@ +--- +title: "Kubernetes Persistent Storage: Volumes and Storage Classes" +description: "Learn about the two ways with which you can create persistent storage in Kubernetes: persistent volumes and storage classes" +--- + + + + + +When deploying an application that needs to retain data, you'll need to create persistent storage. Persistent storage allows you to store application data external from the pod running your application. This storage practice allows you to maintain application data, even if the application's pod fails. + +The documents in this section assume that you understand the Kubernetes concepts of persistent volumes, persistent volume claims, and storage classes. For more information, refer to the section on [how storage works.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage.md) + +### Prerequisites + +To set up persistent storage, the `Manage Volumes` [role](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-role-reference) is required. + +If you are provisioning storage for a cluster hosted in the cloud, the storage and cluster hosts must have the same cloud provider. + +For provisioning new storage with Rancher, the cloud provider must be enabled. For details on enabling cloud providers, refer to [this page.](../pages-for-subheaders/set-up-cloud-providers.md) + +For attaching existing persistent storage to a cluster, the cloud provider does not need to be enabled. + +### Setting up Existing Storage + +The overall workflow for setting up existing storage is as follows: + +1. Set up your persistent storage. This may be storage in an infrastructure provider, or it could be your own storage. +2. Add a persistent volume (PV) that refers to the persistent storage. +3. Add a persistent volume claim (PVC) that refers to the PV. +4. Mount the PVC as a volume in your workload. + +For details and prerequisites, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md) + +### Dynamically Provisioning New Storage in Rancher + +The overall workflow for provisioning new storage is as follows: + +1. Add a StorageClass and configure it to use your storage provider. The StorageClass could refer to storage in an infrastructure provider, or it could refer to your own storage. +2. Add a persistent volume claim (PVC) that refers to the storage class. +3. Mount the PVC as a volume for your workload. + +For details and prerequisites, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md) + +### Longhorn Storage + +[Longhorn](https://longhorn.io/) is a lightweight, reliable and easy-to-use distributed block storage system for Kubernetes. + +Longhorn is free, open source software. Originally developed by Rancher Labs, it is now being developed as a sandbox project of the Cloud Native Computing Foundation. It can be installed on any Kubernetes cluster with Helm, with kubectl, or with the Rancher UI. + +If you have a pool of block storage, Longhorn can help you provide persistent storage to your Kubernetes cluster without relying on cloud providers. For more information about Longhorn features, refer to the [documentation.](https://longhorn.io/docs/latest/what-is-longhorn/) + +Rancher v2.5 simplified the process of installing Longhorn on a Rancher-managed cluster. For more information, see [this page.](../integrations-in-rancher/longhorn.md) + +### Provisioning Storage Examples + +We provide examples of how to provision storage with [NFS,](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) [vSphere,](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) and [Amazon's EBS.](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) + +### GlusterFS Volumes + +In clusters that store data on GlusterFS volumes, you may experience an issue where pods fail to mount volumes after restarting the `kubelet`. For details on preventing this from happening, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes.md) + +### iSCSI Volumes + +In [Rancher Launched Kubernetes clusters](../pages-for-subheaders/launch-kubernetes-with-rancher.md) that store data on iSCSI volumes, you may experience an issue where kubelets fail to automatically connect with iSCSI volumes. For details on resolving this issue, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes.md) + +### hostPath Volumes +Before you create a hostPath volume, you need to set up an [extra_bind](https://rancher.com/docs/rke/latest/en/config-options/services/services-extras/#extra-binds/) in your cluster configuration. This will mount the path as a volume in your kubelets, which can then be used for hostPath volumes in your workloads. + +### Migrating vSphere Cloud Provider from In-tree to Out-of-tree + +Kubernetes is moving away from maintaining cloud providers in-tree. vSphere has an out-of-tree cloud provider that can be used by installing the vSphere cloud provider and cloud storage plugins. + +For instructions on how to migrate from the in-tree vSphere cloud provider to out-of-tree, and manage the existing VMs post migration, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md) + +### Related Links + +- [Kubernetes Documentation: Storage](https://kubernetes.io/docs/concepts/storage/) diff --git a/versioned_docs/version-latest/pages-for-subheaders/custom-resource-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/custom-resource-configuration.md new file mode 100644 index 00000000000..37c2f337d78 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/custom-resource-configuration.md @@ -0,0 +1,12 @@ +--- +title: Custom Resource Configuration +--- + + + + + +The following Custom Resource Definitions are used to configure logging: + +- [Flow and ClusterFlow](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md) +- [Output and ClusterOutput](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/deploy-apps-across-clusters.md b/versioned_docs/version-latest/pages-for-subheaders/deploy-apps-across-clusters.md new file mode 100644 index 00000000000..8089f7a914d --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/deploy-apps-across-clusters.md @@ -0,0 +1,18 @@ +--- +title: Deploying Applications across Clusters +--- + + + + +### Fleet + +Rancher v2.5 introduced Fleet, a new way to deploy applications across clusters. + +Continuous Delivery with Fleet is GitOps at scale. For more information, refer to the [Fleet section.](../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) + +### Multi-cluster Apps + +In Rancher before v2.5, the multi-cluster apps feature was used to deploy applications across clusters. The multi-cluster apps feature is deprecated, but still available in Rancher v2.5. + +Refer to the documentation [here.](../how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/deploy-rancher-manager.md b/versioned_docs/version-latest/pages-for-subheaders/deploy-rancher-manager.md new file mode 100644 index 00000000000..74e282f0832 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/deploy-rancher-manager.md @@ -0,0 +1,23 @@ +--- +title: Deploying Rancher Server +--- + + + + + +Use one of the following guides to deploy and provision Rancher and a Kubernetes cluster in the provider of your choice. + +- [AWS](../getting-started/quick-start-guides/deploy-rancher-manager/aws.md) (uses Terraform) +- [AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md) (uses Amazon EKS) +- [Azure](../getting-started/quick-start-guides/deploy-rancher-manager/azure.md) (uses Terraform) +- [DigitalOcean](../getting-started/quick-start-guides/deploy-rancher-manager/digitalocean.md) (uses Terraform) +- [GCP](../getting-started/quick-start-guides/deploy-rancher-manager/gcp.md) (uses Terraform) +- [Hetzner Cloud](../getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud.md) (uses Terraform) +- [Vagrant](../getting-started/quick-start-guides/deploy-rancher-manager/vagrant.md) +- [Equinix Metal](../getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal.md) +- [Outscale](../getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs.md) (uses Terraform) + +If you prefer, the following guide will take you through the same process in individual steps. Use this if you want to run Rancher in a different provider, on prem, or if you would just like to see how easy it is. + +- [Manual Install](../getting-started/quick-start-guides/deploy-rancher-manager/helm-cli.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/deploy-rancher-workloads.md b/versioned_docs/version-latest/pages-for-subheaders/deploy-rancher-workloads.md new file mode 100644 index 00000000000..b2898cd513b --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/deploy-rancher-workloads.md @@ -0,0 +1,12 @@ +--- +title: Deploying Workloads +--- + + + + + +These guides walk you through the deployment of an application, including how to expose the application for use outside of the cluster. + +- [Workload with Ingress](../getting-started/quick-start-guides/deploy-workloads/workload-ingress.md) +- [Workload with NodePort](../getting-started/quick-start-guides/deploy-workloads/nodeports.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/downstream-cluster-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/downstream-cluster-configuration.md new file mode 100644 index 00000000000..b9fbad0b966 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/downstream-cluster-configuration.md @@ -0,0 +1,9 @@ +--- +title: Downstream Cluster Configuration +--- + + + + + +The following docs will discuss [node template configuration](./node-template-configuration.md) and [machine configuration](./machine-configuration.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/enable-experimental-features.md b/versioned_docs/version-latest/pages-for-subheaders/enable-experimental-features.md new file mode 100644 index 00000000000..0e5ad863608 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/enable-experimental-features.md @@ -0,0 +1,126 @@ +--- +title: Enabling Experimental Features +--- + + + + + +Rancher includes some features that are experimental and disabled by default. You might want to enable these features, for example, if you decide that the benefits of using an [unsupported storage type](../how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers.md) outweighs the risk of using an untested feature. Feature flags were introduced to allow you to try these features that are not enabled by default. + +The features can be enabled in three ways: + +- [Enable features when starting Rancher.](#enabling-features-when-starting-rancher) When installing Rancher with a CLI, you can use a feature flag to enable a feature by default. +- [Enable features from the Rancher UI](#enabling-features-with-the-rancher-ui) by going to the **Settings** page. +- [Enable features with the Rancher API](#enabling-features-with-the-rancher-api) after installing Rancher. + +Each feature has two values: + +- A default value, which can be configured with a flag or environment variable from the command line +- A set value, which can be configured with the Rancher API or UI + +If no value has been set, Rancher uses the default value. + +Because the API sets the actual value and the command line sets the default value, that means that if you enable or disable a feature with the API or UI, it will override any value set with the command line. + +For example, if you install Rancher, then set a feature flag to true with the Rancher API, then upgrade Rancher with a command that sets the feature flag to false, the default value will still be false, but the feature will still be enabled because it was set with the Rancher API. If you then deleted the set value (true) with the Rancher API, setting it to NULL, the default value (false) would take effect. See the [feature flags page](../getting-started/installation-and-upgrade/installation-references/feature-flags.md) for more information. + +## Enabling Features when Starting Rancher + +When you install Rancher, enable the feature you want with a feature flag. The command is different depending on whether you are installing Rancher on a single node or if you are doing a Kubernetes Installation of Rancher. + +### Enabling Features for Kubernetes Installs + +:::note + +Values set from the Rancher API will override the value passed in through the command line. + +::: + +When installing Rancher with a Helm chart, use the `--set` option. In the below example, two features are enabled by passing the feature flag names in a comma separated list: + +For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +:::note + +If you are installing an alpha version, Helm requires adding the `--devel` option to the command. + +::: + +### Enabling Features for Air Gap Installs + +To perform an [air gap installation of Rancher](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md), add a Helm chart repository and download a Helm chart, then install Rancher with Helm. + +When you install the Helm chart, you should pass in feature flag names in a comma separated list, as in the following example: + +``` +helm install rancher ./rancher-.tgz \ + --namespace cattle-system \ + --set hostname= \ + --set rancherImage=/rancher/rancher \ + --set ingress.tls.source=secret \ + --set systemDefaultRegistry= \ # Set a default private registry to be used in Rancher + --set useBundledSystemChart=true # Use the packaged Rancher system charts + --set 'extraEnv[0].name=CATTLE_FEATURES' + --set 'extraEnv[0].value==true,=true' +``` + +### Enabling Features for Docker Installs + +When installing Rancher with Docker, use the `--features` option. In the below example, two features are enabled by passing the feature flag names in a comma separated list: + +``` +docker run -d -p 80:80 -p 443:443 \ + --restart=unless-stopped \ + rancher/rancher:rancher-latest \ + --features==true,=true +``` + + +## Enabling Features with the Rancher UI + +1. In the upper left corner, click **☰ > Global Settings**. +1. Click **Feature Flags**. +1. To enable a feature, go to the disabled feature you want to enable and click **â‹® > Activate**. + +**Result:** The feature is enabled. + +### Disabling Features with the Rancher UI + +1. In the upper left corner, click **☰ > Global Settings**. +1. Click **Feature Flags**. You will see a list of experimental features. +1. To disable a feature, go to the enabled feature you want to disable and click **â‹® > Deactivate**. + +**Result:** The feature is disabled. + +## Enabling Features with the Rancher API + +1. Go to `/v3/features`. +1. In the `data` section, you will see an array containing all of the features that can be turned on with feature flags. The name of the feature is in the `id` field. Click the name of the feature you want to enable. +1. In the upper left corner of the screen, under **Operations,** click **Edit**. +1. In the **Value** drop-down menu, click **True**. +1. Click **Show Request**. +1. Click **Send Request**. +1. Click **Close**. + +**Result:** The feature is enabled. + +### Disabling Features with the Rancher API + +1. Go to `/v3/features`. +1. In the `data` section, you will see an array containing all of the features that can be turned on with feature flags. The name of the feature is in the `id` field. Click the name of the feature you want to enable. +1. In the upper left corner of the screen, under **Operations,** click **Edit**. +1. In the **Value** drop-down menu, click **False**. +1. Click **Show Request**. +1. Click **Send Request**. +1. Click **Close**. + +**Result:** The feature is disabled. diff --git a/versioned_docs/version-latest/pages-for-subheaders/fleet-gitops-at-scale.md b/versioned_docs/version-latest/pages-for-subheaders/fleet-gitops-at-scale.md new file mode 100644 index 00000000000..54e38cc48eb --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/fleet-gitops-at-scale.md @@ -0,0 +1,68 @@ +--- +title: Continuous Delivery with Fleet +--- + + + + + +Continuous Delivery with Fleet is GitOps at scale. Fleet is designed to manage up to a million clusters. It’s also lightweight enough that it works great for a [single cluster](https://fleet.rancher.io/installation#default-install) too, but it really shines when you get to a [large scale](https://fleet.rancher.io/installation#configuration-for-multi-cluster). By large scale we mean either a lot of clusters, a lot of deployments, or a lot of teams in a single organization. + +Fleet is a separate project from Rancher, and can be installed on any Kubernetes cluster with Helm. + + +## Architecture + +For information about how Fleet works, see [this page](../integrations-in-rancher/fleet-gitops-at-scale/architecture.md). + +## Accessing Fleet in the Rancher UI + +Fleet comes preinstalled in Rancher and is managed by the **Continuous Delivery** option in the Rancher UI. For additional information on Continuous Delivery and other Fleet troubleshooting tips, refer [here](https://fleet.rancher.io/troubleshooting). + +Users can leverage continuous delivery to deploy their applications to the Kubernetes clusters in the git repository without any manual operation by following **gitops** practice. + +Follow the steps below to access Continuous Delivery in the Rancher UI: + +1. Click **☰ > Continous Delivery**. + +1. Select your namespace at the top of the menu, noting the following: + + - By default, **fleet-default** is selected which includes all downstream clusters that are registered through Rancher. + + - You may switch to **fleet-local**, which only contains the **local** cluster, or you may create your own workspace to which you may assign and move clusters. + + - You can then manage clusters by clicking on **Clusters** on the left navigation bar. + +1. Click on **Gitrepos** on the left navigation bar to deploy the gitrepo into your clusters in the current workspace. + +1. Select your [git repository](https://fleet.rancher.io/gitrepo-add) and [target clusters/cluster group](https://fleet.rancher.io/gitrepo-targets). You can also create the cluster group in the UI by clicking on **Cluster Groups** from the left navigation bar. + +1. Once the gitrepo is deployed, you can monitor the application through the Rancher UI. + +## Windows Support + +For details on support for clusters with Windows nodes, see [this page](../integrations-in-rancher/fleet-gitops-at-scale/windows-support.md). + +## GitHub Repository + +The Fleet Helm charts are available [here](https://github.com/rancher/fleet/releases). + +## Using Fleet Behind a Proxy + +For details on using Fleet behind a proxy, see [this page](../integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy.md). + +## Helm Chart Dependencies + +In order for Helm charts with dependencies to deploy successfully, you must run a manual command (as listed below), as it is up to the user to fulfill the dependency list. If you do not do this and proceed to clone your repository and run `helm install`, your installation will fail because the dependencies will be missing. + +The Helm chart in the git repository must include its dependencies in the charts subdirectory. You must either manually run `helm dependencies update $chart` OR run `helm dependencies build $chart` locally, then commit the complete charts directory to your git repository. Note that you will update your commands with the applicable parameters + +## Troubleshooting + +- **Known Issue**: clientSecretName and helmSecretName secrets for Fleet gitrepos are not included in the backup nor restore created by the [backup-restore-operator](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher.md#1-install-the-rancher-backup-operator). We will update the community once a permanent solution is in place. + +- **Temporary Workaround**: By default, user-defined secrets are not backed up in Fleet. It is necessary to recreate secrets if performing a disaster recovery restore or migration of Rancher into a fresh cluster. To modify resourceSet to include extra resources you want to backup, refer to docs [here](https://github.com/rancher/backup-restore-operator#user-flow). + +## Documentation + +The Fleet documentation is at https://fleet.rancher.io/. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/gke-cluster-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/gke-cluster-configuration.md new file mode 100644 index 00000000000..de9a1638a9c --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/gke-cluster-configuration.md @@ -0,0 +1,325 @@ +--- +title: GKE Cluster Configuration Reference +--- + + + + + +## Changes in Rancher v2.6 + +- Support for additional configuration options: + - Project network isolation + - Network tags + +## Cluster Location + +| Value | Description | +|--------|--------------| +| Location Type | Zonal or Regional. With GKE, you can create a cluster tailored to the availability requirements of your workload and your budget. By default, a cluster's nodes run in a single compute zone. When multiple zones are selected, the cluster's nodes will span multiple compute zones, while the controlplane is located in a single zone. Regional clusters increase the availability of the controlplane as well. For help choosing the type of cluster availability, refer to [these docs.](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#choosing_a_regional_or_zonal_control_plane) | +| Zone | Each region in Compute engine contains a number of zones. For more information about available regions and zones, refer to [these docs.](https://cloud.google.com/compute/docs/regions-zones#available) | +| Additional Zones | For zonal clusters, you can select additional zones to create a [multi-zone cluster.](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#multi-zonal_clusters) | +| Region | For [regional clusters,](https://cloud.google.com/kubernetes-engine/docs/concepts/types-of-clusters#regional_clusters) you can select a region. For more information about available regions and zones, refer to [this section](https://cloud.google.com/compute/docs/regions-zones#available). The first part of each zone name is the name of the region. | + +## Cluster Options + +### Kubernetes Version + +_Mutable: yes_ + +For more information on GKE Kubernetes versions, refer to [these docs.](https://cloud.google.com/kubernetes-engine/versioning) + +### Container Address Range + +_Mutable: no_ + +The IP address range for pods in the cluster. Must be a valid CIDR range, e.g. 10.42.0.0/16. If not specified, a random range is automatically chosen from 10.0.0.0/8 and will exclude ranges already allocated to VMs, other clusters, or routes. Automatically chosen ranges may conflict with reserved IP addresses, dynamic routes, or routes within VPCs peering with the cluster. + +### Network + +_Mutable: no_ + +The Compute Engine Network that the cluster connects to. Routes and firewalls will be created using this network. If using [Shared VPCs](https://cloud.google.com/vpc/docs/shared-vpc), the VPC networks that are shared to your project will appear here. will be available to select in this field. For more information, refer to [this page](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets). + +### Node Subnet / Subnet + +_Mutable: no_ + +The Compute Engine subnetwork that the cluster connects to. This subnetwork must belong to the network specified in the **Network** field. Select an existing subnetwork, or select "Auto Create Subnetwork" to have one automatically created. If not using an existing network, **Subnetwork Name** is required to generate one. If using [Shared VPCs](https://cloud.google.com/vpc/docs/shared-vpc), the VPC subnets that are shared to your project will appear here. If using a Shared VPC network, you cannot select "Auto Create Subnetwork". For more information, refer to [this page.](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets) + +### Subnetwork Name + +_Mutable: no_ + +Automatically create a subnetwork with the provided name. Required if "Auto Create Subnetwork" is selected for **Node Subnet** or **Subnet**. For more information on subnetworks, refer to [this page.](https://cloud.google.com/vpc/docs/vpc#vpc_networks_and_subnets) + +### Ip Aliases + +_Mutable: no_ + +Enable [alias IPs](https://cloud.google.com/vpc/docs/alias-ip). This enables VPC-native traffic routing. Required if using [Shared VPCs](https://cloud.google.com/vpc/docs/shared-vpc). + +### Network Policy + +_Mutable: yes_ + +Enable network policy enforcement on the cluster. A network policy defines the level of communication that can occur between pods and services in the cluster. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/how-to/network-policy) + +### Project Network Isolation + +_Mutable: yes_ + +choose whether to enable or disable inter-project communication. Note that enabling Project Network Isolation will automatically enable Network Policy and Network Policy Config, but not vice versa. + +### Node Ipv4 CIDR Block + +_Mutable: no_ + +The IP address range of the instance IPs in this cluster. Can be set if "Auto Create Subnetwork" is selected for **Node Subnet** or **Subnet**. Must be a valid CIDR range, e.g. 10.96.0.0/14. For more information on how to determine the IP address range, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing) + +### Cluster Secondary Range Name + +_Mutable: no_ + +The name of an existing secondary range for Pod IP addresses. If selected, **Cluster Pod Address Range** will automatically be populated. Required if using a Shared VPC network. + +### Cluster Pod Address Range + +_Mutable: no_ + +The IP address range assigned to pods in the cluster. Must be a valid CIDR range, e.g. 10.96.0.0/11. If not provided, will be created automatically. Must be provided if using a Shared VPC network. For more information on how to determine the IP address range for your pods, refer to [this section.](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_pods) + +### Services Secondary Range Name + +_Mutable: no_ + +The name of an existing secondary range for service IP addresses. If selected, **Service Address Range** will be automatically populated. Required if using a Shared VPC network. + +### Service Address Range + +_Mutable: no_ + +The address range assigned to the services in the cluster. Must be a valid CIDR range, e.g. 10.94.0.0/18. If not provided, will be created automatically. Must be provided if using a Shared VPC network. For more information on how to determine the IP address range for your services, refer to [this section.](https://cloud.google.com/kubernetes-engine/docs/concepts/alias-ips#cluster_sizing_secondary_range_svcs) + +### Private Cluster + +_Mutable: no_ + +:::caution + +Private clusters require additional planning and configuration outside of Rancher. Refer to the [private cluster guide](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md). + +::: + +Assign nodes only internal IP addresses. Private cluster nodes cannot access the public internet unless additional networking steps are taken in GCP. + +### Enable Private Endpoint + +:::caution + +Private clusters require additional planning and configuration outside of Rancher. Refer to the [private cluster guide](../reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md). + +::: + +_Mutable: no_ + +Locks down external access to the control plane endpoint. Only available if **Private Cluster** is also selected. If selected, and if Rancher does not have direct access to the Virtual Private Cloud network the cluster is running in, Rancher will provide a registration command to run on the cluster to enable Rancher to connect to it. + +### Master IPV4 CIDR Block + +_Mutable: no_ + +The IP range for the control plane VPC. + +### Master Authorized Network + +_Mutable: yes_ + +Enable control plane authorized networks to block untrusted non-GCP source IPs from accessing the Kubernetes master through HTTPS. If selected, additional authorized networks may be added. If the cluster is created with a public endpoint, this option is useful for locking down access to the public endpoint to only certain networks, such as the network where your Rancher service is running. If the cluster only has a private endpoint, this setting is required. + +## Additional Options + +### Cluster Addons + +Additional Kubernetes cluster components. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1/projects.locations.clusters#Cluster.AddonsConfig) + +#### Horizontal Pod Autoscaling + +_Mutable: yes_ + +The Horizontal Pod Autoscaler changes the shape of your Kubernetes workload by automatically increasing or decreasing the number of Pods in response to the workload's CPU or memory consumption, or in response to custom metrics reported from within Kubernetes or external metrics from sources outside of your cluster. For more information, see [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/horizontalpodautoscaler) + +#### HTTP (L7) Load Balancing + +_Mutable: yes_ + +HTTP (L7) Load Balancing distributes HTTP and HTTPS traffic to backends hosted on GKE. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/tutorials/http-balancer) + +#### Network Policy Config (master only) + +_Mutable: yes_ + +Configuration for NetworkPolicy. This only tracks whether the addon is enabled or not on the master, it does not track whether network policy is enabled for the nodes. + +### Cluster Features (Alpha Features) + +_Mutable: no_ + +Turns on all Kubernetes alpha API groups and features for the cluster. When enabled, the cluster cannot be upgraded and will be deleted automatically after 30 days. Alpha clusters are not recommended for production use as they are not covered by the GKE SLA. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/alpha-clusters) + +### Logging Service + +_Mutable: yes_ + +The logging service the cluster uses to write logs. Use either [Cloud Logging](https://cloud.google.com/logging) or no logging service in which case no logs are exported from the cluster. + +### Monitoring Service + +_Mutable: yes_ + +The monitoring service the cluster uses to write metrics. Use either [Cloud Monitoring](https://cloud.google.com/monitoring) or monitoring service in which case no metrics are exported from the cluster. + + +### Maintenance Window + +_Mutable: yes_ + +Set the start time for a 4 hour maintenance window. The time is specified in the UTC time zone using the HH:MM format. For more information, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/maintenance-windows-and-exclusions) + +## Node Pools + +In this section, enter details describing the configuration of each node in the node pool. + +### Kubernetes Version + +_Mutable: yes_ + +The Kubernetes version for each node in the node pool. For more information on GKE Kubernetes versions, refer to [these docs.](https://cloud.google.com/kubernetes-engine/versioning) + +### Image Type + +_Mutable: yes_ + +The node operating system image. For more information for the node image options that GKE offers for each OS, refer to [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/node-images#available_node_images) + +:::note + +The default option is "Container-Optimized OS with Docker". The read-only filesystem on GCP's Container-Optimized OS is not compatible with the [legacy logging](/versioned_docs/version-2.0-2.4/pages-for-subheaders/cluster-logging.md) implementation in Rancher. If you need to use the legacy logging feature, select "Ubuntu with Docker" or "Ubuntu with Containerd". The [current logging feature](logging.md) is compatible with the Container-Optimized OS image. + +::: + +:::note + +If selecting "Windows Long Term Service Channel" or "Windows Semi-Annual Channel" for the node pool image type, you must also add at least one Container-Optimized OS or Ubuntu node pool. + +::: + +### Machine Type + +_Mutable: no_ + +The virtualized hardware resources available to node instances. For more information on Google Cloud machine types, refer to [this page.](https://cloud.google.com/compute/docs/machine-types#machine_types) + +### Root Disk Type + +_Mutable: no_ + +Standard persistent disks are backed by standard hard disk drives (HDD), while SSD persistent disks are backed by solid state drives (SSD). For more information, refer to [this section.](https://cloud.google.com/compute/docs/disks) + +### Local SSD Disks + +_Mutable: no_ + +Configure each node's local SSD disk storage in GB. Local SSDs are physically attached to the server that hosts your VM instance. Local SSDs have higher throughput and lower latency than standard persistent disks or SSD persistent disks. The data that you store on a local SSD persists only until the instance is stopped or deleted. For more information, see [this section.](https://cloud.google.com/compute/docs/disks#localssds) + +### Preemptible nodes (beta) + +_Mutable: no_ + +Preemptible nodes, also called preemptible VMs, are Compute Engine VM instances that last a maximum of 24 hours in general, and provide no availability guarantees. For more information, see [this page.](https://cloud.google.com/kubernetes-engine/docs/how-to/preemptible-vms) + +### Taints + +_Mutable: no_ + +When you apply a taint to a node, only Pods that tolerate the taint are allowed to run on the node. In a GKE cluster, you can apply a taint to a node pool, which applies the taint to all nodes in the pool. + +### Node Labels + +_Mutable: no_ + +You can apply labels to the node pool, which applies the labels to all nodes in the pool. + +Invalid labels can prevent upgrades or can prevent Rancher from starting. For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +### Network Tags + +_Mutable: no_ + +You can add network tags to the node pool to make firewall rules and routes between subnets. Tags will apply to all nodes in the pool. + +For details on tag syntax and requirements, see the [Kubernetes documentation](https://cloud.google.com/vpc/docs/add-remove-network-tags). + +## Group Details + +In this section, enter details describing the node pool. + +### Name + +_Mutable: no_ + +Enter a name for the node pool. + +### Initial Node Count + +_Mutable: yes_ + +Integer for the starting number of nodes in the node pool. + +### Max Pod Per Node + +_Mutable: no_ + +GKE has a hard limit of 110 Pods per node. For more information on the Kubernetes limits, see [this section.](https://cloud.google.com/kubernetes-engine/docs/best-practices/scalability#dimension_limits) + +### Autoscaling + +_Mutable: yes_ + +Node pool autoscaling dynamically creates or deletes nodes based on the demands of your workload. For more information, see [this page.](https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-autoscaler) + +### Auto Repair + +_Mutable: yes_ + +GKE's node auto-repair feature helps you keep the nodes in your cluster in a healthy, running state. When enabled, GKE makes periodic checks on the health state of each node in your cluster. If a node fails consecutive health checks over an extended time period, GKE initiates a repair process for that node. For more information, see the section on [auto-repairing nodes.](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-repair) + +### Auto Upgrade + +_Mutable: yes_ + +When enabled, the auto-upgrade feature keeps the nodes in your cluster up-to-date with the cluster control plane (master) version when your control plane is [updated on your behalf.](https://cloud.google.com/kubernetes-engine/upgrades#automatic_cp_upgrades) For more information about auto-upgrading nodes, see [this page.](https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades) + +### Access Scopes + +_Mutable: no_ + +Access scopes are the legacy method of specifying permissions for your nodes. + +- **Allow default access:** The default access for new clusters is the [Compute Engine default service account.](https://cloud.google.com/compute/docs/access/service-accounts?hl=en_US#default_service_account) +- **Allow full access to all Cloud APIs:** Generally, you can just set the cloud-platform access scope to allow full access to all Cloud APIs, then grant the service account only relevant IAM roles. The combination of access scopes granted to the virtual machine instance and the IAM roles granted to the service account determines the amount of access the service account has for that instance. +- **Set access for each API:** Alternatively, you can choose to set specific scopes that permit access to the particular API methods that the service will call. + +For more information, see the [section about enabling service accounts for a VM.](https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances) + + +### Configuring the Refresh Interval + +The refresh interval can be configured through the setting "gke-refresh", which is an integer representing seconds. + +The default value is 300 seconds. + +The syncing interval can be changed by running `kubectl edit setting gke-refresh`. + +The shorter the refresh window, the less likely any race conditions will occur, but it does increase the likelihood of encountering request limits that may be in place for GCP APIs. + diff --git a/versioned_docs/version-latest/pages-for-subheaders/helm-charts-in-rancher.md b/versioned_docs/version-latest/pages-for-subheaders/helm-charts-in-rancher.md new file mode 100644 index 00000000000..f724ce6f0cf --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/helm-charts-in-rancher.md @@ -0,0 +1,156 @@ +--- +title: Helm Charts in Rancher +--- + + + + + +In this section, you'll learn how to manage Helm chart repositories and applications in Rancher. Helm chart repositories are managed using **Apps**. It uses a catalog-like system to import bundles of charts from repositories and then uses those charts to either deploy custom Helm applications or Rancher's tools such as Monitoring or Istio. Rancher tools come as pre-loaded repositories which deploy as standalone Helm charts. Any additional repositories are only added to the current cluster. + +### Versioning Scheme + +The Rancher feature charts versioning scheme is centered around the major version of the charts and the `+up` annotation for upstream charts, where applicable. + +**Major Version:** The major version of the charts is tied to Rancher minor versions. When you upgrade to a new Rancher minor version, you should ensure that all of your **Apps** charts are also upgraded to the correct release line for the chart. + +**Feature Charts:** + +| **Name** | **Supported Minimum Version** | **Supported Maximum Version** | +| ---------------- | ------------ | ------------ | +| external-ip-webhook | 100.0.0+up1.0.0 | 100.0.1+up1.0.1 | +| harvester-cloud-provider | 100.0.2+up0.1.12 | 100.0.2+up0.1.12 | +| harvester-csi-driver | 100.0.2+up0.1.11 | 100.0.2+up0.1.11 | +| neuvector | 100.0.0+up2.2.0 | 100.0.0+up2.2.0 | +| rancher-alerting-drivers | 100.0.0 | 100.0.2 | +| rancher-backup | 2.0.1 | 2.1.2 | +| rancher-cis-benchmark | 2.0.1 | 2.0.4 | +| rancher-gatekeeper | 100.0.0+up3.6.0 | 100.1.0+up3.7.1 | +| rancher-istio | 100.0.0+up1.10.4 | 100.3.0+up1.13.3 | +| rancher-logging | 100.0.0+up3.12.0 | 100.1.2+up3.17.4 | +| rancher-longhorn | 100.0.0+up1.1.2 | 100.1.2+up1.2.4 | +| rancher-monitoring | 100.0.0+up16.6.0 | 100.1.2+up19.0.3 | +| rancher-sriov (experimental) | 100.0.0+up0.1.0 | 100.0.3+up0.1.0 | +| rancher-vsphere-cpi | 100.3.0+up1.2.1 | 100.3.0+up1.2.1 | +| rancher-vsphere-csi | 100.3.0+up2.5.1-rancher1 | 100.3.0+up2.5.1-rancher1 | +| rancher-wins-upgrader | 0.0.100 | 100.0.1+up0.0.1 | + +
+**Charts based on upstream:** For charts that are based on upstreams, the +up annotation should inform you of what upstream version the Rancher chart is tracking. Check the upstream version compatibility with Rancher during upgrades also. + +- As an example, `100.x.x+up16.6.0` for Monitoring tracks upstream kube-prometheus-stack `16.6.0` with some Rancher patches added to it. + +- On upgrades, ensure that you are not downgrading the version of the chart that you are using. For example, if you are using a version of Monitoring > `16.6.0` in Rancher 2.5, you should not upgrade to `100.x.x+up16.6.0`. Instead, you should upgrade to the appropriate version in the next release. + +### Prerelease Versions + +Prereleases adhere to [the specification](https://semver.org/#spec-item-9) defined by [Semantic Versioning 2.0.0](https://semver.org/). For example, a Helm chart with a version of `0.1.3-dev.12ab4f` is considered a prerelease. Prerelease versions are not displayed by default and must be configured to do so. + +To display prerelease versions: + +1. Click on your user avatar in the upper right corner. +1. Click **Preferences**. +1. Under **Helm Charts**, select **Include Prerelease Versions**. + +### Charts + +From the top-left menu select _"Apps"_ and you will be taken to the Charts page. + +The charts page contains all Rancher, Partner, and Custom Charts. + +* Rancher tools such as Logging or Monitoring are included under the Rancher label +* Partner charts reside under the Partners label +* Custom charts will show up under the name of the repository + +All three types are deployed and managed in the same way. + +:::note + +Apps managed by the Cluster Manager (the global view in the legacy Rancher UI) should continue to be managed only by the Cluster Manager, and apps managed with Apps in the new UI must be managed only by Apps. + +::: + +### Repositories + +From the left sidebar select _"Repositories"_. + +These items represent helm repositories, and can be either traditional helm endpoints which have an index.yaml, or git repositories which will be cloned and can point to a specific branch. In order to use custom charts, simply add your repository here and they will become available in the Charts tab under the name of the repository. + +To add a private CA for Helm Chart repositories: + +- **HTTP-based chart repositories**: You must add a base64 encoded copy of the CA certificate in DER format to the spec.caBundle field of the chart repo, such as `openssl x509 -outform der -in ca.pem | base64 -w0`. Click **Edit YAML** for the chart repo and set, as in the following example:
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + + +- **Git-based chart repositories**: You must add a base64 encoded copy of the CA certificate in DER format to the spec.caBundle field of the chart repo, such as `openssl x509 -outform der -in ca.pem | base64 -w0`. Click **Edit YAML** for the chart repo and set, as in the following example:
+ ``` + [...] + spec: + caBundle: + MIIFXzCCA0egAwIBAgIUWNy8WrvSkgNzV0zdWRP79j9cVcEwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRQwEgYDVQQKDAtNeU9yZywgSW5jLjENMAsGA1UEAwwEcm9vdDAeFw0yMTEyMTQwODMyMTdaFw0yNDEwMDMwODMyMT + ... + nDxZ/tNXt/WPJr/PgEB3hQdInDWYMg7vGO0Oz00G5kWg0sJ0ZTSoA10ZwdjIdGEeKlj1NlPyAqpQ+uDnmx6DW+zqfYtLnc/g6GuLLVPamraqN+gyU8CHwAWPNjZonFN9Vpg0PIk1I2zuOc4EHifoTAXSpnjfzfyAxCaZsnTptimlPFJJqAMj+FfDArGmr4= + [...] + ``` + +:::note Helm chart repositories with authentication + +The Repo.Spec contains a `disableSameOriginCheck` that allows users to bypass the same origin checks, sending the repository Authentication information as a Basic Auth Header with all API calls. This is not recommended but can be used as a temporary solution in cases of non-standard Helm chart repositories such as those that have redirects to a different origin URL. + +To use this feature for an existing Helm chart repository, click ⋮ > Edit YAML. On the `spec` portion of the YAML file, add `disableSameOriginCheck` and set it to `true`. + +```yaml +[...] +spec: + disableSameOriginCheck: true +[...] +``` + +::: + +### Helm Compatibility + +Only Helm 3 compatible charts are supported. + + +### Deployment and Upgrades + +From the _"Charts"_ tab select a Chart to install. Rancher and Partner charts may have extra configurations available through custom pages or questions.yaml files, but all chart installations can modify the values.yaml and other basic settings. Once you click install, a Helm operation job is deployed, and the console for the job is displayed. + +To view all recent changes, go to the _"Recent Operations"_ tab. From there you can view the call that was made, conditions, events, and logs. + +After installing a chart, you can find it in the _"Installed Apps"_ tab. In this section you can upgrade or delete the installation, and see further details. When choosing to upgrade, the form and values presented will be the same as installation. + +Most Rancher tools have additional pages located in the toolbar below the _"Apps"_ section to help manage and use the features. These pages include links to dashboards, forms to easily add Custom Resources, and additional information. + +:::caution + +If you are upgrading your chart using _"Customize Helm options before upgrade"_ , please be aware that using the _"--force"_ option may result in errors if your chart has immutable fields. This is because some objects in Kubernetes cannot be changed once they are created. To ensure you do not get this error you can: + + * use the default upgrade option ( i.e do not use _"--force"_ option ) + * uninstall the existing chart and install the upgraded chart + * delete the resources with immutable fields from the cluster before performing the _"--force"_ upgrade + +::: + +#### Legacy Apps + +The upgrade button has been removed for legacy apps from the **Apps > Installed Apps** page. + +If you have a legacy app installed and want to upgrade it: + +- The legacy [feature flag](enable-experimental-features.md) must be turned on (if it's not turned on automatically because of having a legacy app before upgrading) +- You can upgrade the app from cluster explorer, from the left nav section **Legacy > Project > Apps** +- For multi-cluster apps, you can go to **≡ > Multi-cluster Apps** and upgrade the app from there + +### Limitations + +[Dashboard apps or Rancher feature charts](helm-charts-in-rancher.md) **cannot** be installed using the Rancher CLI. diff --git a/versioned_docs/version-latest/pages-for-subheaders/horizontal-pod-autoscaler.md b/versioned_docs/version-latest/pages-for-subheaders/horizontal-pod-autoscaler.md new file mode 100644 index 00000000000..d18ce147d06 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/horizontal-pod-autoscaler.md @@ -0,0 +1,30 @@ +--- +title: The Horizontal Pod Autoscaler +description: Learn about the horizontal pod autoscaler (HPA). How to manage HPAs and how to test them with a service deployment +--- + + + + + +The [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) (HPA) is a Kubernetes feature that allows you to configure your cluster to automatically scale the services it's running up or down. + +Rancher provides some additional features to help manage HPAs, depending on the version of Rancher. + +You can create, manage, and delete HPAs using the Rancher UI. It only supports HPA in the `autoscaling/v2beta2` API. + +## Managing HPAs + +The way that you manage HPAs is different based on your version of the Kubernetes API: + +- **For Kubernetes API version autoscaling/V2beta1:** This version of the Kubernetes API lets you autoscale your pods based on the CPU and memory utilization of your application. +- **For Kubernetes API Version autoscaling/V2beta2:** This version of the Kubernetes API lets you autoscale your pods based on CPU and memory utilization, in addition to custom metrics. + +You can create, manage, and delete HPAs using the Rancher UI. From the Rancher UI you can configure the HPA to scale based on CPU and memory utilization. For more information, refer to [Managing HPAs with the Rancher UI](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md). To scale the HPA based on custom metrics, you still need to use `kubectl`. For more information, refer to [Configuring HPA to Scale Using Custom Metrics with Prometheus](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl.md#configuring-hpa-to-scale-using-custom-metrics-with-prometheus). + +Clusters created in Rancher v2.0.7 and higher automatically have all the requirements needed (metrics-server and Kubernetes cluster configuration) to use HPA. +## Testing HPAs with a Service Deployment + +You can see your HPA's current number of replicas by going to your project and clicking **Resources > HPA**. For more information, refer to [Get HPA Metrics and Status](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui.md). + +You can also use `kubectl` to get the status of HPAs that you test with your load testing tool. For more information, refer to [Testing HPAs with kubectl](../how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl.md). diff --git a/versioned_docs/version-latest/pages-for-subheaders/infrastructure-setup.md b/versioned_docs/version-latest/pages-for-subheaders/infrastructure-setup.md new file mode 100644 index 00000000000..fabdc72e975 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/infrastructure-setup.md @@ -0,0 +1,12 @@ +--- +title: Don't have infrastructure for your Kubernetes cluster? Try one of these tutorials. +--- + + + + + +To set up infrastructure for a high-availability K3s Kubernetes cluster with an external DB, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md) + + +To set up infrastructure for a high-availability RKE Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/install-cluster-autoscaler.md b/versioned_docs/version-latest/pages-for-subheaders/install-cluster-autoscaler.md new file mode 100644 index 00000000000..8b749aae5ee --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/install-cluster-autoscaler.md @@ -0,0 +1,28 @@ +--- +title: Cluster Autoscaler +--- + + + + + +In this section, you'll learn how to install and use the [Kubernetes cluster-autoscaler](https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/) on Rancher custom clusters using AWS EC2 Auto Scaling Groups. + +The cluster autoscaler is a tool that automatically adjusts the size of the Kubernetes cluster when one of the following conditions is true: + +* There are pods that failed to run in the cluster due to insufficient resources. +* There are nodes in the cluster that have been underutilized for an extended period of time and their pods can be placed on other existing nodes. + +To prevent your pod from being evicted, set a `priorityClassName: system-cluster-critical` property on your pod spec. + +Cluster Autoscaler is designed to run on Kubernetes master nodes. It can run in the `kube-system` namespace. Cluster Autoscaler doesn't scale down nodes with non-mirrored `kube-system` pods running on them. + +It's possible to run a customized deployment of Cluster Autoscaler on worker nodes, but extra care needs to be taken to ensure that Cluster Autoscaler remains up and running. + +## Cloud Providers + +Cluster Autoscaler provides support to distinct cloud providers. For more information, go to [cluster-autoscaler supported cloud providers.](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler#deployment) + +### Setting up Cluster Autoscaler on Amazon Cloud Provider + +For details on running the cluster autoscaler on Amazon cloud provider, refer to [this page.](../how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md b/versioned_docs/version-latest/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md new file mode 100644 index 00000000000..4eda613d25f --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md @@ -0,0 +1,356 @@ +--- +title: Install/Upgrade Rancher on a Kubernetes Cluster +description: Learn how to install Rancher in development and production environments. Read about single node and high availability installation +--- + + + + + +In this section, you'll learn how to deploy Rancher on a Kubernetes cluster using the Helm CLI. + + +## Prerequisites + +- [Kubernetes Cluster](#kubernetes-cluster) +- [Ingress Controller](#ingress-controller) +- [CLI Tools](#cli-tools) + +### Kubernetes Cluster + +Set up the Rancher server's local Kubernetes cluster. + +Rancher can be installed on any Kubernetes cluster. This cluster can use upstream Kubernetes, or it can use one of Rancher's Kubernetes distributions, or it can be a managed Kubernetes cluster from a provider such as Amazon EKS. + +For help setting up a Kubernetes cluster, we provide these tutorials: + +- **RKE:** For the tutorial to install an RKE Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher.md) For help setting up the infrastructure for a high-availability RKE cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster.md) +- **K3s:** For the tutorial to install a K3s Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher.md) For help setting up the infrastructure for a high-availability K3s cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster.md) +- **RKE2:** For the tutorial to install an RKE2 Kubernetes cluster, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher.md) For help setting up the infrastructure for a high-availability RKE2 cluster, refer to [this page.](../how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster.md) +- **Amazon EKS:** For details on how to install Rancher on Amazon EKS, including how to install an Ingress controller so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md) +- **AKS:** For details on how to install Rancher with Azure Kubernetes Service, including how to install an Ingress controller so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md) +- **GKE:** For details on how to install Rancher with Google Kubernetes Engine, including how to install an Ingress controller so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md) GKE has two modes of operation when creating a Kubernetes cluster, Autopilot and Standard mode. The cluster configuration for Autopilot mode has restrictions on editing the kube-system namespace. However, Rancher needs to create resources in the kube-system namespace during installation. As a result, you will not be able to install Rancher on a GKE cluster created in Autopilot mode. + + +### Ingress Controller + +The Rancher UI and API are exposed through an Ingress. This means the Kubernetes cluster that you install Rancher in must contain an Ingress controller. + +For RKE, RKE2, and K3s installations, you don't have to install the Ingress controller manually because one is installed by default. + +For distributions that do not include an Ingress Controller by default, like a hosted Kubernetes cluster such as EKS, GKE, or AKS, you have to deploy an Ingress controller first. Note that the Rancher Helm chart does not set an `ingressClassName` on the ingress by default. Because of this, you have to configure the Ingress controller to also watch ingresses without an `ingressClassName`. + +Examples are included in the **Amazon EKS**, **AKS**, and **GKE** tutorials above. + +### CLI Tools + +The following CLI tools are required for setting up the Kubernetes cluster. Please make sure these tools are installed and available in your `$PATH`. + +- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/#install-kubectl) - Kubernetes command-line tool. +- [helm](https://docs.helm.sh/using_helm/#installing-helm) - Package management for Kubernetes. Refer to the [Helm version requirements](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md) to choose a version of Helm to install Rancher. Refer to the [instructions provided by the Helm project](https://helm.sh/docs/intro/install/) for your specific platform. + +## Install the Rancher Helm Chart + +Rancher is installed using the [Helm](https://helm.sh/) package manager for Kubernetes. Helm charts provide templating syntax for Kubernetes YAML manifest documents. With Helm, we can create configurable deployments instead of just using static files. + +For systems without direct internet access, see [Air Gap: Kubernetes install](../getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha.md). + +To choose a Rancher version to install, refer to [Choosing a Rancher Version.](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md) + +To choose a version of Helm to install Rancher with, refer to the [Helm version requirements](../getting-started/installation-and-upgrade/resources/helm-version-requirements.md) + +:::note + +The installation instructions assume you are using Helm 3. + +::: + +To set up Rancher, + +1. [Add the Helm chart repository](#1-add-the-helm-chart-repository) +2. [Create a namespace for Rancher](#2-create-a-namespace-for-rancher) +3. [Choose your SSL configuration](#3-choose-your-ssl-configuration) +4. [Install cert-manager](#4-install-cert-manager) (unless you are bringing your own certificates, or TLS will be terminated on a load balancer) +5. [Install Rancher with Helm and your chosen certificate option](#5-install-rancher-with-helm-and-your-chosen-certificate-option) +6. [Verify that the Rancher server is successfully deployed](#6-verify-that-the-rancher-server-is-successfully-deployed) +7. [Save your options](#7-save-your-options) + +### 1. Add the Helm Chart Repository + +Use `helm repo add` command to add the Helm chart repository that contains charts to install Rancher. For more information about the repository choices and which is best for your use case, see [Choosing a Rancher Version](../getting-started/installation-and-upgrade/resources/choose-a-rancher-version.md). + +- Latest: Recommended for trying out the newest features + ``` + helm repo add rancher-latest https://releases.rancher.com/server-charts/latest + ``` +- Stable: Recommended for production environments + ``` + helm repo add rancher-stable https://releases.rancher.com/server-charts/stable + ``` +- Alpha: Experimental preview of upcoming releases. + ``` + helm repo add rancher-alpha https://releases.rancher.com/server-charts/alpha + ``` + Note: Upgrades are not supported to, from, or between Alphas. + +### 2. Create a Namespace for Rancher + +We'll need to define a Kubernetes namespace where the resources created by the Chart should be installed. This should always be `cattle-system`: + +``` +kubectl create namespace cattle-system +``` + +### 3. Choose your SSL Configuration + +The Rancher management server is designed to be secure by default and requires SSL/TLS configuration. + +:::note + +If you want terminate SSL/TLS externally, see [TLS termination on an External Load Balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination). + +::: + +There are three recommended options for the source of the certificate used for TLS termination at the Rancher server: + +- **Rancher-generated TLS certificate:** In this case, you will need to install `cert-manager` into the cluster. Rancher utilizes `cert-manager` to issue and maintain its certificates. Rancher will generate a CA certificate of its own, and sign a cert using that CA. `cert-manager` is then responsible for managing that certificate. +- **Let's Encrypt:** The Let's Encrypt option also uses `cert-manager`. However, in this case, cert-manager is combined with a special Issuer for Let's Encrypt that performs all actions (including request and validation) necessary for getting a Let's Encrypt issued cert. This configuration uses HTTP validation (`HTTP-01`), so the load balancer must have a public DNS record and be accessible from the internet. +- **Bring your own certificate:** This option allows you to bring your own public- or private-CA signed certificate. Rancher will use that certificate to secure websocket and HTTPS traffic. In this case, you must upload this certificate (and associated key) as PEM-encoded files with the name `tls.crt` and `tls.key`. If you are using a private CA, you must also upload that certificate. This is due to the fact that this private CA may not be trusted by your nodes. Rancher will take that CA certificate, and generate a checksum from it, which the various Rancher components will use to validate their connection to Rancher. + + +| Configuration | Helm Chart Option | Requires cert-manager | +| ------------------------------ | ----------------------- | ------------------------------------- | +| Rancher Generated Certificates (Default) | `ingress.tls.source=rancher` | [yes](#4-install-cert-manager) | +| Let’s Encrypt | `ingress.tls.source=letsEncrypt` | [yes](#4-install-cert-manager) | +| Certificates from Files | `ingress.tls.source=secret` | no | + +### 4. Install cert-manager + +> You should skip this step if you are bringing your own certificate files (option `ingress.tls.source=secret`), or if you use [TLS termination on an external load balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination). + +This step is only required to use certificates issued by Rancher's generated CA (`ingress.tls.source=rancher`) or to request Let's Encrypt issued certificates (`ingress.tls.source=letsEncrypt`). + +
+ Click to Expand + +:::note Important: + +Recent changes to cert-manager require an upgrade. If you are upgrading Rancher and using a version of cert-manager older than v0.11.0, please see our [upgrade documentation](../getting-started/installation-and-upgrade/resources/upgrade-cert-manager.md). + +::: + +These instructions are adapted from the [official cert-manager documentation](https://cert-manager.io/docs/installation/kubernetes/#installing-with-helm). + +:::note + +To see options on how to customize the cert-manager install (including for cases where your cluster uses PodSecurityPolicies), see the [cert-manager docs](https://artifacthub.io/packages/helm/cert-manager/cert-manager#configuration). + +::: + +``` +# If you have installed the CRDs manually instead of with the `--set installCRDs=true` option added to your Helm install command, you should upgrade your CRD resources before upgrading the Helm chart: +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + +# Add the Jetstack Helm repository +helm repo add jetstack https://charts.jetstack.io + +# Update your local Helm chart repository cache +helm repo update + +# Install the cert-manager Helm chart +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version v1.11.0 +``` + +Once you’ve installed cert-manager, you can verify it is deployed correctly by checking the cert-manager namespace for running pods: + +``` +kubectl get pods --namespace cert-manager + +NAME READY STATUS RESTARTS AGE +cert-manager-5c6866597-zw7kh 1/1 Running 0 2m +cert-manager-cainjector-577f6d9fd7-tr77l 1/1 Running 0 2m +cert-manager-webhook-787858fcdb-nlzsq 1/1 Running 0 2m +``` + +
+ +### 5. Install Rancher with Helm and Your Chosen Certificate Option + +The exact command to install Rancher differs depending on the certificate configuration. + +However, irrespective of the certificate configuration, the name of the Rancher installation in the `cattle-system` namespace should always be `rancher`. + +:::tip Testing and Development: + +This final command to install Rancher requires a domain name that forwards traffic to Rancher. If you are using the Helm CLI to set up a proof-of-concept, you can use a fake domain name when passing the `hostname` option. An example of a fake domain name would be `.sslip.io`, which would expose Rancher on an IP where it is running. Production installs would require a real domain name. + +::: + + + + +The default is for Rancher to generate a CA and uses `cert-manager` to issue the certificate for access to the Rancher server interface. + +Because `rancher` is the default option for `ingress.tls.source`, we are not specifying `ingress.tls.source` when running the `helm install` command. + +- Set the `hostname` to the DNS name you pointed at your load balancer. +- Set the `bootstrapPassword` to something unique for the `admin` user. +- To install a specific Rancher version, use the `--version` flag, example: `--version 2.7.0` +- For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin +``` + +If you are installing an alpha version, Helm requires adding the `--devel` option to the install command: + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +Wait for Rancher to be rolled out: + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + + +This option uses `cert-manager` to automatically request and renew [Let's Encrypt](https://letsencrypt.org/) certificates. This is a free service that provides you with a valid certificate as Let's Encrypt is a trusted CA. + +:::note + +You need to have port 80 open as the HTTP-01 challenge can only be done on port 80. + +::: + +In the following command, + +- `hostname` is set to the public DNS record, +- Set the `bootstrapPassword` to something unique for the `admin` user. +- `ingress.tls.source` is set to `letsEncrypt` +- `letsEncrypt.email` is set to the email address used for communication about your certificate (for example, expiry notices) +- Set `letsEncrypt.ingress.class` to whatever your ingress controller is, e.g., `traefik`, `nginx`, `haproxy`, etc. +- For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=letsEncrypt \ + --set letsEncrypt.email=me@example.org \ + --set letsEncrypt.ingress.class=nginx +``` + +If you are installing an alpha version, Helm requires adding the `--devel` option to the install command: + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +Wait for Rancher to be rolled out: + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + + + +In this option, Kubernetes secrets are created from your own certificates for Rancher to use. + +When you run this command, the `hostname` option must match the `Common Name` or a `Subject Alternative Names` entry in the server certificate or the Ingress controller will fail to configure correctly. + +Although an entry in the `Subject Alternative Names` is technically required, having a matching `Common Name` maximizes compatibility with older browsers and applications. + +:::note + +If you want to check if your certificates are correct, see [How do I check Common Name and Subject Alternative Names in my server certificate?](../faq/technical-items.md#how-do-i-check-common-name-and-subject-alternative-names-in-my-server-certificate) + +::: + +- Set the `hostname`. +- Set the `bootstrapPassword` to something unique for the `admin` user. +- Set `ingress.tls.source` to `secret`. +- For Kubernetes v1.25 or later, set `global.cattle.psp.enabled` to `false` when using Rancher v2.7.2-v2.7.4. This is not necessary for Rancher v2.7.5 and above, but you can still manually set the option if you choose. + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret +``` +If you are installing an alpha version, Helm requires adding the `--devel` option to the install command: + +``` +helm install rancher rancher-alpha/rancher --devel +``` + +If you are using a Private CA signed certificate , add `--set privateCA=true` to the command: + +``` +helm install rancher rancher-/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin \ + --set ingress.tls.source=secret \ + --set privateCA=true +``` + +Now that Rancher is deployed, see [Adding TLS Secrets](../getting-started/installation-and-upgrade/resources/add-tls-secrets.md) to publish the certificate files so Rancher and the Ingress controller can use them. + + + + +The Rancher chart configuration has many options for customizing the installation to suit your specific environment. Here are some common advanced scenarios. + +- [HTTP Proxy](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#http-proxy) +- [Private Container Image Registry](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#private-registry-and-air-gap-installs) +- [TLS Termination on an External Load Balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) + +See the [Chart Options](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md) for the full list of options. + + +### 6. Verify that the Rancher Server is Successfully Deployed + +After adding the secrets, check if Rancher was rolled out successfully: + +``` +kubectl -n cattle-system rollout status deploy/rancher +Waiting for deployment "rancher" rollout to finish: 0 of 3 updated replicas are available... +deployment "rancher" successfully rolled out +``` + +If you see the following error: `error: deployment "rancher" exceeded its progress deadline`, you can check the status of the deployment by running the following command: + +``` +kubectl -n cattle-system get deploy rancher +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +rancher 3 3 3 3 3m +``` + +It should show the same count for `DESIRED` and `AVAILABLE`. + +### 7. Save Your Options + +Make sure you save the `--set` options you used. You will need to use the same options when you upgrade Rancher to new versions with Helm. + +### Finishing Up + +That's it. You should have a functional Rancher server. + +In a web browser, go to the DNS name that forwards traffic to your load balancer. Then you should be greeted by the colorful login page. + +Doesn't work? Take a look at the [Troubleshooting](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting.md) Page diff --git a/versioned_docs/version-latest/pages-for-subheaders/installation-and-upgrade.md b/versioned_docs/version-latest/pages-for-subheaders/installation-and-upgrade.md new file mode 100644 index 00000000000..3077b14edc2 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/installation-and-upgrade.md @@ -0,0 +1,98 @@ +--- +title: Installing/Upgrading Rancher +description: Learn how to install Rancher in development and production environments. Read about single node and high availability installation +--- + + + + + +This section provides an overview of the architecture options of installing Rancher, describing advantages of each option. + +## Terminology + +In this section, + +- **The Rancher server** manages and provisions Kubernetes clusters. You can interact with downstream Kubernetes clusters through the Rancher server's user interface. The Rancher management server can be installed on any Kubernetes cluster, including hosted clusters, such as Amazon EKS clusters. +- **RKE (Rancher Kubernetes Engine)** is a certified Kubernetes distribution and CLI/library which creates and manages a Kubernetes cluster. +- **K3s (Lightweight Kubernetes)** is also a fully compliant Kubernetes distribution. It is newer than RKE, easier to use, and more lightweight, with a binary size of less than 100 MB. +- **RKE2** is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. + +Note the `restrictedAdmin` Helm chart option available for **the Rancher Server**. When this option is set to true, the initial Rancher user has restricted access to the local Kubernetes cluster to prevent privilege escalation. For more information, see the section about the [restricted-admin role.](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md#restricted-admin) + +## Overview of Installation Options + +Rancher can be installed on these main architectures: + +### High-availability Kubernetes Install with the Helm CLI + +We recommend using Helm, a Kubernetes package manager, to install Rancher on multiple nodes on a dedicated Kubernetes cluster. For RKE clusters, three nodes are required to achieve a high-availability cluster. For K3s clusters, only two nodes are required. + +### Rancher on EKS Install with the AWS Marketplace + +Rancher can be installed on to Amazon Elastic Kubernetes Service (EKS) [through the AWS Marketplace](../getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace.md). The EKS cluster deployed is production-ready and follows AWS best practices. + +### Single-node Kubernetes Install + +Rancher can be installed on a single-node Kubernetes cluster. In this case, the Rancher server doesn't have high availability, which is important for running Rancher in production. + +However, this option is useful if you want to save resources by using a single node in the short term, while preserving a high-availability migration path. In the future, you can add nodes to the cluster to get a high-availability Rancher server. + +### Docker Install + +For test and demonstration purposes, Rancher can be installed with Docker on a single node. A local Kubernetes cluster is installed in the single Docker container, and Rancher is installed on the local cluster. + +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +### Other Options + +There are also separate instructions for installing Rancher in an air gap environment or behind an HTTP proxy: + +| Level of Internet Access | Kubernetes Installation - Strongly Recommended | Docker Installation | +| ---------------------------------- | ------------------------------ | ---------- | +| With direct access to the Internet | [Docs](install-upgrade-on-a-kubernetes-cluster.md) | [Docs](rancher-on-a-single-node-with-docker.md) | +| Behind an HTTP proxy | [Docs](rancher-behind-an-http-proxy.md) | These [docs,](rancher-on-a-single-node-with-docker.md) plus this [configuration](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) | +| In an air gap environment | [Docs](air-gapped-helm-cli-install.md) | [Docs](air-gapped-helm-cli-install.md) | + +We recommend installing Rancher on a Kubernetes cluster, because in a multi-node cluster, the Rancher management server becomes highly available. This high-availability configuration helps maintain consistent access to the downstream Kubernetes clusters that Rancher will manage. + +For that reason, we recommend that for a production-grade architecture, you should set up a high-availability Kubernetes cluster, then install Rancher on it. After Rancher is installed, you can use Rancher to deploy and manage Kubernetes clusters. + +For testing or demonstration purposes, you can install Rancher in single Docker container. In this Docker install, you can use Rancher to set up Kubernetes clusters out-of-the-box. The Docker install allows you to explore the Rancher server functionality, but it is intended to be used for development and testing purposes only. + +Our [instructions for installing Rancher on Kubernetes](install-upgrade-on-a-kubernetes-cluster.md) describe how to first use K3s or RKE to create and manage a Kubernetes cluster, then install Rancher onto that cluster. + +When the nodes in your Kubernetes cluster are running and fulfill the [node requirements,](installation-requirements.md) you will use Helm to deploy Rancher onto Kubernetes. Helm uses Rancher's Helm chart to install a replica of Rancher on each node in the Kubernetes cluster. We recommend using a load balancer to direct traffic to each replica of Rancher in the cluster. + +For a longer discussion of Rancher architecture, refer to the [architecture overview,](rancher-manager-architecture.md) [recommendations for production-grade architecture,](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) or our [best practices guide.](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md) + +## Prerequisites + +Before installing Rancher, make sure that your nodes fulfill all of the [installation requirements.](installation-requirements.md) + +## Architecture Tip + +For the best performance and greater security, we recommend a separate, dedicated Kubernetes cluster for the Rancher management server. Running user workloads on this cluster is not advised. After deploying Rancher, you can [create or import clusters](kubernetes-clusters-in-rancher-setup.md) for running your workloads. + +For more architecture recommendations, refer to [this page.](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) + +### More Options for Installations on a Kubernetes Cluster + +Refer to the [Helm chart options](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md) for details on installing Rancher on a Kubernetes cluster with other configurations, including: + +- With [API auditing to record all transactions](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#api-audit-log) +- With [TLS termination on a load balancer](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#external-tls-termination) +- With a [custom Ingress](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md#customizing-your-ingress) + +In the Rancher installation instructions, we recommend using K3s or RKE to set up a Kubernetes cluster before installing Rancher on the cluster. Both K3s and RKE have many configuration options for customizing the Kubernetes cluster to suit your specific environment. For the full list of their capabilities, refer to their documentation: + +- [RKE configuration options](https://rancher.com/docs/rke/latest/en/config-options/) +- [K3s configuration options](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +### More Options for Installations with Docker + +Refer to the [docs about options for Docker installs](rancher-on-a-single-node-with-docker.md) for details about other configurations including: + +- With [API auditing to record all transactions](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) +- With an [external load balancer](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md) +- With a [persistent data store](../reference-guides/single-node-rancher-in-docker/advanced-options.md#persistent-data) diff --git a/versioned_docs/version-latest/pages-for-subheaders/installation-references.md b/versioned_docs/version-latest/pages-for-subheaders/installation-references.md new file mode 100644 index 00000000000..6108728b04f --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/installation-references.md @@ -0,0 +1,9 @@ +--- +title: Installation References +--- + + + + + +Please see the following reference guides for other installation resources: [Rancher Helm chart options](../getting-started/installation-and-upgrade/installation-references/helm-chart-options.md), [TLS settings](../getting-started/installation-and-upgrade/installation-references/tls-settings.md), and [feature flags](../getting-started/installation-and-upgrade/installation-references/feature-flags.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/installation-requirements.md b/versioned_docs/version-latest/pages-for-subheaders/installation-requirements.md new file mode 100644 index 00000000000..b7214336b13 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/installation-requirements.md @@ -0,0 +1,159 @@ +--- +title: Installation Requirements +description: Learn the node requirements for each node running Rancher server when you’re configuring Rancher to run either in a Docker or Kubernetes setup +--- + + + + + +This page describes the software, hardware, and networking requirements for the nodes where the Rancher server will be installed. The Rancher server can be installed on a single node or a high-availability Kubernetes cluster. + +:::note Important: + +If you install Rancher on a Kubernetes cluster, requirements are different from the [node requirements for downstream user clusters,](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) which will run your apps and services. + +::: + +The Rancher UI works best in Firefox or Chromium based browsers (Chrome, Edge, Opera, Brave, etc). + +See our page on [best practices](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md) for a list of recommendations for running a Rancher server in production. + +## Kubernetes Compatibility with Rancher + +Rancher needs to be installed on a supported Kubernetes version. Consult the [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions) to ensure that your intended version of Kubernetes is supported. + +## Operating Systems and Container Runtime Requirements + +All supported operating systems are 64-bit x86. Rancher should work with any modern Linux distribution. + +The [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions) lists which OS and Docker versions were tested for each Rancher version. + +Docker is required for nodes that will run RKE clusters. It is not required for RKE2 or K3s clusters. + +The `ntp` (Network Time Protocol) package should be installed. This prevents errors with certificate validation that can occur when the time is not synchronized between the client and server. + +Some distributions of Linux may have default firewall rules that block communication within the Kubernetes cluster. Since Kubernetes v1.19, firewalld must be turned off, because it conflicts with the Kubernetes networking plugins. + +If you don't feel comfortable doing so, you might check suggestions in the [respective issue](https://github.com/rancher/rancher/issues/28840). Some users were successful [creating a separate firewalld zone with a policy of ACCEPT for the Pod CIDR](https://github.com/rancher/rancher/issues/28840#issuecomment-787404822). + +If you plan to run Rancher on ARM64, see [Running on ARM64 (Experimental).](../how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64.md) + +### RKE Specific Requirements + +For the container runtime, RKE should work with any modern Docker version. + +For more information see [Installing Docker,](../getting-started/installation-and-upgrade/installation-requirements/install-docker.md) + +### K3s Specific Requirements + +For the container runtime, K3s bundles its own containerd by default. Alternatively, you can configure K3s to use an already installed Docker runtime. For more information on using K3s with Docker see the [K3s documentation.](https://docs.k3s.io/advanced#using-docker-as-the-container-runtime) + +Rancher needs to be installed on a supported Kubernetes version. To find out which versions of Kubernetes are supported for your Rancher version, refer to the [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions). To specify the K3s version, use the INSTALL_K3S_VERSION environment variable when running the K3s installation script. + +If you are installing Rancher on a K3s cluster with **Raspbian Buster**, follow [these steps](https://rancher.com/docs/k3s/latest/en/advanced/#enabling-legacy-iptables-on-raspbian-buster) to switch to legacy iptables. + +If you are installing Rancher on a K3s cluster with Alpine Linux, follow [these steps](https://rancher.com/docs/k3s/latest/en/advanced/#additional-preparation-for-alpine-linux-setup) for additional setup. + +### RKE2 Specific Requirements + +For the container runtime, RKE2 bundles its own containerd. Docker is not required for RKE2 installs. + +For details on which OS versions were tested with RKE2, refer to the [Rancher support matrix](https://www.suse.com/suse-rancher/support-matrix/all-supported-versions). + +## Hardware Requirements + +The following sections describe the CPU, memory, and disk requirements for the nodes where the Rancher server is installed. + +## CPU and Memory + +Hardware requirements scale based on the size of your Rancher deployment. Provision each individual node according to the requirements. The requirements are different depending on if you are installing Rancher in a single container with Docker, or if you are installing Rancher on a Kubernetes cluster. + +### RKE and Hosted Kubernetes + +These CPU and memory requirements apply to each host in the Kubernetes cluster where the Rancher server is installed. + +These requirements apply to RKE Kubernetes clusters, as well as to hosted Kubernetes clusters such as EKS. + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | +| --------------- | ---------- | ------------ | -------| ------- | +| Small | Up to 150 | Up to 1500 | 2 | 8 GB | +| Medium | Up to 300 | Up to 3000 | 4 | 16 GB | +| Large | Up to 500 | Up to 5000 | 8 | 32 GB | +| X-Large | Up to 1000 | Up to 10,000 | 16 | 64 GB | +| XX-Large | Up to 2000 | Up to 20,000 | 32 | 128 GB | + +Every use case and environment is different. Please [contact Rancher](https://rancher.com/contact/) to review yours. + +### K3s Kubernetes + +These CPU and memory requirements apply to each host in a [K3s Kubernetes cluster where the Rancher server is installed.](install-upgrade-on-a-kubernetes-cluster.md) + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | Database Size | +| --------------- | ---------- | ------------ | -------| ---------| ------------------------- | +| Small | Up to 150 | Up to 1500 | 2 | 8 GB | 2 cores, 4 GB + 1000 IOPS | +| Medium | Up to 300 | Up to 3000 | 4 | 16 GB | 2 cores, 4 GB + 1000 IOPS | +| Large | Up to 500 | Up to 5000 | 8 | 32 GB | 2 cores, 4 GB + 1000 IOPS | +| X-Large | Up to 1000 | Up to 10,000 | 16 | 64 GB | 2 cores, 4 GB + 1000 IOPS | +| XX-Large | Up to 2000 | Up to 20,000 | 32 | 128 GB | 2 cores, 4 GB + 1000 IOPS | + +Every use case and environment is different. Please [contact Rancher](https://rancher.com/contact/) to review yours. + + +### RKE2 Kubernetes + +These CPU and memory requirements apply to each instance with RKE2 installed. Minimum recommendations are outlined here. + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | +| --------------- | -------- | --------- | ----- | ---- | +| Small | Up to 5 | Up to 50 | 2 | 5 GB | +| Medium | Up to 15 | Up to 200 | 3 | 9 GB | + +### Docker + +These CPU and memory requirements apply to a host with a [single-node](rancher-on-a-single-node-with-docker.md) installation of Rancher. + +| Deployment Size | Clusters | Nodes | vCPUs | RAM | +| --------------- | -------- | --------- | ----- | ---- | +| Small | Up to 5 | Up to 50 | 1 | 4 GB | +| Medium | Up to 15 | Up to 200 | 2 | 8 GB | + +## Ingress + +Each node in the Kubernetes cluster that Rancher is installed on should run an Ingress. + +The Ingress should be deployed as DaemonSet to ensure your load balancer can successfully route traffic to all nodes. + +For RKE, RKE2 and K3s installations, you don't have to install the Ingress manually because it is installed by default. + +For hosted Kubernetes clusters (EKS, GKE, AKS), you will need to set up the ingress. + +- **Amazon EKS:** For details on how to install Rancher on Amazon EKS, including how to install an ingress so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks.md) +- **AKS:** For details on how to install Rancher with Azure Kubernetes Service, including how to install an ingress so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks.md) +- **GKE:** For details on how to install Rancher with Google Kubernetes Engine, including how to install an ingress so that the Rancher server can be accessed, refer to [this page.](../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke.md) + +## Disks + +Rancher performance depends on etcd in the cluster performance. To ensure optimal speed, we recommend always using SSD disks to back your Rancher management Kubernetes cluster. On cloud providers, you will also want to use the minimum size that allows the maximum IOPS. In larger clusters, consider using dedicated storage devices for etcd data and wal directories. + +## Networking Requirements + +This section describes the networking requirements for the node(s) where the Rancher server is installed. + +:::caution + +If a server containing Rancher has the `X-Frame-Options=DENY` header, some pages in the new Rancher UI will not be able to render after upgrading from the legacy UI. This is because some legacy pages are embedded as iFrames in the new UI. + +::: + +### Node IP Addresses + +Each node used should have a static IP configured, regardless of whether you are installing Rancher on a single node or on an HA cluster. In case of DHCP, each node should have a DHCP reservation to make sure the node gets the same IP allocated. + +### Port Requirements + +To operate properly, Rancher requires a number of ports to be open on Rancher nodes and on downstream Kubernetes cluster nodes. [Port Requirements](../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md) lists all the necessary ports for Rancher and Downstream Clusters for the different cluster types. + +## Dockershim Support + +For more information on Dockershim support, refer to [this page](../getting-started/installation-and-upgrade/installation-requirements/dockershim.md). diff --git a/versioned_docs/version-latest/pages-for-subheaders/istio-setup-guide.md b/versioned_docs/version-latest/pages-for-subheaders/istio-setup-guide.md new file mode 100644 index 00000000000..24475f7ffea --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/istio-setup-guide.md @@ -0,0 +1,34 @@ +--- +title: Setup Guide +--- + + + + + +This section describes how to enable Istio and start using it in your projects. + +If you use Istio for traffic management, you will need to allow external traffic to the cluster. In that case, you will need to follow all of the steps below. + +## Prerequisites + +This guide assumes you have already [installed Rancher,](installation-and-upgrade.md) and you have already [provisioned a separate Kubernetes cluster](kubernetes-clusters-in-rancher-setup.md) on which you will install Istio. + +The nodes in your cluster must meet the [CPU and memory requirements.](../integrations-in-rancher/istio/cpu-and-memory-allocations.md) + +The workloads and services that you want to be controlled by Istio must meet [Istio's requirements.](https://istio.io/docs/setup/additional-setup/requirements/) + +## Install + +:::tip Quick Setup Tip: + +If you don't need external traffic to reach Istio, and you just want to set up Istio for monitoring and tracing traffic within the cluster, skip the steps for [setting up the Istio gateway](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) and [setting up Istio's components for traffic management.](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md) + +::: + +1. [Enable Istio in the cluster.](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster.md) +1. [Enable Istio in all the namespaces where you want to use it.](../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md) +1. [Add deployments and services that have the Istio sidecar injected.](../how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar.md) +1. [Set up the Istio gateway. ](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway.md) +1. [Set up Istio's components for traffic management.](../how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management.md) +1. [Generate traffic and see Istio in action.](../how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/istio.md b/versioned_docs/version-latest/pages-for-subheaders/istio.md new file mode 100644 index 00000000000..93f1e9b7cc5 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/istio.md @@ -0,0 +1,141 @@ +--- +title: Istio +--- + + + + + +[Istio](https://istio.io/) is an open-source tool that makes it easier for DevOps teams to observe, secure, control, and troubleshoot the traffic within a complex network of microservices. + +As a network of microservices changes and grows, the interactions between them can become increasingly difficult to manage and understand. In such a situation, it is useful to have a service mesh as a separate infrastructure layer. Istio's service mesh lets you manipulate traffic between microservices without changing the microservices directly. + +Our integration of Istio is designed so that a Rancher operator, such as an administrator or cluster owner, can deliver Istio to a team of developers. Then developers can use Istio to enforce security policies, troubleshoot problems, or manage traffic for green/blue deployments, canary deployments, or A/B testing. + +This core service mesh provides features that include but are not limited to the following: + +- **Traffic Management** such as ingress and egress routing, circuit breaking, mirroring. +- **Security** with resources to authenticate and authorize traffic and users, mTLS included. +- **Observability** of logs, metrics, and distributed traffic flows. + +After [setting up istio](istio-setup-guide.md) you can leverage Istio's control plane functionality through the Rancher UI, `kubectl`, or `istioctl`. + +Istio needs to be set up by a `cluster-admin` before it can be used in a project. + + +## What's New in Rancher v2.5 + +The overall architecture of Istio has been simplified. A single component, Istiod, has been created by combining Pilot, Citadel, Galley and the sidecar injector. Node Agent functionality has also been merged into istio-agent. + +Addons that were previously installed by Istio (cert-manager, Grafana, Jaeger, Kiali, Prometheus, Zipkin) will now need to be installed separately. Istio will support installation of integrations that are from the Istio Project and will maintain compatibility with those that are not. + +A Prometheus integration will still be available through an installation of [Rancher Monitoring](monitoring-and-alerting.md), or by installing your own Prometheus operator. Rancher's Istio chart will also install Kiali by default to ensure you can get a full picture of your microservices out of the box. + +Istio has migrated away from Helm as a way to install Istio and now provides installation through the istioctl binary or Istio Operator. To ensure the easiest interaction with Istio, Rancher's Istio will maintain a Helm chart that utilizes the istioctl binary to manage your Istio installation. + +This Helm chart will be available via the Apps and Marketplace in the UI. A user that has access to the Rancher Chart's catalog will need to set up Istio before it can be used in the project. + +## Tools Bundled with Istio + +Our [Istio](https://istio.io/) installer wraps the istioctl binary commands in a handy Helm chart, including an overlay file option to allow complex customization. + +It also includes the following: + +### Kiali + +Kiali is a comprehensive visualization aid used for graphing traffic flow throughout the service mesh. It allows you to see how they are connected, including the traffic rates and latencies between them. + +You can check the health of the service mesh, or drill down to see the incoming and outgoing requests to a single component. + +### Jaeger + +Our Istio installer includes a quick-start, all-in-one installation of [Jaeger,](https://www.jaegertracing.io/) a tool used for tracing distributed systems. + +Note that this is not a production-qualified deployment of Jaeger. This deployment uses an in-memory storage component, while a persistent storage component is recommended for production. For more information on which deployment strategy you may need, refer to the [Jaeger documentation.](https://www.jaegertracing.io/docs/latest/operator/#production-strategy) + +## Prerequisites + +Before enabling Istio, we recommend that you confirm that your Rancher worker nodes have enough [CPU and memory](../integrations-in-rancher/istio/cpu-and-memory-allocations.md) to run all of the components of Istio. + +If you are installing Istio on RKE2 cluster, some additional steps are required. For details, see [this section.](#additional-steps-for-installing-istio-on-an-rke2-cluster) + +## Setup Guide + +Refer to the [setup guide](istio-setup-guide.md) for instructions on how to set up Istio and use it in a project. + +## Remove Istio + +To remove Istio components from a cluster, namespace, or workload, refer to the section on [uninstalling Istio.](../integrations-in-rancher/istio/disable-istio.md) + +## Accessing Visualizations + +> By default, only cluster-admins have access to Kiali. For instructions on how to allow admin, edit or views roles to access them, see [this section.](../integrations-in-rancher/istio/rbac-for-istio.md) + +After Istio is set up in a cluster, Grafana, Prometheus, and Kiali are available in the Rancher UI. + +To access the Grafana and Prometheus visualizations, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see the visualizations and click **Explore**. +1. In the left navigation bar, click **Monitoring**. +1. Click **Grafana** or any of the other dashboards. + +To access the Kiali visualization, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to see Kiali and click **Explore**. +1. In the left navigation bar, click **Istio**. +1. Click **Kiali**. From here you can access the **Traffic Graph** tab or the **Traffic Metrics** tab to see network visualizations and metrics. + +By default, all namespace will picked up by prometheus and make data available for Kiali graphs. Refer to [selector/scrape config setup](../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) if you would like to use a different configuration for prometheus data scraping. + +Your access to the visualizations depend on your role. Grafana and Prometheus are only available for `cluster-admin` roles. The Kiali UI is available only to `cluster-admin` by default, but `cluster-admin` can allow other roles to access them by editing the Istio values.yaml. + +## Architecture + +Istio installs a service mesh that uses [Envoy](https://www.envoyproxy.io) sidecar proxies to intercept traffic to each workload. These sidecars intercept and manage service-to-service communication, allowing fine-grained observation and control over traffic within the cluster. + +Only workloads that have the Istio sidecar injected can be tracked and controlled by Istio. + +When a namespace has Istio enabled, new workloads deployed in the namespace will automatically have the Istio sidecar. You need to manually enable Istio in preexisting workloads. + +For more information on the Istio sidecar, refer to the [Istio sidecare-injection docs](https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/) and for more information on Istio's architecture, refer to the [Istio Architecture docs](https://istio.io/latest/docs/ops/deployment/architecture/) + +### Multiple Ingresses + +By default, each Rancher-provisioned cluster has one NGINX ingress controller allowing traffic into the cluster. Istio also installs an ingress gateway by default into the `istio-system` namespace. The result is that your cluster will have two ingresses in your cluster. + +![In an Istio-enabled cluster, you can have two ingresses: the default Nginx ingress, and the default Istio controller.](/img/istio-ingress.svg) + + Additional Istio Ingress gateways can be enabled via the [overlay file](configuration-options.md#overlay-file). + +### Egress Support + +By default the Egress gateway is disabled, but can be enabled on install or upgrade through the values.yaml or via the [overlay file](configuration-options.md#overlay-file). + +## Additional Steps for Installing Istio on an RKE2 Cluster + +To install Istio on an RKE2 cluster, follow the steps in [this section.](../integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster.md) + +## Upgrading Istio in an Air-Gapped Environment + +The Istio pod security policy is now enabled by default. A new value, `installer.releaseMirror.enabled`, has been added to the rancher-istio chart to enable and disable the server that supports air-gapped upgrades. Note that `installer.releaseMirror.enabled` is set to `false` by default. You can set this value as needed when you install or upgrade. Follow the steps below: + +1. Provision an air-gapped Rancher instance and an air-gapped custom cluster in the Rancher UI. +2. Install Monitoring in the cluster: **Cluster Explorer -> Apps & Marketplace -> Charts -> Monitoring**. +3. Pull all required images for Istio into the private registry you will use in the air-gapped environment. +4. Install Istio in the cluster: **Cluster Explorer -> Apps & Marketplace -> Charts -> Istio**. + +:::note + +You can enable [Jaeger](https://www.jaegertracing.io/) and [Kiali](https://kiali.io/) on a fresh Istio install. To ensure that Jaeger and Kiali work, set `installer.releaseMirror.enabled` to `true` in `values.yaml` during installation. + +::: + +5. Upgrade the Istio installation. + +:::caution + +If you haven't already, set `installer.releaseMirror.enabled=true` to upgrade Istio. + +::: diff --git a/versioned_docs/version-latest/pages-for-subheaders/k3s-hardening-guide.md b/versioned_docs/version-latest/pages-for-subheaders/k3s-hardening-guide.md new file mode 100644 index 00000000000..285577fbca5 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/k3s-hardening-guide.md @@ -0,0 +1,744 @@ +--- +title: K3s Hardening Guide +--- + + + + + +This document provides prescriptive guidance for how to harden a K3s cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for K3s clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | + +:::note +In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter isn't required anymore, and was removed by CIS. +::: + +For more details on how to evaluate a hardened K3s cluster against the official CIS benchmark, refer to the K3s self-assessment guides for specific Kubernetes and CIS benchmark versions. + +K3s passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. K3s does not modify the host operating system. Any host-level modifications need to be done manually. +2. Certain CIS policy controls for `NetworkPolicies` and `PodSecurityStandards` (`PodSecurityPolicies` on v1.24 and older) restrict cluster functionality. + You must opt into having K3s configure these policies. Add the appropriate options to your command-line flags or configuration file (enable admission plugins), and manually apply the appropriate policies. + See further for more details. + +The first section (1.1) of the CIS Benchmark primarily focuses on pod manifest permissions and ownership. Since everything in the distribution is packaged in a single binary, this section does not apply to the core components of K3s. + +## Host-level Requirements + +### Ensure `protect-kernel-defaults` is set + + + + +The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. + + + + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + protect-kernel-defaults: true +``` + + + + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +This configuration needs to be done before setting the kubelet flag, otherwise K3s will fail to start. + +## Kubernetes Runtime Requirements + +The CIS Benchmark runtime requirements center around pod security (via PSP or PSA), network policies and API Server auditing logs. + +By default, K3s does not include any pod security or network policies. However, K3s ships with a controller that enforces any network policies you create. By default, K3s enables both the `PodSecurity` and `NodeRestriction` admission controllers, among others. + +### Pod Security + + + + +K3s v1.25 and newer support [Pod Security admission (PSA)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) for controlling pod security. + +You can specify the PSA configuration by setting the `defaultPodSecurityAdmissionConfigurationTemplateName` field in the cluster configuration in Rancher: + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted +``` + +The `rancher-restricted` template is provided by Rancher to enforce the highly-restrictive Kubernetes upstream [`Restricted`](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted) profile with best practices for pod hardening. + + + + +K3s v1.24 and older support [Pod Security Policy (PSP)](https://v1-24.docs.kubernetes.io/docs/concepts/security/pod-security-policy/) for controlling pod security. + +You can enable PSPs by passing the following flags in the cluster configuration in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount +``` + +This maintains the `NodeRestriction` plugin and enables the `PodSecurityPolicy`. + +Once you enable PSPs, you can apply a policy to satisfy the necessary controls described in section 5.2 of the CIS Benchmark. + +:::note +These are manual checks in the CIS Benchmark. The CIS scan flags the results as `warning`, because manual inspection is necessary by the cluster operator. +::: + +Here is an example of a compliant PSP: + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false # CIS - 5.2.1 + allowPrivilegeEscalation: false # CIS - 5.2.5 + requiredDropCapabilities: # CIS - 5.2.7/8/9 + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false # CIS - 5.2.4 + hostIPC: false # CIS - 5.2.3 + hostPID: false # CIS - 5.2.2 + runAsUser: + rule: 'MustRunAsNonRoot' # CIS - 5.2.6 + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +``` + +For the example PSP to be effective, we need to create a `ClusterRole` and a `ClusterRoleBinding`. We also need to include a "system unrestricted policy" for system-level pods that require additional privileges, and an additional policy that allows the necessary sysctls for full functionality of ServiceLB. + +```yaml +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted-psp +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: system-unrestricted-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - '*' + fsGroup: + rule: RunAsAny + hostIPC: true + hostNetwork: true + hostPID: true + hostPorts: + - max: 65535 + min: 0 + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: svclb-psp + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_ADMIN + allowedUnsafeSysctls: + - net.ipv4.ip_forward + - net.ipv6.conf.all.forwarding + fsGroup: + rule: RunAsAny + hostPorts: + - max: 65535 + min: 0 + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + verbs: + - use + resourceNames: + - restricted-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:system-unrestricted-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svclb-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - svclb-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-local-path-provisioner-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-coredns-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:svc-cis-operator-psp +rules: +- apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - system-unrestricted-psp + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-psp +subjects: +- kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-unrestricted-node-psp-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-unrestricted-svc-acct-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:system-unrestricted-psp +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svclb-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svclb-psp +subjects: +- kind: ServiceAccount + name: svclb +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-local-path-provisioner-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-local-path-provisioner-psp +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-coredns-psp-rolebinding + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-coredns-psp +subjects: +- kind: ServiceAccount + name: coredns +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: svc-cis-operator-psp-rolebinding + namespace: cis-operator-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:svc-cis-operator-psp +subjects: +- kind: ServiceAccount + name: cis-operator-serviceaccount +``` + +The policies presented above can be placed in a file named `policy.yaml` in the `/var/lib/rancher/k3s/server/manifests` directory. Both the policy file and the its directory hierarchy must be created before starting K3s. A restrictive access permission is recommended to avoid leaking potential sensitive information. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +:::note +The critical Kubernetes additions such as CNI, DNS, and Ingress are run as pods in the `kube-system` namespace. Therefore, this namespace has a less restrictive policy, so that these components can run properly. +::: + + + + +### Network Policies + +CIS requires that all namespaces apply a network policy that reasonably limits traffic into namespaces and pods. + +:::note +This is a manual check in the CIS Benchmark. The CIS scan flags the result as a `warning`, because manual inspection is necessary by the cluster operator. +::: + +The network policies can be placed in the `policy.yaml` file in `/var/lib/rancher/k3s/server/manifests` directory. If the directory was not created as part of the PSP (as described above), it must be created first. + +```shell +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/manifests +``` + +Here is an example of a compliant network policy: + +```yaml +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-system +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-system +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: default +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: default +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: intra-namespace + namespace: kube-public +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: kube-public +``` + +The active restrictions block DNS unless purposely allowed. Below is a network policy that allows DNS-related traffic: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-network-dns-policy + namespace: +spec: + ingress: + - ports: + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + podSelector: + matchLabels: + k8s-app: kube-dns + policyTypes: + - Ingress +``` + +The metrics-server and Traefik ingress controller are blocked by default if network policies are not created to allow access. + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-metrics-server + namespace: kube-system +spec: + podSelector: + matchLabels: + k8s-app: metrics-server + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-svclbtraefik-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + svccontroller.k3s.cattle.io/svcname: traefik + ingress: + - {} + policyTypes: + - Ingress +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-all-traefik-v121-ingress + namespace: kube-system +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: traefik + ingress: + - {} + policyTypes: + - Ingress +``` + +:::note +You must manage network policies as normal for any additional namespaces you create. +::: + +### API Server audit configuration + +CIS requirements 1.2.22 to 1.2.25 are related to configuring audit logs for the API Server. K3s does not create by default the log directory and audit policy, as auditing requirements are specific to each user's policies and environment. + +If you need a log directory, it must be created before you start K3s. We recommend a restrictive access permission to avoid leaking sensitive information. + +```bash +sudo mkdir -p -m 700 /var/lib/rancher/k3s/server/logs +``` + +The following is a starter audit policy to log request metadata. This policy should be written to a file named `audit.yaml` in the `/var/lib/rancher/k3s/server` directory. Detailed information about policy configuration for the API server can be found in the [official Kubernetes documentation](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/). + +```yaml +--- +apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: Metadata +``` + +Further configurations are also needed to pass CIS checks. These are not configured by default in K3s, because they vary based on your environment and needs: + +- Ensure that the `--audit-log-path` argument is set. +- Ensure that the `--audit-log-maxage` argument is set to 30 or as appropriate. +- Ensure that the `--audit-log-maxbackup` argument is set to 10 or as appropriate. +- Ensure that the `--audit-log-maxsize` argument is set to 100 or as appropriate. + +Combined, to enable and configure audit logs, add the following lines to the K3s cluster configuration file in Rancher: + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 +``` + +### Controller Manager Requirements + +CIS requirement 1.3.1 checks for garbage collection settings in the Controller Manager. Garbage collection is important to ensure sufficient resource availability and avoid degraded performance and availability. Based on your system resources and tests, choose an appropriate threshold value to activate garbage collection. + +This can be remediated by setting the following configuration in the K3s cluster file in Rancher. The value below is only an example. The appropriate threshold value is specific to each user's environment. + +```yaml +spec: + rkeConfig: + machineGlobalConfig: + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 +``` + +### Configure `default` Service Account + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. + +For CIS requirement 5.1.5 the `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +This can be remediated by updating the `automountServiceAccountToken` field to `false` for the `default` service account in each namespace. + +For `default` service accounts in the built-in namespaces (`kube-system`, `kube-public`, `kube-node-lease`, and `default)`, K3s does not automatically do this. + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```shell +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Run the script every time a new service account is added to your cluster. + +## Reference Hardened K3s Template Configuration + +The following reference template configuration is used in Rancher to create a hardened K3s custom cluster based on each CIS control in this guide. This reference does not include other required **cluster configuration** directives, which vary based on your environment. + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,ServiceAccount # CIS 1.2.15, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + enableNetworkPolicy: true + kubernetesVersion: # Define K3s version + rkeConfig: + machineGlobalConfig: + kube-apiserver-arg: + - enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount # CIS 1.2.15, 5.2, 1.2.13 + - audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml # CIS 3.2.1 + - audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log # CIS 1.2.18 + - audit-log-maxage=30 # CIS 1.2.19 + - audit-log-maxbackup=10 # CIS 1.2.20 + - audit-log-maxsize=100 # CIS 1.2.21 + - request-timeout=300s # CIS 1.2.22 + - service-account-lookup=true # CIS 1.2.24 + kube-controller-manager-arg: + - terminated-pod-gc-threshold=10 # CIS 1.3.1 + secrets-encryption: true + machineSelectorConfig: + - config: + kubelet-arg: + - make-iptables-util-chains=true # CIS 4.2.7 + protect-kernel-defaults: true # CIS 4.2.6 +``` + + + + +## Conclusion + +If you have followed this guide, your K3s custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our K3s self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-latest/pages-for-subheaders/kubernetes-cluster-setup.md b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-cluster-setup.md new file mode 100644 index 00000000000..9291d55bee4 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-cluster-setup.md @@ -0,0 +1,11 @@ +--- +title: "Don't have a Kubernetes cluster? Try one of these tutorials." +--- + + + + + +This section contains information on how to install a Kubernetes cluster that the Rancher server can be installed on. + +Rancher can run on any Kubernetes cluster. diff --git a/versioned_docs/version-latest/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md new file mode 100644 index 00000000000..899d4bb5937 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md @@ -0,0 +1,80 @@ +--- +title: Setting up Kubernetes Clusters in Rancher +description: Provisioning Kubernetes Clusters +--- + + + + + +Rancher simplifies the creation of clusters by allowing you to create them through the Rancher UI rather than more complex alternatives. Rancher provides multiple options for launching a cluster. Use the option that best fits your use case. + +This section assumes a basic familiarity with Docker and Kubernetes. For a brief explanation of how Kubernetes components work together, refer to the [concepts](../reference-guides/kubernetes-concepts.md) page. + +For a conceptual overview of how the Rancher server provisions clusters and what tools it uses to provision them, refer to the [architecture](rancher-manager-architecture.md) page. + + + +### Cluster Management Capabilities by Cluster Type + +The following table summarizes the options and settings available for each cluster type: + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## Setting up Clusters in a Hosted Kubernetes Provider + +In this scenario, Rancher does not provision Kubernetes because it is installed by providers such as Google Kubernetes Engine (GKE), Amazon Elastic Container Service for Kubernetes, or Azure Kubernetes Service. + +If you use a Kubernetes provider such as Google GKE, Rancher integrates with its cloud APIs, allowing you to create and manage role-based access control for the hosted cluster from the Rancher UI. + +For more information, refer to the section on [hosted Kubernetes clusters.](set-up-clusters-from-hosted-kubernetes-providers.md) + +## Launching Kubernetes with Rancher + +Rancher uses the [Rancher Kubernetes Engine (RKE)](https://rancher.com/docs/rke/latest/en/) as a library when provisioning Kubernetes on your own nodes. RKE is Rancher’s own lightweight Kubernetes installer. + +In RKE clusters, Rancher manages the deployment of Kubernetes. These clusters can be deployed on any bare metal server, cloud provider, or virtualization platform. + +These nodes can be dynamically provisioned through Rancher's UI, which calls [Docker Machine](https://docs.docker.com/machine/) to launch nodes on various cloud providers. + +If you already have a node that you want to add to an RKE cluster, you can add it to the cluster by running a Rancher agent container on it. + +For more information, refer to the section on [RKE clusters.](../pages-for-subheaders/launch-kubernetes-with-rancher.md) + +### Launching Kubernetes and Provisioning Nodes in an Infrastructure Provider + +Rancher can dynamically provision nodes in infrastructure providers such as Amazon EC2, DigitalOcean, Azure, or vSphere, then install Kubernetes on them. + +Using Rancher, you can create pools of nodes based on a [node template](use-new-nodes-in-an-infra-provider.md#node-templates). This template defines the parameters used to launch nodes in your cloud providers. + +One benefit of using nodes hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically replace it, thus maintaining the expected cluster configuration. + +The cloud providers available for creating a node template are decided based on the [node drivers](use-new-nodes-in-an-infra-provider.md#node-drivers) active in the Rancher UI. + +For more information, refer to the section on [nodes hosted by an infrastructure provider](use-new-nodes-in-an-infra-provider.md) + +### Launching Kubernetes on Existing Custom Nodes + +When setting up this type of cluster, Rancher installs Kubernetes on existing [custom nodes,](use-existing-nodes.md) which creates a custom cluster. + +You can bring any nodes you want to Rancher and use them to create a cluster. + +These nodes include on-prem bare metal servers, cloud-hosted virtual machines, or on-prem virtual machines. + +## Registering Existing Clusters + +The cluster registration feature replaces the feature to import clusters. + +Registering EKS clusters now provides additional benefits. For the most part, registered EKS clusters and EKS clusters created in Rancher are treated the same way in the Rancher UI, except for deletion. + +When you delete an EKS cluster that was created in Rancher, the cluster is destroyed. When you delete an EKS cluster that was registered in Rancher, it is disconnected from the Rancher server, but it still exists and you can still access it in the same way you did before it was registered in Rancher. + +For more information, see [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md) + +## Programmatically Creating Clusters + +The most common way to programmatically deploy Kubernetes clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) + +EKS, GKE, AKS clusters and RKE clusters can be created or imported with Terraform. diff --git a/versioned_docs/version-latest/pages-for-subheaders/kubernetes-components.md b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-components.md new file mode 100644 index 00000000000..f048b5ba19b --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-components.md @@ -0,0 +1,21 @@ +--- +title: Kubernetes Components +--- + + + + + +The commands and steps listed in this section apply to the core Kubernetes components on [Rancher Launched Kubernetes](../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters. + +This section includes troubleshooting tips in the following categories: + +- [Troubleshooting etcd Nodes](../troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md) +- [Troubleshooting Controlplane Nodes](../troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md) +- [Troubleshooting nginx-proxy Nodes](../troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md) +- [Troubleshooting Worker Nodes and Generic Components](../troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md) + +## Kubernetes Component Diagram + +![Cluster diagram](/img/clusterdiagram.svg)
+Lines show the traffic flow between components. Colors are used purely for visual aid \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/kubernetes-resources-setup.md b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-resources-setup.md new file mode 100644 index 00000000000..865f5ae5c46 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/kubernetes-resources-setup.md @@ -0,0 +1,67 @@ +--- +title: Kubernetes Resources +--- + + + + + +You can view and manipulate all of the custom resources and CRDs in a Kubernetes cluster from the Rancher UI. + +## Workloads + +Deploy applications to your cluster nodes using [workloads](workloads-and-pods.md), which are objects that contain pods that run your apps, along with metadata that set rules for the deployment's behavior. Workloads can be deployed within the scope of the entire clusters or within a namespace. + +When deploying a workload, you can deploy from any image. There are a variety of [workload types](workloads-and-pods.md#workload-types) to choose from which determine how your application should run. + +Following a workload deployment, you can continue working with it. You can: + +- [Upgrade](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md) the workload to a newer version of the application it's running. +- [Roll back](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md) a workload to a previous version, if an issue occurs during upgrade. +- [Add a sidecar](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar.md), which is a workload that supports a primary workload. + +## Load Balancing and Ingress + +### Load Balancers + +After you launch an application, it's only available within the cluster. It can't be reached externally. + +If you want your applications to be externally accessible, you must add a load balancer to your cluster. Load balancers create a gateway for external connections to access your cluster, provided that the user knows the load balancer's IP address and the application's port number. + +Rancher supports two types of load balancers: + +- [Layer-4 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-4-load-balancer) +- [Layer-7 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-7-load-balancer) + +For more information, see [load balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md). + +#### Ingress + +Load Balancers can only handle one IP address per service, which means if you run multiple services in your cluster, you must have a load balancer for each service. Running multiples load balancers can be expensive. You can get around this issue by using an ingress. + +Ingress is a set of rules that act as a load balancer. Ingress works in conjunction with one or more ingress controllers to dynamically route service requests. When the ingress receives a request, the ingress controller(s) in your cluster program the load balancer to direct the request to the correct service based on service subdomains or path rules that you've configured. + +For more information, see [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md). + +When using ingresses in a project, you can program the ingress hostname to an external DNS by setting up a Global DNS entry. + +## Service Discovery + +After you expose your cluster to external requests using a load balancer and/or ingress, it's only available by IP address. To create a resolveable hostname, you must create a service record, which is a record that maps an IP address, external hostname, DNS record alias, workload(s), or labelled pods to a specific hostname. + +For more information, see [Service Discovery](../how-to-guides/new-user-guides/kubernetes-resources-setup/create-services.md). + +## Applications + +Besides launching individual components of an application, you can use the Rancher catalog to start launching applications, which are Helm charts. + +## Kubernetes Resources + +Within the context of a Rancher project or namespace, _resources_ are files and data that support operation of your pods. Within Rancher, certificates, registries, and secrets are all considered resources. However, Kubernetes classifies resources as different types of [secrets](https://kubernetes.io/docs/concepts/configuration/secret/). Therefore, within a single project or namespace, individual resources must have unique names to avoid conflicts. Although resources are primarily used to carry sensitive information, they have other uses as well. + +Resources include: + +- [Certificates](../how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication.md): Files used to encrypt/decrypt data entering or leaving the cluster. +- [ConfigMaps](../how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps.md): Files that store general configuration information, such as a group of config files. +- [Secrets](../how-to-guides/new-user-guides/kubernetes-resources-setup/secrets.md): Files that store sensitive data like passwords, tokens, or keys. +- [Registries](../how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries.md): Files that carry credentials used to authenticate with private registries. diff --git a/versioned_docs/version-latest/pages-for-subheaders/launch-kubernetes-with-rancher.md b/versioned_docs/version-latest/pages-for-subheaders/launch-kubernetes-with-rancher.md new file mode 100644 index 00000000000..5b7f4363bab --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/launch-kubernetes-with-rancher.md @@ -0,0 +1,84 @@ +--- +title: Launching Kubernetes with Rancher +--- + + + + + +You can have Rancher launch a Kubernetes cluster using any nodes you want. When Rancher deploys Kubernetes onto these nodes, you can choose between [Rancher Kubernetes Engine](https://rancher.com/docs/rke/latest/en/) (RKE) or [RKE2](https://docs.rke2.io) distributions. Rancher can launch Kubernetes on any computers, including: + +- Bare-metal servers +- On-premise virtual machines +- Virtual machines hosted by an infrastructure provider + +Rancher can install Kubernetes on existing nodes, or it can dynamically provision nodes in an infrastructure provider and install Kubernetes on them. + +Rancher can also create pools of nodes. One benefit of installing Kubernetes on node pools hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically create another node to join the cluster to ensure that the count of the node pool is as expected. + +## RKE + +### Requirements + +If you use RKE to set up a cluster, your nodes must meet the [requirements](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) for nodes in downstream user clusters. + +### Launching Kubernetes on New Nodes in an Infrastructure Provider + +Using Rancher, you can create pools of nodes based on a [node template](use-new-nodes-in-an-infra-provider.md#node-templates). This node template defines the parameters you want to use to launch nodes in your cloud providers. + +One benefit of installing Kubernetes on node pools hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically create another node to join the cluster to ensure that the count of the node pool is as expected. + +For more information, refer to the section on [launching Kubernetes on new nodes.](use-new-nodes-in-an-infra-provider.md) + +### Launching Kubernetes on Existing Custom Nodes + +In this scenario, you want to install Kubernetes on bare-metal servers, on-prem virtual machines, or virtual machines that already exist in a cloud provider. With this option, you will run a Rancher agent Docker container on the machine. + +If you want to reuse a node from a previous custom cluster, [clean the node](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) before using it in a cluster again. If you reuse a node that hasn't been cleaned, cluster provisioning may fail. + +For more information, refer to the section on [custom nodes.](use-existing-nodes.md) + +### Programmatically Creating RKE Clusters + +The most common way to programmatically deploy RKE clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster) + +## RKE2 + +Rancher v2.6 introduced provisioning for [RKE2](https://docs.rke2.io/) clusters directly from the Rancher UI. RKE2, also known as RKE Government, is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. In Rancher v.2.6.4 and earlier, RKE2 provisioning was in tech preview. + +Note that in Rancher v2.6.5, RKE2 provisioning became GA. + +### Requirements + +If you use RKE2 to set up a cluster, your nodes must meet the [requirements](https://docs.rke2.io/install/requirements) for nodes in downstream user clusters. + +### Launching Kubernetes on New Nodes in an Infrastructure Provider + +RKE2 provisioning is built on top of a new provisioning framework that leverages the upstream [Cluster API](https://github.com/kubernetes-sigs/cluster-api) project. With this new provisioning framework, you can: + +- Provision RKE2 clusters onto any provider for which Rancher has a node driver +- Fully configure RKE2 clusters within Rancher +- Choose CNI options Calico, Cilium, and Multus in addition to Canal + +RKE2 provisioning also includes installing RKE2 on clusters with Windows nodes. + +Windows features for RKE2 include: + +- Windows supports the vSphere node driver +- Calico CNI for Windows RKE2 custom clusters +- Project Network Isolation (PNI) for Calico +- Windows Containers with RKE2 powered by containerd +- Provisioning of Windows RKE2 clusters through Terraform +- Provisioning of Windows RKE2 custom clusters directly from the Rancher UI + +Windows Support for RKE2 Custom Clusters requires choosing Calico as the CNI. + +### Launching Kubernetes on Existing Custom Nodes + +RKE2 provisioning also allows you to install custom clusters on pre-provisioned VMs or bare-metal nodes. + +If you want to reuse a node from a previous custom cluster, clean the node before using it in a cluster again. If you reuse a node that hasn't been cleaned, cluster provisioning may fail. + +### Programmatically Creating RKE2 Clusters + +The most common way to programmatically deploy RKE2 clusters through Rancher is by using the Rancher2 Terraform provider. The documentation for creating clusters with Terraform is [here.](https://registry.terraform.io/providers/rancher/rancher2/latest/docs/resources/cluster_v2) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/load-balancer-and-ingress-controller.md b/versioned_docs/version-latest/pages-for-subheaders/load-balancer-and-ingress-controller.md new file mode 100644 index 00000000000..41bdf40a323 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/load-balancer-and-ingress-controller.md @@ -0,0 +1,65 @@ +--- +title: Set Up Load Balancer and Ingress Controller within Rancher +description: Learn how you can set up load balancers and ingress controllers to redirect service requests within Rancher, and learn about the limitations of load balancers +--- + + + + + +Within Rancher, you can set up load balancers and ingress controllers to redirect service requests. + +## Load Balancers + +After you launch an application, the app is only available within the cluster. It can't be reached from outside the cluster. + +If you want your applications to be externally accessible, you must add a load balancer or ingress to your cluster. Load balancers create a gateway for external connections to access your cluster, provided that the user knows the load balancer's IP address and the application's port number. + +Rancher supports two types of load balancers: + +- [Layer-4 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-4-load-balancer) +- [Layer-7 Load Balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#layer-7-load-balancer) + +For more information, see [load balancers](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md). + +### Load Balancer Limitations + +Load Balancers have a couple of limitations you should be aware of: + +- Load Balancers can only handle one IP address per service, which means if you run multiple services in your cluster, you must have a load balancer for each service. Running multiples load balancers can be expensive. + +- If you want to use a load balancer with a Hosted Kubernetes cluster (i.e., clusters hosted in GKE, EKS, or AKS), the load balancer must be running within that cloud provider's infrastructure. Please review the compatibility tables regarding support for load balancers based on how you've provisioned your clusters: + +- [Support for Layer-4 Load Balancing](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#support-for-layer-4-load-balancing) + +- [Support for Layer-7 Load Balancing](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing.md#support-for-layer-7-load-balancing) + +## Ingress + +As mentioned in the limitations above, the disadvantages of using a load balancer are: + +- Load Balancers can only handle one IP address per service. +- If you run multiple services in your cluster, you must have a load balancer for each service. +- It can be expensive to have a load balancer for every service. + +In contrast, when an ingress is used as the entrypoint into a cluster, the ingress can route traffic to multiple services with greater flexibility. It can map multiple HTTP requests to services without individual IP addresses for each service. + +Therefore, it is useful to have an ingress if you want multiple services to be exposed with the same IP address, the same Layer 7 protocol, or the same privileged node-ports: 80 and 443. + +Ingress works in conjunction with one or more ingress controllers to dynamically route service requests. When the ingress receives a request, the ingress controller(s) in your cluster direct the request to the correct service based on service subdomains or path rules that you've configured. + +Each Kubernetes Ingress resource corresponds roughly to a file in `/etc/nginx/sites-available/` containing a `server{}` configuration block, where requests for specific files and folders are configured. + +Your ingress, which creates a port of entry to your cluster similar to a load balancer, can reside within your cluster or externally. Ingress and ingress controllers residing in RKE-launched clusters are powered by [Nginx](https://www.nginx.com/). + +Ingress can provide other functionality as well, such as SSL termination, name-based virtual hosting, and more. + +:::note Using Rancher in a High Availability Configuration? + +Refrain from adding an Ingress to the `local` cluster. The Nginx Ingress Controller that Rancher uses acts as a global entry point for _all_ clusters managed by Rancher, including the `local` cluster. Therefore, when users try to access an application, your Rancher connection may drop due to the Nginx configuration being reloaded. We recommend working around this issue by deploying applications only in clusters that you launch using Rancher. + +::: + +- For more information on how to set up ingress in Rancher, see [Ingress](../how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses.md). +- For complete information about ingress and ingress controllers, see the [Kubernetes Ingress Documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) +- When using ingresses in a project, you can program the ingress hostname to an external DNS by setting up a Global DNS entry. diff --git a/versioned_docs/version-latest/pages-for-subheaders/logging.md b/versioned_docs/version-latest/pages-for-subheaders/logging.md new file mode 100644 index 00000000000..427422627f3 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/logging.md @@ -0,0 +1,121 @@ +--- +title: Rancher Integration with Logging Services +description: Rancher integrates with popular logging services. Learn the requirements and benefits of integrating with logging services, and enable logging on your cluster. +--- + + + + + +The [Logging operator](https://kube-logging.github.io/docs/) now powers Rancher's logging solution in place of the former, in-house solution. + +## Enabling Logging + +You can enable the logging for a Rancher managed cluster by going to the Apps page and installing the logging app. + +1. Go to the cluster where you want to install logging and click **Apps**. +1. Click the **Logging** app. +1. Scroll to the bottom of the Helm chart README and click **Install**. + +**Result:** The logging app is deployed in the `cattle-logging-system` namespace. + +## Uninstall Logging + +1. Go to the cluster where you want to install logging and click **Apps**. +1. Click **Installed Apps**. +1. Go to the `cattle-logging-system` namespace and check the boxes for `rancher-logging` and `rancher-logging-crd`. +1. Click **Delete**. +1. Confirm **Delete**. + +**Result** `rancher-logging` is uninstalled. + +## Architecture + +For more information about how the logging application works, see [this section.](../integrations-in-rancher/logging/logging-architecture.md) + + + +## Role-based Access Control + +Rancher logging has two roles, `logging-admin` and `logging-view`. For more information on how and when to use these roles, see [this page.](../integrations-in-rancher/logging/rbac-for-logging.md) + +## Configuring Logging Custom Resources + +To manage `Flows,` `ClusterFlows`, `Outputs`, and `ClusterOutputs`, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to configure logging custom resources and click **Explore**. +1. In the left navigation bar, click **Logging**. + +### Flows and ClusterFlows + +For help with configuring `Flows` and `ClusterFlows`, see [this page.](../integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows.md) + +### Outputs and ClusterOutputs + +For help with configuring `Outputs` and `ClusterOutputs`, see [this page.](../integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs.md) + +## Configuring the Logging Helm Chart + +For a list of options that can be configured when the logging application is installed or upgraded, see [this page.](../integrations-in-rancher/logging/logging-helm-chart-options.md) + +### Windows Support + +You can [enable logging](../integrations-in-rancher/logging/logging-helm-chart-options.md#enabledisable-windows-node-logging) from Windows nodes. + + +### Working with a Custom Docker Root Directory + +For details on using a custom Docker root directory, see [this section.](../integrations-in-rancher/logging/logging-helm-chart-options.md#working-with-a-custom-docker-root-directory) + + +### Working with Taints and Tolerations + +For information on how to use taints and tolerations with the logging application, see [this page.](../integrations-in-rancher/logging/taints-and-tolerations.md) + + +### Logging V2 with SELinux + +For information on enabling the logging application for SELinux-enabled nodes, see [this section.](../integrations-in-rancher/logging/logging-helm-chart-options.md#enabling-the-logging-application-to-work-with-selinux) + +### Additional Logging Sources + +By default, Rancher collects logs for control plane components and node components for all cluster types. In some cases additional logs can be collected. For details, see [this section.](../integrations-in-rancher/logging/logging-helm-chart-options.md#additional-logging-sources) + + +## Troubleshooting + +### The Logging Buffer Overloads Pods + +Depending on your configuration, the default buffer size may be too large and cause pod failures. One way to reduce the load is to lower the logger's flush interval. This prevents logs from overfilling the buffer. You can also add more flush threads to handle moments when many logs are attempting to fill the buffer at once. + +For a more complete description of how to configure the logging buffer to suit your organization's needs, see the official Logging operator documentation on [buffers](https://kube-logging.github.io/docs/configuration/plugins/outputs/buffer/) and on [Fluentd configuration](https://kube-logging.github.io/docs/logging-infrastructure/fluentd/). + +### The `cattle-logging` Namespace Being Recreated + +If your cluster previously deployed logging from the global view in the legacy Rancher UI, you may encounter an issue where its `cattle-logging` namespace is continually being recreated. + +The solution is to delete all `clusterloggings.management.cattle.io` and `projectloggings.management.cattle.io` custom resources from the cluster specific namespace in the management cluster. +The existence of these custom resources causes Rancher to create the `cattle-logging` namespace in the downstream cluster if it does not exist. + +The cluster namespace matches the cluster ID, so we need to find the cluster ID for each cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster you want to get the ID of and click **Explore**. +2. Copy the `` portion from one of the URLs below. The `` portion is the cluster namespace name. + +```bash +# Cluster Management UI +https:///c// + +# Cluster Dashboard +https:///dashboard/c// +``` + +Now that we have the `` namespace, we can delete the CRs that cause `cattle-logging` to be continually recreated. +*Warning:* ensure that logging, the version installed from the global view in the legacy Rancher UI, is not currently in use. + +```bash +kubectl delete crd clusterloggings.management.cattle.io -n +kubectl delete crd projectloggings.management.cattle.io -n +``` diff --git a/versioned_docs/version-latest/pages-for-subheaders/machine-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/machine-configuration.md new file mode 100644 index 00000000000..e1b9bb72f0a --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/machine-configuration.md @@ -0,0 +1,9 @@ +--- +title: Machine Configuration +--- + + + + + +Machine configuration is the arrangement of resources assigned to a virtual machine. Please see the docs for [Amazon EC2](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md), [DigitalOcean](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md), and [Azure](../reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md) to learn more. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/manage-clusters.md b/versioned_docs/version-latest/pages-for-subheaders/manage-clusters.md new file mode 100644 index 00000000000..e6f69885338 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/manage-clusters.md @@ -0,0 +1,36 @@ +--- +title: Cluster Administration +--- + + + + + +After you provision a cluster in Rancher, you can begin using powerful Kubernetes features to deploy and scale your containerized applications in development, testing, or production environments. + +:::note + +This section assumes a basic familiarity with Docker and Kubernetes. For a brief explanation of how Kubernetes components work together, refer to the [concepts](../reference-guides/kubernetes-concepts.md) page. + +::: + +## Managing Clusters in Rancher + +After clusters have been [provisioned into Rancher](kubernetes-clusters-in-rancher-setup.md), [cluster owners](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles) will need to manage these clusters. There are many different options of how to manage your cluster. + +import ClusterCapabilitiesTable from '../shared-files/_cluster-capabilities-table.md'; + + + +## Configuring Tools + +Rancher contains a variety of tools that aren't included in Kubernetes to assist in your DevOps operations. Rancher can integrate with external services to help your clusters run more efficiently. Tools are divided into the following categories: + +- Alerts +- Notifiers +- Logging +- Monitoring +- Istio Service Mesh +- OPA Gatekeeper + +Tools can be installed through **Apps.** diff --git a/versioned_docs/version-latest/pages-for-subheaders/manage-project-resource-quotas.md b/versioned_docs/version-latest/pages-for-subheaders/manage-project-resource-quotas.md new file mode 100644 index 00000000000..fa6f10ec84c --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/manage-project-resource-quotas.md @@ -0,0 +1,50 @@ +--- +title: Project Resource Quotas +--- + + + + + +In situations where several teams share a cluster, one team may overconsume the resources available: CPU, memory, storage, services, Kubernetes objects like pods or secrets, and so on. To prevent this overconsumption, you can apply a _resource quota_, which is a Rancher feature that limits the resources available to a project or namespace. + +This page is a how-to guide for creating resource quotas in existing projects. + +Resource quotas can also be set when a new project is created. For details, refer to the section on [creating new projects.](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md#creating-projects) + +Resource quotas in Rancher include the same functionality as the [native version of Kubernetes](https://kubernetes.io/docs/concepts/policy/resource-quotas/). In Rancher, resource quotas have been extended so that you can apply them to projects. For details on how resource quotas work with projects in Rancher, refer to [this page.](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas.md) + +### Applying Resource Quotas to Existing Projects + +Edit resource quotas when: + +- You want to limit the resources that a project and its namespaces can use. +- You want to scale the resources available to a project up or down when a research quota is already in effect. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to apply a resource quota and click **Explore**. +1. Click **Cluster > Projects/Namespaces**. +1. Make sure that the **Projects/Namespaces** page is in **Group by Project** view mode. + ![Screenshot highlighting the "Group by Project" icon, above the list of projects. It resembles a folder.](/img/edit-project-config-for-resource-quotas-group-by-project.png) + +1. Find the project that you want to add a resource quota to, and select the **⋮** that's on the same row as the project's name. + ![Screenshot highlighting triple dots icon at the end of the same row as the project name.](/img/edit-project-config-for-resource-quotas-dots.png) + +1. Select **Edit Config**. + +1. Expand **Resource Quotas** and click **Add Resource**. Alternatively, you can edit existing quotas. + +1. Select a Resource Type. For more information on types, see the [quota type reference.](../how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types.md) + +1. Enter values for the **Project Limit** and the **Namespace Default Limit**. + + | Field | Description | + | ----------------------- | -------------------------------------------------------------------------------------------------------- | + | Project Limit | The overall resource limit for the project. | + | Namespace Default Limit | The default resource limit available for each namespace. This limit is propagated to each namespace in the project. The combined limit of all project namespaces shouldn't exceed the project limit. | + +1. **Optional:** Add more quotas. + +1. Click **Create**. + +**Result:** The resource quota is applied to your project and namespaces. When you add more namespaces in the future, Rancher validates that the project can accommodate the namespace. If the project can't allocate the resources, you may still create namespaces, but they will be given a resource quota of 0. Subsequently, Rancher will not allow you to create any resources restricted by this quota. diff --git a/versioned_docs/version-latest/pages-for-subheaders/manage-projects.md b/versioned_docs/version-latest/pages-for-subheaders/manage-projects.md new file mode 100644 index 00000000000..be308c7e342 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/manage-projects.md @@ -0,0 +1,41 @@ +--- +title: Project Administration +--- + + + + + +_Projects_ are objects introduced in Rancher that help organize namespaces in your Kubernetes cluster. You can use projects to create multi-tenant clusters, which allows a group of users to share the same underlying resources without interacting with each other's applications. + +In terms of hierarchy: + +- Clusters contain projects +- Projects contain namespaces + +Within Rancher, projects allow you to manage multiple namespaces as a single entity. In native Kubernetes, which does not include projects, features like role-based access rights or cluster resources are assigned to individual namespaces. In clusters where multiple namespaces require the same set of access rights, assigning these rights to each individual namespace can become tedious. Even though all namespaces require the same rights, there's no way to apply those rights to all of your namespaces in a single action. You'd have to repetitively assign these rights to each namespace! + +Rancher projects resolve this issue by allowing you to apply resources and access rights at the project level. Each namespace in the project then inherits these resources and policies, so you only have to assign them to the project once, rather than assigning them to each individual namespace. + +You can use projects to perform actions like: + +- [Assign users access to a group of namespaces](../how-to-guides/new-user-guides/add-users-to-projects.md) +- Assign users [specific roles in a project](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles). A role can be owner, member, read-only, or [custom](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles.md) +- [Set resource quotas](manage-project-resource-quotas.md) +- [Manage namespaces](../how-to-guides/new-user-guides/manage-namespaces.md) +- [Configure tools](../reference-guides/rancher-project-tools.md) +- [Configure pod security policies](../how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies.md) + +### Authorization + +Non-administrative users are only authorized for project access after an [administrator](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md), [cluster owner or member](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#cluster-roles), or [project owner](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles) adds them to the project's **Members** tab. + +Whoever creates the project automatically becomes a [project owner](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md#project-roles). + +## Switching between Projects + +To switch between projects, use the drop-down available in the navigation bar. Alternatively, you can switch between projects directly in the navigation bar. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to switch projects and click **Explore**. +1. In the top navigation bar, select the project that you want to open. diff --git a/versioned_docs/version-latest/pages-for-subheaders/manage-role-based-access-control-rbac.md b/versioned_docs/version-latest/pages-for-subheaders/manage-role-based-access-control-rbac.md new file mode 100644 index 00000000000..c6c87eff5fc --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/manage-role-based-access-control-rbac.md @@ -0,0 +1,29 @@ +--- +title: Role-Based Access Control (RBAC) +--- + + + + + +Within Rancher, each person authenticates as a _user_, which is a login that grants you access to Rancher. As mentioned in [Authentication](authentication-config.md), users can either be local or external. + +After you configure external authentication, the users that display on the **Users** page changes. + +- If you are logged in as a local user, only local users display. + +- If you are logged in as an external user, both external and local users display. + +## Users and Roles + +Once the user logs in to Rancher, their _authorization_, or their access rights within the system, is determined by _global permissions_, and _cluster and project roles_. + +- [Global Permissions](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions.md): + + Define user authorization outside the scope of any particular cluster. + +- [Cluster and Project Roles](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles.md): + + Define user authorization inside the specific cluster or project where they are assigned the role. + +Both global permissions and cluster and project roles are implemented on top of [Kubernetes RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/). Therefore, enforcement of permissions and roles is performed by Kubernetes. diff --git a/versioned_docs/version-latest/pages-for-subheaders/monitoring-alerting-guides.md b/versioned_docs/version-latest/pages-for-subheaders/monitoring-alerting-guides.md new file mode 100644 index 00000000000..97e3e801b26 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/monitoring-alerting-guides.md @@ -0,0 +1,14 @@ +--- +title: Monitoring Guides +--- + + + + + +- [Enable monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [Uninstall monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring workloads](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [Customizing Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [Persistent Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [Debugging high memory usage](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/monitoring-and-alerting.md b/versioned_docs/version-latest/pages-for-subheaders/monitoring-and-alerting.md new file mode 100644 index 00000000000..a4d38ca86dd --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/monitoring-and-alerting.md @@ -0,0 +1,99 @@ +--- +title: Monitoring and Alerting +description: Prometheus lets you view metrics from your different Rancher and Kubernetes objects. Learn about the scope of monitoring and how to enable cluster monitoring +--- + + + + + +The `rancher-monitoring` application can quickly deploy leading open-source monitoring and alerting solutions onto your cluster. + +Introduced in Rancher v2.5, the application is powered by [Prometheus](https://prometheus.io/), [Grafana](https://grafana.com/grafana/), [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/), the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator), and the [Prometheus adapter.](https://github.com/DirectXMan12/k8s-prometheus-adapter) + +For information on V1 monitoring and alerting, available in Rancher v2.2 up to v2.4, please see the Rancher v2.0—v2.4 docs on [cluster monitoring](../../version-2.0-2.4/pages-for-subheaders/cluster-monitoring.md), [alerting](../../version-2.0-2.4/pages-for-subheaders/cluster-alerts.md), [notifiers](../../version-2.0-2.4/explanations/integrations-in-rancher/notifiers.md) and other [tools](../../version-2.0-2.4/pages-for-subheaders/project-tools.md). + +Using the `rancher-monitoring` application, you can quickly deploy leading open-source monitoring and alerting solutions onto your cluster. + +### Features + +Prometheus lets you view metrics from your Rancher and Kubernetes objects. Using timestamps, Prometheus lets you query and view these metrics in easy-to-read graphs and visuals, either through the Rancher UI or Grafana, which is an analytics viewing platform deployed along with Prometheus. + +By viewing data that Prometheus scrapes from your cluster control plane, nodes, and deployments, you can stay on top of everything happening in your cluster. You can then use these analytics to better run your organization: stop system emergencies before they start, develop maintenance strategies, or restore crashed servers. + +The monitoring application: + +- Monitors the state and processes of your cluster nodes, Kubernetes components, and software deployments. +- Defines alerts based on metrics collected via Prometheus. +- Creates custom Grafana dashboards. +- Configures alert-based notifications via email, Slack, PagerDuty, etc. using Prometheus Alertmanager. +- Defines precomputed, frequently needed or computationally expensive expressions as new time series based on metrics collected via Prometheus. +- Exposes collected metrics from Prometheus to the Kubernetes Custom Metrics API via Prometheus Adapter for use in HPA. + +See [How Monitoring Works](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) for an explanation of how the monitoring components work together. + +## Default Components and Deployments + +### Built-in Dashboards + +By default, the monitoring application deploys Grafana dashboards (curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project) onto a cluster. + +It also deploys an Alertmanager UI and a Prometheus UI. For more information about these tools, see [Built-in Dashboards.](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md) +### Default Metrics Exporters + +By default, Rancher Monitoring deploys exporters (such as [node-exporter](https://github.com/prometheus/node_exporter) and [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics)). + +These default exporters automatically scrape metrics for CPU and memory from all components of your Kubernetes cluster, including your workloads. + +### Default Alerts + +The monitoring application deploys some alerts by default. To see the default alerts, go to the [Alertmanager UI](../integrations-in-rancher/monitoring-and-alerting/built-in-dashboards.md#alertmanager-ui) and click **Expand all groups.** + +### Components Exposed in the Rancher UI + +For a list of monitoring components exposed in the Rancher UI, along with common use cases for editing them, see [this section.](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#components-exposed-in-the-rancher-ui) + +## Role-based Access Control + +For information on configuring access to monitoring, see [this page.](../integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring.md) + +## Guides + +- [Enable monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring.md) +- [Uninstall monitoring](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring.md) +- [Monitoring workloads](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads.md) +- [Customizing Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard.md) +- [Persistent Grafana dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard.md) +- [Debugging high memory usage](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) + +## Configuration + +### Configuring Monitoring Resources in Rancher + +The configuration reference assumes familiarity with how monitoring components work together. For more information, see [How Monitoring Works.](../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +- [ServiceMonitor and PodMonitor](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [Receiver](../reference-guides/monitoring-v2-configuration/receivers.md) +- [Route](../reference-guides/monitoring-v2-configuration/routes.md) +- [PrometheusRule](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules.md) +- [Prometheus](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) +- [Alertmanager](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### Configuring Helm Chart Options + +For more information on `rancher-monitoring` chart options, including options to set resource limits and requests, see [Helm Chart Options](../reference-guides/monitoring-v2-configuration/helm-chart-options.md). + +## Windows Cluster Support + +When deployed onto an RKE1 Windows cluster, Monitoring V2 will now automatically deploy a [windows-exporter](https://github.com/prometheus-community/windows_exporter) DaemonSet and set up a ServiceMonitor to collect metrics from each of the deployed Pods. This will populate Prometheus with `windows_` metrics that are akin to the `node_` metrics exported by [node_exporter](https://github.com/prometheus/node_exporter) for Linux hosts. + +To be able to fully deploy Monitoring V2 for Windows, all of your Windows hosts must have a minimum [wins](https://github.com/rancher/wins) version of v0.1.0. + +For more details on how to upgrade wins on existing Windows hosts, see [Windows cluster support for Monitoring V2.](../integrations-in-rancher/monitoring-and-alerting/windows-support.md). + + +## Known Issues + +There is a [known issue](https://github.com/rancher/rancher/issues/28787#issuecomment-693611821) that K3s clusters require more than the allotted default memory. If you enable monitoring on a K3s cluster, set `prometheus.prometheusSpec.resources.memory.limit` to 2500 Mi and `prometheus.prometheusSpec.resources.memory.request` to 1750 Mi. + +See [Debugging High Memory Usage](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage.md) for advice and recommendations. diff --git a/versioned_docs/version-latest/pages-for-subheaders/monitoring-v2-configuration-guides.md b/versioned_docs/version-latest/pages-for-subheaders/monitoring-v2-configuration-guides.md new file mode 100644 index 00000000000..bd0de341f46 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/monitoring-v2-configuration-guides.md @@ -0,0 +1,55 @@ +--- +title: Configuration +--- + + + + + +This page captures some of the most important options for configuring Monitoring V2 in the Rancher UI. + +For information on configuring custom scrape targets and rules for Prometheus, please refer to the upstream documentation for the [Prometheus Operator.](https://github.com/prometheus-operator/prometheus-operator) Some of the most important custom resources are explained in the Prometheus Operator [design documentation.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/design.md) The Prometheus Operator documentation can help also you set up RBAC, Thanos, or custom configuration. + +## Setting Resource Limits and Requests + +The resource requests and limits for the monitoring application can be configured when installing `rancher-monitoring`. For more information about the default limits, see [this page.](../reference-guides/monitoring-v2-configuration/helm-chart-options.md#configuring-resource-limits-and-requests) + +:::tip + +On an idle cluster, Monitoring may have high CPU usage. To improve performance, turn off the Prometheus adapter. + +::: + +## Prometheus Configuration + +It is usually not necessary to directly edit the Prometheus custom resource. + +Instead, to configure Prometheus to scrape custom metrics, you will only need to create a new ServiceMonitor or PodMonitor to configure Prometheus to scrape additional metrics. + + +### ServiceMonitor and PodMonitor Configuration + +For details, see [this page.](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) + +### Advanced Prometheus Configuration + +For more information about directly editing the Prometheus custom resource, which may be helpful in advanced use cases, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus.md) + +## Alertmanager Configuration + +The Alertmanager custom resource usually doesn't need to be edited directly. For most common use cases, you can manage alerts by updating Routes and Receivers. + +Routes and receivers are part of the configuration of the alertmanager custom resource. In the Rancher UI, Routes and Receivers are not true custom resources, but pseudo-custom resources that the Prometheus Operator uses to synchronize your configuration with the Alertmanager custom resource. When routes and receivers are updated, the monitoring application will automatically update Alertmanager to reflect those changes. + +For some advanced use cases, you may want to configure alertmanager directly. For more information, refer to [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) + +### Receivers + +Receivers are used to set up notifications. For details on how to configure receivers, see [this page.](../reference-guides/monitoring-v2-configuration/receivers.md) +### Routes + +Routes filter notifications before they reach receivers. Each route needs to refer to a receiver that has already been configured. For details on how to configure routes, see [this page.](../reference-guides/monitoring-v2-configuration/routes.md) + +### Advanced + +For more information about directly editing the Alertmanager custom resource, which may be helpful in advanced use cases, see [this page.](../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/monitoring-v2-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/monitoring-v2-configuration.md new file mode 100644 index 00000000000..79f97d9513d --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/monitoring-v2-configuration.md @@ -0,0 +1,15 @@ +--- +title: Monitoring V2 Configuration +--- + + + + + +The following sections will explain important options essential to configuring Monitoring V2 in Rancher: + +- [Receiver Configuration](../reference-guides/monitoring-v2-configuration/receivers.md) +- [Route Configuration](../reference-guides/monitoring-v2-configuration/routes.md) +- [ServiceMonitor and PodMonitor Configuration](../reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md) +- [Helm Chart Options](../reference-guides/monitoring-v2-configuration/helm-chart-options.md) +- [Examples](../reference-guides/monitoring-v2-configuration/examples.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/new-user-guides.md b/versioned_docs/version-latest/pages-for-subheaders/new-user-guides.md new file mode 100644 index 00000000000..e2e68ff0089 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/new-user-guides.md @@ -0,0 +1,13 @@ +--- +title: New User Guides +--- + + + + + +New user guides, also known as **tutorials**, describe practical steps for users to follow in order to complete some concrete action. These docs are known as "learning-oriented" docs in which users learn by "doing". + +The new user guides are designed to guide beginners, or the everyday users of Rancher, through a series of steps to learn how to do something. The goal is that the user will be able to learn how to complete tasks by using easy-to-follow, meaningful, and repeatable directions. These guides will assist users to do work to then get the promised results immediately. + +The average Rancher user has a level of technical skill that is above the level of "beginner"; however, the new user guides are designed to help new, or beginner, users as well as the seasoned Rancher customer equally. This is accomplished by using a combination of high-level and technical language to introduce topics and guide the user through general tasks that are essential for every Rancher user to know. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/node-template-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/node-template-configuration.md new file mode 100644 index 00000000000..e6c22d5e852 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/node-template-configuration.md @@ -0,0 +1,9 @@ +--- +title: Node Template Configuration +--- + + + + + +To learn about node template config, refer to [EC2 Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md), [DigitalOcean Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md), [Azure Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md), [vSphere Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md), and [Nutanix Node Template Configuration](../reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md). diff --git a/versioned_docs/version-latest/pages-for-subheaders/nutanix.md b/versioned_docs/version-latest/pages-for-subheaders/nutanix.md new file mode 100644 index 00000000000..b626cf2bd12 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/nutanix.md @@ -0,0 +1,21 @@ +--- +title: Creating a Nutanix AOS Cluster +description: Use Rancher to create a Nutanix AOS (AHV) cluster. It may consist of groups of VMs with distinct properties which allow for fine-grained control over the sizing of nodes. +--- + + + + + +[Nutanix Acropolis Operating System](https://www.nutanix.com/products/acropolis) (Nutanix AOS) is an operating system for the Nutanix hyper-converged infrastructure platform. AOS comes with a built-in hypervisor called [Acropolis Hypervisor](https://www.nutanix.com/products/ahv), or AHV. By using Rancher with Nutanix AOS (AHV), you can bring cloud operations on-premises. + +Rancher can provision nodes in AOS (AHV) and install Kubernetes on them. When creating a Kubernetes cluster in AOS, Rancher first provisions the specified number of virtual machines by communicating with the Prism Central API. Then it installs Kubernetes on top of the VMs. + +A Nutanix cluster may consist of multiple groups of VMs with distinct properties, such as the amount of memory or the number of vCPUs. This grouping allows for fine-grained control over the sizing of nodes for each Kubernetes role. + +- [Creating a Nutanix Cluster](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md#creating-a-nutanix-aos-cluster) +- [Provisioning Storage](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md) + +## Creating a Nutanix Cluster + +In [this section,](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in Nutanix AOS. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/other-installation-methods.md b/versioned_docs/version-latest/pages-for-subheaders/other-installation-methods.md new file mode 100644 index 00000000000..7cd497a8d48 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/other-installation-methods.md @@ -0,0 +1,23 @@ +--- +title: Other Installation Methods +--- + + + + + +### Air Gapped Installations + +Follow [these steps](air-gapped-helm-cli-install.md) to install the Rancher server in an air gapped environment. + +An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +### Docker Installations + +The [single-node Docker installation](rancher-on-a-single-node-with-docker.md) is for Rancher users that are wanting to test out Rancher. Instead of running on a Kubernetes cluster using Helm, you install the Rancher server component on a single node using a `docker run` command. + +The Docker installation is for development and testing environments only. + +Since there is only one node and a single Docker container, if the node goes down, there is no copy of the etcd data available on other nodes and you will lose all the data of your Rancher server. + +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/prometheus-federator-guides.md b/versioned_docs/version-latest/pages-for-subheaders/prometheus-federator-guides.md new file mode 100644 index 00000000000..2d1c0ae8224 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/prometheus-federator-guides.md @@ -0,0 +1,12 @@ +--- +title: Prometheus Federator Guides +--- + + + + + +- [Enable Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator.md) +- [Uninstall Prometheus Operator](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator.md) +- [Customize Grafana Dashboards](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards.md) +- [Set Up Workloads](../how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/prometheus-federator.md b/versioned_docs/version-latest/pages-for-subheaders/prometheus-federator.md new file mode 100644 index 00000000000..efef5f5abae --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/prometheus-federator.md @@ -0,0 +1,108 @@ +--- +title: Prometheus Federator +--- + + + + + +Prometheus Federator, also referred to as Project Monitoring v2, deploys a Helm Project Operator (based on the [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator)), an operator that manages deploying Helm charts each containing a Project Monitoring Stack, where each stack contains: + +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed resources + +:::note Important: + +Prometheus Federator is designed to be deployed alongside an existing Prometheus Operator deployment in a cluster that has already installed the Prometheus Operator CRDs. + +::: + +## How does the operator work? + +1. On deploying this chart, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `monitoring.cattle.io/v1alpha1` (also known as "Project Monitors" in the Rancher UI) in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy a Project Prometheus stack on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--monitoring`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack deployed; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). For more information, see the section on [configuring RBAC](../reference-guides/prometheus-federator/rbac.md). + +### What is a Project? + +In Prometheus Federator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`. By default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given Rancher Project. + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart's resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: + +- View the chart's definition located at [`rancher/prometheus-federator` under `charts/rancher-project-monitoring`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) (where the chart version will be tied to the version of this operator). +- Look for the ConfigMap named `monitoring.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `prometheus-federator` binary). + +### Namespaces + +As a Project Operator based on [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator), Prometheus Federator has three different classifications of namespaces that the operator looks out for: + +1. **Operator / System Namespace**: The namespace that the operator is deployed into (e.g., `cattle-monitoring-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** + +2. **Project Registration Namespace (`cattle-project-`)**: The set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace. For details, refer to the [RBAC page](../reference-guides/prometheus-federator/rbac.md). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace.** + + :::note Notes: + + - Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided, which is set to `field.cattle.io/projectId` by default. This indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g., this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted), or it is no longer watching that project because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects. These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project. + + - If `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace. + + ::: + +3. **Project Release Namespace (`cattle-project--monitoring`):** The set of namespaces that the operator deploys Project Monitoring Stacks within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Project Monitoring Stack based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Prometheus Federator.** + + :::note Notes: + + - Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue`, which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified, whenever a ProjectHelmChart is specified in a Project Registration Namespace. + + - Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above). + + - If `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace. + + ::: + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Prometheus Federator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: + +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): This custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR. This CR is automatically updated on changes to the ProjectHelmChart (e.g., modifying the values.yaml) or changes to the underlying Project definition (e.g., adding or removing namespaces from a project). + +:::note Important: + +If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation. However, this is generally only accessible by a **Cluster Admin.** + +::: + +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): This custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). + +:::note + +HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place. To view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources modifications were attempted on. + +::: + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### Advanced Helm Project Operator Configuration + +For more information on advanced configurations, refer to [this page](https://github.com/rancher/prometheus-federator/blob/main/charts/prometheus-federator/0.0.1/README.md#advanced-helm-project-operator-configuration). + + + +### Prometheus Federator on the Local Cluster + +Prometheus Federator is a resource intensive application. Installing it to the local cluster is possible, but **not recommended**. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/provisioning-storage-examples.md b/versioned_docs/version-latest/pages-for-subheaders/provisioning-storage-examples.md new file mode 100644 index 00000000000..44fd9593fba --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/provisioning-storage-examples.md @@ -0,0 +1,15 @@ +--- +title: Provisioning Storage Examples +--- + + + + + +Rancher supports persistent storage with a variety of volume plugins. However, before you use any of these plugins to bind persistent storage to your workloads, you have to configure the storage itself, whether its a cloud-based solution from a service-provider or an on-prem solution that you manage yourself. + +For your convenience, Rancher offers documentation on how to configure some of the popular storage methods: + +- [NFS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage.md) +- [vSphere](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) +- [EBS](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/quick-start-guides.md b/versioned_docs/version-latest/pages-for-subheaders/quick-start-guides.md new file mode 100644 index 00000000000..8a7f8028dfb --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/quick-start-guides.md @@ -0,0 +1,20 @@ +--- +title: Rancher Deployment Quick Start Guides +--- + + + + +:::caution + +The intent of these guides is to quickly launch a sandbox that you can use to evaluate Rancher. These guides are not intended for production environments. For comprehensive setup instructions, see [Installation](installation-and-upgrade.md). + +::: + +Use this section of the docs to jump start your deployment and testing of Rancher 2.x. It contains instructions for a simple Rancher setup and some common use cases. We plan on adding more content to this section in the future. + +We have Quick Start Guides for: + +- [Deploying Rancher Server](deploy-rancher-manager.md): Get started running Rancher using the method most convenient for you. + +- [Deploying Workloads](deploy-rancher-workloads.md): Deploy a simple [workload](https://kubernetes.io/docs/concepts/workloads/) and expose it, letting you access it from outside the cluster. diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-behind-an-http-proxy.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-behind-an-http-proxy.md new file mode 100644 index 00000000000..fd8a41b8e08 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-behind-an-http-proxy.md @@ -0,0 +1,17 @@ +--- +title: Installing Rancher behind an HTTP Proxy +--- + + + + + +In a lot of enterprise environments, servers or VMs running on premise do not have direct Internet access, but must connect to external services through a HTTP(S) proxy for security reasons. This tutorial shows step by step how to set up a highly available Rancher installation in such an environment. + +Alternatively, it is also possible to set up Rancher completely air-gapped without any Internet access. This process is described in detail in the [Rancher docs](air-gapped-helm-cli-install.md). + +## Installation Outline + +1. [Set up infrastructure](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure.md) +2. [Set up a Kubernetes cluster](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes.md) +3. [Install Rancher](../getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-hardening-guides.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-hardening-guides.md new file mode 100644 index 00000000000..5db6c192945 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-hardening-guides.md @@ -0,0 +1,56 @@ +--- +title: Self-Assessment and Hardening Guides for Rancher +--- + + + + + +Rancher provides specific security hardening guides for each supported Rancher version's Kubernetes distributions. + +## Rancher Kubernetes Distributions + +Rancher uses the following Kubernetes distributions: + +- [**RKE**](https://rancher.com/docs/rke/latest/en/), Rancher Kubernetes Engine, is a CNCF-certified Kubernetes distribution that runs entirely within Docker containers. +- [**RKE2**](https://docs.rke2.io/) is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. +- [**K3s**](https://docs.k3s.io/) is a fully conformant, lightweight Kubernetes distribution. It is easy to install, with half the memory requirement of upstream Kubernetes, all in a binary of less than 100 MB. + +To harden a Kubernetes cluster that's running a distribution other than those listed, refer to your Kubernetes provider docs. + +## Hardening Guides and Benchmark Versions + +Each self-assessment guide is accompanied by a hardening guide. These guides were tested alongside the listed Rancher releases. Each self-assessment guides was tested on a specific Kubernetes version and CIS benchmark version. If a CIS benchmark has not been validated for your Kubernetes version, you can use the existing guides until a guide for your version is added. + +### RKE Guides + +| Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|--------------------|-----------------------|-----------------------|------------------| +| Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke1-hardening-guide.md) | +| Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke1-hardening-guide.md) | + +### RKE2 Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned RKE2 | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](rke2-hardening-guide.md) | +| Rancher provisioned RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](rke2-hardening-guide.md) | +| Standalone RKE2 | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](https://docs.rke2.io/security/cis_self_assessment123) | [Link](https://docs.rke2.io/security/hardening_guide) | + +### K3s Guides + +| Type | Kubernetes Version | CIS Benchmark Version | Self Assessment Guide | Hardening Guides | +|------|--------------------|-----------------------|-----------------------|------------------| +| Rancher provisioned K3s cluster | Kubernetes v1.23 | CIS v1.23 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.24 | CIS v1.24 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md) | [Link](k3s-hardening-guide.md) | +| Rancher provisioned K3s cluster | Kubernetes v1.25/v1.26/v1.27 | CIS v1.7 | [Link](../reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md) | [Link](k3s-hardening-guide.md) | +| Standalone K3s | Kubernetes v1.22 up to v1.24 | CIS v1.23 | [Link](https://docs.k3s.io/security/self-assessment) | [Link](https://docs.k3s.io/security/hardening-guide) | + +## Rancher with SELinux + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a kernel module that adds extra access controls and security tools to Linux. Historically used by government agencies, SELinux is now industry-standard. SELinux is enabled by default on RHEL and CentOS. + +To use Rancher with SELinux, we recommend [installing](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) the `rancher-selinux` RPM. + diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-managed-clusters.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-managed-clusters.md new file mode 100644 index 00000000000..2cdb03fd909 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-managed-clusters.md @@ -0,0 +1,23 @@ +--- +title: Best Practices for Rancher Managed Clusters +--- + + + + + +### Logging + +Refer to [this guide](../reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md) for our recommendations for cluster-level logging and application logging. + +### Monitoring + +Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. Refer to this [guide](../reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md) for our recommendations. + +### Tips for Setting Up Containers + +Running well-built containers can greatly impact the overall performance and security of your environment. Refer to this [guide](../reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md) for tips. + +### Best Practices for Rancher Managed vSphere Clusters + +This [guide](../reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md) outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-manager-architecture.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-manager-architecture.md new file mode 100644 index 00000000000..d7e76f28573 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-manager-architecture.md @@ -0,0 +1,21 @@ +--- +title: Architecture +--- + + + + + +This section focuses on the [Rancher server and its components](../reference-guides/rancher-manager-architecture/rancher-server-and-components.md) and how [Rancher communicates with downstream Kubernetes clusters](../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md). + +For information on the different ways that Rancher can be installed, refer to the [overview of installation options.](installation-and-upgrade.md#overview-of-installation-options) + +For a list of main features of the Rancher API server, refer to the [overview section.](../getting-started/overview.md#features-of-the-rancher-api-server) + +For guidance about setting up the underlying infrastructure for the Rancher server, refer to the [architecture recommendations.](../reference-guides/rancher-manager-architecture/architecture-recommendations.md) + +:::note + +This section assumes a basic familiarity with Docker and Kubernetes. For a brief explanation of how Kubernetes components work together, refer to the [concepts](../reference-guides/kubernetes-concepts.md) page. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-on-a-single-node-with-docker.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-on-a-single-node-with-docker.md new file mode 100644 index 00000000000..da5b39209cf --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-on-a-single-node-with-docker.md @@ -0,0 +1,211 @@ +--- +title: Installing Rancher on a Single Node Using Docker +description: For development and testing environments only, use a Docker install. Install Docker on a single Linux host, and deploy Rancher with a single Docker container. +--- + + + + + +Rancher can be installed by running a single Docker container. + +In this installation scenario, you'll install Docker on a single Linux host, and then deploy Rancher on your host using a single Docker container. + +:::note Want to use an external load balancer? + +See [Docker Install with an External Load Balancer](../how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer.md) instead. + +::: + +A Docker installation of Rancher is recommended only for development and testing purposes. The ability to migrate Rancher to a high-availability cluster depends on the Rancher version: + +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster.](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) + +## Privileged Access for Rancher + +When the Rancher server is deployed in the Docker container, a local Kubernetes cluster is installed within the container for Rancher to use. Because many features of Rancher run as deployments, and privileged mode is required to run containers within containers, you will need to install Rancher with the `--privileged` option. + +## Requirements for OS, Docker, Hardware, and Networking + +Make sure that your node fulfills the general [installation requirements.](installation-requirements.md) + +## 1. Provision Linux Host + +Provision a single Linux host according to our [Requirements](installation-requirements.md) to launch your Rancher server. + +## 2. Choose an SSL Option and Install Rancher + +For security purposes, SSL (Secure Sockets Layer) is required when using Rancher. SSL secures all Rancher network communication, like when you login or interact with a cluster. + +:::tip Do you want to.. + +- Use a proxy? See [HTTP Proxy Configuration](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) +- Configure custom CA root certificate to access your services? See [Custom CA root certificate](../reference-guides/single-node-rancher-in-docker/advanced-options.md#custom-ca-certificate/) +- Complete an Air Gap Installation? See [Air Gap: Docker Install](air-gapped-helm-cli-install.md) +- Record all transactions with the Rancher API? See [API Auditing](../reference-guides/single-node-rancher-in-docker/advanced-options.md#api-audit-log) + +::: + +Choose from the following options: + +- [Option A: Default Rancher-generated Self-signed Certificate](#option-a-default-rancher-generated-self-signed-certificate) +- [Option B: Bring Your Own Certificate, Self-signed](#option-b-bring-your-own-certificate-self-signed) +- [Option C: Bring Your Own Certificate, Signed by a Recognized CA](#option-c-bring-your-own-certificate-signed-by-a-recognized-ca) +- [Option D: Let's Encrypt Certificate](#option-d-lets-encrypt-certificate) +- [Option E: Localhost tunneling, no Certificate](#option-e-localhost-tunneling-no-certificate) + +### Option A: Default Rancher-generated Self-signed Certificate + +If you are installing Rancher in a development or testing environment where identity verification isn't a concern, install Rancher using the self-signed certificate that it generates. This installation option omits the hassle of generating a certificate yourself. + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest +``` + +### Option B: Bring Your Own Certificate, Self-signed +In development or testing environments where your team will access your Rancher server, create a self-signed certificate for use with your install so that your team can verify they're connecting to your instance of Rancher. + +:::note Prerequisites: + +Create a self-signed certificate using [OpenSSL](https://www.openssl.org/) or another method of your choice. + +- The certificate files must be in PEM format. +- In your certificate file, include all intermediate certificates in the chain. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +::: + +After creating your certificate, run the Docker command below to install Rancher. Use the `-v` flag and provide the path to your certificates to mount them in your container. + +| Placeholder | Description | +| ------------------- | --------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | +| `` | The path to the certificate authority's certificate. | + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + -v //:/etc/rancher/ssl/cacerts.pem \ + --privileged \ + rancher/rancher:latest +``` + +### Option C: Bring Your Own Certificate, Signed by a Recognized CA + +In production environments where you're exposing an app publicly, you would use a certificate signed by a recognized CA so that your user base doesn't encounter security warnings. + +The Docker install is not recommended for production. These instructions are provided for testing and development purposes only. + +:::note Prerequisites: + +- The certificate files must be in PEM format. +- In your certificate file, include all intermediate certificates provided by the recognized CA. Order your certificates with your certificate first, followed by the intermediates. For an example, see [Certificate Troubleshooting.](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) + +::: + +After obtaining your certificate, run the Docker command below. + +- Use the `-v` flag and provide the path to your certificates to mount them in your container. Because your certificate is signed by a recognized CA, mounting an additional CA certificate file is unnecessary. +- Use the `--no-cacerts` as argument to the container to disable the default CA certificate generated by Rancher. + +| Placeholder | Description | +| ------------------- | ----------------------------- | +| `` | The path to the directory containing your certificate files. | +| `` | The path to your full certificate chain. | +| `` | The path to the private key for your certificate. | + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v //:/etc/rancher/ssl/cert.pem \ + -v //:/etc/rancher/ssl/key.pem \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +### Option D: Let's Encrypt Certificate + +:::caution + +Let's Encrypt provides rate limits for requesting new certificates. Therefore, limit how often you create or destroy the container. For more information, see [Let's Encrypt documentation on rate limits](https://letsencrypt.org/docs/rate-limits/). + +::: + +For production environments, you also have the option of using [Let's Encrypt](https://letsencrypt.org/) certificates. Let's Encrypt uses an http-01 challenge to verify that you have control over your domain. You can confirm that you control the domain by pointing the hostname that you want to use for Rancher access (for example, `rancher.mydomain.com`) to the IP of the machine it is running on. You can bind the hostname to the IP address by creating an A record in DNS. + +The Docker install is not recommended for production. These instructions are provided for testing and development purposes only. + +:::note Prerequisites: + +- Let's Encrypt is an Internet service. Therefore, this option cannot be used in an internal/air gapped network. +- Create a record in your DNS that binds your Linux host IP address to the hostname that you want to use for Rancher access (`rancher.mydomain.com` for example). +- Open port `TCP/80` on your Linux host. The Let's Encrypt http-01 challenge can come from any source IP address, so port `TCP/80` must be open to all IP addresses. + +::: + +After you fulfill the prerequisites, you can install Rancher using a Let's Encrypt certificate by running the following command. + +| Placeholder | Description | +| ----------------- | ------------------- | +| `` | Your domain address | + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --acme-domain +``` + +### Option E: Localhost tunneling, no Certificate + +If you are installing Rancher in a development or testing environment where you have a localhost tunneling solution running, such as [ngrok](https://ngrok.com/), avoid generating a certificate. This installation option doesn't require a certificate. + +- You will use `--no-cacerts` in the argument to disable the default CA certificate generated by Rancher. + +Log into your host, and run the command below: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + --privileged \ + rancher/rancher:latest \ + --no-cacerts +``` + +## Advanced Options + +When installing Rancher on a single node with Docker, there are several advanced options that can be enabled: + +- Custom CA Certificate +- API Audit Log +- TLS Settings +- Air Gap +- Persistent Data +- Running `rancher/rancher` and `rancher/rancher-agent` on the Same Node + +Refer to [this page](../reference-guides/single-node-rancher-in-docker/advanced-options.md) for details. + +## Troubleshooting + +Refer to [this page](../getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting.md) for frequently asked questions and troubleshooting tips. + +## What's Next? + +- **Recommended:** Review Single Node [Backup](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher.md) and [Restore](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher.md). Although you don't have any data you need to back up right now, we recommend creating backups after regular Rancher use. +- Create a Kubernetes cluster: [Provisioning Kubernetes Clusters](kubernetes-clusters-in-rancher-setup.md). diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-security.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-security.md new file mode 100644 index 00000000000..5a30b09c77b --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-security.md @@ -0,0 +1,89 @@ +--- +title: Security +--- + + + + + + + + + + + +
+

Security policy

+

Rancher Labs supports responsible disclosure, and endeavours to resolve all issues in a reasonable time frame.

+		 +

Reporting process

+

Please submit possible security issues by emailing security-rancher@suse.com .

+		 +

Announcements

+

Subscribe to the Rancher announcements forum for release updates.

+
+ +Security is at the heart of all Rancher features. From integrating with all the popular authentication tools and services, to an enterprise grade [RBAC capability](manage-role-based-access-control-rbac.md), Rancher makes your Kubernetes clusters even more secure. + +On this page, we provide security related documentation along with resources to help you secure your Rancher installation and your downstream Kubernetes clusters. + +### NeuVector Integration with Rancher + +NeuVector is an open-source, container-focused security application that is now integrated into Rancher. NeuVector provides production security, DevOps vulnerability protection, and a container firewall, et al. Please see the [Rancher docs](../integrations-in-rancher/neuvector.md) and the [NeuVector docs](https://open-docs.neuvector.com/) for more information. + +### Running a CIS Security Scan on a Kubernetes Cluster + +Rancher leverages [kube-bench](https://github.com/aquasecurity/kube-bench) to run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the [CIS](https://www.cisecurity.org/cis-benchmarks/) (Center for Internet Security) Kubernetes Benchmark. + +The CIS Kubernetes Benchmark is a reference document that can be used to establish a secure configuration baseline for Kubernetes. + +The Center for Internet Security (CIS) is a 501(c\)(3) non-profit organization, formed in October 2000, with a mission to "identify, develop, validate, promote, and sustain best practice solutions for cyber defense and build and lead communities to enable an environment of trust in cyberspace". + +CIS Benchmarks are best practices for the secure configuration of a target system. CIS Benchmarks are developed through the generous volunteer efforts of subject matter experts, technology vendors, public and private community members, and the CIS Benchmark Development team. + +The Benchmark provides recommendations of two types: Automated and Manual. We run tests related to only Automated recommendations. + +When Rancher runs a CIS security scan on a cluster, it generates a report showing the results of each test, including a summary with the number of passed, skipped and failed tests. The report also includes remediation steps for any failed tests. + +For details, refer to the section on [security scans](cis-scan-guides.md). + +### SELinux RPM + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. + +We provide two RPMs (Red Hat packages) that enable Rancher products to function properly on SELinux-enforcing hosts: `rancher-selinux` and `rke2-selinux`. For details, see [this page](selinux-rpm.md). + +### Rancher Hardening Guide + +The Rancher Hardening Guide is based on controls and best practices found in the CIS Kubernetes Benchmark from the Center for Internet Security. + +The hardening guides provide prescriptive guidance for hardening a production installation of Rancher. See Rancher's guides for [Self Assessment of the CIS Kubernetes Benchmark](#the-cis-benchmark-and-self-assessment) for the full list of security controls. + +> The hardening guides describe how to secure the nodes in your cluster, and it is recommended to follow a hardening guide before installing Kubernetes. + +Each version of the hardening guide is intended to be used with specific versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher. + +### The CIS Benchmark and Self-Assessment + +The benchmark self-assessment is a companion to the Rancher security hardening guide. While the hardening guide shows you how to harden the cluster, the benchmark guide is meant to help you evaluate the level of security of the hardened cluster. + +Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. This guide will walk through the various controls and provide updated example commands to audit compliance in Rancher created clusters. The original benchmark documents can be downloaded from the [CIS website](https://www.cisecurity.org/benchmark/kubernetes/). + +Each version of Rancher's self-assessment guide corresponds to specific versions of the hardening guide, Rancher, Kubernetes, and the CIS Benchmark. + +### Third-party Penetration Test Reports + +Rancher periodically hires third parties to perform security audits and penetration tests of the Rancher 2.x software stack. The environments under test follow the Rancher provided hardening guides at the time of the testing. Previous penetration test reports are available below. + +Results: + +- [Cure53 Pen Test - July 2019](https://releases.rancher.com/documents/security/pen-tests/2019/RAN-01-cure53-report.final.pdf) +- [Untamed Theory Pen Test - March 2019](https://releases.rancher.com/documents/security/pen-tests/2019/UntamedTheory-Rancher_SecurityAssessment-20190712_v5.pdf) + +### Rancher Security Advisories and CVEs + +Rancher is committed to informing the community of security issues in our products. For the list of CVEs (Common Vulnerabilities and Exposures) for issues we have resolved, refer to [this page.](../reference-guides/rancher-security/security-advisories-and-cves.md) + +### Kubernetes Security Best Practices + +For recommendations on securing your Kubernetes cluster, refer to the [Kubernetes Security Best Practices](../reference-guides/rancher-security/kubernetes-security-best-practices.md) guide. diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-server-configuration.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-server-configuration.md new file mode 100644 index 00000000000..5e18f69e740 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-server-configuration.md @@ -0,0 +1,16 @@ +--- +title: Rancher Server Configuration +--- + + + + + +- [RKE1 Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md) +- [RKE2 Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md) +- [K3s Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md) +- [EKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md) +- [AKS Cluster Configuration](../reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md) +- [GKE Cluster Configuration](../pages-for-subheaders/gke-cluster-configuration.md) +- [Use Existing Nodes](../pages-for-subheaders/use-existing-nodes.md) +- [Sync Clusters](../reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/rancher-server.md b/versioned_docs/version-latest/pages-for-subheaders/rancher-server.md new file mode 100644 index 00000000000..45c3917cd58 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rancher-server.md @@ -0,0 +1,21 @@ +--- +title: Best Practices for the Rancher Server +--- + + + + + +This guide contains our recommendations for running the Rancher server, and is intended to be used in situations in which Rancher manages downstream Kubernetes clusters. + +### Recommended Architecture and Infrastructure + +Refer to this [guide](../reference-guides/best-practices/rancher-server/tips-for-running-rancher.md) for our general advice for setting up the Rancher server on a high-availability Kubernetes cluster. + +### Deployment Strategies + +This [guide](../reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md) is designed to help you choose whether a regional deployment strategy or a hub-and-spoke deployment strategy is better for a Rancher server that manages downstream Kubernetes clusters. + +### Installing Rancher in a vSphere Environment + +This [guide](../reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md) outlines a reference architecture for installing Rancher in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/resources.md b/versioned_docs/version-latest/pages-for-subheaders/resources.md new file mode 100644 index 00000000000..52e61353441 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/resources.md @@ -0,0 +1,29 @@ +--- +title: Resources +--- + + + + + +### Docker Installations + +The [single-node Docker installation](rancher-on-a-single-node-with-docker.md) is for Rancher users that are wanting to test out Rancher. Instead of running on a Kubernetes cluster using Helm, you install the Rancher server component on a single node using a `docker run` command. + +Since there is only one node and a single Docker container, if the node goes down, there is no copy of the etcd data available on other nodes and you will lose all the data of your Rancher server. + +### Air-Gapped Installations + +Follow [these steps](air-gapped-helm-cli-install.md) to install the Rancher server in an air gapped environment. + +An air gapped environment could be where Rancher server will be installed offline, behind a firewall, or behind a proxy. + +### Advanced Options + +When installing Rancher, there are several advanced options that can be enabled during installation. Within each install guide, these options are presented. Learn more about these options: + +- [Custom CA Certificate](../getting-started/installation-and-upgrade/resources/custom-ca-root-certificates.md) +- [API Audit Log](../how-to-guides/advanced-user-guides/enable-api-audit-log.md) +- [TLS Settings](../getting-started/installation-and-upgrade/installation-references/tls-settings.md) +- [etcd configuration](../how-to-guides/advanced-user-guides/tune-etcd-for-large-installs.md) +- [Local System Charts for Air Gap Installations](../getting-started/installation-and-upgrade/resources/local-system-charts.md) | v2.3.0 | diff --git a/versioned_docs/version-latest/pages-for-subheaders/rke1-hardening-guide.md b/versioned_docs/version-latest/pages-for-subheaders/rke1-hardening-guide.md new file mode 100644 index 00000000000..4d6f97e18c7 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rke1-hardening-guide.md @@ -0,0 +1,509 @@ +--- +title: RKE Hardening Guide +--- + + + + + +This document provides prescriptive guidance for how to harden an RKE cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | + +:::note +- In Benchmark v1.24 and later, check id `4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated)` might fail, as `/etc/kubernetes/ssl/kube-ca.pem` is set to 644 by default. +- In Benchmark v1.7, the `--protect-kernel-defaults` (`4.2.6`) parameter isn't required anymore, and was removed by CIS. +::: + +For more details on how to evaluate a hardened RKE cluster against the official CIS benchmark, refer to the RKE self-assessment guides for specific Kubernetes and CIS benchmark versions. + +## Host-level requirements + +### Configure Kernel Runtime Parameters + +The following `sysctl` configuration is recommended for all nodes types in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.overcommit_memory=1 +vm.panic_on_oom=0 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Configure `etcd` user and group + +A user account and group for the **etcd** service is required to be set up before installing RKE. + +#### Create `etcd` user and group + +To create the **etcd** user and group run the following console commands. +The commands below use `52034` for **uid** and **gid** for example purposes. +Any valid unused **uid** or **gid** could also be used in lieu of `52034`. + +```bash +groupadd --gid 52034 etcd +useradd --comment "etcd service account" --uid 52034 --gid 52034 etcd --shell /usr/sbin/nologin +``` + +When deploying RKE through its cluster configuration `config.yml` file, update the `uid` and `gid` of the `etcd` user: + +```yaml +services: + etcd: + gid: 52034 + uid: 52034 +``` + +## Kubernetes runtime requirements + +### Configure `default` Service Account + +#### Set `automountServiceAccountToken` to `false` for `default` service accounts + +Kubernetes provides a default service account which is used by cluster workloads where no specific service account is assigned to the pod. +Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. +The default service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +Save the following configuration to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. +Be sure to `chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### Configure Network Policy + +#### Ensure that all Namespaces have Network Policies defined + +Running different applications on the same Kubernetes cluster creates a risk of one compromised application attacking a neighboring application. Network segmentation is important to ensure that containers can communicate only with those they are supposed to. A network policy is a specification of how selections of pods are allowed to communicate with each other and other network endpoints. + +Network Policies are namespace scoped. When a network policy is introduced to a given namespace, all traffic not allowed by the policy is denied. However, if there are no network policies in a namespace all traffic will be allowed into and out of the pods in that namespace. To enforce network policies, a container network interface (CNI) plugin must be enabled. This guide uses [Canal](https://github.com/projectcalico/canal) to provide the policy enforcement. Additional information about CNI providers can be found [here](https://www.suse.com/c/rancher_blog/comparing-kubernetes-cni-providers-flannel-calico-canal-and-weave/). + +Once a CNI provider is enabled on a cluster a default network policy can be applied. For reference purposes a **permissive** example is provided below. If you want to allow all traffic to all pods in a namespace (even if policies are added that cause some pods to be treated as “isolatedâ€), you can create a policy that explicitly allows all traffic in that namespace. Save the following configuration as `default-allow-all.yaml`. Additional [documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/) about network policies can be found on the Kubernetes site. + +:::caution +This network policy is just an example and is not recommended for production use. +::: + +```yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +``` + +Create a bash script file called `apply_networkPolicy_to_all_ns.sh`. Be sure to `chmod +x apply_networkPolicy_to_all_ns.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + kubectl apply -f default-allow-all.yaml -n ${namespace} +done +``` + +Execute this script to apply the `default-allow-all.yaml` configuration with the **permissive** `NetworkPolicy` to all namespaces. + +## Known Limitations + +- Rancher **exec shell** and **view logs** for pods are **not** functional in a hardened setup when only a public IP is provided when registering custom nodes. This functionality requires a private IP to be provided when registering the custom nodes. +- When setting `default_pod_security_policy_template_id:` to `restricted` or `restricted-noroot`, based on the pod security policies (PSP) [provided](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) by Rancher, Rancher creates `RoleBindings` and `ClusterRoleBindings` on the `default` service accounts. The CIS check 5.1.5 requires that the `default` service accounts have no roles or cluster roles bound to it apart from the defaults. In addition, the `default` service accounts should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +## Reference Hardened RKE `cluster.yml` Configuration + +The reference `cluster.yml` is used by the RKE CLI that provides the configuration needed to achieve a hardened installation of RKE. RKE [documentation](https://rancher.com/docs/rke/latest/en/installation/) provides additional details about the configuration items. This reference `cluster.yml` does not include the required `nodes` directive which will vary depending on your environment. Documentation for node configuration in RKE can be found [here](https://rancher.com/docs/rke/latest/en/config-options/nodes/). + +The example `cluster.yml` configuration file contains an Admission Configuration policy in the `services.kube-api.admission_configuration` field. This [sample](../reference-guides/rancher-security/psa-restricted-exemptions.md) policy contains the namespace exemptions necessary for an imported RKE cluster to run properly in Rancher, similar to Rancher's pre-defined [`rancher-restricted`](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) policy. + +If you prefer to use RKE's default `restricted` policy, then leave the `services.kube-api.admission_configuration` field empty and set `services.pod_security_configuration` to `restricted`. See [the RKE docs](https://rke.docs.rancher.com/config-options/services/pod-security-admission) for more information. + + + + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + # Leave `pod_security_configuration` out if you are setting a + # custom policy in `admission_configuration`. Otherwise set + # it to `restricted` to use RKE's pre-defined restricted policy, + # and remove everything inside `admission_configuration` field. + # + # pod_security_configuration: restricted + # + admission_configuration: + apiVersion: apiserver.config.k8s.io/v1 + kind: AdmissionConfiguration + plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [ calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator ] + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + generate_serving_certificate: true +addons: | + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +```yaml +# If you intend to deploy Kubernetes in an air-gapped environment, +# please consult the documentation on how to configure custom RKE images. +nodes: [] +kubernetes_version: # Define RKE version +services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + secrets_encryption_config: + enabled: true + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + generate_serving_certificate: true +addons: | + # Upstream Kubernetes restricted PSP policy + # https://github.com/kubernetes/website/blob/564baf15c102412522e9c8fc6ef2b5ff5b6e766c/content/en/examples/policy/restricted-psp.yaml + apiVersion: policy/v1beta1 + kind: PodSecurityPolicy + metadata: + name: restricted-noroot + spec: + privileged: false + # Required to prevent escalations to root. + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + # Assume that ephemeral CSI drivers & persistentVolumes set up by the cluster admin are safe to use. + - 'csi' + - 'persistentVolumeClaim' + - 'ephemeral' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Require the container to run without root privileges. + rule: 'MustRunAsNonRoot' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: psp:restricted-noroot + rules: + - apiGroups: + - extensions + resourceNames: + - restricted-noroot + resources: + - podsecuritypolicies + verbs: + - use + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: psp:restricted-noroot + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted-noroot + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:authenticated + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: default-allow-all + spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: default + automountServiceAccountToken: false +``` + + + + +## Reference Hardened RKE Cluster Template Configuration + +The reference RKE cluster template provides the minimum required configuration to achieve a hardened installation of Kubernetes. RKE templates are used to provision Kubernetes and define Rancher settings. Follow the Rancher [documentation](installation-and-upgrade.md) for additional information about installing RKE and its template details. + + + + +```yaml +# +# Cluster Config +# +default_pod_security_admission_configuration_template_name: rancher-restricted +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: false + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +```yaml +# +# Cluster Config +# +default_pod_security_policy_template_id: restricted-noroot +enable_network_policy: true +local_cluster_auth_endpoint: + enabled: true +name: # Define cluster name + +# +# Rancher Config +# +rancher_kubernetes_engine_config: + addon_job_timeout: 45 + authentication: + strategy: x509|webhook + kubernetes_version: # Define RKE version + services: + etcd: + uid: 52034 + gid: 52034 + kube-api: + audit_log: + enabled: true + event_rate_limit: + enabled: true + pod_security_policy: true + secrets_encryption_config: + enabled: true + kube-controller: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + kubelet: + extra_args: + feature-gates: RotateKubeletServerCertificate=true + protect-kernel-defaults: true + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 + generate_serving_certificate: true + scheduler: + extra_args: + tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + + + + +## Conclusion + +If you have followed this guide, your RKE custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-latest/pages-for-subheaders/rke2-hardening-guide.md b/versioned_docs/version-latest/pages-for-subheaders/rke2-hardening-guide.md new file mode 100644 index 00000000000..378050c8e45 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/rke2-hardening-guide.md @@ -0,0 +1,274 @@ +--- +title: RKE2 Hardening Guide +--- + + + + + +This document provides prescriptive guidance for how to harden an RKE2 cluster intended for production, before provisioning it with Rancher. It outlines the configurations and controls required for Center for Information Security (CIS) Kubernetes benchmark controls. + +:::note +This hardening guide describes how to secure the nodes in your cluster. We recommended that you follow this guide before you install Kubernetes. +::: + +This hardening guide is intended to be used for RKE2 clusters and is associated with the following versions of the CIS Kubernetes Benchmark, Kubernetes, and Rancher: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|------------------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25 up to v1.26 | + +:::note +- In Benchmark v1.24 and later, some check ids might fail due to new file permission requirements (600 instead of 644). Impacted check ids: `1.1.1`, `1.1.3`, `1.1.5`, `1.1.7`, `1.1.13`, `1.1.15`, `1.1.17`, `4.1.3`, `4.1.5` and `4.1.9`. + - In Benchmark v1.7, the `--protect-kernel-defaults` (4.2.6) parameter is not required anymore, and was removed by CIS. +::: + +For more details on how to evaluate a hardened RKE2 cluster against the official CIS benchmark, refer to the RKE2 self-assessment guides for specific Kubernetes and CIS benchmark versions. + +RKE2 passes a number of the Kubernetes CIS controls without modification, as it applies several security mitigations by default. There are some notable exceptions to this that require manual intervention to fully comply with the CIS Benchmark: + +1. RKE2 will not modify the host operating system. Therefore, you, the operator, must make a few host-level modifications. +2. Certain CIS controls for Network Policies and Pod Security Standards (or Pod Security Policies (PSP) on RKE2 versions prior to v1.25) will restrict the functionality of the cluster. You must opt into having RKE2 configure these for you. To help ensure these requirements are met, RKE2 can be started with the profile flag set to `cis-1.23` for v1.25 and newer or `cis-1.6` for v1.24 and older. + +## Host-level requirements + +There are two areas of host-level requirements: kernel parameters and etcd process/directory configuration. These are outlined in this section. + +### Ensure `protect-kernel-defaults` is set + + + + +The `protect-kernel-defaults` is no longer required since CIS benchmark 1.7. + + + + +This is a kubelet flag that will cause the kubelet to exit if the required kernel parameters are unset or are set to values that are different from the kubelet's defaults. + +The `protect-kernel-defaults` flag can be set in the cluster configuration in Rancher. + +```yaml +spec: + rkeConfig: + machineSelectorConfig: + - config: + protect-kernel-defaults: true +``` + + + + +### Set kernel parameters + +The following `sysctl` configuration is recommended for all nodes type in the cluster. Set the following parameters in `/etc/sysctl.d/90-kubelet.conf`: + +```ini +vm.panic_on_oom=0 +vm.overcommit_memory=1 +kernel.panic=10 +kernel.panic_on_oops=1 +``` + +Run `sudo sysctl -p /etc/sysctl.d/90-kubelet.conf` to enable the settings. + +### Ensure etcd is configured properly + +The CIS Benchmark requires that the etcd data directory be owned by the `etcd` user and group. This implicitly requires the etcd process run as the host-level `etcd` user. To achieve this, RKE2 takes several steps when started with a valid `cis-1.xx` profile: + +1. Check that the `etcd` user and group exists on the host. If they don't, exit with an error. +2. Create etcd's data directory with `etcd` as the user and group owner. +3. Ensure the etcd process is ran as the `etcd` user and group by setting the etcd static pod's `SecurityContext` appropriately. + +To meet the above requirements, you must: + +#### Create the etcd user + +On some Linux distributions, the `useradd` command will not create a group. The `-U` flag is included below to account for that. This flag tells `useradd` to create a group with the same name as the user. + +```bash +sudo useradd -r -c "etcd user" -s /sbin/nologin -M etcd -U +``` + +## Kubernetes runtime requirements + +The runtime requirements to pass the CIS Benchmark are centered around pod security, network policies and kernel parameters. Most of this is automatically handled by RKE2 when using a valid `cis-1.xx` profile, but some additional operator intervention is required. These are outlined in this section. + +### PodSecurity + +RKE2 always runs with some amount of pod security. + + + + +On v1.25 and newer, [Pod Security Admissions (PSAs)](https://kubernetes.io/docs/concepts/security/pod-security-admission/) are used for pod security. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.7-hardened` available in Rancher. + +```yaml +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 +``` + +When both the `defaultPodSecurityAdmissionConfigurationTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures the Pod Security Admission Controller with the PSA configuration template `rancher-restricted`, to enforce restricted mode in all namespaces, except the ones in the template's exemption list. + These namespaces are exempted to allow system pods to run without restrictions, which is required for proper operation of the cluster. + +:::note +If you intend to import an RKE cluster into Rancher, please consult the [documentation](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for how to configure the PSA to exempt Rancher system namespaces. +::: + + + + + +On Kubernetes v1.24 and older, the `PodSecurityPolicy` admission controller is always enabled. + +Below is the minimum necessary configuration needed for hardening RKE2 to pass CIS v1.23 hardened profile `rke2-cis-1.23-hardened` available in Rancher. + +:::note +In the following example the profile is set to `cis-1.6` which is the value defined in the upstream RKE2, but the cluster is actually configured to pass the CIS v1.23 hardened profile +::: + +```yaml +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 +``` + + +When both the `defaultPodSecurityPolicyTemplateName` and `profile` flags are set, Rancher and RKE2 does the following: + +1. Checks that host-level requirements have been met. If they haven't, RKE2 will exit with a fatal error describing the unmet requirements. +2. Applies network policies that allow the cluster to pass associated controls. +3. Configures runtime pod security policies that allow the cluster to pass associated controls. + + + + +:::note +The Kubernetes control plane components and critical additions such as CNI, DNS, and Ingress are ran as pods in the `kube-system` namespace. Therefore, this namespace will have a policy that is less restrictive so that these components can run properly. +::: + +### NetworkPolicies + +When ran with a valid `cis-1.xx` profile, RKE2 will put `NetworkPolicies` in place that passes the CIS Benchmark for Kubernetes' built-in namespaces. These namespaces are: `kube-system`, `kube-public`, `kube-node-lease`, and `default`. + +The `NetworkPolicy` used will only allow pods within the same namespace to talk to each other. The notable exception to this is that it allows DNS requests to be resolved. + +:::note +Operators must manage network policies as normal for additional namespaces that are created. +::: + +### Configure `default` service account + +**Set `automountServiceAccountToken` to `false` for `default` service accounts** + +Kubernetes provides a `default` service account which is used by cluster workloads where no specific service account is assigned to the pod. Where access to the Kubernetes API from a pod is required, a specific service account should be created for that pod, and rights granted to that service account. The `default` service account should be configured such that it does not provide a service account token and does not have any explicit rights assignments. + +For each namespace including `default` and `kube-system` on a standard RKE2 install, the `default` service account must include this value: + +```yaml +automountServiceAccountToken: false +``` + +For namespaces created by the cluster operator, the following script and configuration file can be used to configure the `default` service account. + +The configuration bellow must be saved to a file called `account_update.yaml`. + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: default +automountServiceAccountToken: false +``` + +Create a bash script file called `account_update.sh`. Be sure to `sudo chmod +x account_update.sh` so the script has execute permissions. + +```bash +#!/bin/bash -e + +for namespace in $(kubectl get namespaces -A -o=jsonpath="{.items[*]['metadata.name']}"); do + echo -n "Patching namespace $namespace - " + kubectl patch serviceaccount default -n ${namespace} -p "$(cat account_update.yaml)" +done +``` + +Execute this script to apply the `account_update.yaml` configuration to `default` service account in all namespaces. + +### API Server audit configuration + +CIS requirements 1.2.19 to 1.2.22 are related to configuring audit logs for the API Server. When RKE2 is started with the `profile` flag set, it will automatically configure hardened `--audit-log-` parameters in the API Server to pass those CIS checks. + +RKE2's default audit policy is configured to not log requests in the API Server. This is done to allow cluster operators flexibility to customize an audit policy that suits their auditing requirements and needs, as these are specific to each users' environment and policies. + +A default audit policy is created by RKE2 when started with the `profile` flag set. The policy is defined in `/etc/rancher/rke2/audit-policy.yaml`. + +```yaml +apiVersion: audit.k8s.io/v1 +kind: Policy +metadata: + creationTimestamp: null +rules: +- level: None +``` + +## Reference Hardened RKE2 Template Configuration + +The reference template configuration is used in Rancher to create a hardened RKE2 custom cluster. This reference does not include other required **cluster configuration** directives which will vary depending on your environment. + + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityAdmissionConfigurationTemplateName: rancher-restricted + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.23 +``` + + + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +metadata: + name: # Define cluster name +spec: + defaultPodSecurityPolicyTemplateName: restricted-noroot + kubernetesVersion: # Define RKE2 version + rkeConfig: + machineSelectorConfig: + - config: + profile: cis-1.6 + protect-kernel-defaults: true +``` + + + + +## Conclusion + +If you have followed this guide, your RKE2 custom cluster provisioned by Rancher will be configured to pass the CIS Kubernetes Benchmark. You can review our RKE2 self-assessment guides to understand how we verified each of the benchmarks and how you can do the same on your cluster. diff --git a/versioned_docs/version-latest/pages-for-subheaders/selinux-rpm.md b/versioned_docs/version-latest/pages-for-subheaders/selinux-rpm.md new file mode 100644 index 00000000000..c72c72ff56a --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/selinux-rpm.md @@ -0,0 +1,20 @@ +--- +title: SELinux RPM +--- + + + + + +[Security-Enhanced Linux (SELinux)](https://en.wikipedia.org/wiki/Security-Enhanced_Linux) is a security enhancement to Linux. + +Developed by Red Hat, it is an implementation of mandatory access controls (MAC) on Linux. Mandatory access controls allow an administrator of a system to define how applications and users can access different resources such as files, devices, networks and inter-process communication. SELinux also enhances security by making an OS restrictive by default. + +After being historically used by government agencies, SELinux is now industry standard and is enabled by default on CentOS 7 and 8. To check whether SELinux is enabled and enforcing on your system, use `getenforce`: + +``` +# getenforce +Enforcing +``` + +We provide two RPMs (Red Hat packages) that enable Rancher products to function properly on SELinux-enforcing hosts: [`rancher-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md) and [`rke2-selinux`](../reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md). \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/set-up-cloud-providers.md b/versioned_docs/version-latest/pages-for-subheaders/set-up-cloud-providers.md new file mode 100644 index 00000000000..9a02515dba9 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/set-up-cloud-providers.md @@ -0,0 +1,47 @@ +--- +title: Setting up Cloud Providers +--- + + + + + +A _cloud provider_ is a module in Kubernetes that provides an interface for managing nodes, load balancers, and networking routes. + +When a cloud provider is set up in Rancher, the Rancher server can automatically provision new nodes, load balancers or persistent storage devices when launching Kubernetes definitions, if the cloud provider you're using supports such automation. + +Your cluster will not provision correctly if you configure a cloud provider cluster of nodes that do not meet the prerequisites. + +By default, the **Cloud Provider** option is set to `None`. + +The following cloud providers can be enabled: + +* Amazon +* Azure +* GCE (Google Compute Engine) +* vSphere + +### Setting up the Amazon Cloud Provider + +For details on enabling the Amazon cloud provider, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon.md) + +### Setting up the Azure Cloud Provider + +For details on enabling the Azure cloud provider, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure.md) + +### Setting up the GCE Cloud Provider + +For details on enabling the Google Compute Engine cloud provider, refer to [this page.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md) + +### Setting up the vSphere Cloud Provider + +For details on enabling the vSphere cloud provider, refer to [in-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). + +### Setting up a Custom Cloud Provider + +The `Custom` cloud provider is available if you want to configure any Kubernetes cloud provider. + +For the custom cloud provider option, you can refer to the [RKE docs](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to edit the yaml file for your specific cloud provider. There are specific cloud providers that have more detailed configuration: + +* [vSphere](https://rke.docs.rancher.com/config-options/cloud-providers/vsphere) +* [OpenStack](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/openstack/) diff --git a/versioned_docs/version-latest/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md b/versioned_docs/version-latest/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md new file mode 100644 index 00000000000..fd4f4ed5bda --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md @@ -0,0 +1,33 @@ +--- +title: Setting up Clusters from Hosted Kubernetes Providers +--- + + + + + +In this scenario, Rancher does not provision Kubernetes because it is installed by providers such as Google Kubernetes Engine (GKE), Amazon Elastic Container Service for Kubernetes, or Azure Kubernetes Service. + +If you use a Kubernetes provider such as Google GKE, Rancher integrates with its cloud APIs, allowing you to create and manage role-based access control for the hosted cluster from the Rancher UI. + +In this use case, Rancher sends a request to a hosted provider using the provider's API. The provider then provisions and hosts the cluster for you. When the cluster finishes building, you can manage it from the Rancher UI along with clusters you've provisioned that are hosted on-prem or in an infrastructure provider. + +Rancher supports the following Kubernetes providers: + +- [Google GKE (Google Kubernetes Engine)](https://cloud.google.com/kubernetes-engine/) +- [Amazon EKS (Amazon Elastic Container Service for Kubernetes)](https://aws.amazon.com/eks/) +- [Microsoft AKS (Azure Kubernetes Service)](https://azure.microsoft.com/en-us/services/kubernetes-service/) +- [Alibaba ACK (Alibaba Cloud Container Service for Kubernetes)](https://www.alibabacloud.com/product/kubernetes) +- [Tencent TKE (Tencent Kubernetes Engine)](https://intl.cloud.tencent.com/product/tke) +- [Huawei CCE (Huawei Cloud Container Engine)](https://www.huaweicloud.com/en-us/product/cce.html) + +## Hosted Kubernetes Provider Authentication + +When using Rancher to create a cluster hosted by a provider, you are prompted for authentication information. This information is required to access the provider's API. For more information on how to obtain this information, see the following procedures: + +- [Creating a GKE Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke.md) +- [Creating an EKS Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks.md) +- [Creating an AKS Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md) +- [Creating an ACK Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba.md) +- [Creating a TKE Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent.md) +- [Creating a CCE Cluster](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/single-node-rancher-in-docker.md b/versioned_docs/version-latest/pages-for-subheaders/single-node-rancher-in-docker.md new file mode 100644 index 00000000000..91072d2b3b4 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/single-node-rancher-in-docker.md @@ -0,0 +1,9 @@ +--- +title: Single Node Rancher in Docker +--- + + + + + +The following docs will discuss [HTTP proxy configuration](../reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md) and [advanced options](../reference-guides/single-node-rancher-in-docker/advanced-options.md) for Docker installs. \ No newline at end of file diff --git a/versioned_docs/version-latest/pages-for-subheaders/use-existing-nodes.md b/versioned_docs/version-latest/pages-for-subheaders/use-existing-nodes.md new file mode 100644 index 00000000000..2aeb05bc488 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/use-existing-nodes.md @@ -0,0 +1,141 @@ +--- +title: Launching Kubernetes on Existing Custom Nodes +description: To create a cluster with custom nodes, you’ll need to access servers in your cluster and provision them according to Rancher requirements +--- + + + + + +When you create a custom cluster, Rancher uses RKE (the Rancher Kubernetes Engine) to create a Kubernetes cluster in on-prem bare-metal servers, on-prem virtual machines, or in any node hosted by an infrastructure provider. + +To use this option you'll need access to servers you intend to use in your Kubernetes cluster. Provision each server according to the [requirements](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md), which includes some hardware specifications and Docker. After you install Docker on each server, you willl also run the command provided in the Rancher UI on each server to turn each one into a Kubernetes node. + +This section describes how to set up a custom cluster. + +## Creating a Cluster with Custom Nodes + +:::note Want to use Windows hosts as Kubernetes workers? + +See [Configuring Custom Clusters for Windows](use-windows-clusters.md) before you start. + +::: + +### 1. Provision a Linux Host + +Begin creation of a custom cluster by provisioning a Linux host. Your host can be: + +- A cloud-host virtual machine (VM) +- An on-prem VM +- A bare-metal server + +If you want to reuse a node from a previous custom cluster, [clean the node](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) before using it in a cluster again. If you reuse a node that hasn't been cleaned, cluster provisioning may fail. + +Provision the host according to the [installation requirements](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md) and the [checklist for production-ready clusters.](checklist-for-production-ready-clusters.md) + +If you're using Amazon EC2 as your host and want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) when provisioning the host. + +### 2. Create the Custom Cluster + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Custom**. +1. Enter a **Cluster Name**. +1. Use **Cluster Configuration** section to choose the version of Kubernetes, what network provider will be used and if you want to enable project network isolation. To see more cluster options, click on **Show advanced options**. + + :::note Using Windows nodes as Kubernetes workers? + + - See [Enable the Windows Support Option](use-windows-clusters.md). + - The only Network Provider available for clusters with Windows support is Flannel. + + ::: + + :::note Dual-stack on Amazon EC2: + + If you're using Amazon EC2 as your host and want to use the [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) feature, there are additional [requirements](https://rancher.com/docs/rke//latest/en/config-options/dual-stack#requirements) when configuring RKE. + + ::: + +6. Click **Next**. + +4. Use **Member Roles** to configure user authorization for the cluster. Click **Add Member** to add users that can access the cluster. Use the **Role** drop-down to set permissions for each user. + +7. From **Node Role**, choose the roles that you want filled by a cluster node. You must provision at least one node for each role: `etcd`, `worker`, and `control plane`. All three roles are required for a custom cluster to finish provisioning. For more information on roles, see [this section.](../reference-guides/kubernetes-concepts.md#roles-for-nodes-in-kubernetes-clusters) + +:::note + +- Using Windows nodes as Kubernetes workers? See [this section](use-windows-clusters.md). +- Bare-Metal Server Reminder: If you plan on dedicating bare-metal servers to each role, you must provision a bare-metal server for each role (i.e. provision multiple bare-metal servers). + +::: + +8. **Optional**: Click **[Show advanced options](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md)** to specify IP address(es) to use when registering the node, override the hostname of the node, or to add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) or [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) to the node. + +9. Copy the command displayed on screen to your clipboard. + +10. Log in to your Linux host using your preferred shell, such as PuTTy or a remote Terminal connection. Run the command copied to your clipboard. + +:::note + +Repeat steps 7-10 if you want to dedicate specific hosts to specific node roles. Repeat the steps as many times as needed. + +::: + +11. When you finish running the command(s) on your Linux host(s), click **Done**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +### 3. Amazon Only: Tag Resources + +If you have configured your cluster to use Amazon as **Cloud Provider**, tag your AWS resources with a cluster ID. + +[Amazon Documentation: Tagging Your Amazon EC2 Resources](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html) + +:::note + +You can use Amazon EC2 instances without configuring a cloud provider in Kubernetes. You only have to configure the cloud provider if you want to use specific Kubernetes cloud provider functionality. For more information, see [Kubernetes Cloud Providers](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/) + +::: + +The following resources need to be tagged with a `ClusterID`: + +- **Nodes**: All hosts added in Rancher. +- **Subnet**: The subnet used for your cluster +- **Security Group**: The security group used for your cluster. + +:::note + +Do not tag multiple security groups. Tagging multiple groups generates an error when creating Elastic Load Balancer. + +::: + +The tag that should be used is: + +``` +Key=kubernetes.io/cluster/, Value=owned +``` + +`` can be any string you choose. However, the same string must be used on every resource you tag. Setting the tag value to `owned` informs the cluster that all resources tagged with the `` are owned and managed by this cluster. + +If you share resources between clusters, you can change the tag to: + +``` +Key=kubernetes.io/cluster/CLUSTERID, Value=shared +``` + +## Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through Rancher. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. diff --git a/versioned_docs/version-latest/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md b/versioned_docs/version-latest/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md new file mode 100644 index 00000000000..d875e2325c2 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/use-new-nodes-in-an-infra-provider.md @@ -0,0 +1,156 @@ +--- +title: Launching Kubernetes on New Nodes in an Infrastructure Provider +--- + + + + + +When you create an RKE or RKE2 cluster using a node template in Rancher, each resulting node pool is shown in a new **Machine Pools** tab. You can see the machine pools by doing the following: + +1. Click **☰ > Cluster Management**. +1. Click the name of the RKE or RKE2 cluster. + +## RKE Clusters + +Using Rancher, you can create pools of nodes based on a [node template](use-new-nodes-in-an-infra-provider.md#node-templates). This node template defines the parameters you want to use to launch nodes in your infrastructure providers or cloud providers. + +One benefit of installing Kubernetes on node pools hosted by an infrastructure provider is that if a node loses connectivity with the cluster, Rancher can automatically create another node to join the cluster to ensure that the count of the node pool is as expected. + +The available cloud providers to create a node template are decided based on active [node drivers](use-new-nodes-in-an-infra-provider.md#node-drivers). + +### Node Templates + +A node template is the saved configuration for the parameters to use when provisioning nodes in a specific cloud provider. These nodes can be launched from the UI. Rancher uses [Docker Machine](https://docs.docker.com/machine/) to provision these nodes. The available cloud providers to create node templates are based on the active node drivers in Rancher. + +After you create a node template in Rancher, it's saved so that you can use this template again to create node pools. Node templates are bound to your login. After you add a template, you can remove them from your user profile. + +#### Node Labels + +You can add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) on each node template, so that any nodes created from the node template will automatically have these labels on them. + +Invalid labels can prevent upgrades or can prevent Rancher from starting. For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +#### Node Taints + +You can add [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) on each node template, so that any nodes created from the node template will automatically have these taints on them. + +Since taints can be added at a node template and node pool, if there is no conflict with the same key and effect of the taints, all taints will be added to the nodes. If there are taints with the same key and different effect, the taints from the node pool will override the taints from the node template. + +#### Administrator Control of Node Templates + +Administrators can control all node templates. Admins can now maintain all the node templates within Rancher. When a node template owner is no longer using Rancher, the node templates created by them can be managed by administrators so the cluster can continue to be updated and maintained. + +To access all node templates, an administrator will need to do the following: + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. + +**Result:** All node templates are listed. The templates can be edited or cloned by clicking the **â‹®**. + +### Node Pools + +Using Rancher, you can create pools of nodes based on a [node template](#node-templates). + +A node template defines the configuration of a node, like what operating system to use, number of CPUs, and amount of memory. + +The benefit of using a node pool is that if a node is destroyed or deleted, you can increase the number of live nodes to compensate for the node that was lost. The node pool helps you ensure that the count of the node pool is as expected. + +Each node pool must have one or more nodes roles assigned. + +Each node role (i.e. etcd, controlplane, and worker) should be assigned to a distinct node pool. Although it is possible to assign multiple node roles to a node pool, this should not be done for production clusters. + +The recommended setup is to have: + +- a node pool with the etcd node role and a count of three +- a node pool with the controlplane node role and a count of at least two +- a node pool with the worker node role and a count of at least two + +**RKE1 downstream cluster nodes in an air-gapped environment:** + +By default, Rancher tries to run the Docker Install script when provisioning RKE1 downstream cluster nodes, such as in vSphere. However, the Rancher Docker installation script would fail in air-gapped environments. To work around this issue, you may choose to skip installing Docker when creating a Node Template where Docker is pre-installed onto a VM image. You can accomplish this by selecting **None** in the dropdown list for `Docker Install URL` under **Engine Options** in the Rancher UI. + +
**Engine Options Dropdown:**
+ +![Engine Options Dropdown](/img/node-template-engine-options-rke1.png) + +#### Node Pool Taints + +If you haven't defined [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) on your node template, you can add taints for each node pool. The benefit of adding taints to a node pool is that you can change the node template without having to first ensure that the taint exists in the new template. + +For each taint, they will automatically be added to any created node in the node pool. Therefore, if you add taints to a node pool that have existing nodes, the taints won't apply to existing nodes in the node pool, but any new node added into the node pool will get the taint. + +When there are taints on the node pool and node template, if there is no conflict with the same key and effect of the taints, all taints will be added to the nodes. If there are taints with the same key and different effect, the taints from the node pool will override the taints from the node template. + +#### About Node Auto-replace + +If a node is in a node pool, Rancher can automatically replace unreachable nodes. Rancher will use the existing node template for the given node pool to recreate the node if it becomes inactive for a specified number of minutes. + +:::caution + +Self-healing node pools are designed to help you replace worker nodes for stateless applications. It is not recommended to enable node auto-replace on a node pool of master nodes or nodes with persistent volumes attached, because VMs are treated ephemerally. When a node in a node pool loses connectivity with the cluster, its persistent volumes are destroyed, resulting in data loss for stateful applications. + +::: + +Node auto-replace works on top of the Kubernetes node controller. The node controller periodically checks the status of all the nodes (configurable via the `--node-monitor-period` flag of the `kube-controller`). When a node is unreachable, the node controller will taint that node. When this occurs, Rancher will begin its deletion countdown. You can configure the amount of time Rancher waits to delete the node. If the taint is not removed before the deletion countdown ends, Rancher will proceed to delete the node object. Rancher will then provision a node in accordance with the set quantity of the node pool. + +#### Enabling Node Auto-replace + +When you create the node pool, you can specify the amount of time in minutes that Rancher will wait to replace an unresponsive node. + +1. In the form for creating or editing a cluster, go to the **Node Pools** section. +1. Go to the node pool where you want to enable node auto-replace. In the **Recreate Unreachable After** field, enter the number of minutes that Rancher should wait for a node to respond before replacing the node. +1. Fill out the rest of the form for creating or editing the cluster. + +**Result:** Node auto-replace is enabled for the node pool. + +#### Disabling Node Auto-replace + +You can disable node auto-replace from the Rancher UI with the following steps: + +1. Click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to disable node auto-replace and click **⋮ > Edit Config**. +1. In the **Node Pools** section, go to the node pool where you want to enable node auto-replace. In the **Recreate Unreachable After** field, enter 0. +1. Click **Save**. + +**Result:** Node auto-replace is disabled for the node pool. + +### Cloud Credentials + +Node templates can use cloud credentials to store credentials for launching nodes in your cloud provider, which has some benefits: + +- Credentials are stored as a Kubernetes secret, which is not only more secure, but it also allows you to edit a node template without having to enter your credentials every time. + +- After the cloud credential is created, it can be re-used to create additional node templates. + +- Multiple node templates can share the same cloud credential to create node pools. If your key is compromised or expired, the cloud credential can be updated in a single place, which allows all node templates that are using it to be updated at once. + +After cloud credentials are created, the user can start [managing the cloud credentials that they created](../reference-guides/user-settings/manage-cloud-credentials.md). + +### Node Drivers + +If you don't find the node driver that you want to use, you can see if it is available in Rancher's built-in [node drivers and activate it](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#activatingdeactivating-node-drivers), or you can [add your own custom node driver](../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md#adding-custom-node-drivers). + +## RKE2 Clusters + +Rancher v2.6 introduces provisioning for [RKE2](https://docs.rke2.io/) clusters directly from the Rancher UI. RKE2, also known as RKE Government, is a fully conformant Kubernetes distribution that focuses on security and compliance within the U.S. Federal Government sector. + +:::note + +For RKE2 cluster templates, please refer to [this page](../how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates.md#rke2-cluster-template) for additional information. + +::: + +### Node Roles + +The RKE2 CLI exposes two roles, `server` and `agent`, which represent the Kubernetes node-roles `etcd` + `controlplane` and `worker` respectively. With RKE2 integration in Rancher v2.6, RKE2 node pools can represent more fine-grained role assignments such that `etcd` and `controlplane` roles can be represented. + +The same functionality of using `etcd`, `controlplane` and `worker` nodes is possible in the RKE2 CLI by using flags and node tainting to control where workloads and the Kubernetes master were scheduled. The reason those roles were not implemented as first-class roles in the RKE2 CLI is that RKE2 is conceptualized as a set of raw building blocks that are best leveraged through an orchestration system such as Rancher. + +The implementation of the three node roles in Rancher means that Rancher managed RKE2 clusters are able to easily leverage all of the same architectural best practices that are recommended for RKE clusters. + +In our [recommended cluster architecture](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture.md), we outline how many nodes of each role clusters should have: + +- At least three nodes with the role etcd to survive losing one node +- At least two nodes with the role controlplane for master component high availability +- At least two nodes with the role worker for workload rescheduling upon node failure diff --git a/versioned_docs/version-latest/pages-for-subheaders/use-windows-clusters.md b/versioned_docs/version-latest/pages-for-subheaders/use-windows-clusters.md new file mode 100644 index 00000000000..36fe47c422f --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/use-windows-clusters.md @@ -0,0 +1,281 @@ +--- +title: Launching Kubernetes on Windows Clusters +--- + + + + + +When provisioning a [custom cluster](use-existing-nodes.md) using Rancher, Rancher uses RKE (the Rancher Kubernetes Engine) to install Kubernetes on your existing nodes. + +In a Windows cluster provisioned with Rancher, the cluster must contain both Linux and Windows nodes. The Kubernetes controlplane can only run on Linux nodes, and the Windows nodes can only have the worker role. Windows nodes can only be used for deploying workloads. + +Some other requirements for Windows clusters include: + +- You can only add Windows nodes to a cluster if Windows support is enabled when the cluster is created. Windows support cannot be enabled for existing clusters. +- Kubernetes 1.15+ is required. +- The Flannel network provider must be used. +- Windows nodes must have 50 GB of disk space. + +For the full list of requirements, see [this section.](#requirements-for-windows-clusters) + +For a summary of Kubernetes features supported in Windows, see the Kubernetes documentation on [supported functionality and limitations for using Kubernetes with Windows](https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#supported-functionality-and-limitations) or the [guide for scheduling Windows containers in Kubernetes](https://kubernetes.io/docs/setup/production-environment/windows/user-guide-windows-containers/). + +### RKE2 Windows + +The RKE2 provisioning feature also includes installing RKE2 on Windows clusters. Windows features for RKE2 include: + +- Windows Containers with RKE2 powered by containerd +- Added provisioning of Windows RKE2 custom clusters directly from the Rancher UI +- Calico CNI for Windows RKE2 custom clusters +- SAC releases of Windows Server (2004 and 20H2) are included in the technical preview + +Windows Support for RKE2 Custom Clusters requires choosing Calico as the CNI. + +:::note + +Rancher will allow Windows workload pods to deploy on both Windows and Linux worker nodes by default. When creating mixed clusters in RKE2, you must edit the `nodeSelector` in the chart to direct the pods to be placed onto a compatible Windows node. Refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for more information on how to use `nodeSelector` to assign pods to nodes. + +::: + +- HostProcess containers in Windows RKE2 are supported in Kubernetes v1.24.1 and up. See [the upstream documentation](https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/) for more information. + +## Requirements for Windows Clusters + +The general node requirements for networking, operating systems, and Docker are the same as the node requirements for a [Rancher installation](installation-requirements.md). + +### OS and Docker Requirements + +Our support for Windows Server and Windows containers match the Microsoft official lifecycle for LTSC (Long-Term Servicing Channel) and SAC (Semi-Annual Channel). + +For the support lifecycle dates for Windows Server, see the [Microsoft Documentation.](https://docs.microsoft.com/en-us/windows-server/get-started/windows-server-release-info) + +### Kubernetes Version + +Kubernetes v1.15+ is required. + +If you are using Kubernetes v1.21 with Windows Server 20H2 Standard Core, the patch "2019-08 Servicing Stack Update for Windows Server" must be installed on the node. + +### Node Requirements + +The hosts in the cluster need to have at least: + +- 2 core CPUs +- 5 GB memory +- 50 GB disk space + +Rancher will not provision the node if the node does not meet these requirements. + +### Networking Requirements + +Before provisioning a new cluster, be sure that you have already installed Rancher on a device that accepts inbound network traffic. This is required in order for the cluster nodes to communicate with Rancher. If you have not already installed Rancher, please refer to the [installation documentation](installation-and-upgrade.md) before proceeding with this guide. + +Rancher only supports Windows using Flannel as the network provider. + +There are two network options: [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) and [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan). The default option is **VXLAN (Overlay)** mode. + +For **Host Gateway (L2bridge)** networking, it's best to use the same Layer 2 network for all nodes. Otherwise, you need to configure the route rules for them. For details, refer to the [documentation on configuring cloud-hosted VM routes.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#cloud-hosted-vm-routes-configuration) You will also need to [disable private IP address checks](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#disabling-private-ip-address-checks) if you are using Amazon EC2, Google GCE, or Azure VM. + +For **VXLAN (Overlay)** networking, the [KB4489899](https://support.microsoft.com/en-us/help/4489899) hotfix must be installed. Most cloud-hosted VMs already have this hotfix. + +If you are configuring DHCP options sets for an AWS virtual private cloud, note that in the `domain-name` option field, only one domain name can be specified. According to the DHCP options [documentation:](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_DHCP_Options.html) + +:::note + +Some Linux operating systems accept multiple domain names separated by spaces. However, other Linux operating systems and Windows treat the value as a single domain, which results in unexpected behavior. If your DHCP options set is associated with a VPC that has instances with multiple operating systems, specify only one domain name. + +::: + +### Rancher on vSphere with ESXi 6.7u2 and above + +If you are using Rancher on VMware vSphere with ESXi 6.7u2 or later with Red Hat Enterprise Linux 8.3, CentOS 8.3, or SUSE Enterprise Linux 15 SP2 or later, it is necessary to disable the `vmxnet3` virtual network adapter hardware offloading feature. Failure to do so will result in all network connections between pods on different cluster nodes to fail with timeout errors. All connections from Windows pods to critical services running on Linux nodes, such as CoreDNS, will fail as well. It is also possible that external connections may fail. This issue is the result of Linux distributions enabling the hardware offloading feature in `vmxnet3` and a bug in the `vmxnet3` hardware offloading feature that results in the discarding of packets for guest overlay traffic. To address this issue, it is necessary disable the `vmxnet3` hardware offloading feature. This setting does not survive reboot, so it is necessary to disable on every boot. The recommended course of action is to create a systemd unit file at `/etc/systemd/system/disable_hw_offloading.service`, which disables the `vmxnet3` hardware offloading feature on boot. A sample systemd unit file which disables the `vmxnet3` hardware offloading feature is as follows. Note that `` must be customized to the host `vmxnet3` network interface, e.g., `ens192`: + +``` +[Unit] +Description=Disable vmxnet3 hardware offloading feature + +[Service] +Type=oneshot +ExecStart=ethtool -K tx-udp_tnl-segmentation off +ExecStart=ethtool -K tx-udp_tnl-csum-segmentation off +StandardOutput=journal + +[Install] +WantedBy=multi-user.target +``` +Then set the appropriate permissions on the systemd unit file: +``` +chmod 0644 /etc/systemd/system/disable_hw_offloading.service +``` +Finally, enable the systemd service: +``` +systemctl enable disable_hw_offloading.service +``` + +### Architecture Requirements + +The Kubernetes cluster management nodes (`etcd` and `controlplane`) must be run on Linux nodes. + +The `worker` nodes, which is where your workloads will be deployed on, will typically be Windows nodes, but there must be at least one `worker` node that is run on Linux in order to run the Rancher cluster agent, DNS, metrics server, and Ingress related containers. + +#### Recommended Architecture + +We recommend the minimum three-node architecture listed in the table below, but you can always add additional Linux and Windows workers to scale up your cluster for redundancy: + +| Node | Operating System | Kubernetes Cluster Role(s) | Purpose | +| ------ | --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +| Node 1 | Linux (Ubuntu Server 18.04 recommended) | Control plane, etcd, worker | Manage the Kubernetes cluster | +| Node 2 | Linux (Ubuntu Server 18.04 recommended) | Worker | Support the Rancher Cluster agent, Metrics server, DNS, and Ingress for the cluster | +| Node 3 | Windows (Windows Server core version 1809 or above) | Worker | Run your Windows containers | + +### Container Requirements + +Windows requires that containers must be built on the same Windows Server version that they are being deployed on. Therefore, containers must be built on Windows Server core version 1809 or above. If you have existing containers built for an earlier Windows Server core version, they must be re-built on Windows Server core version 1809 or above. + +### Cloud Provider Specific Requirements + +If you set a Kubernetes cloud provider in your cluster, some additional steps are required. You might want to set a cloud provider if you want to want to leverage a cloud provider's capabilities, for example, to automatically provision storage, load balancers, or other infrastructure for your cluster. Refer to [this page](../pages-for-subheaders/set-up-cloud-providers.md) for details on how to configure a cloud provider cluster of nodes that meet the prerequisites. + +If you are using the GCE (Google Compute Engine) cloud provider, you must do the following: + +- Enable the GCE cloud provider in the `cluster.yml` by following [these steps.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine.md) +- When provisioning the cluster in Rancher, choose **Custom cloud provider** as the cloud provider in the Rancher UI. + +## Tutorial: How to Create a Cluster with Windows Support + +This tutorial describes how to create a Rancher-provisioned cluster with the three nodes in the [recommended architecture.](#recommended-architecture) + +When you provision a cluster with Rancher on existing nodes, you will add nodes to the cluster by installing the [Rancher agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) on each one. When you create or edit your cluster from the Rancher UI, you will see a **Customize Node Run Command** that you can run on each server to add it to your cluster. + +To set up a cluster with support for Windows nodes and containers, you will need to complete the tasks below. + + +### 1. Provision Hosts + +To begin provisioning a cluster on existing nodes with Windows support, prepare your hosts. + +Your hosts can be: + +- Cloud-hosted VMs +- VMs from virtualization clusters +- Bare-metal servers + +You will provision three nodes: + +- One Linux node, which manages the Kubernetes control plane and stores your `etcd` +- A second Linux node, which will be another worker node +- The Windows node, which will run your Windows containers as a worker node + +| Node | Operating System | +| ------ | ------------------------------------------------------------ | +| Node 1 | Linux (Ubuntu Server 18.04 recommended) | +| Node 2 | Linux (Ubuntu Server 18.04 recommended) | +| Node 3 | Windows (Windows Server core version 1809 or above required) | + +If your nodes are hosted by a **Cloud Provider** and you want automation support such as loadbalancers or persistent storage devices, your nodes have additional configuration requirements. For details, see [Selecting Cloud Providers.](../pages-for-subheaders/set-up-cloud-providers.md) + +### 2. Create the Cluster on Existing Nodes + +The instructions for creating a Windows cluster on existing nodes are very similar to the general [instructions for creating a custom cluster](use-existing-nodes.md) with some Windows-specific requirements. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, click **Create**. +1. Click **Custom**. +1. Enter a name for your cluster in the **Cluster Name** field. +1. In the **Kubernetes Version** dropdown menu, select v1.19 or above. +1. In the **Network Provider** field, select **Flannel**. +1. In the **Windows Support** section, click **Enabled**. +1. Optional: After you enable Windows support, you will be able to choose the Flannel backend. There are two network options: [**Host Gateway (L2bridge)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#host-gw) and [**VXLAN (Overlay)**](https://github.com/coreos/flannel/blob/master/Documentation/backends.md#vxlan). The default option is **VXLAN (Overlay)** mode. +1. Click **Next**. + +:::note Important: + +For Host Gateway (L2bridge) networking, it's best to use the same Layer 2 network for all nodes. Otherwise, you need to configure the route rules for them. For details, refer to the [documentation on configuring cloud-hosted VM routes.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#cloud-hosted-vm-routes-configuration) You will also need to [disable private IP address checks](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway.md#disabling-private-ip-address-checks) if you are using Amazon EC2, Google GCE, or Azure VM. + +::: + +### 3. Add Nodes to the Cluster + +This section describes how to register your Linux and Worker nodes to your cluster. You will run a command on each node, which will install the Rancher agent and allow Rancher to manage each node. + +#### Add Linux Master Node + +In this section, we fill out a form on the Rancher UI to get a custom command to install the Rancher agent on the Linux master node. Then we will copy the command and run it on our Linux master node to register the node in the cluster. + +The first node in your cluster should be a Linux host has both the **Control Plane** and **etcd** roles. At a minimum, both of these roles must be enabled for this node, and this node must be added to your cluster before you can add Windows hosts. + +1. In the **Node Operating System** section, click **Linux**. +1. In the **Node Role** section, choose at least **etcd** and **Control Plane**. We recommend selecting all three. +1. Optional: If you click **Show advanced options,** you can customize the settings for the [Rancher agent](../reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md) and [node labels.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +1. Copy the command displayed on the screen to your clipboard. +1. SSH into your Linux host and run the command that you copied to your clipboard. +1. When you are finished provisioning your Linux node(s), select **Done**. + +**Result:** + +Your cluster is created and assigned a state of **Provisioning**. Rancher is standing up your cluster. + +You can access your cluster after its state is updated to **Active**. + +**Active** clusters are assigned two Projects: + +- `Default`, containing the `default` namespace +- `System`, containing the `cattle-system`, `ingress-nginx`, `kube-public`, and `kube-system` namespaces + + +It may take a few minutes for the node to be registered in your cluster. + +#### Add Linux Worker Node + +In this section, we run a command to register the Linux worker node to the cluster. + +After the initial provisioning of your cluster, your cluster only has a single Linux host. Next, we add another Linux `worker` host, which will be used to support _Rancher cluster agent_, _Metrics server_, _DNS_ and _Ingress_ for your cluster. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **⋮ > Edit Config**. +1. Scroll down to **Node Operating System**. Choose **Linux**. +1. In the **Customize Node Run Command** section, go to the **Node Options** and select the **Worker** role. +1. Copy the command displayed on screen to your clipboard. +1. Log in to your Linux host using a remote Terminal connection. Run the command copied to your clipboard. +1. From **Rancher**, click **Save**. + +**Result:** The **Worker** role is installed on your Linux host, and the node registers with Rancher. It may take a few minutes for the node to be registered in your cluster. + +:::note + +Taints on Linux Worker Nodes + +For each Linux worker node added into the cluster, the following taints will be added to Linux worker node. By adding this taint to the Linux worker node, any workloads added to the Windows cluster will be automatically scheduled to the Windows worker node. If you want to schedule workloads specifically onto the Linux worker node, you will need to add tolerations to those workloads. + +| Taint Key | Taint Value | Taint Effect | +| -------------- | ----------- | ------------ | +| `cattle.io/os` | `linux` | `NoSchedule` | + +::: + +#### Add a Windows Worker Node + +In this section, we run a command to register the Windows worker node to the cluster. + +You can add Windows hosts to the cluster by editing the cluster and choosing the **Windows** option. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster that you created and click **⋮ > Edit Config**. +1. Scroll down to **Node Operating System**. Choose **Windows**. Note: You will see that the **worker** role is the only available role. +1. Copy the command displayed on screen to your clipboard. +1. Log in to your Windows host using your preferred tool, such as [Microsoft Remote Desktop](https://docs.microsoft.com/en-us/windows-server/remote/remote-desktop-services/clients/remote-desktop-clients). Run the command copied to your clipboard in the **Command Prompt (CMD)**. +1. From Rancher, click **Save**. +1. Optional: Repeat these instructions if you want to add more Windows nodes to your cluster. + +**Result:** The **Worker** role is installed on your Windows host, and the node registers with Rancher. It may take a few minutes for the node to be registered in your cluster. You now have a Windows Kubernetes cluster. + +### Optional Next Steps + +After creating your cluster, you can access it through the Rancher UI. As a best practice, we recommend setting up these alternate ways of accessing your cluster: + +- **Access your cluster with the kubectl CLI:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#accessing-clusters-with-kubectl-from-your-workstation) to access clusters with kubectl on your workstation. In this case, you will be authenticated through the Rancher server’s authentication proxy, then Rancher will connect you to the downstream cluster. This method lets you manage the cluster without the Rancher UI. +- **Access your cluster with the kubectl CLI, using the authorized cluster endpoint:** Follow [these steps](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md#authenticating-directly-with-a-downstream-cluster) to access your cluster with kubectl directly, without authenticating through the Rancher server. We recommend setting up this alternative method to access your cluster so that in case you can’t connect to Rancher, you can still access the cluster. + +## Configuration for Storage Classes in Azure + +If you are using Azure VMs for your nodes, you can use [Azure files](https://docs.microsoft.com/en-us/azure/aks/azure-files-dynamic-pv) as a StorageClass for the cluster. For details, refer to [this section.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration.md) diff --git a/versioned_docs/version-latest/pages-for-subheaders/user-settings.md b/versioned_docs/version-latest/pages-for-subheaders/user-settings.md new file mode 100644 index 00000000000..a9ed1c72d92 --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/user-settings.md @@ -0,0 +1,19 @@ +--- +title: User Settings +--- + + + + + +Within Rancher, each user has a number of settings associated with their login: personal preferences, API keys, etc. You can configure these settings by choosing from the **User Settings** menu. You can open this menu by clicking your avatar, located within the main menu. + +![User Settings Menu](/img/user-settings.png) + +The available user settings are: + +- [API & Keys](../reference-guides/user-settings/api-keys.md): If you want to interact with Rancher programmatically, you need an API key. Follow the directions in this section to obtain a key. +- [Cloud Credentials](../reference-guides/user-settings/manage-cloud-credentials.md): Manage cloud credentials [used by node templates](use-new-nodes-in-an-infra-provider.md#node-templates) to [provision nodes for clusters](../pages-for-subheaders/launch-kubernetes-with-rancher.md). +- [Node Templates](../reference-guides/user-settings/manage-node-templates.md): Manage templates [used by Rancher to provision nodes for clusters](../pages-for-subheaders/launch-kubernetes-with-rancher.md). +- [Preferences](../reference-guides/user-settings/user-preferences.md): Sets superficial preferences for the Rancher UI. +- Log Out: Ends your user session. diff --git a/versioned_docs/version-latest/pages-for-subheaders/vsphere.md b/versioned_docs/version-latest/pages-for-subheaders/vsphere.md new file mode 100644 index 00000000000..634a037c1cd --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/vsphere.md @@ -0,0 +1,60 @@ +--- +title: Creating a vSphere Cluster +description: Use Rancher to create a vSphere cluster. It may consist of groups of VMs with distinct properties which allow for fine-grained control over the sizing of nodes. +--- + + + + + +import YouTube from '@site/src/components/YouTube' + +By using Rancher with vSphere, you can bring cloud operations on-premises. + +Rancher can provision nodes in vSphere and install Kubernetes on them. When creating a Kubernetes cluster in vSphere, Rancher first provisions the specified number of virtual machines by communicating with the vCenter API. Then it installs Kubernetes on top of them. + +A vSphere cluster may consist of multiple groups of VMs with distinct properties, such as the amount of memory or the number of vCPUs. This grouping allows for fine-grained control over the sizing of nodes for each Kubernetes role. + +## vSphere Enhancements in Rancher v2.3 + +The vSphere node templates have been updated, allowing you to bring cloud operations on-premises with the following enhancements: + +### Self-healing Node Pools + +One of the biggest advantages of provisioning vSphere nodes with Rancher is that it allows you to take advantage of Rancher's self-healing node pools, also called the [node auto-replace feature,](use-new-nodes-in-an-infra-provider.md#about-node-auto-replace) in your on-premises clusters. Self-healing node pools are designed to help you replace worker nodes for stateless applications. When Rancher provisions nodes from a node template, Rancher can automatically replace unreachable nodes. + +:::caution + +It is not recommended to enable node auto-replace on a node pool of master nodes or nodes with persistent volumes attached, because VMs are treated ephemerally. When a node in a node pool loses connectivity with the cluster, its persistent volumes are destroyed, resulting in data loss for stateful applications. + +::: + +### Dynamically Populated Options for Instances and Scheduling + +Node templates for vSphere have been updated so that when you create a node template with your vSphere credentials, the template is automatically populated with the same options for provisioning VMs that you have access to in the vSphere console. + +For the fields to be populated, your setup needs to fulfill the [prerequisites.](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md#preparation-in-vsphere) + +### More Supported Operating Systems + +You can provision VMs with any operating system that supports `cloud-init`. Only YAML format is supported for the [cloud config.](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) + +### Video Walkthrough of v2.3.3 Node Template Features + +In this YouTube video, we demonstrate how to set up a node template with the new features designed to help you bring cloud operations to on-premises clusters. + + + +## Creating a vSphere Cluster + +In [this section,](../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere.md) you'll learn how to use Rancher to install an [RKE](https://rancher.com/docs/rke/latest/en/) Kubernetes cluster in vSphere. + +## Provisioning Storage + +For an example of how to provision storage in vSphere using Rancher, refer to [this section.](../how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage.md) In order to dynamically provision storage in vSphere, the vSphere provider must be enabled. Refer to [in-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). + +## Enabling the vSphere Cloud Provider + +When a cloud provider is set up in Rancher, the Rancher server can automatically provision new infrastructure for the cluster, including new nodes or persistent storage devices. + +For details on enabling the vSphere cloud provider, refer to [in-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere.md) and [out-of-tree vSphere config](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere.md). diff --git a/versioned_docs/version-latest/pages-for-subheaders/workloads-and-pods.md b/versioned_docs/version-latest/pages-for-subheaders/workloads-and-pods.md new file mode 100644 index 00000000000..5cfe84668af --- /dev/null +++ b/versioned_docs/version-latest/pages-for-subheaders/workloads-and-pods.md @@ -0,0 +1,82 @@ +--- +title: "Kubernetes Workloads and Pods" +description: "Learn about the two constructs with which you can build any complex containerized application in Kubernetes: Kubernetes workloads and pods" +--- + + + + + +You can build any complex containerized application in Kubernetes using two basic constructs: pods and workloads. Once you build an application, you can expose it for access either within the same cluster or on the Internet using a third construct: services. + +### Pods + +[_Pods_](https://kubernetes.io/docs/concepts/workloads/pods/pod-overview/) are one or more containers that share network namespaces and storage volumes. Most pods have only one container. Therefore when we discuss _pods_, the term is often synonymous with _containers_. You scale pods the same way you scale containers—by having multiple instances of the same pod that implement a service. Usually pods get scaled and managed by the workload. + +### Workloads + +_Workloads_ are objects that set deployment rules for pods. Based on these rules, Kubernetes performs the deployment and updates the workload with the current state of the application. +Workloads let you define the rules for application scheduling, scaling, and upgrade. + +#### Workload Types + +Kubernetes divides workloads into different types. The most popular types supported by Kubernetes are: + +- [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) + + _Deployments_ are best used for stateless applications (i.e., when you don't have to maintain the workload's state). Pods managed by deployment workloads are treated as independent and disposable. If a pod encounters disruption, Kubernetes removes it and then recreates it. An example application would be an Nginx web server. + +- [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + + _StatefulSets_, in contrast to deployments, are best used when your application needs to maintain its identity and store data. An application would be something like Zookeeper—an application that requires a database for storage. + +- [DaemonSets](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) + + _Daemonsets_ ensures that every node in the cluster runs a copy of pod. For use cases where you're collecting logs or monitoring node performance, this daemon-like workload works best. + +- [Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/) + + _Jobs_ launch one or more pods and ensure that a specified number of them successfully terminate. Jobs are best used to run a finite task to completion as opposed to managing an ongoing desired application state. + +- [CronJobs](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) + + _CronJobs_ are similar to jobs. CronJobs, however, runs to completion on a cron-based schedule. + +### Services + +In many use cases, a workload has to be either: + +- Accessed by other workloads in the cluster. +- Exposed to the outside world. + +You can achieve these goals by creating a _Service_. Services are mapped to the underlying workload's pods using a [selector/label approach (view the code samples)](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#service-and-replicationcontroller). Rancher UI simplifies this mapping process by automatically creating a service along with the workload, using the service port and type that you select. + +#### Service Types + +There are several types of services available in Rancher. The descriptions below are sourced from the [Kubernetes Documentation](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types). + +- **ClusterIP** + + >Exposes the service on a cluster-internal IP. Choosing this value makes the service only reachable from within the cluster. This is the default `ServiceType`. + +- **NodePort** + + >Exposes the service on each Node’s IP at a static port (the `NodePort`). A `ClusterIP` service, to which the `NodePort` service will route, is automatically created. You’ll be able to contact the `NodePort` service, from outside the cluster, by requesting `:`. + +- **LoadBalancer** + + >Exposes the service externally using a cloud provider’s load balancer. `NodePort` and `ClusterIP` services, to which the external load balancer will route, are automatically created. + +## Workload Options + +This section of the documentation contains instructions for deploying workloads and using workload options. + +- [Deploy Workloads](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads.md) +- [Upgrade Workloads](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads.md) +- [Rollback Workloads](../how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads.md) + +## Related Links + +### External Links + +- [Services](https://kubernetes.io/docs/concepts/services-networking/service/) diff --git a/versioned_docs/version-latest/rancher-manager.md b/versioned_docs/version-latest/rancher-manager.md new file mode 100644 index 00000000000..ad5eeb33b87 --- /dev/null +++ b/versioned_docs/version-latest/rancher-manager.md @@ -0,0 +1,20 @@ +--- +slug: / +title: "What is Rancher?" +sidebar_label: What is Rancher? +description: "Rancher adds significant value on top of Kubernetes: managing hundreds of clusters from one interface, centralizing RBAC, enabling monitoring and alerting. Read more." +--- + + + + + +Rancher is a Kubernetes management tool to deploy and run clusters anywhere and on any provider. + +Rancher can provision Kubernetes from a hosted provider, provision compute nodes and then install Kubernetes onto them, or import existing Kubernetes clusters running anywhere. + +Rancher adds significant value on top of Kubernetes, first by centralizing authentication and role-based access control (RBAC) for all of the clusters, giving global admins the ability to control cluster access from one location. + +It then enables detailed monitoring and alerting for clusters and their resources, ships logs to external providers, and integrates directly with Helm via the Application Catalog. If you have an external CI/CD system, you can plug it into Rancher, but if you don't, Rancher even includes Fleet to help you automatically deploy and upgrade workloads. + +Rancher is a _complete_ container management platform for Kubernetes, giving you the tools to successfully run Kubernetes anywhere. diff --git a/versioned_docs/version-latest/reference-guides/about-the-api/api-tokens.md b/versioned_docs/version-latest/reference-guides/about-the-api/api-tokens.md new file mode 100644 index 00000000000..5fe8a0eb5a4 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/about-the-api/api-tokens.md @@ -0,0 +1,90 @@ +--- +title: API Tokens +--- + + + + + +By default, some cluster-level API tokens are generated with infinite time-to-live (`ttl=0`). In other words, API tokens with `ttl=0` never expire unless you invalidate them. Tokens are not invalidated by changing a password. + +You can deactivate API tokens by deleting them or by deactivating the user account. + +### Deleting tokens +To delete a token, + +1. Go to the list of all tokens in the Rancher API view at `https:///v3/tokens`. + +1. Access the token you want to delete by its ID. For example, `https:///v3/tokens/kubectl-shell-user-vqkqt` + +1. Click **Delete**. + +Here is the complete list of tokens that are generated with `ttl=0`: + +| Token | Description | +| ----------------- | -------------------------------------------------------------------------------------- | +| `kubeconfig-*` | Kubeconfig token | +| `kubectl-shell-*` | Access to `kubectl` shell in the browser | +| `agent-*` | Token for agent deployment | +| `compose-token-*` | Token for compose | +| `helm-token-*` | Token for Helm chart deployment | +| `telemetry-*` | Telemetry token | +| `drain-node-*` | Token for drain (we use `kubectl` for drain because there is no native Kubernetes API) | + + +### Setting TTL on Kubeconfig Tokens + +Admins can set a global time-to-live (TTL) on Kubeconfig tokens. Changing the default kubeconfig TTL can be done by navigating to global settings and setting [`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) to the desired duration in minutes. The default value of [`kubeconfig-default-token-ttl-minutes`](#kubeconfig-default-token-ttl-minutes) is 0, which means tokens never expire. + +:::note + +This setting is used by all kubeconfig tokens except those created by the CLI to [generate kubeconfig tokens](#disable-tokens-in-generated-kubeconfigs). + +::: + +### Disable Tokens in Generated Kubeconfigs + +1. Set the `kubeconfig-generate-token` setting to `false`. This setting instructs Rancher to no longer automatically generate a token when a user clicks on download a kubeconfig file. Once this setting is deactivated, a generated kubeconfig will reference the [Rancher CLI](../cli-with-rancher/kubectl-utility.md#authentication-with-kubectl-and-kubeconfig-tokens-with-ttl) to retrieve a short-lived token for the cluster. When this kubeconfig is used in a client, such as `kubectl`, the Rancher CLI needs to be installed to complete the log in request. + +2. Set the `kubeconfig-token-ttl-minutes` setting to the desired duration in minutes. By default, `kubeconfig-token-ttl-minutes` is 960 (16 hours). + +### Token Hashing + +Users can enable token hashing, where tokens will undergo a one-way hash using the SHA256 algorithm. This is a non-reversible process, once enabled, this feature cannot be disabled. It is advisable to take backups prior to enabling and/or evaluating in a test environment first. + +To enable token hashing, refer to [this section](../../pages-for-subheaders/enable-experimental-features.md). + +This feature will affect all tokens which include, but are not limited to, the following: + +- Kubeconfig tokens +- Bearer tokens API keys/calls +- Tokens used by internal operations + +### Token Settings +These global settings affect Rancher token behavior. + +| Setting | Description | +| ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes) | TTL in minutes on a user auth session token. | +| [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes) | Default TTL applied to all kubeconfig tokens except those [generated by Rancher CLI](#disable-tokens-in-generated-kubeconfigs). **Introduced in version 2.6.6.** | +| [`kubeconfig-token-ttl-minutes`](#kubeconfig-token-ttl-minutes) | TTL used for tokens generated via the CLI. **Deprecated since version 2.6.6, and will be removed in 2.8.0.** This setting will be removed, and `kubeconfig-default-token-TTL-minutes` will be used for all kubeconfig tokens. | +| [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes) | Max TTL for all tokens except those controlled by [`auth-user-session-ttl-minutes`](#auth-user-session-ttl-minutes). | +| [`kubeconfig-generate-token`](#kubeconfig-generate-token) | If true, automatically generate tokens when a user downloads a kubeconfig. | + +#### auth-user-session-ttl-minutes +Time to live (TTL) duration in minutes used to determine when a user auth session token expires. When expired, the user will be required to log in and obtain a new token. This setting is not affected by [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes). Session tokens are created when a user logs into Rancher. + +#### kubeconfig-default-token-TTL-minutes +Time to live (TTL) duration in minutes used to determine when a kubeconfig token expires. When the token is expired, the API will reject the token. This setting can not be larger than [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes). This setting applies to a token generated in a requested kubeconfig file. Except those [generated by Rancher CLI](#disable-tokens-in-generated-kubeconfigs). +**Introduced in version 2.6.6**. + +#### kubeconfig-token-ttl-minutes +Time to live (TTL) duration in minutes used to determine when a kubeconfig token that was generated by the CLI expires. Tokens are generated by the CLI when [`kubeconfig-generate-token`](#kubeconfig-generate-token) is false. When the token is expired, the API will reject the token. This setting can not be larger than [`auth-token-max-ttl-minutes`](#auth-token-max-ttl-minutes). +**Deprecated since version 2.6.6, and will be removed in 2.8.0: This setting will be replaced with the value of [`kubeconfig-default-token-TTL-minutes`](#kubeconfig-default-token-ttl-minutes).** + +#### auth-token-max-ttl-minutes +Maximum Time to Live (TTL) in minutes allowed for auth tokens. If a user attempts to create a token with a TTL greater than `auth-token-max-ttl-minutes`, Rancher will set the token TTL to the value of `auth-token-max-ttl-minutes`. Auth tokens are tokens created for authenticating API requests. +**Changed in version 2.6.6: Applies to all kubeconfig tokens and api tokens.** + +#### kubeconfig-generate-token +When true, kubeconfigs requested through the UI will contain a valid token. When false, the kubeconfig will contain a command that uses the Rancher CLI to prompt the user to log in. [The CLI then will retrieve and cache a token for the user](../cli-with-rancher/kubectl-utility.md#authentication-with-kubectl-and-kubeconfig-tokens-with-ttl). diff --git a/versioned_docs/version-latest/reference-guides/backup-restore-configuration/backup-configuration.md b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/backup-configuration.md new file mode 100644 index 00000000000..e7ef2b056bf --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/backup-configuration.md @@ -0,0 +1,186 @@ +--- +title: Backup Configuration +--- + + + + + +The Backup Create page lets you configure a schedule, enable encryption and specify the storage location for your backups. + + +## Schedule + +Select the first option to perform a one-time backup, or select the second option to schedule recurring backups. Selecting **Recurring Backups** lets you configure following two fields: + +- **Schedule**: This field accepts + - Standard [cron expressions](https://en.wikipedia.org/wiki/Cron), such as `"0 * * * *"` + - Descriptors, such as `"@midnight"` or `"@every 1h30m"` +- **Retention Count**: This value specifies how many backup files must be retained. If files exceed the given retentionCount, the oldest files will be deleted. The default value is 10. + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `schedule` | Provide the cron string for scheduling recurring backups. | +| `retentionCount` | Provide the number of backup files to be retained. | + +## Encryption + +The rancher-backup gathers resources by making calls to the kube-apiserver. Objects returned by apiserver are decrypted, so even if [encryption at rest](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/) is enabled, even the encrypted objects gathered by the backup will be in plaintext. + +To avoid storing them in plaintext, you can use the same `EncryptionConfiguration` file that was used for at rest encryption, to encrypt certain resources in your backup. + +:::note Important: + +When encrypting objects in the backup you must save the `EncryptionConfiguration` file for future use, because it won’t be saved by the rancher-backup operator. + +For example, when [migrating Rancher to a new cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md) the file is used to re-create the secret in the new cluster. + +::: + +The operator consumes the `EncryptionConfiguration` as a Kubernetes Secret in the `cattle-resources-system` namespace under the key named `encryption-provider-config.yaml` in the secret data. + +For the `EncryptionConfiguration`, you can use the [sample file provided in the Kubernetes documentation.](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#understanding-the-encryption-at-rest-configuration) + +To ensure the correct key is used in the secret, the encryption configuration file must be named `encryption-provider-config.yaml`. The below command uses the `--from-file` flag to create the secret with the correct key name. + +Save the `EncryptionConfiguration` in a file called `encryption-provider-config.yaml` and run this command: + +```bash +kubectl create secret generic encryptionconfig \ + --from-file=./encryption-provider-config.yaml \ + -n cattle-resources-system +``` + +This will ensure that the secret contains a key named `encryption-provider-config.yaml`, and the operator will use this key to get the encryption configuration. + +The `Encryption Config Secret` dropdown will filter out and list only those Secrets that have this exact key. + +![](/img/backup_restore/backup/encryption.png) + +In the example command above, the name `encryptionconfig` can be changed to anything. + + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | Provide the name of the Secret from `cattle-resources-system` namespace, that contains the encryption config file. | + +## Storage Location + +If the StorageLocation is specified in the Backup, the operator will retrieve the backup location from that particular S3 bucket. If not specified, the operator will try to find this file in the default operator-level S3 store, and in the operator-level PVC store. The default storage location is configured during the deployment of the `rancher-backup` operator. + +Selecting the first option stores this backup in the storage location configured while installing the rancher-backup chart. The second option lets you configure a different S3 compatible storage provider for storing the backup. + +### S3 + +The S3 storage location contains the following configuration fields: + +1. **Credential Secret** (optional): If you need to use the AWS Access keys Secret keys to access s3 bucket, create a secret with your credentials with keys and the directives `accessKey` and `secretKey`. It can be in any namespace. An example secret is [here.](#example-credentialsecret) This directive is unnecessary if the nodes running your operator are in EC2 and set up with IAM permissions that allow them to access S3, as described in [this section.](#iam-permissions-for-ec2-nodes-to-access-s3) The Credential Secret dropdown lists the secrets in all namespaces. +1. **Bucket Name**: The name of the S3 bucket where backup files will be stored. +1. **Region** (optional): The AWS [region](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) where the S3 bucket is located. This field isn't needed for configuring MinIO. +1. **Folder** (optional): The name of the folder in the S3 bucket where backup files will be stored. Nested folders (e.g., `rancher/cluster1`) are not supported. If this field is left empty, the default behavior is to store the backup files in the root folder of the S3 bucket. +1. **Endpoint**: The [endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) that is used to access S3 in the region of your bucket. +1. **Endpoint CA** (optional): This should be the Base64 encoded CA cert. For an example, refer to the [example S3 compatible configuration.](#example-s3-storage-configuration) +1. **Skip TLS Verifications** (optional): Set to true if you are not using TLS. + + +| YAML Directive Name | Description | Required | +| ---------------- | ---------------- | ------------ | +| `credentialSecretName` | If you need to use the AWS Access keys Secret keys to access s3 bucket, create a secret with your credentials with keys and the directives `accessKey` and `secretKey`. It can be in any namespace as long as you provide that namespace in `credentialSecretNamespace`. An example secret is [here.](#example-credentialsecret) This directive is unnecessary if the nodes running your operator are in EC2 and set up with IAM permissions that allow them to access S3, as described in [this section.](#iam-permissions-for-ec2-nodes-to-access-s3) | | +| `credentialSecretNamespace` | The namespace of the secret containing the credentials to access S3. This directive is unnecessary if the nodes running your operator are in EC2 and set up with IAM permissions that allow them to access S3, as described in [this section.](#iam-permissions-for-ec2-nodes-to-access-s3) | | +| `bucketName` | The name of the S3 bucket where backup files will be stored. | ✓ | +| `folder` | The name of the folder in the S3 bucket where backup files will be stored. Nested folders (e.g., `rancher/cluster1`) are not supported. If this field is left empty, the default behavior is to store the backup files in the root folder of the S3 bucket. | | +| `region` | The AWS [region](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) where the S3 bucket is located. | ✓ | +| `endpoint` | The [endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) that is used to access S3 in the region of your bucket. | ✓ | +| `endpointCA` | This should be the Base64 encoded CA cert. For an example, refer to the [example S3 compatible configuration.](#example-s3-storage-configuration) | | +| `insecureTLSSkipVerify` | Set to true if you are not using TLS. | | + +### Example S3 Storage Configuration + +```yaml +s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: rancher + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### Example MinIO Configuration + +```yaml +s3: + credentialSecretName: minio-creds + bucketName: rancherbackups + endpoint: minio.35.202.130.254.xip.io + endpointCA: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURHakNDQWdLZ0F3SUJBZ0lKQUtpWFZpNEpBb0J5TUEwR0NTcUdTSWIzRFFFQkN3VUFNQkl4RURBT0JnTlYKQkFNTUIzUmxjM1F0WTJFd0hoY05NakF3T0RNd01UZ3lOVFE1V2hjTk1qQXhNREk1TVRneU5UUTVXakFTTVJBdwpEZ1lEVlFRRERBZDBaWE4wTFdOaE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBCjA4dnV3Q2Y0SEhtR2Q2azVNTmozRW5NOG00T2RpS3czSGszd1NlOUlXQkwyVzY5WDZxenBhN2I2M3U2L05mMnkKSnZWNDVqeXplRFB6bFJycjlpbEpWaVZ1NFNqWlFjdG9jWmFCaVNsL0xDbEFDdkFaUlYvKzN0TFVTZSs1ZDY0QQpWcUhDQlZObU5xM3E3aVY0TE1aSVpRc3N6K0FxaU1Sd0pOMVVKQTZ6V0tUc2Yzc3ByQ0J2dWxJWmZsVXVETVAyCnRCTCt6cXZEc0pDdWlhNEEvU2JNT29tVmM2WnNtTGkwMjdub3dGRld3MnRpSkM5d0xMRE14NnJoVHQ4a3VvVHYKQXJpUjB4WktiRU45L1Uzb011eUVKbHZyck9YS2ZuUDUwbk8ycGNaQnZCb3pUTStYZnRvQ1d5UnhKUmI5cFNTRApKQjlmUEFtLzNZcFpMMGRKY2sxR1h3SURBUUFCbzNNd2NUQWRCZ05WSFE0RUZnUVU5NHU4WXlMdmE2MTJnT1pyCm44QnlFQ2NucVFjd1FnWURWUjBqQkRzd09ZQVU5NHU4WXlMdmE2MTJnT1pybjhCeUVDY25xUWVoRnFRVU1CSXgKRURBT0JnTlZCQU1NQjNSbGMzUXRZMkdDQ1FDb2wxWXVDUUtBY2pBTUJnTlZIUk1FQlRBREFRSC9NQTBHQ1NxRwpTSWIzRFFFQkN3VUFBNElCQVFER1JRZ1RtdzdVNXRQRHA5Q2psOXlLRW9Vd2pYWWM2UlAwdm1GSHpubXJ3dUVLCjFrTkVJNzhBTUw1MEpuS29CY0ljVDNEeGQ3TGdIbTNCRE5mVVh2anArNnZqaXhJYXR2UWhsSFNVaWIyZjJsSTkKVEMxNzVyNCtROFkzelc1RlFXSDdLK08vY3pJTGh5ei93aHRDUlFkQ29lS1dXZkFiby8wd0VSejZzNkhkVFJzNwpHcWlGNWZtWGp6S0lOcTBjMHRyZ0xtalNKd1hwSnU0ZnNGOEcyZUh4b2pOKzdJQ1FuSkg5cGRIRVpUQUtOL2ppCnIvem04RlZtd1kvdTBndEZneWVQY1ZWbXBqRm03Y0ZOSkc4Y2ZYd0QzcEFwVjhVOGNocTZGeFBHTkVvWFZnclMKY1VRMklaU0RJd1FFY3FvSzFKSGdCUWw2RXBaUVpWMW1DRklrdFBwSQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t +``` +### Example credentialSecret + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +:::note + +To avoid encoding issues, the `credentialSecret` can be created with the below command, updating the values for `accessKey` and `secretKey`. + +```bash +kubectl create secret generic s3-creds \ + --from-literal=accessKey= \ + --from-literal=secretKey= +``` + +::: + +### IAM Permissions for EC2 Nodes to Access S3 + +There are two ways to set up the `rancher-backup` operator to use S3 as the backup storage location. + +One way is to configure the `credentialSecretName` in the Backup custom resource, which refers to AWS credentials that have access to S3. + +If the cluster nodes are in Amazon EC2, the S3 access can also be set up by assigning IAM permissions to the EC2 nodes so that they can access S3. + +To allow a node to access S3, follow the instructions in the [AWS documentation](https://aws.amazon.com/premiumsupport/knowledge-center/ec2-instance-access-s3-bucket/) to create an IAM role for EC2. When you add a custom policy to the role, add the following permissions, and replace the `Resource` with your bucket name: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObjectAcl" + ], + "Resource": [ + "arn:aws:s3:::rancher-backups/*" + ] + } + ] +} +``` + +After the role is created, and you have attached the corresponding instance profile to your EC2 instance(s), the `credentialSecretName` directive can be left empty in the Backup custom resource. + +## Examples + +For example Backup custom resources, refer to [this page.](examples.md#backup) diff --git a/versioned_docs/version-latest/reference-guides/backup-restore-configuration/examples.md b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/examples.md new file mode 100644 index 00000000000..bab7533302e --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/examples.md @@ -0,0 +1,297 @@ +--- +title: Examples +--- + + + + + +This section contains examples of Backup and Restore custom resources. + +The default backup storage location is configured when the `rancher-backup` operator is installed or upgraded. + +Encrypted backups can only be restored if the Restore custom resource uses the same encryption configuration secret that was used to create the backup. + +## Backup + +This section contains example Backup custom resources. + +>**Note:** Refer to the [backup config reference page](./backup-configuration.md) for more information on configuring the options below. + +### Backup in the Default Location with Encryption + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-encrypted-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### Recurring Backup in the Default Location + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-location-recurring-backup +spec: + resourceSetName: rancher-resource-set + schedule: "@every 1h" + retentionCount: 10 +``` + +### Encrypted Recurring Backup in the Default Location + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: default-enc-recurring-backup +spec: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 3 +``` + +### Encrypted Backup in Minio + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: minio-backup +spec: + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### Backup in S3 Using AWS Credential Secret + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +### Recurring Backup in S3 Using AWS Credential Secret + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-recurring-backup +spec: + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig + schedule: "@every 1h" + retentionCount: 10 +``` + +### Backup from EC2 Nodes with IAM Permission to Access S3 + +This example shows that the AWS credential secret does not have to be provided to create a backup if the nodes running `rancher-backup` have [these permissions for access to S3.](backup-configuration.md#iam-permissions-for-ec2-nodes-to-access-s3) + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Backup +metadata: + name: s3-iam-backup +spec: + storageLocation: + s3: + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + resourceSetName: rancher-resource-set + encryptionConfigSecretName: encryptionconfig +``` + +## Restore + +This section contains example Restore custom resources. + +>**Note:** Refer to the [restore config reference page](./restore-configuration.md) for more information on configuring the options below. + +### Restore Using the Default Backup File Location + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-default +spec: + backupFilename: default-location-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-29-54-07-00.tar.gz +# encryptionConfigSecretName: test-encryptionconfig +``` + +### Restore for Rancher Migration +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-migration +spec: + backupFilename: backup-b0450532-cee1-4aa1-a881-f5f48a007b1c-2020-09-15T07-27-09Z.tar.gz + prune: false + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com +``` + +### Restore from Encrypted Backup + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-encrypted +spec: + backupFilename: default-test-s3-def-backup-c583d8f2-6daf-4648-8ead-ed826c591471-2020-08-24T20-47-05Z.tar.gz + encryptionConfigSecretName: encryptionconfig +``` + +### Restore an Encrypted Backup from Minio + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-minio +spec: + backupFilename: default-minio-backup-demo-aa5c04b7-4dba-4c48-9ac4-ab7916812eaa-2020-08-30T13-18-17-07-00.tar.gz + storageLocation: + s3: + credentialSecretName: minio-creds + credentialSecretNamespace: default + bucketName: rancherbackups + endpoint: minio.xip.io + endpointCA: + encryptionConfigSecretName: test-encryptionconfig +``` + +### Restore from Backup Using an AWS Credential Secret to Access S3 + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: test-s3-recurring-backup-752ecd87-d958-4d20-8350-072f8d090045-2020-09-26T12-49-34-07-00.tar.gz.enc + storageLocation: + s3: + credentialSecretName: s3-creds + credentialSecretNamespace: default + bucketName: rancher-backups + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +### Restore from EC2 Nodes with IAM Permissions to Access S3 + +This example shows that the AWS credential secret does not have to be provided to restore from backup if the nodes running `rancher-backup` have [these permissions for access to S3.](backup-configuration.md#iam-permissions-for-ec2-nodes-to-access-s3) + +```yaml +apiVersion: resources.cattle.io/v1 +kind: Restore +metadata: + name: restore-s3-demo +spec: + backupFilename: default-test-s3-recurring-backup-84bf8dd8-0ef3-4240-8ad1-fc7ec308e216-2020-08-24T10#52#44-07#00.tar.gz + storageLocation: + s3: + bucketName: rajashree-backup-test + folder: ecm1 + region: us-west-2 + endpoint: s3.us-west-2.amazonaws.com + encryptionConfigSecretName: test-encryptionconfig +``` + +## Example Credential Secret for Storing Backups in S3 + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: creds +type: Opaque +data: + accessKey: + secretKey: +``` + +## Example EncryptionConfiguration + +The snippet below demonstrates two different types of secrets and their relevance with respect to Backup and Restore of custom resources. + +The first example is that of a secret that is used to encrypt the backup files. The backup operator, in this case, will not be able to read the secrets encryption file. It only uses the contents of the secret. + +The second example is that of a Kubernetes secrets encryption config file that is used to encrypt secrets when stored in etcd. **When backing up the etcd datastore, be sure to also back up the EncryptionConfiguration.** Failure to do so will result in an inability to use the restored data if secrets encryption was in use at the time the data was backed up. + + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: EncryptionConfiguration +resources: + - resources: + - secrets + providers: + - aesgcm: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - aescbc: + keys: + - name: key1 + secret: c2VjcmV0IGlzIHNlY3VyZQ== + - name: key2 + secret: dGhpcyBpcyBwYXNzd29yZA== + - secretbox: + keys: + - name: key1 + secret: YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoxMjM0NTY= +``` + + + diff --git a/versioned_docs/version-latest/reference-guides/backup-restore-configuration/restore-configuration.md b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/restore-configuration.md new file mode 100644 index 00000000000..b162a6ef873 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/restore-configuration.md @@ -0,0 +1,85 @@ +--- +title: Restore Configuration +--- + + + + + +The Restore Create page lets you provide details of the backup to restore from + +![](/img/backup_restore/restore/restore.png) + + +## Backup Source +Provide details of the backup file and its storage location, which the operator will then use to perform the restore. Select from the following options to provide these details + + + + +### An existing backup config + +Selecting this option will populate the **Target Backup** dropdown with the Backups available in this cluster. Select the Backup from the dropdown, and that will fill out the **Backup Filename** field for you, and will also pass the backup source information from the selected Backup to the operator. + +![](/img/backup_restore/restore/existing.png) + +If the Backup custom resource does not exist in the cluster, you need to get the exact filename and provide the backup source details with the default storage target or an S3-compatible object store. + + +### The default storage target + +Select this option if you are restoring from a backup file that exists in the default storage location configured at the operator-level. The operator-level configuration is the storage location that was configured when the `rancher-backup` operator was installed or upgraded. Provide the exact filename in the **Backup Filename** field. + +![](/img/backup_restore/restore/default.png) + +### An S3-compatible object store + +Select this option if no default storage location is configured at the operator-level, OR if the backup file exists in a different S3 bucket than the one configured as the default storage location. Provide the exact filename in the **Backup Filename** field. Refer to [this section](#getting-the-backup-filename-from-s3) for exact steps on getting the backup filename from s3. Fill in all the details for the S3 compatible object store. Its fields are exactly same as ones for the `backup.StorageLocation` configuration in the [Backup custom resource.](backup-configuration.md#storage-location) + +![](/img/backup_restore/restore/s3store.png) + +## Encryption + +If the backup was created with encryption enabled, its file will have `.enc` suffix. Choosing such a Backup, or providing a backup filename with `.enc` suffix will display another dropdown named **Encryption Config Secret**. + +![](/img/backup_restore/restore/encryption.png) + +The Secret selected from this dropdown must have the same contents as the one used for the Backup custom resource while performing the backup. If the encryption configuration doesn't match, the restore will fail + +The `Encryption Config Secret` dropdown will filter out and list only those Secrets that have this exact key + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `encryptionConfigSecretName` | Provide the name of the Secret from `cattle-resources-system` namespace, that contains the encryption config file. | + +:::note Important: + +This field should only be set if the backup was created with encryption enabled. Providing the incorrect encryption config will cause the restore to fail. + +::: + +## Prune During Restore + +* **Prune**: In order to fully restore Rancher from a backup, and to go back to the exact state it was at when the backup was performed, we need to delete any additional resources that were created by Rancher after the backup was taken. The operator does so if the **Prune** flag is enabled. Prune is enabled by default and it is recommended to keep it enabled. +* **Delete Timeout**: This is the amount of time the operator will wait while deleting a resource before editing the resource to remove finalizers and attempt deletion again. + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `prune` | Delete the resources managed by Rancher that are not present in the backup (Recommended). | +| `deleteTimeoutSeconds` | Amount of time the operator will wait while deleting a resource before editing the resource to remove finalizers and attempt deletion again. | + +## Getting the Backup Filename from S3 + +This is the name of the backup file that the `rancher-backup` operator will use to perform the restore. + +To obtain this file name from S3, go to your S3 bucket (and folder if it was specified while performing backup). + +Copy the filename and store it in your Restore custom resource. So assuming the name of your backup file is `backupfile`, + +- If your bucket name is `s3bucket` and no folder was specified, then the `backupFilename` to use will be `backupfile`. +- If your bucket name is `s3bucket` and the base folder is`s3folder`, the `backupFilename` to use is only `backupfile` . +- If there is a subfolder inside `s3Folder` called `s3sub`, and that has your backup file, then the `backupFilename` to use is `s3sub/backupfile`. + +| YAML Directive Name | Description | +| ---------------- | ---------------- | +| `backupFilename` | This is the name of the backup file that the `rancher-backup` operator will use to perform the restore. | diff --git a/versioned_docs/version-latest/reference-guides/backup-restore-configuration/storage-configuration.md b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/storage-configuration.md new file mode 100644 index 00000000000..b1b8bad0896 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/backup-restore-configuration/storage-configuration.md @@ -0,0 +1,60 @@ +--- +title: Backup Storage Location Configuration +--- + + + + + +Configure a storage location where all backups are saved by default. You will have the option to override this with each backup, but will be limited to using an S3-compatible object store. + +Only one storage location can be configured at the operator level. + + +## Storage Location Configuration + +### No Default Storage Location + +You can choose to not have any operator-level storage location configured. If you select this option, you must configure an S3-compatible object store as the storage location for each individual backup. + +### S3-compatible Object Store + +| Parameter | Description | +| -------------- | -------------- | +| Credential Secret | Choose the credentials for S3 from your secrets in Rancher. [Example](examples.md#example-credential-secret-for-storing-backups-in-s3). | +| Bucket Name | Enter the name of the [S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html) where the backups will be stored. Default: `rancherbackups`. | +| Region | The [AWS region](https://aws.amazon.com/about-aws/global-infrastructure/regions_az/) where the S3 bucket is located. | +| Folder | The [folder in the S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html) where the backups will be stored. If this field is left empty, the default behavior is to store the backup files in the root folder of the S3 bucket. | +| Endpoint | The [S3 endpoint](https://docs.aws.amazon.com/general/latest/gr/s3.html) For example, `s3.us-west-2.amazonaws.com`. | +| Endpoint CA | The CA cert used to for the S3 endpoint. Default: base64 encoded CA cert | +| insecureTLSSkipVerify | Set to true if you are not using TLS. | + +### Existing StorageClass + +Installing the `rancher-backup` chart by selecting the StorageClass option will create a Persistent Volume Claim (PVC), and Kubernetes will in turn dynamically provision a Persistent Volume (PV) where all the backups will be saved by default. + +For information about creating storage classes refer to [this section.](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage.md) + +:::note Important: + +It is highly recommended to use a StorageClass with a reclaim policy of "Retain". Otherwise if the PVC created by the `rancher-backup` chart gets deleted (either during app upgrade, or accidentally), the PV will get deleted too, which means all backups saved in it will get deleted. +If no such StorageClass is available, after the PV is provisioned, make sure to edit its reclaim policy and set it to "Retain" before storing backups in it. + +::: + +### Existing Persistent Volume + +Select an existing Persistent Volume (PV) that will be used to store your backups. For information about creating PersistentVolumes in Rancher, refer to [this section.](../../how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage.md#2-add-a-persistentvolume-that-refers-to-the-persistent-storage) + +:::note Important: + +It is highly recommended to use a Persistent Volume with a reclaim policy of "Retain". Otherwise if the PVC created by the `rancher-backup` chart gets deleted (either during app upgrade, or accidentally), the PV will get deleted too, which means all backups saved in it will get deleted. + +::: + +## Example values.yaml for the rancher-backup Helm Chart + +The documented `values.yaml` file that can be used to configure `rancher-backup` operator when the Helm CLI is used can be found in the [backup-restore-operator repository.](https://github.com/rancher/backup-restore-operator/blob/master/charts/rancher-backup/values.yaml) + +For more information about `values.yaml` files and configuring Helm charts during installation, refer to the [Helm documentation.](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing) + diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md new file mode 100644 index 00000000000..dc200ae62b1 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/logging-best-practices.md @@ -0,0 +1,92 @@ +--- +title: Logging Best Practices +--- + + + + + +In this guide, we recommend best practices for cluster-level logging and application logging. + +- [Cluster-level Logging](#cluster-level-logging) +- [Application Logging](#application-logging) +- [General Best Practices](#general-best-practices) + +Before Rancher v2.5, logging in Rancher has historically been a pretty static integration. There were a fixed list of aggregators to choose from (ElasticSearch, Splunk, Kafka, Fluentd and Syslog), and only two configuration points to choose (Cluster-level and Project-level). + +Rancher provides a flexible experience for log aggregation. With the logging feature, administrators and users alike can deploy logging that meets fine-grained collection criteria while offering a wider array of destinations and configuration options. + +"Under the hood", Rancher logging uses the [Logging operator](https://github.com/kube-logging/logging-operator). We provide manageability of this operator (and its resources), and tie that experience in with managing your Rancher clusters. + +## Cluster-level Logging + +### Cluster-wide Scraping + +For some users, it is desirable to scrape logs from every container running in the cluster. This usually coincides with your security team's request (or requirement) to collect all logs from all points of execution. + +In this scenario, it is recommended to create at least two _ClusterOutput_ objects - one for your security team (if you have that requirement), and one for yourselves, the cluster administrators. When creating these objects take care to choose an output endpoint that can handle the significant log traffic coming from the entire cluster. Also make sure to choose an appropriate index to receive all these logs. + +Once you have created these _ClusterOutput_ objects, create a _ClusterFlow_ to collect all the logs. Do not define any _Include_ or _Exclude_ rules on this flow. This will ensure that all logs from across the cluster are collected. If you have two _ClusterOutputs_, make sure to send logs to both of them. + +### Kubernetes Components + +_ClusterFlows_ have the ability to collect logs from all containers on all hosts in the Kubernetes cluster. This works well in cases where those containers are part of a Kubernetes pod; however, RKE containers exist outside of the scope of Kubernetes. + +Currently the logs from RKE containers are collected, but are not able to easily be filtered. This is because those logs do not contain information as to the source container (e.g. `etcd` or `kube-apiserver`). + +A future release of Rancher will include the source container name which will enable filtering of these component logs. Once that change is made, you will be able to customize a _ClusterFlow_ to retrieve **only** the Kubernetes component logs, and direct them to an appropriate output. + +## Application Logging + +Best practice not only in Kubernetes but in all container-based applications is to direct application logs to `stdout`/`stderr`. The container runtime will then trap these logs and do **something** with them - typically writing them to a file. Depending on the container runtime (and its configuration), these logs can end up in any number of locations. + +In the case of writing the logs to a file, Kubernetes helps by creating a `/var/log/containers` directory on each host. This directory symlinks the log files to their actual destination (which can differ based on configuration or container runtime). + +Rancher logging will read all log entries in `/var/log/containers`, ensuring that all log entries from all containers (assuming a default configuration) will have the opportunity to be collected and processed. + +### Specific Log Files + +Log collection only retrieves `stdout`/`stderr` logs from pods in Kubernetes. But what if we want to collect logs from other files that are generated by applications? Here, a log streaming sidecar (or two) may come in handy. + +The goal of setting up a streaming sidecar is to take log files that are written to disk, and have their contents streamed to `stdout`. This way, the Logging Operator can pick up those logs and send them to your desired output. + +To set this up, edit your workload resource (e.g. Deployment) and add the following sidecar definition: + +```yaml +... +containers: +- args: + - -F + - /path/to/your/log/file.log + command: + - tail + image: busybox + name: stream-log-file-[name] + volumeMounts: + - mountPath: /path/to/your/log + name: mounted-log +... +``` + +This will add a container to your workload definition that will now stream the contents of (in this example) `/path/to/your/log/file.log` to `stdout`. + +This log stream is then automatically collected according to any _Flows_ or _ClusterFlows_ you have setup. You may also wish to consider creating a _Flow_ specifically for this log file by targeting the name of the container. See example: + +```yaml +... +spec: + match: + - select: + container_names: + - stream-log-file-name +... +``` + + +## General Best Practices + +- Where possible, output structured log entries (e.g. `syslog`, JSON). This makes handling of the log entry easier as there are already parsers written for these formats. +- Try to provide the name of the application that is creating the log entry, in the entry itself. This can make troubleshooting easier as Kubernetes objects do not always carry the name of the application as the object name. For instance, a pod ID may be something like `myapp-098kjhsdf098sdf98` which does not provide much information about the application running inside the container. +- Except in the case of collecting all logs cluster-wide, try to scope your _Flow_ and _ClusterFlow_ objects tightly. This makes it easier to troubleshoot when problems arise, and also helps ensure unrelated log entries do not show up in your aggregator. An example of tight scoping would be to constrain a _Flow_ to a single _Deployment_ in a namespace, or perhaps even a single container within a _Pod_. +- Keep the log verbosity down except when troubleshooting. High log verbosity poses a number of issues, chief among them being **noise**: significant events can be drowned out in a sea of `DEBUG` messages. This is somewhat mitigated with automated alerting and scripting, but highly verbose logging still places an inordinate amount of stress on the logging infrastructure. +- Where possible, try to provide a transaction or request ID with the log entry. This can make tracing application activity across multiple log sources easier, especially when dealing with distributed applications. diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md new file mode 100644 index 00000000000..a6e34b03a50 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices.md @@ -0,0 +1,115 @@ +--- +title: Monitoring Best Practices +--- + + + + + +Configuring sensible monitoring and alerting rules is vital for running any production workloads securely and reliably. This is not different when using Kubernetes and Rancher. Fortunately the integrated monitoring and alerting functionality makes this whole process a lot easier. + +The [Rancher monitoring documentation](../../../pages-for-subheaders/monitoring-and-alerting.md) describes how you can set up a complete Prometheus and Grafana stack. Out of the box this will scrape monitoring data from all system and Kubernetes components in your cluster and provide sensible dashboards and alerts for them to get started. But for a reliable setup, you also need to monitor your own workloads and adapt Prometheus and Grafana to your own specific use cases and cluster sizes. This document aims to give you best practices for this. + +## What to Monitor + +Kubernetes itself, as well as applications running inside of it, form a distributed system where different components interact with each other. For the whole system and each individual component, you have to ensure performance, availability, reliability and scalability. A good resource with more details and information is Google's free [Site Reliability Engineering Book](https://sre.google/sre-book/table-of-contents/), especially the chapter about [Monitoring distributed systems](https://sre.google/sre-book/monitoring-distributed-systems/). + +## Configuring Prometheus Resource Usage + +When installing the integrated monitoring stack, Rancher allows to configure several settings that are dependent on the size of your cluster and the workloads running in it. This chapter covers these in more detail. + +### Storage and Data Retention + +The amount of storage needed for Prometheus directly correlates to the amount of time series and labels that you store and the data retention you have configured. It is important to note that Prometheus is not meant to be used as a long-term metrics storage. Data retention time is usually only a couple of days and not weeks or months. The reason for this is that Prometheus does not perform any aggregation on its stored metrics. This is great because aggregation can dilute data, but it also means that the needed storage grows linearly over time without retention. + +One way to calculate the necessary storage is to look at the average size of a storage chunk in Prometheus with this query + +``` +rate(prometheus_tsdb_compaction_chunk_size_bytes_sum[1h]) / rate(prometheus_tsdb_compaction_chunk_samples_sum[1h]) +``` + +Next, find out your data ingestion rate per second: + +``` +rate(prometheus_tsdb_head_samples_appended_total[1h]) +``` + +and then multiply this with the retention time, adding a few percentage points as buffer: + +``` +average chunk size in bytes * ingestion rate per second * retention time in seconds * 1.1 = necessary storage in bytes +``` + +You can find more information about how to calculate the necessary storage in this [blog post](https://www.robustperception.io/how-much-disk-space-do-prometheus-blocks-use). + +You can read more about the Prometheus storage concept in the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/storage). + +### CPU and Memory Requests and Limits + +In larger Kubernetes clusters Prometheus can consume quite a bit of memory. The amount of memory Prometheus needs directly correlates to the amount of time series and amount of labels it stores and the scrape interval in which these are filled. + +You can find more information about how to calculate the necessary memory in this [blog post](https://www.robustperception.io/how-much-ram-does-prometheus-2-x-need-for-cardinality-and-ingestion). + +The amount of necessary CPUs correlate with the amount of queries you are performing. + +### Federation and Long-term Storage + +Prometheus is not meant to store metrics for a long amount of time, but should only be used for short term storage. + +In order to store some, or all metrics for a long time, you can leverage Prometheus' [remote read/write](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations) capabilities to connect it to storage systems like [Thanos](https://thanos.io/), [InfluxDB](https://www.influxdata.com/), [M3DB](https://www.m3db.io/), or others. You can find an example setup in this [blog post](https://rancher.com/blog/2020/prometheus-metric-federation). + +## Scraping Custom Workloads + +While the integrated Rancher Monitoring already scrapes system metrics from a cluster's nodes and system components, the custom workloads that you deploy on Kubernetes should also be scraped for data. For that you can configure Prometheus to do an HTTP request to an endpoint of your applications in a certain interval. These endpoints should then return their metrics in a Prometheus format. + +In general, you want to scrape data from all the workloads running in your cluster so that you can use them for alerts or debugging issues. Often, you recognize that you need some data only when you actually need the metrics during an incident. It is good, if it is already scraped and stored. Since Prometheus is only meant to be a short-term metrics storage, scraping and keeping lots of data is usually not that expensive. If you are using a long-term storage solution with Prometheus, you can then still decide which data you are actually persisting and keeping there. + +### About Prometheus Exporters + +A lot of 3rd party workloads like databases, queues or web-servers either already support exposing metrics in a Prometheus format, or there are so called exporters available that translate between the tool's metrics and the format that Prometheus understands. Usually you can add these exporters as additional sidecar containers to the workload's Pods. A lot of helm charts already include options to deploy the correct exporter. Additionally you can find a curated list of exports by SysDig on [promcat.io](https://promcat.io/) and on [ExporterHub](https://exporterhub.io/). + +### Prometheus support in Programming Languages and Frameworks + +To get your own custom application metrics into Prometheus, you have to collect and expose these metrics directly from your application's code. Fortunately, there are already libraries and integrations available to help with this for most popular programming languages and frameworks. One example for this is the Prometheus support in the [Spring Framework](https://docs.spring.io/spring-metrics/docs/current/public/prometheus). + +### ServiceMonitors and PodMonitors + +Once all your workloads expose metrics in a Prometheus format, you have to configure Prometheus to scrape it. Under the hood Rancher is using the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator). This makes it easy to add additional scraping targets with ServiceMonitors and PodMonitors. A lot of helm charts already include an option to create these monitors directly. You can also find more information in the Rancher documentation. + +### Prometheus Push Gateway + +There are some workloads that are traditionally hard to scrape by Prometheus. Examples for these are short lived workloads like Jobs and CronJobs, or applications that do not allow sharing data between individual handled incoming requests, like PHP applications. + +To still get metrics for these use cases, you can set up [prometheus-pushgateways](https://github.com/prometheus/pushgateway). The CronJob or PHP application would push metric updates to the pushgateway. The pushgateway aggregates and exposes them through an HTTP endpoint, which then can be scraped by Prometheus. + +### Prometheus Blackbox Monitor + +Sometimes it is useful to monitor workloads from the outside. For this, you can use the [Prometheus blackbox-exporter](https://github.com/prometheus/blackbox_exporter) which allows probing any kind of endpoint over HTTP, HTTPS, DNS, TCP and ICMP. + +## Monitoring in a (Micro)Service Architecture + +If you have a (micro)service architecture where multiple individual workloads within your cluster are communicating with each other, it is really important to have detailed metrics and traces about this traffic to understand how all these workloads are communicating with each other and where a problem or bottleneck may be. + +Of course you can monitor all this internal traffic in all your workloads and expose these metrics to Prometheus. But this can quickly become quite work intensive. Service Meshes like Istio, which can be installed with [a click](../../../pages-for-subheaders/istio.md) in Rancher, can do this automatically and provide rich telemetry about the traffic between all services. + +## Real User Monitoring + +Monitoring the availability and performance of all your internal workloads is vitally important to run stable, reliable and fast applications. But these metrics only show you parts of the picture. To get a complete view it is also necessary to know how your end users are actually perceiving it. For this you can look into various [Real user monitoring solutions](https://en.wikipedia.org/wiki/Real_user_monitoring). + +## Security Monitoring + +In addition to monitoring workloads to detect performance, availability or scalability problems, the cluster and the workloads running into it should also be monitored for potential security problems. A good starting point is to frequently run and alert on [CIS Scans](../../../pages-for-subheaders/cis-scan-guides.md) which check if the cluster is configured according to security best practices. + +For the workloads, you can have a look at Kubernetes and Container security solutions like [NeuVector](https://www.suse.com/products/neuvector/), [Falco](https://falco.org/), [Aqua Kubernetes Security](https://www.aquasec.com/solutions/kubernetes-container-security/), [SysDig](https://sysdig.com/). + +## Setting up Alerts + +Getting all the metrics into a monitoring systems and visualizing them in dashboards is great, but you also want to be pro-actively alerted if something goes wrong. + +The integrated Rancher monitoring already configures a sensible set of alerts that make sense in any Kubernetes cluster. You should extend these to cover your specific workloads and use cases. + +When setting up alerts, configure them for all the workloads that are critical to the availability of your applications. But also make sure that they are not too noisy. Ideally every alert you are receiving should be because of a problem that needs your attention and needs to be fixed. If you have alerts that are firing all the time but are not that critical, there is a danger that you start ignoring your alerts all together and then miss the real important ones. Less may be more here. Start to focus on the real important metrics first, for example alert if your application is offline. Fix all the problems that start to pop up and then start to create more detailed alerts. + +If an alert starts firing, but there is nothing you can do about it at the moment, it's also fine to silence the alert for a certain amount of time, so that you can look at it later. + +You can find more information on how to set up alerts and notification channels in the [Rancher Documentation](../../../pages-for-subheaders/monitoring-and-alerting.md). diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md new file mode 100644 index 00000000000..9d24fb3c5db --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere.md @@ -0,0 +1,62 @@ +--- +title: Best Practices for Rancher Managed vSphere Clusters +--- + + + + + +This guide outlines a reference architecture for provisioning downstream Rancher clusters in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. + +- [1. VM Considerations](#1-vm-considerations) +- [2. Network Considerations](#2-network-considerations) +- [3. Storage Considerations](#3-storage-considerations) +- [4. Backups and Disaster Recovery](#4-backups-and-disaster-recovery) + +
Solution Overview
+ +![Solution Overview](/img/solution_overview.drawio.svg) + +## 1. VM Considerations + +### Leverage VM Templates to Construct the Environment + +To facilitate consistency across the deployed Virtual Machines across the environment, consider the use of "Golden Images" in the form of VM templates. Packer can be used to accomplish this, adding greater customisation options. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Downstream Cluster Nodes Across ESXi Hosts + +Doing so will ensure node VM's are spread across multiple ESXi hosts - preventing a single point of failure at the host level. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Downstream Cluster Nodes Across Datastores + +Doing so will ensure node VM's are spread across multiple datastores - preventing a single point of failure at the datastore level. + +### Configure VM's as Appropriate for Kubernetes + +It’s important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double-checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node. + +## 2. Network Considerations + +### Leverage Low Latency, High Bandwidth Connectivity Between ETCD Nodes + +Deploy etcd members within a single data center where possible to avoid latency overheads and reduce the likelihood of network partitioning. For most setups, 1Gb connections will suffice. For large clusters, 10Gb connections can reduce the time taken to restore from backup. + +### Consistent IP Addressing for VM's + +Each node used should have a static IP configured. In the case of DHCP, each node should have a DHCP reservation to make sure the node gets the same IP allocated. + +## 3. Storage Considerations + +### Leverage SSD Drives for ETCD Nodes + +ETCD is very sensitive to write latency. Therefore, leverage SSD disks where possible. + +## 4. Backups and Disaster Recovery + +### Perform Regular Downstream Cluster Backups + +Kubernetes uses etcd to store all its data - from configuration, state and metadata. Backing this up is crucial in the event of disaster recovery. + +### Back up Downstream Node VMs + +Incorporate the Rancher downstream node VM's within a standard VM backup policy. \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md new file mode 100644 index 00000000000..0cec0a91864 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers.md @@ -0,0 +1,56 @@ +--- +title: Tips for Setting Up Containers +--- + + + + + +Running well-built containers can greatly impact the overall performance and security of your environment. + +Below are a few tips for setting up your containers. + +For a more detailed discussion of security for containers, you can also refer to Rancher's [Guide to Container Security.](https://rancher.com/complete-guide-container-security) + +### Use a Common Container OS + +When possible, you should try to standardize on a common container base OS. + +Smaller distributions such as Alpine and BusyBox reduce container image size and generally have a smaller attack/vulnerability surface. + +Popular distributions such as Ubuntu, Fedora, and CentOS are more field-tested and offer more functionality. + +### Start with a FROM scratch container +If your microservice is a standalone static binary, you should use a FROM scratch container. + +The FROM scratch container is an [official Docker image](https://hub.docker.com/_/scratch) that is empty so that you can use it to design minimal images. + +This will have the smallest attack surface and smallest image size. + +### Run Container Processes as Unprivileged +When possible, use a non-privileged user when running processes within your container. While container runtimes provide isolation, vulnerabilities and attacks are still possible. Inadvertent or accidental host mounts can also be impacted if the container is running as root. For details on configuring a security context for a pod or container, refer to the [Kubernetes docs](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). + +### Define Resource Limits +Apply CPU and memory limits to your pods. This can help manage the resources on your worker nodes and avoid a malfunctioning microservice from impacting other microservices. + +In standard Kubernetes, you can set resource limits on the namespace level. In Rancher, you can set resource limits on the project level and they will propagate to all the namespaces within the project. For details, refer to the Rancher docs. + +When setting resource quotas, if you set anything related to CPU or Memory (i.e. limits or reservations) on a project or namespace, all containers will require a respective CPU or Memory field set during creation. To avoid setting these limits on each and every container during workload creation, a default container resource limit can be specified on the namespace. + +The Kubernetes docs have more information on how resource limits can be set at the [container level](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container) and the namespace level. + +### Define Resource Requirements +You should apply CPU and memory requirements to your pods. This is crucial for informing the scheduler which type of compute node your pod needs to be placed on, and ensuring it does not over-provision that node. In Kubernetes, you can set a resource requirement by defining `resources.requests` in the resource requests field in a pod's container spec. For details, refer to the [Kubernetes docs](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-requests-and-limits-of-pod-and-container). + +:::note + +If you set a resource limit for the namespace that the pod is deployed in, and the container doesn't have a specific resource request, the pod will not be allowed to start. To avoid setting these fields on each and every container during workload creation, a default container resource limit can be specified on the namespace. + +::: + +It is recommended to define resource requirements on the container level because otherwise, the scheduler makes assumptions that will likely not be helpful to your application when the cluster experiences load. + +### Liveness and Readiness Probes +Set up liveness and readiness probes for your container. Unless your container completely crashes, Kubernetes will not know it's unhealthy unless you create an endpoint or mechanism that can report container status. Alternatively, make sure your container halts and crashes if unhealthy. + +The Kubernetes docs show how to [configure liveness and readiness probes for containers.](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md new file mode 100644 index 00000000000..74243559753 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere.md @@ -0,0 +1,88 @@ +--- +title: Installing Rancher in a vSphere Environment +--- + + + + + +This guide outlines a reference architecture for installing Rancher on an RKE Kubernetes cluster in a vSphere environment, in addition to standard vSphere best practices as documented by VMware. + + +
Solution Overview
+ +![Solution Overview](/img/rancher-on-prem-vsphere.svg) + +## 1. Load Balancer Considerations + +A load balancer is required to direct traffic to the Rancher workloads residing on the RKE nodes. + +### Leverage Fault Tolerance and High Availability + +Leverage the use of an external (hardware or software) load balancer that has inherit high-availability functionality (F5, NSX-T, Keepalived, etc). + +### Back Up Load Balancer Configuration + +In the event of a Disaster Recovery activity, availability of the Load balancer configuration will expedite the recovery process. + +### Configure Health Checks + +Configure the Load balancer to automatically mark nodes as unavailable if a health check is failed. For example, NGINX can facilitate this with: + +`max_fails=3 fail_timeout=5s` + +### Leverage an External Load Balancer + +Avoid implementing a software load balancer within the management cluster. + +### Secure Access to Rancher + +Configure appropriate Firewall / ACL rules to only expose access to Rancher + +## 2. VM Considerations + +### Size the VM's According to Rancher Documentation + +See [Installation Requirements](../../../pages-for-subheaders/installation-requirements.md). + +### Leverage VM Templates to Construct the Environment + +To facilitate the consistency of Virtual Machines deployed across the environment, consider the use of "Golden Images" in the form of VM templates. Packer can be used to accomplish this, adding greater customization options. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Rancher Cluster Nodes Across ESXi Hosts + +Doing so will ensure node VM's are spread across multiple ESXi hosts - preventing a single point of failure at the host level. + +### Leverage DRS Anti-Affinity Rules (Where Possible) to Separate Rancher Cluster Nodes Across Datastores + +Doing so will ensure node VM's are spread across multiple datastores - preventing a single point of failure at the datastore level. + +### Configure VM's as Appropriate for Kubernetes + +It’s important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double-checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node. + +## 3. Network Considerations + +### Leverage Low Latency, High Bandwidth Connectivity Between ETCD Nodes + +Deploy etcd members within a single data center where possible to avoid latency overheads and reduce the likelihood of network partitioning. For most setups, 1Gb connections will suffice. For large clusters, 10Gb connections can reduce the time taken to restore from backup. + +### Consistent IP Addressing for VM's + +Each node used should have a static IP configured. In the case of DHCP, each node should have a DHCP reservation to make sure the node gets the same IP allocated. + +## 4. Storage Considerations + +### Leverage SSD Drives for ETCD Nodes + +ETCD is very sensitive to write latency. Therefore, leverage SSD disks where possible. + +## 5. Backups and Disaster Recovery + +### Perform Regular Management Cluster Backups + +Rancher stores its data in the ETCD datastore of the Kubernetes cluster it resides on. Like with any Kubernetes cluster, perform frequent, tested backups of this cluster. + +### Back up Rancher Cluster Node VMs + +Incorporate the Rancher management node VM's within a standard VM backup policy. diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md new file mode 100644 index 00000000000..99583f75579 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/rancher-deployment-strategy.md @@ -0,0 +1,40 @@ +--- +title: Rancher Deployment Strategy +--- + + + + + +There are two recommended deployment strategies for a Rancher instance that manages downstream Kubernetes clusters. Each one has its own pros and cons. Read more about which one would fit best for your use case. + +## Hub & Spoke Strategy +--- + +In this deployment scenario, there is a single Rancher instance managing Kubernetes clusters across the globe. The Rancher instance would be run on a high-availability Kubernetes cluster, and there would be impact due to latencies. + +### Pros + +* Single control plane interface to view/see all regions and environments. +* Kubernetes does not require Rancher to operate and can tolerate losing connectivity to the Rancher instance. + +### Cons + +* Subject to network latencies. +* If Rancher goes down, global provisioning of new services is unavailable until it is restored. However, each Kubernetes cluster can continue to be managed individually. + +## Regional Strategy +--- +In the regional deployment model a Rancher instance is deployed in close proximity to the downstream Kubernetes clusters. + +### Pros + +* Rancher functionality in regions stay operational if a Rancher instance in another region goes down. +* Network latency between Rancher and downstream clusters is greatly reduced, improving the performance of functionality in Rancher. +* Upgrades of Rancher can be done independently per region. + +### Cons + +* Overhead of managing multiple Rancher installations. +* Visibility into Kubernetes clusters in different regions requires multiple interfaces/panes of glass. +* Deploying multi-cluster apps in Rancher requires repeating the process for each Rancher server. diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md new file mode 100644 index 00000000000..1ed05696fa2 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-running-rancher.md @@ -0,0 +1,40 @@ +--- +title: Tips for Running Rancher +--- + + + + + +This guide is geared toward use cases where Rancher is used to manage downstream Kubernetes clusters. The high-availability setup is intended to prevent losing access to downstream clusters if the Rancher server is not available. + +A high-availability Kubernetes installation, defined as an installation of Rancher on a Kubernetes cluster with at least three nodes, should be used in any production installation of Rancher, as well as any installation deemed "important." Multiple Rancher instances running on multiple nodes ensure high availability that cannot be accomplished with a single node environment. + +If you are installing Rancher in a vSphere environment, refer to the best practices documented [here.](on-premises-rancher-in-vsphere.md) + +When you set up your high-availability Rancher installation, consider the following: + +### Run Rancher on a Separate Cluster +Don't run other workloads or microservices in the Kubernetes cluster that Rancher is installed on. + +### Make sure nodes are configured correctly for Kubernetes +It's important to follow K8s and etcd best practices when deploying your nodes, including disabling swap, double checking you have full network connectivity between all machines in the cluster, using unique hostnames, MAC addresses, and product_uuids for every node, checking that all correct ports are opened, and deploying with ssd backed etcd. More details can be found in the [kubernetes docs](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin) and [etcd's performance op guide](https://etcd.io/docs/v3.4/op-guide/performance/). + +### When using RKE: Back up the Statefile +RKE keeps record of the cluster state in a file called `cluster.rkestate`. This file is important for the recovery of a cluster and/or the continued maintenance of the cluster through RKE. Because this file contains certificate material, we strongly recommend encrypting this file before backing up. After each run of `rke up` you should backup the state file. + +### Run All Nodes in the Cluster in the Same Datacenter +For best performance, run all three of your nodes in the same geographic datacenter. If you are running nodes in the cloud, such as AWS, run each node in a separate Availability Zone. For example, launch node 1 in us-west-2a, node 2 in us-west-2b, and node 3 in us-west-2c. + +### Development and Production Environments Should be Similar +It's strongly recommended to have a "staging" or "pre-production" environment of the Kubernetes cluster that Rancher runs on. This environment should mirror your production environment as closely as possible in terms of software and hardware configuration. + +### Monitor Your Clusters to Plan Capacity +The Rancher server's Kubernetes cluster should run within the [system and hardware requirements](../../../pages-for-subheaders/installation-requirements.md) as closely as possible. The more you deviate from the system and hardware requirements, the more risk you take. + +However, metrics-driven capacity planning analysis should be the ultimate guidance for scaling Rancher, because the published requirements take into account a variety of workload types. + +Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with Prometheus, a leading open-source monitoring solution, and Grafana, which lets you visualize the metrics from Prometheus. + +After you [enable monitoring](../../../pages-for-subheaders/monitoring-and-alerting.md) in the cluster, you can set up alerts to let you know if your cluster is approaching its capacity. You can also use the Prometheus and Grafana monitoring framework to establish a baseline for key metrics as you scale. + diff --git a/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md new file mode 100644 index 00000000000..e8e919bde9b --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/best-practices/rancher-server/tips-for-scaling-rancher.md @@ -0,0 +1,65 @@ +--- +title: Tips for Scaling Rancher +--- + + + + + +This guide aims to introduce the approaches that should be considered to scale Rancher setups, and associated challenges with doing so. As systems grow performance will naturally reduce, but there are steps we can take to minimize the load put on Rancher, as well as optimize Rancher's ability to handle these larger setups. + +## General Tips on Optimizing Rancher's Performance +* It is advisable to keep Rancher up to date with patch releases. Performance improvements and bug fixes are made throughout the life of a minor release. You can review the release notes to help inform your own decisions on whether an upgrade is necessary but we recommend keeping yourself up to date in most cases. + +* Performance will be negatively impacted by increased latency between Rancher's infrastructure and a downstream cluster's infrastructure (eg. geographic distance). If a user or organization requires clusters/nodes all over the world or spread across many regions, it is best to use multiple Rancher installations. + +* Please always try to scale up gradually, monitoring and observing any change in behavior while doing do. It is usually easier to resolve performance problems as soon as they surface, and before other problems confuse symptoms. + +## Minimizing Load on the local cluster +The largest bottleneck when scaling Rancher is resource growth in the local Kubernetes cluster. The local cluster contains information for all downstream clusters. Many operations that apply to downstream clusters will create new objects in the local cluster and require computation from handlers running in the local cluster. + +### Managing Your Object Counts +ETCD eventually encounters limitations to the number of a single Kubernetes resource type it can store. These exact numbers are not well documented. From internal observations we usually see performance issues once a single resource type's object count exceeds 60k, and often that type is Rolebindings. + +Rolebindings are created in the local cluster as a side effect of many operations. + +Considerations when attempting reduce rolebindings in the local cluster: +* Only add users to clusters and projects when necessary +* Remove clusters and projects when they are no longer needed +* Only use custom roles if necessary +* Use as few rules as possible in custom roles +* Consider whether adding a role to a user is redundant +* Consider that using less, but more powerful, clusters may be more efficient +* Experiment to see if creating new projects or creating new clusters manifests in fewer rolebindings for your specific use case. + +### Using New Apps Over Legacy Apps +There are two app kubernetes resources that Rancher uses: apps.projects.cattle.io and apps.cattle.cattle.io. The legacy apps, apps.projects.cattle.io, were introduced first in the Cluster Manager and are now outdated. The new apps, apps.catalog.cattle.io, are found in the Cluster Explorer for their respective cluster. The new apps are preferrable because they live in the downstream cluster while the legacy apps live in the local cluster. + +We recommend removing apps that appear in the Cluster Manager, replacing them with apps in the Cluster Explorer for their target cluster if necessary and creating any future apps in the cluster's Cluster Explorer only. + +### Using the Authorized Cluster Endpoint (ACE) +There is an _Authorized Cluster Endpoint_ option for Rancher provisioned RKE1, RKE2, and K3s clusters. When enabled this adds a context to kubeconfigs generated for the cluster that uses a direct endpoint to the cluster and bypasses Rancher. However, it is not enough to only enable this option. The user of the Kubeconfig needs to use `kubectl use-context ` in order to start using it. + +Without using ACE, all kubeconfig requests first route through Rancher. + +### Experimental: Option to Reduce Event Handler Executions +The bulk of Rancher's logic occurs on event handlers. These event handlers run on an object whenever the object is updated, and when Rancher is started. Additionally, they run every 15 hours when caches are synced. In scaled setups these scheduled runs come with huge performance costs because every handler is being run on every applicable object. However, this scheduled execution of handlers can be disabled using the CATTLE_SYNC_ONLY_CHANGED_OBJECTS environment variable. If resource allocation spikes are seen on an interval of about 15 hours it is possible this setting can help. + +The value for the environment variable can be a comma separated list of the following options. The values refer to types of controllers (the structures that contain and run handlers) and their handlers. Adding the controller types to the variable will disable that set of controllers from running their handlers as part of cache resyncing. + +* `mgmt` refers to management controllers which only run on one Rancher node. +* `user` refers to user controllers which run for every cluster. Some of these are ran on the same node as management controllers, while other run in the downstream cluster. This will option targets the former. +* `scaled` refers to scaled controllers which run on every Rancher node. This is not recommended to be set due to the critical functionality the scaled handlers are responsible for. + +In short, if you notice CPU usage peaks every 15 hours, add the CATTLE_SYNC_ONLY_CHANGED_OBJECTS environment variable to your rancher deployment with the value `mgmt,user`. + +## Optimizations Outside of Rancher +A large component of performance is the local cluster and how it was configured. This cluster can introduce a bottleneck before Rancher software ever runs. When Rancher nodes experience high resource usage, you can use the command "top" to identify whether it is Rancher or a Kubernetes component that is consuming the resource in excess. + +### Keeping Kubernetes Versions Up to Date +Similar to Rancher versions, it is advisable to keep your kubernetes cluster up to date. This will ensure that your cluster contains any available performance enhancements or bug fixes. + +### Optimizing ETCD +The two main bottlenecks to [ETCD performance](https://etcd.io/docs/v3.4/op-guide/performance/) are disk speed and network speed. Optimization to either should improve performance. For information regarding ETCD performance see [Slow etcd performance (performance testing and optimization)](https://www.suse.com/support/kb/doc/?id=000020100) and [Tuning etcd for Large Installations](https://docs.ranchermanager.rancher.io/how-to-guides/advanced-user-guides/tune-etcd-for-large-installs). Information on disks can also be found [in our docs](https://docs.Ranchermanager.Rancher.io/v2.5/pages-for-subheaders/installation-requirements#disks). + +Theoretically, the more nodes in an ETCD cluster the slower it will be due to replication requirements [source](https://etcd.io/docs/v3.3/faq). This may be counter-intuitive to common scaling approaches. It can also be inferred that ETCD performance will be inversely affected by distance between nodes as that will slow down network communication. diff --git a/versioned_docs/version-latest/reference-guides/cli-with-rancher/kubectl-utility.md b/versioned_docs/version-latest/reference-guides/cli-with-rancher/kubectl-utility.md new file mode 100644 index 00000000000..fefa1a66b1b --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cli-with-rancher/kubectl-utility.md @@ -0,0 +1,36 @@ +--- +title: kubectl Utility +--- + + + + + +## kubectl + +Interact with Rancher using kubectl. + +### kubectl Utility + +Install the `kubectl` utility. See [install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/). + +Configure kubectl by visiting your cluster in the Rancher Web UI, clicking on `Kubeconfig`, copying contents, and putting them into your `~/.kube/config` file. + +Run `kubectl cluster-info` or `kubectl get pods` successfully. + +### Authentication with kubectl and kubeconfig Tokens with TTL + +_Requirements_ + +If admins have [kubeconfig token generation turned off](../about-the-api/api-tokens.md#disable-tokens-in-generated-kubeconfigs), the kubeconfig file requires the [Rancher CLI](./rancher-cli.md) to be present in your PATH when you run `kubectl`. Otherwise, you’ll see an error like: +`Unable to connect to the server: getting credentials: exec: exec: "rancher": executable file not found in $PATH`. + +This feature enables kubectl to authenticate with the Rancher server and get a new kubeconfig token when required. The following auth providers are currently supported: + +1. Local +2. Active Directory (LDAP only) +3. FreeIPA +4. OpenLDAP +5. SAML providers: Ping, Okta, ADFS, Keycloak, Shibboleth + +When you first run kubectl, for example, `kubectl get pods`, it will ask you to pick an auth provider and log in with the Rancher server. The kubeconfig token is cached in the path where you run kubectl under `./.cache/token`. This token is valid until [it expires](../about-the-api/api-tokens.md#disable-tokens-in-generated-kubeconfigs), or [gets deleted from the Rancher server](../about-the-api/api-tokens.md#deleting-tokens). Upon expiration, the next `kubectl get pods` will ask you to log in with the Rancher server again. \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/cli-with-rancher/rancher-cli.md b/versioned_docs/version-latest/reference-guides/cli-with-rancher/rancher-cli.md new file mode 100644 index 00000000000..0c9a27c1694 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cli-with-rancher/rancher-cli.md @@ -0,0 +1,91 @@ +--- +title: Rancher CLI +description: Interact with Rancher using command line interface (CLI) tools from your workstation. +--- + + + + + +The Rancher CLI (Command Line Interface) is a unified tool that you can use to interact with Rancher. With this tool, you can operate Rancher using a command line rather than the GUI. + +### Download Rancher CLI + +The binary can be downloaded directly from the UI. + +1. In the upper left corner, click **☰**. +1. At the bottom of the navigation sidebar menu, click **About**. +1. Under the **CLI Downloads section**, there are links to download the binaries for Windows, Mac, and Linux. You can also check the [releases page for our CLI](https://github.com/rancher/cli/releases) for direct downloads of the binary. + +### Requirements + +After you download the Rancher CLI, you need to make a few configurations. Rancher CLI requires: + +- Your Rancher Server URL, which is used to connect to Rancher Server. +- An API Bearer Token, which is used to authenticate with Rancher. For more information about obtaining a Bearer Token, see [Creating an API Key](../user-settings/api-keys.md). + +### CLI Authentication + +Before you can use Rancher CLI to control your Rancher Server, you must authenticate using an API Bearer Token. Log in using the following command (replace `` and `` with your information): + +```bash +$ ./rancher login https:// --token +``` + +If Rancher Server uses a self-signed certificate, Rancher CLI prompts you to continue with the connection. + +### Project Selection + +Before you can perform any commands, you must select a Rancher project to perform those commands against. To select a [project](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) to work on, use the command `./rancher context switch`. When you enter this command, a list of available projects displays. Enter a number to choose your project. + +**Example: `./rancher context switch` Output** +``` +User:rancher-cli-directory user$ ./rancher context switch +NUMBER CLUSTER NAME PROJECT ID PROJECT NAME +1 cluster-2 c-7q96s:p-h4tmb project-2 +2 cluster-2 c-7q96s:project-j6z6d Default +3 cluster-1 c-lchzv:p-xbpdt project-1 +4 cluster-1 c-lchzv:project-s2mch Default +Select a Project: +``` + +After you enter a number, the console displays a message that you've changed projects. + +``` +INFO[0005] Setting new context to project project-1 +INFO[0005] Saving config to /Users/markbishop/.ranchcli2.json +``` + +Ensure you can run `rancher kubectl get pods` successfully. + +### Commands + +The following commands are available for use in Rancher CLI. + +| Command | Result | +|---|---| +| `apps, [app]` | Performs operations on catalog applications (i.e., individual [Helm charts](https://docs.helm.sh/developing_charts/)) or Rancher charts. | +| `catalog` | Performs operations on [catalogs](../../pages-for-subheaders/helm-charts-in-rancher.md). | +| `clusters, [cluster]` | Performs operations on your [clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md). | +| `context` | Switches between Rancher [projects](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md). For an example, see [Project Selection](#project-selection). | +| `inspect [OPTIONS] [RESOURCEID RESOURCENAME]` | Displays details about [Kubernetes resources](https://kubernetes.io/docs/reference/kubectl/cheatsheet/#resource-types) or Rancher resources (i.e.: [projects](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) and [workloads](../../pages-for-subheaders/workloads-and-pods.md)). Specify resources by name or ID. | +| `kubectl` |Runs [kubectl commands](https://kubernetes.io/docs/reference/kubectl/overview/#operations). | +| `login, [l]` | Logs into a Rancher Server. For an example, see [CLI Authentication](#cli-authentication). | +| `namespaces, [namespace]` |Performs operations on namespaces. | +| `nodes, [node]` |Performs operations on nodes. | +| `projects, [project]` | Performs operations on [projects](../../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md). | +| `ps` | Displays [workloads](../../pages-for-subheaders/workloads-and-pods.md) in a project. | +| `settings, [setting]` | Shows the current settings for your Rancher Server. | +| `ssh` | Connects to one of your cluster nodes using the SSH protocol. | +| `help, [h]` | Shows a list of commands or help for one command. | + + +### Rancher CLI Help + +Once logged into Rancher Server using the CLI, enter `./rancher --help` for a list of commands. + +All commands accept the `--help` flag, which documents each command's usage. + +### Limitations + +The Rancher CLI **cannot** be used to install [dashboard apps or Rancher feature charts](../../pages-for-subheaders/helm-charts-in-rancher.md). diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md new file mode 100644 index 00000000000..4640d142697 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2.md @@ -0,0 +1,82 @@ +--- +title: EC2 Machine Configuration Reference +--- + + + + + +For more details about EC2 nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). + +### Region + +The geographical [region](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html) in which to build your cluster. + +### Zone + +The [zone](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-availability-zones), an isolated location within a region to build your cluster + +### Instance Type + +The [instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html), which determines the hardware characteristics, used to provision your cluster. + +### Root Disk Size + +Configure the size (in GB) for your [root device](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/RootDeviceStorage.html). + +### VPC/Subnet + +The [VPC](https://docs.aws.amazon.com/vpc/latest/userguide/configure-your-vpc.html) or specific [subnet](https://docs.aws.amazon.com/vpc/latest/userguide/configure-subnets.html), an IP range in your VPC, to add your resources to. + +### IAM Instance Profile Name + +The name of the [instance profile](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) used to pass an IAM role to an EC2 instance. + +## Advanced Options + +### AMI ID + +The [Amazon Machine Image](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) used for the nodes in your cluster. + +### SSH Username for AMI + +The username for connecting to your launched instances. Refer to [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connection-prereqs.html) for the default usernames to selected AMIs. For AMIs not listed, check with the AMI provider. + +### Security Group + +Choose the default security group or configure a security group. + +Please refer to [Amazon EC2 security group when using Node Driver](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-security-group) to see what rules are created in the `rancher-nodes` Security Group. + +### EBS Root Volume Type + +The [EBS volume type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-volume-types.html) to use for the root device. + +### Encrypt EBS Volume + +Enable [Amazon EBS Encryption](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html). + +### Request Spot Instance + +Enable option to [request spot instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-requests.html) and specify the maximum instance price per hour you're willing to pay. + +### Use only private address + +Enable option on use only [private addresses](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-instance-addressing.html). + +### EBS-Optimized Instance + +Use an [EBS-optimized instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ebs-optimized.html). + +### Allow access to EC2 metadata + +Enable access to [EC2 metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html). + +### Use tokens for metadata + +Use [Instance Metadata Service Version 2 (IMDSv2)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html), a token-based method to access metadata. + +### Add Tag + +Add metadata using [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html) to categorize resources. + diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md new file mode 100644 index 00000000000..ba7e806295e --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure.md @@ -0,0 +1,124 @@ +--- +title: Azure Machine Configuration +--- + + + + + +For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) + +### Environment + +Microsoft provides multiple [clouds](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest) for compliance with regional laws, which are available for your use: + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +### Location + +Configure the cluster and node [location](https://docs.microsoft.com/en-us/azure/virtual-machines/regions). + +### Resource Group + +A resource group is a container that holds related resources for an Azure solution. The resource group can include all the resources for the solution, or only those resources that you want to manage as a group. You decide how you want to allocate resources to resource groups based on what makes the most sense for your organization. Generally, add resources that share the same lifecycle to the same resource group so you can easily deploy, update, and delete them as a group. + +Use an existing resource group or enter a resource group name and one will be created for you. + +For information on managing resource groups, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) + +### Availability Set (unmanaged) + +Name or ID of an existing [availability set](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview) to add the VM to. + +### Image + +The name of the operating system image provided as an ARM resource identifier. Requires using managed disk. + +### VM Size + +Choose a size for each VM in the node pool. For details about each VM size, see [this page.](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) + +## Advanced Options + +### Fault Domain Count + +Fault domains define the group of virtual machines that share a common power source and network switch. If the availability set has already been created, the fault domain count will be ignored. + +For more information on fault domains, see [refer here](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work). + +### Update Domain Count + +Update domains indicate groups of virtual machines and underlying physical hardware that can be rebooted at the same time. If the availability set has already been created, the update domain count will be ignored. + +For more information on update domains, see [refer here](https://docs.microsoft.com/en-us/azure/virtual-machines/availability-set-overview#how-do-availability-sets-work). + +### Purchase Plan + +Some VM images in the Azure Marketplace require a plan. If applicable, select a purchase plan, formatted as `publisher:product:plan`, to use with your chosen image. + +### Subnet + +The name of the subnet when creating a new VNet or referencing an existing one. + +Default: `docker-machine` + +### Subnet Prefix + +The subnet IP address prefix to use when creating a new VNet in CIDR format. + +Default: `192.168.0.0/16` + +### Virtual Network + +The [virtual network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-overview) to use or create if one does not exist. Formatted as `[resourcegroup:]name`. + +### Public IP Options + +#### No Public IP + +Do not allocate a public IP address. + +#### Static Public IP + +Allocate a static public IP address. + +### Use Private IP + +Use a static private IP address. + +### Private IP Address + +Configure a static private IP address to use. + +### Network Security Group + +The [network security group](https://docs.microsoft.com/en-us/azure/virtual-network/network-security-groups-overview) to use. All nodes using this template will use the supplied network security group. If no network security group is provided, a new one will be created for each node. + +### DNS Label + +A unique DNS name label for the public IP address. + +### Storage Type + +The [storage account](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview) type to use with your VMs. Options include Standard LRS, Standard ZRS, Standard GRS, Standard RAGRS, and Premium LRS. + +### Use Managed Disks + +[Azure managed disks](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview) are block-level storage volumes that are managed by Azure and used with Azure Virtual Machines. Managed disks are designed for 99.999% availability. Managed disks achieve this by providing you with three replicas of your data, allowing for high durability. + +### Managed Disk Size + +The size in GB for the disk for each node. + +### SSH Username + +The username used to create an SSH connection to your nodes. + +### Open Port + +Opens inbound traffic on specified ports. When using an existing Network Security Group, Open Ports are ignored. + +Default: `2379/tcp, 2380/tcp, 6443/tcp, 9796/tcp, 10250/tcp, 10251/tcp, 10252/tcp, 10256/tcp` and `8472/udp, 4789/udp` \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md new file mode 100644 index 00000000000..634c7f48561 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean.md @@ -0,0 +1,37 @@ +--- +title: DigitalOcean Machine Configuration +--- + + + + + +For more details about DigitalOcean, Droplets, refer to the [official documentation](https://docs.digitalocean.com/products/compute/). + +### Region + +Configure the [region](https://docs.digitalocean.com/glossary/region/) where Droplets are created. + +### Size + +Configure the [size](https://docs.digitalocean.com/products/droplets/resources/choose-plan/) of Droplets. + +### OS Image + +Configure the operating system [image](https://docs.digitalocean.com/products/images/) Droplets are created from. + +### Monitoring + +Enable the DigitalOcean agent for additional [monitoring](https://docs.digitalocean.com/products/monitoring/). + +### IPv6 + +Enable IPv6 for Droplets. + +### Private Networking + +Enable private networking for Droplets. + +### Droplet Tags + +Apply a tag (label) to a Droplet. Tags may only contain letters, numbers, colons, dashes, and underscores. For example, `my_server`. diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md new file mode 100644 index 00000000000..bd79dabbed5 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2.md @@ -0,0 +1,51 @@ +--- +title: EC2 Node Template Configuration +--- + + + + + +For more details about EC2, nodes, refer to the official documentation for the [EC2 Management Console](https://aws.amazon.com/ec2). + +### Region + +In the **Region** field, select the same region that you used when creating your cloud credentials. + +### Cloud Credentials + +Your AWS account access information, stored in a [cloud credential.](../../../user-settings/manage-cloud-credentials.md) + +See [Amazon Documentation: Creating Access Keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) how to create an Access Key and Secret Key. + +See [Amazon Documentation: Creating IAM Policies (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start) how to create an IAM policy. + +See [Amazon Documentation: Adding Permissions to a User (Console)](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html#users_change_permissions-add-console) how to attach an IAM + +See our three example JSON policies: + +- [Example IAM Policy](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy) +- [Example IAM Policy with PassRole](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy-with-passrole) (needed if you want to use [Kubernetes Cloud Provider](../../../../pages-for-subheaders/set-up-cloud-providers.md) or want to pass an IAM Profile to an instance) +- [Example IAM Policy to allow encrypted EBS volumes](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy-to-allow-encrypted-ebs-volumes) policy to an user. + +### Authenticate & Configure Nodes + +Choose an availability zone and network settings for your cluster. + +### Security Group + +Choose the default security group or configure a security group. + +Please refer to [Amazon EC2 security group when using Node Driver](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#rancher-aws-ec2-security-group) to see what rules are created in the `rancher-nodes` Security Group. + +If you provide your own security group for an EC2 instance, please note that Rancher will not modify it. As such, you will be responsible for ensuring that your security group is set to allow the [necessary ports for Rancher to provision the instance](../../../../getting-started/installation-and-upgrade/installation-requirements/port-requirements.md#ports-for-rancher-server-nodes-on-rke). For more information on controlling inbound and outbound traffic to EC2 instances with security groups, refer [here](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html#WorkingWithSecurityGroups). + +### Instance Options + +Configure the instances that will be created. Make sure you configure the correct **SSH User** for the configured AMI. It is possible that a selected region does not support the default instance type. In this scenario you must select an instance type that does exist, otherwise an error will occur stating the requested configuration is not supported. + +If you need to pass an **IAM Instance Profile Name** (not ARN), for example, when you want to use a [Kubernetes Cloud Provider](../../../../pages-for-subheaders/set-up-cloud-providers.md), you will need an additional permission in your policy. See [Example IAM policy with PassRole](../../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster.md#example-iam-policy-with-passrole) for an example policy. + +### Engine Options + +In the **Engine Options** section of the node template, you can configure the container daemon. You may want to specify the container version or a container image registry mirror. diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md new file mode 100644 index 00000000000..e747dbbb547 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure.md @@ -0,0 +1,37 @@ +--- +title: Azure Node Template Configuration +--- + + + + + +For more information about Azure, refer to the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/?product=featured) + +Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. + +- **Placement** sets the geographical region where your cluster is hosted and other location metadata. +- **Network** configures the networking used in your cluster. +- **Instance** customizes your VM configuration. + + :::note + + If using a VNet in a different Resource Group than the VMs, the VNet name should be prefixed with the Resource Group name. For example, `:`. + + ::: + +If you use Docker, the [Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) configuration options include: + +- **Labels:** For information on labels, refer to the [Docker object label documentation.](https://docs.docker.com/config/labels-custom-metadata/). +- **Docker Engine Install URL:** Determines what Docker version will be installed on the instance. + + :::note + + If you're provisioning Red Hat Enterprise Linux (RHEL) or CentOS nodes, leave the **Docker Install URL** field as the default value, or select **none**. This will bypass a check for Docker installation, as Docker is already installed on these node types. + + If you set **Docker Install URL** to a value other than the default or **none**, you might see an error message such as the following: `Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories.` + + ::: + +- **Registry mirrors:** Docker Registry mirror to be used by the Docker daemon. +- **Other advanced options:** Refer to the [Docker daemon option reference](https://docs.docker.com/engine/reference/commandline/dockerd/). diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md new file mode 100644 index 00000000000..87b5fccdcfb --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean.md @@ -0,0 +1,22 @@ +--- +title: DigitalOcean Node Template Configuration +--- + + + + + +Account access information is stored as a cloud credential. Cloud credentials are stored as Kubernetes secrets. Multiple node templates can use the same cloud credential. You can use an existing cloud credential or create a new one. + +### Droplet Options + +The **Droplet Options** provision your cluster's geographical region and specifications. + +### Docker Daemon + +If you use Docker, the [Docker daemon](https://docs.docker.com/engine/docker-overview/#the-docker-daemon) configuration options include: + +- **Labels:** For information on labels, refer to the [Docker object label documentation.](https://docs.docker.com/config/labels-custom-metadata/) +- **Docker Engine Install URL:** Determines what Docker version will be installed on the instance. +- **Registry mirrors:** Docker Registry mirror to be used by the Docker daemon +- **Other advanced options:** Refer to the [Docker daemon option reference](https://docs.docker.com/engine/reference/commandline/dockerd/). diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md new file mode 100644 index 00000000000..4e44d5ceca7 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix.md @@ -0,0 +1,70 @@ +--- +title: Nutanix Node Template Configuration +--- + + + + + +## Account Access + +| Parameter | Required | Description | Default +|:-----------------------------|:--------:|:-----------------------------------------------------------------|:----- +| Management Endpoint | ✓ | Hostname/IP address of Prism Central | +| Username | ✓ | Username of the Prism Central user | +| Password | ✓ | Password of the Prism Central user | +| Allow insecure communication | | Set to true to allow insecure SSL communication to Prism Central | False + +## Scheduling + +Choose what Nutanix cluster the virtual machine will be scheduled to. + +| Parameter | Required | Description +|:----------|:--------:|:---------------------------------------------------------------------------- +| Cluster | ✓ | Name of the Nutanix cluster where the VM should be deployed (case sensitive) + +## Instance Options + +In the **Instance Options** section, configure the number of vCPUs, memory, and disk size for the VMs created by this template. + +| Parameter | Required | Description | Default +|:---------------------|:--------:|:--------------------------------------------------------------------------------------------|:------- +| CPUs | | Number of vCPUs allocated to the VM (cores) | 2 +| Memory | | Amount of RAM allocated to the VM (MB) | 2 GB +| Template Image | ✓ | Name of the Disk Image template to clone as the VM's primary disk (must support cloud-init) | +| VM Disk Size | | New size of the VM's primary disk (in GiB) | +| Additional Disk Size | | Size of an additional disk to add to the VM (in GiB) | +| Storage Container | | Storage container _UUID_ in which to provision an additional disk | +| Cloud Config YAML | | Cloud-init to provide to the VM (will be patched with Rancher root user) | +| Network | ✓ | Name(s) of the network(s) to attach to the VM | +| VM Categories | | Name(s) of any categories to be applied to the VM | + +The VM may use any modern Linux operating system that is configured with support for [cloud-init](https://cloudinit.readthedocs.io/en/latest/) using the [Config Drive v2 datasource](https://cloudinit.readthedocs.io/en/latest/reference/datasources/configdrive.html). + +## Networks + +The node template allows a VM to be provisioned with multiple networks. In the **Network** field, you can click **Add** to add any networks available to you in AOS. + +## VM Categories + +A category is a grouping of entities into a key value pair. Typically, VMs are assigned to a category based on some criteria. Policies can then be tied to those entities that are assigned (grouped by) a specific category value. + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) allows you to initialize your nodes by applying configuration on the first boot. This may involve things such as creating users or authorizing SSH keys. + +To make use of cloud-init initialization, paste a cloud config using valid YAML syntax into the **Cloud Config YAML** field. Refer to the [cloud-init documentation](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) for a commented set of examples of supported cloud config directives. + +Note that cloud-init based network configuration is not recommended and only supported via user data `runcmd` rather than by NoCloud or other network configuration datasources. + +Nutanix IP Address Management (IPAM) or another DHCP service is recommended. + +## Engine Options + +In the **Engine Options** section of the node template, you can configure the container daemon. You may want to specify the container version or a container image registry mirror. + +:::note +If you're provisioning Red Hat Enterprise Linux (RHEL) or CentOS nodes, leave the **Docker Install URL** field as the default value, or select **none**. This will bypass a check for Docker installation, as Docker is already installed on these node types. + +If you set **Docker Install URL** to a value other than the default or **none**, you might see an error message such as the following: `Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories.` +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md new file mode 100644 index 00000000000..ace79a39ca7 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere.md @@ -0,0 +1,99 @@ +--- +title: VSphere Node Template Configuration +--- + + + + + +## Account Access + +| Parameter | Required | Description | +|:----------------------|:--------:|:-----| +| Cloud Credentials | * | Your vSphere account access information, stored in a [cloud credential.](../../../user-settings/manage-cloud-credentials.md) | + +Your cloud credential has these fields: + +| Credential Field | Description | +|-----------------|--------------| +| vCenter or ESXi Server | Enter the vCenter or ESXi hostname/IP. ESXi is the virtualization platform where you create and run virtual machines and virtual appliances. vCenter Server is the service through which you manage multiple hosts connected in a network and pool host resources. | +| Port | Configure the port of the vCenter or ESXi server. | +| Username and password | Enter your vSphere login username and password. | + +## Scheduling + +Choose what hypervisor the virtual machine will be scheduled to. + +The fields in the **Scheduling** section should auto-populate with the data center and other scheduling options that are available to you in vSphere. + +| Field | Required | Explanation | +|---------|---------------|-----------| +| Data Center | * | Choose the name/path of the data center where the VM will be scheduled. | +| Resource Pool | | Name of the resource pool to schedule the VMs in. Resource pools can be used to partition available CPU and memory resources of a standalone host or cluster, and they can also be nested. Leave blank for standalone ESXi. If not specified, the default resource pool is used. | +| Data Store | * | If you have a data store cluster, you can toggle the **Data Store** field. This lets you select a data store cluster where your VM will be scheduled to. If the field is not toggled, you can select an individual disk. | +| Folder | | Name of a folder in the datacenter to create the VMs in. Must already exist. The VM folders in this dropdown menu directly correspond to your VM folders in vSphere. The folder name should be prefaced with `vm/` in your vSphere config file. | +| Host | | The IP of the host system to schedule VMs in. Leave this field blank for a standalone ESXi or for a cluster with DRS (Distributed Resource Scheduler). If specified, the host system's pool will be used and the **Resource Pool** parameter will be ignored. | + +## Instance Options + +In the **Instance Options** section, configure the number of vCPUs, memory, and disk size for the VMs created by this template. + +| Parameter | Required | Description | +|:----------------|:--------:|:-----------| +| CPUs | * | Number of vCPUS to assign to VMs. | +| Memory | * | Amount of memory to assign to VMs. | +| Disk | * | Size of the disk (in MB) to attach to the VMs. | +| Creation method | * | The method for setting up an operating system on the node. The operating system can be installed from an ISO or from a VM template. Depending on the creation method, you will also have to specify a VM template, content library, existing VM, or ISO. For more information on creation methods, refer to [About VM Creation Methods.](#about-vm-creation-methods) | +| Cloud Init | | URL of a `cloud-config.yml` file or URL to provision VMs with. This file allows further customization of the operating system, such as network configuration, DNS servers, or system daemons. The operating system must support `cloud-init`. | +| Networks | | Name(s) of the network to attach the VM to. | +| Configuration Parameters used for guestinfo | | Additional configuration parameters for the VMs. These correspond to the [Advanced Settings](https://kb.vmware.com/s/article/1016098) in the vSphere console. Example use cases include providing RancherOS [guestinfo](https://rancher.com/docs/os/v1.x/en/installation/cloud/vmware-esxi/#vmware-guestinfo) parameters or enabling disk UUIDs for the VMs (`disk.EnableUUID=TRUE`). | + + +### About VM Creation Methods + +In the **Creation method** field, configure the method used to provision VMs in vSphere. Available options include creating VMs that boot from a RancherOS ISO or creating VMs by cloning from an existing virtual machine or [VM template](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-F7BF0E6B-7C4F-4E46-8BBF-76229AEA7220.html). + +The existing VM or template may use any modern Linux operating system that is configured with support for [cloud-init](https://cloudinit.readthedocs.io/en/latest/) using the [NoCloud datasource](https://canonical-cloud-init.readthedocs-hosted.com/en/latest/reference/datasources/nocloud.html). + +Choose the way that the VM will be created: + +- **Deploy from template: Data Center:** Choose a VM template that exists in the data center that you selected. +- **Deploy from template: Content Library:** First, select the [Content Library](https://docs.vmware.com/en/VMware-vSphere/6.5/com.vmware.vsphere.vm_admin.doc/GUID-254B2CE8-20A8-43F0-90E8-3F6776C2C896.html) that contains your template, then select the template from the populated list **Library templates**. +- **Clone an existing virtual machine:** In the **Virtual machine** field, choose an existing VM that the new VM will be cloned from. +- **Install from boot2docker ISO:** Ensure that the **OS ISO URL** field contains the URL of a VMware ISO release for RancherOS (`rancheros-vmware.iso`). Note that this URL must be accessible from the nodes running your Rancher server installation. + +## Networks + +The node template now allows a VM to be provisioned with multiple networks. In the **Networks** field, you can now click **Add Network** to add any networks available to you in vSphere. + +## Node Tags and Custom Attributes + +Tags allow you to attach metadata to objects in the vSphere inventory to make it easier to sort and search for these objects. + +For tags, all your vSphere tags will show up as options to select from in your node template. + +In the custom attributes, Rancher will let you select all the custom attributes you have already set up in vSphere. The custom attributes are keys and you can enter values for each one. + +:::note + +Custom attributes are a legacy feature that will eventually be removed from vSphere. + +::: + +## cloud-init + +[Cloud-init](https://cloudinit.readthedocs.io/en/latest/) allows you to initialize your nodes by applying configuration on the first boot. This may involve things such as creating users, authorizing SSH keys or setting up the network. + +To make use of cloud-init initialization, create a cloud config file using valid YAML syntax and paste the file content in the the **Cloud Init** field. Refer to the [cloud-init documentation.](https://cloudinit.readthedocs.io/en/latest/topics/examples.html) for a commented set of examples of supported cloud config directives. + +Note that cloud-init is not supported when using the ISO creation method. + +## Engine Options + +In the **Engine Options** section of the node template, you can configure the container daemon. You may want to specify the container version or a container image registry mirror. + +:::note +If you're provisioning Red Hat Enterprise Linux (RHEL) or CentOS nodes, leave the **Docker Install URL** field as the default value, or select **none**. This will bypass a check for Docker installation, as Docker is already installed on these node types. + +If you set **Docker Install URL** to a value other than the default or **none**, you might see an error message such as the following: `Error creating machine: RHEL ssh command error: command: sudo -E yum install -y curl err: exit status 1 output: Updating Subscription Management repositories.` +::: diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md new file mode 100644 index 00000000000..67cd280e470 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration.md @@ -0,0 +1,228 @@ +--- +title: AKS Cluster Configuration Reference +--- + + + + + +## Changes in Rancher v2.6 + +- Support for adding more than one node pool +- Support for private clusters +- Enabled autoscaling node pools +- The AKS permissions are now configured in cloud credentials + +## Role-based Access Control + +When provisioning an AKS cluster in the Rancher UI, RBAC cannot be disabled. If role-based access control is disabled for the cluster in AKS, the cluster cannot be registered or imported into Rancher. + +Rancher can configure member roles for AKS clusters in the same way as any other cluster. For more information, see the section on [role-based access control.](../../../pages-for-subheaders/manage-role-based-access-control-rbac.md) + +## Cloud Credentials + +:::note + +The configuration information in this section assumes you have already set up a service principal for Rancher. For step-by-step instructions for how to set up the service principal, see [this section.](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks.md#prerequisites-in-microsoft-azure) + +::: + +### Subscription ID + +To get the subscription ID, click **All Services** in the left navigation bar. Then click **Subscriptions**. Go to the name of the subscription that you want to associate with your Kubernetes cluster and copy the **Subscription ID**. + +### Client ID + +To get the client ID, go to the Azure Portal, then click **Azure Active Directory**, then click **App registrations,** then click the name of the service principal. The client ID is listed on the app registration detail page as **Application (client) ID**. + +### Client Secret + +You can't retrieve the client secret value after it is created, so if you don't already have a client secret value, you will need to create a new client secret. + +To get a new client secret, go to the Azure Portal, then click **Azure Active Directory**, then click **App registrations,** then click the name of the service principal. + +Then click **Certificates & secrets** and click **New client secret**. Click **Add**. Then copy the **Value** of the new client secret. + +### Environment + +Microsoft provides multiple [clouds](https://docs.microsoft.com/en-us/cli/azure/cloud?view=azure-cli-latest) for compliance with regional laws, which are available for your use: + +- AzurePublicCloud +- AzureGermanCloud +- AzureChinaCloud +- AzureUSGovernmentCloud + +## Account Access + +In this section you will need to select an existing Azure cloud credential or create a new one. + +For help configuring your Azure cloud credential, see [this section.](#cloud-credentials) + +## Cluster Location + +Configure the cluster and node location. For more information on availability zones for AKS, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/availability-zones) + +The high availability locations include multiple availability zones. + +## Cluster Options + +### Kubernetes Version + +The available Kubernetes versions are dynamically fetched from the Azure API. + +### Cluster Resource Group + +A resource group is a container that holds related resources for an Azure solution. The resource group can include all the resources for the solution, or only those resources that you want to manage as a group. You decide how you want to allocate resources to resource groups based on what makes the most sense for your organization. Generally, add resources that share the same lifecycle to the same resource group so you can easily deploy, update, and delete them as a group. + +Use an existing resource group or enter a resource group name and one will be created for you. + +Using a resource group containing an existing AKS cluster will create a new resource group. Azure AKS only allows one AKS cluster per resource group. + +For information on managing resource groups, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/manage-resource-groups-portal) + +### Linux Admin Username + +The username used to create an SSH connection to the Linux nodes. + +The default username for AKS nodes is `azureuser`. + +### SSH Public Key + +The key used to create an SSH connection to the Linux nodes. + +### Tags + +Cluster tags can be useful if your organization uses tags as a way to organize resources across multiple Azure services. These tags don't apply to resources within the cluster. + +## Networking Options + +### LoadBalancer SKU + +Azure load balancers support both standard and basic SKUs (stock keeping units). + +For a comparison of standard and basic load balancers, see the official [Azure documentation.](https://docs.microsoft.com/en-us/azure/load-balancer/skus#skus) Microsoft recommends the Standard load balancer. + +The Standard load balancer is required if you have selected one or more availability zones, or if you have more than one node pool. + +### Network Policy + +All pods in an AKS cluster can send and receive traffic without limitations, by default. To improve security, you can define rules that control the flow of traffic. The Network Policy feature in Kubernetes lets you define rules for ingress and egress traffic between pods in a cluster. + +Azure provides two ways to implement network policy. You choose a network policy option when you create an AKS cluster. The policy option can't be changed after the cluster is created: + +- Azure's own implementation, called Azure Network Policies. The Azure network policy requires the Azure CNI. +- Calico Network Policies, an open-source network and network security solution founded by [Tigera](https://www.tigera.io/). + +You can also choose to have no network policy. + +For more information about the differences between Azure and Calico network policies and their capabilities, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/use-network-policies#differences-between-azure-and-calico-policies-and-their-capabilities) + +### DNS Prefix +Enter a unique DNS prefix for your cluster's Kubernetes API server FQDN. + +### Network Plugin +There are two network plugins: kubenet and Azure CNI. + +The [kubenet](https://kubernetes.io/docs/concepts/cluster-administration/network-plugins/#kubenet) Kubernetes plugin is the default configuration for AKS cluster creation. When kubenet is used, each node in the cluster receives a routable IP address. The pods use NAT to communicate with other resources outside the AKS cluster. This approach reduces the number of IP addresses you need to reserve in your network space for pods to use. + +With the Azure CNI (advanced) networking plugin, pods get full virtual network connectivity and can be directly reached via their private IP address from connected networks. This plugin requires more IP address space. + +For more information on the differences between kubenet and Azure CNI, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/concepts-network#compare-network-models) + +### HTTP Application Routing + +When enabled, the HTTP application routing add-on makes it easier to access applications deployed to the AKS cluster. It deploys two components: a [Kubernetes Ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress/) and an [External-DNS](https://github.com/kubernetes-incubator/external-dns) controller. + +For more information, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/http-application-routing) + +### Set Authorized IP Ranges + +You can secure access to the Kubernetes API server using [authorized IP address ranges.](https://docs.microsoft.com/en-us/azure/aks/api-server-authorized-ip-ranges#overview-of-api-server-authorized-ip-ranges) + +The Kubernetes API server exposes the Kubernetes API. This component provides the interaction for management tools, such as kubectl. AKS provides a single-tenant cluster control plane with a dedicated API server. By default, the API server is assigned a public IP address, and you should control access to it using Kubernetes-based or Azure-based RBAC. + +To secure access to the otherwise publicly accessible AKS control plane and API server, you can enable and use authorized IP ranges. These authorized IP ranges only allow defined IP address ranges to communicate with the API server. + +However, even if you use authorized IP address ranges, you should still use Kubernetes RBAC or Azure RBAC to authorize users and the actions they request. + +### Container Monitoring + +Container monitoring gives you performance visibility by collecting memory and processor metrics from controllers, nodes, and containers that are available in Kubernetes through the Metrics API. Container logs are also collected. After you enable monitoring, metrics and logs are automatically collected for you through a containerized version of the Log Analytics agent for Linux. Metrics are written to the metrics store and log data is written to the logs store associated with your [Log Analytics](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview) workspace. + +### Log Analytics Workspace Resource Group + +The [resource group](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups) containing the Log Analytics Workspace. You must create at least one workspace to use Azure Monitor Logs. + +### Log Analytics Workspace Name + +Data collected by Azure Monitor Logs is stored in one or more [Log Analytics workspaces.](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/design-logs-deployment) The workspace defines the geographic location of the data, access rights defining which users can access data, and configuration settings such as the pricing tier and data retention. + +You must create at least one workspace to use Azure Monitor Logs. A single workspace may be suffxicient for all of your monitoring data, or may choose to create multiple workspaces depending on your requirements. For example, you might have one workspace for your production data and another for testing. + +For more information about Azure Monitor Logs, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/azure-monitor/logs/data-platform-logs) + +### Support Private Kubernetes Service + +Typically, AKS worker nodes do not get public IPs, regardless of whether the cluster is private. In a private cluster, the control plane does not have a public endpoint. + +Rancher can connect to a private AKS cluster in one of two ways. + +The first way to ensure that Rancher is running on the same [NAT](https://docs.microsoft.com/en-us/azure/virtual-network/nat-overview) as the AKS nodes. + +The second way is to run a command to register the cluster with Rancher. Once the cluster is provisioned, you can run the displayed command anywhere you can connect to the cluster’s Kubernetes API. This command is displayed in a pop-up when you provision an AKS cluster with a private API endpoint enabled. + +:::note + +Please be aware that when registering an existing AKS cluster, the cluster might take some time, possibly hours, to appear in the `Cluster To register` dropdown list. This outcome will be based on region. + +::: + +For more information about connecting to an AKS private cluster, see the [AKS documentation.](https://docs.microsoft.com/en-us/azure/aks/private-clusters#options-for-connecting-to-the-private-cluster) + +## Node Pools + +### Mode + +The Azure interface allows users to specify whether a Primary Node Pool relies on either `system` (normally used for control planes) or `user` (what is most typically needed for Rancher). + +For Primary Node Pools, you can specify Mode, OS, Count and Size. + +System node pools always require running nodes, so they cannot be scaled below one node. At least one system node pool is required. + +For subsequent node pools, the Rancher UI forces the default of User. User node pools allow you to scale to zero nodes. User node pools don't run any part of the Kubernetes controlplane. + +AKS doesn't expose the nodes that run the Kubernetes controlplane components. + +### Availability Zones + +[Availability zones](https://docs.microsoft.com/en-us/azure/availability-zones/az-overview) are unique physical locations within a region. Each zone is made up of one or more data centers equipped with independent power, cooling, and networking. + +Not all regions have support for availability zones. For a list of Azure regions with availability zones, see the [Azure documentation.](https://docs.microsoft.com/en-us/azure/availability-zones/az-region#azure-regions-with-availability-zones) + +### VM Size + +Choose a size for each VM in the node pool. For details about each VM size, see [this page.](https://azure.microsoft.com/en-us/pricing/details/virtual-machines/linux/) + +### OS Disk Type + +The nodes in the node pool can have either managed or ephemeral disks. + +[Ephemeral OS disks](https://docs.microsoft.com/en-us/azure/virtual-machines/ephemeral-os-disks) are created on the local virtual machine storage and not saved to the remote Azure Storage. Ephemeral OS disks work well for stateless workloads, where applications are tolerant of individual VM failures, but are more affected by VM deployment time or reimaging the individual VM instances. With Ephemeral OS disk, you get lower read/write latency to the OS disk and faster VM reimage. + +[Azure managed disks](https://docs.microsoft.com/en-us/azure/virtual-machines/managed-disks-overview) are block-level storage volumes that are managed by Azure and used with Azure Virtual Machines. Managed disks are designed for 99.999% availability. Managed disks achieve this by providing you with three replicas of your data, allowing for high durability. + +### OS Disk Size + +The size in GB for the disk for each node. + +### Node Count +The number of nodes in the node pool. The maximum number of nodes may be limited by your [Azure subscription.](https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits) + +### Max Pods Per Node +The maximum number of pods per node defaults to 110 with a maximum of 250. + +### Enable Auto Scaling + +When auto scaling is enabled, you will need to enter a minimum and maximum node count. + +When Auto Scaling is enabled, you can't manually scale the node pool. The scale is controlled by the AKS autoscaler. diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md new file mode 100644 index 00000000000..4a9bbe4f0ef --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration.md @@ -0,0 +1,154 @@ +--- +title: EKS Cluster Configuration Reference +--- + + + + + +### Account Access + +Complete each drop-down and field using the information obtained for your IAM policy. + +| Setting | Description | +| ---------- | -------------------------------------------------------------------------------------------------------------------- | +| Region | From the drop-down choose the geographical region in which to build your cluster. | +| Cloud Credentials | Select the cloud credentials that you created for your IAM policy. For more information on creating cloud credentials in Rancher, refer to [this page.](../../user-settings/manage-cloud-credentials.md) | + +### Service Role + +Choose a [service role](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html). + +Service Role | Description +-------------|--------------------------- +Standard: Rancher generated service role | If you choose this role, Rancher automatically adds a service role for use with the cluster. +Custom: Choose from your existing service roles | If you choose this role, Rancher lets you choose from service roles that you're already created within AWS. For more information on creating a custom service role in AWS, see the [Amazon documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html#create-service-linked-role). + +### Secrets Encryption + +Optional: To encrypt secrets, select or enter a key created in [AWS Key Management Service (KMS)](https://docs.aws.amazon.com/kms/latest/developerguide/overview.html) + +### API Server Endpoint Access + +Configuring Public/Private API access is an advanced use case. For details, refer to the EKS cluster endpoint access control [documentation.](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) + +### Private-only API Endpoints + +If you enable private and disable public API endpoint access when creating a cluster, then there is an extra step you must take in order for Rancher to connect to the cluster successfully. In this case, a pop-up will be displayed with a command that you will run on the cluster to register it with Rancher. Once the cluster is provisioned, you can run the displayed command anywhere you can connect to the cluster's Kubernetes API. + +There are two ways to avoid this extra manual step: +- You can create the cluster with both private and public API endpoint access on cluster creation. You can disable public access after the cluster is created and in an active state and Rancher will continue to communicate with the EKS cluster. +- You can ensure that Rancher shares a subnet with the EKS cluster. Then security groups can be used to enable Rancher to communicate with the cluster's API endpoint. In this case, the command to register the cluster is not needed, and Rancher will be able to communicate with your cluster. For more information on configuring security groups, refer to the [security groups documentation](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html). + +### Public Access Endpoints + +Optionally limit access to the public endpoint via explicit CIDR blocks. + +If you limit access to specific CIDR blocks, then it is recommended that you also enable the private access to avoid losing network communication to the cluster. + +One of the following is required to enable private access: +- Rancher's IP must be part of an allowed CIDR block +- Private access should be enabled, and Rancher must share a subnet with the cluster and have network access to the cluster, which can be configured with a security group + +For more information about public and private access to the cluster endpoint, refer to the [Amazon EKS documentation.](https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html) + +### Subnet + +| Option | Description | +| ------- | ------------ | +| Standard: Rancher generated VPC and Subnet | While provisioning your cluster, Rancher generates a new VPC with 3 public subnets. | +| Custom: Choose from your existing VPC and Subnets | While provisioning your cluster, Rancher configures your Control Plane and nodes to use a VPC and Subnet that you've already [created in AWS](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html). | + + For more information, refer to the AWS documentation for [Cluster VPC Considerations](https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html). Follow one of the sets of instructions below based on your selection from the previous step. + +- [What Is Amazon VPC?](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) +- [VPCs and Subnets](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) + +### Security Group + +Amazon Documentation: + +- [Cluster Security Group Considerations](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) +- [Security Groups for Your VPC](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_SecurityGroups.html) +- [Create a Security Group](https://docs.aws.amazon.com/vpc/latest/userguide/getting-started-ipv4.html#getting-started-create-security-group) + +### Logging + +Configure control plane logs to send to Amazon CloudWatch. You are charged the standard CloudWatch Logs data ingestion and storage costs for any logs sent to CloudWatch Logs from your clusters. + +Each log type corresponds to a component of the Kubernetes control plane. To learn more about these components, see [Kubernetes Components](https://kubernetes.io/docs/concepts/overview/components/) in the Kubernetes documentation. + +For more information on EKS control plane logging, refer to the official [documentation.](https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) + +### Managed Node Groups + +Amazon EKS managed node groups automate the provisioning and lifecycle management of nodes (Amazon EC2 instances) for Amazon EKS Kubernetes clusters. + +For more information about how node groups work and how they are configured, refer to the [EKS documentation.](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) + +#### User-provided Launch Templates + +You can provide your own launch template ID and version to configure the EC2 instances in a node group. If you provide the launch template, none of the template settings will be configurable from Rancher. You must set all of the required options listed below in your launch template. + +Also, if you provide the launch template, you can only update the template version, not the template ID. To use a new template ID, create a new managed node group. + +| Option | Description | Required/Optional | +| ------ | ----------- | ----------------- | +| Instance Type | Choose the [hardware specs](https://aws.amazon.com/ec2/instance-types/) for the instance you're provisioning. | Required | +| Image ID | Specify a custom AMI for the nodes. Custom AMIs used with EKS must be [configured properly](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/) | Optional | +| Node Volume Size | The launch template must specify an EBS volume with the desired size | Required | +| SSH Key | A key to be added to the instances to provide SSH access to the nodes | Optional | +| User Data | Cloud init script in [MIME multi-part format](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-user-data) | Optional | +| Instance Resource Tags | Tag each EC2 instance and its volumes in the node group | Optional | + +#### Rancher-managed Launch Templates + +If you do not specify a launch template, then you will be able to configure the above options in the Rancher UI and all of them can be updated after creation. In order to take advantage of all of these options, Rancher will create and manage a launch template for you. Each cluster in Rancher will have one Rancher-managed launch template and each managed node group that does not have a specified launch template will have one version of the managed launch template. The name of this launch template will have the prefix "rancher-managed-lt-" followed by the display name of the cluster. In addition, the Rancher-managed launch template will be tagged with the key "rancher-managed-template" and value "do-not-modify-or-delete" to help identify it as Rancher-managed. It is important that this launch template and its versions not be modified, deleted, or used with any other clusters or managed node groups. Doing so could result in your node groups being "degraded" and needing to be destroyed and recreated. + +#### Custom AMIs + +If you specify a custom AMI, whether in a launch template or in Rancher, then the image must be [configured properly](https://aws.amazon.com/premiumsupport/knowledge-center/eks-custom-linux-ami/) and you must provide user data to [bootstrap the node](https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami). This is considered an advanced use case and understanding the requirements is imperative. + +If you specify a launch template that does not contain a custom AMI, then Amazon will use the [EKS-optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) for the Kubernetes version and selected region. You can also select a [GPU enabled instance](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html#gpu-ami) for workloads that would benefit from it. + +:::note + +The GPU enabled instance setting in Rancher is ignored if a custom AMI is provided, either in the dropdown or in a launch template. + +::: + +#### Spot instances + +Spot instances are now [supported by EKS](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html#managed-node-group-capacity-types-spot). If a launch template is specified, Amazon recommends that the template not provide an instance type. Instead, Amazon recommends providing multiple instance types. If the "Request Spot Instances" checkbox is enabled for a node group, then you will have the opportunity to provide multiple instance types. + +:::note + +Any selection you made in the instance type dropdown will be ignored in this situation and you must specify at least one instance type to the "Spot Instance Types" section. Furthermore, a launch template used with EKS cannot request spot instances. Requesting spot instances must be part of the EKS configuration. + +::: + +#### Node Group Settings + +The following settings are also configurable. All of these except for the "Node Group Name" are editable after the node group is created. + +| Option | Description | +| ------- | ------------ | +| Node Group Name | The name of the node group. | +| Desired ASG Size | The desired number of instances. | +| Maximum ASG Size | The maximum number of instances. This setting won't take effect until the [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) is installed. | +| Minimum ASG Size | The minimum number of instances. This setting won't take effect until the [Cluster Autoscaler](https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html) is installed. | +| Labels | Kubernetes labels applied to the nodes in the managed node group. | +| Tags | These are tags for the managed node group and do not propagate to any of the associated resources. | + + +### Configuring the Refresh Interval + +The `eks-refresh-cron` setting is deprecated. It has been migrated to the `eks-refresh` setting, which is an integer representing seconds. + +The default value is 300 seconds. + +The syncing interval can be changed by running `kubectl edit setting eks-refresh`. + +If the `eks-refresh-cron` setting was previously set, the migration will happen automatically. + +The shorter the refresh window, the less likely any race conditions will occur, but it does increase the likelihood of encountering request limits that may be in place for AWS APIs. diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md new file mode 100644 index 00000000000..120be47a641 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters.md @@ -0,0 +1,54 @@ +--- +title: Private Clusters +--- + + + + + +In GKE, [private clusters](https://cloud.google.com/kubernetes-engine/docs/concepts/private-cluster-concept) are clusters whose nodes are isolated from inbound and outbound traffic by assigning them internal IP addresses only. Private clusters in GKE have the option of exposing the control plane endpoint as a publicly accessible address or as a private address. This is different from other Kubernetes providers, which may refer to clusters with private control plane endpoints as "private clusters" but still allow traffic to and from nodes. You may want to create a cluster with private nodes, with or without a public control plane endpoint, depending on your organization's networking and security requirements. A GKE cluster provisioned from Rancher can use isolated nodes by selecting "Private Cluster" in the Cluster Options (under "Show advanced options"). The control plane endpoint can optionally be made private by selecting "Enable Private Endpoint". + +### Private Nodes + +Because the nodes in a private cluster only have internal IP addresses, they will not be able to install the cluster agent and Rancher will not be able to fully manage the cluster. This can be overcome in a few ways. + +#### Cloud NAT + +:::caution + +Cloud NAT will [incur charges](https://cloud.google.com/nat/pricing). + +::: + +If restricting outgoing internet access is not a concern for your organization, use Google's [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) service to allow nodes in the private network to access the internet, enabling them to download the required images from Dockerhub and contact the Rancher management server. This is the simplest solution. + +#### Private registry + +:::caution + +This scenario is not officially supported, but is described for cases in which using the Cloud NAT service is not sufficient. + +::: + +If restricting both incoming and outgoing traffic to nodes is a requirement, follow the air-gapped installation instructions to set up a private container image [registry](../../../../pages-for-subheaders/air-gapped-helm-cli-install.md) on the VPC where the cluster is going to be, allowing the cluster nodes to access and download the images they need to run the cluster agent. If the control plane endpoint is also private, Rancher will need [direct access](#direct-access) to it. + +### Private Control Plane Endpoint + +If the cluster has a public endpoint exposed, Rancher will be able to reach the cluster, and no additional steps need to be taken. However, if the cluster has no public endpoint, then considerations must be made to ensure Rancher can access the cluster. + +#### Cloud NAT + +:::caution + +Cloud NAT will [incur charges](https://cloud.google.com/nat/pricing). + +::: + +As above, if restricting outgoing internet access to the nodes is not a concern, then Google's [Cloud NAT](https://cloud.google.com/nat/docs/using-nat) service can be used to allow the nodes to access the internet. While the cluster is provisioning, Rancher will provide a registration command to run on the cluster. Download the [kubeconfig](https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl) for the new cluster and run the provided kubectl command on the cluster. Gaining access +to the cluster in order to run this command can be done by creating a temporary node or using an existing node in the VPC, or by logging on to or creating an SSH tunnel through one of the cluster nodes. + +#### Direct access + +If the Rancher server is run on the same VPC as the cluster's control plane, it will have direct access to the control plane's private endpoint. The cluster nodes will need to have access to a [private registry](#private-registry) to download images as described above. + +You can also use services from Google such as [Cloud VPN](https://cloud.google.com/network-connectivity/docs/vpn/concepts/overview) or [Cloud Interconnect VLAN](https://cloud.google.com/network-connectivity/docs/interconnect) to facilitate connectivity between your organization's network and your Google VPC. diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md new file mode 100644 index 00000000000..ca377342025 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration.md @@ -0,0 +1,434 @@ +--- +title: K3s Cluster Configuration Reference +--- + + + + + +This section covers the configuration options that are available in Rancher for a new or existing K3s Kubernetes cluster. + +## Overview + +You can configure the Kubernetes options one of two ways: + +- [Rancher UI](#configuration-options-in-the-rancher-ui): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. +- [Cluster Config File](#cluster-config-file): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create a K3s config file. Using a config file allows you to set any of the [options](https://rancher.com/docs/k3s/latest/en/installation/install-options/) available in an K3s installation. + +## Editing Clusters in the Rancher UI + +The Rancher UI provides two ways to edit a cluster: +1. With a form. +1. With YAML. + +### Editing Clusters with a Form + +The form covers the most frequently needed options for clusters. + +To edit your cluster, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit Config**. + +### Editing Clusters in YAML + +For a complete reference of configurable options for K3s clusters in YAML, see the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +To edit your cluster with YAML: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit as YAML**. +1. Edit the RKE options under the `rkeConfig` directive. + +## Configuration Options in the Rancher UI + +### Machine Pool Configuration + +This subsection covers generic machine pool configurations. For specific infrastructure provider configurations, refer to the following: + +- [Azure](../downstream-cluster-configuration/machine-configuration/azure.md) +- [DigitalOcean](../downstream-cluster-configuration/machine-configuration/digitalocean.md) +- [EC2](../downstream-cluster-configuration/machine-configuration/amazon-ec2.md) + +##### Pool Name + +The name of the machine pool. + +##### Machine Count + +The number of machines in the pool. + +##### Roles + +Option to assign etcd, control plane, and worker roles to nodes. + +#### Advanced + +##### Auto Replace + +The amount of time nodes can be unreachable before they are automatically deleted and replaced. + +##### Drain Before Delete + +Enables draining nodes by evicting all pods before the node is deleted. + +##### Kubernetes Node Labels + +Add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to nodes to help with organization and object selection. + +For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +##### Taints + +Add [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) to nodes, to prevent pods from being scheduled to or executed on the nodes, unless the pods have matching tolerations. + +### Cluster Configuration +#### Basics +##### Kubernetes Version + +The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). + +For more detail, see [Upgrading Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md). + +##### Pod Security Admission Configuration Template + +The default [pod security admission configuration template](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for the cluster. + +##### Encrypt Secrets + +Option to enable or disable secrets encryption. When enabled, secrets will be encrypted using a AES-CBC key. If disabled, any previously secrets will not be readable until encryption is enabled again. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/advanced/#secrets-encryption-config-experimental) for details. + +##### Project Network Isolation + +If your network provider allows project network isolation, you can choose whether to enable or disable inter-project communication. + +##### SELinux + +Option to enable or disable [SELinux](https://rancher.com/docs/k3s/latest/en/advanced/#selinux-support) support. + +##### CoreDNS + +By default, [CoreDNS](https://coredns.io/) is installed as the default DNS provider. If CoreDNS is not installed, an alternate DNS provider must be installed yourself. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/networking/#coredns) for details.. + +##### Klipper Service LB + +Option to enable or disable the [Klipper](https://github.com/rancher/klipper-lb) service load balancer. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/networking/#service-load-balancer) for details. + +##### Traefik Ingress + +Option to enable or disable the [Traefik](https://traefik.io/) HTTP reverse proxy and load balancer. For more details and configuration options, see the [K3s documentation](https://rancher.com/docs/k3s/latest/en/networking/#traefik-ingress-controller). + +##### Local Storage + +Option to enable or disable [local storage](https://rancher.com/docs/k3s/latest/en/storage/) on the node(s). + +##### Metrics Server + +Option to enable or disable the [metrics server](https://github.com/kubernetes-incubator/metrics-server). If enabled, ensure port 10250 is opened for inbound TCP traffic. + +#### Add-On Config + +Additional Kubernetes manifests, managed as a [Add-on](https://kubernetes.io/docs/concepts/cluster-administration/addons/), to apply to the cluster on startup. Refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/helm/#automatically-deploying-manifests-and-helm-charts) for details. + +#### Agent Environment Vars + +Option to set environment variables for [K3s agents](https://rancher.com/docs/k3s/latest/en/architecture/). The environment variables can be set using key value pairs. Refer to the [K3 documentation](https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/) for more details. + +#### etcd + +##### Automatic Snapshots + +Option to enable or disable recurring etcd snapshots. If enabled, users have the option to configure the frequency of snapshots. For details, refer to the [K3s documentation](https://rancher.com/docs/k3s/latest/en/backup-restore/#creating-snapshots). + +##### Metrics + +Option to choose whether to expose etcd metrics to the public or only within the cluster. + +#### Networking + +##### Cluster CIDR + +IPv4/IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16). + +##### Service CIDR + +IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16). + +##### Cluster DNS + +IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10). + +##### Cluster Domain + +Select the domain for the cluster. The default is `cluster.local`. + +##### NodePort Service Port Range + +Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). The default is `30000-32767`. + +##### Truncate Hostnames + +Option to truncate hostnames to 15 characters or less. You can only set this field during the initial creation of the cluster. You can't enable or disable the 15 character limit after cluster creation. + +This setting only affects machine-provisioned clusters. Since custom clusters set hostnames during their own node creation process, which occurs outside of Rancher, this field doesn't restrict custom cluster hostname length. + +Truncating hostnames in a cluster improves compatibility with Windows-based systems. Although Kubernetes allows hostnames up to 63 characters in length, systems that use NetBIOS restrict hostnames to 15 characters or less. + +##### TLS Alternate Names + +Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert. + +##### Authorized Cluster Endpoint + +Authorized Cluster Endpoint can be used to directly access the Kubernetes API server, without requiring communication through Rancher. + +For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +#### Registries + +Select the image repository to pull Rancher images from. For more details and configuration options, see the [K3s documentation](https://rancher.com/docs/k3s/latest/en/installation/private-registry/). + +#### Upgrade Strategy + +##### Control Plane Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Worker Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Drain Nodes (Control Plane) + +Option to remove all pods from the node prior to upgrading. + +##### Drain Nodes (Worker Nodes) + +Option to remove all pods from the node prior to upgrading. + +#### Advanced + +Option to set kubelet options for different nodes. For available options, refer to the [Kubernetes documentation](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/). + +## Cluster Config File Reference + +Editing clusters in YAML allows you to set configurations that are already listed in [Configuration Options in the Rancher UI](#configuration-options-in-the-rancher-ui), as well as set Rancher-specific parameters. + +
+ + Example Cluster Config File Snippet + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +spec: + cloudCredentialSecretName: cattle-global-data:cc-fllv6 + clusterAgentDeploymentCustomization: {} + fleetAgentDeploymentCustomization: {} + kubernetesVersion: v1.26.7+k3s1 + localClusterAuthEndpoint: {} + rkeConfig: + chartValues: {} + etcd: + snapshotRetention: 5 + snapshotScheduleCron: 0 */5 * * * + machineGlobalConfig: + disable-apiserver: false + disable-cloud-controller: false + disable-controller-manager: false + disable-etcd: false + disable-kube-proxy: false + disable-network-policy: false + disable-scheduler: false + etcd-expose-metrics: false + kube-apiserver-arg: + - audit-policy-file=/etc/rancher/k3s/user-audit-policy.yaml + - audit-log-path=/etc/rancher/k3s/user-audit.logs + profile: null + secrets-encryption: false + machinePools: + - controlPlaneRole: true + etcdRole: true + machineConfigRef: + kind: Amazonec2Config + name: nc-test-pool1-pwl5h + name: pool1 + quantity: 1 + unhealthyNodeTimeout: 0s + workerRole: true + machineSelectorConfig: + - config: + docker: false + protect-kernel-defaults: false + selinux: false + machineSelectorFiles: + - fileSources: + - configMap: + name: '' + secret: + name: audit-policy + items: + - key: audit-policy + path: /etc/rancher/k3s/user-audit-policy.yaml + machineLabelSelector: + matchLabels: + rke.cattle.io/control-plane-role: 'true' + registries: {} + upgradeStrategy: + controlPlaneConcurrency: '1' + controlPlaneDrainOptions: + deleteEmptyDirData: true + disableEviction: false + enabled: false + force: false + gracePeriod: -1 + ignoreDaemonSets: true + ignoreErrors: false + postDrainHooks: null + preDrainHooks: null + skipWaitForDeleteTimeoutSeconds: 0 + timeout: 120 + workerConcurrency: '1' + workerDrainOptions: + deleteEmptyDirData: true + disableEviction: false + enabled: false + force: false + gracePeriod: -1 + ignoreDaemonSets: true + ignoreErrors: false + postDrainHooks: null + preDrainHooks: null + skipWaitForDeleteTimeoutSeconds: 0 + timeout: 120 +``` +
+ +### chartValues + +Specify the values for the system charts installed by K3s. + +Example: + +```yaml +chartValues: + chart-name: + key: value +``` +### machineGlobalConfig + +Specify K3s configurations. Any configuration change made here will apply to every node. The configuration options available in the [standalone version of k3s](https://docs.k3s.io/cli/server) can be applied here. + +Example: + +```yaml +machineGlobalConfig: + etcd-arg: + - key1=value1 + - key2=value2 +``` + +### machineSelectorConfig + +`machineSelectorConfig` is the same as [`machineGlobalConfig`](#machineglobalconfig) except that a [label](#kubernetes-node-labels) selector can be specified with the configuration. The configuration will only be applied to nodes that match the provided label selector. + +Multiple `config` entries are allowed, each specifying their own `machineLabelSelector`. A user can specify `matchExpressions`, `matchLabels`, both, or neither. Omitting the `machineLabelSelector` section of this field has the same effect as putting the config in the `machineGlobalConfig` section. + +Example: + +```yaml +machineSelectorConfig + - config: + config-key: config-value + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` +### machineSelectorFiles + +:::note + +This feature is available in Rancher v2.7.2 and later. + +::: + +Deliver files to nodes, so that the files can be in place before initiating K3s server or agent processes. +The content of the file is retrieved from either a secret or a configmap. The target nodes are filtered by the `machineLabelSelector`. + +Example : + +```yaml +machineSelectorFiles: + - fileSources: + - secret: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-secret-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 + - fileSources: + - configMap: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-configmap-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` + +The secret or configmap must meet the following requirements: + +1. It must be in the `fleet-default` namespace where the Cluster object exists. +2. It must have the annotation `rke.cattle.io/object-authorized-for-clusters: cluster-name1,cluster-name2`, which permits the target clusters to use it. + +:::tip + +Rancher Dashboard provides an easy-to-use form for creating the secret or configmap. + +::: + +Example: + +```yaml +apiVersion: v1 +data: + audit-policy: >- + IyBMb2cgYWxsIHJlcXVlc3RzIGF0IHRoZSBNZXRhZGF0YSBsZXZlbC4KYXBpVmVyc2lvbjogYXVkaXQuazhzLmlvL3YxCmtpbmQ6IFBvbGljeQpydWxlczoKLSBsZXZlbDogTWV0YWRhdGE= +kind: Secret +metadata: + annotations: + rke.cattle.io/object-authorized-for-clusters: cluster1 + name: name1 + namespace: fleet-default +``` diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md new file mode 100644 index 00000000000..b0047715921 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md @@ -0,0 +1,365 @@ +--- +title: RKE Cluster Configuration Reference +--- + + + + + +When Rancher installs Kubernetes, it uses [RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md) or [RKE2](https://docs.rke2.io/) as the Kubernetes distribution. + +This section covers the configuration options that are available in Rancher for a new or existing RKE Kubernetes cluster. + + +## Overview + +You can configure the Kubernetes options one of two ways: + +- [Rancher UI](#configuration-options-in-the-rancher-ui): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. +- [Cluster Config File](#rke-cluster-config-file-reference): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the options available in an RKE installation, except for system_images configuration, by specifying them in YAML. + +The RKE cluster config options are nested under the `rancher_kubernetes_engine_config` directive. For more information, see the section about the [cluster config file.](#rke-cluster-config-file-reference) + +In [clusters launched by RKE](../../../pages-for-subheaders/launch-kubernetes-with-rancher.md), you can edit any of the remaining options that follow. + +For an example of RKE config file syntax, see the [RKE documentation](https://rancher.com/docs/rke/latest/en/example-yamls/). + +The forms in the Rancher UI don't include all advanced options for configuring RKE. For the complete reference of configurable options for RKE Kubernetes clusters in YAML, see the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/) + +## Editing Clusters with a Form in the Rancher UI + +To edit your cluster, + +1. In the upper left corner, click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit Config**. + + +## Editing Clusters with YAML + +Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the options available in an RKE installation, except for system_images configuration, by specifying them in YAML. + +RKE clusters (also called RKE1 clusters) are edited differently than RKE2 and K3s clusters. + +To edit an RKE config file directly from the Rancher UI, + +1. Click **☰ > Cluster Management**. +1. Go to the RKE cluster you want to configure. Click and click **⋮ > Edit Config**. This take you to the RKE configuration form. Note: Because cluster provisioning changed in Rancher 2.6, the **⋮ > Edit as YAML** can be used for configuring RKE2 clusters, but it can't be used for editing RKE1 configuration. +1. In the configuration form, scroll down and click **Edit as YAML**. +1. Edit the RKE options under the `rancher_kubernetes_engine_config` directive. + +## Configuration Options in the Rancher UI + +:::tip + +Some advanced configuration options are not exposed in the Rancher UI forms, but they can be enabled by editing the RKE cluster configuration file in YAML. For the complete reference of configurable options for RKE Kubernetes clusters in YAML, see the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/) + +::: + +### Kubernetes Version + +The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). + +For more detail, see [Upgrading Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md). + +### Network Provider + +The [Network Provider](https://kubernetes.io/docs/concepts/cluster-administration/networking/) that the cluster uses. For more details on the different networking providers, please view our [Networking FAQ](../../../faq/container-network-interface-providers.md). + +:::caution + +After you launch the cluster, you cannot change your network provider. Therefore, choose which network provider you want to use carefully, as Kubernetes doesn't allow switching between network providers. Once a cluster is created with a network provider, changing network providers would require you tear down the entire cluster and all its applications. + +::: + +Out of the box, Rancher is compatible with the following network providers: + +- [Canal](https://github.com/projectcalico/canal) +- [Flannel](https://github.com/coreos/flannel#flannel) +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Weave](https://github.com/weaveworks/weave) + +:::note Notes on Weave: + +When Weave is selected as network provider, Rancher will automatically enable encryption by generating a random password. If you want to specify the password manually, please see how to configure your cluster using a [Config File](#rke-cluster-config-file-reference) and the [Weave Network Plug-in Options](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/#weave-network-plug-in-options). + +::: + +### Project Network Isolation + +If your network provider allows project network isolation, you can choose whether to enable or disable inter-project communication. + +Project network isolation is available if you are using any RKE network plugin that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin. + +### Kubernetes Cloud Providers + +You can configure a [Kubernetes cloud provider](../../../pages-for-subheaders/set-up-cloud-providers.md). If you want to use dynamically provisioned [volumes and storage](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) in Kubernetes, typically you must select the specific cloud provider in order to use it. For example, if you want to use Amazon EBS, you would need to select the `aws` cloud provider. + +:::note + +If the cloud provider you want to use is not listed as an option, you will need to use the [config file option](#rke-cluster-config-file-reference) to configure the cloud provider. Please reference the [RKE cloud provider documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to configure the cloud provider. + +::: + +### Private Registries + +The cluster-level private registry configuration is only used for provisioning clusters. + +There are two main ways to set up private registries in Rancher: by setting up the [global default registry](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry.md) through the **Settings** tab in the global view, and by setting up a private registry in the advanced options in the cluster-level settings. The global default registry is intended to be used for air-gapped setups, for registries that do not require credentials. The cluster-level private registry is intended to be used in all setups in which the private registry requires credentials. + +If your private registry requires credentials, you need to pass the credentials to Rancher by editing the cluster options for each cluster that needs to pull images from the registry. + +The private registry configuration option tells Rancher where to pull the [system images](https://rancher.com/docs/rke/latest/en/config-options/system-images/) or [addon images](https://rancher.com/docs/rke/latest/en/config-options/add-ons/) that will be used in your cluster. + +- **System images** are components needed to maintain the Kubernetes cluster. +- **Add-ons** are used to deploy several cluster components, including network plug-ins, the ingress controller, the DNS provider, or the metrics server. + +For more information on setting up a private registry for components applied during the provisioning of the cluster, see the [RKE documentation on private registries](https://rancher.com/docs/rke/latest/en/config-options/private-registries/). + +Rancher v2.6 introduced the ability to configure [ECR registries for RKE clusters](https://rancher.com/docs/rke/latest/en/config-options/private-registries/#amazon-elastic-container-registry-ecr-private-registry-setup). + +### Authorized Cluster Endpoint + +Authorized Cluster Endpoint (ACE) can be used to directly access the Kubernetes API server, without requiring communication through Rancher. + +:::note + +ACE is available on RKE, RKE2, and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. + +::: + +ACE must be set up [manually](../../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters) on RKE2 and K3s clusters. In RKE, ACE is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self-signed certificates. + +For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +### Node Pools + +For information on using the Rancher UI to set up node pools in an RKE cluster, refer to [this page.](../../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md) + +### NGINX Ingress + +If you want to publish your applications in a high-availability configuration, and you're hosting your nodes with a cloud-provider that doesn't have a native load-balancing feature, enable this option to use NGINX Ingress within the cluster. + +### Metrics Server Monitoring + +Option to enable or disable [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/). + +Each cloud provider capable of launching a cluster using RKE can collect metrics and monitor for your cluster nodes. Enable this option to view your node metrics from your cloud provider's portal. + +### Pod Security Policy Support + +Enables [pod security policies](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) for the cluster. After enabling this option, choose a policy using the **Default Pod Security Policy** drop-down. + +You must have an existing Pod Security Policy configured before you can use this option. + +### Docker Version on Nodes + +Configures whether nodes are allowed to run versions of Docker that Rancher doesn't officially support. + +If you choose to require a supported Docker version, Rancher will stop pods from running on nodes that don't have a supported Docker version installed. + +For details on which Docker versions were tested with each Rancher version, refer to the [support maintenance terms.](https://rancher.com/support-maintenance-terms/) + +### Docker Root Directory + +If the nodes you are adding to the cluster have Docker configured with a non-default Docker Root Directory (default is `/var/lib/docker`), specify the correct Docker Root Directory in this option. + +### Default Pod Security Policy + +If you enable **Pod Security Policy Support**, use this drop-down to choose the pod security policy that's applied to the cluster. + +### Node Port Range + +Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). Default is `30000-32767`. + +### Recurring etcd Snapshots + +Option to enable or disable [recurring etcd snapshots](https://rancher.com/docs/rke/latest/en/etcd-snapshots/#etcd-recurring-snapshots). + +### Agent Environment Variables + +Option to set environment variables for [rancher agents](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. If rancher agent requires use of proxy to communicate with Rancher server, `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` environment variables can be set using agent environment variables. + +### Updating ingress-nginx + +Clusters that were created before Kubernetes 1.16 will have an `ingress-nginx` `updateStrategy` of `OnDelete`. Clusters that were created with Kubernetes 1.16 or newer will have `RollingUpdate`. + +If the `updateStrategy` of `ingress-nginx` is `OnDelete`, you will need to delete these pods to get the correct version for your deployment. + +### Cluster Agent Configuration and Fleet Agent Configuration + +You can configure the scheduling fields and resource limits for the Cluster Agent and the cluster's Fleet Agent. You can use these fields to customize tolerations, affinity rules, and resource requirements. Additional tolerations are appended to a list of default tolerations and control plane node taints. If you define custom affinity rules, they override the global default affinity setting. Defining resource requirements sets requests or limits where there previously were none. + +:::note + +With this option, it's possible to override or remove rules that are required for the functioning of the cluster. We strongly recommend against removing or overriding these and any other affinity rules, as this may cause unwanted side effects: + +- `affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution` for `cattle-cluster-agent` +- `cluster-agent-default-affinity` for `cattle-cluster-agent` +- `fleet-agent-default-affinity` for `fleet-agent` + +::: + +If you downgrade Rancher to v2.7.4 or below, your changes will be lost and the agents will re-deploy without your customizations. The Fleet agent will fallback to using its built-in default values when it re-deploys. If the Fleet version doesn't change during the downgrade, the re-deploy won't be immediate. + + +## RKE Cluster Config File Reference + +Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE config file. Using a config file allows you to set any of the [options available](https://rancher.com/docs/rke/latest/en/config-options/) in an RKE installation, except for `system_images` configuration. The `system_images` option is not supported when creating a cluster with the Rancher UI or API. + +For the complete reference for configurable options for RKE Kubernetes clusters in YAML, see the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/) + +### Config File Structure in Rancher + +RKE (Rancher Kubernetes Engine) is the tool that Rancher uses to provision Kubernetes clusters. Rancher's cluster config files used to have the same structure as [RKE config files,](https://rancher.com/docs/rke/latest/en/example-yamls/) but the structure changed so that in Rancher, RKE cluster config items are separated from non-RKE config items. Therefore, configuration for your cluster needs to be nested under the `rancher_kubernetes_engine_config` directive in the cluster config file. Cluster config files created with earlier versions of Rancher will need to be updated for this format. An example cluster config file is included below. + +
+ Example Cluster Config File + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker +enable_cluster_alerting: false +enable_cluster_monitoring: false +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # Currently only nginx ingress provider is supported. +# # To disable ingress controller, set `provider: none` +# # To enable ingress on specific nodes, use the node_selector, eg: +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` +
+ +### Default DNS provider + +The table below indicates what DNS provider is deployed by default. See [RKE documentation on DNS provider](https://rancher.com/docs/rke/latest/en/config-options/add-ons/dns/) for more information how to configure a different DNS provider. CoreDNS can only be used on Kubernetes v1.12.0 and higher. + +| Rancher version | Kubernetes version | Default DNS provider | +|-------------|--------------------|----------------------| +| v2.2.5 and higher | v1.14.0 and higher | CoreDNS | +| v2.2.5 and higher | v1.13.x and lower | kube-dns | +| v2.2.4 and lower | any | kube-dns | + +## Rancher Specific Parameters in YAML + +Besides the RKE config file options, there are also Rancher specific settings that can be configured in the Config File (YAML): + +### docker_root_dir + +See [Docker Root Directory](#docker-root-directory). + +### enable_cluster_monitoring + +Option to enable or disable [Cluster Monitoring](../../../pages-for-subheaders/monitoring-and-alerting.md). + +### enable_network_policy + +Option to enable or disable Project Network Isolation. + +Project network isolation is available if you are using any RKE network plugin that supports the enforcement of Kubernetes network policies, such as Canal or the Cisco ACI plugin. + +### local_cluster_auth_endpoint + +See [Authorized Cluster Endpoint](#authorized-cluster-endpoint). + +Example: + +```yaml +local_cluster_auth_endpoint: + enabled: true + fqdn: "FQDN" + ca_certs: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +### Custom Network Plug-in + +You can add a custom network plug-in by using the [user-defined add-on functionality](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/) of RKE. You define any add-on that you want deployed after the Kubernetes cluster is deployed. + +There are two ways that you can specify an add-on: + +- [In-line Add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#in-line-add-ons) +- [Referencing YAML Files for Add-ons](https://rancher.com/docs/rke/latest/en/config-options/add-ons/user-defined-add-ons/#referencing-yaml-files-for-add-ons) + +For an example of how to configure a custom network plug-in by editing the `cluster.yml`, refer to the [RKE documentation.](https://rancher.com/docs/rke/latest/en/config-options/add-ons/network-plugins/custom-network-plugin-example) \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md new file mode 100644 index 00000000000..402d426b02c --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration.md @@ -0,0 +1,464 @@ +--- +title: RKE2 Cluster Configuration Reference +--- + + + + + +This section covers the configuration options that are available in Rancher for a new or existing RKE2 Kubernetes cluster. + +## Overview + +You can configure the Kubernetes options in one of the two following ways: + +- [Rancher UI](#configuration-options-in-the-rancher-ui): Use the Rancher UI to select options that are commonly customized when setting up a Kubernetes cluster. +- [Cluster Config File](#cluster-config-file-reference): Instead of using the Rancher UI to choose Kubernetes options for the cluster, advanced users can create an RKE2 config file. Using a config file allows you to set many additional [options](https://docs.rke2.io/install/configuration) available for an RKE2 installation. + +## Editing Clusters in the Rancher UI + +The Rancher UI provides two ways to edit a cluster: +1. With a form. +1. With YAML. + +### Editing Clusters with a Form + +The form covers the most frequently needed options for clusters. + +To edit your cluster, + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit Config**. + +### Editing Clusters in YAML + +For a complete reference of configurable options for K3s clusters in YAML, see the [K3s documentation.](https://rancher.com/docs/k3s/latest/en/installation/install-options/) + +To edit your cluster in YAML: + +1. Click **☰ > Cluster Management**. +1. Go to the cluster you want to configure and click **⋮ > Edit as YAML**. +1. Edit the RKE options under the `rkeConfig` directive. + +## Configuration Options in the Rancher UI + +### Machine Pool Configuration + +This subsection covers generic machine pool configurations. For specific infrastructure provider configurations, refer to the following: + +- [Azure](../downstream-cluster-configuration/machine-configuration/azure.md) +- [DigitalOcean](../downstream-cluster-configuration/machine-configuration/digitalocean.md) +- [EC2](../downstream-cluster-configuration/machine-configuration/amazon-ec2.md) + +##### Pool Name + +The name of the machine pool. + +##### Machine Count + +The number of machines in the pool. + +##### Roles + +Option to assign etcd, control plane, and worker roles to nodes. + +#### Advanced + +##### Auto Replace + +The amount of time nodes can be unreachable before they are automatically deleted and replaced. + +##### Drain Before Delete + +Enables draining nodes by evicting all pods before the node is deleted. + +##### Kubernetes Node Labels + +Add [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to nodes to help with organization and object selection. + +For details on label syntax requirements, see the [Kubernetes documentation.](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set) + +##### Taints + +Add [taints](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) to nodes, to prevent pods from being scheduled to or executed on the nodes, unless the pods have matching tolerations. + +### Cluster Configuration +#### Basics +##### Kubernetes Version + +The version of Kubernetes installed on your cluster nodes. Rancher packages its own version of Kubernetes based on [hyperkube](https://github.com/rancher/hyperkube). + +For more detail, see [Upgrading Kubernetes](../../../getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes.md). + +##### Container Network Provider + +The [Network Provider](https://kubernetes.io/docs/concepts/cluster-administration/networking/) that the cluster uses. + +:::caution + +After you launch the cluster, you cannot change your network provider. Therefore, choose which network provider you want to use carefully, as Kubernetes doesn't allow switching between network providers. Once a cluster is created with a network provider, changing network providers would require you to tear down the entire cluster and all its applications. + +::: + +Out of the box, Rancher is compatible with the following network providers: + +- [Canal](https://github.com/projectcalico/canal) +- [Cilium](https://cilium.io/)* +- [Calico](https://docs.projectcalico.org/v3.11/introduction/) +- [Multus](https://github.com/k8snetworkplumbingwg/multus-cni) + +\* When using [project network isolation](#project-network-isolation) in the [Cilium CNI](../../../faq/container-network-interface-providers.md#cilium), it is possible to enable cross-node ingress routing. Click the [CNI provider docs](../../../faq/container-network-interface-providers.md#ingress-routing-across-nodes-in-cilium) to learn more. + +For more details on the different networking providers and how to configure them, please view our [RKE2 documentation](https://docs.rke2.io/install/network_options). + +###### Dual-stack Networking + +[Dual-stack](https://docs.rke2.io/install/network_options#dual-stack-configuration) networking is supported for all CNI providers. To configure RKE2 in dual-stack mode, set valid IPv4/IPv6 CIDRs for your [Cluster CIDR](#cluster-cidr) and/or [Service CIDR](#service-cidr). + +###### Additional Configuration {#dual-stack-additional-config} + +When using `cilium` or `multus,cilium` as your container network interface provider, ensure the **Enable IPv6 Support** option is also enabled. + +##### Cloud Provider + +You can configure a [Kubernetes cloud provider](../../../pages-for-subheaders/set-up-cloud-providers.md). If you want to use dynamically provisioned [volumes and storage](../../../pages-for-subheaders/create-kubernetes-persistent-storage.md) in Kubernetes, typically you must select the specific cloud provider in order to use it. For example, if you want to use Amazon EBS, you would need to select the `aws` cloud provider. + +:::note + +If the cloud provider you want to use is not listed as an option, you will need to use the [config file option](#cluster-config-file-reference) to configure the cloud provider. Please reference [this documentation](https://rancher.com/docs/rke/latest/en/config-options/cloud-providers/) on how to configure the cloud provider. + +::: + +##### Default Pod Security Policy + +The default [pod security policy](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies.md) for the cluster. Please refer to the [RKE2 documentation](https://docs.rke2.io/security/pod_security_policies) on the specifications of each available policy. + +##### Pod Security Admission Configuration Template + +The default [pod security admission configuration template](../../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates.md) for the cluster. + +##### Worker CIS Profile + +Select a [CIS benchmark](../../../pages-for-subheaders/cis-scan-guides.md) to validate the system configuration against. + +##### Project Network Isolation + +If your network provider allows project network isolation, you can choose whether to enable or disable inter-project communication. + +Project network isolation is available if you are using any RKE2 network plugin that supports the enforcement of Kubernetes network policies, such as Canal. + +##### CoreDNS + +By default, [CoreDNS](https://coredns.io/) is installed as the default DNS provider. If CoreDNS is not installed, an alternate DNS provider must be installed yourself. Refer to the [RKE2 documentation](https://docs.rke2.io/networking#coredns) for additional CoreDNS configurations. + +##### NGINX Ingress + +If you want to publish your applications in a high-availability configuration, and you're hosting your nodes with a cloud-provider that doesn't have a native load-balancing feature, enable this option to use NGINX Ingress within the cluster. Refer to the [RKE2 documentation](https://docs.rke2.io/networking#nginx-ingress-controller) for additional configuration options. + +Refer to the [RKE2 documentation](https://docs.rke2.io/networking#nginx-ingress-controller) for additional configuration options. + +##### Metrics Server + +Option to enable or disable [Metrics Server](https://rancher.com/docs/rke/latest/en/config-options/add-ons/metrics-server/). + +Each cloud provider capable of launching a cluster using RKE2 can collect metrics and monitor for your cluster nodes. Enable this option to view your node metrics from your cloud provider's portal. + +#### Add-On Config + +Additional Kubernetes manifests, managed as an [Add-on](https://kubernetes.io/docs/concepts/cluster-administration/addons/), to apply to the cluster on startup. Refer to the [RKE2 documentation](https://docs.rke2.io/helm#automatically-deploying-manifests-and-helm-charts) for details. + +#### Agent Environment Vars + +Option to set environment variables for [Rancher agents](../../../how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents.md). The environment variables can be set using key value pairs. Refer to the [RKE2 documentation](https://docs.rke2.io/reference/linux_agent_config) for more details. + +#### etcd + +##### Automatic Snapshots + +Option to enable or disable recurring etcd snapshots. If enabled, users have the option to configure the frequency of snapshots. For details, refer to the [RKE2 documentation](https://docs.rke2.io/backup_restore#creating-snapshots). Note that with RKE2, snapshots are stored on each etcd node. This varies from RKE1 which only stores one snapshot per cluster. + +##### Metrics + +Option to choose whether to expose etcd metrics to the public or only within the cluster. + +#### Networking + +##### Cluster CIDR + +IPv4 and/or IPv6 network CIDRs to use for pod IPs (default: 10.42.0.0/16). + +###### Dual-stack Networking + +To configure [dual-stack](https://docs.rke2.io/install/network_options#dual-stack-configuration) mode, enter a valid IPv4/IPv6 CIDR. For example `10.42.0.0/16,2001:cafe:42:0::/56`. + +[Additional configuration](#dual-stack-additional-config) is required when using `cilium` or `multus,cilium` as your [container network](#container-network-provider) interface provider. + +##### Service CIDR + +IPv4/IPv6 network CIDRs to use for service IPs (default: 10.43.0.0/16). + +###### Dual-stack Networking + +To configure [dual-stack](https://docs.rke2.io/install/network_options#dual-stack-configuration) mode, enter a valid IPv4/IPv6 CIDR. For example `10.42.0.0/16,2001:cafe:42:0::/56`. + +[Additional configuration](#dual-stack-additional-config) is required when using `cilium ` or `multus,cilium` as your [container network](#container-network-provider) interface provider. + +##### Cluster DNS + +IPv4 Cluster IP for coredns service. Should be in your service-cidr range (default: 10.43.0.10). + +##### Cluster Domain + +Select the domain for the cluster. The default is `cluster.local`. + +##### NodePort Service Port Range + +Option to change the range of ports that can be used for [NodePort services](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). The default is `30000-32767`. + +##### Truncate Hostnames + +Option to truncate hostnames to 15 characters or less. You can only set this field during the initial creation of the cluster. You can't enable or disable the 15 character limit after cluster creation. + +This setting only affects machine-provisioned clusters. Since custom clusters set hostnames during their own node creation process, which occurs outside of Rancher, this field doesn't restrict custom cluster hostname length. + +Truncating hostnames in a cluster improves compatibility with Windows-based systems. Although Kubernetes allows hostnames up to 63 characters in length, systems that use NetBIOS restrict hostnames to 15 characters or less. + +##### TLS Alternate Names + +Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert. + +##### Authorized Cluster Endpoint + +Authorized Cluster Endpoint can be used to directly access the Kubernetes API server, without requiring communication through Rancher. + +This is enabled by default in Rancher-launched Kubernetes clusters, using the IP of the node with the `controlplane` role and the default Kubernetes self signed certificates. + +For more detail on how an authorized cluster endpoint works and why it is used, refer to the [architecture section.](../../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) + +We recommend using a load balancer with the authorized cluster endpoint. For details, refer to the [recommended architecture section.](../../rancher-manager-architecture/architecture-recommendations.md#architecture-for-an-authorized-cluster-endpoint-ace) + +#### Registries + +Select the image repository to pull Rancher images from. For more details and configuration options, see the [RKE2 documentation](https://docs.rke2.io/install/containerd_registry_configuration). + +#### Upgrade Strategy + +##### Control Plane Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Worker Concurrency + +Select how many nodes can be upgraded at the same time. Can be a fixed number or percentage. + +##### Drain Nodes (Control Plane) + +Option to remove all pods from the node prior to upgrading. + +##### Drain Nodes (Worker Nodes) + +Option to remove all pods from the node prior to upgrading. + +#### Advanced + +Option to set kubelet options for different nodes. For available options, refer to the [Kubernetes documentation](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/). + +## Cluster Config File Reference + +Editing clusters in YAML allows you to set the [options available](https://docs.rke2.io/install/configuration) in an RKE2 installation, including those already listed in [Configuration Options in the Rancher UI](#configuration-options-in-the-rancher-ui), as well as set Rancher-specific parameters. + +
+ + Example Cluster Config File Snippet + + +```yaml +apiVersion: provisioning.cattle.io/v1 +kind: Cluster +spec: + cloudCredentialSecretName: cattle-global-data:cc-s879v + kubernetesVersion: v1.25.12+rke2r1 + localClusterAuthEndpoint: {} + rkeConfig: + chartValues: + rke2-calico: {} + etcd: + snapshotRetention: 5 + snapshotScheduleCron: 0 */5 * * * + machineGlobalConfig: + cni: calico + disable-kube-proxy: false + etcd-expose-metrics: false + profile: null + kube-apiserver-arg: + - audit-policy-file=/etc/rancher/rke2/user-audit-policy.yaml + - audit-log-path=/etc/rancher/rke2/user-audit.logs + machinePools: + - controlPlaneRole: true + etcdRole: true + machineConfigRef: + kind: Amazonec2Config + name: nc-test-pool1-pwl5h + name: pool1 + quantity: 1 + unhealthyNodeTimeout: 0s + workerRole: true + machineSelectorConfig: + - config: + protect-kernel-defaults: false + machineSelectorFiles: + - fileSources: + - configMap: + name: '' + secret: + name: audit-policy + items: + - key: audit-policy + path: /etc/rancher/rke2/user-audit-policy.yaml + machineLabelSelector: + matchLabels: + rke.cattle.io/control-plane-role: 'true' + registries: {} + upgradeStrategy: + controlPlaneConcurrency: "1" + controlPlaneDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 + workerConcurrency: "1" + workerDrainOptions: + deleteEmptyDirData: true + enabled: true + gracePeriod: -1 + ignoreDaemonSets: true + timeout: 120 +``` +
+ +### chartValues + +Specify the values for the system charts installed by RKE2. + +Example: + +```yaml +chartValues: + chart-name: + key: value +``` +### machineGlobalConfig + +Specify RKE2 configurations. Any configuration change made here will apply to every node. The configuration options available in the [standalone version of RKE2](https://docs.rke2.io/reference/server_config) can be applied here. + +Example: + +```yaml +machineGlobalConfig: + etcd-arg: + - key1=value1 + - key2=value2 +``` + +### machineSelectorConfig + +`machineSelectorConfig` is the same as [`machineGlobalConfig`](#machineglobalconfig) except that a [label](#kubernetes-node-labels) selector can be specified with the configuration. The configuration will only be applied to nodes that match the provided label selector. + +Multiple `config` entries are allowed, each specifying their own `machineLabelSelector`. A user can specify `matchExpressions`, `matchLabels`, both, or neither. Omitting the `machineLabelSelector` section of this field has the same effect as putting the config in the `machineGlobalConfig` section. + +Example: + +```yaml +machineSelectorConfig + - config: + config-key: config-value + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` +### machineSelectorFiles + +:::note + +This feature is available in Rancher v2.7.2 and later. + +::: + +Deliver files to nodes, so that the files can be in place before initiating RKE2 server or agent processes. +The content of the file is retrieved from either a secret or a configmap. The target nodes are filtered by the `machineLabelSelector`. + +Example : + +```yaml +machineSelectorFiles: + - fileSources: + - secret: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-secret-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 + - fileSources: + - configMap: + items: + - key: example-key + path: path-to-put-the-file-on-nodes + permissions: 644 (optional) + hash: base64-encoded-hash-of-the-content (optional) + name: example-configmap-name + machineLabelSelector: + matchExpressions: + - key: example-key + operator: string # Valid operators are In, NotIn, Exists and DoesNotExist. + values: + - example-value1 + - example-value2 + matchLabels: + key1: value1 + key2: value2 +``` + +The secret or configmap must meet the following requirements: + +1. It must be in the `fleet-default` namespace where the Cluster object exists. +2. It must have the annotation `rke.cattle.io/object-authorized-for-clusters: cluster-name1,cluster-name2`, which permits the target clusters to use it. + +:::tip + +Rancher Dashboard provides an easy-to-use form for creating the secret or configmap. + +::: + +Example: + +```yaml +apiVersion: v1 +data: + audit-policy: >- + IyBMb2cgYWxsIHJlcXVlc3RzIGF0IHRoZSBNZXRhZGF0YSBsZXZlbC4KYXBpVmVyc2lvbjogYXVkaXQuazhzLmlvL3YxCmtpbmQ6IFBvbGljeQpydWxlczoKLSBsZXZlbDogTWV0YWRhdGE= +kind: Secret +metadata: + annotations: + rke.cattle.io/object-authorized-for-clusters: cluster1 + name: name1 + namespace: fleet-default +``` diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md new file mode 100644 index 00000000000..63fa8df3511 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters.md @@ -0,0 +1,45 @@ +--- +title: Syncing +--- + + + + + +Syncing allows Rancher to update cluster values so that they're up to date with the corresponding cluster object hosted in AKS, EKS or GKE. This enables sources other than Rancher to own a hosted cluster’s state. + +:::warning +You may accidentally overwrite the state from one source if you simultaneously process an update from another source. This may also occur if you process an update from one source within 5 minutes of finishing an update from another source. +::: + +### How it works + +There are two fields on the Rancher Cluster object that must be understood to understand how syncing works: + +1. The config object for the cluster, located on the Spec of the Cluster: + + * For AKS, the field is called AKSConfig + * For EKS, the field is called EKSConfig + * For GKE, the field is called GKEConfig + +2. The UpstreamSpec object + + * For AKS, this is located on the AKSStatus field on the Status of the Cluster. + * For EKS, this is located on the EKSStatus field on the Status of the Cluster. + * For GKE, this is located on the GKEStatus field on the Status of the Cluster. + +The struct types that define these objects can be found in their corresponding operator projects: + + * [aks-operator](https://github.com/rancher/aks-operator/blob/master/pkg/apis/aks.cattle.io/v1/types.go) + * [eks-operator](https://github.com/rancher/eks-operator/blob/master/pkg/apis/eks.cattle.io/v1/types.go) + * [gke-operator](https://github.com/rancher/gke-operator/blob/master/pkg/apis/gke.cattle.io/v1/types.go) + +All fields are nillable, except for the following: the cluster name, the location (region or zone), Imported, and the cloud credential reference. + +The AKSConfig, EKSConfig or GKEConfig represents the desired state. Nil values are ignored. Fields that are non-nil in the config object can be thought of as managed. When a cluster is created in Rancher, all fields are non-nil and therefore managed. When a pre-existing cluster is registered in Rancher all nillable fields are set to nil and aren't managed. Those fields become managed once their value has been changed by Rancher. + +UpstreamSpec represents the cluster as it is in the hosted Kubernetes provider. It's refreshed every 5 minutes. After the UpstreamSpec is refreshed, Rancher checks if the cluster has an update in progress. If it's currently updating, nothing further is done. If it is not currently updating, any managed fields on AKSConfig, EKSConfig or GKEConfig are overwritten with their corresponding value from the recently updated UpstreamSpec. + +The effective desired state can be thought of as the UpstreamSpec, plus all non-nil fields in the AKSConfig, EKSConfig or GKEConfig. This is what is displayed in the UI. + +If Rancher and another source attempt to update a cluster at the same time, or within 5 minutes of an update finishing, any managed fields are likely to get caught in a race condition. To use EKS as an example, a cluster may have PrivateAccess as a managed field. If PrivateAccess is false and then enabled in EKS console at 11:01, and tags are updated from Rancher before 11:05, then the value is likely to be overwritten. This can also occur if tags are updated while the cluster is still processing the update. The issue described in this example shouldn't occur if the cluster is registered and the PrivateAccess fields are nil. diff --git a/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md new file mode 100644 index 00000000000..3c363d053cf --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options.md @@ -0,0 +1,57 @@ +--- +title: Rancher Agent Options +--- + + + + + +Rancher deploys an agent on each node to communicate with the node. This pages describes the options that can be passed to the agent. To use these options, you will need to [create a cluster with custom nodes](../../../../pages-for-subheaders/use-existing-nodes.md) and add the options to the generated `docker run` command when adding a node. + +For an overview of how Rancher communicates with downstream clusters using node agents, refer to the [architecture section.](../../../rancher-manager-architecture/communicating-with-downstream-user-clusters.md#3-node-agents) + +## General options + +| Parameter | Environment variable | Description | +| ---------- | -------------------- | ----------- | +| `--server` | `CATTLE_SERVER` | The configured Rancher `server-url` setting which the agent connects to | +| `--token` | `CATTLE_TOKEN` | Token that is needed to register the node in Rancher | +| `--ca-checksum` | `CATTLE_CA_CHECKSUM` | The SHA256 checksum of the configured Rancher `cacerts` setting to validate | +| `--node-name` | `CATTLE_NODE_NAME` | Override the hostname that is used to register the node (defaults to `hostname -s`) | +| `--label` | `CATTLE_NODE_LABEL` | Add node labels to the node. For multiple labels, pass additional `--label` options. (`--label key=value`) | +| `--taints` | `CATTLE_NODE_TAINTS` | Add node taints to the node. For multiple taints, pass additional `--taints` options. (`--taints key=value:effect`) | + +## Role options + +| Parameter | Environment variable | Description | +| ---------- | -------------------- | ----------- | +| `--all-roles` | `ALL=true` | Apply all roles (`etcd`,`controlplane`,`worker`) to the node | +| `--etcd` | `ETCD=true` | Apply the role `etcd` to the node | +| `--controlplane` | `CONTROL=true` | Apply the role `controlplane` to the node | +| `--worker` | `WORKER=true` | Apply the role `worker` to the node | + +## IP address options + +| Parameter | Environment variable | Description | +| ---------- | -------------------- | ----------- | +| `--address` | `CATTLE_ADDRESS` | The IP address the node will be registered with (defaults to the IP used to reach `8.8.8.8`) | +| `--internal-address` | `CATTLE_INTERNAL_ADDRESS` | The IP address used for inter-host communication on a private network | + +### Dynamic IP address options + +For automation purposes, you can't have a specific IP address in a command as it has to be generic to be used for every node. For this, we have dynamic IP address options. They are used as a value to the existing IP address options. This is supported for `--address` and `--internal-address`. + +| Value | Example | Description | +| ---------- | -------------------- | ----------- | +| Interface name | `--address eth0` | The first configured IP address will be retrieved from the given interface | +| `ipify` | `--address ipify` | Value retrieved from `https://api.ipify.org` will be used | +| `awslocal` | `--address awslocal` | Value retrieved from `http://169.254.169.254/latest/meta-data/local-ipv4` will be used | +| `awspublic` | `--address awspublic` | Value retrieved from `http://169.254.169.254/latest/meta-data/public-ipv4` will be used | +| `doprivate` | `--address doprivate` | Value retrieved from `http://169.254.169.254/metadata/v1/interfaces/private/0/ipv4/address` will be used | +| `dopublic` | `--address dopublic` | Value retrieved from `http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address` will be used | +| `azprivate` | `--address azprivate` | Value retrieved from `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/privateIpAddress?api-version=2017-08-01&format=text` will be used | +| `azpublic` | `--address azpublic` | Value retrieved from `http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-08-01&format=text` will be used | +| `gceinternal` | `--address gceinternal` | Value retrieved from `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip` will be used | +| `gceexternal` | `--address gceexternal` | Value retrieved from `http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip` will be used | +| `packetlocal` | `--address packetlocal` | Value retrieved from `https://metadata.packet.net/2009-04-04/meta-data/local-ipv4` will be used | +| `packetpublic` | `--address packetlocal` | Value retrieved from `https://metadata.packet.net/2009-04-04/meta-data/public-ipv4` will be used | diff --git a/versioned_docs/version-latest/reference-guides/kubernetes-concepts.md b/versioned_docs/version-latest/reference-guides/kubernetes-concepts.md new file mode 100644 index 00000000000..631b7cfd118 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/kubernetes-concepts.md @@ -0,0 +1,64 @@ +--- +title: Kubernetes Concepts +--- + +This page explains concepts related to Kubernetes that are important for understanding how Rancher works. The descriptions below provide a simplified overview of Kubernetes components. For more details, refer to the [official documentation on Kubernetes components.](https://kubernetes.io/docs/concepts/overview/components/) + +## About Docker + +Docker is the container packaging and runtime standard. Developers build container images from Dockerfiles and distribute container images from Docker registries. [Docker Hub](https://hub.docker.com) is the most popular public registry. Many organizations also set up private Docker registries. Docker is primarily used to manage containers on individual nodes. + +:::note + +Although Rancher 1.6 supported Docker Swarm clustering technology, it is no longer supported in Rancher 2.x due to the success of Kubernetes. + +::: + +## About Kubernetes + +Kubernetes is the container cluster management standard. YAML files specify containers and other resources that form an application. Kubernetes performs functions such as scheduling, scaling, service discovery, health check, secret management, and configuration management. + +## What is a Kubernetes Cluster? + +A cluster is a group of computers that work together as a single system. + +A _Kubernetes Cluster_ is a cluster that uses the [Kubernetes container-orchestration system](https://kubernetes.io/) to deploy, maintain, and scale Docker containers, allowing your organization to automate application operations. + +## Roles for Nodes in Kubernetes Clusters + +Each computing resource in a Kubernetes cluster is called a _node_. Nodes can be either bare-metal servers or virtual machines. Kubernetes classifies nodes into three types: _etcd_ nodes, _control plane_ nodes, and _worker_ nodes. + +A Kubernetes cluster consists of at least one etcd, controlplane, and worker node. + +### etcd Nodes + +Rancher uses etcd as a data store in both single node and high-availability installations. In Kubernetes, etcd is also a role for nodes that store the cluster state. + +The state of a Kubernetes cluster is maintained in [etcd.](https://kubernetes.io/docs/concepts/overview/components/#etcd) The etcd nodes run the etcd database. + +The etcd database component is a distributed key-value store used as Kubernetes storage for all cluster data, such as cluster coordination and state management. It is recommended to run etcd on multiple nodes so that there's always a backup available for failover. + +Although you can run etcd on just one node, etcd requires a majority of nodes, a quorum, to agree on updates to the cluster state. The cluster should always contain enough healthy etcd nodes to form a quorum. For a cluster with n members, a quorum is (n/2)+1. For any odd-sized cluster, adding one node will always increase the number of nodes necessary for a quorum. + +Three etcd nodes is generally sufficient for smaller clusters and five etcd nodes for large clusters. + +### Controlplane Nodes + +Controlplane nodes run the Kubernetes API server, scheduler, and controller manager. These nodes take care of routine tasks to ensure that your cluster maintains your configuration. Because all cluster data is stored on your etcd nodes, control plane nodes are stateless. You can run control plane on a single node, although three or more nodes are recommended for redundancy. Additionally, a single node can share the control plane and etcd roles. + +### Worker Nodes + +Each [worker node](https://kubernetes.io/docs/concepts/architecture/nodes/) runs the following: + +- **Kubelets:** An agent that monitors the state of the node, ensuring your containers are healthy. +- **Workloads:** The containers and pods that hold your apps, as well as other types of deployments. + +Worker nodes also run storage and networking drivers, and ingress controllers when required. You create as many worker nodes as necessary to run your [workloads](../pages-for-subheaders/workloads-and-pods.md). + +## About Helm + +For high-availability installations of Rancher, Helm is the tool used to install Rancher on a Kubernetes cluster. + +Helm is the package management tool of choice for Kubernetes. Helm charts provide templating syntax for Kubernetes YAML manifest documents. With Helm we can create configurable deployments instead of just using static files. For more information about creating your own catalog of deployments, check out the docs at [https://helm.sh/](https://helm.sh). + +For more information on service accounts and cluster role binding, refer to the [Kubernetes documentation.](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) diff --git a/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/examples.md b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/examples.md new file mode 100644 index 00000000000..c65722cfa97 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/examples.md @@ -0,0 +1,27 @@ +--- +title: Examples +--- + + + + + +### ServiceMonitor + +An example ServiceMonitor custom resource can be found [here.](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + +### PodMonitor + +An example PodMonitor can be found [here.](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/example-app-pod-monitor.yaml) An example Prometheus resource that refers to it can be found [here.](https://github.com/prometheus-operator/prometheus-operator/blob/master/example/user-guides/getting-started/prometheus-pod-monitor.yaml) + +### PrometheusRule + +For users who are familiar with Prometheus, a PrometheusRule contains the alerting and recording rules that you would normally place in a [Prometheus rule file](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/). + +For a more fine-grained application of PrometheusRules within your cluster, the ruleSelector field on a Prometheus resource allows you to select which PrometheusRules should be loaded onto Prometheus based on the labels attached to the PrometheusRules resources. + +An example PrometheusRule is on [this page.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/alerting.md) + +### Alertmanager Config + +For an example configuration, refer to [this section](./receivers.md#example-alertmanager-configs). diff --git a/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/helm-chart-options.md b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/helm-chart-options.md new file mode 100644 index 00000000000..112c41c9ea0 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/helm-chart-options.md @@ -0,0 +1,72 @@ +--- +title: Helm Chart Options +--- + + + + + +## Configuring Resource Limits and Requests + +The resource requests and limits can be configured when installing `rancher-monitoring`. + +The default values are in the [values.yaml](https://github.com/rancher/charts/blob/main/charts/rancher-monitoring/values.yaml) in the `rancher-monitoring` Helm chart. + +The default values in the table below are the minimum required resource limits and requests. + +| Resource Name | Memory Limit | CPU Limit | Memory Request | CPU Request | +| ------------- | ------------ | ----------- | ---------------- | ------------------ | +| alertmanager | 500Mi | 1000m | 100Mi | 100m | +| grafana | 200Mi | 200m | 100Mi | 100m | +| kube-state-metrics subchart | 200Mi | 100m | 130Mi | 100m | +| prometheus-node-exporter subchart | 50Mi | 200m | 30Mi | 100m | +| prometheusOperator | 500Mi | 200m | 100Mi | 100m | +| prometheus | 2500Mi | 1000m | 1750Mi | 750m | +| **Total** | **3950Mi** | **2700m** | **2210Mi** | **1250m** | + +At least 50Gi storage is recommended. + + +## Trusted CA for Notifiers + +If you need to add a trusted CA to your notifier, follow these steps: + +1. Create the `cattle-monitoring-system` namespace. +1. Add your trusted CA secret to the `cattle-monitoring-system` namespace. +1. Deploy or upgrade the `rancher-monitoring` Helm chart. In the chart options, reference the secret in **Alerting > Additional Secrets**. + +**Result:** The default Alertmanager custom resource will have access to your trusted CA. + + +## Additional Scrape Configurations + +If the scrape configuration you want cannot be specified via a ServiceMonitor or PodMonitor at the moment, you can provide an `additionalScrapeConfigSecret` on deploying or upgrading `rancher-monitoring`. + +A [scrape_config section](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) specifies a set of targets and parameters describing how to scrape them. In the general case, one scrape configuration specifies a single job. + +An example of where this might be used is with Istio. For more information, see [this section.](../../integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations.md) + + +## Configuring Applications Packaged within Monitoring v2 + +We deploy kube-state-metrics and node-exporter with monitoring v2. Node exporter are deployed as DaemonSets. In the monitoring v2 helm chart, in the values.yaml, each of the things are deployed as sub charts. + +We also deploy grafana which is not managed by prometheus. + +If you look at what the helm chart is doing like in kube-state-metrics, there are plenty more values that you can set that aren’t exposed in the top level chart. + +But in the top level chart you can add values that override values that exist in the sub chart. + +### Increase the Replicas of Alertmanager + +As part of the chart deployment options, you can opt to increase the number of replicas of the Alertmanager deployed onto your cluster. The replicas can all be managed using the same underlying Alertmanager Config Secret. For more information on the Alertmanager Config Secret, refer to [this section.](../../how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager.md#multiple-alertmanager-replicas) + +### Configuring the Namespace for a Persistent Grafana Dashboard + +To specify that you would like Grafana to watch for ConfigMaps across all namespaces, set this value in the `rancher-monitoring` Helm chart: + +``` +grafana.sidecar.dashboards.searchNamespace=ALL +``` + +Note that the RBAC roles exposed by the Monitoring chart to add Grafana Dashboards are still restricted to giving permissions for users to add dashboards in the namespace defined in `grafana.dashboards.namespace`, which defaults to `cattle-dashboards`. \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/receivers.md b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/receivers.md new file mode 100644 index 00000000000..dc459f34378 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/receivers.md @@ -0,0 +1,303 @@ +--- +title: Receiver Configuration +--- + + + + + +The [Alertmanager Config](https://prometheus.io/docs/alerting/latest/configuration/#configuration-file) Secret contains the configuration of an Alertmanager instance that sends out notifications based on alerts it receives from Prometheus. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [this section.](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md#3-how-alertmanager-works) + +::: + + +## Creating Receivers in the Rancher UI + +:::note Prerequisites: + +- The monitoring application needs to be installed. +- If you configured monitoring with an existing Alertmanager Secret, it must have a format that is supported by Rancher's UI. Otherwise you will only be able to make changes based on modifying the Alertmanager Secret directly. Note: We are continuing to make enhancements to what kinds of Alertmanager Configurations we can support using the Routes and Receivers UI, so please [file an issue](https://github.com/rancher/rancher/issues/new) if you have a request for a feature enhancement. + +::: + +To create notification receivers in the Rancher UI, + + + + +1. Go to the cluster where you want to create receivers. Click **Monitoring -> Alerting -> AlertManagerConfigs**. +1. Click **Create**. +1. Click **Add Receiver**. +1. Enter a **Name** for the receiver. +1. Configure one or more providers for the receiver. For help filling out the forms, refer to the configuration options below. +1. Click **Create**. + + + + +1. Go to the cluster where you want to create receivers. Click **Monitoring** and click **Receiver**. +2. Enter a name for the receiver. +3. Configure one or more providers for the receiver. For help filling out the forms, refer to the configuration options below. +4. Click **Create**. + + + + +**Result:** Alerts can be configured to send notifications to the receiver(s). + +## Receiver Configuration + +The notification integrations are configured with the `receiver`, which is explained in the [Prometheus documentation.](https://prometheus.io/docs/alerting/latest/configuration/#receiver) + +### Native vs. Non-native Receivers + +By default, AlertManager provides native integration with some receivers, which are listed in [this section.](https://prometheus.io/docs/alerting/latest/configuration/#receiver) All natively supported receivers are configurable through the Rancher UI. + +For notification mechanisms not natively supported by AlertManager, integration is achieved using the [webhook receiver.](https://prometheus.io/docs/alerting/latest/configuration/#webhook_config) A list of third-party drivers providing such integrations can be found [here.](https://prometheus.io/docs/operating/integrations/#alertmanager-webhook-receiver) Access to these drivers, and their associated integrations, is provided through the Alerting Drivers app. Once enabled, configuring non-native receivers can also be done through the Rancher UI. + +Currently the Rancher Alerting Drivers app provides access to the following integrations: +- Microsoft Teams, based on the [prom2teams](https://github.com/idealista/prom2teams) driver +- SMS, based on the [Sachet](https://github.com/messagebird/sachet) driver + +The following types of receivers can be configured in the Rancher UI: + +- Slack +- Email +- PagerDuty +- Opsgenie +- Webhook +- Custom +- Teams +- SMS + +The custom receiver option can be used to configure any receiver in YAML that cannot be configured by filling out the other forms in the Rancher UI. + +## Slack + +| Field | Type | Description | +|------|--------------|------| +| URL | String | Enter your Slack webhook URL. For instructions to create a Slack webhook, see the [Slack documentation.](https://get.slack.help/hc/en-us/articles/115005265063-Incoming-WebHooks-for-Slack) | +| Default Channel | String | Enter the name of the channel that you want to send alert notifications in the following format: `#`. | +| Proxy URL | String | Proxy for the webhook notifications. | +| Enable Send Resolved Alerts | Bool | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +## Email + +| Field | Type | Description | +|------|--------------|------| +| Default Recipient Address | String | The email address that will receive notifications. | +| Enable Send Resolved Alerts | Bool | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +SMTP options: + +| Field | Type | Description | +|------|--------------|------| +| Sender | String | Enter an email address available on your SMTP mail server that you want to send the notification from. | +| Host | String | Enter the IP address or hostname for your SMTP server. Example: `smtp.email.com`. | +| Use TLS | Bool | Use TLS for encryption. | +| Username | String | Enter a username to authenticate with the SMTP server. | +| Password | String | Enter a password to authenticate with the SMTP server. | + +## PagerDuty + +| Field | Type | Description | +|------|------|-------| +| Integration Type | String | `Events API v2` or `Prometheus`. | +| Default Integration Key | String | For instructions to get an integration key, see the [PagerDuty documentation.](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/) | +| Proxy URL | String | Proxy for the PagerDuty notifications. | +| Enable Send Resolved Alerts | Bool | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +## Opsgenie + +| Field | Description | +|------|-------------| +| API Key | For instructions to get an API key, refer to the [Opsgenie documentation.](https://docs.opsgenie.com/docs/api-key-management) | +| Proxy URL | Proxy for the Opsgenie notifications. | +| Enable Send Resolved Alerts | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + +Opsgenie Responders: + +| Field | Type | Description | +|-------|------|--------| +| Type | String | Schedule, Team, User, or Escalation. For more information on alert responders, refer to the [Opsgenie documentation.](https://docs.opsgenie.com/docs/alert-recipients-and-teams) | +| Send To | String | Id, Name, or Username of the Opsgenie recipient. | + +## Webhook + +| Field | Description | +|-------|--------------| +| URL | Webhook URL for the app of your choice. | +| Proxy URL | Proxy for the webhook notification. | +| Enable Send Resolved Alerts | Whether to send a follow-up notification if an alert has been resolved (e.g. [Resolved] High CPU Usage). | + + + +## Custom + +The YAML provided here will be directly appended to your receiver within the Alertmanager Config Secret. + +## Teams + +### Enabling the Teams Receiver for Rancher Managed Clusters + +The Teams receiver is not a native receiver and must be enabled before it can be used. You can enable the Teams receiver for a Rancher managed cluster by going to the Apps page and installing the rancher-alerting-drivers app with the Teams option selected. + +1. In the Rancher UI, go to the cluster where you want to install rancher-alerting-drivers and click **Apps**. +1. Click the **Alerting Drivers** app. +1. Click the **Helm Deploy Options** tab. +1. Select the **Teams** option and click **Install**. +1. Take note of the namespace used as it will be required in a later step. + +### Configure the Teams Receiver + +The Teams receiver can be configured by updating its ConfigMap. For example, the following is a minimal Teams receiver configuration. + +```yaml +[Microsoft Teams] +teams-instance-1: https://your-teams-webhook-url +``` + +When configuration is complete, add the receiver using the steps in [this section](#creating-receivers-in-the-rancher-ui). + +Use the example below as the URL where: + +- `ns-1` is replaced with the namespace where the `rancher-alerting-drivers` app is installed + +```yaml +url: http://rancher-alerting-drivers-prom2teams.ns-1.svc:8089/v2/teams-instance-1 +``` + + + +## SMS + +### Enabling the SMS Receiver for Rancher Managed Clusters + +The SMS receiver is not a native receiver and must be enabled before it can be used. You can enable the SMS receiver for a Rancher managed cluster by going to the Apps page and installing the rancher-alerting-drivers app with the SMS option selected. + +1. In the upper left corner, click **☰ > Cluster Management**. +1. On the **Clusters** page, go to the cluster where you want to install `rancher-alerting-drivers` and click **Explore**. +1. In the left navigation bar, click +1. Click the **Alerting Drivers** app. +1. Click the **Helm Deploy Options** tab +1. Select the **SMS** option and click **Install**. +1. Take note of the namespace used as it will be required in a later step. + +### Configure the SMS Receiver + +The SMS receiver can be configured by updating its ConfigMap. For example, the following is a minimal SMS receiver configuration. + +```yaml +providers: + telegram: + token: 'your-token-from-telegram' + +receivers: +- name: 'telegram-receiver-1' + provider: 'telegram' + to: + - '123456789' +``` + +When configuration is complete, add the receiver using the steps in [this section](#creating-receivers-in-the-rancher-ui). + +Use the example below as the name and URL, where: + +- the name assigned to the receiver, e.g. `telegram-receiver-1`, must match the name in the `receivers.name` field in the ConfigMap, e.g. `telegram-receiver-1` +- `ns-1` in the URL is replaced with the namespace where the `rancher-alerting-drivers` app is installed + +```yaml +name: telegram-receiver-1 +url http://rancher-alerting-drivers-sachet.ns-1.svc:9876/alert +``` + + + + +## Configuring Multiple Receivers + +By editing the forms in the Rancher UI, you can set up a Receiver resource with all the information Alertmanager needs to send alerts to your notification system. + +It is also possible to send alerts to multiple notification systems. One way is to configure the Receiver using custom YAML, in which case you can add the configuration for multiple notification systems, as long as you are sure that both systems should receive the same messages. + +You can also set up multiple receivers by using the `continue` option for a route, so that the alerts sent to a receiver continue being evaluated in the next level of the routing tree, which could contain another receiver. + + +## Example Alertmanager Configs + +### Slack +To set up notifications via Slack, the following Alertmanager Config YAML can be placed into the `alertmanager.yaml` key of the Alertmanager Config Secret, where the `api_url` should be updated to use your Webhook URL from Slack: + +```yaml +route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + receiver: 'slack-notifications' +receivers: +- name: 'slack-notifications' + slack_configs: + - send_resolved: true + text: '{{ template "slack.rancher.text" . }}' + api_url: +templates: +- /etc/alertmanager/config/*.tmpl +``` + +### PagerDuty +To set up notifications via PagerDuty, use the example below from the [PagerDuty documentation](https://www.pagerduty.com/docs/guides/prometheus-integration-guide/) as a guideline. This example sets up a route that captures alerts for a database service and sends them to a receiver linked to a service that will directly notify the DBAs in PagerDuty, while all other alerts will be directed to a default receiver with a different PagerDuty integration key. + +The following Alertmanager Config YAML can be placed into the `alertmanager.yaml` key of the Alertmanager Config Secret. The `service_key` should be updated to use your PagerDuty integration key and can be found as per the "Integrating with Global Event Routing" section of the PagerDuty documentation. For the full list of configuration options, refer to the [Prometheus documentation](https://prometheus.io/docs/alerting/latest/configuration/#pagerduty_config). + +```yaml +route: + group_by: [cluster] + receiver: 'pagerduty-notifications' + group_interval: 5m + routes: + - match: + service: database + receiver: 'database-notifcations' + +receivers: +- name: 'pagerduty-notifications' + pagerduty_configs: + - service_key: 'primary-integration-key' + +- name: 'database-notifcations' + pagerduty_configs: + - service_key: 'database-integration-key' +``` + +## Example Route Config for CIS Scan Alerts + +While configuring the routes for `rancher-cis-benchmark` alerts, you can specify the matching using the key-value pair `job: rancher-cis-scan`. + +For example, the following example route configuration could be used with a Slack receiver named `test-cis`: + +```yaml +spec: + receiver: test-cis + group_by: +# - string + group_wait: 30s + group_interval: 30s + repeat_interval: 30s + match: + job: rancher-cis-scan +# key: string + match_re: + {} +# key: string +``` + +For more information on enabling alerting for `rancher-cis-benchmark`, see [this section.](../../how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark.md) + +## Trusted CA for Notifiers + +If you need to add a trusted CA to your notifier, follow the steps in [this section.](helm-chart-options.md#trusted-ca-for-notifiers) diff --git a/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/routes.md b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/routes.md new file mode 100644 index 00000000000..3f538d92112 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/routes.md @@ -0,0 +1,68 @@ +--- +title: Route Configuration +--- + + + + + +The route configuration is the section of the Alertmanager custom resource that controls how the alerts fired by Prometheus are grouped and filtered before they reach the receiver. + +When a Route is changed, the Prometheus Operator regenerates the Alertmanager custom resource to reflect the changes. + +For more information about configuring routes, refer to the [official Alertmanager documentation.](https://www.prometheus.io/docs/alerting/latest/configuration/#route) + +:::note + +This section assumes familiarity with how monitoring components work together. For more information, see [this section.](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + + + +## Route Restrictions + +Alertmanager proxies alerts for Prometheus based on its receivers and a routing tree that filters alerts to certain receivers based on labels. + +Alerting drivers proxy alerts for Alertmanager to non-native receivers, such as Microsoft Teams and SMS. + +In the Rancher UI for configuring routes and receivers, you can configure routing trees with one root and then a depth of one more level, for a tree with a depth of two. But if you use a `continue` route when configuring Alertmanager directly, you can make the tree deeper. + +Each receiver is for one or more notification providers. So if you know that every alert for Slack should also go to PagerDuty, you can configure both in the same receiver. + +## Route Configuration + +### Note on Labels and Annotations + +Labels should be used for identifying information that can affect the routing of notifications. Identifying information about the alert could consist of a container name, or the name of the team that should be notified. + +Annotations should be used for information that does not affect who receives the alert, such as a runbook url or error message. + + +### Receiver +The route needs to refer to a [receiver](./receivers.md) that has already been configured. + +### Grouping + +| Field | Default | Description | +|-------|--------------|---------| +| Group By | N/a | List of labels to group by. Labels must be unique. Special label "..." (aggregate by all possible labels), if provided, must be the only element in the list. Accepts a list of strings. See the [upstream documentation](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#route) for details. | +| Group Wait | 30s | How long to wait to buffer alerts of the same group before sending initially. | +| Group Interval | 5m | How long to wait before sending an alert that has been added to a group of alerts for which an initial notification has already been sent. | +| Repeat Interval | 4h | How long to wait before re-sending a given alert that has already been sent. | + +### Matching + +The **Match** field refers to a set of equality matchers used to identify which alerts to send to a given Route based on labels defined on that alert. When you add key-value pairs to the Rancher UI, they correspond to the YAML in this format: + +```yaml +match: + [ : , ... ] +``` + +The **Match Regex** field refers to a set of regex-matchers used to identify which alerts to send to a given Route based on labels defined on that alert. When you add key-value pairs in the Rancher UI, they correspond to the YAML in this format: + +```yaml +match_re: + [ : , ... ] +``` diff --git a/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md new file mode 100644 index 00000000000..30fccb634c8 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors.md @@ -0,0 +1,37 @@ +--- +title: ServiceMonitor and PodMonitor Configuration +--- + + + + + +ServiceMonitors and PodMonitors are both pseudo-CRDs that map the scrape configuration of the Prometheus custom resource. + +These configuration objects declaratively specify the endpoints that Prometheus will scrape metrics from. + +ServiceMonitors are more commonly used than PodMonitors, and we recommend them for most use cases. + +:::note + +This section assumes familiarity with how monitoring components work together. For more information about Alertmanager, see [this section.](../../integrations-in-rancher/monitoring-and-alerting/how-monitoring-works.md) + +::: + +### ServiceMonitors + +This pseudo-CRD maps to a section of the Prometheus custom resource configuration. It declaratively specifies how groups of Kubernetes services should be monitored. + +When a ServiceMonitor is created, the Prometheus Operator updates the Prometheus scrape configuration to include the ServiceMonitor configuration. Then Prometheus begins scraping metrics from the endpoint defined in the ServiceMonitor. + +Any Services in your cluster that match the labels located within the ServiceMonitor `selector` field will be monitored based on the `endpoints` specified on the ServiceMonitor. For more information on what fields can be specified, please look at the [spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) provided by Prometheus Operator. + +For more information about how ServiceMonitors work, refer to the [Prometheus Operator documentation.](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/running-exporters.md) + +### PodMonitors + +This pseudo-CRD maps to a section of the Prometheus custom resource configuration. It declaratively specifies how group of pods should be monitored. + +When a PodMonitor is created, the Prometheus Operator updates the Prometheus scrape configuration to include the PodMonitor configuration. Then Prometheus begins scraping metrics from the endpoint defined in the PodMonitor. + +Any Pods in your cluster that match the labels located within the PodMonitor `selector` field will be monitored based on the `podMetricsEndpoints` specified on the PodMonitor. For more information on what fields can be specified, please look at the [spec](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#podmonitorspec) provided by Prometheus Operator. diff --git a/versioned_docs/version-latest/reference-guides/prometheus-federator/rbac.md b/versioned_docs/version-latest/reference-guides/prometheus-federator/rbac.md new file mode 100644 index 00000000000..8b54ce9559f --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/prometheus-federator/rbac.md @@ -0,0 +1,35 @@ +--- +title: Role-Based Access Control +--- + + + + + +This section describes the expectations for Role-Based Access Control (RBAC) for Prometheus Federator. + +As described in the section on [namespaces](../../pages-for-subheaders/prometheus-federator.md#namespaces), Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: + +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under: + +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit` +- `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view` + +By default, these roleRefs will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +:::note + +For Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +::: + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: + +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, `rancher-project-monitoring`, the underlying chart deployed by Prometheus Federator, creates three default Roles per Project Release Namespace that provide `admin`, `edit`, and `view` users to permissions to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack to provide least privilege. However, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or create Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/rancher-cluster-tools.md b/versioned_docs/version-latest/reference-guides/rancher-cluster-tools.md new file mode 100644 index 00000000000..41305db554b --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-cluster-tools.md @@ -0,0 +1,48 @@ +--- +title: Cluster Tools for Logging, Monitoring, and Visibility +--- + +Rancher contains a variety of tools that aren't included in Kubernetes to assist in your DevOps operations. Rancher can integrate with external services to help your clusters run more efficiently. Tools are divided into following categories: + + +## Logging + +Logging is helpful because it allows you to: + +- Capture and analyze the state of your cluster +- Look for trends in your environment +- Save your logs to a safe location outside of your cluster +- Stay informed of events like a container crashing, a pod eviction, or a node dying +- More easily debugg and troubleshoot problems + +Rancher can integrate with Elasticsearch, splunk, kafka, syslog, and fluentd. + +For more information, refer to the logging documentation [here.](../pages-for-subheaders/logging.md) +## Monitoring and Alerts + +Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with [Prometheus](https://prometheus.io/), a leading open-source monitoring solution. + +After monitoring is enabled, you can set up alerts and notifiers that provide the mechanism to receive them. + +Notifiers are services that inform you of alert events. You can configure notifiers to send alert notifications to staff best suited to take corrective action. Notifications can be sent with Slack, email, PagerDuty, WeChat, and webhooks. + +Alerts are rules that trigger those notifications. Before you can receive alerts, you must configure one or more notifier in Rancher. The scope for alerts can be set at either the cluster or project level. + +For more information, refer to the monitoring documentation [here.](../pages-for-subheaders/monitoring-and-alerting.md) + +## Istio + +[Istio](https://istio.io/) is an open-source tool that makes it easier for DevOps teams to observe, control, troubleshoot, and secure the traffic within a complex network of microservices. + +Rancher's integration with Istio was improved in Rancher v2.5. + +For more information, refer to the Istio documentation [here.](../pages-for-subheaders/istio.md) +## OPA Gatekeeper + +[OPA Gatekeeper](https://github.com/open-policy-agent/gatekeeper) is an open-source project that provides integration between OPA and Kubernetes to provide policy control via admission controller webhooks. For details on how to enable Gatekeeper in Rancher, refer to the [OPA Gatekeeper section.](../integrations-in-rancher/opa-gatekeeper.md) + +## CIS Scans + +Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark. + +For more information, refer to the CIS scan documentation [here.](../pages-for-subheaders/cis-scan-guides.md) \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/architecture-recommendations.md b/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/architecture-recommendations.md new file mode 100644 index 00000000000..1ece990ff8c --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/architecture-recommendations.md @@ -0,0 +1,110 @@ +--- +title: Architecture Recommendations +--- + + + + + +If you are installing Rancher on a single node, the main architecture recommendation that applies to your installation is that the node running Rancher should be [separate from downstream clusters.](#separation-of-rancher-and-user-clusters) + +## Separation of Rancher and User Clusters + +A user cluster is a downstream Kubernetes cluster that runs your apps and services. + +If you have a Docker installation of Rancher, the node running the Rancher server should be separate from your downstream clusters. + +If Rancher is intended to manage downstream Kubernetes clusters, the Kubernetes cluster that the Rancher server runs on should also be separate from the downstream user clusters. + +![Separation of Rancher Server from User Clusters](/img/rancher-architecture-separation-of-rancher-server.svg) + +## Why HA is Better for Rancher in Production + +We recommend installing the Rancher server on a high-availability Kubernetes cluster, primarily because it protects the Rancher server data. In a high-availability installation, a load balancer serves as the single point of contact for clients, distributing network traffic across multiple servers in the cluster and helping to prevent any one server from becoming a point of failure. + +We don't recommend installing Rancher in a single Docker container, because if the node goes down, there is no copy of the cluster data available on other nodes and you could lose the data on your Rancher server. + +### K3s Kubernetes Cluster Installations + +One option for the underlying Kubernetes cluster is to use K3s Kubernetes. K3s is Rancher's CNCF certified Kubernetes distribution. It is easy to install and uses half the memory of Kubernetes, all in a binary of less than 100 MB. Another advantage of K3s is that it allows an external datastore to hold the cluster data, allowing the K3s server nodes to be treated as ephemeral. + +
Architecture of a K3s Kubernetes Cluster Running the Rancher Management Server
+ +![Architecture of a K3s Kubernetes Cluster Running the Rancher Management Server](/img/k3s-server-storage.svg) + +### RKE Kubernetes Cluster Installations + +In an RKE installation, the cluster data is replicated on each of three etcd nodes in the cluster, providing redundancy and data duplication in case one of the nodes fails. + +
Architecture of an RKE Kubernetes Cluster Running the Rancher Management Server
+ +![Architecture of an RKE Kubernetes cluster running the Rancher management server](/img/rke-server-storage.svg) + +## Recommended Load Balancer Configuration for Kubernetes Installations + +We recommend the following configurations for the load balancer and Ingress controllers: + +* The DNS for Rancher should resolve to a Layer 4 load balancer (TCP). +* The Load Balancer should forward port TCP/80 and TCP/443 to all 3 nodes in the Kubernetes cluster. +* The Ingress controller will redirect HTTP to HTTPS and terminate SSL/TLS on port TCP/443. +* The Ingress controller will forward traffic to port TCP/80 on the pod in the Rancher deployment. + +
Rancher installed on a Kubernetes cluster with layer 4 load balancer, depicting SSL termination at Ingress controllers
+ +![Rancher HA](/img/ha/rancher2ha.svg) + +## Environment for Kubernetes Installations + +It is strongly recommended to install Rancher on a Kubernetes cluster on hosted infrastructure such as Amazon's EC2 or Google Compute Engine. + +For the best performance and greater security, we recommend a dedicated Kubernetes cluster for the Rancher management server. Running user workloads on this cluster is not advised. After deploying Rancher, you can [create or import clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) for running your workloads. + +## Recommended Node Roles for Kubernetes Installations + +The below recommendations apply when Rancher is installed on a K3s Kubernetes cluster or an RKE Kubernetes cluster. + +### K3s Cluster Roles + +In K3s clusters, there are two types of nodes: server nodes and agent nodes. Both servers and agents can have workloads scheduled on them. Server nodes run the Kubernetes master. + +For the cluster running the Rancher management server, we recommend using two server nodes. Agent nodes are not required. + +### RKE Cluster Roles + +If Rancher is installed on an RKE Kubernetes cluster, the cluster should have three nodes, and each node should have all three Kubernetes roles: etcd, controlplane, and worker. + +### Contrasting RKE Cluster Architecture for Rancher Server and for Downstream Kubernetes Clusters + +Our recommendation for RKE node roles on the Rancher server cluster contrasts with our recommendations for the downstream user clusters that run your apps and services. + +Rancher uses RKE as a library when provisioning downstream Kubernetes clusters. Note: The capability to provision downstream K3s clusters will be added in a future version of Rancher. + +For downstream Kubernetes clusters, we recommend that each node in a user cluster should have a single role for stability and scalability. + +![Kubernetes Roles for Nodes in Rancher Server Cluster vs. User Clusters](/img/rancher-architecture-node-roles.svg) + +RKE only requires at least one node with each role and does not require nodes to be restricted to one role. However, for the clusters that run your apps, we recommend separate roles for each node so that workloads on worker nodes don't interfere with the Kubernetes master or cluster data as your services scale. + +We recommend that downstream user clusters should have at least: + +- **Three nodes with only the etcd role** to maintain a quorum if one node is lost, making the state of your cluster highly available +- **Two nodes with only the controlplane role** to make the master component highly available +- **One or more nodes with only the worker role** to run the Kubernetes node components, as well as the workloads for your apps and services + +With that said, it is safe to use all three roles on three nodes when setting up the Rancher server because: + +* It allows one `etcd` node failure. +* It maintains multiple instances of the master components by having multiple `controlplane` nodes. +* No other workloads than Rancher itself should be created on this cluster. + +Because no additional workloads will be deployed on the Rancher server cluster, in most cases it is not necessary to use the same architecture that we recommend for the scalability and reliability of downstream clusters. + +For more best practices for downstream clusters, refer to the [production checklist](../../pages-for-subheaders/checklist-for-production-ready-clusters.md) or our [best practices guide.](../../pages-for-subheaders/best-practices.md) + +## Architecture for an Authorized Cluster Endpoint (ACE) + +If you are using an [authorized cluster endpoint (ACE),](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) we recommend creating an FQDN pointing to a load balancer which balances traffic across your nodes with the `controlplane` role. + +If you are using private CA signed certificates on the load balancer, you have to supply the CA certificate, which will be included in the generated kubeconfig file to validate the certificate chain. See the documentation on [kubeconfig files](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) and [API keys](../user-settings/api-keys.md#creating-an-api-key) for more information. + +ACE support is available for registered RKE2 and K3s clusters. To view the manual steps to perform on the downstream cluster to enable the ACE, click [here](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md#authorized-cluster-endpoint-support-for-rke2-and-k3s-clusters). diff --git a/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md b/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md new file mode 100644 index 00000000000..8abfd0c9f6c --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md @@ -0,0 +1,217 @@ +--- +title: Communicating with Downstream User Clusters +--- + + + + + +This section describes how Rancher provisions and manages the downstream user clusters that run your apps and services. + +The below diagram shows how the cluster controllers, cluster agents, and node agents allow Rancher to control downstream clusters. + +
Communicating with Downstream Clusters
+ +![Rancher Components](/img/rancher-architecture-cluster-controller.svg) + +The following descriptions correspond to the numbers in the diagram above: + +1. [The Authentication Proxy](#1-the-authentication-proxy) +2. [Cluster Controllers and Cluster Agents](#2-cluster-controllers-and-cluster-agents) +3. [Node Agents](#3-node-agents) +4. [Authorized Cluster Endpoint](#4-authorized-cluster-endpoint) + +### 1. The Authentication Proxy + +In this diagram, a user named Bob wants to see all pods running on a downstream user cluster called User Cluster 1. From within Rancher, he can run a `kubectl` command to see +the pods. Bob is authenticated through Rancher's authentication proxy. + +The authentication proxy forwards all Kubernetes API calls to downstream clusters. It integrates with authentication services like local authentication, Active Directory, and GitHub. On every Kubernetes API call, the authentication proxy authenticates the caller and sets the proper Kubernetes impersonation headers before forwarding the call to Kubernetes masters. + +Rancher communicates with Kubernetes clusters using a [service account](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/). Each user account in Rancher correlates with an equivalent service account in the downstream cluster. Rancher uses the service account to [impersonate](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation) the user, which provides all the permissions the user is intended to have. + +By default, Rancher generates a [kubeconfig file](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) that contains credentials for proxying through the Rancher server to connect to the Kubernetes API server on a downstream user cluster. The kubeconfig file (`kube_config_rancher-cluster.yml`) contains full access to the cluster. + +### 2. Cluster Controllers and Cluster Agents + +Each downstream user cluster has a cluster agent, which opens a tunnel to the corresponding cluster controller within the Rancher server. + +There is one cluster controller and one cluster agent for each downstream cluster. Each cluster controller: + +- Watches for resource changes in the downstream cluster +- Brings the current state of the downstream cluster to the desired state +- Configures access control policies to clusters and projects +- Provisions clusters by calling the required Docker machine drivers and Kubernetes engines, such as RKE and GKE + +By default, to enable Rancher to communicate with a downstream cluster, the cluster controller connects to the cluster agent. If the cluster agent is not available, the cluster controller can connect to a [node agent](#3-node-agents) instead. + +The cluster agent, also called `cattle-cluster-agent`, is a component that runs in a downstream user cluster. It performs the following tasks: + +- Connects to the Kubernetes API of Rancher-launched Kubernetes clusters +- Manages workloads, pod creation and deployment within each cluster +- Applies the roles and bindings defined in each cluster's global policies +- Communicates between the cluster and Rancher server (through a tunnel to the cluster controller) about events, stats, node info, and health + +### 3. Node Agents + +If the cluster agent (also called `cattle-cluster-agent`) is not available, one of the node agents creates a tunnel to the cluster controller to communicate with Rancher. + +The `cattle-node-agent` is deployed using a [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) resource to make sure it runs on every node in a Rancher-launched Kubernetes cluster. It is used to interact with the nodes when performing cluster operations. Examples of cluster operations include upgrading the Kubernetes version and creating or restoring etcd snapshots. + +### 4. Authorized Cluster Endpoint + +An authorized cluster endpoint (ACE) allows users to connect to the Kubernetes API server of a downstream cluster without having to route their requests through the Rancher authentication proxy. + +> ACE is available on RKE, RKE2, and K3s clusters that are provisioned or registered with Rancher. It's not available on clusters in a hosted Kubernetes provider, such as Amazon's EKS. + +There are two main reasons why a user might need the authorized cluster endpoint: + +- To access a downstream user cluster while Rancher is down +- To reduce latency in situations where the Rancher server and downstream cluster are separated by a long distance + +The `kube-api-auth` microservice is deployed to provide the user authentication functionality for the authorized cluster endpoint. When you access the user cluster using `kubectl`, the cluster's Kubernetes API server authenticates you by using the `kube-api-auth` service as a webhook. + +Like the authorized cluster endpoint, the `kube-api-auth` authentication service is also only available for Rancher-launched Kubernetes clusters. + +> **Example scenario:** Let's say that the Rancher server is located in the United States, and User Cluster 1 is located in Australia. A user, Alice, also lives in Australia. Alice can manipulate resources in User Cluster 1 by using the Rancher UI, but her requests will have to be sent from Australia to the Rancher server in the United States, then be proxied back to Australia, where the downstream user cluster is. The geographical distance may cause significant latency, which Alice can reduce by using the authorized cluster endpoint. + +With this endpoint enabled for the downstream cluster, Rancher generates an extra Kubernetes context in the kubeconfig file in order to connect directly to the cluster. This file has the credentials for `kubectl` and `helm`. + +You will need to use a context defined in this kubeconfig file to access the cluster if Rancher goes down. Therefore, we recommend exporting the kubeconfig file so that if Rancher goes down, you can still use the credentials in the file to access your cluster. For more information, refer to the section on accessing your cluster with [kubectl and the kubeconfig file.](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) + +## Impersonation + +Users technically exist only on the upstream cluster. Rancher creates [RoleBindings and ClusterRoleBindings](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#rolebinding-and-clusterrolebinding) that refer to Rancher users, even though there is [no actual User resource](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes) on the downstream cluster. + +When users interact with a downstream cluster through the authentication proxy, there needs to be some entity downstream to serve as the actor for those requests. Rancher creates service accounts to be that entity. Each service account is only granted one permission, which is to **impersonate** the user they belong to. If there was only one service account that could impersonate any user, then it would be possible for a malicious user to corrupt that account and escalate their privileges by impersonating another user. This issue was the basis for a [CVE](https://github.com/rancher/rancher/security/advisories/GHSA-pvxj-25m6-7vqr). + +### Impersonation Troubleshooting + +On the downstream cluster, five resources handle impersonation: + +- namespace: `cattle-impersonation-system` +- service account: `cattle-impersonation-system/cattle-impersonation-` +- account token secret: `cattle-impersonation-system/cattle-impersonation--token-` +- cluster role: `cattle-impersonation-` +- cluster role binding: `cattle-impersonation-` + +In this example of a typical impersonation cluster role, the system is configured to use `github` as the auth provider: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: "2021-10-06T18:20:13Z" + labels: + authz.cluster.cattle.io/impersonator: "true" + cattle.io/creator: norman + name: cattle-impersonation-user-abcde + resourceVersion: "3528" + uid: a7478731-72a0-4343-b09f-c3bf12552d77 +rules: +# allowed to impersonate user user-abcde +- apiGroups: + - "" + resourceNames: + - user-abcde + resources: + - users + verbs: + - impersonate +# allowed to impersonate listed groups +- apiGroups: + - "" + resourceNames: + - github_team://123 # group from GitHub auth provider + - system:authenticated # automatic group from Kubernetes + - system:cattle:authenticated # automatic group from Rancher + resources: + - groups + verbs: + - impersonate +# allowed to impersonate principal ID github_user://098 +- apiGroups: + - authentication.k8s.io + resourceNames: + - github_user://098 # principal ID from GitHub auth provider + resources: + - userextras/principalid + verbs: + - impersonate +# allowed to impersonate username example +- apiGroups: + - authentication.k8s.io + resourceNames: + - example # username from GitHub auth provider + resources: + - userextras/username + verbs: + - impersonate +``` + +When you troubleshoot impersonation issues, check whether these resources exist for the user, and whether the rules in the cluster role look similar to the above. For example: + +```bash +kubectl --namespace cattle-impersonation-system get serviceaccount cattle-impersonation- +kubectl --namespace cattle-impersonation-system get secret cattle-impersonation--token- +kubectl get clusterrole cattle-impersonation- --output yaml +kubectl get clusterrolebinding cattle-impersonation- +``` + +If you see an error related to "impersonation" in the UI, pay close attention to the *end* of the error message, which should indicate the real reason that the request failed. + +## Important Files + +The files mentioned below are needed to maintain, troubleshoot and upgrade your cluster: + +- `rancher-cluster.yml`: The RKE cluster configuration file. +- `kube_config_rancher-cluster.yml`: The Kubeconfig file for the cluster, this file contains credentials for full access to the cluster. You can use this file to authenticate with a Rancher-launched Kubernetes cluster if Rancher goes down. +- `rancher-cluster.rkestate`: The Kubernetes cluster state file. This file contains credentials for full access to the cluster. Note: This state file is only created when using RKE v0.2.0 or higher. + +> **Note:** The "rancher-cluster" parts of the two latter file names are dependent on how you name the RKE cluster configuration file. + +For more information on connecting to a cluster without the Rancher authentication proxy and other configuration options, refer to the [kubeconfig file](../../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) documentation. + +## Tools for Provisioning Kubernetes Clusters + +The tools that Rancher uses to provision downstream user clusters depends on the type of cluster that is being provisioned. + +### Rancher Launched Kubernetes for Nodes Hosted in an Infrastructure Provider + +Rancher can dynamically provision nodes in a provider such as Amazon EC2, DigitalOcean, Azure, or vSphere, then install Kubernetes on them. + +Rancher provisions this type of cluster using [RKE](https://github.com/rancher/rke) and [docker-machine.](https://github.com/rancher/machine) + +### Rancher Launched Kubernetes for Custom Nodes + +When setting up this type of cluster, Rancher installs Kubernetes on existing nodes, which creates a custom cluster. + +Rancher provisions this type of cluster using [RKE.](https://github.com/rancher/rke) + +### Hosted Kubernetes Providers + +When setting up this type of cluster, Kubernetes is installed by providers such as Google Kubernetes Engine, Amazon Elastic Container Service for Kubernetes, or Azure Kubernetes Service. + +Rancher provisions this type of cluster using [kontainer-engine.](https://github.com/rancher/kontainer-engine) + +### Imported Kubernetes Clusters + +In this type of cluster, Rancher connects to a Kubernetes cluster that has already been set up. Therefore, Rancher does not provision Kubernetes, but only sets up the Rancher agents to communicate with the cluster. + +## Rancher Server Components and Source Code + +This diagram shows each component that the Rancher server is composed of: + +![Rancher Components](/img/rancher-architecture-rancher-components.svg) + +The GitHub repositories for Rancher can be found at the following links: + +- [Main Rancher server repository](https://github.com/rancher/rancher) +- [Rancher UI](https://github.com/rancher/ui) +- [Rancher API UI](https://github.com/rancher/api-ui) +- [Norman,](https://github.com/rancher/norman) Rancher's API framework +- [Types](https://github.com/rancher/types) +- [Rancher CLI](https://github.com/rancher/cli) +- [Catalog applications](https://github.com/rancher/helm) + +This is a partial list of the most important Rancher repositories. For more details about Rancher source code, refer to the section on [contributing to Rancher.](../../contribute-to-rancher.md#rancher-repositories) To see all libraries and projects used in Rancher, see the [`go.mod` file](https://github.com/rancher/rancher/blob/master/go.mod) in the `rancher/rancher` repository. \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/rancher-server-and-components.md b/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/rancher-server-and-components.md new file mode 100644 index 00000000000..e9fec332622 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-manager-architecture/rancher-server-and-components.md @@ -0,0 +1,39 @@ +--- +title: Rancher Server and Components +--- + + + + + +The majority of Rancher 2.x software runs on the Rancher Server. Rancher Server includes all the software components used to manage the entire Rancher deployment. + +The figure below illustrates the high-level architecture of Rancher 2.x. The figure depicts a Rancher Server installation that manages two downstream Kubernetes clusters: one created by RKE and another created by Amazon EKS (Elastic Kubernetes Service). + +For the best performance and security, we recommend a dedicated Kubernetes cluster for the Rancher management server. Running user workloads on this cluster is not advised. After deploying Rancher, you can [create or import clusters](../../pages-for-subheaders/kubernetes-clusters-in-rancher-setup.md) for running your workloads. + +The diagram below shows how users can manipulate both [Rancher-launched Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters and [hosted Kubernetes](../../pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers.md) clusters through Rancher's authentication proxy: + +
Managing Kubernetes Clusters through Rancher's Authentication Proxy
+ +![Architecture](/img/rancher-architecture-rancher-api-server.svg) + +You can install Rancher on a single node, or on a high-availability Kubernetes cluster. + +A high-availability Kubernetes installation is recommended for production. + +A Docker installation of Rancher is recommended only for development and testing purposes. The ability to migrate Rancher to a high-availability cluster depends on the Rancher version. + +To see deployed resources, run the following command: + +```bash +kubectl get all -n +``` +If you have administrator privileges, you can also see these resources listed in the Rancher UI: + +1. Click **☰** and select a cluster. +1. Select **More Resources** from the side navigation menu to view deployed resources by type. +1. Select **Cluster** > **Projects/Namespaces** from the side navigation menu, then select a namespace, to view deployed resources by namespace. +The Rancher backup operator can be used to migrate Rancher from the single Docker container install to an installation on a high-availability Kubernetes cluster. For details, refer to the documentation on [migrating Rancher to a new cluster](../../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster.md). + +The Rancher server, regardless of the installation method, should always run on nodes that are separate from the downstream user clusters that it manages. If Rancher is installed on a high-availability Kubernetes cluster, it should run on a separate cluster from the cluster(s) it manages. diff --git a/versioned_docs/version-latest/reference-guides/rancher-project-tools.md b/versioned_docs/version-latest/reference-guides/rancher-project-tools.md new file mode 100644 index 00000000000..567cc5cf408 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-project-tools.md @@ -0,0 +1,32 @@ +--- +title: Project Tools for Logging, Monitoring, and Visibility +--- + +Rancher contains a variety of tools that aren't included in Kubernetes to assist in your DevOps operations. Rancher can integrate with external services to help your clusters run more efficiently. + + +## Notifiers and Alerts + +Notifiers and alerts are two features that work together to inform you of events in the Rancher system. Before they can be enabled, the monitoring application must be installed. + +Notifiers are services that inform you of alert events. You can configure notifiers to send alert notifications to staff best suited to take corrective action. Notifications can be sent with Slack, email, PagerDuty, WeChat, and webhooks. + +Alerts are rules that trigger those notifications. Before you can receive alerts, you must configure one or more notifier in Rancher. The scope for alerts can be set at either the cluster or project level. + +## Logging + +Logging is helpful because it allows you to: + +- Capture and analyze the state of your cluster +- Look for trends in your environment +- Save your logs to a safe location outside of your cluster +- Stay informed of events like a container crashing, a pod eviction, or a node dying +- More easily debugg and troubleshoot problems + +Rancher can integrate with Elasticsearch, splunk, kafka, syslog, and fluentd. + +For details, refer to the [logging section.](../pages-for-subheaders/logging.md) + +## Monitoring + +Using Rancher, you can monitor the state and processes of your cluster nodes, Kubernetes components, and software deployments through integration with [Prometheus](https://prometheus.io/), a leading open-source monitoring solution. For details, refer to the [monitoring section.](../pages-for-subheaders/monitoring-and-alerting.md) diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..d0ddba7d1e7 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3152 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 518 1 0 Feb26 ? 00:03:06 /usr/bin/containerd root 794 1 0 Feb26 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 94093 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7927fe0998264b8fa49789e8e2793a21ea8f22001cb580de46e1fcc01b1d1455 -address /run/k3s/containerd/containerd.sock root 94107 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ccec5bdaf56b4aad4018e8fefabfa103122905ecf4f908dbddaf9fbb985bc5c -address /run/k3s/containerd/containerd.sock root 94188 1 0 21:13 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d0f840cefd5b5ba804b2795420963e2c43a0c8b7b6b2c796e859c57ba0d38b3f -address /run/k3s/containerd/containerd.sock root 95349 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3849ee071d3f5a8e8a63ee1c4d590f88e0d21bc5fc25e20bbecb1e104b2835d0 -address /run/k3s/containerd/containerd.sock root 95421 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id a608e22cc1520272c91a2aca6a029e662a5d13defc7cf67de6a59594b859c550 -address /run/k3s/containerd/containerd.sock root 96569 1 0 21:14 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id f46f18b427a026156a41a54daad5c55a7d04bee1d112ae32e9c3d07c8cfc5a4e -address /run/k3s/containerd/containerd.sock root 97820 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id aa5731bfdbd6ff9c87043ccd4e30adc1c384eb8653b97169629598852931375d -address /run/k3s/containerd/containerd.sock root 97868 1 0 21:15 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 99f90b82253554453d0d1e57a23e770f000cf8972ca751387fd5682e9e1607f2 -address /run/k3s/containerd/containerd.sock root 100124 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 632f8fcf8174c12048354376e1f22e87ac95baa26a46e77f4cb8c1549f00fc4f -address /run/k3s/containerd/containerd.sock root 101049 1 0 21:31 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 05cc13a10b18601034393b12dcb0b140799cef2f3c7f712de671054cb65a4cff -address /run/k3s/containerd/containerd.sock root 105484 105441 2 21:41 ? 00:00:10 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 107618 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id d7baeb7e8d463b1afd831ca4367d4c552c8e87afc29f7cae7cb5bf987c2e4b1f -address /run/k3s/containerd/containerd.sock root 107751 1 0 21:50 ? 00:00:00 /var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin/containerd-shim-runc-v2 -namespace k8s.io -id 8fd7e8824474bb98e13d39dca0e130444b7b6fb917b7dc5e662c5a5f669fe584 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'644' is equal to '644' OR '640' is present OR '600' is present OR '444' is present OR '440' is present OR '400' is present OR '000' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:34 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:34Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Feb 27 21:41:36 ip-172-31-18-153 k3s[105441]: time="2023-02-27T21:41:36Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --cni-bin-dir=/var/lib/rancher/k3s/data/c3a9bd3231e69d1914e40f148c79b53e5081f1d9e26e35153bfd5d31ac7f6c85/bin --cni-conf-dir=/var/lib/rancher/k3s/agent/etc/cni/net.d --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-153 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=a53c5620-13b6-49f8-ae02-2e9bf44a3cf0 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..1d348069a2a --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3208 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +find /var/lib/rancher/k3s/server/tls | xargs stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 539 1 0 Sep11 ? 00:00:09 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 45213 45195 3 Sep11 ? 00:45:14 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 47188 1 0 Sep11 ? 00:01:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 02b8fdd94b7628d575ace92f337dcc93202151d07f33858341cfcb0178fea586 -address /run/k3s/containerd/containerd.sock root 47235 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id eb0f03c4bc1125ae34e5de77c7eaeb9c2c1e46bec0bb106b99281df7353f9ded -address /run/k3s/containerd/containerd.sock root 47948 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5eece7110bd3876414f890ed1b7158eb91f60d1f14450a9f8c77a0fc62b73c4a -address /run/k3s/containerd/containerd.sock root 48047 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4ef831f4bb8830a0a7b46a1e8be58197be00846ea0caa094014ea7ce3adf008e -address /run/k3s/containerd/containerd.sock root 48220 1 0 Sep11 ? 00:00:33 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id f8d5d3791ad2b1a8534476aa2fbdf8da1e4f0b9fdcdd26b1ae161142458a5336 -address /run/k3s/containerd/containerd.sock root 48878 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 41b93a6581e272b0852257b3d7c44ab4cadcdf465a3579a8b29b493ea75feaf7 -address /run/k3s/containerd/containerd.sock root 49870 1 0 Sep11 ? 00:00:30 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id be329cdccda4049f048c759a2b1915cd72526473ff7768e895bcfc153a66c125 -address /run/k3s/containerd/containerd.sock root 50271 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id cceb5d26eba7d7d3c474a2dc842b72495689f4e40055505ea2598e4ac6849d40 -address /run/k3s/containerd/containerd.sock root 50571 1 0 Sep11 ? 00:00:32 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3a26e691e0d0186165d523bef22b0af71f175c39df82845b32ce7b4e3d652d4a -address /run/k3s/containerd/containerd.sock root 96693 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9ed77e6e58410f1295f730ba4a8f8daf7aff8f4e896e24219a8fa710729c2287 -address /run/k3s/containerd/containerd.sock root 97726 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 1be4fe50e6bb29d4549f8b03102b5dc15b7669fe56b32beae3f701cf6d2e58cc -address /run/k3s/containerd/containerd.sock root 97923 1 1 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/484bf694b486d93cc93bcf90f74d5c77628550d4456f21760fd720b88c93881e/bin/containerd-shim-runc-v2 -namespace k8s.io -id 7c65c47376f36a8980a5bebad7ffdc1abe1853821cf6c4a1d45f1b6be3452062 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'ServiceAccount' +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-path' +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxage' +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxbackup' +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-log-maxsize' +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.23 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'request-timeout' +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-lookup' +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.25 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'service-account-key-file' +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'encryption-provider-config' +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +grep aescbc /path/to/encryption-config.json +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Manual) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** warn + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'600' is equal to '600' +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:31:58 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:31:58Z" level=info msg="Running kube-apiserver --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'protect-kernel-defaults' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--event-qps' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:32:02 ip-172-31-3-32 k3s[45195]: time="2023-09-11T20:32:02Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-3-32 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=cc5faddd-9c87-4a04-97c2-dcf4cbcbe10f --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 410 1 0 Sep11 ? 00:01:50 /usr/bin/containerd root 45213 45195 3 Sep11 ? 00:45:15 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..e7373288b35 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,3215 @@ +--- +title: K3s Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [K3s Hardening Guide](../../../../pages-for-subheaders/k3s-hardening-guide.md), which provides prescriptive guidance on how to harden K3s clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Each control in the CIS Kubernetes Benchmark was evaluated against a K3s cluster that was configured according to the accompanying hardening guide. + +Where control audits differ from the original CIS benchmark, the audit commands specific to K3s are provided for testing. + +These are the possible results for each control: + +- **Pass** - The K3s cluster passes the audit outlined in the benchmark. +- **Not Applicable** - The control is not applicable to K3s because of how it is designed to operate. The remediation section explains why. +- **Warn** - The control is manual in the CIS benchmark and it depends on the cluster's use-case or some other factor that must be determined by the cluster operator. These controls have been evaluated to ensure K3s doesn't prevent their implementation, but no further configuration or auditing of the cluster has been performed. + +This guide makes the assumption that K3s is running as a Systemd unit. Your installation may vary. Adjust the "audit" commands to fit your scenario. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 +Not Applicable. + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root +Not Applicable. + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.1.11 +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd +Not Applicable. + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/k3s/server/tls + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.crt + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.crt +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/pki/*.key + +**Audit:** + +```bash +stat -c %n %a /var/lib/rancher/k3s/server/tls/*.key +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'anonymous-auth' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'kubelet-certificate-authority' +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'authorization-mode' +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' is present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... +Permissive - Enabling Pod Security Policy can cause applications to unexpectedly fail. + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'enable-admission-plugins' +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'secure-port' +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log +Permissive. + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 +Permissive. + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 +Permissive. + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 +Permissive. + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s +Permissive. + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 1.2.29 +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep -A1 'Running kube-apiserver' | tail -n2 +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'client-ca-file' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-cafile' +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. +Permissive - Enabling encryption changes how data can be recovered as data is encrypted. + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'tls-cipher-suites' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'terminated-pod-gc-threshold' +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'profiling' +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'use-service-account-credentials' +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'service-account-private-key-file' +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-controller-manager' | tail -n1 | grep 'root-ca-file' +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/k3s/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/k3s/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/k3s/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/k3s/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/k3s/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/k3s/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --terminated-pod-gc-threshold=10 --use-service-account-credentials=true" +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Not Applicable. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep containerd | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 663 1 0 Sep11 ? 00:00:08 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd root 3021 1 0 Sep11 ? 00:00:29 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 4790d392966915d995e666002c56ed4cce6dd86f305ce2ee390547a1fcbf6c82 -address /run/k3s/containerd/containerd.sock root 3035 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 9d73e6a160ccde6c7c7d4ba0df3b9f696e3de2ebfc0f19a1dcbdf13aea496427 -address /run/k3s/containerd/containerd.sock root 3235 1 0 Sep11 ? 00:00:31 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 587f6221ee9f36c877231ada8d816a799dfda186332c33378d1eb16c72cdc87d -address /run/k3s/containerd/containerd.sock root 4435 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a74c9ec7d99785c2f2d4e6826aa80c22eb8b38249e8f99679ece00a818e9b7b3 -address /run/k3s/containerd/containerd.sock root 4985 1 0 Sep11 ? 00:00:53 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b0c9784dbe0fcbe8be10c857976e70ec84a208cb814e87b5ca085a02d434f8c -address /run/k3s/containerd/containerd.sock root 5056 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id f3231ff35f18056e74eda14907f296be15f7ea1c6ae5ab7904e27d4d18183301 -address /run/k3s/containerd/containerd.sock root 5868 1 0 Sep11 ? 00:00:27 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3e908c4d0b10df275bdef6f72fbcfa09517d11cf749236ad020364e14d77bc93 -address /run/k3s/containerd/containerd.sock root 6158 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id bec5780a5c73fa3154aa3b5ee26cdf23202db821205893e7e66ae17e6103e97b -address /run/k3s/containerd/containerd.sock root 7366 1 0 Sep11 ? 00:00:28 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id a81b78845bdcaef710314c93e5ea0d0617f37a8929472f7b570ab90c6667f57f -address /run/k3s/containerd/containerd.sock root 97274 1 0 16:13 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id c90652c935d9e79af45b9a9ac2b4fe315e2a761a0525737fdb95c680123a164c -address /run/k3s/containerd/containerd.sock root 98309 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 431a2763636488efd7d104c228d41f4f2ecd8c06a7fc375d8977ab0d238936a8 -address /run/k3s/containerd/containerd.sock root 98493 1 0 16:16 ? 00:00:00 /var/lib/rancher/k3s/data/3cdacaf539fc388d8e542a8d643948e3c7bfa4a7e91b7521102325e0ce8581b6/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5b480d2fa55c8cc0105ec1902887040c017f03d5b1eb6c73c24bb9d523ad9b37 -address /run/k3s/containerd/containerd.sock +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-scheduler' | tail -n1 | grep 'bind-address' +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-scheduler --authentication-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/k3s/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259" +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.1 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/server-client.key +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.2 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.3 +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.4 +``` + +**Expected Result**: + +```console +'cert-file' is present AND 'key-file' is present +``` + +**Returned Value**: + +```console +cert-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt key-file: /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.5 +``` + +**Expected Result**: + +```console +'--client-cert-auth' is present OR 'client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +client-cert-auth: true +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.6 +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). cat: /proc//environ: No such file or directory +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/k3s/server/db/etcd/config on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit Script:** `check_for_k3s_etcd.sh` + +```bash +#!/bin/bash + +# This script is used to ensure that k3s is actually running etcd (and not other databases like sqlite3) +# before it checks the requirement +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + + +if [[ "$(journalctl -D /var/log/journal -u k3s | grep 'Managed etcd cluster initializing' | grep -v grep | wc -l)" -gt 0 ]]; then + case $1 in + "1.1.11") + echo $(stat -c %a /var/lib/rancher/k3s/server/db/etcd);; + "1.2.29") + echo $(journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'etcd-');; + "2.1") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.2") + echo $(grep -A 5 'client-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.3") + echo $(grep 'auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.4") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep -E 'cert-file|key-file');; + "2.5") + echo $(grep -A 5 'peer-transport-security' /var/lib/rancher/k3s/server/db/etcd/config | grep 'client-cert-auth');; + "2.6") + echo $(grep 'peer-auto-tls' /var/lib/rancher/k3s/server/db/etcd/config);; + "2.7") + echo $(grep 'trusted-ca-file' /var/lib/rancher/k3s/server/db/etcd/config);; + esac +else +# If another database is running, return whatever is required to pass the scan + case $1 in + "1.1.11") + echo "700";; + "1.2.29") + echo "--etcd-certfile AND --etcd-keyfile";; + "2.1") + echo "cert-file AND key-file";; + "2.2") + echo "--client-cert-auth=true";; + "2.3") + echo "false";; + "2.4") + echo "peer-cert-file AND peer-key-file";; + "2.5") + echo "--client-cert-auth=true";; + "2.6") + echo "--peer-auto-tls=false";; + "2.7") + echo "--trusted-ca-file";; + esac +fi + +``` + +**Audit Execution:** + +```bash +./check_for_k3s_etcd.sh 2.7 +``` + +**Expected Result**: + +```console +'trusted-ca-file' is present +``` + +**Returned Value**: + +```console +trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt trusted-ca-file: /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Manual) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kube-apiserver' | tail -n1 | grep 'audit-policy-file' +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/k3s/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/k3s/server/cred/admin.kubeconfig; then stat -c permissions=%a /var/lib/rancher/k3s/server/cred/admin.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/k3s/server/cred/admin.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c %a /var/lib/rancher/k3s/server/tls/server-ca.crt +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/k3s/server/tls/client-ca.crt +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "anonymous-auth" | grep -v grep; else echo "--anonymous-auth=false"; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "authorization-mode" | grep -v grep; else echo "--authorization-mode=Webhook"; fi' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/sh -c 'if test $(journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | wc -l) -gt 0; then journalctl -D /var/log/journal -u k3s | grep "Running kube-apiserver" | tail -n1 | grep "client-ca-file" | grep -v grep; else echo "--client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt"; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +Sep 11 20:52:00 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:00Z" level=info msg="Running kube-apiserver --admission-control-config-file=/etc/rancher/k3s/config/rancher-psact.yaml --advertise-port=6443 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,k3s --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log --audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml --authorization-mode=Node,RBAC --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/k3s/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/k3s/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/k3s/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,ServiceAccount --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/k3s/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/k3s/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/k3s/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/k3s/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/k3s/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/k3s/server/tls/client-auth-proxy.key --request-timeout=300s --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/k3s/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6444 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/k3s/server/tls/service.key --service-account-lookup=true --service-account-signing-key-file=/var/lib/rancher/k3s/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/k3s/server/tls/serving-kube-apiserver.key" +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'read-only-port' +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'streaming-connection-idle-timeout' +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +journalctl -D /var/log/journal -u k3s | grep 'Running kubelet' | tail -n1 | grep 'make-iptables-util-chains' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +Sep 11 20:52:15 ip-172-31-12-34 k3s[2340]: time="2023-09-11T20:52:15Z" level=info msg="Running kubelet --address=0.0.0.0 --allowed-unsafe-sysctls=net.ipv4.ip_forward,net.ipv6.conf.all.forwarding --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/k3s/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-12-34 --kubeconfig=/var/lib/rancher/k3s/agent/kubelet.kubeconfig --make-iptables-util-chains=true --node-labels=cattle.io/os=linux,rke.cattle.io/machine=af02ecbc-1e4e-422e-8b4d-4b2aa24a9d46 --pod-infra-container-image=rancher/mirrored-pause:3.6 --pod-manifest-path=/var/lib/rancher/k3s/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --tls-cert-file=/var/lib/rancher/k3s/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/k3s/agent/serving-kubelet.key" +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is present OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** Not Applicable + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Permissive - When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/kubelet/config.yaml +``` + +**Expected Result**: + +```console +'RotateKubeletServerCertificate' is present OR 'RotateKubeletServerCertificate' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC containerd +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 527 1 0 Sep11 ? 00:01:28 /usr/bin/containerd root 2361 2340 3 Sep11 ? 00:40:19 containerd -c /var/lib/rancher/k3s/agent/etc/containerd/config.toml -a /run/k3s/containerd/containerd.sock --state /run/k3s/containerd --root /var/lib/rancher/k3s/agent/containerd +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..9c6d1369a90 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3087 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chmod -R 644 /etc/kubernetes/pki/*.crt + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/!(*key).pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5411 5390 2 21:34 ? 00:00:06 kube-controller-manager --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --allow-untagged-cloud=true --leader-elect=true --terminated-pod-gc-threshold=1000 --allocate-node-cidrs=true --node-monitor-grace-period=40s --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --profiling=false --configure-cloud-routes=false --enable-hostpath-provisioner=false --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --address=0.0.0.0 --v=2 --cloud-provider= --pod-eviction-timeout=5m0s --cluster-cidr=10.42.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 5564 5543 0 21:34 ? 00:00:01 kube-scheduler --v=2 --address=0.0.0.0 --leader-elect=true --profiling=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 5 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 3 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-26-226 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem ETCDCTL_ENDPOINTS=https://172.31.26.226:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 5096 5076 3 21:34 ? 00:00:08 /usr/local/bin/etcd --initial-cluster=etcd-ip-172-31-26-226=https://172.31.26.226:2380 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226.pem --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-26-226 --listen-client-urls=https://172.31.26.226:2379 --listen-peer-urls=https://172.31.26.226:2380 --client-cert-auth=true --election-timeout=5000 --initial-advertise-peer-urls=https://172.31.26.226:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --initial-cluster-state=new --advertise-client-urls=https://172.31.26.226:2379 --heartbeat-interval=500 --peer-client-cert-auth=true --data-dir=/var/lib/rancher/etcd/ --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-26-226-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 5258 5238 18 21:34 ? 00:00:40 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 root 17615 17594 2 21:38 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.23-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 5258 5238 17 21:34 ? 00:00:41 kube-apiserver --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --api-audiences=unknown --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --requestheader-group-headers=X-Remote-Group --advertise-address=172.31.26.226 --audit-log-format=json --requestheader-allowed-names=kube-apiserver-proxy-client --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --anonymous-auth=false --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --secure-port=6443 --audit-log-path=/var/log/kube-audit/audit-log.json --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --etcd-prefix=/registry --etcd-servers=https://172.31.26.226:2379 --insecure-port=0 --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --audit-log-maxsize=100 --cloud-provider= --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-cache-ttl=5s --profiling=false --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --storage-backend=etcd3 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --authorization-mode=Node,RBAC --admission-control-config-file=/etc/kubernetes/admission.yaml --service-node-port-range=30000-32767 --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-issuer=rke --allow-privileged=true --service-account-lookup=true --audit-log-maxbackup=10 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-cluster-ip-range=10.43.0.0/16 +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml' is not present +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 5730 5709 4 21:34 ? 00:00:08 kubelet --pod-infra-container-image=rancher/mirrored-pause:3.6 --network-plugin=cni --protect-kernel-defaults=true --fail-swap-on=false --hostname-override=ip-172-31-26-226 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cni-conf-dir=/etc/cni/net.d --anonymous-auth=false --streaming-connection-idle-timeout=30m --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --cloud-provider= --cluster-domain=cluster.local --resolv-conf=/etc/resolv.conf --make-iptables-util-chains=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --root-dir=/var/lib/kubelet --read-only-port=0 --v=2 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --feature-gates=RotateKubeletServerCertificate=true --cgroups-per-qos=True --cluster-dns=10.43.0.10 --cni-bin-dir=/opt/cni/bin --address=0.0.0.0 --node-ip=172.31.26.226 --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-26-226-key.pem --event-qps=0 --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): replicationcontrollers is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicationcontrollers" in API group "" in the namespace "default" Error from server (Forbidden): services is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "services" in API group "" in the namespace "default" Error from server (Forbidden): daemonsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "daemonsets" in API group "apps" in the namespace "default" Error from server (Forbidden): deployments.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "deployments" in API group "apps" in the namespace "default" Error from server (Forbidden): replicasets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "replicasets" in API group "apps" in the namespace "default" Error from server (Forbidden): statefulsets.apps is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "statefulsets" in API group "apps" in the namespace "default" Error from server (Forbidden): horizontalpodautoscalers.autoscaling is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "horizontalpodautoscalers" in API group "autoscaling" in the namespace "default" Error from server (Forbidden): cronjobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "cronjobs" in API group "batch" in the namespace "default" Error from server (Forbidden): jobs.batch is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "jobs" in API group "batch" in the namespace "default" true +``` diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..5f3c20fb37f --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3047 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for etcd. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the master node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /etc/kubernetes/ssl/*key.pem + +**Audit Script:** `check_files_permissions.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the file permissions are set to 644 or +# more restrictive for all files in a given directory or a wildcard +# selection of files +# +# inputs: +# $1 = /full/path/to/directory or /path/to/fileswithpattern +# ex: !(*key).pem +# +# $2 (optional) = permission (ex: 600) +# +# outputs: +# true/false + +# Turn on "extended glob" for use of '!' in wildcard +shopt -s extglob + +# Turn off history to avoid surprises when using '!' +set -H + +USER_INPUT=$1 + +if [[ "${USER_INPUT}" == "" ]]; then + echo "false" + exit +fi + + +if [[ -d ${USER_INPUT} ]]; then + PATTERN="${USER_INPUT}/*" +else + PATTERN="${USER_INPUT}" +fi + +PERMISSION="" +if [[ "$2" != "" ]]; then + PERMISSION=$2 +fi + +FILES_PERMISSIONS=$(stat -c %n\ %a ${PATTERN}) + +while read -r fileInfo; do + p=$(echo ${fileInfo} | cut -d' ' -f2) + + if [[ "${PERMISSION}" != "" ]]; then + if [[ "$p" != "${PERMISSION}" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "644" && "$p" != "640" && "$p" != "600" ]]; then + echo "false" + exit + fi + fi +done <<< "${FILES_PERMISSIONS}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_permissions.sh '/node/etc/kubernetes/ssl/*key.pem' +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 1.2.31 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit Script:** `check_encryption_provider_config.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to check the encrption provider config is set to aesbc +# +# outputs: +# true/false + +# TODO: Figure out the file location from the kube-apiserver commandline args +ENCRYPTION_CONFIG_FILE="/node/etc/kubernetes/ssl/encryption.yaml" + +if [[ ! -f "${ENCRYPTION_CONFIG_FILE}" ]]; then + echo "false" + exit +fi + +for provider in "$@" +do + if grep "$provider" "${ENCRYPTION_CONFIG_FILE}"; then + echo "true" + exit + fi +done + +echo "false" +exit + +``` + +**Audit Execution:** + +```bash +./check_encryption_provider_config.sh aescbc +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +- aescbc: true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3690 3671 1 Sep11 ? 00:20:42 kube-controller-manager --service-cluster-ip-range=10.43.0.0/16 --configure-cloud-routes=false --enable-hostpath-provisioner=false --terminated-pod-gc-threshold=1000 --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --v=2 --pod-eviction-timeout=5m0s --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --cloud-provider= --cluster-cidr=10.42.0.0/16 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --feature-gates=RotateKubeletServerCertificate=true --leader-elect=true --profiling=false --node-monitor-grace-period=40s --allow-untagged-cloud=true --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3859 3838 0 Sep11 ? 00:03:44 kube-scheduler --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --leader-elect=true --profiling=false --v=2 --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 5 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:08 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 4 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-7-100 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3369 3348 2 Sep11 ? 00:26:05 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-cluster-token=etcd-cluster-1 --initial-cluster=etcd-ip-172-31-7-100=https://172.31.7.100:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --client-cert-auth=true --election-timeout=5000 --name=etcd-ip-172-31-7-100 --listen-client-urls=https://0.0.0.0:2379 --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-state=new --advertise-client-urls=https://172.31.7.100:2379 --heartbeat-interval=500 --initial-advertise-peer-urls=https://172.31.7.100:2380 --listen-peer-urls=https://0.0.0.0:2380 --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-7-100-key.pem --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 root 3528 3509 7 Sep11 ? 01:24:09 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s root 1057543 1057522 2 16:15 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.24-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 3528 3509 7 Sep11 ? 01:24:10 kube-apiserver --service-cluster-ip-range=10.43.0.0/16 --anonymous-auth=false --profiling=false --advertise-address=172.31.7.100 --audit-log-maxsize=100 --cloud-provider= --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --etcd-prefix=/registry --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-username-headers=X-Remote-User --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --audit-policy-file=/etc/kubernetes/audit-policy.yaml --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --storage-backend=etcd3 --service-node-port-range=30000-32767 --bind-address=0.0.0.0 --api-audiences=unknown --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --runtime-config=policy/v1beta1/podsecuritypolicy=true --allow-privileged=true --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,PodSecurityPolicy,EventRateLimit --service-account-issuer=rke --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-format=json --authorization-mode=Node,RBAC --etcd-servers=https://172.31.7.100:2379 --service-account-lookup=true --secure-port=6443 --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --audit-log-maxage=30 --admission-control-config-file=/etc/kubernetes/admission.yaml --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --audit-log-maxbackup=10 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --authentication-token-webhook-cache-ttl=5s +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Cluster provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml + +Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is equal to '0' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4429 4031 3 Sep11 ? 00:38:25 kubelet --read-only-port=0 --event-qps=0 --root-dir=/var/lib/kubelet --pod-infra-container-image=rancher/mirrored-pause:3.7 --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100-key.pem --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --cloud-provider= --address=0.0.0.0 --protect-kernel-defaults=true --hostname-override=ip-172-31-7-100 --resolv-conf=/etc/resolv.conf --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-7-100.pem --make-iptables-util-chains=true --feature-gates=RotateKubeletServerCertificate=true --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --container-runtime=remote --v=2 --anonymous-auth=false --authentication-token-webhook=true --fail-swap-on=false --cgroups-per-qos=True --authorization-mode=Webhook --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "ingress-nginx" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): roles.rbac.authorization.k8s.io "default-psp-role" is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=1 +``` + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in $(kubectl get namespaces --all-namespaces -o json | jq -r '.items[].metadata.name'); do + policy_count=$(kubectl get networkpolicy -n ${namespace} -o json | jq '.items | length') + if [[ ${policy_count} -eq 0 ]]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Automated) + + +**Result:** pass + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + +**Audit Script:** `check_for_default_ns.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count=$(kubectl get all -n default -o json | jq .items[] | jq -r 'select((.metadata.name!="kubernetes"))' | jq .metadata.name | wc -l) +if [[ ${count} -gt 0 ]]; then + echo "false" + exit +fi + +echo "true" + + +``` + +**Audit Execution:** + +```bash +./check_for_default_ns.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..9fe9ca2045f --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2863 @@ +--- +title: RKE Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE Hardening Guide](../../../../pages-for-subheaders/rke1-hardening-guide.md), which provides prescriptive guidance on how to harden RKE clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +Rancher and RKE install Kubernetes services via Docker containers. Configuration is defined by arguments passed to the container at the time of initialization, not via configuration files. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-apiserver.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-controller-manager.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/manifests/kube-scheduler.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /etc/kubernetes/manifests/etcd.yaml +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for kube-apiserver. +All configuration is passed in as arguments at container run time. + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +'permissions' is present +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -ef | grep kubelet | grep -- --cni-conf-dir | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c %a /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'700' is equal to '700' +``` + +**Returned Value**: + +```console +700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /node/var/lib/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf +Not Applicable - Cluster provisioned by RKE does not store the kubernetes default kubeconfig credentials file on the nodes. + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for scheduler. +All configuration is passed in as arguments at container run time. + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root controllermanager +Not Applicable - Cluster provisioned by RKE doesn't require or maintain a configuration file for controller-manager. +All configuration is passed in as arguments at container run time. + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit Script:** `check_files_owner_in_dir.sh` + +```bash +#!/usr/bin/env bash + +# This script is used to ensure the owner is set to root:root for +# the given directory and all the files in it +# +# inputs: +# $1 = /full/path/to/directory +# +# outputs: +# true/false + +INPUT_DIR=$1 + +if [[ "${INPUT_DIR}" == "" ]]; then + echo "false" + exit +fi + +if [[ $(stat -c %U:%G ${INPUT_DIR}) != "root:root" ]]; then + echo "false" + exit +fi + +statInfoLines=$(stat -c "%n %U:%G" ${INPUT_DIR}/*) +while read -r statInfoLine; do + f=$(echo ${statInfoLine} | cut -d' ' -f1) + p=$(echo ${statInfoLine} | cut -d' ' -f2) + + if [[ $(basename "$f" .pem) == "kube-etcd-"* ]]; then + if [[ "$p" != "root:root" && "$p" != "etcd:etcd" ]]; then + echo "false" + exit + fi + else + if [[ "$p" != "root:root" ]]; then + echo "false" + exit + fi + fi +done <<< "${statInfoLines}" + + +echo "true" +exit + +``` + +**Audit Execution:** + +```bash +./check_files_owner_in_dir.sh /node/etc/kubernetes/ssl +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*.pem' ! -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=644 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +find /node/etc/kubernetes/ssl/ -name '*key.pem' -exec chmod -R 600 {} + + +**Audit:** + +```bash +find /node/etc/kubernetes/ssl/ -name '*key.pem' | xargs stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/etc/kubernetes/manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= +When generating serving certificates, functionality could break in conjunction with hostname overrides which are required for certain cloud providers. + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is equal to 'true' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384' +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true +Cluster provisioned by RKE handles certificate rotation directly through RKE. + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /etc/kubernetes/manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4184 4163 1 Sep11 ? 00:20:06 kube-controller-manager --configure-cloud-routes=false --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --allocate-node-cidrs=true --enable-hostpath-provisioner=false --pod-eviction-timeout=5m0s --terminated-pod-gc-threshold=1000 --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --service-cluster-ip-range=10.43.0.0/16 --cluster-cidr=10.42.0.0/16 --root-ca-file=/etc/kubernetes/ssl/kube-ca.pem --service-account-private-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --node-monitor-grace-period=40s --v=2 --profiling=false --cloud-provider= --allow-untagged-cloud=true --leader-elect=true --feature-gates=RotateKubeletServerCertificate=true --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-controller-manager.yaml --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /etc/kubernetes/manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is present OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 4339 4318 0 Sep11 ? 00:03:28 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --authorization-kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-scheduler.yaml --profiling=false --v=2 --leader-elect=true +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--cert-file' is present AND '--key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-cert-file' is present AND '--peer-key-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 2 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--peer-client-cert-auth' is equal to 'true' +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-4-224 ETCDCTL_API=3 ETCDCTL_CACERT=/etc/kubernetes/ssl/kube-ca.pem ETCDCTL_CERT=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem ETCDCTL_KEY=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 ETCD_UNSUPPORTED_ARCH=x86_64 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'--trusted-ca-file' is present +``` + +**Returned Value**: + +```console +etcd 3847 3824 2 Sep11 ? 00:29:36 /usr/local/bin/etcd --peer-client-cert-auth=true --initial-advertise-peer-urls=https://172.31.4.224:2380 --initial-cluster=etcd-ip-172-31-4-224=https://172.31.4.224:2380 --initial-cluster-state=new --trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --client-cert-auth=true --heartbeat-interval=500 --listen-client-urls=https://0.0.0.0:2379 --peer-trusted-ca-file=/etc/kubernetes/ssl/kube-ca.pem --listen-peer-urls=https://0.0.0.0:2380 --cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --peer-cert-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224.pem --peer-key-file=/etc/kubernetes/ssl/kube-etcd-172-31-4-224-key.pem --data-dir=/var/lib/rancher/etcd/ --initial-cluster-token=etcd-cluster-1 --name=etcd-ip-172-31-4-224 --advertise-client-urls=https://172.31.4.224:2379 --cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 --election-timeout=5000 root 4018 3998 5 Sep11 ? 01:03:21 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml root 1034677 1034607 1 16:16 ? 00:00:00 kube-bench run --targets etcd --scored --nosummary --noremediations --v=0 --config-dir=/etc/kube-bench/cfg --benchmark rke-cis-1.7-hardened --json --log_dir /tmp/sonobuoy/logs --outputfile /tmp/sonobuoy/etcd.json +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 4018 3998 5 Sep11 ? 01:03:22 kube-apiserver --advertise-address=172.31.4.224 --audit-log-path=/var/log/kube-audit/audit-log.json --audit-log-maxbackup=10 --requestheader-allowed-names=kube-apiserver-proxy-client --service-cluster-ip-range=10.43.0.0/16 --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize,EventRateLimit --requestheader-extra-headers-prefix=X-Remote-Extra- --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --storage-backend=etcd3 --anonymous-auth=false --bind-address=0.0.0.0 --cloud-provider= --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --service-node-port-range=30000-32767 --profiling=false --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --encryption-provider-config=/etc/kubernetes/ssl/encryption.yaml --runtime-config=authorization.k8s.io/v1beta1=true --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --service-account-lookup=true --etcd-servers=https://172.31.4.224:2379 --api-audiences=unknown --requestheader-group-headers=X-Remote-Group --service-account-issuer=rke --audit-log-maxsize=100 --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --secure-port=6443 --service-account-signing-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --authorization-mode=Node,RBAC --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --audit-log-maxage=30 --audit-log-format=json --etcd-prefix=/registry --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-config-file=/etc/kubernetes/kube-api-authn-webhook.yaml --kubelet-certificate-authority=/etc/kubernetes/ssl/kube-ca.pem --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --authentication-token-webhook-cache-ttl=5s --admission-control-config-file=/etc/kubernetes/admission.yaml --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --requestheader-username-headers=X-Remote-User --allow-privileged=true --audit-policy-file=/etc/kubernetes/audit-policy.yaml +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. +All configuration is passed in as arguments at container run time. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet service. + All configuration is passed in as arguments at container run time. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-proxy.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c permissions=%a /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; then stat -c %U:%G /node/etc/kubernetes/ssl/kubecfg-kube-node.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit:** + +```bash +stat -c permissions=%a /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit:** + +```bash +stat -c %U:%G /node/etc/kubernetes/ssl/kube-ca.pem +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE do not require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/kubelet/config.yaml +Not Applicable - Clusters provisioned by RKE doesn’t require or maintain a configuration file for the kubelet. +All configuration is passed in as arguments at container run time. + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--streaming-connection-idle-timeout' is not equal to '0' OR '--streaming-connection-idle-timeout' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--make-iptables-util-chains' is equal to 'true' OR '--make-iptables-util-chains' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE set the --hostname-override to avoid any hostname configuration errors + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--event-qps' is greater or equal to 0 OR '--event-qps' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--rotate-certificates' is present OR '--rotate-certificates' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable - Clusters provisioned by RKE handles certificate rotation directly through RKE. + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--tls-cipher-suites' contains valid elements from 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/sh -c 'if test -e /var/lib/kubelet/config.yaml; then /bin/cat /var/lib/kubelet/config.yaml; fi' +``` + +**Expected Result**: + +```console +'--pod-max-pids' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 4903 4499 3 Sep11 ? 00:36:52 kubelet --v=2 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224-key.pem --event-qps=0 --address=0.0.0.0 --cgroups-per-qos=True --pod-infra-container-image=rancher/mirrored-pause:3.7 --root-dir=/var/lib/kubelet --container-runtime=remote --make-iptables-util-chains=true --authorization-mode=Webhook --resolv-conf=/etc/resolv.conf --cloud-provider= --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 --authentication-token-webhook=true --anonymous-auth=false --read-only-port=0 --volume-plugin-dir=/var/lib/kubelet/volumeplugins --protect-kernel-defaults=true --feature-gates=RotateKubeletServerCertificate=true --cluster-dns=10.43.0.10 --fail-swap-on=false --hostname-override=ip-172-31-4-224 --kubeconfig=/etc/kubernetes/ssl/kubecfg-kube-node.yaml --cluster-domain=cluster.local --tls-cert-file=/etc/kubernetes/ssl/kube-kubelet-172-31-4-224.pem --streaming-connection-idle-timeout=30m --cgroup-driver=cgroupfs --resolv-conf=/run/systemd/resolve/resolv.conf +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md new file mode 100644 index 00000000000..3099612cacb --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23.md @@ -0,0 +1,3200 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.23 - K8s v1.23 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.23 | Kubernetes v1.23 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.23. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Master Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 644 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 3707 3642 17 23:48 ? 00:01:53 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --insecure-port=0 --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"zF6yWmUNDvLZW5QnHG580wVBYOwq9S+VXJ3G4JjDaEk="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3887 3772 2 23:48 ? 00:00:13 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 3872 3747 0 23:48 ? 00:00:03 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. +--auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-18-103 ETCD_UNSUPPORTED_ARCH= FILE_HASH=57978886cc67e0107d32d86955fc94bb4b76cba8a712bc69f19a3708dd4302a3 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=f6bd3035e1aec68ad6bd4411b8a9bfe7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. + For most requests, minimally logging at the Metadata level is recommended + (the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 644 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 644 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present OR '/var/lib/rancher/rke2/agent/kubeproxy.kubeconfig' is not present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 644 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 644 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 Ensure that the kubelet --config configuration file has permissions set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 644 /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c permissions=%a /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.10 Ensure that the kubelet --config configuration file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /etc/rancher/rke2/rke2.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /etc/rancher/rke2/rke2.yaml; then stat -c %U:%G /etc/rancher/rke2/rke2.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Ensure that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the --event-qps argument is set to 0 or a level which ensures appropriate event capture (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 3383 3288 3 23:47 ? 00:00:23 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime=remote --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-18-103 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=36a58c5b-e767-494b-b311-dfbb86d1de88 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /etc/rancher/rke2/rke2.yaml +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlRENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TmpjM05EVTFNakk1TUI0WERUSXpNREl5TmpJek5EY3dPVm9YRFRNek1ESXlNekl6TkRjdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRZM056UTFOVEl5T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJHL2oraitwN1FRNElpd2NQc2gzclE2Ny9rUXNxTjZsUjgzbDkxaloKZ1FiVlhRbUwvdkRvcGxvNHIvaXQ1bGVrSE9PMGNkQXlXMWtxMlNDVmxDZExHZitqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUWxYWWkrK0pTMzBudjVnUE43CklEYmNrMEpDWERBS0JnZ3Foa2pPUFFRREFnTkhBREJFQWlCd3dId1o0cndrOUxWMVZUTmxGZFZQOVNBOGQ1WmUKTllaajZIc0JnZGV0bWdJZ0JlOCtBZnpMTzAydkJTQVVnSE40RkFHQ1BtUGhMLy9aeXpBaU02cDExSU09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K server: https://127.0.0.1:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrekNDQVRpZ0F3SUJBZ0lJRUNQa2R1SWEvQjR3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVFkzTnpRMU5USXlPVEFlRncweU16QXlNall5TXpRM01EbGFGdzB5TkRBeQpNall5TXpRM01EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBUURnUXQxSDQwbTRlSHQKZnBvMWhwTklhb1A4cWRJb0JzYmlTOUhxYlhlRlRwV3NnM21QNm40ZGN4dTdrN29NcVI5QmdNelpzRVV3Z2E0TgpUY1dJS0QwUG8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVrM29Ub1ArV2ZmSDR0cjNVVnB6dGtBNm5PZEF3Q2dZSUtvWkl6ajBFQXdJRFNRQXcKUmdJaEFMVHRRV0lUSExrVWRtVFlUZXIxOFNhRTU3d0tIditTMUVCMFVBV3U5NDloQWlFQWl6TklCU2wzRDIzbQoxOHcrbkxKM2tIOUdISXFVN2l3YTRXQkw4M21nT05FPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZURDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE5qYzNORFUxTWpJNU1CNFhEVEl6TURJeU5qSXpORGN3T1ZvWERUTXpNREl5TXpJek5EY3cKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UWTNOelExTlRJeU9UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCQ3Yza01hdUIzUzlBYVNiNldnK3E4cGZBL09HeTQzZ2NSQWJpMS9tCi9oblhNY2lhem5rNTFmZ1lTejhoc09hM2FwQS9zRFNzZVNiN3dDVXZoNUdydWVxalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlNUZWhPZy81Wjk4ZmkydmRSVwpuTzJRRHFjNTBEQUtCZ2dxaGtqT1BRUURBZ05IQURCRUFpQXlSU0VoUzdlNHAzT3BSZ2dVdjZwQ0NYY1dHakJoCjJaSGxRNWhIVDhBekpnSWdSdi9DUFA3QUNyWWlJL3BoVG1OVlRwdTVtekhubkhjS0pMeG8wNHVwU3FVPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUovNlAyKzlyU1A5QjR4VnpHQlhZT0FBTEE5ZytKK2ZaUWViVEQ3eU5YVnRvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQTRFTGRSK05KdUhoN1g2YU5ZYVRTR3FEL0tuU0tBYkc0a3ZSNm0xM2hVNlZySU41aitwKwpIWE1idTVPNkRLa2ZRWURNMmJCRk1JR3VEVTNGaUNnOUR3PT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[].kind=="ServiceAccount" and .subjects[].name=="default") or (.subjects[].kind=="Group" and .subjects[].name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[] != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +Error from server (Forbidden): serviceaccounts is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "serviceaccounts" in API group "" at the cluster scope Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "calico-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-fleet-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-impersonation-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cattle-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "cis-operator-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "default" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-node-lease" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-public" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "kube-system" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "local" Error from server (Forbidden): clusterrolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "clusterrolebindings" in API group "rbac.authorization.k8s.io" at the cluster scope Error from server (Forbidden): rolebindings.rbac.authorization.k8s.io is forbidden: User "system:serviceaccount:cis-operator-system:cis-serviceaccount" cannot list resource "rolebindings" in API group "rbac.authorization.k8s.io" in the namespace "tigera-operator" true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r '.spec.runAsUser.rule' +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is present +``` + +**Returned Value**: + +```console +MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is present +``` + +**Returned Value**: + +```console +ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is present +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: +securityContext: +seccompProfile: +type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md new file mode 100644 index 00000000000..ba80c0a516a --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24.md @@ -0,0 +1,3202 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.24 - K8s v1.24 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.24 | Kubernetes v1.24 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.24. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 644 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 644 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 644 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'644' is equal to '644' +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +'etcd:etcd' is present +``` + +**Returned Value**: + +```console +etcd:etcd +``` + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 scheduler + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root scheduler + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 controllermanager + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /var/lib/rancher/rke2/server/tls/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Automated) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 644 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +'permissions' is equal to '600' +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-https argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --kubelet-https parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-https' is present OR '--kubelet-https' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.14 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --secure-port argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.22 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxsize' is greater or equal to 100 +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1017917 2489 83 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz root 1017945 2489 75 16:16 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/livez +``` + +### 1.2.26 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that the --encryption-provider-config argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2548 2489 10 Sep11 ? 02:10:01 kube-apiserver --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction,PodSecurityPolicy --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.32 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 + +### 1.2.33 Ensure that encryption providers are appropriately configured (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +/bin/sh -c 'if grep aescbc /var/lib/rancher/rke2/server/cred/encryption-config.json; then echo 0; fi' +``` + +**Expected Result**: + +```console +'0' is present +``` + +**Returned Value**: + +```console +{"kind":"EncryptionConfiguration","apiVersion":"apiserver.config.k8s.io/v1","resources":[{"resources":["secrets"],"providers":[{"aescbc":{"keys":[{"name":"aescbckey","secret":"akmMjUAq94YvsUftJpiA3b+9SClu0ESPBeckAI7KZBY="}]}},{"identity":{}}]}]} 0 +``` + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2743 2649 2 Sep11 ? 00:28:36 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2707 2593 0 Sep11 ? 00:06:20 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-10-113 ETCD_UNSUPPORTED_ARCH= NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=560b915d9afb672c20c2c1e8664bdf8f FILE_HASH=166aebdce42ff62fcdd2cefc9e7ed8f7c5b562d219ca6afec8f73adc654f65e7 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep | grep -o audit-policy-file +``` + +**Expected Result**: + +```console +'audit-policy-file' is equal to 'audit-policy-file' +``` + +**Returned Value**: + +```console +audit-policy-file +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --protect-kernel-defaults argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `protectKernelDefaults` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--protect-kernel-defaults=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--protect-kernel-defaults' is equal to 'true' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.7 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.8 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +### 4.2.9 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Automated) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.10 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2291 2246 4 Sep11 ? 00:50:01 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-10-113 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=6685035f-32be-4d1b-a06d-7ea5f42467f5 --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.11 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Automated) + + +**Result:** pass + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +**Audit Script:** `check_for_default_sa.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +count_sa=$(kubectl get serviceaccounts --all-namespaces -o json | jq -r '.items[] | select(.metadata.name=="default") | select((.automountServiceAccountToken == null) or (.automountServiceAccountToken == true))' | jq .metadata.namespace | wc -l) +if [[ ${count_sa} -gt 0 ]]; then + echo "false" + exit +fi + +for ns in $(kubectl get ns --no-headers -o custom-columns=":metadata.name") +do + for result in $(kubectl get clusterrolebinding,rolebinding -n $ns -o json | jq -r '.items[] | select((.subjects[]?.kind=="ServiceAccount" and .subjects[]?.name=="default") or (.subjects[]?.kind=="Group" and .subjects[]?.name=="system:serviceaccounts"))' | jq -r '"\(.roleRef.kind),\(.roleRef.name)"') + do + read kind name <<<$(IFS=","; echo $result) + resource_count=$(kubectl get $kind $name -n $ns -o json | jq -r '.rules[] | select(.resources[]? != "podsecuritypolicies")' | wc -l) + if [[ ${resource_count} -gt 0 ]]; then + echo "false" + exit + fi + done +done + + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_default_sa.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r ".spec.runAsUser.rule" || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r ".spec.runAsUser.rule" +``` + +**Expected Result**: + +```console +'MustRunAsNonRoot' is equal to 'MustRunAsNonRoot' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ MustRunAsNonRoot +``` + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostPID == null) or (.spec.hostPID == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostIPC == null) or (.spec.hostIPC == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=5 +``` + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.hostNetwork == null) or (.spec.hostNetwork == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=2 +``` + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.7 Minimize the admission of root containers (Automated) + + +**Result:** pass + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +**Audit:** + +```bash +kubectl get psp -o json | jq .items[] | jq -r 'select((.spec.allowPrivilegeEscalation == null) or (.spec.allowPrivilegeEscalation == false))' | jq .metadata.name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ --count=4 +``` + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Automated) + + +**Result:** pass + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +**Audit:** + +```bash +kubectl get psp global-restricted-psp && kubectl get psp global-restricted-psp -o json | jq -r .spec.requiredDropCapabilities[] || kubectl get psp restricted-noroot-psp && kubectl get psp restricted-noroot-psp -o json | jq -r .spec.requiredDropCapabilities[] +``` + +**Expected Result**: + +```console +'ALL' is equal to 'ALL' +``` + +**Returned Value**: + +```console +Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ Error from server (NotFound): podsecuritypolicies.policy "global-restricted-psp" not found Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES restricted-noroot-psp false RunAsAny MustRunAsNonRoot MustRunAs MustRunAs false configMap,emptyDir,projected,secret,downwardAPI,csi,persistentVolumeClaim Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ ALL +``` + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports Network Policies (Automated) + + +**Result:** pass + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +**Audit:** + +```bash +kubectl get pods --all-namespaces --selector='k8s-app in (calico-node, canal, cilium)' -o name | wc -l | xargs -I {} echo '--count={}' +``` + +**Expected Result**: + +```console +'count' is greater than 0 +``` + +**Returned Value**: + +```console +--count=1 +``` + +### 5.3.2 Ensure that all Namespaces have Network Policies defined (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +**Audit Script:** `check_for_rke2_network_policies.sh` + +```bash +#!/bin/bash + +set -eE + +handle_error() { + echo "false" +} + +trap 'handle_error' ERR + +for namespace in kube-system kube-public default; do + policy_count=$(/var/lib/rancher/rke2/bin/kubectl get networkpolicy -n ${namespace} -o json | jq -r '.items | length') + if [ ${policy_count} -eq 0 ]; then + echo "false" + exit + fi +done + +echo "true" + +``` + +**Audit Execution:** + +```bash +./check_for_rke2_network_policies.sh +``` + +**Expected Result**: + +```console +'true' is equal to 'true' +``` + +**Returned Value**: + +```console +true +``` + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md new file mode 100644 index 00000000000..b7d4a36ea4b --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27.md @@ -0,0 +1,2967 @@ +--- +title: RKE2 Self-Assessment Guide - CIS Benchmark v1.7 - K8s v1.25/v1.26/v1.27 +--- + + + + + +This document is a companion to the [RKE2 Hardening Guide](../../../../pages-for-subheaders/rke2-hardening-guide.md), which provides prescriptive guidance on how to harden RKE2 clusters that are running in production and managed by Rancher. This benchmark guide helps you evaluate the security of a hardened cluster against each control in the CIS Kubernetes Benchmark. + +This guide corresponds to the following versions of Rancher, CIS Benchmarks, and Kubernetes: + +| Rancher Version | CIS Benchmark Version | Kubernetes Version | +|-----------------|-----------------------|--------------------| +| Rancher v2.7 | Benchmark v1.7 | Kubernetes v1.25/v1.26/v1.27 | + +This guide walks through the various controls and provide updated example commands to audit compliance in Rancher created clusters. Because Rancher and RKE2 install Kubernetes services as Docker containers, many of the control verification checks in the CIS Kubernetes Benchmark don't apply. These checks will return a result of `Not Applicable`. + +This document is for Rancher operators, security teams, auditors and decision makers. + +For more information about each control, including detailed descriptions and remediations for failing tests, refer to the corresponding section of the CIS Kubernetes Benchmark v1.7. You can download the benchmark, after creating a free account, at [Center for Internet Security (CIS)](https://www.cisecurity.org/benchmark/kubernetes/). + +## Testing Methodology + +RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as the container runtime. Configuration is defined by arguments passed to the container at the time of initialization or via configuration file. + +Where control audits differ from the original CIS benchmark, the audit commands specific to Rancher are provided for testing. When performing the tests, you will need access to the command line on the hosts of all RKE2 nodes. The commands also make use of the [kubectl](https://kubernetes.io/docs/tasks/tools/) (with a valid configuration file) and [jq](https://stedolan.github.io/jq/) tools, which are required in the testing and evaluation of test results. + +:::note + +This guide only covers `automated` (previously called `scored`) tests. + +::: + +### Controls + +## 1.1 Control Plane Node Configuration Files +### 1.1.1 Ensure that the API server pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the +control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.2 Ensure that the API server pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.3 Ensure that the controller manager pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.4 Ensure that the controller manager pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.5 Ensure that the scheduler pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c permissions=%a /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.6 Ensure that the scheduler pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.7 Ensure that the etcd pod specification file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then find /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml -name '*etcd*' | xargs stat -c permissions=%a; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.8 Ensure that the etcd pod specification file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; then stat -c %U:%G /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.9 Ensure that the Container Network Interface file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c permissions=%a find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c permissions=%a +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=644 +``` + +### 1.1.10 Ensure that the Container Network Interface file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root + +**Audit:** + +```bash +ps -fC ${kubeletbin:-kubelet} | grep -- --cni-conf-dir || echo "/etc/cni/net.d" | sed 's%.*cni-conf-dir[= ]\([^ ]*\).*%\1%' | xargs -I{} find {} -mindepth 1 | xargs --no-run-if-empty stat -c %U:%G find /var/lib/cni/networks -type f 2> /dev/null | xargs --no-run-if-empty stat -c %U:%G +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root root:root +``` + +### 1.1.11 Ensure that the etcd data directory permissions are set to 700 or more restrictive (Automated) + + +**Result:** pass + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). For example, +chmod 700 /var/lib/etcd + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/db/etcd +``` + +**Expected Result**: + +```console +permissions has permissions 700, expected 700 or more restrictive +``` + +**Returned Value**: + +```console +permissions=700 +``` + +### 1.1.12 Ensure that the etcd data directory ownership is set to etcd:etcd (Automated) + + +**Result:** Not Applicable + +**Remediation:** +On the etcd server node, get the etcd data directory, passed as an argument --data-dir, +from the command 'ps -ef | grep etcd'. +Run the below command (based on the etcd data directory found above). +For example, chown etcd:etcd /var/lib/etcd + +### 1.1.13 Ensure that the admin.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chmod 600 /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.14 Ensure that the admin.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, chown root:root /etc/kubernetes/admin.conf + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/admin.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.15 Ensure that the scheduler.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.16 Ensure that the scheduler.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/scheduler.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.17 Ensure that the controller-manager.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod 600 /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/server/cred/controller.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/server/cred/controller.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'600' is present +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 1.1.18 Ensure that the controller-manager.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown root:root /var/lib/rancher/rke2/server/cred/controller.kubeconfig + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/cred/controller.kubeconfig +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.19 Ensure that the Kubernetes PKI directory and file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chown -R root:root /etc/kubernetes/pki/ + +**Audit:** + +```bash +stat -c %U:%G /var/lib/rancher/rke2/server/tls +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 1.1.20 Ensure that the Kubernetes PKI certificate file permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.crt + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.crt +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 permissions=644 +``` + +### 1.1.21 Ensure that the Kubernetes PKI key file permissions are set to 600 (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the control plane node. +For example, +chmod -R 600 /var/lib/rancher/rke2/server/tls/*.key + +**Audit:** + +```bash +stat -c permissions=%a /var/lib/rancher/rke2/server/tls/*.key +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 permissions=600 +``` + +## 1.2 API Server +### 1.2.1 Ensure that the --anonymous-auth argument is set to false (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--anonymous-auth=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +### 1.2.2 Ensure that the --token-auth-file parameter is not set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and configure alternate mechanisms for authentication. Then, +edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the --token-auth-file= parameter. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--token-auth-file' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.3 Ensure that the --DenyServiceExternalIPs is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and remove the `DenyServiceExternalIPs` +from enabled admission plugins. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'DenyServiceExternalIPs' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.4 Ensure that the --kubelet-client-certificate and --kubelet-client-key arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the +apiserver and kubelets. Then, edit API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +kubelet client certificate and key parameters as below. +--kubelet-client-certificate= +--kubelet-client-key= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-client-certificate' is present AND '--kubelet-client-key' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.5 Ensure that the --kubelet-certificate-authority argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and setup the TLS connection between +the apiserver and kubelets. Then, edit the API server pod specification file +/var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml on the control plane node and set the +--kubelet-certificate-authority parameter to the path to the cert file for the certificate authority. +--kubelet-certificate-authority= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--kubelet-certificate-authority' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.6 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to values other than AlwaysAllow. +One such example could be as below. +--authorization-mode=RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.7 Ensure that the --authorization-mode argument includes Node (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes Node. +--authorization-mode=Node,RBAC + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'Node' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.8 Ensure that the --authorization-mode argument includes RBAC (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --authorization-mode parameter to a value that includes RBAC, +for example `--authorization-mode=Node,RBAC`. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--authorization-mode' has 'RBAC' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.9 Ensure that the admission control plugin EventRateLimit is set (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and set the desired limits in a configuration file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameters. +--enable-admission-plugins=...,EventRateLimit,... +--admission-control-config-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'EventRateLimit' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.10 Ensure that the admission control plugin AlwaysAdmit is not set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --enable-admission-plugins parameter, or set it to a +value that does not include AlwaysAdmit. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' does not have 'AlwaysAdmit' OR '--enable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.11 Ensure that the admission control plugin AlwaysPullImages is set (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +AlwaysPullImages. +--enable-admission-plugins=...,AlwaysPullImages,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'AlwaysPullImages' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.12 Ensure that the admission control plugin SecurityContextDeny is set if PodSecurityPolicy is not used (Manual) + + +**Result:** warn + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to include +SecurityContextDeny, unless PodSecurityPolicy is already in place. +--enable-admission-plugins=...,SecurityContextDeny,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'SecurityContextDeny' OR '--enable-admission-plugins' has 'PodSecurityPolicy' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.13 Ensure that the admission control plugin ServiceAccount is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the documentation and create ServiceAccount objects as per your environment. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and ensure that the --disable-admission-plugins parameter is set to a +value that does not include ServiceAccount. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true root 1018768 2419 99 16:17 ? 00:00:00 kubectl get --server=https://localhost:6443/ --client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --raw=/readyz +``` + +### 1.2.14 Ensure that the admission control plugin NamespaceLifecycle is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --disable-admission-plugins parameter to +ensure it does not include NamespaceLifecycle. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--disable-admission-plugins' is present OR '--disable-admission-plugins' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.15 Ensure that the admission control plugin NodeRestriction is set (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure NodeRestriction plug-in on kubelets. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --enable-admission-plugins parameter to a +value that includes NodeRestriction. +--enable-admission-plugins=...,NodeRestriction,... + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--enable-admission-plugins' has 'NodeRestriction' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.16 Ensure that the --secure-port argument is not set to 0 - NoteThis recommendation is obsolete and will be deleted per the consensus process (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and either remove the --secure-port parameter or +set it to a different (non-zero) desired port. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--secure-port' is greater than 0 OR '--secure-port' is not present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.17 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.18 Ensure that the --audit-log-path argument is set (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-path parameter to a suitable path and +file where you would like audit logs to be written, for example, +--audit-log-path=/var/log/apiserver/audit.log + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-path' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.19 Ensure that the --audit-log-maxage argument is set to 30 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxage parameter to 30 +or as an appropriate number of days, for example, +--audit-log-maxage=30 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxage' is greater or equal to 30 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.20 Ensure that the --audit-log-maxbackup argument is set to 10 or as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxbackup parameter to 10 or to an appropriate +value. For example, +--audit-log-maxbackup=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-log-maxbackup' is greater or equal to 10 +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.21 Ensure that the --audit-log-maxsize argument is set to 100 or as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --audit-log-maxsize parameter to an appropriate size in MB. +For example, to set it as 100 MB, --audit-log-maxsize=100 + +### 1.2.22 Ensure that the --request-timeout argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +and set the below parameter as appropriate and if needed. +For example, --request-timeout=300s + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--request-timeout' is not present OR '--request-timeout' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.23 Ensure that the --service-account-lookup argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--service-account-lookup=true +Alternatively, you can delete the --service-account-lookup parameter from this file so +that the default takes effect. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-lookup' is not present OR '--service-account-lookup' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.24 Ensure that the --service-account-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --service-account-key-file parameter +to the public key file for service accounts. For example, +--service-account-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.25 Ensure that the --etcd-certfile and --etcd-keyfile arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate and key file parameters. +--etcd-certfile= +--etcd-keyfile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-certfile' is present AND '--etcd-keyfile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.26 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the TLS certificate and private key file parameters. +--tls-cert-file= +--tls-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.27 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection on the apiserver. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the client certificate authority file. +--client-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.28 Ensure that the --etcd-cafile argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and set up the TLS connection between the apiserver and etcd. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the etcd certificate authority file parameter. +--etcd-cafile= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--etcd-cafile' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.29 Ensure that the --encryption-provider-config argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +Then, edit the API server pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-apiserver.yaml +on the control plane node and set the --encryption-provider-config parameter to the path of that file. +For example, --encryption-provider-config= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--encryption-provider-config' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.2.30 Ensure that encryption providers are appropriately configured (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and configure a EncryptionConfig file. +In this file, choose aescbc, kms or secretbox as the encryption provider. + +**Audit:** + +```bash +ENCRYPTION_PROVIDER_CONFIG=$(ps -ef | grep kube-apiserver | grep -- --encryption-provider-config | sed 's%.*encryption-provider-config[= ]\([^ ]*\).*%\1%') if test -e $ENCRYPTION_PROVIDER_CONFIG; then grep -A1 'providers:' $ENCRYPTION_PROVIDER_CONFIG | tail -n1 | grep -o "[A-Za-z]*" | sed 's/^/provider=/'; fi +``` + +**Expected Result**: + +```console +'provider' is present +``` + +### 1.2.31 Ensure that the API Server only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the API server pod specification file /etc/kubernetes/manifests/kube-apiserver.yaml +on the control plane node and set the below parameter. +--tls-cipher-suites=TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, +TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, +TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, +TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, +TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_RSA_WITH_3DES_EDE_CBC_SHA,TLS_RSA_WITH_AES_128_CBC_SHA, +TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_256_GCM_SHA384 +Not Applicable. + +## 1.3 Controller Manager +### 1.3.1 Ensure that the --terminated-pod-gc-threshold argument is set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --terminated-pod-gc-threshold to an appropriate threshold, +for example, --terminated-pod-gc-threshold=10 + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--terminated-pod-gc-threshold' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.2 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.3 Ensure that the --use-service-account-credentials argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node to set the below parameter. +--use-service-account-credentials=true + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--use-service-account-credentials' is not equal to 'false' +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.4 Ensure that the --service-account-private-key-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --service-account-private-key-file parameter +to the private key file for service accounts. +--service-account-private-key-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--service-account-private-key-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.5 Ensure that the --root-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --root-ca-file parameter to the certificate bundle file`. +--root-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--root-ca-file' is present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 1.3.6 Ensure that the RotateKubeletServerCertificate argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and set the --feature-gates parameter to include RotateKubeletServerCertificate=true. +--feature-gates=RotateKubeletServerCertificate=true + +### 1.3.7 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Controller Manager pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-controller-manager.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-controller-manager | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +## 1.4 Scheduler +### 1.4.1 Ensure that the --profiling argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml file +on the control plane node and set the below parameter. +--profiling=false + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--profiling' is equal to 'false' +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +### 1.4.2 Ensure that the --bind-address argument is set to 127.0.0.1 (Automated) + + +**Result:** pass + +**Remediation:** +Edit the Scheduler pod specification file /var/lib/rancher/rke2/agent/pod-manifests/kube-scheduler.yaml +on the control plane node and ensure the correct value for the --bind-address parameter + +**Audit:** + +```bash +/bin/ps -ef | grep kube-scheduler | grep -v grep +``` + +**Expected Result**: + +```console +'--bind-address' is equal to '127.0.0.1' OR '--bind-address' is not present +``` + +**Returned Value**: + +```console +root 2645 2538 0 Sep11 ? 00:05:26 kube-scheduler --permit-port-sharing=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-scheduler --kubeconfig=/var/lib/rancher/rke2/server/cred/scheduler.kubeconfig --profiling=false --secure-port=10259 +``` + +## 2 Etcd Node Configuration +### 2.1 Ensure that the --cert-file and --key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure TLS encryption. +Then, edit the etcd pod specification file /etc/kubernetes/manifests/etcd.yaml +on the master node and set the below parameters. +--cert-file= +--key-file= +Not Applicable. + +### 2.2 Ensure that the --client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--client-cert-auth="true" +Not Applicable. + +### 2.3 Ensure that the --auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --auto-tls parameter or set it to false. + --auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_AUTO_TLS' is not present OR 'ETCD_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.4 Ensure that the --peer-cert-file and --peer-key-file arguments are set as appropriate (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Follow the etcd service documentation and configure peer TLS encryption as appropriate +for your etcd cluster. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameters. +--peer-client-file= +--peer-key-file= +Not Applicable. + +### 2.5 Ensure that the --peer-client-cert-auth argument is set to true (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and set the below parameter. +--peer-client-cert-auth=true +Not Applicable. + +### 2.6 Ensure that the --peer-auto-tls argument is not set to true (Automated) + + +**Result:** pass + +**Remediation:** +Edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the master +node and either remove the --peer-auto-tls parameter or set it to false. +--peer-auto-tls=false + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Expected Result**: + +```console +'ETCD_PEER_AUTO_TLS' is not present OR 'ETCD_PEER_AUTO_TLS' is present +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +### 2.7 Ensure that a unique Certificate Authority is used for etcd (Automated) + + +**Result:** pass + +**Remediation:** +[Manual test] +Follow the etcd documentation and create a dedicated certificate authority setup for the +etcd service. +Then, edit the etcd pod specification file /var/lib/rancher/rke2/agent/pod-manifests/etcd.yaml on the +master node and set the below parameter. +--trusted-ca-file= + +**Audit:** + +```bash +/bin/ps -ef | /bin/grep etcd | /bin/grep -v grep +``` + +**Audit Config:** + +```bash +cat /var/lib/rancher/rke2/server/db/etcd/config +``` + +**Expected Result**: + +```console +'ETCD_TRUSTED_CA_FILE' is present OR '{.peer-transport-security.trusted-ca-file}' is equal to '/var/lib/rancher/rke2/server/tls/etcd/peer-ca.crt' +``` + +**Returned Value**: + +```console +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOSTNAME=ip-172-31-14-226 ETCD_UNSUPPORTED_ARCH= FILE_HASH=e9ca6f328e70dd3c17ba78c302ee32927a4961198e95d1d948114a5d7e350d99 NO_PROXY=.svc,.cluster.local,10.42.0.0/16,10.43.0.0/16 POD_HASH=aa1658a59ab75324ef67c786d08307e3 HOME=/ +``` + +## 3.1 Authentication and Authorization +### 3.1.1 Client certificate authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be +implemented in place of client certificates. + +### 3.1.2 Service account token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of service account tokens. + +### 3.1.3 Bootstrap token authentication should not be used for users (Manual) + + +**Result:** warn + +**Remediation:** +Alternative mechanisms provided by Kubernetes such as the use of OIDC should be implemented +in place of bootstrap tokens. + +## 3.2 Logging +### 3.2.1 Ensure that a minimal audit policy is created (Automated) + + +**Result:** pass + +**Remediation:** +Create an audit policy file for your cluster. + +**Audit:** + +```bash +/bin/ps -ef | grep kube-apiserver | grep -v grep +``` + +**Expected Result**: + +```console +'--audit-policy-file' is present +``` + +**Returned Value**: + +```console +root 2489 2419 8 Sep11 ? 01:41:54 kube-apiserver --admission-control-config-file=/etc/rancher/rke2/rke2-pss.yaml --audit-policy-file=/etc/rancher/rke2/audit-policy.yaml --audit-log-maxage=30 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/lib/rancher/rke2/server/logs/audit.log --admission-control-config-file=/etc/rancher/rke2/config/rancher-psact.yaml --allow-privileged=true --anonymous-auth=false --api-audiences=https://kubernetes.default.svc.cluster.local,rke2 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --cert-dir=/var/lib/rancher/rke2/server/tls/temporary-certs --client-ca-file=/var/lib/rancher/rke2/server/tls/client-ca.crt --egress-selector-config-file=/var/lib/rancher/rke2/server/etc/egress-selector-config.yaml --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --encryption-provider-config=/var/lib/rancher/rke2/server/cred/encryption-config.json --etcd-cafile=/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt --etcd-certfile=/var/lib/rancher/rke2/server/tls/etcd/client.crt --etcd-keyfile=/var/lib/rancher/rke2/server/tls/etcd/client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates=JobTrackingWithFinalizers=true --kubelet-certificate-authority=/var/lib/rancher/rke2/server/tls/server-ca.crt --kubelet-client-certificate=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.crt --kubelet-client-key=/var/lib/rancher/rke2/server/tls/client-kube-apiserver.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --profiling=false --proxy-client-cert-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.crt --proxy-client-key-file=/var/lib/rancher/rke2/server/tls/client-auth-proxy.key --requestheader-allowed-names=system:auth-proxy --requestheader-client-ca-file=/var/lib/rancher/rke2/server/tls/request-header-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/var/lib/rancher/rke2/server/tls/service.key --service-account-signing-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --service-node-port-range=30000-32767 --storage-backend=etcd3 --tls-cert-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.crt --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --tls-private-key-file=/var/lib/rancher/rke2/server/tls/serving-kube-apiserver.key root 2652 2539 2 Sep11 ? 00:24:53 kube-controller-manager --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins --terminated-pod-gc-threshold=1000 --permit-port-sharing=true --allocate-node-cidrs=true --authentication-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --authorization-kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --bind-address=127.0.0.1 --cert-dir=/var/lib/rancher/rke2/server/tls/kube-controller-manager --cluster-cidr=10.42.0.0/16 --cluster-signing-kube-apiserver-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kube-apiserver-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-client-cert-file=/var/lib/rancher/rke2/server/tls/client-ca.nochain.crt --cluster-signing-kubelet-client-key-file=/var/lib/rancher/rke2/server/tls/client-ca.key --cluster-signing-kubelet-serving-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-kubelet-serving-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --cluster-signing-legacy-unknown-cert-file=/var/lib/rancher/rke2/server/tls/server-ca.nochain.crt --cluster-signing-legacy-unknown-key-file=/var/lib/rancher/rke2/server/tls/server-ca.key --configure-cloud-routes=false --controllers=*,tokencleaner,-service,-route,-cloud-node-lifecycle --feature-gates=JobTrackingWithFinalizers=true --kubeconfig=/var/lib/rancher/rke2/server/cred/controller.kubeconfig --profiling=false --root-ca-file=/var/lib/rancher/rke2/server/tls/server-ca.crt --secure-port=10257 --service-account-private-key-file=/var/lib/rancher/rke2/server/tls/service.current.key --service-cluster-ip-range=10.43.0.0/16 --use-service-account-credentials=true +``` + +### 3.2.2 Ensure that the audit policy covers key security concerns (Manual) + + +**Result:** warn + +**Remediation:** +Review the audit policy provided for the cluster and ensure that it covers +at least the following areas, +- Access to Secrets managed by the cluster. Care should be taken to only + log Metadata for requests to Secrets, ConfigMaps, and TokenReviews, in + order to avoid risk of logging sensitive data. +- Modification of Pod and Deployment objects. +- Use of `pods/exec`, `pods/portforward`, `pods/proxy` and `services/proxy`. +For most requests, minimally logging at the Metadata level is recommended +(the most basic level of logging). + +## 4.1 Worker Node Configuration Files +### 4.1.1 Ensure that the kubelet service file permissions are set to 600 or more restrictive (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chmod 600 /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not Applicable. + +### 4.1.2 Ensure that the kubelet service file ownership is set to root:root (Automated) + + +**Result:** Not Applicable + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +Not applicable. + +### 4.1.3 If proxy kubeconfig file exists ensure permissions are set to 600 or more restrictive (Manual) + + +**Result:** warn + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.4 If proxy kubeconfig file exists ensure ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, chown root:root /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubeproxy.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.5 Ensure that the --kubeconfig kubelet.conf file permissions are set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.6 Ensure that the --kubeconfig kubelet.conf file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the below command (based on the file location on your system) on the each worker node. +For example, +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.7 Ensure that the certificate authorities file permissions are set to 600 or more restrictive (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the file permissions of the +--client-ca-file chmod 600 + +**Audit Script:** `check_cafile_permissions.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c permissions=%a $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_permissions.sh +``` + +**Expected Result**: + +```console +permissions has permissions 600, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=600 +``` + +### 4.1.8 Ensure that the client certificate authorities file ownership is set to root:root (Manual) + + +**Result:** pass + +**Remediation:** +Run the following command to modify the ownership of the --client-ca-file. +chown root:root + +**Audit Script:** `check_cafile_ownership.sh` + +```bash +#!/usr/bin/env bash + +CAFILE=$(ps -ef | grep kubelet | grep -v apiserver | grep -- --client-ca-file= | awk -F '--client-ca-file=' '{print $2}' | awk '{print $1}') +CAFILE=/node$CAFILE +if test -z $CAFILE; then CAFILE=$kubeletcafile; fi +if test -e $CAFILE; then stat -c %U:%G $CAFILE; fi + +``` + +**Audit Execution:** + +```bash +./check_cafile_ownership.sh +``` + +**Expected Result**: + +```console +'root:root' is equal to 'root:root' +``` + +**Returned Value**: + +```console +root:root +``` + +### 4.1.9 If the kubelet config.yaml configuration file is being used validate permissions set to 600 or more restrictive (Automated) + + +**Result:** fail + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chmod 600 /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c permissions=%a /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +permissions has permissions 644, expected 600 or more restrictive +``` + +**Returned Value**: + +```console +permissions=644 +``` + +### 4.1.10 If the kubelet config.yaml configuration file is being used validate file ownership is set to root:root (Automated) + + +**Result:** pass + +**Remediation:** +Run the following command (using the config file location identified in the Audit step) +chown root:root /var/lib/rancher/rke2/agent/kubelet.kubeconfig + +**Audit:** + +```bash +/bin/sh -c 'if test -e /var/lib/rancher/rke2/agent/kubelet.kubeconfig; then stat -c %U:%G /var/lib/rancher/rke2/agent/kubelet.kubeconfig; fi' +``` + +**Expected Result**: + +```console +'root:root' is present +``` + +**Returned Value**: + +```console +root:root +``` + +## 4.2 Kubelet +### 4.2.1 Ensure that the --anonymous-auth argument is set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication: anonymous: enabled` to +`false`. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +`--anonymous-auth=false` +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--anonymous-auth' is equal to 'false' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.2 Ensure that the --authorization-mode argument is not set to AlwaysAllow (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authorization.mode` to Webhook. If +using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--authorization-mode=Webhook +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--authorization-mode' does not have 'AlwaysAllow' +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.3 Ensure that the --client-ca-file argument is set as appropriate (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `authentication.x509.clientCAFile` to +the location of the client CA file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_AUTHZ_ARGS variable. +--client-ca-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--client-ca-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.4 Verify that the --read-only-port argument is set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `readOnlyPort` to 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--read-only-port=0 +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--read-only-port' is equal to '0' OR '--read-only-port' is not present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.5 Ensure that the --streaming-connection-idle-timeout argument is not set to 0 (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `streamingConnectionIdleTimeout` to a +value other than 0. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +--streaming-connection-idle-timeout=5m +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.streamingConnectionIdleTimeout}' is present OR '{.streamingConnectionIdleTimeout}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.6 Ensure that the --make-iptables-util-chains argument is set to true (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `makeIPTablesUtilChains` to `true`. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove the --make-iptables-util-chains argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.makeIPTablesUtilChains}' is present OR '{.makeIPTablesUtilChains}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.7 Ensure that the --hostname-override argument is not set (Manual) + + +**Result:** Not Applicable + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and remove the --hostname-override argument from the +KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service +Not Applicable. + +### 4.2.8 Ensure that the eventRecordQPS argument is set to a level which ensures appropriate event capture (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `eventRecordQPS` to an appropriate level. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameter in KUBELET_SYSTEM_PODS_ARGS variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.eventRecordQPS}' is present OR '{.eventRecordQPS}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.9 Ensure that the --tls-cert-file and --tls-private-key-file arguments are set as appropriate (Manual) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to set `tlsCertFile` to the location +of the certificate file to use to identify this Kubelet, and `tlsPrivateKeyFile` +to the location of the corresponding private key file. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the below parameters in KUBELET_CERTIFICATE_ARGS variable. +--tls-cert-file= +--tls-private-key-file= +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'--tls-cert-file' is present AND '--tls-private-key-file' is present +``` + +**Returned Value**: + +```console +UID PID PPID C STIME TTY TIME CMD root 2236 2075 4 Sep11 ? 00:47:51 kubelet --volume-plugin-dir=/var/lib/kubelet/volumeplugins --file-check-frequency=5s --sync-frequency=30s --address=0.0.0.0 --alsologtostderr=false --anonymous-auth=false --authentication-token-webhook=true --authorization-mode=Webhook --cgroup-driver=systemd --client-ca-file=/var/lib/rancher/rke2/agent/client-ca.crt --cloud-provider=external --cluster-dns=10.43.0.10 --cluster-domain=cluster.local --container-runtime-endpoint=unix:///run/k3s/containerd/containerd.sock --containerd=/run/k3s/containerd/containerd.sock --eviction-hard=imagefs.available<5%,nodefs.available<5% --eviction-minimum-reclaim=imagefs.available=10%,nodefs.available=10% --fail-swap-on=false --healthz-bind-address=127.0.0.1 --hostname-override=ip-172-31-14-226 --kubeconfig=/var/lib/rancher/rke2/agent/kubelet.kubeconfig --log-file=/var/lib/rancher/rke2/agent/logs/kubelet.log --log-file-max-size=50 --logtostderr=false --node-labels=cattle.io/os=linux,rke.cattle.io/machine=1adcc1f7-66f1-4503-b9bc-dfb6e808f27b --pod-infra-container-image=index.docker.io/rancher/pause:3.6 --pod-manifest-path=/var/lib/rancher/rke2/agent/pod-manifests --protect-kernel-defaults=true --read-only-port=0 --resolv-conf=/run/systemd/resolve/resolv.conf --serialize-image-pulls=false --stderrthreshold=FATAL --tls-cert-file=/var/lib/rancher/rke2/agent/serving-kubelet.crt --tls-private-key-file=/var/lib/rancher/rke2/agent/serving-kubelet.key +``` + +### 4.2.10 Ensure that the --rotate-certificates argument is not set to false (Automated) + + +**Result:** pass + +**Remediation:** +If using a Kubelet config file, edit the file to add the line `rotateCertificates` to `true` or +remove it altogether to use the default value. +If using command line arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +remove --rotate-certificates=false argument from the KUBELET_CERTIFICATE_ARGS +variable. +Based on your system, restart the kubelet service. For example, +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.rotateCertificates}' is present OR '{.rotateCertificates}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.11 Verify that the RotateKubeletServerCertificate argument is set to true (Manual) + + +**Result:** pass + +**Remediation:** +Edit the kubelet service file /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +on each worker node and set the below parameter in KUBELET_CERTIFICATE_ARGS variable. +--feature-gates=RotateKubeletServerCertificate=true +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.featureGates.RotateKubeletServerCertificate}' is present OR '{.featureGates.RotateKubeletServerCertificate}' is not present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.12 Ensure that the Kubelet only makes use of Strong Cryptographic Ciphers (Manual) + + +**Result:** warn + +**Remediation:** +If using a Kubelet config file, edit the file to set `TLSCipherSuites` to +TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +or to a subset of these values. +If using executable arguments, edit the kubelet service file +/etc/systemd/system/kubelet.service.d/10-kubeadm.conf on each worker node and +set the --tls-cipher-suites parameter as follows, or to a subset of these values. +--tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256 +Based on your system, restart the kubelet service. For example: +systemctl daemon-reload +systemctl restart kubelet.service + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{range .tlsCipherSuites[:]}{}{','}{end}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +### 4.2.13 Ensure that a limit is set on pod PIDs (Manual) + + +**Result:** warn + +**Remediation:** +Decide on an appropriate level for this parameter and set it, +either via the --pod-max-pids command line parameter or the PodPidsLimit configuration file setting. + +**Audit:** + +```bash +/bin/ps -fC kubelet +``` + +**Audit Config:** + +```bash +/bin/cat /var/lib/rancher/rke2/agent/kubelet.kubeconfig +``` + +**Expected Result**: + +```console +'{.podPidsLimit}' is present +``` + +**Returned Value**: + +```console +apiVersion: v1 clusters: - cluster: server: https://127.0.0.1:6443 certificate-authority: /var/lib/rancher/rke2/agent/server-ca.crt name: local contexts: - context: cluster: local namespace: default user: user name: Default current-context: Default kind: Config preferences: {} users: - name: user user: client-certificate: /var/lib/rancher/rke2/agent/client-kubelet.crt client-key: /var/lib/rancher/rke2/agent/client-kubelet.key +``` + +## 5.1 RBAC and Service Accounts +### 5.1.1 Ensure that the cluster-admin role is only used where required (Manual) + + +**Result:** warn + +**Remediation:** +Identify all clusterrolebindings to the cluster-admin role. Check if they are used and +if they need this role or if they could use a role with fewer privileges. +Where possible, first bind users to a lower privileged role and then remove the +clusterrolebinding to the cluster-admin role : +kubectl delete clusterrolebinding [name] + +### 5.1.2 Minimize access to secrets (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove get, list and watch access to Secret objects in the cluster. + +### 5.1.3 Minimize wildcard use in Roles and ClusterRoles (Manual) + + +**Result:** warn + +**Remediation:** +Where possible replace any use of wildcards in clusterroles and roles with specific +objects or actions. + +### 5.1.4 Minimize access to create pods (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to pod objects in the cluster. + +### 5.1.5 Ensure that default service accounts are not actively used. (Manual) + + +**Result:** warn + +**Remediation:** +Create explicit service accounts wherever a Kubernetes workload requires specific access +to the Kubernetes API server. +Modify the configuration of each default service account to include this value +automountServiceAccountToken: false + +### 5.1.6 Ensure that Service Account Tokens are only mounted where necessary (Manual) + + +**Result:** warn + +**Remediation:** +Modify the definition of pods and service accounts which do not need to mount service +account tokens to disable it. + +### 5.1.7 Avoid use of system:masters group (Manual) + + +**Result:** warn + +**Remediation:** +Remove the system:masters group from all users in the cluster. + +### 5.1.8 Limit use of the Bind, Impersonate and Escalate permissions in the Kubernetes cluster (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove the impersonate, bind and escalate rights from subjects. + +### 5.1.9 Minimize access to create persistent volumes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove create access to PersistentVolume objects in the cluster. + +### 5.1.10 Minimize access to the proxy sub-resource of nodes (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the proxy sub-resource of node objects. + +### 5.1.11 Minimize access to the approval sub-resource of certificatesigningrequests objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the approval sub-resource of certificatesigningrequest objects. + +### 5.1.12 Minimize access to webhook configuration objects (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the validatingwebhookconfigurations or mutatingwebhookconfigurations objects + +### 5.1.13 Minimize access to the service account token creation (Manual) + + +**Result:** warn + +**Remediation:** +Where possible, remove access to the token sub-resource of serviceaccount objects. + +## 5.2 Pod Security Standards +### 5.2.1 Ensure that the cluster has at least one active policy control mechanism in place (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that either Pod Security Admission or an external policy control system is in place +for every namespace which contains user workloads. + +### 5.2.2 Minimize the admission of privileged containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of privileged containers. + +### 5.2.3 Minimize the admission of containers wishing to share the host process ID namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostPID` containers. + +### 5.2.4 Minimize the admission of containers wishing to share the host IPC namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostIPC` containers. + +### 5.2.5 Minimize the admission of containers wishing to share the host network namespace (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of `hostNetwork` containers. + +### 5.2.6 Minimize the admission of containers with allowPrivilegeEscalation (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `.spec.allowPrivilegeEscalation` set to `true`. + +### 5.2.7 Minimize the admission of root containers (Manual) + + +**Result:** warn + +**Remediation:** +Create a policy for each namespace in the cluster, ensuring that either `MustRunAsNonRoot` +or `MustRunAs` with the range of UIDs not including 0, is set. + +### 5.2.8 Minimize the admission of containers with the NET_RAW capability (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with the `NET_RAW` capability. + +### 5.2.9 Minimize the admission of containers with added capabilities (Automated) + + +**Result:** warn + +**Remediation:** +Ensure that `allowedCapabilities` is not present in policies for the cluster unless +it is set to an empty array. + +### 5.2.10 Minimize the admission of containers with capabilities assigned (Manual) + + +**Result:** warn + +**Remediation:** +Review the use of capabilites in applications running on your cluster. Where a namespace +contains applicaions which do not require any Linux capabities to operate consider adding +a PSP which forbids the admission of containers which do not drop all capabilities. + +### 5.2.11 Minimize the admission of Windows HostProcess containers (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers that have `.securityContext.windowsOptions.hostProcess` set to `true`. + +### 5.2.12 Minimize the admission of HostPath volumes (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers with `hostPath` volumes. + +### 5.2.13 Minimize the admission of containers which use HostPorts (Manual) + + +**Result:** warn + +**Remediation:** +Add policies to each namespace in the cluster which has user workloads to restrict the +admission of containers which use `hostPort` sections. + +## 5.3 Network Policies and CNI +### 5.3.1 Ensure that the CNI in use supports NetworkPolicies (Manual) + + +**Result:** warn + +**Remediation:** +If the CNI plugin in use does not support network policies, consideration should be given to +making use of a different plugin, or finding an alternate mechanism for restricting traffic +in the Kubernetes cluster. + +### 5.3.2 Ensure that all Namespaces have NetworkPolicies defined (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create NetworkPolicy objects as you need them. + +## 5.4 Secrets Management +### 5.4.1 Prefer using Secrets as files over Secrets as environment variables (Manual) + + +**Result:** warn + +**Remediation:** +If possible, rewrite application code to read Secrets from mounted secret files, rather than +from environment variables. + +### 5.4.2 Consider external secret storage (Manual) + + +**Result:** warn + +**Remediation:** +Refer to the Secrets management options offered by your cloud provider or a third-party +secrets management solution. + +## 5.5 Extensible Admission Control +### 5.5.1 Configure Image Provenance using ImagePolicyWebhook admission controller (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and setup image provenance. + +## 5.7 General Policies +### 5.7.1 Create administrative boundaries between resources using namespaces (Manual) + + +**Result:** warn + +**Remediation:** +Follow the documentation and create namespaces for objects in your deployment as you need +them. + +### 5.7.2 Ensure that the seccomp profile is set to docker/default in your Pod definitions (Manual) + + +**Result:** warn + +**Remediation:** +Use `securityContext` to enable the docker/default seccomp profile in your pod definitions. +An example is as below: + securityContext: + seccompProfile: + type: RuntimeDefault + +### 5.7.3 Apply SecurityContext to your Pods and Containers (Manual) + + +**Result:** warn + +**Remediation:** +Follow the Kubernetes documentation and apply SecurityContexts to your Pods. For a +suggested list of SecurityContexts, you may refer to the CIS Security Benchmark for Docker +Containers. + +### 5.7.4 The default namespace should not be used (Manual) + + +**Result:** warn + +**Remediation:** +Ensure that namespaces are created to allow for appropriate segregation of Kubernetes +resources and that all new resources are created in a specific namespace. + diff --git a/docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md b/versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md similarity index 100% rename from docs/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md rename to versioned_docs/version-latest/reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25.md diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/kubernetes-security-best-practices.md b/versioned_docs/version-latest/reference-guides/rancher-security/kubernetes-security-best-practices.md new file mode 100644 index 00000000000..ace8bd95fab --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/kubernetes-security-best-practices.md @@ -0,0 +1,15 @@ +--- +title: Kubernetes Security Best Practices +--- + + + + + +### Restricting cloud metadata API access + +Cloud providers such as AWS, Azure, DigitalOcean or GCP often expose metadata services locally to instances. By default, this endpoint is accessible by pods running on a cloud instance, including pods in hosted Kubernetes providers such as EKS, AKS, DigitalOcean Kubernetes or GKE, and can contain cloud credentials for that node, provisioning data such as kubelet credentials, or other sensitive data. To mitigate this risk when running on a cloud platform, follow the [Kubernetes security recommendations](https://kubernetes.io/docs/tasks/administer-cluster/securing-a-cluster/#restricting-cloud-metadata-api-access): limit permissions given to instance credentials, use network policies to restrict pod access to the metadata API, and avoid using provisioning data to deliver secrets. + +It is advised to consult your cloud provider's security best practices for further recommendations and specific details on how to restrict access to cloud instance metadata API. + +Further references: MITRE ATT&CK knowledge base on - [Unsecured Credentials: Cloud Instance Metadata API](https://attack.mitre.org/techniques/T1552/005/). diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/psa-restricted-exemptions.md b/versioned_docs/version-latest/reference-guides/rancher-security/psa-restricted-exemptions.md new file mode 100644 index 00000000000..0f35551cf31 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/psa-restricted-exemptions.md @@ -0,0 +1,66 @@ +--- +title: Sample PodSecurityConfiguration +--- + + + + + +The following PodSecurityConfiguration contains the required Rancher namespace exemptions for a `rancher-restricted` cluster to run properly. + +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1 + kind: PodSecurityConfiguration + defaults: + enforce: "restricted" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: [calico-apiserver, + calico-system, + cattle-alerting, + cattle-csp-adapter-system, + cattle-elemental-system, + cattle-epinio-system, + cattle-externalip-system, + cattle-fleet-local-system, + cattle-fleet-system, + cattle-gatekeeper-system, + cattle-global-data, + cattle-global-nt, + cattle-impersonation-system, + cattle-istio, + cattle-istio-system, + cattle-logging, + cattle-logging-system, + cattle-monitoring-system, + cattle-neuvector-system, + cattle-prometheus, + cattle-resources-system, + cattle-sriov-system, + cattle-system, + cattle-ui-plugin-system, + cattle-windows-gmsa-system, + cert-manager, + cis-operator-system, + fleet-default, + ingress-nginx, + istio-system, + kube-node-lease, + kube-public, + kube-system, + longhorn-system, + rancher-alerting-drivers, + security-scan, + tigera-operator] +``` diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/security-advisories-and-cves.md b/versioned_docs/version-latest/reference-guides/rancher-security/security-advisories-and-cves.md new file mode 100644 index 00000000000..8f3649e7a85 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/security-advisories-and-cves.md @@ -0,0 +1,45 @@ +--- +title: Security Advisories and CVEs +--- + + + + + +Rancher is committed to informing the community of security issues in our products. Rancher will publish security advisories and CVEs (Common Vulnerabilities and Exposures) for issues we have resolved. New security advisories are also published in Rancher's GitHub [security page](https://github.com/rancher/rancher/security/advisories). + +| ID | Description | Date | Resolution | +|----|-------------|------|------------| +| [CVE-2023-22648](https://github.com/rancher/rancher/security/advisories/GHSA-vf6j-6739-78m8) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, in which permission changes in Azure AD are not reflected to users until they logout and log back into the Rancher UI. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2022-43760](https://github.com/rancher/rancher/security/advisories/GHSA-46v3-ggjg-qq3x) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, where multiple Cross-Site Scripting (XSS) vulnerabilities can be exploited via the Rancher UI. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2020-10676](https://github.com/rancher/rancher/security/advisories/GHSA-8vhc-hwhc-cpj4) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, in which users with update privileges on a namespace, can move that namespace into a project they don't have access to. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22647](https://github.com/rancher/rancher/security/advisories/GHSA-p976-h52c-26p6) | An issue was discovered in Rancher versions up to and including 2.6.12 and 2.7.3, where Standard users or above are able to elevate their permissions to Administrator in the local cluster. | 31 May 2023 | Rancher [v2.7.4](https://github.com/rancher/rancher/releases/tag/v2.7.4) | +| [CVE-2023-22651](https://github.com/rancher/rancher/security/advisories/GHSA-6m9f-pj6w-w87g) | The Rancher admissions webhook may become misconfigured due to a failure in the webhook's update logic. The admissions webhook enforces validation rules and security checks before resources are admitted into the Kubernetes cluster. When the webhook is operating in a degraded state, it no longer validates any resources, which can result in severe privilege escalations and data corruption. | 24 April 2023 | Rancher [v2.7.3](https://github.com/rancher/rancher/releases/tag/v2.7.3) | +| [CVE-2022-43758](https://github.com/rancher/rancher/security/advisories/GHSA-34p5-jp77-fcrc) | An issue was discovered in Rancher from versions 2.5.0 up to and including 2.5.16, 2.6.0 up to and including 2.6.9 and 2.7.0, where a command injection vulnerability is present in the Rancher Git package. This package uses the underlying Git binary available in the Rancher container image to execute Git operations. Specially crafted commands, when not properly disambiguated, can cause confusion when executed through Git, resulting in command injection in the underlying Rancher host. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43757](https://github.com/rancher/rancher/security/advisories/GHSA-cq4p-vp5q-4522) | This issue affects Rancher versions from 2.5.0 up to and including 2.5.16, from 2.6.0 up to and including 2.6.9 and 2.7.0. It was discovered that the security advisory [CVE-2021-36782](https://github.com/advisories/GHSA-g7j7-h4q8-8w2f), previously released by Rancher, missed addressing some sensitive fields, secret tokens, encryption keys, and SSH keys that were still being stored in plaintext directly on Kubernetes objects like `Clusters`. The exposed credentials are visible in Rancher to authenticated `Cluster Owners`, `Cluster Members`, `Project Owners` and `Project Members` of that cluster. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-43755](https://github.com/rancher/rancher/security/advisories/GHSA-8c69-r38j-rpfj) | An issue was discovered in Rancher versions up to and including 2.6.9 and 2.7.0, where the `cattle-token` secret, used by the `cattle-cluster-agent`, is predictable. Even after the token is regenerated, it will have the same value. This can pose a serious problem if the token is compromised and needs to be recreated for security purposes. The `cattle-token` is used by Rancher's `cattle-cluster-agent` to connect to the Kubernetes API of Rancher provisioned downstream clusters. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1) and [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) | +| [CVE-2022-21953](https://github.com/rancher/rancher/security/advisories/GHSA-g25r-gvq3-wrq7) | An issue was discovered in Rancher versions up to and including 2.5.16, 2.6.9 and 2.7.0, where an authorization logic flaw allows an authenticated user on any downstream cluster to (1) open a shell pod in the Rancher `local` cluster and (2) have limited kubectl access to it. The expected behavior is that a user does not have such access in the Rancher `local` cluster unless explicitly granted. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [GHSA-c45c-39f6-6gw9](https://github.com/rancher/rancher/security/advisories/GHSA-c45c-39f6-6gw9) | This issue affects Rancher versions from 2.5.0 up to and including 2.5.16, from 2.6.0 up to and including 2.6.9 and 2.7.0. It only affects Rancher setups that have an external authentication provider configured or had one configured in the past. It was discovered that when an external authentication provider is configured in Rancher and then disabled, the Rancher generated tokens associated with users who had access granted through the now disabled auth provider are not revoked. | 24 January 2023 | Rancher [v2.7.1](https://github.com/rancher/rancher/releases/tag/v2.7.1), [v2.6.10](https://github.com/rancher/rancher/releases/tag/v2.6.10) and [v2.5.17](https://github.com/rancher/rancher/releases/tag/v2.5.17) | +| [CVE-2022-31247](https://github.com/rancher/rancher/security/advisories/GHSA-6x34-89p7-95wg) | An issue was discovered in Rancher versions up to and including 2.5.15 and 2.6.6 where a flaw with authorization logic allows privilege escalation in downstream clusters through cluster role template binding (CRTB) and project role template binding (PRTB). The vulnerability can be exploited by any user who has permissions to create/edit CRTB or PRTB (such as `cluster-owner`, `manage cluster members`, `project-owner`, and `manage project members`) to gain owner permission in another project in the same cluster or in another project on a different downstream cluster. | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36783](https://github.com/rancher/rancher/security/advisories/GHSA-8w87-58w6-hfv8) | It was discovered that in Rancher versions up to and including 2.5.12 and 2.6.3, there is a failure to properly sanitize credentials in cluster template answers. This failure can lead to plaintext storage and exposure of credentials, passwords, and API tokens. The exposed credentials are visible in Rancher to authenticated `Cluster Owners`, `Cluster Members`, `Project Owners`, and `Project Members` on the endpoints `/v1/management.cattle.io.clusters`, `/v3/clusters`, and `/k8s/clusters/local/apis/management.cattle.io/v3/clusters`. | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2021-36782](https://github.com/rancher/rancher/security/advisories/GHSA-g7j7-h4q8-8w2f) | An issue was discovered in Rancher versions up to and including 2.5.15 and 2.6.6 where sensitive fields like passwords, API keys, and Rancher's service account token (used to provision clusters) were stored in plaintext directly on Kubernetes objects like `Clusters` (e.g., `cluster.management.cattle.io`). Anyone with read access to those objects in the Kubernetes API could retrieve the plaintext version of those sensitive data. The issue was partially found and reported by Florian Struck (from [Continum AG](https://www.continum.net/)) and [Marco Stuurman](https://github.com/fe-ax) (from [Shock Media B.V.](https://www.shockmedia.nl/)). | 18 August 2022 | [Rancher v2.6.7](https://github.com/rancher/rancher/releases/tag/v2.6.7) and [Rancher v2.5.16](https://github.com/rancher/rancher/releases/tag/v2.5.16) | +| [CVE-2022-21951](https://github.com/rancher/rancher/security/advisories/GHSA-vrph-m5jj-c46c) | This vulnerability only affects customers using [Weave](../../faq/container-network-interface-providers.md#weave) Container Network Interface (CNI) when configured through [RKE templates](../../pages-for-subheaders/about-rke1-templates.md). A vulnerability was discovered in Rancher versions 2.5.0 up to and including 2.5.13, and 2.6.0 up to and including 2.6.4, where a user interface (UI) issue with RKE templates does not include a value for the Weave password when Weave is chosen as the CNI. If a cluster is created based on the mentioned template, and Weave is configured as the CNI, no password will be created for [network encryption](https://www.weave.works/docs/net/latest/tasks/manage/security-untrusted-networks/) in Weave; therefore, network traffic in the cluster will be sent unencrypted. | 24 May 2022 | [Rancher v2.6.5](https://github.com/rancher/rancher/releases/tag/v2.6.5) and [Rancher v2.5.14](https://github.com/rancher/rancher/releases/tag/v2.5.14) | +| [CVE-2021-36784](https://github.com/rancher/rancher/security/advisories/GHSA-jwvr-vv7p-gpwq) | A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3 which allows users who have create or update permissions on [Global Roles](../../pages-for-subheaders/manage-role-based-access-control-rbac.md) to escalate their permissions, or those of another user, to admin-level permissions. Global Roles grant users Rancher-wide permissions, such as the ability to create clusters. In the identified versions of Rancher, when users are given permission to edit or create Global Roles, they are not restricted to only granting permissions which they already posses. This vulnerability affects customers who utilize non-admin users that are able to create or edit Global Roles. The most common use case for this scenario is the `restricted-admin` role. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-4200](https://github.com/rancher/rancher/security/advisories/GHSA-hx8w-ghh8-r4xf) | This vulnerability only affects customers using the `restricted-admin` role in Rancher. A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3 where the `global-data` role in `cattle-global-data` namespace grants write access to the Catalogs. Since each user with any level of catalog access was bound to the `global-data` role, this grants write access to templates (`CatalogTemplates`) and template versions (`CatalogTemplateVersions`) for any user with any level of catalog access. New users created in Rancher are by default assigned to the `user` role (standard user), which is not designed to grant write catalog access. This vulnerability effectively elevates the privilege of any user to write access for the catalog template and catalog template version resources. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [GHSA-wm2r-rp98-8pmh](https://github.com/rancher/rancher/security/advisories/GHSA-wm2r-rp98-8pmh) | This vulnerability only affects customers using [Fleet](../../how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet.md) for continuous delivery with authenticated Git and/or Helm repositories. An issue was discovered in `go-getter` library in versions prior to [`v1.5.11`](https://github.com/hashicorp/go-getter/releases/tag/v1.5.11) that exposes SSH private keys in base64 format due to a failure in redacting such information from error messages. The vulnerable version of this library is used in Rancher through Fleet in versions of Fleet prior to [`v0.3.9`](https://github.com/rancher/fleet/releases/tag/v0.3.9). This issue affects Rancher versions 2.5.0 up to and including 2.5.12 and from 2.6.0 up to and including 2.6.3. The issue was found and reported by Dagan Henderson from Raft Engineering. | 14 Apr 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) and [Rancher v2.5.13](https://github.com/rancher/rancher/releases/tag/v2.5.13) | +| [CVE-2021-36778](https://github.com/rancher/rancher/security/advisories/GHSA-4fc7-hc63-7fjg) | A vulnerability was discovered in Rancher versions from 2.5.0 up to and including 2.5.11 and from 2.6.0 up to and including 2.6.2, where an insufficient check of the same-origin policy when downloading Helm charts from a configured private repository can lead to exposure of the repository credentials to a third-party provider. This issue only happens when the user configures access credentials to a private repository in Rancher inside `Apps & Marketplace > Repositories`. The issue was found and reported by Martin Andreas Ullrich. | 14 Apr 2022 | [Rancher v2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3) and [Rancher v2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) | +| [GHSA-hwm2-4ph6-w6m5](https://github.com/rancher/rancher/security/advisories/GHSA-hwm2-4ph6-w6m5) | A vulnerability was discovered in versions of Rancher starting 2.0 up to and including 2.6.3. The `restricted` pod security policy (PSP) provided in Rancher deviated from the upstream `restricted` policy provided in Kubernetes on account of which Rancher's PSP had `runAsUser` set to `runAsAny`, while upstream had `runAsUser` set to `MustRunAsNonRoot`. This allowed containers to run as any user, including a privileged user (`root`), even when Rancher's `restricted` policy was enforced on a project or at the cluster level. | 31 Mar 2022 | [Rancher v2.6.4](https://github.com/rancher/rancher/releases/tag/v2.6.4) | +| [CVE-2021-36775](https://github.com/rancher/rancher/security/advisories/GHSA-28g7-896h-695v) | A vulnerability was discovered in Rancher versions up to and including 2.4.17, 2.5.11 and 2.6.2. After removing a `Project Role` associated with a group from the project, the bindings that granted access to cluster-scoped resources for those subjects were not deleted. This was due to an incomplete authorization logic check. A user who was a member of the affected group with authenticated access to Rancher could exploit this vulnerability to access resources they shouldn't have had access to. The exposure level would depend on the original permission level granted to the affected project role. This vulnerability only affected customers using group based authentication in Rancher. | 31 Mar 2022 | [Rancher v2.6.3](https://github.com/rancher/rancher/releases/tag/v2.6.3), [Rancher v2.5.12](https://github.com/rancher/rancher/releases/tag/v2.5.12) and [Rancher v2.4.18](https://github.com/rancher/rancher/releases/tag/v2.4.18) | +| [CVE-2021-36776](https://github.com/rancher/rancher/security/advisories/GHSA-gvh9-xgrq-r8hw) | A vulnerability was discovered in Rancher versions starting 2.5.0 up to and including 2.5.9, that allowed an authenticated user to impersonate any user on a cluster through an API proxy, without requiring knowledge of the impersonated user's credentials. This was due to the API proxy not dropping the impersonation header before sending the request to the Kubernetes API. A malicious user with authenticated access to Rancher could use this to impersonate another user with administrator access in Rancher, thereby gaining administrator level access to the cluster. | 31 Mar 2022 | [Rancher v2.6.0](https://github.com/rancher/rancher/releases/tag/v2.6.0) and [Rancher v2.5.10](https://github.com/rancher/rancher/releases/tag/v2.5.10) | +| [CVE-2021-25318](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25318) | A vulnerability was discovered in Rancher versions 2.0 through the aforementioned fixed versions, where users were granted access to resources regardless of the resource's API group. For example, Rancher should have allowed users access to `apps.catalog.cattle.io`, but instead incorrectly gave access to `apps.*`. Resources affected in the **Downstream clusters** and **Rancher management cluster** can be found [here](https://github.com/rancher/rancher/security/advisories/GHSA-f9xf-jq4j-vqw4). There is not a direct mitigation besides upgrading to the patched Rancher versions. | 14 Jul 2021 | [Rancher v2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) and [Rancher v2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-31999](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-31999) | A vulnerability was discovered in Rancher 2.0.0 through the aforementioned patched versions, where a malicious Rancher user could craft an API request directed at the proxy for the Kubernetes API of a managed cluster to gain access to information they do not have access to. This is done by passing the "Impersonate-User" or "Impersonate-Group" header in the Connection header, which is then correctly removed by the proxy. At this point, instead of impersonating the user and their permissions, the request will act as if it was from the Rancher management server and incorrectly return the information. The vulnerability is limited to valid Rancher users with some level of permissions on the cluster. There is not a direct mitigation besides upgrading to the patched Rancher versions. | 14 Jul 2021 | [Rancher v2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) and [Rancher v2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25320](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25320) | A vulnerability was discovered in Rancher 2.2.0 through the aforementioned patched versions, where cloud credentials weren't being properly validated through the Rancher API. Specifically through a proxy designed to communicate with cloud providers. Any Rancher user that was logged-in and aware of a cloud-credential ID that was valid for a given cloud provider, could call that cloud provider's API through the proxy API, and the cloud-credential would be attached. The exploit is limited to valid Rancher users. There is not a direct mitigation outside of upgrading to the patched Rancher versions. | 14 Jul 2021 | [Rancher v2.5.9](https://github.com/rancher/rancher/releases/tag/v2.5.9) and [Rancher v2.4.16](https://github.com/rancher/rancher/releases/tag/v2.4.16) | +| [CVE-2021-25313](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25313) | A security vulnerability was discovered on all Rancher 2 versions. When accessing the Rancher API with a browser, the URL was not properly escaped, making it vulnerable to an XSS attack. Specially crafted URLs to these API endpoints could include JavaScript which would be embedded in the page and execute in a browser. There is no direct mitigation. Avoid clicking on untrusted links to your Rancher server. | 2 Mar 2021 | [Rancher v2.5.6](https://github.com/rancher/rancher/releases/tag/v2.5.6), [Rancher v2.4.14](https://github.com/rancher/rancher/releases/tag/v2.4.14), and [Rancher v2.3.11](https://github.com/rancher/rancher/releases/tag/v2.3.11) | +| [CVE-2019-14435](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14435) | This vulnerability allows authenticated users to potentially extract otherwise private data out of IPs reachable from system service containers used by Rancher. This can include but not only limited to services such as cloud provider metadata services. Although Rancher allow users to configure whitelisted domains for system service access, this flaw can still be exploited by a carefully crafted HTTP request. The issue was found and reported by Matt Belisle and Alex Stevenson at Workiva. | 5 Aug 2019 | [Rancher v2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) and [Rancher v2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-14436](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-14436) | The vulnerability allows a member of a project that has access to edit role bindings to be able to assign themselves or others a cluster level role granting them administrator access to that cluster. The issue was found and reported by Michal Lipinski at Nokia. | 5 Aug 2019 | [Rancher v2.2.7](https://github.com/rancher/rancher/releases/tag/v2.2.7) and [Rancher v2.1.12](https://github.com/rancher/rancher/releases/tag/v2.1.12) | +| [CVE-2019-13209](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-13209) | The vulnerability is known as a [Cross-Site Websocket Hijacking attack](https://www.christian-schneider.net/CrossSiteWebSocketHijacking.html). This attack allows an exploiter to gain access to clusters managed by Rancher with the roles/permissions of a victim. It requires that a victim to be logged into a Rancher server and then access a third-party site hosted by the exploiter. Once that is accomplished, the exploiter is able to execute commands against the Kubernetes API with the permissions and identity of the victim. Reported by Matt Belisle and Alex Stevenson from Workiva. | 15 Jul 2019 | [Rancher v2.2.5](https://github.com/rancher/rancher/releases/tag/v2.2.5), [Rancher v2.1.11](https://github.com/rancher/rancher/releases/tag/v2.1.11) and [Rancher v2.0.16](https://github.com/rancher/rancher/releases/tag/v2.0.16) | +| [CVE-2019-12303](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12303) | Project owners can inject extra fluentd logging configurations that makes it possible to read files or execute arbitrary commands inside the fluentd container. Reported by Tyler Welton from Untamed Theory. | 5 Jun 2019 | [Rancher v2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4), [Rancher v2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) and [Rancher v2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-12274](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-12274) | Nodes using the built-in node drivers using a file path option allows the machine to read arbitrary files including sensitive ones from inside the Rancher server container. | 5 Jun 2019 | [Rancher v2.2.4](https://github.com/rancher/rancher/releases/tag/v2.2.4), [Rancher v2.1.10](https://github.com/rancher/rancher/releases/tag/v2.1.10) and [Rancher v2.0.15](https://github.com/rancher/rancher/releases/tag/v2.0.15) | +| [CVE-2019-11202](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11202) | The default admin, that is shipped with Rancher, will be re-created upon restart of Rancher despite being explicitly deleted. | 16 Apr 2019 | [Rancher v2.2.2](https://github.com/rancher/rancher/releases/tag/v2.2.2), [Rancher v2.1.9](https://github.com/rancher/rancher/releases/tag/v2.1.9) and [Rancher v2.0.14](https://github.com/rancher/rancher/releases/tag/v2.0.14) | +| [CVE-2019-6287](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-6287) | Project members continue to get access to namespaces from projects that they were removed from if they were added to more than one project. | 29 Jan 2019 | [Rancher v2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) and [Rancher v2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) | +| [CVE-2018-20321](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2018-20321) | Any project member with access to the `default` namespace can mount the `netes-default` service account in a pod and then use that pod to execute administrative privileged commands against the Kubernetes cluster. | 29 Jan 2019 | [Rancher v2.1.6](https://github.com/rancher/rancher/releases/tag/v2.1.6) and [Rancher v2.0.11](https://github.com/rancher/rancher/releases/tag/v2.0.11) - Rolling back from these versions or greater have specific [instructions](../../getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks.md). | diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md b/versioned_docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md new file mode 100644 index 00000000000..1d9d46c75d4 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rancher-selinux.md @@ -0,0 +1,70 @@ +--- +title: About rancher-selinux +--- + + + + + +To allow Rancher to work with SELinux, some functionality has to be manually enabled for the SELinux nodes. To help with that, Rancher provides a SELinux RPM. + +The `rancher-selinux` RPM only contains policies for the [rancher-logging application.](https://github.com/rancher/charts/tree/dev-v2.5/charts/rancher-logging) + +The `rancher-selinux` GitHub repository is [here.](https://github.com/rancher/rancher-selinux) + +## Installing the rancher-selinux RPM + +:::note Requirement: + +The rancher-selinux RPM was tested with CentOS 7 and 8. + +::: + +### 1. Set up the yum repo + +Set up the yum repo to install `rancher-selinux` directly on all hosts in the cluster. + +In order to use the RPM repository, on a CentOS 7 or RHEL 7 system, run the following bash snippet: + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/7/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` + +In order to use the RPM repository, on a CentOS 8 or RHEL 8 system, run the following bash snippet: + +``` +# cat << EOF > /etc/yum.repos.d/rancher.repo +[rancher] +name=Rancher +baseurl=https://rpm.rancher.io/rancher/production/centos/8/noarch +enabled=1 +gpgcheck=1 +gpgkey=https://rpm.rancher.io/public.key +EOF +``` +### 2. Installing the RPM + +Install the RPM: + +``` +yum -y install rancher-selinux +``` + +## Configuring the Logging Application to Work with SELinux + +:::note Requirement: + +Logging v2 was tested with SELinux on RHEL/CentOS 7 and 8. + +::: + +Applications do not automatically work once the `rancher-selinux` RPM is installed on the host. They need to be configured to run in an allowed SELinux container domain provided by the RPM. + +To configure the `rancher-logging` chart to be SELinux aware, change `global.seLinux.enabled` to true in the `values.yaml` when installing the chart. \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md b/versioned_docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md new file mode 100644 index 00000000000..c5edf5e0017 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-security/selinux-rpm/about-rke2-selinux.md @@ -0,0 +1,13 @@ +--- +title: About rke2-selinux +--- + + + + + +`rke2-selinux` provides policies for RKE2. It is installed automatically when the RKE2 installer script detects that it is running on an RPM-based distro. + +The `rke2-selinux` GitHub repository is [here.](https://github.com/rancher/rke2-selinux) + +For more information about installing RKE2 on SELinux-enabled hosts, see the [RKE2 documentation.](https://docs.rke2.io/install/methods#rpm) \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/rancher-webhook.md b/versioned_docs/version-latest/reference-guides/rancher-webhook.md new file mode 100644 index 00000000000..1d2a17275b4 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rancher-webhook.md @@ -0,0 +1,106 @@ +--- +title: Rancher Webhook +--- + + + + + +Rancher-Webhook is an essential component of Rancher that works in conjunction with Kubernetes to enhance security and enable critical features for Rancher-managed clusters. + +It integrates with Kubernetes' extensible admission controllers, as described in the [Kubernetes documentation](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), which allows Rancher-Webhook to inspect specific requests sent to the Kubernetes API server, and add custom, Rancher-specific validation and mutations to the requests that are specific to Rancher. Rancher-Webhook manages the resources to be validated using the `rancher.cattle.io` `ValidatingWebhookConfiguration` and the `rancher.cattle.io` `MutatingWebhookConfiguration`, and will override any manual edits. +Rancher deploys Rancher-Webhook as a separate deployment and service in both local and downstream clusters. Rancher manages Rancher-Webhook using Helm. It's important to note that Rancher may override modifications made by users to the Helm release. + +Each Rancher version is designed to be compatible with a single version of the webhook. The compatible versions are provided below for convenience. + +**Note:** Rancher manages deployment and upgrade of the webhook. Under most circumstances, no user intervention should be needed to ensure that the webhook version is compatible with the version of Rancher that you are running. + +| Rancher Version | Webhook Version | +|-----------------|:---------------:| +| v2.7.0 | v0.3.0 | +| v2.7.1 | v0.3.0 | +| v2.7.2 | v0.3.2 | +| v2.7.3 | v0.3.3 | +| v2.7.4 | v0.3.4 | +| v2.7.5 | v0.3.5 | +| v2.7.6 | v0.3.5 | + + +## Why Do We Need It? + +Rancher-Webhook is crucial for Rancher to protect clusters against malicious attacks and enable various features. +Rancher relies on the Rancher-Webhook as an integral part of its functionality. Without the webhook, Rancher would not be a complete product. +It provides essential protection for Rancher-managed clusters, preventing security vulnerabilities and ensuring the consistency and stability of the cluster. + +## What Resources Does the Webhook Validate? + +An in-progress list of the resources that the webhook validates can be found in the [webhook's repo](https://github.com/rancher/webhook/blob/release/v0.4/docs.md). Checks specific to one version can be found by viewing the `docs.md` file associated with a particular tag (note that webhook versions prior to `v0.3.6` won't have this file). + +## Bypassing the Webhook + +Sometimes, it may be necessary to bypass Rancher's webhook validation to perform emergency restore operations, or fix other critical issues. The bypass operation is exhaustive, meaning that no webhook checks will apply when this is used. It is not possible to bypass some checks and have others still apply - they are either all bypassed, or all active. + +:::danger + +Rancher's webhook provides critical security protections. Bypassing the webhook should only be done by administrators in specific scenarios, after all other options have been exhausted. In addition, permission to bypass the webhook should be carefully controlled, and never given to users who are not admins. + +::: + +To bypass the webhook, impersonate both the `rancher-webhook-sudo` service account and the `system:masters` group (both are required): + +```bash +kubectl create -f example.yaml --as=system:serviceaccount:cattle-system:rancher-webhook-sudo --as-group=system:masters +``` + +## Common Issues + +### EKS Cluster with Calico CNI + +Users running an EKS cluster with Calico CNI may run into errors when the Kubernetes API server attempts to contact the Rancher-Webhook. +One workaround for this issue [documented by Calico](https://docs.tigera.io/calico/latest/getting-started/kubernetes/managed-public-cloud/eks#install-eks-with-calico-networking) involves setting `hostNetwork=true` for the webhook deployment. Users can change this using the Helm commands below on the affected clusters. + +``` bash +helm repo add rancher-charts https://charts.rancher.io +helm upgrade --reuse-values rancher-webhook rancher-chart/rancher-webhook -n cattle-system --set global.hostNetwork=true +``` +**Note:** This temporary workaround may violate an environment's security policy. This workaround also requires that port 9443 is unused on the host network. + +### Private GKE Cluster + +When using a private GKE cluster, errors may occur that prevent the Kubernetes API server from communicating with the webhook. The following error message may appear: + +``` +Internal error occurred: failed calling webhook "rancher.cattle.io.namespaces.create-non-kubesystem": failed to call webhook: Post "https://rancher-webhook.cattle-system.svc:443/v1/webhook/validation/namespaces?timeout=10s": context deadline exceeded +``` +This issue occurs because firewall rules restrict communication between the API server and the private cluster. To resolve this communication problem, users must add firewall rules to allow the GKE control plane to communicate with the Rancher-Webhook on port 9443. Please refer to the [GKE documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) for detailed information and steps on updating the firewall rules. + +### Application Fails to Deploy Due to rancher-webhook Blocking Access + +The webhook provides extra validations on [namespaces](https://github.com/rancher/webhook/blob/release/v0.4/docs.md#psa-label-validation). One of these validations ensures that users can only update PSA relevant labels if they have the proper permissions (`updatepsa` for `projects` in `management.cattle.io`). This can result in specific operators, such as Tigera or Trident, failing when they attempt to deploy namespaces with PSA labels. There are several ways to resolve this issue: + +- Configure the application to create a namespace with no PSA labels. If users wish to apply a PSA to these namespaces, they can add them to a project with the desired PSA after configuration. See the [docs on PSS and PSA resources](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards) for instructions on how. + - This is the preferred option, though not all applications can be configured in this fashion. +- Manually grant the operator permissions to manage PSAs for namespaces. + - This option will introduce security risks, since the operator will now be able to set the PSA for the namespaces it has access to. This could allow the operator to deploy a privileged pod, or effect cluster takeover through other means. +- A user account with the proper permissions can pre-create the namespace with the appropriate configuration. + - This option depends on the ability of the application to handle existing resources. + +## Issues on Specific Versions + +**Note:** The following is an incomplete list of high-severity issues affecting specific Rancher/webhook versions. In most cases, these issues can be resolved by upgrading to a more recent Rancher version. + +### Incompatible Webhook Version on Rollback + +**Note:** This affects rolling back to Rancher v2.7.5 or earlier. + +If you roll back to Rancher v2.7.5 or earlier, you may see webhook versions that are too recent to be compatible with downstream clusters running pre-v2.7.5 version of Rancher. This may cause various incompatibility issues. For example, project members may be unable to create namespaces. In addition, when you roll back to versions before the webhook was installed in downstream clusters, the webhook may remain installed, which can result in similar incompatibility issues. + +To help alleviate these issues, you can run the [adjust-downstream-webhook](https://github.com/rancherlabs/support-tools/tree/master/adjust-downstream-webhook) shell script after roll back. This script selects and installs the proper webhook version (or removes the webhook entirely) for the corresponding Rancher version. + +### Project Members Can't Create Namespaces + +**Note:** This affects Rancher versions `v2.7.2 - v2.7.4` + +Project users who aren't owners may not be able to create namespaces in projects. This issue is caused by Rancher automatically upgrading the webhook to a version compatible with a more recent version of Rancher than the one currently installed. + +To help alleviate these issues, you can run the [adjust-downstream-webhook](https://github.com/rancherlabs/support-tools/tree/master/adjust-downstream-webhook) shell script after roll back. This script selects and installs the proper webhook version (or removes the webhook entirely) for the corresponding Rancher version. diff --git a/versioned_docs/version-latest/reference-guides/rke1-template-example-yaml.md b/versioned_docs/version-latest/reference-guides/rke1-template-example-yaml.md new file mode 100644 index 00000000000..d14e3863ff9 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/rke1-template-example-yaml.md @@ -0,0 +1,111 @@ +--- +title: RKE1 Example YAML +--- + +Below is an example RKE template configuration file for reference. + +The YAML in the RKE template uses the same customization that is used when you create an RKE cluster. However, since the YAML is within the context of a Rancher provisioned RKE cluster, the customization from the RKE docs needs to be nested under the `rancher_kubernetes_engine` directive. + +```yaml +# +# Cluster Config +# +docker_root_dir: /var/lib/docker + +enable_cluster_alerting: false +# This setting is not enforced. Clusters +# created with this sample template +# would have alerting turned off by default, +# but end users could still turn alerting +# on or off. + +enable_cluster_monitoring: true +# This setting is not enforced. Clusters +# created with this sample template +# would have monitoring turned on +# by default, but end users could still +# turn monitoring on or off. + +enable_network_policy: false +local_cluster_auth_endpoint: + enabled: true +# +# Rancher Config +# +rancher_kubernetes_engine_config: # Your RKE template config goes here. + addon_job_timeout: 30 + authentication: + strategy: x509 + ignore_docker_version: true +# +# # Currently only nginx ingress provider is supported. +# # To disable ingress controller, set `provider: none` +# # To enable ingress on specific nodes, use the node_selector, eg: +# provider: nginx +# node_selector: +# app: ingress +# + ingress: + provider: nginx + kubernetes_version: v1.15.3-rancher3-1 + monitoring: + provider: metrics-server +# +# If you are using calico on AWS +# +# network: +# plugin: calico +# calico_network_provider: +# cloud_provider: aws +# +# # To specify flannel interface +# +# network: +# plugin: flannel +# flannel_network_provider: +# iface: eth1 +# +# # To specify flannel interface for canal plugin +# +# network: +# plugin: canal +# canal_network_provider: +# iface: eth1 +# + network: + options: + flannel_backend_type: vxlan + plugin: canal +# +# services: +# kube-api: +# service_cluster_ip_range: 10.43.0.0/16 +# kube-controller: +# cluster_cidr: 10.42.0.0/16 +# service_cluster_ip_range: 10.43.0.0/16 +# kubelet: +# cluster_domain: cluster.local +# cluster_dns_server: 10.43.0.10 +# + services: + etcd: + backup_config: + enabled: true + interval_hours: 12 + retention: 6 + safe_timestamp: false + creation: 12h + extra_args: + election-timeout: 5000 + heartbeat-interval: 500 + gid: 0 + retention: 72h + snapshot: false + uid: 0 + kube_api: + always_pull_images: false + pod_security_policy: false + service_node_port_range: 30000-32767 + ssh_agent_auth: false +windows_prefered_cluster: false +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/single-node-rancher-in-docker/advanced-options.md b/versioned_docs/version-latest/reference-guides/single-node-rancher-in-docker/advanced-options.md new file mode 100644 index 00000000000..d020910622b --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/single-node-rancher-in-docker/advanced-options.md @@ -0,0 +1,105 @@ +--- +title: Advanced Options for Docker Installs +--- + +### Custom CA Certificate + +If you want to configure Rancher to use a CA root certificate to be used when validating services, you would start the Rancher container sharing the directory that contains the CA root certificate. + +Use the command example to start a Rancher container with your private CA certificates mounted. + +- The volume flag (`-v`) should specify the host directory containing the CA root certificates. +- The environment variable flag (`-e`) in combination with `SSL_CERT_DIR` and directory declares an environment variable that specifies the mounted CA root certificates directory location inside the container. +- Passing environment variables to the Rancher container can be done using `-e KEY=VALUE` or `--env KEY=VALUE`. +- Mounting a host directory inside the container can be done using `-v host-source-directory:container-destination-directory` or `--volume host-source-directory:container-destination-directory`. + +The example below is based on having the CA root certificates in the `/host/certs` directory on the host and mounting this directory on `/container/certs` inside the Rancher container. + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /host/certs:/container/certs \ + -e SSL_CERT_DIR="/container/certs" \ + --privileged \ + rancher/rancher:latest +``` + +### API Audit Log + +The API Audit Log records all the user and system transactions made through Rancher server. + +The API Audit Log writes to `/var/log/auditlog` inside the rancher container by default. Share that directory as a volume and set your `AUDIT_LEVEL` to enable the log. + +See [API Audit Log](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) for more information and options. + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /var/log/rancher/auditlog:/var/log/auditlog \ + -e AUDIT_LEVEL=1 \ + --privileged \ + rancher/rancher:latest +``` + +### TLS settings + +To set a different TLS configuration, you can use the `CATTLE_TLS_MIN_VERSION` and `CATTLE_TLS_CIPHERS` environment variables. For example, to configure TLS 1.0 as minimum accepted TLS version: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e CATTLE_TLS_MIN_VERSION="1.0" \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +See [TLS settings](../../getting-started/installation-and-upgrade/installation-references/tls-settings.md) for more information and options. + +### Air Gap + +If you are visiting this page to complete an air gap installation, you must prepend your private registry URL to the server tag when running the installation command in the option that you choose. Add `` with your private registry URL in front of `rancher/rancher:latest`. + +**Example:** + + /rancher/rancher:latest + +### Persistent Data + +Rancher uses etcd as a datastore. When Rancher is installed with Docker, the embedded etcd is being used. The persistent data is at the following path in the container: `/var/lib/rancher`. + +You can bind mount a host volume to this location to preserve data on the host it is running on: + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -v /opt/rancher:/var/lib/rancher \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +### Running `rancher/rancher` and `rancher/rancher-agent` on the Same Node + +In the situation where you want to use a single node to run Rancher and to be able to add the same node to a cluster, you have to adjust the host ports mapped for the `rancher/rancher` container. + +If a node is added to a cluster, it deploys the nginx ingress controller which will use port 80 and 443. This will conflict with the default ports we advise to expose for the `rancher/rancher` container. + +Please note that this setup is not recommended for production use, but can be convenient for development/demo purposes. + +To change the host ports mapping, replace the following part `-p 80:80 -p 443:443` with `-p 8080:80 -p 8443:443`: + +``` +docker run -d --restart=unless-stopped \ + -p 8080:80 -p 8443:443 \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) diff --git a/versioned_docs/version-latest/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md b/versioned_docs/version-latest/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md new file mode 100644 index 00000000000..22210ccd8ff --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/single-node-rancher-in-docker/http-proxy-configuration.md @@ -0,0 +1,61 @@ +--- +title: HTTP Proxy Configuration +--- + +If you operate Rancher behind a proxy and you want to access services through the proxy (such as retrieving catalogs), you must provide Rancher information about your proxy. As Rancher is written in Go, it uses the common proxy environment variables as shown below. + +Make sure `NO_PROXY` contains the network addresses, network address ranges and domains that should be excluded from using the proxy. + +| Environment variable | Purpose | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| HTTP_PROXY | Proxy address to use when initiating HTTP connection(s) | +| HTTPS_PROXY | Proxy address to use when initiating HTTPS connection(s) | +| NO_PROXY | Network address(es), network address range(s) and domains to exclude from using the proxy when initiating connection(s) | + +:::note Important: + +NO_PROXY must be in uppercase to use network range (CIDR) notation. + +::: + +## Docker Installation + +Passing environment variables to the Rancher container can be done using `-e KEY=VALUE` or `--env KEY=VALUE`. Required values for `NO_PROXY` in a [Docker Installation](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md) are: + +- `localhost` +- `127.0.0.1` +- `0.0.0.0` +- `10.0.0.0/8` +- `cattle-system.svc` +- `.svc` +- `.cluster.local` + +The example below is based on a proxy server accessible at `http://192.168.0.1:3128`, and excluding usage the proxy when accessing network range `192.168.10.0/24` and every hostname under the domain `example.com`. + +``` +docker run -d --restart=unless-stopped \ + -p 80:80 -p 443:443 \ + -e HTTP_PROXY="http://192.168.10.1:3128" \ + -e HTTPS_PROXY="http://192.168.10.1:3128" \ + -e NO_PROXY="localhost,127.0.0.1,0.0.0.0,10.0.0.0/8,cattle-system.svc,192.168.10.0/24,.svc,.cluster.local,example.com" \ + --privileged \ + rancher/rancher:latest +``` + +Privileged access is [required.](../../pages-for-subheaders/rancher-on-a-single-node-with-docker.md#privileged-access-for-rancher) + +### Air-gapped proxy configuration + +You can now provision node driver clusters from an air-gapped cluster configured to use a proxy for outbound connections. + +In addition to setting the default rules for a proxy server as shown above, you will need to add additional rules, shown below, to provision node driver clusters from a proxied Rancher environment. + +You will configure your filepath according to your setup, e.g., `/etc/apt/apt.conf.d/proxy.conf`: + +``` +acl SSL_ports port 22 +acl SSL_ports port 2376 + +acl Safe_ports port 22 # ssh +acl Safe_ports port 2376 # docker port +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/reference-guides/system-tools.md b/versioned_docs/version-latest/reference-guides/system-tools.md new file mode 100644 index 00000000000..d9480976757 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/system-tools.md @@ -0,0 +1,31 @@ +--- +title: System Tools +--- + +:::note + +System Tools has been deprecated since June 2022. + +::: + +## Logs + +Please use [logs-collector](https://github.com/rancherlabs/support-tools/tree/master/collection/rancher/v2.x/logs-collector) to collect logs from your cluster. + +## Stats + +If you want to replicate the stats command, you can run the following command on your cluster nodes: + +:::note + +This command below requires the package `sysstat` on the cluster node. + +::: + +``` +/usr/bin/sar -u -r -F 1 1 +``` + +## Remove + +Please use the [Rancher Cleanup](https://github.com/rancher/rancher-cleanup) tool. diff --git a/versioned_docs/version-latest/reference-guides/user-settings/api-keys.md b/versioned_docs/version-latest/reference-guides/user-settings/api-keys.md new file mode 100644 index 00000000000..ade1aee3d9b --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/user-settings/api-keys.md @@ -0,0 +1,59 @@ +--- +title: API Keys +--- + +## API Keys and User Authentication + +If you want to access your Rancher clusters, projects, or other objects using external applications, you can do so using the Rancher API. However, before your application can access the API, you must provide the app with a key used to authenticate with Rancher. You can obtain a key using the Rancher UI. + +An API key is also required for using Rancher CLI. + +API Keys are composed of four components: + +- **Endpoint:** This is the IP address and path that other applications use to send requests to the Rancher API. +- **Access Key:** The token's username. +- **Secret Key:** The token's password. For applications that prompt you for two different strings for API authentication, you usually enter the two keys together. +- **Bearer Token:** The token username and password concatenated together. Use this string for applications that prompt you for one authentication string. + +:::note + +Users may opt to enable [token hashing](../about-the-api/api-tokens.md). + +::: + +## Creating an API Key + +1. Select **User Avatar > Account & API Keys** from upper right corner. + +2. Click **Create API Key**. + +3. **Optional:** Enter a description for the API key and select an expiration period or a scope. We recommend setting an expiration date. + + The API key won't be valid after expiration. Shorter expiration periods are more secure. + + Expiration period will be bound by `v3/settings/auth-token-max-ttl-minutes`. If it exceeds the max-ttl, API key will be created with max-ttl as the expiration period. + + A scope will limit the API key so that it will only work against the Kubernetes API of the specified cluster. If the cluster is configured with an Authorized Cluster Endpoint, you will be able to use a scoped token directly against the cluster's API without proxying through the Rancher server. See [Authorized Cluster Endpoints](../../reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters.md#4-authorized-cluster-endpoint) for more information. + +4. Click **Create**. + + **Step Result:** Your API Key is created. Your API **Endpoint**, **Access Key**, **Secret Key**, and **Bearer Token** are displayed. + + Use the **Bearer Token** to authenticate with Rancher CLI. + +5. Copy the information displayed to a secure location. This information is only displayed once, so if you lose your key, you'll have to make a new one. + +## What's Next? + +- Enter your API key information into the application that will send requests to the Rancher API. +- Learn more about the Rancher endpoints and parameters by selecting **View in API** for an object in the Rancher UI. +- API keys are used for API calls and [Rancher CLI](../../pages-for-subheaders/cli-with-rancher.md). + +## Deleting API Keys + +If you need to revoke an API key, delete it. You should delete API keys: + +- That may have been compromised. +- That have expired. + +To delete an API, select the stale key and click **Delete**. diff --git a/versioned_docs/version-latest/reference-guides/user-settings/manage-cloud-credentials.md b/versioned_docs/version-latest/reference-guides/user-settings/manage-cloud-credentials.md new file mode 100644 index 00000000000..e387346c02c --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/user-settings/manage-cloud-credentials.md @@ -0,0 +1,51 @@ +--- +title: Managing Cloud Credentials +--- + +When you create a cluster [hosted by an infrastructure provider](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md), [node templates](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) are used to provision the cluster nodes. These templates use Docker Machine configuration options to define an operating system image and settings/parameters for the node. + +Node templates can use cloud credentials to access the credential information required to provision nodes in the infrastructure providers. The same cloud credential can be used by multiple node templates. By using a cloud credential, you do not have to re-enter access keys for the same cloud provider. Cloud credentials are stored as Kubernetes secrets. + +Cloud credentials are only used by node templates if there are fields marked as `password`. The default `active` node drivers have their account access fields marked as `password`, but there may be some `inactive` node drivers, which are not using them yet. These node drivers will not use cloud credentials. + +You can create cloud credentials in two contexts: + +- [During creation of a node template](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) for a cluster. +- In the **User Settings** + +All cloud credentials are bound to the user profile of who created it. They **cannot** be shared across users. + +## Creating a Cloud Credential from User Settings + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Click **Create**. +1. Click a cloud credential type. The values of this dropdown is based on the `active` [node drivers](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md) in Rancher. +1. Enter a name for the cloud credential. +1. Based on the selected cloud credential type, enter the required values to authenticate with the infrastructure provider. +1. Click **Create**. + +**Result:** The cloud credential is created and can immediately be used to [create node templates](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates). + +## Updating a Cloud Credential + +When access credentials are changed or compromised, updating a cloud credential allows you to rotate those credentials while keeping the same node template. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. Choose the cloud credential you want to edit and click the **⋮ > Edit Config**. +1. Update the credential information and click **Save**. + +**Result:** The cloud credential is updated with the new access credentials. All existing node templates using this cloud credential will automatically use the updated information whenever [new nodes are added](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +## Deleting a Cloud Credential + +In order to delete cloud credentials, there must not be any node template associated with it. If you are unable to delete the cloud credential, [delete any node templates](manage-node-templates.md#deleting-a-node-template) that are still associated to that cloud credential. + +1. Click **☰ > Cluster Management**. +1. Click **Cloud Credentials**. +1. You can either individually delete a cloud credential or bulk delete. + + - To individually delete one, choose the cloud credential you want to edit and click the **⋮ > Delete**. + - To bulk delete cloud credentials, select one or more cloud credentials from the list. Click **Delete**. +1. Confirm that you want to delete these cloud credentials. diff --git a/versioned_docs/version-latest/reference-guides/user-settings/manage-node-templates.md b/versioned_docs/version-latest/reference-guides/user-settings/manage-node-templates.md new file mode 100644 index 00000000000..22c29e0ae54 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/user-settings/manage-node-templates.md @@ -0,0 +1,54 @@ +--- +title: Managing Node Templates +--- + +When you provision a cluster [hosted by an infrastructure provider](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md), [node templates](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#node-templates) are used to provision the cluster nodes. These templates use Docker Machine configuration options to define an operating system image and settings/parameters for the node. You can create node templates in two contexts: + +- While [provisioning a node pool cluster](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). +- At any time, from your [user settings](../../pages-for-subheaders/user-settings.md). + +When you create a node template, it is bound to your user profile. Node templates cannot be shared among users. You can delete stale node templates that you no longer user from your user settings. + +## Creating a Node Template + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Click **Add Template**. +1. Select one of the cloud providers available. Then follow the instructions on screen to configure the template. + +**Result:** The template is configured. You can use the template later when you [provision a node pool cluster](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +## Updating a Node Template + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Choose the node template that you want to edit and click the **⋮ > Edit**. + + :::note + + The default `active` [node drivers](../../how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers.md) and any node driver, that has fields marked as `password`, are required to use [cloud credentials](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md#cloud-credentials). + + ::: + +1. Edit the required information and click **Save**. + +**Result:** The node template is updated. All node pools using this node template will automatically use the updated information when new nodes are added. + +## Cloning Node Templates + +When creating new node templates from your user settings, you can clone an existing template and quickly update its settings rather than creating a new one from scratch. Cloning templates saves you the hassle of re-entering access keys for the cloud provider. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Find the template you want to clone. Then select **⋮ > Clone**. +1. Complete the rest of the form. + +**Result:** The template is cloned and configured. You can use the template later when you [provision a node pool cluster](../../pages-for-subheaders/use-new-nodes-in-an-infra-provider.md). + +## Deleting a Node Template + +When you no longer use a node template, you can delete it from your user settings. + +1. Click **☰ > Cluster Management**. +1. Click **RKE1 Configuration > Node Templates**. +1. Select one or more template from the list. Then click **Delete**. Confirm the delete when prompted. diff --git a/versioned_docs/version-latest/reference-guides/user-settings/user-preferences.md b/versioned_docs/version-latest/reference-guides/user-settings/user-preferences.md new file mode 100644 index 00000000000..3a36abd6e18 --- /dev/null +++ b/versioned_docs/version-latest/reference-guides/user-settings/user-preferences.md @@ -0,0 +1,62 @@ +--- +title: User Preferences +--- + +You can set preferences to personalize your Rancher experience. To change preference settings: + +1. Click on your user avatar in the upper right corner. +1. Click **Preferences**. + +## Language + +Choose your language for the Rancher UI. Options include: + +- English +- 简体中文 (Simplified Chinese) + +## Theme + +Choose your background color for the Rancher UI. If you choose **Auto**, the background color changes from light to dark at 6 PM, and then changes back at 6 AM. + +## Login Landing Page + +Choose the page displayed after logging in. Options include: + +- The home page. +- The area you last visited. +- A specific cluster of your choosing. + +## Display Settings + +Choose how certain information is displayed: + +- Date format +- Time format +- Table rows per page +- Number of clusters to show in side menu + +## Confirmation Setting + +_Available as of v2.7.2_ + +Choose whether to ask for confirmation when scaling down node pools. + +## Advanced Features + +- Enable "View in API". +- Show system Namespaces managed by Rancher (not intended for editing or deletion). +- Enable Dark/Light Theme keyboard shortcut toggle (shift+T). +- Hide All Type Descriptions. +- Enable Extension developer features. + +![](/img/user-preferences-2-7.png) + +## YAML Editor Key Mapping + +- Normal human (Sublime) +- Emacs +- Vim + +## Helm Charts + +Choose whether to display released Helm charts only or to include prerelease versions as well. A version is considered to be a prerelease if it adheres to [the specification](https://semver.org/#spec-item-9) defined by [Semantic Versioning 2.0.0](https://semver.org/). For example, a Helm chart with a version of `0.1.3-dev.12ab4f` requires `Include Prerelease Versions` to be selected in order to be displayed. diff --git a/versioned_docs/version-latest/security/security-scan/security-scan.md b/versioned_docs/version-latest/security/security-scan/security-scan.md new file mode 100644 index 00000000000..061d0af8edd --- /dev/null +++ b/versioned_docs/version-latest/security/security-scan/security-scan.md @@ -0,0 +1,9 @@ +--- +title: Security Scans +--- + + + + + +The documentation about CIS security scans has moved [here.](../../pages-for-subheaders/cis-scan-guides.md) diff --git a/versioned_docs/version-latest/shared-files/_cluster-capabilities-table.md b/versioned_docs/version-latest/shared-files/_cluster-capabilities-table.md new file mode 100644 index 00000000000..b74973f5a67 --- /dev/null +++ b/versioned_docs/version-latest/shared-files/_cluster-capabilities-table.md @@ -0,0 +1,23 @@ +| Action | Rancher Launched Kubernetes Clusters | EKS, GKE and AKS Clusters1 | Other Hosted Kubernetes Clusters | Non-EKS or GKE Registered Clusters | +| --- | --- | ---| ---|----| +| [Using kubectl and a kubeconfig file to Access a Cluster](../how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig.md) | ✓ | ✓ | ✓ | ✓ | +| [Managing Cluster Members](../how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters.md) | ✓ | ✓ | ✓ | ✓ | +| [Editing and Upgrading Clusters](../pages-for-subheaders/cluster-configuration.md) | ✓ | ✓ | ✓ | ✓2 | +| [Managing Nodes](../how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools.md) | ✓ | ✓ | ✓ | ✓3 | +| [Managing Persistent Volumes and Storage Classes](../pages-for-subheaders/create-kubernetes-persistent-storage.md) | ✓ | ✓ | ✓ | ✓ | +| [Managing Projects, Namespaces and Workloads](../how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces.md) | ✓ | ✓ | ✓ | ✓ | +| [Using App Catalogs](../pages-for-subheaders/helm-charts-in-rancher.md) | ✓ | ✓ | ✓ | ✓ | +| Configuring Tools ([Alerts, Notifiers, Monitoring](../pages-for-subheaders/monitoring-and-alerting.md), [Logging](../pages-for-subheaders/logging.md), [Istio](../pages-for-subheaders/istio.md)) | ✓ | ✓ | ✓ | ✓ | +| [Running Security Scans](../pages-for-subheaders/cis-scan-guides.md) | ✓ | ✓ | ✓ | ✓ | +| [Ability to rotate certificates](../how-to-guides/new-user-guides/manage-clusters/rotate-certificates.md) | ✓ | ✓ | | | +| Ability to [backup](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters.md) and [restore](../how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup.md) Rancher-launched clusters | ✓ | ✓ | | ✓4 | +| [Cleaning Kubernetes components when clusters are no longer reachable from Rancher](../how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes.md) | ✓ | | | | +| [Configuring Pod Security Policies](../how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy.md) | ✓ | ✓ | || + +1. Registered EKS, GKE and AKS clusters have the same options available as EKS, GKE and AKS clusters created from the Rancher UI. The difference is that when a registered cluster is deleted from the Rancher UI, it is not destroyed. + +2. Cluster configuration options can't be edited for registered clusters, except for [K3s and RKE2 clusters.](../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters.md) + +3. For registered cluster nodes, the Rancher UI exposes the ability to cordon, drain, and edit the node. + +4. For registered clusters using etcd as a control plane, snapshots must be taken manually outside of the Rancher UI to use for backup and recovery. diff --git a/versioned_docs/version-latest/shared-files/_common-ports-table.md b/versioned_docs/version-latest/shared-files/_common-ports-table.md new file mode 100644 index 00000000000..1835beba031 --- /dev/null +++ b/versioned_docs/version-latest/shared-files/_common-ports-table.md @@ -0,0 +1,19 @@ +| Protocol | Port | Description | +|:--------: |:----------------: |---------------------------------------------------------------------------------- | +| TCP | 22 | Node driver SSH provisioning | +| TCP | 179 | Calico BGP Port | +| TCP | 2376 | Node driver Docker daemon TLS port | +| TCP | 2379 | etcd client requests | +| TCP | 2380 | etcd peer communication | +| UDP | 8472 | Canal/Flannel VXLAN overlay networking | +| UDP | 4789 | Flannel VXLAN overlay networking on Windows cluster | +| TCP | 8443 | Rancher webhook | +| TCP | 9099 | Canal/Flannel livenessProbe/readinessProbe | +| TCP | 9100 | Default port required by Monitoring to scrape metrics from Linux node-exporters | +| TCP | 9443 | Rancher webhook | +| TCP | 9796 | Default port required by Monitoring to scrape metrics from Windows node-exporters | +| TCP | 6783 | Weave Port | +| UDP | 6783-6784 | Weave UDP Ports | +| TCP | 10250 | Metrics server communication with all nodes API | +| TCP | 10254 | Ingress controller livenessProbe/readinessProbe | +| TCP/UDP | 30000-32767 | NodePort port range | diff --git a/versioned_docs/version-latest/troubleshooting/general-troubleshooting.md b/versioned_docs/version-latest/troubleshooting/general-troubleshooting.md new file mode 100644 index 00000000000..77ab7e24806 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/general-troubleshooting.md @@ -0,0 +1,48 @@ +--- +title: General Troubleshooting +--- + + + + + +This section contains information to help you troubleshoot issues when using Rancher. + +- [Kubernetes components](../pages-for-subheaders/kubernetes-components.md) + + If you need help troubleshooting core Kubernetes cluster components like: + * `etcd` + * `kube-apiserver` + * `kube-controller-manager` + * `kube-scheduler` + * `kubelet` + * `kube-proxy` + * `nginx-proxy` + +- [Kubernetes resources](other-troubleshooting-tips/kubernetes-resources.md) + + Options for troubleshooting Kubernetes resources like Nodes, Ingress Controller and Rancher Agents are described in this section. + +- [Networking](other-troubleshooting-tips/networking.md) + + Steps to troubleshoot networking issues can be found here. + +- [DNS](other-troubleshooting-tips/dns.md) + + When you experience name resolution issues in your cluster. + +- [Troubleshooting Rancher installed on Kubernetes](other-troubleshooting-tips/rancher-ha.md) + + If you experience issues with your [Rancher server installed on Kubernetes](../pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster.md) + +- [Logging](other-troubleshooting-tips/logging.md) + + Read more about what log levels can be configured and how to configure a log level. + +- [User ID Tracking in Audit Logs](other-troubleshooting-tips/user-id-tracking-in-audit-logs.md) + + Read more about how a Rancher Admin can trace an event from the Rancher audit logs and into the Kubernetes audit logs using the external Identity Provider username. + +- [Expired Webhook Certificates](other-troubleshooting-tips/expired-webhook-certificate-rotation.md) + + Read more about how to rotate a Rancher webhook certificate secret after it expires on an annual basis. diff --git a/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md new file mode 100644 index 00000000000..84877bc8fab --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes.md @@ -0,0 +1,55 @@ +--- +title: Troubleshooting Controlplane Nodes +--- + + + + + +This section applies to nodes with the `controlplane` role. + +## Check if the Controlplane Containers are Running + +There are three specific containers launched on nodes with the `controlplane` role: + +* `kube-apiserver` +* `kube-controller-manager` +* `kube-scheduler` + +The containers should have status **Up**. The duration shown after **Up** is the time the container has been running. + +``` +docker ps -a -f=name='kube-apiserver|kube-controller-manager|kube-scheduler' +``` + +Example output: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +26c7159abbcc rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-apiserver +f3d287ca4549 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-scheduler +bdf3898b8063 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-controller-manager +``` + +## Controlplane Container Logging + +:::note + +If you added multiple nodes with the `controlplane` role, both `kube-controller-manager` and `kube-scheduler` use a leader election process to determine the leader. Only the current leader will log the performed actions. See [Kubernetes leader election](../other-troubleshooting-tips/kubernetes-resources.md#kubernetes-leader-election) how to retrieve the current leader. + +::: + +The logging of the containers can contain information on what the problem could be. + +``` +docker logs kube-apiserver +docker logs kube-controller-manager +docker logs kube-scheduler +``` + +## RKE2 Server Logging + +If Rancher provisions an RKE2 cluster that can't communicate with Rancher, you can run this command on a server node in the downstream cluster to get the RKE2 server logs: + +``` +journalctl -u rke2-server -f +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md new file mode 100644 index 00000000000..9785d8e4f68 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-etcd-nodes.md @@ -0,0 +1,369 @@ +--- +title: Troubleshooting etcd Nodes +--- + + + + + +This section contains commands and tips for troubleshooting nodes with the `etcd` role. + + +## Checking if the etcd Container is Running + +The container for etcd should have status **Up**. The duration shown after **Up** is the time the container has been running. + +``` +docker ps -a -f=name=etcd$ +``` + +Example output: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +605a124503b9 rancher/coreos-etcd:v3.2.18 "/usr/local/bin/et..." 2 hours ago Up 2 hours etcd +``` + +## etcd Container Logging + +The logging of the container can contain information on what the problem could be. + +``` +docker logs etcd +``` +| Log | Explanation | +|-----|------------------| +| `health check for peer xxx could not connect: dial tcp IP:2380: getsockopt: connection refused` | A connection to the address shown on port 2380 cannot be established. Check if the etcd container is running on the host with the address shown. | +| `xxx is starting a new election at term x` | The etcd cluster has lost its quorum and is trying to establish a new leader. This can happen when the majority of the nodes running etcd go down/unreachable. | +| `connection error: desc = "transport: Error while dialing dial tcp 0.0.0.0:2379: i/o timeout"; Reconnecting to {0.0.0.0:2379 0 }` | The host firewall is preventing network communication. | +| `rafthttp: request cluster ID mismatch` | The node with the etcd instance logging `rafthttp: request cluster ID mismatch` is trying to join a cluster that has already been formed with another peer. The node should be removed from the cluster, and re-added. | +| `rafthttp: failed to find member` | The cluster state (`/var/lib/etcd`) contains wrong information to join the cluster. The node should be removed from the cluster, the state directory should be cleaned and the node should be re-added. + +## etcd Cluster and Connectivity Checks + +The address where etcd is listening depends on the address configuration of the host etcd is running on. If an internal address is configured for the host etcd is running on, the endpoint for `etcdctl` needs to be specified explicitly. If any of the commands respond with `Error: context deadline exceeded`, the etcd instance is unhealthy (either quorum is lost or the instance is not correctly joined in the cluster) + +### Check etcd Members on all Nodes + +Output should contain all the nodes with the `etcd` role and the output should be identical on all nodes. + +Command: +``` +docker exec etcd etcdctl member list +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list" +``` + +Example output: +``` +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +xxx, started, etcd-xxx, https://IP:2380, https://IP:2379,https://IP:4001 +``` + +### Check Endpoint Status + +The values for `RAFT TERM` should be equal and `RAFT INDEX` should be not be too far apart from each other. + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table +``` + +Example output: +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | 333ef673fc4add56 | 3.2.18 | 24 MB | false | 72 | 66887 | +| https://IP:2379 | 5feed52d940ce4cf | 3.2.18 | 24 MB | true | 72 | 66887 | +| https://IP:2379 | db6b3bdb559a848d | 3.2.18 | 25 MB | false | 72 | 66887 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### Check Endpoint Health + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint health +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl endpoint health --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") +``` + +Example output: +``` +https://IP:2379 is healthy: successfully committed proposal: took = 2.113189ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.649963ms +https://IP:2379 is healthy: successfully committed proposal: took = 2.451201ms +``` + +### Check Connectivity on Port TCP/2379 + +Command: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health" + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5"); do + echo "Validating connection to ${endpoint}/health"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/health" +done +``` + +Example output: +``` +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +Validating connection to https://IP:2379/health +{"health": "true"} +``` + +### Check Connectivity on Port TCP/2380 + +Command: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +for endpoint in $(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f4"); do + echo "Validating connection to ${endpoint}/version"; + docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl --http1.1 -s -w "\n" --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) "${endpoint}/version" +done +``` + +Example output: +``` +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +Validating connection to https://IP:2380/version +{"etcdserver":"3.2.18","etcdcluster":"3.2.0"} +``` + +## etcd Alarms + +etcd will trigger alarms, for instance when it runs out of space. + +Command: +``` +docker exec etcd etcdctl alarm list +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +Example output when NOSPACE alarm is triggered: +``` +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +``` + +## etcd Space Errors + +Related error messages are `etcdserver: mvcc: database space exceeded` or `applying raft message exceeded backend quota`. Alarm `NOSPACE` will be triggered. + +Resolutions: + +- [Compact the Keyspace](#compact-the-keyspace) +- [Defrag All etcd Members](#defrag-all-etcd-members) +- [Check Endpoint Status](#check-endpoint-status) +- [Disarm Alarm](#disarm-alarm) + +### Compact the Keyspace + +Command: +``` +rev=$(docker exec etcd etcdctl endpoint status --write-out json | egrep -o '"revision":[0-9]*' | egrep -o '[0-9]*') +docker exec etcd etcdctl compact "$rev" +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +rev=$(docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT endpoint status --write-out json | egrep -o '\"revision\":[0-9]*' | egrep -o '[0-9]*'") +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT compact \"$rev\"" +``` + +Example output: +``` +compacted revision xxx +``` + +### Defrag All etcd Members + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl defrag +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl defrag --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','")" +``` + +Example output: +``` +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +Finished defragmenting etcd member[https://IP:2379] +``` + +### Check Endpoint Status + +Command: +``` +docker exec -e ETCDCTL_ENDPOINTS=$(docker exec etcd /bin/sh -c "etcdctl member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") etcd etcdctl endpoint status --write-out table +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl endpoint status --endpoints=$(docker exec etcd /bin/sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT member list | cut -d, -f5 | sed -e 's/ //g' | paste -sd ','") --write-out table" +``` + +Example output: +``` ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +| https://IP:2379 | e973e4419737125 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | 4a509c997b26c206 | 3.2.18 | 553 kB | false | 32 | 2449410 | +| https://IP:2379 | b217e736575e9dd3 | 3.2.18 | 553 kB | true | 32 | 2449410 | ++-----------------+------------------+---------+---------+-----------+-----------+------------+ +``` + +### Disarm Alarm + +After verifying that the DB size went down after compaction and defragmenting, the alarm needs to be disarmed for etcd to allow writes again. + +Command: +``` +docker exec etcd etcdctl alarm list +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm disarm" +docker exec etcd sh -c "etcdctl --endpoints=\$ETCDCTL_ENDPOINT alarm list" +``` + +Example output: +``` +docker exec etcd etcdctl alarm list +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +memberID:x alarm:NOSPACE +docker exec etcd etcdctl alarm disarm +docker exec etcd etcdctl alarm list +``` + +## Configure Log Level + +:::note + +You can no longer dynamically change the log level in etcd v3.5 or later. + +::: + +### etcd v3.5 And Later + +To configure the log level for etcd, edit the cluster YAML: + +``` +services: + etcd: + extra_args: + log-level: "debug" +``` + +### etcd v3.4 And Earlier + +In earlier etcd versions, you can use the API to dynamically change the log level. Configure debug logging using the commands below: + +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"DEBUG"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +To reset the log level back to the default (`INFO`), you can use the following command. + +Command: +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINTS)/config/local/log +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker run --net=host -v $(docker inspect kubelet --format '{{ range .Mounts }}{{ if eq .Destination "/etc/kubernetes" }}{{ .Source }}{{ end }}{{ end }}')/ssl:/etc/kubernetes/ssl:ro appropriate/curl -s -XPUT -d '{"Level":"INFO"}' --cacert $(docker exec etcd printenv ETCDCTL_CACERT) --cert $(docker exec etcd printenv ETCDCTL_CERT) --key $(docker exec etcd printenv ETCDCTL_KEY) $(docker exec etcd printenv ETCDCTL_ENDPOINT)/config/local/log +``` + +## etcd Content + +If you want to investigate the contents of your etcd, you can either watch streaming events or you can query etcd directly, see below for examples. + +### Watch Streaming Events + +Command: +``` +docker exec etcd etcdctl watch --prefix /registry +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT watch --prefix /registry +``` + +If you only want to see the affected keys (and not the binary data), you can append `| grep -a ^/registry` to the command to filter for keys only. + +### Query etcd Directly + +Command: +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only +``` + +Command when using etcd version lower than 3.3.x (Kubernetes 1.13.x and lower) and `--internal-address` was specified when adding the node: +``` +docker exec etcd etcdctl --endpoints=\$ETCDCTL_ENDPOINT get /registry --prefix=true --keys-only +``` + +You can process the data to get a summary of count per key, using the command below: + +``` +docker exec etcd etcdctl get /registry --prefix=true --keys-only | grep -v ^$ | awk -F'/' '{ if ($3 ~ /cattle.io/) {h[$3"/"$4]++} else { h[$3]++ }} END { for(k in h) print h[k], k }' | sort -nr +``` + +## Replacing Unhealthy etcd Nodes + +When a node in your etcd cluster becomes unhealthy, the recommended approach is to fix or remove the failed or unhealthy node before adding a new etcd node to the cluster. diff --git a/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md new file mode 100644 index 00000000000..6c381ee5bb0 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-nginx-proxy.md @@ -0,0 +1,72 @@ +--- +title: Troubleshooting nginx-proxy +--- + + + + + +The `nginx-proxy` container is deployed on every node that does not have the `controlplane` role. It provides access to all the nodes with the `controlplane` role by dynamically generating the NGINX configuration based on available nodes with the `controlplane` role. + +## Check if the Container is Running + +The container is called `nginx-proxy` and should have status `Up`. The duration shown after `Up` is the time the container has been running. + +``` +docker ps -a -f=name=nginx-proxy +``` + +Example output: + +``` +docker ps -a -f=name=nginx-proxy +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +c3e933687c0e rancher/rke-tools:v0.1.15 "nginx-proxy CP_HO..." 3 hours ago Up 3 hours nginx-proxy +``` + +## Check Generated NGINX Configuration + +The generated configuration should include the IP addresses of the nodes with the `controlplane` role. The configuration can be checked using the following command: + +``` +docker exec nginx-proxy cat /etc/nginx/nginx.conf +``` + +Example output: +``` +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + + server ip_of_controlplane_node1:6443; + + server ip_of_controlplane_node2:6443; + + } + + server { + listen 6443; + proxy_pass kube_apiserver; + proxy_timeout 30; + proxy_connect_timeout 2s; + + } + +} +``` + +## nginx-proxy Container Logging + +The logging of the containers can contain information on what the problem could be. + +``` +docker logs nginx-proxy +``` \ No newline at end of file diff --git a/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md new file mode 100644 index 00000000000..5a5de6c0156 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components.md @@ -0,0 +1,38 @@ +--- +title: Troubleshooting Worker Nodes and Generic Components +--- + + + + + +This section applies to every node as it includes components that run on nodes with any role. + +## Check if the Containers are Running + +There are two specific containers launched on nodes with the `worker` role: + +* kubelet +* kube-proxy + +The containers should have status `Up`. The duration shown after `Up` is the time the container has been running. + +``` +docker ps -a -f=name='kubelet|kube-proxy' +``` + +Example output: +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +158d0dcc33a5 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kube-proxy +a30717ecfb55 rancher/hyperkube:v1.11.5-rancher1 "/opt/rke-tools/en..." 3 hours ago Up 3 hours kubelet +``` + +## Container Logging + +The logging of the containers can contain information on what the problem could be. + +``` +docker logs kubelet +docker logs kube-proxy +``` diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/dns.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/dns.md new file mode 100644 index 00000000000..af1108b6f80 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/dns.md @@ -0,0 +1,224 @@ +--- +title: DNS +--- + + + + + +The commands/steps listed on this page can be used to check name resolution issues in your cluster. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` for Rancher HA) or are using the embedded kubectl via the UI. + +Before running the DNS checks, check the [default DNS provider](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#default-dns-provider) for your cluster and make sure that [the overlay network is functioning correctly](networking.md#check-if-overlay-network-is-functioning-correctly) as this can also be the reason why DNS resolution (partly) fails. + +### Check if DNS pods are running + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns +``` + +Example output when using CoreDNS: +``` +NAME READY STATUS RESTARTS AGE +coredns-799dffd9c4-6jhlz 1/1 Running 0 76m +``` + +Example output when using kube-dns: +``` +NAME READY STATUS RESTARTS AGE +kube-dns-5fd74c7488-h6f7n 3/3 Running 0 4m13s +``` + +### Check if the DNS service is present with the correct cluster-ip + +``` +kubectl -n kube-system get svc -l k8s-app=kube-dns +``` + +``` +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/kube-dns ClusterIP 10.43.0.10 53/UDP,53/TCP 4m13s +``` + +### Check if domain names are resolving + +Check if internal cluster names are resolving (in this example, `kubernetes.default`), the IP shown after `Server:` should be the same as the `CLUSTER-IP` from the `kube-dns` service. + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup kubernetes.default +``` + +Example output: +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: kubernetes.default +Address 1: 10.43.0.1 kubernetes.default.svc.cluster.local +pod "busybox" deleted +``` + +Check if external names are resolving (in this example, `www.google.com`) + +``` +kubectl run -it --rm --restart=Never busybox --image=busybox:1.28 -- nslookup www.google.com +``` + +Example output: +``` +Server: 10.43.0.10 +Address 1: 10.43.0.10 kube-dns.kube-system.svc.cluster.local + +Name: www.google.com +Address 1: 2a00:1450:4009:80b::2004 lhr35s04-in-x04.1e100.net +Address 2: 216.58.211.100 ams15s32-in-f4.1e100.net +pod "busybox" deleted +``` + +If you want to check resolving of domain names on all of the hosts, execute the following steps: + +1. Save the following file as `ds-dnstest.yml` + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: dnstest + spec: + selector: + matchLabels: + name: dnstest + template: + metadata: + labels: + name: dnstest + spec: + tolerations: + - operator: Exists + containers: + - image: busybox:1.28 + imagePullPolicy: Always + name: alpine + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + ``` + +2. Launch it using `kubectl create -f ds-dnstest.yml` +3. Wait until `kubectl rollout status ds/dnstest -w` returns: `daemon set "dnstest" successfully rolled out`. +4. Configure the environment variable `DOMAIN` to a fully qualified domain name (FQDN) that the host should be able to resolve (`www.google.com` is used as an example) and run the following command to let each container on every host resolve the configured domain name (it's a single line command). + + ``` + export DOMAIN=www.google.com; echo "=> Start DNS resolve test"; kubectl get pods -l name=dnstest --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do kubectl exec $pod -- /bin/sh -c "nslookup $DOMAIN > /dev/null 2>&1"; RC=$?; if [ $RC -ne 0 ]; then echo $host cannot resolve $DOMAIN; fi; done; echo "=> End DNS resolve test" + ``` + +5. When this command has finished running, the output indicating everything is correct is: + + ``` + => Start DNS resolve test + => End DNS resolve test + ``` + +If you see error in the output, that means that the mentioned host(s) is/are not able to resolve the given FQDN. + +Example error output of a situation where host with IP 209.97.182.150 had the UDP ports blocked. + +``` +=> Start DNS resolve test +command terminated with exit code 1 +209.97.182.150 cannot resolve www.google.com +=> End DNS resolve test +``` + +Cleanup the alpine DaemonSet by running `kubectl delete ds/dnstest`. + +### CoreDNS specific + +#### Check CoreDNS logging + +``` +kubectl -n kube-system logs -l k8s-app=kube-dns +``` + +#### Check configuration + +CoreDNS configuration is stored in the configmap `coredns` in the `kube-system` namespace. + +``` +kubectl -n kube-system get configmap coredns -o go-template={{.data.Corefile}} +``` + +#### Check upstream nameservers in resolv.conf + +By default, the configured nameservers on the host (in `/etc/resolv.conf`) will be used as upstream nameservers for CoreDNS. You can check this file on the host or run the following Pod with `dnsPolicy` set to `Default`, which will inherit the `/etc/resolv.conf` from the host it is running on. + +``` +kubectl run -i --restart=Never --rm test-${RANDOM} --image=ubuntu --overrides='{"kind":"Pod", "apiVersion":"v1", "spec": {"dnsPolicy":"Default"}}' -- sh -c 'cat /etc/resolv.conf' +``` + +#### Enable query logging + +Enabling query logging can be done by enabling the [log plugin](https://coredns.io/plugins/log/) in the Corefile configuration in the configmap `coredns`. You can do so by using `kubectl -n kube-system edit configmap coredns` or use the command below to replace the configuration in place: + +``` +kubectl get configmap -n kube-system coredns -o json | sed -e 's_loadbalance_log\\n loadbalance_g' | kubectl apply -f - +``` + +All queries will now be logged and can be checked using the command in [Check CoreDNS logging](#check-coredns-logging). + +### kube-dns specific + +#### Check upstream nameservers in kubedns container + +By default, the configured nameservers on the host (in `/etc/resolv.conf`) will be used as upstream nameservers for kube-dns. Sometimes the host will run a local caching DNS nameserver, which means the address in `/etc/resolv.conf` will point to an address in the loopback range (`127.0.0.0/8`) which will be unreachable by the container. In case of Ubuntu 18.04, this is done by `systemd-resolved`. We detect if `systemd-resolved` is running, and will automatically use the `/etc/resolv.conf` file with the correct upstream nameservers (which is located at `/run/systemd/resolve/resolv.conf`). + +Use the following command to check the upstream nameservers used by the kubedns container: + +``` +kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers -o custom-columns=NAME:.metadata.name,HOSTIP:.status.hostIP | while read pod host; do echo "Pod ${pod} on host ${host}"; kubectl -n kube-system exec $pod -c kubedns cat /etc/resolv.conf; done +``` + +Example output: +``` +Pod kube-dns-667c7cb9dd-z4dsf on host x.x.x.x +nameserver 1.1.1.1 +nameserver 8.8.4.4 +``` + +If the output shows an address in the loopback range (`127.0.0.0/8`), you can correct this in two ways: + +* Make sure the correct nameservers are listed in `/etc/resolv.conf` on your nodes in the cluster, please consult your operating system documentation on how to do this. Make sure you execute this before provisioning a cluster, or reboot the nodes after making the modification. +* Configure the `kubelet` to use a different file for resolving names, by using `extra_args` as shown below (where `/run/resolvconf/resolv.conf` is the file with the correct nameservers): + +``` +services: + kubelet: + extra_args: + resolv-conf: "/run/resolvconf/resolv.conf" +``` + +:::note + +As the `kubelet` is running inside a container, the path for files located in `/etc` and `/usr` are in `/host/etc` and `/host/usr` inside the `kubelet` container. + +::: + +See [Editing Cluster as YAML](../../reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration.md#editing-clusters-with-yaml) how to apply this change. When the provisioning of the cluster has finished, you have to remove the kube-dns pod to activate the new setting in the pod: + +``` +kubectl delete pods -n kube-system -l k8s-app=kube-dns +pod "kube-dns-5fd74c7488-6pwsf" deleted +``` + +Try to resolve name again using [Check if domain names are resolving](#check-if-domain-names-are-resolving). + +If you want to check the kube-dns configuration in your cluster (for example, to check if there are different upstream nameservers configured), you can run the following command to list the kube-dns configuration: + +``` +kubectl -n kube-system get configmap kube-dns -o go-template='{{range $key, $value := .data}}{{ $key }}{{":"}}{{ $value }}{{"\n"}}{{end}}' +``` + +Example output: +``` +upstreamNameservers:["1.1.1.1"] +``` diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md new file mode 100644 index 00000000000..106479c0bb7 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation.md @@ -0,0 +1,32 @@ +--- +title: Rotation of Expired Webhook Certificates +--- + + + + + +For Rancher versions that have `rancher-webhook` installed, certain versions created certificates that will expire after one year. It will be necessary for you to rotate your webhook certificate if the certificate did not renew. + +In Rancher v2.6.3 and up, rancher-webhook deployments will automatically renew their TLS certificate when it is within 30 or fewer days of its expiration date. If you are using v2.6.2 or below, there are two methods to work around this issue: + +##### 1. Users with cluster access, run the following commands: +``` +kubectl delete secret -n cattle-system cattle-webhook-tls +kubectl delete mutatingwebhookconfigurations.admissionregistration.k8s.io --ignore-not-found=true rancher.cattle.io +kubectl delete pod -n cattle-system -l app=rancher-webhook +``` + +##### 2. Users with no cluster access via `kubectl`: + +1. Delete the `cattle-webhook-tls` secret in the `cattle-system` namespace in the local cluster. + +2. Delete the `rancher.cattle.io` mutating webhook + +3. Delete the `rancher-webhook` pod in the `cattle-system` namespace in the local cluster. + +:::note + +The webhook certificate expiration issue is not specific to `cattle-webhook-tls` as listed in the examples. You will fill in your expired certificate secret accordingly. + +::: \ No newline at end of file diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md new file mode 100644 index 00000000000..f1e6b8f8594 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/kubernetes-resources.md @@ -0,0 +1,259 @@ +--- +title: Kubernetes Resources +--- + + + + + +The commands/steps listed on this page can be used to check the most important Kubernetes resources and apply to [Rancher Launched Kubernetes](../../pages-for-subheaders/launch-kubernetes-with-rancher.md) clusters. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` for Rancher HA) or are using the embedded kubectl via the UI. + + +## Nodes + +### Get nodes + +Run the command below and check the following: + +- All nodes in your cluster should be listed, make sure there is not one missing. +- All nodes should have the **Ready** status (if not in **Ready** state, check the `kubelet` container logs on that node using `docker logs kubelet`) +- Check if all nodes report the correct version. +- Check if OS/Kernel/Docker values are shown as expected (possibly you can relate issues due to upgraded OS/Kernel/Docker) + + +``` +kubectl get nodes -o wide +``` + +Example output: + +``` +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +controlplane-0 Ready controlplane 31m v1.13.5 138.68.188.91 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +etcd-0 Ready etcd 31m v1.13.5 138.68.180.33 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +worker-0 Ready worker 30m v1.13.5 139.59.179.88 Ubuntu 18.04.2 LTS 4.15.0-47-generic docker://18.9.5 +``` + +### Get node conditions + +Run the command below to list nodes with [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{end}}' +``` + +Run the command below to list nodes with [Node Conditions](https://kubernetes.io/docs/concepts/architecture/nodes/#condition) that are active that could prevent normal operation. + +``` +kubectl get nodes -o go-template='{{range .items}}{{$node := .}}{{range .status.conditions}}{{if ne .type "Ready"}}{{if eq .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{":"}}{{.status}}{{"\n"}}{{end}}{{else}}{{if ne .status "True"}}{{$node.metadata.name}}{{": "}}{{.type}}{{": "}}{{.status}}{{"\n"}}{{end}}{{end}}{{end}}{{end}}' +``` + +Example output: + +``` +worker-0: DiskPressure:True +``` + +## Kubernetes leader election + +### Kubernetes Controller Manager leader + +The leader is determined by a leader election process. After the leader has been determined, the leader (`holderIdentity`) is saved in the `kube-controller-manager` endpoint (in this example, `controlplane-0`). + +``` +kubectl -n kube-system get endpoints kube-controller-manager -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +### Kubernetes Scheduler leader + +The leader is determined by a leader election process. After the leader has been determined, the leader (`holderIdentity`) is saved in the `kube-scheduler` endpoint (in this example, `controlplane-0`). + +``` +kubectl -n kube-system get endpoints kube-scheduler -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"controlplane-0_xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx","leaseDurationSeconds":15,"acquireTime":"2018-12-27T08:59:45Z","renewTime":"2018-12-27T09:44:57Z","leaderTransitions":0}> +``` + +## Ingress Controller + +The default Ingress Controller is NGINX and is deployed as a DaemonSet in the `ingress-nginx` namespace. The pods are only scheduled to nodes with the `worker` role. + +Check if the pods are running on all nodes: + +``` +kubectl -n ingress-nginx get pods -o wide +``` + +Example output: + +``` +kubectl -n ingress-nginx get pods -o wide +NAME READY STATUS RESTARTS AGE IP NODE +default-http-backend-797c5bc547-kwwlq 1/1 Running 0 17m x.x.x.x worker-1 +nginx-ingress-controller-4qd64 1/1 Running 0 14m x.x.x.x worker-1 +nginx-ingress-controller-8wxhm 1/1 Running 0 13m x.x.x.x worker-0 +``` + +If a pod is unable to run (Status is not **Running**, Ready status is not showing `1/1` or you see a high count of Restarts), check the pod details, logs and namespace events. + +### Pod details + +``` +kubectl -n ingress-nginx describe pods -l app=ingress-nginx +``` + +### Pod container logs + +The below command can show the logs of all the pods labeled "app=ingress-nginx", but it will display only 10 lines of log because of the restrictions of the `kubectl logs` command. Refer to `--tail` of `kubectl logs -h` for more information. + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +If the full log is needed, specify the pod name in the trailing command: + +``` +kubectl -n ingress-nginx logs +``` + +### Namespace events + +``` +kubectl -n ingress-nginx get events +``` + +### Debug logging + +To enable debug logging: + +``` +kubectl -n ingress-nginx patch ds nginx-ingress-controller --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--v=5"}]' +``` + +### Check configuration + +Retrieve generated configuration in each pod: + +``` +kubectl -n ingress-nginx get pods -l app=ingress-nginx --no-headers -o custom-columns=.NAME:.metadata.name | while read pod; do kubectl -n ingress-nginx exec $pod -- cat /etc/nginx/nginx.conf; done +``` + +## Rancher agents + +Communication to the cluster (Kubernetes API via `cattle-cluster-agent`) and communication to the nodes (cluster provisioning via `cattle-node-agent`) is done through Rancher agents. + +#### cattle-node-agent + +Check if the cattle-node-agent pods are present on each node, have status **Running** and don't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +Check logging of a specific cattle-node-agent pod or all cattle-node-agent pods: + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +Check if the cattle-cluster-agent pod is present in the cluster, has status **Running** and doesn't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +Check logging of cattle-cluster-agent pod: + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` + +## Jobs and Pods + +### Check that pods or jobs have status **Running**/**Completed** + +To check, run the command: + +``` +kubectl get pods --all-namespaces +``` + +If a pod is not in **Running** state, you can dig into the root cause by running: + +### Describe pod + +``` +kubectl describe pod POD_NAME -n NAMESPACE +``` + +### Pod container logs + +``` +kubectl logs POD_NAME -n NAMESPACE +``` + +If a job is not in **Completed** state, you can dig into the root cause by running: + +### Describe job + +``` +kubectl describe job JOB_NAME -n NAMESPACE +``` + +### Logs from the containers of pods of the job + +``` +kubectl logs -l job-name=JOB_NAME -n NAMESPACE +``` + +### Evicted pods + +Pods can be evicted based on [eviction signals](https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/#eviction-policy). + +Retrieve a list of evicted pods (podname and namespace): + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' +``` + +To delete all evicted pods: + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace delete pod $epod; done +``` + +Retrieve a list of evicted pods, scheduled node and the reason: + +``` +kubectl get pods --all-namespaces -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{if eq .status.reason "Evicted"}}{{.metadata.name}}{{" "}}{{.metadata.namespace}}{{"\n"}}{{end}}{{end}}{{end}}' | while read epod enamespace; do kubectl -n $enamespace get pod $epod -o=custom-columns=NAME:.metadata.name,NODE:.spec.nodeName,MSG:.status.message; done +``` + +### Job does not complete + +If you have enabled Istio, and you are having issues with a Job you deployed not completing, you will need to add an annotation to your pod using [these steps.](../../how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace.md) + +Since Istio Sidecars run indefinitely, a Job cannot be considered complete even after its task has completed. This is a temporary workaround and will disable Istio for any traffic to/from the annotated Pod. Keep in mind this may not allow you to continue to use a Job for integration testing, as the Job will not have access to the service mesh. \ No newline at end of file diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/logging.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/logging.md new file mode 100644 index 00000000000..f4ba45226bc --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/logging.md @@ -0,0 +1,96 @@ +--- +title: Logging +--- + + + + + +## Log Levels + +The following log levels are used in Rancher: + +| Name | Description | +|---------|-------------| +| `info` | Logs informational messages. This is the default log level. | +| `debug` | Logs more detailed messages that can be used to debug. | +| `trace` | Logs very detailed messages on internal functions. This is very verbose and can contain sensitive information. | + +### How to Configure a Log Level + +#### Kubernetes Install + +* Configure debug log level + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set debug; done +OK +OK +OK +$ kubectl -n cattle-system logs -l app=rancher -c rancher +``` + +* Configure info log level + +``` +$ KUBECONFIG=./kube_config_cluster.yml +$ kubectl -n cattle-system get pods -l app=rancher --no-headers -o custom-columns=name:.metadata.name | while read rancherpod; do kubectl -n cattle-system exec $rancherpod -c rancher -- loglevel --set info; done +OK +OK +OK +``` + +#### Docker Install + +* Configure debug log level + +``` +$ docker exec -ti loglevel --set debug +OK +$ docker logs -f +``` + +* Configure info log level + +``` +$ docker exec -ti loglevel --set info +OK +``` + +## Rancher Machine Debug Logs +If you need to troubleshoot the creation of objects in your infrastructure provider of choice, `rancher-machine` +debug logs might be helpful to you. + +It's possible to enable debug logs for `rancher-machine` by setting environment variables when launching Rancher. + +The `CATTLE_WHITELIST_ENVVARS` environment variable allows users to whitelist specific environment variables to be +passed down to `rancher-machine` during provisioning. + +The `MACHINE_DEBUG` variable enables debug logs in `rancher-machine`. + +Thus, by setting `MACHINE_DEBUG=true` and adding `MACHINE_DEBUG` to the default list of variables in +`CATTLE_WHITELIST_ENVVARS` (e.g. `CATTLE_WHITELIST_ENVVARS=HTTP_PROXY,HTTPS_PROXY,NO_PROXY,MACHINE_DEBUG`) it is +possible to enable debug logs in `rancher-machine` when provisioning RKE1, RKE2 and k3s clusters. + +:::caution + +Just like the `trace` log level above, `rancher-machine` debug logs can contain sensitive information. + +::: + + +## Cattle-cluster-agent Debug Logs + +The `cattle-cluster-agent` log levels can be set when you initialize downstream clusters. + +When you create a cluster under **Cluster Configuration > Agent Environment Vars** you can set variables to define the log level. +- Trace-level logging: Set `CATTLE_TRACE` or `RANCHER_TRACE` to `true` + +- Debug-level logging: Set `CATTLE_DEBUG` or `RANCHER_DEBUG` to `true` + +:::caution + +The `cattle-cluster-agent` debug logs may contain sensitive information. + +::: diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/networking.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/networking.md new file mode 100644 index 00000000000..bea850ffe13 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/networking.md @@ -0,0 +1,125 @@ +--- +title: Networking +--- + + + + + +The commands/steps listed on this page can be used to check networking related issues in your cluster. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml` for Rancher HA) or are using the embedded kubectl via the UI. + +### Double check if all the required ports are opened in your (host) firewall + +Double check if all the [required ports](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) are opened in your (host) firewall. The overlay network uses UDP in comparison to all other required ports which are TCP. +### Check if overlay network is functioning correctly + +The pod can be scheduled to any of the hosts you used for your cluster, but that means that the NGINX ingress controller needs to be able to route the request from `NODE_1` to `NODE_2`. This happens over the overlay network. If the overlay network is not functioning, you will experience intermittent TCP/HTTP connection failures due to the NGINX ingress controller not being able to route to the pod. + +To test the overlay network, you can launch the following `DaemonSet` definition. This will run a `swiss-army-knife` container on every host (image was developed by Rancher engineers and can be found here: https://github.com/rancherlabs/swiss-army-knife), which we will use to run a `ping` test between containers on all hosts. + +:::note + +This container [does not support ARM nodes](https://github.com/leodotcloud/swiss-army-knife/issues/18), such as a Raspberry Pi. This will be seen in the pod logs as `exec user process caused: exec format error`. + +::: + +1. Save the following file as `overlaytest.yml` + + ``` + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: overlaytest + spec: + selector: + matchLabels: + name: overlaytest + template: + metadata: + labels: + name: overlaytest + spec: + tolerations: + - operator: Exists + containers: + - image: rancherlabs/swiss-army-knife + imagePullPolicy: Always + name: overlaytest + command: ["sh", "-c", "tail -f /dev/null"] + terminationMessagePath: /dev/termination-log + + ``` + +2. Launch it using `kubectl create -f overlaytest.yml` +3. Wait until `kubectl rollout status ds/overlaytest -w` returns: `daemon set "overlaytest" successfully rolled out`. +4. Run the following script, from the same location. It will have each `overlaytest` container on every host ping each other: + ``` + #!/bin/bash + echo "=> Start network overlay test" + kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.metadata.name}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read spod shost + do kubectl get pods -l name=overlaytest -o jsonpath='{range .items[*]}{@.status.podIP}{" "}{@.spec.nodeName}{"\n"}{end}' | + while read tip thost + do kubectl --request-timeout='10s' exec $spod -c overlaytest -- /bin/sh -c "ping -c2 $tip > /dev/null 2>&1" + RC=$? + if [ $RC -ne 0 ] + then echo FAIL: $spod on $shost cannot reach pod IP $tip on $thost + else echo $shost can reach $thost + fi + done + done + echo "=> End network overlay test" + ``` + +5. When this command has finished running, it will output the state of each route: + + ``` + => Start network overlay test + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.7.3 on wk2 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.0.5 on cp1 + Error from server (NotFound): pods "wk2" not found + FAIL: overlaytest-5bglp on wk2 cannot reach pod IP 10.42.2.12 on wk1 + command terminated with exit code 1 + FAIL: overlaytest-v4qkl on cp1 cannot reach pod IP 10.42.7.3 on wk2 + cp1 can reach cp1 + cp1 can reach wk1 + command terminated with exit code 1 + FAIL: overlaytest-xpxwp on wk1 cannot reach pod IP 10.42.7.3 on wk2 + wk1 can reach cp1 + wk1 can reach wk1 + => End network overlay test + ``` + If you see error in the output, there is some issue with the route between the pods on the two hosts. In the above output the node `wk2` has no connectivity over the overlay network. This could be because the [required ports](../../how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters.md#networking-requirements) for overlay networking are not opened for `wk2`. +6. You can now clean up the DaemonSet by running `kubectl delete ds/overlaytest`. + + +### Check if MTU is correctly configured on hosts and on peering/tunnel appliances/devices + +When the MTU is incorrectly configured (either on hosts running Rancher, nodes in created/imported clusters or on appliances/devices in between), error messages will be logged in Rancher and in the agents, similar to: + +* `websocket: bad handshake` +* `Failed to connect to proxy` +* `read tcp: i/o timeout` + +See [Google Cloud VPN: MTU Considerations](https://cloud.google.com/vpn/docs/concepts/mtu-considerations#gateway_mtu_vs_system_mtu) for an example how to configure MTU correctly when using Google Cloud VPN between Rancher and cluster nodes. + +### Resolved issues + +#### Overlay network broken when using Canal/Flannel due to missing node annotations + +| | | +|------------|------------| +| GitHub issue | [#13644](https://github.com/rancher/rancher/issues/13644) | +| Resolved in | v2.1.2 | + +To check if your cluster is affected, the following command will list nodes that are broken (this command requires `jq` to be installed): + +``` +kubectl get nodes -o json | jq '.items[].metadata | select(.annotations["flannel.alpha.coreos.com/public-ip"] == null or .annotations["flannel.alpha.coreos.com/kube-subnet-manager"] == null or .annotations["flannel.alpha.coreos.com/backend-type"] == null or .annotations["flannel.alpha.coreos.com/backend-data"] == null) | .name' +``` + +If there is no output, the cluster is not affected. diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/rancher-ha.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/rancher-ha.md new file mode 100644 index 00000000000..8917f80da4d --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/rancher-ha.md @@ -0,0 +1,83 @@ +--- +title: Rancher HA +--- + + + + + +The commands/steps listed on this page can be used to check your Rancher Kubernetes Installation. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kube_config_cluster.yml`). + +### Check Rancher pods + +Rancher pods are deployed as a Deployment in the `cattle-system` namespace. + +Check if the pods are running on all nodes: + +``` +kubectl -n cattle-system get pods -l app=rancher -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +rancher-7dbd7875f7-n6t5t 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qbj5k 1/1 Running 0 8m x.x.x.x x.x.x.x +rancher-7dbd7875f7-qw7wb 1/1 Running 0 8m x.x.x.x x.x.x.x +``` + +If a pod is unable to run (Status is not **Running**, Ready status is not showing `1/1` or you see a high count of Restarts), check the pod details, logs and namespace events. + +#### Pod details + +``` +kubectl -n cattle-system describe pods -l app=rancher +``` + +#### Pod container logs + +``` +kubectl -n cattle-system logs -l app=rancher +``` + +#### Namespace events + +``` +kubectl -n cattle-system get events +``` + +### Check ingress + +Ingress should have the correct `HOSTS` (showing the configured FQDN) and `ADDRESS` (host address(es) it will be routed to). + +``` +kubectl -n cattle-system get ingress +``` + +Example output: + +``` +NAME HOSTS ADDRESS PORTS AGE +rancher rancher.yourdomain.com x.x.x.x,x.x.x.x,x.x.x.x 80, 443 2m +``` + +### Check ingress controller logs + +When accessing your configured Rancher FQDN does not show you the UI, check the ingress controller logging to see what happens when you try to access Rancher: + +``` +kubectl -n ingress-nginx logs -l app=ingress-nginx +``` + +### Leader election + +The leader is determined by a leader election process. After the leader has been determined, the leader (`holderIdentity`) is saved in the `cattle-controllers` ConfigMap (in this example, `rancher-7dbd7875f7-qbj5k`). + +``` +kubectl -n kube-system get configmap cattle-controllers -o jsonpath='{.metadata.annotations.control-plane\.alpha\.kubernetes\.io/leader}' +{"holderIdentity":"rancher-7dbd7875f7-qbj5k","leaseDurationSeconds":45,"acquireTime":"2019-04-04T11:53:12Z","renewTime":"2019-04-04T12:24:08Z","leaderTransitions":0} +``` + diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/registered-clusters.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/registered-clusters.md new file mode 100644 index 00000000000..cce0e089621 --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/registered-clusters.md @@ -0,0 +1,71 @@ +--- +title: Registered Clusters +--- + + + + + +The commands/steps listed on this page can be used to check clusters that you are registering or that are registered in Rancher. + +Make sure you configured the correct kubeconfig (for example, `export KUBECONFIG=$PWD/kubeconfig_from_imported_cluster.yml`) + +### Rancher agents + +Communication to the cluster (Kubernetes API via cattle-cluster-agent) and communication to the nodes is done through Rancher agents. + +If the cattle-cluster-agent cannot connect to the configured `server-url`, the cluster will remain in **Pending** state, showing `Waiting for full cluster configuration`. + +#### cattle-node-agent + +:::note + +cattle-node-agents are only present in clusters created in Rancher with RKE. + +::: + +Check if the cattle-node-agent pods are present on each node, have status **Running** and don't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-node-agent-4gc2p 1/1 Running 0 2h x.x.x.x worker-1 +cattle-node-agent-8cxkk 1/1 Running 0 2h x.x.x.x etcd-1 +cattle-node-agent-kzrlg 1/1 Running 0 2h x.x.x.x etcd-0 +cattle-node-agent-nclz9 1/1 Running 0 2h x.x.x.x controlplane-0 +cattle-node-agent-pwxp7 1/1 Running 0 2h x.x.x.x worker-0 +cattle-node-agent-t5484 1/1 Running 0 2h x.x.x.x controlplane-1 +cattle-node-agent-t8mtz 1/1 Running 0 2h x.x.x.x etcd-2 +``` + +Check logging of a specific cattle-node-agent pod or all cattle-node-agent pods: + +``` +kubectl -n cattle-system logs -l app=cattle-agent +``` + +#### cattle-cluster-agent + +Check if the cattle-cluster-agent pod is present in the cluster, has status **Running** and doesn't have a high count of Restarts: + +``` +kubectl -n cattle-system get pods -l app=cattle-cluster-agent -o wide +``` + +Example output: + +``` +NAME READY STATUS RESTARTS AGE IP NODE +cattle-cluster-agent-54d7c6c54d-ht9h4 1/1 Running 0 2h x.x.x.x worker-1 +``` + +Check logging of cattle-cluster-agent pod: + +``` +kubectl -n cattle-system logs -l app=cattle-cluster-agent +``` diff --git a/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md new file mode 100644 index 00000000000..6a25ae1565e --- /dev/null +++ b/versioned_docs/version-latest/troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs.md @@ -0,0 +1,26 @@ +--- +title: User ID Tracking in Audit Logs +--- + + + + + +The following audit logs are used in Rancher to track events occuring on the local and downstream clusters: + +* [Kubernetes Audit Logs](https://rancher.com/docs/rke/latest/en/config-options/audit-log/) +* [Rancher API Audit Logs](../../how-to-guides/advanced-user-guides/enable-api-audit-log.md) + +Audit logs in Rancher v2.6 have been enhanced to include the external Identity Provider name (common name of the user in the external Auth provider) in both the Rancher and downstream Kubernetes audit logs. + +Before v2.6, a Rancher Admin could not trace an event from the Rancher audit logs and into the Kubernetes audit logs without knowing the mapping of the external Identity Provider username to the userId (`u-xXXX`) used in Rancher. +To know this mapping, the cluster admins needed to have access to Rancher API, UI, and the local management cluster. + +Now with this feature, a downstream cluster admin should be able to look at the Kubernetes audit logs and know which specific external Identity Provider (IDP) user performed an action without needing to view anything in Rancher. +If the audit logs are shipped off of the cluster, a user of the logging system should be able to identify the user in the external Identity Provider system. +A Rancher Admin should now be able to view Rancher audit logs and follow through to the Kubernetes audit log by using the external Identity Provider username. + +### Feature Description + +- When Kubernetes Audit logs are enabled on the downstream cluster, in each event that is logged, the external Identity Provider's username is now logged for each request, at the "metadata" level. +- When Rancher API Audit logs are enabled on the Rancher installation, the external Identity Provider's username is also logged now at the `auditLog.level=1` for each request that hits the Rancher API server, including the login requests. diff --git a/versioned_sidebars/version-2.7-sidebars.json b/versioned_sidebars/version-2.7-sidebars.json index 3c543404682..5625fdaf67d 100644 --- a/versioned_sidebars/version-2.7-sidebars.json +++ b/versioned_sidebars/version-2.7-sidebars.json @@ -1029,8 +1029,8 @@ }, "items": [ "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23", - "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.24", - "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25" + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" ] }, { @@ -1042,8 +1042,8 @@ }, "items": [ "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23", - "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.24", - "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25" + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" ] }, { @@ -1055,8 +1055,8 @@ }, "items": [ "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23", - "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.24", - "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25" + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" ] } ] diff --git a/versioned_sidebars/version-2.8-sidebars.json b/versioned_sidebars/version-2.8-sidebars.json new file mode 100644 index 00000000000..2219c8e78bb --- /dev/null +++ b/versioned_sidebars/version-2.8-sidebars.json @@ -0,0 +1,1267 @@ +{ + "tutorialSidebar": [ + "rancher-manager", + { + "type": "category", + "label": "Getting Started", + "items": [ + "getting-started/overview", + { + "type": "category", + "label": "Quick Start Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/quick-start-guides" + }, + "items": [ + { + "type": "category", + "label": "Deploy Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/deploy-rancher-manager" + }, + "items": [ + "getting-started/quick-start-guides/deploy-rancher-manager/aws", + "getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace", + "getting-started/quick-start-guides/deploy-rancher-manager/azure", + "getting-started/quick-start-guides/deploy-rancher-manager/digitalocean", + "getting-started/quick-start-guides/deploy-rancher-manager/gcp", + "getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud", + "getting-started/quick-start-guides/deploy-rancher-manager/vagrant", + "getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal", + "getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs", + "getting-started/quick-start-guides/deploy-rancher-manager/helm-cli" + ] + }, + "getting-started/quick-start-guides/deploy-rancher-manager/prime", + { + "type": "category", + "label": "Deploy Rancher Workloads", + "link": { + "type": "doc", + "id": "pages-for-subheaders/deploy-rancher-workloads" + }, + "items": [ + "getting-started/quick-start-guides/deploy-workloads/workload-ingress", + "getting-started/quick-start-guides/deploy-workloads/nodeports" + ] + } + ] + }, + { + "type": "category", + "label": "Installation and Upgrade", + "link": { + "type": "doc", + "id": "pages-for-subheaders/installation-and-upgrade" + }, + "items": [ + { + "type": "category", + "label": "Installation Requirements", + "link": { + "type": "doc", + "id": "pages-for-subheaders/installation-requirements" + }, + "items": [ + "getting-started/installation-and-upgrade/installation-requirements/install-docker", + "getting-started/installation-and-upgrade/installation-requirements/dockershim", + "getting-started/installation-and-upgrade/installation-requirements/port-requirements" + ] + }, + { + "type": "category", + "label": "Installation References", + "link": { + "type": "doc", + "id": "pages-for-subheaders/installation-references" + }, + "items": [ + "getting-started/installation-and-upgrade/installation-references/helm-chart-options", + "getting-started/installation-and-upgrade/installation-references/tls-settings", + "getting-started/installation-and-upgrade/installation-references/feature-flags" + ] + }, + { + "type": "category", + "label": "Install/Upgrade on a Kubernetes Cluster", + "link": { + "type": "doc", + "id": "pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster" + }, + "items": [ + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting" + ] + }, + { + "type": "category", + "label": "Other Installation Methods", + "link": { + "type": "doc", + "id": "pages-for-subheaders/other-installation-methods" + }, + "items": [ + { + "type": "category", + "label": "Air-Gapped Helm CLI Install", + "link": { + "type": "doc", + "id": "pages-for-subheaders/air-gapped-helm-cli-install" + }, + "items": [ + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands" + ] + }, + { + "type": "category", + "label": "Rancher on a Single Node with Docker", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-on-a-single-node-with-docker" + }, + "items": [ + "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting" + ] + }, + { + "type": "category", + "label": "Rancher Behind an HTTP Proxy", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-behind-an-http-proxy" + }, + "items": [ + "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher" + ] + } + ] + }, + { + "type": "category", + "label": "Resources", + "link": { + "type": "doc", + "id": "pages-for-subheaders/resources" + }, + "items": [ + "getting-started/installation-and-upgrade/resources/choose-a-rancher-version", + "getting-started/installation-and-upgrade/resources/helm-version-requirements", + "getting-started/installation-and-upgrade/resources/add-tls-secrets", + "getting-started/installation-and-upgrade/resources/custom-ca-root-certificates", + "getting-started/installation-and-upgrade/resources/upgrade-cert-manager", + "getting-started/installation-and-upgrade/resources/update-rancher-certificate", + "getting-started/installation-and-upgrade/resources/bootstrap-password", + "getting-started/installation-and-upgrade/resources/local-system-charts" + ] + }, + "getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes", + "getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher" + ] + } + ] + }, + { + "type": "category", + "label": "How-to Guides", + "items": [ + { + "type": "category", + "label": "New User Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/new-user-guides" + }, + "items": [ + { + "type": "category", + "label": "Authentication, Permissions, and Global Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/authentication-permissions-and-global-configuration" + }, + "items": [ + { + "type": "category", + "label": "Authentication Config", + "link": { + "type": "doc", + "id": "pages-for-subheaders/authentication-config" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml" + ] + }, + { + "type": "category", + "label": "Configure OpenLDAP", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configure-openldap" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference" + ] + }, + { + "type": "category", + "label": "Configure Microsoft AD Federation Service (SAML)", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configure-microsoft-ad-federation-service-saml" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs" + ] + }, + { + "type": "category", + "label": "Configure Shibboleth (SAML)", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configure-shibboleth-saml" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions" + ] + }, + { + "type": "category", + "label": "Manage Role-Based Access Control (RBAC)", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-role-based-access-control-rbac" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles" + ] + }, + { + "type": "category", + "label": "About Provisioning Drivers", + "link": { + "type": "doc", + "id": "pages-for-subheaders/about-provisioning-drivers" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers" + ] + }, + { + "type": "category", + "label": "About RKE1 Templates", + "link": { + "type": "doc", + "id": "pages-for-subheaders/about-rke1-templates" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases" + ] + }, + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding" + ] + }, + { + "type": "category", + "label": "Manage Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-clusters" + }, + "items": [ + { + "type": "category", + "label": "Access Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/access-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig", + "how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint", + "how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters" + ] + }, + { + "type": "category", + "label": "Install Cluster Autoscaler", + "link": { + "type": "doc", + "id": "pages-for-subheaders/install-cluster-autoscaler" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups" + ] + }, + { + "type": "category", + "label": "Create Kubernetes Persistent Storage", + "link": { + "type": "doc", + "id": "pages-for-subheaders/create-kubernetes-persistent-storage" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes" + ] + }, + { + "type": "category", + "label": "Provisioning Storage Examples", + "link": { + "type": "doc", + "id": "pages-for-subheaders/provisioning-storage-examples" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs", + "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage", + "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage" + ] + }, + "how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces", + "how-to-guides/new-user-guides/manage-clusters/rotate-certificates", + "how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key", + "how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates", + "how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools", + "how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes", + "how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy", + "how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies" + ] + }, + { + "type": "category", + "label": "Kubernetes Cluster Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-cluster-setup" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs", + "how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher", + "how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher", + "how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher" + ] + }, + { + "type": "category", + "label": "Infrastructure Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/infrastructure-setup" + }, + "items": [ + "how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster", + "how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster", + "how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster", + "how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2", + "how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds", + "how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer", + "how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer" + ] + }, + { + "type": "category", + "label": "Kubernetes Clusters in Rancher Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-clusters-in-rancher-setup" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters", + { + "type": "category", + "label": "Checklist for Production-Ready Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/checklist-for-production-ready-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes" + ] + }, + { + "type": "category", + "label": "Set Up Clusters from Hosted Kubernetes Providers", + "link": { + "type": "doc", + "id": "pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei" + ] + }, + { + "type": "category", + "label": "Use Windows Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/use-windows-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance" + ] + }, + { + "type": "category", + "label": "Set Up Cloud Providers", + "link": { + "type": "doc", + "id": "pages-for-subheaders/set-up-cloud-providers" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree" + ] + }, + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters" + ] + }, + { + "type": "category", + "label": "Launch Kubernetes with Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/launch-kubernetes-with-rancher" + }, + "items": [ + { + "type": "category", + "label": "Use New Nodes in an Infra Provider", + "link": { + "type": "doc", + "id": "pages-for-subheaders/use-new-nodes-in-an-infra-provider" + }, + "items": [ + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster", + { + "type": "category", + "label": "vSphere", + "link": { + "type": "doc", + "id": "pages-for-subheaders/vsphere" + }, + "items": [ + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template" + ] + }, + { + "type": "category", + "label": "Nutanix", + "link": { + "type": "doc", + "id": "pages-for-subheaders/nutanix" + }, + "items": [ + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos" + ] + } + ] + }, + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents" + ] + }, + { + "type": "category", + "label": "Kubernetes Resources Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-resources-setup" + }, + "items": [ + { + "type": "category", + "label": "Workloads and Pods", + "link": { + "type": "doc", + "id": "pages-for-subheaders/workloads-and-pods" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads", + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads", + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads", + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar" + ] + }, + { + "type": "category", + "label": "Horizontal Pod Autoscaler", + "link": { + "type": "doc", + "id": "pages-for-subheaders/horizontal-pod-autoscaler" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas", + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui", + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl", + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl" + ] + }, + { + "type": "category", + "label": "Load Balancer and Ingress Controller", + "link": { + "type": "doc", + "id": "pages-for-subheaders/load-balancer-and-ingress-controller" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing", + "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses", + "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration" + ] + }, + "how-to-guides/new-user-guides/kubernetes-resources-setup/create-services", + "how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication", + "how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps", + "how-to-guides/new-user-guides/kubernetes-resources-setup/secrets", + "how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries" + ] + }, + { + "type": "category", + "label": "Helm Charts in Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/helm-charts-in-rancher" + }, + "items": [ + "how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps" + ] + }, + { + "type": "category", + "label": "Deploy Apps Across Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/deploy-apps-across-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet", + "how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps" + ] + }, + { + "type": "category", + "label": "Backup, Restore, and Disaster Recovery", + "link": { + "type": "doc", + "id": "pages-for-subheaders/backup-restore-and-disaster-recovery" + }, + "items": [ + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup" + ] + }, + "how-to-guides/new-user-guides/add-users-to-projects", + "how-to-guides/new-user-guides/manage-namespaces" + ] + }, + { + "type": "category", + "label": "Advanced User Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/advanced-user-guides" + }, + "items": [ + { + "type": "category", + "label": "Manage Projects", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-projects" + }, + "items": [ + "how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies", + { + "type": "category", + "label": "Manage Project Resource Quotas", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-project-resource-quotas" + }, + "items": [ + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas", + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces", + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits", + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types" + ] + } + ] + }, + { + "type": "category", + "label": "Monitoring/Alerting Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-alerting-guides" + }, + "items": [ + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage", + { + "type": "category", + "label": "Prometheus Federator Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/prometheus-federator-guides" + }, + "items": [ + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors" + ] + } + ] + }, + { + "type": "category", + "label": "Monitoring V2 Configuration Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-v2-configuration-guides" + }, + "items": [ + { + "type": "category", + "label": "Advanced Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/advanced-configuration" + }, + "items": [ + "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager", + "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus", + "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules" + ] + } + ] + }, + { + "type": "category", + "label": "Istio Setup Guide", + "link": { + "type": "doc", + "id": "pages-for-subheaders/istio-setup-guide" + }, + "items": [ + "how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster", + "how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace", + "how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar", + "how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway", + "how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management", + "how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic" + ] + }, + { + "type": "category", + "label": "CIS Scan Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cis-scan-guides" + }, + "items": [ + "how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark", + "how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark", + "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan", + "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule", + "how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests", + "how-to-guides/advanced-user-guides/cis-scan-guides/view-reports", + "how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark", + "how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", + "how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run" + ] + }, + { + "type": "category", + "label": "Enable Experimental Features", + "link": { + "type": "doc", + "id": "pages-for-subheaders/enable-experimental-features" + }, + "items": [ + "how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64", + "how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers", + "how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features", + "how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery" + ] + }, + "how-to-guides/advanced-user-guides/open-ports-with-firewalld", + "how-to-guides/advanced-user-guides/tune-etcd-for-large-installs", + "how-to-guides/advanced-user-guides/enable-api-audit-log", + "how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters", + "how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer" + ] + } + ] + }, + { + "type": "category", + "label": "Reference Guides", + "items": [ + { + "type": "category", + "label": "Best Practices", + "link": { + "type": "doc", + "id": "pages-for-subheaders/best-practices" + }, + "items": [ + { + "type": "category", + "label": "Rancher Server", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-server" + }, + "items": [ + "reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere", + "reference-guides/best-practices/rancher-server/rancher-deployment-strategy", + "reference-guides/best-practices/rancher-server/tips-for-running-rancher" + ] + }, + { + "type": "category", + "label": "Rancher-Managed Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-managed-clusters" + }, + "items": [ + "reference-guides/best-practices/rancher-managed-clusters/logging-best-practices", + "reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices", + "reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers", + "reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere" + ] + } + ] + }, + { + "type": "category", + "label": "Rancher Architecture", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-manager-architecture" + }, + "items": [ + "reference-guides/rancher-manager-architecture/rancher-server-and-components", + "reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters", + "reference-guides/rancher-manager-architecture/architecture-recommendations" + ] + }, + { + "type": "category", + "label": "Cluster Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cluster-configuration" + }, + "items": [ + { + "type": "category", + "label": "Rancher Server Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-server-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration", + { + "type": "category", + "label": "GKE Cluster Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/gke-cluster-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters" + ] + }, + { + "type": "category", + "label": "Use Existing Nodes", + "link": { + "type": "doc", + "id": "pages-for-subheaders/use-existing-nodes" + }, + "items": [ + "reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options" + ] + }, + "reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters" + ] + }, + { + "type": "category", + "label": "Downstream Cluster Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/downstream-cluster-configuration" + }, + "items": [ + { + "type": "category", + "label": "Node Template Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/node-template-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix" + ] + }, + { + "type": "category", + "label": "Machine Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/machine-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2", + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean", + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure" + ] + } + ] + } + ] + }, + { + "type": "category", + "label": "Single-Node Rancher in Docker", + "link": { + "type": "doc", + "id": "pages-for-subheaders/single-node-rancher-in-docker" + }, + "items": [ + "reference-guides/single-node-rancher-in-docker/http-proxy-configuration", + "reference-guides/single-node-rancher-in-docker/advanced-options" + ] + }, + { + "type": "category", + "label": "Backup & Restore Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/backup-restore-configuration" + }, + "items": [ + "reference-guides/backup-restore-configuration/backup-configuration", + "reference-guides/backup-restore-configuration/restore-configuration", + "reference-guides/backup-restore-configuration/storage-configuration", + "reference-guides/backup-restore-configuration/examples" + ] + }, + "reference-guides/kubernetes-concepts", + { + "type": "category", + "label": "Monitoring V2 Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-v2-configuration" + }, + "items": [ + "reference-guides/monitoring-v2-configuration/receivers", + "reference-guides/monitoring-v2-configuration/routes", + "reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors", + "reference-guides/monitoring-v2-configuration/helm-chart-options", + "reference-guides/monitoring-v2-configuration/examples" + ] + }, + { + "type": "category", + "label": "Prometheus Federator", + "link": { + "type": "doc", + "id": "pages-for-subheaders/prometheus-federator" + }, + "items": [ + "reference-guides/prometheus-federator/rbac" + ] + }, + { + "type": "category", + "label": "User Settings", + "link": { + "type": "doc", + "id": "pages-for-subheaders/user-settings" + }, + "items": [ + "reference-guides/user-settings/api-keys", + "reference-guides/user-settings/manage-node-templates", + "reference-guides/user-settings/manage-cloud-credentials", + "reference-guides/user-settings/user-preferences" + ] + }, + { + "type": "category", + "label": "CLI with Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cli-with-rancher" + }, + "items": [ + "reference-guides/cli-with-rancher/rancher-cli", + "reference-guides/cli-with-rancher/kubectl-utility" + ] + }, + { + "type": "category", + "label": "About the API", + "link": { + "type": "doc", + "id": "pages-for-subheaders/about-the-api" + }, + "items": [ + "reference-guides/about-the-api/api-tokens" + ] + }, + "reference-guides/rancher-cluster-tools", + "reference-guides/rancher-project-tools", + "reference-guides/system-tools", + "reference-guides/rke1-template-example-yaml", + "reference-guides/rancher-webhook", + { + "type": "category", + "label": "Rancher Security", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-security" + }, + "items": [ + { + "type": "category", + "label": "Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-hardening-guides" + }, + "items": [ + { + "type": "category", + "label": "RKE Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rke1-hardening-guide" + }, + "items": [ + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" + ] + }, + { + "type": "category", + "label": "RKE2 Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rke2-hardening-guide" + }, + "items": [ + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" + ] + }, + { + "type": "category", + "label": "K3s Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/k3s-hardening-guide" + }, + "items": [ + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" + ] + } + ] + }, + { + "type": "category", + "label": "SELinux RPM", + "link": { + "type": "doc", + "id": "pages-for-subheaders/selinux-rpm" + }, + "items": [ + "reference-guides/rancher-security/selinux-rpm/about-rancher-selinux", + "reference-guides/rancher-security/selinux-rpm/about-rke2-selinux" + ] + }, + "reference-guides/rancher-security/kubernetes-security-best-practices", + "reference-guides/rancher-security/security-advisories-and-cves", + "reference-guides/rancher-security/psa-restricted-exemptions", + "reference-guides/rancher-security/rancher-webhook-hardening" + ] + } + ] + }, + { + "type": "category", + "label": "Integrations in Rancher", + "items": [ + { + "type": "category", + "label": "Cloud Marketplace Integration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cloud-marketplace" + }, + "items": [ + { + "type": "category", + "label": "AWS Marketplace Integration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/aws-cloud-marketplace" + }, + "items": [ + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues" + ] + }, + "integrations-in-rancher/cloud-marketplace/supportconfig" + ] + }, + { + "type": "category", + "label": "CIS Scans", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cis-scans" + }, + "items": [ + "integrations-in-rancher/cis-scans/configuration-reference", + "integrations-in-rancher/cis-scans/rbac-for-cis-scans", + "integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests", + "integrations-in-rancher/cis-scans/custom-benchmark" + ] + }, + { + "type": "category", + "label": "Continuous Delivery with Fleet", + "link": { + "type": "doc", + "id": "pages-for-subheaders/fleet-gitops-at-scale" + }, + "items": [ + "integrations-in-rancher/fleet-gitops-at-scale/architecture", + "integrations-in-rancher/fleet-gitops-at-scale/windows-support", + "integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy" + ] + }, + "integrations-in-rancher/harvester", + { + "type": "category", + "label": "Istio", + "link": { + "type": "doc", + "id": "pages-for-subheaders/istio" + }, + "items": [ + "integrations-in-rancher/istio/cpu-and-memory-allocations", + "integrations-in-rancher/istio/rbac-for-istio", + "integrations-in-rancher/istio/disable-istio", + { + "type": "category", + "label": "Configuration Options", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configuration-options" + }, + "items": [ + "integrations-in-rancher/istio/configuration-options/pod-security-policies", + "integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations", + "integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster", + "integrations-in-rancher/istio/configuration-options/project-network-isolation" + ] + } + ] + }, + "integrations-in-rancher/longhorn", + { + "type": "category", + "label": "Logging", + "link": { + "type": "doc", + "id": "pages-for-subheaders/logging" + }, + "items": [ + "integrations-in-rancher/logging/logging-architecture", + "integrations-in-rancher/logging/rbac-for-logging", + "integrations-in-rancher/logging/logging-helm-chart-options", + "integrations-in-rancher/logging/taints-and-tolerations", + { + "type": "category", + "label": "Custom Resource Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/custom-resource-configuration" + }, + "items": [ + "integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows", + "integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs" + ] + } + ] + }, + { + "type": "category", + "label": "Monitoring and Alerting", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-and-alerting" + }, + "items": [ + "integrations-in-rancher/monitoring-and-alerting/how-monitoring-works", + "integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring", + "integrations-in-rancher/monitoring-and-alerting/built-in-dashboards", + "integrations-in-rancher/monitoring-and-alerting/windows-support", + "integrations-in-rancher/monitoring-and-alerting/promql-expressions" + ] + }, + "integrations-in-rancher/neuvector", + "integrations-in-rancher/opa-gatekeeper", + "integrations-in-rancher/rancher-extensions" + ] + }, + { + "type": "category", + "label": "FAQ", + "items": [ + "faq/general-faq", + "faq/deprecated-features-in-v2.5", + "faq/install-and-configure-kubectl", + "faq/dockershim", + "faq/technical-items", + "faq/security", + "faq/telemetry", + "faq/container-network-interface-providers", + "faq/rancher-is-no-longer-needed" + ] + }, + { + "type": "category", + "label": "Troubleshooting", + "items": [ + "troubleshooting/general-troubleshooting", + { + "type": "category", + "label": "Kubernetes Components", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-components" + }, + "items": [ + "troubleshooting/kubernetes-components/troubleshooting-etcd-nodes", + "troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes", + "troubleshooting/kubernetes-components/troubleshooting-nginx-proxy", + "troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components" + ] + }, + { + "type": "category", + "label": "Other Troubleshooting Tips", + "items": [ + "troubleshooting/other-troubleshooting-tips/kubernetes-resources", + "troubleshooting/other-troubleshooting-tips/networking", + "troubleshooting/other-troubleshooting-tips/dns", + "troubleshooting/other-troubleshooting-tips/rancher-ha", + "troubleshooting/other-troubleshooting-tips/registered-clusters", + "troubleshooting/other-troubleshooting-tips/logging", + "troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs", + "troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation" + ] + } + ] + }, + "contribute-to-rancher" + ] +} diff --git a/versioned_sidebars/version-latest-sidebars.json b/versioned_sidebars/version-latest-sidebars.json new file mode 100644 index 00000000000..5625fdaf67d --- /dev/null +++ b/versioned_sidebars/version-latest-sidebars.json @@ -0,0 +1,1266 @@ +{ + "tutorialSidebar": [ + "rancher-manager", + { + "type": "category", + "label": "Getting Started", + "items": [ + "getting-started/overview", + { + "type": "category", + "label": "Quick Start Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/quick-start-guides" + }, + "items": [ + { + "type": "category", + "label": "Deploy Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/deploy-rancher-manager" + }, + "items": [ + "getting-started/quick-start-guides/deploy-rancher-manager/aws", + "getting-started/quick-start-guides/deploy-rancher-manager/aws-marketplace", + "getting-started/quick-start-guides/deploy-rancher-manager/azure", + "getting-started/quick-start-guides/deploy-rancher-manager/digitalocean", + "getting-started/quick-start-guides/deploy-rancher-manager/gcp", + "getting-started/quick-start-guides/deploy-rancher-manager/hetzner-cloud", + "getting-started/quick-start-guides/deploy-rancher-manager/vagrant", + "getting-started/quick-start-guides/deploy-rancher-manager/equinix-metal", + "getting-started/quick-start-guides/deploy-rancher-manager/outscale-qs", + "getting-started/quick-start-guides/deploy-rancher-manager/helm-cli" + ] + }, + "getting-started/quick-start-guides/deploy-rancher-manager/prime", + { + "type": "category", + "label": "Deploy Rancher Workloads", + "link": { + "type": "doc", + "id": "pages-for-subheaders/deploy-rancher-workloads" + }, + "items": [ + "getting-started/quick-start-guides/deploy-workloads/workload-ingress", + "getting-started/quick-start-guides/deploy-workloads/nodeports" + ] + } + ] + }, + { + "type": "category", + "label": "Installation and Upgrade", + "link": { + "type": "doc", + "id": "pages-for-subheaders/installation-and-upgrade" + }, + "items": [ + { + "type": "category", + "label": "Installation Requirements", + "link": { + "type": "doc", + "id": "pages-for-subheaders/installation-requirements" + }, + "items": [ + "getting-started/installation-and-upgrade/installation-requirements/install-docker", + "getting-started/installation-and-upgrade/installation-requirements/dockershim", + "getting-started/installation-and-upgrade/installation-requirements/port-requirements" + ] + }, + { + "type": "category", + "label": "Installation References", + "link": { + "type": "doc", + "id": "pages-for-subheaders/installation-references" + }, + "items": [ + "getting-started/installation-and-upgrade/installation-references/helm-chart-options", + "getting-started/installation-and-upgrade/installation-references/tls-settings", + "getting-started/installation-and-upgrade/installation-references/feature-flags" + ] + }, + { + "type": "category", + "label": "Install/Upgrade on a Kubernetes Cluster", + "link": { + "type": "doc", + "id": "pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster" + }, + "items": [ + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rollbacks", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrades", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/air-gapped-upgrades", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-amazon-eks", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-aks", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/rancher-on-gke", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/upgrade-a-hardened-cluster-to-k8s-v1-25", + "getting-started/installation-and-upgrade/install-upgrade-on-a-kubernetes-cluster/troubleshooting" + ] + }, + { + "type": "category", + "label": "Other Installation Methods", + "link": { + "type": "doc", + "id": "pages-for-subheaders/other-installation-methods" + }, + "items": [ + { + "type": "category", + "label": "Air-Gapped Helm CLI Install", + "link": { + "type": "doc", + "id": "pages-for-subheaders/air-gapped-helm-cli-install" + }, + "items": [ + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/infrastructure-private-registry", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/publish-images", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-kubernetes", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/install-rancher-ha", + "getting-started/installation-and-upgrade/other-installation-methods/air-gapped-helm-cli-install/docker-install-commands" + ] + }, + { + "type": "category", + "label": "Rancher on a Single Node with Docker", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-on-a-single-node-with-docker" + }, + "items": [ + "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/upgrade-docker-installed-rancher", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/roll-back-docker-installed-rancher", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-on-a-single-node-with-docker/certificate-troubleshooting" + ] + }, + { + "type": "category", + "label": "Rancher Behind an HTTP Proxy", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-behind-an-http-proxy" + }, + "items": [ + "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/set-up-infrastructure", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-kubernetes", + "getting-started/installation-and-upgrade/other-installation-methods/rancher-behind-an-http-proxy/install-rancher" + ] + } + ] + }, + { + "type": "category", + "label": "Resources", + "link": { + "type": "doc", + "id": "pages-for-subheaders/resources" + }, + "items": [ + "getting-started/installation-and-upgrade/resources/choose-a-rancher-version", + "getting-started/installation-and-upgrade/resources/helm-version-requirements", + "getting-started/installation-and-upgrade/resources/add-tls-secrets", + "getting-started/installation-and-upgrade/resources/custom-ca-root-certificates", + "getting-started/installation-and-upgrade/resources/upgrade-cert-manager", + "getting-started/installation-and-upgrade/resources/update-rancher-certificate", + "getting-started/installation-and-upgrade/resources/bootstrap-password", + "getting-started/installation-and-upgrade/resources/local-system-charts" + ] + }, + "getting-started/installation-and-upgrade/upgrade-and-roll-back-kubernetes", + "getting-started/installation-and-upgrade/upgrade-kubernetes-without-upgrading-rancher" + ] + } + ] + }, + { + "type": "category", + "label": "How-to Guides", + "items": [ + { + "type": "category", + "label": "New User Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/new-user-guides" + }, + "items": [ + { + "type": "category", + "label": "Authentication, Permissions, and Global Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/authentication-permissions-and-global-configuration" + }, + "items": [ + { + "type": "category", + "label": "Authentication Config", + "link": { + "type": "doc", + "id": "pages-for-subheaders/authentication-config" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/manage-users-and-groups", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/create-local-users", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-google-oauth", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-active-directory", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-freeipa", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-azure-ad", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-github", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-oidc", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-keycloak-saml", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-pingidentity", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/authentication-config/configure-okta-saml" + ] + }, + { + "type": "category", + "label": "Configure OpenLDAP", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configure-openldap" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-openldap/openldap-config-reference" + ] + }, + { + "type": "category", + "label": "Configure Microsoft AD Federation Service (SAML)", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configure-microsoft-ad-federation-service-saml" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-ms-adfs-for-rancher", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-microsoft-ad-federation-service-saml/configure-rancher-for-ms-adfs" + ] + }, + { + "type": "category", + "label": "Configure Shibboleth (SAML)", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configure-shibboleth-saml" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/configure-shibboleth-saml/about-group-permissions" + ] + }, + { + "type": "category", + "label": "Manage Role-Based Access Control (RBAC)", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-role-based-access-control-rbac" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/global-permissions", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/cluster-and-project-roles", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/custom-roles", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/manage-role-based-access-control-rbac/locked-roles" + ] + }, + { + "type": "category", + "label": "About Provisioning Drivers", + "link": { + "type": "doc", + "id": "pages-for-subheaders/about-provisioning-drivers" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-cluster-drivers", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-provisioning-drivers/manage-node-drivers" + ] + }, + { + "type": "category", + "label": "About RKE1 Templates", + "link": { + "type": "doc", + "id": "pages-for-subheaders/about-rke1-templates" + }, + "items": [ + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/creator-permissions", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/access-or-share-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/manage-rke1-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/enforce-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/override-template-settings", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/apply-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/infrastructure", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/about-rke1-templates/example-use-cases" + ] + }, + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/create-pod-security-policies", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/pod-security-standards", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/psa-config-templates", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/global-default-private-registry", + "how-to-guides/new-user-guides/authentication-permissions-and-global-configuration/custom-branding" + ] + }, + { + "type": "category", + "label": "Manage Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-clusters" + }, + "items": [ + { + "type": "category", + "label": "Access Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/access-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/access-clusters/use-kubectl-and-kubeconfig", + "how-to-guides/new-user-guides/manage-clusters/access-clusters/authorized-cluster-endpoint", + "how-to-guides/new-user-guides/manage-clusters/access-clusters/add-users-to-clusters" + ] + }, + { + "type": "category", + "label": "Install Cluster Autoscaler", + "link": { + "type": "doc", + "id": "pages-for-subheaders/install-cluster-autoscaler" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/install-cluster-autoscaler/use-aws-ec2-auto-scaling-groups" + ] + }, + { + "type": "category", + "label": "Create Kubernetes Persistent Storage", + "link": { + "type": "doc", + "id": "pages-for-subheaders/create-kubernetes-persistent-storage" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-persistent-storage", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/set-up-existing-storage", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/dynamically-provision-new-storage", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/use-external-ceph-driver", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/about-glusterfs-volumes", + "how-to-guides/new-user-guides/manage-clusters/create-kubernetes-persistent-storage/manage-persistent-storage/install-iscsi-volumes" + ] + }, + { + "type": "category", + "label": "Provisioning Storage Examples", + "link": { + "type": "doc", + "id": "pages-for-subheaders/provisioning-storage-examples" + }, + "items": [ + "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/persistent-storage-in-amazon-ebs", + "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/nfs-storage", + "how-to-guides/new-user-guides/manage-clusters/provisioning-storage-examples/vsphere-storage" + ] + }, + "how-to-guides/new-user-guides/manage-clusters/projects-and-namespaces", + "how-to-guides/new-user-guides/manage-clusters/rotate-certificates", + "how-to-guides/new-user-guides/manage-clusters/rotate-encryption-key", + "how-to-guides/new-user-guides/manage-clusters/manage-cluster-templates", + "how-to-guides/new-user-guides/manage-clusters/nodes-and-node-pools", + "how-to-guides/new-user-guides/manage-clusters/clean-cluster-nodes", + "how-to-guides/new-user-guides/manage-clusters/add-a-pod-security-policy", + "how-to-guides/new-user-guides/manage-clusters/assign-pod-security-policies" + ] + }, + { + "type": "category", + "label": "Kubernetes Cluster Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-cluster-setup" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-cluster-setup/high-availability-installs", + "how-to-guides/new-user-guides/kubernetes-cluster-setup/k3s-for-rancher", + "how-to-guides/new-user-guides/kubernetes-cluster-setup/rke1-for-rancher", + "how-to-guides/new-user-guides/kubernetes-cluster-setup/rke2-for-rancher" + ] + }, + { + "type": "category", + "label": "Infrastructure Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/infrastructure-setup" + }, + "items": [ + "how-to-guides/new-user-guides/infrastructure-setup/ha-k3s-kubernetes-cluster", + "how-to-guides/new-user-guides/infrastructure-setup/ha-rke1-kubernetes-cluster", + "how-to-guides/new-user-guides/infrastructure-setup/ha-rke2-kubernetes-cluster", + "how-to-guides/new-user-guides/infrastructure-setup/nodes-in-amazon-ec2", + "how-to-guides/new-user-guides/infrastructure-setup/mysql-database-in-amazon-rds", + "how-to-guides/new-user-guides/infrastructure-setup/nginx-load-balancer", + "how-to-guides/new-user-guides/infrastructure-setup/amazon-elb-load-balancer" + ] + }, + { + "type": "category", + "label": "Kubernetes Clusters in Rancher Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-clusters-in-rancher-setup" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/node-requirements-for-rancher-managed-clusters", + { + "type": "category", + "label": "Checklist for Production-Ready Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/checklist-for-production-ready-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/recommended-cluster-architecture", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/checklist-for-production-ready-clusters/roles-for-nodes-in-kubernetes" + ] + }, + { + "type": "category", + "label": "Set Up Clusters from Hosted Kubernetes Providers", + "link": { + "type": "doc", + "id": "pages-for-subheaders/set-up-clusters-from-hosted-kubernetes-providers" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/eks", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/gke", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/aks", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/alibaba", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/tencent", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-clusters-from-hosted-kubernetes-providers/huawei" + ] + }, + { + "type": "category", + "label": "Use Windows Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/use-windows-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/azure-storageclass-configuration", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/windows-linux-cluster-feature-parity", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/network-requirements-for-host-gateway", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/use-windows-clusters/workload-migration-guidance" + ] + }, + { + "type": "category", + "label": "Set Up Cloud Providers", + "link": { + "type": "doc", + "id": "pages-for-subheaders/set-up-cloud-providers" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/amazon", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/azure", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/google-compute-engine", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-in-tree-vsphere", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/configure-out-of-tree-vsphere", + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/set-up-cloud-providers/migrate-from-in-tree-to-out-of-tree" + ] + }, + "how-to-guides/new-user-guides/kubernetes-clusters-in-rancher-setup/register-existing-clusters" + ] + }, + { + "type": "category", + "label": "Launch Kubernetes with Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/launch-kubernetes-with-rancher" + }, + "items": [ + { + "type": "category", + "label": "Use New Nodes in an Infra Provider", + "link": { + "type": "doc", + "id": "pages-for-subheaders/use-new-nodes-in-an-infra-provider" + }, + "items": [ + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-amazon-ec2-cluster", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-a-digitalocean-cluster", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/create-an-azure-cluster", + { + "type": "category", + "label": "vSphere", + "link": { + "type": "doc", + "id": "pages-for-subheaders/vsphere" + }, + "items": [ + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/provision-kubernetes-clusters-in-vsphere", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-credentials", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/vsphere/create-a-vm-template" + ] + }, + { + "type": "category", + "label": "Nutanix", + "link": { + "type": "doc", + "id": "pages-for-subheaders/nutanix" + }, + "items": [ + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/use-new-nodes-in-an-infra-provider/nutanix/provision-kubernetes-clusters-in-aos" + ] + } + ] + }, + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/rke1-vs-rke2-differences", + "how-to-guides/new-user-guides/launch-kubernetes-with-rancher/about-rancher-agents" + ] + }, + { + "type": "category", + "label": "Kubernetes Resources Setup", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-resources-setup" + }, + "items": [ + { + "type": "category", + "label": "Workloads and Pods", + "link": { + "type": "doc", + "id": "pages-for-subheaders/workloads-and-pods" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/deploy-workloads", + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/roll-back-workloads", + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/upgrade-workloads", + "how-to-guides/new-user-guides/kubernetes-resources-setup/workloads-and-pods/add-a-sidecar" + ] + }, + { + "type": "category", + "label": "Horizontal Pod Autoscaler", + "link": { + "type": "doc", + "id": "pages-for-subheaders/horizontal-pod-autoscaler" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/about-hpas", + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-ui", + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/manage-hpas-with-kubectl", + "how-to-guides/new-user-guides/kubernetes-resources-setup/horizontal-pod-autoscaler/test-hpas-with-kubectl" + ] + }, + { + "type": "category", + "label": "Load Balancer and Ingress Controller", + "link": { + "type": "doc", + "id": "pages-for-subheaders/load-balancer-and-ingress-controller" + }, + "items": [ + "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/layer-4-and-layer-7-load-balancing", + "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/add-ingresses", + "how-to-guides/new-user-guides/kubernetes-resources-setup/load-balancer-and-ingress-controller/ingress-configuration" + ] + }, + "how-to-guides/new-user-guides/kubernetes-resources-setup/create-services", + "how-to-guides/new-user-guides/kubernetes-resources-setup/encrypt-http-communication", + "how-to-guides/new-user-guides/kubernetes-resources-setup/configmaps", + "how-to-guides/new-user-guides/kubernetes-resources-setup/secrets", + "how-to-guides/new-user-guides/kubernetes-resources-setup/kubernetes-and-docker-registries" + ] + }, + { + "type": "category", + "label": "Helm Charts in Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/helm-charts-in-rancher" + }, + "items": [ + "how-to-guides/new-user-guides/helm-charts-in-rancher/create-apps" + ] + }, + { + "type": "category", + "label": "Deploy Apps Across Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/deploy-apps-across-clusters" + }, + "items": [ + "how-to-guides/new-user-guides/deploy-apps-across-clusters/fleet", + "how-to-guides/new-user-guides/deploy-apps-across-clusters/multi-cluster-apps" + ] + }, + { + "type": "category", + "label": "Backup, Restore, and Disaster Recovery", + "link": { + "type": "doc", + "id": "pages-for-subheaders/backup-restore-and-disaster-recovery" + }, + "items": [ + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-restore-usage-guide", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/migrate-rancher-to-new-cluster", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-docker-installed-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-docker-installed-rancher", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/back-up-rancher-launched-kubernetes-clusters", + "how-to-guides/new-user-guides/backup-restore-and-disaster-recovery/restore-rancher-launched-kubernetes-clusters-from-backup" + ] + }, + "how-to-guides/new-user-guides/add-users-to-projects", + "how-to-guides/new-user-guides/manage-namespaces" + ] + }, + { + "type": "category", + "label": "Advanced User Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/advanced-user-guides" + }, + "items": [ + { + "type": "category", + "label": "Manage Projects", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-projects" + }, + "items": [ + "how-to-guides/advanced-user-guides/manage-projects/manage-pod-security-policies", + { + "type": "category", + "label": "Manage Project Resource Quotas", + "link": { + "type": "doc", + "id": "pages-for-subheaders/manage-project-resource-quotas" + }, + "items": [ + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/about-project-resource-quotas", + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/override-default-limit-in-namespaces", + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/set-container-default-resource-limits", + "how-to-guides/advanced-user-guides/manage-projects/manage-project-resource-quotas/resource-quota-types" + ] + } + ] + }, + { + "type": "category", + "label": "Monitoring/Alerting Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-alerting-guides" + }, + "items": [ + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/enable-monitoring", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/uninstall-monitoring", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/set-up-monitoring-for-workloads", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/customize-grafana-dashboard", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/create-persistent-grafana-dashboard", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/debug-high-memory-usage", + { + "type": "category", + "label": "Prometheus Federator Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/prometheus-federator-guides" + }, + "items": [ + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/enable-prometheus-federator", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/uninstall-prometheus-federator", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/customize-grafana-dashboards", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/set-up-workloads", + "how-to-guides/advanced-user-guides/monitoring-alerting-guides/prometheus-federator-guides/project-monitors" + ] + } + ] + }, + { + "type": "category", + "label": "Monitoring V2 Configuration Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-v2-configuration-guides" + }, + "items": [ + { + "type": "category", + "label": "Advanced Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/advanced-configuration" + }, + "items": [ + "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/alertmanager", + "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheus", + "how-to-guides/advanced-user-guides/monitoring-v2-configuration-guides/advanced-configuration/prometheusrules" + ] + } + ] + }, + { + "type": "category", + "label": "Istio Setup Guide", + "link": { + "type": "doc", + "id": "pages-for-subheaders/istio-setup-guide" + }, + "items": [ + "how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-cluster", + "how-to-guides/advanced-user-guides/istio-setup-guide/enable-istio-in-namespace", + "how-to-guides/advanced-user-guides/istio-setup-guide/use-istio-sidecar", + "how-to-guides/advanced-user-guides/istio-setup-guide/set-up-istio-gateway", + "how-to-guides/advanced-user-guides/istio-setup-guide/set-up-traffic-management", + "how-to-guides/advanced-user-guides/istio-setup-guide/generate-and-view-traffic" + ] + }, + { + "type": "category", + "label": "CIS Scan Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cis-scan-guides" + }, + "items": [ + "how-to-guides/advanced-user-guides/cis-scan-guides/install-rancher-cis-benchmark", + "how-to-guides/advanced-user-guides/cis-scan-guides/uninstall-rancher-cis-benchmark", + "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan", + "how-to-guides/advanced-user-guides/cis-scan-guides/run-a-scan-periodically-on-a-schedule", + "how-to-guides/advanced-user-guides/cis-scan-guides/skip-tests", + "how-to-guides/advanced-user-guides/cis-scan-guides/view-reports", + "how-to-guides/advanced-user-guides/cis-scan-guides/enable-alerting-for-rancher-cis-benchmark", + "how-to-guides/advanced-user-guides/cis-scan-guides/configure-alerts-for-periodic-scan-on-a-schedule", + "how-to-guides/advanced-user-guides/cis-scan-guides/create-a-custom-benchmark-version-to-run" + ] + }, + { + "type": "category", + "label": "Enable Experimental Features", + "link": { + "type": "doc", + "id": "pages-for-subheaders/enable-experimental-features" + }, + "items": [ + "how-to-guides/advanced-user-guides/enable-experimental-features/rancher-on-arm64", + "how-to-guides/advanced-user-guides/enable-experimental-features/unsupported-storage-drivers", + "how-to-guides/advanced-user-guides/enable-experimental-features/istio-traffic-management-features", + "how-to-guides/advanced-user-guides/enable-experimental-features/continuous-delivery" + ] + }, + "how-to-guides/advanced-user-guides/open-ports-with-firewalld", + "how-to-guides/advanced-user-guides/tune-etcd-for-large-installs", + "how-to-guides/advanced-user-guides/enable-api-audit-log", + "how-to-guides/advanced-user-guides/enable-api-audit-log-in-downstream-clusters", + "how-to-guides/advanced-user-guides/configure-layer-7-nginx-load-balancer" + ] + } + ] + }, + { + "type": "category", + "label": "Reference Guides", + "items": [ + { + "type": "category", + "label": "Best Practices", + "link": { + "type": "doc", + "id": "pages-for-subheaders/best-practices" + }, + "items": [ + { + "type": "category", + "label": "Rancher Server", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-server" + }, + "items": [ + "reference-guides/best-practices/rancher-server/on-premises-rancher-in-vsphere", + "reference-guides/best-practices/rancher-server/rancher-deployment-strategy", + "reference-guides/best-practices/rancher-server/tips-for-running-rancher" + ] + }, + { + "type": "category", + "label": "Rancher-Managed Clusters", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-managed-clusters" + }, + "items": [ + "reference-guides/best-practices/rancher-managed-clusters/logging-best-practices", + "reference-guides/best-practices/rancher-managed-clusters/monitoring-best-practices", + "reference-guides/best-practices/rancher-managed-clusters/tips-to-set-up-containers", + "reference-guides/best-practices/rancher-managed-clusters/rancher-managed-clusters-in-vsphere" + ] + } + ] + }, + { + "type": "category", + "label": "Rancher Architecture", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-manager-architecture" + }, + "items": [ + "reference-guides/rancher-manager-architecture/rancher-server-and-components", + "reference-guides/rancher-manager-architecture/communicating-with-downstream-user-clusters", + "reference-guides/rancher-manager-architecture/architecture-recommendations" + ] + }, + { + "type": "category", + "label": "Cluster Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cluster-configuration" + }, + "items": [ + { + "type": "category", + "label": "Rancher Server Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-server-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/rancher-server-configuration/rke1-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/rke2-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/k3s-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/eks-cluster-configuration", + "reference-guides/cluster-configuration/rancher-server-configuration/aks-cluster-configuration", + { + "type": "category", + "label": "GKE Cluster Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/gke-cluster-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/rancher-server-configuration/gke-cluster-configuration/gke-private-clusters" + ] + }, + { + "type": "category", + "label": "Use Existing Nodes", + "link": { + "type": "doc", + "id": "pages-for-subheaders/use-existing-nodes" + }, + "items": [ + "reference-guides/cluster-configuration/rancher-server-configuration/use-existing-nodes/rancher-agent-options" + ] + }, + "reference-guides/cluster-configuration/rancher-server-configuration/sync-clusters" + ] + }, + { + "type": "category", + "label": "Downstream Cluster Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/downstream-cluster-configuration" + }, + "items": [ + { + "type": "category", + "label": "Node Template Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/node-template-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/amazon-ec2", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/digitalocean", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/azure", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/vsphere", + "reference-guides/cluster-configuration/downstream-cluster-configuration/node-template-configuration/nutanix" + ] + }, + { + "type": "category", + "label": "Machine Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/machine-configuration" + }, + "items": [ + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/amazon-ec2", + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/digitalocean", + "reference-guides/cluster-configuration/downstream-cluster-configuration/machine-configuration/azure" + ] + } + ] + } + ] + }, + { + "type": "category", + "label": "Single-Node Rancher in Docker", + "link": { + "type": "doc", + "id": "pages-for-subheaders/single-node-rancher-in-docker" + }, + "items": [ + "reference-guides/single-node-rancher-in-docker/http-proxy-configuration", + "reference-guides/single-node-rancher-in-docker/advanced-options" + ] + }, + { + "type": "category", + "label": "Backup & Restore Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/backup-restore-configuration" + }, + "items": [ + "reference-guides/backup-restore-configuration/backup-configuration", + "reference-guides/backup-restore-configuration/restore-configuration", + "reference-guides/backup-restore-configuration/storage-configuration", + "reference-guides/backup-restore-configuration/examples" + ] + }, + "reference-guides/kubernetes-concepts", + { + "type": "category", + "label": "Monitoring V2 Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-v2-configuration" + }, + "items": [ + "reference-guides/monitoring-v2-configuration/receivers", + "reference-guides/monitoring-v2-configuration/routes", + "reference-guides/monitoring-v2-configuration/servicemonitors-and-podmonitors", + "reference-guides/monitoring-v2-configuration/helm-chart-options", + "reference-guides/monitoring-v2-configuration/examples" + ] + }, + { + "type": "category", + "label": "Prometheus Federator", + "link": { + "type": "doc", + "id": "pages-for-subheaders/prometheus-federator" + }, + "items": [ + "reference-guides/prometheus-federator/rbac" + ] + }, + { + "type": "category", + "label": "User Settings", + "link": { + "type": "doc", + "id": "pages-for-subheaders/user-settings" + }, + "items": [ + "reference-guides/user-settings/api-keys", + "reference-guides/user-settings/manage-node-templates", + "reference-guides/user-settings/manage-cloud-credentials", + "reference-guides/user-settings/user-preferences" + ] + }, + { + "type": "category", + "label": "CLI with Rancher", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cli-with-rancher" + }, + "items": [ + "reference-guides/cli-with-rancher/rancher-cli", + "reference-guides/cli-with-rancher/kubectl-utility" + ] + }, + { + "type": "category", + "label": "About the API", + "link": { + "type": "doc", + "id": "pages-for-subheaders/about-the-api" + }, + "items": [ + "reference-guides/about-the-api/api-tokens" + ] + }, + "reference-guides/rancher-cluster-tools", + "reference-guides/rancher-project-tools", + "reference-guides/system-tools", + "reference-guides/rke1-template-example-yaml", + "reference-guides/rancher-webhook", + { + "type": "category", + "label": "Rancher Security", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-security" + }, + "items": [ + { + "type": "category", + "label": "Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rancher-hardening-guides" + }, + "items": [ + { + "type": "category", + "label": "RKE Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rke1-hardening-guide" + }, + "items": [ + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.23-k8s-v1.23", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke1-hardening-guide/rke1-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" + ] + }, + { + "type": "category", + "label": "RKE2 Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/rke2-hardening-guide" + }, + "items": [ + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.23-k8s-v1.23", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/rke2-hardening-guide/rke2-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" + ] + }, + { + "type": "category", + "label": "K3s Hardening Guides", + "link": { + "type": "doc", + "id": "pages-for-subheaders/k3s-hardening-guide" + }, + "items": [ + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.23-k8s-v1.23", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.24-k8s-v1.24", + "reference-guides/rancher-security/hardening-guides/k3s-hardening-guide/k3s-self-assessment-guide-with-cis-v1.7-k8s-v1.25-v1.26-v1.27" + ] + } + ] + }, + { + "type": "category", + "label": "SELinux RPM", + "link": { + "type": "doc", + "id": "pages-for-subheaders/selinux-rpm" + }, + "items": [ + "reference-guides/rancher-security/selinux-rpm/about-rancher-selinux", + "reference-guides/rancher-security/selinux-rpm/about-rke2-selinux" + ] + }, + "reference-guides/rancher-security/kubernetes-security-best-practices", + "reference-guides/rancher-security/security-advisories-and-cves", + "reference-guides/rancher-security/psa-restricted-exemptions" + ] + } + ] + }, + { + "type": "category", + "label": "Integrations in Rancher", + "items": [ + { + "type": "category", + "label": "Cloud Marketplace Integration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cloud-marketplace" + }, + "items": [ + { + "type": "category", + "label": "AWS Marketplace Integration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/aws-cloud-marketplace" + }, + "items": [ + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/adapter-requirements", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/install-adapter", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/uninstall-adapter", + "integrations-in-rancher/cloud-marketplace/aws-cloud-marketplace/common-issues" + ] + }, + "integrations-in-rancher/cloud-marketplace/supportconfig" + ] + }, + { + "type": "category", + "label": "CIS Scans", + "link": { + "type": "doc", + "id": "pages-for-subheaders/cis-scans" + }, + "items": [ + "integrations-in-rancher/cis-scans/configuration-reference", + "integrations-in-rancher/cis-scans/rbac-for-cis-scans", + "integrations-in-rancher/cis-scans/skipped-and-not-applicable-tests", + "integrations-in-rancher/cis-scans/custom-benchmark" + ] + }, + { + "type": "category", + "label": "Continuous Delivery with Fleet", + "link": { + "type": "doc", + "id": "pages-for-subheaders/fleet-gitops-at-scale" + }, + "items": [ + "integrations-in-rancher/fleet-gitops-at-scale/architecture", + "integrations-in-rancher/fleet-gitops-at-scale/windows-support", + "integrations-in-rancher/fleet-gitops-at-scale/use-fleet-behind-a-proxy" + ] + }, + "integrations-in-rancher/harvester", + { + "type": "category", + "label": "Istio", + "link": { + "type": "doc", + "id": "pages-for-subheaders/istio" + }, + "items": [ + "integrations-in-rancher/istio/cpu-and-memory-allocations", + "integrations-in-rancher/istio/rbac-for-istio", + "integrations-in-rancher/istio/disable-istio", + { + "type": "category", + "label": "Configuration Options", + "link": { + "type": "doc", + "id": "pages-for-subheaders/configuration-options" + }, + "items": [ + "integrations-in-rancher/istio/configuration-options/pod-security-policies", + "integrations-in-rancher/istio/configuration-options/selectors-and-scrape-configurations", + "integrations-in-rancher/istio/configuration-options/install-istio-on-rke2-cluster", + "integrations-in-rancher/istio/configuration-options/project-network-isolation" + ] + } + ] + }, + "integrations-in-rancher/longhorn", + { + "type": "category", + "label": "Logging", + "link": { + "type": "doc", + "id": "pages-for-subheaders/logging" + }, + "items": [ + "integrations-in-rancher/logging/logging-architecture", + "integrations-in-rancher/logging/rbac-for-logging", + "integrations-in-rancher/logging/logging-helm-chart-options", + "integrations-in-rancher/logging/taints-and-tolerations", + { + "type": "category", + "label": "Custom Resource Configuration", + "link": { + "type": "doc", + "id": "pages-for-subheaders/custom-resource-configuration" + }, + "items": [ + "integrations-in-rancher/logging/custom-resource-configuration/flows-and-clusterflows", + "integrations-in-rancher/logging/custom-resource-configuration/outputs-and-clusteroutputs" + ] + } + ] + }, + { + "type": "category", + "label": "Monitoring and Alerting", + "link": { + "type": "doc", + "id": "pages-for-subheaders/monitoring-and-alerting" + }, + "items": [ + "integrations-in-rancher/monitoring-and-alerting/how-monitoring-works", + "integrations-in-rancher/monitoring-and-alerting/rbac-for-monitoring", + "integrations-in-rancher/monitoring-and-alerting/built-in-dashboards", + "integrations-in-rancher/monitoring-and-alerting/windows-support", + "integrations-in-rancher/monitoring-and-alerting/promql-expressions" + ] + }, + "integrations-in-rancher/neuvector", + "integrations-in-rancher/opa-gatekeeper", + "integrations-in-rancher/rancher-extensions" + ] + }, + { + "type": "category", + "label": "FAQ", + "items": [ + "faq/general-faq", + "faq/deprecated-features-in-v2.5", + "faq/install-and-configure-kubectl", + "faq/dockershim", + "faq/technical-items", + "faq/security", + "faq/telemetry", + "faq/container-network-interface-providers", + "faq/rancher-is-no-longer-needed" + ] + }, + { + "type": "category", + "label": "Troubleshooting", + "items": [ + "troubleshooting/general-troubleshooting", + { + "type": "category", + "label": "Kubernetes Components", + "link": { + "type": "doc", + "id": "pages-for-subheaders/kubernetes-components" + }, + "items": [ + "troubleshooting/kubernetes-components/troubleshooting-etcd-nodes", + "troubleshooting/kubernetes-components/troubleshooting-controlplane-nodes", + "troubleshooting/kubernetes-components/troubleshooting-nginx-proxy", + "troubleshooting/kubernetes-components/troubleshooting-worker-nodes-and-generic-components" + ] + }, + { + "type": "category", + "label": "Other Troubleshooting Tips", + "items": [ + "troubleshooting/other-troubleshooting-tips/kubernetes-resources", + "troubleshooting/other-troubleshooting-tips/networking", + "troubleshooting/other-troubleshooting-tips/dns", + "troubleshooting/other-troubleshooting-tips/rancher-ha", + "troubleshooting/other-troubleshooting-tips/registered-clusters", + "troubleshooting/other-troubleshooting-tips/logging", + "troubleshooting/other-troubleshooting-tips/user-id-tracking-in-audit-logs", + "troubleshooting/other-troubleshooting-tips/expired-webhook-certificate-rotation" + ] + } + ] + }, + "contribute-to-rancher" + ] +} diff --git a/versions.json b/versions.json index 385dca16647..c1dec51d76b 100644 --- a/versions.json +++ b/versions.json @@ -1,4 +1,6 @@ [ + "latest", + "2.8", "2.7", "2.6", "2.5",